option(GMX_FAHCORE "Build a library with mdrun functionality" OFF)
mark_as_advanced(GMX_FAHCORE)
-if(NOT DEFINED GMX_CPU_ACCELERATION AND NOT CMAKE_CROSSCOMPILING)
- include(gmxDetectAcceleration)
- gmx_detect_acceleration(GMX_SUGGESTED_CPU_ACCELERATION)
-endif()
+option(GMX_COOL_QUOTES "Enable Gromacs cool quotes" ON)
+mark_as_advanced(GMX_COOL_QUOTES)
+gmx_add_cache_dependency(GMX_COOL_QUOTES BOOL "NOT GMX_FAHCORE" OFF)
+
+# decide on GPU settings based on user-settings and GPU/CUDA detection
+include(gmxManageGPU)
+ # Detect the architecture the compiler is targetting, detect
+ # acceleration possibilities on that hardware, suggest an acceleration
+ # to use if none is specified, and populate the cache option for CPU
+ # accleration.
+ include(gmxDetectTargetArchitecture)
+ gmx_detect_target_architecture()
include(gmxDetectAcceleration)
- if(NOT DEFINED GMX_CPU_ACCELERATION)
- if(CMAKE_CROSSCOMPILING)
- if("${CMAKE_SYSTEM_NAME}" MATCHES "BlueGeneQ")
- set(GMX_SUGGESTED_CPU_ACCELERATION "IBM_QPX")
- else()
- set(GMX_SUGGESTED_CPU_ACCELERATION "None")
- endif()
- else(CMAKE_CROSSCOMPILING)
- gmx_detect_acceleration(GMX_SUGGESTED_CPU_ACCELERATION)
- endif(CMAKE_CROSSCOMPILING)
- endif(NOT DEFINED GMX_CPU_ACCELERATION)
+ gmx_detect_acceleration(GMX_SUGGESTED_CPU_ACCELERATION)
-set(GMX_CPU_ACCELERATION "@GMX_SUGGESTED_CPU_ACCELERATION@"
- CACHE STRING "Accelerated CPU kernels. Pick one of: None, SSE2, SSE4.1, AVX_128_FMA, AVX_256, IBM_QPX, Sparc64_HPC_ACE")
-set(GMX_FFT_LIBRARY "fftw3"
- CACHE STRING "FFT library choices: fftw3,mkl,fftpack[built-in]")
-option(GMX_BUILD_OWN_FFTW "Download and build FFTW 3 during the GROMACS build process, rather than fall back on the really slow fftpack." OFF)
+gmx_option_multichoice(
+ GMX_CPU_ACCELERATION
+ "Acceleration for CPU kernels and compiler optimization"
+ "${GMX_SUGGESTED_CPU_ACCELERATION}"
+ None SSE2 SSE4.1 AVX_128_FMA AVX_256 IBM_QPX Sparc64_HPC_ACE)
+
+gmx_option_multichoice(
+ GMX_FFT_LIBRARY
+ "FFT library"
+ "fftw3"
+ fftw3 mkl "fftpack[built-in]")
+gmx_dependent_option(
+ GMX_BUILD_OWN_FFTW
+ "Download and build FFTW 3 during the GROMACS build process, rather than fall back on the really slow fftpack."
+ OFF
+ "GMX_FFT_LIBRARY STREQUAL FFTW3")
+gmx_dependent_option(
+ GMX_DISABLE_FFTW_MEASURE
+ "Do not optimize FFTW setups (not needed with SSE)"
+ OFF
+ "GMX_FFT_LIBRARY STREQUAL FFTW3")
mark_as_advanced(GMX_BUILD_OWN_FFTW)
-option(GMX_DISABLE_FFTW_MEASURE
- "Do not optimize FFTW setups (not needed with SSE)" OFF)
mark_as_advanced(GMX_DISABLE_FFTW_MEASURE)
-set(GMX_QMMM_PROGRAM "none"
- CACHE STRING "QM package choices: none,gaussian,mopac,gamess,orca")
+
+gmx_option_multichoice(
+ GMX_QMMM_PROGRAM
+ "QM package for QM/MM"
+ None
+ none gaussian mopac gamess orca)
+
option(GMX_BROKEN_CALLOC "Work around broken calloc()" OFF)
mark_as_advanced(GMX_BROKEN_CALLOC)
-option(GMX_MPI_IN_PLACE "Enable MPI_IN_PLACE for MPIs that have it defined" ON)
-mark_as_advanced(GMX_MPI_IN_PLACE)
option(GMX_LOAD_PLUGINS "Compile with plugin support, needed to read VMD supported file formats" ON)
mark_as_advanced(GMX_LOAD_PLUGINS)
set(GMX_CPU_ACCELERATION_X86_SSE2 1)
# The user should not be able to set this orthogonally to the acceleration
set(GMX_X86_SSE2 1)
- if (NOT ACCELERATION_QUIETLY)
- message(STATUS "Enabling SSE2 Gromacs acceleration")
- endif()
+ set(ACCELERATION_STATUS_MESSAGE
- "Enabling SSE2 Gromacs acceleration, and it will help compiler optimization.")
++ "Enabling SSE2 Gromacs acceleration")
elseif(${GMX_CPU_ACCELERATION} STREQUAL "SSE4.1")
# The user should not be able to set this orthogonally to the acceleration
set(GMX_X86_SSE4_1 1)
set(GMX_X86_SSE2 1)
- if (NOT ACCELERATION_QUIETLY)
- message(STATUS "Enabling SSE4.1 Gromacs acceleration")
- endif()
+ set(ACCELERATION_STATUS_MESSAGE
- "Enabling SSE4.1 Gromacs acceleration, and it will help compiler optimization.")
++ "Enabling SSE4.1 Gromacs acceleration")
- if(CMAKE_C_COMPILER_ID MATCHES "Intel" AND C_COMPILER_VERSION VERSION_EQUAL "11.1")
- message(FATAL_ERROR "You are using Intel compiler version 11.1, and that compiler is known to produce incorrect results with SSE4.1 acceleration. You need to use another compiler (e.g. icc 12 or newer) or different acceleration (probably slower simulations).")
- endif()
elseif(${GMX_CPU_ACCELERATION} STREQUAL "AVX_128_FMA" OR ${GMX_CPU_ACCELERATION} STREQUAL "AVX_256")
# Set the AVX compiler flag for both these choices!
message(STATUS "Clang ${C_COMPILER_VERSION} detected, enabling FMA bug workaround")
set(EXTRA_C_FLAGS "${EXTRA_C_FLAGS} -no-integrated-as")
endif()
- "Enabling 128-bit AVX Gromacs acceleration (with fused-multiply add), and it will help compiler optimization.")
+ if (GMX_USE_CLANG_CXX_FMA_BUG_WORKAROUND)
+ # we assume that we have an external assembler that supports AVX
+ message(STATUS "Clang ${CXX_COMPILER_VERSION} detected, enabling FMA bug workaround")
+ set(EXTRA_CXX_FLAGS "${EXTRA_CXX_FLAGS} -no-integrated-as")
+ endif()
+
+ set(GMX_CPU_ACCELERATION_X86_AVX_128_FMA 1)
+ set(GMX_X86_AVX_128_FMA 1)
+ set(ACCELERATION_STATUS_MESSAGE
++ "Enabling 128-bit AVX Gromacs acceleration (with fused-multiply add)")
else()
# If we are not doing AVX_128, it must be AVX_256...
set(GMX_CPU_ACCELERATION_X86_AVX_256 1)
set(GMX_X86_AVX_256 1)
- if (NOT ACCELERATION_QUIETLY)
- message(STATUS "Enabling 256-bit AVX Gromacs acceleration")
- endif()
+ set(ACCELERATION_STATUS_MESSAGE
- "Enabling 256-bit AVX Gromacs acceleration, and it will help compiler optimization.")
++ "Enabling 256-bit AVX Gromacs acceleration")
endif()
# Unfortunately gcc-4.5.2 and gcc-4.6.0 has a bug where they use the wrong datatype for the formal
endif(GMX_LOAD_PLUGINS)
set(VMD_QUIETLY TRUE CACHE INTERNAL "")
-if(EXISTS "${CMAKE_SOURCE_DIR}/admin/.isreposource")
- if(NOT CMAKE_CROSSCOMPILING)
- option(GMX_BUILD_MANPAGES "Build man pages" ON)
- else()
- message(STATUS "Building the man pages is not available when "
- "cross-compiling the developer version from git")
- endif()
-else()
- #make sure source package contains all man pages
- if(NOT EXISTS "${CMAKE_SOURCE_DIR}/man/man1/ngmx.1")
- message(FATAL_ERROR "Man pages are missing from source package.")
- endif()
-endif()
-mark_as_advanced(GMX_BUILD_MANPAGES)
-
# Math and thread libraries must often come after all others when linking...
if(HAVE_LIBM)
- list(APPEND GMX_EXTRA_LIBRARIES m)
+ list(APPEND GMX_EXTRA_LIBRARIES m)
endif(HAVE_LIBM)
+ if (${CMAKE_SYSTEM_NAME} MATCHES "BlueGene")
+ check_library_exists(mass_simd atan2f4 "" HAVE_MASS_SIMD)
+ if(HAVE_MASS_SIMD)
+ list(APPEND GMX_EXTRA_LIBRARIES mass_simd)
+ else()
+ message(FATAL_ERROR "Could not link to the SIMD version of the IBM MASS library. Please adjust your CMAKE_PREFIX_PATH to contain it")
+ endif()
+ endif()
if(GMX_FAHCORE)
- set(COREWRAP_INCLUDE_DIR "${CMAKE_SOURCE_DIR}/../corewrap" CACHE STRING
+ set(COREWRAP_INCLUDE_DIR "${CMAKE_SOURCE_DIR}/../corewrap" CACHE STRING
"Path to swindirect.h")
include_directories(${COREWRAP_INCLUDE_DIR})
- set_property(CACHE GMX_COOL_QUOTES PROPERTY VALUE OFF)
endif(GMX_FAHCORE)
# # # # # # # # # # NO MORE TESTS AFTER THIS LINE! # # # # # # # # # # #
above instructions.
mpicc is used for compiling and linking. This can make it awkward to
- attempt to use IBM's optimized BLAS/LAPACK called ESSL. Since mdrun is
- the only part of \gromacs{} that should normally run on the compute
- nodes, and there is nearly no need for linear algebra support for
- mdrun, it is recommended to use the \gromacs{} built-in linear algebra
- routines - it is rare for this to be a bottleneck.
-
+ attempt to use IBM's optimized BLAS/LAPACK called ESSL (see the
+ section on linear algebra). Since mdrun is the only part of \gromacs{}
+ that should normally run on the compute nodes, and there is nearly no
+ need for linear algebra support for mdrun, it is recommended to use
+ the \gromacs{} built-in linear algebra routines - it is rare for this
+ to be a bottleneck.
+
+ The recommended configuration is to use
\begin{verbatim}
- cmake .. -DCMAKE_TOOLCHAIN_FILE=BlueGeneQ-static-XL-C \
+ cmake .. -DCMAKE_TOOLCHAIN_FILE=Platform/BlueGeneQ-static-XL-CXX \
-DCMAKE_PREFIX_PATH=/your/fftw/installation/prefix \
- -DGMX_MPI=on
-make mdrun
-make install-mdrun
++ -DGMX_MPI=ON \
+ -DGMX_BUILD_MDRUN_ONLY=ON
+make
+make install
\end{verbatim}
+ which will build a statically-linked MPI-enabled mdrun for the back
+ end. Otherwise, GROMACS default configuration behaviour applies.
+
It is possible to configure and make the remaining \gromacs{} tools
- with the compute node toolchain, but as none of those tools are
- \mpi{}-aware, this would not normally be useful. Instead, these should
- be planned to run on the login node, and a seperate \gromacs{}
- installation performed for that using the login node's toolchain.
+ with the compute-node toolchain, but as none of those tools are
+ \mpi{}-aware and could then only run on the compute nodes, this
+ would not normally be useful. Instead, these should be planned
+ to run on the login node, and a separate \gromacs{} installation
+ performed for that using the login node's toolchain - not the
+ above platform file, or any other compute-node toolchain.
+
+ Note that only the MPI build is available for the compute-node
+ toolchains. The GROMACS thread-MPI or serial builds are not useful at
+ all on BlueGene/Q.
\subsubsection{Fujitsu PRIMEHPC}
-#
-# This file is part of the GROMACS molecular simulation package.
-#
-# Copyright (c) 2012,2013, by the GROMACS development team, led by
-# David van der Spoel, Berk Hess, Erik Lindahl, and including many
-# others, as listed in the AUTHORS file in the top-level source
-# directory and at http://www.gromacs.org.
-#
-# GROMACS is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public License
-# as published by the Free Software Foundation; either version 2.1
-# of the License, or (at your option) any later version.
-#
-# GROMACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with GROMACS; if not, see
-# http://www.gnu.org/licenses, or write to the Free Software Foundation,
-# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# If you want to redistribute modifications to GROMACS, please
-# consider that scientific software is very special. Version
-# control is crucial - bugs must be traceable. We will be happy to
-# consider code for inclusion in the official distribution, but
-# derived work must not be called official GROMACS. Details are found
-# in the README & COPYING files - if they are missing, get the
-# official version at http://www.gromacs.org.
-#
-# To help us fund GROMACS development, we humbly ask that you cite
-# the research papers on the package. Check out http://www.gromacs.org.
-#
# - Check the username performing the build, as well as date and time
#
- # GMX_DETECT_ACCELERATION(GMX_SUGGESTED_ACCELERATION)
+ # gmx_detect_acceleration(GMX_SUGGESTED_CPU_ACCELERATION)
#
# Try to detect CPU information and suggest an acceleration option
- # (such as SSE/AVX) that fits the current CPU.
+ # (such as SSE/AVX) that fits the current CPU. These functions assume
+ # that gmx_detect_target_architecture() has already been run, so that
+ # things like GMX_IS_X86 are already available.
#
- # GMX_SUGGESTED_ACCELERATION
+ # Sets ${GMX_SUGGESTED_CPU_ACCELERATION} in the parent scope if
+ # GMX_CPU_ACCELERATION is not set (e.g. by the user, or a previous run
+ # of CMake).
#
# we rely on inline asm support for GNU!
# Get CPU acceleration information
try_run(GMX_CPUID_RUN_ACC GMX_CPUID_COMPILED
${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}/src/gmxlib/gmx_cpuid.c
- COMPILE_DEFINITIONS "@GCC_INLINE_ASM_DEFINE@ -I${CMAKE_SOURCE_DIR}/include -DGMX_CPUID_STANDALONE -DGMX_IS_X86"
+ ${CMAKE_SOURCE_DIR}/src/gromacs/gmxlib/gmx_cpuid.c
- COMPILE_DEFINITIONS "@GCC_INLINE_ASM_DEFINE@ -I${CMAKE_SOURCE_DIR}/src/gromacs/legacyheaders/ -DGMX_CPUID_STANDALONE"
++ COMPILE_DEFINITIONS "@GCC_INLINE_ASM_DEFINE@ -I${CMAKE_SOURCE_DIR}/src/gromacs/legacyheaders -DGMX_CPUID_STANDALONE -DGMX_IS_X86"
RUN_OUTPUT_VARIABLE OUTPUT_TMP
COMPILE_OUTPUT_VARIABLE GMX_CPUID_COMPILE_OUTPUT
ARGS "-acceleration")
if(NOT CMAKE_CROSSCOMPILING)
# Get CPU acceleration information
- set(_compile_definitions "@GCC_INLINE_ASM_DEFINE@ -I${CMAKE_SOURCE_DIR}/include -DGMX_CPUID_STANDALONE -DGMX_IS_X86")
++ set(_compile_definitions "@GCC_INLINE_ASM_DEFINE@ -I${CMAKE_SOURCE_DIR}/src/gromacs/legacyheaders -DGMX_CPUID_STANDALONE -DGMX_IS_X86")
try_run(GMX_CPUID_RUN_VENDOR GMX_CPUID_COMPILED
${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}/src/gmxlib/gmx_cpuid.c
+ ${CMAKE_SOURCE_DIR}/src/gromacs/gmxlib/gmx_cpuid.c
- COMPILE_DEFINITIONS "@GCC_INLINE_ASM_DEFINE@ -I${CMAKE_SOURCE_DIR}/src/gromacs/legacyheaders/ -DGMX_CPUID_STANDALONE"
+ COMPILE_DEFINITIONS ${_compile_definitions}
RUN_OUTPUT_VARIABLE OUTPUT_CPU_VENDOR ARGS "-vendor")
try_run(GMX_CPUID_RUN_BRAND GMX_CPUID_COMPILED
${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}/src/gmxlib/gmx_cpuid.c
+ ${CMAKE_SOURCE_DIR}/src/gromacs/gmxlib/gmx_cpuid.c
- COMPILE_DEFINITIONS "@GCC_INLINE_ASM_DEFINE@ -I${CMAKE_SOURCE_DIR}/src/gromacs/legacyheaders/ -DGMX_CPUID_STANDALONE"
+ COMPILE_DEFINITIONS ${_compile_definitions}
RUN_OUTPUT_VARIABLE OUTPUT_CPU_BRAND ARGS "-brand")
try_run(GMX_CPUID_RUN_FAMILY GMX_CPUID_COMPILED
${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}/src/gmxlib/gmx_cpuid.c
+ ${CMAKE_SOURCE_DIR}/src/gromacs/gmxlib/gmx_cpuid.c
- COMPILE_DEFINITIONS "@GCC_INLINE_ASM_DEFINE@ -I${CMAKE_SOURCE_DIR}/src/gromacs/legacyheaders/ -DGMX_CPUID_STANDALONE"
+ COMPILE_DEFINITIONS ${_compile_definitions}
RUN_OUTPUT_VARIABLE OUTPUT_CPU_FAMILY ARGS "-family")
try_run(GMX_CPUID_RUN_MODEL GMX_CPUID_COMPILED
${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}/src/gmxlib/gmx_cpuid.c
+ ${CMAKE_SOURCE_DIR}/src/gromacs/gmxlib/gmx_cpuid.c
- COMPILE_DEFINITIONS "@GCC_INLINE_ASM_DEFINE@ -I${CMAKE_SOURCE_DIR}/src/gromacs/legacyheaders/ -DGMX_CPUID_STANDALONE"
+ COMPILE_DEFINITIONS ${_compile_definitions}
RUN_OUTPUT_VARIABLE OUTPUT_CPU_MODEL ARGS "-model")
try_run(GMX_CPUID_RUN_STEPPING GMX_CPUID_COMPILED
${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}/src/gmxlib/gmx_cpuid.c
+ ${CMAKE_SOURCE_DIR}/src/gromacs/gmxlib/gmx_cpuid.c
- COMPILE_DEFINITIONS "@GCC_INLINE_ASM_DEFINE@ -I${CMAKE_SOURCE_DIR}/src/gromacs/legacyheaders/ -DGMX_CPUID_STANDALONE"
+ COMPILE_DEFINITIONS ${_compile_definitions}
RUN_OUTPUT_VARIABLE OUTPUT_CPU_STEPPING ARGS "-stepping")
try_run(GMX_CPUID_RUN_FEATURES GMX_CPUID_COMPILED
${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}/src/gmxlib/gmx_cpuid.c
+ ${CMAKE_SOURCE_DIR}/src/gromacs/gmxlib/gmx_cpuid.c
- COMPILE_DEFINITIONS "@GCC_INLINE_ASM_DEFINE@ -I${CMAKE_SOURCE_DIR}/src/gromacs/legacyheaders/ -DGMX_CPUID_STANDALONE"
+ COMPILE_DEFINITIONS ${_compile_definitions}
RUN_OUTPUT_VARIABLE OUTPUT_CPU_FEATURES ARGS "-features")
+ unset(_compile_definitions)
string(STRIP "@OUTPUT_CPU_VENDOR@" OUTPUT_CPU_VENDOR)
string(STRIP "@OUTPUT_CPU_BRAND@" OUTPUT_CPU_BRAND)
--- /dev/null
+#
+# This file is part of the GROMACS molecular simulation package.
+#
+# Copyright (c) 2010,2011,2012,2013, by the GROMACS development team, led by
+# David van der Spoel, Berk Hess, Erik Lindahl, and including many
+# others, as listed in the AUTHORS file in the top-level source
+# directory and at http://www.gromacs.org.
+#
+# GROMACS is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public License
+# as published by the Free Software Foundation; either version 2.1
+# of the License, or (at your option) any later version.
+#
+# GROMACS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with GROMACS; if not, see
+# http://www.gnu.org/licenses, or write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# If you want to redistribute modifications to GROMACS, please
+# consider that scientific software is very special. Version
+# control is crucial - bugs must be traceable. We will be happy to
+# consider code for inclusion in the official distribution, but
+# derived work must not be called official GROMACS. Details are found
+# in the README & COPYING files - if they are missing, get the
+# official version at http://www.gromacs.org.
+#
+# To help us fund GROMACS development, we humbly ask that you cite
+# the research papers on the package. Check out http://www.gromacs.org.
+
+set(LIBGROMACS_SOURCES)
+
+function (gmx_install_headers DESTINATION)
+ if (NOT GMX_BUILD_MDRUN_ONLY)
+ if (DESTINATION)
+ set(DESTINATION ${INCL_INSTALL_DIR}/gromacs/${DESTINATION})
+ else()
+ set(DESTINATION ${INCL_INSTALL_DIR}/gromacs)
+ endif()
+ install(FILES ${ARGN} DESTINATION ${DESTINATION} COMPONENT development)
+ endif()
+endfunction ()
+
+add_subdirectory(gmxlib)
+add_subdirectory(mdlib)
+add_subdirectory(gmxpreprocess)
+add_subdirectory(commandline)
+add_subdirectory(fft)
+add_subdirectory(linearalgebra)
+add_subdirectory(onlinehelp)
+add_subdirectory(options)
+add_subdirectory(utility)
+if (NOT GMX_BUILD_MDRUN_ONLY)
+ add_subdirectory(legacyheaders)
+ add_subdirectory(gmxana)
+ add_subdirectory(analysisdata)
+ add_subdirectory(selection)
+ add_subdirectory(trajectoryanalysis)
+endif ()
+
+list(APPEND LIBGROMACS_SOURCES ${GMXLIB_SOURCES} ${MDLIB_SOURCES})
+
+file(GLOB LIBGROMACS_HEADERS *.h)
+configure_file(version.h.cmakein version.h)
+gmx_install_headers("" ${LIBGROMACS_HEADERS})
+gmx_install_headers("" ${CMAKE_CURRENT_BINARY_DIR}/version.h)
+
+# Add target that generates gitversion.c every time make is run
+# if git version info is requested
+# This code is here instead of utility/CMakeLists.txt because CMake
+# ignores set_source_file_properties from subdirectories.
+if (GMX_GIT_VERSION_INFO)
+ set(GENERATED_VERSION_FILE ${CMAKE_CURRENT_BINARY_DIR}/utility/gitversion.c)
+ add_custom_target(gmx-version ALL
+ COMMAND ${CMAKE_COMMAND}
+ -D GIT_EXECUTABLE="${GIT_EXECUTABLE}"
+ -D PROJECT_VERSION="${PROJECT_VERSION}"
+ -D PROJECT_SOURCE_DIR="${PROJECT_SOURCE_DIR}"
+ -D VERSION_CMAKEIN="${CMAKE_CURRENT_SOURCE_DIR}/utility/gitversion.c.cmakein"
+ -D VERSION_OUT=${GENERATED_VERSION_FILE}
+ -P ${CMAKE_SOURCE_DIR}/cmake/gmxGenerateVersionInfo.cmake
+ WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+ DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/utility/gitversion.c.cmakein
+ COMMENT "Generating git version information")
+ set_source_files_properties(${GENERATED_VERSION_FILE}
+ PROPERTIES GENERATED true)
+ list(APPEND LIBGROMACS_SOURCES ${GENERATED_VERSION_FILE})
+endif()
+
+# apply gcc 4.4.x bug workaround
+if(GMX_USE_GCC44_BUG_WORKAROUND)
+ include(gmxGCC44O3BugWorkaround)
+ gmx_apply_gcc44_bug_workaround("gmxlib/bondfree.c")
+ gmx_apply_gcc44_bug_workaround("mdlib/force.c")
+ gmx_apply_gcc44_bug_workaround("mdlib/constr.c")
+endif()
+
+add_library(libgromacs ${LIBGROMACS_SOURCES})
+if (GMX_GIT_VERSION_INFO)
+ add_dependencies(libgromacs gmx-version)
+endif ()
+
+if(GMX_BUILD_OWN_FFTW)
++ # Only needed for cmake 2.8.7, otherwise this should be automatic
+ # This dependency has to be made here rather than the CMakeLists.txt that
+ # does the FFTW build, because of the order in which
+ # add_subdirectory() calls are made in the top-level CMakeLists.txt; the
+ # md library target does not necessarily exist yet. Also enabling and
+ # disabling GMX_BUILD_OWN_FFTW changes dependencies correctly.
+ add_dependencies(libgromacs gmxfftw)
+endif()
+
+target_link_libraries(libgromacs ${GMX_GPU_LIBRARIES}
+ ${GMX_EXTRA_LIBRARIES}
+ ${FFT_LIBRARIES} ${LINEAR_ALGEBRA_LIBRARIES}
+ ${XML_LIBRARIES} ${GSL_LIBRARIES}
+ ${THREAD_LIB} ${GMX_SHARED_LINKER_FLAGS})
+set_target_properties(libgromacs PROPERTIES
+ OUTPUT_NAME "gromacs${GMX_LIBS_SUFFIX}"
+ SOVERSION ${SOVERSION}
+ COMPILE_FLAGS "${OpenMP_C_FLAGS}")
+
+# Only install the library in mdrun-only mode if it is actually necessary
+# for the binary
+if (NOT GMX_BUILD_MDRUN_ONLY OR BUILD_SHARED_LIBS)
+ install(TARGETS libgromacs DESTINATION ${LIB_INSTALL_DIR} COMPONENT libraries)
+endif()
+
+if (NOT GMX_BUILD_MDRUN_ONLY)
+ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/libgromacs.pc.cmakein
+ ${CMAKE_CURRENT_BINARY_DIR}/libgromacs.pc @ONLY)
+ install(FILES ${CMAKE_CURRENT_BINARY_DIR}/libgromacs.pc
+ DESTINATION ${LIB_INSTALL_DIR}/pkgconfig
+ RENAME "libgromacs${GMX_LIBS_SUFFIX}.pc"
+ COMPONENT development)
+endif()
+
+if (INSTALL_CUDART_LIB) #can be set manual by user
+ if (GMX_GPU)
+ foreach(CUDA_LIB ${CUDA_LIBRARIES})
+ string(REGEX MATCH "cudart" IS_CUDART ${CUDA_LIB})
+ if(IS_CUDART) #libcuda should not be installed
+ #install also name-links (linker uses those)
+ file(GLOB CUDA_LIBS ${CUDA_LIB}*)
+ install(FILES ${CUDA_LIBS} DESTINATION
+ ${LIB_INSTALL_DIR} COMPONENT libraries)
+ endif()
+ endforeach()
+ else()
+ message(WARNING "INSTALL_CUDART_LIB only makes sense with GMX_GPU")
+ endif()
+endif ()
--- /dev/null
- MPI_Alltoall(lout2, N[s]*pM[s]*K[s]*sizeof(t_complex)/sizeof(real), GMX_MPI_REAL, lout3, N[s]*pM[s]*K[s]*sizeof(t_complex)/sizeof(real), GMX_MPI_REAL, cart[s]);
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2012, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * Groningen Machine for Chemical Simulation
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <algorithm>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#ifdef NOGMX
+#define GMX_PARALLEL_ENV_INITIALIZED 1
+#else
+#ifdef GMX_MPI
+#define GMX_PARALLEL_ENV_INITIALIZED 1
+#else
+#define GMX_PARALLEL_ENV_INITIALIZED 0
+#endif
+#endif
+
+#include "gromacs/utility/gmxmpi.h"
+
+#ifdef GMX_OPENMP
+/* TODO: Do we still need this? Are we still planning ot use fftw + OpenMP? */
+#define FFT5D_THREADS
+/* requires fftw compiled with openmp */
+/* #define FFT5D_FFTW_THREADS (now set by cmake) */
+#endif
+
+#include "fft5d.h"
+#include <float.h>
+#include <math.h>
+#include <assert.h>
+#include "smalloc.h"
+
+#ifndef __FLT_EPSILON__
+#define __FLT_EPSILON__ FLT_EPSILON
+#define __DBL_EPSILON__ DBL_EPSILON
+#endif
+
+#ifdef NOGMX
+FILE* debug = 0;
+#endif
+
+#include "gmx_fatal.h"
+
+
+#ifdef GMX_FFT_FFTW3
+#include "thread_mpi/mutex.h"
+#include "gromacs/utility/exceptions.h"
+/* none of the fftw3 calls, except execute(), are thread-safe, so
+ we need to serialize them with this mutex. */
+static tMPI::mutex big_fftw_mutex;
+#define FFTW_LOCK try { big_fftw_mutex.lock(); } GMX_CATCH_ALL_AND_EXIT_WITH_FATAL_ERROR
+#define FFTW_UNLOCK try { big_fftw_mutex.unlock(); } GMX_CATCH_ALL_AND_EXIT_WITH_FATAL_ERROR
+#endif /* GMX_FFT_FFTW3 */
+
+/* largest factor smaller than sqrt */
+static int lfactor(int z)
+{
+ int i;
+ for (i = static_cast<int>(sqrt(static_cast<double>(z)));; i--)
+ {
+ if (z%i == 0)
+ {
+ return i;
+ }
+ }
+ return 1;
+}
+
+/* largest factor */
+static int l2factor(int z)
+{
+ int i;
+ if (z == 1)
+ {
+ return 1;
+ }
+ for (i = z/2;; i--)
+ {
+ if (z%i == 0)
+ {
+ return i;
+ }
+ }
+ return 1;
+}
+
+/* largest prime factor: WARNING: slow recursion, only use for small numbers */
+static int lpfactor(int z)
+{
+ int f = l2factor(z);
+ if (f == 1)
+ {
+ return z;
+ }
+ return std::max(lpfactor(f), lpfactor(z/f));
+}
+
+#ifndef GMX_MPI
+#ifdef HAVE_GETTIMEOFDAY
+#include <sys/time.h>
+double MPI_Wtime()
+{
+ struct timeval tv;
+ gettimeofday(&tv, 0);
+ return tv.tv_sec+tv.tv_usec*1e-6;
+}
+#else
+double MPI_Wtime()
+{
+ return 0.0;
+}
+#endif
+#endif
+
+static int vmax(int* a, int s)
+{
+ int i, max = 0;
+ for (i = 0; i < s; i++)
+ {
+ if (a[i] > max)
+ {
+ max = a[i];
+ }
+ }
+ return max;
+}
+
+
+/* NxMxK the size of the data
+ * comm communicator to use for fft5d
+ * P0 number of processor in 1st axes (can be null for automatic)
+ * lin is allocated by fft5d because size of array is only known after planning phase
+ * rlout2 is only used as intermediate buffer - only returned after allocation to reuse for back transform - should not be used by caller
+ */
+fft5d_plan fft5d_plan_3d(int NG, int MG, int KG, MPI_Comm comm[2], int flags, t_complex** rlin, t_complex** rlout, t_complex** rlout2, t_complex** rlout3, int nthreads)
+{
+
+ int P[2], bMaster, prank[2], i, t;
+ int rNG, rMG, rKG;
+ int *N0 = 0, *N1 = 0, *M0 = 0, *M1 = 0, *K0 = 0, *K1 = 0, *oN0 = 0, *oN1 = 0, *oM0 = 0, *oM1 = 0, *oK0 = 0, *oK1 = 0;
+ int N[3], M[3], K[3], pN[3], pM[3], pK[3], oM[3], oK[3], *iNin[3] = {0}, *oNin[3] = {0}, *iNout[3] = {0}, *oNout[3] = {0};
+ int C[3], rC[3], nP[2];
+ int lsize;
+ t_complex *lin = 0, *lout = 0, *lout2 = 0, *lout3 = 0;
+ fft5d_plan plan;
+ int s;
+
+ /* comm, prank and P are in the order of the decomposition (plan->cart is in the order of transposes) */
+#ifdef GMX_MPI
+ if (GMX_PARALLEL_ENV_INITIALIZED && comm[0] != MPI_COMM_NULL)
+ {
+ MPI_Comm_size(comm[0], &P[0]);
+ MPI_Comm_rank(comm[0], &prank[0]);
+ }
+ else
+#endif
+ {
+ P[0] = 1;
+ prank[0] = 0;
+ }
+#ifdef GMX_MPI
+ if (GMX_PARALLEL_ENV_INITIALIZED && comm[1] != MPI_COMM_NULL)
+ {
+ MPI_Comm_size(comm[1], &P[1]);
+ MPI_Comm_rank(comm[1], &prank[1]);
+ }
+ else
+#endif
+ {
+ P[1] = 1;
+ prank[1] = 0;
+ }
+
+ bMaster = (prank[0] == 0 && prank[1] == 0);
+
+
+ if (debug)
+ {
+ fprintf(debug, "FFT5D: Using %dx%d processor grid, rank %d,%d\n",
+ P[0], P[1], prank[0], prank[1]);
+ }
+
+ if (bMaster)
+ {
+ if (debug)
+ {
+ fprintf(debug, "FFT5D: N: %d, M: %d, K: %d, P: %dx%d, real2complex: %d, backward: %d, order yz: %d, debug %d\n",
+ NG, MG, KG, P[0], P[1], (flags&FFT5D_REALCOMPLEX) > 0, (flags&FFT5D_BACKWARD) > 0, (flags&FFT5D_ORDER_YZ) > 0, (flags&FFT5D_DEBUG) > 0);
+ }
+ /* The check below is not correct, one prime factor 11 or 13 is ok.
+ if (fft5d_fmax(fft5d_fmax(lpfactor(NG),lpfactor(MG)),lpfactor(KG))>7) {
+ printf("WARNING: FFT very slow with prime factors larger 7\n");
+ printf("Change FFT size or in case you cannot change it look at\n");
+ printf("http://www.fftw.org/fftw3_doc/Generating-your-own-code.html\n");
+ }
+ */
+ }
+
+ if (NG == 0 || MG == 0 || KG == 0)
+ {
+ if (bMaster)
+ {
+ printf("FFT5D: FATAL: Datasize cannot be zero in any dimension\n");
+ }
+ return 0;
+ }
+
+ rNG = NG; rMG = MG; rKG = KG;
+
+ if (flags&FFT5D_REALCOMPLEX)
+ {
+ if (!(flags&FFT5D_BACKWARD))
+ {
+ NG = NG/2+1;
+ }
+ else
+ {
+ if (!(flags&FFT5D_ORDER_YZ))
+ {
+ MG = MG/2+1;
+ }
+ else
+ {
+ KG = KG/2+1;
+ }
+ }
+ }
+
+
+ /*for transpose we need to know the size for each processor not only our own size*/
+
+ N0 = (int*)malloc(P[0]*sizeof(int)); N1 = (int*)malloc(P[1]*sizeof(int));
+ M0 = (int*)malloc(P[0]*sizeof(int)); M1 = (int*)malloc(P[1]*sizeof(int));
+ K0 = (int*)malloc(P[0]*sizeof(int)); K1 = (int*)malloc(P[1]*sizeof(int));
+ oN0 = (int*)malloc(P[0]*sizeof(int)); oN1 = (int*)malloc(P[1]*sizeof(int));
+ oM0 = (int*)malloc(P[0]*sizeof(int)); oM1 = (int*)malloc(P[1]*sizeof(int));
+ oK0 = (int*)malloc(P[0]*sizeof(int)); oK1 = (int*)malloc(P[1]*sizeof(int));
+
+ for (i = 0; i < P[0]; i++)
+ {
+ #define EVENDIST
+ #ifndef EVENDIST
+ oN0[i] = i*ceil((double)NG/P[0]);
+ oM0[i] = i*ceil((double)MG/P[0]);
+ oK0[i] = i*ceil((double)KG/P[0]);
+ #else
+ oN0[i] = (NG*i)/P[0];
+ oM0[i] = (MG*i)/P[0];
+ oK0[i] = (KG*i)/P[0];
+ #endif
+ }
+ for (i = 0; i < P[1]; i++)
+ {
+ #ifndef EVENDIST
+ oN1[i] = i*ceil((double)NG/P[1]);
+ oM1[i] = i*ceil((double)MG/P[1]);
+ oK1[i] = i*ceil((double)KG/P[1]);
+ #else
+ oN1[i] = (NG*i)/P[1];
+ oM1[i] = (MG*i)/P[1];
+ oK1[i] = (KG*i)/P[1];
+ #endif
+ }
+ for (i = 0; i < P[0]-1; i++)
+ {
+ N0[i] = oN0[i+1]-oN0[i];
+ M0[i] = oM0[i+1]-oM0[i];
+ K0[i] = oK0[i+1]-oK0[i];
+ }
+ N0[P[0]-1] = NG-oN0[P[0]-1];
+ M0[P[0]-1] = MG-oM0[P[0]-1];
+ K0[P[0]-1] = KG-oK0[P[0]-1];
+ for (i = 0; i < P[1]-1; i++)
+ {
+ N1[i] = oN1[i+1]-oN1[i];
+ M1[i] = oM1[i+1]-oM1[i];
+ K1[i] = oK1[i+1]-oK1[i];
+ }
+ N1[P[1]-1] = NG-oN1[P[1]-1];
+ M1[P[1]-1] = MG-oM1[P[1]-1];
+ K1[P[1]-1] = KG-oK1[P[1]-1];
+
+ /* for step 1-3 the local N,M,K sizes of the transposed system
+ C: contiguous dimension, and nP: number of processor in subcommunicator
+ for that step */
+
+
+ pM[0] = M0[prank[0]];
+ oM[0] = oM0[prank[0]];
+ pK[0] = K1[prank[1]];
+ oK[0] = oK1[prank[1]];
+ C[0] = NG;
+ rC[0] = rNG;
+ if (!(flags&FFT5D_ORDER_YZ))
+ {
+ N[0] = vmax(N1, P[1]);
+ M[0] = M0[prank[0]];
+ K[0] = vmax(K1, P[1]);
+ pN[0] = N1[prank[1]];
+ iNout[0] = N1;
+ oNout[0] = oN1;
+ nP[0] = P[1];
+ C[1] = KG;
+ rC[1] = rKG;
+ N[1] = vmax(K0, P[0]);
+ pN[1] = K0[prank[0]];
+ iNin[1] = K1;
+ oNin[1] = oK1;
+ iNout[1] = K0;
+ oNout[1] = oK0;
+ M[1] = vmax(M0, P[0]);
+ pM[1] = M0[prank[0]];
+ oM[1] = oM0[prank[0]];
+ K[1] = N1[prank[1]];
+ pK[1] = N1[prank[1]];
+ oK[1] = oN1[prank[1]];
+ nP[1] = P[0];
+ C[2] = MG;
+ rC[2] = rMG;
+ iNin[2] = M0;
+ oNin[2] = oM0;
+ M[2] = vmax(K0, P[0]);
+ pM[2] = K0[prank[0]];
+ oM[2] = oK0[prank[0]];
+ K[2] = vmax(N1, P[1]);
+ pK[2] = N1[prank[1]];
+ oK[2] = oN1[prank[1]];
+ free(N0); free(oN0); /*these are not used for this order*/
+ free(M1); free(oM1); /*the rest is freed in destroy*/
+ }
+ else
+ {
+ N[0] = vmax(N0, P[0]);
+ M[0] = vmax(M0, P[0]);
+ K[0] = K1[prank[1]];
+ pN[0] = N0[prank[0]];
+ iNout[0] = N0;
+ oNout[0] = oN0;
+ nP[0] = P[0];
+ C[1] = MG;
+ rC[1] = rMG;
+ N[1] = vmax(M1, P[1]);
+ pN[1] = M1[prank[1]];
+ iNin[1] = M0;
+ oNin[1] = oM0;
+ iNout[1] = M1;
+ oNout[1] = oM1;
+ M[1] = N0[prank[0]];
+ pM[1] = N0[prank[0]];
+ oM[1] = oN0[prank[0]];
+ K[1] = vmax(K1, P[1]);
+ pK[1] = K1[prank[1]];
+ oK[1] = oK1[prank[1]];
+ nP[1] = P[1];
+ C[2] = KG;
+ rC[2] = rKG;
+ iNin[2] = K1;
+ oNin[2] = oK1;
+ M[2] = vmax(N0, P[0]);
+ pM[2] = N0[prank[0]];
+ oM[2] = oN0[prank[0]];
+ K[2] = vmax(M1, P[1]);
+ pK[2] = M1[prank[1]];
+ oK[2] = oM1[prank[1]];
+ free(N1); free(oN1); /*these are not used for this order*/
+ free(K0); free(oK0); /*the rest is freed in destroy*/
+ }
+ N[2] = pN[2] = -1; /*not used*/
+
+ /*
+ Difference between x-y-z regarding 2d decomposition is whether they are
+ distributed along axis 1, 2 or both
+ */
+
+ /* int lsize = fmax(N[0]*M[0]*K[0]*nP[0],N[1]*M[1]*K[1]*nP[1]); */
+ lsize = std::max(N[0]*M[0]*K[0]*nP[0], std::max(N[1]*M[1]*K[1]*nP[1], C[2]*M[2]*K[2]));
+ /* int lsize = fmax(C[0]*M[0]*K[0],fmax(C[1]*M[1]*K[1],C[2]*M[2]*K[2])); */
+ if (!(flags&FFT5D_NOMALLOC))
+ {
+ snew_aligned(lin, lsize, 32);
+ snew_aligned(lout, lsize, 32);
+ if (nthreads > 1)
+ {
+ /* We need extra transpose buffers to avoid OpenMP barriers */
+ snew_aligned(lout2, lsize, 32);
+ snew_aligned(lout3, lsize, 32);
+ }
+ else
+ {
+ /* We can reuse the buffers to avoid cache misses */
+ lout2 = lin;
+ lout3 = lout;
+ }
+ }
+ else
+ {
+ lin = *rlin;
+ lout = *rlout;
+ if (nthreads > 1)
+ {
+ lout2 = *rlout2;
+ lout3 = *rlout3;
+ }
+ else
+ {
+ lout2 = lin;
+ lout3 = lout;
+ }
+ }
+
+ plan = (fft5d_plan)calloc(1, sizeof(struct fft5d_plan_t));
+
+
+ if (debug)
+ {
+ fprintf(debug, "Running on %d threads\n", nthreads);
+ }
+
+#ifdef GMX_FFT_FFTW3 /*if not FFTW - then we don't do a 3d plan but instead use only 1D plans */
+ /* It is possible to use the 3d plan with OMP threads - but in that case it is not allowed to be called from
+ * within a parallel region. For now deactivated. If it should be supported it has to made sure that
+ * that the execute of the 3d plan is in a master/serial block (since it contains it own parallel region)
+ * and that the 3d plan is faster than the 1d plan.
+ */
+ if ((!(flags&FFT5D_INPLACE)) && (!(P[0] > 1 || P[1] > 1)) && nthreads == 1) /*don't do 3d plan in parallel or if in_place requested */
+ {
+ int fftwflags = FFTW_DESTROY_INPUT;
+ FFTW(iodim) dims[3];
+ int inNG = NG, outMG = MG, outKG = KG;
+
+ FFTW_LOCK;
+ if (!(flags&FFT5D_NOMEASURE))
+ {
+ fftwflags |= FFTW_MEASURE;
+ }
+ if (flags&FFT5D_REALCOMPLEX)
+ {
+ if (!(flags&FFT5D_BACKWARD)) /*input pointer is not complex*/
+ {
+ inNG *= 2;
+ }
+ else /*output pointer is not complex*/
+ {
+ if (!(flags&FFT5D_ORDER_YZ))
+ {
+ outMG *= 2;
+ }
+ else
+ {
+ outKG *= 2;
+ }
+ }
+ }
+
+ if (!(flags&FFT5D_BACKWARD))
+ {
+ dims[0].n = KG;
+ dims[1].n = MG;
+ dims[2].n = rNG;
+
+ dims[0].is = inNG*MG; /*N M K*/
+ dims[1].is = inNG;
+ dims[2].is = 1;
+ if (!(flags&FFT5D_ORDER_YZ))
+ {
+ dims[0].os = MG; /*M K N*/
+ dims[1].os = 1;
+ dims[2].os = MG*KG;
+ }
+ else
+ {
+ dims[0].os = 1; /*K N M*/
+ dims[1].os = KG*NG;
+ dims[2].os = KG;
+ }
+ }
+ else
+ {
+ if (!(flags&FFT5D_ORDER_YZ))
+ {
+ dims[0].n = NG;
+ dims[1].n = KG;
+ dims[2].n = rMG;
+
+ dims[0].is = 1;
+ dims[1].is = NG*MG;
+ dims[2].is = NG;
+
+ dims[0].os = outMG*KG;
+ dims[1].os = outMG;
+ dims[2].os = 1;
+ }
+ else
+ {
+ dims[0].n = MG;
+ dims[1].n = NG;
+ dims[2].n = rKG;
+
+ dims[0].is = NG;
+ dims[1].is = 1;
+ dims[2].is = NG*MG;
+
+ dims[0].os = outKG*NG;
+ dims[1].os = outKG;
+ dims[2].os = 1;
+ }
+ }
+#ifdef FFT5D_THREADS
+#ifdef FFT5D_FFTW_THREADS
+ FFTW(plan_with_nthreads)(nthreads);
+#endif
+#endif
+ if ((flags&FFT5D_REALCOMPLEX) && !(flags&FFT5D_BACKWARD))
+ {
+ plan->p3d = FFTW(plan_guru_dft_r2c)(/*rank*/ 3, dims,
+ /*howmany*/ 0, /*howmany_dims*/ 0,
+ (real*)lin, (FFTW(complex) *) lout,
+ /*flags*/ fftwflags);
+ }
+ else if ((flags&FFT5D_REALCOMPLEX) && (flags&FFT5D_BACKWARD))
+ {
+ plan->p3d = FFTW(plan_guru_dft_c2r)(/*rank*/ 3, dims,
+ /*howmany*/ 0, /*howmany_dims*/ 0,
+ (FFTW(complex) *) lin, (real*)lout,
+ /*flags*/ fftwflags);
+ }
+ else
+ {
+ plan->p3d = FFTW(plan_guru_dft)(/*rank*/ 3, dims,
+ /*howmany*/ 0, /*howmany_dims*/ 0,
+ (FFTW(complex) *) lin, (FFTW(complex) *) lout,
+ /*sign*/ (flags&FFT5D_BACKWARD) ? 1 : -1, /*flags*/ fftwflags);
+ }
+#ifdef FFT5D_THREADS
+#ifdef FFT5D_FFTW_THREADS
+ FFTW(plan_with_nthreads)(1);
+#endif
+#endif
+ FFTW_UNLOCK;
+ }
+ if (!plan->p3d) /* for decomposition and if 3d plan did not work */
+ {
+#endif /* GMX_FFT_FFTW3 */
+ for (s = 0; s < 3; s++)
+ {
+ if (debug)
+ {
+ fprintf(debug, "FFT5D: Plan s %d rC %d M %d pK %d C %d lsize %d\n",
+ s, rC[s], M[s], pK[s], C[s], lsize);
+ }
+ plan->p1d[s] = (gmx_fft_t*)malloc(sizeof(gmx_fft_t)*nthreads);
+
+ /* Make sure that the init routines are only called by one thread at a time and in order
+ (later is only important to not confuse valgrind)
+ */
+#pragma omp parallel for num_threads(nthreads) schedule(static) ordered
+ for (t = 0; t < nthreads; t++)
+ {
+#pragma omp ordered
+ {
+ int tsize = ((t+1)*pM[s]*pK[s]/nthreads)-(t*pM[s]*pK[s]/nthreads);
+
+ if ((flags&FFT5D_REALCOMPLEX) && ((!(flags&FFT5D_BACKWARD) && s == 0) || ((flags&FFT5D_BACKWARD) && s == 2)))
+ {
+ gmx_fft_init_many_1d_real( &plan->p1d[s][t], rC[s], tsize, (flags&FFT5D_NOMEASURE) ? GMX_FFT_FLAG_CONSERVATIVE : 0 );
+ }
+ else
+ {
+ gmx_fft_init_many_1d ( &plan->p1d[s][t], C[s], tsize, (flags&FFT5D_NOMEASURE) ? GMX_FFT_FLAG_CONSERVATIVE : 0 );
+ }
+ }
+ }
+ }
+
+#ifdef GMX_FFT_FFTW3
+}
+#endif
+ if ((flags&FFT5D_ORDER_YZ)) /*plan->cart is in the order of transposes */
+ {
+ plan->cart[0] = comm[0]; plan->cart[1] = comm[1];
+ }
+ else
+ {
+ plan->cart[1] = comm[0]; plan->cart[0] = comm[1];
+ }
+#ifdef FFT5D_MPI_TRANSPOSE
+ FFTW_LOCK;
+ for (s = 0; s < 2; s++)
+ {
+ if ((s == 0 && !(flags&FFT5D_ORDER_YZ)) || (s == 1 && (flags&FFT5D_ORDER_YZ)))
+ {
+ plan->mpip[s] = FFTW(mpi_plan_many_transpose)(nP[s], nP[s], N[s]*K[s]*pM[s]*2, 1, 1, (real*)lout2, (real*)lout3, plan->cart[s], FFTW_PATIENT);
+ }
+ else
+ {
+ plan->mpip[s] = FFTW(mpi_plan_many_transpose)(nP[s], nP[s], N[s]*pK[s]*M[s]*2, 1, 1, (real*)lout2, (real*)lout3, plan->cart[s], FFTW_PATIENT);
+ }
+ }
+ FFTW_UNLOCK;
+#endif
+
+
+ plan->lin = lin;
+ plan->lout = lout;
+ plan->lout2 = lout2;
+ plan->lout3 = lout3;
+
+ plan->NG = NG; plan->MG = MG; plan->KG = KG;
+
+ for (s = 0; s < 3; s++)
+ {
+ plan->N[s] = N[s]; plan->M[s] = M[s]; plan->K[s] = K[s]; plan->pN[s] = pN[s]; plan->pM[s] = pM[s]; plan->pK[s] = pK[s];
+ plan->oM[s] = oM[s]; plan->oK[s] = oK[s];
+ plan->C[s] = C[s]; plan->rC[s] = rC[s];
+ plan->iNin[s] = iNin[s]; plan->oNin[s] = oNin[s]; plan->iNout[s] = iNout[s]; plan->oNout[s] = oNout[s];
+ }
+ for (s = 0; s < 2; s++)
+ {
+ plan->P[s] = nP[s]; plan->coor[s] = prank[s];
+ }
+
+/* plan->fftorder=fftorder;
+ plan->direction=direction;
+ plan->realcomplex=realcomplex;
+ */
+ plan->flags = flags;
+ plan->nthreads = nthreads;
+ *rlin = lin;
+ *rlout = lout;
+ *rlout2 = lout2;
+ *rlout3 = lout3;
+ return plan;
+}
+
+
+enum order {
+ XYZ,
+ XZY,
+ YXZ,
+ YZX,
+ ZXY,
+ ZYX
+};
+
+
+
+/*here x,y,z and N,M,K is in rotated coordinate system!!
+ x (and N) is mayor (consecutive) dimension, y (M) middle and z (K) major
+ maxN,maxM,maxK is max size of local data
+ pN, pM, pK is local size specific to current processor (only different to max if not divisible)
+ NG, MG, KG is size of global data*/
+static void splitaxes(t_complex* lout, const t_complex* lin,
+ int maxN, int maxM, int maxK, int pM,
+ int P, int NG, int *N, int* oN, int starty, int startz, int endy, int endz)
+{
+ int x, y, z, i;
+ int in_i, out_i, in_z, out_z, in_y, out_y;
+ int s_y, e_y;
+
+ for (z = startz; z < endz+1; z++) /*3. z l*/
+ {
+ if (z == startz)
+ {
+ s_y = starty;
+ }
+ else
+ {
+ s_y = 0;
+ }
+ if (z == endz)
+ {
+ e_y = endy;
+ }
+ else
+ {
+ e_y = pM;
+ }
+ out_z = z*maxN*maxM;
+ in_z = z*NG*pM;
+
+ for (i = 0; i < P; i++) /*index cube along long axis*/
+ {
+ out_i = out_z + i*maxN*maxM*maxK;
+ in_i = in_z + oN[i];
+ for (y = s_y; y < e_y; y++) /*2. y k*/
+ {
+ out_y = out_i + y*maxN;
+ in_y = in_i + y*NG;
+ for (x = 0; x < N[i]; x++) /*1. x j*/
+ {
+ lout[out_y+x] = lin[in_y+x]; /*in=z*NG*pM+oN[i]+y*NG+x*/
+ /*after split important that each processor chunk i has size maxN*maxM*maxK and thus being the same size*/
+ /*before split data contiguos - thus if different processor get different amount oN is different*/
+ }
+ }
+ }
+ }
+}
+
+/*make axis contiguous again (after AllToAll) and also do local transpose*/
+/*transpose mayor and major dimension
+ variables see above
+ the major, middle, minor order is only correct for x,y,z (N,M,K) for the input
+ N,M,K local dimensions
+ KG global size*/
+static void joinAxesTrans13(t_complex* lout, const t_complex* lin,
+ int maxN, int maxM, int maxK, int pM,
+ int P, int KG, int* K, int* oK, int starty, int startx, int endy, int endx)
+{
+ int i, x, y, z;
+ int out_i, in_i, out_x, in_x, out_z, in_z;
+ int s_y, e_y;
+
+ for (x = startx; x < endx+1; x++) /*1.j*/
+ {
+ if (x == startx)
+ {
+ s_y = starty;
+ }
+ else
+ {
+ s_y = 0;
+ }
+ if (x == endx)
+ {
+ e_y = endy;
+ }
+ else
+ {
+ e_y = pM;
+ }
+
+ out_x = x*KG*pM;
+ in_x = x;
+
+ for (i = 0; i < P; i++) /*index cube along long axis*/
+ {
+ out_i = out_x + oK[i];
+ in_i = in_x + i*maxM*maxN*maxK;
+ for (z = 0; z < K[i]; z++) /*3.l*/
+ {
+ out_z = out_i + z;
+ in_z = in_i + z*maxM*maxN;
+ for (y = s_y; y < e_y; y++) /*2.k*/
+ {
+ lout[out_z+y*KG] = lin[in_z+y*maxN]; /*out=x*KG*pM+oK[i]+z+y*KG*/
+ }
+ }
+ }
+ }
+}
+
+/*make axis contiguous again (after AllToAll) and also do local transpose
+ tranpose mayor and middle dimension
+ variables see above
+ the minor, middle, major order is only correct for x,y,z (N,M,K) for the input
+ N,M,K local size
+ MG, global size*/
+static void joinAxesTrans12(t_complex* lout, const t_complex* lin, int maxN, int maxM, int maxK, int pN,
+ int P, int MG, int* M, int* oM, int startx, int startz, int endx, int endz)
+{
+ int i, z, y, x;
+ int out_i, in_i, out_z, in_z, out_x, in_x;
+ int s_x, e_x;
+
+ for (z = startz; z < endz+1; z++)
+ {
+ if (z == startz)
+ {
+ s_x = startx;
+ }
+ else
+ {
+ s_x = 0;
+ }
+ if (z == endz)
+ {
+ e_x = endx;
+ }
+ else
+ {
+ e_x = pN;
+ }
+ out_z = z*MG*pN;
+ in_z = z*maxM*maxN;
+
+ for (i = 0; i < P; i++) /*index cube along long axis*/
+ {
+ out_i = out_z + oM[i];
+ in_i = in_z + i*maxM*maxN*maxK;
+ for (x = s_x; x < e_x; x++)
+ {
+ out_x = out_i + x*MG;
+ in_x = in_i + x;
+ for (y = 0; y < M[i]; y++)
+ {
+ lout[out_x+y] = lin[in_x+y*maxN]; /*out=z*MG*pN+oM[i]+x*MG+y*/
+ }
+ }
+ }
+ }
+}
+
+
+static void rotate_offsets(int x[])
+{
+ int t = x[0];
+/* x[0]=x[2];
+ x[2]=x[1];
+ x[1]=t;*/
+ x[0] = x[1];
+ x[1] = x[2];
+ x[2] = t;
+}
+
+/*compute the offset to compare or print transposed local data in original input coordinates
+ xs matrix dimension size, xl dimension length, xc decomposition offset
+ s: step in computation = number of transposes*/
+static void compute_offsets(fft5d_plan plan, int xs[], int xl[], int xc[], int NG[], int s)
+{
+/* int direction = plan->direction;
+ int fftorder = plan->fftorder;*/
+
+ int o = 0;
+ int pos[3], i;
+ int *pM = plan->pM, *pK = plan->pK, *oM = plan->oM, *oK = plan->oK,
+ *C = plan->C, *rC = plan->rC;
+
+ NG[0] = plan->NG; NG[1] = plan->MG; NG[2] = plan->KG;
+
+ if (!(plan->flags&FFT5D_ORDER_YZ))
+ {
+ switch (s)
+ {
+ case 0: o = XYZ; break;
+ case 1: o = ZYX; break;
+ case 2: o = YZX; break;
+ default: assert(0);
+ }
+ }
+ else
+ {
+ switch (s)
+ {
+ case 0: o = XYZ; break;
+ case 1: o = YXZ; break;
+ case 2: o = ZXY; break;
+ default: assert(0);
+ }
+ }
+
+ switch (o)
+ {
+ case XYZ: pos[0] = 1; pos[1] = 2; pos[2] = 3; break;
+ case XZY: pos[0] = 1; pos[1] = 3; pos[2] = 2; break;
+ case YXZ: pos[0] = 2; pos[1] = 1; pos[2] = 3; break;
+ case YZX: pos[0] = 3; pos[1] = 1; pos[2] = 2; break;
+ case ZXY: pos[0] = 2; pos[1] = 3; pos[2] = 1; break;
+ case ZYX: pos[0] = 3; pos[1] = 2; pos[2] = 1; break;
+ }
+ /*if (debug) printf("pos: %d %d %d\n",pos[0],pos[1],pos[2]);*/
+
+ /*xs, xl give dimension size and data length in local transposed coordinate system
+ for 0(/1/2): x(/y/z) in original coordinate system*/
+ for (i = 0; i < 3; i++)
+ {
+ switch (pos[i])
+ {
+ case 1: xs[i] = 1; xc[i] = 0; xl[i] = C[s]; break;
+ case 2: xs[i] = C[s]; xc[i] = oM[s]; xl[i] = pM[s]; break;
+ case 3: xs[i] = C[s]*pM[s]; xc[i] = oK[s]; xl[i] = pK[s]; break;
+ }
+ }
+ /*input order is different for test program to match FFTW order
+ (important for complex to real)*/
+ if (plan->flags&FFT5D_BACKWARD)
+ {
+ rotate_offsets(xs);
+ rotate_offsets(xl);
+ rotate_offsets(xc);
+ rotate_offsets(NG);
+ if (plan->flags&FFT5D_ORDER_YZ)
+ {
+ rotate_offsets(xs);
+ rotate_offsets(xl);
+ rotate_offsets(xc);
+ rotate_offsets(NG);
+ }
+ }
+ if ((plan->flags&FFT5D_REALCOMPLEX) && ((!(plan->flags&FFT5D_BACKWARD) && s == 0) || ((plan->flags&FFT5D_BACKWARD) && s == 2)))
+ {
+ xl[0] = rC[s];
+ }
+}
+
+static void print_localdata(const t_complex* lin, const char* txt, int s, fft5d_plan plan)
+{
+ int x, y, z, l;
+ int *coor = plan->coor;
+ int xs[3], xl[3], xc[3], NG[3];
+ int ll = (plan->flags&FFT5D_REALCOMPLEX) ? 1 : 2;
+ compute_offsets(plan, xs, xl, xc, NG, s);
+ fprintf(debug, txt, coor[0], coor[1], s);
+ /*printf("xs: %d %d %d, xl: %d %d %d\n",xs[0],xs[1],xs[2],xl[0],xl[1],xl[2]);*/
+ for (z = 0; z < xl[2]; z++)
+ {
+ for (y = 0; y < xl[1]; y++)
+ {
+ fprintf(debug, "%d %d: ", coor[0], coor[1]);
+ for (x = 0; x < xl[0]; x++)
+ {
+ for (l = 0; l < ll; l++)
+ {
+ fprintf(debug, "%f ", ((real*)lin)[(z*xs[2]+y*xs[1])*2+(x*xs[0])*ll+l]);
+ }
+ fprintf(debug, ",");
+ }
+ fprintf(debug, "\n");
+ }
+ }
+}
+
+void fft5d_execute(fft5d_plan plan, int thread, fft5d_time times)
+{
+ t_complex *lin = plan->lin;
+ t_complex *lout = plan->lout;
+ t_complex *lout2 = plan->lout2;
+ t_complex *lout3 = plan->lout3;
+ t_complex *fftout, *joinin;
+
+ gmx_fft_t **p1d = plan->p1d;
+#ifdef FFT5D_MPI_TRANSPOSE
+ FFTW(plan) *mpip = plan->mpip;
+#endif
+#ifdef GMX_MPI
+ MPI_Comm *cart = plan->cart;
+#endif
+#ifdef NOGMX
+ double time_fft = 0, time_local = 0, time_mpi[2] = {0}, time = 0;
+#endif
+ int *N = plan->N, *M = plan->M, *K = plan->K, *pN = plan->pN, *pM = plan->pM, *pK = plan->pK,
+ *C = plan->C, *P = plan->P, **iNin = plan->iNin, **oNin = plan->oNin, **iNout = plan->iNout, **oNout = plan->oNout;
+ int s = 0, tstart, tend, bParallelDim;
+
+
+#ifdef GMX_FFT_FFTW3
+ if (plan->p3d)
+ {
+ if (thread == 0)
+ {
+#ifdef NOGMX
+ if (times != 0)
+ {
+ time = MPI_Wtime();
+ }
+#endif
+ FFTW(execute)(plan->p3d);
+#ifdef NOGMX
+ if (times != 0)
+ {
+ times->fft += MPI_Wtime()-time;
+ }
+#endif
+ }
+ return;
+ }
+#endif
+
+ s = 0;
+
+ /*lin: x,y,z*/
+ if (plan->flags&FFT5D_DEBUG && thread == 0)
+ {
+ print_localdata(lin, "%d %d: copy in lin\n", s, plan);
+ }
+
+ for (s = 0; s < 2; s++) /*loop over first two FFT steps (corner rotations)*/
+
+ {
+#ifdef GMX_MPI
+ if (GMX_PARALLEL_ENV_INITIALIZED && cart[s] != MPI_COMM_NULL && P[s] > 1)
+ {
+ bParallelDim = 1;
+ }
+ else
+#endif
+ {
+ bParallelDim = 0;
+ }
+
+ /* ---------- START FFT ------------ */
+#ifdef NOGMX
+ if (times != 0 && thread == 0)
+ {
+ time = MPI_Wtime();
+ }
+#endif
+
+ if (bParallelDim || plan->nthreads == 1)
+ {
+ fftout = lout;
+ }
+ else
+ {
+ if (s == 0)
+ {
+ fftout = lout3;
+ }
+ else
+ {
+ fftout = lout2;
+ }
+ }
+
+ tstart = (thread*pM[s]*pK[s]/plan->nthreads)*C[s];
+ if ((plan->flags&FFT5D_REALCOMPLEX) && !(plan->flags&FFT5D_BACKWARD) && s == 0)
+ {
+ gmx_fft_many_1d_real(p1d[s][thread], (plan->flags&FFT5D_BACKWARD) ? GMX_FFT_COMPLEX_TO_REAL : GMX_FFT_REAL_TO_COMPLEX, lin+tstart, fftout+tstart);
+ }
+ else
+ {
+ gmx_fft_many_1d( p1d[s][thread], (plan->flags&FFT5D_BACKWARD) ? GMX_FFT_BACKWARD : GMX_FFT_FORWARD, lin+tstart, fftout+tstart);
+
+ }
+
+#ifdef NOGMX
+ if (times != NULL && thread == 0)
+ {
+ time_fft += MPI_Wtime()-time;
+ }
+#endif
+ if (plan->flags&FFT5D_DEBUG && thread == 0)
+ {
+ print_localdata(lout, "%d %d: FFT %d\n", s, plan);
+ }
+ /* ---------- END FFT ------------ */
+
+ /* ---------- START SPLIT + TRANSPOSE------------ (if parallel in in this dimension)*/
+ if (bParallelDim)
+ {
+#ifdef NOGMX
+ if (times != NULL && thread == 0)
+ {
+ time = MPI_Wtime();
+ }
+#endif
+ /*prepare for A
+ llToAll
+ 1. (most outer) axes (x) is split into P[s] parts of size N[s]
+ for sending*/
+ if (pM[s] > 0)
+ {
+ tend = ((thread+1)*pM[s]*pK[s]/plan->nthreads);
+ tstart /= C[s];
+ splitaxes(lout2, lout, N[s], M[s], K[s], pM[s], P[s], C[s], iNout[s], oNout[s], tstart%pM[s], tstart/pM[s], tend%pM[s], tend/pM[s]);
+ }
+#pragma omp barrier /*barrier required before AllToAll (all input has to be their) - before timing to make timing more acurate*/
+#ifdef NOGMX
+ if (times != NULL && thread == 0)
+ {
+ time_local += MPI_Wtime()-time;
+ }
+#endif
+
+ /* ---------- END SPLIT , START TRANSPOSE------------ */
+
+ if (thread == 0)
+ {
+#ifdef NOGMX
+ if (times != 0)
+ {
+ time = MPI_Wtime();
+ }
+#else
+ wallcycle_start(times, ewcPME_FFTCOMM);
+#endif
+#ifdef FFT5D_MPI_TRANSPOSE
+ FFTW(execute)(mpip[s]);
+#else
+#ifdef GMX_MPI
+ if ((s == 0 && !(plan->flags&FFT5D_ORDER_YZ)) || (s == 1 && (plan->flags&FFT5D_ORDER_YZ)))
+ {
- MPI_Alltoall(lout2, N[s]*M[s]*pK[s]*sizeof(t_complex)/sizeof(real), GMX_MPI_REAL, lout3, N[s]*M[s]*pK[s]*sizeof(t_complex)/sizeof(real), GMX_MPI_REAL, cart[s]);
++ MPI_Alltoall((real *)lout2, N[s]*pM[s]*K[s]*sizeof(t_complex)/sizeof(real), GMX_MPI_REAL, (real *)lout3, N[s]*pM[s]*K[s]*sizeof(t_complex)/sizeof(real), GMX_MPI_REAL, cart[s]);
+ }
+ else
+ {
++ MPI_Alltoall((real *)lout2, N[s]*M[s]*pK[s]*sizeof(t_complex)/sizeof(real), GMX_MPI_REAL, (real *)lout3, N[s]*M[s]*pK[s]*sizeof(t_complex)/sizeof(real), GMX_MPI_REAL, cart[s]);
+ }
+#else
+ gmx_incons("fft5d MPI call without MPI configuration");
+#endif /*GMX_MPI*/
+#endif /*FFT5D_MPI_TRANSPOSE*/
+#ifdef NOGMX
+ if (times != 0)
+ {
+ time_mpi[s] = MPI_Wtime()-time;
+ }
+#else
+ wallcycle_stop(times, ewcPME_FFTCOMM);
+#endif
+ } /*master*/
+ } /* bPrallelDim */
+#pragma omp barrier /*both needed for parallel and non-parallel dimension (either have to wait on data from AlltoAll or from last FFT*/
+
+ /* ---------- END SPLIT + TRANSPOSE------------ */
+
+ /* ---------- START JOIN ------------ */
+#ifdef NOGMX
+ if (times != NULL && thread == 0)
+ {
+ time = MPI_Wtime();
+ }
+#endif
+
+ if (bParallelDim)
+ {
+ joinin = lout3;
+ }
+ else
+ {
+ joinin = fftout;
+ }
+ /*bring back in matrix form
+ thus make new 1. axes contiguos
+ also local transpose 1 and 2/3
+ runs on thread used for following FFT (thus needing a barrier before but not afterwards)
+ */
+ if ((s == 0 && !(plan->flags&FFT5D_ORDER_YZ)) || (s == 1 && (plan->flags&FFT5D_ORDER_YZ)))
+ {
+ if (pM[s] > 0)
+ {
+ tstart = ( thread *pM[s]*pN[s]/plan->nthreads);
+ tend = ((thread+1)*pM[s]*pN[s]/plan->nthreads);
+ joinAxesTrans13(lin, joinin, N[s], pM[s], K[s], pM[s], P[s], C[s+1], iNin[s+1], oNin[s+1], tstart%pM[s], tstart/pM[s], tend%pM[s], tend/pM[s]);
+ }
+ }
+ else
+ {
+ if (pN[s] > 0)
+ {
+ tstart = ( thread *pK[s]*pN[s]/plan->nthreads);
+ tend = ((thread+1)*pK[s]*pN[s]/plan->nthreads);
+ joinAxesTrans12(lin, joinin, N[s], M[s], pK[s], pN[s], P[s], C[s+1], iNin[s+1], oNin[s+1], tstart%pN[s], tstart/pN[s], tend%pN[s], tend/pN[s]);
+ }
+ }
+
+#ifdef NOGMX
+ if (times != NULL && thread == 0)
+ {
+ time_local += MPI_Wtime()-time;
+ }
+#endif
+ if (plan->flags&FFT5D_DEBUG && thread == 0)
+ {
+ print_localdata(lin, "%d %d: tranposed %d\n", s+1, plan);
+ }
+ /* ---------- END JOIN ------------ */
+
+ /*if (debug) print_localdata(lin, "%d %d: transposed x-z\n", N1, M0, K, ZYX, coor);*/
+ } /* for(s=0;s<2;s++) */
+#ifdef NOGMX
+ if (times != NULL && thread == 0)
+ {
+ time = MPI_Wtime();
+ }
+#endif
+
+ if (plan->flags&FFT5D_INPLACE)
+ {
+ lout = lin; /*in place currently not supported*/
+
+ }
+ /* ----------- FFT ----------- */
+ tstart = (thread*pM[s]*pK[s]/plan->nthreads)*C[s];
+ if ((plan->flags&FFT5D_REALCOMPLEX) && (plan->flags&FFT5D_BACKWARD))
+ {
+ gmx_fft_many_1d_real(p1d[s][thread], (plan->flags&FFT5D_BACKWARD) ? GMX_FFT_COMPLEX_TO_REAL : GMX_FFT_REAL_TO_COMPLEX, lin+tstart, lout+tstart);
+ }
+ else
+ {
+ gmx_fft_many_1d( p1d[s][thread], (plan->flags&FFT5D_BACKWARD) ? GMX_FFT_BACKWARD : GMX_FFT_FORWARD, lin+tstart, lout+tstart);
+ }
+ /* ------------ END FFT ---------*/
+
+#ifdef NOGMX
+ if (times != NULL && thread == 0)
+ {
+ time_fft += MPI_Wtime()-time;
+
+ times->fft += time_fft;
+ times->local += time_local;
+ times->mpi2 += time_mpi[1];
+ times->mpi1 += time_mpi[0];
+ }
+#endif
+
+ if (plan->flags&FFT5D_DEBUG && thread == 0)
+ {
+ print_localdata(lout, "%d %d: FFT %d\n", s, plan);
+ }
+}
+
+void fft5d_destroy(fft5d_plan plan)
+{
+ int s, t;
+
+ for (s = 0; s < 3; s++)
+ {
+ if (plan->p1d[s])
+ {
+ for (t = 0; t < plan->nthreads; t++)
+ {
+ gmx_many_fft_destroy(plan->p1d[s][t]);
+ }
+ free(plan->p1d[s]);
+ }
+ if (plan->iNin[s])
+ {
+ free(plan->iNin[s]);
+ plan->iNin[s] = 0;
+ }
+ if (plan->oNin[s])
+ {
+ free(plan->oNin[s]);
+ plan->oNin[s] = 0;
+ }
+ if (plan->iNout[s])
+ {
+ free(plan->iNout[s]);
+ plan->iNout[s] = 0;
+ }
+ if (plan->oNout[s])
+ {
+ free(plan->oNout[s]);
+ plan->oNout[s] = 0;
+ }
+ }
+#ifdef GMX_FFT_FFTW3
+ FFTW_LOCK;
+#ifdef FFT5D_MPI_TRANSPOS
+ for (s = 0; s < 2; s++)
+ {
+ FFTW(destroy_plan)(plan->mpip[s]);
+ }
+#endif /* FFT5D_MPI_TRANSPOS */
+ if (plan->p3d)
+ {
+ FFTW(destroy_plan)(plan->p3d);
+ }
+ FFTW_UNLOCK;
+#endif /* GMX_FFT_FFTW3 */
+
+ if (!(plan->flags&FFT5D_NOMALLOC))
+ {
+ sfree_aligned(plan->lin);
+ sfree_aligned(plan->lout);
+ if (plan->nthreads > 1)
+ {
+ sfree_aligned(plan->lout2);
+ sfree_aligned(plan->lout3);
+ }
+ }
+
+#ifdef FFT5D_THREADS
+#ifdef FFT5D_FFTW_THREADS
+ /*FFTW(cleanup_threads)();*/
+#endif
+#endif
+
+ free(plan);
+}
+
+/*Is this better than direct access of plan? enough data?
+ here 0,1 reference divided by which processor grid dimension (not FFT step!)*/
+void fft5d_local_size(fft5d_plan plan, int* N1, int* M0, int* K0, int* K1, int** coor)
+{
+ *N1 = plan->N[0];
+ *M0 = plan->M[0];
+ *K1 = plan->K[0];
+ *K0 = plan->N[1];
+
+ *coor = plan->coor;
+}
+
+
+/*same as fft5d_plan_3d but with cartesian coordinator and automatic splitting
+ of processor dimensions*/
+fft5d_plan fft5d_plan_3d_cart(int NG, int MG, int KG, MPI_Comm comm, int P0, int flags, t_complex** rlin, t_complex** rlout, t_complex** rlout2, t_complex** rlout3, int nthreads)
+{
+ MPI_Comm cart[2] = {0};
+#ifdef GMX_MPI
+ int size = 1, prank = 0;
+ int P[2];
+ int coor[2];
+ int wrap[] = {0, 0};
+ MPI_Comm gcart;
+ int rdim1[] = {0, 1}, rdim2[] = {1, 0};
+
+ MPI_Comm_size(comm, &size);
+ MPI_Comm_rank(comm, &prank);
+
+ if (P0 == 0)
+ {
+ P0 = lfactor(size);
+ }
+ if (size%P0 != 0)
+ {
+ if (prank == 0)
+ {
+ printf("FFT5D: WARNING: Number of processors %d not evenly dividable by %d\n", size, P0);
+ }
+ P0 = lfactor(size);
+ }
+
+ P[0] = P0; P[1] = size/P0; /*number of processors in the two dimensions*/
+
+ /*Difference between x-y-z regarding 2d decomposition is whether they are
+ distributed along axis 1, 2 or both*/
+
+ MPI_Cart_create(comm, 2, P, wrap, 1, &gcart); /*parameter 4: value 1: reorder*/
+ MPI_Cart_get(gcart, 2, P, wrap, coor);
+ MPI_Cart_sub(gcart, rdim1, &cart[0]);
+ MPI_Cart_sub(gcart, rdim2, &cart[1]);
+#endif
+ return fft5d_plan_3d(NG, MG, KG, cart, flags, rlin, rlout, rlout2, rlout3, nthreads);
+}
+
+
+
+/*prints in original coordinate system of data (as the input to FFT)*/
+void fft5d_compare_data(const t_complex* lin, const t_complex* in, fft5d_plan plan, int bothLocal, int normalize)
+{
+ int xs[3], xl[3], xc[3], NG[3];
+ int x, y, z, l;
+ int *coor = plan->coor;
+ int ll = 2; /*compare ll values per element (has to be 2 for complex)*/
+ if ((plan->flags&FFT5D_REALCOMPLEX) && (plan->flags&FFT5D_BACKWARD))
+ {
+ ll = 1;
+ }
+
+ compute_offsets(plan, xs, xl, xc, NG, 2);
+ if (plan->flags&FFT5D_DEBUG)
+ {
+ printf("Compare2\n");
+ }
+ for (z = 0; z < xl[2]; z++)
+ {
+ for (y = 0; y < xl[1]; y++)
+ {
+ if (plan->flags&FFT5D_DEBUG)
+ {
+ printf("%d %d: ", coor[0], coor[1]);
+ }
+ for (x = 0; x < xl[0]; x++)
+ {
+ for (l = 0; l < ll; l++) /*loop over real/complex parts*/
+ {
+ real a, b;
+ a = ((real*)lin)[(z*xs[2]+y*xs[1])*2+x*xs[0]*ll+l];
+ if (normalize)
+ {
+ a /= plan->rC[0]*plan->rC[1]*plan->rC[2];
+ }
+ if (!bothLocal)
+ {
+ b = ((real*)in)[((z+xc[2])*NG[0]*NG[1]+(y+xc[1])*NG[0])*2+(x+xc[0])*ll+l];
+ }
+ else
+ {
+ b = ((real*)in)[(z*xs[2]+y*xs[1])*2+x*xs[0]*ll+l];
+ }
+ if (plan->flags&FFT5D_DEBUG)
+ {
+ printf("%f %f, ", a, b);
+ }
+ else
+ {
+ if (fabs(a-b) > 2*NG[0]*NG[1]*NG[2]*GMX_REAL_EPS)
+ {
+ printf("result incorrect on %d,%d at %d,%d,%d: FFT5D:%f reference:%f\n", coor[0], coor[1], x, y, z, a, b);
+ }
+/* assert(fabs(a-b)<2*NG[0]*NG[1]*NG[2]*GMX_REAL_EPS);*/
+ }
+ }
+ if (plan->flags&FFT5D_DEBUG)
+ {
+ printf(",");
+ }
+ }
+ if (plan->flags&FFT5D_DEBUG)
+ {
+ printf("\n");
+ }
+ }
+ }
+
+}
--- /dev/null
- int i, i1, i2, j, nf, nrms;
+/*
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * VERSION 3.2.0
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2004, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * Green Red Orange Magenta Azure Cyan Skyblue
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+#include <math.h>
+#include <string.h>
+#include <ctype.h>
+#include "macros.h"
+#include "smalloc.h"
+#include "typedefs.h"
+#include "statutil.h"
+#include "tpxio.h"
+#include "string2.h"
+#include "vec.h"
+#include "macros.h"
+#include "index.h"
+#include "gmx_random.h"
+#include "pbc.h"
+#include "rmpbc.h"
+#include "xvgr.h"
+#include "futil.h"
+#include "matio.h"
+#include "cmat.h"
+#include "do_fit.h"
+#include "trnio.h"
+#include "viewit.h"
+#include "gmx_ana.h"
+
+#include "gromacs/linearalgebra/eigensolver.h"
+
+/* print to two file pointers at once (i.e. stderr and log) */
+static inline
+void lo_ffprintf(FILE *fp1, FILE *fp2, const char *buf)
+{
+ fprintf(fp1, "%s", buf);
+ fprintf(fp2, "%s", buf);
+}
+
+/* just print a prepared buffer to fp1 and fp2 */
+static inline
+void ffprintf(FILE *fp1, FILE *fp2, const char *buf)
+{
+ lo_ffprintf(fp1, fp2, buf);
+}
+
+/* prepare buffer with one argument, then print to fp1 and fp2 */
+static inline
+void ffprintf_d(FILE *fp1, FILE *fp2, char *buf, const char *fmt, int arg)
+{
+ sprintf(buf, fmt, arg);
+ lo_ffprintf(fp1, fp2, buf);
+}
+
+/* prepare buffer with one argument, then print to fp1 and fp2 */
+static inline
+void ffprintf_g(FILE *fp1, FILE *fp2, char *buf, const char *fmt, real arg)
+{
+ sprintf(buf, fmt, arg);
+ lo_ffprintf(fp1, fp2, buf);
+}
+
+/* prepare buffer with one argument, then print to fp1 and fp2 */
+static inline
+void ffprintf_s(FILE *fp1, FILE *fp2, char *buf, const char *fmt, const char *arg)
+{
+ sprintf(buf, fmt, arg);
+ lo_ffprintf(fp1, fp2, buf);
+}
+
+/* prepare buffer with two arguments, then print to fp1 and fp2 */
+static inline
+void ffprintf_dd(FILE *fp1, FILE *fp2, char *buf, const char *fmt, int arg1, int arg2)
+{
+ sprintf(buf, fmt, arg1, arg2);
+ lo_ffprintf(fp1, fp2, buf);
+}
+
+/* prepare buffer with two arguments, then print to fp1 and fp2 */
+static inline
+void ffprintf_gg(FILE *fp1, FILE *fp2, char *buf, const char *fmt, real arg1, real arg2)
+{
+ sprintf(buf, fmt, arg1, arg2);
+ lo_ffprintf(fp1, fp2, buf);
+}
+
+/* prepare buffer with two arguments, then print to fp1 and fp2 */
+static inline
+void ffprintf_ss(FILE *fp1, FILE *fp2, char *buf, const char *fmt, const char *arg1, const char *arg2)
+{
+ sprintf(buf, fmt, arg1, arg2);
+ lo_ffprintf(fp1, fp2, buf);
+}
+
+typedef struct {
+ int ncl;
+ int *cl;
+} t_clusters;
+
+typedef struct {
+ int nr;
+ int *nb;
+} t_nnb;
+
+void pr_energy(FILE *fp, real e)
+{
+ fprintf(fp, "Energy: %8.4f\n", e);
+}
+
+void cp_index(int nn, int from[], int to[])
+{
+ int i;
+
+ for (i = 0; (i < nn); i++)
+ {
+ to[i] = from[i];
+ }
+}
+
+void mc_optimize(FILE *log, t_mat *m, real *time,
+ int maxiter, int nrandom,
+ int seed, real kT,
+ const char *conv, output_env_t oenv)
+{
+ FILE *fp = NULL;
+ real ecur, enext, emin, prob, enorm;
+ int i, j, iswap, jswap, nn, nuphill = 0;
+ gmx_rng_t rng;
+ t_mat *minimum;
+
+ if (m->n1 != m->nn)
+ {
+ fprintf(stderr, "Can not do Monte Carlo optimization with a non-square matrix.\n");
+ return;
+ }
+ printf("\nDoing Monte Carlo optimization to find the smoothest trajectory\n");
+ printf("by reordering the frames to minimize the path between the two structures\n");
+ printf("that have the largest pairwise RMSD.\n");
+
+ iswap = jswap = -1;
+ enorm = m->mat[0][0];
+ for (i = 0; (i < m->n1); i++)
+ {
+ for (j = 0; (j < m->nn); j++)
+ {
+ if (m->mat[i][j] > enorm)
+ {
+ enorm = m->mat[i][j];
+ iswap = i;
+ jswap = j;
+ }
+ }
+ }
+ if ((iswap == -1) || (jswap == -1))
+ {
+ fprintf(stderr, "Matrix contains identical values in all fields\n");
+ return;
+ }
+ swap_rows(m, 0, iswap);
+ swap_rows(m, m->n1-1, jswap);
+ emin = ecur = mat_energy(m);
+ printf("Largest distance %g between %d and %d. Energy: %g.\n",
+ enorm, iswap, jswap, emin);
+
+ rng = gmx_rng_init(seed);
+ nn = m->nn;
+
+ /* Initiate and store global minimum */
+ minimum = init_mat(nn, m->b1D);
+ minimum->nn = nn;
+ copy_t_mat(minimum, m);
+
+ if (NULL != conv)
+ {
+ fp = xvgropen(conv, "Convergence of the MC optimization",
+ "Energy", "Step", oenv);
+ }
+ for (i = 0; (i < maxiter); i++)
+ {
+ /* Generate new swapping candidates */
+ do
+ {
+ iswap = 1+(nn-2)*gmx_rng_uniform_real(rng);
+ jswap = 1+(nn-2)*gmx_rng_uniform_real(rng);
+ }
+ while ((iswap == jswap) || (iswap >= nn-1) || (jswap >= nn-1));
+
+ /* Apply swap and compute energy */
+ swap_rows(m, iswap, jswap);
+ enext = mat_energy(m);
+
+ /* Compute probability */
+ prob = 0;
+ if ((enext < ecur) || (i < nrandom))
+ {
+ prob = 1;
+ if (enext < emin)
+ {
+ /* Store global minimum */
+ copy_t_mat(minimum, m);
+ emin = enext;
+ }
+ }
+ else if (kT > 0)
+ {
+ /* Try Monte Carlo step */
+ prob = exp(-(enext-ecur)/(enorm*kT));
+ }
+
+ if ((prob == 1) || (gmx_rng_uniform_real(rng) < prob))
+ {
+ if (enext > ecur)
+ {
+ nuphill++;
+ }
+
+ fprintf(log, "Iter: %d Swapped %4d and %4d (energy: %g prob: %g)\n",
+ i, iswap, jswap, enext, prob);
+ if (NULL != fp)
+ {
+ fprintf(fp, "%6d %10g\n", i, enext);
+ }
+ ecur = enext;
+ }
+ else
+ {
+ swap_rows(m, jswap, iswap);
+ }
+ }
+ fprintf(log, "%d uphill steps were taken during optimization\n",
+ nuphill);
+
+ /* Now swap the matrix to get it into global minimum mode */
+ copy_t_mat(m, minimum);
+
+ fprintf(log, "Global minimum energy %g\n", mat_energy(minimum));
+ fprintf(log, "Global minimum energy %g\n", mat_energy(m));
+ fprintf(log, "Swapped time and frame indices and RMSD to next neighbor:\n");
+ for (i = 0; (i < m->nn); i++)
+ {
+ fprintf(log, "%10g %5d %10g\n",
+ time[m->m_ind[i]],
+ m->m_ind[i],
+ (i < m->nn-1) ? m->mat[m->m_ind[i]][m->m_ind[i+1]] : 0);
+ }
+
+ if (NULL != fp)
+ {
+ fclose(fp);
+ }
+}
+
+static void calc_dist(int nind, rvec x[], real **d)
+{
+ int i, j;
+ real *xi;
+ rvec dx;
+
+ for (i = 0; (i < nind-1); i++)
+ {
+ xi = x[i];
+ for (j = i+1; (j < nind); j++)
+ {
+ /* Should use pbc_dx when analysing multiple molecueles,
+ * but the box is not stored for every frame.
+ */
+ rvec_sub(xi, x[j], dx);
+ d[i][j] = norm(dx);
+ }
+ }
+}
+
+static real rms_dist(int isize, real **d, real **d_r)
+{
+ int i, j;
+ real r, r2;
+
+ r2 = 0.0;
+ for (i = 0; (i < isize-1); i++)
+ {
+ for (j = i+1; (j < isize); j++)
+ {
+ r = d[i][j]-d_r[i][j];
+ r2 += r*r;
+ }
+ }
+ r2 /= (isize*(isize-1))/2;
+
+ return sqrt(r2);
+}
+
+static int rms_dist_comp(const void *a, const void *b)
+{
+ t_dist *da, *db;
+
+ da = (t_dist *)a;
+ db = (t_dist *)b;
+
+ if (da->dist - db->dist < 0)
+ {
+ return -1;
+ }
+ else if (da->dist - db->dist > 0)
+ {
+ return 1;
+ }
+ return 0;
+}
+
+static int clust_id_comp(const void *a, const void *b)
+{
+ t_clustid *da, *db;
+
+ da = (t_clustid *)a;
+ db = (t_clustid *)b;
+
+ return da->clust - db->clust;
+}
+
+static int nrnb_comp(const void *a, const void *b)
+{
+ t_nnb *da, *db;
+
+ da = (t_nnb *)a;
+ db = (t_nnb *)b;
+
+ /* return the b-a, we want highest first */
+ return db->nr - da->nr;
+}
+
+void gather(t_mat *m, real cutoff, t_clusters *clust)
+{
+ t_clustid *c;
+ t_dist *d;
+ int i, j, k, nn, cid, n1, diff;
+ gmx_bool bChange;
+
+ /* First we sort the entries in the RMSD matrix */
+ n1 = m->nn;
+ nn = ((n1-1)*n1)/2;
+ snew(d, nn);
+ for (i = k = 0; (i < n1); i++)
+ {
+ for (j = i+1; (j < n1); j++, k++)
+ {
+ d[k].i = i;
+ d[k].j = j;
+ d[k].dist = m->mat[i][j];
+ }
+ }
+ if (k != nn)
+ {
+ gmx_incons("gather algortihm");
+ }
+ qsort(d, nn, sizeof(d[0]), rms_dist_comp);
+
+ /* Now we make a cluster index for all of the conformations */
+ c = new_clustid(n1);
+
+ /* Now we check the closest structures, and equalize their cluster numbers */
+ fprintf(stderr, "Linking structures ");
+ do
+ {
+ fprintf(stderr, "*");
+ bChange = FALSE;
+ for (k = 0; (k < nn) && (d[k].dist < cutoff); k++)
+ {
+ diff = c[d[k].j].clust - c[d[k].i].clust;
+ if (diff)
+ {
+ bChange = TRUE;
+ if (diff > 0)
+ {
+ c[d[k].j].clust = c[d[k].i].clust;
+ }
+ else
+ {
+ c[d[k].i].clust = c[d[k].j].clust;
+ }
+ }
+ }
+ }
+ while (bChange);
+ fprintf(stderr, "\nSorting and renumbering clusters\n");
+ /* Sort on cluster number */
+ qsort(c, n1, sizeof(c[0]), clust_id_comp);
+
+ /* Renumber clusters */
+ cid = 1;
+ for (k = 1; k < n1; k++)
+ {
+ if (c[k].clust != c[k-1].clust)
+ {
+ c[k-1].clust = cid;
+ cid++;
+ }
+ else
+ {
+ c[k-1].clust = cid;
+ }
+ }
+ c[k-1].clust = cid;
+ if (debug)
+ {
+ for (k = 0; (k < n1); k++)
+ {
+ fprintf(debug, "Cluster index for conformation %d: %d\n",
+ c[k].conf, c[k].clust);
+ }
+ }
+ clust->ncl = cid;
+ for (k = 0; k < n1; k++)
+ {
+ clust->cl[c[k].conf] = c[k].clust;
+ }
+
+ sfree(c);
+ sfree(d);
+}
+
+gmx_bool jp_same(int **nnb, int i, int j, int P)
+{
+ gmx_bool bIn;
+ int k, ii, jj, pp;
+
+ bIn = FALSE;
+ for (k = 0; nnb[i][k] >= 0; k++)
+ {
+ bIn = bIn || (nnb[i][k] == j);
+ }
+ if (!bIn)
+ {
+ return FALSE;
+ }
+
+ bIn = FALSE;
+ for (k = 0; nnb[j][k] >= 0; k++)
+ {
+ bIn = bIn || (nnb[j][k] == i);
+ }
+ if (!bIn)
+ {
+ return FALSE;
+ }
+
+ pp = 0;
+ for (ii = 0; nnb[i][ii] >= 0; ii++)
+ {
+ for (jj = 0; nnb[j][jj] >= 0; jj++)
+ {
+ if ((nnb[i][ii] == nnb[j][jj]) && (nnb[i][ii] != -1))
+ {
+ pp++;
+ }
+ }
+ }
+
+ return (pp >= P);
+}
+
+static void jarvis_patrick(int n1, real **mat, int M, int P,
+ real rmsdcut, t_clusters *clust)
+{
+ t_dist *row;
+ t_clustid *c;
+ int **nnb;
+ int i, j, k, cid, diff, max;
+ gmx_bool bChange;
+ real **mcpy = NULL;
+
+ if (rmsdcut < 0)
+ {
+ rmsdcut = 10000;
+ }
+
+ /* First we sort the entries in the RMSD matrix row by row.
+ * This gives us the nearest neighbor list.
+ */
+ snew(nnb, n1);
+ snew(row, n1);
+ for (i = 0; (i < n1); i++)
+ {
+ for (j = 0; (j < n1); j++)
+ {
+ row[j].j = j;
+ row[j].dist = mat[i][j];
+ }
+ qsort(row, n1, sizeof(row[0]), rms_dist_comp);
+ if (M > 0)
+ {
+ /* Put the M nearest neighbors in the list */
+ snew(nnb[i], M+1);
+ for (j = k = 0; (k < M) && (j < n1) && (mat[i][row[j].j] < rmsdcut); j++)
+ {
+ if (row[j].j != i)
+ {
+ nnb[i][k] = row[j].j;
+ k++;
+ }
+ }
+ nnb[i][k] = -1;
+ }
+ else
+ {
+ /* Put all neighbors nearer than rmsdcut in the list */
+ max = 0;
+ k = 0;
+ for (j = 0; (j < n1) && (mat[i][row[j].j] < rmsdcut); j++)
+ {
+ if (row[j].j != i)
+ {
+ if (k >= max)
+ {
+ max += 10;
+ srenew(nnb[i], max);
+ }
+ nnb[i][k] = row[j].j;
+ k++;
+ }
+ }
+ if (k == max)
+ {
+ srenew(nnb[i], max+1);
+ }
+ nnb[i][k] = -1;
+ }
+ }
+ sfree(row);
+ if (debug)
+ {
+ fprintf(debug, "Nearest neighborlist. M = %d, P = %d\n", M, P);
+ for (i = 0; (i < n1); i++)
+ {
+ fprintf(debug, "i:%5d nbs:", i);
+ for (j = 0; nnb[i][j] >= 0; j++)
+ {
+ fprintf(debug, "%5d[%5.3f]", nnb[i][j], mat[i][nnb[i][j]]);
+ }
+ fprintf(debug, "\n");
+ }
+ }
+
+ c = new_clustid(n1);
+ fprintf(stderr, "Linking structures ");
+ /* Use mcpy for temporary storage of booleans */
+ mcpy = mk_matrix(n1, n1, FALSE);
+ for (i = 0; i < n1; i++)
+ {
+ for (j = i+1; j < n1; j++)
+ {
+ mcpy[i][j] = jp_same(nnb, i, j, P);
+ }
+ }
+ do
+ {
+ fprintf(stderr, "*");
+ bChange = FALSE;
+ for (i = 0; i < n1; i++)
+ {
+ for (j = i+1; j < n1; j++)
+ {
+ if (mcpy[i][j])
+ {
+ diff = c[j].clust - c[i].clust;
+ if (diff)
+ {
+ bChange = TRUE;
+ if (diff > 0)
+ {
+ c[j].clust = c[i].clust;
+ }
+ else
+ {
+ c[i].clust = c[j].clust;
+ }
+ }
+ }
+ }
+ }
+ }
+ while (bChange);
+
+ fprintf(stderr, "\nSorting and renumbering clusters\n");
+ /* Sort on cluster number */
+ qsort(c, n1, sizeof(c[0]), clust_id_comp);
+
+ /* Renumber clusters */
+ cid = 1;
+ for (k = 1; k < n1; k++)
+ {
+ if (c[k].clust != c[k-1].clust)
+ {
+ c[k-1].clust = cid;
+ cid++;
+ }
+ else
+ {
+ c[k-1].clust = cid;
+ }
+ }
+ c[k-1].clust = cid;
+ clust->ncl = cid;
+ for (k = 0; k < n1; k++)
+ {
+ clust->cl[c[k].conf] = c[k].clust;
+ }
+ if (debug)
+ {
+ for (k = 0; (k < n1); k++)
+ {
+ fprintf(debug, "Cluster index for conformation %d: %d\n",
+ c[k].conf, c[k].clust);
+ }
+ }
+
+/* Again, I don't see the point in this... (AF) */
+/* for(i=0; (i<n1); i++) { */
+/* for(j=0; (j<n1); j++) */
+/* mcpy[c[i].conf][c[j].conf] = mat[i][j]; */
+/* } */
+/* for(i=0; (i<n1); i++) { */
+/* for(j=0; (j<n1); j++) */
+/* mat[i][j] = mcpy[i][j]; */
+/* } */
+ done_matrix(n1, &mcpy);
+
+ sfree(c);
+ for (i = 0; (i < n1); i++)
+ {
+ sfree(nnb[i]);
+ }
+ sfree(nnb);
+}
+
+static void dump_nnb (FILE *fp, const char *title, int n1, t_nnb *nnb)
+{
+ int i, j;
+
+ /* dump neighbor list */
+ fprintf(fp, "%s", title);
+ for (i = 0; (i < n1); i++)
+ {
+ fprintf(fp, "i:%5d #:%5d nbs:", i, nnb[i].nr);
+ for (j = 0; j < nnb[i].nr; j++)
+ {
+ fprintf(fp, "%5d", nnb[i].nb[j]);
+ }
+ fprintf(fp, "\n");
+ }
+}
+
+static void gromos(int n1, real **mat, real rmsdcut, t_clusters *clust)
+{
+ t_dist *row;
+ t_nnb *nnb;
+ int i, j, k, j1, max;
+
+ /* Put all neighbors nearer than rmsdcut in the list */
+ fprintf(stderr, "Making list of neighbors within cutoff ");
+ snew(nnb, n1);
+ snew(row, n1);
+ for (i = 0; (i < n1); i++)
+ {
+ max = 0;
+ k = 0;
+ /* put all neighbors within cut-off in list */
+ for (j = 0; j < n1; j++)
+ {
+ if (mat[i][j] < rmsdcut)
+ {
+ if (k >= max)
+ {
+ max += 10;
+ srenew(nnb[i].nb, max);
+ }
+ nnb[i].nb[k] = j;
+ k++;
+ }
+ }
+ /* store nr of neighbors, we'll need that */
+ nnb[i].nr = k;
+ if (i%(1+n1/100) == 0)
+ {
+ fprintf(stderr, "%3d%%\b\b\b\b", (i*100+1)/n1);
+ }
+ }
+ fprintf(stderr, "%3d%%\n", 100);
+ sfree(row);
+
+ /* sort neighbor list on number of neighbors, largest first */
+ qsort(nnb, n1, sizeof(nnb[0]), nrnb_comp);
+
+ if (debug)
+ {
+ dump_nnb(debug, "Nearest neighborlist after sort.\n", n1, nnb);
+ }
+
+ /* turn first structure with all its neighbors (largest) into cluster
+ remove them from pool of structures and repeat for all remaining */
+ fprintf(stderr, "Finding clusters %4d", 0);
+ /* cluster id's start at 1: */
+ k = 1;
+ while (nnb[0].nr)
+ {
+ /* set cluster id (k) for first item in neighborlist */
+ for (j = 0; j < nnb[0].nr; j++)
+ {
+ clust->cl[nnb[0].nb[j]] = k;
+ }
+ /* mark as done */
+ nnb[0].nr = 0;
+ sfree(nnb[0].nb);
+
+ /* adjust number of neighbors for others, taking removals into account: */
+ for (i = 1; i < n1 && nnb[i].nr; i++)
+ {
+ j1 = 0;
+ for (j = 0; j < nnb[i].nr; j++)
+ {
+ /* if this neighbor wasn't removed */
+ if (clust->cl[nnb[i].nb[j]] == 0)
+ {
+ /* shift the rest (j1<=j) */
+ nnb[i].nb[j1] = nnb[i].nb[j];
+ /* next */
+ j1++;
+ }
+ }
+ /* now j1 is the new number of neighbors */
+ nnb[i].nr = j1;
+ }
+ /* sort again on nnb[].nr, because we have new # neighbors: */
+ /* but we only need to sort upto i, i.e. when nnb[].nr>0 */
+ qsort(nnb, i, sizeof(nnb[0]), nrnb_comp);
+
+ fprintf(stderr, "\b\b\b\b%4d", k);
+ /* new cluster id */
+ k++;
+ }
+ fprintf(stderr, "\n");
+ sfree(nnb);
+ if (debug)
+ {
+ fprintf(debug, "Clusters (%d):\n", k);
+ for (i = 0; i < n1; i++)
+ {
+ fprintf(debug, " %3d", clust->cl[i]);
+ }
+ fprintf(debug, "\n");
+ }
+
+ clust->ncl = k-1;
+}
+
+rvec **read_whole_trj(const char *fn, int isize, atom_id index[], int skip,
+ int *nframe, real **time, const output_env_t oenv, gmx_bool bPBC, gmx_rmpbc_t gpbc)
+{
+ rvec **xx, *x;
+ matrix box;
+ real t;
+ int i, i0, j, max_nf;
+ int natom;
+ t_trxstatus *status;
+
+
+ max_nf = 0;
+ xx = NULL;
+ *time = NULL;
+ natom = read_first_x(oenv, &status, fn, &t, &x, box);
+ i = 0;
+ i0 = 0;
+ do
+ {
+ if (bPBC)
+ {
+ gmx_rmpbc(gpbc, natom, box, x);
+ }
+ if (i0 >= max_nf)
+ {
+ max_nf += 10;
+ srenew(xx, max_nf);
+ srenew(*time, max_nf);
+ }
+ if ((i % skip) == 0)
+ {
+ snew(xx[i0], isize);
+ /* Store only the interesting atoms */
+ for (j = 0; (j < isize); j++)
+ {
+ copy_rvec(x[index[j]], xx[i0][j]);
+ }
+ (*time)[i0] = t;
+ i0++;
+ }
+ i++;
+ }
+ while (read_next_x(oenv, status, &t, x, box));
+ fprintf(stderr, "Allocated %lu bytes for frames\n",
+ (unsigned long) (max_nf*isize*sizeof(**xx)));
+ fprintf(stderr, "Read %d frames from trajectory %s\n", i0, fn);
+ *nframe = i0;
+ sfree(x);
+
+ return xx;
+}
+
+static int plot_clusters(int nf, real **mat, t_clusters *clust,
+ int minstruct)
+{
+ int i, j, ncluster, ci;
+ int *cl_id, *nstruct, *strind;
+
+ snew(cl_id, nf);
+ snew(nstruct, nf);
+ snew(strind, nf);
+ for (i = 0; i < nf; i++)
+ {
+ strind[i] = 0;
+ cl_id[i] = clust->cl[i];
+ nstruct[cl_id[i]]++;
+ }
+ ncluster = 0;
+ for (i = 0; i < nf; i++)
+ {
+ if (nstruct[i] >= minstruct)
+ {
+ ncluster++;
+ for (j = 0; (j < nf); j++)
+ {
+ if (cl_id[j] == i)
+ {
+ strind[j] = ncluster;
+ }
+ }
+ }
+ }
+ ncluster++;
+ fprintf(stderr, "There are %d clusters with at least %d conformations\n",
+ ncluster, minstruct);
+
+ for (i = 0; (i < nf); i++)
+ {
+ ci = cl_id[i];
+ for (j = 0; j < i; j++)
+ {
+ if ((ci == cl_id[j]) && (nstruct[ci] >= minstruct))
+ {
+ /* color different clusters with different colors, as long as
+ we don't run out of colors */
+ mat[i][j] = strind[i];
+ }
+ else
+ {
+ mat[i][j] = 0;
+ }
+ }
+ }
+ sfree(strind);
+ sfree(nstruct);
+ sfree(cl_id);
+
+ return ncluster;
+}
+
+static void mark_clusters(int nf, real **mat, real val, t_clusters *clust)
+{
+ int i, j, v;
+
+ for (i = 0; i < nf; i++)
+ {
+ for (j = 0; j < i; j++)
+ {
+ if (clust->cl[i] == clust->cl[j])
+ {
+ mat[i][j] = val;
+ }
+ else
+ {
+ mat[i][j] = 0;
+ }
+ }
+ }
+}
+
+static char *parse_filename(const char *fn, int maxnr)
+{
+ int i;
+ char *fnout;
+ const char *ext;
+ char buf[STRLEN];
+
+ if (strchr(fn, '%'))
+ {
+ gmx_fatal(FARGS, "will not number filename %s containing '%c'", fn, '%');
+ }
+ /* number of digits needed in numbering */
+ i = (int)(log(maxnr)/log(10)) + 1;
+ /* split fn and ext */
+ ext = strrchr(fn, '.');
+ if (!ext)
+ {
+ gmx_fatal(FARGS, "cannot separate extension in filename %s", fn);
+ }
+ ext++;
+ /* insert e.g. '%03d' between fn and ext */
+ sprintf(buf, "%s%%0%dd.%s", fn, i, ext);
+ snew(fnout, strlen(buf)+1);
+ strcpy(fnout, buf);
+
+ return fnout;
+}
+
+static void ana_trans(t_clusters *clust, int nf,
+ const char *transfn, const char *ntransfn, FILE *log,
+ t_rgb rlo, t_rgb rhi, const output_env_t oenv)
+{
+ FILE *fp;
+ real **trans, *axis;
+ int *ntrans;
+ int i, ntranst, maxtrans;
+ char buf[STRLEN];
+
+ snew(ntrans, clust->ncl);
+ snew(trans, clust->ncl);
+ snew(axis, clust->ncl);
+ for (i = 0; i < clust->ncl; i++)
+ {
+ axis[i] = i+1;
+ snew(trans[i], clust->ncl);
+ }
+ ntranst = 0;
+ maxtrans = 0;
+ for (i = 1; i < nf; i++)
+ {
+ if (clust->cl[i] != clust->cl[i-1])
+ {
+ ntranst++;
+ ntrans[clust->cl[i-1]-1]++;
+ ntrans[clust->cl[i]-1]++;
+ trans[clust->cl[i-1]-1][clust->cl[i]-1]++;
+ maxtrans = max(maxtrans, trans[clust->cl[i]-1][clust->cl[i-1]-1]);
+ }
+ }
+ ffprintf_dd(stderr, log, buf, "Counted %d transitions in total, "
+ "max %d between two specific clusters\n", ntranst, maxtrans);
+ if (transfn)
+ {
+ fp = ffopen(transfn, "w");
+ i = min(maxtrans+1, 80);
+ write_xpm(fp, 0, "Cluster Transitions", "# transitions",
+ "from cluster", "to cluster",
+ clust->ncl, clust->ncl, axis, axis, trans,
+ 0, maxtrans, rlo, rhi, &i);
+ ffclose(fp);
+ }
+ if (ntransfn)
+ {
+ fp = xvgropen(ntransfn, "Cluster Transitions", "Cluster #", "# transitions",
+ oenv);
+ for (i = 0; i < clust->ncl; i++)
+ {
+ fprintf(fp, "%5d %5d\n", i+1, ntrans[i]);
+ }
+ ffclose(fp);
+ }
+ sfree(ntrans);
+ for (i = 0; i < clust->ncl; i++)
+ {
+ sfree(trans[i]);
+ }
+ sfree(trans);
+ sfree(axis);
+}
+
+static void analyze_clusters(int nf, t_clusters *clust, real **rmsd,
+ int natom, t_atoms *atoms, rvec *xtps,
+ real *mass, rvec **xx, real *time,
+ int ifsize, atom_id *fitidx,
+ int iosize, atom_id *outidx,
+ const char *trxfn, const char *sizefn,
+ const char *transfn, const char *ntransfn,
+ const char *clustidfn, gmx_bool bAverage,
+ int write_ncl, int write_nst, real rmsmin,
+ gmx_bool bFit, FILE *log, t_rgb rlo, t_rgb rhi,
+ const output_env_t oenv)
+{
+ FILE *fp = NULL;
+ char buf[STRLEN], buf1[40], buf2[40], buf3[40], *trxsfn;
+ t_trxstatus *trxout = NULL;
+ t_trxstatus *trxsout = NULL;
+ int i, i1, cl, nstr, *structure, first = 0, midstr;
+ gmx_bool *bWrite = NULL;
+ real r, clrmsd, midrmsd;
+ rvec *xav = NULL;
+ matrix zerobox;
+
+ clear_mat(zerobox);
+
+ ffprintf_d(stderr, log, buf, "\nFound %d clusters\n\n", clust->ncl);
+ trxsfn = NULL;
+ if (trxfn)
+ {
+ /* do we write all structures? */
+ if (write_ncl)
+ {
+ trxsfn = parse_filename(trxfn, max(write_ncl, clust->ncl));
+ snew(bWrite, nf);
+ }
+ ffprintf_ss(stderr, log, buf, "Writing %s structure for each cluster to %s\n",
+ bAverage ? "average" : "middle", trxfn);
+ if (write_ncl)
+ {
+ /* find out what we want to tell the user:
+ Writing [all structures|structures with rmsd > %g] for
+ {all|first %d} clusters {with more than %d structures} to %s */
+ if (rmsmin > 0.0)
+ {
+ sprintf(buf1, "structures with rmsd > %g", rmsmin);
+ }
+ else
+ {
+ sprintf(buf1, "all structures");
+ }
+ buf2[0] = buf3[0] = '\0';
+ if (write_ncl >= clust->ncl)
+ {
+ if (write_nst == 0)
+ {
+ sprintf(buf2, "all ");
+ }
+ }
+ else
+ {
+ sprintf(buf2, "the first %d ", write_ncl);
+ }
+ if (write_nst)
+ {
+ sprintf(buf3, " with more than %d structures", write_nst);
+ }
+ sprintf(buf, "Writing %s for %sclusters%s to %s\n", buf1, buf2, buf3, trxsfn);
+ ffprintf(stderr, log, buf);
+ }
+
+ /* Prepare a reference structure for the orientation of the clusters */
+ if (bFit)
+ {
+ reset_x(ifsize, fitidx, natom, NULL, xtps, mass);
+ }
+ trxout = open_trx(trxfn, "w");
+ /* Calculate the average structure in each cluster, *
+ * all structures are fitted to the first struture of the cluster */
+ snew(xav, natom);
+ }
+
+ if (transfn || ntransfn)
+ {
+ ana_trans(clust, nf, transfn, ntransfn, log, rlo, rhi, oenv);
+ }
+
+ if (clustidfn)
+ {
+ fp = xvgropen(clustidfn, "Clusters", output_env_get_xvgr_tlabel(oenv), "Cluster #", oenv);
+ fprintf(fp, "@ s0 symbol 2\n");
+ fprintf(fp, "@ s0 symbol size 0.2\n");
+ fprintf(fp, "@ s0 linestyle 0\n");
+ for (i = 0; i < nf; i++)
+ {
+ fprintf(fp, "%8g %8d\n", time[i], clust->cl[i]);
+ }
+ ffclose(fp);
+ }
+ if (sizefn)
+ {
+ fp = xvgropen(sizefn, "Cluster Sizes", "Cluster #", "# Structures", oenv);
+ fprintf(fp, "@g%d type %s\n", 0, "bar");
+ }
+ snew(structure, nf);
+ fprintf(log, "\n%3s | %3s %4s | %6s %4s | cluster members\n",
+ "cl.", "#st", "rmsd", "middle", "rmsd");
+ for (cl = 1; cl <= clust->ncl; cl++)
+ {
+ /* prepare structures (fit, middle, average) */
+ if (xav)
+ {
+ for (i = 0; i < natom; i++)
+ {
+ clear_rvec(xav[i]);
+ }
+ }
+ nstr = 0;
+ for (i1 = 0; i1 < nf; i1++)
+ {
+ if (clust->cl[i1] == cl)
+ {
+ structure[nstr] = i1;
+ nstr++;
+ if (trxfn && (bAverage || write_ncl) )
+ {
+ if (bFit)
+ {
+ reset_x(ifsize, fitidx, natom, NULL, xx[i1], mass);
+ }
+ if (nstr == 1)
+ {
+ first = i1;
+ }
+ else if (bFit)
+ {
+ do_fit(natom, mass, xx[first], xx[i1]);
+ }
+ if (xav)
+ {
+ for (i = 0; i < natom; i++)
+ {
+ rvec_inc(xav[i], xx[i1][i]);
+ }
+ }
+ }
+ }
+ }
+ if (sizefn)
+ {
+ fprintf(fp, "%8d %8d\n", cl, nstr);
+ }
+ clrmsd = 0;
+ midstr = 0;
+ midrmsd = 10000;
+ for (i1 = 0; i1 < nstr; i1++)
+ {
+ r = 0;
+ if (nstr > 1)
+ {
+ for (i = 0; i < nstr; i++)
+ {
+ if (i < i1)
+ {
+ r += rmsd[structure[i]][structure[i1]];
+ }
+ else
+ {
+ r += rmsd[structure[i1]][structure[i]];
+ }
+ }
+ r /= (nstr - 1);
+ }
+ if (r < midrmsd)
+ {
+ midstr = structure[i1];
+ midrmsd = r;
+ }
+ clrmsd += r;
+ }
+ clrmsd /= nstr;
+
+ /* dump cluster info to logfile */
+ if (nstr > 1)
+ {
+ sprintf(buf1, "%6.3f", clrmsd);
+ if (buf1[0] == '0')
+ {
+ buf1[0] = ' ';
+ }
+ sprintf(buf2, "%5.3f", midrmsd);
+ if (buf2[0] == '0')
+ {
+ buf2[0] = ' ';
+ }
+ }
+ else
+ {
+ sprintf(buf1, "%5s", "");
+ sprintf(buf2, "%5s", "");
+ }
+ fprintf(log, "%3d | %3d %s | %6g%s |", cl, nstr, buf1, time[midstr], buf2);
+ for (i = 0; i < nstr; i++)
+ {
+ if ((i % 7 == 0) && i)
+ {
+ sprintf(buf, "\n%3s | %3s %4s | %6s %4s |", "", "", "", "", "");
+ }
+ else
+ {
+ buf[0] = '\0';
+ }
+ i1 = structure[i];
+ fprintf(log, "%s %6g", buf, time[i1]);
+ }
+ fprintf(log, "\n");
+
+ /* write structures to trajectory file(s) */
+ if (trxfn)
+ {
+ if (write_ncl)
+ {
+ for (i = 0; i < nstr; i++)
+ {
+ bWrite[i] = FALSE;
+ }
+ }
+ if (cl < write_ncl+1 && nstr > write_nst)
+ {
+ /* Dump all structures for this cluster */
+ /* generate numbered filename (there is a %d in trxfn!) */
+ sprintf(buf, trxsfn, cl);
+ trxsout = open_trx(buf, "w");
+ for (i = 0; i < nstr; i++)
+ {
+ bWrite[i] = TRUE;
+ if (rmsmin > 0.0)
+ {
+ for (i1 = 0; i1 < i && bWrite[i]; i1++)
+ {
+ if (bWrite[i1])
+ {
+ bWrite[i] = rmsd[structure[i1]][structure[i]] > rmsmin;
+ }
+ }
+ }
+ if (bWrite[i])
+ {
+ write_trx(trxsout, iosize, outidx, atoms, i, time[structure[i]], zerobox,
+ xx[structure[i]], NULL, NULL);
+ }
+ }
+ close_trx(trxsout);
+ }
+ /* Dump the average structure for this cluster */
+ if (bAverage)
+ {
+ for (i = 0; i < natom; i++)
+ {
+ svmul(1.0/nstr, xav[i], xav[i]);
+ }
+ }
+ else
+ {
+ for (i = 0; i < natom; i++)
+ {
+ copy_rvec(xx[midstr][i], xav[i]);
+ }
+ if (bFit)
+ {
+ reset_x(ifsize, fitidx, natom, NULL, xav, mass);
+ }
+ }
+ if (bFit)
+ {
+ do_fit(natom, mass, xtps, xav);
+ }
+ r = cl;
+ write_trx(trxout, iosize, outidx, atoms, cl, time[midstr], zerobox, xav, NULL, NULL);
+ }
+ }
+ /* clean up */
+ if (trxfn)
+ {
+ close_trx(trxout);
+ sfree(xav);
+ if (write_ncl)
+ {
+ sfree(bWrite);
+ }
+ }
+ sfree(structure);
+ if (trxsfn)
+ {
+ sfree(trxsfn);
+ }
+}
+
+static void convert_mat(t_matrix *mat, t_mat *rms)
+{
+ int i, j;
+
+ rms->n1 = mat->nx;
+ matrix2real(mat, rms->mat);
+ /* free input xpm matrix data */
+ for (i = 0; i < mat->nx; i++)
+ {
+ sfree(mat->matrix[i]);
+ }
+ sfree(mat->matrix);
+
+ for (i = 0; i < mat->nx; i++)
+ {
+ for (j = i; j < mat->nx; j++)
+ {
+ rms->sumrms += rms->mat[i][j];
+ rms->maxrms = max(rms->maxrms, rms->mat[i][j]);
+ if (j != i)
+ {
+ rms->minrms = min(rms->minrms, rms->mat[i][j]);
+ }
+ }
+ }
+ rms->nn = mat->nx;
+}
+
+int gmx_cluster(int argc, char *argv[])
+{
+ const char *desc[] = {
+ "[TT]g_cluster[tt] can cluster structures using several different methods.",
+ "Distances between structures can be determined from a trajectory",
+ "or read from an [TT].xpm[tt] matrix file with the [TT]-dm[tt] option.",
+ "RMS deviation after fitting or RMS deviation of atom-pair distances",
+ "can be used to define the distance between structures.[PAR]",
+
+ "single linkage: add a structure to a cluster when its distance to any",
+ "element of the cluster is less than [TT]cutoff[tt].[PAR]",
+
+ "Jarvis Patrick: add a structure to a cluster when this structure",
+ "and a structure in the cluster have each other as neighbors and",
+ "they have a least [TT]P[tt] neighbors in common. The neighbors",
+ "of a structure are the M closest structures or all structures within",
+ "[TT]cutoff[tt].[PAR]",
+
+ "Monte Carlo: reorder the RMSD matrix using Monte Carlo such that",
+ "the order of the frames is using the smallest possible increments.",
+ "With this it is possible to make a smooth animation going from one",
+ "structure to another with the largest possible (e.g.) RMSD between",
+ "them, however the intermediate steps should be as small as possible.",
+ "Applications could be to visualize a potential of mean force",
+ "ensemble of simulations or a pulling simulation. Obviously the user",
+ "has to prepare the trajectory well (e.g. by not superimposing frames).",
+ "The final result can be inspect visually by looking at the matrix",
+ "[TT].xpm[tt] file, which should vary smoothly from bottom to top.[PAR]",
+
+ "diagonalization: diagonalize the RMSD matrix.[PAR]",
+
+ "gromos: use algorithm as described in Daura [IT]et al.[it]",
+ "([IT]Angew. Chem. Int. Ed.[it] [BB]1999[bb], [IT]38[it], pp 236-240).",
+ "Count number of neighbors using cut-off, take structure with",
+ "largest number of neighbors with all its neighbors as cluster",
+ "and eliminate it from the pool of clusters. Repeat for remaining",
+ "structures in pool.[PAR]",
+
+ "When the clustering algorithm assigns each structure to exactly one",
+ "cluster (single linkage, Jarvis Patrick and gromos) and a trajectory",
+ "file is supplied, the structure with",
+ "the smallest average distance to the others or the average structure",
+ "or all structures for each cluster will be written to a trajectory",
+ "file. When writing all structures, separate numbered files are made",
+ "for each cluster.[PAR]",
+
+ "Two output files are always written:[BR]",
+ "[TT]-o[tt] writes the RMSD values in the upper left half of the matrix",
+ "and a graphical depiction of the clusters in the lower right half",
+ "When [TT]-minstruct[tt] = 1 the graphical depiction is black",
+ "when two structures are in the same cluster.",
+ "When [TT]-minstruct[tt] > 1 different colors will be used for each",
+ "cluster.[BR]",
+ "[TT]-g[tt] writes information on the options used and a detailed list",
+ "of all clusters and their members.[PAR]",
+
+ "Additionally, a number of optional output files can be written:[BR]",
+ "[TT]-dist[tt] writes the RMSD distribution.[BR]",
+ "[TT]-ev[tt] writes the eigenvectors of the RMSD matrix",
+ "diagonalization.[BR]",
+ "[TT]-sz[tt] writes the cluster sizes.[BR]",
+ "[TT]-tr[tt] writes a matrix of the number transitions between",
+ "cluster pairs.[BR]",
+ "[TT]-ntr[tt] writes the total number of transitions to or from",
+ "each cluster.[BR]",
+ "[TT]-clid[tt] writes the cluster number as a function of time.[BR]",
+ "[TT]-cl[tt] writes average (with option [TT]-av[tt]) or central",
+ "structure of each cluster or writes numbered files with cluster members",
+ "for a selected set of clusters (with option [TT]-wcl[tt], depends on",
+ "[TT]-nst[tt] and [TT]-rmsmin[tt]). The center of a cluster is the",
+ "structure with the smallest average RMSD from all other structures",
+ "of the cluster.[BR]",
+ };
+
+ FILE *fp, *log;
- /* Initiate arrays */
- snew(d1, isize);
- snew(d2, isize);
- for (i = 0; (i < isize); i++)
- {
- snew(d1[i], isize);
- snew(d2[i], isize);
- }
++ int nf, i, i1, i2, j;
++ gmx_large_int_t nrms = 0;
+
+ matrix box;
+ rvec *xtps, *usextps, *x1, **xx = NULL;
+ const char *fn, *trx_out_fn;
+ t_clusters clust;
+ t_mat *rms, *orig = NULL;
+ real *eigenvalues;
+ t_topology top;
+ int ePBC;
+ t_atoms useatoms;
+ t_matrix *readmat = NULL;
+ real *eigenvectors;
+
+ int isize = 0, ifsize = 0, iosize = 0;
+ atom_id *index = NULL, *fitidx, *outidx;
+ char *grpname;
+ real rmsd, **d1, **d2, *time = NULL, time_invfac, *mass = NULL;
+ char buf[STRLEN], buf1[80], title[STRLEN];
+ gmx_bool bAnalyze, bUseRmsdCut, bJP_RMSD = FALSE, bReadMat, bReadTraj, bPBC = TRUE;
+
+ int method, ncluster = 0;
+ static const char *methodname[] = {
+ NULL, "linkage", "jarvis-patrick", "monte-carlo",
+ "diagonalization", "gromos", NULL
+ };
+ enum {
+ m_null, m_linkage, m_jarvis_patrick,
+ m_monte_carlo, m_diagonalize, m_gromos, m_nr
+ };
+ /* Set colors for plotting: white = zero RMS, black = maximum */
+ static t_rgb rlo_top = { 1.0, 1.0, 1.0 };
+ static t_rgb rhi_top = { 0.0, 0.0, 0.0 };
+ static t_rgb rlo_bot = { 1.0, 1.0, 1.0 };
+ static t_rgb rhi_bot = { 0.0, 0.0, 1.0 };
+ static int nlevels = 40, skip = 1;
+ static real scalemax = -1.0, rmsdcut = 0.1, rmsmin = 0.0;
+ gmx_bool bRMSdist = FALSE, bBinary = FALSE, bAverage = FALSE, bFit = TRUE;
+ static int niter = 10000, nrandom = 0, seed = 1993, write_ncl = 0, write_nst = 1, minstruct = 1;
+ static real kT = 1e-3;
+ static int M = 10, P = 3;
+ output_env_t oenv;
+ gmx_rmpbc_t gpbc = NULL;
+
+ t_pargs pa[] = {
+ { "-dista", FALSE, etBOOL, {&bRMSdist},
+ "Use RMSD of distances instead of RMS deviation" },
+ { "-nlevels", FALSE, etINT, {&nlevels},
+ "Discretize RMSD matrix in this number of levels" },
+ { "-cutoff", FALSE, etREAL, {&rmsdcut},
+ "RMSD cut-off (nm) for two structures to be neighbor" },
+ { "-fit", FALSE, etBOOL, {&bFit},
+ "Use least squares fitting before RMSD calculation" },
+ { "-max", FALSE, etREAL, {&scalemax},
+ "Maximum level in RMSD matrix" },
+ { "-skip", FALSE, etINT, {&skip},
+ "Only analyze every nr-th frame" },
+ { "-av", FALSE, etBOOL, {&bAverage},
+ "Write average iso middle structure for each cluster" },
+ { "-wcl", FALSE, etINT, {&write_ncl},
+ "Write the structures for this number of clusters to numbered files" },
+ { "-nst", FALSE, etINT, {&write_nst},
+ "Only write all structures if more than this number of structures per cluster" },
+ { "-rmsmin", FALSE, etREAL, {&rmsmin},
+ "minimum rms difference with rest of cluster for writing structures" },
+ { "-method", FALSE, etENUM, {methodname},
+ "Method for cluster determination" },
+ { "-minstruct", FALSE, etINT, {&minstruct},
+ "Minimum number of structures in cluster for coloring in the [TT].xpm[tt] file" },
+ { "-binary", FALSE, etBOOL, {&bBinary},
+ "Treat the RMSD matrix as consisting of 0 and 1, where the cut-off "
+ "is given by [TT]-cutoff[tt]" },
+ { "-M", FALSE, etINT, {&M},
+ "Number of nearest neighbors considered for Jarvis-Patrick algorithm, "
+ "0 is use cutoff" },
+ { "-P", FALSE, etINT, {&P},
+ "Number of identical nearest neighbors required to form a cluster" },
+ { "-seed", FALSE, etINT, {&seed},
+ "Random number seed for Monte Carlo clustering algorithm: <= 0 means generate" },
+ { "-niter", FALSE, etINT, {&niter},
+ "Number of iterations for MC" },
+ { "-nrandom", FALSE, etINT, {&nrandom},
+ "The first iterations for MC may be done complete random, to shuffle the frames" },
+ { "-kT", FALSE, etREAL, {&kT},
+ "Boltzmann weighting factor for Monte Carlo optimization "
+ "(zero turns off uphill steps)" },
+ { "-pbc", FALSE, etBOOL,
+ { &bPBC }, "PBC check" }
+ };
+ t_filenm fnm[] = {
+ { efTRX, "-f", NULL, ffOPTRD },
+ { efTPS, "-s", NULL, ffOPTRD },
+ { efNDX, NULL, NULL, ffOPTRD },
+ { efXPM, "-dm", "rmsd", ffOPTRD },
+ { efXPM, "-om", "rmsd-raw", ffWRITE },
+ { efXPM, "-o", "rmsd-clust", ffWRITE },
+ { efLOG, "-g", "cluster", ffWRITE },
+ { efXVG, "-dist", "rmsd-dist", ffOPTWR },
+ { efXVG, "-ev", "rmsd-eig", ffOPTWR },
+ { efXVG, "-conv", "mc-conv", ffOPTWR },
+ { efXVG, "-sz", "clust-size", ffOPTWR},
+ { efXPM, "-tr", "clust-trans", ffOPTWR},
+ { efXVG, "-ntr", "clust-trans", ffOPTWR},
+ { efXVG, "-clid", "clust-id.xvg", ffOPTWR},
+ { efTRX, "-cl", "clusters.pdb", ffOPTWR }
+ };
+#define NFILE asize(fnm)
+
+ if (!parse_common_args(&argc, argv,
+ PCA_CAN_VIEW | PCA_CAN_TIME | PCA_TIME_UNIT | PCA_BE_NICE,
+ NFILE, fnm, asize(pa), pa, asize(desc), desc, 0, NULL,
+ &oenv))
+ {
+ return 0;
+ }
+
+ /* parse options */
+ bReadMat = opt2bSet("-dm", NFILE, fnm);
+ bReadTraj = opt2bSet("-f", NFILE, fnm) || !bReadMat;
+ if (opt2parg_bSet("-av", asize(pa), pa) ||
+ opt2parg_bSet("-wcl", asize(pa), pa) ||
+ opt2parg_bSet("-nst", asize(pa), pa) ||
+ opt2parg_bSet("-rmsmin", asize(pa), pa) ||
+ opt2bSet("-cl", NFILE, fnm) )
+ {
+ trx_out_fn = opt2fn("-cl", NFILE, fnm);
+ }
+ else
+ {
+ trx_out_fn = NULL;
+ }
+ if (bReadMat && output_env_get_time_factor(oenv) != 1)
+ {
+ fprintf(stderr,
+ "\nWarning: assuming the time unit in %s is %s\n",
+ opt2fn("-dm", NFILE, fnm), output_env_get_time_unit(oenv));
+ }
+ if (trx_out_fn && !bReadTraj)
+ {
+ fprintf(stderr, "\nWarning: "
+ "cannot write cluster structures without reading trajectory\n"
+ " ignoring option -cl %s\n", trx_out_fn);
+ }
+
+ method = 1;
+ while (method < m_nr && gmx_strcasecmp(methodname[0], methodname[method]) != 0)
+ {
+ method++;
+ }
+ if (method == m_nr)
+ {
+ gmx_fatal(FARGS, "Invalid method");
+ }
+
+ bAnalyze = (method == m_linkage || method == m_jarvis_patrick ||
+ method == m_gromos );
+
+ /* Open log file */
+ log = ftp2FILE(efLOG, NFILE, fnm, "w");
+
+ fprintf(stderr, "Using %s method for clustering\n", methodname[0]);
+ fprintf(log, "Using %s method for clustering\n", methodname[0]);
+
+ /* check input and write parameters to log file */
+ bUseRmsdCut = FALSE;
+ if (method == m_jarvis_patrick)
+ {
+ bJP_RMSD = (M == 0) || opt2parg_bSet("-cutoff", asize(pa), pa);
+ if ((M < 0) || (M == 1))
+ {
+ gmx_fatal(FARGS, "M (%d) must be 0 or larger than 1", M);
+ }
+ if (M < 2)
+ {
+ sprintf(buf1, "Will use P=%d and RMSD cutoff (%g)", P, rmsdcut);
+ bUseRmsdCut = TRUE;
+ }
+ else
+ {
+ if (P >= M)
+ {
+ gmx_fatal(FARGS, "Number of neighbors required (P) must be less than M");
+ }
+ if (bJP_RMSD)
+ {
+ sprintf(buf1, "Will use P=%d, M=%d and RMSD cutoff (%g)", P, M, rmsdcut);
+ bUseRmsdCut = TRUE;
+ }
+ else
+ {
+ sprintf(buf1, "Will use P=%d, M=%d", P, M);
+ }
+ }
+ ffprintf_s(stderr, log, buf, "%s for determining the neighbors\n\n", buf1);
+ }
+ else /* method != m_jarvis */
+ {
+ bUseRmsdCut = ( bBinary || method == m_linkage || method == m_gromos );
+ }
+ if (bUseRmsdCut && method != m_jarvis_patrick)
+ {
+ fprintf(log, "Using RMSD cutoff %g nm\n", rmsdcut);
+ }
+ if (method == m_monte_carlo)
+ {
+ fprintf(log, "Using %d iterations\n", niter);
+ }
+
+ if (skip < 1)
+ {
+ gmx_fatal(FARGS, "skip (%d) should be >= 1", skip);
+ }
+
+ /* get input */
+ if (bReadTraj)
+ {
+ /* don't read mass-database as masses (and top) are not used */
+ read_tps_conf(ftp2fn(efTPS, NFILE, fnm), buf, &top, &ePBC, &xtps, NULL, box,
+ TRUE);
+ if (bPBC)
+ {
+ gpbc = gmx_rmpbc_init(&top.idef, ePBC, top.atoms.nr);
+ }
+
+ fprintf(stderr, "\nSelect group for least squares fit%s:\n",
+ bReadMat ? "" : " and RMSD calculation");
+ get_index(&(top.atoms), ftp2fn_null(efNDX, NFILE, fnm),
+ 1, &ifsize, &fitidx, &grpname);
+ if (trx_out_fn)
+ {
+ fprintf(stderr, "\nSelect group for output:\n");
+ get_index(&(top.atoms), ftp2fn_null(efNDX, NFILE, fnm),
+ 1, &iosize, &outidx, &grpname);
+ /* merge and convert both index groups: */
+ /* first copy outidx to index. let outidx refer to elements in index */
+ snew(index, iosize);
+ isize = iosize;
+ for (i = 0; i < iosize; i++)
+ {
+ index[i] = outidx[i];
+ outidx[i] = i;
+ }
+ /* now lookup elements from fitidx in index, add them if necessary
+ and also let fitidx refer to elements in index */
+ for (i = 0; i < ifsize; i++)
+ {
+ j = 0;
+ while (j < isize && index[j] != fitidx[i])
+ {
+ j++;
+ }
+ if (j >= isize)
+ {
+ /* slow this way, but doesn't matter much */
+ isize++;
+ srenew(index, isize);
+ }
+ index[j] = fitidx[i];
+ fitidx[i] = j;
+ }
+ }
+ else /* !trx_out_fn */
+ {
+ isize = ifsize;
+ snew(index, isize);
+ for (i = 0; i < ifsize; i++)
+ {
+ index[i] = fitidx[i];
+ fitidx[i] = i;
+ }
+ }
+ }
- nrms = (nf*(nf-1))/2;
+
+ if (bReadTraj)
+ {
+ /* Loop over first coordinate file */
+ fn = opt2fn("-f", NFILE, fnm);
+
+ xx = read_whole_trj(fn, isize, index, skip, &nf, &time, oenv, bPBC, gpbc);
+ output_env_conv_times(oenv, nf, time);
+ if (!bRMSdist || bAnalyze)
+ {
+ /* Center all frames on zero */
+ snew(mass, isize);
+ for (i = 0; i < ifsize; i++)
+ {
+ mass[fitidx[i]] = top.atoms.atom[index[fitidx[i]]].m;
+ }
+ if (bFit)
+ {
+ for (i = 0; i < nf; i++)
+ {
+ reset_x(ifsize, fitidx, isize, NULL, xx[i], mass);
+ }
+ }
+ }
+ if (bPBC)
+ {
+ gmx_rmpbc_done(gpbc);
+ }
+ }
+
+ if (bReadMat)
+ {
+ fprintf(stderr, "Reading rms distance matrix ");
+ read_xpm_matrix(opt2fn("-dm", NFILE, fnm), &readmat);
+ fprintf(stderr, "\n");
+ if (readmat[0].nx != readmat[0].ny)
+ {
+ gmx_fatal(FARGS, "Matrix (%dx%d) is not square",
+ readmat[0].nx, readmat[0].ny);
+ }
+ if (bReadTraj && bAnalyze && (readmat[0].nx != nf))
+ {
+ gmx_fatal(FARGS, "Matrix size (%dx%d) does not match the number of "
+ "frames (%d)", readmat[0].nx, readmat[0].ny, nf);
+ }
+
+ nf = readmat[0].nx;
+ sfree(time);
+ time = readmat[0].axis_x;
+ time_invfac = output_env_get_time_invfactor(oenv);
+ for (i = 0; i < nf; i++)
+ {
+ time[i] *= time_invfac;
+ }
+
+ rms = init_mat(readmat[0].nx, method == m_diagonalize);
+ convert_mat(&(readmat[0]), rms);
+
+ nlevels = readmat[0].nmap;
+ }
+ else /* !bReadMat */
+ {
+ rms = init_mat(nf, method == m_diagonalize);
- for (i1 = 0; (i1 < nf); i1++)
++ nrms = ((gmx_large_int_t)nf*((gmx_large_int_t)nf-1))/2;
+ if (!bRMSdist)
+ {
+ fprintf(stderr, "Computing %dx%d RMS deviation matrix\n", nf, nf);
++ /* Initialize work array */
+ snew(x1, isize);
- for (i2 = i1+1; (i2 < nf); i2++)
++ for (i1 = 0; i1 < nf; i1++)
+ {
- nrms -= (nf-i1-1);
- fprintf(stderr, "\r# RMSD calculations left: %d ", nrms);
++ for (i2 = i1+1; i2 < nf; i2++)
+ {
+ for (i = 0; i < isize; i++)
+ {
+ copy_rvec(xx[i1][i], x1[i]);
+ }
+ if (bFit)
+ {
+ do_fit(isize, mass, xx[i2], x1);
+ }
+ rmsd = rmsdev(isize, mass, xx[i2], x1);
+ set_mat_entry(rms, i1, i2, rmsd);
+ }
- for (i1 = 0; (i1 < nf); i1++)
++ nrms -= (gmx_large_int_t) (nf-i1-1);
++ fprintf(stderr, "\r# RMSD calculations left: "gmx_large_int_pfmt" ", nrms);
+ }
++ sfree(x1);
+ }
+ else /* bRMSdist */
+ {
+ fprintf(stderr, "Computing %dx%d RMS distance deviation matrix\n", nf, nf);
- fprintf(stderr, "\r# RMSD calculations left: %d ", nrms);
++
++ /* Initiate work arrays */
++ snew(d1, isize);
++ snew(d2, isize);
++ for (i = 0; (i < isize); i++)
++ {
++ snew(d1[i], isize);
++ snew(d2[i], isize);
++ }
++ for (i1 = 0; i1 < nf ; i1++)
+ {
+ calc_dist(isize, xx[i1], d1);
+ for (i2 = i1+1; (i2 < nf); i2++)
+ {
+ calc_dist(isize, xx[i2], d2);
+ set_mat_entry(rms, i1, i2, rms_dist(isize, d1, d2));
+ }
+ nrms -= (nf-i1-1);
++ fprintf(stderr, "\r# RMSD calculations left: "gmx_large_int_pfmt" ", nrms);
++ }
++ /* Clean up work arrays */
++ for (i = 0; (i < isize); i++)
++ {
++ sfree(d1[i]);
++ sfree(d2[i]);
+ }
++ sfree(d1);
++ sfree(d2);
+ }
+ fprintf(stderr, "\n\n");
+ }
+ ffprintf_gg(stderr, log, buf, "The RMSD ranges from %g to %g nm\n",
+ rms->minrms, rms->maxrms);
+ ffprintf_g(stderr, log, buf, "Average RMSD is %g\n", 2*rms->sumrms/(nf*(nf-1)));
+ ffprintf_d(stderr, log, buf, "Number of structures for matrix %d\n", nf);
+ ffprintf_g(stderr, log, buf, "Energy of the matrix is %g.\n", mat_energy(rms));
+ if (bUseRmsdCut && (rmsdcut < rms->minrms || rmsdcut > rms->maxrms) )
+ {
+ fprintf(stderr, "WARNING: rmsd cutoff %g is outside range of rmsd values "
+ "%g to %g\n", rmsdcut, rms->minrms, rms->maxrms);
+ }
+ if (bAnalyze && (rmsmin < rms->minrms) )
+ {
+ fprintf(stderr, "WARNING: rmsd minimum %g is below lowest rmsd value %g\n",
+ rmsmin, rms->minrms);
+ }
+ if (bAnalyze && (rmsmin > rmsdcut) )
+ {
+ fprintf(stderr, "WARNING: rmsd minimum %g is above rmsd cutoff %g\n",
+ rmsmin, rmsdcut);
+ }
+
+ /* Plot the rmsd distribution */
+ rmsd_distribution(opt2fn("-dist", NFILE, fnm), rms, oenv);
+
+ if (bBinary)
+ {
+ for (i1 = 0; (i1 < nf); i1++)
+ {
+ for (i2 = 0; (i2 < nf); i2++)
+ {
+ if (rms->mat[i1][i2] < rmsdcut)
+ {
+ rms->mat[i1][i2] = 0;
+ }
+ else
+ {
+ rms->mat[i1][i2] = 1;
+ }
+ }
+ }
+ }
+
+ snew(clust.cl, nf);
+ switch (method)
+ {
+ case m_linkage:
+ /* Now sort the matrix and write it out again */
+ gather(rms, rmsdcut, &clust);
+ break;
+ case m_diagonalize:
+ /* Do a diagonalization */
+ snew(eigenvalues, nf);
+ snew(eigenvectors, nf*nf);
+ memcpy(eigenvectors, rms->mat[0], nf*nf*sizeof(real));
+ eigensolver(eigenvectors, nf, 0, nf, eigenvalues, rms->mat[0]);
+ sfree(eigenvectors);
+
+ fp = xvgropen(opt2fn("-ev", NFILE, fnm), "RMSD matrix Eigenvalues",
+ "Eigenvector index", "Eigenvalues (nm\\S2\\N)", oenv);
+ for (i = 0; (i < nf); i++)
+ {
+ fprintf(fp, "%10d %10g\n", i, eigenvalues[i]);
+ }
+ ffclose(fp);
+ break;
+ case m_monte_carlo:
+ orig = init_mat(rms->nn, FALSE);
+ orig->nn = rms->nn;
+ copy_t_mat(orig, rms);
+ mc_optimize(log, rms, time, niter, nrandom, seed, kT,
+ opt2fn_null("-conv", NFILE, fnm), oenv);
+ break;
+ case m_jarvis_patrick:
+ jarvis_patrick(rms->nn, rms->mat, M, P, bJP_RMSD ? rmsdcut : -1, &clust);
+ break;
+ case m_gromos:
+ gromos(rms->nn, rms->mat, rmsdcut, &clust);
+ break;
+ default:
+ gmx_fatal(FARGS, "DEATH HORROR unknown method \"%s\"", methodname[0]);
+ }
+
+ if (method == m_monte_carlo || method == m_diagonalize)
+ {
+ fprintf(stderr, "Energy of the matrix after clustering is %g.\n",
+ mat_energy(rms));
+ }
+
+ if (bAnalyze)
+ {
+ if (minstruct > 1)
+ {
+ ncluster = plot_clusters(nf, rms->mat, &clust, minstruct);
+ }
+ else
+ {
+ mark_clusters(nf, rms->mat, rms->maxrms, &clust);
+ }
+ init_t_atoms(&useatoms, isize, FALSE);
+ snew(usextps, isize);
+ useatoms.resinfo = top.atoms.resinfo;
+ for (i = 0; i < isize; i++)
+ {
+ useatoms.atomname[i] = top.atoms.atomname[index[i]];
+ useatoms.atom[i].resind = top.atoms.atom[index[i]].resind;
+ useatoms.nres = max(useatoms.nres, useatoms.atom[i].resind+1);
+ copy_rvec(xtps[index[i]], usextps[i]);
+ }
+ useatoms.nr = isize;
+ analyze_clusters(nf, &clust, rms->mat, isize, &useatoms, usextps, mass, xx, time,
+ ifsize, fitidx, iosize, outidx,
+ bReadTraj ? trx_out_fn : NULL,
+ opt2fn_null("-sz", NFILE, fnm),
+ opt2fn_null("-tr", NFILE, fnm),
+ opt2fn_null("-ntr", NFILE, fnm),
+ opt2fn_null("-clid", NFILE, fnm),
+ bAverage, write_ncl, write_nst, rmsmin, bFit, log,
+ rlo_bot, rhi_bot, oenv);
+ }
+ ffclose(log);
+
+ if (bBinary && !bAnalyze)
+ {
+ /* Make the clustering visible */
+ for (i2 = 0; (i2 < nf); i2++)
+ {
+ for (i1 = i2+1; (i1 < nf); i1++)
+ {
+ if (rms->mat[i1][i2])
+ {
+ rms->mat[i1][i2] = rms->maxrms;
+ }
+ }
+ }
+ }
+
+ fp = opt2FILE("-o", NFILE, fnm, "w");
+ fprintf(stderr, "Writing rms distance/clustering matrix ");
+ if (bReadMat)
+ {
+ write_xpm(fp, 0, readmat[0].title, readmat[0].legend, readmat[0].label_x,
+ readmat[0].label_y, nf, nf, readmat[0].axis_x, readmat[0].axis_y,
+ rms->mat, 0.0, rms->maxrms, rlo_top, rhi_top, &nlevels);
+ }
+ else
+ {
+ sprintf(buf, "Time (%s)", output_env_get_time_unit(oenv));
+ sprintf(title, "RMS%sDeviation / Cluster Index",
+ bRMSdist ? " Distance " : " ");
+ if (minstruct > 1)
+ {
+ write_xpm_split(fp, 0, title, "RMSD (nm)", buf, buf,
+ nf, nf, time, time, rms->mat, 0.0, rms->maxrms, &nlevels,
+ rlo_top, rhi_top, 0.0, (real) ncluster,
+ &ncluster, TRUE, rlo_bot, rhi_bot);
+ }
+ else
+ {
+ write_xpm(fp, 0, title, "RMSD (nm)", buf, buf,
+ nf, nf, time, time, rms->mat, 0.0, rms->maxrms,
+ rlo_top, rhi_top, &nlevels);
+ }
+ }
+ fprintf(stderr, "\n");
+ ffclose(fp);
+ if (NULL != orig)
+ {
+ fp = opt2FILE("-om", NFILE, fnm, "w");
+ sprintf(buf, "Time (%s)", output_env_get_time_unit(oenv));
+ sprintf(title, "RMS%sDeviation", bRMSdist ? " Distance " : " ");
+ write_xpm(fp, 0, title, "RMSD (nm)", buf, buf,
+ nf, nf, time, time, orig->mat, 0.0, orig->maxrms,
+ rlo_top, rhi_top, &nlevels);
+ ffclose(fp);
+ done_mat(&orig);
+ sfree(orig);
+ }
+ /* now show what we've done */
+ do_view(oenv, opt2fn("-o", NFILE, fnm), "-nxy");
+ do_view(oenv, opt2fn_null("-sz", NFILE, fnm), "-nxy");
+ if (method == m_diagonalize)
+ {
+ do_view(oenv, opt2fn_null("-ev", NFILE, fnm), "-nxy");
+ }
+ do_view(oenv, opt2fn("-dist", NFILE, fnm), "-nxy");
+ if (bAnalyze)
+ {
+ do_view(oenv, opt2fn_null("-tr", NFILE, fnm), "-nxy");
+ do_view(oenv, opt2fn_null("-ntr", NFILE, fnm), "-nxy");
+ do_view(oenv, opt2fn_null("-clid", NFILE, fnm), "-nxy");
+ }
+ do_view(oenv, opt2fn_null("-conv", NFILE, fnm), NULL);
+
+ return 0;
+}
--- /dev/null
- " - [TT]energygrp = Protein[tt] (or other group that you want to insert)[BR]",
+/*
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2012, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * Gallium Rubidium Oxygen Manganese Argon Carbon Silicon
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <signal.h>
+#include <stdlib.h>
+#include "typedefs.h"
+#include "sysstuff.h"
+#include "statutil.h"
+#include "macros.h"
+#include "main.h"
+#include "gmx_ana.h"
+#include "string2.h"
+
+int gmx_membed(int argc, char *argv[])
+{
+ const char *desc[] = {
+ "[TT]g_membed[tt] embeds a membrane protein into an equilibrated lipid bilayer at the position",
+ "and orientation specified by the user.[PAR]",
+ "SHORT MANUAL[BR]------------[BR]",
+ "The user should merge the structure files of the protein and membrane (+solvent), creating a",
+ "single structure file with the protein overlapping the membrane at the desired position and",
+ "orientation. The box size is taken from the membrane structure file. The corresponding topology",
+ "files should also be merged. Consecutively, create a [TT].tpr[tt] file (input for [TT]g_membed[tt]) from these files,"
+ "with the following options included in the [TT].mdp[tt] file.[BR]",
+ " - [TT]integrator = md[tt][BR]",
++ " - [TT]energygrps = Protein[tt] (or other group that you want to insert)[BR]",
+ " - [TT]freezegrps = Protein[tt][BR]",
+ " - [TT]freezedim = Y Y Y[tt][BR]",
+ " - [TT]energygrp_excl = Protein Protein[tt][BR]",
+ "The output is a structure file containing the protein embedded in the membrane. If a topology",
+ "file is provided, the number of lipid and ",
+ "solvent molecules will be updated to match the new structure file.[BR]",
+ "For a more extensive manual see Wolf et al, J Comp Chem 31 (2010) 2169-2174, Appendix.[PAR]",
+ "SHORT METHOD DESCRIPTION[BR]",
+ "------------------------[BR]",
+ "1. The protein is resized around its center of mass by a factor [TT]-xy[tt] in the xy-plane",
+ "(the membrane plane) and a factor [TT]-z[tt] in the [IT]z[it]-direction (if the size of the",
+ "protein in the z-direction is the same or smaller than the width of the membrane, a",
+ "[TT]-z[tt] value larger than 1 can prevent that the protein will be enveloped by the lipids).[BR]",
+ "2. All lipid and solvent molecules overlapping with the resized protein are removed. All",
+ "intraprotein interactions are turned off to prevent numerical issues for small values of [TT]-xy[tt]",
+ " or [TT]-z[tt][BR]",
+ "3. One md step is performed.[BR]",
+ "4. The resize factor ([TT]-xy[tt] or [TT]-z[tt]) is incremented by a small amount ((1-xy)/nxy or (1-z)/nz) and the",
+ "protein is resized again around its center of mass. The resize factor for the xy-plane",
+ "is incremented first. The resize factor for the z-direction is not changed until the [TT]-xy[tt] factor",
+ "is 1 (thus after [TT]-nxy[tt] iterations).[BR]",
+ "5. Repeat step 3 and 4 until the protein reaches its original size ([TT]-nxy[tt] + [TT]-nz[tt] iterations).[BR]",
+ "For a more extensive method description see Wolf et al, J Comp Chem, 31 (2010) 2169-2174.[PAR]",
+ "NOTE[BR]----[BR]",
+ " - Protein can be any molecule you want to insert in the membrane.[BR]",
+ " - It is recommended to perform a short equilibration run after the embedding",
+ "(see Wolf et al, J Comp Chem 31 (2010) 2169-2174), to re-equilibrate the membrane. Clearly",
+ "protein equilibration might require longer.[PAR]"
+ };
+ t_filenm fnm[] = {
+ { efTPX, "-f", "into_mem", ffREAD },
+ { efNDX, "-n", "index", ffOPTRD },
+ { efTOP, "-p", "topol", ffOPTRW },
+ { efTRN, "-o", NULL, ffWRITE },
+ { efXTC, "-x", NULL, ffOPTWR },
+ { efSTO, "-c", "membedded", ffWRITE },
+ { efEDR, "-e", "ener", ffWRITE },
+ { efDAT, "-dat", "membed", ffWRITE }
+ };
+#define NFILE asize(fnm)
+
+ /* Command line options ! */
+ real xy_fac = 0.5;
+ real xy_max = 1.0;
+ real z_fac = 1.0;
+ real z_max = 1.0;
+ int it_xy = 1000;
+ int it_z = 0;
+ real probe_rad = 0.22;
+ int low_up_rm = 0;
+ int maxwarn = 0;
+ int pieces = 1;
+ gmx_bool bALLOW_ASYMMETRY = FALSE;
+ gmx_bool bStart = FALSE;
+ int nstepout = 100;
+ gmx_bool bVerbose = FALSE;
+ char *mdrun_path = NULL;
+
+ t_pargs pa[] = {
+ { "-xyinit", FALSE, etREAL, {&xy_fac},
+ "Resize factor for the protein in the xy dimension before starting embedding" },
+ { "-xyend", FALSE, etREAL, {&xy_max},
+ "Final resize factor in the xy dimension" },
+ { "-zinit", FALSE, etREAL, {&z_fac},
+ "Resize factor for the protein in the z dimension before starting embedding" },
+ { "-zend", FALSE, etREAL, {&z_max},
+ "Final resize faction in the z dimension" },
+ { "-nxy", FALSE, etINT, {&it_xy},
+ "Number of iteration for the xy dimension" },
+ { "-nz", FALSE, etINT, {&it_z},
+ "Number of iterations for the z dimension" },
+ { "-rad", FALSE, etREAL, {&probe_rad},
+ "Probe radius to check for overlap between the group to embed and the membrane"},
+ { "-pieces", FALSE, etINT, {&pieces},
+ "Perform piecewise resize. Select parts of the group to insert and resize these with respect to their own geometrical center." },
+ { "-asymmetry", FALSE, etBOOL, {&bALLOW_ASYMMETRY},
+ "Allow asymmetric insertion, i.e. the number of lipids removed from the upper and lower leaflet will not be checked." },
+ { "-ndiff", FALSE, etINT, {&low_up_rm},
+ "Number of lipids that will additionally be removed from the lower (negative number) or upper (positive number) membrane leaflet." },
+ { "-maxwarn", FALSE, etINT, {&maxwarn},
+ "Maximum number of warning allowed" },
+ { "-start", FALSE, etBOOL, {&bStart},
+ "Call mdrun with membed options" },
+ { "-stepout", FALSE, etINT, {&nstepout},
+ "HIDDENFrequency of writing the remaining runtime" },
+ { "-v", FALSE, etBOOL, {&bVerbose},
+ "Be loud and noisy" },
+ { "-mdrun_path", FALSE, etSTR, {&mdrun_path},
+ "Path to the mdrun executable compiled with this g_membed version" }
+ };
+
+ FILE *data_out;
+ output_env_t oenv;
+ char buf[256], buf2[64];
+ gmx_bool bSucces;
+
+ if (!parse_common_args(&argc, argv, 0, NFILE, fnm, asize(pa), pa,
+ asize(desc), desc, 0, NULL, &oenv))
+ {
+ return 0;
+ }
+
+ data_out = ffopen(opt2fn("-dat", NFILE, fnm), "w");
+
+ fprintf(data_out, "nxy = %d\nnz = %d\nxyinit = %f\nxyend = %f\nzinit = %f\nzend = %f\n"
+ "rad = %f\npieces = %d\nasymmetry = %s\nndiff = %d\nmaxwarn = %d\n",
+ it_xy, it_z, xy_fac, xy_max, z_fac, z_max, probe_rad, pieces,
+ bALLOW_ASYMMETRY ? "yes" : "no", low_up_rm, maxwarn);
+
+ fclose(data_out);
+
+ sprintf(buf, "%s -s %s -membed %s -o %s -c %s -e %s -nt 1 -cpt -1",
+ (mdrun_path == NULL) ? "mdrun" : mdrun_path,
+ opt2fn("-f", NFILE, fnm), opt2fn("-dat", NFILE, fnm), opt2fn("-o", NFILE, fnm),
+ opt2fn("-c", NFILE, fnm), opt2fn("-e", NFILE, fnm));
+
+ if (opt2bSet("-n", NFILE, fnm))
+ {
+ sprintf(buf2, " -mn %s", opt2fn("-n", NFILE, fnm));
+ strcat(buf, buf2);
+ }
+
+ if (opt2bSet("-x", NFILE, fnm))
+ {
+ sprintf(buf2, " -x %s", opt2fn("-x", NFILE, fnm));
+ strcat(buf, buf2);
+ }
+
+ if (opt2bSet("-p", NFILE, fnm))
+ {
+ sprintf(buf2, " -mp %s", opt2fn("-p", NFILE, fnm));
+ strcat(buf, buf2);
+ }
+
+ if (bVerbose)
+ {
+ sprintf(buf2, " -v -stepout %d", nstepout);
+ strcat(buf, buf2);
+ }
+
+ if (bStart)
+ {
+ printf("Start run with:\n%s\n", buf);
+ bSucces = system(buf);
+ }
+ else
+ {
+ printf("You can membed your protein now by:\n%s\n", buf);
+ }
+
+ fprintf(stderr, "Please cite:\nWolf et al, J Comp Chem 31 (2010) 2169-2174.\n");
+
+ return 0;
+}
--- /dev/null
- "2. [TT]trjconv -s a.tpr -f a.xtc -o b.xtc -center tric -ur compact -pbc none[tt] \n",
+/*
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * VERSION 3.0
+ *
+ * Copyright (c) 1991-2001
+ * BIOSON Research Institute, Dept. of Biophysical Chemistry
+ * University of Groningen, The Netherlands
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * Do check out http://www.gromacs.org , or mail us at gromacs@gromacs.org .
+ *
+ * And Hey:
+ * Gyas ROwers Mature At Cryogenic Speed
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+
+
+#include "statutil.h"
+#include "typedefs.h"
+#include "smalloc.h"
+#include "vec.h"
+#include "statutil.h"
+#include "tpxio.h"
+#include <math.h>
+#include "index.h"
+#include "pbc.h"
+#include "rmpbc.h"
+#include "gmx_ana.h"
+#include "macros.h"
+
+
+static const double bohr = 0.529177249; /* conversion factor to compensate for VMD plugin conversion... */
+
+static void mequit(void)
+{
+ printf("Memory allocation error\n");
+ exit(1);
+}
+
+int gmx_spatial(int argc, char *argv[])
+{
+ const char *desc[] = {
+ "[TT]g_spatial[tt] calculates the spatial distribution function and ",
+ "outputs it in a form that can be read by VMD as Gaussian98 cube format. ",
+ "This was developed from template.c (GROMACS-3.3). ",
+ "For a system of 32,000 atoms and a 50 ns trajectory, the SDF can be generated ",
+ "in about 30 minutes, with most of the time dedicated to the two runs through ",
+ "[TT]trjconv[tt] that are required to center everything properly. ",
+ "This also takes a whole bunch of space (3 copies of the [TT].xtc[tt] file). ",
+ "Still, the pictures are pretty and very informative when the fitted selection is properly made. ",
+ "3-4 atoms in a widely mobile group (like a free amino acid in solution) works ",
+ "well, or select the protein backbone in a stable folded structure to get the SDF ",
+ "of solvent and look at the time-averaged solvation shell. ",
+ "It is also possible using this program to generate the SDF based on some arbitrary ",
+ "Cartesian coordinate. To do that, simply omit the preliminary [TT]trjconv[tt] steps. \n",
+ "USAGE: \n",
+ "1. Use [TT]make_ndx[tt] to create a group containing the atoms around which you want the SDF \n",
++ "2. [TT]trjconv -s a.tpr -f a.xtc -o b.xtc -boxcenter tric -ur compact -pbc none[tt] \n",
+ "3. [TT]trjconv -s a.tpr -f b.xtc -o c.xtc -fit rot+trans[tt] \n",
+ "4. run [TT]g_spatial[tt] on the [TT].xtc[tt] output of step #3. \n",
+ "5. Load [TT]grid.cube[tt] into VMD and view as an isosurface. \n",
+ "[BB]Note[bb] that systems such as micelles will require [TT]trjconv -pbc cluster[tt] between steps 1 and 2\n",
+ "WARNINGS:[BR]",
+ "The SDF will be generated for a cube that contains all bins that have some non-zero occupancy. ",
+ "However, the preparatory [TT]-fit rot+trans[tt] option to [TT]trjconv[tt] implies that your system will be rotating ",
+ "and translating in space (in order that the selected group does not). Therefore the values that are ",
+ "returned will only be valid for some region around your central group/coordinate that has full overlap ",
+ "with system volume throughout the entire translated/rotated system over the course of the trajectory. ",
+ "It is up to the user to ensure that this is the case. \n",
+ "BUGS:[BR]",
+ "When the allocated memory is not large enough, a segmentation fault may occur. This is usually detected ",
+ "and the program is halted prior to the fault while displaying a warning message suggesting the use of the [TT]-nab[tt] (Number of Additional Bins)",
+ "option. However, the program does not detect all such events. If you encounter a segmentation fault, run it again ",
+ "with an increased [TT]-nab[tt] value. \n",
+ "RISKY OPTIONS:[BR]",
+ "To reduce the amount of space and time required, you can output only the coords ",
+ "that are going to be used in the first and subsequent run through [TT]trjconv[tt]. ",
+ "However, be sure to set the [TT]-nab[tt] option to a sufficiently high value since ",
+ "memory is allocated for cube bins based on the initial coordinates and the [TT]-nab[tt] ",
+ "option value. \n"
+ };
+
+ static gmx_bool bPBC = FALSE;
+ static gmx_bool bSHIFT = FALSE;
+ static int iIGNOREOUTER = -1; /*Positive values may help if the surface is spikey */
+ static gmx_bool bCUTDOWN = TRUE;
+ static real rBINWIDTH = 0.05; /* nm */
+ static gmx_bool bCALCDIV = TRUE;
+ static int iNAB = 4;
+
+ t_pargs pa[] = {
+ { "-pbc", FALSE, etBOOL, {&bPBC},
+ "Use periodic boundary conditions for computing distances" },
+ { "-div", FALSE, etBOOL, {&bCALCDIV},
+ "Calculate and apply the divisor for bin occupancies based on atoms/minimal cube size. Set as TRUE for visualization and as FALSE ([TT]-nodiv[tt]) to get accurate counts per frame" },
+ { "-ign", FALSE, etINT, {&iIGNOREOUTER},
+ "Do not display this number of outer cubes (positive values may reduce boundary speckles; -1 ensures outer surface is visible)" },
+ /* { "-cut", bCUTDOWN, etBOOL, {&bCUTDOWN},*/
+ /* "Display a total cube that is of minimal size" }, */
+ { "-bin", FALSE, etREAL, {&rBINWIDTH},
+ "Width of the bins (nm)" },
+ { "-nab", FALSE, etINT, {&iNAB},
+ "Number of additional bins to ensure proper memory allocation" }
+ };
+
+ double MINBIN[3];
+ double MAXBIN[3];
+ t_topology top;
+ int ePBC;
+ char title[STRLEN];
+ t_trxframe fr;
+ rvec *xtop, *shx[26];
+ matrix box, box_pbc;
+ t_trxstatus *status;
+ int flags = TRX_READ_X;
+ t_pbc pbc;
+ t_atoms *atoms;
+ int natoms;
+ char *grpnm, *grpnmp;
+ atom_id *index, *indexp;
+ int i, nidx, nidxp;
+ int v;
+ int j, k;
+ long ***bin = (long ***)NULL;
+ long nbin[3];
+ FILE *flp;
+ long x, y, z, minx, miny, minz, maxx, maxy, maxz;
+ long numfr, numcu;
+ long tot, max, min;
+ double norm;
+ output_env_t oenv;
+ gmx_rmpbc_t gpbc = NULL;
+
+ t_filenm fnm[] = {
+ { efTPS, NULL, NULL, ffREAD }, /* this is for the topology */
+ { efTRX, "-f", NULL, ffREAD }, /* and this for the trajectory */
+ { efNDX, NULL, NULL, ffOPTRD }
+ };
+
+#define NFILE asize(fnm)
+
+ /* This is the routine responsible for adding default options,
+ * calling the X/motif interface, etc. */
+ if (!parse_common_args(&argc, argv, PCA_CAN_TIME | PCA_CAN_VIEW,
+ NFILE, fnm, asize(pa), pa, asize(desc), desc, 0, NULL, &oenv))
+ {
+ return 0;
+ }
+
+ read_tps_conf(ftp2fn(efTPS, NFILE, fnm), title, &top, &ePBC, &xtop, NULL, box, TRUE);
+ sfree(xtop);
+
+ atoms = &(top.atoms);
+ printf("Select group to generate SDF:\n");
+ get_index(atoms, ftp2fn_null(efNDX, NFILE, fnm), 1, &nidx, &index, &grpnm);
+ printf("Select group to output coords (e.g. solute):\n");
+ get_index(atoms, ftp2fn_null(efNDX, NFILE, fnm), 1, &nidxp, &indexp, &grpnmp);
+
+ /* The first time we read data is a little special */
+ natoms = read_first_frame(oenv, &status, ftp2fn(efTRX, NFILE, fnm), &fr, flags);
+
+ /* Memory Allocation */
+ MINBIN[XX] = MAXBIN[XX] = fr.x[0][XX];
+ MINBIN[YY] = MAXBIN[YY] = fr.x[0][YY];
+ MINBIN[ZZ] = MAXBIN[ZZ] = fr.x[0][ZZ];
+ for (i = 1; i < top.atoms.nr; ++i)
+ {
+ if (fr.x[i][XX] < MINBIN[XX])
+ {
+ MINBIN[XX] = fr.x[i][XX];
+ }
+ if (fr.x[i][XX] > MAXBIN[XX])
+ {
+ MAXBIN[XX] = fr.x[i][XX];
+ }
+ if (fr.x[i][YY] < MINBIN[YY])
+ {
+ MINBIN[YY] = fr.x[i][YY];
+ }
+ if (fr.x[i][YY] > MAXBIN[YY])
+ {
+ MAXBIN[YY] = fr.x[i][YY];
+ }
+ if (fr.x[i][ZZ] < MINBIN[ZZ])
+ {
+ MINBIN[ZZ] = fr.x[i][ZZ];
+ }
+ if (fr.x[i][ZZ] > MAXBIN[ZZ])
+ {
+ MAXBIN[ZZ] = fr.x[i][ZZ];
+ }
+ }
+ for (i = ZZ; i >= XX; --i)
+ {
+ MAXBIN[i] = (ceil((MAXBIN[i]-MINBIN[i])/rBINWIDTH)+(double)iNAB)*rBINWIDTH+MINBIN[i];
+ MINBIN[i] -= (double)iNAB*rBINWIDTH;
+ nbin[i] = (long)ceil((MAXBIN[i]-MINBIN[i])/rBINWIDTH);
+ }
+ bin = (long ***)malloc(nbin[XX]*sizeof(long **));
+ if (!bin)
+ {
+ mequit();
+ }
+ for (i = 0; i < nbin[XX]; ++i)
+ {
+ bin[i] = (long **)malloc(nbin[YY]*sizeof(long *));
+ if (!bin[i])
+ {
+ mequit();
+ }
+ for (j = 0; j < nbin[YY]; ++j)
+ {
+ bin[i][j] = (long *)calloc(nbin[ZZ], sizeof(long));
+ if (!bin[i][j])
+ {
+ mequit();
+ }
+ }
+ }
+ copy_mat(box, box_pbc);
+ numfr = 0;
+ minx = miny = minz = 999;
+ maxx = maxy = maxz = 0;
+
+ if (bPBC)
+ {
+ gpbc = gmx_rmpbc_init(&top.idef, ePBC, natoms);
+ }
+ /* This is the main loop over frames */
+ do
+ {
+ /* Must init pbc every step because of pressure coupling */
+
+ copy_mat(box, box_pbc);
+ if (bPBC)
+ {
+ gmx_rmpbc_trxfr(gpbc, &fr);
+ set_pbc(&pbc, ePBC, box_pbc);
+ }
+
+ for (i = 0; i < nidx; i++)
+ {
+ if (fr.x[index[i]][XX] < MINBIN[XX] || fr.x[index[i]][XX] > MAXBIN[XX] ||
+ fr.x[index[i]][YY] < MINBIN[YY] || fr.x[index[i]][YY] > MAXBIN[YY] ||
+ fr.x[index[i]][ZZ] < MINBIN[ZZ] || fr.x[index[i]][ZZ] > MAXBIN[ZZ])
+ {
+ printf("There was an item outside of the allocated memory. Increase the value given with the -nab option.\n");
+ printf("Memory was allocated for [%f,%f,%f]\tto\t[%f,%f,%f]\n", MINBIN[XX], MINBIN[YY], MINBIN[ZZ], MAXBIN[XX], MAXBIN[YY], MAXBIN[ZZ]);
+ printf("Memory was required for [%f,%f,%f]\n", fr.x[index[i]][XX], fr.x[index[i]][YY], fr.x[index[i]][ZZ]);
+ exit(1);
+ }
+ x = (long)ceil((fr.x[index[i]][XX]-MINBIN[XX])/rBINWIDTH);
+ y = (long)ceil((fr.x[index[i]][YY]-MINBIN[YY])/rBINWIDTH);
+ z = (long)ceil((fr.x[index[i]][ZZ]-MINBIN[ZZ])/rBINWIDTH);
+ ++bin[x][y][z];
+ if (x < minx)
+ {
+ minx = x;
+ }
+ if (x > maxx)
+ {
+ maxx = x;
+ }
+ if (y < miny)
+ {
+ miny = y;
+ }
+ if (y > maxy)
+ {
+ maxy = y;
+ }
+ if (z < minz)
+ {
+ minz = z;
+ }
+ if (z > maxz)
+ {
+ maxz = z;
+ }
+ }
+ numfr++;
+ /* printf("%f\t%f\t%f\n",box[XX][XX],box[YY][YY],box[ZZ][ZZ]); */
+
+ }
+ while (read_next_frame(oenv, status, &fr));
+
+ if (bPBC)
+ {
+ gmx_rmpbc_done(gpbc);
+ }
+
+ if (!bCUTDOWN)
+ {
+ minx = miny = minz = 0;
+ maxx = nbin[XX];
+ maxy = nbin[YY];
+ maxz = nbin[ZZ];
+ }
+
+ /* OUTPUT */
+ flp = ffopen("grid.cube", "w");
+ fprintf(flp, "Spatial Distribution Function\n");
+ fprintf(flp, "test\n");
+ fprintf(flp, "%5d%12.6f%12.6f%12.6f\n", nidxp, (MINBIN[XX]+(minx+iIGNOREOUTER)*rBINWIDTH)*10./bohr, (MINBIN[YY]+(miny+iIGNOREOUTER)*rBINWIDTH)*10./bohr, (MINBIN[ZZ]+(minz+iIGNOREOUTER)*rBINWIDTH)*10./bohr);
+ fprintf(flp, "%5ld%12.6f%12.6f%12.6f\n", maxx-minx+1-(2*iIGNOREOUTER), rBINWIDTH*10./bohr, 0., 0.);
+ fprintf(flp, "%5ld%12.6f%12.6f%12.6f\n", maxy-miny+1-(2*iIGNOREOUTER), 0., rBINWIDTH*10./bohr, 0.);
+ fprintf(flp, "%5ld%12.6f%12.6f%12.6f\n", maxz-minz+1-(2*iIGNOREOUTER), 0., 0., rBINWIDTH*10./bohr);
+ for (i = 0; i < nidxp; i++)
+ {
+ v = 2;
+ if (*(top.atoms.atomname[indexp[i]][0]) == 'C')
+ {
+ v = 6;
+ }
+ if (*(top.atoms.atomname[indexp[i]][0]) == 'N')
+ {
+ v = 7;
+ }
+ if (*(top.atoms.atomname[indexp[i]][0]) == 'O')
+ {
+ v = 8;
+ }
+ if (*(top.atoms.atomname[indexp[i]][0]) == 'H')
+ {
+ v = 1;
+ }
+ if (*(top.atoms.atomname[indexp[i]][0]) == 'S')
+ {
+ v = 16;
+ }
+ fprintf(flp, "%5d%12.6f%12.6f%12.6f%12.6f\n", v, 0., (double)fr.x[indexp[i]][XX]*10./bohr, (double)fr.x[indexp[i]][YY]*10./bohr, (double)fr.x[indexp[i]][ZZ]*10./bohr);
+ }
+
+ tot = 0;
+ for (k = 0; k < nbin[XX]; k++)
+ {
+ if (!(k < minx || k > maxx))
+ {
+ continue;
+ }
+ for (j = 0; j < nbin[YY]; j++)
+ {
+ if (!(j < miny || j > maxy))
+ {
+ continue;
+ }
+ for (i = 0; i < nbin[ZZ]; i++)
+ {
+ if (!(i < minz || i > maxz))
+ {
+ continue;
+ }
+ if (bin[k][j][i] != 0)
+ {
+ printf("A bin was not empty when it should have been empty. Programming error.\n");
+ printf("bin[%d][%d][%d] was = %ld\n", k, j, i, bin[k][j][i]);
+ exit(1);
+ }
+ }
+ }
+ }
+
+ min = 999;
+ max = 0;
+ for (k = 0; k < nbin[XX]; k++)
+ {
+ if (k < minx+iIGNOREOUTER || k > maxx-iIGNOREOUTER)
+ {
+ continue;
+ }
+ for (j = 0; j < nbin[YY]; j++)
+ {
+ if (j < miny+iIGNOREOUTER || j > maxy-iIGNOREOUTER)
+ {
+ continue;
+ }
+ for (i = 0; i < nbin[ZZ]; i++)
+ {
+ if (i < minz+iIGNOREOUTER || i > maxz-iIGNOREOUTER)
+ {
+ continue;
+ }
+ tot += bin[k][j][i];
+ if (bin[k][j][i] > max)
+ {
+ max = bin[k][j][i];
+ }
+ if (bin[k][j][i] < min)
+ {
+ min = bin[k][j][i];
+ }
+ }
+ }
+ }
+
+ numcu = (maxx-minx+1-(2*iIGNOREOUTER))*(maxy-miny+1-(2*iIGNOREOUTER))*(maxz-minz+1-(2*iIGNOREOUTER));
+ if (bCALCDIV)
+ {
+ norm = ((double)numcu*(double)numfr) / (double)tot;
+ }
+ else
+ {
+ norm = 1.0;
+ }
+
+ for (k = 0; k < nbin[XX]; k++)
+ {
+ if (k < minx+iIGNOREOUTER || k > maxx-iIGNOREOUTER)
+ {
+ continue;
+ }
+ for (j = 0; j < nbin[YY]; j++)
+ {
+ if (j < miny+iIGNOREOUTER || j > maxy-iIGNOREOUTER)
+ {
+ continue;
+ }
+ for (i = 0; i < nbin[ZZ]; i++)
+ {
+ if (i < minz+iIGNOREOUTER || i > maxz-iIGNOREOUTER)
+ {
+ continue;
+ }
+ fprintf(flp, "%12.6f ", norm*(double)bin[k][j][i]/(double)numfr);
+ }
+ fprintf(flp, "\n");
+ }
+ fprintf(flp, "\n");
+ }
+ ffclose(flp);
+
+ /* printf("x=%d to %d\n",minx,maxx); */
+ /* printf("y=%d to %d\n",miny,maxy); */
+ /* printf("z=%d to %d\n",minz,maxz); */
+
+ if (bCALCDIV)
+ {
+ printf("Counts per frame in all %ld cubes divided by %le\n", numcu, 1.0/norm);
+ printf("Normalized data: average %le, min %le, max %le\n", 1.0, norm*(double)min/(double)numfr, norm*(double)max/(double)numfr);
+ }
+ else
+ {
+ printf("grid.cube contains counts per frame in all %ld cubes\n", numcu);
+ printf("Raw data: average %le, min %le, max %le\n", 1.0/norm, (double)min/(double)numfr, (double)max/(double)numfr);
+ }
+
+ return 0;
+}
--- /dev/null
- bin = (int)(sqrt(d2)*invbin);
+/*
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * VERSION 3.2.0
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2004, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * Green Red Orange Magenta Azure Cyan Skyblue
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <string.h>
+#include <ctype.h>
+#include <math.h>
+
+#include "sysstuff.h"
+#include "smalloc.h"
+#include "macros.h"
+#include "statutil.h"
+#include "maths.h"
+#include "futil.h"
+#include "index.h"
+#include "typedefs.h"
+#include "xvgr.h"
+#include "gstat.h"
+#include "tpxio.h"
+#include "vec.h"
+#include "matio.h"
+#include "gmx_ana.h"
+
+
+int gmx_vanhove(int argc, char *argv[])
+{
+ const char *desc[] = {
+ "[TT]g_vanhove[tt] computes the Van Hove correlation function.",
+ "The Van Hove G(r,t) is the probability that a particle that is at r[SUB]0[sub]",
+ "at time zero can be found at position r[SUB]0[sub]+r at time t.",
+ "[TT]g_vanhove[tt] determines G not for a vector r, but for the length of r.",
+ "Thus it gives the probability that a particle moves a distance of r",
+ "in time t.",
+ "Jumps across the periodic boundaries are removed.",
+ "Corrections are made for scaling due to isotropic",
+ "or anisotropic pressure coupling.",
+ "[PAR]",
+ "With option [TT]-om[tt] the whole matrix can be written as a function",
+ "of t and r or as a function of [SQRT]t[sqrt] and r (option [TT]-sqrt[tt]).",
+ "[PAR]",
+ "With option [TT]-or[tt] the Van Hove function is plotted for one",
+ "or more values of t. Option [TT]-nr[tt] sets the number of times,",
+ "option [TT]-fr[tt] the number spacing between the times.",
+ "The binwidth is set with option [TT]-rbin[tt]. The number of bins",
+ "is determined automatically.",
+ "[PAR]",
+ "With option [TT]-ot[tt] the integral up to a certain distance",
+ "(option [TT]-rt[tt]) is plotted as a function of time.",
+ "[PAR]",
+ "For all frames that are read the coordinates of the selected particles",
+ "are stored in memory. Therefore the program may use a lot of memory.",
+ "For options [TT]-om[tt] and [TT]-ot[tt] the program may be slow.",
+ "This is because the calculation scales as the number of frames times",
+ "[TT]-fm[tt] or [TT]-ft[tt].",
+ "Note that with the [TT]-dt[tt] option the memory usage and calculation",
+ "time can be reduced."
+ };
+ static int fmmax = 0, ftmax = 0, nlev = 81, nr = 1, fshift = 0;
+ static real sbin = 0, rmax = 2, rbin = 0.01, mmax = 0, rint = 0;
+ t_pargs pa[] = {
+ { "-sqrt", FALSE, etREAL, {&sbin},
+ "Use [SQRT]t[sqrt] on the matrix axis which binspacing # in [SQRT]ps[sqrt]" },
+ { "-fm", FALSE, etINT, {&fmmax},
+ "Number of frames in the matrix, 0 is plot all" },
+ { "-rmax", FALSE, etREAL, {&rmax},
+ "Maximum r in the matrix (nm)" },
+ { "-rbin", FALSE, etREAL, {&rbin},
+ "Binwidth in the matrix and for [TT]-or[tt] (nm)" },
+ { "-mmax", FALSE, etREAL, {&mmax},
+ "Maximum density in the matrix, 0 is calculate (1/nm)" },
+ { "-nlevels", FALSE, etINT, {&nlev},
+ "Number of levels in the matrix" },
+ { "-nr", FALSE, etINT, {&nr},
+ "Number of curves for the [TT]-or[tt] output" },
+ { "-fr", FALSE, etINT, {&fshift},
+ "Frame spacing for the [TT]-or[tt] output" },
+ { "-rt", FALSE, etREAL, {&rint},
+ "Integration limit for the [TT]-ot[tt] output (nm)" },
+ { "-ft", FALSE, etINT, {&ftmax},
+ "Number of frames in the [TT]-ot[tt] output, 0 is plot all" }
+ };
+#define NPA asize(pa)
+
+ t_filenm fnm[] = {
+ { efTRX, NULL, NULL, ffREAD },
+ { efTPS, NULL, NULL, ffREAD },
+ { efNDX, NULL, NULL, ffOPTRD },
+ { efXPM, "-om", "vanhove", ffOPTWR },
+ { efXVG, "-or", "vanhove_r", ffOPTWR },
+ { efXVG, "-ot", "vanhove_t", ffOPTWR }
+ };
+#define NFILE asize(fnm)
+
+ output_env_t oenv;
+ const char *matfile, *otfile, *orfile;
+ char title[256];
+ t_topology top;
+ int ePBC;
+ matrix boxtop, box, *sbox, avbox, corr;
+ rvec *xtop, *x, **sx;
+ int isize, nalloc, nallocn, natom;
+ t_trxstatus *status;
+ atom_id *index;
+ char *grpname;
+ int nfr, f, ff, i, m, mat_nx = 0, nbin = 0, bin, mbin, fbin;
+ real *time, t, invbin = 0, rmax2 = 0, rint2 = 0, d2;
+ real invsbin = 0, matmax, normfac, dt, *tickx, *ticky;
+ char buf[STRLEN], **legend;
+ real **mat = NULL;
+ int *pt = NULL, **pr = NULL, *mcount = NULL, *tcount = NULL, *rcount = NULL;
+ FILE *fp;
+ t_rgb rlo = {1, 1, 1}, rhi = {0, 0, 0};
+
+ if (!parse_common_args(&argc, argv, PCA_CAN_VIEW | PCA_CAN_TIME | PCA_BE_NICE,
+ NFILE, fnm, asize(pa), pa, asize(desc), desc, 0, NULL, &oenv))
+ {
+ return 0;
+ }
+
+ matfile = opt2fn_null("-om", NFILE, fnm);
+ if (opt2parg_bSet("-fr", NPA, pa))
+ {
+ orfile = opt2fn("-or", NFILE, fnm);
+ }
+ else
+ {
+ orfile = opt2fn_null("-or", NFILE, fnm);
+ }
+ if (opt2parg_bSet("-rt", NPA, pa))
+ {
+ otfile = opt2fn("-ot", NFILE, fnm);
+ }
+ else
+ {
+ otfile = opt2fn_null("-ot", NFILE, fnm);
+ }
+
+ if (!matfile && !otfile && !orfile)
+ {
+ fprintf(stderr,
+ "For output set one (or more) of the output file options\n");
+ exit(0);
+ }
+
+ read_tps_conf(ftp2fn(efTPS, NFILE, fnm), title, &top, &ePBC, &xtop, NULL, boxtop,
+ FALSE);
+ get_index(&top.atoms, ftp2fn_null(efNDX, NFILE, fnm), 1, &isize, &index, &grpname);
+
+ nalloc = 0;
+ time = NULL;
+ sbox = NULL;
+ sx = NULL;
+ clear_mat(avbox);
+
+ natom = read_first_x(oenv, &status, ftp2fn(efTRX, NFILE, fnm), &t, &x, box);
+ nfr = 0;
+ do
+ {
+ if (nfr >= nalloc)
+ {
+ nalloc += 100;
+ srenew(time, nalloc);
+ srenew(sbox, nalloc);
+ srenew(sx, nalloc);
+ }
+
+ time[nfr] = t;
+ copy_mat(box, sbox[nfr]);
+ /* This assumes that the off-diagonal box elements
+ * are not affected by jumps across the periodic boundaries.
+ */
+ m_add(avbox, box, avbox);
+ snew(sx[nfr], isize);
+ for (i = 0; i < isize; i++)
+ {
+ copy_rvec(x[index[i]], sx[nfr][i]);
+ }
+
+ nfr++;
+ }
+ while (read_next_x(oenv, status, &t, x, box));
+
+ /* clean up */
+ sfree(x);
+ close_trj(status);
+
+ fprintf(stderr, "Read %d frames\n", nfr);
+
+ dt = (time[nfr-1] - time[0])/(nfr - 1);
+ /* Some ugly rounding to get nice nice times in the output */
+ dt = (int)(10000.0*dt + 0.5)/10000.0;
+
+ invbin = 1.0/rbin;
+
+ if (matfile)
+ {
+ if (fmmax <= 0 || fmmax >= nfr)
+ {
+ fmmax = nfr - 1;
+ }
+ snew(mcount, fmmax);
+ nbin = (int)(rmax*invbin + 0.5);
+ if (sbin == 0)
+ {
+ mat_nx = fmmax + 1;
+ }
+ else
+ {
+ invsbin = 1.0/sbin;
+ mat_nx = sqrt(fmmax*dt)*invsbin + 1;
+ }
+ snew(mat, mat_nx);
+ for (f = 0; f < mat_nx; f++)
+ {
+ snew(mat[f], nbin);
+ }
+ rmax2 = sqr(nbin*rbin);
+ /* Initialize time zero */
+ mat[0][0] = nfr*isize;
+ mcount[0] += nfr;
+ }
+ else
+ {
+ fmmax = 0;
+ }
+
+ if (orfile)
+ {
+ snew(pr, nr);
+ nalloc = 0;
+ snew(rcount, nr);
+ }
+
+ if (otfile)
+ {
+ if (ftmax <= 0)
+ {
+ ftmax = nfr - 1;
+ }
+ snew(tcount, ftmax);
+ snew(pt, nfr);
+ rint2 = rint*rint;
+ /* Initialize time zero */
+ pt[0] = nfr*isize;
+ tcount[0] += nfr;
+ }
+ else
+ {
+ ftmax = 0;
+ }
+
+ msmul(avbox, 1.0/nfr, avbox);
+ for (f = 0; f < nfr; f++)
+ {
+ if (f % 100 == 0)
+ {
+ fprintf(stderr, "\rProcessing frame %d", f);
+ }
+ /* Scale all the configuration to the average box */
+ m_inv_ur0(sbox[f], corr);
+ mmul_ur0(avbox, corr, corr);
+ for (i = 0; i < isize; i++)
+ {
+ mvmul_ur0(corr, sx[f][i], sx[f][i]);
+ if (f > 0)
+ {
+ /* Correct for periodic jumps */
+ for (m = DIM-1; m >= 0; m--)
+ {
+ while (sx[f][i][m] - sx[f-1][i][m] > 0.5*avbox[m][m])
+ {
+ rvec_dec(sx[f][i], avbox[m]);
+ }
+ while (sx[f][i][m] - sx[f-1][i][m] <= -0.5*avbox[m][m])
+ {
+ rvec_inc(sx[f][i], avbox[m]);
+ }
+ }
+ }
+ }
+ for (ff = 0; ff < f; ff++)
+ {
+ fbin = f - ff;
+ if (fbin <= fmmax || fbin <= ftmax)
+ {
+ if (sbin == 0)
+ {
+ mbin = fbin;
+ }
+ else
+ {
+ mbin = (int)(sqrt(fbin*dt)*invsbin + 0.5);
+ }
+ for (i = 0; i < isize; i++)
+ {
+ d2 = distance2(sx[f][i], sx[ff][i]);
+ if (mbin < mat_nx && d2 < rmax2)
+ {
+ bin = (int)(sqrt(d2)*invbin + 0.5);
+ if (bin < nbin)
+ {
+ mat[mbin][bin] += 1;
+ }
+ }
+ if (fbin <= ftmax && d2 <= rint2)
+ {
+ pt[fbin]++;
+ }
+ }
+ if (matfile)
+ {
+ mcount[mbin]++;
+ }
+ if (otfile)
+ {
+ tcount[fbin]++;
+ }
+ }
+ }
+ if (orfile)
+ {
+ for (fbin = 0; fbin < nr; fbin++)
+ {
+ ff = f - (fbin + 1)*fshift;
+ if (ff >= 0)
+ {
+ for (i = 0; i < isize; i++)
+ {
+ d2 = distance2(sx[f][i], sx[ff][i]);
++ bin = (int)(sqrt(d2)*invbin + 0.5);
+ if (bin >= nalloc)
+ {
+ nallocn = 10*(bin/10) + 11;
+ for (m = 0; m < nr; m++)
+ {
+ srenew(pr[m], nallocn);
+ for (i = nalloc; i < nallocn; i++)
+ {
+ pr[m][i] = 0;
+ }
+ }
+ nalloc = nallocn;
+ }
+ pr[fbin][bin]++;
+ }
+ rcount[fbin]++;
+ }
+ }
+ }
+ }
+ fprintf(stderr, "\n");
+
+ if (matfile)
+ {
+ matmax = 0;
+ for (f = 0; f < mat_nx; f++)
+ {
+ normfac = 1.0/(mcount[f]*isize*rbin);
+ for (i = 0; i < nbin; i++)
+ {
+ mat[f][i] *= normfac;
+ if (mat[f][i] > matmax && (f != 0 || i != 0))
+ {
+ matmax = mat[f][i];
+ }
+ }
+ }
+ fprintf(stdout, "Value at (0,0): %.3f, maximum of the rest %.3f\n",
+ mat[0][0], matmax);
+ if (mmax > 0)
+ {
+ matmax = mmax;
+ }
+ snew(tickx, mat_nx);
+ for (f = 0; f < mat_nx; f++)
+ {
+ if (sbin == 0)
+ {
+ tickx[f] = f*dt;
+ }
+ else
+ {
+ tickx[f] = f*sbin;
+ }
+ }
+ snew(ticky, nbin+1);
+ for (i = 0; i <= nbin; i++)
+ {
+ ticky[i] = i*rbin;
+ }
+ fp = ffopen(matfile, "w");
+ write_xpm(fp, MAT_SPATIAL_Y, "Van Hove function", "G (1/nm)",
+ sbin == 0 ? "time (ps)" : "sqrt(time) (ps^1/2)", "r (nm)",
+ mat_nx, nbin, tickx, ticky, mat, 0, matmax, rlo, rhi, &nlev);
+ ffclose(fp);
+ }
+
+ if (orfile)
+ {
+ fp = xvgropen(orfile, "Van Hove function", "r (nm)", "G (nm\\S-1\\N)", oenv);
+ fprintf(fp, "@ subtitle \"for particles in group %s\"\n", grpname);
+ snew(legend, nr);
+ for (fbin = 0; fbin < nr; fbin++)
+ {
+ sprintf(buf, "%g ps", (fbin + 1)*fshift*dt);
+ legend[fbin] = strdup(buf);
+ }
+ xvgr_legend(fp, nr, (const char**)legend, oenv);
+ for (i = 0; i < nalloc; i++)
+ {
+ fprintf(fp, "%g", i*rbin);
+ for (fbin = 0; fbin < nr; fbin++)
+ {
+ fprintf(fp, " %g",
+ (real)pr[fbin][i]/(rcount[fbin]*isize*rbin*(i == 0 ? 0.5 : 1)));
+ }
+ fprintf(fp, "\n");
+ }
+ ffclose(fp);
+ }
+
+ if (otfile)
+ {
+ sprintf(buf, "Probability of moving less than %g nm", rint);
+ fp = xvgropen(otfile, buf, "t (ps)", "", oenv);
+ fprintf(fp, "@ subtitle \"for particles in group %s\"\n", grpname);
+ for (f = 0; f <= ftmax; f++)
+ {
+ fprintf(fp, "%g %g\n", f*dt, (real)pt[f]/(tcount[f]*isize));
+ }
+ ffclose(fp);
+ }
+
+ do_view(oenv, matfile, NULL);
+ do_view(oenv, orfile, NULL);
+ do_view(oenv, otfile, NULL);
+
+ return 0;
+}
--- /dev/null
- gmx_mm_pr cos_S, sin_S;
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * VERSION 3.2.0
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2004, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * GROningen Mixture of Alchemy and Childrens' Stories
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <math.h>
+#include <assert.h>
+#include "physics.h"
+#include "vec.h"
+#include "maths.h"
+#include "txtdump.h"
+#include "bondf.h"
+#include "smalloc.h"
+#include "pbc.h"
+#include "ns.h"
+#include "macros.h"
+#include "names.h"
+#include "gmx_fatal.h"
+#include "mshift.h"
+#include "main.h"
+#include "disre.h"
+#include "orires.h"
+#include "force.h"
+#include "nonbonded.h"
+
+/* Include the SIMD macro file and then check for support */
+#include "gmx_simd_macros.h"
+#if defined GMX_HAVE_SIMD_MACROS && defined GMX_SIMD_HAVE_TRIGONOMETRIC
+#define SIMD_BONDEDS
+#include "gmx_simd_vec.h"
+#endif
+
+/* Find a better place for this? */
+const int cmap_coeff_matrix[] = {
+ 1, 0, -3, 2, 0, 0, 0, 0, -3, 0, 9, -6, 2, 0, -6, 4,
+ 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, -9, 6, -2, 0, 6, -4,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, -6, 0, 0, -6, 4,
+ 0, 0, 3, -2, 0, 0, 0, 0, 0, 0, -9, 6, 0, 0, 6, -4,
+ 0, 0, 0, 0, 1, 0, -3, 2, -2, 0, 6, -4, 1, 0, -3, 2,
+ 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 3, -2, 1, 0, -3, 2,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -3, 2, 0, 0, 3, -2,
+ 0, 0, 0, 0, 0, 0, 3, -2, 0, 0, -6, 4, 0, 0, 3, -2,
+ 0, 1, -2, 1, 0, 0, 0, 0, 0, -3, 6, -3, 0, 2, -4, 2,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, -6, 3, 0, -2, 4, -2,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -3, 3, 0, 0, 2, -2,
+ 0, 0, -1, 1, 0, 0, 0, 0, 0, 0, 3, -3, 0, 0, -2, 2,
+ 0, 0, 0, 0, 0, 1, -2, 1, 0, -2, 4, -2, 0, 1, -2, 1,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 2, -1, 0, 1, -2, 1,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, -1, 0, 0, -1, 1,
+ 0, 0, 0, 0, 0, 0, -1, 1, 0, 0, 2, -2, 0, 0, -1, 1
+};
+
+
+
+int glatnr(int *global_atom_index, int i)
+{
+ int atnr;
+
+ if (global_atom_index == NULL)
+ {
+ atnr = i + 1;
+ }
+ else
+ {
+ atnr = global_atom_index[i] + 1;
+ }
+
+ return atnr;
+}
+
+static int pbc_rvec_sub(const t_pbc *pbc, const rvec xi, const rvec xj, rvec dx)
+{
+ if (pbc)
+ {
+ return pbc_dx_aiuc(pbc, xi, xj, dx);
+ }
+ else
+ {
+ rvec_sub(xi, xj, dx);
+ return CENTRAL;
+ }
+}
+
+#ifdef SIMD_BONDEDS
+
+/* SIMD PBC data structure, containing 1/boxdiag and the box vectors */
+typedef struct {
+ gmx_mm_pr inv_bzz;
+ gmx_mm_pr inv_byy;
+ gmx_mm_pr inv_bxx;
+ gmx_mm_pr bzx;
+ gmx_mm_pr bzy;
+ gmx_mm_pr bzz;
+ gmx_mm_pr byx;
+ gmx_mm_pr byy;
+ gmx_mm_pr bxx;
+} pbc_simd_t;
+
+/* Set the SIMD pbc data from a normal t_pbc struct */
+static void set_pbc_simd(const t_pbc *pbc, pbc_simd_t *pbc_simd)
+{
+ rvec inv_bdiag;
+ int d;
+
+ /* Setting inv_bdiag to 0 effectively turns off PBC */
+ clear_rvec(inv_bdiag);
+ if (pbc != NULL)
+ {
+ for (d = 0; d < pbc->ndim_ePBC; d++)
+ {
+ inv_bdiag[d] = 1.0/pbc->box[d][d];
+ }
+ }
+
+ pbc_simd->inv_bzz = gmx_set1_pr(inv_bdiag[ZZ]);
+ pbc_simd->inv_byy = gmx_set1_pr(inv_bdiag[YY]);
+ pbc_simd->inv_bxx = gmx_set1_pr(inv_bdiag[XX]);
+
+ if (pbc != NULL)
+ {
+ pbc_simd->bzx = gmx_set1_pr(pbc->box[ZZ][XX]);
+ pbc_simd->bzy = gmx_set1_pr(pbc->box[ZZ][YY]);
+ pbc_simd->bzz = gmx_set1_pr(pbc->box[ZZ][ZZ]);
+ pbc_simd->byx = gmx_set1_pr(pbc->box[YY][XX]);
+ pbc_simd->byy = gmx_set1_pr(pbc->box[YY][YY]);
+ pbc_simd->bxx = gmx_set1_pr(pbc->box[XX][XX]);
+ }
+ else
+ {
+ pbc_simd->bzx = gmx_setzero_pr();
+ pbc_simd->bzy = gmx_setzero_pr();
+ pbc_simd->bzz = gmx_setzero_pr();
+ pbc_simd->byx = gmx_setzero_pr();
+ pbc_simd->byy = gmx_setzero_pr();
+ pbc_simd->bxx = gmx_setzero_pr();
+ }
+}
+
+/* Correct distance vector *dx,*dy,*dz for PBC using SIMD */
+static gmx_inline void
+pbc_dx_simd(gmx_mm_pr *dx, gmx_mm_pr *dy, gmx_mm_pr *dz,
+ const pbc_simd_t *pbc)
+{
+ gmx_mm_pr sh;
+
+ sh = gmx_round_pr(gmx_mul_pr(*dz, pbc->inv_bzz));
+ *dx = gmx_nmsub_pr(sh, pbc->bzx, *dx);
+ *dy = gmx_nmsub_pr(sh, pbc->bzy, *dy);
+ *dz = gmx_nmsub_pr(sh, pbc->bzz, *dz);
+
+ sh = gmx_round_pr(gmx_mul_pr(*dy, pbc->inv_byy));
+ *dx = gmx_nmsub_pr(sh, pbc->byx, *dx);
+ *dy = gmx_nmsub_pr(sh, pbc->byy, *dy);
+
+ sh = gmx_round_pr(gmx_mul_pr(*dx, pbc->inv_bxx));
+ *dx = gmx_nmsub_pr(sh, pbc->bxx, *dx);
+}
+
+#endif /* SIMD_BONDEDS */
+
+/*
+ * Morse potential bond by Frank Everdij
+ *
+ * Three parameters needed:
+ *
+ * b0 = equilibrium distance in nm
+ * be = beta in nm^-1 (actually, it's nu_e*Sqrt(2*pi*pi*mu/D_e))
+ * cb = well depth in kJ/mol
+ *
+ * Note: the potential is referenced to be +cb at infinite separation
+ * and zero at the equilibrium distance!
+ */
+
+real morse_bonds(int nbonds,
+ const t_iatom forceatoms[], const t_iparams forceparams[],
+ const rvec x[], rvec f[], rvec fshift[],
+ const t_pbc *pbc, const t_graph *g,
+ real lambda, real *dvdlambda,
+ const t_mdatoms gmx_unused *md, t_fcdata gmx_unused *fcd,
+ int gmx_unused *global_atom_index)
+{
+ const real one = 1.0;
+ const real two = 2.0;
+ real dr, dr2, temp, omtemp, cbomtemp, fbond, vbond, fij, vtot;
+ real b0, be, cb, b0A, beA, cbA, b0B, beB, cbB, L1;
+ rvec dx;
+ int i, m, ki, type, ai, aj;
+ ivec dt;
+
+ vtot = 0.0;
+ for (i = 0; (i < nbonds); )
+ {
+ type = forceatoms[i++];
+ ai = forceatoms[i++];
+ aj = forceatoms[i++];
+
+ b0A = forceparams[type].morse.b0A;
+ beA = forceparams[type].morse.betaA;
+ cbA = forceparams[type].morse.cbA;
+
+ b0B = forceparams[type].morse.b0B;
+ beB = forceparams[type].morse.betaB;
+ cbB = forceparams[type].morse.cbB;
+
+ L1 = one-lambda; /* 1 */
+ b0 = L1*b0A + lambda*b0B; /* 3 */
+ be = L1*beA + lambda*beB; /* 3 */
+ cb = L1*cbA + lambda*cbB; /* 3 */
+
+ ki = pbc_rvec_sub(pbc, x[ai], x[aj], dx); /* 3 */
+ dr2 = iprod(dx, dx); /* 5 */
+ dr = dr2*gmx_invsqrt(dr2); /* 10 */
+ temp = exp(-be*(dr-b0)); /* 12 */
+
+ if (temp == one)
+ {
+ /* bonds are constrainted. This may _not_ include bond constraints if they are lambda dependent */
+ *dvdlambda += cbB-cbA;
+ continue;
+ }
+
+ omtemp = one-temp; /* 1 */
+ cbomtemp = cb*omtemp; /* 1 */
+ vbond = cbomtemp*omtemp; /* 1 */
+ fbond = -two*be*temp*cbomtemp*gmx_invsqrt(dr2); /* 9 */
+ vtot += vbond; /* 1 */
+
+ *dvdlambda += (cbB - cbA) * omtemp * omtemp - (2-2*omtemp)*omtemp * cb * ((b0B-b0A)*be - (beB-beA)*(dr-b0)); /* 15 */
+
+ if (g)
+ {
+ ivec_sub(SHIFT_IVEC(g, ai), SHIFT_IVEC(g, aj), dt);
+ ki = IVEC2IS(dt);
+ }
+
+ for (m = 0; (m < DIM); m++) /* 15 */
+ {
+ fij = fbond*dx[m];
+ f[ai][m] += fij;
+ f[aj][m] -= fij;
+ fshift[ki][m] += fij;
+ fshift[CENTRAL][m] -= fij;
+ }
+ } /* 83 TOTAL */
+ return vtot;
+}
+
+real cubic_bonds(int nbonds,
+ const t_iatom forceatoms[], const t_iparams forceparams[],
+ const rvec x[], rvec f[], rvec fshift[],
+ const t_pbc *pbc, const t_graph *g,
+ real gmx_unused lambda, real gmx_unused *dvdlambda,
+ const t_mdatoms gmx_unused *md, t_fcdata gmx_unused *fcd,
+ int gmx_unused *global_atom_index)
+{
+ const real three = 3.0;
+ const real two = 2.0;
+ real kb, b0, kcub;
+ real dr, dr2, dist, kdist, kdist2, fbond, vbond, fij, vtot;
+ rvec dx;
+ int i, m, ki, type, ai, aj;
+ ivec dt;
+
+ vtot = 0.0;
+ for (i = 0; (i < nbonds); )
+ {
+ type = forceatoms[i++];
+ ai = forceatoms[i++];
+ aj = forceatoms[i++];
+
+ b0 = forceparams[type].cubic.b0;
+ kb = forceparams[type].cubic.kb;
+ kcub = forceparams[type].cubic.kcub;
+
+ ki = pbc_rvec_sub(pbc, x[ai], x[aj], dx); /* 3 */
+ dr2 = iprod(dx, dx); /* 5 */
+
+ if (dr2 == 0.0)
+ {
+ continue;
+ }
+
+ dr = dr2*gmx_invsqrt(dr2); /* 10 */
+ dist = dr-b0;
+ kdist = kb*dist;
+ kdist2 = kdist*dist;
+
+ vbond = kdist2 + kcub*kdist2*dist;
+ fbond = -(two*kdist + three*kdist2*kcub)/dr;
+
+ vtot += vbond; /* 21 */
+
+ if (g)
+ {
+ ivec_sub(SHIFT_IVEC(g, ai), SHIFT_IVEC(g, aj), dt);
+ ki = IVEC2IS(dt);
+ }
+ for (m = 0; (m < DIM); m++) /* 15 */
+ {
+ fij = fbond*dx[m];
+ f[ai][m] += fij;
+ f[aj][m] -= fij;
+ fshift[ki][m] += fij;
+ fshift[CENTRAL][m] -= fij;
+ }
+ } /* 54 TOTAL */
+ return vtot;
+}
+
+real FENE_bonds(int nbonds,
+ const t_iatom forceatoms[], const t_iparams forceparams[],
+ const rvec x[], rvec f[], rvec fshift[],
+ const t_pbc *pbc, const t_graph *g,
+ real gmx_unused lambda, real gmx_unused *dvdlambda,
+ const t_mdatoms gmx_unused *md, t_fcdata gmx_unused *fcd,
+ int *global_atom_index)
+{
+ const real half = 0.5;
+ const real one = 1.0;
+ real bm, kb;
+ real dr, dr2, bm2, omdr2obm2, fbond, vbond, fij, vtot;
+ rvec dx;
+ int i, m, ki, type, ai, aj;
+ ivec dt;
+
+ vtot = 0.0;
+ for (i = 0; (i < nbonds); )
+ {
+ type = forceatoms[i++];
+ ai = forceatoms[i++];
+ aj = forceatoms[i++];
+
+ bm = forceparams[type].fene.bm;
+ kb = forceparams[type].fene.kb;
+
+ ki = pbc_rvec_sub(pbc, x[ai], x[aj], dx); /* 3 */
+ dr2 = iprod(dx, dx); /* 5 */
+
+ if (dr2 == 0.0)
+ {
+ continue;
+ }
+
+ bm2 = bm*bm;
+
+ if (dr2 >= bm2)
+ {
+ gmx_fatal(FARGS,
+ "r^2 (%f) >= bm^2 (%f) in FENE bond between atoms %d and %d",
+ dr2, bm2,
+ glatnr(global_atom_index, ai),
+ glatnr(global_atom_index, aj));
+ }
+
+ omdr2obm2 = one - dr2/bm2;
+
+ vbond = -half*kb*bm2*log(omdr2obm2);
+ fbond = -kb/omdr2obm2;
+
+ vtot += vbond; /* 35 */
+
+ if (g)
+ {
+ ivec_sub(SHIFT_IVEC(g, ai), SHIFT_IVEC(g, aj), dt);
+ ki = IVEC2IS(dt);
+ }
+ for (m = 0; (m < DIM); m++) /* 15 */
+ {
+ fij = fbond*dx[m];
+ f[ai][m] += fij;
+ f[aj][m] -= fij;
+ fshift[ki][m] += fij;
+ fshift[CENTRAL][m] -= fij;
+ }
+ } /* 58 TOTAL */
+ return vtot;
+}
+
+real harmonic(real kA, real kB, real xA, real xB, real x, real lambda,
+ real *V, real *F)
+{
+ const real half = 0.5;
+ real L1, kk, x0, dx, dx2;
+ real v, f, dvdlambda;
+
+ L1 = 1.0-lambda;
+ kk = L1*kA+lambda*kB;
+ x0 = L1*xA+lambda*xB;
+
+ dx = x-x0;
+ dx2 = dx*dx;
+
+ f = -kk*dx;
+ v = half*kk*dx2;
+ dvdlambda = half*(kB-kA)*dx2 + (xA-xB)*kk*dx;
+
+ *F = f;
+ *V = v;
+
+ return dvdlambda;
+
+ /* That was 19 flops */
+}
+
+
+real bonds(int nbonds,
+ const t_iatom forceatoms[], const t_iparams forceparams[],
+ const rvec x[], rvec f[], rvec fshift[],
+ const t_pbc *pbc, const t_graph *g,
+ real lambda, real *dvdlambda,
+ const t_mdatoms gmx_unused *md, t_fcdata gmx_unused *fcd,
+ int gmx_unused *global_atom_index)
+{
+ int i, m, ki, ai, aj, type;
+ real dr, dr2, fbond, vbond, fij, vtot;
+ rvec dx;
+ ivec dt;
+
+ vtot = 0.0;
+ for (i = 0; (i < nbonds); )
+ {
+ type = forceatoms[i++];
+ ai = forceatoms[i++];
+ aj = forceatoms[i++];
+
+ ki = pbc_rvec_sub(pbc, x[ai], x[aj], dx); /* 3 */
+ dr2 = iprod(dx, dx); /* 5 */
+ dr = dr2*gmx_invsqrt(dr2); /* 10 */
+
+ *dvdlambda += harmonic(forceparams[type].harmonic.krA,
+ forceparams[type].harmonic.krB,
+ forceparams[type].harmonic.rA,
+ forceparams[type].harmonic.rB,
+ dr, lambda, &vbond, &fbond); /* 19 */
+
+ if (dr2 == 0.0)
+ {
+ continue;
+ }
+
+
+ vtot += vbond; /* 1*/
+ fbond *= gmx_invsqrt(dr2); /* 6 */
+#ifdef DEBUG
+ if (debug)
+ {
+ fprintf(debug, "BONDS: dr = %10g vbond = %10g fbond = %10g\n",
+ dr, vbond, fbond);
+ }
+#endif
+ if (g)
+ {
+ ivec_sub(SHIFT_IVEC(g, ai), SHIFT_IVEC(g, aj), dt);
+ ki = IVEC2IS(dt);
+ }
+ for (m = 0; (m < DIM); m++) /* 15 */
+ {
+ fij = fbond*dx[m];
+ f[ai][m] += fij;
+ f[aj][m] -= fij;
+ fshift[ki][m] += fij;
+ fshift[CENTRAL][m] -= fij;
+ }
+ } /* 59 TOTAL */
+ return vtot;
+}
+
+real restraint_bonds(int nbonds,
+ const t_iatom forceatoms[], const t_iparams forceparams[],
+ const rvec x[], rvec f[], rvec fshift[],
+ const t_pbc *pbc, const t_graph *g,
+ real lambda, real *dvdlambda,
+ const t_mdatoms gmx_unused *md, t_fcdata gmx_unused *fcd,
+ int gmx_unused *global_atom_index)
+{
+ int i, m, ki, ai, aj, type;
+ real dr, dr2, fbond, vbond, fij, vtot;
+ real L1;
+ real low, dlow, up1, dup1, up2, dup2, k, dk;
+ real drh, drh2;
+ rvec dx;
+ ivec dt;
+
+ L1 = 1.0 - lambda;
+
+ vtot = 0.0;
+ for (i = 0; (i < nbonds); )
+ {
+ type = forceatoms[i++];
+ ai = forceatoms[i++];
+ aj = forceatoms[i++];
+
+ ki = pbc_rvec_sub(pbc, x[ai], x[aj], dx); /* 3 */
+ dr2 = iprod(dx, dx); /* 5 */
+ dr = dr2*gmx_invsqrt(dr2); /* 10 */
+
+ low = L1*forceparams[type].restraint.lowA + lambda*forceparams[type].restraint.lowB;
+ dlow = -forceparams[type].restraint.lowA + forceparams[type].restraint.lowB;
+ up1 = L1*forceparams[type].restraint.up1A + lambda*forceparams[type].restraint.up1B;
+ dup1 = -forceparams[type].restraint.up1A + forceparams[type].restraint.up1B;
+ up2 = L1*forceparams[type].restraint.up2A + lambda*forceparams[type].restraint.up2B;
+ dup2 = -forceparams[type].restraint.up2A + forceparams[type].restraint.up2B;
+ k = L1*forceparams[type].restraint.kA + lambda*forceparams[type].restraint.kB;
+ dk = -forceparams[type].restraint.kA + forceparams[type].restraint.kB;
+ /* 24 */
+
+ if (dr < low)
+ {
+ drh = dr - low;
+ drh2 = drh*drh;
+ vbond = 0.5*k*drh2;
+ fbond = -k*drh;
+ *dvdlambda += 0.5*dk*drh2 - k*dlow*drh;
+ } /* 11 */
+ else if (dr <= up1)
+ {
+ vbond = 0;
+ fbond = 0;
+ }
+ else if (dr <= up2)
+ {
+ drh = dr - up1;
+ drh2 = drh*drh;
+ vbond = 0.5*k*drh2;
+ fbond = -k*drh;
+ *dvdlambda += 0.5*dk*drh2 - k*dup1*drh;
+ } /* 11 */
+ else
+ {
+ drh = dr - up2;
+ vbond = k*(up2 - up1)*(0.5*(up2 - up1) + drh);
+ fbond = -k*(up2 - up1);
+ *dvdlambda += dk*(up2 - up1)*(0.5*(up2 - up1) + drh)
+ + k*(dup2 - dup1)*(up2 - up1 + drh)
+ - k*(up2 - up1)*dup2;
+ }
+
+ if (dr2 == 0.0)
+ {
+ continue;
+ }
+
+ vtot += vbond; /* 1*/
+ fbond *= gmx_invsqrt(dr2); /* 6 */
+#ifdef DEBUG
+ if (debug)
+ {
+ fprintf(debug, "BONDS: dr = %10g vbond = %10g fbond = %10g\n",
+ dr, vbond, fbond);
+ }
+#endif
+ if (g)
+ {
+ ivec_sub(SHIFT_IVEC(g, ai), SHIFT_IVEC(g, aj), dt);
+ ki = IVEC2IS(dt);
+ }
+ for (m = 0; (m < DIM); m++) /* 15 */
+ {
+ fij = fbond*dx[m];
+ f[ai][m] += fij;
+ f[aj][m] -= fij;
+ fshift[ki][m] += fij;
+ fshift[CENTRAL][m] -= fij;
+ }
+ } /* 59 TOTAL */
+
+ return vtot;
+}
+
+real polarize(int nbonds,
+ const t_iatom forceatoms[], const t_iparams forceparams[],
+ const rvec x[], rvec f[], rvec fshift[],
+ const t_pbc *pbc, const t_graph *g,
+ real lambda, real *dvdlambda,
+ const t_mdatoms *md, t_fcdata gmx_unused *fcd,
+ int gmx_unused *global_atom_index)
+{
+ int i, m, ki, ai, aj, type;
+ real dr, dr2, fbond, vbond, fij, vtot, ksh;
+ rvec dx;
+ ivec dt;
+
+ vtot = 0.0;
+ for (i = 0; (i < nbonds); )
+ {
+ type = forceatoms[i++];
+ ai = forceatoms[i++];
+ aj = forceatoms[i++];
+ ksh = sqr(md->chargeA[aj])*ONE_4PI_EPS0/forceparams[type].polarize.alpha;
+ if (debug)
+ {
+ fprintf(debug, "POL: local ai = %d aj = %d ksh = %.3f\n", ai, aj, ksh);
+ }
+
+ ki = pbc_rvec_sub(pbc, x[ai], x[aj], dx); /* 3 */
+ dr2 = iprod(dx, dx); /* 5 */
+ dr = dr2*gmx_invsqrt(dr2); /* 10 */
+
+ *dvdlambda += harmonic(ksh, ksh, 0, 0, dr, lambda, &vbond, &fbond); /* 19 */
+
+ if (dr2 == 0.0)
+ {
+ continue;
+ }
+
+ vtot += vbond; /* 1*/
+ fbond *= gmx_invsqrt(dr2); /* 6 */
+
+ if (g)
+ {
+ ivec_sub(SHIFT_IVEC(g, ai), SHIFT_IVEC(g, aj), dt);
+ ki = IVEC2IS(dt);
+ }
+ for (m = 0; (m < DIM); m++) /* 15 */
+ {
+ fij = fbond*dx[m];
+ f[ai][m] += fij;
+ f[aj][m] -= fij;
+ fshift[ki][m] += fij;
+ fshift[CENTRAL][m] -= fij;
+ }
+ } /* 59 TOTAL */
+ return vtot;
+}
+
+real anharm_polarize(int nbonds,
+ const t_iatom forceatoms[], const t_iparams forceparams[],
+ const rvec x[], rvec f[], rvec fshift[],
+ const t_pbc *pbc, const t_graph *g,
+ real lambda, real *dvdlambda,
+ const t_mdatoms *md, t_fcdata gmx_unused *fcd,
+ int gmx_unused *global_atom_index)
+{
+ int i, m, ki, ai, aj, type;
+ real dr, dr2, fbond, vbond, fij, vtot, ksh, khyp, drcut, ddr, ddr3;
+ rvec dx;
+ ivec dt;
+
+ vtot = 0.0;
+ for (i = 0; (i < nbonds); )
+ {
+ type = forceatoms[i++];
+ ai = forceatoms[i++];
+ aj = forceatoms[i++];
+ ksh = sqr(md->chargeA[aj])*ONE_4PI_EPS0/forceparams[type].anharm_polarize.alpha; /* 7*/
+ khyp = forceparams[type].anharm_polarize.khyp;
+ drcut = forceparams[type].anharm_polarize.drcut;
+ if (debug)
+ {
+ fprintf(debug, "POL: local ai = %d aj = %d ksh = %.3f\n", ai, aj, ksh);
+ }
+
+ ki = pbc_rvec_sub(pbc, x[ai], x[aj], dx); /* 3 */
+ dr2 = iprod(dx, dx); /* 5 */
+ dr = dr2*gmx_invsqrt(dr2); /* 10 */
+
+ *dvdlambda += harmonic(ksh, ksh, 0, 0, dr, lambda, &vbond, &fbond); /* 19 */
+
+ if (dr2 == 0.0)
+ {
+ continue;
+ }
+
+ if (dr > drcut)
+ {
+ ddr = dr-drcut;
+ ddr3 = ddr*ddr*ddr;
+ vbond += khyp*ddr*ddr3;
+ fbond -= 4*khyp*ddr3;
+ }
+ fbond *= gmx_invsqrt(dr2); /* 6 */
+ vtot += vbond; /* 1*/
+
+ if (g)
+ {
+ ivec_sub(SHIFT_IVEC(g, ai), SHIFT_IVEC(g, aj), dt);
+ ki = IVEC2IS(dt);
+ }
+ for (m = 0; (m < DIM); m++) /* 15 */
+ {
+ fij = fbond*dx[m];
+ f[ai][m] += fij;
+ f[aj][m] -= fij;
+ fshift[ki][m] += fij;
+ fshift[CENTRAL][m] -= fij;
+ }
+ } /* 72 TOTAL */
+ return vtot;
+}
+
+real water_pol(int nbonds,
+ const t_iatom forceatoms[], const t_iparams forceparams[],
+ const rvec x[], rvec f[], rvec gmx_unused fshift[],
+ const t_pbc gmx_unused *pbc, const t_graph gmx_unused *g,
+ real gmx_unused lambda, real gmx_unused *dvdlambda,
+ const t_mdatoms gmx_unused *md, t_fcdata gmx_unused *fcd,
+ int gmx_unused *global_atom_index)
+{
+ /* This routine implements anisotropic polarizibility for water, through
+ * a shell connected to a dummy with spring constant that differ in the
+ * three spatial dimensions in the molecular frame.
+ */
+ int i, m, aO, aH1, aH2, aD, aS, type, type0;
+ rvec dOH1, dOH2, dHH, dOD, dDS, nW, kk, dx, kdx, proj;
+#ifdef DEBUG
+ rvec df;
+#endif
+ real vtot, fij, r_HH, r_OD, r_nW, tx, ty, tz, qS;
+
+ vtot = 0.0;
+ if (nbonds > 0)
+ {
+ type0 = forceatoms[0];
+ aS = forceatoms[5];
+ qS = md->chargeA[aS];
+ kk[XX] = sqr(qS)*ONE_4PI_EPS0/forceparams[type0].wpol.al_x;
+ kk[YY] = sqr(qS)*ONE_4PI_EPS0/forceparams[type0].wpol.al_y;
+ kk[ZZ] = sqr(qS)*ONE_4PI_EPS0/forceparams[type0].wpol.al_z;
+ r_HH = 1.0/forceparams[type0].wpol.rHH;
+ r_OD = 1.0/forceparams[type0].wpol.rOD;
+ if (debug)
+ {
+ fprintf(debug, "WPOL: qS = %10.5f aS = %5d\n", qS, aS);
+ fprintf(debug, "WPOL: kk = %10.3f %10.3f %10.3f\n",
+ kk[XX], kk[YY], kk[ZZ]);
+ fprintf(debug, "WPOL: rOH = %10.3f rHH = %10.3f rOD = %10.3f\n",
+ forceparams[type0].wpol.rOH,
+ forceparams[type0].wpol.rHH,
+ forceparams[type0].wpol.rOD);
+ }
+ for (i = 0; (i < nbonds); i += 6)
+ {
+ type = forceatoms[i];
+ if (type != type0)
+ {
+ gmx_fatal(FARGS, "Sorry, type = %d, type0 = %d, file = %s, line = %d",
+ type, type0, __FILE__, __LINE__);
+ }
+ aO = forceatoms[i+1];
+ aH1 = forceatoms[i+2];
+ aH2 = forceatoms[i+3];
+ aD = forceatoms[i+4];
+ aS = forceatoms[i+5];
+
+ /* Compute vectors describing the water frame */
+ rvec_sub(x[aH1], x[aO], dOH1);
+ rvec_sub(x[aH2], x[aO], dOH2);
+ rvec_sub(x[aH2], x[aH1], dHH);
+ rvec_sub(x[aD], x[aO], dOD);
+ rvec_sub(x[aS], x[aD], dDS);
+ cprod(dOH1, dOH2, nW);
+
+ /* Compute inverse length of normal vector
+ * (this one could be precomputed, but I'm too lazy now)
+ */
+ r_nW = gmx_invsqrt(iprod(nW, nW));
+ /* This is for precision, but does not make a big difference,
+ * it can go later.
+ */
+ r_OD = gmx_invsqrt(iprod(dOD, dOD));
+
+ /* Normalize the vectors in the water frame */
+ svmul(r_nW, nW, nW);
+ svmul(r_HH, dHH, dHH);
+ svmul(r_OD, dOD, dOD);
+
+ /* Compute displacement of shell along components of the vector */
+ dx[ZZ] = iprod(dDS, dOD);
+ /* Compute projection on the XY plane: dDS - dx[ZZ]*dOD */
+ for (m = 0; (m < DIM); m++)
+ {
+ proj[m] = dDS[m]-dx[ZZ]*dOD[m];
+ }
+
+ /*dx[XX] = iprod(dDS,nW);
+ dx[YY] = iprod(dDS,dHH);*/
+ dx[XX] = iprod(proj, nW);
+ for (m = 0; (m < DIM); m++)
+ {
+ proj[m] -= dx[XX]*nW[m];
+ }
+ dx[YY] = iprod(proj, dHH);
+ /*#define DEBUG*/
+#ifdef DEBUG
+ if (debug)
+ {
+ fprintf(debug, "WPOL: dx2=%10g dy2=%10g dz2=%10g sum=%10g dDS^2=%10g\n",
+ sqr(dx[XX]), sqr(dx[YY]), sqr(dx[ZZ]), iprod(dx, dx), iprod(dDS, dDS));
+ fprintf(debug, "WPOL: dHH=(%10g,%10g,%10g)\n", dHH[XX], dHH[YY], dHH[ZZ]);
+ fprintf(debug, "WPOL: dOD=(%10g,%10g,%10g), 1/r_OD = %10g\n",
+ dOD[XX], dOD[YY], dOD[ZZ], 1/r_OD);
+ fprintf(debug, "WPOL: nW =(%10g,%10g,%10g), 1/r_nW = %10g\n",
+ nW[XX], nW[YY], nW[ZZ], 1/r_nW);
+ fprintf(debug, "WPOL: dx =%10g, dy =%10g, dz =%10g\n",
+ dx[XX], dx[YY], dx[ZZ]);
+ fprintf(debug, "WPOL: dDSx=%10g, dDSy=%10g, dDSz=%10g\n",
+ dDS[XX], dDS[YY], dDS[ZZ]);
+ }
+#endif
+ /* Now compute the forces and energy */
+ kdx[XX] = kk[XX]*dx[XX];
+ kdx[YY] = kk[YY]*dx[YY];
+ kdx[ZZ] = kk[ZZ]*dx[ZZ];
+ vtot += iprod(dx, kdx);
+ for (m = 0; (m < DIM); m++)
+ {
+ /* This is a tensor operation but written out for speed */
+ tx = nW[m]*kdx[XX];
+ ty = dHH[m]*kdx[YY];
+ tz = dOD[m]*kdx[ZZ];
+ fij = -tx-ty-tz;
+#ifdef DEBUG
+ df[m] = fij;
+#endif
+ f[aS][m] += fij;
+ f[aD][m] -= fij;
+ }
+#ifdef DEBUG
+ if (debug)
+ {
+ fprintf(debug, "WPOL: vwpol=%g\n", 0.5*iprod(dx, kdx));
+ fprintf(debug, "WPOL: df = (%10g, %10g, %10g)\n", df[XX], df[YY], df[ZZ]);
+ }
+#endif
+ }
+ }
+ return 0.5*vtot;
+}
+
+static real do_1_thole(const rvec xi, const rvec xj, rvec fi, rvec fj,
+ const t_pbc *pbc, real qq,
+ rvec fshift[], real afac)
+{
+ rvec r12;
+ real r12sq, r12_1, r12n, r12bar, v0, v1, fscal, ebar, fff;
+ int m, t;
+
+ t = pbc_rvec_sub(pbc, xi, xj, r12); /* 3 */
+
+ r12sq = iprod(r12, r12); /* 5 */
+ r12_1 = gmx_invsqrt(r12sq); /* 5 */
+ r12bar = afac/r12_1; /* 5 */
+ v0 = qq*ONE_4PI_EPS0*r12_1; /* 2 */
+ ebar = exp(-r12bar); /* 5 */
+ v1 = (1-(1+0.5*r12bar)*ebar); /* 4 */
+ fscal = ((v0*r12_1)*v1 - v0*0.5*afac*ebar*(r12bar+1))*r12_1; /* 9 */
+ if (debug)
+ {
+ fprintf(debug, "THOLE: v0 = %.3f v1 = %.3f r12= % .3f r12bar = %.3f fscal = %.3f ebar = %.3f\n", v0, v1, 1/r12_1, r12bar, fscal, ebar);
+ }
+
+ for (m = 0; (m < DIM); m++)
+ {
+ fff = fscal*r12[m];
+ fi[m] += fff;
+ fj[m] -= fff;
+ fshift[t][m] += fff;
+ fshift[CENTRAL][m] -= fff;
+ } /* 15 */
+
+ return v0*v1; /* 1 */
+ /* 54 */
+}
+
+real thole_pol(int nbonds,
+ const t_iatom forceatoms[], const t_iparams forceparams[],
+ const rvec x[], rvec f[], rvec fshift[],
+ const t_pbc *pbc, const t_graph gmx_unused *g,
+ real gmx_unused lambda, real gmx_unused *dvdlambda,
+ const t_mdatoms *md, t_fcdata gmx_unused *fcd,
+ int gmx_unused *global_atom_index)
+{
+ /* Interaction between two pairs of particles with opposite charge */
+ int i, type, a1, da1, a2, da2;
+ real q1, q2, qq, a, al1, al2, afac;
+ real V = 0;
+
+ for (i = 0; (i < nbonds); )
+ {
+ type = forceatoms[i++];
+ a1 = forceatoms[i++];
+ da1 = forceatoms[i++];
+ a2 = forceatoms[i++];
+ da2 = forceatoms[i++];
+ q1 = md->chargeA[da1];
+ q2 = md->chargeA[da2];
+ a = forceparams[type].thole.a;
+ al1 = forceparams[type].thole.alpha1;
+ al2 = forceparams[type].thole.alpha2;
+ qq = q1*q2;
+ afac = a*pow(al1*al2, -1.0/6.0);
+ V += do_1_thole(x[a1], x[a2], f[a1], f[a2], pbc, qq, fshift, afac);
+ V += do_1_thole(x[da1], x[a2], f[da1], f[a2], pbc, -qq, fshift, afac);
+ V += do_1_thole(x[a1], x[da2], f[a1], f[da2], pbc, -qq, fshift, afac);
+ V += do_1_thole(x[da1], x[da2], f[da1], f[da2], pbc, qq, fshift, afac);
+ }
+ /* 290 flops */
+ return V;
+}
+
+real bond_angle(const rvec xi, const rvec xj, const rvec xk, const t_pbc *pbc,
+ rvec r_ij, rvec r_kj, real *costh,
+ int *t1, int *t2)
+/* Return value is the angle between the bonds i-j and j-k */
+{
+ /* 41 FLOPS */
+ real th;
+
+ *t1 = pbc_rvec_sub(pbc, xi, xj, r_ij); /* 3 */
+ *t2 = pbc_rvec_sub(pbc, xk, xj, r_kj); /* 3 */
+
+ *costh = cos_angle(r_ij, r_kj); /* 25 */
+ th = acos(*costh); /* 10 */
+ /* 41 TOTAL */
+ return th;
+}
+
+real angles(int nbonds,
+ const t_iatom forceatoms[], const t_iparams forceparams[],
+ const rvec x[], rvec f[], rvec fshift[],
+ const t_pbc *pbc, const t_graph *g,
+ real lambda, real *dvdlambda,
+ const t_mdatoms gmx_unused *md, t_fcdata gmx_unused *fcd,
+ int gmx_unused *global_atom_index)
+{
+ int i, ai, aj, ak, t1, t2, type;
+ rvec r_ij, r_kj;
+ real cos_theta, cos_theta2, theta, dVdt, va, vtot;
+ ivec jt, dt_ij, dt_kj;
+
+ vtot = 0.0;
+ for (i = 0; i < nbonds; )
+ {
+ type = forceatoms[i++];
+ ai = forceatoms[i++];
+ aj = forceatoms[i++];
+ ak = forceatoms[i++];
+
+ theta = bond_angle(x[ai], x[aj], x[ak], pbc,
+ r_ij, r_kj, &cos_theta, &t1, &t2); /* 41 */
+
+ *dvdlambda += harmonic(forceparams[type].harmonic.krA,
+ forceparams[type].harmonic.krB,
+ forceparams[type].harmonic.rA*DEG2RAD,
+ forceparams[type].harmonic.rB*DEG2RAD,
+ theta, lambda, &va, &dVdt); /* 21 */
+ vtot += va;
+
+ cos_theta2 = sqr(cos_theta);
+ if (cos_theta2 < 1)
+ {
+ int m;
+ real st, sth;
+ real cik, cii, ckk;
+ real nrkj2, nrij2;
+ real nrkj_1, nrij_1;
+ rvec f_i, f_j, f_k;
+
+ st = dVdt*gmx_invsqrt(1 - cos_theta2); /* 12 */
+ sth = st*cos_theta; /* 1 */
+#ifdef DEBUG
+ if (debug)
+ {
+ fprintf(debug, "ANGLES: theta = %10g vth = %10g dV/dtheta = %10g\n",
+ theta*RAD2DEG, va, dVdt);
+ }
+#endif
+ nrij2 = iprod(r_ij, r_ij); /* 5 */
+ nrkj2 = iprod(r_kj, r_kj); /* 5 */
+
+ nrij_1 = gmx_invsqrt(nrij2); /* 10 */
+ nrkj_1 = gmx_invsqrt(nrkj2); /* 10 */
+
+ cik = st*nrij_1*nrkj_1; /* 2 */
+ cii = sth*nrij_1*nrij_1; /* 2 */
+ ckk = sth*nrkj_1*nrkj_1; /* 2 */
+
+ for (m = 0; m < DIM; m++)
+ { /* 39 */
+ f_i[m] = -(cik*r_kj[m] - cii*r_ij[m]);
+ f_k[m] = -(cik*r_ij[m] - ckk*r_kj[m]);
+ f_j[m] = -f_i[m] - f_k[m];
+ f[ai][m] += f_i[m];
+ f[aj][m] += f_j[m];
+ f[ak][m] += f_k[m];
+ }
+ if (g != NULL)
+ {
+ copy_ivec(SHIFT_IVEC(g, aj), jt);
+
+ ivec_sub(SHIFT_IVEC(g, ai), jt, dt_ij);
+ ivec_sub(SHIFT_IVEC(g, ak), jt, dt_kj);
+ t1 = IVEC2IS(dt_ij);
+ t2 = IVEC2IS(dt_kj);
+ }
+ rvec_inc(fshift[t1], f_i);
+ rvec_inc(fshift[CENTRAL], f_j);
+ rvec_inc(fshift[t2], f_k);
+ } /* 161 TOTAL */
+ }
+
+ return vtot;
+}
+
+#ifdef SIMD_BONDEDS
+
+/* As angles, but using SIMD to calculate many dihedrals at once.
+ * This routines does not calculate energies and shift forces.
+ */
+static gmx_inline void
+angles_noener_simd(int nbonds,
+ const t_iatom forceatoms[], const t_iparams forceparams[],
+ const rvec x[], rvec f[],
+ const t_pbc *pbc, const t_graph gmx_unused *g,
+ real gmx_unused lambda,
+ const t_mdatoms gmx_unused *md, t_fcdata gmx_unused *fcd,
+ int gmx_unused *global_atom_index)
+{
+#define UNROLL GMX_SIMD_WIDTH_HERE
+ const int nfa1 = 4;
+ int i, iu, s, m;
+ int type, ai[UNROLL], aj[UNROLL], ak[UNROLL];
+ real coeff_array[2*UNROLL+UNROLL], *coeff;
+ real dr_array[2*DIM*UNROLL+UNROLL], *dr;
+ real f_buf_array[6*UNROLL+UNROLL], *f_buf;
+ gmx_mm_pr k_S, theta0_S;
+ gmx_mm_pr rijx_S, rijy_S, rijz_S;
+ gmx_mm_pr rkjx_S, rkjy_S, rkjz_S;
+ gmx_mm_pr one_S;
++ gmx_mm_pr min_one_plus_eps_S;
+ gmx_mm_pr rij_rkj_S;
+ gmx_mm_pr nrij2_S, nrij_1_S;
+ gmx_mm_pr nrkj2_S, nrkj_1_S;
- sin_S = gmx_invsqrt_pr(gmx_max_pr(gmx_sub_pr(one_S, gmx_mul_pr(cos_S, cos_S)),
- gmx_setzero_pr()));
++ gmx_mm_pr cos_S, invsin_S;
+ gmx_mm_pr theta_S;
+ gmx_mm_pr st_S, sth_S;
+ gmx_mm_pr cik_S, cii_S, ckk_S;
+ gmx_mm_pr f_ix_S, f_iy_S, f_iz_S;
+ gmx_mm_pr f_kx_S, f_ky_S, f_kz_S;
+ pbc_simd_t pbc_simd;
+
+ /* Ensure register memory alignment */
+ coeff = gmx_simd_align_real(coeff_array);
+ dr = gmx_simd_align_real(dr_array);
+ f_buf = gmx_simd_align_real(f_buf_array);
+
+ set_pbc_simd(pbc, &pbc_simd);
+
+ one_S = gmx_set1_pr(1.0);
+
++ /* The smallest number > -1 */
++ min_one_plus_eps_S = gmx_set1_pr(-1.0 + 2*GMX_REAL_EPS);
++
+ /* nbonds is the number of angles times nfa1, here we step UNROLL angles */
+ for (i = 0; (i < nbonds); i += UNROLL*nfa1)
+ {
+ /* Collect atoms for UNROLL angles.
+ * iu indexes into forceatoms, we should not let iu go beyond nbonds.
+ */
+ iu = i;
+ for (s = 0; s < UNROLL; s++)
+ {
+ type = forceatoms[iu];
+ ai[s] = forceatoms[iu+1];
+ aj[s] = forceatoms[iu+2];
+ ak[s] = forceatoms[iu+3];
+
+ coeff[s] = forceparams[type].harmonic.krA;
+ coeff[UNROLL+s] = forceparams[type].harmonic.rA*DEG2RAD;
+
+ /* If you can't use pbc_dx_simd below for PBC, e.g. because
+ * you can't round in SIMD, use pbc_rvec_sub here.
+ */
+ /* Store the non PBC corrected distances packed and aligned */
+ for (m = 0; m < DIM; m++)
+ {
+ dr[s + m *UNROLL] = x[ai[s]][m] - x[aj[s]][m];
+ dr[s + (DIM+m)*UNROLL] = x[ak[s]][m] - x[aj[s]][m];
+ }
+
+ /* At the end fill the arrays with identical entries */
+ if (iu + nfa1 < nbonds)
+ {
+ iu += nfa1;
+ }
+ }
+
+ k_S = gmx_load_pr(coeff);
+ theta0_S = gmx_load_pr(coeff+UNROLL);
+
+ rijx_S = gmx_load_pr(dr + 0*UNROLL);
+ rijy_S = gmx_load_pr(dr + 1*UNROLL);
+ rijz_S = gmx_load_pr(dr + 2*UNROLL);
+ rkjx_S = gmx_load_pr(dr + 3*UNROLL);
+ rkjy_S = gmx_load_pr(dr + 4*UNROLL);
+ rkjz_S = gmx_load_pr(dr + 5*UNROLL);
+
+ pbc_dx_simd(&rijx_S, &rijy_S, &rijz_S, &pbc_simd);
+ pbc_dx_simd(&rkjx_S, &rkjy_S, &rkjz_S, &pbc_simd);
+
+ rij_rkj_S = gmx_iprod_pr(rijx_S, rijy_S, rijz_S,
+ rkjx_S, rkjy_S, rkjz_S);
+
+ nrij2_S = gmx_norm2_pr(rijx_S, rijy_S, rijz_S);
+ nrkj2_S = gmx_norm2_pr(rkjx_S, rkjy_S, rkjz_S);
+
+ nrij_1_S = gmx_invsqrt_pr(nrij2_S);
+ nrkj_1_S = gmx_invsqrt_pr(nrkj2_S);
+
+ cos_S = gmx_mul_pr(rij_rkj_S, gmx_mul_pr(nrij_1_S, nrkj_1_S));
+
++ /* To allow for 180 degrees, we take the max of cos and -1 + 1bit,
++ * so we can safely get the 1/sin from 1/sqrt(1 - cos^2).
++ * This also ensures that rounding errors would cause the argument
++ * of gmx_acos_pr to be < -1.
++ * Note that we do not take precautions for cos(0)=1, so the outer
++ * atoms in an angle should not be on top of each other.
++ */
++ cos_S = gmx_max_pr(cos_S, min_one_plus_eps_S);
++
+ theta_S = gmx_acos_pr(cos_S);
+
- sin_S);
++ invsin_S = gmx_invsqrt_pr(gmx_sub_pr(one_S, gmx_mul_pr(cos_S, cos_S)));
++
+ st_S = gmx_mul_pr(gmx_mul_pr(k_S, gmx_sub_pr(theta0_S, theta_S)),
- gmx_mm_pr fmin_S = gmx_set1_pr(GMX_FLOAT_MIN);
++ invsin_S);
+ sth_S = gmx_mul_pr(st_S, cos_S);
+
+ cik_S = gmx_mul_pr(st_S, gmx_mul_pr(nrij_1_S, nrkj_1_S));
+ cii_S = gmx_mul_pr(sth_S, gmx_mul_pr(nrij_1_S, nrij_1_S));
+ ckk_S = gmx_mul_pr(sth_S, gmx_mul_pr(nrkj_1_S, nrkj_1_S));
+
+ f_ix_S = gmx_mul_pr(cii_S, rijx_S);
+ f_ix_S = gmx_nmsub_pr(cik_S, rkjx_S, f_ix_S);
+ f_iy_S = gmx_mul_pr(cii_S, rijy_S);
+ f_iy_S = gmx_nmsub_pr(cik_S, rkjy_S, f_iy_S);
+ f_iz_S = gmx_mul_pr(cii_S, rijz_S);
+ f_iz_S = gmx_nmsub_pr(cik_S, rkjz_S, f_iz_S);
+ f_kx_S = gmx_mul_pr(ckk_S, rkjx_S);
+ f_kx_S = gmx_nmsub_pr(cik_S, rijx_S, f_kx_S);
+ f_ky_S = gmx_mul_pr(ckk_S, rkjy_S);
+ f_ky_S = gmx_nmsub_pr(cik_S, rijy_S, f_ky_S);
+ f_kz_S = gmx_mul_pr(ckk_S, rkjz_S);
+ f_kz_S = gmx_nmsub_pr(cik_S, rijz_S, f_kz_S);
+
+ gmx_store_pr(f_buf + 0*UNROLL, f_ix_S);
+ gmx_store_pr(f_buf + 1*UNROLL, f_iy_S);
+ gmx_store_pr(f_buf + 2*UNROLL, f_iz_S);
+ gmx_store_pr(f_buf + 3*UNROLL, f_kx_S);
+ gmx_store_pr(f_buf + 4*UNROLL, f_ky_S);
+ gmx_store_pr(f_buf + 5*UNROLL, f_kz_S);
+
+ iu = i;
+ s = 0;
+ do
+ {
+ for (m = 0; m < DIM; m++)
+ {
+ f[ai[s]][m] += f_buf[s + m*UNROLL];
+ f[aj[s]][m] -= f_buf[s + m*UNROLL] + f_buf[s + (DIM+m)*UNROLL];
+ f[ak[s]][m] += f_buf[s + (DIM+m)*UNROLL];
+ }
+ s++;
+ iu += nfa1;
+ }
+ while (s < UNROLL && iu < nbonds);
+ }
+#undef UNROLL
+}
+
+#endif /* SIMD_BONDEDS */
+
+real linear_angles(int nbonds,
+ const t_iatom forceatoms[], const t_iparams forceparams[],
+ const rvec x[], rvec f[], rvec fshift[],
+ const t_pbc *pbc, const t_graph *g,
+ real lambda, real *dvdlambda,
+ const t_mdatoms gmx_unused *md, t_fcdata gmx_unused *fcd,
+ int gmx_unused *global_atom_index)
+{
+ int i, m, ai, aj, ak, t1, t2, type;
+ rvec f_i, f_j, f_k;
+ real L1, kA, kB, aA, aB, dr, dr2, va, vtot, a, b, klin;
+ ivec jt, dt_ij, dt_kj;
+ rvec r_ij, r_kj, r_ik, dx;
+
+ L1 = 1-lambda;
+ vtot = 0.0;
+ for (i = 0; (i < nbonds); )
+ {
+ type = forceatoms[i++];
+ ai = forceatoms[i++];
+ aj = forceatoms[i++];
+ ak = forceatoms[i++];
+
+ kA = forceparams[type].linangle.klinA;
+ kB = forceparams[type].linangle.klinB;
+ klin = L1*kA + lambda*kB;
+
+ aA = forceparams[type].linangle.aA;
+ aB = forceparams[type].linangle.aB;
+ a = L1*aA+lambda*aB;
+ b = 1-a;
+
+ t1 = pbc_rvec_sub(pbc, x[ai], x[aj], r_ij);
+ t2 = pbc_rvec_sub(pbc, x[ak], x[aj], r_kj);
+ rvec_sub(r_ij, r_kj, r_ik);
+
+ dr2 = 0;
+ for (m = 0; (m < DIM); m++)
+ {
+ dr = -a * r_ij[m] - b * r_kj[m];
+ dr2 += dr*dr;
+ dx[m] = dr;
+ f_i[m] = a*klin*dr;
+ f_k[m] = b*klin*dr;
+ f_j[m] = -(f_i[m]+f_k[m]);
+ f[ai][m] += f_i[m];
+ f[aj][m] += f_j[m];
+ f[ak][m] += f_k[m];
+ }
+ va = 0.5*klin*dr2;
+ *dvdlambda += 0.5*(kB-kA)*dr2 + klin*(aB-aA)*iprod(dx, r_ik);
+
+ vtot += va;
+
+ if (g)
+ {
+ copy_ivec(SHIFT_IVEC(g, aj), jt);
+
+ ivec_sub(SHIFT_IVEC(g, ai), jt, dt_ij);
+ ivec_sub(SHIFT_IVEC(g, ak), jt, dt_kj);
+ t1 = IVEC2IS(dt_ij);
+ t2 = IVEC2IS(dt_kj);
+ }
+ rvec_inc(fshift[t1], f_i);
+ rvec_inc(fshift[CENTRAL], f_j);
+ rvec_inc(fshift[t2], f_k);
+ } /* 57 TOTAL */
+ return vtot;
+}
+
+real urey_bradley(int nbonds,
+ const t_iatom forceatoms[], const t_iparams forceparams[],
+ const rvec x[], rvec f[], rvec fshift[],
+ const t_pbc *pbc, const t_graph *g,
+ real lambda, real *dvdlambda,
+ const t_mdatoms gmx_unused *md, t_fcdata gmx_unused *fcd,
+ int gmx_unused *global_atom_index)
+{
+ int i, m, ai, aj, ak, t1, t2, type, ki;
+ rvec r_ij, r_kj, r_ik;
+ real cos_theta, cos_theta2, theta;
+ real dVdt, va, vtot, dr, dr2, vbond, fbond, fik;
+ real kthA, th0A, kUBA, r13A, kthB, th0B, kUBB, r13B;
+ ivec jt, dt_ij, dt_kj, dt_ik;
+
+ vtot = 0.0;
+ for (i = 0; (i < nbonds); )
+ {
+ type = forceatoms[i++];
+ ai = forceatoms[i++];
+ aj = forceatoms[i++];
+ ak = forceatoms[i++];
+ th0A = forceparams[type].u_b.thetaA*DEG2RAD;
+ kthA = forceparams[type].u_b.kthetaA;
+ r13A = forceparams[type].u_b.r13A;
+ kUBA = forceparams[type].u_b.kUBA;
+ th0B = forceparams[type].u_b.thetaB*DEG2RAD;
+ kthB = forceparams[type].u_b.kthetaB;
+ r13B = forceparams[type].u_b.r13B;
+ kUBB = forceparams[type].u_b.kUBB;
+
+ theta = bond_angle(x[ai], x[aj], x[ak], pbc,
+ r_ij, r_kj, &cos_theta, &t1, &t2); /* 41 */
+
+ *dvdlambda += harmonic(kthA, kthB, th0A, th0B, theta, lambda, &va, &dVdt); /* 21 */
+ vtot += va;
+
+ ki = pbc_rvec_sub(pbc, x[ai], x[ak], r_ik); /* 3 */
+ dr2 = iprod(r_ik, r_ik); /* 5 */
+ dr = dr2*gmx_invsqrt(dr2); /* 10 */
+
+ *dvdlambda += harmonic(kUBA, kUBB, r13A, r13B, dr, lambda, &vbond, &fbond); /* 19 */
+
+ cos_theta2 = sqr(cos_theta); /* 1 */
+ if (cos_theta2 < 1)
+ {
+ real st, sth;
+ real cik, cii, ckk;
+ real nrkj2, nrij2;
+ rvec f_i, f_j, f_k;
+
+ st = dVdt*gmx_invsqrt(1 - cos_theta2); /* 12 */
+ sth = st*cos_theta; /* 1 */
+#ifdef DEBUG
+ if (debug)
+ {
+ fprintf(debug, "ANGLES: theta = %10g vth = %10g dV/dtheta = %10g\n",
+ theta*RAD2DEG, va, dVdt);
+ }
+#endif
+ nrkj2 = iprod(r_kj, r_kj); /* 5 */
+ nrij2 = iprod(r_ij, r_ij);
+
+ cik = st*gmx_invsqrt(nrkj2*nrij2); /* 12 */
+ cii = sth/nrij2; /* 10 */
+ ckk = sth/nrkj2; /* 10 */
+
+ for (m = 0; (m < DIM); m++) /* 39 */
+ {
+ f_i[m] = -(cik*r_kj[m]-cii*r_ij[m]);
+ f_k[m] = -(cik*r_ij[m]-ckk*r_kj[m]);
+ f_j[m] = -f_i[m]-f_k[m];
+ f[ai][m] += f_i[m];
+ f[aj][m] += f_j[m];
+ f[ak][m] += f_k[m];
+ }
+ if (g)
+ {
+ copy_ivec(SHIFT_IVEC(g, aj), jt);
+
+ ivec_sub(SHIFT_IVEC(g, ai), jt, dt_ij);
+ ivec_sub(SHIFT_IVEC(g, ak), jt, dt_kj);
+ t1 = IVEC2IS(dt_ij);
+ t2 = IVEC2IS(dt_kj);
+ }
+ rvec_inc(fshift[t1], f_i);
+ rvec_inc(fshift[CENTRAL], f_j);
+ rvec_inc(fshift[t2], f_k);
+ } /* 161 TOTAL */
+ /* Time for the bond calculations */
+ if (dr2 == 0.0)
+ {
+ continue;
+ }
+
+ vtot += vbond; /* 1*/
+ fbond *= gmx_invsqrt(dr2); /* 6 */
+
+ if (g)
+ {
+ ivec_sub(SHIFT_IVEC(g, ai), SHIFT_IVEC(g, ak), dt_ik);
+ ki = IVEC2IS(dt_ik);
+ }
+ for (m = 0; (m < DIM); m++) /* 15 */
+ {
+ fik = fbond*r_ik[m];
+ f[ai][m] += fik;
+ f[ak][m] -= fik;
+ fshift[ki][m] += fik;
+ fshift[CENTRAL][m] -= fik;
+ }
+ }
+ return vtot;
+}
+
+real quartic_angles(int nbonds,
+ const t_iatom forceatoms[], const t_iparams forceparams[],
+ const rvec x[], rvec f[], rvec fshift[],
+ const t_pbc *pbc, const t_graph *g,
+ real gmx_unused lambda, real gmx_unused *dvdlambda,
+ const t_mdatoms gmx_unused *md, t_fcdata gmx_unused *fcd,
+ int gmx_unused *global_atom_index)
+{
+ int i, j, ai, aj, ak, t1, t2, type;
+ rvec r_ij, r_kj;
+ real cos_theta, cos_theta2, theta, dt, dVdt, va, dtp, c, vtot;
+ ivec jt, dt_ij, dt_kj;
+
+ vtot = 0.0;
+ for (i = 0; (i < nbonds); )
+ {
+ type = forceatoms[i++];
+ ai = forceatoms[i++];
+ aj = forceatoms[i++];
+ ak = forceatoms[i++];
+
+ theta = bond_angle(x[ai], x[aj], x[ak], pbc,
+ r_ij, r_kj, &cos_theta, &t1, &t2); /* 41 */
+
+ dt = theta - forceparams[type].qangle.theta*DEG2RAD; /* 2 */
+
+ dVdt = 0;
+ va = forceparams[type].qangle.c[0];
+ dtp = 1.0;
+ for (j = 1; j <= 4; j++)
+ {
+ c = forceparams[type].qangle.c[j];
+ dVdt -= j*c*dtp;
+ dtp *= dt;
+ va += c*dtp;
+ }
+ /* 20 */
+
+ vtot += va;
+
+ cos_theta2 = sqr(cos_theta); /* 1 */
+ if (cos_theta2 < 1)
+ {
+ int m;
+ real st, sth;
+ real cik, cii, ckk;
+ real nrkj2, nrij2;
+ rvec f_i, f_j, f_k;
+
+ st = dVdt*gmx_invsqrt(1 - cos_theta2); /* 12 */
+ sth = st*cos_theta; /* 1 */
+#ifdef DEBUG
+ if (debug)
+ {
+ fprintf(debug, "ANGLES: theta = %10g vth = %10g dV/dtheta = %10g\n",
+ theta*RAD2DEG, va, dVdt);
+ }
+#endif
+ nrkj2 = iprod(r_kj, r_kj); /* 5 */
+ nrij2 = iprod(r_ij, r_ij);
+
+ cik = st*gmx_invsqrt(nrkj2*nrij2); /* 12 */
+ cii = sth/nrij2; /* 10 */
+ ckk = sth/nrkj2; /* 10 */
+
+ for (m = 0; (m < DIM); m++) /* 39 */
+ {
+ f_i[m] = -(cik*r_kj[m]-cii*r_ij[m]);
+ f_k[m] = -(cik*r_ij[m]-ckk*r_kj[m]);
+ f_j[m] = -f_i[m]-f_k[m];
+ f[ai][m] += f_i[m];
+ f[aj][m] += f_j[m];
+ f[ak][m] += f_k[m];
+ }
+ if (g)
+ {
+ copy_ivec(SHIFT_IVEC(g, aj), jt);
+
+ ivec_sub(SHIFT_IVEC(g, ai), jt, dt_ij);
+ ivec_sub(SHIFT_IVEC(g, ak), jt, dt_kj);
+ t1 = IVEC2IS(dt_ij);
+ t2 = IVEC2IS(dt_kj);
+ }
+ rvec_inc(fshift[t1], f_i);
+ rvec_inc(fshift[CENTRAL], f_j);
+ rvec_inc(fshift[t2], f_k);
+ } /* 153 TOTAL */
+ }
+ return vtot;
+}
+
+real dih_angle(const rvec xi, const rvec xj, const rvec xk, const rvec xl,
+ const t_pbc *pbc,
+ rvec r_ij, rvec r_kj, rvec r_kl, rvec m, rvec n,
+ real *sign, int *t1, int *t2, int *t3)
+{
+ real ipr, phi;
+
+ *t1 = pbc_rvec_sub(pbc, xi, xj, r_ij); /* 3 */
+ *t2 = pbc_rvec_sub(pbc, xk, xj, r_kj); /* 3 */
+ *t3 = pbc_rvec_sub(pbc, xk, xl, r_kl); /* 3 */
+
+ cprod(r_ij, r_kj, m); /* 9 */
+ cprod(r_kj, r_kl, n); /* 9 */
+ phi = gmx_angle(m, n); /* 49 (assuming 25 for atan2) */
+ ipr = iprod(r_ij, n); /* 5 */
+ (*sign) = (ipr < 0.0) ? -1.0 : 1.0;
+ phi = (*sign)*phi; /* 1 */
+ /* 82 TOTAL */
+ return phi;
+}
+
+
+#ifdef SIMD_BONDEDS
+
+/* As dih_angle above, but calculates 4 dihedral angles at once using SIMD,
+ * also calculates the pre-factor required for the dihedral force update.
+ * Note that bv and buf should be register aligned.
+ */
+static gmx_inline void
+dih_angle_simd(const rvec *x,
+ const int *ai, const int *aj, const int *ak, const int *al,
+ const pbc_simd_t *pbc,
+ real *dr,
+ gmx_mm_pr *phi_S,
+ gmx_mm_pr *mx_S, gmx_mm_pr *my_S, gmx_mm_pr *mz_S,
+ gmx_mm_pr *nx_S, gmx_mm_pr *ny_S, gmx_mm_pr *nz_S,
+ gmx_mm_pr *nrkj_m2_S,
+ gmx_mm_pr *nrkj_n2_S,
+ real *p,
+ real *q)
+{
+#define UNROLL GMX_SIMD_WIDTH_HERE
+ int s, m;
+ gmx_mm_pr rijx_S, rijy_S, rijz_S;
+ gmx_mm_pr rkjx_S, rkjy_S, rkjz_S;
+ gmx_mm_pr rklx_S, rkly_S, rklz_S;
+ gmx_mm_pr cx_S, cy_S, cz_S;
+ gmx_mm_pr cn_S;
+ gmx_mm_pr s_S;
+ gmx_mm_pr ipr_S;
+ gmx_mm_pr iprm_S, iprn_S;
+ gmx_mm_pr nrkj2_S, nrkj_1_S, nrkj_2_S, nrkj_S;
++ gmx_mm_pr toler_S;
+ gmx_mm_pr p_S, q_S;
- nrkj2_S = gmx_max_pr(nrkj2_S, fmin_S);
++ gmx_mm_pr nrkj2_min_S;
++ gmx_mm_pr real_eps_S;
++
++ /* Used to avoid division by zero.
++ * We take into acount that we multiply the result by real_eps_S.
++ */
++ nrkj2_min_S = gmx_set1_pr(GMX_REAL_MIN/(2*GMX_REAL_EPS));
++
++ /* The value of the last significant bit (GMX_REAL_EPS is half of that) */
++ real_eps_S = gmx_set1_pr(2*GMX_REAL_EPS);
+
+ for (s = 0; s < UNROLL; s++)
+ {
+ /* If you can't use pbc_dx_simd below for PBC, e.g. because
+ * you can't round in SIMD, use pbc_rvec_sub here.
+ */
+ for (m = 0; m < DIM; m++)
+ {
+ dr[s + (0*DIM + m)*UNROLL] = x[ai[s]][m] - x[aj[s]][m];
+ dr[s + (1*DIM + m)*UNROLL] = x[ak[s]][m] - x[aj[s]][m];
+ dr[s + (2*DIM + m)*UNROLL] = x[ak[s]][m] - x[al[s]][m];
+ }
+ }
+
+ rijx_S = gmx_load_pr(dr + 0*UNROLL);
+ rijy_S = gmx_load_pr(dr + 1*UNROLL);
+ rijz_S = gmx_load_pr(dr + 2*UNROLL);
+ rkjx_S = gmx_load_pr(dr + 3*UNROLL);
+ rkjy_S = gmx_load_pr(dr + 4*UNROLL);
+ rkjz_S = gmx_load_pr(dr + 5*UNROLL);
+ rklx_S = gmx_load_pr(dr + 6*UNROLL);
+ rkly_S = gmx_load_pr(dr + 7*UNROLL);
+ rklz_S = gmx_load_pr(dr + 8*UNROLL);
+
+ pbc_dx_simd(&rijx_S, &rijy_S, &rijz_S, pbc);
+ pbc_dx_simd(&rkjx_S, &rkjy_S, &rkjz_S, pbc);
+ pbc_dx_simd(&rklx_S, &rkly_S, &rklz_S, pbc);
+
+ gmx_cprod_pr(rijx_S, rijy_S, rijz_S,
+ rkjx_S, rkjy_S, rkjz_S,
+ mx_S, my_S, mz_S);
+
+ gmx_cprod_pr(rkjx_S, rkjy_S, rkjz_S,
+ rklx_S, rkly_S, rklz_S,
+ nx_S, ny_S, nz_S);
+
+ gmx_cprod_pr(*mx_S, *my_S, *mz_S,
+ *nx_S, *ny_S, *nz_S,
+ &cx_S, &cy_S, &cz_S);
+
+ cn_S = gmx_sqrt_pr(gmx_norm2_pr(cx_S, cy_S, cz_S));
+
+ s_S = gmx_iprod_pr(*mx_S, *my_S, *mz_S, *nx_S, *ny_S, *nz_S);
+
+ /* Determine the dihedral angle, the sign might need correction */
+ *phi_S = gmx_atan2_pr(cn_S, s_S);
+
+ ipr_S = gmx_iprod_pr(rijx_S, rijy_S, rijz_S,
+ *nx_S, *ny_S, *nz_S);
+
+ iprm_S = gmx_norm2_pr(*mx_S, *my_S, *mz_S);
+ iprn_S = gmx_norm2_pr(*nx_S, *ny_S, *nz_S);
+
+ nrkj2_S = gmx_norm2_pr(rkjx_S, rkjy_S, rkjz_S);
+
+ /* Avoid division by zero. When zero, the result is multiplied by 0
+ * anyhow, so the 3 max below do not affect the final result.
+ */
- iprm_S = gmx_max_pr(iprm_S, fmin_S);
- iprn_S = gmx_max_pr(iprn_S, fmin_S);
++ nrkj2_S = gmx_max_pr(nrkj2_S, nrkj2_min_S);
+ nrkj_1_S = gmx_invsqrt_pr(nrkj2_S);
+ nrkj_2_S = gmx_mul_pr(nrkj_1_S, nrkj_1_S);
+ nrkj_S = gmx_mul_pr(nrkj2_S, nrkj_1_S);
+
++ toler_S = gmx_mul_pr(nrkj2_S, real_eps_S);
++
++ /* Here the plain-C code uses a conditional, but we can't do that in SIMD.
++ * So we take a max with the tolerance instead. Since we multiply with
++ * m or n later, the max does not affect the results.
++ */
++ iprm_S = gmx_max_pr(iprm_S, toler_S);
++ iprn_S = gmx_max_pr(iprn_S, toler_S);
+ *nrkj_m2_S = gmx_mul_pr(nrkj_S, gmx_inv_pr(iprm_S));
+ *nrkj_n2_S = gmx_mul_pr(nrkj_S, gmx_inv_pr(iprn_S));
+
+ /* Set sign of phi_S with the sign of ipr_S; phi_S is currently positive */
+ *phi_S = gmx_cpsgn_nonneg_pr(ipr_S, *phi_S);
+
+ p_S = gmx_iprod_pr(rijx_S, rijy_S, rijz_S,
+ rkjx_S, rkjy_S, rkjz_S);
+ p_S = gmx_mul_pr(p_S, nrkj_2_S);
+
+ q_S = gmx_iprod_pr(rklx_S, rkly_S, rklz_S,
+ rkjx_S, rkjy_S, rkjz_S);
+ q_S = gmx_mul_pr(q_S, nrkj_2_S);
+
+ gmx_store_pr(p, p_S);
+ gmx_store_pr(q, q_S);
+#undef UNROLL
+}
+
+#endif /* SIMD_BONDEDS */
+
+
+void do_dih_fup(int i, int j, int k, int l, real ddphi,
+ rvec r_ij, rvec r_kj, rvec r_kl,
+ rvec m, rvec n, rvec f[], rvec fshift[],
+ const t_pbc *pbc, const t_graph *g,
+ const rvec x[], int t1, int t2, int t3)
+{
+ /* 143 FLOPS */
+ rvec f_i, f_j, f_k, f_l;
+ rvec uvec, vvec, svec, dx_jl;
+ real iprm, iprn, nrkj, nrkj2, nrkj_1, nrkj_2;
+ real a, b, p, q, toler;
+ ivec jt, dt_ij, dt_kj, dt_lj;
+
+ iprm = iprod(m, m); /* 5 */
+ iprn = iprod(n, n); /* 5 */
+ nrkj2 = iprod(r_kj, r_kj); /* 5 */
+ toler = nrkj2*GMX_REAL_EPS;
+ if ((iprm > toler) && (iprn > toler))
+ {
+ nrkj_1 = gmx_invsqrt(nrkj2); /* 10 */
+ nrkj_2 = nrkj_1*nrkj_1; /* 1 */
+ nrkj = nrkj2*nrkj_1; /* 1 */
+ a = -ddphi*nrkj/iprm; /* 11 */
+ svmul(a, m, f_i); /* 3 */
+ b = ddphi*nrkj/iprn; /* 11 */
+ svmul(b, n, f_l); /* 3 */
+ p = iprod(r_ij, r_kj); /* 5 */
+ p *= nrkj_2; /* 1 */
+ q = iprod(r_kl, r_kj); /* 5 */
+ q *= nrkj_2; /* 1 */
+ svmul(p, f_i, uvec); /* 3 */
+ svmul(q, f_l, vvec); /* 3 */
+ rvec_sub(uvec, vvec, svec); /* 3 */
+ rvec_sub(f_i, svec, f_j); /* 3 */
+ rvec_add(f_l, svec, f_k); /* 3 */
+ rvec_inc(f[i], f_i); /* 3 */
+ rvec_dec(f[j], f_j); /* 3 */
+ rvec_dec(f[k], f_k); /* 3 */
+ rvec_inc(f[l], f_l); /* 3 */
+
+ if (g)
+ {
+ copy_ivec(SHIFT_IVEC(g, j), jt);
+ ivec_sub(SHIFT_IVEC(g, i), jt, dt_ij);
+ ivec_sub(SHIFT_IVEC(g, k), jt, dt_kj);
+ ivec_sub(SHIFT_IVEC(g, l), jt, dt_lj);
+ t1 = IVEC2IS(dt_ij);
+ t2 = IVEC2IS(dt_kj);
+ t3 = IVEC2IS(dt_lj);
+ }
+ else if (pbc)
+ {
+ t3 = pbc_rvec_sub(pbc, x[l], x[j], dx_jl);
+ }
+ else
+ {
+ t3 = CENTRAL;
+ }
+
+ rvec_inc(fshift[t1], f_i);
+ rvec_dec(fshift[CENTRAL], f_j);
+ rvec_dec(fshift[t2], f_k);
+ rvec_inc(fshift[t3], f_l);
+ }
+ /* 112 TOTAL */
+}
+
+/* As do_dih_fup above, but without shift forces */
+static void
+do_dih_fup_noshiftf(int i, int j, int k, int l, real ddphi,
+ rvec r_ij, rvec r_kj, rvec r_kl,
+ rvec m, rvec n, rvec f[])
+{
+ rvec f_i, f_j, f_k, f_l;
+ rvec uvec, vvec, svec, dx_jl;
+ real iprm, iprn, nrkj, nrkj2, nrkj_1, nrkj_2;
+ real a, b, p, q, toler;
+ ivec jt, dt_ij, dt_kj, dt_lj;
+
+ iprm = iprod(m, m); /* 5 */
+ iprn = iprod(n, n); /* 5 */
+ nrkj2 = iprod(r_kj, r_kj); /* 5 */
+ toler = nrkj2*GMX_REAL_EPS;
+ if ((iprm > toler) && (iprn > toler))
+ {
+ nrkj_1 = gmx_invsqrt(nrkj2); /* 10 */
+ nrkj_2 = nrkj_1*nrkj_1; /* 1 */
+ nrkj = nrkj2*nrkj_1; /* 1 */
+ a = -ddphi*nrkj/iprm; /* 11 */
+ svmul(a, m, f_i); /* 3 */
+ b = ddphi*nrkj/iprn; /* 11 */
+ svmul(b, n, f_l); /* 3 */
+ p = iprod(r_ij, r_kj); /* 5 */
+ p *= nrkj_2; /* 1 */
+ q = iprod(r_kl, r_kj); /* 5 */
+ q *= nrkj_2; /* 1 */
+ svmul(p, f_i, uvec); /* 3 */
+ svmul(q, f_l, vvec); /* 3 */
+ rvec_sub(uvec, vvec, svec); /* 3 */
+ rvec_sub(f_i, svec, f_j); /* 3 */
+ rvec_add(f_l, svec, f_k); /* 3 */
+ rvec_inc(f[i], f_i); /* 3 */
+ rvec_dec(f[j], f_j); /* 3 */
+ rvec_dec(f[k], f_k); /* 3 */
+ rvec_inc(f[l], f_l); /* 3 */
+ }
+}
+
+/* As do_dih_fup_noshiftf above, but with pre-calculated pre-factors */
+static gmx_inline void
+do_dih_fup_noshiftf_precalc(int i, int j, int k, int l,
+ real p, real q,
+ real f_i_x, real f_i_y, real f_i_z,
+ real mf_l_x, real mf_l_y, real mf_l_z,
+ rvec f[])
+{
+ rvec f_i, f_j, f_k, f_l;
+ rvec uvec, vvec, svec;
+
+ f_i[XX] = f_i_x;
+ f_i[YY] = f_i_y;
+ f_i[ZZ] = f_i_z;
+ f_l[XX] = -mf_l_x;
+ f_l[YY] = -mf_l_y;
+ f_l[ZZ] = -mf_l_z;
+ svmul(p, f_i, uvec);
+ svmul(q, f_l, vvec);
+ rvec_sub(uvec, vvec, svec);
+ rvec_sub(f_i, svec, f_j);
+ rvec_add(f_l, svec, f_k);
+ rvec_inc(f[i], f_i);
+ rvec_dec(f[j], f_j);
+ rvec_dec(f[k], f_k);
+ rvec_inc(f[l], f_l);
+}
+
+
+real dopdihs(real cpA, real cpB, real phiA, real phiB, int mult,
+ real phi, real lambda, real *V, real *F)
+{
+ real v, dvdlambda, mdphi, v1, sdphi, ddphi;
+ real L1 = 1.0 - lambda;
+ real ph0 = (L1*phiA + lambda*phiB)*DEG2RAD;
+ real dph0 = (phiB - phiA)*DEG2RAD;
+ real cp = L1*cpA + lambda*cpB;
+
+ mdphi = mult*phi - ph0;
+ sdphi = sin(mdphi);
+ ddphi = -cp*mult*sdphi;
+ v1 = 1.0 + cos(mdphi);
+ v = cp*v1;
+
+ dvdlambda = (cpB - cpA)*v1 + cp*dph0*sdphi;
+
+ *V = v;
+ *F = ddphi;
+
+ return dvdlambda;
+
+ /* That was 40 flops */
+}
+
+static void
+dopdihs_noener(real cpA, real cpB, real phiA, real phiB, int mult,
+ real phi, real lambda, real *F)
+{
+ real mdphi, sdphi, ddphi;
+ real L1 = 1.0 - lambda;
+ real ph0 = (L1*phiA + lambda*phiB)*DEG2RAD;
+ real cp = L1*cpA + lambda*cpB;
+
+ mdphi = mult*phi - ph0;
+ sdphi = sin(mdphi);
+ ddphi = -cp*mult*sdphi;
+
+ *F = ddphi;
+
+ /* That was 20 flops */
+}
+
+static void
+dopdihs_mdphi(real cpA, real cpB, real phiA, real phiB, int mult,
+ real phi, real lambda, real *cp, real *mdphi)
+{
+ real L1 = 1.0 - lambda;
+ real ph0 = (L1*phiA + lambda*phiB)*DEG2RAD;
+
+ *cp = L1*cpA + lambda*cpB;
+
+ *mdphi = mult*phi - ph0;
+}
+
+static real dopdihs_min(real cpA, real cpB, real phiA, real phiB, int mult,
+ real phi, real lambda, real *V, real *F)
+/* similar to dopdihs, except for a minus sign *
+ * and a different treatment of mult/phi0 */
+{
+ real v, dvdlambda, mdphi, v1, sdphi, ddphi;
+ real L1 = 1.0 - lambda;
+ real ph0 = (L1*phiA + lambda*phiB)*DEG2RAD;
+ real dph0 = (phiB - phiA)*DEG2RAD;
+ real cp = L1*cpA + lambda*cpB;
+
+ mdphi = mult*(phi-ph0);
+ sdphi = sin(mdphi);
+ ddphi = cp*mult*sdphi;
+ v1 = 1.0-cos(mdphi);
+ v = cp*v1;
+
+ dvdlambda = (cpB-cpA)*v1 + cp*dph0*sdphi;
+
+ *V = v;
+ *F = ddphi;
+
+ return dvdlambda;
+
+ /* That was 40 flops */
+}
+
+real pdihs(int nbonds,
+ const t_iatom forceatoms[], const t_iparams forceparams[],
+ const rvec x[], rvec f[], rvec fshift[],
+ const t_pbc *pbc, const t_graph *g,
+ real lambda, real *dvdlambda,
+ const t_mdatoms gmx_unused *md, t_fcdata gmx_unused *fcd,
+ int gmx_unused *global_atom_index)
+{
+ int i, type, ai, aj, ak, al;
+ int t1, t2, t3;
+ rvec r_ij, r_kj, r_kl, m, n;
+ real phi, sign, ddphi, vpd, vtot;
+
+ vtot = 0.0;
+
+ for (i = 0; (i < nbonds); )
+ {
+ type = forceatoms[i++];
+ ai = forceatoms[i++];
+ aj = forceatoms[i++];
+ ak = forceatoms[i++];
+ al = forceatoms[i++];
+
+ phi = dih_angle(x[ai], x[aj], x[ak], x[al], pbc, r_ij, r_kj, r_kl, m, n,
+ &sign, &t1, &t2, &t3); /* 84 */
+ *dvdlambda += dopdihs(forceparams[type].pdihs.cpA,
+ forceparams[type].pdihs.cpB,
+ forceparams[type].pdihs.phiA,
+ forceparams[type].pdihs.phiB,
+ forceparams[type].pdihs.mult,
+ phi, lambda, &vpd, &ddphi);
+
+ vtot += vpd;
+ do_dih_fup(ai, aj, ak, al, ddphi, r_ij, r_kj, r_kl, m, n,
+ f, fshift, pbc, g, x, t1, t2, t3); /* 112 */
+
+#ifdef DEBUG
+ fprintf(debug, "pdih: (%d,%d,%d,%d) phi=%g\n",
+ ai, aj, ak, al, phi);
+#endif
+ } /* 223 TOTAL */
+
+ return vtot;
+}
+
+void make_dp_periodic(real *dp) /* 1 flop? */
+{
+ /* dp cannot be outside (-pi,pi) */
+ if (*dp >= M_PI)
+ {
+ *dp -= 2*M_PI;
+ }
+ else if (*dp < -M_PI)
+ {
+ *dp += 2*M_PI;
+ }
+ return;
+}
+
+/* As pdihs above, but without calculating energies and shift forces */
+static void
+pdihs_noener(int nbonds,
+ const t_iatom forceatoms[], const t_iparams forceparams[],
+ const rvec x[], rvec f[],
+ const t_pbc gmx_unused *pbc, const t_graph gmx_unused *g,
+ real lambda,
+ const t_mdatoms gmx_unused *md, t_fcdata gmx_unused *fcd,
+ int gmx_unused *global_atom_index)
+{
+ int i, type, ai, aj, ak, al;
+ int t1, t2, t3;
+ rvec r_ij, r_kj, r_kl, m, n;
+ real phi, sign, ddphi_tot, ddphi;
+
+ for (i = 0; (i < nbonds); )
+ {
+ ai = forceatoms[i+1];
+ aj = forceatoms[i+2];
+ ak = forceatoms[i+3];
+ al = forceatoms[i+4];
+
+ phi = dih_angle(x[ai], x[aj], x[ak], x[al], pbc, r_ij, r_kj, r_kl, m, n,
+ &sign, &t1, &t2, &t3);
+
+ ddphi_tot = 0;
+
+ /* Loop over dihedrals working on the same atoms,
+ * so we avoid recalculating angles and force distributions.
+ */
+ do
+ {
+ type = forceatoms[i];
+ dopdihs_noener(forceparams[type].pdihs.cpA,
+ forceparams[type].pdihs.cpB,
+ forceparams[type].pdihs.phiA,
+ forceparams[type].pdihs.phiB,
+ forceparams[type].pdihs.mult,
+ phi, lambda, &ddphi);
+ ddphi_tot += ddphi;
+
+ i += 5;
+ }
+ while (i < nbonds &&
+ forceatoms[i+1] == ai &&
+ forceatoms[i+2] == aj &&
+ forceatoms[i+3] == ak &&
+ forceatoms[i+4] == al);
+
+ do_dih_fup_noshiftf(ai, aj, ak, al, ddphi_tot, r_ij, r_kj, r_kl, m, n, f);
+ }
+}
+
+
+#ifdef SIMD_BONDEDS
+
+/* As pdihs_noner above, but using SIMD to calculate many dihedrals at once */
+static void
+pdihs_noener_simd(int nbonds,
+ const t_iatom forceatoms[], const t_iparams forceparams[],
+ const rvec x[], rvec f[],
+ const t_pbc *pbc, const t_graph gmx_unused *g,
+ real gmx_unused lambda,
+ const t_mdatoms gmx_unused *md, t_fcdata gmx_unused *fcd,
+ int gmx_unused *global_atom_index)
+{
+#define UNROLL GMX_SIMD_WIDTH_HERE
+ const int nfa1 = 5;
+ int i, iu, s;
+ int type, ai[UNROLL], aj[UNROLL], ak[UNROLL], al[UNROLL];
+ int t1[UNROLL], t2[UNROLL], t3[UNROLL];
+ real ddphi;
+ real dr_array[3*DIM*UNROLL+UNROLL], *dr;
+ real buf_array[7*UNROLL+UNROLL], *buf;
+ real *cp, *phi0, *mult, *phi, *p, *q, *sf_i, *msf_l;
+ gmx_mm_pr phi0_S, phi_S;
+ gmx_mm_pr mx_S, my_S, mz_S;
+ gmx_mm_pr nx_S, ny_S, nz_S;
+ gmx_mm_pr nrkj_m2_S, nrkj_n2_S;
+ gmx_mm_pr cp_S, mdphi_S, mult_S;
+ gmx_mm_pr sin_S, cos_S;
+ gmx_mm_pr mddphi_S;
+ gmx_mm_pr sf_i_S, msf_l_S;
+ pbc_simd_t pbc_simd;
+
+ /* Ensure SIMD register alignment */
+ dr = gmx_simd_align_real(dr_array);
+ buf = gmx_simd_align_real(buf_array);
+
+ /* Extract aligned pointer for parameters and variables */
+ cp = buf + 0*UNROLL;
+ phi0 = buf + 1*UNROLL;
+ mult = buf + 2*UNROLL;
+ p = buf + 3*UNROLL;
+ q = buf + 4*UNROLL;
+ sf_i = buf + 5*UNROLL;
+ msf_l = buf + 6*UNROLL;
+
+ set_pbc_simd(pbc, &pbc_simd);
+
+ /* nbonds is the number of dihedrals times nfa1, here we step UNROLL dihs */
+ for (i = 0; (i < nbonds); i += UNROLL*nfa1)
+ {
+ /* Collect atoms quadruplets for UNROLL dihedrals.
+ * iu indexes into forceatoms, we should not let iu go beyond nbonds.
+ */
+ iu = i;
+ for (s = 0; s < UNROLL; s++)
+ {
+ type = forceatoms[iu];
+ ai[s] = forceatoms[iu+1];
+ aj[s] = forceatoms[iu+2];
+ ak[s] = forceatoms[iu+3];
+ al[s] = forceatoms[iu+4];
+
+ cp[s] = forceparams[type].pdihs.cpA;
+ phi0[s] = forceparams[type].pdihs.phiA*DEG2RAD;
+ mult[s] = forceparams[type].pdihs.mult;
+
+ /* At the end fill the arrays with identical entries */
+ if (iu + nfa1 < nbonds)
+ {
+ iu += nfa1;
+ }
+ }
+
+ /* Caclulate UNROLL dihedral angles at once */
+ dih_angle_simd(x, ai, aj, ak, al, &pbc_simd,
+ dr,
+ &phi_S,
+ &mx_S, &my_S, &mz_S,
+ &nx_S, &ny_S, &nz_S,
+ &nrkj_m2_S,
+ &nrkj_n2_S,
+ p, q);
+
+ cp_S = gmx_load_pr(cp);
+ phi0_S = gmx_load_pr(phi0);
+ mult_S = gmx_load_pr(mult);
+
+ mdphi_S = gmx_sub_pr(gmx_mul_pr(mult_S, phi_S), phi0_S);
+
+ /* Calculate UNROLL sines at once */
+ gmx_sincos_pr(mdphi_S, &sin_S, &cos_S);
+ mddphi_S = gmx_mul_pr(gmx_mul_pr(cp_S, mult_S), sin_S);
+ sf_i_S = gmx_mul_pr(mddphi_S, nrkj_m2_S);
+ msf_l_S = gmx_mul_pr(mddphi_S, nrkj_n2_S);
+
+ /* After this m?_S will contain f[i] */
+ mx_S = gmx_mul_pr(sf_i_S, mx_S);
+ my_S = gmx_mul_pr(sf_i_S, my_S);
+ mz_S = gmx_mul_pr(sf_i_S, mz_S);
+
+ /* After this m?_S will contain -f[l] */
+ nx_S = gmx_mul_pr(msf_l_S, nx_S);
+ ny_S = gmx_mul_pr(msf_l_S, ny_S);
+ nz_S = gmx_mul_pr(msf_l_S, nz_S);
+
+ gmx_store_pr(dr + 0*UNROLL, mx_S);
+ gmx_store_pr(dr + 1*UNROLL, my_S);
+ gmx_store_pr(dr + 2*UNROLL, mz_S);
+ gmx_store_pr(dr + 3*UNROLL, nx_S);
+ gmx_store_pr(dr + 4*UNROLL, ny_S);
+ gmx_store_pr(dr + 5*UNROLL, nz_S);
+
+ iu = i;
+ s = 0;
+ do
+ {
+ do_dih_fup_noshiftf_precalc(ai[s], aj[s], ak[s], al[s],
+ p[s], q[s],
+ dr[ XX *UNROLL+s],
+ dr[ YY *UNROLL+s],
+ dr[ ZZ *UNROLL+s],
+ dr[(DIM+XX)*UNROLL+s],
+ dr[(DIM+YY)*UNROLL+s],
+ dr[(DIM+ZZ)*UNROLL+s],
+ f);
+ s++;
+ iu += nfa1;
+ }
+ while (s < UNROLL && iu < nbonds);
+ }
+#undef UNROLL
+}
+
+#endif /* SIMD_BONDEDS */
+
+
+real idihs(int nbonds,
+ const t_iatom forceatoms[], const t_iparams forceparams[],
+ const rvec x[], rvec f[], rvec fshift[],
+ const t_pbc *pbc, const t_graph *g,
+ real lambda, real *dvdlambda,
+ const t_mdatoms gmx_unused *md, t_fcdata gmx_unused *fcd,
+ int gmx_unused *global_atom_index)
+{
+ int i, type, ai, aj, ak, al;
+ int t1, t2, t3;
+ real phi, phi0, dphi0, ddphi, sign, vtot;
+ rvec r_ij, r_kj, r_kl, m, n;
+ real L1, kk, dp, dp2, kA, kB, pA, pB, dvdl_term;
+
+ L1 = 1.0-lambda;
+ dvdl_term = 0;
+ vtot = 0.0;
+ for (i = 0; (i < nbonds); )
+ {
+ type = forceatoms[i++];
+ ai = forceatoms[i++];
+ aj = forceatoms[i++];
+ ak = forceatoms[i++];
+ al = forceatoms[i++];
+
+ phi = dih_angle(x[ai], x[aj], x[ak], x[al], pbc, r_ij, r_kj, r_kl, m, n,
+ &sign, &t1, &t2, &t3); /* 84 */
+
+ /* phi can jump if phi0 is close to Pi/-Pi, which will cause huge
+ * force changes if we just apply a normal harmonic.
+ * Instead, we first calculate phi-phi0 and take it modulo (-Pi,Pi).
+ * This means we will never have the periodicity problem, unless
+ * the dihedral is Pi away from phiO, which is very unlikely due to
+ * the potential.
+ */
+ kA = forceparams[type].harmonic.krA;
+ kB = forceparams[type].harmonic.krB;
+ pA = forceparams[type].harmonic.rA;
+ pB = forceparams[type].harmonic.rB;
+
+ kk = L1*kA + lambda*kB;
+ phi0 = (L1*pA + lambda*pB)*DEG2RAD;
+ dphi0 = (pB - pA)*DEG2RAD;
+
+ dp = phi-phi0;
+
+ make_dp_periodic(&dp);
+
+ dp2 = dp*dp;
+
+ vtot += 0.5*kk*dp2;
+ ddphi = -kk*dp;
+
+ dvdl_term += 0.5*(kB - kA)*dp2 - kk*dphi0*dp;
+
+ do_dih_fup(ai, aj, ak, al, (real)(-ddphi), r_ij, r_kj, r_kl, m, n,
+ f, fshift, pbc, g, x, t1, t2, t3); /* 112 */
+ /* 218 TOTAL */
+#ifdef DEBUG
+ if (debug)
+ {
+ fprintf(debug, "idih: (%d,%d,%d,%d) phi=%g\n",
+ ai, aj, ak, al, phi);
+ }
+#endif
+ }
+
+ *dvdlambda += dvdl_term;
+ return vtot;
+}
+
+
+/*! \brief returns dx, rdist, and dpdl for functions posres() and fbposres()
+ */
+static void posres_dx(const rvec x, const rvec pos0A, const rvec pos0B,
+ const rvec comA_sc, const rvec comB_sc,
+ real lambda,
+ t_pbc *pbc, int refcoord_scaling, int npbcdim,
+ rvec dx, rvec rdist, rvec dpdl)
+{
+ int m, d;
+ real posA, posB, L1, ref = 0.;
+ rvec pos;
+
+ L1 = 1.0-lambda;
+
+ for (m = 0; m < DIM; m++)
+ {
+ posA = pos0A[m];
+ posB = pos0B[m];
+ if (m < npbcdim)
+ {
+ switch (refcoord_scaling)
+ {
+ case erscNO:
+ ref = 0;
+ rdist[m] = L1*posA + lambda*posB;
+ dpdl[m] = posB - posA;
+ break;
+ case erscALL:
+ /* Box relative coordinates are stored for dimensions with pbc */
+ posA *= pbc->box[m][m];
+ posB *= pbc->box[m][m];
+ for (d = m+1; d < npbcdim; d++)
+ {
+ posA += pos0A[d]*pbc->box[d][m];
+ posB += pos0B[d]*pbc->box[d][m];
+ }
+ ref = L1*posA + lambda*posB;
+ rdist[m] = 0;
+ dpdl[m] = posB - posA;
+ break;
+ case erscCOM:
+ ref = L1*comA_sc[m] + lambda*comB_sc[m];
+ rdist[m] = L1*posA + lambda*posB;
+ dpdl[m] = comB_sc[m] - comA_sc[m] + posB - posA;
+ break;
+ default:
+ gmx_fatal(FARGS, "No such scaling method implemented");
+ }
+ }
+ else
+ {
+ ref = L1*posA + lambda*posB;
+ rdist[m] = 0;
+ dpdl[m] = posB - posA;
+ }
+
+ /* We do pbc_dx with ref+rdist,
+ * since with only ref we can be up to half a box vector wrong.
+ */
+ pos[m] = ref + rdist[m];
+ }
+
+ if (pbc)
+ {
+ pbc_dx(pbc, x, pos, dx);
+ }
+ else
+ {
+ rvec_sub(x, pos, dx);
+ }
+}
+
+/*! \brief Adds forces of flat-bottomed positions restraints to f[]
+ * and fixes vir_diag. Returns the flat-bottomed potential. */
+real fbposres(int nbonds,
+ const t_iatom forceatoms[], const t_iparams forceparams[],
+ const rvec x[], rvec f[], rvec vir_diag,
+ t_pbc *pbc,
+ int refcoord_scaling, int ePBC, rvec com)
+/* compute flat-bottomed positions restraints */
+{
+ int i, ai, m, d, type, npbcdim = 0, fbdim;
+ const t_iparams *pr;
+ real vtot, kk, v;
+ real ref = 0, dr, dr2, rpot, rfb, rfb2, fact, invdr;
+ rvec com_sc, rdist, pos, dx, dpdl, fm;
+ gmx_bool bInvert;
+
+ npbcdim = ePBC2npbcdim(ePBC);
+
+ if (refcoord_scaling == erscCOM)
+ {
+ clear_rvec(com_sc);
+ for (m = 0; m < npbcdim; m++)
+ {
+ for (d = m; d < npbcdim; d++)
+ {
+ com_sc[m] += com[d]*pbc->box[d][m];
+ }
+ }
+ }
+
+ vtot = 0.0;
+ for (i = 0; (i < nbonds); )
+ {
+ type = forceatoms[i++];
+ ai = forceatoms[i++];
+ pr = &forceparams[type];
+
+ /* same calculation as for normal posres, but with identical A and B states, and lambda==0 */
+ posres_dx(x[ai], forceparams[type].fbposres.pos0, forceparams[type].fbposres.pos0,
+ com_sc, com_sc, 0.0,
+ pbc, refcoord_scaling, npbcdim,
+ dx, rdist, dpdl);
+
+ clear_rvec(fm);
+ v = 0.0;
+
+ kk = pr->fbposres.k;
+ rfb = pr->fbposres.r;
+ rfb2 = sqr(rfb);
+
+ /* with rfb<0, push particle out of the sphere/cylinder/layer */
+ bInvert = FALSE;
+ if (rfb < 0.)
+ {
+ bInvert = TRUE;
+ rfb = -rfb;
+ }
+
+ switch (pr->fbposres.geom)
+ {
+ case efbposresSPHERE:
+ /* spherical flat-bottom posres */
+ dr2 = norm2(dx);
+ if (dr2 > 0.0 &&
+ ( (dr2 > rfb2 && bInvert == FALSE ) || (dr2 < rfb2 && bInvert == TRUE ) )
+ )
+ {
+ dr = sqrt(dr2);
+ v = 0.5*kk*sqr(dr - rfb);
+ fact = -kk*(dr-rfb)/dr; /* Force pointing to the center pos0 */
+ svmul(fact, dx, fm);
+ }
+ break;
+ case efbposresCYLINDER:
+ /* cylidrical flat-bottom posres in x-y plane. fm[ZZ] = 0. */
+ dr2 = sqr(dx[XX])+sqr(dx[YY]);
+ if (dr2 > 0.0 &&
+ ( (dr2 > rfb2 && bInvert == FALSE ) || (dr2 < rfb2 && bInvert == TRUE ) )
+ )
+ {
+ dr = sqrt(dr2);
+ invdr = 1./dr;
+ v = 0.5*kk*sqr(dr - rfb);
+ fm[XX] = -kk*(dr-rfb)*dx[XX]*invdr; /* Force pointing to the center */
+ fm[YY] = -kk*(dr-rfb)*dx[YY]*invdr;
+ }
+ break;
+ case efbposresX: /* fbdim=XX */
+ case efbposresY: /* fbdim=YY */
+ case efbposresZ: /* fbdim=ZZ */
+ /* 1D flat-bottom potential */
+ fbdim = pr->fbposres.geom - efbposresX;
+ dr = dx[fbdim];
+ if ( ( dr > rfb && bInvert == FALSE ) || ( 0 < dr && dr < rfb && bInvert == TRUE ) )
+ {
+ v = 0.5*kk*sqr(dr - rfb);
+ fm[fbdim] = -kk*(dr - rfb);
+ }
+ else if ( (dr < (-rfb) && bInvert == FALSE ) || ( (-rfb) < dr && dr < 0 && bInvert == TRUE ))
+ {
+ v = 0.5*kk*sqr(dr + rfb);
+ fm[fbdim] = -kk*(dr + rfb);
+ }
+ break;
+ }
+
+ vtot += v;
+
+ for (m = 0; (m < DIM); m++)
+ {
+ f[ai][m] += fm[m];
+ /* Here we correct for the pbc_dx which included rdist */
+ vir_diag[m] -= 0.5*(dx[m] + rdist[m])*fm[m];
+ }
+ }
+
+ return vtot;
+}
+
+
+real posres(int nbonds,
+ const t_iatom forceatoms[], const t_iparams forceparams[],
+ const rvec x[], rvec f[], rvec vir_diag,
+ t_pbc *pbc,
+ real lambda, real *dvdlambda,
+ int refcoord_scaling, int ePBC, rvec comA, rvec comB)
+{
+ int i, ai, m, d, type, ki, npbcdim = 0;
+ const t_iparams *pr;
+ real L1;
+ real vtot, kk, fm;
+ real posA, posB, ref = 0;
+ rvec comA_sc, comB_sc, rdist, dpdl, pos, dx;
+ gmx_bool bForceValid = TRUE;
+
+ if ((f == NULL) || (vir_diag == NULL)) /* should both be null together! */
+ {
+ bForceValid = FALSE;
+ }
+
+ npbcdim = ePBC2npbcdim(ePBC);
+
+ if (refcoord_scaling == erscCOM)
+ {
+ clear_rvec(comA_sc);
+ clear_rvec(comB_sc);
+ for (m = 0; m < npbcdim; m++)
+ {
+ for (d = m; d < npbcdim; d++)
+ {
+ comA_sc[m] += comA[d]*pbc->box[d][m];
+ comB_sc[m] += comB[d]*pbc->box[d][m];
+ }
+ }
+ }
+
+ L1 = 1.0 - lambda;
+
+ vtot = 0.0;
+ for (i = 0; (i < nbonds); )
+ {
+ type = forceatoms[i++];
+ ai = forceatoms[i++];
+ pr = &forceparams[type];
+
+ /* return dx, rdist, and dpdl */
+ posres_dx(x[ai], forceparams[type].posres.pos0A, forceparams[type].posres.pos0B,
+ comA_sc, comB_sc, lambda,
+ pbc, refcoord_scaling, npbcdim,
+ dx, rdist, dpdl);
+
+ for (m = 0; (m < DIM); m++)
+ {
+ kk = L1*pr->posres.fcA[m] + lambda*pr->posres.fcB[m];
+ fm = -kk*dx[m];
+ vtot += 0.5*kk*dx[m]*dx[m];
+ *dvdlambda +=
+ 0.5*(pr->posres.fcB[m] - pr->posres.fcA[m])*dx[m]*dx[m]
+ -fm*dpdl[m];
+
+ /* Here we correct for the pbc_dx which included rdist */
+ if (bForceValid)
+ {
+ f[ai][m] += fm;
+ vir_diag[m] -= 0.5*(dx[m] + rdist[m])*fm;
+ }
+ }
+ }
+
+ return vtot;
+}
+
+static real low_angres(int nbonds,
+ const t_iatom forceatoms[], const t_iparams forceparams[],
+ const rvec x[], rvec f[], rvec fshift[],
+ const t_pbc *pbc, const t_graph *g,
+ real lambda, real *dvdlambda,
+ gmx_bool bZAxis)
+{
+ int i, m, type, ai, aj, ak, al;
+ int t1, t2;
+ real phi, cos_phi, cos_phi2, vid, vtot, dVdphi;
+ rvec r_ij, r_kl, f_i, f_k = {0, 0, 0};
+ real st, sth, nrij2, nrkl2, c, cij, ckl;
+
+ ivec dt;
+ t2 = 0; /* avoid warning with gcc-3.3. It is never used uninitialized */
+
+ vtot = 0.0;
+ ak = al = 0; /* to avoid warnings */
+ for (i = 0; i < nbonds; )
+ {
+ type = forceatoms[i++];
+ ai = forceatoms[i++];
+ aj = forceatoms[i++];
+ t1 = pbc_rvec_sub(pbc, x[aj], x[ai], r_ij); /* 3 */
+ if (!bZAxis)
+ {
+ ak = forceatoms[i++];
+ al = forceatoms[i++];
+ t2 = pbc_rvec_sub(pbc, x[al], x[ak], r_kl); /* 3 */
+ }
+ else
+ {
+ r_kl[XX] = 0;
+ r_kl[YY] = 0;
+ r_kl[ZZ] = 1;
+ }
+
+ cos_phi = cos_angle(r_ij, r_kl); /* 25 */
+ phi = acos(cos_phi); /* 10 */
+
+ *dvdlambda += dopdihs_min(forceparams[type].pdihs.cpA,
+ forceparams[type].pdihs.cpB,
+ forceparams[type].pdihs.phiA,
+ forceparams[type].pdihs.phiB,
+ forceparams[type].pdihs.mult,
+ phi, lambda, &vid, &dVdphi); /* 40 */
+
+ vtot += vid;
+
+ cos_phi2 = sqr(cos_phi); /* 1 */
+ if (cos_phi2 < 1)
+ {
+ st = -dVdphi*gmx_invsqrt(1 - cos_phi2); /* 12 */
+ sth = st*cos_phi; /* 1 */
+ nrij2 = iprod(r_ij, r_ij); /* 5 */
+ nrkl2 = iprod(r_kl, r_kl); /* 5 */
+
+ c = st*gmx_invsqrt(nrij2*nrkl2); /* 11 */
+ cij = sth/nrij2; /* 10 */
+ ckl = sth/nrkl2; /* 10 */
+
+ for (m = 0; m < DIM; m++) /* 18+18 */
+ {
+ f_i[m] = (c*r_kl[m]-cij*r_ij[m]);
+ f[ai][m] += f_i[m];
+ f[aj][m] -= f_i[m];
+ if (!bZAxis)
+ {
+ f_k[m] = (c*r_ij[m]-ckl*r_kl[m]);
+ f[ak][m] += f_k[m];
+ f[al][m] -= f_k[m];
+ }
+ }
+
+ if (g)
+ {
+ ivec_sub(SHIFT_IVEC(g, ai), SHIFT_IVEC(g, aj), dt);
+ t1 = IVEC2IS(dt);
+ }
+ rvec_inc(fshift[t1], f_i);
+ rvec_dec(fshift[CENTRAL], f_i);
+ if (!bZAxis)
+ {
+ if (g)
+ {
+ ivec_sub(SHIFT_IVEC(g, ak), SHIFT_IVEC(g, al), dt);
+ t2 = IVEC2IS(dt);
+ }
+ rvec_inc(fshift[t2], f_k);
+ rvec_dec(fshift[CENTRAL], f_k);
+ }
+ }
+ }
+
+ return vtot; /* 184 / 157 (bZAxis) total */
+}
+
+real angres(int nbonds,
+ const t_iatom forceatoms[], const t_iparams forceparams[],
+ const rvec x[], rvec f[], rvec fshift[],
+ const t_pbc *pbc, const t_graph *g,
+ real lambda, real *dvdlambda,
+ const t_mdatoms gmx_unused *md, t_fcdata gmx_unused *fcd,
+ int gmx_unused *global_atom_index)
+{
+ return low_angres(nbonds, forceatoms, forceparams, x, f, fshift, pbc, g,
+ lambda, dvdlambda, FALSE);
+}
+
+real angresz(int nbonds,
+ const t_iatom forceatoms[], const t_iparams forceparams[],
+ const rvec x[], rvec f[], rvec fshift[],
+ const t_pbc *pbc, const t_graph *g,
+ real lambda, real *dvdlambda,
+ const t_mdatoms gmx_unused *md, t_fcdata gmx_unused *fcd,
+ int gmx_unused *global_atom_index)
+{
+ return low_angres(nbonds, forceatoms, forceparams, x, f, fshift, pbc, g,
+ lambda, dvdlambda, TRUE);
+}
+
+real dihres(int nbonds,
+ const t_iatom forceatoms[], const t_iparams forceparams[],
+ const rvec x[], rvec f[], rvec fshift[],
+ const t_pbc *pbc, const t_graph *g,
+ real lambda, real *dvdlambda,
+ const t_mdatoms gmx_unused *md, t_fcdata gmx_unused *fcd,
+ int gmx_unused *global_atom_index)
+{
+ real vtot = 0;
+ int ai, aj, ak, al, i, k, type, t1, t2, t3;
+ real phi0A, phi0B, dphiA, dphiB, kfacA, kfacB, phi0, dphi, kfac;
+ real phi, ddphi, ddp, ddp2, dp, sign, d2r, fc, L1;
+ rvec r_ij, r_kj, r_kl, m, n;
+
+ L1 = 1.0-lambda;
+
+ d2r = DEG2RAD;
+ k = 0;
+
+ for (i = 0; (i < nbonds); )
+ {
+ type = forceatoms[i++];
+ ai = forceatoms[i++];
+ aj = forceatoms[i++];
+ ak = forceatoms[i++];
+ al = forceatoms[i++];
+
+ phi0A = forceparams[type].dihres.phiA*d2r;
+ dphiA = forceparams[type].dihres.dphiA*d2r;
+ kfacA = forceparams[type].dihres.kfacA;
+
+ phi0B = forceparams[type].dihres.phiB*d2r;
+ dphiB = forceparams[type].dihres.dphiB*d2r;
+ kfacB = forceparams[type].dihres.kfacB;
+
+ phi0 = L1*phi0A + lambda*phi0B;
+ dphi = L1*dphiA + lambda*dphiB;
+ kfac = L1*kfacA + lambda*kfacB;
+
+ phi = dih_angle(x[ai], x[aj], x[ak], x[al], pbc, r_ij, r_kj, r_kl, m, n,
+ &sign, &t1, &t2, &t3);
+ /* 84 flops */
+
+ if (debug)
+ {
+ fprintf(debug, "dihres[%d]: %d %d %d %d : phi=%f, dphi=%f, kfac=%f\n",
+ k++, ai, aj, ak, al, phi0, dphi, kfac);
+ }
+ /* phi can jump if phi0 is close to Pi/-Pi, which will cause huge
+ * force changes if we just apply a normal harmonic.
+ * Instead, we first calculate phi-phi0 and take it modulo (-Pi,Pi).
+ * This means we will never have the periodicity problem, unless
+ * the dihedral is Pi away from phiO, which is very unlikely due to
+ * the potential.
+ */
+ dp = phi-phi0;
+ make_dp_periodic(&dp);
+
+ if (dp > dphi)
+ {
+ ddp = dp-dphi;
+ }
+ else if (dp < -dphi)
+ {
+ ddp = dp+dphi;
+ }
+ else
+ {
+ ddp = 0;
+ }
+
+ if (ddp != 0.0)
+ {
+ ddp2 = ddp*ddp;
+ vtot += 0.5*kfac*ddp2;
+ ddphi = kfac*ddp;
+
+ *dvdlambda += 0.5*(kfacB - kfacA)*ddp2;
+ /* lambda dependence from changing restraint distances */
+ if (ddp > 0)
+ {
+ *dvdlambda -= kfac*ddp*((dphiB - dphiA)+(phi0B - phi0A));
+ }
+ else if (ddp < 0)
+ {
+ *dvdlambda += kfac*ddp*((dphiB - dphiA)-(phi0B - phi0A));
+ }
+ do_dih_fup(ai, aj, ak, al, ddphi, r_ij, r_kj, r_kl, m, n,
+ f, fshift, pbc, g, x, t1, t2, t3); /* 112 */
+ }
+ }
+ return vtot;
+}
+
+
+real unimplemented(int gmx_unused nbonds,
+ const t_iatom gmx_unused forceatoms[], const t_iparams gmx_unused forceparams[],
+ const rvec gmx_unused x[], rvec gmx_unused f[], rvec gmx_unused fshift[],
+ const t_pbc gmx_unused *pbc, const t_graph gmx_unused *g,
+ real gmx_unused lambda, real gmx_unused *dvdlambda,
+ const t_mdatoms gmx_unused *md, t_fcdata gmx_unused *fcd,
+ int gmx_unused *global_atom_index)
+{
+ gmx_impl("*** you are using a not implemented function");
+
+ return 0.0; /* To make the compiler happy */
+}
+
+real rbdihs(int nbonds,
+ const t_iatom forceatoms[], const t_iparams forceparams[],
+ const rvec x[], rvec f[], rvec fshift[],
+ const t_pbc *pbc, const t_graph *g,
+ real lambda, real *dvdlambda,
+ const t_mdatoms gmx_unused *md, t_fcdata gmx_unused *fcd,
+ int gmx_unused *global_atom_index)
+{
+ const real c0 = 0.0, c1 = 1.0, c2 = 2.0, c3 = 3.0, c4 = 4.0, c5 = 5.0;
+ int type, ai, aj, ak, al, i, j;
+ int t1, t2, t3;
+ rvec r_ij, r_kj, r_kl, m, n;
+ real parmA[NR_RBDIHS];
+ real parmB[NR_RBDIHS];
+ real parm[NR_RBDIHS];
+ real cos_phi, phi, rbp, rbpBA;
+ real v, sign, ddphi, sin_phi;
+ real cosfac, vtot;
+ real L1 = 1.0-lambda;
+ real dvdl_term = 0;
+
+ vtot = 0.0;
+ for (i = 0; (i < nbonds); )
+ {
+ type = forceatoms[i++];
+ ai = forceatoms[i++];
+ aj = forceatoms[i++];
+ ak = forceatoms[i++];
+ al = forceatoms[i++];
+
+ phi = dih_angle(x[ai], x[aj], x[ak], x[al], pbc, r_ij, r_kj, r_kl, m, n,
+ &sign, &t1, &t2, &t3); /* 84 */
+
+ /* Change to polymer convention */
+ if (phi < c0)
+ {
+ phi += M_PI;
+ }
+ else
+ {
+ phi -= M_PI; /* 1 */
+
+ }
+ cos_phi = cos(phi);
+ /* Beware of accuracy loss, cannot use 1-sqrt(cos^2) ! */
+ sin_phi = sin(phi);
+
+ for (j = 0; (j < NR_RBDIHS); j++)
+ {
+ parmA[j] = forceparams[type].rbdihs.rbcA[j];
+ parmB[j] = forceparams[type].rbdihs.rbcB[j];
+ parm[j] = L1*parmA[j]+lambda*parmB[j];
+ }
+ /* Calculate cosine powers */
+ /* Calculate the energy */
+ /* Calculate the derivative */
+
+ v = parm[0];
+ dvdl_term += (parmB[0]-parmA[0]);
+ ddphi = c0;
+ cosfac = c1;
+
+ rbp = parm[1];
+ rbpBA = parmB[1]-parmA[1];
+ ddphi += rbp*cosfac;
+ cosfac *= cos_phi;
+ v += cosfac*rbp;
+ dvdl_term += cosfac*rbpBA;
+ rbp = parm[2];
+ rbpBA = parmB[2]-parmA[2];
+ ddphi += c2*rbp*cosfac;
+ cosfac *= cos_phi;
+ v += cosfac*rbp;
+ dvdl_term += cosfac*rbpBA;
+ rbp = parm[3];
+ rbpBA = parmB[3]-parmA[3];
+ ddphi += c3*rbp*cosfac;
+ cosfac *= cos_phi;
+ v += cosfac*rbp;
+ dvdl_term += cosfac*rbpBA;
+ rbp = parm[4];
+ rbpBA = parmB[4]-parmA[4];
+ ddphi += c4*rbp*cosfac;
+ cosfac *= cos_phi;
+ v += cosfac*rbp;
+ dvdl_term += cosfac*rbpBA;
+ rbp = parm[5];
+ rbpBA = parmB[5]-parmA[5];
+ ddphi += c5*rbp*cosfac;
+ cosfac *= cos_phi;
+ v += cosfac*rbp;
+ dvdl_term += cosfac*rbpBA;
+
+ ddphi = -ddphi*sin_phi; /* 11 */
+
+ do_dih_fup(ai, aj, ak, al, ddphi, r_ij, r_kj, r_kl, m, n,
+ f, fshift, pbc, g, x, t1, t2, t3); /* 112 */
+ vtot += v;
+ }
+ *dvdlambda += dvdl_term;
+
+ return vtot;
+}
+
+int cmap_setup_grid_index(int ip, int grid_spacing, int *ipm1, int *ipp1, int *ipp2)
+{
+ int im1, ip1, ip2;
+
+ if (ip < 0)
+ {
+ ip = ip + grid_spacing - 1;
+ }
+ else if (ip > grid_spacing)
+ {
+ ip = ip - grid_spacing - 1;
+ }
+
+ im1 = ip - 1;
+ ip1 = ip + 1;
+ ip2 = ip + 2;
+
+ if (ip == 0)
+ {
+ im1 = grid_spacing - 1;
+ }
+ else if (ip == grid_spacing-2)
+ {
+ ip2 = 0;
+ }
+ else if (ip == grid_spacing-1)
+ {
+ ip1 = 0;
+ ip2 = 1;
+ }
+
+ *ipm1 = im1;
+ *ipp1 = ip1;
+ *ipp2 = ip2;
+
+ return ip;
+
+}
+
+real cmap_dihs(int nbonds,
+ const t_iatom forceatoms[], const t_iparams forceparams[],
+ const gmx_cmap_t *cmap_grid,
+ const rvec x[], rvec f[], rvec fshift[],
+ const t_pbc *pbc, const t_graph *g,
+ real gmx_unused lambda, real gmx_unused *dvdlambda,
+ const t_mdatoms gmx_unused *md, t_fcdata gmx_unused *fcd,
+ int gmx_unused *global_atom_index)
+{
+ int i, j, k, n, idx;
+ int ai, aj, ak, al, am;
+ int a1i, a1j, a1k, a1l, a2i, a2j, a2k, a2l;
+ int type, cmapA;
+ int t11, t21, t31, t12, t22, t32;
+ int iphi1, ip1m1, ip1p1, ip1p2;
+ int iphi2, ip2m1, ip2p1, ip2p2;
+ int l1, l2, l3, l4;
+ int pos1, pos2, pos3, pos4, tmp;
+
+ real ty[4], ty1[4], ty2[4], ty12[4], tc[16], tx[16];
+ real phi1, psi1, cos_phi1, sin_phi1, sign1, xphi1;
+ real phi2, psi2, cos_phi2, sin_phi2, sign2, xphi2;
+ real dx, xx, tt, tu, e, df1, df2, ddf1, ddf2, ddf12, vtot;
+ real ra21, rb21, rg21, rg1, rgr1, ra2r1, rb2r1, rabr1;
+ real ra22, rb22, rg22, rg2, rgr2, ra2r2, rb2r2, rabr2;
+ real fg1, hg1, fga1, hgb1, gaa1, gbb1;
+ real fg2, hg2, fga2, hgb2, gaa2, gbb2;
+ real fac;
+
+ rvec r1_ij, r1_kj, r1_kl, m1, n1;
+ rvec r2_ij, r2_kj, r2_kl, m2, n2;
+ rvec f1_i, f1_j, f1_k, f1_l;
+ rvec f2_i, f2_j, f2_k, f2_l;
+ rvec a1, b1, a2, b2;
+ rvec f1, g1, h1, f2, g2, h2;
+ rvec dtf1, dtg1, dth1, dtf2, dtg2, dth2;
+ ivec jt1, dt1_ij, dt1_kj, dt1_lj;
+ ivec jt2, dt2_ij, dt2_kj, dt2_lj;
+
+ const real *cmapd;
+
+ int loop_index[4][4] = {
+ {0, 4, 8, 12},
+ {1, 5, 9, 13},
+ {2, 6, 10, 14},
+ {3, 7, 11, 15}
+ };
+
+ /* Total CMAP energy */
+ vtot = 0;
+
+ for (n = 0; n < nbonds; )
+ {
+ /* Five atoms are involved in the two torsions */
+ type = forceatoms[n++];
+ ai = forceatoms[n++];
+ aj = forceatoms[n++];
+ ak = forceatoms[n++];
+ al = forceatoms[n++];
+ am = forceatoms[n++];
+
+ /* Which CMAP type is this */
+ cmapA = forceparams[type].cmap.cmapA;
+ cmapd = cmap_grid->cmapdata[cmapA].cmap;
+
+ /* First torsion */
+ a1i = ai;
+ a1j = aj;
+ a1k = ak;
+ a1l = al;
+
+ phi1 = dih_angle(x[a1i], x[a1j], x[a1k], x[a1l], pbc, r1_ij, r1_kj, r1_kl, m1, n1,
+ &sign1, &t11, &t21, &t31); /* 84 */
+
+ cos_phi1 = cos(phi1);
+
+ a1[0] = r1_ij[1]*r1_kj[2]-r1_ij[2]*r1_kj[1];
+ a1[1] = r1_ij[2]*r1_kj[0]-r1_ij[0]*r1_kj[2];
+ a1[2] = r1_ij[0]*r1_kj[1]-r1_ij[1]*r1_kj[0]; /* 9 */
+
+ b1[0] = r1_kl[1]*r1_kj[2]-r1_kl[2]*r1_kj[1];
+ b1[1] = r1_kl[2]*r1_kj[0]-r1_kl[0]*r1_kj[2];
+ b1[2] = r1_kl[0]*r1_kj[1]-r1_kl[1]*r1_kj[0]; /* 9 */
+
+ tmp = pbc_rvec_sub(pbc, x[a1l], x[a1k], h1);
+
+ ra21 = iprod(a1, a1); /* 5 */
+ rb21 = iprod(b1, b1); /* 5 */
+ rg21 = iprod(r1_kj, r1_kj); /* 5 */
+ rg1 = sqrt(rg21);
+
+ rgr1 = 1.0/rg1;
+ ra2r1 = 1.0/ra21;
+ rb2r1 = 1.0/rb21;
+ rabr1 = sqrt(ra2r1*rb2r1);
+
+ sin_phi1 = rg1 * rabr1 * iprod(a1, h1) * (-1);
+
+ if (cos_phi1 < -0.5 || cos_phi1 > 0.5)
+ {
+ phi1 = asin(sin_phi1);
+
+ if (cos_phi1 < 0)
+ {
+ if (phi1 > 0)
+ {
+ phi1 = M_PI - phi1;
+ }
+ else
+ {
+ phi1 = -M_PI - phi1;
+ }
+ }
+ }
+ else
+ {
+ phi1 = acos(cos_phi1);
+
+ if (sin_phi1 < 0)
+ {
+ phi1 = -phi1;
+ }
+ }
+
+ xphi1 = phi1 + M_PI; /* 1 */
+
+ /* Second torsion */
+ a2i = aj;
+ a2j = ak;
+ a2k = al;
+ a2l = am;
+
+ phi2 = dih_angle(x[a2i], x[a2j], x[a2k], x[a2l], pbc, r2_ij, r2_kj, r2_kl, m2, n2,
+ &sign2, &t12, &t22, &t32); /* 84 */
+
+ cos_phi2 = cos(phi2);
+
+ a2[0] = r2_ij[1]*r2_kj[2]-r2_ij[2]*r2_kj[1];
+ a2[1] = r2_ij[2]*r2_kj[0]-r2_ij[0]*r2_kj[2];
+ a2[2] = r2_ij[0]*r2_kj[1]-r2_ij[1]*r2_kj[0]; /* 9 */
+
+ b2[0] = r2_kl[1]*r2_kj[2]-r2_kl[2]*r2_kj[1];
+ b2[1] = r2_kl[2]*r2_kj[0]-r2_kl[0]*r2_kj[2];
+ b2[2] = r2_kl[0]*r2_kj[1]-r2_kl[1]*r2_kj[0]; /* 9 */
+
+ tmp = pbc_rvec_sub(pbc, x[a2l], x[a2k], h2);
+
+ ra22 = iprod(a2, a2); /* 5 */
+ rb22 = iprod(b2, b2); /* 5 */
+ rg22 = iprod(r2_kj, r2_kj); /* 5 */
+ rg2 = sqrt(rg22);
+
+ rgr2 = 1.0/rg2;
+ ra2r2 = 1.0/ra22;
+ rb2r2 = 1.0/rb22;
+ rabr2 = sqrt(ra2r2*rb2r2);
+
+ sin_phi2 = rg2 * rabr2 * iprod(a2, h2) * (-1);
+
+ if (cos_phi2 < -0.5 || cos_phi2 > 0.5)
+ {
+ phi2 = asin(sin_phi2);
+
+ if (cos_phi2 < 0)
+ {
+ if (phi2 > 0)
+ {
+ phi2 = M_PI - phi2;
+ }
+ else
+ {
+ phi2 = -M_PI - phi2;
+ }
+ }
+ }
+ else
+ {
+ phi2 = acos(cos_phi2);
+
+ if (sin_phi2 < 0)
+ {
+ phi2 = -phi2;
+ }
+ }
+
+ xphi2 = phi2 + M_PI; /* 1 */
+
+ /* Range mangling */
+ if (xphi1 < 0)
+ {
+ xphi1 = xphi1 + 2*M_PI;
+ }
+ else if (xphi1 >= 2*M_PI)
+ {
+ xphi1 = xphi1 - 2*M_PI;
+ }
+
+ if (xphi2 < 0)
+ {
+ xphi2 = xphi2 + 2*M_PI;
+ }
+ else if (xphi2 >= 2*M_PI)
+ {
+ xphi2 = xphi2 - 2*M_PI;
+ }
+
+ /* Number of grid points */
+ dx = 2*M_PI / cmap_grid->grid_spacing;
+
+ /* Where on the grid are we */
+ iphi1 = (int)(xphi1/dx);
+ iphi2 = (int)(xphi2/dx);
+
+ iphi1 = cmap_setup_grid_index(iphi1, cmap_grid->grid_spacing, &ip1m1, &ip1p1, &ip1p2);
+ iphi2 = cmap_setup_grid_index(iphi2, cmap_grid->grid_spacing, &ip2m1, &ip2p1, &ip2p2);
+
+ pos1 = iphi1*cmap_grid->grid_spacing+iphi2;
+ pos2 = ip1p1*cmap_grid->grid_spacing+iphi2;
+ pos3 = ip1p1*cmap_grid->grid_spacing+ip2p1;
+ pos4 = iphi1*cmap_grid->grid_spacing+ip2p1;
+
+ ty[0] = cmapd[pos1*4];
+ ty[1] = cmapd[pos2*4];
+ ty[2] = cmapd[pos3*4];
+ ty[3] = cmapd[pos4*4];
+
+ ty1[0] = cmapd[pos1*4+1];
+ ty1[1] = cmapd[pos2*4+1];
+ ty1[2] = cmapd[pos3*4+1];
+ ty1[3] = cmapd[pos4*4+1];
+
+ ty2[0] = cmapd[pos1*4+2];
+ ty2[1] = cmapd[pos2*4+2];
+ ty2[2] = cmapd[pos3*4+2];
+ ty2[3] = cmapd[pos4*4+2];
+
+ ty12[0] = cmapd[pos1*4+3];
+ ty12[1] = cmapd[pos2*4+3];
+ ty12[2] = cmapd[pos3*4+3];
+ ty12[3] = cmapd[pos4*4+3];
+
+ /* Switch to degrees */
+ dx = 360.0 / cmap_grid->grid_spacing;
+ xphi1 = xphi1 * RAD2DEG;
+ xphi2 = xphi2 * RAD2DEG;
+
+ for (i = 0; i < 4; i++) /* 16 */
+ {
+ tx[i] = ty[i];
+ tx[i+4] = ty1[i]*dx;
+ tx[i+8] = ty2[i]*dx;
+ tx[i+12] = ty12[i]*dx*dx;
+ }
+
+ idx = 0;
+ for (i = 0; i < 4; i++) /* 1056 */
+ {
+ for (j = 0; j < 4; j++)
+ {
+ xx = 0;
+ for (k = 0; k < 16; k++)
+ {
+ xx = xx + cmap_coeff_matrix[k*16+idx]*tx[k];
+ }
+
+ idx++;
+ tc[i*4+j] = xx;
+ }
+ }
+
+ tt = (xphi1-iphi1*dx)/dx;
+ tu = (xphi2-iphi2*dx)/dx;
+
+ e = 0;
+ df1 = 0;
+ df2 = 0;
+ ddf1 = 0;
+ ddf2 = 0;
+ ddf12 = 0;
+
+ for (i = 3; i >= 0; i--)
+ {
+ l1 = loop_index[i][3];
+ l2 = loop_index[i][2];
+ l3 = loop_index[i][1];
+
+ e = tt * e + ((tc[i*4+3]*tu+tc[i*4+2])*tu + tc[i*4+1])*tu+tc[i*4];
+ df1 = tu * df1 + (3.0*tc[l1]*tt+2.0*tc[l2])*tt+tc[l3];
+ df2 = tt * df2 + (3.0*tc[i*4+3]*tu+2.0*tc[i*4+2])*tu+tc[i*4+1];
+ ddf1 = tu * ddf1 + 2.0*3.0*tc[l1]*tt+2.0*tc[l2];
+ ddf2 = tt * ddf2 + 2.0*3.0*tc[4*i+3]*tu+2.0*tc[4*i+2];
+ }
+
+ ddf12 = tc[5] + 2.0*tc[9]*tt + 3.0*tc[13]*tt*tt + 2.0*tu*(tc[6]+2.0*tc[10]*tt+3.0*tc[14]*tt*tt) +
+ 3.0*tu*tu*(tc[7]+2.0*tc[11]*tt+3.0*tc[15]*tt*tt);
+
+ fac = RAD2DEG/dx;
+ df1 = df1 * fac;
+ df2 = df2 * fac;
+ ddf1 = ddf1 * fac * fac;
+ ddf2 = ddf2 * fac * fac;
+ ddf12 = ddf12 * fac * fac;
+
+ /* CMAP energy */
+ vtot += e;
+
+ /* Do forces - first torsion */
+ fg1 = iprod(r1_ij, r1_kj);
+ hg1 = iprod(r1_kl, r1_kj);
+ fga1 = fg1*ra2r1*rgr1;
+ hgb1 = hg1*rb2r1*rgr1;
+ gaa1 = -ra2r1*rg1;
+ gbb1 = rb2r1*rg1;
+
+ for (i = 0; i < DIM; i++)
+ {
+ dtf1[i] = gaa1 * a1[i];
+ dtg1[i] = fga1 * a1[i] - hgb1 * b1[i];
+ dth1[i] = gbb1 * b1[i];
+
+ f1[i] = df1 * dtf1[i];
+ g1[i] = df1 * dtg1[i];
+ h1[i] = df1 * dth1[i];
+
+ f1_i[i] = f1[i];
+ f1_j[i] = -f1[i] - g1[i];
+ f1_k[i] = h1[i] + g1[i];
+ f1_l[i] = -h1[i];
+
+ f[a1i][i] = f[a1i][i] + f1_i[i];
+ f[a1j][i] = f[a1j][i] + f1_j[i]; /* - f1[i] - g1[i] */
+ f[a1k][i] = f[a1k][i] + f1_k[i]; /* h1[i] + g1[i] */
+ f[a1l][i] = f[a1l][i] + f1_l[i]; /* h1[i] */
+ }
+
+ /* Do forces - second torsion */
+ fg2 = iprod(r2_ij, r2_kj);
+ hg2 = iprod(r2_kl, r2_kj);
+ fga2 = fg2*ra2r2*rgr2;
+ hgb2 = hg2*rb2r2*rgr2;
+ gaa2 = -ra2r2*rg2;
+ gbb2 = rb2r2*rg2;
+
+ for (i = 0; i < DIM; i++)
+ {
+ dtf2[i] = gaa2 * a2[i];
+ dtg2[i] = fga2 * a2[i] - hgb2 * b2[i];
+ dth2[i] = gbb2 * b2[i];
+
+ f2[i] = df2 * dtf2[i];
+ g2[i] = df2 * dtg2[i];
+ h2[i] = df2 * dth2[i];
+
+ f2_i[i] = f2[i];
+ f2_j[i] = -f2[i] - g2[i];
+ f2_k[i] = h2[i] + g2[i];
+ f2_l[i] = -h2[i];
+
+ f[a2i][i] = f[a2i][i] + f2_i[i]; /* f2[i] */
+ f[a2j][i] = f[a2j][i] + f2_j[i]; /* - f2[i] - g2[i] */
+ f[a2k][i] = f[a2k][i] + f2_k[i]; /* h2[i] + g2[i] */
+ f[a2l][i] = f[a2l][i] + f2_l[i]; /* - h2[i] */
+ }
+
+ /* Shift forces */
+ if (g)
+ {
+ copy_ivec(SHIFT_IVEC(g, a1j), jt1);
+ ivec_sub(SHIFT_IVEC(g, a1i), jt1, dt1_ij);
+ ivec_sub(SHIFT_IVEC(g, a1k), jt1, dt1_kj);
+ ivec_sub(SHIFT_IVEC(g, a1l), jt1, dt1_lj);
+ t11 = IVEC2IS(dt1_ij);
+ t21 = IVEC2IS(dt1_kj);
+ t31 = IVEC2IS(dt1_lj);
+
+ copy_ivec(SHIFT_IVEC(g, a2j), jt2);
+ ivec_sub(SHIFT_IVEC(g, a2i), jt2, dt2_ij);
+ ivec_sub(SHIFT_IVEC(g, a2k), jt2, dt2_kj);
+ ivec_sub(SHIFT_IVEC(g, a2l), jt2, dt2_lj);
+ t12 = IVEC2IS(dt2_ij);
+ t22 = IVEC2IS(dt2_kj);
+ t32 = IVEC2IS(dt2_lj);
+ }
+ else if (pbc)
+ {
+ t31 = pbc_rvec_sub(pbc, x[a1l], x[a1j], h1);
+ t32 = pbc_rvec_sub(pbc, x[a2l], x[a2j], h2);
+ }
+ else
+ {
+ t31 = CENTRAL;
+ t32 = CENTRAL;
+ }
+
+ rvec_inc(fshift[t11], f1_i);
+ rvec_inc(fshift[CENTRAL], f1_j);
+ rvec_inc(fshift[t21], f1_k);
+ rvec_inc(fshift[t31], f1_l);
+
+ rvec_inc(fshift[t21], f2_i);
+ rvec_inc(fshift[CENTRAL], f2_j);
+ rvec_inc(fshift[t22], f2_k);
+ rvec_inc(fshift[t32], f2_l);
+ }
+ return vtot;
+}
+
+
+
+/***********************************************************
+ *
+ * G R O M O S 9 6 F U N C T I O N S
+ *
+ ***********************************************************/
+real g96harmonic(real kA, real kB, real xA, real xB, real x, real lambda,
+ real *V, real *F)
+{
+ const real half = 0.5;
+ real L1, kk, x0, dx, dx2;
+ real v, f, dvdlambda;
+
+ L1 = 1.0-lambda;
+ kk = L1*kA+lambda*kB;
+ x0 = L1*xA+lambda*xB;
+
+ dx = x-x0;
+ dx2 = dx*dx;
+
+ f = -kk*dx;
+ v = half*kk*dx2;
+ dvdlambda = half*(kB-kA)*dx2 + (xA-xB)*kk*dx;
+
+ *F = f;
+ *V = v;
+
+ return dvdlambda;
+
+ /* That was 21 flops */
+}
+
+real g96bonds(int nbonds,
+ const t_iatom forceatoms[], const t_iparams forceparams[],
+ const rvec x[], rvec f[], rvec fshift[],
+ const t_pbc *pbc, const t_graph *g,
+ real lambda, real *dvdlambda,
+ const t_mdatoms gmx_unused *md, t_fcdata gmx_unused *fcd,
+ int gmx_unused *global_atom_index)
+{
+ int i, m, ki, ai, aj, type;
+ real dr2, fbond, vbond, fij, vtot;
+ rvec dx;
+ ivec dt;
+
+ vtot = 0.0;
+ for (i = 0; (i < nbonds); )
+ {
+ type = forceatoms[i++];
+ ai = forceatoms[i++];
+ aj = forceatoms[i++];
+
+ ki = pbc_rvec_sub(pbc, x[ai], x[aj], dx); /* 3 */
+ dr2 = iprod(dx, dx); /* 5 */
+
+ *dvdlambda += g96harmonic(forceparams[type].harmonic.krA,
+ forceparams[type].harmonic.krB,
+ forceparams[type].harmonic.rA,
+ forceparams[type].harmonic.rB,
+ dr2, lambda, &vbond, &fbond);
+
+ vtot += 0.5*vbond; /* 1*/
+#ifdef DEBUG
+ if (debug)
+ {
+ fprintf(debug, "G96-BONDS: dr = %10g vbond = %10g fbond = %10g\n",
+ sqrt(dr2), vbond, fbond);
+ }
+#endif
+
+ if (g)
+ {
+ ivec_sub(SHIFT_IVEC(g, ai), SHIFT_IVEC(g, aj), dt);
+ ki = IVEC2IS(dt);
+ }
+ for (m = 0; (m < DIM); m++) /* 15 */
+ {
+ fij = fbond*dx[m];
+ f[ai][m] += fij;
+ f[aj][m] -= fij;
+ fshift[ki][m] += fij;
+ fshift[CENTRAL][m] -= fij;
+ }
+ } /* 44 TOTAL */
+ return vtot;
+}
+
+real g96bond_angle(const rvec xi, const rvec xj, const rvec xk, const t_pbc *pbc,
+ rvec r_ij, rvec r_kj,
+ int *t1, int *t2)
+/* Return value is the angle between the bonds i-j and j-k */
+{
+ real costh;
+
+ *t1 = pbc_rvec_sub(pbc, xi, xj, r_ij); /* 3 */
+ *t2 = pbc_rvec_sub(pbc, xk, xj, r_kj); /* 3 */
+
+ costh = cos_angle(r_ij, r_kj); /* 25 */
+ /* 41 TOTAL */
+ return costh;
+}
+
+real g96angles(int nbonds,
+ const t_iatom forceatoms[], const t_iparams forceparams[],
+ const rvec x[], rvec f[], rvec fshift[],
+ const t_pbc *pbc, const t_graph *g,
+ real lambda, real *dvdlambda,
+ const t_mdatoms gmx_unused *md, t_fcdata gmx_unused *fcd,
+ int gmx_unused *global_atom_index)
+{
+ int i, ai, aj, ak, type, m, t1, t2;
+ rvec r_ij, r_kj;
+ real cos_theta, dVdt, va, vtot;
+ real rij_1, rij_2, rkj_1, rkj_2, rijrkj_1;
+ rvec f_i, f_j, f_k;
+ ivec jt, dt_ij, dt_kj;
+
+ vtot = 0.0;
+ for (i = 0; (i < nbonds); )
+ {
+ type = forceatoms[i++];
+ ai = forceatoms[i++];
+ aj = forceatoms[i++];
+ ak = forceatoms[i++];
+
+ cos_theta = g96bond_angle(x[ai], x[aj], x[ak], pbc, r_ij, r_kj, &t1, &t2);
+
+ *dvdlambda += g96harmonic(forceparams[type].harmonic.krA,
+ forceparams[type].harmonic.krB,
+ forceparams[type].harmonic.rA,
+ forceparams[type].harmonic.rB,
+ cos_theta, lambda, &va, &dVdt);
+ vtot += va;
+
+ rij_1 = gmx_invsqrt(iprod(r_ij, r_ij));
+ rkj_1 = gmx_invsqrt(iprod(r_kj, r_kj));
+ rij_2 = rij_1*rij_1;
+ rkj_2 = rkj_1*rkj_1;
+ rijrkj_1 = rij_1*rkj_1; /* 23 */
+
+#ifdef DEBUG
+ if (debug)
+ {
+ fprintf(debug, "G96ANGLES: costheta = %10g vth = %10g dV/dct = %10g\n",
+ cos_theta, va, dVdt);
+ }
+#endif
+ for (m = 0; (m < DIM); m++) /* 42 */
+ {
+ f_i[m] = dVdt*(r_kj[m]*rijrkj_1 - r_ij[m]*rij_2*cos_theta);
+ f_k[m] = dVdt*(r_ij[m]*rijrkj_1 - r_kj[m]*rkj_2*cos_theta);
+ f_j[m] = -f_i[m]-f_k[m];
+ f[ai][m] += f_i[m];
+ f[aj][m] += f_j[m];
+ f[ak][m] += f_k[m];
+ }
+
+ if (g)
+ {
+ copy_ivec(SHIFT_IVEC(g, aj), jt);
+
+ ivec_sub(SHIFT_IVEC(g, ai), jt, dt_ij);
+ ivec_sub(SHIFT_IVEC(g, ak), jt, dt_kj);
+ t1 = IVEC2IS(dt_ij);
+ t2 = IVEC2IS(dt_kj);
+ }
+ rvec_inc(fshift[t1], f_i);
+ rvec_inc(fshift[CENTRAL], f_j);
+ rvec_inc(fshift[t2], f_k); /* 9 */
+ /* 163 TOTAL */
+ }
+ return vtot;
+}
+
+real cross_bond_bond(int nbonds,
+ const t_iatom forceatoms[], const t_iparams forceparams[],
+ const rvec x[], rvec f[], rvec fshift[],
+ const t_pbc *pbc, const t_graph *g,
+ real gmx_unused lambda, real gmx_unused *dvdlambda,
+ const t_mdatoms gmx_unused *md, t_fcdata gmx_unused *fcd,
+ int gmx_unused *global_atom_index)
+{
+ /* Potential from Lawrence and Skimmer, Chem. Phys. Lett. 372 (2003)
+ * pp. 842-847
+ */
+ int i, ai, aj, ak, type, m, t1, t2;
+ rvec r_ij, r_kj;
+ real vtot, vrr, s1, s2, r1, r2, r1e, r2e, krr;
+ rvec f_i, f_j, f_k;
+ ivec jt, dt_ij, dt_kj;
+
+ vtot = 0.0;
+ for (i = 0; (i < nbonds); )
+ {
+ type = forceatoms[i++];
+ ai = forceatoms[i++];
+ aj = forceatoms[i++];
+ ak = forceatoms[i++];
+ r1e = forceparams[type].cross_bb.r1e;
+ r2e = forceparams[type].cross_bb.r2e;
+ krr = forceparams[type].cross_bb.krr;
+
+ /* Compute distance vectors ... */
+ t1 = pbc_rvec_sub(pbc, x[ai], x[aj], r_ij);
+ t2 = pbc_rvec_sub(pbc, x[ak], x[aj], r_kj);
+
+ /* ... and their lengths */
+ r1 = norm(r_ij);
+ r2 = norm(r_kj);
+
+ /* Deviations from ideality */
+ s1 = r1-r1e;
+ s2 = r2-r2e;
+
+ /* Energy (can be negative!) */
+ vrr = krr*s1*s2;
+ vtot += vrr;
+
+ /* Forces */
+ svmul(-krr*s2/r1, r_ij, f_i);
+ svmul(-krr*s1/r2, r_kj, f_k);
+
+ for (m = 0; (m < DIM); m++) /* 12 */
+ {
+ f_j[m] = -f_i[m] - f_k[m];
+ f[ai][m] += f_i[m];
+ f[aj][m] += f_j[m];
+ f[ak][m] += f_k[m];
+ }
+
+ /* Virial stuff */
+ if (g)
+ {
+ copy_ivec(SHIFT_IVEC(g, aj), jt);
+
+ ivec_sub(SHIFT_IVEC(g, ai), jt, dt_ij);
+ ivec_sub(SHIFT_IVEC(g, ak), jt, dt_kj);
+ t1 = IVEC2IS(dt_ij);
+ t2 = IVEC2IS(dt_kj);
+ }
+ rvec_inc(fshift[t1], f_i);
+ rvec_inc(fshift[CENTRAL], f_j);
+ rvec_inc(fshift[t2], f_k); /* 9 */
+ /* 163 TOTAL */
+ }
+ return vtot;
+}
+
+real cross_bond_angle(int nbonds,
+ const t_iatom forceatoms[], const t_iparams forceparams[],
+ const rvec x[], rvec f[], rvec fshift[],
+ const t_pbc *pbc, const t_graph *g,
+ real gmx_unused lambda, real gmx_unused *dvdlambda,
+ const t_mdatoms gmx_unused *md, t_fcdata gmx_unused *fcd,
+ int gmx_unused *global_atom_index)
+{
+ /* Potential from Lawrence and Skimmer, Chem. Phys. Lett. 372 (2003)
+ * pp. 842-847
+ */
+ int i, ai, aj, ak, type, m, t1, t2, t3;
+ rvec r_ij, r_kj, r_ik;
+ real vtot, vrt, s1, s2, s3, r1, r2, r3, r1e, r2e, r3e, krt, k1, k2, k3;
+ rvec f_i, f_j, f_k;
+ ivec jt, dt_ij, dt_kj;
+
+ vtot = 0.0;
+ for (i = 0; (i < nbonds); )
+ {
+ type = forceatoms[i++];
+ ai = forceatoms[i++];
+ aj = forceatoms[i++];
+ ak = forceatoms[i++];
+ r1e = forceparams[type].cross_ba.r1e;
+ r2e = forceparams[type].cross_ba.r2e;
+ r3e = forceparams[type].cross_ba.r3e;
+ krt = forceparams[type].cross_ba.krt;
+
+ /* Compute distance vectors ... */
+ t1 = pbc_rvec_sub(pbc, x[ai], x[aj], r_ij);
+ t2 = pbc_rvec_sub(pbc, x[ak], x[aj], r_kj);
+ t3 = pbc_rvec_sub(pbc, x[ai], x[ak], r_ik);
+
+ /* ... and their lengths */
+ r1 = norm(r_ij);
+ r2 = norm(r_kj);
+ r3 = norm(r_ik);
+
+ /* Deviations from ideality */
+ s1 = r1-r1e;
+ s2 = r2-r2e;
+ s3 = r3-r3e;
+
+ /* Energy (can be negative!) */
+ vrt = krt*s3*(s1+s2);
+ vtot += vrt;
+
+ /* Forces */
+ k1 = -krt*(s3/r1);
+ k2 = -krt*(s3/r2);
+ k3 = -krt*(s1+s2)/r3;
+ for (m = 0; (m < DIM); m++)
+ {
+ f_i[m] = k1*r_ij[m] + k3*r_ik[m];
+ f_k[m] = k2*r_kj[m] - k3*r_ik[m];
+ f_j[m] = -f_i[m] - f_k[m];
+ }
+
+ for (m = 0; (m < DIM); m++) /* 12 */
+ {
+ f[ai][m] += f_i[m];
+ f[aj][m] += f_j[m];
+ f[ak][m] += f_k[m];
+ }
+
+ /* Virial stuff */
+ if (g)
+ {
+ copy_ivec(SHIFT_IVEC(g, aj), jt);
+
+ ivec_sub(SHIFT_IVEC(g, ai), jt, dt_ij);
+ ivec_sub(SHIFT_IVEC(g, ak), jt, dt_kj);
+ t1 = IVEC2IS(dt_ij);
+ t2 = IVEC2IS(dt_kj);
+ }
+ rvec_inc(fshift[t1], f_i);
+ rvec_inc(fshift[CENTRAL], f_j);
+ rvec_inc(fshift[t2], f_k); /* 9 */
+ /* 163 TOTAL */
+ }
+ return vtot;
+}
+
+static real bonded_tab(const char *type, int table_nr,
+ const bondedtable_t *table, real kA, real kB, real r,
+ real lambda, real *V, real *F)
+{
+ real k, tabscale, *VFtab, rt, eps, eps2, Yt, Ft, Geps, Heps2, Fp, VV, FF;
+ int n0, nnn;
+ real v, f, dvdlambda;
+
+ k = (1.0 - lambda)*kA + lambda*kB;
+
+ tabscale = table->scale;
+ VFtab = table->data;
+
+ rt = r*tabscale;
+ n0 = rt;
+ if (n0 >= table->n)
+ {
+ gmx_fatal(FARGS, "A tabulated %s interaction table number %d is out of the table range: r %f, between table indices %d and %d, table length %d",
+ type, table_nr, r, n0, n0+1, table->n);
+ }
+ eps = rt - n0;
+ eps2 = eps*eps;
+ nnn = 4*n0;
+ Yt = VFtab[nnn];
+ Ft = VFtab[nnn+1];
+ Geps = VFtab[nnn+2]*eps;
+ Heps2 = VFtab[nnn+3]*eps2;
+ Fp = Ft + Geps + Heps2;
+ VV = Yt + Fp*eps;
+ FF = Fp + Geps + 2.0*Heps2;
+
+ *F = -k*FF*tabscale;
+ *V = k*VV;
+ dvdlambda = (kB - kA)*VV;
+
+ return dvdlambda;
+
+ /* That was 22 flops */
+}
+
+real tab_bonds(int nbonds,
+ const t_iatom forceatoms[], const t_iparams forceparams[],
+ const rvec x[], rvec f[], rvec fshift[],
+ const t_pbc *pbc, const t_graph *g,
+ real lambda, real *dvdlambda,
+ const t_mdatoms gmx_unused *md, t_fcdata *fcd,
+ int gmx_unused *global_atom_index)
+{
+ int i, m, ki, ai, aj, type, table;
+ real dr, dr2, fbond, vbond, fij, vtot;
+ rvec dx;
+ ivec dt;
+
+ vtot = 0.0;
+ for (i = 0; (i < nbonds); )
+ {
+ type = forceatoms[i++];
+ ai = forceatoms[i++];
+ aj = forceatoms[i++];
+
+ ki = pbc_rvec_sub(pbc, x[ai], x[aj], dx); /* 3 */
+ dr2 = iprod(dx, dx); /* 5 */
+ dr = dr2*gmx_invsqrt(dr2); /* 10 */
+
+ table = forceparams[type].tab.table;
+
+ *dvdlambda += bonded_tab("bond", table,
+ &fcd->bondtab[table],
+ forceparams[type].tab.kA,
+ forceparams[type].tab.kB,
+ dr, lambda, &vbond, &fbond); /* 22 */
+
+ if (dr2 == 0.0)
+ {
+ continue;
+ }
+
+
+ vtot += vbond; /* 1*/
+ fbond *= gmx_invsqrt(dr2); /* 6 */
+#ifdef DEBUG
+ if (debug)
+ {
+ fprintf(debug, "TABBONDS: dr = %10g vbond = %10g fbond = %10g\n",
+ dr, vbond, fbond);
+ }
+#endif
+ if (g)
+ {
+ ivec_sub(SHIFT_IVEC(g, ai), SHIFT_IVEC(g, aj), dt);
+ ki = IVEC2IS(dt);
+ }
+ for (m = 0; (m < DIM); m++) /* 15 */
+ {
+ fij = fbond*dx[m];
+ f[ai][m] += fij;
+ f[aj][m] -= fij;
+ fshift[ki][m] += fij;
+ fshift[CENTRAL][m] -= fij;
+ }
+ } /* 62 TOTAL */
+ return vtot;
+}
+
+real tab_angles(int nbonds,
+ const t_iatom forceatoms[], const t_iparams forceparams[],
+ const rvec x[], rvec f[], rvec fshift[],
+ const t_pbc *pbc, const t_graph *g,
+ real lambda, real *dvdlambda,
+ const t_mdatoms gmx_unused *md, t_fcdata *fcd,
+ int gmx_unused *global_atom_index)
+{
+ int i, ai, aj, ak, t1, t2, type, table;
+ rvec r_ij, r_kj;
+ real cos_theta, cos_theta2, theta, dVdt, va, vtot;
+ ivec jt, dt_ij, dt_kj;
+
+ vtot = 0.0;
+ for (i = 0; (i < nbonds); )
+ {
+ type = forceatoms[i++];
+ ai = forceatoms[i++];
+ aj = forceatoms[i++];
+ ak = forceatoms[i++];
+
+ theta = bond_angle(x[ai], x[aj], x[ak], pbc,
+ r_ij, r_kj, &cos_theta, &t1, &t2); /* 41 */
+
+ table = forceparams[type].tab.table;
+
+ *dvdlambda += bonded_tab("angle", table,
+ &fcd->angletab[table],
+ forceparams[type].tab.kA,
+ forceparams[type].tab.kB,
+ theta, lambda, &va, &dVdt); /* 22 */
+ vtot += va;
+
+ cos_theta2 = sqr(cos_theta); /* 1 */
+ if (cos_theta2 < 1)
+ {
+ int m;
+ real snt, st, sth;
+ real cik, cii, ckk;
+ real nrkj2, nrij2;
+ rvec f_i, f_j, f_k;
+
+ st = dVdt*gmx_invsqrt(1 - cos_theta2); /* 12 */
+ sth = st*cos_theta; /* 1 */
+#ifdef DEBUG
+ if (debug)
+ {
+ fprintf(debug, "ANGLES: theta = %10g vth = %10g dV/dtheta = %10g\n",
+ theta*RAD2DEG, va, dVdt);
+ }
+#endif
+ nrkj2 = iprod(r_kj, r_kj); /* 5 */
+ nrij2 = iprod(r_ij, r_ij);
+
+ cik = st*gmx_invsqrt(nrkj2*nrij2); /* 12 */
+ cii = sth/nrij2; /* 10 */
+ ckk = sth/nrkj2; /* 10 */
+
+ for (m = 0; (m < DIM); m++) /* 39 */
+ {
+ f_i[m] = -(cik*r_kj[m]-cii*r_ij[m]);
+ f_k[m] = -(cik*r_ij[m]-ckk*r_kj[m]);
+ f_j[m] = -f_i[m]-f_k[m];
+ f[ai][m] += f_i[m];
+ f[aj][m] += f_j[m];
+ f[ak][m] += f_k[m];
+ }
+ if (g)
+ {
+ copy_ivec(SHIFT_IVEC(g, aj), jt);
+
+ ivec_sub(SHIFT_IVEC(g, ai), jt, dt_ij);
+ ivec_sub(SHIFT_IVEC(g, ak), jt, dt_kj);
+ t1 = IVEC2IS(dt_ij);
+ t2 = IVEC2IS(dt_kj);
+ }
+ rvec_inc(fshift[t1], f_i);
+ rvec_inc(fshift[CENTRAL], f_j);
+ rvec_inc(fshift[t2], f_k);
+ } /* 169 TOTAL */
+ }
+ return vtot;
+}
+
+real tab_dihs(int nbonds,
+ const t_iatom forceatoms[], const t_iparams forceparams[],
+ const rvec x[], rvec f[], rvec fshift[],
+ const t_pbc *pbc, const t_graph *g,
+ real lambda, real *dvdlambda,
+ const t_mdatoms gmx_unused *md, t_fcdata *fcd,
+ int gmx_unused *global_atom_index)
+{
+ int i, type, ai, aj, ak, al, table;
+ int t1, t2, t3;
+ rvec r_ij, r_kj, r_kl, m, n;
+ real phi, sign, ddphi, vpd, vtot;
+
+ vtot = 0.0;
+ for (i = 0; (i < nbonds); )
+ {
+ type = forceatoms[i++];
+ ai = forceatoms[i++];
+ aj = forceatoms[i++];
+ ak = forceatoms[i++];
+ al = forceatoms[i++];
+
+ phi = dih_angle(x[ai], x[aj], x[ak], x[al], pbc, r_ij, r_kj, r_kl, m, n,
+ &sign, &t1, &t2, &t3); /* 84 */
+
+ table = forceparams[type].tab.table;
+
+ /* Hopefully phi+M_PI never results in values < 0 */
+ *dvdlambda += bonded_tab("dihedral", table,
+ &fcd->dihtab[table],
+ forceparams[type].tab.kA,
+ forceparams[type].tab.kB,
+ phi+M_PI, lambda, &vpd, &ddphi);
+
+ vtot += vpd;
+ do_dih_fup(ai, aj, ak, al, -ddphi, r_ij, r_kj, r_kl, m, n,
+ f, fshift, pbc, g, x, t1, t2, t3); /* 112 */
+
+#ifdef DEBUG
+ fprintf(debug, "pdih: (%d,%d,%d,%d) phi=%g\n",
+ ai, aj, ak, al, phi);
+#endif
+ } /* 227 TOTAL */
+
+ return vtot;
+}
+
+/* Return if this is a potential calculated in bondfree.c,
+ * i.e. an interaction that actually calculates a potential and
+ * works on multiple atoms (not e.g. a connection or a position restraint).
+ */
+static gmx_inline gmx_bool ftype_is_bonded_potential(int ftype)
+{
+ return
+ (interaction_function[ftype].flags & IF_BOND) &&
+ !(ftype == F_CONNBONDS || ftype == F_POSRES) &&
+ (ftype < F_GB12 || ftype > F_GB14);
+}
+
+static void divide_bondeds_over_threads(t_idef *idef, int nthreads)
+{
+ int ftype;
+ int nat1;
+ int t;
+ int il_nr_thread;
+
+ idef->nthreads = nthreads;
+
+ if (F_NRE*(nthreads+1) > idef->il_thread_division_nalloc)
+ {
+ idef->il_thread_division_nalloc = F_NRE*(nthreads+1);
+ snew(idef->il_thread_division, idef->il_thread_division_nalloc);
+ }
+
+ for (ftype = 0; ftype < F_NRE; ftype++)
+ {
+ if (ftype_is_bonded_potential(ftype))
+ {
+ nat1 = interaction_function[ftype].nratoms + 1;
+
+ for (t = 0; t <= nthreads; t++)
+ {
+ /* Divide the interactions equally over the threads.
+ * When the different types of bonded interactions
+ * are distributed roughly equally over the threads,
+ * this should lead to well localized output into
+ * the force buffer on each thread.
+ * If this is not the case, a more advanced scheme
+ * (not implemented yet) will do better.
+ */
+ il_nr_thread = (((idef->il[ftype].nr/nat1)*t)/nthreads)*nat1;
+
+ /* Ensure that distance restraint pairs with the same label
+ * end up on the same thread.
+ * This is slighlty tricky code, since the next for iteration
+ * may have an initial il_nr_thread lower than the final value
+ * in the previous iteration, but this will anyhow be increased
+ * to the approriate value again by this while loop.
+ */
+ while (ftype == F_DISRES &&
+ il_nr_thread > 0 &&
+ il_nr_thread < idef->il[ftype].nr &&
+ idef->iparams[idef->il[ftype].iatoms[il_nr_thread]].disres.label ==
+ idef->iparams[idef->il[ftype].iatoms[il_nr_thread-nat1]].disres.label)
+ {
+ il_nr_thread += nat1;
+ }
+
+ idef->il_thread_division[ftype*(nthreads+1)+t] = il_nr_thread;
+ }
+ }
+ }
+}
+
+static unsigned
+calc_bonded_reduction_mask(const t_idef *idef,
+ int shift,
+ int t, int nt)
+{
+ unsigned mask;
+ int ftype, nb, nat1, nb0, nb1, i, a;
+
+ mask = 0;
+
+ for (ftype = 0; ftype < F_NRE; ftype++)
+ {
+ if (ftype_is_bonded_potential(ftype))
+ {
+ nb = idef->il[ftype].nr;
+ if (nb > 0)
+ {
+ nat1 = interaction_function[ftype].nratoms + 1;
+
+ /* Divide this interaction equally over the threads.
+ * This is not stored: should match division in calc_bonds.
+ */
+ nb0 = idef->il_thread_division[ftype*(nt+1)+t];
+ nb1 = idef->il_thread_division[ftype*(nt+1)+t+1];
+
+ for (i = nb0; i < nb1; i += nat1)
+ {
+ for (a = 1; a < nat1; a++)
+ {
+ mask |= (1U << (idef->il[ftype].iatoms[i+a]>>shift));
+ }
+ }
+ }
+ }
+ }
+
+ return mask;
+}
+
+void setup_bonded_threading(t_forcerec *fr, t_idef *idef)
+{
+#define MAX_BLOCK_BITS 32
+ int t;
+ int ctot, c, b;
+
+ assert(fr->nthreads >= 1);
+
+ /* Divide the bonded interaction over the threads */
+ divide_bondeds_over_threads(idef, fr->nthreads);
+
+ if (fr->nthreads == 1)
+ {
+ fr->red_nblock = 0;
+
+ return;
+ }
+
+ /* We divide the force array in a maximum of 32 blocks.
+ * Minimum force block reduction size is 2^6=64.
+ */
+ fr->red_ashift = 6;
+ while (fr->natoms_force > (int)(MAX_BLOCK_BITS*(1U<<fr->red_ashift)))
+ {
+ fr->red_ashift++;
+ }
+ if (debug)
+ {
+ fprintf(debug, "bonded force buffer block atom shift %d bits\n",
+ fr->red_ashift);
+ }
+
+ /* Determine to which blocks each thread's bonded force calculation
+ * contributes. Store this is a mask for each thread.
+ */
+#pragma omp parallel for num_threads(fr->nthreads) schedule(static)
+ for (t = 1; t < fr->nthreads; t++)
+ {
+ fr->f_t[t].red_mask =
+ calc_bonded_reduction_mask(idef, fr->red_ashift, t, fr->nthreads);
+ }
+
+ /* Determine the maximum number of blocks we need to reduce over */
+ fr->red_nblock = 0;
+ ctot = 0;
+ for (t = 0; t < fr->nthreads; t++)
+ {
+ c = 0;
+ for (b = 0; b < MAX_BLOCK_BITS; b++)
+ {
+ if (fr->f_t[t].red_mask & (1U<<b))
+ {
+ fr->red_nblock = max(fr->red_nblock, b+1);
+ c++;
+ }
+ }
+ if (debug)
+ {
+ fprintf(debug, "thread %d flags %x count %d\n",
+ t, fr->f_t[t].red_mask, c);
+ }
+ ctot += c;
+ }
+ if (debug)
+ {
+ fprintf(debug, "Number of blocks to reduce: %d of size %d\n",
+ fr->red_nblock, 1<<fr->red_ashift);
+ fprintf(debug, "Reduction density %.2f density/#thread %.2f\n",
+ ctot*(1<<fr->red_ashift)/(double)fr->natoms_force,
+ ctot*(1<<fr->red_ashift)/(double)(fr->natoms_force*fr->nthreads));
+ }
+}
+
+static void zero_thread_forces(f_thread_t *f_t, int n,
+ int nblock, int blocksize)
+{
+ int b, a0, a1, a, i, j;
+
+ if (n > f_t->f_nalloc)
+ {
+ f_t->f_nalloc = over_alloc_large(n);
+ srenew(f_t->f, f_t->f_nalloc);
+ }
+
+ if (f_t->red_mask != 0)
+ {
+ for (b = 0; b < nblock; b++)
+ {
+ if (f_t->red_mask && (1U<<b))
+ {
+ a0 = b*blocksize;
+ a1 = min((b+1)*blocksize, n);
+ for (a = a0; a < a1; a++)
+ {
+ clear_rvec(f_t->f[a]);
+ }
+ }
+ }
+ }
+ for (i = 0; i < SHIFTS; i++)
+ {
+ clear_rvec(f_t->fshift[i]);
+ }
+ for (i = 0; i < F_NRE; i++)
+ {
+ f_t->ener[i] = 0;
+ }
+ for (i = 0; i < egNR; i++)
+ {
+ for (j = 0; j < f_t->grpp.nener; j++)
+ {
+ f_t->grpp.ener[i][j] = 0;
+ }
+ }
+ for (i = 0; i < efptNR; i++)
+ {
+ f_t->dvdl[i] = 0;
+ }
+}
+
+static void reduce_thread_force_buffer(int n, rvec *f,
+ int nthreads, f_thread_t *f_t,
+ int nblock, int block_size)
+{
+ /* The max thread number is arbitrary,
+ * we used a fixed number to avoid memory management.
+ * Using more than 16 threads is probably never useful performance wise.
+ */
+#define MAX_BONDED_THREADS 256
+ int b;
+
+ if (nthreads > MAX_BONDED_THREADS)
+ {
+ gmx_fatal(FARGS, "Can not reduce bonded forces on more than %d threads",
+ MAX_BONDED_THREADS);
+ }
+
+ /* This reduction can run on any number of threads,
+ * independently of nthreads.
+ */
+#pragma omp parallel for num_threads(nthreads) schedule(static)
+ for (b = 0; b < nblock; b++)
+ {
+ rvec *fp[MAX_BONDED_THREADS];
+ int nfb, ft, fb;
+ int a0, a1, a;
+
+ /* Determine which threads contribute to this block */
+ nfb = 0;
+ for (ft = 1; ft < nthreads; ft++)
+ {
+ if (f_t[ft].red_mask & (1U<<b))
+ {
+ fp[nfb++] = f_t[ft].f;
+ }
+ }
+ if (nfb > 0)
+ {
+ /* Reduce force buffers for threads that contribute */
+ a0 = b *block_size;
+ a1 = (b+1)*block_size;
+ a1 = min(a1, n);
+ for (a = a0; a < a1; a++)
+ {
+ for (fb = 0; fb < nfb; fb++)
+ {
+ rvec_inc(f[a], fp[fb][a]);
+ }
+ }
+ }
+ }
+}
+
+static void reduce_thread_forces(int n, rvec *f, rvec *fshift,
+ real *ener, gmx_grppairener_t *grpp, real *dvdl,
+ int nthreads, f_thread_t *f_t,
+ int nblock, int block_size,
+ gmx_bool bCalcEnerVir,
+ gmx_bool bDHDL)
+{
+ if (nblock > 0)
+ {
+ /* Reduce the bonded force buffer */
+ reduce_thread_force_buffer(n, f, nthreads, f_t, nblock, block_size);
+ }
+
+ /* When necessary, reduce energy and virial using one thread only */
+ if (bCalcEnerVir)
+ {
+ int t, i, j;
+
+ for (i = 0; i < SHIFTS; i++)
+ {
+ for (t = 1; t < nthreads; t++)
+ {
+ rvec_inc(fshift[i], f_t[t].fshift[i]);
+ }
+ }
+ for (i = 0; i < F_NRE; i++)
+ {
+ for (t = 1; t < nthreads; t++)
+ {
+ ener[i] += f_t[t].ener[i];
+ }
+ }
+ for (i = 0; i < egNR; i++)
+ {
+ for (j = 0; j < f_t[1].grpp.nener; j++)
+ {
+ for (t = 1; t < nthreads; t++)
+ {
+
+ grpp->ener[i][j] += f_t[t].grpp.ener[i][j];
+ }
+ }
+ }
+ if (bDHDL)
+ {
+ for (i = 0; i < efptNR; i++)
+ {
+
+ for (t = 1; t < nthreads; t++)
+ {
+ dvdl[i] += f_t[t].dvdl[i];
+ }
+ }
+ }
+ }
+}
+
+static real calc_one_bond(FILE *fplog, int thread,
+ int ftype, const t_idef *idef,
+ rvec x[], rvec f[], rvec fshift[],
+ t_forcerec *fr,
+ const t_pbc *pbc, const t_graph *g,
+ gmx_grppairener_t *grpp,
+ t_nrnb *nrnb,
+ real *lambda, real *dvdl,
+ const t_mdatoms *md, t_fcdata *fcd,
+ gmx_bool bCalcEnerVir,
+ int *global_atom_index, gmx_bool bPrintSepPot)
+{
+ int nat1, nbonds, efptFTYPE;
+ real v = 0;
+ t_iatom *iatoms;
+ int nb0, nbn;
+
+ if (IS_RESTRAINT_TYPE(ftype))
+ {
+ efptFTYPE = efptRESTRAINT;
+ }
+ else
+ {
+ efptFTYPE = efptBONDED;
+ }
+
+ nat1 = interaction_function[ftype].nratoms + 1;
+ nbonds = idef->il[ftype].nr/nat1;
+ iatoms = idef->il[ftype].iatoms;
+
+ nb0 = idef->il_thread_division[ftype*(idef->nthreads+1)+thread];
+ nbn = idef->il_thread_division[ftype*(idef->nthreads+1)+thread+1] - nb0;
+
+ if (!IS_LISTED_LJ_C(ftype))
+ {
+ if (ftype == F_CMAP)
+ {
+ v = cmap_dihs(nbn, iatoms+nb0,
+ idef->iparams, &idef->cmap_grid,
+ (const rvec*)x, f, fshift,
+ pbc, g, lambda[efptFTYPE], &(dvdl[efptFTYPE]),
+ md, fcd, global_atom_index);
+ }
+#ifdef SIMD_BONDEDS
+ else if (ftype == F_ANGLES &&
+ !bCalcEnerVir && fr->efep == efepNO)
+ {
+ /* No energies, shift forces, dvdl */
+ angles_noener_simd(nbn, idef->il[ftype].iatoms+nb0,
+ idef->iparams,
+ (const rvec*)x, f,
+ pbc, g, lambda[efptFTYPE], md, fcd,
+ global_atom_index);
+ v = 0;
+ }
+#endif
+ else if (ftype == F_PDIHS &&
+ !bCalcEnerVir && fr->efep == efepNO)
+ {
+ /* No energies, shift forces, dvdl */
+#ifndef SIMD_BONDEDS
+ pdihs_noener
+#else
+ pdihs_noener_simd
+#endif
+ (nbn, idef->il[ftype].iatoms+nb0,
+ idef->iparams,
+ (const rvec*)x, f,
+ pbc, g, lambda[efptFTYPE], md, fcd,
+ global_atom_index);
+ v = 0;
+ }
+ else
+ {
+ v = interaction_function[ftype].ifunc(nbn, iatoms+nb0,
+ idef->iparams,
+ (const rvec*)x, f, fshift,
+ pbc, g, lambda[efptFTYPE], &(dvdl[efptFTYPE]),
+ md, fcd, global_atom_index);
+ }
+ if (bPrintSepPot)
+ {
+ fprintf(fplog, " %-23s #%4d V %12.5e dVdl %12.5e\n",
+ interaction_function[ftype].longname,
+ nbonds, v, lambda[efptFTYPE]);
+ }
+ }
+ else
+ {
+ v = do_nonbonded_listed(ftype, nbn, iatoms+nb0, idef->iparams, (const rvec*)x, f, fshift,
+ pbc, g, lambda, dvdl, md, fr, grpp, global_atom_index);
+
+ if (bPrintSepPot)
+ {
+ fprintf(fplog, " %-5s + %-15s #%4d dVdl %12.5e\n",
+ interaction_function[ftype].longname,
+ interaction_function[F_LJ14].longname, nbonds, dvdl[efptVDW]);
+ fprintf(fplog, " %-5s + %-15s #%4d dVdl %12.5e\n",
+ interaction_function[ftype].longname,
+ interaction_function[F_COUL14].longname, nbonds, dvdl[efptCOUL]);
+ }
+ }
+
+ if (thread == 0)
+ {
+ inc_nrnb(nrnb, interaction_function[ftype].nrnb_ind, nbonds);
+ }
+
+ return v;
+}
+
+void calc_bonds(FILE *fplog, const gmx_multisim_t *ms,
+ const t_idef *idef,
+ rvec x[], history_t *hist,
+ rvec f[], t_forcerec *fr,
+ const t_pbc *pbc, const t_graph *g,
+ gmx_enerdata_t *enerd, t_nrnb *nrnb,
+ real *lambda,
+ const t_mdatoms *md,
+ t_fcdata *fcd, int *global_atom_index,
+ t_atomtypes gmx_unused *atype, gmx_genborn_t gmx_unused *born,
+ int force_flags,
+ gmx_bool bPrintSepPot, gmx_large_int_t step)
+{
+ gmx_bool bCalcEnerVir;
+ int i;
+ real v, dvdl[efptNR], dvdl_dum[efptNR]; /* The dummy array is to have a place to store the dhdl at other values
+ of lambda, which will be thrown away in the end*/
+ const t_pbc *pbc_null;
+ char buf[22];
+ int thread;
+
+ assert(fr->nthreads == idef->nthreads);
+
+ bCalcEnerVir = (force_flags & (GMX_FORCE_VIRIAL | GMX_FORCE_ENERGY));
+
+ for (i = 0; i < efptNR; i++)
+ {
+ dvdl[i] = 0.0;
+ }
+ if (fr->bMolPBC)
+ {
+ pbc_null = pbc;
+ }
+ else
+ {
+ pbc_null = NULL;
+ }
+ if (bPrintSepPot)
+ {
+ fprintf(fplog, "Step %s: bonded V and dVdl for this node\n",
+ gmx_step_str(step, buf));
+ }
+
+#ifdef DEBUG
+ if (g && debug)
+ {
+ p_graph(debug, "Bondage is fun", g);
+ }
+#endif
+
+ /* Do pre force calculation stuff which might require communication */
+ if (idef->il[F_ORIRES].nr)
+ {
+ enerd->term[F_ORIRESDEV] =
+ calc_orires_dev(ms, idef->il[F_ORIRES].nr,
+ idef->il[F_ORIRES].iatoms,
+ idef->iparams, md, (const rvec*)x,
+ pbc_null, fcd, hist);
+ }
+ if (idef->il[F_DISRES].nr)
+ {
+ calc_disres_R_6(idef->il[F_DISRES].nr,
+ idef->il[F_DISRES].iatoms,
+ idef->iparams, (const rvec*)x, pbc_null,
+ fcd, hist);
+#ifdef GMX_MPI
+ if (fcd->disres.nsystems > 1)
+ {
+ gmx_sum_sim(2*fcd->disres.nres, fcd->disres.Rt_6, ms);
+ }
+#endif
+ }
+
+#pragma omp parallel for num_threads(fr->nthreads) schedule(static)
+ for (thread = 0; thread < fr->nthreads; thread++)
+ {
+ int ftype;
+ real *epot, v;
+ /* thread stuff */
+ rvec *ft, *fshift;
+ real *dvdlt;
+ gmx_grppairener_t *grpp;
+
+ if (thread == 0)
+ {
+ ft = f;
+ fshift = fr->fshift;
+ epot = enerd->term;
+ grpp = &enerd->grpp;
+ dvdlt = dvdl;
+ }
+ else
+ {
+ zero_thread_forces(&fr->f_t[thread], fr->natoms_force,
+ fr->red_nblock, 1<<fr->red_ashift);
+
+ ft = fr->f_t[thread].f;
+ fshift = fr->f_t[thread].fshift;
+ epot = fr->f_t[thread].ener;
+ grpp = &fr->f_t[thread].grpp;
+ dvdlt = fr->f_t[thread].dvdl;
+ }
+ /* Loop over all bonded force types to calculate the bonded forces */
+ for (ftype = 0; (ftype < F_NRE); ftype++)
+ {
+ if (idef->il[ftype].nr > 0 && ftype_is_bonded_potential(ftype))
+ {
+ v = calc_one_bond(fplog, thread, ftype, idef, x,
+ ft, fshift, fr, pbc_null, g, grpp,
+ nrnb, lambda, dvdlt,
+ md, fcd, bCalcEnerVir,
+ global_atom_index, bPrintSepPot);
+ epot[ftype] += v;
+ }
+ }
+ }
+ if (fr->nthreads > 1)
+ {
+ reduce_thread_forces(fr->natoms_force, f, fr->fshift,
+ enerd->term, &enerd->grpp, dvdl,
+ fr->nthreads, fr->f_t,
+ fr->red_nblock, 1<<fr->red_ashift,
+ bCalcEnerVir,
+ force_flags & GMX_FORCE_DHDL);
+ }
+ if (force_flags & GMX_FORCE_DHDL)
+ {
+ for (i = 0; i < efptNR; i++)
+ {
+ enerd->dvdl_nonlin[i] += dvdl[i];
+ }
+ }
+
+ /* Copy the sum of violations for the distance restraints from fcd */
+ if (fcd)
+ {
+ enerd->term[F_DISRESVIOL] = fcd->disres.sumviol;
+
+ }
+}
+
+void calc_bonds_lambda(FILE *fplog,
+ const t_idef *idef,
+ rvec x[],
+ t_forcerec *fr,
+ const t_pbc *pbc, const t_graph *g,
+ gmx_grppairener_t *grpp, real *epot, t_nrnb *nrnb,
+ real *lambda,
+ const t_mdatoms *md,
+ t_fcdata *fcd,
+ int *global_atom_index)
+{
+ int i, ftype, nr_nonperturbed, nr;
+ real v;
+ real dvdl_dum[efptNR];
+ rvec *f, *fshift;
+ const t_pbc *pbc_null;
+ t_idef idef_fe;
+
+ if (fr->bMolPBC)
+ {
+ pbc_null = pbc;
+ }
+ else
+ {
+ pbc_null = NULL;
+ }
+
+ /* Copy the whole idef, so we can modify the contents locally */
+ idef_fe = *idef;
+ idef_fe.nthreads = 1;
+ snew(idef_fe.il_thread_division, F_NRE*(idef_fe.nthreads+1));
+
+ /* We already have the forces, so we use temp buffers here */
+ snew(f, fr->natoms_force);
+ snew(fshift, SHIFTS);
+
+ /* Loop over all bonded force types to calculate the bonded energies */
+ for (ftype = 0; (ftype < F_NRE); ftype++)
+ {
+ if (ftype_is_bonded_potential(ftype))
+ {
+ /* Set the work range of thread 0 to the perturbed bondeds only */
+ nr_nonperturbed = idef->il[ftype].nr_nonperturbed;
+ nr = idef->il[ftype].nr;
+ idef_fe.il_thread_division[ftype*2+0] = nr_nonperturbed;
+ idef_fe.il_thread_division[ftype*2+1] = nr;
+
+ /* This is only to get the flop count correct */
+ idef_fe.il[ftype].nr = nr - nr_nonperturbed;
+
+ if (nr - nr_nonperturbed > 0)
+ {
+ v = calc_one_bond(fplog, 0, ftype, &idef_fe,
+ x, f, fshift, fr, pbc_null, g,
+ grpp, nrnb, lambda, dvdl_dum,
+ md, fcd, TRUE,
+ global_atom_index, FALSE);
+ epot[ftype] += v;
+ }
+ }
+ }
+
+ sfree(fshift);
+ sfree(f);
+
+ sfree(idef_fe.il_thread_division);
+}
--- /dev/null
- "bEquilibrated", "N_at_state", "Wang-Landau_Histogram", "Wang-Landau-delta", "Weights", "Free Energies", "minvar", "variance",
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This file is part of Gromacs Copyright (c) 1991-2008
+ * David van der Spoel, Erik Lindahl, Berk Hess, University of Groningen.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the research papers on the package. Check out http://www.gromacs.org
+ *
+ * And Hey:
+ * Gnomes, ROck Monsters And Chili Sauce
+ */
+
+/* The source code in this file should be thread-safe.
+ Please keep it that way. */
+
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <string.h>
+#include <time.h>
+
+#ifdef HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#ifdef GMX_NATIVE_WINDOWS
+/* _chsize_s */
+#include <io.h>
+#include <sys/locking.h>
+#endif
+
+
+#include "filenm.h"
+#include "names.h"
+#include "typedefs.h"
+#include "smalloc.h"
+#include "gmxfio.h"
+#include "xdrf.h"
+#include "statutil.h"
+#include "txtdump.h"
+#include "vec.h"
+#include "network.h"
+#include "gmx_random.h"
+#include "checkpoint.h"
+#include "futil.h"
+#include "string2.h"
+#include <fcntl.h>
+
+#include "buildinfo.h"
+
+#ifdef GMX_FAHCORE
+#include "corewrap.h"
+#endif
+
+#define CPT_MAGIC1 171817
+#define CPT_MAGIC2 171819
+#define CPTSTRLEN 1024
+
+#ifdef GMX_DOUBLE
+#define GMX_CPT_BUILD_DP 1
+#else
+#define GMX_CPT_BUILD_DP 0
+#endif
+
+/* cpt_version should normally only be changed
+ * when the header of footer format changes.
+ * The state data format itself is backward and forward compatible.
+ * But old code can not read a new entry that is present in the file
+ * (but can read a new format when new entries are not present).
+ */
+static const int cpt_version = 15;
+
+
+const char *est_names[estNR] =
+{
+ "FE-lambda",
+ "box", "box-rel", "box-v", "pres_prev",
+ "nosehoover-xi", "thermostat-integral",
+ "x", "v", "SDx", "CGp", "LD-rng", "LD-rng-i",
+ "disre_initf", "disre_rm3tav",
+ "orire_initf", "orire_Dtav",
+ "svir_prev", "nosehoover-vxi", "v_eta", "vol0", "nhpres_xi", "nhpres_vxi", "fvir_prev", "fep_state", "MC-rng", "MC-rng-i"
+};
+
+enum {
+ eeksEKIN_N, eeksEKINH, eeksDEKINDL, eeksMVCOS, eeksEKINF, eeksEKINO, eeksEKINSCALEF, eeksEKINSCALEH, eeksVSCALE, eeksEKINTOTAL, eeksNR
+};
+
+const char *eeks_names[eeksNR] =
+{
+ "Ekin_n", "Ekinh", "dEkindlambda", "mv_cos",
+ "Ekinf", "Ekinh_old", "EkinScaleF_NHC", "EkinScaleH_NHC", "Vscale_NHC", "Ekin_Total"
+};
+
+enum {
+ eenhENERGY_N, eenhENERGY_AVER, eenhENERGY_SUM, eenhENERGY_NSUM,
+ eenhENERGY_SUM_SIM, eenhENERGY_NSUM_SIM,
+ eenhENERGY_NSTEPS, eenhENERGY_NSTEPS_SIM,
+ eenhENERGY_DELTA_H_NN,
+ eenhENERGY_DELTA_H_LIST,
+ eenhENERGY_DELTA_H_STARTTIME,
+ eenhENERGY_DELTA_H_STARTLAMBDA,
+ eenhNR
+};
+
+const char *eenh_names[eenhNR] =
+{
+ "energy_n", "energy_aver", "energy_sum", "energy_nsum",
+ "energy_sum_sim", "energy_nsum_sim",
+ "energy_nsteps", "energy_nsteps_sim",
+ "energy_delta_h_nn",
+ "energy_delta_h_list",
+ "energy_delta_h_start_time",
+ "energy_delta_h_start_lambda"
+};
+
+/* free energy history variables -- need to be preserved over checkpoint */
+enum {
+ edfhBEQUIL, edfhNATLAMBDA, edfhWLHISTO, edfhWLDELTA, edfhSUMWEIGHTS, edfhSUMDG, edfhSUMMINVAR, edfhSUMVAR,
+ edfhACCUMP, edfhACCUMM, edfhACCUMP2, edfhACCUMM2, edfhTIJ, edfhTIJEMP, edfhNR
+};
+/* free energy history variable names */
+const char *edfh_names[edfhNR] =
+{
- static int do_cpt_state(XDR *xd, int fflags, t_state *state,
++ "bEquilibrated", "N_at_state", "Wang-Landau Histogram", "Wang-Landau Delta", "Weights", "Free Energies", "minvar", "variance",
+ "accumulated_plus", "accumulated_minus", "accumulated_plus_2", "accumulated_minus_2", "Tij", "Tij_empirical"
+};
+
+#ifdef GMX_NATIVE_WINDOWS
+static int
+gmx_wintruncate(const char *filename, __int64 size)
+{
+#ifdef GMX_FAHCORE
+ /*we do this elsewhere*/
+ return 0;
+#else
+ FILE *fp;
+ int rc;
+
+ fp = fopen(filename, "rb+");
+
+ if (fp == NULL)
+ {
+ return -1;
+ }
+
+ return _chsize_s( fileno(fp), size);
+#endif
+}
+#endif
+
+
+enum {
+ ecprREAL, ecprRVEC, ecprMATRIX
+};
+
+enum {
+ cptpEST, cptpEEKS, cptpEENH, cptpEDFH
+};
+/* enums for the different components of checkpoint variables, replacing the hard coded ones.
+ cptpEST - state variables.
+ cptpEEKS - Kinetic energy state variables.
+ cptpEENH - Energy history state variables.
+ cptpEDFH - free energy history variables.
+ */
+
+
+static const char *st_names(int cptp, int ecpt)
+{
+ switch (cptp)
+ {
+ case cptpEST: return est_names [ecpt]; break;
+ case cptpEEKS: return eeks_names[ecpt]; break;
+ case cptpEENH: return eenh_names[ecpt]; break;
+ case cptpEDFH: return edfh_names[ecpt]; break;
+ }
+
+ return NULL;
+}
+
+static void cp_warning(FILE *fp)
+{
+ fprintf(fp, "\nWARNING: Checkpoint file is corrupted or truncated\n\n");
+}
+
+static void cp_error()
+{
+ gmx_fatal(FARGS, "Checkpoint file corrupted/truncated, or maybe you are out of disk space?");
+}
+
+static void do_cpt_string_err(XDR *xd, gmx_bool bRead, const char *desc, char **s, FILE *list)
+{
+ bool_t res = 0;
+
+ if (bRead)
+ {
+ snew(*s, CPTSTRLEN);
+ }
+ res = xdr_string(xd, s, CPTSTRLEN);
+ if (res == 0)
+ {
+ cp_error();
+ }
+ if (list)
+ {
+ fprintf(list, "%s = %s\n", desc, *s);
+ sfree(*s);
+ }
+}
+
+static int do_cpt_int(XDR *xd, const char *desc, int *i, FILE *list)
+{
+ bool_t res = 0;
+
+ res = xdr_int(xd, i);
+ if (res == 0)
+ {
+ return -1;
+ }
+ if (list)
+ {
+ fprintf(list, "%s = %d\n", desc, *i);
+ }
+ return 0;
+}
+
+static int do_cpt_u_chars(XDR *xd, const char *desc, int n, unsigned char *i, FILE *list)
+{
+ bool_t res = 1;
+ int j;
+ if (list)
+ {
+ fprintf(list, "%s = ", desc);
+ }
+ for (j = 0; j < n && res; j++)
+ {
+ res &= xdr_u_char(xd, &i[j]);
+ if (list)
+ {
+ fprintf(list, "%02x", i[j]);
+ }
+ }
+ if (list)
+ {
+ fprintf(list, "\n");
+ }
+ if (res == 0)
+ {
+ return -1;
+ }
+
+ return 0;
+}
+
+static void do_cpt_int_err(XDR *xd, const char *desc, int *i, FILE *list)
+{
+ if (do_cpt_int(xd, desc, i, list) < 0)
+ {
+ cp_error();
+ }
+}
+
+static void do_cpt_step_err(XDR *xd, const char *desc, gmx_large_int_t *i, FILE *list)
+{
+ bool_t res = 0;
+ char buf[STEPSTRSIZE];
+
+ res = xdr_gmx_large_int(xd, i);
+ if (res == 0)
+ {
+ cp_error();
+ }
+ if (list)
+ {
+ fprintf(list, "%s = %s\n", desc, gmx_step_str(*i, buf));
+ }
+}
+
+static void do_cpt_double_err(XDR *xd, const char *desc, double *f, FILE *list)
+{
+ bool_t res = 0;
+
+ res = xdr_double(xd, f);
+ if (res == 0)
+ {
+ cp_error();
+ }
+ if (list)
+ {
+ fprintf(list, "%s = %f\n", desc, *f);
+ }
+}
+
+static void do_cpt_real_err(XDR *xd, real *f)
+{
+ bool_t res = 0;
+
+#ifdef GMX_DOUBLE
+ res = xdr_double(xd, f);
+#else
+ res = xdr_float(xd, f);
+#endif
+ if (res == 0)
+ {
+ cp_error();
+ }
+}
+
+static void do_cpt_n_rvecs_err(XDR *xd, const char *desc, int n, rvec f[], FILE *list)
+{
+ int i, j;
+
+ for (i = 0; i < n; i++)
+ {
+ for (j = 0; j < DIM; j++)
+ {
+ do_cpt_real_err(xd, &f[i][j]);
+ }
+ }
+
+ if (list)
+ {
+ pr_rvecs(list, 0, desc, f, n);
+ }
+}
+
+/* If nval >= 0, nval is used; on read this should match the passed value.
+ * If nval n<0, *nptr is used; on read the value is stored in nptr
+ */
+static int do_cpte_reals_low(XDR *xd, int cptp, int ecpt, int sflags,
+ int nval, int *nptr, real **v,
+ FILE *list, int erealtype)
+{
+ bool_t res = 0;
+#ifndef GMX_DOUBLE
+ int dtc = xdr_datatype_float;
+#else
+ int dtc = xdr_datatype_double;
+#endif
+ real *vp, *va = NULL;
+ float *vf;
+ double *vd;
+ int nf, dt, i;
+
+ if (list == NULL)
+ {
+ if (nval >= 0)
+ {
+ nf = nval;
+ }
+ else
+ {
+ if (nptr == NULL)
+ {
+ gmx_incons("*ntpr=NULL in do_cpte_reals_low");
+ }
+ nf = *nptr;
+ }
+ }
+ res = xdr_int(xd, &nf);
+ if (res == 0)
+ {
+ return -1;
+ }
+ if (list == NULL)
+ {
+ if (nval >= 0)
+ {
+ if (nf != nval)
+ {
+ gmx_fatal(FARGS, "Count mismatch for state entry %s, code count is %d, file count is %d\n", st_names(cptp, ecpt), nval, nf);
+ }
+ }
+ else
+ {
+ *nptr = nf;
+ }
+ }
+ dt = dtc;
+ res = xdr_int(xd, &dt);
+ if (res == 0)
+ {
+ return -1;
+ }
+ if (dt != dtc)
+ {
+ fprintf(stderr, "Precision mismatch for state entry %s, code precision is %s, file precision is %s\n",
+ st_names(cptp, ecpt), xdr_datatype_names[dtc],
+ xdr_datatype_names[dt]);
+ }
+ if (list || !(sflags & (1<<ecpt)))
+ {
+ snew(va, nf);
+ vp = va;
+ }
+ else
+ {
+ if (*v == NULL)
+ {
+ snew(*v, nf);
+ }
+ vp = *v;
+ }
+ if (dt == xdr_datatype_float)
+ {
+ if (dtc == xdr_datatype_float)
+ {
+ vf = (float *)vp;
+ }
+ else
+ {
+ snew(vf, nf);
+ }
+ res = xdr_vector(xd, (char *)vf, nf,
+ (unsigned int)sizeof(float), (xdrproc_t)xdr_float);
+ if (res == 0)
+ {
+ return -1;
+ }
+ if (dtc != xdr_datatype_float)
+ {
+ for (i = 0; i < nf; i++)
+ {
+ vp[i] = vf[i];
+ }
+ sfree(vf);
+ }
+ }
+ else
+ {
+ if (dtc == xdr_datatype_double)
+ {
+ vd = (double *)vp;
+ }
+ else
+ {
+ snew(vd, nf);
+ }
+ res = xdr_vector(xd, (char *)vd, nf,
+ (unsigned int)sizeof(double), (xdrproc_t)xdr_double);
+ if (res == 0)
+ {
+ return -1;
+ }
+ if (dtc != xdr_datatype_double)
+ {
+ for (i = 0; i < nf; i++)
+ {
+ vp[i] = vd[i];
+ }
+ sfree(vd);
+ }
+ }
+
+ if (list)
+ {
+ switch (erealtype)
+ {
+ case ecprREAL:
+ pr_reals(list, 0, st_names(cptp, ecpt), vp, nf);
+ break;
+ case ecprRVEC:
+ pr_rvecs(list, 0, st_names(cptp, ecpt), (rvec *)vp, nf/3);
+ break;
+ default:
+ gmx_incons("Unknown checkpoint real type");
+ }
+ }
+ if (va)
+ {
+ sfree(va);
+ }
+
+ return 0;
+}
+
+
+/* This function stores n along with the reals for reading,
+ * but on reading it assumes that n matches the value in the checkpoint file,
+ * a fatal error is generated when this is not the case.
+ */
+static int do_cpte_reals(XDR *xd, int cptp, int ecpt, int sflags,
+ int n, real **v, FILE *list)
+{
+ return do_cpte_reals_low(xd, cptp, ecpt, sflags, n, NULL, v, list, ecprREAL);
+}
+
+/* This function does the same as do_cpte_reals,
+ * except that on reading it ignores the passed value of *n
+ * and stored the value read from the checkpoint file in *n.
+ */
+static int do_cpte_n_reals(XDR *xd, int cptp, int ecpt, int sflags,
+ int *n, real **v, FILE *list)
+{
+ return do_cpte_reals_low(xd, cptp, ecpt, sflags, -1, n, v, list, ecprREAL);
+}
+
+static int do_cpte_real(XDR *xd, int cptp, int ecpt, int sflags,
+ real *r, FILE *list)
+{
+ int n;
+
+ return do_cpte_reals_low(xd, cptp, ecpt, sflags, 1, NULL, &r, list, ecprREAL);
+}
+
+static int do_cpte_ints(XDR *xd, int cptp, int ecpt, int sflags,
+ int n, int **v, FILE *list)
+{
+ bool_t res = 0;
+ int dtc = xdr_datatype_int;
+ int *vp, *va = NULL;
+ int nf, dt, i;
+
+ nf = n;
+ res = xdr_int(xd, &nf);
+ if (res == 0)
+ {
+ return -1;
+ }
+ if (list == NULL && v != NULL && nf != n)
+ {
+ gmx_fatal(FARGS, "Count mismatch for state entry %s, code count is %d, file count is %d\n", st_names(cptp, ecpt), n, nf);
+ }
+ dt = dtc;
+ res = xdr_int(xd, &dt);
+ if (res == 0)
+ {
+ return -1;
+ }
+ if (dt != dtc)
+ {
+ gmx_fatal(FARGS, "Type mismatch for state entry %s, code type is %s, file type is %s\n",
+ st_names(cptp, ecpt), xdr_datatype_names[dtc],
+ xdr_datatype_names[dt]);
+ }
+ if (list || !(sflags & (1<<ecpt)) || v == NULL)
+ {
+ snew(va, nf);
+ vp = va;
+ }
+ else
+ {
+ if (*v == NULL)
+ {
+ snew(*v, nf);
+ }
+ vp = *v;
+ }
+ res = xdr_vector(xd, (char *)vp, nf,
+ (unsigned int)sizeof(int), (xdrproc_t)xdr_int);
+ if (res == 0)
+ {
+ return -1;
+ }
+ if (list)
+ {
+ pr_ivec(list, 0, st_names(cptp, ecpt), vp, nf, TRUE);
+ }
+ if (va)
+ {
+ sfree(va);
+ }
+
+ return 0;
+}
+
+static int do_cpte_int(XDR *xd, int cptp, int ecpt, int sflags,
+ int *i, FILE *list)
+{
+ return do_cpte_ints(xd, cptp, ecpt, sflags, 1, &i, list);
+}
+
+static int do_cpte_doubles(XDR *xd, int cptp, int ecpt, int sflags,
+ int n, double **v, FILE *list)
+{
+ bool_t res = 0;
+ int dtc = xdr_datatype_double;
+ double *vp, *va = NULL;
+ int nf, dt, i;
+
+ nf = n;
+ res = xdr_int(xd, &nf);
+ if (res == 0)
+ {
+ return -1;
+ }
+ if (list == NULL && nf != n)
+ {
+ gmx_fatal(FARGS, "Count mismatch for state entry %s, code count is %d, file count is %d\n", st_names(cptp, ecpt), n, nf);
+ }
+ dt = dtc;
+ res = xdr_int(xd, &dt);
+ if (res == 0)
+ {
+ return -1;
+ }
+ if (dt != dtc)
+ {
+ gmx_fatal(FARGS, "Precision mismatch for state entry %s, code precision is %s, file precision is %s\n",
+ st_names(cptp, ecpt), xdr_datatype_names[dtc],
+ xdr_datatype_names[dt]);
+ }
+ if (list || !(sflags & (1<<ecpt)))
+ {
+ snew(va, nf);
+ vp = va;
+ }
+ else
+ {
+ if (*v == NULL)
+ {
+ snew(*v, nf);
+ }
+ vp = *v;
+ }
+ res = xdr_vector(xd, (char *)vp, nf,
+ (unsigned int)sizeof(double), (xdrproc_t)xdr_double);
+ if (res == 0)
+ {
+ return -1;
+ }
+ if (list)
+ {
+ pr_doubles(list, 0, st_names(cptp, ecpt), vp, nf);
+ }
+ if (va)
+ {
+ sfree(va);
+ }
+
+ return 0;
+}
+
+static int do_cpte_double(XDR *xd, int cptp, int ecpt, int sflags,
+ double *r, FILE *list)
+{
+ return do_cpte_doubles(xd, cptp, ecpt, sflags, 1, &r, list);
+}
+
+
+static int do_cpte_rvecs(XDR *xd, int cptp, int ecpt, int sflags,
+ int n, rvec **v, FILE *list)
+{
+ int n3;
+
+ return do_cpte_reals_low(xd, cptp, ecpt, sflags,
+ n*DIM, NULL, (real **)v, list, ecprRVEC);
+}
+
+static int do_cpte_matrix(XDR *xd, int cptp, int ecpt, int sflags,
+ matrix v, FILE *list)
+{
+ real *vr;
+ real ret;
+
+ vr = (real *)&(v[0][0]);
+ ret = do_cpte_reals_low(xd, cptp, ecpt, sflags,
+ DIM*DIM, NULL, &vr, NULL, ecprMATRIX);
+
+ if (list && ret == 0)
+ {
+ pr_rvecs(list, 0, st_names(cptp, ecpt), v, DIM);
+ }
+
+ return ret;
+}
+
+
+static int do_cpte_nmatrix(XDR *xd, int cptp, int ecpt, int sflags,
+ int n, real **v, FILE *list)
+{
+ int i;
+ real *vr;
+ real ret, reti;
+ char name[CPTSTRLEN];
+
+ ret = 0;
+ if (v == NULL)
+ {
+ snew(v, n);
+ }
+ for (i = 0; i < n; i++)
+ {
+ reti = 0;
+ vr = v[i];
+ reti = do_cpte_reals_low(xd, cptp, ecpt, sflags, n, NULL, &(v[i]), NULL, ecprREAL);
+ if (list && reti == 0)
+ {
+ sprintf(name, "%s[%d]", st_names(cptp, ecpt), i);
+ pr_reals(list, 0, name, v[i], n);
+ }
+ if (reti == 0)
+ {
+ ret = 0;
+ }
+ }
+ return ret;
+}
+
+static int do_cpte_matrices(XDR *xd, int cptp, int ecpt, int sflags,
+ int n, matrix **v, FILE *list)
+{
+ bool_t res = 0;
+ matrix *vp, *va = NULL;
+ real *vr;
+ int nf, i, j, k;
+ int ret;
+
+ nf = n;
+ res = xdr_int(xd, &nf);
+ if (res == 0)
+ {
+ return -1;
+ }
+ if (list == NULL && nf != n)
+ {
+ gmx_fatal(FARGS, "Count mismatch for state entry %s, code count is %d, file count is %d\n", st_names(cptp, ecpt), n, nf);
+ }
+ if (list || !(sflags & (1<<ecpt)))
+ {
+ snew(va, nf);
+ vp = va;
+ }
+ else
+ {
+ if (*v == NULL)
+ {
+ snew(*v, nf);
+ }
+ vp = *v;
+ }
+ snew(vr, nf*DIM*DIM);
+ for (i = 0; i < nf; i++)
+ {
+ for (j = 0; j < DIM; j++)
+ {
+ for (k = 0; k < DIM; k++)
+ {
+ vr[(i*DIM+j)*DIM+k] = vp[i][j][k];
+ }
+ }
+ }
+ ret = do_cpte_reals_low(xd, cptp, ecpt, sflags,
+ nf*DIM*DIM, NULL, &vr, NULL, ecprMATRIX);
+ for (i = 0; i < nf; i++)
+ {
+ for (j = 0; j < DIM; j++)
+ {
+ for (k = 0; k < DIM; k++)
+ {
+ vp[i][j][k] = vr[(i*DIM+j)*DIM+k];
+ }
+ }
+ }
+ sfree(vr);
+
+ if (list && ret == 0)
+ {
+ for (i = 0; i < nf; i++)
+ {
+ pr_rvecs(list, 0, st_names(cptp, ecpt), vp[i], DIM);
+ }
+ }
+ if (va)
+ {
+ sfree(va);
+ }
+
+ return ret;
+}
+
+static void do_cpt_header(XDR *xd, gmx_bool bRead, int *file_version,
+ char **version, char **btime, char **buser, char **bhost,
+ int *double_prec,
+ char **fprog, char **ftime,
+ int *eIntegrator, int *simulation_part,
+ gmx_large_int_t *step, double *t,
+ int *nnodes, int *dd_nc, int *npme,
+ int *natoms, int *ngtc, int *nnhpres, int *nhchainlength,
+ int *nlambda, int *flags_state,
+ int *flags_eks, int *flags_enh, int *flags_dfh,
+ int *nED,
+ FILE *list)
+{
+ bool_t res = 0;
+ int magic;
+ int idum = 0;
+ int i;
+ char *fhost;
+
+ if (bRead)
+ {
+ magic = -1;
+ }
+ else
+ {
+ magic = CPT_MAGIC1;
+ }
+ res = xdr_int(xd, &magic);
+ if (res == 0)
+ {
+ gmx_fatal(FARGS, "The checkpoint file is empty/corrupted, or maybe you are out of disk space?");
+ }
+ if (magic != CPT_MAGIC1)
+ {
+ gmx_fatal(FARGS, "Start of file magic number mismatch, checkpoint file has %d, should be %d\n"
+ "The checkpoint file is corrupted or not a checkpoint file",
+ magic, CPT_MAGIC1);
+ }
+ if (!bRead)
+ {
+ snew(fhost, 255);
+#ifdef HAVE_UNISTD_H
+ if (gethostname(fhost, 255) != 0)
+ {
+ sprintf(fhost, "unknown");
+ }
+#else
+ sprintf(fhost, "unknown");
+#endif
+ }
+ do_cpt_string_err(xd, bRead, "GROMACS version", version, list);
+ do_cpt_string_err(xd, bRead, "GROMACS build time", btime, list);
+ do_cpt_string_err(xd, bRead, "GROMACS build user", buser, list);
+ do_cpt_string_err(xd, bRead, "GROMACS build host", bhost, list);
+ do_cpt_string_err(xd, bRead, "generating program", fprog, list);
+ do_cpt_string_err(xd, bRead, "generation time", ftime, list);
+ *file_version = cpt_version;
+ do_cpt_int_err(xd, "checkpoint file version", file_version, list);
+ if (*file_version > cpt_version)
+ {
+ gmx_fatal(FARGS, "Attempting to read a checkpoint file of version %d with code of version %d\n", *file_version, cpt_version);
+ }
+ if (*file_version >= 13)
+ {
+ do_cpt_int_err(xd, "GROMACS double precision", double_prec, list);
+ }
+ else
+ {
+ *double_prec = -1;
+ }
+ if (*file_version >= 12)
+ {
+ do_cpt_string_err(xd, bRead, "generating host", &fhost, list);
+ if (list == NULL)
+ {
+ sfree(fhost);
+ }
+ }
+ do_cpt_int_err(xd, "#atoms", natoms, list);
+ do_cpt_int_err(xd, "#T-coupling groups", ngtc, list);
+ if (*file_version >= 10)
+ {
+ do_cpt_int_err(xd, "#Nose-Hoover T-chains", nhchainlength, list);
+ }
+ else
+ {
+ *nhchainlength = 1;
+ }
+ if (*file_version >= 11)
+ {
+ do_cpt_int_err(xd, "#Nose-Hoover T-chains for barostat ", nnhpres, list);
+ }
+ else
+ {
+ *nnhpres = 0;
+ }
+ if (*file_version >= 14)
+ {
+ do_cpt_int_err(xd, "# of total lambda states ", nlambda, list);
+ }
+ else
+ {
+ *nlambda = 0;
+ }
+ do_cpt_int_err(xd, "integrator", eIntegrator, list);
+ if (*file_version >= 3)
+ {
+ do_cpt_int_err(xd, "simulation part #", simulation_part, list);
+ }
+ else
+ {
+ *simulation_part = 1;
+ }
+ if (*file_version >= 5)
+ {
+ do_cpt_step_err(xd, "step", step, list);
+ }
+ else
+ {
+ do_cpt_int_err(xd, "step", &idum, list);
+ *step = idum;
+ }
+ do_cpt_double_err(xd, "t", t, list);
+ do_cpt_int_err(xd, "#PP-nodes", nnodes, list);
+ idum = 1;
+ do_cpt_int_err(xd, "dd_nc[x]", dd_nc ? &(dd_nc[0]) : &idum, list);
+ do_cpt_int_err(xd, "dd_nc[y]", dd_nc ? &(dd_nc[1]) : &idum, list);
+ do_cpt_int_err(xd, "dd_nc[z]", dd_nc ? &(dd_nc[2]) : &idum, list);
+ do_cpt_int_err(xd, "#PME-only nodes", npme, list);
+ do_cpt_int_err(xd, "state flags", flags_state, list);
+ if (*file_version >= 4)
+ {
+ do_cpt_int_err(xd, "ekin data flags", flags_eks, list);
+ do_cpt_int_err(xd, "energy history flags", flags_enh, list);
+ }
+ else
+ {
+ *flags_eks = 0;
+ *flags_enh = (*flags_state >> (estORIRE_DTAV+1));
+ *flags_state = (*flags_state & ~((1<<(estORIRE_DTAV+1)) |
+ (1<<(estORIRE_DTAV+2)) |
+ (1<<(estORIRE_DTAV+3))));
+ }
+ if (*file_version >= 14)
+ {
+ do_cpt_int_err(xd, "df history flags", flags_dfh, list);
+ }
+ else
+ {
+ *flags_dfh = 0;
+ }
+
+ if (*file_version >= 15)
+ {
+ do_cpt_int_err(xd, "ED data sets", nED, list);
+ }
+ else
+ {
+ *nED = 0;
+ }
+}
+
+static int do_cpt_footer(XDR *xd, int file_version)
+{
+ bool_t res = 0;
+ int magic;
+
+ if (file_version >= 2)
+ {
+ magic = CPT_MAGIC2;
+ res = xdr_int(xd, &magic);
+ if (res == 0)
+ {
+ cp_error();
+ }
+ if (magic != CPT_MAGIC2)
+ {
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
- if ((do_cpt_state(gmx_fio_getxdr(fp), state->flags, state, TRUE, NULL) < 0) ||
++static int do_cpt_state(XDR *xd, gmx_bool bRead,
++ int fflags, t_state *state,
+ gmx_bool bReadRNG, FILE *list)
+{
+ int sflags;
+ int **rng_p, **rngi_p;
+ int i;
+ int ret;
+ int nnht, nnhtp;
+
+ ret = 0;
+
+ nnht = state->nhchainlength*state->ngtc;
+ nnhtp = state->nhchainlength*state->nnhpres;
+
+ if (bReadRNG)
+ {
+ rng_p = (int **)&state->ld_rng;
+ rngi_p = &state->ld_rngi;
+ }
+ else
+ {
+ /* Do not read the RNG data */
+ rng_p = NULL;
+ rngi_p = NULL;
+ }
++
++ if (bRead) /* we need to allocate space for dfhist if we are reading */
++ {
++ init_df_history(&state->dfhist,state->dfhist.nlambda);
++ }
++
+ /* We want the MC_RNG the same across all the notes for now -- lambda MC is global */
+
+ sflags = state->flags;
+ for (i = 0; (i < estNR && ret == 0); i++)
+ {
+ if (fflags & (1<<i))
+ {
+ switch (i)
+ {
+ case estLAMBDA: ret = do_cpte_reals(xd, cptpEST, i, sflags, efptNR, &(state->lambda), list); break;
+ case estFEPSTATE: ret = do_cpte_int (xd, cptpEST, i, sflags, &state->fep_state, list); break;
+ case estBOX: ret = do_cpte_matrix(xd, cptpEST, i, sflags, state->box, list); break;
+ case estBOX_REL: ret = do_cpte_matrix(xd, cptpEST, i, sflags, state->box_rel, list); break;
+ case estBOXV: ret = do_cpte_matrix(xd, cptpEST, i, sflags, state->boxv, list); break;
+ case estPRES_PREV: ret = do_cpte_matrix(xd, cptpEST, i, sflags, state->pres_prev, list); break;
+ case estSVIR_PREV: ret = do_cpte_matrix(xd, cptpEST, i, sflags, state->svir_prev, list); break;
+ case estFVIR_PREV: ret = do_cpte_matrix(xd, cptpEST, i, sflags, state->fvir_prev, list); break;
+ case estNH_XI: ret = do_cpte_doubles(xd, cptpEST, i, sflags, nnht, &state->nosehoover_xi, list); break;
+ case estNH_VXI: ret = do_cpte_doubles(xd, cptpEST, i, sflags, nnht, &state->nosehoover_vxi, list); break;
+ case estNHPRES_XI: ret = do_cpte_doubles(xd, cptpEST, i, sflags, nnhtp, &state->nhpres_xi, list); break;
+ case estNHPRES_VXI: ret = do_cpte_doubles(xd, cptpEST, i, sflags, nnhtp, &state->nhpres_vxi, list); break;
+ case estTC_INT: ret = do_cpte_doubles(xd, cptpEST, i, sflags, state->ngtc, &state->therm_integral, list); break;
+ case estVETA: ret = do_cpte_real(xd, cptpEST, i, sflags, &state->veta, list); break;
+ case estVOL0: ret = do_cpte_real(xd, cptpEST, i, sflags, &state->vol0, list); break;
+ case estX: ret = do_cpte_rvecs(xd, cptpEST, i, sflags, state->natoms, &state->x, list); break;
+ case estV: ret = do_cpte_rvecs(xd, cptpEST, i, sflags, state->natoms, &state->v, list); break;
+ case estSDX: ret = do_cpte_rvecs(xd, cptpEST, i, sflags, state->natoms, &state->sd_X, list); break;
+ case estLD_RNG: ret = do_cpte_ints(xd, cptpEST, i, sflags, state->nrng, rng_p, list); break;
+ case estLD_RNGI: ret = do_cpte_ints(xd, cptpEST, i, sflags, state->nrngi, rngi_p, list); break;
+ case estMC_RNG: ret = do_cpte_ints(xd, cptpEST, i, sflags, state->nmcrng, (int **)&state->mc_rng, list); break;
+ case estMC_RNGI: ret = do_cpte_ints(xd, cptpEST, i, sflags, 1, &state->mc_rngi, list); break;
+ case estDISRE_INITF: ret = do_cpte_real (xd, cptpEST, i, sflags, &state->hist.disre_initf, list); break;
+ case estDISRE_RM3TAV: ret = do_cpte_n_reals(xd, cptpEST, i, sflags, &state->hist.ndisrepairs, &state->hist.disre_rm3tav, list); break;
+ case estORIRE_INITF: ret = do_cpte_real (xd, cptpEST, i, sflags, &state->hist.orire_initf, list); break;
+ case estORIRE_DTAV: ret = do_cpte_n_reals(xd, cptpEST, i, sflags, &state->hist.norire_Dtav, &state->hist.orire_Dtav, list); break;
+ default:
+ gmx_fatal(FARGS, "Unknown state entry %d\n"
+ "You are probably reading a new checkpoint file with old code", i);
+ }
+ }
+ }
+
+ return ret;
+}
+
+static int do_cpt_ekinstate(XDR *xd, int fflags, ekinstate_t *ekins,
+ FILE *list)
+{
+ int i;
+ int ret;
+
+ ret = 0;
+
+ for (i = 0; (i < eeksNR && ret == 0); i++)
+ {
+ if (fflags & (1<<i))
+ {
+ switch (i)
+ {
+
+ case eeksEKIN_N: ret = do_cpte_int(xd, cptpEEKS, i, fflags, &ekins->ekin_n, list); break;
+ case eeksEKINH: ret = do_cpte_matrices(xd, cptpEEKS, i, fflags, ekins->ekin_n, &ekins->ekinh, list); break;
+ case eeksEKINF: ret = do_cpte_matrices(xd, cptpEEKS, i, fflags, ekins->ekin_n, &ekins->ekinf, list); break;
+ case eeksEKINO: ret = do_cpte_matrices(xd, cptpEEKS, i, fflags, ekins->ekin_n, &ekins->ekinh_old, list); break;
+ case eeksEKINTOTAL: ret = do_cpte_matrix(xd, cptpEEKS, i, fflags, ekins->ekin_total, list); break;
+ case eeksEKINSCALEF: ret = do_cpte_doubles(xd, cptpEEKS, i, fflags, ekins->ekin_n, &ekins->ekinscalef_nhc, list); break;
+ case eeksVSCALE: ret = do_cpte_doubles(xd, 1, cptpEEKS, fflags, ekins->ekin_n, &ekins->vscale_nhc, list); break;
+ case eeksEKINSCALEH: ret = do_cpte_doubles(xd, 1, cptpEEKS, fflags, ekins->ekin_n, &ekins->ekinscaleh_nhc, list); break;
+ case eeksDEKINDL: ret = do_cpte_real(xd, 1, cptpEEKS, fflags, &ekins->dekindl, list); break;
+ case eeksMVCOS: ret = do_cpte_real(xd, 1, cptpEEKS, fflags, &ekins->mvcos, list); break;
+ default:
+ gmx_fatal(FARGS, "Unknown ekin data state entry %d\n"
+ "You are probably reading a new checkpoint file with old code", i);
+ }
+ }
+ }
+
+ return ret;
+}
+
+
+static int do_cpt_enerhist(XDR *xd, gmx_bool bRead,
+ int fflags, energyhistory_t *enerhist,
+ FILE *list)
+{
+ int i;
+ int j;
+ int ret;
+
+ ret = 0;
+
+ if (bRead)
+ {
+ enerhist->nsteps = 0;
+ enerhist->nsum = 0;
+ enerhist->nsteps_sim = 0;
+ enerhist->nsum_sim = 0;
+ enerhist->dht = NULL;
+
+ if (fflags & (1<< eenhENERGY_DELTA_H_NN) )
+ {
+ snew(enerhist->dht, 1);
+ enerhist->dht->ndh = NULL;
+ enerhist->dht->dh = NULL;
+ enerhist->dht->start_lambda_set = FALSE;
+ }
+ }
+
+ for (i = 0; (i < eenhNR && ret == 0); i++)
+ {
+ if (fflags & (1<<i))
+ {
+ switch (i)
+ {
+ case eenhENERGY_N: ret = do_cpte_int(xd, cptpEENH, i, fflags, &enerhist->nener, list); break;
+ case eenhENERGY_AVER: ret = do_cpte_doubles(xd, cptpEENH, i, fflags, enerhist->nener, &enerhist->ener_ave, list); break;
+ case eenhENERGY_SUM: ret = do_cpte_doubles(xd, cptpEENH, i, fflags, enerhist->nener, &enerhist->ener_sum, list); break;
+ case eenhENERGY_NSUM: do_cpt_step_err(xd, eenh_names[i], &enerhist->nsum, list); break;
+ case eenhENERGY_SUM_SIM: ret = do_cpte_doubles(xd, cptpEENH, i, fflags, enerhist->nener, &enerhist->ener_sum_sim, list); break;
+ case eenhENERGY_NSUM_SIM: do_cpt_step_err(xd, eenh_names[i], &enerhist->nsum_sim, list); break;
+ case eenhENERGY_NSTEPS: do_cpt_step_err(xd, eenh_names[i], &enerhist->nsteps, list); break;
+ case eenhENERGY_NSTEPS_SIM: do_cpt_step_err(xd, eenh_names[i], &enerhist->nsteps_sim, list); break;
+ case eenhENERGY_DELTA_H_NN: do_cpt_int_err(xd, eenh_names[i], &(enerhist->dht->nndh), list);
+ if (bRead) /* now allocate memory for it */
+ {
+ snew(enerhist->dht->dh, enerhist->dht->nndh);
+ snew(enerhist->dht->ndh, enerhist->dht->nndh);
+ for (j = 0; j < enerhist->dht->nndh; j++)
+ {
+ enerhist->dht->ndh[j] = 0;
+ enerhist->dht->dh[j] = NULL;
+ }
+ }
+ break;
+ case eenhENERGY_DELTA_H_LIST:
+ for (j = 0; j < enerhist->dht->nndh; j++)
+ {
+ ret = do_cpte_n_reals(xd, cptpEENH, i, fflags, &enerhist->dht->ndh[j], &(enerhist->dht->dh[j]), list);
+ }
+ break;
+ case eenhENERGY_DELTA_H_STARTTIME:
+ ret = do_cpte_double(xd, cptpEENH, i, fflags, &(enerhist->dht->start_time), list); break;
+ case eenhENERGY_DELTA_H_STARTLAMBDA:
+ ret = do_cpte_double(xd, cptpEENH, i, fflags, &(enerhist->dht->start_lambda), list); break;
+ default:
+ gmx_fatal(FARGS, "Unknown energy history entry %d\n"
+ "You are probably reading a new checkpoint file with old code", i);
+ }
+ }
+ }
+
+ if ((fflags & (1<<eenhENERGY_SUM)) && !(fflags & (1<<eenhENERGY_SUM_SIM)))
+ {
+ /* Assume we have an old file format and copy sum to sum_sim */
+ srenew(enerhist->ener_sum_sim, enerhist->nener);
+ for (i = 0; i < enerhist->nener; i++)
+ {
+ enerhist->ener_sum_sim[i] = enerhist->ener_sum[i];
+ }
+ fflags |= (1<<eenhENERGY_SUM_SIM);
+ }
+
+ if ( (fflags & (1<<eenhENERGY_NSUM)) &&
+ !(fflags & (1<<eenhENERGY_NSTEPS)))
+ {
+ /* Assume we have an old file format and copy nsum to nsteps */
+ enerhist->nsteps = enerhist->nsum;
+ fflags |= (1<<eenhENERGY_NSTEPS);
+ }
+ if ( (fflags & (1<<eenhENERGY_NSUM_SIM)) &&
+ !(fflags & (1<<eenhENERGY_NSTEPS_SIM)))
+ {
+ /* Assume we have an old file format and copy nsum to nsteps */
+ enerhist->nsteps_sim = enerhist->nsum_sim;
+ fflags |= (1<<eenhENERGY_NSTEPS_SIM);
+ }
+
+ return ret;
+}
+
+static int do_cpt_df_hist(XDR *xd, int fflags, df_history_t *dfhist, FILE *list)
+{
+ int i, nlambda;
+ int ret;
+
+ nlambda = dfhist->nlambda;
+ ret = 0;
+
+ for (i = 0; (i < edfhNR && ret == 0); i++)
+ {
+ if (fflags & (1<<i))
+ {
+ switch (i)
+ {
+ case edfhBEQUIL: ret = do_cpte_int(xd, cptpEDFH, i, fflags, &dfhist->bEquil, list); break;
+ case edfhNATLAMBDA: ret = do_cpte_ints(xd, cptpEDFH, i, fflags, nlambda, &dfhist->n_at_lam, list); break;
+ case edfhWLHISTO: ret = do_cpte_reals(xd, cptpEDFH, i, fflags, nlambda, &dfhist->wl_histo, list); break;
+ case edfhWLDELTA: ret = do_cpte_real(xd, cptpEDFH, i, fflags, &dfhist->wl_delta, list); break;
+ case edfhSUMWEIGHTS: ret = do_cpte_reals(xd, cptpEDFH, i, fflags, nlambda, &dfhist->sum_weights, list); break;
+ case edfhSUMDG: ret = do_cpte_reals(xd, cptpEDFH, i, fflags, nlambda, &dfhist->sum_dg, list); break;
+ case edfhSUMMINVAR: ret = do_cpte_reals(xd, cptpEDFH, i, fflags, nlambda, &dfhist->sum_minvar, list); break;
+ case edfhSUMVAR: ret = do_cpte_reals(xd, cptpEDFH, i, fflags, nlambda, &dfhist->sum_variance, list); break;
+ case edfhACCUMP: ret = do_cpte_nmatrix(xd, cptpEDFH, i, fflags, nlambda, dfhist->accum_p, list); break;
+ case edfhACCUMM: ret = do_cpte_nmatrix(xd, cptpEDFH, i, fflags, nlambda, dfhist->accum_m, list); break;
+ case edfhACCUMP2: ret = do_cpte_nmatrix(xd, cptpEDFH, i, fflags, nlambda, dfhist->accum_p2, list); break;
+ case edfhACCUMM2: ret = do_cpte_nmatrix(xd, cptpEDFH, i, fflags, nlambda, dfhist->accum_m2, list); break;
+ case edfhTIJ: ret = do_cpte_nmatrix(xd, cptpEDFH, i, fflags, nlambda, dfhist->Tij, list); break;
+ case edfhTIJEMP: ret = do_cpte_nmatrix(xd, cptpEDFH, i, fflags, nlambda, dfhist->Tij_empirical, list); break;
+
+ default:
+ gmx_fatal(FARGS, "Unknown df history entry %d\n"
+ "You are probably reading a new checkpoint file with old code", i);
+ }
+ }
+ }
+
+ return ret;
+}
+
+
+/* This function stores the last whole configuration of the reference and
+ * average structure in the .cpt file
+ */
+static int do_cpt_EDstate(XDR *xd, gmx_bool bRead,
+ edsamstate_t *EDstate, FILE *list)
+{
+ int i, j;
+ int ret = 0;
+ char buf[STRLEN];
+
+
+ EDstate->bFromCpt = bRead;
+
+ if (EDstate->nED <= 0)
+ {
+ return ret;
+ }
+
+ /* When reading, init_edsam has not been called yet,
+ * so we have to allocate memory first. */
+ if (bRead)
+ {
+ snew(EDstate->nref, EDstate->nED);
+ snew(EDstate->old_sref, EDstate->nED);
+ snew(EDstate->nav, EDstate->nED);
+ snew(EDstate->old_sav, EDstate->nED);
+ }
+
+ /* Read/write the last whole conformation of SREF and SAV for each ED dataset (usually only one) */
+ for (i = 0; i < EDstate->nED; i++)
+ {
+ /* Reference structure SREF */
+ sprintf(buf, "ED%d # of atoms in reference structure", i+1);
+ do_cpt_int_err(xd, buf, &EDstate->nref[i], list);
+ sprintf(buf, "ED%d x_ref", i+1);
+ if (bRead)
+ {
+ snew(EDstate->old_sref[i], EDstate->nref[i]);
+ do_cpt_n_rvecs_err(xd, buf, EDstate->nref[i], EDstate->old_sref[i], list);
+ }
+ else
+ {
+ do_cpt_n_rvecs_err(xd, buf, EDstate->nref[i], EDstate->old_sref_p[i], list);
+ }
+
+ /* Average structure SAV */
+ sprintf(buf, "ED%d # of atoms in average structure", i+1);
+ do_cpt_int_err(xd, buf, &EDstate->nav[i], list);
+ sprintf(buf, "ED%d x_av", i+1);
+ if (bRead)
+ {
+ snew(EDstate->old_sav[i], EDstate->nav[i]);
+ do_cpt_n_rvecs_err(xd, buf, EDstate->nav[i], EDstate->old_sav[i], list);
+ }
+ else
+ {
+ do_cpt_n_rvecs_err(xd, buf, EDstate->nav[i], EDstate->old_sav_p[i], list);
+ }
+ }
+
+ return ret;
+}
+
+
+static int do_cpt_files(XDR *xd, gmx_bool bRead,
+ gmx_file_position_t **p_outputfiles, int *nfiles,
+ FILE *list, int file_version)
+{
+ int i, j;
+ gmx_off_t offset;
+ gmx_off_t mask = 0xFFFFFFFFL;
+ int offset_high, offset_low;
+ char *buf;
+ gmx_file_position_t *outputfiles;
+
+ if (do_cpt_int(xd, "number of output files", nfiles, list) != 0)
+ {
+ return -1;
+ }
+
+ if (bRead)
+ {
+ snew(*p_outputfiles, *nfiles);
+ }
+
+ outputfiles = *p_outputfiles;
+
+ for (i = 0; i < *nfiles; i++)
+ {
+ /* 64-bit XDR numbers are not portable, so it is stored as separate high/low fractions */
+ if (bRead)
+ {
+ do_cpt_string_err(xd, bRead, "output filename", &buf, list);
+ strncpy(outputfiles[i].filename, buf, CPTSTRLEN-1);
+ if (list == NULL)
+ {
+ sfree(buf);
+ }
+
+ if (do_cpt_int(xd, "file_offset_high", &offset_high, list) != 0)
+ {
+ return -1;
+ }
+ if (do_cpt_int(xd, "file_offset_low", &offset_low, list) != 0)
+ {
+ return -1;
+ }
+#if (SIZEOF_GMX_OFF_T > 4)
+ outputfiles[i].offset = ( ((gmx_off_t) offset_high) << 32 ) | ( (gmx_off_t) offset_low & mask );
+#else
+ outputfiles[i].offset = offset_low;
+#endif
+ }
+ else
+ {
+ buf = outputfiles[i].filename;
+ do_cpt_string_err(xd, bRead, "output filename", &buf, list);
+ /* writing */
+ offset = outputfiles[i].offset;
+ if (offset == -1)
+ {
+ offset_low = -1;
+ offset_high = -1;
+ }
+ else
+ {
+#if (SIZEOF_GMX_OFF_T > 4)
+ offset_low = (int) (offset & mask);
+ offset_high = (int) ((offset >> 32) & mask);
+#else
+ offset_low = offset;
+ offset_high = 0;
+#endif
+ }
+ if (do_cpt_int(xd, "file_offset_high", &offset_high, list) != 0)
+ {
+ return -1;
+ }
+ if (do_cpt_int(xd, "file_offset_low", &offset_low, list) != 0)
+ {
+ return -1;
+ }
+ }
+ if (file_version >= 8)
+ {
+ if (do_cpt_int(xd, "file_checksum_size", &(outputfiles[i].chksum_size),
+ list) != 0)
+ {
+ return -1;
+ }
+ if (do_cpt_u_chars(xd, "file_checksum", 16, outputfiles[i].chksum, list) != 0)
+ {
+ return -1;
+ }
+ }
+ else
+ {
+ outputfiles[i].chksum_size = -1;
+ }
+ }
+ return 0;
+}
+
+
+void write_checkpoint(const char *fn, gmx_bool bNumberAndKeep,
+ FILE *fplog, t_commrec *cr,
+ int eIntegrator, int simulation_part,
+ gmx_bool bExpanded, int elamstats,
+ gmx_large_int_t step, double t, t_state *state)
+{
+ t_fileio *fp;
+ int file_version;
+ char *version;
+ char *btime;
+ char *buser;
+ char *bhost;
+ int double_prec;
+ char *fprog;
+ char *fntemp; /* the temporary checkpoint file name */
+ time_t now;
+ char timebuf[STRLEN];
+ int nppnodes, npmenodes, flag_64bit;
+ char buf[1024], suffix[5+STEPSTRSIZE], sbuf[STEPSTRSIZE];
+ gmx_file_position_t *outputfiles;
+ int noutputfiles;
+ char *ftime;
+ int flags_eks, flags_enh, flags_dfh, i;
+ t_fileio *ret;
+
+ if (PAR(cr))
+ {
+ if (DOMAINDECOMP(cr))
+ {
+ nppnodes = cr->dd->nnodes;
+ npmenodes = cr->npmenodes;
+ }
+ else
+ {
+ nppnodes = cr->nnodes;
+ npmenodes = 0;
+ }
+ }
+ else
+ {
+ nppnodes = 1;
+ npmenodes = 0;
+ }
+
+ /* make the new temporary filename */
+ snew(fntemp, strlen(fn)+5+STEPSTRSIZE);
+ strcpy(fntemp, fn);
+ fntemp[strlen(fn) - strlen(ftp2ext(fn2ftp(fn))) - 1] = '\0';
+ sprintf(suffix, "_%s%s", "step", gmx_step_str(step, sbuf));
+ strcat(fntemp, suffix);
+ strcat(fntemp, fn+strlen(fn) - strlen(ftp2ext(fn2ftp(fn))) - 1);
+
+ time(&now);
+ gmx_ctime_r(&now, timebuf, STRLEN);
+
+ if (fplog)
+ {
+ fprintf(fplog, "Writing checkpoint, step %s at %s\n\n",
+ gmx_step_str(step, buf), timebuf);
+ }
+
+ /* Get offsets for open files */
+ gmx_fio_get_output_file_positions(&outputfiles, &noutputfiles);
+
+ fp = gmx_fio_open(fntemp, "w");
+
+ if (state->ekinstate.bUpToDate)
+ {
+ flags_eks =
+ ((1<<eeksEKIN_N) | (1<<eeksEKINH) | (1<<eeksEKINF) |
+ (1<<eeksEKINO) | (1<<eeksEKINSCALEF) | (1<<eeksEKINSCALEH) |
+ (1<<eeksVSCALE) | (1<<eeksDEKINDL) | (1<<eeksMVCOS));
+ }
+ else
+ {
+ flags_eks = 0;
+ }
+
+ flags_enh = 0;
+ if (state->enerhist.nsum > 0 || state->enerhist.nsum_sim > 0)
+ {
+ flags_enh |= (1<<eenhENERGY_N);
+ if (state->enerhist.nsum > 0)
+ {
+ flags_enh |= ((1<<eenhENERGY_AVER) | (1<<eenhENERGY_SUM) |
+ (1<<eenhENERGY_NSTEPS) | (1<<eenhENERGY_NSUM));
+ }
+ if (state->enerhist.nsum_sim > 0)
+ {
+ flags_enh |= ((1<<eenhENERGY_SUM_SIM) | (1<<eenhENERGY_NSTEPS_SIM) |
+ (1<<eenhENERGY_NSUM_SIM));
+ }
+ if (state->enerhist.dht)
+ {
+ flags_enh |= ( (1<< eenhENERGY_DELTA_H_NN) |
+ (1<< eenhENERGY_DELTA_H_LIST) |
+ (1<< eenhENERGY_DELTA_H_STARTTIME) |
+ (1<< eenhENERGY_DELTA_H_STARTLAMBDA) );
+ }
+ }
+
+ if (bExpanded)
+ {
+ flags_dfh = ((1<<edfhBEQUIL) | (1<<edfhNATLAMBDA) | (1<<edfhSUMWEIGHTS) | (1<<edfhSUMDG) |
+ (1<<edfhTIJ) | (1<<edfhTIJEMP));
+ if (EWL(elamstats))
+ {
+ flags_dfh |= ((1<<edfhWLDELTA) | (1<<edfhWLHISTO));
+ }
+ if ((elamstats == elamstatsMINVAR) || (elamstats == elamstatsBARKER) || (elamstats == elamstatsMETROPOLIS))
+ {
+ flags_dfh |= ((1<<edfhACCUMP) | (1<<edfhACCUMM) | (1<<edfhACCUMP2) | (1<<edfhACCUMM2)
+ | (1<<edfhSUMMINVAR) | (1<<edfhSUMVAR));
+ }
+ }
+ else
+ {
+ flags_dfh = 0;
+ }
+
+ /* We can check many more things now (CPU, acceleration, etc), but
+ * it is highly unlikely to have two separate builds with exactly
+ * the same version, user, time, and build host!
+ */
+
+ version = gmx_strdup(VERSION);
+ btime = gmx_strdup(BUILD_TIME);
+ buser = gmx_strdup(BUILD_USER);
+ bhost = gmx_strdup(BUILD_HOST);
+
+ double_prec = GMX_CPT_BUILD_DP;
+ fprog = gmx_strdup(Program());
+
+ ftime = &(timebuf[0]);
+
+ do_cpt_header(gmx_fio_getxdr(fp), FALSE, &file_version,
+ &version, &btime, &buser, &bhost, &double_prec, &fprog, &ftime,
+ &eIntegrator, &simulation_part, &step, &t, &nppnodes,
+ DOMAINDECOMP(cr) ? cr->dd->nc : NULL, &npmenodes,
+ &state->natoms, &state->ngtc, &state->nnhpres,
+ &state->nhchainlength, &(state->dfhist.nlambda), &state->flags, &flags_eks, &flags_enh, &flags_dfh,
+ &state->edsamstate.nED,
+ NULL);
+
+ sfree(version);
+ sfree(btime);
+ sfree(buser);
+ sfree(bhost);
+ sfree(fprog);
+
- ret = do_cpt_state(gmx_fio_getxdr(fp), fflags, state, *bReadRNG, NULL);
++ if ((do_cpt_state(gmx_fio_getxdr(fp), FALSE, state->flags, state, TRUE, NULL) < 0) ||
+ (do_cpt_ekinstate(gmx_fio_getxdr(fp), flags_eks, &state->ekinstate, NULL) < 0) ||
+ (do_cpt_enerhist(gmx_fio_getxdr(fp), FALSE, flags_enh, &state->enerhist, NULL) < 0) ||
+ (do_cpt_df_hist(gmx_fio_getxdr(fp), flags_dfh, &state->dfhist, NULL) < 0) ||
+ (do_cpt_EDstate(gmx_fio_getxdr(fp), FALSE, &state->edsamstate, NULL) < 0) ||
+ (do_cpt_files(gmx_fio_getxdr(fp), FALSE, &outputfiles, &noutputfiles, NULL,
+ file_version) < 0))
+ {
+ gmx_file("Cannot read/write checkpoint; corrupt file, or maybe you are out of disk space?");
+ }
+
+ do_cpt_footer(gmx_fio_getxdr(fp), file_version);
+
+ /* we really, REALLY, want to make sure to physically write the checkpoint,
+ and all the files it depends on, out to disk. Because we've
+ opened the checkpoint with gmx_fio_open(), it's in our list
+ of open files. */
+ ret = gmx_fio_all_output_fsync();
+
+ if (ret)
+ {
+ char buf[STRLEN];
+ sprintf(buf,
+ "Cannot fsync '%s'; maybe you are out of disk space?",
+ gmx_fio_getname(ret));
+
+ if (getenv(GMX_IGNORE_FSYNC_FAILURE_ENV) == NULL)
+ {
+ gmx_file(buf);
+ }
+ else
+ {
+ gmx_warning(buf);
+ }
+ }
+
+ if (gmx_fio_close(fp) != 0)
+ {
+ gmx_file("Cannot read/write checkpoint; corrupt file, or maybe you are out of disk space?");
+ }
+
+ /* we don't move the checkpoint if the user specified they didn't want it,
+ or if the fsyncs failed */
+ if (!bNumberAndKeep && !ret)
+ {
+ if (gmx_fexist(fn))
+ {
+ /* Rename the previous checkpoint file */
+ strcpy(buf, fn);
+ buf[strlen(fn) - strlen(ftp2ext(fn2ftp(fn))) - 1] = '\0';
+ strcat(buf, "_prev");
+ strcat(buf, fn+strlen(fn) - strlen(ftp2ext(fn2ftp(fn))) - 1);
+#ifndef GMX_FAHCORE
+ /* we copy here so that if something goes wrong between now and
+ * the rename below, there's always a state.cpt.
+ * If renames are atomic (such as in POSIX systems),
+ * this copying should be unneccesary.
+ */
+ gmx_file_copy(fn, buf, FALSE);
+ /* We don't really care if this fails:
+ * there's already a new checkpoint.
+ */
+#else
+ gmx_file_rename(fn, buf);
+#endif
+ }
+ if (gmx_file_rename(fntemp, fn) != 0)
+ {
+ gmx_file("Cannot rename checkpoint file; maybe you are out of disk space?");
+ }
+ }
+
+ sfree(outputfiles);
+ sfree(fntemp);
+
+#ifdef GMX_FAHCORE
+ /*code for alternate checkpointing scheme. moved from top of loop over
+ steps */
+ fcRequestCheckPoint();
+ if (fcCheckPointParallel( cr->nodeid, NULL, 0) == 0)
+ {
+ gmx_fatal( 3, __FILE__, __LINE__, "Checkpoint error on step %d\n", step );
+ }
+#endif /* end GMX_FAHCORE block */
+}
+
+static void print_flag_mismatch(FILE *fplog, int sflags, int fflags)
+{
+ int i;
+
+ fprintf(fplog, "\nState entry mismatch between the simulation and the checkpoint file\n");
+ fprintf(fplog, "Entries which are not present in the checkpoint file will not be updated\n");
+ fprintf(fplog, " %24s %11s %11s\n", "", "simulation", "checkpoint");
+ for (i = 0; i < estNR; i++)
+ {
+ if ((sflags & (1<<i)) || (fflags & (1<<i)))
+ {
+ fprintf(fplog, " %24s %11s %11s\n",
+ est_names[i],
+ (sflags & (1<<i)) ? " present " : "not present",
+ (fflags & (1<<i)) ? " present " : "not present");
+ }
+ }
+}
+
+static void check_int(FILE *fplog, const char *type, int p, int f, gmx_bool *mm)
+{
+ FILE *fp = fplog ? fplog : stderr;
+
+ if (p != f)
+ {
+ fprintf(fp, " %s mismatch,\n", type);
+ fprintf(fp, " current program: %d\n", p);
+ fprintf(fp, " checkpoint file: %d\n", f);
+ fprintf(fp, "\n");
+ *mm = TRUE;
+ }
+}
+
+static void check_string(FILE *fplog, const char *type, const char *p,
+ const char *f, gmx_bool *mm)
+{
+ FILE *fp = fplog ? fplog : stderr;
+
+ if (strcmp(p, f) != 0)
+ {
+ fprintf(fp, " %s mismatch,\n", type);
+ fprintf(fp, " current program: %s\n", p);
+ fprintf(fp, " checkpoint file: %s\n", f);
+ fprintf(fp, "\n");
+ *mm = TRUE;
+ }
+}
+
+static void check_match(FILE *fplog,
+ char *version,
+ char *btime, char *buser, char *bhost, int double_prec,
+ char *fprog,
+ t_commrec *cr, gmx_bool bPartDecomp, int npp_f, int npme_f,
+ ivec dd_nc, ivec dd_nc_f)
+{
+ int npp;
+ gmx_bool mm;
+
+ mm = FALSE;
+
+ check_string(fplog, "Version", VERSION, version, &mm);
+ check_string(fplog, "Build time", BUILD_TIME, btime, &mm);
+ check_string(fplog, "Build user", BUILD_USER, buser, &mm);
+ check_string(fplog, "Build host", BUILD_HOST, bhost, &mm);
+ check_int (fplog, "Double prec.", GMX_CPT_BUILD_DP, double_prec, &mm);
+ check_string(fplog, "Program name", Program(), fprog, &mm);
+
+ check_int (fplog, "#nodes", cr->nnodes, npp_f+npme_f, &mm);
+ if (bPartDecomp)
+ {
+ dd_nc[XX] = 1;
+ dd_nc[YY] = 1;
+ dd_nc[ZZ] = 1;
+ }
+ if (cr->nnodes > 1)
+ {
+ check_int (fplog, "#PME-nodes", cr->npmenodes, npme_f, &mm);
+
+ npp = cr->nnodes;
+ if (cr->npmenodes >= 0)
+ {
+ npp -= cr->npmenodes;
+ }
+ if (npp == npp_f)
+ {
+ check_int (fplog, "#DD-cells[x]", dd_nc[XX], dd_nc_f[XX], &mm);
+ check_int (fplog, "#DD-cells[y]", dd_nc[YY], dd_nc_f[YY], &mm);
+ check_int (fplog, "#DD-cells[z]", dd_nc[ZZ], dd_nc_f[ZZ], &mm);
+ }
+ }
+
+ if (mm)
+ {
+ fprintf(stderr,
+ "Gromacs binary or parallel settings not identical to previous run.\n"
+ "Continuation is exact, but is not guaranteed to be binary identical%s.\n\n",
+ fplog ? ",\n see the log file for details" : "");
+
+ if (fplog)
+ {
+ fprintf(fplog,
+ "Gromacs binary or parallel settings not identical to previous run.\n"
+ "Continuation is exact, but is not guaranteed to be binary identical.\n\n");
+ }
+ }
+}
+
+static void read_checkpoint(const char *fn, FILE **pfplog,
+ t_commrec *cr, gmx_bool bPartDecomp, ivec dd_nc,
+ int eIntegrator, int *init_fep_state, gmx_large_int_t *step, double *t,
+ t_state *state, gmx_bool *bReadRNG, gmx_bool *bReadEkin,
+ int *simulation_part,
+ gmx_bool bAppendOutputFiles, gmx_bool bForceAppend)
+{
+ t_fileio *fp;
+ int i, j, rc;
+ int file_version;
+ char *version, *btime, *buser, *bhost, *fprog, *ftime;
+ int double_prec;
+ char filename[STRLEN], buf[STEPSTRSIZE];
+ int nppnodes, eIntegrator_f, nppnodes_f, npmenodes_f;
+ ivec dd_nc_f;
+ int natoms, ngtc, nnhpres, nhchainlength, nlambda, fflags, flags_eks, flags_enh, flags_dfh;
+ int d;
+ int ret;
+ gmx_file_position_t *outputfiles;
+ int nfiles;
+ t_fileio *chksum_file;
+ FILE * fplog = *pfplog;
+ unsigned char digest[16];
+#ifndef GMX_NATIVE_WINDOWS
+ struct flock fl; /* don't initialize here: the struct order is OS
+ dependent! */
+#endif
+
+ const char *int_warn =
+ "WARNING: The checkpoint file was generated with integrator %s,\n"
+ " while the simulation uses integrator %s\n\n";
+ const char *sd_note =
+ "NOTE: The checkpoint file was for %d nodes doing SD or BD,\n"
+ " while the simulation uses %d SD or BD nodes,\n"
+ " continuation will be exact, except for the random state\n\n";
+
+#ifndef GMX_NATIVE_WINDOWS
+ fl.l_type = F_WRLCK;
+ fl.l_whence = SEEK_SET;
+ fl.l_start = 0;
+ fl.l_len = 0;
+ fl.l_pid = 0;
+#endif
+
+ if (PARTDECOMP(cr))
+ {
+ gmx_fatal(FARGS,
+ "read_checkpoint not (yet) supported with particle decomposition");
+ }
+
+ fp = gmx_fio_open(fn, "r");
+ do_cpt_header(gmx_fio_getxdr(fp), TRUE, &file_version,
+ &version, &btime, &buser, &bhost, &double_prec, &fprog, &ftime,
+ &eIntegrator_f, simulation_part, step, t,
+ &nppnodes_f, dd_nc_f, &npmenodes_f,
+ &natoms, &ngtc, &nnhpres, &nhchainlength, &nlambda,
+ &fflags, &flags_eks, &flags_enh, &flags_dfh,
+ &state->edsamstate.nED, NULL);
+
+ if (bAppendOutputFiles &&
+ file_version >= 13 && double_prec != GMX_CPT_BUILD_DP)
+ {
+ gmx_fatal(FARGS, "Output file appending requested, but the code and checkpoint file precision (single/double) don't match");
+ }
+
+ if (cr == NULL || MASTER(cr))
+ {
+ fprintf(stderr, "\nReading checkpoint file %s generated: %s\n\n",
+ fn, ftime);
+ }
+
+ /* This will not be written if we do appending, since fplog is still NULL then */
+ if (fplog)
+ {
+ fprintf(fplog, "\n");
+ fprintf(fplog, "Reading checkpoint file %s\n", fn);
+ fprintf(fplog, " file generated by: %s\n", fprog);
+ fprintf(fplog, " file generated at: %s\n", ftime);
+ fprintf(fplog, " GROMACS build time: %s\n", btime);
+ fprintf(fplog, " GROMACS build user: %s\n", buser);
+ fprintf(fplog, " GROMACS build host: %s\n", bhost);
+ fprintf(fplog, " GROMACS double prec.: %d\n", double_prec);
+ fprintf(fplog, " simulation part #: %d\n", *simulation_part);
+ fprintf(fplog, " step: %s\n", gmx_step_str(*step, buf));
+ fprintf(fplog, " time: %f\n", *t);
+ fprintf(fplog, "\n");
+ }
+
+ if (natoms != state->natoms)
+ {
+ gmx_fatal(FARGS, "Checkpoint file is for a system of %d atoms, while the current system consists of %d atoms", natoms, state->natoms);
+ }
+ if (ngtc != state->ngtc)
+ {
+ gmx_fatal(FARGS, "Checkpoint file is for a system of %d T-coupling groups, while the current system consists of %d T-coupling groups", ngtc, state->ngtc);
+ }
+ if (nnhpres != state->nnhpres)
+ {
+ gmx_fatal(FARGS, "Checkpoint file is for a system of %d NH-pressure-coupling variables, while the current system consists of %d NH-pressure-coupling variables", nnhpres, state->nnhpres);
+ }
+
+ if (nlambda != state->dfhist.nlambda)
+ {
+ gmx_fatal(FARGS, "Checkpoint file is for a system with %d lambda states, while the current system consists of %d lambda states", nlambda, state->dfhist.nlambda);
+ }
+
+ init_gtc_state(state, state->ngtc, state->nnhpres, nhchainlength); /* need to keep this here to keep the tpr format working */
+ /* write over whatever was read; we use the number of Nose-Hoover chains from the checkpoint */
+
+ if (eIntegrator_f != eIntegrator)
+ {
+ if (MASTER(cr))
+ {
+ fprintf(stderr, int_warn, EI(eIntegrator_f), EI(eIntegrator));
+ }
+ if (bAppendOutputFiles)
+ {
+ gmx_fatal(FARGS,
+ "Output file appending requested, but input/checkpoint integrators do not match.\n"
+ "Stopping the run to prevent you from ruining all your data...\n"
+ "If you _really_ know what you are doing, try with the -noappend option.\n");
+ }
+ if (fplog)
+ {
+ fprintf(fplog, int_warn, EI(eIntegrator_f), EI(eIntegrator));
+ }
+ }
+
+ if (!PAR(cr))
+ {
+ nppnodes = 1;
+ cr->npmenodes = 0;
+ }
+ else if (bPartDecomp)
+ {
+ nppnodes = cr->nnodes;
+ cr->npmenodes = 0;
+ }
+ else if (cr->nnodes == nppnodes_f + npmenodes_f)
+ {
+ if (cr->npmenodes < 0)
+ {
+ cr->npmenodes = npmenodes_f;
+ }
+ nppnodes = cr->nnodes - cr->npmenodes;
+ if (nppnodes == nppnodes_f)
+ {
+ for (d = 0; d < DIM; d++)
+ {
+ if (dd_nc[d] == 0)
+ {
+ dd_nc[d] = dd_nc_f[d];
+ }
+ }
+ }
+ }
+ else
+ {
+ /* The number of PP nodes has not been set yet */
+ nppnodes = -1;
+ }
+
+ if ((EI_SD(eIntegrator) || eIntegrator == eiBD) && nppnodes > 0)
+ {
+ /* Correct the RNG state size for the number of PP nodes.
+ * Such assignments should all be moved to one central function.
+ */
+ state->nrng = nppnodes*gmx_rng_n();
+ state->nrngi = nppnodes;
+ }
+
+ *bReadRNG = TRUE;
+ if (fflags != state->flags)
+ {
+
+ if (MASTER(cr))
+ {
+ if (bAppendOutputFiles)
+ {
+ gmx_fatal(FARGS,
+ "Output file appending requested, but input and checkpoint states are not identical.\n"
+ "Stopping the run to prevent you from ruining all your data...\n"
+ "You can try with the -noappend option, and get more info in the log file.\n");
+ }
+
+ if (getenv("GMX_ALLOW_CPT_MISMATCH") == NULL)
+ {
+ gmx_fatal(FARGS, "You seem to have switched ensemble, integrator, T and/or P-coupling algorithm between the cpt and tpr file. The recommended way of doing this is passing the cpt file to grompp (with option -t) instead of to mdrun. If you know what you are doing, you can override this error by setting the env.var. GMX_ALLOW_CPT_MISMATCH");
+ }
+ else
+ {
+ fprintf(stderr,
+ "WARNING: The checkpoint state entries do not match the simulation,\n"
+ " see the log file for details\n\n");
+ }
+ }
+
+ if (fplog)
+ {
+ print_flag_mismatch(fplog, state->flags, fflags);
+ }
+ }
+ else
+ {
+ if ((EI_SD(eIntegrator) || eIntegrator == eiBD) &&
+ nppnodes != nppnodes_f)
+ {
+ *bReadRNG = FALSE;
+ if (MASTER(cr))
+ {
+ fprintf(stderr, sd_note, nppnodes_f, nppnodes);
+ }
+ if (fplog)
+ {
+ fprintf(fplog, sd_note, nppnodes_f, nppnodes);
+ }
+ }
+ if (MASTER(cr))
+ {
+ check_match(fplog, version, btime, buser, bhost, double_prec, fprog,
+ cr, bPartDecomp, nppnodes_f, npmenodes_f, dd_nc, dd_nc_f);
+ }
+ }
- do_cpt_state(gmx_fio_getxdr(fp), state->flags, state, bReadRNG, NULL);
++ ret = do_cpt_state(gmx_fio_getxdr(fp), TRUE, fflags, state, *bReadRNG, NULL);
+ *init_fep_state = state->fep_state; /* there should be a better way to do this than setting it here.
+ Investigate for 5.0. */
+ if (ret)
+ {
+ cp_error();
+ }
+ ret = do_cpt_ekinstate(gmx_fio_getxdr(fp), flags_eks, &state->ekinstate, NULL);
+ if (ret)
+ {
+ cp_error();
+ }
+ *bReadEkin = ((flags_eks & (1<<eeksEKINH)) || (flags_eks & (1<<eeksEKINF)) || (flags_eks & (1<<eeksEKINO)) ||
+ ((flags_eks & (1<<eeksEKINSCALEF)) | (flags_eks & (1<<eeksEKINSCALEH)) | (flags_eks & (1<<eeksVSCALE))));
+
+ ret = do_cpt_enerhist(gmx_fio_getxdr(fp), TRUE,
+ flags_enh, &state->enerhist, NULL);
+ if (ret)
+ {
+ cp_error();
+ }
+
+ ret = do_cpt_EDstate(gmx_fio_getxdr(fp), TRUE, &state->edsamstate, NULL);
+ if (ret)
+ {
+ cp_error();
+ }
+
+ if (file_version < 6)
+ {
+ const char *warn = "Reading checkpoint file in old format, assuming that the run that generated this file started at step 0, if this is not the case the averages stored in the energy file will be incorrect.";
+
+ fprintf(stderr, "\nWARNING: %s\n\n", warn);
+ if (fplog)
+ {
+ fprintf(fplog, "\nWARNING: %s\n\n", warn);
+ }
+ state->enerhist.nsum = *step;
+ state->enerhist.nsum_sim = *step;
+ }
+
+ ret = do_cpt_df_hist(gmx_fio_getxdr(fp), flags_dfh, &state->dfhist, NULL);
+ if (ret)
+ {
+ cp_error();
+ }
+
+ ret = do_cpt_files(gmx_fio_getxdr(fp), TRUE, &outputfiles, &nfiles, NULL, file_version);
+ if (ret)
+ {
+ cp_error();
+ }
+
+ ret = do_cpt_footer(gmx_fio_getxdr(fp), file_version);
+ if (ret)
+ {
+ cp_error();
+ }
+ if (gmx_fio_close(fp) != 0)
+ {
+ gmx_file("Cannot read/write checkpoint; corrupt file, or maybe you are out of disk space?");
+ }
+
+ sfree(fprog);
+ sfree(ftime);
+ sfree(btime);
+ sfree(buser);
+ sfree(bhost);
+
+ /* If the user wants to append to output files,
+ * we use the file pointer positions of the output files stored
+ * in the checkpoint file and truncate the files such that any frames
+ * written after the checkpoint time are removed.
+ * All files are md5sum checked such that we can be sure that
+ * we do not truncate other (maybe imprortant) files.
+ */
+ if (bAppendOutputFiles)
+ {
+ if (fn2ftp(outputfiles[0].filename) != efLOG)
+ {
+ /* make sure first file is log file so that it is OK to use it for
+ * locking
+ */
+ gmx_fatal(FARGS, "The first output file should always be the log "
+ "file but instead is: %s. Cannot do appending because of this condition.", outputfiles[0].filename);
+ }
+ for (i = 0; i < nfiles; i++)
+ {
+ if (outputfiles[i].offset < 0)
+ {
+ gmx_fatal(FARGS, "The original run wrote a file called '%s' which "
+ "is larger than 2 GB, but mdrun did not support large file"
+ " offsets. Can not append. Run mdrun with -noappend",
+ outputfiles[i].filename);
+ }
+#ifdef GMX_FAHCORE
+ chksum_file = gmx_fio_open(outputfiles[i].filename, "a");
+
+#else
+ chksum_file = gmx_fio_open(outputfiles[i].filename, "r+");
+
+ /* lock log file */
+ if (i == 0)
+ {
+ /* Note that there are systems where the lock operation
+ * will succeed, but a second process can also lock the file.
+ * We should probably try to detect this.
+ */
+#ifndef GMX_NATIVE_WINDOWS
+ if (fcntl(fileno(gmx_fio_getfp(chksum_file)), F_SETLK, &fl)
+ == -1)
+#else
+ if (_locking(fileno(gmx_fio_getfp(chksum_file)), _LK_NBLCK, LONG_MAX) == -1)
+#endif
+ {
+ if (errno == ENOSYS)
+ {
+ if (!bForceAppend)
+ {
+ gmx_fatal(FARGS, "File locking is not supported on this system. Use -noappend or specify -append explicitly to append anyhow.");
+ }
+ else
+ {
+ fprintf(stderr, "\nNOTE: File locking is not supported on this system, will not lock %s\n\n", outputfiles[i].filename);
+ if (fplog)
+ {
+ fprintf(fplog, "\nNOTE: File locking not supported on this system, will not lock %s\n\n", outputfiles[i].filename);
+ }
+ }
+ }
+ else if (errno == EACCES || errno == EAGAIN)
+ {
+ gmx_fatal(FARGS, "Failed to lock: %s. Already running "
+ "simulation?", outputfiles[i].filename);
+ }
+ else
+ {
+ gmx_fatal(FARGS, "Failed to lock: %s. %s.",
+ outputfiles[i].filename, strerror(errno));
+ }
+ }
+ }
+
+ /* compute md5 chksum */
+ if (outputfiles[i].chksum_size != -1)
+ {
+ if (gmx_fio_get_file_md5(chksum_file, outputfiles[i].offset,
+ digest) != outputfiles[i].chksum_size) /*at the end of the call the file position is at the end of the file*/
+ {
+ gmx_fatal(FARGS, "Can't read %d bytes of '%s' to compute checksum. The file has been replaced or its contents have been modified. Cannot do appending because of this condition.",
+ outputfiles[i].chksum_size,
+ outputfiles[i].filename);
+ }
+ }
+ if (i == 0) /*log file needs to be seeked in case we need to truncate (other files are truncated below)*/
+ {
+ if (gmx_fio_seek(chksum_file, outputfiles[i].offset))
+ {
+ gmx_fatal(FARGS, "Seek error! Failed to truncate log-file: %s.", strerror(errno));
+ }
+ }
+#endif
+
+ if (i == 0) /*open log file here - so that lock is never lifted
+ after chksum is calculated */
+ {
+ *pfplog = gmx_fio_getfp(chksum_file);
+ }
+ else
+ {
+ gmx_fio_close(chksum_file);
+ }
+#ifndef GMX_FAHCORE
+ /* compare md5 chksum */
+ if (outputfiles[i].chksum_size != -1 &&
+ memcmp(digest, outputfiles[i].chksum, 16) != 0)
+ {
+ if (debug)
+ {
+ fprintf(debug, "chksum for %s: ", outputfiles[i].filename);
+ for (j = 0; j < 16; j++)
+ {
+ fprintf(debug, "%02x", digest[j]);
+ }
+ fprintf(debug, "\n");
+ }
+ gmx_fatal(FARGS, "Checksum wrong for '%s'. The file has been replaced or its contents have been modified. Cannot do appending because of this condition.",
+ outputfiles[i].filename);
+ }
+#endif
+
+
+ if (i != 0) /*log file is already seeked to correct position */
+ {
+#ifdef GMX_NATIVE_WINDOWS
+ rc = gmx_wintruncate(outputfiles[i].filename, outputfiles[i].offset);
+#else
+ rc = truncate(outputfiles[i].filename, outputfiles[i].offset);
+#endif
+ if (rc != 0)
+ {
+ gmx_fatal(FARGS, "Truncation of file %s failed. Cannot do appending because of this failure.", outputfiles[i].filename);
+ }
+ }
+ }
+ }
+
+ sfree(outputfiles);
+}
+
+
+void load_checkpoint(const char *fn, FILE **fplog,
+ t_commrec *cr, gmx_bool bPartDecomp, ivec dd_nc,
+ t_inputrec *ir, t_state *state,
+ gmx_bool *bReadRNG, gmx_bool *bReadEkin,
+ gmx_bool bAppend, gmx_bool bForceAppend)
+{
+ gmx_large_int_t step;
+ double t;
+
+ if (SIMMASTER(cr))
+ {
+ /* Read the state from the checkpoint file */
+ read_checkpoint(fn, fplog,
+ cr, bPartDecomp, dd_nc,
+ ir->eI, &(ir->fepvals->init_fep_state), &step, &t, state, bReadRNG, bReadEkin,
+ &ir->simulation_part, bAppend, bForceAppend);
+ }
+ if (PAR(cr))
+ {
+ gmx_bcast(sizeof(cr->npmenodes), &cr->npmenodes, cr);
+ gmx_bcast(DIM*sizeof(dd_nc[0]), dd_nc, cr);
+ gmx_bcast(sizeof(step), &step, cr);
+ gmx_bcast(sizeof(*bReadRNG), bReadRNG, cr);
+ gmx_bcast(sizeof(*bReadEkin), bReadEkin, cr);
+ }
+ ir->bContinuation = TRUE;
+ if (ir->nsteps >= 0)
+ {
+ ir->nsteps += ir->init_step - step;
+ }
+ ir->init_step = step;
+ ir->simulation_part += 1;
+}
+
+static void read_checkpoint_data(t_fileio *fp, int *simulation_part,
+ gmx_large_int_t *step, double *t, t_state *state,
+ gmx_bool bReadRNG,
+ int *nfiles, gmx_file_position_t **outputfiles)
+{
+ int file_version;
+ char *version, *btime, *buser, *bhost, *fprog, *ftime;
+ int double_prec;
+ int eIntegrator;
+ int nppnodes, npme;
+ ivec dd_nc;
+ int flags_eks, flags_enh, flags_dfh;
+ int nfiles_loc;
+ gmx_file_position_t *files_loc = NULL;
+ int ret;
+
+ do_cpt_header(gmx_fio_getxdr(fp), TRUE, &file_version,
+ &version, &btime, &buser, &bhost, &double_prec, &fprog, &ftime,
+ &eIntegrator, simulation_part, step, t, &nppnodes, dd_nc, &npme,
+ &state->natoms, &state->ngtc, &state->nnhpres, &state->nhchainlength,
+ &(state->dfhist.nlambda), &state->flags, &flags_eks, &flags_enh, &flags_dfh,
+ &state->edsamstate.nED, NULL);
+ ret =
- ret = do_cpt_state(gmx_fio_getxdr(fp), state.flags, &state, TRUE, out);
++ do_cpt_state(gmx_fio_getxdr(fp), TRUE, state->flags, state, bReadRNG, NULL);
+ if (ret)
+ {
+ cp_error();
+ }
+ ret = do_cpt_ekinstate(gmx_fio_getxdr(fp), flags_eks, &state->ekinstate, NULL);
+ if (ret)
+ {
+ cp_error();
+ }
+ ret = do_cpt_enerhist(gmx_fio_getxdr(fp), TRUE,
+ flags_enh, &state->enerhist, NULL);
+ if (ret)
+ {
+ cp_error();
+ }
+ ret = do_cpt_df_hist(gmx_fio_getxdr(fp), flags_dfh, &state->dfhist, NULL);
+ if (ret)
+ {
+ cp_error();
+ }
+
+ ret = do_cpt_EDstate(gmx_fio_getxdr(fp), TRUE, &state->edsamstate, NULL);
+ if (ret)
+ {
+ cp_error();
+ }
+
+ ret = do_cpt_files(gmx_fio_getxdr(fp), TRUE,
+ outputfiles != NULL ? outputfiles : &files_loc,
+ outputfiles != NULL ? nfiles : &nfiles_loc,
+ NULL, file_version);
+ if (files_loc != NULL)
+ {
+ sfree(files_loc);
+ }
+
+ if (ret)
+ {
+ cp_error();
+ }
+
+ ret = do_cpt_footer(gmx_fio_getxdr(fp), file_version);
+ if (ret)
+ {
+ cp_error();
+ }
+
+ sfree(fprog);
+ sfree(ftime);
+ sfree(btime);
+ sfree(buser);
+ sfree(bhost);
+}
+
+void
+read_checkpoint_state(const char *fn, int *simulation_part,
+ gmx_large_int_t *step, double *t, t_state *state)
+{
+ t_fileio *fp;
+
+ fp = gmx_fio_open(fn, "r");
+ read_checkpoint_data(fp, simulation_part, step, t, state, FALSE, NULL, NULL);
+ if (gmx_fio_close(fp) != 0)
+ {
+ gmx_file("Cannot read/write checkpoint; corrupt file, or maybe you are out of disk space?");
+ }
+}
+
+void read_checkpoint_trxframe(t_fileio *fp, t_trxframe *fr)
+{
+ /* This next line is nasty because the sub-structures of t_state
+ * cannot be assumed to be zeroed (or even initialized in ways the
+ * rest of the code might assume). Using snew would be better, but
+ * this will all go away for 5.0. */
+ t_state state;
+ int simulation_part;
+ gmx_large_int_t step;
+ double t;
+
+ init_state(&state, 0, 0, 0, 0, 0);
+
+ read_checkpoint_data(fp, &simulation_part, &step, &t, &state, FALSE, NULL, NULL);
+
+ fr->natoms = state.natoms;
+ fr->bTitle = FALSE;
+ fr->bStep = TRUE;
+ fr->step = gmx_large_int_to_int(step,
+ "conversion of checkpoint to trajectory");
+ fr->bTime = TRUE;
+ fr->time = t;
+ fr->bLambda = TRUE;
+ fr->lambda = state.lambda[efptFEP];
+ fr->fep_state = state.fep_state;
+ fr->bAtoms = FALSE;
+ fr->bX = (state.flags & (1<<estX));
+ if (fr->bX)
+ {
+ fr->x = state.x;
+ state.x = NULL;
+ }
+ fr->bV = (state.flags & (1<<estV));
+ if (fr->bV)
+ {
+ fr->v = state.v;
+ state.v = NULL;
+ }
+ fr->bF = FALSE;
+ fr->bBox = (state.flags & (1<<estBOX));
+ if (fr->bBox)
+ {
+ copy_mat(state.box, fr->box);
+ }
+ done_state(&state);
+}
+
+void list_checkpoint(const char *fn, FILE *out)
+{
+ t_fileio *fp;
+ int file_version;
+ char *version, *btime, *buser, *bhost, *fprog, *ftime;
+ int double_prec;
+ int eIntegrator, simulation_part, nppnodes, npme;
+ gmx_large_int_t step;
+ double t;
+ ivec dd_nc;
+ t_state state;
+ int flags_eks, flags_enh, flags_dfh;
+ int indent;
+ int i, j;
+ int ret;
+ gmx_file_position_t *outputfiles;
+ int nfiles;
+
+ init_state(&state, -1, -1, -1, -1, 0);
+
+ fp = gmx_fio_open(fn, "r");
+ do_cpt_header(gmx_fio_getxdr(fp), TRUE, &file_version,
+ &version, &btime, &buser, &bhost, &double_prec, &fprog, &ftime,
+ &eIntegrator, &simulation_part, &step, &t, &nppnodes, dd_nc, &npme,
+ &state.natoms, &state.ngtc, &state.nnhpres, &state.nhchainlength,
+ &(state.dfhist.nlambda), &state.flags,
+ &flags_eks, &flags_enh, &flags_dfh, &state.edsamstate.nED, out);
- init_df_history(&state.dfhist, state.dfhist.nlambda, 0); /* reinitialize state with correct sizes */
- ret = do_cpt_df_hist(gmx_fio_getxdr(fp), flags_dfh, &state.dfhist, out);
++ ret = do_cpt_state(gmx_fio_getxdr(fp), TRUE, state.flags, &state, TRUE, out);
+ if (ret)
+ {
+ cp_error();
+ }
+ ret = do_cpt_ekinstate(gmx_fio_getxdr(fp), flags_eks, &state.ekinstate, out);
+ if (ret)
+ {
+ cp_error();
+ }
+ ret = do_cpt_enerhist(gmx_fio_getxdr(fp), TRUE,
+ flags_enh, &state.enerhist, out);
+
+ if (ret == 0)
+ {
++ ret = do_cpt_df_hist(gmx_fio_getxdr(fp),
++ flags_dfh, &state.dfhist, out);
+ }
+
+ if (ret == 0)
+ {
+ ret = do_cpt_EDstate(gmx_fio_getxdr(fp), TRUE, &state.edsamstate, out);
+ }
+
+ if (ret == 0)
+ {
+ do_cpt_files(gmx_fio_getxdr(fp), TRUE, &outputfiles, &nfiles, out, file_version);
+ }
+
+ if (ret == 0)
+ {
+ ret = do_cpt_footer(gmx_fio_getxdr(fp), file_version);
+ }
+
+ if (ret)
+ {
+ cp_warning(out);
+ }
+ if (gmx_fio_close(fp) != 0)
+ {
+ gmx_file("Cannot read/write checkpoint; corrupt file, or maybe you are out of disk space?");
+ }
+
+ done_state(&state);
+}
+
+
+static gmx_bool exist_output_file(const char *fnm_cp, int nfile, const t_filenm fnm[])
+{
+ int i;
+
+ /* Check if the output file name stored in the checkpoint file
+ * is one of the output file names of mdrun.
+ */
+ i = 0;
+ while (i < nfile &&
+ !(is_output(&fnm[i]) && strcmp(fnm_cp, fnm[i].fns[0]) == 0))
+ {
+ i++;
+ }
+
+ return (i < nfile && gmx_fexist(fnm_cp));
+}
+
+/* This routine cannot print tons of data, since it is called before the log file is opened. */
+gmx_bool read_checkpoint_simulation_part(const char *filename, int *simulation_part,
+ gmx_large_int_t *cpt_step, t_commrec *cr,
+ gmx_bool bAppendReq,
+ int nfile, const t_filenm fnm[],
+ const char *part_suffix, gmx_bool *bAddPart)
+{
+ t_fileio *fp;
+ gmx_large_int_t step = 0;
+ double t;
+ /* This next line is nasty because the sub-structures of t_state
+ * cannot be assumed to be zeroed (or even initialized in ways the
+ * rest of the code might assume). Using snew would be better, but
+ * this will all go away for 5.0. */
+ t_state state;
+ int nfiles;
+ gmx_file_position_t *outputfiles;
+ int nexist, f;
+ gmx_bool bAppend;
+ char *fn, suf_up[STRLEN];
+
+ bAppend = FALSE;
+
+ if (SIMMASTER(cr))
+ {
+ if (!gmx_fexist(filename) || (!(fp = gmx_fio_open(filename, "r")) ))
+ {
+ *simulation_part = 0;
+ }
+ else
+ {
+ init_state(&state, 0, 0, 0, 0, 0);
+
+ read_checkpoint_data(fp, simulation_part, &step, &t, &state, FALSE,
+ &nfiles, &outputfiles);
+ if (gmx_fio_close(fp) != 0)
+ {
+ gmx_file("Cannot read/write checkpoint; corrupt file, or maybe you are out of disk space?");
+ }
+ done_state(&state);
+
+ if (bAppendReq)
+ {
+ nexist = 0;
+ for (f = 0; f < nfiles; f++)
+ {
+ if (exist_output_file(outputfiles[f].filename, nfile, fnm))
+ {
+ nexist++;
+ }
+ }
+ if (nexist == nfiles)
+ {
+ bAppend = bAppendReq;
+ }
+ else if (nexist > 0)
+ {
+ fprintf(stderr,
+ "Output file appending has been requested,\n"
+ "but some output files listed in the checkpoint file %s\n"
+ "are not present or are named differently by the current program:\n",
+ filename);
+ fprintf(stderr, "output files present:");
+ for (f = 0; f < nfiles; f++)
+ {
+ if (exist_output_file(outputfiles[f].filename,
+ nfile, fnm))
+ {
+ fprintf(stderr, " %s", outputfiles[f].filename);
+ }
+ }
+ fprintf(stderr, "\n");
+ fprintf(stderr, "output files not present or named differently:");
+ for (f = 0; f < nfiles; f++)
+ {
+ if (!exist_output_file(outputfiles[f].filename,
+ nfile, fnm))
+ {
+ fprintf(stderr, " %s", outputfiles[f].filename);
+ }
+ }
+ fprintf(stderr, "\n");
+
+ gmx_fatal(FARGS, "File appending requested, but only %d of the %d output files are present", nexist, nfiles);
+ }
+ }
+
+ if (bAppend)
+ {
+ if (nfiles == 0)
+ {
+ gmx_fatal(FARGS, "File appending requested, but no output file information is stored in the checkpoint file");
+ }
+ fn = outputfiles[0].filename;
+ if (strlen(fn) < 4 ||
+ gmx_strcasecmp(fn+strlen(fn)-4, ftp2ext(efLOG)) == 0)
+ {
+ gmx_fatal(FARGS, "File appending requested, but the log file is not the first file listed in the checkpoint file");
+ }
+ /* Set bAddPart to whether the suffix string '.part' is present
+ * in the log file name.
+ */
+ strcpy(suf_up, part_suffix);
+ upstring(suf_up);
+ *bAddPart = (strstr(fn, part_suffix) != NULL ||
+ strstr(fn, suf_up) != NULL);
+ }
+
+ sfree(outputfiles);
+ }
+ }
+ if (PAR(cr))
+ {
+ gmx_bcast(sizeof(*simulation_part), simulation_part, cr);
+
+ if (*simulation_part > 0 && bAppendReq)
+ {
+ gmx_bcast(sizeof(bAppend), &bAppend, cr);
+ gmx_bcast(sizeof(*bAddPart), bAddPart, cr);
+ }
+ }
+ if (NULL != cpt_step)
+ {
+ *cpt_step = step;
+ }
+
+ return bAppend;
+}
--- /dev/null
- #if defined (__i386__) || defined (__x86_64__) || defined (_M_IX86) || defined (_M_X64)
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This file is part of GROMACS.
+ * Copyright (c) 2012-
+ *
+ * Written by the Gromacs development team under coordination of
+ * David van der Spoel, Berk Hess, and Erik Lindahl.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the research papers on the package. Check out http://www.gromacs.org
+ *
+ * And Hey:
+ * Gnomes, ROck Monsters And Chili Sauce
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#ifdef HAVE_SCHED_H
+#define _GNU_SOURCE
+#include <sched.h>
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+#ifdef _MSC_VER
+/* MSVC definition for __cpuid() */
+#include <intrin.h>
+/* sysinfo functions */
+#include <windows.h>
+#endif
+#ifdef HAVE_UNISTD_H
+/* sysconf() definition */
+#include <unistd.h>
+#endif
+
+#include "gmx_cpuid.h"
+
+
+
+/* For convenience, and to enable configure-time invocation, we keep all architectures
+ * in a single file, but to avoid repeated ifdefs we set the overall architecture here.
+ */
++#ifdef GMX_IS_X86
+/* OK, it is x86, but can we execute cpuid? */
+#if defined(GMX_X86_GCC_INLINE_ASM) || ( defined(_MSC_VER) && ( (_MSC_VER > 1500) || (_MSC_VER==1500 & _MSC_FULL_VER >= 150030729)))
+# define GMX_CPUID_X86
+#endif
+#endif
+
+/* Global constant character strings corresponding to our enumerated types */
+const char *
+gmx_cpuid_vendor_string[GMX_CPUID_NVENDORS] =
+{
+ "CannotDetect",
+ "Unknown",
+ "GenuineIntel",
+ "AuthenticAMD",
+ "Fujitsu",
+ "IBM"
+};
+
+const char *
+gmx_cpuid_vendor_string_alternative[GMX_CPUID_NVENDORS] =
+{
+ "CannotDetect",
+ "Unknown",
+ "GenuineIntel",
+ "AuthenticAMD",
+ "Fujitsu",
+ "ibm" /* Used on BlueGene/Q */
+};
+
+const char *
+gmx_cpuid_feature_string[GMX_CPUID_NFEATURES] =
+{
+ "CannotDetect",
+ "aes",
+ "apic",
+ "avx",
+ "avx2",
+ "clfsh",
+ "cmov",
+ "cx8",
+ "cx16",
+ "f16c",
+ "fma",
+ "fma4",
+ "htt",
+ "lahf_lm",
+ "misalignsse",
+ "mmx",
+ "msr",
+ "nonstop_tsc",
+ "pcid",
+ "pclmuldq",
+ "pdcm",
+ "pdpe1gb",
+ "popcnt",
+ "pse",
+ "rdrnd",
+ "rdtscp",
+ "sse2",
+ "sse3",
+ "sse4a",
+ "sse4.1",
+ "sse4.2",
+ "ssse3",
+ "tdt",
+ "x2apic",
+ "xop"
+};
+
+const char *
+gmx_cpuid_acceleration_string[GMX_CPUID_NACCELERATIONS] =
+{
+ "CannotDetect",
+ "None",
+ "SSE2",
+ "SSE4.1",
+ "AVX_128_FMA",
+ "AVX_256",
+ "Sparc64 HPC-ACE",
+ "IBM_QPX"
+};
+
+/* Max length of brand string */
+#define GMX_CPUID_BRAND_MAXLEN 256
+
+
+/* Contents of the abstract datatype */
+struct gmx_cpuid
+{
+ enum gmx_cpuid_vendor vendor;
+ char brand[GMX_CPUID_BRAND_MAXLEN];
+ int family;
+ int model;
+ int stepping;
+ /* Not using gmx_bool here, since this file must be possible to compile without simple.h */
+ char feature[GMX_CPUID_NFEATURES];
+
+ /* Basic CPU topology information. For x86 this is a bit complicated since the topology differs between
+ * operating systems and sometimes even settings. For most other architectures you can likely just check
+ * the documentation and then write static information to these arrays rather than detecting on-the-fly.
+ */
+ int have_cpu_topology;
+ int nproc; /* total number of logical processors from OS */
+ int npackages;
+ int ncores_per_package;
+ int nhwthreads_per_core;
+ int * package_id;
+ int * core_id; /* Local core id in each package */
+ int * hwthread_id; /* Local hwthread id in each core */
+ int * locality_order; /* Processor indices sorted in locality order */
+};
+
+
+/* Simple routines to access the data structure. The initialization routine is
+ * further down since that needs to call other static routines in this file.
+ */
+enum gmx_cpuid_vendor
+gmx_cpuid_vendor (gmx_cpuid_t cpuid)
+{
+ return cpuid->vendor;
+}
+
+
+const char *
+gmx_cpuid_brand (gmx_cpuid_t cpuid)
+{
+ return cpuid->brand;
+}
+
+int
+gmx_cpuid_family (gmx_cpuid_t cpuid)
+{
+ return cpuid->family;
+}
+
+int
+gmx_cpuid_model (gmx_cpuid_t cpuid)
+{
+ return cpuid->model;
+}
+
+int
+gmx_cpuid_stepping (gmx_cpuid_t cpuid)
+{
+ return cpuid->stepping;
+}
+
+int
+gmx_cpuid_feature (gmx_cpuid_t cpuid,
+ enum gmx_cpuid_feature feature)
+{
+ return (cpuid->feature[feature] != 0);
+}
+
+
+
+
+/* What type of acceleration was compiled in, if any?
+ * This is set from Cmake. Note that the SSE2 and SSE4_1 macros are set for
+ * AVX too, so it is important that they appear last in the list.
+ */
+#ifdef GMX_X86_AVX_256
+static const
+enum gmx_cpuid_acceleration
+ compiled_acc = GMX_CPUID_ACCELERATION_X86_AVX_256;
+#elif defined GMX_X86_AVX_128_FMA
+static const
+enum gmx_cpuid_acceleration
+ compiled_acc = GMX_CPUID_ACCELERATION_X86_AVX_128_FMA;
+#elif defined GMX_X86_SSE4_1
+static const
+enum gmx_cpuid_acceleration
+ compiled_acc = GMX_CPUID_ACCELERATION_X86_SSE4_1;
+#elif defined GMX_X86_SSE2
+static const
+enum gmx_cpuid_acceleration
+ compiled_acc = GMX_CPUID_ACCELERATION_X86_SSE2;
+#elif defined GMX_CPU_ACCELERATION_SPARC64_HPC_ACE
+static const
+enum gmx_cpuid_acceleration
+ compiled_acc = GMX_CPUID_ACCELERATION_SPARC64_HPC_ACE;
+#elif defined GMX_CPU_ACCELERATION_IBM_QPX
+static const
+enum gmx_cpuid_acceleration
+ compiled_acc = GMX_CPUID_ACCELERATION_IBM_QPX;
+#else
+static const
+enum gmx_cpuid_acceleration
+ compiled_acc = GMX_CPUID_ACCELERATION_NONE;
+#endif
+
+
+#ifdef GMX_CPUID_X86
+
+/* Execute CPUID on x86 class CPUs. level sets function to exec, and the
+ * contents of register output is returned. See Intel/AMD docs for details.
+ *
+ * This version supports extended information where we can also have an input
+ * value in the ecx register. This is ignored for most levels, but some of them
+ * (e.g. level 0xB on Intel) use it.
+ */
+static int
+execute_x86cpuid(unsigned int level,
+ unsigned int ecxval,
+ unsigned int * eax,
+ unsigned int * ebx,
+ unsigned int * ecx,
+ unsigned int * edx)
+{
+ int rc = 0;
+
+ /* Currently CPUID is only supported (1) if we can use an instruction on MSVC, or (2)
+ * if the compiler handles GNU-style inline assembly.
+ */
+
+#if (defined _MSC_VER)
+ int CPUInfo[4];
+
+#if (_MSC_VER > 1500) || (_MSC_VER == 1500 & _MSC_FULL_VER >= 150030729)
+ /* MSVC 9.0 SP1 or later */
+ __cpuidex(CPUInfo, level, ecxval);
+ rc = 0;
+#else
+ __cpuid(CPUInfo, level);
+ /* Set an error code if the user wanted a non-zero ecxval, since we did not have cpuidex */
+ rc = (ecxval > 0) ? -1 : 0;
+#endif
+ *eax = CPUInfo[0];
+ *ebx = CPUInfo[1];
+ *ecx = CPUInfo[2];
+ *edx = CPUInfo[3];
+
+#elif (defined GMX_X86_GCC_INLINE_ASM)
+ /* for now this means GMX_X86_GCC_INLINE_ASM should be defined,
+ * but there might be more options added in the future.
+ */
+ *eax = level;
+ *ecx = ecxval;
+ *ebx = 0;
+ *edx = 0;
+#if defined(__i386__) && defined(__PIC__)
+ /* Avoid clobbering the global offset table in 32-bit pic code (ebx register) */
+ __asm__ __volatile__ ("xchgl %%ebx, %1 \n\t"
+ "cpuid \n\t"
+ "xchgl %%ebx, %1 \n\t"
+ : "+a" (*eax), "+r" (*ebx), "+c" (*ecx), "+d" (*edx));
+#else
+ /* i386 without PIC, or x86-64. Things are easy and we can clobber any reg we want :-) */
+ __asm__ __volatile__ ("cpuid \n\t"
+ : "+a" (*eax), "+b" (*ebx), "+c" (*ecx), "+d" (*edx));
+#endif
+ rc = 0;
+#else
+ /* Death and horror!
+ * Apparently this is an x86 platform where we don't know how to call cpuid.
+ *
+ * This is REALLY bad, since we will lose all Gromacs acceleration.
+ */
+ *eax = 0;
+ *ebx = 0;
+ *ecx = 0;
+ *edx = 0;
+
+ rc = -1;
+#endif
+ return rc;
+}
+
+
+/* Identify CPU features common to Intel & AMD - mainly brand string,
+ * version and some features. Vendor has already been detected outside this.
+ */
+static int
+cpuid_check_common_x86(gmx_cpuid_t cpuid)
+{
+ int fn, max_stdfn, max_extfn;
+ unsigned int eax, ebx, ecx, edx;
+ char str[GMX_CPUID_BRAND_MAXLEN];
+ char * p;
+
+ /* Find largest standard/extended function input value */
+ execute_x86cpuid(0x0, 0, &eax, &ebx, &ecx, &edx);
+ max_stdfn = eax;
+ execute_x86cpuid(0x80000000, 0, &eax, &ebx, &ecx, &edx);
+ max_extfn = eax;
+
+ p = str;
+ if (max_extfn >= 0x80000005)
+ {
+ /* Get CPU brand string */
+ for (fn = 0x80000002; fn < 0x80000005; fn++)
+ {
+ execute_x86cpuid(fn, 0, &eax, &ebx, &ecx, &edx);
+ memcpy(p, &eax, 4);
+ memcpy(p+4, &ebx, 4);
+ memcpy(p+8, &ecx, 4);
+ memcpy(p+12, &edx, 4);
+ p += 16;
+ }
+ *p = '\0';
+
+ /* Remove empty initial space */
+ p = str;
+ while (isspace(*(p)))
+ {
+ p++;
+ }
+ strncpy(cpuid->brand, p, GMX_CPUID_BRAND_MAXLEN);
+ }
+ else
+ {
+ strncpy(cpuid->brand, "Unknown CPU brand", GMX_CPUID_BRAND_MAXLEN);
+ }
+
+ /* Find basic CPU properties */
+ if (max_stdfn >= 1)
+ {
+ execute_x86cpuid(0x1, 0, &eax, &ebx, &ecx, &edx);
+
+ cpuid->family = ((eax & 0x0FF00000) >> 20) + ((eax & 0x00000F00) >> 8);
+ /* Note that extended model should be shifted left 4, so only shift right 12 iso 16. */
+ cpuid->model = ((eax & 0x000F0000) >> 12) + ((eax & 0x000000F0) >> 4);
+ cpuid->stepping = (eax & 0x0000000F);
+
+ /* Feature flags common to AMD and intel */
+ cpuid->feature[GMX_CPUID_FEATURE_X86_SSE3] = (ecx & (1 << 0)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_PCLMULDQ] = (ecx & (1 << 1)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_SSSE3] = (ecx & (1 << 9)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_FMA] = (ecx & (1 << 12)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_CX16] = (ecx & (1 << 13)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_SSE4_1] = (ecx & (1 << 19)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_SSE4_2] = (ecx & (1 << 20)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_POPCNT] = (ecx & (1 << 23)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_AES] = (ecx & (1 << 25)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_AVX] = (ecx & (1 << 28)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_F16C] = (ecx & (1 << 29)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_RDRND] = (ecx & (1 << 30)) != 0;
+
+ cpuid->feature[GMX_CPUID_FEATURE_X86_PSE] = (edx & (1 << 3)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_MSR] = (edx & (1 << 5)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_CX8] = (edx & (1 << 8)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_APIC] = (edx & (1 << 9)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_CMOV] = (edx & (1 << 15)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_CLFSH] = (edx & (1 << 19)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_MMX] = (edx & (1 << 23)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_SSE2] = (edx & (1 << 26)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_HTT] = (edx & (1 << 28)) != 0;
+ }
+ else
+ {
+ cpuid->family = -1;
+ cpuid->model = -1;
+ cpuid->stepping = -1;
+ }
+
+ if (max_extfn >= 0x80000001)
+ {
+ execute_x86cpuid(0x80000001, 0, &eax, &ebx, &ecx, &edx);
+ cpuid->feature[GMX_CPUID_FEATURE_X86_LAHF_LM] = (ecx & (1 << 0)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_PDPE1GB] = (edx & (1 << 26)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_RDTSCP] = (edx & (1 << 27)) != 0;
+ }
+
+ if (max_extfn >= 0x80000007)
+ {
+ execute_x86cpuid(0x80000007, 0, &eax, &ebx, &ecx, &edx);
+ cpuid->feature[GMX_CPUID_FEATURE_X86_NONSTOP_TSC] = (edx & (1 << 8)) != 0;
+ }
+ return 0;
+}
+
+/* This routine returns the number of unique different elements found in the array,
+ * and renumbers these starting from 0. For example, the array {0,1,2,8,9,10,8,9,10,0,1,2}
+ * will be rewritten to {0,1,2,3,4,5,3,4,5,0,1,2}, and it returns 6 for the
+ * number of unique elements.
+ */
+static int
+cpuid_renumber_elements(int *data, int n)
+{
+ int *unique;
+ int i, j, nunique, found;
+
+ unique = malloc(sizeof(int)*n);
+
+ nunique = 0;
+ for (i = 0; i < n; i++)
+ {
+ for (j = 0, found = 0; j < nunique && !found; j++)
+ {
+ found = (data[i] == unique[j]);
+ }
+ if (!found)
+ {
+ /* Insert in sorted order! */
+ for (j = nunique++; j > 0 && unique[j-1] > data[i]; j--)
+ {
+ unique[j] = unique[j-1];
+ }
+ unique[j] = data[i];
+ }
+ }
+ /* renumber */
+ for (i = 0; i < n; i++)
+ {
+ for (j = 0; j < nunique; j++)
+ {
+ if (data[i] == unique[j])
+ {
+ data[i] = j;
+ }
+ }
+ }
+ return nunique;
+}
+
+/* APIC IDs, or everything you wanted to know about your x86 cores but were afraid to ask...
+ *
+ * Raw APIC IDs are unfortunately somewhat dirty. For technical reasons they are assigned
+ * in power-of-2 chunks, and even then there are no guarantees about specific numbers - all
+ * we know is that the part for each thread/core/package is unique, and how many bits are
+ * reserved for that part.
+ * This routine does internal renumbering so we get continuous indices, and also
+ * decodes the actual number of packages,cores-per-package and hwthreads-per-core.
+ * Returns: 0 on success, non-zero on failure.
+ */
+static int
+cpuid_x86_decode_apic_id(gmx_cpuid_t cpuid, int *apic_id, int core_bits, int hwthread_bits)
+{
+ int i, idx;
+ int hwthread_mask, core_mask_after_shift;
+
+ cpuid->hwthread_id = malloc(sizeof(int)*cpuid->nproc);
+ cpuid->core_id = malloc(sizeof(int)*cpuid->nproc);
+ cpuid->package_id = malloc(sizeof(int)*cpuid->nproc);
+ cpuid->locality_order = malloc(sizeof(int)*cpuid->nproc);
+
+ hwthread_mask = (1 << hwthread_bits) - 1;
+ core_mask_after_shift = (1 << core_bits) - 1;
+
+ for (i = 0; i < cpuid->nproc; i++)
+ {
+ cpuid->hwthread_id[i] = apic_id[i] & hwthread_mask;
+ cpuid->core_id[i] = (apic_id[i] >> hwthread_bits) & core_mask_after_shift;
+ cpuid->package_id[i] = apic_id[i] >> (core_bits + hwthread_bits);
+ }
+
+ cpuid->npackages = cpuid_renumber_elements(cpuid->package_id, cpuid->nproc);
+ cpuid->ncores_per_package = cpuid_renumber_elements(cpuid->core_id, cpuid->nproc);
+ cpuid->nhwthreads_per_core = cpuid_renumber_elements(cpuid->hwthread_id, cpuid->nproc);
+
+ /* now check for consistency */
+ if ( (cpuid->npackages * cpuid->ncores_per_package *
+ cpuid->nhwthreads_per_core) != cpuid->nproc )
+ {
+ /* the packages/cores-per-package/hwthreads-per-core counts are
+ inconsistent. */
+ return -1;
+ }
+
+ /* Create a locality order array, i.e. first all resources in package0, which in turn
+ * are sorted so we first have all resources in core0, where threads are sorted in order, etc.
+ */
+
+ for (i = 0; i < cpuid->nproc; i++)
+ {
+ idx = (cpuid->package_id[i]*cpuid->ncores_per_package + cpuid->core_id[i])*cpuid->nhwthreads_per_core + cpuid->hwthread_id[i];
+ cpuid->locality_order[idx] = i;
+ }
+ return 0;
+}
+
+
+/* Detection of AMD-specific CPU features */
+static int
+cpuid_check_amd_x86(gmx_cpuid_t cpuid)
+{
+ int max_stdfn, max_extfn, ret;
+ unsigned int eax, ebx, ecx, edx;
+ int hwthread_bits, core_bits;
+ int * apic_id;
+
+ cpuid_check_common_x86(cpuid);
+
+ execute_x86cpuid(0x0, 0, &eax, &ebx, &ecx, &edx);
+ max_stdfn = eax;
+
+ execute_x86cpuid(0x80000000, 0, &eax, &ebx, &ecx, &edx);
+ max_extfn = eax;
+
+ if (max_extfn >= 0x80000001)
+ {
+ execute_x86cpuid(0x80000001, 0, &eax, &ebx, &ecx, &edx);
+
+ cpuid->feature[GMX_CPUID_FEATURE_X86_SSE4A] = (ecx & (1 << 6)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_MISALIGNSSE] = (ecx & (1 << 7)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_XOP] = (ecx & (1 << 11)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_FMA4] = (ecx & (1 << 16)) != 0;
+ }
+
+ /* Query APIC information on AMD */
+ if (max_extfn >= 0x80000008)
+ {
+#if (defined HAVE_SCHED_H && defined HAVE_SCHED_SETAFFINITY && defined HAVE_SYSCONF && defined __linux__)
+ /* Linux */
+ unsigned int i;
+ cpu_set_t cpuset, save_cpuset;
+ cpuid->nproc = sysconf(_SC_NPROCESSORS_ONLN);
+ apic_id = malloc(sizeof(int)*cpuid->nproc);
+ sched_getaffinity(0, sizeof(cpu_set_t), &save_cpuset);
+ /* Get APIC id from each core */
+ CPU_ZERO(&cpuset);
+ for (i = 0; i < cpuid->nproc; i++)
+ {
+ CPU_SET(i, &cpuset);
+ sched_setaffinity(0, sizeof(cpu_set_t), &cpuset);
+ execute_x86cpuid(0x1, 0, &eax, &ebx, &ecx, &edx);
+ apic_id[i] = ebx >> 24;
+ CPU_CLR(i, &cpuset);
+ }
+ /* Reset affinity to the value it had when calling this routine */
+ sched_setaffinity(0, sizeof(cpu_set_t), &save_cpuset);
+#define CPUID_HAVE_APIC
+#elif defined GMX_NATIVE_WINDOWS
+ /* Windows */
+ DWORD_PTR i;
+ SYSTEM_INFO sysinfo;
+ unsigned int save_affinity, affinity;
+ GetSystemInfo( &sysinfo );
+ cpuid->nproc = sysinfo.dwNumberOfProcessors;
+ apic_id = malloc(sizeof(int)*cpuid->nproc);
+ /* Get previous affinity mask */
+ save_affinity = SetThreadAffinityMask(GetCurrentThread(), 1);
+ for (i = 0; i < cpuid->nproc; i++)
+ {
+ SetThreadAffinityMask(GetCurrentThread(), (((DWORD_PTR)1)<<i));
+ Sleep(0);
+ execute_x86cpuid(0x1, 0, &eax, &ebx, &ecx, &edx);
+ apic_id[i] = ebx >> 24;
+ }
+ SetThreadAffinityMask(GetCurrentThread(), save_affinity);
+#define CPUID_HAVE_APIC
+#endif
+#ifdef CPUID_HAVE_APIC
+ /* AMD does not support SMT yet - there are no hwthread bits in apic ID */
+ hwthread_bits = 0;
+ /* Get number of core bits in apic ID - try modern extended method first */
+ execute_x86cpuid(0x80000008, 0, &eax, &ebx, &ecx, &edx);
+ core_bits = (ecx >> 12) & 0xf;
+ if (core_bits == 0)
+ {
+ /* Legacy method for old single/dual core AMD CPUs */
+ int i = ecx & 0xF;
+ for (core_bits = 0; (i>>core_bits) > 0; core_bits++)
+ {
+ ;
+ }
+ }
+ ret = cpuid_x86_decode_apic_id(cpuid, apic_id, core_bits,
+ hwthread_bits);
+ cpuid->have_cpu_topology = (ret == 0);
+#endif
+ }
+ return 0;
+}
+
+/* Detection of Intel-specific CPU features */
+static int
+cpuid_check_intel_x86(gmx_cpuid_t cpuid)
+{
+ unsigned int max_stdfn, max_extfn, ret;
+ unsigned int eax, ebx, ecx, edx;
+ unsigned int max_logical_cores, max_physical_cores;
+ int hwthread_bits, core_bits;
+ int * apic_id;
+
+ cpuid_check_common_x86(cpuid);
+
+ execute_x86cpuid(0x0, 0, &eax, &ebx, &ecx, &edx);
+ max_stdfn = eax;
+
+ execute_x86cpuid(0x80000000, 0, &eax, &ebx, &ecx, &edx);
+ max_extfn = eax;
+
+ if (max_stdfn >= 1)
+ {
+ execute_x86cpuid(0x1, 0, &eax, &ebx, &ecx, &edx);
+ cpuid->feature[GMX_CPUID_FEATURE_X86_PDCM] = (ecx & (1 << 15)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_PCID] = (ecx & (1 << 17)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_X2APIC] = (ecx & (1 << 21)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_TDT] = (ecx & (1 << 24)) != 0;
+ }
+
+ if (max_stdfn >= 7)
+ {
+ execute_x86cpuid(0x7, 0, &eax, &ebx, &ecx, &edx);
+ cpuid->feature[GMX_CPUID_FEATURE_X86_AVX2] = (ebx & (1 << 5)) != 0;
+ }
+
+ /* Check whether Hyper-Threading is enabled, not only supported */
+ if (cpuid->feature[GMX_CPUID_FEATURE_X86_HTT] && max_stdfn >= 4)
+ {
+ execute_x86cpuid(0x1, 0, &eax, &ebx, &ecx, &edx);
+ max_logical_cores = (ebx >> 16) & 0x0FF;
+ execute_x86cpuid(0x4, 0, &eax, &ebx, &ecx, &edx);
+ max_physical_cores = ((eax >> 26) & 0x3F) + 1;
+
+ /* Clear HTT flag if we only have 1 logical core per physical */
+ if (max_logical_cores/max_physical_cores < 2)
+ {
+ cpuid->feature[GMX_CPUID_FEATURE_X86_HTT] = 0;
+ }
+ }
+
+ if (max_stdfn >= 0xB)
+ {
+ /* Query x2 APIC information from cores */
+#if (defined HAVE_SCHED_H && defined HAVE_SCHED_SETAFFINITY && defined HAVE_SYSCONF && defined __linux__)
+ /* Linux */
+ unsigned int i;
+ cpu_set_t cpuset, save_cpuset;
+ cpuid->nproc = sysconf(_SC_NPROCESSORS_ONLN);
+ apic_id = malloc(sizeof(int)*cpuid->nproc);
+ sched_getaffinity(0, sizeof(cpu_set_t), &save_cpuset);
+ /* Get x2APIC ID from each hardware thread */
+ CPU_ZERO(&cpuset);
+ for (i = 0; i < cpuid->nproc; i++)
+ {
+ CPU_SET(i, &cpuset);
+ sched_setaffinity(0, sizeof(cpu_set_t), &cpuset);
+ execute_x86cpuid(0xB, 0, &eax, &ebx, &ecx, &edx);
+ apic_id[i] = edx;
+ CPU_CLR(i, &cpuset);
+ }
+ /* Reset affinity to the value it had when calling this routine */
+ sched_setaffinity(0, sizeof(cpu_set_t), &save_cpuset);
+#define CPUID_HAVE_APIC
+#elif defined GMX_NATIVE_WINDOWS
+ /* Windows */
+ DWORD_PTR i;
+ SYSTEM_INFO sysinfo;
+ unsigned int save_affinity, affinity;
+ GetSystemInfo( &sysinfo );
+ cpuid->nproc = sysinfo.dwNumberOfProcessors;
+ apic_id = malloc(sizeof(int)*cpuid->nproc);
+ /* Get previous affinity mask */
+ save_affinity = SetThreadAffinityMask(GetCurrentThread(), 1);
+ for (i = 0; i < cpuid->nproc; i++)
+ {
+ SetThreadAffinityMask(GetCurrentThread(), (((DWORD_PTR)1)<<i));
+ Sleep(0);
+ execute_x86cpuid(0xB, 0, &eax, &ebx, &ecx, &edx);
+ apic_id[i] = edx;
+ }
+ SetThreadAffinityMask(GetCurrentThread(), save_affinity);
+#define CPUID_HAVE_APIC
+#endif
+#ifdef CPUID_HAVE_APIC
+ execute_x86cpuid(0xB, 0, &eax, &ebx, &ecx, &edx);
+ hwthread_bits = eax & 0x1F;
+ execute_x86cpuid(0xB, 1, &eax, &ebx, &ecx, &edx);
+ core_bits = (eax & 0x1F) - hwthread_bits;
+ ret = cpuid_x86_decode_apic_id(cpuid, apic_id, core_bits,
+ hwthread_bits);
+ cpuid->have_cpu_topology = (ret == 0);
+#endif
+ }
+ return 0;
+}
+#endif /* GMX_CPUID_X86 */
+
+
+
+
+static void
+chomp_substring_before_colon(const char *in, char *s, int maxlength)
+{
+ char *p;
+ strncpy(s,in,maxlength);
+ p = strchr(s,':');
+ if(p!=NULL)
+ {
+ *p='\0';
+ while(isspace(*(--p)) && (p>=s))
+ {
+ *p='\0';
+ }
+ }
+ else
+ {
+ *s='\0';
+ }
+}
+
+static void
+chomp_substring_after_colon(const char *in, char *s, int maxlength)
+{
+ char *p;
+ if( (p = strchr(in,':'))!=NULL)
+ {
+ p++;
+ while(isspace(*p)) p++;
+ strncpy(s,p,maxlength);
+ p = s+strlen(s);
+ while(isspace(*(--p)) && (p>=s))
+ {
+ *p='\0';
+ }
+ }
+ else
+ {
+ *s='\0';
+ }
+}
+
+/* Try to find the vendor of the current CPU, so we know what specific
+ * detection routine to call.
+ */
+static enum gmx_cpuid_vendor
+cpuid_check_vendor(void)
+{
+ enum gmx_cpuid_vendor i, vendor;
+ /* Register data used on x86 */
+ unsigned int eax, ebx, ecx, edx;
+ char vendorstring[13];
+ FILE * fp;
+ char buffer[255],before_colon[255], after_colon[255];
+
+ /* Set default first */
+ vendor = GMX_CPUID_VENDOR_UNKNOWN;
+
+#ifdef GMX_CPUID_X86
+ execute_x86cpuid(0x0, 0, &eax, &ebx, &ecx, &edx);
+
+ memcpy(vendorstring, &ebx, 4);
+ memcpy(vendorstring+4, &edx, 4);
+ memcpy(vendorstring+8, &ecx, 4);
+
+ vendorstring[12] = '\0';
+
+ for (i = GMX_CPUID_VENDOR_UNKNOWN; i < GMX_CPUID_NVENDORS; i++)
+ {
+ if (!strncmp(vendorstring, gmx_cpuid_vendor_string[i], 12))
+ {
+ vendor = i;
+ }
+ }
+#elif defined(__linux__) || defined(__linux)
+ /* General Linux. Try to get CPU vendor from /proc/cpuinfo */
+ if( (fp = fopen("/proc/cpuinfo","r")) != NULL)
+ {
+ while( (vendor == GMX_CPUID_VENDOR_UNKNOWN) && (fgets(buffer,sizeof(buffer),fp) != NULL))
+ {
+ chomp_substring_before_colon(buffer,before_colon,sizeof(before_colon));
+ /* Intel/AMD use "vendor_id", IBM "vendor"(?) or "model". Fujitsu "manufacture". Add others if you have them! */
+ if( !strcmp(before_colon,"vendor_id")
+ || !strcmp(before_colon,"vendor")
+ || !strcmp(before_colon,"manufacture")
+ || !strcmp(before_colon,"model"))
+ {
+ chomp_substring_after_colon(buffer,after_colon,sizeof(after_colon));
+ for(i=GMX_CPUID_VENDOR_UNKNOWN; i<GMX_CPUID_NVENDORS; i++)
+ {
+ /* Be liberal and accept if we find the vendor
+ * string (or alternative string) anywhere. Using
+ * strcasestr() would be non-portable. */
+ if(strstr(after_colon,gmx_cpuid_vendor_string[i])
+ || strstr(after_colon,gmx_cpuid_vendor_string_alternative[i]))
+ {
+ vendor = i;
+ }
+ }
+ }
+ }
+ }
+ fclose(fp);
+#endif
+
+ return vendor;
+}
+
+
+
+int
+gmx_cpuid_topology(gmx_cpuid_t cpuid,
+ int * nprocessors,
+ int * npackages,
+ int * ncores_per_package,
+ int * nhwthreads_per_core,
+ const int ** package_id,
+ const int ** core_id,
+ const int ** hwthread_id,
+ const int ** locality_order)
+{
+ int rc;
+
+ if (cpuid->have_cpu_topology)
+ {
+ *nprocessors = cpuid->nproc;
+ *npackages = cpuid->npackages;
+ *ncores_per_package = cpuid->ncores_per_package;
+ *nhwthreads_per_core = cpuid->nhwthreads_per_core;
+ *package_id = cpuid->package_id;
+ *core_id = cpuid->core_id;
+ *hwthread_id = cpuid->hwthread_id;
+ *locality_order = cpuid->locality_order;
+ rc = 0;
+ }
+ else
+ {
+ rc = -1;
+ }
+ return rc;
+}
+
+
+enum gmx_cpuid_x86_smt
+gmx_cpuid_x86_smt(gmx_cpuid_t cpuid)
+{
+ enum gmx_cpuid_x86_smt rc;
+
+ if (cpuid->have_cpu_topology)
+ {
+ rc = (cpuid->nhwthreads_per_core > 1) ? GMX_CPUID_X86_SMT_ENABLED : GMX_CPUID_X86_SMT_DISABLED;
+ }
+ else if (cpuid->vendor == GMX_CPUID_VENDOR_AMD || gmx_cpuid_feature(cpuid, GMX_CPUID_FEATURE_X86_HTT) == 0)
+ {
+ rc = GMX_CPUID_X86_SMT_DISABLED;
+ }
+ else
+ {
+ rc = GMX_CPUID_X86_SMT_CANNOTDETECT;
+ }
+ return rc;
+}
+
+
+int
+gmx_cpuid_init (gmx_cpuid_t * pcpuid)
+{
+ gmx_cpuid_t cpuid;
+ int i;
+ FILE * fp;
+ char buffer[255],buffer2[255];
+ int found_brand;
+
+ cpuid = malloc(sizeof(*cpuid));
+
+ *pcpuid = cpuid;
+
+ for (i = 0; i < GMX_CPUID_NFEATURES; i++)
+ {
+ cpuid->feature[i] = 0;
+ }
+
+ cpuid->have_cpu_topology = 0;
+ cpuid->nproc = 0;
+ cpuid->npackages = 0;
+ cpuid->ncores_per_package = 0;
+ cpuid->nhwthreads_per_core = 0;
+ cpuid->package_id = NULL;
+ cpuid->core_id = NULL;
+ cpuid->hwthread_id = NULL;
+ cpuid->locality_order = NULL;
+
+ cpuid->vendor = cpuid_check_vendor();
+
+ switch (cpuid->vendor)
+ {
+#ifdef GMX_CPUID_X86
+ case GMX_CPUID_VENDOR_INTEL:
+ cpuid_check_intel_x86(cpuid);
+ break;
+ case GMX_CPUID_VENDOR_AMD:
+ cpuid_check_amd_x86(cpuid);
+ break;
+#endif
+ default:
+ /* Default value */
+ strncpy(cpuid->brand,"Unknown CPU brand",GMX_CPUID_BRAND_MAXLEN);
+#if defined(__linux__) || defined(__linux)
+ /* General Linux. Try to get CPU type from /proc/cpuinfo */
+ if( (fp = fopen("/proc/cpuinfo","r")) != NULL)
+ {
+ found_brand = 0;
+ while( (found_brand==0) && (fgets(buffer,sizeof(buffer),fp) !=NULL))
+ {
+ chomp_substring_before_colon(buffer,buffer2,sizeof(buffer2));
+ /* Intel uses "model name", Fujitsu and IBM "cpu". */
+ if( !strcmp(buffer2,"model name") || !strcmp(buffer2,"cpu"))
+ {
+ chomp_substring_after_colon(buffer,cpuid->brand,GMX_CPUID_BRAND_MAXLEN);
+ found_brand = 1;
+ }
+ }
+ }
+ fclose(fp);
+#endif
+ cpuid->family = 0;
+ cpuid->model = 0;
+ cpuid->stepping = 0;
+
+ for(i=0; i<GMX_CPUID_NFEATURES; i++)
+ {
+ cpuid->feature[i]=0;
+ }
+ cpuid->feature[GMX_CPUID_FEATURE_CANNOTDETECT] = 1;
+ break;
+ }
+ return 0;
+}
+
+
+
+void
+gmx_cpuid_done (gmx_cpuid_t cpuid)
+{
+ free(cpuid);
+}
+
+
+int
+gmx_cpuid_formatstring (gmx_cpuid_t cpuid,
+ char * str,
+ int n)
+{
+ int c;
+ int i;
+ enum gmx_cpuid_feature feature;
+
+#ifdef _MSC_VER
+ _snprintf(str, n,
+ "Vendor: %s\n"
+ "Brand: %s\n"
+ "Family: %2d Model: %2d Stepping: %2d\n"
+ "Features:",
+ gmx_cpuid_vendor_string[gmx_cpuid_vendor(cpuid)],
+ gmx_cpuid_brand(cpuid),
+ gmx_cpuid_family(cpuid), gmx_cpuid_model(cpuid), gmx_cpuid_stepping(cpuid));
+#else
+ snprintf(str, n,
+ "Vendor: %s\n"
+ "Brand: %s\n"
+ "Family: %2d Model: %2d Stepping: %2d\n"
+ "Features:",
+ gmx_cpuid_vendor_string[gmx_cpuid_vendor(cpuid)],
+ gmx_cpuid_brand(cpuid),
+ gmx_cpuid_family(cpuid), gmx_cpuid_model(cpuid), gmx_cpuid_stepping(cpuid));
+#endif
+
+ str[n-1] = '\0';
+ c = strlen(str);
+ n -= c;
+ str += c;
+
+ for (feature = GMX_CPUID_FEATURE_CANNOTDETECT; feature < GMX_CPUID_NFEATURES; feature++)
+ {
+ if (gmx_cpuid_feature(cpuid, feature) == 1)
+ {
+#ifdef _MSC_VER
+ _snprintf(str, n, " %s", gmx_cpuid_feature_string[feature]);
+#else
+ snprintf(str, n, " %s", gmx_cpuid_feature_string[feature]);
+#endif
+ str[n-1] = '\0';
+ c = strlen(str);
+ n -= c;
+ str += c;
+ }
+ }
+#ifdef _MSC_VER
+ _snprintf(str, n, "\n");
+#else
+ snprintf(str, n, "\n");
+#endif
+ str[n-1] = '\0';
+
+ return 0;
+}
+
+
+
+enum gmx_cpuid_acceleration
+gmx_cpuid_acceleration_suggest (gmx_cpuid_t cpuid)
+{
+ enum gmx_cpuid_acceleration tmpacc;
+
+ tmpacc = GMX_CPUID_ACCELERATION_NONE;
+
+ if (gmx_cpuid_vendor(cpuid) == GMX_CPUID_VENDOR_INTEL)
+ {
+ if (gmx_cpuid_feature(cpuid, GMX_CPUID_FEATURE_X86_AVX))
+ {
+ tmpacc = GMX_CPUID_ACCELERATION_X86_AVX_256;
+ }
+ else if (gmx_cpuid_feature(cpuid, GMX_CPUID_FEATURE_X86_SSE4_1))
+ {
+ tmpacc = GMX_CPUID_ACCELERATION_X86_SSE4_1;
+ }
+ else if (gmx_cpuid_feature(cpuid, GMX_CPUID_FEATURE_X86_SSE2))
+ {
+ tmpacc = GMX_CPUID_ACCELERATION_X86_SSE2;
+ }
+ }
+ else if (gmx_cpuid_vendor(cpuid) == GMX_CPUID_VENDOR_AMD)
+ {
+ if (gmx_cpuid_feature(cpuid, GMX_CPUID_FEATURE_X86_AVX))
+ {
+ tmpacc = GMX_CPUID_ACCELERATION_X86_AVX_128_FMA;
+ }
+ else if (gmx_cpuid_feature(cpuid, GMX_CPUID_FEATURE_X86_SSE4_1))
+ {
+ tmpacc = GMX_CPUID_ACCELERATION_X86_SSE4_1;
+ }
+ else if (gmx_cpuid_feature(cpuid, GMX_CPUID_FEATURE_X86_SSE2))
+ {
+ tmpacc = GMX_CPUID_ACCELERATION_X86_SSE2;
+ }
+ }
+ else if(gmx_cpuid_vendor(cpuid)==GMX_CPUID_VENDOR_FUJITSU)
+ {
+ if(strstr(gmx_cpuid_brand(cpuid),"SPARC64"))
+ {
+ tmpacc = GMX_CPUID_ACCELERATION_SPARC64_HPC_ACE;
+ }
+ }
+ else if(gmx_cpuid_vendor(cpuid)==GMX_CPUID_VENDOR_IBM)
+ {
+ if(strstr(gmx_cpuid_brand(cpuid),"A2"))
+ {
+ tmpacc = GMX_CPUID_ACCELERATION_IBM_QPX;
+ }
+ }
+ return tmpacc;
+}
+
+
+
+int
+gmx_cpuid_acceleration_check(gmx_cpuid_t cpuid,
+ FILE * log)
+{
+ int rc;
+ char str[1024];
+ enum gmx_cpuid_acceleration acc;
+
+ acc = gmx_cpuid_acceleration_suggest(cpuid);
+
+ rc = (acc != compiled_acc);
+
+ gmx_cpuid_formatstring(cpuid, str, 1023);
+ str[1023] = '\0';
+
+ if (log != NULL)
+ {
+ fprintf(log,
+ "\nDetecting CPU-specific acceleration.\nPresent hardware specification:\n"
+ "%s"
+ "Acceleration most likely to fit this hardware: %s\n"
+ "Acceleration selected at GROMACS compile time: %s\n\n",
+ str,
+ gmx_cpuid_acceleration_string[acc],
+ gmx_cpuid_acceleration_string[compiled_acc]);
+ }
+
+ if (rc != 0)
+ {
+ if (log != NULL)
+ {
+ fprintf(log, "\nBinary not matching hardware - you might be losing performance.\n"
+ "Acceleration most likely to fit this hardware: %s\n"
+ "Acceleration selected at GROMACS compile time: %s\n\n",
+ gmx_cpuid_acceleration_string[acc],
+ gmx_cpuid_acceleration_string[compiled_acc]);
+ }
+ printf("Compiled acceleration: %s (Gromacs could use %s on this machine, which is better)\n",
+ gmx_cpuid_acceleration_string[compiled_acc],
+ gmx_cpuid_acceleration_string[acc]);
+ }
+ return rc;
+}
+
+
+#ifdef GMX_CPUID_STANDALONE
+/* Stand-alone program to enable queries of CPU features from Cmake.
+ * Note that you need to check inline ASM capabilities before compiling and set
+ * -DGMX_X86_GCC_INLINE_ASM for the cpuid instruction to work...
+ */
+int
+main(int argc, char **argv)
+{
+ gmx_cpuid_t cpuid;
+ enum gmx_cpuid_acceleration acc;
+ int i, cnt;
+
+ if (argc < 2)
+ {
+ fprintf(stdout,
+ "Usage:\n\n%s [flags]\n\n"
+ "Available flags:\n"
+ "-vendor Print CPU vendor.\n"
+ "-brand Print CPU brand string.\n"
+ "-family Print CPU family version.\n"
+ "-model Print CPU model version.\n"
+ "-stepping Print CPU stepping version.\n"
+ "-features Print CPU feature flags.\n"
+ "-acceleration Print suggested GROMACS acceleration.\n",
+ argv[0]);
+ exit(0);
+ }
+
+ gmx_cpuid_init(&cpuid);
+
+ if (!strncmp(argv[1], "-vendor", 3))
+ {
+ printf("%s\n", gmx_cpuid_vendor_string[cpuid->vendor]);
+ }
+ else if (!strncmp(argv[1], "-brand", 3))
+ {
+ printf("%s\n", cpuid->brand);
+ }
+ else if (!strncmp(argv[1], "-family", 3))
+ {
+ printf("%d\n", cpuid->family);
+ }
+ else if (!strncmp(argv[1], "-model", 3))
+ {
+ printf("%d\n", cpuid->model);
+ }
+ else if (!strncmp(argv[1], "-stepping", 3))
+ {
+ printf("%d\n", cpuid->stepping);
+ }
+ else if (!strncmp(argv[1], "-features", 3))
+ {
+ cnt = 0;
+ for (i = 0; i < GMX_CPUID_NFEATURES; i++)
+ {
+ if (cpuid->feature[i] == 1)
+ {
+ if (cnt++ > 0)
+ {
+ printf(" ");
+ }
+ printf("%s", gmx_cpuid_feature_string[i]);
+ }
+ }
+ printf("\n");
+ }
+ else if (!strncmp(argv[1], "-acceleration", 3))
+ {
+ acc = gmx_cpuid_acceleration_suggest(cpuid);
+ fprintf(stdout, "%s\n", gmx_cpuid_acceleration_string[acc]);
+ }
+
+ gmx_cpuid_done(cpuid);
+
+
+ return 0;
+}
+
+#endif
--- /dev/null
- int nth_affinity_set, thread_id_node, thread_id,
+/*
+ * This file is part of the GROMACS molecular simulation package.
+ *
+ * Copyright (c) 2012, by the GROMACS development team, led by
+ * David van der Spoel, Berk Hess, Erik Lindahl, and including many
+ * others, as listed in the AUTHORS file in the top-level source
+ * directory and at http://www.gromacs.org.
+ *
+ * GROMACS is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ *
+ * GROMACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with GROMACS; if not, see
+ * http://www.gnu.org/licenses, or write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * If you want to redistribute modifications to GROMACS, please
+ * consider that scientific software is very special. Version
+ * control is crucial - bugs must be traceable. We will be happy to
+ * consider code for inclusion in the official distribution, but
+ * derived work must not be called official GROMACS. Details are found
+ * in the README & COPYING files - if they are missing, get the
+ * official version at http://www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the research papers on the package. Check out http://www.gromacs.org.
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+#if defined(HAVE_SCHED_H) && defined(HAVE_SCHED_GETAFFINITY)
+#define _GNU_SOURCE
+#include <sched.h>
+#include <sys/syscall.h>
+#endif
+#include <string.h>
+#include <errno.h>
+#include <assert.h>
+#include <stdio.h>
+#include "typedefs.h"
+#include "types/commrec.h"
+#include "types/hw_info.h"
+#include "gmx_cpuid.h"
+#include "gmx_omp.h"
+#include "gmx_omp_nthreads.h"
+#include "mdrun.h"
+#include "md_logging.h"
+#include "statutil.h"
+#include "gmx_thread_affinity.h"
+
+#include "thread_mpi/threads.h"
+
+
+static int
+get_thread_affinity_layout(FILE *fplog,
+ const t_commrec *cr,
+ const gmx_hw_info_t * hwinfo,
+ int nthreads,
+ int pin_offset, int * pin_stride,
+ const int **locality_order)
+{
+ int nhwthreads, npkg, ncores, nhwthreads_per_core, rc;
+ const int * pkg_id;
+ const int * core_id;
+ const int * hwthread_id;
+ gmx_bool bPickPinStride;
+
+ if (pin_offset < 0)
+ {
+ gmx_fatal(FARGS, "Negative thread pinning offset requested");
+ }
+ if (*pin_stride < 0)
+ {
+ gmx_fatal(FARGS, "Negative thread pinning stride requested");
+ }
+
+ rc = gmx_cpuid_topology(hwinfo->cpuid_info, &nhwthreads, &npkg, &ncores,
+ &nhwthreads_per_core,
+ &pkg_id, &core_id, &hwthread_id, locality_order);
+
+ if (rc != 0)
+ {
+ /* topology information not available or invalid, ignore it */
+ nhwthreads = hwinfo->nthreads_hw_avail;
+ *locality_order = NULL;
+
+ if (nhwthreads <= 0)
+ {
+ /* We don't know anything about the hardware, don't pin */
+ md_print_warn(cr, fplog,
+ "NOTE: We don't know how many logical cores we have, will not pin threads");
+
+ return -1;
+ }
+ }
+
+ if (nthreads > nhwthreads)
+ {
+ /* We are oversubscribing, don't pin */
+ md_print_warn(NULL, fplog,
+ "WARNING: Oversubscribing the CPU, will not pin threads");
+
+ return -1;
+ }
+
+ if (pin_offset + nthreads > nhwthreads)
+ {
+ /* We are oversubscribing, don't pin */
+ md_print_warn(NULL, fplog,
+ "WARNING: The requested pin offset is too large for the available logical cores,\n"
+ " will not pin threads");
+
+ return -1;
+ }
+
+
+ /* do we need to choose the pinning stride? */
+ bPickPinStride = (*pin_stride == 0);
+
+ if (bPickPinStride)
+ {
+ if (rc == 0 && pin_offset + nthreads*nhwthreads_per_core <= nhwthreads)
+ {
+ /* Put one thread on each physical core */
+ *pin_stride = nhwthreads_per_core;
+ }
+ else
+ {
+ /* We don't know if we have SMT, and if we do, we don't know
+ * if hw threads in the same physical core are consecutive.
+ * Without SMT the pinning layout should not matter too much.
+ * so we assume a consecutive layout and maximally spread out"
+ * the threads at equal threads per core.
+ * Note that IBM is the major non-x86 case with cpuid support
+ * and probably threads are already pinned by the queuing system,
+ * so we wouldn't end up here in the first place.
+ */
+ *pin_stride = (nhwthreads - pin_offset)/nthreads;
+ }
+ }
+ else
+ {
+ /* Check the placement of the thread with the largest index to make sure
+ * that the offset & stride doesn't cause pinning beyond the last hardware thread. */
+ if (pin_offset + (nthreads-1)*(*pin_stride) >= nhwthreads)
+ {
+ /* We are oversubscribing, don't pin */
+ md_print_warn(NULL, fplog,
+ "WARNING: The requested pinning stride is too large for the available logical cores,\n"
+ " will not pin threads");
+
+ return -1;
+ }
+ }
+
+ if (fplog != NULL)
+ {
+ fprintf(fplog, "Pinning threads with a%s logical core stride of %d\n",
+ bPickPinStride ? "n auto-selected" : " user-specified",
+ *pin_stride);
+ }
+
+ return 0;
+}
+
+/* Set CPU affinity. Can be important for performance.
+ On some systems (e.g. Cray) CPU Affinity is set by default.
+ But default assigning doesn't work (well) with only some ranks
+ having threads. This causes very low performance.
+ External tools have cumbersome syntax for setting affinity
+ in the case that only some ranks have threads.
+ Thus it is important that GROMACS sets the affinity internally
+ if only PME is using threads.
+ */
+void
+gmx_set_thread_affinity(FILE *fplog,
+ const t_commrec *cr,
+ gmx_hw_opt_t *hw_opt,
+ const gmx_hw_info_t *hwinfo)
+{
- thread_id_node = 0;
- nthread_node = nthread_local;
++ int nth_affinity_set, thread0_id_node,
+ nthread_local, nthread_node, nthread_hw_max, nphyscore;
+ int offset;
+ const int *locality_order;
+ int rc;
+
+ if (hw_opt->thread_affinity == threadaffOFF)
+ {
+ /* Nothing to do */
+ return;
+ }
+
+ /* If the tMPI thread affinity setting is not supported encourage the user
+ * to report it as it's either a bug or an exotic platform which we might
+ * want to support. */
+ if (tMPI_Thread_setaffinity_support() != TMPI_SETAFFINITY_SUPPORT_YES)
+ {
+ /* we know Mac OS doesn't support setting thread affinity, so there's
+ no point in warning the user in that case. In any other case
+ the user might be able to do something about it. */
+#ifndef __APPLE__
+ md_print_warn(NULL, fplog,
+ "Can not set thread affinities on the current platform. On NUMA systems this\n"
+ "can cause performance degradation. If you think your platform should support\n"
+ "setting affinities, contact the GROMACS developers.");
+#endif /* __APPLE__ */
+ return;
+ }
+
+ /* threads on this MPI process or TMPI thread */
+ if (cr->duty & DUTY_PP)
+ {
+ nthread_local = gmx_omp_nthreads_get(emntNonbonded);
+ }
+ else
+ {
+ nthread_local = gmx_omp_nthreads_get(emntPME);
+ }
+
+ /* map the current process to cores */
- MPI_Scan(&nthread_local, &thread_id_node, 1, MPI_INT, MPI_SUM, comm_intra);
++ thread0_id_node = 0;
++ nthread_node = nthread_local;
+#ifdef GMX_MPI
+ if (PAR(cr) || MULTISIM(cr))
+ {
+ /* We need to determine a scan of the thread counts in this
+ * compute node.
+ */
+ MPI_Comm comm_intra;
+
+ MPI_Comm_split(MPI_COMM_WORLD, gmx_hostname_num(), cr->rank_intranode,
+ &comm_intra);
- thread_id_node -= nthread_local;
++ MPI_Scan(&nthread_local, &thread0_id_node, 1, MPI_INT, MPI_SUM, comm_intra);
+ /* MPI_Scan is inclusive, but here we need exclusive */
- #pragma omp parallel firstprivate(thread_id_node) num_threads(nthread_local) \
- reduction(+:nth_affinity_set)
++ thread0_id_node -= nthread_local;
+ /* Get the total number of threads on this physical node */
+ MPI_Allreduce(&nthread_local, &nthread_node, 1, MPI_INT, MPI_SUM, comm_intra);
+ MPI_Comm_free(&comm_intra);
+ }
+#endif
+
+ if (hw_opt->thread_affinity == threadaffAUTO &&
+ nthread_node != hwinfo->nthreads_hw_avail)
+ {
+ if (nthread_node > 1 && nthread_node < hwinfo->nthreads_hw_avail)
+ {
+ md_print_warn(cr, fplog,
+ "NOTE: The number of threads is not equal to the number of (logical) cores\n"
+ " and the -pin option is set to auto: will not pin thread to cores.\n"
+ " This can lead to significant performance degradation.\n"
+ " Consider using -pin on (and -pinoffset in case you run multiple jobs).\n");
+ }
+
+ return;
+ }
+
+ offset = 0;
+ if (hw_opt->core_pinning_offset != 0)
+ {
+ offset = hw_opt->core_pinning_offset;
+ md_print_info(cr, fplog, "Applying core pinning offset %d\n", offset);
+ }
+
+ rc = get_thread_affinity_layout(fplog, cr, hwinfo,
+ nthread_node,
+ offset, &hw_opt->core_pinning_stride,
+ &locality_order);
+
+ if (rc != 0)
+ {
+ /* Incompatible layout, don't pin, warning was already issued */
+ return;
+ }
+
+ /* Set the per-thread affinity. In order to be able to check the success
+ * of affinity settings, we will set nth_affinity_set to 1 on threads
+ * where the affinity setting succeded and to 0 where it failed.
+ * Reducing these 0/1 values over the threads will give the total number
+ * of threads on which we succeeded.
+ */
+ nth_affinity_set = 0;
- thread_id = gmx_omp_get_thread_num();
- thread_id_node += thread_id;
- index = offset + thread_id_node*hw_opt->core_pinning_stride;
++#pragma omp parallel num_threads(nthread_local) reduction(+:nth_affinity_set)
+ {
++ int thread_id, thread_id_node;
+ int index, core;
+ gmx_bool setaffinity_ret;
+
- fprintf(debug, "On rank %2d, thread %2d, core %2d the affinity setting returned %d\n",
- cr->nodeid, gmx_omp_get_thread_num(), core, setaffinity_ret);
++ thread_id = gmx_omp_get_thread_num();
++ thread_id_node = thread0_id_node + thread_id;
++ index = offset + thread_id_node*hw_opt->core_pinning_stride;
+ if (locality_order != NULL)
+ {
+ core = locality_order[index];
+ }
+ else
+ {
+ core = index;
+ }
+
+ setaffinity_ret = tMPI_Thread_setaffinity_single(tMPI_Thread_self(), core);
+
+ /* store the per-thread success-values of the setaffinity */
+ nth_affinity_set = (setaffinity_ret == 0);
+
+ if (debug)
+ {
++ fprintf(debug, "On rank %2d, thread %2d, index %2d, core %2d the affinity setting returned %d\n",
++ cr->nodeid, gmx_omp_get_thread_num(), index, core, setaffinity_ret);
+ }
+ }
+
+ if (nth_affinity_set > nthread_local)
+ {
+ char msg[STRLEN];
+
+ sprintf(msg, "Looks like we have set affinity for more threads than "
+ "we have (%d > %d)!\n", nth_affinity_set, nthread_local);
+ gmx_incons(msg);
+ }
+ else
+ {
+ /* check & warn if some threads failed to set their affinities */
+ if (nth_affinity_set != nthread_local)
+ {
+ char sbuf1[STRLEN], sbuf2[STRLEN];
+
+ /* sbuf1 contains rank info, while sbuf2 OpenMP thread info */
+ sbuf1[0] = sbuf2[0] = '\0';
+ /* Only add rank info if we have more than one rank. */
+ if (cr->nnodes > 1)
+ {
+#ifdef GMX_MPI
+#ifdef GMX_THREAD_MPI
+ sprintf(sbuf1, "In tMPI thread #%d: ", cr->nodeid);
+#else /* GMX_LIB_MPI */
+ sprintf(sbuf1, "In MPI process #%d: ", cr->nodeid);
+#endif
+#endif /* GMX_MPI */
+ }
+
+ if (nthread_local > 1)
+ {
+ sprintf(sbuf2, "for %d/%d thread%s ",
+ nthread_local - nth_affinity_set, nthread_local,
+ nthread_local > 1 ? "s" : "");
+ }
+
+ md_print_warn(NULL, fplog,
+ "WARNING: %sAffinity setting %sfailed.\n"
+ " This can cause performance degradation! If you think your setting are\n"
+ " correct, contact the GROMACS developers.",
+ sbuf1, sbuf2);
+ }
+ }
+ return;
+}
+
+/* Check the process affinity mask and if it is found to be non-zero,
+ * will honor it and disable mdrun internal affinity setting.
+ * Note that this will only work on Linux as we use a GNU feature.
+ */
+void
+gmx_check_thread_affinity_set(FILE *fplog, const t_commrec *cr,
+ gmx_hw_opt_t *hw_opt, int ncpus,
+ gmx_bool bAfterOpenmpInit)
+{
+#ifdef HAVE_SCHED_GETAFFINITY
+ cpu_set_t mask_current;
+ int i, ret, cpu_count, cpu_set;
+ gmx_bool bAllSet;
+
+ assert(hw_opt);
+ if (hw_opt->thread_affinity == threadaffOFF)
+ {
+ /* internal affinity setting is off, don't bother checking process affinity */
+ return;
+ }
+
+ CPU_ZERO(&mask_current);
+ if ((ret = sched_getaffinity(0, sizeof(cpu_set_t), &mask_current)) != 0)
+ {
+ /* failed to query affinity mask, will just return */
+ if (debug)
+ {
+ fprintf(debug, "Failed to query affinity mask (error %d)", ret);
+ }
+ return;
+ }
+
+ /* Before proceeding with the actual check, make sure that the number of
+ * detected CPUs is >= the CPUs in the current set.
+ * We need to check for CPU_COUNT as it was added only in glibc 2.6. */
+#ifdef CPU_COUNT
+ if (ncpus < CPU_COUNT(&mask_current))
+ {
+ if (debug)
+ {
+ fprintf(debug, "%d CPUs detected, but %d was returned by CPU_COUNT",
+ ncpus, CPU_COUNT(&mask_current));
+ }
+ return;
+ }
+#endif /* CPU_COUNT */
+
+ bAllSet = TRUE;
+ for (i = 0; (i < ncpus && i < CPU_SETSIZE); i++)
+ {
+ bAllSet = bAllSet && (CPU_ISSET(i, &mask_current) != 0);
+ }
+
+ if (!bAllSet)
+ {
+ if (hw_opt->thread_affinity == threadaffAUTO)
+ {
+ if (!bAfterOpenmpInit)
+ {
+ md_print_warn(cr, fplog,
+ "Non-default thread affinity set, disabling internal thread affinity");
+ }
+ else
+ {
+ md_print_warn(cr, fplog,
+ "Non-default thread affinity set probably by the OpenMP library,\n"
+ "disabling internal thread affinity");
+ }
+ hw_opt->thread_affinity = threadaffOFF;
+ }
+ else
+ {
+ /* Only warn once, at the last check (bAfterOpenmpInit==TRUE) */
+ if (bAfterOpenmpInit)
+ {
+ md_print_warn(cr, fplog,
+ "Overriding thread affinity set outside %s\n",
+ ShortProgram());
+ }
+ }
+
+ if (debug)
+ {
+ fprintf(debug, "Non-default affinity mask found\n");
+ }
+ }
+ else
+ {
+ if (debug)
+ {
+ fprintf(debug, "Default affinity mask found\n");
+ }
+ }
+#endif /* HAVE_SCHED_GETAFFINITY */
+}
--- /dev/null
- fprintf(debug, "In gmx_setup_nodecomm: hostname '%s', hostnum %d\n",
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * VERSION 3.2.0
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2004, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * GROningen Mixture of Alchemy and Childrens' Stories
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <string.h>
+#include "gmx_fatal.h"
+#include "main.h"
+#include "smalloc.h"
+#include "network.h"
+#include "copyrite.h"
+#include "statutil.h"
+#include <ctype.h>
+#include "macros.h"
+
+#include "gromacs/utility/gmxmpi.h"
+
+
+/* The source code in this file should be thread-safe.
+ Please keep it that way. */
+
+gmx_bool gmx_mpi_initialized(void)
+{
+ int n;
+#ifndef GMX_MPI
+ return 0;
+#else
+ MPI_Initialized(&n);
+
+ return n;
+#endif
+}
+
+void gmx_fill_commrec_from_mpi(t_commrec *cr)
+{
+#ifndef GMX_MPI
+ gmx_call("gmx_fill_commrec_from_mpi");
+#else
+ if (!gmx_mpi_initialized())
+ {
+ gmx_comm("MPI has not been initialized properly");
+ }
+
+ cr->nnodes = gmx_node_num();
+ cr->nodeid = gmx_node_rank();
+ cr->sim_nodeid = cr->nodeid;
+ cr->mpi_comm_mysim = MPI_COMM_WORLD;
+ cr->mpi_comm_mygroup = MPI_COMM_WORLD;
+
+#endif
+}
+
+t_commrec *init_commrec()
+{
+ t_commrec *cr;
+
+ snew(cr, 1);
+
+#ifdef GMX_LIB_MPI
+ gmx_fill_commrec_from_mpi(cr);
+#else
+ cr->mpi_comm_mysim = NULL;
+ cr->mpi_comm_mygroup = NULL;
+ cr->nnodes = 1;
+ cr->sim_nodeid = 0;
+ cr->nodeid = cr->sim_nodeid;
+#endif
+
+ // TODO cr->duty should not be initialized here
+ cr->duty = (DUTY_PP | DUTY_PME);
+
+#if defined GMX_LIB_MPI && !defined MPI_IN_PLACE_EXISTS
+ /* initialize the MPI_IN_PLACE replacement buffers */
+ snew(cr->mpb, 1);
+ cr->mpb->ibuf = NULL;
+ cr->mpb->libuf = NULL;
+ cr->mpb->fbuf = NULL;
+ cr->mpb->dbuf = NULL;
+ cr->mpb->ibuf_alloc = 0;
+ cr->mpb->libuf_alloc = 0;
+ cr->mpb->fbuf_alloc = 0;
+ cr->mpb->dbuf_alloc = 0;
+#endif
+
+ return cr;
+}
+
+t_commrec *reinitialize_commrec_for_this_thread(const t_commrec *cro)
+{
+#ifdef GMX_THREAD_MPI
+ t_commrec *cr;
+
+ /* make a thread-specific commrec */
+ snew(cr, 1);
+ /* now copy the whole thing, so settings like the number of PME nodes
+ get propagated. */
+ *cr = *cro;
+
+ /* and we start setting our own thread-specific values for things */
+ gmx_fill_commrec_from_mpi(cr);
+
+ // TODO cr->duty should not be initialized here
+ cr->duty = (DUTY_PP | DUTY_PME);
+
+ return cr;
+#else
+ return NULL;
+#endif
+}
+
+int gmx_node_num(void)
+{
+#ifndef GMX_MPI
+ return 1;
+#else
+ int i;
+ (void) MPI_Comm_size(MPI_COMM_WORLD, &i);
+ return i;
+#endif
+}
+
+int gmx_node_rank(void)
+{
+#ifndef GMX_MPI
+ return 0;
+#else
+ int i;
+ (void) MPI_Comm_rank(MPI_COMM_WORLD, &i);
+ return i;
+#endif
+}
+
++#if defined GMX_LIB_MPI && defined GMX_IS_BGQ
++#include <spi/include/kernel/location.h>
++#endif
+
+int gmx_hostname_num()
+{
+#ifndef GMX_MPI
+ return 0;
+#else
+#ifdef GMX_THREAD_MPI
+ /* thread-MPI currently puts the thread number in the process name,
+ * we might want to change this, as this is inconsistent with what
+ * most MPI implementations would do when running on a single node.
+ */
+ return 0;
+#else
+ int resultlen, hostnum, i, j;
+ char mpi_hostname[MPI_MAX_PROCESSOR_NAME], hostnum_str[MPI_MAX_PROCESSOR_NAME];
+
+ MPI_Get_processor_name(mpi_hostname, &resultlen);
++#ifdef GMX_IS_BGQ
++ Personality_t personality;
++ Kernel_GetPersonality(&personality, sizeof(personality));
++ /* Each MPI rank has a unique coordinate in a 6-dimensional space
++ (A,B,C,D,E,T), with dimensions A-E corresponding to different
++ physical nodes, and T within each node. Each node has sixteen
++ physical cores, each of which can have up to four hardware
++ threads, so 0 <= T <= 63 (but the maximum value of T depends on
++ the confituration of ranks and OpenMP threads per
++ node). However, T is irrelevant for computing a suitable return
++ value for gmx_hostname_num().
++ */
++ hostnum = personality.Network_Config.Acoord;
++ hostnum *= personality.Network_Config.Bnodes;
++ hostnum += personality.Network_Config.Bcoord;
++ hostnum *= personality.Network_Config.Cnodes;
++ hostnum += personality.Network_Config.Ccoord;
++ hostnum *= personality.Network_Config.Dnodes;
++ hostnum += personality.Network_Config.Dcoord;
++ hostnum *= personality.Network_Config.Enodes;
++ hostnum += personality.Network_Config.Ecoord;
++#else
+ /* This procedure can only differentiate nodes with host names
+ * that end on unique numbers.
+ */
+ i = 0;
+ j = 0;
+ /* Only parse the host name up to the first dot */
+ while (i < resultlen && mpi_hostname[i] != '.')
+ {
+ if (isdigit(mpi_hostname[i]))
+ {
+ hostnum_str[j++] = mpi_hostname[i];
+ }
+ i++;
+ }
+ hostnum_str[j] = '\0';
+ if (j == 0)
+ {
+ hostnum = 0;
+ }
+ else
+ {
+ /* Use only the last 9 decimals, so we don't overflow an int */
+ hostnum = strtol(hostnum_str + max(0, j-9), NULL, 10);
+ }
++#endif
+
+ if (debug)
+ {
++ fprintf(debug, "In gmx_hostname_num: hostname '%s', hostnum %d\n",
+ mpi_hostname, hostnum);
++#ifdef GMX_IS_BGQ
++ fprintf(debug,
++ "Torus ID A: %d / %d B: %d / %d C: %d / %d D: %d / %d E: %d / %d\nNode ID T: %d / %d core: %d / %d hardware thread: %d / %d\n",
++ personality.Network_Config.Acoord,
++ personality.Network_Config.Anodes,
++ personality.Network_Config.Bcoord,
++ personality.Network_Config.Bnodes,
++ personality.Network_Config.Ccoord,
++ personality.Network_Config.Cnodes,
++ personality.Network_Config.Dcoord,
++ personality.Network_Config.Dnodes,
++ personality.Network_Config.Ecoord,
++ personality.Network_Config.Enodes,
++ Kernel_ProcessorCoreID(),
++ 16,
++ Kernel_ProcessorID(),
++ 64,
++ Kernel_ProcessorThreadID(),
++ 4);
++#endif
+ }
+ return hostnum;
+#endif
+#endif
+}
+
+void gmx_setup_nodecomm(FILE gmx_unused *fplog, t_commrec *cr)
+{
+ gmx_nodecomm_t *nc;
+ int n, rank, hostnum, ng, ni;
+
+ /* Many MPI implementations do not optimize MPI_Allreduce
+ * (and probably also other global communication calls)
+ * for multi-core nodes connected by a network.
+ * We can optimize such communication by using one MPI call
+ * within each node and one between the nodes.
+ * For MVAPICH2 and Intel MPI this reduces the time for
+ * the global_stat communication by 25%
+ * for 2x2-core 3 GHz Woodcrest connected by mixed DDR/SDR Infiniband.
+ * B. Hess, November 2007
+ */
+
+ nc = &cr->nc;
+
+ nc->bUse = FALSE;
+#ifndef GMX_THREAD_MPI
+#ifdef GMX_MPI
+ MPI_Comm_size(cr->mpi_comm_mygroup, &n);
+ MPI_Comm_rank(cr->mpi_comm_mygroup, &rank);
+
+ hostnum = gmx_hostname_num();
+
+ if (debug)
+ {
+ fprintf(debug, "In gmx_setup_nodecomm: splitting communicator of size %d\n", n);
+ }
+
+
+ /* The intra-node communicator, split on node number */
+ MPI_Comm_split(cr->mpi_comm_mygroup, hostnum, rank, &nc->comm_intra);
+ MPI_Comm_rank(nc->comm_intra, &nc->rank_intra);
+ if (debug)
+ {
+ fprintf(debug, "In gmx_setup_nodecomm: node rank %d rank_intra %d\n",
+ rank, nc->rank_intra);
+ }
+ /* The inter-node communicator, split on rank_intra.
+ * We actually only need the one for rank=0,
+ * but it is easier to create them all.
+ */
+ MPI_Comm_split(cr->mpi_comm_mygroup, nc->rank_intra, rank, &nc->comm_inter);
+ /* Check if this really created two step communication */
+ MPI_Comm_size(nc->comm_inter, &ng);
+ MPI_Comm_size(nc->comm_intra, &ni);
+ if (debug)
+ {
+ fprintf(debug, "In gmx_setup_nodecomm: groups %d, my group size %d\n",
+ ng, ni);
+ }
+
+ if (getenv("GMX_NO_NODECOMM") == NULL &&
+ ((ng > 1 && ng < n) || (ni > 1 && ni < n)))
+ {
+ nc->bUse = TRUE;
+ if (fplog)
+ {
+ fprintf(fplog, "Using two step summing over %d groups of on average %.1f processes\n\n",
+ ng, (real)n/(real)ng);
+ }
+ if (nc->rank_intra > 0)
+ {
+ MPI_Comm_free(&nc->comm_inter);
+ }
+ }
+ else
+ {
+ /* One group or all processes in a separate group, use normal summing */
+ MPI_Comm_free(&nc->comm_inter);
+ MPI_Comm_free(&nc->comm_intra);
+ if (debug)
+ {
+ fprintf(debug, "In gmx_setup_nodecomm: not unsing separate inter- and intra-node communicators.\n");
+ }
+ }
+#endif
+#else
+ /* tMPI runs only on a single node so just use the nodeid */
+ nc->rank_intra = cr->nodeid;
+#endif
+}
+
+void gmx_init_intranode_counters(t_commrec *cr)
+{
+ /* counters for PP+PME and PP-only processes on my physical node */
+ int nrank_intranode, rank_intranode;
+ int nrank_pp_intranode, rank_pp_intranode;
+ /* thread-MPI is not initialized when not running in parallel */
+#if defined GMX_MPI && !defined GMX_THREAD_MPI
+ int nrank_world, rank_world;
+ int i, mynum, *num, *num_s, *num_pp, *num_pp_s;
+
+ MPI_Comm_size(MPI_COMM_WORLD, &nrank_world);
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank_world);
+
+ /* Get the node number from the hostname to identify the nodes */
+ mynum = gmx_hostname_num();
+
+ /* We can't rely on MPI_IN_PLACE, so we need send and receive buffers */
+ snew(num, nrank_world);
+ snew(num_s, nrank_world);
+ snew(num_pp, nrank_world);
+ snew(num_pp_s, nrank_world);
+
+ num_s[rank_world] = mynum;
+ num_pp_s[rank_world] = (cr->duty & DUTY_PP) ? mynum : -1;
+
+ MPI_Allreduce(num_s, num, nrank_world, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+ MPI_Allreduce(num_pp_s, num_pp, nrank_world, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+
+ nrank_intranode = 0;
+ rank_intranode = 0;
+ nrank_pp_intranode = 0;
+ rank_pp_intranode = 0;
+ for (i = 0; i < nrank_world; i++)
+ {
+ if (num[i] == mynum)
+ {
+ nrank_intranode++;
+ if (i < rank_world)
+ {
+ rank_intranode++;
+ }
+ }
+ if ((cr->duty & DUTY_PP) && num_pp[i] == mynum)
+ {
+ nrank_pp_intranode++;
+ if (i < rank_world)
+ {
+ rank_pp_intranode++;
+ }
+ }
+ }
+ sfree(num);
+ sfree(num_s);
+ sfree(num_pp);
+ sfree(num_pp_s);
+#else
+ /* Serial or thread-MPI code: we run within a single physical node */
+ nrank_intranode = cr->nnodes;
+ rank_intranode = cr->sim_nodeid;
+ nrank_pp_intranode = cr->nnodes - cr->npmenodes;
+ rank_pp_intranode = cr->nodeid;
+#endif
+
+ if (debug)
+ {
+ char sbuf[STRLEN];
+ if (cr->duty & DUTY_PP && cr->duty & DUTY_PME)
+ {
+ sprintf(sbuf, "PP+PME");
+ }
+ else
+ {
+ sprintf(sbuf, "%s", cr->duty & DUTY_PP ? "PP" : "PME");
+ }
+ fprintf(debug, "On %3s node %d: nrank_intranode=%d, rank_intranode=%d, "
+ "nrank_pp_intranode=%d, rank_pp_intranode=%d\n",
+ sbuf, cr->sim_nodeid,
+ nrank_intranode, rank_intranode,
+ nrank_pp_intranode, rank_pp_intranode);
+ }
+
+ cr->nrank_intranode = nrank_intranode;
+ cr->rank_intranode = rank_intranode;
+ cr->nrank_pp_intranode = nrank_pp_intranode;
+ cr->rank_pp_intranode = rank_pp_intranode;
+}
+
+
+void gmx_barrier(const t_commrec *cr)
+{
+#ifndef GMX_MPI
+ gmx_call("gmx_barrier");
+#else
+ MPI_Barrier(cr->mpi_comm_mygroup);
+#endif
+}
+
+void gmx_abort(int gmx_unused noderank, int gmx_unused nnodes, int gmx_unused errorno)
+{
+#ifndef GMX_MPI
+ gmx_call("gmx_abort");
+#else
+#ifdef GMX_THREAD_MPI
+ fprintf(stderr, "Halting program %s\n", ShortProgram());
+ gmx_thanx(stderr);
+ exit(1);
+#else
+ if (nnodes > 1)
+ {
+ fprintf(stderr, "Halting parallel program %s on CPU %d out of %d\n",
+ ShortProgram(), noderank, nnodes);
+ }
+ else
+ {
+ fprintf(stderr, "Halting program %s\n", ShortProgram());
+ }
+
+ gmx_thanx(stderr);
+ MPI_Abort(MPI_COMM_WORLD, errorno);
+ exit(1);
+#endif
+#endif
+}
+
+void gmx_bcast(int nbytes, void *b, const t_commrec *cr)
+{
+#ifndef GMX_MPI
+ gmx_call("gmx_bast");
+#else
+ MPI_Bcast(b, nbytes, MPI_BYTE, MASTERRANK(cr), cr->mpi_comm_mygroup);
+#endif
+}
+
+void gmx_bcast_sim(int nbytes, void *b, const t_commrec *cr)
+{
+#ifndef GMX_MPI
+ gmx_call("gmx_bast");
+#else
+ MPI_Bcast(b, nbytes, MPI_BYTE, MASTERRANK(cr), cr->mpi_comm_mysim);
+#endif
+}
+
+void gmx_sumd(int nr, double r[], const t_commrec *cr)
+{
+#ifndef GMX_MPI
+ gmx_call("gmx_sumd");
+#else
+#if defined(MPI_IN_PLACE_EXISTS) || defined(GMX_THREAD_MPI)
+ if (cr->nc.bUse)
+ {
+ if (cr->nc.rank_intra == 0)
+ {
+ /* Use two step summing. */
+ MPI_Reduce(MPI_IN_PLACE, r, nr, MPI_DOUBLE, MPI_SUM, 0,
+ cr->nc.comm_intra);
+ /* Sum the roots of the internal (intra) buffers. */
+ MPI_Allreduce(MPI_IN_PLACE, r, nr, MPI_DOUBLE, MPI_SUM,
+ cr->nc.comm_inter);
+ }
+ else
+ {
+ /* This is here because of the silly MPI specification
+ that MPI_IN_PLACE should be put in sendbuf instead of recvbuf */
+ MPI_Reduce(r, NULL, nr, MPI_DOUBLE, MPI_SUM, 0, cr->nc.comm_intra);
+ }
+ MPI_Bcast(r, nr, MPI_DOUBLE, 0, cr->nc.comm_intra);
+ }
+ else
+ {
+ MPI_Allreduce(MPI_IN_PLACE, r, nr, MPI_DOUBLE, MPI_SUM,
+ cr->mpi_comm_mygroup);
+ }
+#else
+ int i;
+
+ if (nr > cr->mpb->dbuf_alloc)
+ {
+ cr->mpb->dbuf_alloc = nr;
+ srenew(cr->mpb->dbuf, cr->mpb->dbuf_alloc);
+ }
+ if (cr->nc.bUse)
+ {
+ /* Use two step summing */
+ MPI_Allreduce(r, cr->mpb->dbuf, nr, MPI_DOUBLE, MPI_SUM, cr->nc.comm_intra);
+ if (cr->nc.rank_intra == 0)
+ {
+ /* Sum with the buffers reversed */
+ MPI_Allreduce(cr->mpb->dbuf, r, nr, MPI_DOUBLE, MPI_SUM,
+ cr->nc.comm_inter);
+ }
+ MPI_Bcast(r, nr, MPI_DOUBLE, 0, cr->nc.comm_intra);
+ }
+ else
+ {
+ MPI_Allreduce(r, cr->mpb->dbuf, nr, MPI_DOUBLE, MPI_SUM,
+ cr->mpi_comm_mygroup);
+ for (i = 0; i < nr; i++)
+ {
+ r[i] = cr->mpb->dbuf[i];
+ }
+ }
+#endif
+#endif
+}
+
+void gmx_sumf(int nr, float r[], const t_commrec *cr)
+{
+#ifndef GMX_MPI
+ gmx_call("gmx_sumf");
+#else
+#if defined(MPI_IN_PLACE_EXISTS) || defined(GMX_THREAD_MPI)
+ if (cr->nc.bUse)
+ {
+ /* Use two step summing. */
+ if (cr->nc.rank_intra == 0)
+ {
+ MPI_Reduce(MPI_IN_PLACE, r, nr, MPI_FLOAT, MPI_SUM, 0,
+ cr->nc.comm_intra);
+ /* Sum the roots of the internal (intra) buffers */
+ MPI_Allreduce(MPI_IN_PLACE, r, nr, MPI_FLOAT, MPI_SUM,
+ cr->nc.comm_inter);
+ }
+ else
+ {
+ /* This is here because of the silly MPI specification
+ that MPI_IN_PLACE should be put in sendbuf instead of recvbuf */
+ MPI_Reduce(r, NULL, nr, MPI_FLOAT, MPI_SUM, 0, cr->nc.comm_intra);
+ }
+ MPI_Bcast(r, nr, MPI_FLOAT, 0, cr->nc.comm_intra);
+ }
+ else
+ {
+ MPI_Allreduce(MPI_IN_PLACE, r, nr, MPI_FLOAT, MPI_SUM, cr->mpi_comm_mygroup);
+ }
+#else
+ int i;
+
+ if (nr > cr->mpb->fbuf_alloc)
+ {
+ cr->mpb->fbuf_alloc = nr;
+ srenew(cr->mpb->fbuf, cr->mpb->fbuf_alloc);
+ }
+ if (cr->nc.bUse)
+ {
+ /* Use two step summing */
+ MPI_Allreduce(r, cr->mpb->fbuf, nr, MPI_FLOAT, MPI_SUM, cr->nc.comm_intra);
+ if (cr->nc.rank_intra == 0)
+ {
+ /* Sum with the buffers reversed */
+ MPI_Allreduce(cr->mpb->fbuf, r, nr, MPI_FLOAT, MPI_SUM,
+ cr->nc.comm_inter);
+ }
+ MPI_Bcast(r, nr, MPI_FLOAT, 0, cr->nc.comm_intra);
+ }
+ else
+ {
+ MPI_Allreduce(r, cr->mpb->fbuf, nr, MPI_FLOAT, MPI_SUM,
+ cr->mpi_comm_mygroup);
+ for (i = 0; i < nr; i++)
+ {
+ r[i] = cr->mpb->fbuf[i];
+ }
+ }
+#endif
+#endif
+}
+
+void gmx_sumi(int nr, int r[], const t_commrec *cr)
+{
+#ifndef GMX_MPI
+ gmx_call("gmx_sumi");
+#else
+#if defined(MPI_IN_PLACE_EXISTS) || defined(GMX_THREAD_MPI)
+ if (cr->nc.bUse)
+ {
+ /* Use two step summing */
+ if (cr->nc.rank_intra == 0)
+ {
+ MPI_Reduce(MPI_IN_PLACE, r, nr, MPI_INT, MPI_SUM, 0, cr->nc.comm_intra);
+ /* Sum with the buffers reversed */
+ MPI_Allreduce(MPI_IN_PLACE, r, nr, MPI_INT, MPI_SUM, cr->nc.comm_inter);
+ }
+ else
+ {
+ /* This is here because of the silly MPI specification
+ that MPI_IN_PLACE should be put in sendbuf instead of recvbuf */
+ MPI_Reduce(r, NULL, nr, MPI_INT, MPI_SUM, 0, cr->nc.comm_intra);
+ }
+ MPI_Bcast(r, nr, MPI_INT, 0, cr->nc.comm_intra);
+ }
+ else
+ {
+ MPI_Allreduce(MPI_IN_PLACE, r, nr, MPI_INT, MPI_SUM, cr->mpi_comm_mygroup);
+ }
+#else
+ int i;
+
+ if (nr > cr->mpb->ibuf_alloc)
+ {
+ cr->mpb->ibuf_alloc = nr;
+ srenew(cr->mpb->ibuf, cr->mpb->ibuf_alloc);
+ }
+ if (cr->nc.bUse)
+ {
+ /* Use two step summing */
+ MPI_Allreduce(r, cr->mpb->ibuf, nr, MPI_INT, MPI_SUM, cr->nc.comm_intra);
+ if (cr->nc.rank_intra == 0)
+ {
+ /* Sum with the buffers reversed */
+ MPI_Allreduce(cr->mpb->ibuf, r, nr, MPI_INT, MPI_SUM, cr->nc.comm_inter);
+ }
+ MPI_Bcast(r, nr, MPI_INT, 0, cr->nc.comm_intra);
+ }
+ else
+ {
+ MPI_Allreduce(r, cr->mpb->ibuf, nr, MPI_INT, MPI_SUM, cr->mpi_comm_mygroup);
+ for (i = 0; i < nr; i++)
+ {
+ r[i] = cr->mpb->ibuf[i];
+ }
+ }
+#endif
+#endif
+}
+
+void gmx_sumli(int nr, gmx_large_int_t r[], const t_commrec *cr)
+{
+#ifndef GMX_MPI
+ gmx_call("gmx_sumli");
+#else
+#if defined(MPI_IN_PLACE_EXISTS) || defined(GMX_THREAD_MPI)
+ if (cr->nc.bUse)
+ {
+ /* Use two step summing */
+ if (cr->nc.rank_intra == 0)
+ {
+ MPI_Reduce(MPI_IN_PLACE, r, nr, GMX_MPI_LARGE_INT, MPI_SUM, 0,
+ cr->nc.comm_intra);
+ /* Sum with the buffers reversed */
+ MPI_Allreduce(MPI_IN_PLACE, r, nr, GMX_MPI_LARGE_INT, MPI_SUM,
+ cr->nc.comm_inter);
+ }
+ else
+ {
+ /* This is here because of the silly MPI specification
+ that MPI_IN_PLACE should be put in sendbuf instead of recvbuf */
+ MPI_Reduce(r, NULL, nr, GMX_MPI_LARGE_INT, MPI_SUM, 0, cr->nc.comm_intra);
+ }
+ MPI_Bcast(r, nr, GMX_MPI_LARGE_INT, 0, cr->nc.comm_intra);
+ }
+ else
+ {
+ MPI_Allreduce(MPI_IN_PLACE, r, nr, GMX_MPI_LARGE_INT, MPI_SUM, cr->mpi_comm_mygroup);
+ }
+#else
+ int i;
+
+ if (nr > cr->mpb->libuf_alloc)
+ {
+ cr->mpb->libuf_alloc = nr;
+ srenew(cr->mpb->libuf, cr->mpb->libuf_alloc);
+ }
+ if (cr->nc.bUse)
+ {
+ /* Use two step summing */
+ MPI_Allreduce(r, cr->mpb->libuf, nr, GMX_MPI_LARGE_INT, MPI_SUM,
+ cr->nc.comm_intra);
+ if (cr->nc.rank_intra == 0)
+ {
+ /* Sum with the buffers reversed */
+ MPI_Allreduce(cr->mpb->libuf, r, nr, GMX_MPI_LARGE_INT, MPI_SUM,
+ cr->nc.comm_inter);
+ }
+ MPI_Bcast(r, nr, GMX_MPI_LARGE_INT, 0, cr->nc.comm_intra);
+ }
+ else
+ {
+ MPI_Allreduce(r, cr->mpb->libuf, nr, GMX_MPI_LARGE_INT, MPI_SUM,
+ cr->mpi_comm_mygroup);
+ for (i = 0; i < nr; i++)
+ {
+ r[i] = cr->mpb->libuf[i];
+ }
+ }
+#endif
+#endif
+}
+
+
+
+#ifdef GMX_MPI
+void gmx_sumd_comm(int nr, double r[], MPI_Comm mpi_comm)
+{
+#if defined(MPI_IN_PLACE_EXISTS) || defined(GMX_THREAD_MPI)
+ MPI_Allreduce(MPI_IN_PLACE, r, nr, MPI_DOUBLE, MPI_SUM, mpi_comm);
+#else
+ /* this function is only used in code that is not performance critical,
+ (during setup, when comm_rec is not the appropriate communication
+ structure), so this isn't as bad as it looks. */
+ double *buf;
+ int i;
+
+ snew(buf, nr);
+ MPI_Allreduce(r, buf, nr, MPI_DOUBLE, MPI_SUM, mpi_comm);
+ for (i = 0; i < nr; i++)
+ {
+ r[i] = buf[i];
+ }
+ sfree(buf);
+#endif
+}
+#endif
+
+#ifdef GMX_MPI
+void gmx_sumf_comm(int nr, float r[], MPI_Comm mpi_comm)
+{
+#if defined(MPI_IN_PLACE_EXISTS) || defined(GMX_THREAD_MPI)
+ MPI_Allreduce(MPI_IN_PLACE, r, nr, MPI_FLOAT, MPI_SUM, mpi_comm);
+#else
+ /* this function is only used in code that is not performance critical,
+ (during setup, when comm_rec is not the appropriate communication
+ structure), so this isn't as bad as it looks. */
+ float *buf;
+ int i;
+
+ snew(buf, nr);
+ MPI_Allreduce(r, buf, nr, MPI_FLOAT, MPI_SUM, mpi_comm);
+ for (i = 0; i < nr; i++)
+ {
+ r[i] = buf[i];
+ }
+ sfree(buf);
+#endif
+}
+#endif
+
+void gmx_sumd_sim(int nr, double r[], const gmx_multisim_t *ms)
+{
+#ifndef GMX_MPI
+ gmx_call("gmx_sumd_sim");
+#else
+ gmx_sumd_comm(nr, r, ms->mpi_comm_masters);
+#endif
+}
+
+void gmx_sumf_sim(int nr, float r[], const gmx_multisim_t *ms)
+{
+#ifndef GMX_MPI
+ gmx_call("gmx_sumf_sim");
+#else
+ gmx_sumf_comm(nr, r, ms->mpi_comm_masters);
+#endif
+}
+
+void gmx_sumi_sim(int nr, int r[], const gmx_multisim_t *ms)
+{
+#ifndef GMX_MPI
+ gmx_call("gmx_sumi_sim");
+#else
+#if defined(MPI_IN_PLACE_EXISTS) || defined(GMX_THREAD_MPI)
+ MPI_Allreduce(MPI_IN_PLACE, r, nr, MPI_INT, MPI_SUM, ms->mpi_comm_masters);
+#else
+ /* this is thread-unsafe, but it will do for now: */
+ int i;
+
+ if (nr > ms->mpb->ibuf_alloc)
+ {
+ ms->mpb->ibuf_alloc = nr;
+ srenew(ms->mpb->ibuf, ms->mpb->ibuf_alloc);
+ }
+ MPI_Allreduce(r, ms->mpb->ibuf, nr, MPI_INT, MPI_SUM, ms->mpi_comm_masters);
+ for (i = 0; i < nr; i++)
+ {
+ r[i] = ms->mpb->ibuf[i];
+ }
+#endif
+#endif
+}
+
+void gmx_sumli_sim(int nr, gmx_large_int_t r[], const gmx_multisim_t *ms)
+{
+#ifndef GMX_MPI
+ gmx_call("gmx_sumli_sim");
+#else
+#if defined(MPI_IN_PLACE_EXISTS) || defined(GMX_THREAD_MPI)
+ MPI_Allreduce(MPI_IN_PLACE, r, nr, GMX_MPI_LARGE_INT, MPI_SUM,
+ ms->mpi_comm_masters);
+#else
+ /* this is thread-unsafe, but it will do for now: */
+ int i;
+
+ if (nr > ms->mpb->libuf_alloc)
+ {
+ ms->mpb->libuf_alloc = nr;
+ srenew(ms->mpb->libuf, ms->mpb->libuf_alloc);
+ }
+ MPI_Allreduce(r, ms->mpb->libuf, nr, GMX_MPI_LARGE_INT, MPI_SUM,
+ ms->mpi_comm_masters);
+ for (i = 0; i < nr; i++)
+ {
+ r[i] = ms->mpb->libuf[i];
+ }
+#endif
+#endif
+}
--- /dev/null
- FscalC[i] = Vcoul[i]*rpinvC;
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * VERSION 3.2.0
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2004, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * GROningen Mixture of Alchemy and Childrens' Stories
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <math.h>
+
+#include "vec.h"
+#include "typedefs.h"
+#include "nonbonded.h"
+#include "nb_kernel.h"
+#include "nrnb.h"
+#include "nb_free_energy.h"
+
+void
+gmx_nb_free_energy_kernel(const t_nblist * gmx_restrict nlist,
+ rvec * gmx_restrict xx,
+ rvec * gmx_restrict ff,
+ t_forcerec * gmx_restrict fr,
+ const t_mdatoms * gmx_restrict mdatoms,
+ nb_kernel_data_t * gmx_restrict kernel_data,
+ t_nrnb * gmx_restrict nrnb)
+{
+
+#define STATE_A 0
+#define STATE_B 1
+#define NSTATES 2
+ int i, j, n, ii, is3, ii3, k, nj0, nj1, jnr, j3, ggid;
+ real shX, shY, shZ;
+ real Fscal, FscalC[NSTATES], FscalV[NSTATES], tx, ty, tz;
+ real Vcoul[NSTATES], Vvdw[NSTATES];
+ real rinv6, r, rt, rtC, rtV;
+ real iqA, iqB;
+ real qq[NSTATES], vctot, krsq;
+ int ntiA, ntiB, tj[NSTATES];
+ real Vvdw6, Vvdw12, vvtot;
+ real ix, iy, iz, fix, fiy, fiz;
+ real dx, dy, dz, rsq, rinv;
+ real c6[NSTATES], c12[NSTATES];
+ real LFC[NSTATES], LFV[NSTATES], DLF[NSTATES];
+ double dvdl_coul, dvdl_vdw;
+ real lfac_coul[NSTATES], dlfac_coul[NSTATES], lfac_vdw[NSTATES], dlfac_vdw[NSTATES];
+ real sigma6[NSTATES], alpha_vdw_eff, alpha_coul_eff, sigma2_def, sigma2_min;
+ real rp, rpm2, rC, rV, rinvC, rpinvC, rinvV, rpinvV;
+ real sigma2[NSTATES], sigma_pow[NSTATES], sigma_powm2[NSTATES], rs, rs2;
+ int do_tab, tab_elemsize;
+ int n0, n1C, n1V, nnn;
+ real Y, F, G, H, Fp, Geps, Heps2, epsC, eps2C, epsV, eps2V, VV, FF;
+ int icoul, ivdw;
+ int nri;
+ const int * iinr;
+ const int * jindex;
+ const int * jjnr;
+ const int * shift;
+ const int * gid;
+ const int * typeA;
+ const int * typeB;
+ int ntype;
+ const real * shiftvec;
+ real dvdl_part;
+ real * fshift;
+ real tabscale;
+ const real * VFtab;
+ const real * x;
+ real * f;
+ real facel, krf, crf;
+ const real * chargeA;
+ const real * chargeB;
+ real sigma6_min, sigma6_def, lam_power, sc_power, sc_r_power;
+ real alpha_coul, alpha_vdw, lambda_coul, lambda_vdw, ewc;
+ const real * nbfp;
+ real * dvdl;
+ real * Vv;
+ real * Vc;
+ gmx_bool bDoForces;
+ real rcoulomb, rvdw, sh_invrc6;
+ gmx_bool bExactElecCutoff, bExactVdwCutoff;
+ real rcutoff, rcutoff2, rswitch, d, d2, swV3, swV4, swV5, swF2, swF3, swF4, sw, dsw, rinvcorr;
+ const real * tab_ewald_F;
+ const real * tab_ewald_V;
+ real tab_ewald_scale, tab_ewald_halfsp;
+
+ x = xx[0];
+ f = ff[0];
+
+ fshift = fr->fshift[0];
+
+ nri = nlist->nri;
+ iinr = nlist->iinr;
+ jindex = nlist->jindex;
+ jjnr = nlist->jjnr;
+ icoul = nlist->ielec;
+ ivdw = nlist->ivdw;
+ shift = nlist->shift;
+ gid = nlist->gid;
+
+ shiftvec = fr->shift_vec[0];
+ chargeA = mdatoms->chargeA;
+ chargeB = mdatoms->chargeB;
+ facel = fr->epsfac;
+ krf = fr->k_rf;
+ crf = fr->c_rf;
+ ewc = fr->ewaldcoeff;
+ Vc = kernel_data->energygrp_elec;
+ typeA = mdatoms->typeA;
+ typeB = mdatoms->typeB;
+ ntype = fr->ntype;
+ nbfp = fr->nbfp;
+ Vv = kernel_data->energygrp_vdw;
+ tabscale = kernel_data->table_elec_vdw->scale;
+ VFtab = kernel_data->table_elec_vdw->data;
+ lambda_coul = kernel_data->lambda[efptCOUL];
+ lambda_vdw = kernel_data->lambda[efptVDW];
+ dvdl = kernel_data->dvdl;
+ alpha_coul = fr->sc_alphacoul;
+ alpha_vdw = fr->sc_alphavdw;
+ lam_power = fr->sc_power;
+ sc_r_power = fr->sc_r_power;
+ sigma6_def = fr->sc_sigma6_def;
+ sigma6_min = fr->sc_sigma6_min;
+ bDoForces = kernel_data->flags & GMX_NONBONDED_DO_FORCE;
+
+ rcoulomb = fr->rcoulomb;
+ rvdw = fr->rvdw;
+ sh_invrc6 = fr->ic->sh_invrc6;
+
+ /* Ewald (PME) reciprocal force and energy quadratic spline tables */
+ tab_ewald_F = fr->ic->tabq_coul_F;
+ tab_ewald_V = fr->ic->tabq_coul_V;
+ tab_ewald_scale = fr->ic->tabq_scale;
+ tab_ewald_halfsp = 0.5/tab_ewald_scale;
+
+ if (fr->coulomb_modifier == eintmodPOTSWITCH || fr->vdw_modifier == eintmodPOTSWITCH)
+ {
+ rcutoff = (fr->coulomb_modifier == eintmodPOTSWITCH) ? fr->rcoulomb : fr->rvdw;
+ rcutoff2 = rcutoff*rcutoff;
+ rswitch = (fr->coulomb_modifier == eintmodPOTSWITCH) ? fr->rcoulomb_switch : fr->rvdw_switch;
+ d = rcutoff-rswitch;
+ swV3 = -10.0/(d*d*d);
+ swV4 = 15.0/(d*d*d*d);
+ swV5 = -6.0/(d*d*d*d*d);
+ swF2 = -30.0/(d*d*d);
+ swF3 = 60.0/(d*d*d*d);
+ swF4 = -30.0/(d*d*d*d*d);
+ }
+ else
+ {
+ /* Stupid compilers dont realize these variables will not be used */
+ rswitch = 0.0;
+ swV3 = 0.0;
+ swV4 = 0.0;
+ swV5 = 0.0;
+ swF2 = 0.0;
+ swF3 = 0.0;
+ swF4 = 0.0;
+ }
+
+ bExactElecCutoff = (fr->coulomb_modifier != eintmodNONE) || fr->eeltype == eelRF_ZERO;
+ bExactVdwCutoff = (fr->vdw_modifier != eintmodNONE);
+
+ /* fix compiler warnings */
+ nj1 = 0;
+ n1C = n1V = 0;
+ epsC = epsV = 0;
+ eps2C = eps2V = 0;
+
+ dvdl_coul = 0;
+ dvdl_vdw = 0;
+
+ /* Lambda factor for state A, 1-lambda*/
+ LFC[STATE_A] = 1.0 - lambda_coul;
+ LFV[STATE_A] = 1.0 - lambda_vdw;
+
+ /* Lambda factor for state B, lambda*/
+ LFC[STATE_B] = lambda_coul;
+ LFV[STATE_B] = lambda_vdw;
+
+ /*derivative of the lambda factor for state A and B */
+ DLF[STATE_A] = -1;
+ DLF[STATE_B] = 1;
+
+ for (i = 0; i < NSTATES; i++)
+ {
+ lfac_coul[i] = (lam_power == 2 ? (1-LFC[i])*(1-LFC[i]) : (1-LFC[i]));
+ dlfac_coul[i] = DLF[i]*lam_power/sc_r_power*(lam_power == 2 ? (1-LFC[i]) : 1);
+ lfac_vdw[i] = (lam_power == 2 ? (1-LFV[i])*(1-LFV[i]) : (1-LFV[i]));
+ dlfac_vdw[i] = DLF[i]*lam_power/sc_r_power*(lam_power == 2 ? (1-LFV[i]) : 1);
+ }
+ /* precalculate */
+ sigma2_def = pow(sigma6_def, 1.0/3.0);
+ sigma2_min = pow(sigma6_min, 1.0/3.0);
+
+ /* Ewald (not PME) table is special (icoul==enbcoulFEWALD) */
+
+ do_tab = (icoul == GMX_NBKERNEL_ELEC_CUBICSPLINETABLE ||
+ ivdw == GMX_NBKERNEL_VDW_CUBICSPLINETABLE);
+
+ /* we always use the combined table here */
+ tab_elemsize = 12;
+
+ for (n = 0; (n < nri); n++)
+ {
+ is3 = 3*shift[n];
+ shX = shiftvec[is3];
+ shY = shiftvec[is3+1];
+ shZ = shiftvec[is3+2];
+ nj0 = jindex[n];
+ nj1 = jindex[n+1];
+ ii = iinr[n];
+ ii3 = 3*ii;
+ ix = shX + x[ii3+0];
+ iy = shY + x[ii3+1];
+ iz = shZ + x[ii3+2];
+ iqA = facel*chargeA[ii];
+ iqB = facel*chargeB[ii];
+ ntiA = 2*ntype*typeA[ii];
+ ntiB = 2*ntype*typeB[ii];
+ vctot = 0;
+ vvtot = 0;
+ fix = 0;
+ fiy = 0;
+ fiz = 0;
+
+ for (k = nj0; (k < nj1); k++)
+ {
+ jnr = jjnr[k];
+ j3 = 3*jnr;
+ dx = ix - x[j3];
+ dy = iy - x[j3+1];
+ dz = iz - x[j3+2];
+ rsq = dx*dx+dy*dy+dz*dz;
+ rinv = gmx_invsqrt(rsq);
+ r = rsq*rinv;
+ if (sc_r_power == 6.0)
+ {
+ rpm2 = rsq*rsq; /* r4 */
+ rp = rpm2*rsq; /* r6 */
+ }
+ else if (sc_r_power == 48.0)
+ {
+ rp = rsq*rsq*rsq; /* r6 */
+ rp = rp*rp; /* r12 */
+ rp = rp*rp; /* r24 */
+ rp = rp*rp; /* r48 */
+ rpm2 = rp/rsq; /* r46 */
+ }
+ else
+ {
+ rp = pow(r, sc_r_power); /* not currently supported as input, but can handle it */
+ rpm2 = rp/rsq;
+ }
+
+ tj[STATE_A] = ntiA+2*typeA[jnr];
+ tj[STATE_B] = ntiB+2*typeB[jnr];
+ qq[STATE_A] = iqA*chargeA[jnr];
+ qq[STATE_B] = iqB*chargeB[jnr];
+
+ for (i = 0; i < NSTATES; i++)
+ {
+
+ c6[i] = nbfp[tj[i]];
+ c12[i] = nbfp[tj[i]+1];
+ if ((c6[i] > 0) && (c12[i] > 0))
+ {
+ /* c12 is stored scaled with 12.0 and c6 is scaled with 6.0 - correct for this */
+ sigma6[i] = 0.5*c12[i]/c6[i];
+ sigma2[i] = pow(sigma6[i], 1.0/3.0);
+ /* should be able to get rid of this ^^^ internal pow call eventually. Will require agreement on
+ what data to store externally. Can't be fixed without larger scale changes, so not 4.6 */
+ if (sigma6[i] < sigma6_min) /* for disappearing coul and vdw with soft core at the same time */
+ {
+ sigma6[i] = sigma6_min;
+ sigma2[i] = sigma2_min;
+ }
+ }
+ else
+ {
+ sigma6[i] = sigma6_def;
+ sigma2[i] = sigma2_def;
+ }
+ if (sc_r_power == 6.0)
+ {
+ sigma_pow[i] = sigma6[i];
+ sigma_powm2[i] = sigma6[i]/sigma2[i];
+ }
+ else if (sc_r_power == 48.0)
+ {
+ sigma_pow[i] = sigma6[i]*sigma6[i]; /* sigma^12 */
+ sigma_pow[i] = sigma_pow[i]*sigma_pow[i]; /* sigma^24 */
+ sigma_pow[i] = sigma_pow[i]*sigma_pow[i]; /* sigma^48 */
+ sigma_powm2[i] = sigma_pow[i]/sigma2[i];
+ }
+ else
+ { /* not really supported as input, but in here for testing the general case*/
+ sigma_pow[i] = pow(sigma2[i], sc_r_power/2);
+ sigma_powm2[i] = sigma_pow[i]/(sigma2[i]);
+ }
+ }
+
+ /* only use softcore if one of the states has a zero endstate - softcore is for avoiding infinities!*/
+ if ((c12[STATE_A] > 0) && (c12[STATE_B] > 0))
+ {
+ alpha_vdw_eff = 0;
+ alpha_coul_eff = 0;
+ }
+ else
+ {
+ alpha_vdw_eff = alpha_vdw;
+ alpha_coul_eff = alpha_coul;
+ }
+
+ for (i = 0; i < NSTATES; i++)
+ {
+ FscalC[i] = 0;
+ FscalV[i] = 0;
+ Vcoul[i] = 0;
+ Vvdw[i] = 0;
+
+ /* Only spend time on A or B state if it is non-zero */
+ if ( (qq[i] != 0) || (c6[i] != 0) || (c12[i] != 0) )
+ {
+
+ /* this section has to be inside the loop becaue of the dependence on sigma_pow */
+ rpinvC = 1.0/(alpha_coul_eff*lfac_coul[i]*sigma_pow[i]+rp);
+ rinvC = pow(rpinvC, 1.0/sc_r_power);
+ rC = 1.0/rinvC;
+
+ rpinvV = 1.0/(alpha_vdw_eff*lfac_vdw[i]*sigma_pow[i]+rp);
+ rinvV = pow(rpinvV, 1.0/sc_r_power);
+ rV = 1.0/rinvV;
+
+ if (do_tab)
+ {
+ rtC = rC*tabscale;
+ n0 = rtC;
+ epsC = rtC-n0;
+ eps2C = epsC*epsC;
+ n1C = tab_elemsize*n0;
+
+ rtV = rV*tabscale;
+ n0 = rtV;
+ epsV = rtV-n0;
+ eps2V = epsV*epsV;
+ n1V = tab_elemsize*n0;
+ }
+
+ /* With Ewald and soft-core we should put the cut-off on r,
+ * not on the soft-cored rC, as the real-space and
+ * reciprocal space contributions should (almost) cancel.
+ */
+ if (qq[i] != 0 &&
+ !(bExactElecCutoff &&
+ ((icoul != GMX_NBKERNEL_ELEC_EWALD && rC >= rcoulomb) ||
+ (icoul == GMX_NBKERNEL_ELEC_EWALD && r >= rcoulomb))))
+ {
+ switch (icoul)
+ {
+ case GMX_NBKERNEL_ELEC_COULOMB:
+ case GMX_NBKERNEL_ELEC_EWALD:
+ /* simple cutoff (yes, ewald is done all on direct space for free energy) */
+ Vcoul[i] = qq[i]*rinvC;
- Vcoul[i] = qq[i]*(rinvC+krf*rC*rC-crf);
- FscalC[i] = qq[i]*(rinvC*rpinvC-2.0*krf);
++ FscalC[i] = Vcoul[i];
+ break;
+
+ case GMX_NBKERNEL_ELEC_REACTIONFIELD:
+ /* reaction-field */
- FscalC[i] = -qq[i]*tabscale*FF*rC*rpinvC;
++ Vcoul[i] = qq[i]*(rinvC + krf*rC*rC-crf);
++ FscalC[i] = qq[i]*(rinvC - 2.0*krf*rC*rC);
+ break;
+
+ case GMX_NBKERNEL_ELEC_CUBICSPLINETABLE:
+ /* non-Ewald tabulated coulomb */
+ nnn = n1C;
+ Y = VFtab[nnn];
+ F = VFtab[nnn+1];
+ Geps = epsC*VFtab[nnn+2];
+ Heps2 = eps2C*VFtab[nnn+3];
+ Fp = F+Geps+Heps2;
+ VV = Y+epsC*Fp;
+ FF = Fp+Geps+2.0*Heps2;
+ Vcoul[i] = qq[i]*VV;
- Vvdw[i] = Vvdw12*(1.0/12.0)-Vvdw6*(1.0/6.0);
++ FscalC[i] = -qq[i]*tabscale*FF*rC;
+ break;
+
+ case GMX_NBKERNEL_ELEC_GENERALIZEDBORN:
+ gmx_fatal(FARGS, "Free energy and GB not implemented.\n");
+ break;
+
+ case GMX_NBKERNEL_ELEC_NONE:
+ FscalC[i] = 0.0;
+ Vcoul[i] = 0.0;
+ break;
+
+ default:
+ gmx_incons("Invalid icoul in free energy kernel");
+ break;
+ }
+
+ if (fr->coulomb_modifier == eintmodPOTSWITCH)
+ {
+ d = rC-rswitch;
+ d = (d > 0.0) ? d : 0.0;
+ d2 = d*d;
+ sw = 1.0+d2*d*(swV3+d*(swV4+d*swV5));
+ dsw = d2*(swF2+d*(swF3+d*swF4));
+
+ Vcoul[i] *= sw;
+ FscalC[i] = FscalC[i]*sw + Vcoul[i]*dsw;
+ }
+ }
+
+ if ((c6[i] != 0 || c12[i] != 0) &&
+ !(bExactVdwCutoff && rV >= rvdw))
+ {
+ switch (ivdw)
+ {
+ case GMX_NBKERNEL_VDW_LENNARDJONES:
+ /* cutoff LJ */
+ if (sc_r_power == 6.0)
+ {
+ rinv6 = rpinvV;
+ }
+ else
+ {
+ rinv6 = pow(rinvV, 6.0);
+ }
+ Vvdw6 = c6[i]*rinv6;
+ Vvdw12 = c12[i]*rinv6*rinv6;
+ if (fr->vdw_modifier == eintmodPOTSHIFT)
+ {
+ Vvdw[i] = ( (Vvdw12-c12[i]*sh_invrc6*sh_invrc6)*(1.0/12.0)
+ -(Vvdw6-c6[i]*sh_invrc6)*(1.0/6.0));
+ }
+ else
+ {
- FscalV[i] = (Vvdw12-Vvdw6)*rpinvV;
++ Vvdw[i] = Vvdw12*(1.0/12.0) - Vvdw6*(1.0/6.0);
+ }
- FscalV[i] -= c6[i]*tabscale*FF*rV*rpinvV;
++ FscalV[i] = Vvdw12 - Vvdw6;
+ break;
+
+ case GMX_NBKERNEL_VDW_BUCKINGHAM:
+ gmx_fatal(FARGS, "Buckingham free energy not supported.");
+ break;
+
+ case GMX_NBKERNEL_VDW_CUBICSPLINETABLE:
+ /* Table LJ */
+ nnn = n1V+4;
+ /* dispersion */
+ Y = VFtab[nnn];
+ F = VFtab[nnn+1];
+ Geps = epsV*VFtab[nnn+2];
+ Heps2 = eps2V*VFtab[nnn+3];
+ Fp = F+Geps+Heps2;
+ VV = Y+epsV*Fp;
+ FF = Fp+Geps+2.0*Heps2;
+ Vvdw[i] += c6[i]*VV;
- FscalV[i] -= c12[i]*tabscale*FF*rV*rpinvV;
++ FscalV[i] -= c6[i]*tabscale*FF*rV;
+
+ /* repulsion */
+ Y = VFtab[nnn+4];
+ F = VFtab[nnn+5];
+ Geps = epsV*VFtab[nnn+6];
+ Heps2 = eps2V*VFtab[nnn+7];
+ Fp = F+Geps+Heps2;
+ VV = Y+epsV*Fp;
+ FF = Fp+Geps+2.0*Heps2;
+ Vvdw[i] += c12[i]*VV;
++ FscalV[i] -= c12[i]*tabscale*FF*rV;
+ break;
+
+ case GMX_NBKERNEL_VDW_NONE:
+ Vvdw[i] = 0.0;
+ FscalV[i] = 0.0;
+ break;
+
+ default:
+ gmx_incons("Invalid ivdw in free energy kernel");
+ break;
+ }
+
+ if (fr->vdw_modifier == eintmodPOTSWITCH)
+ {
+ d = rV-rswitch;
+ d = (d > 0.0) ? d : 0.0;
+ d2 = d*d;
+ sw = 1.0+d2*d*(swV3+d*(swV4+d*swV5));
+ dsw = d2*(swF2+d*(swF3+d*swF4));
+
+ Vvdw[i] *= sw;
+ FscalV[i] = FscalV[i]*sw + Vvdw[i]*dsw;
+
+ FscalV[i] = (rV < rvdw) ? FscalV[i] : 0.0;
+ Vvdw[i] = (rV < rvdw) ? Vvdw[i] : 0.0;
+ }
+ }
++
++ /* FscalC (and FscalV) now contain: dV/drC * rC
++ * Now we multiply by rC^-p, so it will be: dV/drC * rC^1-p
++ * Further down we first multiply by r^p-2 and then by
++ * the vector r, which in total gives: dV/drC * (r/rC)^1-p
++ */
++ FscalC[i] *= rpinvC;
++ FscalV[i] *= rpinvV;
+ }
+ }
+
+ Fscal = 0;
+
+ if (icoul == GMX_NBKERNEL_ELEC_EWALD &&
+ !(bExactElecCutoff && r >= rcoulomb))
+ {
+ /* Because we compute the soft-core normally,
+ * we have to remove the Ewald short range portion.
+ * Done outside of the states loop because this part
+ * doesn't depend on the scaled R.
+ */
+ real rs, frac, f_lr;
+ int ri;
+
+ rs = rsq*rinv*tab_ewald_scale;
+ ri = (int)rs;
+ frac = rs - ri;
+ f_lr = (1 - frac)*tab_ewald_F[ri] + frac*tab_ewald_F[ri+1];
+ FF = f_lr*rinv;
+ VV = tab_ewald_V[ri] - tab_ewald_halfsp*frac*(tab_ewald_F[ri] + f_lr);
+
+ for (i = 0; i < NSTATES; i++)
+ {
+ vctot -= LFC[i]*qq[i]*VV;
+ Fscal -= LFC[i]*qq[i]*FF;
+ dvdl_coul -= (DLF[i]*qq[i])*VV;
+ }
+ }
+
+ /* Assemble A and B states */
+ for (i = 0; i < NSTATES; i++)
+ {
+ vctot += LFC[i]*Vcoul[i];
+ vvtot += LFV[i]*Vvdw[i];
+
+ Fscal += LFC[i]*FscalC[i]*rpm2;
+ Fscal += LFV[i]*FscalV[i]*rpm2;
+
+ dvdl_coul += Vcoul[i]*DLF[i] + LFC[i]*alpha_coul_eff*dlfac_coul[i]*FscalC[i]*sigma_pow[i];
+ dvdl_vdw += Vvdw[i]*DLF[i] + LFV[i]*alpha_vdw_eff*dlfac_vdw[i]*FscalV[i]*sigma_pow[i];
+ }
+
+ if (bDoForces)
+ {
+ tx = Fscal*dx;
+ ty = Fscal*dy;
+ tz = Fscal*dz;
+ fix = fix + tx;
+ fiy = fiy + ty;
+ fiz = fiz + tz;
+ f[j3] = f[j3] - tx;
+ f[j3+1] = f[j3+1] - ty;
+ f[j3+2] = f[j3+2] - tz;
+ }
+ }
+
+ if (bDoForces)
+ {
+ f[ii3] = f[ii3] + fix;
+ f[ii3+1] = f[ii3+1] + fiy;
+ f[ii3+2] = f[ii3+2] + fiz;
+ fshift[is3] = fshift[is3] + fix;
+ fshift[is3+1] = fshift[is3+1] + fiy;
+ fshift[is3+2] = fshift[is3+2] + fiz;
+ }
+ ggid = gid[n];
+ Vc[ggid] = Vc[ggid] + vctot;
+ Vv[ggid] = Vv[ggid] + vvtot;
+ }
+
+ dvdl[efptCOUL] += dvdl_coul;
+ dvdl[efptVDW] += dvdl_vdw;
+
+ /* Estimate flops, average for free energy stuff:
+ * 12 flops per outer iteration
+ * 150 flops per inner iteration
+ */
+ inc_nrnb(nrnb, eNR_NBKERNEL_FREE_ENERGY, nlist->nri*12 + nlist->jindex[n]*150);
+}
+
+real
+nb_free_energy_evaluate_single(real r2, real sc_r_power, real alpha_coul, real alpha_vdw,
+ real tabscale, real *vftab,
+ real qqA, real c6A, real c12A, real qqB, real c6B, real c12B,
+ real LFC[2], real LFV[2], real DLF[2],
+ real lfac_coul[2], real lfac_vdw[2], real dlfac_coul[2], real dlfac_vdw[2],
+ real sigma6_def, real sigma6_min, real sigma2_def, real sigma2_min,
+ real *velectot, real *vvdwtot, real *dvdl)
+{
+ real r, rp, rpm2, rtab, eps, eps2, Y, F, Geps, Heps2, Fp, VV, FF, fscal;
+ real qq[2], c6[2], c12[2], sigma6[2], sigma2[2], sigma_pow[2], sigma_powm2[2];
+ real alpha_coul_eff, alpha_vdw_eff, dvdl_coul, dvdl_vdw;
+ real rpinv, r_coul, r_vdw, velecsum, vvdwsum;
+ real fscal_vdw[2], fscal_elec[2];
+ real velec[2], vvdw[2];
+ int i, ntab;
+
+ qq[0] = qqA;
+ qq[1] = qqB;
+ c6[0] = c6A;
+ c6[1] = c6B;
+ c12[0] = c12A;
+ c12[1] = c12B;
+
+ if (sc_r_power == 6.0)
+ {
+ rpm2 = r2*r2; /* r4 */
+ rp = rpm2*r2; /* r6 */
+ }
+ else if (sc_r_power == 48.0)
+ {
+ rp = r2*r2*r2; /* r6 */
+ rp = rp*rp; /* r12 */
+ rp = rp*rp; /* r24 */
+ rp = rp*rp; /* r48 */
+ rpm2 = rp/r2; /* r46 */
+ }
+ else
+ {
+ rp = pow(r2, 0.5*sc_r_power); /* not currently supported as input, but can handle it */
+ rpm2 = rp/r2;
+ }
+
+ /* Loop over state A(0) and B(1) */
+ for (i = 0; i < 2; i++)
+ {
+ if ((c6[i] > 0) && (c12[i] > 0))
+ {
+ /* The c6 & c12 coefficients now contain the constants 6.0 and 12.0, respectively.
+ * Correct for this by multiplying with (1/12.0)/(1/6.0)=6.0/12.0=0.5.
+ */
+ sigma6[i] = 0.5*c12[i]/c6[i];
+ sigma2[i] = pow(0.5*c12[i]/c6[i], 1.0/3.0);
+ /* should be able to get rid of this ^^^ internal pow call eventually. Will require agreement on
+ what data to store externally. Can't be fixed without larger scale changes, so not 4.6 */
+ if (sigma6[i] < sigma6_min) /* for disappearing coul and vdw with soft core at the same time */
+ {
+ sigma6[i] = sigma6_min;
+ sigma2[i] = sigma2_min;
+ }
+ }
+ else
+ {
+ sigma6[i] = sigma6_def;
+ sigma2[i] = sigma2_def;
+ }
+ if (sc_r_power == 6.0)
+ {
+ sigma_pow[i] = sigma6[i];
+ sigma_powm2[i] = sigma6[i]/sigma2[i];
+ }
+ else if (sc_r_power == 48.0)
+ {
+ sigma_pow[i] = sigma6[i]*sigma6[i]; /* sigma^12 */
+ sigma_pow[i] = sigma_pow[i]*sigma_pow[i]; /* sigma^24 */
+ sigma_pow[i] = sigma_pow[i]*sigma_pow[i]; /* sigma^48 */
+ sigma_powm2[i] = sigma_pow[i]/sigma2[i];
+ }
+ else
+ { /* not really supported as input, but in here for testing the general case*/
+ sigma_pow[i] = pow(sigma2[i], sc_r_power/2);
+ sigma_powm2[i] = sigma_pow[i]/(sigma2[i]);
+ }
+ }
+
+ /* only use softcore if one of the states has a zero endstate - softcore is for avoiding infinities!*/
+ if ((c12[0] > 0) && (c12[1] > 0))
+ {
+ alpha_vdw_eff = 0;
+ alpha_coul_eff = 0;
+ }
+ else
+ {
+ alpha_vdw_eff = alpha_vdw;
+ alpha_coul_eff = alpha_coul;
+ }
+
+ /* Loop over A and B states again */
+ for (i = 0; i < 2; i++)
+ {
+ fscal_elec[i] = 0;
+ fscal_vdw[i] = 0;
+ velec[i] = 0;
+ vvdw[i] = 0;
+
+ /* Only spend time on A or B state if it is non-zero */
+ if ( (qq[i] != 0) || (c6[i] != 0) || (c12[i] != 0) )
+ {
+ /* Coulomb */
+ rpinv = 1.0/(alpha_coul_eff*lfac_coul[i]*sigma_pow[i]+rp);
+ r_coul = pow(rpinv, -1.0/sc_r_power);
+
+ /* Electrostatics table lookup data */
+ rtab = r_coul*tabscale;
+ ntab = rtab;
+ eps = rtab-ntab;
+ eps2 = eps*eps;
+ ntab = 12*ntab;
+ /* Electrostatics */
+ Y = vftab[ntab];
+ F = vftab[ntab+1];
+ Geps = eps*vftab[ntab+2];
+ Heps2 = eps2*vftab[ntab+3];
+ Fp = F+Geps+Heps2;
+ VV = Y+eps*Fp;
+ FF = Fp+Geps+2.0*Heps2;
+ velec[i] = qq[i]*VV;
+ fscal_elec[i] = -qq[i]*FF*r_coul*rpinv*tabscale;
+
+ /* Vdw */
+ rpinv = 1.0/(alpha_vdw_eff*lfac_vdw[i]*sigma_pow[i]+rp);
+ r_vdw = pow(rpinv, -1.0/sc_r_power);
+ /* Vdw table lookup data */
+ rtab = r_vdw*tabscale;
+ ntab = rtab;
+ eps = rtab-ntab;
+ eps2 = eps*eps;
+ ntab = 12*ntab;
+ /* Dispersion */
+ Y = vftab[ntab+4];
+ F = vftab[ntab+5];
+ Geps = eps*vftab[ntab+6];
+ Heps2 = eps2*vftab[ntab+7];
+ Fp = F+Geps+Heps2;
+ VV = Y+eps*Fp;
+ FF = Fp+Geps+2.0*Heps2;
+ vvdw[i] = c6[i]*VV;
+ fscal_vdw[i] = -c6[i]*FF;
+
+ /* Repulsion */
+ Y = vftab[ntab+8];
+ F = vftab[ntab+9];
+ Geps = eps*vftab[ntab+10];
+ Heps2 = eps2*vftab[ntab+11];
+ Fp = F+Geps+Heps2;
+ VV = Y+eps*Fp;
+ FF = Fp+Geps+2.0*Heps2;
+ vvdw[i] += c12[i]*VV;
+ fscal_vdw[i] -= c12[i]*FF;
+ fscal_vdw[i] *= r_vdw*rpinv*tabscale;
+ }
+ }
+ /* Now we have velec[i], vvdw[i], and fscal[i] for both states */
+ /* Assemble A and B states */
+ velecsum = 0;
+ vvdwsum = 0;
+ dvdl_coul = 0;
+ dvdl_vdw = 0;
+ fscal = 0;
+ for (i = 0; i < 2; i++)
+ {
+ velecsum += LFC[i]*velec[i];
+ vvdwsum += LFV[i]*vvdw[i];
+
+ fscal += (LFC[i]*fscal_elec[i]+LFV[i]*fscal_vdw[i])*rpm2;
+
+ dvdl_coul += velec[i]*DLF[i] + LFC[i]*alpha_coul_eff*dlfac_coul[i]*fscal_elec[i]*sigma_pow[i];
+ dvdl_vdw += vvdw[i]*DLF[i] + LFV[i]*alpha_vdw_eff*dlfac_vdw[i]*fscal_vdw[i]*sigma_pow[i];
+ }
+
+ dvdl[efptCOUL] += dvdl_coul;
+ dvdl[efptVDW] += dvdl_vdw;
+
+ *velectot = velecsum;
+ *vvdwtot = vvdwsum;
+
+ return fscal;
+}
--- /dev/null
- init_df_history(&state->dfhist, nlambda, 0);
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * VERSION 3.2.0
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2004, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * GROningen Mixture of Alchemy and Childrens' Stories
+ */
+/* This file is completely threadsafe - keep it that way! */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include "smalloc.h"
+#include "symtab.h"
+#include "vec.h"
+#include "pbc.h"
+#include "macros.h"
+#include <string.h>
+
+#ifdef GMX_THREAD_MPI
+#include "thread_mpi.h"
+#endif
+
+/* The source code in this file should be thread-safe.
+ Please keep it that way. */
+
+
+
+static gmx_bool bOverAllocDD = FALSE;
+#ifdef GMX_THREAD_MPI
+static tMPI_Thread_mutex_t over_alloc_mutex = TMPI_THREAD_MUTEX_INITIALIZER;
+#endif
+
+
+void set_over_alloc_dd(gmx_bool set)
+{
+#ifdef GMX_THREAD_MPI
+ tMPI_Thread_mutex_lock(&over_alloc_mutex);
+ /* we just make sure that we don't set this at the same time.
+ We don't worry too much about reading this rarely-set variable */
+#endif
+ bOverAllocDD = set;
+#ifdef GMX_THREAD_MPI
+ tMPI_Thread_mutex_unlock(&over_alloc_mutex);
+#endif
+}
+
+int over_alloc_dd(int n)
+{
+ if (bOverAllocDD)
+ {
+ return OVER_ALLOC_FAC*n + 100;
+ }
+ else
+ {
+ return n;
+ }
+}
+
+int gmx_large_int_to_int(gmx_large_int_t step, const char *warn)
+{
+ int i;
+
+ i = (int)step;
+
+ if (warn != NULL && (step < INT_MIN || step > INT_MAX))
+ {
+ fprintf(stderr, "\nWARNING during %s:\n", warn);
+ fprintf(stderr, "step value ");
+ fprintf(stderr, gmx_large_int_pfmt, step);
+ fprintf(stderr, " does not fit in int, converted to %d\n\n", i);
+ }
+
+ return i;
+}
+
+char *gmx_step_str(gmx_large_int_t i, char *buf)
+{
+ sprintf(buf, gmx_large_int_pfmt, i);
+
+ return buf;
+}
+
+void init_block(t_block *block)
+{
+ int i;
+
+ block->nr = 0;
+ block->nalloc_index = 1;
+ snew(block->index, block->nalloc_index);
+ block->index[0] = 0;
+}
+
+void init_blocka(t_blocka *block)
+{
+ int i;
+
+ block->nr = 0;
+ block->nra = 0;
+ block->nalloc_index = 1;
+ snew(block->index, block->nalloc_index);
+ block->index[0] = 0;
+ block->nalloc_a = 0;
+ block->a = NULL;
+}
+
+void init_atom(t_atoms *at)
+{
+ int i;
+
+ at->nr = 0;
+ at->nres = 0;
+ at->atom = NULL;
+ at->resinfo = NULL;
+ at->atomname = NULL;
+ at->atomtype = NULL;
+ at->atomtypeB = NULL;
+ at->pdbinfo = NULL;
+}
+
+void init_atomtypes(t_atomtypes *at)
+{
+ at->nr = 0;
+ at->radius = NULL;
+ at->vol = NULL;
+ at->atomnumber = NULL;
+ at->gb_radius = NULL;
+ at->S_hct = NULL;
+}
+
+void init_groups(gmx_groups_t *groups)
+{
+ int g;
+
+ groups->ngrpname = 0;
+ groups->grpname = NULL;
+ for (g = 0; (g < egcNR); g++)
+ {
+ groups->grps[g].nm_ind = NULL;
+ groups->ngrpnr[g] = 0;
+ groups->grpnr[g] = NULL;
+ }
+
+}
+
+void init_mtop(gmx_mtop_t *mtop)
+{
+ mtop->name = NULL;
+ mtop->nmoltype = 0;
+ mtop->moltype = NULL;
+ mtop->nmolblock = 0;
+ mtop->molblock = NULL;
+ mtop->maxres_renum = 0;
+ mtop->maxresnr = -1;
+ init_groups(&mtop->groups);
+ init_block(&mtop->mols);
+ open_symtab(&mtop->symtab);
+}
+
+void init_top (t_topology *top)
+{
+ int i;
+
+ top->name = NULL;
+ init_atom (&(top->atoms));
+ init_atomtypes(&(top->atomtypes));
+ init_block(&top->cgs);
+ init_block(&top->mols);
+ init_blocka(&top->excls);
+ open_symtab(&top->symtab);
+}
+
+void init_inputrec(t_inputrec *ir)
+{
+ memset(ir, 0, (size_t)sizeof(*ir));
+ snew(ir->fepvals, 1);
+ snew(ir->expandedvals, 1);
+ snew(ir->simtempvals, 1);
+}
+
+void stupid_fill_block(t_block *grp, int natom, gmx_bool bOneIndexGroup)
+{
+ int i;
+
+ if (bOneIndexGroup)
+ {
+ grp->nalloc_index = 2;
+ snew(grp->index, grp->nalloc_index);
+ grp->index[0] = 0;
+ grp->index[1] = natom;
+ grp->nr = 1;
+ }
+ else
+ {
+ grp->nalloc_index = natom+1;
+ snew(grp->index, grp->nalloc_index);
+ snew(grp->index, natom+1);
+ for (i = 0; (i <= natom); i++)
+ {
+ grp->index[i] = i;
+ }
+ grp->nr = natom;
+ }
+}
+
+void stupid_fill_blocka(t_blocka *grp, int natom)
+{
+ int i;
+
+ grp->nalloc_a = natom;
+ snew(grp->a, grp->nalloc_a);
+ for (i = 0; (i < natom); i++)
+ {
+ grp->a[i] = i;
+ }
+ grp->nra = natom;
+
+ grp->nalloc_index = natom + 1;
+ snew(grp->index, grp->nalloc_index);
+ for (i = 0; (i <= natom); i++)
+ {
+ grp->index[i] = i;
+ }
+ grp->nr = natom;
+}
+
+void copy_blocka(const t_blocka *src, t_blocka *dest)
+{
+ int i;
+
+ dest->nr = src->nr;
+ dest->nalloc_index = dest->nr + 1;
+ snew(dest->index, dest->nalloc_index);
+ for (i = 0; i < dest->nr+1; i++)
+ {
+ dest->index[i] = src->index[i];
+ }
+ dest->nra = src->nra;
+ dest->nalloc_a = dest->nra + 1;
+ snew(dest->a, dest->nalloc_a);
+ for (i = 0; i < dest->nra+1; i++)
+ {
+ dest->a[i] = src->a[i];
+ }
+}
+
+void done_block(t_block *block)
+{
+ block->nr = 0;
+ sfree(block->index);
+ block->nalloc_index = 0;
+}
+
+void done_blocka(t_blocka *block)
+{
+ block->nr = 0;
+ block->nra = 0;
+ sfree(block->index);
+ sfree(block->a);
+ block->index = NULL;
+ block->a = NULL;
+ block->nalloc_index = 0;
+ block->nalloc_a = 0;
+}
+
+void done_atom (t_atoms *at)
+{
+ at->nr = 0;
+ at->nres = 0;
+ sfree(at->atom);
+ sfree(at->resinfo);
+ sfree(at->atomname);
+ sfree(at->atomtype);
+ sfree(at->atomtypeB);
+ if (at->pdbinfo)
+ {
+ sfree(at->pdbinfo);
+ }
+}
+
+void done_atomtypes(t_atomtypes *atype)
+{
+ atype->nr = 0;
+ sfree(atype->radius);
+ sfree(atype->vol);
+ sfree(atype->surftens);
+ sfree(atype->atomnumber);
+ sfree(atype->gb_radius);
+ sfree(atype->S_hct);
+}
+
+void done_moltype(gmx_moltype_t *molt)
+{
+ int f;
+
+ done_atom(&molt->atoms);
+ done_block(&molt->cgs);
+ done_blocka(&molt->excls);
+
+ for (f = 0; f < F_NRE; f++)
+ {
+ sfree(molt->ilist[f].iatoms);
+ molt->ilist[f].nalloc = 0;
+ }
+}
+
+void done_molblock(gmx_molblock_t *molb)
+{
+ if (molb->nposres_xA > 0)
+ {
+ molb->nposres_xA = 0;
+ free(molb->posres_xA);
+ }
+ if (molb->nposres_xB > 0)
+ {
+ molb->nposres_xB = 0;
+ free(molb->posres_xB);
+ }
+}
+
+void done_mtop(gmx_mtop_t *mtop, gmx_bool bDoneSymtab)
+{
+ int i;
+
+ if (bDoneSymtab)
+ {
+ done_symtab(&mtop->symtab);
+ }
+
+ sfree(mtop->ffparams.functype);
+ sfree(mtop->ffparams.iparams);
+
+ for (i = 0; i < mtop->nmoltype; i++)
+ {
+ done_moltype(&mtop->moltype[i]);
+ }
+ sfree(mtop->moltype);
+ for (i = 0; i < mtop->nmolblock; i++)
+ {
+ done_molblock(&mtop->molblock[i]);
+ }
+ sfree(mtop->molblock);
+ done_block(&mtop->mols);
+}
+
+void done_top(t_topology *top)
+{
+ int f;
+
+ sfree(top->idef.functype);
+ sfree(top->idef.iparams);
+ for (f = 0; f < F_NRE; ++f)
+ {
+ sfree(top->idef.il[f].iatoms);
+ top->idef.il[f].iatoms = NULL;
+ top->idef.il[f].nalloc = 0;
+ }
+
+ done_atom (&(top->atoms));
+
+ /* For GB */
+ done_atomtypes(&(top->atomtypes));
+
+ done_symtab(&(top->symtab));
+ done_block(&(top->cgs));
+ done_block(&(top->mols));
+ done_blocka(&(top->excls));
+}
+
+static void done_pullgrp(t_pullgrp *pgrp)
+{
+ sfree(pgrp->ind);
+ sfree(pgrp->ind_loc);
+ sfree(pgrp->weight);
+ sfree(pgrp->weight_loc);
+}
+
+static void done_pull(t_pull *pull)
+{
+ int i;
+
+ for (i = 0; i < pull->ngrp+1; i++)
+ {
+ done_pullgrp(pull->grp);
+ done_pullgrp(pull->dyna);
+ }
+}
+
+void done_inputrec(t_inputrec *ir)
+{
+ int m;
+
+ for (m = 0; (m < DIM); m++)
+ {
+ if (ir->ex[m].a)
+ {
+ sfree(ir->ex[m].a);
+ }
+ if (ir->ex[m].phi)
+ {
+ sfree(ir->ex[m].phi);
+ }
+ if (ir->et[m].a)
+ {
+ sfree(ir->et[m].a);
+ }
+ if (ir->et[m].phi)
+ {
+ sfree(ir->et[m].phi);
+ }
+ }
+
+ sfree(ir->opts.nrdf);
+ sfree(ir->opts.ref_t);
+ sfree(ir->opts.annealing);
+ sfree(ir->opts.anneal_npoints);
+ sfree(ir->opts.anneal_time);
+ sfree(ir->opts.anneal_temp);
+ sfree(ir->opts.tau_t);
+ sfree(ir->opts.acc);
+ sfree(ir->opts.nFreeze);
+ sfree(ir->opts.QMmethod);
+ sfree(ir->opts.QMbasis);
+ sfree(ir->opts.QMcharge);
+ sfree(ir->opts.QMmult);
+ sfree(ir->opts.bSH);
+ sfree(ir->opts.CASorbitals);
+ sfree(ir->opts.CASelectrons);
+ sfree(ir->opts.SAon);
+ sfree(ir->opts.SAoff);
+ sfree(ir->opts.SAsteps);
+ sfree(ir->opts.bOPT);
+ sfree(ir->opts.bTS);
+
+ if (ir->pull)
+ {
+ done_pull(ir->pull);
+ sfree(ir->pull);
+ }
+}
+
+static void zero_history(history_t *hist)
+{
+ hist->disre_initf = 0;
+ hist->ndisrepairs = 0;
+ hist->disre_rm3tav = NULL;
+ hist->orire_initf = 0;
+ hist->norire_Dtav = 0;
+ hist->orire_Dtav = NULL;
+}
+
+static void zero_ekinstate(ekinstate_t *eks)
+{
+ eks->ekin_n = 0;
+ eks->ekinh = NULL;
+ eks->ekinf = NULL;
+ eks->ekinh_old = NULL;
+ eks->ekinscalef_nhc = NULL;
+ eks->ekinscaleh_nhc = NULL;
+ eks->vscale_nhc = NULL;
+ eks->dekindl = 0;
+ eks->mvcos = 0;
+}
+
+void init_energyhistory(energyhistory_t * enerhist)
+{
+ enerhist->nener = 0;
+
+ enerhist->ener_ave = NULL;
+ enerhist->ener_sum = NULL;
+ enerhist->ener_sum_sim = NULL;
+ enerhist->dht = NULL;
+
+ enerhist->nsteps = 0;
+ enerhist->nsum = 0;
+ enerhist->nsteps_sim = 0;
+ enerhist->nsum_sim = 0;
+
+ enerhist->dht = NULL;
+}
+
+static void done_delta_h_history(delta_h_history_t *dht)
+{
+ int i;
+
+ for (i = 0; i < dht->nndh; i++)
+ {
+ sfree(dht->dh[i]);
+ }
+ sfree(dht->dh);
+ sfree(dht->ndh);
+}
+
+void done_energyhistory(energyhistory_t * enerhist)
+{
+ sfree(enerhist->ener_ave);
+ sfree(enerhist->ener_sum);
+ sfree(enerhist->ener_sum_sim);
+
+ if (enerhist->dht != NULL)
+ {
+ done_delta_h_history(enerhist->dht);
+ sfree(enerhist->dht);
+ }
+}
+
+void init_gtc_state(t_state *state, int ngtc, int nnhpres, int nhchainlength)
+{
+ int i, j;
+
+ state->ngtc = ngtc;
+ state->nnhpres = nnhpres;
+ state->nhchainlength = nhchainlength;
+ if (state->ngtc > 0)
+ {
+ snew(state->nosehoover_xi, state->nhchainlength*state->ngtc);
+ snew(state->nosehoover_vxi, state->nhchainlength*state->ngtc);
+ snew(state->therm_integral, state->ngtc);
+ for (i = 0; i < state->ngtc; i++)
+ {
+ for (j = 0; j < state->nhchainlength; j++)
+ {
+ state->nosehoover_xi[i*state->nhchainlength + j] = 0.0;
+ state->nosehoover_vxi[i*state->nhchainlength + j] = 0.0;
+ }
+ }
+ for (i = 0; i < state->ngtc; i++)
+ {
+ state->therm_integral[i] = 0.0;
+ }
+ }
+ else
+ {
+ state->nosehoover_xi = NULL;
+ state->nosehoover_vxi = NULL;
+ state->therm_integral = NULL;
+ }
+
+ if (state->nnhpres > 0)
+ {
+ snew(state->nhpres_xi, state->nhchainlength*nnhpres);
+ snew(state->nhpres_vxi, state->nhchainlength*nnhpres);
+ for (i = 0; i < nnhpres; i++)
+ {
+ for (j = 0; j < state->nhchainlength; j++)
+ {
+ state->nhpres_xi[i*nhchainlength + j] = 0.0;
+ state->nhpres_vxi[i*nhchainlength + j] = 0.0;
+ }
+ }
+ }
+ else
+ {
+ state->nhpres_xi = NULL;
+ state->nhpres_vxi = NULL;
+ }
+}
+
+
+void init_state(t_state *state, int natoms, int ngtc, int nnhpres, int nhchainlength, int nlambda)
+{
+ int i;
+
+ state->natoms = natoms;
+ state->nrng = 0;
+ state->flags = 0;
+ state->lambda = 0;
+ snew(state->lambda, efptNR);
+ for (i = 0; i < efptNR; i++)
+ {
+ state->lambda[i] = 0;
+ }
+ state->veta = 0;
+ clear_mat(state->box);
+ clear_mat(state->box_rel);
+ clear_mat(state->boxv);
+ clear_mat(state->pres_prev);
+ clear_mat(state->svir_prev);
+ clear_mat(state->fvir_prev);
+ init_gtc_state(state, ngtc, nnhpres, nhchainlength);
+ state->nalloc = state->natoms;
+ if (state->nalloc > 0)
+ {
+ snew(state->x, state->nalloc);
+ snew(state->v, state->nalloc);
+ }
+ else
+ {
+ state->x = NULL;
+ state->v = NULL;
+ }
+ state->sd_X = NULL;
+ state->cg_p = NULL;
+ zero_history(&state->hist);
+ zero_ekinstate(&state->ekinstate);
+ init_energyhistory(&state->enerhist);
- extern void init_df_history(df_history_t *dfhist, int nlambda, real wl_delta)
++ init_df_history(&state->dfhist,nlambda);
+ state->ddp_count = 0;
+ state->ddp_count_cg_gl = 0;
+ state->cg_gl = NULL;
+ state->cg_gl_nalloc = 0;
+}
+
+void done_state(t_state *state)
+{
+ if (state->x)
+ {
+ sfree(state->x);
+ }
+ if (state->v)
+ {
+ sfree(state->v);
+ }
+ if (state->sd_X)
+ {
+ sfree(state->sd_X);
+ }
+ if (state->cg_p)
+ {
+ sfree(state->cg_p);
+ }
+ state->nalloc = 0;
+ if (state->cg_gl)
+ {
+ sfree(state->cg_gl);
+ }
+ state->cg_gl_nalloc = 0;
+ if (state->lambda)
+ {
+ sfree(state->lambda);
+ }
+ if (state->ngtc > 0)
+ {
+ sfree(state->nosehoover_xi);
+ sfree(state->nosehoover_vxi);
+ sfree(state->therm_integral);
+ }
+}
+
+static void do_box_rel(t_inputrec *ir, matrix box_rel, matrix b, gmx_bool bInit)
+{
+ int d, d2;
+
+ for (d = YY; d <= ZZ; d++)
+ {
+ for (d2 = XX; d2 <= (ir->epct == epctSEMIISOTROPIC ? YY : ZZ); d2++)
+ {
+ /* We need to check if this box component is deformed
+ * or if deformation of another component might cause
+ * changes in this component due to box corrections.
+ */
+ if (ir->deform[d][d2] == 0 &&
+ !(d == ZZ && d2 == XX && ir->deform[d][YY] != 0 &&
+ (b[YY][d2] != 0 || ir->deform[YY][d2] != 0)))
+ {
+ if (bInit)
+ {
+ box_rel[d][d2] = b[d][d2]/b[XX][XX];
+ }
+ else
+ {
+ b[d][d2] = b[XX][XX]*box_rel[d][d2];
+ }
+ }
+ }
+ }
+}
+
+void set_box_rel(t_inputrec *ir, t_state *state)
+{
+ /* Make sure the box obeys the restrictions before we fix the ratios */
+ correct_box(NULL, 0, state->box, NULL);
+
+ clear_mat(state->box_rel);
+
+ if (PRESERVE_SHAPE(*ir))
+ {
+ do_box_rel(ir, state->box_rel, state->box, TRUE);
+ }
+}
+
+void preserve_box_shape(t_inputrec *ir, matrix box_rel, matrix b)
+{
+ if (PRESERVE_SHAPE(*ir))
+ {
+ do_box_rel(ir, box_rel, b, FALSE);
+ }
+}
+
+void add_t_atoms(t_atoms *atoms, int natom_extra, int nres_extra)
+{
+ int i;
+
+ if (natom_extra > 0)
+ {
+ srenew(atoms->atomname, atoms->nr+natom_extra);
+ srenew(atoms->atom, atoms->nr+natom_extra);
+ if (NULL != atoms->pdbinfo)
+ {
+ srenew(atoms->pdbinfo, atoms->nr+natom_extra);
+ }
+ if (NULL != atoms->atomtype)
+ {
+ srenew(atoms->atomtype, atoms->nr+natom_extra);
+ }
+ if (NULL != atoms->atomtypeB)
+ {
+ srenew(atoms->atomtypeB, atoms->nr+natom_extra);
+ }
+ for (i = atoms->nr; (i < atoms->nr+natom_extra); i++)
+ {
+ atoms->atomname[i] = NULL;
+ memset(&atoms->atom[i], 0, sizeof(atoms->atom[i]));
+ if (NULL != atoms->pdbinfo)
+ {
+ memset(&atoms->pdbinfo[i], 0, sizeof(atoms->pdbinfo[i]));
+ }
+ if (NULL != atoms->atomtype)
+ {
+ atoms->atomtype[i] = NULL;
+ }
+ if (NULL != atoms->atomtypeB)
+ {
+ atoms->atomtypeB[i] = NULL;
+ }
+ }
+ atoms->nr += natom_extra;
+ }
+ if (nres_extra > 0)
+ {
+ srenew(atoms->resinfo, atoms->nres+nres_extra);
+ for (i = atoms->nres; (i < atoms->nres+nres_extra); i++)
+ {
+ memset(&atoms->resinfo[i], 0, sizeof(atoms->resinfo[i]));
+ }
+ atoms->nres += nres_extra;
+ }
+}
+
+void init_t_atoms(t_atoms *atoms, int natoms, gmx_bool bPdbinfo)
+{
+ atoms->nr = natoms;
+ atoms->nres = 0;
+ snew(atoms->atomname, natoms);
+ atoms->atomtype = NULL;
+ atoms->atomtypeB = NULL;
+ snew(atoms->resinfo, natoms);
+ snew(atoms->atom, natoms);
+ if (bPdbinfo)
+ {
+ snew(atoms->pdbinfo, natoms);
+ }
+ else
+ {
+ atoms->pdbinfo = NULL;
+ }
+}
+
+t_atoms *copy_t_atoms(t_atoms *src)
+{
+ t_atoms *dst;
+ int i;
+
+ snew(dst, 1);
+ init_t_atoms(dst, src->nr, (NULL != src->pdbinfo));
+ dst->nr = src->nr;
+ if (NULL != src->atomname)
+ {
+ snew(dst->atomname, src->nr);
+ }
+ if (NULL != src->atomtype)
+ {
+ snew(dst->atomtype, src->nr);
+ }
+ if (NULL != src->atomtypeB)
+ {
+ snew(dst->atomtypeB, src->nr);
+ }
+ for (i = 0; (i < src->nr); i++)
+ {
+ dst->atom[i] = src->atom[i];
+ if (NULL != src->pdbinfo)
+ {
+ dst->pdbinfo[i] = src->pdbinfo[i];
+ }
+ if (NULL != src->atomname)
+ {
+ dst->atomname[i] = src->atomname[i];
+ }
+ if (NULL != src->atomtype)
+ {
+ dst->atomtype[i] = src->atomtype[i];
+ }
+ if (NULL != src->atomtypeB)
+ {
+ dst->atomtypeB[i] = src->atomtypeB[i];
+ }
+ }
+ dst->nres = src->nres;
+ for (i = 0; (i < src->nres); i++)
+ {
+ dst->resinfo[i] = src->resinfo[i];
+ }
+ return dst;
+}
+
+void t_atoms_set_resinfo(t_atoms *atoms, int atom_ind, t_symtab *symtab,
+ const char *resname, int resnr, unsigned char ic,
+ int chainnum, char chainid)
+{
+ t_resinfo *ri;
+
+ ri = &atoms->resinfo[atoms->atom[atom_ind].resind];
+ ri->name = put_symtab(symtab, resname);
+ ri->rtp = NULL;
+ ri->nr = resnr;
+ ri->ic = ic;
+ ri->chainnum = chainnum;
+ ri->chainid = chainid;
+}
+
+void free_t_atoms(t_atoms *atoms, gmx_bool bFreeNames)
+{
+ int i;
+
+ if (bFreeNames && atoms->atomname != NULL)
+ {
+ for (i = 0; i < atoms->nr; i++)
+ {
+ if (atoms->atomname[i] != NULL)
+ {
+ sfree(*atoms->atomname[i]);
+ *atoms->atomname[i] = NULL;
+ }
+ }
+ }
+ if (bFreeNames && atoms->resinfo != NULL)
+ {
+ for (i = 0; i < atoms->nres; i++)
+ {
+ if (atoms->resinfo[i].name != NULL)
+ {
+ sfree(*atoms->resinfo[i].name);
+ *atoms->resinfo[i].name = NULL;
+ }
+ }
+ }
+ if (bFreeNames && atoms->atomtype != NULL)
+ {
+ for (i = 0; i < atoms->nr; i++)
+ {
+ if (atoms->atomtype[i] != NULL)
+ {
+ sfree(*atoms->atomtype[i]);
+ *atoms->atomtype[i] = NULL;
+ }
+ }
+ }
+ if (bFreeNames && atoms->atomtypeB != NULL)
+ {
+ for (i = 0; i < atoms->nr; i++)
+ {
+ if (atoms->atomtypeB[i] != NULL)
+ {
+ sfree(*atoms->atomtypeB[i]);
+ *atoms->atomtypeB[i] = NULL;
+ }
+ }
+ }
+ sfree(atoms->atomname);
+ sfree(atoms->atomtype);
+ sfree(atoms->atomtypeB);
+ sfree(atoms->resinfo);
+ sfree(atoms->atom);
+ sfree(atoms->pdbinfo);
+ atoms->nr = 0;
+ atoms->nres = 0;
+ atoms->atomname = NULL;
+ atoms->atomtype = NULL;
+ atoms->atomtypeB = NULL;
+ atoms->resinfo = NULL;
+ atoms->atom = NULL;
+ atoms->pdbinfo = NULL;
+}
+
+real max_cutoff(real cutoff1, real cutoff2)
+{
+ if (cutoff1 == 0 || cutoff2 == 0)
+ {
+ return 0;
+ }
+ else
+ {
+ return max(cutoff1, cutoff2);
+ }
+}
+
- dfhist->bEquil = 0;
++void init_df_history(df_history_t *dfhist, int nlambda)
+{
+ int i;
+
- dfhist->wl_delta = wl_delta;
- snew(dfhist->sum_weights, dfhist->nlambda);
- snew(dfhist->sum_dg, dfhist->nlambda);
- snew(dfhist->sum_minvar, dfhist->nlambda);
- snew(dfhist->sum_variance, dfhist->nlambda);
- snew(dfhist->n_at_lam, dfhist->nlambda);
- snew(dfhist->wl_histo, dfhist->nlambda);
-
- /* allocate transition matrices here */
- snew(dfhist->Tij, dfhist->nlambda);
- snew(dfhist->Tij_empirical, dfhist->nlambda);
-
- for (i = 0; i < dfhist->nlambda; i++)
- {
- snew(dfhist->Tij[i], dfhist->nlambda);
- snew(dfhist->Tij_empirical[i], dfhist->nlambda);
- }
-
- snew(dfhist->accum_p, dfhist->nlambda);
- snew(dfhist->accum_m, dfhist->nlambda);
- snew(dfhist->accum_p2, dfhist->nlambda);
- snew(dfhist->accum_m2, dfhist->nlambda);
+ dfhist->nlambda = nlambda;
- for (i = 0; i < dfhist->nlambda; i++)
++ dfhist->bEquil = 0;
++ dfhist->wl_delta = 0;
+
- snew((dfhist->accum_p)[i], dfhist->nlambda);
- snew((dfhist->accum_m)[i], dfhist->nlambda);
- snew((dfhist->accum_p2)[i], dfhist->nlambda);
- snew((dfhist->accum_m2)[i], dfhist->nlambda);
++ if (nlambda > 0)
+ {
- init_df_history(df_dest, df_source->nlambda, df_source->wl_delta);
++ snew(dfhist->sum_weights, dfhist->nlambda);
++ snew(dfhist->sum_dg, dfhist->nlambda);
++ snew(dfhist->sum_minvar, dfhist->nlambda);
++ snew(dfhist->sum_variance, dfhist->nlambda);
++ snew(dfhist->n_at_lam, dfhist->nlambda);
++ snew(dfhist->wl_histo, dfhist->nlambda);
++
++ /* allocate transition matrices here */
++ snew(dfhist->Tij, dfhist->nlambda);
++ snew(dfhist->Tij_empirical, dfhist->nlambda);
++
++ /* allocate accumulators for various transition matrix
++ free energy methods here */
++ snew(dfhist->accum_p, dfhist->nlambda);
++ snew(dfhist->accum_m, dfhist->nlambda);
++ snew(dfhist->accum_p2, dfhist->nlambda);
++ snew(dfhist->accum_m2, dfhist->nlambda);
++
++ for (i = 0; i < dfhist->nlambda; i++)
++ {
++ snew(dfhist->Tij[i], dfhist->nlambda);
++ snew(dfhist->Tij_empirical[i], dfhist->nlambda);
++ snew((dfhist->accum_p)[i], dfhist->nlambda);
++ snew((dfhist->accum_m)[i], dfhist->nlambda);
++ snew((dfhist->accum_p2)[i], dfhist->nlambda);
++ snew((dfhist->accum_m2)[i], dfhist->nlambda);
++ }
+ }
+}
+
+extern void copy_df_history(df_history_t *df_dest, df_history_t *df_source)
+{
+ int i, j;
+
- df_dest->accum_p[i] = df_source->accum_p[i];
- df_dest->accum_m[i] = df_source->accum_m[i];
- df_dest->accum_p2[i] = df_source->accum_p2[i];
- df_dest->accum_m2[i] = df_source->accum_m2[i];
++ /* Currently, there should not be any difference in nlambda between the two,
++ but this is included for completeness for potential later functionality */
+ df_dest->nlambda = df_source->nlambda;
+ df_dest->bEquil = df_source->bEquil;
++ df_dest->wl_delta = df_source->wl_delta;
++
+ for (i = 0; i < df_dest->nlambda; i++)
+ {
+ df_dest->sum_weights[i] = df_source->sum_weights[i];
+ df_dest->sum_dg[i] = df_source->sum_dg[i];
+ df_dest->sum_minvar[i] = df_source->sum_minvar[i];
+ df_dest->sum_variance[i] = df_source->sum_variance[i];
+ df_dest->n_at_lam[i] = df_source->n_at_lam[i];
+ df_dest->wl_histo[i] = df_source->wl_histo[i];
+ }
+
+ for (i = 0; i < df_dest->nlambda; i++)
+ {
+ for (j = 0; j < df_dest->nlambda; j++)
+ {
++ df_dest->accum_p[i][j] = df_source->accum_p[i][j];
++ df_dest->accum_m[i][j] = df_source->accum_m[i][j];
++ df_dest->accum_p2[i][j] = df_source->accum_p2[i][j];
++ df_dest->accum_m2[i][j] = df_source->accum_m2[i][j];
+ df_dest->Tij[i][j] = df_source->Tij[i][j];
+ df_dest->Tij_empirical[i][j] = df_source->Tij_empirical[i][j];
+ }
+ }
+}
++
++void done_df_history(df_history_t *dfhist)
++{
++ int i;
++
++ if (dfhist->nlambda > 0)
++ {
++ sfree(dfhist->n_at_lam);
++ sfree(dfhist->wl_histo);
++ sfree(dfhist->sum_weights);
++ sfree(dfhist->sum_dg);
++ sfree(dfhist->sum_minvar);
++ sfree(dfhist->sum_variance);
++
++ for (i=0;i<dfhist->nlambda;i++)
++ {
++ sfree(dfhist->Tij[i]);
++ sfree(dfhist->Tij_empirical[i]);
++ sfree(dfhist->accum_p[i]);
++ sfree(dfhist->accum_m[i]);
++ sfree(dfhist->accum_p2[i]);
++ sfree(dfhist->accum_m2[i]);
++ }
++ }
++ dfhist->bEquil = 0;
++ dfhist->nlambda = 0;
++ dfhist->wl_delta = 0;
++}
--- /dev/null
- expand->bInit_weights = FALSE;
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * VERSION 3.2.0
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2004, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * Gallium Rubidium Oxygen Manganese Argon Carbon Silicon
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <ctype.h>
+#include <stdlib.h>
+#include <limits.h>
+#include "sysstuff.h"
+#include "smalloc.h"
+#include "typedefs.h"
+#include "physics.h"
+#include "names.h"
+#include "gmx_fatal.h"
+#include "macros.h"
+#include "index.h"
+#include "symtab.h"
+#include "string2.h"
+#include "readinp.h"
+#include "warninp.h"
+#include "readir.h"
+#include "toputil.h"
+#include "index.h"
+#include "network.h"
+#include "vec.h"
+#include "pbc.h"
+#include "mtop_util.h"
+#include "chargegroup.h"
+#include "inputrec.h"
+
+#define MAXPTR 254
+#define NOGID 255
+#define MAXLAMBDAS 1024
+
+/* Resource parameters
+ * Do not change any of these until you read the instruction
+ * in readinp.h. Some cpp's do not take spaces after the backslash
+ * (like the c-shell), which will give you a very weird compiler
+ * message.
+ */
+
+static char tcgrps[STRLEN], tau_t[STRLEN], ref_t[STRLEN],
+ acc[STRLEN], accgrps[STRLEN], freeze[STRLEN], frdim[STRLEN],
+ energy[STRLEN], user1[STRLEN], user2[STRLEN], vcm[STRLEN], xtc_grps[STRLEN],
+ couple_moltype[STRLEN], orirefitgrp[STRLEN], egptable[STRLEN], egpexcl[STRLEN],
+ wall_atomtype[STRLEN], wall_density[STRLEN], deform[STRLEN], QMMM[STRLEN];
+static char fep_lambda[efptNR][STRLEN];
+static char lambda_weights[STRLEN];
+static char **pull_grp;
+static char **rot_grp;
+static char anneal[STRLEN], anneal_npoints[STRLEN],
+ anneal_time[STRLEN], anneal_temp[STRLEN];
+static char QMmethod[STRLEN], QMbasis[STRLEN], QMcharge[STRLEN], QMmult[STRLEN],
+ bSH[STRLEN], CASorbitals[STRLEN], CASelectrons[STRLEN], SAon[STRLEN],
+ SAoff[STRLEN], SAsteps[STRLEN], bTS[STRLEN], bOPT[STRLEN];
+static char efield_x[STRLEN], efield_xt[STRLEN], efield_y[STRLEN],
+ efield_yt[STRLEN], efield_z[STRLEN], efield_zt[STRLEN];
+
+enum {
+ egrptpALL, /* All particles have to be a member of a group. */
+ egrptpALL_GENREST, /* A rest group with name is generated for particles *
+ * that are not part of any group. */
+ egrptpPART, /* As egrptpALL_GENREST, but no name is generated *
+ * for the rest group. */
+ egrptpONE /* Merge all selected groups into one group, *
+ * make a rest group for the remaining particles. */
+};
+
+static const char *constraints[eshNR+1] = {
+ "none", "h-bonds", "all-bonds", "h-angles", "all-angles", NULL
+};
+
+static const char *couple_lam[ecouplamNR+1] = {
+ "vdw-q", "vdw", "q", "none", NULL
+};
+
+void init_ir(t_inputrec *ir, t_gromppopts *opts)
+{
+ snew(opts->include, STRLEN);
+ snew(opts->define, STRLEN);
+ snew(ir->fepvals, 1);
+ snew(ir->expandedvals, 1);
+ snew(ir->simtempvals, 1);
+}
+
+static void GetSimTemps(int ntemps, t_simtemp *simtemp, double *temperature_lambdas)
+{
+
+ int i;
+
+ for (i = 0; i < ntemps; i++)
+ {
+ /* simple linear scaling -- allows more control */
+ if (simtemp->eSimTempScale == esimtempLINEAR)
+ {
+ simtemp->temperatures[i] = simtemp->simtemp_low + (simtemp->simtemp_high-simtemp->simtemp_low)*temperature_lambdas[i];
+ }
+ else if (simtemp->eSimTempScale == esimtempGEOMETRIC) /* should give roughly equal acceptance for constant heat capacity . . . */
+ {
+ simtemp->temperatures[i] = simtemp->simtemp_low * pow(simtemp->simtemp_high/simtemp->simtemp_low, (1.0*i)/(ntemps-1));
+ }
+ else if (simtemp->eSimTempScale == esimtempEXPONENTIAL)
+ {
+ simtemp->temperatures[i] = simtemp->simtemp_low + (simtemp->simtemp_high-simtemp->simtemp_low)*((exp(temperature_lambdas[i])-1)/(exp(1.0)-1));
+ }
+ else
+ {
+ char errorstr[128];
+ sprintf(errorstr, "eSimTempScale=%d not defined", simtemp->eSimTempScale);
+ gmx_fatal(FARGS, errorstr);
+ }
+ }
+}
+
+
+
+static void _low_check(gmx_bool b, char *s, warninp_t wi)
+{
+ if (b)
+ {
+ warning_error(wi, s);
+ }
+}
+
+static void check_nst(const char *desc_nst, int nst,
+ const char *desc_p, int *p,
+ warninp_t wi)
+{
+ char buf[STRLEN];
+
+ if (*p > 0 && *p % nst != 0)
+ {
+ /* Round up to the next multiple of nst */
+ *p = ((*p)/nst + 1)*nst;
+ sprintf(buf, "%s should be a multiple of %s, changing %s to %d\n",
+ desc_p, desc_nst, desc_p, *p);
+ warning(wi, buf);
+ }
+}
+
+static gmx_bool ir_NVE(const t_inputrec *ir)
+{
+ return ((ir->eI == eiMD || EI_VV(ir->eI)) && ir->etc == etcNO);
+}
+
+static int lcd(int n1, int n2)
+{
+ int d, i;
+
+ d = 1;
+ for (i = 2; (i <= n1 && i <= n2); i++)
+ {
+ if (n1 % i == 0 && n2 % i == 0)
+ {
+ d = i;
+ }
+ }
+
+ return d;
+}
+
+static void process_interaction_modifier(const t_inputrec *ir, int *eintmod)
+{
+ if (*eintmod == eintmodPOTSHIFT_VERLET)
+ {
+ if (ir->cutoff_scheme == ecutsVERLET)
+ {
+ *eintmod = eintmodPOTSHIFT;
+ }
+ else
+ {
+ *eintmod = eintmodNONE;
+ }
+ }
+}
+
+void check_ir(const char *mdparin, t_inputrec *ir, t_gromppopts *opts,
+ warninp_t wi)
+/* Check internal consistency */
+{
+ /* Strange macro: first one fills the err_buf, and then one can check
+ * the condition, which will print the message and increase the error
+ * counter.
+ */
+#define CHECK(b) _low_check(b, err_buf, wi)
+ char err_buf[256], warn_buf[STRLEN];
+ int i, j;
+ int ns_type = 0;
+ real dt_coupl = 0;
+ real dt_pcoupl;
+ int nstcmin;
+ t_lambda *fep = ir->fepvals;
+ t_expanded *expand = ir->expandedvals;
+
+ set_warning_line(wi, mdparin, -1);
+
+ /* BASIC CUT-OFF STUFF */
+ if (ir->rcoulomb < 0)
+ {
+ warning_error(wi, "rcoulomb should be >= 0");
+ }
+ if (ir->rvdw < 0)
+ {
+ warning_error(wi, "rvdw should be >= 0");
+ }
+ if (ir->rlist < 0 &&
+ !(ir->cutoff_scheme == ecutsVERLET && ir->verletbuf_drift > 0))
+ {
+ warning_error(wi, "rlist should be >= 0");
+ }
+
+ process_interaction_modifier(ir, &ir->coulomb_modifier);
+ process_interaction_modifier(ir, &ir->vdw_modifier);
+
+ if (ir->cutoff_scheme == ecutsGROUP)
+ {
+ /* BASIC CUT-OFF STUFF */
+ if (ir->rlist == 0 ||
+ !((EEL_MIGHT_BE_ZERO_AT_CUTOFF(ir->coulombtype) && ir->rcoulomb > ir->rlist) ||
+ (EVDW_MIGHT_BE_ZERO_AT_CUTOFF(ir->vdwtype) && ir->rvdw > ir->rlist)))
+ {
+ /* No switched potential and/or no twin-range:
+ * we can set the long-range cut-off to the maximum of the other cut-offs.
+ */
+ ir->rlistlong = max_cutoff(ir->rlist, max_cutoff(ir->rvdw, ir->rcoulomb));
+ }
+ else if (ir->rlistlong < 0)
+ {
+ ir->rlistlong = max_cutoff(ir->rlist, max_cutoff(ir->rvdw, ir->rcoulomb));
+ sprintf(warn_buf, "rlistlong was not set, setting it to %g (no buffer)",
+ ir->rlistlong);
+ warning(wi, warn_buf);
+ }
+ if (ir->rlistlong == 0 && ir->ePBC != epbcNONE)
+ {
+ warning_error(wi, "Can not have an infinite cut-off with PBC");
+ }
+ if (ir->rlistlong > 0 && (ir->rlist == 0 || ir->rlistlong < ir->rlist))
+ {
+ warning_error(wi, "rlistlong can not be shorter than rlist");
+ }
+ if (IR_TWINRANGE(*ir) && ir->nstlist <= 0)
+ {
+ warning_error(wi, "Can not have nstlist<=0 with twin-range interactions");
+ }
+ }
+
+ if (ir->rlistlong == ir->rlist)
+ {
+ ir->nstcalclr = 0;
+ }
+ else if (ir->rlistlong > ir->rlist && ir->nstcalclr == 0)
+ {
+ warning_error(wi, "With different cutoffs for electrostatics and VdW, nstcalclr must be -1 or a positive number");
+ }
+
+ if (ir->cutoff_scheme == ecutsVERLET)
+ {
+ real rc_max;
+
+ /* Normal Verlet type neighbor-list, currently only limited feature support */
+ if (inputrec2nboundeddim(ir) < 3)
+ {
+ warning_error(wi, "With Verlet lists only full pbc or pbc=xy with walls is supported");
+ }
+ if (ir->rcoulomb != ir->rvdw)
+ {
+ warning_error(wi, "With Verlet lists rcoulomb!=rvdw is not supported");
+ }
+ if (ir->vdwtype != evdwCUT)
+ {
+ warning_error(wi, "With Verlet lists only cut-off LJ interactions are supported");
+ }
+ if (!(ir->coulombtype == eelCUT ||
+ (EEL_RF(ir->coulombtype) && ir->coulombtype != eelRF_NEC) ||
+ EEL_PME(ir->coulombtype) || ir->coulombtype == eelEWALD))
+ {
+ warning_error(wi, "With Verlet lists only cut-off, reaction-field, PME and Ewald electrostatics are supported");
+ }
+
+ if (ir->nstlist <= 0)
+ {
+ warning_error(wi, "With Verlet lists nstlist should be larger than 0");
+ }
+
+ if (ir->nstlist < 10)
+ {
+ warning_note(wi, "With Verlet lists the optimal nstlist is >= 10, with GPUs >= 20. Note that with the Verlet scheme, nstlist has no effect on the accuracy of your simulation.");
+ }
+
+ rc_max = max(ir->rvdw, ir->rcoulomb);
+
+ if (ir->verletbuf_drift <= 0)
+ {
+ if (ir->verletbuf_drift == 0)
+ {
+ warning_error(wi, "Can not have an energy drift of exactly 0");
+ }
+
+ if (ir->rlist < rc_max)
+ {
+ warning_error(wi, "With verlet lists rlist can not be smaller than rvdw or rcoulomb");
+ }
+
+ if (ir->rlist == rc_max && ir->nstlist > 1)
+ {
+ warning_note(wi, "rlist is equal to rvdw and/or rcoulomb: there is no explicit Verlet buffer. The cluster pair list does have a buffering effect, but choosing a larger rlist might be necessary for good energy conservation.");
+ }
+ }
+ else
+ {
+ if (ir->rlist > rc_max)
+ {
+ warning_note(wi, "You have set rlist larger than the interaction cut-off, but you also have verlet-buffer-drift > 0. Will set rlist using verlet-buffer-drift.");
+ }
+
+ if (ir->nstlist == 1)
+ {
+ /* No buffer required */
+ ir->rlist = rc_max;
+ }
+ else
+ {
+ if (EI_DYNAMICS(ir->eI))
+ {
+ if (EI_MD(ir->eI) && ir->etc == etcNO)
+ {
+ warning_error(wi, "Temperature coupling is required for calculating rlist using the energy drift with verlet-buffer-drift > 0. Either use temperature coupling or set rlist yourself together with verlet-buffer-drift = -1.");
+ }
+
+ if (inputrec2nboundeddim(ir) < 3)
+ {
+ warning_error(wi, "The box volume is required for calculating rlist from the energy drift with verlet-buffer-drift > 0. You are using at least one unbounded dimension, so no volume can be computed. Either use a finite box, or set rlist yourself together with verlet-buffer-drift = -1.");
+ }
+ /* Set rlist temporarily so we can continue processing */
+ ir->rlist = rc_max;
+ }
+ else
+ {
+ /* Set the buffer to 5% of the cut-off */
+ ir->rlist = 1.05*rc_max;
+ }
+ }
+ }
+
+ /* No twin-range calculations with Verlet lists */
+ ir->rlistlong = ir->rlist;
+ }
+
+ if (ir->nstcalclr == -1)
+ {
+ /* if rlist=rlistlong, this will later be changed to nstcalclr=0 */
+ ir->nstcalclr = ir->nstlist;
+ }
+ else if (ir->nstcalclr > 0)
+ {
+ if (ir->nstlist > 0 && (ir->nstlist % ir->nstcalclr != 0))
+ {
+ warning_error(wi, "nstlist must be evenly divisible by nstcalclr. Use nstcalclr = -1 to automatically follow nstlist");
+ }
+ }
+ else if (ir->nstcalclr < -1)
+ {
+ warning_error(wi, "nstcalclr must be a positive number (divisor of nstcalclr), or -1 to follow nstlist.");
+ }
+
+ if (EEL_PME(ir->coulombtype) && ir->rcoulomb > ir->rvdw && ir->nstcalclr > 1)
+ {
+ warning_error(wi, "When used with PME, the long-range component of twin-range interactions must be updated every step (nstcalclr)");
+ }
+
+ /* GENERAL INTEGRATOR STUFF */
+ if (!(ir->eI == eiMD || EI_VV(ir->eI)))
+ {
+ ir->etc = etcNO;
+ }
+ if (ir->eI == eiVVAK)
+ {
+ sprintf(warn_buf, "Integrator method %s is implemented primarily for validation purposes; for molecular dynamics, you should probably be using %s or %s", ei_names[eiVVAK], ei_names[eiMD], ei_names[eiVV]);
+ warning_note(wi, warn_buf);
+ }
+ if (!EI_DYNAMICS(ir->eI))
+ {
+ ir->epc = epcNO;
+ }
+ if (EI_DYNAMICS(ir->eI))
+ {
+ if (ir->nstcalcenergy < 0)
+ {
+ ir->nstcalcenergy = ir_optimal_nstcalcenergy(ir);
+ if (ir->nstenergy != 0 && ir->nstenergy < ir->nstcalcenergy)
+ {
+ /* nstcalcenergy larger than nstener does not make sense.
+ * We ideally want nstcalcenergy=nstener.
+ */
+ if (ir->nstlist > 0)
+ {
+ ir->nstcalcenergy = lcd(ir->nstenergy, ir->nstlist);
+ }
+ else
+ {
+ ir->nstcalcenergy = ir->nstenergy;
+ }
+ }
+ }
+ else if ( (ir->nstenergy > 0 && ir->nstcalcenergy > ir->nstenergy) ||
+ (ir->efep != efepNO && ir->fepvals->nstdhdl > 0 &&
+ (ir->nstcalcenergy > ir->fepvals->nstdhdl) ) )
+
+ {
+ const char *nsten = "nstenergy";
+ const char *nstdh = "nstdhdl";
+ const char *min_name = nsten;
+ int min_nst = ir->nstenergy;
+
+ /* find the smallest of ( nstenergy, nstdhdl ) */
+ if (ir->efep != efepNO && ir->fepvals->nstdhdl > 0 &&
+ (ir->fepvals->nstdhdl < ir->nstenergy) )
+ {
+ min_nst = ir->fepvals->nstdhdl;
+ min_name = nstdh;
+ }
+ /* If the user sets nstenergy small, we should respect that */
+ sprintf(warn_buf,
+ "Setting nstcalcenergy (%d) equal to %s (%d)",
+ ir->nstcalcenergy, min_name, min_nst);
+ warning_note(wi, warn_buf);
+ ir->nstcalcenergy = min_nst;
+ }
+
+ if (ir->epc != epcNO)
+ {
+ if (ir->nstpcouple < 0)
+ {
+ ir->nstpcouple = ir_optimal_nstpcouple(ir);
+ }
+ }
+ if (IR_TWINRANGE(*ir))
+ {
+ check_nst("nstlist", ir->nstlist,
+ "nstcalcenergy", &ir->nstcalcenergy, wi);
+ if (ir->epc != epcNO)
+ {
+ check_nst("nstlist", ir->nstlist,
+ "nstpcouple", &ir->nstpcouple, wi);
+ }
+ }
+
+ if (ir->nstcalcenergy > 0)
+ {
+ if (ir->efep != efepNO)
+ {
+ /* nstdhdl should be a multiple of nstcalcenergy */
+ check_nst("nstcalcenergy", ir->nstcalcenergy,
+ "nstdhdl", &ir->fepvals->nstdhdl, wi);
+ /* nstexpanded should be a multiple of nstcalcenergy */
+ check_nst("nstcalcenergy", ir->nstcalcenergy,
+ "nstexpanded", &ir->expandedvals->nstexpanded, wi);
+ }
+ /* for storing exact averages nstenergy should be
+ * a multiple of nstcalcenergy
+ */
+ check_nst("nstcalcenergy", ir->nstcalcenergy,
+ "nstenergy", &ir->nstenergy, wi);
+ }
+ }
+
+ /* LD STUFF */
+ if ((EI_SD(ir->eI) || ir->eI == eiBD) &&
+ ir->bContinuation && ir->ld_seed != -1)
+ {
+ warning_note(wi, "You are doing a continuation with SD or BD, make sure that ld_seed is different from the previous run (using ld_seed=-1 will ensure this)");
+ }
+
+ /* TPI STUFF */
+ if (EI_TPI(ir->eI))
+ {
+ sprintf(err_buf, "TPI only works with pbc = %s", epbc_names[epbcXYZ]);
+ CHECK(ir->ePBC != epbcXYZ);
+ sprintf(err_buf, "TPI only works with ns = %s", ens_names[ensGRID]);
+ CHECK(ir->ns_type != ensGRID);
+ sprintf(err_buf, "with TPI nstlist should be larger than zero");
+ CHECK(ir->nstlist <= 0);
+ sprintf(err_buf, "TPI does not work with full electrostatics other than PME");
+ CHECK(EEL_FULL(ir->coulombtype) && !EEL_PME(ir->coulombtype));
+ }
+
+ /* SHAKE / LINCS */
+ if ( (opts->nshake > 0) && (opts->bMorse) )
+ {
+ sprintf(warn_buf,
+ "Using morse bond-potentials while constraining bonds is useless");
+ warning(wi, warn_buf);
+ }
+
+ if ((EI_SD(ir->eI) || ir->eI == eiBD) &&
+ ir->bContinuation && ir->ld_seed != -1)
+ {
+ warning_note(wi, "You are doing a continuation with SD or BD, make sure that ld_seed is different from the previous run (using ld_seed=-1 will ensure this)");
+ }
+ /* verify simulated tempering options */
+
+ if (ir->bSimTemp)
+ {
+ gmx_bool bAllTempZero = TRUE;
+ for (i = 0; i < fep->n_lambda; i++)
+ {
+ sprintf(err_buf, "Entry %d for %s must be between 0 and 1, instead is %g", i, efpt_names[efptTEMPERATURE], fep->all_lambda[efptTEMPERATURE][i]);
+ CHECK((fep->all_lambda[efptTEMPERATURE][i] < 0) || (fep->all_lambda[efptTEMPERATURE][i] > 1));
+ if (fep->all_lambda[efptTEMPERATURE][i] > 0)
+ {
+ bAllTempZero = FALSE;
+ }
+ }
+ sprintf(err_buf, "if simulated tempering is on, temperature-lambdas may not be all zero");
+ CHECK(bAllTempZero == TRUE);
+
+ sprintf(err_buf, "Simulated tempering is currently only compatible with md-vv");
+ CHECK(ir->eI != eiVV);
+
+ /* check compatability of the temperature coupling with simulated tempering */
+
+ if (ir->etc == etcNOSEHOOVER)
+ {
+ sprintf(warn_buf, "Nose-Hoover based temperature control such as [%s] my not be entirelyconsistent with simulated tempering", etcoupl_names[ir->etc]);
+ warning_note(wi, warn_buf);
+ }
+
+ /* check that the temperatures make sense */
+
+ sprintf(err_buf, "Higher simulated tempering temperature (%g) must be >= than the simulated tempering lower temperature (%g)", ir->simtempvals->simtemp_high, ir->simtempvals->simtemp_low);
+ CHECK(ir->simtempvals->simtemp_high <= ir->simtempvals->simtemp_low);
+
+ sprintf(err_buf, "Higher simulated tempering temperature (%g) must be >= zero", ir->simtempvals->simtemp_high);
+ CHECK(ir->simtempvals->simtemp_high <= 0);
+
+ sprintf(err_buf, "Lower simulated tempering temperature (%g) must be >= zero", ir->simtempvals->simtemp_low);
+ CHECK(ir->simtempvals->simtemp_low <= 0);
+ }
+
+ /* verify free energy options */
+
+ if (ir->efep != efepNO)
+ {
+ fep = ir->fepvals;
+ sprintf(err_buf, "The soft-core power is %d and can only be 1 or 2",
+ fep->sc_power);
+ CHECK(fep->sc_alpha != 0 && fep->sc_power != 1 && fep->sc_power != 2);
+
+ sprintf(err_buf, "The soft-core sc-r-power is %d and can only be 6 or 48",
+ (int)fep->sc_r_power);
+ CHECK(fep->sc_alpha != 0 && fep->sc_r_power != 6.0 && fep->sc_r_power != 48.0);
+
+ sprintf(err_buf, "Can't use postive delta-lambda (%g) if initial state/lambda does not start at zero", fep->delta_lambda);
+ CHECK(fep->delta_lambda > 0 && ((fep->init_fep_state > 0) || (fep->init_lambda > 0)));
+
+ sprintf(err_buf, "Can't use postive delta-lambda (%g) with expanded ensemble simulations", fep->delta_lambda);
+ CHECK(fep->delta_lambda > 0 && (ir->efep == efepEXPANDED));
+
++ sprintf(err_buf, "Can only use expanded ensemble with md-vv for now; should be supported for other integrators in 5.0");
++ CHECK(!(EI_VV(ir->eI)) && (ir->efep == efepEXPANDED));
++
+ sprintf(err_buf, "Free-energy not implemented for Ewald");
+ CHECK(ir->coulombtype == eelEWALD);
+
+ /* check validty of lambda inputs */
+ if (fep->n_lambda == 0)
+ {
+ /* Clear output in case of no states:*/
+ sprintf(err_buf, "init-lambda-state set to %d: no lambda states are defined.", fep->init_fep_state);
+ CHECK((fep->init_fep_state >= 0) && (fep->n_lambda == 0));
+ }
+ else
+ {
+ sprintf(err_buf, "initial thermodynamic state %d does not exist, only goes to %d", fep->init_fep_state, fep->n_lambda-1);
+ CHECK((fep->init_fep_state >= fep->n_lambda));
+ }
+
+ sprintf(err_buf, "Lambda state must be set, either with init-lambda-state or with init-lambda");
+ CHECK((fep->init_fep_state < 0) && (fep->init_lambda < 0));
+
+ sprintf(err_buf, "init-lambda=%g while init-lambda-state=%d. Lambda state must be set either with init-lambda-state or with init-lambda, but not both",
+ fep->init_lambda, fep->init_fep_state);
+ CHECK((fep->init_fep_state >= 0) && (fep->init_lambda >= 0));
+
+
+
+ if ((fep->init_lambda >= 0) && (fep->delta_lambda == 0))
+ {
+ int n_lambda_terms;
+ n_lambda_terms = 0;
+ for (i = 0; i < efptNR; i++)
+ {
+ if (fep->separate_dvdl[i])
+ {
+ n_lambda_terms++;
+ }
+ }
+ if (n_lambda_terms > 1)
+ {
+ sprintf(warn_buf, "If lambda vector states (fep-lambdas, coul-lambdas etc.) are set, don't use init-lambda to set lambda state (except for slow growth). Use init-lambda-state instead.");
+ warning(wi, warn_buf);
+ }
+
+ if (n_lambda_terms < 2 && fep->n_lambda > 0)
+ {
+ warning_note(wi,
+ "init-lambda is deprecated for setting lambda state (except for slow growth). Use init-lambda-state instead.");
+ }
+ }
+
+ for (j = 0; j < efptNR; j++)
+ {
+ for (i = 0; i < fep->n_lambda; i++)
+ {
+ sprintf(err_buf, "Entry %d for %s must be between 0 and 1, instead is %g", i, efpt_names[j], fep->all_lambda[j][i]);
+ CHECK((fep->all_lambda[j][i] < 0) || (fep->all_lambda[j][i] > 1));
+ }
+ }
+
+ if ((fep->sc_alpha > 0) && (!fep->bScCoul))
+ {
+ for (i = 0; i < fep->n_lambda; i++)
+ {
+ sprintf(err_buf, "For state %d, vdw-lambdas (%f) is changing with vdw softcore, while coul-lambdas (%f) is nonzero without coulomb softcore: this will lead to crashes, and is not supported.", i, fep->all_lambda[efptVDW][i],
+ fep->all_lambda[efptCOUL][i]);
+ CHECK((fep->sc_alpha > 0) &&
+ (((fep->all_lambda[efptCOUL][i] > 0.0) &&
+ (fep->all_lambda[efptCOUL][i] < 1.0)) &&
+ ((fep->all_lambda[efptVDW][i] > 0.0) &&
+ (fep->all_lambda[efptVDW][i] < 1.0))));
+ }
+ }
+
+ if ((fep->bScCoul) && (EEL_PME(ir->coulombtype)))
+ {
+ real sigma, lambda, r_sc;
+
+ sigma = 0.34;
+ /* Maximum estimate for A and B charges equal with lambda power 1 */
+ lambda = 0.5;
+ r_sc = pow(lambda*fep->sc_alpha*pow(sigma/ir->rcoulomb, fep->sc_r_power) + 1.0, 1.0/fep->sc_r_power);
+ sprintf(warn_buf, "With PME there is a minor soft core effect present at the cut-off, proportional to (LJsigma/rcoulomb)^%g. This could have a minor effect on energy conservation, but usually other effects dominate. With a common sigma value of %g nm the fraction of the particle-particle potential at the cut-off at lambda=%g is around %.1e, while ewald-rtol is %.1e.",
+ fep->sc_r_power,
+ sigma, lambda, r_sc - 1.0, ir->ewald_rtol);
+ warning_note(wi, warn_buf);
+ }
+
+ /* Free Energy Checks -- In an ideal world, slow growth and FEP would
+ be treated differently, but that's the next step */
+
+ for (i = 0; i < efptNR; i++)
+ {
+ for (j = 0; j < fep->n_lambda; j++)
+ {
+ sprintf(err_buf, "%s[%d] must be between 0 and 1", efpt_names[i], j);
+ CHECK((fep->all_lambda[i][j] < 0) || (fep->all_lambda[i][j] > 1));
+ }
+ }
+ }
+
+ if ((ir->bSimTemp) || (ir->efep == efepEXPANDED))
+ {
+ fep = ir->fepvals;
+ expand = ir->expandedvals;
+
+ /* checking equilibration of weights inputs for validity */
+
+ sprintf(err_buf, "weight-equil-number-all-lambda (%d) is ignored if lmc-weights-equil is not equal to %s",
+ expand->equil_n_at_lam, elmceq_names[elmceqNUMATLAM]);
+ CHECK((expand->equil_n_at_lam > 0) && (expand->elmceq != elmceqNUMATLAM));
+
+ sprintf(err_buf, "weight-equil-number-samples (%d) is ignored if lmc-weights-equil is not equal to %s",
+ expand->equil_samples, elmceq_names[elmceqSAMPLES]);
+ CHECK((expand->equil_samples > 0) && (expand->elmceq != elmceqSAMPLES));
+
+ sprintf(err_buf, "weight-equil-number-steps (%d) is ignored if lmc-weights-equil is not equal to %s",
+ expand->equil_steps, elmceq_names[elmceqSTEPS]);
+ CHECK((expand->equil_steps > 0) && (expand->elmceq != elmceqSTEPS));
+
+ sprintf(err_buf, "weight-equil-wl-delta (%d) is ignored if lmc-weights-equil is not equal to %s",
+ expand->equil_samples, elmceq_names[elmceqWLDELTA]);
+ CHECK((expand->equil_wl_delta > 0) && (expand->elmceq != elmceqWLDELTA));
+
+ sprintf(err_buf, "weight-equil-count-ratio (%f) is ignored if lmc-weights-equil is not equal to %s",
+ expand->equil_ratio, elmceq_names[elmceqRATIO]);
+ CHECK((expand->equil_ratio > 0) && (expand->elmceq != elmceqRATIO));
+
+ sprintf(err_buf, "weight-equil-number-all-lambda (%d) must be a positive integer if lmc-weights-equil=%s",
+ expand->equil_n_at_lam, elmceq_names[elmceqNUMATLAM]);
+ CHECK((expand->equil_n_at_lam <= 0) && (expand->elmceq == elmceqNUMATLAM));
+
+ sprintf(err_buf, "weight-equil-number-samples (%d) must be a positive integer if lmc-weights-equil=%s",
+ expand->equil_samples, elmceq_names[elmceqSAMPLES]);
+ CHECK((expand->equil_samples <= 0) && (expand->elmceq == elmceqSAMPLES));
+
+ sprintf(err_buf, "weight-equil-number-steps (%d) must be a positive integer if lmc-weights-equil=%s",
+ expand->equil_steps, elmceq_names[elmceqSTEPS]);
+ CHECK((expand->equil_steps <= 0) && (expand->elmceq == elmceqSTEPS));
+
+ sprintf(err_buf, "weight-equil-wl-delta (%f) must be > 0 if lmc-weights-equil=%s",
+ expand->equil_wl_delta, elmceq_names[elmceqWLDELTA]);
+ CHECK((expand->equil_wl_delta <= 0) && (expand->elmceq == elmceqWLDELTA));
+
+ sprintf(err_buf, "weight-equil-count-ratio (%f) must be > 0 if lmc-weights-equil=%s",
+ expand->equil_ratio, elmceq_names[elmceqRATIO]);
+ CHECK((expand->equil_ratio <= 0) && (expand->elmceq == elmceqRATIO));
+
+ sprintf(err_buf, "lmc-weights-equil=%s only possible when lmc-stats = %s or lmc-stats %s",
+ elmceq_names[elmceqWLDELTA], elamstats_names[elamstatsWL], elamstats_names[elamstatsWWL]);
+ CHECK((expand->elmceq == elmceqWLDELTA) && (!EWL(expand->elamstats)));
+
+ sprintf(err_buf, "lmc-repeats (%d) must be greater than 0", expand->lmc_repeats);
+ CHECK((expand->lmc_repeats <= 0));
+ sprintf(err_buf, "minimum-var-min (%d) must be greater than 0", expand->minvarmin);
+ CHECK((expand->minvarmin <= 0));
+ sprintf(err_buf, "weight-c-range (%d) must be greater or equal to 0", expand->c_range);
+ CHECK((expand->c_range < 0));
+ sprintf(err_buf, "init-lambda-state (%d) must be zero if lmc-forced-nstart (%d)> 0 and lmc-move != 'no'",
+ fep->init_fep_state, expand->lmc_forced_nstart);
+ CHECK((fep->init_fep_state != 0) && (expand->lmc_forced_nstart > 0) && (expand->elmcmove != elmcmoveNO));
+ sprintf(err_buf, "lmc-forced-nstart (%d) must not be negative", expand->lmc_forced_nstart);
+ CHECK((expand->lmc_forced_nstart < 0));
+ sprintf(err_buf, "init-lambda-state (%d) must be in the interval [0,number of lambdas)", fep->init_fep_state);
+ CHECK((fep->init_fep_state < 0) || (fep->init_fep_state >= fep->n_lambda));
+
+ sprintf(err_buf, "init-wl-delta (%f) must be greater than or equal to 0", expand->init_wl_delta);
+ CHECK((expand->init_wl_delta < 0));
+ sprintf(err_buf, "wl-ratio (%f) must be between 0 and 1", expand->wl_ratio);
+ CHECK((expand->wl_ratio <= 0) || (expand->wl_ratio >= 1));
+ sprintf(err_buf, "wl-scale (%f) must be between 0 and 1", expand->wl_scale);
+ CHECK((expand->wl_scale <= 0) || (expand->wl_scale >= 1));
+
+ /* if there is no temperature control, we need to specify an MC temperature */
+ sprintf(err_buf, "If there is no temperature control, and lmc-mcmove!= 'no',mc_temperature must be set to a positive number");
+ if (expand->nstTij > 0)
+ {
+ sprintf(err_buf, "nst-transition-matrix (%d) must be an integer multiple of nstlog (%d)",
+ expand->nstTij, ir->nstlog);
+ CHECK((mod(expand->nstTij, ir->nstlog) != 0));
+ }
+ }
+
+ /* PBC/WALLS */
+ sprintf(err_buf, "walls only work with pbc=%s", epbc_names[epbcXY]);
+ CHECK(ir->nwall && ir->ePBC != epbcXY);
+
+ /* VACUUM STUFF */
+ if (ir->ePBC != epbcXYZ && ir->nwall != 2)
+ {
+ if (ir->ePBC == epbcNONE)
+ {
+ if (ir->epc != epcNO)
+ {
+ warning(wi, "Turning off pressure coupling for vacuum system");
+ ir->epc = epcNO;
+ }
+ }
+ else
+ {
+ sprintf(err_buf, "Can not have pressure coupling with pbc=%s",
+ epbc_names[ir->ePBC]);
+ CHECK(ir->epc != epcNO);
+ }
+ sprintf(err_buf, "Can not have Ewald with pbc=%s", epbc_names[ir->ePBC]);
+ CHECK(EEL_FULL(ir->coulombtype));
+
+ sprintf(err_buf, "Can not have dispersion correction with pbc=%s",
+ epbc_names[ir->ePBC]);
+ CHECK(ir->eDispCorr != edispcNO);
+ }
+
+ if (ir->rlist == 0.0)
+ {
+ sprintf(err_buf, "can only have neighborlist cut-off zero (=infinite)\n"
+ "with coulombtype = %s or coulombtype = %s\n"
+ "without periodic boundary conditions (pbc = %s) and\n"
+ "rcoulomb and rvdw set to zero",
+ eel_names[eelCUT], eel_names[eelUSER], epbc_names[epbcNONE]);
+ CHECK(((ir->coulombtype != eelCUT) && (ir->coulombtype != eelUSER)) ||
+ (ir->ePBC != epbcNONE) ||
+ (ir->rcoulomb != 0.0) || (ir->rvdw != 0.0));
+
+ if (ir->nstlist < 0)
+ {
+ warning_error(wi, "Can not have heuristic neighborlist updates without cut-off");
+ }
+ if (ir->nstlist > 0)
+ {
+ warning_note(wi, "Simulating without cut-offs is usually (slightly) faster with nstlist=0, nstype=simple and particle decomposition");
+ }
+ }
+
+ /* COMM STUFF */
+ if (ir->nstcomm == 0)
+ {
+ ir->comm_mode = ecmNO;
+ }
+ if (ir->comm_mode != ecmNO)
+ {
+ if (ir->nstcomm < 0)
+ {
+ warning(wi, "If you want to remove the rotation around the center of mass, you should set comm_mode = Angular instead of setting nstcomm < 0. nstcomm is modified to its absolute value");
+ ir->nstcomm = abs(ir->nstcomm);
+ }
+
+ if (ir->nstcalcenergy > 0 && ir->nstcomm < ir->nstcalcenergy)
+ {
+ warning_note(wi, "nstcomm < nstcalcenergy defeats the purpose of nstcalcenergy, setting nstcomm to nstcalcenergy");
+ ir->nstcomm = ir->nstcalcenergy;
+ }
+
+ if (ir->comm_mode == ecmANGULAR)
+ {
+ sprintf(err_buf, "Can not remove the rotation around the center of mass with periodic molecules");
+ CHECK(ir->bPeriodicMols);
+ if (ir->ePBC != epbcNONE)
+ {
+ warning(wi, "Removing the rotation around the center of mass in a periodic system (this is not a problem when you have only one molecule).");
+ }
+ }
+ }
+
+ if (EI_STATE_VELOCITY(ir->eI) && ir->ePBC == epbcNONE && ir->comm_mode != ecmANGULAR)
+ {
+ warning_note(wi, "Tumbling and or flying ice-cubes: We are not removing rotation around center of mass in a non-periodic system. You should probably set comm_mode = ANGULAR.");
+ }
+
+ sprintf(err_buf, "Twin-range neighbour searching (NS) with simple NS"
+ " algorithm not implemented");
+ CHECK(((ir->rcoulomb > ir->rlist) || (ir->rvdw > ir->rlist))
+ && (ir->ns_type == ensSIMPLE));
+
+ /* TEMPERATURE COUPLING */
+ if (ir->etc == etcYES)
+ {
+ ir->etc = etcBERENDSEN;
+ warning_note(wi, "Old option for temperature coupling given: "
+ "changing \"yes\" to \"Berendsen\"\n");
+ }
+
+ if ((ir->etc == etcNOSEHOOVER) || (ir->epc == epcMTTK))
+ {
+ if (ir->opts.nhchainlength < 1)
+ {
+ sprintf(warn_buf, "number of Nose-Hoover chains (currently %d) cannot be less than 1,reset to 1\n", ir->opts.nhchainlength);
+ ir->opts.nhchainlength = 1;
+ warning(wi, warn_buf);
+ }
+
+ if (ir->etc == etcNOSEHOOVER && !EI_VV(ir->eI) && ir->opts.nhchainlength > 1)
+ {
+ warning_note(wi, "leapfrog does not yet support Nose-Hoover chains, nhchainlength reset to 1");
+ ir->opts.nhchainlength = 1;
+ }
+ }
+ else
+ {
+ ir->opts.nhchainlength = 0;
+ }
+
+ if (ir->eI == eiVVAK)
+ {
+ sprintf(err_buf, "%s implemented primarily for validation, and requires nsttcouple = 1 and nstpcouple = 1.",
+ ei_names[eiVVAK]);
+ CHECK((ir->nsttcouple != 1) || (ir->nstpcouple != 1));
+ }
+
+ if (ETC_ANDERSEN(ir->etc))
+ {
+ sprintf(err_buf, "%s temperature control not supported for integrator %s.", etcoupl_names[ir->etc], ei_names[ir->eI]);
+ CHECK(!(EI_VV(ir->eI)));
+
+ for (i = 0; i < ir->opts.ngtc; i++)
+ {
+ sprintf(err_buf, "all tau_t must currently be equal using Andersen temperature control, violated for group %d", i);
+ CHECK(ir->opts.tau_t[0] != ir->opts.tau_t[i]);
+ sprintf(err_buf, "all tau_t must be postive using Andersen temperature control, tau_t[%d]=%10.6f",
+ i, ir->opts.tau_t[i]);
+ CHECK(ir->opts.tau_t[i] < 0);
+ }
+ if (ir->nstcomm > 0 && (ir->etc == etcANDERSEN))
+ {
+ sprintf(warn_buf, "Center of mass removal not necessary for %s. All velocities of coupled groups are rerandomized periodically, so flying ice cube errors will not occur.", etcoupl_names[ir->etc]);
+ warning_note(wi, warn_buf);
+ }
+
+ sprintf(err_buf, "nstcomm must be 1, not %d for %s, as velocities of atoms in coupled groups are randomized every time step", ir->nstcomm, etcoupl_names[ir->etc]);
+ CHECK(ir->nstcomm > 1 && (ir->etc == etcANDERSEN));
+
+ for (i = 0; i < ir->opts.ngtc; i++)
+ {
+ int nsteps = (int)(ir->opts.tau_t[i]/ir->delta_t);
+ sprintf(err_buf, "tau_t/delta_t for group %d for temperature control method %s must be a multiple of nstcomm (%d), as velocities of atoms in coupled groups are randomized every time step. The input tau_t (%8.3f) leads to %d steps per randomization", i, etcoupl_names[ir->etc], ir->nstcomm, ir->opts.tau_t[i], nsteps);
+ CHECK((nsteps % ir->nstcomm) && (ir->etc == etcANDERSENMASSIVE));
+ }
+ }
+ if (ir->etc == etcBERENDSEN)
+ {
+ sprintf(warn_buf, "The %s thermostat does not generate the correct kinetic energy distribution. You might want to consider using the %s thermostat.",
+ ETCOUPLTYPE(ir->etc), ETCOUPLTYPE(etcVRESCALE));
+ warning_note(wi, warn_buf);
+ }
+
+ if ((ir->etc == etcNOSEHOOVER || ETC_ANDERSEN(ir->etc))
+ && ir->epc == epcBERENDSEN)
+ {
+ sprintf(warn_buf, "Using Berendsen pressure coupling invalidates the "
+ "true ensemble for the thermostat");
+ warning(wi, warn_buf);
+ }
+
+ /* PRESSURE COUPLING */
+ if (ir->epc == epcISOTROPIC)
+ {
+ ir->epc = epcBERENDSEN;
+ warning_note(wi, "Old option for pressure coupling given: "
+ "changing \"Isotropic\" to \"Berendsen\"\n");
+ }
+
+ if (ir->epc != epcNO)
+ {
+ dt_pcoupl = ir->nstpcouple*ir->delta_t;
+
+ sprintf(err_buf, "tau-p must be > 0 instead of %g\n", ir->tau_p);
+ CHECK(ir->tau_p <= 0);
+
+ if (ir->tau_p/dt_pcoupl < pcouple_min_integration_steps(ir->epc))
+ {
+ sprintf(warn_buf, "For proper integration of the %s barostat, tau-p (%g) should be at least %d times larger than nstpcouple*dt (%g)",
+ EPCOUPLTYPE(ir->epc), ir->tau_p, pcouple_min_integration_steps(ir->epc), dt_pcoupl);
+ warning(wi, warn_buf);
+ }
+
+ sprintf(err_buf, "compressibility must be > 0 when using pressure"
+ " coupling %s\n", EPCOUPLTYPE(ir->epc));
+ CHECK(ir->compress[XX][XX] < 0 || ir->compress[YY][YY] < 0 ||
+ ir->compress[ZZ][ZZ] < 0 ||
+ (trace(ir->compress) == 0 && ir->compress[YY][XX] <= 0 &&
+ ir->compress[ZZ][XX] <= 0 && ir->compress[ZZ][YY] <= 0));
+
+ if (epcPARRINELLORAHMAN == ir->epc && opts->bGenVel)
+ {
+ sprintf(warn_buf,
+ "You are generating velocities so I am assuming you "
+ "are equilibrating a system. You are using "
+ "%s pressure coupling, but this can be "
+ "unstable for equilibration. If your system crashes, try "
+ "equilibrating first with Berendsen pressure coupling. If "
+ "you are not equilibrating the system, you can probably "
+ "ignore this warning.",
+ epcoupl_names[ir->epc]);
+ warning(wi, warn_buf);
+ }
+ }
+
+ if (EI_VV(ir->eI))
+ {
+ if (ir->epc > epcNO)
+ {
+ if ((ir->epc != epcBERENDSEN) && (ir->epc != epcMTTK))
+ {
+ warning_error(wi, "for md-vv and md-vv-avek, can only use Berendsen and Martyna-Tuckerman-Tobias-Klein (MTTK) equations for pressure control; MTTK is equivalent to Parrinello-Rahman.");
+ }
+ }
+ }
+
+ /* ELECTROSTATICS */
+ /* More checks are in triple check (grompp.c) */
+
+ if (ir->coulombtype == eelSWITCH)
+ {
+ sprintf(warn_buf, "coulombtype = %s is only for testing purposes and can lead to serious "
+ "artifacts, advice: use coulombtype = %s",
+ eel_names[ir->coulombtype],
+ eel_names[eelRF_ZERO]);
+ warning(wi, warn_buf);
+ }
+
+ if (ir->epsilon_r != 1 && ir->implicit_solvent == eisGBSA)
+ {
+ sprintf(warn_buf, "epsilon-r = %g with GB implicit solvent, will use this value for inner dielectric", ir->epsilon_r);
+ warning_note(wi, warn_buf);
+ }
+
+ if (EEL_RF(ir->coulombtype) && ir->epsilon_rf == 1 && ir->epsilon_r != 1)
+ {
+ sprintf(warn_buf, "epsilon-r = %g and epsilon-rf = 1 with reaction field, proceeding assuming old format and exchanging epsilon-r and epsilon-rf", ir->epsilon_r);
+ warning(wi, warn_buf);
+ ir->epsilon_rf = ir->epsilon_r;
+ ir->epsilon_r = 1.0;
+ }
+
+ if (getenv("GALACTIC_DYNAMICS") == NULL)
+ {
+ sprintf(err_buf, "epsilon-r must be >= 0 instead of %g\n", ir->epsilon_r);
+ CHECK(ir->epsilon_r < 0);
+ }
+
+ if (EEL_RF(ir->coulombtype))
+ {
+ /* reaction field (at the cut-off) */
+
+ if (ir->coulombtype == eelRF_ZERO)
+ {
+ sprintf(warn_buf, "With coulombtype = %s, epsilon-rf must be 0, assuming you meant epsilon_rf=0",
+ eel_names[ir->coulombtype]);
+ CHECK(ir->epsilon_rf != 0);
+ ir->epsilon_rf = 0.0;
+ }
+
+ sprintf(err_buf, "epsilon-rf must be >= epsilon-r");
+ CHECK((ir->epsilon_rf < ir->epsilon_r && ir->epsilon_rf != 0) ||
+ (ir->epsilon_r == 0));
+ if (ir->epsilon_rf == ir->epsilon_r)
+ {
+ sprintf(warn_buf, "Using epsilon-rf = epsilon-r with %s does not make sense",
+ eel_names[ir->coulombtype]);
+ warning(wi, warn_buf);
+ }
+ }
+ /* Allow rlist>rcoulomb for tabulated long range stuff. This just
+ * means the interaction is zero outside rcoulomb, but it helps to
+ * provide accurate energy conservation.
+ */
+ if (EEL_MIGHT_BE_ZERO_AT_CUTOFF(ir->coulombtype))
+ {
+ if (EEL_SWITCHED(ir->coulombtype))
+ {
+ sprintf(err_buf,
+ "With coulombtype = %s rcoulomb_switch must be < rcoulomb. Or, better: Use the potential modifier options!",
+ eel_names[ir->coulombtype]);
+ CHECK(ir->rcoulomb_switch >= ir->rcoulomb);
+ }
+ }
+ else if (ir->coulombtype == eelCUT || EEL_RF(ir->coulombtype))
+ {
+ if (ir->cutoff_scheme == ecutsGROUP && ir->coulomb_modifier == eintmodNONE)
+ {
+ sprintf(err_buf, "With coulombtype = %s, rcoulomb should be >= rlist unless you use a potential modifier",
+ eel_names[ir->coulombtype]);
+ CHECK(ir->rlist > ir->rcoulomb);
+ }
+ }
+
+ if (ir->coulombtype == eelSWITCH || ir->coulombtype == eelSHIFT ||
+ ir->vdwtype == evdwSWITCH || ir->vdwtype == evdwSHIFT)
+ {
+ sprintf(warn_buf,
+ "The switch/shift interaction settings are just for compatibility; you will get better "
+ "performance from applying potential modifiers to your interactions!\n");
+ warning_note(wi, warn_buf);
+ }
+
+ if (ir->coulombtype == eelPMESWITCH)
+ {
+ if (ir->rcoulomb_switch/ir->rcoulomb < 0.9499)
+ {
+ sprintf(warn_buf, "The switching range for %s should be 5%% or less, energy conservation will be good anyhow, since ewald_rtol = %g",
+ eel_names[ir->coulombtype],
+ ir->ewald_rtol);
+ warning(wi, warn_buf);
+ }
+ }
+
+ if (EEL_FULL(ir->coulombtype))
+ {
+ if (ir->coulombtype == eelPMESWITCH || ir->coulombtype == eelPMEUSER ||
+ ir->coulombtype == eelPMEUSERSWITCH)
+ {
+ sprintf(err_buf, "With coulombtype = %s, rcoulomb must be <= rlist",
+ eel_names[ir->coulombtype]);
+ CHECK(ir->rcoulomb > ir->rlist);
+ }
+ else if (ir->cutoff_scheme == ecutsGROUP && ir->coulomb_modifier == eintmodNONE)
+ {
+ if (ir->coulombtype == eelPME || ir->coulombtype == eelP3M_AD)
+ {
+ sprintf(err_buf,
+ "With coulombtype = %s (without modifier), rcoulomb must be equal to rlist,\n"
+ "or rlistlong if nstcalclr=1. For optimal energy conservation,consider using\n"
+ "a potential modifier.", eel_names[ir->coulombtype]);
+ if (ir->nstcalclr == 1)
+ {
+ CHECK(ir->rcoulomb != ir->rlist && ir->rcoulomb != ir->rlistlong);
+ }
+ else
+ {
+ CHECK(ir->rcoulomb != ir->rlist);
+ }
+ }
+ }
+ }
+
+ if (EEL_PME(ir->coulombtype))
+ {
+ if (ir->pme_order < 3)
+ {
+ warning_error(wi, "pme-order can not be smaller than 3");
+ }
+ }
+
+ if (ir->nwall == 2 && EEL_FULL(ir->coulombtype))
+ {
+ if (ir->ewald_geometry == eewg3D)
+ {
+ sprintf(warn_buf, "With pbc=%s you should use ewald-geometry=%s",
+ epbc_names[ir->ePBC], eewg_names[eewg3DC]);
+ warning(wi, warn_buf);
+ }
+ /* This check avoids extra pbc coding for exclusion corrections */
+ sprintf(err_buf, "wall-ewald-zfac should be >= 2");
+ CHECK(ir->wall_ewald_zfac < 2);
+ }
+
+ if (EVDW_SWITCHED(ir->vdwtype))
+ {
+ sprintf(err_buf, "With vdwtype = %s rvdw-switch must be < rvdw. Or, better - use a potential modifier.",
+ evdw_names[ir->vdwtype]);
+ CHECK(ir->rvdw_switch >= ir->rvdw);
+ }
+ else if (ir->vdwtype == evdwCUT)
+ {
+ if (ir->cutoff_scheme == ecutsGROUP && ir->vdw_modifier == eintmodNONE)
+ {
+ sprintf(err_buf, "With vdwtype = %s, rvdw must be >= rlist unless you use a potential modifier", evdw_names[ir->vdwtype]);
+ CHECK(ir->rlist > ir->rvdw);
+ }
+ }
+ if (ir->cutoff_scheme == ecutsGROUP)
+ {
+ if (((ir->coulomb_modifier != eintmodNONE && ir->rcoulomb == ir->rlist) ||
+ (ir->vdw_modifier != eintmodNONE && ir->rvdw == ir->rlist)) &&
+ ir->nstlist != 1)
+ {
+ warning_note(wi, "With exact cut-offs, rlist should be "
+ "larger than rcoulomb and rvdw, so that there "
+ "is a buffer region for particle motion "
+ "between neighborsearch steps");
+ }
+
+ if (EEL_IS_ZERO_AT_CUTOFF(ir->coulombtype)
+ && (ir->rlistlong <= ir->rcoulomb))
+ {
+ sprintf(warn_buf, "For energy conservation with switch/shift potentials, %s should be 0.1 to 0.3 nm larger than rcoulomb.",
+ IR_TWINRANGE(*ir) ? "rlistlong" : "rlist");
+ warning_note(wi, warn_buf);
+ }
+ if (EVDW_SWITCHED(ir->vdwtype) && (ir->rlistlong <= ir->rvdw))
+ {
+ sprintf(warn_buf, "For energy conservation with switch/shift potentials, %s should be 0.1 to 0.3 nm larger than rvdw.",
+ IR_TWINRANGE(*ir) ? "rlistlong" : "rlist");
+ warning_note(wi, warn_buf);
+ }
+ }
+
+ if (ir->vdwtype == evdwUSER && ir->eDispCorr != edispcNO)
+ {
+ warning_note(wi, "You have selected user tables with dispersion correction, the dispersion will be corrected to -C6/r^6 beyond rvdw_switch (the tabulated interaction between rvdw_switch and rvdw will not be double counted). Make sure that you really want dispersion correction to -C6/r^6.");
+ }
+
+ if (ir->nstlist == -1)
+ {
+ sprintf(err_buf, "With nstlist=-1 rvdw and rcoulomb should be smaller than rlist to account for diffusion and possibly charge-group radii");
+ CHECK(ir->rvdw >= ir->rlist || ir->rcoulomb >= ir->rlist);
+ }
+ sprintf(err_buf, "nstlist can not be smaller than -1");
+ CHECK(ir->nstlist < -1);
+
+ if (ir->eI == eiLBFGS && (ir->coulombtype == eelCUT || ir->vdwtype == evdwCUT)
+ && ir->rvdw != 0)
+ {
+ warning(wi, "For efficient BFGS minimization, use switch/shift/pme instead of cut-off.");
+ }
+
+ if (ir->eI == eiLBFGS && ir->nbfgscorr <= 0)
+ {
+ warning(wi, "Using L-BFGS with nbfgscorr<=0 just gets you steepest descent.");
+ }
+
+ /* ENERGY CONSERVATION */
+ if (ir_NVE(ir) && ir->cutoff_scheme == ecutsGROUP)
+ {
+ if (!EVDW_MIGHT_BE_ZERO_AT_CUTOFF(ir->vdwtype) && ir->rvdw > 0 && ir->vdw_modifier == eintmodNONE)
+ {
+ sprintf(warn_buf, "You are using a cut-off for VdW interactions with NVE, for good energy conservation use vdwtype = %s (possibly with DispCorr)",
+ evdw_names[evdwSHIFT]);
+ warning_note(wi, warn_buf);
+ }
+ if (!EEL_MIGHT_BE_ZERO_AT_CUTOFF(ir->coulombtype) && ir->rcoulomb > 0 && ir->coulomb_modifier == eintmodNONE)
+ {
+ sprintf(warn_buf, "You are using a cut-off for electrostatics with NVE, for good energy conservation use coulombtype = %s or %s",
+ eel_names[eelPMESWITCH], eel_names[eelRF_ZERO]);
+ warning_note(wi, warn_buf);
+ }
+ }
+
+ /* IMPLICIT SOLVENT */
+ if (ir->coulombtype == eelGB_NOTUSED)
+ {
+ ir->coulombtype = eelCUT;
+ ir->implicit_solvent = eisGBSA;
+ fprintf(stderr, "Note: Old option for generalized born electrostatics given:\n"
+ "Changing coulombtype from \"generalized-born\" to \"cut-off\" and instead\n"
+ "setting implicit-solvent value to \"GBSA\" in input section.\n");
+ }
+
+ if (ir->sa_algorithm == esaSTILL)
+ {
+ sprintf(err_buf, "Still SA algorithm not available yet, use %s or %s instead\n", esa_names[esaAPPROX], esa_names[esaNO]);
+ CHECK(ir->sa_algorithm == esaSTILL);
+ }
+
+ if (ir->implicit_solvent == eisGBSA)
+ {
+ sprintf(err_buf, "With GBSA implicit solvent, rgbradii must be equal to rlist.");
+ CHECK(ir->rgbradii != ir->rlist);
+
+ if (ir->coulombtype != eelCUT)
+ {
+ sprintf(err_buf, "With GBSA, coulombtype must be equal to %s\n", eel_names[eelCUT]);
+ CHECK(ir->coulombtype != eelCUT);
+ }
+ if (ir->vdwtype != evdwCUT)
+ {
+ sprintf(err_buf, "With GBSA, vdw-type must be equal to %s\n", evdw_names[evdwCUT]);
+ CHECK(ir->vdwtype != evdwCUT);
+ }
+ if (ir->nstgbradii < 1)
+ {
+ sprintf(warn_buf, "Using GBSA with nstgbradii<1, setting nstgbradii=1");
+ warning_note(wi, warn_buf);
+ ir->nstgbradii = 1;
+ }
+ if (ir->sa_algorithm == esaNO)
+ {
+ sprintf(warn_buf, "No SA (non-polar) calculation requested together with GB. Are you sure this is what you want?\n");
+ warning_note(wi, warn_buf);
+ }
+ if (ir->sa_surface_tension < 0 && ir->sa_algorithm != esaNO)
+ {
+ sprintf(warn_buf, "Value of sa_surface_tension is < 0. Changing it to 2.05016 or 2.25936 kJ/nm^2/mol for Still and HCT/OBC respectively\n");
+ warning_note(wi, warn_buf);
+
+ if (ir->gb_algorithm == egbSTILL)
+ {
+ ir->sa_surface_tension = 0.0049 * CAL2JOULE * 100;
+ }
+ else
+ {
+ ir->sa_surface_tension = 0.0054 * CAL2JOULE * 100;
+ }
+ }
+ if (ir->sa_surface_tension == 0 && ir->sa_algorithm != esaNO)
+ {
+ sprintf(err_buf, "Surface tension set to 0 while SA-calculation requested\n");
+ CHECK(ir->sa_surface_tension == 0 && ir->sa_algorithm != esaNO);
+ }
+
+ }
+
+ if (ir->bAdress)
+ {
+ if (ir->cutoff_scheme != ecutsGROUP)
+ {
+ warning_error(wi, "AdresS simulation supports only cutoff-scheme=group");
+ }
+ if (!EI_SD(ir->eI))
+ {
+ warning_error(wi, "AdresS simulation supports only stochastic dynamics");
+ }
+ if (ir->epc != epcNO)
+ {
+ warning_error(wi, "AdresS simulation does not support pressure coupling");
+ }
+ if (EEL_FULL(ir->coulombtype))
+ {
+ warning_error(wi, "AdresS simulation does not support long-range electrostatics");
+ }
+ }
+}
+
+/* count the number of text elemets separated by whitespace in a string.
+ str = the input string
+ maxptr = the maximum number of allowed elements
+ ptr = the output array of pointers to the first character of each element
+ returns: the number of elements. */
+int str_nelem(const char *str, int maxptr, char *ptr[])
+{
+ int np = 0;
+ char *copy0, *copy;
+
+ copy0 = strdup(str);
+ copy = copy0;
+ ltrim(copy);
+ while (*copy != '\0')
+ {
+ if (np >= maxptr)
+ {
+ gmx_fatal(FARGS, "Too many groups on line: '%s' (max is %d)",
+ str, maxptr);
+ }
+ if (ptr)
+ {
+ ptr[np] = copy;
+ }
+ np++;
+ while ((*copy != '\0') && !isspace(*copy))
+ {
+ copy++;
+ }
+ if (*copy != '\0')
+ {
+ *copy = '\0';
+ copy++;
+ }
+ ltrim(copy);
+ }
+ if (ptr == NULL)
+ {
+ sfree(copy0);
+ }
+
+ return np;
+}
+
+/* interpret a number of doubles from a string and put them in an array,
+ after allocating space for them.
+ str = the input string
+ n = the (pre-allocated) number of doubles read
+ r = the output array of doubles. */
+static void parse_n_real(char *str, int *n, real **r)
+{
+ char *ptr[MAXPTR];
+ int i;
+
+ *n = str_nelem(str, MAXPTR, ptr);
+
+ snew(*r, *n);
+ for (i = 0; i < *n; i++)
+ {
+ (*r)[i] = strtod(ptr[i], NULL);
+ }
+}
+
+static void do_fep_params(t_inputrec *ir, char fep_lambda[][STRLEN], char weights[STRLEN])
+{
+
+ int i, j, max_n_lambda, nweights, nfep[efptNR];
+ t_lambda *fep = ir->fepvals;
+ t_expanded *expand = ir->expandedvals;
+ real **count_fep_lambdas;
+ gmx_bool bOneLambda = TRUE;
+
+ snew(count_fep_lambdas, efptNR);
+
+ /* FEP input processing */
+ /* first, identify the number of lambda values for each type.
+ All that are nonzero must have the same number */
+
+ for (i = 0; i < efptNR; i++)
+ {
+ parse_n_real(fep_lambda[i], &(nfep[i]), &(count_fep_lambdas[i]));
+ }
+
+ /* now, determine the number of components. All must be either zero, or equal. */
+
+ max_n_lambda = 0;
+ for (i = 0; i < efptNR; i++)
+ {
+ if (nfep[i] > max_n_lambda)
+ {
+ max_n_lambda = nfep[i]; /* here's a nonzero one. All of them
+ must have the same number if its not zero.*/
+ break;
+ }
+ }
+
+ for (i = 0; i < efptNR; i++)
+ {
+ if (nfep[i] == 0)
+ {
+ ir->fepvals->separate_dvdl[i] = FALSE;
+ }
+ else if (nfep[i] == max_n_lambda)
+ {
+ if (i != efptTEMPERATURE) /* we treat this differently -- not really a reason to compute the derivative with
+ respect to the temperature currently */
+ {
+ ir->fepvals->separate_dvdl[i] = TRUE;
+ }
+ }
+ else
+ {
+ gmx_fatal(FARGS, "Number of lambdas (%d) for FEP type %s not equal to number of other types (%d)",
+ nfep[i], efpt_names[i], max_n_lambda);
+ }
+ }
+ /* we don't print out dhdl if the temperature is changing, since we can't correctly define dhdl in this case */
+ ir->fepvals->separate_dvdl[efptTEMPERATURE] = FALSE;
+
+ /* the number of lambdas is the number we've read in, which is either zero
+ or the same for all */
+ fep->n_lambda = max_n_lambda;
+
+ /* allocate space for the array of lambda values */
+ snew(fep->all_lambda, efptNR);
+ /* if init_lambda is defined, we need to set lambda */
+ if ((fep->init_lambda > 0) && (fep->n_lambda == 0))
+ {
+ ir->fepvals->separate_dvdl[efptFEP] = TRUE;
+ }
+ /* otherwise allocate the space for all of the lambdas, and transfer the data */
+ for (i = 0; i < efptNR; i++)
+ {
+ snew(fep->all_lambda[i], fep->n_lambda);
+ if (nfep[i] > 0) /* if it's zero, then the count_fep_lambda arrays
+ are zero */
+ {
+ for (j = 0; j < fep->n_lambda; j++)
+ {
+ fep->all_lambda[i][j] = (double)count_fep_lambdas[i][j];
+ }
+ sfree(count_fep_lambdas[i]);
+ }
+ }
+ sfree(count_fep_lambdas);
+
+ /* "fep-vals" is either zero or the full number. If zero, we'll need to define fep-lambdas for internal
+ bookkeeping -- for now, init_lambda */
+
+ if ((nfep[efptFEP] == 0) && (fep->init_lambda >= 0))
+ {
+ for (i = 0; i < fep->n_lambda; i++)
+ {
+ fep->all_lambda[efptFEP][i] = fep->init_lambda;
+ }
+ }
+
+ /* check to see if only a single component lambda is defined, and soft core is defined.
+ In this case, turn on coulomb soft core */
+
+ if (max_n_lambda == 0)
+ {
+ bOneLambda = TRUE;
+ }
+ else
+ {
+ for (i = 0; i < efptNR; i++)
+ {
+ if ((nfep[i] != 0) && (i != efptFEP))
+ {
+ bOneLambda = FALSE;
+ }
+ }
+ }
+ if ((bOneLambda) && (fep->sc_alpha > 0))
+ {
+ fep->bScCoul = TRUE;
+ }
+
+ /* Fill in the others with the efptFEP if they are not explicitly
+ specified (i.e. nfep[i] == 0). This means if fep is not defined,
+ they are all zero. */
+
+ for (i = 0; i < efptNR; i++)
+ {
+ if ((nfep[i] == 0) && (i != efptFEP))
+ {
+ for (j = 0; j < fep->n_lambda; j++)
+ {
+ fep->all_lambda[i][j] = fep->all_lambda[efptFEP][j];
+ }
+ }
+ }
+
+
+ /* make it easier if sc_r_power = 48 by increasing it to the 4th power, to be in the right scale. */
+ if (fep->sc_r_power == 48)
+ {
+ if (fep->sc_alpha > 0.1)
+ {
+ gmx_fatal(FARGS, "sc_alpha (%f) for sc_r_power = 48 should usually be between 0.001 and 0.004", fep->sc_alpha);
+ }
+ }
+
+ expand = ir->expandedvals;
+ /* now read in the weights */
+ parse_n_real(weights, &nweights, &(expand->init_lambda_weights));
+ if (nweights == 0)
+ {
- else
- {
- expand->bInit_weights = TRUE;
- }
+ snew(expand->init_lambda_weights, fep->n_lambda); /* initialize to zero */
+ }
+ else if (nweights != fep->n_lambda)
+ {
+ gmx_fatal(FARGS, "Number of weights (%d) is not equal to number of lambda values (%d)",
+ nweights, fep->n_lambda);
+ }
+ if ((expand->nstexpanded < 0) && (ir->efep != efepNO))
+ {
+ expand->nstexpanded = fep->nstdhdl;
+ /* if you don't specify nstexpanded when doing expanded ensemble free energy calcs, it is set to nstdhdl */
+ }
+ if ((expand->nstexpanded < 0) && ir->bSimTemp)
+ {
+ expand->nstexpanded = 2*(int)(ir->opts.tau_t[0]/ir->delta_t);
+ /* if you don't specify nstexpanded when doing expanded ensemble simulated tempering, it is set to
+ 2*tau_t just to be careful so it's not to frequent */
+ }
+}
+
+
+static void do_simtemp_params(t_inputrec *ir)
+{
+
+ snew(ir->simtempvals->temperatures, ir->fepvals->n_lambda);
+ GetSimTemps(ir->fepvals->n_lambda, ir->simtempvals, ir->fepvals->all_lambda[efptTEMPERATURE]);
+
+ return;
+}
+
+static void do_wall_params(t_inputrec *ir,
+ char *wall_atomtype, char *wall_density,
+ t_gromppopts *opts)
+{
+ int nstr, i;
+ char *names[MAXPTR];
+ double dbl;
+
+ opts->wall_atomtype[0] = NULL;
+ opts->wall_atomtype[1] = NULL;
+
+ ir->wall_atomtype[0] = -1;
+ ir->wall_atomtype[1] = -1;
+ ir->wall_density[0] = 0;
+ ir->wall_density[1] = 0;
+
+ if (ir->nwall > 0)
+ {
+ nstr = str_nelem(wall_atomtype, MAXPTR, names);
+ if (nstr != ir->nwall)
+ {
+ gmx_fatal(FARGS, "Expected %d elements for wall_atomtype, found %d",
+ ir->nwall, nstr);
+ }
+ for (i = 0; i < ir->nwall; i++)
+ {
+ opts->wall_atomtype[i] = strdup(names[i]);
+ }
+
+ if (ir->wall_type == ewt93 || ir->wall_type == ewt104)
+ {
+ nstr = str_nelem(wall_density, MAXPTR, names);
+ if (nstr != ir->nwall)
+ {
+ gmx_fatal(FARGS, "Expected %d elements for wall-density, found %d", ir->nwall, nstr);
+ }
+ for (i = 0; i < ir->nwall; i++)
+ {
+ sscanf(names[i], "%lf", &dbl);
+ if (dbl <= 0)
+ {
+ gmx_fatal(FARGS, "wall-density[%d] = %f\n", i, dbl);
+ }
+ ir->wall_density[i] = dbl;
+ }
+ }
+ }
+}
+
+static void add_wall_energrps(gmx_groups_t *groups, int nwall, t_symtab *symtab)
+{
+ int i;
+ t_grps *grps;
+ char str[STRLEN];
+
+ if (nwall > 0)
+ {
+ srenew(groups->grpname, groups->ngrpname+nwall);
+ grps = &(groups->grps[egcENER]);
+ srenew(grps->nm_ind, grps->nr+nwall);
+ for (i = 0; i < nwall; i++)
+ {
+ sprintf(str, "wall%d", i);
+ groups->grpname[groups->ngrpname] = put_symtab(symtab, str);
+ grps->nm_ind[grps->nr++] = groups->ngrpname++;
+ }
+ }
+}
+
+void read_expandedparams(int *ninp_p, t_inpfile **inp_p,
+ t_expanded *expand, warninp_t wi)
+{
+ int ninp, nerror = 0;
+ t_inpfile *inp;
+
+ ninp = *ninp_p;
+ inp = *inp_p;
+
+ /* read expanded ensemble parameters */
+ CCTYPE ("expanded ensemble variables");
+ ITYPE ("nstexpanded", expand->nstexpanded, -1);
+ EETYPE("lmc-stats", expand->elamstats, elamstats_names);
+ EETYPE("lmc-move", expand->elmcmove, elmcmove_names);
+ EETYPE("lmc-weights-equil", expand->elmceq, elmceq_names);
+ ITYPE ("weight-equil-number-all-lambda", expand->equil_n_at_lam, -1);
+ ITYPE ("weight-equil-number-samples", expand->equil_samples, -1);
+ ITYPE ("weight-equil-number-steps", expand->equil_steps, -1);
+ RTYPE ("weight-equil-wl-delta", expand->equil_wl_delta, -1);
+ RTYPE ("weight-equil-count-ratio", expand->equil_ratio, -1);
+ CCTYPE("Seed for Monte Carlo in lambda space");
+ ITYPE ("lmc-seed", expand->lmc_seed, -1);
+ RTYPE ("mc-temperature", expand->mc_temp, -1);
+ ITYPE ("lmc-repeats", expand->lmc_repeats, 1);
+ ITYPE ("lmc-gibbsdelta", expand->gibbsdeltalam, -1);
+ ITYPE ("lmc-forced-nstart", expand->lmc_forced_nstart, 0);
+ EETYPE("symmetrized-transition-matrix", expand->bSymmetrizedTMatrix, yesno_names);
+ ITYPE("nst-transition-matrix", expand->nstTij, -1);
+ ITYPE ("mininum-var-min", expand->minvarmin, 100); /*default is reasonable */
+ ITYPE ("weight-c-range", expand->c_range, 0); /* default is just C=0 */
+ RTYPE ("wl-scale", expand->wl_scale, 0.8);
+ RTYPE ("wl-ratio", expand->wl_ratio, 0.8);
+ RTYPE ("init-wl-delta", expand->init_wl_delta, 1.0);
+ EETYPE("wl-oneovert", expand->bWLoneovert, yesno_names);
+
+ *ninp_p = ninp;
+ *inp_p = inp;
+
+ return;
+}
+
+void get_ir(const char *mdparin, const char *mdparout,
+ t_inputrec *ir, t_gromppopts *opts,
+ warninp_t wi)
+{
+ char *dumstr[2];
+ double dumdub[2][6];
+ t_inpfile *inp;
+ const char *tmp;
+ int i, j, m, ninp;
+ char warn_buf[STRLEN];
+ t_lambda *fep = ir->fepvals;
+ t_expanded *expand = ir->expandedvals;
+
+ inp = read_inpfile(mdparin, &ninp, wi);
+
+ snew(dumstr[0], STRLEN);
+ snew(dumstr[1], STRLEN);
+
+ /* remove the following deprecated commands */
+ REM_TYPE("title");
+ REM_TYPE("cpp");
+ REM_TYPE("domain-decomposition");
+ REM_TYPE("andersen-seed");
+ REM_TYPE("dihre");
+ REM_TYPE("dihre-fc");
+ REM_TYPE("dihre-tau");
+ REM_TYPE("nstdihreout");
+ REM_TYPE("nstcheckpoint");
+
+ /* replace the following commands with the clearer new versions*/
+ REPL_TYPE("unconstrained-start", "continuation");
+ REPL_TYPE("foreign-lambda", "fep-lambdas");
+
+ CCTYPE ("VARIOUS PREPROCESSING OPTIONS");
+ CTYPE ("Preprocessor information: use cpp syntax.");
+ CTYPE ("e.g.: -I/home/joe/doe -I/home/mary/roe");
+ STYPE ("include", opts->include, NULL);
+ CTYPE ("e.g.: -DPOSRES -DFLEXIBLE (note these variable names are case sensitive)");
+ STYPE ("define", opts->define, NULL);
+
+ CCTYPE ("RUN CONTROL PARAMETERS");
+ EETYPE("integrator", ir->eI, ei_names);
+ CTYPE ("Start time and timestep in ps");
+ RTYPE ("tinit", ir->init_t, 0.0);
+ RTYPE ("dt", ir->delta_t, 0.001);
+ STEPTYPE ("nsteps", ir->nsteps, 0);
+ CTYPE ("For exact run continuation or redoing part of a run");
+ STEPTYPE ("init-step", ir->init_step, 0);
+ CTYPE ("Part index is updated automatically on checkpointing (keeps files separate)");
+ ITYPE ("simulation-part", ir->simulation_part, 1);
+ CTYPE ("mode for center of mass motion removal");
+ EETYPE("comm-mode", ir->comm_mode, ecm_names);
+ CTYPE ("number of steps for center of mass motion removal");
+ ITYPE ("nstcomm", ir->nstcomm, 100);
+ CTYPE ("group(s) for center of mass motion removal");
+ STYPE ("comm-grps", vcm, NULL);
+
+ CCTYPE ("LANGEVIN DYNAMICS OPTIONS");
+ CTYPE ("Friction coefficient (amu/ps) and random seed");
+ RTYPE ("bd-fric", ir->bd_fric, 0.0);
+ ITYPE ("ld-seed", ir->ld_seed, 1993);
+
+ /* Em stuff */
+ CCTYPE ("ENERGY MINIMIZATION OPTIONS");
+ CTYPE ("Force tolerance and initial step-size");
+ RTYPE ("emtol", ir->em_tol, 10.0);
+ RTYPE ("emstep", ir->em_stepsize, 0.01);
+ CTYPE ("Max number of iterations in relax-shells");
+ ITYPE ("niter", ir->niter, 20);
+ CTYPE ("Step size (ps^2) for minimization of flexible constraints");
+ RTYPE ("fcstep", ir->fc_stepsize, 0);
+ CTYPE ("Frequency of steepest descents steps when doing CG");
+ ITYPE ("nstcgsteep", ir->nstcgsteep, 1000);
+ ITYPE ("nbfgscorr", ir->nbfgscorr, 10);
+
+ CCTYPE ("TEST PARTICLE INSERTION OPTIONS");
+ RTYPE ("rtpi", ir->rtpi, 0.05);
+
+ /* Output options */
+ CCTYPE ("OUTPUT CONTROL OPTIONS");
+ CTYPE ("Output frequency for coords (x), velocities (v) and forces (f)");
+ ITYPE ("nstxout", ir->nstxout, 0);
+ ITYPE ("nstvout", ir->nstvout, 0);
+ ITYPE ("nstfout", ir->nstfout, 0);
+ ir->nstcheckpoint = 1000;
+ CTYPE ("Output frequency for energies to log file and energy file");
+ ITYPE ("nstlog", ir->nstlog, 1000);
+ ITYPE ("nstcalcenergy", ir->nstcalcenergy, 100);
+ ITYPE ("nstenergy", ir->nstenergy, 1000);
+ CTYPE ("Output frequency and precision for .xtc file");
+ ITYPE ("nstxtcout", ir->nstxtcout, 0);
+ RTYPE ("xtc-precision", ir->xtcprec, 1000.0);
+ CTYPE ("This selects the subset of atoms for the .xtc file. You can");
+ CTYPE ("select multiple groups. By default all atoms will be written.");
+ STYPE ("xtc-grps", xtc_grps, NULL);
+ CTYPE ("Selection of energy groups");
+ STYPE ("energygrps", energy, NULL);
+
+ /* Neighbor searching */
+ CCTYPE ("NEIGHBORSEARCHING PARAMETERS");
+ CTYPE ("cut-off scheme (group: using charge groups, Verlet: particle based cut-offs)");
+ EETYPE("cutoff-scheme", ir->cutoff_scheme, ecutscheme_names);
+ CTYPE ("nblist update frequency");
+ ITYPE ("nstlist", ir->nstlist, 10);
+ CTYPE ("ns algorithm (simple or grid)");
+ EETYPE("ns-type", ir->ns_type, ens_names);
+ /* set ndelta to the optimal value of 2 */
+ ir->ndelta = 2;
+ CTYPE ("Periodic boundary conditions: xyz, no, xy");
+ EETYPE("pbc", ir->ePBC, epbc_names);
+ EETYPE("periodic-molecules", ir->bPeriodicMols, yesno_names);
+ CTYPE ("Allowed energy drift due to the Verlet buffer in kJ/mol/ps per atom,");
+ CTYPE ("a value of -1 means: use rlist");
+ RTYPE("verlet-buffer-drift", ir->verletbuf_drift, 0.005);
+ CTYPE ("nblist cut-off");
+ RTYPE ("rlist", ir->rlist, 1.0);
+ CTYPE ("long-range cut-off for switched potentials");
+ RTYPE ("rlistlong", ir->rlistlong, -1);
+ ITYPE ("nstcalclr", ir->nstcalclr, -1);
+
+ /* Electrostatics */
+ CCTYPE ("OPTIONS FOR ELECTROSTATICS AND VDW");
+ CTYPE ("Method for doing electrostatics");
+ EETYPE("coulombtype", ir->coulombtype, eel_names);
+ EETYPE("coulomb-modifier", ir->coulomb_modifier, eintmod_names);
+ CTYPE ("cut-off lengths");
+ RTYPE ("rcoulomb-switch", ir->rcoulomb_switch, 0.0);
+ RTYPE ("rcoulomb", ir->rcoulomb, 1.0);
+ CTYPE ("Relative dielectric constant for the medium and the reaction field");
+ RTYPE ("epsilon-r", ir->epsilon_r, 1.0);
+ RTYPE ("epsilon-rf", ir->epsilon_rf, 0.0);
+ CTYPE ("Method for doing Van der Waals");
+ EETYPE("vdw-type", ir->vdwtype, evdw_names);
+ EETYPE("vdw-modifier", ir->vdw_modifier, eintmod_names);
+ CTYPE ("cut-off lengths");
+ RTYPE ("rvdw-switch", ir->rvdw_switch, 0.0);
+ RTYPE ("rvdw", ir->rvdw, 1.0);
+ CTYPE ("Apply long range dispersion corrections for Energy and Pressure");
+ EETYPE("DispCorr", ir->eDispCorr, edispc_names);
+ CTYPE ("Extension of the potential lookup tables beyond the cut-off");
+ RTYPE ("table-extension", ir->tabext, 1.0);
+ CTYPE ("Separate tables between energy group pairs");
+ STYPE ("energygrp-table", egptable, NULL);
+ CTYPE ("Spacing for the PME/PPPM FFT grid");
+ RTYPE ("fourierspacing", ir->fourier_spacing, 0.12);
+ CTYPE ("FFT grid size, when a value is 0 fourierspacing will be used");
+ ITYPE ("fourier-nx", ir->nkx, 0);
+ ITYPE ("fourier-ny", ir->nky, 0);
+ ITYPE ("fourier-nz", ir->nkz, 0);
+ CTYPE ("EWALD/PME/PPPM parameters");
+ ITYPE ("pme-order", ir->pme_order, 4);
+ RTYPE ("ewald-rtol", ir->ewald_rtol, 0.00001);
+ EETYPE("ewald-geometry", ir->ewald_geometry, eewg_names);
+ RTYPE ("epsilon-surface", ir->epsilon_surface, 0.0);
+ EETYPE("optimize-fft", ir->bOptFFT, yesno_names);
+
+ CCTYPE("IMPLICIT SOLVENT ALGORITHM");
+ EETYPE("implicit-solvent", ir->implicit_solvent, eis_names);
+
+ CCTYPE ("GENERALIZED BORN ELECTROSTATICS");
+ CTYPE ("Algorithm for calculating Born radii");
+ EETYPE("gb-algorithm", ir->gb_algorithm, egb_names);
+ CTYPE ("Frequency of calculating the Born radii inside rlist");
+ ITYPE ("nstgbradii", ir->nstgbradii, 1);
+ CTYPE ("Cutoff for Born radii calculation; the contribution from atoms");
+ CTYPE ("between rlist and rgbradii is updated every nstlist steps");
+ RTYPE ("rgbradii", ir->rgbradii, 1.0);
+ CTYPE ("Dielectric coefficient of the implicit solvent");
+ RTYPE ("gb-epsilon-solvent", ir->gb_epsilon_solvent, 80.0);
+ CTYPE ("Salt concentration in M for Generalized Born models");
+ RTYPE ("gb-saltconc", ir->gb_saltconc, 0.0);
+ CTYPE ("Scaling factors used in the OBC GB model. Default values are OBC(II)");
+ RTYPE ("gb-obc-alpha", ir->gb_obc_alpha, 1.0);
+ RTYPE ("gb-obc-beta", ir->gb_obc_beta, 0.8);
+ RTYPE ("gb-obc-gamma", ir->gb_obc_gamma, 4.85);
+ RTYPE ("gb-dielectric-offset", ir->gb_dielectric_offset, 0.009);
+ EETYPE("sa-algorithm", ir->sa_algorithm, esa_names);
+ CTYPE ("Surface tension (kJ/mol/nm^2) for the SA (nonpolar surface) part of GBSA");
+ CTYPE ("The value -1 will set default value for Still/HCT/OBC GB-models.");
+ RTYPE ("sa-surface-tension", ir->sa_surface_tension, -1);
+
+ /* Coupling stuff */
+ CCTYPE ("OPTIONS FOR WEAK COUPLING ALGORITHMS");
+ CTYPE ("Temperature coupling");
+ EETYPE("tcoupl", ir->etc, etcoupl_names);
+ ITYPE ("nsttcouple", ir->nsttcouple, -1);
+ ITYPE("nh-chain-length", ir->opts.nhchainlength, NHCHAINLENGTH);
+ EETYPE("print-nose-hoover-chain-variables", ir->bPrintNHChains, yesno_names);
+ CTYPE ("Groups to couple separately");
+ STYPE ("tc-grps", tcgrps, NULL);
+ CTYPE ("Time constant (ps) and reference temperature (K)");
+ STYPE ("tau-t", tau_t, NULL);
+ STYPE ("ref-t", ref_t, NULL);
+ CTYPE ("pressure coupling");
+ EETYPE("pcoupl", ir->epc, epcoupl_names);
+ EETYPE("pcoupltype", ir->epct, epcoupltype_names);
+ ITYPE ("nstpcouple", ir->nstpcouple, -1);
+ CTYPE ("Time constant (ps), compressibility (1/bar) and reference P (bar)");
+ RTYPE ("tau-p", ir->tau_p, 1.0);
+ STYPE ("compressibility", dumstr[0], NULL);
+ STYPE ("ref-p", dumstr[1], NULL);
+ CTYPE ("Scaling of reference coordinates, No, All or COM");
+ EETYPE ("refcoord-scaling", ir->refcoord_scaling, erefscaling_names);
+
+ /* QMMM */
+ CCTYPE ("OPTIONS FOR QMMM calculations");
+ EETYPE("QMMM", ir->bQMMM, yesno_names);
+ CTYPE ("Groups treated Quantum Mechanically");
+ STYPE ("QMMM-grps", QMMM, NULL);
+ CTYPE ("QM method");
+ STYPE("QMmethod", QMmethod, NULL);
+ CTYPE ("QMMM scheme");
+ EETYPE("QMMMscheme", ir->QMMMscheme, eQMMMscheme_names);
+ CTYPE ("QM basisset");
+ STYPE("QMbasis", QMbasis, NULL);
+ CTYPE ("QM charge");
+ STYPE ("QMcharge", QMcharge, NULL);
+ CTYPE ("QM multiplicity");
+ STYPE ("QMmult", QMmult, NULL);
+ CTYPE ("Surface Hopping");
+ STYPE ("SH", bSH, NULL);
+ CTYPE ("CAS space options");
+ STYPE ("CASorbitals", CASorbitals, NULL);
+ STYPE ("CASelectrons", CASelectrons, NULL);
+ STYPE ("SAon", SAon, NULL);
+ STYPE ("SAoff", SAoff, NULL);
+ STYPE ("SAsteps", SAsteps, NULL);
+ CTYPE ("Scale factor for MM charges");
+ RTYPE ("MMChargeScaleFactor", ir->scalefactor, 1.0);
+ CTYPE ("Optimization of QM subsystem");
+ STYPE ("bOPT", bOPT, NULL);
+ STYPE ("bTS", bTS, NULL);
+
+ /* Simulated annealing */
+ CCTYPE("SIMULATED ANNEALING");
+ CTYPE ("Type of annealing for each temperature group (no/single/periodic)");
+ STYPE ("annealing", anneal, NULL);
+ CTYPE ("Number of time points to use for specifying annealing in each group");
+ STYPE ("annealing-npoints", anneal_npoints, NULL);
+ CTYPE ("List of times at the annealing points for each group");
+ STYPE ("annealing-time", anneal_time, NULL);
+ CTYPE ("Temp. at each annealing point, for each group.");
+ STYPE ("annealing-temp", anneal_temp, NULL);
+
+ /* Startup run */
+ CCTYPE ("GENERATE VELOCITIES FOR STARTUP RUN");
+ EETYPE("gen-vel", opts->bGenVel, yesno_names);
+ RTYPE ("gen-temp", opts->tempi, 300.0);
+ ITYPE ("gen-seed", opts->seed, 173529);
+
+ /* Shake stuff */
+ CCTYPE ("OPTIONS FOR BONDS");
+ EETYPE("constraints", opts->nshake, constraints);
+ CTYPE ("Type of constraint algorithm");
+ EETYPE("constraint-algorithm", ir->eConstrAlg, econstr_names);
+ CTYPE ("Do not constrain the start configuration");
+ EETYPE("continuation", ir->bContinuation, yesno_names);
+ CTYPE ("Use successive overrelaxation to reduce the number of shake iterations");
+ EETYPE("Shake-SOR", ir->bShakeSOR, yesno_names);
+ CTYPE ("Relative tolerance of shake");
+ RTYPE ("shake-tol", ir->shake_tol, 0.0001);
+ CTYPE ("Highest order in the expansion of the constraint coupling matrix");
+ ITYPE ("lincs-order", ir->nProjOrder, 4);
+ CTYPE ("Number of iterations in the final step of LINCS. 1 is fine for");
+ CTYPE ("normal simulations, but use 2 to conserve energy in NVE runs.");
+ CTYPE ("For energy minimization with constraints it should be 4 to 8.");
+ ITYPE ("lincs-iter", ir->nLincsIter, 1);
+ CTYPE ("Lincs will write a warning to the stderr if in one step a bond");
+ CTYPE ("rotates over more degrees than");
+ RTYPE ("lincs-warnangle", ir->LincsWarnAngle, 30.0);
+ CTYPE ("Convert harmonic bonds to morse potentials");
+ EETYPE("morse", opts->bMorse, yesno_names);
+
+ /* Energy group exclusions */
+ CCTYPE ("ENERGY GROUP EXCLUSIONS");
+ CTYPE ("Pairs of energy groups for which all non-bonded interactions are excluded");
+ STYPE ("energygrp-excl", egpexcl, NULL);
+
+ /* Walls */
+ CCTYPE ("WALLS");
+ CTYPE ("Number of walls, type, atom types, densities and box-z scale factor for Ewald");
+ ITYPE ("nwall", ir->nwall, 0);
+ EETYPE("wall-type", ir->wall_type, ewt_names);
+ RTYPE ("wall-r-linpot", ir->wall_r_linpot, -1);
+ STYPE ("wall-atomtype", wall_atomtype, NULL);
+ STYPE ("wall-density", wall_density, NULL);
+ RTYPE ("wall-ewald-zfac", ir->wall_ewald_zfac, 3);
+
+ /* COM pulling */
+ CCTYPE("COM PULLING");
+ CTYPE("Pull type: no, umbrella, constraint or constant-force");
+ EETYPE("pull", ir->ePull, epull_names);
+ if (ir->ePull != epullNO)
+ {
+ snew(ir->pull, 1);
+ pull_grp = read_pullparams(&ninp, &inp, ir->pull, &opts->pull_start, wi);
+ }
+
+ /* Enforced rotation */
+ CCTYPE("ENFORCED ROTATION");
+ CTYPE("Enforced rotation: No or Yes");
+ EETYPE("rotation", ir->bRot, yesno_names);
+ if (ir->bRot)
+ {
+ snew(ir->rot, 1);
+ rot_grp = read_rotparams(&ninp, &inp, ir->rot, wi);
+ }
+
+ /* Refinement */
+ CCTYPE("NMR refinement stuff");
+ CTYPE ("Distance restraints type: No, Simple or Ensemble");
+ EETYPE("disre", ir->eDisre, edisre_names);
+ CTYPE ("Force weighting of pairs in one distance restraint: Conservative or Equal");
+ EETYPE("disre-weighting", ir->eDisreWeighting, edisreweighting_names);
+ CTYPE ("Use sqrt of the time averaged times the instantaneous violation");
+ EETYPE("disre-mixed", ir->bDisreMixed, yesno_names);
+ RTYPE ("disre-fc", ir->dr_fc, 1000.0);
+ RTYPE ("disre-tau", ir->dr_tau, 0.0);
+ CTYPE ("Output frequency for pair distances to energy file");
+ ITYPE ("nstdisreout", ir->nstdisreout, 100);
+ CTYPE ("Orientation restraints: No or Yes");
+ EETYPE("orire", opts->bOrire, yesno_names);
+ CTYPE ("Orientation restraints force constant and tau for time averaging");
+ RTYPE ("orire-fc", ir->orires_fc, 0.0);
+ RTYPE ("orire-tau", ir->orires_tau, 0.0);
+ STYPE ("orire-fitgrp", orirefitgrp, NULL);
+ CTYPE ("Output frequency for trace(SD) and S to energy file");
+ ITYPE ("nstorireout", ir->nstorireout, 100);
+
+ /* free energy variables */
+ CCTYPE ("Free energy variables");
+ EETYPE("free-energy", ir->efep, efep_names);
+ STYPE ("couple-moltype", couple_moltype, NULL);
+ EETYPE("couple-lambda0", opts->couple_lam0, couple_lam);
+ EETYPE("couple-lambda1", opts->couple_lam1, couple_lam);
+ EETYPE("couple-intramol", opts->bCoupleIntra, yesno_names);
+
+ RTYPE ("init-lambda", fep->init_lambda, -1); /* start with -1 so
+ we can recognize if
+ it was not entered */
+ ITYPE ("init-lambda-state", fep->init_fep_state, -1);
+ RTYPE ("delta-lambda", fep->delta_lambda, 0.0);
+ ITYPE ("nstdhdl", fep->nstdhdl, 50);
+ STYPE ("fep-lambdas", fep_lambda[efptFEP], NULL);
+ STYPE ("mass-lambdas", fep_lambda[efptMASS], NULL);
+ STYPE ("coul-lambdas", fep_lambda[efptCOUL], NULL);
+ STYPE ("vdw-lambdas", fep_lambda[efptVDW], NULL);
+ STYPE ("bonded-lambdas", fep_lambda[efptBONDED], NULL);
+ STYPE ("restraint-lambdas", fep_lambda[efptRESTRAINT], NULL);
+ STYPE ("temperature-lambdas", fep_lambda[efptTEMPERATURE], NULL);
+ ITYPE ("calc-lambda-neighbors", fep->lambda_neighbors, 1);
+ STYPE ("init-lambda-weights", lambda_weights, NULL);
+ EETYPE("dhdl-print-energy", fep->bPrintEnergy, yesno_names);
+ RTYPE ("sc-alpha", fep->sc_alpha, 0.0);
+ ITYPE ("sc-power", fep->sc_power, 1);
+ RTYPE ("sc-r-power", fep->sc_r_power, 6.0);
+ RTYPE ("sc-sigma", fep->sc_sigma, 0.3);
+ EETYPE("sc-coul", fep->bScCoul, yesno_names);
+ ITYPE ("dh_hist_size", fep->dh_hist_size, 0);
+ RTYPE ("dh_hist_spacing", fep->dh_hist_spacing, 0.1);
+ EETYPE("separate-dhdl-file", fep->separate_dhdl_file,
+ separate_dhdl_file_names);
+ EETYPE("dhdl-derivatives", fep->dhdl_derivatives, dhdl_derivatives_names);
+ ITYPE ("dh_hist_size", fep->dh_hist_size, 0);
+ RTYPE ("dh_hist_spacing", fep->dh_hist_spacing, 0.1);
+
+ /* Non-equilibrium MD stuff */
+ CCTYPE("Non-equilibrium MD stuff");
+ STYPE ("acc-grps", accgrps, NULL);
+ STYPE ("accelerate", acc, NULL);
+ STYPE ("freezegrps", freeze, NULL);
+ STYPE ("freezedim", frdim, NULL);
+ RTYPE ("cos-acceleration", ir->cos_accel, 0);
+ STYPE ("deform", deform, NULL);
+
+ /* simulated tempering variables */
+ CCTYPE("simulated tempering variables");
+ EETYPE("simulated-tempering", ir->bSimTemp, yesno_names);
+ EETYPE("simulated-tempering-scaling", ir->simtempvals->eSimTempScale, esimtemp_names);
+ RTYPE("sim-temp-low", ir->simtempvals->simtemp_low, 300.0);
+ RTYPE("sim-temp-high", ir->simtempvals->simtemp_high, 300.0);
+
+ /* expanded ensemble variables */
+ if (ir->efep == efepEXPANDED || ir->bSimTemp)
+ {
+ read_expandedparams(&ninp, &inp, expand, wi);
+ }
+
+ /* Electric fields */
+ CCTYPE("Electric fields");
+ CTYPE ("Format is number of terms (int) and for all terms an amplitude (real)");
+ CTYPE ("and a phase angle (real)");
+ STYPE ("E-x", efield_x, NULL);
+ STYPE ("E-xt", efield_xt, NULL);
+ STYPE ("E-y", efield_y, NULL);
+ STYPE ("E-yt", efield_yt, NULL);
+ STYPE ("E-z", efield_z, NULL);
+ STYPE ("E-zt", efield_zt, NULL);
+
+ /* AdResS defined thingies */
+ CCTYPE ("AdResS parameters");
+ EETYPE("adress", ir->bAdress, yesno_names);
+ if (ir->bAdress)
+ {
+ snew(ir->adress, 1);
+ read_adressparams(&ninp, &inp, ir->adress, wi);
+ }
+
+ /* User defined thingies */
+ CCTYPE ("User defined thingies");
+ STYPE ("user1-grps", user1, NULL);
+ STYPE ("user2-grps", user2, NULL);
+ ITYPE ("userint1", ir->userint1, 0);
+ ITYPE ("userint2", ir->userint2, 0);
+ ITYPE ("userint3", ir->userint3, 0);
+ ITYPE ("userint4", ir->userint4, 0);
+ RTYPE ("userreal1", ir->userreal1, 0);
+ RTYPE ("userreal2", ir->userreal2, 0);
+ RTYPE ("userreal3", ir->userreal3, 0);
+ RTYPE ("userreal4", ir->userreal4, 0);
+#undef CTYPE
+
+ write_inpfile(mdparout, ninp, inp, FALSE, wi);
+ for (i = 0; (i < ninp); i++)
+ {
+ sfree(inp[i].name);
+ sfree(inp[i].value);
+ }
+ sfree(inp);
+
+ /* Process options if necessary */
+ for (m = 0; m < 2; m++)
+ {
+ for (i = 0; i < 2*DIM; i++)
+ {
+ dumdub[m][i] = 0.0;
+ }
+ if (ir->epc)
+ {
+ switch (ir->epct)
+ {
+ case epctISOTROPIC:
+ if (sscanf(dumstr[m], "%lf", &(dumdub[m][XX])) != 1)
+ {
+ warning_error(wi, "Pressure coupling not enough values (I need 1)");
+ }
+ dumdub[m][YY] = dumdub[m][ZZ] = dumdub[m][XX];
+ break;
+ case epctSEMIISOTROPIC:
+ case epctSURFACETENSION:
+ if (sscanf(dumstr[m], "%lf%lf",
+ &(dumdub[m][XX]), &(dumdub[m][ZZ])) != 2)
+ {
+ warning_error(wi, "Pressure coupling not enough values (I need 2)");
+ }
+ dumdub[m][YY] = dumdub[m][XX];
+ break;
+ case epctANISOTROPIC:
+ if (sscanf(dumstr[m], "%lf%lf%lf%lf%lf%lf",
+ &(dumdub[m][XX]), &(dumdub[m][YY]), &(dumdub[m][ZZ]),
+ &(dumdub[m][3]), &(dumdub[m][4]), &(dumdub[m][5])) != 6)
+ {
+ warning_error(wi, "Pressure coupling not enough values (I need 6)");
+ }
+ break;
+ default:
+ gmx_fatal(FARGS, "Pressure coupling type %s not implemented yet",
+ epcoupltype_names[ir->epct]);
+ }
+ }
+ }
+ clear_mat(ir->ref_p);
+ clear_mat(ir->compress);
+ for (i = 0; i < DIM; i++)
+ {
+ ir->ref_p[i][i] = dumdub[1][i];
+ ir->compress[i][i] = dumdub[0][i];
+ }
+ if (ir->epct == epctANISOTROPIC)
+ {
+ ir->ref_p[XX][YY] = dumdub[1][3];
+ ir->ref_p[XX][ZZ] = dumdub[1][4];
+ ir->ref_p[YY][ZZ] = dumdub[1][5];
+ if (ir->ref_p[XX][YY] != 0 && ir->ref_p[XX][ZZ] != 0 && ir->ref_p[YY][ZZ] != 0)
+ {
+ warning(wi, "All off-diagonal reference pressures are non-zero. Are you sure you want to apply a threefold shear stress?\n");
+ }
+ ir->compress[XX][YY] = dumdub[0][3];
+ ir->compress[XX][ZZ] = dumdub[0][4];
+ ir->compress[YY][ZZ] = dumdub[0][5];
+ for (i = 0; i < DIM; i++)
+ {
+ for (m = 0; m < i; m++)
+ {
+ ir->ref_p[i][m] = ir->ref_p[m][i];
+ ir->compress[i][m] = ir->compress[m][i];
+ }
+ }
+ }
+
+ if (ir->comm_mode == ecmNO)
+ {
+ ir->nstcomm = 0;
+ }
+
+ opts->couple_moltype = NULL;
+ if (strlen(couple_moltype) > 0)
+ {
+ if (ir->efep != efepNO)
+ {
+ opts->couple_moltype = strdup(couple_moltype);
+ if (opts->couple_lam0 == opts->couple_lam1)
+ {
+ warning(wi, "The lambda=0 and lambda=1 states for coupling are identical");
+ }
+ if (ir->eI == eiMD && (opts->couple_lam0 == ecouplamNONE ||
+ opts->couple_lam1 == ecouplamNONE))
+ {
+ warning(wi, "For proper sampling of the (nearly) decoupled state, stochastic dynamics should be used");
+ }
+ }
+ else
+ {
+ warning(wi, "Can not couple a molecule with free_energy = no");
+ }
+ }
+ /* FREE ENERGY AND EXPANDED ENSEMBLE OPTIONS */
+ if (ir->efep != efepNO)
+ {
+ if (fep->delta_lambda > 0)
+ {
+ ir->efep = efepSLOWGROWTH;
+ }
+ }
+
+ if (ir->bSimTemp)
+ {
+ fep->bPrintEnergy = TRUE;
+ /* always print out the energy to dhdl if we are doing expanded ensemble, since we need the total energy
+ if the temperature is changing. */
+ }
+
+ if ((ir->efep != efepNO) || ir->bSimTemp)
+ {
+ ir->bExpanded = FALSE;
+ if ((ir->efep == efepEXPANDED) || ir->bSimTemp)
+ {
+ ir->bExpanded = TRUE;
+ }
+ do_fep_params(ir, fep_lambda, lambda_weights);
+ if (ir->bSimTemp) /* done after fep params */
+ {
+ do_simtemp_params(ir);
+ }
+ }
+ else
+ {
+ ir->fepvals->n_lambda = 0;
+ }
+
+ /* WALL PARAMETERS */
+
+ do_wall_params(ir, wall_atomtype, wall_density, opts);
+
+ /* ORIENTATION RESTRAINT PARAMETERS */
+
+ if (opts->bOrire && str_nelem(orirefitgrp, MAXPTR, NULL) != 1)
+ {
+ warning_error(wi, "ERROR: Need one orientation restraint fit group\n");
+ }
+
+ /* DEFORMATION PARAMETERS */
+
+ clear_mat(ir->deform);
+ for (i = 0; i < 6; i++)
+ {
+ dumdub[0][i] = 0;
+ }
+ m = sscanf(deform, "%lf %lf %lf %lf %lf %lf",
+ &(dumdub[0][0]), &(dumdub[0][1]), &(dumdub[0][2]),
+ &(dumdub[0][3]), &(dumdub[0][4]), &(dumdub[0][5]));
+ for (i = 0; i < 3; i++)
+ {
+ ir->deform[i][i] = dumdub[0][i];
+ }
+ ir->deform[YY][XX] = dumdub[0][3];
+ ir->deform[ZZ][XX] = dumdub[0][4];
+ ir->deform[ZZ][YY] = dumdub[0][5];
+ if (ir->epc != epcNO)
+ {
+ for (i = 0; i < 3; i++)
+ {
+ for (j = 0; j <= i; j++)
+ {
+ if (ir->deform[i][j] != 0 && ir->compress[i][j] != 0)
+ {
+ warning_error(wi, "A box element has deform set and compressibility > 0");
+ }
+ }
+ }
+ for (i = 0; i < 3; i++)
+ {
+ for (j = 0; j < i; j++)
+ {
+ if (ir->deform[i][j] != 0)
+ {
+ for (m = j; m < DIM; m++)
+ {
+ if (ir->compress[m][j] != 0)
+ {
+ sprintf(warn_buf, "An off-diagonal box element has deform set while compressibility > 0 for the same component of another box vector, this might lead to spurious periodicity effects.");
+ warning(wi, warn_buf);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ sfree(dumstr[0]);
+ sfree(dumstr[1]);
+}
+
+static int search_QMstring(char *s, int ng, const char *gn[])
+{
+ /* same as normal search_string, but this one searches QM strings */
+ int i;
+
+ for (i = 0; (i < ng); i++)
+ {
+ if (gmx_strcasecmp(s, gn[i]) == 0)
+ {
+ return i;
+ }
+ }
+
+ gmx_fatal(FARGS, "this QM method or basisset (%s) is not implemented\n!", s);
+
+ return -1;
+
+} /* search_QMstring */
+
+
+int search_string(char *s, int ng, char *gn[])
+{
+ int i;
+
+ for (i = 0; (i < ng); i++)
+ {
+ if (gmx_strcasecmp(s, gn[i]) == 0)
+ {
+ return i;
+ }
+ }
+
+ gmx_fatal(FARGS,
+ "Group %s referenced in the .mdp file was not found in the index file.\n"
+ "Group names must match either [moleculetype] names or custom index group\n"
+ "names, in which case you must supply an index file to the '-n' option\n"
+ "of grompp.",
+ s);
+
+ return -1;
+}
+
+static gmx_bool do_numbering(int natoms, gmx_groups_t *groups, int ng, char *ptrs[],
+ t_blocka *block, char *gnames[],
+ int gtype, int restnm,
+ int grptp, gmx_bool bVerbose,
+ warninp_t wi)
+{
+ unsigned short *cbuf;
+ t_grps *grps = &(groups->grps[gtype]);
+ int i, j, gid, aj, ognr, ntot = 0;
+ const char *title;
+ gmx_bool bRest;
+ char warn_buf[STRLEN];
+
+ if (debug)
+ {
+ fprintf(debug, "Starting numbering %d groups of type %d\n", ng, gtype);
+ }
+
+ title = gtypes[gtype];
+
+ snew(cbuf, natoms);
+ /* Mark all id's as not set */
+ for (i = 0; (i < natoms); i++)
+ {
+ cbuf[i] = NOGID;
+ }
+
+ snew(grps->nm_ind, ng+1); /* +1 for possible rest group */
+ for (i = 0; (i < ng); i++)
+ {
+ /* Lookup the group name in the block structure */
+ gid = search_string(ptrs[i], block->nr, gnames);
+ if ((grptp != egrptpONE) || (i == 0))
+ {
+ grps->nm_ind[grps->nr++] = gid;
+ }
+ if (debug)
+ {
+ fprintf(debug, "Found gid %d for group %s\n", gid, ptrs[i]);
+ }
+
+ /* Now go over the atoms in the group */
+ for (j = block->index[gid]; (j < block->index[gid+1]); j++)
+ {
+
+ aj = block->a[j];
+
+ /* Range checking */
+ if ((aj < 0) || (aj >= natoms))
+ {
+ gmx_fatal(FARGS, "Invalid atom number %d in indexfile", aj);
+ }
+ /* Lookup up the old group number */
+ ognr = cbuf[aj];
+ if (ognr != NOGID)
+ {
+ gmx_fatal(FARGS, "Atom %d in multiple %s groups (%d and %d)",
+ aj+1, title, ognr+1, i+1);
+ }
+ else
+ {
+ /* Store the group number in buffer */
+ if (grptp == egrptpONE)
+ {
+ cbuf[aj] = 0;
+ }
+ else
+ {
+ cbuf[aj] = i;
+ }
+ ntot++;
+ }
+ }
+ }
+
+ /* Now check whether we have done all atoms */
+ bRest = FALSE;
+ if (ntot != natoms)
+ {
+ if (grptp == egrptpALL)
+ {
+ gmx_fatal(FARGS, "%d atoms are not part of any of the %s groups",
+ natoms-ntot, title);
+ }
+ else if (grptp == egrptpPART)
+ {
+ sprintf(warn_buf, "%d atoms are not part of any of the %s groups",
+ natoms-ntot, title);
+ warning_note(wi, warn_buf);
+ }
+ /* Assign all atoms currently unassigned to a rest group */
+ for (j = 0; (j < natoms); j++)
+ {
+ if (cbuf[j] == NOGID)
+ {
+ cbuf[j] = grps->nr;
+ bRest = TRUE;
+ }
+ }
+ if (grptp != egrptpPART)
+ {
+ if (bVerbose)
+ {
+ fprintf(stderr,
+ "Making dummy/rest group for %s containing %d elements\n",
+ title, natoms-ntot);
+ }
+ /* Add group name "rest" */
+ grps->nm_ind[grps->nr] = restnm;
+
+ /* Assign the rest name to all atoms not currently assigned to a group */
+ for (j = 0; (j < natoms); j++)
+ {
+ if (cbuf[j] == NOGID)
+ {
+ cbuf[j] = grps->nr;
+ }
+ }
+ grps->nr++;
+ }
+ }
+
+ if (grps->nr == 1 && (ntot == 0 || ntot == natoms))
+ {
+ /* All atoms are part of one (or no) group, no index required */
+ groups->ngrpnr[gtype] = 0;
+ groups->grpnr[gtype] = NULL;
+ }
+ else
+ {
+ groups->ngrpnr[gtype] = natoms;
+ snew(groups->grpnr[gtype], natoms);
+ for (j = 0; (j < natoms); j++)
+ {
+ groups->grpnr[gtype][j] = cbuf[j];
+ }
+ }
+
+ sfree(cbuf);
+
+ return (bRest && grptp == egrptpPART);
+}
+
+static void calc_nrdf(gmx_mtop_t *mtop, t_inputrec *ir, char **gnames)
+{
+ t_grpopts *opts;
+ gmx_groups_t *groups;
+ t_pull *pull;
+ int natoms, ai, aj, i, j, d, g, imin, jmin, nc;
+ t_iatom *ia;
+ int *nrdf2, *na_vcm, na_tot;
+ double *nrdf_tc, *nrdf_vcm, nrdf_uc, n_sub = 0;
+ gmx_mtop_atomloop_all_t aloop;
+ t_atom *atom;
+ int mb, mol, ftype, as;
+ gmx_molblock_t *molb;
+ gmx_moltype_t *molt;
+
+ /* Calculate nrdf.
+ * First calc 3xnr-atoms for each group
+ * then subtract half a degree of freedom for each constraint
+ *
+ * Only atoms and nuclei contribute to the degrees of freedom...
+ */
+
+ opts = &ir->opts;
+
+ groups = &mtop->groups;
+ natoms = mtop->natoms;
+
+ /* Allocate one more for a possible rest group */
+ /* We need to sum degrees of freedom into doubles,
+ * since floats give too low nrdf's above 3 million atoms.
+ */
+ snew(nrdf_tc, groups->grps[egcTC].nr+1);
+ snew(nrdf_vcm, groups->grps[egcVCM].nr+1);
+ snew(na_vcm, groups->grps[egcVCM].nr+1);
+
+ for (i = 0; i < groups->grps[egcTC].nr; i++)
+ {
+ nrdf_tc[i] = 0;
+ }
+ for (i = 0; i < groups->grps[egcVCM].nr+1; i++)
+ {
+ nrdf_vcm[i] = 0;
+ }
+
+ snew(nrdf2, natoms);
+ aloop = gmx_mtop_atomloop_all_init(mtop);
+ while (gmx_mtop_atomloop_all_next(aloop, &i, &atom))
+ {
+ nrdf2[i] = 0;
+ if (atom->ptype == eptAtom || atom->ptype == eptNucleus)
+ {
+ g = ggrpnr(groups, egcFREEZE, i);
+ /* Double count nrdf for particle i */
+ for (d = 0; d < DIM; d++)
+ {
+ if (opts->nFreeze[g][d] == 0)
+ {
+ nrdf2[i] += 2;
+ }
+ }
+ nrdf_tc [ggrpnr(groups, egcTC, i)] += 0.5*nrdf2[i];
+ nrdf_vcm[ggrpnr(groups, egcVCM, i)] += 0.5*nrdf2[i];
+ }
+ }
+
+ as = 0;
+ for (mb = 0; mb < mtop->nmolblock; mb++)
+ {
+ molb = &mtop->molblock[mb];
+ molt = &mtop->moltype[molb->type];
+ atom = molt->atoms.atom;
+ for (mol = 0; mol < molb->nmol; mol++)
+ {
+ for (ftype = F_CONSTR; ftype <= F_CONSTRNC; ftype++)
+ {
+ ia = molt->ilist[ftype].iatoms;
+ for (i = 0; i < molt->ilist[ftype].nr; )
+ {
+ /* Subtract degrees of freedom for the constraints,
+ * if the particles still have degrees of freedom left.
+ * If one of the particles is a vsite or a shell, then all
+ * constraint motion will go there, but since they do not
+ * contribute to the constraints the degrees of freedom do not
+ * change.
+ */
+ ai = as + ia[1];
+ aj = as + ia[2];
+ if (((atom[ia[1]].ptype == eptNucleus) ||
+ (atom[ia[1]].ptype == eptAtom)) &&
+ ((atom[ia[2]].ptype == eptNucleus) ||
+ (atom[ia[2]].ptype == eptAtom)))
+ {
+ if (nrdf2[ai] > 0)
+ {
+ jmin = 1;
+ }
+ else
+ {
+ jmin = 2;
+ }
+ if (nrdf2[aj] > 0)
+ {
+ imin = 1;
+ }
+ else
+ {
+ imin = 2;
+ }
+ imin = min(imin, nrdf2[ai]);
+ jmin = min(jmin, nrdf2[aj]);
+ nrdf2[ai] -= imin;
+ nrdf2[aj] -= jmin;
+ nrdf_tc [ggrpnr(groups, egcTC, ai)] -= 0.5*imin;
+ nrdf_tc [ggrpnr(groups, egcTC, aj)] -= 0.5*jmin;
+ nrdf_vcm[ggrpnr(groups, egcVCM, ai)] -= 0.5*imin;
+ nrdf_vcm[ggrpnr(groups, egcVCM, aj)] -= 0.5*jmin;
+ }
+ ia += interaction_function[ftype].nratoms+1;
+ i += interaction_function[ftype].nratoms+1;
+ }
+ }
+ ia = molt->ilist[F_SETTLE].iatoms;
+ for (i = 0; i < molt->ilist[F_SETTLE].nr; )
+ {
+ /* Subtract 1 dof from every atom in the SETTLE */
+ for (j = 0; j < 3; j++)
+ {
+ ai = as + ia[1+j];
+ imin = min(2, nrdf2[ai]);
+ nrdf2[ai] -= imin;
+ nrdf_tc [ggrpnr(groups, egcTC, ai)] -= 0.5*imin;
+ nrdf_vcm[ggrpnr(groups, egcVCM, ai)] -= 0.5*imin;
+ }
+ ia += 4;
+ i += 4;
+ }
+ as += molt->atoms.nr;
+ }
+ }
+
+ if (ir->ePull == epullCONSTRAINT)
+ {
+ /* Correct nrdf for the COM constraints.
+ * We correct using the TC and VCM group of the first atom
+ * in the reference and pull group. If atoms in one pull group
+ * belong to different TC or VCM groups it is anyhow difficult
+ * to determine the optimal nrdf assignment.
+ */
+ pull = ir->pull;
+ if (pull->eGeom == epullgPOS)
+ {
+ nc = 0;
+ for (i = 0; i < DIM; i++)
+ {
+ if (pull->dim[i])
+ {
+ nc++;
+ }
+ }
+ }
+ else
+ {
+ nc = 1;
+ }
+ for (i = 0; i < pull->ngrp; i++)
+ {
+ imin = 2*nc;
+ if (pull->grp[0].nat > 0)
+ {
+ /* Subtract 1/2 dof from the reference group */
+ ai = pull->grp[0].ind[0];
+ if (nrdf_tc[ggrpnr(groups, egcTC, ai)] > 1)
+ {
+ nrdf_tc [ggrpnr(groups, egcTC, ai)] -= 0.5;
+ nrdf_vcm[ggrpnr(groups, egcVCM, ai)] -= 0.5;
+ imin--;
+ }
+ }
+ /* Subtract 1/2 dof from the pulled group */
+ ai = pull->grp[1+i].ind[0];
+ nrdf_tc [ggrpnr(groups, egcTC, ai)] -= 0.5*imin;
+ nrdf_vcm[ggrpnr(groups, egcVCM, ai)] -= 0.5*imin;
+ if (nrdf_tc[ggrpnr(groups, egcTC, ai)] < 0)
+ {
+ gmx_fatal(FARGS, "Center of mass pulling constraints caused the number of degrees of freedom for temperature coupling group %s to be negative", gnames[groups->grps[egcTC].nm_ind[ggrpnr(groups, egcTC, ai)]]);
+ }
+ }
+ }
+
+ if (ir->nstcomm != 0)
+ {
+ /* Subtract 3 from the number of degrees of freedom in each vcm group
+ * when com translation is removed and 6 when rotation is removed
+ * as well.
+ */
+ switch (ir->comm_mode)
+ {
+ case ecmLINEAR:
+ n_sub = ndof_com(ir);
+ break;
+ case ecmANGULAR:
+ n_sub = 6;
+ break;
+ default:
+ n_sub = 0;
+ gmx_incons("Checking comm_mode");
+ }
+
+ for (i = 0; i < groups->grps[egcTC].nr; i++)
+ {
+ /* Count the number of atoms of TC group i for every VCM group */
+ for (j = 0; j < groups->grps[egcVCM].nr+1; j++)
+ {
+ na_vcm[j] = 0;
+ }
+ na_tot = 0;
+ for (ai = 0; ai < natoms; ai++)
+ {
+ if (ggrpnr(groups, egcTC, ai) == i)
+ {
+ na_vcm[ggrpnr(groups, egcVCM, ai)]++;
+ na_tot++;
+ }
+ }
+ /* Correct for VCM removal according to the fraction of each VCM
+ * group present in this TC group.
+ */
+ nrdf_uc = nrdf_tc[i];
+ if (debug)
+ {
+ fprintf(debug, "T-group[%d] nrdf_uc = %g, n_sub = %g\n",
+ i, nrdf_uc, n_sub);
+ }
+ nrdf_tc[i] = 0;
+ for (j = 0; j < groups->grps[egcVCM].nr+1; j++)
+ {
+ if (nrdf_vcm[j] > n_sub)
+ {
+ nrdf_tc[i] += nrdf_uc*((double)na_vcm[j]/(double)na_tot)*
+ (nrdf_vcm[j] - n_sub)/nrdf_vcm[j];
+ }
+ if (debug)
+ {
+ fprintf(debug, " nrdf_vcm[%d] = %g, nrdf = %g\n",
+ j, nrdf_vcm[j], nrdf_tc[i]);
+ }
+ }
+ }
+ }
+ for (i = 0; (i < groups->grps[egcTC].nr); i++)
+ {
+ opts->nrdf[i] = nrdf_tc[i];
+ if (opts->nrdf[i] < 0)
+ {
+ opts->nrdf[i] = 0;
+ }
+ fprintf(stderr,
+ "Number of degrees of freedom in T-Coupling group %s is %.2f\n",
+ gnames[groups->grps[egcTC].nm_ind[i]], opts->nrdf[i]);
+ }
+
+ sfree(nrdf2);
+ sfree(nrdf_tc);
+ sfree(nrdf_vcm);
+ sfree(na_vcm);
+}
+
+static void decode_cos(char *s, t_cosines *cosine)
+{
+ char *t;
+ char format[STRLEN], f1[STRLEN];
+ double a, phi;
+ int i;
+
+ t = strdup(s);
+ trim(t);
+
+ cosine->n = 0;
+ cosine->a = NULL;
+ cosine->phi = NULL;
+ if (strlen(t))
+ {
+ sscanf(t, "%d", &(cosine->n));
+ if (cosine->n <= 0)
+ {
+ cosine->n = 0;
+ }
+ else
+ {
+ snew(cosine->a, cosine->n);
+ snew(cosine->phi, cosine->n);
+
+ sprintf(format, "%%*d");
+ for (i = 0; (i < cosine->n); i++)
+ {
+ strcpy(f1, format);
+ strcat(f1, "%lf%lf");
+ if (sscanf(t, f1, &a, &phi) < 2)
+ {
+ gmx_fatal(FARGS, "Invalid input for electric field shift: '%s'", t);
+ }
+ cosine->a[i] = a;
+ cosine->phi[i] = phi;
+ strcat(format, "%*lf%*lf");
+ }
+ }
+ }
+ sfree(t);
+}
+
+static gmx_bool do_egp_flag(t_inputrec *ir, gmx_groups_t *groups,
+ const char *option, const char *val, int flag)
+{
+ /* The maximum number of energy group pairs would be MAXPTR*(MAXPTR+1)/2.
+ * But since this is much larger than STRLEN, such a line can not be parsed.
+ * The real maximum is the number of names that fit in a string: STRLEN/2.
+ */
+#define EGP_MAX (STRLEN/2)
+ int nelem, i, j, k, nr;
+ char *names[EGP_MAX];
+ char ***gnames;
+ gmx_bool bSet;
+
+ gnames = groups->grpname;
+
+ nelem = str_nelem(val, EGP_MAX, names);
+ if (nelem % 2 != 0)
+ {
+ gmx_fatal(FARGS, "The number of groups for %s is odd", option);
+ }
+ nr = groups->grps[egcENER].nr;
+ bSet = FALSE;
+ for (i = 0; i < nelem/2; i++)
+ {
+ j = 0;
+ while ((j < nr) &&
+ gmx_strcasecmp(names[2*i], *(gnames[groups->grps[egcENER].nm_ind[j]])))
+ {
+ j++;
+ }
+ if (j == nr)
+ {
+ gmx_fatal(FARGS, "%s in %s is not an energy group\n",
+ names[2*i], option);
+ }
+ k = 0;
+ while ((k < nr) &&
+ gmx_strcasecmp(names[2*i+1], *(gnames[groups->grps[egcENER].nm_ind[k]])))
+ {
+ k++;
+ }
+ if (k == nr)
+ {
+ gmx_fatal(FARGS, "%s in %s is not an energy group\n",
+ names[2*i+1], option);
+ }
+ if ((j < nr) && (k < nr))
+ {
+ ir->opts.egp_flags[nr*j+k] |= flag;
+ ir->opts.egp_flags[nr*k+j] |= flag;
+ bSet = TRUE;
+ }
+ }
+
+ return bSet;
+}
+
+void do_index(const char* mdparin, const char *ndx,
+ gmx_mtop_t *mtop,
+ gmx_bool bVerbose,
+ t_inputrec *ir, rvec *v,
+ warninp_t wi)
+{
+ t_blocka *grps;
+ gmx_groups_t *groups;
+ int natoms;
+ t_symtab *symtab;
+ t_atoms atoms_all;
+ char warnbuf[STRLEN], **gnames;
+ int nr, ntcg, ntau_t, nref_t, nacc, nofg, nSA, nSA_points, nSA_time, nSA_temp;
+ real tau_min;
+ int nstcmin;
+ int nacg, nfreeze, nfrdim, nenergy, nvcm, nuser;
+ char *ptr1[MAXPTR], *ptr2[MAXPTR], *ptr3[MAXPTR];
+ int i, j, k, restnm;
+ real SAtime;
+ gmx_bool bExcl, bTable, bSetTCpar, bAnneal, bRest;
+ int nQMmethod, nQMbasis, nQMcharge, nQMmult, nbSH, nCASorb, nCASelec,
+ nSAon, nSAoff, nSAsteps, nQMg, nbOPT, nbTS;
+ char warn_buf[STRLEN];
+
+ if (bVerbose)
+ {
+ fprintf(stderr, "processing index file...\n");
+ }
+ debug_gmx();
+ if (ndx == NULL)
+ {
+ snew(grps, 1);
+ snew(grps->index, 1);
+ snew(gnames, 1);
+ atoms_all = gmx_mtop_global_atoms(mtop);
+ analyse(&atoms_all, grps, &gnames, FALSE, TRUE);
+ free_t_atoms(&atoms_all, FALSE);
+ }
+ else
+ {
+ grps = init_index(ndx, &gnames);
+ }
+
+ groups = &mtop->groups;
+ natoms = mtop->natoms;
+ symtab = &mtop->symtab;
+
+ snew(groups->grpname, grps->nr+1);
+
+ for (i = 0; (i < grps->nr); i++)
+ {
+ groups->grpname[i] = put_symtab(symtab, gnames[i]);
+ }
+ groups->grpname[i] = put_symtab(symtab, "rest");
+ restnm = i;
+ srenew(gnames, grps->nr+1);
+ gnames[restnm] = *(groups->grpname[i]);
+ groups->ngrpname = grps->nr+1;
+
+ set_warning_line(wi, mdparin, -1);
+
+ ntau_t = str_nelem(tau_t, MAXPTR, ptr1);
+ nref_t = str_nelem(ref_t, MAXPTR, ptr2);
+ ntcg = str_nelem(tcgrps, MAXPTR, ptr3);
+ if ((ntau_t != ntcg) || (nref_t != ntcg))
+ {
+ gmx_fatal(FARGS, "Invalid T coupling input: %d groups, %d ref-t values and "
+ "%d tau-t values", ntcg, nref_t, ntau_t);
+ }
+
+ bSetTCpar = (ir->etc || EI_SD(ir->eI) || ir->eI == eiBD || EI_TPI(ir->eI));
+ do_numbering(natoms, groups, ntcg, ptr3, grps, gnames, egcTC,
+ restnm, bSetTCpar ? egrptpALL : egrptpALL_GENREST, bVerbose, wi);
+ nr = groups->grps[egcTC].nr;
+ ir->opts.ngtc = nr;
+ snew(ir->opts.nrdf, nr);
+ snew(ir->opts.tau_t, nr);
+ snew(ir->opts.ref_t, nr);
+ if (ir->eI == eiBD && ir->bd_fric == 0)
+ {
+ fprintf(stderr, "bd-fric=0, so tau-t will be used as the inverse friction constant(s)\n");
+ }
+
+ if (bSetTCpar)
+ {
+ if (nr != nref_t)
+ {
+ gmx_fatal(FARGS, "Not enough ref-t and tau-t values!");
+ }
+
+ tau_min = 1e20;
+ for (i = 0; (i < nr); i++)
+ {
+ ir->opts.tau_t[i] = strtod(ptr1[i], NULL);
+ if ((ir->eI == eiBD || ir->eI == eiSD2) && ir->opts.tau_t[i] <= 0)
+ {
+ sprintf(warn_buf, "With integrator %s tau-t should be larger than 0", ei_names[ir->eI]);
+ warning_error(wi, warn_buf);
+ }
+
+ if (ir->etc != etcVRESCALE && ir->opts.tau_t[i] == 0)
+ {
+ warning_note(wi, "tau-t = -1 is the value to signal that a group should not have temperature coupling. Treating your use of tau-t = 0 as if you used -1.");
+ }
+
+ if (ir->opts.tau_t[i] >= 0)
+ {
+ tau_min = min(tau_min, ir->opts.tau_t[i]);
+ }
+ }
+ if (ir->etc != etcNO && ir->nsttcouple == -1)
+ {
+ ir->nsttcouple = ir_optimal_nsttcouple(ir);
+ }
+
+ if (EI_VV(ir->eI))
+ {
+ if ((ir->etc == etcNOSEHOOVER) && (ir->epc == epcBERENDSEN))
+ {
+ gmx_fatal(FARGS, "Cannot do Nose-Hoover temperature with Berendsen pressure control with md-vv; use either vrescale temperature with berendsen pressure or Nose-Hoover temperature with MTTK pressure");
+ }
+ if ((ir->epc == epcMTTK) && (ir->etc > etcNO))
+ {
+ if (ir->nstpcouple != ir->nsttcouple)
+ {
+ int mincouple = min(ir->nstpcouple, ir->nsttcouple);
+ ir->nstpcouple = ir->nsttcouple = mincouple;
+ sprintf(warn_buf, "for current Trotter decomposition methods with vv, nsttcouple and nstpcouple must be equal. Both have been reset to min(nsttcouple,nstpcouple) = %d", mincouple);
+ warning_note(wi, warn_buf);
+ }
+ }
+ }
+ /* velocity verlet with averaged kinetic energy KE = 0.5*(v(t+1/2) - v(t-1/2)) is implemented
+ primarily for testing purposes, and does not work with temperature coupling other than 1 */
+
+ if (ETC_ANDERSEN(ir->etc))
+ {
+ if (ir->nsttcouple != 1)
+ {
+ ir->nsttcouple = 1;
+ sprintf(warn_buf, "Andersen temperature control methods assume nsttcouple = 1; there is no need for larger nsttcouple > 1, since no global parameters are computed. nsttcouple has been reset to 1");
+ warning_note(wi, warn_buf);
+ }
+ }
+ nstcmin = tcouple_min_integration_steps(ir->etc);
+ if (nstcmin > 1)
+ {
+ if (tau_min/(ir->delta_t*ir->nsttcouple) < nstcmin)
+ {
+ sprintf(warn_buf, "For proper integration of the %s thermostat, tau-t (%g) should be at least %d times larger than nsttcouple*dt (%g)",
+ ETCOUPLTYPE(ir->etc),
+ tau_min, nstcmin,
+ ir->nsttcouple*ir->delta_t);
+ warning(wi, warn_buf);
+ }
+ }
+ for (i = 0; (i < nr); i++)
+ {
+ ir->opts.ref_t[i] = strtod(ptr2[i], NULL);
+ if (ir->opts.ref_t[i] < 0)
+ {
+ gmx_fatal(FARGS, "ref-t for group %d negative", i);
+ }
+ }
+ /* set the lambda mc temperature to the md integrator temperature (which should be defined
+ if we are in this conditional) if mc_temp is negative */
+ if (ir->expandedvals->mc_temp < 0)
+ {
+ ir->expandedvals->mc_temp = ir->opts.ref_t[0]; /*for now, set to the first reft */
+ }
+ }
+
+ /* Simulated annealing for each group. There are nr groups */
+ nSA = str_nelem(anneal, MAXPTR, ptr1);
+ if (nSA == 1 && (ptr1[0][0] == 'n' || ptr1[0][0] == 'N'))
+ {
+ nSA = 0;
+ }
+ if (nSA > 0 && nSA != nr)
+ {
+ gmx_fatal(FARGS, "Not enough annealing values: %d (for %d groups)\n", nSA, nr);
+ }
+ else
+ {
+ snew(ir->opts.annealing, nr);
+ snew(ir->opts.anneal_npoints, nr);
+ snew(ir->opts.anneal_time, nr);
+ snew(ir->opts.anneal_temp, nr);
+ for (i = 0; i < nr; i++)
+ {
+ ir->opts.annealing[i] = eannNO;
+ ir->opts.anneal_npoints[i] = 0;
+ ir->opts.anneal_time[i] = NULL;
+ ir->opts.anneal_temp[i] = NULL;
+ }
+ if (nSA > 0)
+ {
+ bAnneal = FALSE;
+ for (i = 0; i < nr; i++)
+ {
+ if (ptr1[i][0] == 'n' || ptr1[i][0] == 'N')
+ {
+ ir->opts.annealing[i] = eannNO;
+ }
+ else if (ptr1[i][0] == 's' || ptr1[i][0] == 'S')
+ {
+ ir->opts.annealing[i] = eannSINGLE;
+ bAnneal = TRUE;
+ }
+ else if (ptr1[i][0] == 'p' || ptr1[i][0] == 'P')
+ {
+ ir->opts.annealing[i] = eannPERIODIC;
+ bAnneal = TRUE;
+ }
+ }
+ if (bAnneal)
+ {
+ /* Read the other fields too */
+ nSA_points = str_nelem(anneal_npoints, MAXPTR, ptr1);
+ if (nSA_points != nSA)
+ {
+ gmx_fatal(FARGS, "Found %d annealing-npoints values for %d groups\n", nSA_points, nSA);
+ }
+ for (k = 0, i = 0; i < nr; i++)
+ {
+ ir->opts.anneal_npoints[i] = strtol(ptr1[i], NULL, 10);
+ if (ir->opts.anneal_npoints[i] == 1)
+ {
+ gmx_fatal(FARGS, "Please specify at least a start and an end point for annealing\n");
+ }
+ snew(ir->opts.anneal_time[i], ir->opts.anneal_npoints[i]);
+ snew(ir->opts.anneal_temp[i], ir->opts.anneal_npoints[i]);
+ k += ir->opts.anneal_npoints[i];
+ }
+
+ nSA_time = str_nelem(anneal_time, MAXPTR, ptr1);
+ if (nSA_time != k)
+ {
+ gmx_fatal(FARGS, "Found %d annealing-time values, wanter %d\n", nSA_time, k);
+ }
+ nSA_temp = str_nelem(anneal_temp, MAXPTR, ptr2);
+ if (nSA_temp != k)
+ {
+ gmx_fatal(FARGS, "Found %d annealing-temp values, wanted %d\n", nSA_temp, k);
+ }
+
+ for (i = 0, k = 0; i < nr; i++)
+ {
+
+ for (j = 0; j < ir->opts.anneal_npoints[i]; j++)
+ {
+ ir->opts.anneal_time[i][j] = strtod(ptr1[k], NULL);
+ ir->opts.anneal_temp[i][j] = strtod(ptr2[k], NULL);
+ if (j == 0)
+ {
+ if (ir->opts.anneal_time[i][0] > (ir->init_t+GMX_REAL_EPS))
+ {
+ gmx_fatal(FARGS, "First time point for annealing > init_t.\n");
+ }
+ }
+ else
+ {
+ /* j>0 */
+ if (ir->opts.anneal_time[i][j] < ir->opts.anneal_time[i][j-1])
+ {
+ gmx_fatal(FARGS, "Annealing timepoints out of order: t=%f comes after t=%f\n",
+ ir->opts.anneal_time[i][j], ir->opts.anneal_time[i][j-1]);
+ }
+ }
+ if (ir->opts.anneal_temp[i][j] < 0)
+ {
+ gmx_fatal(FARGS, "Found negative temperature in annealing: %f\n", ir->opts.anneal_temp[i][j]);
+ }
+ k++;
+ }
+ }
+ /* Print out some summary information, to make sure we got it right */
+ for (i = 0, k = 0; i < nr; i++)
+ {
+ if (ir->opts.annealing[i] != eannNO)
+ {
+ j = groups->grps[egcTC].nm_ind[i];
+ fprintf(stderr, "Simulated annealing for group %s: %s, %d timepoints\n",
+ *(groups->grpname[j]), eann_names[ir->opts.annealing[i]],
+ ir->opts.anneal_npoints[i]);
+ fprintf(stderr, "Time (ps) Temperature (K)\n");
+ /* All terms except the last one */
+ for (j = 0; j < (ir->opts.anneal_npoints[i]-1); j++)
+ {
+ fprintf(stderr, "%9.1f %5.1f\n", ir->opts.anneal_time[i][j], ir->opts.anneal_temp[i][j]);
+ }
+
+ /* Finally the last one */
+ j = ir->opts.anneal_npoints[i]-1;
+ if (ir->opts.annealing[i] == eannSINGLE)
+ {
+ fprintf(stderr, "%9.1f- %5.1f\n", ir->opts.anneal_time[i][j], ir->opts.anneal_temp[i][j]);
+ }
+ else
+ {
+ fprintf(stderr, "%9.1f %5.1f\n", ir->opts.anneal_time[i][j], ir->opts.anneal_temp[i][j]);
+ if (fabs(ir->opts.anneal_temp[i][j]-ir->opts.anneal_temp[i][0]) > GMX_REAL_EPS)
+ {
+ warning_note(wi, "There is a temperature jump when your annealing loops back.\n");
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ if (ir->ePull != epullNO)
+ {
+ make_pull_groups(ir->pull, pull_grp, grps, gnames);
+ }
+
+ if (ir->bRot)
+ {
+ make_rotation_groups(ir->rot, rot_grp, grps, gnames);
+ }
+
+ nacc = str_nelem(acc, MAXPTR, ptr1);
+ nacg = str_nelem(accgrps, MAXPTR, ptr2);
+ if (nacg*DIM != nacc)
+ {
+ gmx_fatal(FARGS, "Invalid Acceleration input: %d groups and %d acc. values",
+ nacg, nacc);
+ }
+ do_numbering(natoms, groups, nacg, ptr2, grps, gnames, egcACC,
+ restnm, egrptpALL_GENREST, bVerbose, wi);
+ nr = groups->grps[egcACC].nr;
+ snew(ir->opts.acc, nr);
+ ir->opts.ngacc = nr;
+
+ for (i = k = 0; (i < nacg); i++)
+ {
+ for (j = 0; (j < DIM); j++, k++)
+ {
+ ir->opts.acc[i][j] = strtod(ptr1[k], NULL);
+ }
+ }
+ for (; (i < nr); i++)
+ {
+ for (j = 0; (j < DIM); j++)
+ {
+ ir->opts.acc[i][j] = 0;
+ }
+ }
+
+ nfrdim = str_nelem(frdim, MAXPTR, ptr1);
+ nfreeze = str_nelem(freeze, MAXPTR, ptr2);
+ if (nfrdim != DIM*nfreeze)
+ {
+ gmx_fatal(FARGS, "Invalid Freezing input: %d groups and %d freeze values",
+ nfreeze, nfrdim);
+ }
+ do_numbering(natoms, groups, nfreeze, ptr2, grps, gnames, egcFREEZE,
+ restnm, egrptpALL_GENREST, bVerbose, wi);
+ nr = groups->grps[egcFREEZE].nr;
+ ir->opts.ngfrz = nr;
+ snew(ir->opts.nFreeze, nr);
+ for (i = k = 0; (i < nfreeze); i++)
+ {
+ for (j = 0; (j < DIM); j++, k++)
+ {
+ ir->opts.nFreeze[i][j] = (gmx_strncasecmp(ptr1[k], "Y", 1) == 0);
+ if (!ir->opts.nFreeze[i][j])
+ {
+ if (gmx_strncasecmp(ptr1[k], "N", 1) != 0)
+ {
+ sprintf(warnbuf, "Please use Y(ES) or N(O) for freezedim only "
+ "(not %s)", ptr1[k]);
+ warning(wi, warn_buf);
+ }
+ }
+ }
+ }
+ for (; (i < nr); i++)
+ {
+ for (j = 0; (j < DIM); j++)
+ {
+ ir->opts.nFreeze[i][j] = 0;
+ }
+ }
+
+ nenergy = str_nelem(energy, MAXPTR, ptr1);
+ do_numbering(natoms, groups, nenergy, ptr1, grps, gnames, egcENER,
+ restnm, egrptpALL_GENREST, bVerbose, wi);
+ add_wall_energrps(groups, ir->nwall, symtab);
+ ir->opts.ngener = groups->grps[egcENER].nr;
+ nvcm = str_nelem(vcm, MAXPTR, ptr1);
+ bRest =
+ do_numbering(natoms, groups, nvcm, ptr1, grps, gnames, egcVCM,
+ restnm, nvcm == 0 ? egrptpALL_GENREST : egrptpPART, bVerbose, wi);
+ if (bRest)
+ {
+ warning(wi, "Some atoms are not part of any center of mass motion removal group.\n"
+ "This may lead to artifacts.\n"
+ "In most cases one should use one group for the whole system.");
+ }
+
+ /* Now we have filled the freeze struct, so we can calculate NRDF */
+ calc_nrdf(mtop, ir, gnames);
+
+ if (v && NULL)
+ {
+ real fac, ntot = 0;
+
+ /* Must check per group! */
+ for (i = 0; (i < ir->opts.ngtc); i++)
+ {
+ ntot += ir->opts.nrdf[i];
+ }
+ if (ntot != (DIM*natoms))
+ {
+ fac = sqrt(ntot/(DIM*natoms));
+ if (bVerbose)
+ {
+ fprintf(stderr, "Scaling velocities by a factor of %.3f to account for constraints\n"
+ "and removal of center of mass motion\n", fac);
+ }
+ for (i = 0; (i < natoms); i++)
+ {
+ svmul(fac, v[i], v[i]);
+ }
+ }
+ }
+
+ nuser = str_nelem(user1, MAXPTR, ptr1);
+ do_numbering(natoms, groups, nuser, ptr1, grps, gnames, egcUser1,
+ restnm, egrptpALL_GENREST, bVerbose, wi);
+ nuser = str_nelem(user2, MAXPTR, ptr1);
+ do_numbering(natoms, groups, nuser, ptr1, grps, gnames, egcUser2,
+ restnm, egrptpALL_GENREST, bVerbose, wi);
+ nuser = str_nelem(xtc_grps, MAXPTR, ptr1);
+ do_numbering(natoms, groups, nuser, ptr1, grps, gnames, egcXTC,
+ restnm, egrptpONE, bVerbose, wi);
+ nofg = str_nelem(orirefitgrp, MAXPTR, ptr1);
+ do_numbering(natoms, groups, nofg, ptr1, grps, gnames, egcORFIT,
+ restnm, egrptpALL_GENREST, bVerbose, wi);
+
+ /* QMMM input processing */
+ nQMg = str_nelem(QMMM, MAXPTR, ptr1);
+ nQMmethod = str_nelem(QMmethod, MAXPTR, ptr2);
+ nQMbasis = str_nelem(QMbasis, MAXPTR, ptr3);
+ if ((nQMmethod != nQMg) || (nQMbasis != nQMg))
+ {
+ gmx_fatal(FARGS, "Invalid QMMM input: %d groups %d basissets"
+ " and %d methods\n", nQMg, nQMbasis, nQMmethod);
+ }
+ /* group rest, if any, is always MM! */
+ do_numbering(natoms, groups, nQMg, ptr1, grps, gnames, egcQMMM,
+ restnm, egrptpALL_GENREST, bVerbose, wi);
+ nr = nQMg; /*atoms->grps[egcQMMM].nr;*/
+ ir->opts.ngQM = nQMg;
+ snew(ir->opts.QMmethod, nr);
+ snew(ir->opts.QMbasis, nr);
+ for (i = 0; i < nr; i++)
+ {
+ /* input consists of strings: RHF CASSCF PM3 .. These need to be
+ * converted to the corresponding enum in names.c
+ */
+ ir->opts.QMmethod[i] = search_QMstring(ptr2[i], eQMmethodNR,
+ eQMmethod_names);
+ ir->opts.QMbasis[i] = search_QMstring(ptr3[i], eQMbasisNR,
+ eQMbasis_names);
+
+ }
+ nQMmult = str_nelem(QMmult, MAXPTR, ptr1);
+ nQMcharge = str_nelem(QMcharge, MAXPTR, ptr2);
+ nbSH = str_nelem(bSH, MAXPTR, ptr3);
+ snew(ir->opts.QMmult, nr);
+ snew(ir->opts.QMcharge, nr);
+ snew(ir->opts.bSH, nr);
+
+ for (i = 0; i < nr; i++)
+ {
+ ir->opts.QMmult[i] = strtol(ptr1[i], NULL, 10);
+ ir->opts.QMcharge[i] = strtol(ptr2[i], NULL, 10);
+ ir->opts.bSH[i] = (gmx_strncasecmp(ptr3[i], "Y", 1) == 0);
+ }
+
+ nCASelec = str_nelem(CASelectrons, MAXPTR, ptr1);
+ nCASorb = str_nelem(CASorbitals, MAXPTR, ptr2);
+ snew(ir->opts.CASelectrons, nr);
+ snew(ir->opts.CASorbitals, nr);
+ for (i = 0; i < nr; i++)
+ {
+ ir->opts.CASelectrons[i] = strtol(ptr1[i], NULL, 10);
+ ir->opts.CASorbitals[i] = strtol(ptr2[i], NULL, 10);
+ }
+ /* special optimization options */
+
+ nbOPT = str_nelem(bOPT, MAXPTR, ptr1);
+ nbTS = str_nelem(bTS, MAXPTR, ptr2);
+ snew(ir->opts.bOPT, nr);
+ snew(ir->opts.bTS, nr);
+ for (i = 0; i < nr; i++)
+ {
+ ir->opts.bOPT[i] = (gmx_strncasecmp(ptr1[i], "Y", 1) == 0);
+ ir->opts.bTS[i] = (gmx_strncasecmp(ptr2[i], "Y", 1) == 0);
+ }
+ nSAon = str_nelem(SAon, MAXPTR, ptr1);
+ nSAoff = str_nelem(SAoff, MAXPTR, ptr2);
+ nSAsteps = str_nelem(SAsteps, MAXPTR, ptr3);
+ snew(ir->opts.SAon, nr);
+ snew(ir->opts.SAoff, nr);
+ snew(ir->opts.SAsteps, nr);
+
+ for (i = 0; i < nr; i++)
+ {
+ ir->opts.SAon[i] = strtod(ptr1[i], NULL);
+ ir->opts.SAoff[i] = strtod(ptr2[i], NULL);
+ ir->opts.SAsteps[i] = strtol(ptr3[i], NULL, 10);
+ }
+ /* end of QMMM input */
+
+ if (bVerbose)
+ {
+ for (i = 0; (i < egcNR); i++)
+ {
+ fprintf(stderr, "%-16s has %d element(s):", gtypes[i], groups->grps[i].nr);
+ for (j = 0; (j < groups->grps[i].nr); j++)
+ {
+ fprintf(stderr, " %s", *(groups->grpname[groups->grps[i].nm_ind[j]]));
+ }
+ fprintf(stderr, "\n");
+ }
+ }
+
+ nr = groups->grps[egcENER].nr;
+ snew(ir->opts.egp_flags, nr*nr);
+
+ bExcl = do_egp_flag(ir, groups, "energygrp-excl", egpexcl, EGP_EXCL);
+ if (bExcl && ir->cutoff_scheme == ecutsVERLET)
+ {
+ warning_error(wi, "Energy group exclusions are not (yet) implemented for the Verlet scheme");
+ }
+ if (bExcl && EEL_FULL(ir->coulombtype))
+ {
+ warning(wi, "Can not exclude the lattice Coulomb energy between energy groups");
+ }
+
+ bTable = do_egp_flag(ir, groups, "energygrp-table", egptable, EGP_TABLE);
+ if (bTable && !(ir->vdwtype == evdwUSER) &&
+ !(ir->coulombtype == eelUSER) && !(ir->coulombtype == eelPMEUSER) &&
+ !(ir->coulombtype == eelPMEUSERSWITCH))
+ {
+ gmx_fatal(FARGS, "Can only have energy group pair tables in combination with user tables for VdW and/or Coulomb");
+ }
+
+ decode_cos(efield_x, &(ir->ex[XX]));
+ decode_cos(efield_xt, &(ir->et[XX]));
+ decode_cos(efield_y, &(ir->ex[YY]));
+ decode_cos(efield_yt, &(ir->et[YY]));
+ decode_cos(efield_z, &(ir->ex[ZZ]));
+ decode_cos(efield_zt, &(ir->et[ZZ]));
+
+ if (ir->bAdress)
+ {
+ do_adress_index(ir->adress, groups, gnames, &(ir->opts), wi);
+ }
+
+ for (i = 0; (i < grps->nr); i++)
+ {
+ sfree(gnames[i]);
+ }
+ sfree(gnames);
+ done_blocka(grps);
+ sfree(grps);
+
+}
+
+
+
+static void check_disre(gmx_mtop_t *mtop)
+{
+ gmx_ffparams_t *ffparams;
+ t_functype *functype;
+ t_iparams *ip;
+ int i, ndouble, ftype;
+ int label, old_label;
+
+ if (gmx_mtop_ftype_count(mtop, F_DISRES) > 0)
+ {
+ ffparams = &mtop->ffparams;
+ functype = ffparams->functype;
+ ip = ffparams->iparams;
+ ndouble = 0;
+ old_label = -1;
+ for (i = 0; i < ffparams->ntypes; i++)
+ {
+ ftype = functype[i];
+ if (ftype == F_DISRES)
+ {
+ label = ip[i].disres.label;
+ if (label == old_label)
+ {
+ fprintf(stderr, "Distance restraint index %d occurs twice\n", label);
+ ndouble++;
+ }
+ old_label = label;
+ }
+ }
+ if (ndouble > 0)
+ {
+ gmx_fatal(FARGS, "Found %d double distance restraint indices,\n"
+ "probably the parameters for multiple pairs in one restraint "
+ "are not identical\n", ndouble);
+ }
+ }
+}
+
+static gmx_bool absolute_reference(t_inputrec *ir, gmx_mtop_t *sys,
+ gmx_bool posres_only,
+ ivec AbsRef)
+{
+ int d, g, i;
+ gmx_mtop_ilistloop_t iloop;
+ t_ilist *ilist;
+ int nmol;
+ t_iparams *pr;
+
+ clear_ivec(AbsRef);
+
+ if (!posres_only)
+ {
+ /* Check the COM */
+ for (d = 0; d < DIM; d++)
+ {
+ AbsRef[d] = (d < ndof_com(ir) ? 0 : 1);
+ }
+ /* Check for freeze groups */
+ for (g = 0; g < ir->opts.ngfrz; g++)
+ {
+ for (d = 0; d < DIM; d++)
+ {
+ if (ir->opts.nFreeze[g][d] != 0)
+ {
+ AbsRef[d] = 1;
+ }
+ }
+ }
+ }
+
+ /* Check for position restraints */
+ iloop = gmx_mtop_ilistloop_init(sys);
+ while (gmx_mtop_ilistloop_next(iloop, &ilist, &nmol))
+ {
+ if (nmol > 0 &&
+ (AbsRef[XX] == 0 || AbsRef[YY] == 0 || AbsRef[ZZ] == 0))
+ {
+ for (i = 0; i < ilist[F_POSRES].nr; i += 2)
+ {
+ pr = &sys->ffparams.iparams[ilist[F_POSRES].iatoms[i]];
+ for (d = 0; d < DIM; d++)
+ {
+ if (pr->posres.fcA[d] != 0)
+ {
+ AbsRef[d] = 1;
+ }
+ }
+ }
+ for (i = 0; i < ilist[F_FBPOSRES].nr; i += 2)
+ {
+ /* Check for flat-bottom posres */
+ pr = &sys->ffparams.iparams[ilist[F_FBPOSRES].iatoms[i]];
+ if (pr->fbposres.k != 0)
+ {
+ switch (pr->fbposres.geom)
+ {
+ case efbposresSPHERE:
+ AbsRef[XX] = AbsRef[YY] = AbsRef[ZZ] = 1;
+ break;
+ case efbposresCYLINDER:
+ AbsRef[XX] = AbsRef[YY] = 1;
+ break;
+ case efbposresX: /* d=XX */
+ case efbposresY: /* d=YY */
+ case efbposresZ: /* d=ZZ */
+ d = pr->fbposres.geom - efbposresX;
+ AbsRef[d] = 1;
+ break;
+ default:
+ gmx_fatal(FARGS, " Invalid geometry for flat-bottom position restraint.\n"
+ "Expected nr between 1 and %d. Found %d\n", efbposresNR-1,
+ pr->fbposres.geom);
+ }
+ }
+ }
+ }
+ }
+
+ return (AbsRef[XX] != 0 && AbsRef[YY] != 0 && AbsRef[ZZ] != 0);
+}
+
+void triple_check(const char *mdparin, t_inputrec *ir, gmx_mtop_t *sys,
+ warninp_t wi)
+{
+ char err_buf[256];
+ int i, m, g, nmol, npct;
+ gmx_bool bCharge, bAcc;
+ real gdt_max, *mgrp, mt;
+ rvec acc;
+ gmx_mtop_atomloop_block_t aloopb;
+ gmx_mtop_atomloop_all_t aloop;
+ t_atom *atom;
+ ivec AbsRef;
+ char warn_buf[STRLEN];
+
+ set_warning_line(wi, mdparin, -1);
+
+ if (EI_DYNAMICS(ir->eI) && !EI_SD(ir->eI) && ir->eI != eiBD &&
+ ir->comm_mode == ecmNO &&
+ !(absolute_reference(ir, sys, FALSE, AbsRef) || ir->nsteps <= 10))
+ {
+ warning(wi, "You are not using center of mass motion removal (mdp option comm-mode), numerical rounding errors can lead to build up of kinetic energy of the center of mass");
+ }
+
+ /* Check for pressure coupling with absolute position restraints */
+ if (ir->epc != epcNO && ir->refcoord_scaling == erscNO)
+ {
+ absolute_reference(ir, sys, TRUE, AbsRef);
+ {
+ for (m = 0; m < DIM; m++)
+ {
+ if (AbsRef[m] && norm2(ir->compress[m]) > 0)
+ {
+ warning(wi, "You are using pressure coupling with absolute position restraints, this will give artifacts. Use the refcoord_scaling option.");
+ break;
+ }
+ }
+ }
+ }
+
+ bCharge = FALSE;
+ aloopb = gmx_mtop_atomloop_block_init(sys);
+ while (gmx_mtop_atomloop_block_next(aloopb, &atom, &nmol))
+ {
+ if (atom->q != 0 || atom->qB != 0)
+ {
+ bCharge = TRUE;
+ }
+ }
+
+ if (!bCharge)
+ {
+ if (EEL_FULL(ir->coulombtype))
+ {
+ sprintf(err_buf,
+ "You are using full electrostatics treatment %s for a system without charges.\n"
+ "This costs a lot of performance for just processing zeros, consider using %s instead.\n",
+ EELTYPE(ir->coulombtype), EELTYPE(eelCUT));
+ warning(wi, err_buf);
+ }
+ }
+ else
+ {
+ if (ir->coulombtype == eelCUT && ir->rcoulomb > 0 && !ir->implicit_solvent)
+ {
+ sprintf(err_buf,
+ "You are using a plain Coulomb cut-off, which might produce artifacts.\n"
+ "You might want to consider using %s electrostatics.\n",
+ EELTYPE(eelPME));
+ warning_note(wi, err_buf);
+ }
+ }
+
+ /* Generalized reaction field */
+ if (ir->opts.ngtc == 0)
+ {
+ sprintf(err_buf, "No temperature coupling while using coulombtype %s",
+ eel_names[eelGRF]);
+ CHECK(ir->coulombtype == eelGRF);
+ }
+ else
+ {
+ sprintf(err_buf, "When using coulombtype = %s"
+ " ref-t for temperature coupling should be > 0",
+ eel_names[eelGRF]);
+ CHECK((ir->coulombtype == eelGRF) && (ir->opts.ref_t[0] <= 0));
+ }
+
+ if (ir->eI == eiSD1 &&
+ (gmx_mtop_ftype_count(sys, F_CONSTR) > 0 ||
+ gmx_mtop_ftype_count(sys, F_SETTLE) > 0))
+ {
+ sprintf(warn_buf, "With constraints integrator %s is less accurate, consider using %s instead", ei_names[ir->eI], ei_names[eiSD2]);
+ warning_note(wi, warn_buf);
+ }
+
+ bAcc = FALSE;
+ for (i = 0; (i < sys->groups.grps[egcACC].nr); i++)
+ {
+ for (m = 0; (m < DIM); m++)
+ {
+ if (fabs(ir->opts.acc[i][m]) > 1e-6)
+ {
+ bAcc = TRUE;
+ }
+ }
+ }
+ if (bAcc)
+ {
+ clear_rvec(acc);
+ snew(mgrp, sys->groups.grps[egcACC].nr);
+ aloop = gmx_mtop_atomloop_all_init(sys);
+ while (gmx_mtop_atomloop_all_next(aloop, &i, &atom))
+ {
+ mgrp[ggrpnr(&sys->groups, egcACC, i)] += atom->m;
+ }
+ mt = 0.0;
+ for (i = 0; (i < sys->groups.grps[egcACC].nr); i++)
+ {
+ for (m = 0; (m < DIM); m++)
+ {
+ acc[m] += ir->opts.acc[i][m]*mgrp[i];
+ }
+ mt += mgrp[i];
+ }
+ for (m = 0; (m < DIM); m++)
+ {
+ if (fabs(acc[m]) > 1e-6)
+ {
+ const char *dim[DIM] = { "X", "Y", "Z" };
+ fprintf(stderr,
+ "Net Acceleration in %s direction, will %s be corrected\n",
+ dim[m], ir->nstcomm != 0 ? "" : "not");
+ if (ir->nstcomm != 0 && m < ndof_com(ir))
+ {
+ acc[m] /= mt;
+ for (i = 0; (i < sys->groups.grps[egcACC].nr); i++)
+ {
+ ir->opts.acc[i][m] -= acc[m];
+ }
+ }
+ }
+ }
+ sfree(mgrp);
+ }
+
+ if (ir->efep != efepNO && ir->fepvals->sc_alpha != 0 &&
+ !gmx_within_tol(sys->ffparams.reppow, 12.0, 10*GMX_DOUBLE_EPS))
+ {
+ gmx_fatal(FARGS, "Soft-core interactions are only supported with VdW repulsion power 12");
+ }
+
+ if (ir->ePull != epullNO)
+ {
+ if (ir->pull->grp[0].nat == 0)
+ {
+ absolute_reference(ir, sys, FALSE, AbsRef);
+ for (m = 0; m < DIM; m++)
+ {
+ if (ir->pull->dim[m] && !AbsRef[m])
+ {
+ warning(wi, "You are using an absolute reference for pulling, but the rest of the system does not have an absolute reference. This will lead to artifacts.");
+ break;
+ }
+ }
+ }
+
+ if (ir->pull->eGeom == epullgDIRPBC)
+ {
+ for (i = 0; i < 3; i++)
+ {
+ for (m = 0; m <= i; m++)
+ {
+ if ((ir->epc != epcNO && ir->compress[i][m] != 0) ||
+ ir->deform[i][m] != 0)
+ {
+ for (g = 1; g < ir->pull->ngrp; g++)
+ {
+ if (ir->pull->grp[g].vec[m] != 0)
+ {
+ gmx_fatal(FARGS, "Can not have dynamic box while using pull geometry '%s' (dim %c)", EPULLGEOM(ir->pull->eGeom), 'x'+m);
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ check_disre(sys);
+}
+
+void double_check(t_inputrec *ir, matrix box, gmx_bool bConstr, warninp_t wi)
+{
+ real min_size;
+ gmx_bool bTWIN;
+ char warn_buf[STRLEN];
+ const char *ptr;
+
+ ptr = check_box(ir->ePBC, box);
+ if (ptr)
+ {
+ warning_error(wi, ptr);
+ }
+
+ if (bConstr && ir->eConstrAlg == econtSHAKE)
+ {
+ if (ir->shake_tol <= 0.0)
+ {
+ sprintf(warn_buf, "ERROR: shake-tol must be > 0 instead of %g\n",
+ ir->shake_tol);
+ warning_error(wi, warn_buf);
+ }
+
+ if (IR_TWINRANGE(*ir) && ir->nstlist > 1)
+ {
+ sprintf(warn_buf, "With twin-range cut-off's and SHAKE the virial and the pressure are incorrect.");
+ if (ir->epc == epcNO)
+ {
+ warning(wi, warn_buf);
+ }
+ else
+ {
+ warning_error(wi, warn_buf);
+ }
+ }
+ }
+
+ if ( (ir->eConstrAlg == econtLINCS) && bConstr)
+ {
+ /* If we have Lincs constraints: */
+ if (ir->eI == eiMD && ir->etc == etcNO &&
+ ir->eConstrAlg == econtLINCS && ir->nLincsIter == 1)
+ {
+ sprintf(warn_buf, "For energy conservation with LINCS, lincs_iter should be 2 or larger.\n");
+ warning_note(wi, warn_buf);
+ }
+
+ if ((ir->eI == eiCG || ir->eI == eiLBFGS) && (ir->nProjOrder < 8))
+ {
+ sprintf(warn_buf, "For accurate %s with LINCS constraints, lincs-order should be 8 or more.", ei_names[ir->eI]);
+ warning_note(wi, warn_buf);
+ }
+ if (ir->epc == epcMTTK)
+ {
+ warning_error(wi, "MTTK not compatible with lincs -- use shake instead.");
+ }
+ }
+
+ if (ir->LincsWarnAngle > 90.0)
+ {
+ sprintf(warn_buf, "lincs-warnangle can not be larger than 90 degrees, setting it to 90.\n");
+ warning(wi, warn_buf);
+ ir->LincsWarnAngle = 90.0;
+ }
+
+ if (ir->ePBC != epbcNONE)
+ {
+ if (ir->nstlist == 0)
+ {
+ warning(wi, "With nstlist=0 atoms are only put into the box at step 0, therefore drifting atoms might cause the simulation to crash.");
+ }
+ bTWIN = (ir->rlistlong > ir->rlist);
+ if (ir->ns_type == ensGRID)
+ {
+ if (sqr(ir->rlistlong) >= max_cutoff2(ir->ePBC, box))
+ {
+ sprintf(warn_buf, "ERROR: The cut-off length is longer than half the shortest box vector or longer than the smallest box diagonal element. Increase the box size or decrease %s.\n",
+ bTWIN ? (ir->rcoulomb == ir->rlistlong ? "rcoulomb" : "rvdw") : "rlist");
+ warning_error(wi, warn_buf);
+ }
+ }
+ else
+ {
+ min_size = min(box[XX][XX], min(box[YY][YY], box[ZZ][ZZ]));
+ if (2*ir->rlistlong >= min_size)
+ {
+ sprintf(warn_buf, "ERROR: One of the box lengths is smaller than twice the cut-off length. Increase the box size or decrease rlist.");
+ warning_error(wi, warn_buf);
+ if (TRICLINIC(box))
+ {
+ fprintf(stderr, "Grid search might allow larger cut-off's than simple search with triclinic boxes.");
+ }
+ }
+ }
+ }
+}
+
+void check_chargegroup_radii(const gmx_mtop_t *mtop, const t_inputrec *ir,
+ rvec *x,
+ warninp_t wi)
+{
+ real rvdw1, rvdw2, rcoul1, rcoul2;
+ char warn_buf[STRLEN];
+
+ calc_chargegroup_radii(mtop, x, &rvdw1, &rvdw2, &rcoul1, &rcoul2);
+
+ if (rvdw1 > 0)
+ {
+ printf("Largest charge group radii for Van der Waals: %5.3f, %5.3f nm\n",
+ rvdw1, rvdw2);
+ }
+ if (rcoul1 > 0)
+ {
+ printf("Largest charge group radii for Coulomb: %5.3f, %5.3f nm\n",
+ rcoul1, rcoul2);
+ }
+
+ if (ir->rlist > 0)
+ {
+ if (rvdw1 + rvdw2 > ir->rlist ||
+ rcoul1 + rcoul2 > ir->rlist)
+ {
+ sprintf(warn_buf,
+ "The sum of the two largest charge group radii (%f) "
+ "is larger than rlist (%f)\n",
+ max(rvdw1+rvdw2, rcoul1+rcoul2), ir->rlist);
+ warning(wi, warn_buf);
+ }
+ else
+ {
+ /* Here we do not use the zero at cut-off macro,
+ * since user defined interactions might purposely
+ * not be zero at the cut-off.
+ */
+ if ((EVDW_IS_ZERO_AT_CUTOFF(ir->vdwtype) ||
+ ir->vdw_modifier != eintmodNONE) &&
+ rvdw1 + rvdw2 > ir->rlistlong - ir->rvdw)
+ {
+ sprintf(warn_buf, "The sum of the two largest charge group "
+ "radii (%f) is larger than %s (%f) - rvdw (%f).\n"
+ "With exact cut-offs, better performance can be "
+ "obtained with cutoff-scheme = %s, because it "
+ "does not use charge groups at all.",
+ rvdw1+rvdw2,
+ ir->rlistlong > ir->rlist ? "rlistlong" : "rlist",
+ ir->rlistlong, ir->rvdw,
+ ecutscheme_names[ecutsVERLET]);
+ if (ir_NVE(ir))
+ {
+ warning(wi, warn_buf);
+ }
+ else
+ {
+ warning_note(wi, warn_buf);
+ }
+ }
+ if ((EEL_IS_ZERO_AT_CUTOFF(ir->coulombtype) ||
+ ir->coulomb_modifier != eintmodNONE) &&
+ rcoul1 + rcoul2 > ir->rlistlong - ir->rcoulomb)
+ {
+ sprintf(warn_buf, "The sum of the two largest charge group radii (%f) is larger than %s (%f) - rcoulomb (%f).\n"
+ "With exact cut-offs, better performance can be obtained with cutoff-scheme = %s, because it does not use charge groups at all.",
+ rcoul1+rcoul2,
+ ir->rlistlong > ir->rlist ? "rlistlong" : "rlist",
+ ir->rlistlong, ir->rcoulomb,
+ ecutscheme_names[ecutsVERLET]);
+ if (ir_NVE(ir))
+ {
+ warning(wi, warn_buf);
+ }
+ else
+ {
+ warning_note(wi, warn_buf);
+ }
+ }
+ }
+ }
+}
--- /dev/null
- t_param *param;
- int i;
- real v, w;
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * VERSION 3.2.0
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2004, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * Gallium Rubidium Oxygen Manganese Argon Carbon Silicon
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <ctype.h>
+#include <math.h>
+
+#include "sysstuff.h"
+#include "smalloc.h"
+#include "macros.h"
+#include "string2.h"
+#include "names.h"
+#include "toputil.h"
+#include "toppush.h"
+#include "topdirs.h"
+#include "readir.h"
+#include "symtab.h"
+#include "gmx_fatal.h"
+#include "warninp.h"
+#include "gpp_atomtype.h"
+#include "gpp_bond_atomtype.h"
+
+void generate_nbparams(int comb, int ftype, t_params *plist, gpp_atomtype_t atype,
+ warninp_t wi)
+{
+ int i, j, k = -1, nf;
+ int nr, nrfp;
+ real c, bi, bj, ci, cj, ci0, ci1, ci2, cj0, cj1, cj2;
+ char errbuf[256];
+
+ /* Lean mean shortcuts */
+ nr = get_atomtype_ntypes(atype);
+ nrfp = NRFP(ftype);
+ snew(plist->param, nr*nr);
+ plist->nr = nr*nr;
+
+ /* Fill the matrix with force parameters */
+ switch (ftype)
+ {
+ case F_LJ:
+ switch (comb)
+ {
+ case eCOMB_GEOMETRIC:
+ /* Gromos rules */
+ for (i = k = 0; (i < nr); i++)
+ {
+ for (j = 0; (j < nr); j++, k++)
+ {
+ for (nf = 0; (nf < nrfp); nf++)
+ {
+ ci = get_atomtype_nbparam(i, nf, atype);
+ cj = get_atomtype_nbparam(j, nf, atype);
+ c = sqrt(ci * cj);
+ plist->param[k].c[nf] = c;
+ }
+ }
+ }
+ break;
+
+ case eCOMB_ARITHMETIC:
+ /* c0 and c1 are epsilon and sigma */
+ for (i = k = 0; (i < nr); i++)
+ {
+ for (j = 0; (j < nr); j++, k++)
+ {
+ ci0 = get_atomtype_nbparam(i, 0, atype);
+ cj0 = get_atomtype_nbparam(j, 0, atype);
+ ci1 = get_atomtype_nbparam(i, 1, atype);
+ cj1 = get_atomtype_nbparam(j, 1, atype);
+ plist->param[k].c[0] = (ci0+cj0)*0.5;
+ plist->param[k].c[1] = sqrt(ci1*cj1);
+ }
+ }
+
+ break;
+ case eCOMB_GEOM_SIG_EPS:
+ /* c0 and c1 are epsilon and sigma */
+ for (i = k = 0; (i < nr); i++)
+ {
+ for (j = 0; (j < nr); j++, k++)
+ {
+ ci0 = get_atomtype_nbparam(i, 0, atype);
+ cj0 = get_atomtype_nbparam(j, 0, atype);
+ ci1 = get_atomtype_nbparam(i, 1, atype);
+ cj1 = get_atomtype_nbparam(j, 1, atype);
+ plist->param[k].c[0] = sqrt(ci0*cj0);
+ plist->param[k].c[1] = sqrt(ci1*cj1);
+ }
+ }
+
+ break;
+ default:
+ gmx_fatal(FARGS, "No such combination rule %d", comb);
+ }
+ if (plist->nr != k)
+ {
+ gmx_incons("Topology processing, generate nb parameters");
+ }
+ break;
+
+ case F_BHAM:
+ /* Buckingham rules */
+ for (i = k = 0; (i < nr); i++)
+ {
+ for (j = 0; (j < nr); j++, k++)
+ {
+ ci0 = get_atomtype_nbparam(i, 0, atype);
+ cj0 = get_atomtype_nbparam(j, 0, atype);
+ ci2 = get_atomtype_nbparam(i, 2, atype);
+ cj2 = get_atomtype_nbparam(j, 2, atype);
+ bi = get_atomtype_nbparam(i, 1, atype);
+ bj = get_atomtype_nbparam(j, 1, atype);
+ plist->param[k].c[0] = sqrt(ci0 * cj0);
+ if ((bi == 0) || (bj == 0))
+ {
+ plist->param[k].c[1] = 0;
+ }
+ else
+ {
+ plist->param[k].c[1] = 2.0/(1/bi+1/bj);
+ }
+ plist->param[k].c[2] = sqrt(ci2 * cj2);
+ }
+ }
+
+ break;
+ default:
+ sprintf(errbuf, "Invalid nonbonded type %s",
+ interaction_function[ftype].longname);
+ warning_error(wi, errbuf);
+ }
+}
+
+static void realloc_nb_params(gpp_atomtype_t at,
+ t_nbparam ***nbparam, t_nbparam ***pair)
+{
+ /* Add space in the non-bonded parameters matrix */
+ int atnr = get_atomtype_ntypes(at);
+ srenew(*nbparam, atnr);
+ snew((*nbparam)[atnr-1], atnr);
+ if (pair)
+ {
+ srenew(*pair, atnr);
+ snew((*pair)[atnr-1], atnr);
+ }
+}
+
+static void copy_B_from_A(int ftype, double *c)
+{
+ int nrfpA, nrfpB, i;
+
+ nrfpA = NRFPA(ftype);
+ nrfpB = NRFPB(ftype);
+
+ /* Copy the B parameters from the first nrfpB A parameters */
+ for (i = 0; (i < nrfpB); i++)
+ {
+ c[nrfpA+i] = c[i];
+ }
+}
+
+void push_at (t_symtab *symtab, gpp_atomtype_t at, t_bond_atomtype bat,
+ char *line, int nb_funct,
+ t_nbparam ***nbparam, t_nbparam ***pair,
+ warninp_t wi)
+{
+ typedef struct {
+ const char *entry;
+ int ptype;
+ } t_xlate;
+ t_xlate xl[eptNR] = {
+ { "A", eptAtom },
+ { "N", eptNucleus },
+ { "S", eptShell },
+ { "B", eptBond },
+ { "V", eptVSite },
+ };
+
+ int nr, i, nfields, j, pt, nfp0 = -1;
+ int batype_nr, nread;
+ char type[STRLEN], btype[STRLEN], ptype[STRLEN];
+ double m, q;
+ double c[MAXFORCEPARAM];
+ double radius, vol, surftens, gb_radius, S_hct;
+ char tmpfield[12][100]; /* Max 12 fields of width 100 */
+ char errbuf[256];
+ t_atom *atom;
+ t_param *param;
+ int atomnr;
+ gmx_bool have_atomic_number;
+ gmx_bool have_bonded_type;
+
+ snew(atom, 1);
+ snew(param, 1);
+
+ /* First assign input line to temporary array */
+ nfields = sscanf(line, "%s%s%s%s%s%s%s%s%s%s%s%s",
+ tmpfield[0], tmpfield[1], tmpfield[2], tmpfield[3], tmpfield[4], tmpfield[5],
+ tmpfield[6], tmpfield[7], tmpfield[8], tmpfield[9], tmpfield[10], tmpfield[11]);
+
+ /* Comments on optional fields in the atomtypes section:
+ *
+ * The force field format is getting a bit old. For OPLS-AA we needed
+ * to add a special bonded atomtype, and for Gerrit Groenhofs QM/MM stuff
+ * we also needed the atomic numbers.
+ * To avoid making all old or user-generated force fields unusable we
+ * have introduced both these quantities as optional columns, and do some
+ * acrobatics to check whether they are present or not.
+ * This will all look much nicer when we switch to XML... sigh.
+ *
+ * Field 0 (mandatory) is the nonbonded type name. (string)
+ * Field 1 (optional) is the bonded type (string)
+ * Field 2 (optional) is the atomic number (int)
+ * Field 3 (mandatory) is the mass (numerical)
+ * Field 4 (mandatory) is the charge (numerical)
+ * Field 5 (mandatory) is the particle type (single character)
+ * This is followed by a number of nonbonded parameters.
+ *
+ * The safest way to identify the format is the particle type field.
+ *
+ * So, here is what we do:
+ *
+ * A. Read in the first six fields as strings
+ * B. If field 3 (starting from 0) is a single char, we have neither
+ * bonded_type or atomic numbers.
+ * C. If field 5 is a single char we have both.
+ * D. If field 4 is a single char we check field 1. If this begins with
+ * an alphabetical character we have bonded types, otherwise atomic numbers.
+ */
+
+ if (nfields < 6)
+ {
+ too_few(wi);
+ return;
+ }
+
+ if ( (strlen(tmpfield[5]) == 1) && isalpha(tmpfield[5][0]) )
+ {
+ have_bonded_type = TRUE;
+ have_atomic_number = TRUE;
+ }
+ else if ( (strlen(tmpfield[3]) == 1) && isalpha(tmpfield[3][0]) )
+ {
+ have_bonded_type = FALSE;
+ have_atomic_number = FALSE;
+ }
+ else
+ {
+ have_bonded_type = ( isalpha(tmpfield[1][0]) != 0 );
+ have_atomic_number = !have_bonded_type;
+ }
+
+ /* optional fields */
+ surftens = -1;
+ vol = -1;
+ radius = -1;
+ gb_radius = -1;
+ atomnr = -1;
+ S_hct = -1;
+
+ switch (nb_funct)
+ {
+
+ case F_LJ:
+ nfp0 = 2;
+
+ if (have_atomic_number)
+ {
+ if (have_bonded_type)
+ {
+ nread = sscanf(line, "%s%s%d%lf%lf%s%lf%lf%lf%lf%lf%lf",
+ type, btype, &atomnr, &m, &q, ptype, &c[0], &c[1],
+ &radius, &vol, &surftens, &gb_radius);
+ if (nread < 8)
+ {
+ too_few(wi);
+ return;
+ }
+ }
+ else
+ {
+ /* have_atomic_number && !have_bonded_type */
+ nread = sscanf(line, "%s%d%lf%lf%s%lf%lf%lf%lf%lf%lf",
+ type, &atomnr, &m, &q, ptype, &c[0], &c[1],
+ &radius, &vol, &surftens, &gb_radius);
+ if (nread < 7)
+ {
+ too_few(wi);
+ return;
+ }
+ }
+ }
+ else
+ {
+ if (have_bonded_type)
+ {
+ /* !have_atomic_number && have_bonded_type */
+ nread = sscanf(line, "%s%s%lf%lf%s%lf%lf%lf%lf%lf%lf",
+ type, btype, &m, &q, ptype, &c[0], &c[1],
+ &radius, &vol, &surftens, &gb_radius);
+ if (nread < 7)
+ {
+ too_few(wi);
+ return;
+ }
+ }
+ else
+ {
+ /* !have_atomic_number && !have_bonded_type */
+ nread = sscanf(line, "%s%lf%lf%s%lf%lf%lf%lf%lf%lf",
+ type, &m, &q, ptype, &c[0], &c[1],
+ &radius, &vol, &surftens, &gb_radius);
+ if (nread < 6)
+ {
+ too_few(wi);
+ return;
+ }
+ }
+ }
+
+ if (!have_bonded_type)
+ {
+ strcpy(btype, type);
+ }
+
+ if (!have_atomic_number)
+ {
+ atomnr = -1;
+ }
+
+ break;
+
+ case F_BHAM:
+ nfp0 = 3;
+
+ if (have_atomic_number)
+ {
+ if (have_bonded_type)
+ {
+ nread = sscanf(line, "%s%s%d%lf%lf%s%lf%lf%lf%lf%lf%lf%lf",
+ type, btype, &atomnr, &m, &q, ptype, &c[0], &c[1], &c[2],
+ &radius, &vol, &surftens, &gb_radius);
+ if (nread < 9)
+ {
+ too_few(wi);
+ return;
+ }
+ }
+ else
+ {
+ /* have_atomic_number && !have_bonded_type */
+ nread = sscanf(line, "%s%d%lf%lf%s%lf%lf%lf%lf%lf%lf%lf",
+ type, &atomnr, &m, &q, ptype, &c[0], &c[1], &c[2],
+ &radius, &vol, &surftens, &gb_radius);
+ if (nread < 8)
+ {
+ too_few(wi);
+ return;
+ }
+ }
+ }
+ else
+ {
+ if (have_bonded_type)
+ {
+ /* !have_atomic_number && have_bonded_type */
+ nread = sscanf(line, "%s%s%lf%lf%s%lf%lf%lf%lf%lf%lf%lf",
+ type, btype, &m, &q, ptype, &c[0], &c[1], &c[2],
+ &radius, &vol, &surftens, &gb_radius);
+ if (nread < 8)
+ {
+ too_few(wi);
+ return;
+ }
+ }
+ else
+ {
+ /* !have_atomic_number && !have_bonded_type */
+ nread = sscanf(line, "%s%lf%lf%s%lf%lf%lf%lf%lf%lf%lf",
+ type, &m, &q, ptype, &c[0], &c[1], &c[2],
+ &radius, &vol, &surftens, &gb_radius);
+ if (nread < 7)
+ {
+ too_few(wi);
+ return;
+ }
+ }
+ }
+
+ if (!have_bonded_type)
+ {
+ strcpy(btype, type);
+ }
+
+ if (!have_atomic_number)
+ {
+ atomnr = -1;
+ }
+
+ break;
+
+ default:
+ gmx_fatal(FARGS, "Invalid function type %d in push_at %s %d", nb_funct,
+ __FILE__, __LINE__);
+ }
+ for (j = nfp0; (j < MAXFORCEPARAM); j++)
+ {
+ c[j] = 0.0;
+ }
+
+ if (strlen(type) == 1 && isdigit(type[0]))
+ {
+ gmx_fatal(FARGS, "Atom type names can't be single digits.");
+ }
+
+ if (strlen(btype) == 1 && isdigit(btype[0]))
+ {
+ gmx_fatal(FARGS, "Bond atom type names can't be single digits.");
+ }
+
+ /* Hack to read old topologies */
+ if (gmx_strcasecmp(ptype, "D") == 0)
+ {
+ sprintf(ptype, "V");
+ }
+ for (j = 0; (j < eptNR); j++)
+ {
+ if (gmx_strcasecmp(ptype, xl[j].entry) == 0)
+ {
+ break;
+ }
+ }
+ if (j == eptNR)
+ {
+ gmx_fatal(FARGS, "Invalid particle type %s on line %s",
+ ptype, line);
+ }
+ pt = xl[j].ptype;
+ if (debug)
+ {
+ fprintf(debug, "ptype: %s\n", ptype_str[pt]);
+ }
+
+ atom->q = q;
+ atom->m = m;
+ atom->ptype = pt;
+ for (i = 0; (i < MAXFORCEPARAM); i++)
+ {
+ param->c[i] = c[i];
+ }
+
+ if ((batype_nr = get_bond_atomtype_type(btype, bat)) == NOTSET)
+ {
+ add_bond_atomtype(bat, symtab, btype);
+ }
+ batype_nr = get_bond_atomtype_type(btype, bat);
+
+ if ((nr = get_atomtype_type(type, at)) != NOTSET)
+ {
+ sprintf(errbuf, "Overriding atomtype %s", type);
+ warning(wi, errbuf);
+ if ((nr = set_atomtype(nr, at, symtab, atom, type, param, batype_nr,
+ radius, vol, surftens, atomnr, gb_radius, S_hct)) == NOTSET)
+ {
+ gmx_fatal(FARGS, "Replacing atomtype %s failed", type);
+ }
+ }
+ else if ((nr = add_atomtype(at, symtab, atom, type, param,
+ batype_nr, radius, vol,
+ surftens, atomnr, gb_radius, S_hct)) == NOTSET)
+ {
+ gmx_fatal(FARGS, "Adding atomtype %s failed", type);
+ }
+ else
+ {
+ /* Add space in the non-bonded parameters matrix */
+ realloc_nb_params(at, nbparam, pair);
+ }
+ sfree(atom);
+ sfree(param);
+}
+
+static void push_bondtype(t_params * bt,
+ t_param * b,
+ int nral,
+ int ftype,
+ gmx_bool bAllowRepeat,
+ char * line,
+ warninp_t wi)
+{
+ int i, j;
+ gmx_bool bTest, bFound, bCont, bId;
+ int nr = bt->nr;
+ int nrfp = NRFP(ftype);
+ char errbuf[256];
+
+ /* If bAllowRepeat is TRUE, we allow multiple entries as long as they
+ are on directly _adjacent_ lines.
+ */
+
+ /* First check if our atomtypes are _identical_ (not reversed) to the previous
+ entry. If they are not identical we search for earlier duplicates. If they are
+ we can skip it, since we already searched for the first line
+ in this group.
+ */
+
+ bFound = FALSE;
+ bCont = FALSE;
+
+ if (bAllowRepeat && nr > 1)
+ {
+ for (j = 0, bCont = TRUE; (j < nral); j++)
+ {
+ bCont = bCont && (b->a[j] == bt->param[nr-2].a[j]);
+ }
+ }
+
+ /* Search for earlier duplicates if this entry was not a continuation
+ from the previous line.
+ */
+ if (!bCont)
+ {
+ bFound = FALSE;
+ for (i = 0; (i < nr); i++)
+ {
+ bTest = TRUE;
+ for (j = 0; (j < nral); j++)
+ {
+ bTest = (bTest && (b->a[j] == bt->param[i].a[j]));
+ }
+ if (!bTest)
+ {
+ bTest = TRUE;
+ for (j = 0; (j < nral); j++)
+ {
+ bTest = (bTest && (b->a[nral-1-j] == bt->param[i].a[j]));
+ }
+ }
+ if (bTest)
+ {
+ if (!bFound)
+ {
+ bId = TRUE;
+ for (j = 0; (j < nrfp); j++)
+ {
+ bId = bId && (bt->param[i].c[j] == b->c[j]);
+ }
+ if (!bId)
+ {
+ sprintf(errbuf, "Overriding %s parameters.%s",
+ interaction_function[ftype].longname,
+ (ftype == F_PDIHS) ? "\nUse dihedraltype 4 to allow several multiplicity terms." : "");
+ warning(wi, errbuf);
+ fprintf(stderr, " old:");
+ for (j = 0; (j < nrfp); j++)
+ {
+ fprintf(stderr, " %g", bt->param[i].c[j]);
+ }
+ fprintf(stderr, " \n new: %s\n\n", line);
+ }
+ }
+ /* Overwrite it! */
+ for (j = 0; (j < nrfp); j++)
+ {
+ bt->param[i].c[j] = b->c[j];
+ }
+ bFound = TRUE;
+ }
+ }
+ }
+ if (!bFound)
+ {
+ /* alloc */
+ pr_alloc (2, bt);
+
+ /* fill the arrays up and down */
+ memcpy(bt->param[bt->nr].c, b->c, sizeof(b->c));
+ memcpy(bt->param[bt->nr].a, b->a, sizeof(b->a));
+ memcpy(bt->param[bt->nr+1].c, b->c, sizeof(b->c));
+
+ /* The definitions of linear angles depend on the order of atoms,
+ * that means that for atoms i-j-k, with certain parameter a, the
+ * corresponding k-j-i angle will have parameter 1-a.
+ */
+ if (ftype == F_LINEAR_ANGLES)
+ {
+ bt->param[bt->nr+1].c[0] = 1-bt->param[bt->nr+1].c[0];
+ bt->param[bt->nr+1].c[2] = 1-bt->param[bt->nr+1].c[2];
+ }
+
+ for (j = 0; (j < nral); j++)
+ {
+ bt->param[bt->nr+1].a[j] = b->a[nral-1-j];
+ }
+
+ bt->nr += 2;
+ }
+}
+
+void push_bt(directive d, t_params bt[], int nral,
+ gpp_atomtype_t at,
+ t_bond_atomtype bat, char *line,
+ warninp_t wi)
+{
+ const char *formal[MAXATOMLIST+1] = {
+ "%s",
+ "%s%s",
+ "%s%s%s",
+ "%s%s%s%s",
+ "%s%s%s%s%s",
+ "%s%s%s%s%s%s",
+ "%s%s%s%s%s%s%s"
+ };
+ const char *formnl[MAXATOMLIST+1] = {
+ "%*s",
+ "%*s%*s",
+ "%*s%*s%*s",
+ "%*s%*s%*s%*s",
+ "%*s%*s%*s%*s%*s",
+ "%*s%*s%*s%*s%*s%*s",
+ "%*s%*s%*s%*s%*s%*s%*s"
+ };
+ const char *formlf = "%lf%lf%lf%lf%lf%lf%lf%lf%lf%lf%lf%lf%lf";
+ int i, ft, ftype, nn, nrfp, nrfpA, nrfpB;
+ char f1[STRLEN];
+ char alc[MAXATOMLIST+1][20];
+ /* One force parameter more, so we can check if we read too many */
+ double c[MAXFORCEPARAM+1];
+ t_param p;
+ char errbuf[256];
+
+ if ((bat && at) || (!bat && !at))
+ {
+ gmx_incons("You should pass either bat or at to push_bt");
+ }
+
+ /* Make format string (nral ints+functype) */
+ if ((nn = sscanf(line, formal[nral],
+ alc[0], alc[1], alc[2], alc[3], alc[4], alc[5])) != nral+1)
+ {
+ sprintf(errbuf, "Not enough atomtypes (%d instead of %d)", nn-1, nral);
+ warning_error(wi, errbuf);
+ return;
+ }
+
+ ft = strtol(alc[nral], NULL, 10);
+ ftype = ifunc_index(d, ft);
+ nrfp = NRFP(ftype);
+ nrfpA = interaction_function[ftype].nrfpA;
+ nrfpB = interaction_function[ftype].nrfpB;
+ strcpy(f1, formnl[nral]);
+ strcat(f1, formlf);
+ if ((nn = sscanf(line, f1, &c[0], &c[1], &c[2], &c[3], &c[4], &c[5], &c[6], &c[7], &c[8], &c[9], &c[10], &c[11], &c[12]))
+ != nrfp)
+ {
+ if (nn == nrfpA)
+ {
+ /* Copy the B-state from the A-state */
+ copy_B_from_A(ftype, c);
+ }
+ else
+ {
+ if (nn < nrfpA)
+ {
+ warning_error(wi, "Not enough parameters");
+ }
+ else if (nn > nrfpA && nn < nrfp)
+ {
+ warning_error(wi, "Too many parameters or not enough parameters for topology B");
+ }
+ else if (nn > nrfp)
+ {
+ warning_error(wi, "Too many parameters");
+ }
+ for (i = nn; (i < nrfp); i++)
+ {
+ c[i] = 0.0;
+ }
+ }
+ }
+ for (i = 0; (i < nral); i++)
+ {
+ if (at && ((p.a[i] = get_atomtype_type(alc[i], at)) == NOTSET))
+ {
+ gmx_fatal(FARGS, "Unknown atomtype %s\n", alc[i]);
+ }
+ else if (bat && ((p.a[i] = get_bond_atomtype_type(alc[i], bat)) == NOTSET))
+ {
+ gmx_fatal(FARGS, "Unknown bond_atomtype %s\n", alc[i]);
+ }
+ }
+ for (i = 0; (i < nrfp); i++)
+ {
+ p.c[i] = c[i];
+ }
+ push_bondtype (&(bt[ftype]), &p, nral, ftype, FALSE, line, wi);
+}
+
+
+void push_dihedraltype(directive d, t_params bt[],
+ t_bond_atomtype bat, char *line,
+ warninp_t wi)
+{
+ const char *formal[MAXATOMLIST+1] = {
+ "%s",
+ "%s%s",
+ "%s%s%s",
+ "%s%s%s%s",
+ "%s%s%s%s%s",
+ "%s%s%s%s%s%s",
+ "%s%s%s%s%s%s%s"
+ };
+ const char *formnl[MAXATOMLIST+1] = {
+ "%*s",
+ "%*s%*s",
+ "%*s%*s%*s",
+ "%*s%*s%*s%*s",
+ "%*s%*s%*s%*s%*s",
+ "%*s%*s%*s%*s%*s%*s",
+ "%*s%*s%*s%*s%*s%*s%*s"
+ };
+ const char *formlf[MAXFORCEPARAM] = {
+ "%lf",
+ "%lf%lf",
+ "%lf%lf%lf",
+ "%lf%lf%lf%lf",
+ "%lf%lf%lf%lf%lf",
+ "%lf%lf%lf%lf%lf%lf",
+ "%lf%lf%lf%lf%lf%lf%lf",
+ "%lf%lf%lf%lf%lf%lf%lf%lf",
+ "%lf%lf%lf%lf%lf%lf%lf%lf%lf",
+ "%lf%lf%lf%lf%lf%lf%lf%lf%lf%lf",
+ "%lf%lf%lf%lf%lf%lf%lf%lf%lf%lf%lf",
+ "%lf%lf%lf%lf%lf%lf%lf%lf%lf%lf%lf%lf",
+ };
+ int i, ft, ftype, nn, nrfp, nrfpA, nrfpB, nral;
+ char f1[STRLEN];
+ char alc[MAXATOMLIST+1][20];
+ double c[MAXFORCEPARAM];
+ t_param p;
+ gmx_bool bAllowRepeat;
+ char errbuf[256];
+
+ /* This routine accepts dihedraltypes defined from either 2 or 4 atoms.
+ *
+ * We first check for 2 atoms with the 3th column being an integer
+ * defining the type. If this isn't the case, we try it with 4 atoms
+ * and the 5th column defining the dihedral type.
+ */
+ nn = sscanf(line, formal[4], alc[0], alc[1], alc[2], alc[3], alc[4]);
+ if (nn >= 3 && strlen(alc[2]) == 1 && isdigit(alc[2][0]))
+ {
+ nral = 2;
+ ft = strtol(alc[nral], NULL, 10);
+ /* Move atom types around a bit and use 'X' for wildcard atoms
+ * to create a 4-atom dihedral definition with arbitrary atoms in
+ * position 1 and 4.
+ */
+ if (alc[2][0] == '2')
+ {
+ /* improper - the two atomtypes are 1,4. Use wildcards for 2,3 */
+ strcpy(alc[3], alc[1]);
+ sprintf(alc[2], "X");
+ sprintf(alc[1], "X");
+ /* alc[0] stays put */
+ }
+ else
+ {
+ /* proper - the two atomtypes are 2,3. Use wildcards for 1,4 */
+ sprintf(alc[3], "X");
+ strcpy(alc[2], alc[1]);
+ strcpy(alc[1], alc[0]);
+ sprintf(alc[0], "X");
+ }
+ }
+ else if (nn == 5 && strlen(alc[4]) == 1 && isdigit(alc[4][0]))
+ {
+ nral = 4;
+ ft = strtol(alc[nral], NULL, 10);
+ }
+ else
+ {
+ sprintf(errbuf, "Incorrect number of atomtypes for dihedral (%d instead of 2 or 4)", nn-1);
+ warning_error(wi, errbuf);
+ return;
+ }
+
+ if (ft == 9)
+ {
+ /* Previously, we have always overwritten parameters if e.g. a torsion
+ with the same atomtypes occurs on multiple lines. However, CHARMM and
+ some other force fields specify multiple dihedrals over some bonds,
+ including cosines with multiplicity 6 and somethimes even higher.
+ Thus, they cannot be represented with Ryckaert-Bellemans terms.
+ To add support for these force fields, Dihedral type 9 is identical to
+ normal proper dihedrals, but repeated entries are allowed.
+ */
+ bAllowRepeat = TRUE;
+ ft = 1;
+ }
+ else
+ {
+ bAllowRepeat = FALSE;
+ }
+
+
+ ftype = ifunc_index(d, ft);
+ nrfp = NRFP(ftype);
+ nrfpA = interaction_function[ftype].nrfpA;
+ nrfpB = interaction_function[ftype].nrfpB;
+
+ strcpy(f1, formnl[nral]);
+ strcat(f1, formlf[nrfp-1]);
+
+ /* Check number of parameters given */
+ if ((nn = sscanf(line, f1, &c[0], &c[1], &c[2], &c[3], &c[4], &c[5], &c[6], &c[7], &c[8], &c[9], &c[10], &c[11]))
+ != nrfp)
+ {
+ if (nn == nrfpA)
+ {
+ /* Copy the B-state from the A-state */
+ copy_B_from_A(ftype, c);
+ }
+ else
+ {
+ if (nn < nrfpA)
+ {
+ warning_error(wi, "Not enough parameters");
+ }
+ else if (nn > nrfpA && nn < nrfp)
+ {
+ warning_error(wi, "Too many parameters or not enough parameters for topology B");
+ }
+ else if (nn > nrfp)
+ {
+ warning_error(wi, "Too many parameters");
+ }
+ for (i = nn; (i < nrfp); i++)
+ {
+ c[i] = 0.0;
+ }
+ }
+ }
+
+ for (i = 0; (i < 4); i++)
+ {
+ if (!strcmp(alc[i], "X"))
+ {
+ p.a[i] = -1;
+ }
+ else
+ {
+ if ((p.a[i] = get_bond_atomtype_type(alc[i], bat)) == NOTSET)
+ {
+ gmx_fatal(FARGS, "Unknown bond_atomtype %s", alc[i]);
+ }
+ }
+ }
+ for (i = 0; (i < nrfp); i++)
+ {
+ p.c[i] = c[i];
+ }
+ /* Always use 4 atoms here, since we created two wildcard atoms
+ * if there wasn't of them 4 already.
+ */
+ push_bondtype (&(bt[ftype]), &p, 4, ftype, bAllowRepeat, line, wi);
+}
+
+
+void push_nbt(directive d, t_nbparam **nbt, gpp_atomtype_t atype,
+ char *pline, int nb_funct,
+ warninp_t wi)
+{
+ /* swap the atoms */
+ const char *form2 = "%*s%*s%*s%lf%lf";
+ const char *form3 = "%*s%*s%*s%lf%lf%lf";
+ const char *form4 = "%*s%*s%*s%lf%lf%lf%lf";
+ const char *form5 = "%*s%*s%*s%lf%lf%lf%lf%lf";
+ char a0[80], a1[80];
+ int i, f, n, ftype, atnr, nrfp;
+ double c[4], dum;
+ real cr[4], sig6;
+ atom_id ai, aj;
+ t_nbparam *nbp;
+ gmx_bool bId;
+ char errbuf[256];
+
+ if (sscanf (pline, "%s%s%d", a0, a1, &f) != 3)
+ {
+ too_few(wi);
+ return;
+ }
+
+ ftype = ifunc_index(d, f);
+
+ if (ftype != nb_funct)
+ {
+ sprintf(errbuf, "Trying to add %s while the default nonbond type is %s",
+ interaction_function[ftype].longname,
+ interaction_function[nb_funct].longname);
+ warning_error(wi, errbuf);
+ return;
+ }
+
+ /* Get the force parameters */
+ nrfp = NRFP(ftype);
+ if (ftype == F_LJ14)
+ {
+ n = sscanf(pline, form4, &c[0], &c[1], &c[2], &c[3]);
+ if (n < 2)
+ {
+ too_few(wi);
+ return;
+ }
+ /* When the B topology parameters are not set,
+ * copy them from topology A
+ */
+ for (i = n; i < nrfp; i++)
+ {
+ c[i] = c[i-2];
+ }
+ }
+ else if (ftype == F_LJC14_Q)
+ {
+ n = sscanf(pline, form5, &c[0], &c[1], &c[2], &c[3], &dum);
+ if (n != 4)
+ {
+ incorrect_n_param(wi);
+ return;
+ }
+ }
+ else if (nrfp == 2)
+ {
+ if (sscanf(pline, form3, &c[0], &c[1], &dum) != 2)
+ {
+ incorrect_n_param(wi);
+ return;
+ }
+ }
+ else if (nrfp == 3)
+ {
+ if (sscanf(pline, form4, &c[0], &c[1], &c[2], &dum) != 3)
+ {
+ incorrect_n_param(wi);
+ return;
+ }
+ }
+ else
+ {
+ gmx_fatal(FARGS, "Number of force parameters for nonbonded interactions is %d"
+ " in file %s, line %d", nrfp, __FILE__, __LINE__);
+ }
+ for (i = 0; (i < nrfp); i++)
+ {
+ cr[i] = c[i];
+ }
+
+ /* Put the parameters in the matrix */
+ if ((ai = get_atomtype_type (a0, atype)) == NOTSET)
+ {
+ gmx_fatal(FARGS, "Atomtype %s not found", a0);
+ }
+ if ((aj = get_atomtype_type (a1, atype)) == NOTSET)
+ {
+ gmx_fatal(FARGS, "Atomtype %s not found", a1);
+ }
+ nbp = &(nbt[max(ai, aj)][min(ai, aj)]);
+
+ if (nbp->bSet)
+ {
+ bId = TRUE;
+ for (i = 0; i < nrfp; i++)
+ {
+ bId = bId && (nbp->c[i] == cr[i]);
+ }
+ if (!bId)
+ {
+ sprintf(errbuf, "Overriding non-bonded parameters,");
+ warning(wi, errbuf);
+ fprintf(stderr, " old:");
+ for (i = 0; i < nrfp; i++)
+ {
+ fprintf(stderr, " %g", nbp->c[i]);
+ }
+ fprintf(stderr, " new\n%s\n", pline);
+ }
+ }
+ nbp->bSet = TRUE;
+ for (i = 0; i < nrfp; i++)
+ {
+ nbp->c[i] = cr[i];
+ }
+}
+
+void
+push_gb_params (gpp_atomtype_t at, char *line,
+ warninp_t wi)
+{
+ int nfield;
+ int atype;
+ double radius, vol, surftens, gb_radius, S_hct;
+ char atypename[STRLEN];
+ char errbuf[STRLEN];
+
+ if ( (nfield = sscanf(line, "%s%lf%lf%lf%lf%lf", atypename, &radius, &vol, &surftens, &gb_radius, &S_hct)) != 6)
+ {
+ sprintf(errbuf, "Too few gb parameters for type %s\n", atypename);
+ warning(wi, errbuf);
+ }
+
+ /* Search for atomtype */
+ atype = get_atomtype_type(atypename, at);
+
+ if (atype == NOTSET)
+ {
+ printf("Couldn't find topology match for atomtype %s\n", atypename);
+ abort();
+ }
+
+ set_atomtype_gbparam(at, atype, radius, vol, surftens, gb_radius, S_hct);
+}
+
+void
+push_cmaptype(directive d, t_params bt[], int nral, gpp_atomtype_t at,
+ t_bond_atomtype bat, char *line,
+ warninp_t wi)
+{
+ const char *formal = "%s%s%s%s%s%s%s%s";
+
+ int i, j, ft, ftype, nn, nrfp, nrfpA, nrfpB;
+ int start;
+ int nxcmap, nycmap, ncmap, read_cmap, sl, nct;
+ char s[20], alc[MAXATOMLIST+2][20];
+ t_param p;
+ gmx_bool bAllowRepeat;
+ char errbuf[256];
+
+ /* Keep the compiler happy */
+ read_cmap = 0;
+ start = 0;
+
+ if ((nn = sscanf(line, formal, alc[0], alc[1], alc[2], alc[3], alc[4], alc[5], alc[6], alc[7])) != nral+3)
+ {
+ sprintf(errbuf, "Incorrect number of atomtypes for cmap (%d instead of 5)", nn-1);
+ warning_error(wi, errbuf);
+ return;
+ }
+
+ /* Compute an offset for each line where the cmap parameters start
+ * ie. where the atom types and grid spacing information ends
+ */
+ for (i = 0; i < nn; i++)
+ {
+ start += (int)strlen(alc[i]);
+ }
+
+ /* There are nn-1 spaces between the atom types and the grid spacing info in the cmap.itp file */
+ /* start is the position on the line where we start to read the actual cmap grid data from the itp file */
+ start = start + nn -1;
+
+ ft = strtol(alc[nral], NULL, 10);
+ nxcmap = strtol(alc[nral+1], NULL, 10);
+ nycmap = strtol(alc[nral+2], NULL, 10);
+
+ /* Check for equal grid spacing in x and y dims */
+ if (nxcmap != nycmap)
+ {
+ gmx_fatal(FARGS, "Not the same grid spacing in x and y for cmap grid: x=%d, y=%d", nxcmap, nycmap);
+ }
+
+ ncmap = nxcmap*nycmap;
+ ftype = ifunc_index(d, ft);
+ nrfpA = strtol(alc[6], NULL, 10)*strtol(alc[6], NULL, 10);
+ nrfpB = strtol(alc[7], NULL, 10)*strtol(alc[7], NULL, 10);
+ nrfp = nrfpA+nrfpB;
+
+ /* Allocate memory for the CMAP grid */
+ bt->ncmap += nrfp;
+ srenew(bt->cmap, bt->ncmap);
+
+ /* Read in CMAP parameters */
+ sl = 0;
+ for (i = 0; i < ncmap; i++)
+ {
+ while (isspace(*(line+start+sl)))
+ {
+ sl++;
+ }
+ nn = sscanf(line+start+sl, " %s ", s);
+ sl += strlen(s);
+ bt->cmap[i+(bt->ncmap)-nrfp] = strtod(s, NULL);
+
+ if (nn == 1)
+ {
+ read_cmap++;
+ }
+ else
+ {
+ gmx_fatal(FARGS, "Error in reading cmap parameter for angle %s %s %s %s %s", alc[0], alc[1], alc[2], alc[3], alc[4]);
+ }
+
+ }
+
+ /* Check do that we got the number of parameters we expected */
+ if (read_cmap == nrfpA)
+ {
+ for (i = 0; i < ncmap; i++)
+ {
+ bt->cmap[i+ncmap] = bt->cmap[i];
+ }
+ }
+ else
+ {
+ if (read_cmap < nrfpA)
+ {
+ warning_error(wi, "Not enough cmap parameters");
+ }
+ else if (read_cmap > nrfpA && read_cmap < nrfp)
+ {
+ warning_error(wi, "Too many cmap parameters or not enough parameters for topology B");
+ }
+ else if (read_cmap > nrfp)
+ {
+ warning_error(wi, "Too many cmap parameters");
+ }
+ }
+
+
+ /* Set grid spacing and the number of grids (we assume these numbers to be the same for all grids
+ * so we can safely assign them each time
+ */
+ bt->grid_spacing = nxcmap; /* Or nycmap, they need to be equal */
+ bt->nc = bt->nc + 1; /* Since we are incrementing here, we need to subtract later, see (*****) */
+ nct = (nral+1) * bt->nc;
+
+ /* Allocate memory for the cmap_types information */
+ srenew(bt->cmap_types, nct);
+
+ for (i = 0; (i < nral); i++)
+ {
+ if (at && ((p.a[i] = get_bond_atomtype_type(alc[i], bat)) == NOTSET))
+ {
+ gmx_fatal(FARGS, "Unknown atomtype %s\n", alc[i]);
+ }
+ else if (bat && ((p.a[i] = get_bond_atomtype_type(alc[i], bat)) == NOTSET))
+ {
+ gmx_fatal(FARGS, "Unknown bond_atomtype %s\n", alc[i]);
+ }
+
+ /* Assign a grid number to each cmap_type */
+ bt->cmap_types[bt->nct++] = get_bond_atomtype_type(alc[i], bat);
+ }
+
+ /* Assign a type number to this cmap */
+ bt->cmap_types[bt->nct++] = bt->nc-1; /* Since we inremented earlier, we need to subtrac here, to get the types right (****) */
+
+ /* Check for the correct number of atoms (again) */
+ if (bt->nct != nct)
+ {
+ gmx_fatal(FARGS, "Incorrect number of atom types (%d) in cmap type %d\n", nct, bt->nc);
+ }
+
+ /* Is this correct?? */
+ for (i = 0; i < MAXFORCEPARAM; i++)
+ {
+ p.c[i] = NOTSET;
+ }
+
+ /* Push the bond to the bondlist */
+ push_bondtype (&(bt[ftype]), &p, nral, ftype, FALSE, line, wi);
+}
+
+
+static void push_atom_now(t_symtab *symtab, t_atoms *at, int atomnr,
+ int atomicnumber,
+ int type, char *ctype, int ptype,
+ char *resnumberic,
+ char *resname, char *name, real m0, real q0,
+ int typeB, char *ctypeB, real mB, real qB)
+{
+ int j, resind = 0, resnr;
+ unsigned char ric;
+ int nr = at->nr;
+
+ if (((nr == 0) && (atomnr != 1)) || (nr && (atomnr != at->nr+1)))
+ {
+ gmx_fatal(FARGS, "Atoms in the .top are not numbered consecutively from 1 (rather, atomnr = %d, while at->nr = %d)", atomnr, at->nr);
+ }
+
+ j = strlen(resnumberic) - 1;
+ if (isdigit(resnumberic[j]))
+ {
+ ric = ' ';
+ }
+ else
+ {
+ ric = resnumberic[j];
+ if (j == 0 || !isdigit(resnumberic[j-1]))
+ {
+ gmx_fatal(FARGS, "Invalid residue number '%s' for atom %d",
+ resnumberic, atomnr);
+ }
+ }
+ resnr = strtol(resnumberic, NULL, 10);
+
+ if (nr > 0)
+ {
+ resind = at->atom[nr-1].resind;
+ }
+ if (nr == 0 || strcmp(resname, *at->resinfo[resind].name) != 0 ||
+ resnr != at->resinfo[resind].nr ||
+ ric != at->resinfo[resind].ic)
+ {
+ if (nr == 0)
+ {
+ resind = 0;
+ }
+ else
+ {
+ resind++;
+ }
+ at->nres = resind + 1;
+ srenew(at->resinfo, at->nres);
+ at->resinfo[resind].name = put_symtab(symtab, resname);
+ at->resinfo[resind].nr = resnr;
+ at->resinfo[resind].ic = ric;
+ }
+ else
+ {
+ resind = at->atom[at->nr-1].resind;
+ }
+
+ /* New atom instance
+ * get new space for arrays
+ */
+ srenew(at->atom, nr+1);
+ srenew(at->atomname, nr+1);
+ srenew(at->atomtype, nr+1);
+ srenew(at->atomtypeB, nr+1);
+
+ /* fill the list */
+ at->atom[nr].type = type;
+ at->atom[nr].ptype = ptype;
+ at->atom[nr].q = q0;
+ at->atom[nr].m = m0;
+ at->atom[nr].typeB = typeB;
+ at->atom[nr].qB = qB;
+ at->atom[nr].mB = mB;
+
+ at->atom[nr].resind = resind;
+ at->atom[nr].atomnumber = atomicnumber;
+ at->atomname[nr] = put_symtab(symtab, name);
+ at->atomtype[nr] = put_symtab(symtab, ctype);
+ at->atomtypeB[nr] = put_symtab(symtab, ctypeB);
+ at->nr++;
+}
+
+void push_cg(t_block *block, int *lastindex, int index, int a)
+{
+ if (debug)
+ {
+ fprintf (debug, "Index %d, Atom %d\n", index, a);
+ }
+
+ if (((block->nr) && (*lastindex != index)) || (!block->nr))
+ {
+ /* add a new block */
+ block->nr++;
+ srenew(block->index, block->nr+1);
+ }
+ block->index[block->nr] = a + 1;
+ *lastindex = index;
+}
+
+void push_atom(t_symtab *symtab, t_block *cgs,
+ t_atoms *at, gpp_atomtype_t atype, char *line, int *lastcg,
+ warninp_t wi)
+{
+ int nr, ptype;
+ int cgnumber, atomnr, type, typeB, nscan;
+ char id[STRLEN], ctype[STRLEN], ctypeB[STRLEN],
+ resnumberic[STRLEN], resname[STRLEN], name[STRLEN], check[STRLEN];
+ double m, q, mb, qb;
+ real m0, q0, mB, qB;
+
+ /* Make a shortcut for writing in this molecule */
+ nr = at->nr;
+
+ /* Fixed parameters */
+ if (sscanf(line, "%s%s%s%s%s%d",
+ id, ctype, resnumberic, resname, name, &cgnumber) != 6)
+ {
+ too_few(wi);
+ return;
+ }
+ sscanf(id, "%d", &atomnr);
+ if ((type = get_atomtype_type(ctype, atype)) == NOTSET)
+ {
+ gmx_fatal(FARGS, "Atomtype %s not found", ctype);
+ }
+ ptype = get_atomtype_ptype(type, atype);
+
+ /* Set default from type */
+ q0 = get_atomtype_qA(type, atype);
+ m0 = get_atomtype_massA(type, atype);
+ typeB = type;
+ qB = q0;
+ mB = m0;
+
+ /* Optional parameters */
+ nscan = sscanf(line, "%*s%*s%*s%*s%*s%*s%lf%lf%s%lf%lf%s",
+ &q, &m, ctypeB, &qb, &mb, check);
+
+ /* Nasty switch that falls thru all the way down! */
+ if (nscan > 0)
+ {
+ q0 = qB = q;
+ if (nscan > 1)
+ {
+ m0 = mB = m;
+ if (nscan > 2)
+ {
+ if ((typeB = get_atomtype_type(ctypeB, atype)) == NOTSET)
+ {
+ gmx_fatal(FARGS, "Atomtype %s not found", ctypeB);
+ }
+ qB = get_atomtype_qA(typeB, atype);
+ mB = get_atomtype_massA(typeB, atype);
+ if (nscan > 3)
+ {
+ qB = qb;
+ if (nscan > 4)
+ {
+ mB = mb;
+ if (nscan > 5)
+ {
+ warning_error(wi, "Too many parameters");
+ }
+ }
+ }
+ }
+ }
+ }
+ if (debug)
+ {
+ fprintf(debug, "mB=%g, qB=%g, typeB=%d\n", mB, qB, typeB);
+ }
+
+ push_cg(cgs, lastcg, cgnumber, nr);
+
+ push_atom_now(symtab, at, atomnr, get_atomtype_atomnumber(type, atype),
+ type, ctype, ptype, resnumberic,
+ resname, name, m0, q0, typeB,
+ typeB == type ? ctype : ctypeB, mB, qB);
+}
+
+void push_molt(t_symtab *symtab, int *nmol, t_molinfo **mol, char *line,
+ warninp_t wi)
+{
+ char type[STRLEN];
+ int nrexcl, i;
+ t_molinfo *newmol;
+
+ if ((sscanf(line, "%s%d", type, &nrexcl)) != 2)
+ {
+ warning_error(wi, "Expected a molecule type name and nrexcl");
+ }
+
+ /* Test if this atomtype overwrites another */
+ i = 0;
+ while (i < *nmol)
+ {
+ if (gmx_strcasecmp(*((*mol)[i].name), type) == 0)
+ {
+ gmx_fatal(FARGS, "moleculetype %s is redefined", type);
+ }
+ i++;
+ }
+
+ (*nmol)++;
+ srenew(*mol, *nmol);
+ newmol = &((*mol)[*nmol-1]);
+ init_molinfo(newmol);
+
+ /* Fill in the values */
+ newmol->name = put_symtab(symtab, type);
+ newmol->nrexcl = nrexcl;
+ newmol->excl_set = FALSE;
+}
+
+static gmx_bool default_nb_params(int ftype, t_params bt[], t_atoms *at,
+ t_param *p, int c_start, gmx_bool bB, gmx_bool bGenPairs)
+{
+ int i, j, ti, tj, ntype;
+ gmx_bool bFound;
+ t_param *pi = NULL;
+ int nr = bt[ftype].nr;
+ int nral = NRAL(ftype);
+ int nrfp = interaction_function[ftype].nrfpA;
+ int nrfpB = interaction_function[ftype].nrfpB;
+
+ if ((!bB && nrfp == 0) || (bB && nrfpB == 0))
+ {
+ return TRUE;
+ }
+
+ bFound = FALSE;
+ if (bGenPairs)
+ {
+ /* First test the generated-pair position to save
+ * time when we have 1000*1000 entries for e.g. OPLS...
+ */
+ ntype = sqrt(nr);
+ if (bB)
+ {
+ ti = at->atom[p->a[0]].typeB;
+ tj = at->atom[p->a[1]].typeB;
+ }
+ else
+ {
+ ti = at->atom[p->a[0]].type;
+ tj = at->atom[p->a[1]].type;
+ }
+ pi = &(bt[ftype].param[ntype*ti+tj]);
+ bFound = ((ti == pi->a[0]) && (tj == pi->a[1]));
+ }
+
+ /* Search explicitly if we didnt find it */
+ if (!bFound)
+ {
+ for (i = 0; ((i < nr) && !bFound); i++)
+ {
+ pi = &(bt[ftype].param[i]);
+ if (bB)
+ {
+ for (j = 0; ((j < nral) &&
+ (at->atom[p->a[j]].typeB == pi->a[j])); j++)
+ {
+ ;
+ }
+ }
+ else
+ {
+ for (j = 0; ((j < nral) &&
+ (at->atom[p->a[j]].type == pi->a[j])); j++)
+ {
+ ;
+ }
+ }
+ bFound = (j == nral);
+ }
+ }
+
+ if (bFound)
+ {
+ if (bB)
+ {
+ if (nrfp+nrfpB > MAXFORCEPARAM)
+ {
+ gmx_incons("Too many force parameters");
+ }
+ for (j = c_start; (j < nrfpB); j++)
+ {
+ p->c[nrfp+j] = pi->c[j];
+ }
+ }
+ else
+ {
+ for (j = c_start; (j < nrfp); j++)
+ {
+ p->c[j] = pi->c[j];
+ }
+ }
+ }
+ else
+ {
+ for (j = c_start; (j < nrfp); j++)
+ {
+ p->c[j] = 0.0;
+ }
+ }
+ return bFound;
+}
+
+static gmx_bool default_cmap_params(t_params bondtype[],
+ t_atoms *at, gpp_atomtype_t atype,
+ t_param *p, gmx_bool bB,
+ int *cmap_type, int *nparam_def)
+{
+ int i, j, nparam_found;
+ int ct;
+ gmx_bool bFound = FALSE;
+
+ nparam_found = 0;
+ ct = 0;
+
+ /* Match the current cmap angle against the list of cmap_types */
+ for (i = 0; i < bondtype->nct && !bFound; i += 6)
+ {
+ if (bB)
+ {
+
+ }
+ else
+ {
+ if (
+ (get_atomtype_batype(at->atom[p->a[0]].type, atype) == bondtype->cmap_types[i]) &&
+ (get_atomtype_batype(at->atom[p->a[1]].type, atype) == bondtype->cmap_types[i+1]) &&
+ (get_atomtype_batype(at->atom[p->a[2]].type, atype) == bondtype->cmap_types[i+2]) &&
+ (get_atomtype_batype(at->atom[p->a[3]].type, atype) == bondtype->cmap_types[i+3]) &&
+ (get_atomtype_batype(at->atom[p->a[4]].type, atype) == bondtype->cmap_types[i+4]))
+ {
+ /* Found cmap torsion */
+ bFound = TRUE;
+ ct = bondtype->cmap_types[i+5];
+ nparam_found = 1;
+ }
+ }
+ }
+
+ /* If we did not find a matching type for this cmap torsion */
+ if (!bFound)
+ {
+ gmx_fatal(FARGS, "Unknown cmap torsion between atoms %d %d %d %d %d\n",
+ p->a[0]+1, p->a[1]+1, p->a[2]+1, p->a[3]+1, p->a[4]+1);
+ }
+
+ *nparam_def = nparam_found;
+ *cmap_type = ct;
+
+ return bFound;
+}
+
+static gmx_bool default_params(int ftype, t_params bt[],
+ t_atoms *at, gpp_atomtype_t atype,
+ t_param *p, gmx_bool bB,
+ t_param **param_def,
+ int *nparam_def)
+{
+ int i, j, nparam_found;
+ gmx_bool bFound, bSame;
+ t_param *pi = NULL;
+ t_param *pj = NULL;
+ int nr = bt[ftype].nr;
+ int nral = NRAL(ftype);
+ int nrfpA = interaction_function[ftype].nrfpA;
+ int nrfpB = interaction_function[ftype].nrfpB;
+
+ if ((!bB && nrfpA == 0) || (bB && nrfpB == 0))
+ {
+ return TRUE;
+ }
+
+
+ /* We allow wildcards now. The first type (with or without wildcards) that
+ * fits is used, so you should probably put the wildcarded bondtypes
+ * at the end of each section.
+ */
+ bFound = FALSE;
+ nparam_found = 0;
+ /* OPLS uses 1000s of dihedraltypes, so in order to speed up the scanning we have a
+ * special case for this. Check for B state outside loop to speed it up.
+ */
+ if (ftype == F_PDIHS || ftype == F_RBDIHS || ftype == F_IDIHS || ftype == F_PIDIHS)
+ {
+ if (bB)
+ {
+ for (i = 0; ((i < nr) && !bFound); i++)
+ {
+ pi = &(bt[ftype].param[i]);
+ bFound =
+ (
+ ((pi->AI == -1) || (get_atomtype_batype(at->atom[p->AI].typeB, atype) == pi->AI)) &&
+ ((pi->AJ == -1) || (get_atomtype_batype(at->atom[p->AJ].typeB, atype) == pi->AJ)) &&
+ ((pi->AK == -1) || (get_atomtype_batype(at->atom[p->AK].typeB, atype) == pi->AK)) &&
+ ((pi->AL == -1) || (get_atomtype_batype(at->atom[p->AL].typeB, atype) == pi->AL))
+ );
+ }
+ }
+ else
+ {
+ /* State A */
+ for (i = 0; ((i < nr) && !bFound); i++)
+ {
+ pi = &(bt[ftype].param[i]);
+ bFound =
+ (
+ ((pi->AI == -1) || (get_atomtype_batype(at->atom[p->AI].type, atype) == pi->AI)) &&
+ ((pi->AJ == -1) || (get_atomtype_batype(at->atom[p->AJ].type, atype) == pi->AJ)) &&
+ ((pi->AK == -1) || (get_atomtype_batype(at->atom[p->AK].type, atype) == pi->AK)) &&
+ ((pi->AL == -1) || (get_atomtype_batype(at->atom[p->AL].type, atype) == pi->AL))
+ );
+ }
+ }
+ /* Find additional matches for this dihedral - necessary for ftype==9 which is used e.g. for charmm.
+ * The rules in that case is that additional matches HAVE to be on adjacent lines!
+ */
+ if (bFound == TRUE)
+ {
+ nparam_found++;
+ bSame = TRUE;
+ /* Continue from current i value */
+ for (j = i+1; j < nr && bSame; j += 2)
+ {
+ pj = &(bt[ftype].param[j]);
+ bSame = (pi->AI == pj->AI && pi->AJ == pj->AJ && pi->AK == pj->AK && pi->AL == pj->AL);
+ if (bSame)
+ {
+ nparam_found++;
+ }
+ /* nparam_found will be increased as long as the numbers match */
+ }
+ }
+ }
+ else /* Not a dihedral */
+ {
+ for (i = 0; ((i < nr) && !bFound); i++)
+ {
+ pi = &(bt[ftype].param[i]);
+ if (bB)
+ {
+ for (j = 0; ((j < nral) &&
+ (get_atomtype_batype(at->atom[p->a[j]].typeB, atype) == pi->a[j])); j++)
+ {
+ ;
+ }
+ }
+ else
+ {
+ for (j = 0; ((j < nral) &&
+ (get_atomtype_batype(at->atom[p->a[j]].type, atype) == pi->a[j])); j++)
+ {
+ ;
+ }
+ }
+ bFound = (j == nral);
+ }
+ if (bFound)
+ {
+ nparam_found = 1;
+ }
+ }
+
+ *param_def = pi;
+ *nparam_def = nparam_found;
+
+ return bFound;
+}
+
+
+
+void push_bond(directive d, t_params bondtype[], t_params bond[],
+ t_atoms *at, gpp_atomtype_t atype, char *line,
+ gmx_bool bBonded, gmx_bool bGenPairs, real fudgeQQ,
+ gmx_bool bZero, gmx_bool *bWarn_copy_A_B,
+ warninp_t wi)
+{
+ const char *aaformat[MAXATOMLIST] = {
+ "%d%d",
+ "%d%d%d",
+ "%d%d%d%d",
+ "%d%d%d%d%d",
+ "%d%d%d%d%d%d",
+ "%d%d%d%d%d%d%d"
+ };
+ const char *asformat[MAXATOMLIST] = {
+ "%*s%*s",
+ "%*s%*s%*s",
+ "%*s%*s%*s%*s",
+ "%*s%*s%*s%*s%*s",
+ "%*s%*s%*s%*s%*s%*s",
+ "%*s%*s%*s%*s%*s%*s%*s"
+ };
+ const char *ccformat = "%lf%lf%lf%lf%lf%lf%lf%lf%lf%lf%lf%lf%lf";
+ int nr, i, j, nral, nral_fmt, nread, ftype;
+ char format[STRLEN];
+ /* One force parameter more, so we can check if we read too many */
+ double cc[MAXFORCEPARAM+1];
+ int aa[MAXATOMLIST+1];
+ t_param param, paramB, *param_defA, *param_defB;
+ gmx_bool bFoundA = FALSE, bFoundB = FALSE, bDef, bPert, bSwapParity = FALSE;
+ int nparam_defA, nparam_defB;
+ char errbuf[256];
+
+ nparam_defA = nparam_defB = 0;
+
+ ftype = ifunc_index(d, 1);
+ nral = NRAL(ftype);
+ for (j = 0; j < MAXATOMLIST; j++)
+ {
+ aa[j] = NOTSET;
+ }
+ bDef = (NRFP(ftype) > 0);
+
+ if (ftype == F_SETTLE)
+ {
+ /* SETTLE acts on 3 atoms, but the topology format only specifies
+ * the first atom (for historical reasons).
+ */
+ nral_fmt = 1;
+ }
+ else
+ {
+ nral_fmt = nral;
+ }
+
+ nread = sscanf(line, aaformat[nral_fmt-1],
+ &aa[0], &aa[1], &aa[2], &aa[3], &aa[4], &aa[5]);
+
+ if (ftype == F_SETTLE)
+ {
+ aa[3] = aa[1];
+ aa[1] = aa[0] + 1;
+ aa[2] = aa[0] + 2;
+ }
+
+ if (nread < nral_fmt)
+ {
+ too_few(wi);
+ return;
+ }
+ else if (nread > nral_fmt)
+ {
+ /* this is a hack to allow for virtual sites with swapped parity */
+ bSwapParity = (aa[nral] < 0);
+ if (bSwapParity)
+ {
+ aa[nral] = -aa[nral];
+ }
+ ftype = ifunc_index(d, aa[nral]);
+ if (bSwapParity)
+ {
+ switch (ftype)
+ {
+ case F_VSITE3FAD:
+ case F_VSITE3OUT:
+ break;
+ default:
+ gmx_fatal(FARGS, "Negative function types only allowed for %s and %s",
+ interaction_function[F_VSITE3FAD].longname,
+ interaction_function[F_VSITE3OUT].longname);
+ }
+ }
+ }
+
+
+ /* Check for double atoms and atoms out of bounds */
+ for (i = 0; (i < nral); i++)
+ {
+ if (aa[i] < 1 || aa[i] > at->nr)
+ {
+ gmx_fatal(FARGS, "[ file %s, line %d ]:\n"
+ "Atom index (%d) in %s out of bounds (1-%d).\n"
+ "This probably means that you have inserted topology section \"%s\"\n"
+ "in a part belonging to a different molecule than you intended to.\n"
+ "In that case move the \"%s\" section to the right molecule.",
+ get_warning_file(wi), get_warning_line(wi),
+ aa[i], dir2str(d), at->nr, dir2str(d), dir2str(d));
+ }
+ for (j = i+1; (j < nral); j++)
+ {
+ if (aa[i] == aa[j])
+ {
+ sprintf(errbuf, "Duplicate atom index (%d) in %s", aa[i], dir2str(d));
+ warning(wi, errbuf);
+ }
+ }
+ }
+
+ /* default force parameters */
+ for (j = 0; (j < MAXATOMLIST); j++)
+ {
+ param.a[j] = aa[j]-1;
+ }
+ for (j = 0; (j < MAXFORCEPARAM); j++)
+ {
+ param.c[j] = 0.0;
+ }
+
+ /* Get force params for normal and free energy perturbation
+ * studies, as determined by types!
+ */
+
+ if (bBonded)
+ {
+ bFoundA = default_params(ftype, bondtype, at, atype, ¶m, FALSE, ¶m_defA, &nparam_defA);
+ if (bFoundA)
+ {
+ /* Copy the A-state and B-state default parameters */
+ for (j = 0; (j < NRFPA(ftype)+NRFPB(ftype)); j++)
+ {
+ param.c[j] = param_defA->c[j];
+ }
+ }
+ bFoundB = default_params(ftype, bondtype, at, atype, ¶m, TRUE, ¶m_defB, &nparam_defB);
+ if (bFoundB)
+ {
+ /* Copy only the B-state default parameters */
+ for (j = NRFPA(ftype); (j < NRFP(ftype)); j++)
+ {
+ param.c[j] = param_defB->c[j];
+ }
+ }
+ }
+ else if (ftype == F_LJ14)
+ {
+ bFoundA = default_nb_params(ftype, bondtype, at, ¶m, 0, FALSE, bGenPairs);
+ bFoundB = default_nb_params(ftype, bondtype, at, ¶m, 0, TRUE, bGenPairs);
+ }
+ else if (ftype == F_LJC14_Q)
+ {
+ param.c[0] = fudgeQQ;
+ /* Fill in the A-state charges as default parameters */
+ param.c[1] = at->atom[param.a[0]].q;
+ param.c[2] = at->atom[param.a[1]].q;
+ /* The default LJ parameters are the standard 1-4 parameters */
+ bFoundA = default_nb_params(F_LJ14, bondtype, at, ¶m, 3, FALSE, bGenPairs);
+ bFoundB = TRUE;
+ }
+ else if (ftype == F_LJC_PAIRS_NB)
+ {
+ /* Defaults are not supported here */
+ bFoundA = FALSE;
+ bFoundB = TRUE;
+ }
+ else
+ {
+ gmx_incons("Unknown function type in push_bond");
+ }
+
+ if (nread > nral_fmt)
+ {
+ /* Manually specified parameters - in this case we discard multiple torsion info! */
+
+ strcpy(format, asformat[nral_fmt-1]);
+ strcat(format, ccformat);
+
+ nread = sscanf(line, format, &cc[0], &cc[1], &cc[2], &cc[3], &cc[4], &cc[5],
+ &cc[6], &cc[7], &cc[8], &cc[9], &cc[10], &cc[11], &cc[12]);
+
+ if ((nread == NRFPA(ftype)) && (NRFPB(ftype) != 0))
+ {
+ /* We only have to issue a warning if these atoms are perturbed! */
+ bPert = FALSE;
+ for (j = 0; (j < nral); j++)
+ {
+ bPert = bPert || PERTURBED(at->atom[param.a[j]]);
+ }
+
+ if (bPert && *bWarn_copy_A_B)
+ {
+ sprintf(errbuf,
+ "Some parameters for bonded interaction involving perturbed atoms are specified explicitly in state A, but not B - copying A to B");
+ warning(wi, errbuf);
+ *bWarn_copy_A_B = FALSE;
+ }
+
+ /* If only the A parameters were specified, copy them to the B state */
+ /* The B-state parameters correspond to the first nrfpB
+ * A-state parameters.
+ */
+ for (j = 0; (j < NRFPB(ftype)); j++)
+ {
+ cc[nread++] = cc[j];
+ }
+ }
+
+ /* If nread was 0 or EOF, no parameters were read => use defaults.
+ * If nread was nrfpA we copied above so nread=nrfp.
+ * If nread was nrfp we are cool.
+ * For F_LJC14_Q we allow supplying fudgeQQ only.
+ * Anything else is an error!
+ */
+ if ((nread != 0) && (nread != EOF) && (nread != NRFP(ftype)) &&
+ !(ftype == F_LJC14_Q && nread == 1))
+ {
+ gmx_fatal(FARGS, "Incorrect number of parameters - found %d, expected %d or %d for %s.",
+ nread, NRFPA(ftype), NRFP(ftype),
+ interaction_function[ftype].longname);
+ }
+
+ for (j = 0; (j < nread); j++)
+ {
+ param.c[j] = cc[j];
+ }
+
+ /* Check whether we have to use the defaults */
+ if (nread == NRFP(ftype))
+ {
+ bDef = FALSE;
+ }
+ }
+ else
+ {
+ nread = 0;
+ }
+ /* nread now holds the number of force parameters read! */
+
+ if (bDef)
+ {
+ /* Use defaults */
+ /* When we have multiple terms it would be very dangerous to allow perturbations to a different atom type! */
+ if (ftype == F_PDIHS)
+ {
+ if ((nparam_defA != nparam_defB) || ((nparam_defA > 1 || nparam_defB > 1) && (param_defA != param_defB)))
+ {
+ sprintf(errbuf,
+ "Cannot automatically perturb a torsion with multiple terms to different form.\n"
+ "Please specify perturbed parameters manually for this torsion in your topology!");
+ warning_error(wi, errbuf);
+ }
+ }
+
+ if (nread > 0 && nread < NRFPA(ftype))
+ {
+ /* Issue an error, do not use defaults */
+ sprintf(errbuf, "Not enough parameters, there should be at least %d (or 0 for defaults)", NRFPA(ftype));
+ warning_error(wi, errbuf);
+ }
+
+ if (nread == 0 || nread == EOF)
+ {
+ if (!bFoundA)
+ {
+ if (interaction_function[ftype].flags & IF_VSITE)
+ {
+ /* set them to NOTSET, will be calculated later */
+ for (j = 0; (j < MAXFORCEPARAM); j++)
+ {
+ param.c[j] = NOTSET;
+ }
+
+ if (bSwapParity)
+ {
+ param.C1 = -1; /* flag to swap parity of vsite construction */
+ }
+ }
+ else
+ {
+ if (bZero)
+ {
+ fprintf(stderr, "NOTE: No default %s types, using zeroes\n",
+ interaction_function[ftype].longname);
+ }
+ else
+ {
+ sprintf(errbuf, "No default %s types", interaction_function[ftype].longname);
+ warning_error(wi, errbuf);
+ }
+ }
+ }
+ else
+ {
+ if (bSwapParity)
+ {
+ switch (ftype)
+ {
+ case F_VSITE3FAD:
+ param.C0 = 360 - param.C0;
+ break;
+ case F_VSITE3OUT:
+ param.C2 = -param.C2;
+ break;
+ }
+ }
+ }
+ if (!bFoundB)
+ {
+ /* We only have to issue a warning if these atoms are perturbed! */
+ bPert = FALSE;
+ for (j = 0; (j < nral); j++)
+ {
+ bPert = bPert || PERTURBED(at->atom[param.a[j]]);
+ }
+
+ if (bPert)
+ {
+ sprintf(errbuf, "No default %s types for perturbed atoms, "
+ "using normal values", interaction_function[ftype].longname);
+ warning(wi, errbuf);
+ }
+ }
+ }
+ }
+
+ if ((ftype == F_PDIHS || ftype == F_ANGRES || ftype == F_ANGRESZ)
+ && param.c[5] != param.c[2])
+ {
+ gmx_fatal(FARGS, "[ file %s, line %d ]:\n"
+ " %s multiplicity can not be perturbed %f!=%f",
+ get_warning_file(wi), get_warning_line(wi),
+ interaction_function[ftype].longname,
+ param.c[2], param.c[5]);
+ }
+
+ if (IS_TABULATED(ftype) && param.c[2] != param.c[0])
+ {
+ gmx_fatal(FARGS, "[ file %s, line %d ]:\n"
+ " %s table number can not be perturbed %d!=%d",
+ get_warning_file(wi), get_warning_line(wi),
+ interaction_function[ftype].longname,
+ (int)(param.c[0]+0.5), (int)(param.c[2]+0.5));
+ }
+
+ /* Dont add R-B dihedrals where all parameters are zero (no interaction) */
+ if (ftype == F_RBDIHS)
+ {
+ nr = 0;
+ for (i = 0; i < NRFP(ftype); i++)
+ {
+ if (param.c[i] != 0)
+ {
+ nr++;
+ }
+ }
+ if (nr == 0)
+ {
+ return;
+ }
+ }
+
+ /* Put the values in the appropriate arrays */
+ add_param_to_list (&bond[ftype], ¶m);
+
+ /* Push additional torsions from FF for ftype==9 if we have them.
+ * We have already checked that the A/B states do not differ in this case,
+ * so we do not have to double-check that again, or the vsite stuff.
+ * In addition, those torsions cannot be automatically perturbed.
+ */
+ if (bDef && ftype == F_PDIHS)
+ {
+ for (i = 1; i < nparam_defA; i++)
+ {
+ /* Advance pointer! */
+ param_defA += 2;
+ for (j = 0; (j < NRFPA(ftype)+NRFPB(ftype)); j++)
+ {
+ param.c[j] = param_defA->c[j];
+ }
+ /* And push the next term for this torsion */
+ add_param_to_list (&bond[ftype], ¶m);
+ }
+ }
+}
+
+void push_cmap(directive d, t_params bondtype[], t_params bond[],
+ t_atoms *at, gpp_atomtype_t atype, char *line,
+ warninp_t wi)
+{
+ const char *aaformat[MAXATOMLIST+1] =
+ {
+ "%d",
+ "%d%d",
+ "%d%d%d",
+ "%d%d%d%d",
+ "%d%d%d%d%d",
+ "%d%d%d%d%d%d",
+ "%d%d%d%d%d%d%d"
+ };
+
+ int i, j, nr, ftype, nral, nread, ncmap_params;
+ int cmap_type;
+ int aa[MAXATOMLIST+1];
+ char errbuf[256];
+ gmx_bool bFound;
+ t_param param, paramB, *param_defA, *param_defB;
+
+ ftype = ifunc_index(d, 1);
+ nral = NRAL(ftype);
+ nr = bondtype[ftype].nr;
+ ncmap_params = 0;
+
+ nread = sscanf(line, aaformat[nral-1],
+ &aa[0], &aa[1], &aa[2], &aa[3], &aa[4], &aa[5]);
+
+ if (nread < nral)
+ {
+ too_few(wi);
+ return;
+ }
+ else if (nread == nral)
+ {
+ ftype = ifunc_index(d, 1);
+ }
+
+ /* Check for double atoms and atoms out of bounds */
+ for (i = 0; i < nral; i++)
+ {
+ if (aa[i] < 1 || aa[i] > at->nr)
+ {
+ gmx_fatal(FARGS, "[ file %s, line %d ]:\n"
+ "Atom index (%d) in %s out of bounds (1-%d).\n"
+ "This probably means that you have inserted topology section \"%s\"\n"
+ "in a part belonging to a different molecule than you intended to.\n"
+ "In that case move the \"%s\" section to the right molecule.",
+ get_warning_file(wi), get_warning_line(wi),
+ aa[i], dir2str(d), at->nr, dir2str(d), dir2str(d));
+ }
+
+ for (j = i+1; (j < nral); j++)
+ {
+ if (aa[i] == aa[j])
+ {
+ sprintf(errbuf, "Duplicate atom index (%d) in %s", aa[i], dir2str(d));
+ warning(wi, errbuf);
+ }
+ }
+ }
+
+ /* default force parameters */
+ for (j = 0; (j < MAXATOMLIST); j++)
+ {
+ param.a[j] = aa[j]-1;
+ }
+ for (j = 0; (j < MAXFORCEPARAM); j++)
+ {
+ param.c[j] = 0.0;
+ }
+
+ /* Get the cmap type for this cmap angle */
+ bFound = default_cmap_params(bondtype, at, atype, ¶m, FALSE, &cmap_type, &ncmap_params);
+
+ /* We want exactly one parameter (the cmap type in state A (currently no state B) back */
+ if (bFound && ncmap_params == 1)
+ {
+ /* Put the values in the appropriate arrays */
+ param.c[0] = cmap_type;
+ add_param_to_list(&bond[ftype], ¶m);
+ }
+ else
+ {
+ /* This is essentially the same check as in default_cmap_params() done one more time */
+ gmx_fatal(FARGS, "Unable to assign a cmap type to torsion %d %d %d %d and %d\n",
+ param.a[0]+1, param.a[1]+1, param.a[2]+1, param.a[3]+1, param.a[4]+1);
+ }
+}
+
+
+
+void push_vsitesn(directive d, t_params bond[],
+ t_atoms *at, char *line,
+ warninp_t wi)
+{
+ char *ptr;
+ int type, ftype, j, n, ret, nj, a;
+ int *atc = NULL;
+ double *weight = NULL, weight_tot;
+ t_param param;
+
+ /* default force parameters */
+ for (j = 0; (j < MAXATOMLIST); j++)
+ {
+ param.a[j] = NOTSET;
+ }
+ for (j = 0; (j < MAXFORCEPARAM); j++)
+ {
+ param.c[j] = 0.0;
+ }
+
+ ptr = line;
+ ret = sscanf(ptr, "%d%n", &a, &n);
+ ptr += n;
+ if (ret == 0)
+ {
+ gmx_fatal(FARGS, "[ file %s, line %d ]:\n"
+ " Expected an atom index in section \"%s\"",
+ get_warning_file(wi), get_warning_line(wi),
+ dir2str(d));
+ }
+
+ param.a[0] = a - 1;
+
+ ret = sscanf(ptr, "%d%n", &type, &n);
+ ptr += n;
+ ftype = ifunc_index(d, type);
+
+ weight_tot = 0;
+ nj = 0;
+ do
+ {
+ ret = sscanf(ptr, "%d%n", &a, &n);
+ ptr += n;
+ if (ret > 0)
+ {
+ if (nj % 20 == 0)
+ {
+ srenew(atc, nj+20);
+ srenew(weight, nj+20);
+ }
+ atc[nj] = a - 1;
+ switch (type)
+ {
+ case 1:
+ weight[nj] = 1;
+ break;
+ case 2:
+ /* Here we use the A-state mass as a parameter.
+ * Note that the B-state mass has no influence.
+ */
+ weight[nj] = at->atom[atc[nj]].m;
+ break;
+ case 3:
+ weight[nj] = -1;
+ ret = sscanf(ptr, "%lf%n", &(weight[nj]), &n);
+ ptr += n;
+ if (weight[nj] < 0)
+ {
+ gmx_fatal(FARGS, "[ file %s, line %d ]:\n"
+ " No weight or negative weight found for vsiten constructing atom %d (atom index %d)",
+ get_warning_file(wi), get_warning_line(wi),
+ nj+1, atc[nj]+1);
+ }
+ break;
+ default:
+ gmx_fatal(FARGS, "Unknown vsiten type %d", type);
+ }
+ weight_tot += weight[nj];
+ nj++;
+ }
+ }
+ while (ret > 0);
+
+ if (nj == 0)
+ {
+ gmx_fatal(FARGS, "[ file %s, line %d ]:\n"
+ " Expected more than one atom index in section \"%s\"",
+ get_warning_file(wi), get_warning_line(wi),
+ dir2str(d));
+ }
+
+ if (weight_tot == 0)
+ {
+ gmx_fatal(FARGS, "[ file %s, line %d ]:\n"
+ " The total mass of the construting atoms is zero",
+ get_warning_file(wi), get_warning_line(wi));
+ }
+
+ for (j = 0; j < nj; j++)
+ {
+ param.a[1] = atc[j];
+ param.c[0] = nj;
+ param.c[1] = weight[j]/weight_tot;
+ /* Put the values in the appropriate arrays */
+ add_param_to_list (&bond[ftype], ¶m);
+ }
+
+ sfree(atc);
+ sfree(weight);
+}
+
+void push_mol(int nrmols, t_molinfo mols[], char *pline, int *whichmol,
+ int *nrcopies,
+ warninp_t wi)
+{
+ int i, copies;
+ char type[STRLEN];
+
+ *nrcopies = 0;
+ if (sscanf(pline, "%s%d", type, &copies) != 2)
+ {
+ too_few(wi);
+ return;
+ }
+
+ /* search moleculename */
+ for (i = 0; ((i < nrmols) && gmx_strcasecmp(type, *(mols[i].name))); i++)
+ {
+ ;
+ }
+
+ if (i < nrmols)
+ {
+ *nrcopies = copies;
+ *whichmol = i;
+ }
+ else
+ {
+ gmx_fatal(FARGS, "No such moleculetype %s", type);
+ }
+}
+
+void init_block2(t_block2 *b2, int natom)
+{
+ int i;
+
+ b2->nr = natom;
+ snew(b2->nra, b2->nr);
+ snew(b2->a, b2->nr);
+ for (i = 0; (i < b2->nr); i++)
+ {
+ b2->a[i] = NULL;
+ }
+}
+
+void done_block2(t_block2 *b2)
+{
+ int i;
+
+ if (b2->nr)
+ {
+ for (i = 0; (i < b2->nr); i++)
+ {
+ sfree(b2->a[i]);
+ }
+ sfree(b2->a);
+ sfree(b2->nra);
+ b2->nr = 0;
+ }
+}
+
+void push_excl(char *line, t_block2 *b2)
+{
+ int i, j;
+ int n;
+ char base[STRLEN], format[STRLEN];
+
+ if (sscanf(line, "%d", &i) == 0)
+ {
+ return;
+ }
+
+ if ((1 <= i) && (i <= b2->nr))
+ {
+ i--;
+ }
+ else
+ {
+ if (debug)
+ {
+ fprintf(debug, "Unbound atom %d\n", i-1);
+ }
+ return;
+ }
+ strcpy(base, "%*d");
+ do
+ {
+ strcpy(format, base);
+ strcat(format, "%d");
+ n = sscanf(line, format, &j);
+ if (n == 1)
+ {
+ if ((1 <= j) && (j <= b2->nr))
+ {
+ j--;
+ srenew(b2->a[i], ++(b2->nra[i]));
+ b2->a[i][b2->nra[i]-1] = j;
+ /* also add the reverse exclusion! */
+ srenew(b2->a[j], ++(b2->nra[j]));
+ b2->a[j][b2->nra[j]-1] = i;
+ strcat(base, "%*d");
+ }
+ else
+ {
+ gmx_fatal(FARGS, "Invalid Atomnr j: %d, b2->nr: %d\n", j, b2->nr);
+ }
+ }
+ }
+ while (n == 1);
+}
+
+void b_to_b2(t_blocka *b, t_block2 *b2)
+{
+ int i;
+ atom_id j, a;
+
+ for (i = 0; (i < b->nr); i++)
+ {
+ for (j = b->index[i]; (j < b->index[i+1]); j++)
+ {
+ a = b->a[j];
+ srenew(b2->a[i], ++b2->nra[i]);
+ b2->a[i][b2->nra[i]-1] = a;
+ }
+ }
+}
+
+void b2_to_b(t_block2 *b2, t_blocka *b)
+{
+ int i, nra;
+ atom_id j;
+
+ nra = 0;
+ for (i = 0; (i < b2->nr); i++)
+ {
+ b->index[i] = nra;
+ for (j = 0; (j < b2->nra[i]); j++)
+ {
+ b->a[nra+j] = b2->a[i][j];
+ }
+ nra += b2->nra[i];
+ }
+ /* terminate list */
+ b->index[i] = nra;
+}
+
+static int icomp(const void *v1, const void *v2)
+{
+ return (*((atom_id *) v1))-(*((atom_id *) v2));
+}
+
+void merge_excl(t_blocka *excl, t_block2 *b2)
+{
+ int i, k;
+ atom_id j;
+ int nra;
+
+ if (!b2->nr)
+ {
+ return;
+ }
+ else if (b2->nr != excl->nr)
+ {
+ gmx_fatal(FARGS, "DEATH HORROR: b2->nr = %d, while excl->nr = %d",
+ b2->nr, excl->nr);
+ }
+ else if (debug)
+ {
+ fprintf(debug, "Entering merge_excl\n");
+ }
+
+ /* First copy all entries from excl to b2 */
+ b_to_b2(excl, b2);
+
+ /* Count and sort the exclusions */
+ nra = 0;
+ for (i = 0; (i < b2->nr); i++)
+ {
+ if (b2->nra[i] > 0)
+ {
+ /* remove double entries */
+ qsort(b2->a[i], (size_t)b2->nra[i], (size_t)sizeof(b2->a[i][0]), icomp);
+ k = 1;
+ for (j = 1; (j < b2->nra[i]); j++)
+ {
+ if (b2->a[i][j] != b2->a[i][k-1])
+ {
+ b2->a[i][k] = b2->a[i][j];
+ k++;
+ }
+ }
+ b2->nra[i] = k;
+ nra += b2->nra[i];
+ }
+ }
+ excl->nra = nra;
+ srenew(excl->a, excl->nra);
+
+ b2_to_b(b2, excl);
+}
+
+int add_atomtype_decoupled(t_symtab *symtab, gpp_atomtype_t at,
+ t_nbparam ***nbparam, t_nbparam ***pair)
+{
+ t_atom atom;
+ t_param param;
+ int i, nr;
+
+ /* Define an atom type with all parameters set to zero (no interactions) */
+ atom.q = 0.0;
+ atom.m = 0.0;
+ /* Type for decoupled atoms could be anything,
+ * this should be changed automatically later when required.
+ */
+ atom.ptype = eptAtom;
+ for (i = 0; (i < MAXFORCEPARAM); i++)
+ {
+ param.c[i] = 0.0;
+ }
+
+ nr = add_atomtype(at, symtab, &atom, "decoupled", ¶m, -1, 0.0, 0.0, 0.0, 0, 0, 0);
+
+ /* Add space in the non-bonded parameters matrix */
+ realloc_nb_params(at, nbparam, pair);
+
+ return nr;
+}
+
+static void convert_pairs_to_pairsQ(t_params *plist,
+ real fudgeQQ, t_atoms *atoms)
+{
- /* Copy the pair list to the pairQ list */
- plist[F_LJC14_Q] = plist[F_LJ14];
- /* Empty the pair list */
- plist[F_LJ14].nr = 0;
- plist[F_LJ14].param = NULL;
- param = plist[F_LJC14_Q].param;
- for (i = 0; i < plist[F_LJC14_Q].nr; i++)
++ t_param *paramp1,*paramp2,*paramnew;
++ int i,j,p1nr,p2nr,p2newnr;
+
- v = param[i].c[0];
- w = param[i].c[1];
- param[i].c[0] = fudgeQQ;
- param[i].c[1] = atoms->atom[param[i].a[0]].q;
- param[i].c[2] = atoms->atom[param[i].a[1]].q;
- param[i].c[3] = v;
- param[i].c[4] = w;
++ /* Add the pair list to the pairQ list */
++ p1nr = plist[F_LJ14].nr;
++ p2nr = plist[F_LJC14_Q].nr;
++ p2newnr = p1nr + p2nr;
++ snew(paramnew,p2newnr);
++
++ paramp1 = plist[F_LJ14].param;
++ paramp2 = plist[F_LJC14_Q].param;
++
++ /* Fill in the new F_LJC14_Q array with the old one. NOTE:
++ it may be possible to just ADD the converted F_LJ14 array
++ to the old F_LJC14_Q array, but since we have to create
++ a new sized memory structure, better just to deep copy it all.
++ */
++
++ for (i = 0; i < p2nr; i++)
++ {
++ /* Copy over parameters */
++ for (j=0;j<5;j++) /* entries are 0-4 for F_LJC14_Q */
++ {
++ paramnew[i].c[j] = paramp2[i].c[j];
++ }
++
++ /* copy over atoms */
++ for (j=0;j<2;j++)
++ {
++ paramnew[i].a[j] = paramp2[i].a[j];
++ }
++ }
++
++ for (i = p2nr; i < p2newnr; i++)
+ {
++ j = i-p2nr;
++
++ /* Copy over parameters */
++ paramnew[i].c[0] = fudgeQQ;
++ paramnew[i].c[1] = atoms->atom[paramp1[j].a[0]].q;
++ paramnew[i].c[2] = atoms->atom[paramp1[j].a[1]].q;
++ paramnew[i].c[3] = paramp1[j].c[0];
++ paramnew[i].c[4] = paramp1[j].c[1];
++
++ /* copy over atoms */
++ paramnew[i].a[0] = paramp1[j].a[0];
++ paramnew[i].a[1] = paramp1[j].a[1];
+ }
++
++ /* free the old pairlists */
++ sfree(plist[F_LJC14_Q].param);
++ sfree(plist[F_LJ14].param);
++
++ /* now assign the new data to the F_LJC14_Q structure */
++ plist[F_LJC14_Q].param = paramnew;
++ plist[F_LJC14_Q].nr = p2newnr;
++
++ /* Empty the LJ14 pairlist */
++ plist[F_LJ14].nr = 0;
++ plist[F_LJ14].param = NULL;
+}
+
+static void generate_LJCpairsNB(t_molinfo *mol, int nb_funct, t_params *nbp)
+{
+ int n, ntype, i, j, k;
+ t_atom *atom;
+ t_blocka *excl;
+ gmx_bool bExcl;
+ t_param param;
+
+ n = mol->atoms.nr;
+ atom = mol->atoms.atom;
+
+ ntype = sqrt(nbp->nr);
+
+ for (i = 0; i < MAXATOMLIST; i++)
+ {
+ param.a[i] = NOTSET;
+ }
+ for (i = 0; i < MAXFORCEPARAM; i++)
+ {
+ param.c[i] = NOTSET;
+ }
+
+ /* Add a pair interaction for all non-excluded atom pairs */
+ excl = &mol->excls;
+ for (i = 0; i < n; i++)
+ {
+ for (j = i+1; j < n; j++)
+ {
+ bExcl = FALSE;
+ for (k = excl->index[i]; k < excl->index[i+1]; k++)
+ {
+ if (excl->a[k] == j)
+ {
+ bExcl = TRUE;
+ }
+ }
+ if (!bExcl)
+ {
+ if (nb_funct != F_LJ)
+ {
+ gmx_fatal(FARGS, "Can only generate non-bonded pair interactions for Van der Waals type Lennard-Jones");
+ }
+ param.a[0] = i;
+ param.a[1] = j;
+ param.c[0] = atom[i].q;
+ param.c[1] = atom[j].q;
+ param.c[2] = nbp->param[ntype*atom[i].type+atom[j].type].c[0];
+ param.c[3] = nbp->param[ntype*atom[i].type+atom[j].type].c[1];
+ add_param_to_list(&mol->plist[F_LJC_PAIRS_NB], ¶m);
+ }
+ }
+ }
+}
+
+static void set_excl_all(t_blocka *excl)
+{
+ int nat, i, j, k;
+
+ /* Get rid of the current exclusions and exclude all atom pairs */
+ nat = excl->nr;
+ excl->nra = nat*nat;
+ srenew(excl->a, excl->nra);
+ k = 0;
+ for (i = 0; i < nat; i++)
+ {
+ excl->index[i] = k;
+ for (j = 0; j < nat; j++)
+ {
+ excl->a[k++] = j;
+ }
+ }
+ excl->index[nat] = k;
+}
+
+static void decouple_atoms(t_atoms *atoms, int atomtype_decouple,
+ int couple_lam0, int couple_lam1)
+{
+ int i;
+
+ for (i = 0; i < atoms->nr; i++)
+ {
+ if (couple_lam0 == ecouplamNONE || couple_lam0 == ecouplamVDW)
+ {
+ atoms->atom[i].q = 0.0;
+ }
+ if (couple_lam0 == ecouplamNONE || couple_lam0 == ecouplamQ)
+ {
+ atoms->atom[i].type = atomtype_decouple;
+ }
+ if (couple_lam1 == ecouplamNONE || couple_lam1 == ecouplamVDW)
+ {
+ atoms->atom[i].qB = 0.0;
+ }
+ if (couple_lam1 == ecouplamNONE || couple_lam1 == ecouplamQ)
+ {
+ atoms->atom[i].typeB = atomtype_decouple;
+ }
+ }
+}
+
+void convert_moltype_couple(t_molinfo *mol, int atomtype_decouple, real fudgeQQ,
+ int couple_lam0, int couple_lam1,
+ gmx_bool bCoupleIntra, int nb_funct, t_params *nbp)
+{
+ convert_pairs_to_pairsQ(mol->plist, fudgeQQ, &mol->atoms);
+ if (!bCoupleIntra)
+ {
+ generate_LJCpairsNB(mol, nb_funct, nbp);
+ set_excl_all(&mol->excls);
+ }
+ decouple_atoms(&mol->atoms, atomtype_decouple, couple_lam0, couple_lam1);
+}
--- /dev/null
- * For the nbnxn 4xn kernels all widths (2, 4 and 8) are supported.
- * The nbnxn 2xnn kernels are currently not supported.
+/*
+ * This file is part of the GROMACS molecular simulation package.
+ *
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2012, The GROMACS Development Team
+ * Copyright (c) 2012, by the GROMACS development team, led by
+ * David van der Spoel, Berk Hess, Erik Lindahl, and including many
+ * others, as listed in the AUTHORS file in the top-level source
+ * directory and at http://www.gromacs.org.
+ *
+ * GROMACS is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ *
+ * GROMACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with GROMACS; if not, see
+ * http://www.gnu.org/licenses, or write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * If you want to redistribute modifications to GROMACS, please
+ * consider that scientific software is very special. Version
+ * control is crucial - bugs must be traceable. We will be happy to
+ * consider code for inclusion in the official distribution, but
+ * derived work must not be called official GROMACS. Details are found
+ * in the README & COPYING files - if they are missing, get the
+ * official version at http://www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the research papers on the package. Check out http://www.gromacs.org.
+ */
+
+/* The macros in this file are intended to be used for writing
+ * architecture-independent SIMD intrinsics code.
+ * To support a new architecture, adding macros here should be (nearly)
+ * all that is needed.
+ */
+
+#ifdef _gmx_simd_macros_h_
+#error "gmx_simd_macros.h included twice"
+#else
+#define _gmx_simd_macros_h_
+
+/* NOTE: SSE2 acceleration does not include floor or blendv */
+
+
+/* Uncomment the next line, without other SIMD active, for testing plain-C */
+/* #define GMX_SIMD_REFERENCE_PLAIN_C */
+#ifdef GMX_SIMD_REFERENCE_PLAIN_C
+/* Plain C SIMD reference implementation, also serves as documentation */
+#define GMX_HAVE_SIMD_MACROS
+
+/* In general the reference SIMD supports any SIMD width, including 1.
- /* Not required, gmx_anytrue_pb(x) returns if any of the boolean is x is True.
- * If this is not present, define GMX_SIMD_IS_TRUE(real x),
- * which should return x==True, where True is True as defined in SIMD.
- */
- #define GMX_SIMD_HAVE_ANYTRUE
- #ifdef GMX_SIMD_HAVE_ANYTRUE
++ * See types/nb_verlet.h for details
+ */
+#define GMX_SIMD_REF_WIDTH 4
+
+/* Include plain-C reference implementation, also serves as documentation */
+#include "gmx_simd_ref.h"
+
+#define GMX_SIMD_WIDTH_HERE GMX_SIMD_REF_WIDTH
+
+/* float/double SIMD register type */
+#define gmx_mm_pr gmx_simd_ref_pr
+
+/* boolean SIMD register type */
+#define gmx_mm_pb gmx_simd_ref_pb
+
+/* integer SIMD register type, only for table indexing and exclusion masks */
+#define gmx_epi32 gmx_simd_ref_epi32
+#define GMX_SIMD_EPI32_WIDTH GMX_SIMD_REF_EPI32_WIDTH
+
+/* Load GMX_SIMD_WIDTH_HERE reals for memory starting at r */
+#define gmx_load_pr gmx_simd_ref_load_pr
+/* Set all SIMD register elements to *r */
+#define gmx_load1_pr gmx_simd_ref_load1_pr
+#define gmx_set1_pr gmx_simd_ref_set1_pr
+#define gmx_setzero_pr gmx_simd_ref_setzero_pr
+#define gmx_store_pr gmx_simd_ref_store_pr
+
+#define gmx_add_pr gmx_simd_ref_add_pr
+#define gmx_sub_pr gmx_simd_ref_sub_pr
+#define gmx_mul_pr gmx_simd_ref_mul_pr
+/* For the FMA macros below, aim for c=d in code, so FMA3 uses 1 instruction */
+#define gmx_madd_pr gmx_simd_ref_madd_pr
+#define gmx_nmsub_pr gmx_simd_ref_nmsub_pr
+
+#define gmx_max_pr gmx_simd_ref_max_pr
+#define gmx_blendzero_pr gmx_simd_ref_blendzero_pr
+
+#define gmx_round_pr gmx_simd_ref_round_pr
+
+/* Not required, only used to speed up the nbnxn tabulated PME kernels */
+#define GMX_SIMD_HAVE_FLOOR
+#ifdef GMX_SIMD_HAVE_FLOOR
+#define gmx_floor_pr gmx_simd_ref_floor_pr
+#endif
+
+/* Not required, only used when blendv is faster than comparison */
+#define GMX_SIMD_HAVE_BLENDV
+#ifdef GMX_SIMD_HAVE_BLENDV
+#define gmx_blendv_pr gmx_simd_ref_blendv_pr
+#endif
+
+/* Copy the sign of a to b, assumes b >= 0 for efficiency */
+#define gmx_cpsgn_nonneg_pr gmx_simd_ref_cpsgn_nonneg_pr
+
+/* Very specific operation required in the non-bonded kernels */
+#define gmx_masknot_add_pr gmx_simd_ref_masknot_add_pr
+
+/* Comparison */
+#define gmx_cmplt_pr gmx_simd_ref_cmplt_pr
+
+/* Logical operations on SIMD booleans */
+#define gmx_and_pb gmx_simd_ref_and_pb
+#define gmx_or_pb gmx_simd_ref_or_pb
+
- #else
- /* If we don't have gmx_anytrue_pb, we need to store gmx_mm_pb */
- #define gmx_store_pb gmx_simd_ref_store_pb
- #endif
++/* Returns a single int (0/1) which tells if any of the 4 booleans is True */
+#define gmx_anytrue_pb gmx_simd_ref_anytrue_pb
- #else
+
+/* Conversions only used for PME table lookup */
+#define gmx_cvttpr_epi32 gmx_simd_ref_cvttpr_epi32
+#define gmx_cvtepi32_pr gmx_simd_ref_cvtepi32_pr
+
+/* These two function only need to be approximate, Newton-Raphson iteration
+ * is used for full accuracy in gmx_invsqrt_pr and gmx_inv_pr.
+ */
+#define gmx_rsqrt_pr gmx_simd_ref_rsqrt_pr
+#define gmx_rcp_pr gmx_simd_ref_rcp_pr
+
+/* sqrt+inv+sin+cos+acos+atan2 are used for bonded potentials, exp for PME */
+#define GMX_SIMD_HAVE_EXP
+#ifdef GMX_SIMD_HAVE_EXP
+#define gmx_exp_pr gmx_simd_ref_exp_pr
+#endif
+#define GMX_SIMD_HAVE_TRIGONOMETRIC
+#ifdef GMX_SIMD_HAVE_TRIGONOMETRIC
+#define gmx_sqrt_pr gmx_simd_ref_sqrt_pr
+#define gmx_sincos_pr gmx_simd_ref_sincos_pr
+#define gmx_acos_pr gmx_simd_ref_acos_pr
+#define gmx_atan2_pr gmx_simd_ref_atan2_pr
+#endif
+
+#endif /* GMX_SIMD_REFERENCE_PLAIN_C */
+
+
+/* The same SIMD macros can be translated to SIMD intrinsics (and compiled
+ * to instructions for) different SIMD width and float precision.
+ *
+ * On x86: The gmx_ prefix is replaced by _mm_ or _mm256_ (SSE or AVX).
+ * The _pr suffix is replaced by _ps or _pd (for single or double precision).
+ * Compiler settings will decide if 128-bit intrinsics will
+ * be translated into SSE or AVX instructions.
+ */
+
+
+#ifdef GMX_USE_HALF_WIDTH_SIMD_HERE
+#if defined GMX_X86_AVX_256
+/* We have half SIMD width support, continue */
+#else
+#error "half SIMD width intrinsics are not supported"
+#endif
+#endif
+
++#ifdef GMX_IS_X86
+
+#ifdef GMX_X86_SSE2
+/* This is for general x86 SIMD instruction sets that also support SSE2 */
+#define GMX_HAVE_SIMD_MACROS
+
+/* Include the highest supported x86 SIMD intrisics + math functions */
+#ifdef GMX_X86_AVX_256
+#include "gmx_x86_avx_256.h"
+#ifdef GMX_DOUBLE
+#include "gmx_math_x86_avx_256_double.h"
- #endif
- #else
++#else /* GMX_DOUBLE */
+#include "gmx_math_x86_avx_256_single.h"
- #else
++#endif /* GMX_DOUBLE */
++#else /* GMX_X86_AVX_256 */
+#ifdef GMX_X86_AVX_128_FMA
+#include "gmx_x86_avx_128_fma.h"
+#ifdef GMX_DOUBLE
+#include "gmx_math_x86_avx_128_fma_double.h"
- #endif
- #else
++#else /* GMX_DOUBLE */
+#include "gmx_math_x86_avx_128_fma_single.h"
- #else
++#endif /* GMX_DOUBLE */
++#else /* GMX_X86_AVX_128_FMA */
+#ifdef GMX_X86_SSE4_1
+#include "gmx_x86_sse4_1.h"
+#ifdef GMX_DOUBLE
+#include "gmx_math_x86_sse4_1_double.h"
- #endif
- #else
++#else /* GMX_DOUBLE */
+#include "gmx_math_x86_sse4_1_single.h"
- #else
++#endif /* GMX_DOUBLE */
++#else /* GMX_X86_SSE4_1 */
+#ifdef GMX_X86_SSE2
+#include "gmx_x86_sse2.h"
+#ifdef GMX_DOUBLE
+#include "gmx_math_x86_sse2_double.h"
- #endif
- #else
++#else /* GMX_DOUBLE */
+#include "gmx_math_x86_sse2_single.h"
- #endif
- #endif
- #endif
- #endif
++#endif /* GMX_DOUBLE */
++#else /* GMX_X86_SSE2 */
+#error No x86 acceleration defined
- static gmx_inline gmx_mm_pr gmx_masknot_add_pr(gmx_mm_pb a, gmx_mm_pr b, gmx_mm_pr c) { return _mm_add_ps(b, _mm_andnot_ps(a, c)); };
++#endif /* GMX_X86_SSE2 */
++#endif /* GMX_X86_SSE4_1 */
++#endif /* GMX_X86_AVX_128_FMA */
++#endif /* GMX_X86_AVX_256 */
++
+/* exp and trigonometric functions are included above */
+#define GMX_SIMD_HAVE_EXP
+#define GMX_SIMD_HAVE_TRIGONOMETRIC
+
+#if !defined GMX_X86_AVX_256 || defined GMX_USE_HALF_WIDTH_SIMD_HERE
+
+#ifndef GMX_DOUBLE
+
+#define GMX_SIMD_WIDTH_HERE 4
+
+#define gmx_mm_pr __m128
+
+#define gmx_mm_pb __m128
+
+#define gmx_epi32 __m128i
+#define GMX_SIMD_EPI32_WIDTH 4
+
+#define gmx_load_pr _mm_load_ps
+#define gmx_load1_pr _mm_load1_ps
+#define gmx_set1_pr _mm_set1_ps
+#define gmx_setzero_pr _mm_setzero_ps
+#define gmx_store_pr _mm_store_ps
+
+#define gmx_add_pr _mm_add_ps
+#define gmx_sub_pr _mm_sub_ps
+#define gmx_mul_pr _mm_mul_ps
+#ifdef GMX_X86_AVX_128_FMA
+#define GMX_SIMD_HAVE_FMA
+#define gmx_madd_pr(a, b, c) _mm_macc_ps(a, b, c)
+#define gmx_nmsub_pr(a, b, c) _mm_nmacc_ps(a, b, c)
+#else
+#define gmx_madd_pr(a, b, c) _mm_add_ps(c, _mm_mul_ps(a, b))
+#define gmx_nmsub_pr(a, b, c) _mm_sub_ps(c, _mm_mul_ps(a, b))
+#endif
+#define gmx_max_pr _mm_max_ps
+#define gmx_blendzero_pr _mm_and_ps
+
+#define gmx_cmplt_pr _mm_cmplt_ps
+#define gmx_and_pb _mm_and_ps
+#define gmx_or_pb _mm_or_ps
+
+#ifdef GMX_X86_SSE4_1
+#define gmx_round_pr(x) _mm_round_ps(x, 0x0)
+#define GMX_SIMD_HAVE_FLOOR
+#define gmx_floor_pr _mm_floor_ps
+#else
+#define gmx_round_pr(x) _mm_cvtepi32_ps(_mm_cvtps_epi32(x))
+#endif
+
+#ifdef GMX_X86_SSE4_1
+#define GMX_SIMD_HAVE_BLENDV
+#define gmx_blendv_pr _mm_blendv_ps
+#endif
+
+static gmx_inline gmx_mm_pr gmx_cpsgn_nonneg_pr(gmx_mm_pr a, gmx_mm_pr b)
+{
+ /* The value -0.0 has only the sign-bit set */
+ gmx_mm_pr sign_mask = _mm_set1_ps(-0.0);
+ return _mm_or_ps(_mm_and_ps(a, sign_mask), b);
+};
+
- #define GMX_SIMD_HAVE_ANYTRUE
++static gmx_inline gmx_mm_pr gmx_masknot_add_pr(gmx_mm_pb a, gmx_mm_pr b, gmx_mm_pr c)
++{
++ return _mm_add_ps(b, _mm_andnot_ps(a, c));
++};
+
- static gmx_inline gmx_mm_pr gmx_masknot_add_pr(gmx_mm_pb a, gmx_mm_pr b, gmx_mm_pr c) { return _mm_add_pd(b, _mm_andnot_pd(a, c)); };
+#define gmx_anytrue_pb _mm_movemask_ps
+
+#define gmx_cvttpr_epi32 _mm_cvttps_epi32
+#define gmx_cvtepi32_pr _mm_cvtepi32_ps
+
+#define gmx_rsqrt_pr _mm_rsqrt_ps
+#define gmx_rcp_pr _mm_rcp_ps
+
+#define gmx_exp_pr gmx_mm_exp_ps
+#define gmx_sqrt_pr gmx_mm_sqrt_ps
+#define gmx_sincos_pr gmx_mm_sincos_ps
+#define gmx_acos_pr gmx_mm_acos_ps
+#define gmx_atan2_pr gmx_mm_atan2_ps
+
+#else /* ifndef GMX_DOUBLE */
+
+#define GMX_SIMD_WIDTH_HERE 2
+
+#define gmx_mm_pr __m128d
+
+#define gmx_mm_pb __m128d
+
+#define gmx_epi32 __m128i
+#define GMX_SIMD_EPI32_WIDTH 4
+
+#define gmx_load_pr _mm_load_pd
+#define gmx_load1_pr _mm_load1_pd
+#define gmx_set1_pr _mm_set1_pd
+#define gmx_setzero_pr _mm_setzero_pd
+#define gmx_store_pr _mm_store_pd
+
+#define gmx_add_pr _mm_add_pd
+#define gmx_sub_pr _mm_sub_pd
+#define gmx_mul_pr _mm_mul_pd
+#ifdef GMX_X86_AVX_128_FMA
+#define GMX_SIMD_HAVE_FMA
+#define gmx_madd_pr(a, b, c) _mm_macc_pd(a, b, c)
+#define gmx_nmsub_pr(a, b, c) _mm_nmacc_pd(a, b, c)
+#else
+#define gmx_madd_pr(a, b, c) _mm_add_pd(c, _mm_mul_pd(a, b))
+#define gmx_nmsub_pr(a, b, c) _mm_sub_pd(c, _mm_mul_pd(a, b))
+#endif
+#define gmx_max_pr _mm_max_pd
+#define gmx_blendzero_pr _mm_and_pd
+
+#ifdef GMX_X86_SSE4_1
+#define gmx_round_pr(x) _mm_round_pd(x, 0x0)
+#define GMX_SIMD_HAVE_FLOOR
+#define gmx_floor_pr _mm_floor_pd
+#else
+#define gmx_round_pr(x) _mm_cvtepi32_pd(_mm_cvtpd_epi32(x))
+/* gmx_floor_pr is not used in code for pre-SSE4_1 hardware */
+#endif
+
+#ifdef GMX_X86_SSE4_1
+#define GMX_SIMD_HAVE_BLENDV
+#define gmx_blendv_pr _mm_blendv_pd
+#endif
+
+static gmx_inline gmx_mm_pr gmx_cpsgn_nonneg_pr(gmx_mm_pr a, gmx_mm_pr b)
+{
+ gmx_mm_pr sign_mask = _mm_set1_pd(-0.0);
+ return _mm_or_pd(_mm_and_pd(a, sign_mask), b);
+};
+
- #define GMX_SIMD_HAVE_ANYTRUE
++static gmx_inline gmx_mm_pr gmx_masknot_add_pr(gmx_mm_pb a, gmx_mm_pr b, gmx_mm_pr c)
++{
++ return _mm_add_pd(b, _mm_andnot_pd(a, c));
++};
+
+#define gmx_cmplt_pr _mm_cmplt_pd
+
+#define gmx_and_pb _mm_and_pd
+#define gmx_or_pb _mm_or_pd
+
- static gmx_inline gmx_mm_pr gmx_masknot_add_pr(gmx_mm_pb a, gmx_mm_pr b, gmx_mm_pr c) { return _mm256_add_ps(b, _mm256_andnot_ps(a, c)); };
+#define gmx_anytrue_pb _mm_movemask_pd
+
+#define gmx_cvttpr_epi32 _mm_cvttpd_epi32
+#define gmx_cvtepi32_pr _mm_cvtepi32_pd
+
+#define gmx_rsqrt_pr(r) _mm_cvtps_pd(_mm_rsqrt_ps(_mm_cvtpd_ps(r)))
+#define gmx_rcp_pr(r) _mm_cvtps_pd(_mm_rcp_ps(_mm_cvtpd_ps(r)))
+
+#define gmx_exp_pr gmx_mm_exp_pd
+#define gmx_sqrt_pr gmx_mm_sqrt_pd
+#define gmx_sincos_pr gmx_mm_sincos_pd
+#define gmx_acos_pr gmx_mm_acos_pd
+#define gmx_atan2_pr gmx_mm_atan2_pd
+
+#endif /* ifndef GMX_DOUBLE */
+
+#else
+/* We have GMX_X86_AVX_256 and not GMX_USE_HALF_WIDTH_SIMD_HERE,
+ * so we use 256-bit SIMD.
+ */
+
+#ifndef GMX_DOUBLE
+
+#define GMX_SIMD_WIDTH_HERE 8
+
+#define gmx_mm_pr __m256
+
+#define gmx_mm_pb __m256
+
+#define gmx_epi32 __m256i
+#define GMX_SIMD_EPI32_WIDTH 8
+
+#define gmx_load_pr _mm256_load_ps
+#define gmx_load1_pr(x) _mm256_set1_ps((x)[0])
+#define gmx_set1_pr _mm256_set1_ps
+#define gmx_setzero_pr _mm256_setzero_ps
+#define gmx_store_pr _mm256_store_ps
+
+#define gmx_add_pr _mm256_add_ps
+#define gmx_sub_pr _mm256_sub_ps
+#define gmx_mul_pr _mm256_mul_ps
+#define gmx_madd_pr(a, b, c) _mm256_add_ps(c, _mm256_mul_ps(a, b))
+#define gmx_nmsub_pr(a, b, c) _mm256_sub_ps(c, _mm256_mul_ps(a, b))
+#define gmx_max_pr _mm256_max_ps
+#define gmx_blendzero_pr _mm256_and_ps
+
+#define gmx_round_pr(x) _mm256_round_ps(x, 0x0)
+#define GMX_SIMD_HAVE_FLOOR
+#define gmx_floor_pr _mm256_floor_ps
+
+#define GMX_SIMD_HAVE_BLENDV
+#define gmx_blendv_pr _mm256_blendv_ps
+
+static gmx_inline gmx_mm_pr gmx_cpsgn_nonneg_pr(gmx_mm_pr a, gmx_mm_pr b)
+{
+ gmx_mm_pr sign_mask = _mm256_set1_ps(-0.0);
+ return _mm256_or_ps(_mm256_and_ps(a, sign_mask), b);
+};
+
- #define GMX_SIMD_HAVE_ANYTRUE
++static gmx_inline gmx_mm_pr gmx_masknot_add_pr(gmx_mm_pb a, gmx_mm_pr b, gmx_mm_pr c)
++{
++ return _mm256_add_ps(b, _mm256_andnot_ps(a, c));
++};
+
+/* Less-than (we use ordered, non-signaling, but that's not required) */
+#define gmx_cmplt_pr(x, y) _mm256_cmp_ps(x, y, 0x11)
+#define gmx_and_pb _mm256_and_ps
+#define gmx_or_pb _mm256_or_ps
+
- static gmx_inline gmx_mm_pr gmx_masknot_add_pr(gmx_mm_pb a, gmx_mm_pr b, gmx_mm_pr c) { return _mm256_add_pd(b, _mm256_andnot_pd(a, c)); };
+#define gmx_anytrue_pb _mm256_movemask_ps
+
+#define gmx_cvttpr_epi32 _mm256_cvttps_epi32
+
+#define gmx_rsqrt_pr _mm256_rsqrt_ps
+#define gmx_rcp_pr _mm256_rcp_ps
+
+#define gmx_exp_pr gmx_mm256_exp_ps
+#define gmx_sqrt_pr gmx_mm256_sqrt_ps
+#define gmx_sincos_pr gmx_mm256_sincos_ps
+#define gmx_acos_pr gmx_mm256_acos_ps
+#define gmx_atan2_pr gmx_mm256_atan2_ps
+
+#else /* ifndef GMX_DOUBLE */
+
+#define GMX_SIMD_WIDTH_HERE 4
+
+#define gmx_mm_pr __m256d
+
+#define gmx_mm_pb __m256d
+
+/* We use 128-bit integer registers because of missing 256-bit operations */
+#define gmx_epi32 __m128i
+#define GMX_SIMD_EPI32_WIDTH 4
+
+#define gmx_load_pr _mm256_load_pd
+#define gmx_load1_pr(x) _mm256_set1_pd((x)[0])
+#define gmx_set1_pr _mm256_set1_pd
+#define gmx_setzero_pr _mm256_setzero_pd
+#define gmx_store_pr _mm256_store_pd
+
+#define gmx_add_pr _mm256_add_pd
+#define gmx_sub_pr _mm256_sub_pd
+#define gmx_mul_pr _mm256_mul_pd
+#define gmx_madd_pr(a, b, c) _mm256_add_pd(c, _mm256_mul_pd(a, b))
+#define gmx_nmsub_pr(a, b, c) _mm256_sub_pd(c, _mm256_mul_pd(a, b))
+#define gmx_max_pr _mm256_max_pd
+#define gmx_blendzero_pr _mm256_and_pd
+
+#define gmx_round_pr(x) _mm256_round_pd(x, 0x0)
+#define GMX_SIMD_HAVE_FLOOR
+#define gmx_floor_pr _mm256_floor_pd
+
+#define GMX_SIMD_HAVE_BLENDV
+#define gmx_blendv_pr _mm256_blendv_pd
+
+static gmx_inline gmx_mm_pr gmx_cpsgn_nonneg_pr(gmx_mm_pr a, gmx_mm_pr b)
+{
+ gmx_mm_pr sign_mask = _mm256_set1_pd(-0.0);
+ return _mm256_or_pd(_mm256_and_pd(a, sign_mask), b);
+};
+
- #define GMX_SIMD_HAVE_ANYTRUE
++static gmx_inline gmx_mm_pr gmx_masknot_add_pr(gmx_mm_pb a, gmx_mm_pr b, gmx_mm_pr c)
++{
++ return _mm256_add_pd(b, _mm256_andnot_pd(a, c));
++};
+
+/* Less-than (we use ordered, non-signaling, but that's not required) */
+#define gmx_cmplt_pr(x, y) _mm256_cmp_pd(x, y, 0x11)
+
+#define gmx_and_pb _mm256_and_pd
+#define gmx_or_pb _mm256_or_pd
+
+#define gmx_anytrue_pb _mm256_movemask_pd
+
+#define gmx_cvttpr_epi32 _mm256_cvttpd_epi32
+
+#define gmx_rsqrt_pr(r) _mm256_cvtps_pd(_mm_rsqrt_ps(_mm256_cvtpd_ps(r)))
+#define gmx_rcp_pr(r) _mm256_cvtps_pd(_mm_rcp_ps(_mm256_cvtpd_ps(r)))
+
+#define gmx_exp_pr gmx_mm256_exp_pd
+#define gmx_sqrt_pr gmx_mm256_sqrt_pd
+#define gmx_sincos_pr gmx_mm256_sincos_pd
+#define gmx_acos_pr gmx_mm256_acos_pd
+#define gmx_atan2_pr gmx_mm256_atan2_pd
+
+#endif /* ifndef GMX_DOUBLE */
+
+#endif /* 128- or 256-bit x86 SIMD */
+
+#endif /* GMX_X86_SSE2 */
+
++#endif /* GMX_IS_X86 */
++
++#ifdef GMX_CPU_ACCELERATION_IBM_QPX
++
++/* This hack works on the compilers that can reach this code. A real
++ solution with broader scope will be proposed in master branch. */
++#define gmx_always_inline __attribute__((always_inline))
++
++/* This is for the A2 core on BlueGene/Q that supports IBM's QPX
++ vector built-in functions */
++#define GMX_HAVE_SIMD_MACROS
++#ifdef __clang__
++#include <qpxmath.h>
++#else
++#include "mass_simd.h"
++#endif
++
++/* No need to version the code by the precision, because the QPX AXU
++ extends to and truncates from double precision for free. */
++
++#define GMX_SIMD_WIDTH_HERE 4
++typedef vector4double gmx_mm_pr;
++typedef vector4double gmx_mm_pb;
++typedef vector4double gmx_epi32;
++#define GMX_SIMD_EPI32_WIDTH 4
++
++static gmx_inline gmx_mm_pr gmx_always_inline gmx_load_pr(const real *a)
++{
++#ifdef NDEBUG
++ return vec_ld(0, (real *) a);
++#else
++ return vec_lda(0, (real *) a);
++#endif
++}
++
++static gmx_inline gmx_mm_pr gmx_always_inline gmx_load1_pr(const real *a)
++{
++ return vec_splats(*a);
++}
++
++static gmx_inline gmx_mm_pr gmx_always_inline gmx_set1_pr(real a)
++{
++ return vec_splats(a);
++}
++
++static gmx_inline gmx_mm_pr gmx_always_inline gmx_setzero_pr()
++{
++ return vec_splats(0.0);
++}
++
++static gmx_inline void gmx_always_inline gmx_store_pr(real *a, gmx_mm_pr b)
++{
++#ifdef NDEBUG
++ vec_st(b, 0, a);
++#else
++ vec_sta(b, 0, a);
++#endif
++}
++
++static gmx_inline gmx_mm_pr gmx_always_inline gmx_add_pr(gmx_mm_pr a, gmx_mm_pr b)
++{
++ return vec_add(a, b);
++}
++
++static gmx_inline gmx_mm_pr gmx_always_inline gmx_sub_pr(gmx_mm_pr a, gmx_mm_pr b)
++{
++ return vec_sub(a, b);
++}
++
++static gmx_inline gmx_mm_pr gmx_always_inline gmx_mul_pr(gmx_mm_pr a, gmx_mm_pr b)
++{
++ return vec_mul(a, b);
++}
++
++static gmx_inline gmx_mm_pr gmx_always_inline gmx_madd_pr(gmx_mm_pr a, gmx_mm_pr b, gmx_mm_pr c)
++{
++ return vec_madd(a, b, c);
++}
++
++static gmx_inline gmx_mm_pr gmx_always_inline gmx_nmsub_pr(gmx_mm_pr a, gmx_mm_pr b, gmx_mm_pr c)
++{
++ return vec_nmsub(a, b, c);
++}
++
++static gmx_inline gmx_mm_pr gmx_always_inline gmx_max_pr(gmx_mm_pr a, gmx_mm_pr b)
++{
++ return vec_sel(b, a, vec_sub(a, b));
++}
++
++static gmx_inline gmx_mm_pr gmx_always_inline gmx_blendzero_pr(gmx_mm_pr a, gmx_mm_pr b)
++{
++ return vec_sel(gmx_setzero_pr(), a, b);
++}
++
++static gmx_inline gmx_mm_pb gmx_always_inline gmx_cmplt_pr(gmx_mm_pr a, gmx_mm_pr b)
++{
++ return vec_cmplt(a, b);
++}
++
++static gmx_inline gmx_mm_pb gmx_always_inline gmx_and_pb(gmx_mm_pb a, gmx_mm_pb b)
++{
++ return vec_and(a, b);
++}
++
++static gmx_inline gmx_mm_pb gmx_always_inline gmx_or_pb(gmx_mm_pb a, gmx_mm_pb b)
++{
++ return vec_or(a, b);
++}
++
++static gmx_inline gmx_mm_pr gmx_always_inline gmx_round_pr(gmx_mm_pr a)
++{
++ return vec_round(a);
++}
++
++#define GMX_SIMD_HAVE_FLOOR
++static gmx_inline gmx_mm_pr gmx_always_inline gmx_floor_pr(gmx_mm_pr a)
++{
++ return vec_floor(a);
++}
++
++#define GMX_SIMD_HAVE_BLENDV
++static gmx_inline gmx_mm_pr gmx_always_inline gmx_blendv_pr(gmx_mm_pr a, gmx_mm_pr b, gmx_mm_pr c)
++{
++ return vec_sel(b, a, gmx_cmplt_pr(gmx_setzero_pr(), c));
++}
++
++static gmx_inline gmx_mm_pr gmx_always_inline gmx_cpsgn_nonneg_pr(gmx_mm_pr a, gmx_mm_pr b)
++{
++ return vec_cpsgn(a, b);
++};
++
++static gmx_inline gmx_mm_pr gmx_always_inline gmx_masknot_add_pr(gmx_mm_pb a, gmx_mm_pr b, gmx_mm_pr c)
++{
++ return vec_add(b, vec_sel(c, gmx_setzero_pr(), a));
++};
++
++static gmx_inline gmx_bool gmx_always_inline
++GMX_SIMD_IS_TRUE(real x)
++{
++ return x >= 0.0;
++}
++
++static gmx_inline gmx_epi32 gmx_always_inline gmx_cvttpr_epi32(gmx_mm_pr a)
++{
++ return vec_ctiwuz(a);
++}
++/* Don't want this, we have floor */
++/* #define gmx_cvtepi32_pr vec_cvtepi32 */
++
++/* A2 core on BG/Q delivers relative error of 2^-14, whereas Power ISA
++ Architecture only promises 2^-8. So probably no need for
++ Newton-Raphson iterates at single or double. */
++static gmx_inline gmx_mm_pr gmx_always_inline gmx_rsqrt_pr(gmx_mm_pr a)
++{
++ return vec_rsqrte(a);
++}
++
++/* A2 core on BG/Q delivers relative error of 2^-14, whereas Power ISA
++ Architecture only promises 2^-5. So probably no need for
++ Newton-Raphson iterates at single or double. */
++static gmx_inline gmx_mm_pr gmx_always_inline gmx_rcp_pr(gmx_mm_pr a)
++{
++ return vec_re(a);
++}
++
++/* Note that here, and below, we use the built-in SLEEF port when
++ compiling on BlueGene/Q with clang */
++
++#define GMX_SIMD_HAVE_EXP
++static gmx_inline gmx_mm_pr gmx_always_inline gmx_exp_pr(gmx_mm_pr a)
++{
++#ifdef __clang__
++#ifndef GMX_DOUBLE
++ return xexpf(a);
++#else
++ return xexp(a);
++#endif
++#else
++#ifndef GMX_DOUBLE
++ return expf4(a);
++#else
++ return expd4(a);
++#endif
++#endif
++}
++
++static gmx_inline gmx_mm_pr gmx_always_inline gmx_sqrt_pr(gmx_mm_pr a)
++{
++#ifdef NDEBUG
++ return vec_swsqrt_nochk(a);
++#else
++ return vec_swsqrt(a);
++#endif
++}
++
++#define GMX_SIMD_HAVE_TRIGONOMETRIC
++static gmx_inline int gmx_always_inline gmx_sincos_pr(gmx_mm_pr a, gmx_mm_pr *b, gmx_mm_pr *c)
++{
++#ifdef __clang__
++#ifndef GMX_DOUBLE
++ xsincosf(a, b, c);
++#else
++ xsincos(a, b, c);
++#endif
++#else
++#ifndef GMX_DOUBLE
++ sincosf4(a, b, c);
++#else
++ sincosd4(a, b, c);
++#endif
++#endif
++ return 1;
++}
++
++static gmx_inline gmx_mm_pr gmx_always_inline gmx_acos_pr(gmx_mm_pr a)
++{
++#ifdef __clang__
++#ifndef GMX_DOUBLE
++ return xacosf(a);
++#else
++ return xacos(a);
++#endif
++#else
++#ifndef GMX_DOUBLE
++ return acosf4(a);
++#else
++ return acosd4(a);
++#endif
++#endif
++}
++
++/* NB The order of parameters here is correct; the
++ documentation of atan2[df]4 in SIMD MASS is wrong. */
++static gmx_inline gmx_mm_pr gmx_always_inline gmx_atan2_pr(gmx_mm_pr a, gmx_mm_pr b)
++{
++#ifdef __clang__
++#ifndef GMX_DOUBLE
++ return xatan2f(a, b);
++#else
++ return xatan2(a, b);
++#endif
++#else
++#ifndef GMX_DOUBLE
++ return atan2f4(a, b);
++#else
++ return atan2d4(a, b);
++#endif
++#endif
++}
++
++static gmx_inline int gmx_always_inline
++gmx_anytrue_pb(gmx_mm_pb a)
++{
++ /* The "anytrue" is done solely on the QPX AXU (which is the only
++ available FPU). This is awkward, because pretty much no
++ "horizontal" SIMD-vector operations exist, unlike x86 where
++ SSE4.1 added various kinds of horizontal operations. So we have
++ to make do with shifting vector elements and operating on the
++ results. This makes for lots of data dependency, but the main
++ alternative of storing to memory and reloading is not going to
++ help, either. OpenMP over 2 or 4 hardware threads per core will
++ hide much of the latency from the data dependency. The
++ vec_extract() lets the compiler correctly use a floating-point
++ comparison on the zeroth vector element, which avoids needing
++ memory at all.
++ */
++ gmx_mm_pb vec_shifted_left_0 = a;
++ gmx_mm_pb vec_shifted_left_1 = vec_sldw(a, a, 1);
++ gmx_mm_pb vec_shifted_left_2 = vec_sldw(a, a, 2);
++ gmx_mm_pb vec_shifted_left_3 = vec_sldw(a, a, 3);
++
++ gmx_mm_pb vec_return = vec_or(vec_or(vec_shifted_left_2, vec_shifted_left_3),
++ vec_or(vec_shifted_left_0, vec_shifted_left_1));
++ return (0.0 < vec_extract(vec_return, 0));
++};
++
++#undef gmx_always_inline
++
++#endif /* GMX_CPU_ACCELERATION_IBM_QPX */
+
+#ifdef GMX_HAVE_SIMD_MACROS
+/* Generic functions to extract a SIMD aligned pointer from a pointer x.
+ * x should have at least GMX_SIMD_WIDTH_HERE elements extra compared
+ * to how many you want to use, to avoid indexing outside the aligned region.
+ */
+
+static gmx_inline real *
+gmx_simd_align_real(const real *x)
+{
+ return (real *)(((size_t)((x)+GMX_SIMD_WIDTH_HERE)) & (~((size_t)(GMX_SIMD_WIDTH_HERE*sizeof(real)-1))));
+}
+
+static gmx_inline int *
+gmx_simd_align_int(const int *x)
+{
+ return (int *)(((size_t)((x)+GMX_SIMD_WIDTH_HERE)) & (~((size_t)(GMX_SIMD_WIDTH_HERE*sizeof(int )-1))));
+}
+
+
+/* Include the math functions which only need the above macros,
+ * generally these are the ones that don't need masking operations.
+ */
+#ifdef GMX_DOUBLE
+#include "gmx_simd_math_double.h"
+#else
+#include "gmx_simd_math_single.h"
+#endif
+
++
+#endif /* GMX_HAVE_SIMD_MACROS */
+
+#endif /* _gmx_simd_macros_h_ */
--- /dev/null
- t_state *state, t_extmass *MassQ, df_history_t *dfhist,
+/*
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * VERSION 3.2.0
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2004, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * Gromacs Runs On Most of All Computer Systems
+ */
+
+#ifndef _mdrun_h
+#define _mdrun_h
+
+#include <stdio.h>
+#include <time.h>
+#include "typedefs.h"
+#include "network.h"
+#include "sim_util.h"
+#include "tgroup.h"
+#include "filenm.h"
+#include "mshift.h"
+#include "edsam.h"
+#include "mdebin.h"
+#include "vcm.h"
+#include "vsite.h"
+#include "pull.h"
+#include "update.h"
+#include "types/membedt.h"
+#include "types/globsig.h"
+
+
+#ifdef GMX_THREAD_MPI
+#include "thread_mpi/threads.h"
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define MD_POLARISE (1<<2)
+#define MD_RERUN (1<<4)
+#define MD_RERUN_VSITE (1<<5)
+#define MD_SEPPOT (1<<7)
+#define MD_PARTDEC (1<<9)
+#define MD_DDBONDCHECK (1<<10)
+#define MD_DDBONDCOMM (1<<11)
+#define MD_CONFOUT (1<<12)
+#define MD_REPRODUCIBLE (1<<13)
+#define MD_READ_RNG (1<<14)
+#define MD_APPENDFILES (1<<15)
+#define MD_APPENDFILESSET (1<<21)
+#define MD_KEEPANDNUMCPT (1<<16)
+#define MD_READ_EKIN (1<<17)
+#define MD_STARTFROMCPT (1<<18)
+#define MD_RESETCOUNTERSHALFWAY (1<<19)
+#define MD_TUNEPME (1<<20)
+#define MD_TESTVERLET (1<<22)
+
+/* The options for the domain decomposition MPI task ordering */
+enum {
+ ddnoSEL, ddnoINTERLEAVE, ddnoPP_PME, ddnoCARTESIAN, ddnoNR
+};
+
+/* The options for the thread affinity setting, default: auto */
+enum {
+ threadaffSEL, threadaffAUTO, threadaffON, threadaffOFF, threadaffNR
+};
+
+typedef struct {
+ int nthreads_tot; /* Total number of threads requested (TMPI) */
+ int nthreads_tmpi; /* Number of TMPI threads requested */
+ int nthreads_omp; /* Number of OpenMP threads requested */
+ int nthreads_omp_pme; /* As nthreads_omp, but for PME only nodes */
+ int thread_affinity; /* Thread affinity switch, see enum above */
+ int core_pinning_stride; /* Logical core pinning stride */
+ int core_pinning_offset; /* Logical core pinning offset */
+ char *gpu_id; /* GPU id's to use, each specified as chars */
+} gmx_hw_opt_t;
+
+/* Variables for temporary use with the deform option,
+ * used in runner.c and md.c.
+ * (These variables should be stored in the tpx file.)
+ */
+extern gmx_large_int_t deform_init_init_step_tpx;
+extern matrix deform_init_box_tpx;
+#ifdef GMX_THREAD_MPI
+extern tMPI_Thread_mutex_t deform_init_box_mutex;
+
+/* The minimum number of atoms per tMPI thread. With fewer atoms than this,
+ * the number of threads will get lowered.
+ */
+#define MIN_ATOMS_PER_MPI_THREAD 90
+#define MIN_ATOMS_PER_GPU 900
+#endif
+
+
+typedef double gmx_integrator_t (FILE *log, t_commrec *cr,
+ int nfile, const t_filenm fnm[],
+ const output_env_t oenv, gmx_bool bVerbose,
+ gmx_bool bCompact, int nstglobalcomm,
+ gmx_vsite_t *vsite, gmx_constr_t constr,
+ int stepout,
+ t_inputrec *inputrec,
+ gmx_mtop_t *mtop, t_fcdata *fcd,
+ t_state *state,
+ t_mdatoms *mdatoms,
+ t_nrnb *nrnb, gmx_wallcycle_t wcycle,
+ gmx_edsam_t ed,
+ t_forcerec *fr,
+ int repl_ex_nst, int repl_ex_nex, int repl_ex_seed,
+ gmx_membed_t membed,
+ real cpt_period, real max_hours,
+ const char *deviceOptions,
+ unsigned long Flags,
+ gmx_runtime_t *runtime);
+
+/* ROUTINES from md.c */
+
+gmx_integrator_t do_md;
+
+
+/* ROUTINES from minimize.c */
+
+gmx_integrator_t do_steep;
+/* Do steepest descents EM */
+
+gmx_integrator_t do_cg;
+/* Do conjugate gradient EM */
+
+gmx_integrator_t do_lbfgs;
+/* Do conjugate gradient L-BFGS */
+
+gmx_integrator_t do_nm;
+/* Do normal mode analysis */
+
+/* ROUTINES from tpi.c */
+
+gmx_integrator_t do_tpi;
+/* Do test particle insertion */
+
+void init_npt_masses(t_inputrec *ir, t_state *state, t_extmass *MassQ, gmx_bool bInit);
+
++void init_expanded_ensemble(gmx_bool bStateFromCP, t_inputrec *ir, gmx_rng_t *mcrng, df_history_t *dfhist);
++
+int ExpandedEnsembleDynamics(FILE *log, t_inputrec *ir, gmx_enerdata_t *enerd,
- int nlam, int frequency, gmx_large_int_t step);
++ t_state *state, t_extmass *MassQ, int fep_state, df_history_t *dfhist,
+ gmx_large_int_t step, gmx_rng_t mcrng,
+ rvec *v, t_mdatoms *mdatoms);
+
+void PrintFreeEnergyInfoToFile(FILE *outfile, t_lambda *fep, t_expanded *expand, t_simtemp *simtemp, df_history_t *dfhist,
- df_history_t df_history,
++ int fep_state, int frequency, gmx_large_int_t step);
+
+void get_mc_state(gmx_rng_t rng, t_state *state);
+
+void set_mc_state(gmx_rng_t rng, t_state *state);
+
+/* check the version */
+void check_ir_old_tpx_versions(t_commrec *cr, FILE *fplog,
+ t_inputrec *ir, gmx_mtop_t *mtop);
+
+/* Allocate and initialize node-local state entries. */
+void set_state_entries(t_state *state, const t_inputrec *ir, int nnodes);
+
+/* Broadcast the data for a simulation, and allocate node-specific settings
+ such as rng generators. */
+void init_parallel(t_commrec *cr, t_inputrec *inputrec,
+ gmx_mtop_t *mtop);
+
+int mdrunner(gmx_hw_opt_t *hw_opt,
+ FILE *fplog, t_commrec *cr, int nfile,
+ const t_filenm fnm[], const output_env_t oenv, gmx_bool bVerbose,
+ gmx_bool bCompact, int nstglobalcomm, ivec ddxyz, int dd_node_order,
+ real rdd, real rconstr, const char *dddlb_opt, real dlb_scale,
+ const char *ddcsx, const char *ddcsy, const char *ddcsz,
+ const char *nbpu_opt,
+ gmx_large_int_t nsteps_cmdline, int nstepout, int resetstep,
+ int nmultisim, int repl_ex_nst, int repl_ex_nex,
+ int repl_ex_seed, real pforce, real cpt_period, real max_hours,
+ const char *deviceOptions, unsigned long Flags);
+/* Driver routine, that calls the different methods */
+
+void
+do_trajectory_writing(FILE *fplog,
+ t_commrec *cr,
+ int nfile,
+ const t_filenm fnm[],
+ gmx_large_int_t step,
+ gmx_large_int_t step_rel,
+ double t,
+ t_inputrec *ir,
+ t_state *state,
+ t_state *state_global,
+ gmx_mtop_t *top_global,
+ t_forcerec *fr,
+ gmx_update_t upd,
+ gmx_mdoutf_t *outf,
+ t_mdebin *mdebin,
+ gmx_ekindata_t *ekind,
+ rvec *f,
+ rvec *f_global,
+ gmx_wallcycle_t wcycle,
+ gmx_rng_t mcrng,
+ int *nchkpt,
+ gmx_bool bCPT,
+ gmx_bool bRerunMD,
+ gmx_bool bLastStep,
+ gmx_bool bDoConfOut,
+ gmx_bool bSumEkinhOld
+ );
+/* Wrapper routine for trajectory writing */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _mdrun_h */
--- /dev/null
- /* If the first part of the hostname (up to the first dot) ends with a number, returns this number.
- If the first part of the hostname does not ends in a number (0-9 characters), returns 0.
+/*
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * VERSION 3.2.0
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2004, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * Gromacs Runs On Most of All Computer Systems
+ */
+
+#ifndef _network_h
+#define _network_h
+
+
+/*
+ * This module defines the interface of the actual communication routines.
+ */
+
+#include <stdio.h>
+
+#include "types/simple.h"
+#include "types/commrec.h"
+#include "typedefs.h"
+#include "main.h"
+#include "gmx_fatal.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+t_commrec *init_commrec(void);
+/* Allocate, initialize and return the commrec. */
+
+t_commrec *reinitialize_commrec_for_this_thread(const t_commrec *cro);
+/* Initialize communication records for thread-parallel simulations.
+ Must be called on all threads before any communication takes place by
+ the individual threads. Copies the original commrec to
+ thread-local versions (a small memory leak results because we don't
+ deallocate the old shared version). */
+
+void gmx_fill_commrec_from_mpi(t_commrec *cr);
+/* Continues t_commrec construction */
+
+int gmx_node_num(void);
+/* return the number of nodes in the ring */
+
+int gmx_node_rank(void);
+/* return the rank of the node */
+
+int gmx_hostname_num(void);
++/* Ostensibly, returns a integer characteristic of and unique to each
++ physical node in the MPI system. If the first part of the MPI
++ hostname (up to the first dot) ends with a number, returns this
++ number. If the first part of the MPI hostname does not ends in a
++ number (0-9 characters), returns 0.
+ */
+
+void gmx_setup_nodecomm(FILE *fplog, t_commrec *cr);
+/* Sets up fast global communication for clusters with multi-core nodes */
+
+void gmx_init_intranode_counters(t_commrec *cr);
+/* Initializes intra-physical-node MPI process/thread counts and ID. */
+
+gmx_bool gmx_mpi_initialized(void);
+/* return TRUE when MPI_Init has been called.
+ * return FALSE when MPI_Init has not been called OR
+ * when GROMACS was compiled without MPI support.
+ */
+
+void gmx_barrier(const t_commrec *cr);
+/* Wait till all processes in cr->mpi_comm_mygroup have reached the barrier */
+
+void gmx_bcast(int nbytes, void *b, const t_commrec *cr);
+/* Broadcast nbytes bytes from the master to cr->mpi_comm_mygroup */
+
+void gmx_bcast_sim(int nbytes, void *b, const t_commrec *cr);
+/* Broadcast nbytes bytes from the sim master to cr->mpi_comm_mysim */
+
+void gmx_sumi(int nr, int r[], const t_commrec *cr);
+/* Calculate the global sum of an array of ints */
+
+void gmx_sumli(int nr, gmx_large_int_t r[], const t_commrec *cr);
+/* Calculate the global sum of an array of large ints */
+
+void gmx_sumf(int nr, float r[], const t_commrec *cr);
+/* Calculate the global sum of an array of floats */
+
+void gmx_sumd(int nr, double r[], const t_commrec *cr);
+/* Calculate the global sum of an array of doubles */
+
+void gmx_sumf_comm(int nr, float r[], MPI_Comm mpi_comm);
+/* Calculate the global sum of an array of floats */
+
+void gmx_sumd_comm(int nr, double r[], MPI_Comm mpi_comm);
+/* Calculate the global sum of an array of doubles */
+
+void gmx_sumi_sim(int nr, int r[], const gmx_multisim_t *ms);
+/* Calculate the sum over the simulations of an array of ints */
+
+void gmx_sumli_sim(int nr, gmx_large_int_t r[], const gmx_multisim_t *ms);
+/* Calculate the sum over the simulations of an array of large ints */
+
+void gmx_sumf_sim(int nr, float r[], const gmx_multisim_t *ms);
+/* Calculate the sum over the simulations of an array of floats */
+
+void gmx_sumd_sim(int nr, double r[], const gmx_multisim_t *ms);
+/* Calculate the sum over the simulations of an array of doubles */
+
+void gmx_abort(int nodeid, int nnodes, int errorno);
+/* Abort the parallel run */
+
+#ifdef GMX_DOUBLE
+#define gmx_sum_comm gmx_sumd_comm
+#define gmx_sum gmx_sumd
+#define gmx_sum_sim gmx_sumd_sim
+#else
+#define gmx_sum_comm gmx_sumf_comm
+#define gmx_sum gmx_sumf
+#define gmx_sum_sim gmx_sumf_sim
+#endif
+
+#ifdef DEBUG_GMX
+#define debug_gmx() do { FILE *fp = debug ? debug : stderr; \
+ if (bDebugMode()) { fprintf(fp, "NODEID=%d, %s %d\n", gmx_mpi_initialized() ? gmx_node_rank() : -1, __FILE__, __LINE__); } fflush(fp); } while (0)
+#else
+#define debug_gmx()
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif /* _network_h */
--- /dev/null
- void init_df_history(df_history_t *dfhist, int nlambda, real wl_delta);
+/*
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * VERSION 3.2.0
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2004, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * Gromacs Runs On Most of All Computer Systems
+ */
+
+#ifndef _typedefs_h
+#define _typedefs_h
+
+
+/* DEPRECATED! value for signaling unitialized variables */
+#define NOTSET -12345
+
+#include <sys/types.h>
+#include "sysstuff.h"
+#include "types/simple.h"
+#include "types/enums.h"
+#include "types/block.h"
+#include "types/symtab.h"
+#include "types/idef.h"
+#include "types/atoms.h"
+#include "types/trx.h"
+#include "types/topology.h"
+#include "types/energy.h"
+#include "types/inputrec.h"
+#include "types/ishift.h"
+#include "types/graph.h"
+#include "types/nrnb.h"
+#include "types/nblist.h"
+#include "types/nbnxn_pairlist.h"
+#include "types/nsgrid.h"
+#include "types/forcerec.h"
+#include "types/fcdata.h"
+#include "types/mdatom.h"
+#include "types/pbc.h"
+#include "types/ifunc.h"
+#include "types/filenm.h"
+#include "types/group.h"
+#include "types/state.h"
+#include "types/shellfc.h"
+#include "types/constr.h"
+#include "types/matrix.h"
+#include "types/oenv.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Memory (re)allocation can be VERY slow, especially with some
+ * MPI libraries that replace the standard malloc and realloc calls.
+ * To avoid slow memory allocation we use over_alloc to set the memory
+ * allocation size for large data blocks. Since this scales the size
+ * with a factor, we use log(n) realloc calls instead of n.
+ * This can reduce allocation times from minutes to seconds.
+ */
+/* This factor leads to 4 realloc calls to double the array size */
+#define OVER_ALLOC_FAC 1.19
+
+void set_over_alloc_dd(gmx_bool set);
+/* Turns over allocation for variable size atoms/cg/top arrays on or off,
+ * default is off.
+ */
+
+int over_alloc_dd(int n);
+/* Returns n when domain decomposition over allocation is off.
+ * Returns OVER_ALLOC_FAC*n + 100 when over allocation in on.
+ * This is to avoid frequent reallocation
+ * during domain decomposition in mdrun.
+ */
+
+/* Over allocation for small data types: int, real etc. */
+#define over_alloc_small(n) (int)(OVER_ALLOC_FAC*(n) + 8000)
+
+/* Over allocation for large data types: complex structs */
+#define over_alloc_large(n) (int)(OVER_ALLOC_FAC*(n) + 1000)
+
+int gmx_large_int_to_int(gmx_large_int_t step, const char *warn);
+/* Convert a gmx_large_int_t value to int.
+ * If warn!=NULL a warning message will be written
+ * to stderr when step does not fit in an int,
+ * the first line is:
+ * "WARNING during %s:", where warn is printed in %s.
+ */
+
+#define STEPSTRSIZE 22
+
+char *gmx_step_str(gmx_large_int_t i, char *buf);
+/* Prints a gmx_large_int_t value in buf and returns the pointer to buf.
+ * buf should be large enough to contain i: STEPSTRSIZE (22) chars.
+ * When multiple gmx_large_int_t values are printed in the same printf call,
+ * be sure to call gmx_step_str with different buffers.
+ */
+
+/* Functions to initiate and delete structures *
+ * These functions are defined in gmxlib/typedefs.c
+ */
+void init_block(t_block *block);
+void init_blocka(t_blocka *block);
+void init_atom (t_atoms *at);
+void init_mtop(gmx_mtop_t *mtop);
+void init_top (t_topology *top);
+void init_inputrec(t_inputrec *ir);
+void init_energyhistory(energyhistory_t * enerhist);
+void done_energyhistory(energyhistory_t * enerhist);
+void init_gtc_state(t_state *state, int ngtc, int nnhpres, int nhchainlength);
+void init_state(t_state *state, int natoms, int ngtc, int nnhpres, int nhchainlength, int nlambda);
++void init_df_history(df_history_t *dfhist, int nlambda);
++void done_df_history(df_history_t *dfhist);
+void copy_df_history(df_history_t * df_dest, df_history_t *df_source);
+
+void copy_blocka(const t_blocka *src, t_blocka *dest);
+
+void done_block(t_block *block);
+void done_blocka(t_blocka *block);
+void done_atom (t_atoms *at);
+void done_moltype(gmx_moltype_t *molt);
+void done_molblock(gmx_molblock_t *molb);
+void done_mtop(gmx_mtop_t *mtop, gmx_bool bDoneSymtab);
+void done_top(t_topology *top);
+void done_inputrec(t_inputrec *ir);
+void done_state(t_state *state);
+
+void set_box_rel(t_inputrec *ir, t_state *state);
+/* Set state->box_rel used in mdrun to preserve the box shape */
+
+void preserve_box_shape(t_inputrec *ir, matrix box_rel, matrix b);
+/* Preserve the box shape, b can be box or boxv */
+
+void stupid_fill_block(t_block *grp, int natom, gmx_bool bOneIndexGroup);
+/* Fill a block structure with numbers identical to the index
+ * (0, 1, 2, .. natom-1)
+ * If bOneIndexGroup, then all atoms are lumped in one index group,
+ * otherwise there is one atom per index entry
+ */
+
+void stupid_fill_blocka(t_blocka *grp, int natom);
+/* Fill a block structure with numbers identical to the index
+ * (0, 1, 2, .. natom-1)
+ * There is one atom per index entry
+ */
+
+void init_t_atoms(t_atoms *atoms, int natoms, gmx_bool bPdbinfo);
+/* allocate memory for the arrays, set nr to natoms and nres to 0
+ * set pdbinfo to NULL or allocate memory for it */
+
+t_atoms *copy_t_atoms(t_atoms *src);
+/* copy an atoms struct from src to a new one */
+
+void add_t_atoms(t_atoms *atoms, int natom_extra, int nres_extra);
+/* allocate extra space for more atoms and or residues */
+
+void t_atoms_set_resinfo(t_atoms *atoms, int atom_ind, t_symtab *symtab,
+ const char *resname, int resnr, unsigned char ic,
+ int chainnum, char chainid);
+/* Set the residue name, number, insertion code and chain identifier
+ * of atom index atom_ind.
+ */
+
+void free_t_atoms(t_atoms *atoms, gmx_bool bFreeNames);
+/* Free all the arrays and set the nr and nres to 0.
+ * bFreeNames tells if to free the atom and residue name strings,
+ * don't free them if they still need to be used in e.g. the topology struct.
+ */
+
+t_atoms *mtop2atoms(gmx_mtop_t *mtop);
+/* generate a t_atoms struct for the system from gmx_mtop_t */
+
+real max_cutoff(real cutoff1, real cutoff2);
+/* Returns the maximum of the cut-off's, taking into account that 0=inf. */
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif /* _typedefs_h */
--- /dev/null
- gmx_bool bInit_weights; /* did we initialize the weights? */
+/*
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * VERSION 3.2.0
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2004, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * GRoups of Organic Molecules in ACtion for Science
+ */
+#ifndef _inputrec_h_
+#define _inputrec_h_
+
+
+#include "simple.h"
+#include "../sysstuff.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+typedef struct {
+ int n; /* Number of terms */
+ real *a; /* Coeffients (V / nm ) */
+ real *phi; /* Phase angles */
+} t_cosines;
+
+typedef struct {
+ real E0; /* Field strength (V/nm) */
+ real omega; /* Frequency (1/ps) */
+ real t0; /* Centre of the Gaussian pulse (ps) */
+ real sigma; /* Width of the Gaussian pulse (FWHM) (ps) */
+} t_efield;
+
+#define EGP_EXCL (1<<0)
+#define EGP_TABLE (1<<1)
+
+typedef struct {
+ int ngtc; /* # T-Coupl groups */
+ int nhchainlength; /* # of nose-hoover chains per group */
+ int ngacc; /* # Accelerate groups */
+ int ngfrz; /* # Freeze groups */
+ int ngener; /* # Ener groups */
+ real *nrdf; /* Nr of degrees of freedom in a group */
+ real *ref_t; /* Coupling temperature per group */
+ int *annealing; /* No/simple/periodic SA for each group */
+ int *anneal_npoints; /* Number of annealing time points per grp */
+ real **anneal_time; /* For ea. group: Time points */
+ real **anneal_temp; /* For ea. grp: Temperature at these times */
+ /* Final temp after all intervals is ref_t */
+ real *tau_t; /* Tau coupling time */
+ rvec *acc; /* Acceleration per group */
+ ivec *nFreeze; /* Freeze the group in each direction ? */
+ int *egp_flags; /* Exclusions/tables of energy group pairs */
+
+ /* QMMM stuff */
+ int ngQM; /* nr of QM groups */
+ int *QMmethod; /* Level of theory in the QM calculation */
+ int *QMbasis; /* Basisset in the QM calculation */
+ int *QMcharge; /* Total charge in the QM region */
+ int *QMmult; /* Spin multiplicicty in the QM region */
+ gmx_bool *bSH; /* surface hopping (diabatic hop only) */
+ int *CASorbitals; /* number of orbiatls in the active space */
+ int *CASelectrons; /* number of electrons in the active space */
+ real *SAon; /* at which gap (A.U.) the SA is switched on */
+ real *SAoff;
+ int *SAsteps; /* in how many steps SA goes from 1-1 to 0.5-0.5*/
+ gmx_bool *bOPT;
+ gmx_bool *bTS;
+} t_grpopts;
+
+enum {
+ epgrppbcNONE, epgrppbcREFAT, epgrppbcCOS
+};
+
+typedef struct {
+ int nat; /* Number of atoms in the pull group */
+ atom_id *ind; /* The global atoms numbers */
+ int nat_loc; /* Number of local pull atoms */
+ int nalloc_loc; /* Allocation size for ind_loc and weight_loc */
+ atom_id *ind_loc; /* Local pull indices */
+ int nweight; /* The number of weights (0 or nat) */
+ real *weight; /* Weights (use all 1 when weight==NULL) */
+ real *weight_loc; /* Weights for the local indices */
+ int epgrppbc; /* The type of pbc for this pull group, see enum above */
+ atom_id pbcatom; /* The reference atom for pbc (global number) */
+ rvec vec; /* The pull vector, direction or position */
+ rvec init; /* Initial reference displacement */
+ real rate; /* Rate of motion (nm/ps) */
+ real k; /* force constant */
+ real kB; /* force constant for state B */
+ real wscale; /* scaling factor for the weights: sum w m/sum w w m */
+ real invtm; /* inverse total mass of the group: 1/wscale sum w m */
+ dvec x; /* center of mass before update */
+ dvec xp; /* center of mass after update before constraining */
+ dvec dr; /* The distance from the reference group */
+ double f_scal; /* Scalar force for directional pulling */
+ dvec f; /* force due to the pulling/constraining */
+} t_pullgrp;
+
+typedef struct {
+ int eSimTempScale; /* simulated temperature scaling; linear or exponential */
+ real simtemp_low; /* the low temperature for simulated tempering */
+ real simtemp_high; /* the high temperature for simulated tempering */
+ real *temperatures; /* the range of temperatures used for simulated tempering */
+} t_simtemp;
+
+typedef struct {
+ int nstdhdl; /* The frequency for calculating dhdl */
+ double init_lambda; /* fractional value of lambda (usually will use
+ init_fep_state, this will only be for slow growth,
+ and for legacy free energy code. Only has a
+ valid value if positive) */
+ int init_fep_state; /* the initial number of the state */
+ double delta_lambda; /* change of lambda per time step (fraction of (0.1) */
+ gmx_bool bPrintEnergy; /* Whether to print the energy in the dhdl */
+ int n_lambda; /* The number of foreign lambda points */
+ double **all_lambda; /* The array of all lambda values */
+ int lambda_neighbors; /* The number of neighboring lambda states to
+ calculate the energy for in up and down directions
+ (-1 for all) */
+ int lambda_start_n; /* The first lambda to calculate energies for */
+ int lambda_stop_n; /* The last lambda +1 to calculate energies for */
+ real sc_alpha; /* free energy soft-core parameter */
+ int sc_power; /* lambda power for soft-core interactions */
+ real sc_r_power; /* r power for soft-core interactions */
+ real sc_sigma; /* free energy soft-core sigma when c6 or c12=0 */
+ real sc_sigma_min; /* free energy soft-core sigma for ????? */
+ gmx_bool bScCoul; /* use softcore for the coulomb portion as well (default FALSE) */
+ gmx_bool separate_dvdl[efptNR]; /* whether to print the dvdl term associated with
+ this term; if it is not specified as separate,
+ it is lumped with the FEP term */
+ int separate_dhdl_file; /* whether to write a separate dhdl.xvg file
+ note: NOT a gmx_bool, but an enum */
+ int dhdl_derivatives; /* whether to calculate+write dhdl derivatives
+ note: NOT a gmx_bool, but an enum */
+ int dh_hist_size; /* The maximum table size for the dH histogram */
+ double dh_hist_spacing; /* The spacing for the dH histogram */
+} t_lambda;
+
+typedef struct {
+ int nstexpanded; /* The frequency of expanded ensemble state changes */
+ int elamstats; /* which type of move updating do we use for lambda monte carlo (or no for none) */
+ int elmcmove; /* what move set will be we using for state space moves */
+ int elmceq; /* the method we use to decide of we have equilibrated the weights */
+ int equil_n_at_lam; /* the minumum number of samples at each lambda for deciding whether we have reached a minimum */
+ real equil_wl_delta; /* WL delta at which we stop equilibrating weights */
+ real equil_ratio; /* use the ratio of weights (ratio of minimum to maximum) to decide when to stop equilibrating */
+ int equil_steps; /* after equil_steps steps we stop equilibrating the weights */
+ int equil_samples; /* after equil_samples total samples (steps/nstfep), we stop equilibrating the weights */
+ int lmc_seed; /* random number seed for lambda mc switches */
+ gmx_bool minvar; /* whether to use minumum variance weighting */
+ int minvarmin; /* the number of samples needed before kicking into minvar routine */
+ real minvar_const; /* the offset for the variance in MinVar */
+ int c_range; /* range of cvalues used for BAR */
+ gmx_bool bSymmetrizedTMatrix; /* whether to print symmetrized matrices */
+ int nstTij; /* How frequently to print the transition matrices */
+ int lmc_repeats; /* number of repetitions in the MC lambda jumps */ /*MRS -- VERIFY THIS */
+ int lmc_forced_nstart; /* minimum number of samples for each state before free sampling */ /* MRS -- VERIFY THIS! */
+ int gibbsdeltalam; /* distance in lambda space for the gibbs interval */
+ real wl_scale; /* scaling factor for wang-landau */
+ real wl_ratio; /* ratio between largest and smallest number for freezing the weights */
+ real init_wl_delta; /* starting delta for wang-landau */
+ gmx_bool bWLoneovert; /* use one over t convergece for wang-landau when the delta get sufficiently small */
++ gmx_bool bInit_weights; /* did we initialize the weights? TODO: REMOVE FOR 5.0, no longer needed with new logic */
+ real mc_temp; /* To override the main temperature, or define it if it's not defined */
+ real *init_lambda_weights; /* user-specified initial weights to start with */
+} t_expanded;
+
+typedef struct {
+ int ngrp; /* number of groups */
+ int eGeom; /* pull geometry */
+ ivec dim; /* used to select components for constraint */
+ real cyl_r1; /* radius of cylinder for dynamic COM */
+ real cyl_r0; /* radius of cylinder including switch length */
+ real constr_tol; /* absolute tolerance for constraints in (nm) */
+ int nstxout; /* Output frequency for pull x */
+ int nstfout; /* Output frequency for pull f */
+ int ePBC; /* the boundary conditions */
+ int npbcdim; /* do pbc in dims 0 <= dim < npbcdim */
+ gmx_bool bRefAt; /* do we need reference atoms for a group COM ? */
+ int cosdim; /* dimension for cosine weighting, -1 if none */
+ gmx_bool bVirial; /* do we need to add the pull virial? */
+ t_pullgrp *grp; /* groups to pull/restrain/etc/ */
+ t_pullgrp *dyna; /* dynamic groups for use with local constraints */
+ rvec *rbuf; /* COM calculation buffer */
+ dvec *dbuf; /* COM calculation buffer */
+ double *dbuf_cyl; /* cylinder ref. groups COM calculation buffer */
+
+ FILE *out_x; /* output file for pull data */
+ FILE *out_f; /* output file for pull data */
+} t_pull;
+
+
+/* Abstract types for enforced rotation only defined in pull_rotation.c */
+typedef struct gmx_enfrot *gmx_enfrot_t;
+typedef struct gmx_enfrotgrp *gmx_enfrotgrp_t;
+
+typedef struct {
+ int eType; /* Rotation type for this group */
+ int bMassW; /* Use mass-weighed positions? */
+ int nat; /* Number of atoms in the group */
+ atom_id *ind; /* The global atoms numbers */
+ rvec *x_ref; /* The reference positions */
+ rvec vec; /* The normalized rotation vector */
+ real rate; /* Rate of rotation (degree/ps) */
+ real k; /* Force constant (kJ/(mol nm^2) */
+ rvec pivot; /* Pivot point of rotation axis (nm) */
+ int eFittype; /* Type of fit to determine actual group angle */
+ int PotAngle_nstep; /* Number of angles around the reference angle
+ for which the rotation potential is also
+ evaluated (for fit type 'potential' only) */
+ real PotAngle_step; /* Distance between two angles in degrees (for
+ fit type 'potential' only) */
+ real slab_dist; /* Slab distance (nm) */
+ real min_gaussian; /* Minimum value the gaussian must have so that
+ the force is actually evaluated */
+ real eps; /* Additive constant for radial motion2 and
+ flexible2 potentials (nm^2) */
+ gmx_enfrotgrp_t enfrotgrp; /* Stores non-inputrec rotation data per group */
+} t_rotgrp;
+
+typedef struct {
+ int ngrp; /* Number of rotation groups */
+ int nstrout; /* Output frequency for main rotation outfile */
+ int nstsout; /* Output frequency for per-slab data */
+ t_rotgrp *grp; /* Groups to rotate */
+ gmx_enfrot_t enfrot; /* Stores non-inputrec enforced rotation data */
+} t_rot;
+
+
+typedef struct {
+ int type; /* type of AdResS simulation */
+ gmx_bool bnew_wf; /* enable new AdResS weighting function */
+ gmx_bool bchempot_dx; /*true:interaction table format input is F=-dmu/dx false: dmu_dwp */
+ gmx_bool btf_full_box; /* true: appy therm force everywhere in the box according to table false: only in hybrid region */
+ real const_wf; /* value of weighting function for eAdressConst */
+ real ex_width; /* center of the explicit zone */
+ real hy_width; /* width of the hybrid zone */
+ int icor; /* type of interface correction */
+ int site; /* AdResS CG site location */
+ rvec refs; /* Coordinates for AdResS reference */
+ real ex_forcecap; /* in the hybrid zone, cap forces large then this to adress_ex_forcecap */
+ gmx_bool do_hybridpairs; /* If true pair interaction forces are also scaled in an adress way*/
+
+ int * tf_table_index; /* contains mapping of energy group index -> i-th adress tf table*/
+ int n_tf_grps;
+ int *group_explicit;
+ int n_energy_grps;
+} t_adress;
+
+typedef struct {
+ int eI; /* Integration method */
+ gmx_large_int_t nsteps; /* number of steps to be taken */
+ int simulation_part; /* Used in checkpointing to separate chunks */
+ gmx_large_int_t init_step; /* start at a stepcount >0 (used w. tpbconv) */
+ int nstcalcenergy; /* frequency of energy calc. and T/P coupl. upd. */
+ int cutoff_scheme; /* group or verlet cutoffs */
+ int ns_type; /* which ns method should we use? */
+ int nstlist; /* number of steps before pairlist is generated */
+ int ndelta; /* number of cells per rlong */
+ int nstcomm; /* number of steps after which center of mass */
+ /* motion is removed */
+ int comm_mode; /* Center of mass motion removal algorithm */
+ int nstcheckpoint; /* checkpointing frequency */
+ int nstlog; /* number of steps after which print to logfile */
+ int nstxout; /* number of steps after which X is output */
+ int nstvout; /* id. for V */
+ int nstfout; /* id. for F */
+ int nstenergy; /* number of steps after which energies printed */
+ int nstxtcout; /* id. for compressed trj (.xtc) */
+ double init_t; /* initial time (ps) */
+ double delta_t; /* time step (ps) */
+ real xtcprec; /* precision of xtc file */
+ real fourier_spacing; /* requested fourier_spacing, when nk? not set */
+ int nkx, nky, nkz; /* number of k vectors in each spatial dimension*/
+ /* for fourier methods for long range electrost.*/
+ int pme_order; /* interpolation order for PME */
+ real ewald_rtol; /* Real space tolerance for Ewald, determines */
+ /* the real/reciprocal space relative weight */
+ int ewald_geometry; /* normal/3d ewald, or pseudo-2d LR corrections */
+ real epsilon_surface; /* Epsilon for PME dipole correction */
+ gmx_bool bOptFFT; /* optimize the fft plan at start */
+ int ePBC; /* Type of periodic boundary conditions */
+ int bPeriodicMols; /* Periodic molecules */
+ gmx_bool bContinuation; /* Continuation run: starting state is correct */
+ int etc; /* temperature coupling */
+ int nsttcouple; /* interval in steps for temperature coupling */
+ gmx_bool bPrintNHChains; /* whether to print nose-hoover chains */
+ int epc; /* pressure coupling */
+ int epct; /* pressure coupling type */
+ int nstpcouple; /* interval in steps for pressure coupling */
+ real tau_p; /* pressure coupling time (ps) */
+ tensor ref_p; /* reference pressure (kJ/(mol nm^3)) */
+ tensor compress; /* compressability ((mol nm^3)/kJ) */
+ int refcoord_scaling; /* How to scale absolute reference coordinates */
+ rvec posres_com; /* The COM of the posres atoms */
+ rvec posres_comB; /* The B-state COM of the posres atoms */
+ int andersen_seed; /* Random seed for Andersen thermostat (obsolete) */
+ real verletbuf_drift; /* Max. drift (kJ/mol/ps/atom) for list buffer */
+ real rlist; /* short range pairlist cut-off (nm) */
+ real rlistlong; /* long range pairlist cut-off (nm) */
+ int nstcalclr; /* Frequency of evaluating direct space long-range interactions */
+ real rtpi; /* Radius for test particle insertion */
+ int coulombtype; /* Type of electrostatics treatment */
+ int coulomb_modifier; /* Modify the Coulomb interaction */
+ real rcoulomb_switch; /* Coulomb switch range start (nm) */
+ real rcoulomb; /* Coulomb cutoff (nm) */
+ real epsilon_r; /* relative dielectric constant */
+ real epsilon_rf; /* relative dielectric constant of the RF */
+ int implicit_solvent; /* No (=explicit water), or GBSA solvent models */
+ int gb_algorithm; /* Algorithm to use for calculation Born radii */
+ int nstgbradii; /* Frequency of updating Generalized Born radii */
+ real rgbradii; /* Cutoff for GB radii calculation */
+ real gb_saltconc; /* Salt concentration (M) for GBSA models */
+ real gb_epsilon_solvent; /* dielectric coeff. of implicit solvent */
+ real gb_obc_alpha; /* 1st scaling factor for Bashford-Case GB */
+ real gb_obc_beta; /* 2nd scaling factor for Bashford-Case GB */
+ real gb_obc_gamma; /* 3rd scaling factor for Bashford-Case GB */
+ real gb_dielectric_offset; /* Dielectric offset for Still/HCT/OBC */
+ int sa_algorithm; /* Algorithm for SA part of GBSA */
+ real sa_surface_tension; /* Energy factor for SA part of GBSA */
+ int vdwtype; /* Type of Van der Waals treatment */
+ int vdw_modifier; /* Modify the VdW interaction */
+ real rvdw_switch; /* Van der Waals switch range start (nm) */
+ real rvdw; /* Van der Waals cutoff (nm) */
+ int eDispCorr; /* Perform Long range dispersion corrections */
+ real tabext; /* Extension of the table beyond the cut-off, *
+ * as well as the table length for 1-4 interac. */
+ real shake_tol; /* tolerance for shake */
+ int efep; /* free energy calculations */
+ t_lambda *fepvals; /* Data for the FEP state */
+ gmx_bool bSimTemp; /* Whether to do simulated tempering */
+ t_simtemp *simtempvals; /* Variables for simulated tempering */
+ gmx_bool bExpanded; /* Whether expanded ensembles are used */
+ t_expanded *expandedvals; /* Expanded ensemble parameters */
+ int eDisre; /* Type of distance restraining */
+ real dr_fc; /* force constant for ta_disre */
+ int eDisreWeighting; /* type of weighting of pairs in one restraints */
+ gmx_bool bDisreMixed; /* Use comb of time averaged and instan. viol's */
+ int nstdisreout; /* frequency of writing pair distances to enx */
+ real dr_tau; /* time constant for memory function in disres */
+ real orires_fc; /* force constant for orientational restraints */
+ real orires_tau; /* time constant for memory function in orires */
+ int nstorireout; /* frequency of writing tr(SD) to enx */
+ real dihre_fc; /* force constant for dihedral restraints (obsolete) */
+ real em_stepsize; /* The stepsize for updating */
+ real em_tol; /* The tolerance */
+ int niter; /* Number of iterations for convergence of */
+ /* steepest descent in relax_shells */
+ real fc_stepsize; /* Stepsize for directional minimization */
+ /* in relax_shells */
+ int nstcgsteep; /* number of steps after which a steepest */
+ /* descents step is done while doing cg */
+ int nbfgscorr; /* Number of corrections to the hessian to keep */
+ int eConstrAlg; /* Type of constraint algorithm */
+ int nProjOrder; /* Order of the LINCS Projection Algorithm */
+ real LincsWarnAngle; /* If bond rotates more than %g degrees, warn */
+ int nLincsIter; /* Number of iterations in the final Lincs step */
+ gmx_bool bShakeSOR; /* Use successive overrelaxation for shake */
+ real bd_fric; /* Friction coefficient for BD (amu/ps) */
+ int ld_seed; /* Random seed for SD and BD */
+ int nwall; /* The number of walls */
+ int wall_type; /* The type of walls */
+ real wall_r_linpot; /* The potentail is linear for r<=wall_r_linpot */
+ int wall_atomtype[2]; /* The atom type for walls */
+ real wall_density[2]; /* Number density for walls */
+ real wall_ewald_zfac; /* Scaling factor for the box for Ewald */
+ int ePull; /* Type of pulling: no, umbrella or constraint */
+ t_pull *pull; /* The data for center of mass pulling */
+ gmx_bool bRot; /* Calculate enforced rotation potential(s)? */
+ t_rot *rot; /* The data for enforced rotation potentials */
+ real cos_accel; /* Acceleration for viscosity calculation */
+ tensor deform; /* Triclinic deformation velocities (nm/ps) */
+ int userint1; /* User determined parameters */
+ int userint2;
+ int userint3;
+ int userint4;
+ real userreal1;
+ real userreal2;
+ real userreal3;
+ real userreal4;
+ t_grpopts opts; /* Group options */
+ t_cosines ex[DIM]; /* Electric field stuff (spatial part) */
+ t_cosines et[DIM]; /* Electric field stuff (time part) */
+ gmx_bool bQMMM; /* QM/MM calculation */
+ int QMconstraints; /* constraints on QM bonds */
+ int QMMMscheme; /* Scheme: ONIOM or normal */
+ real scalefactor; /* factor for scaling the MM charges in QM calc.*/
+ /* parameter needed for AdResS simulation */
+ gmx_bool bAdress; /* Is AdResS enabled ? */
+ t_adress *adress; /* The data for adress simulations */
+} t_inputrec;
+
+#define DEFORM(ir) ((ir).deform[XX][XX] != 0 || (ir).deform[YY][YY] != 0 || (ir).deform[ZZ][ZZ] != 0 || (ir).deform[YY][XX] != 0 || (ir).deform[ZZ][XX] != 0 || (ir).deform[ZZ][YY] != 0)
+
+#define DYNAMIC_BOX(ir) ((ir).epc != epcNO || (ir).eI == eiTPI || DEFORM(ir))
+
+#define PRESERVE_SHAPE(ir) ((ir).epc != epcNO && (ir).deform[XX][XX] == 0 && ((ir).epct == epctISOTROPIC || (ir).epct == epctSEMIISOTROPIC))
+
+#define NEED_MUTOT(ir) (((ir).coulombtype == eelEWALD || EEL_PME((ir).coulombtype)) && ((ir).ewald_geometry == eewg3DC || (ir).epsilon_surface != 0))
+
+#define IR_TWINRANGE(ir) ((ir).rlist > 0 && ((ir).rlistlong == 0 || (ir).rlistlong > (ir).rlist))
+
+#define IR_ELEC_FIELD(ir) ((ir).ex[XX].n > 0 || (ir).ex[YY].n > 0 || (ir).ex[ZZ].n > 0)
+
+#define IR_EXCL_FORCES(ir) (EEL_FULL((ir).coulombtype) || (EEL_RF((ir).coulombtype) && (ir).coulombtype != eelRF_NEC) || (ir).implicit_solvent != eisNO)
+/* use pointer definitions of ir here, since that's what's usually used in the code */
+#define IR_NPT_TROTTER(ir) ((((ir)->eI == eiVV) || ((ir)->eI == eiVVAK)) && (((ir)->epc == epcMTTK) && ((ir)->etc == etcNOSEHOOVER)))
+
+#define IR_NVT_TROTTER(ir) ((((ir)->eI == eiVV) || ((ir)->eI == eiVVAK)) && ((!((ir)->epc == epcMTTK)) && ((ir)->etc == etcNOSEHOOVER)))
+
+#define IR_NPH_TROTTER(ir) ((((ir)->eI == eiVV) || ((ir)->eI == eiVVAK)) && (((ir)->epc == epcMTTK) && (!(((ir)->etc == etcNOSEHOOVER)))))
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif
--- /dev/null
- #ifdef GMX_X86_SSE2
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2012, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * Gallium Rubidium Oxygen Manganese Argon Carbon Silicon
+ */
+
+#ifndef NB_VERLET_H
+#define NB_VERLET_H
+
+#include "nbnxn_pairlist.h"
+#include "nbnxn_cuda_types_ext.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+/* For testing the reference plain-C SIMD kernels, uncomment the next lines,
+ * as well as the GMX_SIMD_REFERENCE_PLAIN_C define in gmx_simd_macros.h
+ * The actual SIMD width is set in gmx_simd_macros.h
+ * The 4xN reference kernels support 2-, 4- and 8-way SIMD.
+ * The 2x(N+N) reference kernels support 8- and 16-way SIMD.
+ */
+/* #define GMX_NBNXN_SIMD */
+/* #define GMX_NBNXN_SIMD_4XN */
+/* #define GMX_NBNXN_SIMD_2XNN */
+
+
++#if (defined GMX_X86_SSE2) || (defined GMX_CPU_ACCELERATION_IBM_QPX)
+/* Use SIMD accelerated nbnxn search and kernels */
+#define GMX_NBNXN_SIMD
+
+/* Uncomment the next line to use, slower, 128-bit SIMD with AVX-256 */
+/* #define GMX_NBNXN_HALF_WIDTH_SIMD */
+
+/* The nbnxn SIMD 4xN and 2x(N+N) kernels can be added independently.
+ * Currently the 2xNN SIMD kernels only make sense with:
+ * 8-way SIMD: 4x4 setup, works with AVX-256 in single precision
+ * 16-way SIMD: 4x8 setup, not used, but most of the kernel code is there
+ */
+#define GMX_NBNXN_SIMD_4XN
+#if defined GMX_X86_AVX_256 && !(defined GMX_DOUBLE || defined GMX_NBNXN_HALF_WIDTH_SIMD)
+#define GMX_NBNXN_SIMD_2XNN
+#endif
+
+#endif
+
+
+/*! Nonbonded NxN kernel types: plain C, CPU SIMD, GPU CUDA, GPU emulation */
+typedef enum
+{
+ nbnxnkNotSet = 0,
+ nbnxnk4x4_PlainC,
+ nbnxnk4xN_SIMD_4xN,
+ nbnxnk4xN_SIMD_2xNN,
+ nbnxnk8x8x8_CUDA,
+ nbnxnk8x8x8_PlainC,
+ nbnxnkNR
+} nbnxn_kernel_type;
+
+/*! Return a string indentifying the kernel type */
+const char *lookup_nbnxn_kernel_name(int kernel_type);
+
+enum {
+ ewaldexclTable, ewaldexclAnalytical
+};
+
+/* Atom locality indicator: local, non-local, all, used for calls to:
+ gridding, pair-search, force calculation, x/f buffer operations */
+enum {
+ eatLocal = 0, eatNonlocal = 1, eatAll
+};
+
+#define LOCAL_A(x) ((x) == eatLocal)
+#define NONLOCAL_A(x) ((x) == eatNonlocal)
+#define LOCAL_OR_NONLOCAL_A(x) (LOCAL_A(x) || NONLOCAL_A(x))
+
+/* Interaction locality indicator (used in pair-list search/calculations):
+ - local interactions require local atom data and affect local output only;
+ - non-local interactions require both local and non-local atom data and
+ affect both local- and non-local output. */
+enum {
+ eintLocal = 0, eintNonlocal = 1
+};
+
+#define LOCAL_I(x) ((x) == eintLocal)
+#define NONLOCAL_I(x) ((x) == eintNonlocal)
+
+enum {
+ enbvClearFNo, enbvClearFYes
+};
+
+typedef struct {
+ nbnxn_pairlist_set_t nbl_lists; /* pair list(s) */
+ nbnxn_atomdata_t *nbat; /* atom data */
+ int kernel_type; /* non-bonded kernel - see enum above */
+ int ewald_excl; /* Ewald exclusion - see enum above */
+} nonbonded_verlet_group_t;
+
+/* non-bonded data structure with Verlet-type cut-off */
+typedef struct {
+ nbnxn_search_t nbs; /* n vs n atom pair searching data */
+ int ngrp; /* number of interaction groups */
+ nonbonded_verlet_group_t grp[2]; /* local and non-local interaction group */
+
+ gmx_bool bUseGPU; /* TRUE when GPU acceleration is used */
+ nbnxn_cuda_ptr_t cu_nbv; /* pointer to CUDA nb verlet data */
+ int min_ci_balanced; /* pair list balancing parameter
+ used for the 8x8x8 CUDA kernels */
+} nonbonded_verlet_t;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* NB_VERLET_H */
--- /dev/null
- int cj; /* The j-cluster */
- unsigned excl; /* The topology exclusion (interaction) bits */
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2012, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * Gallium Rubidium Oxygen Manganese Argon Carbon Silicon
+ */
+
+#ifndef _nbnxn_pairlist_h
+#define _nbnxn_pairlist_h
+
++#ifdef HAVE_CONFIG_H
++# include <config.h>
++#endif
++
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* A buffer data structure of 64 bytes
+ * to be placed at the beginning and end of structs
+ * to avoid cache invalidation of the real contents
+ * of the struct by writes to neighboring memory.
+ */
+typedef struct {
+ int dummy[16];
+} gmx_cache_protect_t;
+
+/* Abstract type for pair searching data */
+typedef struct nbnxn_search * nbnxn_search_t;
+
+/* Function that should return a pointer *ptr to memory
+ * of size nbytes.
+ * Error handling should be done within this function.
+ */
+typedef void nbnxn_alloc_t (void **ptr, size_t nbytes);
+
+/* Function that should free the memory pointed to by *ptr.
+ * NULL should not be passed to this function.
+ */
+typedef void nbnxn_free_t (void *ptr);
+
+/* This is the actual cluster-pair list j-entry.
+ * cj is the j-cluster.
+ * The interaction bits in excl are indexed i-major, j-minor.
+ * The cj entries are sorted such that ones with exclusions come first.
+ * This means that once a full mask (=NBNXN_INTERACTION_MASK_ALL)
+ * is found, all subsequent j-entries in the i-entry also have full masks.
+ */
+typedef struct {
-
- int nout; /* The number of force arrays */
- nbnxn_atomdata_output_t *out; /* Output data structures */
- int nalloc; /* Allocation size of all arrays (for x/f *x/fstride) */
- gmx_bool bUseBufferFlags; /* Use the flags or operate on all atoms */
- nbnxn_buffer_flags_t buffer_flags; /* Flags for buffer zeroing+reduc. */
++ int cj; /* The j-cluster */
++ unsigned excl; /* The exclusion (interaction) bits */
++#ifdef GMX_CPU_ACCELERATION_IBM_QPX
++ /* Indices into the arrays of SIMD interaction masks. */
++ char interaction_mask_indices[4];
++#endif
+} nbnxn_cj_t;
+
+/* In nbnxn_ci_t the integer shift contains the shift in the lower 7 bits.
+ * The upper bits contain information for non-bonded kernel optimization.
+ * Simply calculating LJ and Coulomb for all pairs in a cluster pair is fine.
+ * But three flags can be used to skip interactions, currently only for subc=0
+ * !(shift & NBNXN_CI_DO_LJ(subc)) => we can skip LJ for all pairs
+ * shift & NBNXN_CI_HALF_LJ(subc) => we can skip LJ for the second half of i
+ * !(shift & NBNXN_CI_DO_COUL(subc)) => we can skip Coulomb for all pairs
+ */
+#define NBNXN_CI_SHIFT 127
+#define NBNXN_CI_DO_LJ(subc) (1<<(7+3*(subc)))
+#define NBNXN_CI_HALF_LJ(subc) (1<<(8+3*(subc)))
+#define NBNXN_CI_DO_COUL(subc) (1<<(9+3*(subc)))
+
+/* Simple pair-list i-unit */
+typedef struct {
+ int ci; /* i-cluster */
+ int shift; /* Shift vector index plus possible flags, see above */
+ int cj_ind_start; /* Start index into cj */
+ int cj_ind_end; /* End index into cj */
+} nbnxn_ci_t;
+
+/* Grouped pair-list i-unit */
+typedef struct {
+ int sci; /* i-super-cluster */
+ int shift; /* Shift vector index plus possible flags */
+ int cj4_ind_start; /* Start index into cj4 */
+ int cj4_ind_end; /* End index into cj4 */
+} nbnxn_sci_t;
+
+typedef struct {
+ unsigned imask; /* The i-cluster interactions mask for 1 warp */
+ int excl_ind; /* Index into the exclusion array for 1 warp */
+} nbnxn_im_ei_t;
+
+typedef struct {
+ int cj[4]; /* The 4 j-clusters */
+ nbnxn_im_ei_t imei[2]; /* The i-cluster mask data for 2 warps */
+} nbnxn_cj4_t;
+
+typedef struct {
+ unsigned pair[32]; /* Topology exclusion interaction bits for one warp,
+ * each unsigned has bitS for 4*8 i clusters
+ */
+} nbnxn_excl_t;
+
+typedef struct {
+ gmx_cache_protect_t cp0;
+
+ nbnxn_alloc_t *alloc;
+ nbnxn_free_t *free;
+
+ gmx_bool bSimple; /* Simple list has na_sc=na_s and uses cj *
+ * Complex list uses cj4 */
+
+ int na_ci; /* The number of atoms per i-cluster */
+ int na_cj; /* The number of atoms per j-cluster */
+ int na_sc; /* The number of atoms per super cluster */
+ real rlist; /* The radius for constructing the list */
+ int nci; /* The number of i-clusters in the list */
+ nbnxn_ci_t *ci; /* The i-cluster list, size nci */
+ int ci_nalloc; /* The allocation size of ci */
+ int nsci; /* The number of i-super-clusters in the list */
+ nbnxn_sci_t *sci; /* The i-super-cluster list */
+ int sci_nalloc; /* The allocation size of sci */
+
+ int ncj; /* The number of j-clusters in the list */
+ nbnxn_cj_t *cj; /* The j-cluster list, size ncj */
+ int cj_nalloc; /* The allocation size of cj */
+
+ int ncj4; /* The total number of 4*j clusters */
+ nbnxn_cj4_t *cj4; /* The 4*j cluster list, size ncj4 */
+ int cj4_nalloc; /* The allocation size of cj4 */
+ int nexcl; /* The count for excl */
+ nbnxn_excl_t *excl; /* Atom interaction bits (non-exclusions) */
+ int excl_nalloc; /* The allocation size for excl */
+ int nci_tot; /* The total number of i clusters */
+
+ struct nbnxn_list_work *work;
+
+ gmx_cache_protect_t cp1;
+} nbnxn_pairlist_t;
+
+typedef struct {
+ int nnbl; /* number of lists */
+ nbnxn_pairlist_t **nbl; /* lists */
+ gmx_bool bCombined; /* TRUE if lists get combined into one (the 1st) */
+ gmx_bool bSimple; /* TRUE if the list of of type "simple"
+ (na_sc=na_s, no super-clusters used) */
+ int natpair_ljq; /* Total number of atom pairs for LJ+Q kernel */
+ int natpair_lj; /* Total number of atom pairs for LJ kernel */
+ int natpair_q; /* Total number of atom pairs for Q kernel */
+} nbnxn_pairlist_set_t;
+
+enum {
+ nbatXYZ, nbatXYZQ, nbatX4, nbatX8
+};
+
+typedef struct {
+ real *f; /* f, size natoms*fstride */
+ real *fshift; /* Shift force array, size SHIFTS*DIM */
+ int nV; /* The size of *Vvdw and *Vc */
+ real *Vvdw; /* Temporary Van der Waals group energy storage */
+ real *Vc; /* Temporary Coulomb group energy storage */
+ int nVS; /* The size of *VSvdw and *VSc */
+ real *VSvdw; /* Temporary SIMD Van der Waals group energy storage */
+ real *VSc; /* Temporary SIMD Coulomb group energy storage */
+} nbnxn_atomdata_output_t;
+
+/* Block size in atoms for the non-bonded thread force-buffer reduction,
+ * should be a multiple of all cell and x86 SIMD sizes (i.e. 2, 4 and 8).
+ * Should be small to reduce the reduction and zeroing cost,
+ * but too small will result in overhead.
+ * Currently the block size is NBNXN_BUFFERFLAG_SIZE*3*sizeof(real)=192 bytes.
+ */
+#ifdef GMX_DOUBLE
+#define NBNXN_BUFFERFLAG_SIZE 8
+#else
+#define NBNXN_BUFFERFLAG_SIZE 16
+#endif
+
+/* We currently store the reduction flags as bits in an unsigned int.
+ * In most cases this limits the number of flags to 32.
+ * The reduction will automatically disable the flagging and do a full
+ * reduction when the flags won't fit, but this will lead to very slow
+ * reduction. As we anyhow don't expect reasonable performance with
+ * more than 32 threads, we put in this hard limit.
+ * You can increase this number, but the reduction will be very slow.
+ */
+#define NBNXN_BUFFERFLAG_MAX_THREADS 32
+
+/* Flags for telling if threads write to force output buffers */
+typedef struct {
+ int nflag; /* The number of flag blocks */
+ unsigned *flag; /* Bit i is set when thread i writes to a cell-block */
+ int flag_nalloc; /* Allocation size of cxy_flag */
+} nbnxn_buffer_flags_t;
+
+/* LJ combination rules: geometric, Lorentz-Berthelot, none */
+enum {
+ ljcrGEOM, ljcrLB, ljcrNONE, ljcrNR
+};
+
+typedef struct {
+ nbnxn_alloc_t *alloc;
+ nbnxn_free_t *free;
+ int ntype; /* The number of different atom types */
+ real *nbfp; /* Lennard-Jones 6*C6 and 12*C12 params, size ntype^2*2 */
+ int comb_rule; /* Combination rule, see enum above */
+ real *nbfp_comb; /* LJ parameter per atom type, size ntype*2 */
+ real *nbfp_s4; /* As nbfp, but with stride 4, size ntype^2*4. This
+ * might suit 4-wide SIMD loads of two values (e.g.
+ * two floats in single precision on x86). */
+ int natoms; /* Number of atoms */
+ int natoms_local; /* Number of local atoms */
+ int *type; /* Atom types */
+ real *lj_comb; /* LJ parameters per atom for combining for pairs */
+ int XFormat; /* The format of x (and q), enum */
+ int FFormat; /* The format of f, enum */
+ real *q; /* Charges, can be NULL if incorporated in x */
+ int na_c; /* The number of atoms per cluster */
+ int nenergrp; /* The number of energy groups */
+ int neg_2log; /* Log2 of nenergrp */
+ int *energrp; /* The energy groups per cluster, can be NULL */
+ gmx_bool bDynamicBox; /* Do we need to update shift_vec every step? */
+ rvec *shift_vec; /* Shift vectors, copied from t_forcerec */
+ int xstride; /* stride for a coordinate in x (usually 3 or 4) */
+ int fstride; /* stride for a coordinate in f (usually 3 or 4) */
+ real *x; /* x and possibly q, size natoms*xstride */
+
+ /* j-atom minus i-atom index for generating self and Newton exclusions
+ * cluster-cluster pairs of the diagonal, for 4xn and 2xnn kernels.
+ */
+ real *simd_4xn_diagonal_j_minus_i;
+ real *simd_2xnn_diagonal_j_minus_i;
+ /* Filters for topology exclusion masks for the SIMD kernels.
+ * filter2 is the same as filter1, but with each element duplicated.
+ */
+ unsigned *simd_exclusion_filter1;
+ unsigned *simd_exclusion_filter2;
++#ifdef GMX_CPU_ACCELERATION_IBM_QPX
++ real *simd_interaction_array; /* Array of masks needed for exclusions on QPX */
++#endif
++ int nout; /* The number of force arrays */
++ nbnxn_atomdata_output_t *out; /* Output data structures */
++ int nalloc; /* Allocation size of all arrays (for x/f *x/fstride) */
++ gmx_bool bUseBufferFlags; /* Use the flags or operate on all atoms */
++ nbnxn_buffer_flags_t buffer_flags; /* Flags for buffer zeroing+reduc. */
+} nbnxn_atomdata_t;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
--- /dev/null
- gmx_bool bEquil; /* reached equilibration */
+/*
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * VERSION 3.2.0
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2004, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * GRoups of Organic Molecules in ACtion for Science
+ */
+#ifndef _state_h_
+#define _state_h_
+
+
+#include "simple.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * The t_state struct should contain all the (possibly) non-static
+ * information required to define the state of the system.
+ * Currently the random seeds for SD and BD are missing.
+ */
+
+/* for now, define the length of the NH chains here */
+#define NHCHAINLENGTH 10
+#define MAXLAMBDAS 1024
+
+/* These enums are used in flags as (1<<est...).
+ * The order of these enums should not be changed,
+ * since that affects the checkpoint (.cpt) file format.
+ */
+enum {
+ estLAMBDA,
+ estBOX, estBOX_REL, estBOXV, estPRES_PREV, estNH_XI, estTC_INT,
+ estX, estV, estSDX, estCGP, estLD_RNG, estLD_RNGI,
+ estDISRE_INITF, estDISRE_RM3TAV,
+ estORIRE_INITF, estORIRE_DTAV,
+ estSVIR_PREV, estNH_VXI, estVETA, estVOL0, estNHPRES_XI, estNHPRES_VXI, estFVIR_PREV,
+ estFEPSTATE, estMC_RNG, estMC_RNGI,
+ estNR
+};
+
+#define EST_DISTR(e) (!(((e) >= estLAMBDA && (e) <= estTC_INT) || ((e) >= estSVIR_PREV && (e) <= estMC_RNGI)))
+
+/* The names of the state entries, defined in src/gmxlib/checkpoint.c */
+extern const char *est_names[estNR];
+
+typedef struct
+{
+ real disre_initf; /* The scaling factor for initializing the time av. */
+ int ndisrepairs; /* The number of distance restraints */
+ real *disre_rm3tav; /* The r^-3 time averaged pair distances */
+ real orire_initf; /* The scaling factor for initializing the time av. */
+ int norire_Dtav; /* The number of matrix element in dtav (npair*5) */
+ real *orire_Dtav; /* The time averaged orientation tensors */
+} history_t;
+
+/* Struct used for checkpointing only.
+ * This struct would not be required with unlimited precision.
+ * But because of limited precision, the COM motion removal implementation
+ * can cause the kinetic energy in the MD loop to differ by a few bits from
+ * the kinetic energy one would determine from state.v.
+ */
+typedef struct
+{
+ gmx_bool bUpToDate;
+ int ekin_n;
+ tensor *ekinh;
+ tensor *ekinf;
+ tensor *ekinh_old;
+ tensor ekin_total;
+ double *ekinscalef_nhc;
+ double *ekinscaleh_nhc;
+ double *vscale_nhc;
+ real dekindl;
+ real mvcos;
+} ekinstate_t;
+
+/* energy history for delta_h histograms */
+typedef struct
+{
+ int nndh; /* the number of energy difference lists */
+ int *ndh; /* the number in each energy difference list */
+ real **dh; /* the energy difference lists */
+
+ double start_time; /* the start time of these energy diff blocks */
+ double start_lambda; /* lambda at start time */
+
+ gmx_bool start_lambda_set; /* whether the lambda value is set. Here
+ For backward-compatibility. */
+} delta_h_history_t;
+
+typedef struct
+{
+ int nlambda; /* total number of lambda states - for history*/
+
++ gmx_bool bEquil; /* Have we reached equilibration */
+ int *n_at_lam; /* number of points observed at each lambda */
+ real *wl_histo; /* histogram for WL flatness determination */
+ real wl_delta; /* current wang-landau delta */
+
+ real *sum_weights; /* weights of the states */
+ real *sum_dg; /* free energies of the states -- not actually used for weighting, but informational */
+ real *sum_minvar; /* corrections to weights for minimum variance */
+ real *sum_variance; /* variances of the states */
+
+ real **accum_p; /* accumulated bennett weights for n+1 */
+ real **accum_m; /* accumulated bennett weights for n-1 */
+ real **accum_p2; /* accumulated squared bennett weights for n+1 */
+ real **accum_m2; /* accumulated squared bennett weights for n-1 */
+
+ real **Tij; /* transition matrix */
+ real **Tij_empirical; /* Empirical transition matrix */
++
+} df_history_t;
+
+typedef struct
+{
+ gmx_large_int_t nsteps; /* The number of steps in the history */
+ gmx_large_int_t nsum; /* The nr. of steps in the ener_ave and ener_sum */
+ double * ener_ave; /* Energy term history sum to get fluctuations */
+ double * ener_sum; /* Energy term history sum to get fluctuations */
+ int nener; /* Number of energy terms in two previous arrays */
+ gmx_large_int_t nsteps_sim; /* The number of steps in ener_sum_sim */
+ gmx_large_int_t nsum_sim; /* The number of frames in ener_sum_sim */
+ double * ener_sum_sim; /* Energy term history sum of the whole sim */
+
+ delta_h_history_t *dht; /* The BAR energy differences */
+}
+energyhistory_t;
+
+typedef struct
+{
+ /* If one uses essential dynamics or flooding on a group of atoms from
+ * more than one molecule, we cannot make this group whole with
+ * do_pbc_first_mtop(). We assume that the ED group has the correct PBC
+ * representation at the beginning of the simulation and keep track
+ * of the shifts to always get it into that representation.
+ * For proper restarts from a checkpoint we store the positions of the
+ * reference group at the time of checkpoint writing */
+ gmx_bool bFromCpt; /* Did we start from a checkpoint file? */
+ int nED; /* No. of ED/Flooding data sets, if <1 no ED */
+ int *nref; /* No. of atoms in i'th reference structure */
+ int *nav; /* Same for average structure */
+ rvec **old_sref; /* Positions of the reference atoms
+ at the last time step (with correct PBC
+ representation) */
+ rvec **old_sref_p; /* Pointer to these positions */
+ rvec **old_sav; /* Same for the average positions */
+ rvec **old_sav_p;
+}
+edsamstate_t;
+
+typedef struct
+{
+ int natoms;
+ int ngtc;
+ int nnhpres;
+ int nhchainlength; /* number of nose-hoover chains */
+ int nrng;
+ int nrngi;
+ int flags; /* Flags telling which entries are present */
+ int fep_state; /* indicates which of the alchemical states we are in */
+ real *lambda; /* lambda vector */
+ matrix box; /* box vector coordinates */
+ matrix box_rel; /* Relitaive box vectors to preserve shape */
+ matrix boxv; /* box velocitites for Parrinello-Rahman pcoupl */
+ matrix pres_prev; /* Pressure of the previous step for pcoupl */
+ matrix svir_prev; /* Shake virial for previous step for pcoupl */
+ matrix fvir_prev; /* Force virial of the previous step for pcoupl */
+ double *nosehoover_xi; /* for Nose-Hoover tcoupl (ngtc) */
+ double *nosehoover_vxi; /* for N-H tcoupl (ngtc) */
+ double *nhpres_xi; /* for Nose-Hoover pcoupl for barostat */
+ double *nhpres_vxi; /* for Nose-Hoover pcoupl for barostat */
+ double *therm_integral; /* for N-H/V-rescale tcoupl (ngtc) */
+ real veta; /* trotter based isotropic P-coupling */
+ real vol0; /* initial volume,required for computing NPT conserverd quantity */
+ int nalloc; /* Allocation size for x, v and sd_x when !=NULL*/
+ rvec *x; /* the coordinates (natoms) */
+ rvec *v; /* the velocities (natoms) */
+ rvec *sd_X; /* random part of the x update for stoch. dyn. */
+ rvec *cg_p; /* p vector for conjugate gradient minimization */
+
+ unsigned int *ld_rng; /* RNG random state */
+ int *ld_rngi; /* RNG index */
+
+ int nmcrng; /* number of RNG states */
+ unsigned int *mc_rng; /* lambda MC RNG random state */
+ int *mc_rngi; /* lambda MC RNG index */
+
+ history_t hist; /* Time history for restraints */
+
+ ekinstate_t ekinstate; /* The state of the kinetic energy data */
+
+ energyhistory_t enerhist; /* Energy history for statistics */
+ df_history_t dfhist; /*Free energy history for free energy analysis */
+ edsamstate_t edsamstate; /* Essential dynamics / flooding history */
+
+ int ddp_count; /* The DD partitioning count for this state */
+ int ddp_count_cg_gl; /* The DD part. count for index_gl */
+ int ncg_gl; /* The number of local charge groups */
+ int *cg_gl; /* The global cg number of the local cgs */
+ int cg_gl_nalloc; /* Allocation size of cg_gl; */
+} t_state;
+
+typedef struct
+{
+ double *Qinv; /* inverse mass of thermostat -- computed from inputs, but a good place to store */
+ double *QPinv; /* inverse mass of thermostat for barostat -- computed from inputs, but a good place to store */
+ double Winv; /* Pressure mass inverse -- computed, not input, but a good place to store. Need to make a matrix later */
+ tensor Winvm; /* inverse pressure mass tensor, computed */
+} t_extmass;
+
+
+typedef struct
+{
+ real veta;
+ double rscale;
+ double vscale;
+ double rvscale;
+ double alpha;
+ double *vscale_nhc;
+} t_vetavars;
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif /* _state_h_ */
--- /dev/null
-
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This file is part of Gromacs Copyright (c) 1991-2008
+ * David van der Spoel, Erik Lindahl, Berk Hess, University of Groningen.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the research papers on the package. Check out http://www.gromacs.org
+ *
+ * And Hey:
+ * Gnomes, ROck Monsters And Chili Sauce
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdio.h>
+#include <time.h>
+#include <math.h>
+#include <string.h>
+#include <stdlib.h>
+#include "typedefs.h"
+#include "smalloc.h"
+#include "gmx_fatal.h"
+#include "gmx_fatal_collective.h"
+#include "vec.h"
+#include "domdec.h"
+#include "domdec_network.h"
+#include "nrnb.h"
+#include "pbc.h"
+#include "chargegroup.h"
+#include "constr.h"
+#include "mdatoms.h"
+#include "names.h"
+#include "pdbio.h"
+#include "futil.h"
+#include "force.h"
+#include "pme.h"
+#include "pull.h"
+#include "pull_rotation.h"
+#include "gmx_wallcycle.h"
+#include "mdrun.h"
+#include "nsgrid.h"
+#include "shellfc.h"
+#include "mtop_util.h"
+#include "gmxfio.h"
+#include "gmx_ga2la.h"
+#include "gmx_sort.h"
+#include "macros.h"
+#include "nbnxn_search.h"
+#include "bondf.h"
+#include "gmx_omp_nthreads.h"
+
+#include "gromacs/utility/gmxmpi.h"
+
+#define DDRANK(dd, rank) (rank)
+#define DDMASTERRANK(dd) (dd->masterrank)
+
+typedef struct gmx_domdec_master
+{
+ /* The cell boundaries */
+ real **cell_x;
+ /* The global charge group division */
+ int *ncg; /* Number of home charge groups for each node */
+ int *index; /* Index of nnodes+1 into cg */
+ int *cg; /* Global charge group index */
+ int *nat; /* Number of home atoms for each node. */
+ int *ibuf; /* Buffer for communication */
+ rvec *vbuf; /* Buffer for state scattering and gathering */
+} gmx_domdec_master_t;
+
+typedef struct
+{
+ /* The numbers of charge groups to send and receive for each cell
+ * that requires communication, the last entry contains the total
+ * number of atoms that needs to be communicated.
+ */
+ int nsend[DD_MAXIZONE+2];
+ int nrecv[DD_MAXIZONE+2];
+ /* The charge groups to send */
+ int *index;
+ int nalloc;
+ /* The atom range for non-in-place communication */
+ int cell2at0[DD_MAXIZONE];
+ int cell2at1[DD_MAXIZONE];
+} gmx_domdec_ind_t;
+
+typedef struct
+{
+ int np; /* Number of grid pulses in this dimension */
+ int np_dlb; /* For dlb, for use with edlbAUTO */
+ gmx_domdec_ind_t *ind; /* The indices to communicate, size np */
+ int np_nalloc;
+ gmx_bool bInPlace; /* Can we communicate in place? */
+} gmx_domdec_comm_dim_t;
+
+typedef struct
+{
+ gmx_bool *bCellMin; /* Temp. var.: is this cell size at the limit */
+ real *cell_f; /* State var.: cell boundaries, box relative */
+ real *old_cell_f; /* Temp. var.: old cell size */
+ real *cell_f_max0; /* State var.: max lower boundary, incl neighbors */
+ real *cell_f_min1; /* State var.: min upper boundary, incl neighbors */
+ real *bound_min; /* Temp. var.: lower limit for cell boundary */
+ real *bound_max; /* Temp. var.: upper limit for cell boundary */
+ gmx_bool bLimited; /* State var.: is DLB limited in this dim and row */
+ real *buf_ncd; /* Temp. var. */
+} gmx_domdec_root_t;
+
+#define DD_NLOAD_MAX 9
+
+/* Here floats are accurate enough, since these variables
+ * only influence the load balancing, not the actual MD results.
+ */
+typedef struct
+{
+ int nload;
+ float *load;
+ float sum;
+ float max;
+ float sum_m;
+ float cvol_min;
+ float mdf;
+ float pme;
+ int flags;
+} gmx_domdec_load_t;
+
+typedef struct
+{
+ int nsc;
+ int ind_gl;
+ int ind;
+} gmx_cgsort_t;
+
+typedef struct
+{
+ gmx_cgsort_t *sort;
+ gmx_cgsort_t *sort2;
+ int sort_nalloc;
+ gmx_cgsort_t *sort_new;
+ int sort_new_nalloc;
+ int *ibuf;
+ int ibuf_nalloc;
+} gmx_domdec_sort_t;
+
+typedef struct
+{
+ rvec *v;
+ int nalloc;
+} vec_rvec_t;
+
+/* This enum determines the order of the coordinates.
+ * ddnatHOME and ddnatZONE should be first and second,
+ * the others can be ordered as wanted.
+ */
+enum {
+ ddnatHOME, ddnatZONE, ddnatVSITE, ddnatCON, ddnatNR
+};
+
+enum {
+ edlbAUTO, edlbNO, edlbYES, edlbNR
+};
+const char *edlb_names[edlbNR] = { "auto", "no", "yes" };
+
+typedef struct
+{
+ int dim; /* The dimension */
+ gmx_bool dim_match; /* Tells if DD and PME dims match */
+ int nslab; /* The number of PME slabs in this dimension */
+ real *slb_dim_f; /* Cell sizes for determining the PME comm. with SLB */
+ int *pp_min; /* The minimum pp node location, size nslab */
+ int *pp_max; /* The maximum pp node location,size nslab */
+ int maxshift; /* The maximum shift for coordinate redistribution in PME */
+} gmx_ddpme_t;
+
+typedef struct
+{
+ real min0; /* The minimum bottom of this zone */
+ real max1; /* The maximum top of this zone */
+ real min1; /* The minimum top of this zone */
+ real mch0; /* The maximum bottom communicaton height for this zone */
+ real mch1; /* The maximum top communicaton height for this zone */
+ real p1_0; /* The bottom value of the first cell in this zone */
+ real p1_1; /* The top value of the first cell in this zone */
+} gmx_ddzone_t;
+
+typedef struct
+{
+ gmx_domdec_ind_t ind;
+ int *ibuf;
+ int ibuf_nalloc;
+ vec_rvec_t vbuf;
+ int nsend;
+ int nat;
+ int nsend_zone;
+} dd_comm_setup_work_t;
+
+typedef struct gmx_domdec_comm
+{
+ /* All arrays are indexed with 0 to dd->ndim (not Cartesian indexing),
+ * unless stated otherwise.
+ */
+
+ /* The number of decomposition dimensions for PME, 0: no PME */
+ int npmedecompdim;
+ /* The number of nodes doing PME (PP/PME or only PME) */
+ int npmenodes;
+ int npmenodes_x;
+ int npmenodes_y;
+ /* The communication setup including the PME only nodes */
+ gmx_bool bCartesianPP_PME;
+ ivec ntot;
+ int cartpmedim;
+ int *pmenodes; /* size npmenodes */
+ int *ddindex2simnodeid; /* size npmenodes, only with bCartesianPP
+ * but with bCartesianPP_PME */
+ gmx_ddpme_t ddpme[2];
+
+ /* The DD particle-particle nodes only */
+ gmx_bool bCartesianPP;
+ int *ddindex2ddnodeid; /* size npmenode, only with bCartesianPP_PME */
+
+ /* The global charge groups */
+ t_block cgs_gl;
+
+ /* Should we sort the cgs */
+ int nstSortCG;
+ gmx_domdec_sort_t *sort;
+
+ /* Are there charge groups? */
+ gmx_bool bCGs;
+
+ /* Are there bonded and multi-body interactions between charge groups? */
+ gmx_bool bInterCGBondeds;
+ gmx_bool bInterCGMultiBody;
+
+ /* Data for the optional bonded interaction atom communication range */
+ gmx_bool bBondComm;
+ t_blocka *cglink;
+ char *bLocalCG;
+
+ /* The DLB option */
+ int eDLB;
+ /* Are we actually using DLB? */
+ gmx_bool bDynLoadBal;
+
+ /* Cell sizes for static load balancing, first index cartesian */
+ real **slb_frac;
+
+ /* The width of the communicated boundaries */
+ real cutoff_mbody;
+ real cutoff;
+ /* The minimum cell size (including triclinic correction) */
+ rvec cellsize_min;
+ /* For dlb, for use with edlbAUTO */
+ rvec cellsize_min_dlb;
+ /* The lower limit for the DD cell size with DLB */
+ real cellsize_limit;
+ /* Effectively no NB cut-off limit with DLB for systems without PBC? */
+ gmx_bool bVacDLBNoLimit;
+
+ /* With PME load balancing we set limits on DLB */
+ gmx_bool bPMELoadBalDLBLimits;
+ /* DLB needs to take into account that we want to allow this maximum
+ * cut-off (for PME load balancing), this could limit cell boundaries.
+ */
+ real PMELoadBal_max_cutoff;
+
+ /* tric_dir is only stored here because dd_get_ns_ranges needs it */
+ ivec tric_dir;
+ /* box0 and box_size are required with dim's without pbc and -gcom */
+ rvec box0;
+ rvec box_size;
+
+ /* The cell boundaries */
+ rvec cell_x0;
+ rvec cell_x1;
+
+ /* The old location of the cell boundaries, to check cg displacements */
+ rvec old_cell_x0;
+ rvec old_cell_x1;
+
+ /* The communication setup and charge group boundaries for the zones */
+ gmx_domdec_zones_t zones;
+
+ /* The zone limits for DD dimensions 1 and 2 (not 0), determined from
+ * cell boundaries of neighboring cells for dynamic load balancing.
+ */
+ gmx_ddzone_t zone_d1[2];
+ gmx_ddzone_t zone_d2[2][2];
+
+ /* The coordinate/force communication setup and indices */
+ gmx_domdec_comm_dim_t cd[DIM];
+ /* The maximum number of cells to communicate with in one dimension */
+ int maxpulse;
+
+ /* Which cg distribution is stored on the master node */
+ int master_cg_ddp_count;
+
+ /* The number of cg's received from the direct neighbors */
+ int zone_ncg1[DD_MAXZONE];
+
+ /* The atom counts, the range for each type t is nat[t-1] <= at < nat[t] */
+ int nat[ddnatNR];
+
+ /* Array for signalling if atoms have moved to another domain */
+ int *moved;
+ int moved_nalloc;
+
+ /* Communication buffer for general use */
+ int *buf_int;
+ int nalloc_int;
+
+ /* Communication buffer for general use */
+ vec_rvec_t vbuf;
+
+ /* Temporary storage for thread parallel communication setup */
+ int nth;
+ dd_comm_setup_work_t *dth;
+
+ /* Communication buffers only used with multiple grid pulses */
+ int *buf_int2;
+ int nalloc_int2;
+ vec_rvec_t vbuf2;
+
+ /* Communication buffers for local redistribution */
+ int **cggl_flag;
+ int cggl_flag_nalloc[DIM*2];
+ rvec **cgcm_state;
+ int cgcm_state_nalloc[DIM*2];
+
+ /* Cell sizes for dynamic load balancing */
+ gmx_domdec_root_t **root;
+ real *cell_f_row;
+ real cell_f0[DIM];
+ real cell_f1[DIM];
+ real cell_f_max0[DIM];
+ real cell_f_min1[DIM];
+
+ /* Stuff for load communication */
+ gmx_bool bRecordLoad;
+ gmx_domdec_load_t *load;
+#ifdef GMX_MPI
+ MPI_Comm *mpi_comm_load;
+#endif
+
+ /* Maximum DLB scaling per load balancing step in percent */
+ int dlb_scale_lim;
+
+ /* Cycle counters */
+ float cycl[ddCyclNr];
+ int cycl_n[ddCyclNr];
+ float cycl_max[ddCyclNr];
+ /* Flop counter (0=no,1=yes,2=with (eFlop-1)*5% noise */
+ int eFlop;
+ double flop;
+ int flop_n;
+ /* Have often have did we have load measurements */
+ int n_load_have;
+ /* Have often have we collected the load measurements */
+ int n_load_collect;
+
+ /* Statistics */
+ double sum_nat[ddnatNR-ddnatZONE];
+ int ndecomp;
+ int nload;
+ double load_step;
+ double load_sum;
+ double load_max;
+ ivec load_lim;
+ double load_mdf;
+ double load_pme;
+
+ /* The last partition step */
+ gmx_large_int_t partition_step;
+
+ /* Debugging */
+ int nstDDDump;
+ int nstDDDumpGrid;
+ int DD_debug;
+} gmx_domdec_comm_t;
+
+/* The size per charge group of the cggl_flag buffer in gmx_domdec_comm_t */
+#define DD_CGIBS 2
+
+/* The flags for the cggl_flag buffer in gmx_domdec_comm_t */
+#define DD_FLAG_NRCG 65535
+#define DD_FLAG_FW(d) (1<<(16+(d)*2))
+#define DD_FLAG_BW(d) (1<<(16+(d)*2+1))
+
+/* Zone permutation required to obtain consecutive charge groups
+ * for neighbor searching.
+ */
+static const int zone_perm[3][4] = { {0, 0, 0, 0}, {1, 0, 0, 0}, {3, 0, 1, 2} };
+
+/* dd_zo and dd_zp3/dd_zp2 are set up such that i zones with non-zero
+ * components see only j zones with that component 0.
+ */
+
+/* The DD zone order */
+static const ivec dd_zo[DD_MAXZONE] =
+{{0, 0, 0}, {1, 0, 0}, {1, 1, 0}, {0, 1, 0}, {0, 1, 1}, {0, 0, 1}, {1, 0, 1}, {1, 1, 1}};
+
+/* The 3D setup */
+#define dd_z3n 8
+#define dd_zp3n 4
+static const ivec dd_zp3[dd_zp3n] = {{0, 0, 8}, {1, 3, 6}, {2, 5, 6}, {3, 5, 7}};
+
+/* The 2D setup */
+#define dd_z2n 4
+#define dd_zp2n 2
+static const ivec dd_zp2[dd_zp2n] = {{0, 0, 4}, {1, 3, 4}};
+
+/* The 1D setup */
+#define dd_z1n 2
+#define dd_zp1n 1
+static const ivec dd_zp1[dd_zp1n] = {{0, 0, 2}};
+
+/* Factors used to avoid problems due to rounding issues */
+#define DD_CELL_MARGIN 1.0001
+#define DD_CELL_MARGIN2 1.00005
+/* Factor to account for pressure scaling during nstlist steps */
+#define DD_PRES_SCALE_MARGIN 1.02
+
+/* Allowed performance loss before we DLB or warn */
+#define DD_PERF_LOSS 0.05
+
+#define DD_CELL_F_SIZE(dd, di) ((dd)->nc[(dd)->dim[(di)]]+1+(di)*2+1+(di))
+
+/* Use separate MPI send and receive commands
+ * when nnodes <= GMX_DD_NNODES_SENDRECV.
+ * This saves memory (and some copying for small nnodes).
+ * For high parallelization scatter and gather calls are used.
+ */
+#define GMX_DD_NNODES_SENDRECV 4
+
+
+/*
+ #define dd_index(n,i) ((((i)[ZZ]*(n)[YY] + (i)[YY])*(n)[XX]) + (i)[XX])
+
+ static void index2xyz(ivec nc,int ind,ivec xyz)
+ {
+ xyz[XX] = ind % nc[XX];
+ xyz[YY] = (ind / nc[XX]) % nc[YY];
+ xyz[ZZ] = ind / (nc[YY]*nc[XX]);
+ }
+ */
+
+/* This order is required to minimize the coordinate communication in PME
+ * which uses decomposition in the x direction.
+ */
+#define dd_index(n, i) ((((i)[XX]*(n)[YY] + (i)[YY])*(n)[ZZ]) + (i)[ZZ])
+
+static void ddindex2xyz(ivec nc, int ind, ivec xyz)
+{
+ xyz[XX] = ind / (nc[YY]*nc[ZZ]);
+ xyz[YY] = (ind / nc[ZZ]) % nc[YY];
+ xyz[ZZ] = ind % nc[ZZ];
+}
+
+static int ddcoord2ddnodeid(gmx_domdec_t *dd, ivec c)
+{
+ int ddindex;
+ int ddnodeid = -1;
+
+ ddindex = dd_index(dd->nc, c);
+ if (dd->comm->bCartesianPP_PME)
+ {
+ ddnodeid = dd->comm->ddindex2ddnodeid[ddindex];
+ }
+ else if (dd->comm->bCartesianPP)
+ {
+#ifdef GMX_MPI
+ MPI_Cart_rank(dd->mpi_comm_all, c, &ddnodeid);
+#endif
+ }
+ else
+ {
+ ddnodeid = ddindex;
+ }
+
+ return ddnodeid;
+}
+
+static gmx_bool dynamic_dd_box(gmx_ddbox_t *ddbox, t_inputrec *ir)
+{
+ return (ddbox->nboundeddim < DIM || DYNAMIC_BOX(*ir));
+}
+
+int ddglatnr(gmx_domdec_t *dd, int i)
+{
+ int atnr;
+
+ if (dd == NULL)
+ {
+ atnr = i + 1;
+ }
+ else
+ {
+ if (i >= dd->comm->nat[ddnatNR-1])
+ {
+ gmx_fatal(FARGS, "glatnr called with %d, which is larger than the local number of atoms (%d)", i, dd->comm->nat[ddnatNR-1]);
+ }
+ atnr = dd->gatindex[i] + 1;
+ }
+
+ return atnr;
+}
+
+t_block *dd_charge_groups_global(gmx_domdec_t *dd)
+{
+ return &dd->comm->cgs_gl;
+}
+
+static void vec_rvec_init(vec_rvec_t *v)
+{
+ v->nalloc = 0;
+ v->v = NULL;
+}
+
+static void vec_rvec_check_alloc(vec_rvec_t *v, int n)
+{
+ if (n > v->nalloc)
+ {
+ v->nalloc = over_alloc_dd(n);
+ srenew(v->v, v->nalloc);
+ }
+}
+
+void dd_store_state(gmx_domdec_t *dd, t_state *state)
+{
+ int i;
+
+ if (state->ddp_count != dd->ddp_count)
+ {
+ gmx_incons("The state does not the domain decomposition state");
+ }
+
+ state->ncg_gl = dd->ncg_home;
+ if (state->ncg_gl > state->cg_gl_nalloc)
+ {
+ state->cg_gl_nalloc = over_alloc_dd(state->ncg_gl);
+ srenew(state->cg_gl, state->cg_gl_nalloc);
+ }
+ for (i = 0; i < state->ncg_gl; i++)
+ {
+ state->cg_gl[i] = dd->index_gl[i];
+ }
+
+ state->ddp_count_cg_gl = dd->ddp_count;
+}
+
+gmx_domdec_zones_t *domdec_zones(gmx_domdec_t *dd)
+{
+ return &dd->comm->zones;
+}
+
+void dd_get_ns_ranges(gmx_domdec_t *dd, int icg,
+ int *jcg0, int *jcg1, ivec shift0, ivec shift1)
+{
+ gmx_domdec_zones_t *zones;
+ int izone, d, dim;
+
+ zones = &dd->comm->zones;
+
+ izone = 0;
+ while (icg >= zones->izone[izone].cg1)
+ {
+ izone++;
+ }
+
+ if (izone == 0)
+ {
+ *jcg0 = icg;
+ }
+ else if (izone < zones->nizone)
+ {
+ *jcg0 = zones->izone[izone].jcg0;
+ }
+ else
+ {
+ gmx_fatal(FARGS, "DD icg %d out of range: izone (%d) >= nizone (%d)",
+ icg, izone, zones->nizone);
+ }
+
+ *jcg1 = zones->izone[izone].jcg1;
+
+ for (d = 0; d < dd->ndim; d++)
+ {
+ dim = dd->dim[d];
+ shift0[dim] = zones->izone[izone].shift0[dim];
+ shift1[dim] = zones->izone[izone].shift1[dim];
+ if (dd->comm->tric_dir[dim] || (dd->bGridJump && d > 0))
+ {
+ /* A conservative approach, this can be optimized */
+ shift0[dim] -= 1;
+ shift1[dim] += 1;
+ }
+ }
+}
+
+int dd_natoms_vsite(gmx_domdec_t *dd)
+{
+ return dd->comm->nat[ddnatVSITE];
+}
+
+void dd_get_constraint_range(gmx_domdec_t *dd, int *at_start, int *at_end)
+{
+ *at_start = dd->comm->nat[ddnatCON-1];
+ *at_end = dd->comm->nat[ddnatCON];
+}
+
+void dd_move_x(gmx_domdec_t *dd, matrix box, rvec x[])
+{
+ int nzone, nat_tot, n, d, p, i, j, at0, at1, zone;
+ int *index, *cgindex;
+ gmx_domdec_comm_t *comm;
+ gmx_domdec_comm_dim_t *cd;
+ gmx_domdec_ind_t *ind;
+ rvec shift = {0, 0, 0}, *buf, *rbuf;
+ gmx_bool bPBC, bScrew;
+
+ comm = dd->comm;
+
+ cgindex = dd->cgindex;
+
+ buf = comm->vbuf.v;
+
+ nzone = 1;
+ nat_tot = dd->nat_home;
+ for (d = 0; d < dd->ndim; d++)
+ {
+ bPBC = (dd->ci[dd->dim[d]] == 0);
+ bScrew = (bPBC && dd->bScrewPBC && dd->dim[d] == XX);
+ if (bPBC)
+ {
+ copy_rvec(box[dd->dim[d]], shift);
+ }
+ cd = &comm->cd[d];
+ for (p = 0; p < cd->np; p++)
+ {
+ ind = &cd->ind[p];
+ index = ind->index;
+ n = 0;
+ if (!bPBC)
+ {
+ for (i = 0; i < ind->nsend[nzone]; i++)
+ {
+ at0 = cgindex[index[i]];
+ at1 = cgindex[index[i]+1];
+ for (j = at0; j < at1; j++)
+ {
+ copy_rvec(x[j], buf[n]);
+ n++;
+ }
+ }
+ }
+ else if (!bScrew)
+ {
+ for (i = 0; i < ind->nsend[nzone]; i++)
+ {
+ at0 = cgindex[index[i]];
+ at1 = cgindex[index[i]+1];
+ for (j = at0; j < at1; j++)
+ {
+ /* We need to shift the coordinates */
+ rvec_add(x[j], shift, buf[n]);
+ n++;
+ }
+ }
+ }
+ else
+ {
+ for (i = 0; i < ind->nsend[nzone]; i++)
+ {
+ at0 = cgindex[index[i]];
+ at1 = cgindex[index[i]+1];
+ for (j = at0; j < at1; j++)
+ {
+ /* Shift x */
+ buf[n][XX] = x[j][XX] + shift[XX];
+ /* Rotate y and z.
+ * This operation requires a special shift force
+ * treatment, which is performed in calc_vir.
+ */
+ buf[n][YY] = box[YY][YY] - x[j][YY];
+ buf[n][ZZ] = box[ZZ][ZZ] - x[j][ZZ];
+ n++;
+ }
+ }
+ }
+
+ if (cd->bInPlace)
+ {
+ rbuf = x + nat_tot;
+ }
+ else
+ {
+ rbuf = comm->vbuf2.v;
+ }
+ /* Send and receive the coordinates */
+ dd_sendrecv_rvec(dd, d, dddirBackward,
+ buf, ind->nsend[nzone+1],
+ rbuf, ind->nrecv[nzone+1]);
+ if (!cd->bInPlace)
+ {
+ j = 0;
+ for (zone = 0; zone < nzone; zone++)
+ {
+ for (i = ind->cell2at0[zone]; i < ind->cell2at1[zone]; i++)
+ {
+ copy_rvec(rbuf[j], x[i]);
+ j++;
+ }
+ }
+ }
+ nat_tot += ind->nrecv[nzone+1];
+ }
+ nzone += nzone;
+ }
+}
+
+void dd_move_f(gmx_domdec_t *dd, rvec f[], rvec *fshift)
+{
+ int nzone, nat_tot, n, d, p, i, j, at0, at1, zone;
+ int *index, *cgindex;
+ gmx_domdec_comm_t *comm;
+ gmx_domdec_comm_dim_t *cd;
+ gmx_domdec_ind_t *ind;
+ rvec *buf, *sbuf;
+ ivec vis;
+ int is;
+ gmx_bool bPBC, bScrew;
+
+ comm = dd->comm;
+
+ cgindex = dd->cgindex;
+
+ buf = comm->vbuf.v;
+
+ n = 0;
+ nzone = comm->zones.n/2;
+ nat_tot = dd->nat_tot;
+ for (d = dd->ndim-1; d >= 0; d--)
+ {
+ bPBC = (dd->ci[dd->dim[d]] == 0);
+ bScrew = (bPBC && dd->bScrewPBC && dd->dim[d] == XX);
+ if (fshift == NULL && !bScrew)
+ {
+ bPBC = FALSE;
+ }
+ /* Determine which shift vector we need */
+ clear_ivec(vis);
+ vis[dd->dim[d]] = 1;
+ is = IVEC2IS(vis);
+
+ cd = &comm->cd[d];
+ for (p = cd->np-1; p >= 0; p--)
+ {
+ ind = &cd->ind[p];
+ nat_tot -= ind->nrecv[nzone+1];
+ if (cd->bInPlace)
+ {
+ sbuf = f + nat_tot;
+ }
+ else
+ {
+ sbuf = comm->vbuf2.v;
+ j = 0;
+ for (zone = 0; zone < nzone; zone++)
+ {
+ for (i = ind->cell2at0[zone]; i < ind->cell2at1[zone]; i++)
+ {
+ copy_rvec(f[i], sbuf[j]);
+ j++;
+ }
+ }
+ }
+ /* Communicate the forces */
+ dd_sendrecv_rvec(dd, d, dddirForward,
+ sbuf, ind->nrecv[nzone+1],
+ buf, ind->nsend[nzone+1]);
+ index = ind->index;
+ /* Add the received forces */
+ n = 0;
+ if (!bPBC)
+ {
+ for (i = 0; i < ind->nsend[nzone]; i++)
+ {
+ at0 = cgindex[index[i]];
+ at1 = cgindex[index[i]+1];
+ for (j = at0; j < at1; j++)
+ {
+ rvec_inc(f[j], buf[n]);
+ n++;
+ }
+ }
+ }
+ else if (!bScrew)
+ {
+ for (i = 0; i < ind->nsend[nzone]; i++)
+ {
+ at0 = cgindex[index[i]];
+ at1 = cgindex[index[i]+1];
+ for (j = at0; j < at1; j++)
+ {
+ rvec_inc(f[j], buf[n]);
+ /* Add this force to the shift force */
+ rvec_inc(fshift[is], buf[n]);
+ n++;
+ }
+ }
+ }
+ else
+ {
+ for (i = 0; i < ind->nsend[nzone]; i++)
+ {
+ at0 = cgindex[index[i]];
+ at1 = cgindex[index[i]+1];
+ for (j = at0; j < at1; j++)
+ {
+ /* Rotate the force */
+ f[j][XX] += buf[n][XX];
+ f[j][YY] -= buf[n][YY];
+ f[j][ZZ] -= buf[n][ZZ];
+ if (fshift)
+ {
+ /* Add this force to the shift force */
+ rvec_inc(fshift[is], buf[n]);
+ }
+ n++;
+ }
+ }
+ }
+ }
+ nzone /= 2;
+ }
+}
+
+void dd_atom_spread_real(gmx_domdec_t *dd, real v[])
+{
+ int nzone, nat_tot, n, d, p, i, j, at0, at1, zone;
+ int *index, *cgindex;
+ gmx_domdec_comm_t *comm;
+ gmx_domdec_comm_dim_t *cd;
+ gmx_domdec_ind_t *ind;
+ real *buf, *rbuf;
+
+ comm = dd->comm;
+
+ cgindex = dd->cgindex;
+
+ buf = &comm->vbuf.v[0][0];
+
+ nzone = 1;
+ nat_tot = dd->nat_home;
+ for (d = 0; d < dd->ndim; d++)
+ {
+ cd = &comm->cd[d];
+ for (p = 0; p < cd->np; p++)
+ {
+ ind = &cd->ind[p];
+ index = ind->index;
+ n = 0;
+ for (i = 0; i < ind->nsend[nzone]; i++)
+ {
+ at0 = cgindex[index[i]];
+ at1 = cgindex[index[i]+1];
+ for (j = at0; j < at1; j++)
+ {
+ buf[n] = v[j];
+ n++;
+ }
+ }
+
+ if (cd->bInPlace)
+ {
+ rbuf = v + nat_tot;
+ }
+ else
+ {
+ rbuf = &comm->vbuf2.v[0][0];
+ }
+ /* Send and receive the coordinates */
+ dd_sendrecv_real(dd, d, dddirBackward,
+ buf, ind->nsend[nzone+1],
+ rbuf, ind->nrecv[nzone+1]);
+ if (!cd->bInPlace)
+ {
+ j = 0;
+ for (zone = 0; zone < nzone; zone++)
+ {
+ for (i = ind->cell2at0[zone]; i < ind->cell2at1[zone]; i++)
+ {
+ v[i] = rbuf[j];
+ j++;
+ }
+ }
+ }
+ nat_tot += ind->nrecv[nzone+1];
+ }
+ nzone += nzone;
+ }
+}
+
+void dd_atom_sum_real(gmx_domdec_t *dd, real v[])
+{
+ int nzone, nat_tot, n, d, p, i, j, at0, at1, zone;
+ int *index, *cgindex;
+ gmx_domdec_comm_t *comm;
+ gmx_domdec_comm_dim_t *cd;
+ gmx_domdec_ind_t *ind;
+ real *buf, *sbuf;
+
+ comm = dd->comm;
+
+ cgindex = dd->cgindex;
+
+ buf = &comm->vbuf.v[0][0];
+
+ n = 0;
+ nzone = comm->zones.n/2;
+ nat_tot = dd->nat_tot;
+ for (d = dd->ndim-1; d >= 0; d--)
+ {
+ cd = &comm->cd[d];
+ for (p = cd->np-1; p >= 0; p--)
+ {
+ ind = &cd->ind[p];
+ nat_tot -= ind->nrecv[nzone+1];
+ if (cd->bInPlace)
+ {
+ sbuf = v + nat_tot;
+ }
+ else
+ {
+ sbuf = &comm->vbuf2.v[0][0];
+ j = 0;
+ for (zone = 0; zone < nzone; zone++)
+ {
+ for (i = ind->cell2at0[zone]; i < ind->cell2at1[zone]; i++)
+ {
+ sbuf[j] = v[i];
+ j++;
+ }
+ }
+ }
+ /* Communicate the forces */
+ dd_sendrecv_real(dd, d, dddirForward,
+ sbuf, ind->nrecv[nzone+1],
+ buf, ind->nsend[nzone+1]);
+ index = ind->index;
+ /* Add the received forces */
+ n = 0;
+ for (i = 0; i < ind->nsend[nzone]; i++)
+ {
+ at0 = cgindex[index[i]];
+ at1 = cgindex[index[i]+1];
+ for (j = at0; j < at1; j++)
+ {
+ v[j] += buf[n];
+ n++;
+ }
+ }
+ }
+ nzone /= 2;
+ }
+}
+
+static void print_ddzone(FILE *fp, int d, int i, int j, gmx_ddzone_t *zone)
+{
+ fprintf(fp, "zone d0 %d d1 %d d2 %d min0 %6.3f max1 %6.3f mch0 %6.3f mch1 %6.3f p1_0 %6.3f p1_1 %6.3f\n",
+ d, i, j,
+ zone->min0, zone->max1,
+ zone->mch0, zone->mch0,
+ zone->p1_0, zone->p1_1);
+}
+
+
+#define DDZONECOMM_MAXZONE 5
+#define DDZONECOMM_BUFSIZE 3
+
+static void dd_sendrecv_ddzone(const gmx_domdec_t *dd,
+ int ddimind, int direction,
+ gmx_ddzone_t *buf_s, int n_s,
+ gmx_ddzone_t *buf_r, int n_r)
+{
+#define ZBS DDZONECOMM_BUFSIZE
+ rvec vbuf_s[DDZONECOMM_MAXZONE*ZBS];
+ rvec vbuf_r[DDZONECOMM_MAXZONE*ZBS];
+ int i;
+
+ for (i = 0; i < n_s; i++)
+ {
+ vbuf_s[i*ZBS ][0] = buf_s[i].min0;
+ vbuf_s[i*ZBS ][1] = buf_s[i].max1;
+ vbuf_s[i*ZBS ][2] = buf_s[i].min1;
+ vbuf_s[i*ZBS+1][0] = buf_s[i].mch0;
+ vbuf_s[i*ZBS+1][1] = buf_s[i].mch1;
+ vbuf_s[i*ZBS+1][2] = 0;
+ vbuf_s[i*ZBS+2][0] = buf_s[i].p1_0;
+ vbuf_s[i*ZBS+2][1] = buf_s[i].p1_1;
+ vbuf_s[i*ZBS+2][2] = 0;
+ }
+
+ dd_sendrecv_rvec(dd, ddimind, direction,
+ vbuf_s, n_s*ZBS,
+ vbuf_r, n_r*ZBS);
+
+ for (i = 0; i < n_r; i++)
+ {
+ buf_r[i].min0 = vbuf_r[i*ZBS ][0];
+ buf_r[i].max1 = vbuf_r[i*ZBS ][1];
+ buf_r[i].min1 = vbuf_r[i*ZBS ][2];
+ buf_r[i].mch0 = vbuf_r[i*ZBS+1][0];
+ buf_r[i].mch1 = vbuf_r[i*ZBS+1][1];
+ buf_r[i].p1_0 = vbuf_r[i*ZBS+2][0];
+ buf_r[i].p1_1 = vbuf_r[i*ZBS+2][1];
+ }
+
+#undef ZBS
+}
+
+static void dd_move_cellx(gmx_domdec_t *dd, gmx_ddbox_t *ddbox,
+ rvec cell_ns_x0, rvec cell_ns_x1)
+{
+ int d, d1, dim, dim1, pos, buf_size, i, j, k, p, npulse, npulse_min;
+ gmx_ddzone_t *zp;
+ gmx_ddzone_t buf_s[DDZONECOMM_MAXZONE];
+ gmx_ddzone_t buf_r[DDZONECOMM_MAXZONE];
+ gmx_ddzone_t buf_e[DDZONECOMM_MAXZONE];
+ rvec extr_s[2], extr_r[2];
+ rvec dh;
+ real dist_d, c = 0, det;
+ gmx_domdec_comm_t *comm;
+ gmx_bool bPBC, bUse;
+
+ comm = dd->comm;
+
+ for (d = 1; d < dd->ndim; d++)
+ {
+ dim = dd->dim[d];
+ zp = (d == 1) ? &comm->zone_d1[0] : &comm->zone_d2[0][0];
+ zp->min0 = cell_ns_x0[dim];
+ zp->max1 = cell_ns_x1[dim];
+ zp->min1 = cell_ns_x1[dim];
+ zp->mch0 = cell_ns_x0[dim];
+ zp->mch1 = cell_ns_x1[dim];
+ zp->p1_0 = cell_ns_x0[dim];
+ zp->p1_1 = cell_ns_x1[dim];
+ }
+
+ for (d = dd->ndim-2; d >= 0; d--)
+ {
+ dim = dd->dim[d];
+ bPBC = (dim < ddbox->npbcdim);
+
+ /* Use an rvec to store two reals */
+ extr_s[d][0] = comm->cell_f0[d+1];
+ extr_s[d][1] = comm->cell_f1[d+1];
+ extr_s[d][2] = comm->cell_f1[d+1];
+
+ pos = 0;
+ /* Store the extremes in the backward sending buffer,
+ * so the get updated separately from the forward communication.
+ */
+ for (d1 = d; d1 < dd->ndim-1; d1++)
+ {
+ /* We invert the order to be able to use the same loop for buf_e */
+ buf_s[pos].min0 = extr_s[d1][1];
+ buf_s[pos].max1 = extr_s[d1][0];
+ buf_s[pos].min1 = extr_s[d1][2];
+ buf_s[pos].mch0 = 0;
+ buf_s[pos].mch1 = 0;
+ /* Store the cell corner of the dimension we communicate along */
+ buf_s[pos].p1_0 = comm->cell_x0[dim];
+ buf_s[pos].p1_1 = 0;
+ pos++;
+ }
+
+ buf_s[pos] = (dd->ndim == 2) ? comm->zone_d1[0] : comm->zone_d2[0][0];
+ pos++;
+
+ if (dd->ndim == 3 && d == 0)
+ {
+ buf_s[pos] = comm->zone_d2[0][1];
+ pos++;
+ buf_s[pos] = comm->zone_d1[0];
+ pos++;
+ }
+
+ /* We only need to communicate the extremes
+ * in the forward direction
+ */
+ npulse = comm->cd[d].np;
+ if (bPBC)
+ {
+ /* Take the minimum to avoid double communication */
+ npulse_min = min(npulse, dd->nc[dim]-1-npulse);
+ }
+ else
+ {
+ /* Without PBC we should really not communicate over
+ * the boundaries, but implementing that complicates
+ * the communication setup and therefore we simply
+ * do all communication, but ignore some data.
+ */
+ npulse_min = npulse;
+ }
+ for (p = 0; p < npulse_min; p++)
+ {
+ /* Communicate the extremes forward */
+ bUse = (bPBC || dd->ci[dim] > 0);
+
+ dd_sendrecv_rvec(dd, d, dddirForward,
+ extr_s+d, dd->ndim-d-1,
+ extr_r+d, dd->ndim-d-1);
+
+ if (bUse)
+ {
+ for (d1 = d; d1 < dd->ndim-1; d1++)
+ {
+ extr_s[d1][0] = max(extr_s[d1][0], extr_r[d1][0]);
+ extr_s[d1][1] = min(extr_s[d1][1], extr_r[d1][1]);
+ extr_s[d1][2] = min(extr_s[d1][2], extr_r[d1][2]);
+ }
+ }
+ }
+
+ buf_size = pos;
+ for (p = 0; p < npulse; p++)
+ {
+ /* Communicate all the zone information backward */
+ bUse = (bPBC || dd->ci[dim] < dd->nc[dim] - 1);
+
+ dd_sendrecv_ddzone(dd, d, dddirBackward,
+ buf_s, buf_size,
+ buf_r, buf_size);
+
+ clear_rvec(dh);
+ if (p > 0)
+ {
+ for (d1 = d+1; d1 < dd->ndim; d1++)
+ {
+ /* Determine the decrease of maximum required
+ * communication height along d1 due to the distance along d,
+ * this avoids a lot of useless atom communication.
+ */
+ dist_d = comm->cell_x1[dim] - buf_r[0].p1_0;
+
+ if (ddbox->tric_dir[dim])
+ {
+ /* c is the off-diagonal coupling between the cell planes
+ * along directions d and d1.
+ */
+ c = ddbox->v[dim][dd->dim[d1]][dim];
+ }
+ else
+ {
+ c = 0;
+ }
+ det = (1 + c*c)*comm->cutoff*comm->cutoff - dist_d*dist_d;
+ if (det > 0)
+ {
+ dh[d1] = comm->cutoff - (c*dist_d + sqrt(det))/(1 + c*c);
+ }
+ else
+ {
+ /* A negative value signals out of range */
+ dh[d1] = -1;
+ }
+ }
+ }
+
+ /* Accumulate the extremes over all pulses */
+ for (i = 0; i < buf_size; i++)
+ {
+ if (p == 0)
+ {
+ buf_e[i] = buf_r[i];
+ }
+ else
+ {
+ if (bUse)
+ {
+ buf_e[i].min0 = min(buf_e[i].min0, buf_r[i].min0);
+ buf_e[i].max1 = max(buf_e[i].max1, buf_r[i].max1);
+ buf_e[i].min1 = min(buf_e[i].min1, buf_r[i].min1);
+ }
+
+ if (dd->ndim == 3 && d == 0 && i == buf_size - 1)
+ {
+ d1 = 1;
+ }
+ else
+ {
+ d1 = d + 1;
+ }
+ if (bUse && dh[d1] >= 0)
+ {
+ buf_e[i].mch0 = max(buf_e[i].mch0, buf_r[i].mch0-dh[d1]);
+ buf_e[i].mch1 = max(buf_e[i].mch1, buf_r[i].mch1-dh[d1]);
+ }
+ }
+ /* Copy the received buffer to the send buffer,
+ * to pass the data through with the next pulse.
+ */
+ buf_s[i] = buf_r[i];
+ }
+ if (((bPBC || dd->ci[dim]+npulse < dd->nc[dim]) && p == npulse-1) ||
+ (!bPBC && dd->ci[dim]+1+p == dd->nc[dim]-1))
+ {
+ /* Store the extremes */
+ pos = 0;
+
+ for (d1 = d; d1 < dd->ndim-1; d1++)
+ {
+ extr_s[d1][1] = min(extr_s[d1][1], buf_e[pos].min0);
+ extr_s[d1][0] = max(extr_s[d1][0], buf_e[pos].max1);
+ extr_s[d1][2] = min(extr_s[d1][2], buf_e[pos].min1);
+ pos++;
+ }
+
+ if (d == 1 || (d == 0 && dd->ndim == 3))
+ {
+ for (i = d; i < 2; i++)
+ {
+ comm->zone_d2[1-d][i] = buf_e[pos];
+ pos++;
+ }
+ }
+ if (d == 0)
+ {
+ comm->zone_d1[1] = buf_e[pos];
+ pos++;
+ }
+ }
+ }
+ }
+
+ if (dd->ndim >= 2)
+ {
+ dim = dd->dim[1];
+ for (i = 0; i < 2; i++)
+ {
+ if (debug)
+ {
+ print_ddzone(debug, 1, i, 0, &comm->zone_d1[i]);
+ }
+ cell_ns_x0[dim] = min(cell_ns_x0[dim], comm->zone_d1[i].min0);
+ cell_ns_x1[dim] = max(cell_ns_x1[dim], comm->zone_d1[i].max1);
+ }
+ }
+ if (dd->ndim >= 3)
+ {
+ dim = dd->dim[2];
+ for (i = 0; i < 2; i++)
+ {
+ for (j = 0; j < 2; j++)
+ {
+ if (debug)
+ {
+ print_ddzone(debug, 2, i, j, &comm->zone_d2[i][j]);
+ }
+ cell_ns_x0[dim] = min(cell_ns_x0[dim], comm->zone_d2[i][j].min0);
+ cell_ns_x1[dim] = max(cell_ns_x1[dim], comm->zone_d2[i][j].max1);
+ }
+ }
+ }
+ for (d = 1; d < dd->ndim; d++)
+ {
+ comm->cell_f_max0[d] = extr_s[d-1][0];
+ comm->cell_f_min1[d] = extr_s[d-1][1];
+ if (debug)
+ {
+ fprintf(debug, "Cell fraction d %d, max0 %f, min1 %f\n",
+ d, comm->cell_f_max0[d], comm->cell_f_min1[d]);
+ }
+ }
+}
+
+static void dd_collect_cg(gmx_domdec_t *dd,
+ t_state *state_local)
+{
+ gmx_domdec_master_t *ma = NULL;
+ int buf2[2], *ibuf, i, ncg_home = 0, *cg = NULL, nat_home = 0;
+ t_block *cgs_gl;
+
+ if (state_local->ddp_count == dd->comm->master_cg_ddp_count)
+ {
+ /* The master has the correct distribution */
+ return;
+ }
+
+ if (state_local->ddp_count == dd->ddp_count)
+ {
+ ncg_home = dd->ncg_home;
+ cg = dd->index_gl;
+ nat_home = dd->nat_home;
+ }
+ else if (state_local->ddp_count_cg_gl == state_local->ddp_count)
+ {
+ cgs_gl = &dd->comm->cgs_gl;
+
+ ncg_home = state_local->ncg_gl;
+ cg = state_local->cg_gl;
+ nat_home = 0;
+ for (i = 0; i < ncg_home; i++)
+ {
+ nat_home += cgs_gl->index[cg[i]+1] - cgs_gl->index[cg[i]];
+ }
+ }
+ else
+ {
+ gmx_incons("Attempted to collect a vector for a state for which the charge group distribution is unknown");
+ }
+
+ buf2[0] = dd->ncg_home;
+ buf2[1] = dd->nat_home;
+ if (DDMASTER(dd))
+ {
+ ma = dd->ma;
+ ibuf = ma->ibuf;
+ }
+ else
+ {
+ ibuf = NULL;
+ }
+ /* Collect the charge group and atom counts on the master */
+ dd_gather(dd, 2*sizeof(int), buf2, ibuf);
+
+ if (DDMASTER(dd))
+ {
+ ma->index[0] = 0;
+ for (i = 0; i < dd->nnodes; i++)
+ {
+ ma->ncg[i] = ma->ibuf[2*i];
+ ma->nat[i] = ma->ibuf[2*i+1];
+ ma->index[i+1] = ma->index[i] + ma->ncg[i];
+
+ }
+ /* Make byte counts and indices */
+ for (i = 0; i < dd->nnodes; i++)
+ {
+ ma->ibuf[i] = ma->ncg[i]*sizeof(int);
+ ma->ibuf[dd->nnodes+i] = ma->index[i]*sizeof(int);
+ }
+ if (debug)
+ {
+ fprintf(debug, "Initial charge group distribution: ");
+ for (i = 0; i < dd->nnodes; i++)
+ {
+ fprintf(debug, " %d", ma->ncg[i]);
+ }
+ fprintf(debug, "\n");
+ }
+ }
+
+ /* Collect the charge group indices on the master */
+ dd_gatherv(dd,
+ dd->ncg_home*sizeof(int), dd->index_gl,
+ DDMASTER(dd) ? ma->ibuf : NULL,
+ DDMASTER(dd) ? ma->ibuf+dd->nnodes : NULL,
+ DDMASTER(dd) ? ma->cg : NULL);
+
+ dd->comm->master_cg_ddp_count = state_local->ddp_count;
+}
+
+static void dd_collect_vec_sendrecv(gmx_domdec_t *dd,
+ rvec *lv, rvec *v)
+{
+ gmx_domdec_master_t *ma;
+ int n, i, c, a, nalloc = 0;
+ rvec *buf = NULL;
+ t_block *cgs_gl;
+
+ ma = dd->ma;
+
+ if (!DDMASTER(dd))
+ {
+#ifdef GMX_MPI
+ MPI_Send(lv, dd->nat_home*sizeof(rvec), MPI_BYTE, DDMASTERRANK(dd),
+ dd->rank, dd->mpi_comm_all);
+#endif
+ }
+ else
+ {
+ /* Copy the master coordinates to the global array */
+ cgs_gl = &dd->comm->cgs_gl;
+
+ n = DDMASTERRANK(dd);
+ a = 0;
+ for (i = ma->index[n]; i < ma->index[n+1]; i++)
+ {
+ for (c = cgs_gl->index[ma->cg[i]]; c < cgs_gl->index[ma->cg[i]+1]; c++)
+ {
+ copy_rvec(lv[a++], v[c]);
+ }
+ }
+
+ for (n = 0; n < dd->nnodes; n++)
+ {
+ if (n != dd->rank)
+ {
+ if (ma->nat[n] > nalloc)
+ {
+ nalloc = over_alloc_dd(ma->nat[n]);
+ srenew(buf, nalloc);
+ }
+#ifdef GMX_MPI
+ MPI_Recv(buf, ma->nat[n]*sizeof(rvec), MPI_BYTE, DDRANK(dd, n),
+ n, dd->mpi_comm_all, MPI_STATUS_IGNORE);
+#endif
+ a = 0;
+ for (i = ma->index[n]; i < ma->index[n+1]; i++)
+ {
+ for (c = cgs_gl->index[ma->cg[i]]; c < cgs_gl->index[ma->cg[i]+1]; c++)
+ {
+ copy_rvec(buf[a++], v[c]);
+ }
+ }
+ }
+ }
+ sfree(buf);
+ }
+}
+
+static void get_commbuffer_counts(gmx_domdec_t *dd,
+ int **counts, int **disps)
+{
+ gmx_domdec_master_t *ma;
+ int n;
+
+ ma = dd->ma;
+
+ /* Make the rvec count and displacment arrays */
+ *counts = ma->ibuf;
+ *disps = ma->ibuf + dd->nnodes;
+ for (n = 0; n < dd->nnodes; n++)
+ {
+ (*counts)[n] = ma->nat[n]*sizeof(rvec);
+ (*disps)[n] = (n == 0 ? 0 : (*disps)[n-1] + (*counts)[n-1]);
+ }
+}
+
+static void dd_collect_vec_gatherv(gmx_domdec_t *dd,
+ rvec *lv, rvec *v)
+{
+ gmx_domdec_master_t *ma;
+ int *rcounts = NULL, *disps = NULL;
+ int n, i, c, a;
+ rvec *buf = NULL;
+ t_block *cgs_gl;
+
+ ma = dd->ma;
+
+ if (DDMASTER(dd))
+ {
+ get_commbuffer_counts(dd, &rcounts, &disps);
+
+ buf = ma->vbuf;
+ }
+
+ dd_gatherv(dd, dd->nat_home*sizeof(rvec), lv, rcounts, disps, buf);
+
+ if (DDMASTER(dd))
+ {
+ cgs_gl = &dd->comm->cgs_gl;
+
+ a = 0;
+ for (n = 0; n < dd->nnodes; n++)
+ {
+ for (i = ma->index[n]; i < ma->index[n+1]; i++)
+ {
+ for (c = cgs_gl->index[ma->cg[i]]; c < cgs_gl->index[ma->cg[i]+1]; c++)
+ {
+ copy_rvec(buf[a++], v[c]);
+ }
+ }
+ }
+ }
+}
+
+void dd_collect_vec(gmx_domdec_t *dd,
+ t_state *state_local, rvec *lv, rvec *v)
+{
+ gmx_domdec_master_t *ma;
+ int n, i, c, a, nalloc = 0;
+ rvec *buf = NULL;
+
+ dd_collect_cg(dd, state_local);
+
+ if (dd->nnodes <= GMX_DD_NNODES_SENDRECV)
+ {
+ dd_collect_vec_sendrecv(dd, lv, v);
+ }
+ else
+ {
+ dd_collect_vec_gatherv(dd, lv, v);
+ }
+}
+
+
+void dd_collect_state(gmx_domdec_t *dd,
+ t_state *state_local, t_state *state)
+{
+ int est, i, j, nh;
+
+ nh = state->nhchainlength;
+
+ if (DDMASTER(dd))
+ {
+ for (i = 0; i < efptNR; i++)
+ {
+ state->lambda[i] = state_local->lambda[i];
+ }
+ state->fep_state = state_local->fep_state;
+ state->veta = state_local->veta;
+ state->vol0 = state_local->vol0;
+ copy_mat(state_local->box, state->box);
+ copy_mat(state_local->boxv, state->boxv);
+ copy_mat(state_local->svir_prev, state->svir_prev);
+ copy_mat(state_local->fvir_prev, state->fvir_prev);
+ copy_mat(state_local->pres_prev, state->pres_prev);
+
+ for (i = 0; i < state_local->ngtc; i++)
+ {
+ for (j = 0; j < nh; j++)
+ {
+ state->nosehoover_xi[i*nh+j] = state_local->nosehoover_xi[i*nh+j];
+ state->nosehoover_vxi[i*nh+j] = state_local->nosehoover_vxi[i*nh+j];
+ }
+ state->therm_integral[i] = state_local->therm_integral[i];
+ }
+ for (i = 0; i < state_local->nnhpres; i++)
+ {
+ for (j = 0; j < nh; j++)
+ {
+ state->nhpres_xi[i*nh+j] = state_local->nhpres_xi[i*nh+j];
+ state->nhpres_vxi[i*nh+j] = state_local->nhpres_vxi[i*nh+j];
+ }
+ }
+ }
+ for (est = 0; est < estNR; est++)
+ {
+ if (EST_DISTR(est) && (state_local->flags & (1<<est)))
+ {
+ switch (est)
+ {
+ case estX:
+ dd_collect_vec(dd, state_local, state_local->x, state->x);
+ break;
+ case estV:
+ dd_collect_vec(dd, state_local, state_local->v, state->v);
+ break;
+ case estSDX:
+ dd_collect_vec(dd, state_local, state_local->sd_X, state->sd_X);
+ break;
+ case estCGP:
+ dd_collect_vec(dd, state_local, state_local->cg_p, state->cg_p);
+ break;
+ case estLD_RNG:
+ if (state->nrngi == 1)
+ {
+ if (DDMASTER(dd))
+ {
+ for (i = 0; i < state_local->nrng; i++)
+ {
+ state->ld_rng[i] = state_local->ld_rng[i];
+ }
+ }
+ }
+ else
+ {
+ dd_gather(dd, state_local->nrng*sizeof(state->ld_rng[0]),
+ state_local->ld_rng, state->ld_rng);
+ }
+ break;
+ case estLD_RNGI:
+ if (state->nrngi == 1)
+ {
+ if (DDMASTER(dd))
+ {
+ state->ld_rngi[0] = state_local->ld_rngi[0];
+ }
+ }
+ else
+ {
+ dd_gather(dd, sizeof(state->ld_rngi[0]),
+ state_local->ld_rngi, state->ld_rngi);
+ }
+ break;
+ case estDISRE_INITF:
+ case estDISRE_RM3TAV:
+ case estORIRE_INITF:
+ case estORIRE_DTAV:
+ break;
+ default:
+ gmx_incons("Unknown state entry encountered in dd_collect_state");
+ }
+ }
+ }
+}
+
+static void dd_realloc_state(t_state *state, rvec **f, int nalloc)
+{
+ int est;
+
+ if (debug)
+ {
+ fprintf(debug, "Reallocating state: currently %d, required %d, allocating %d\n", state->nalloc, nalloc, over_alloc_dd(nalloc));
+ }
+
+ state->nalloc = over_alloc_dd(nalloc);
+
+ for (est = 0; est < estNR; est++)
+ {
+ if (EST_DISTR(est) && (state->flags & (1<<est)))
+ {
+ switch (est)
+ {
+ case estX:
+ srenew(state->x, state->nalloc);
+ break;
+ case estV:
+ srenew(state->v, state->nalloc);
+ break;
+ case estSDX:
+ srenew(state->sd_X, state->nalloc);
+ break;
+ case estCGP:
+ srenew(state->cg_p, state->nalloc);
+ break;
+ case estLD_RNG:
+ case estLD_RNGI:
+ case estDISRE_INITF:
+ case estDISRE_RM3TAV:
+ case estORIRE_INITF:
+ case estORIRE_DTAV:
+ /* No reallocation required */
+ break;
+ default:
+ gmx_incons("Unknown state entry encountered in dd_realloc_state");
+ }
+ }
+ }
+
+ if (f != NULL)
+ {
+ srenew(*f, state->nalloc);
+ }
+}
+
+static void dd_check_alloc_ncg(t_forcerec *fr, t_state *state, rvec **f,
+ int nalloc)
+{
+ if (nalloc > fr->cg_nalloc)
+ {
+ if (debug)
+ {
+ fprintf(debug, "Reallocating forcerec: currently %d, required %d, allocating %d\n", fr->cg_nalloc, nalloc, over_alloc_dd(nalloc));
+ }
+ fr->cg_nalloc = over_alloc_dd(nalloc);
+ srenew(fr->cginfo, fr->cg_nalloc);
+ if (fr->cutoff_scheme == ecutsGROUP)
+ {
+ srenew(fr->cg_cm, fr->cg_nalloc);
+ }
+ }
+ if (fr->cutoff_scheme == ecutsVERLET && nalloc > state->nalloc)
+ {
+ /* We don't use charge groups, we use x in state to set up
+ * the atom communication.
+ */
+ dd_realloc_state(state, f, nalloc);
+ }
+}
+
+static void dd_distribute_vec_sendrecv(gmx_domdec_t *dd, t_block *cgs,
+ rvec *v, rvec *lv)
+{
+ gmx_domdec_master_t *ma;
+ int n, i, c, a, nalloc = 0;
+ rvec *buf = NULL;
+
+ if (DDMASTER(dd))
+ {
+ ma = dd->ma;
+
+ for (n = 0; n < dd->nnodes; n++)
+ {
+ if (n != dd->rank)
+ {
+ if (ma->nat[n] > nalloc)
+ {
+ nalloc = over_alloc_dd(ma->nat[n]);
+ srenew(buf, nalloc);
+ }
+ /* Use lv as a temporary buffer */
+ a = 0;
+ for (i = ma->index[n]; i < ma->index[n+1]; i++)
+ {
+ for (c = cgs->index[ma->cg[i]]; c < cgs->index[ma->cg[i]+1]; c++)
+ {
+ copy_rvec(v[c], buf[a++]);
+ }
+ }
+ if (a != ma->nat[n])
+ {
+ gmx_fatal(FARGS, "Internal error a (%d) != nat (%d)",
+ a, ma->nat[n]);
+ }
+
+#ifdef GMX_MPI
+ MPI_Send(buf, ma->nat[n]*sizeof(rvec), MPI_BYTE,
+ DDRANK(dd, n), n, dd->mpi_comm_all);
+#endif
+ }
+ }
+ sfree(buf);
+ n = DDMASTERRANK(dd);
+ a = 0;
+ for (i = ma->index[n]; i < ma->index[n+1]; i++)
+ {
+ for (c = cgs->index[ma->cg[i]]; c < cgs->index[ma->cg[i]+1]; c++)
+ {
+ copy_rvec(v[c], lv[a++]);
+ }
+ }
+ }
+ else
+ {
+#ifdef GMX_MPI
+ MPI_Recv(lv, dd->nat_home*sizeof(rvec), MPI_BYTE, DDMASTERRANK(dd),
+ MPI_ANY_TAG, dd->mpi_comm_all, MPI_STATUS_IGNORE);
+#endif
+ }
+}
+
+static void dd_distribute_vec_scatterv(gmx_domdec_t *dd, t_block *cgs,
+ rvec *v, rvec *lv)
+{
+ gmx_domdec_master_t *ma;
+ int *scounts = NULL, *disps = NULL;
+ int n, i, c, a, nalloc = 0;
+ rvec *buf = NULL;
+
+ if (DDMASTER(dd))
+ {
+ ma = dd->ma;
+
+ get_commbuffer_counts(dd, &scounts, &disps);
+
+ buf = ma->vbuf;
+ a = 0;
+ for (n = 0; n < dd->nnodes; n++)
+ {
+ for (i = ma->index[n]; i < ma->index[n+1]; i++)
+ {
+ for (c = cgs->index[ma->cg[i]]; c < cgs->index[ma->cg[i]+1]; c++)
+ {
+ copy_rvec(v[c], buf[a++]);
+ }
+ }
+ }
+ }
+
+ dd_scatterv(dd, scounts, disps, buf, dd->nat_home*sizeof(rvec), lv);
+}
+
+static void dd_distribute_vec(gmx_domdec_t *dd, t_block *cgs, rvec *v, rvec *lv)
+{
+ if (dd->nnodes <= GMX_DD_NNODES_SENDRECV)
+ {
+ dd_distribute_vec_sendrecv(dd, cgs, v, lv);
+ }
+ else
+ {
+ dd_distribute_vec_scatterv(dd, cgs, v, lv);
+ }
+}
+
++static void dd_distribute_dfhist(gmx_domdec_t *dd, df_history_t *dfhist)
++{
++ int i;
++ dd_bcast(dd, sizeof(int), &dfhist->bEquil);
++ dd_bcast(dd, sizeof(int), &dfhist->nlambda);
++ dd_bcast(dd, sizeof(real), &dfhist->wl_delta);
++
++ if (dfhist->nlambda > 0)
++ {
++ int nlam = dfhist->nlambda;
++ dd_bcast(dd, sizeof(int)*nlam, dfhist->n_at_lam);
++ dd_bcast(dd, sizeof(real)*nlam, dfhist->wl_histo);
++ dd_bcast(dd, sizeof(real)*nlam, dfhist->sum_weights);
++ dd_bcast(dd, sizeof(real)*nlam, dfhist->sum_dg);
++ dd_bcast(dd, sizeof(real)*nlam, dfhist->sum_minvar);
++ dd_bcast(dd, sizeof(real)*nlam, dfhist->sum_variance);
++
++ for (i = 0; i<nlam; i++) {
++ dd_bcast(dd, sizeof(real)*nlam, dfhist->accum_p[i]);
++ dd_bcast(dd, sizeof(real)*nlam, dfhist->accum_m[i]);
++ dd_bcast(dd, sizeof(real)*nlam, dfhist->accum_p2[i]);
++ dd_bcast(dd, sizeof(real)*nlam, dfhist->accum_m2[i]);
++ dd_bcast(dd, sizeof(real)*nlam, dfhist->Tij[i]);
++ dd_bcast(dd, sizeof(real)*nlam, dfhist->Tij_empirical[i]);
++ }
++ }
++}
++
+static void dd_distribute_state(gmx_domdec_t *dd, t_block *cgs,
+ t_state *state, t_state *state_local,
+ rvec **f)
+{
+ int i, j, nh;
+
+ nh = state->nhchainlength;
+
+ if (DDMASTER(dd))
+ {
+ for (i = 0; i < efptNR; i++)
+ {
+ state_local->lambda[i] = state->lambda[i];
+ }
+ state_local->fep_state = state->fep_state;
+ state_local->veta = state->veta;
+ state_local->vol0 = state->vol0;
+ copy_mat(state->box, state_local->box);
+ copy_mat(state->box_rel, state_local->box_rel);
+ copy_mat(state->boxv, state_local->boxv);
+ copy_mat(state->svir_prev, state_local->svir_prev);
+ copy_mat(state->fvir_prev, state_local->fvir_prev);
++ copy_df_history(&state_local->dfhist,&state->dfhist);
+ for (i = 0; i < state_local->ngtc; i++)
+ {
+ for (j = 0; j < nh; j++)
+ {
+ state_local->nosehoover_xi[i*nh+j] = state->nosehoover_xi[i*nh+j];
+ state_local->nosehoover_vxi[i*nh+j] = state->nosehoover_vxi[i*nh+j];
+ }
+ state_local->therm_integral[i] = state->therm_integral[i];
+ }
+ for (i = 0; i < state_local->nnhpres; i++)
+ {
+ for (j = 0; j < nh; j++)
+ {
+ state_local->nhpres_xi[i*nh+j] = state->nhpres_xi[i*nh+j];
+ state_local->nhpres_vxi[i*nh+j] = state->nhpres_vxi[i*nh+j];
+ }
+ }
+ }
+ dd_bcast(dd, ((efptNR)*sizeof(real)), state_local->lambda);
+ dd_bcast(dd, sizeof(int), &state_local->fep_state);
+ dd_bcast(dd, sizeof(real), &state_local->veta);
+ dd_bcast(dd, sizeof(real), &state_local->vol0);
+ dd_bcast(dd, sizeof(state_local->box), state_local->box);
+ dd_bcast(dd, sizeof(state_local->box_rel), state_local->box_rel);
+ dd_bcast(dd, sizeof(state_local->boxv), state_local->boxv);
+ dd_bcast(dd, sizeof(state_local->svir_prev), state_local->svir_prev);
+ dd_bcast(dd, sizeof(state_local->fvir_prev), state_local->fvir_prev);
+ dd_bcast(dd, ((state_local->ngtc*nh)*sizeof(double)), state_local->nosehoover_xi);
+ dd_bcast(dd, ((state_local->ngtc*nh)*sizeof(double)), state_local->nosehoover_vxi);
+ dd_bcast(dd, state_local->ngtc*sizeof(double), state_local->therm_integral);
+ dd_bcast(dd, ((state_local->nnhpres*nh)*sizeof(double)), state_local->nhpres_xi);
+ dd_bcast(dd, ((state_local->nnhpres*nh)*sizeof(double)), state_local->nhpres_vxi);
+
++ /* communicate df_history -- required for restarting from checkpoint */
++ dd_distribute_dfhist(dd,&state_local->dfhist);
++
+ if (dd->nat_home > state_local->nalloc)
+ {
+ dd_realloc_state(state_local, f, dd->nat_home);
+ }
+ for (i = 0; i < estNR; i++)
+ {
+ if (EST_DISTR(i) && (state_local->flags & (1<<i)))
+ {
+ switch (i)
+ {
+ case estX:
+ dd_distribute_vec(dd, cgs, state->x, state_local->x);
+ break;
+ case estV:
+ dd_distribute_vec(dd, cgs, state->v, state_local->v);
+ break;
+ case estSDX:
+ dd_distribute_vec(dd, cgs, state->sd_X, state_local->sd_X);
+ break;
+ case estCGP:
+ dd_distribute_vec(dd, cgs, state->cg_p, state_local->cg_p);
+ break;
+ case estLD_RNG:
+ if (state->nrngi == 1)
+ {
+ dd_bcastc(dd,
+ state_local->nrng*sizeof(state_local->ld_rng[0]),
+ state->ld_rng, state_local->ld_rng);
+ }
+ else
+ {
+ dd_scatter(dd,
+ state_local->nrng*sizeof(state_local->ld_rng[0]),
+ state->ld_rng, state_local->ld_rng);
+ }
+ break;
+ case estLD_RNGI:
+ if (state->nrngi == 1)
+ {
+ dd_bcastc(dd, sizeof(state_local->ld_rngi[0]),
+ state->ld_rngi, state_local->ld_rngi);
+ }
+ else
+ {
+ dd_scatter(dd, sizeof(state_local->ld_rngi[0]),
+ state->ld_rngi, state_local->ld_rngi);
+ }
+ break;
+ case estDISRE_INITF:
+ case estDISRE_RM3TAV:
+ case estORIRE_INITF:
+ case estORIRE_DTAV:
+ /* Not implemented yet */
+ break;
+ default:
+ gmx_incons("Unknown state entry encountered in dd_distribute_state");
+ }
+ }
+ }
+}
+
+static char dim2char(int dim)
+{
+ char c = '?';
+
+ switch (dim)
+ {
+ case XX: c = 'X'; break;
+ case YY: c = 'Y'; break;
+ case ZZ: c = 'Z'; break;
+ default: gmx_fatal(FARGS, "Unknown dim %d", dim);
+ }
+
+ return c;
+}
+
+static void write_dd_grid_pdb(const char *fn, gmx_large_int_t step,
+ gmx_domdec_t *dd, matrix box, gmx_ddbox_t *ddbox)
+{
+ rvec grid_s[2], *grid_r = NULL, cx, r;
+ char fname[STRLEN], format[STRLEN], buf[22];
+ FILE *out;
+ int a, i, d, z, y, x;
+ matrix tric;
+ real vol;
+
+ copy_rvec(dd->comm->cell_x0, grid_s[0]);
+ copy_rvec(dd->comm->cell_x1, grid_s[1]);
+
+ if (DDMASTER(dd))
+ {
+ snew(grid_r, 2*dd->nnodes);
+ }
+
+ dd_gather(dd, 2*sizeof(rvec), grid_s[0], DDMASTER(dd) ? grid_r[0] : NULL);
+
+ if (DDMASTER(dd))
+ {
+ for (d = 0; d < DIM; d++)
+ {
+ for (i = 0; i < DIM; i++)
+ {
+ if (d == i)
+ {
+ tric[d][i] = 1;
+ }
+ else
+ {
+ if (d < ddbox->npbcdim && dd->nc[d] > 1)
+ {
+ tric[d][i] = box[i][d]/box[i][i];
+ }
+ else
+ {
+ tric[d][i] = 0;
+ }
+ }
+ }
+ }
+ sprintf(fname, "%s_%s.pdb", fn, gmx_step_str(step, buf));
+ sprintf(format, "%s%s\n", get_pdbformat(), "%6.2f%6.2f");
+ out = gmx_fio_fopen(fname, "w");
+ gmx_write_pdb_box(out, dd->bScrewPBC ? epbcSCREW : epbcXYZ, box);
+ a = 1;
+ for (i = 0; i < dd->nnodes; i++)
+ {
+ vol = dd->nnodes/(box[XX][XX]*box[YY][YY]*box[ZZ][ZZ]);
+ for (d = 0; d < DIM; d++)
+ {
+ vol *= grid_r[i*2+1][d] - grid_r[i*2][d];
+ }
+ for (z = 0; z < 2; z++)
+ {
+ for (y = 0; y < 2; y++)
+ {
+ for (x = 0; x < 2; x++)
+ {
+ cx[XX] = grid_r[i*2+x][XX];
+ cx[YY] = grid_r[i*2+y][YY];
+ cx[ZZ] = grid_r[i*2+z][ZZ];
+ mvmul(tric, cx, r);
+ fprintf(out, format, "ATOM", a++, "CA", "GLY", ' ', 1+i,
+ ' ', 10*r[XX], 10*r[YY], 10*r[ZZ], 1.0, vol);
+ }
+ }
+ }
+ for (d = 0; d < DIM; d++)
+ {
+ for (x = 0; x < 4; x++)
+ {
+ switch (d)
+ {
+ case 0: y = 1 + i*8 + 2*x; break;
+ case 1: y = 1 + i*8 + 2*x - (x % 2); break;
+ case 2: y = 1 + i*8 + x; break;
+ }
+ fprintf(out, "%6s%5d%5d\n", "CONECT", y, y+(1<<d));
+ }
+ }
+ }
+ gmx_fio_fclose(out);
+ sfree(grid_r);
+ }
+}
+
+void write_dd_pdb(const char *fn, gmx_large_int_t step, const char *title,
+ gmx_mtop_t *mtop, t_commrec *cr,
+ int natoms, rvec x[], matrix box)
+{
+ char fname[STRLEN], format[STRLEN], format4[STRLEN], buf[22];
+ FILE *out;
+ int i, ii, resnr, c;
+ char *atomname, *resname;
+ real b;
+ gmx_domdec_t *dd;
+
+ dd = cr->dd;
+ if (natoms == -1)
+ {
+ natoms = dd->comm->nat[ddnatVSITE];
+ }
+
+ sprintf(fname, "%s_%s_n%d.pdb", fn, gmx_step_str(step, buf), cr->sim_nodeid);
+
+ sprintf(format, "%s%s\n", get_pdbformat(), "%6.2f%6.2f");
+ sprintf(format4, "%s%s\n", get_pdbformat4(), "%6.2f%6.2f");
+
+ out = gmx_fio_fopen(fname, "w");
+
+ fprintf(out, "TITLE %s\n", title);
+ gmx_write_pdb_box(out, dd->bScrewPBC ? epbcSCREW : epbcXYZ, box);
+ for (i = 0; i < natoms; i++)
+ {
+ ii = dd->gatindex[i];
+ gmx_mtop_atominfo_global(mtop, ii, &atomname, &resnr, &resname);
+ if (i < dd->comm->nat[ddnatZONE])
+ {
+ c = 0;
+ while (i >= dd->cgindex[dd->comm->zones.cg_range[c+1]])
+ {
+ c++;
+ }
+ b = c;
+ }
+ else if (i < dd->comm->nat[ddnatVSITE])
+ {
+ b = dd->comm->zones.n;
+ }
+ else
+ {
+ b = dd->comm->zones.n + 1;
+ }
+ fprintf(out, strlen(atomname) < 4 ? format : format4,
+ "ATOM", (ii+1)%100000,
+ atomname, resname, ' ', resnr%10000, ' ',
+ 10*x[i][XX], 10*x[i][YY], 10*x[i][ZZ], 1.0, b);
+ }
+ fprintf(out, "TER\n");
+
+ gmx_fio_fclose(out);
+}
+
+real dd_cutoff_mbody(gmx_domdec_t *dd)
+{
+ gmx_domdec_comm_t *comm;
+ int di;
+ real r;
+
+ comm = dd->comm;
+
+ r = -1;
+ if (comm->bInterCGBondeds)
+ {
+ if (comm->cutoff_mbody > 0)
+ {
+ r = comm->cutoff_mbody;
+ }
+ else
+ {
+ /* cutoff_mbody=0 means we do not have DLB */
+ r = comm->cellsize_min[dd->dim[0]];
+ for (di = 1; di < dd->ndim; di++)
+ {
+ r = min(r, comm->cellsize_min[dd->dim[di]]);
+ }
+ if (comm->bBondComm)
+ {
+ r = max(r, comm->cutoff_mbody);
+ }
+ else
+ {
+ r = min(r, comm->cutoff);
+ }
+ }
+ }
+
+ return r;
+}
+
+real dd_cutoff_twobody(gmx_domdec_t *dd)
+{
+ real r_mb;
+
+ r_mb = dd_cutoff_mbody(dd);
+
+ return max(dd->comm->cutoff, r_mb);
+}
+
+
+static void dd_cart_coord2pmecoord(gmx_domdec_t *dd, ivec coord, ivec coord_pme)
+{
+ int nc, ntot;
+
+ nc = dd->nc[dd->comm->cartpmedim];
+ ntot = dd->comm->ntot[dd->comm->cartpmedim];
+ copy_ivec(coord, coord_pme);
+ coord_pme[dd->comm->cartpmedim] =
+ nc + (coord[dd->comm->cartpmedim]*(ntot - nc) + (ntot - nc)/2)/nc;
+}
+
+static int low_ddindex2pmeindex(int ndd, int npme, int ddindex)
+{
+ /* Here we assign a PME node to communicate with this DD node
+ * by assuming that the major index of both is x.
+ * We add cr->npmenodes/2 to obtain an even distribution.
+ */
+ return (ddindex*npme + npme/2)/ndd;
+}
+
+static int ddindex2pmeindex(const gmx_domdec_t *dd, int ddindex)
+{
+ return low_ddindex2pmeindex(dd->nnodes, dd->comm->npmenodes, ddindex);
+}
+
+static int cr_ddindex2pmeindex(const t_commrec *cr, int ddindex)
+{
+ return low_ddindex2pmeindex(cr->dd->nnodes, cr->npmenodes, ddindex);
+}
+
+static int *dd_pmenodes(t_commrec *cr)
+{
+ int *pmenodes;
+ int n, i, p0, p1;
+
+ snew(pmenodes, cr->npmenodes);
+ n = 0;
+ for (i = 0; i < cr->dd->nnodes; i++)
+ {
+ p0 = cr_ddindex2pmeindex(cr, i);
+ p1 = cr_ddindex2pmeindex(cr, i+1);
+ if (i+1 == cr->dd->nnodes || p1 > p0)
+ {
+ if (debug)
+ {
+ fprintf(debug, "pmenode[%d] = %d\n", n, i+1+n);
+ }
+ pmenodes[n] = i + 1 + n;
+ n++;
+ }
+ }
+
+ return pmenodes;
+}
+
+static int gmx_ddcoord2pmeindex(t_commrec *cr, int x, int y, int z)
+{
+ gmx_domdec_t *dd;
+ ivec coords, coords_pme, nc;
+ int slab;
+
+ dd = cr->dd;
+ /*
+ if (dd->comm->bCartesian) {
+ gmx_ddindex2xyz(dd->nc,ddindex,coords);
+ dd_coords2pmecoords(dd,coords,coords_pme);
+ copy_ivec(dd->ntot,nc);
+ nc[dd->cartpmedim] -= dd->nc[dd->cartpmedim];
+ coords_pme[dd->cartpmedim] -= dd->nc[dd->cartpmedim];
+
+ slab = (coords_pme[XX]*nc[YY] + coords_pme[YY])*nc[ZZ] + coords_pme[ZZ];
+ } else {
+ slab = (ddindex*cr->npmenodes + cr->npmenodes/2)/dd->nnodes;
+ }
+ */
+ coords[XX] = x;
+ coords[YY] = y;
+ coords[ZZ] = z;
+ slab = ddindex2pmeindex(dd, dd_index(dd->nc, coords));
+
+ return slab;
+}
+
+static int ddcoord2simnodeid(t_commrec *cr, int x, int y, int z)
+{
+ gmx_domdec_comm_t *comm;
+ ivec coords;
+ int ddindex, nodeid = -1;
+
+ comm = cr->dd->comm;
+
+ coords[XX] = x;
+ coords[YY] = y;
+ coords[ZZ] = z;
+ if (comm->bCartesianPP_PME)
+ {
+#ifdef GMX_MPI
+ MPI_Cart_rank(cr->mpi_comm_mysim, coords, &nodeid);
+#endif
+ }
+ else
+ {
+ ddindex = dd_index(cr->dd->nc, coords);
+ if (comm->bCartesianPP)
+ {
+ nodeid = comm->ddindex2simnodeid[ddindex];
+ }
+ else
+ {
+ if (comm->pmenodes)
+ {
+ nodeid = ddindex + gmx_ddcoord2pmeindex(cr, x, y, z);
+ }
+ else
+ {
+ nodeid = ddindex;
+ }
+ }
+ }
+
+ return nodeid;
+}
+
+static int dd_simnode2pmenode(t_commrec *cr, int sim_nodeid)
+{
+ gmx_domdec_t *dd;
+ gmx_domdec_comm_t *comm;
+ ivec coord, coord_pme;
+ int i;
+ int pmenode = -1;
+
+ dd = cr->dd;
+ comm = dd->comm;
+
+ /* This assumes a uniform x domain decomposition grid cell size */
+ if (comm->bCartesianPP_PME)
+ {
+#ifdef GMX_MPI
+ MPI_Cart_coords(cr->mpi_comm_mysim, sim_nodeid, DIM, coord);
+ if (coord[comm->cartpmedim] < dd->nc[comm->cartpmedim])
+ {
+ /* This is a PP node */
+ dd_cart_coord2pmecoord(dd, coord, coord_pme);
+ MPI_Cart_rank(cr->mpi_comm_mysim, coord_pme, &pmenode);
+ }
+#endif
+ }
+ else if (comm->bCartesianPP)
+ {
+ if (sim_nodeid < dd->nnodes)
+ {
+ pmenode = dd->nnodes + ddindex2pmeindex(dd, sim_nodeid);
+ }
+ }
+ else
+ {
+ /* This assumes DD cells with identical x coordinates
+ * are numbered sequentially.
+ */
+ if (dd->comm->pmenodes == NULL)
+ {
+ if (sim_nodeid < dd->nnodes)
+ {
+ /* The DD index equals the nodeid */
+ pmenode = dd->nnodes + ddindex2pmeindex(dd, sim_nodeid);
+ }
+ }
+ else
+ {
+ i = 0;
+ while (sim_nodeid > dd->comm->pmenodes[i])
+ {
+ i++;
+ }
+ if (sim_nodeid < dd->comm->pmenodes[i])
+ {
+ pmenode = dd->comm->pmenodes[i];
+ }
+ }
+ }
+
+ return pmenode;
+}
+
+void get_pme_nnodes(const gmx_domdec_t *dd,
+ int *npmenodes_x, int *npmenodes_y)
+{
+ if (dd != NULL)
+ {
+ *npmenodes_x = dd->comm->npmenodes_x;
+ *npmenodes_y = dd->comm->npmenodes_y;
+ }
+ else
+ {
+ *npmenodes_x = 1;
+ *npmenodes_y = 1;
+ }
+}
+
+gmx_bool gmx_pmeonlynode(t_commrec *cr, int sim_nodeid)
+{
+ gmx_bool bPMEOnlyNode;
+
+ if (DOMAINDECOMP(cr))
+ {
+ bPMEOnlyNode = (dd_simnode2pmenode(cr, sim_nodeid) == -1);
+ }
+ else
+ {
+ bPMEOnlyNode = FALSE;
+ }
+
+ return bPMEOnlyNode;
+}
+
+void get_pme_ddnodes(t_commrec *cr, int pmenodeid,
+ int *nmy_ddnodes, int **my_ddnodes, int *node_peer)
+{
+ gmx_domdec_t *dd;
+ int x, y, z;
+ ivec coord, coord_pme;
+
+ dd = cr->dd;
+
+ snew(*my_ddnodes, (dd->nnodes+cr->npmenodes-1)/cr->npmenodes);
+
+ *nmy_ddnodes = 0;
+ for (x = 0; x < dd->nc[XX]; x++)
+ {
+ for (y = 0; y < dd->nc[YY]; y++)
+ {
+ for (z = 0; z < dd->nc[ZZ]; z++)
+ {
+ if (dd->comm->bCartesianPP_PME)
+ {
+ coord[XX] = x;
+ coord[YY] = y;
+ coord[ZZ] = z;
+ dd_cart_coord2pmecoord(dd, coord, coord_pme);
+ if (dd->ci[XX] == coord_pme[XX] &&
+ dd->ci[YY] == coord_pme[YY] &&
+ dd->ci[ZZ] == coord_pme[ZZ])
+ {
+ (*my_ddnodes)[(*nmy_ddnodes)++] = ddcoord2simnodeid(cr, x, y, z);
+ }
+ }
+ else
+ {
+ /* The slab corresponds to the nodeid in the PME group */
+ if (gmx_ddcoord2pmeindex(cr, x, y, z) == pmenodeid)
+ {
+ (*my_ddnodes)[(*nmy_ddnodes)++] = ddcoord2simnodeid(cr, x, y, z);
+ }
+ }
+ }
+ }
+ }
+
+ /* The last PP-only node is the peer node */
+ *node_peer = (*my_ddnodes)[*nmy_ddnodes-1];
+
+ if (debug)
+ {
+ fprintf(debug, "Receive coordinates from PP nodes:");
+ for (x = 0; x < *nmy_ddnodes; x++)
+ {
+ fprintf(debug, " %d", (*my_ddnodes)[x]);
+ }
+ fprintf(debug, "\n");
+ }
+}
+
+static gmx_bool receive_vir_ener(t_commrec *cr)
+{
+ gmx_domdec_comm_t *comm;
+ int pmenode, coords[DIM], rank;
+ gmx_bool bReceive;
+
+ bReceive = TRUE;
+ if (cr->npmenodes < cr->dd->nnodes)
+ {
+ comm = cr->dd->comm;
+ if (comm->bCartesianPP_PME)
+ {
+ pmenode = dd_simnode2pmenode(cr, cr->sim_nodeid);
+#ifdef GMX_MPI
+ MPI_Cart_coords(cr->mpi_comm_mysim, cr->sim_nodeid, DIM, coords);
+ coords[comm->cartpmedim]++;
+ if (coords[comm->cartpmedim] < cr->dd->nc[comm->cartpmedim])
+ {
+ MPI_Cart_rank(cr->mpi_comm_mysim, coords, &rank);
+ if (dd_simnode2pmenode(cr, rank) == pmenode)
+ {
+ /* This is not the last PP node for pmenode */
+ bReceive = FALSE;
+ }
+ }
+#endif
+ }
+ else
+ {
+ pmenode = dd_simnode2pmenode(cr, cr->sim_nodeid);
+ if (cr->sim_nodeid+1 < cr->nnodes &&
+ dd_simnode2pmenode(cr, cr->sim_nodeid+1) == pmenode)
+ {
+ /* This is not the last PP node for pmenode */
+ bReceive = FALSE;
+ }
+ }
+ }
+
+ return bReceive;
+}
+
+static void set_zones_ncg_home(gmx_domdec_t *dd)
+{
+ gmx_domdec_zones_t *zones;
+ int i;
+
+ zones = &dd->comm->zones;
+
+ zones->cg_range[0] = 0;
+ for (i = 1; i < zones->n+1; i++)
+ {
+ zones->cg_range[i] = dd->ncg_home;
+ }
+ /* zone_ncg1[0] should always be equal to ncg_home */
+ dd->comm->zone_ncg1[0] = dd->ncg_home;
+}
+
+static void rebuild_cgindex(gmx_domdec_t *dd,
+ const int *gcgs_index, t_state *state)
+{
+ int nat, i, *ind, *dd_cg_gl, *cgindex, cg_gl;
+
+ ind = state->cg_gl;
+ dd_cg_gl = dd->index_gl;
+ cgindex = dd->cgindex;
+ nat = 0;
+ cgindex[0] = nat;
+ for (i = 0; i < state->ncg_gl; i++)
+ {
+ cgindex[i] = nat;
+ cg_gl = ind[i];
+ dd_cg_gl[i] = cg_gl;
+ nat += gcgs_index[cg_gl+1] - gcgs_index[cg_gl];
+ }
+ cgindex[i] = nat;
+
+ dd->ncg_home = state->ncg_gl;
+ dd->nat_home = nat;
+
+ set_zones_ncg_home(dd);
+}
+
+static int ddcginfo(const cginfo_mb_t *cginfo_mb, int cg)
+{
+ while (cg >= cginfo_mb->cg_end)
+ {
+ cginfo_mb++;
+ }
+
+ return cginfo_mb->cginfo[(cg - cginfo_mb->cg_start) % cginfo_mb->cg_mod];
+}
+
+static void dd_set_cginfo(int *index_gl, int cg0, int cg1,
+ t_forcerec *fr, char *bLocalCG)
+{
+ cginfo_mb_t *cginfo_mb;
+ int *cginfo;
+ int cg;
+
+ if (fr != NULL)
+ {
+ cginfo_mb = fr->cginfo_mb;
+ cginfo = fr->cginfo;
+
+ for (cg = cg0; cg < cg1; cg++)
+ {
+ cginfo[cg] = ddcginfo(cginfo_mb, index_gl[cg]);
+ }
+ }
+
+ if (bLocalCG != NULL)
+ {
+ for (cg = cg0; cg < cg1; cg++)
+ {
+ bLocalCG[index_gl[cg]] = TRUE;
+ }
+ }
+}
+
+static void make_dd_indices(gmx_domdec_t *dd,
+ const int *gcgs_index, int cg_start)
+{
+ int nzone, zone, zone1, cg0, cg1, cg1_p1, cg, cg_gl, a, a_gl;
+ int *zone2cg, *zone_ncg1, *index_gl, *gatindex;
+ gmx_ga2la_t *ga2la;
+ char *bLocalCG;
+ gmx_bool bCGs;
+
+ bLocalCG = dd->comm->bLocalCG;
+
+ if (dd->nat_tot > dd->gatindex_nalloc)
+ {
+ dd->gatindex_nalloc = over_alloc_dd(dd->nat_tot);
+ srenew(dd->gatindex, dd->gatindex_nalloc);
+ }
+
+ nzone = dd->comm->zones.n;
+ zone2cg = dd->comm->zones.cg_range;
+ zone_ncg1 = dd->comm->zone_ncg1;
+ index_gl = dd->index_gl;
+ gatindex = dd->gatindex;
+ bCGs = dd->comm->bCGs;
+
+ if (zone2cg[1] != dd->ncg_home)
+ {
+ gmx_incons("dd->ncg_zone is not up to date");
+ }
+
+ /* Make the local to global and global to local atom index */
+ a = dd->cgindex[cg_start];
+ for (zone = 0; zone < nzone; zone++)
+ {
+ if (zone == 0)
+ {
+ cg0 = cg_start;
+ }
+ else
+ {
+ cg0 = zone2cg[zone];
+ }
+ cg1 = zone2cg[zone+1];
+ cg1_p1 = cg0 + zone_ncg1[zone];
+
+ for (cg = cg0; cg < cg1; cg++)
+ {
+ zone1 = zone;
+ if (cg >= cg1_p1)
+ {
+ /* Signal that this cg is from more than one pulse away */
+ zone1 += nzone;
+ }
+ cg_gl = index_gl[cg];
+ if (bCGs)
+ {
+ for (a_gl = gcgs_index[cg_gl]; a_gl < gcgs_index[cg_gl+1]; a_gl++)
+ {
+ gatindex[a] = a_gl;
+ ga2la_set(dd->ga2la, a_gl, a, zone1);
+ a++;
+ }
+ }
+ else
+ {
+ gatindex[a] = cg_gl;
+ ga2la_set(dd->ga2la, cg_gl, a, zone1);
+ a++;
+ }
+ }
+ }
+}
+
+static int check_bLocalCG(gmx_domdec_t *dd, int ncg_sys, const char *bLocalCG,
+ const char *where)
+{
+ int ncg, i, ngl, nerr;
+
+ nerr = 0;
+ if (bLocalCG == NULL)
+ {
+ return nerr;
+ }
+ for (i = 0; i < dd->ncg_tot; i++)
+ {
+ if (!bLocalCG[dd->index_gl[i]])
+ {
+ fprintf(stderr,
+ "DD node %d, %s: cg %d, global cg %d is not marked in bLocalCG (ncg_home %d)\n", dd->rank, where, i+1, dd->index_gl[i]+1, dd->ncg_home);
+ nerr++;
+ }
+ }
+ ngl = 0;
+ for (i = 0; i < ncg_sys; i++)
+ {
+ if (bLocalCG[i])
+ {
+ ngl++;
+ }
+ }
+ if (ngl != dd->ncg_tot)
+ {
+ fprintf(stderr, "DD node %d, %s: In bLocalCG %d cgs are marked as local, whereas there are %d\n", dd->rank, where, ngl, dd->ncg_tot);
+ nerr++;
+ }
+
+ return nerr;
+}
+
+static void check_index_consistency(gmx_domdec_t *dd,
+ int natoms_sys, int ncg_sys,
+ const char *where)
+{
+ int nerr, ngl, i, a, cell;
+ int *have;
+
+ nerr = 0;
+
+ if (dd->comm->DD_debug > 1)
+ {
+ snew(have, natoms_sys);
+ for (a = 0; a < dd->nat_tot; a++)
+ {
+ if (have[dd->gatindex[a]] > 0)
+ {
+ fprintf(stderr, "DD node %d: global atom %d occurs twice: index %d and %d\n", dd->rank, dd->gatindex[a]+1, have[dd->gatindex[a]], a+1);
+ }
+ else
+ {
+ have[dd->gatindex[a]] = a + 1;
+ }
+ }
+ sfree(have);
+ }
+
+ snew(have, dd->nat_tot);
+
+ ngl = 0;
+ for (i = 0; i < natoms_sys; i++)
+ {
+ if (ga2la_get(dd->ga2la, i, &a, &cell))
+ {
+ if (a >= dd->nat_tot)
+ {
+ fprintf(stderr, "DD node %d: global atom %d marked as local atom %d, which is larger than nat_tot (%d)\n", dd->rank, i+1, a+1, dd->nat_tot);
+ nerr++;
+ }
+ else
+ {
+ have[a] = 1;
+ if (dd->gatindex[a] != i)
+ {
+ fprintf(stderr, "DD node %d: global atom %d marked as local atom %d, which has global atom index %d\n", dd->rank, i+1, a+1, dd->gatindex[a]+1);
+ nerr++;
+ }
+ }
+ ngl++;
+ }
+ }
+ if (ngl != dd->nat_tot)
+ {
+ fprintf(stderr,
+ "DD node %d, %s: %d global atom indices, %d local atoms\n",
+ dd->rank, where, ngl, dd->nat_tot);
+ }
+ for (a = 0; a < dd->nat_tot; a++)
+ {
+ if (have[a] == 0)
+ {
+ fprintf(stderr,
+ "DD node %d, %s: local atom %d, global %d has no global index\n",
+ dd->rank, where, a+1, dd->gatindex[a]+1);
+ }
+ }
+ sfree(have);
+
+ nerr += check_bLocalCG(dd, ncg_sys, dd->comm->bLocalCG, where);
+
+ if (nerr > 0)
+ {
+ gmx_fatal(FARGS, "DD node %d, %s: %d atom/cg index inconsistencies",
+ dd->rank, where, nerr);
+ }
+}
+
+static void clear_dd_indices(gmx_domdec_t *dd, int cg_start, int a_start)
+{
+ int i;
+ char *bLocalCG;
+
+ if (a_start == 0)
+ {
+ /* Clear the whole list without searching */
+ ga2la_clear(dd->ga2la);
+ }
+ else
+ {
+ for (i = a_start; i < dd->nat_tot; i++)
+ {
+ ga2la_del(dd->ga2la, dd->gatindex[i]);
+ }
+ }
+
+ bLocalCG = dd->comm->bLocalCG;
+ if (bLocalCG)
+ {
+ for (i = cg_start; i < dd->ncg_tot; i++)
+ {
+ bLocalCG[dd->index_gl[i]] = FALSE;
+ }
+ }
+
+ dd_clear_local_vsite_indices(dd);
+
+ if (dd->constraints)
+ {
+ dd_clear_local_constraint_indices(dd);
+ }
+}
+
+/* This function should be used for moving the domain boudaries during DLB,
+ * for obtaining the minimum cell size. It checks the initially set limit
+ * comm->cellsize_min, for bonded and initial non-bonded cut-offs,
+ * and, possibly, a longer cut-off limit set for PME load balancing.
+ */
+static real cellsize_min_dlb(gmx_domdec_comm_t *comm, int dim_ind, int dim)
+{
+ real cellsize_min;
+
+ cellsize_min = comm->cellsize_min[dim];
+
+ if (!comm->bVacDLBNoLimit)
+ {
+ /* The cut-off might have changed, e.g. by PME load balacning,
+ * from the value used to set comm->cellsize_min, so check it.
+ */
+ cellsize_min = max(cellsize_min, comm->cutoff/comm->cd[dim_ind].np_dlb);
+
+ if (comm->bPMELoadBalDLBLimits)
+ {
+ /* Check for the cut-off limit set by the PME load balancing */
+ cellsize_min = max(cellsize_min, comm->PMELoadBal_max_cutoff/comm->cd[dim_ind].np_dlb);
+ }
+ }
+
+ return cellsize_min;
+}
+
+static real grid_jump_limit(gmx_domdec_comm_t *comm, real cutoff,
+ int dim_ind)
+{
+ real grid_jump_limit;
+
+ /* The distance between the boundaries of cells at distance
+ * x+-1,y+-1 or y+-1,z+-1 is limited by the cut-off restrictions
+ * and by the fact that cells should not be shifted by more than
+ * half their size, such that cg's only shift by one cell
+ * at redecomposition.
+ */
+ grid_jump_limit = comm->cellsize_limit;
+ if (!comm->bVacDLBNoLimit)
+ {
+ if (comm->bPMELoadBalDLBLimits)
+ {
+ cutoff = max(cutoff, comm->PMELoadBal_max_cutoff);
+ }
+ grid_jump_limit = max(grid_jump_limit,
+ cutoff/comm->cd[dim_ind].np);
+ }
+
+ return grid_jump_limit;
+}
+
+static gmx_bool check_grid_jump(gmx_large_int_t step,
+ gmx_domdec_t *dd,
+ real cutoff,
+ gmx_ddbox_t *ddbox,
+ gmx_bool bFatal)
+{
+ gmx_domdec_comm_t *comm;
+ int d, dim;
+ real limit, bfac;
+ gmx_bool bInvalid;
+
+ bInvalid = FALSE;
+
+ comm = dd->comm;
+
+ for (d = 1; d < dd->ndim; d++)
+ {
+ dim = dd->dim[d];
+ limit = grid_jump_limit(comm, cutoff, d);
+ bfac = ddbox->box_size[dim];
+ if (ddbox->tric_dir[dim])
+ {
+ bfac *= ddbox->skew_fac[dim];
+ }
+ if ((comm->cell_f1[d] - comm->cell_f_max0[d])*bfac < limit ||
+ (comm->cell_f0[d] - comm->cell_f_min1[d])*bfac > -limit)
+ {
+ bInvalid = TRUE;
+
+ if (bFatal)
+ {
+ char buf[22];
+
+ /* This error should never be triggered under normal
+ * circumstances, but you never know ...
+ */
+ gmx_fatal(FARGS, "Step %s: The domain decomposition grid has shifted too much in the %c-direction around cell %d %d %d. This should not have happened. Running with less nodes might avoid this issue.",
+ gmx_step_str(step, buf),
+ dim2char(dim), dd->ci[XX], dd->ci[YY], dd->ci[ZZ]);
+ }
+ }
+ }
+
+ return bInvalid;
+}
+
+static int dd_load_count(gmx_domdec_comm_t *comm)
+{
+ return (comm->eFlop ? comm->flop_n : comm->cycl_n[ddCyclF]);
+}
+
+static float dd_force_load(gmx_domdec_comm_t *comm)
+{
+ float load;
+
+ if (comm->eFlop)
+ {
+ load = comm->flop;
+ if (comm->eFlop > 1)
+ {
+ load *= 1.0 + (comm->eFlop - 1)*(0.1*rand()/RAND_MAX - 0.05);
+ }
+ }
+ else
+ {
+ load = comm->cycl[ddCyclF];
+ if (comm->cycl_n[ddCyclF] > 1)
+ {
+ /* Subtract the maximum of the last n cycle counts
+ * to get rid of possible high counts due to other soures,
+ * for instance system activity, that would otherwise
+ * affect the dynamic load balancing.
+ */
+ load -= comm->cycl_max[ddCyclF];
+ }
+ }
+
+ return load;
+}
+
+static void set_slb_pme_dim_f(gmx_domdec_t *dd, int dim, real **dim_f)
+{
+ gmx_domdec_comm_t *comm;
+ int i;
+
+ comm = dd->comm;
+
+ snew(*dim_f, dd->nc[dim]+1);
+ (*dim_f)[0] = 0;
+ for (i = 1; i < dd->nc[dim]; i++)
+ {
+ if (comm->slb_frac[dim])
+ {
+ (*dim_f)[i] = (*dim_f)[i-1] + comm->slb_frac[dim][i-1];
+ }
+ else
+ {
+ (*dim_f)[i] = (real)i/(real)dd->nc[dim];
+ }
+ }
+ (*dim_f)[dd->nc[dim]] = 1;
+}
+
+static void init_ddpme(gmx_domdec_t *dd, gmx_ddpme_t *ddpme, int dimind)
+{
+ int pmeindex, slab, nso, i;
+ ivec xyz;
+
+ if (dimind == 0 && dd->dim[0] == YY && dd->comm->npmenodes_x == 1)
+ {
+ ddpme->dim = YY;
+ }
+ else
+ {
+ ddpme->dim = dimind;
+ }
+ ddpme->dim_match = (ddpme->dim == dd->dim[dimind]);
+
+ ddpme->nslab = (ddpme->dim == 0 ?
+ dd->comm->npmenodes_x :
+ dd->comm->npmenodes_y);
+
+ if (ddpme->nslab <= 1)
+ {
+ return;
+ }
+
+ nso = dd->comm->npmenodes/ddpme->nslab;
+ /* Determine for each PME slab the PP location range for dimension dim */
+ snew(ddpme->pp_min, ddpme->nslab);
+ snew(ddpme->pp_max, ddpme->nslab);
+ for (slab = 0; slab < ddpme->nslab; slab++)
+ {
+ ddpme->pp_min[slab] = dd->nc[dd->dim[dimind]] - 1;
+ ddpme->pp_max[slab] = 0;
+ }
+ for (i = 0; i < dd->nnodes; i++)
+ {
+ ddindex2xyz(dd->nc, i, xyz);
+ /* For y only use our y/z slab.
+ * This assumes that the PME x grid size matches the DD grid size.
+ */
+ if (dimind == 0 || xyz[XX] == dd->ci[XX])
+ {
+ pmeindex = ddindex2pmeindex(dd, i);
+ if (dimind == 0)
+ {
+ slab = pmeindex/nso;
+ }
+ else
+ {
+ slab = pmeindex % ddpme->nslab;
+ }
+ ddpme->pp_min[slab] = min(ddpme->pp_min[slab], xyz[dimind]);
+ ddpme->pp_max[slab] = max(ddpme->pp_max[slab], xyz[dimind]);
+ }
+ }
+
+ set_slb_pme_dim_f(dd, ddpme->dim, &ddpme->slb_dim_f);
+}
+
+int dd_pme_maxshift_x(gmx_domdec_t *dd)
+{
+ if (dd->comm->ddpme[0].dim == XX)
+ {
+ return dd->comm->ddpme[0].maxshift;
+ }
+ else
+ {
+ return 0;
+ }
+}
+
+int dd_pme_maxshift_y(gmx_domdec_t *dd)
+{
+ if (dd->comm->ddpme[0].dim == YY)
+ {
+ return dd->comm->ddpme[0].maxshift;
+ }
+ else if (dd->comm->npmedecompdim >= 2 && dd->comm->ddpme[1].dim == YY)
+ {
+ return dd->comm->ddpme[1].maxshift;
+ }
+ else
+ {
+ return 0;
+ }
+}
+
+static void set_pme_maxshift(gmx_domdec_t *dd, gmx_ddpme_t *ddpme,
+ gmx_bool bUniform, gmx_ddbox_t *ddbox, real *cell_f)
+{
+ gmx_domdec_comm_t *comm;
+ int nc, ns, s;
+ int *xmin, *xmax;
+ real range, pme_boundary;
+ int sh;
+
+ comm = dd->comm;
+ nc = dd->nc[ddpme->dim];
+ ns = ddpme->nslab;
+
+ if (!ddpme->dim_match)
+ {
+ /* PP decomposition is not along dim: the worst situation */
+ sh = ns/2;
+ }
+ else if (ns <= 3 || (bUniform && ns == nc))
+ {
+ /* The optimal situation */
+ sh = 1;
+ }
+ else
+ {
+ /* We need to check for all pme nodes which nodes they
+ * could possibly need to communicate with.
+ */
+ xmin = ddpme->pp_min;
+ xmax = ddpme->pp_max;
+ /* Allow for atoms to be maximally 2/3 times the cut-off
+ * out of their DD cell. This is a reasonable balance between
+ * between performance and support for most charge-group/cut-off
+ * combinations.
+ */
+ range = 2.0/3.0*comm->cutoff/ddbox->box_size[ddpme->dim];
+ /* Avoid extra communication when we are exactly at a boundary */
+ range *= 0.999;
+
+ sh = 1;
+ for (s = 0; s < ns; s++)
+ {
+ /* PME slab s spreads atoms between box frac. s/ns and (s+1)/ns */
+ pme_boundary = (real)s/ns;
+ while (sh+1 < ns &&
+ ((s-(sh+1) >= 0 &&
+ cell_f[xmax[s-(sh+1) ]+1] + range > pme_boundary) ||
+ (s-(sh+1) < 0 &&
+ cell_f[xmax[s-(sh+1)+ns]+1] - 1 + range > pme_boundary)))
+ {
+ sh++;
+ }
+ pme_boundary = (real)(s+1)/ns;
+ while (sh+1 < ns &&
+ ((s+(sh+1) < ns &&
+ cell_f[xmin[s+(sh+1) ] ] - range < pme_boundary) ||
+ (s+(sh+1) >= ns &&
+ cell_f[xmin[s+(sh+1)-ns] ] + 1 - range < pme_boundary)))
+ {
+ sh++;
+ }
+ }
+ }
+
+ ddpme->maxshift = sh;
+
+ if (debug)
+ {
+ fprintf(debug, "PME slab communication range for dim %d is %d\n",
+ ddpme->dim, ddpme->maxshift);
+ }
+}
+
+static void check_box_size(gmx_domdec_t *dd, gmx_ddbox_t *ddbox)
+{
+ int d, dim;
+
+ for (d = 0; d < dd->ndim; d++)
+ {
+ dim = dd->dim[d];
+ if (dim < ddbox->nboundeddim &&
+ ddbox->box_size[dim]*ddbox->skew_fac[dim] <
+ dd->nc[dim]*dd->comm->cellsize_limit*DD_CELL_MARGIN)
+ {
+ gmx_fatal(FARGS, "The %c-size of the box (%f) times the triclinic skew factor (%f) is smaller than the number of DD cells (%d) times the smallest allowed cell size (%f)\n",
+ dim2char(dim), ddbox->box_size[dim], ddbox->skew_fac[dim],
+ dd->nc[dim], dd->comm->cellsize_limit);
+ }
+ }
+}
+
+static void set_dd_cell_sizes_slb(gmx_domdec_t *dd, gmx_ddbox_t *ddbox,
+ gmx_bool bMaster, ivec npulse)
+{
+ gmx_domdec_comm_t *comm;
+ int d, j;
+ rvec cellsize_min;
+ real *cell_x, cell_dx, cellsize;
+
+ comm = dd->comm;
+
+ for (d = 0; d < DIM; d++)
+ {
+ cellsize_min[d] = ddbox->box_size[d]*ddbox->skew_fac[d];
+ npulse[d] = 1;
+ if (dd->nc[d] == 1 || comm->slb_frac[d] == NULL)
+ {
+ /* Uniform grid */
+ cell_dx = ddbox->box_size[d]/dd->nc[d];
+ if (bMaster)
+ {
+ for (j = 0; j < dd->nc[d]+1; j++)
+ {
+ dd->ma->cell_x[d][j] = ddbox->box0[d] + j*cell_dx;
+ }
+ }
+ else
+ {
+ comm->cell_x0[d] = ddbox->box0[d] + (dd->ci[d] )*cell_dx;
+ comm->cell_x1[d] = ddbox->box0[d] + (dd->ci[d]+1)*cell_dx;
+ }
+ cellsize = cell_dx*ddbox->skew_fac[d];
+ while (cellsize*npulse[d] < comm->cutoff && npulse[d] < dd->nc[d]-1)
+ {
+ npulse[d]++;
+ }
+ cellsize_min[d] = cellsize;
+ }
+ else
+ {
+ /* Statically load balanced grid */
+ /* Also when we are not doing a master distribution we determine
+ * all cell borders in a loop to obtain identical values
+ * to the master distribution case and to determine npulse.
+ */
+ if (bMaster)
+ {
+ cell_x = dd->ma->cell_x[d];
+ }
+ else
+ {
+ snew(cell_x, dd->nc[d]+1);
+ }
+ cell_x[0] = ddbox->box0[d];
+ for (j = 0; j < dd->nc[d]; j++)
+ {
+ cell_dx = ddbox->box_size[d]*comm->slb_frac[d][j];
+ cell_x[j+1] = cell_x[j] + cell_dx;
+ cellsize = cell_dx*ddbox->skew_fac[d];
+ while (cellsize*npulse[d] < comm->cutoff &&
+ npulse[d] < dd->nc[d]-1)
+ {
+ npulse[d]++;
+ }
+ cellsize_min[d] = min(cellsize_min[d], cellsize);
+ }
+ if (!bMaster)
+ {
+ comm->cell_x0[d] = cell_x[dd->ci[d]];
+ comm->cell_x1[d] = cell_x[dd->ci[d]+1];
+ sfree(cell_x);
+ }
+ }
+ /* The following limitation is to avoid that a cell would receive
+ * some of its own home charge groups back over the periodic boundary.
+ * Double charge groups cause trouble with the global indices.
+ */
+ if (d < ddbox->npbcdim &&
+ dd->nc[d] > 1 && npulse[d] >= dd->nc[d])
+ {
+ gmx_fatal_collective(FARGS, NULL, dd,
+ "The box size in direction %c (%f) times the triclinic skew factor (%f) is too small for a cut-off of %f with %d domain decomposition cells, use 1 or more than %d %s or increase the box size in this direction",
+ dim2char(d), ddbox->box_size[d], ddbox->skew_fac[d],
+ comm->cutoff,
+ dd->nc[d], dd->nc[d],
+ dd->nnodes > dd->nc[d] ? "cells" : "processors");
+ }
+ }
+
+ if (!comm->bDynLoadBal)
+ {
+ copy_rvec(cellsize_min, comm->cellsize_min);
+ }
+
+ for (d = 0; d < comm->npmedecompdim; d++)
+ {
+ set_pme_maxshift(dd, &comm->ddpme[d],
+ comm->slb_frac[dd->dim[d]] == NULL, ddbox,
+ comm->ddpme[d].slb_dim_f);
+ }
+}
+
+
+static void dd_cell_sizes_dlb_root_enforce_limits(gmx_domdec_t *dd,
+ int d, int dim, gmx_domdec_root_t *root,
+ gmx_ddbox_t *ddbox,
+ gmx_bool bUniform, gmx_large_int_t step, real cellsize_limit_f, int range[])
+{
+ gmx_domdec_comm_t *comm;
+ int ncd, i, j, nmin, nmin_old;
+ gmx_bool bLimLo, bLimHi;
+ real *cell_size;
+ real fac, halfway, cellsize_limit_f_i, region_size;
+ gmx_bool bPBC, bLastHi = FALSE;
+ int nrange[] = {range[0], range[1]};
+
+ region_size = root->cell_f[range[1]]-root->cell_f[range[0]];
+
+ comm = dd->comm;
+
+ ncd = dd->nc[dim];
+
+ bPBC = (dim < ddbox->npbcdim);
+
+ cell_size = root->buf_ncd;
+
+ if (debug)
+ {
+ fprintf(debug, "enforce_limits: %d %d\n", range[0], range[1]);
+ }
+
+ /* First we need to check if the scaling does not make cells
+ * smaller than the smallest allowed size.
+ * We need to do this iteratively, since if a cell is too small,
+ * it needs to be enlarged, which makes all the other cells smaller,
+ * which could in turn make another cell smaller than allowed.
+ */
+ for (i = range[0]; i < range[1]; i++)
+ {
+ root->bCellMin[i] = FALSE;
+ }
+ nmin = 0;
+ do
+ {
+ nmin_old = nmin;
+ /* We need the total for normalization */
+ fac = 0;
+ for (i = range[0]; i < range[1]; i++)
+ {
+ if (root->bCellMin[i] == FALSE)
+ {
+ fac += cell_size[i];
+ }
+ }
+ fac = ( region_size - nmin*cellsize_limit_f)/fac; /* substracting cells already set to cellsize_limit_f */
+ /* Determine the cell boundaries */
+ for (i = range[0]; i < range[1]; i++)
+ {
+ if (root->bCellMin[i] == FALSE)
+ {
+ cell_size[i] *= fac;
+ if (!bPBC && (i == 0 || i == dd->nc[dim] -1))
+ {
+ cellsize_limit_f_i = 0;
+ }
+ else
+ {
+ cellsize_limit_f_i = cellsize_limit_f;
+ }
+ if (cell_size[i] < cellsize_limit_f_i)
+ {
+ root->bCellMin[i] = TRUE;
+ cell_size[i] = cellsize_limit_f_i;
+ nmin++;
+ }
+ }
+ root->cell_f[i+1] = root->cell_f[i] + cell_size[i];
+ }
+ }
+ while (nmin > nmin_old);
+
+ i = range[1]-1;
+ cell_size[i] = root->cell_f[i+1] - root->cell_f[i];
+ /* For this check we should not use DD_CELL_MARGIN,
+ * but a slightly smaller factor,
+ * since rounding could get use below the limit.
+ */
+ if (bPBC && cell_size[i] < cellsize_limit_f*DD_CELL_MARGIN2/DD_CELL_MARGIN)
+ {
+ char buf[22];
+ gmx_fatal(FARGS, "Step %s: the dynamic load balancing could not balance dimension %c: box size %f, triclinic skew factor %f, #cells %d, minimum cell size %f\n",
+ gmx_step_str(step, buf),
+ dim2char(dim), ddbox->box_size[dim], ddbox->skew_fac[dim],
+ ncd, comm->cellsize_min[dim]);
+ }
+
+ root->bLimited = (nmin > 0) || (range[0] > 0) || (range[1] < ncd);
+
+ if (!bUniform)
+ {
+ /* Check if the boundary did not displace more than halfway
+ * each of the cells it bounds, as this could cause problems,
+ * especially when the differences between cell sizes are large.
+ * If changes are applied, they will not make cells smaller
+ * than the cut-off, as we check all the boundaries which
+ * might be affected by a change and if the old state was ok,
+ * the cells will at most be shrunk back to their old size.
+ */
+ for (i = range[0]+1; i < range[1]; i++)
+ {
+ halfway = 0.5*(root->old_cell_f[i] + root->old_cell_f[i-1]);
+ if (root->cell_f[i] < halfway)
+ {
+ root->cell_f[i] = halfway;
+ /* Check if the change also causes shifts of the next boundaries */
+ for (j = i+1; j < range[1]; j++)
+ {
+ if (root->cell_f[j] < root->cell_f[j-1] + cellsize_limit_f)
+ {
+ root->cell_f[j] = root->cell_f[j-1] + cellsize_limit_f;
+ }
+ }
+ }
+ halfway = 0.5*(root->old_cell_f[i] + root->old_cell_f[i+1]);
+ if (root->cell_f[i] > halfway)
+ {
+ root->cell_f[i] = halfway;
+ /* Check if the change also causes shifts of the next boundaries */
+ for (j = i-1; j >= range[0]+1; j--)
+ {
+ if (root->cell_f[j] > root->cell_f[j+1] - cellsize_limit_f)
+ {
+ root->cell_f[j] = root->cell_f[j+1] - cellsize_limit_f;
+ }
+ }
+ }
+ }
+ }
+
+ /* nrange is defined as [lower, upper) range for new call to enforce_limits */
+ /* find highest violation of LimLo (a) and the following violation of LimHi (thus the lowest following) (b)
+ * then call enforce_limits for (oldb,a), (a,b). In the next step: (b,nexta). oldb and nexta can be the boundaries.
+ * for a and b nrange is used */
+ if (d > 0)
+ {
+ /* Take care of the staggering of the cell boundaries */
+ if (bUniform)
+ {
+ for (i = range[0]; i < range[1]; i++)
+ {
+ root->cell_f_max0[i] = root->cell_f[i];
+ root->cell_f_min1[i] = root->cell_f[i+1];
+ }
+ }
+ else
+ {
+ for (i = range[0]+1; i < range[1]; i++)
+ {
+ bLimLo = (root->cell_f[i] < root->bound_min[i]);
+ bLimHi = (root->cell_f[i] > root->bound_max[i]);
+ if (bLimLo && bLimHi)
+ {
+ /* Both limits violated, try the best we can */
+ /* For this case we split the original range (range) in two parts and care about the other limitiations in the next iteration. */
+ root->cell_f[i] = 0.5*(root->bound_min[i] + root->bound_max[i]);
+ nrange[0] = range[0];
+ nrange[1] = i;
+ dd_cell_sizes_dlb_root_enforce_limits(dd, d, dim, root, ddbox, bUniform, step, cellsize_limit_f, nrange);
+
+ nrange[0] = i;
+ nrange[1] = range[1];
+ dd_cell_sizes_dlb_root_enforce_limits(dd, d, dim, root, ddbox, bUniform, step, cellsize_limit_f, nrange);
+
+ return;
+ }
+ else if (bLimLo)
+ {
+ /* root->cell_f[i] = root->bound_min[i]; */
+ nrange[1] = i; /* only store violation location. There could be a LimLo violation following with an higher index */
+ bLastHi = FALSE;
+ }
+ else if (bLimHi && !bLastHi)
+ {
+ bLastHi = TRUE;
+ if (nrange[1] < range[1]) /* found a LimLo before */
+ {
+ root->cell_f[nrange[1]] = root->bound_min[nrange[1]];
+ dd_cell_sizes_dlb_root_enforce_limits(dd, d, dim, root, ddbox, bUniform, step, cellsize_limit_f, nrange);
+ nrange[0] = nrange[1];
+ }
+ root->cell_f[i] = root->bound_max[i];
+ nrange[1] = i;
+ dd_cell_sizes_dlb_root_enforce_limits(dd, d, dim, root, ddbox, bUniform, step, cellsize_limit_f, nrange);
+ nrange[0] = i;
+ nrange[1] = range[1];
+ }
+ }
+ if (nrange[1] < range[1]) /* found last a LimLo */
+ {
+ root->cell_f[nrange[1]] = root->bound_min[nrange[1]];
+ dd_cell_sizes_dlb_root_enforce_limits(dd, d, dim, root, ddbox, bUniform, step, cellsize_limit_f, nrange);
+ nrange[0] = nrange[1];
+ nrange[1] = range[1];
+ dd_cell_sizes_dlb_root_enforce_limits(dd, d, dim, root, ddbox, bUniform, step, cellsize_limit_f, nrange);
+ }
+ else if (nrange[0] > range[0]) /* found at least one LimHi */
+ {
+ dd_cell_sizes_dlb_root_enforce_limits(dd, d, dim, root, ddbox, bUniform, step, cellsize_limit_f, nrange);
+ }
+ }
+ }
+}
+
+
+static void set_dd_cell_sizes_dlb_root(gmx_domdec_t *dd,
+ int d, int dim, gmx_domdec_root_t *root,
+ gmx_ddbox_t *ddbox, gmx_bool bDynamicBox,
+ gmx_bool bUniform, gmx_large_int_t step)
+{
+ gmx_domdec_comm_t *comm;
+ int ncd, d1, i, j, pos;
+ real *cell_size;
+ real load_aver, load_i, imbalance, change, change_max, sc;
+ real cellsize_limit_f, dist_min_f, dist_min_f_hard, space;
+ real change_limit;
+ real relax = 0.5;
+ gmx_bool bPBC;
+ int range[] = { 0, 0 };
+
+ comm = dd->comm;
+
+ /* Convert the maximum change from the input percentage to a fraction */
+ change_limit = comm->dlb_scale_lim*0.01;
+
+ ncd = dd->nc[dim];
+
+ bPBC = (dim < ddbox->npbcdim);
+
+ cell_size = root->buf_ncd;
+
+ /* Store the original boundaries */
+ for (i = 0; i < ncd+1; i++)
+ {
+ root->old_cell_f[i] = root->cell_f[i];
+ }
+ if (bUniform)
+ {
+ for (i = 0; i < ncd; i++)
+ {
+ cell_size[i] = 1.0/ncd;
+ }
+ }
+ else if (dd_load_count(comm))
+ {
+ load_aver = comm->load[d].sum_m/ncd;
+ change_max = 0;
+ for (i = 0; i < ncd; i++)
+ {
+ /* Determine the relative imbalance of cell i */
+ load_i = comm->load[d].load[i*comm->load[d].nload+2];
+ imbalance = (load_i - load_aver)/(load_aver > 0 ? load_aver : 1);
+ /* Determine the change of the cell size using underrelaxation */
+ change = -relax*imbalance;
+ change_max = max(change_max, max(change, -change));
+ }
+ /* Limit the amount of scaling.
+ * We need to use the same rescaling for all cells in one row,
+ * otherwise the load balancing might not converge.
+ */
+ sc = relax;
+ if (change_max > change_limit)
+ {
+ sc *= change_limit/change_max;
+ }
+ for (i = 0; i < ncd; i++)
+ {
+ /* Determine the relative imbalance of cell i */
+ load_i = comm->load[d].load[i*comm->load[d].nload+2];
+ imbalance = (load_i - load_aver)/(load_aver > 0 ? load_aver : 1);
+ /* Determine the change of the cell size using underrelaxation */
+ change = -sc*imbalance;
+ cell_size[i] = (root->cell_f[i+1]-root->cell_f[i])*(1 + change);
+ }
+ }
+
+ cellsize_limit_f = cellsize_min_dlb(comm, d, dim)/ddbox->box_size[dim];
+ cellsize_limit_f *= DD_CELL_MARGIN;
+ dist_min_f_hard = grid_jump_limit(comm, comm->cutoff, d)/ddbox->box_size[dim];
+ dist_min_f = dist_min_f_hard * DD_CELL_MARGIN;
+ if (ddbox->tric_dir[dim])
+ {
+ cellsize_limit_f /= ddbox->skew_fac[dim];
+ dist_min_f /= ddbox->skew_fac[dim];
+ }
+ if (bDynamicBox && d > 0)
+ {
+ dist_min_f *= DD_PRES_SCALE_MARGIN;
+ }
+ if (d > 0 && !bUniform)
+ {
+ /* Make sure that the grid is not shifted too much */
+ for (i = 1; i < ncd; i++)
+ {
+ if (root->cell_f_min1[i] - root->cell_f_max0[i-1] < 2 * dist_min_f_hard)
+ {
+ gmx_incons("Inconsistent DD boundary staggering limits!");
+ }
+ root->bound_min[i] = root->cell_f_max0[i-1] + dist_min_f;
+ space = root->cell_f[i] - (root->cell_f_max0[i-1] + dist_min_f);
+ if (space > 0)
+ {
+ root->bound_min[i] += 0.5*space;
+ }
+ root->bound_max[i] = root->cell_f_min1[i] - dist_min_f;
+ space = root->cell_f[i] - (root->cell_f_min1[i] - dist_min_f);
+ if (space < 0)
+ {
+ root->bound_max[i] += 0.5*space;
+ }
+ if (debug)
+ {
+ fprintf(debug,
+ "dim %d boundary %d %.3f < %.3f < %.3f < %.3f < %.3f\n",
+ d, i,
+ root->cell_f_max0[i-1] + dist_min_f,
+ root->bound_min[i], root->cell_f[i], root->bound_max[i],
+ root->cell_f_min1[i] - dist_min_f);
+ }
+ }
+ }
+ range[1] = ncd;
+ root->cell_f[0] = 0;
+ root->cell_f[ncd] = 1;
+ dd_cell_sizes_dlb_root_enforce_limits(dd, d, dim, root, ddbox, bUniform, step, cellsize_limit_f, range);
+
+
+ /* After the checks above, the cells should obey the cut-off
+ * restrictions, but it does not hurt to check.
+ */
+ for (i = 0; i < ncd; i++)
+ {
+ if (debug)
+ {
+ fprintf(debug, "Relative bounds dim %d cell %d: %f %f\n",
+ dim, i, root->cell_f[i], root->cell_f[i+1]);
+ }
+
+ if ((bPBC || (i != 0 && i != dd->nc[dim]-1)) &&
+ root->cell_f[i+1] - root->cell_f[i] <
+ cellsize_limit_f/DD_CELL_MARGIN)
+ {
+ char buf[22];
+ fprintf(stderr,
+ "\nWARNING step %s: direction %c, cell %d too small: %f\n",
+ gmx_step_str(step, buf), dim2char(dim), i,
+ (root->cell_f[i+1] - root->cell_f[i])
+ *ddbox->box_size[dim]*ddbox->skew_fac[dim]);
+ }
+ }
+
+ pos = ncd + 1;
+ /* Store the cell boundaries of the lower dimensions at the end */
+ for (d1 = 0; d1 < d; d1++)
+ {
+ root->cell_f[pos++] = comm->cell_f0[d1];
+ root->cell_f[pos++] = comm->cell_f1[d1];
+ }
+
+ if (d < comm->npmedecompdim)
+ {
+ /* The master determines the maximum shift for
+ * the coordinate communication between separate PME nodes.
+ */
+ set_pme_maxshift(dd, &comm->ddpme[d], bUniform, ddbox, root->cell_f);
+ }
+ root->cell_f[pos++] = comm->ddpme[0].maxshift;
+ if (d >= 1)
+ {
+ root->cell_f[pos++] = comm->ddpme[1].maxshift;
+ }
+}
+
+static void relative_to_absolute_cell_bounds(gmx_domdec_t *dd,
+ gmx_ddbox_t *ddbox, int dimind)
+{
+ gmx_domdec_comm_t *comm;
+ int dim;
+
+ comm = dd->comm;
+
+ /* Set the cell dimensions */
+ dim = dd->dim[dimind];
+ comm->cell_x0[dim] = comm->cell_f0[dimind]*ddbox->box_size[dim];
+ comm->cell_x1[dim] = comm->cell_f1[dimind]*ddbox->box_size[dim];
+ if (dim >= ddbox->nboundeddim)
+ {
+ comm->cell_x0[dim] += ddbox->box0[dim];
+ comm->cell_x1[dim] += ddbox->box0[dim];
+ }
+}
+
+static void distribute_dd_cell_sizes_dlb(gmx_domdec_t *dd,
+ int d, int dim, real *cell_f_row,
+ gmx_ddbox_t *ddbox)
+{
+ gmx_domdec_comm_t *comm;
+ int d1, dim1, pos;
+
+ comm = dd->comm;
+
+#ifdef GMX_MPI
+ /* Each node would only need to know two fractions,
+ * but it is probably cheaper to broadcast the whole array.
+ */
+ MPI_Bcast(cell_f_row, DD_CELL_F_SIZE(dd, d)*sizeof(real), MPI_BYTE,
+ 0, comm->mpi_comm_load[d]);
+#endif
+ /* Copy the fractions for this dimension from the buffer */
+ comm->cell_f0[d] = cell_f_row[dd->ci[dim] ];
+ comm->cell_f1[d] = cell_f_row[dd->ci[dim]+1];
+ /* The whole array was communicated, so set the buffer position */
+ pos = dd->nc[dim] + 1;
+ for (d1 = 0; d1 <= d; d1++)
+ {
+ if (d1 < d)
+ {
+ /* Copy the cell fractions of the lower dimensions */
+ comm->cell_f0[d1] = cell_f_row[pos++];
+ comm->cell_f1[d1] = cell_f_row[pos++];
+ }
+ relative_to_absolute_cell_bounds(dd, ddbox, d1);
+ }
+ /* Convert the communicated shift from float to int */
+ comm->ddpme[0].maxshift = (int)(cell_f_row[pos++] + 0.5);
+ if (d >= 1)
+ {
+ comm->ddpme[1].maxshift = (int)(cell_f_row[pos++] + 0.5);
+ }
+}
+
+static void set_dd_cell_sizes_dlb_change(gmx_domdec_t *dd,
+ gmx_ddbox_t *ddbox, gmx_bool bDynamicBox,
+ gmx_bool bUniform, gmx_large_int_t step)
+{
+ gmx_domdec_comm_t *comm;
+ int d, dim, d1;
+ gmx_bool bRowMember, bRowRoot;
+ real *cell_f_row;
+
+ comm = dd->comm;
+
+ for (d = 0; d < dd->ndim; d++)
+ {
+ dim = dd->dim[d];
+ bRowMember = TRUE;
+ bRowRoot = TRUE;
+ for (d1 = d; d1 < dd->ndim; d1++)
+ {
+ if (dd->ci[dd->dim[d1]] > 0)
+ {
+ if (d1 > d)
+ {
+ bRowMember = FALSE;
+ }
+ bRowRoot = FALSE;
+ }
+ }
+ if (bRowMember)
+ {
+ if (bRowRoot)
+ {
+ set_dd_cell_sizes_dlb_root(dd, d, dim, comm->root[d],
+ ddbox, bDynamicBox, bUniform, step);
+ cell_f_row = comm->root[d]->cell_f;
+ }
+ else
+ {
+ cell_f_row = comm->cell_f_row;
+ }
+ distribute_dd_cell_sizes_dlb(dd, d, dim, cell_f_row, ddbox);
+ }
+ }
+}
+
+static void set_dd_cell_sizes_dlb_nochange(gmx_domdec_t *dd, gmx_ddbox_t *ddbox)
+{
+ int d;
+
+ /* This function assumes the box is static and should therefore
+ * not be called when the box has changed since the last
+ * call to dd_partition_system.
+ */
+ for (d = 0; d < dd->ndim; d++)
+ {
+ relative_to_absolute_cell_bounds(dd, ddbox, d);
+ }
+}
+
+
+
+static void set_dd_cell_sizes_dlb(gmx_domdec_t *dd,
+ gmx_ddbox_t *ddbox, gmx_bool bDynamicBox,
+ gmx_bool bUniform, gmx_bool bDoDLB, gmx_large_int_t step,
+ gmx_wallcycle_t wcycle)
+{
+ gmx_domdec_comm_t *comm;
+ int dim;
+
+ comm = dd->comm;
+
+ if (bDoDLB)
+ {
+ wallcycle_start(wcycle, ewcDDCOMMBOUND);
+ set_dd_cell_sizes_dlb_change(dd, ddbox, bDynamicBox, bUniform, step);
+ wallcycle_stop(wcycle, ewcDDCOMMBOUND);
+ }
+ else if (bDynamicBox)
+ {
+ set_dd_cell_sizes_dlb_nochange(dd, ddbox);
+ }
+
+ /* Set the dimensions for which no DD is used */
+ for (dim = 0; dim < DIM; dim++)
+ {
+ if (dd->nc[dim] == 1)
+ {
+ comm->cell_x0[dim] = 0;
+ comm->cell_x1[dim] = ddbox->box_size[dim];
+ if (dim >= ddbox->nboundeddim)
+ {
+ comm->cell_x0[dim] += ddbox->box0[dim];
+ comm->cell_x1[dim] += ddbox->box0[dim];
+ }
+ }
+ }
+}
+
+static void realloc_comm_ind(gmx_domdec_t *dd, ivec npulse)
+{
+ int d, np, i;
+ gmx_domdec_comm_dim_t *cd;
+
+ for (d = 0; d < dd->ndim; d++)
+ {
+ cd = &dd->comm->cd[d];
+ np = npulse[dd->dim[d]];
+ if (np > cd->np_nalloc)
+ {
+ if (debug)
+ {
+ fprintf(debug, "(Re)allocing cd for %c to %d pulses\n",
+ dim2char(dd->dim[d]), np);
+ }
+ if (DDMASTER(dd) && cd->np_nalloc > 0)
+ {
+ fprintf(stderr, "\nIncreasing the number of cell to communicate in dimension %c to %d for the first time\n", dim2char(dd->dim[d]), np);
+ }
+ srenew(cd->ind, np);
+ for (i = cd->np_nalloc; i < np; i++)
+ {
+ cd->ind[i].index = NULL;
+ cd->ind[i].nalloc = 0;
+ }
+ cd->np_nalloc = np;
+ }
+ cd->np = np;
+ }
+}
+
+
+static void set_dd_cell_sizes(gmx_domdec_t *dd,
+ gmx_ddbox_t *ddbox, gmx_bool bDynamicBox,
+ gmx_bool bUniform, gmx_bool bDoDLB, gmx_large_int_t step,
+ gmx_wallcycle_t wcycle)
+{
+ gmx_domdec_comm_t *comm;
+ int d;
+ ivec npulse;
+
+ comm = dd->comm;
+
+ /* Copy the old cell boundaries for the cg displacement check */
+ copy_rvec(comm->cell_x0, comm->old_cell_x0);
+ copy_rvec(comm->cell_x1, comm->old_cell_x1);
+
+ if (comm->bDynLoadBal)
+ {
+ if (DDMASTER(dd))
+ {
+ check_box_size(dd, ddbox);
+ }
+ set_dd_cell_sizes_dlb(dd, ddbox, bDynamicBox, bUniform, bDoDLB, step, wcycle);
+ }
+ else
+ {
+ set_dd_cell_sizes_slb(dd, ddbox, FALSE, npulse);
+ realloc_comm_ind(dd, npulse);
+ }
+
+ if (debug)
+ {
+ for (d = 0; d < DIM; d++)
+ {
+ fprintf(debug, "cell_x[%d] %f - %f skew_fac %f\n",
+ d, comm->cell_x0[d], comm->cell_x1[d], ddbox->skew_fac[d]);
+ }
+ }
+}
+
+static void comm_dd_ns_cell_sizes(gmx_domdec_t *dd,
+ gmx_ddbox_t *ddbox,
+ rvec cell_ns_x0, rvec cell_ns_x1,
+ gmx_large_int_t step)
+{
+ gmx_domdec_comm_t *comm;
+ int dim_ind, dim;
+
+ comm = dd->comm;
+
+ for (dim_ind = 0; dim_ind < dd->ndim; dim_ind++)
+ {
+ dim = dd->dim[dim_ind];
+
+ /* Without PBC we don't have restrictions on the outer cells */
+ if (!(dim >= ddbox->npbcdim &&
+ (dd->ci[dim] == 0 || dd->ci[dim] == dd->nc[dim] - 1)) &&
+ comm->bDynLoadBal &&
+ (comm->cell_x1[dim] - comm->cell_x0[dim])*ddbox->skew_fac[dim] <
+ comm->cellsize_min[dim])
+ {
+ char buf[22];
+ gmx_fatal(FARGS, "Step %s: The %c-size (%f) times the triclinic skew factor (%f) is smaller than the smallest allowed cell size (%f) for domain decomposition grid cell %d %d %d",
+ gmx_step_str(step, buf), dim2char(dim),
+ comm->cell_x1[dim] - comm->cell_x0[dim],
+ ddbox->skew_fac[dim],
+ dd->comm->cellsize_min[dim],
+ dd->ci[XX], dd->ci[YY], dd->ci[ZZ]);
+ }
+ }
+
+ if ((dd->bGridJump && dd->ndim > 1) || ddbox->nboundeddim < DIM)
+ {
+ /* Communicate the boundaries and update cell_ns_x0/1 */
+ dd_move_cellx(dd, ddbox, cell_ns_x0, cell_ns_x1);
+ if (dd->bGridJump && dd->ndim > 1)
+ {
+ check_grid_jump(step, dd, dd->comm->cutoff, ddbox, TRUE);
+ }
+ }
+}
+
+static void make_tric_corr_matrix(int npbcdim, matrix box, matrix tcm)
+{
+ if (YY < npbcdim)
+ {
+ tcm[YY][XX] = -box[YY][XX]/box[YY][YY];
+ }
+ else
+ {
+ tcm[YY][XX] = 0;
+ }
+ if (ZZ < npbcdim)
+ {
+ tcm[ZZ][XX] = -(box[ZZ][YY]*tcm[YY][XX] + box[ZZ][XX])/box[ZZ][ZZ];
+ tcm[ZZ][YY] = -box[ZZ][YY]/box[ZZ][ZZ];
+ }
+ else
+ {
+ tcm[ZZ][XX] = 0;
+ tcm[ZZ][YY] = 0;
+ }
+}
+
+static void check_screw_box(matrix box)
+{
+ /* Mathematical limitation */
+ if (box[YY][XX] != 0 || box[ZZ][XX] != 0)
+ {
+ gmx_fatal(FARGS, "With screw pbc the unit cell can not have non-zero off-diagonal x-components");
+ }
+
+ /* Limitation due to the asymmetry of the eighth shell method */
+ if (box[ZZ][YY] != 0)
+ {
+ gmx_fatal(FARGS, "pbc=screw with non-zero box_zy is not supported");
+ }
+}
+
+static void distribute_cg(FILE *fplog, gmx_large_int_t step,
+ matrix box, ivec tric_dir, t_block *cgs, rvec pos[],
+ gmx_domdec_t *dd)
+{
+ gmx_domdec_master_t *ma;
+ int **tmp_ind = NULL, *tmp_nalloc = NULL;
+ int i, icg, j, k, k0, k1, d, npbcdim;
+ matrix tcm;
+ rvec box_size, cg_cm;
+ ivec ind;
+ real nrcg, inv_ncg, pos_d;
+ atom_id *cgindex;
+ gmx_bool bUnbounded, bScrew;
+
+ ma = dd->ma;
+
+ if (tmp_ind == NULL)
+ {
+ snew(tmp_nalloc, dd->nnodes);
+ snew(tmp_ind, dd->nnodes);
+ for (i = 0; i < dd->nnodes; i++)
+ {
+ tmp_nalloc[i] = over_alloc_large(cgs->nr/dd->nnodes+1);
+ snew(tmp_ind[i], tmp_nalloc[i]);
+ }
+ }
+
+ /* Clear the count */
+ for (i = 0; i < dd->nnodes; i++)
+ {
+ ma->ncg[i] = 0;
+ ma->nat[i] = 0;
+ }
+
+ make_tric_corr_matrix(dd->npbcdim, box, tcm);
+
+ cgindex = cgs->index;
+
+ /* Compute the center of geometry for all charge groups */
+ for (icg = 0; icg < cgs->nr; icg++)
+ {
+ k0 = cgindex[icg];
+ k1 = cgindex[icg+1];
+ nrcg = k1 - k0;
+ if (nrcg == 1)
+ {
+ copy_rvec(pos[k0], cg_cm);
+ }
+ else
+ {
+ inv_ncg = 1.0/nrcg;
+
+ clear_rvec(cg_cm);
+ for (k = k0; (k < k1); k++)
+ {
+ rvec_inc(cg_cm, pos[k]);
+ }
+ for (d = 0; (d < DIM); d++)
+ {
+ cg_cm[d] *= inv_ncg;
+ }
+ }
+ /* Put the charge group in the box and determine the cell index */
+ for (d = DIM-1; d >= 0; d--)
+ {
+ pos_d = cg_cm[d];
+ if (d < dd->npbcdim)
+ {
+ bScrew = (dd->bScrewPBC && d == XX);
+ if (tric_dir[d] && dd->nc[d] > 1)
+ {
+ /* Use triclinic coordintates for this dimension */
+ for (j = d+1; j < DIM; j++)
+ {
+ pos_d += cg_cm[j]*tcm[j][d];
+ }
+ }
+ while (pos_d >= box[d][d])
+ {
+ pos_d -= box[d][d];
+ rvec_dec(cg_cm, box[d]);
+ if (bScrew)
+ {
+ cg_cm[YY] = box[YY][YY] - cg_cm[YY];
+ cg_cm[ZZ] = box[ZZ][ZZ] - cg_cm[ZZ];
+ }
+ for (k = k0; (k < k1); k++)
+ {
+ rvec_dec(pos[k], box[d]);
+ if (bScrew)
+ {
+ pos[k][YY] = box[YY][YY] - pos[k][YY];
+ pos[k][ZZ] = box[ZZ][ZZ] - pos[k][ZZ];
+ }
+ }
+ }
+ while (pos_d < 0)
+ {
+ pos_d += box[d][d];
+ rvec_inc(cg_cm, box[d]);
+ if (bScrew)
+ {
+ cg_cm[YY] = box[YY][YY] - cg_cm[YY];
+ cg_cm[ZZ] = box[ZZ][ZZ] - cg_cm[ZZ];
+ }
+ for (k = k0; (k < k1); k++)
+ {
+ rvec_inc(pos[k], box[d]);
+ if (bScrew)
+ {
+ pos[k][YY] = box[YY][YY] - pos[k][YY];
+ pos[k][ZZ] = box[ZZ][ZZ] - pos[k][ZZ];
+ }
+ }
+ }
+ }
+ /* This could be done more efficiently */
+ ind[d] = 0;
+ while (ind[d]+1 < dd->nc[d] && pos_d >= ma->cell_x[d][ind[d]+1])
+ {
+ ind[d]++;
+ }
+ }
+ i = dd_index(dd->nc, ind);
+ if (ma->ncg[i] == tmp_nalloc[i])
+ {
+ tmp_nalloc[i] = over_alloc_large(ma->ncg[i]+1);
+ srenew(tmp_ind[i], tmp_nalloc[i]);
+ }
+ tmp_ind[i][ma->ncg[i]] = icg;
+ ma->ncg[i]++;
+ ma->nat[i] += cgindex[icg+1] - cgindex[icg];
+ }
+
+ k1 = 0;
+ for (i = 0; i < dd->nnodes; i++)
+ {
+ ma->index[i] = k1;
+ for (k = 0; k < ma->ncg[i]; k++)
+ {
+ ma->cg[k1++] = tmp_ind[i][k];
+ }
+ }
+ ma->index[dd->nnodes] = k1;
+
+ for (i = 0; i < dd->nnodes; i++)
+ {
+ sfree(tmp_ind[i]);
+ }
+ sfree(tmp_ind);
+ sfree(tmp_nalloc);
+
+ if (fplog)
+ {
+ char buf[22];
+ fprintf(fplog, "Charge group distribution at step %s:",
+ gmx_step_str(step, buf));
+ for (i = 0; i < dd->nnodes; i++)
+ {
+ fprintf(fplog, " %d", ma->ncg[i]);
+ }
+ fprintf(fplog, "\n");
+ }
+}
+
+static void get_cg_distribution(FILE *fplog, gmx_large_int_t step, gmx_domdec_t *dd,
+ t_block *cgs, matrix box, gmx_ddbox_t *ddbox,
+ rvec pos[])
+{
+ gmx_domdec_master_t *ma = NULL;
+ ivec npulse;
+ int i, cg_gl;
+ int *ibuf, buf2[2] = { 0, 0 };
+ gmx_bool bMaster = DDMASTER(dd);
+ if (bMaster)
+ {
+ ma = dd->ma;
+
+ if (dd->bScrewPBC)
+ {
+ check_screw_box(box);
+ }
+
+ set_dd_cell_sizes_slb(dd, ddbox, TRUE, npulse);
+
+ distribute_cg(fplog, step, box, ddbox->tric_dir, cgs, pos, dd);
+ for (i = 0; i < dd->nnodes; i++)
+ {
+ ma->ibuf[2*i] = ma->ncg[i];
+ ma->ibuf[2*i+1] = ma->nat[i];
+ }
+ ibuf = ma->ibuf;
+ }
+ else
+ {
+ ibuf = NULL;
+ }
+ dd_scatter(dd, 2*sizeof(int), ibuf, buf2);
+
+ dd->ncg_home = buf2[0];
+ dd->nat_home = buf2[1];
+ dd->ncg_tot = dd->ncg_home;
+ dd->nat_tot = dd->nat_home;
+ if (dd->ncg_home > dd->cg_nalloc || dd->cg_nalloc == 0)
+ {
+ dd->cg_nalloc = over_alloc_dd(dd->ncg_home);
+ srenew(dd->index_gl, dd->cg_nalloc);
+ srenew(dd->cgindex, dd->cg_nalloc+1);
+ }
+ if (bMaster)
+ {
+ for (i = 0; i < dd->nnodes; i++)
+ {
+ ma->ibuf[i] = ma->ncg[i]*sizeof(int);
+ ma->ibuf[dd->nnodes+i] = ma->index[i]*sizeof(int);
+ }
+ }
+
+ dd_scatterv(dd,
+ DDMASTER(dd) ? ma->ibuf : NULL,
+ DDMASTER(dd) ? ma->ibuf+dd->nnodes : NULL,
+ DDMASTER(dd) ? ma->cg : NULL,
+ dd->ncg_home*sizeof(int), dd->index_gl);
+
+ /* Determine the home charge group sizes */
+ dd->cgindex[0] = 0;
+ for (i = 0; i < dd->ncg_home; i++)
+ {
+ cg_gl = dd->index_gl[i];
+ dd->cgindex[i+1] =
+ dd->cgindex[i] + cgs->index[cg_gl+1] - cgs->index[cg_gl];
+ }
+
+ if (debug)
+ {
+ fprintf(debug, "Home charge groups:\n");
+ for (i = 0; i < dd->ncg_home; i++)
+ {
+ fprintf(debug, " %d", dd->index_gl[i]);
+ if (i % 10 == 9)
+ {
+ fprintf(debug, "\n");
+ }
+ }
+ fprintf(debug, "\n");
+ }
+}
+
+static int compact_and_copy_vec_at(int ncg, int *move,
+ int *cgindex,
+ int nvec, int vec,
+ rvec *src, gmx_domdec_comm_t *comm,
+ gmx_bool bCompact)
+{
+ int m, icg, i, i0, i1, nrcg;
+ int home_pos;
+ int pos_vec[DIM*2];
+
+ home_pos = 0;
+
+ for (m = 0; m < DIM*2; m++)
+ {
+ pos_vec[m] = 0;
+ }
+
+ i0 = 0;
+ for (icg = 0; icg < ncg; icg++)
+ {
+ i1 = cgindex[icg+1];
+ m = move[icg];
+ if (m == -1)
+ {
+ if (bCompact)
+ {
+ /* Compact the home array in place */
+ for (i = i0; i < i1; i++)
+ {
+ copy_rvec(src[i], src[home_pos++]);
+ }
+ }
+ }
+ else
+ {
+ /* Copy to the communication buffer */
+ nrcg = i1 - i0;
+ pos_vec[m] += 1 + vec*nrcg;
+ for (i = i0; i < i1; i++)
+ {
+ copy_rvec(src[i], comm->cgcm_state[m][pos_vec[m]++]);
+ }
+ pos_vec[m] += (nvec - vec - 1)*nrcg;
+ }
+ if (!bCompact)
+ {
+ home_pos += i1 - i0;
+ }
+ i0 = i1;
+ }
+
+ return home_pos;
+}
+
+static int compact_and_copy_vec_cg(int ncg, int *move,
+ int *cgindex,
+ int nvec, rvec *src, gmx_domdec_comm_t *comm,
+ gmx_bool bCompact)
+{
+ int m, icg, i0, i1, nrcg;
+ int home_pos;
+ int pos_vec[DIM*2];
+
+ home_pos = 0;
+
+ for (m = 0; m < DIM*2; m++)
+ {
+ pos_vec[m] = 0;
+ }
+
+ i0 = 0;
+ for (icg = 0; icg < ncg; icg++)
+ {
+ i1 = cgindex[icg+1];
+ m = move[icg];
+ if (m == -1)
+ {
+ if (bCompact)
+ {
+ /* Compact the home array in place */
+ copy_rvec(src[icg], src[home_pos++]);
+ }
+ }
+ else
+ {
+ nrcg = i1 - i0;
+ /* Copy to the communication buffer */
+ copy_rvec(src[icg], comm->cgcm_state[m][pos_vec[m]]);
+ pos_vec[m] += 1 + nrcg*nvec;
+ }
+ i0 = i1;
+ }
+ if (!bCompact)
+ {
+ home_pos = ncg;
+ }
+
+ return home_pos;
+}
+
+static int compact_ind(int ncg, int *move,
+ int *index_gl, int *cgindex,
+ int *gatindex,
+ gmx_ga2la_t ga2la, char *bLocalCG,
+ int *cginfo)
+{
+ int cg, nat, a0, a1, a, a_gl;
+ int home_pos;
+
+ home_pos = 0;
+ nat = 0;
+ for (cg = 0; cg < ncg; cg++)
+ {
+ a0 = cgindex[cg];
+ a1 = cgindex[cg+1];
+ if (move[cg] == -1)
+ {
+ /* Compact the home arrays in place.
+ * Anything that can be done here avoids access to global arrays.
+ */
+ cgindex[home_pos] = nat;
+ for (a = a0; a < a1; a++)
+ {
+ a_gl = gatindex[a];
+ gatindex[nat] = a_gl;
+ /* The cell number stays 0, so we don't need to set it */
+ ga2la_change_la(ga2la, a_gl, nat);
+ nat++;
+ }
+ index_gl[home_pos] = index_gl[cg];
+ cginfo[home_pos] = cginfo[cg];
+ /* The charge group remains local, so bLocalCG does not change */
+ home_pos++;
+ }
+ else
+ {
+ /* Clear the global indices */
+ for (a = a0; a < a1; a++)
+ {
+ ga2la_del(ga2la, gatindex[a]);
+ }
+ if (bLocalCG)
+ {
+ bLocalCG[index_gl[cg]] = FALSE;
+ }
+ }
+ }
+ cgindex[home_pos] = nat;
+
+ return home_pos;
+}
+
+static void clear_and_mark_ind(int ncg, int *move,
+ int *index_gl, int *cgindex, int *gatindex,
+ gmx_ga2la_t ga2la, char *bLocalCG,
+ int *cell_index)
+{
+ int cg, a0, a1, a;
+
+ for (cg = 0; cg < ncg; cg++)
+ {
+ if (move[cg] >= 0)
+ {
+ a0 = cgindex[cg];
+ a1 = cgindex[cg+1];
+ /* Clear the global indices */
+ for (a = a0; a < a1; a++)
+ {
+ ga2la_del(ga2la, gatindex[a]);
+ }
+ if (bLocalCG)
+ {
+ bLocalCG[index_gl[cg]] = FALSE;
+ }
+ /* Signal that this cg has moved using the ns cell index.
+ * Here we set it to -1. fill_grid will change it
+ * from -1 to NSGRID_SIGNAL_MOVED_FAC*grid->ncells.
+ */
+ cell_index[cg] = -1;
+ }
+ }
+}
+
+static void print_cg_move(FILE *fplog,
+ gmx_domdec_t *dd,
+ gmx_large_int_t step, int cg, int dim, int dir,
+ gmx_bool bHaveLimitdAndCMOld, real limitd,
+ rvec cm_old, rvec cm_new, real pos_d)
+{
+ gmx_domdec_comm_t *comm;
+ char buf[22];
+
+ comm = dd->comm;
+
+ fprintf(fplog, "\nStep %s:\n", gmx_step_str(step, buf));
+ if (bHaveLimitdAndCMOld)
+ {
+ fprintf(fplog, "The charge group starting at atom %d moved more than the distance allowed by the domain decomposition (%f) in direction %c\n",
+ ddglatnr(dd, dd->cgindex[cg]), limitd, dim2char(dim));
+ }
+ else
+ {
+ fprintf(fplog, "The charge group starting at atom %d moved than the distance allowed by the domain decomposition in direction %c\n",
+ ddglatnr(dd, dd->cgindex[cg]), dim2char(dim));
+ }
+ fprintf(fplog, "distance out of cell %f\n",
+ dir == 1 ? pos_d - comm->cell_x1[dim] : pos_d - comm->cell_x0[dim]);
+ if (bHaveLimitdAndCMOld)
+ {
+ fprintf(fplog, "Old coordinates: %8.3f %8.3f %8.3f\n",
+ cm_old[XX], cm_old[YY], cm_old[ZZ]);
+ }
+ fprintf(fplog, "New coordinates: %8.3f %8.3f %8.3f\n",
+ cm_new[XX], cm_new[YY], cm_new[ZZ]);
+ fprintf(fplog, "Old cell boundaries in direction %c: %8.3f %8.3f\n",
+ dim2char(dim),
+ comm->old_cell_x0[dim], comm->old_cell_x1[dim]);
+ fprintf(fplog, "New cell boundaries in direction %c: %8.3f %8.3f\n",
+ dim2char(dim),
+ comm->cell_x0[dim], comm->cell_x1[dim]);
+}
+
+static void cg_move_error(FILE *fplog,
+ gmx_domdec_t *dd,
+ gmx_large_int_t step, int cg, int dim, int dir,
+ gmx_bool bHaveLimitdAndCMOld, real limitd,
+ rvec cm_old, rvec cm_new, real pos_d)
+{
+ if (fplog)
+ {
+ print_cg_move(fplog, dd, step, cg, dim, dir,
+ bHaveLimitdAndCMOld, limitd, cm_old, cm_new, pos_d);
+ }
+ print_cg_move(stderr, dd, step, cg, dim, dir,
+ bHaveLimitdAndCMOld, limitd, cm_old, cm_new, pos_d);
+ gmx_fatal(FARGS,
+ "A charge group moved too far between two domain decomposition steps\n"
+ "This usually means that your system is not well equilibrated");
+}
+
+static void rotate_state_atom(t_state *state, int a)
+{
+ int est;
+
+ for (est = 0; est < estNR; est++)
+ {
+ if (EST_DISTR(est) && (state->flags & (1<<est)))
+ {
+ switch (est)
+ {
+ case estX:
+ /* Rotate the complete state; for a rectangular box only */
+ state->x[a][YY] = state->box[YY][YY] - state->x[a][YY];
+ state->x[a][ZZ] = state->box[ZZ][ZZ] - state->x[a][ZZ];
+ break;
+ case estV:
+ state->v[a][YY] = -state->v[a][YY];
+ state->v[a][ZZ] = -state->v[a][ZZ];
+ break;
+ case estSDX:
+ state->sd_X[a][YY] = -state->sd_X[a][YY];
+ state->sd_X[a][ZZ] = -state->sd_X[a][ZZ];
+ break;
+ case estCGP:
+ state->cg_p[a][YY] = -state->cg_p[a][YY];
+ state->cg_p[a][ZZ] = -state->cg_p[a][ZZ];
+ break;
+ case estDISRE_INITF:
+ case estDISRE_RM3TAV:
+ case estORIRE_INITF:
+ case estORIRE_DTAV:
+ /* These are distances, so not affected by rotation */
+ break;
+ default:
+ gmx_incons("Unknown state entry encountered in rotate_state_atom");
+ }
+ }
+ }
+}
+
+static int *get_moved(gmx_domdec_comm_t *comm, int natoms)
+{
+ if (natoms > comm->moved_nalloc)
+ {
+ /* Contents should be preserved here */
+ comm->moved_nalloc = over_alloc_dd(natoms);
+ srenew(comm->moved, comm->moved_nalloc);
+ }
+
+ return comm->moved;
+}
+
+static void calc_cg_move(FILE *fplog, gmx_large_int_t step,
+ gmx_domdec_t *dd,
+ t_state *state,
+ ivec tric_dir, matrix tcm,
+ rvec cell_x0, rvec cell_x1,
+ rvec limitd, rvec limit0, rvec limit1,
+ const int *cgindex,
+ int cg_start, int cg_end,
+ rvec *cg_cm,
+ int *move)
+{
+ int npbcdim;
+ int c, i, cg, k, k0, k1, d, dim, dim2, dir, d2, d3, d4, cell_d;
+ int mc, cdd, nrcg, ncg_recv, nat_recv, nvs, nvr, nvec, vec;
+ int flag;
+ gmx_bool bScrew;
+ ivec dev;
+ real inv_ncg, pos_d;
+ rvec cm_new;
+
+ npbcdim = dd->npbcdim;
+
+ for (cg = cg_start; cg < cg_end; cg++)
+ {
+ k0 = cgindex[cg];
+ k1 = cgindex[cg+1];
+ nrcg = k1 - k0;
+ if (nrcg == 1)
+ {
+ copy_rvec(state->x[k0], cm_new);
+ }
+ else
+ {
+ inv_ncg = 1.0/nrcg;
+
+ clear_rvec(cm_new);
+ for (k = k0; (k < k1); k++)
+ {
+ rvec_inc(cm_new, state->x[k]);
+ }
+ for (d = 0; (d < DIM); d++)
+ {
+ cm_new[d] = inv_ncg*cm_new[d];
+ }
+ }
+
+ clear_ivec(dev);
+ /* Do pbc and check DD cell boundary crossings */
+ for (d = DIM-1; d >= 0; d--)
+ {
+ if (dd->nc[d] > 1)
+ {
+ bScrew = (dd->bScrewPBC && d == XX);
+ /* Determine the location of this cg in lattice coordinates */
+ pos_d = cm_new[d];
+ if (tric_dir[d])
+ {
+ for (d2 = d+1; d2 < DIM; d2++)
+ {
+ pos_d += cm_new[d2]*tcm[d2][d];
+ }
+ }
+ /* Put the charge group in the triclinic unit-cell */
+ if (pos_d >= cell_x1[d])
+ {
+ if (pos_d >= limit1[d])
+ {
+ cg_move_error(fplog, dd, step, cg, d, 1, TRUE, limitd[d],
+ cg_cm[cg], cm_new, pos_d);
+ }
+ dev[d] = 1;
+ if (dd->ci[d] == dd->nc[d] - 1)
+ {
+ rvec_dec(cm_new, state->box[d]);
+ if (bScrew)
+ {
+ cm_new[YY] = state->box[YY][YY] - cm_new[YY];
+ cm_new[ZZ] = state->box[ZZ][ZZ] - cm_new[ZZ];
+ }
+ for (k = k0; (k < k1); k++)
+ {
+ rvec_dec(state->x[k], state->box[d]);
+ if (bScrew)
+ {
+ rotate_state_atom(state, k);
+ }
+ }
+ }
+ }
+ else if (pos_d < cell_x0[d])
+ {
+ if (pos_d < limit0[d])
+ {
+ cg_move_error(fplog, dd, step, cg, d, -1, TRUE, limitd[d],
+ cg_cm[cg], cm_new, pos_d);
+ }
+ dev[d] = -1;
+ if (dd->ci[d] == 0)
+ {
+ rvec_inc(cm_new, state->box[d]);
+ if (bScrew)
+ {
+ cm_new[YY] = state->box[YY][YY] - cm_new[YY];
+ cm_new[ZZ] = state->box[ZZ][ZZ] - cm_new[ZZ];
+ }
+ for (k = k0; (k < k1); k++)
+ {
+ rvec_inc(state->x[k], state->box[d]);
+ if (bScrew)
+ {
+ rotate_state_atom(state, k);
+ }
+ }
+ }
+ }
+ }
+ else if (d < npbcdim)
+ {
+ /* Put the charge group in the rectangular unit-cell */
+ while (cm_new[d] >= state->box[d][d])
+ {
+ rvec_dec(cm_new, state->box[d]);
+ for (k = k0; (k < k1); k++)
+ {
+ rvec_dec(state->x[k], state->box[d]);
+ }
+ }
+ while (cm_new[d] < 0)
+ {
+ rvec_inc(cm_new, state->box[d]);
+ for (k = k0; (k < k1); k++)
+ {
+ rvec_inc(state->x[k], state->box[d]);
+ }
+ }
+ }
+ }
+
+ copy_rvec(cm_new, cg_cm[cg]);
+
+ /* Determine where this cg should go */
+ flag = 0;
+ mc = -1;
+ for (d = 0; d < dd->ndim; d++)
+ {
+ dim = dd->dim[d];
+ if (dev[dim] == 1)
+ {
+ flag |= DD_FLAG_FW(d);
+ if (mc == -1)
+ {
+ mc = d*2;
+ }
+ }
+ else if (dev[dim] == -1)
+ {
+ flag |= DD_FLAG_BW(d);
+ if (mc == -1)
+ {
+ if (dd->nc[dim] > 2)
+ {
+ mc = d*2 + 1;
+ }
+ else
+ {
+ mc = d*2;
+ }
+ }
+ }
+ }
+ /* Temporarily store the flag in move */
+ move[cg] = mc + flag;
+ }
+}
+
+static void dd_redistribute_cg(FILE *fplog, gmx_large_int_t step,
+ gmx_domdec_t *dd, ivec tric_dir,
+ t_state *state, rvec **f,
+ t_forcerec *fr,
+ gmx_bool bCompact,
+ t_nrnb *nrnb,
+ int *ncg_stay_home,
+ int *ncg_moved)
+{
+ int *move;
+ int npbcdim;
+ int ncg[DIM*2], nat[DIM*2];
+ int c, i, cg, k, k0, k1, d, dim, dim2, dir, d2, d3, d4, cell_d;
+ int mc, cdd, nrcg, ncg_recv, nat_recv, nvs, nvr, nvec, vec;
+ int sbuf[2], rbuf[2];
+ int home_pos_cg, home_pos_at, buf_pos;
+ int flag;
+ gmx_bool bV = FALSE, bSDX = FALSE, bCGP = FALSE;
+ gmx_bool bScrew;
+ ivec dev;
+ real inv_ncg, pos_d;
+ matrix tcm;
+ rvec *cg_cm = NULL, cell_x0, cell_x1, limitd, limit0, limit1, cm_new;
+ atom_id *cgindex;
+ cginfo_mb_t *cginfo_mb;
+ gmx_domdec_comm_t *comm;
+ int *moved;
+ int nthread, thread;
+
+ if (dd->bScrewPBC)
+ {
+ check_screw_box(state->box);
+ }
+
+ comm = dd->comm;
+ if (fr->cutoff_scheme == ecutsGROUP)
+ {
+ cg_cm = fr->cg_cm;
+ }
+
+ for (i = 0; i < estNR; i++)
+ {
+ if (EST_DISTR(i))
+ {
+ switch (i)
+ {
+ case estX: /* Always present */ break;
+ case estV: bV = (state->flags & (1<<i)); break;
+ case estSDX: bSDX = (state->flags & (1<<i)); break;
+ case estCGP: bCGP = (state->flags & (1<<i)); break;
+ case estLD_RNG:
+ case estLD_RNGI:
+ case estDISRE_INITF:
+ case estDISRE_RM3TAV:
+ case estORIRE_INITF:
+ case estORIRE_DTAV:
+ /* No processing required */
+ break;
+ default:
+ gmx_incons("Unknown state entry encountered in dd_redistribute_cg");
+ }
+ }
+ }
+
+ if (dd->ncg_tot > comm->nalloc_int)
+ {
+ comm->nalloc_int = over_alloc_dd(dd->ncg_tot);
+ srenew(comm->buf_int, comm->nalloc_int);
+ }
+ move = comm->buf_int;
+
+ /* Clear the count */
+ for (c = 0; c < dd->ndim*2; c++)
+ {
+ ncg[c] = 0;
+ nat[c] = 0;
+ }
+
+ npbcdim = dd->npbcdim;
+
+ for (d = 0; (d < DIM); d++)
+ {
+ limitd[d] = dd->comm->cellsize_min[d];
+ if (d >= npbcdim && dd->ci[d] == 0)
+ {
+ cell_x0[d] = -GMX_FLOAT_MAX;
+ }
+ else
+ {
+ cell_x0[d] = comm->cell_x0[d];
+ }
+ if (d >= npbcdim && dd->ci[d] == dd->nc[d] - 1)
+ {
+ cell_x1[d] = GMX_FLOAT_MAX;
+ }
+ else
+ {
+ cell_x1[d] = comm->cell_x1[d];
+ }
+ if (d < npbcdim)
+ {
+ limit0[d] = comm->old_cell_x0[d] - limitd[d];
+ limit1[d] = comm->old_cell_x1[d] + limitd[d];
+ }
+ else
+ {
+ /* We check after communication if a charge group moved
+ * more than one cell. Set the pre-comm check limit to float_max.
+ */
+ limit0[d] = -GMX_FLOAT_MAX;
+ limit1[d] = GMX_FLOAT_MAX;
+ }
+ }
+
+ make_tric_corr_matrix(npbcdim, state->box, tcm);
+
+ cgindex = dd->cgindex;
+
+ nthread = gmx_omp_nthreads_get(emntDomdec);
+
+ /* Compute the center of geometry for all home charge groups
+ * and put them in the box and determine where they should go.
+ */
+#pragma omp parallel for num_threads(nthread) schedule(static)
+ for (thread = 0; thread < nthread; thread++)
+ {
+ calc_cg_move(fplog, step, dd, state, tric_dir, tcm,
+ cell_x0, cell_x1, limitd, limit0, limit1,
+ cgindex,
+ ( thread *dd->ncg_home)/nthread,
+ ((thread+1)*dd->ncg_home)/nthread,
+ fr->cutoff_scheme == ecutsGROUP ? cg_cm : state->x,
+ move);
+ }
+
+ for (cg = 0; cg < dd->ncg_home; cg++)
+ {
+ if (move[cg] >= 0)
+ {
+ mc = move[cg];
+ flag = mc & ~DD_FLAG_NRCG;
+ mc = mc & DD_FLAG_NRCG;
+ move[cg] = mc;
+
+ if (ncg[mc]+1 > comm->cggl_flag_nalloc[mc])
+ {
+ comm->cggl_flag_nalloc[mc] = over_alloc_dd(ncg[mc]+1);
+ srenew(comm->cggl_flag[mc], comm->cggl_flag_nalloc[mc]*DD_CGIBS);
+ }
+ comm->cggl_flag[mc][ncg[mc]*DD_CGIBS ] = dd->index_gl[cg];
+ /* We store the cg size in the lower 16 bits
+ * and the place where the charge group should go
+ * in the next 6 bits. This saves some communication volume.
+ */
+ nrcg = cgindex[cg+1] - cgindex[cg];
+ comm->cggl_flag[mc][ncg[mc]*DD_CGIBS+1] = nrcg | flag;
+ ncg[mc] += 1;
+ nat[mc] += nrcg;
+ }
+ }
+
+ inc_nrnb(nrnb, eNR_CGCM, dd->nat_home);
+ inc_nrnb(nrnb, eNR_RESETX, dd->ncg_home);
+
+ *ncg_moved = 0;
+ for (i = 0; i < dd->ndim*2; i++)
+ {
+ *ncg_moved += ncg[i];
+ }
+
+ nvec = 1;
+ if (bV)
+ {
+ nvec++;
+ }
+ if (bSDX)
+ {
+ nvec++;
+ }
+ if (bCGP)
+ {
+ nvec++;
+ }
+
+ /* Make sure the communication buffers are large enough */
+ for (mc = 0; mc < dd->ndim*2; mc++)
+ {
+ nvr = ncg[mc] + nat[mc]*nvec;
+ if (nvr > comm->cgcm_state_nalloc[mc])
+ {
+ comm->cgcm_state_nalloc[mc] = over_alloc_dd(nvr);
+ srenew(comm->cgcm_state[mc], comm->cgcm_state_nalloc[mc]);
+ }
+ }
+
+ switch (fr->cutoff_scheme)
+ {
+ case ecutsGROUP:
+ /* Recalculating cg_cm might be cheaper than communicating,
+ * but that could give rise to rounding issues.
+ */
+ home_pos_cg =
+ compact_and_copy_vec_cg(dd->ncg_home, move, cgindex,
+ nvec, cg_cm, comm, bCompact);
+ break;
+ case ecutsVERLET:
+ /* Without charge groups we send the moved atom coordinates
+ * over twice. This is so the code below can be used without
+ * many conditionals for both for with and without charge groups.
+ */
+ home_pos_cg =
+ compact_and_copy_vec_cg(dd->ncg_home, move, cgindex,
+ nvec, state->x, comm, FALSE);
+ if (bCompact)
+ {
+ home_pos_cg -= *ncg_moved;
+ }
+ break;
+ default:
+ gmx_incons("unimplemented");
+ home_pos_cg = 0;
+ }
+
+ vec = 0;
+ home_pos_at =
+ compact_and_copy_vec_at(dd->ncg_home, move, cgindex,
+ nvec, vec++, state->x, comm, bCompact);
+ if (bV)
+ {
+ compact_and_copy_vec_at(dd->ncg_home, move, cgindex,
+ nvec, vec++, state->v, comm, bCompact);
+ }
+ if (bSDX)
+ {
+ compact_and_copy_vec_at(dd->ncg_home, move, cgindex,
+ nvec, vec++, state->sd_X, comm, bCompact);
+ }
+ if (bCGP)
+ {
+ compact_and_copy_vec_at(dd->ncg_home, move, cgindex,
+ nvec, vec++, state->cg_p, comm, bCompact);
+ }
+
+ if (bCompact)
+ {
+ compact_ind(dd->ncg_home, move,
+ dd->index_gl, dd->cgindex, dd->gatindex,
+ dd->ga2la, comm->bLocalCG,
+ fr->cginfo);
+ }
+ else
+ {
+ if (fr->cutoff_scheme == ecutsVERLET)
+ {
+ moved = get_moved(comm, dd->ncg_home);
+
+ for (k = 0; k < dd->ncg_home; k++)
+ {
+ moved[k] = 0;
+ }
+ }
+ else
+ {
+ moved = fr->ns.grid->cell_index;
+ }
+
+ clear_and_mark_ind(dd->ncg_home, move,
+ dd->index_gl, dd->cgindex, dd->gatindex,
+ dd->ga2la, comm->bLocalCG,
+ moved);
+ }
+
+ cginfo_mb = fr->cginfo_mb;
+
+ *ncg_stay_home = home_pos_cg;
+ for (d = 0; d < dd->ndim; d++)
+ {
+ dim = dd->dim[d];
+ ncg_recv = 0;
+ nat_recv = 0;
+ nvr = 0;
+ for (dir = 0; dir < (dd->nc[dim] == 2 ? 1 : 2); dir++)
+ {
+ cdd = d*2 + dir;
+ /* Communicate the cg and atom counts */
+ sbuf[0] = ncg[cdd];
+ sbuf[1] = nat[cdd];
+ if (debug)
+ {
+ fprintf(debug, "Sending ddim %d dir %d: ncg %d nat %d\n",
+ d, dir, sbuf[0], sbuf[1]);
+ }
+ dd_sendrecv_int(dd, d, dir, sbuf, 2, rbuf, 2);
+
+ if ((ncg_recv+rbuf[0])*DD_CGIBS > comm->nalloc_int)
+ {
+ comm->nalloc_int = over_alloc_dd((ncg_recv+rbuf[0])*DD_CGIBS);
+ srenew(comm->buf_int, comm->nalloc_int);
+ }
+
+ /* Communicate the charge group indices, sizes and flags */
+ dd_sendrecv_int(dd, d, dir,
+ comm->cggl_flag[cdd], sbuf[0]*DD_CGIBS,
+ comm->buf_int+ncg_recv*DD_CGIBS, rbuf[0]*DD_CGIBS);
+
+ nvs = ncg[cdd] + nat[cdd]*nvec;
+ i = rbuf[0] + rbuf[1] *nvec;
+ vec_rvec_check_alloc(&comm->vbuf, nvr+i);
+
+ /* Communicate cgcm and state */
+ dd_sendrecv_rvec(dd, d, dir,
+ comm->cgcm_state[cdd], nvs,
+ comm->vbuf.v+nvr, i);
+ ncg_recv += rbuf[0];
+ nat_recv += rbuf[1];
+ nvr += i;
+ }
+
+ /* Process the received charge groups */
+ buf_pos = 0;
+ for (cg = 0; cg < ncg_recv; cg++)
+ {
+ flag = comm->buf_int[cg*DD_CGIBS+1];
+
+ if (dim >= npbcdim && dd->nc[dim] > 2)
+ {
+ /* No pbc in this dim and more than one domain boundary.
+ * We do a separate check if a charge group didn't move too far.
+ */
+ if (((flag & DD_FLAG_FW(d)) &&
+ comm->vbuf.v[buf_pos][dim] > cell_x1[dim]) ||
+ ((flag & DD_FLAG_BW(d)) &&
+ comm->vbuf.v[buf_pos][dim] < cell_x0[dim]))
+ {
+ cg_move_error(fplog, dd, step, cg, dim,
+ (flag & DD_FLAG_FW(d)) ? 1 : 0,
+ FALSE, 0,
+ comm->vbuf.v[buf_pos],
+ comm->vbuf.v[buf_pos],
+ comm->vbuf.v[buf_pos][dim]);
+ }
+ }
+
+ mc = -1;
+ if (d < dd->ndim-1)
+ {
+ /* Check which direction this cg should go */
+ for (d2 = d+1; (d2 < dd->ndim && mc == -1); d2++)
+ {
+ if (dd->bGridJump)
+ {
+ /* The cell boundaries for dimension d2 are not equal
+ * for each cell row of the lower dimension(s),
+ * therefore we might need to redetermine where
+ * this cg should go.
+ */
+ dim2 = dd->dim[d2];
+ /* If this cg crosses the box boundary in dimension d2
+ * we can use the communicated flag, so we do not
+ * have to worry about pbc.
+ */
+ if (!((dd->ci[dim2] == dd->nc[dim2]-1 &&
+ (flag & DD_FLAG_FW(d2))) ||
+ (dd->ci[dim2] == 0 &&
+ (flag & DD_FLAG_BW(d2)))))
+ {
+ /* Clear the two flags for this dimension */
+ flag &= ~(DD_FLAG_FW(d2) | DD_FLAG_BW(d2));
+ /* Determine the location of this cg
+ * in lattice coordinates
+ */
+ pos_d = comm->vbuf.v[buf_pos][dim2];
+ if (tric_dir[dim2])
+ {
+ for (d3 = dim2+1; d3 < DIM; d3++)
+ {
+ pos_d +=
+ comm->vbuf.v[buf_pos][d3]*tcm[d3][dim2];
+ }
+ }
+ /* Check of we are not at the box edge.
+ * pbc is only handled in the first step above,
+ * but this check could move over pbc while
+ * the first step did not due to different rounding.
+ */
+ if (pos_d >= cell_x1[dim2] &&
+ dd->ci[dim2] != dd->nc[dim2]-1)
+ {
+ flag |= DD_FLAG_FW(d2);
+ }
+ else if (pos_d < cell_x0[dim2] &&
+ dd->ci[dim2] != 0)
+ {
+ flag |= DD_FLAG_BW(d2);
+ }
+ comm->buf_int[cg*DD_CGIBS+1] = flag;
+ }
+ }
+ /* Set to which neighboring cell this cg should go */
+ if (flag & DD_FLAG_FW(d2))
+ {
+ mc = d2*2;
+ }
+ else if (flag & DD_FLAG_BW(d2))
+ {
+ if (dd->nc[dd->dim[d2]] > 2)
+ {
+ mc = d2*2+1;
+ }
+ else
+ {
+ mc = d2*2;
+ }
+ }
+ }
+ }
+
+ nrcg = flag & DD_FLAG_NRCG;
+ if (mc == -1)
+ {
+ if (home_pos_cg+1 > dd->cg_nalloc)
+ {
+ dd->cg_nalloc = over_alloc_dd(home_pos_cg+1);
+ srenew(dd->index_gl, dd->cg_nalloc);
+ srenew(dd->cgindex, dd->cg_nalloc+1);
+ }
+ /* Set the global charge group index and size */
+ dd->index_gl[home_pos_cg] = comm->buf_int[cg*DD_CGIBS];
+ dd->cgindex[home_pos_cg+1] = dd->cgindex[home_pos_cg] + nrcg;
+ /* Copy the state from the buffer */
+ dd_check_alloc_ncg(fr, state, f, home_pos_cg+1);
+ if (fr->cutoff_scheme == ecutsGROUP)
+ {
+ cg_cm = fr->cg_cm;
+ copy_rvec(comm->vbuf.v[buf_pos], cg_cm[home_pos_cg]);
+ }
+ buf_pos++;
+
+ /* Set the cginfo */
+ fr->cginfo[home_pos_cg] = ddcginfo(cginfo_mb,
+ dd->index_gl[home_pos_cg]);
+ if (comm->bLocalCG)
+ {
+ comm->bLocalCG[dd->index_gl[home_pos_cg]] = TRUE;
+ }
+
+ if (home_pos_at+nrcg > state->nalloc)
+ {
+ dd_realloc_state(state, f, home_pos_at+nrcg);
+ }
+ for (i = 0; i < nrcg; i++)
+ {
+ copy_rvec(comm->vbuf.v[buf_pos++],
+ state->x[home_pos_at+i]);
+ }
+ if (bV)
+ {
+ for (i = 0; i < nrcg; i++)
+ {
+ copy_rvec(comm->vbuf.v[buf_pos++],
+ state->v[home_pos_at+i]);
+ }
+ }
+ if (bSDX)
+ {
+ for (i = 0; i < nrcg; i++)
+ {
+ copy_rvec(comm->vbuf.v[buf_pos++],
+ state->sd_X[home_pos_at+i]);
+ }
+ }
+ if (bCGP)
+ {
+ for (i = 0; i < nrcg; i++)
+ {
+ copy_rvec(comm->vbuf.v[buf_pos++],
+ state->cg_p[home_pos_at+i]);
+ }
+ }
+ home_pos_cg += 1;
+ home_pos_at += nrcg;
+ }
+ else
+ {
+ /* Reallocate the buffers if necessary */
+ if (ncg[mc]+1 > comm->cggl_flag_nalloc[mc])
+ {
+ comm->cggl_flag_nalloc[mc] = over_alloc_dd(ncg[mc]+1);
+ srenew(comm->cggl_flag[mc], comm->cggl_flag_nalloc[mc]*DD_CGIBS);
+ }
+ nvr = ncg[mc] + nat[mc]*nvec;
+ if (nvr + 1 + nrcg*nvec > comm->cgcm_state_nalloc[mc])
+ {
+ comm->cgcm_state_nalloc[mc] = over_alloc_dd(nvr + 1 + nrcg*nvec);
+ srenew(comm->cgcm_state[mc], comm->cgcm_state_nalloc[mc]);
+ }
+ /* Copy from the receive to the send buffers */
+ memcpy(comm->cggl_flag[mc] + ncg[mc]*DD_CGIBS,
+ comm->buf_int + cg*DD_CGIBS,
+ DD_CGIBS*sizeof(int));
+ memcpy(comm->cgcm_state[mc][nvr],
+ comm->vbuf.v[buf_pos],
+ (1+nrcg*nvec)*sizeof(rvec));
+ buf_pos += 1 + nrcg*nvec;
+ ncg[mc] += 1;
+ nat[mc] += nrcg;
+ }
+ }
+ }
+
+ /* With sorting (!bCompact) the indices are now only partially up to date
+ * and ncg_home and nat_home are not the real count, since there are
+ * "holes" in the arrays for the charge groups that moved to neighbors.
+ */
+ if (fr->cutoff_scheme == ecutsVERLET)
+ {
+ moved = get_moved(comm, home_pos_cg);
+
+ for (i = dd->ncg_home; i < home_pos_cg; i++)
+ {
+ moved[i] = 0;
+ }
+ }
+ dd->ncg_home = home_pos_cg;
+ dd->nat_home = home_pos_at;
+
+ if (debug)
+ {
+ fprintf(debug,
+ "Finished repartitioning: cgs moved out %d, new home %d\n",
+ *ncg_moved, dd->ncg_home-*ncg_moved);
+
+ }
+}
+
+void dd_cycles_add(gmx_domdec_t *dd, float cycles, int ddCycl)
+{
+ dd->comm->cycl[ddCycl] += cycles;
+ dd->comm->cycl_n[ddCycl]++;
+ if (cycles > dd->comm->cycl_max[ddCycl])
+ {
+ dd->comm->cycl_max[ddCycl] = cycles;
+ }
+}
+
+static double force_flop_count(t_nrnb *nrnb)
+{
+ int i;
+ double sum;
+ const char *name;
+
+ sum = 0;
+ for (i = 0; i < eNR_NBKERNEL_FREE_ENERGY; i++)
+ {
+ /* To get closer to the real timings, we half the count
+ * for the normal loops and again half it for water loops.
+ */
+ name = nrnb_str(i);
+ if (strstr(name, "W3") != NULL || strstr(name, "W4") != NULL)
+ {
+ sum += nrnb->n[i]*0.25*cost_nrnb(i);
+ }
+ else
+ {
+ sum += nrnb->n[i]*0.50*cost_nrnb(i);
+ }
+ }
+ for (i = eNR_NBKERNEL_FREE_ENERGY; i <= eNR_NB14; i++)
+ {
+ name = nrnb_str(i);
+ if (strstr(name, "W3") != NULL || strstr(name, "W4") != NULL)
+ {
+ sum += nrnb->n[i]*cost_nrnb(i);
+ }
+ }
+ for (i = eNR_BONDS; i <= eNR_WALLS; i++)
+ {
+ sum += nrnb->n[i]*cost_nrnb(i);
+ }
+
+ return sum;
+}
+
+void dd_force_flop_start(gmx_domdec_t *dd, t_nrnb *nrnb)
+{
+ if (dd->comm->eFlop)
+ {
+ dd->comm->flop -= force_flop_count(nrnb);
+ }
+}
+void dd_force_flop_stop(gmx_domdec_t *dd, t_nrnb *nrnb)
+{
+ if (dd->comm->eFlop)
+ {
+ dd->comm->flop += force_flop_count(nrnb);
+ dd->comm->flop_n++;
+ }
+}
+
+static void clear_dd_cycle_counts(gmx_domdec_t *dd)
+{
+ int i;
+
+ for (i = 0; i < ddCyclNr; i++)
+ {
+ dd->comm->cycl[i] = 0;
+ dd->comm->cycl_n[i] = 0;
+ dd->comm->cycl_max[i] = 0;
+ }
+ dd->comm->flop = 0;
+ dd->comm->flop_n = 0;
+}
+
+static void get_load_distribution(gmx_domdec_t *dd, gmx_wallcycle_t wcycle)
+{
+ gmx_domdec_comm_t *comm;
+ gmx_domdec_load_t *load;
+ gmx_domdec_root_t *root = NULL;
+ int d, dim, cid, i, pos;
+ float cell_frac = 0, sbuf[DD_NLOAD_MAX];
+ gmx_bool bSepPME;
+
+ if (debug)
+ {
+ fprintf(debug, "get_load_distribution start\n");
+ }
+
+ wallcycle_start(wcycle, ewcDDCOMMLOAD);
+
+ comm = dd->comm;
+
+ bSepPME = (dd->pme_nodeid >= 0);
+
+ for (d = dd->ndim-1; d >= 0; d--)
+ {
+ dim = dd->dim[d];
+ /* Check if we participate in the communication in this dimension */
+ if (d == dd->ndim-1 ||
+ (dd->ci[dd->dim[d+1]] == 0 && dd->ci[dd->dim[dd->ndim-1]] == 0))
+ {
+ load = &comm->load[d];
+ if (dd->bGridJump)
+ {
+ cell_frac = comm->cell_f1[d] - comm->cell_f0[d];
+ }
+ pos = 0;
+ if (d == dd->ndim-1)
+ {
+ sbuf[pos++] = dd_force_load(comm);
+ sbuf[pos++] = sbuf[0];
+ if (dd->bGridJump)
+ {
+ sbuf[pos++] = sbuf[0];
+ sbuf[pos++] = cell_frac;
+ if (d > 0)
+ {
+ sbuf[pos++] = comm->cell_f_max0[d];
+ sbuf[pos++] = comm->cell_f_min1[d];
+ }
+ }
+ if (bSepPME)
+ {
+ sbuf[pos++] = comm->cycl[ddCyclPPduringPME];
+ sbuf[pos++] = comm->cycl[ddCyclPME];
+ }
+ }
+ else
+ {
+ sbuf[pos++] = comm->load[d+1].sum;
+ sbuf[pos++] = comm->load[d+1].max;
+ if (dd->bGridJump)
+ {
+ sbuf[pos++] = comm->load[d+1].sum_m;
+ sbuf[pos++] = comm->load[d+1].cvol_min*cell_frac;
+ sbuf[pos++] = comm->load[d+1].flags;
+ if (d > 0)
+ {
+ sbuf[pos++] = comm->cell_f_max0[d];
+ sbuf[pos++] = comm->cell_f_min1[d];
+ }
+ }
+ if (bSepPME)
+ {
+ sbuf[pos++] = comm->load[d+1].mdf;
+ sbuf[pos++] = comm->load[d+1].pme;
+ }
+ }
+ load->nload = pos;
+ /* Communicate a row in DD direction d.
+ * The communicators are setup such that the root always has rank 0.
+ */
+#ifdef GMX_MPI
+ MPI_Gather(sbuf, load->nload*sizeof(float), MPI_BYTE,
+ load->load, load->nload*sizeof(float), MPI_BYTE,
+ 0, comm->mpi_comm_load[d]);
+#endif
+ if (dd->ci[dim] == dd->master_ci[dim])
+ {
+ /* We are the root, process this row */
+ if (comm->bDynLoadBal)
+ {
+ root = comm->root[d];
+ }
+ load->sum = 0;
+ load->max = 0;
+ load->sum_m = 0;
+ load->cvol_min = 1;
+ load->flags = 0;
+ load->mdf = 0;
+ load->pme = 0;
+ pos = 0;
+ for (i = 0; i < dd->nc[dim]; i++)
+ {
+ load->sum += load->load[pos++];
+ load->max = max(load->max, load->load[pos]);
+ pos++;
+ if (dd->bGridJump)
+ {
+ if (root->bLimited)
+ {
+ /* This direction could not be load balanced properly,
+ * therefore we need to use the maximum iso the average load.
+ */
+ load->sum_m = max(load->sum_m, load->load[pos]);
+ }
+ else
+ {
+ load->sum_m += load->load[pos];
+ }
+ pos++;
+ load->cvol_min = min(load->cvol_min, load->load[pos]);
+ pos++;
+ if (d < dd->ndim-1)
+ {
+ load->flags = (int)(load->load[pos++] + 0.5);
+ }
+ if (d > 0)
+ {
+ root->cell_f_max0[i] = load->load[pos++];
+ root->cell_f_min1[i] = load->load[pos++];
+ }
+ }
+ if (bSepPME)
+ {
+ load->mdf = max(load->mdf, load->load[pos]);
+ pos++;
+ load->pme = max(load->pme, load->load[pos]);
+ pos++;
+ }
+ }
+ if (comm->bDynLoadBal && root->bLimited)
+ {
+ load->sum_m *= dd->nc[dim];
+ load->flags |= (1<<d);
+ }
+ }
+ }
+ }
+
+ if (DDMASTER(dd))
+ {
+ comm->nload += dd_load_count(comm);
+ comm->load_step += comm->cycl[ddCyclStep];
+ comm->load_sum += comm->load[0].sum;
+ comm->load_max += comm->load[0].max;
+ if (comm->bDynLoadBal)
+ {
+ for (d = 0; d < dd->ndim; d++)
+ {
+ if (comm->load[0].flags & (1<<d))
+ {
+ comm->load_lim[d]++;
+ }
+ }
+ }
+ if (bSepPME)
+ {
+ comm->load_mdf += comm->load[0].mdf;
+ comm->load_pme += comm->load[0].pme;
+ }
+ }
+
+ wallcycle_stop(wcycle, ewcDDCOMMLOAD);
+
+ if (debug)
+ {
+ fprintf(debug, "get_load_distribution finished\n");
+ }
+}
+
+static float dd_force_imb_perf_loss(gmx_domdec_t *dd)
+{
+ /* Return the relative performance loss on the total run time
+ * due to the force calculation load imbalance.
+ */
+ if (dd->comm->nload > 0)
+ {
+ return
+ (dd->comm->load_max*dd->nnodes - dd->comm->load_sum)/
+ (dd->comm->load_step*dd->nnodes);
+ }
+ else
+ {
+ return 0;
+ }
+}
+
+static void print_dd_load_av(FILE *fplog, gmx_domdec_t *dd)
+{
+ char buf[STRLEN];
+ int npp, npme, nnodes, d, limp;
+ float imbal, pme_f_ratio, lossf, lossp = 0;
+ gmx_bool bLim;
+ gmx_domdec_comm_t *comm;
+
+ comm = dd->comm;
+ if (DDMASTER(dd) && comm->nload > 0)
+ {
+ npp = dd->nnodes;
+ npme = (dd->pme_nodeid >= 0) ? comm->npmenodes : 0;
+ nnodes = npp + npme;
+ imbal = comm->load_max*npp/comm->load_sum - 1;
+ lossf = dd_force_imb_perf_loss(dd);
+ sprintf(buf, " Average load imbalance: %.1f %%\n", imbal*100);
+ fprintf(fplog, "%s", buf);
+ fprintf(stderr, "\n");
+ fprintf(stderr, "%s", buf);
+ sprintf(buf, " Part of the total run time spent waiting due to load imbalance: %.1f %%\n", lossf*100);
+ fprintf(fplog, "%s", buf);
+ fprintf(stderr, "%s", buf);
+ bLim = FALSE;
+ if (comm->bDynLoadBal)
+ {
+ sprintf(buf, " Steps where the load balancing was limited by -rdd, -rcon and/or -dds:");
+ for (d = 0; d < dd->ndim; d++)
+ {
+ limp = (200*comm->load_lim[d]+1)/(2*comm->nload);
+ sprintf(buf+strlen(buf), " %c %d %%", dim2char(dd->dim[d]), limp);
+ if (limp >= 50)
+ {
+ bLim = TRUE;
+ }
+ }
+ sprintf(buf+strlen(buf), "\n");
+ fprintf(fplog, "%s", buf);
+ fprintf(stderr, "%s", buf);
+ }
+ if (npme > 0)
+ {
+ pme_f_ratio = comm->load_pme/comm->load_mdf;
+ lossp = (comm->load_pme -comm->load_mdf)/comm->load_step;
+ if (lossp <= 0)
+ {
+ lossp *= (float)npme/(float)nnodes;
+ }
+ else
+ {
+ lossp *= (float)npp/(float)nnodes;
+ }
+ sprintf(buf, " Average PME mesh/force load: %5.3f\n", pme_f_ratio);
+ fprintf(fplog, "%s", buf);
+ fprintf(stderr, "%s", buf);
+ sprintf(buf, " Part of the total run time spent waiting due to PP/PME imbalance: %.1f %%\n", fabs(lossp)*100);
+ fprintf(fplog, "%s", buf);
+ fprintf(stderr, "%s", buf);
+ }
+ fprintf(fplog, "\n");
+ fprintf(stderr, "\n");
+
+ if (lossf >= DD_PERF_LOSS)
+ {
+ sprintf(buf,
+ "NOTE: %.1f %% of the available CPU time was lost due to load imbalance\n"
+ " in the domain decomposition.\n", lossf*100);
+ if (!comm->bDynLoadBal)
+ {
+ sprintf(buf+strlen(buf), " You might want to use dynamic load balancing (option -dlb.)\n");
+ }
+ else if (bLim)
+ {
+ sprintf(buf+strlen(buf), " You might want to decrease the cell size limit (options -rdd, -rcon and/or -dds).\n");
+ }
+ fprintf(fplog, "%s\n", buf);
+ fprintf(stderr, "%s\n", buf);
+ }
+ if (npme > 0 && fabs(lossp) >= DD_PERF_LOSS)
+ {
+ sprintf(buf,
+ "NOTE: %.1f %% performance was lost because the PME nodes\n"
+ " had %s work to do than the PP nodes.\n"
+ " You might want to %s the number of PME nodes\n"
+ " or %s the cut-off and the grid spacing.\n",
+ fabs(lossp*100),
+ (lossp < 0) ? "less" : "more",
+ (lossp < 0) ? "decrease" : "increase",
+ (lossp < 0) ? "decrease" : "increase");
+ fprintf(fplog, "%s\n", buf);
+ fprintf(stderr, "%s\n", buf);
+ }
+ }
+}
+
+static float dd_vol_min(gmx_domdec_t *dd)
+{
+ return dd->comm->load[0].cvol_min*dd->nnodes;
+}
+
+static gmx_bool dd_load_flags(gmx_domdec_t *dd)
+{
+ return dd->comm->load[0].flags;
+}
+
+static float dd_f_imbal(gmx_domdec_t *dd)
+{
+ return dd->comm->load[0].max*dd->nnodes/dd->comm->load[0].sum - 1;
+}
+
+float dd_pme_f_ratio(gmx_domdec_t *dd)
+{
+ if (dd->comm->cycl_n[ddCyclPME] > 0)
+ {
+ return dd->comm->load[0].pme/dd->comm->load[0].mdf;
+ }
+ else
+ {
+ return -1.0;
+ }
+}
+
+static void dd_print_load(FILE *fplog, gmx_domdec_t *dd, gmx_large_int_t step)
+{
+ int flags, d;
+ char buf[22];
+
+ flags = dd_load_flags(dd);
+ if (flags)
+ {
+ fprintf(fplog,
+ "DD load balancing is limited by minimum cell size in dimension");
+ for (d = 0; d < dd->ndim; d++)
+ {
+ if (flags & (1<<d))
+ {
+ fprintf(fplog, " %c", dim2char(dd->dim[d]));
+ }
+ }
+ fprintf(fplog, "\n");
+ }
+ fprintf(fplog, "DD step %s", gmx_step_str(step, buf));
+ if (dd->comm->bDynLoadBal)
+ {
+ fprintf(fplog, " vol min/aver %5.3f%c",
+ dd_vol_min(dd), flags ? '!' : ' ');
+ }
+ fprintf(fplog, " load imb.: force %4.1f%%", dd_f_imbal(dd)*100);
+ if (dd->comm->cycl_n[ddCyclPME])
+ {
+ fprintf(fplog, " pme mesh/force %5.3f", dd_pme_f_ratio(dd));
+ }
+ fprintf(fplog, "\n\n");
+}
+
+static void dd_print_load_verbose(gmx_domdec_t *dd)
+{
+ if (dd->comm->bDynLoadBal)
+ {
+ fprintf(stderr, "vol %4.2f%c ",
+ dd_vol_min(dd), dd_load_flags(dd) ? '!' : ' ');
+ }
+ fprintf(stderr, "imb F %2d%% ", (int)(dd_f_imbal(dd)*100+0.5));
+ if (dd->comm->cycl_n[ddCyclPME])
+ {
+ fprintf(stderr, "pme/F %4.2f ", dd_pme_f_ratio(dd));
+ }
+}
+
+#ifdef GMX_MPI
+static void make_load_communicator(gmx_domdec_t *dd, int dim_ind, ivec loc)
+{
+ MPI_Comm c_row;
+ int dim, i, rank;
+ ivec loc_c;
+ gmx_domdec_root_t *root;
+ gmx_bool bPartOfGroup = FALSE;
+
+ dim = dd->dim[dim_ind];
+ copy_ivec(loc, loc_c);
+ for (i = 0; i < dd->nc[dim]; i++)
+ {
+ loc_c[dim] = i;
+ rank = dd_index(dd->nc, loc_c);
+ if (rank == dd->rank)
+ {
+ /* This process is part of the group */
+ bPartOfGroup = TRUE;
+ }
+ }
+ MPI_Comm_split(dd->mpi_comm_all, bPartOfGroup ? 0 : MPI_UNDEFINED, dd->rank,
+ &c_row);
+ if (bPartOfGroup)
+ {
+ dd->comm->mpi_comm_load[dim_ind] = c_row;
+ if (dd->comm->eDLB != edlbNO)
+ {
+ if (dd->ci[dim] == dd->master_ci[dim])
+ {
+ /* This is the root process of this row */
+ snew(dd->comm->root[dim_ind], 1);
+ root = dd->comm->root[dim_ind];
+ snew(root->cell_f, DD_CELL_F_SIZE(dd, dim_ind));
+ snew(root->old_cell_f, dd->nc[dim]+1);
+ snew(root->bCellMin, dd->nc[dim]);
+ if (dim_ind > 0)
+ {
+ snew(root->cell_f_max0, dd->nc[dim]);
+ snew(root->cell_f_min1, dd->nc[dim]);
+ snew(root->bound_min, dd->nc[dim]);
+ snew(root->bound_max, dd->nc[dim]);
+ }
+ snew(root->buf_ncd, dd->nc[dim]);
+ }
+ else
+ {
+ /* This is not a root process, we only need to receive cell_f */
+ snew(dd->comm->cell_f_row, DD_CELL_F_SIZE(dd, dim_ind));
+ }
+ }
+ if (dd->ci[dim] == dd->master_ci[dim])
+ {
+ snew(dd->comm->load[dim_ind].load, dd->nc[dim]*DD_NLOAD_MAX);
+ }
+ }
+}
+#endif
+
+static void make_load_communicators(gmx_domdec_t *dd)
+{
+#ifdef GMX_MPI
+ int dim0, dim1, i, j;
+ ivec loc;
+
+ if (debug)
+ {
+ fprintf(debug, "Making load communicators\n");
+ }
+
+ snew(dd->comm->load, dd->ndim);
+ snew(dd->comm->mpi_comm_load, dd->ndim);
+
+ clear_ivec(loc);
+ make_load_communicator(dd, 0, loc);
+ if (dd->ndim > 1)
+ {
+ dim0 = dd->dim[0];
+ for (i = 0; i < dd->nc[dim0]; i++)
+ {
+ loc[dim0] = i;
+ make_load_communicator(dd, 1, loc);
+ }
+ }
+ if (dd->ndim > 2)
+ {
+ dim0 = dd->dim[0];
+ for (i = 0; i < dd->nc[dim0]; i++)
+ {
+ loc[dim0] = i;
+ dim1 = dd->dim[1];
+ for (j = 0; j < dd->nc[dim1]; j++)
+ {
+ loc[dim1] = j;
+ make_load_communicator(dd, 2, loc);
+ }
+ }
+ }
+
+ if (debug)
+ {
+ fprintf(debug, "Finished making load communicators\n");
+ }
+#endif
+}
+
+void setup_dd_grid(FILE *fplog, gmx_domdec_t *dd)
+{
+ gmx_bool bZYX;
+ int d, dim, i, j, m;
+ ivec tmp, s;
+ int nzone, nzonep;
+ ivec dd_zp[DD_MAXIZONE];
+ gmx_domdec_zones_t *zones;
+ gmx_domdec_ns_ranges_t *izone;
+
+ for (d = 0; d < dd->ndim; d++)
+ {
+ dim = dd->dim[d];
+ copy_ivec(dd->ci, tmp);
+ tmp[dim] = (tmp[dim] + 1) % dd->nc[dim];
+ dd->neighbor[d][0] = ddcoord2ddnodeid(dd, tmp);
+ copy_ivec(dd->ci, tmp);
+ tmp[dim] = (tmp[dim] - 1 + dd->nc[dim]) % dd->nc[dim];
+ dd->neighbor[d][1] = ddcoord2ddnodeid(dd, tmp);
+ if (debug)
+ {
+ fprintf(debug, "DD rank %d neighbor ranks in dir %d are + %d - %d\n",
+ dd->rank, dim,
+ dd->neighbor[d][0],
+ dd->neighbor[d][1]);
+ }
+ }
+
+ if (fplog)
+ {
+ fprintf(fplog, "\nMaking %dD domain decomposition grid %d x %d x %d, home cell index %d %d %d\n\n",
+ dd->ndim,
+ dd->nc[XX], dd->nc[YY], dd->nc[ZZ],
+ dd->ci[XX], dd->ci[YY], dd->ci[ZZ]);
+ }
+ switch (dd->ndim)
+ {
+ case 3:
+ nzone = dd_z3n;
+ nzonep = dd_zp3n;
+ for (i = 0; i < nzonep; i++)
+ {
+ copy_ivec(dd_zp3[i], dd_zp[i]);
+ }
+ break;
+ case 2:
+ nzone = dd_z2n;
+ nzonep = dd_zp2n;
+ for (i = 0; i < nzonep; i++)
+ {
+ copy_ivec(dd_zp2[i], dd_zp[i]);
+ }
+ break;
+ case 1:
+ nzone = dd_z1n;
+ nzonep = dd_zp1n;
+ for (i = 0; i < nzonep; i++)
+ {
+ copy_ivec(dd_zp1[i], dd_zp[i]);
+ }
+ break;
+ default:
+ gmx_fatal(FARGS, "Can only do 1, 2 or 3D domain decomposition");
+ nzone = 0;
+ nzonep = 0;
+ }
+
+ zones = &dd->comm->zones;
+
+ for (i = 0; i < nzone; i++)
+ {
+ m = 0;
+ clear_ivec(zones->shift[i]);
+ for (d = 0; d < dd->ndim; d++)
+ {
+ zones->shift[i][dd->dim[d]] = dd_zo[i][m++];
+ }
+ }
+
+ zones->n = nzone;
+ for (i = 0; i < nzone; i++)
+ {
+ for (d = 0; d < DIM; d++)
+ {
+ s[d] = dd->ci[d] - zones->shift[i][d];
+ if (s[d] < 0)
+ {
+ s[d] += dd->nc[d];
+ }
+ else if (s[d] >= dd->nc[d])
+ {
+ s[d] -= dd->nc[d];
+ }
+ }
+ }
+ zones->nizone = nzonep;
+ for (i = 0; i < zones->nizone; i++)
+ {
+ if (dd_zp[i][0] != i)
+ {
+ gmx_fatal(FARGS, "Internal inconsistency in the dd grid setup");
+ }
+ izone = &zones->izone[i];
+ izone->j0 = dd_zp[i][1];
+ izone->j1 = dd_zp[i][2];
+ for (dim = 0; dim < DIM; dim++)
+ {
+ if (dd->nc[dim] == 1)
+ {
+ /* All shifts should be allowed */
+ izone->shift0[dim] = -1;
+ izone->shift1[dim] = 1;
+ }
+ else
+ {
+ /*
+ izone->shift0[d] = 0;
+ izone->shift1[d] = 0;
+ for(j=izone->j0; j<izone->j1; j++) {
+ if (dd->shift[j][d] > dd->shift[i][d])
+ izone->shift0[d] = -1;
+ if (dd->shift[j][d] < dd->shift[i][d])
+ izone->shift1[d] = 1;
+ }
+ */
+
+ int shift_diff;
+
+ /* Assume the shift are not more than 1 cell */
+ izone->shift0[dim] = 1;
+ izone->shift1[dim] = -1;
+ for (j = izone->j0; j < izone->j1; j++)
+ {
+ shift_diff = zones->shift[j][dim] - zones->shift[i][dim];
+ if (shift_diff < izone->shift0[dim])
+ {
+ izone->shift0[dim] = shift_diff;
+ }
+ if (shift_diff > izone->shift1[dim])
+ {
+ izone->shift1[dim] = shift_diff;
+ }
+ }
+ }
+ }
+ }
+
+ if (dd->comm->eDLB != edlbNO)
+ {
+ snew(dd->comm->root, dd->ndim);
+ }
+
+ if (dd->comm->bRecordLoad)
+ {
+ make_load_communicators(dd);
+ }
+}
+
+static void make_pp_communicator(FILE *fplog, t_commrec *cr, int reorder)
+{
+ gmx_domdec_t *dd;
+ gmx_domdec_comm_t *comm;
+ int i, rank, *buf;
+ ivec periods;
+#ifdef GMX_MPI
+ MPI_Comm comm_cart;
+#endif
+
+ dd = cr->dd;
+ comm = dd->comm;
+
+#ifdef GMX_MPI
+ if (comm->bCartesianPP)
+ {
+ /* Set up cartesian communication for the particle-particle part */
+ if (fplog)
+ {
+ fprintf(fplog, "Will use a Cartesian communicator: %d x %d x %d\n",
+ dd->nc[XX], dd->nc[YY], dd->nc[ZZ]);
+ }
+
+ for (i = 0; i < DIM; i++)
+ {
+ periods[i] = TRUE;
+ }
+ MPI_Cart_create(cr->mpi_comm_mygroup, DIM, dd->nc, periods, reorder,
+ &comm_cart);
+ /* We overwrite the old communicator with the new cartesian one */
+ cr->mpi_comm_mygroup = comm_cart;
+ }
+
+ dd->mpi_comm_all = cr->mpi_comm_mygroup;
+ MPI_Comm_rank(dd->mpi_comm_all, &dd->rank);
+
+ if (comm->bCartesianPP_PME)
+ {
+ /* Since we want to use the original cartesian setup for sim,
+ * and not the one after split, we need to make an index.
+ */
+ snew(comm->ddindex2ddnodeid, dd->nnodes);
+ comm->ddindex2ddnodeid[dd_index(dd->nc, dd->ci)] = dd->rank;
+ gmx_sumi(dd->nnodes, comm->ddindex2ddnodeid, cr);
+ /* Get the rank of the DD master,
+ * above we made sure that the master node is a PP node.
+ */
+ if (MASTER(cr))
+ {
+ rank = dd->rank;
+ }
+ else
+ {
+ rank = 0;
+ }
+ MPI_Allreduce(&rank, &dd->masterrank, 1, MPI_INT, MPI_SUM, dd->mpi_comm_all);
+ }
+ else if (comm->bCartesianPP)
+ {
+ if (cr->npmenodes == 0)
+ {
+ /* The PP communicator is also
+ * the communicator for this simulation
+ */
+ cr->mpi_comm_mysim = cr->mpi_comm_mygroup;
+ }
+ cr->nodeid = dd->rank;
+
+ MPI_Cart_coords(dd->mpi_comm_all, dd->rank, DIM, dd->ci);
+
+ /* We need to make an index to go from the coordinates
+ * to the nodeid of this simulation.
+ */
+ snew(comm->ddindex2simnodeid, dd->nnodes);
+ snew(buf, dd->nnodes);
+ if (cr->duty & DUTY_PP)
+ {
+ buf[dd_index(dd->nc, dd->ci)] = cr->sim_nodeid;
+ }
+ /* Communicate the ddindex to simulation nodeid index */
+ MPI_Allreduce(buf, comm->ddindex2simnodeid, dd->nnodes, MPI_INT, MPI_SUM,
+ cr->mpi_comm_mysim);
+ sfree(buf);
+
+ /* Determine the master coordinates and rank.
+ * The DD master should be the same node as the master of this sim.
+ */
+ for (i = 0; i < dd->nnodes; i++)
+ {
+ if (comm->ddindex2simnodeid[i] == 0)
+ {
+ ddindex2xyz(dd->nc, i, dd->master_ci);
+ MPI_Cart_rank(dd->mpi_comm_all, dd->master_ci, &dd->masterrank);
+ }
+ }
+ if (debug)
+ {
+ fprintf(debug, "The master rank is %d\n", dd->masterrank);
+ }
+ }
+ else
+ {
+ /* No Cartesian communicators */
+ /* We use the rank in dd->comm->all as DD index */
+ ddindex2xyz(dd->nc, dd->rank, dd->ci);
+ /* The simulation master nodeid is 0, so the DD master rank is also 0 */
+ dd->masterrank = 0;
+ clear_ivec(dd->master_ci);
+ }
+#endif
+
+ if (fplog)
+ {
+ fprintf(fplog,
+ "Domain decomposition nodeid %d, coordinates %d %d %d\n\n",
+ dd->rank, dd->ci[XX], dd->ci[YY], dd->ci[ZZ]);
+ }
+ if (debug)
+ {
+ fprintf(debug,
+ "Domain decomposition nodeid %d, coordinates %d %d %d\n\n",
+ dd->rank, dd->ci[XX], dd->ci[YY], dd->ci[ZZ]);
+ }
+}
+
+static void receive_ddindex2simnodeid(t_commrec *cr)
+{
+ gmx_domdec_t *dd;
+
+ gmx_domdec_comm_t *comm;
+ int *buf;
+
+ dd = cr->dd;
+ comm = dd->comm;
+
+#ifdef GMX_MPI
+ if (!comm->bCartesianPP_PME && comm->bCartesianPP)
+ {
+ snew(comm->ddindex2simnodeid, dd->nnodes);
+ snew(buf, dd->nnodes);
+ if (cr->duty & DUTY_PP)
+ {
+ buf[dd_index(dd->nc, dd->ci)] = cr->sim_nodeid;
+ }
+#ifdef GMX_MPI
+ /* Communicate the ddindex to simulation nodeid index */
+ MPI_Allreduce(buf, comm->ddindex2simnodeid, dd->nnodes, MPI_INT, MPI_SUM,
+ cr->mpi_comm_mysim);
+#endif
+ sfree(buf);
+ }
+#endif
+}
+
+static gmx_domdec_master_t *init_gmx_domdec_master_t(gmx_domdec_t *dd,
+ int ncg, int natoms)
+{
+ gmx_domdec_master_t *ma;
+ int i;
+
+ snew(ma, 1);
+
+ snew(ma->ncg, dd->nnodes);
+ snew(ma->index, dd->nnodes+1);
+ snew(ma->cg, ncg);
+ snew(ma->nat, dd->nnodes);
+ snew(ma->ibuf, dd->nnodes*2);
+ snew(ma->cell_x, DIM);
+ for (i = 0; i < DIM; i++)
+ {
+ snew(ma->cell_x[i], dd->nc[i]+1);
+ }
+
+ if (dd->nnodes <= GMX_DD_NNODES_SENDRECV)
+ {
+ ma->vbuf = NULL;
+ }
+ else
+ {
+ snew(ma->vbuf, natoms);
+ }
+
+ return ma;
+}
+
+static void split_communicator(FILE *fplog, t_commrec *cr, int dd_node_order,
+ int reorder)
+{
+ gmx_domdec_t *dd;
+ gmx_domdec_comm_t *comm;
+ int i, rank;
+ gmx_bool bDiv[DIM];
+ ivec periods;
+#ifdef GMX_MPI
+ MPI_Comm comm_cart;
+#endif
+
+ dd = cr->dd;
+ comm = dd->comm;
+
+ if (comm->bCartesianPP)
+ {
+ for (i = 1; i < DIM; i++)
+ {
+ bDiv[i] = ((cr->npmenodes*dd->nc[i]) % (dd->nnodes) == 0);
+ }
+ if (bDiv[YY] || bDiv[ZZ])
+ {
+ comm->bCartesianPP_PME = TRUE;
+ /* If we have 2D PME decomposition, which is always in x+y,
+ * we stack the PME only nodes in z.
+ * Otherwise we choose the direction that provides the thinnest slab
+ * of PME only nodes as this will have the least effect
+ * on the PP communication.
+ * But for the PME communication the opposite might be better.
+ */
+ if (bDiv[ZZ] && (comm->npmenodes_y > 1 ||
+ !bDiv[YY] ||
+ dd->nc[YY] > dd->nc[ZZ]))
+ {
+ comm->cartpmedim = ZZ;
+ }
+ else
+ {
+ comm->cartpmedim = YY;
+ }
+ comm->ntot[comm->cartpmedim]
+ += (cr->npmenodes*dd->nc[comm->cartpmedim])/dd->nnodes;
+ }
+ else if (fplog)
+ {
+ fprintf(fplog, "#pmenodes (%d) is not a multiple of nx*ny (%d*%d) or nx*nz (%d*%d)\n", cr->npmenodes, dd->nc[XX], dd->nc[YY], dd->nc[XX], dd->nc[ZZ]);
+ fprintf(fplog,
+ "Will not use a Cartesian communicator for PP <-> PME\n\n");
+ }
+ }
+
+#ifdef GMX_MPI
+ if (comm->bCartesianPP_PME)
+ {
+ if (fplog)
+ {
+ fprintf(fplog, "Will use a Cartesian communicator for PP <-> PME: %d x %d x %d\n", comm->ntot[XX], comm->ntot[YY], comm->ntot[ZZ]);
+ }
+
+ for (i = 0; i < DIM; i++)
+ {
+ periods[i] = TRUE;
+ }
+ MPI_Cart_create(cr->mpi_comm_mysim, DIM, comm->ntot, periods, reorder,
+ &comm_cart);
+
+ MPI_Comm_rank(comm_cart, &rank);
+ if (MASTERNODE(cr) && rank != 0)
+ {
+ gmx_fatal(FARGS, "MPI rank 0 was renumbered by MPI_Cart_create, we do not allow this");
+ }
+
+ /* With this assigment we loose the link to the original communicator
+ * which will usually be MPI_COMM_WORLD, unless have multisim.
+ */
+ cr->mpi_comm_mysim = comm_cart;
+ cr->sim_nodeid = rank;
+
+ MPI_Cart_coords(cr->mpi_comm_mysim, cr->sim_nodeid, DIM, dd->ci);
+
+ if (fplog)
+ {
+ fprintf(fplog, "Cartesian nodeid %d, coordinates %d %d %d\n\n",
+ cr->sim_nodeid, dd->ci[XX], dd->ci[YY], dd->ci[ZZ]);
+ }
+
+ if (dd->ci[comm->cartpmedim] < dd->nc[comm->cartpmedim])
+ {
+ cr->duty = DUTY_PP;
+ }
+ if (cr->npmenodes == 0 ||
+ dd->ci[comm->cartpmedim] >= dd->nc[comm->cartpmedim])
+ {
+ cr->duty = DUTY_PME;
+ }
+
+ /* Split the sim communicator into PP and PME only nodes */
+ MPI_Comm_split(cr->mpi_comm_mysim,
+ cr->duty,
+ dd_index(comm->ntot, dd->ci),
+ &cr->mpi_comm_mygroup);
+ }
+ else
+ {
+ switch (dd_node_order)
+ {
+ case ddnoPP_PME:
+ if (fplog)
+ {
+ fprintf(fplog, "Order of the nodes: PP first, PME last\n");
+ }
+ break;
+ case ddnoINTERLEAVE:
+ /* Interleave the PP-only and PME-only nodes,
+ * as on clusters with dual-core machines this will double
+ * the communication bandwidth of the PME processes
+ * and thus speed up the PP <-> PME and inter PME communication.
+ */
+ if (fplog)
+ {
+ fprintf(fplog, "Interleaving PP and PME nodes\n");
+ }
+ comm->pmenodes = dd_pmenodes(cr);
+ break;
+ case ddnoCARTESIAN:
+ break;
+ default:
+ gmx_fatal(FARGS, "Unknown dd_node_order=%d", dd_node_order);
+ }
+
+ if (dd_simnode2pmenode(cr, cr->sim_nodeid) == -1)
+ {
+ cr->duty = DUTY_PME;
+ }
+ else
+ {
+ cr->duty = DUTY_PP;
+ }
+
+ /* Split the sim communicator into PP and PME only nodes */
+ MPI_Comm_split(cr->mpi_comm_mysim,
+ cr->duty,
+ cr->nodeid,
+ &cr->mpi_comm_mygroup);
+ MPI_Comm_rank(cr->mpi_comm_mygroup, &cr->nodeid);
+ }
+#endif
+
+ if (fplog)
+ {
+ fprintf(fplog, "This is a %s only node\n\n",
+ (cr->duty & DUTY_PP) ? "particle-particle" : "PME-mesh");
+ }
+}
+
+void make_dd_communicators(FILE *fplog, t_commrec *cr, int dd_node_order)
+{
+ gmx_domdec_t *dd;
+ gmx_domdec_comm_t *comm;
+ int CartReorder;
+
+ dd = cr->dd;
+ comm = dd->comm;
+
+ copy_ivec(dd->nc, comm->ntot);
+
+ comm->bCartesianPP = (dd_node_order == ddnoCARTESIAN);
+ comm->bCartesianPP_PME = FALSE;
+
+ /* Reorder the nodes by default. This might change the MPI ranks.
+ * Real reordering is only supported on very few architectures,
+ * Blue Gene is one of them.
+ */
+ CartReorder = (getenv("GMX_NO_CART_REORDER") == NULL);
+
+ if (cr->npmenodes > 0)
+ {
+ /* Split the communicator into a PP and PME part */
+ split_communicator(fplog, cr, dd_node_order, CartReorder);
+ if (comm->bCartesianPP_PME)
+ {
+ /* We (possibly) reordered the nodes in split_communicator,
+ * so it is no longer required in make_pp_communicator.
+ */
+ CartReorder = FALSE;
+ }
+ }
+ else
+ {
+ /* All nodes do PP and PME */
+#ifdef GMX_MPI
+ /* We do not require separate communicators */
+ cr->mpi_comm_mygroup = cr->mpi_comm_mysim;
+#endif
+ }
+
+ if (cr->duty & DUTY_PP)
+ {
+ /* Copy or make a new PP communicator */
+ make_pp_communicator(fplog, cr, CartReorder);
+ }
+ else
+ {
+ receive_ddindex2simnodeid(cr);
+ }
+
+ if (!(cr->duty & DUTY_PME))
+ {
+ /* Set up the commnuication to our PME node */
+ dd->pme_nodeid = dd_simnode2pmenode(cr, cr->sim_nodeid);
+ dd->pme_receive_vir_ener = receive_vir_ener(cr);
+ if (debug)
+ {
+ fprintf(debug, "My pme_nodeid %d receive ener %d\n",
+ dd->pme_nodeid, dd->pme_receive_vir_ener);
+ }
+ }
+ else
+ {
+ dd->pme_nodeid = -1;
+ }
+
+ if (DDMASTER(dd))
+ {
+ dd->ma = init_gmx_domdec_master_t(dd,
+ comm->cgs_gl.nr,
+ comm->cgs_gl.index[comm->cgs_gl.nr]);
+ }
+}
+
+static real *get_slb_frac(FILE *fplog, const char *dir, int nc, const char *size_string)
+{
+ real *slb_frac, tot;
+ int i, n;
+ double dbl;
+
+ slb_frac = NULL;
+ if (nc > 1 && size_string != NULL)
+ {
+ if (fplog)
+ {
+ fprintf(fplog, "Using static load balancing for the %s direction\n",
+ dir);
+ }
+ snew(slb_frac, nc);
+ tot = 0;
+ for (i = 0; i < nc; i++)
+ {
+ dbl = 0;
+ sscanf(size_string, "%lf%n", &dbl, &n);
+ if (dbl == 0)
+ {
+ gmx_fatal(FARGS, "Incorrect or not enough DD cell size entries for direction %s: '%s'", dir, size_string);
+ }
+ slb_frac[i] = dbl;
+ size_string += n;
+ tot += slb_frac[i];
+ }
+ /* Normalize */
+ if (fplog)
+ {
+ fprintf(fplog, "Relative cell sizes:");
+ }
+ for (i = 0; i < nc; i++)
+ {
+ slb_frac[i] /= tot;
+ if (fplog)
+ {
+ fprintf(fplog, " %5.3f", slb_frac[i]);
+ }
+ }
+ if (fplog)
+ {
+ fprintf(fplog, "\n");
+ }
+ }
+
+ return slb_frac;
+}
+
+static int multi_body_bondeds_count(gmx_mtop_t *mtop)
+{
+ int n, nmol, ftype;
+ gmx_mtop_ilistloop_t iloop;
+ t_ilist *il;
+
+ n = 0;
+ iloop = gmx_mtop_ilistloop_init(mtop);
+ while (gmx_mtop_ilistloop_next(iloop, &il, &nmol))
+ {
+ for (ftype = 0; ftype < F_NRE; ftype++)
+ {
+ if ((interaction_function[ftype].flags & IF_BOND) &&
+ NRAL(ftype) > 2)
+ {
+ n += nmol*il[ftype].nr/(1 + NRAL(ftype));
+ }
+ }
+ }
+
+ return n;
+}
+
+static int dd_nst_env(FILE *fplog, const char *env_var, int def)
+{
+ char *val;
+ int nst;
+
+ nst = def;
+ val = getenv(env_var);
+ if (val)
+ {
+ if (sscanf(val, "%d", &nst) <= 0)
+ {
+ nst = 1;
+ }
+ if (fplog)
+ {
+ fprintf(fplog, "Found env.var. %s = %s, using value %d\n",
+ env_var, val, nst);
+ }
+ }
+
+ return nst;
+}
+
+static void dd_warning(t_commrec *cr, FILE *fplog, const char *warn_string)
+{
+ if (MASTER(cr))
+ {
+ fprintf(stderr, "\n%s\n", warn_string);
+ }
+ if (fplog)
+ {
+ fprintf(fplog, "\n%s\n", warn_string);
+ }
+}
+
+static void check_dd_restrictions(t_commrec *cr, gmx_domdec_t *dd,
+ t_inputrec *ir, FILE *fplog)
+{
+ if (ir->ePBC == epbcSCREW &&
+ (dd->nc[XX] == 1 || dd->nc[YY] > 1 || dd->nc[ZZ] > 1))
+ {
+ gmx_fatal(FARGS, "With pbc=%s can only do domain decomposition in the x-direction", epbc_names[ir->ePBC]);
+ }
+
+ if (ir->ns_type == ensSIMPLE)
+ {
+ gmx_fatal(FARGS, "Domain decomposition does not support simple neighbor searching, use grid searching or use particle decomposition");
+ }
+
+ if (ir->nstlist == 0)
+ {
+ gmx_fatal(FARGS, "Domain decomposition does not work with nstlist=0");
+ }
+
+ if (ir->comm_mode == ecmANGULAR && ir->ePBC != epbcNONE)
+ {
+ dd_warning(cr, fplog, "comm-mode angular will give incorrect results when the comm group partially crosses a periodic boundary");
+ }
+}
+
+static real average_cellsize_min(gmx_domdec_t *dd, gmx_ddbox_t *ddbox)
+{
+ int di, d;
+ real r;
+
+ r = ddbox->box_size[XX];
+ for (di = 0; di < dd->ndim; di++)
+ {
+ d = dd->dim[di];
+ /* Check using the initial average cell size */
+ r = min(r, ddbox->box_size[d]*ddbox->skew_fac[d]/dd->nc[d]);
+ }
+
+ return r;
+}
+
+static int check_dlb_support(FILE *fplog, t_commrec *cr,
+ const char *dlb_opt, gmx_bool bRecordLoad,
+ unsigned long Flags, t_inputrec *ir)
+{
+ gmx_domdec_t *dd;
+ int eDLB = -1;
+ char buf[STRLEN];
+
+ switch (dlb_opt[0])
+ {
+ case 'a': eDLB = edlbAUTO; break;
+ case 'n': eDLB = edlbNO; break;
+ case 'y': eDLB = edlbYES; break;
+ default: gmx_incons("Unknown dlb_opt");
+ }
+
+ if (Flags & MD_RERUN)
+ {
+ return edlbNO;
+ }
+
+ if (!EI_DYNAMICS(ir->eI))
+ {
+ if (eDLB == edlbYES)
+ {
+ sprintf(buf, "NOTE: dynamic load balancing is only supported with dynamics, not with integrator '%s'\n", EI(ir->eI));
+ dd_warning(cr, fplog, buf);
+ }
+
+ return edlbNO;
+ }
+
+ if (!bRecordLoad)
+ {
+ dd_warning(cr, fplog, "NOTE: Cycle counting is not supported on this architecture, will not use dynamic load balancing\n");
+
+ return edlbNO;
+ }
+
+ if (Flags & MD_REPRODUCIBLE)
+ {
+ switch (eDLB)
+ {
+ case edlbNO:
+ break;
+ case edlbAUTO:
+ dd_warning(cr, fplog, "NOTE: reproducibility requested, will not use dynamic load balancing\n");
+ eDLB = edlbNO;
+ break;
+ case edlbYES:
+ dd_warning(cr, fplog, "WARNING: reproducibility requested with dynamic load balancing, the simulation will NOT be binary reproducible\n");
+ break;
+ default:
+ gmx_fatal(FARGS, "Death horror: undefined case (%d) for load balancing choice", eDLB);
+ break;
+ }
+ }
+
+ return eDLB;
+}
+
+static void set_dd_dim(FILE *fplog, gmx_domdec_t *dd)
+{
+ int dim;
+
+ dd->ndim = 0;
+ if (getenv("GMX_DD_ORDER_ZYX") != NULL)
+ {
+ /* Decomposition order z,y,x */
+ if (fplog)
+ {
+ fprintf(fplog, "Using domain decomposition order z, y, x\n");
+ }
+ for (dim = DIM-1; dim >= 0; dim--)
+ {
+ if (dd->nc[dim] > 1)
+ {
+ dd->dim[dd->ndim++] = dim;
+ }
+ }
+ }
+ else
+ {
+ /* Decomposition order x,y,z */
+ for (dim = 0; dim < DIM; dim++)
+ {
+ if (dd->nc[dim] > 1)
+ {
+ dd->dim[dd->ndim++] = dim;
+ }
+ }
+ }
+}
+
+static gmx_domdec_comm_t *init_dd_comm()
+{
+ gmx_domdec_comm_t *comm;
+ int i;
+
+ snew(comm, 1);
+ snew(comm->cggl_flag, DIM*2);
+ snew(comm->cgcm_state, DIM*2);
+ for (i = 0; i < DIM*2; i++)
+ {
+ comm->cggl_flag_nalloc[i] = 0;
+ comm->cgcm_state_nalloc[i] = 0;
+ }
+
+ comm->nalloc_int = 0;
+ comm->buf_int = NULL;
+
+ vec_rvec_init(&comm->vbuf);
+
+ comm->n_load_have = 0;
+ comm->n_load_collect = 0;
+
+ for (i = 0; i < ddnatNR-ddnatZONE; i++)
+ {
+ comm->sum_nat[i] = 0;
+ }
+ comm->ndecomp = 0;
+ comm->nload = 0;
+ comm->load_step = 0;
+ comm->load_sum = 0;
+ comm->load_max = 0;
+ clear_ivec(comm->load_lim);
+ comm->load_mdf = 0;
+ comm->load_pme = 0;
+
+ return comm;
+}
+
+gmx_domdec_t *init_domain_decomposition(FILE *fplog, t_commrec *cr,
+ unsigned long Flags,
+ ivec nc,
+ real comm_distance_min, real rconstr,
+ const char *dlb_opt, real dlb_scale,
+ const char *sizex, const char *sizey, const char *sizez,
+ gmx_mtop_t *mtop, t_inputrec *ir,
+ matrix box, rvec *x,
+ gmx_ddbox_t *ddbox,
+ int *npme_x, int *npme_y)
+{
+ gmx_domdec_t *dd;
+ gmx_domdec_comm_t *comm;
+ int recload;
+ int d, i, j;
+ real r_2b, r_mb, r_bonded = -1, r_bonded_limit = -1, limit, acs;
+ gmx_bool bC;
+ char buf[STRLEN];
+
+ if (fplog)
+ {
+ fprintf(fplog,
+ "\nInitializing Domain Decomposition on %d nodes\n", cr->nnodes);
+ }
+
+ snew(dd, 1);
+
+ dd->comm = init_dd_comm();
+ comm = dd->comm;
+ snew(comm->cggl_flag, DIM*2);
+ snew(comm->cgcm_state, DIM*2);
+
+ dd->npbcdim = ePBC2npbcdim(ir->ePBC);
+ dd->bScrewPBC = (ir->ePBC == epbcSCREW);
+
+ dd->bSendRecv2 = dd_nst_env(fplog, "GMX_DD_SENDRECV2", 0);
+ comm->dlb_scale_lim = dd_nst_env(fplog, "GMX_DLB_MAX", 10);
+ comm->eFlop = dd_nst_env(fplog, "GMX_DLB_FLOP", 0);
+ recload = dd_nst_env(fplog, "GMX_DD_LOAD", 1);
+ comm->nstSortCG = dd_nst_env(fplog, "GMX_DD_SORT", 1);
+ comm->nstDDDump = dd_nst_env(fplog, "GMX_DD_DUMP", 0);
+ comm->nstDDDumpGrid = dd_nst_env(fplog, "GMX_DD_DUMP_GRID", 0);
+ comm->DD_debug = dd_nst_env(fplog, "GMX_DD_DEBUG", 0);
+
+ dd->pme_recv_f_alloc = 0;
+ dd->pme_recv_f_buf = NULL;
+
+ if (dd->bSendRecv2 && fplog)
+ {
+ fprintf(fplog, "Will use two sequential MPI_Sendrecv calls instead of two simultaneous non-blocking MPI_Irecv and MPI_Isend pairs for constraint and vsite communication\n");
+ }
+ if (comm->eFlop)
+ {
+ if (fplog)
+ {
+ fprintf(fplog, "Will load balance based on FLOP count\n");
+ }
+ if (comm->eFlop > 1)
+ {
+ srand(1+cr->nodeid);
+ }
+ comm->bRecordLoad = TRUE;
+ }
+ else
+ {
+ comm->bRecordLoad = (wallcycle_have_counter() && recload > 0);
+
+ }
+
+ comm->eDLB = check_dlb_support(fplog, cr, dlb_opt, comm->bRecordLoad, Flags, ir);
+
+ comm->bDynLoadBal = (comm->eDLB == edlbYES);
+ if (fplog)
+ {
+ fprintf(fplog, "Dynamic load balancing: %s\n", edlb_names[comm->eDLB]);
+ }
+ dd->bGridJump = comm->bDynLoadBal;
+ comm->bPMELoadBalDLBLimits = FALSE;
+
+ if (comm->nstSortCG)
+ {
+ if (fplog)
+ {
+ if (comm->nstSortCG == 1)
+ {
+ fprintf(fplog, "Will sort the charge groups at every domain (re)decomposition\n");
+ }
+ else
+ {
+ fprintf(fplog, "Will sort the charge groups every %d steps\n",
+ comm->nstSortCG);
+ }
+ }
+ snew(comm->sort, 1);
+ }
+ else
+ {
+ if (fplog)
+ {
+ fprintf(fplog, "Will not sort the charge groups\n");
+ }
+ }
+
+ comm->bCGs = (ncg_mtop(mtop) < mtop->natoms);
+
+ comm->bInterCGBondeds = (ncg_mtop(mtop) > mtop->mols.nr);
+ if (comm->bInterCGBondeds)
+ {
+ comm->bInterCGMultiBody = (multi_body_bondeds_count(mtop) > 0);
+ }
+ else
+ {
+ comm->bInterCGMultiBody = FALSE;
+ }
+
+ dd->bInterCGcons = inter_charge_group_constraints(mtop);
+ dd->bInterCGsettles = inter_charge_group_settles(mtop);
+
+ if (ir->rlistlong == 0)
+ {
+ /* Set the cut-off to some very large value,
+ * so we don't need if statements everywhere in the code.
+ * We use sqrt, since the cut-off is squared in some places.
+ */
+ comm->cutoff = GMX_CUTOFF_INF;
+ }
+ else
+ {
+ comm->cutoff = ir->rlistlong;
+ }
+ comm->cutoff_mbody = 0;
+
+ comm->cellsize_limit = 0;
+ comm->bBondComm = FALSE;
+
+ if (comm->bInterCGBondeds)
+ {
+ if (comm_distance_min > 0)
+ {
+ comm->cutoff_mbody = comm_distance_min;
+ if (Flags & MD_DDBONDCOMM)
+ {
+ comm->bBondComm = (comm->cutoff_mbody > comm->cutoff);
+ }
+ else
+ {
+ comm->cutoff = max(comm->cutoff, comm->cutoff_mbody);
+ }
+ r_bonded_limit = comm->cutoff_mbody;
+ }
+ else if (ir->bPeriodicMols)
+ {
+ /* Can not easily determine the required cut-off */
+ dd_warning(cr, fplog, "NOTE: Periodic molecules are present in this system. Because of this, the domain decomposition algorithm cannot easily determine the minimum cell size that it requires for treating bonded interactions. Instead, domain decomposition will assume that half the non-bonded cut-off will be a suitable lower bound.\n");
+ comm->cutoff_mbody = comm->cutoff/2;
+ r_bonded_limit = comm->cutoff_mbody;
+ }
+ else
+ {
+ if (MASTER(cr))
+ {
+ dd_bonded_cg_distance(fplog, mtop, ir, x, box,
+ Flags & MD_DDBONDCHECK, &r_2b, &r_mb);
+ }
+ gmx_bcast(sizeof(r_2b), &r_2b, cr);
+ gmx_bcast(sizeof(r_mb), &r_mb, cr);
+
+ /* We use an initial margin of 10% for the minimum cell size,
+ * except when we are just below the non-bonded cut-off.
+ */
+ if (Flags & MD_DDBONDCOMM)
+ {
+ if (max(r_2b, r_mb) > comm->cutoff)
+ {
+ r_bonded = max(r_2b, r_mb);
+ r_bonded_limit = 1.1*r_bonded;
+ comm->bBondComm = TRUE;
+ }
+ else
+ {
+ r_bonded = r_mb;
+ r_bonded_limit = min(1.1*r_bonded, comm->cutoff);
+ }
+ /* We determine cutoff_mbody later */
+ }
+ else
+ {
+ /* No special bonded communication,
+ * simply increase the DD cut-off.
+ */
+ r_bonded_limit = 1.1*max(r_2b, r_mb);
+ comm->cutoff_mbody = r_bonded_limit;
+ comm->cutoff = max(comm->cutoff, comm->cutoff_mbody);
+ }
+ }
+ comm->cellsize_limit = max(comm->cellsize_limit, r_bonded_limit);
+ if (fplog)
+ {
+ fprintf(fplog,
+ "Minimum cell size due to bonded interactions: %.3f nm\n",
+ comm->cellsize_limit);
+ }
+ }
+
+ if (dd->bInterCGcons && rconstr <= 0)
+ {
+ /* There is a cell size limit due to the constraints (P-LINCS) */
+ rconstr = constr_r_max(fplog, mtop, ir);
+ if (fplog)
+ {
+ fprintf(fplog,
+ "Estimated maximum distance required for P-LINCS: %.3f nm\n",
+ rconstr);
+ if (rconstr > comm->cellsize_limit)
+ {
+ fprintf(fplog, "This distance will limit the DD cell size, you can override this with -rcon\n");
+ }
+ }
+ }
+ else if (rconstr > 0 && fplog)
+ {
+ /* Here we do not check for dd->bInterCGcons,
+ * because one can also set a cell size limit for virtual sites only
+ * and at this point we don't know yet if there are intercg v-sites.
+ */
+ fprintf(fplog,
+ "User supplied maximum distance required for P-LINCS: %.3f nm\n",
+ rconstr);
+ }
+ comm->cellsize_limit = max(comm->cellsize_limit, rconstr);
+
+ comm->cgs_gl = gmx_mtop_global_cgs(mtop);
+
+ if (nc[XX] > 0)
+ {
+ copy_ivec(nc, dd->nc);
+ set_dd_dim(fplog, dd);
+ set_ddbox_cr(cr, &dd->nc, ir, box, &comm->cgs_gl, x, ddbox);
+
+ if (cr->npmenodes == -1)
+ {
+ cr->npmenodes = 0;
+ }
+ acs = average_cellsize_min(dd, ddbox);
+ if (acs < comm->cellsize_limit)
+ {
+ if (fplog)
+ {
+ fprintf(fplog, "ERROR: The initial cell size (%f) is smaller than the cell size limit (%f)\n", acs, comm->cellsize_limit);
+ }
+ gmx_fatal_collective(FARGS, cr, NULL,
+ "The initial cell size (%f) is smaller than the cell size limit (%f), change options -dd, -rdd or -rcon, see the log file for details",
+ acs, comm->cellsize_limit);
+ }
+ }
+ else
+ {
+ set_ddbox_cr(cr, NULL, ir, box, &comm->cgs_gl, x, ddbox);
+
+ /* We need to choose the optimal DD grid and possibly PME nodes */
+ limit = dd_choose_grid(fplog, cr, dd, ir, mtop, box, ddbox,
+ comm->eDLB != edlbNO, dlb_scale,
+ comm->cellsize_limit, comm->cutoff,
+ comm->bInterCGBondeds);
+
+ if (dd->nc[XX] == 0)
+ {
+ bC = (dd->bInterCGcons && rconstr > r_bonded_limit);
+ sprintf(buf, "Change the number of nodes or mdrun option %s%s%s",
+ !bC ? "-rdd" : "-rcon",
+ comm->eDLB != edlbNO ? " or -dds" : "",
+ bC ? " or your LINCS settings" : "");
+
+ gmx_fatal_collective(FARGS, cr, NULL,
+ "There is no domain decomposition for %d nodes that is compatible with the given box and a minimum cell size of %g nm\n"
+ "%s\n"
+ "Look in the log file for details on the domain decomposition",
+ cr->nnodes-cr->npmenodes, limit, buf);
+ }
+ set_dd_dim(fplog, dd);
+ }
+
+ if (fplog)
+ {
+ fprintf(fplog,
+ "Domain decomposition grid %d x %d x %d, separate PME nodes %d\n",
+ dd->nc[XX], dd->nc[YY], dd->nc[ZZ], cr->npmenodes);
+ }
+
+ dd->nnodes = dd->nc[XX]*dd->nc[YY]*dd->nc[ZZ];
+ if (cr->nnodes - dd->nnodes != cr->npmenodes)
+ {
+ gmx_fatal_collective(FARGS, cr, NULL,
+ "The size of the domain decomposition grid (%d) does not match the number of nodes (%d). The total number of nodes is %d",
+ dd->nnodes, cr->nnodes - cr->npmenodes, cr->nnodes);
+ }
+ if (cr->npmenodes > dd->nnodes)
+ {
+ gmx_fatal_collective(FARGS, cr, NULL,
+ "The number of separate PME nodes (%d) is larger than the number of PP nodes (%d), this is not supported.", cr->npmenodes, dd->nnodes);
+ }
+ if (cr->npmenodes > 0)
+ {
+ comm->npmenodes = cr->npmenodes;
+ }
+ else
+ {
+ comm->npmenodes = dd->nnodes;
+ }
+
+ if (EEL_PME(ir->coulombtype))
+ {
+ /* The following choices should match those
+ * in comm_cost_est in domdec_setup.c.
+ * Note that here the checks have to take into account
+ * that the decomposition might occur in a different order than xyz
+ * (for instance through the env.var. GMX_DD_ORDER_ZYX),
+ * in which case they will not match those in comm_cost_est,
+ * but since that is mainly for testing purposes that's fine.
+ */
+ if (dd->ndim >= 2 && dd->dim[0] == XX && dd->dim[1] == YY &&
+ comm->npmenodes > dd->nc[XX] && comm->npmenodes % dd->nc[XX] == 0 &&
+ getenv("GMX_PMEONEDD") == NULL)
+ {
+ comm->npmedecompdim = 2;
+ comm->npmenodes_x = dd->nc[XX];
+ comm->npmenodes_y = comm->npmenodes/comm->npmenodes_x;
+ }
+ else
+ {
+ /* In case nc is 1 in both x and y we could still choose to
+ * decompose pme in y instead of x, but we use x for simplicity.
+ */
+ comm->npmedecompdim = 1;
+ if (dd->dim[0] == YY)
+ {
+ comm->npmenodes_x = 1;
+ comm->npmenodes_y = comm->npmenodes;
+ }
+ else
+ {
+ comm->npmenodes_x = comm->npmenodes;
+ comm->npmenodes_y = 1;
+ }
+ }
+ if (fplog)
+ {
+ fprintf(fplog, "PME domain decomposition: %d x %d x %d\n",
+ comm->npmenodes_x, comm->npmenodes_y, 1);
+ }
+ }
+ else
+ {
+ comm->npmedecompdim = 0;
+ comm->npmenodes_x = 0;
+ comm->npmenodes_y = 0;
+ }
+
+ /* Technically we don't need both of these,
+ * but it simplifies code not having to recalculate it.
+ */
+ *npme_x = comm->npmenodes_x;
+ *npme_y = comm->npmenodes_y;
+
+ snew(comm->slb_frac, DIM);
+ if (comm->eDLB == edlbNO)
+ {
+ comm->slb_frac[XX] = get_slb_frac(fplog, "x", dd->nc[XX], sizex);
+ comm->slb_frac[YY] = get_slb_frac(fplog, "y", dd->nc[YY], sizey);
+ comm->slb_frac[ZZ] = get_slb_frac(fplog, "z", dd->nc[ZZ], sizez);
+ }
+
+ if (comm->bInterCGBondeds && comm->cutoff_mbody == 0)
+ {
+ if (comm->bBondComm || comm->eDLB != edlbNO)
+ {
+ /* Set the bonded communication distance to halfway
+ * the minimum and the maximum,
+ * since the extra communication cost is nearly zero.
+ */
+ acs = average_cellsize_min(dd, ddbox);
+ comm->cutoff_mbody = 0.5*(r_bonded + acs);
+ if (comm->eDLB != edlbNO)
+ {
+ /* Check if this does not limit the scaling */
+ comm->cutoff_mbody = min(comm->cutoff_mbody, dlb_scale*acs);
+ }
+ if (!comm->bBondComm)
+ {
+ /* Without bBondComm do not go beyond the n.b. cut-off */
+ comm->cutoff_mbody = min(comm->cutoff_mbody, comm->cutoff);
+ if (comm->cellsize_limit >= comm->cutoff)
+ {
+ /* We don't loose a lot of efficieny
+ * when increasing it to the n.b. cut-off.
+ * It can even be slightly faster, because we need
+ * less checks for the communication setup.
+ */
+ comm->cutoff_mbody = comm->cutoff;
+ }
+ }
+ /* Check if we did not end up below our original limit */
+ comm->cutoff_mbody = max(comm->cutoff_mbody, r_bonded_limit);
+
+ if (comm->cutoff_mbody > comm->cellsize_limit)
+ {
+ comm->cellsize_limit = comm->cutoff_mbody;
+ }
+ }
+ /* Without DLB and cutoff_mbody<cutoff, cutoff_mbody is dynamic */
+ }
+
+ if (debug)
+ {
+ fprintf(debug, "Bonded atom communication beyond the cut-off: %d\n"
+ "cellsize limit %f\n",
+ comm->bBondComm, comm->cellsize_limit);
+ }
+
+ if (MASTER(cr))
+ {
+ check_dd_restrictions(cr, dd, ir, fplog);
+ }
+
+ comm->partition_step = INT_MIN;
+ dd->ddp_count = 0;
+
+ clear_dd_cycle_counts(dd);
+
+ return dd;
+}
+
+static void set_dlb_limits(gmx_domdec_t *dd)
+
+{
+ int d;
+
+ for (d = 0; d < dd->ndim; d++)
+ {
+ dd->comm->cd[d].np = dd->comm->cd[d].np_dlb;
+ dd->comm->cellsize_min[dd->dim[d]] =
+ dd->comm->cellsize_min_dlb[dd->dim[d]];
+ }
+}
+
+
+static void turn_on_dlb(FILE *fplog, t_commrec *cr, gmx_large_int_t step)
+{
+ gmx_domdec_t *dd;
+ gmx_domdec_comm_t *comm;
+ real cellsize_min;
+ int d, nc, i;
+ char buf[STRLEN];
+
+ dd = cr->dd;
+ comm = dd->comm;
+
+ if (fplog)
+ {
+ fprintf(fplog, "At step %s the performance loss due to force load imbalance is %.1f %%\n", gmx_step_str(step, buf), dd_force_imb_perf_loss(dd)*100);
+ }
+
+ cellsize_min = comm->cellsize_min[dd->dim[0]];
+ for (d = 1; d < dd->ndim; d++)
+ {
+ cellsize_min = min(cellsize_min, comm->cellsize_min[dd->dim[d]]);
+ }
+
+ if (cellsize_min < comm->cellsize_limit*1.05)
+ {
+ dd_warning(cr, fplog, "NOTE: the minimum cell size is smaller than 1.05 times the cell size limit, will not turn on dynamic load balancing\n");
+
+ /* Change DLB from "auto" to "no". */
+ comm->eDLB = edlbNO;
+
+ return;
+ }
+
+ dd_warning(cr, fplog, "NOTE: Turning on dynamic load balancing\n");
+ comm->bDynLoadBal = TRUE;
+ dd->bGridJump = TRUE;
+
+ set_dlb_limits(dd);
+
+ /* We can set the required cell size info here,
+ * so we do not need to communicate this.
+ * The grid is completely uniform.
+ */
+ for (d = 0; d < dd->ndim; d++)
+ {
+ if (comm->root[d])
+ {
+ comm->load[d].sum_m = comm->load[d].sum;
+
+ nc = dd->nc[dd->dim[d]];
+ for (i = 0; i < nc; i++)
+ {
+ comm->root[d]->cell_f[i] = i/(real)nc;
+ if (d > 0)
+ {
+ comm->root[d]->cell_f_max0[i] = i /(real)nc;
+ comm->root[d]->cell_f_min1[i] = (i+1)/(real)nc;
+ }
+ }
+ comm->root[d]->cell_f[nc] = 1.0;
+ }
+ }
+}
+
+static char *init_bLocalCG(gmx_mtop_t *mtop)
+{
+ int ncg, cg;
+ char *bLocalCG;
+
+ ncg = ncg_mtop(mtop);
+ snew(bLocalCG, ncg);
+ for (cg = 0; cg < ncg; cg++)
+ {
+ bLocalCG[cg] = FALSE;
+ }
+
+ return bLocalCG;
+}
+
+void dd_init_bondeds(FILE *fplog,
+ gmx_domdec_t *dd, gmx_mtop_t *mtop,
+ gmx_vsite_t *vsite,
+ t_inputrec *ir, gmx_bool bBCheck, cginfo_mb_t *cginfo_mb)
+{
+ gmx_domdec_comm_t *comm;
+ gmx_bool bBondComm;
+ int d;
+
+ dd_make_reverse_top(fplog, dd, mtop, vsite, ir, bBCheck);
+
+ comm = dd->comm;
+
+ if (comm->bBondComm)
+ {
+ /* Communicate atoms beyond the cut-off for bonded interactions */
+ comm = dd->comm;
+
+ comm->cglink = make_charge_group_links(mtop, dd, cginfo_mb);
+
+ comm->bLocalCG = init_bLocalCG(mtop);
+ }
+ else
+ {
+ /* Only communicate atoms based on cut-off */
+ comm->cglink = NULL;
+ comm->bLocalCG = NULL;
+ }
+}
+
+static void print_dd_settings(FILE *fplog, gmx_domdec_t *dd,
+ t_inputrec *ir,
+ gmx_bool bDynLoadBal, real dlb_scale,
+ gmx_ddbox_t *ddbox)
+{
+ gmx_domdec_comm_t *comm;
+ int d;
+ ivec np;
+ real limit, shrink;
+ char buf[64];
+
+ if (fplog == NULL)
+ {
+ return;
+ }
+
+ comm = dd->comm;
+
+ if (bDynLoadBal)
+ {
+ fprintf(fplog, "The maximum number of communication pulses is:");
+ for (d = 0; d < dd->ndim; d++)
+ {
+ fprintf(fplog, " %c %d", dim2char(dd->dim[d]), comm->cd[d].np_dlb);
+ }
+ fprintf(fplog, "\n");
+ fprintf(fplog, "The minimum size for domain decomposition cells is %.3f nm\n", comm->cellsize_limit);
+ fprintf(fplog, "The requested allowed shrink of DD cells (option -dds) is: %.2f\n", dlb_scale);
+ fprintf(fplog, "The allowed shrink of domain decomposition cells is:");
+ for (d = 0; d < DIM; d++)
+ {
+ if (dd->nc[d] > 1)
+ {
+ if (d >= ddbox->npbcdim && dd->nc[d] == 2)
+ {
+ shrink = 0;
+ }
+ else
+ {
+ shrink =
+ comm->cellsize_min_dlb[d]/
+ (ddbox->box_size[d]*ddbox->skew_fac[d]/dd->nc[d]);
+ }
+ fprintf(fplog, " %c %.2f", dim2char(d), shrink);
+ }
+ }
+ fprintf(fplog, "\n");
+ }
+ else
+ {
+ set_dd_cell_sizes_slb(dd, ddbox, FALSE, np);
+ fprintf(fplog, "The initial number of communication pulses is:");
+ for (d = 0; d < dd->ndim; d++)
+ {
+ fprintf(fplog, " %c %d", dim2char(dd->dim[d]), np[dd->dim[d]]);
+ }
+ fprintf(fplog, "\n");
+ fprintf(fplog, "The initial domain decomposition cell size is:");
+ for (d = 0; d < DIM; d++)
+ {
+ if (dd->nc[d] > 1)
+ {
+ fprintf(fplog, " %c %.2f nm",
+ dim2char(d), dd->comm->cellsize_min[d]);
+ }
+ }
+ fprintf(fplog, "\n\n");
+ }
+
+ if (comm->bInterCGBondeds || dd->vsite_comm || dd->constraint_comm)
+ {
+ fprintf(fplog, "The maximum allowed distance for charge groups involved in interactions is:\n");
+ fprintf(fplog, "%40s %-7s %6.3f nm\n",
+ "non-bonded interactions", "", comm->cutoff);
+
+ if (bDynLoadBal)
+ {
+ limit = dd->comm->cellsize_limit;
+ }
+ else
+ {
+ if (dynamic_dd_box(ddbox, ir))
+ {
+ fprintf(fplog, "(the following are initial values, they could change due to box deformation)\n");
+ }
+ limit = dd->comm->cellsize_min[XX];
+ for (d = 1; d < DIM; d++)
+ {
+ limit = min(limit, dd->comm->cellsize_min[d]);
+ }
+ }
+
+ if (comm->bInterCGBondeds)
+ {
+ fprintf(fplog, "%40s %-7s %6.3f nm\n",
+ "two-body bonded interactions", "(-rdd)",
+ max(comm->cutoff, comm->cutoff_mbody));
+ fprintf(fplog, "%40s %-7s %6.3f nm\n",
+ "multi-body bonded interactions", "(-rdd)",
+ (comm->bBondComm || dd->bGridJump) ? comm->cutoff_mbody : min(comm->cutoff, limit));
+ }
+ if (dd->vsite_comm)
+ {
+ fprintf(fplog, "%40s %-7s %6.3f nm\n",
+ "virtual site constructions", "(-rcon)", limit);
+ }
+ if (dd->constraint_comm)
+ {
+ sprintf(buf, "atoms separated by up to %d constraints",
+ 1+ir->nProjOrder);
+ fprintf(fplog, "%40s %-7s %6.3f nm\n",
+ buf, "(-rcon)", limit);
+ }
+ fprintf(fplog, "\n");
+ }
+
+ fflush(fplog);
+}
+
+static void set_cell_limits_dlb(gmx_domdec_t *dd,
+ real dlb_scale,
+ const t_inputrec *ir,
+ const gmx_ddbox_t *ddbox)
+{
+ gmx_domdec_comm_t *comm;
+ int d, dim, npulse, npulse_d_max, npulse_d;
+ gmx_bool bNoCutOff;
+
+ comm = dd->comm;
+
+ bNoCutOff = (ir->rvdw == 0 || ir->rcoulomb == 0);
+
+ /* Determine the maximum number of comm. pulses in one dimension */
+
+ comm->cellsize_limit = max(comm->cellsize_limit, comm->cutoff_mbody);
+
+ /* Determine the maximum required number of grid pulses */
+ if (comm->cellsize_limit >= comm->cutoff)
+ {
+ /* Only a single pulse is required */
+ npulse = 1;
+ }
+ else if (!bNoCutOff && comm->cellsize_limit > 0)
+ {
+ /* We round down slightly here to avoid overhead due to the latency
+ * of extra communication calls when the cut-off
+ * would be only slightly longer than the cell size.
+ * Later cellsize_limit is redetermined,
+ * so we can not miss interactions due to this rounding.
+ */
+ npulse = (int)(0.96 + comm->cutoff/comm->cellsize_limit);
+ }
+ else
+ {
+ /* There is no cell size limit */
+ npulse = max(dd->nc[XX]-1, max(dd->nc[YY]-1, dd->nc[ZZ]-1));
+ }
+
+ if (!bNoCutOff && npulse > 1)
+ {
+ /* See if we can do with less pulses, based on dlb_scale */
+ npulse_d_max = 0;
+ for (d = 0; d < dd->ndim; d++)
+ {
+ dim = dd->dim[d];
+ npulse_d = (int)(1 + dd->nc[dim]*comm->cutoff
+ /(ddbox->box_size[dim]*ddbox->skew_fac[dim]*dlb_scale));
+ npulse_d_max = max(npulse_d_max, npulse_d);
+ }
+ npulse = min(npulse, npulse_d_max);
+ }
+
+ /* This env var can override npulse */
+ d = dd_nst_env(debug, "GMX_DD_NPULSE", 0);
+ if (d > 0)
+ {
+ npulse = d;
+ }
+
+ comm->maxpulse = 1;
+ comm->bVacDLBNoLimit = (ir->ePBC == epbcNONE);
+ for (d = 0; d < dd->ndim; d++)
+ {
+ comm->cd[d].np_dlb = min(npulse, dd->nc[dd->dim[d]]-1);
+ comm->cd[d].np_nalloc = comm->cd[d].np_dlb;
+ snew(comm->cd[d].ind, comm->cd[d].np_nalloc);
+ comm->maxpulse = max(comm->maxpulse, comm->cd[d].np_dlb);
+ if (comm->cd[d].np_dlb < dd->nc[dd->dim[d]]-1)
+ {
+ comm->bVacDLBNoLimit = FALSE;
+ }
+ }
+
+ /* cellsize_limit is set for LINCS in init_domain_decomposition */
+ if (!comm->bVacDLBNoLimit)
+ {
+ comm->cellsize_limit = max(comm->cellsize_limit,
+ comm->cutoff/comm->maxpulse);
+ }
+ comm->cellsize_limit = max(comm->cellsize_limit, comm->cutoff_mbody);
+ /* Set the minimum cell size for each DD dimension */
+ for (d = 0; d < dd->ndim; d++)
+ {
+ if (comm->bVacDLBNoLimit ||
+ comm->cd[d].np_dlb*comm->cellsize_limit >= comm->cutoff)
+ {
+ comm->cellsize_min_dlb[dd->dim[d]] = comm->cellsize_limit;
+ }
+ else
+ {
+ comm->cellsize_min_dlb[dd->dim[d]] =
+ comm->cutoff/comm->cd[d].np_dlb;
+ }
+ }
+ if (comm->cutoff_mbody <= 0)
+ {
+ comm->cutoff_mbody = min(comm->cutoff, comm->cellsize_limit);
+ }
+ if (comm->bDynLoadBal)
+ {
+ set_dlb_limits(dd);
+ }
+}
+
+gmx_bool dd_bonded_molpbc(gmx_domdec_t *dd, int ePBC)
+{
+ /* If each molecule is a single charge group
+ * or we use domain decomposition for each periodic dimension,
+ * we do not need to take pbc into account for the bonded interactions.
+ */
+ return (ePBC != epbcNONE && dd->comm->bInterCGBondeds &&
+ !(dd->nc[XX] > 1 &&
+ dd->nc[YY] > 1 &&
+ (dd->nc[ZZ] > 1 || ePBC == epbcXY)));
+}
+
+void set_dd_parameters(FILE *fplog, gmx_domdec_t *dd, real dlb_scale,
+ t_inputrec *ir, gmx_ddbox_t *ddbox)
+{
+ gmx_domdec_comm_t *comm;
+ int natoms_tot;
+ real vol_frac;
+
+ comm = dd->comm;
+
+ /* Initialize the thread data.
+ * This can not be done in init_domain_decomposition,
+ * as the numbers of threads is determined later.
+ */
+ comm->nth = gmx_omp_nthreads_get(emntDomdec);
+ if (comm->nth > 1)
+ {
+ snew(comm->dth, comm->nth);
+ }
+
+ if (EEL_PME(ir->coulombtype))
+ {
+ init_ddpme(dd, &comm->ddpme[0], 0);
+ if (comm->npmedecompdim >= 2)
+ {
+ init_ddpme(dd, &comm->ddpme[1], 1);
+ }
+ }
+ else
+ {
+ comm->npmenodes = 0;
+ if (dd->pme_nodeid >= 0)
+ {
+ gmx_fatal_collective(FARGS, NULL, dd,
+ "Can not have separate PME nodes without PME electrostatics");
+ }
+ }
+
+ if (debug)
+ {
+ fprintf(debug, "The DD cut-off is %f\n", comm->cutoff);
+ }
+ if (comm->eDLB != edlbNO)
+ {
+ set_cell_limits_dlb(dd, dlb_scale, ir, ddbox);
+ }
+
+ print_dd_settings(fplog, dd, ir, comm->bDynLoadBal, dlb_scale, ddbox);
+ if (comm->eDLB == edlbAUTO)
+ {
+ if (fplog)
+ {
+ fprintf(fplog, "When dynamic load balancing gets turned on, these settings will change to:\n");
+ }
+ print_dd_settings(fplog, dd, ir, TRUE, dlb_scale, ddbox);
+ }
+
+ if (ir->ePBC == epbcNONE)
+ {
+ vol_frac = 1 - 1/(double)dd->nnodes;
+ }
+ else
+ {
+ vol_frac =
+ (1 + comm_box_frac(dd->nc, comm->cutoff, ddbox))/(double)dd->nnodes;
+ }
+ if (debug)
+ {
+ fprintf(debug, "Volume fraction for all DD zones: %f\n", vol_frac);
+ }
+ natoms_tot = comm->cgs_gl.index[comm->cgs_gl.nr];
+
+ dd->ga2la = ga2la_init(natoms_tot, vol_frac*natoms_tot);
+}
+
+static gmx_bool test_dd_cutoff(t_commrec *cr,
+ t_state *state, t_inputrec *ir,
+ real cutoff_req)
+{
+ gmx_domdec_t *dd;
+ gmx_ddbox_t ddbox;
+ int d, dim, np;
+ real inv_cell_size;
+ int LocallyLimited;
+
+ dd = cr->dd;
+
+ set_ddbox(dd, FALSE, cr, ir, state->box,
+ TRUE, &dd->comm->cgs_gl, state->x, &ddbox);
+
+ LocallyLimited = 0;
+
+ for (d = 0; d < dd->ndim; d++)
+ {
+ dim = dd->dim[d];
+
+ inv_cell_size = DD_CELL_MARGIN*dd->nc[dim]/ddbox.box_size[dim];
+ if (dynamic_dd_box(&ddbox, ir))
+ {
+ inv_cell_size *= DD_PRES_SCALE_MARGIN;
+ }
+
+ np = 1 + (int)(cutoff_req*inv_cell_size*ddbox.skew_fac[dim]);
+
+ if (dd->comm->eDLB != edlbNO && dim < ddbox.npbcdim &&
+ dd->comm->cd[d].np_dlb > 0)
+ {
+ if (np > dd->comm->cd[d].np_dlb)
+ {
+ return FALSE;
+ }
+
+ /* If a current local cell size is smaller than the requested
+ * cut-off, we could still fix it, but this gets very complicated.
+ * Without fixing here, we might actually need more checks.
+ */
+ if ((dd->comm->cell_x1[dim] - dd->comm->cell_x0[dim])*ddbox.skew_fac[dim]*dd->comm->cd[d].np_dlb < cutoff_req)
+ {
+ LocallyLimited = 1;
+ }
+ }
+ }
+
+ if (dd->comm->eDLB != edlbNO)
+ {
+ /* If DLB is not active yet, we don't need to check the grid jumps.
+ * Actually we shouldn't, because then the grid jump data is not set.
+ */
+ if (dd->comm->bDynLoadBal &&
+ check_grid_jump(0, dd, cutoff_req, &ddbox, FALSE))
+ {
+ LocallyLimited = 1;
+ }
+
+ gmx_sumi(1, &LocallyLimited, cr);
+
+ if (LocallyLimited > 0)
+ {
+ return FALSE;
+ }
+ }
+
+ return TRUE;
+}
+
+gmx_bool change_dd_cutoff(t_commrec *cr, t_state *state, t_inputrec *ir,
+ real cutoff_req)
+{
+ gmx_bool bCutoffAllowed;
+
+ bCutoffAllowed = test_dd_cutoff(cr, state, ir, cutoff_req);
+
+ if (bCutoffAllowed)
+ {
+ cr->dd->comm->cutoff = cutoff_req;
+ }
+
+ return bCutoffAllowed;
+}
+
+void change_dd_dlb_cutoff_limit(t_commrec *cr)
+{
+ gmx_domdec_comm_t *comm;
+
+ comm = cr->dd->comm;
+
+ /* Turn on the DLB limiting (might have been on already) */
+ comm->bPMELoadBalDLBLimits = TRUE;
+
+ /* Change the cut-off limit */
+ comm->PMELoadBal_max_cutoff = comm->cutoff;
+}
+
+static void merge_cg_buffers(int ncell,
+ gmx_domdec_comm_dim_t *cd, int pulse,
+ int *ncg_cell,
+ int *index_gl, int *recv_i,
+ rvec *cg_cm, rvec *recv_vr,
+ int *cgindex,
+ cginfo_mb_t *cginfo_mb, int *cginfo)
+{
+ gmx_domdec_ind_t *ind, *ind_p;
+ int p, cell, c, cg, cg0, cg1, cg_gl, nat;
+ int shift, shift_at;
+
+ ind = &cd->ind[pulse];
+
+ /* First correct the already stored data */
+ shift = ind->nrecv[ncell];
+ for (cell = ncell-1; cell >= 0; cell--)
+ {
+ shift -= ind->nrecv[cell];
+ if (shift > 0)
+ {
+ /* Move the cg's present from previous grid pulses */
+ cg0 = ncg_cell[ncell+cell];
+ cg1 = ncg_cell[ncell+cell+1];
+ cgindex[cg1+shift] = cgindex[cg1];
+ for (cg = cg1-1; cg >= cg0; cg--)
+ {
+ index_gl[cg+shift] = index_gl[cg];
+ copy_rvec(cg_cm[cg], cg_cm[cg+shift]);
+ cgindex[cg+shift] = cgindex[cg];
+ cginfo[cg+shift] = cginfo[cg];
+ }
+ /* Correct the already stored send indices for the shift */
+ for (p = 1; p <= pulse; p++)
+ {
+ ind_p = &cd->ind[p];
+ cg0 = 0;
+ for (c = 0; c < cell; c++)
+ {
+ cg0 += ind_p->nsend[c];
+ }
+ cg1 = cg0 + ind_p->nsend[cell];
+ for (cg = cg0; cg < cg1; cg++)
+ {
+ ind_p->index[cg] += shift;
+ }
+ }
+ }
+ }
+
+ /* Merge in the communicated buffers */
+ shift = 0;
+ shift_at = 0;
+ cg0 = 0;
+ for (cell = 0; cell < ncell; cell++)
+ {
+ cg1 = ncg_cell[ncell+cell+1] + shift;
+ if (shift_at > 0)
+ {
+ /* Correct the old cg indices */
+ for (cg = ncg_cell[ncell+cell]; cg < cg1; cg++)
+ {
+ cgindex[cg+1] += shift_at;
+ }
+ }
+ for (cg = 0; cg < ind->nrecv[cell]; cg++)
+ {
+ /* Copy this charge group from the buffer */
+ index_gl[cg1] = recv_i[cg0];
+ copy_rvec(recv_vr[cg0], cg_cm[cg1]);
+ /* Add it to the cgindex */
+ cg_gl = index_gl[cg1];
+ cginfo[cg1] = ddcginfo(cginfo_mb, cg_gl);
+ nat = GET_CGINFO_NATOMS(cginfo[cg1]);
+ cgindex[cg1+1] = cgindex[cg1] + nat;
+ cg0++;
+ cg1++;
+ shift_at += nat;
+ }
+ shift += ind->nrecv[cell];
+ ncg_cell[ncell+cell+1] = cg1;
+ }
+}
+
+static void make_cell2at_index(gmx_domdec_comm_dim_t *cd,
+ int nzone, int cg0, const int *cgindex)
+{
+ int cg, zone, p;
+
+ /* Store the atom block boundaries for easy copying of communication buffers
+ */
+ cg = cg0;
+ for (zone = 0; zone < nzone; zone++)
+ {
+ for (p = 0; p < cd->np; p++)
+ {
+ cd->ind[p].cell2at0[zone] = cgindex[cg];
+ cg += cd->ind[p].nrecv[zone];
+ cd->ind[p].cell2at1[zone] = cgindex[cg];
+ }
+ }
+}
+
+static gmx_bool missing_link(t_blocka *link, int cg_gl, char *bLocalCG)
+{
+ int i;
+ gmx_bool bMiss;
+
+ bMiss = FALSE;
+ for (i = link->index[cg_gl]; i < link->index[cg_gl+1]; i++)
+ {
+ if (!bLocalCG[link->a[i]])
+ {
+ bMiss = TRUE;
+ }
+ }
+
+ return bMiss;
+}
+
+/* Domain corners for communication, a maximum of 4 i-zones see a j domain */
+typedef struct {
+ real c[DIM][4]; /* the corners for the non-bonded communication */
+ real cr0; /* corner for rounding */
+ real cr1[4]; /* corners for rounding */
+ real bc[DIM]; /* corners for bounded communication */
+ real bcr1; /* corner for rounding for bonded communication */
+} dd_corners_t;
+
+/* Determine the corners of the domain(s) we are communicating with */
+static void
+set_dd_corners(const gmx_domdec_t *dd,
+ int dim0, int dim1, int dim2,
+ gmx_bool bDistMB,
+ dd_corners_t *c)
+{
+ const gmx_domdec_comm_t *comm;
+ const gmx_domdec_zones_t *zones;
+ int i, j;
+
+ comm = dd->comm;
+
+ zones = &comm->zones;
+
+ /* Keep the compiler happy */
+ c->cr0 = 0;
+ c->bcr1 = 0;
+
+ /* The first dimension is equal for all cells */
+ c->c[0][0] = comm->cell_x0[dim0];
+ if (bDistMB)
+ {
+ c->bc[0] = c->c[0][0];
+ }
+ if (dd->ndim >= 2)
+ {
+ dim1 = dd->dim[1];
+ /* This cell row is only seen from the first row */
+ c->c[1][0] = comm->cell_x0[dim1];
+ /* All rows can see this row */
+ c->c[1][1] = comm->cell_x0[dim1];
+ if (dd->bGridJump)
+ {
+ c->c[1][1] = max(comm->cell_x0[dim1], comm->zone_d1[1].mch0);
+ if (bDistMB)
+ {
+ /* For the multi-body distance we need the maximum */
+ c->bc[1] = max(comm->cell_x0[dim1], comm->zone_d1[1].p1_0);
+ }
+ }
+ /* Set the upper-right corner for rounding */
+ c->cr0 = comm->cell_x1[dim0];
+
+ if (dd->ndim >= 3)
+ {
+ dim2 = dd->dim[2];
+ for (j = 0; j < 4; j++)
+ {
+ c->c[2][j] = comm->cell_x0[dim2];
+ }
+ if (dd->bGridJump)
+ {
+ /* Use the maximum of the i-cells that see a j-cell */
+ for (i = 0; i < zones->nizone; i++)
+ {
+ for (j = zones->izone[i].j0; j < zones->izone[i].j1; j++)
+ {
+ if (j >= 4)
+ {
+ c->c[2][j-4] =
+ max(c->c[2][j-4],
+ comm->zone_d2[zones->shift[i][dim0]][zones->shift[i][dim1]].mch0);
+ }
+ }
+ }
+ if (bDistMB)
+ {
+ /* For the multi-body distance we need the maximum */
+ c->bc[2] = comm->cell_x0[dim2];
+ for (i = 0; i < 2; i++)
+ {
+ for (j = 0; j < 2; j++)
+ {
+ c->bc[2] = max(c->bc[2], comm->zone_d2[i][j].p1_0);
+ }
+ }
+ }
+ }
+
+ /* Set the upper-right corner for rounding */
+ /* Cell (0,0,0) and cell (1,0,0) can see cell 4 (0,1,1)
+ * Only cell (0,0,0) can see cell 7 (1,1,1)
+ */
+ c->cr1[0] = comm->cell_x1[dim1];
+ c->cr1[3] = comm->cell_x1[dim1];
+ if (dd->bGridJump)
+ {
+ c->cr1[0] = max(comm->cell_x1[dim1], comm->zone_d1[1].mch1);
+ if (bDistMB)
+ {
+ /* For the multi-body distance we need the maximum */
+ c->bcr1 = max(comm->cell_x1[dim1], comm->zone_d1[1].p1_1);
+ }
+ }
+ }
+ }
+}
+
+/* Determine which cg's we need to send in this pulse from this zone */
+static void
+get_zone_pulse_cgs(gmx_domdec_t *dd,
+ int zonei, int zone,
+ int cg0, int cg1,
+ const int *index_gl,
+ const int *cgindex,
+ int dim, int dim_ind,
+ int dim0, int dim1, int dim2,
+ real r_comm2, real r_bcomm2,
+ matrix box,
+ ivec tric_dist,
+ rvec *normal,
+ real skew_fac2_d, real skew_fac_01,
+ rvec *v_d, rvec *v_0, rvec *v_1,
+ const dd_corners_t *c,
+ rvec sf2_round,
+ gmx_bool bDistBonded,
+ gmx_bool bBondComm,
+ gmx_bool bDist2B,
+ gmx_bool bDistMB,
+ rvec *cg_cm,
+ int *cginfo,
+ gmx_domdec_ind_t *ind,
+ int **ibuf, int *ibuf_nalloc,
+ vec_rvec_t *vbuf,
+ int *nsend_ptr,
+ int *nat_ptr,
+ int *nsend_z_ptr)
+{
+ gmx_domdec_comm_t *comm;
+ gmx_bool bScrew;
+ gmx_bool bDistMB_pulse;
+ int cg, i;
+ real r2, rb2, r, tric_sh;
+ rvec rn, rb;
+ int dimd;
+ int nsend_z, nsend, nat;
+
+ comm = dd->comm;
+
+ bScrew = (dd->bScrewPBC && dim == XX);
+
+ bDistMB_pulse = (bDistMB && bDistBonded);
+
+ nsend_z = 0;
+ nsend = *nsend_ptr;
+ nat = *nat_ptr;
+
+ for (cg = cg0; cg < cg1; cg++)
+ {
+ r2 = 0;
+ rb2 = 0;
+ if (tric_dist[dim_ind] == 0)
+ {
+ /* Rectangular direction, easy */
+ r = cg_cm[cg][dim] - c->c[dim_ind][zone];
+ if (r > 0)
+ {
+ r2 += r*r;
+ }
+ if (bDistMB_pulse)
+ {
+ r = cg_cm[cg][dim] - c->bc[dim_ind];
+ if (r > 0)
+ {
+ rb2 += r*r;
+ }
+ }
+ /* Rounding gives at most a 16% reduction
+ * in communicated atoms
+ */
+ if (dim_ind >= 1 && (zonei == 1 || zonei == 2))
+ {
+ r = cg_cm[cg][dim0] - c->cr0;
+ /* This is the first dimension, so always r >= 0 */
+ r2 += r*r;
+ if (bDistMB_pulse)
+ {
+ rb2 += r*r;
+ }
+ }
+ if (dim_ind == 2 && (zonei == 2 || zonei == 3))
+ {
+ r = cg_cm[cg][dim1] - c->cr1[zone];
+ if (r > 0)
+ {
+ r2 += r*r;
+ }
+ if (bDistMB_pulse)
+ {
+ r = cg_cm[cg][dim1] - c->bcr1;
+ if (r > 0)
+ {
+ rb2 += r*r;
+ }
+ }
+ }
+ }
+ else
+ {
+ /* Triclinic direction, more complicated */
+ clear_rvec(rn);
+ clear_rvec(rb);
+ /* Rounding, conservative as the skew_fac multiplication
+ * will slightly underestimate the distance.
+ */
+ if (dim_ind >= 1 && (zonei == 1 || zonei == 2))
+ {
+ rn[dim0] = cg_cm[cg][dim0] - c->cr0;
+ for (i = dim0+1; i < DIM; i++)
+ {
+ rn[dim0] -= cg_cm[cg][i]*v_0[i][dim0];
+ }
+ r2 = rn[dim0]*rn[dim0]*sf2_round[dim0];
+ if (bDistMB_pulse)
+ {
+ rb[dim0] = rn[dim0];
+ rb2 = r2;
+ }
+ /* Take care that the cell planes along dim0 might not
+ * be orthogonal to those along dim1 and dim2.
+ */
+ for (i = 1; i <= dim_ind; i++)
+ {
+ dimd = dd->dim[i];
+ if (normal[dim0][dimd] > 0)
+ {
+ rn[dimd] -= rn[dim0]*normal[dim0][dimd];
+ if (bDistMB_pulse)
+ {
+ rb[dimd] -= rb[dim0]*normal[dim0][dimd];
+ }
+ }
+ }
+ }
+ if (dim_ind == 2 && (zonei == 2 || zonei == 3))
+ {
+ rn[dim1] += cg_cm[cg][dim1] - c->cr1[zone];
+ tric_sh = 0;
+ for (i = dim1+1; i < DIM; i++)
+ {
+ tric_sh -= cg_cm[cg][i]*v_1[i][dim1];
+ }
+ rn[dim1] += tric_sh;
+ if (rn[dim1] > 0)
+ {
+ r2 += rn[dim1]*rn[dim1]*sf2_round[dim1];
+ /* Take care of coupling of the distances
+ * to the planes along dim0 and dim1 through dim2.
+ */
+ r2 -= rn[dim0]*rn[dim1]*skew_fac_01;
+ /* Take care that the cell planes along dim1
+ * might not be orthogonal to that along dim2.
+ */
+ if (normal[dim1][dim2] > 0)
+ {
+ rn[dim2] -= rn[dim1]*normal[dim1][dim2];
+ }
+ }
+ if (bDistMB_pulse)
+ {
+ rb[dim1] +=
+ cg_cm[cg][dim1] - c->bcr1 + tric_sh;
+ if (rb[dim1] > 0)
+ {
+ rb2 += rb[dim1]*rb[dim1]*sf2_round[dim1];
+ /* Take care of coupling of the distances
+ * to the planes along dim0 and dim1 through dim2.
+ */
+ rb2 -= rb[dim0]*rb[dim1]*skew_fac_01;
+ /* Take care that the cell planes along dim1
+ * might not be orthogonal to that along dim2.
+ */
+ if (normal[dim1][dim2] > 0)
+ {
+ rb[dim2] -= rb[dim1]*normal[dim1][dim2];
+ }
+ }
+ }
+ }
+ /* The distance along the communication direction */
+ rn[dim] += cg_cm[cg][dim] - c->c[dim_ind][zone];
+ tric_sh = 0;
+ for (i = dim+1; i < DIM; i++)
+ {
+ tric_sh -= cg_cm[cg][i]*v_d[i][dim];
+ }
+ rn[dim] += tric_sh;
+ if (rn[dim] > 0)
+ {
+ r2 += rn[dim]*rn[dim]*skew_fac2_d;
+ /* Take care of coupling of the distances
+ * to the planes along dim0 and dim1 through dim2.
+ */
+ if (dim_ind == 1 && zonei == 1)
+ {
+ r2 -= rn[dim0]*rn[dim]*skew_fac_01;
+ }
+ }
+ if (bDistMB_pulse)
+ {
+ clear_rvec(rb);
+ rb[dim] += cg_cm[cg][dim] - c->bc[dim_ind] + tric_sh;
+ if (rb[dim] > 0)
+ {
+ rb2 += rb[dim]*rb[dim]*skew_fac2_d;
+ /* Take care of coupling of the distances
+ * to the planes along dim0 and dim1 through dim2.
+ */
+ if (dim_ind == 1 && zonei == 1)
+ {
+ rb2 -= rb[dim0]*rb[dim]*skew_fac_01;
+ }
+ }
+ }
+ }
+
+ if (r2 < r_comm2 ||
+ (bDistBonded &&
+ ((bDistMB && rb2 < r_bcomm2) ||
+ (bDist2B && r2 < r_bcomm2)) &&
+ (!bBondComm ||
+ (GET_CGINFO_BOND_INTER(cginfo[cg]) &&
+ missing_link(comm->cglink, index_gl[cg],
+ comm->bLocalCG)))))
+ {
+ /* Make an index to the local charge groups */
+ if (nsend+1 > ind->nalloc)
+ {
+ ind->nalloc = over_alloc_large(nsend+1);
+ srenew(ind->index, ind->nalloc);
+ }
+ if (nsend+1 > *ibuf_nalloc)
+ {
+ *ibuf_nalloc = over_alloc_large(nsend+1);
+ srenew(*ibuf, *ibuf_nalloc);
+ }
+ ind->index[nsend] = cg;
+ (*ibuf)[nsend] = index_gl[cg];
+ nsend_z++;
+ vec_rvec_check_alloc(vbuf, nsend+1);
+
+ if (dd->ci[dim] == 0)
+ {
+ /* Correct cg_cm for pbc */
+ rvec_add(cg_cm[cg], box[dim], vbuf->v[nsend]);
+ if (bScrew)
+ {
+ vbuf->v[nsend][YY] = box[YY][YY] - vbuf->v[nsend][YY];
+ vbuf->v[nsend][ZZ] = box[ZZ][ZZ] - vbuf->v[nsend][ZZ];
+ }
+ }
+ else
+ {
+ copy_rvec(cg_cm[cg], vbuf->v[nsend]);
+ }
+ nsend++;
+ nat += cgindex[cg+1] - cgindex[cg];
+ }
+ }
+
+ *nsend_ptr = nsend;
+ *nat_ptr = nat;
+ *nsend_z_ptr = nsend_z;
+}
+
+static void setup_dd_communication(gmx_domdec_t *dd,
+ matrix box, gmx_ddbox_t *ddbox,
+ t_forcerec *fr, t_state *state, rvec **f)
+{
+ int dim_ind, dim, dim0, dim1, dim2, dimd, p, nat_tot;
+ int nzone, nzone_send, zone, zonei, cg0, cg1;
+ int c, i, j, cg, cg_gl, nrcg;
+ int *zone_cg_range, pos_cg, *index_gl, *cgindex, *recv_i;
+ gmx_domdec_comm_t *comm;
+ gmx_domdec_zones_t *zones;
+ gmx_domdec_comm_dim_t *cd;
+ gmx_domdec_ind_t *ind;
+ cginfo_mb_t *cginfo_mb;
+ gmx_bool bBondComm, bDist2B, bDistMB, bDistBonded;
+ real r_mb, r_comm2, r_scomm2, r_bcomm2, r_0, r_1, r2inc, inv_ncg;
+ dd_corners_t corners;
+ ivec tric_dist;
+ rvec *cg_cm, *normal, *v_d, *v_0 = NULL, *v_1 = NULL, *recv_vr;
+ real skew_fac2_d, skew_fac_01;
+ rvec sf2_round;
+ int nsend, nat;
+ int th;
+
+ if (debug)
+ {
+ fprintf(debug, "Setting up DD communication\n");
+ }
+
+ comm = dd->comm;
+
+ switch (fr->cutoff_scheme)
+ {
+ case ecutsGROUP:
+ cg_cm = fr->cg_cm;
+ break;
+ case ecutsVERLET:
+ cg_cm = state->x;
+ break;
+ default:
+ gmx_incons("unimplemented");
+ cg_cm = NULL;
+ }
+
+ for (dim_ind = 0; dim_ind < dd->ndim; dim_ind++)
+ {
+ dim = dd->dim[dim_ind];
+
+ /* Check if we need to use triclinic distances */
+ tric_dist[dim_ind] = 0;
+ for (i = 0; i <= dim_ind; i++)
+ {
+ if (ddbox->tric_dir[dd->dim[i]])
+ {
+ tric_dist[dim_ind] = 1;
+ }
+ }
+ }
+
+ bBondComm = comm->bBondComm;
+
+ /* Do we need to determine extra distances for multi-body bondeds? */
+ bDistMB = (comm->bInterCGMultiBody && dd->bGridJump && dd->ndim > 1);
+
+ /* Do we need to determine extra distances for only two-body bondeds? */
+ bDist2B = (bBondComm && !bDistMB);
+
+ r_comm2 = sqr(comm->cutoff);
+ r_bcomm2 = sqr(comm->cutoff_mbody);
+
+ if (debug)
+ {
+ fprintf(debug, "bBondComm %d, r_bc %f\n", bBondComm, sqrt(r_bcomm2));
+ }
+
+ zones = &comm->zones;
+
+ dim0 = dd->dim[0];
+ dim1 = (dd->ndim >= 2 ? dd->dim[1] : -1);
+ dim2 = (dd->ndim >= 3 ? dd->dim[2] : -1);
+
+ set_dd_corners(dd, dim0, dim1, dim2, bDistMB, &corners);
+
+ /* Triclinic stuff */
+ normal = ddbox->normal;
+ skew_fac_01 = 0;
+ if (dd->ndim >= 2)
+ {
+ v_0 = ddbox->v[dim0];
+ if (ddbox->tric_dir[dim0] && ddbox->tric_dir[dim1])
+ {
+ /* Determine the coupling coefficient for the distances
+ * to the cell planes along dim0 and dim1 through dim2.
+ * This is required for correct rounding.
+ */
+ skew_fac_01 =
+ ddbox->v[dim0][dim1+1][dim0]*ddbox->v[dim1][dim1+1][dim1];
+ if (debug)
+ {
+ fprintf(debug, "\nskew_fac_01 %f\n", skew_fac_01);
+ }
+ }
+ }
+ if (dd->ndim >= 3)
+ {
+ v_1 = ddbox->v[dim1];
+ }
+
+ zone_cg_range = zones->cg_range;
+ index_gl = dd->index_gl;
+ cgindex = dd->cgindex;
+ cginfo_mb = fr->cginfo_mb;
+
+ zone_cg_range[0] = 0;
+ zone_cg_range[1] = dd->ncg_home;
+ comm->zone_ncg1[0] = dd->ncg_home;
+ pos_cg = dd->ncg_home;
+
+ nat_tot = dd->nat_home;
+ nzone = 1;
+ for (dim_ind = 0; dim_ind < dd->ndim; dim_ind++)
+ {
+ dim = dd->dim[dim_ind];
+ cd = &comm->cd[dim_ind];
+
+ if (dim >= ddbox->npbcdim && dd->ci[dim] == 0)
+ {
+ /* No pbc in this dimension, the first node should not comm. */
+ nzone_send = 0;
+ }
+ else
+ {
+ nzone_send = nzone;
+ }
+
+ v_d = ddbox->v[dim];
+ skew_fac2_d = sqr(ddbox->skew_fac[dim]);
+
+ cd->bInPlace = TRUE;
+ for (p = 0; p < cd->np; p++)
+ {
+ /* Only atoms communicated in the first pulse are used
+ * for multi-body bonded interactions or for bBondComm.
+ */
+ bDistBonded = ((bDistMB || bDist2B) && p == 0);
+
+ ind = &cd->ind[p];
+ nsend = 0;
+ nat = 0;
+ for (zone = 0; zone < nzone_send; zone++)
+ {
+ if (tric_dist[dim_ind] && dim_ind > 0)
+ {
+ /* Determine slightly more optimized skew_fac's
+ * for rounding.
+ * This reduces the number of communicated atoms
+ * by about 10% for 3D DD of rhombic dodecahedra.
+ */
+ for (dimd = 0; dimd < dim; dimd++)
+ {
+ sf2_round[dimd] = 1;
+ if (ddbox->tric_dir[dimd])
+ {
+ for (i = dd->dim[dimd]+1; i < DIM; i++)
+ {
+ /* If we are shifted in dimension i
+ * and the cell plane is tilted forward
+ * in dimension i, skip this coupling.
+ */
+ if (!(zones->shift[nzone+zone][i] &&
+ ddbox->v[dimd][i][dimd] >= 0))
+ {
+ sf2_round[dimd] +=
+ sqr(ddbox->v[dimd][i][dimd]);
+ }
+ }
+ sf2_round[dimd] = 1/sf2_round[dimd];
+ }
+ }
+ }
+
+ zonei = zone_perm[dim_ind][zone];
+ if (p == 0)
+ {
+ /* Here we permutate the zones to obtain a convenient order
+ * for neighbor searching
+ */
+ cg0 = zone_cg_range[zonei];
+ cg1 = zone_cg_range[zonei+1];
+ }
+ else
+ {
+ /* Look only at the cg's received in the previous grid pulse
+ */
+ cg1 = zone_cg_range[nzone+zone+1];
+ cg0 = cg1 - cd->ind[p-1].nrecv[zone];
+ }
+
+#pragma omp parallel for num_threads(comm->nth) schedule(static)
+ for (th = 0; th < comm->nth; th++)
+ {
+ gmx_domdec_ind_t *ind_p;
+ int **ibuf_p, *ibuf_nalloc_p;
+ vec_rvec_t *vbuf_p;
+ int *nsend_p, *nat_p;
+ int *nsend_zone_p;
+ int cg0_th, cg1_th;
+
+ if (th == 0)
+ {
+ /* Thread 0 writes in the comm buffers */
+ ind_p = ind;
+ ibuf_p = &comm->buf_int;
+ ibuf_nalloc_p = &comm->nalloc_int;
+ vbuf_p = &comm->vbuf;
+ nsend_p = &nsend;
+ nat_p = &nat;
+ nsend_zone_p = &ind->nsend[zone];
+ }
+ else
+ {
+ /* Other threads write into temp buffers */
+ ind_p = &comm->dth[th].ind;
+ ibuf_p = &comm->dth[th].ibuf;
+ ibuf_nalloc_p = &comm->dth[th].ibuf_nalloc;
+ vbuf_p = &comm->dth[th].vbuf;
+ nsend_p = &comm->dth[th].nsend;
+ nat_p = &comm->dth[th].nat;
+ nsend_zone_p = &comm->dth[th].nsend_zone;
+
+ comm->dth[th].nsend = 0;
+ comm->dth[th].nat = 0;
+ comm->dth[th].nsend_zone = 0;
+ }
+
+ if (comm->nth == 1)
+ {
+ cg0_th = cg0;
+ cg1_th = cg1;
+ }
+ else
+ {
+ cg0_th = cg0 + ((cg1 - cg0)* th )/comm->nth;
+ cg1_th = cg0 + ((cg1 - cg0)*(th+1))/comm->nth;
+ }
+
+ /* Get the cg's for this pulse in this zone */
+ get_zone_pulse_cgs(dd, zonei, zone, cg0_th, cg1_th,
+ index_gl, cgindex,
+ dim, dim_ind, dim0, dim1, dim2,
+ r_comm2, r_bcomm2,
+ box, tric_dist,
+ normal, skew_fac2_d, skew_fac_01,
+ v_d, v_0, v_1, &corners, sf2_round,
+ bDistBonded, bBondComm,
+ bDist2B, bDistMB,
+ cg_cm, fr->cginfo,
+ ind_p,
+ ibuf_p, ibuf_nalloc_p,
+ vbuf_p,
+ nsend_p, nat_p,
+ nsend_zone_p);
+ }
+
+ /* Append data of threads>=1 to the communication buffers */
+ for (th = 1; th < comm->nth; th++)
+ {
+ dd_comm_setup_work_t *dth;
+ int i, ns1;
+
+ dth = &comm->dth[th];
+
+ ns1 = nsend + dth->nsend_zone;
+ if (ns1 > ind->nalloc)
+ {
+ ind->nalloc = over_alloc_dd(ns1);
+ srenew(ind->index, ind->nalloc);
+ }
+ if (ns1 > comm->nalloc_int)
+ {
+ comm->nalloc_int = over_alloc_dd(ns1);
+ srenew(comm->buf_int, comm->nalloc_int);
+ }
+ if (ns1 > comm->vbuf.nalloc)
+ {
+ comm->vbuf.nalloc = over_alloc_dd(ns1);
+ srenew(comm->vbuf.v, comm->vbuf.nalloc);
+ }
+
+ for (i = 0; i < dth->nsend_zone; i++)
+ {
+ ind->index[nsend] = dth->ind.index[i];
+ comm->buf_int[nsend] = dth->ibuf[i];
+ copy_rvec(dth->vbuf.v[i],
+ comm->vbuf.v[nsend]);
+ nsend++;
+ }
+ nat += dth->nat;
+ ind->nsend[zone] += dth->nsend_zone;
+ }
+ }
+ /* Clear the counts in case we do not have pbc */
+ for (zone = nzone_send; zone < nzone; zone++)
+ {
+ ind->nsend[zone] = 0;
+ }
+ ind->nsend[nzone] = nsend;
+ ind->nsend[nzone+1] = nat;
+ /* Communicate the number of cg's and atoms to receive */
+ dd_sendrecv_int(dd, dim_ind, dddirBackward,
+ ind->nsend, nzone+2,
+ ind->nrecv, nzone+2);
+
+ /* The rvec buffer is also required for atom buffers of size nsend
+ * in dd_move_x and dd_move_f.
+ */
+ vec_rvec_check_alloc(&comm->vbuf, ind->nsend[nzone+1]);
+
+ if (p > 0)
+ {
+ /* We can receive in place if only the last zone is not empty */
+ for (zone = 0; zone < nzone-1; zone++)
+ {
+ if (ind->nrecv[zone] > 0)
+ {
+ cd->bInPlace = FALSE;
+ }
+ }
+ if (!cd->bInPlace)
+ {
+ /* The int buffer is only required here for the cg indices */
+ if (ind->nrecv[nzone] > comm->nalloc_int2)
+ {
+ comm->nalloc_int2 = over_alloc_dd(ind->nrecv[nzone]);
+ srenew(comm->buf_int2, comm->nalloc_int2);
+ }
+ /* The rvec buffer is also required for atom buffers
+ * of size nrecv in dd_move_x and dd_move_f.
+ */
+ i = max(cd->ind[0].nrecv[nzone+1], ind->nrecv[nzone+1]);
+ vec_rvec_check_alloc(&comm->vbuf2, i);
+ }
+ }
+
+ /* Make space for the global cg indices */
+ if (pos_cg + ind->nrecv[nzone] > dd->cg_nalloc
+ || dd->cg_nalloc == 0)
+ {
+ dd->cg_nalloc = over_alloc_dd(pos_cg + ind->nrecv[nzone]);
+ srenew(index_gl, dd->cg_nalloc);
+ srenew(cgindex, dd->cg_nalloc+1);
+ }
+ /* Communicate the global cg indices */
+ if (cd->bInPlace)
+ {
+ recv_i = index_gl + pos_cg;
+ }
+ else
+ {
+ recv_i = comm->buf_int2;
+ }
+ dd_sendrecv_int(dd, dim_ind, dddirBackward,
+ comm->buf_int, nsend,
+ recv_i, ind->nrecv[nzone]);
+
+ /* Make space for cg_cm */
+ dd_check_alloc_ncg(fr, state, f, pos_cg + ind->nrecv[nzone]);
+ if (fr->cutoff_scheme == ecutsGROUP)
+ {
+ cg_cm = fr->cg_cm;
+ }
+ else
+ {
+ cg_cm = state->x;
+ }
+ /* Communicate cg_cm */
+ if (cd->bInPlace)
+ {
+ recv_vr = cg_cm + pos_cg;
+ }
+ else
+ {
+ recv_vr = comm->vbuf2.v;
+ }
+ dd_sendrecv_rvec(dd, dim_ind, dddirBackward,
+ comm->vbuf.v, nsend,
+ recv_vr, ind->nrecv[nzone]);
+
+ /* Make the charge group index */
+ if (cd->bInPlace)
+ {
+ zone = (p == 0 ? 0 : nzone - 1);
+ while (zone < nzone)
+ {
+ for (cg = 0; cg < ind->nrecv[zone]; cg++)
+ {
+ cg_gl = index_gl[pos_cg];
+ fr->cginfo[pos_cg] = ddcginfo(cginfo_mb, cg_gl);
+ nrcg = GET_CGINFO_NATOMS(fr->cginfo[pos_cg]);
+ cgindex[pos_cg+1] = cgindex[pos_cg] + nrcg;
+ if (bBondComm)
+ {
+ /* Update the charge group presence,
+ * so we can use it in the next pass of the loop.
+ */
+ comm->bLocalCG[cg_gl] = TRUE;
+ }
+ pos_cg++;
+ }
+ if (p == 0)
+ {
+ comm->zone_ncg1[nzone+zone] = ind->nrecv[zone];
+ }
+ zone++;
+ zone_cg_range[nzone+zone] = pos_cg;
+ }
+ }
+ else
+ {
+ /* This part of the code is never executed with bBondComm. */
+ merge_cg_buffers(nzone, cd, p, zone_cg_range,
+ index_gl, recv_i, cg_cm, recv_vr,
+ cgindex, fr->cginfo_mb, fr->cginfo);
+ pos_cg += ind->nrecv[nzone];
+ }
+ nat_tot += ind->nrecv[nzone+1];
+ }
+ if (!cd->bInPlace)
+ {
+ /* Store the atom block for easy copying of communication buffers */
+ make_cell2at_index(cd, nzone, zone_cg_range[nzone], cgindex);
+ }
+ nzone += nzone;
+ }
+ dd->index_gl = index_gl;
+ dd->cgindex = cgindex;
+
+ dd->ncg_tot = zone_cg_range[zones->n];
+ dd->nat_tot = nat_tot;
+ comm->nat[ddnatHOME] = dd->nat_home;
+ for (i = ddnatZONE; i < ddnatNR; i++)
+ {
+ comm->nat[i] = dd->nat_tot;
+ }
+
+ if (!bBondComm)
+ {
+ /* We don't need to update cginfo, since that was alrady done above.
+ * So we pass NULL for the forcerec.
+ */
+ dd_set_cginfo(dd->index_gl, dd->ncg_home, dd->ncg_tot,
+ NULL, comm->bLocalCG);
+ }
+
+ if (debug)
+ {
+ fprintf(debug, "Finished setting up DD communication, zones:");
+ for (c = 0; c < zones->n; c++)
+ {
+ fprintf(debug, " %d", zones->cg_range[c+1]-zones->cg_range[c]);
+ }
+ fprintf(debug, "\n");
+ }
+}
+
+static void set_cg_boundaries(gmx_domdec_zones_t *zones)
+{
+ int c;
+
+ for (c = 0; c < zones->nizone; c++)
+ {
+ zones->izone[c].cg1 = zones->cg_range[c+1];
+ zones->izone[c].jcg0 = zones->cg_range[zones->izone[c].j0];
+ zones->izone[c].jcg1 = zones->cg_range[zones->izone[c].j1];
+ }
+}
+
+static void set_zones_size(gmx_domdec_t *dd,
+ matrix box, const gmx_ddbox_t *ddbox,
+ int zone_start, int zone_end)
+{
+ gmx_domdec_comm_t *comm;
+ gmx_domdec_zones_t *zones;
+ gmx_bool bDistMB;
+ int z, zi, zj0, zj1, d, dim;
+ real rcs, rcmbs;
+ int i, j;
+ real size_j, add_tric;
+ real vol;
+
+ comm = dd->comm;
+
+ zones = &comm->zones;
+
+ /* Do we need to determine extra distances for multi-body bondeds? */
+ bDistMB = (comm->bInterCGMultiBody && dd->bGridJump && dd->ndim > 1);
+
+ for (z = zone_start; z < zone_end; z++)
+ {
+ /* Copy cell limits to zone limits.
+ * Valid for non-DD dims and non-shifted dims.
+ */
+ copy_rvec(comm->cell_x0, zones->size[z].x0);
+ copy_rvec(comm->cell_x1, zones->size[z].x1);
+ }
+
+ for (d = 0; d < dd->ndim; d++)
+ {
+ dim = dd->dim[d];
+
+ for (z = 0; z < zones->n; z++)
+ {
+ /* With a staggered grid we have different sizes
+ * for non-shifted dimensions.
+ */
+ if (dd->bGridJump && zones->shift[z][dim] == 0)
+ {
+ if (d == 1)
+ {
+ zones->size[z].x0[dim] = comm->zone_d1[zones->shift[z][dd->dim[d-1]]].min0;
+ zones->size[z].x1[dim] = comm->zone_d1[zones->shift[z][dd->dim[d-1]]].max1;
+ }
+ else if (d == 2)
+ {
+ zones->size[z].x0[dim] = comm->zone_d2[zones->shift[z][dd->dim[d-2]]][zones->shift[z][dd->dim[d-1]]].min0;
+ zones->size[z].x1[dim] = comm->zone_d2[zones->shift[z][dd->dim[d-2]]][zones->shift[z][dd->dim[d-1]]].max1;
+ }
+ }
+ }
+
+ rcs = comm->cutoff;
+ rcmbs = comm->cutoff_mbody;
+ if (ddbox->tric_dir[dim])
+ {
+ rcs /= ddbox->skew_fac[dim];
+ rcmbs /= ddbox->skew_fac[dim];
+ }
+
+ /* Set the lower limit for the shifted zone dimensions */
+ for (z = zone_start; z < zone_end; z++)
+ {
+ if (zones->shift[z][dim] > 0)
+ {
+ dim = dd->dim[d];
+ if (!dd->bGridJump || d == 0)
+ {
+ zones->size[z].x0[dim] = comm->cell_x1[dim];
+ zones->size[z].x1[dim] = comm->cell_x1[dim] + rcs;
+ }
+ else
+ {
+ /* Here we take the lower limit of the zone from
+ * the lowest domain of the zone below.
+ */
+ if (z < 4)
+ {
+ zones->size[z].x0[dim] =
+ comm->zone_d1[zones->shift[z][dd->dim[d-1]]].min1;
+ }
+ else
+ {
+ if (d == 1)
+ {
+ zones->size[z].x0[dim] =
+ zones->size[zone_perm[2][z-4]].x0[dim];
+ }
+ else
+ {
+ zones->size[z].x0[dim] =
+ comm->zone_d2[zones->shift[z][dd->dim[d-2]]][zones->shift[z][dd->dim[d-1]]].min1;
+ }
+ }
+ /* A temporary limit, is updated below */
+ zones->size[z].x1[dim] = zones->size[z].x0[dim];
+
+ if (bDistMB)
+ {
+ for (zi = 0; zi < zones->nizone; zi++)
+ {
+ if (zones->shift[zi][dim] == 0)
+ {
+ /* This takes the whole zone into account.
+ * With multiple pulses this will lead
+ * to a larger zone then strictly necessary.
+ */
+ zones->size[z].x1[dim] = max(zones->size[z].x1[dim],
+ zones->size[zi].x1[dim]+rcmbs);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ /* Loop over the i-zones to set the upper limit of each
+ * j-zone they see.
+ */
+ for (zi = 0; zi < zones->nizone; zi++)
+ {
+ if (zones->shift[zi][dim] == 0)
+ {
+ for (z = zones->izone[zi].j0; z < zones->izone[zi].j1; z++)
+ {
+ if (zones->shift[z][dim] > 0)
+ {
+ zones->size[z].x1[dim] = max(zones->size[z].x1[dim],
+ zones->size[zi].x1[dim]+rcs);
+ }
+ }
+ }
+ }
+ }
+
+ for (z = zone_start; z < zone_end; z++)
+ {
+ /* Initialization only required to keep the compiler happy */
+ rvec corner_min = {0, 0, 0}, corner_max = {0, 0, 0}, corner;
+ int nc, c;
+
+ /* To determine the bounding box for a zone we need to find
+ * the extreme corners of 4, 2 or 1 corners.
+ */
+ nc = 1 << (ddbox->npbcdim - 1);
+
+ for (c = 0; c < nc; c++)
+ {
+ /* Set up a zone corner at x=0, ignoring trilinic couplings */
+ corner[XX] = 0;
+ if ((c & 1) == 0)
+ {
+ corner[YY] = zones->size[z].x0[YY];
+ }
+ else
+ {
+ corner[YY] = zones->size[z].x1[YY];
+ }
+ if ((c & 2) == 0)
+ {
+ corner[ZZ] = zones->size[z].x0[ZZ];
+ }
+ else
+ {
+ corner[ZZ] = zones->size[z].x1[ZZ];
+ }
+ if (dd->ndim == 1 && box[ZZ][YY] != 0)
+ {
+ /* With 1D domain decomposition the cg's are not in
+ * the triclinic box, but triclinic x-y and rectangular y-z.
+ * Shift y back, so it will later end up at 0.
+ */
+ corner[YY] -= corner[ZZ]*box[ZZ][YY]/box[ZZ][ZZ];
+ }
+ /* Apply the triclinic couplings */
+ for (i = YY; i < ddbox->npbcdim; i++)
+ {
+ for (j = XX; j < i; j++)
+ {
+ corner[j] += corner[i]*box[i][j]/box[i][i];
+ }
+ }
+ if (c == 0)
+ {
+ copy_rvec(corner, corner_min);
+ copy_rvec(corner, corner_max);
+ }
+ else
+ {
+ for (i = 0; i < DIM; i++)
+ {
+ corner_min[i] = min(corner_min[i], corner[i]);
+ corner_max[i] = max(corner_max[i], corner[i]);
+ }
+ }
+ }
+ /* Copy the extreme cornes without offset along x */
+ for (i = 0; i < DIM; i++)
+ {
+ zones->size[z].bb_x0[i] = corner_min[i];
+ zones->size[z].bb_x1[i] = corner_max[i];
+ }
+ /* Add the offset along x */
+ zones->size[z].bb_x0[XX] += zones->size[z].x0[XX];
+ zones->size[z].bb_x1[XX] += zones->size[z].x1[XX];
+ }
+
+ if (zone_start == 0)
+ {
+ vol = 1;
+ for (dim = 0; dim < DIM; dim++)
+ {
+ vol *= zones->size[0].x1[dim] - zones->size[0].x0[dim];
+ }
+ zones->dens_zone0 = (zones->cg_range[1] - zones->cg_range[0])/vol;
+ }
+
+ if (debug)
+ {
+ for (z = zone_start; z < zone_end; z++)
+ {
+ fprintf(debug, "zone %d %6.3f - %6.3f %6.3f - %6.3f %6.3f - %6.3f\n",
+ z,
+ zones->size[z].x0[XX], zones->size[z].x1[XX],
+ zones->size[z].x0[YY], zones->size[z].x1[YY],
+ zones->size[z].x0[ZZ], zones->size[z].x1[ZZ]);
+ fprintf(debug, "zone %d bb %6.3f - %6.3f %6.3f - %6.3f %6.3f - %6.3f\n",
+ z,
+ zones->size[z].bb_x0[XX], zones->size[z].bb_x1[XX],
+ zones->size[z].bb_x0[YY], zones->size[z].bb_x1[YY],
+ zones->size[z].bb_x0[ZZ], zones->size[z].bb_x1[ZZ]);
+ }
+ }
+}
+
+static int comp_cgsort(const void *a, const void *b)
+{
+ int comp;
+
+ gmx_cgsort_t *cga, *cgb;
+ cga = (gmx_cgsort_t *)a;
+ cgb = (gmx_cgsort_t *)b;
+
+ comp = cga->nsc - cgb->nsc;
+ if (comp == 0)
+ {
+ comp = cga->ind_gl - cgb->ind_gl;
+ }
+
+ return comp;
+}
+
+static void order_int_cg(int n, const gmx_cgsort_t *sort,
+ int *a, int *buf)
+{
+ int i;
+
+ /* Order the data */
+ for (i = 0; i < n; i++)
+ {
+ buf[i] = a[sort[i].ind];
+ }
+
+ /* Copy back to the original array */
+ for (i = 0; i < n; i++)
+ {
+ a[i] = buf[i];
+ }
+}
+
+static void order_vec_cg(int n, const gmx_cgsort_t *sort,
+ rvec *v, rvec *buf)
+{
+ int i;
+
+ /* Order the data */
+ for (i = 0; i < n; i++)
+ {
+ copy_rvec(v[sort[i].ind], buf[i]);
+ }
+
+ /* Copy back to the original array */
+ for (i = 0; i < n; i++)
+ {
+ copy_rvec(buf[i], v[i]);
+ }
+}
+
+static void order_vec_atom(int ncg, const int *cgindex, const gmx_cgsort_t *sort,
+ rvec *v, rvec *buf)
+{
+ int a, atot, cg, cg0, cg1, i;
+
+ if (cgindex == NULL)
+ {
+ /* Avoid the useless loop of the atoms within a cg */
+ order_vec_cg(ncg, sort, v, buf);
+
+ return;
+ }
+
+ /* Order the data */
+ a = 0;
+ for (cg = 0; cg < ncg; cg++)
+ {
+ cg0 = cgindex[sort[cg].ind];
+ cg1 = cgindex[sort[cg].ind+1];
+ for (i = cg0; i < cg1; i++)
+ {
+ copy_rvec(v[i], buf[a]);
+ a++;
+ }
+ }
+ atot = a;
+
+ /* Copy back to the original array */
+ for (a = 0; a < atot; a++)
+ {
+ copy_rvec(buf[a], v[a]);
+ }
+}
+
+static void ordered_sort(int nsort2, gmx_cgsort_t *sort2,
+ int nsort_new, gmx_cgsort_t *sort_new,
+ gmx_cgsort_t *sort1)
+{
+ int i1, i2, i_new;
+
+ /* The new indices are not very ordered, so we qsort them */
+ qsort_threadsafe(sort_new, nsort_new, sizeof(sort_new[0]), comp_cgsort);
+
+ /* sort2 is already ordered, so now we can merge the two arrays */
+ i1 = 0;
+ i2 = 0;
+ i_new = 0;
+ while (i2 < nsort2 || i_new < nsort_new)
+ {
+ if (i2 == nsort2)
+ {
+ sort1[i1++] = sort_new[i_new++];
+ }
+ else if (i_new == nsort_new)
+ {
+ sort1[i1++] = sort2[i2++];
+ }
+ else if (sort2[i2].nsc < sort_new[i_new].nsc ||
+ (sort2[i2].nsc == sort_new[i_new].nsc &&
+ sort2[i2].ind_gl < sort_new[i_new].ind_gl))
+ {
+ sort1[i1++] = sort2[i2++];
+ }
+ else
+ {
+ sort1[i1++] = sort_new[i_new++];
+ }
+ }
+}
+
+static int dd_sort_order(gmx_domdec_t *dd, t_forcerec *fr, int ncg_home_old)
+{
+ gmx_domdec_sort_t *sort;
+ gmx_cgsort_t *cgsort, *sort_i;
+ int ncg_new, nsort2, nsort_new, i, *a, moved, *ibuf;
+ int sort_last, sort_skip;
+
+ sort = dd->comm->sort;
+
+ a = fr->ns.grid->cell_index;
+
+ moved = NSGRID_SIGNAL_MOVED_FAC*fr->ns.grid->ncells;
+
+ if (ncg_home_old >= 0)
+ {
+ /* The charge groups that remained in the same ns grid cell
+ * are completely ordered. So we can sort efficiently by sorting
+ * the charge groups that did move into the stationary list.
+ */
+ ncg_new = 0;
+ nsort2 = 0;
+ nsort_new = 0;
+ for (i = 0; i < dd->ncg_home; i++)
+ {
+ /* Check if this cg did not move to another node */
+ if (a[i] < moved)
+ {
+ if (i >= ncg_home_old || a[i] != sort->sort[i].nsc)
+ {
+ /* This cg is new on this node or moved ns grid cell */
+ if (nsort_new >= sort->sort_new_nalloc)
+ {
+ sort->sort_new_nalloc = over_alloc_dd(nsort_new+1);
+ srenew(sort->sort_new, sort->sort_new_nalloc);
+ }
+ sort_i = &(sort->sort_new[nsort_new++]);
+ }
+ else
+ {
+ /* This cg did not move */
+ sort_i = &(sort->sort2[nsort2++]);
+ }
+ /* Sort on the ns grid cell indices
+ * and the global topology index.
+ * index_gl is irrelevant with cell ns,
+ * but we set it here anyhow to avoid a conditional.
+ */
+ sort_i->nsc = a[i];
+ sort_i->ind_gl = dd->index_gl[i];
+ sort_i->ind = i;
+ ncg_new++;
+ }
+ }
+ if (debug)
+ {
+ fprintf(debug, "ordered sort cgs: stationary %d moved %d\n",
+ nsort2, nsort_new);
+ }
+ /* Sort efficiently */
+ ordered_sort(nsort2, sort->sort2, nsort_new, sort->sort_new,
+ sort->sort);
+ }
+ else
+ {
+ cgsort = sort->sort;
+ ncg_new = 0;
+ for (i = 0; i < dd->ncg_home; i++)
+ {
+ /* Sort on the ns grid cell indices
+ * and the global topology index
+ */
+ cgsort[i].nsc = a[i];
+ cgsort[i].ind_gl = dd->index_gl[i];
+ cgsort[i].ind = i;
+ if (cgsort[i].nsc < moved)
+ {
+ ncg_new++;
+ }
+ }
+ if (debug)
+ {
+ fprintf(debug, "qsort cgs: %d new home %d\n", dd->ncg_home, ncg_new);
+ }
+ /* Determine the order of the charge groups using qsort */
+ qsort_threadsafe(cgsort, dd->ncg_home, sizeof(cgsort[0]), comp_cgsort);
+ }
+
+ return ncg_new;
+}
+
+static int dd_sort_order_nbnxn(gmx_domdec_t *dd, t_forcerec *fr)
+{
+ gmx_cgsort_t *sort;
+ int ncg_new, i, *a, na;
+
+ sort = dd->comm->sort->sort;
+
+ nbnxn_get_atomorder(fr->nbv->nbs, &a, &na);
+
+ ncg_new = 0;
+ for (i = 0; i < na; i++)
+ {
+ if (a[i] >= 0)
+ {
+ sort[ncg_new].ind = a[i];
+ ncg_new++;
+ }
+ }
+
+ return ncg_new;
+}
+
+static void dd_sort_state(gmx_domdec_t *dd, rvec *cgcm, t_forcerec *fr, t_state *state,
+ int ncg_home_old)
+{
+ gmx_domdec_sort_t *sort;
+ gmx_cgsort_t *cgsort, *sort_i;
+ int *cgindex;
+ int ncg_new, i, *ibuf, cgsize;
+ rvec *vbuf;
+
+ sort = dd->comm->sort;
+
+ if (dd->ncg_home > sort->sort_nalloc)
+ {
+ sort->sort_nalloc = over_alloc_dd(dd->ncg_home);
+ srenew(sort->sort, sort->sort_nalloc);
+ srenew(sort->sort2, sort->sort_nalloc);
+ }
+ cgsort = sort->sort;
+
+ switch (fr->cutoff_scheme)
+ {
+ case ecutsGROUP:
+ ncg_new = dd_sort_order(dd, fr, ncg_home_old);
+ break;
+ case ecutsVERLET:
+ ncg_new = dd_sort_order_nbnxn(dd, fr);
+ break;
+ default:
+ gmx_incons("unimplemented");
+ ncg_new = 0;
+ }
+
+ /* We alloc with the old size, since cgindex is still old */
+ vec_rvec_check_alloc(&dd->comm->vbuf, dd->cgindex[dd->ncg_home]);
+ vbuf = dd->comm->vbuf.v;
+
+ if (dd->comm->bCGs)
+ {
+ cgindex = dd->cgindex;
+ }
+ else
+ {
+ cgindex = NULL;
+ }
+
+ /* Remove the charge groups which are no longer at home here */
+ dd->ncg_home = ncg_new;
+ if (debug)
+ {
+ fprintf(debug, "Set the new home charge group count to %d\n",
+ dd->ncg_home);
+ }
+
+ /* Reorder the state */
+ for (i = 0; i < estNR; i++)
+ {
+ if (EST_DISTR(i) && (state->flags & (1<<i)))
+ {
+ switch (i)
+ {
+ case estX:
+ order_vec_atom(dd->ncg_home, cgindex, cgsort, state->x, vbuf);
+ break;
+ case estV:
+ order_vec_atom(dd->ncg_home, cgindex, cgsort, state->v, vbuf);
+ break;
+ case estSDX:
+ order_vec_atom(dd->ncg_home, cgindex, cgsort, state->sd_X, vbuf);
+ break;
+ case estCGP:
+ order_vec_atom(dd->ncg_home, cgindex, cgsort, state->cg_p, vbuf);
+ break;
+ case estLD_RNG:
+ case estLD_RNGI:
+ case estDISRE_INITF:
+ case estDISRE_RM3TAV:
+ case estORIRE_INITF:
+ case estORIRE_DTAV:
+ /* No ordering required */
+ break;
+ default:
+ gmx_incons("Unknown state entry encountered in dd_sort_state");
+ break;
+ }
+ }
+ }
+ if (fr->cutoff_scheme == ecutsGROUP)
+ {
+ /* Reorder cgcm */
+ order_vec_cg(dd->ncg_home, cgsort, cgcm, vbuf);
+ }
+
+ if (dd->ncg_home+1 > sort->ibuf_nalloc)
+ {
+ sort->ibuf_nalloc = over_alloc_dd(dd->ncg_home+1);
+ srenew(sort->ibuf, sort->ibuf_nalloc);
+ }
+ ibuf = sort->ibuf;
+ /* Reorder the global cg index */
+ order_int_cg(dd->ncg_home, cgsort, dd->index_gl, ibuf);
+ /* Reorder the cginfo */
+ order_int_cg(dd->ncg_home, cgsort, fr->cginfo, ibuf);
+ /* Rebuild the local cg index */
+ if (dd->comm->bCGs)
+ {
+ ibuf[0] = 0;
+ for (i = 0; i < dd->ncg_home; i++)
+ {
+ cgsize = dd->cgindex[cgsort[i].ind+1] - dd->cgindex[cgsort[i].ind];
+ ibuf[i+1] = ibuf[i] + cgsize;
+ }
+ for (i = 0; i < dd->ncg_home+1; i++)
+ {
+ dd->cgindex[i] = ibuf[i];
+ }
+ }
+ else
+ {
+ for (i = 0; i < dd->ncg_home+1; i++)
+ {
+ dd->cgindex[i] = i;
+ }
+ }
+ /* Set the home atom number */
+ dd->nat_home = dd->cgindex[dd->ncg_home];
+
+ if (fr->cutoff_scheme == ecutsVERLET)
+ {
+ /* The atoms are now exactly in grid order, update the grid order */
+ nbnxn_set_atomorder(fr->nbv->nbs);
+ }
+ else
+ {
+ /* Copy the sorted ns cell indices back to the ns grid struct */
+ for (i = 0; i < dd->ncg_home; i++)
+ {
+ fr->ns.grid->cell_index[i] = cgsort[i].nsc;
+ }
+ fr->ns.grid->nr = dd->ncg_home;
+ }
+}
+
+static void add_dd_statistics(gmx_domdec_t *dd)
+{
+ gmx_domdec_comm_t *comm;
+ int ddnat;
+
+ comm = dd->comm;
+
+ for (ddnat = ddnatZONE; ddnat < ddnatNR; ddnat++)
+ {
+ comm->sum_nat[ddnat-ddnatZONE] +=
+ comm->nat[ddnat] - comm->nat[ddnat-1];
+ }
+ comm->ndecomp++;
+}
+
+void reset_dd_statistics_counters(gmx_domdec_t *dd)
+{
+ gmx_domdec_comm_t *comm;
+ int ddnat;
+
+ comm = dd->comm;
+
+ /* Reset all the statistics and counters for total run counting */
+ for (ddnat = ddnatZONE; ddnat < ddnatNR; ddnat++)
+ {
+ comm->sum_nat[ddnat-ddnatZONE] = 0;
+ }
+ comm->ndecomp = 0;
+ comm->nload = 0;
+ comm->load_step = 0;
+ comm->load_sum = 0;
+ comm->load_max = 0;
+ clear_ivec(comm->load_lim);
+ comm->load_mdf = 0;
+ comm->load_pme = 0;
+}
+
+void print_dd_statistics(t_commrec *cr, t_inputrec *ir, FILE *fplog)
+{
+ gmx_domdec_comm_t *comm;
+ int ddnat;
+ double av;
+
+ comm = cr->dd->comm;
+
+ gmx_sumd(ddnatNR-ddnatZONE, comm->sum_nat, cr);
+
+ if (fplog == NULL)
+ {
+ return;
+ }
+
+ fprintf(fplog, "\n D O M A I N D E C O M P O S I T I O N S T A T I S T I C S\n\n");
+
+ for (ddnat = ddnatZONE; ddnat < ddnatNR; ddnat++)
+ {
+ av = comm->sum_nat[ddnat-ddnatZONE]/comm->ndecomp;
+ switch (ddnat)
+ {
+ case ddnatZONE:
+ fprintf(fplog,
+ " av. #atoms communicated per step for force: %d x %.1f\n",
+ 2, av);
+ break;
+ case ddnatVSITE:
+ if (cr->dd->vsite_comm)
+ {
+ fprintf(fplog,
+ " av. #atoms communicated per step for vsites: %d x %.1f\n",
+ (EEL_PME(ir->coulombtype) || ir->coulombtype == eelEWALD) ? 3 : 2,
+ av);
+ }
+ break;
+ case ddnatCON:
+ if (cr->dd->constraint_comm)
+ {
+ fprintf(fplog,
+ " av. #atoms communicated per step for LINCS: %d x %.1f\n",
+ 1 + ir->nLincsIter, av);
+ }
+ break;
+ default:
+ gmx_incons(" Unknown type for DD statistics");
+ }
+ }
+ fprintf(fplog, "\n");
+
+ if (comm->bRecordLoad && EI_DYNAMICS(ir->eI))
+ {
+ print_dd_load_av(fplog, cr->dd);
+ }
+}
+
+void dd_partition_system(FILE *fplog,
+ gmx_large_int_t step,
+ t_commrec *cr,
+ gmx_bool bMasterState,
+ int nstglobalcomm,
+ t_state *state_global,
+ gmx_mtop_t *top_global,
+ t_inputrec *ir,
+ t_state *state_local,
+ rvec **f,
+ t_mdatoms *mdatoms,
+ gmx_localtop_t *top_local,
+ t_forcerec *fr,
+ gmx_vsite_t *vsite,
+ gmx_shellfc_t shellfc,
+ gmx_constr_t constr,
+ t_nrnb *nrnb,
+ gmx_wallcycle_t wcycle,
+ gmx_bool bVerbose)
+{
+ gmx_domdec_t *dd;
+ gmx_domdec_comm_t *comm;
+ gmx_ddbox_t ddbox = {0};
+ t_block *cgs_gl;
+ gmx_large_int_t step_pcoupl;
+ rvec cell_ns_x0, cell_ns_x1;
+ int i, j, n, ncgindex_set, ncg_home_old = -1, ncg_moved, nat_f_novirsum;
+ gmx_bool bBoxChanged, bNStGlobalComm, bDoDLB, bCheckDLB, bTurnOnDLB, bLogLoad;
+ gmx_bool bRedist, bSortCG, bResortAll;
+ ivec ncells_old = {0, 0, 0}, ncells_new = {0, 0, 0}, np;
+ real grid_density;
+ char sbuf[22];
+
+ dd = cr->dd;
+ comm = dd->comm;
+
+ bBoxChanged = (bMasterState || DEFORM(*ir));
+ if (ir->epc != epcNO)
+ {
+ /* With nstpcouple > 1 pressure coupling happens.
+ * one step after calculating the pressure.
+ * Box scaling happens at the end of the MD step,
+ * after the DD partitioning.
+ * We therefore have to do DLB in the first partitioning
+ * after an MD step where P-coupling occured.
+ * We need to determine the last step in which p-coupling occurred.
+ * MRS -- need to validate this for vv?
+ */
+ n = ir->nstpcouple;
+ if (n == 1)
+ {
+ step_pcoupl = step - 1;
+ }
+ else
+ {
+ step_pcoupl = ((step - 1)/n)*n + 1;
+ }
+ if (step_pcoupl >= comm->partition_step)
+ {
+ bBoxChanged = TRUE;
+ }
+ }
+
+ bNStGlobalComm = (step % nstglobalcomm == 0);
+
+ if (!comm->bDynLoadBal)
+ {
+ bDoDLB = FALSE;
+ }
+ else
+ {
+ /* Should we do dynamic load balacing this step?
+ * Since it requires (possibly expensive) global communication,
+ * we might want to do DLB less frequently.
+ */
+ if (bBoxChanged || ir->epc != epcNO)
+ {
+ bDoDLB = bBoxChanged;
+ }
+ else
+ {
+ bDoDLB = bNStGlobalComm;
+ }
+ }
+
+ /* Check if we have recorded loads on the nodes */
+ if (comm->bRecordLoad && dd_load_count(comm))
+ {
+ if (comm->eDLB == edlbAUTO && !comm->bDynLoadBal)
+ {
+ /* Check if we should use DLB at the second partitioning
+ * and every 100 partitionings,
+ * so the extra communication cost is negligible.
+ */
+ n = max(100, nstglobalcomm);
+ bCheckDLB = (comm->n_load_collect == 0 ||
+ comm->n_load_have % n == n-1);
+ }
+ else
+ {
+ bCheckDLB = FALSE;
+ }
+
+ /* Print load every nstlog, first and last step to the log file */
+ bLogLoad = ((ir->nstlog > 0 && step % ir->nstlog == 0) ||
+ comm->n_load_collect == 0 ||
+ (ir->nsteps >= 0 &&
+ (step + ir->nstlist > ir->init_step + ir->nsteps)));
+
+ /* Avoid extra communication due to verbose screen output
+ * when nstglobalcomm is set.
+ */
+ if (bDoDLB || bLogLoad || bCheckDLB ||
+ (bVerbose && (ir->nstlist == 0 || nstglobalcomm <= ir->nstlist)))
+ {
+ get_load_distribution(dd, wcycle);
+ if (DDMASTER(dd))
+ {
+ if (bLogLoad)
+ {
+ dd_print_load(fplog, dd, step-1);
+ }
+ if (bVerbose)
+ {
+ dd_print_load_verbose(dd);
+ }
+ }
+ comm->n_load_collect++;
+
+ if (bCheckDLB)
+ {
+ /* Since the timings are node dependent, the master decides */
+ if (DDMASTER(dd))
+ {
+ bTurnOnDLB =
+ (dd_force_imb_perf_loss(dd) >= DD_PERF_LOSS);
+ if (debug)
+ {
+ fprintf(debug, "step %s, imb loss %f\n",
+ gmx_step_str(step, sbuf),
+ dd_force_imb_perf_loss(dd));
+ }
+ }
+ dd_bcast(dd, sizeof(bTurnOnDLB), &bTurnOnDLB);
+ if (bTurnOnDLB)
+ {
+ turn_on_dlb(fplog, cr, step);
+ bDoDLB = TRUE;
+ }
+ }
+ }
+ comm->n_load_have++;
+ }
+
+ cgs_gl = &comm->cgs_gl;
+
+ bRedist = FALSE;
+ if (bMasterState)
+ {
+ /* Clear the old state */
+ clear_dd_indices(dd, 0, 0);
+ ncgindex_set = 0;
+
+ set_ddbox(dd, bMasterState, cr, ir, state_global->box,
+ TRUE, cgs_gl, state_global->x, &ddbox);
+
+ get_cg_distribution(fplog, step, dd, cgs_gl,
+ state_global->box, &ddbox, state_global->x);
+
+ dd_distribute_state(dd, cgs_gl,
+ state_global, state_local, f);
+
+ dd_make_local_cgs(dd, &top_local->cgs);
+
+ /* Ensure that we have space for the new distribution */
+ dd_check_alloc_ncg(fr, state_local, f, dd->ncg_home);
+
+ if (fr->cutoff_scheme == ecutsGROUP)
+ {
+ calc_cgcm(fplog, 0, dd->ncg_home,
+ &top_local->cgs, state_local->x, fr->cg_cm);
+ }
+
+ inc_nrnb(nrnb, eNR_CGCM, dd->nat_home);
+
+ dd_set_cginfo(dd->index_gl, 0, dd->ncg_home, fr, comm->bLocalCG);
+ }
+ else if (state_local->ddp_count != dd->ddp_count)
+ {
+ if (state_local->ddp_count > dd->ddp_count)
+ {
+ gmx_fatal(FARGS, "Internal inconsistency state_local->ddp_count (%d) > dd->ddp_count (%d)", state_local->ddp_count, dd->ddp_count);
+ }
+
+ if (state_local->ddp_count_cg_gl != state_local->ddp_count)
+ {
+ gmx_fatal(FARGS, "Internal inconsistency state_local->ddp_count_cg_gl (%d) != state_local->ddp_count (%d)", state_local->ddp_count_cg_gl, state_local->ddp_count);
+ }
+
+ /* Clear the old state */
+ clear_dd_indices(dd, 0, 0);
+
+ /* Build the new indices */
+ rebuild_cgindex(dd, cgs_gl->index, state_local);
+ make_dd_indices(dd, cgs_gl->index, 0);
+ ncgindex_set = dd->ncg_home;
+
+ if (fr->cutoff_scheme == ecutsGROUP)
+ {
+ /* Redetermine the cg COMs */
+ calc_cgcm(fplog, 0, dd->ncg_home,
+ &top_local->cgs, state_local->x, fr->cg_cm);
+ }
+
+ inc_nrnb(nrnb, eNR_CGCM, dd->nat_home);
+
+ dd_set_cginfo(dd->index_gl, 0, dd->ncg_home, fr, comm->bLocalCG);
+
+ set_ddbox(dd, bMasterState, cr, ir, state_local->box,
+ TRUE, &top_local->cgs, state_local->x, &ddbox);
+
+ bRedist = comm->bDynLoadBal;
+ }
+ else
+ {
+ /* We have the full state, only redistribute the cgs */
+
+ /* Clear the non-home indices */
+ clear_dd_indices(dd, dd->ncg_home, dd->nat_home);
+ ncgindex_set = 0;
+
+ /* Avoid global communication for dim's without pbc and -gcom */
+ if (!bNStGlobalComm)
+ {
+ copy_rvec(comm->box0, ddbox.box0 );
+ copy_rvec(comm->box_size, ddbox.box_size);
+ }
+ set_ddbox(dd, bMasterState, cr, ir, state_local->box,
+ bNStGlobalComm, &top_local->cgs, state_local->x, &ddbox);
+
+ bBoxChanged = TRUE;
+ bRedist = TRUE;
+ }
+ /* For dim's without pbc and -gcom */
+ copy_rvec(ddbox.box0, comm->box0 );
+ copy_rvec(ddbox.box_size, comm->box_size);
+
+ set_dd_cell_sizes(dd, &ddbox, dynamic_dd_box(&ddbox, ir), bMasterState, bDoDLB,
+ step, wcycle);
+
+ if (comm->nstDDDumpGrid > 0 && step % comm->nstDDDumpGrid == 0)
+ {
+ write_dd_grid_pdb("dd_grid", step, dd, state_local->box, &ddbox);
+ }
+
+ /* Check if we should sort the charge groups */
+ if (comm->nstSortCG > 0)
+ {
+ bSortCG = (bMasterState ||
+ (bRedist && (step % comm->nstSortCG == 0)));
+ }
+ else
+ {
+ bSortCG = FALSE;
+ }
+
+ ncg_home_old = dd->ncg_home;
+
+ ncg_moved = 0;
+ if (bRedist)
+ {
+ wallcycle_sub_start(wcycle, ewcsDD_REDIST);
+
+ dd_redistribute_cg(fplog, step, dd, ddbox.tric_dir,
+ state_local, f, fr,
+ !bSortCG, nrnb, &ncgindex_set, &ncg_moved);
+
+ wallcycle_sub_stop(wcycle, ewcsDD_REDIST);
+ }
+
+ get_nsgrid_boundaries(ddbox.nboundeddim, state_local->box,
+ dd, &ddbox,
+ &comm->cell_x0, &comm->cell_x1,
+ dd->ncg_home, fr->cg_cm,
+ cell_ns_x0, cell_ns_x1, &grid_density);
+
+ if (bBoxChanged)
+ {
+ comm_dd_ns_cell_sizes(dd, &ddbox, cell_ns_x0, cell_ns_x1, step);
+ }
+
+ switch (fr->cutoff_scheme)
+ {
+ case ecutsGROUP:
+ copy_ivec(fr->ns.grid->n, ncells_old);
+ grid_first(fplog, fr->ns.grid, dd, &ddbox,
+ state_local->box, cell_ns_x0, cell_ns_x1,
+ fr->rlistlong, grid_density);
+ break;
+ case ecutsVERLET:
+ nbnxn_get_ncells(fr->nbv->nbs, &ncells_old[XX], &ncells_old[YY]);
+ break;
+ default:
+ gmx_incons("unimplemented");
+ }
+ /* We need to store tric_dir for dd_get_ns_ranges called from ns.c */
+ copy_ivec(ddbox.tric_dir, comm->tric_dir);
+
+ if (bSortCG)
+ {
+ wallcycle_sub_start(wcycle, ewcsDD_GRID);
+
+ /* Sort the state on charge group position.
+ * This enables exact restarts from this step.
+ * It also improves performance by about 15% with larger numbers
+ * of atoms per node.
+ */
+
+ /* Fill the ns grid with the home cell,
+ * so we can sort with the indices.
+ */
+ set_zones_ncg_home(dd);
+
+ switch (fr->cutoff_scheme)
+ {
+ case ecutsVERLET:
+ set_zones_size(dd, state_local->box, &ddbox, 0, 1);
+
+ nbnxn_put_on_grid(fr->nbv->nbs, fr->ePBC, state_local->box,
+ 0,
+ comm->zones.size[0].bb_x0,
+ comm->zones.size[0].bb_x1,
+ 0, dd->ncg_home,
+ comm->zones.dens_zone0,
+ fr->cginfo,
+ state_local->x,
+ ncg_moved, bRedist ? comm->moved : NULL,
+ fr->nbv->grp[eintLocal].kernel_type,
+ fr->nbv->grp[eintLocal].nbat);
+
+ nbnxn_get_ncells(fr->nbv->nbs, &ncells_new[XX], &ncells_new[YY]);
+ break;
+ case ecutsGROUP:
+ fill_grid(&comm->zones, fr->ns.grid, dd->ncg_home,
+ 0, dd->ncg_home, fr->cg_cm);
+
+ copy_ivec(fr->ns.grid->n, ncells_new);
+ break;
+ default:
+ gmx_incons("unimplemented");
+ }
+
+ bResortAll = bMasterState;
+
+ /* Check if we can user the old order and ns grid cell indices
+ * of the charge groups to sort the charge groups efficiently.
+ */
+ if (ncells_new[XX] != ncells_old[XX] ||
+ ncells_new[YY] != ncells_old[YY] ||
+ ncells_new[ZZ] != ncells_old[ZZ])
+ {
+ bResortAll = TRUE;
+ }
+
+ if (debug)
+ {
+ fprintf(debug, "Step %s, sorting the %d home charge groups\n",
+ gmx_step_str(step, sbuf), dd->ncg_home);
+ }
+ dd_sort_state(dd, fr->cg_cm, fr, state_local,
+ bResortAll ? -1 : ncg_home_old);
+ /* Rebuild all the indices */
+ ga2la_clear(dd->ga2la);
+ ncgindex_set = 0;
+
+ wallcycle_sub_stop(wcycle, ewcsDD_GRID);
+ }
+
+ wallcycle_sub_start(wcycle, ewcsDD_SETUPCOMM);
+
+ /* Setup up the communication and communicate the coordinates */
+ setup_dd_communication(dd, state_local->box, &ddbox, fr, state_local, f);
+
+ /* Set the indices */
+ make_dd_indices(dd, cgs_gl->index, ncgindex_set);
+
+ /* Set the charge group boundaries for neighbor searching */
+ set_cg_boundaries(&comm->zones);
+
+ if (fr->cutoff_scheme == ecutsVERLET)
+ {
+ set_zones_size(dd, state_local->box, &ddbox,
+ bSortCG ? 1 : 0, comm->zones.n);
+ }
+
+ wallcycle_sub_stop(wcycle, ewcsDD_SETUPCOMM);
+
+ /*
+ write_dd_pdb("dd_home",step,"dump",top_global,cr,
+ -1,state_local->x,state_local->box);
+ */
+
+ wallcycle_sub_start(wcycle, ewcsDD_MAKETOP);
+
+ /* Extract a local topology from the global topology */
+ for (i = 0; i < dd->ndim; i++)
+ {
+ np[dd->dim[i]] = comm->cd[i].np;
+ }
+ dd_make_local_top(dd, &comm->zones, dd->npbcdim, state_local->box,
+ comm->cellsize_min, np,
+ fr,
+ fr->cutoff_scheme == ecutsGROUP ? fr->cg_cm : state_local->x,
+ vsite, top_global, top_local);
+
+ wallcycle_sub_stop(wcycle, ewcsDD_MAKETOP);
+
+ wallcycle_sub_start(wcycle, ewcsDD_MAKECONSTR);
+
+ /* Set up the special atom communication */
+ n = comm->nat[ddnatZONE];
+ for (i = ddnatZONE+1; i < ddnatNR; i++)
+ {
+ switch (i)
+ {
+ case ddnatVSITE:
+ if (vsite && vsite->n_intercg_vsite)
+ {
+ n = dd_make_local_vsites(dd, n, top_local->idef.il);
+ }
+ break;
+ case ddnatCON:
+ if (dd->bInterCGcons || dd->bInterCGsettles)
+ {
+ /* Only for inter-cg constraints we need special code */
+ n = dd_make_local_constraints(dd, n, top_global, fr->cginfo,
+ constr, ir->nProjOrder,
+ top_local->idef.il);
+ }
+ break;
+ default:
+ gmx_incons("Unknown special atom type setup");
+ }
+ comm->nat[i] = n;
+ }
+
+ wallcycle_sub_stop(wcycle, ewcsDD_MAKECONSTR);
+
+ wallcycle_sub_start(wcycle, ewcsDD_TOPOTHER);
+
+ /* Make space for the extra coordinates for virtual site
+ * or constraint communication.
+ */
+ state_local->natoms = comm->nat[ddnatNR-1];
+ if (state_local->natoms > state_local->nalloc)
+ {
+ dd_realloc_state(state_local, f, state_local->natoms);
+ }
+
+ if (fr->bF_NoVirSum)
+ {
+ if (vsite && vsite->n_intercg_vsite)
+ {
+ nat_f_novirsum = comm->nat[ddnatVSITE];
+ }
+ else
+ {
+ if (EEL_FULL(ir->coulombtype) && dd->n_intercg_excl > 0)
+ {
+ nat_f_novirsum = dd->nat_tot;
+ }
+ else
+ {
+ nat_f_novirsum = dd->nat_home;
+ }
+ }
+ }
+ else
+ {
+ nat_f_novirsum = 0;
+ }
+
+ /* Set the number of atoms required for the force calculation.
+ * Forces need to be constrained when using a twin-range setup
+ * or with energy minimization. For simple simulations we could
+ * avoid some allocation, zeroing and copying, but this is
+ * probably not worth the complications ande checking.
+ */
+ forcerec_set_ranges(fr, dd->ncg_home, dd->ncg_tot,
+ dd->nat_tot, comm->nat[ddnatCON], nat_f_novirsum);
+
+ /* We make the all mdatoms up to nat_tot_con.
+ * We could save some work by only setting invmass
+ * between nat_tot and nat_tot_con.
+ */
+ /* This call also sets the new number of home particles to dd->nat_home */
+ atoms2md(top_global, ir,
+ comm->nat[ddnatCON], dd->gatindex, 0, dd->nat_home, mdatoms);
+
+ /* Now we have the charges we can sort the FE interactions */
+ dd_sort_local_top(dd, mdatoms, top_local);
+
+ if (vsite != NULL)
+ {
+ /* Now we have updated mdatoms, we can do the last vsite bookkeeping */
+ split_vsites_over_threads(top_local->idef.il, mdatoms, FALSE, vsite);
+ }
+
+ if (shellfc)
+ {
+ /* Make the local shell stuff, currently no communication is done */
+ make_local_shells(cr, mdatoms, shellfc);
+ }
+
+ if (ir->implicit_solvent)
+ {
+ make_local_gb(cr, fr->born, ir->gb_algorithm);
+ }
+
+ setup_bonded_threading(fr, &top_local->idef);
+
+ if (!(cr->duty & DUTY_PME))
+ {
+ /* Send the charges to our PME only node */
+ gmx_pme_send_q(cr, mdatoms->nChargePerturbed,
+ mdatoms->chargeA, mdatoms->chargeB,
+ dd_pme_maxshift_x(dd), dd_pme_maxshift_y(dd));
+ }
+
+ if (constr)
+ {
+ set_constraints(constr, top_local, ir, mdatoms, cr);
+ }
+
+ if (ir->ePull != epullNO)
+ {
+ /* Update the local pull groups */
+ dd_make_local_pull_groups(dd, ir->pull, mdatoms);
+ }
+
+ if (ir->bRot)
+ {
+ /* Update the local rotation groups */
+ dd_make_local_rotation_groups(dd, ir->rot);
+ }
+
+
+ add_dd_statistics(dd);
+
+ /* Make sure we only count the cycles for this DD partitioning */
+ clear_dd_cycle_counts(dd);
+
+ /* Because the order of the atoms might have changed since
+ * the last vsite construction, we need to communicate the constructing
+ * atom coordinates again (for spreading the forces this MD step).
+ */
+ dd_move_x_vsites(dd, state_local->box, state_local->x);
+
+ wallcycle_sub_stop(wcycle, ewcsDD_TOPOTHER);
+
+ if (comm->nstDDDump > 0 && step % comm->nstDDDump == 0)
+ {
+ dd_move_x(dd, state_local->box, state_local->x);
+ write_dd_pdb("dd_dump", step, "dump", top_global, cr,
+ -1, state_local->x, state_local->box);
+ }
+
+ /* Store the partitioning step */
+ comm->partition_step = step;
+
+ /* Increase the DD partitioning counter */
+ dd->ddp_count++;
+ /* The state currently matches this DD partitioning count, store it */
+ state_local->ddp_count = dd->ddp_count;
+ if (bMasterState)
+ {
+ /* The DD master node knows the complete cg distribution,
+ * store the count so we can possibly skip the cg info communication.
+ */
+ comm->master_cg_ddp_count = (bSortCG ? 0 : dd->ddp_count);
+ }
+
+ if (comm->DD_debug > 0)
+ {
+ /* Set the env var GMX_DD_DEBUG if you suspect corrupted indices */
+ check_index_consistency(dd, top_global->natoms, ncg_mtop(top_global),
+ "after partitioning");
+ }
+}
--- /dev/null
- void GenerateGibbsProbabilities(real *ene, double *p_k, double *pks, int minfep, int maxfep)
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2012, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * GROwing Monsters And Cloning Shrimps
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#ifdef GMX_CRAY_XT3
+#include <catamount/dclock.h>
+#endif
+
+
+#include <stdio.h>
+#include <time.h>
+#ifdef HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif
+#include <math.h>
+#include "typedefs.h"
+#include "string2.h"
+#include "gmxfio.h"
+#include "smalloc.h"
+#include "names.h"
+#include "confio.h"
+#include "mvdata.h"
+#include "txtdump.h"
+#include "pbc.h"
+#include "chargegroup.h"
+#include "vec.h"
+#include "nrnb.h"
+#include "mshift.h"
+#include "mdrun.h"
+#include "update.h"
+#include "physics.h"
+#include "main.h"
+#include "mdatoms.h"
+#include "force.h"
+#include "bondf.h"
+#include "pme.h"
+#include "disre.h"
+#include "orires.h"
+#include "network.h"
+#include "calcmu.h"
+#include "constr.h"
+#include "xvgr.h"
+#include "trnio.h"
+#include "xtcio.h"
+#include "gmx_random.h"
+#include "domdec.h"
+#include "partdec.h"
+#include "gmx_wallcycle.h"
+#include "macros.h"
+
+#include "gromacs/utility/gmxmpi.h"
+
- void GenerateWeightedGibbsProbabilities(real *ene, double *p_k, double *pks, int nlim, real *nvals, real delta)
++static void init_df_history_weights(df_history_t *dfhist, t_expanded *expand, int nlim)
++{
++ int i;
++ dfhist->wl_delta = expand->init_wl_delta;
++ for (i = 0; i < nlim; i++)
++ {
++ dfhist->sum_weights[i] = expand->init_lambda_weights[i];
++ dfhist->sum_dg[i] = expand->init_lambda_weights[i];
++ }
++}
++
++/* Eventually should contain all the functions needed to initialize expanded ensemble
++ before the md loop starts */
++extern void init_expanded_ensemble(gmx_bool bStateFromCP, t_inputrec *ir, gmx_rng_t *mcrng, df_history_t *dfhist)
++{
++ *mcrng = gmx_rng_init(ir->expandedvals->lmc_seed);
++ if (!bStateFromCP)
++ {
++ init_df_history_weights(dfhist,ir->expandedvals,ir->fepvals->n_lambda);
++ }
++}
++
++static void GenerateGibbsProbabilities(real *ene, double *p_k, double *pks, int minfep, int maxfep)
+{
+
+ int i;
+ real maxene;
+
+ *pks = 0.0;
+ maxene = ene[minfep];
+ /* find the maximum value */
+ for (i = minfep; i <= maxfep; i++)
+ {
+ if (ene[i] > maxene)
+ {
+ maxene = ene[i];
+ }
+ }
+ /* find the denominator */
+ for (i = minfep; i <= maxfep; i++)
+ {
+ *pks += exp(ene[i]-maxene);
+ }
+ /*numerators*/
+ for (i = minfep; i <= maxfep; i++)
+ {
+ p_k[i] = exp(ene[i]-maxene) / *pks;
+ }
+}
+
- int i, k, n, nz, indexi, indexk, min_n, max_n, nlam, totali;
++static void GenerateWeightedGibbsProbabilities(real *ene, double *p_k, double *pks, int nlim, real *nvals, real delta)
+{
+
+ int i;
+ real maxene;
+ real *nene;
+ *pks = 0.0;
+
+ snew(nene, nlim);
+ for (i = 0; i < nlim; i++)
+ {
+ if (nvals[i] == 0)
+ {
+ /* add the delta, since we need to make sure it's greater than zero, and
+ we need a non-arbitrary number? */
+ nene[i] = ene[i] + log(nvals[i]+delta);
+ }
+ else
+ {
+ nene[i] = ene[i] + log(nvals[i]);
+ }
+ }
+
+ /* find the maximum value */
+ maxene = nene[0];
+ for (i = 0; i < nlim; i++)
+ {
+ if (nene[i] > maxene)
+ {
+ maxene = nene[i];
+ }
+ }
+
+ /* subtract off the maximum, avoiding overflow */
+ for (i = 0; i < nlim; i++)
+ {
+ nene[i] -= maxene;
+ }
+
+ /* find the denominator */
+ for (i = 0; i < nlim; i++)
+ {
+ *pks += exp(nene[i]);
+ }
+
+ /*numerators*/
+ for (i = 0; i < nlim; i++)
+ {
+ p_k[i] = exp(nene[i]) / *pks;
+ }
+ sfree(nene);
+}
+
+real do_logsum(int N, real *a_n)
+{
+
+ /* RETURN VALUE */
+ /* log(\sum_{i=0}^(N-1) exp[a_n]) */
+ real maxarg;
+ real sum;
+ int i;
+ real logsum;
+ /* compute maximum argument to exp(.) */
+
+ maxarg = a_n[0];
+ for (i = 1; i < N; i++)
+ {
+ maxarg = max(maxarg, a_n[i]);
+ }
+
+ /* compute sum of exp(a_n - maxarg) */
+ sum = 0.0;
+ for (i = 0; i < N; i++)
+ {
+ sum = sum + exp(a_n[i] - maxarg);
+ }
+
+ /* compute log sum */
+ logsum = log(sum) + maxarg;
+ return logsum;
+}
+
+int FindMinimum(real *min_metric, int N)
+{
+
+ real min_val;
+ int min_nval, nval;
+
+ min_nval = 0;
+ min_val = min_metric[0];
+
+ for (nval = 0; nval < N; nval++)
+ {
+ if (min_metric[nval] < min_val)
+ {
+ min_val = min_metric[nval];
+ min_nval = nval;
+ }
+ }
+ return min_nval;
+}
+
+static gmx_bool CheckHistogramRatios(int nhisto, real *histo, real ratio)
+{
+
+ int i;
+ real nmean;
+ gmx_bool bIfFlat;
+
+ nmean = 0;
+ for (i = 0; i < nhisto; i++)
+ {
+ nmean += histo[i];
+ }
+
+ if (nmean == 0)
+ {
+ /* no samples! is bad!*/
+ bIfFlat = FALSE;
+ return bIfFlat;
+ }
+ nmean /= (real)nhisto;
+
+ bIfFlat = TRUE;
+ for (i = 0; i < nhisto; i++)
+ {
+ /* make sure that all points are in the ratio < x < 1/ratio range */
+ if (!((histo[i]/nmean < 1.0/ratio) && (histo[i]/nmean > ratio)))
+ {
+ bIfFlat = FALSE;
+ break;
+ }
+ }
+ return bIfFlat;
+}
+
+static gmx_bool CheckIfDoneEquilibrating(int nlim, t_expanded *expand, df_history_t *dfhist, gmx_large_int_t step)
+{
+
+ int i, totalsamples;
+ gmx_bool bDoneEquilibrating = TRUE;
+ gmx_bool bIfFlat;
+
+ /* assume we have equilibrated the weights, then check to see if any of the conditions are not met */
+
+ /* calculate the total number of samples */
+ switch (expand->elmceq)
+ {
+ case elmceqNO:
+ /* We have not equilibrated, and won't, ever. */
+ return FALSE;
+ case elmceqYES:
+ /* we have equilibrated -- we're done */
+ return TRUE;
+ case elmceqSTEPS:
+ /* first, check if we are equilibrating by steps, if we're still under */
+ if (step < expand->equil_steps)
+ {
+ bDoneEquilibrating = FALSE;
+ }
+ break;
+ case elmceqSAMPLES:
+ totalsamples = 0;
+ for (i = 0; i < nlim; i++)
+ {
+ totalsamples += dfhist->n_at_lam[i];
+ }
+ if (totalsamples < expand->equil_samples)
+ {
+ bDoneEquilibrating = FALSE;
+ }
+ break;
+ case elmceqNUMATLAM:
+ for (i = 0; i < nlim; i++)
+ {
+ if (dfhist->n_at_lam[i] < expand->equil_n_at_lam) /* we are still doing the initial sweep, so we're definitely not
+ done equilibrating*/
+ {
+ bDoneEquilibrating = FALSE;
+ break;
+ }
+ }
+ break;
+ case elmceqWLDELTA:
+ if (EWL(expand->elamstats)) /* This check is in readir as well, but
+ just to be sure */
+ {
+ if (dfhist->wl_delta > expand->equil_wl_delta)
+ {
+ bDoneEquilibrating = FALSE;
+ }
+ }
+ break;
+ case elmceqRATIO:
+ /* we can use the flatness as a judge of good weights, as long as
+ we're not doing minvar, or Wang-Landau.
+ But turn off for now until we figure out exactly how we do this.
+ */
+
+ if (!(EWL(expand->elamstats) || expand->elamstats == elamstatsMINVAR))
+ {
+ /* we want to use flatness -avoiding- the forced-through samples. Plus, we need to convert to
+ floats for this histogram function. */
+
+ real *modhisto;
+ snew(modhisto, nlim);
+ for (i = 0; i < nlim; i++)
+ {
+ modhisto[i] = 1.0*(dfhist->n_at_lam[i]-expand->lmc_forced_nstart);
+ }
+ bIfFlat = CheckHistogramRatios(nlim, modhisto, expand->equil_ratio);
+ sfree(modhisto);
+ if (!bIfFlat)
+ {
+ bDoneEquilibrating = FALSE;
+ }
+ }
+ default:
+ bDoneEquilibrating = TRUE;
+ }
+ /* one last case to go though, if we are doing slow growth to get initial values, we haven't finished equilibrating */
+
+ if (expand->lmc_forced_nstart > 0)
+ {
+ for (i = 0; i < nlim; i++)
+ {
+ if (dfhist->n_at_lam[i] < expand->lmc_forced_nstart) /* we are still doing the initial sweep, so we're definitely not
+ done equilibrating*/
+ {
+ bDoneEquilibrating = FALSE;
+ break;
+ }
+ }
+ }
+ return bDoneEquilibrating;
+}
+
+static gmx_bool UpdateWeights(int nlim, t_expanded *expand, df_history_t *dfhist,
+ int fep_state, real *scaled_lamee, real *weighted_lamee, gmx_large_int_t step)
+{
+ real maxdiff = 0.000000001;
+ gmx_bool bSufficientSamples;
- int nlam, int frequency, gmx_large_int_t step)
++ int i, k, n, nz, indexi, indexk, min_n, max_n, totali;
+ int n0, np1, nm1, nval, min_nvalm, min_nvalp, maxc;
+ real omega_m1_0, omega_p1_m1, omega_m1_p1, omega_p1_0, clam_osum;
+ real de, de_function, dr, denom, maxdr;
+ real min_val, cnval, zero_sum_weights;
+ real *omegam_array, *weightsm_array, *omegap_array, *weightsp_array, *varm_array, *varp_array, *dwp_array, *dwm_array;
+ real clam_varm, clam_varp, clam_weightsm, clam_weightsp, clam_minvar;
+ real *lam_weights, *lam_minvar_corr, *lam_variance, *lam_dg;
+ double *p_k;
+ double pks = 0;
+ real *numweighted_lamee, *logfrac;
+ int *nonzero;
+ real chi_m1_0, chi_p1_0, chi_m2_0, chi_p2_0, chi_p1_m1, chi_p2_m1, chi_m1_p1, chi_m2_p1;
+
+ /* if we have equilibrated the weights, exit now */
+ if (dfhist->bEquil)
+ {
+ return FALSE;
+ }
+
+ if (CheckIfDoneEquilibrating(nlim, expand, dfhist, step))
+ {
+ dfhist->bEquil = TRUE;
+ /* zero out the visited states so we know how many equilibrated states we have
+ from here on out.*/
+ for (i = 0; i < nlim; i++)
+ {
+ dfhist->n_at_lam[i] = 0;
+ }
+ return TRUE;
+ }
+
+ /* If we reached this far, we have not equilibrated yet, keep on
+ going resetting the weights */
+
+ if (EWL(expand->elamstats))
+ {
+ if (expand->elamstats == elamstatsWL) /* Standard Wang-Landau */
+ {
+ dfhist->sum_weights[fep_state] -= dfhist->wl_delta;
+ dfhist->wl_histo[fep_state] += 1.0;
+ }
+ else if (expand->elamstats == elamstatsWWL) /* Weighted Wang-Landau */
+ {
+ snew(p_k, nlim);
+
+ /* first increment count */
+ GenerateGibbsProbabilities(weighted_lamee, p_k, &pks, 0, nlim-1);
+ for (i = 0; i < nlim; i++)
+ {
+ dfhist->wl_histo[i] += (real)p_k[i];
+ }
+
+ /* then increment weights (uses count) */
+ pks = 0.0;
+ GenerateWeightedGibbsProbabilities(weighted_lamee, p_k, &pks, nlim, dfhist->wl_histo, dfhist->wl_delta);
+
+ for (i = 0; i < nlim; i++)
+ {
+ dfhist->sum_weights[i] -= dfhist->wl_delta*(real)p_k[i];
+ }
+ /* Alternate definition, using logarithms. Shouldn't make very much difference! */
+ /*
+ real di;
+ for (i=0;i<nlim;i++)
+ {
+ di = (real)1.0 + dfhist->wl_delta*(real)p_k[i];
+ dfhist->sum_weights[i] -= log(di);
+ }
+ */
+ sfree(p_k);
+ }
+
+ zero_sum_weights = dfhist->sum_weights[0];
+ for (i = 0; i < nlim; i++)
+ {
+ dfhist->sum_weights[i] -= zero_sum_weights;
+ }
+ }
+
+ if (expand->elamstats == elamstatsBARKER || expand->elamstats == elamstatsMETROPOLIS || expand->elamstats == elamstatsMINVAR)
+ {
+
+ de_function = 0; /* to get rid of warnings, but this value will not be used because of the logic */
+ maxc = 2*expand->c_range+1;
+
+ snew(lam_dg, nlim);
+ snew(lam_variance, nlim);
+
+ snew(omegap_array, maxc);
+ snew(weightsp_array, maxc);
+ snew(varp_array, maxc);
+ snew(dwp_array, maxc);
+
+ snew(omegam_array, maxc);
+ snew(weightsm_array, maxc);
+ snew(varm_array, maxc);
+ snew(dwm_array, maxc);
+
+ /* unpack the current lambdas -- we will only update 2 of these */
+
+ for (i = 0; i < nlim-1; i++)
+ { /* only through the second to last */
+ lam_dg[i] = dfhist->sum_dg[i+1] - dfhist->sum_dg[i];
+ lam_variance[i] = pow(dfhist->sum_variance[i+1], 2) - pow(dfhist->sum_variance[i], 2);
+ }
+
+ /* accumulate running averages */
+ for (nval = 0; nval < maxc; nval++)
+ {
+ /* constants for later use */
+ cnval = (real)(nval-expand->c_range);
+ /* actually, should be able to rewrite it w/o exponential, for better numerical stability */
+ if (fep_state > 0)
+ {
+ de = exp(cnval - (scaled_lamee[fep_state]-scaled_lamee[fep_state-1]));
+ if (expand->elamstats == elamstatsBARKER || expand->elamstats == elamstatsMINVAR)
+ {
+ de_function = 1.0/(1.0+de);
+ }
+ else if (expand->elamstats == elamstatsMETROPOLIS)
+ {
+ if (de < 1.0)
+ {
+ de_function = 1.0;
+ }
+ else
+ {
+ de_function = 1.0/de;
+ }
+ }
+ dfhist->accum_m[fep_state][nval] += de_function;
+ dfhist->accum_m2[fep_state][nval] += de_function*de_function;
+ }
+
+ if (fep_state < nlim-1)
+ {
+ de = exp(-cnval + (scaled_lamee[fep_state+1]-scaled_lamee[fep_state]));
+ if (expand->elamstats == elamstatsBARKER || expand->elamstats == elamstatsMINVAR)
+ {
+ de_function = 1.0/(1.0+de);
+ }
+ else if (expand->elamstats == elamstatsMETROPOLIS)
+ {
+ if (de < 1.0)
+ {
+ de_function = 1.0;
+ }
+ else
+ {
+ de_function = 1.0/de;
+ }
+ }
+ dfhist->accum_p[fep_state][nval] += de_function;
+ dfhist->accum_p2[fep_state][nval] += de_function*de_function;
+ }
+
+ /* Metropolis transition and Barker transition (unoptimized Bennett) acceptance weight determination */
+
+ n0 = dfhist->n_at_lam[fep_state];
+ if (fep_state > 0)
+ {
+ nm1 = dfhist->n_at_lam[fep_state-1];
+ }
+ else
+ {
+ nm1 = 0;
+ }
+ if (fep_state < nlim-1)
+ {
+ np1 = dfhist->n_at_lam[fep_state+1];
+ }
+ else
+ {
+ np1 = 0;
+ }
+
+ /* logic SHOULD keep these all set correctly whatever the logic, but apparently it can't figure it out. */
+ chi_m1_0 = chi_p1_0 = chi_m2_0 = chi_p2_0 = chi_p1_m1 = chi_p2_m1 = chi_m1_p1 = chi_m2_p1 = 0;
+
+ if (n0 > 0)
+ {
+ chi_m1_0 = dfhist->accum_m[fep_state][nval]/n0;
+ chi_p1_0 = dfhist->accum_p[fep_state][nval]/n0;
+ chi_m2_0 = dfhist->accum_m2[fep_state][nval]/n0;
+ chi_p2_0 = dfhist->accum_p2[fep_state][nval]/n0;
+ }
+
+ if ((fep_state > 0 ) && (nm1 > 0))
+ {
+ chi_p1_m1 = dfhist->accum_p[fep_state-1][nval]/nm1;
+ chi_p2_m1 = dfhist->accum_p2[fep_state-1][nval]/nm1;
+ }
+
+ if ((fep_state < nlim-1) && (np1 > 0))
+ {
+ chi_m1_p1 = dfhist->accum_m[fep_state+1][nval]/np1;
+ chi_m2_p1 = dfhist->accum_m2[fep_state+1][nval]/np1;
+ }
+
+ omega_m1_0 = 0;
+ omega_p1_0 = 0;
+ clam_weightsm = 0;
+ clam_weightsp = 0;
+ clam_varm = 0;
+ clam_varp = 0;
+
+ if (fep_state > 0)
+ {
+ if (n0 > 0)
+ {
+ omega_m1_0 = chi_m2_0/(chi_m1_0*chi_m1_0) - 1.0;
+ }
+ if (nm1 > 0)
+ {
+ omega_p1_m1 = chi_p2_m1/(chi_p1_m1*chi_p1_m1) - 1.0;
+ }
+ if ((n0 > 0) && (nm1 > 0))
+ {
+ clam_weightsm = (log(chi_m1_0) - log(chi_p1_m1)) + cnval;
+ clam_varm = (1.0/n0)*(omega_m1_0) + (1.0/nm1)*(omega_p1_m1);
+ }
+ }
+
+ if (fep_state < nlim-1)
+ {
+ if (n0 > 0)
+ {
+ omega_p1_0 = chi_p2_0/(chi_p1_0*chi_p1_0) - 1.0;
+ }
+ if (np1 > 0)
+ {
+ omega_m1_p1 = chi_m2_p1/(chi_m1_p1*chi_m1_p1) - 1.0;
+ }
+ if ((n0 > 0) && (np1 > 0))
+ {
+ clam_weightsp = (log(chi_m1_p1) - log(chi_p1_0)) + cnval;
+ clam_varp = (1.0/np1)*(omega_m1_p1) + (1.0/n0)*(omega_p1_0);
+ }
+ }
+
+ if (n0 > 0)
+ {
+ omegam_array[nval] = omega_m1_0;
+ }
+ else
+ {
+ omegam_array[nval] = 0;
+ }
+ weightsm_array[nval] = clam_weightsm;
+ varm_array[nval] = clam_varm;
+ if (nm1 > 0)
+ {
+ dwm_array[nval] = fabs( (cnval + log((1.0*n0)/nm1)) - lam_dg[fep_state-1] );
+ }
+ else
+ {
+ dwm_array[nval] = fabs( cnval - lam_dg[fep_state-1] );
+ }
+
+ if (n0 > 0)
+ {
+ omegap_array[nval] = omega_p1_0;
+ }
+ else
+ {
+ omegap_array[nval] = 0;
+ }
+ weightsp_array[nval] = clam_weightsp;
+ varp_array[nval] = clam_varp;
+ if ((np1 > 0) && (n0 > 0))
+ {
+ dwp_array[nval] = fabs( (cnval + log((1.0*np1)/n0)) - lam_dg[fep_state] );
+ }
+ else
+ {
+ dwp_array[nval] = fabs( cnval - lam_dg[fep_state] );
+ }
+
+ }
+
+ /* find the C's closest to the old weights value */
+
+ min_nvalm = FindMinimum(dwm_array, maxc);
+ omega_m1_0 = omegam_array[min_nvalm];
+ clam_weightsm = weightsm_array[min_nvalm];
+ clam_varm = varm_array[min_nvalm];
+
+ min_nvalp = FindMinimum(dwp_array, maxc);
+ omega_p1_0 = omegap_array[min_nvalp];
+ clam_weightsp = weightsp_array[min_nvalp];
+ clam_varp = varp_array[min_nvalp];
+
+ clam_osum = omega_m1_0 + omega_p1_0;
+ clam_minvar = 0;
+ if (clam_osum > 0)
+ {
+ clam_minvar = 0.5*log(clam_osum);
+ }
+
+ if (fep_state > 0)
+ {
+ lam_dg[fep_state-1] = clam_weightsm;
+ lam_variance[fep_state-1] = clam_varm;
+ }
+
+ if (fep_state < nlim-1)
+ {
+ lam_dg[fep_state] = clam_weightsp;
+ lam_variance[fep_state] = clam_varp;
+ }
+
+ if (expand->elamstats == elamstatsMINVAR)
+ {
+ bSufficientSamples = TRUE;
+ /* make sure they are all past a threshold */
+ for (i = 0; i < nlim; i++)
+ {
+ if (dfhist->n_at_lam[i] < expand->minvarmin)
+ {
+ bSufficientSamples = FALSE;
+ }
+ }
+ if (bSufficientSamples)
+ {
+ dfhist->sum_minvar[fep_state] = clam_minvar;
+ if (fep_state == 0)
+ {
+ for (i = 0; i < nlim; i++)
+ {
+ dfhist->sum_minvar[i] += (expand->minvar_const-clam_minvar);
+ }
+ expand->minvar_const = clam_minvar;
+ dfhist->sum_minvar[fep_state] = 0.0;
+ }
+ else
+ {
+ dfhist->sum_minvar[fep_state] -= expand->minvar_const;
+ }
+ }
+ }
+
+ /* we need to rezero minvar now, since it could change at fep_state = 0 */
+ dfhist->sum_dg[0] = 0.0;
+ dfhist->sum_variance[0] = 0.0;
+ dfhist->sum_weights[0] = dfhist->sum_dg[0] + dfhist->sum_minvar[0]; /* should be zero */
+
+ for (i = 1; i < nlim; i++)
+ {
+ dfhist->sum_dg[i] = lam_dg[i-1] + dfhist->sum_dg[i-1];
+ dfhist->sum_variance[i] = sqrt(lam_variance[i-1] + pow(dfhist->sum_variance[i-1], 2));
+ dfhist->sum_weights[i] = dfhist->sum_dg[i] + dfhist->sum_minvar[i];
+ }
+
+ sfree(lam_dg);
+ sfree(lam_variance);
+
+ sfree(omegam_array);
+ sfree(weightsm_array);
+ sfree(varm_array);
+ sfree(dwm_array);
+
+ sfree(omegap_array);
+ sfree(weightsp_array);
+ sfree(varp_array);
+ sfree(dwp_array);
+ }
+ return FALSE;
+}
+
+static int ChooseNewLambda(int nlim, t_expanded *expand, df_history_t *dfhist, int fep_state, real *weighted_lamee, double *p_k, gmx_rng_t rng)
+{
+ /* Choose new lambda value, and update transition matrix */
+
+ int i, ifep, jfep, minfep, maxfep, lamnew, lamtrial, starting_fep_state;
+ real r1, r2, de_old, de_new, de, trialprob, tprob = 0;
+ real **Tij;
+ double *propose, *accept, *remainder;
+ double pks;
+ real sum, pnorm;
+ gmx_bool bRestricted;
+
+ starting_fep_state = fep_state;
+ lamnew = fep_state; /* so that there is a default setting -- stays the same */
+
+ if (!EWL(expand->elamstats)) /* ignore equilibrating the weights if using WL */
+ {
+ if ((expand->lmc_forced_nstart > 0) && (dfhist->n_at_lam[nlim-1] <= expand->lmc_forced_nstart))
+ {
+ /* Use a marching method to run through the lambdas and get preliminary free energy data,
+ before starting 'free' sampling. We start free sampling when we have enough at each lambda */
+
+ /* if we have enough at this lambda, move on to the next one */
+
+ if (dfhist->n_at_lam[fep_state] == expand->lmc_forced_nstart)
+ {
+ lamnew = fep_state+1;
+ if (lamnew == nlim) /* whoops, stepped too far! */
+ {
+ lamnew -= 1;
+ }
+ }
+ else
+ {
+ lamnew = fep_state;
+ }
+ return lamnew;
+ }
+ }
+
+ snew(propose, nlim);
+ snew(accept, nlim);
+ snew(remainder, nlim);
+
+ for (i = 0; i < expand->lmc_repeats; i++)
+ {
+
+ for (ifep = 0; ifep < nlim; ifep++)
+ {
+ propose[ifep] = 0;
+ accept[ifep] = 0;
+ }
+
+ if ((expand->elmcmove == elmcmoveGIBBS) || (expand->elmcmove == elmcmoveMETGIBBS))
+ {
+ bRestricted = TRUE;
+ /* use the Gibbs sampler, with restricted range */
+ if (expand->gibbsdeltalam < 0)
+ {
+ minfep = 0;
+ maxfep = nlim-1;
+ bRestricted = FALSE;
+ }
+ else
+ {
+ minfep = fep_state - expand->gibbsdeltalam;
+ maxfep = fep_state + expand->gibbsdeltalam;
+ if (minfep < 0)
+ {
+ minfep = 0;
+ }
+ if (maxfep > nlim-1)
+ {
+ maxfep = nlim-1;
+ }
+ }
+
+ GenerateGibbsProbabilities(weighted_lamee, p_k, &pks, minfep, maxfep);
+
+ if (expand->elmcmove == elmcmoveGIBBS)
+ {
+ for (ifep = minfep; ifep <= maxfep; ifep++)
+ {
+ propose[ifep] = p_k[ifep];
+ accept[ifep] = 1.0;
+ }
+ /* Gibbs sampling */
+ r1 = gmx_rng_uniform_real(rng);
+ for (lamnew = minfep; lamnew <= maxfep; lamnew++)
+ {
+ if (r1 <= p_k[lamnew])
+ {
+ break;
+ }
+ r1 -= p_k[lamnew];
+ }
+ }
+ else if (expand->elmcmove == elmcmoveMETGIBBS)
+ {
+
+ /* Metropolized Gibbs sampling */
+ for (ifep = minfep; ifep <= maxfep; ifep++)
+ {
+ remainder[ifep] = 1 - p_k[ifep];
+ }
+
+ /* find the proposal probabilities */
+
+ if (remainder[fep_state] == 0)
+ {
+ /* only the current state has any probability */
+ /* we have to stay at the current state */
+ lamnew = fep_state;
+ }
+ else
+ {
+ for (ifep = minfep; ifep <= maxfep; ifep++)
+ {
+ if (ifep != fep_state)
+ {
+ propose[ifep] = p_k[ifep]/remainder[fep_state];
+ }
+ else
+ {
+ propose[ifep] = 0;
+ }
+ }
+
+ r1 = gmx_rng_uniform_real(rng);
+ for (lamtrial = minfep; lamtrial <= maxfep; lamtrial++)
+ {
+ pnorm = p_k[lamtrial]/remainder[fep_state];
+ if (lamtrial != fep_state)
+ {
+ if (r1 <= pnorm)
+ {
+ break;
+ }
+ r1 -= pnorm;
+ }
+ }
+
+ /* we have now selected lamtrial according to p(lamtrial)/1-p(fep_state) */
+ tprob = 1.0;
+ /* trial probability is min{1,\frac{1 - p(old)}{1-p(new)} MRS 1/8/2008 */
+ trialprob = (remainder[fep_state])/(remainder[lamtrial]);
+ if (trialprob < tprob)
+ {
+ tprob = trialprob;
+ }
+ r2 = gmx_rng_uniform_real(rng);
+ if (r2 < tprob)
+ {
+ lamnew = lamtrial;
+ }
+ else
+ {
+ lamnew = fep_state;
+ }
+ }
+
+ /* now figure out the acceptance probability for each */
+ for (ifep = minfep; ifep <= maxfep; ifep++)
+ {
+ tprob = 1.0;
+ if (remainder[ifep] != 0)
+ {
+ trialprob = (remainder[fep_state])/(remainder[ifep]);
+ }
+ else
+ {
+ trialprob = 1.0; /* this state is the only choice! */
+ }
+ if (trialprob < tprob)
+ {
+ tprob = trialprob;
+ }
+ /* probability for fep_state=0, but that's fine, it's never proposed! */
+ accept[ifep] = tprob;
+ }
+ }
+
+ if (lamnew > maxfep)
+ {
+ /* it's possible some rounding is failing */
+ if (gmx_within_tol(remainder[fep_state],0,50*GMX_DOUBLE_EPS))
+ {
+ /* numerical rounding error -- no state other than the original has weight */
+ lamnew = fep_state;
+ }
+ else
+ {
+ /* probably not a numerical issue */
+ int loc = 0;
+ int nerror = 200+(maxfep-minfep+1)*60;
+ char *errorstr;
+ snew(errorstr, nerror);
+ /* if its greater than maxfep, then something went wrong -- probably underflow in the calculation
+ of sum weights. Generated detailed info for failure */
+ loc += sprintf(errorstr, "Something wrong in choosing new lambda state with a Gibbs move -- probably underflow in weight determination.\nDenominator is: %3d%17.10e\n i dE numerator weights\n", 0, pks);
+ for (ifep = minfep; ifep <= maxfep; ifep++)
+ {
+ loc += sprintf(&errorstr[loc], "%3d %17.10e%17.10e%17.10e\n", ifep, weighted_lamee[ifep], p_k[ifep], dfhist->sum_weights[ifep]);
+ }
+ gmx_fatal(FARGS, errorstr);
+ }
+ }
+ }
+ else if ((expand->elmcmove == elmcmoveMETROPOLIS) || (expand->elmcmove == elmcmoveBARKER))
+ {
+ /* use the metropolis sampler with trial +/- 1 */
+ r1 = gmx_rng_uniform_real(rng);
+ if (r1 < 0.5)
+ {
+ if (fep_state == 0)
+ {
+ lamtrial = fep_state;
+ }
+ else
+ {
+ lamtrial = fep_state-1;
+ }
+ }
+ else
+ {
+ if (fep_state == nlim-1)
+ {
+ lamtrial = fep_state;
+ }
+ else
+ {
+ lamtrial = fep_state+1;
+ }
+ }
+
+ de = weighted_lamee[lamtrial] - weighted_lamee[fep_state];
+ if (expand->elmcmove == elmcmoveMETROPOLIS)
+ {
+ tprob = 1.0;
+ trialprob = exp(de);
+ if (trialprob < tprob)
+ {
+ tprob = trialprob;
+ }
+ propose[fep_state] = 0;
+ propose[lamtrial] = 1.0; /* note that this overwrites the above line if fep_state = ntrial, which only occurs at the ends */
+ accept[fep_state] = 1.0; /* doesn't actually matter, never proposed unless fep_state = ntrial, in which case it's 1.0 anyway */
+ accept[lamtrial] = tprob;
+
+ }
+ else if (expand->elmcmove == elmcmoveBARKER)
+ {
+ tprob = 1.0/(1.0+exp(-de));
+
+ propose[fep_state] = (1-tprob);
+ propose[lamtrial] += tprob; /* we add, to account for the fact that at the end, they might be the same point */
+ accept[fep_state] = 1.0;
+ accept[lamtrial] = 1.0;
+ }
+
+ r2 = gmx_rng_uniform_real(rng);
+ if (r2 < tprob)
+ {
+ lamnew = lamtrial;
+ }
+ else
+ {
+ lamnew = fep_state;
+ }
+ }
+
+ for (ifep = 0; ifep < nlim; ifep++)
+ {
+ dfhist->Tij[fep_state][ifep] += propose[ifep]*accept[ifep];
+ dfhist->Tij[fep_state][fep_state] += propose[ifep]*(1.0-accept[ifep]);
+ }
+ fep_state = lamnew;
+ }
+
+ dfhist->Tij_empirical[starting_fep_state][lamnew] += 1.0;
+
+ sfree(propose);
+ sfree(accept);
+ sfree(remainder);
+
+ return lamnew;
+}
+
+/* print out the weights to the log, along with current state */
+extern void PrintFreeEnergyInfoToFile(FILE *outfile, t_lambda *fep, t_expanded *expand, t_simtemp *simtemp, df_history_t *dfhist,
- if (ifep == nlam)
++ int fep_state, int frequency, gmx_large_int_t step)
+{
+ int nlim, i, ifep, jfep;
+ real dw, dg, dv, dm, Tprint;
+ real *temps;
+ const char *print_names[efptNR] = {" FEPL", "MassL", "CoulL", " VdwL", "BondL", "RestT", "Temp.(K)"};
+ gmx_bool bSimTemp = FALSE;
+
+ nlim = fep->n_lambda;
+ if (simtemp != NULL)
+ {
+ bSimTemp = TRUE;
+ }
+
+ if (mod(step, frequency) == 0)
+ {
+ fprintf(outfile, " MC-lambda information\n");
+ if (EWL(expand->elamstats) && (!(dfhist->bEquil)))
+ {
+ fprintf(outfile, " Wang-Landau incrementor is: %11.5g\n", dfhist->wl_delta);
+ }
+ fprintf(outfile, " N");
+ for (i = 0; i < efptNR; i++)
+ {
+ if (fep->separate_dvdl[i])
+ {
+ fprintf(outfile, "%7s", print_names[i]);
+ }
+ else if ((i == efptTEMPERATURE) && bSimTemp)
+ {
+ fprintf(outfile, "%10s", print_names[i]); /* more space for temperature formats */
+ }
+ }
+ fprintf(outfile, " Count ");
+ if (expand->elamstats == elamstatsMINVAR)
+ {
+ fprintf(outfile, "W(in kT) G(in kT) dG(in kT) dV(in kT)\n");
+ }
+ else
+ {
+ fprintf(outfile, "G(in kT) dG(in kT)\n");
+ }
+ for (ifep = 0; ifep < nlim; ifep++)
+ {
+ if (ifep == nlim-1)
+ {
+ dw = 0.0;
+ dg = 0.0;
+ dv = 0.0;
+ dm = 0.0;
+ }
+ else
+ {
+ dw = dfhist->sum_weights[ifep+1] - dfhist->sum_weights[ifep];
+ dg = dfhist->sum_dg[ifep+1] - dfhist->sum_dg[ifep];
+ dv = sqrt(pow(dfhist->sum_variance[ifep+1], 2) - pow(dfhist->sum_variance[ifep], 2));
+ dm = dfhist->sum_minvar[ifep+1] - dfhist->sum_minvar[ifep];
+
+ }
+ fprintf(outfile, "%3d", (ifep+1));
+ for (i = 0; i < efptNR; i++)
+ {
+ if (fep->separate_dvdl[i])
+ {
+ fprintf(outfile, "%7.3f", fep->all_lambda[i][ifep]);
+ }
+ else if (i == efptTEMPERATURE && bSimTemp)
+ {
+ fprintf(outfile, "%9.3f", simtemp->temperatures[ifep]);
+ }
+ }
+ if (EWL(expand->elamstats) && (!(dfhist->bEquil))) /* if performing WL and still haven't equilibrated */
+ {
+ if (expand->elamstats == elamstatsWL)
+ {
+ fprintf(outfile, " %8d", (int)dfhist->wl_histo[ifep]);
+ }
+ else
+ {
+ fprintf(outfile, " %8.3f", dfhist->wl_histo[ifep]);
+ }
+ }
+ else /* we have equilibrated weights */
+ {
+ fprintf(outfile, " %8d", dfhist->n_at_lam[ifep]);
+ }
+ if (expand->elamstats == elamstatsMINVAR)
+ {
+ fprintf(outfile, " %10.5f %10.5f %10.5f %10.5f", dfhist->sum_weights[ifep], dfhist->sum_dg[ifep], dg, dv);
+ }
+ else
+ {
+ fprintf(outfile, " %10.5f %10.5f", dfhist->sum_weights[ifep], dw);
+ }
- t_state *state, t_extmass *MassQ, df_history_t *dfhist,
++ if (ifep == fep_state)
+ {
+ fprintf(outfile, " <<\n");
+ }
+ else
+ {
+ fprintf(outfile, " \n");
+ }
+ }
+ fprintf(outfile, "\n");
+
+ if ((mod(step, expand->nstTij) == 0) && (expand->nstTij > 0) && (step > 0))
+ {
+ fprintf(outfile, " Transition Matrix\n");
+ for (ifep = 0; ifep < nlim; ifep++)
+ {
+ fprintf(outfile, "%12d", (ifep+1));
+ }
+ fprintf(outfile, "\n");
+ for (ifep = 0; ifep < nlim; ifep++)
+ {
+ for (jfep = 0; jfep < nlim; jfep++)
+ {
+ if (dfhist->n_at_lam[ifep] > 0)
+ {
+ if (expand->bSymmetrizedTMatrix)
+ {
+ Tprint = (dfhist->Tij[ifep][jfep]+dfhist->Tij[jfep][ifep])/(dfhist->n_at_lam[ifep]+dfhist->n_at_lam[jfep]);
+ }
+ else
+ {
+ Tprint = (dfhist->Tij[ifep][jfep])/(dfhist->n_at_lam[ifep]);
+ }
+ }
+ else
+ {
+ Tprint = 0.0;
+ }
+ fprintf(outfile, "%12.8f", Tprint);
+ }
+ fprintf(outfile, "%3d\n", (ifep+1));
+ }
+
+ fprintf(outfile, " Empirical Transition Matrix\n");
+ for (ifep = 0; ifep < nlim; ifep++)
+ {
+ fprintf(outfile, "%12d", (ifep+1));
+ }
+ fprintf(outfile, "\n");
+ for (ifep = 0; ifep < nlim; ifep++)
+ {
+ for (jfep = 0; jfep < nlim; jfep++)
+ {
+ if (dfhist->n_at_lam[ifep] > 0)
+ {
+ if (expand->bSymmetrizedTMatrix)
+ {
+ Tprint = (dfhist->Tij_empirical[ifep][jfep]+dfhist->Tij_empirical[jfep][ifep])/(dfhist->n_at_lam[ifep]+dfhist->n_at_lam[jfep]);
+ }
+ else
+ {
+ Tprint = dfhist->Tij_empirical[ifep][jfep]/(dfhist->n_at_lam[ifep]);
+ }
+ }
+ else
+ {
+ Tprint = 0.0;
+ }
+ fprintf(outfile, "%12.8f", Tprint);
+ }
+ fprintf(outfile, "%3d\n", (ifep+1));
+ }
+ }
+ }
+}
+
+extern void get_mc_state(gmx_rng_t rng, t_state *state)
+{
+ gmx_rng_get_state(rng, state->mc_rng, state->mc_rngi);
+}
+
+extern void set_mc_state(gmx_rng_t rng, t_state *state)
+{
+ gmx_rng_set_state(rng, state->mc_rng, state->mc_rngi[0]);
+}
+
+extern int ExpandedEnsembleDynamics(FILE *log, t_inputrec *ir, gmx_enerdata_t *enerd,
- int i, nlam, nlim, lamnew, totalsamples;
++ t_state *state, t_extmass *MassQ, int fep_state, df_history_t *dfhist,
+ gmx_large_int_t step, gmx_rng_t mcrng,
+ rvec *v, t_mdatoms *mdatoms)
++/* Note that the state variable is only needed for simulated tempering, not
++ Hamiltonian expanded ensemble. May be able to remove it after integrator refactoring. */
+{
+ real *pfep_lamee, *scaled_lamee, *weighted_lamee;
+ double *p_k;
- nlam = state->fep_state;
++ int i, nlim, lamnew, totalsamples;
+ real oneovert, maxscaled = 0, maxweighted = 0;
+ t_expanded *expand;
+ t_simtemp *simtemp;
+ double *temperature_lambdas;
+ gmx_bool bIfReset, bSwitchtoOneOverT, bDoneEquilibrating = FALSE;
+
+ expand = ir->expandedvals;
+ simtemp = ir->simtempvals;
+ nlim = ir->fepvals->n_lambda;
- if (expand->bInit_weights) /* if initialized weights, we need to fill them in */
- {
- dfhist->wl_delta = expand->init_wl_delta; /* MRS -- this would fit better somewhere else? */
- for (i = 0; i < nlim; i++)
- {
- dfhist->sum_weights[i] = expand->init_lambda_weights[i];
- dfhist->sum_dg[i] = expand->init_lambda_weights[i];
- }
- expand->bInit_weights = FALSE;
- }
-
+
+ snew(scaled_lamee, nlim);
+ snew(weighted_lamee, nlim);
+ snew(pfep_lamee, nlim);
+ snew(p_k, nlim);
+
- dfhist->n_at_lam[nlam]++;
+ /* update the count at the current lambda*/
- + enerd->term[F_EPOT]*(1.0/(simtemp->temperatures[i])- 1.0/(simtemp->temperatures[nlam]))/BOLTZ;
++ dfhist->n_at_lam[fep_state]++;
+
+ /* need to calculate the PV term somewhere, but not needed here? Not until there's a lambda state that's
+ pressure controlled.*/
+ /*
+ pVTerm = 0;
+ where does this PV term go?
+ for (i=0;i<nlim;i++)
+ {
+ fep_lamee[i] += pVTerm;
+ }
+ */
+
+ /* determine the minimum value to avoid overflow. Probably a better way to do this */
+ /* we don't need to include the pressure term, since the volume is the same between the two.
+ is there some term we are neglecting, however? */
+
+ if (ir->efep != efepNO)
+ {
+ for (i = 0; i < nlim; i++)
+ {
+ if (ir->bSimTemp)
+ {
+ /* Note -- this assumes no mass changes, since kinetic energy is not added . . . */
+ scaled_lamee[i] = (enerd->enerpart_lambda[i+1]-enerd->enerpart_lambda[0])/(simtemp->temperatures[i]*BOLTZ)
- scaled_lamee[i] = enerd->term[F_EPOT]*(1.0/simtemp->temperatures[i] - 1.0/simtemp->temperatures[nlam])/BOLTZ;
++ + enerd->term[F_EPOT]*(1.0/(simtemp->temperatures[i])- 1.0/(simtemp->temperatures[fep_state]))/BOLTZ;
+ }
+ else
+ {
+ scaled_lamee[i] = (enerd->enerpart_lambda[i+1]-enerd->enerpart_lambda[0])/(expand->mc_temp*BOLTZ);
+ /* mc_temp is currently set to the system reft unless otherwise defined */
+ }
+
+ /* save these energies for printing, so they don't get overwritten by the next step */
+ /* they aren't overwritten in the non-free energy case, but we always print with these
+ for simplicity */
+ }
+ }
+ else
+ {
+ if (ir->bSimTemp)
+ {
+ for (i = 0; i < nlim; i++)
+ {
- bDoneEquilibrating = UpdateWeights(nlim, expand, dfhist, nlam, scaled_lamee, weighted_lamee, step);
++ scaled_lamee[i] = enerd->term[F_EPOT]*(1.0/simtemp->temperatures[i] - 1.0/simtemp->temperatures[fep_state])/BOLTZ;
+ }
+ }
+ }
+
+ for (i = 0; i < nlim; i++)
+ {
+ pfep_lamee[i] = scaled_lamee[i];
+
+ weighted_lamee[i] = dfhist->sum_weights[i] - scaled_lamee[i];
+ if (i == 0)
+ {
+ maxscaled = scaled_lamee[i];
+ maxweighted = weighted_lamee[i];
+ }
+ else
+ {
+ if (scaled_lamee[i] > maxscaled)
+ {
+ maxscaled = scaled_lamee[i];
+ }
+ if (weighted_lamee[i] > maxweighted)
+ {
+ maxweighted = weighted_lamee[i];
+ }
+ }
+ }
+
+ for (i = 0; i < nlim; i++)
+ {
+ scaled_lamee[i] -= maxscaled;
+ weighted_lamee[i] -= maxweighted;
+ }
+
+ /* update weights - we decide whether or not to actually do this inside */
+
- lamnew = ChooseNewLambda(nlim, expand, dfhist, nlam, weighted_lamee, p_k, mcrng);
++ bDoneEquilibrating = UpdateWeights(nlim, expand, dfhist, fep_state, scaled_lamee, weighted_lamee, step);
+ if (bDoneEquilibrating)
+ {
+ if (log)
+ {
+ fprintf(log, "\nStep %d: Weights have equilibrated, using criteria: %s\n", (int)step, elmceq_names[expand->elmceq]);
+ }
+ }
+
- if (ir->bSimTemp && (lamnew != nlam)) /* only need to change the temperatures if we change the state */
++ lamnew = ChooseNewLambda(nlim, expand, dfhist, fep_state, weighted_lamee, p_k, mcrng);
+ /* if using simulated tempering, we need to adjust the temperatures */
++ if (ir->bSimTemp && (lamnew != fep_state)) /* only need to change the temperatures if we change the state */
+ {
+ int i, j, n, d;
+ real *buf_ngtc;
+ real told;
+ int nstart, nend, gt;
+
+ snew(buf_ngtc, ir->opts.ngtc);
+
+ for (i = 0; i < ir->opts.ngtc; i++)
+ {
+ if (ir->opts.ref_t[i] > 0)
+ {
+ told = ir->opts.ref_t[i];
+ ir->opts.ref_t[i] = simtemp->temperatures[lamnew];
+ buf_ngtc[i] = sqrt(ir->opts.ref_t[i]/told); /* using the buffer as temperature scaling */
+ }
+ }
+
+ /* we don't need to manipulate the ekind information, as it isn't due to be reset until the next step anyway */
+
+ nstart = mdatoms->start;
+ nend = nstart + mdatoms->homenr;
+ for (n = nstart; n < nend; n++)
+ {
+ gt = 0;
+ if (mdatoms->cTC)
+ {
+ gt = mdatoms->cTC[n];
+ }
+ for (d = 0; d < DIM; d++)
+ {
+ v[n][d] *= buf_ngtc[gt];
+ }
+ }
+
+ if (IR_NPT_TROTTER(ir) || IR_NPH_TROTTER(ir) || IR_NVT_TROTTER(ir))
+ {
+ /* we need to recalculate the masses if the temperature has changed */
+ init_npt_masses(ir, state, MassQ, FALSE);
+ for (i = 0; i < state->nnhpres; i++)
+ {
+ for (j = 0; j < ir->opts.nhchainlength; j++)
+ {
+ state->nhpres_vxi[i+j] *= buf_ngtc[i];
+ }
+ }
+ for (i = 0; i < ir->opts.ngtc; i++)
+ {
+ for (j = 0; j < ir->opts.nhchainlength; j++)
+ {
+ state->nosehoover_vxi[i+j] *= buf_ngtc[i];
+ }
+ }
+ }
+ sfree(buf_ngtc);
+ }
+
+ /* now check on the Wang-Landau updating critera */
+
+ if (EWL(expand->elamstats))
+ {
+ bSwitchtoOneOverT = FALSE;
+ if (expand->bWLoneovert)
+ {
+ totalsamples = 0;
+ for (i = 0; i < nlim; i++)
+ {
+ totalsamples += dfhist->n_at_lam[i];
+ }
+ oneovert = (1.0*nlim)/totalsamples;
+ /* oneovert has decreasd by a bit since last time, so we actually make sure its within one of this number */
+ /* switch to 1/t incrementing when wl_delta has decreased at least once, and wl_delta is now less than 1/t */
+ if ((dfhist->wl_delta <= ((totalsamples)/(totalsamples-1.00001))*oneovert) &&
+ (dfhist->wl_delta < expand->init_wl_delta))
+ {
+ bSwitchtoOneOverT = TRUE;
+ }
+ }
+ if (bSwitchtoOneOverT)
+ {
+ dfhist->wl_delta = oneovert; /* now we reduce by this each time, instead of only at flatness */
+ }
+ else
+ {
+ bIfReset = CheckHistogramRatios(nlim, dfhist->wl_histo, expand->wl_ratio);
+ if (bIfReset)
+ {
+ for (i = 0; i < nlim; i++)
+ {
+ dfhist->wl_histo[i] = 0;
+ }
+ dfhist->wl_delta *= expand->wl_scale;
+ if (log)
+ {
+ fprintf(log, "\nStep %d: weights are now:", (int)step);
+ for (i = 0; i < nlim; i++)
+ {
+ fprintf(log, " %.5f", dfhist->sum_weights[i]);
+ }
+ fprintf(log, "\n");
+ }
+ }
+ }
+ }
+ sfree(pfep_lamee);
+ sfree(scaled_lamee);
+ sfree(weighted_lamee);
+ sfree(p_k);
+
+ return lamnew;
+}
--- /dev/null
- /* Analytical Ewald exclusion correction is only an option in the
- * x86 SIMD kernel. This is faster in single precision
- * on Bulldozer and slightly faster on Sandy Bridge.
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * VERSION 3.2.0
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2004, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * GROwing Monsters And Cloning Shrimps
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <math.h>
+#include <string.h>
+#include <assert.h>
+#include "sysstuff.h"
+#include "typedefs.h"
+#include "vec.h"
+#include "maths.h"
+#include "macros.h"
+#include "smalloc.h"
+#include "macros.h"
+#include "gmx_fatal.h"
+#include "gmx_fatal_collective.h"
+#include "physics.h"
+#include "force.h"
+#include "tables.h"
+#include "nonbonded.h"
+#include "invblock.h"
+#include "names.h"
+#include "network.h"
+#include "pbc.h"
+#include "ns.h"
+#include "mshift.h"
+#include "txtdump.h"
+#include "coulomb.h"
+#include "md_support.h"
+#include "md_logging.h"
+#include "domdec.h"
+#include "partdec.h"
+#include "qmmm.h"
+#include "copyrite.h"
+#include "mtop_util.h"
+#include "nbnxn_search.h"
+#include "nbnxn_atomdata.h"
+#include "nbnxn_consts.h"
+#include "statutil.h"
+#include "gmx_omp_nthreads.h"
+#include "gmx_detect_hardware.h"
+
+#ifdef _MSC_VER
+/* MSVC definition for __cpuid() */
+#include <intrin.h>
+#endif
+
+#include "types/nbnxn_cuda_types_ext.h"
+#include "gpu_utils.h"
+#include "nbnxn_cuda_data_mgmt.h"
+#include "pmalloc_cuda.h"
+
+t_forcerec *mk_forcerec(void)
+{
+ t_forcerec *fr;
+
+ snew(fr, 1);
+
+ return fr;
+}
+
+#ifdef DEBUG
+static void pr_nbfp(FILE *fp, real *nbfp, gmx_bool bBHAM, int atnr)
+{
+ int i, j;
+
+ for (i = 0; (i < atnr); i++)
+ {
+ for (j = 0; (j < atnr); j++)
+ {
+ fprintf(fp, "%2d - %2d", i, j);
+ if (bBHAM)
+ {
+ fprintf(fp, " a=%10g, b=%10g, c=%10g\n", BHAMA(nbfp, atnr, i, j),
+ BHAMB(nbfp, atnr, i, j), BHAMC(nbfp, atnr, i, j)/6.0);
+ }
+ else
+ {
+ fprintf(fp, " c6=%10g, c12=%10g\n", C6(nbfp, atnr, i, j)/6.0,
+ C12(nbfp, atnr, i, j)/12.0);
+ }
+ }
+ }
+}
+#endif
+
+static real *mk_nbfp(const gmx_ffparams_t *idef, gmx_bool bBHAM)
+{
+ real *nbfp;
+ int i, j, k, atnr;
+
+ atnr = idef->atnr;
+ if (bBHAM)
+ {
+ snew(nbfp, 3*atnr*atnr);
+ for (i = k = 0; (i < atnr); i++)
+ {
+ for (j = 0; (j < atnr); j++, k++)
+ {
+ BHAMA(nbfp, atnr, i, j) = idef->iparams[k].bham.a;
+ BHAMB(nbfp, atnr, i, j) = idef->iparams[k].bham.b;
+ /* nbfp now includes the 6.0 derivative prefactor */
+ BHAMC(nbfp, atnr, i, j) = idef->iparams[k].bham.c*6.0;
+ }
+ }
+ }
+ else
+ {
+ snew(nbfp, 2*atnr*atnr);
+ for (i = k = 0; (i < atnr); i++)
+ {
+ for (j = 0; (j < atnr); j++, k++)
+ {
+ /* nbfp now includes the 6.0/12.0 derivative prefactors */
+ C6(nbfp, atnr, i, j) = idef->iparams[k].lj.c6*6.0;
+ C12(nbfp, atnr, i, j) = idef->iparams[k].lj.c12*12.0;
+ }
+ }
+ }
+
+ return nbfp;
+}
+
+/* This routine sets fr->solvent_opt to the most common solvent in the
+ * system, e.g. esolSPC or esolTIP4P. It will also mark each charge group in
+ * the fr->solvent_type array with the correct type (or esolNO).
+ *
+ * Charge groups that fulfill the conditions but are not identical to the
+ * most common one will be marked as esolNO in the solvent_type array.
+ *
+ * TIP3p is identical to SPC for these purposes, so we call it
+ * SPC in the arrays (Apologies to Bill Jorgensen ;-)
+ *
+ * NOTE: QM particle should not
+ * become an optimized solvent. Not even if there is only one charge
+ * group in the Qm
+ */
+
+typedef struct
+{
+ int model;
+ int count;
+ int vdwtype[4];
+ real charge[4];
+} solvent_parameters_t;
+
+static void
+check_solvent_cg(const gmx_moltype_t *molt,
+ int cg0,
+ int nmol,
+ const unsigned char *qm_grpnr,
+ const t_grps *qm_grps,
+ t_forcerec * fr,
+ int *n_solvent_parameters,
+ solvent_parameters_t **solvent_parameters_p,
+ int cginfo,
+ int *cg_sp)
+{
+ const t_blocka * excl;
+ t_atom *atom;
+ int j, k;
+ int j0, j1, nj;
+ gmx_bool perturbed;
+ gmx_bool has_vdw[4];
+ gmx_bool match;
+ real tmp_charge[4];
+ int tmp_vdwtype[4];
+ int tjA;
+ gmx_bool qm;
+ solvent_parameters_t *solvent_parameters;
+
+ /* We use a list with parameters for each solvent type.
+ * Every time we discover a new molecule that fulfills the basic
+ * conditions for a solvent we compare with the previous entries
+ * in these lists. If the parameters are the same we just increment
+ * the counter for that type, and otherwise we create a new type
+ * based on the current molecule.
+ *
+ * Once we've finished going through all molecules we check which
+ * solvent is most common, and mark all those molecules while we
+ * clear the flag on all others.
+ */
+
+ solvent_parameters = *solvent_parameters_p;
+
+ /* Mark the cg first as non optimized */
+ *cg_sp = -1;
+
+ /* Check if this cg has no exclusions with atoms in other charge groups
+ * and all atoms inside the charge group excluded.
+ * We only have 3 or 4 atom solvent loops.
+ */
+ if (GET_CGINFO_EXCL_INTER(cginfo) ||
+ !GET_CGINFO_EXCL_INTRA(cginfo))
+ {
+ return;
+ }
+
+ /* Get the indices of the first atom in this charge group */
+ j0 = molt->cgs.index[cg0];
+ j1 = molt->cgs.index[cg0+1];
+
+ /* Number of atoms in our molecule */
+ nj = j1 - j0;
+
+ if (debug)
+ {
+ fprintf(debug,
+ "Moltype '%s': there are %d atoms in this charge group\n",
+ *molt->name, nj);
+ }
+
+ /* Check if it could be an SPC (3 atoms) or TIP4p (4) water,
+ * otherwise skip it.
+ */
+ if (nj < 3 || nj > 4)
+ {
+ return;
+ }
+
+ /* Check if we are doing QM on this group */
+ qm = FALSE;
+ if (qm_grpnr != NULL)
+ {
+ for (j = j0; j < j1 && !qm; j++)
+ {
+ qm = (qm_grpnr[j] < qm_grps->nr - 1);
+ }
+ }
+ /* Cannot use solvent optimization with QM */
+ if (qm)
+ {
+ return;
+ }
+
+ atom = molt->atoms.atom;
+
+ /* Still looks like a solvent, time to check parameters */
+
+ /* If it is perturbed (free energy) we can't use the solvent loops,
+ * so then we just skip to the next molecule.
+ */
+ perturbed = FALSE;
+
+ for (j = j0; j < j1 && !perturbed; j++)
+ {
+ perturbed = PERTURBED(atom[j]);
+ }
+
+ if (perturbed)
+ {
+ return;
+ }
+
+ /* Now it's only a question if the VdW and charge parameters
+ * are OK. Before doing the check we compare and see if they are
+ * identical to a possible previous solvent type.
+ * First we assign the current types and charges.
+ */
+ for (j = 0; j < nj; j++)
+ {
+ tmp_vdwtype[j] = atom[j0+j].type;
+ tmp_charge[j] = atom[j0+j].q;
+ }
+
+ /* Does it match any previous solvent type? */
+ for (k = 0; k < *n_solvent_parameters; k++)
+ {
+ match = TRUE;
+
+
+ /* We can only match SPC with 3 atoms and TIP4p with 4 atoms */
+ if ( (solvent_parameters[k].model == esolSPC && nj != 3) ||
+ (solvent_parameters[k].model == esolTIP4P && nj != 4) )
+ {
+ match = FALSE;
+ }
+
+ /* Check that types & charges match for all atoms in molecule */
+ for (j = 0; j < nj && match == TRUE; j++)
+ {
+ if (tmp_vdwtype[j] != solvent_parameters[k].vdwtype[j])
+ {
+ match = FALSE;
+ }
+ if (tmp_charge[j] != solvent_parameters[k].charge[j])
+ {
+ match = FALSE;
+ }
+ }
+ if (match == TRUE)
+ {
+ /* Congratulations! We have a matched solvent.
+ * Flag it with this type for later processing.
+ */
+ *cg_sp = k;
+ solvent_parameters[k].count += nmol;
+
+ /* We are done with this charge group */
+ return;
+ }
+ }
+
+ /* If we get here, we have a tentative new solvent type.
+ * Before we add it we must check that it fulfills the requirements
+ * of the solvent optimized loops. First determine which atoms have
+ * VdW interactions.
+ */
+ for (j = 0; j < nj; j++)
+ {
+ has_vdw[j] = FALSE;
+ tjA = tmp_vdwtype[j];
+
+ /* Go through all other tpes and see if any have non-zero
+ * VdW parameters when combined with this one.
+ */
+ for (k = 0; k < fr->ntype && (has_vdw[j] == FALSE); k++)
+ {
+ /* We already checked that the atoms weren't perturbed,
+ * so we only need to check state A now.
+ */
+ if (fr->bBHAM)
+ {
+ has_vdw[j] = (has_vdw[j] ||
+ (BHAMA(fr->nbfp, fr->ntype, tjA, k) != 0.0) ||
+ (BHAMB(fr->nbfp, fr->ntype, tjA, k) != 0.0) ||
+ (BHAMC(fr->nbfp, fr->ntype, tjA, k) != 0.0));
+ }
+ else
+ {
+ /* Standard LJ */
+ has_vdw[j] = (has_vdw[j] ||
+ (C6(fr->nbfp, fr->ntype, tjA, k) != 0.0) ||
+ (C12(fr->nbfp, fr->ntype, tjA, k) != 0.0));
+ }
+ }
+ }
+
+ /* Now we know all we need to make the final check and assignment. */
+ if (nj == 3)
+ {
+ /* So, is it an SPC?
+ * For this we require thatn all atoms have charge,
+ * the charges on atom 2 & 3 should be the same, and only
+ * atom 1 might have VdW.
+ */
+ if (has_vdw[1] == FALSE &&
+ has_vdw[2] == FALSE &&
+ tmp_charge[0] != 0 &&
+ tmp_charge[1] != 0 &&
+ tmp_charge[2] == tmp_charge[1])
+ {
+ srenew(solvent_parameters, *n_solvent_parameters+1);
+ solvent_parameters[*n_solvent_parameters].model = esolSPC;
+ solvent_parameters[*n_solvent_parameters].count = nmol;
+ for (k = 0; k < 3; k++)
+ {
+ solvent_parameters[*n_solvent_parameters].vdwtype[k] = tmp_vdwtype[k];
+ solvent_parameters[*n_solvent_parameters].charge[k] = tmp_charge[k];
+ }
+
+ *cg_sp = *n_solvent_parameters;
+ (*n_solvent_parameters)++;
+ }
+ }
+ else if (nj == 4)
+ {
+ /* Or could it be a TIP4P?
+ * For this we require thatn atoms 2,3,4 have charge, but not atom 1.
+ * Only atom 1 mght have VdW.
+ */
+ if (has_vdw[1] == FALSE &&
+ has_vdw[2] == FALSE &&
+ has_vdw[3] == FALSE &&
+ tmp_charge[0] == 0 &&
+ tmp_charge[1] != 0 &&
+ tmp_charge[2] == tmp_charge[1] &&
+ tmp_charge[3] != 0)
+ {
+ srenew(solvent_parameters, *n_solvent_parameters+1);
+ solvent_parameters[*n_solvent_parameters].model = esolTIP4P;
+ solvent_parameters[*n_solvent_parameters].count = nmol;
+ for (k = 0; k < 4; k++)
+ {
+ solvent_parameters[*n_solvent_parameters].vdwtype[k] = tmp_vdwtype[k];
+ solvent_parameters[*n_solvent_parameters].charge[k] = tmp_charge[k];
+ }
+
+ *cg_sp = *n_solvent_parameters;
+ (*n_solvent_parameters)++;
+ }
+ }
+
+ *solvent_parameters_p = solvent_parameters;
+}
+
+static void
+check_solvent(FILE * fp,
+ const gmx_mtop_t * mtop,
+ t_forcerec * fr,
+ cginfo_mb_t *cginfo_mb)
+{
+ const t_block * cgs;
+ const t_block * mols;
+ const gmx_moltype_t *molt;
+ int mb, mol, cg_mol, at_offset, cg_offset, am, cgm, i, nmol_ch, nmol;
+ int n_solvent_parameters;
+ solvent_parameters_t *solvent_parameters;
+ int **cg_sp;
+ int bestsp, bestsol;
+
+ if (debug)
+ {
+ fprintf(debug, "Going to determine what solvent types we have.\n");
+ }
+
+ mols = &mtop->mols;
+
+ n_solvent_parameters = 0;
+ solvent_parameters = NULL;
+ /* Allocate temporary array for solvent type */
+ snew(cg_sp, mtop->nmolblock);
+
+ cg_offset = 0;
+ at_offset = 0;
+ for (mb = 0; mb < mtop->nmolblock; mb++)
+ {
+ molt = &mtop->moltype[mtop->molblock[mb].type];
+ cgs = &molt->cgs;
+ /* Here we have to loop over all individual molecules
+ * because we need to check for QMMM particles.
+ */
+ snew(cg_sp[mb], cginfo_mb[mb].cg_mod);
+ nmol_ch = cginfo_mb[mb].cg_mod/cgs->nr;
+ nmol = mtop->molblock[mb].nmol/nmol_ch;
+ for (mol = 0; mol < nmol_ch; mol++)
+ {
+ cgm = mol*cgs->nr;
+ am = mol*cgs->index[cgs->nr];
+ for (cg_mol = 0; cg_mol < cgs->nr; cg_mol++)
+ {
+ check_solvent_cg(molt, cg_mol, nmol,
+ mtop->groups.grpnr[egcQMMM] ?
+ mtop->groups.grpnr[egcQMMM]+at_offset+am : 0,
+ &mtop->groups.grps[egcQMMM],
+ fr,
+ &n_solvent_parameters, &solvent_parameters,
+ cginfo_mb[mb].cginfo[cgm+cg_mol],
+ &cg_sp[mb][cgm+cg_mol]);
+ }
+ }
+ cg_offset += cgs->nr;
+ at_offset += cgs->index[cgs->nr];
+ }
+
+ /* Puh! We finished going through all charge groups.
+ * Now find the most common solvent model.
+ */
+
+ /* Most common solvent this far */
+ bestsp = -2;
+ for (i = 0; i < n_solvent_parameters; i++)
+ {
+ if (bestsp == -2 ||
+ solvent_parameters[i].count > solvent_parameters[bestsp].count)
+ {
+ bestsp = i;
+ }
+ }
+
+ if (bestsp >= 0)
+ {
+ bestsol = solvent_parameters[bestsp].model;
+ }
+ else
+ {
+ bestsol = esolNO;
+ }
+
+#ifdef DISABLE_WATER_NLIST
+ bestsol = esolNO;
+#endif
+
+ fr->nWatMol = 0;
+ for (mb = 0; mb < mtop->nmolblock; mb++)
+ {
+ cgs = &mtop->moltype[mtop->molblock[mb].type].cgs;
+ nmol = (mtop->molblock[mb].nmol*cgs->nr)/cginfo_mb[mb].cg_mod;
+ for (i = 0; i < cginfo_mb[mb].cg_mod; i++)
+ {
+ if (cg_sp[mb][i] == bestsp)
+ {
+ SET_CGINFO_SOLOPT(cginfo_mb[mb].cginfo[i], bestsol);
+ fr->nWatMol += nmol;
+ }
+ else
+ {
+ SET_CGINFO_SOLOPT(cginfo_mb[mb].cginfo[i], esolNO);
+ }
+ }
+ sfree(cg_sp[mb]);
+ }
+ sfree(cg_sp);
+
+ if (bestsol != esolNO && fp != NULL)
+ {
+ fprintf(fp, "\nEnabling %s-like water optimization for %d molecules.\n\n",
+ esol_names[bestsol],
+ solvent_parameters[bestsp].count);
+ }
+
+ sfree(solvent_parameters);
+ fr->solvent_opt = bestsol;
+}
+
+enum {
+ acNONE = 0, acCONSTRAINT, acSETTLE
+};
+
+static cginfo_mb_t *init_cginfo_mb(FILE *fplog, const gmx_mtop_t *mtop,
+ t_forcerec *fr, gmx_bool bNoSolvOpt,
+ gmx_bool *bExcl_IntraCGAll_InterCGNone)
+{
+ const t_block *cgs;
+ const t_blocka *excl;
+ const gmx_moltype_t *molt;
+ const gmx_molblock_t *molb;
+ cginfo_mb_t *cginfo_mb;
+ gmx_bool *type_VDW;
+ int *cginfo;
+ int cg_offset, a_offset, cgm, am;
+ int mb, m, ncg_tot, cg, a0, a1, gid, ai, j, aj, excl_nalloc;
+ int *a_con;
+ int ftype;
+ int ia;
+ gmx_bool bId, *bExcl, bExclIntraAll, bExclInter, bHaveVDW, bHaveQ;
+
+ ncg_tot = ncg_mtop(mtop);
+ snew(cginfo_mb, mtop->nmolblock);
+
+ snew(type_VDW, fr->ntype);
+ for (ai = 0; ai < fr->ntype; ai++)
+ {
+ type_VDW[ai] = FALSE;
+ for (j = 0; j < fr->ntype; j++)
+ {
+ type_VDW[ai] = type_VDW[ai] ||
+ fr->bBHAM ||
+ C6(fr->nbfp, fr->ntype, ai, j) != 0 ||
+ C12(fr->nbfp, fr->ntype, ai, j) != 0;
+ }
+ }
+
+ *bExcl_IntraCGAll_InterCGNone = TRUE;
+
+ excl_nalloc = 10;
+ snew(bExcl, excl_nalloc);
+ cg_offset = 0;
+ a_offset = 0;
+ for (mb = 0; mb < mtop->nmolblock; mb++)
+ {
+ molb = &mtop->molblock[mb];
+ molt = &mtop->moltype[molb->type];
+ cgs = &molt->cgs;
+ excl = &molt->excls;
+
+ /* Check if the cginfo is identical for all molecules in this block.
+ * If so, we only need an array of the size of one molecule.
+ * Otherwise we make an array of #mol times #cgs per molecule.
+ */
+ bId = TRUE;
+ am = 0;
+ for (m = 0; m < molb->nmol; m++)
+ {
+ am = m*cgs->index[cgs->nr];
+ for (cg = 0; cg < cgs->nr; cg++)
+ {
+ a0 = cgs->index[cg];
+ a1 = cgs->index[cg+1];
+ if (ggrpnr(&mtop->groups, egcENER, a_offset+am+a0) !=
+ ggrpnr(&mtop->groups, egcENER, a_offset +a0))
+ {
+ bId = FALSE;
+ }
+ if (mtop->groups.grpnr[egcQMMM] != NULL)
+ {
+ for (ai = a0; ai < a1; ai++)
+ {
+ if (mtop->groups.grpnr[egcQMMM][a_offset+am+ai] !=
+ mtop->groups.grpnr[egcQMMM][a_offset +ai])
+ {
+ bId = FALSE;
+ }
+ }
+ }
+ }
+ }
+
+ cginfo_mb[mb].cg_start = cg_offset;
+ cginfo_mb[mb].cg_end = cg_offset + molb->nmol*cgs->nr;
+ cginfo_mb[mb].cg_mod = (bId ? 1 : molb->nmol)*cgs->nr;
+ snew(cginfo_mb[mb].cginfo, cginfo_mb[mb].cg_mod);
+ cginfo = cginfo_mb[mb].cginfo;
+
+ /* Set constraints flags for constrained atoms */
+ snew(a_con, molt->atoms.nr);
+ for (ftype = 0; ftype < F_NRE; ftype++)
+ {
+ if (interaction_function[ftype].flags & IF_CONSTRAINT)
+ {
+ int nral;
+
+ nral = NRAL(ftype);
+ for (ia = 0; ia < molt->ilist[ftype].nr; ia += 1+nral)
+ {
+ int a;
+
+ for (a = 0; a < nral; a++)
+ {
+ a_con[molt->ilist[ftype].iatoms[ia+1+a]] =
+ (ftype == F_SETTLE ? acSETTLE : acCONSTRAINT);
+ }
+ }
+ }
+ }
+
+ for (m = 0; m < (bId ? 1 : molb->nmol); m++)
+ {
+ cgm = m*cgs->nr;
+ am = m*cgs->index[cgs->nr];
+ for (cg = 0; cg < cgs->nr; cg++)
+ {
+ a0 = cgs->index[cg];
+ a1 = cgs->index[cg+1];
+
+ /* Store the energy group in cginfo */
+ gid = ggrpnr(&mtop->groups, egcENER, a_offset+am+a0);
+ SET_CGINFO_GID(cginfo[cgm+cg], gid);
+
+ /* Check the intra/inter charge group exclusions */
+ if (a1-a0 > excl_nalloc)
+ {
+ excl_nalloc = a1 - a0;
+ srenew(bExcl, excl_nalloc);
+ }
+ /* bExclIntraAll: all intra cg interactions excluded
+ * bExclInter: any inter cg interactions excluded
+ */
+ bExclIntraAll = TRUE;
+ bExclInter = FALSE;
+ bHaveVDW = FALSE;
+ bHaveQ = FALSE;
+ for (ai = a0; ai < a1; ai++)
+ {
+ /* Check VDW and electrostatic interactions */
+ bHaveVDW = bHaveVDW || (type_VDW[molt->atoms.atom[ai].type] ||
+ type_VDW[molt->atoms.atom[ai].typeB]);
+ bHaveQ = bHaveQ || (molt->atoms.atom[ai].q != 0 ||
+ molt->atoms.atom[ai].qB != 0);
+
+ /* Clear the exclusion list for atom ai */
+ for (aj = a0; aj < a1; aj++)
+ {
+ bExcl[aj-a0] = FALSE;
+ }
+ /* Loop over all the exclusions of atom ai */
+ for (j = excl->index[ai]; j < excl->index[ai+1]; j++)
+ {
+ aj = excl->a[j];
+ if (aj < a0 || aj >= a1)
+ {
+ bExclInter = TRUE;
+ }
+ else
+ {
+ bExcl[aj-a0] = TRUE;
+ }
+ }
+ /* Check if ai excludes a0 to a1 */
+ for (aj = a0; aj < a1; aj++)
+ {
+ if (!bExcl[aj-a0])
+ {
+ bExclIntraAll = FALSE;
+ }
+ }
+
+ switch (a_con[ai])
+ {
+ case acCONSTRAINT:
+ SET_CGINFO_CONSTR(cginfo[cgm+cg]);
+ break;
+ case acSETTLE:
+ SET_CGINFO_SETTLE(cginfo[cgm+cg]);
+ break;
+ default:
+ break;
+ }
+ }
+ if (bExclIntraAll)
+ {
+ SET_CGINFO_EXCL_INTRA(cginfo[cgm+cg]);
+ }
+ if (bExclInter)
+ {
+ SET_CGINFO_EXCL_INTER(cginfo[cgm+cg]);
+ }
+ if (a1 - a0 > MAX_CHARGEGROUP_SIZE)
+ {
+ /* The size in cginfo is currently only read with DD */
+ gmx_fatal(FARGS, "A charge group has size %d which is larger than the limit of %d atoms", a1-a0, MAX_CHARGEGROUP_SIZE);
+ }
+ if (bHaveVDW)
+ {
+ SET_CGINFO_HAS_VDW(cginfo[cgm+cg]);
+ }
+ if (bHaveQ)
+ {
+ SET_CGINFO_HAS_Q(cginfo[cgm+cg]);
+ }
+ /* Store the charge group size */
+ SET_CGINFO_NATOMS(cginfo[cgm+cg], a1-a0);
+
+ if (!bExclIntraAll || bExclInter)
+ {
+ *bExcl_IntraCGAll_InterCGNone = FALSE;
+ }
+ }
+ }
+
+ sfree(a_con);
+
+ cg_offset += molb->nmol*cgs->nr;
+ a_offset += molb->nmol*cgs->index[cgs->nr];
+ }
+ sfree(bExcl);
+
+ /* the solvent optimizer is called after the QM is initialized,
+ * because we don't want to have the QM subsystemto become an
+ * optimized solvent
+ */
+
+ check_solvent(fplog, mtop, fr, cginfo_mb);
+
+ if (getenv("GMX_NO_SOLV_OPT"))
+ {
+ if (fplog)
+ {
+ fprintf(fplog, "Found environment variable GMX_NO_SOLV_OPT.\n"
+ "Disabling all solvent optimization\n");
+ }
+ fr->solvent_opt = esolNO;
+ }
+ if (bNoSolvOpt)
+ {
+ fr->solvent_opt = esolNO;
+ }
+ if (!fr->solvent_opt)
+ {
+ for (mb = 0; mb < mtop->nmolblock; mb++)
+ {
+ for (cg = 0; cg < cginfo_mb[mb].cg_mod; cg++)
+ {
+ SET_CGINFO_SOLOPT(cginfo_mb[mb].cginfo[cg], esolNO);
+ }
+ }
+ }
+
+ return cginfo_mb;
+}
+
+static int *cginfo_expand(int nmb, cginfo_mb_t *cgi_mb)
+{
+ int ncg, mb, cg;
+ int *cginfo;
+
+ ncg = cgi_mb[nmb-1].cg_end;
+ snew(cginfo, ncg);
+ mb = 0;
+ for (cg = 0; cg < ncg; cg++)
+ {
+ while (cg >= cgi_mb[mb].cg_end)
+ {
+ mb++;
+ }
+ cginfo[cg] =
+ cgi_mb[mb].cginfo[(cg - cgi_mb[mb].cg_start) % cgi_mb[mb].cg_mod];
+ }
+
+ return cginfo;
+}
+
+static void set_chargesum(FILE *log, t_forcerec *fr, const gmx_mtop_t *mtop)
+{
+ double qsum, q2sum, q;
+ int mb, nmol, i;
+ const t_atoms *atoms;
+
+ qsum = 0;
+ q2sum = 0;
+ for (mb = 0; mb < mtop->nmolblock; mb++)
+ {
+ nmol = mtop->molblock[mb].nmol;
+ atoms = &mtop->moltype[mtop->molblock[mb].type].atoms;
+ for (i = 0; i < atoms->nr; i++)
+ {
+ q = atoms->atom[i].q;
+ qsum += nmol*q;
+ q2sum += nmol*q*q;
+ }
+ }
+ fr->qsum[0] = qsum;
+ fr->q2sum[0] = q2sum;
+ if (fr->efep != efepNO)
+ {
+ qsum = 0;
+ q2sum = 0;
+ for (mb = 0; mb < mtop->nmolblock; mb++)
+ {
+ nmol = mtop->molblock[mb].nmol;
+ atoms = &mtop->moltype[mtop->molblock[mb].type].atoms;
+ for (i = 0; i < atoms->nr; i++)
+ {
+ q = atoms->atom[i].qB;
+ qsum += nmol*q;
+ q2sum += nmol*q*q;
+ }
+ fr->qsum[1] = qsum;
+ fr->q2sum[1] = q2sum;
+ }
+ }
+ else
+ {
+ fr->qsum[1] = fr->qsum[0];
+ fr->q2sum[1] = fr->q2sum[0];
+ }
+ if (log)
+ {
+ if (fr->efep == efepNO)
+ {
+ fprintf(log, "System total charge: %.3f\n", fr->qsum[0]);
+ }
+ else
+ {
+ fprintf(log, "System total charge, top. A: %.3f top. B: %.3f\n",
+ fr->qsum[0], fr->qsum[1]);
+ }
+ }
+}
+
+void update_forcerec(t_forcerec *fr, matrix box)
+{
+ if (fr->eeltype == eelGRF)
+ {
+ calc_rffac(NULL, fr->eeltype, fr->epsilon_r, fr->epsilon_rf,
+ fr->rcoulomb, fr->temp, fr->zsquare, box,
+ &fr->kappa, &fr->k_rf, &fr->c_rf);
+ }
+}
+
+void set_avcsixtwelve(FILE *fplog, t_forcerec *fr, const gmx_mtop_t *mtop)
+{
+ const t_atoms *atoms, *atoms_tpi;
+ const t_blocka *excl;
+ int mb, nmol, nmolc, i, j, tpi, tpj, j1, j2, k, n, nexcl, q;
+#if (defined SIZEOF_LONG_LONG_INT) && (SIZEOF_LONG_LONG_INT >= 8)
+ long long int npair, npair_ij, tmpi, tmpj;
+#else
+ double npair, npair_ij, tmpi, tmpj;
+#endif
+ double csix, ctwelve;
+ int ntp, *typecount;
+ gmx_bool bBHAM;
+ real *nbfp;
+
+ ntp = fr->ntype;
+ bBHAM = fr->bBHAM;
+ nbfp = fr->nbfp;
+
+ for (q = 0; q < (fr->efep == efepNO ? 1 : 2); q++)
+ {
+ csix = 0;
+ ctwelve = 0;
+ npair = 0;
+ nexcl = 0;
+ if (!fr->n_tpi)
+ {
+ /* Count the types so we avoid natoms^2 operations */
+ snew(typecount, ntp);
+ for (mb = 0; mb < mtop->nmolblock; mb++)
+ {
+ nmol = mtop->molblock[mb].nmol;
+ atoms = &mtop->moltype[mtop->molblock[mb].type].atoms;
+ for (i = 0; i < atoms->nr; i++)
+ {
+ if (q == 0)
+ {
+ tpi = atoms->atom[i].type;
+ }
+ else
+ {
+ tpi = atoms->atom[i].typeB;
+ }
+ typecount[tpi] += nmol;
+ }
+ }
+ for (tpi = 0; tpi < ntp; tpi++)
+ {
+ for (tpj = tpi; tpj < ntp; tpj++)
+ {
+ tmpi = typecount[tpi];
+ tmpj = typecount[tpj];
+ if (tpi != tpj)
+ {
+ npair_ij = tmpi*tmpj;
+ }
+ else
+ {
+ npair_ij = tmpi*(tmpi - 1)/2;
+ }
+ if (bBHAM)
+ {
+ /* nbfp now includes the 6.0 derivative prefactor */
+ csix += npair_ij*BHAMC(nbfp, ntp, tpi, tpj)/6.0;
+ }
+ else
+ {
+ /* nbfp now includes the 6.0/12.0 derivative prefactors */
+ csix += npair_ij* C6(nbfp, ntp, tpi, tpj)/6.0;
+ ctwelve += npair_ij* C12(nbfp, ntp, tpi, tpj)/12.0;
+ }
+ npair += npair_ij;
+ }
+ }
+ sfree(typecount);
+ /* Subtract the excluded pairs.
+ * The main reason for substracting exclusions is that in some cases
+ * some combinations might never occur and the parameters could have
+ * any value. These unused values should not influence the dispersion
+ * correction.
+ */
+ for (mb = 0; mb < mtop->nmolblock; mb++)
+ {
+ nmol = mtop->molblock[mb].nmol;
+ atoms = &mtop->moltype[mtop->molblock[mb].type].atoms;
+ excl = &mtop->moltype[mtop->molblock[mb].type].excls;
+ for (i = 0; (i < atoms->nr); i++)
+ {
+ if (q == 0)
+ {
+ tpi = atoms->atom[i].type;
+ }
+ else
+ {
+ tpi = atoms->atom[i].typeB;
+ }
+ j1 = excl->index[i];
+ j2 = excl->index[i+1];
+ for (j = j1; j < j2; j++)
+ {
+ k = excl->a[j];
+ if (k > i)
+ {
+ if (q == 0)
+ {
+ tpj = atoms->atom[k].type;
+ }
+ else
+ {
+ tpj = atoms->atom[k].typeB;
+ }
+ if (bBHAM)
+ {
+ /* nbfp now includes the 6.0 derivative prefactor */
+ csix -= nmol*BHAMC(nbfp, ntp, tpi, tpj)/6.0;
+ }
+ else
+ {
+ /* nbfp now includes the 6.0/12.0 derivative prefactors */
+ csix -= nmol*C6 (nbfp, ntp, tpi, tpj)/6.0;
+ ctwelve -= nmol*C12(nbfp, ntp, tpi, tpj)/12.0;
+ }
+ nexcl += nmol;
+ }
+ }
+ }
+ }
+ }
+ else
+ {
+ /* Only correct for the interaction of the test particle
+ * with the rest of the system.
+ */
+ atoms_tpi =
+ &mtop->moltype[mtop->molblock[mtop->nmolblock-1].type].atoms;
+
+ npair = 0;
+ for (mb = 0; mb < mtop->nmolblock; mb++)
+ {
+ nmol = mtop->molblock[mb].nmol;
+ atoms = &mtop->moltype[mtop->molblock[mb].type].atoms;
+ for (j = 0; j < atoms->nr; j++)
+ {
+ nmolc = nmol;
+ /* Remove the interaction of the test charge group
+ * with itself.
+ */
+ if (mb == mtop->nmolblock-1)
+ {
+ nmolc--;
+
+ if (mb == 0 && nmol == 1)
+ {
+ gmx_fatal(FARGS, "Old format tpr with TPI, please generate a new tpr file");
+ }
+ }
+ if (q == 0)
+ {
+ tpj = atoms->atom[j].type;
+ }
+ else
+ {
+ tpj = atoms->atom[j].typeB;
+ }
+ for (i = 0; i < fr->n_tpi; i++)
+ {
+ if (q == 0)
+ {
+ tpi = atoms_tpi->atom[i].type;
+ }
+ else
+ {
+ tpi = atoms_tpi->atom[i].typeB;
+ }
+ if (bBHAM)
+ {
+ /* nbfp now includes the 6.0 derivative prefactor */
+ csix += nmolc*BHAMC(nbfp, ntp, tpi, tpj)/6.0;
+ }
+ else
+ {
+ /* nbfp now includes the 6.0/12.0 derivative prefactors */
+ csix += nmolc*C6 (nbfp, ntp, tpi, tpj)/6.0;
+ ctwelve += nmolc*C12(nbfp, ntp, tpi, tpj)/12.0;
+ }
+ npair += nmolc;
+ }
+ }
+ }
+ }
+ if (npair - nexcl <= 0 && fplog)
+ {
+ fprintf(fplog, "\nWARNING: There are no atom pairs for dispersion correction\n\n");
+ csix = 0;
+ ctwelve = 0;
+ }
+ else
+ {
+ csix /= npair - nexcl;
+ ctwelve /= npair - nexcl;
+ }
+ if (debug)
+ {
+ fprintf(debug, "Counted %d exclusions\n", nexcl);
+ fprintf(debug, "Average C6 parameter is: %10g\n", (double)csix);
+ fprintf(debug, "Average C12 parameter is: %10g\n", (double)ctwelve);
+ }
+ fr->avcsix[q] = csix;
+ fr->avctwelve[q] = ctwelve;
+ }
+ if (fplog != NULL)
+ {
+ if (fr->eDispCorr == edispcAllEner ||
+ fr->eDispCorr == edispcAllEnerPres)
+ {
+ fprintf(fplog, "Long Range LJ corr.: <C6> %10.4e, <C12> %10.4e\n",
+ fr->avcsix[0], fr->avctwelve[0]);
+ }
+ else
+ {
+ fprintf(fplog, "Long Range LJ corr.: <C6> %10.4e\n", fr->avcsix[0]);
+ }
+ }
+}
+
+
+static void set_bham_b_max(FILE *fplog, t_forcerec *fr,
+ const gmx_mtop_t *mtop)
+{
+ const t_atoms *at1, *at2;
+ int mt1, mt2, i, j, tpi, tpj, ntypes;
+ real b, bmin;
+ real *nbfp;
+
+ if (fplog)
+ {
+ fprintf(fplog, "Determining largest Buckingham b parameter for table\n");
+ }
+ nbfp = fr->nbfp;
+ ntypes = fr->ntype;
+
+ bmin = -1;
+ fr->bham_b_max = 0;
+ for (mt1 = 0; mt1 < mtop->nmoltype; mt1++)
+ {
+ at1 = &mtop->moltype[mt1].atoms;
+ for (i = 0; (i < at1->nr); i++)
+ {
+ tpi = at1->atom[i].type;
+ if (tpi >= ntypes)
+ {
+ gmx_fatal(FARGS, "Atomtype[%d] = %d, maximum = %d", i, tpi, ntypes);
+ }
+
+ for (mt2 = mt1; mt2 < mtop->nmoltype; mt2++)
+ {
+ at2 = &mtop->moltype[mt2].atoms;
+ for (j = 0; (j < at2->nr); j++)
+ {
+ tpj = at2->atom[j].type;
+ if (tpj >= ntypes)
+ {
+ gmx_fatal(FARGS, "Atomtype[%d] = %d, maximum = %d", j, tpj, ntypes);
+ }
+ b = BHAMB(nbfp, ntypes, tpi, tpj);
+ if (b > fr->bham_b_max)
+ {
+ fr->bham_b_max = b;
+ }
+ if ((b < bmin) || (bmin == -1))
+ {
+ bmin = b;
+ }
+ }
+ }
+ }
+ }
+ if (fplog)
+ {
+ fprintf(fplog, "Buckingham b parameters, min: %g, max: %g\n",
+ bmin, fr->bham_b_max);
+ }
+}
+
+static void make_nbf_tables(FILE *fp, const output_env_t oenv,
+ t_forcerec *fr, real rtab,
+ const t_commrec *cr,
+ const char *tabfn, char *eg1, char *eg2,
+ t_nblists *nbl)
+{
+ char buf[STRLEN];
+ int i, j;
+
+ if (tabfn == NULL)
+ {
+ if (debug)
+ {
+ fprintf(debug, "No table file name passed, can not read table, can not do non-bonded interactions\n");
+ }
+ return;
+ }
+
+ sprintf(buf, "%s", tabfn);
+ if (eg1 && eg2)
+ {
+ /* Append the two energy group names */
+ sprintf(buf + strlen(tabfn) - strlen(ftp2ext(efXVG)) - 1, "_%s_%s.%s",
+ eg1, eg2, ftp2ext(efXVG));
+ }
+ nbl->table_elec_vdw = make_tables(fp, oenv, fr, MASTER(cr), buf, rtab, 0);
+ /* Copy the contents of the table to separate coulomb and LJ tables too,
+ * to improve cache performance.
+ */
+ /* For performance reasons we want
+ * the table data to be aligned to 16-byte. The pointers could be freed
+ * but currently aren't.
+ */
+ nbl->table_elec.interaction = GMX_TABLE_INTERACTION_ELEC;
+ nbl->table_elec.format = nbl->table_elec_vdw.format;
+ nbl->table_elec.r = nbl->table_elec_vdw.r;
+ nbl->table_elec.n = nbl->table_elec_vdw.n;
+ nbl->table_elec.scale = nbl->table_elec_vdw.scale;
+ nbl->table_elec.scale_exp = nbl->table_elec_vdw.scale_exp;
+ nbl->table_elec.formatsize = nbl->table_elec_vdw.formatsize;
+ nbl->table_elec.ninteractions = 1;
+ nbl->table_elec.stride = nbl->table_elec.formatsize * nbl->table_elec.ninteractions;
+ snew_aligned(nbl->table_elec.data, nbl->table_elec.stride*(nbl->table_elec.n+1), 32);
+
+ nbl->table_vdw.interaction = GMX_TABLE_INTERACTION_VDWREP_VDWDISP;
+ nbl->table_vdw.format = nbl->table_elec_vdw.format;
+ nbl->table_vdw.r = nbl->table_elec_vdw.r;
+ nbl->table_vdw.n = nbl->table_elec_vdw.n;
+ nbl->table_vdw.scale = nbl->table_elec_vdw.scale;
+ nbl->table_vdw.scale_exp = nbl->table_elec_vdw.scale_exp;
+ nbl->table_vdw.formatsize = nbl->table_elec_vdw.formatsize;
+ nbl->table_vdw.ninteractions = 2;
+ nbl->table_vdw.stride = nbl->table_vdw.formatsize * nbl->table_vdw.ninteractions;
+ snew_aligned(nbl->table_vdw.data, nbl->table_vdw.stride*(nbl->table_vdw.n+1), 32);
+
+ for (i = 0; i <= nbl->table_elec_vdw.n; i++)
+ {
+ for (j = 0; j < 4; j++)
+ {
+ nbl->table_elec.data[4*i+j] = nbl->table_elec_vdw.data[12*i+j];
+ }
+ for (j = 0; j < 8; j++)
+ {
+ nbl->table_vdw.data[8*i+j] = nbl->table_elec_vdw.data[12*i+4+j];
+ }
+ }
+}
+
+static void count_tables(int ftype1, int ftype2, const gmx_mtop_t *mtop,
+ int *ncount, int **count)
+{
+ const gmx_moltype_t *molt;
+ const t_ilist *il;
+ int mt, ftype, stride, i, j, tabnr;
+
+ for (mt = 0; mt < mtop->nmoltype; mt++)
+ {
+ molt = &mtop->moltype[mt];
+ for (ftype = 0; ftype < F_NRE; ftype++)
+ {
+ if (ftype == ftype1 || ftype == ftype2)
+ {
+ il = &molt->ilist[ftype];
+ stride = 1 + NRAL(ftype);
+ for (i = 0; i < il->nr; i += stride)
+ {
+ tabnr = mtop->ffparams.iparams[il->iatoms[i]].tab.table;
+ if (tabnr < 0)
+ {
+ gmx_fatal(FARGS, "A bonded table number is smaller than 0: %d\n", tabnr);
+ }
+ if (tabnr >= *ncount)
+ {
+ srenew(*count, tabnr+1);
+ for (j = *ncount; j < tabnr+1; j++)
+ {
+ (*count)[j] = 0;
+ }
+ *ncount = tabnr+1;
+ }
+ (*count)[tabnr]++;
+ }
+ }
+ }
+ }
+}
+
+static bondedtable_t *make_bonded_tables(FILE *fplog,
+ int ftype1, int ftype2,
+ const gmx_mtop_t *mtop,
+ const char *basefn, const char *tabext)
+{
+ int i, ncount, *count;
+ char tabfn[STRLEN];
+ bondedtable_t *tab;
+
+ tab = NULL;
+
+ ncount = 0;
+ count = NULL;
+ count_tables(ftype1, ftype2, mtop, &ncount, &count);
+
+ if (ncount > 0)
+ {
+ snew(tab, ncount);
+ for (i = 0; i < ncount; i++)
+ {
+ if (count[i] > 0)
+ {
+ sprintf(tabfn, "%s", basefn);
+ sprintf(tabfn + strlen(basefn) - strlen(ftp2ext(efXVG)) - 1, "_%s%d.%s",
+ tabext, i, ftp2ext(efXVG));
+ tab[i] = make_bonded_table(fplog, tabfn, NRAL(ftype1)-2);
+ }
+ }
+ sfree(count);
+ }
+
+ return tab;
+}
+
+void forcerec_set_ranges(t_forcerec *fr,
+ int ncg_home, int ncg_force,
+ int natoms_force,
+ int natoms_force_constr, int natoms_f_novirsum)
+{
+ fr->cg0 = 0;
+ fr->hcg = ncg_home;
+
+ /* fr->ncg_force is unused in the standard code,
+ * but it can be useful for modified code dealing with charge groups.
+ */
+ fr->ncg_force = ncg_force;
+ fr->natoms_force = natoms_force;
+ fr->natoms_force_constr = natoms_force_constr;
+
+ if (fr->natoms_force_constr > fr->nalloc_force)
+ {
+ fr->nalloc_force = over_alloc_dd(fr->natoms_force_constr);
+
+ if (fr->bTwinRange)
+ {
+ srenew(fr->f_twin, fr->nalloc_force);
+ }
+ }
+
+ if (fr->bF_NoVirSum)
+ {
+ fr->f_novirsum_n = natoms_f_novirsum;
+ if (fr->f_novirsum_n > fr->f_novirsum_nalloc)
+ {
+ fr->f_novirsum_nalloc = over_alloc_dd(fr->f_novirsum_n);
+ srenew(fr->f_novirsum_alloc, fr->f_novirsum_nalloc);
+ }
+ }
+ else
+ {
+ fr->f_novirsum_n = 0;
+ }
+}
+
+static real cutoff_inf(real cutoff)
+{
+ if (cutoff == 0)
+ {
+ cutoff = GMX_CUTOFF_INF;
+ }
+
+ return cutoff;
+}
+
+static void make_adress_tf_tables(FILE *fp, const output_env_t oenv,
+ t_forcerec *fr, const t_inputrec *ir,
+ const char *tabfn, const gmx_mtop_t *mtop,
+ matrix box)
+{
+ char buf[STRLEN];
+ int i, j;
+
+ if (tabfn == NULL)
+ {
+ gmx_fatal(FARGS, "No thermoforce table file given. Use -tabletf to specify a file\n");
+ return;
+ }
+
+ snew(fr->atf_tabs, ir->adress->n_tf_grps);
+
+ sprintf(buf, "%s", tabfn);
+ for (i = 0; i < ir->adress->n_tf_grps; i++)
+ {
+ j = ir->adress->tf_table_index[i]; /* get energy group index */
+ sprintf(buf + strlen(tabfn) - strlen(ftp2ext(efXVG)) - 1, "tf_%s.%s",
+ *(mtop->groups.grpname[mtop->groups.grps[egcENER].nm_ind[j]]), ftp2ext(efXVG));
+ if (fp)
+ {
+ fprintf(fp, "loading tf table for energygrp index %d from %s\n", ir->adress->tf_table_index[i], buf);
+ }
+ fr->atf_tabs[i] = make_atf_table(fp, oenv, fr, buf, box);
+ }
+
+}
+
+gmx_bool can_use_allvsall(const t_inputrec *ir, gmx_bool bPrintNote, t_commrec *cr, FILE *fp)
+{
+ gmx_bool bAllvsAll;
+
+ bAllvsAll =
+ (
+ ir->rlist == 0 &&
+ ir->rcoulomb == 0 &&
+ ir->rvdw == 0 &&
+ ir->ePBC == epbcNONE &&
+ ir->vdwtype == evdwCUT &&
+ ir->coulombtype == eelCUT &&
+ ir->efep == efepNO &&
+ (ir->implicit_solvent == eisNO ||
+ (ir->implicit_solvent == eisGBSA && (ir->gb_algorithm == egbSTILL ||
+ ir->gb_algorithm == egbHCT ||
+ ir->gb_algorithm == egbOBC))) &&
+ getenv("GMX_NO_ALLVSALL") == NULL
+ );
+
+ if (bAllvsAll && ir->opts.ngener > 1)
+ {
+ const char *note = "NOTE: Can not use all-vs-all force loops, because there are multiple energy monitor groups; you might get significantly higher performance when using only a single energy monitor group.\n";
+
+ if (bPrintNote)
+ {
+ if (MASTER(cr))
+ {
+ fprintf(stderr, "\n%s\n", note);
+ }
+ if (fp != NULL)
+ {
+ fprintf(fp, "\n%s\n", note);
+ }
+ }
+ bAllvsAll = FALSE;
+ }
+
+ if (bAllvsAll && fp && MASTER(cr))
+ {
+ fprintf(fp, "\nUsing accelerated all-vs-all kernels.\n\n");
+ }
+
+ return bAllvsAll;
+}
+
+
+static void init_forcerec_f_threads(t_forcerec *fr, int nenergrp)
+{
+ int t, i;
+
+ /* These thread local data structures are used for bondeds only */
+ fr->nthreads = gmx_omp_nthreads_get(emntBonded);
+
+ if (fr->nthreads > 1)
+ {
+ snew(fr->f_t, fr->nthreads);
+ /* Thread 0 uses the global force and energy arrays */
+ for (t = 1; t < fr->nthreads; t++)
+ {
+ fr->f_t[t].f = NULL;
+ fr->f_t[t].f_nalloc = 0;
+ snew(fr->f_t[t].fshift, SHIFTS);
+ fr->f_t[t].grpp.nener = nenergrp*nenergrp;
+ for (i = 0; i < egNR; i++)
+ {
+ snew(fr->f_t[t].grpp.ener[i], fr->f_t[t].grpp.nener);
+ }
+ }
+ }
+}
+
+
+static void pick_nbnxn_kernel_cpu(const t_inputrec gmx_unused *ir,
+ int *kernel_type,
+ int *ewald_excl)
+{
+ *kernel_type = nbnxnk4x4_PlainC;
+ *ewald_excl = ewaldexclTable;
+
+#ifdef GMX_NBNXN_SIMD
+ {
+#ifdef GMX_NBNXN_SIMD_4XN
+ *kernel_type = nbnxnk4xN_SIMD_4xN;
+#endif
+#ifdef GMX_NBNXN_SIMD_2XNN
+ /* We expect the 2xNN kernels to be faster in most cases */
+ *kernel_type = nbnxnk4xN_SIMD_2xNN;
+#endif
+
+#if defined GMX_NBNXN_SIMD_4XN && defined GMX_X86_AVX_256
+ if (EEL_RF(ir->coulombtype) || ir->coulombtype == eelCUT)
+ {
+ /* The raw pair rate of the 4x8 kernel is higher than 2x(4+4),
+ * 10% with HT, 50% without HT, but extra zeros interactions
+ * can compensate. As we currently don't detect the actual use
+ * of HT, switch to 4x8 to avoid a potential performance hit.
+ */
+ *kernel_type = nbnxnk4xN_SIMD_4xN;
+ }
+#endif
+ if (getenv("GMX_NBNXN_SIMD_4XN") != NULL)
+ {
+#ifdef GMX_NBNXN_SIMD_4XN
+ *kernel_type = nbnxnk4xN_SIMD_4xN;
+#else
+ gmx_fatal(FARGS, "SIMD 4xN kernels requested, but Gromacs has been compiled without support for these kernels");
+#endif
+ }
+ if (getenv("GMX_NBNXN_SIMD_2XNN") != NULL)
+ {
+#ifdef GMX_NBNXN_SIMD_2XNN
+ *kernel_type = nbnxnk4xN_SIMD_2xNN;
+#else
+ gmx_fatal(FARGS, "SIMD 2x(N+N) kernels requested, but Gromacs has been compiled without support for these kernels");
+#endif
+ }
+
- #if (defined GMX_X86_AVX_128_FMA || defined GMX_X86_AVX_256) && !defined GMX_DOUBLE
++ /* Analytical Ewald exclusion correction is only an option in
++ * the SIMD kernel. On BlueGene/Q, this is faster regardless
++ * of precision. In single precision, this is faster on
++ * Bulldozer, and slightly faster on Sandy Bridge.
+ */
- #endif /* GMX_X86_SSE2 */
++#if ((defined GMX_X86_AVX_128_FMA || defined GMX_X86_AVX_256) && !defined GMX_DOUBLE) || (defined GMX_CPU_ACCELERATION_IBM_QPX)
+ *ewald_excl = ewaldexclAnalytical;
+#endif
+ if (getenv("GMX_NBNXN_EWALD_TABLE") != NULL)
+ {
+ *ewald_excl = ewaldexclTable;
+ }
+ if (getenv("GMX_NBNXN_EWALD_ANALYTICAL") != NULL)
+ {
+ *ewald_excl = ewaldexclAnalytical;
+ }
+
+ }
- #else /* GMX_X86_SSE2 */
++#endif /* GMX_NBNXN_SIMD */
+}
+
+
+const char *lookup_nbnxn_kernel_name(int kernel_type)
+{
+ const char *returnvalue = NULL;
+ switch (kernel_type)
+ {
+ case nbnxnkNotSet:
+ returnvalue = "not set";
+ break;
+ case nbnxnk4x4_PlainC:
+ returnvalue = "plain C";
+ break;
+ case nbnxnk4xN_SIMD_4xN:
+ case nbnxnk4xN_SIMD_2xNN:
+#ifdef GMX_NBNXN_SIMD
+#ifdef GMX_X86_SSE2
+ /* We have x86 SSE2 compatible SIMD */
+#ifdef GMX_X86_AVX_128_FMA
+ returnvalue = "AVX-128-FMA";
+#else
+#if defined GMX_X86_AVX_256 || defined __AVX__
+ /* x86 SIMD intrinsics can be converted to SSE or AVX depending
+ * on compiler flags. As we use nearly identical intrinsics,
+ * compiling for AVX without an AVX macros effectively results
+ * in AVX kernels.
+ * For gcc we check for __AVX__
+ * At least a check for icc should be added (if there is a macro)
+ */
+#if defined GMX_X86_AVX_256 && !defined GMX_NBNXN_HALF_WIDTH_SIMD
+ returnvalue = "AVX-256";
+#else
+ returnvalue = "AVX-128";
+#endif
+#else
+#ifdef GMX_X86_SSE4_1
+ returnvalue = "SSE4.1";
+#else
+ returnvalue = "SSE2";
+#endif
+#endif
+#endif
- #else /* GMX_NBNXN_SIMD */
++#else /* GMX_X86_SSE2 */
+ /* not GMX_X86_SSE2, but other SIMD */
+ returnvalue = "SIMD";
+#endif /* GMX_X86_SSE2 */
++#else /* GMX_NBNXN_SIMD */
+ returnvalue = "not available";
+#endif /* GMX_NBNXN_SIMD */
+ break;
+ case nbnxnk8x8x8_CUDA: returnvalue = "CUDA"; break;
+ case nbnxnk8x8x8_PlainC: returnvalue = "plain C"; break;
+
+ case nbnxnkNR:
+ default:
+ gmx_fatal(FARGS, "Illegal kernel type selected");
+ returnvalue = NULL;
+ break;
+ }
+ return returnvalue;
+};
+
+static void pick_nbnxn_kernel(FILE *fp,
+ const t_commrec *cr,
+ gmx_bool use_cpu_acceleration,
+ gmx_bool bUseGPU,
+ gmx_bool bEmulateGPU,
+ const t_inputrec *ir,
+ int *kernel_type,
+ int *ewald_excl,
+ gmx_bool bDoNonbonded)
+{
+ assert(kernel_type);
+
+ *kernel_type = nbnxnkNotSet;
+ *ewald_excl = ewaldexclTable;
+
+ if (bEmulateGPU)
+ {
+ *kernel_type = nbnxnk8x8x8_PlainC;
+
+ if (bDoNonbonded)
+ {
+ md_print_warn(cr, fp, "Emulating a GPU run on the CPU (slow)");
+ }
+ }
+ else if (bUseGPU)
+ {
+ *kernel_type = nbnxnk8x8x8_CUDA;
+ }
+
+ if (*kernel_type == nbnxnkNotSet)
+ {
+ if (use_cpu_acceleration)
+ {
+ pick_nbnxn_kernel_cpu(ir, kernel_type, ewald_excl);
+ }
+ else
+ {
+ *kernel_type = nbnxnk4x4_PlainC;
+ }
+ }
+
+ if (bDoNonbonded && fp != NULL)
+ {
+ fprintf(fp, "\nUsing %s %dx%d non-bonded kernels\n\n",
+ lookup_nbnxn_kernel_name(*kernel_type),
+ nbnxn_kernel_pairlist_simple(*kernel_type) ? NBNXN_CPU_CLUSTER_I_SIZE : NBNXN_GPU_CLUSTER_SIZE,
+ nbnxn_kernel_to_cj_size(*kernel_type));
+ }
+}
+
+static void pick_nbnxn_resources(const t_commrec *cr,
+ const gmx_hw_info_t *hwinfo,
+ gmx_bool bDoNonbonded,
+ gmx_bool *bUseGPU,
+ gmx_bool *bEmulateGPU)
+{
+ gmx_bool bEmulateGPUEnvVarSet;
+ char gpu_err_str[STRLEN];
+
+ *bUseGPU = FALSE;
+
+ bEmulateGPUEnvVarSet = (getenv("GMX_EMULATE_GPU") != NULL);
+
+ /* Run GPU emulation mode if GMX_EMULATE_GPU is defined. Because
+ * GPUs (currently) only handle non-bonded calculations, we will
+ * automatically switch to emulation if non-bonded calculations are
+ * turned off via GMX_NO_NONBONDED - this is the simple and elegant
+ * way to turn off GPU initialization, data movement, and cleanup.
+ *
+ * GPU emulation can be useful to assess the performance one can expect by
+ * adding GPU(s) to the machine. The conditional below allows this even
+ * if mdrun is compiled without GPU acceleration support.
+ * Note that you should freezing the system as otherwise it will explode.
+ */
+ *bEmulateGPU = (bEmulateGPUEnvVarSet ||
+ (!bDoNonbonded && hwinfo->bCanUseGPU));
+
+ /* Enable GPU mode when GPUs are available or no GPU emulation is requested.
+ */
+ if (hwinfo->bCanUseGPU && !(*bEmulateGPU))
+ {
+ /* Each PP node will use the intra-node id-th device from the
+ * list of detected/selected GPUs. */
+ if (!init_gpu(cr->rank_pp_intranode, gpu_err_str, &hwinfo->gpu_info))
+ {
+ /* At this point the init should never fail as we made sure that
+ * we have all the GPUs we need. If it still does, we'll bail. */
+ gmx_fatal(FARGS, "On node %d failed to initialize GPU #%d: %s",
+ cr->nodeid,
+ get_gpu_device_id(&hwinfo->gpu_info, cr->rank_pp_intranode),
+ gpu_err_str);
+ }
+
+ /* Here we actually turn on hardware GPU acceleration */
+ *bUseGPU = TRUE;
+ }
+}
+
+gmx_bool uses_simple_tables(int cutoff_scheme,
+ nonbonded_verlet_t *nbv,
+ int group)
+{
+ gmx_bool bUsesSimpleTables = TRUE;
+ int grp_index;
+
+ switch (cutoff_scheme)
+ {
+ case ecutsGROUP:
+ bUsesSimpleTables = TRUE;
+ break;
+ case ecutsVERLET:
+ assert(NULL != nbv && NULL != nbv->grp);
+ grp_index = (group < 0) ? 0 : (nbv->ngrp - 1);
+ bUsesSimpleTables = nbnxn_kernel_pairlist_simple(nbv->grp[grp_index].kernel_type);
+ break;
+ default:
+ gmx_incons("unimplemented");
+ }
+ return bUsesSimpleTables;
+}
+
+static void init_ewald_f_table(interaction_const_t *ic,
+ gmx_bool bUsesSimpleTables,
+ real rtab)
+{
+ real maxr;
+
+ if (bUsesSimpleTables)
+ {
+ /* With a spacing of 0.0005 we are at the force summation accuracy
+ * for the SSE kernels for "normal" atomistic simulations.
+ */
+ ic->tabq_scale = ewald_spline3_table_scale(ic->ewaldcoeff,
+ ic->rcoulomb);
+
+ maxr = (rtab > ic->rcoulomb) ? rtab : ic->rcoulomb;
+ ic->tabq_size = (int)(maxr*ic->tabq_scale) + 2;
+ }
+ else
+ {
+ ic->tabq_size = GPU_EWALD_COULOMB_FORCE_TABLE_SIZE;
+ /* Subtract 2 iso 1 to avoid access out of range due to rounding */
+ ic->tabq_scale = (ic->tabq_size - 2)/ic->rcoulomb;
+ }
+
+ sfree_aligned(ic->tabq_coul_FDV0);
+ sfree_aligned(ic->tabq_coul_F);
+ sfree_aligned(ic->tabq_coul_V);
+
+ /* Create the original table data in FDV0 */
+ snew_aligned(ic->tabq_coul_FDV0, ic->tabq_size*4, 32);
+ snew_aligned(ic->tabq_coul_F, ic->tabq_size, 32);
+ snew_aligned(ic->tabq_coul_V, ic->tabq_size, 32);
+ table_spline3_fill_ewald_lr(ic->tabq_coul_F, ic->tabq_coul_V, ic->tabq_coul_FDV0,
+ ic->tabq_size, 1/ic->tabq_scale, ic->ewaldcoeff);
+}
+
+void init_interaction_const_tables(FILE *fp,
+ interaction_const_t *ic,
+ gmx_bool bUsesSimpleTables,
+ real rtab)
+{
+ real spacing;
+
+ if (ic->eeltype == eelEWALD || EEL_PME(ic->eeltype))
+ {
+ init_ewald_f_table(ic, bUsesSimpleTables, rtab);
+
+ if (fp != NULL)
+ {
+ fprintf(fp, "Initialized non-bonded Ewald correction tables, spacing: %.2e size: %d\n\n",
+ 1/ic->tabq_scale, ic->tabq_size);
+ }
+ }
+}
+
+void init_interaction_const(FILE *fp,
+ interaction_const_t **interaction_const,
+ const t_forcerec *fr,
+ real rtab)
+{
+ interaction_const_t *ic;
+ gmx_bool bUsesSimpleTables = TRUE;
+
+ snew(ic, 1);
+
+ /* Just allocate something so we can free it */
+ snew_aligned(ic->tabq_coul_FDV0, 16, 32);
+ snew_aligned(ic->tabq_coul_F, 16, 32);
+ snew_aligned(ic->tabq_coul_V, 16, 32);
+
+ ic->rlist = fr->rlist;
+ ic->rlistlong = fr->rlistlong;
+
+ /* Lennard-Jones */
+ ic->rvdw = fr->rvdw;
+ if (fr->vdw_modifier == eintmodPOTSHIFT)
+ {
+ ic->sh_invrc6 = pow(ic->rvdw, -6.0);
+ }
+ else
+ {
+ ic->sh_invrc6 = 0;
+ }
+
+ /* Electrostatics */
+ ic->eeltype = fr->eeltype;
+ ic->rcoulomb = fr->rcoulomb;
+ ic->epsilon_r = fr->epsilon_r;
+ ic->epsfac = fr->epsfac;
+
+ /* Ewald */
+ ic->ewaldcoeff = fr->ewaldcoeff;
+ if (fr->coulomb_modifier == eintmodPOTSHIFT)
+ {
+ ic->sh_ewald = gmx_erfc(ic->ewaldcoeff*ic->rcoulomb);
+ }
+ else
+ {
+ ic->sh_ewald = 0;
+ }
+
+ /* Reaction-field */
+ if (EEL_RF(ic->eeltype))
+ {
+ ic->epsilon_rf = fr->epsilon_rf;
+ ic->k_rf = fr->k_rf;
+ ic->c_rf = fr->c_rf;
+ }
+ else
+ {
+ /* For plain cut-off we might use the reaction-field kernels */
+ ic->epsilon_rf = ic->epsilon_r;
+ ic->k_rf = 0;
+ if (fr->coulomb_modifier == eintmodPOTSHIFT)
+ {
+ ic->c_rf = 1/ic->rcoulomb;
+ }
+ else
+ {
+ ic->c_rf = 0;
+ }
+ }
+
+ if (fp != NULL)
+ {
+ fprintf(fp, "Potential shift: LJ r^-12: %.3f r^-6 %.3f",
+ sqr(ic->sh_invrc6), ic->sh_invrc6);
+ if (ic->eeltype == eelCUT)
+ {
+ fprintf(fp, ", Coulomb %.3f", ic->c_rf);
+ }
+ else if (EEL_PME(ic->eeltype))
+ {
+ fprintf(fp, ", Ewald %.3e", ic->sh_ewald);
+ }
+ fprintf(fp, "\n");
+ }
+
+ *interaction_const = ic;
+
+ if (fr->nbv != NULL && fr->nbv->bUseGPU)
+ {
+ nbnxn_cuda_init_const(fr->nbv->cu_nbv, ic, fr->nbv->grp);
+ }
+
+ bUsesSimpleTables = uses_simple_tables(fr->cutoff_scheme, fr->nbv, -1);
+ init_interaction_const_tables(fp, ic, bUsesSimpleTables, rtab);
+}
+
+static void init_nb_verlet(FILE *fp,
+ nonbonded_verlet_t **nb_verlet,
+ const t_inputrec *ir,
+ const t_forcerec *fr,
+ const t_commrec *cr,
+ const char *nbpu_opt)
+{
+ nonbonded_verlet_t *nbv;
+ int i;
+ char *env;
+ gmx_bool bEmulateGPU, bHybridGPURun = FALSE;
+
+ nbnxn_alloc_t *nb_alloc;
+ nbnxn_free_t *nb_free;
+
+ snew(nbv, 1);
+
+ pick_nbnxn_resources(cr, fr->hwinfo,
+ fr->bNonbonded,
+ &nbv->bUseGPU,
+ &bEmulateGPU);
+
+ nbv->nbs = NULL;
+
+ nbv->ngrp = (DOMAINDECOMP(cr) ? 2 : 1);
+ for (i = 0; i < nbv->ngrp; i++)
+ {
+ nbv->grp[i].nbl_lists.nnbl = 0;
+ nbv->grp[i].nbat = NULL;
+ nbv->grp[i].kernel_type = nbnxnkNotSet;
+
+ if (i == 0) /* local */
+ {
+ pick_nbnxn_kernel(fp, cr, fr->use_cpu_acceleration,
+ nbv->bUseGPU, bEmulateGPU, ir,
+ &nbv->grp[i].kernel_type,
+ &nbv->grp[i].ewald_excl,
+ fr->bNonbonded);
+ }
+ else /* non-local */
+ {
+ if (nbpu_opt != NULL && strcmp(nbpu_opt, "gpu_cpu") == 0)
+ {
+ /* Use GPU for local, select a CPU kernel for non-local */
+ pick_nbnxn_kernel(fp, cr, fr->use_cpu_acceleration,
+ FALSE, FALSE, ir,
+ &nbv->grp[i].kernel_type,
+ &nbv->grp[i].ewald_excl,
+ fr->bNonbonded);
+
+ bHybridGPURun = TRUE;
+ }
+ else
+ {
+ /* Use the same kernel for local and non-local interactions */
+ nbv->grp[i].kernel_type = nbv->grp[0].kernel_type;
+ nbv->grp[i].ewald_excl = nbv->grp[0].ewald_excl;
+ }
+ }
+ }
+
+ if (nbv->bUseGPU)
+ {
+ /* init the NxN GPU data; the last argument tells whether we'll have
+ * both local and non-local NB calculation on GPU */
+ nbnxn_cuda_init(fp, &nbv->cu_nbv,
+ &fr->hwinfo->gpu_info, cr->rank_pp_intranode,
+ (nbv->ngrp > 1) && !bHybridGPURun);
+
+ if ((env = getenv("GMX_NB_MIN_CI")) != NULL)
+ {
+ char *end;
+
+ nbv->min_ci_balanced = strtol(env, &end, 10);
+ if (!end || (*end != 0) || nbv->min_ci_balanced <= 0)
+ {
+ gmx_fatal(FARGS, "Invalid value passed in GMX_NB_MIN_CI=%s, positive integer required", env);
+ }
+
+ if (debug)
+ {
+ fprintf(debug, "Neighbor-list balancing parameter: %d (passed as env. var.)\n",
+ nbv->min_ci_balanced);
+ }
+ }
+ else
+ {
+ nbv->min_ci_balanced = nbnxn_cuda_min_ci_balanced(nbv->cu_nbv);
+ if (debug)
+ {
+ fprintf(debug, "Neighbor-list balancing parameter: %d (auto-adjusted to the number of GPU multi-processors)\n",
+ nbv->min_ci_balanced);
+ }
+ }
+ }
+ else
+ {
+ nbv->min_ci_balanced = 0;
+ }
+
+ *nb_verlet = nbv;
+
+ nbnxn_init_search(&nbv->nbs,
+ DOMAINDECOMP(cr) ? &cr->dd->nc : NULL,
+ DOMAINDECOMP(cr) ? domdec_zones(cr->dd) : NULL,
+ gmx_omp_nthreads_get(emntNonbonded));
+
+ for (i = 0; i < nbv->ngrp; i++)
+ {
+ if (nbv->grp[0].kernel_type == nbnxnk8x8x8_CUDA)
+ {
+ nb_alloc = &pmalloc;
+ nb_free = &pfree;
+ }
+ else
+ {
+ nb_alloc = NULL;
+ nb_free = NULL;
+ }
+
+ nbnxn_init_pairlist_set(&nbv->grp[i].nbl_lists,
+ nbnxn_kernel_pairlist_simple(nbv->grp[i].kernel_type),
+ /* 8x8x8 "non-simple" lists are ATM always combined */
+ !nbnxn_kernel_pairlist_simple(nbv->grp[i].kernel_type),
+ nb_alloc, nb_free);
+
+ if (i == 0 ||
+ nbv->grp[0].kernel_type != nbv->grp[i].kernel_type)
+ {
+ snew(nbv->grp[i].nbat, 1);
+ nbnxn_atomdata_init(fp,
+ nbv->grp[i].nbat,
+ nbv->grp[i].kernel_type,
+ fr->ntype, fr->nbfp,
+ ir->opts.ngener,
+ nbnxn_kernel_pairlist_simple(nbv->grp[i].kernel_type) ? gmx_omp_nthreads_get(emntNonbonded) : 1,
+ nb_alloc, nb_free);
+ }
+ else
+ {
+ nbv->grp[i].nbat = nbv->grp[0].nbat;
+ }
+ }
+}
+
+void init_forcerec(FILE *fp,
+ const output_env_t oenv,
+ t_forcerec *fr,
+ t_fcdata *fcd,
+ const t_inputrec *ir,
+ const gmx_mtop_t *mtop,
+ const t_commrec *cr,
+ matrix box,
+ const char *tabfn,
+ const char *tabafn,
+ const char *tabpfn,
+ const char *tabbfn,
+ const char *nbpu_opt,
+ gmx_bool bNoSolvOpt,
+ real print_force)
+{
+ int i, j, m, natoms, ngrp, negp_pp, negptable, egi, egj;
+ real rtab;
+ char *env;
+ double dbl;
+ const t_block *cgs;
+ gmx_bool bGenericKernelOnly;
+ gmx_bool bTab, bSep14tab, bNormalnblists;
+ t_nblists *nbl;
+ int *nm_ind, egp_flags;
+
+ if (fr->hwinfo == NULL)
+ {
+ /* Detect hardware, gather information.
+ * In mdrun, hwinfo has already been set before calling init_forcerec.
+ * Here we ignore GPUs, as tools will not use them anyhow.
+ */
+ fr->hwinfo = gmx_detect_hardware(fp, cr, FALSE, FALSE, NULL);
+ }
+
+ /* By default we turn acceleration on, but it might be turned off further down... */
+ fr->use_cpu_acceleration = TRUE;
+
+ fr->bDomDec = DOMAINDECOMP(cr);
+
+ natoms = mtop->natoms;
+
+ if (check_box(ir->ePBC, box))
+ {
+ gmx_fatal(FARGS, check_box(ir->ePBC, box));
+ }
+
+ /* Test particle insertion ? */
+ if (EI_TPI(ir->eI))
+ {
+ /* Set to the size of the molecule to be inserted (the last one) */
+ /* Because of old style topologies, we have to use the last cg
+ * instead of the last molecule type.
+ */
+ cgs = &mtop->moltype[mtop->molblock[mtop->nmolblock-1].type].cgs;
+ fr->n_tpi = cgs->index[cgs->nr] - cgs->index[cgs->nr-1];
+ if (fr->n_tpi != mtop->mols.index[mtop->mols.nr] - mtop->mols.index[mtop->mols.nr-1])
+ {
+ gmx_fatal(FARGS, "The molecule to insert can not consist of multiple charge groups.\nMake it a single charge group.");
+ }
+ }
+ else
+ {
+ fr->n_tpi = 0;
+ }
+
+ /* Copy AdResS parameters */
+ if (ir->bAdress)
+ {
+ fr->adress_type = ir->adress->type;
+ fr->adress_const_wf = ir->adress->const_wf;
+ fr->adress_ex_width = ir->adress->ex_width;
+ fr->adress_hy_width = ir->adress->hy_width;
+ fr->adress_icor = ir->adress->icor;
+ fr->adress_site = ir->adress->site;
+ fr->adress_ex_forcecap = ir->adress->ex_forcecap;
+ fr->adress_do_hybridpairs = ir->adress->do_hybridpairs;
+
+
+ snew(fr->adress_group_explicit, ir->adress->n_energy_grps);
+ for (i = 0; i < ir->adress->n_energy_grps; i++)
+ {
+ fr->adress_group_explicit[i] = ir->adress->group_explicit[i];
+ }
+
+ fr->n_adress_tf_grps = ir->adress->n_tf_grps;
+ snew(fr->adress_tf_table_index, fr->n_adress_tf_grps);
+ for (i = 0; i < fr->n_adress_tf_grps; i++)
+ {
+ fr->adress_tf_table_index[i] = ir->adress->tf_table_index[i];
+ }
+ copy_rvec(ir->adress->refs, fr->adress_refs);
+ }
+ else
+ {
+ fr->adress_type = eAdressOff;
+ fr->adress_do_hybridpairs = FALSE;
+ }
+
+ /* Copy the user determined parameters */
+ fr->userint1 = ir->userint1;
+ fr->userint2 = ir->userint2;
+ fr->userint3 = ir->userint3;
+ fr->userint4 = ir->userint4;
+ fr->userreal1 = ir->userreal1;
+ fr->userreal2 = ir->userreal2;
+ fr->userreal3 = ir->userreal3;
+ fr->userreal4 = ir->userreal4;
+
+ /* Shell stuff */
+ fr->fc_stepsize = ir->fc_stepsize;
+
+ /* Free energy */
+ fr->efep = ir->efep;
+ fr->sc_alphavdw = ir->fepvals->sc_alpha;
+ if (ir->fepvals->bScCoul)
+ {
+ fr->sc_alphacoul = ir->fepvals->sc_alpha;
+ fr->sc_sigma6_min = pow(ir->fepvals->sc_sigma_min, 6);
+ }
+ else
+ {
+ fr->sc_alphacoul = 0;
+ fr->sc_sigma6_min = 0; /* only needed when bScCoul is on */
+ }
+ fr->sc_power = ir->fepvals->sc_power;
+ fr->sc_r_power = ir->fepvals->sc_r_power;
+ fr->sc_sigma6_def = pow(ir->fepvals->sc_sigma, 6);
+
+ env = getenv("GMX_SCSIGMA_MIN");
+ if (env != NULL)
+ {
+ dbl = 0;
+ sscanf(env, "%lf", &dbl);
+ fr->sc_sigma6_min = pow(dbl, 6);
+ if (fp)
+ {
+ fprintf(fp, "Setting the minimum soft core sigma to %g nm\n", dbl);
+ }
+ }
+
+ fr->bNonbonded = TRUE;
+ if (getenv("GMX_NO_NONBONDED") != NULL)
+ {
+ /* turn off non-bonded calculations */
+ fr->bNonbonded = FALSE;
+ md_print_warn(cr, fp,
+ "Found environment variable GMX_NO_NONBONDED.\n"
+ "Disabling nonbonded calculations.\n");
+ }
+
+ bGenericKernelOnly = FALSE;
+
+ /* We now check in the NS code whether a particular combination of interactions
+ * can be used with water optimization, and disable it if that is not the case.
+ */
+
+ if (getenv("GMX_NB_GENERIC") != NULL)
+ {
+ if (fp != NULL)
+ {
+ fprintf(fp,
+ "Found environment variable GMX_NB_GENERIC.\n"
+ "Disabling all interaction-specific nonbonded kernels, will only\n"
+ "use the slow generic ones in src/gmxlib/nonbonded/nb_generic.c\n\n");
+ }
+ bGenericKernelOnly = TRUE;
+ }
+
+ if (bGenericKernelOnly == TRUE)
+ {
+ bNoSolvOpt = TRUE;
+ }
+
+ if ( (getenv("GMX_DISABLE_CPU_ACCELERATION") != NULL) || (getenv("GMX_NOOPTIMIZEDKERNELS") != NULL) )
+ {
+ fr->use_cpu_acceleration = FALSE;
+ if (fp != NULL)
+ {
+ fprintf(fp,
+ "\nFound environment variable GMX_DISABLE_CPU_ACCELERATION.\n"
+ "Disabling all CPU architecture-specific (e.g. SSE2/SSE4/AVX) routines.\n\n");
+ }
+ }
+
+ fr->bBHAM = (mtop->ffparams.functype[0] == F_BHAM);
+
+ /* Check if we can/should do all-vs-all kernels */
+ fr->bAllvsAll = can_use_allvsall(ir, FALSE, NULL, NULL);
+ fr->AllvsAll_work = NULL;
+ fr->AllvsAll_workgb = NULL;
+
+ /* All-vs-all kernels have not been implemented in 4.6, and
+ * the SIMD group kernels are also buggy in this case. Non-accelerated
+ * group kernels are OK. See Redmine #1249. */
+ if (fr->bAllvsAll)
+ {
+ fr->bAllvsAll = FALSE;
+ fr->use_cpu_acceleration = FALSE;
+ if (fp != NULL)
+ {
+ fprintf(fp,
+ "\nYour simulation settings would have triggered the efficient all-vs-all\n"
+ "kernels in GROMACS 4.5, but these have not been implemented in GROMACS\n"
+ "4.6. Also, we can't use the accelerated SIMD kernels here because\n"
+ "of an unfixed bug. The reference C kernels are correct, though, so\n"
+ "we are proceeding by disabling all CPU architecture-specific\n"
+ "(e.g. SSE2/SSE4/AVX) routines. If performance is important, please\n"
+ "use GROMACS 4.5.7 or try cutoff-scheme = Verlet.\n\n");
+ }
+ }
+
+ /* Neighbour searching stuff */
+ fr->cutoff_scheme = ir->cutoff_scheme;
+ fr->bGrid = (ir->ns_type == ensGRID);
+ fr->ePBC = ir->ePBC;
+
+ /* Determine if we will do PBC for distances in bonded interactions */
+ if (fr->ePBC == epbcNONE)
+ {
+ fr->bMolPBC = FALSE;
+ }
+ else
+ {
+ if (!DOMAINDECOMP(cr))
+ {
+ /* The group cut-off scheme and SHAKE assume charge groups
+ * are whole, but not using molpbc is faster in most cases.
+ */
+ if (fr->cutoff_scheme == ecutsGROUP ||
+ (ir->eConstrAlg == econtSHAKE &&
+ (gmx_mtop_ftype_count(mtop, F_CONSTR) > 0 ||
+ gmx_mtop_ftype_count(mtop, F_CONSTRNC) > 0)))
+ {
+ fr->bMolPBC = ir->bPeriodicMols;
+ }
+ else
+ {
+ fr->bMolPBC = TRUE;
+ if (getenv("GMX_USE_GRAPH") != NULL)
+ {
+ fr->bMolPBC = FALSE;
+ if (fp)
+ {
+ fprintf(fp, "\nGMX_MOLPBC is set, using the graph for bonded interactions\n\n");
+ }
+ }
+ }
+ }
+ else
+ {
+ fr->bMolPBC = dd_bonded_molpbc(cr->dd, fr->ePBC);
+ }
+ }
+ fr->bGB = (ir->implicit_solvent == eisGBSA);
+
+ fr->rc_scaling = ir->refcoord_scaling;
+ copy_rvec(ir->posres_com, fr->posres_com);
+ copy_rvec(ir->posres_comB, fr->posres_comB);
+ fr->rlist = cutoff_inf(ir->rlist);
+ fr->rlistlong = cutoff_inf(ir->rlistlong);
+ fr->eeltype = ir->coulombtype;
+ fr->vdwtype = ir->vdwtype;
+
+ fr->coulomb_modifier = ir->coulomb_modifier;
+ fr->vdw_modifier = ir->vdw_modifier;
+
+ /* Electrostatics: Translate from interaction-setting-in-mdp-file to kernel interaction format */
+ switch (fr->eeltype)
+ {
+ case eelCUT:
+ fr->nbkernel_elec_interaction = (fr->bGB) ? GMX_NBKERNEL_ELEC_GENERALIZEDBORN : GMX_NBKERNEL_ELEC_COULOMB;
+ break;
+
+ case eelRF:
+ case eelGRF:
+ case eelRF_NEC:
+ fr->nbkernel_elec_interaction = GMX_NBKERNEL_ELEC_REACTIONFIELD;
+ break;
+
+ case eelRF_ZERO:
+ fr->nbkernel_elec_interaction = GMX_NBKERNEL_ELEC_REACTIONFIELD;
+ fr->coulomb_modifier = eintmodEXACTCUTOFF;
+ break;
+
+ case eelSWITCH:
+ case eelSHIFT:
+ case eelUSER:
+ case eelENCADSHIFT:
+ case eelPMESWITCH:
+ case eelPMEUSER:
+ case eelPMEUSERSWITCH:
+ fr->nbkernel_elec_interaction = GMX_NBKERNEL_ELEC_CUBICSPLINETABLE;
+ break;
+
+ case eelPME:
+ case eelEWALD:
+ fr->nbkernel_elec_interaction = GMX_NBKERNEL_ELEC_EWALD;
+ break;
+
+ default:
+ gmx_fatal(FARGS, "Unsupported electrostatic interaction: %s", eel_names[fr->eeltype]);
+ break;
+ }
+
+ /* Vdw: Translate from mdp settings to kernel format */
+ switch (fr->vdwtype)
+ {
+ case evdwCUT:
+ if (fr->bBHAM)
+ {
+ fr->nbkernel_vdw_interaction = GMX_NBKERNEL_VDW_BUCKINGHAM;
+ }
+ else
+ {
+ fr->nbkernel_vdw_interaction = GMX_NBKERNEL_VDW_LENNARDJONES;
+ }
+ break;
+
+ case evdwSWITCH:
+ case evdwSHIFT:
+ case evdwUSER:
+ case evdwENCADSHIFT:
+ fr->nbkernel_vdw_interaction = GMX_NBKERNEL_VDW_CUBICSPLINETABLE;
+ break;
+
+ default:
+ gmx_fatal(FARGS, "Unsupported vdw interaction: %s", evdw_names[fr->vdwtype]);
+ break;
+ }
+
+ /* These start out identical to ir, but might be altered if we e.g. tabulate the interaction in the kernel */
+ fr->nbkernel_elec_modifier = fr->coulomb_modifier;
+ fr->nbkernel_vdw_modifier = fr->vdw_modifier;
+
+ fr->bTwinRange = fr->rlistlong > fr->rlist;
+ fr->bEwald = (EEL_PME(fr->eeltype) || fr->eeltype == eelEWALD);
+
+ fr->reppow = mtop->ffparams.reppow;
+
+ if (ir->cutoff_scheme == ecutsGROUP)
+ {
+ fr->bvdwtab = (fr->vdwtype != evdwCUT ||
+ !gmx_within_tol(fr->reppow, 12.0, 10*GMX_DOUBLE_EPS));
+ /* We have special kernels for standard Ewald and PME, but the pme-switch ones are tabulated above */
+ fr->bcoultab = !(fr->eeltype == eelCUT ||
+ fr->eeltype == eelEWALD ||
+ fr->eeltype == eelPME ||
+ fr->eeltype == eelRF ||
+ fr->eeltype == eelRF_ZERO);
+
+ /* If the user absolutely wants different switch/shift settings for coul/vdw, it is likely
+ * going to be faster to tabulate the interaction than calling the generic kernel.
+ */
+ if (fr->nbkernel_elec_modifier == eintmodPOTSWITCH && fr->nbkernel_vdw_modifier == eintmodPOTSWITCH)
+ {
+ if ((fr->rcoulomb_switch != fr->rvdw_switch) || (fr->rcoulomb != fr->rvdw))
+ {
+ fr->bcoultab = TRUE;
+ }
+ }
+ else if ((fr->nbkernel_elec_modifier == eintmodPOTSHIFT && fr->nbkernel_vdw_modifier == eintmodPOTSHIFT) ||
+ ((fr->nbkernel_elec_interaction == GMX_NBKERNEL_ELEC_REACTIONFIELD &&
+ fr->nbkernel_elec_modifier == eintmodEXACTCUTOFF &&
+ (fr->nbkernel_vdw_modifier == eintmodPOTSWITCH || fr->nbkernel_vdw_modifier == eintmodPOTSHIFT))))
+ {
+ if (fr->rcoulomb != fr->rvdw)
+ {
+ fr->bcoultab = TRUE;
+ }
+ }
+
+ if (getenv("GMX_REQUIRE_TABLES"))
+ {
+ fr->bvdwtab = TRUE;
+ fr->bcoultab = TRUE;
+ }
+
+ if (fp)
+ {
+ fprintf(fp, "Table routines are used for coulomb: %s\n", bool_names[fr->bcoultab]);
+ fprintf(fp, "Table routines are used for vdw: %s\n", bool_names[fr->bvdwtab ]);
+ }
+
+ if (fr->bvdwtab == TRUE)
+ {
+ fr->nbkernel_vdw_interaction = GMX_NBKERNEL_VDW_CUBICSPLINETABLE;
+ fr->nbkernel_vdw_modifier = eintmodNONE;
+ }
+ if (fr->bcoultab == TRUE)
+ {
+ fr->nbkernel_elec_interaction = GMX_NBKERNEL_ELEC_CUBICSPLINETABLE;
+ fr->nbkernel_elec_modifier = eintmodNONE;
+ }
+ }
+
+ if (ir->cutoff_scheme == ecutsVERLET)
+ {
+ if (!gmx_within_tol(fr->reppow, 12.0, 10*GMX_DOUBLE_EPS))
+ {
+ gmx_fatal(FARGS, "Cut-off scheme %S only supports LJ repulsion power 12", ecutscheme_names[ir->cutoff_scheme]);
+ }
+ fr->bvdwtab = FALSE;
+ fr->bcoultab = FALSE;
+ }
+
+ /* Tables are used for direct ewald sum */
+ if (fr->bEwald)
+ {
+ if (EEL_PME(ir->coulombtype))
+ {
+ if (fp)
+ {
+ fprintf(fp, "Will do PME sum in reciprocal space.\n");
+ }
+ if (ir->coulombtype == eelP3M_AD)
+ {
+ please_cite(fp, "Hockney1988");
+ please_cite(fp, "Ballenegger2012");
+ }
+ else
+ {
+ please_cite(fp, "Essmann95a");
+ }
+
+ if (ir->ewald_geometry == eewg3DC)
+ {
+ if (fp)
+ {
+ fprintf(fp, "Using the Ewald3DC correction for systems with a slab geometry.\n");
+ }
+ please_cite(fp, "In-Chul99a");
+ }
+ }
+ fr->ewaldcoeff = calc_ewaldcoeff(ir->rcoulomb, ir->ewald_rtol);
+ init_ewald_tab(&(fr->ewald_table), ir, fp);
+ if (fp)
+ {
+ fprintf(fp, "Using a Gaussian width (1/beta) of %g nm for Ewald\n",
+ 1/fr->ewaldcoeff);
+ }
+ }
+
+ /* Electrostatics */
+ fr->epsilon_r = ir->epsilon_r;
+ fr->epsilon_rf = ir->epsilon_rf;
+ fr->fudgeQQ = mtop->ffparams.fudgeQQ;
+ fr->rcoulomb_switch = ir->rcoulomb_switch;
+ fr->rcoulomb = cutoff_inf(ir->rcoulomb);
+
+ /* Parameters for generalized RF */
+ fr->zsquare = 0.0;
+ fr->temp = 0.0;
+
+ if (fr->eeltype == eelGRF)
+ {
+ init_generalized_rf(fp, mtop, ir, fr);
+ }
+
+ fr->bF_NoVirSum = (EEL_FULL(fr->eeltype) ||
+ gmx_mtop_ftype_count(mtop, F_POSRES) > 0 ||
+ gmx_mtop_ftype_count(mtop, F_FBPOSRES) > 0 ||
+ IR_ELEC_FIELD(*ir) ||
+ (fr->adress_icor != eAdressICOff)
+ );
+
+ if (fr->cutoff_scheme == ecutsGROUP &&
+ ncg_mtop(mtop) > fr->cg_nalloc && !DOMAINDECOMP(cr))
+ {
+ /* Count the total number of charge groups */
+ fr->cg_nalloc = ncg_mtop(mtop);
+ srenew(fr->cg_cm, fr->cg_nalloc);
+ }
+ if (fr->shift_vec == NULL)
+ {
+ snew(fr->shift_vec, SHIFTS);
+ }
+
+ if (fr->fshift == NULL)
+ {
+ snew(fr->fshift, SHIFTS);
+ }
+
+ if (fr->nbfp == NULL)
+ {
+ fr->ntype = mtop->ffparams.atnr;
+ fr->nbfp = mk_nbfp(&mtop->ffparams, fr->bBHAM);
+ }
+
+ /* Copy the energy group exclusions */
+ fr->egp_flags = ir->opts.egp_flags;
+
+ /* Van der Waals stuff */
+ fr->rvdw = cutoff_inf(ir->rvdw);
+ fr->rvdw_switch = ir->rvdw_switch;
+ if ((fr->vdwtype != evdwCUT) && (fr->vdwtype != evdwUSER) && !fr->bBHAM)
+ {
+ if (fr->rvdw_switch >= fr->rvdw)
+ {
+ gmx_fatal(FARGS, "rvdw_switch (%f) must be < rvdw (%f)",
+ fr->rvdw_switch, fr->rvdw);
+ }
+ if (fp)
+ {
+ fprintf(fp, "Using %s Lennard-Jones, switch between %g and %g nm\n",
+ (fr->eeltype == eelSWITCH) ? "switched" : "shifted",
+ fr->rvdw_switch, fr->rvdw);
+ }
+ }
+
+ if (fr->bBHAM && (fr->vdwtype == evdwSHIFT || fr->vdwtype == evdwSWITCH))
+ {
+ gmx_fatal(FARGS, "Switch/shift interaction not supported with Buckingham");
+ }
+
+ if (fp)
+ {
+ fprintf(fp, "Cut-off's: NS: %g Coulomb: %g %s: %g\n",
+ fr->rlist, fr->rcoulomb, fr->bBHAM ? "BHAM" : "LJ", fr->rvdw);
+ }
+
+ fr->eDispCorr = ir->eDispCorr;
+ if (ir->eDispCorr != edispcNO)
+ {
+ set_avcsixtwelve(fp, fr, mtop);
+ }
+
+ if (fr->bBHAM)
+ {
+ set_bham_b_max(fp, fr, mtop);
+ }
+
+ fr->gb_epsilon_solvent = ir->gb_epsilon_solvent;
+
+ /* Copy the GBSA data (radius, volume and surftens for each
+ * atomtype) from the topology atomtype section to forcerec.
+ */
+ snew(fr->atype_radius, fr->ntype);
+ snew(fr->atype_vol, fr->ntype);
+ snew(fr->atype_surftens, fr->ntype);
+ snew(fr->atype_gb_radius, fr->ntype);
+ snew(fr->atype_S_hct, fr->ntype);
+
+ if (mtop->atomtypes.nr > 0)
+ {
+ for (i = 0; i < fr->ntype; i++)
+ {
+ fr->atype_radius[i] = mtop->atomtypes.radius[i];
+ }
+ for (i = 0; i < fr->ntype; i++)
+ {
+ fr->atype_vol[i] = mtop->atomtypes.vol[i];
+ }
+ for (i = 0; i < fr->ntype; i++)
+ {
+ fr->atype_surftens[i] = mtop->atomtypes.surftens[i];
+ }
+ for (i = 0; i < fr->ntype; i++)
+ {
+ fr->atype_gb_radius[i] = mtop->atomtypes.gb_radius[i];
+ }
+ for (i = 0; i < fr->ntype; i++)
+ {
+ fr->atype_S_hct[i] = mtop->atomtypes.S_hct[i];
+ }
+ }
+
+ /* Generate the GB table if needed */
+ if (fr->bGB)
+ {
+#ifdef GMX_DOUBLE
+ fr->gbtabscale = 2000;
+#else
+ fr->gbtabscale = 500;
+#endif
+
+ fr->gbtabr = 100;
+ fr->gbtab = make_gb_table(oenv, fr);
+
+ init_gb(&fr->born, cr, fr, ir, mtop, ir->gb_algorithm);
+
+ /* Copy local gb data (for dd, this is done in dd_partition_system) */
+ if (!DOMAINDECOMP(cr))
+ {
+ make_local_gb(cr, fr->born, ir->gb_algorithm);
+ }
+ }
+
+ /* Set the charge scaling */
+ if (fr->epsilon_r != 0)
+ {
+ fr->epsfac = ONE_4PI_EPS0/fr->epsilon_r;
+ }
+ else
+ {
+ /* eps = 0 is infinite dieletric: no coulomb interactions */
+ fr->epsfac = 0;
+ }
+
+ /* Reaction field constants */
+ if (EEL_RF(fr->eeltype))
+ {
+ calc_rffac(fp, fr->eeltype, fr->epsilon_r, fr->epsilon_rf,
+ fr->rcoulomb, fr->temp, fr->zsquare, box,
+ &fr->kappa, &fr->k_rf, &fr->c_rf);
+ }
+
+ set_chargesum(fp, fr, mtop);
+
+ /* if we are using LR electrostatics, and they are tabulated,
+ * the tables will contain modified coulomb interactions.
+ * Since we want to use the non-shifted ones for 1-4
+ * coulombic interactions, we must have an extra set of tables.
+ */
+
+ /* Construct tables.
+ * A little unnecessary to make both vdw and coul tables sometimes,
+ * but what the heck... */
+
+ bTab = fr->bcoultab || fr->bvdwtab || fr->bEwald;
+
+ bSep14tab = ((!bTab || fr->eeltype != eelCUT || fr->vdwtype != evdwCUT ||
+ fr->bBHAM || fr->bEwald) &&
+ (gmx_mtop_ftype_count(mtop, F_LJ14) > 0 ||
+ gmx_mtop_ftype_count(mtop, F_LJC14_Q) > 0 ||
+ gmx_mtop_ftype_count(mtop, F_LJC_PAIRS_NB) > 0));
+
+ negp_pp = ir->opts.ngener - ir->nwall;
+ negptable = 0;
+ if (!bTab)
+ {
+ bNormalnblists = TRUE;
+ fr->nnblists = 1;
+ }
+ else
+ {
+ bNormalnblists = (ir->eDispCorr != edispcNO);
+ for (egi = 0; egi < negp_pp; egi++)
+ {
+ for (egj = egi; egj < negp_pp; egj++)
+ {
+ egp_flags = ir->opts.egp_flags[GID(egi, egj, ir->opts.ngener)];
+ if (!(egp_flags & EGP_EXCL))
+ {
+ if (egp_flags & EGP_TABLE)
+ {
+ negptable++;
+ }
+ else
+ {
+ bNormalnblists = TRUE;
+ }
+ }
+ }
+ }
+ if (bNormalnblists)
+ {
+ fr->nnblists = negptable + 1;
+ }
+ else
+ {
+ fr->nnblists = negptable;
+ }
+ if (fr->nnblists > 1)
+ {
+ snew(fr->gid2nblists, ir->opts.ngener*ir->opts.ngener);
+ }
+ }
+
+ if (ir->adress)
+ {
+ fr->nnblists *= 2;
+ }
+
+ snew(fr->nblists, fr->nnblists);
+
+ /* This code automatically gives table length tabext without cut-off's,
+ * in that case grompp should already have checked that we do not need
+ * normal tables and we only generate tables for 1-4 interactions.
+ */
+ rtab = ir->rlistlong + ir->tabext;
+
+ if (bTab)
+ {
+ /* make tables for ordinary interactions */
+ if (bNormalnblists)
+ {
+ make_nbf_tables(fp, oenv, fr, rtab, cr, tabfn, NULL, NULL, &fr->nblists[0]);
+ if (ir->adress)
+ {
+ make_nbf_tables(fp, oenv, fr, rtab, cr, tabfn, NULL, NULL, &fr->nblists[fr->nnblists/2]);
+ }
+ if (!bSep14tab)
+ {
+ fr->tab14 = fr->nblists[0].table_elec_vdw;
+ }
+ m = 1;
+ }
+ else
+ {
+ m = 0;
+ }
+ if (negptable > 0)
+ {
+ /* Read the special tables for certain energy group pairs */
+ nm_ind = mtop->groups.grps[egcENER].nm_ind;
+ for (egi = 0; egi < negp_pp; egi++)
+ {
+ for (egj = egi; egj < negp_pp; egj++)
+ {
+ egp_flags = ir->opts.egp_flags[GID(egi, egj, ir->opts.ngener)];
+ if ((egp_flags & EGP_TABLE) && !(egp_flags & EGP_EXCL))
+ {
+ nbl = &(fr->nblists[m]);
+ if (fr->nnblists > 1)
+ {
+ fr->gid2nblists[GID(egi, egj, ir->opts.ngener)] = m;
+ }
+ /* Read the table file with the two energy groups names appended */
+ make_nbf_tables(fp, oenv, fr, rtab, cr, tabfn,
+ *mtop->groups.grpname[nm_ind[egi]],
+ *mtop->groups.grpname[nm_ind[egj]],
+ &fr->nblists[m]);
+ if (ir->adress)
+ {
+ make_nbf_tables(fp, oenv, fr, rtab, cr, tabfn,
+ *mtop->groups.grpname[nm_ind[egi]],
+ *mtop->groups.grpname[nm_ind[egj]],
+ &fr->nblists[fr->nnblists/2+m]);
+ }
+ m++;
+ }
+ else if (fr->nnblists > 1)
+ {
+ fr->gid2nblists[GID(egi, egj, ir->opts.ngener)] = 0;
+ }
+ }
+ }
+ }
+ }
+ if (bSep14tab)
+ {
+ /* generate extra tables with plain Coulomb for 1-4 interactions only */
+ fr->tab14 = make_tables(fp, oenv, fr, MASTER(cr), tabpfn, rtab,
+ GMX_MAKETABLES_14ONLY);
+ }
+
+ /* Read AdResS Thermo Force table if needed */
+ if (fr->adress_icor == eAdressICThermoForce)
+ {
+ /* old todo replace */
+
+ if (ir->adress->n_tf_grps > 0)
+ {
+ make_adress_tf_tables(fp, oenv, fr, ir, tabfn, mtop, box);
+
+ }
+ else
+ {
+ /* load the default table */
+ snew(fr->atf_tabs, 1);
+ fr->atf_tabs[DEFAULT_TF_TABLE] = make_atf_table(fp, oenv, fr, tabafn, box);
+ }
+ }
+
+ /* Wall stuff */
+ fr->nwall = ir->nwall;
+ if (ir->nwall && ir->wall_type == ewtTABLE)
+ {
+ make_wall_tables(fp, oenv, ir, tabfn, &mtop->groups, fr);
+ }
+
+ if (fcd && tabbfn)
+ {
+ fcd->bondtab = make_bonded_tables(fp,
+ F_TABBONDS, F_TABBONDSNC,
+ mtop, tabbfn, "b");
+ fcd->angletab = make_bonded_tables(fp,
+ F_TABANGLES, -1,
+ mtop, tabbfn, "a");
+ fcd->dihtab = make_bonded_tables(fp,
+ F_TABDIHS, -1,
+ mtop, tabbfn, "d");
+ }
+ else
+ {
+ if (debug)
+ {
+ fprintf(debug, "No fcdata or table file name passed, can not read table, can not do bonded interactions\n");
+ }
+ }
+
+ /* QM/MM initialization if requested
+ */
+ if (ir->bQMMM)
+ {
+ fprintf(stderr, "QM/MM calculation requested.\n");
+ }
+
+ fr->bQMMM = ir->bQMMM;
+ fr->qr = mk_QMMMrec();
+
+ /* Set all the static charge group info */
+ fr->cginfo_mb = init_cginfo_mb(fp, mtop, fr, bNoSolvOpt,
+ &fr->bExcl_IntraCGAll_InterCGNone);
+ if (DOMAINDECOMP(cr))
+ {
+ fr->cginfo = NULL;
+ }
+ else
+ {
+ fr->cginfo = cginfo_expand(mtop->nmolblock, fr->cginfo_mb);
+ }
+
+ if (!DOMAINDECOMP(cr))
+ {
+ /* When using particle decomposition, the effect of the second argument,
+ * which sets fr->hcg, is corrected later in do_md and init_em.
+ */
+ forcerec_set_ranges(fr, ncg_mtop(mtop), ncg_mtop(mtop),
+ mtop->natoms, mtop->natoms, mtop->natoms);
+ }
+
+ fr->print_force = print_force;
+
+
+ /* coarse load balancing vars */
+ fr->t_fnbf = 0.;
+ fr->t_wait = 0.;
+ fr->timesteps = 0;
+
+ /* Initialize neighbor search */
+ init_ns(fp, cr, &fr->ns, fr, mtop);
+
+ if (cr->duty & DUTY_PP)
+ {
+ gmx_nonbonded_setup(fr, bGenericKernelOnly);
+ /*
+ if (ir->bAdress)
+ {
+ gmx_setup_adress_kernels(fp,bGenericKernelOnly);
+ }
+ */
+ }
+
+ /* Initialize the thread working data for bonded interactions */
+ init_forcerec_f_threads(fr, mtop->groups.grps[egcENER].nr);
+
+ snew(fr->excl_load, fr->nthreads+1);
+
+ if (fr->cutoff_scheme == ecutsVERLET)
+ {
+ if (ir->rcoulomb != ir->rvdw)
+ {
+ gmx_fatal(FARGS, "With Verlet lists rcoulomb and rvdw should be identical");
+ }
+
+ init_nb_verlet(fp, &fr->nbv, ir, fr, cr, nbpu_opt);
+ }
+
+ /* fr->ic is used both by verlet and group kernels (to some extent) now */
+ init_interaction_const(fp, &fr->ic, fr, rtab);
+ if (ir->eDispCorr != edispcNO)
+ {
+ calc_enervirdiff(fp, ir->eDispCorr, fr);
+ }
+}
+
+#define pr_real(fp, r) fprintf(fp, "%s: %e\n",#r, r)
+#define pr_int(fp, i) fprintf((fp), "%s: %d\n",#i, i)
+#define pr_bool(fp, b) fprintf((fp), "%s: %s\n",#b, bool_names[b])
+
+void pr_forcerec(FILE *fp, t_forcerec *fr)
+{
+ int i;
+
+ pr_real(fp, fr->rlist);
+ pr_real(fp, fr->rcoulomb);
+ pr_real(fp, fr->fudgeQQ);
+ pr_bool(fp, fr->bGrid);
+ pr_bool(fp, fr->bTwinRange);
+ /*pr_int(fp,fr->cg0);
+ pr_int(fp,fr->hcg);*/
+ for (i = 0; i < fr->nnblists; i++)
+ {
+ pr_int(fp, fr->nblists[i].table_elec_vdw.n);
+ }
+ pr_real(fp, fr->rcoulomb_switch);
+ pr_real(fp, fr->rcoulomb);
+
+ fflush(fp);
+}
+
+void forcerec_set_excl_load(t_forcerec *fr,
+ const gmx_localtop_t *top, const t_commrec *cr)
+{
+ const int *ind, *a;
+ int t, i, j, ntot, n, ntarget;
+
+ if (cr != NULL && PARTDECOMP(cr))
+ {
+ /* No OpenMP with particle decomposition */
+ pd_at_range(cr,
+ &fr->excl_load[0],
+ &fr->excl_load[1]);
+
+ return;
+ }
+
+ ind = top->excls.index;
+ a = top->excls.a;
+
+ ntot = 0;
+ for (i = 0; i < top->excls.nr; i++)
+ {
+ for (j = ind[i]; j < ind[i+1]; j++)
+ {
+ if (a[j] > i)
+ {
+ ntot++;
+ }
+ }
+ }
+
+ fr->excl_load[0] = 0;
+ n = 0;
+ i = 0;
+ for (t = 1; t <= fr->nthreads; t++)
+ {
+ ntarget = (ntot*t)/fr->nthreads;
+ while (i < top->excls.nr && n < ntarget)
+ {
+ for (j = ind[i]; j < ind[i+1]; j++)
+ {
+ if (a[j] > i)
+ {
+ n++;
+ }
+ }
+ i++;
+ }
+ fr->excl_load[t] = i;
+ }
+}
--- /dev/null
- init_df_history(&state->dfhist, ir->fepvals->n_lambda, ir->expandedvals->init_wl_delta);
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * VERSION 3.2.0
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2004, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * GROwing Monsters And Cloning Shrimps
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdio.h>
+#include "typedefs.h"
+#include "tpxio.h"
+#include "smalloc.h"
+#include "vec.h"
+#include "main.h"
+#include "mvdata.h"
+#include "gmx_fatal.h"
+#include "symtab.h"
+#include "txtdump.h"
+#include "mdatoms.h"
+#include "mdrun.h"
+#include "statutil.h"
+#include "names.h"
+#include "calcgrid.h"
+#include "gmx_random.h"
+#include "update.h"
+#include "mdebin.h"
+
+#define BUFSIZE 256
+
+#define NOT_FINISHED(l1, l2) \
+ printf("not finished yet: lines %d .. %d in %s\n", l1, l2, __FILE__)
+
+void set_state_entries(t_state *state, const t_inputrec *ir, int nnodes)
+{
+ int nnhpres;
+
+ /* The entries in the state in the tpx file might not correspond
+ * with what is needed, so we correct this here.
+ */
+ state->flags = 0;
+ if (ir->efep != efepNO || ir->bExpanded)
+ {
+ state->flags |= (1<<estLAMBDA);
+ state->flags |= (1<<estFEPSTATE);
+ }
+ state->flags |= (1<<estX);
+ if (state->lambda == NULL)
+ {
+ snew(state->lambda, efptNR);
+ }
+ if (state->x == NULL)
+ {
+ snew(state->x, state->nalloc);
+ }
+ if (EI_DYNAMICS(ir->eI))
+ {
+ state->flags |= (1<<estV);
+ if (state->v == NULL)
+ {
+ snew(state->v, state->nalloc);
+ }
+ }
+ if (ir->eI == eiSD2)
+ {
+ state->flags |= (1<<estSDX);
+ if (state->sd_X == NULL)
+ {
+ /* sd_X is not stored in the tpx file, so we need to allocate it */
+ snew(state->sd_X, state->nalloc);
+ }
+ }
+ if (ir->eI == eiCG)
+ {
+ state->flags |= (1<<estCGP);
+ if (state->cg_p == NULL)
+ {
+ /* cg_p is not stored in the tpx file, so we need to allocate it */
+ snew(state->cg_p, state->nalloc);
+ }
+ }
+ if (EI_SD(ir->eI) || ir->eI == eiBD || ir->etc == etcVRESCALE || ETC_ANDERSEN(ir->etc))
+ {
+ state->nrng = gmx_rng_n();
+ state->nrngi = 1;
+ if (EI_SD(ir->eI) || ir->eI == eiBD || ETC_ANDERSEN(ir->etc))
+ {
+ /* This will be correct later with DD */
+ state->nrng *= nnodes;
+ state->nrngi *= nnodes;
+ }
+ state->flags |= ((1<<estLD_RNG) | (1<<estLD_RNGI));
+ snew(state->ld_rng, state->nrng);
+ snew(state->ld_rngi, state->nrngi);
+ }
+ else
+ {
+ state->nrng = 0;
+ }
+
+ if (ir->bExpanded)
+ {
+ state->nmcrng = gmx_rng_n();
+ snew(state->mc_rng, state->nmcrng);
+ snew(state->mc_rngi, 1);
+ }
+
+ state->nnhpres = 0;
+ if (ir->ePBC != epbcNONE)
+ {
+ state->flags |= (1<<estBOX);
+ if (PRESERVE_SHAPE(*ir))
+ {
+ state->flags |= (1<<estBOX_REL);
+ }
+ if ((ir->epc == epcPARRINELLORAHMAN) || (ir->epc == epcMTTK))
+ {
+ state->flags |= (1<<estBOXV);
+ }
+ if (ir->epc != epcNO)
+ {
+ if (IR_NPT_TROTTER(ir) || (IR_NPH_TROTTER(ir)))
+ {
+ state->nnhpres = 1;
+ state->flags |= (1<<estNHPRES_XI);
+ state->flags |= (1<<estNHPRES_VXI);
+ state->flags |= (1<<estSVIR_PREV);
+ state->flags |= (1<<estFVIR_PREV);
+ state->flags |= (1<<estVETA);
+ state->flags |= (1<<estVOL0);
+ }
+ else
+ {
+ state->flags |= (1<<estPRES_PREV);
+ }
+ }
+ }
+
+ if (ir->etc == etcNOSEHOOVER)
+ {
+ state->flags |= (1<<estNH_XI);
+ state->flags |= (1<<estNH_VXI);
+ }
+
+ if (ir->etc == etcVRESCALE)
+ {
+ state->flags |= (1<<estTC_INT);
+ }
+
+ init_gtc_state(state, state->ngtc, state->nnhpres, ir->opts.nhchainlength); /* allocate the space for nose-hoover chains */
+ init_ekinstate(&state->ekinstate, ir);
+
+ init_energyhistory(&state->enerhist);
++ init_df_history(&state->dfhist, ir->fepvals->n_lambda);
+}
+
+
+void init_parallel(t_commrec *cr, t_inputrec *inputrec,
+ gmx_mtop_t *mtop)
+{
+ bcast_ir_mtop(cr, inputrec, mtop);
+
+ if (inputrec->eI == eiBD || EI_SD(inputrec->eI) || ETC_ANDERSEN(inputrec->etc))
+ {
+ /* Make sure the random seeds are different on each node */
+ inputrec->ld_seed += cr->nodeid;
+ }
+}
--- /dev/null
- if (fepvals->delta_lambda != 0)
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * VERSION 3.2.0
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2004, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * Gallium Rubidium Oxygen Manganese Argon Carbon Silicon
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include "typedefs.h"
+#include "string2.h"
+#include "smalloc.h"
+#include "mdrun.h"
+#include "domdec.h"
+#include "mtop_util.h"
+#include "gmx_wallcycle.h"
+#include "vcm.h"
+#include "nrnb.h"
+#include "macros.h"
+#include "md_logging.h"
+#include "md_support.h"
+
+/* Is the signal in one simulation independent of other simulations? */
+gmx_bool gs_simlocal[eglsNR] = { TRUE, FALSE, FALSE, TRUE };
+
+/* check which of the multisim simulations has the shortest number of
+ steps and return that number of nsteps */
+gmx_large_int_t get_multisim_nsteps(const t_commrec *cr,
+ gmx_large_int_t nsteps)
+{
+ gmx_large_int_t steps_out;
+
+ if MASTER(cr)
+ {
+ gmx_large_int_t *buf;
+ int s;
+
+ snew(buf, cr->ms->nsim);
+
+ buf[cr->ms->sim] = nsteps;
+ gmx_sumli_sim(cr->ms->nsim, buf, cr->ms);
+
+ steps_out = -1;
+ for (s = 0; s < cr->ms->nsim; s++)
+ {
+ /* find the smallest positive number */
+ if (buf[s] >= 0 && ((steps_out < 0) || (buf[s] < steps_out)) )
+ {
+ steps_out = buf[s];
+ }
+ }
+ sfree(buf);
+
+ /* if we're the limiting simulation, don't do anything */
+ if (steps_out >= 0 && steps_out < nsteps)
+ {
+ char strbuf[255];
+ snprintf(strbuf, 255, "Will stop simulation %%d after %s steps (another simulation will end then).\n", gmx_large_int_pfmt);
+ fprintf(stderr, strbuf, cr->ms->sim, steps_out);
+ }
+ }
+ /* broadcast to non-masters */
+ gmx_bcast(sizeof(gmx_large_int_t), &steps_out, cr);
+ return steps_out;
+}
+
+int multisim_min(const gmx_multisim_t *ms, int nmin, int n)
+{
+ int *buf;
+ gmx_bool bPos, bEqual;
+ int s, d;
+
+ snew(buf, ms->nsim);
+ buf[ms->sim] = n;
+ gmx_sumi_sim(ms->nsim, buf, ms);
+ bPos = TRUE;
+ bEqual = TRUE;
+ for (s = 0; s < ms->nsim; s++)
+ {
+ bPos = bPos && (buf[s] > 0);
+ bEqual = bEqual && (buf[s] == buf[0]);
+ }
+ if (bPos)
+ {
+ if (bEqual)
+ {
+ nmin = min(nmin, buf[0]);
+ }
+ else
+ {
+ /* Find the least common multiple */
+ for (d = 2; d < nmin; d++)
+ {
+ s = 0;
+ while (s < ms->nsim && d % buf[s] == 0)
+ {
+ s++;
+ }
+ if (s == ms->nsim)
+ {
+ /* We found the LCM and it is less than nmin */
+ nmin = d;
+ break;
+ }
+ }
+ }
+ }
+ sfree(buf);
+
+ return nmin;
+}
+
+int multisim_nstsimsync(const t_commrec *cr,
+ const t_inputrec *ir, int repl_ex_nst)
+{
+ int nmin;
+
+ if (MASTER(cr))
+ {
+ nmin = INT_MAX;
+ nmin = multisim_min(cr->ms, nmin, ir->nstlist);
+ nmin = multisim_min(cr->ms, nmin, ir->nstcalcenergy);
+ nmin = multisim_min(cr->ms, nmin, repl_ex_nst);
+ if (nmin == INT_MAX)
+ {
+ gmx_fatal(FARGS, "Can not find an appropriate interval for inter-simulation communication, since nstlist, nstcalcenergy and -replex are all <= 0");
+ }
+ /* Avoid inter-simulation communication at every (second) step */
+ if (nmin <= 2)
+ {
+ nmin = 10;
+ }
+ }
+
+ gmx_bcast(sizeof(int), &nmin, cr);
+
+ return nmin;
+}
+
+void init_global_signals(globsig_t *gs, const t_commrec *cr,
+ const t_inputrec *ir, int repl_ex_nst)
+{
+ int i;
+
+ if (MULTISIM(cr))
+ {
+ gs->nstms = multisim_nstsimsync(cr, ir, repl_ex_nst);
+ if (debug)
+ {
+ fprintf(debug, "Syncing simulations for checkpointing and termination every %d steps\n", gs->nstms);
+ }
+ }
+ else
+ {
+ gs->nstms = 1;
+ }
+
+ for (i = 0; i < eglsNR; i++)
+ {
+ gs->sig[i] = 0;
+ gs->set[i] = 0;
+ }
+}
+
+void copy_coupling_state(t_state *statea, t_state *stateb,
+ gmx_ekindata_t *ekinda, gmx_ekindata_t *ekindb, t_grpopts* opts)
+{
+
+ /* MRS note -- might be able to get rid of some of the arguments. Look over it when it's all debugged */
+
+ int i, j, nc;
+
+ /* Make sure we have enough space for x and v */
+ if (statea->nalloc > stateb->nalloc)
+ {
+ stateb->nalloc = statea->nalloc;
+ srenew(stateb->x, stateb->nalloc);
+ srenew(stateb->v, stateb->nalloc);
+ }
+
+ stateb->natoms = statea->natoms;
+ stateb->ngtc = statea->ngtc;
+ stateb->nnhpres = statea->nnhpres;
+ stateb->veta = statea->veta;
+ if (ekinda)
+ {
+ copy_mat(ekinda->ekin, ekindb->ekin);
+ for (i = 0; i < stateb->ngtc; i++)
+ {
+ ekindb->tcstat[i].T = ekinda->tcstat[i].T;
+ ekindb->tcstat[i].Th = ekinda->tcstat[i].Th;
+ copy_mat(ekinda->tcstat[i].ekinh, ekindb->tcstat[i].ekinh);
+ copy_mat(ekinda->tcstat[i].ekinf, ekindb->tcstat[i].ekinf);
+ ekindb->tcstat[i].ekinscalef_nhc = ekinda->tcstat[i].ekinscalef_nhc;
+ ekindb->tcstat[i].ekinscaleh_nhc = ekinda->tcstat[i].ekinscaleh_nhc;
+ ekindb->tcstat[i].vscale_nhc = ekinda->tcstat[i].vscale_nhc;
+ }
+ }
+ copy_rvecn(statea->x, stateb->x, 0, stateb->natoms);
+ copy_rvecn(statea->v, stateb->v, 0, stateb->natoms);
+ copy_mat(statea->box, stateb->box);
+ copy_mat(statea->box_rel, stateb->box_rel);
+ copy_mat(statea->boxv, stateb->boxv);
+
+ for (i = 0; i < stateb->ngtc; i++)
+ {
+ nc = i*opts->nhchainlength;
+ for (j = 0; j < opts->nhchainlength; j++)
+ {
+ stateb->nosehoover_xi[nc+j] = statea->nosehoover_xi[nc+j];
+ stateb->nosehoover_vxi[nc+j] = statea->nosehoover_vxi[nc+j];
+ }
+ }
+ if (stateb->nhpres_xi != NULL)
+ {
+ for (i = 0; i < stateb->nnhpres; i++)
+ {
+ nc = i*opts->nhchainlength;
+ for (j = 0; j < opts->nhchainlength; j++)
+ {
+ stateb->nhpres_xi[nc+j] = statea->nhpres_xi[nc+j];
+ stateb->nhpres_vxi[nc+j] = statea->nhpres_vxi[nc+j];
+ }
+ }
+ }
+}
+
+real compute_conserved_from_auxiliary(t_inputrec *ir, t_state *state, t_extmass *MassQ)
+{
+ real quantity = 0;
+ switch (ir->etc)
+ {
+ case etcNO:
+ break;
+ case etcBERENDSEN:
+ break;
+ case etcNOSEHOOVER:
+ quantity = NPT_energy(ir, state, MassQ);
+ break;
+ case etcVRESCALE:
+ quantity = vrescale_energy(&(ir->opts), state->therm_integral);
+ break;
+ default:
+ break;
+ }
+ return quantity;
+}
+
+void compute_globals(FILE *fplog, gmx_global_stat_t gstat, t_commrec *cr, t_inputrec *ir,
+ t_forcerec *fr, gmx_ekindata_t *ekind,
+ t_state *state, t_state *state_global, t_mdatoms *mdatoms,
+ t_nrnb *nrnb, t_vcm *vcm, gmx_wallcycle_t wcycle,
+ gmx_enerdata_t *enerd, tensor force_vir, tensor shake_vir, tensor total_vir,
+ tensor pres, rvec mu_tot, gmx_constr_t constr,
+ globsig_t *gs, gmx_bool bInterSimGS,
+ matrix box, gmx_mtop_t *top_global,
+ gmx_bool *bSumEkinhOld, int flags)
+{
+ int i, gsi;
+ real gs_buf[eglsNR];
+ tensor corr_vir, corr_pres;
+ gmx_bool bEner, bPres, bTemp, bVV;
+ gmx_bool bRerunMD, bStopCM, bGStat, bIterate,
+ bFirstIterate, bReadEkin, bEkinAveVel, bScaleEkin, bConstrain;
+ real ekin, temp, prescorr, enercorr, dvdlcorr, dvdl_ekin;
+
+ /* translate CGLO flags to gmx_booleans */
+ bRerunMD = flags & CGLO_RERUNMD;
+ bStopCM = flags & CGLO_STOPCM;
+ bGStat = flags & CGLO_GSTAT;
+
+ bReadEkin = (flags & CGLO_READEKIN);
+ bScaleEkin = (flags & CGLO_SCALEEKIN);
+ bEner = flags & CGLO_ENERGY;
+ bTemp = flags & CGLO_TEMPERATURE;
+ bPres = (flags & CGLO_PRESSURE);
+ bConstrain = (flags & CGLO_CONSTRAINT);
+ bIterate = (flags & CGLO_ITERATE);
+ bFirstIterate = (flags & CGLO_FIRSTITERATE);
+
+ /* we calculate a full state kinetic energy either with full-step velocity verlet
+ or half step where we need the pressure */
+
+ bEkinAveVel = (ir->eI == eiVV || (ir->eI == eiVVAK && bPres) || bReadEkin);
+
+ /* in initalization, it sums the shake virial in vv, and to
+ sums ekinh_old in leapfrog (or if we are calculating ekinh_old) for other reasons */
+
+ /* ########## Kinetic energy ############## */
+
+ if (bTemp)
+ {
+ /* Non-equilibrium MD: this is parallellized, but only does communication
+ * when there really is NEMD.
+ */
+
+ if (PAR(cr) && (ekind->bNEMD))
+ {
+ accumulate_u(cr, &(ir->opts), ekind);
+ }
+ debug_gmx();
+ if (bReadEkin)
+ {
+ restore_ekinstate_from_state(cr, ekind, &state_global->ekinstate);
+ }
+ else
+ {
+
+ calc_ke_part(state, &(ir->opts), mdatoms, ekind, nrnb, bEkinAveVel, bIterate);
+ }
+
+ debug_gmx();
+ }
+
+ /* Calculate center of mass velocity if necessary, also parallellized */
+ if (bStopCM)
+ {
+ calc_vcm_grp(mdatoms->start, mdatoms->homenr, mdatoms,
+ state->x, state->v, vcm);
+ }
+
+ if (bTemp || bStopCM || bPres || bEner || bConstrain)
+ {
+ if (!bGStat)
+ {
+ /* We will not sum ekinh_old,
+ * so signal that we still have to do it.
+ */
+ *bSumEkinhOld = TRUE;
+
+ }
+ else
+ {
+ if (gs != NULL)
+ {
+ for (i = 0; i < eglsNR; i++)
+ {
+ gs_buf[i] = gs->sig[i];
+ }
+ }
+ if (PAR(cr))
+ {
+ wallcycle_start(wcycle, ewcMoveE);
+ global_stat(fplog, gstat, cr, enerd, force_vir, shake_vir, mu_tot,
+ ir, ekind, constr, bStopCM ? vcm : NULL,
+ gs != NULL ? eglsNR : 0, gs_buf,
+ top_global, state,
+ *bSumEkinhOld, flags);
+ wallcycle_stop(wcycle, ewcMoveE);
+ }
+ if (gs != NULL)
+ {
+ if (MULTISIM(cr) && bInterSimGS)
+ {
+ if (MASTER(cr))
+ {
+ /* Communicate the signals between the simulations */
+ gmx_sum_sim(eglsNR, gs_buf, cr->ms);
+ }
+ /* Communicate the signals form the master to the others */
+ gmx_bcast(eglsNR*sizeof(gs_buf[0]), gs_buf, cr);
+ }
+ for (i = 0; i < eglsNR; i++)
+ {
+ if (bInterSimGS || gs_simlocal[i])
+ {
+ /* Set the communicated signal only when it is non-zero,
+ * since signals might not be processed at each MD step.
+ */
+ gsi = (gs_buf[i] >= 0 ?
+ (int)(gs_buf[i] + 0.5) :
+ (int)(gs_buf[i] - 0.5));
+ if (gsi != 0)
+ {
+ gs->set[i] = gsi;
+ }
+ /* Turn off the local signal */
+ gs->sig[i] = 0;
+ }
+ }
+ }
+ *bSumEkinhOld = FALSE;
+ }
+ }
+
+ if (!ekind->bNEMD && debug && bTemp && (vcm->nr > 0))
+ {
+ correct_ekin(debug,
+ mdatoms->start, mdatoms->start+mdatoms->homenr,
+ state->v, vcm->group_p[0],
+ mdatoms->massT, mdatoms->tmass, ekind->ekin);
+ }
+
+ /* Do center of mass motion removal */
+ if (bStopCM)
+ {
+ check_cm_grp(fplog, vcm, ir, 1);
+ do_stopcm_grp(mdatoms->start, mdatoms->homenr, mdatoms->cVCM,
+ state->x, state->v, vcm);
+ inc_nrnb(nrnb, eNR_STOPCM, mdatoms->homenr);
+ }
+
+ if (bEner)
+ {
+ /* Calculate the amplitude of the cosine velocity profile */
+ ekind->cosacc.vcos = ekind->cosacc.mvcos/mdatoms->tmass;
+ }
+
+ if (bTemp)
+ {
+ /* Sum the kinetic energies of the groups & calc temp */
+ /* compute full step kinetic energies if vv, or if vv-avek and we are computing the pressure with IR_NPT_TROTTER */
+ /* three maincase: VV with AveVel (md-vv), vv with AveEkin (md-vv-avek), leap with AveEkin (md).
+ Leap with AveVel is not supported; it's not clear that it will actually work.
+ bEkinAveVel: If TRUE, we simply multiply ekin by ekinscale to get a full step kinetic energy.
+ If FALSE, we average ekinh_old and ekinh*ekinscale_nhc to get an averaged half step kinetic energy.
+ bSaveEkinOld: If TRUE (in the case of iteration = bIterate is TRUE), we don't reset the ekinscale_nhc.
+ If FALSE, we go ahead and erase over it.
+ */
+ enerd->term[F_TEMP] = sum_ekin(&(ir->opts), ekind, &dvdl_ekin,
+ bEkinAveVel, bScaleEkin);
+ enerd->dvdl_lin[efptMASS] = (double) dvdl_ekin;
+
+ enerd->term[F_EKIN] = trace(ekind->ekin);
+ }
+
+ /* ########## Long range energy information ###### */
+
+ if (bEner || bPres || bConstrain)
+ {
+ calc_dispcorr(fplog, ir, fr, 0, top_global->natoms, box, state->lambda[efptVDW],
+ corr_pres, corr_vir, &prescorr, &enercorr, &dvdlcorr);
+ }
+
+ if (bEner && bFirstIterate)
+ {
+ enerd->term[F_DISPCORR] = enercorr;
+ enerd->term[F_EPOT] += enercorr;
+ enerd->term[F_DVDL_VDW] += dvdlcorr;
+ }
+
+ /* ########## Now pressure ############## */
+ if (bPres || bConstrain)
+ {
+
+ m_add(force_vir, shake_vir, total_vir);
+
+ /* Calculate pressure and apply LR correction if PPPM is used.
+ * Use the box from last timestep since we already called update().
+ */
+
+ enerd->term[F_PRES] = calc_pres(fr->ePBC, ir->nwall, box, ekind->ekin, total_vir, pres);
+
+ /* Calculate long range corrections to pressure and energy */
+ /* this adds to enerd->term[F_PRES] and enerd->term[F_ETOT],
+ and computes enerd->term[F_DISPCORR]. Also modifies the
+ total_vir and pres tesors */
+
+ m_add(total_vir, corr_vir, total_vir);
+ m_add(pres, corr_pres, pres);
+ enerd->term[F_PDISPCORR] = prescorr;
+ enerd->term[F_PRES] += prescorr;
+ }
+}
+
+void check_nst_param(FILE *fplog, t_commrec *cr,
+ const char *desc_nst, int nst,
+ const char *desc_p, int *p)
+{
+ if (*p > 0 && *p % nst != 0)
+ {
+ /* Round up to the next multiple of nst */
+ *p = ((*p)/nst + 1)*nst;
+ md_print_warn(cr, fplog,
+ "NOTE: %s changes %s to %d\n", desc_nst, desc_p, *p);
+ }
+}
+
+void set_current_lambdas(gmx_large_int_t step, t_lambda *fepvals, gmx_bool bRerunMD,
+ t_trxframe *rerun_fr, t_state *state_global, t_state *state, double lam0[])
+/* find the current lambdas. If rerunning, we either read in a state, or a lambda value,
+ requiring different logic. */
+{
+ real frac;
+ int i, fep_state = 0;
+ if (bRerunMD)
+ {
+ if (rerun_fr->bLambda)
+ {
++ if (fepvals->delta_lambda==0)
+ {
+ state_global->lambda[efptFEP] = rerun_fr->lambda;
+ for (i = 0; i < efptNR; i++)
+ {
+ if (i != efptFEP)
+ {
+ state->lambda[i] = state_global->lambda[i];
+ }
+ }
+ }
+ else
+ {
+ /* find out between which two value of lambda we should be */
+ frac = (step*fepvals->delta_lambda);
+ fep_state = floor(frac*fepvals->n_lambda);
+ /* interpolate between this state and the next */
+ /* this assumes that the initial lambda corresponds to lambda==0, which is verified in grompp */
+ frac = (frac*fepvals->n_lambda)-fep_state;
+ for (i = 0; i < efptNR; i++)
+ {
+ state_global->lambda[i] = lam0[i] + (fepvals->all_lambda[i][fep_state]) +
+ frac*(fepvals->all_lambda[i][fep_state+1]-fepvals->all_lambda[i][fep_state]);
+ }
+ }
+ }
+ else if (rerun_fr->bFepState)
+ {
+ state_global->fep_state = rerun_fr->fep_state;
+ for (i = 0; i < efptNR; i++)
+ {
+ state_global->lambda[i] = fepvals->all_lambda[i][fep_state];
+ }
+ }
+ }
+ else
+ {
+ if (fepvals->delta_lambda != 0)
+ {
+ /* find out between which two value of lambda we should be */
+ frac = (step*fepvals->delta_lambda);
+ if (fepvals->n_lambda > 0)
+ {
+ fep_state = floor(frac*fepvals->n_lambda);
+ /* interpolate between this state and the next */
+ /* this assumes that the initial lambda corresponds to lambda==0, which is verified in grompp */
+ frac = (frac*fepvals->n_lambda)-fep_state;
+ for (i = 0; i < efptNR; i++)
+ {
+ state_global->lambda[i] = lam0[i] + (fepvals->all_lambda[i][fep_state]) +
+ frac*(fepvals->all_lambda[i][fep_state+1]-fepvals->all_lambda[i][fep_state]);
+ }
+ }
+ else
+ {
+ for (i = 0; i < efptNR; i++)
+ {
+ state_global->lambda[i] = lam0[i] + frac;
+ }
+ }
+ }
++ else
++ {
++ if (state->fep_state > 0) {
++ state_global->fep_state = state->fep_state; /* state->fep is the one updated by bExpanded */
++ for (i = 0; i < efptNR; i++)
++ {
++ state_global->lambda[i] = fepvals->all_lambda[i][state_global->fep_state];
++ }
++ }
++ }
+ }
+ for (i = 0; i < efptNR; i++)
+ {
+ state->lambda[i] = state_global->lambda[i];
+ }
+}
+
+static void min_zero(int *n, int i)
+{
+ if (i > 0 && (*n == 0 || i < *n))
+ {
+ *n = i;
+ }
+}
+
+static int lcd4(int i1, int i2, int i3, int i4)
+{
+ int nst;
+
+ nst = 0;
+ min_zero(&nst, i1);
+ min_zero(&nst, i2);
+ min_zero(&nst, i3);
+ min_zero(&nst, i4);
+ if (nst == 0)
+ {
+ gmx_incons("All 4 inputs for determininig nstglobalcomm are <= 0");
+ }
+
+ while (nst > 1 && ((i1 > 0 && i1 % nst != 0) ||
+ (i2 > 0 && i2 % nst != 0) ||
+ (i3 > 0 && i3 % nst != 0) ||
+ (i4 > 0 && i4 % nst != 0)))
+ {
+ nst--;
+ }
+
+ return nst;
+}
+
+int check_nstglobalcomm(FILE *fplog, t_commrec *cr,
+ int nstglobalcomm, t_inputrec *ir)
+{
+ if (!EI_DYNAMICS(ir->eI))
+ {
+ nstglobalcomm = 1;
+ }
+
+ if (nstglobalcomm == -1)
+ {
+ if (!(ir->nstcalcenergy > 0 ||
+ ir->nstlist > 0 ||
+ ir->etc != etcNO ||
+ ir->epc != epcNO))
+ {
+ nstglobalcomm = 10;
+ if (ir->nstenergy > 0 && ir->nstenergy < nstglobalcomm)
+ {
+ nstglobalcomm = ir->nstenergy;
+ }
+ }
+ else
+ {
+ /* Ensure that we do timely global communication for
+ * (possibly) each of the four following options.
+ */
+ nstglobalcomm = lcd4(ir->nstcalcenergy,
+ ir->nstlist,
+ ir->etc != etcNO ? ir->nsttcouple : 0,
+ ir->epc != epcNO ? ir->nstpcouple : 0);
+ }
+ }
+ else
+ {
+ if (ir->nstlist > 0 &&
+ nstglobalcomm > ir->nstlist && nstglobalcomm % ir->nstlist != 0)
+ {
+ nstglobalcomm = (nstglobalcomm / ir->nstlist)*ir->nstlist;
+ md_print_warn(cr, fplog, "WARNING: nstglobalcomm is larger than nstlist, but not a multiple, setting it to %d\n", nstglobalcomm);
+ }
+ if (ir->nstcalcenergy > 0)
+ {
+ check_nst_param(fplog, cr, "-gcom", nstglobalcomm,
+ "nstcalcenergy", &ir->nstcalcenergy);
+ }
+ if (ir->etc != etcNO && ir->nsttcouple > 0)
+ {
+ check_nst_param(fplog, cr, "-gcom", nstglobalcomm,
+ "nsttcouple", &ir->nsttcouple);
+ }
+ if (ir->epc != epcNO && ir->nstpcouple > 0)
+ {
+ check_nst_param(fplog, cr, "-gcom", nstglobalcomm,
+ "nstpcouple", &ir->nstpcouple);
+ }
+
+ check_nst_param(fplog, cr, "-gcom", nstglobalcomm,
+ "nstenergy", &ir->nstenergy);
+
+ check_nst_param(fplog, cr, "-gcom", nstglobalcomm,
+ "nstlog", &ir->nstlog);
+ }
+
+ if (ir->comm_mode != ecmNO && ir->nstcomm < nstglobalcomm)
+ {
+ md_print_warn(cr, fplog, "WARNING: Changing nstcomm from %d to %d\n",
+ ir->nstcomm, nstglobalcomm);
+ ir->nstcomm = nstglobalcomm;
+ }
+
+ return nstglobalcomm;
+}
+
+void check_ir_old_tpx_versions(t_commrec *cr, FILE *fplog,
+ t_inputrec *ir, gmx_mtop_t *mtop)
+{
+ /* Check required for old tpx files */
+ if (IR_TWINRANGE(*ir) && ir->nstlist > 1 &&
+ ir->nstcalcenergy % ir->nstlist != 0)
+ {
+ md_print_warn(cr, fplog, "Old tpr file with twin-range settings: modifying energy calculation and/or T/P-coupling frequencies\n");
+
+ if (gmx_mtop_ftype_count(mtop, F_CONSTR) +
+ gmx_mtop_ftype_count(mtop, F_CONSTRNC) > 0 &&
+ ir->eConstrAlg == econtSHAKE)
+ {
+ md_print_warn(cr, fplog, "With twin-range cut-off's and SHAKE the virial and pressure are incorrect\n");
+ if (ir->epc != epcNO)
+ {
+ gmx_fatal(FARGS, "Can not do pressure coupling with twin-range cut-off's and SHAKE");
+ }
+ }
+ check_nst_param(fplog, cr, "nstlist", ir->nstlist,
+ "nstcalcenergy", &ir->nstcalcenergy);
+ if (ir->epc != epcNO)
+ {
+ check_nst_param(fplog, cr, "nstlist", ir->nstlist,
+ "nstpcouple", &ir->nstpcouple);
+ }
+ check_nst_param(fplog, cr, "nstcalcenergy", ir->nstcalcenergy,
+ "nstenergy", &ir->nstenergy);
+ check_nst_param(fplog, cr, "nstcalcenergy", ir->nstcalcenergy,
+ "nstlog", &ir->nstlog);
+ if (ir->efep != efepNO)
+ {
+ check_nst_param(fplog, cr, "nstcalcenergy", ir->nstcalcenergy,
+ "nstdhdl", &ir->fepvals->nstdhdl);
+ }
+ }
+}
+
+void rerun_parallel_comm(t_commrec *cr, t_trxframe *fr,
+ gmx_bool *bNotLastFrame)
+{
+ gmx_bool bAlloc;
+ rvec *xp, *vp;
+
+ bAlloc = (fr->natoms == 0);
+
+ if (MASTER(cr) && !*bNotLastFrame)
+ {
+ fr->natoms = -1;
+ }
+ xp = fr->x;
+ vp = fr->v;
+ gmx_bcast(sizeof(*fr), fr, cr);
+ fr->x = xp;
+ fr->v = vp;
+
+ *bNotLastFrame = (fr->natoms >= 0);
+
+ if (*bNotLastFrame && PARTDECOMP(cr))
+ {
+ /* x and v are the only variable size quantities stored in trr
+ * that are required for rerun (f is not needed).
+ */
+ if (bAlloc)
+ {
+ snew(fr->x, fr->natoms);
+ snew(fr->v, fr->natoms);
+ }
+ if (fr->bX)
+ {
+ gmx_bcast(fr->natoms*sizeof(fr->x[0]), fr->x[0], cr);
+ }
+ if (fr->bV)
+ {
+ gmx_bcast(fr->natoms*sizeof(fr->v[0]), fr->v[0], cr);
+ }
+ }
+}
--- /dev/null
- #include "nbnxn_atomdata.h"
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2012, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <math.h>
+#include <string.h>
+#include "smalloc.h"
+#include "macros.h"
+#include "vec.h"
+#include "nbnxn_consts.h"
+#include "nbnxn_internal.h"
+#include "nbnxn_search.h"
- /* Set the diagonal cluster pair interaction mask setup data.
- * In the kernel we check 0 < j - i to generate the masks.
- * Here we store j - i for generating the mask for the first i (i=0);
- * we substract 0.5 to avoid rounding issues.
- * In the kernel we can subtract 1 to generate the mask for the next i.
- */
- const int simd_width = GMX_SIMD_WIDTH_HERE;
- int simd_4xn_diag_ind_size, simd_interaction_size, j;
-
- simd_4xn_diag_ind_size = max(NBNXN_CPU_CLUSTER_I_SIZE, simd_width);
- snew_aligned(nbat->simd_4xn_diagonal_j_minus_i,
- simd_4xn_diag_ind_size, NBNXN_MEM_ALIGN);
- for (j = 0; j < simd_4xn_diag_ind_size; j++)
- {
- nbat->simd_4xn_diagonal_j_minus_i[j] = j - 0.5;
- }
-
- snew_aligned(nbat->simd_2xnn_diagonal_j_minus_i,
- simd_width, NBNXN_MEM_ALIGN);
- for (j = 0; j < simd_width/2; j++)
- {
- /* The j-cluster size is half the SIMD width */
- nbat->simd_2xnn_diagonal_j_minus_i[j] = j - 0.5;
- /* The next half of the SIMD width is for i + 1 */
- nbat->simd_2xnn_diagonal_j_minus_i[simd_width/2+j] = j - 1 - 0.5;
- }
-
- /* We use up to 32 bits for exclusion masking.
- * The same masks are used for the 4xN and 2x(N+N) kernels.
- * The masks are read either into epi32 SIMD registers or into
- * real SIMD registers (together with a cast).
- * In single precision this means the real and epi32 SIMD registers
- * are of equal size.
- * In double precision the epi32 registers can be smaller than
- * the real registers, so depending on the architecture, we might
- * need to use two, identical, 32-bit masks per real.
- */
- simd_interaction_size = NBNXN_CPU_CLUSTER_I_SIZE*simd_width;
- snew_aligned(nbat->simd_exclusion_filter1, simd_interaction_size, NBNXN_MEM_ALIGN);
- snew_aligned(nbat->simd_exclusion_filter2, simd_interaction_size*2, NBNXN_MEM_ALIGN);
-
- for (j = 0; j < simd_interaction_size; j++)
- {
- /* Set the consecutive bits for filters pair exclusions masks */
- nbat->simd_exclusion_filter1[j] = (1U << j);
- nbat->simd_exclusion_filter2[j*2 + 0] = (1U << j);
- nbat->simd_exclusion_filter2[j*2 + 1] = (1U << j);
- }
+#include "gmx_omp_nthreads.h"
+
+/* Default nbnxn allocation routine, allocates NBNXN_MEM_ALIGN byte aligned */
+void nbnxn_alloc_aligned(void **ptr, size_t nbytes)
+{
+ *ptr = save_malloc_aligned("ptr", __FILE__, __LINE__, nbytes, 1, NBNXN_MEM_ALIGN);
+}
+
+/* Free function for memory allocated with nbnxn_alloc_aligned */
+void nbnxn_free_aligned(void *ptr)
+{
+ sfree_aligned(ptr);
+}
+
+/* Reallocation wrapper function for nbnxn data structures */
+void nbnxn_realloc_void(void **ptr,
+ int nbytes_copy, int nbytes_new,
+ nbnxn_alloc_t *ma,
+ nbnxn_free_t *mf)
+{
+ void *ptr_new;
+
+ ma(&ptr_new, nbytes_new);
+
+ if (nbytes_new > 0 && ptr_new == NULL)
+ {
+ gmx_fatal(FARGS, "Allocation of %d bytes failed", nbytes_new);
+ }
+
+ if (nbytes_copy > 0)
+ {
+ if (nbytes_new < nbytes_copy)
+ {
+ gmx_incons("In nbnxn_realloc_void: new size less than copy size");
+ }
+ memcpy(ptr_new, *ptr, nbytes_copy);
+ }
+ if (*ptr != NULL)
+ {
+ mf(*ptr);
+ }
+ *ptr = ptr_new;
+}
+
+/* Reallocate the nbnxn_atomdata_t for a size of n atoms */
+void nbnxn_atomdata_realloc(nbnxn_atomdata_t *nbat, int n)
+{
+ int t;
+
+ nbnxn_realloc_void((void **)&nbat->type,
+ nbat->natoms*sizeof(*nbat->type),
+ n*sizeof(*nbat->type),
+ nbat->alloc, nbat->free);
+ nbnxn_realloc_void((void **)&nbat->lj_comb,
+ nbat->natoms*2*sizeof(*nbat->lj_comb),
+ n*2*sizeof(*nbat->lj_comb),
+ nbat->alloc, nbat->free);
+ if (nbat->XFormat != nbatXYZQ)
+ {
+ nbnxn_realloc_void((void **)&nbat->q,
+ nbat->natoms*sizeof(*nbat->q),
+ n*sizeof(*nbat->q),
+ nbat->alloc, nbat->free);
+ }
+ if (nbat->nenergrp > 1)
+ {
+ nbnxn_realloc_void((void **)&nbat->energrp,
+ nbat->natoms/nbat->na_c*sizeof(*nbat->energrp),
+ n/nbat->na_c*sizeof(*nbat->energrp),
+ nbat->alloc, nbat->free);
+ }
+ nbnxn_realloc_void((void **)&nbat->x,
+ nbat->natoms*nbat->xstride*sizeof(*nbat->x),
+ n*nbat->xstride*sizeof(*nbat->x),
+ nbat->alloc, nbat->free);
+ for (t = 0; t < nbat->nout; t++)
+ {
+ /* Allocate one element extra for possible signaling with CUDA */
+ nbnxn_realloc_void((void **)&nbat->out[t].f,
+ nbat->natoms*nbat->fstride*sizeof(*nbat->out[t].f),
+ n*nbat->fstride*sizeof(*nbat->out[t].f),
+ nbat->alloc, nbat->free);
+ }
+ nbat->nalloc = n;
+}
+
+/* Initializes an nbnxn_atomdata_output_t data structure */
+static void nbnxn_atomdata_output_init(nbnxn_atomdata_output_t *out,
+ int nb_kernel_type,
+ int nenergrp, int stride,
+ nbnxn_alloc_t *ma)
+{
+ int cj_size;
+
+ out->f = NULL;
+ ma((void **)&out->fshift, SHIFTS*DIM*sizeof(*out->fshift));
+ out->nV = nenergrp*nenergrp;
+ ma((void **)&out->Vvdw, out->nV*sizeof(*out->Vvdw));
+ ma((void **)&out->Vc, out->nV*sizeof(*out->Vc ));
+
+ if (nb_kernel_type == nbnxnk4xN_SIMD_4xN ||
+ nb_kernel_type == nbnxnk4xN_SIMD_2xNN)
+ {
+ cj_size = nbnxn_kernel_to_cj_size(nb_kernel_type);
+ out->nVS = nenergrp*nenergrp*stride*(cj_size>>1)*cj_size;
+ ma((void **)&out->VSvdw, out->nVS*sizeof(*out->VSvdw));
+ ma((void **)&out->VSc, out->nVS*sizeof(*out->VSc ));
+ }
+ else
+ {
+ out->nVS = 0;
+ }
+}
+
+static void copy_int_to_nbat_int(const int *a, int na, int na_round,
+ const int *in, int fill, int *innb)
+{
+ int i, j;
+
+ j = 0;
+ for (i = 0; i < na; i++)
+ {
+ innb[j++] = in[a[i]];
+ }
+ /* Complete the partially filled last cell with fill */
+ for (; i < na_round; i++)
+ {
+ innb[j++] = fill;
+ }
+}
+
+static void clear_nbat_real(int na, int nbatFormat, real *xnb, int a0)
+{
+ int a, d, j, c;
+
+ switch (nbatFormat)
+ {
+ case nbatXYZ:
+ for (a = 0; a < na; a++)
+ {
+ for (d = 0; d < DIM; d++)
+ {
+ xnb[(a0+a)*STRIDE_XYZ+d] = 0;
+ }
+ }
+ break;
+ case nbatXYZQ:
+ for (a = 0; a < na; a++)
+ {
+ for (d = 0; d < DIM; d++)
+ {
+ xnb[(a0+a)*STRIDE_XYZQ+d] = 0;
+ }
+ }
+ break;
+ case nbatX4:
+ j = X4_IND_A(a0);
+ c = a0 & (PACK_X4-1);
+ for (a = 0; a < na; a++)
+ {
+ xnb[j+XX*PACK_X4] = 0;
+ xnb[j+YY*PACK_X4] = 0;
+ xnb[j+ZZ*PACK_X4] = 0;
+ j++;
+ c++;
+ if (c == PACK_X4)
+ {
+ j += (DIM-1)*PACK_X4;
+ c = 0;
+ }
+ }
+ break;
+ case nbatX8:
+ j = X8_IND_A(a0);
+ c = a0 & (PACK_X8-1);
+ for (a = 0; a < na; a++)
+ {
+ xnb[j+XX*PACK_X8] = 0;
+ xnb[j+YY*PACK_X8] = 0;
+ xnb[j+ZZ*PACK_X8] = 0;
+ j++;
+ c++;
+ if (c == PACK_X8)
+ {
+ j += (DIM-1)*PACK_X8;
+ c = 0;
+ }
+ }
+ break;
+ }
+}
+
+void copy_rvec_to_nbat_real(const int *a, int na, int na_round,
+ rvec *x, int nbatFormat, real *xnb, int a0,
+ int cx, int cy, int cz)
+{
+ int i, j, c;
+
+/* We might need to place filler particles to fill up the cell to na_round.
+ * The coefficients (LJ and q) for such particles are zero.
+ * But we might still get NaN as 0*NaN when distances are too small.
+ * We hope that -107 nm is far away enough from to zero
+ * to avoid accidental short distances to particles shifted down for pbc.
+ */
+#define NBAT_FAR_AWAY 107
+
+ switch (nbatFormat)
+ {
+ case nbatXYZ:
+ j = a0*STRIDE_XYZ;
+ for (i = 0; i < na; i++)
+ {
+ xnb[j++] = x[a[i]][XX];
+ xnb[j++] = x[a[i]][YY];
+ xnb[j++] = x[a[i]][ZZ];
+ }
+ /* Complete the partially filled last cell with copies of the last element.
+ * This simplifies the bounding box calculation and avoid
+ * numerical issues with atoms that are coincidentally close.
+ */
+ for (; i < na_round; i++)
+ {
+ xnb[j++] = -NBAT_FAR_AWAY*(1 + cx);
+ xnb[j++] = -NBAT_FAR_AWAY*(1 + cy);
+ xnb[j++] = -NBAT_FAR_AWAY*(1 + cz + i);
+ }
+ break;
+ case nbatXYZQ:
+ j = a0*STRIDE_XYZQ;
+ for (i = 0; i < na; i++)
+ {
+ xnb[j++] = x[a[i]][XX];
+ xnb[j++] = x[a[i]][YY];
+ xnb[j++] = x[a[i]][ZZ];
+ j++;
+ }
+ /* Complete the partially filled last cell with particles far apart */
+ for (; i < na_round; i++)
+ {
+ xnb[j++] = -NBAT_FAR_AWAY*(1 + cx);
+ xnb[j++] = -NBAT_FAR_AWAY*(1 + cy);
+ xnb[j++] = -NBAT_FAR_AWAY*(1 + cz + i);
+ j++;
+ }
+ break;
+ case nbatX4:
+ j = X4_IND_A(a0);
+ c = a0 & (PACK_X4-1);
+ for (i = 0; i < na; i++)
+ {
+ xnb[j+XX*PACK_X4] = x[a[i]][XX];
+ xnb[j+YY*PACK_X4] = x[a[i]][YY];
+ xnb[j+ZZ*PACK_X4] = x[a[i]][ZZ];
+ j++;
+ c++;
+ if (c == PACK_X4)
+ {
+ j += (DIM-1)*PACK_X4;
+ c = 0;
+ }
+ }
+ /* Complete the partially filled last cell with particles far apart */
+ for (; i < na_round; i++)
+ {
+ xnb[j+XX*PACK_X4] = -NBAT_FAR_AWAY*(1 + cx);
+ xnb[j+YY*PACK_X4] = -NBAT_FAR_AWAY*(1 + cy);
+ xnb[j+ZZ*PACK_X4] = -NBAT_FAR_AWAY*(1 + cz + i);
+ j++;
+ c++;
+ if (c == PACK_X4)
+ {
+ j += (DIM-1)*PACK_X4;
+ c = 0;
+ }
+ }
+ break;
+ case nbatX8:
+ j = X8_IND_A(a0);
+ c = a0 & (PACK_X8 - 1);
+ for (i = 0; i < na; i++)
+ {
+ xnb[j+XX*PACK_X8] = x[a[i]][XX];
+ xnb[j+YY*PACK_X8] = x[a[i]][YY];
+ xnb[j+ZZ*PACK_X8] = x[a[i]][ZZ];
+ j++;
+ c++;
+ if (c == PACK_X8)
+ {
+ j += (DIM-1)*PACK_X8;
+ c = 0;
+ }
+ }
+ /* Complete the partially filled last cell with particles far apart */
+ for (; i < na_round; i++)
+ {
+ xnb[j+XX*PACK_X8] = -NBAT_FAR_AWAY*(1 + cx);
+ xnb[j+YY*PACK_X8] = -NBAT_FAR_AWAY*(1 + cy);
+ xnb[j+ZZ*PACK_X8] = -NBAT_FAR_AWAY*(1 + cz + i);
+ j++;
+ c++;
+ if (c == PACK_X8)
+ {
+ j += (DIM-1)*PACK_X8;
+ c = 0;
+ }
+ }
+ break;
+ default:
+ gmx_incons("Unsupported nbnxn_atomdata_t format");
+ }
+}
+
+/* Determines the combination rule (or none) to be used, stores it,
+ * and sets the LJ parameters required with the rule.
+ */
+static void set_combination_rule_data(nbnxn_atomdata_t *nbat)
+{
+ int nt, i, j;
+ real c6, c12;
+
+ nt = nbat->ntype;
+
+ switch (nbat->comb_rule)
+ {
+ case ljcrGEOM:
+ nbat->comb_rule = ljcrGEOM;
+
+ for (i = 0; i < nt; i++)
+ {
+ /* Copy the diagonal from the nbfp matrix */
+ nbat->nbfp_comb[i*2 ] = sqrt(nbat->nbfp[(i*nt+i)*2 ]);
+ nbat->nbfp_comb[i*2+1] = sqrt(nbat->nbfp[(i*nt+i)*2+1]);
+ }
+ break;
+ case ljcrLB:
+ for (i = 0; i < nt; i++)
+ {
+ /* Get 6*C6 and 12*C12 from the diagonal of the nbfp matrix */
+ c6 = nbat->nbfp[(i*nt+i)*2 ];
+ c12 = nbat->nbfp[(i*nt+i)*2+1];
+ if (c6 > 0 && c12 > 0)
+ {
+ /* We store 0.5*2^1/6*sigma and sqrt(4*3*eps),
+ * so we get 6*C6 and 12*C12 after combining.
+ */
+ nbat->nbfp_comb[i*2 ] = 0.5*pow(c12/c6, 1.0/6.0);
+ nbat->nbfp_comb[i*2+1] = sqrt(c6*c6/c12);
+ }
+ else
+ {
+ nbat->nbfp_comb[i*2 ] = 0;
+ nbat->nbfp_comb[i*2+1] = 0;
+ }
+ }
+ break;
+ case ljcrNONE:
+ /* nbfp_s4 stores two parameters using a stride of 4,
+ * because this would suit x86 SIMD single-precision
+ * quad-load intrinsics. There's a slight inefficiency in
+ * allocating and initializing nbfp_s4 when it might not
+ * be used, but introducing the conditional code is not
+ * really worth it. */
+ nbat->alloc((void **)&nbat->nbfp_s4, nt*nt*4*sizeof(*nbat->nbfp_s4));
+ for (i = 0; i < nt; i++)
+ {
+ for (j = 0; j < nt; j++)
+ {
+ nbat->nbfp_s4[(i*nt+j)*4+0] = nbat->nbfp[(i*nt+j)*2+0];
+ nbat->nbfp_s4[(i*nt+j)*4+1] = nbat->nbfp[(i*nt+j)*2+1];
+ nbat->nbfp_s4[(i*nt+j)*4+2] = 0;
+ nbat->nbfp_s4[(i*nt+j)*4+3] = 0;
+ }
+ }
+ break;
+ default:
+ gmx_incons("Unknown combination rule");
+ break;
+ }
+}
+
++#ifdef GMX_NBNXN_SIMD
++static void
++nbnxn_atomdata_init_simple_exclusion_masks(nbnxn_atomdata_t *nbat)
++{
++ int i, j;
++ const int simd_width = GMX_SIMD_WIDTH_HERE;
++ int simd_excl_size;
++ /* Set the diagonal cluster pair exclusion mask setup data.
++ * In the kernel we check 0 < j - i to generate the masks.
++ * Here we store j - i for generating the mask for the first i,
++ * we substract 0.5 to avoid rounding issues.
++ * In the kernel we can subtract 1 to generate the subsequent mask.
++ */
++ int simd_4xn_diag_size;
++ const real simdFalse = -1, simdTrue = 1;
++ real *simd_interaction_array;
++
++ simd_4xn_diag_size = max(NBNXN_CPU_CLUSTER_I_SIZE, simd_width);
++ snew_aligned(nbat->simd_4xn_diagonal_j_minus_i, simd_4xn_diag_size, NBNXN_MEM_ALIGN);
++ for (j = 0; j < simd_4xn_diag_size; j++)
++ {
++ nbat->simd_4xn_diagonal_j_minus_i[j] = j - 0.5;
++ }
++
++ snew_aligned(nbat->simd_2xnn_diagonal_j_minus_i, simd_width, NBNXN_MEM_ALIGN);
++ for (j = 0; j < simd_width/2; j++)
++ {
++ /* The j-cluster size is half the SIMD width */
++ nbat->simd_2xnn_diagonal_j_minus_i[j] = j - 0.5;
++ /* The next half of the SIMD width is for i + 1 */
++ nbat->simd_2xnn_diagonal_j_minus_i[simd_width/2+j] = j - 1 - 0.5;
++ }
++
++ /* We use up to 32 bits for exclusion masking.
++ * The same masks are used for the 4xN and 2x(N+N) kernels.
++ * The masks are read either into epi32 SIMD registers or into
++ * real SIMD registers (together with a cast).
++ * In single precision this means the real and epi32 SIMD registers
++ * are of equal size.
++ * In double precision the epi32 registers can be smaller than
++ * the real registers, so depending on the architecture, we might
++ * need to use two, identical, 32-bit masks per real.
++ */
++ simd_excl_size = NBNXN_CPU_CLUSTER_I_SIZE*simd_width;
++ snew_aligned(nbat->simd_exclusion_filter1, simd_excl_size, NBNXN_MEM_ALIGN);
++ snew_aligned(nbat->simd_exclusion_filter2, simd_excl_size*2, NBNXN_MEM_ALIGN);
++
++ for (j = 0; j < simd_excl_size; j++)
++ {
++ /* Set the consecutive bits for masking pair exclusions */
++ nbat->simd_exclusion_filter1[j] = (1U << j);
++ nbat->simd_exclusion_filter2[j*2 + 0] = (1U << j);
++ nbat->simd_exclusion_filter2[j*2 + 1] = (1U << j);
++ }
++
++#if (defined GMX_CPU_ACCELERATION_IBM_QPX)
++ /* The QPX kernels shouldn't do the bit masking that is done on
++ * x86, because the SIMD units lack bit-wise operations. Instead,
++ * we generate a vector of all 2^4 possible ways an i atom
++ * interacts with its 4 j atoms. Each array entry contains
++ * simd_width signed ints that are read in a single SIMD
++ * load. These ints must contain values that will be interpreted
++ * as true and false when loaded in the SIMD floating-point
++ * registers, ie. any positive or any negative value,
++ * respectively. Each array entry encodes how this i atom will
++ * interact with the 4 j atoms. Matching code exists in
++ * set_ci_top_excls() to generate indices into this array. Those
++ * indices are used in the kernels. */
++
++ simd_excl_size = NBNXN_CPU_CLUSTER_I_SIZE*NBNXN_CPU_CLUSTER_I_SIZE;
++ const int qpx_simd_width = GMX_SIMD_WIDTH_HERE;
++ snew_aligned(simd_interaction_array, simd_excl_size * qpx_simd_width, NBNXN_MEM_ALIGN);
++ for (j = 0; j < simd_excl_size; j++)
++ {
++ int index = j * qpx_simd_width;
++ for (i = 0; i < qpx_simd_width; i++)
++ {
++ simd_interaction_array[index + i] = (j & (1 << i)) ? simdTrue : simdFalse;
++ }
++ }
++ nbat->simd_interaction_array = simd_interaction_array;
++#endif
++}
++#endif
++
+/* Initializes an nbnxn_atomdata_t data structure */
+void nbnxn_atomdata_init(FILE *fp,
+ nbnxn_atomdata_t *nbat,
+ int nb_kernel_type,
+ int ntype, const real *nbfp,
+ int n_energygroups,
+ int nout,
+ nbnxn_alloc_t *alloc,
+ nbnxn_free_t *free)
+{
+ int i, j;
+ real c6, c12, tol;
+ char *ptr;
+ gmx_bool simple, bCombGeom, bCombLB;
+
+ if (alloc == NULL)
+ {
+ nbat->alloc = nbnxn_alloc_aligned;
+ }
+ else
+ {
+ nbat->alloc = alloc;
+ }
+ if (free == NULL)
+ {
+ nbat->free = nbnxn_free_aligned;
+ }
+ else
+ {
+ nbat->free = free;
+ }
+
+ if (debug)
+ {
+ fprintf(debug, "There are %d atom types in the system, adding one for nbnxn_atomdata_t\n", ntype);
+ }
+ nbat->ntype = ntype + 1;
+ nbat->alloc((void **)&nbat->nbfp,
+ nbat->ntype*nbat->ntype*2*sizeof(*nbat->nbfp));
+ nbat->alloc((void **)&nbat->nbfp_comb, nbat->ntype*2*sizeof(*nbat->nbfp_comb));
+
+ /* A tolerance of 1e-5 seems reasonable for (possibly hand-typed)
+ * force-field floating point parameters.
+ */
+ tol = 1e-5;
+ ptr = getenv("GMX_LJCOMB_TOL");
+ if (ptr != NULL)
+ {
+ double dbl;
+
+ sscanf(ptr, "%lf", &dbl);
+ tol = dbl;
+ }
+ bCombGeom = TRUE;
+ bCombLB = TRUE;
+
+ /* Temporarily fill nbat->nbfp_comb with sigma and epsilon
+ * to check for the LB rule.
+ */
+ for (i = 0; i < ntype; i++)
+ {
+ c6 = nbfp[(i*ntype+i)*2 ]/6.0;
+ c12 = nbfp[(i*ntype+i)*2+1]/12.0;
+ if (c6 > 0 && c12 > 0)
+ {
+ nbat->nbfp_comb[i*2 ] = pow(c12/c6, 1.0/6.0);
+ nbat->nbfp_comb[i*2+1] = 0.25*c6*c6/c12;
+ }
+ else if (c6 == 0 && c12 == 0)
+ {
+ nbat->nbfp_comb[i*2 ] = 0;
+ nbat->nbfp_comb[i*2+1] = 0;
+ }
+ else
+ {
+ /* Can not use LB rule with only dispersion or repulsion */
+ bCombLB = FALSE;
+ }
+ }
+
+ for (i = 0; i < nbat->ntype; i++)
+ {
+ for (j = 0; j < nbat->ntype; j++)
+ {
+ if (i < ntype && j < ntype)
+ {
+ /* fr->nbfp has been updated, so that array too now stores c6/c12 including
+ * the 6.0/12.0 prefactors to save 2 flops in the most common case (force-only).
+ */
+ c6 = nbfp[(i*ntype+j)*2 ];
+ c12 = nbfp[(i*ntype+j)*2+1];
+ nbat->nbfp[(i*nbat->ntype+j)*2 ] = c6;
+ nbat->nbfp[(i*nbat->ntype+j)*2+1] = c12;
+
+ /* Compare 6*C6 and 12*C12 for geometric cobination rule */
+ bCombGeom = bCombGeom &&
+ gmx_within_tol(c6*c6, nbfp[(i*ntype+i)*2 ]*nbfp[(j*ntype+j)*2 ], tol) &&
+ gmx_within_tol(c12*c12, nbfp[(i*ntype+i)*2+1]*nbfp[(j*ntype+j)*2+1], tol);
+
+ /* Compare C6 and C12 for Lorentz-Berthelot combination rule */
+ c6 /= 6.0;
+ c12 /= 12.0;
+ bCombLB = bCombLB &&
+ ((c6 == 0 && c12 == 0 &&
+ (nbat->nbfp_comb[i*2+1] == 0 || nbat->nbfp_comb[j*2+1] == 0)) ||
+ (c6 > 0 && c12 > 0 &&
+ gmx_within_tol(pow(c12/c6, 1.0/6.0), 0.5*(nbat->nbfp_comb[i*2]+nbat->nbfp_comb[j*2]), tol) &&
+ gmx_within_tol(0.25*c6*c6/c12, sqrt(nbat->nbfp_comb[i*2+1]*nbat->nbfp_comb[j*2+1]), tol)));
+ }
+ else
+ {
+ /* Add zero parameters for the additional dummy atom type */
+ nbat->nbfp[(i*nbat->ntype+j)*2 ] = 0;
+ nbat->nbfp[(i*nbat->ntype+j)*2+1] = 0;
+ }
+ }
+ }
+ if (debug)
+ {
+ fprintf(debug, "Combination rules: geometric %d Lorentz-Berthelot %d\n",
+ bCombGeom, bCombLB);
+ }
+
+ simple = nbnxn_kernel_pairlist_simple(nb_kernel_type);
+
+ if (simple)
+ {
+ /* We prefer the geometic combination rule,
+ * as that gives a slightly faster kernel than the LB rule.
+ */
+ if (bCombGeom)
+ {
+ nbat->comb_rule = ljcrGEOM;
+ }
+ else if (bCombLB)
+ {
+ nbat->comb_rule = ljcrLB;
+ }
+ else
+ {
+ nbat->comb_rule = ljcrNONE;
+
+ nbat->free(nbat->nbfp_comb);
+ }
+
+ if (fp)
+ {
+ if (nbat->comb_rule == ljcrNONE)
+ {
+ fprintf(fp, "Using full Lennard-Jones parameter combination matrix\n\n");
+ }
+ else
+ {
+ fprintf(fp, "Using %s Lennard-Jones combination rule\n\n",
+ nbat->comb_rule == ljcrGEOM ? "geometric" : "Lorentz-Berthelot");
+ }
+ }
+
+ set_combination_rule_data(nbat);
+ }
+ else
+ {
+ nbat->comb_rule = ljcrNONE;
+
+ nbat->free(nbat->nbfp_comb);
+ }
+
+ nbat->natoms = 0;
+ nbat->type = NULL;
+ nbat->lj_comb = NULL;
+ if (simple)
+ {
+ int pack_x;
+
+ switch (nb_kernel_type)
+ {
+ case nbnxnk4xN_SIMD_4xN:
+ case nbnxnk4xN_SIMD_2xNN:
+ pack_x = max(NBNXN_CPU_CLUSTER_I_SIZE,
+ nbnxn_kernel_to_cj_size(nb_kernel_type));
+ switch (pack_x)
+ {
+ case 4:
+ nbat->XFormat = nbatX4;
+ break;
+ case 8:
+ nbat->XFormat = nbatX8;
+ break;
+ default:
+ gmx_incons("Unsupported packing width");
+ }
+ break;
+ default:
+ nbat->XFormat = nbatXYZ;
+ break;
+ }
+
+ nbat->FFormat = nbat->XFormat;
+ }
+ else
+ {
+ nbat->XFormat = nbatXYZQ;
+ nbat->FFormat = nbatXYZ;
+ }
+ nbat->q = NULL;
+ nbat->nenergrp = n_energygroups;
+ if (!simple)
+ {
+ /* Energy groups not supported yet for super-sub lists */
+ if (n_energygroups > 1 && fp != NULL)
+ {
+ fprintf(fp, "\nNOTE: With GPUs, reporting energy group contributions is not supported\n\n");
+ }
+ nbat->nenergrp = 1;
+ }
+ /* Temporary storage goes as #grp^3*simd_width^2/2, so limit to 64 */
+ if (nbat->nenergrp > 64)
+ {
+ gmx_fatal(FARGS, "With NxN kernels not more than 64 energy groups are supported\n");
+ }
+ nbat->neg_2log = 1;
+ while (nbat->nenergrp > (1<<nbat->neg_2log))
+ {
+ nbat->neg_2log++;
+ }
+ nbat->energrp = NULL;
+ nbat->alloc((void **)&nbat->shift_vec, SHIFTS*sizeof(*nbat->shift_vec));
+ nbat->xstride = (nbat->XFormat == nbatXYZQ ? STRIDE_XYZQ : DIM);
+ nbat->fstride = (nbat->FFormat == nbatXYZQ ? STRIDE_XYZQ : DIM);
+ nbat->x = NULL;
+
+#ifdef GMX_NBNXN_SIMD
+ if (simple)
+ {
++ nbnxn_atomdata_init_simple_exclusion_masks(nbat);
+ }
+#endif
+
+ /* Initialize the output data structures */
+ nbat->nout = nout;
+ snew(nbat->out, nbat->nout);
+ nbat->nalloc = 0;
+ for (i = 0; i < nbat->nout; i++)
+ {
+ nbnxn_atomdata_output_init(&nbat->out[i],
+ nb_kernel_type,
+ nbat->nenergrp, 1<<nbat->neg_2log,
+ nbat->alloc);
+ }
+ nbat->buffer_flags.flag = NULL;
+ nbat->buffer_flags.flag_nalloc = 0;
+}
+
+static void copy_lj_to_nbat_lj_comb_x4(const real *ljparam_type,
+ const int *type, int na,
+ real *ljparam_at)
+{
+ int is, k, i;
+
+ /* The LJ params follow the combination rule:
+ * copy the params for the type array to the atom array.
+ */
+ for (is = 0; is < na; is += PACK_X4)
+ {
+ for (k = 0; k < PACK_X4; k++)
+ {
+ i = is + k;
+ ljparam_at[is*2 +k] = ljparam_type[type[i]*2 ];
+ ljparam_at[is*2+PACK_X4+k] = ljparam_type[type[i]*2+1];
+ }
+ }
+}
+
+static void copy_lj_to_nbat_lj_comb_x8(const real *ljparam_type,
+ const int *type, int na,
+ real *ljparam_at)
+{
+ int is, k, i;
+
+ /* The LJ params follow the combination rule:
+ * copy the params for the type array to the atom array.
+ */
+ for (is = 0; is < na; is += PACK_X8)
+ {
+ for (k = 0; k < PACK_X8; k++)
+ {
+ i = is + k;
+ ljparam_at[is*2 +k] = ljparam_type[type[i]*2 ];
+ ljparam_at[is*2+PACK_X8+k] = ljparam_type[type[i]*2+1];
+ }
+ }
+}
+
+/* Sets the atom type and LJ data in nbnxn_atomdata_t */
+static void nbnxn_atomdata_set_atomtypes(nbnxn_atomdata_t *nbat,
+ int ngrid,
+ const nbnxn_search_t nbs,
+ const int *type)
+{
+ int g, i, ncz, ash;
+ const nbnxn_grid_t *grid;
+
+ for (g = 0; g < ngrid; g++)
+ {
+ grid = &nbs->grid[g];
+
+ /* Loop over all columns and copy and fill */
+ for (i = 0; i < grid->ncx*grid->ncy; i++)
+ {
+ ncz = grid->cxy_ind[i+1] - grid->cxy_ind[i];
+ ash = (grid->cell0 + grid->cxy_ind[i])*grid->na_sc;
+
+ copy_int_to_nbat_int(nbs->a+ash, grid->cxy_na[i], ncz*grid->na_sc,
+ type, nbat->ntype-1, nbat->type+ash);
+
+ if (nbat->comb_rule != ljcrNONE)
+ {
+ if (nbat->XFormat == nbatX4)
+ {
+ copy_lj_to_nbat_lj_comb_x4(nbat->nbfp_comb,
+ nbat->type+ash, ncz*grid->na_sc,
+ nbat->lj_comb+ash*2);
+ }
+ else if (nbat->XFormat == nbatX8)
+ {
+ copy_lj_to_nbat_lj_comb_x8(nbat->nbfp_comb,
+ nbat->type+ash, ncz*grid->na_sc,
+ nbat->lj_comb+ash*2);
+ }
+ }
+ }
+ }
+}
+
+/* Sets the charges in nbnxn_atomdata_t *nbat */
+static void nbnxn_atomdata_set_charges(nbnxn_atomdata_t *nbat,
+ int ngrid,
+ const nbnxn_search_t nbs,
+ const real *charge)
+{
+ int g, cxy, ncz, ash, na, na_round, i, j;
+ real *q;
+ const nbnxn_grid_t *grid;
+
+ for (g = 0; g < ngrid; g++)
+ {
+ grid = &nbs->grid[g];
+
+ /* Loop over all columns and copy and fill */
+ for (cxy = 0; cxy < grid->ncx*grid->ncy; cxy++)
+ {
+ ash = (grid->cell0 + grid->cxy_ind[cxy])*grid->na_sc;
+ na = grid->cxy_na[cxy];
+ na_round = (grid->cxy_ind[cxy+1] - grid->cxy_ind[cxy])*grid->na_sc;
+
+ if (nbat->XFormat == nbatXYZQ)
+ {
+ q = nbat->x + ash*STRIDE_XYZQ + ZZ + 1;
+ for (i = 0; i < na; i++)
+ {
+ *q = charge[nbs->a[ash+i]];
+ q += STRIDE_XYZQ;
+ }
+ /* Complete the partially filled last cell with zeros */
+ for (; i < na_round; i++)
+ {
+ *q = 0;
+ q += STRIDE_XYZQ;
+ }
+ }
+ else
+ {
+ q = nbat->q + ash;
+ for (i = 0; i < na; i++)
+ {
+ *q = charge[nbs->a[ash+i]];
+ q++;
+ }
+ /* Complete the partially filled last cell with zeros */
+ for (; i < na_round; i++)
+ {
+ *q = 0;
+ q++;
+ }
+ }
+ }
+ }
+}
+
+/* Copies the energy group indices to a reordered and packed array */
+static void copy_egp_to_nbat_egps(const int *a, int na, int na_round,
+ int na_c, int bit_shift,
+ const int *in, int *innb)
+{
+ int i, j, sa, at;
+ int comb;
+
+ j = 0;
+ for (i = 0; i < na; i += na_c)
+ {
+ /* Store na_c energy group numbers into one int */
+ comb = 0;
+ for (sa = 0; sa < na_c; sa++)
+ {
+ at = a[i+sa];
+ if (at >= 0)
+ {
+ comb |= (GET_CGINFO_GID(in[at]) << (sa*bit_shift));
+ }
+ }
+ innb[j++] = comb;
+ }
+ /* Complete the partially filled last cell with fill */
+ for (; i < na_round; i += na_c)
+ {
+ innb[j++] = 0;
+ }
+}
+
+/* Set the energy group indices for atoms in nbnxn_atomdata_t */
+static void nbnxn_atomdata_set_energygroups(nbnxn_atomdata_t *nbat,
+ int ngrid,
+ const nbnxn_search_t nbs,
+ const int *atinfo)
+{
+ int g, i, ncz, ash;
+ const nbnxn_grid_t *grid;
+
+ for (g = 0; g < ngrid; g++)
+ {
+ grid = &nbs->grid[g];
+
+ /* Loop over all columns and copy and fill */
+ for (i = 0; i < grid->ncx*grid->ncy; i++)
+ {
+ ncz = grid->cxy_ind[i+1] - grid->cxy_ind[i];
+ ash = (grid->cell0 + grid->cxy_ind[i])*grid->na_sc;
+
+ copy_egp_to_nbat_egps(nbs->a+ash, grid->cxy_na[i], ncz*grid->na_sc,
+ nbat->na_c, nbat->neg_2log,
+ atinfo, nbat->energrp+(ash>>grid->na_c_2log));
+ }
+ }
+}
+
+/* Sets all required atom parameter data in nbnxn_atomdata_t */
+void nbnxn_atomdata_set(nbnxn_atomdata_t *nbat,
+ int locality,
+ const nbnxn_search_t nbs,
+ const t_mdatoms *mdatoms,
+ const int *atinfo)
+{
+ int ngrid;
+
+ if (locality == eatLocal)
+ {
+ ngrid = 1;
+ }
+ else
+ {
+ ngrid = nbs->ngrid;
+ }
+
+ nbnxn_atomdata_set_atomtypes(nbat, ngrid, nbs, mdatoms->typeA);
+
+ nbnxn_atomdata_set_charges(nbat, ngrid, nbs, mdatoms->chargeA);
+
+ if (nbat->nenergrp > 1)
+ {
+ nbnxn_atomdata_set_energygroups(nbat, ngrid, nbs, atinfo);
+ }
+}
+
+/* Copies the shift vector array to nbnxn_atomdata_t */
+void nbnxn_atomdata_copy_shiftvec(gmx_bool bDynamicBox,
+ rvec *shift_vec,
+ nbnxn_atomdata_t *nbat)
+{
+ int i;
+
+ nbat->bDynamicBox = bDynamicBox;
+ for (i = 0; i < SHIFTS; i++)
+ {
+ copy_rvec(shift_vec[i], nbat->shift_vec[i]);
+ }
+}
+
+/* Copies (and reorders) the coordinates to nbnxn_atomdata_t */
+void nbnxn_atomdata_copy_x_to_nbat_x(const nbnxn_search_t nbs,
+ int locality,
+ gmx_bool FillLocal,
+ rvec *x,
+ nbnxn_atomdata_t *nbat)
+{
+ int g0 = 0, g1 = 0;
+ int nth, th;
+
+ switch (locality)
+ {
+ case eatAll:
+ g0 = 0;
+ g1 = nbs->ngrid;
+ break;
+ case eatLocal:
+ g0 = 0;
+ g1 = 1;
+ break;
+ case eatNonlocal:
+ g0 = 1;
+ g1 = nbs->ngrid;
+ break;
+ }
+
+ if (FillLocal)
+ {
+ nbat->natoms_local = nbs->grid[0].nc*nbs->grid[0].na_sc;
+ }
+
+ nth = gmx_omp_nthreads_get(emntPairsearch);
+
+#pragma omp parallel for num_threads(nth) schedule(static)
+ for (th = 0; th < nth; th++)
+ {
+ int g;
+
+ for (g = g0; g < g1; g++)
+ {
+ const nbnxn_grid_t *grid;
+ int cxy0, cxy1, cxy;
+
+ grid = &nbs->grid[g];
+
+ cxy0 = (grid->ncx*grid->ncy* th +nth-1)/nth;
+ cxy1 = (grid->ncx*grid->ncy*(th+1)+nth-1)/nth;
+
+ for (cxy = cxy0; cxy < cxy1; cxy++)
+ {
+ int na, ash, na_fill;
+
+ na = grid->cxy_na[cxy];
+ ash = (grid->cell0 + grid->cxy_ind[cxy])*grid->na_sc;
+
+ if (g == 0 && FillLocal)
+ {
+ na_fill =
+ (grid->cxy_ind[cxy+1] - grid->cxy_ind[cxy])*grid->na_sc;
+ }
+ else
+ {
+ /* We fill only the real particle locations.
+ * We assume the filling entries at the end have been
+ * properly set before during ns.
+ */
+ na_fill = na;
+ }
+ copy_rvec_to_nbat_real(nbs->a+ash, na, na_fill, x,
+ nbat->XFormat, nbat->x, ash,
+ 0, 0, 0);
+ }
+ }
+ }
+}
+
+static void
+nbnxn_atomdata_clear_reals(real * gmx_restrict dest,
+ int i0, int i1)
+{
+ int i;
+
+ for (i = i0; i < i1; i++)
+ {
+ dest[i] = 0;
+ }
+}
+
+static void
+nbnxn_atomdata_reduce_reals(real * gmx_restrict dest,
+ gmx_bool bDestSet,
+ real ** gmx_restrict src,
+ int nsrc,
+ int i0, int i1)
+{
+ int i, s;
+
+ if (bDestSet)
+ {
+ /* The destination buffer contains data, add to it */
+ for (i = i0; i < i1; i++)
+ {
+ for (s = 0; s < nsrc; s++)
+ {
+ dest[i] += src[s][i];
+ }
+ }
+ }
+ else
+ {
+ /* The destination buffer is unitialized, set it first */
+ for (i = i0; i < i1; i++)
+ {
+ dest[i] = src[0][i];
+ for (s = 1; s < nsrc; s++)
+ {
+ dest[i] += src[s][i];
+ }
+ }
+ }
+}
+
+static void
+nbnxn_atomdata_reduce_reals_simd(real * gmx_restrict dest,
+ gmx_bool bDestSet,
+ real ** gmx_restrict src,
+ int nsrc,
+ int i0, int i1)
+{
+#ifdef GMX_NBNXN_SIMD
+/* The SIMD width here is actually independent of that in the kernels,
+ * but we use the same width for simplicity (usually optimal anyhow).
+ */
+ int i, s;
+ gmx_mm_pr dest_SSE, src_SSE;
+
+ if (bDestSet)
+ {
+ for (i = i0; i < i1; i += GMX_SIMD_WIDTH_HERE)
+ {
+ dest_SSE = gmx_load_pr(dest+i);
+ for (s = 0; s < nsrc; s++)
+ {
+ src_SSE = gmx_load_pr(src[s]+i);
+ dest_SSE = gmx_add_pr(dest_SSE, src_SSE);
+ }
+ gmx_store_pr(dest+i, dest_SSE);
+ }
+ }
+ else
+ {
+ for (i = i0; i < i1; i += GMX_SIMD_WIDTH_HERE)
+ {
+ dest_SSE = gmx_load_pr(src[0]+i);
+ for (s = 1; s < nsrc; s++)
+ {
+ src_SSE = gmx_load_pr(src[s]+i);
+ dest_SSE = gmx_add_pr(dest_SSE, src_SSE);
+ }
+ gmx_store_pr(dest+i, dest_SSE);
+ }
+ }
+#endif
+}
+
+/* Add part of the force array(s) from nbnxn_atomdata_t to f */
+static void
+nbnxn_atomdata_add_nbat_f_to_f_part(const nbnxn_search_t nbs,
+ const nbnxn_atomdata_t *nbat,
+ nbnxn_atomdata_output_t *out,
+ int nfa,
+ int a0, int a1,
+ rvec *f)
+{
+ int a, i, fa;
+ const int *cell;
+ const real *fnb;
+
+ cell = nbs->cell;
+
+ /* Loop over all columns and copy and fill */
+ switch (nbat->FFormat)
+ {
+ case nbatXYZ:
+ case nbatXYZQ:
+ if (nfa == 1)
+ {
+ fnb = out[0].f;
+
+ for (a = a0; a < a1; a++)
+ {
+ i = cell[a]*nbat->fstride;
+
+ f[a][XX] += fnb[i];
+ f[a][YY] += fnb[i+1];
+ f[a][ZZ] += fnb[i+2];
+ }
+ }
+ else
+ {
+ for (a = a0; a < a1; a++)
+ {
+ i = cell[a]*nbat->fstride;
+
+ for (fa = 0; fa < nfa; fa++)
+ {
+ f[a][XX] += out[fa].f[i];
+ f[a][YY] += out[fa].f[i+1];
+ f[a][ZZ] += out[fa].f[i+2];
+ }
+ }
+ }
+ break;
+ case nbatX4:
+ if (nfa == 1)
+ {
+ fnb = out[0].f;
+
+ for (a = a0; a < a1; a++)
+ {
+ i = X4_IND_A(cell[a]);
+
+ f[a][XX] += fnb[i+XX*PACK_X4];
+ f[a][YY] += fnb[i+YY*PACK_X4];
+ f[a][ZZ] += fnb[i+ZZ*PACK_X4];
+ }
+ }
+ else
+ {
+ for (a = a0; a < a1; a++)
+ {
+ i = X4_IND_A(cell[a]);
+
+ for (fa = 0; fa < nfa; fa++)
+ {
+ f[a][XX] += out[fa].f[i+XX*PACK_X4];
+ f[a][YY] += out[fa].f[i+YY*PACK_X4];
+ f[a][ZZ] += out[fa].f[i+ZZ*PACK_X4];
+ }
+ }
+ }
+ break;
+ case nbatX8:
+ if (nfa == 1)
+ {
+ fnb = out[0].f;
+
+ for (a = a0; a < a1; a++)
+ {
+ i = X8_IND_A(cell[a]);
+
+ f[a][XX] += fnb[i+XX*PACK_X8];
+ f[a][YY] += fnb[i+YY*PACK_X8];
+ f[a][ZZ] += fnb[i+ZZ*PACK_X8];
+ }
+ }
+ else
+ {
+ for (a = a0; a < a1; a++)
+ {
+ i = X8_IND_A(cell[a]);
+
+ for (fa = 0; fa < nfa; fa++)
+ {
+ f[a][XX] += out[fa].f[i+XX*PACK_X8];
+ f[a][YY] += out[fa].f[i+YY*PACK_X8];
+ f[a][ZZ] += out[fa].f[i+ZZ*PACK_X8];
+ }
+ }
+ }
+ break;
+ default:
+ gmx_incons("Unsupported nbnxn_atomdata_t format");
+ }
+}
+
+/* Add the force array(s) from nbnxn_atomdata_t to f */
+void nbnxn_atomdata_add_nbat_f_to_f(const nbnxn_search_t nbs,
+ int locality,
+ const nbnxn_atomdata_t *nbat,
+ rvec *f)
+{
+ int a0 = 0, na = 0;
+ int nth, th;
+
+ nbs_cycle_start(&nbs->cc[enbsCCreducef]);
+
+ switch (locality)
+ {
+ case eatAll:
+ a0 = 0;
+ na = nbs->natoms_nonlocal;
+ break;
+ case eatLocal:
+ a0 = 0;
+ na = nbs->natoms_local;
+ break;
+ case eatNonlocal:
+ a0 = nbs->natoms_local;
+ na = nbs->natoms_nonlocal - nbs->natoms_local;
+ break;
+ }
+
+ nth = gmx_omp_nthreads_get(emntNonbonded);
+
+ if (nbat->nout > 1)
+ {
+ if (locality != eatAll)
+ {
+ gmx_incons("add_f_to_f called with nout>1 and locality!=eatAll");
+ }
+
+ /* Reduce the force thread output buffers into buffer 0, before adding
+ * them to the, differently ordered, "real" force buffer.
+ */
+#pragma omp parallel for num_threads(nth) schedule(static)
+ for (th = 0; th < nth; th++)
+ {
+ const nbnxn_buffer_flags_t *flags;
+ int b0, b1, b;
+ int i0, i1;
+ int nfptr;
+ real *fptr[NBNXN_BUFFERFLAG_MAX_THREADS];
+ int out;
+
+ flags = &nbat->buffer_flags;
+
+ /* Calculate the cell-block range for our thread */
+ b0 = (flags->nflag* th )/nth;
+ b1 = (flags->nflag*(th+1))/nth;
+
+ for (b = b0; b < b1; b++)
+ {
+ i0 = b *NBNXN_BUFFERFLAG_SIZE*nbat->fstride;
+ i1 = (b+1)*NBNXN_BUFFERFLAG_SIZE*nbat->fstride;
+
+ nfptr = 0;
+ for (out = 1; out < nbat->nout; out++)
+ {
+ if (flags->flag[b] & (1U<<out))
+ {
+ fptr[nfptr++] = nbat->out[out].f;
+ }
+ }
+ if (nfptr > 0)
+ {
+#ifdef GMX_NBNXN_SIMD
+ nbnxn_atomdata_reduce_reals_simd
+#else
+ nbnxn_atomdata_reduce_reals
+#endif
+ (nbat->out[0].f,
+ flags->flag[b] & (1U<<0),
+ fptr, nfptr,
+ i0, i1);
+ }
+ else if (!(flags->flag[b] & (1U<<0)))
+ {
+ nbnxn_atomdata_clear_reals(nbat->out[0].f,
+ i0, i1);
+ }
+ }
+ }
+ }
+
+#pragma omp parallel for num_threads(nth) schedule(static)
+ for (th = 0; th < nth; th++)
+ {
+ nbnxn_atomdata_add_nbat_f_to_f_part(nbs, nbat,
+ nbat->out,
+ 1,
+ a0+((th+0)*na)/nth,
+ a0+((th+1)*na)/nth,
+ f);
+ }
+
+ nbs_cycle_stop(&nbs->cc[enbsCCreducef]);
+}
+
+/* Adds the shift forces from nbnxn_atomdata_t to fshift */
+void nbnxn_atomdata_add_nbat_fshift_to_fshift(const nbnxn_atomdata_t *nbat,
+ rvec *fshift)
+{
+ const nbnxn_atomdata_output_t *out;
+ int th;
+ int s;
+ rvec sum;
+
+ out = nbat->out;
+
+ for (s = 0; s < SHIFTS; s++)
+ {
+ clear_rvec(sum);
+ for (th = 0; th < nbat->nout; th++)
+ {
+ sum[XX] += out[th].fshift[s*DIM+XX];
+ sum[YY] += out[th].fshift[s*DIM+YY];
+ sum[ZZ] += out[th].fshift[s*DIM+ZZ];
+ }
+ rvec_inc(fshift[s], sum);
+ }
+}
--- /dev/null
- texture<float, 1, cudaReadModeElementType> tex_nbfp;
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2012, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * Gallium Rubidium Oxygen Manganese Argon Carbon Silicon
+ */
+
+#include <stdlib.h>
+#include <assert.h>
+
+#if defined(_MSVC)
+#include <limits>
+#endif
+
+#include <cuda.h>
+
+#include "types/simple.h"
+#include "types/nbnxn_pairlist.h"
+#include "types/nb_verlet.h"
+#include "types/ishift.h"
+#include "types/force_flags.h"
+#include "../nbnxn_consts.h"
+
+#ifdef TMPI_ATOMICS
+#include "thread_mpi/atomic.h"
+#endif
+
+#include "nbnxn_cuda_types.h"
+#include "../../gmxlib/cuda_tools/cudautils.cuh"
+#include "nbnxn_cuda.h"
+#include "nbnxn_cuda_data_mgmt.h"
+
++#if defined TEXOBJ_SUPPORTED && __CUDA_ARCH__ > 300
++#define USE_TEXOBJ
++#endif
+
+/*! Texture reference for nonbonded parameters; bound to cu_nbparam_t.nbfp*/
- texture<float, 1, cudaReadModeElementType> tex_coulomb_tab;
++texture<float, 1, cudaReadModeElementType> nbfp_texref;
+
+/*! Texture reference for Ewald coulomb force table; bound to cu_nbparam_t.coulomb_tab */
- return tex_nbfp;
++texture<float, 1, cudaReadModeElementType> coulomb_tab_texref;
+
+/* Convenience defines */
+#define NCL_PER_SUPERCL (NBNXN_GPU_NCLUSTER_PER_SUPERCLUSTER)
+#define CL_SIZE (NBNXN_GPU_CLUSTER_SIZE)
+
+/***** The kernels come here *****/
+#include "nbnxn_cuda_kernel_utils.cuh"
+
+/* Top-level kernel generation: will generate through multiple inclusion the
+ * following flavors for all kernels:
+ * - force-only output;
+ * - force and energy output;
+ * - force-only with pair list pruning;
+ * - force and energy output with pair list pruning.
+ */
+/** Force only **/
+#include "nbnxn_cuda_kernels.cuh"
+/** Force & energy **/
+#define CALC_ENERGIES
+#include "nbnxn_cuda_kernels.cuh"
+#undef CALC_ENERGIES
+
+/*** Pair-list pruning kernels ***/
+/** Force only **/
+#define PRUNE_NBL
+#include "nbnxn_cuda_kernels.cuh"
+/** Force & energy **/
+#define CALC_ENERGIES
+#include "nbnxn_cuda_kernels.cuh"
+#undef CALC_ENERGIES
+#undef PRUNE_NBL
+
+/*! Nonbonded kernel function pointer type */
+typedef void (*nbnxn_cu_kfunc_ptr_t)(const cu_atomdata_t,
+ const cu_nbparam_t,
+ const cu_plist_t,
+ bool);
+
+/*********************************/
+
+/* XXX always/never run the energy/pruning kernels -- only for benchmarking purposes */
+static bool always_ener = (getenv("GMX_GPU_ALWAYS_ENER") != NULL);
+static bool never_ener = (getenv("GMX_GPU_NEVER_ENER") != NULL);
+static bool always_prune = (getenv("GMX_GPU_ALWAYS_PRUNE") != NULL);
+
+
+/* Bit-pattern used for polling-based GPU synchronization. It is used as a float
+ * and corresponds to having the exponent set to the maximum (127 -- single
+ * precision) and the mantissa to 0.
+ */
+static unsigned int poll_wait_pattern = (0x7FU << 23);
+
+/*! Returns the number of blocks to be used for the nonbonded GPU kernel. */
+static inline int calc_nb_kernel_nblock(int nwork_units, cuda_dev_info_t *dinfo)
+{
+ int max_grid_x_size;
+
+ assert(dinfo);
+
+ max_grid_x_size = dinfo->prop.maxGridSize[0];
+
+ /* do we exceed the grid x dimension limit? */
+ if (nwork_units > max_grid_x_size)
+ {
+ gmx_fatal(FARGS, "Watch out, the input system is too large to simulate!\n"
+ "The number of nonbonded work units (=number of super-clusters) exceeds the"
+ "maximum grid size in x dimension (%d > %d)!", nwork_units, max_grid_x_size);
+ }
+
+ return nwork_units;
+}
+
+
+/* Constant arrays listing all kernel function pointers and enabling selection
+ of a kernel in an elegant manner. */
+
+static const int nEnergyKernelTypes = 2; /* 0 - no energy, 1 - energy */
+static const int nPruneKernelTypes = 2; /* 0 - no prune, 1 - prune */
+
+/*! Pointers to the default kernels organized in a 3 dim array by:
+ * electrostatics type, energy calculation on/off, and pruning on/off.
+ *
+ * Note that the order of electrostatics (1st dimension) has to match the
+ * order of corresponding enumerated types defined in nbnxn_cuda_types.h.
+ */
+static const nbnxn_cu_kfunc_ptr_t
+nb_default_kfunc_ptr[eelCuNR][nEnergyKernelTypes][nPruneKernelTypes] =
+{
+ { { k_nbnxn_cutoff, k_nbnxn_cutoff_prune },
+ { k_nbnxn_cutoff_ener, k_nbnxn_cutoff_ener_prune } },
+ { { k_nbnxn_rf, k_nbnxn_rf_prune },
+ { k_nbnxn_rf_ener, k_nbnxn_rf_ener_prune } },
+ { { k_nbnxn_ewald_tab, k_nbnxn_ewald_tab_prune },
+ { k_nbnxn_ewald_tab_ener, k_nbnxn_ewald_tab_ener_prune } },
+ { { k_nbnxn_ewald_tab_twin, k_nbnxn_ewald_tab_twin_prune },
+ { k_nbnxn_ewald_tab_twin_ener, k_nbnxn_ewald_twin_ener_prune } },
+ { { k_nbnxn_ewald, k_nbnxn_ewald_prune },
+ { k_nbnxn_ewald_ener, k_nbnxn_ewald_ener_prune } },
+ { { k_nbnxn_ewald_twin, k_nbnxn_ewald_twin_prune },
+ { k_nbnxn_ewald_twin_ener, k_nbnxn_ewald_twin_ener_prune } },
+};
+
+/*! Pointers to the legacy kernels organized in a 3 dim array by:
+ * electrostatics type, energy calculation on/off, and pruning on/off.
+ *
+ * Note that the order of electrostatics (1st dimension) has to match the
+ * order of corresponding enumerated types defined in nbnxn_cuda_types.h.
+ */
+static const nbnxn_cu_kfunc_ptr_t
+nb_legacy_kfunc_ptr[eelCuNR][nEnergyKernelTypes][nPruneKernelTypes] =
+{
+ { { k_nbnxn_cutoff_legacy, k_nbnxn_cutoff_prune_legacy },
+ { k_nbnxn_cutoff_ener_legacy, k_nbnxn_cutoff_ener_prune_legacy } },
+ { { k_nbnxn_rf_legacy, k_nbnxn_rf_prune_legacy },
+ { k_nbnxn_rf_ener_legacy, k_nbnxn_rf_ener_prune_legacy } },
+ { { k_nbnxn_ewald_tab_legacy, k_nbnxn_ewald_tab_prune_legacy },
+ { k_nbnxn_ewald_tab_ener_legacy, k_nbnxn_ewald_tab_ener_prune_legacy } },
+ { { k_nbnxn_ewald_tab_twin_legacy, k_nbnxn_ewald_tab_twin_prune_legacy },
+ { k_nbnxn_ewald_tab_twin_ener_legacy, k_nbnxn_ewald_tab_twin_ener_prune_legacy } },
+};
+
+/*! Return a pointer to the kernel version to be executed at the current step. */
+static inline nbnxn_cu_kfunc_ptr_t select_nbnxn_kernel(int kver, int eeltype,
+ bool bDoEne, bool bDoPrune)
+{
+ assert(kver < eNbnxnCuKNR);
+ assert(eeltype < eelCuNR);
+
+ if (NBNXN_KVER_LEGACY(kver))
+ {
+ /* no analytical Ewald with legacy kernels */
+ assert(eeltype <= eelCuEWALD_TAB_TWIN);
+
+ return nb_legacy_kfunc_ptr[eeltype][bDoEne][bDoPrune];
+ }
+ else
+ {
+ return nb_default_kfunc_ptr[eeltype][bDoEne][bDoPrune];
+ }
+}
+
+/*! Calculates the amount of shared memory required for kernel version in use. */
+static inline int calc_shmem_required(int kver)
+{
+ int shmem;
+
+ /* size of shmem (force-buffers/xq/atom type preloading) */
+ if (NBNXN_KVER_LEGACY(kver))
+ {
+ /* i-atom x+q in shared memory */
+ shmem = NCL_PER_SUPERCL * CL_SIZE * sizeof(float4);
+ /* force reduction buffers in shared memory */
+ shmem += CL_SIZE * CL_SIZE * 3 * sizeof(float);
+ }
+ else
+ {
+ /* NOTE: with the default kernel on sm3.0 we need shmem only for pre-loading */
+ /* i-atom x+q in shared memory */
+ shmem = NCL_PER_SUPERCL * CL_SIZE * sizeof(float4);
+ /* cj in shared memory, for both warps separately */
+ shmem += 2 * NBNXN_GPU_JGROUP_SIZE * sizeof(int);
+#ifdef IATYPE_SHMEM
+ /* i-atom types in shared memory */
+ shmem += NCL_PER_SUPERCL * CL_SIZE * sizeof(int);
+#endif
+#if __CUDA_ARCH__ < 300
+ /* force reduction buffers in shared memory */
+ shmem += CL_SIZE * CL_SIZE * 3 * sizeof(float);
+#endif
+ }
+
+ return shmem;
+}
+
+/*! As we execute nonbonded workload in separate streams, before launching
+ the kernel we need to make sure that he following operations have completed:
+ - atomdata allocation and related H2D transfers (every nstlist step);
+ - pair list H2D transfer (every nstlist step);
+ - shift vector H2D transfer (every nstlist step);
+ - force (+shift force and energy) output clearing (every step).
+
+ These operations are issued in the local stream at the beginning of the step
+ and therefore always complete before the local kernel launch. The non-local
+ kernel is launched after the local on the same device/context, so this is
+ inherently scheduled after the operations in the local stream (including the
+ above "misc_ops").
+ However, for the sake of having a future-proof implementation, we use the
+ misc_ops_done event to record the point in time when the above operations
+ are finished and synchronize with this event in the non-local stream.
+*/
+void nbnxn_cuda_launch_kernel(nbnxn_cuda_ptr_t cu_nb,
+ const nbnxn_atomdata_t *nbatom,
+ int flags,
+ int iloc)
+{
+ cudaError_t stat;
+ int adat_begin, adat_len; /* local/nonlocal offset and length used for xq and f */
+ /* CUDA kernel launch-related stuff */
+ int shmem, nblock;
+ dim3 dim_block, dim_grid;
+ nbnxn_cu_kfunc_ptr_t nb_kernel = NULL; /* fn pointer to the nonbonded kernel */
+
+ cu_atomdata_t *adat = cu_nb->atdat;
+ cu_nbparam_t *nbp = cu_nb->nbparam;
+ cu_plist_t *plist = cu_nb->plist[iloc];
+ cu_timers_t *t = cu_nb->timers;
+ cudaStream_t stream = cu_nb->stream[iloc];
+
+ bool bCalcEner = flags & GMX_FORCE_VIRIAL;
+ bool bCalcFshift = flags & GMX_FORCE_VIRIAL;
+ bool bDoTime = cu_nb->bDoTime;
+
+ /* turn energy calculation always on/off (for debugging/testing only) */
+ bCalcEner = (bCalcEner || always_ener) && !never_ener;
+
+ /* don't launch the kernel if there is no work to do */
+ if (plist->nsci == 0)
+ {
+ return;
+ }
+
+ /* calculate the atom data index range based on locality */
+ if (LOCAL_I(iloc))
+ {
+ adat_begin = 0;
+ adat_len = adat->natoms_local;
+ }
+ else
+ {
+ adat_begin = adat->natoms_local;
+ adat_len = adat->natoms - adat->natoms_local;
+ }
+
+ /* When we get here all misc operations issues in the local stream are done,
+ so we record that in the local stream and wait for it in the nonlocal one. */
+ if (cu_nb->bUseTwoStreams)
+ {
+ if (iloc == eintLocal)
+ {
+ stat = cudaEventRecord(cu_nb->misc_ops_done, stream);
+ CU_RET_ERR(stat, "cudaEventRecord on misc_ops_done failed");
+ }
+ else
+ {
+ stat = cudaStreamWaitEvent(stream, cu_nb->misc_ops_done, 0);
+ CU_RET_ERR(stat, "cudaStreamWaitEvent on misc_ops_done failed");
+ }
+ }
+
+ /* beginning of timed HtoD section */
+ if (bDoTime)
+ {
+ stat = cudaEventRecord(t->start_nb_h2d[iloc], stream);
+ CU_RET_ERR(stat, "cudaEventRecord failed");
+ }
+
+ /* HtoD x, q */
+ cu_copy_H2D_async(adat->xq + adat_begin, nbatom->x + adat_begin * 4,
+ adat_len * sizeof(*adat->xq), stream);
+
+ if (bDoTime)
+ {
+ stat = cudaEventRecord(t->stop_nb_h2d[iloc], stream);
+ CU_RET_ERR(stat, "cudaEventRecord failed");
+ }
+
+ /* beginning of timed nonbonded calculation section */
+ if (bDoTime)
+ {
+ stat = cudaEventRecord(t->start_nb_k[iloc], stream);
+ CU_RET_ERR(stat, "cudaEventRecord failed");
+ }
+
+ /* get the pointer to the kernel flavor we need to use */
+ nb_kernel = select_nbnxn_kernel(cu_nb->kernel_ver, nbp->eeltype, bCalcEner,
+ plist->bDoPrune || always_prune);
+
+ /* kernel launch config */
+ nblock = calc_nb_kernel_nblock(plist->nsci, cu_nb->dev_info);
+ dim_block = dim3(CL_SIZE, CL_SIZE, 1);
+ dim_grid = dim3(nblock, 1, 1);
+ shmem = calc_shmem_required(cu_nb->kernel_ver);
+
+ if (debug)
+ {
+ fprintf(debug, "GPU launch configuration:\n\tThread block: %dx%dx%d\n\t"
+ "Grid: %dx%d\n\t#Super-clusters/clusters: %d/%d (%d)\n",
+ dim_block.x, dim_block.y, dim_block.z,
+ dim_grid.x, dim_grid.y, plist->nsci*NCL_PER_SUPERCL,
+ NCL_PER_SUPERCL, plist->na_c);
+ }
+
+ nb_kernel<<<dim_grid, dim_block, shmem, stream>>>(*adat, *nbp, *plist, bCalcFshift);
+ CU_LAUNCH_ERR("k_calc_nb");
+
+ if (bDoTime)
+ {
+ stat = cudaEventRecord(t->stop_nb_k[iloc], stream);
+ CU_RET_ERR(stat, "cudaEventRecord failed");
+ }
+}
+
+void nbnxn_cuda_launch_cpyback(nbnxn_cuda_ptr_t cu_nb,
+ const nbnxn_atomdata_t *nbatom,
+ int flags,
+ int aloc)
+{
+ cudaError_t stat;
+ int adat_begin, adat_len, adat_end; /* local/nonlocal offset and length used for xq and f */
+ int iloc = -1;
+
+ /* determine interaction locality from atom locality */
+ if (LOCAL_A(aloc))
+ {
+ iloc = eintLocal;
+ }
+ else if (NONLOCAL_A(aloc))
+ {
+ iloc = eintNonlocal;
+ }
+ else
+ {
+ char stmp[STRLEN];
+ sprintf(stmp, "Invalid atom locality passed (%d); valid here is only "
+ "local (%d) or nonlocal (%d)", aloc, eatLocal, eatNonlocal);
+ gmx_incons(stmp);
+ }
+
+ cu_atomdata_t *adat = cu_nb->atdat;
+ cu_timers_t *t = cu_nb->timers;
+ bool bDoTime = cu_nb->bDoTime;
+ cudaStream_t stream = cu_nb->stream[iloc];
+
+ bool bCalcEner = flags & GMX_FORCE_VIRIAL;
+ bool bCalcFshift = flags & GMX_FORCE_VIRIAL;
+
+ /* don't launch copy-back if there was no work to do */
+ if (cu_nb->plist[iloc]->nsci == 0)
+ {
+ return;
+ }
+
+ /* calculate the atom data index range based on locality */
+ if (LOCAL_A(aloc))
+ {
+ adat_begin = 0;
+ adat_len = adat->natoms_local;
+ adat_end = cu_nb->atdat->natoms_local;
+ }
+ else
+ {
+ adat_begin = adat->natoms_local;
+ adat_len = adat->natoms - adat->natoms_local;
+ adat_end = cu_nb->atdat->natoms;
+ }
+
+ /* beginning of timed D2H section */
+ if (bDoTime)
+ {
+ stat = cudaEventRecord(t->start_nb_d2h[iloc], stream);
+ CU_RET_ERR(stat, "cudaEventRecord failed");
+ }
+
+ if (!cu_nb->bUseStreamSync)
+ {
+ /* For safety reasons set a few (5%) forces to NaN. This way even if the
+ polling "hack" fails with some future NVIDIA driver we'll get a crash. */
+ for (int i = adat_begin; i < 3*adat_end + 2; i += adat_len/20)
+ {
+#ifdef NAN
+ nbatom->out[0].f[i] = NAN;
+#else
+# ifdef _MSVC
+ if (numeric_limits<float>::has_quiet_NaN)
+ {
+ nbatom->out[0].f[i] = numeric_limits<float>::quiet_NaN();
+ }
+ else
+# endif
+ {
+ nbatom->out[0].f[i] = GMX_REAL_MAX;
+ }
+#endif
+ }
+
+ /* Set the last four bytes of the force array to a bit pattern
+ which can't be the result of the force calculation:
+ max exponent (127) and zero mantissa. */
+ *(unsigned int*)&nbatom->out[0].f[adat_end*3 - 1] = poll_wait_pattern;
+ }
+
+ /* With DD the local D2H transfer can only start after the non-local
+ has been launched. */
+ if (iloc == eintLocal && cu_nb->bUseTwoStreams)
+ {
+ stat = cudaStreamWaitEvent(stream, cu_nb->nonlocal_done, 0);
+ CU_RET_ERR(stat, "cudaStreamWaitEvent on nonlocal_done failed");
+ }
+
+ /* DtoH f */
+ cu_copy_D2H_async(nbatom->out[0].f + adat_begin * 3, adat->f + adat_begin,
+ (adat_len)*sizeof(*adat->f), stream);
+
+ /* After the non-local D2H is launched the nonlocal_done event can be
+ recorded which signals that the local D2H can proceed. This event is not
+ placed after the non-local kernel because we first need the non-local
+ data back first. */
+ if (iloc == eintNonlocal)
+ {
+ stat = cudaEventRecord(cu_nb->nonlocal_done, stream);
+ CU_RET_ERR(stat, "cudaEventRecord on nonlocal_done failed");
+ }
+
+ /* only transfer energies in the local stream */
+ if (LOCAL_I(iloc))
+ {
+ /* DtoH fshift */
+ if (bCalcFshift)
+ {
+ cu_copy_D2H_async(cu_nb->nbst.fshift, adat->fshift,
+ SHIFTS * sizeof(*cu_nb->nbst.fshift), stream);
+ }
+
+ /* DtoH energies */
+ if (bCalcEner)
+ {
+ cu_copy_D2H_async(cu_nb->nbst.e_lj, adat->e_lj,
+ sizeof(*cu_nb->nbst.e_lj), stream);
+ cu_copy_D2H_async(cu_nb->nbst.e_el, adat->e_el,
+ sizeof(*cu_nb->nbst.e_el), stream);
+ }
+ }
+
+ if (bDoTime)
+ {
+ stat = cudaEventRecord(t->stop_nb_d2h[iloc], stream);
+ CU_RET_ERR(stat, "cudaEventRecord failed");
+ }
+}
+
+/* Atomic compare-exchange operation on unsigned values. It is used in
+ * polling wait for the GPU.
+ */
+static inline bool atomic_cas(volatile unsigned int *ptr,
+ unsigned int oldval,
+ unsigned int newval)
+{
+ assert(ptr);
+
+#ifdef TMPI_ATOMICS
+ return tMPI_Atomic_cas((tMPI_Atomic_t *)ptr, oldval, newval);
+#else
+ gmx_incons("Atomic operations not available, atomic_cas() should not have been called!");
+ return true;
+#endif
+}
+
+void nbnxn_cuda_wait_gpu(nbnxn_cuda_ptr_t cu_nb,
+ const nbnxn_atomdata_t *nbatom,
+ int flags, int aloc,
+ real *e_lj, real *e_el, rvec *fshift)
+{
+ /* NOTE: only implemented for single-precision at this time */
+ cudaError_t stat;
+ int i, adat_end, iloc = -1;
+ volatile unsigned int *poll_word;
+
+ /* determine interaction locality from atom locality */
+ if (LOCAL_A(aloc))
+ {
+ iloc = eintLocal;
+ }
+ else if (NONLOCAL_A(aloc))
+ {
+ iloc = eintNonlocal;
+ }
+ else
+ {
+ char stmp[STRLEN];
+ sprintf(stmp, "Invalid atom locality passed (%d); valid here is only "
+ "local (%d) or nonlocal (%d)", aloc, eatLocal, eatNonlocal);
+ gmx_incons(stmp);
+ }
+
+ cu_plist_t *plist = cu_nb->plist[iloc];
+ cu_timers_t *timers = cu_nb->timers;
+ wallclock_gpu_t *timings = cu_nb->timings;
+ nb_staging nbst = cu_nb->nbst;
+
+ bool bCalcEner = flags & GMX_FORCE_VIRIAL;
+ bool bCalcFshift = flags & GMX_FORCE_VIRIAL;
+
+ /* turn energy calculation always on/off (for debugging/testing only) */
+ bCalcEner = (bCalcEner || always_ener) && !never_ener;
+
+ /* don't launch wait/update timers & counters if there was no work to do
+
+ NOTE: if timing with multiple GPUs (streams) becomes possible, the
+ counters could end up being inconsistent due to not being incremented
+ on some of the nodes! */
+ if (cu_nb->plist[iloc]->nsci == 0)
+ {
+ return;
+ }
+
+ /* calculate the atom data index range based on locality */
+ if (LOCAL_A(aloc))
+ {
+ adat_end = cu_nb->atdat->natoms_local;
+ }
+ else
+ {
+ adat_end = cu_nb->atdat->natoms;
+ }
+
+ if (cu_nb->bUseStreamSync)
+ {
+ stat = cudaStreamSynchronize(cu_nb->stream[iloc]);
+ CU_RET_ERR(stat, "cudaStreamSynchronize failed in cu_blockwait_nb");
+ }
+ else
+ {
+ /* Busy-wait until we get the signal pattern set in last byte
+ * of the l/nl float vector. This pattern corresponds to a floating
+ * point number which can't be the result of the force calculation
+ * (maximum, 127 exponent and 0 mantissa).
+ * The polling uses atomic compare-exchange.
+ */
+ poll_word = (volatile unsigned int*)&nbatom->out[0].f[adat_end*3 - 1];
+ while (atomic_cas(poll_word, poll_wait_pattern, poll_wait_pattern)) {}
+ }
+
+ /* timing data accumulation */
+ if (cu_nb->bDoTime)
+ {
+ /* only increase counter once (at local F wait) */
+ if (LOCAL_I(iloc))
+ {
+ timings->nb_c++;
+ timings->ktime[plist->bDoPrune ? 1 : 0][bCalcEner ? 1 : 0].c += 1;
+ }
+
+ /* kernel timings */
+ timings->ktime[plist->bDoPrune ? 1 : 0][bCalcEner ? 1 : 0].t +=
+ cu_event_elapsed(timers->start_nb_k[iloc], timers->stop_nb_k[iloc]);
+
+ /* X/q H2D and F D2H timings */
+ timings->nb_h2d_t += cu_event_elapsed(timers->start_nb_h2d[iloc],
+ timers->stop_nb_h2d[iloc]);
+ timings->nb_d2h_t += cu_event_elapsed(timers->start_nb_d2h[iloc],
+ timers->stop_nb_d2h[iloc]);
+
+ /* only count atdat and pair-list H2D at pair-search step */
+ if (plist->bDoPrune)
+ {
+ /* atdat transfer timing (add only once, at local F wait) */
+ if (LOCAL_A(aloc))
+ {
+ timings->pl_h2d_c++;
+ timings->pl_h2d_t += cu_event_elapsed(timers->start_atdat,
+ timers->stop_atdat);
+ }
+
+ timings->pl_h2d_t += cu_event_elapsed(timers->start_pl_h2d[iloc],
+ timers->stop_pl_h2d[iloc]);
+ }
+ }
+
+ /* add up energies and shift forces (only once at local F wait) */
+ if (LOCAL_I(iloc))
+ {
+ if (bCalcEner)
+ {
+ *e_lj += *nbst.e_lj;
+ *e_el += *nbst.e_el;
+ }
+
+ if (bCalcFshift)
+ {
+ for (i = 0; i < SHIFTS; i++)
+ {
+ fshift[i][0] += nbst.fshift[i].x;
+ fshift[i][1] += nbst.fshift[i].y;
+ fshift[i][2] += nbst.fshift[i].z;
+ }
+ }
+ }
+
+ /* turn off pruning (doesn't matter if this is pair-search step or not) */
+ plist->bDoPrune = false;
+}
+
+/*! Return the reference to the nbfp texture. */
+const struct texture<float, 1, cudaReadModeElementType>& nbnxn_cuda_get_nbfp_texref()
+{
- return tex_coulomb_tab;
++ return nbfp_texref;
+}
+
+/*! Return the reference to the coulomb_tab. */
+const struct texture<float, 1, cudaReadModeElementType>& nbnxn_cuda_get_coulomb_tab_texref()
+{
++ return coulomb_tab_texref;
+}
+
+/*! Set up the cache configuration for the non-bonded kernels,
+ */
+void nbnxn_cuda_set_cacheconfig(cuda_dev_info_t *devinfo)
+{
+ cudaError_t stat;
+
+ for (int i = 0; i < eelCuNR; i++)
+ {
+ for (int j = 0; j < nEnergyKernelTypes; j++)
+ {
+ for (int k = 0; k < nPruneKernelTypes; k++)
+ {
+ /* Legacy kernel 16/48 kB Shared/L1
+ * No analytical Ewald!
+ */
+ if (i != eelCuEWALD_ANA && i != eelCuEWALD_ANA_TWIN)
+ {
+ stat = cudaFuncSetCacheConfig(nb_legacy_kfunc_ptr[i][j][k], cudaFuncCachePreferL1);
+ CU_RET_ERR(stat, "cudaFuncSetCacheConfig failed");
+ }
+
+ if (devinfo->prop.major >= 3)
+ {
+ /* Default kernel on sm 3.x 48/16 kB Shared/L1 */
+ stat = cudaFuncSetCacheConfig(nb_default_kfunc_ptr[i][j][k], cudaFuncCachePreferShared);
+ }
+ else
+ {
+ /* On Fermi prefer L1 gives 2% higher performance */
+ /* Default kernel on sm_2.x 16/48 kB Shared/L1 */
+ stat = cudaFuncSetCacheConfig(nb_default_kfunc_ptr[i][j][k], cudaFuncCachePreferL1);
+ }
+ CU_RET_ERR(stat, "cudaFuncSetCacheConfig failed");
+ }
+ }
+ }
+}
--- /dev/null
- static void init_ewald_coulomb_force_table(cu_nbparam_t *nbp)
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2012, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * Gallium Rubidium Oxygen Manganese Argon Carbon Silicon
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <assert.h>
+
+#include <cuda.h>
+
+#include "gmx_fatal.h"
+#include "smalloc.h"
+#include "tables.h"
+#include "typedefs.h"
+#include "types/nb_verlet.h"
+#include "types/interaction_const.h"
+#include "types/force_flags.h"
+#include "../nbnxn_consts.h"
+#include "gmx_detect_hardware.h"
+
+#include "nbnxn_cuda_types.h"
+#include "../../gmxlib/cuda_tools/cudautils.cuh"
+#include "nbnxn_cuda_data_mgmt.h"
+#include "pmalloc_cuda.h"
+#include "gpu_utils.h"
+
+static bool bUseCudaEventBlockingSync = false; /* makes the CPU thread block */
+
+/* This is a heuristically determined parameter for the Fermi architecture for
+ * the minimum size of ci lists by multiplying this constant with the # of
+ * multiprocessors on the current device.
+ */
+static unsigned int gpu_min_ci_balanced_factor = 40;
+
+/* Functions from nbnxn_cuda.cu */
+extern void nbnxn_cuda_set_cacheconfig(cuda_dev_info_t *devinfo);
+extern const struct texture<float, 1, cudaReadModeElementType>& nbnxn_cuda_get_nbfp_texref();
+extern const struct texture<float, 1, cudaReadModeElementType>& nbnxn_cuda_get_coulomb_tab_texref();
+
+/* We should actually be using md_print_warn in md_logging.c,
+ * but we can't include mpi.h in CUDA code.
+ */
+static void md_print_warn(FILE *fplog,
+ const char *fmt, ...)
+{
+ va_list ap;
+
+ if (fplog != NULL)
+ {
+ /* We should only print to stderr on the master node,
+ * in most cases fplog is only set on the master node, so this works.
+ */
+ va_start(ap, fmt);
+ fprintf(stderr, "\n");
+ vfprintf(stderr, fmt, ap);
+ fprintf(stderr, "\n");
+ va_end(ap);
+
+ va_start(ap, fmt);
+ fprintf(fplog, "\n");
+ vfprintf(fplog, fmt, ap);
+ fprintf(fplog, "\n");
+ va_end(ap);
+ }
+}
+
+
+/* Fw. decl. */
+static void nbnxn_cuda_clear_e_fshift(nbnxn_cuda_ptr_t cu_nb);
+
+
+/*! Tabulates the Ewald Coulomb force and initializes the size/scale
+ and the table GPU array. If called with an already allocated table,
+ it just re-uploads the table.
+ */
- cudaChannelFormatDesc cd = cudaCreateChannelDesc<float>();
- stat = cudaBindTexture(NULL, &nbnxn_cuda_get_coulomb_tab_texref(),
- coul_tab, &cd, tabsize*sizeof(*coul_tab));
- CU_RET_ERR(stat, "cudaBindTexture on coul_tab failed");
++static void init_ewald_coulomb_force_table(cu_nbparam_t *nbp,
++ const cuda_dev_info_t *dev_info)
+{
+ float *ftmp, *coul_tab;
+ int tabsize;
+ double tabscale;
+ cudaError_t stat;
+
+ tabsize = GPU_EWALD_COULOMB_FORCE_TABLE_SIZE;
+ /* Subtract 2 iso 1 to avoid access out of range due to rounding */
+ tabscale = (tabsize - 2) / sqrt(nbp->rcoulomb_sq);
+
+ pmalloc((void**)&ftmp, tabsize*sizeof(*ftmp));
+
+ table_spline3_fill_ewald_lr(ftmp, NULL, NULL, tabsize,
+ 1/tabscale, nbp->ewald_beta);
+
+ /* If the table pointer == NULL the table is generated the first time =>
+ the array pointer will be saved to nbparam and the texture is bound.
+ */
+ coul_tab = nbp->coulomb_tab;
+ if (coul_tab == NULL)
+ {
+ stat = cudaMalloc((void **)&coul_tab, tabsize*sizeof(*coul_tab));
+ CU_RET_ERR(stat, "cudaMalloc failed on coul_tab");
+
+ nbp->coulomb_tab = coul_tab;
+
- init_ewald_coulomb_force_table(nbp);
++#ifdef TEXOBJ_SUPPORTED
++ /* Only device CC >= 3.0 (Kepler and later) support texture objects */
++ if (dev_info->prop.major >= 3)
++ {
++ cudaResourceDesc rd;
++ memset(&rd, 0, sizeof(rd));
++ rd.resType = cudaResourceTypeLinear;
++ rd.res.linear.devPtr = nbp->coulomb_tab;
++ rd.res.linear.desc.f = cudaChannelFormatKindFloat;
++ rd.res.linear.desc.x = 32;
++ rd.res.linear.sizeInBytes = tabsize*sizeof(*coul_tab);
++
++ cudaTextureDesc td;
++ memset(&td, 0, sizeof(td));
++ td.readMode = cudaReadModeElementType;
++ stat = cudaCreateTextureObject(&nbp->coulomb_tab_texobj, &rd, &td, NULL);
++ CU_RET_ERR(stat, "cudaCreateTextureObject on coulomb_tab_texobj failed");
++ }
++ else
++#endif
++ {
++ cudaChannelFormatDesc cd = cudaCreateChannelDesc<float>();
++ stat = cudaBindTexture(NULL, &nbnxn_cuda_get_coulomb_tab_texref(),
++ coul_tab, &cd, tabsize*sizeof(*coul_tab));
++ CU_RET_ERR(stat, "cudaBindTexture on coulomb_tab_texref failed");
++ }
+ }
+
+ cu_copy_H2D(coul_tab, ftmp, tabsize*sizeof(*coul_tab));
+
+ nbp->coulomb_tab_size = tabsize;
+ nbp->coulomb_tab_scale = tabscale;
+
+ pfree(ftmp);
+}
+
+
+/*! Initializes the atomdata structure first time, it only gets filled at
+ pair-search. */
+static void init_atomdata_first(cu_atomdata_t *ad, int ntypes)
+{
+ cudaError_t stat;
+
+ ad->ntypes = ntypes;
+ stat = cudaMalloc((void**)&ad->shift_vec, SHIFTS*sizeof(*ad->shift_vec));
+ CU_RET_ERR(stat, "cudaMalloc failed on ad->shift_vec");
+ ad->bShiftVecUploaded = false;
+
+ stat = cudaMalloc((void**)&ad->fshift, SHIFTS*sizeof(*ad->fshift));
+ CU_RET_ERR(stat, "cudaMalloc failed on ad->fshift");
+
+ stat = cudaMalloc((void**)&ad->e_lj, sizeof(*ad->e_lj));
+ CU_RET_ERR(stat, "cudaMalloc failed on ad->e_lj");
+ stat = cudaMalloc((void**)&ad->e_el, sizeof(*ad->e_el));
+ CU_RET_ERR(stat, "cudaMalloc failed on ad->e_el");
+
+ /* initialize to NULL poiters to data that is not allocated here and will
+ need reallocation in nbnxn_cuda_init_atomdata */
+ ad->xq = NULL;
+ ad->f = NULL;
+
+ /* size -1 indicates that the respective array hasn't been initialized yet */
+ ad->natoms = -1;
+ ad->nalloc = -1;
+}
+
+/*! Selects the Ewald kernel type, analytical on SM 3.0 and later, tabulated on
+ earlier GPUs, single or twin cut-off. */
+static int pick_ewald_kernel_type(bool bTwinCut,
+ const cuda_dev_info_t *dev_info)
+{
+ bool bUseAnalyticalEwald, bForceAnalyticalEwald, bForceTabulatedEwald;
+ int kernel_type;
+
+ /* Benchmarking/development environment variables to force the use of
+ analytical or tabulated Ewald kernel. */
+ bForceAnalyticalEwald = (getenv("GMX_CUDA_NB_ANA_EWALD") != NULL);
+ bForceTabulatedEwald = (getenv("GMX_CUDA_NB_TAB_EWALD") != NULL);
+
+ if (bForceAnalyticalEwald && bForceTabulatedEwald)
+ {
+ gmx_incons("Both analytical and tabulated Ewald CUDA non-bonded kernels "
+ "requested through environment variables.");
+ }
+
+ /* By default, on SM 3.0 and later use analytical Ewald, on earlier tabulated. */
+ if ((dev_info->prop.major >= 3 || bForceAnalyticalEwald) && !bForceTabulatedEwald)
+ {
+ bUseAnalyticalEwald = true;
+
+ if (debug)
+ {
+ fprintf(debug, "Using analytical Ewald CUDA kernels\n");
+ }
+ }
+ else
+ {
+ bUseAnalyticalEwald = false;
+
+ if (debug)
+ {
+ fprintf(debug, "Using tabulated Ewald CUDA kernels\n");
+ }
+ }
+
+ /* Use twin cut-off kernels if requested by bTwinCut or the env. var.
+ forces it (use it for debugging/benchmarking only). */
+ if (!bTwinCut && (getenv("GMX_CUDA_NB_EWALD_TWINCUT") == NULL))
+ {
+ kernel_type = bUseAnalyticalEwald ? eelCuEWALD_ANA : eelCuEWALD_TAB;
+ }
+ else
+ {
+ kernel_type = bUseAnalyticalEwald ? eelCuEWALD_ANA_TWIN : eelCuEWALD_TAB_TWIN;
+ }
+
+ return kernel_type;
+}
+
+
+/*! Initializes the nonbonded parameter data structure. */
+static void init_nbparam(cu_nbparam_t *nbp,
+ const interaction_const_t *ic,
+ const nbnxn_atomdata_t *nbat,
+ const cuda_dev_info_t *dev_info)
+{
+ cudaError_t stat;
+ int ntypes, nnbfp;
+
+ ntypes = nbat->ntype;
+
+ nbp->ewald_beta = ic->ewaldcoeff;
+ nbp->sh_ewald = ic->sh_ewald;
+ nbp->epsfac = ic->epsfac;
+ nbp->two_k_rf = 2.0 * ic->k_rf;
+ nbp->c_rf = ic->c_rf;
+ nbp->rvdw_sq = ic->rvdw * ic->rvdw;
+ nbp->rcoulomb_sq= ic->rcoulomb * ic->rcoulomb;
+ nbp->rlist_sq = ic->rlist * ic->rlist;
+ nbp->sh_invrc6 = ic->sh_invrc6;
+
+ if (ic->eeltype == eelCUT)
+ {
+ nbp->eeltype = eelCuCUT;
+ }
+ else if (EEL_RF(ic->eeltype))
+ {
+ nbp->eeltype = eelCuRF;
+ }
+ else if ((EEL_PME(ic->eeltype) || ic->eeltype==eelEWALD))
+ {
+ /* Initially rcoulomb == rvdw, so it's surely not twin cut-off. */
+ nbp->eeltype = pick_ewald_kernel_type(false, dev_info);
+ }
+ else
+ {
+ /* Shouldn't happen, as this is checked when choosing Verlet-scheme */
+ gmx_incons("The requested electrostatics type is not implemented in the CUDA GPU accelerated kernels!");
+ }
+
+ /* generate table for PME */
+ nbp->coulomb_tab = NULL;
+ if (nbp->eeltype == eelCuEWALD_TAB || nbp->eeltype == eelCuEWALD_TAB_TWIN)
+ {
- cudaChannelFormatDesc cd = cudaCreateChannelDesc<float>();
- stat = cudaBindTexture(NULL, &nbnxn_cuda_get_nbfp_texref(),
- nbp->nbfp, &cd, nnbfp*sizeof(*nbp->nbfp));
- CU_RET_ERR(stat, "cudaBindTexture on nbfp failed");
++ init_ewald_coulomb_force_table(nbp, dev_info);
+ }
+
+ nnbfp = 2*ntypes*ntypes;
+ stat = cudaMalloc((void **)&nbp->nbfp, nnbfp*sizeof(*nbp->nbfp));
+ CU_RET_ERR(stat, "cudaMalloc failed on nbp->nbfp");
+ cu_copy_H2D(nbp->nbfp, nbat->nbfp, nnbfp*sizeof(*nbp->nbfp));
+
- init_ewald_coulomb_force_table(cu_nb->nbparam);
++#ifdef TEXOBJ_SUPPORTED
++ /* Only device CC >= 3.0 (Kepler and later) support texture objects */
++ if (dev_info->prop.major >= 3)
++ {
++ cudaResourceDesc rd;
++ memset(&rd, 0, sizeof(rd));
++ rd.resType = cudaResourceTypeLinear;
++ rd.res.linear.devPtr = nbp->nbfp;
++ rd.res.linear.desc.f = cudaChannelFormatKindFloat;
++ rd.res.linear.desc.x = 32;
++ rd.res.linear.sizeInBytes = nnbfp*sizeof(*nbp->nbfp);
++
++ cudaTextureDesc td;
++ memset(&td, 0, sizeof(td));
++ td.readMode = cudaReadModeElementType;
++ stat = cudaCreateTextureObject(&nbp->nbfp_texobj, &rd, &td, NULL);
++ CU_RET_ERR(stat, "cudaCreateTextureObject on nbfp_texobj failed");
++ }
++ else
++#endif
++ {
++ cudaChannelFormatDesc cd = cudaCreateChannelDesc<float>();
++ stat = cudaBindTexture(NULL, &nbnxn_cuda_get_nbfp_texref(),
++ nbp->nbfp, &cd, nnbfp*sizeof(*nbp->nbfp));
++ CU_RET_ERR(stat, "cudaBindTexture on nbfp_texref failed");
++ }
+}
+
+/*! Re-generate the GPU Ewald force table, resets rlist, and update the
+ * electrostatic type switching to twin cut-off (or back) if needed. */
+void nbnxn_cuda_pme_loadbal_update_param(nbnxn_cuda_ptr_t cu_nb,
+ const interaction_const_t *ic)
+{
+ cu_nbparam_t *nbp = cu_nb->nbparam;
+
+ nbp->rlist_sq = ic->rlist * ic->rlist;
+ nbp->rcoulomb_sq = ic->rcoulomb * ic->rcoulomb;
+ nbp->ewald_beta = ic->ewaldcoeff;
+
+ nbp->eeltype = pick_ewald_kernel_type(ic->rcoulomb != ic->rvdw,
+ cu_nb->dev_info);
+
- /* set device info, just point it to the right GPU among the detected ones */
- nb->dev_info = &gpu_info->cuda_dev[get_gpu_device_id(gpu_info, my_gpu_index)];
-
++ init_ewald_coulomb_force_table(cu_nb->nbparam, cu_nb->dev_info);
+}
+
+/*! Initializes the pair list data structure. */
+static void init_plist(cu_plist_t *pl)
+{
+ /* initialize to NULL pointers to data that is not allocated here and will
+ need reallocation in nbnxn_cuda_init_pairlist */
+ pl->sci = NULL;
+ pl->cj4 = NULL;
+ pl->excl = NULL;
+
+ /* size -1 indicates that the respective array hasn't been initialized yet */
+ pl->na_c = -1;
+ pl->nsci = -1;
+ pl->sci_nalloc = -1;
+ pl->ncj4 = -1;
+ pl->cj4_nalloc = -1;
+ pl->nexcl = -1;
+ pl->excl_nalloc = -1;
+ pl->bDoPrune = false;
+}
+
+/*! Initializes the timer data structure. */
+static void init_timers(cu_timers_t *t, bool bUseTwoStreams)
+{
+ cudaError_t stat;
+ int eventflags = ( bUseCudaEventBlockingSync ? cudaEventBlockingSync: cudaEventDefault );
+
+ stat = cudaEventCreateWithFlags(&(t->start_atdat), eventflags);
+ CU_RET_ERR(stat, "cudaEventCreate on start_atdat failed");
+ stat = cudaEventCreateWithFlags(&(t->stop_atdat), eventflags);
+ CU_RET_ERR(stat, "cudaEventCreate on stop_atdat failed");
+
+ /* The non-local counters/stream (second in the array) are needed only with DD. */
+ for (int i = 0; i <= (bUseTwoStreams ? 1 : 0); i++)
+ {
+ stat = cudaEventCreateWithFlags(&(t->start_nb_k[i]), eventflags);
+ CU_RET_ERR(stat, "cudaEventCreate on start_nb_k failed");
+ stat = cudaEventCreateWithFlags(&(t->stop_nb_k[i]), eventflags);
+ CU_RET_ERR(stat, "cudaEventCreate on stop_nb_k failed");
+
+
+ stat = cudaEventCreateWithFlags(&(t->start_pl_h2d[i]), eventflags);
+ CU_RET_ERR(stat, "cudaEventCreate on start_pl_h2d failed");
+ stat = cudaEventCreateWithFlags(&(t->stop_pl_h2d[i]), eventflags);
+ CU_RET_ERR(stat, "cudaEventCreate on stop_pl_h2d failed");
+
+ stat = cudaEventCreateWithFlags(&(t->start_nb_h2d[i]), eventflags);
+ CU_RET_ERR(stat, "cudaEventCreate on start_nb_h2d failed");
+ stat = cudaEventCreateWithFlags(&(t->stop_nb_h2d[i]), eventflags);
+ CU_RET_ERR(stat, "cudaEventCreate on stop_nb_h2d failed");
+
+ stat = cudaEventCreateWithFlags(&(t->start_nb_d2h[i]), eventflags);
+ CU_RET_ERR(stat, "cudaEventCreate on start_nb_d2h failed");
+ stat = cudaEventCreateWithFlags(&(t->stop_nb_d2h[i]), eventflags);
+ CU_RET_ERR(stat, "cudaEventCreate on stop_nb_d2h failed");
+ }
+}
+
+/*! Initializes the timings data structure. */
+static void init_timings(wallclock_gpu_t *t)
+{
+ int i, j;
+
+ t->nb_h2d_t = 0.0;
+ t->nb_d2h_t = 0.0;
+ t->nb_c = 0;
+ t->pl_h2d_t = 0.0;
+ t->pl_h2d_c = 0;
+ for (i = 0; i < 2; i++)
+ {
+ for(j = 0; j < 2; j++)
+ {
+ t->ktime[i][j].t = 0.0;
+ t->ktime[i][j].c = 0;
+ }
+ }
+}
+
+/* Decide which kernel version to use (default or legacy) based on:
+ * - CUDA version used for compilation
+ * - non-bonded kernel selector environment variables
+ * - GPU architecture version
+ */
+static int pick_nbnxn_kernel_version(FILE *fplog,
+ cuda_dev_info_t *devinfo)
+{
+ bool bForceLegacyKernel, bForceDefaultKernel, bCUDA40, bCUDA32;
+ char sbuf[STRLEN];
+ int kver;
+
+ /* Legacy kernel (former k2), kept for backward compatibility as it is
+ faster than the default with CUDA 3.2/4.0 on Fermi (not on Kepler). */
+ bForceLegacyKernel = (getenv("GMX_CUDA_NB_LEGACY") != NULL);
+ /* default kernel (former k3). */
+ bForceDefaultKernel = (getenv("GMX_CUDA_NB_DEFAULT") != NULL);
+
+ if ((unsigned)(bForceLegacyKernel + bForceDefaultKernel) > 1)
+ {
+ gmx_fatal(FARGS, "Multiple CUDA non-bonded kernels requested; to manually pick a kernel set only one \n"
+ "of the following environment variables: \n"
+ "GMX_CUDA_NB_DEFAULT, GMX_CUDA_NB_LEGACY");
+ }
+
+ bCUDA32 = bCUDA40 = false;
+#if CUDA_VERSION == 3200
+ bCUDA32 = true;
+ sprintf(sbuf, "3.2");
+#elif CUDA_VERSION == 4000
+ bCUDA40 = true;
+ sprintf(sbuf, "4.0");
+#endif
+
+ /* default is default ;) */
+ kver = eNbnxnCuKDefault;
+
+ /* Consider switching to legacy kernels only on Fermi */
+ if (devinfo->prop.major < 3 && (bCUDA32 || bCUDA40))
+ {
+ /* use legacy kernel unless something else is forced by an env. var */
+ if (bForceDefaultKernel)
+ {
+ md_print_warn(fplog,
+ "NOTE: CUDA %s compilation detected; with this compiler version the legacy\n"
+ " non-bonded kernels perform best. However, the default kernels were\n"
+ " selected by the GMX_CUDA_NB_DEFAULT environment variable.\n"
+ " For best performance upgrade your CUDA toolkit.\n",
+ sbuf);
+ }
+ else
+ {
+ kver = eNbnxnCuKLegacy;
+ }
+ }
+ else
+ {
+ /* issue note if the non-default kernel is forced by an env. var */
+ if (bForceLegacyKernel)
+ {
+ md_print_warn(fplog,
+ "NOTE: Legacy non-bonded CUDA kernels selected by the GMX_CUDA_NB_LEGACY\n"
+ " env. var. Consider using using the default kernels which should be faster!\n");
+
+ kver = eNbnxnCuKLegacy;
+ }
+ }
+
+ return kver;
+}
+
+void nbnxn_cuda_init(FILE *fplog,
+ nbnxn_cuda_ptr_t *p_cu_nb,
+ const gmx_gpu_info_t *gpu_info, int my_gpu_index,
+ gmx_bool bLocalAndNonlocal)
+{
+ cudaError_t stat;
+ nbnxn_cuda_ptr_t nb;
+ char sbuf[STRLEN];
+ bool bStreamSync, bNoStreamSync, bTMPIAtomics, bX86, bOldDriver;
+ int cuda_drv_ver;
+
+ assert(gpu_info);
+
+ if (p_cu_nb == NULL) return;
+
+ snew(nb, 1);
+ snew(nb->atdat, 1);
+ snew(nb->nbparam, 1);
+ snew(nb->plist[eintLocal], 1);
+ if (bLocalAndNonlocal)
+ {
+ snew(nb->plist[eintNonlocal], 1);
+ }
+
+ nb->bUseTwoStreams = bLocalAndNonlocal;
+
+ snew(nb->timers, 1);
+ snew(nb->timings, 1);
+
+ /* init nbst */
+ pmalloc((void**)&nb->nbst.e_lj, sizeof(*nb->nbst.e_lj));
+ pmalloc((void**)&nb->nbst.e_el, sizeof(*nb->nbst.e_el));
+ pmalloc((void**)&nb->nbst.fshift, SHIFTS * sizeof(*nb->nbst.fshift));
+
+ init_plist(nb->plist[eintLocal]);
+
++ /* set device info, just point it to the right GPU among the detected ones */
++ nb->dev_info = &gpu_info->cuda_dev[get_gpu_device_id(gpu_info, my_gpu_index)];
++
+ /* local/non-local GPU streams */
+ stat = cudaStreamCreate(&nb->stream[eintLocal]);
+ CU_RET_ERR(stat, "cudaStreamCreate on stream[eintLocal] failed");
+ if (nb->bUseTwoStreams)
+ {
+ init_plist(nb->plist[eintNonlocal]);
++
++ /* CUDA stream priority available in the CUDA RT 5.5 API.
++ * Note that the device we're running on does not have to support
++ * priorities, because we are querying the priority range which in this
++ * case will be a single value.
++ */
++#if CUDA_VERSION >= 5500
++ {
++ int highest_priority;
++ stat = cudaDeviceGetStreamPriorityRange(NULL, &highest_priority);
++ CU_RET_ERR(stat, "cudaDeviceGetStreamPriorityRange failed");
++
++ stat = cudaStreamCreateWithPriority(&nb->stream[eintNonlocal],
++ cudaStreamDefault,
++ highest_priority);
++ CU_RET_ERR(stat, "cudaStreamCreateWithPriority on stream[eintNonlocal] failed");
++ }
++#else
+ stat = cudaStreamCreate(&nb->stream[eintNonlocal]);
+ CU_RET_ERR(stat, "cudaStreamCreate on stream[eintNonlocal] failed");
++#endif
+ }
+
+ /* init events for sychronization (timing disabled for performance reasons!) */
+ stat = cudaEventCreateWithFlags(&nb->nonlocal_done, cudaEventDisableTiming);
+ CU_RET_ERR(stat, "cudaEventCreate on nonlocal_done failed");
+ stat = cudaEventCreateWithFlags(&nb->misc_ops_done, cudaEventDisableTiming);
+ CU_RET_ERR(stat, "cudaEventCreate on misc_ops_one failed");
+
- #if defined(i386) || defined(__x86_64__)
+ /* On GPUs with ECC enabled, cudaStreamSynchronize shows a large overhead
+ * (which increases with shorter time/step) caused by a known CUDA driver bug.
+ * To work around the issue we'll use an (admittedly fragile) memory polling
+ * waiting to preserve performance. This requires support for atomic
+ * operations and only works on x86/x86_64.
+ * With polling wait event-timing also needs to be disabled.
+ *
+ * The overhead is greatly reduced in API v5.0 drivers and the improvement
+ $ is independent of runtime version. Hence, with API v5.0 drivers and later
+ * we won't switch to polling.
+ *
+ * NOTE: Unfortunately, this is known to fail when GPUs are shared by (t)MPI,
+ * ranks so we will also disable it in that case.
+ */
+
+ bStreamSync = getenv("GMX_CUDA_STREAMSYNC") != NULL;
+ bNoStreamSync = getenv("GMX_NO_CUDA_STREAMSYNC") != NULL;
+
+#ifdef TMPI_ATOMICS
+ bTMPIAtomics = true;
+#else
+ bTMPIAtomics = false;
+#endif
+
- stat = cudaUnbindTexture(nbnxn_cuda_get_coulomb_tab_texref());
- CU_RET_ERR(stat, "cudaUnbindTexture on coulomb_tab failed");
- cu_free_buffered(nbparam->coulomb_tab, &nbparam->coulomb_tab_size);
++#ifdef GMX_IS_X86
+ bX86 = true;
+#else
+ bX86 = false;
+#endif
+
+ if (bStreamSync && bNoStreamSync)
+ {
+ gmx_fatal(FARGS, "Conflicting environment variables: both GMX_CUDA_STREAMSYNC and GMX_NO_CUDA_STREAMSYNC defined");
+ }
+
+ stat = cudaDriverGetVersion(&cuda_drv_ver);
+ CU_RET_ERR(stat, "cudaDriverGetVersion failed");
+
+ bOldDriver = (cuda_drv_ver < 5000);
+
+ if ((nb->dev_info->prop.ECCEnabled == 1) && bOldDriver)
+ {
+ /* Polling wait should be used instead of cudaStreamSynchronize only if:
+ * - ECC is ON & driver is old (checked above),
+ * - we're on x86/x86_64,
+ * - atomics are available, and
+ * - GPUs are not being shared.
+ */
+ bool bShouldUsePollSync = (bX86 && bTMPIAtomics &&
+ (gmx_count_gpu_dev_shared(gpu_info) < 1));
+
+ if (bStreamSync)
+ {
+ nb->bUseStreamSync = true;
+
+ /* only warn if polling should be used */
+ if (bShouldUsePollSync)
+ {
+ md_print_warn(fplog,
+ "NOTE: Using a GPU with ECC enabled and CUDA driver API version <5.0, but\n"
+ " cudaStreamSynchronize waiting is forced by the GMX_CUDA_STREAMSYNC env. var.\n");
+ }
+ }
+ else
+ {
+ nb->bUseStreamSync = !bShouldUsePollSync;
+
+ if (bShouldUsePollSync)
+ {
+ md_print_warn(fplog,
+ "NOTE: Using a GPU with ECC enabled and CUDA driver API version <5.0, known to\n"
+ " cause performance loss. Switching to the alternative polling GPU wait.\n"
+ " If you encounter issues, switch back to standard GPU waiting by setting\n"
+ " the GMX_CUDA_STREAMSYNC environment variable.\n");
+ }
+ else
+ {
+ /* Tell the user that the ECC+old driver combination can be bad */
+ sprintf(sbuf,
+ "NOTE: Using a GPU with ECC enabled and CUDA driver API version <5.0.\n"
+ " A known bug in this driver version can cause performance loss.\n"
+ " However, the polling wait workaround can not be used because\n%s\n"
+ " Consider updating the driver or turning ECC off.",
+ (bX86 && bTMPIAtomics) ?
+ " GPU(s) are being oversubscribed." :
+ " atomic operations are not supported by the platform/CPU+compiler.");
+ md_print_warn(fplog, sbuf);
+ }
+ }
+ }
+ else
+ {
+ if (bNoStreamSync)
+ {
+ nb->bUseStreamSync = false;
+
+ md_print_warn(fplog,
+ "NOTE: Polling wait for GPU synchronization requested by GMX_NO_CUDA_STREAMSYNC\n");
+ }
+ else
+ {
+ /* no/off ECC, cudaStreamSynchronize not turned off by env. var. */
+ nb->bUseStreamSync = true;
+ }
+ }
+
+ /* CUDA timing disabled as event timers don't work:
+ - with multiple streams = domain-decomposition;
+ - with the polling waiting hack (without cudaStreamSynchronize);
+ - when turned off by GMX_DISABLE_CUDA_TIMING.
+ */
+ nb->bDoTime = (!nb->bUseTwoStreams && nb->bUseStreamSync &&
+ (getenv("GMX_DISABLE_CUDA_TIMING") == NULL));
+
+ if (nb->bDoTime)
+ {
+ init_timers(nb->timers, nb->bUseTwoStreams);
+ init_timings(nb->timings);
+ }
+
+ /* set the kernel type for the current GPU */
+ nb->kernel_ver = pick_nbnxn_kernel_version(fplog, nb->dev_info);
+ /* pick L1 cache configuration */
+ nbnxn_cuda_set_cacheconfig(nb->dev_info);
+
+ *p_cu_nb = nb;
+
+ if (debug)
+ {
+ fprintf(debug, "Initialized CUDA data structures.\n");
+ }
+}
+
+void nbnxn_cuda_init_const(nbnxn_cuda_ptr_t cu_nb,
+ const interaction_const_t *ic,
+ const nonbonded_verlet_group_t *nbv_group)
+{
+ init_atomdata_first(cu_nb->atdat, nbv_group[0].nbat->ntype);
+ init_nbparam(cu_nb->nbparam, ic, nbv_group[0].nbat, cu_nb->dev_info);
+
+ /* clear energy and shift force outputs */
+ nbnxn_cuda_clear_e_fshift(cu_nb);
+}
+
+void nbnxn_cuda_init_pairlist(nbnxn_cuda_ptr_t cu_nb,
+ const nbnxn_pairlist_t *h_plist,
+ int iloc)
+{
+ char sbuf[STRLEN];
+ cudaError_t stat;
+ bool bDoTime = cu_nb->bDoTime;
+ cudaStream_t stream = cu_nb->stream[iloc];
+ cu_plist_t *d_plist = cu_nb->plist[iloc];
+
+ if (d_plist->na_c < 0)
+ {
+ d_plist->na_c = h_plist->na_ci;
+ }
+ else
+ {
+ if (d_plist->na_c != h_plist->na_ci)
+ {
+ sprintf(sbuf, "In cu_init_plist: the #atoms per cell has changed (from %d to %d)",
+ d_plist->na_c, h_plist->na_ci);
+ gmx_incons(sbuf);
+ }
+ }
+
+ if (bDoTime)
+ {
+ stat = cudaEventRecord(cu_nb->timers->start_pl_h2d[iloc], stream);
+ CU_RET_ERR(stat, "cudaEventRecord failed");
+ }
+
+ cu_realloc_buffered((void **)&d_plist->sci, h_plist->sci, sizeof(*d_plist->sci),
+ &d_plist->nsci, &d_plist->sci_nalloc,
+ h_plist->nsci,
+ stream, true);
+
+ cu_realloc_buffered((void **)&d_plist->cj4, h_plist->cj4, sizeof(*d_plist->cj4),
+ &d_plist->ncj4, &d_plist->cj4_nalloc,
+ h_plist->ncj4,
+ stream, true);
+
+ cu_realloc_buffered((void **)&d_plist->excl, h_plist->excl, sizeof(*d_plist->excl),
+ &d_plist->nexcl, &d_plist->excl_nalloc,
+ h_plist->nexcl,
+ stream, true);
+
+ if (bDoTime)
+ {
+ stat = cudaEventRecord(cu_nb->timers->stop_pl_h2d[iloc], stream);
+ CU_RET_ERR(stat, "cudaEventRecord failed");
+ }
+
+ /* need to prune the pair list during the next step */
+ d_plist->bDoPrune = true;
+}
+
+void nbnxn_cuda_upload_shiftvec(nbnxn_cuda_ptr_t cu_nb,
+ const nbnxn_atomdata_t *nbatom)
+{
+ cu_atomdata_t *adat = cu_nb->atdat;
+ cudaStream_t ls = cu_nb->stream[eintLocal];
+
+ /* only if we have a dynamic box */
+ if (nbatom->bDynamicBox || !adat->bShiftVecUploaded)
+ {
+ cu_copy_H2D_async(adat->shift_vec, nbatom->shift_vec,
+ SHIFTS * sizeof(*adat->shift_vec), ls);
+ adat->bShiftVecUploaded = true;
+ }
+}
+
+/*! Clears the first natoms_clear elements of the GPU nonbonded force output array. */
+static void nbnxn_cuda_clear_f(nbnxn_cuda_ptr_t cu_nb, int natoms_clear)
+{
+ cudaError_t stat;
+ cu_atomdata_t *adat = cu_nb->atdat;
+ cudaStream_t ls = cu_nb->stream[eintLocal];
+
+ stat = cudaMemsetAsync(adat->f, 0, natoms_clear * sizeof(*adat->f), ls);
+ CU_RET_ERR(stat, "cudaMemsetAsync on f falied");
+}
+
+/*! Clears nonbonded shift force output array and energy outputs on the GPU. */
+static void nbnxn_cuda_clear_e_fshift(nbnxn_cuda_ptr_t cu_nb)
+{
+ cudaError_t stat;
+ cu_atomdata_t *adat = cu_nb->atdat;
+ cudaStream_t ls = cu_nb->stream[eintLocal];
+
+ stat = cudaMemsetAsync(adat->fshift, 0, SHIFTS * sizeof(*adat->fshift), ls);
+ CU_RET_ERR(stat, "cudaMemsetAsync on fshift falied");
+ stat = cudaMemsetAsync(adat->e_lj, 0, sizeof(*adat->e_lj), ls);
+ CU_RET_ERR(stat, "cudaMemsetAsync on e_lj falied");
+ stat = cudaMemsetAsync(adat->e_el, 0, sizeof(*adat->e_el), ls);
+ CU_RET_ERR(stat, "cudaMemsetAsync on e_el falied");
+}
+
+void nbnxn_cuda_clear_outputs(nbnxn_cuda_ptr_t cu_nb, int flags)
+{
+ nbnxn_cuda_clear_f(cu_nb, cu_nb->atdat->natoms);
+ /* clear shift force array and energies if the outputs were
+ used in the current step */
+ if (flags & GMX_FORCE_VIRIAL)
+ {
+ nbnxn_cuda_clear_e_fshift(cu_nb);
+ }
+}
+
+void nbnxn_cuda_init_atomdata(nbnxn_cuda_ptr_t cu_nb,
+ const nbnxn_atomdata_t *nbat)
+{
+ cudaError_t stat;
+ int nalloc, natoms;
+ bool realloced;
+ bool bDoTime = cu_nb->bDoTime;
+ cu_timers_t *timers = cu_nb->timers;
+ cu_atomdata_t *d_atdat = cu_nb->atdat;
+ cudaStream_t ls = cu_nb->stream[eintLocal];
+
+ natoms = nbat->natoms;
+ realloced = false;
+
+ if (bDoTime)
+ {
+ /* time async copy */
+ stat = cudaEventRecord(timers->start_atdat, ls);
+ CU_RET_ERR(stat, "cudaEventRecord failed");
+ }
+
+ /* need to reallocate if we have to copy more atoms than the amount of space
+ available and only allocate if we haven't initialized yet, i.e d_atdat->natoms == -1 */
+ if (natoms > d_atdat->nalloc)
+ {
+ nalloc = over_alloc_small(natoms);
+
+ /* free up first if the arrays have already been initialized */
+ if (d_atdat->nalloc != -1)
+ {
+ cu_free_buffered(d_atdat->f, &d_atdat->natoms, &d_atdat->nalloc);
+ cu_free_buffered(d_atdat->xq);
+ cu_free_buffered(d_atdat->atom_types);
+ }
+
+ stat = cudaMalloc((void **)&d_atdat->f, nalloc*sizeof(*d_atdat->f));
+ CU_RET_ERR(stat, "cudaMalloc failed on d_atdat->f");
+ stat = cudaMalloc((void **)&d_atdat->xq, nalloc*sizeof(*d_atdat->xq));
+ CU_RET_ERR(stat, "cudaMalloc failed on d_atdat->xq");
+
+ stat = cudaMalloc((void **)&d_atdat->atom_types, nalloc*sizeof(*d_atdat->atom_types));
+ CU_RET_ERR(stat, "cudaMalloc failed on d_atdat->atom_types");
+
+ d_atdat->nalloc = nalloc;
+ realloced = true;
+ }
+
+ d_atdat->natoms = natoms;
+ d_atdat->natoms_local = nbat->natoms_local;
+
+ /* need to clear GPU f output if realloc happened */
+ if (realloced)
+ {
+ nbnxn_cuda_clear_f(cu_nb, nalloc);
+ }
+
+ cu_copy_H2D_async(d_atdat->atom_types, nbat->type,
+ natoms*sizeof(*d_atdat->atom_types), ls);
+
+ if (bDoTime)
+ {
+ stat = cudaEventRecord(timers->stop_atdat, ls);
+ CU_RET_ERR(stat, "cudaEventRecord failed");
+ }
+}
+
+void nbnxn_cuda_free(FILE *fplog, nbnxn_cuda_ptr_t cu_nb)
+{
+ cudaError_t stat;
+ cu_atomdata_t *atdat;
+ cu_nbparam_t *nbparam;
+ cu_plist_t *plist, *plist_nl;
+ cu_timers_t *timers;
+
+ if (cu_nb == NULL) return;
+
+ atdat = cu_nb->atdat;
+ nbparam = cu_nb->nbparam;
+ plist = cu_nb->plist[eintLocal];
+ plist_nl = cu_nb->plist[eintNonlocal];
+ timers = cu_nb->timers;
+
+ if (nbparam->eeltype == eelCuEWALD_TAB || nbparam->eeltype == eelCuEWALD_TAB_TWIN)
+ {
- stat = cudaUnbindTexture(nbnxn_cuda_get_nbfp_texref());
- CU_RET_ERR(stat, "cudaUnbindTexture on coulomb_tab failed");
++
++#ifdef TEXOBJ_SUPPORTED
++ /* Only device CC >= 3.0 (Kepler and later) support texture objects */
++ if (cu_nb->dev_info->prop.major >= 3)
++ {
++ stat = cudaDestroyTextureObject(nbparam->coulomb_tab_texobj);
++ CU_RET_ERR(stat, "cudaDestroyTextureObject on coulomb_tab_texobj failed");
++ }
++ else
++#endif
++ {
++ stat = cudaUnbindTexture(nbnxn_cuda_get_coulomb_tab_texref());
++ CU_RET_ERR(stat, "cudaUnbindTexture on coulomb_tab_texref failed");
++ }
++ cu_free_buffered(nbparam->coulomb_tab, &nbparam->coulomb_tab_size);
+ }
+
+ stat = cudaEventDestroy(cu_nb->nonlocal_done);
+ CU_RET_ERR(stat, "cudaEventDestroy failed on timers->nonlocal_done");
+ stat = cudaEventDestroy(cu_nb->misc_ops_done);
+ CU_RET_ERR(stat, "cudaEventDestroy failed on timers->misc_ops_done");
+
+ if (cu_nb->bDoTime)
+ {
+ stat = cudaEventDestroy(timers->start_atdat);
+ CU_RET_ERR(stat, "cudaEventDestroy failed on timers->start_atdat");
+ stat = cudaEventDestroy(timers->stop_atdat);
+ CU_RET_ERR(stat, "cudaEventDestroy failed on timers->stop_atdat");
+
+ /* The non-local counters/stream (second in the array) are needed only with DD. */
+ for (int i = 0; i <= (cu_nb->bUseTwoStreams ? 1 : 0); i++)
+ {
+ stat = cudaEventDestroy(timers->start_nb_k[i]);
+ CU_RET_ERR(stat, "cudaEventDestroy failed on timers->start_nb_k");
+ stat = cudaEventDestroy(timers->stop_nb_k[i]);
+ CU_RET_ERR(stat, "cudaEventDestroy failed on timers->stop_nb_k");
+
+ stat = cudaEventDestroy(timers->start_pl_h2d[i]);
+ CU_RET_ERR(stat, "cudaEventDestroy failed on timers->start_pl_h2d");
+ stat = cudaEventDestroy(timers->stop_pl_h2d[i]);
+ CU_RET_ERR(stat, "cudaEventDestroy failed on timers->stop_pl_h2d");
+
+ stat = cudaStreamDestroy(cu_nb->stream[i]);
+ CU_RET_ERR(stat, "cudaStreamDestroy failed on stream");
+
+ stat = cudaEventDestroy(timers->start_nb_h2d[i]);
+ CU_RET_ERR(stat, "cudaEventDestroy failed on timers->start_nb_h2d");
+ stat = cudaEventDestroy(timers->stop_nb_h2d[i]);
+ CU_RET_ERR(stat, "cudaEventDestroy failed on timers->stop_nb_h2d");
+
+ stat = cudaEventDestroy(timers->start_nb_d2h[i]);
+ CU_RET_ERR(stat, "cudaEventDestroy failed on timers->start_nb_d2h");
+ stat = cudaEventDestroy(timers->stop_nb_d2h[i]);
+ CU_RET_ERR(stat, "cudaEventDestroy failed on timers->stop_nb_d2h");
+ }
+ }
+
++#ifdef TEXOBJ_SUPPORTED
++ /* Only device CC >= 3.0 (Kepler and later) support texture objects */
++ if (cu_nb->dev_info->prop.major >= 3)
++ {
++ stat = cudaDestroyTextureObject(nbparam->nbfp_texobj);
++ CU_RET_ERR(stat, "cudaDestroyTextureObject on nbfp_texobj failed");
++ }
++ else
++#endif
++ {
++ stat = cudaUnbindTexture(nbnxn_cuda_get_nbfp_texref());
++ CU_RET_ERR(stat, "cudaUnbindTexture on nbfp_texref failed");
++ }
+ cu_free_buffered(nbparam->nbfp);
+
+ stat = cudaFree(atdat->shift_vec);
+ CU_RET_ERR(stat, "cudaFree failed on atdat->shift_vec");
+ stat = cudaFree(atdat->fshift);
+ CU_RET_ERR(stat, "cudaFree failed on atdat->fshift");
+
+ stat = cudaFree(atdat->e_lj);
+ CU_RET_ERR(stat, "cudaFree failed on atdat->e_lj");
+ stat = cudaFree(atdat->e_el);
+ CU_RET_ERR(stat, "cudaFree failed on atdat->e_el");
+
+ cu_free_buffered(atdat->f, &atdat->natoms, &atdat->nalloc);
+ cu_free_buffered(atdat->xq);
+ cu_free_buffered(atdat->atom_types, &atdat->ntypes);
+
+ cu_free_buffered(plist->sci, &plist->nsci, &plist->sci_nalloc);
+ cu_free_buffered(plist->cj4, &plist->ncj4, &plist->cj4_nalloc);
+ cu_free_buffered(plist->excl, &plist->nexcl, &plist->excl_nalloc);
+ if (cu_nb->bUseTwoStreams)
+ {
+ cu_free_buffered(plist_nl->sci, &plist_nl->nsci, &plist_nl->sci_nalloc);
+ cu_free_buffered(plist_nl->cj4, &plist_nl->ncj4, &plist_nl->cj4_nalloc);
+ cu_free_buffered(plist_nl->excl, &plist_nl->nexcl, &plist->excl_nalloc);
+ }
+
+ sfree(atdat);
+ sfree(nbparam);
+ sfree(plist);
+ if (cu_nb->bUseTwoStreams)
+ {
+ sfree(plist_nl);
+ }
+ sfree(timers);
+ sfree(cu_nb->timings);
+ sfree(cu_nb);
+
+ if (debug)
+ {
+ fprintf(debug, "Cleaned up CUDA data structures.\n");
+ }
+}
+
+void cu_synchstream_atdat(nbnxn_cuda_ptr_t cu_nb, int iloc)
+{
+ cudaError_t stat;
+ cudaStream_t stream = cu_nb->stream[iloc];
+
+ stat = cudaStreamWaitEvent(stream, cu_nb->timers->stop_atdat, 0);
+ CU_RET_ERR(stat, "cudaStreamWaitEvent failed");
+}
+
+wallclock_gpu_t * nbnxn_cuda_get_timings(nbnxn_cuda_ptr_t cu_nb)
+{
+ return (cu_nb != NULL && cu_nb->bDoTime) ? cu_nb->timings : NULL;
+}
+
+void nbnxn_cuda_reset_timings(nbnxn_cuda_ptr_t cu_nb)
+{
+ if (cu_nb->bDoTime)
+ {
+ init_timings(cu_nb->timings);
+ }
+}
+
+int nbnxn_cuda_min_ci_balanced(nbnxn_cuda_ptr_t cu_nb)
+{
+ return cu_nb != NULL ?
+ gpu_min_ci_balanced_factor*cu_nb->dev_info->prop.multiProcessorCount : 0;
+
+}
+
+gmx_bool nbnxn_cuda_is_kernel_ewald_analytical(const nbnxn_cuda_ptr_t cu_nb)
+{
+ return ((cu_nb->nbparam->eeltype == eelCuEWALD_ANA) ||
+ (cu_nb->nbparam->eeltype == eelCuEWALD_ANA_TWIN));
+}
--- /dev/null
- c6 = tex1Dfetch(tex_nbfp, 2 * (ntypes * typei + typej));
- c12 = tex1Dfetch(tex_nbfp, 2 * (ntypes * typei + typej) + 1);
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2012, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * Gallium Rubidium Oxygen Manganese Argon Carbon Silicon
+ */
+
+#include "maths.h"
+/* Note that floating-point constants in CUDA code should be suffixed
+ * with f (e.g. 0.5f), to stop the compiler producing intermediate
+ * code that is in double precision.
+ */
+
+#if __CUDA_ARCH__ >= 300
+#define REDUCE_SHUFFLE
+/* On Kepler pre-loading i-atom types to shmem gives a few %,
+ but on Fermi it does not */
+#define IATYPE_SHMEM
+#endif
+
+#if defined EL_EWALD_ANA || defined EL_EWALD_TAB
+/* Note: convenience macro, needs to be undef-ed at the end of the file. */
+#define EL_EWALD_ANY
+#endif
+
+/*
+ Kernel launch parameters:
+ - #blocks = #pair lists, blockId = pair list Id
+ - #threads = CL_SIZE^2
+ - shmem = CL_SIZE^2 * sizeof(float)
+
+ Each thread calculates an i force-component taking one pair of i-j atoms.
+ */
++#if __CUDA_ARCH__ >= 350
++__launch_bounds__(64,16)
++#endif
+#ifdef PRUNE_NBL
+#ifdef CALC_ENERGIES
+__global__ void NB_KERNEL_FUNC_NAME(k_nbnxn, _ener_prune)
+#else
+__global__ void NB_KERNEL_FUNC_NAME(k_nbnxn, _prune)
+#endif
+#else
+#ifdef CALC_ENERGIES
+__global__ void NB_KERNEL_FUNC_NAME(k_nbnxn, _ener)
+#else
+__global__ void NB_KERNEL_FUNC_NAME(k_nbnxn)
+#endif
+#endif
+ (const cu_atomdata_t atdat,
+ const cu_nbparam_t nbparam,
+ const cu_plist_t plist,
+ bool bCalcFshift)
+{
+ /* convenience variables */
+ const nbnxn_sci_t *pl_sci = plist.sci;
+#ifndef PRUNE_NBL
+ const
+#endif
+ nbnxn_cj4_t *pl_cj4 = plist.cj4;
+ const nbnxn_excl_t *excl = plist.excl;
+ const int *atom_types = atdat.atom_types;
+ int ntypes = atdat.ntypes;
+ const float4 *xq = atdat.xq;
+ float3 *f = atdat.f;
+ const float3 *shift_vec = atdat.shift_vec;
+ float rcoulomb_sq = nbparam.rcoulomb_sq;
+#ifdef VDW_CUTOFF_CHECK
+ float rvdw_sq = nbparam.rvdw_sq;
+ float vdw_in_range;
+#endif
+#ifdef EL_RF
+ float two_k_rf = nbparam.two_k_rf;
+#endif
+#ifdef EL_EWALD_TAB
+ float coulomb_tab_scale = nbparam.coulomb_tab_scale;
+#endif
+#ifdef EL_EWALD_ANA
+ float beta2 = nbparam.ewald_beta*nbparam.ewald_beta;
+ float beta3 = nbparam.ewald_beta*nbparam.ewald_beta*nbparam.ewald_beta;
+#endif
+#ifdef PRUNE_NBL
+ float rlist_sq = nbparam.rlist_sq;
+#endif
+
+#ifdef CALC_ENERGIES
+ float lj_shift = nbparam.sh_invrc6;
+#ifdef EL_EWALD_ANY
+ float beta = nbparam.ewald_beta;
+ float ewald_shift = nbparam.sh_ewald;
+#else
+ float c_rf = nbparam.c_rf;
+#endif
+ float *e_lj = atdat.e_lj;
+ float *e_el = atdat.e_el;
+#endif
+
+ /* thread/block/warp id-s */
+ unsigned int tidxi = threadIdx.x;
+ unsigned int tidxj = threadIdx.y;
+ unsigned int tidx = threadIdx.y * blockDim.x + threadIdx.x;
+ unsigned int bidx = blockIdx.x;
+ unsigned int widx = tidx / WARP_SIZE; /* warp index */
+
+ int sci, ci, cj, ci_offset,
+ ai, aj,
+ cij4_start, cij4_end,
+ typei, typej,
+ i, jm, j4, wexcl_idx;
+ float qi, qj_f,
+ r2, inv_r, inv_r2, inv_r6,
+ c6, c12,
+ int_bit,
+#ifdef CALC_ENERGIES
+ E_lj, E_el, E_lj_p,
+#endif
+ F_invr;
+ unsigned int wexcl, imask, mask_ji;
+ float4 xqbuf;
+ float3 xi, xj, rv, f_ij, fcj_buf, fshift_buf;
+ float3 fci_buf[NCL_PER_SUPERCL]; /* i force buffer */
+ nbnxn_sci_t nb_sci;
+
+ /* shmem buffer for i x+q pre-loading */
+ extern __shared__ float4 xqib[];
+ /* shmem buffer for cj, for both warps separately */
+ int *cjs = (int *)(xqib + NCL_PER_SUPERCL * CL_SIZE);
+#ifdef IATYPE_SHMEM
+ /* shmem buffer for i atom-type pre-loading */
+ int *atib = (int *)(cjs + 2 * NBNXN_GPU_JGROUP_SIZE);
+#endif
+
+#ifndef REDUCE_SHUFFLE
+ /* shmem j force buffer */
+#ifdef IATYPE_SHMEM
+ float *f_buf = (float *)(atib + NCL_PER_SUPERCL * CL_SIZE);
+#else
+ float *f_buf = (float *)(cjs + 2 * NBNXN_GPU_JGROUP_SIZE);
+#endif
+#endif
+
+ nb_sci = pl_sci[bidx]; /* my i super-cluster's index = current bidx */
+ sci = nb_sci.sci; /* super-cluster */
+ cij4_start = nb_sci.cj4_ind_start; /* first ...*/
+ cij4_end = nb_sci.cj4_ind_end; /* and last index of j clusters */
+
+ /* Store the i-atom x and q in shared memory */
+ /* Note: the thread indexing here is inverted with respect to the
+ inner-loop as this results in slightly higher performance */
+ ci = sci * NCL_PER_SUPERCL + tidxi;
+ ai = ci * CL_SIZE + tidxj;
+ xqib[tidxi * CL_SIZE + tidxj] = xq[ai] + shift_vec[nb_sci.shift];
+#ifdef IATYPE_SHMEM
+ ci = sci * NCL_PER_SUPERCL + tidxj;
+ ai = ci * CL_SIZE + tidxi;
+ atib[tidxj * CL_SIZE + tidxi] = atom_types[ai];
+#endif
+ __syncthreads();
+
+ for(ci_offset = 0; ci_offset < NCL_PER_SUPERCL; ci_offset++)
+ {
+ fci_buf[ci_offset] = make_float3(0.0f);
+ }
+
+#ifdef CALC_ENERGIES
+ E_lj = 0.0f;
+ E_el = 0.0f;
+
+#if defined EL_EWALD_ANY || defined EL_RF
+ if (nb_sci.shift == CENTRAL && pl_cj4[cij4_start].cj[0] == sci*NCL_PER_SUPERCL)
+ {
+ /* we have the diagonal: add the charge self interaction energy term */
+ for (i = 0; i < NCL_PER_SUPERCL; i++)
+ {
+ qi = xqib[i * CL_SIZE + tidxi].w;
+ E_el += qi*qi;
+ }
+ /* divide the self term equally over the j-threads */
+ E_el /= CL_SIZE;
+#ifdef EL_RF
+ E_el *= -nbparam.epsfac*0.5f*c_rf;
+#else
+ E_el *= -nbparam.epsfac*beta*M_FLOAT_1_SQRTPI; /* last factor 1/sqrt(pi) */
+#endif
+ }
+#endif
+#endif
+
+ /* skip central shifts when summing shift forces */
+ if (nb_sci.shift == CENTRAL)
+ {
+ bCalcFshift = false;
+ }
+
+ fshift_buf = make_float3(0.0f);
+
+ /* loop over the j clusters = seen by any of the atoms in the current super-cluster */
+ for (j4 = cij4_start; j4 < cij4_end; j4++)
+ {
+ wexcl_idx = pl_cj4[j4].imei[widx].excl_ind;
+ imask = pl_cj4[j4].imei[widx].imask;
+ wexcl = excl[wexcl_idx].pair[(tidx) & (WARP_SIZE - 1)];
+
+#ifndef PRUNE_NBL
+ if (imask)
+#endif
+ {
+ /* Pre-load cj into shared memory on both warps separately */
+ if ((tidxj == 0 || tidxj == 4) && tidxi < NBNXN_GPU_JGROUP_SIZE)
+ {
+ cjs[tidxi + tidxj * NBNXN_GPU_JGROUP_SIZE / 4] = pl_cj4[j4].cj[tidxi];
+ }
+
+ /* Unrolling this loop
+ - with pruning leads to register spilling;
+ - on Kepler is much slower;
+ - doesn't work on CUDA <v4.1
+ Tested with nvcc 3.2 - 5.0.7 */
+#if !defined PRUNE_NBL && __CUDA_ARCH__ < 300 && CUDA_VERSION >= 4010
+#pragma unroll 4
+#endif
+ for (jm = 0; jm < NBNXN_GPU_JGROUP_SIZE; jm++)
+ {
+ if (imask & (supercl_interaction_mask << (jm * NCL_PER_SUPERCL)))
+ {
+ mask_ji = (1U << (jm * NCL_PER_SUPERCL));
+
+ cj = cjs[jm + (tidxj & 4) * NBNXN_GPU_JGROUP_SIZE / 4];
+ aj = cj * CL_SIZE + tidxj;
+
+ /* load j atom data */
+ xqbuf = xq[aj];
+ xj = make_float3(xqbuf.x, xqbuf.y, xqbuf.z);
+ qj_f = nbparam.epsfac * xqbuf.w;
+ typej = atom_types[aj];
+
+ fcj_buf = make_float3(0.0f);
+
+ /* The PME and RF kernels don't unroll with CUDA <v4.1. */
+#if !defined PRUNE_NBL && !(CUDA_VERSION < 4010 && (defined EL_EWALD_ANY || defined EL_RF))
+#pragma unroll 8
+#endif
+ for(i = 0; i < NCL_PER_SUPERCL; i++)
+ {
+ if (imask & mask_ji)
+ {
+ ci_offset = i; /* i force buffer offset */
+
+ ci = sci * NCL_PER_SUPERCL + i; /* i cluster index */
+ ai = ci * CL_SIZE + tidxi; /* i atom index */
+
+ /* all threads load an atom from i cluster ci into shmem! */
+ xqbuf = xqib[i * CL_SIZE + tidxi];
+ xi = make_float3(xqbuf.x, xqbuf.y, xqbuf.z);
+
+ /* distance between i and j atoms */
+ rv = xi - xj;
+ r2 = norm2(rv);
+
+#ifdef PRUNE_NBL
+ /* If _none_ of the atoms pairs are in cutoff range,
+ the bit corresponding to the current
+ cluster-pair in imask gets set to 0. */
+ if (!__any(r2 < rlist_sq))
+ {
+ imask &= ~mask_ji;
+ }
+#endif
+
+ int_bit = (wexcl & mask_ji) ? 1.0f : 0.0f;
+
+ /* cutoff & exclusion check */
+#if defined EL_EWALD_ANY || defined EL_RF
+ if (r2 < rcoulomb_sq *
+ (nb_sci.shift != CENTRAL || ci != cj || tidxj > tidxi))
+#else
+ if (r2 < rcoulomb_sq * int_bit)
+#endif
+ {
+ /* load the rest of the i-atom parameters */
+ qi = xqbuf.w;
+#ifdef IATYPE_SHMEM
+ typei = atib[i * CL_SIZE + tidxi];
+#else
+ typei = atom_types[ai];
+#endif
+
+ /* LJ 6*C6 and 12*C12 */
- F_invr += qi * qj_f * (int_bit*inv_r2 - interpolate_coulomb_force_r(r2 * inv_r, coulomb_tab_scale)) * inv_r;
++#ifdef USE_TEXOBJ
++ c6 = tex1Dfetch<float>(nbparam.nbfp_texobj, 2 * (ntypes * typei + typej));
++ c12 = tex1Dfetch<float>(nbparam.nbfp_texobj, 2 * (ntypes * typei + typej) + 1);
++#else
++ c6 = tex1Dfetch(nbfp_texref, 2 * (ntypes * typei + typej));
++ c12 = tex1Dfetch(nbfp_texref, 2 * (ntypes * typei + typej) + 1);
++#endif /* USE_TEXOBJ */
++
+
+ /* avoid NaN for excluded pairs at r=0 */
+ r2 += (1.0f - int_bit) * NBNXN_AVOID_SING_R2_INC;
+
+ inv_r = rsqrt(r2);
+ inv_r2 = inv_r * inv_r;
+ inv_r6 = inv_r2 * inv_r2 * inv_r2;
+#if defined EL_EWALD_ANY || defined EL_RF
+ /* We could mask inv_r2, but with Ewald
+ * masking both inv_r6 and F_invr is faster */
+ inv_r6 *= int_bit;
+#endif
+
+ F_invr = inv_r6 * (c12 * inv_r6 - c6) * inv_r2;
+
+#ifdef CALC_ENERGIES
+ E_lj_p = int_bit * (c12 * (inv_r6 * inv_r6 - lj_shift * lj_shift) * 0.08333333f - c6 * (inv_r6 - lj_shift) * 0.16666667f);
+#endif
+
+#ifdef VDW_CUTOFF_CHECK
+ /* this enables twin-range cut-offs (rvdw < rcoulomb <= rlist) */
+ vdw_in_range = (r2 < rvdw_sq) ? 1.0f : 0.0f;
+ F_invr *= vdw_in_range;
+#ifdef CALC_ENERGIES
+ E_lj_p *= vdw_in_range;
+#endif
+#endif
+#ifdef CALC_ENERGIES
+ E_lj += E_lj_p;
+#endif
+
+
+#ifdef EL_CUTOFF
+ F_invr += qi * qj_f * inv_r2 * inv_r;
+#endif
+#ifdef EL_RF
+ F_invr += qi * qj_f * (int_bit*inv_r2 * inv_r - two_k_rf);
+#endif
+#if defined EL_EWALD_ANA
+ F_invr += qi * qj_f * (int_bit*inv_r2*inv_r + pmecorrF(beta2*r2)*beta3);
+#elif defined EL_EWALD_TAB
++ F_invr += qi * qj_f * (int_bit*inv_r2 -
++#ifdef USE_TEXOBJ
++ interpolate_coulomb_force_r(nbparam.coulomb_tab_texobj, r2 * inv_r, coulomb_tab_scale)
++#else
++ interpolate_coulomb_force_r(r2 * inv_r, coulomb_tab_scale)
++#endif /* USE_TEXOBJ */
++ ) * inv_r;
+#endif /* EL_EWALD_ANA/TAB */
+
+#ifdef CALC_ENERGIES
+#ifdef EL_CUTOFF
+ E_el += qi * qj_f * (inv_r - c_rf);
+#endif
+#ifdef EL_RF
+ E_el += qi * qj_f * (int_bit*inv_r + 0.5f * two_k_rf * r2 - c_rf);
+#endif
+#ifdef EL_EWALD_ANY
+ /* 1.0f - erff is faster than erfcf */
+ E_el += qi * qj_f * (inv_r * (int_bit - erff(r2 * inv_r * beta)) - int_bit * ewald_shift);
+#endif /* EL_EWALD_ANY */
+#endif
+ f_ij = rv * F_invr;
+
+ /* accumulate j forces in registers */
+ fcj_buf -= f_ij;
+
+ /* accumulate i forces in registers */
+ fci_buf[ci_offset] += f_ij;
+ }
+ }
+
+ /* shift the mask bit by 1 */
+ mask_ji += mask_ji;
+ }
+
+ /* reduce j forces */
+#ifdef REDUCE_SHUFFLE
+ reduce_force_j_warp_shfl(fcj_buf, f, tidxi, aj);
+#else
+ /* store j forces in shmem */
+ f_buf[ tidx] = fcj_buf.x;
+ f_buf[ FBUF_STRIDE + tidx] = fcj_buf.y;
+ f_buf[2 * FBUF_STRIDE + tidx] = fcj_buf.z;
+
+ reduce_force_j_generic(f_buf, f, tidxi, tidxj, aj);
+#endif
+ }
+ }
+#ifdef PRUNE_NBL
+ /* Update the imask with the new one which does not contain the
+ out of range clusters anymore. */
+ pl_cj4[j4].imei[widx].imask = imask;
+#endif
+ }
+ }
+
+ /* reduce i forces */
+ for(ci_offset = 0; ci_offset < NCL_PER_SUPERCL; ci_offset++)
+ {
+ ai = (sci * NCL_PER_SUPERCL + ci_offset) * CL_SIZE + tidxi;
+#ifdef REDUCE_SHUFFLE
+ reduce_force_i_warp_shfl(fci_buf[ci_offset], f,
+ &fshift_buf, bCalcFshift,
+ tidxj, ai);
+#else
+ f_buf[ tidx] = fci_buf[ci_offset].x;
+ f_buf[ FBUF_STRIDE + tidx] = fci_buf[ci_offset].y;
+ f_buf[2 * FBUF_STRIDE + tidx] = fci_buf[ci_offset].z;
+ __syncthreads();
+ reduce_force_i(f_buf, f,
+ &fshift_buf, bCalcFshift,
+ tidxi, tidxj, ai);
+ __syncthreads();
+#endif
+ }
+
+ /* add up local shift forces into global mem */
+#ifdef REDUCE_SHUFFLE
+ if (bCalcFshift && (tidxj == 0 || tidxj == 4))
+#else
+ if (bCalcFshift && tidxj == 0)
+#endif
+ {
+ atomicAdd(&atdat.fshift[nb_sci.shift].x, fshift_buf.x);
+ atomicAdd(&atdat.fshift[nb_sci.shift].y, fshift_buf.y);
+ atomicAdd(&atdat.fshift[nb_sci.shift].z, fshift_buf.z);
+ }
+
+#ifdef CALC_ENERGIES
+#ifdef REDUCE_SHUFFLE
+ /* reduce the energies over warps and store into global memory */
+ reduce_energy_warp_shfl(E_lj, E_el, e_lj, e_el, tidx);
+#else
+ /* flush the energies to shmem and reduce them */
+ f_buf[ tidx] = E_lj;
+ f_buf[FBUF_STRIDE + tidx] = E_el;
+ reduce_energy_pow2(f_buf + (tidx & WARP_SIZE), e_lj, e_el, tidx & ~WARP_SIZE);
+#endif
+#endif
+}
+
+#undef EL_EWALD_ANY
--- /dev/null
- c6 = tex1Dfetch(tex_nbfp, 2 * (ntypes * typei + typej));
- c12 = tex1Dfetch(tex_nbfp, 2 * (ntypes * typei + typej) + 1);
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2012, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * Gallium Rubidium Oxygen Manganese Argon Carbon Silicon
+ */
+
+#include "maths.h"
+/* Note that floating-point constants in CUDA code should be suffixed
+ * with f (e.g. 0.5f), to stop the compiler producing intermediate
+ * code that is in double precision.
+ */
+
+/* Analytical Ewald is not implemented for the legacy kernels (as it is anyway
+ slower than the tabulated kernel on Fermi). */
+#ifdef EL_EWALD_ANA
+#error Trying to generate Analytical Ewald legacy kernels which is not implemented in the legacy kernels!
+#endif
+
+/*
+ Kernel launch parameters:
+ - #blocks = #pair lists, blockId = pair list Id
+ - #threads = CL_SIZE^2
+ - shmem = CL_SIZE^2 * sizeof(float)
+
+ Each thread calculates an i force-component taking one pair of i-j atoms.
+ */
++#if __CUDA_ARCH__ >= 350
++__launch_bounds__(64,16)
++#endif
+#ifdef PRUNE_NBL
+#ifdef CALC_ENERGIES
+__global__ void NB_KERNEL_FUNC_NAME(k_nbnxn, _ener_prune_legacy)
+#else
+__global__ void NB_KERNEL_FUNC_NAME(k_nbnxn, _prune_legacy)
+#endif
+#else
+#ifdef CALC_ENERGIES
+__global__ void NB_KERNEL_FUNC_NAME(k_nbnxn, _ener_legacy)
+#else
+__global__ void NB_KERNEL_FUNC_NAME(k_nbnxn, _legacy)
+#endif
+#endif
+ (const cu_atomdata_t atdat,
+ const cu_nbparam_t nbparam,
+ const cu_plist_t plist,
+ bool bCalcFshift)
+{
+ /* convenience variables */
+ const nbnxn_sci_t *pl_sci = plist.sci;
+#ifndef PRUNE_NBL
+ const
+#endif
+ nbnxn_cj4_t *pl_cj4 = plist.cj4;
+ const nbnxn_excl_t *excl = plist.excl;
+ const int *atom_types = atdat.atom_types;
+ int ntypes = atdat.ntypes;
+ const float4 *xq = atdat.xq;
+ float3 *f = atdat.f;
+ const float3 *shift_vec = atdat.shift_vec;
+ float rcoulomb_sq = nbparam.rcoulomb_sq;
+#ifdef VDW_CUTOFF_CHECK
+ float rvdw_sq = nbparam.rvdw_sq;
+ float vdw_in_range;
+#endif
+#ifdef EL_RF
+ float two_k_rf = nbparam.two_k_rf;
+#endif
+#ifdef EL_EWALD_TAB
+ float coulomb_tab_scale = nbparam.coulomb_tab_scale;
+#endif
+#ifdef PRUNE_NBL
+ float rlist_sq = nbparam.rlist_sq;
+#endif
+
+#ifdef CALC_ENERGIES
+ float lj_shift = nbparam.sh_invrc6;
+#ifdef EL_EWALD_TAB
+ float beta = nbparam.ewald_beta;
+ float ewald_shift = nbparam.sh_ewald;
+#else
+ float c_rf = nbparam.c_rf;
+#endif
+ float *e_lj = atdat.e_lj;
+ float *e_el = atdat.e_el;
+#endif
+
+ /* thread/block/warp id-s */
+ unsigned int tidxi = threadIdx.x;
+ unsigned int tidxj = threadIdx.y;
+ unsigned int tidx = threadIdx.y * blockDim.x + threadIdx.x;
+ unsigned int bidx = blockIdx.x;
+ unsigned int widx = tidx / WARP_SIZE; /* warp index */
+
+ int sci, ci, cj, ci_offset,
+ ai, aj,
+ cij4_start, cij4_end,
+ typei, typej,
+ i, cii, jm, j4, nsubi, wexcl_idx;
+ float qi, qj_f,
+ r2, inv_r, inv_r2, inv_r6,
+ c6, c12,
+ int_bit,
+#ifdef CALC_ENERGIES
+ E_lj, E_el, E_lj_p,
+#endif
+ F_invr;
+ unsigned int wexcl, imask, mask_ji;
+ float4 xqbuf;
+ float3 xi, xj, rv, f_ij, fcj_buf, fshift_buf;
+ float3 fci_buf[NCL_PER_SUPERCL]; /* i force buffer */
+ nbnxn_sci_t nb_sci;
+
+ /* shmem buffer for i x+q pre-loading */
+ extern __shared__ float4 xqib[];
+ /* shmem j force buffer */
+ float *f_buf = (float *)(xqib + NCL_PER_SUPERCL * CL_SIZE);
+
+ nb_sci = pl_sci[bidx]; /* my i super-cluster's index = current bidx */
+ sci = nb_sci.sci; /* super-cluster */
+ cij4_start = nb_sci.cj4_ind_start; /* first ...*/
+ cij4_end = nb_sci.cj4_ind_end; /* and last index of j clusters */
+
+ /* Store the i-atom x and q in shared memory */
+ /* Note: the thread indexing here is inverted with respect to the
+ inner-loop as this results in slightly higher performance */
+ ci = sci * NCL_PER_SUPERCL + tidxi;
+ ai = ci * CL_SIZE + tidxj;
+ xqib[tidxi * CL_SIZE + tidxj] = xq[ai] + shift_vec[nb_sci.shift];
+ __syncthreads();
+
+ for(ci_offset = 0; ci_offset < NCL_PER_SUPERCL; ci_offset++)
+ {
+ fci_buf[ci_offset] = make_float3(0.0f);
+ }
+
+#ifdef CALC_ENERGIES
+ E_lj = 0.0f;
+ E_el = 0.0f;
+
+#if defined EL_EWALD_TAB || defined EL_RF
+ if (nb_sci.shift == CENTRAL && pl_cj4[cij4_start].cj[0] == sci*NCL_PER_SUPERCL)
+ {
+ /* we have the diagonal: add the charge self interaction energy term */
+ for (i = 0; i < NCL_PER_SUPERCL; i++)
+ {
+ qi = xqib[i * CL_SIZE + tidxi].w;
+ E_el += qi*qi;
+ }
+ /* divide the self term equally over the j-threads */
+ E_el /= CL_SIZE;
+#ifdef EL_RF
+ E_el *= -nbparam.epsfac*0.5f*c_rf;
+#else
+ E_el *= -nbparam.epsfac*beta*M_FLOAT_1_SQRTPI; /* last factor 1/sqrt(pi) */
+#endif
+ }
+#endif
+#endif
+
+ /* skip central shifts when summing shift forces */
+ if (nb_sci.shift == CENTRAL)
+ {
+ bCalcFshift = false;
+ }
+
+ fshift_buf = make_float3(0.0f);
+
+ /* loop over the j clusters = seen by any of the atoms in the current super-cluster */
+ for (j4 = cij4_start; j4 < cij4_end; j4++)
+ {
+ wexcl_idx = pl_cj4[j4].imei[widx].excl_ind;
+ imask = pl_cj4[j4].imei[widx].imask;
+ wexcl = excl[wexcl_idx].pair[(tidx) & (WARP_SIZE - 1)];
+
+#ifndef PRUNE_NBL
+ if (imask)
+#endif
+ {
+ /* nvcc >v4.1 doesn't like this loop, it refuses to unroll it */
+#if CUDA_VERSION >= 4010
+ #pragma unroll 4
+#endif
+ for (jm = 0; jm < NBNXN_GPU_JGROUP_SIZE; jm++)
+ {
+ mask_ji = (imask >> (jm * CL_SIZE)) & supercl_interaction_mask;
+ if (mask_ji)
+ {
+ nsubi = __popc(mask_ji);
+
+ cj = pl_cj4[j4].cj[jm];
+ aj = cj * CL_SIZE + tidxj;
+
+ /* load j atom data */
+ xqbuf = xq[aj];
+ xj = make_float3(xqbuf.x, xqbuf.y, xqbuf.z);
+ qj_f = nbparam.epsfac * xqbuf.w;
+ typej = atom_types[aj];
+
+ fcj_buf = make_float3(0.0f);
+
+ /* loop over the i-clusters in sci */
+ /* #pragma unroll 8
+ -- nvcc doesn't like my code, it refuses to unroll it
+ which is a pity because here unrolling could help. */
+ for (cii = 0; cii < nsubi; cii++)
+ {
+ i = __ffs(mask_ji) - 1;
+ mask_ji &= ~(1U << i);
+
+ ci_offset = i; /* i force buffer offset */
+
+ ci = sci * NCL_PER_SUPERCL + i; /* i cluster index */
+ ai = ci * CL_SIZE + tidxi; /* i atom index */
+
+ /* all threads load an atom from i cluster ci into shmem! */
+ xqbuf = xqib[i * CL_SIZE + tidxi];
+ xi = make_float3(xqbuf.x, xqbuf.y, xqbuf.z);
+
+ /* distance between i and j atoms */
+ rv = xi - xj;
+ r2 = norm2(rv);
+
+#ifdef PRUNE_NBL
+ /* If _none_ of the atoms pairs are in cutoff range,
+ the bit corresponding to the current
+ cluster-pair in imask gets set to 0. */
+ if (!__any(r2 < rlist_sq))
+ {
+ imask &= ~(1U << (jm * NCL_PER_SUPERCL + i));
+ }
+#endif
+
+ int_bit = ((wexcl >> (jm * NCL_PER_SUPERCL + i)) & 1) ? 1.0f : 0.0f;
+
+ /* cutoff & exclusion check */
+#if defined EL_EWALD_TAB || defined EL_RF
+ if (r2 < rcoulomb_sq *
+ (nb_sci.shift != CENTRAL || ci != cj || tidxj > tidxi))
+#else
+ if (r2 < rcoulomb_sq * int_bit)
+#endif
+ {
+ /* load the rest of the i-atom parameters */
+ qi = xqbuf.w;
+ typei = atom_types[ai];
+
+ /* LJ 6*C6 and 12*C12 */
++ c6 = tex1Dfetch(nbfp_texref, 2 * (ntypes * typei + typej));
++ c12 = tex1Dfetch(nbfp_texref, 2 * (ntypes * typei + typej) + 1);
+
+ /* avoid NaN for excluded pairs at r=0 */
+ r2 += (1.0f - int_bit) * NBNXN_AVOID_SING_R2_INC;
+
+ inv_r = rsqrt(r2);
+ inv_r2 = inv_r * inv_r;
+ inv_r6 = inv_r2 * inv_r2 * inv_r2;
+#if defined EL_EWALD_TAB || defined EL_RF
+ /* We could mask inv_r2, but with Ewald
+ * masking both inv_r6 and F_invr is faster */
+ inv_r6 *= int_bit;
+#endif
+
+ F_invr = inv_r6 * (c12 * inv_r6 - c6) * inv_r2;
+
+#ifdef CALC_ENERGIES
+ E_lj_p = int_bit * (c12 * (inv_r6 * inv_r6 - lj_shift * lj_shift) * 0.08333333f - c6 * (inv_r6 - lj_shift) * 0.16666667f);
+#endif
+
+#ifdef VDW_CUTOFF_CHECK
+ /* this enables twin-range cut-offs (rvdw < rcoulomb <= rlist) */
+ vdw_in_range = (r2 < rvdw_sq) ? 1.0f : 0.0f;
+ F_invr *= vdw_in_range;
+#ifdef CALC_ENERGIES
+ E_lj_p *= vdw_in_range;
+#endif
+#endif
+#ifdef CALC_ENERGIES
+ E_lj += E_lj_p;
+#endif
+
+
+#ifdef EL_CUTOFF
+ F_invr += qi * qj_f * inv_r2 * inv_r;
+#endif
+#ifdef EL_RF
+ F_invr += qi * qj_f * (int_bit*inv_r2 * inv_r - two_k_rf);
+#endif
+#ifdef EL_EWALD_TAB
+ F_invr += qi * qj_f * (int_bit*inv_r2 - interpolate_coulomb_force_r(r2 * inv_r, coulomb_tab_scale)) * inv_r;
+#endif /* EL_EWALD_TAB */
+
+#ifdef CALC_ENERGIES
+#ifdef EL_CUTOFF
+ E_el += qi * qj_f * (inv_r - c_rf);
+#endif
+#ifdef EL_RF
+ E_el += qi * qj_f * (int_bit*inv_r + 0.5f * two_k_rf * r2 - c_rf);
+#endif
+#ifdef EL_EWALD_TAB
+ /* 1.0f - erff is faster than erfcf */
+ E_el += qi * qj_f * (inv_r * (int_bit - erff(r2 * inv_r * beta)) - int_bit * ewald_shift);
+#endif
+#endif
+ f_ij = rv * F_invr;
+
+ /* accumulate j forces in registers */
+ fcj_buf -= f_ij;
+
+ /* accumulate i forces in registers */
+ fci_buf[ci_offset] += f_ij;
+ }
+ }
+
+ /* store j forces in shmem */
+ f_buf[ tidx] = fcj_buf.x;
+ f_buf[ FBUF_STRIDE + tidx] = fcj_buf.y;
+ f_buf[2 * FBUF_STRIDE + tidx] = fcj_buf.z;
+
+ /* reduce j forces */
+ reduce_force_j_generic(f_buf, f, tidxi, tidxj, aj);
+ }
+ }
+#ifdef PRUNE_NBL
+ /* Update the imask with the new one which does not contain the
+ out of range clusters anymore. */
+ pl_cj4[j4].imei[widx].imask = imask;
+#endif
+ }
+ }
+
+ /* reduce i forces */
+ for(ci_offset = 0; ci_offset < NCL_PER_SUPERCL; ci_offset++)
+ {
+ ai = (sci * NCL_PER_SUPERCL + ci_offset) * CL_SIZE + tidxi;
+ f_buf[ tidx] = fci_buf[ci_offset].x;
+ f_buf[ FBUF_STRIDE + tidx] = fci_buf[ci_offset].y;
+ f_buf[2 * FBUF_STRIDE + tidx] = fci_buf[ci_offset].z;
+ __syncthreads();
+ reduce_force_i(f_buf, f,
+ &fshift_buf, bCalcFshift,
+ tidxi, tidxj, ai);
+ __syncthreads();
+ }
+
+ /* add up local shift forces into global mem */
+ if (bCalcFshift && tidxj == 0)
+ {
+ atomicAdd(&atdat.fshift[nb_sci.shift].x, fshift_buf.x);
+ atomicAdd(&atdat.fshift[nb_sci.shift].y, fshift_buf.y);
+ atomicAdd(&atdat.fshift[nb_sci.shift].z, fshift_buf.z);
+ }
+
+#ifdef CALC_ENERGIES
+ /* flush the energies to shmem and reduce them */
+ f_buf[ tidx] = E_lj;
+ f_buf[FBUF_STRIDE + tidx] = E_el;
+ reduce_energy_pow2(f_buf + (tidx & WARP_SIZE), e_lj, e_el, tidx & ~WARP_SIZE);
+#endif
+}
--- /dev/null
- return fract1 * tex1Dfetch(tex_coulomb_tab, index)
- + fract2 * tex1Dfetch(tex_coulomb_tab, index + 1);
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2012, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * Gallium Rubidium Oxygen Manganese Argon Carbon Silicon
+ */
+
+/* Note that floating-point constants in CUDA code should be suffixed
+ * with f (e.g. 0.5f), to stop the compiler producing intermediate
+ * code that is in double precision.
+ */
+
+#include "../../gmxlib/cuda_tools/vectype_ops.cuh"
+
+#ifndef NBNXN_CUDA_KERNEL_UTILS_CUH
+#define NBNXN_CUDA_KERNEL_UTILS_CUH
+
+#define WARP_SIZE_POW2_EXPONENT (5)
+#define CL_SIZE_POW2_EXPONENT (3) /* change this together with GPU_NS_CLUSTER_SIZE !*/
+#define CL_SIZE_SQ (CL_SIZE * CL_SIZE)
+#define FBUF_STRIDE (CL_SIZE_SQ)
+
+/*! i-cluster interaction mask for a super-cluster with all NCL_PER_SUPERCL bits set */
+const unsigned supercl_interaction_mask = ((1U << NCL_PER_SUPERCL) - 1U);
+
+/*! Interpolate Ewald coulomb force using the table through the tex_nbfp texture.
+ * Original idea: from the OpenMM project
+ */
+static inline __device__
+float interpolate_coulomb_force_r(float r, float scale)
+{
+ float normalized = scale * r;
+ int index = (int) normalized;
+ float fract2 = normalized - index;
+ float fract1 = 1.0f - fract2;
+
++ return fract1 * tex1Dfetch(coulomb_tab_texref, index)
++ + fract2 * tex1Dfetch(coulomb_tab_texref, index + 1);
+}
+
++#ifdef TEXOBJ_SUPPORTED
++static inline __device__
++float interpolate_coulomb_force_r(cudaTextureObject_t texobj_coulomb_tab,
++ float r, float scale)
++{
++ float normalized = scale * r;
++ int index = (int) normalized;
++ float fract2 = normalized - index;
++ float fract1 = 1.0f - fract2;
++
++ return fract1 * tex1Dfetch<float>(texobj_coulomb_tab, index) +
++ fract2 * tex1Dfetch<float>(texobj_coulomb_tab, index + 1);
++}
++#endif
++
++
+/*! Calculate analytical Ewald correction term. */
+static inline __device__
+float pmecorrF(float z2)
+{
+ const float FN6 = -1.7357322914161492954e-8f;
+ const float FN5 = 1.4703624142580877519e-6f;
+ const float FN4 = -0.000053401640219807709149f;
+ const float FN3 = 0.0010054721316683106153f;
+ const float FN2 = -0.019278317264888380590f;
+ const float FN1 = 0.069670166153766424023f;
+ const float FN0 = -0.75225204789749321333f;
+
+ const float FD4 = 0.0011193462567257629232f;
+ const float FD3 = 0.014866955030185295499f;
+ const float FD2 = 0.11583842382862377919f;
+ const float FD1 = 0.50736591960530292870f;
+ const float FD0 = 1.0f;
+
+ float z4;
+ float polyFN0,polyFN1,polyFD0,polyFD1;
+
+ z4 = z2*z2;
+
+ polyFD0 = FD4*z4 + FD2;
+ polyFD1 = FD3*z4 + FD1;
+ polyFD0 = polyFD0*z4 + FD0;
+ polyFD0 = polyFD1*z2 + polyFD0;
+
+ polyFD0 = 1.0f/polyFD0;
+
+ polyFN0 = FN6*z4 + FN4;
+ polyFN1 = FN5*z4 + FN3;
+ polyFN0 = polyFN0*z4 + FN2;
+ polyFN1 = polyFN1*z4 + FN1;
+ polyFN0 = polyFN0*z4 + FN0;
+ polyFN0 = polyFN1*z2 + polyFN0;
+
+ return polyFN0*polyFD0;
+}
+
+/*! Final j-force reduction; this generic implementation works with
+ * arbitrary array sizes.
+ */
+static inline __device__
+void reduce_force_j_generic(float *f_buf, float3 *fout,
+ int tidxi, int tidxj, int aidx)
+{
+ if (tidxi == 0)
+ {
+ float3 f = make_float3(0.0f);
+ for (int j = tidxj * CL_SIZE; j < (tidxj + 1) * CL_SIZE; j++)
+ {
+ f.x += f_buf[ j];
+ f.y += f_buf[ FBUF_STRIDE + j];
+ f.z += f_buf[2 * FBUF_STRIDE + j];
+ }
+
+ atomicAdd(&fout[aidx], f);
+ }
+}
+
+/*! Final j-force reduction; this implementation only with power of two
+ * array sizes and with sm >= 3.0
+ */
+#if __CUDA_ARCH__ >= 300
+static inline __device__
+void reduce_force_j_warp_shfl(float3 f, float3 *fout,
+ int tidxi, int aidx)
+{
+ int i;
+
+#pragma unroll 3
+ for (i = 0; i < 3; i++)
+ {
+ f.x += __shfl_down(f.x, 1<<i);
+ f.y += __shfl_down(f.y, 1<<i);
+ f.z += __shfl_down(f.z, 1<<i);
+ }
+
+ /* Write the reduced j-force on one thread for each j */
+ if (tidxi == 0)
+ {
+ atomicAdd(&fout[aidx], f);
+ }
+}
+#endif
+
+/*! Final i-force reduction; this generic implementation works with
+ * arbitrary array sizes.
+ */
+static inline __device__
+void reduce_force_i_generic(float *f_buf, float3 *fout,
+ float3 *fshift_buf, bool bCalcFshift,
+ int tidxi, int tidxj, int aidx)
+{
+ if (tidxj == 0)
+ {
+ float3 f = make_float3(0.0f);
+ for (int j = tidxi; j < CL_SIZE_SQ; j += CL_SIZE)
+ {
+ f.x += f_buf[ j];
+ f.y += f_buf[ FBUF_STRIDE + j];
+ f.z += f_buf[2 * FBUF_STRIDE + j];
+ }
+
+ atomicAdd(&fout[aidx], f);
+
+ if (bCalcFshift)
+ {
+ *fshift_buf += f;
+ }
+ }
+}
+
+/*! Final i-force reduction; this implementation works only with power of two
+ * array sizes.
+ */
+static inline __device__
+void reduce_force_i_pow2(volatile float *f_buf, float3 *fout,
+ float3 *fshift_buf, bool bCalcFshift,
+ int tidxi, int tidxj, int aidx)
+{
+ int i, j;
+ float3 f = make_float3(0.0f);
+
+ /* Reduce the initial CL_SIZE values for each i atom to half
+ * every step by using CL_SIZE * i threads.
+ * Can't just use i as loop variable because than nvcc refuses to unroll.
+ */
+ i = CL_SIZE/2;
+ # pragma unroll 5
+ for (j = CL_SIZE_POW2_EXPONENT - 1; j > 0; j--)
+ {
+ if (tidxj < i)
+ {
+
+ f_buf[ tidxj * CL_SIZE + tidxi] += f_buf[ (tidxj + i) * CL_SIZE + tidxi];
+ f_buf[ FBUF_STRIDE + tidxj * CL_SIZE + tidxi] += f_buf[ FBUF_STRIDE + (tidxj + i) * CL_SIZE + tidxi];
+ f_buf[2 * FBUF_STRIDE + tidxj * CL_SIZE + tidxi] += f_buf[2 * FBUF_STRIDE + (tidxj + i) * CL_SIZE + tidxi];
+ }
+ i >>= 1;
+ }
+
+ /* i == 1, last reduction step, writing to global mem */
+ if (tidxj == 0)
+ {
+ f.x = f_buf[ tidxj * CL_SIZE + tidxi] + f_buf[ (tidxj + i) * CL_SIZE + tidxi];
+ f.y = f_buf[ FBUF_STRIDE + tidxj * CL_SIZE + tidxi] + f_buf[ FBUF_STRIDE + (tidxj + i) * CL_SIZE + tidxi];
+ f.z = f_buf[2 * FBUF_STRIDE + tidxj * CL_SIZE + tidxi] + f_buf[2 * FBUF_STRIDE + (tidxj + i) * CL_SIZE + tidxi];
+
+ atomicAdd(&fout[aidx], f);
+
+ if (bCalcFshift)
+ {
+ *fshift_buf += f;
+ }
+ }
+}
+
+/*! Final i-force reduction wrapper; calls the generic or pow2 reduction depending
+ * on whether the size of the array to be reduced is power of two or not.
+ */
+static inline __device__
+void reduce_force_i(float *f_buf, float3 *f,
+ float3 *fshift_buf, bool bCalcFshift,
+ int tidxi, int tidxj, int ai)
+{
+ if ((CL_SIZE & (CL_SIZE - 1)))
+ {
+ reduce_force_i_generic(f_buf, f, fshift_buf, bCalcFshift, tidxi, tidxj, ai);
+ }
+ else
+ {
+ reduce_force_i_pow2(f_buf, f, fshift_buf, bCalcFshift, tidxi, tidxj, ai);
+ }
+}
+
+/*! Final i-force reduction; this implementation works only with power of two
+ * array sizes and with sm >= 3.0
+ */
+#if __CUDA_ARCH__ >= 300
+static inline __device__
+void reduce_force_i_warp_shfl(float3 fin, float3 *fout,
+ float3 *fshift_buf, bool bCalcFshift,
+ int tidxj, int aidx)
+{
+ int j;
+
+#pragma unroll 2
+ for (j = 0; j < 2; j++)
+ {
+ fin.x += __shfl_down(fin.x, CL_SIZE<<j);
+ fin.y += __shfl_down(fin.y, CL_SIZE<<j);
+ fin.z += __shfl_down(fin.z, CL_SIZE<<j);
+ }
+
+ /* The first thread in the warp writes the reduced force */
+ if (tidxj == 0 || tidxj == 4)
+ {
+ atomicAdd(&fout[aidx], fin);
+
+ if (bCalcFshift)
+ {
+ fshift_buf->x += fin.x;
+ fshift_buf->y += fin.y;
+ fshift_buf->z += fin.z;
+ }
+ }
+}
+#endif
+
+/*! Energy reduction; this implementation works only with power of two
+ * array sizes.
+ */
+static inline __device__
+void reduce_energy_pow2(volatile float *buf,
+ float *e_lj, float *e_el,
+ unsigned int tidx)
+{
+ int i, j;
+ float e1, e2;
+
+ i = WARP_SIZE/2;
+
+ /* Can't just use i as loop variable because than nvcc refuses to unroll. */
+# pragma unroll 10
+ for (j = WARP_SIZE_POW2_EXPONENT - 1; j > 0; j--)
+ {
+ if (tidx < i)
+ {
+ buf[ tidx] += buf[ tidx + i];
+ buf[FBUF_STRIDE + tidx] += buf[FBUF_STRIDE + tidx + i];
+ }
+ i >>= 1;
+ }
+
+ /* last reduction step, writing to global mem */
+ if (tidx == 0)
+ {
+ e1 = buf[ tidx] + buf[ tidx + i];
+ e2 = buf[FBUF_STRIDE + tidx] + buf[FBUF_STRIDE + tidx + i];
+
+ atomicAdd(e_lj, e1);
+ atomicAdd(e_el, e2);
+ }
+}
+
+/*! Energy reduction; this implementation works only with power of two
+ * array sizes and with sm >= 3.0
+ */
+#if __CUDA_ARCH__ >= 300
+static inline __device__
+void reduce_energy_warp_shfl(float E_lj, float E_el,
+ float *e_lj, float *e_el,
+ int tidx)
+{
+ int i, sh;
+
+ sh = 1;
+#pragma unroll 5
+ for (i = 0; i < 5; i++)
+ {
+ E_lj += __shfl_down(E_lj,sh);
+ E_el += __shfl_down(E_el,sh);
+ sh += sh;
+ }
+
+ /* The first thread in the warp writes the reduced energies */
+ if (tidx == 0 || tidx == WARP_SIZE)
+ {
+ atomicAdd(e_lj,E_lj);
+ atomicAdd(e_el,E_el);
+ }
+}
+#endif /* __CUDA_ARCH__ */
+
+#endif /* NBNXN_CUDA_KERNEL_UTILS_CUH */
--- /dev/null
- /* All structs prefixed with "cu_" hold data used in GPU calculations and
- * are passed to the kernels, except cu_timers_t. */
+/*
+ * This file is part of the GROMACS molecular simulation package.
+ *
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2012, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+ * Copyright (c) 2012,2013, by the GROMACS development team, led by
+ * David van der Spoel, Berk Hess, Erik Lindahl, and including many
+ * others, as listed in the AUTHORS file in the top-level source
+ * directory and at http://www.gromacs.org.
+ *
+ * GROMACS is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ *
+ * GROMACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with GROMACS; if not, see
+ * http://www.gnu.org/licenses, or write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * If you want to redistribute modifications to GROMACS, please
+ * consider that scientific software is very special. Version
+ * control is crucial - bugs must be traceable. We will be happy to
+ * consider code for inclusion in the official distribution, but
+ * derived work must not be called official GROMACS. Details are found
+ * in the README & COPYING files - if they are missing, get the
+ * official version at http://www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the research papers on the package. Check out http://www.gromacs.org.
+ */
+
+#ifndef NBNXN_CUDA_TYPES_H
+#define NBNXN_CUDA_TYPES_H
+
+#include "types/nbnxn_pairlist.h"
+#include "types/nbnxn_cuda_types_ext.h"
+#include "../../gmxlib/cuda_tools/cudautils.cuh"
+
++/* CUDA versions from 5.0 above support texture objects. */
++#if CUDA_VERSION >= 5000
++#define TEXOBJ_SUPPORTED
++#else /* CUDA_VERSION */
++/* This typedef allows us to define only one version of struct cu_nbparam */
++typedef int cudaTextureObject_t;
++#endif /* CUDA_VERSION */
++
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** Types of electrostatics implementations available in the CUDA non-bonded
+ * force kernels. These represent both the electrostatics types implemented
+ * by the kernels (cut-off, RF, and Ewald - a subset of what's defined in
+ * enums.h) as well as encode implementation details analytical/tabulated
+ * and single or twin cut-off (for Ewald kernels).
+ * Note that the cut-off and RF kernels have only analytical flavor and unlike
+ * in the CPU kernels, the tabulated kernels are ATM Ewald-only.
+ *
+ * The order of pointers to different electrostatic kernels defined in
+ * nbnxn_cuda.cu by the nb_default_kfunc_ptr and nb_legacy_kfunc_ptr arrays
+ * should match the order of enumerated types below. */
+enum {
+ eelCuCUT, eelCuRF, eelCuEWALD_TAB, eelCuEWALD_TAB_TWIN, eelCuEWALD_ANA, eelCuEWALD_ANA_TWIN, eelCuNR
+};
+
+/** Kernel flavors with different set of optimizations: default for CUDA <=v4.1
+ * compilers and legacy for earlier, 3.2 and 4.0 CUDA compilers. */
+enum {
+ eNbnxnCuKDefault, eNbnxnCuKLegacy, eNbnxnCuKNR
+};
+
+#define NBNXN_KVER_OLD(k) (k == eNbnxnCuKOld)
+#define NBNXN_KVER_LEGACY(k) (k == eNbnxnCuKLegacy)
+#define NBNXN_KVER_DEFAULT(k) (k == eNbnxnCuKDefault)
+
+/* Non-bonded kernel versions. */
+
- * before getting added to the CPU-side aggregate values.
++/* All structs prefixed with "cu_" hold data used in GPU calculations and
++ * are passed to the kernels, except cu_timers_t. */
+typedef struct cu_plist cu_plist_t;
+typedef struct cu_atomdata cu_atomdata_t;
+typedef struct cu_nbparam cu_nbparam_t;
+typedef struct cu_timers cu_timers_t;
+typedef struct nb_staging nb_staging_t;
+
+
+/** Staging area for temporary data. The energies get downloaded here first,
- int eeltype; /**< type of electrostatics */
-
- float epsfac; /**< charge multiplication factor */
- float c_rf, /**< Reaction-field/plain cutoff electrostatics const. */
- two_k_rf; /**< Reaction-field electrostatics constant */
- float ewald_beta; /**< Ewald/PME parameter */
- float sh_ewald; /**< Ewald/PME correction term */
- float rvdw_sq; /**< VdW cut-off */
- float rcoulomb_sq; /**< Coulomb cut-off */
- float rlist_sq; /**< pair-list cut-off */
- float sh_invrc6; /**< LJ potential correction term */
-
- float *nbfp; /**< nonbonded parameter table with C6/C12 pairs */
-
- /* Ewald Coulomb force table data */
- int coulomb_tab_size; /**< table size (s.t. it fits in texture cache) */
- float coulomb_tab_scale; /**< table scale/spacing */
- float *coulomb_tab; /**< pointer to the table in the device memory */
++ * before getting added to the CPU-side aggregate values.
+ */
+struct nb_staging
+{
+ float *e_lj; /**< LJ energy */
+ float *e_el; /**< electrostatic energy */
+ float3 *fshift; /**< shift forces */
+};
+
+/** Nonbonded atom data -- both inputs and outputs. */
+struct cu_atomdata
+{
+ int natoms; /**< number of atoms */
+ int natoms_local; /**< number of local atoms */
+ int nalloc; /**< allocation size for the atom data (xq, f) */
+
+ float4 *xq; /**< atom coordinates + charges, size natoms */
+ float3 *f; /**< force output array, size natoms */
+ /* TODO: try float2 for the energies */
+ float *e_lj, /**< LJ energy output, size 1 */
+ *e_el; /**< Electrostatics energy input, size 1 */
+
+ float3 *fshift; /**< shift forces */
+
+ int ntypes; /**< number of atom types */
+ int *atom_types; /**< atom type indices, size natoms */
+
+ float3 *shift_vec; /**< shifts */
+ bool bShiftVecUploaded; /**< true if the shift vector has been uploaded */
+};
+
+/** Parameters required for the CUDA nonbonded calculations. */
+struct cu_nbparam
+{
- cudaEvent_t nonlocal_done; /**< event triggered when the non-local non-bonded kernel
- is done (and the local transfer can proceed) */
- cudaEvent_t misc_ops_done; /**< event triggered when the operations that precede the
- main force calculations are done (e.g. buffer 0-ing) */
++ int eeltype; /**< type of electrostatics */
++
++ float epsfac; /**< charge multiplication factor */
++ float c_rf; /**< Reaction-field/plain cutoff electrostatics const. */
++ float two_k_rf; /**< Reaction-field electrostatics constant */
++ float ewald_beta; /**< Ewald/PME parameter */
++ float sh_ewald; /**< Ewald/PME correction term */
++ float rvdw_sq; /**< VdW cut-off */
++ float rcoulomb_sq; /**< Coulomb cut-off */
++ float rlist_sq; /**< pair-list cut-off */
++ float sh_invrc6; /**< LJ potential correction term */
++
++ /* Non-bonded parameters - accessed through texture memory */
++ float *nbfp; /**< nonbonded parameter table with C6/C12 pairs */
++ cudaTextureObject_t nbfp_texobj; /**< texture object bound to nbfp */
++
++ /* Ewald Coulomb force table data - accessed through texture memory */
++ int coulomb_tab_size; /**< table size (s.t. it fits in texture cache) */
++ float coulomb_tab_scale; /**< table scale/spacing */
++ float *coulomb_tab; /**< pointer to the table in the device memory */
++ cudaTextureObject_t coulomb_tab_texobj; /**< texture object bound to coulomb_tab */
+};
+
+/** Pair list data */
+struct cu_plist
+{
+ int na_c; /**< number of atoms per cluster */
+
+ int nsci; /**< size of sci, # of i clusters in the list */
+ int sci_nalloc; /**< allocation size of sci */
+ nbnxn_sci_t *sci; /**< list of i-cluster ("super-clusters") */
+
+ int ncj4; /**< total # of 4*j clusters */
+ int cj4_nalloc; /**< allocation size of cj4 */
+ nbnxn_cj4_t *cj4; /**< 4*j cluster list, contains j cluster number
+ and index into the i cluster list */
+ nbnxn_excl_t *excl; /**< atom interaction bits */
+ int nexcl; /**< count for excl */
+ int excl_nalloc; /**< allocation size of excl */
+
+ bool bDoPrune; /**< true if pair-list pruning needs to be
+ done during the current step */
+};
+
+/** CUDA events used for timing GPU kernels and H2D/D2H transfers.
+ * The two-sized arrays hold the local and non-local values and should always
+ * be indexed with eintLocal/eintNonlocal.
+ */
+struct cu_timers
+{
+ cudaEvent_t start_atdat; /**< start event for atom data transfer (every PS step) */
+ cudaEvent_t stop_atdat; /**< stop event for atom data transfer (every PS step) */
+ cudaEvent_t start_nb_h2d[2]; /**< start events for x/q H2D transfers (l/nl, every step) */
+ cudaEvent_t stop_nb_h2d[2]; /**< stop events for x/q H2D transfers (l/nl, every step) */
+ cudaEvent_t start_nb_d2h[2]; /**< start events for f D2H transfer (l/nl, every step) */
+ cudaEvent_t stop_nb_d2h[2]; /**< stop events for f D2H transfer (l/nl, every step) */
+ cudaEvent_t start_pl_h2d[2]; /**< start events for pair-list H2D transfers (l/nl, every PS step) */
+ cudaEvent_t stop_pl_h2d[2]; /**< start events for pair-list H2D transfers (l/nl, every PS step) */
+ cudaEvent_t start_nb_k[2]; /**< start event for non-bonded kernels (l/nl, every step) */
+ cudaEvent_t stop_nb_k[2]; /**< stop event non-bonded kernels (l/nl, every step) */
+};
+
+/** Main data structure for CUDA nonbonded force calculations. */
+struct nbnxn_cuda
+{
+ cuda_dev_info_t *dev_info; /**< CUDA device information */
+ int kernel_ver; /**< The version of the kernel to be executed on the
+ device in use, possible values: eNbnxnCuK* */
+ bool bUseTwoStreams; /**< true if doing both local/non-local NB work on GPU */
+ bool bUseStreamSync; /**< true if the standard cudaStreamSynchronize is used
+ and not memory polling-based waiting */
+ cu_atomdata_t *atdat; /**< atom data */
+ cu_nbparam_t *nbparam; /**< parameters required for the non-bonded calc. */
+ cu_plist_t *plist[2]; /**< pair-list data structures (local and non-local) */
+ nb_staging_t nbst; /**< staging area where fshift/energies get downloaded */
+
+ cudaStream_t stream[2]; /**< local and non-local GPU streams */
+
+ /** events used for synchronization */
++ cudaEvent_t nonlocal_done; /**< event triggered when the non-local non-bonded kernel
++ is done (and the local transfer can proceed) */
++ cudaEvent_t misc_ops_done; /**< event triggered when the operations that precede the
++ main force calculations are done (e.g. buffer 0-ing) */
+
+ /* NOTE: With current CUDA versions (<=5.0) timing doesn't work with multiple
+ * concurrent streams, so we won't time if both l/nl work is done on GPUs.
+ * Timer init/uninit is still done even with timing off so only the condition
+ * setting bDoTime needs to be change if this CUDA "feature" gets fixed. */
+ bool bDoTime; /**< True if event-based timing is enabled. */
+ cu_timers_t *timers; /**< CUDA event-based timers. */
+ wallclock_gpu_t *timings; /**< Timing data. */
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* NBNXN_CUDA_TYPES_H */
--- /dev/null
- #ifdef __cplusplus
- extern "C" {
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustr
+ *
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2012, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * Gallium Rubidium Oxygen Manganese Argon Carbon Silicon
+ */
+
+#ifndef _nbnxn_internal_h
+#define _nbnxn_internal_h
+
+#include "typedefs.h"
+#include "domdec.h"
+#include "gmx_cyclecounter.h"
+
+#ifdef GMX_NBNXN_SIMD
+/* The include below sets the SIMD instruction type (precision+width)
+ * for all nbnxn SIMD search and non-bonded kernel code.
+ */
+#ifdef GMX_NBNXN_HALF_WIDTH_SIMD
+#define GMX_USE_HALF_WIDTH_SIMD_HERE
+#endif
+#include "gmx_simd_macros.h"
+#endif
+
- #ifdef GMX_X86_SSE2
- /* Use 4-way SIMD for, always, single precision bounding box calculations */
- #define NBNXN_SEARCH_BB_SSE
++
++/* Bounding box calculations are (currently) always in single precision.
++ * This uses less (cache-)memory and SIMD is faster, at least on x86.
++ */
++#define GMX_SIMD4_SINGLE
++/* Include the 4-wide SIMD macro file */
++#include "gmx_simd4_macros.h"
++/* Check if we have 4-wide SIMD macro support */
++#ifdef GMX_HAVE_SIMD4_MACROS
++#define NBNXN_SEARCH_BB_SIMD4
+#endif
+
+
- * Store this in 4 iso 3 reals, which is useful with SSE.
- * To avoid complicating the code we also use 4 without SSE.
++#ifdef __cplusplus
++extern "C" {
+#endif
+
+
+#ifdef GMX_NBNXN_SIMD
+/* Memory alignment in bytes as required by SIMD aligned loads/stores */
+#define NBNXN_MEM_ALIGN (GMX_SIMD_WIDTH_HERE*sizeof(real))
+#else
+/* No alignment required, but set it so we can call the same routines */
+#define NBNXN_MEM_ALIGN 32
+#endif
+
+
++#ifdef NBNXN_SEARCH_BB_SIMD4
++/* Memory alignment in bytes as required by SIMD aligned loads/stores */
++#define NBNXN_SEARCH_BB_MEM_ALIGN (GMX_SIMD4_WIDTH*sizeof(float))
++#else
++/* No alignment required, but set it so we can call the same routines */
++#define NBNXN_SEARCH_BB_MEM_ALIGN 32
++#endif
++
++
+/* Pair search box lower and upper corner in x,y,z.
- nbnxn_bb_t *bbj; /* 3D j-b.boxes for SSE-double or AVX-single */
++ * Store this in 4 iso 3 reals, which is useful with 4-wide SIMD.
++ * To avoid complicating the code we also use 4 without 4-wide SIMD.
+ */
+#define NNBSBB_C 4
+/* Pair search box lower and upper bound in z only. */
+#define NNBSBB_D 2
+/* Pair search box lower and upper corner x,y,z indices, entry 3 is unused */
+#define BB_X 0
+#define BB_Y 1
+#define BB_Z 2
+
+/* Bounding box for a nbnxn atom cluster */
+typedef struct {
+ float lower[NNBSBB_C];
+ float upper[NNBSBB_C];
+} nbnxn_bb_t;
+
+
+/* A pair-search grid struct for one domain decomposition zone */
+typedef struct {
+ rvec c0; /* The lower corner of the (local) grid */
+ rvec c1; /* The upper corner of the (local) grid */
+ real atom_density; /* The atom number density for the local grid */
+
+ gmx_bool bSimple; /* Is this grid simple or super/sub */
+ int na_c; /* Number of atoms per cluster */
+ int na_cj; /* Number of atoms for list j-clusters */
+ int na_sc; /* Number of atoms per super-cluster */
+ int na_c_2log; /* 2log of na_c */
+
+ int ncx; /* Number of (super-)cells along x */
+ int ncy; /* Number of (super-)cells along y */
+ int nc; /* Total number of (super-)cells */
+
+ real sx; /* x-size of a (super-)cell */
+ real sy; /* y-size of a (super-)cell */
+ real inv_sx; /* 1/sx */
+ real inv_sy; /* 1/sy */
+
+ int cell0; /* Index in nbs->cell corresponding to cell 0 */
+
+ int *cxy_na; /* The number of atoms for each column in x,y */
+ int *cxy_ind; /* Grid (super)cell index, offset from cell0 */
+ int cxy_nalloc; /* Allocation size for cxy_na and cxy_ind */
+
+ int *nsubc; /* The number of sub cells for each super cell */
+ float *bbcz; /* Bounding boxes in z for the super cells */
+ nbnxn_bb_t *bb; /* 3D bounding boxes for the sub cells */
- gmx_mm_pr ix_SSE0, iy_SSE0, iz_SSE0;
- gmx_mm_pr ix_SSE1, iy_SSE1, iz_SSE1;
- gmx_mm_pr ix_SSE2, iy_SSE2, iz_SSE2;
- gmx_mm_pr ix_SSE3, iy_SSE3, iz_SSE3;
++ nbnxn_bb_t *bbj; /* 3D j-bounding boxes for the case where *
++ * the i- and j-cluster sizes are different */
+ float *pbb; /* 3D b. boxes in xxxx format per super cell */
+ int *flags; /* Flag for the super cells */
+ int nc_nalloc; /* Allocation size for the pointers above */
+
+ float *bbcz_simple; /* bbcz for simple grid converted from super */
+ nbnxn_bb_t *bb_simple; /* bb for simple grid converted from super */
+ int *flags_simple; /* flags for simple grid converted from super */
+ int nc_nalloc_simple; /* Allocation size for the pointers above */
+
+ int nsubc_tot; /* Total number of subcell, used for printing */
+} nbnxn_grid_t;
+
+#ifdef GMX_NBNXN_SIMD
+
+typedef struct nbnxn_x_ci_simd_4xn {
+ /* The i-cluster coordinates for simple search */
- gmx_mm_pr ix_SSE0, iy_SSE0, iz_SSE0;
- gmx_mm_pr ix_SSE2, iy_SSE2, iz_SSE2;
++ gmx_mm_pr ix_S0, iy_S0, iz_S0;
++ gmx_mm_pr ix_S1, iy_S1, iz_S1;
++ gmx_mm_pr ix_S2, iy_S2, iz_S2;
++ gmx_mm_pr ix_S3, iy_S3, iz_S3;
+} nbnxn_x_ci_simd_4xn_t;
+
+typedef struct nbnxn_x_ci_simd_2xnn {
+ /* The i-cluster coordinates for simple search */
++ gmx_mm_pr ix_S0, iy_S0, iz_S0;
++ gmx_mm_pr ix_S2, iy_S2, iz_S2;
+} nbnxn_x_ci_simd_2xnn_t;
+
+#endif
+
+/* Working data for the actual i-supercell during pair search */
+typedef struct nbnxn_list_work {
+ gmx_cache_protect_t cp0; /* Protect cache between threads */
+
+ nbnxn_bb_t *bb_ci; /* The bounding boxes, pbc shifted, for each cluster */
+ float *pbb_ci; /* As bb_ci, but in xxxx packed format */
+ real *x_ci; /* The coordinates, pbc shifted, for each atom */
+#ifdef GMX_NBNXN_SIMD
+ nbnxn_x_ci_simd_4xn_t *x_ci_simd_4xn;
+ nbnxn_x_ci_simd_2xnn_t *x_ci_simd_2xnn;
+#endif
+ int cj_ind; /* The current cj_ind index for the current list */
+ int cj4_init; /* The first unitialized cj4 block */
+
+ float *d2; /* Bounding box distance work array */
+
+ nbnxn_cj_t *cj; /* The j-cell list */
+ int cj_nalloc; /* Allocation size of cj */
+
+ int ncj_noq; /* Nr. of cluster pairs without Coul for flop count */
+ int ncj_hlj; /* Nr. of cluster pairs with 1/2 LJ for flop count */
+
+ int *sort; /* Sort index */
+ int sort_nalloc; /* Allocation size of sort */
+
+ nbnxn_sci_t *sci_sort; /* Second sci array, for sorting */
+ int sci_sort_nalloc; /* Allocation size of sci_sort */
+
+ gmx_cache_protect_t cp1; /* Protect cache between threads */
+} nbnxn_list_work_t;
+
+/* Function type for setting the i-atom coordinate working data */
+typedef void
+ gmx_icell_set_x_t (int ci,
+ real shx, real shy, real shz,
+ int na_c,
+ int stride, const real *x,
+ nbnxn_list_work_t *work);
+
+static gmx_icell_set_x_t icell_set_x_simple;
+#ifdef GMX_NBNXN_SIMD
+static gmx_icell_set_x_t icell_set_x_simple_simd_4xn;
+static gmx_icell_set_x_t icell_set_x_simple_simd_2xnn;
+#endif
+static gmx_icell_set_x_t icell_set_x_supersub;
+#ifdef NBNXN_SEARCH_SSE
+static gmx_icell_set_x_t icell_set_x_supersub_sse8;
+#endif
+
+/* Local cycle count struct for profiling */
+typedef struct {
+ int count;
+ gmx_cycles_t c;
+ gmx_cycles_t start;
+} nbnxn_cycle_t;
+
+/* Local cycle count enum for profiling */
+enum {
+ enbsCCgrid, enbsCCsearch, enbsCCcombine, enbsCCreducef, enbsCCnr
+};
+
+/* Thread-local work struct, contains part of nbnxn_grid_t */
+typedef struct {
+ gmx_cache_protect_t cp0;
+
+ int *cxy_na;
+ int cxy_na_nalloc;
+
+ int *sort_work;
+ int sort_work_nalloc;
+
+ nbnxn_buffer_flags_t buffer_flags; /* Flags for force buffer access */
+
+ int ndistc; /* Number of distance checks for flop counting */
+
+ nbnxn_cycle_t cc[enbsCCnr];
+
+ gmx_cache_protect_t cp1;
+} nbnxn_search_work_t;
+
+/* Main pair-search struct, contains the grid(s), not the pair-list(s) */
+typedef struct nbnxn_search {
+ int ePBC; /* PBC type enum */
+ matrix box; /* The periodic unit-cell */
+
+ gmx_bool DomDec; /* Are we doing domain decomposition? */
+ ivec dd_dim; /* Are we doing DD in x,y,z? */
+ gmx_domdec_zones_t *zones; /* The domain decomposition zones */
+
+ int ngrid; /* The number of grids, equal to #DD-zones */
+ nbnxn_grid_t *grid; /* Array of grids, size ngrid */
+ int *cell; /* Actual allocated cell array for all grids */
+ int cell_nalloc; /* Allocation size of cell */
+ int *a; /* Atom index for grid, the inverse of cell */
+ int a_nalloc; /* Allocation size of a */
+
+ int natoms_local; /* The local atoms run from 0 to natoms_local */
+ int natoms_nonlocal; /* The non-local atoms run from natoms_local
+ * to natoms_nonlocal */
+
+ gmx_bool print_cycles;
+ int search_count;
+ nbnxn_cycle_t cc[enbsCCnr];
+
+ gmx_icell_set_x_t *icell_set_x; /* Function for setting i-coords */
+
+ int nthread_max; /* Maximum number of threads for pair-search */
+ nbnxn_search_work_t *work; /* Work array, size nthread_max */
+} nbnxn_search_t_t;
+
+
+static void nbs_cycle_start(nbnxn_cycle_t *cc)
+{
+ cc->start = gmx_cycles_read();
+}
+
+static void nbs_cycle_stop(nbnxn_cycle_t *cc)
+{
+ cc->c += gmx_cycles_read() - cc->start;
+ cc->count++;
+}
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
--- /dev/null
- (padded) array. Only strides of 2 and 4 are currently supported. */
- #if defined GMX_NBNXN_SIMD_2XNN
- static const int nbfp_stride = 4;
- #elif defined GMX_DOUBLE
+/*
+ * This file is part of the GROMACS molecular simulation package.
+ *
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2012, The GROMACS Development Team
+ * Copyright (c) 2012, by the GROMACS development team, led by
+ * David van der Spoel, Berk Hess, Erik Lindahl, and including many
+ * others, as listed in the AUTHORS file in the top-level source
+ * directory and at http://www.gromacs.org.
+ *
+ * GROMACS is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ *
+ * GROMACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with GROMACS; if not, see
+ * http://www.gnu.org/licenses, or write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * If you want to redistribute modifications to GROMACS, please
+ * consider that scientific software is very special. Version
+ * control is crucial - bugs must be traceable. We will be happy to
+ * consider code for inclusion in the official distribution, but
+ * derived work must not be called official GROMACS. Details are found
+ * in the README & COPYING files - if they are missing, get the
+ * official version at http://www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the research papers on the package. Check out http://www.gromacs.org.
+ */
+#ifndef _nbnxn_kernel_simd_utils_h_
+#define _nbnxn_kernel_simd_utils_h_
+
+/*! \brief Provides hardware-specific utility routines for the SIMD kernels.
+ *
+ * Defines all functions, typedefs, constants and macros that have
+ * explicit dependencies on the j-cluster size, precision, or SIMD
+ * width. This includes handling diagonal, Newton and topology
+ * exclusions.
+ *
+ * The functionality which depends on the j-cluster size is:
+ * LJ-parameter lookup
+ * force table lookup
+ * energy group pair energy storage
+ */
+
+#if !defined GMX_NBNXN_SIMD_2XNN && !defined GMX_NBNXN_SIMD_4XN
+#error "Must define an NBNxN kernel flavour before including NBNxN kernel utility functions"
+#endif
+
+#ifdef GMX_SIMD_REFERENCE_PLAIN_C
+
++/* Set the stride for the lookup of the two LJ parameters from their
++ * (padded) array.
++ * Note that currently only arrays with stride 2 and 4 are available.
++ * Since the reference code does not require alignment, we can always use 2.
++ */
++static const int nbfp_stride = 2;
++
++/* Align a stack-based thread-local working array. */
++static gmx_inline int *
++prepare_table_load_buffer(const int *array)
++{
++ return NULL;
++}
++
+#include "nbnxn_kernel_simd_utils_ref.h"
+
+#else /* GMX_SIMD_REFERENCE_PLAIN_C */
+
+#ifdef GMX_X86_SSE2
+/* Include x86 SSE2 compatible SIMD functions */
+
+/* Set the stride for the lookup of the two LJ parameters from their
++ * (padded) array. We use the minimum supported SIMD memory alignment.
++ */
++#if defined GMX_DOUBLE
+static const int nbfp_stride = 2;
+#else
+static const int nbfp_stride = 4;
+#endif
+
+/* Align a stack-based thread-local working array. Table loads on
+ * full-width AVX_256 use the array, but other implementations do
+ * not. */
+static gmx_inline int *
+prepare_table_load_buffer(const int *array)
+{
+#if defined GMX_X86_AVX_256 && !defined GMX_USE_HALF_WIDTH_SIMD_HERE
+ return gmx_simd_align_int(array);
+#else
+ return NULL;
+#endif
+}
+
+#if defined GMX_X86_AVX_256 && !defined GMX_USE_HALF_WIDTH_SIMD_HERE
+
+/* With full AVX-256 SIMD, half SIMD-width table loads are optimal */
+#if GMX_SIMD_WIDTH_HERE == 8
+#define TAB_FDV0
+#endif
+
+/*
+Berk, 2xnn.c had the following code, but I think it is safe to remove now, given the code immediately above.
+
+#if defined GMX_X86_AVX_256 && !defined GMX_DOUBLE
+/ * AVX-256 single precision 2x(4+4) kernel,
+ * we can do half SIMD-width aligned FDV0 table loads.
+ * /
+#define TAB_FDV0
+#endif
+*/
+
+#ifdef GMX_DOUBLE
+#include "nbnxn_kernel_simd_utils_x86_256d.h"
+#else /* GMX_DOUBLE */
+#include "nbnxn_kernel_simd_utils_x86_256s.h"
+#endif /* GMX_DOUBLE */
+
+#else /* defined GMX_X86_AVX_256 && !defined GMX_USE_HALF_WIDTH_SIMD_HERE */
+
+/* We use the FDV0 table layout when we can use aligned table loads */
+#if GMX_SIMD_WIDTH_HERE == 4
+#define TAB_FDV0
+#endif
+
+#ifdef GMX_DOUBLE
+#include "nbnxn_kernel_simd_utils_x86_128d.h"
+#else /* GMX_DOUBLE */
+#include "nbnxn_kernel_simd_utils_x86_128s.h"
+#endif /* GMX_DOUBLE */
+
+#endif /* defined GMX_X86_AVX_256 && !defined GMX_USE_HALF_WIDTH_SIMD_HERE */
+
+#else /* GMX_X86_SSE2 */
+
+#if GMX_SIMD_WIDTH_HERE > 4
+static const int nbfp_stride = 4;
+#else
+static const int nbfp_stride = GMX_SIMD_WIDTH_HERE;
+#endif
+
++/* We use the FDV0 table layout when we can use aligned table loads */
++#if GMX_SIMD_WIDTH_HERE == 4
++#define TAB_FDV0
++#endif
++
++#ifdef GMX_CPU_ACCELERATION_IBM_QPX
++#include "nbnxn_kernel_simd_utils_ibm_qpx.h"
++#endif /* GMX_CPU_ACCELERATION_IBM_QPX */
++
+#endif /* GMX_X86_SSE2 */
+#endif /* GMX_SIMD_REFERENCE_PLAIN_C */
+
+
+#ifdef UNROLLJ
+/* Add energy register to possibly multiple terms in the energy array */
+static inline void add_ener_grp(gmx_mm_pr e_S, real *v, const int *offset_jj)
+{
+ int jj;
+
+ /* We need to balance the number of store operations with
+ * the rapidly increases number of combinations of energy groups.
+ * We add to a temporary buffer for 1 i-group vs 2 j-groups.
+ */
+ for (jj = 0; jj < (UNROLLJ/2); jj++)
+ {
+ gmx_mm_pr v_S;
+
+ v_S = gmx_load_pr(v+offset_jj[jj]+jj*GMX_SIMD_WIDTH_HERE);
+ gmx_store_pr(v+offset_jj[jj]+jj*GMX_SIMD_WIDTH_HERE, gmx_add_pr(v_S, e_S));
+ }
+}
+#endif
+
+#if defined GMX_NBNXN_SIMD_2XNN && defined UNROLLJ
+/* As add_ener_grp, but for two groups of UNROLLJ/2 stored in
+ * a single SIMD register.
+ */
+static inline void
+add_ener_grp_halves(gmx_mm_pr e_S, real *v0, real *v1, const int *offset_jj)
+{
+ gmx_mm_hpr e_S0, e_S1;
+ int jj;
+
+ gmx_pr_to_2hpr(e_S, &e_S0, &e_S1);
+
+ for (jj = 0; jj < (UNROLLJ/2); jj++)
+ {
+ gmx_mm_hpr v_S;
+
+ gmx_load_hpr(&v_S, v0+offset_jj[jj]+jj*GMX_SIMD_WIDTH_HERE/2);
+ gmx_store_hpr(v0+offset_jj[jj]+jj*GMX_SIMD_WIDTH_HERE/2, gmx_add_hpr(v_S, e_S0));
+ }
+ for (jj = 0; jj < (UNROLLJ/2); jj++)
+ {
+ gmx_mm_hpr v_S;
+
+ gmx_load_hpr(&v_S, v1+offset_jj[jj]+jj*GMX_SIMD_WIDTH_HERE/2);
+ gmx_store_hpr(v1+offset_jj[jj]+jj*GMX_SIMD_WIDTH_HERE/2, gmx_add_hpr(v_S, e_S1));
+ }
+}
+#endif
+
+#endif /* _nbnxn_kernel_simd_utils_h_ */
--- /dev/null
- #ifdef NBNXN_SEARCH_BB_SSE
- /* We use SSE or AVX-128bit for bounding box calculations */
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2012, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <math.h>
+#include <string.h>
+#include "sysstuff.h"
+#include "smalloc.h"
+#include "macros.h"
+#include "maths.h"
+#include "vec.h"
+#include "pbc.h"
+#include "nbnxn_consts.h"
+/* nbnxn_internal.h included gmx_simd_macros.h */
+#include "nbnxn_internal.h"
+#ifdef GMX_NBNXN_SIMD
+#include "gmx_simd_vec.h"
+#endif
+#include "nbnxn_atomdata.h"
+#include "nbnxn_search.h"
+#include "gmx_cyclecounter.h"
+#include "gmxfio.h"
+#include "gmx_omp_nthreads.h"
+#include "nrnb.h"
+
+
- /* Single precision BBs + coordinates, we can also load coordinates using SSE */
- #define NBNXN_SEARCH_SSE_SINGLE
++#ifdef NBNXN_SEARCH_BB_SIMD4
++/* We use 4-wide SIMD for bounding box calculations */
+
+#ifndef GMX_DOUBLE
- /* Include basic SSE2 stuff */
- #include <emmintrin.h>
-
- #if defined NBNXN_SEARCH_SSE_SINGLE && (GPU_NSUBCELL == 4 || GPU_NSUBCELL == 8)
++/* Single precision BBs + coordinates, we can also load coordinates with SIMD */
++#define NBNXN_SEARCH_SIMD4_FLOAT_X_BB
+#endif
+
- #define NBNXN_PBB_SSE
++#if defined NBNXN_SEARCH_SIMD4_FLOAT_X_BB && (GPU_NSUBCELL == 4 || GPU_NSUBCELL == 8)
+/* Store bounding boxes with x, y and z coordinates in packs of 4 */
- /* The width of SSE/AVX128 with single precision for bounding boxes with GPU.
- * Here AVX-256 turns out to be slightly slower than AVX-128.
++#define NBNXN_PBB_SIMD4
+#endif
+
- #endif /* NBNXN_SEARCH_BB_SSE */
++/* The packed bounding box coordinate stride is always set to 4.
++ * With AVX we could use 8, but that turns out not to be faster.
+ */
+#define STRIDE_PBB 4
+#define STRIDE_PBB_2LOG 2
+
- #ifdef NBNXN_SEARCH_BB_SSE
++#endif /* NBNXN_SEARCH_BB_SIMD4 */
+
+#ifdef GMX_NBNXN_SIMD
+
+/* The functions below are macros as they are performance sensitive */
+
+/* 4x4 list, pack=4: no complex conversion required */
+/* i-cluster to j-cluster conversion */
+#define CI_TO_CJ_J4(ci) (ci)
+/* cluster index to coordinate array index conversion */
+#define X_IND_CI_J4(ci) ((ci)*STRIDE_P4)
+#define X_IND_CJ_J4(cj) ((cj)*STRIDE_P4)
+
+/* 4x2 list, pack=4: j-cluster size is half the packing width */
+/* i-cluster to j-cluster conversion */
+#define CI_TO_CJ_J2(ci) ((ci)<<1)
+/* cluster index to coordinate array index conversion */
+#define X_IND_CI_J2(ci) ((ci)*STRIDE_P4)
+#define X_IND_CJ_J2(cj) (((cj)>>1)*STRIDE_P4 + ((cj) & 1)*(PACK_X4>>1))
+
+/* 4x8 list, pack=8: i-cluster size is half the packing width */
+/* i-cluster to j-cluster conversion */
+#define CI_TO_CJ_J8(ci) ((ci)>>1)
+/* cluster index to coordinate array index conversion */
+#define X_IND_CI_J8(ci) (((ci)>>1)*STRIDE_P8 + ((ci) & 1)*(PACK_X8>>1))
+#define X_IND_CJ_J8(cj) ((cj)*STRIDE_P8)
+
+/* The j-cluster size is matched to the SIMD width */
+#if GMX_SIMD_WIDTH_HERE == 2
+#define CI_TO_CJ_SIMD_4XN(ci) CI_TO_CJ_J2(ci)
+#define X_IND_CI_SIMD_4XN(ci) X_IND_CI_J2(ci)
+#define X_IND_CJ_SIMD_4XN(cj) X_IND_CJ_J2(cj)
+#else
+#if GMX_SIMD_WIDTH_HERE == 4
+#define CI_TO_CJ_SIMD_4XN(ci) CI_TO_CJ_J4(ci)
+#define X_IND_CI_SIMD_4XN(ci) X_IND_CI_J4(ci)
+#define X_IND_CJ_SIMD_4XN(cj) X_IND_CJ_J4(cj)
+#else
+#if GMX_SIMD_WIDTH_HERE == 8
+#define CI_TO_CJ_SIMD_4XN(ci) CI_TO_CJ_J8(ci)
+#define X_IND_CI_SIMD_4XN(ci) X_IND_CI_J8(ci)
+#define X_IND_CJ_SIMD_4XN(cj) X_IND_CJ_J8(cj)
+/* Half SIMD with j-cluster size */
+#define CI_TO_CJ_SIMD_2XNN(ci) CI_TO_CJ_J4(ci)
+#define X_IND_CI_SIMD_2XNN(ci) X_IND_CI_J4(ci)
+#define X_IND_CJ_SIMD_2XNN(cj) X_IND_CJ_J4(cj)
+#else
+#if GMX_SIMD_WIDTH_HERE == 16
+#define CI_TO_CJ_SIMD_2XNN(ci) CI_TO_CJ_J8(ci)
+#define X_IND_CI_SIMD_2XNN(ci) X_IND_CI_J8(ci)
+#define X_IND_CJ_SIMD_2XNN(cj) X_IND_CJ_J8(cj)
+#else
+#error "unsupported GMX_NBNXN_SIMD_WIDTH"
+#endif
+#endif
+#endif
+#endif
+
+#endif /* GMX_NBNXN_SIMD */
+
+
- * floating exceptions in SSE with the unused bb elements.
++#ifdef NBNXN_SEARCH_BB_SIMD4
+/* Store bounding boxes corners as quadruplets: xxxxyyyyzzzz */
+#define NBNXN_BBXXXX
+/* Size of bounding box corners quadruplet */
+#define NNBSBB_XXXX (NNBSBB_D*DIM*STRIDE_PBB)
+#endif
+
+/* We shift the i-particles backward for PBC.
+ * This leads to more conditionals than shifting forward.
+ * We do this to get more balanced pair lists.
+ */
+#define NBNXN_SHIFT_BACKWARD
+
+
+/* This define is a lazy way to avoid interdependence of the grid
+ * and searching data structures.
+ */
+#define NBNXN_NA_SC_MAX (GPU_NSUBCELL*NBNXN_GPU_CLUSTER_SIZE)
+
+
+static void nbs_cycle_clear(nbnxn_cycle_t *cc)
+{
+ int i;
+
+ for (i = 0; i < enbsCCnr; i++)
+ {
+ cc[i].count = 0;
+ cc[i].c = 0;
+ }
+}
+
+static double Mcyc_av(const nbnxn_cycle_t *cc)
+{
+ return (double)cc->c*1e-6/cc->count;
+}
+
+static void nbs_cycle_print(FILE *fp, const nbnxn_search_t nbs)
+{
+ int n;
+ int t;
+
+ fprintf(fp, "\n");
+ fprintf(fp, "ns %4d grid %4.1f search %4.1f red.f %5.3f",
+ nbs->cc[enbsCCgrid].count,
+ Mcyc_av(&nbs->cc[enbsCCgrid]),
+ Mcyc_av(&nbs->cc[enbsCCsearch]),
+ Mcyc_av(&nbs->cc[enbsCCreducef]));
+
+ if (nbs->nthread_max > 1)
+ {
+ if (nbs->cc[enbsCCcombine].count > 0)
+ {
+ fprintf(fp, " comb %5.2f",
+ Mcyc_av(&nbs->cc[enbsCCcombine]));
+ }
+ fprintf(fp, " s. th");
+ for (t = 0; t < nbs->nthread_max; t++)
+ {
+ fprintf(fp, " %4.1f",
+ Mcyc_av(&nbs->work[t].cc[enbsCCsearch]));
+ }
+ }
+ fprintf(fp, "\n");
+}
+
+static void nbnxn_grid_init(nbnxn_grid_t * grid)
+{
+ grid->cxy_na = NULL;
+ grid->cxy_ind = NULL;
+ grid->cxy_nalloc = 0;
+ grid->bb = NULL;
+ grid->bbj = NULL;
+ grid->nc_nalloc = 0;
+}
+
+static int get_2log(int n)
+{
+ int log2;
+
+ log2 = 0;
+ while ((1<<log2) < n)
+ {
+ log2++;
+ }
+ if ((1<<log2) != n)
+ {
+ gmx_fatal(FARGS, "nbnxn na_c (%d) is not a power of 2", n);
+ }
+
+ return log2;
+}
+
+static int nbnxn_kernel_to_ci_size(int nb_kernel_type)
+{
+ switch (nb_kernel_type)
+ {
+ case nbnxnk4x4_PlainC:
+ case nbnxnk4xN_SIMD_4xN:
+ case nbnxnk4xN_SIMD_2xNN:
+ return NBNXN_CPU_CLUSTER_I_SIZE;
+ case nbnxnk8x8x8_CUDA:
+ case nbnxnk8x8x8_PlainC:
+ /* The cluster size for super/sub lists is only set here.
+ * Any value should work for the pair-search and atomdata code.
+ * The kernels, of course, might require a particular value.
+ */
+ return NBNXN_GPU_CLUSTER_SIZE;
+ default:
+ gmx_incons("unknown kernel type");
+ }
+
+ return 0;
+}
+
+int nbnxn_kernel_to_cj_size(int nb_kernel_type)
+{
+ int nbnxn_simd_width = 0;
+ int cj_size = 0;
+
+#ifdef GMX_NBNXN_SIMD
+ nbnxn_simd_width = GMX_SIMD_WIDTH_HERE;
+#endif
+
+ switch (nb_kernel_type)
+ {
+ case nbnxnk4x4_PlainC:
+ cj_size = NBNXN_CPU_CLUSTER_I_SIZE;
+ break;
+ case nbnxnk4xN_SIMD_4xN:
+ cj_size = nbnxn_simd_width;
+ break;
+ case nbnxnk4xN_SIMD_2xNN:
+ cj_size = nbnxn_simd_width/2;
+ break;
+ case nbnxnk8x8x8_CUDA:
+ case nbnxnk8x8x8_PlainC:
+ cj_size = nbnxn_kernel_to_ci_size(nb_kernel_type);
+ break;
+ default:
+ gmx_incons("unknown kernel type");
+ }
+
+ return cj_size;
+}
+
+static int ci_to_cj(int na_cj_2log, int ci)
+{
+ switch (na_cj_2log)
+ {
+ case 2: return ci; break;
+ case 1: return (ci<<1); break;
+ case 3: return (ci>>1); break;
+ }
+
+ return 0;
+}
+
+gmx_bool nbnxn_kernel_pairlist_simple(int nb_kernel_type)
+{
+ if (nb_kernel_type == nbnxnkNotSet)
+ {
+ gmx_fatal(FARGS, "Non-bonded kernel type not set for Verlet-style pair-list.");
+ }
+
+ switch (nb_kernel_type)
+ {
+ case nbnxnk8x8x8_CUDA:
+ case nbnxnk8x8x8_PlainC:
+ return FALSE;
+
+ case nbnxnk4x4_PlainC:
+ case nbnxnk4xN_SIMD_4xN:
+ case nbnxnk4xN_SIMD_2xNN:
+ return TRUE;
+
+ default:
+ gmx_incons("Invalid nonbonded kernel type passed!");
+ return FALSE;
+ }
+}
+
+void nbnxn_init_search(nbnxn_search_t * nbs_ptr,
+ ivec *n_dd_cells,
+ gmx_domdec_zones_t *zones,
+ int nthread_max)
+{
+ nbnxn_search_t nbs;
+ int d, g, t;
+
+ snew(nbs, 1);
+ *nbs_ptr = nbs;
+
+ nbs->DomDec = (n_dd_cells != NULL);
+
+ clear_ivec(nbs->dd_dim);
+ nbs->ngrid = 1;
+ if (nbs->DomDec)
+ {
+ nbs->zones = zones;
+
+ for (d = 0; d < DIM; d++)
+ {
+ if ((*n_dd_cells)[d] > 1)
+ {
+ nbs->dd_dim[d] = 1;
+ /* Each grid matches a DD zone */
+ nbs->ngrid *= 2;
+ }
+ }
+ }
+
+ snew(nbs->grid, nbs->ngrid);
+ for (g = 0; g < nbs->ngrid; g++)
+ {
+ nbnxn_grid_init(&nbs->grid[g]);
+ }
+ nbs->cell = NULL;
+ nbs->cell_nalloc = 0;
+ nbs->a = NULL;
+ nbs->a_nalloc = 0;
+
+ nbs->nthread_max = nthread_max;
+
+ /* Initialize the work data structures for each thread */
+ snew(nbs->work, nbs->nthread_max);
+ for (t = 0; t < nbs->nthread_max; t++)
+ {
+ nbs->work[t].cxy_na = NULL;
+ nbs->work[t].cxy_na_nalloc = 0;
+ nbs->work[t].sort_work = NULL;
+ nbs->work[t].sort_work_nalloc = 0;
+ }
+
+ /* Initialize detailed nbsearch cycle counting */
+ nbs->print_cycles = (getenv("GMX_NBNXN_CYCLE") != 0);
+ nbs->search_count = 0;
+ nbs_cycle_clear(nbs->cc);
+ for (t = 0; t < nbs->nthread_max; t++)
+ {
+ nbs_cycle_clear(nbs->work[t].cc);
+ }
+}
+
+static real grid_atom_density(int n, rvec corner0, rvec corner1)
+{
+ rvec size;
+
+ rvec_sub(corner1, corner0, size);
+
+ return n/(size[XX]*size[YY]*size[ZZ]);
+}
+
+static int set_grid_size_xy(const nbnxn_search_t nbs,
+ nbnxn_grid_t *grid,
+ int dd_zone,
+ int n, rvec corner0, rvec corner1,
+ real atom_density)
+{
+ rvec size;
+ int na_c;
+ real adens, tlen, tlen_x, tlen_y, nc_max;
+ int t;
+
+ rvec_sub(corner1, corner0, size);
+
+ if (n > grid->na_sc)
+ {
+ /* target cell length */
+ if (grid->bSimple)
+ {
+ /* To minimize the zero interactions, we should make
+ * the largest of the i/j cell cubic.
+ */
+ na_c = max(grid->na_c, grid->na_cj);
+
+ /* Approximately cubic cells */
+ tlen = pow(na_c/atom_density, 1.0/3.0);
+ tlen_x = tlen;
+ tlen_y = tlen;
+ }
+ else
+ {
+ /* Approximately cubic sub cells */
+ tlen = pow(grid->na_c/atom_density, 1.0/3.0);
+ tlen_x = tlen*GPU_NSUBCELL_X;
+ tlen_y = tlen*GPU_NSUBCELL_Y;
+ }
+ /* We round ncx and ncy down, because we get less cell pairs
+ * in the nbsist when the fixed cell dimensions (x,y) are
+ * larger than the variable one (z) than the other way around.
+ */
+ grid->ncx = max(1, (int)(size[XX]/tlen_x));
+ grid->ncy = max(1, (int)(size[YY]/tlen_y));
+ }
+ else
+ {
+ grid->ncx = 1;
+ grid->ncy = 1;
+ }
+
+ grid->sx = size[XX]/grid->ncx;
+ grid->sy = size[YY]/grid->ncy;
+ grid->inv_sx = 1/grid->sx;
+ grid->inv_sy = 1/grid->sy;
+
+ if (dd_zone > 0)
+ {
+ /* This is a non-home zone, add an extra row of cells
+ * for particles communicated for bonded interactions.
+ * These can be beyond the cut-off. It doesn't matter where
+ * they end up on the grid, but for performance it's better
+ * if they don't end up in cells that can be within cut-off range.
+ */
+ grid->ncx++;
+ grid->ncy++;
+ }
+
+ /* We need one additional cell entry for particles moved by DD */
+ if (grid->ncx*grid->ncy+1 > grid->cxy_nalloc)
+ {
+ grid->cxy_nalloc = over_alloc_large(grid->ncx*grid->ncy+1);
+ srenew(grid->cxy_na, grid->cxy_nalloc);
+ srenew(grid->cxy_ind, grid->cxy_nalloc+1);
+ }
+ for (t = 0; t < nbs->nthread_max; t++)
+ {
+ if (grid->ncx*grid->ncy+1 > nbs->work[t].cxy_na_nalloc)
+ {
+ nbs->work[t].cxy_na_nalloc = over_alloc_large(grid->ncx*grid->ncy+1);
+ srenew(nbs->work[t].cxy_na, nbs->work[t].cxy_na_nalloc);
+ }
+ }
+
+ /* Worst case scenario of 1 atom in each last cell */
+ if (grid->na_cj <= grid->na_c)
+ {
+ nc_max = n/grid->na_sc + grid->ncx*grid->ncy;
+ }
+ else
+ {
+ nc_max = n/grid->na_sc + grid->ncx*grid->ncy*grid->na_cj/grid->na_c;
+ }
+
+ if (nc_max > grid->nc_nalloc)
+ {
+ grid->nc_nalloc = over_alloc_large(nc_max);
+ srenew(grid->nsubc, grid->nc_nalloc);
+ srenew(grid->bbcz, grid->nc_nalloc*NNBSBB_D);
+
+ sfree_aligned(grid->bb);
+ /* This snew also zeros the contents, this avoid possible
- #ifdef NBNXN_SEARCH_BB_SSE
- _mm_store_ps(&bbj[1].lower[0], _mm_load_ps(&bbj[0].lower[0]));
- _mm_store_ps(&bbj[1].upper[0], _mm_load_ps(&bbj[0].upper[0]));
++ * floating exceptions in SIMD with the unused bb elements.
+ */
+ if (grid->bSimple)
+ {
+ snew_aligned(grid->bb, grid->nc_nalloc, 16);
+ }
+ else
+ {
+#ifdef NBNXN_BBXXXX
+ int pbb_nalloc;
+
+ pbb_nalloc = grid->nc_nalloc*GPU_NSUBCELL/STRIDE_PBB*NNBSBB_XXXX;
+ snew_aligned(grid->pbb, pbb_nalloc, 16);
+#else
+ snew_aligned(grid->bb, grid->nc_nalloc*GPU_NSUBCELL, 16);
+#endif
+ }
+
+ if (grid->bSimple)
+ {
+ if (grid->na_cj == grid->na_c)
+ {
+ grid->bbj = grid->bb;
+ }
+ else
+ {
+ sfree_aligned(grid->bbj);
+ snew_aligned(grid->bbj, grid->nc_nalloc*grid->na_c/grid->na_cj, 16);
+ }
+ }
+
+ srenew(grid->flags, grid->nc_nalloc);
+ }
+
+ copy_rvec(corner0, grid->c0);
+ copy_rvec(corner1, grid->c1);
+
+ return nc_max;
+}
+
+/* We need to sort paricles in grid columns on z-coordinate.
+ * As particle are very often distributed homogeneously, we a sorting
+ * algorithm similar to pigeonhole sort. We multiply the z-coordinate
+ * by a factor, cast to an int and try to store in that hole. If the hole
+ * is full, we move this or another particle. A second pass is needed to make
+ * contiguous elements. SORT_GRID_OVERSIZE is the ratio of holes to particles.
+ * 4 is the optimal value for homogeneous particle distribution and allows
+ * for an O(#particles) sort up till distributions were all particles are
+ * concentrated in 1/4 of the space. No NlogN fallback is implemented,
+ * as it can be expensive to detect imhomogeneous particle distributions.
+ * SGSF is the maximum ratio of holes used, in the worst case all particles
+ * end up in the last hole and we need #particles extra holes at the end.
+ */
+#define SORT_GRID_OVERSIZE 4
+#define SGSF (SORT_GRID_OVERSIZE + 1)
+
+/* Sort particle index a on coordinates x along dim.
+ * Backwards tells if we want decreasing iso increasing coordinates.
+ * h0 is the minimum of the coordinate range.
+ * invh is the 1/length of the sorting range.
+ * n_per_h (>=n) is the expected average number of particles per 1/invh
+ * sort is the sorting work array.
+ * sort should have a size of at least n_per_h*SORT_GRID_OVERSIZE + n,
+ * or easier, allocate at least n*SGSF elements.
+ */
+static void sort_atoms(int dim, gmx_bool Backwards,
+ int *a, int n, rvec *x,
+ real h0, real invh, int n_per_h,
+ int *sort)
+{
+ int nsort, i, c;
+ int zi, zim, zi_min, zi_max;
+ int cp, tmp;
+
+ if (n <= 1)
+ {
+ /* Nothing to do */
+ return;
+ }
+
+#ifndef NDEBUG
+ if (n > n_per_h)
+ {
+ gmx_incons("n > n_per_h");
+ }
+#endif
+
+ /* Transform the inverse range height into the inverse hole height */
+ invh *= n_per_h*SORT_GRID_OVERSIZE;
+
+ /* Set nsort to the maximum possible number of holes used.
+ * In worst case all n elements end up in the last bin.
+ */
+ nsort = n_per_h*SORT_GRID_OVERSIZE + n;
+
+ /* Determine the index range used, so we can limit it for the second pass */
+ zi_min = INT_MAX;
+ zi_max = -1;
+
+ /* Sort the particles using a simple index sort */
+ for (i = 0; i < n; i++)
+ {
+ /* The cast takes care of float-point rounding effects below zero.
+ * This code assumes particles are less than 1/SORT_GRID_OVERSIZE
+ * times the box height out of the box.
+ */
+ zi = (int)((x[a[i]][dim] - h0)*invh);
+
+#ifndef NDEBUG
+ /* As we can have rounding effect, we use > iso >= here */
+ if (zi < 0 || zi > n_per_h*SORT_GRID_OVERSIZE)
+ {
+ gmx_fatal(FARGS, "(int)((x[%d][%c]=%f - %f)*%f) = %d, not in 0 - %d*%d\n",
+ a[i], 'x'+dim, x[a[i]][dim], h0, invh, zi,
+ n_per_h, SORT_GRID_OVERSIZE);
+ }
+#endif
+
+ /* Ideally this particle should go in sort cell zi,
+ * but that might already be in use,
+ * in that case find the first empty cell higher up
+ */
+ if (sort[zi] < 0)
+ {
+ sort[zi] = a[i];
+ zi_min = min(zi_min, zi);
+ zi_max = max(zi_max, zi);
+ }
+ else
+ {
+ /* We have multiple atoms in the same sorting slot.
+ * Sort on real z for minimal bounding box size.
+ * There is an extra check for identical z to ensure
+ * well-defined output order, independent of input order
+ * to ensure binary reproducibility after restarts.
+ */
+ while (sort[zi] >= 0 && ( x[a[i]][dim] > x[sort[zi]][dim] ||
+ (x[a[i]][dim] == x[sort[zi]][dim] &&
+ a[i] > sort[zi])))
+ {
+ zi++;
+ }
+
+ if (sort[zi] >= 0)
+ {
+ /* Shift all elements by one slot until we find an empty slot */
+ cp = sort[zi];
+ zim = zi + 1;
+ while (sort[zim] >= 0)
+ {
+ tmp = sort[zim];
+ sort[zim] = cp;
+ cp = tmp;
+ zim++;
+ }
+ sort[zim] = cp;
+ zi_max = max(zi_max, zim);
+ }
+ sort[zi] = a[i];
+ zi_max = max(zi_max, zi);
+ }
+ }
+
+ c = 0;
+ if (!Backwards)
+ {
+ for (zi = 0; zi < nsort; zi++)
+ {
+ if (sort[zi] >= 0)
+ {
+ a[c++] = sort[zi];
+ sort[zi] = -1;
+ }
+ }
+ }
+ else
+ {
+ for (zi = zi_max; zi >= zi_min; zi--)
+ {
+ if (sort[zi] >= 0)
+ {
+ a[c++] = sort[zi];
+ sort[zi] = -1;
+ }
+ }
+ }
+ if (c < n)
+ {
+ gmx_incons("Lost particles while sorting");
+ }
+}
+
+#ifdef GMX_DOUBLE
+#define R2F_D(x) ((float)((x) >= 0 ? ((1-GMX_FLOAT_EPS)*(x)) : ((1+GMX_FLOAT_EPS)*(x))))
+#define R2F_U(x) ((float)((x) >= 0 ? ((1+GMX_FLOAT_EPS)*(x)) : ((1-GMX_FLOAT_EPS)*(x))))
+#else
+#define R2F_D(x) (x)
+#define R2F_U(x) (x)
+#endif
+
+/* Coordinate order x,y,z, bb order xyz0 */
+static void calc_bounding_box(int na, int stride, const real *x, nbnxn_bb_t *bb)
+{
+ int i, j;
+ real xl, xh, yl, yh, zl, zh;
+
+ i = 0;
+ xl = x[i+XX];
+ xh = x[i+XX];
+ yl = x[i+YY];
+ yh = x[i+YY];
+ zl = x[i+ZZ];
+ zh = x[i+ZZ];
+ i += stride;
+ for (j = 1; j < na; j++)
+ {
+ xl = min(xl, x[i+XX]);
+ xh = max(xh, x[i+XX]);
+ yl = min(yl, x[i+YY]);
+ yh = max(yh, x[i+YY]);
+ zl = min(zl, x[i+ZZ]);
+ zh = max(zh, x[i+ZZ]);
+ i += stride;
+ }
+ /* Note: possible double to float conversion here */
+ bb->lower[BB_X] = R2F_D(xl);
+ bb->lower[BB_Y] = R2F_D(yl);
+ bb->lower[BB_Z] = R2F_D(zl);
+ bb->upper[BB_X] = R2F_U(xh);
+ bb->upper[BB_Y] = R2F_U(yh);
+ bb->upper[BB_Z] = R2F_U(zh);
+}
+
+/* Packed coordinates, bb order xyz0 */
+static void calc_bounding_box_x_x4(int na, const real *x, nbnxn_bb_t *bb)
+{
+ int j;
+ real xl, xh, yl, yh, zl, zh;
+
+ xl = x[XX*PACK_X4];
+ xh = x[XX*PACK_X4];
+ yl = x[YY*PACK_X4];
+ yh = x[YY*PACK_X4];
+ zl = x[ZZ*PACK_X4];
+ zh = x[ZZ*PACK_X4];
+ for (j = 1; j < na; j++)
+ {
+ xl = min(xl, x[j+XX*PACK_X4]);
+ xh = max(xh, x[j+XX*PACK_X4]);
+ yl = min(yl, x[j+YY*PACK_X4]);
+ yh = max(yh, x[j+YY*PACK_X4]);
+ zl = min(zl, x[j+ZZ*PACK_X4]);
+ zh = max(zh, x[j+ZZ*PACK_X4]);
+ }
+ /* Note: possible double to float conversion here */
+ bb->lower[BB_X] = R2F_D(xl);
+ bb->lower[BB_Y] = R2F_D(yl);
+ bb->lower[BB_Z] = R2F_D(zl);
+ bb->upper[BB_X] = R2F_U(xh);
+ bb->upper[BB_Y] = R2F_U(yh);
+ bb->upper[BB_Z] = R2F_U(zh);
+}
+
+/* Packed coordinates, bb order xyz0 */
+static void calc_bounding_box_x_x8(int na, const real *x, nbnxn_bb_t *bb)
+{
+ int j;
+ real xl, xh, yl, yh, zl, zh;
+
+ xl = x[XX*PACK_X8];
+ xh = x[XX*PACK_X8];
+ yl = x[YY*PACK_X8];
+ yh = x[YY*PACK_X8];
+ zl = x[ZZ*PACK_X8];
+ zh = x[ZZ*PACK_X8];
+ for (j = 1; j < na; j++)
+ {
+ xl = min(xl, x[j+XX*PACK_X8]);
+ xh = max(xh, x[j+XX*PACK_X8]);
+ yl = min(yl, x[j+YY*PACK_X8]);
+ yh = max(yh, x[j+YY*PACK_X8]);
+ zl = min(zl, x[j+ZZ*PACK_X8]);
+ zh = max(zh, x[j+ZZ*PACK_X8]);
+ }
+ /* Note: possible double to float conversion here */
+ bb->lower[BB_X] = R2F_D(xl);
+ bb->lower[BB_Y] = R2F_D(yl);
+ bb->lower[BB_Z] = R2F_D(zl);
+ bb->upper[BB_X] = R2F_U(xh);
+ bb->upper[BB_Y] = R2F_U(yh);
+ bb->upper[BB_Z] = R2F_U(zh);
+}
+
+/* Packed coordinates, bb order xyz0 */
+static void calc_bounding_box_x_x4_halves(int na, const real *x,
+ nbnxn_bb_t *bb, nbnxn_bb_t *bbj)
+{
+ calc_bounding_box_x_x4(min(na, 2), x, bbj);
+
+ if (na > 2)
+ {
+ calc_bounding_box_x_x4(min(na-2, 2), x+(PACK_X4>>1), bbj+1);
+ }
+ else
+ {
+ /* Set the "empty" bounding box to the same as the first one,
+ * so we don't need to treat special cases in the rest of the code.
+ */
- #ifdef NBNXN_SEARCH_BB_SSE
- _mm_store_ps(&bb->lower[0], _mm_min_ps(_mm_load_ps(&bbj[0].lower[0]),
- _mm_load_ps(&bbj[1].lower[0])));
- _mm_store_ps(&bb->upper[0], _mm_max_ps(_mm_load_ps(&bbj[0].upper[0]),
- _mm_load_ps(&bbj[1].upper[0])));
++#ifdef NBNXN_SEARCH_BB_SIMD4
++ gmx_simd4_store_pr(&bbj[1].lower[0], gmx_simd4_load_bb_pr(&bbj[0].lower[0]));
++ gmx_simd4_store_pr(&bbj[1].upper[0], gmx_simd4_load_bb_pr(&bbj[0].upper[0]));
+#else
+ bbj[1] = bbj[0];
+#endif
+ }
+
- #ifdef NBNXN_SEARCH_BB_SSE
++#ifdef NBNXN_SEARCH_BB_SIMD4
++ gmx_simd4_store_pr(&bb->lower[0],
++ gmx_simd4_min_pr(gmx_simd4_load_bb_pr(&bbj[0].lower[0]),
++ gmx_simd4_load_bb_pr(&bbj[1].lower[0])));
++ gmx_simd4_store_pr(&bb->upper[0],
++ gmx_simd4_max_pr(gmx_simd4_load_bb_pr(&bbj[0].upper[0]),
++ gmx_simd4_load_bb_pr(&bbj[1].upper[0])));
+#else
+ {
+ int i;
+
+ for (i = 0; i < NNBSBB_C; i++)
+ {
+ bb->lower[i] = min(bbj[0].lower[i], bbj[1].lower[i]);
+ bb->upper[i] = max(bbj[0].upper[i], bbj[1].upper[i]);
+ }
+ }
+#endif
+}
+
- #endif /* NBNXN_SEARCH_BB_SSE */
++#ifdef NBNXN_SEARCH_BB_SIMD4
+
+/* Coordinate order xyz, bb order xxxxyyyyzzzz */
+static void calc_bounding_box_xxxx(int na, int stride, const real *x, float *bb)
+{
+ int i, j;
+ real xl, xh, yl, yh, zl, zh;
+
+ i = 0;
+ xl = x[i+XX];
+ xh = x[i+XX];
+ yl = x[i+YY];
+ yh = x[i+YY];
+ zl = x[i+ZZ];
+ zh = x[i+ZZ];
+ i += stride;
+ for (j = 1; j < na; j++)
+ {
+ xl = min(xl, x[i+XX]);
+ xh = max(xh, x[i+XX]);
+ yl = min(yl, x[i+YY]);
+ yh = max(yh, x[i+YY]);
+ zl = min(zl, x[i+ZZ]);
+ zh = max(zh, x[i+ZZ]);
+ i += stride;
+ }
+ /* Note: possible double to float conversion here */
+ bb[0*STRIDE_PBB] = R2F_D(xl);
+ bb[1*STRIDE_PBB] = R2F_D(yl);
+ bb[2*STRIDE_PBB] = R2F_D(zl);
+ bb[3*STRIDE_PBB] = R2F_U(xh);
+ bb[4*STRIDE_PBB] = R2F_U(yh);
+ bb[5*STRIDE_PBB] = R2F_U(zh);
+}
+
- #ifdef NBNXN_SEARCH_SSE_SINGLE
++#endif /* NBNXN_SEARCH_BB_SIMD4 */
+
- static void calc_bounding_box_sse(int na, const float *x, nbnxn_bb_t *bb)
++#ifdef NBNXN_SEARCH_SIMD4_FLOAT_X_BB
+
+/* Coordinate order xyz?, bb order xyz0 */
- __m128 bb_0_SSE, bb_1_SSE;
- __m128 x_SSE;
++static void calc_bounding_box_simd4(int na, const float *x, nbnxn_bb_t *bb)
+{
- bb_0_SSE = _mm_load_ps(x);
- bb_1_SSE = bb_0_SSE;
++ gmx_simd4_pr bb_0_S, bb_1_S;
++ gmx_simd4_pr x_S;
+
+ int i;
+
- x_SSE = _mm_load_ps(x+i*NNBSBB_C);
- bb_0_SSE = _mm_min_ps(bb_0_SSE, x_SSE);
- bb_1_SSE = _mm_max_ps(bb_1_SSE, x_SSE);
++ bb_0_S = gmx_simd4_load_bb_pr(x);
++ bb_1_S = bb_0_S;
+
+ for (i = 1; i < na; i++)
+ {
- _mm_store_ps(&bb->lower[0], bb_0_SSE);
- _mm_store_ps(&bb->upper[0], bb_1_SSE);
++ x_S = gmx_simd4_load_bb_pr(x+i*NNBSBB_C);
++ bb_0_S = gmx_simd4_min_pr(bb_0_S, x_S);
++ bb_1_S = gmx_simd4_max_pr(bb_1_S, x_S);
+ }
+
- static void calc_bounding_box_xxxx_sse(int na, const float *x,
- nbnxn_bb_t *bb_work_aligned,
- real *bb)
++ gmx_simd4_store_pr(&bb->lower[0], bb_0_S);
++ gmx_simd4_store_pr(&bb->upper[0], bb_1_S);
+}
+
+/* Coordinate order xyz?, bb order xxxxyyyyzzzz */
- calc_bounding_box_sse(na, x, bb_work_aligned);
++static void calc_bounding_box_xxxx_simd4(int na, const float *x,
++ nbnxn_bb_t *bb_work_aligned,
++ real *bb)
+{
- #endif /* NBNXN_SEARCH_SSE_SINGLE */
++ calc_bounding_box_simd4(na, x, bb_work_aligned);
+
+ bb[0*STRIDE_PBB] = bb_work_aligned->lower[BB_X];
+ bb[1*STRIDE_PBB] = bb_work_aligned->lower[BB_Y];
+ bb[2*STRIDE_PBB] = bb_work_aligned->lower[BB_Z];
+ bb[3*STRIDE_PBB] = bb_work_aligned->upper[BB_X];
+ bb[4*STRIDE_PBB] = bb_work_aligned->upper[BB_Y];
+ bb[5*STRIDE_PBB] = bb_work_aligned->upper[BB_Z];
+}
+
- #ifdef NBNXN_SEARCH_BB_SSE
- __m128 min_SSE, max_SSE;
-
- min_SSE = _mm_min_ps(_mm_load_ps(&bb[c2*2+0].lower[0]),
- _mm_load_ps(&bb[c2*2+1].lower[0]));
- max_SSE = _mm_max_ps(_mm_load_ps(&bb[c2*2+0].upper[0]),
- _mm_load_ps(&bb[c2*2+1].upper[0]));
- _mm_store_ps(&grid->bbj[c2].lower[0], min_SSE);
- _mm_store_ps(&grid->bbj[c2].upper[0], max_SSE);
++#endif /* NBNXN_SEARCH_SIMD4_FLOAT_X_BB */
+
+
+/* Combines pairs of consecutive bounding boxes */
+static void combine_bounding_box_pairs(nbnxn_grid_t *grid, const nbnxn_bb_t *bb)
+{
+ int i, j, sc2, nc2, c2;
+
+ for (i = 0; i < grid->ncx*grid->ncy; i++)
+ {
+ /* Starting bb in a column is expected to be 2-aligned */
+ sc2 = grid->cxy_ind[i]>>1;
+ /* For odd numbers skip the last bb here */
+ nc2 = (grid->cxy_na[i]+3)>>(2+1);
+ for (c2 = sc2; c2 < sc2+nc2; c2++)
+ {
- grid->bbj[c2].lower[j] = min(bb[c2*2].lower[j],
- bb[c2*2].lower[j]);
- grid->bbj[c2].upper[j] = max(bb[c2*2].upper[j],
- bb[c2*2].upper[j]);
++#ifdef NBNXN_SEARCH_BB_SIMD4
++ gmx_simd4_pr min_S, max_S;
++
++ min_S = gmx_simd4_min_pr(gmx_simd4_load_bb_pr(&bb[c2*2+0].lower[0]),
++ gmx_simd4_load_bb_pr(&bb[c2*2+1].lower[0]));
++ max_S = gmx_simd4_max_pr(gmx_simd4_load_bb_pr(&bb[c2*2+0].upper[0]),
++ gmx_simd4_load_bb_pr(&bb[c2*2+1].upper[0]));
++ gmx_simd4_store_pr(&grid->bbj[c2].lower[0], min_S);
++ gmx_simd4_store_pr(&grid->bbj[c2].upper[0], max_S);
+#else
+ for (j = 0; j < NNBSBB_C; j++)
+ {
- /* Copy the last bb for odd bb count in this column */
++ grid->bbj[c2].lower[j] = min(bb[c2*2+0].lower[j],
++ bb[c2*2+1].lower[j]);
++ grid->bbj[c2].upper[j] = max(bb[c2*2+0].upper[j],
++ bb[c2*2+1].upper[j]);
+ }
+#endif
+ }
+ if (((grid->cxy_na[i]+3)>>2) & 1)
+ {
- * for SSE calculations: xxxxyyyyzzzz...
++ /* The bb count in this column is odd: duplicate the last bb */
+ for (j = 0; j < NNBSBB_C; j++)
+ {
+ grid->bbj[c2].lower[j] = bb[c2*2].lower[j];
+ grid->bbj[c2].upper[j] = bb[c2*2].upper[j];
+ }
+ }
+ }
+}
+
+
+/* Prints the average bb size, used for debug output */
+static void print_bbsizes_simple(FILE *fp,
+ const nbnxn_search_t nbs,
+ const nbnxn_grid_t *grid)
+{
+ int c, d;
+ dvec ba;
+
+ clear_dvec(ba);
+ for (c = 0; c < grid->nc; c++)
+ {
+ for (d = 0; d < DIM; d++)
+ {
+ ba[d] += grid->bb[c].upper[d] - grid->bb[c].lower[d];
+ }
+ }
+ dsvmul(1.0/grid->nc, ba, ba);
+
+ fprintf(fp, "ns bb: %4.2f %4.2f %4.2f %4.2f %4.2f %4.2f rel %4.2f %4.2f %4.2f\n",
+ nbs->box[XX][XX]/grid->ncx,
+ nbs->box[YY][YY]/grid->ncy,
+ nbs->box[ZZ][ZZ]*grid->ncx*grid->ncy/grid->nc,
+ ba[XX], ba[YY], ba[ZZ],
+ ba[XX]*grid->ncx/nbs->box[XX][XX],
+ ba[YY]*grid->ncy/nbs->box[YY][YY],
+ ba[ZZ]*grid->nc/(grid->ncx*grid->ncy*nbs->box[ZZ][ZZ]));
+}
+
+/* Prints the average bb size, used for debug output */
+static void print_bbsizes_supersub(FILE *fp,
+ const nbnxn_search_t nbs,
+ const nbnxn_grid_t *grid)
+{
+ int ns, c, s;
+ dvec ba;
+
+ clear_dvec(ba);
+ ns = 0;
+ for (c = 0; c < grid->nc; c++)
+ {
+#ifdef NBNXN_BBXXXX
+ for (s = 0; s < grid->nsubc[c]; s += STRIDE_PBB)
+ {
+ int cs_w, i, d;
+
+ cs_w = (c*GPU_NSUBCELL + s)/STRIDE_PBB;
+ for (i = 0; i < STRIDE_PBB; i++)
+ {
+ for (d = 0; d < DIM; d++)
+ {
+ ba[d] +=
+ grid->pbb[cs_w*NNBSBB_XXXX+(DIM+d)*STRIDE_PBB+i] -
+ grid->pbb[cs_w*NNBSBB_XXXX+ d *STRIDE_PBB+i];
+ }
+ }
+ }
+#else
+ for (s = 0; s < grid->nsubc[c]; s++)
+ {
+ int cs, d;
+
+ cs = c*GPU_NSUBCELL + s;
+ for (d = 0; d < DIM; d++)
+ {
+ ba[d] += grid->bb[cs].upper[d] - grid->bb[cs].lower[d];
+ }
+ }
+#endif
+ ns += grid->nsubc[c];
+ }
+ dsvmul(1.0/ns, ba, ba);
+
+ fprintf(fp, "ns bb: %4.2f %4.2f %4.2f %4.2f %4.2f %4.2f rel %4.2f %4.2f %4.2f\n",
+ nbs->box[XX][XX]/(grid->ncx*GPU_NSUBCELL_X),
+ nbs->box[YY][YY]/(grid->ncy*GPU_NSUBCELL_Y),
+ nbs->box[ZZ][ZZ]*grid->ncx*grid->ncy/(grid->nc*GPU_NSUBCELL_Z),
+ ba[XX], ba[YY], ba[ZZ],
+ ba[XX]*grid->ncx*GPU_NSUBCELL_X/nbs->box[XX][XX],
+ ba[YY]*grid->ncy*GPU_NSUBCELL_Y/nbs->box[YY][YY],
+ ba[ZZ]*grid->nc*GPU_NSUBCELL_Z/(grid->ncx*grid->ncy*nbs->box[ZZ][ZZ]));
+}
+
+/* Potentially sorts atoms on LJ coefficients !=0 and ==0.
+ * Also sets interaction flags.
+ */
+void sort_on_lj(int na_c,
+ int a0, int a1, const int *atinfo,
+ int *order,
+ int *flags)
+{
+ int subc, s, a, n1, n2, a_lj_max, i, j;
+ int sort1[NBNXN_NA_SC_MAX/GPU_NSUBCELL];
+ int sort2[NBNXN_NA_SC_MAX/GPU_NSUBCELL];
+ gmx_bool haveQ;
+
+ *flags = 0;
+
+ subc = 0;
+ for (s = a0; s < a1; s += na_c)
+ {
+ /* Make lists for this (sub-)cell on atoms with and without LJ */
+ n1 = 0;
+ n2 = 0;
+ haveQ = FALSE;
+ a_lj_max = -1;
+ for (a = s; a < min(s+na_c, a1); a++)
+ {
+ haveQ = haveQ || GET_CGINFO_HAS_Q(atinfo[order[a]]);
+
+ if (GET_CGINFO_HAS_VDW(atinfo[order[a]]))
+ {
+ sort1[n1++] = order[a];
+ a_lj_max = a;
+ }
+ else
+ {
+ sort2[n2++] = order[a];
+ }
+ }
+
+ /* If we don't have atom with LJ, there's nothing to sort */
+ if (n1 > 0)
+ {
+ *flags |= NBNXN_CI_DO_LJ(subc);
+
+ if (2*n1 <= na_c)
+ {
+ /* Only sort when strictly necessary. Ordering particles
+ * Ordering particles can lead to less accurate summation
+ * due to rounding, both for LJ and Coulomb interactions.
+ */
+ if (2*(a_lj_max - s) >= na_c)
+ {
+ for (i = 0; i < n1; i++)
+ {
+ order[a0+i] = sort1[i];
+ }
+ for (j = 0; j < n2; j++)
+ {
+ order[a0+n1+j] = sort2[j];
+ }
+ }
+
+ *flags |= NBNXN_CI_HALF_LJ(subc);
+ }
+ }
+ if (haveQ)
+ {
+ *flags |= NBNXN_CI_DO_COUL(subc);
+ }
+ subc++;
+ }
+}
+
+/* Fill a pair search cell with atoms.
+ * Potentially sorts atoms and sets the interaction flags.
+ */
+void fill_cell(const nbnxn_search_t nbs,
+ nbnxn_grid_t *grid,
+ nbnxn_atomdata_t *nbat,
+ int a0, int a1,
+ const int *atinfo,
+ rvec *x,
+ int sx, int sy, int sz,
+ nbnxn_bb_t *bb_work_aligned)
+{
+ int na, a;
+ size_t offset;
+ nbnxn_bb_t *bb_ptr;
+#ifdef NBNXN_BBXXXX
+ float *pbb_ptr;
+#endif
+
+ na = a1 - a0;
+
+ if (grid->bSimple)
+ {
+ sort_on_lj(grid->na_c, a0, a1, atinfo, nbs->a,
+ grid->flags+(a0>>grid->na_c_2log)-grid->cell0);
+ }
+
+ /* Now we have sorted the atoms, set the cell indices */
+ for (a = a0; a < a1; a++)
+ {
+ nbs->cell[nbs->a[a]] = a;
+ }
+
+ copy_rvec_to_nbat_real(nbs->a+a0, a1-a0, grid->na_c, x,
+ nbat->XFormat, nbat->x, a0,
+ sx, sy, sz);
+
+ if (nbat->XFormat == nbatX4)
+ {
+ /* Store the bounding boxes as xyz.xyz. */
+ offset = (a0 - grid->cell0*grid->na_sc) >> grid->na_c_2log;
+ bb_ptr = grid->bb + offset;
+
+#if defined GMX_NBNXN_SIMD && GMX_SIMD_WIDTH_HERE == 2
+ if (2*grid->na_cj == grid->na_c)
+ {
+ calc_bounding_box_x_x4_halves(na, nbat->x+X4_IND_A(a0), bb_ptr,
+ grid->bbj+offset*2);
+ }
+ else
+#endif
+ {
+ calc_bounding_box_x_x4(na, nbat->x+X4_IND_A(a0), bb_ptr);
+ }
+ }
+ else if (nbat->XFormat == nbatX8)
+ {
+ /* Store the bounding boxes as xyz.xyz. */
+ offset = (a0 - grid->cell0*grid->na_sc) >> grid->na_c_2log;
+ bb_ptr = grid->bb + offset;
+
+ calc_bounding_box_x_x8(na, nbat->x+X8_IND_A(a0), bb_ptr);
+ }
+#ifdef NBNXN_BBXXXX
+ else if (!grid->bSimple)
+ {
+ /* Store the bounding boxes in a format convenient
- #ifdef NBNXN_SEARCH_SSE_SINGLE
++ * for SIMD4 calculations: xxxxyyyyzzzz...
+ */
+ pbb_ptr =
+ grid->pbb +
+ ((a0-grid->cell0*grid->na_sc)>>(grid->na_c_2log+STRIDE_PBB_2LOG))*NNBSBB_XXXX +
+ (((a0-grid->cell0*grid->na_sc)>>grid->na_c_2log) & (STRIDE_PBB-1));
+
- calc_bounding_box_xxxx_sse(na, nbat->x+a0*nbat->xstride,
- bb_work_aligned, pbb_ptr);
++#ifdef NBNXN_SEARCH_SIMD4_FLOAT_X_BB
+ if (nbat->XFormat == nbatXYZQ)
+ {
- #ifdef NBNXN_SEARCH_BB_SSE
++ calc_bounding_box_xxxx_simd4(na, nbat->x+a0*nbat->xstride,
++ bb_work_aligned, pbb_ptr);
+ }
+ else
+#endif
+ {
+ calc_bounding_box_xxxx(na, nbat->xstride, nbat->x+a0*nbat->xstride,
+ pbb_ptr);
+ }
+ if (gmx_debug_at)
+ {
+ fprintf(debug, "%2d %2d %2d bb %5.2f %5.2f %5.2f %5.2f %5.2f %5.2f\n",
+ sx, sy, sz,
+ pbb_ptr[0*STRIDE_PBB], pbb_ptr[3*STRIDE_PBB],
+ pbb_ptr[1*STRIDE_PBB], pbb_ptr[4*STRIDE_PBB],
+ pbb_ptr[2*STRIDE_PBB], pbb_ptr[5*STRIDE_PBB]);
+ }
+ }
+#endif
+ else
+ {
+ /* Store the bounding boxes as xyz.xyz. */
+ bb_ptr = grid->bb+((a0-grid->cell0*grid->na_sc)>>grid->na_c_2log);
+
+ calc_bounding_box(na, nbat->xstride, nbat->x+a0*nbat->xstride,
+ bb_ptr);
+
+ if (gmx_debug_at)
+ {
+ int bbo;
+ bbo = (a0 - grid->cell0*grid->na_sc)/grid->na_c;
+ fprintf(debug, "%2d %2d %2d bb %5.2f %5.2f %5.2f %5.2f %5.2f %5.2f\n",
+ sx, sy, sz,
+ grid->bb[bbo].lower[BB_X],
+ grid->bb[bbo].lower[BB_Y],
+ grid->bb[bbo].lower[BB_Z],
+ grid->bb[bbo].upper[BB_X],
+ grid->bb[bbo].upper[BB_Y],
+ grid->bb[bbo].upper[BB_Z]);
+ }
+ }
+}
+
+/* Spatially sort the atoms within one grid column */
+static void sort_columns_simple(const nbnxn_search_t nbs,
+ int dd_zone,
+ nbnxn_grid_t *grid,
+ int a0, int a1,
+ const int *atinfo,
+ rvec *x,
+ nbnxn_atomdata_t *nbat,
+ int cxy_start, int cxy_end,
+ int *sort_work)
+{
+ int cxy;
+ int cx, cy, cz, ncz, cfilled, c;
+ int na, ash, ind, a;
+ int na_c, ash_c;
+
+ if (debug)
+ {
+ fprintf(debug, "cell0 %d sorting columns %d - %d, atoms %d - %d\n",
+ grid->cell0, cxy_start, cxy_end, a0, a1);
+ }
+
+ /* Sort the atoms within each x,y column in 3 dimensions */
+ for (cxy = cxy_start; cxy < cxy_end; cxy++)
+ {
+ cx = cxy/grid->ncy;
+ cy = cxy - cx*grid->ncy;
+
+ na = grid->cxy_na[cxy];
+ ncz = grid->cxy_ind[cxy+1] - grid->cxy_ind[cxy];
+ ash = (grid->cell0 + grid->cxy_ind[cxy])*grid->na_sc;
+
+ /* Sort the atoms within each x,y column on z coordinate */
+ sort_atoms(ZZ, FALSE,
+ nbs->a+ash, na, x,
+ grid->c0[ZZ],
+ 1.0/nbs->box[ZZ][ZZ], ncz*grid->na_sc,
+ sort_work);
+
+ /* Fill the ncz cells in this column */
+ cfilled = grid->cxy_ind[cxy];
+ for (cz = 0; cz < ncz; cz++)
+ {
+ c = grid->cxy_ind[cxy] + cz;
+
+ ash_c = ash + cz*grid->na_sc;
+ na_c = min(grid->na_sc, na-(ash_c-ash));
+
+ fill_cell(nbs, grid, nbat,
+ ash_c, ash_c+na_c, atinfo, x,
+ grid->na_sc*cx + (dd_zone >> 2),
+ grid->na_sc*cy + (dd_zone & 3),
+ grid->na_sc*cz,
+ NULL);
+
+ /* This copy to bbcz is not really necessary.
+ * But it allows to use the same grid search code
+ * for the simple and supersub cell setups.
+ */
+ if (na_c > 0)
+ {
+ cfilled = c;
+ }
+ grid->bbcz[c*NNBSBB_D ] = grid->bb[cfilled].lower[BB_Z];
+ grid->bbcz[c*NNBSBB_D+1] = grid->bb[cfilled].upper[BB_Z];
+ }
+
+ /* Set the unused atom indices to -1 */
+ for (ind = na; ind < ncz*grid->na_sc; ind++)
+ {
+ nbs->a[ash+ind] = -1;
+ }
+ }
+}
+
+/* Spatially sort the atoms within one grid column */
+static void sort_columns_supersub(const nbnxn_search_t nbs,
+ int dd_zone,
+ nbnxn_grid_t *grid,
+ int a0, int a1,
+ const int *atinfo,
+ rvec *x,
+ nbnxn_atomdata_t *nbat,
+ int cxy_start, int cxy_end,
+ int *sort_work)
+{
+ int cxy;
+ int cx, cy, cz = -1, c = -1, ncz;
+ int na, ash, na_c, ind, a;
+ int subdiv_z, sub_z, na_z, ash_z;
+ int subdiv_y, sub_y, na_y, ash_y;
+ int subdiv_x, sub_x, na_x, ash_x;
+
+ /* cppcheck-suppress unassignedVariable */
+ nbnxn_bb_t bb_work_array[2], *bb_work_aligned;
+
+ bb_work_aligned = (nbnxn_bb_t *)(((size_t)(bb_work_array+1)) & (~((size_t)15)));
+
+ if (debug)
+ {
+ fprintf(debug, "cell0 %d sorting columns %d - %d, atoms %d - %d\n",
+ grid->cell0, cxy_start, cxy_end, a0, a1);
+ }
+
+ subdiv_x = grid->na_c;
+ subdiv_y = GPU_NSUBCELL_X*subdiv_x;
+ subdiv_z = GPU_NSUBCELL_Y*subdiv_y;
+
+ /* Sort the atoms within each x,y column in 3 dimensions */
+ for (cxy = cxy_start; cxy < cxy_end; cxy++)
+ {
+ cx = cxy/grid->ncy;
+ cy = cxy - cx*grid->ncy;
+
+ na = grid->cxy_na[cxy];
+ ncz = grid->cxy_ind[cxy+1] - grid->cxy_ind[cxy];
+ ash = (grid->cell0 + grid->cxy_ind[cxy])*grid->na_sc;
+
+ /* Sort the atoms within each x,y column on z coordinate */
+ sort_atoms(ZZ, FALSE,
+ nbs->a+ash, na, x,
+ grid->c0[ZZ],
+ 1.0/nbs->box[ZZ][ZZ], ncz*grid->na_sc,
+ sort_work);
+
+ /* This loop goes over the supercells and subcells along z at once */
+ for (sub_z = 0; sub_z < ncz*GPU_NSUBCELL_Z; sub_z++)
+ {
+ ash_z = ash + sub_z*subdiv_z;
+ na_z = min(subdiv_z, na-(ash_z-ash));
+
+ /* We have already sorted on z */
+
+ if (sub_z % GPU_NSUBCELL_Z == 0)
+ {
+ cz = sub_z/GPU_NSUBCELL_Z;
+ c = grid->cxy_ind[cxy] + cz;
+
+ /* The number of atoms in this supercell */
+ na_c = min(grid->na_sc, na-(ash_z-ash));
+
+ grid->nsubc[c] = min(GPU_NSUBCELL, (na_c+grid->na_c-1)/grid->na_c);
+
+ /* Store the z-boundaries of the super cell */
+ grid->bbcz[c*NNBSBB_D ] = x[nbs->a[ash_z]][ZZ];
+ grid->bbcz[c*NNBSBB_D+1] = x[nbs->a[ash_z+na_c-1]][ZZ];
+ }
+
+#if GPU_NSUBCELL_Y > 1
+ /* Sort the atoms along y */
+ sort_atoms(YY, (sub_z & 1),
+ nbs->a+ash_z, na_z, x,
+ grid->c0[YY]+cy*grid->sy,
+ grid->inv_sy, subdiv_z,
+ sort_work);
+#endif
+
+ for (sub_y = 0; sub_y < GPU_NSUBCELL_Y; sub_y++)
+ {
+ ash_y = ash_z + sub_y*subdiv_y;
+ na_y = min(subdiv_y, na-(ash_y-ash));
+
+#if GPU_NSUBCELL_X > 1
+ /* Sort the atoms along x */
+ sort_atoms(XX, ((cz*GPU_NSUBCELL_Y + sub_y) & 1),
+ nbs->a+ash_y, na_y, x,
+ grid->c0[XX]+cx*grid->sx,
+ grid->inv_sx, subdiv_y,
+ sort_work);
+#endif
+
+ for (sub_x = 0; sub_x < GPU_NSUBCELL_X; sub_x++)
+ {
+ ash_x = ash_y + sub_x*subdiv_x;
+ na_x = min(subdiv_x, na-(ash_x-ash));
+
+ fill_cell(nbs, grid, nbat,
+ ash_x, ash_x+na_x, atinfo, x,
+ grid->na_c*(cx*GPU_NSUBCELL_X+sub_x) + (dd_zone >> 2),
+ grid->na_c*(cy*GPU_NSUBCELL_Y+sub_y) + (dd_zone & 3),
+ grid->na_c*sub_z,
+ bb_work_aligned);
+ }
+ }
+ }
+
+ /* Set the unused atom indices to -1 */
+ for (ind = na; ind < ncz*grid->na_sc; ind++)
+ {
+ nbs->a[ash+ind] = -1;
+ }
+ }
+}
+
+/* Determine in which grid column atoms should go */
+static void calc_column_indices(nbnxn_grid_t *grid,
+ int a0, int a1,
+ rvec *x,
+ int dd_zone, const int *move,
+ int thread, int nthread,
+ int *cell,
+ int *cxy_na)
+{
+ int n0, n1, i;
+ int cx, cy;
+
+ /* We add one extra cell for particles which moved during DD */
+ for (i = 0; i < grid->ncx*grid->ncy+1; i++)
+ {
+ cxy_na[i] = 0;
+ }
+
+ n0 = a0 + (int)((thread+0)*(a1 - a0))/nthread;
+ n1 = a0 + (int)((thread+1)*(a1 - a0))/nthread;
+ if (dd_zone == 0)
+ {
+ /* Home zone */
+ for (i = n0; i < n1; i++)
+ {
+ if (move == NULL || move[i] >= 0)
+ {
+ /* We need to be careful with rounding,
+ * particles might be a few bits outside the local zone.
+ * The int cast takes care of the lower bound,
+ * we will explicitly take care of the upper bound.
+ */
+ cx = (int)((x[i][XX] - grid->c0[XX])*grid->inv_sx);
+ cy = (int)((x[i][YY] - grid->c0[YY])*grid->inv_sy);
+
+#ifndef NDEBUG
+ if (cx < 0 || cx > grid->ncx ||
+ cy < 0 || cy > grid->ncy)
+ {
+ gmx_fatal(FARGS,
+ "grid cell cx %d cy %d out of range (max %d %d)\n"
+ "atom %f %f %f, grid->c0 %f %f",
+ cx, cy, grid->ncx, grid->ncy,
+ x[i][XX], x[i][YY], x[i][ZZ], grid->c0[XX], grid->c0[YY]);
+ }
+#endif
+ /* Take care of potential rouding issues */
+ cx = min(cx, grid->ncx - 1);
+ cy = min(cy, grid->ncy - 1);
+
+ /* For the moment cell will contain only the, grid local,
+ * x and y indices, not z.
+ */
+ cell[i] = cx*grid->ncy + cy;
+ }
+ else
+ {
+ /* Put this moved particle after the end of the grid,
+ * so we can process it later without using conditionals.
+ */
+ cell[i] = grid->ncx*grid->ncy;
+ }
+
+ cxy_na[cell[i]]++;
+ }
+ }
+ else
+ {
+ /* Non-home zone */
+ for (i = n0; i < n1; i++)
+ {
+ cx = (int)((x[i][XX] - grid->c0[XX])*grid->inv_sx);
+ cy = (int)((x[i][YY] - grid->c0[YY])*grid->inv_sy);
+
+ /* For non-home zones there could be particles outside
+ * the non-bonded cut-off range, which have been communicated
+ * for bonded interactions only. For the result it doesn't
+ * matter where these end up on the grid. For performance
+ * we put them in an extra row at the border.
+ */
+ cx = max(cx, 0);
+ cx = min(cx, grid->ncx - 1);
+ cy = max(cy, 0);
+ cy = min(cy, grid->ncy - 1);
+
+ /* For the moment cell will contain only the, grid local,
+ * x and y indices, not z.
+ */
+ cell[i] = cx*grid->ncy + cy;
+
+ cxy_na[cell[i]]++;
+ }
+ }
+}
+
+/* Determine in which grid cells the atoms should go */
+static void calc_cell_indices(const nbnxn_search_t nbs,
+ int dd_zone,
+ nbnxn_grid_t *grid,
+ int a0, int a1,
+ const int *atinfo,
+ rvec *x,
+ const int *move,
+ nbnxn_atomdata_t *nbat)
+{
+ int n0, n1, i;
+ int cx, cy, cxy, ncz_max, ncz;
+ int nthread, thread;
+ int *cxy_na, cxy_na_i;
+
+ nthread = gmx_omp_nthreads_get(emntPairsearch);
+
+#pragma omp parallel for num_threads(nthread) schedule(static)
+ for (thread = 0; thread < nthread; thread++)
+ {
+ calc_column_indices(grid, a0, a1, x, dd_zone, move, thread, nthread,
+ nbs->cell, nbs->work[thread].cxy_na);
+ }
+
+ /* Make the cell index as a function of x and y */
+ ncz_max = 0;
+ ncz = 0;
+ grid->cxy_ind[0] = 0;
+ for (i = 0; i < grid->ncx*grid->ncy+1; i++)
+ {
+ /* We set ncz_max at the beginning of the loop iso at the end
+ * to skip i=grid->ncx*grid->ncy which are moved particles
+ * that do not need to be ordered on the grid.
+ */
+ if (ncz > ncz_max)
+ {
+ ncz_max = ncz;
+ }
+ cxy_na_i = nbs->work[0].cxy_na[i];
+ for (thread = 1; thread < nthread; thread++)
+ {
+ cxy_na_i += nbs->work[thread].cxy_na[i];
+ }
+ ncz = (cxy_na_i + grid->na_sc - 1)/grid->na_sc;
+ if (nbat->XFormat == nbatX8)
+ {
+ /* Make the number of cell a multiple of 2 */
+ ncz = (ncz + 1) & ~1;
+ }
+ grid->cxy_ind[i+1] = grid->cxy_ind[i] + ncz;
+ /* Clear cxy_na, so we can reuse the array below */
+ grid->cxy_na[i] = 0;
+ }
+ grid->nc = grid->cxy_ind[grid->ncx*grid->ncy] - grid->cxy_ind[0];
+
+ nbat->natoms = (grid->cell0 + grid->nc)*grid->na_sc;
+
+ if (debug)
+ {
+ fprintf(debug, "ns na_sc %d na_c %d super-cells: %d x %d y %d z %.1f maxz %d\n",
+ grid->na_sc, grid->na_c, grid->nc,
+ grid->ncx, grid->ncy, grid->nc/((double)(grid->ncx*grid->ncy)),
+ ncz_max);
+ if (gmx_debug_at)
+ {
+ i = 0;
+ for (cy = 0; cy < grid->ncy; cy++)
+ {
+ for (cx = 0; cx < grid->ncx; cx++)
+ {
+ fprintf(debug, " %2d", grid->cxy_ind[i+1]-grid->cxy_ind[i]);
+ i++;
+ }
+ fprintf(debug, "\n");
+ }
+ }
+ }
+
+ /* Make sure the work array for sorting is large enough */
+ if (ncz_max*grid->na_sc*SGSF > nbs->work[0].sort_work_nalloc)
+ {
+ for (thread = 0; thread < nbs->nthread_max; thread++)
+ {
+ nbs->work[thread].sort_work_nalloc =
+ over_alloc_large(ncz_max*grid->na_sc*SGSF);
+ srenew(nbs->work[thread].sort_work,
+ nbs->work[thread].sort_work_nalloc);
+ /* When not in use, all elements should be -1 */
+ for (i = 0; i < nbs->work[thread].sort_work_nalloc; i++)
+ {
+ nbs->work[thread].sort_work[i] = -1;
+ }
+ }
+ }
+
+ /* Now we know the dimensions we can fill the grid.
+ * This is the first, unsorted fill. We sort the columns after this.
+ */
+ for (i = a0; i < a1; i++)
+ {
+ /* At this point nbs->cell contains the local grid x,y indices */
+ cxy = nbs->cell[i];
+ nbs->a[(grid->cell0 + grid->cxy_ind[cxy])*grid->na_sc + grid->cxy_na[cxy]++] = i;
+ }
+
+ if (dd_zone == 0)
+ {
+ /* Set the cell indices for the moved particles */
+ n0 = grid->nc*grid->na_sc;
+ n1 = grid->nc*grid->na_sc+grid->cxy_na[grid->ncx*grid->ncy];
+ if (dd_zone == 0)
+ {
+ for (i = n0; i < n1; i++)
+ {
+ nbs->cell[nbs->a[i]] = i;
+ }
+ }
+ }
+
+ /* Sort the super-cell columns along z into the sub-cells. */
+#pragma omp parallel for num_threads(nbs->nthread_max) schedule(static)
+ for (thread = 0; thread < nbs->nthread_max; thread++)
+ {
+ if (grid->bSimple)
+ {
+ sort_columns_simple(nbs, dd_zone, grid, a0, a1, atinfo, x, nbat,
+ ((thread+0)*grid->ncx*grid->ncy)/nthread,
+ ((thread+1)*grid->ncx*grid->ncy)/nthread,
+ nbs->work[thread].sort_work);
+ }
+ else
+ {
+ sort_columns_supersub(nbs, dd_zone, grid, a0, a1, atinfo, x, nbat,
+ ((thread+0)*grid->ncx*grid->ncy)/nthread,
+ ((thread+1)*grid->ncx*grid->ncy)/nthread,
+ nbs->work[thread].sort_work);
+ }
+ }
+
+ if (grid->bSimple && nbat->XFormat == nbatX8)
+ {
+ combine_bounding_box_pairs(grid, grid->bb);
+ }
+
+ if (!grid->bSimple)
+ {
+ grid->nsubc_tot = 0;
+ for (i = 0; i < grid->nc; i++)
+ {
+ grid->nsubc_tot += grid->nsubc[i];
+ }
+ }
+
+ if (debug)
+ {
+ if (grid->bSimple)
+ {
+ print_bbsizes_simple(debug, nbs, grid);
+ }
+ else
+ {
+ fprintf(debug, "ns non-zero sub-cells: %d average atoms %.2f\n",
+ grid->nsubc_tot, (a1-a0)/(double)grid->nsubc_tot);
+
+ print_bbsizes_supersub(debug, nbs, grid);
+ }
+ }
+}
+
+static void init_buffer_flags(nbnxn_buffer_flags_t *flags,
+ int natoms)
+{
+ int b;
+
+ flags->nflag = (natoms + NBNXN_BUFFERFLAG_SIZE - 1)/NBNXN_BUFFERFLAG_SIZE;
+ if (flags->nflag > flags->flag_nalloc)
+ {
+ flags->flag_nalloc = over_alloc_large(flags->nflag);
+ srenew(flags->flag, flags->flag_nalloc);
+ }
+ for (b = 0; b < flags->nflag; b++)
+ {
+ flags->flag[b] = 0;
+ }
+}
+
+/* Sets up a grid and puts the atoms on the grid.
+ * This function only operates on one domain of the domain decompostion.
+ * Note that without domain decomposition there is only one domain.
+ */
+void nbnxn_put_on_grid(nbnxn_search_t nbs,
+ int ePBC, matrix box,
+ int dd_zone,
+ rvec corner0, rvec corner1,
+ int a0, int a1,
+ real atom_density,
+ const int *atinfo,
+ rvec *x,
+ int nmoved, int *move,
+ int nb_kernel_type,
+ nbnxn_atomdata_t *nbat)
+{
+ nbnxn_grid_t *grid;
+ int n;
+ int nc_max_grid, nc_max;
+
+ grid = &nbs->grid[dd_zone];
+
+ nbs_cycle_start(&nbs->cc[enbsCCgrid]);
+
+ grid->bSimple = nbnxn_kernel_pairlist_simple(nb_kernel_type);
+
+ grid->na_c = nbnxn_kernel_to_ci_size(nb_kernel_type);
+ grid->na_cj = nbnxn_kernel_to_cj_size(nb_kernel_type);
+ grid->na_sc = (grid->bSimple ? 1 : GPU_NSUBCELL)*grid->na_c;
+ grid->na_c_2log = get_2log(grid->na_c);
+
+ nbat->na_c = grid->na_c;
+
+ if (dd_zone == 0)
+ {
+ grid->cell0 = 0;
+ }
+ else
+ {
+ grid->cell0 =
+ (nbs->grid[dd_zone-1].cell0 + nbs->grid[dd_zone-1].nc)*
+ nbs->grid[dd_zone-1].na_sc/grid->na_sc;
+ }
+
+ n = a1 - a0;
+
+ if (dd_zone == 0)
+ {
+ nbs->ePBC = ePBC;
+ copy_mat(box, nbs->box);
+
+ if (atom_density >= 0)
+ {
+ grid->atom_density = atom_density;
+ }
+ else
+ {
+ grid->atom_density = grid_atom_density(n-nmoved, corner0, corner1);
+ }
+
+ grid->cell0 = 0;
+
+ nbs->natoms_local = a1 - nmoved;
+ /* We assume that nbnxn_put_on_grid is called first
+ * for the local atoms (dd_zone=0).
+ */
+ nbs->natoms_nonlocal = a1 - nmoved;
+ }
+ else
+ {
+ nbs->natoms_nonlocal = max(nbs->natoms_nonlocal, a1);
+ }
+
+ nc_max_grid = set_grid_size_xy(nbs, grid,
+ dd_zone, n-nmoved, corner0, corner1,
+ nbs->grid[0].atom_density);
+
+ nc_max = grid->cell0 + nc_max_grid;
+
+ if (a1 > nbs->cell_nalloc)
+ {
+ nbs->cell_nalloc = over_alloc_large(a1);
+ srenew(nbs->cell, nbs->cell_nalloc);
+ }
+
+ /* To avoid conditionals we store the moved particles at the end of a,
+ * make sure we have enough space.
+ */
+ if (nc_max*grid->na_sc + nmoved > nbs->a_nalloc)
+ {
+ nbs->a_nalloc = over_alloc_large(nc_max*grid->na_sc + nmoved);
+ srenew(nbs->a, nbs->a_nalloc);
+ }
+
+ /* We need padding up to a multiple of the buffer flag size: simply add */
+ if (nc_max*grid->na_sc + NBNXN_BUFFERFLAG_SIZE > nbat->nalloc)
+ {
+ nbnxn_atomdata_realloc(nbat, nc_max*grid->na_sc+NBNXN_BUFFERFLAG_SIZE);
+ }
+
+ calc_cell_indices(nbs, dd_zone, grid, a0, a1, atinfo, x, move, nbat);
+
+ if (dd_zone == 0)
+ {
+ nbat->natoms_local = nbat->natoms;
+ }
+
+ nbs_cycle_stop(&nbs->cc[enbsCCgrid]);
+}
+
+/* Calls nbnxn_put_on_grid for all non-local domains */
+void nbnxn_put_on_grid_nonlocal(nbnxn_search_t nbs,
+ const gmx_domdec_zones_t *zones,
+ const int *atinfo,
+ rvec *x,
+ int nb_kernel_type,
+ nbnxn_atomdata_t *nbat)
+{
+ int zone, d;
+ rvec c0, c1;
+
+ for (zone = 1; zone < zones->n; zone++)
+ {
+ for (d = 0; d < DIM; d++)
+ {
+ c0[d] = zones->size[zone].bb_x0[d];
+ c1[d] = zones->size[zone].bb_x1[d];
+ }
+
+ nbnxn_put_on_grid(nbs, nbs->ePBC, NULL,
+ zone, c0, c1,
+ zones->cg_range[zone],
+ zones->cg_range[zone+1],
+ -1,
+ atinfo,
+ x,
+ 0, NULL,
+ nb_kernel_type,
+ nbat);
+ }
+}
+
+/* Add simple grid type information to the local super/sub grid */
+void nbnxn_grid_add_simple(nbnxn_search_t nbs,
+ nbnxn_atomdata_t *nbat)
+{
+ nbnxn_grid_t *grid;
+ float *bbcz;
+ nbnxn_bb_t *bb;
+ int ncd, sc;
+
+ grid = &nbs->grid[0];
+
+ if (grid->bSimple)
+ {
+ gmx_incons("nbnxn_grid_simple called with a simple grid");
+ }
+
+ ncd = grid->na_sc/NBNXN_CPU_CLUSTER_I_SIZE;
+
+ if (grid->nc*ncd > grid->nc_nalloc_simple)
+ {
+ grid->nc_nalloc_simple = over_alloc_large(grid->nc*ncd);
+ srenew(grid->bbcz_simple, grid->nc_nalloc_simple*NNBSBB_D);
+ srenew(grid->bb_simple, grid->nc_nalloc_simple);
+ srenew(grid->flags_simple, grid->nc_nalloc_simple);
+ if (nbat->XFormat)
+ {
+ sfree_aligned(grid->bbj);
+ snew_aligned(grid->bbj, grid->nc_nalloc_simple/2, 16);
+ }
+ }
+
+ bbcz = grid->bbcz_simple;
+ bb = grid->bb_simple;
+
+#pragma omp parallel for num_threads(gmx_omp_nthreads_get(emntPairsearch)) schedule(static)
+ for (sc = 0; sc < grid->nc; sc++)
+ {
+ int c, tx, na;
+
+ for (c = 0; c < ncd; c++)
+ {
+ tx = sc*ncd + c;
+
+ na = NBNXN_CPU_CLUSTER_I_SIZE;
+ while (na > 0 &&
+ nbat->type[tx*NBNXN_CPU_CLUSTER_I_SIZE+na-1] == nbat->ntype-1)
+ {
+ na--;
+ }
+
+ if (na > 0)
+ {
+ switch (nbat->XFormat)
+ {
+ case nbatX4:
+ /* PACK_X4==NBNXN_CPU_CLUSTER_I_SIZE, so this is simple */
+ calc_bounding_box_x_x4(na, nbat->x+tx*STRIDE_P4,
+ bb+tx);
+ break;
+ case nbatX8:
+ /* PACK_X8>NBNXN_CPU_CLUSTER_I_SIZE, more complicated */
+ calc_bounding_box_x_x8(na, nbat->x+X8_IND_A(tx*NBNXN_CPU_CLUSTER_I_SIZE),
+ bb+tx);
+ break;
+ default:
+ calc_bounding_box(na, nbat->xstride,
+ nbat->x+tx*NBNXN_CPU_CLUSTER_I_SIZE*nbat->xstride,
+ bb+tx);
+ break;
+ }
+ bbcz[tx*NNBSBB_D+0] = bb[tx].lower[BB_Z];
+ bbcz[tx*NNBSBB_D+1] = bb[tx].upper[BB_Z];
+
+ /* No interaction optimization yet here */
+ grid->flags_simple[tx] = NBNXN_CI_DO_LJ(0) | NBNXN_CI_DO_COUL(0);
+ }
+ else
+ {
+ grid->flags_simple[tx] = 0;
+ }
+ }
+ }
+
+ if (grid->bSimple && nbat->XFormat == nbatX8)
+ {
+ combine_bounding_box_pairs(grid, grid->bb_simple);
+ }
+}
+
+void nbnxn_get_ncells(nbnxn_search_t nbs, int *ncx, int *ncy)
+{
+ *ncx = nbs->grid[0].ncx;
+ *ncy = nbs->grid[0].ncy;
+}
+
+void nbnxn_get_atomorder(nbnxn_search_t nbs, int **a, int *n)
+{
+ const nbnxn_grid_t *grid;
+
+ grid = &nbs->grid[0];
+
+ /* Return the atom order for the home cell (index 0) */
+ *a = nbs->a;
+
+ *n = grid->cxy_ind[grid->ncx*grid->ncy]*grid->na_sc;
+}
+
+void nbnxn_set_atomorder(nbnxn_search_t nbs)
+{
+ nbnxn_grid_t *grid;
+ int ao, cx, cy, cxy, cz, j;
+
+ /* Set the atom order for the home cell (index 0) */
+ grid = &nbs->grid[0];
+
+ ao = 0;
+ for (cx = 0; cx < grid->ncx; cx++)
+ {
+ for (cy = 0; cy < grid->ncy; cy++)
+ {
+ cxy = cx*grid->ncy + cy;
+ j = grid->cxy_ind[cxy]*grid->na_sc;
+ for (cz = 0; cz < grid->cxy_na[cxy]; cz++)
+ {
+ nbs->a[j] = ao;
+ nbs->cell[ao] = j;
+ ao++;
+ j++;
+ }
+ }
+ }
+}
+
+/* Determines the cell range along one dimension that
+ * the bounding box b0 - b1 sees.
+ */
+static void get_cell_range(real b0, real b1,
+ int nc, real c0, real s, real invs,
+ real d2, real r2, int *cf, int *cl)
+{
+ *cf = max((int)((b0 - c0)*invs), 0);
+
+ while (*cf > 0 && d2 + sqr((b0 - c0) - (*cf-1+1)*s) < r2)
+ {
+ (*cf)--;
+ }
+
+ *cl = min((int)((b1 - c0)*invs), nc-1);
+ while (*cl < nc-1 && d2 + sqr((*cl+1)*s - (b1 - c0)) < r2)
+ {
+ (*cl)++;
+ }
+}
+
+/* Reference code calculating the distance^2 between two bounding boxes */
+static float box_dist2(float bx0, float bx1, float by0,
+ float by1, float bz0, float bz1,
+ const nbnxn_bb_t *bb)
+{
+ float d2;
+ float dl, dh, dm, dm0;
+
+ d2 = 0;
+
+ dl = bx0 - bb->upper[BB_X];
+ dh = bb->lower[BB_X] - bx1;
+ dm = max(dl, dh);
+ dm0 = max(dm, 0);
+ d2 += dm0*dm0;
+
+ dl = by0 - bb->upper[BB_Y];
+ dh = bb->lower[BB_Y] - by1;
+ dm = max(dl, dh);
+ dm0 = max(dm, 0);
+ d2 += dm0*dm0;
+
+ dl = bz0 - bb->upper[BB_Z];
+ dh = bb->lower[BB_Z] - bz1;
+ dm = max(dl, dh);
+ dm0 = max(dm, 0);
+ d2 += dm0*dm0;
+
+ return d2;
+}
+
+/* Plain C code calculating the distance^2 between two bounding boxes */
+static float subc_bb_dist2(int si, const nbnxn_bb_t *bb_i_ci,
+ int csj, const nbnxn_bb_t *bb_j_all)
+{
+ const nbnxn_bb_t *bb_i, *bb_j;
+ float d2;
+ float dl, dh, dm, dm0;
+
+ bb_i = bb_i_ci + si;
+ bb_j = bb_j_all + csj;
+
+ d2 = 0;
+
+ dl = bb_i->lower[BB_X] - bb_j->upper[BB_X];
+ dh = bb_j->lower[BB_X] - bb_i->upper[BB_X];
+ dm = max(dl, dh);
+ dm0 = max(dm, 0);
+ d2 += dm0*dm0;
+
+ dl = bb_i->lower[BB_Y] - bb_j->upper[BB_Y];
+ dh = bb_j->lower[BB_Y] - bb_i->upper[BB_Y];
+ dm = max(dl, dh);
+ dm0 = max(dm, 0);
+ d2 += dm0*dm0;
+
+ dl = bb_i->lower[BB_Z] - bb_j->upper[BB_Z];
+ dh = bb_j->lower[BB_Z] - bb_i->upper[BB_Z];
+ dm = max(dl, dh);
+ dm0 = max(dm, 0);
+ d2 += dm0*dm0;
+
+ return d2;
+}
+
- /* SSE code for bb distance for bb format xyz0 */
- static float subc_bb_dist2_sse(int si, const nbnxn_bb_t *bb_i_ci,
- int csj, const nbnxn_bb_t *bb_j_all)
++#ifdef NBNXN_SEARCH_BB_SIMD4
+
- __m128 bb_i_SSE0, bb_i_SSE1;
- __m128 bb_j_SSE0, bb_j_SSE1;
- __m128 dl_SSE;
- __m128 dh_SSE;
- __m128 dm_SSE;
- __m128 dm0_SSE;
- __m128 d2_SSE;
- #ifndef GMX_X86_SSE4_1
- float d2_array[7], *d2_align;
-
- d2_align = (float *)(((size_t)(d2_array+3)) & (~((size_t)15)));
- #else
- float d2;
- #endif
++/* 4-wide SIMD code for bb distance for bb format xyz0 */
++static float subc_bb_dist2_simd4(int si, const nbnxn_bb_t *bb_i_ci,
++ int csj, const nbnxn_bb_t *bb_j_all)
+{
- bb_i_SSE0 = _mm_load_ps(&bb_i_ci[si].lower[0]);
- bb_i_SSE1 = _mm_load_ps(&bb_i_ci[si].upper[0]);
- bb_j_SSE0 = _mm_load_ps(&bb_j_all[csj].lower[0]);
- bb_j_SSE1 = _mm_load_ps(&bb_j_all[csj].upper[0]);
++ gmx_simd4_pr bb_i_S0, bb_i_S1;
++ gmx_simd4_pr bb_j_S0, bb_j_S1;
++ gmx_simd4_pr dl_S;
++ gmx_simd4_pr dh_S;
++ gmx_simd4_pr dm_S;
++ gmx_simd4_pr dm0_S;
+
- dl_SSE = _mm_sub_ps(bb_i_SSE0, bb_j_SSE1);
- dh_SSE = _mm_sub_ps(bb_j_SSE0, bb_i_SSE1);
++ bb_i_S0 = gmx_simd4_load_bb_pr(&bb_i_ci[si].lower[0]);
++ bb_i_S1 = gmx_simd4_load_bb_pr(&bb_i_ci[si].upper[0]);
++ bb_j_S0 = gmx_simd4_load_bb_pr(&bb_j_all[csj].lower[0]);
++ bb_j_S1 = gmx_simd4_load_bb_pr(&bb_j_all[csj].upper[0]);
+
- dm_SSE = _mm_max_ps(dl_SSE, dh_SSE);
- dm0_SSE = _mm_max_ps(dm_SSE, _mm_setzero_ps());
- #ifndef GMX_X86_SSE4_1
- d2_SSE = _mm_mul_ps(dm0_SSE, dm0_SSE);
-
- _mm_store_ps(d2_align, d2_SSE);
-
- return d2_align[0] + d2_align[1] + d2_align[2];
- #else
- /* SSE4.1 dot product of components 0,1,2 */
- d2_SSE = _mm_dp_ps(dm0_SSE, dm0_SSE, 0x71);
++ dl_S = gmx_simd4_sub_pr(bb_i_S0, bb_j_S1);
++ dh_S = gmx_simd4_sub_pr(bb_j_S0, bb_i_S1);
+
- _mm_store_ss(&d2, d2_SSE);
-
- return d2;
- #endif
++ dm_S = gmx_simd4_max_pr(dl_S, dh_S);
++ dm0_S = gmx_simd4_max_pr(dm_S, gmx_simd4_setzero_pr());
+
- #define SUBC_BB_DIST2_SSE_XXXX_INNER(si, bb_i, d2) \
++ return gmx_simd4_dotproduct3(dm0_S, dm0_S);
+}
+
+/* Calculate bb bounding distances of bb_i[si,...,si+3] and store them in d2 */
- __m128 dx_0, dy_0, dz_0; \
- __m128 dx_1, dy_1, dz_1; \
++#define SUBC_BB_DIST2_SIMD4_XXXX_INNER(si, bb_i, d2) \
+ { \
+ int shi; \
+ \
- __m128 mx, my, mz; \
- __m128 m0x, m0y, m0z; \
++ gmx_simd4_pr dx_0, dy_0, dz_0; \
++ gmx_simd4_pr dx_1, dy_1, dz_1; \
+ \
- __m128 d2x, d2y, d2z; \
- __m128 d2s, d2t; \
++ gmx_simd4_pr mx, my, mz; \
++ gmx_simd4_pr m0x, m0y, m0z; \
+ \
- xi_l = _mm_load_ps(bb_i+shi+0*STRIDE_PBB); \
- yi_l = _mm_load_ps(bb_i+shi+1*STRIDE_PBB); \
- zi_l = _mm_load_ps(bb_i+shi+2*STRIDE_PBB); \
- xi_h = _mm_load_ps(bb_i+shi+3*STRIDE_PBB); \
- yi_h = _mm_load_ps(bb_i+shi+4*STRIDE_PBB); \
- zi_h = _mm_load_ps(bb_i+shi+5*STRIDE_PBB); \
++ gmx_simd4_pr d2x, d2y, d2z; \
++ gmx_simd4_pr d2s, d2t; \
+ \
+ shi = si*NNBSBB_D*DIM; \
+ \
- dx_0 = _mm_sub_ps(xi_l, xj_h); \
- dy_0 = _mm_sub_ps(yi_l, yj_h); \
- dz_0 = _mm_sub_ps(zi_l, zj_h); \
++ xi_l = gmx_simd4_load_bb_pr(bb_i+shi+0*STRIDE_PBB); \
++ yi_l = gmx_simd4_load_bb_pr(bb_i+shi+1*STRIDE_PBB); \
++ zi_l = gmx_simd4_load_bb_pr(bb_i+shi+2*STRIDE_PBB); \
++ xi_h = gmx_simd4_load_bb_pr(bb_i+shi+3*STRIDE_PBB); \
++ yi_h = gmx_simd4_load_bb_pr(bb_i+shi+4*STRIDE_PBB); \
++ zi_h = gmx_simd4_load_bb_pr(bb_i+shi+5*STRIDE_PBB); \
+ \
- dx_1 = _mm_sub_ps(xj_l, xi_h); \
- dy_1 = _mm_sub_ps(yj_l, yi_h); \
- dz_1 = _mm_sub_ps(zj_l, zi_h); \
++ dx_0 = gmx_simd4_sub_pr(xi_l, xj_h); \
++ dy_0 = gmx_simd4_sub_pr(yi_l, yj_h); \
++ dz_0 = gmx_simd4_sub_pr(zi_l, zj_h); \
+ \
- mx = _mm_max_ps(dx_0, dx_1); \
- my = _mm_max_ps(dy_0, dy_1); \
- mz = _mm_max_ps(dz_0, dz_1); \
++ dx_1 = gmx_simd4_sub_pr(xj_l, xi_h); \
++ dy_1 = gmx_simd4_sub_pr(yj_l, yi_h); \
++ dz_1 = gmx_simd4_sub_pr(zj_l, zi_h); \
+ \
- m0x = _mm_max_ps(mx, zero); \
- m0y = _mm_max_ps(my, zero); \
- m0z = _mm_max_ps(mz, zero); \
++ mx = gmx_simd4_max_pr(dx_0, dx_1); \
++ my = gmx_simd4_max_pr(dy_0, dy_1); \
++ mz = gmx_simd4_max_pr(dz_0, dz_1); \
+ \
- d2x = _mm_mul_ps(m0x, m0x); \
- d2y = _mm_mul_ps(m0y, m0y); \
- d2z = _mm_mul_ps(m0z, m0z); \
++ m0x = gmx_simd4_max_pr(mx, zero); \
++ m0y = gmx_simd4_max_pr(my, zero); \
++ m0z = gmx_simd4_max_pr(mz, zero); \
+ \
- d2s = _mm_add_ps(d2x, d2y); \
- d2t = _mm_add_ps(d2s, d2z); \
++ d2x = gmx_simd4_mul_pr(m0x, m0x); \
++ d2y = gmx_simd4_mul_pr(m0y, m0y); \
++ d2z = gmx_simd4_mul_pr(m0z, m0z); \
+ \
- _mm_store_ps(d2+si, d2t); \
++ d2s = gmx_simd4_add_pr(d2x, d2y); \
++ d2t = gmx_simd4_add_pr(d2s, d2z); \
+ \
- /* SSE code for nsi bb distances for bb format xxxxyyyyzzzz */
- static void subc_bb_dist2_sse_xxxx(const float *bb_j,
- int nsi, const float *bb_i,
- float *d2)
++ gmx_simd4_store_pr(d2+si, d2t); \
+ }
+
- __m128 xj_l, yj_l, zj_l;
- __m128 xj_h, yj_h, zj_h;
- __m128 xi_l, yi_l, zi_l;
- __m128 xi_h, yi_h, zi_h;
++/* 4-wide SIMD code for nsi bb distances for bb format xxxxyyyyzzzz */
++static void subc_bb_dist2_simd4_xxxx(const float *bb_j,
++ int nsi, const float *bb_i,
++ float *d2)
+{
- __m128 zero;
++ gmx_simd4_pr xj_l, yj_l, zj_l;
++ gmx_simd4_pr xj_h, yj_h, zj_h;
++ gmx_simd4_pr xi_l, yi_l, zi_l;
++ gmx_simd4_pr xi_h, yi_h, zi_h;
+
- zero = _mm_setzero_ps();
++ gmx_simd4_pr zero;
+
- xj_l = _mm_set1_ps(bb_j[0*STRIDE_PBB]);
- yj_l = _mm_set1_ps(bb_j[1*STRIDE_PBB]);
- zj_l = _mm_set1_ps(bb_j[2*STRIDE_PBB]);
- xj_h = _mm_set1_ps(bb_j[3*STRIDE_PBB]);
- yj_h = _mm_set1_ps(bb_j[4*STRIDE_PBB]);
- zj_h = _mm_set1_ps(bb_j[5*STRIDE_PBB]);
++ zero = gmx_simd4_setzero_pr();
+
- SUBC_BB_DIST2_SSE_XXXX_INNER(0, bb_i, d2);
++ xj_l = gmx_simd4_set1_pr(bb_j[0*STRIDE_PBB]);
++ yj_l = gmx_simd4_set1_pr(bb_j[1*STRIDE_PBB]);
++ zj_l = gmx_simd4_set1_pr(bb_j[2*STRIDE_PBB]);
++ xj_h = gmx_simd4_set1_pr(bb_j[3*STRIDE_PBB]);
++ yj_h = gmx_simd4_set1_pr(bb_j[4*STRIDE_PBB]);
++ zj_h = gmx_simd4_set1_pr(bb_j[5*STRIDE_PBB]);
+
+ /* Here we "loop" over si (0,STRIDE_PBB) from 0 to nsi with step STRIDE_PBB.
+ * But as we know the number of iterations is 1 or 2, we unroll manually.
+ */
- SUBC_BB_DIST2_SSE_XXXX_INNER(STRIDE_PBB, bb_i, d2);
++ SUBC_BB_DIST2_SIMD4_XXXX_INNER(0, bb_i, d2);
+ if (STRIDE_PBB < nsi)
+ {
- #endif /* NBNXN_SEARCH_BB_SSE */
++ SUBC_BB_DIST2_SIMD4_XXXX_INNER(STRIDE_PBB, bb_i, d2);
+ }
+}
+
- #ifdef NBNXN_SEARCH_SSE_SINGLE
++#endif /* NBNXN_SEARCH_BB_SIMD4 */
+
+/* Plain C function which determines if any atom pair between two cells
+ * is within distance sqrt(rl2).
+ */
+static gmx_bool subc_in_range_x(int na_c,
+ int si, const real *x_i,
+ int csj, int stride, const real *x_j,
+ real rl2)
+{
+ int i, j, i0, j0;
+ real d2;
+
+ for (i = 0; i < na_c; i++)
+ {
+ i0 = (si*na_c + i)*DIM;
+ for (j = 0; j < na_c; j++)
+ {
+ j0 = (csj*na_c + j)*stride;
+
+ d2 = sqr(x_i[i0 ] - x_j[j0 ]) +
+ sqr(x_i[i0+1] - x_j[j0+1]) +
+ sqr(x_i[i0+2] - x_j[j0+2]);
+
+ if (d2 < rl2)
+ {
+ return TRUE;
+ }
+ }
+ }
+
+ return FALSE;
+}
+
- static inline __m128
- gmx_mm_calc_rsq_ps(__m128 x, __m128 y, __m128 z)
++#ifdef NBNXN_SEARCH_SIMD4_FLOAT_X_BB
+/* When we make seperate single/double precision SIMD vector operation
+ * include files, this function should be moved there (also using FMA).
+ */
- return _mm_add_ps( _mm_add_ps( _mm_mul_ps(x, x), _mm_mul_ps(y, y) ), _mm_mul_ps(z, z) );
++static inline gmx_simd4_pr
++gmx_simd4_calc_rsq_pr(gmx_simd4_pr x, gmx_simd4_pr y, gmx_simd4_pr z)
+{
- /* SSE function which determines if any atom pair between two cells,
++ return gmx_simd4_add_pr( gmx_simd4_add_pr( gmx_simd4_mul_pr(x, x), gmx_simd4_mul_pr(y, y) ), gmx_simd4_mul_pr(z, z) );
+}
+#endif
+
- * Not performance critical, so only uses plain SSE.
++/* 4-wide SIMD function which determines if any atom pair between two cells,
+ * both with 8 atoms, is within distance sqrt(rl2).
- static gmx_bool subc_in_range_sse8(int na_c,
- int si, const real *x_i,
- int csj, int stride, const real *x_j,
- real rl2)
++ * Using 8-wide AVX is not faster on Intel Sandy Bridge.
+ */
- #ifdef NBNXN_SEARCH_SSE_SINGLE
- __m128 ix_SSE0, iy_SSE0, iz_SSE0;
- __m128 ix_SSE1, iy_SSE1, iz_SSE1;
++static gmx_bool subc_in_range_simd4(int na_c,
++ int si, const real *x_i,
++ int csj, int stride, const real *x_j,
++ real rl2)
+{
- __m128 rc2_SSE;
++#ifdef NBNXN_SEARCH_SIMD4_FLOAT_X_BB
++ gmx_simd4_pr ix_S0, iy_S0, iz_S0;
++ gmx_simd4_pr ix_S1, iy_S1, iz_S1;
+
- int na_c_sse;
++ gmx_simd4_pr rc2_S;
+
- rc2_SSE = _mm_set1_ps(rl2);
++ int dim_stride;
+ int j0, j1;
+
- na_c_sse = NBNXN_GPU_CLUSTER_SIZE/STRIDE_PBB;
- ix_SSE0 = _mm_load_ps(x_i+(si*na_c_sse*DIM+0)*STRIDE_PBB);
- iy_SSE0 = _mm_load_ps(x_i+(si*na_c_sse*DIM+1)*STRIDE_PBB);
- iz_SSE0 = _mm_load_ps(x_i+(si*na_c_sse*DIM+2)*STRIDE_PBB);
- ix_SSE1 = _mm_load_ps(x_i+(si*na_c_sse*DIM+3)*STRIDE_PBB);
- iy_SSE1 = _mm_load_ps(x_i+(si*na_c_sse*DIM+4)*STRIDE_PBB);
- iz_SSE1 = _mm_load_ps(x_i+(si*na_c_sse*DIM+5)*STRIDE_PBB);
++ rc2_S = gmx_simd4_set1_pr(rl2);
+
- __m128 jx0_SSE, jy0_SSE, jz0_SSE;
- __m128 jx1_SSE, jy1_SSE, jz1_SSE;
++ dim_stride = NBNXN_GPU_CLUSTER_SIZE/STRIDE_PBB*DIM;
++ ix_S0 = gmx_simd4_load_bb_pr(x_i+(si*dim_stride+0)*STRIDE_PBB);
++ iy_S0 = gmx_simd4_load_bb_pr(x_i+(si*dim_stride+1)*STRIDE_PBB);
++ iz_S0 = gmx_simd4_load_bb_pr(x_i+(si*dim_stride+2)*STRIDE_PBB);
++ ix_S1 = gmx_simd4_load_bb_pr(x_i+(si*dim_stride+3)*STRIDE_PBB);
++ iy_S1 = gmx_simd4_load_bb_pr(x_i+(si*dim_stride+4)*STRIDE_PBB);
++ iz_S1 = gmx_simd4_load_bb_pr(x_i+(si*dim_stride+5)*STRIDE_PBB);
+
+ /* We loop from the outer to the inner particles to maximize
+ * the chance that we find a pair in range quickly and return.
+ */
+ j0 = csj*na_c;
+ j1 = j0 + na_c - 1;
+ while (j0 < j1)
+ {
- __m128 dx_SSE0, dy_SSE0, dz_SSE0;
- __m128 dx_SSE1, dy_SSE1, dz_SSE1;
- __m128 dx_SSE2, dy_SSE2, dz_SSE2;
- __m128 dx_SSE3, dy_SSE3, dz_SSE3;
++ gmx_simd4_pr jx0_S, jy0_S, jz0_S;
++ gmx_simd4_pr jx1_S, jy1_S, jz1_S;
+
- __m128 rsq_SSE0;
- __m128 rsq_SSE1;
- __m128 rsq_SSE2;
- __m128 rsq_SSE3;
++ gmx_simd4_pr dx_S0, dy_S0, dz_S0;
++ gmx_simd4_pr dx_S1, dy_S1, dz_S1;
++ gmx_simd4_pr dx_S2, dy_S2, dz_S2;
++ gmx_simd4_pr dx_S3, dy_S3, dz_S3;
+
- __m128 wco_SSE0;
- __m128 wco_SSE1;
- __m128 wco_SSE2;
- __m128 wco_SSE3;
- __m128 wco_any_SSE01, wco_any_SSE23, wco_any_SSE;
++ gmx_simd4_pr rsq_S0;
++ gmx_simd4_pr rsq_S1;
++ gmx_simd4_pr rsq_S2;
++ gmx_simd4_pr rsq_S3;
+
- jx0_SSE = _mm_load1_ps(x_j+j0*stride+0);
- jy0_SSE = _mm_load1_ps(x_j+j0*stride+1);
- jz0_SSE = _mm_load1_ps(x_j+j0*stride+2);
++ gmx_simd4_pb wco_S0;
++ gmx_simd4_pb wco_S1;
++ gmx_simd4_pb wco_S2;
++ gmx_simd4_pb wco_S3;
++ gmx_simd4_pb wco_any_S01, wco_any_S23, wco_any_S;
+
- jx1_SSE = _mm_load1_ps(x_j+j1*stride+0);
- jy1_SSE = _mm_load1_ps(x_j+j1*stride+1);
- jz1_SSE = _mm_load1_ps(x_j+j1*stride+2);
++ jx0_S = gmx_simd4_set1_pr(x_j[j0*stride+0]);
++ jy0_S = gmx_simd4_set1_pr(x_j[j0*stride+1]);
++ jz0_S = gmx_simd4_set1_pr(x_j[j0*stride+2]);
+
- dx_SSE0 = _mm_sub_ps(ix_SSE0, jx0_SSE);
- dy_SSE0 = _mm_sub_ps(iy_SSE0, jy0_SSE);
- dz_SSE0 = _mm_sub_ps(iz_SSE0, jz0_SSE);
- dx_SSE1 = _mm_sub_ps(ix_SSE1, jx0_SSE);
- dy_SSE1 = _mm_sub_ps(iy_SSE1, jy0_SSE);
- dz_SSE1 = _mm_sub_ps(iz_SSE1, jz0_SSE);
- dx_SSE2 = _mm_sub_ps(ix_SSE0, jx1_SSE);
- dy_SSE2 = _mm_sub_ps(iy_SSE0, jy1_SSE);
- dz_SSE2 = _mm_sub_ps(iz_SSE0, jz1_SSE);
- dx_SSE3 = _mm_sub_ps(ix_SSE1, jx1_SSE);
- dy_SSE3 = _mm_sub_ps(iy_SSE1, jy1_SSE);
- dz_SSE3 = _mm_sub_ps(iz_SSE1, jz1_SSE);
++ jx1_S = gmx_simd4_set1_pr(x_j[j1*stride+0]);
++ jy1_S = gmx_simd4_set1_pr(x_j[j1*stride+1]);
++ jz1_S = gmx_simd4_set1_pr(x_j[j1*stride+2]);
+
+ /* Calculate distance */
- rsq_SSE0 = gmx_mm_calc_rsq_ps(dx_SSE0, dy_SSE0, dz_SSE0);
- rsq_SSE1 = gmx_mm_calc_rsq_ps(dx_SSE1, dy_SSE1, dz_SSE1);
- rsq_SSE2 = gmx_mm_calc_rsq_ps(dx_SSE2, dy_SSE2, dz_SSE2);
- rsq_SSE3 = gmx_mm_calc_rsq_ps(dx_SSE3, dy_SSE3, dz_SSE3);
++ dx_S0 = gmx_simd4_sub_pr(ix_S0, jx0_S);
++ dy_S0 = gmx_simd4_sub_pr(iy_S0, jy0_S);
++ dz_S0 = gmx_simd4_sub_pr(iz_S0, jz0_S);
++ dx_S1 = gmx_simd4_sub_pr(ix_S1, jx0_S);
++ dy_S1 = gmx_simd4_sub_pr(iy_S1, jy0_S);
++ dz_S1 = gmx_simd4_sub_pr(iz_S1, jz0_S);
++ dx_S2 = gmx_simd4_sub_pr(ix_S0, jx1_S);
++ dy_S2 = gmx_simd4_sub_pr(iy_S0, jy1_S);
++ dz_S2 = gmx_simd4_sub_pr(iz_S0, jz1_S);
++ dx_S3 = gmx_simd4_sub_pr(ix_S1, jx1_S);
++ dy_S3 = gmx_simd4_sub_pr(iy_S1, jy1_S);
++ dz_S3 = gmx_simd4_sub_pr(iz_S1, jz1_S);
+
+ /* rsq = dx*dx+dy*dy+dz*dz */
- wco_SSE0 = _mm_cmplt_ps(rsq_SSE0, rc2_SSE);
- wco_SSE1 = _mm_cmplt_ps(rsq_SSE1, rc2_SSE);
- wco_SSE2 = _mm_cmplt_ps(rsq_SSE2, rc2_SSE);
- wco_SSE3 = _mm_cmplt_ps(rsq_SSE3, rc2_SSE);
++ rsq_S0 = gmx_simd4_calc_rsq_pr(dx_S0, dy_S0, dz_S0);
++ rsq_S1 = gmx_simd4_calc_rsq_pr(dx_S1, dy_S1, dz_S1);
++ rsq_S2 = gmx_simd4_calc_rsq_pr(dx_S2, dy_S2, dz_S2);
++ rsq_S3 = gmx_simd4_calc_rsq_pr(dx_S3, dy_S3, dz_S3);
+
- wco_any_SSE01 = _mm_or_ps(wco_SSE0, wco_SSE1);
- wco_any_SSE23 = _mm_or_ps(wco_SSE2, wco_SSE3);
- wco_any_SSE = _mm_or_ps(wco_any_SSE01, wco_any_SSE23);
++ wco_S0 = gmx_simd4_cmplt_pr(rsq_S0, rc2_S);
++ wco_S1 = gmx_simd4_cmplt_pr(rsq_S1, rc2_S);
++ wco_S2 = gmx_simd4_cmplt_pr(rsq_S2, rc2_S);
++ wco_S3 = gmx_simd4_cmplt_pr(rsq_S3, rc2_S);
+
- if (_mm_movemask_ps(wco_any_SSE))
++ wco_any_S01 = gmx_simd4_or_pb(wco_S0, wco_S1);
++ wco_any_S23 = gmx_simd4_or_pb(wco_S2, wco_S3);
++ wco_any_S = gmx_simd4_or_pb(wco_any_S01, wco_any_S23);
+
- /* No SSE */
- gmx_incons("SSE function called without SSE support");
++ if (gmx_simd4_anytrue_pb(wco_any_S))
+ {
+ return TRUE;
+ }
+
+ j0++;
+ j1--;
+ }
+ return FALSE;
+
+#else
- snew_aligned(nbl->work->bb_ci, 1, NBNXN_MEM_ALIGN);
++ /* No SIMD4 */
++ gmx_incons("SIMD4 function called without 4-wide SIMD support");
+
+ return TRUE;
+#endif
+}
+
+/* Returns the j sub-cell for index cj_ind */
+static int nbl_cj(const nbnxn_pairlist_t *nbl, int cj_ind)
+{
+ return nbl->cj4[cj_ind >> NBNXN_GPU_JGROUP_SIZE_2LOG].cj[cj_ind & (NBNXN_GPU_JGROUP_SIZE - 1)];
+}
+
+/* Returns the i-interaction mask of the j sub-cell for index cj_ind */
+static unsigned nbl_imask0(const nbnxn_pairlist_t *nbl, int cj_ind)
+{
+ return nbl->cj4[cj_ind >> NBNXN_GPU_JGROUP_SIZE_2LOG].imei[0].imask;
+}
+
+/* Ensures there is enough space for extra extra exclusion masks */
+static void check_excl_space(nbnxn_pairlist_t *nbl, int extra)
+{
+ if (nbl->nexcl+extra > nbl->excl_nalloc)
+ {
+ nbl->excl_nalloc = over_alloc_small(nbl->nexcl+extra);
+ nbnxn_realloc_void((void **)&nbl->excl,
+ nbl->nexcl*sizeof(*nbl->excl),
+ nbl->excl_nalloc*sizeof(*nbl->excl),
+ nbl->alloc, nbl->free);
+ }
+}
+
+/* Ensures there is enough space for ncell extra j-cells in the list */
+static void check_subcell_list_space_simple(nbnxn_pairlist_t *nbl,
+ int ncell)
+{
+ int cj_max;
+
+ cj_max = nbl->ncj + ncell;
+
+ if (cj_max > nbl->cj_nalloc)
+ {
+ nbl->cj_nalloc = over_alloc_small(cj_max);
+ nbnxn_realloc_void((void **)&nbl->cj,
+ nbl->ncj*sizeof(*nbl->cj),
+ nbl->cj_nalloc*sizeof(*nbl->cj),
+ nbl->alloc, nbl->free);
+ }
+}
+
+/* Ensures there is enough space for ncell extra j-subcells in the list */
+static void check_subcell_list_space_supersub(nbnxn_pairlist_t *nbl,
+ int nsupercell)
+{
+ int ncj4_max, j4, j, w, t;
+
+#define NWARP 2
+#define WARP_SIZE 32
+
+ /* We can have maximally nsupercell*GPU_NSUBCELL sj lists */
+ /* We can store 4 j-subcell - i-supercell pairs in one struct.
+ * since we round down, we need one extra entry.
+ */
+ ncj4_max = ((nbl->work->cj_ind + nsupercell*GPU_NSUBCELL + NBNXN_GPU_JGROUP_SIZE - 1) >> NBNXN_GPU_JGROUP_SIZE_2LOG);
+
+ if (ncj4_max > nbl->cj4_nalloc)
+ {
+ nbl->cj4_nalloc = over_alloc_small(ncj4_max);
+ nbnxn_realloc_void((void **)&nbl->cj4,
+ nbl->work->cj4_init*sizeof(*nbl->cj4),
+ nbl->cj4_nalloc*sizeof(*nbl->cj4),
+ nbl->alloc, nbl->free);
+ }
+
+ if (ncj4_max > nbl->work->cj4_init)
+ {
+ for (j4 = nbl->work->cj4_init; j4 < ncj4_max; j4++)
+ {
+ /* No i-subcells and no excl's in the list initially */
+ for (w = 0; w < NWARP; w++)
+ {
+ nbl->cj4[j4].imei[w].imask = 0U;
+ nbl->cj4[j4].imei[w].excl_ind = 0;
+
+ }
+ }
+ nbl->work->cj4_init = ncj4_max;
+ }
+}
+
+/* Set all excl masks for one GPU warp no exclusions */
+static void set_no_excls(nbnxn_excl_t *excl)
+{
+ int t;
+
+ for (t = 0; t < WARP_SIZE; t++)
+ {
+ /* Turn all interaction bits on */
+ excl->pair[t] = NBNXN_INTERACTION_MASK_ALL;
+ }
+}
+
+/* Initializes a single nbnxn_pairlist_t data structure */
+static void nbnxn_init_pairlist(nbnxn_pairlist_t *nbl,
+ gmx_bool bSimple,
+ nbnxn_alloc_t *alloc,
+ nbnxn_free_t *free)
+{
+ if (alloc == NULL)
+ {
+ nbl->alloc = nbnxn_alloc_aligned;
+ }
+ else
+ {
+ nbl->alloc = alloc;
+ }
+ if (free == NULL)
+ {
+ nbl->free = nbnxn_free_aligned;
+ }
+ else
+ {
+ nbl->free = free;
+ }
+
+ nbl->bSimple = bSimple;
+ nbl->na_sc = 0;
+ nbl->na_ci = 0;
+ nbl->na_cj = 0;
+ nbl->nci = 0;
+ nbl->ci = NULL;
+ nbl->ci_nalloc = 0;
+ nbl->ncj = 0;
+ nbl->cj = NULL;
+ nbl->cj_nalloc = 0;
+ nbl->ncj4 = 0;
+ /* We need one element extra in sj, so alloc initially with 1 */
+ nbl->cj4_nalloc = 0;
+ nbl->cj4 = NULL;
+ nbl->nci_tot = 0;
+
+ if (!nbl->bSimple)
+ {
+ nbl->excl = NULL;
+ nbl->excl_nalloc = 0;
+ nbl->nexcl = 0;
+ check_excl_space(nbl, 1);
+ nbl->nexcl = 1;
+ set_no_excls(&nbl->excl[0]);
+ }
+
+ snew(nbl->work, 1);
+ if (nbl->bSimple)
+ {
- snew_aligned(nbl->work->pbb_ci, GPU_NSUBCELL/STRIDE_PBB*NNBSBB_XXXX, NBNXN_MEM_ALIGN);
++ snew_aligned(nbl->work->bb_ci, 1, NBNXN_SEARCH_BB_MEM_ALIGN);
+ }
+ else
+ {
+#ifdef NBNXN_BBXXXX
- snew_aligned(nbl->work->bb_ci, GPU_NSUBCELL, NBNXN_MEM_ALIGN);
++ snew_aligned(nbl->work->pbb_ci, GPU_NSUBCELL/STRIDE_PBB*NNBSBB_XXXX, NBNXN_SEARCH_BB_MEM_ALIGN);
+#else
- snew_aligned(nbl->work->x_ci, NBNXN_NA_SC_MAX*DIM, NBNXN_MEM_ALIGN);
++ snew_aligned(nbl->work->bb_ci, GPU_NSUBCELL, NBNXN_SEARCH_BB_MEM_ALIGN);
+#endif
+ }
- snew_aligned(nbl->work->d2, GPU_NSUBCELL, NBNXN_MEM_ALIGN);
++ snew_aligned(nbl->work->x_ci, NBNXN_NA_SC_MAX*DIM, NBNXN_SEARCH_BB_MEM_ALIGN);
+#ifdef GMX_NBNXN_SIMD
+ snew_aligned(nbl->work->x_ci_simd_4xn, 1, NBNXN_MEM_ALIGN);
+ snew_aligned(nbl->work->x_ci_simd_2xnn, 1, NBNXN_MEM_ALIGN);
+#endif
- /* Plain C or SSE code for making a pair list of super-cell sci vs scj.
++ snew_aligned(nbl->work->d2, GPU_NSUBCELL, NBNXN_SEARCH_BB_MEM_ALIGN);
+
+ nbl->work->sort = NULL;
+ nbl->work->sort_nalloc = 0;
+ nbl->work->sci_sort = NULL;
+ nbl->work->sci_sort_nalloc = 0;
+}
+
+void nbnxn_init_pairlist_set(nbnxn_pairlist_set_t *nbl_list,
+ gmx_bool bSimple, gmx_bool bCombined,
+ nbnxn_alloc_t *alloc,
+ nbnxn_free_t *free)
+{
+ int i;
+
+ nbl_list->bSimple = bSimple;
+ nbl_list->bCombined = bCombined;
+
+ nbl_list->nnbl = gmx_omp_nthreads_get(emntNonbonded);
+
+ if (!nbl_list->bCombined &&
+ nbl_list->nnbl > NBNXN_BUFFERFLAG_MAX_THREADS)
+ {
+ gmx_fatal(FARGS, "%d OpenMP threads were requested. Since the non-bonded force buffer reduction is prohibitively slow with more than %d threads, we do not allow this. Use %d or less OpenMP threads.",
+ nbl_list->nnbl, NBNXN_BUFFERFLAG_MAX_THREADS, NBNXN_BUFFERFLAG_MAX_THREADS);
+ }
+
+ snew(nbl_list->nbl, nbl_list->nnbl);
+ /* Execute in order to avoid memory interleaving between threads */
+#pragma omp parallel for num_threads(nbl_list->nnbl) schedule(static)
+ for (i = 0; i < nbl_list->nnbl; i++)
+ {
+ /* Allocate the nblist data structure locally on each thread
+ * to optimize memory access for NUMA architectures.
+ */
+ snew(nbl_list->nbl[i], 1);
+
+ /* Only list 0 is used on the GPU, use normal allocation for i>0 */
+ if (i == 0)
+ {
+ nbnxn_init_pairlist(nbl_list->nbl[i], nbl_list->bSimple, alloc, free);
+ }
+ else
+ {
+ nbnxn_init_pairlist(nbl_list->nbl[i], nbl_list->bSimple, NULL, NULL);
+ }
+ }
+}
+
+/* Print statistics of a pair list, used for debug output */
+static void print_nblist_statistics_simple(FILE *fp, const nbnxn_pairlist_t *nbl,
+ const nbnxn_search_t nbs, real rl)
+{
+ const nbnxn_grid_t *grid;
+ int cs[SHIFTS];
+ int s, i, j;
+ int npexcl;
+
+ /* This code only produces correct statistics with domain decomposition */
+ grid = &nbs->grid[0];
+
+ fprintf(fp, "nbl nci %d ncj %d\n",
+ nbl->nci, nbl->ncj);
+ fprintf(fp, "nbl na_sc %d rl %g ncp %d per cell %.1f atoms %.1f ratio %.2f\n",
+ nbl->na_sc, rl, nbl->ncj, nbl->ncj/(double)grid->nc,
+ nbl->ncj/(double)grid->nc*grid->na_sc,
+ nbl->ncj/(double)grid->nc*grid->na_sc/(0.5*4.0/3.0*M_PI*rl*rl*rl*grid->nc*grid->na_sc/det(nbs->box)));
+
+ fprintf(fp, "nbl average j cell list length %.1f\n",
+ 0.25*nbl->ncj/(double)nbl->nci);
+
+ for (s = 0; s < SHIFTS; s++)
+ {
+ cs[s] = 0;
+ }
+ npexcl = 0;
+ for (i = 0; i < nbl->nci; i++)
+ {
+ cs[nbl->ci[i].shift & NBNXN_CI_SHIFT] +=
+ nbl->ci[i].cj_ind_end - nbl->ci[i].cj_ind_start;
+
+ j = nbl->ci[i].cj_ind_start;
+ while (j < nbl->ci[i].cj_ind_end &&
+ nbl->cj[j].excl != NBNXN_INTERACTION_MASK_ALL)
+ {
+ npexcl++;
+ j++;
+ }
+ }
+ fprintf(fp, "nbl cell pairs, total: %d excl: %d %.1f%%\n",
+ nbl->ncj, npexcl, 100*npexcl/(double)nbl->ncj);
+ for (s = 0; s < SHIFTS; s++)
+ {
+ if (cs[s] > 0)
+ {
+ fprintf(fp, "nbl shift %2d ncj %3d\n", s, cs[s]);
+ }
+ }
+}
+
+/* Print statistics of a pair lists, used for debug output */
+static void print_nblist_statistics_supersub(FILE *fp, const nbnxn_pairlist_t *nbl,
+ const nbnxn_search_t nbs, real rl)
+{
+ const nbnxn_grid_t *grid;
+ int i, j4, j, si, b;
+ int c[GPU_NSUBCELL+1];
+
+ /* This code only produces correct statistics with domain decomposition */
+ grid = &nbs->grid[0];
+
+ fprintf(fp, "nbl nsci %d ncj4 %d nsi %d excl4 %d\n",
+ nbl->nsci, nbl->ncj4, nbl->nci_tot, nbl->nexcl);
+ fprintf(fp, "nbl na_c %d rl %g ncp %d per cell %.1f atoms %.1f ratio %.2f\n",
+ nbl->na_ci, rl, nbl->nci_tot, nbl->nci_tot/(double)grid->nsubc_tot,
+ nbl->nci_tot/(double)grid->nsubc_tot*grid->na_c,
+ nbl->nci_tot/(double)grid->nsubc_tot*grid->na_c/(0.5*4.0/3.0*M_PI*rl*rl*rl*grid->nsubc_tot*grid->na_c/det(nbs->box)));
+
+ fprintf(fp, "nbl average j super cell list length %.1f\n",
+ 0.25*nbl->ncj4/(double)nbl->nsci);
+ fprintf(fp, "nbl average i sub cell list length %.1f\n",
+ nbl->nci_tot/((double)nbl->ncj4));
+
+ for (si = 0; si <= GPU_NSUBCELL; si++)
+ {
+ c[si] = 0;
+ }
+ for (i = 0; i < nbl->nsci; i++)
+ {
+ for (j4 = nbl->sci[i].cj4_ind_start; j4 < nbl->sci[i].cj4_ind_end; j4++)
+ {
+ for (j = 0; j < NBNXN_GPU_JGROUP_SIZE; j++)
+ {
+ b = 0;
+ for (si = 0; si < GPU_NSUBCELL; si++)
+ {
+ if (nbl->cj4[j4].imei[0].imask & (1U << (j*GPU_NSUBCELL + si)))
+ {
+ b++;
+ }
+ }
+ c[b]++;
+ }
+ }
+ }
+ for (b = 0; b <= GPU_NSUBCELL; b++)
+ {
+ fprintf(fp, "nbl j-list #i-subcell %d %7d %4.1f\n",
+ b, c[b], 100.0*c[b]/(double)(nbl->ncj4*NBNXN_GPU_JGROUP_SIZE));
+ }
+}
+
+/* Returns a pointer to the exclusion mask for cj4-unit cj4, warp warp */
+static void low_get_nbl_exclusions(nbnxn_pairlist_t *nbl, int cj4,
+ int warp, nbnxn_excl_t **excl)
+{
+ if (nbl->cj4[cj4].imei[warp].excl_ind == 0)
+ {
+ /* No exclusions set, make a new list entry */
+ nbl->cj4[cj4].imei[warp].excl_ind = nbl->nexcl;
+ nbl->nexcl++;
+ *excl = &nbl->excl[nbl->cj4[cj4].imei[warp].excl_ind];
+ set_no_excls(*excl);
+ }
+ else
+ {
+ /* We already have some exclusions, new ones can be added to the list */
+ *excl = &nbl->excl[nbl->cj4[cj4].imei[warp].excl_ind];
+ }
+}
+
+/* Returns a pointer to the exclusion mask for cj4-unit cj4, warp warp,
+ * allocates extra memory, if necessary.
+ */
+static void get_nbl_exclusions_1(nbnxn_pairlist_t *nbl, int cj4,
+ int warp, nbnxn_excl_t **excl)
+{
+ if (nbl->cj4[cj4].imei[warp].excl_ind == 0)
+ {
+ /* We need to make a new list entry, check if we have space */
+ check_excl_space(nbl, 1);
+ }
+ low_get_nbl_exclusions(nbl, cj4, warp, excl);
+}
+
+/* Returns pointers to the exclusion mask for cj4-unit cj4 for both warps,
+ * allocates extra memory, if necessary.
+ */
+static void get_nbl_exclusions_2(nbnxn_pairlist_t *nbl, int cj4,
+ nbnxn_excl_t **excl_w0,
+ nbnxn_excl_t **excl_w1)
+{
+ /* Check for space we might need */
+ check_excl_space(nbl, 2);
+
+ low_get_nbl_exclusions(nbl, cj4, 0, excl_w0);
+ low_get_nbl_exclusions(nbl, cj4, 1, excl_w1);
+}
+
+/* Sets the self exclusions i=j and pair exclusions i>j */
+static void set_self_and_newton_excls_supersub(nbnxn_pairlist_t *nbl,
+ int cj4_ind, int sj_offset,
+ int si)
+{
+ nbnxn_excl_t *excl[2];
+ int ei, ej, w;
+
+ /* Here we only set the set self and double pair exclusions */
+
+ get_nbl_exclusions_2(nbl, cj4_ind, &excl[0], &excl[1]);
+
+ /* Only minor < major bits set */
+ for (ej = 0; ej < nbl->na_ci; ej++)
+ {
+ w = (ej>>2);
+ for (ei = ej; ei < nbl->na_ci; ei++)
+ {
+ excl[w]->pair[(ej & (NBNXN_GPU_JGROUP_SIZE-1))*nbl->na_ci + ei] &=
+ ~(1U << (sj_offset*GPU_NSUBCELL + si));
+ }
+ }
+}
+
+/* Returns a diagonal or off-diagonal interaction mask for plain C lists */
+static unsigned int get_imask(gmx_bool rdiag, int ci, int cj)
+{
+ return (rdiag && ci == cj ? NBNXN_INTERACTION_MASK_DIAG : NBNXN_INTERACTION_MASK_ALL);
+}
+
+/* Returns a diagonal or off-diagonal interaction mask for cj-size=2 */
+static unsigned int get_imask_simd_j2(gmx_bool rdiag, int ci, int cj)
+{
+ return (rdiag && ci*2 == cj ? NBNXN_INTERACTION_MASK_DIAG_J2_0 :
+ (rdiag && ci*2+1 == cj ? NBNXN_INTERACTION_MASK_DIAG_J2_1 :
+ NBNXN_INTERACTION_MASK_ALL));
+}
+
+/* Returns a diagonal or off-diagonal interaction mask for cj-size=4 */
+static unsigned int get_imask_simd_j4(gmx_bool rdiag, int ci, int cj)
+{
+ return (rdiag && ci == cj ? NBNXN_INTERACTION_MASK_DIAG : NBNXN_INTERACTION_MASK_ALL);
+}
+
+/* Returns a diagonal or off-diagonal interaction mask for cj-size=8 */
+static unsigned int get_imask_simd_j8(gmx_bool rdiag, int ci, int cj)
+{
+ return (rdiag && ci == cj*2 ? NBNXN_INTERACTION_MASK_DIAG_J8_0 :
+ (rdiag && ci == cj*2+1 ? NBNXN_INTERACTION_MASK_DIAG_J8_1 :
+ NBNXN_INTERACTION_MASK_ALL));
+}
+
+#ifdef GMX_NBNXN_SIMD
+#if GMX_SIMD_WIDTH_HERE == 2
+#define get_imask_simd_4xn get_imask_simd_j2
+#endif
+#if GMX_SIMD_WIDTH_HERE == 4
+#define get_imask_simd_4xn get_imask_simd_j4
+#endif
+#if GMX_SIMD_WIDTH_HERE == 8
+#define get_imask_simd_4xn get_imask_simd_j8
+#define get_imask_simd_2xnn get_imask_simd_j4
+#endif
+#if GMX_SIMD_WIDTH_HERE == 16
+#define get_imask_simd_2xnn get_imask_simd_j8
+#endif
+#endif
+
+/* Plain C code for making a pair list of cell ci vs cell cjf-cjl.
+ * Checks bounding box distances and possibly atom pair distances.
+ */
+static void make_cluster_list_simple(const nbnxn_grid_t *gridj,
+ nbnxn_pairlist_t *nbl,
+ int ci, int cjf, int cjl,
+ gmx_bool remove_sub_diag,
+ const real *x_j,
+ real rl2, float rbb2,
+ int *ndistc)
+{
+ const nbnxn_list_work_t *work;
+
+ const nbnxn_bb_t *bb_ci;
+ const real *x_ci;
+
+ gmx_bool InRange;
+ real d2;
+ int cjf_gl, cjl_gl, cj;
+
+ work = nbl->work;
+
+ bb_ci = nbl->work->bb_ci;
+ x_ci = nbl->work->x_ci;
+
+ InRange = FALSE;
+ while (!InRange && cjf <= cjl)
+ {
+ d2 = subc_bb_dist2(0, bb_ci, cjf, gridj->bb);
+ *ndistc += 2;
+
+ /* Check if the distance is within the distance where
+ * we use only the bounding box distance rbb,
+ * or within the cut-off and there is at least one atom pair
+ * within the cut-off.
+ */
+ if (d2 < rbb2)
+ {
+ InRange = TRUE;
+ }
+ else if (d2 < rl2)
+ {
+ int i, j;
+
+ cjf_gl = gridj->cell0 + cjf;
+ for (i = 0; i < NBNXN_CPU_CLUSTER_I_SIZE && !InRange; i++)
+ {
+ for (j = 0; j < NBNXN_CPU_CLUSTER_I_SIZE; j++)
+ {
+ InRange = InRange ||
+ (sqr(x_ci[i*STRIDE_XYZ+XX] - x_j[(cjf_gl*NBNXN_CPU_CLUSTER_I_SIZE+j)*STRIDE_XYZ+XX]) +
+ sqr(x_ci[i*STRIDE_XYZ+YY] - x_j[(cjf_gl*NBNXN_CPU_CLUSTER_I_SIZE+j)*STRIDE_XYZ+YY]) +
+ sqr(x_ci[i*STRIDE_XYZ+ZZ] - x_j[(cjf_gl*NBNXN_CPU_CLUSTER_I_SIZE+j)*STRIDE_XYZ+ZZ]) < rl2);
+ }
+ }
+ *ndistc += NBNXN_CPU_CLUSTER_I_SIZE*NBNXN_CPU_CLUSTER_I_SIZE;
+ }
+ if (!InRange)
+ {
+ cjf++;
+ }
+ }
+ if (!InRange)
+ {
+ return;
+ }
+
+ InRange = FALSE;
+ while (!InRange && cjl > cjf)
+ {
+ d2 = subc_bb_dist2(0, bb_ci, cjl, gridj->bb);
+ *ndistc += 2;
+
+ /* Check if the distance is within the distance where
+ * we use only the bounding box distance rbb,
+ * or within the cut-off and there is at least one atom pair
+ * within the cut-off.
+ */
+ if (d2 < rbb2)
+ {
+ InRange = TRUE;
+ }
+ else if (d2 < rl2)
+ {
+ int i, j;
+
+ cjl_gl = gridj->cell0 + cjl;
+ for (i = 0; i < NBNXN_CPU_CLUSTER_I_SIZE && !InRange; i++)
+ {
+ for (j = 0; j < NBNXN_CPU_CLUSTER_I_SIZE; j++)
+ {
+ InRange = InRange ||
+ (sqr(x_ci[i*STRIDE_XYZ+XX] - x_j[(cjl_gl*NBNXN_CPU_CLUSTER_I_SIZE+j)*STRIDE_XYZ+XX]) +
+ sqr(x_ci[i*STRIDE_XYZ+YY] - x_j[(cjl_gl*NBNXN_CPU_CLUSTER_I_SIZE+j)*STRIDE_XYZ+YY]) +
+ sqr(x_ci[i*STRIDE_XYZ+ZZ] - x_j[(cjl_gl*NBNXN_CPU_CLUSTER_I_SIZE+j)*STRIDE_XYZ+ZZ]) < rl2);
+ }
+ }
+ *ndistc += NBNXN_CPU_CLUSTER_I_SIZE*NBNXN_CPU_CLUSTER_I_SIZE;
+ }
+ if (!InRange)
+ {
+ cjl--;
+ }
+ }
+
+ if (cjf <= cjl)
+ {
+ for (cj = cjf; cj <= cjl; cj++)
+ {
+ /* Store cj and the interaction mask */
+ nbl->cj[nbl->ncj].cj = gridj->cell0 + cj;
+ nbl->cj[nbl->ncj].excl = get_imask(remove_sub_diag, ci, cj);
+ nbl->ncj++;
+ }
+ /* Increase the closing index in i super-cell list */
+ nbl->ci[nbl->nci].cj_ind_end = nbl->ncj;
+ }
+}
+
+#ifdef GMX_NBNXN_SIMD_4XN
+#include "nbnxn_search_simd_4xn.h"
+#endif
+#ifdef GMX_NBNXN_SIMD_2XNN
+#include "nbnxn_search_simd_2xnn.h"
+#endif
+
- /* Determine all ci1 bb distances in one call with SSE */
- subc_bb_dist2_sse_xxxx(gridj->pbb+(cj>>STRIDE_PBB_2LOG)*NNBSBB_XXXX+(cj & (STRIDE_PBB-1)),
++/* Plain C or SIMD4 code for making a pair list of super-cell sci vs scj.
+ * Checks bounding box distances and possibly atom pair distances.
+ */
+static void make_cluster_list_supersub(const nbnxn_grid_t *gridi,
+ const nbnxn_grid_t *gridj,
+ nbnxn_pairlist_t *nbl,
+ int sci, int scj,
+ gmx_bool sci_equals_scj,
+ int stride, const real *x,
+ real rl2, float rbb2,
+ int *ndistc)
+{
+ int na_c;
+ int npair;
+ int cjo, ci1, ci, cj, cj_gl;
+ int cj4_ind, cj_offset;
+ unsigned imask;
+ nbnxn_cj4_t *cj4;
+#ifdef NBNXN_BBXXXX
+ const float *pbb_ci;
+#else
+ const nbnxn_bb_t *bb_ci;
+#endif
+ const real *x_ci;
+ float *d2l, d2;
+ int w;
+#define PRUNE_LIST_CPU_ONE
+#ifdef PRUNE_LIST_CPU_ONE
+ int ci_last = -1;
+#endif
+
+ d2l = nbl->work->d2;
+
+#ifdef NBNXN_BBXXXX
+ pbb_ci = nbl->work->pbb_ci;
+#else
+ bb_ci = nbl->work->bb_ci;
+#endif
+ x_ci = nbl->work->x_ci;
+
+ na_c = gridj->na_c;
+
+ for (cjo = 0; cjo < gridj->nsubc[scj]; cjo++)
+ {
+ cj4_ind = (nbl->work->cj_ind >> NBNXN_GPU_JGROUP_SIZE_2LOG);
+ cj_offset = nbl->work->cj_ind - cj4_ind*NBNXN_GPU_JGROUP_SIZE;
+ cj4 = &nbl->cj4[cj4_ind];
+
+ cj = scj*GPU_NSUBCELL + cjo;
+
+ cj_gl = gridj->cell0*GPU_NSUBCELL + cj;
+
+ /* Initialize this j-subcell i-subcell list */
+ cj4->cj[cj_offset] = cj_gl;
+ imask = 0;
+
+ if (sci_equals_scj)
+ {
+ ci1 = cjo + 1;
+ }
+ else
+ {
+ ci1 = gridi->nsubc[sci];
+ }
+
+#ifdef NBNXN_BBXXXX
- #ifdef NBNXN_PBB_SSE
- subc_in_range_sse8
++ /* Determine all ci1 bb distances in one call with SIMD4 */
++ subc_bb_dist2_simd4_xxxx(gridj->pbb+(cj>>STRIDE_PBB_2LOG)*NNBSBB_XXXX+(cj & (STRIDE_PBB-1)),
+ ci1, pbb_ci, d2l);
+ *ndistc += na_c*2;
+#endif
+
+ npair = 0;
+ /* We use a fixed upper-bound instead of ci1 to help optimization */
+ for (ci = 0; ci < GPU_NSUBCELL; ci++)
+ {
+ if (ci == ci1)
+ {
+ break;
+ }
+
+#ifndef NBNXN_BBXXXX
+ /* Determine the bb distance between ci and cj */
+ d2l[ci] = subc_bb_dist2(ci, bb_ci, cj, gridj->bb);
+ *ndistc += 2;
+#endif
+ d2 = d2l[ci];
+
+#ifdef PRUNE_LIST_CPU_ALL
+ /* Check if the distance is within the distance where
+ * we use only the bounding box distance rbb,
+ * or within the cut-off and there is at least one atom pair
+ * within the cut-off. This check is very costly.
+ */
+ *ndistc += na_c*na_c;
+ if (d2 < rbb2 ||
+ (d2 < rl2 &&
- #ifdef NBNXN_PBB_SSE
- !subc_in_range_sse8
++#ifdef NBNXN_PBB_SIMD4
++ subc_in_range_simd4
+#else
+ subc_in_range_x
+#endif
+ (na_c, ci, x_ci, cj_gl, stride, x, rl2)))
+#else
+ /* Check if the distance between the two bounding boxes
+ * in within the pair-list cut-off.
+ */
+ if (d2 < rl2)
+#endif
+ {
+ /* Flag this i-subcell to be taken into account */
+ imask |= (1U << (cj_offset*GPU_NSUBCELL+ci));
+
+#ifdef PRUNE_LIST_CPU_ONE
+ ci_last = ci;
+#endif
+
+ npair++;
+ }
+ }
+
+#ifdef PRUNE_LIST_CPU_ONE
+ /* If we only found 1 pair, check if any atoms are actually
+ * within the cut-off, so we could get rid of it.
+ */
+ if (npair == 1 && d2l[ci_last] >= rbb2)
+ {
+ /* Avoid using function pointers here, as it's slower */
+ if (
- #ifdef NBNXN_SEARCH_BB_SSE
++#ifdef NBNXN_PBB_SIMD4
++ !subc_in_range_simd4
+#else
+ !subc_in_range_x
+#endif
+ (na_c, ci_last, x_ci, cj_gl, stride, x, rl2))
+ {
+ imask &= ~(1U << (cj_offset*GPU_NSUBCELL+ci_last));
+ npair--;
+ }
+ }
+#endif
+
+ if (npair > 0)
+ {
+ /* We have a useful sj entry, close it now */
+
+ /* Set the exclucions for the ci== sj entry.
+ * Here we don't bother to check if this entry is actually flagged,
+ * as it will nearly always be in the list.
+ */
+ if (sci_equals_scj)
+ {
+ set_self_and_newton_excls_supersub(nbl, cj4_ind, cj_offset, cjo);
+ }
+
+ /* Copy the cluster interaction mask to the list */
+ for (w = 0; w < NWARP; w++)
+ {
+ cj4->imei[w].imask |= imask;
+ }
+
+ nbl->work->cj_ind++;
+
+ /* Keep the count */
+ nbl->nci_tot += npair;
+
+ /* Increase the closing index in i super-cell list */
+ nbl->sci[nbl->nsci].cj4_ind_end =
+ ((nbl->work->cj_ind+NBNXN_GPU_JGROUP_SIZE-1) >> NBNXN_GPU_JGROUP_SIZE_2LOG);
+ }
+ }
+}
+
+/* Set all atom-pair exclusions from the topology stored in excl
+ * as masks in the pair-list for simple list i-entry nbl_ci
+ */
+static void set_ci_top_excls(const nbnxn_search_t nbs,
+ nbnxn_pairlist_t *nbl,
+ gmx_bool diagRemoved,
+ int na_ci_2log,
+ int na_cj_2log,
+ const nbnxn_ci_t *nbl_ci,
+ const t_blocka *excl)
+{
+ const int *cell;
+ int ci;
+ int cj_ind_first, cj_ind_last;
+ int cj_first, cj_last;
+ int ndirect;
+ int i, ai, aj, si, eind, ge, se;
+ int found, cj_ind_0, cj_ind_1, cj_ind_m;
+ int cj_m;
+ gmx_bool Found_si;
+ int si_ind;
+ nbnxn_excl_t *nbl_excl;
+ int inner_i, inner_e;
+
+ cell = nbs->cell;
+
+ if (nbl_ci->cj_ind_end == nbl_ci->cj_ind_start)
+ {
+ /* Empty list */
+ return;
+ }
+
+ ci = nbl_ci->ci;
+
+ cj_ind_first = nbl_ci->cj_ind_start;
+ cj_ind_last = nbl->ncj - 1;
+
+ cj_first = nbl->cj[cj_ind_first].cj;
+ cj_last = nbl->cj[cj_ind_last].cj;
+
+ /* Determine how many contiguous j-cells we have starting
+ * from the first i-cell. This number can be used to directly
+ * calculate j-cell indices for excluded atoms.
+ */
+ ndirect = 0;
+ if (na_ci_2log == na_cj_2log)
+ {
+ while (cj_ind_first + ndirect <= cj_ind_last &&
+ nbl->cj[cj_ind_first+ndirect].cj == ci + ndirect)
+ {
+ ndirect++;
+ }
+ }
- #ifdef NBNXN_SEARCH_BB_SSE
++#ifdef NBNXN_SEARCH_BB_SIMD4
+ else
+ {
+ while (cj_ind_first + ndirect <= cj_ind_last &&
+ nbl->cj[cj_ind_first+ndirect].cj == ci_to_cj(na_cj_2log, ci) + ndirect)
+ {
+ ndirect++;
+ }
+ }
+#endif
+
+ /* Loop over the atoms in the i super-cell */
+ for (i = 0; i < nbl->na_sc; i++)
+ {
+ ai = nbs->a[ci*nbl->na_sc+i];
+ if (ai >= 0)
+ {
+ si = (i>>na_ci_2log);
+
+ /* Loop over the topology-based exclusions for this i-atom */
+ for (eind = excl->index[ai]; eind < excl->index[ai+1]; eind++)
+ {
+ aj = excl->a[eind];
+
+ if (aj == ai)
+ {
+ /* The self exclusion are already set, save some time */
+ continue;
+ }
+
+ ge = cell[aj];
+
+ /* Without shifts we only calculate interactions j>i
+ * for one-way pair-lists.
+ */
+ if (diagRemoved && ge <= ci*nbl->na_sc + i)
+ {
+ continue;
+ }
+
+ se = (ge >> na_cj_2log);
+
+ /* Could the cluster se be in our list? */
+ if (se >= cj_first && se <= cj_last)
+ {
+ if (se < cj_first + ndirect)
+ {
+ /* We can calculate cj_ind directly from se */
+ found = cj_ind_first + se - cj_first;
+ }
+ else
+ {
+ /* Search for se using bisection */
+ found = -1;
+ cj_ind_0 = cj_ind_first + ndirect;
+ cj_ind_1 = cj_ind_last + 1;
+ while (found == -1 && cj_ind_0 < cj_ind_1)
+ {
+ cj_ind_m = (cj_ind_0 + cj_ind_1)>>1;
+
+ cj_m = nbl->cj[cj_ind_m].cj;
+
+ if (se == cj_m)
+ {
+ found = cj_ind_m;
+ }
+ else if (se < cj_m)
+ {
+ cj_ind_1 = cj_ind_m;
+ }
+ else
+ {
+ cj_ind_0 = cj_ind_m + 1;
+ }
+ }
+ }
+
+ if (found >= 0)
+ {
+ inner_i = i - (si << na_ci_2log);
+ inner_e = ge - (se << na_cj_2log);
+
+ nbl->cj[found].excl &= ~(1U<<((inner_i<<na_cj_2log) + inner_e));
++/* The next code line is usually not needed. We do not want to version
++ * away the above line, because there is logic that relies on being
++ * able to detect easily whether any exclusions exist. */
++#if (defined GMX_CPU_ACCELERATION_IBM_QPX)
++ nbl->cj[found].interaction_mask_indices[inner_i] &= ~(1U << inner_e);
++#endif
+ }
+ }
+ }
+ }
+ }
+}
+
+/* Set all atom-pair exclusions from the topology stored in excl
+ * as masks in the pair-list for i-super-cell entry nbl_sci
+ */
+static void set_sci_top_excls(const nbnxn_search_t nbs,
+ nbnxn_pairlist_t *nbl,
+ gmx_bool diagRemoved,
+ int na_c_2log,
+ const nbnxn_sci_t *nbl_sci,
+ const t_blocka *excl)
+{
+ const int *cell;
+ int na_c;
+ int sci;
+ int cj_ind_first, cj_ind_last;
+ int cj_first, cj_last;
+ int ndirect;
+ int i, ai, aj, si, eind, ge, se;
+ int found, cj_ind_0, cj_ind_1, cj_ind_m;
+ int cj_m;
+ gmx_bool Found_si;
+ int si_ind;
+ nbnxn_excl_t *nbl_excl;
+ int inner_i, inner_e, w;
+
+ cell = nbs->cell;
+
+ na_c = nbl->na_ci;
+
+ if (nbl_sci->cj4_ind_end == nbl_sci->cj4_ind_start)
+ {
+ /* Empty list */
+ return;
+ }
+
+ sci = nbl_sci->sci;
+
+ cj_ind_first = nbl_sci->cj4_ind_start*NBNXN_GPU_JGROUP_SIZE;
+ cj_ind_last = nbl->work->cj_ind - 1;
+
+ cj_first = nbl->cj4[nbl_sci->cj4_ind_start].cj[0];
+ cj_last = nbl_cj(nbl, cj_ind_last);
+
+ /* Determine how many contiguous j-clusters we have starting
+ * from the first i-cluster. This number can be used to directly
+ * calculate j-cluster indices for excluded atoms.
+ */
+ ndirect = 0;
+ while (cj_ind_first + ndirect <= cj_ind_last &&
+ nbl_cj(nbl, cj_ind_first+ndirect) == sci*GPU_NSUBCELL + ndirect)
+ {
+ ndirect++;
+ }
+
+ /* Loop over the atoms in the i super-cell */
+ for (i = 0; i < nbl->na_sc; i++)
+ {
+ ai = nbs->a[sci*nbl->na_sc+i];
+ if (ai >= 0)
+ {
+ si = (i>>na_c_2log);
+
+ /* Loop over the topology-based exclusions for this i-atom */
+ for (eind = excl->index[ai]; eind < excl->index[ai+1]; eind++)
+ {
+ aj = excl->a[eind];
+
+ if (aj == ai)
+ {
+ /* The self exclusion are already set, save some time */
+ continue;
+ }
+
+ ge = cell[aj];
+
+ /* Without shifts we only calculate interactions j>i
+ * for one-way pair-lists.
+ */
+ if (diagRemoved && ge <= sci*nbl->na_sc + i)
+ {
+ continue;
+ }
+
+ se = ge>>na_c_2log;
+ /* Could the cluster se be in our list? */
+ if (se >= cj_first && se <= cj_last)
+ {
+ if (se < cj_first + ndirect)
+ {
+ /* We can calculate cj_ind directly from se */
+ found = cj_ind_first + se - cj_first;
+ }
+ else
+ {
+ /* Search for se using bisection */
+ found = -1;
+ cj_ind_0 = cj_ind_first + ndirect;
+ cj_ind_1 = cj_ind_last + 1;
+ while (found == -1 && cj_ind_0 < cj_ind_1)
+ {
+ cj_ind_m = (cj_ind_0 + cj_ind_1)>>1;
+
+ cj_m = nbl_cj(nbl, cj_ind_m);
+
+ if (se == cj_m)
+ {
+ found = cj_ind_m;
+ }
+ else if (se < cj_m)
+ {
+ cj_ind_1 = cj_ind_m;
+ }
+ else
+ {
+ cj_ind_0 = cj_ind_m + 1;
+ }
+ }
+ }
+
+ if (found >= 0)
+ {
+ inner_i = i - si*na_c;
+ inner_e = ge - se*na_c;
+
+/* Macro for getting the index of atom a within a cluster */
+#define AMODCJ4(a) ((a) & (NBNXN_GPU_JGROUP_SIZE - 1))
+/* Macro for converting an atom number to a cluster number */
+#define A2CJ4(a) ((a) >> NBNXN_GPU_JGROUP_SIZE_2LOG)
+/* Macro for getting the index of an i-atom within a warp */
+#define AMODWI(a) ((a) & (NBNXN_GPU_CLUSTER_SIZE/2 - 1))
+
+ if (nbl_imask0(nbl, found) & (1U << (AMODCJ4(found)*GPU_NSUBCELL + si)))
+ {
+ w = (inner_e >> 2);
+
+ get_nbl_exclusions_1(nbl, A2CJ4(found), w, &nbl_excl);
+
+ nbl_excl->pair[AMODWI(inner_e)*nbl->na_ci+inner_i] &=
+ ~(1U << (AMODCJ4(found)*GPU_NSUBCELL + si));
+ }
+
+#undef AMODCJ4
+#undef A2CJ4
+#undef AMODWI
+ }
+ }
+ }
+ }
+ }
+}
+
+/* Reallocate the simple ci list for at least n entries */
+static void nb_realloc_ci(nbnxn_pairlist_t *nbl, int n)
+{
+ nbl->ci_nalloc = over_alloc_small(n);
+ nbnxn_realloc_void((void **)&nbl->ci,
+ nbl->nci*sizeof(*nbl->ci),
+ nbl->ci_nalloc*sizeof(*nbl->ci),
+ nbl->alloc, nbl->free);
+}
+
+/* Reallocate the super-cell sci list for at least n entries */
+static void nb_realloc_sci(nbnxn_pairlist_t *nbl, int n)
+{
+ nbl->sci_nalloc = over_alloc_small(n);
+ nbnxn_realloc_void((void **)&nbl->sci,
+ nbl->nsci*sizeof(*nbl->sci),
+ nbl->sci_nalloc*sizeof(*nbl->sci),
+ nbl->alloc, nbl->free);
+}
+
+/* Make a new ci entry at index nbl->nci */
+static void new_ci_entry(nbnxn_pairlist_t *nbl, int ci, int shift, int flags)
+{
+ if (nbl->nci + 1 > nbl->ci_nalloc)
+ {
+ nb_realloc_ci(nbl, nbl->nci+1);
+ }
+ nbl->ci[nbl->nci].ci = ci;
+ nbl->ci[nbl->nci].shift = shift;
+ /* Store the interaction flags along with the shift */
+ nbl->ci[nbl->nci].shift |= flags;
+ nbl->ci[nbl->nci].cj_ind_start = nbl->ncj;
+ nbl->ci[nbl->nci].cj_ind_end = nbl->ncj;
+}
+
+/* Make a new sci entry at index nbl->nsci */
+static void new_sci_entry(nbnxn_pairlist_t *nbl, int sci, int shift)
+{
+ if (nbl->nsci + 1 > nbl->sci_nalloc)
+ {
+ nb_realloc_sci(nbl, nbl->nsci+1);
+ }
+ nbl->sci[nbl->nsci].sci = sci;
+ nbl->sci[nbl->nsci].shift = shift;
+ nbl->sci[nbl->nsci].cj4_ind_start = nbl->ncj4;
+ nbl->sci[nbl->nsci].cj4_ind_end = nbl->ncj4;
+}
+
+/* Sort the simple j-list cj on exclusions.
+ * Entries with exclusions will all be sorted to the beginning of the list.
+ */
+static void sort_cj_excl(nbnxn_cj_t *cj, int ncj,
+ nbnxn_list_work_t *work)
+{
+ int jnew, j;
+
+ if (ncj > work->cj_nalloc)
+ {
+ work->cj_nalloc = over_alloc_large(ncj);
+ srenew(work->cj, work->cj_nalloc);
+ }
+
+ /* Make a list of the j-cells involving exclusions */
+ jnew = 0;
+ for (j = 0; j < ncj; j++)
+ {
+ if (cj[j].excl != NBNXN_INTERACTION_MASK_ALL)
+ {
+ work->cj[jnew++] = cj[j];
+ }
+ }
+ /* Check if there are exclusions at all or not just the first entry */
+ if (!((jnew == 0) ||
+ (jnew == 1 && cj[0].excl != NBNXN_INTERACTION_MASK_ALL)))
+ {
+ for (j = 0; j < ncj; j++)
+ {
+ if (cj[j].excl == NBNXN_INTERACTION_MASK_ALL)
+ {
+ work->cj[jnew++] = cj[j];
+ }
+ }
+ for (j = 0; j < ncj; j++)
+ {
+ cj[j] = work->cj[j];
+ }
+ }
+}
+
+/* Close this simple list i entry */
+static void close_ci_entry_simple(nbnxn_pairlist_t *nbl)
+{
+ int jlen;
+
+ /* All content of the new ci entry have already been filled correctly,
+ * we only need to increase the count here (for non empty lists).
+ */
+ jlen = nbl->ci[nbl->nci].cj_ind_end - nbl->ci[nbl->nci].cj_ind_start;
+ if (jlen > 0)
+ {
+ sort_cj_excl(nbl->cj+nbl->ci[nbl->nci].cj_ind_start, jlen, nbl->work);
+
+ /* The counts below are used for non-bonded pair/flop counts
+ * and should therefore match the available kernel setups.
+ */
+ if (!(nbl->ci[nbl->nci].shift & NBNXN_CI_DO_COUL(0)))
+ {
+ nbl->work->ncj_noq += jlen;
+ }
+ else if ((nbl->ci[nbl->nci].shift & NBNXN_CI_HALF_LJ(0)) ||
+ !(nbl->ci[nbl->nci].shift & NBNXN_CI_DO_LJ(0)))
+ {
+ nbl->work->ncj_hlj += jlen;
+ }
+
+ nbl->nci++;
+ }
+}
+
+/* Split sci entry for load balancing on the GPU.
+ * Splitting ensures we have enough lists to fully utilize the whole GPU.
+ * With progBal we generate progressively smaller lists, which improves
+ * load balancing. As we only know the current count on our own thread,
+ * we will need to estimate the current total amount of i-entries.
+ * As the lists get concatenated later, this estimate depends
+ * both on nthread and our own thread index.
+ */
+static void split_sci_entry(nbnxn_pairlist_t *nbl,
+ int nsp_max_av, gmx_bool progBal, int nc_bal,
+ int thread, int nthread)
+{
+ int nsci_est;
+ int nsp_max;
+ int cj4_start, cj4_end, j4len, cj4;
+ int sci;
+ int nsp, nsp_sci, nsp_cj4, nsp_cj4_e, nsp_cj4_p;
+ int p;
+
+ if (progBal)
+ {
+ /* Estimate the total numbers of ci's of the nblist combined
+ * over all threads using the target number of ci's.
+ */
+ nsci_est = nc_bal*thread/nthread + nbl->nsci;
+
+ /* The first ci blocks should be larger, to avoid overhead.
+ * The last ci blocks should be smaller, to improve load balancing.
+ */
+ nsp_max = max(1,
+ nsp_max_av*nc_bal*3/(2*(nsci_est - 1 + nc_bal)));
+ }
+ else
+ {
+ nsp_max = nsp_max_av;
+ }
+
+ cj4_start = nbl->sci[nbl->nsci-1].cj4_ind_start;
+ cj4_end = nbl->sci[nbl->nsci-1].cj4_ind_end;
+ j4len = cj4_end - cj4_start;
+
+ if (j4len > 1 && j4len*GPU_NSUBCELL*NBNXN_GPU_JGROUP_SIZE > nsp_max)
+ {
+ /* Remove the last ci entry and process the cj4's again */
+ nbl->nsci -= 1;
+
+ sci = nbl->nsci;
+ nsp = 0;
+ nsp_sci = 0;
+ nsp_cj4_e = 0;
+ nsp_cj4 = 0;
+ for (cj4 = cj4_start; cj4 < cj4_end; cj4++)
+ {
+ nsp_cj4_p = nsp_cj4;
+ /* Count the number of cluster pairs in this cj4 group */
+ nsp_cj4 = 0;
+ for (p = 0; p < GPU_NSUBCELL*NBNXN_GPU_JGROUP_SIZE; p++)
+ {
+ nsp_cj4 += (nbl->cj4[cj4].imei[0].imask >> p) & 1;
+ }
+
+ if (nsp_cj4 > 0 && nsp + nsp_cj4 > nsp_max)
+ {
+ /* Split the list at cj4 */
+ nbl->sci[sci].cj4_ind_end = cj4;
+ /* Create a new sci entry */
+ sci++;
+ nbl->nsci++;
+ if (nbl->nsci+1 > nbl->sci_nalloc)
+ {
+ nb_realloc_sci(nbl, nbl->nsci+1);
+ }
+ nbl->sci[sci].sci = nbl->sci[nbl->nsci-1].sci;
+ nbl->sci[sci].shift = nbl->sci[nbl->nsci-1].shift;
+ nbl->sci[sci].cj4_ind_start = cj4;
+ nsp_sci = nsp;
+ nsp_cj4_e = nsp_cj4_p;
+ nsp = 0;
+ }
+ nsp += nsp_cj4;
+ }
+
+ /* Put the remaining cj4's in the last sci entry */
+ nbl->sci[sci].cj4_ind_end = cj4_end;
+
+ /* Possibly balance out the last two sci's
+ * by moving the last cj4 of the second last sci.
+ */
+ if (nsp_sci - nsp_cj4_e >= nsp + nsp_cj4_e)
+ {
+ nbl->sci[sci-1].cj4_ind_end--;
+ nbl->sci[sci].cj4_ind_start--;
+ }
+
+ nbl->nsci++;
+ }
+}
+
+/* Clost this super/sub list i entry */
+static void close_ci_entry_supersub(nbnxn_pairlist_t *nbl,
+ int nsp_max_av,
+ gmx_bool progBal, int nc_bal,
+ int thread, int nthread)
+{
+ int j4len, tlen;
+ int nb, b;
+
+ /* All content of the new ci entry have already been filled correctly,
+ * we only need to increase the count here (for non empty lists).
+ */
+ j4len = nbl->sci[nbl->nsci].cj4_ind_end - nbl->sci[nbl->nsci].cj4_ind_start;
+ if (j4len > 0)
+ {
+ /* We can only have complete blocks of 4 j-entries in a list,
+ * so round the count up before closing.
+ */
+ nbl->ncj4 = ((nbl->work->cj_ind + NBNXN_GPU_JGROUP_SIZE - 1) >> NBNXN_GPU_JGROUP_SIZE_2LOG);
+ nbl->work->cj_ind = nbl->ncj4*NBNXN_GPU_JGROUP_SIZE;
+
+ nbl->nsci++;
+
+ if (nsp_max_av > 0)
+ {
+ /* Measure the size of the new entry and potentially split it */
+ split_sci_entry(nbl, nsp_max_av, progBal, nc_bal, thread, nthread);
+ }
+ }
+}
+
+/* Syncs the working array before adding another grid pair to the list */
+static void sync_work(nbnxn_pairlist_t *nbl)
+{
+ if (!nbl->bSimple)
+ {
+ nbl->work->cj_ind = nbl->ncj4*NBNXN_GPU_JGROUP_SIZE;
+ nbl->work->cj4_init = nbl->ncj4;
+ }
+}
+
+/* Clears an nbnxn_pairlist_t data structure */
+static void clear_pairlist(nbnxn_pairlist_t *nbl)
+{
+ nbl->nci = 0;
+ nbl->nsci = 0;
+ nbl->ncj = 0;
+ nbl->ncj4 = 0;
+ nbl->nci_tot = 0;
+ nbl->nexcl = 1;
+
+ nbl->work->ncj_noq = 0;
+ nbl->work->ncj_hlj = 0;
+}
+
+/* Sets a simple list i-cell bounding box, including PBC shift */
+static gmx_inline void set_icell_bb_simple(const nbnxn_bb_t *bb, int ci,
+ real shx, real shy, real shz,
+ nbnxn_bb_t *bb_ci)
+{
+ bb_ci->lower[BB_X] = bb[ci].lower[BB_X] + shx;
+ bb_ci->lower[BB_Y] = bb[ci].lower[BB_Y] + shy;
+ bb_ci->lower[BB_Z] = bb[ci].lower[BB_Z] + shz;
+ bb_ci->upper[BB_X] = bb[ci].upper[BB_X] + shx;
+ bb_ci->upper[BB_Y] = bb[ci].upper[BB_Y] + shy;
+ bb_ci->upper[BB_Z] = bb[ci].upper[BB_Z] + shz;
+}
+
+#ifdef NBNXN_BBXXXX
+/* Sets a super-cell and sub cell bounding boxes, including PBC shift */
+static void set_icell_bbxxxx_supersub(const float *bb, int ci,
+ real shx, real shy, real shz,
+ float *bb_ci)
+{
+ int ia, m, i;
+
+ ia = ci*(GPU_NSUBCELL>>STRIDE_PBB_2LOG)*NNBSBB_XXXX;
+ for (m = 0; m < (GPU_NSUBCELL>>STRIDE_PBB_2LOG)*NNBSBB_XXXX; m += NNBSBB_XXXX)
+ {
+ for (i = 0; i < STRIDE_PBB; i++)
+ {
+ bb_ci[m+0*STRIDE_PBB+i] = bb[ia+m+0*STRIDE_PBB+i] + shx;
+ bb_ci[m+1*STRIDE_PBB+i] = bb[ia+m+1*STRIDE_PBB+i] + shy;
+ bb_ci[m+2*STRIDE_PBB+i] = bb[ia+m+2*STRIDE_PBB+i] + shz;
+ bb_ci[m+3*STRIDE_PBB+i] = bb[ia+m+3*STRIDE_PBB+i] + shx;
+ bb_ci[m+4*STRIDE_PBB+i] = bb[ia+m+4*STRIDE_PBB+i] + shy;
+ bb_ci[m+5*STRIDE_PBB+i] = bb[ia+m+5*STRIDE_PBB+i] + shz;
+ }
+ }
+}
+#endif
+
+/* Sets a super-cell and sub cell bounding boxes, including PBC shift */
+static void set_icell_bb_supersub(const nbnxn_bb_t *bb, int ci,
+ real shx, real shy, real shz,
+ nbnxn_bb_t *bb_ci)
+{
+ int i;
+
+ for (i = 0; i < GPU_NSUBCELL; i++)
+ {
+ set_icell_bb_simple(bb, ci*GPU_NSUBCELL+i,
+ shx, shy, shz,
+ &bb_ci[i]);
+ }
+}
+
+/* Copies PBC shifted i-cell atom coordinates x,y,z to working array */
+static void icell_set_x_simple(int ci,
+ real shx, real shy, real shz,
+ int gmx_unused na_c,
+ int stride, const real *x,
+ nbnxn_list_work_t *work)
+{
+ int ia, i;
+
+ ia = ci*NBNXN_CPU_CLUSTER_I_SIZE;
+
+ for (i = 0; i < NBNXN_CPU_CLUSTER_I_SIZE; i++)
+ {
+ work->x_ci[i*STRIDE_XYZ+XX] = x[(ia+i)*stride+XX] + shx;
+ work->x_ci[i*STRIDE_XYZ+YY] = x[(ia+i)*stride+YY] + shy;
+ work->x_ci[i*STRIDE_XYZ+ZZ] = x[(ia+i)*stride+ZZ] + shz;
+ }
+}
+
+/* Copies PBC shifted super-cell atom coordinates x,y,z to working array */
+static void icell_set_x_supersub(int ci,
+ real shx, real shy, real shz,
+ int na_c,
+ int stride, const real *x,
+ nbnxn_list_work_t *work)
+{
+ int ia, i;
+ real *x_ci;
+
+ x_ci = work->x_ci;
+
+ ia = ci*GPU_NSUBCELL*na_c;
+ for (i = 0; i < GPU_NSUBCELL*na_c; i++)
+ {
+ x_ci[i*DIM + XX] = x[(ia+i)*stride + XX] + shx;
+ x_ci[i*DIM + YY] = x[(ia+i)*stride + YY] + shy;
+ x_ci[i*DIM + ZZ] = x[(ia+i)*stride + ZZ] + shz;
+ }
+}
+
- static void icell_set_x_supersub_sse8(int ci,
- real shx, real shy, real shz,
- int na_c,
- int stride, const real *x,
- nbnxn_list_work_t *work)
++#ifdef NBNXN_SEARCH_BB_SIMD4
+/* Copies PBC shifted super-cell packed atom coordinates to working array */
- #ifdef NBNXN_SEARCH_BB_SSE
- nbs->icell_set_x = icell_set_x_supersub_sse8;
++static void icell_set_x_supersub_simd4(int ci,
++ real shx, real shy, real shz,
++ int na_c,
++ int stride, const real *x,
++ nbnxn_list_work_t *work)
+{
+ int si, io, ia, i, j;
+ real *x_ci;
+
+ x_ci = work->x_ci;
+
+ for (si = 0; si < GPU_NSUBCELL; si++)
+ {
+ for (i = 0; i < na_c; i += STRIDE_PBB)
+ {
+ io = si*na_c + i;
+ ia = ci*GPU_NSUBCELL*na_c + io;
+ for (j = 0; j < STRIDE_PBB; j++)
+ {
+ x_ci[io*DIM + j + XX*STRIDE_PBB] = x[(ia+j)*stride+XX] + shx;
+ x_ci[io*DIM + j + YY*STRIDE_PBB] = x[(ia+j)*stride+YY] + shy;
+ x_ci[io*DIM + j + ZZ*STRIDE_PBB] = x[(ia+j)*stride+ZZ] + shz;
+ }
+ }
+ }
+}
+#endif
+
+static real nbnxn_rlist_inc_nonloc_fac = 0.6;
+
+/* Due to the cluster size the effective pair-list is longer than
+ * that of a simple atom pair-list. This function gives the extra distance.
+ */
+real nbnxn_get_rlist_effective_inc(int cluster_size, real atom_density)
+{
+ return ((0.5 + nbnxn_rlist_inc_nonloc_fac)*sqr(((cluster_size) - 1.0)/(cluster_size))*pow((cluster_size)/(atom_density), 1.0/3.0));
+}
+
+/* Estimates the interaction volume^2 for non-local interactions */
+static real nonlocal_vol2(const gmx_domdec_zones_t *zones, rvec ls, real r)
+{
+ int z, d;
+ real cl, ca, za;
+ real vold_est;
+ real vol2_est_tot;
+
+ vol2_est_tot = 0;
+
+ /* Here we simply add up the volumes of 1, 2 or 3 1D decomposition
+ * not home interaction volume^2. As these volumes are not additive,
+ * this is an overestimate, but it would only be significant in the limit
+ * of small cells, where we anyhow need to split the lists into
+ * as small parts as possible.
+ */
+
+ for (z = 0; z < zones->n; z++)
+ {
+ if (zones->shift[z][XX] + zones->shift[z][YY] + zones->shift[z][ZZ] == 1)
+ {
+ cl = 0;
+ ca = 1;
+ za = 1;
+ for (d = 0; d < DIM; d++)
+ {
+ if (zones->shift[z][d] == 0)
+ {
+ cl += 0.5*ls[d];
+ ca *= ls[d];
+ za *= zones->size[z].x1[d] - zones->size[z].x0[d];
+ }
+ }
+
+ /* 4 octants of a sphere */
+ vold_est = 0.25*M_PI*r*r*r*r;
+ /* 4 quarter pie slices on the edges */
+ vold_est += 4*cl*M_PI/6.0*r*r*r;
+ /* One rectangular volume on a face */
+ vold_est += ca*0.5*r*r;
+
+ vol2_est_tot += vold_est*za;
+ }
+ }
+
+ return vol2_est_tot;
+}
+
+/* Estimates the average size of a full j-list for super/sub setup */
+static int get_nsubpair_max(const nbnxn_search_t nbs,
+ int iloc,
+ real rlist,
+ int min_ci_balanced)
+{
+ const nbnxn_grid_t *grid;
+ rvec ls;
+ real xy_diag2, r_eff_sup, vol_est, nsp_est, nsp_est_nl;
+ int nsubpair_max;
+
+ grid = &nbs->grid[0];
+
+ ls[XX] = (grid->c1[XX] - grid->c0[XX])/(grid->ncx*GPU_NSUBCELL_X);
+ ls[YY] = (grid->c1[YY] - grid->c0[YY])/(grid->ncy*GPU_NSUBCELL_Y);
+ ls[ZZ] = (grid->c1[ZZ] - grid->c0[ZZ])*grid->ncx*grid->ncy/(grid->nc*GPU_NSUBCELL_Z);
+
+ /* The average squared length of the diagonal of a sub cell */
+ xy_diag2 = ls[XX]*ls[XX] + ls[YY]*ls[YY] + ls[ZZ]*ls[ZZ];
+
+ /* The formulas below are a heuristic estimate of the average nsj per si*/
+ r_eff_sup = rlist + nbnxn_rlist_inc_nonloc_fac*sqr((grid->na_c - 1.0)/grid->na_c)*sqrt(xy_diag2/3);
+
+ if (!nbs->DomDec || nbs->zones->n == 1)
+ {
+ nsp_est_nl = 0;
+ }
+ else
+ {
+ nsp_est_nl =
+ sqr(grid->atom_density/grid->na_c)*
+ nonlocal_vol2(nbs->zones, ls, r_eff_sup);
+ }
+
+ if (LOCAL_I(iloc))
+ {
+ /* Sub-cell interacts with itself */
+ vol_est = ls[XX]*ls[YY]*ls[ZZ];
+ /* 6/2 rectangular volume on the faces */
+ vol_est += (ls[XX]*ls[YY] + ls[XX]*ls[ZZ] + ls[YY]*ls[ZZ])*r_eff_sup;
+ /* 12/2 quarter pie slices on the edges */
+ vol_est += 2*(ls[XX] + ls[YY] + ls[ZZ])*0.25*M_PI*sqr(r_eff_sup);
+ /* 4 octants of a sphere */
+ vol_est += 0.5*4.0/3.0*M_PI*pow(r_eff_sup, 3);
+
+ nsp_est = grid->nsubc_tot*vol_est*grid->atom_density/grid->na_c;
+
+ /* Subtract the non-local pair count */
+ nsp_est -= nsp_est_nl;
+
+ if (debug)
+ {
+ fprintf(debug, "nsp_est local %5.1f non-local %5.1f\n",
+ nsp_est, nsp_est_nl);
+ }
+ }
+ else
+ {
+ nsp_est = nsp_est_nl;
+ }
+
+ if (min_ci_balanced <= 0 || grid->nc >= min_ci_balanced || grid->nc == 0)
+ {
+ /* We don't need to worry */
+ nsubpair_max = -1;
+ }
+ else
+ {
+ /* Thus the (average) maximum j-list size should be as follows */
+ nsubpair_max = max(1, (int)(nsp_est/min_ci_balanced+0.5));
+
+ /* Since the target value is a maximum (this avoids high outliers,
+ * which lead to load imbalance), not average, we add half the
+ * number of pairs in a cj4 block to get the average about right.
+ */
+ nsubpair_max += GPU_NSUBCELL*NBNXN_GPU_JGROUP_SIZE/2;
+ }
+
+ if (debug)
+ {
+ fprintf(debug, "nbl nsp estimate %.1f, nsubpair_max %d\n",
+ nsp_est, nsubpair_max);
+ }
+
+ return nsubpair_max;
+}
+
+/* Debug list print function */
+static void print_nblist_ci_cj(FILE *fp, const nbnxn_pairlist_t *nbl)
+{
+ int i, j;
+
+ for (i = 0; i < nbl->nci; i++)
+ {
+ fprintf(fp, "ci %4d shift %2d ncj %3d\n",
+ nbl->ci[i].ci, nbl->ci[i].shift,
+ nbl->ci[i].cj_ind_end - nbl->ci[i].cj_ind_start);
+
+ for (j = nbl->ci[i].cj_ind_start; j < nbl->ci[i].cj_ind_end; j++)
+ {
+ fprintf(fp, " cj %5d imask %x\n",
+ nbl->cj[j].cj,
+ nbl->cj[j].excl);
+ }
+ }
+}
+
+/* Debug list print function */
+static void print_nblist_sci_cj(FILE *fp, const nbnxn_pairlist_t *nbl)
+{
+ int i, j4, j, ncp, si;
+
+ for (i = 0; i < nbl->nsci; i++)
+ {
+ fprintf(fp, "ci %4d shift %2d ncj4 %2d\n",
+ nbl->sci[i].sci, nbl->sci[i].shift,
+ nbl->sci[i].cj4_ind_end - nbl->sci[i].cj4_ind_start);
+
+ ncp = 0;
+ for (j4 = nbl->sci[i].cj4_ind_start; j4 < nbl->sci[i].cj4_ind_end; j4++)
+ {
+ for (j = 0; j < NBNXN_GPU_JGROUP_SIZE; j++)
+ {
+ fprintf(fp, " sj %5d imask %x\n",
+ nbl->cj4[j4].cj[j],
+ nbl->cj4[j4].imei[0].imask);
+ for (si = 0; si < GPU_NSUBCELL; si++)
+ {
+ if (nbl->cj4[j4].imei[0].imask & (1U << (j*GPU_NSUBCELL + si)))
+ {
+ ncp++;
+ }
+ }
+ }
+ }
+ fprintf(fp, "ci %4d shift %2d ncj4 %2d ncp %3d\n",
+ nbl->sci[i].sci, nbl->sci[i].shift,
+ nbl->sci[i].cj4_ind_end - nbl->sci[i].cj4_ind_start,
+ ncp);
+ }
+}
+
+/* Combine pair lists *nbl generated on multiple threads nblc */
+static void combine_nblists(int nnbl, nbnxn_pairlist_t **nbl,
+ nbnxn_pairlist_t *nblc)
+{
+ int nsci, ncj4, nexcl;
+ int n, i;
+
+ if (nblc->bSimple)
+ {
+ gmx_incons("combine_nblists does not support simple lists");
+ }
+
+ nsci = nblc->nsci;
+ ncj4 = nblc->ncj4;
+ nexcl = nblc->nexcl;
+ for (i = 0; i < nnbl; i++)
+ {
+ nsci += nbl[i]->nsci;
+ ncj4 += nbl[i]->ncj4;
+ nexcl += nbl[i]->nexcl;
+ }
+
+ if (nsci > nblc->sci_nalloc)
+ {
+ nb_realloc_sci(nblc, nsci);
+ }
+ if (ncj4 > nblc->cj4_nalloc)
+ {
+ nblc->cj4_nalloc = over_alloc_small(ncj4);
+ nbnxn_realloc_void((void **)&nblc->cj4,
+ nblc->ncj4*sizeof(*nblc->cj4),
+ nblc->cj4_nalloc*sizeof(*nblc->cj4),
+ nblc->alloc, nblc->free);
+ }
+ if (nexcl > nblc->excl_nalloc)
+ {
+ nblc->excl_nalloc = over_alloc_small(nexcl);
+ nbnxn_realloc_void((void **)&nblc->excl,
+ nblc->nexcl*sizeof(*nblc->excl),
+ nblc->excl_nalloc*sizeof(*nblc->excl),
+ nblc->alloc, nblc->free);
+ }
+
+ /* Each thread should copy its own data to the combined arrays,
+ * as otherwise data will go back and forth between different caches.
+ */
+#pragma omp parallel for num_threads(gmx_omp_nthreads_get(emntPairsearch)) schedule(static)
+ for (n = 0; n < nnbl; n++)
+ {
+ int sci_offset;
+ int cj4_offset;
+ int ci_offset;
+ int excl_offset;
+ int i, j4;
+ const nbnxn_pairlist_t *nbli;
+
+ /* Determine the offset in the combined data for our thread */
+ sci_offset = nblc->nsci;
+ cj4_offset = nblc->ncj4;
+ ci_offset = nblc->nci_tot;
+ excl_offset = nblc->nexcl;
+
+ for (i = 0; i < n; i++)
+ {
+ sci_offset += nbl[i]->nsci;
+ cj4_offset += nbl[i]->ncj4;
+ ci_offset += nbl[i]->nci_tot;
+ excl_offset += nbl[i]->nexcl;
+ }
+
+ nbli = nbl[n];
+
+ for (i = 0; i < nbli->nsci; i++)
+ {
+ nblc->sci[sci_offset+i] = nbli->sci[i];
+ nblc->sci[sci_offset+i].cj4_ind_start += cj4_offset;
+ nblc->sci[sci_offset+i].cj4_ind_end += cj4_offset;
+ }
+
+ for (j4 = 0; j4 < nbli->ncj4; j4++)
+ {
+ nblc->cj4[cj4_offset+j4] = nbli->cj4[j4];
+ nblc->cj4[cj4_offset+j4].imei[0].excl_ind += excl_offset;
+ nblc->cj4[cj4_offset+j4].imei[1].excl_ind += excl_offset;
+ }
+
+ for (j4 = 0; j4 < nbli->nexcl; j4++)
+ {
+ nblc->excl[excl_offset+j4] = nbli->excl[j4];
+ }
+ }
+
+ for (n = 0; n < nnbl; n++)
+ {
+ nblc->nsci += nbl[n]->nsci;
+ nblc->ncj4 += nbl[n]->ncj4;
+ nblc->nci_tot += nbl[n]->nci_tot;
+ nblc->nexcl += nbl[n]->nexcl;
+ }
+}
+
+/* Returns the next ci to be processes by our thread */
+static gmx_bool next_ci(const nbnxn_grid_t *grid,
+ int conv,
+ int nth, int ci_block,
+ int *ci_x, int *ci_y,
+ int *ci_b, int *ci)
+{
+ (*ci_b)++;
+ (*ci)++;
+
+ if (*ci_b == ci_block)
+ {
+ /* Jump to the next block assigned to this task */
+ *ci += (nth - 1)*ci_block;
+ *ci_b = 0;
+ }
+
+ if (*ci >= grid->nc*conv)
+ {
+ return FALSE;
+ }
+
+ while (*ci >= grid->cxy_ind[*ci_x*grid->ncy + *ci_y + 1]*conv)
+ {
+ *ci_y += 1;
+ if (*ci_y == grid->ncy)
+ {
+ *ci_x += 1;
+ *ci_y = 0;
+ }
+ }
+
+ return TRUE;
+}
+
+/* Returns the distance^2 for which we put cell pairs in the list
+ * without checking atom pair distances. This is usually < rlist^2.
+ */
+static float boundingbox_only_distance2(const nbnxn_grid_t *gridi,
+ const nbnxn_grid_t *gridj,
+ real rlist,
+ gmx_bool simple)
+{
+ /* If the distance between two sub-cell bounding boxes is less
+ * than this distance, do not check the distance between
+ * all particle pairs in the sub-cell, since then it is likely
+ * that the box pair has atom pairs within the cut-off.
+ * We use the nblist cut-off minus 0.5 times the average x/y diagonal
+ * spacing of the sub-cells. Around 40% of the checked pairs are pruned.
+ * Using more than 0.5 gains at most 0.5%.
+ * If forces are calculated more than twice, the performance gain
+ * in the force calculation outweighs the cost of checking.
+ * Note that with subcell lists, the atom-pair distance check
+ * is only performed when only 1 out of 8 sub-cells in within range,
+ * this is because the GPU is much faster than the cpu.
+ */
+ real bbx, bby;
+ real rbb2;
+
+ bbx = 0.5*(gridi->sx + gridj->sx);
+ bby = 0.5*(gridi->sy + gridj->sy);
+ if (!simple)
+ {
+ bbx /= GPU_NSUBCELL_X;
+ bby /= GPU_NSUBCELL_Y;
+ }
+
+ rbb2 = sqr(max(0, rlist - 0.5*sqrt(bbx*bbx + bby*bby)));
+
+#ifndef GMX_DOUBLE
+ return rbb2;
+#else
+ return (float)((1+GMX_FLOAT_EPS)*rbb2);
+#endif
+}
+
+static int get_ci_block_size(const nbnxn_grid_t *gridi,
+ gmx_bool bDomDec, int nth)
+{
+ const int ci_block_enum = 5;
+ const int ci_block_denom = 11;
+ const int ci_block_min_atoms = 16;
+ int ci_block;
+
+ /* Here we decide how to distribute the blocks over the threads.
+ * We use prime numbers to try to avoid that the grid size becomes
+ * a multiple of the number of threads, which would lead to some
+ * threads getting "inner" pairs and others getting boundary pairs,
+ * which in turns will lead to load imbalance between threads.
+ * Set the block size as 5/11/ntask times the average number of cells
+ * in a y,z slab. This should ensure a quite uniform distribution
+ * of the grid parts of the different thread along all three grid
+ * zone boundaries with 3D domain decomposition. At the same time
+ * the blocks will not become too small.
+ */
+ ci_block = (gridi->nc*ci_block_enum)/(ci_block_denom*gridi->ncx*nth);
+
+ /* Ensure the blocks are not too small: avoids cache invalidation */
+ if (ci_block*gridi->na_sc < ci_block_min_atoms)
+ {
+ ci_block = (ci_block_min_atoms + gridi->na_sc - 1)/gridi->na_sc;
+ }
+
+ /* Without domain decomposition
+ * or with less than 3 blocks per task, divide in nth blocks.
+ */
+ if (!bDomDec || ci_block*3*nth > gridi->nc)
+ {
+ ci_block = (gridi->nc + nth - 1)/nth;
+ }
+
+ return ci_block;
+}
+
+/* Generates the part of pair-list nbl assigned to our thread */
+static void nbnxn_make_pairlist_part(const nbnxn_search_t nbs,
+ const nbnxn_grid_t *gridi,
+ const nbnxn_grid_t *gridj,
+ nbnxn_search_work_t *work,
+ const nbnxn_atomdata_t *nbat,
+ const t_blocka *excl,
+ real rlist,
+ int nb_kernel_type,
+ int ci_block,
+ gmx_bool bFBufferFlag,
+ int nsubpair_max,
+ gmx_bool progBal,
+ int min_ci_balanced,
+ int th, int nth,
+ nbnxn_pairlist_t *nbl)
+{
+ int na_cj_2log;
+ matrix box;
+ real rl2;
+ float rbb2;
+ int d;
+ int ci_b, ci, ci_x, ci_y, ci_xy, cj;
+ ivec shp;
+ int tx, ty, tz;
+ int shift;
+ gmx_bool bMakeList;
+ real shx, shy, shz;
+ int conv_i, cell0_i;
+ const nbnxn_bb_t *bb_i=NULL;
+#ifdef NBNXN_BBXXXX
+ const float *pbb_i=NULL;
+#endif
+ const float *bbcz_i, *bbcz_j;
+ const int *flags_i;
+ real bx0, bx1, by0, by1, bz0, bz1;
+ real bz1_frac;
+ real d2cx, d2z, d2z_cx, d2z_cy, d2zx, d2zxy, d2xy;
+ int cxf, cxl, cyf, cyf_x, cyl;
+ int cx, cy;
+ int c0, c1, cs, cf, cl;
+ int ndistc;
+ int ncpcheck;
+ int gridi_flag_shift = 0, gridj_flag_shift = 0;
+ unsigned *gridj_flag = NULL;
+ int ncj_old_i, ncj_old_j;
+
+ nbs_cycle_start(&work->cc[enbsCCsearch]);
+
+ if (gridj->bSimple != nbl->bSimple)
+ {
+ gmx_incons("Grid incompatible with pair-list");
+ }
+
+ sync_work(nbl);
+ nbl->na_sc = gridj->na_sc;
+ nbl->na_ci = gridj->na_c;
+ nbl->na_cj = nbnxn_kernel_to_cj_size(nb_kernel_type);
+ na_cj_2log = get_2log(nbl->na_cj);
+
+ nbl->rlist = rlist;
+
+ if (bFBufferFlag)
+ {
+ /* Determine conversion of clusters to flag blocks */
+ gridi_flag_shift = 0;
+ while ((nbl->na_ci<<gridi_flag_shift) < NBNXN_BUFFERFLAG_SIZE)
+ {
+ gridi_flag_shift++;
+ }
+ gridj_flag_shift = 0;
+ while ((nbl->na_cj<<gridj_flag_shift) < NBNXN_BUFFERFLAG_SIZE)
+ {
+ gridj_flag_shift++;
+ }
+
+ gridj_flag = work->buffer_flags.flag;
+ }
+
+ copy_mat(nbs->box, box);
+
+ rl2 = nbl->rlist*nbl->rlist;
+
+ rbb2 = boundingbox_only_distance2(gridi, gridj, nbl->rlist, nbl->bSimple);
+
+ if (debug)
+ {
+ fprintf(debug, "nbl bounding box only distance %f\n", sqrt(rbb2));
+ }
+
+ /* Set the shift range */
+ for (d = 0; d < DIM; d++)
+ {
+ /* Check if we need periodicity shifts.
+ * Without PBC or with domain decomposition we don't need them.
+ */
+ if (d >= ePBC2npbcdim(nbs->ePBC) || nbs->dd_dim[d])
+ {
+ shp[d] = 0;
+ }
+ else
+ {
+ if (d == XX &&
+ box[XX][XX] - fabs(box[YY][XX]) - fabs(box[ZZ][XX]) < sqrt(rl2))
+ {
+ shp[d] = 2;
+ }
+ else
+ {
+ shp[d] = 1;
+ }
+ }
+ }
+
+ if (nbl->bSimple && !gridi->bSimple)
+ {
+ conv_i = gridi->na_sc/gridj->na_sc;
+ bb_i = gridi->bb_simple;
+ bbcz_i = gridi->bbcz_simple;
+ flags_i = gridi->flags_simple;
+ }
+ else
+ {
+ conv_i = 1;
+#ifdef NBNXN_BBXXXX
+ if (gridi->bSimple)
+ {
+ bb_i = gridi->bb;
+ }
+ else
+ {
+ pbb_i = gridi->pbb;
+ }
+#else
+ /* We use the normal bounding box format for both grid types */
+ bb_i = gridi->bb;
+#endif
+ bbcz_i = gridi->bbcz;
+ flags_i = gridi->flags;
+ }
+ cell0_i = gridi->cell0*conv_i;
+
+ bbcz_j = gridj->bbcz;
+
+ if (conv_i != 1)
+ {
+ /* Blocks of the conversion factor - 1 give a large repeat count
+ * combined with a small block size. This should result in good
+ * load balancing for both small and large domains.
+ */
+ ci_block = conv_i - 1;
+ }
+ if (debug)
+ {
+ fprintf(debug, "nbl nc_i %d col.av. %.1f ci_block %d\n",
+ gridi->nc, gridi->nc/(double)(gridi->ncx*gridi->ncy), ci_block);
+ }
+
+ ndistc = 0;
+ ncpcheck = 0;
+
+ /* Initially ci_b and ci to 1 before where we want them to start,
+ * as they will both be incremented in next_ci.
+ */
+ ci_b = -1;
+ ci = th*ci_block - 1;
+ ci_x = 0;
+ ci_y = 0;
+ while (next_ci(gridi, conv_i, nth, ci_block, &ci_x, &ci_y, &ci_b, &ci))
+ {
+ if (nbl->bSimple && flags_i[ci] == 0)
+ {
+ continue;
+ }
+
+ ncj_old_i = nbl->ncj;
+
+ d2cx = 0;
+ if (gridj != gridi && shp[XX] == 0)
+ {
+ if (nbl->bSimple)
+ {
+ bx1 = bb_i[ci].upper[BB_X];
+ }
+ else
+ {
+ bx1 = gridi->c0[XX] + (ci_x+1)*gridi->sx;
+ }
+ if (bx1 < gridj->c0[XX])
+ {
+ d2cx = sqr(gridj->c0[XX] - bx1);
+
+ if (d2cx >= rl2)
+ {
+ continue;
+ }
+ }
+ }
+
+ ci_xy = ci_x*gridi->ncy + ci_y;
+
+ /* Loop over shift vectors in three dimensions */
+ for (tz = -shp[ZZ]; tz <= shp[ZZ]; tz++)
+ {
+ shz = tz*box[ZZ][ZZ];
+
+ bz0 = bbcz_i[ci*NNBSBB_D ] + shz;
+ bz1 = bbcz_i[ci*NNBSBB_D+1] + shz;
+
+ if (tz == 0)
+ {
+ d2z = 0;
+ }
+ else if (tz < 0)
+ {
+ d2z = sqr(bz1);
+ }
+ else
+ {
+ d2z = sqr(bz0 - box[ZZ][ZZ]);
+ }
+
+ d2z_cx = d2z + d2cx;
+
+ if (d2z_cx >= rl2)
+ {
+ continue;
+ }
+
+ bz1_frac =
+ bz1/((real)(gridi->cxy_ind[ci_xy+1] - gridi->cxy_ind[ci_xy]));
+ if (bz1_frac < 0)
+ {
+ bz1_frac = 0;
+ }
+ /* The check with bz1_frac close to or larger than 1 comes later */
+
+ for (ty = -shp[YY]; ty <= shp[YY]; ty++)
+ {
+ shy = ty*box[YY][YY] + tz*box[ZZ][YY];
+
+ if (nbl->bSimple)
+ {
+ by0 = bb_i[ci].lower[BB_Y] + shy;
+ by1 = bb_i[ci].upper[BB_Y] + shy;
+ }
+ else
+ {
+ by0 = gridi->c0[YY] + (ci_y )*gridi->sy + shy;
+ by1 = gridi->c0[YY] + (ci_y+1)*gridi->sy + shy;
+ }
+
+ get_cell_range(by0, by1,
+ gridj->ncy, gridj->c0[YY], gridj->sy, gridj->inv_sy,
+ d2z_cx, rl2,
+ &cyf, &cyl);
+
+ if (cyf > cyl)
+ {
+ continue;
+ }
+
+ d2z_cy = d2z;
+ if (by1 < gridj->c0[YY])
+ {
+ d2z_cy += sqr(gridj->c0[YY] - by1);
+ }
+ else if (by0 > gridj->c1[YY])
+ {
+ d2z_cy += sqr(by0 - gridj->c1[YY]);
+ }
+
+ for (tx = -shp[XX]; tx <= shp[XX]; tx++)
+ {
+ shift = XYZ2IS(tx, ty, tz);
+
+#ifdef NBNXN_SHIFT_BACKWARD
+ if (gridi == gridj && shift > CENTRAL)
+ {
+ continue;
+ }
+#endif
+
+ shx = tx*box[XX][XX] + ty*box[YY][XX] + tz*box[ZZ][XX];
+
+ if (nbl->bSimple)
+ {
+ bx0 = bb_i[ci].lower[BB_X] + shx;
+ bx1 = bb_i[ci].upper[BB_X] + shx;
+ }
+ else
+ {
+ bx0 = gridi->c0[XX] + (ci_x )*gridi->sx + shx;
+ bx1 = gridi->c0[XX] + (ci_x+1)*gridi->sx + shx;
+ }
+
+ get_cell_range(bx0, bx1,
+ gridj->ncx, gridj->c0[XX], gridj->sx, gridj->inv_sx,
+ d2z_cy, rl2,
+ &cxf, &cxl);
+
+ if (cxf > cxl)
+ {
+ continue;
+ }
+
+ if (nbl->bSimple)
+ {
+ new_ci_entry(nbl, cell0_i+ci, shift, flags_i[ci]);
+ }
+ else
+ {
+ new_sci_entry(nbl, cell0_i+ci, shift);
+ }
+
+#ifndef NBNXN_SHIFT_BACKWARD
+ if (cxf < ci_x)
+#else
+ if (shift == CENTRAL && gridi == gridj &&
+ cxf < ci_x)
+#endif
+ {
+ /* Leave the pairs with i > j.
+ * x is the major index, so skip half of it.
+ */
+ cxf = ci_x;
+ }
+
+ if (nbl->bSimple)
+ {
+ set_icell_bb_simple(bb_i, ci, shx, shy, shz,
+ nbl->work->bb_ci);
+ }
+ else
+ {
+#ifdef NBNXN_BBXXXX
+ set_icell_bbxxxx_supersub(pbb_i, ci, shx, shy, shz,
+ nbl->work->pbb_ci);
+#else
+ set_icell_bb_supersub(bb_i, ci, shx, shy, shz,
+ nbl->work->bb_ci);
+#endif
+ }
+
+ nbs->icell_set_x(cell0_i+ci, shx, shy, shz,
+ gridi->na_c, nbat->xstride, nbat->x,
+ nbl->work);
+
+ for (cx = cxf; cx <= cxl; cx++)
+ {
+ d2zx = d2z;
+ if (gridj->c0[XX] + cx*gridj->sx > bx1)
+ {
+ d2zx += sqr(gridj->c0[XX] + cx*gridj->sx - bx1);
+ }
+ else if (gridj->c0[XX] + (cx+1)*gridj->sx < bx0)
+ {
+ d2zx += sqr(gridj->c0[XX] + (cx+1)*gridj->sx - bx0);
+ }
+
+#ifndef NBNXN_SHIFT_BACKWARD
+ if (gridi == gridj &&
+ cx == 0 && cyf < ci_y)
+#else
+ if (gridi == gridj &&
+ cx == 0 && shift == CENTRAL && cyf < ci_y)
+#endif
+ {
+ /* Leave the pairs with i > j.
+ * Skip half of y when i and j have the same x.
+ */
+ cyf_x = ci_y;
+ }
+ else
+ {
+ cyf_x = cyf;
+ }
+
+ for (cy = cyf_x; cy <= cyl; cy++)
+ {
+ c0 = gridj->cxy_ind[cx*gridj->ncy+cy];
+ c1 = gridj->cxy_ind[cx*gridj->ncy+cy+1];
+#ifdef NBNXN_SHIFT_BACKWARD
+ if (gridi == gridj &&
+ shift == CENTRAL && c0 < ci)
+ {
+ c0 = ci;
+ }
+#endif
+
+ d2zxy = d2zx;
+ if (gridj->c0[YY] + cy*gridj->sy > by1)
+ {
+ d2zxy += sqr(gridj->c0[YY] + cy*gridj->sy - by1);
+ }
+ else if (gridj->c0[YY] + (cy+1)*gridj->sy < by0)
+ {
+ d2zxy += sqr(gridj->c0[YY] + (cy+1)*gridj->sy - by0);
+ }
+ if (c1 > c0 && d2zxy < rl2)
+ {
+ cs = c0 + (int)(bz1_frac*(c1 - c0));
+ if (cs >= c1)
+ {
+ cs = c1 - 1;
+ }
+
+ d2xy = d2zxy - d2z;
+
+ /* Find the lowest cell that can possibly
+ * be within range.
+ */
+ cf = cs;
+ while (cf > c0 &&
+ (bbcz_j[cf*NNBSBB_D+1] >= bz0 ||
+ d2xy + sqr(bbcz_j[cf*NNBSBB_D+1] - bz0) < rl2))
+ {
+ cf--;
+ }
+
+ /* Find the highest cell that can possibly
+ * be within range.
+ */
+ cl = cs;
+ while (cl < c1-1 &&
+ (bbcz_j[cl*NNBSBB_D] <= bz1 ||
+ d2xy + sqr(bbcz_j[cl*NNBSBB_D] - bz1) < rl2))
+ {
+ cl++;
+ }
+
+#ifdef NBNXN_REFCODE
+ {
+ /* Simple reference code, for debugging,
+ * overrides the more complex code above.
+ */
+ int k;
+ cf = c1;
+ cl = -1;
+ for (k = c0; k < c1; k++)
+ {
+ if (box_dist2(bx0, bx1, by0, by1, bz0, bz1, bb+k) < rl2 &&
+ k < cf)
+ {
+ cf = k;
+ }
+ if (box_dist2(bx0, bx1, by0, by1, bz0, bz1, bb+k) < rl2 &&
+ k > cl)
+ {
+ cl = k;
+ }
+ }
+ }
+#endif
+
+ if (gridi == gridj)
+ {
+ /* We want each atom/cell pair only once,
+ * only use cj >= ci.
+ */
+#ifndef NBNXN_SHIFT_BACKWARD
+ cf = max(cf, ci);
+#else
+ if (shift == CENTRAL)
+ {
+ cf = max(cf, ci);
+ }
+#endif
+ }
+
+ if (cf <= cl)
+ {
+ /* For f buffer flags with simple lists */
+ ncj_old_j = nbl->ncj;
+
+ switch (nb_kernel_type)
+ {
+ case nbnxnk4x4_PlainC:
+ check_subcell_list_space_simple(nbl, cl-cf+1);
+
+ make_cluster_list_simple(gridj,
+ nbl, ci, cf, cl,
+ (gridi == gridj && shift == CENTRAL),
+ nbat->x,
+ rl2, rbb2,
+ &ndistc);
+ break;
+#ifdef GMX_NBNXN_SIMD_4XN
+ case nbnxnk4xN_SIMD_4xN:
+ check_subcell_list_space_simple(nbl, ci_to_cj(na_cj_2log, cl-cf)+2);
+ make_cluster_list_simd_4xn(gridj,
+ nbl, ci, cf, cl,
+ (gridi == gridj && shift == CENTRAL),
+ nbat->x,
+ rl2, rbb2,
+ &ndistc);
+ break;
+#endif
+#ifdef GMX_NBNXN_SIMD_2XNN
+ case nbnxnk4xN_SIMD_2xNN:
+ check_subcell_list_space_simple(nbl, ci_to_cj(na_cj_2log, cl-cf)+2);
+ make_cluster_list_simd_2xnn(gridj,
+ nbl, ci, cf, cl,
+ (gridi == gridj && shift == CENTRAL),
+ nbat->x,
+ rl2, rbb2,
+ &ndistc);
+ break;
+#endif
+ case nbnxnk8x8x8_PlainC:
+ case nbnxnk8x8x8_CUDA:
+ check_subcell_list_space_supersub(nbl, cl-cf+1);
+ for (cj = cf; cj <= cl; cj++)
+ {
+ make_cluster_list_supersub(gridi, gridj,
+ nbl, ci, cj,
+ (gridi == gridj && shift == CENTRAL && ci == cj),
+ nbat->xstride, nbat->x,
+ rl2, rbb2,
+ &ndistc);
+ }
+ break;
+ }
+ ncpcheck += cl - cf + 1;
+
+ if (bFBufferFlag && nbl->ncj > ncj_old_j)
+ {
+ int cbf, cbl, cb;
+
+ cbf = nbl->cj[ncj_old_j].cj >> gridj_flag_shift;
+ cbl = nbl->cj[nbl->ncj-1].cj >> gridj_flag_shift;
+ for (cb = cbf; cb <= cbl; cb++)
+ {
+ gridj_flag[cb] = 1U<<th;
+ }
+ }
+ }
+ }
+ }
+ }
+
+ /* Set the exclusions for this ci list */
+ if (nbl->bSimple)
+ {
+ set_ci_top_excls(nbs,
+ nbl,
+ shift == CENTRAL && gridi == gridj,
+ gridj->na_c_2log,
+ na_cj_2log,
+ &(nbl->ci[nbl->nci]),
+ excl);
+ }
+ else
+ {
+ set_sci_top_excls(nbs,
+ nbl,
+ shift == CENTRAL && gridi == gridj,
+ gridj->na_c_2log,
+ &(nbl->sci[nbl->nsci]),
+ excl);
+ }
+
+ /* Close this ci list */
+ if (nbl->bSimple)
+ {
+ close_ci_entry_simple(nbl);
+ }
+ else
+ {
+ close_ci_entry_supersub(nbl,
+ nsubpair_max,
+ progBal, min_ci_balanced,
+ th, nth);
+ }
+ }
+ }
+ }
+
+ if (bFBufferFlag && nbl->ncj > ncj_old_i)
+ {
+ work->buffer_flags.flag[(gridi->cell0+ci)>>gridi_flag_shift] = 1U<<th;
+ }
+ }
+
+ work->ndistc = ndistc;
+
+ nbs_cycle_stop(&work->cc[enbsCCsearch]);
+
+ if (debug)
+ {
+ fprintf(debug, "number of distance checks %d\n", ndistc);
+ fprintf(debug, "ncpcheck %s %d\n", gridi == gridj ? "local" : "non-local",
+ ncpcheck);
+
+ if (nbl->bSimple)
+ {
+ print_nblist_statistics_simple(debug, nbl, nbs, rlist);
+ }
+ else
+ {
+ print_nblist_statistics_supersub(debug, nbl, nbs, rlist);
+ }
+
+ }
+}
+
+static void reduce_buffer_flags(const nbnxn_search_t nbs,
+ int nsrc,
+ const nbnxn_buffer_flags_t *dest)
+{
+ int s, b;
+ const unsigned *flag;
+
+ for (s = 0; s < nsrc; s++)
+ {
+ flag = nbs->work[s].buffer_flags.flag;
+
+ for (b = 0; b < dest->nflag; b++)
+ {
+ dest->flag[b] |= flag[b];
+ }
+ }
+}
+
+static void print_reduction_cost(const nbnxn_buffer_flags_t *flags, int nout)
+{
+ int nelem, nkeep, ncopy, nred, b, c, out;
+
+ nelem = 0;
+ nkeep = 0;
+ ncopy = 0;
+ nred = 0;
+ for (b = 0; b < flags->nflag; b++)
+ {
+ if (flags->flag[b] == 1)
+ {
+ /* Only flag 0 is set, no copy of reduction required */
+ nelem++;
+ nkeep++;
+ }
+ else if (flags->flag[b] > 0)
+ {
+ c = 0;
+ for (out = 0; out < nout; out++)
+ {
+ if (flags->flag[b] & (1U<<out))
+ {
+ c++;
+ }
+ }
+ nelem += c;
+ if (c == 1)
+ {
+ ncopy++;
+ }
+ else
+ {
+ nred += c;
+ }
+ }
+ }
+
+ fprintf(debug, "nbnxn reduction: #flag %d #list %d elem %4.2f, keep %4.2f copy %4.2f red %4.2f\n",
+ flags->nflag, nout,
+ nelem/(double)(flags->nflag),
+ nkeep/(double)(flags->nflag),
+ ncopy/(double)(flags->nflag),
+ nred/(double)(flags->nflag));
+}
+
+/* Perform a count (linear) sort to sort the smaller lists to the end.
+ * This avoids load imbalance on the GPU, as large lists will be
+ * scheduled and executed first and the smaller lists later.
+ * Load balancing between multi-processors only happens at the end
+ * and there smaller lists lead to more effective load balancing.
+ * The sorting is done on the cj4 count, not on the actual pair counts.
+ * Not only does this make the sort faster, but it also results in
+ * better load balancing than using a list sorted on exact load.
+ * This function swaps the pointer in the pair list to avoid a copy operation.
+ */
+static void sort_sci(nbnxn_pairlist_t *nbl)
+{
+ nbnxn_list_work_t *work;
+ int m, i, s, s0, s1;
+ nbnxn_sci_t *sci_sort;
+
+ if (nbl->ncj4 <= nbl->nsci)
+ {
+ /* nsci = 0 or all sci have size 1, sorting won't change the order */
+ return;
+ }
+
+ work = nbl->work;
+
+ /* We will distinguish differences up to double the average */
+ m = (2*nbl->ncj4)/nbl->nsci;
+
+ if (m + 1 > work->sort_nalloc)
+ {
+ work->sort_nalloc = over_alloc_large(m + 1);
+ srenew(work->sort, work->sort_nalloc);
+ }
+
+ if (work->sci_sort_nalloc != nbl->sci_nalloc)
+ {
+ work->sci_sort_nalloc = nbl->sci_nalloc;
+ nbnxn_realloc_void((void **)&work->sci_sort,
+ 0,
+ work->sci_sort_nalloc*sizeof(*work->sci_sort),
+ nbl->alloc, nbl->free);
+ }
+
+ /* Count the entries of each size */
+ for (i = 0; i <= m; i++)
+ {
+ work->sort[i] = 0;
+ }
+ for (s = 0; s < nbl->nsci; s++)
+ {
+ i = min(m, nbl->sci[s].cj4_ind_end - nbl->sci[s].cj4_ind_start);
+ work->sort[i]++;
+ }
+ /* Calculate the offset for each count */
+ s0 = work->sort[m];
+ work->sort[m] = 0;
+ for (i = m - 1; i >= 0; i--)
+ {
+ s1 = work->sort[i];
+ work->sort[i] = work->sort[i + 1] + s0;
+ s0 = s1;
+ }
+
+ /* Sort entries directly into place */
+ sci_sort = work->sci_sort;
+ for (s = 0; s < nbl->nsci; s++)
+ {
+ i = min(m, nbl->sci[s].cj4_ind_end - nbl->sci[s].cj4_ind_start);
+ sci_sort[work->sort[i]++] = nbl->sci[s];
+ }
+
+ /* Swap the sci pointers so we use the new, sorted list */
+ work->sci_sort = nbl->sci;
+ nbl->sci = sci_sort;
+}
+
+/* Make a local or non-local pair-list, depending on iloc */
+void nbnxn_make_pairlist(const nbnxn_search_t nbs,
+ nbnxn_atomdata_t *nbat,
+ const t_blocka *excl,
+ real rlist,
+ int min_ci_balanced,
+ nbnxn_pairlist_set_t *nbl_list,
+ int iloc,
+ int nb_kernel_type,
+ t_nrnb *nrnb)
+{
+ nbnxn_grid_t *gridi, *gridj;
+ gmx_bool bGPUCPU;
+ int nzi, zi, zj0, zj1, zj;
+ int nsubpair_max;
+ int th;
+ int nnbl;
+ nbnxn_pairlist_t **nbl;
+ int ci_block;
+ gmx_bool CombineNBLists;
+ gmx_bool progBal;
+ int np_tot, np_noq, np_hlj, nap;
+
+ /* Check if we are running hybrid GPU + CPU nbnxn mode */
+ bGPUCPU = (!nbs->grid[0].bSimple && nbl_list->bSimple);
+
+ nnbl = nbl_list->nnbl;
+ nbl = nbl_list->nbl;
+ CombineNBLists = nbl_list->bCombined;
+
+ if (debug)
+ {
+ fprintf(debug, "ns making %d nblists\n", nnbl);
+ }
+
+ nbat->bUseBufferFlags = (nbat->nout > 1);
+ /* We should re-init the flags before making the first list */
+ if (nbat->bUseBufferFlags && (LOCAL_I(iloc) || bGPUCPU))
+ {
+ init_buffer_flags(&nbat->buffer_flags, nbat->natoms);
+ }
+
+ if (nbl_list->bSimple)
+ {
+ switch (nb_kernel_type)
+ {
+#ifdef GMX_NBNXN_SIMD_4XN
+ case nbnxnk4xN_SIMD_4xN:
+ nbs->icell_set_x = icell_set_x_simd_4xn;
+ break;
+#endif
+#ifdef GMX_NBNXN_SIMD_2XNN
+ case nbnxnk4xN_SIMD_2xNN:
+ nbs->icell_set_x = icell_set_x_simd_2xnn;
+ break;
+#endif
+ default:
+ nbs->icell_set_x = icell_set_x_simple;
+ break;
+ }
+ }
+ else
+ {
++#ifdef NBNXN_SEARCH_BB_SIMD4
++ nbs->icell_set_x = icell_set_x_supersub_simd4;
+#else
+ nbs->icell_set_x = icell_set_x_supersub;
+#endif
+ }
+
+ if (LOCAL_I(iloc))
+ {
+ /* Only zone (grid) 0 vs 0 */
+ nzi = 1;
+ zj0 = 0;
+ zj1 = 1;
+ }
+ else
+ {
+ nzi = nbs->zones->nizone;
+ }
+
+ if (!nbl_list->bSimple && min_ci_balanced > 0)
+ {
+ nsubpair_max = get_nsubpair_max(nbs, iloc, rlist, min_ci_balanced);
+ }
+ else
+ {
+ nsubpair_max = 0;
+ }
+
+ /* Clear all pair-lists */
+ for (th = 0; th < nnbl; th++)
+ {
+ clear_pairlist(nbl[th]);
+ }
+
+ for (zi = 0; zi < nzi; zi++)
+ {
+ gridi = &nbs->grid[zi];
+
+ if (NONLOCAL_I(iloc))
+ {
+ zj0 = nbs->zones->izone[zi].j0;
+ zj1 = nbs->zones->izone[zi].j1;
+ if (zi == 0)
+ {
+ zj0++;
+ }
+ }
+ for (zj = zj0; zj < zj1; zj++)
+ {
+ gridj = &nbs->grid[zj];
+
+ if (debug)
+ {
+ fprintf(debug, "ns search grid %d vs %d\n", zi, zj);
+ }
+
+ nbs_cycle_start(&nbs->cc[enbsCCsearch]);
+
+ if (nbl[0]->bSimple && !gridi->bSimple)
+ {
+ /* Hybrid list, determine blocking later */
+ ci_block = 0;
+ }
+ else
+ {
+ ci_block = get_ci_block_size(gridi, nbs->DomDec, nnbl);
+ }
+
+#pragma omp parallel for num_threads(nnbl) schedule(static)
+ for (th = 0; th < nnbl; th++)
+ {
+ /* Re-init the thread-local work flag data before making
+ * the first list (not an elegant conditional).
+ */
+ if (nbat->bUseBufferFlags && ((zi == 0 && zj == 0) ||
+ (bGPUCPU && zi == 0 && zj == 1)))
+ {
+ init_buffer_flags(&nbs->work[th].buffer_flags, nbat->natoms);
+ }
+
+ if (CombineNBLists && th > 0)
+ {
+ clear_pairlist(nbl[th]);
+ }
+
+ /* With GPU: generate progressively smaller lists for
+ * load balancing for local only or non-local with 2 zones.
+ */
+ progBal = (LOCAL_I(iloc) || nbs->zones->n <= 2);
+
+ /* Divide the i super cell equally over the nblists */
+ nbnxn_make_pairlist_part(nbs, gridi, gridj,
+ &nbs->work[th], nbat, excl,
+ rlist,
+ nb_kernel_type,
+ ci_block,
+ nbat->bUseBufferFlags,
+ nsubpair_max,
+ progBal, min_ci_balanced,
+ th, nnbl,
+ nbl[th]);
+ }
+ nbs_cycle_stop(&nbs->cc[enbsCCsearch]);
+
+ np_tot = 0;
+ np_noq = 0;
+ np_hlj = 0;
+ for (th = 0; th < nnbl; th++)
+ {
+ inc_nrnb(nrnb, eNR_NBNXN_DIST2, nbs->work[th].ndistc);
+
+ if (nbl_list->bSimple)
+ {
+ np_tot += nbl[th]->ncj;
+ np_noq += nbl[th]->work->ncj_noq;
+ np_hlj += nbl[th]->work->ncj_hlj;
+ }
+ else
+ {
+ /* This count ignores potential subsequent pair pruning */
+ np_tot += nbl[th]->nci_tot;
+ }
+ }
+ nap = nbl[0]->na_ci*nbl[0]->na_cj;
+ nbl_list->natpair_ljq = (np_tot - np_noq)*nap - np_hlj*nap/2;
+ nbl_list->natpair_lj = np_noq*nap;
+ nbl_list->natpair_q = np_hlj*nap/2;
+
+ if (CombineNBLists && nnbl > 1)
+ {
+ nbs_cycle_start(&nbs->cc[enbsCCcombine]);
+
+ combine_nblists(nnbl-1, nbl+1, nbl[0]);
+
+ nbs_cycle_stop(&nbs->cc[enbsCCcombine]);
+ }
+ }
+ }
+
+ if (!nbl_list->bSimple)
+ {
+ /* Sort the entries on size, large ones first */
+ if (CombineNBLists || nnbl == 1)
+ {
+ sort_sci(nbl[0]);
+ }
+ else
+ {
+#pragma omp parallel for num_threads(nnbl) schedule(static)
+ for (th = 0; th < nnbl; th++)
+ {
+ sort_sci(nbl[th]);
+ }
+ }
+ }
+
+ if (nbat->bUseBufferFlags)
+ {
+ reduce_buffer_flags(nbs, nnbl, &nbat->buffer_flags);
+ }
+
+ /* Special performance logging stuff (env.var. GMX_NBNXN_CYCLE) */
+ if (LOCAL_I(iloc))
+ {
+ nbs->search_count++;
+ }
+ if (nbs->print_cycles &&
+ (!nbs->DomDec || (nbs->DomDec && !LOCAL_I(iloc))) &&
+ nbs->search_count % 100 == 0)
+ {
+ nbs_cycle_print(stderr, nbs);
+ }
+
+ if (debug && (CombineNBLists && nnbl > 1))
+ {
+ if (nbl[0]->bSimple)
+ {
+ print_nblist_statistics_simple(debug, nbl[0], nbs, rlist);
+ }
+ else
+ {
+ print_nblist_statistics_supersub(debug, nbl[0], nbs, rlist);
+ }
+ }
+
+ if (debug)
+ {
+ if (gmx_debug_at)
+ {
+ if (nbl[0]->bSimple)
+ {
+ print_nblist_ci_cj(debug, nbl[0]);
+ }
+ else
+ {
+ print_nblist_sci_cj(debug, nbl[0]);
+ }
+ }
+
+ if (nbat->bUseBufferFlags)
+ {
+ print_reduction_cost(&nbat->buffer_flags, nnbl);
+ }
+ }
+}
--- /dev/null
-
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustr
+ *
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2012, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * Gallium Rubidium Oxygen Manganese Argon Carbon Silicon
+ */
+
+#ifndef _nbnxn_search_h
+#define _nbnxn_search_h
+
+#include "typedefs.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Returns the j-cluster size for kernel of type nb_kernel_type */
+int nbnxn_kernel_to_cj_size(int nb_kernel_type);
+
+/* Tells if the pair-list corresponding to nb_kernel_type is simple.
+ * Returns FALSE for super-sub type pair-list.
+ */
+gmx_bool nbnxn_kernel_pairlist_simple(int nb_kernel_type);
+
+/* Due to the cluster size the effective pair-list is longer than
+ * that of a simple atom pair-list. This function gives the extra distance.
+ */
+real nbnxn_get_rlist_effective_inc(int cluster_size, real atom_density);
+
+/* Allocates and initializes a pair search data structure */
+void nbnxn_init_search(nbnxn_search_t * nbs_ptr,
+ ivec *n_dd_cells,
+ gmx_domdec_zones_t *zones,
+ int nthread_max);
+
+/* Put the atoms on the pair search grid.
+ * Only atoms a0 to a1 in x are put on the grid.
+ * The atom_density is used to determine the grid size.
+ * When atom_density=-1, the density is determined from a1-a0 and the corners.
+ * With domain decomposition part of the n particles might have migrated,
+ * but have not been removed yet. This count is given by nmoved.
+ * When move[i] < 0 particle i has migrated and will not be put on the grid.
+ * Without domain decomposition move will be NULL.
+ */
+void nbnxn_put_on_grid(nbnxn_search_t nbs,
+ int ePBC, matrix box,
+ int dd_zone,
+ rvec corner0, rvec corner1,
+ int a0, int a1,
+ real atom_density,
+ const int *atinfo,
+ rvec *x,
+ int nmoved, int *move,
+ int nb_kernel_type,
+ nbnxn_atomdata_t *nbat);
+
+/* As nbnxn_put_on_grid, but for the non-local atoms
+ * with domain decomposition. Should be called after calling
+ * nbnxn_search_put_on_grid for the local atoms / home zone.
+ */
+void nbnxn_put_on_grid_nonlocal(nbnxn_search_t nbs,
+ const gmx_domdec_zones_t *zones,
+ const int *atinfo,
+ rvec *x,
+ int nb_kernel_type,
+ nbnxn_atomdata_t *nbat);
+
+/* Add simple grid type information to the local super/sub grid */
+void nbnxn_grid_add_simple(nbnxn_search_t nbs,
+ nbnxn_atomdata_t *nbat);
+
+/* Return the number of x and y cells in the local grid */
+void nbnxn_get_ncells(nbnxn_search_t nbs, int *ncx, int *ncy);
+
+/* Return the order indices *a of the atoms on the ns grid, size n */
+void nbnxn_get_atomorder(nbnxn_search_t nbs, int **a, int *n);
+
+/* Renumber the atom indices on the grid to consecutive order */
+void nbnxn_set_atomorder(nbnxn_search_t nbs);
+
+/* Initializes a set of pair lists stored in nbnxn_pairlist_set_t */
+void nbnxn_init_pairlist_set(nbnxn_pairlist_set_t *nbl_list,
+ gmx_bool simple, gmx_bool combined,
+ nbnxn_alloc_t *alloc,
+ nbnxn_free_t *free);
+
+/* Make a apir-list with radius rlist, store it in nbl.
+ * The parameter min_ci_balanced sets the minimum required
+ * number or roughly equally sized ci blocks in nbl.
+ * When set >0 ci lists will be chopped up when the estimate
+ * for the number of equally sized lists is below min_ci_balanced.
+ */
+void nbnxn_make_pairlist(const nbnxn_search_t nbs,
+ nbnxn_atomdata_t *nbat,
+ const t_blocka *excl,
+ real rlist,
+ int min_ci_balanced,
+ nbnxn_pairlist_set_t *nbl_list,
+ int iloc,
+ int nb_kernel_type,
+ t_nrnb *nrnb);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
--- /dev/null
- x_ci->ix_SSE0 = gmx_set_2real_shift_pr(x + ia + 0*STRIDE_S + 0, shx);
- x_ci->iy_SSE0 = gmx_set_2real_shift_pr(x + ia + 1*STRIDE_S + 0, shy);
- x_ci->iz_SSE0 = gmx_set_2real_shift_pr(x + ia + 2*STRIDE_S + 0, shz);
- x_ci->ix_SSE2 = gmx_set_2real_shift_pr(x + ia + 0*STRIDE_S + 2, shx);
- x_ci->iy_SSE2 = gmx_set_2real_shift_pr(x + ia + 1*STRIDE_S + 2, shy);
- x_ci->iz_SSE2 = gmx_set_2real_shift_pr(x + ia + 2*STRIDE_S + 2, shz);
+/*
+ * This file is part of the GROMACS molecular simulation package.
+ *
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2012, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+ * Copyright (c) 2012, by the GROMACS development team, led by
+ * David van der Spoel, Berk Hess, Erik Lindahl, and including many
+ * others, as listed in the AUTHORS file in the top-level source
+ * directory and at http://www.gromacs.org.
+ *
+ * GROMACS is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ *
+ * GROMACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with GROMACS; if not, see
+ * http://www.gnu.org/licenses, or write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * If you want to redistribute modifications to GROMACS, please
+ * consider that scientific software is very special. Version
+ * control is crucial - bugs must be traceable. We will be happy to
+ * consider code for inclusion in the official distribution, but
+ * derived work must not be called official GROMACS. Details are found
+ * in the README & COPYING files - if they are missing, get the
+ * official version at http://www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the research papers on the package. Check out http://www.gromacs.org.
+ */
+
+/* Get the half-width SIMD stuff from the kernel utils files */
+#include "nbnxn_kernels/nbnxn_kernel_simd_utils.h"
+
+
+#if GMX_SIMD_WIDTH_HERE >= 2*NBNXN_CPU_CLUSTER_I_SIZE
+#define STRIDE_S (GMX_SIMD_WIDTH_HERE/2)
+#else
+#define STRIDE_S NBNXN_CPU_CLUSTER_I_SIZE
+#endif
+
+static gmx_inline gmx_mm_pr gmx_load_hpr_hilo_pr(const real *a)
+{
+ gmx_mm_hpr a_S;
+ gmx_mm_pr a_a_S;
+
+ gmx_load_hpr(&a_S, a);
+
+ gmx_2hpr_to_pr(a_S, a_S, &a_a_S);
+
+ return a_a_S;
+}
+
+static gmx_inline gmx_mm_pr gmx_set_2real_shift_pr(const real *a, real shift)
+{
+ gmx_mm_hpr a0_S, a1_S;
+ gmx_mm_pr a0_a1_S;
+
+ gmx_set1_hpr(&a0_S, a[0] + shift);
+ gmx_set1_hpr(&a1_S, a[1] + shift);
+
+ gmx_2hpr_to_pr(a0_S, a1_S, &a0_a1_S);
+
+ return a0_a1_S;
+}
+
+/* Copies PBC shifted i-cell packed atom coordinates to working array */
+static gmx_inline void
+icell_set_x_simd_2xnn(int ci,
+ real shx, real shy, real shz,
+ int gmx_unused na_c,
+ int gmx_unused stride, const real *x,
+ nbnxn_list_work_t *work)
+{
+ int ia;
+ nbnxn_x_ci_simd_2xnn_t *x_ci;
+
+ x_ci = work->x_ci_simd_2xnn;
+
+ ia = X_IND_CI_SIMD_2XNN(ci);
+
- #ifndef GMX_SIMD_HAVE_ANYTRUE
- /* Fallback function in case gmx_anytrue_pr is not present */
- static gmx_inline gmx_bool
- gmx_anytrue_2xn_pb(gmx_mm_pb bool_S)
- {
- real bools_array[2*GMX_SIMD_WIDTH_HERE], *bools;
- gmx_bool any;
- int s;
-
- bools = gmx_simd_align_real(bools_array);
-
- gmx_store_pb(bools, bool_S);
-
- any = FALSE;
- for (s = 0; s < GMX_SIMD_WIDTH_HERE; s++)
- {
- if (GMX_SIMD_IS_TRUE(s))
- {
- any = TRUE;
- }
- }
-
- return any;
- }
- #endif
-
++ x_ci->ix_S0 = gmx_set_2real_shift_pr(x + ia + 0*STRIDE_S + 0, shx);
++ x_ci->iy_S0 = gmx_set_2real_shift_pr(x + ia + 1*STRIDE_S + 0, shy);
++ x_ci->iz_S0 = gmx_set_2real_shift_pr(x + ia + 2*STRIDE_S + 0, shz);
++ x_ci->ix_S2 = gmx_set_2real_shift_pr(x + ia + 0*STRIDE_S + 2, shx);
++ x_ci->iy_S2 = gmx_set_2real_shift_pr(x + ia + 1*STRIDE_S + 2, shy);
++ x_ci->iz_S2 = gmx_set_2real_shift_pr(x + ia + 2*STRIDE_S + 2, shz);
+}
+
- gmx_mm_pr jx_SSE, jy_SSE, jz_SSE;
+/* SIMD code for making a pair list of cell ci vs cell cjf-cjl
+ * for coordinates in packed format.
+ * Checks bouding box distances and possibly atom pair distances.
+ * This is an accelerated version of make_cluster_list_simple.
+ */
+static gmx_inline void
+make_cluster_list_simd_2xnn(const nbnxn_grid_t *gridj,
+ nbnxn_pairlist_t *nbl,
+ int ci, int cjf, int cjl,
+ gmx_bool remove_sub_diag,
+ const real *x_j,
+ real rl2, float rbb2,
+ int *ndistc)
+{
+ const nbnxn_x_ci_simd_2xnn_t *work;
+ const nbnxn_bb_t *bb_ci;
+
- gmx_mm_pr dx_SSE0, dy_SSE0, dz_SSE0;
- gmx_mm_pr dx_SSE2, dy_SSE2, dz_SSE2;
++ gmx_mm_pr jx_S, jy_S, jz_S;
+
- gmx_mm_pr rsq_SSE0;
- gmx_mm_pr rsq_SSE2;
++ gmx_mm_pr dx_S0, dy_S0, dz_S0;
++ gmx_mm_pr dx_S2, dy_S2, dz_S2;
+
- gmx_mm_pb wco_SSE0;
- gmx_mm_pb wco_SSE2;
- gmx_mm_pb wco_any_SSE;
++ gmx_mm_pr rsq_S0;
++ gmx_mm_pr rsq_S2;
+
- gmx_mm_pr rc2_SSE;
++ gmx_mm_pb wco_S0;
++ gmx_mm_pb wco_S2;
++ gmx_mm_pb wco_any_S;
+
- rc2_SSE = gmx_set1_pr(rl2);
++ gmx_mm_pr rc2_S;
+
+ gmx_bool InRange;
+ float d2;
+ int xind_f, xind_l, cj;
+
+ cjf = CI_TO_CJ_SIMD_2XNN(cjf);
+ cjl = CI_TO_CJ_SIMD_2XNN(cjl+1) - 1;
+
+ work = nbl->work->x_ci_simd_2xnn;
+
+ bb_ci = nbl->work->bb_ci;
+
- #ifdef NBNXN_SEARCH_BB_SSE
- d2 = subc_bb_dist2_sse(0, bb_ci, cjf, gridj->bbj);
++ rc2_S = gmx_set1_pr(rl2);
+
+ InRange = FALSE;
+ while (!InRange && cjf <= cjl)
+ {
- jx_SSE = gmx_load_hpr_hilo_pr(x_j+xind_f+0*STRIDE_S);
- jy_SSE = gmx_load_hpr_hilo_pr(x_j+xind_f+1*STRIDE_S);
- jz_SSE = gmx_load_hpr_hilo_pr(x_j+xind_f+2*STRIDE_S);
++#ifdef NBNXN_SEARCH_BB_SIMD4
++ d2 = subc_bb_dist2_simd4(0, bb_ci, cjf, gridj->bbj);
+#else
+ d2 = subc_bb_dist2(0, bb_ci, cjf, gridj->bbj);
+#endif
+ *ndistc += 2;
+
+ /* Check if the distance is within the distance where
+ * we use only the bounding box distance rbb,
+ * or within the cut-off and there is at least one atom pair
+ * within the cut-off.
+ */
+ if (d2 < rbb2)
+ {
+ InRange = TRUE;
+ }
+ else if (d2 < rl2)
+ {
+ xind_f = X_IND_CJ_SIMD_2XNN(CI_TO_CJ_SIMD_2XNN(gridj->cell0) + cjf);
+
- dx_SSE0 = gmx_sub_pr(work->ix_SSE0, jx_SSE);
- dy_SSE0 = gmx_sub_pr(work->iy_SSE0, jy_SSE);
- dz_SSE0 = gmx_sub_pr(work->iz_SSE0, jz_SSE);
- dx_SSE2 = gmx_sub_pr(work->ix_SSE2, jx_SSE);
- dy_SSE2 = gmx_sub_pr(work->iy_SSE2, jy_SSE);
- dz_SSE2 = gmx_sub_pr(work->iz_SSE2, jz_SSE);
++ jx_S = gmx_load_hpr_hilo_pr(x_j+xind_f+0*STRIDE_S);
++ jy_S = gmx_load_hpr_hilo_pr(x_j+xind_f+1*STRIDE_S);
++ jz_S = gmx_load_hpr_hilo_pr(x_j+xind_f+2*STRIDE_S);
+
+ /* Calculate distance */
- rsq_SSE0 = gmx_calc_rsq_pr(dx_SSE0, dy_SSE0, dz_SSE0);
- rsq_SSE2 = gmx_calc_rsq_pr(dx_SSE2, dy_SSE2, dz_SSE2);
++ dx_S0 = gmx_sub_pr(work->ix_S0, jx_S);
++ dy_S0 = gmx_sub_pr(work->iy_S0, jy_S);
++ dz_S0 = gmx_sub_pr(work->iz_S0, jz_S);
++ dx_S2 = gmx_sub_pr(work->ix_S2, jx_S);
++ dy_S2 = gmx_sub_pr(work->iy_S2, jy_S);
++ dz_S2 = gmx_sub_pr(work->iz_S2, jz_S);
+
+ /* rsq = dx*dx+dy*dy+dz*dz */
- wco_SSE0 = gmx_cmplt_pr(rsq_SSE0, rc2_SSE);
- wco_SSE2 = gmx_cmplt_pr(rsq_SSE2, rc2_SSE);
++ rsq_S0 = gmx_calc_rsq_pr(dx_S0, dy_S0, dz_S0);
++ rsq_S2 = gmx_calc_rsq_pr(dx_S2, dy_S2, dz_S2);
+
- wco_any_SSE = gmx_or_pb(wco_SSE0, wco_SSE2);
++ wco_S0 = gmx_cmplt_pr(rsq_S0, rc2_S);
++ wco_S2 = gmx_cmplt_pr(rsq_S2, rc2_S);
+
- #ifdef GMX_SIMD_HAVE_ANYTRUE
- InRange = gmx_anytrue_pb(wco_any_SSE);
- #else
- InRange = gmx_anytrue_2xn_pb(wco_any_SSE);
- #endif
++ wco_any_S = gmx_or_pb(wco_S0, wco_S2);
+
- #ifdef NBNXN_SEARCH_BB_SSE
- d2 = subc_bb_dist2_sse(0, bb_ci, cjl, gridj->bbj);
++ InRange = gmx_anytrue_pb(wco_any_S);
+
+ *ndistc += 2*GMX_SIMD_WIDTH_HERE;
+ }
+ if (!InRange)
+ {
+ cjf++;
+ }
+ }
+ if (!InRange)
+ {
+ return;
+ }
+
+ InRange = FALSE;
+ while (!InRange && cjl > cjf)
+ {
- jx_SSE = gmx_load_hpr_hilo_pr(x_j+xind_l+0*STRIDE_S);
- jy_SSE = gmx_load_hpr_hilo_pr(x_j+xind_l+1*STRIDE_S);
- jz_SSE = gmx_load_hpr_hilo_pr(x_j+xind_l+2*STRIDE_S);
++#ifdef NBNXN_SEARCH_BB_SIMD4
++ d2 = subc_bb_dist2_simd4(0, bb_ci, cjl, gridj->bbj);
+#else
+ d2 = subc_bb_dist2(0, bb_ci, cjl, gridj->bbj);
+#endif
+ *ndistc += 2;
+
+ /* Check if the distance is within the distance where
+ * we use only the bounding box distance rbb,
+ * or within the cut-off and there is at least one atom pair
+ * within the cut-off.
+ */
+ if (d2 < rbb2)
+ {
+ InRange = TRUE;
+ }
+ else if (d2 < rl2)
+ {
+ xind_l = X_IND_CJ_SIMD_2XNN(CI_TO_CJ_SIMD_2XNN(gridj->cell0) + cjl);
+
- dx_SSE0 = gmx_sub_pr(work->ix_SSE0, jx_SSE);
- dy_SSE0 = gmx_sub_pr(work->iy_SSE0, jy_SSE);
- dz_SSE0 = gmx_sub_pr(work->iz_SSE0, jz_SSE);
- dx_SSE2 = gmx_sub_pr(work->ix_SSE2, jx_SSE);
- dy_SSE2 = gmx_sub_pr(work->iy_SSE2, jy_SSE);
- dz_SSE2 = gmx_sub_pr(work->iz_SSE2, jz_SSE);
++ jx_S = gmx_load_hpr_hilo_pr(x_j+xind_l+0*STRIDE_S);
++ jy_S = gmx_load_hpr_hilo_pr(x_j+xind_l+1*STRIDE_S);
++ jz_S = gmx_load_hpr_hilo_pr(x_j+xind_l+2*STRIDE_S);
+
+ /* Calculate distance */
- rsq_SSE0 = gmx_calc_rsq_pr(dx_SSE0, dy_SSE0, dz_SSE0);
- rsq_SSE2 = gmx_calc_rsq_pr(dx_SSE2, dy_SSE2, dz_SSE2);
++ dx_S0 = gmx_sub_pr(work->ix_S0, jx_S);
++ dy_S0 = gmx_sub_pr(work->iy_S0, jy_S);
++ dz_S0 = gmx_sub_pr(work->iz_S0, jz_S);
++ dx_S2 = gmx_sub_pr(work->ix_S2, jx_S);
++ dy_S2 = gmx_sub_pr(work->iy_S2, jy_S);
++ dz_S2 = gmx_sub_pr(work->iz_S2, jz_S);
+
+ /* rsq = dx*dx+dy*dy+dz*dz */
- wco_SSE0 = gmx_cmplt_pr(rsq_SSE0, rc2_SSE);
- wco_SSE2 = gmx_cmplt_pr(rsq_SSE2, rc2_SSE);
++ rsq_S0 = gmx_calc_rsq_pr(dx_S0, dy_S0, dz_S0);
++ rsq_S2 = gmx_calc_rsq_pr(dx_S2, dy_S2, dz_S2);
+
- wco_any_SSE = gmx_or_pb(wco_SSE0, wco_SSE2);
++ wco_S0 = gmx_cmplt_pr(rsq_S0, rc2_S);
++ wco_S2 = gmx_cmplt_pr(rsq_S2, rc2_S);
+
- #ifdef GMX_SIMD_HAVE_ANYTRUE
- InRange = gmx_anytrue_pb(wco_any_SSE);
- #else
- InRange = gmx_anytrue_2xn_pb(wco_any_SSE);
- #endif
++ wco_any_S = gmx_or_pb(wco_S0, wco_S2);
+
++ InRange = gmx_anytrue_pb(wco_any_S);
+
+ *ndistc += 2*GMX_SIMD_WIDTH_HERE;
+ }
+ if (!InRange)
+ {
+ cjl--;
+ }
+ }
+
+ if (cjf <= cjl)
+ {
+ for (cj = cjf; cj <= cjl; cj++)
+ {
+ /* Store cj and the interaction mask */
+ nbl->cj[nbl->ncj].cj = CI_TO_CJ_SIMD_2XNN(gridj->cell0) + cj;
+ nbl->cj[nbl->ncj].excl = get_imask_simd_2xnn(remove_sub_diag, ci, cj);
+ nbl->ncj++;
+ }
+ /* Increase the closing index in i super-cell list */
+ nbl->ci[nbl->nci].cj_ind_end = nbl->ncj;
+ }
+}
+
+#undef STRIDE_S
--- /dev/null
- x_ci->ix_SSE0 = gmx_set1_pr(x[ia + 0*STRIDE_S ] + shx);
- x_ci->iy_SSE0 = gmx_set1_pr(x[ia + 1*STRIDE_S ] + shy);
- x_ci->iz_SSE0 = gmx_set1_pr(x[ia + 2*STRIDE_S ] + shz);
- x_ci->ix_SSE1 = gmx_set1_pr(x[ia + 0*STRIDE_S + 1] + shx);
- x_ci->iy_SSE1 = gmx_set1_pr(x[ia + 1*STRIDE_S + 1] + shy);
- x_ci->iz_SSE1 = gmx_set1_pr(x[ia + 2*STRIDE_S + 1] + shz);
- x_ci->ix_SSE2 = gmx_set1_pr(x[ia + 0*STRIDE_S + 2] + shx);
- x_ci->iy_SSE2 = gmx_set1_pr(x[ia + 1*STRIDE_S + 2] + shy);
- x_ci->iz_SSE2 = gmx_set1_pr(x[ia + 2*STRIDE_S + 2] + shz);
- x_ci->ix_SSE3 = gmx_set1_pr(x[ia + 0*STRIDE_S + 3] + shx);
- x_ci->iy_SSE3 = gmx_set1_pr(x[ia + 1*STRIDE_S + 3] + shy);
- x_ci->iz_SSE3 = gmx_set1_pr(x[ia + 2*STRIDE_S + 3] + shz);
+/*
+ * This file is part of the GROMACS molecular simulation package.
+ *
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2012, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+ * Copyright (c) 2012, by the GROMACS development team, led by
+ * David van der Spoel, Berk Hess, Erik Lindahl, and including many
+ * others, as listed in the AUTHORS file in the top-level source
+ * directory and at http://www.gromacs.org.
+ *
+ * GROMACS is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ *
+ * GROMACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with GROMACS; if not, see
+ * http://www.gnu.org/licenses, or write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * If you want to redistribute modifications to GROMACS, please
+ * consider that scientific software is very special. Version
+ * control is crucial - bugs must be traceable. We will be happy to
+ * consider code for inclusion in the official distribution, but
+ * derived work must not be called official GROMACS. Details are found
+ * in the README & COPYING files - if they are missing, get the
+ * official version at http://www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the research papers on the package. Check out http://www.gromacs.org.
+ */
+
+
+#if GMX_SIMD_WIDTH_HERE >= NBNXN_CPU_CLUSTER_I_SIZE
+#define STRIDE_S (GMX_SIMD_WIDTH_HERE)
+#else
+#define STRIDE_S NBNXN_CPU_CLUSTER_I_SIZE
+#endif
+
+/* Copies PBC shifted i-cell packed atom coordinates to working array */
+static gmx_inline void
+icell_set_x_simd_4xn(int ci,
+ real shx, real shy, real shz,
+ int gmx_unused na_c,
+ int gmx_unused stride, const real *x,
+ nbnxn_list_work_t *work)
+{
+ int ia;
+ nbnxn_x_ci_simd_4xn_t *x_ci;
+
+ x_ci = work->x_ci_simd_4xn;
+
+ ia = X_IND_CI_SIMD_4XN(ci);
+
- #ifndef GMX_SIMD_HAVE_ANYTRUE
- /* Fallback function in case gmx_anytrue_pr is not present */
- static gmx_inline gmx_bool
- gmx_anytrue_4xn_pb(gmx_mm_pb bool_S)
- {
- real bools_array[2*GMX_SIMD_WIDTH_HERE], *bools;
- gmx_bool any;
- int s;
-
- bools = gmx_simd_align_real(bools_array);
-
- gmx_store_pb(bools, bool_S);
-
- any = FALSE;
- for (s = 0; s < GMX_SIMD_WIDTH_HERE; s++)
- {
- if (GMX_SIMD_IS_TRUE(bools[s]))
- {
- any = TRUE;
- }
- }
-
- return any;
- }
- #endif
-
++ x_ci->ix_S0 = gmx_set1_pr(x[ia + 0*STRIDE_S ] + shx);
++ x_ci->iy_S0 = gmx_set1_pr(x[ia + 1*STRIDE_S ] + shy);
++ x_ci->iz_S0 = gmx_set1_pr(x[ia + 2*STRIDE_S ] + shz);
++ x_ci->ix_S1 = gmx_set1_pr(x[ia + 0*STRIDE_S + 1] + shx);
++ x_ci->iy_S1 = gmx_set1_pr(x[ia + 1*STRIDE_S + 1] + shy);
++ x_ci->iz_S1 = gmx_set1_pr(x[ia + 2*STRIDE_S + 1] + shz);
++ x_ci->ix_S2 = gmx_set1_pr(x[ia + 0*STRIDE_S + 2] + shx);
++ x_ci->iy_S2 = gmx_set1_pr(x[ia + 1*STRIDE_S + 2] + shy);
++ x_ci->iz_S2 = gmx_set1_pr(x[ia + 2*STRIDE_S + 2] + shz);
++ x_ci->ix_S3 = gmx_set1_pr(x[ia + 0*STRIDE_S + 3] + shx);
++ x_ci->iy_S3 = gmx_set1_pr(x[ia + 1*STRIDE_S + 3] + shy);
++ x_ci->iz_S3 = gmx_set1_pr(x[ia + 2*STRIDE_S + 3] + shz);
+}
+
- gmx_mm_pr jx_SSE, jy_SSE, jz_SSE;
+/* SIMD code for making a pair list of cell ci vs cell cjf-cjl
+ * for coordinates in packed format.
+ * Checks bouding box distances and possibly atom pair distances.
+ * This is an accelerated version of make_cluster_list_simple.
+ */
+static gmx_inline void
+make_cluster_list_simd_4xn(const nbnxn_grid_t *gridj,
+ nbnxn_pairlist_t *nbl,
+ int ci, int cjf, int cjl,
+ gmx_bool remove_sub_diag,
+ const real *x_j,
+ real rl2, float rbb2,
+ int *ndistc)
+{
+ const nbnxn_x_ci_simd_4xn_t *work;
+ const nbnxn_bb_t *bb_ci;
+
- gmx_mm_pr dx_SSE0, dy_SSE0, dz_SSE0;
- gmx_mm_pr dx_SSE1, dy_SSE1, dz_SSE1;
- gmx_mm_pr dx_SSE2, dy_SSE2, dz_SSE2;
- gmx_mm_pr dx_SSE3, dy_SSE3, dz_SSE3;
++ gmx_mm_pr jx_S, jy_S, jz_S;
+
- gmx_mm_pr rsq_SSE0;
- gmx_mm_pr rsq_SSE1;
- gmx_mm_pr rsq_SSE2;
- gmx_mm_pr rsq_SSE3;
++ gmx_mm_pr dx_S0, dy_S0, dz_S0;
++ gmx_mm_pr dx_S1, dy_S1, dz_S1;
++ gmx_mm_pr dx_S2, dy_S2, dz_S2;
++ gmx_mm_pr dx_S3, dy_S3, dz_S3;
+
- gmx_mm_pb wco_SSE0;
- gmx_mm_pb wco_SSE1;
- gmx_mm_pb wco_SSE2;
- gmx_mm_pb wco_SSE3;
- gmx_mm_pb wco_any_SSE01, wco_any_SSE23, wco_any_SSE;
++ gmx_mm_pr rsq_S0;
++ gmx_mm_pr rsq_S1;
++ gmx_mm_pr rsq_S2;
++ gmx_mm_pr rsq_S3;
+
- gmx_mm_pr rc2_SSE;
++ gmx_mm_pb wco_S0;
++ gmx_mm_pb wco_S1;
++ gmx_mm_pb wco_S2;
++ gmx_mm_pb wco_S3;
++ gmx_mm_pb wco_any_S01, wco_any_S23, wco_any_S;
+
- rc2_SSE = gmx_set1_pr(rl2);
++ gmx_mm_pr rc2_S;
+
+ gmx_bool InRange;
+ float d2;
+ int xind_f, xind_l, cj;
+
+ cjf = CI_TO_CJ_SIMD_4XN(cjf);
+ cjl = CI_TO_CJ_SIMD_4XN(cjl+1) - 1;
+
+ work = nbl->work->x_ci_simd_4xn;
+
+ bb_ci = nbl->work->bb_ci;
+
- #ifdef NBNXN_SEARCH_BB_SSE
- d2 = subc_bb_dist2_sse(0, bb_ci, cjf, gridj->bbj);
++ rc2_S = gmx_set1_pr(rl2);
+
+ InRange = FALSE;
+ while (!InRange && cjf <= cjl)
+ {
- jx_SSE = gmx_load_pr(x_j+xind_f+0*STRIDE_S);
- jy_SSE = gmx_load_pr(x_j+xind_f+1*STRIDE_S);
- jz_SSE = gmx_load_pr(x_j+xind_f+2*STRIDE_S);
++#ifdef NBNXN_SEARCH_BB_SIMD4
++ d2 = subc_bb_dist2_simd4(0, bb_ci, cjf, gridj->bbj);
+#else
+ d2 = subc_bb_dist2(0, bb_ci, cjf, gridj->bbj);
+#endif
+ *ndistc += 2;
+
+ /* Check if the distance is within the distance where
+ * we use only the bounding box distance rbb,
+ * or within the cut-off and there is at least one atom pair
+ * within the cut-off.
+ */
+ if (d2 < rbb2)
+ {
+ InRange = TRUE;
+ }
+ else if (d2 < rl2)
+ {
+ xind_f = X_IND_CJ_SIMD_4XN(CI_TO_CJ_SIMD_4XN(gridj->cell0) + cjf);
+
- dx_SSE0 = gmx_sub_pr(work->ix_SSE0, jx_SSE);
- dy_SSE0 = gmx_sub_pr(work->iy_SSE0, jy_SSE);
- dz_SSE0 = gmx_sub_pr(work->iz_SSE0, jz_SSE);
- dx_SSE1 = gmx_sub_pr(work->ix_SSE1, jx_SSE);
- dy_SSE1 = gmx_sub_pr(work->iy_SSE1, jy_SSE);
- dz_SSE1 = gmx_sub_pr(work->iz_SSE1, jz_SSE);
- dx_SSE2 = gmx_sub_pr(work->ix_SSE2, jx_SSE);
- dy_SSE2 = gmx_sub_pr(work->iy_SSE2, jy_SSE);
- dz_SSE2 = gmx_sub_pr(work->iz_SSE2, jz_SSE);
- dx_SSE3 = gmx_sub_pr(work->ix_SSE3, jx_SSE);
- dy_SSE3 = gmx_sub_pr(work->iy_SSE3, jy_SSE);
- dz_SSE3 = gmx_sub_pr(work->iz_SSE3, jz_SSE);
++ jx_S = gmx_load_pr(x_j+xind_f+0*STRIDE_S);
++ jy_S = gmx_load_pr(x_j+xind_f+1*STRIDE_S);
++ jz_S = gmx_load_pr(x_j+xind_f+2*STRIDE_S);
+
+
+ /* Calculate distance */
- rsq_SSE0 = gmx_calc_rsq_pr(dx_SSE0, dy_SSE0, dz_SSE0);
- rsq_SSE1 = gmx_calc_rsq_pr(dx_SSE1, dy_SSE1, dz_SSE1);
- rsq_SSE2 = gmx_calc_rsq_pr(dx_SSE2, dy_SSE2, dz_SSE2);
- rsq_SSE3 = gmx_calc_rsq_pr(dx_SSE3, dy_SSE3, dz_SSE3);
-
- wco_SSE0 = gmx_cmplt_pr(rsq_SSE0, rc2_SSE);
- wco_SSE1 = gmx_cmplt_pr(rsq_SSE1, rc2_SSE);
- wco_SSE2 = gmx_cmplt_pr(rsq_SSE2, rc2_SSE);
- wco_SSE3 = gmx_cmplt_pr(rsq_SSE3, rc2_SSE);
-
- wco_any_SSE01 = gmx_or_pb(wco_SSE0, wco_SSE1);
- wco_any_SSE23 = gmx_or_pb(wco_SSE2, wco_SSE3);
- wco_any_SSE = gmx_or_pb(wco_any_SSE01, wco_any_SSE23);
-
- #ifdef GMX_SIMD_HAVE_ANYTRUE
- InRange = gmx_anytrue_pb(wco_any_SSE);
- #else
- InRange = gmx_anytrue_4xn_pb(wco_any_SSE);
- #endif
++ dx_S0 = gmx_sub_pr(work->ix_S0, jx_S);
++ dy_S0 = gmx_sub_pr(work->iy_S0, jy_S);
++ dz_S0 = gmx_sub_pr(work->iz_S0, jz_S);
++ dx_S1 = gmx_sub_pr(work->ix_S1, jx_S);
++ dy_S1 = gmx_sub_pr(work->iy_S1, jy_S);
++ dz_S1 = gmx_sub_pr(work->iz_S1, jz_S);
++ dx_S2 = gmx_sub_pr(work->ix_S2, jx_S);
++ dy_S2 = gmx_sub_pr(work->iy_S2, jy_S);
++ dz_S2 = gmx_sub_pr(work->iz_S2, jz_S);
++ dx_S3 = gmx_sub_pr(work->ix_S3, jx_S);
++ dy_S3 = gmx_sub_pr(work->iy_S3, jy_S);
++ dz_S3 = gmx_sub_pr(work->iz_S3, jz_S);
+
+ /* rsq = dx*dx+dy*dy+dz*dz */
- #ifdef NBNXN_SEARCH_BB_SSE
- d2 = subc_bb_dist2_sse(0, bb_ci, cjl, gridj->bbj);
++ rsq_S0 = gmx_calc_rsq_pr(dx_S0, dy_S0, dz_S0);
++ rsq_S1 = gmx_calc_rsq_pr(dx_S1, dy_S1, dz_S1);
++ rsq_S2 = gmx_calc_rsq_pr(dx_S2, dy_S2, dz_S2);
++ rsq_S3 = gmx_calc_rsq_pr(dx_S3, dy_S3, dz_S3);
++
++ wco_S0 = gmx_cmplt_pr(rsq_S0, rc2_S);
++ wco_S1 = gmx_cmplt_pr(rsq_S1, rc2_S);
++ wco_S2 = gmx_cmplt_pr(rsq_S2, rc2_S);
++ wco_S3 = gmx_cmplt_pr(rsq_S3, rc2_S);
++
++ wco_any_S01 = gmx_or_pb(wco_S0, wco_S1);
++ wco_any_S23 = gmx_or_pb(wco_S2, wco_S3);
++ wco_any_S = gmx_or_pb(wco_any_S01, wco_any_S23);
++
++ InRange = gmx_anytrue_pb(wco_any_S);
+
+ *ndistc += 4*GMX_SIMD_WIDTH_HERE;
+ }
+ if (!InRange)
+ {
+ cjf++;
+ }
+ }
+ if (!InRange)
+ {
+ return;
+ }
+
+ InRange = FALSE;
+ while (!InRange && cjl > cjf)
+ {
- jx_SSE = gmx_load_pr(x_j+xind_l+0*STRIDE_S);
- jy_SSE = gmx_load_pr(x_j+xind_l+1*STRIDE_S);
- jz_SSE = gmx_load_pr(x_j+xind_l+2*STRIDE_S);
++#ifdef NBNXN_SEARCH_BB_SIMD4
++ d2 = subc_bb_dist2_simd4(0, bb_ci, cjl, gridj->bbj);
+#else
+ d2 = subc_bb_dist2(0, bb_ci, cjl, gridj->bbj);
+#endif
+ *ndistc += 2;
+
+ /* Check if the distance is within the distance where
+ * we use only the bounding box distance rbb,
+ * or within the cut-off and there is at least one atom pair
+ * within the cut-off.
+ */
+ if (d2 < rbb2)
+ {
+ InRange = TRUE;
+ }
+ else if (d2 < rl2)
+ {
+ xind_l = X_IND_CJ_SIMD_4XN(CI_TO_CJ_SIMD_4XN(gridj->cell0) + cjl);
+
- dx_SSE0 = gmx_sub_pr(work->ix_SSE0, jx_SSE);
- dy_SSE0 = gmx_sub_pr(work->iy_SSE0, jy_SSE);
- dz_SSE0 = gmx_sub_pr(work->iz_SSE0, jz_SSE);
- dx_SSE1 = gmx_sub_pr(work->ix_SSE1, jx_SSE);
- dy_SSE1 = gmx_sub_pr(work->iy_SSE1, jy_SSE);
- dz_SSE1 = gmx_sub_pr(work->iz_SSE1, jz_SSE);
- dx_SSE2 = gmx_sub_pr(work->ix_SSE2, jx_SSE);
- dy_SSE2 = gmx_sub_pr(work->iy_SSE2, jy_SSE);
- dz_SSE2 = gmx_sub_pr(work->iz_SSE2, jz_SSE);
- dx_SSE3 = gmx_sub_pr(work->ix_SSE3, jx_SSE);
- dy_SSE3 = gmx_sub_pr(work->iy_SSE3, jy_SSE);
- dz_SSE3 = gmx_sub_pr(work->iz_SSE3, jz_SSE);
++ jx_S = gmx_load_pr(x_j+xind_l+0*STRIDE_S);
++ jy_S = gmx_load_pr(x_j+xind_l+1*STRIDE_S);
++ jz_S = gmx_load_pr(x_j+xind_l+2*STRIDE_S);
+
+ /* Calculate distance */
- rsq_SSE0 = gmx_calc_rsq_pr(dx_SSE0, dy_SSE0, dz_SSE0);
- rsq_SSE1 = gmx_calc_rsq_pr(dx_SSE1, dy_SSE1, dz_SSE1);
- rsq_SSE2 = gmx_calc_rsq_pr(dx_SSE2, dy_SSE2, dz_SSE2);
- rsq_SSE3 = gmx_calc_rsq_pr(dx_SSE3, dy_SSE3, dz_SSE3);
-
- wco_SSE0 = gmx_cmplt_pr(rsq_SSE0, rc2_SSE);
- wco_SSE1 = gmx_cmplt_pr(rsq_SSE1, rc2_SSE);
- wco_SSE2 = gmx_cmplt_pr(rsq_SSE2, rc2_SSE);
- wco_SSE3 = gmx_cmplt_pr(rsq_SSE3, rc2_SSE);
-
- wco_any_SSE01 = gmx_or_pb(wco_SSE0, wco_SSE1);
- wco_any_SSE23 = gmx_or_pb(wco_SSE2, wco_SSE3);
- wco_any_SSE = gmx_or_pb(wco_any_SSE01, wco_any_SSE23);
-
- #ifdef GMX_SIMD_HAVE_ANYTRUE
- InRange = gmx_anytrue_pb(wco_any_SSE);
- #else
- InRange = gmx_anytrue_4xn_pb(wco_any_SSE);
- #endif
++ dx_S0 = gmx_sub_pr(work->ix_S0, jx_S);
++ dy_S0 = gmx_sub_pr(work->iy_S0, jy_S);
++ dz_S0 = gmx_sub_pr(work->iz_S0, jz_S);
++ dx_S1 = gmx_sub_pr(work->ix_S1, jx_S);
++ dy_S1 = gmx_sub_pr(work->iy_S1, jy_S);
++ dz_S1 = gmx_sub_pr(work->iz_S1, jz_S);
++ dx_S2 = gmx_sub_pr(work->ix_S2, jx_S);
++ dy_S2 = gmx_sub_pr(work->iy_S2, jy_S);
++ dz_S2 = gmx_sub_pr(work->iz_S2, jz_S);
++ dx_S3 = gmx_sub_pr(work->ix_S3, jx_S);
++ dy_S3 = gmx_sub_pr(work->iy_S3, jy_S);
++ dz_S3 = gmx_sub_pr(work->iz_S3, jz_S);
+
+ /* rsq = dx*dx+dy*dy+dz*dz */
-
++ rsq_S0 = gmx_calc_rsq_pr(dx_S0, dy_S0, dz_S0);
++ rsq_S1 = gmx_calc_rsq_pr(dx_S1, dy_S1, dz_S1);
++ rsq_S2 = gmx_calc_rsq_pr(dx_S2, dy_S2, dz_S2);
++ rsq_S3 = gmx_calc_rsq_pr(dx_S3, dy_S3, dz_S3);
++
++ wco_S0 = gmx_cmplt_pr(rsq_S0, rc2_S);
++ wco_S1 = gmx_cmplt_pr(rsq_S1, rc2_S);
++ wco_S2 = gmx_cmplt_pr(rsq_S2, rc2_S);
++ wco_S3 = gmx_cmplt_pr(rsq_S3, rc2_S);
++
++ wco_any_S01 = gmx_or_pb(wco_S0, wco_S1);
++ wco_any_S23 = gmx_or_pb(wco_S2, wco_S3);
++ wco_any_S = gmx_or_pb(wco_any_S01, wco_any_S23);
++
++ InRange = gmx_anytrue_pb(wco_any_S);
+
+ *ndistc += 4*GMX_SIMD_WIDTH_HERE;
+ }
+ if (!InRange)
+ {
+ cjl--;
+ }
+ }
+
+ if (cjf <= cjl)
+ {
+ for (cj = cjf; cj <= cjl; cj++)
+ {
+ /* Store cj and the interaction mask */
+ nbl->cj[nbl->ncj].cj = CI_TO_CJ_SIMD_4XN(gridj->cell0) + cj;
+ nbl->cj[nbl->ncj].excl = get_imask_simd_4xn(remove_sub_diag, ci, cj);
++#ifdef GMX_CPU_ACCELERATION_IBM_QPX
++ nbl->cj[nbl->ncj].interaction_mask_indices[0] = (nbl->cj[nbl->ncj].excl & 0x000F) >> (0 * 4);
++ nbl->cj[nbl->ncj].interaction_mask_indices[1] = (nbl->cj[nbl->ncj].excl & 0x00F0) >> (1 * 4);
++ nbl->cj[nbl->ncj].interaction_mask_indices[2] = (nbl->cj[nbl->ncj].excl & 0x0F00) >> (2 * 4);
++ nbl->cj[nbl->ncj].interaction_mask_indices[3] = (nbl->cj[nbl->ncj].excl & 0xF000) >> (3 * 4);
++#endif
+ nbl->ncj++;
+ }
+ /* Increase the closing index in i super-cell list */
+ nbl->ci[nbl->nci].cj_ind_end = nbl->ncj;
+ }
+}
+
+#undef STRIDE_S
--- /dev/null
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * VERSION 3.2.0
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2004, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * GROwing Monsters And Cloning Shrimps
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <math.h>
+#include <string.h>
+#include "sysstuff.h"
+#include "smalloc.h"
+#include "macros.h"
+#include "maths.h"
+#include "vec.h"
+#include "network.h"
+#include "nsgrid.h"
+#include "force.h"
+#include "nonbonded.h"
+#include "ns.h"
+#include "pbc.h"
+#include "names.h"
+#include "gmx_fatal.h"
+#include "nrnb.h"
+#include "txtdump.h"
+#include "mtop_util.h"
+
+#include "domdec.h"
+#include "adress.h"
+
+
+/*
+ * E X C L U S I O N H A N D L I N G
+ */
+
+#ifdef DEBUG
+static void SETEXCL_(t_excl e[], atom_id i, atom_id j)
+{
+ e[j] = e[j] | (1<<i);
+}
+static void RMEXCL_(t_excl e[], atom_id i, atom_id j)
+{
+ e[j] = e[j] & ~(1<<i);
+}
+static gmx_bool ISEXCL_(t_excl e[], atom_id i, atom_id j)
+{
+ return (gmx_bool)(e[j] & (1<<i));
+}
+static gmx_bool NOTEXCL_(t_excl e[], atom_id i, atom_id j)
+{
+ return !(ISEXCL(e, i, j));
+}
+#else
+#define SETEXCL(e, i, j) (e)[((atom_id) (j))] |= (1<<((atom_id) (i)))
+#define RMEXCL(e, i, j) (e)[((atom_id) (j))] &= (~(1<<((atom_id) (i))))
+#define ISEXCL(e, i, j) (gmx_bool) ((e)[((atom_id) (j))] & (1<<((atom_id) (i))))
+#define NOTEXCL(e, i, j) !(ISEXCL(e, i, j))
+#endif
+
+static int
+round_up_to_simd_width(int length, int simd_width)
+{
+ int offset, newlength;
+
+ offset = (simd_width > 0) ? length % simd_width : 0;
+
+ return (offset == 0) ? length : length-offset+simd_width;
+}
+/************************************************
+ *
+ * U T I L I T I E S F O R N S
+ *
+ ************************************************/
+
+static void reallocate_nblist(t_nblist *nl)
+{
+ if (gmx_debug_at)
+ {
+ fprintf(debug, "reallocating neigborlist (ielec=%d, ivdw=%d, igeometry=%d, type=%d), maxnri=%d\n",
+ nl->ielec, nl->ivdw, nl->igeometry, nl->type, nl->maxnri);
+ }
+ srenew(nl->iinr, nl->maxnri);
+ if (nl->igeometry == GMX_NBLIST_GEOMETRY_CG_CG)
+ {
+ srenew(nl->iinr_end, nl->maxnri);
+ }
+ srenew(nl->gid, nl->maxnri);
+ srenew(nl->shift, nl->maxnri);
+ srenew(nl->jindex, nl->maxnri+1);
+}
+
+
+static void init_nblist(FILE *log, t_nblist *nl_sr, t_nblist *nl_lr,
+ int maxsr, int maxlr,
+ int ivdw, int ivdwmod,
+ int ielec, int ielecmod,
+ int igeometry, int type)
+{
+ t_nblist *nl;
+ int homenr;
+ int i, nn;
+
+ for (i = 0; (i < 2); i++)
+ {
+ nl = (i == 0) ? nl_sr : nl_lr;
+ homenr = (i == 0) ? maxsr : maxlr;
+
+ if (nl == NULL)
+ {
+ continue;
+ }
+
+
+ /* Set coul/vdw in neighborlist, and for the normal loops we determine
+ * an index of which one to call.
+ */
+ nl->ivdw = ivdw;
+ nl->ivdwmod = ivdwmod;
+ nl->ielec = ielec;
+ nl->ielecmod = ielecmod;
+ nl->type = type;
+ nl->igeometry = igeometry;
+
+ if (nl->type == GMX_NBLIST_INTERACTION_FREE_ENERGY)
+ {
+ nl->igeometry = GMX_NBLIST_GEOMETRY_PARTICLE_PARTICLE;
+ }
+
+ /* This will also set the simd_padding_width field */
+ gmx_nonbonded_set_kernel_pointers( (i == 0) ? log : NULL, nl);
+
+ /* maxnri is influenced by the number of shifts (maximum is 8)
+ * and the number of energy groups.
+ * If it is not enough, nl memory will be reallocated during the run.
+ * 4 seems to be a reasonable factor, which only causes reallocation
+ * during runs with tiny and many energygroups.
+ */
+ nl->maxnri = homenr*4;
+ nl->maxnrj = 0;
+ nl->maxlen = 0;
+ nl->nri = -1;
+ nl->nrj = 0;
+ nl->iinr = NULL;
+ nl->gid = NULL;
+ nl->shift = NULL;
+ nl->jindex = NULL;
+ reallocate_nblist(nl);
+ nl->jindex[0] = 0;
+
+ if (debug)
+ {
+ fprintf(debug, "Initiating neighbourlist (ielec=%d, ivdw=%d, type=%d) for %s interactions,\nwith %d SR, %d LR atoms.\n",
+ nl->ielec, nl->ivdw, nl->type, gmx_nblist_geometry_names[nl->igeometry], maxsr, maxlr);
+ }
+ }
+}
+
+void init_neighbor_list(FILE *log, t_forcerec *fr, int homenr)
+{
+ /* Make maxlr tunable! (does not seem to be a big difference though)
+ * This parameter determines the number of i particles in a long range
+ * neighbourlist. Too few means many function calls, too many means
+ * cache trashing.
+ */
+ int maxsr, maxsr_wat, maxlr, maxlr_wat;
+ int ielec, ielecf, ivdw, ielecmod, ielecmodf, ivdwmod, type;
+ int solvent;
+ int igeometry_def, igeometry_w, igeometry_ww;
+ int i;
+ t_nblists *nbl;
+
+ /* maxsr = homenr-fr->nWatMol*3; */
+ maxsr = homenr;
+
+ if (maxsr < 0)
+ {
+ gmx_fatal(FARGS, "%s, %d: Negative number of short range atoms.\n"
+ "Call your Gromacs dealer for assistance.", __FILE__, __LINE__);
+ }
+ /* This is just for initial allocation, so we do not reallocate
+ * all the nlist arrays many times in a row.
+ * The numbers seem very accurate, but they are uncritical.
+ */
+ maxsr_wat = min(fr->nWatMol, (homenr+2)/3);
+ if (fr->bTwinRange)
+ {
+ maxlr = 50;
+ maxlr_wat = min(maxsr_wat, maxlr);
+ }
+ else
+ {
+ maxlr = maxlr_wat = 0;
+ }
+
+ /* Determine the values for ielec/ivdw. */
+ ielec = fr->nbkernel_elec_interaction;
+ ivdw = fr->nbkernel_vdw_interaction;
+ ielecmod = fr->nbkernel_elec_modifier;
+ ivdwmod = fr->nbkernel_vdw_modifier;
+ type = GMX_NBLIST_INTERACTION_STANDARD;
+
+ fr->ns.bCGlist = (getenv("GMX_NBLISTCG") != 0);
+ if (!fr->ns.bCGlist)
+ {
+ igeometry_def = GMX_NBLIST_GEOMETRY_PARTICLE_PARTICLE;
+ }
+ else
+ {
+ igeometry_def = GMX_NBLIST_GEOMETRY_CG_CG;
+ if (log != NULL)
+ {
+ fprintf(log, "\nUsing charge-group - charge-group neighbor lists and kernels\n\n");
+ }
+ }
+
+ if (fr->solvent_opt == esolTIP4P)
+ {
+ igeometry_w = GMX_NBLIST_GEOMETRY_WATER4_PARTICLE;
+ igeometry_ww = GMX_NBLIST_GEOMETRY_WATER4_WATER4;
+ }
+ else
+ {
+ igeometry_w = GMX_NBLIST_GEOMETRY_WATER3_PARTICLE;
+ igeometry_ww = GMX_NBLIST_GEOMETRY_WATER3_WATER3;
+ }
+
+ for (i = 0; i < fr->nnblists; i++)
+ {
+ nbl = &(fr->nblists[i]);
+
+ if ((fr->adress_type != eAdressOff) && (i >= fr->nnblists/2))
+ {
+ type = GMX_NBLIST_INTERACTION_ADRESS;
+ }
+ init_nblist(log, &nbl->nlist_sr[eNL_VDWQQ], &nbl->nlist_lr[eNL_VDWQQ],
+ maxsr, maxlr, ivdw, ivdwmod, ielec, ielecmod, igeometry_def, type);
+ init_nblist(log, &nbl->nlist_sr[eNL_VDW], &nbl->nlist_lr[eNL_VDW],
+ maxsr, maxlr, ivdw, ivdwmod, GMX_NBKERNEL_ELEC_NONE, eintmodNONE, igeometry_def, type);
+ init_nblist(log, &nbl->nlist_sr[eNL_QQ], &nbl->nlist_lr[eNL_QQ],
+ maxsr, maxlr, GMX_NBKERNEL_VDW_NONE, eintmodNONE, ielec, ielecmod, igeometry_def, type);
+ init_nblist(log, &nbl->nlist_sr[eNL_VDWQQ_WATER], &nbl->nlist_lr[eNL_VDWQQ_WATER],
+ maxsr_wat, maxlr_wat, ivdw, ivdwmod, ielec, ielecmod, igeometry_w, type);
+ init_nblist(log, &nbl->nlist_sr[eNL_QQ_WATER], &nbl->nlist_lr[eNL_QQ_WATER],
+ maxsr_wat, maxlr_wat, GMX_NBKERNEL_VDW_NONE, eintmodNONE, ielec, ielecmod, igeometry_w, type);
+ init_nblist(log, &nbl->nlist_sr[eNL_VDWQQ_WATERWATER], &nbl->nlist_lr[eNL_VDWQQ_WATERWATER],
+ maxsr_wat, maxlr_wat, ivdw, ivdwmod, ielec, ielecmod, igeometry_ww, type);
+ init_nblist(log, &nbl->nlist_sr[eNL_QQ_WATERWATER], &nbl->nlist_lr[eNL_QQ_WATERWATER],
+ maxsr_wat, maxlr_wat, GMX_NBKERNEL_VDW_NONE, eintmodNONE, ielec, ielecmod, igeometry_ww, type);
+
+ /* Did we get the solvent loops so we can use optimized water kernels? */
+ if (nbl->nlist_sr[eNL_VDWQQ_WATER].kernelptr_vf == NULL
+ || nbl->nlist_sr[eNL_QQ_WATER].kernelptr_vf == NULL
+#ifndef DISABLE_WATERWATER_NLIST
+ || nbl->nlist_sr[eNL_VDWQQ_WATERWATER].kernelptr_vf == NULL
+ || nbl->nlist_sr[eNL_QQ_WATERWATER].kernelptr_vf == NULL
+#endif
+ )
+ {
+ fr->solvent_opt = esolNO;
+ fprintf(log, "Note: The available nonbonded kernels do not support water optimization - disabling.\n");
+ }
+
+ if (fr->efep != efepNO)
+ {
+ if ((fr->bEwald) && (fr->sc_alphacoul > 0)) /* need to handle long range differently if using softcore */
+ {
+ ielecf = GMX_NBKERNEL_ELEC_EWALD;
+ ielecmodf = eintmodNONE;
+ }
+ else
+ {
+ ielecf = ielec;
+ ielecmodf = ielecmod;
+ }
+
+ init_nblist(log, &nbl->nlist_sr[eNL_VDWQQ_FREE], &nbl->nlist_lr[eNL_VDWQQ_FREE],
+ maxsr, maxlr, ivdw, ivdwmod, ielecf, ielecmod, GMX_NBLIST_GEOMETRY_PARTICLE_PARTICLE, GMX_NBLIST_INTERACTION_FREE_ENERGY);
+ init_nblist(log, &nbl->nlist_sr[eNL_VDW_FREE], &nbl->nlist_lr[eNL_VDW_FREE],
+ maxsr, maxlr, ivdw, ivdwmod, GMX_NBKERNEL_ELEC_NONE, eintmodNONE, GMX_NBLIST_GEOMETRY_PARTICLE_PARTICLE, GMX_NBLIST_INTERACTION_FREE_ENERGY);
+ init_nblist(log, &nbl->nlist_sr[eNL_QQ_FREE], &nbl->nlist_lr[eNL_QQ_FREE],
+ maxsr, maxlr, GMX_NBKERNEL_VDW_NONE, eintmodNONE, ielecf, ielecmod, GMX_NBLIST_GEOMETRY_PARTICLE_PARTICLE, GMX_NBLIST_INTERACTION_FREE_ENERGY);
+ }
+ }
+ /* QMMM MM list */
+ if (fr->bQMMM && fr->qr->QMMMscheme != eQMMMschemeoniom)
+ {
+ init_nblist(log, &fr->QMMMlist, NULL,
+ maxsr, maxlr, 0, 0, ielec, ielecmod, GMX_NBLIST_GEOMETRY_PARTICLE_PARTICLE, GMX_NBLIST_INTERACTION_STANDARD);
+ }
+
+ if (log != NULL)
+ {
+ fprintf(log, "\n");
+ }
+
+ fr->ns.nblist_initialized = TRUE;
+}
+
+static void reset_nblist(t_nblist *nl)
+{
+ nl->nri = -1;
+ nl->nrj = 0;
+ nl->maxlen = 0;
+ if (nl->jindex)
+ {
+ nl->jindex[0] = 0;
+ }
+}
+
+static void reset_neighbor_lists(t_forcerec *fr, gmx_bool bResetSR, gmx_bool bResetLR)
+{
+ int n, i;
+
+ if (fr->bQMMM)
+ {
+ /* only reset the short-range nblist */
+ reset_nblist(&(fr->QMMMlist));
+ }
+
+ for (n = 0; n < fr->nnblists; n++)
+ {
+ for (i = 0; i < eNL_NR; i++)
+ {
+ if (bResetSR)
+ {
+ reset_nblist( &(fr->nblists[n].nlist_sr[i]) );
+ }
+ if (bResetLR)
+ {
+ reset_nblist( &(fr->nblists[n].nlist_lr[i]) );
+ }
+ }
+ }
+}
+
+
+
+
+static inline void new_i_nblist(t_nblist *nlist, atom_id i_atom, int shift, int gid)
+{
+ int i, k, nri, nshift;
+
+ nri = nlist->nri;
+
+ /* Check whether we have to increase the i counter */
+ if ((nri == -1) ||
+ (nlist->iinr[nri] != i_atom) ||
+ (nlist->shift[nri] != shift) ||
+ (nlist->gid[nri] != gid))
+ {
+ /* This is something else. Now see if any entries have
+ * been added in the list of the previous atom.
+ */
+ if ((nri == -1) ||
+ ((nlist->jindex[nri+1] > nlist->jindex[nri]) &&
+ (nlist->gid[nri] != -1)))
+ {
+ /* If so increase the counter */
+ nlist->nri++;
+ nri++;
+ if (nlist->nri >= nlist->maxnri)
+ {
+ nlist->maxnri += over_alloc_large(nlist->nri);
+ reallocate_nblist(nlist);
+ }
+ }
+ /* Set the number of neighbours and the atom number */
+ nlist->jindex[nri+1] = nlist->jindex[nri];
+ nlist->iinr[nri] = i_atom;
+ nlist->gid[nri] = gid;
+ nlist->shift[nri] = shift;
+ }
++ else
++ {
++ /* Adding to previous list. First remove possible previous padding */
++ if(nlist->simd_padding_width>1)
++ {
++ while(nlist->nrj>0 && nlist->jjnr[nlist->nrj-1]<0)
++ {
++ nlist->nrj--;
++ }
++ }
++ }
+}
+
+static inline void close_i_nblist(t_nblist *nlist)
+{
+ int nri = nlist->nri;
+ int len;
+
+ if (nri >= 0)
+ {
+ /* Add elements up to padding. Since we allocate memory in units
+ * of the simd_padding width, we do not have to check for possible
+ * list reallocation here.
+ */
+ while ((nlist->nrj % nlist->simd_padding_width) != 0)
+ {
+ /* Use -4 here, so we can write forces for 4 atoms before real data */
+ nlist->jjnr[nlist->nrj++] = -4;
+ }
+ nlist->jindex[nri+1] = nlist->nrj;
+
+ len = nlist->nrj - nlist->jindex[nri];
+
+ /* nlist length for water i molecules is treated statically
+ * in the innerloops
+ */
+ if (len > nlist->maxlen)
+ {
+ nlist->maxlen = len;
+ }
+ }
+}
+
+static inline void close_nblist(t_nblist *nlist)
+{
+ /* Only close this nblist when it has been initialized.
+ * Avoid the creation of i-lists with no j-particles.
+ */
+ if (nlist->nrj == 0)
+ {
+ /* Some assembly kernels do not support empty lists,
+ * make sure here that we don't generate any empty lists.
+ * With the current ns code this branch is taken in two cases:
+ * No i-particles at all: nri=-1 here
+ * There are i-particles, but no j-particles; nri=0 here
+ */
+ nlist->nri = 0;
+ }
+ else
+ {
+ /* Close list number nri by incrementing the count */
+ nlist->nri++;
+ }
+}
+
+static inline void close_neighbor_lists(t_forcerec *fr, gmx_bool bMakeQMMMnblist)
+{
+ int n, i;
+
+ if (bMakeQMMMnblist)
+ {
+ close_nblist(&(fr->QMMMlist));
+ }
+
+ for (n = 0; n < fr->nnblists; n++)
+ {
+ for (i = 0; (i < eNL_NR); i++)
+ {
+ close_nblist(&(fr->nblists[n].nlist_sr[i]));
+ close_nblist(&(fr->nblists[n].nlist_lr[i]));
+ }
+ }
+}
+
+
+static inline void add_j_to_nblist(t_nblist *nlist, atom_id j_atom, gmx_bool bLR)
+{
+ int nrj = nlist->nrj;
+
+ if (nlist->nrj >= nlist->maxnrj)
+ {
+ nlist->maxnrj = round_up_to_simd_width(over_alloc_small(nlist->nrj + 1), nlist->simd_padding_width);
+
+ if (gmx_debug_at)
+ {
+ fprintf(debug, "Increasing %s nblist (ielec=%d,ivdw=%d,type=%d,igeometry=%d) j size to %d\n",
+ bLR ? "LR" : "SR", nlist->ielec, nlist->ivdw, nlist->type, nlist->igeometry, nlist->maxnrj);
+ }
+
+ srenew(nlist->jjnr, nlist->maxnrj);
+ }
+
+ nlist->jjnr[nrj] = j_atom;
+ nlist->nrj++;
+}
+
+static inline void add_j_to_nblist_cg(t_nblist *nlist,
+ atom_id j_start, int j_end,
+ t_excl *bexcl, gmx_bool i_is_j,
+ gmx_bool bLR)
+{
+ int nrj = nlist->nrj;
+ int j;
+
+ if (nlist->nrj >= nlist->maxnrj)
+ {
+ nlist->maxnrj = over_alloc_small(nlist->nrj + 1);
+ if (gmx_debug_at)
+ {
+ fprintf(debug, "Increasing %s nblist (ielec=%d,ivdw=%d,type=%d,igeometry=%d) j size to %d\n",
+ bLR ? "LR" : "SR", nlist->ielec, nlist->ivdw, nlist->type, nlist->igeometry, nlist->maxnrj);
+ }
+
+ srenew(nlist->jjnr, nlist->maxnrj);
+ srenew(nlist->jjnr_end, nlist->maxnrj);
+ srenew(nlist->excl, nlist->maxnrj*MAX_CGCGSIZE);
+ }
+
+ nlist->jjnr[nrj] = j_start;
+ nlist->jjnr_end[nrj] = j_end;
+
+ if (j_end - j_start > MAX_CGCGSIZE)
+ {
+ gmx_fatal(FARGS, "The charge-group - charge-group neighborlist do not support charge groups larger than %d, found a charge group of size %d", MAX_CGCGSIZE, j_end-j_start);
+ }
+
+ /* Set the exclusions */
+ for (j = j_start; j < j_end; j++)
+ {
+ nlist->excl[nrj*MAX_CGCGSIZE + j - j_start] = bexcl[j];
+ }
+ if (i_is_j)
+ {
+ /* Avoid double counting of intra-cg interactions */
+ for (j = 1; j < j_end-j_start; j++)
+ {
+ nlist->excl[nrj*MAX_CGCGSIZE + j] |= (1<<j) - 1;
+ }
+ }
+
+ nlist->nrj++;
+}
+
+typedef void
+ put_in_list_t (gmx_bool bHaveVdW[],
+ int ngid,
+ t_mdatoms * md,
+ int icg,
+ int jgid,
+ int nj,
+ atom_id jjcg[],
+ atom_id index[],
+ t_excl bExcl[],
+ int shift,
+ t_forcerec * fr,
+ gmx_bool bLR,
+ gmx_bool bDoVdW,
+ gmx_bool bDoCoul,
+ int solvent_opt);
+
+static void
+put_in_list_at(gmx_bool bHaveVdW[],
+ int ngid,
+ t_mdatoms * md,
+ int icg,
+ int jgid,
+ int nj,
+ atom_id jjcg[],
+ atom_id index[],
+ t_excl bExcl[],
+ int shift,
+ t_forcerec * fr,
+ gmx_bool bLR,
+ gmx_bool bDoVdW,
+ gmx_bool bDoCoul,
+ int solvent_opt)
+{
+ /* The a[] index has been removed,
+ * to put it back in i_atom should be a[i0] and jj should be a[jj].
+ */
+ t_nblist * vdwc;
+ t_nblist * vdw;
+ t_nblist * coul;
+ t_nblist * vdwc_free = NULL;
+ t_nblist * vdw_free = NULL;
+ t_nblist * coul_free = NULL;
+ t_nblist * vdwc_ww = NULL;
+ t_nblist * coul_ww = NULL;
+
+ int i, j, jcg, igid, gid, nbl_ind, ind_ij;
+ atom_id jj, jj0, jj1, i_atom;
+ int i0, nicg, len;
+
+ int *cginfo;
+ int *type, *typeB;
+ real *charge, *chargeB;
+ real qi, qiB, qq, rlj;
+ gmx_bool bFreeEnergy, bFree, bFreeJ, bNotEx, *bPert;
+ gmx_bool bDoVdW_i, bDoCoul_i, bDoCoul_i_sol;
+ int iwater, jwater;
+ t_nblist *nlist;
+
+ /* Copy some pointers */
+ cginfo = fr->cginfo;
+ charge = md->chargeA;
+ chargeB = md->chargeB;
+ type = md->typeA;
+ typeB = md->typeB;
+ bPert = md->bPerturbed;
+
+ /* Get atom range */
+ i0 = index[icg];
+ nicg = index[icg+1]-i0;
+
+ /* Get the i charge group info */
+ igid = GET_CGINFO_GID(cginfo[icg]);
+
+ iwater = (solvent_opt != esolNO) ? GET_CGINFO_SOLOPT(cginfo[icg]) : esolNO;
+
+ bFreeEnergy = FALSE;
+ if (md->nPerturbed)
+ {
+ /* Check if any of the particles involved are perturbed.
+ * If not we can do the cheaper normal put_in_list
+ * and use more solvent optimization.
+ */
+ for (i = 0; i < nicg; i++)
+ {
+ bFreeEnergy |= bPert[i0+i];
+ }
+ /* Loop over the j charge groups */
+ for (j = 0; (j < nj && !bFreeEnergy); j++)
+ {
+ jcg = jjcg[j];
+ jj0 = index[jcg];
+ jj1 = index[jcg+1];
+ /* Finally loop over the atoms in the j-charge group */
+ for (jj = jj0; jj < jj1; jj++)
+ {
+ bFreeEnergy |= bPert[jj];
+ }
+ }
+ }
+
+ /* Unpack pointers to neighbourlist structs */
+ if (fr->nnblists == 1)
+ {
+ nbl_ind = 0;
+ }
+ else
+ {
+ nbl_ind = fr->gid2nblists[GID(igid, jgid, ngid)];
+ }
+ if (bLR)
+ {
+ nlist = fr->nblists[nbl_ind].nlist_lr;
+ }
+ else
+ {
+ nlist = fr->nblists[nbl_ind].nlist_sr;
+ }
+
+ if (iwater != esolNO)
+ {
+ vdwc = &nlist[eNL_VDWQQ_WATER];
+ vdw = &nlist[eNL_VDW];
+ coul = &nlist[eNL_QQ_WATER];
+#ifndef DISABLE_WATERWATER_NLIST
+ vdwc_ww = &nlist[eNL_VDWQQ_WATERWATER];
+ coul_ww = &nlist[eNL_QQ_WATERWATER];
+#endif
+ }
+ else
+ {
+ vdwc = &nlist[eNL_VDWQQ];
+ vdw = &nlist[eNL_VDW];
+ coul = &nlist[eNL_QQ];
+ }
+
+ if (!bFreeEnergy)
+ {
+ if (iwater != esolNO)
+ {
+ /* Loop over the atoms in the i charge group */
+ i_atom = i0;
+ gid = GID(igid, jgid, ngid);
+ /* Create new i_atom for each energy group */
+ if (bDoCoul && bDoVdW)
+ {
+ new_i_nblist(vdwc, i_atom, shift, gid);
+#ifndef DISABLE_WATERWATER_NLIST
+ new_i_nblist(vdwc_ww, i_atom, shift, gid);
+#endif
+ }
+ if (bDoVdW)
+ {
+ new_i_nblist(vdw, i_atom, shift, gid);
+ }
+ if (bDoCoul)
+ {
+ new_i_nblist(coul, i_atom, shift, gid);
+#ifndef DISABLE_WATERWATER_NLIST
+ new_i_nblist(coul_ww, i_atom, shift, gid);
+#endif
+ }
+ /* Loop over the j charge groups */
+ for (j = 0; (j < nj); j++)
+ {
+ jcg = jjcg[j];
+
+ if (jcg == icg)
+ {
+ continue;
+ }
+
+ jj0 = index[jcg];
+ jwater = GET_CGINFO_SOLOPT(cginfo[jcg]);
+
+ if (iwater == esolSPC && jwater == esolSPC)
+ {
+ /* Interaction between two SPC molecules */
+ if (!bDoCoul)
+ {
+ /* VdW only - only first atoms in each water interact */
+ add_j_to_nblist(vdw, jj0, bLR);
+ }
+ else
+ {
+#ifdef DISABLE_WATERWATER_NLIST
+ /* Add entries for the three atoms - only do VdW if we need to */
+ if (!bDoVdW)
+ {
+ add_j_to_nblist(coul, jj0, bLR);
+ }
+ else
+ {
+ add_j_to_nblist(vdwc, jj0, bLR);
+ }
+ add_j_to_nblist(coul, jj0+1, bLR);
+ add_j_to_nblist(coul, jj0+2, bLR);
+#else
+ /* One entry for the entire water-water interaction */
+ if (!bDoVdW)
+ {
+ add_j_to_nblist(coul_ww, jj0, bLR);
+ }
+ else
+ {
+ add_j_to_nblist(vdwc_ww, jj0, bLR);
+ }
+#endif
+ }
+ }
+ else if (iwater == esolTIP4P && jwater == esolTIP4P)
+ {
+ /* Interaction between two TIP4p molecules */
+ if (!bDoCoul)
+ {
+ /* VdW only - only first atoms in each water interact */
+ add_j_to_nblist(vdw, jj0, bLR);
+ }
+ else
+ {
+#ifdef DISABLE_WATERWATER_NLIST
+ /* Add entries for the four atoms - only do VdW if we need to */
+ if (bDoVdW)
+ {
+ add_j_to_nblist(vdw, jj0, bLR);
+ }
+ add_j_to_nblist(coul, jj0+1, bLR);
+ add_j_to_nblist(coul, jj0+2, bLR);
+ add_j_to_nblist(coul, jj0+3, bLR);
+#else
+ /* One entry for the entire water-water interaction */
+ if (!bDoVdW)
+ {
+ add_j_to_nblist(coul_ww, jj0, bLR);
+ }
+ else
+ {
+ add_j_to_nblist(vdwc_ww, jj0, bLR);
+ }
+#endif
+ }
+ }
+ else
+ {
+ /* j charge group is not water, but i is.
+ * Add entries to the water-other_atom lists; the geometry of the water
+ * molecule doesn't matter - that is taken care of in the nonbonded kernel,
+ * so we don't care if it is SPC or TIP4P...
+ */
+
+ jj1 = index[jcg+1];
+
+ if (!bDoVdW)
+ {
+ for (jj = jj0; (jj < jj1); jj++)
+ {
+ if (charge[jj] != 0)
+ {
+ add_j_to_nblist(coul, jj, bLR);
+ }
+ }
+ }
+ else if (!bDoCoul)
+ {
+ for (jj = jj0; (jj < jj1); jj++)
+ {
+ if (bHaveVdW[type[jj]])
+ {
+ add_j_to_nblist(vdw, jj, bLR);
+ }
+ }
+ }
+ else
+ {
+ /* _charge_ _groups_ interact with both coulomb and LJ */
+ /* Check which atoms we should add to the lists! */
+ for (jj = jj0; (jj < jj1); jj++)
+ {
+ if (bHaveVdW[type[jj]])
+ {
+ if (charge[jj] != 0)
+ {
+ add_j_to_nblist(vdwc, jj, bLR);
+ }
+ else
+ {
+ add_j_to_nblist(vdw, jj, bLR);
+ }
+ }
+ else if (charge[jj] != 0)
+ {
+ add_j_to_nblist(coul, jj, bLR);
+ }
+ }
+ }
+ }
+ }
+ close_i_nblist(vdw);
+ close_i_nblist(coul);
+ close_i_nblist(vdwc);
+#ifndef DISABLE_WATERWATER_NLIST
+ close_i_nblist(coul_ww);
+ close_i_nblist(vdwc_ww);
+#endif
+ }
+ else
+ {
+ /* no solvent as i charge group */
+ /* Loop over the atoms in the i charge group */
+ for (i = 0; i < nicg; i++)
+ {
+ i_atom = i0+i;
+ gid = GID(igid, jgid, ngid);
+ qi = charge[i_atom];
+
+ /* Create new i_atom for each energy group */
+ if (bDoVdW && bDoCoul)
+ {
+ new_i_nblist(vdwc, i_atom, shift, gid);
+ }
+ if (bDoVdW)
+ {
+ new_i_nblist(vdw, i_atom, shift, gid);
+ }
+ if (bDoCoul)
+ {
+ new_i_nblist(coul, i_atom, shift, gid);
+ }
+ bDoVdW_i = (bDoVdW && bHaveVdW[type[i_atom]]);
+ bDoCoul_i = (bDoCoul && qi != 0);
+
+ if (bDoVdW_i || bDoCoul_i)
+ {
+ /* Loop over the j charge groups */
+ for (j = 0; (j < nj); j++)
+ {
+ jcg = jjcg[j];
+
+ /* Check for large charge groups */
+ if (jcg == icg)
+ {
+ jj0 = i0 + i + 1;
+ }
+ else
+ {
+ jj0 = index[jcg];
+ }
+
+ jj1 = index[jcg+1];
+ /* Finally loop over the atoms in the j-charge group */
+ for (jj = jj0; jj < jj1; jj++)
+ {
+ bNotEx = NOTEXCL(bExcl, i, jj);
+
+ if (bNotEx)
+ {
+ if (!bDoVdW_i)
+ {
+ if (charge[jj] != 0)
+ {
+ add_j_to_nblist(coul, jj, bLR);
+ }
+ }
+ else if (!bDoCoul_i)
+ {
+ if (bHaveVdW[type[jj]])
+ {
+ add_j_to_nblist(vdw, jj, bLR);
+ }
+ }
+ else
+ {
+ if (bHaveVdW[type[jj]])
+ {
+ if (charge[jj] != 0)
+ {
+ add_j_to_nblist(vdwc, jj, bLR);
+ }
+ else
+ {
+ add_j_to_nblist(vdw, jj, bLR);
+ }
+ }
+ else if (charge[jj] != 0)
+ {
+ add_j_to_nblist(coul, jj, bLR);
+ }
+ }
+ }
+ }
+ }
+ }
+ close_i_nblist(vdw);
+ close_i_nblist(coul);
+ close_i_nblist(vdwc);
+ }
+ }
+ }
+ else
+ {
+ /* we are doing free energy */
+ vdwc_free = &nlist[eNL_VDWQQ_FREE];
+ vdw_free = &nlist[eNL_VDW_FREE];
+ coul_free = &nlist[eNL_QQ_FREE];
+ /* Loop over the atoms in the i charge group */
+ for (i = 0; i < nicg; i++)
+ {
+ i_atom = i0+i;
+ gid = GID(igid, jgid, ngid);
+ qi = charge[i_atom];
+ qiB = chargeB[i_atom];
+
+ /* Create new i_atom for each energy group */
+ if (bDoVdW && bDoCoul)
+ {
+ new_i_nblist(vdwc, i_atom, shift, gid);
+ }
+ if (bDoVdW)
+ {
+ new_i_nblist(vdw, i_atom, shift, gid);
+ }
+ if (bDoCoul)
+ {
+ new_i_nblist(coul, i_atom, shift, gid);
+ }
+
+ new_i_nblist(vdw_free, i_atom, shift, gid);
+ new_i_nblist(coul_free, i_atom, shift, gid);
+ new_i_nblist(vdwc_free, i_atom, shift, gid);
+
+ bDoVdW_i = (bDoVdW &&
+ (bHaveVdW[type[i_atom]] || bHaveVdW[typeB[i_atom]]));
+ bDoCoul_i = (bDoCoul && (qi != 0 || qiB != 0));
+ /* For TIP4P the first atom does not have a charge,
+ * but the last three do. So we should still put an atom
+ * without LJ but with charge in the water-atom neighborlist
+ * for a TIP4p i charge group.
+ * For SPC type water the first atom has LJ and charge,
+ * so there is no such problem.
+ */
+ if (iwater == esolNO)
+ {
+ bDoCoul_i_sol = bDoCoul_i;
+ }
+ else
+ {
+ bDoCoul_i_sol = bDoCoul;
+ }
+
+ if (bDoVdW_i || bDoCoul_i_sol)
+ {
+ /* Loop over the j charge groups */
+ for (j = 0; (j < nj); j++)
+ {
+ jcg = jjcg[j];
+
+ /* Check for large charge groups */
+ if (jcg == icg)
+ {
+ jj0 = i0 + i + 1;
+ }
+ else
+ {
+ jj0 = index[jcg];
+ }
+
+ jj1 = index[jcg+1];
+ /* Finally loop over the atoms in the j-charge group */
+ bFree = bPert[i_atom];
+ for (jj = jj0; (jj < jj1); jj++)
+ {
+ bFreeJ = bFree || bPert[jj];
+ /* Complicated if, because the water H's should also
+ * see perturbed j-particles
+ */
+ if (iwater == esolNO || i == 0 || bFreeJ)
+ {
+ bNotEx = NOTEXCL(bExcl, i, jj);
+
+ if (bNotEx)
+ {
+ if (bFreeJ)
+ {
+ if (!bDoVdW_i)
+ {
+ if (charge[jj] != 0 || chargeB[jj] != 0)
+ {
+ add_j_to_nblist(coul_free, jj, bLR);
+ }
+ }
+ else if (!bDoCoul_i)
+ {
+ if (bHaveVdW[type[jj]] || bHaveVdW[typeB[jj]])
+ {
+ add_j_to_nblist(vdw_free, jj, bLR);
+ }
+ }
+ else
+ {
+ if (bHaveVdW[type[jj]] || bHaveVdW[typeB[jj]])
+ {
+ if (charge[jj] != 0 || chargeB[jj] != 0)
+ {
+ add_j_to_nblist(vdwc_free, jj, bLR);
+ }
+ else
+ {
+ add_j_to_nblist(vdw_free, jj, bLR);
+ }
+ }
+ else if (charge[jj] != 0 || chargeB[jj] != 0)
+ {
+ add_j_to_nblist(coul_free, jj, bLR);
+ }
+ }
+ }
+ else if (!bDoVdW_i)
+ {
+ /* This is done whether or not bWater is set */
+ if (charge[jj] != 0)
+ {
+ add_j_to_nblist(coul, jj, bLR);
+ }
+ }
+ else if (!bDoCoul_i_sol)
+ {
+ if (bHaveVdW[type[jj]])
+ {
+ add_j_to_nblist(vdw, jj, bLR);
+ }
+ }
+ else
+ {
+ if (bHaveVdW[type[jj]])
+ {
+ if (charge[jj] != 0)
+ {
+ add_j_to_nblist(vdwc, jj, bLR);
+ }
+ else
+ {
+ add_j_to_nblist(vdw, jj, bLR);
+ }
+ }
+ else if (charge[jj] != 0)
+ {
+ add_j_to_nblist(coul, jj, bLR);
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ close_i_nblist(vdw);
+ close_i_nblist(coul);
+ close_i_nblist(vdwc);
+ close_i_nblist(vdw_free);
+ close_i_nblist(coul_free);
+ close_i_nblist(vdwc_free);
+ }
+ }
+}
+
+static void
+put_in_list_adress(gmx_bool bHaveVdW[],
+ int ngid,
+ t_mdatoms * md,
+ int icg,
+ int jgid,
+ int nj,
+ atom_id jjcg[],
+ atom_id index[],
+ t_excl bExcl[],
+ int shift,
+ t_forcerec * fr,
+ gmx_bool bLR,
+ gmx_bool bDoVdW,
+ gmx_bool bDoCoul,
+ int solvent_opt)
+{
+ /* The a[] index has been removed,
+ * to put it back in i_atom should be a[i0] and jj should be a[jj].
+ */
+ t_nblist * vdwc;
+ t_nblist * vdw;
+ t_nblist * coul;
+ t_nblist * vdwc_adress = NULL;
+ t_nblist * vdw_adress = NULL;
+ t_nblist * coul_adress = NULL;
+ t_nblist * vdwc_ww = NULL;
+ t_nblist * coul_ww = NULL;
+
+ int i, j, jcg, igid, gid, nbl_ind, nbl_ind_adress;
+ atom_id jj, jj0, jj1, i_atom;
+ int i0, nicg, len;
+
+ int *cginfo;
+ int *type, *typeB;
+ real *charge, *chargeB;
+ real *wf;
+ real qi, qiB, qq, rlj;
+ gmx_bool bFreeEnergy, bFree, bFreeJ, bNotEx, *bPert;
+ gmx_bool bDoVdW_i, bDoCoul_i, bDoCoul_i_sol;
+ gmx_bool b_hybrid;
+ gmx_bool j_all_atom;
+ int iwater, jwater;
+ t_nblist *nlist, *nlist_adress;
+ gmx_bool bEnergyGroupCG;
+
+ /* Copy some pointers */
+ cginfo = fr->cginfo;
+ charge = md->chargeA;
+ chargeB = md->chargeB;
+ type = md->typeA;
+ typeB = md->typeB;
+ bPert = md->bPerturbed;
+ wf = md->wf;
+
+ /* Get atom range */
+ i0 = index[icg];
+ nicg = index[icg+1]-i0;
+
+ /* Get the i charge group info */
+ igid = GET_CGINFO_GID(cginfo[icg]);
+
+ iwater = (solvent_opt != esolNO) ? GET_CGINFO_SOLOPT(cginfo[icg]) : esolNO;
+
+ if (md->nPerturbed)
+ {
+ gmx_fatal(FARGS, "AdResS does not support free energy pertubation\n");
+ }
+
+ /* Unpack pointers to neighbourlist structs */
+ if (fr->nnblists == 2)
+ {
+ nbl_ind = 0;
+ nbl_ind_adress = 1;
+ }
+ else
+ {
+ nbl_ind = fr->gid2nblists[GID(igid, jgid, ngid)];
+ nbl_ind_adress = nbl_ind+fr->nnblists/2;
+ }
+ if (bLR)
+ {
+ nlist = fr->nblists[nbl_ind].nlist_lr;
+ nlist_adress = fr->nblists[nbl_ind_adress].nlist_lr;
+ }
+ else
+ {
+ nlist = fr->nblists[nbl_ind].nlist_sr;
+ nlist_adress = fr->nblists[nbl_ind_adress].nlist_sr;
+ }
+
+
+ vdwc = &nlist[eNL_VDWQQ];
+ vdw = &nlist[eNL_VDW];
+ coul = &nlist[eNL_QQ];
+
+ vdwc_adress = &nlist_adress[eNL_VDWQQ];
+ vdw_adress = &nlist_adress[eNL_VDW];
+ coul_adress = &nlist_adress[eNL_QQ];
+
+ /* We do not support solvent optimization with AdResS for now.
+ For this we would need hybrid solvent-other kernels */
+
+ /* no solvent as i charge group */
+ /* Loop over the atoms in the i charge group */
+ for (i = 0; i < nicg; i++)
+ {
+ i_atom = i0+i;
+ gid = GID(igid, jgid, ngid);
+ qi = charge[i_atom];
+
+ /* Create new i_atom for each energy group */
+ if (bDoVdW && bDoCoul)
+ {
+ new_i_nblist(vdwc, i_atom, shift, gid);
+ new_i_nblist(vdwc_adress, i_atom, shift, gid);
+
+ }
+ if (bDoVdW)
+ {
+ new_i_nblist(vdw, i_atom, shift, gid);
+ new_i_nblist(vdw_adress, i_atom, shift, gid);
+
+ }
+ if (bDoCoul)
+ {
+ new_i_nblist(coul, i_atom, shift, gid);
+ new_i_nblist(coul_adress, i_atom, shift, gid);
+ }
+ bDoVdW_i = (bDoVdW && bHaveVdW[type[i_atom]]);
+ bDoCoul_i = (bDoCoul && qi != 0);
+
+ /* Here we find out whether the energy groups interaction belong to a
+ * coarse-grained (vsite) or atomistic interaction. Note that, beacuse
+ * interactions between coarse-grained and other (atomistic) energygroups
+ * are excluded automatically by grompp, it is sufficient to check for
+ * the group id of atom i (igid) */
+ bEnergyGroupCG = !egp_explicit(fr, igid);
+
+ if (bDoVdW_i || bDoCoul_i)
+ {
+ /* Loop over the j charge groups */
+ for (j = 0; (j < nj); j++)
+ {
+ jcg = jjcg[j];
+
+ /* Check for large charge groups */
+ if (jcg == icg)
+ {
+ jj0 = i0 + i + 1;
+ }
+ else
+ {
+ jj0 = index[jcg];
+ }
+
+ jj1 = index[jcg+1];
+ /* Finally loop over the atoms in the j-charge group */
+ for (jj = jj0; jj < jj1; jj++)
+ {
+ bNotEx = NOTEXCL(bExcl, i, jj);
+
+ /* Now we have to exclude interactions which will be zero
+ * anyway due to the AdResS weights (in previous implementations
+ * this was done in the force kernel). This is necessary as
+ * pure interactions (those with b_hybrid=false, i.e. w_i*w_j==1 or 0)
+ * are put into neighbour lists which will be passed to the
+ * standard (optimized) kernels for speed. The interactions with
+ * b_hybrid=true are placed into the _adress neighbour lists and
+ * processed by the generic AdResS kernel.
+ */
+ if ( (bEnergyGroupCG &&
+ wf[i_atom] >= 1-GMX_REAL_EPS && wf[jj] >= 1-GMX_REAL_EPS ) ||
+ ( !bEnergyGroupCG && wf[jj] <= GMX_REAL_EPS ) )
+ {
+ continue;
+ }
+
+ b_hybrid = !((wf[i_atom] >= 1-GMX_REAL_EPS && wf[jj] >= 1-GMX_REAL_EPS) ||
+ (wf[i_atom] <= GMX_REAL_EPS && wf[jj] <= GMX_REAL_EPS));
+
+ if (bNotEx)
+ {
+ if (!bDoVdW_i)
+ {
+ if (charge[jj] != 0)
+ {
+ if (!b_hybrid)
+ {
+ add_j_to_nblist(coul, jj, bLR);
+ }
+ else
+ {
+ add_j_to_nblist(coul_adress, jj, bLR);
+ }
+ }
+ }
+ else if (!bDoCoul_i)
+ {
+ if (bHaveVdW[type[jj]])
+ {
+ if (!b_hybrid)
+ {
+ add_j_to_nblist(vdw, jj, bLR);
+ }
+ else
+ {
+ add_j_to_nblist(vdw_adress, jj, bLR);
+ }
+ }
+ }
+ else
+ {
+ if (bHaveVdW[type[jj]])
+ {
+ if (charge[jj] != 0)
+ {
+ if (!b_hybrid)
+ {
+ add_j_to_nblist(vdwc, jj, bLR);
+ }
+ else
+ {
+ add_j_to_nblist(vdwc_adress, jj, bLR);
+ }
+ }
+ else
+ {
+ if (!b_hybrid)
+ {
+ add_j_to_nblist(vdw, jj, bLR);
+ }
+ else
+ {
+ add_j_to_nblist(vdw_adress, jj, bLR);
+ }
+
+ }
+ }
+ else if (charge[jj] != 0)
+ {
+ if (!b_hybrid)
+ {
+ add_j_to_nblist(coul, jj, bLR);
+ }
+ else
+ {
+ add_j_to_nblist(coul_adress, jj, bLR);
+ }
+
+ }
+ }
+ }
+ }
+ }
+
+ close_i_nblist(vdw);
+ close_i_nblist(coul);
+ close_i_nblist(vdwc);
+ close_i_nblist(vdw_adress);
+ close_i_nblist(coul_adress);
+ close_i_nblist(vdwc_adress);
+ }
+ }
+}
+
+static void
+put_in_list_qmmm(gmx_bool gmx_unused bHaveVdW[],
+ int ngid,
+ t_mdatoms gmx_unused * md,
+ int icg,
+ int jgid,
+ int nj,
+ atom_id jjcg[],
+ atom_id index[],
+ t_excl bExcl[],
+ int shift,
+ t_forcerec * fr,
+ gmx_bool bLR,
+ gmx_bool gmx_unused bDoVdW,
+ gmx_bool gmx_unused bDoCoul,
+ int gmx_unused solvent_opt)
+{
+ t_nblist * coul;
+ int i, j, jcg, igid, gid;
+ atom_id jj, jj0, jj1, i_atom;
+ int i0, nicg;
+ gmx_bool bNotEx;
+
+ /* Get atom range */
+ i0 = index[icg];
+ nicg = index[icg+1]-i0;
+
+ /* Get the i charge group info */
+ igid = GET_CGINFO_GID(fr->cginfo[icg]);
+
+ coul = &fr->QMMMlist;
+
+ /* Loop over atoms in the ith charge group */
+ for (i = 0; i < nicg; i++)
+ {
+ i_atom = i0+i;
+ gid = GID(igid, jgid, ngid);
+ /* Create new i_atom for each energy group */
+ new_i_nblist(coul, i_atom, shift, gid);
+
+ /* Loop over the j charge groups */
+ for (j = 0; j < nj; j++)
+ {
+ jcg = jjcg[j];
+
+ /* Charge groups cannot have QM and MM atoms simultaneously */
+ if (jcg != icg)
+ {
+ jj0 = index[jcg];
+ jj1 = index[jcg+1];
+ /* Finally loop over the atoms in the j-charge group */
+ for (jj = jj0; jj < jj1; jj++)
+ {
+ bNotEx = NOTEXCL(bExcl, i, jj);
+ if (bNotEx)
+ {
+ add_j_to_nblist(coul, jj, bLR);
+ }
+ }
+ }
+ }
+ close_i_nblist(coul);
+ }
+}
+
+static void
+put_in_list_cg(gmx_bool gmx_unused bHaveVdW[],
+ int ngid,
+ t_mdatoms gmx_unused * md,
+ int icg,
+ int jgid,
+ int nj,
+ atom_id jjcg[],
+ atom_id index[],
+ t_excl bExcl[],
+ int shift,
+ t_forcerec * fr,
+ gmx_bool bLR,
+ gmx_bool gmx_unused bDoVdW,
+ gmx_bool gmx_unused bDoCoul,
+ int gmx_unused solvent_opt)
+{
+ int cginfo;
+ int igid, gid, nbl_ind;
+ t_nblist * vdwc;
+ int j, jcg;
+
+ cginfo = fr->cginfo[icg];
+
+ igid = GET_CGINFO_GID(cginfo);
+ gid = GID(igid, jgid, ngid);
+
+ /* Unpack pointers to neighbourlist structs */
+ if (fr->nnblists == 1)
+ {
+ nbl_ind = 0;
+ }
+ else
+ {
+ nbl_ind = fr->gid2nblists[gid];
+ }
+ if (bLR)
+ {
+ vdwc = &fr->nblists[nbl_ind].nlist_lr[eNL_VDWQQ];
+ }
+ else
+ {
+ vdwc = &fr->nblists[nbl_ind].nlist_sr[eNL_VDWQQ];
+ }
+
+ /* Make a new neighbor list for charge group icg.
+ * Currently simply one neighbor list is made with LJ and Coulomb.
+ * If required, zero interactions could be removed here
+ * or in the force loop.
+ */
+ new_i_nblist(vdwc, index[icg], shift, gid);
+ vdwc->iinr_end[vdwc->nri] = index[icg+1];
+
+ for (j = 0; (j < nj); j++)
+ {
+ jcg = jjcg[j];
+ /* Skip the icg-icg pairs if all self interactions are excluded */
+ if (!(jcg == icg && GET_CGINFO_EXCL_INTRA(cginfo)))
+ {
+ /* Here we add the j charge group jcg to the list,
+ * exclusions are also added to the list.
+ */
+ add_j_to_nblist_cg(vdwc, index[jcg], index[jcg+1], bExcl, icg == jcg, bLR);
+ }
+ }
+
+ close_i_nblist(vdwc);
+}
+
+static void setexcl(atom_id start, atom_id end, t_blocka *excl, gmx_bool b,
+ t_excl bexcl[])
+{
+ atom_id i, k;
+
+ if (b)
+ {
+ for (i = start; i < end; i++)
+ {
+ for (k = excl->index[i]; k < excl->index[i+1]; k++)
+ {
+ SETEXCL(bexcl, i-start, excl->a[k]);
+ }
+ }
+ }
+ else
+ {
+ for (i = start; i < end; i++)
+ {
+ for (k = excl->index[i]; k < excl->index[i+1]; k++)
+ {
+ RMEXCL(bexcl, i-start, excl->a[k]);
+ }
+ }
+ }
+}
+
+int calc_naaj(int icg, int cgtot)
+{
+ int naaj;
+
+ if ((cgtot % 2) == 1)
+ {
+ /* Odd number of charge groups, easy */
+ naaj = 1 + (cgtot/2);
+ }
+ else if ((cgtot % 4) == 0)
+ {
+ /* Multiple of four is hard */
+ if (icg < cgtot/2)
+ {
+ if ((icg % 2) == 0)
+ {
+ naaj = 1+(cgtot/2);
+ }
+ else
+ {
+ naaj = cgtot/2;
+ }
+ }
+ else
+ {
+ if ((icg % 2) == 1)
+ {
+ naaj = 1+(cgtot/2);
+ }
+ else
+ {
+ naaj = cgtot/2;
+ }
+ }
+ }
+ else
+ {
+ /* cgtot/2 = odd */
+ if ((icg % 2) == 0)
+ {
+ naaj = 1+(cgtot/2);
+ }
+ else
+ {
+ naaj = cgtot/2;
+ }
+ }
+#ifdef DEBUG
+ fprintf(log, "naaj=%d\n", naaj);
+#endif
+
+ return naaj;
+}
+
+/************************************************
+ *
+ * S I M P L E C O R E S T U F F
+ *
+ ************************************************/
+
+static real calc_image_tric(rvec xi, rvec xj, matrix box,
+ rvec b_inv, int *shift)
+{
+ /* This code assumes that the cut-off is smaller than
+ * a half times the smallest diagonal element of the box.
+ */
+ const real h25 = 2.5;
+ real dx, dy, dz;
+ real r2;
+ int tx, ty, tz;
+
+ /* Compute diff vector */
+ dz = xj[ZZ] - xi[ZZ];
+ dy = xj[YY] - xi[YY];
+ dx = xj[XX] - xi[XX];
+
+ /* Perform NINT operation, using trunc operation, therefore
+ * we first add 2.5 then subtract 2 again
+ */
+ tz = dz*b_inv[ZZ] + h25;
+ tz -= 2;
+ dz -= tz*box[ZZ][ZZ];
+ dy -= tz*box[ZZ][YY];
+ dx -= tz*box[ZZ][XX];
+
+ ty = dy*b_inv[YY] + h25;
+ ty -= 2;
+ dy -= ty*box[YY][YY];
+ dx -= ty*box[YY][XX];
+
+ tx = dx*b_inv[XX]+h25;
+ tx -= 2;
+ dx -= tx*box[XX][XX];
+
+ /* Distance squared */
+ r2 = (dx*dx) + (dy*dy) + (dz*dz);
+
+ *shift = XYZ2IS(tx, ty, tz);
+
+ return r2;
+}
+
+static real calc_image_rect(rvec xi, rvec xj, rvec box_size,
+ rvec b_inv, int *shift)
+{
+ const real h15 = 1.5;
+ real ddx, ddy, ddz;
+ real dx, dy, dz;
+ real r2;
+ int tx, ty, tz;
+
+ /* Compute diff vector */
+ dx = xj[XX] - xi[XX];
+ dy = xj[YY] - xi[YY];
+ dz = xj[ZZ] - xi[ZZ];
+
+ /* Perform NINT operation, using trunc operation, therefore
+ * we first add 1.5 then subtract 1 again
+ */
+ tx = dx*b_inv[XX] + h15;
+ ty = dy*b_inv[YY] + h15;
+ tz = dz*b_inv[ZZ] + h15;
+ tx--;
+ ty--;
+ tz--;
+
+ /* Correct diff vector for translation */
+ ddx = tx*box_size[XX] - dx;
+ ddy = ty*box_size[YY] - dy;
+ ddz = tz*box_size[ZZ] - dz;
+
+ /* Distance squared */
+ r2 = (ddx*ddx) + (ddy*ddy) + (ddz*ddz);
+
+ *shift = XYZ2IS(tx, ty, tz);
+
+ return r2;
+}
+
+static void add_simple(t_ns_buf *nsbuf, int nrj, atom_id cg_j,
+ gmx_bool bHaveVdW[], int ngid, t_mdatoms *md,
+ int icg, int jgid, t_block *cgs, t_excl bexcl[],
+ int shift, t_forcerec *fr, put_in_list_t *put_in_list)
+{
+ if (nsbuf->nj + nrj > MAX_CG)
+ {
+ put_in_list(bHaveVdW, ngid, md, icg, jgid, nsbuf->ncg, nsbuf->jcg,
+ cgs->index, bexcl, shift, fr, FALSE, TRUE, TRUE, fr->solvent_opt);
+ /* Reset buffer contents */
+ nsbuf->ncg = nsbuf->nj = 0;
+ }
+ nsbuf->jcg[nsbuf->ncg++] = cg_j;
+ nsbuf->nj += nrj;
+}
+
+static void ns_inner_tric(rvec x[], int icg, int *i_egp_flags,
+ int njcg, atom_id jcg[],
+ matrix box, rvec b_inv, real rcut2,
+ t_block *cgs, t_ns_buf **ns_buf,
+ gmx_bool bHaveVdW[], int ngid, t_mdatoms *md,
+ t_excl bexcl[], t_forcerec *fr,
+ put_in_list_t *put_in_list)
+{
+ int shift;
+ int j, nrj, jgid;
+ int *cginfo = fr->cginfo;
+ atom_id cg_j, *cgindex;
+ t_ns_buf *nsbuf;
+
+ cgindex = cgs->index;
+ shift = CENTRAL;
+ for (j = 0; (j < njcg); j++)
+ {
+ cg_j = jcg[j];
+ nrj = cgindex[cg_j+1]-cgindex[cg_j];
+ if (calc_image_tric(x[icg], x[cg_j], box, b_inv, &shift) < rcut2)
+ {
+ jgid = GET_CGINFO_GID(cginfo[cg_j]);
+ if (!(i_egp_flags[jgid] & EGP_EXCL))
+ {
+ add_simple(&ns_buf[jgid][shift], nrj, cg_j,
+ bHaveVdW, ngid, md, icg, jgid, cgs, bexcl, shift, fr,
+ put_in_list);
+ }
+ }
+ }
+}
+
+static void ns_inner_rect(rvec x[], int icg, int *i_egp_flags,
+ int njcg, atom_id jcg[],
+ gmx_bool bBox, rvec box_size, rvec b_inv, real rcut2,
+ t_block *cgs, t_ns_buf **ns_buf,
+ gmx_bool bHaveVdW[], int ngid, t_mdatoms *md,
+ t_excl bexcl[], t_forcerec *fr,
+ put_in_list_t *put_in_list)
+{
+ int shift;
+ int j, nrj, jgid;
+ int *cginfo = fr->cginfo;
+ atom_id cg_j, *cgindex;
+ t_ns_buf *nsbuf;
+
+ cgindex = cgs->index;
+ if (bBox)
+ {
+ shift = CENTRAL;
+ for (j = 0; (j < njcg); j++)
+ {
+ cg_j = jcg[j];
+ nrj = cgindex[cg_j+1]-cgindex[cg_j];
+ if (calc_image_rect(x[icg], x[cg_j], box_size, b_inv, &shift) < rcut2)
+ {
+ jgid = GET_CGINFO_GID(cginfo[cg_j]);
+ if (!(i_egp_flags[jgid] & EGP_EXCL))
+ {
+ add_simple(&ns_buf[jgid][shift], nrj, cg_j,
+ bHaveVdW, ngid, md, icg, jgid, cgs, bexcl, shift, fr,
+ put_in_list);
+ }
+ }
+ }
+ }
+ else
+ {
+ for (j = 0; (j < njcg); j++)
+ {
+ cg_j = jcg[j];
+ nrj = cgindex[cg_j+1]-cgindex[cg_j];
+ if ((rcut2 == 0) || (distance2(x[icg], x[cg_j]) < rcut2))
+ {
+ jgid = GET_CGINFO_GID(cginfo[cg_j]);
+ if (!(i_egp_flags[jgid] & EGP_EXCL))
+ {
+ add_simple(&ns_buf[jgid][CENTRAL], nrj, cg_j,
+ bHaveVdW, ngid, md, icg, jgid, cgs, bexcl, CENTRAL, fr,
+ put_in_list);
+ }
+ }
+ }
+ }
+}
+
+/* ns_simple_core needs to be adapted for QMMM still 2005 */
+
+static int ns_simple_core(t_forcerec *fr,
+ gmx_localtop_t *top,
+ t_mdatoms *md,
+ matrix box, rvec box_size,
+ t_excl bexcl[], atom_id *aaj,
+ int ngid, t_ns_buf **ns_buf,
+ put_in_list_t *put_in_list, gmx_bool bHaveVdW[])
+{
+ int naaj, k;
+ real rlist2;
+ int nsearch, icg, jcg, igid, i0, nri, nn;
+ int *cginfo;
+ t_ns_buf *nsbuf;
+ /* atom_id *i_atoms; */
+ t_block *cgs = &(top->cgs);
+ t_blocka *excl = &(top->excls);
+ rvec b_inv;
+ int m;
+ gmx_bool bBox, bTriclinic;
+ int *i_egp_flags;
+
+ rlist2 = sqr(fr->rlist);
+
+ bBox = (fr->ePBC != epbcNONE);
+ if (bBox)
+ {
+ for (m = 0; (m < DIM); m++)
+ {
+ b_inv[m] = divide_err(1.0, box_size[m]);
+ }
+ bTriclinic = TRICLINIC(box);
+ }
+ else
+ {
+ bTriclinic = FALSE;
+ }
+
+ cginfo = fr->cginfo;
+
+ nsearch = 0;
+ for (icg = fr->cg0; (icg < fr->hcg); icg++)
+ {
+ /*
+ i0 = cgs->index[icg];
+ nri = cgs->index[icg+1]-i0;
+ i_atoms = &(cgs->a[i0]);
+ i_eg_excl = fr->eg_excl + ngid*md->cENER[*i_atoms];
+ setexcl(nri,i_atoms,excl,TRUE,bexcl);
+ */
+ igid = GET_CGINFO_GID(cginfo[icg]);
+ i_egp_flags = fr->egp_flags + ngid*igid;
+ setexcl(cgs->index[icg], cgs->index[icg+1], excl, TRUE, bexcl);
+
+ naaj = calc_naaj(icg, cgs->nr);
+ if (bTriclinic)
+ {
+ ns_inner_tric(fr->cg_cm, icg, i_egp_flags, naaj, &(aaj[icg]),
+ box, b_inv, rlist2, cgs, ns_buf,
+ bHaveVdW, ngid, md, bexcl, fr, put_in_list);
+ }
+ else
+ {
+ ns_inner_rect(fr->cg_cm, icg, i_egp_flags, naaj, &(aaj[icg]),
+ bBox, box_size, b_inv, rlist2, cgs, ns_buf,
+ bHaveVdW, ngid, md, bexcl, fr, put_in_list);
+ }
+ nsearch += naaj;
+
+ for (nn = 0; (nn < ngid); nn++)
+ {
+ for (k = 0; (k < SHIFTS); k++)
+ {
+ nsbuf = &(ns_buf[nn][k]);
+ if (nsbuf->ncg > 0)
+ {
+ put_in_list(bHaveVdW, ngid, md, icg, nn, nsbuf->ncg, nsbuf->jcg,
+ cgs->index, bexcl, k, fr, FALSE, TRUE, TRUE, fr->solvent_opt);
+ nsbuf->ncg = nsbuf->nj = 0;
+ }
+ }
+ }
+ /* setexcl(nri,i_atoms,excl,FALSE,bexcl); */
+ setexcl(cgs->index[icg], cgs->index[icg+1], excl, FALSE, bexcl);
+ }
+ close_neighbor_lists(fr, FALSE);
+
+ return nsearch;
+}
+
+/************************************************
+ *
+ * N S 5 G R I D S T U F F
+ *
+ ************************************************/
+
+static inline void get_dx(int Nx, real gridx, real rc2, int xgi, real x,
+ int *dx0, int *dx1, real *dcx2)
+{
+ real dcx, tmp;
+ int xgi0, xgi1, i;
+
+ if (xgi < 0)
+ {
+ *dx0 = 0;
+ xgi0 = -1;
+ *dx1 = -1;
+ xgi1 = 0;
+ }
+ else if (xgi >= Nx)
+ {
+ *dx0 = Nx;
+ xgi0 = Nx-1;
+ *dx1 = Nx-1;
+ xgi1 = Nx;
+ }
+ else
+ {
+ dcx2[xgi] = 0;
+ *dx0 = xgi;
+ xgi0 = xgi-1;
+ *dx1 = xgi;
+ xgi1 = xgi+1;
+ }
+
+ for (i = xgi0; i >= 0; i--)
+ {
+ dcx = (i+1)*gridx-x;
+ tmp = dcx*dcx;
+ if (tmp >= rc2)
+ {
+ break;
+ }
+ *dx0 = i;
+ dcx2[i] = tmp;
+ }
+ for (i = xgi1; i < Nx; i++)
+ {
+ dcx = i*gridx-x;
+ tmp = dcx*dcx;
+ if (tmp >= rc2)
+ {
+ break;
+ }
+ *dx1 = i;
+ dcx2[i] = tmp;
+ }
+}
+
+static inline void get_dx_dd(int Nx, real gridx, real rc2, int xgi, real x,
+ int ncpddc, int shift_min, int shift_max,
+ int *g0, int *g1, real *dcx2)
+{
+ real dcx, tmp;
+ int g_min, g_max, shift_home;
+
+ if (xgi < 0)
+ {
+ g_min = 0;
+ g_max = Nx - 1;
+ *g0 = 0;
+ *g1 = -1;
+ }
+ else if (xgi >= Nx)
+ {
+ g_min = 0;
+ g_max = Nx - 1;
+ *g0 = Nx;
+ *g1 = Nx - 1;
+ }
+ else
+ {
+ if (ncpddc == 0)
+ {
+ g_min = 0;
+ g_max = Nx - 1;
+ }
+ else
+ {
+ if (xgi < ncpddc)
+ {
+ shift_home = 0;
+ }
+ else
+ {
+ shift_home = -1;
+ }
+ g_min = (shift_min == shift_home ? 0 : ncpddc);
+ g_max = (shift_max == shift_home ? ncpddc - 1 : Nx - 1);
+ }
+ if (shift_min > 0)
+ {
+ *g0 = g_min;
+ *g1 = g_min - 1;
+ }
+ else if (shift_max < 0)
+ {
+ *g0 = g_max + 1;
+ *g1 = g_max;
+ }
+ else
+ {
+ *g0 = xgi;
+ *g1 = xgi;
+ dcx2[xgi] = 0;
+ }
+ }
+
+ while (*g0 > g_min)
+ {
+ /* Check one grid cell down */
+ dcx = ((*g0 - 1) + 1)*gridx - x;
+ tmp = dcx*dcx;
+ if (tmp >= rc2)
+ {
+ break;
+ }
+ (*g0)--;
+ dcx2[*g0] = tmp;
+ }
+
+ while (*g1 < g_max)
+ {
+ /* Check one grid cell up */
+ dcx = (*g1 + 1)*gridx - x;
+ tmp = dcx*dcx;
+ if (tmp >= rc2)
+ {
+ break;
+ }
+ (*g1)++;
+ dcx2[*g1] = tmp;
+ }
+}
+
+
+#define sqr(x) ((x)*(x))
+#define calc_dx2(XI, YI, ZI, y) (sqr(XI-y[XX]) + sqr(YI-y[YY]) + sqr(ZI-y[ZZ]))
+#define calc_cyl_dx2(XI, YI, y) (sqr(XI-y[XX]) + sqr(YI-y[YY]))
+/****************************************************
+ *
+ * F A S T N E I G H B O R S E A R C H I N G
+ *
+ * Optimized neighboursearching routine using grid
+ * at least 1x1x1, see GROMACS manual
+ *
+ ****************************************************/
+
+
+static void get_cutoff2(t_forcerec *fr, gmx_bool bDoLongRange,
+ real *rvdw2, real *rcoul2,
+ real *rs2, real *rm2, real *rl2)
+{
+ *rs2 = sqr(fr->rlist);
+
+ if (bDoLongRange && fr->bTwinRange)
+ {
+ /* The VdW and elec. LR cut-off's could be different,
+ * so we can not simply set them to rlistlong.
+ */
+ if (EVDW_MIGHT_BE_ZERO_AT_CUTOFF(fr->vdwtype) &&
+ fr->rvdw > fr->rlist)
+ {
+ *rvdw2 = sqr(fr->rlistlong);
+ }
+ else
+ {
+ *rvdw2 = sqr(fr->rvdw);
+ }
+ if (EEL_MIGHT_BE_ZERO_AT_CUTOFF(fr->eeltype) &&
+ fr->rcoulomb > fr->rlist)
+ {
+ *rcoul2 = sqr(fr->rlistlong);
+ }
+ else
+ {
+ *rcoul2 = sqr(fr->rcoulomb);
+ }
+ }
+ else
+ {
+ /* Workaround for a gcc -O3 or -ffast-math problem */
+ *rvdw2 = *rs2;
+ *rcoul2 = *rs2;
+ }
+ *rm2 = min(*rvdw2, *rcoul2);
+ *rl2 = max(*rvdw2, *rcoul2);
+}
+
+static void init_nsgrid_lists(t_forcerec *fr, int ngid, gmx_ns_t *ns)
+{
+ real rvdw2, rcoul2, rs2, rm2, rl2;
+ int j;
+
+ get_cutoff2(fr, TRUE, &rvdw2, &rcoul2, &rs2, &rm2, &rl2);
+
+ /* Short range buffers */
+ snew(ns->nl_sr, ngid);
+ /* Counters */
+ snew(ns->nsr, ngid);
+ snew(ns->nlr_ljc, ngid);
+ snew(ns->nlr_one, ngid);
+
+ /* Always allocate both list types, since rcoulomb might now change with PME load balancing */
+ /* Long range VdW and Coul buffers */
+ snew(ns->nl_lr_ljc, ngid);
+ /* Long range VdW or Coul only buffers */
+ snew(ns->nl_lr_one, ngid);
+
+ for (j = 0; (j < ngid); j++)
+ {
+ snew(ns->nl_sr[j], MAX_CG);
+ snew(ns->nl_lr_ljc[j], MAX_CG);
+ snew(ns->nl_lr_one[j], MAX_CG);
+ }
+ if (debug)
+ {
+ fprintf(debug,
+ "ns5_core: rs2 = %g, rm2 = %g, rl2 = %g (nm^2)\n",
+ rs2, rm2, rl2);
+ }
+}
+
+static int nsgrid_core(t_commrec *cr, t_forcerec *fr,
+ matrix box, int ngid,
+ gmx_localtop_t *top,
+ t_grid *grid,
+ t_excl bexcl[], gmx_bool *bExcludeAlleg,
+ t_mdatoms *md,
+ put_in_list_t *put_in_list,
+ gmx_bool bHaveVdW[],
+ gmx_bool bDoLongRange, gmx_bool bMakeQMMMnblist)
+{
+ gmx_ns_t *ns;
+ atom_id **nl_lr_ljc, **nl_lr_one, **nl_sr;
+ int *nlr_ljc, *nlr_one, *nsr;
+ gmx_domdec_t *dd = NULL;
+ t_block *cgs = &(top->cgs);
+ int *cginfo = fr->cginfo;
+ /* atom_id *i_atoms,*cgsindex=cgs->index; */
+ ivec sh0, sh1, shp;
+ int cell_x, cell_y, cell_z;
+ int d, tx, ty, tz, dx, dy, dz, cj;
+#ifdef ALLOW_OFFDIAG_LT_HALFDIAG
+ int zsh_ty, zsh_tx, ysh_tx;
+#endif
+ int dx0, dx1, dy0, dy1, dz0, dz1;
+ int Nx, Ny, Nz, shift = -1, j, nrj, nns, nn = -1;
+ real gridx, gridy, gridz, grid_x, grid_y, grid_z;
+ real *dcx2, *dcy2, *dcz2;
+ int zgi, ygi, xgi;
+ int cg0, cg1, icg = -1, cgsnr, i0, igid, nri, naaj, max_jcg;
+ int jcg0, jcg1, jjcg, cgj0, jgid;
+ int *grida, *gridnra, *gridind;
+ gmx_bool rvdw_lt_rcoul, rcoul_lt_rvdw;
+ rvec xi, *cgcm, grid_offset;
+ real r2, rs2, rvdw2, rcoul2, rm2, rl2, XI, YI, ZI, dcx, dcy, dcz, tmp1, tmp2;
+ int *i_egp_flags;
+ gmx_bool bDomDec, bTriclinicX, bTriclinicY;
+ ivec ncpddc;
+
+ ns = &fr->ns;
+
+ bDomDec = DOMAINDECOMP(cr);
+ if (bDomDec)
+ {
+ dd = cr->dd;
+ }
+
+ bTriclinicX = ((YY < grid->npbcdim &&
+ (!bDomDec || dd->nc[YY] == 1) && box[YY][XX] != 0) ||
+ (ZZ < grid->npbcdim &&
+ (!bDomDec || dd->nc[ZZ] == 1) && box[ZZ][XX] != 0));
+ bTriclinicY = (ZZ < grid->npbcdim &&
+ (!bDomDec || dd->nc[ZZ] == 1) && box[ZZ][YY] != 0);
+
+ cgsnr = cgs->nr;
+
+ get_cutoff2(fr, bDoLongRange, &rvdw2, &rcoul2, &rs2, &rm2, &rl2);
+
+ rvdw_lt_rcoul = (rvdw2 >= rcoul2);
+ rcoul_lt_rvdw = (rcoul2 >= rvdw2);
+
+ if (bMakeQMMMnblist)
+ {
+ rm2 = rl2;
+ rs2 = rl2;
+ }
+
+ nl_sr = ns->nl_sr;
+ nsr = ns->nsr;
+ nl_lr_ljc = ns->nl_lr_ljc;
+ nl_lr_one = ns->nl_lr_one;
+ nlr_ljc = ns->nlr_ljc;
+ nlr_one = ns->nlr_one;
+
+ /* Unpack arrays */
+ cgcm = fr->cg_cm;
+ Nx = grid->n[XX];
+ Ny = grid->n[YY];
+ Nz = grid->n[ZZ];
+ grida = grid->a;
+ gridind = grid->index;
+ gridnra = grid->nra;
+ nns = 0;
+
+ gridx = grid->cell_size[XX];
+ gridy = grid->cell_size[YY];
+ gridz = grid->cell_size[ZZ];
+ grid_x = 1/gridx;
+ grid_y = 1/gridy;
+ grid_z = 1/gridz;
+ copy_rvec(grid->cell_offset, grid_offset);
+ copy_ivec(grid->ncpddc, ncpddc);
+ dcx2 = grid->dcx2;
+ dcy2 = grid->dcy2;
+ dcz2 = grid->dcz2;
+
+#ifdef ALLOW_OFFDIAG_LT_HALFDIAG
+ zsh_ty = floor(-box[ZZ][YY]/box[YY][YY]+0.5);
+ zsh_tx = floor(-box[ZZ][XX]/box[XX][XX]+0.5);
+ ysh_tx = floor(-box[YY][XX]/box[XX][XX]+0.5);
+ if (zsh_tx != 0 && ysh_tx != 0)
+ {
+ /* This could happen due to rounding, when both ratios are 0.5 */
+ ysh_tx = 0;
+ }
+#endif
+
+ debug_gmx();
+
+ if (fr->n_tpi)
+ {
+ /* We only want a list for the test particle */
+ cg0 = cgsnr - 1;
+ }
+ else
+ {
+ cg0 = grid->icg0;
+ }
+ cg1 = grid->icg1;
+
+ /* Set the shift range */
+ for (d = 0; d < DIM; d++)
+ {
+ sh0[d] = -1;
+ sh1[d] = 1;
+ /* Check if we need periodicity shifts.
+ * Without PBC or with domain decomposition we don't need them.
+ */
+ if (d >= ePBC2npbcdim(fr->ePBC) || (bDomDec && dd->nc[d] > 1))
+ {
+ shp[d] = 0;
+ }
+ else
+ {
+ if (d == XX &&
+ box[XX][XX] - fabs(box[YY][XX]) - fabs(box[ZZ][XX]) < sqrt(rl2))
+ {
+ shp[d] = 2;
+ }
+ else
+ {
+ shp[d] = 1;
+ }
+ }
+ }
+
+ /* Loop over charge groups */
+ for (icg = cg0; (icg < cg1); icg++)
+ {
+ igid = GET_CGINFO_GID(cginfo[icg]);
+ /* Skip this charge group if all energy groups are excluded! */
+ if (bExcludeAlleg[igid])
+ {
+ continue;
+ }
+
+ i0 = cgs->index[icg];
+
+ if (bMakeQMMMnblist)
+ {
+ /* Skip this charge group if it is not a QM atom while making a
+ * QM/MM neighbourlist
+ */
+ if (md->bQM[i0] == FALSE)
+ {
+ continue; /* MM particle, go to next particle */
+ }
+
+ /* Compute the number of charge groups that fall within the control
+ * of this one (icg)
+ */
+ naaj = calc_naaj(icg, cgsnr);
+ jcg0 = icg;
+ jcg1 = icg + naaj;
+ max_jcg = cgsnr;
+ }
+ else
+ {
+ /* make a normal neighbourlist */
+
+ if (bDomDec)
+ {
+ /* Get the j charge-group and dd cell shift ranges */
+ dd_get_ns_ranges(cr->dd, icg, &jcg0, &jcg1, sh0, sh1);
+ max_jcg = 0;
+ }
+ else
+ {
+ /* Compute the number of charge groups that fall within the control
+ * of this one (icg)
+ */
+ naaj = calc_naaj(icg, cgsnr);
+ jcg0 = icg;
+ jcg1 = icg + naaj;
+
+ if (fr->n_tpi)
+ {
+ /* The i-particle is awlways the test particle,
+ * so we want all j-particles
+ */
+ max_jcg = cgsnr - 1;
+ }
+ else
+ {
+ max_jcg = jcg1 - cgsnr;
+ }
+ }
+ }
+
+ i_egp_flags = fr->egp_flags + igid*ngid;
+
+ /* Set the exclusions for the atoms in charge group icg using a bitmask */
+ setexcl(i0, cgs->index[icg+1], &top->excls, TRUE, bexcl);
+
+ ci2xyz(grid, icg, &cell_x, &cell_y, &cell_z);
+
+ /* Changed iicg to icg, DvdS 990115
+ * (but see consistency check above, DvdS 990330)
+ */
+#ifdef NS5DB
+ fprintf(log, "icg=%5d, naaj=%5d, cell %d %d %d\n",
+ icg, naaj, cell_x, cell_y, cell_z);
+#endif
+ /* Loop over shift vectors in three dimensions */
+ for (tz = -shp[ZZ]; tz <= shp[ZZ]; tz++)
+ {
+ ZI = cgcm[icg][ZZ]+tz*box[ZZ][ZZ];
+ /* Calculate range of cells in Z direction that have the shift tz */
+ zgi = cell_z + tz*Nz;
+#define FAST_DD_NS
+#ifndef FAST_DD_NS
+ get_dx(Nz, gridz, rl2, zgi, ZI, &dz0, &dz1, dcz2);
+#else
+ get_dx_dd(Nz, gridz, rl2, zgi, ZI-grid_offset[ZZ],
+ ncpddc[ZZ], sh0[ZZ], sh1[ZZ], &dz0, &dz1, dcz2);
+#endif
+ if (dz0 > dz1)
+ {
+ continue;
+ }
+ for (ty = -shp[YY]; ty <= shp[YY]; ty++)
+ {
+ YI = cgcm[icg][YY]+ty*box[YY][YY]+tz*box[ZZ][YY];
+ /* Calculate range of cells in Y direction that have the shift ty */
+ if (bTriclinicY)
+ {
+ ygi = (int)(Ny + (YI - grid_offset[YY])*grid_y) - Ny;
+ }
+ else
+ {
+ ygi = cell_y + ty*Ny;
+ }
+#ifndef FAST_DD_NS
+ get_dx(Ny, gridy, rl2, ygi, YI, &dy0, &dy1, dcy2);
+#else
+ get_dx_dd(Ny, gridy, rl2, ygi, YI-grid_offset[YY],
+ ncpddc[YY], sh0[YY], sh1[YY], &dy0, &dy1, dcy2);
+#endif
+ if (dy0 > dy1)
+ {
+ continue;
+ }
+ for (tx = -shp[XX]; tx <= shp[XX]; tx++)
+ {
+ XI = cgcm[icg][XX]+tx*box[XX][XX]+ty*box[YY][XX]+tz*box[ZZ][XX];
+ /* Calculate range of cells in X direction that have the shift tx */
+ if (bTriclinicX)
+ {
+ xgi = (int)(Nx + (XI - grid_offset[XX])*grid_x) - Nx;
+ }
+ else
+ {
+ xgi = cell_x + tx*Nx;
+ }
+#ifndef FAST_DD_NS
+ get_dx(Nx, gridx, rl2, xgi*Nx, XI, &dx0, &dx1, dcx2);
+#else
+ get_dx_dd(Nx, gridx, rl2, xgi, XI-grid_offset[XX],
+ ncpddc[XX], sh0[XX], sh1[XX], &dx0, &dx1, dcx2);
+#endif
+ if (dx0 > dx1)
+ {
+ continue;
+ }
+ /* Adress: an explicit cg that has a weigthing function of 0 is excluded
+ * from the neigbour list as it will not interact */
+ if (fr->adress_type != eAdressOff)
+ {
+ if (md->wf[cgs->index[icg]] <= GMX_REAL_EPS && egp_explicit(fr, igid))
+ {
+ continue;
+ }
+ }
+ /* Get shift vector */
+ shift = XYZ2IS(tx, ty, tz);
+#ifdef NS5DB
+ range_check(shift, 0, SHIFTS);
+#endif
+ for (nn = 0; (nn < ngid); nn++)
+ {
+ nsr[nn] = 0;
+ nlr_ljc[nn] = 0;
+ nlr_one[nn] = 0;
+ }
+#ifdef NS5DB
+ fprintf(log, "shift: %2d, dx0,1: %2d,%2d, dy0,1: %2d,%2d, dz0,1: %2d,%2d\n",
+ shift, dx0, dx1, dy0, dy1, dz0, dz1);
+ fprintf(log, "cgcm: %8.3f %8.3f %8.3f\n", cgcm[icg][XX],
+ cgcm[icg][YY], cgcm[icg][ZZ]);
+ fprintf(log, "xi: %8.3f %8.3f %8.3f\n", XI, YI, ZI);
+#endif
+ for (dx = dx0; (dx <= dx1); dx++)
+ {
+ tmp1 = rl2 - dcx2[dx];
+ for (dy = dy0; (dy <= dy1); dy++)
+ {
+ tmp2 = tmp1 - dcy2[dy];
+ if (tmp2 > 0)
+ {
+ for (dz = dz0; (dz <= dz1); dz++)
+ {
+ if (tmp2 > dcz2[dz])
+ {
+ /* Find grid-cell cj in which possible neighbours are */
+ cj = xyz2ci(Ny, Nz, dx, dy, dz);
+
+ /* Check out how many cgs (nrj) there in this cell */
+ nrj = gridnra[cj];
+
+ /* Find the offset in the cg list */
+ cgj0 = gridind[cj];
+
+ /* Check if all j's are out of range so we
+ * can skip the whole cell.
+ * Should save some time, especially with DD.
+ */
+ if (nrj == 0 ||
+ (grida[cgj0] >= max_jcg &&
+ (grida[cgj0] >= jcg1 || grida[cgj0+nrj-1] < jcg0)))
+ {
+ continue;
+ }
+
+ /* Loop over cgs */
+ for (j = 0; (j < nrj); j++)
+ {
+ jjcg = grida[cgj0+j];
+
+ /* check whether this guy is in range! */
+ if ((jjcg >= jcg0 && jjcg < jcg1) ||
+ (jjcg < max_jcg))
+ {
+ r2 = calc_dx2(XI, YI, ZI, cgcm[jjcg]);
+ if (r2 < rl2)
+ {
+ /* jgid = gid[cgsatoms[cgsindex[jjcg]]]; */
+ jgid = GET_CGINFO_GID(cginfo[jjcg]);
+ /* check energy group exclusions */
+ if (!(i_egp_flags[jgid] & EGP_EXCL))
+ {
+ if (r2 < rs2)
+ {
+ if (nsr[jgid] >= MAX_CG)
+ {
+ /* Add to short-range list */
+ put_in_list(bHaveVdW, ngid, md, icg, jgid,
+ nsr[jgid], nl_sr[jgid],
+ cgs->index, /* cgsatoms, */ bexcl,
+ shift, fr, FALSE, TRUE, TRUE, fr->solvent_opt);
+ nsr[jgid] = 0;
+ }
+ nl_sr[jgid][nsr[jgid]++] = jjcg;
+ }
+ else if (r2 < rm2)
+ {
+ if (nlr_ljc[jgid] >= MAX_CG)
+ {
+ /* Add to LJ+coulomb long-range list */
+ put_in_list(bHaveVdW, ngid, md, icg, jgid,
+ nlr_ljc[jgid], nl_lr_ljc[jgid], top->cgs.index,
+ bexcl, shift, fr, TRUE, TRUE, TRUE, fr->solvent_opt);
+ nlr_ljc[jgid] = 0;
+ }
+ nl_lr_ljc[jgid][nlr_ljc[jgid]++] = jjcg;
+ }
+ else
+ {
+ if (nlr_one[jgid] >= MAX_CG)
+ {
+ /* Add to long-range list with only coul, or only LJ */
+ put_in_list(bHaveVdW, ngid, md, icg, jgid,
+ nlr_one[jgid], nl_lr_one[jgid], top->cgs.index,
+ bexcl, shift, fr, TRUE, rvdw_lt_rcoul, rcoul_lt_rvdw, fr->solvent_opt);
+ nlr_one[jgid] = 0;
+ }
+ nl_lr_one[jgid][nlr_one[jgid]++] = jjcg;
+ }
+ }
+ }
+ nns++;
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ /* CHECK whether there is anything left in the buffers */
+ for (nn = 0; (nn < ngid); nn++)
+ {
+ if (nsr[nn] > 0)
+ {
+ put_in_list(bHaveVdW, ngid, md, icg, nn, nsr[nn], nl_sr[nn],
+ cgs->index, /* cgsatoms, */ bexcl,
+ shift, fr, FALSE, TRUE, TRUE, fr->solvent_opt);
+ }
+
+ if (nlr_ljc[nn] > 0)
+ {
+ put_in_list(bHaveVdW, ngid, md, icg, nn, nlr_ljc[nn],
+ nl_lr_ljc[nn], top->cgs.index,
+ bexcl, shift, fr, TRUE, TRUE, TRUE, fr->solvent_opt);
+ }
+
+ if (nlr_one[nn] > 0)
+ {
+ put_in_list(bHaveVdW, ngid, md, icg, nn, nlr_one[nn],
+ nl_lr_one[nn], top->cgs.index,
+ bexcl, shift, fr, TRUE, rvdw_lt_rcoul, rcoul_lt_rvdw, fr->solvent_opt);
+ }
+ }
+ }
+ }
+ }
+ /* setexcl(nri,i_atoms,&top->atoms.excl,FALSE,bexcl); */
+ setexcl(cgs->index[icg], cgs->index[icg+1], &top->excls, FALSE, bexcl);
+ }
+ /* No need to perform any left-over force calculations anymore (as we used to do here)
+ * since we now save the proper long-range lists for later evaluation.
+ */
+
+ debug_gmx();
+
+ /* Close neighbourlists */
+ close_neighbor_lists(fr, bMakeQMMMnblist);
+
+ return nns;
+}
+
+void ns_realloc_natoms(gmx_ns_t *ns, int natoms)
+{
+ int i;
+
+ if (natoms > ns->nra_alloc)
+ {
+ ns->nra_alloc = over_alloc_dd(natoms);
+ srenew(ns->bexcl, ns->nra_alloc);
+ for (i = 0; i < ns->nra_alloc; i++)
+ {
+ ns->bexcl[i] = 0;
+ }
+ }
+}
+
+void init_ns(FILE *fplog, const t_commrec *cr,
+ gmx_ns_t *ns, t_forcerec *fr,
+ const gmx_mtop_t *mtop)
+{
+ int mt, icg, nr_in_cg, maxcg, i, j, jcg, ngid, ncg;
+ t_block *cgs;
+ char *ptr;
+
+ /* Compute largest charge groups size (# atoms) */
+ nr_in_cg = 1;
+ for (mt = 0; mt < mtop->nmoltype; mt++)
+ {
+ cgs = &mtop->moltype[mt].cgs;
+ for (icg = 0; (icg < cgs->nr); icg++)
+ {
+ nr_in_cg = max(nr_in_cg, (int)(cgs->index[icg+1]-cgs->index[icg]));
+ }
+ }
+
+ /* Verify whether largest charge group is <= max cg.
+ * This is determined by the type of the local exclusion type
+ * Exclusions are stored in bits. (If the type is not large
+ * enough, enlarge it, unsigned char -> unsigned short -> unsigned long)
+ */
+ maxcg = sizeof(t_excl)*8;
+ if (nr_in_cg > maxcg)
+ {
+ gmx_fatal(FARGS, "Max #atoms in a charge group: %d > %d\n",
+ nr_in_cg, maxcg);
+ }
+
+ ngid = mtop->groups.grps[egcENER].nr;
+ snew(ns->bExcludeAlleg, ngid);
+ for (i = 0; i < ngid; i++)
+ {
+ ns->bExcludeAlleg[i] = TRUE;
+ for (j = 0; j < ngid; j++)
+ {
+ if (!(fr->egp_flags[i*ngid+j] & EGP_EXCL))
+ {
+ ns->bExcludeAlleg[i] = FALSE;
+ }
+ }
+ }
+
+ if (fr->bGrid)
+ {
+ /* Grid search */
+ ns->grid = init_grid(fplog, fr);
+ init_nsgrid_lists(fr, ngid, ns);
+ }
+ else
+ {
+ /* Simple search */
+ snew(ns->ns_buf, ngid);
+ for (i = 0; (i < ngid); i++)
+ {
+ snew(ns->ns_buf[i], SHIFTS);
+ }
+ ncg = ncg_mtop(mtop);
+ snew(ns->simple_aaj, 2*ncg);
+ for (jcg = 0; (jcg < ncg); jcg++)
+ {
+ ns->simple_aaj[jcg] = jcg;
+ ns->simple_aaj[jcg+ncg] = jcg;
+ }
+ }
+
+ /* Create array that determines whether or not atoms have VdW */
+ snew(ns->bHaveVdW, fr->ntype);
+ for (i = 0; (i < fr->ntype); i++)
+ {
+ for (j = 0; (j < fr->ntype); j++)
+ {
+ ns->bHaveVdW[i] = (ns->bHaveVdW[i] ||
+ (fr->bBHAM ?
+ ((BHAMA(fr->nbfp, fr->ntype, i, j) != 0) ||
+ (BHAMB(fr->nbfp, fr->ntype, i, j) != 0) ||
+ (BHAMC(fr->nbfp, fr->ntype, i, j) != 0)) :
+ ((C6(fr->nbfp, fr->ntype, i, j) != 0) ||
+ (C12(fr->nbfp, fr->ntype, i, j) != 0))));
+ }
+ }
+ if (debug)
+ {
+ pr_bvec(debug, 0, "bHaveVdW", ns->bHaveVdW, fr->ntype, TRUE);
+ }
+
+ ns->nra_alloc = 0;
+ ns->bexcl = NULL;
+ if (!DOMAINDECOMP(cr))
+ {
+ /* This could be reduced with particle decomposition */
+ ns_realloc_natoms(ns, mtop->natoms);
+ }
+
+ ns->nblist_initialized = FALSE;
+
+ /* nbr list debug dump */
+ {
+ char *ptr = getenv("GMX_DUMP_NL");
+ if (ptr)
+ {
+ ns->dump_nl = strtol(ptr, NULL, 10);
+ if (fplog)
+ {
+ fprintf(fplog, "GMX_DUMP_NL = %d", ns->dump_nl);
+ }
+ }
+ else
+ {
+ ns->dump_nl = 0;
+ }
+ }
+}
+
+
+int search_neighbours(FILE *log, t_forcerec *fr,
+ matrix box,
+ gmx_localtop_t *top,
+ gmx_groups_t *groups,
+ t_commrec *cr,
+ t_nrnb *nrnb, t_mdatoms *md,
+ gmx_bool bFillGrid,
+ gmx_bool bDoLongRangeNS)
+{
+ t_block *cgs = &(top->cgs);
+ rvec box_size, grid_x0, grid_x1;
+ int i, j, m, ngid;
+ real min_size, grid_dens;
+ int nsearch;
+ gmx_bool bGrid;
+ char *ptr;
+ gmx_bool *i_egp_flags;
+ int cg_start, cg_end, start, end;
+ gmx_ns_t *ns;
+ t_grid *grid;
+ gmx_domdec_zones_t *dd_zones;
+ put_in_list_t *put_in_list;
+
+ ns = &fr->ns;
+
+ /* Set some local variables */
+ bGrid = fr->bGrid;
+ ngid = groups->grps[egcENER].nr;
+
+ for (m = 0; (m < DIM); m++)
+ {
+ box_size[m] = box[m][m];
+ }
+
+ if (fr->ePBC != epbcNONE)
+ {
+ if (sqr(fr->rlistlong) >= max_cutoff2(fr->ePBC, box))
+ {
+ gmx_fatal(FARGS, "One of the box vectors has become shorter than twice the cut-off length or box_yy-|box_zy| or box_zz has become smaller than the cut-off.");
+ }
+ if (!bGrid)
+ {
+ min_size = min(box_size[XX], min(box_size[YY], box_size[ZZ]));
+ if (2*fr->rlistlong >= min_size)
+ {
+ gmx_fatal(FARGS, "One of the box diagonal elements has become smaller than twice the cut-off length.");
+ }
+ }
+ }
+
+ if (DOMAINDECOMP(cr))
+ {
+ ns_realloc_natoms(ns, cgs->index[cgs->nr]);
+ }
+ debug_gmx();
+
+ /* Reset the neighbourlists */
+ reset_neighbor_lists(fr, TRUE, TRUE);
+
+ if (bGrid && bFillGrid)
+ {
+
+ grid = ns->grid;
+ if (DOMAINDECOMP(cr))
+ {
+ dd_zones = domdec_zones(cr->dd);
+ }
+ else
+ {
+ dd_zones = NULL;
+
+ get_nsgrid_boundaries(grid->nboundeddim, box, NULL, NULL, NULL, NULL,
+ cgs->nr, fr->cg_cm, grid_x0, grid_x1, &grid_dens);
+
+ grid_first(log, grid, NULL, NULL, box, grid_x0, grid_x1,
+ fr->rlistlong, grid_dens);
+ }
+ debug_gmx();
+
+ /* Don't know why this all is... (DvdS 3/99) */
+#ifndef SEGV
+ start = 0;
+ end = cgs->nr;
+#else
+ start = fr->cg0;
+ end = (cgs->nr+1)/2;
+#endif
+
+ if (DOMAINDECOMP(cr))
+ {
+ end = cgs->nr;
+ fill_grid(dd_zones, grid, end, -1, end, fr->cg_cm);
+ grid->icg0 = 0;
+ grid->icg1 = dd_zones->izone[dd_zones->nizone-1].cg1;
+ }
+ else
+ {
+ fill_grid(NULL, grid, cgs->nr, fr->cg0, fr->hcg, fr->cg_cm);
+ grid->icg0 = fr->cg0;
+ grid->icg1 = fr->hcg;
+ debug_gmx();
+
+ if (PARTDECOMP(cr))
+ {
+ mv_grid(cr, grid);
+ }
+ debug_gmx();
+ }
+
+ calc_elemnr(grid, start, end, cgs->nr);
+ calc_ptrs(grid);
+ grid_last(grid, start, end, cgs->nr);
+
+ if (gmx_debug_at)
+ {
+ check_grid(grid);
+ print_grid(debug, grid);
+ }
+ }
+ else if (fr->n_tpi)
+ {
+ /* Set the grid cell index for the test particle only.
+ * The cell to cg index is not corrected, but that does not matter.
+ */
+ fill_grid(NULL, ns->grid, fr->hcg, fr->hcg-1, fr->hcg, fr->cg_cm);
+ }
+ debug_gmx();
+
+ if (fr->adress_type == eAdressOff)
+ {
+ if (!fr->ns.bCGlist)
+ {
+ put_in_list = put_in_list_at;
+ }
+ else
+ {
+ put_in_list = put_in_list_cg;
+ }
+ }
+ else
+ {
+ put_in_list = put_in_list_adress;
+ }
+
+ /* Do the core! */
+ if (bGrid)
+ {
+ grid = ns->grid;
+ nsearch = nsgrid_core(cr, fr, box, ngid, top,
+ grid, ns->bexcl, ns->bExcludeAlleg,
+ md, put_in_list, ns->bHaveVdW,
+ bDoLongRangeNS, FALSE);
+
+ /* neighbour searching withouth QMMM! QM atoms have zero charge in
+ * the classical calculation. The charge-charge interaction
+ * between QM and MM atoms is handled in the QMMM core calculation
+ * (see QMMM.c). The VDW however, we'd like to compute classically
+ * and the QM MM atom pairs have just been put in the
+ * corresponding neighbourlists. in case of QMMM we still need to
+ * fill a special QMMM neighbourlist that contains all neighbours
+ * of the QM atoms. If bQMMM is true, this list will now be made:
+ */
+ if (fr->bQMMM && fr->qr->QMMMscheme != eQMMMschemeoniom)
+ {
+ nsearch += nsgrid_core(cr, fr, box, ngid, top,
+ grid, ns->bexcl, ns->bExcludeAlleg,
+ md, put_in_list_qmmm, ns->bHaveVdW,
+ bDoLongRangeNS, TRUE);
+ }
+ }
+ else
+ {
+ nsearch = ns_simple_core(fr, top, md, box, box_size,
+ ns->bexcl, ns->simple_aaj,
+ ngid, ns->ns_buf, put_in_list, ns->bHaveVdW);
+ }
+ debug_gmx();
+
+#ifdef DEBUG
+ pr_nsblock(log);
+#endif
+
+ inc_nrnb(nrnb, eNR_NS, nsearch);
+ /* inc_nrnb(nrnb,eNR_LR,fr->nlr); */
+
+ return nsearch;
+}
+
+int natoms_beyond_ns_buffer(t_inputrec *ir, t_forcerec *fr, t_block *cgs,
+ matrix scale_tot, rvec *x)
+{
+ int cg0, cg1, cg, a0, a1, a, i, j;
+ real rint, hbuf2, scale;
+ rvec *cg_cm, cgsc;
+ gmx_bool bIsotropic;
+ int nBeyond;
+
+ nBeyond = 0;
+
+ rint = max(ir->rcoulomb, ir->rvdw);
+ if (ir->rlist < rint)
+ {
+ gmx_fatal(FARGS, "The neighbor search buffer has negative size: %f nm",
+ ir->rlist - rint);
+ }
+ cg_cm = fr->cg_cm;
+
+ cg0 = fr->cg0;
+ cg1 = fr->hcg;
+
+ if (!EI_DYNAMICS(ir->eI) || !DYNAMIC_BOX(*ir))
+ {
+ hbuf2 = sqr(0.5*(ir->rlist - rint));
+ for (cg = cg0; cg < cg1; cg++)
+ {
+ a0 = cgs->index[cg];
+ a1 = cgs->index[cg+1];
+ for (a = a0; a < a1; a++)
+ {
+ if (distance2(cg_cm[cg], x[a]) > hbuf2)
+ {
+ nBeyond++;
+ }
+ }
+ }
+ }
+ else
+ {
+ bIsotropic = TRUE;
+ scale = scale_tot[0][0];
+ for (i = 1; i < DIM; i++)
+ {
+ /* With anisotropic scaling, the original spherical ns volumes become
+ * ellipsoids. To avoid costly transformations we use the minimum
+ * eigenvalue of the scaling matrix for determining the buffer size.
+ * Since the lower half is 0, the eigenvalues are the diagonal elements.
+ */
+ scale = min(scale, scale_tot[i][i]);
+ if (scale_tot[i][i] != scale_tot[i-1][i-1])
+ {
+ bIsotropic = FALSE;
+ }
+ for (j = 0; j < i; j++)
+ {
+ if (scale_tot[i][j] != 0)
+ {
+ bIsotropic = FALSE;
+ }
+ }
+ }
+ hbuf2 = sqr(0.5*(scale*ir->rlist - rint));
+ if (bIsotropic)
+ {
+ for (cg = cg0; cg < cg1; cg++)
+ {
+ svmul(scale, cg_cm[cg], cgsc);
+ a0 = cgs->index[cg];
+ a1 = cgs->index[cg+1];
+ for (a = a0; a < a1; a++)
+ {
+ if (distance2(cgsc, x[a]) > hbuf2)
+ {
+ nBeyond++;
+ }
+ }
+ }
+ }
+ else
+ {
+ /* Anistropic scaling */
+ for (cg = cg0; cg < cg1; cg++)
+ {
+ /* Since scale_tot contains the transpose of the scaling matrix,
+ * we need to multiply with the transpose.
+ */
+ tmvmul_ur0(scale_tot, cg_cm[cg], cgsc);
+ a0 = cgs->index[cg];
+ a1 = cgs->index[cg+1];
+ for (a = a0; a < a1; a++)
+ {
+ if (distance2(cgsc, x[a]) > hbuf2)
+ {
+ nBeyond++;
+ }
+ }
+ }
+ }
+ }
+
+ return nBeyond;
+}
--- /dev/null
- /* Turn on SIMD intrinsics for PME solve */
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * VERSION 3.2.0
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2004, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * GROwing Monsters And Cloning Shrimps
+ */
+/* IMPORTANT FOR DEVELOPERS:
+ *
+ * Triclinic pme stuff isn't entirely trivial, and we've experienced
+ * some bugs during development (many of them due to me). To avoid
+ * this in the future, please check the following things if you make
+ * changes in this file:
+ *
+ * 1. You should obtain identical (at least to the PME precision)
+ * energies, forces, and virial for
+ * a rectangular box and a triclinic one where the z (or y) axis is
+ * tilted a whole box side. For instance you could use these boxes:
+ *
+ * rectangular triclinic
+ * 2 0 0 2 0 0
+ * 0 2 0 0 2 0
+ * 0 0 6 2 2 6
+ *
+ * 2. You should check the energy conservation in a triclinic box.
+ *
+ * It might seem an overkill, but better safe than sorry.
+ * /Erik 001109
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include "gromacs/fft/parallel_3dfft.h"
+#include "gromacs/utility/gmxmpi.h"
+
+#include <stdio.h>
+#include <string.h>
+#include <math.h>
+#include <assert.h>
+#include "typedefs.h"
+#include "txtdump.h"
+#include "vec.h"
+#include "gmxcomplex.h"
+#include "smalloc.h"
+#include "futil.h"
+#include "coulomb.h"
+#include "gmx_fatal.h"
+#include "pme.h"
+#include "network.h"
+#include "physics.h"
+#include "nrnb.h"
+#include "gmx_wallcycle.h"
+#include "pdbio.h"
+#include "gmx_cyclecounter.h"
+#include "gmx_omp.h"
+#include "macros.h"
+
+
+/* Include the SIMD macro file and then check for support */
+#include "gmx_simd_macros.h"
+#if defined GMX_HAVE_SIMD_MACROS && defined GMX_SIMD_HAVE_EXP
- /* SIMD spread+gather only in single precision with SSE2 or higher available.
- * We might want to switch to use gmx_simd_macros.h, but this is somewhat
- * complicated, as we use unaligned and/or 4-wide only loads.
++/* Turn on arbitrary width SIMD intrinsics for PME solve */
+#define PME_SIMD
+#endif
+
- #if defined(GMX_X86_SSE2) && !defined(GMX_DOUBLE)
- #define PME_SSE_SPREAD_GATHER
- #include <emmintrin.h>
- /* Some old AMD processors could have problems with unaligned loads+stores */
- #ifndef GMX_FAHCORE
- #define PME_SSE_UNALIGNED
++/* Include the 4-wide SIMD macro file */
++#include "gmx_simd4_macros.h"
++/* Check if we have 4-wide SIMD macro support */
++#ifdef GMX_HAVE_SIMD4_MACROS
++/* Do PME spread and gather with 4-wide SIMD.
++ * NOTE: SIMD is only used with PME order 4 and 5 (which are the most common).
+ */
- /* GMX_CACHE_SEP should be a multiple of 16 to preserve alignment */
++#define PME_SIMD4_SPREAD_GATHER
++
++#ifdef GMX_SIMD4_HAVE_UNALIGNED
++/* With PME-order=4 on x86, unaligned load+store is slightly faster
++ * than doubling all SIMD operations when using aligned load+store.
++ */
++#define PME_SIMD4_UNALIGNED
+#endif
+#endif
+
+#define DFT_TOL 1e-7
+/* #define PRT_FORCE */
+/* conditions for on the fly time-measurement */
+/* #define TAKETIME (step > 1 && timesteps < 10) */
+#define TAKETIME FALSE
+
+/* #define PME_TIME_THREADS */
+
+#ifdef GMX_DOUBLE
+#define mpi_type MPI_DOUBLE
+#else
+#define mpi_type MPI_FLOAT
+#endif
+
- #ifdef PME_SSE_SPREAD_GATHER
- /* Masks for SSE aligned spreading and gathering */
- __m128 mask_SSE0[6], mask_SSE1[6];
++#ifdef PME_SIMD4_SPREAD_GATHER
++#define SIMD4_ALIGNMENT (GMX_SIMD4_WIDTH*sizeof(real))
++#else
++/* We can use any alignment, apart from 0, so we use 4 reals */
++#define SIMD4_ALIGNMENT (4*sizeof(real))
++#endif
++
++/* GMX_CACHE_SEP should be a multiple of the SIMD and SIMD4 register size
++ * to preserve alignment.
++ */
+#define GMX_CACHE_SEP 64
+
+/* We only define a maximum to be able to use local arrays without allocation.
+ * An order larger than 12 should never be needed, even for test cases.
+ * If needed it can be changed here.
+ */
+#define PME_ORDER_MAX 12
+
+/* Internal datastructures */
+typedef struct {
+ int send_index0;
+ int send_nindex;
+ int recv_index0;
+ int recv_nindex;
+ int recv_size; /* Receive buffer width, used with OpenMP */
+} pme_grid_comm_t;
+
+typedef struct {
+#ifdef GMX_MPI
+ MPI_Comm mpi_comm;
+#endif
+ int nnodes, nodeid;
+ int *s2g0;
+ int *s2g1;
+ int noverlap_nodes;
+ int *send_id, *recv_id;
+ int send_size; /* Send buffer width, used with OpenMP */
+ pme_grid_comm_t *comm_data;
+ real *sendbuf;
+ real *recvbuf;
+} pme_overlap_t;
+
+typedef struct {
+ int *n; /* Cumulative counts of the number of particles per thread */
+ int nalloc; /* Allocation size of i */
+ int *i; /* Particle indices ordered on thread index (n) */
+} thread_plist_t;
+
+typedef struct {
+ int *thread_one;
+ int n;
+ int *ind;
+ splinevec theta;
+ real *ptr_theta_z;
+ splinevec dtheta;
+ real *ptr_dtheta_z;
+} splinedata_t;
+
+typedef struct {
+ int dimind; /* The index of the dimension, 0=x, 1=y */
+ int nslab;
+ int nodeid;
+#ifdef GMX_MPI
+ MPI_Comm mpi_comm;
+#endif
+
+ int *node_dest; /* The nodes to send x and q to with DD */
+ int *node_src; /* The nodes to receive x and q from with DD */
+ int *buf_index; /* Index for commnode into the buffers */
+
+ int maxshift;
+
+ int npd;
+ int pd_nalloc;
+ int *pd;
+ int *count; /* The number of atoms to send to each node */
+ int **count_thread;
+ int *rcount; /* The number of atoms to receive */
+
+ int n;
+ int nalloc;
+ rvec *x;
+ real *q;
+ rvec *f;
+ gmx_bool bSpread; /* These coordinates are used for spreading */
+ int pme_order;
+ ivec *idx;
+ rvec *fractx; /* Fractional coordinate relative to the
+ * lower cell boundary
+ */
+ int nthread;
+ int *thread_idx; /* Which thread should spread which charge */
+ thread_plist_t *thread_plist;
+ splinedata_t *spline;
+} pme_atomcomm_t;
+
+#define FLBS 3
+#define FLBSZ 4
+
+typedef struct {
+ ivec ci; /* The spatial location of this grid */
+ ivec n; /* The used size of *grid, including order-1 */
+ ivec offset; /* The grid offset from the full node grid */
+ int order; /* PME spreading order */
+ ivec s; /* The allocated size of *grid, s >= n */
+ real *grid; /* The grid local thread, size n */
+} pmegrid_t;
+
+typedef struct {
+ pmegrid_t grid; /* The full node grid (non thread-local) */
+ int nthread; /* The number of threads operating on this grid */
+ ivec nc; /* The local spatial decomposition over the threads */
+ pmegrid_t *grid_th; /* Array of grids for each thread */
+ real *grid_all; /* Allocated array for the grids in *grid_th */
+ int **g2t; /* The grid to thread index */
+ ivec nthread_comm; /* The number of threads to communicate with */
+} pmegrids_t;
+
+
+typedef struct {
- /* In z we add padding, this is only required for the aligned SSE code */
- srenew(*ptr_z, nalloc+2*padding);
++#ifdef PME_SIMD4_SPREAD_GATHER
++ /* Masks for 4-wide SIMD aligned spreading and gathering */
++ gmx_simd4_pb mask_S0[6], mask_S1[6];
+#else
+ int dummy; /* C89 requires that struct has at least one member */
+#endif
+} pme_spline_work_t;
+
+typedef struct {
+ /* work data for solve_pme */
+ int nalloc;
+ real * mhx;
+ real * mhy;
+ real * mhz;
+ real * m2;
+ real * denom;
+ real * tmp1_alloc;
+ real * tmp1;
+ real * eterm;
+ real * m2inv;
+
+ real energy;
+ matrix vir;
+} pme_work_t;
+
+typedef struct gmx_pme {
+ int ndecompdim; /* The number of decomposition dimensions */
+ int nodeid; /* Our nodeid in mpi->mpi_comm */
+ int nodeid_major;
+ int nodeid_minor;
+ int nnodes; /* The number of nodes doing PME */
+ int nnodes_major;
+ int nnodes_minor;
+
+ MPI_Comm mpi_comm;
+ MPI_Comm mpi_comm_d[2]; /* Indexed on dimension, 0=x, 1=y */
+#ifdef GMX_MPI
+ MPI_Datatype rvec_mpi; /* the pme vector's MPI type */
+#endif
+
+ gmx_bool bUseThreads; /* Does any of the PME ranks have nthread>1 ? */
+ int nthread; /* The number of threads doing PME on our rank */
+
+ gmx_bool bPPnode; /* Node also does particle-particle forces */
+ gmx_bool bFEP; /* Compute Free energy contribution */
+ int nkx, nky, nkz; /* Grid dimensions */
+ gmx_bool bP3M; /* Do P3M: optimize the influence function */
+ int pme_order;
+ real epsilon_r;
+
+ pmegrids_t pmegridA; /* Grids on which we do spreading/interpolation, includes overlap */
+ pmegrids_t pmegridB;
+ /* The PME charge spreading grid sizes/strides, includes pme_order-1 */
+ int pmegrid_nx, pmegrid_ny, pmegrid_nz;
+ /* pmegrid_nz might be larger than strictly necessary to ensure
+ * memory alignment, pmegrid_nz_base gives the real base size.
+ */
+ int pmegrid_nz_base;
+ /* The local PME grid starting indices */
+ int pmegrid_start_ix, pmegrid_start_iy, pmegrid_start_iz;
+
+ /* Work data for spreading and gathering */
+ pme_spline_work_t *spline_work;
+
+ real *fftgridA; /* Grids for FFT. With 1D FFT decomposition this can be a pointer */
+ real *fftgridB; /* inside the interpolation grid, but separate for 2D PME decomp. */
+ int fftgrid_nx, fftgrid_ny, fftgrid_nz;
+
+ t_complex *cfftgridA; /* Grids for complex FFT data */
+ t_complex *cfftgridB;
+ int cfftgrid_nx, cfftgrid_ny, cfftgrid_nz;
+
+ gmx_parallel_3dfft_t pfft_setupA;
+ gmx_parallel_3dfft_t pfft_setupB;
+
+ int *nnx, *nny, *nnz;
+ real *fshx, *fshy, *fshz;
+
+ pme_atomcomm_t atc[2]; /* Indexed on decomposition index */
+ matrix recipbox;
+ splinevec bsp_mod;
+
+ pme_overlap_t overlap[2]; /* Indexed on dimension, 0=x, 1=y */
+
+ pme_atomcomm_t atc_energy; /* Only for gmx_pme_calc_energy */
+
+ rvec *bufv; /* Communication buffer */
+ real *bufr; /* Communication buffer */
+ int buf_nalloc; /* The communication buffer size */
+
+ /* thread local work data for solve_pme */
+ pme_work_t *work;
+
+ /* Work data for PME_redist */
+ gmx_bool redist_init;
+ int * scounts;
+ int * rcounts;
+ int * sdispls;
+ int * rdispls;
+ int * sidx;
+ int * idxa;
+ real * redist_buf;
+ int redist_buf_nalloc;
+
+ /* Work data for sum_qgrid */
+ real * sum_qgrid_tmp;
+ real * sum_qgrid_dd_tmp;
+} t_gmx_pme;
+
+
+static void calc_interpolation_idx(gmx_pme_t pme, pme_atomcomm_t *atc,
+ int start, int end, int thread)
+{
+ int i;
+ int *idxptr, tix, tiy, tiz;
+ real *xptr, *fptr, tx, ty, tz;
+ real rxx, ryx, ryy, rzx, rzy, rzz;
+ int nx, ny, nz;
+ int start_ix, start_iy, start_iz;
+ int *g2tx, *g2ty, *g2tz;
+ gmx_bool bThreads;
+ int *thread_idx = NULL;
+ thread_plist_t *tpl = NULL;
+ int *tpl_n = NULL;
+ int thread_i;
+
+ nx = pme->nkx;
+ ny = pme->nky;
+ nz = pme->nkz;
+
+ start_ix = pme->pmegrid_start_ix;
+ start_iy = pme->pmegrid_start_iy;
+ start_iz = pme->pmegrid_start_iz;
+
+ rxx = pme->recipbox[XX][XX];
+ ryx = pme->recipbox[YY][XX];
+ ryy = pme->recipbox[YY][YY];
+ rzx = pme->recipbox[ZZ][XX];
+ rzy = pme->recipbox[ZZ][YY];
+ rzz = pme->recipbox[ZZ][ZZ];
+
+ g2tx = pme->pmegridA.g2t[XX];
+ g2ty = pme->pmegridA.g2t[YY];
+ g2tz = pme->pmegridA.g2t[ZZ];
+
+ bThreads = (atc->nthread > 1);
+ if (bThreads)
+ {
+ thread_idx = atc->thread_idx;
+
+ tpl = &atc->thread_plist[thread];
+ tpl_n = tpl->n;
+ for (i = 0; i < atc->nthread; i++)
+ {
+ tpl_n[i] = 0;
+ }
+ }
+
+ for (i = start; i < end; i++)
+ {
+ xptr = atc->x[i];
+ idxptr = atc->idx[i];
+ fptr = atc->fractx[i];
+
+ /* Fractional coordinates along box vectors, add 2.0 to make 100% sure we are positive for triclinic boxes */
+ tx = nx * ( xptr[XX] * rxx + xptr[YY] * ryx + xptr[ZZ] * rzx + 2.0 );
+ ty = ny * ( xptr[YY] * ryy + xptr[ZZ] * rzy + 2.0 );
+ tz = nz * ( xptr[ZZ] * rzz + 2.0 );
+
+ tix = (int)(tx);
+ tiy = (int)(ty);
+ tiz = (int)(tz);
+
+ /* Because decomposition only occurs in x and y,
+ * we never have a fraction correction in z.
+ */
+ fptr[XX] = tx - tix + pme->fshx[tix];
+ fptr[YY] = ty - tiy + pme->fshy[tiy];
+ fptr[ZZ] = tz - tiz;
+
+ idxptr[XX] = pme->nnx[tix];
+ idxptr[YY] = pme->nny[tiy];
+ idxptr[ZZ] = pme->nnz[tiz];
+
+#ifdef DEBUG
+ range_check(idxptr[XX], 0, pme->pmegrid_nx);
+ range_check(idxptr[YY], 0, pme->pmegrid_ny);
+ range_check(idxptr[ZZ], 0, pme->pmegrid_nz);
+#endif
+
+ if (bThreads)
+ {
+ thread_i = g2tx[idxptr[XX]] + g2ty[idxptr[YY]] + g2tz[idxptr[ZZ]];
+ thread_idx[i] = thread_i;
+ tpl_n[thread_i]++;
+ }
+ }
+
+ if (bThreads)
+ {
+ /* Make a list of particle indices sorted on thread */
+
+ /* Get the cumulative count */
+ for (i = 1; i < atc->nthread; i++)
+ {
+ tpl_n[i] += tpl_n[i-1];
+ }
+ /* The current implementation distributes particles equally
+ * over the threads, so we could actually allocate for that
+ * in pme_realloc_atomcomm_things.
+ */
+ if (tpl_n[atc->nthread-1] > tpl->nalloc)
+ {
+ tpl->nalloc = over_alloc_large(tpl_n[atc->nthread-1]);
+ srenew(tpl->i, tpl->nalloc);
+ }
+ /* Set tpl_n to the cumulative start */
+ for (i = atc->nthread-1; i >= 1; i--)
+ {
+ tpl_n[i] = tpl_n[i-1];
+ }
+ tpl_n[0] = 0;
+
+ /* Fill our thread local array with indices sorted on thread */
+ for (i = start; i < end; i++)
+ {
+ tpl->i[tpl_n[atc->thread_idx[i]]++] = i;
+ }
+ /* Now tpl_n contains the cummulative count again */
+ }
+}
+
+static void make_thread_local_ind(pme_atomcomm_t *atc,
+ int thread, splinedata_t *spline)
+{
+ int n, t, i, start, end;
+ thread_plist_t *tpl;
+
+ /* Combine the indices made by each thread into one index */
+
+ n = 0;
+ start = 0;
+ for (t = 0; t < atc->nthread; t++)
+ {
+ tpl = &atc->thread_plist[t];
+ /* Copy our part (start - end) from the list of thread t */
+ if (thread > 0)
+ {
+ start = tpl->n[thread-1];
+ }
+ end = tpl->n[thread];
+ for (i = start; i < end; i++)
+ {
+ spline->ind[n++] = tpl->i[i];
+ }
+ }
+
+ spline->n = n;
+}
+
+
+static void pme_calc_pidx(int start, int end,
+ matrix recipbox, rvec x[],
+ pme_atomcomm_t *atc, int *count)
+{
+ int nslab, i;
+ int si;
+ real *xptr, s;
+ real rxx, ryx, rzx, ryy, rzy;
+ int *pd;
+
+ /* Calculate PME task index (pidx) for each grid index.
+ * Here we always assign equally sized slabs to each node
+ * for load balancing reasons (the PME grid spacing is not used).
+ */
+
+ nslab = atc->nslab;
+ pd = atc->pd;
+
+ /* Reset the count */
+ for (i = 0; i < nslab; i++)
+ {
+ count[i] = 0;
+ }
+
+ if (atc->dimind == 0)
+ {
+ rxx = recipbox[XX][XX];
+ ryx = recipbox[YY][XX];
+ rzx = recipbox[ZZ][XX];
+ /* Calculate the node index in x-dimension */
+ for (i = start; i < end; i++)
+ {
+ xptr = x[i];
+ /* Fractional coordinates along box vectors */
+ s = nslab*(xptr[XX]*rxx + xptr[YY]*ryx + xptr[ZZ]*rzx);
+ si = (int)(s + 2*nslab) % nslab;
+ pd[i] = si;
+ count[si]++;
+ }
+ }
+ else
+ {
+ ryy = recipbox[YY][YY];
+ rzy = recipbox[ZZ][YY];
+ /* Calculate the node index in y-dimension */
+ for (i = start; i < end; i++)
+ {
+ xptr = x[i];
+ /* Fractional coordinates along box vectors */
+ s = nslab*(xptr[YY]*ryy + xptr[ZZ]*rzy);
+ si = (int)(s + 2*nslab) % nslab;
+ pd[i] = si;
+ count[si]++;
+ }
+ }
+}
+
+static void pme_calc_pidx_wrapper(int natoms, matrix recipbox, rvec x[],
+ pme_atomcomm_t *atc)
+{
+ int nthread, thread, slab;
+
+ nthread = atc->nthread;
+
+#pragma omp parallel for num_threads(nthread) schedule(static)
+ for (thread = 0; thread < nthread; thread++)
+ {
+ pme_calc_pidx(natoms* thread /nthread,
+ natoms*(thread+1)/nthread,
+ recipbox, x, atc, atc->count_thread[thread]);
+ }
+ /* Non-parallel reduction, since nslab is small */
+
+ for (thread = 1; thread < nthread; thread++)
+ {
+ for (slab = 0; slab < atc->nslab; slab++)
+ {
+ atc->count_thread[0][slab] += atc->count_thread[thread][slab];
+ }
+ }
+}
+
+static void realloc_splinevec(splinevec th, real **ptr_z, int nalloc)
+{
+ const int padding = 4;
+ int i;
+
+ srenew(th[XX], nalloc);
+ srenew(th[YY], nalloc);
- #ifdef PME_SSE_SPREAD_GATHER
- #ifdef PME_SSE_UNALIGNED
- #define PME_SPREAD_SSE_ORDER4
++ /* In z we add padding, this is only required for the aligned SIMD code */
++ sfree_aligned(*ptr_z);
++ snew_aligned(*ptr_z, nalloc+2*padding, SIMD4_ALIGNMENT);
+ th[ZZ] = *ptr_z + padding;
+
+ for (i = 0; i < padding; i++)
+ {
+ (*ptr_z)[ i] = 0;
+ (*ptr_z)[padding+nalloc+i] = 0;
+ }
+}
+
+static void pme_realloc_splinedata(splinedata_t *spline, pme_atomcomm_t *atc)
+{
+ int i, d;
+
+ srenew(spline->ind, atc->nalloc);
+ /* Initialize the index to identity so it works without threads */
+ for (i = 0; i < atc->nalloc; i++)
+ {
+ spline->ind[i] = i;
+ }
+
+ realloc_splinevec(spline->theta, &spline->ptr_theta_z,
+ atc->pme_order*atc->nalloc);
+ realloc_splinevec(spline->dtheta, &spline->ptr_dtheta_z,
+ atc->pme_order*atc->nalloc);
+}
+
+static void pme_realloc_atomcomm_things(pme_atomcomm_t *atc)
+{
+ int nalloc_old, i, j, nalloc_tpl;
+
+ /* We have to avoid a NULL pointer for atc->x to avoid
+ * possible fatal errors in MPI routines.
+ */
+ if (atc->n > atc->nalloc || atc->nalloc == 0)
+ {
+ nalloc_old = atc->nalloc;
+ atc->nalloc = over_alloc_dd(max(atc->n, 1));
+
+ if (atc->nslab > 1)
+ {
+ srenew(atc->x, atc->nalloc);
+ srenew(atc->q, atc->nalloc);
+ srenew(atc->f, atc->nalloc);
+ for (i = nalloc_old; i < atc->nalloc; i++)
+ {
+ clear_rvec(atc->f[i]);
+ }
+ }
+ if (atc->bSpread)
+ {
+ srenew(atc->fractx, atc->nalloc);
+ srenew(atc->idx, atc->nalloc);
+
+ if (atc->nthread > 1)
+ {
+ srenew(atc->thread_idx, atc->nalloc);
+ }
+
+ for (i = 0; i < atc->nthread; i++)
+ {
+ pme_realloc_splinedata(&atc->spline[i], atc);
+ }
+ }
+ }
+}
+
+static void pmeredist_pd(gmx_pme_t pme, gmx_bool forw,
+ int n, gmx_bool bXF, rvec *x_f, real *charge,
+ pme_atomcomm_t *atc)
+/* Redistribute particle data for PME calculation */
+/* domain decomposition by x coordinate */
+{
+ int *idxa;
+ int i, ii;
+
+ if (FALSE == pme->redist_init)
+ {
+ snew(pme->scounts, atc->nslab);
+ snew(pme->rcounts, atc->nslab);
+ snew(pme->sdispls, atc->nslab);
+ snew(pme->rdispls, atc->nslab);
+ snew(pme->sidx, atc->nslab);
+ pme->redist_init = TRUE;
+ }
+ if (n > pme->redist_buf_nalloc)
+ {
+ pme->redist_buf_nalloc = over_alloc_dd(n);
+ srenew(pme->redist_buf, pme->redist_buf_nalloc*DIM);
+ }
+
+ pme->idxa = atc->pd;
+
+#ifdef GMX_MPI
+ if (forw && bXF)
+ {
+ /* forward, redistribution from pp to pme */
+
+ /* Calculate send counts and exchange them with other nodes */
+ for (i = 0; (i < atc->nslab); i++)
+ {
+ pme->scounts[i] = 0;
+ }
+ for (i = 0; (i < n); i++)
+ {
+ pme->scounts[pme->idxa[i]]++;
+ }
+ MPI_Alltoall( pme->scounts, 1, MPI_INT, pme->rcounts, 1, MPI_INT, atc->mpi_comm);
+
+ /* Calculate send and receive displacements and index into send
+ buffer */
+ pme->sdispls[0] = 0;
+ pme->rdispls[0] = 0;
+ pme->sidx[0] = 0;
+ for (i = 1; i < atc->nslab; i++)
+ {
+ pme->sdispls[i] = pme->sdispls[i-1]+pme->scounts[i-1];
+ pme->rdispls[i] = pme->rdispls[i-1]+pme->rcounts[i-1];
+ pme->sidx[i] = pme->sdispls[i];
+ }
+ /* Total # of particles to be received */
+ atc->n = pme->rdispls[atc->nslab-1] + pme->rcounts[atc->nslab-1];
+
+ pme_realloc_atomcomm_things(atc);
+
+ /* Copy particle coordinates into send buffer and exchange*/
+ for (i = 0; (i < n); i++)
+ {
+ ii = DIM*pme->sidx[pme->idxa[i]];
+ pme->sidx[pme->idxa[i]]++;
+ pme->redist_buf[ii+XX] = x_f[i][XX];
+ pme->redist_buf[ii+YY] = x_f[i][YY];
+ pme->redist_buf[ii+ZZ] = x_f[i][ZZ];
+ }
+ MPI_Alltoallv(pme->redist_buf, pme->scounts, pme->sdispls,
+ pme->rvec_mpi, atc->x, pme->rcounts, pme->rdispls,
+ pme->rvec_mpi, atc->mpi_comm);
+ }
+ if (forw)
+ {
+ /* Copy charge into send buffer and exchange*/
+ for (i = 0; i < atc->nslab; i++)
+ {
+ pme->sidx[i] = pme->sdispls[i];
+ }
+ for (i = 0; (i < n); i++)
+ {
+ ii = pme->sidx[pme->idxa[i]];
+ pme->sidx[pme->idxa[i]]++;
+ pme->redist_buf[ii] = charge[i];
+ }
+ MPI_Alltoallv(pme->redist_buf, pme->scounts, pme->sdispls, mpi_type,
+ atc->q, pme->rcounts, pme->rdispls, mpi_type,
+ atc->mpi_comm);
+ }
+ else /* backward, redistribution from pme to pp */
+ {
+ MPI_Alltoallv(atc->f, pme->rcounts, pme->rdispls, pme->rvec_mpi,
+ pme->redist_buf, pme->scounts, pme->sdispls,
+ pme->rvec_mpi, atc->mpi_comm);
+
+ /* Copy data from receive buffer */
+ for (i = 0; i < atc->nslab; i++)
+ {
+ pme->sidx[i] = pme->sdispls[i];
+ }
+ for (i = 0; (i < n); i++)
+ {
+ ii = DIM*pme->sidx[pme->idxa[i]];
+ x_f[i][XX] += pme->redist_buf[ii+XX];
+ x_f[i][YY] += pme->redist_buf[ii+YY];
+ x_f[i][ZZ] += pme->redist_buf[ii+ZZ];
+ pme->sidx[pme->idxa[i]]++;
+ }
+ }
+#endif
+}
+
+static void pme_dd_sendrecv(pme_atomcomm_t *atc,
+ gmx_bool bBackward, int shift,
+ void *buf_s, int nbyte_s,
+ void *buf_r, int nbyte_r)
+{
+#ifdef GMX_MPI
+ int dest, src;
+ MPI_Status stat;
+
+ if (bBackward == FALSE)
+ {
+ dest = atc->node_dest[shift];
+ src = atc->node_src[shift];
+ }
+ else
+ {
+ dest = atc->node_src[shift];
+ src = atc->node_dest[shift];
+ }
+
+ if (nbyte_s > 0 && nbyte_r > 0)
+ {
+ MPI_Sendrecv(buf_s, nbyte_s, MPI_BYTE,
+ dest, shift,
+ buf_r, nbyte_r, MPI_BYTE,
+ src, shift,
+ atc->mpi_comm, &stat);
+ }
+ else if (nbyte_s > 0)
+ {
+ MPI_Send(buf_s, nbyte_s, MPI_BYTE,
+ dest, shift,
+ atc->mpi_comm);
+ }
+ else if (nbyte_r > 0)
+ {
+ MPI_Recv(buf_r, nbyte_r, MPI_BYTE,
+ src, shift,
+ atc->mpi_comm, &stat);
+ }
+#endif
+}
+
+static void dd_pmeredist_x_q(gmx_pme_t pme,
+ int n, gmx_bool bX, rvec *x, real *charge,
+ pme_atomcomm_t *atc)
+{
+ int *commnode, *buf_index;
+ int nnodes_comm, i, nsend, local_pos, buf_pos, node, scount, rcount;
+
+ commnode = atc->node_dest;
+ buf_index = atc->buf_index;
+
+ nnodes_comm = min(2*atc->maxshift, atc->nslab-1);
+
+ nsend = 0;
+ for (i = 0; i < nnodes_comm; i++)
+ {
+ buf_index[commnode[i]] = nsend;
+ nsend += atc->count[commnode[i]];
+ }
+ if (bX)
+ {
+ if (atc->count[atc->nodeid] + nsend != n)
+ {
+ gmx_fatal(FARGS, "%d particles communicated to PME node %d are more than 2/3 times the cut-off out of the domain decomposition cell of their charge group in dimension %c.\n"
+ "This usually means that your system is not well equilibrated.",
+ n - (atc->count[atc->nodeid] + nsend),
+ pme->nodeid, 'x'+atc->dimind);
+ }
+
+ if (nsend > pme->buf_nalloc)
+ {
+ pme->buf_nalloc = over_alloc_dd(nsend);
+ srenew(pme->bufv, pme->buf_nalloc);
+ srenew(pme->bufr, pme->buf_nalloc);
+ }
+
+ atc->n = atc->count[atc->nodeid];
+ for (i = 0; i < nnodes_comm; i++)
+ {
+ scount = atc->count[commnode[i]];
+ /* Communicate the count */
+ if (debug)
+ {
+ fprintf(debug, "dimind %d PME node %d send to node %d: %d\n",
+ atc->dimind, atc->nodeid, commnode[i], scount);
+ }
+ pme_dd_sendrecv(atc, FALSE, i,
+ &scount, sizeof(int),
+ &atc->rcount[i], sizeof(int));
+ atc->n += atc->rcount[i];
+ }
+
+ pme_realloc_atomcomm_things(atc);
+ }
+
+ local_pos = 0;
+ for (i = 0; i < n; i++)
+ {
+ node = atc->pd[i];
+ if (node == atc->nodeid)
+ {
+ /* Copy direct to the receive buffer */
+ if (bX)
+ {
+ copy_rvec(x[i], atc->x[local_pos]);
+ }
+ atc->q[local_pos] = charge[i];
+ local_pos++;
+ }
+ else
+ {
+ /* Copy to the send buffer */
+ if (bX)
+ {
+ copy_rvec(x[i], pme->bufv[buf_index[node]]);
+ }
+ pme->bufr[buf_index[node]] = charge[i];
+ buf_index[node]++;
+ }
+ }
+
+ buf_pos = 0;
+ for (i = 0; i < nnodes_comm; i++)
+ {
+ scount = atc->count[commnode[i]];
+ rcount = atc->rcount[i];
+ if (scount > 0 || rcount > 0)
+ {
+ if (bX)
+ {
+ /* Communicate the coordinates */
+ pme_dd_sendrecv(atc, FALSE, i,
+ pme->bufv[buf_pos], scount*sizeof(rvec),
+ atc->x[local_pos], rcount*sizeof(rvec));
+ }
+ /* Communicate the charges */
+ pme_dd_sendrecv(atc, FALSE, i,
+ pme->bufr+buf_pos, scount*sizeof(real),
+ atc->q+local_pos, rcount*sizeof(real));
+ buf_pos += scount;
+ local_pos += atc->rcount[i];
+ }
+ }
+}
+
+static void dd_pmeredist_f(gmx_pme_t pme, pme_atomcomm_t *atc,
+ int n, rvec *f,
+ gmx_bool bAddF)
+{
+ int *commnode, *buf_index;
+ int nnodes_comm, local_pos, buf_pos, i, scount, rcount, node;
+
+ commnode = atc->node_dest;
+ buf_index = atc->buf_index;
+
+ nnodes_comm = min(2*atc->maxshift, atc->nslab-1);
+
+ local_pos = atc->count[atc->nodeid];
+ buf_pos = 0;
+ for (i = 0; i < nnodes_comm; i++)
+ {
+ scount = atc->rcount[i];
+ rcount = atc->count[commnode[i]];
+ if (scount > 0 || rcount > 0)
+ {
+ /* Communicate the forces */
+ pme_dd_sendrecv(atc, TRUE, i,
+ atc->f[local_pos], scount*sizeof(rvec),
+ pme->bufv[buf_pos], rcount*sizeof(rvec));
+ local_pos += scount;
+ }
+ buf_index[commnode[i]] = buf_pos;
+ buf_pos += rcount;
+ }
+
+ local_pos = 0;
+ if (bAddF)
+ {
+ for (i = 0; i < n; i++)
+ {
+ node = atc->pd[i];
+ if (node == atc->nodeid)
+ {
+ /* Add from the local force array */
+ rvec_inc(f[i], atc->f[local_pos]);
+ local_pos++;
+ }
+ else
+ {
+ /* Add from the receive buffer */
+ rvec_inc(f[i], pme->bufv[buf_index[node]]);
+ buf_index[node]++;
+ }
+ }
+ }
+ else
+ {
+ for (i = 0; i < n; i++)
+ {
+ node = atc->pd[i];
+ if (node == atc->nodeid)
+ {
+ /* Copy from the local force array */
+ copy_rvec(atc->f[local_pos], f[i]);
+ local_pos++;
+ }
+ else
+ {
+ /* Copy from the receive buffer */
+ copy_rvec(pme->bufv[buf_index[node]], f[i]);
+ buf_index[node]++;
+ }
+ }
+ }
+}
+
+#ifdef GMX_MPI
+static void
+gmx_sum_qgrid_dd(gmx_pme_t pme, real *grid, int direction)
+{
+ pme_overlap_t *overlap;
+ int send_index0, send_nindex;
+ int recv_index0, recv_nindex;
+ MPI_Status stat;
+ int i, j, k, ix, iy, iz, icnt;
+ int ipulse, send_id, recv_id, datasize;
+ real *p;
+ real *sendptr, *recvptr;
+
+ /* Start with minor-rank communication. This is a bit of a pain since it is not contiguous */
+ overlap = &pme->overlap[1];
+
+ for (ipulse = 0; ipulse < overlap->noverlap_nodes; ipulse++)
+ {
+ /* Since we have already (un)wrapped the overlap in the z-dimension,
+ * we only have to communicate 0 to nkz (not pmegrid_nz).
+ */
+ if (direction == GMX_SUM_QGRID_FORWARD)
+ {
+ send_id = overlap->send_id[ipulse];
+ recv_id = overlap->recv_id[ipulse];
+ send_index0 = overlap->comm_data[ipulse].send_index0;
+ send_nindex = overlap->comm_data[ipulse].send_nindex;
+ recv_index0 = overlap->comm_data[ipulse].recv_index0;
+ recv_nindex = overlap->comm_data[ipulse].recv_nindex;
+ }
+ else
+ {
+ send_id = overlap->recv_id[ipulse];
+ recv_id = overlap->send_id[ipulse];
+ send_index0 = overlap->comm_data[ipulse].recv_index0;
+ send_nindex = overlap->comm_data[ipulse].recv_nindex;
+ recv_index0 = overlap->comm_data[ipulse].send_index0;
+ recv_nindex = overlap->comm_data[ipulse].send_nindex;
+ }
+
+ /* Copy data to contiguous send buffer */
+ if (debug)
+ {
+ fprintf(debug, "PME send node %d %d -> %d grid start %d Communicating %d to %d\n",
+ pme->nodeid, overlap->nodeid, send_id,
+ pme->pmegrid_start_iy,
+ send_index0-pme->pmegrid_start_iy,
+ send_index0-pme->pmegrid_start_iy+send_nindex);
+ }
+ icnt = 0;
+ for (i = 0; i < pme->pmegrid_nx; i++)
+ {
+ ix = i;
+ for (j = 0; j < send_nindex; j++)
+ {
+ iy = j + send_index0 - pme->pmegrid_start_iy;
+ for (k = 0; k < pme->nkz; k++)
+ {
+ iz = k;
+ overlap->sendbuf[icnt++] = grid[ix*(pme->pmegrid_ny*pme->pmegrid_nz)+iy*(pme->pmegrid_nz)+iz];
+ }
+ }
+ }
+
+ datasize = pme->pmegrid_nx * pme->nkz;
+
+ MPI_Sendrecv(overlap->sendbuf, send_nindex*datasize, GMX_MPI_REAL,
+ send_id, ipulse,
+ overlap->recvbuf, recv_nindex*datasize, GMX_MPI_REAL,
+ recv_id, ipulse,
+ overlap->mpi_comm, &stat);
+
+ /* Get data from contiguous recv buffer */
+ if (debug)
+ {
+ fprintf(debug, "PME recv node %d %d <- %d grid start %d Communicating %d to %d\n",
+ pme->nodeid, overlap->nodeid, recv_id,
+ pme->pmegrid_start_iy,
+ recv_index0-pme->pmegrid_start_iy,
+ recv_index0-pme->pmegrid_start_iy+recv_nindex);
+ }
+ icnt = 0;
+ for (i = 0; i < pme->pmegrid_nx; i++)
+ {
+ ix = i;
+ for (j = 0; j < recv_nindex; j++)
+ {
+ iy = j + recv_index0 - pme->pmegrid_start_iy;
+ for (k = 0; k < pme->nkz; k++)
+ {
+ iz = k;
+ if (direction == GMX_SUM_QGRID_FORWARD)
+ {
+ grid[ix*(pme->pmegrid_ny*pme->pmegrid_nz)+iy*(pme->pmegrid_nz)+iz] += overlap->recvbuf[icnt++];
+ }
+ else
+ {
+ grid[ix*(pme->pmegrid_ny*pme->pmegrid_nz)+iy*(pme->pmegrid_nz)+iz] = overlap->recvbuf[icnt++];
+ }
+ }
+ }
+ }
+ }
+
+ /* Major dimension is easier, no copying required,
+ * but we might have to sum to separate array.
+ * Since we don't copy, we have to communicate up to pmegrid_nz,
+ * not nkz as for the minor direction.
+ */
+ overlap = &pme->overlap[0];
+
+ for (ipulse = 0; ipulse < overlap->noverlap_nodes; ipulse++)
+ {
+ if (direction == GMX_SUM_QGRID_FORWARD)
+ {
+ send_id = overlap->send_id[ipulse];
+ recv_id = overlap->recv_id[ipulse];
+ send_index0 = overlap->comm_data[ipulse].send_index0;
+ send_nindex = overlap->comm_data[ipulse].send_nindex;
+ recv_index0 = overlap->comm_data[ipulse].recv_index0;
+ recv_nindex = overlap->comm_data[ipulse].recv_nindex;
+ recvptr = overlap->recvbuf;
+ }
+ else
+ {
+ send_id = overlap->recv_id[ipulse];
+ recv_id = overlap->send_id[ipulse];
+ send_index0 = overlap->comm_data[ipulse].recv_index0;
+ send_nindex = overlap->comm_data[ipulse].recv_nindex;
+ recv_index0 = overlap->comm_data[ipulse].send_index0;
+ recv_nindex = overlap->comm_data[ipulse].send_nindex;
+ recvptr = grid + (recv_index0-pme->pmegrid_start_ix)*(pme->pmegrid_ny*pme->pmegrid_nz);
+ }
+
+ sendptr = grid + (send_index0-pme->pmegrid_start_ix)*(pme->pmegrid_ny*pme->pmegrid_nz);
+ datasize = pme->pmegrid_ny * pme->pmegrid_nz;
+
+ if (debug)
+ {
+ fprintf(debug, "PME send node %d %d -> %d grid start %d Communicating %d to %d\n",
+ pme->nodeid, overlap->nodeid, send_id,
+ pme->pmegrid_start_ix,
+ send_index0-pme->pmegrid_start_ix,
+ send_index0-pme->pmegrid_start_ix+send_nindex);
+ fprintf(debug, "PME recv node %d %d <- %d grid start %d Communicating %d to %d\n",
+ pme->nodeid, overlap->nodeid, recv_id,
+ pme->pmegrid_start_ix,
+ recv_index0-pme->pmegrid_start_ix,
+ recv_index0-pme->pmegrid_start_ix+recv_nindex);
+ }
+
+ MPI_Sendrecv(sendptr, send_nindex*datasize, GMX_MPI_REAL,
+ send_id, ipulse,
+ recvptr, recv_nindex*datasize, GMX_MPI_REAL,
+ recv_id, ipulse,
+ overlap->mpi_comm, &stat);
+
+ /* ADD data from contiguous recv buffer */
+ if (direction == GMX_SUM_QGRID_FORWARD)
+ {
+ p = grid + (recv_index0-pme->pmegrid_start_ix)*(pme->pmegrid_ny*pme->pmegrid_nz);
+ for (i = 0; i < recv_nindex*datasize; i++)
+ {
+ p[i] += overlap->recvbuf[i];
+ }
+ }
+ }
+}
+#endif
+
+
+static int
+copy_pmegrid_to_fftgrid(gmx_pme_t pme, real *pmegrid, real *fftgrid)
+{
+ ivec local_fft_ndata, local_fft_offset, local_fft_size;
+ ivec local_pme_size;
+ int i, ix, iy, iz;
+ int pmeidx, fftidx;
+
+ /* Dimensions should be identical for A/B grid, so we just use A here */
+ gmx_parallel_3dfft_real_limits(pme->pfft_setupA,
+ local_fft_ndata,
+ local_fft_offset,
+ local_fft_size);
+
+ local_pme_size[0] = pme->pmegrid_nx;
+ local_pme_size[1] = pme->pmegrid_ny;
+ local_pme_size[2] = pme->pmegrid_nz;
+
+ /* The fftgrid is always 'justified' to the lower-left corner of the PME grid,
+ the offset is identical, and the PME grid always has more data (due to overlap)
+ */
+ {
+#ifdef DEBUG_PME
+ FILE *fp, *fp2;
+ char fn[STRLEN], format[STRLEN];
+ real val;
+ sprintf(fn, "pmegrid%d.pdb", pme->nodeid);
+ fp = ffopen(fn, "w");
+ sprintf(fn, "pmegrid%d.txt", pme->nodeid);
+ fp2 = ffopen(fn, "w");
+ sprintf(format, "%s%s\n", pdbformat, "%6.2f%6.2f");
+#endif
+
+ for (ix = 0; ix < local_fft_ndata[XX]; ix++)
+ {
+ for (iy = 0; iy < local_fft_ndata[YY]; iy++)
+ {
+ for (iz = 0; iz < local_fft_ndata[ZZ]; iz++)
+ {
+ pmeidx = ix*(local_pme_size[YY]*local_pme_size[ZZ])+iy*(local_pme_size[ZZ])+iz;
+ fftidx = ix*(local_fft_size[YY]*local_fft_size[ZZ])+iy*(local_fft_size[ZZ])+iz;
+ fftgrid[fftidx] = pmegrid[pmeidx];
+#ifdef DEBUG_PME
+ val = 100*pmegrid[pmeidx];
+ if (pmegrid[pmeidx] != 0)
+ {
+ fprintf(fp, format, "ATOM", pmeidx, "CA", "GLY", ' ', pmeidx, ' ',
+ 5.0*ix, 5.0*iy, 5.0*iz, 1.0, val);
+ }
+ if (pmegrid[pmeidx] != 0)
+ {
+ fprintf(fp2, "%-12s %5d %5d %5d %12.5e\n",
+ "qgrid",
+ pme->pmegrid_start_ix + ix,
+ pme->pmegrid_start_iy + iy,
+ pme->pmegrid_start_iz + iz,
+ pmegrid[pmeidx]);
+ }
+#endif
+ }
+ }
+ }
+#ifdef DEBUG_PME
+ ffclose(fp);
+ ffclose(fp2);
+#endif
+ }
+ return 0;
+}
+
+
+static gmx_cycles_t omp_cyc_start()
+{
+ return gmx_cycles_read();
+}
+
+static gmx_cycles_t omp_cyc_end(gmx_cycles_t c)
+{
+ return gmx_cycles_read() - c;
+}
+
+
+static int
+copy_fftgrid_to_pmegrid(gmx_pme_t pme, const real *fftgrid, real *pmegrid,
+ int nthread, int thread)
+{
+ ivec local_fft_ndata, local_fft_offset, local_fft_size;
+ ivec local_pme_size;
+ int ixy0, ixy1, ixy, ix, iy, iz;
+ int pmeidx, fftidx;
+#ifdef PME_TIME_THREADS
+ gmx_cycles_t c1;
+ static double cs1 = 0;
+ static int cnt = 0;
+#endif
+
+#ifdef PME_TIME_THREADS
+ c1 = omp_cyc_start();
+#endif
+ /* Dimensions should be identical for A/B grid, so we just use A here */
+ gmx_parallel_3dfft_real_limits(pme->pfft_setupA,
+ local_fft_ndata,
+ local_fft_offset,
+ local_fft_size);
+
+ local_pme_size[0] = pme->pmegrid_nx;
+ local_pme_size[1] = pme->pmegrid_ny;
+ local_pme_size[2] = pme->pmegrid_nz;
+
+ /* The fftgrid is always 'justified' to the lower-left corner of the PME grid,
+ the offset is identical, and the PME grid always has more data (due to overlap)
+ */
+ ixy0 = ((thread )*local_fft_ndata[XX]*local_fft_ndata[YY])/nthread;
+ ixy1 = ((thread+1)*local_fft_ndata[XX]*local_fft_ndata[YY])/nthread;
+
+ for (ixy = ixy0; ixy < ixy1; ixy++)
+ {
+ ix = ixy/local_fft_ndata[YY];
+ iy = ixy - ix*local_fft_ndata[YY];
+
+ pmeidx = (ix*local_pme_size[YY] + iy)*local_pme_size[ZZ];
+ fftidx = (ix*local_fft_size[YY] + iy)*local_fft_size[ZZ];
+ for (iz = 0; iz < local_fft_ndata[ZZ]; iz++)
+ {
+ pmegrid[pmeidx+iz] = fftgrid[fftidx+iz];
+ }
+ }
+
+#ifdef PME_TIME_THREADS
+ c1 = omp_cyc_end(c1);
+ cs1 += (double)c1;
+ cnt++;
+ if (cnt % 20 == 0)
+ {
+ printf("copy %.2f\n", cs1*1e-9);
+ }
+#endif
+
+ return 0;
+}
+
+
+static void
+wrap_periodic_pmegrid(gmx_pme_t pme, real *pmegrid)
+{
+ int nx, ny, nz, pnx, pny, pnz, ny_x, overlap, ix, iy, iz;
+
+ nx = pme->nkx;
+ ny = pme->nky;
+ nz = pme->nkz;
+
+ pnx = pme->pmegrid_nx;
+ pny = pme->pmegrid_ny;
+ pnz = pme->pmegrid_nz;
+
+ overlap = pme->pme_order - 1;
+
+ /* Add periodic overlap in z */
+ for (ix = 0; ix < pme->pmegrid_nx; ix++)
+ {
+ for (iy = 0; iy < pme->pmegrid_ny; iy++)
+ {
+ for (iz = 0; iz < overlap; iz++)
+ {
+ pmegrid[(ix*pny+iy)*pnz+iz] +=
+ pmegrid[(ix*pny+iy)*pnz+nz+iz];
+ }
+ }
+ }
+
+ if (pme->nnodes_minor == 1)
+ {
+ for (ix = 0; ix < pme->pmegrid_nx; ix++)
+ {
+ for (iy = 0; iy < overlap; iy++)
+ {
+ for (iz = 0; iz < nz; iz++)
+ {
+ pmegrid[(ix*pny+iy)*pnz+iz] +=
+ pmegrid[(ix*pny+ny+iy)*pnz+iz];
+ }
+ }
+ }
+ }
+
+ if (pme->nnodes_major == 1)
+ {
+ ny_x = (pme->nnodes_minor == 1 ? ny : pme->pmegrid_ny);
+
+ for (ix = 0; ix < overlap; ix++)
+ {
+ for (iy = 0; iy < ny_x; iy++)
+ {
+ for (iz = 0; iz < nz; iz++)
+ {
+ pmegrid[(ix*pny+iy)*pnz+iz] +=
+ pmegrid[((nx+ix)*pny+iy)*pnz+iz];
+ }
+ }
+ }
+ }
+}
+
+
+static void
+unwrap_periodic_pmegrid(gmx_pme_t pme, real *pmegrid)
+{
+ int nx, ny, nz, pnx, pny, pnz, ny_x, overlap, ix;
+
+ nx = pme->nkx;
+ ny = pme->nky;
+ nz = pme->nkz;
+
+ pnx = pme->pmegrid_nx;
+ pny = pme->pmegrid_ny;
+ pnz = pme->pmegrid_nz;
+
+ overlap = pme->pme_order - 1;
+
+ if (pme->nnodes_major == 1)
+ {
+ ny_x = (pme->nnodes_minor == 1 ? ny : pme->pmegrid_ny);
+
+ for (ix = 0; ix < overlap; ix++)
+ {
+ int iy, iz;
+
+ for (iy = 0; iy < ny_x; iy++)
+ {
+ for (iz = 0; iz < nz; iz++)
+ {
+ pmegrid[((nx+ix)*pny+iy)*pnz+iz] =
+ pmegrid[(ix*pny+iy)*pnz+iz];
+ }
+ }
+ }
+ }
+
+ if (pme->nnodes_minor == 1)
+ {
+#pragma omp parallel for num_threads(pme->nthread) schedule(static)
+ for (ix = 0; ix < pme->pmegrid_nx; ix++)
+ {
+ int iy, iz;
+
+ for (iy = 0; iy < overlap; iy++)
+ {
+ for (iz = 0; iz < nz; iz++)
+ {
+ pmegrid[(ix*pny+ny+iy)*pnz+iz] =
+ pmegrid[(ix*pny+iy)*pnz+iz];
+ }
+ }
+ }
+ }
+
+ /* Copy periodic overlap in z */
+#pragma omp parallel for num_threads(pme->nthread) schedule(static)
+ for (ix = 0; ix < pme->pmegrid_nx; ix++)
+ {
+ int iy, iz;
+
+ for (iy = 0; iy < pme->pmegrid_ny; iy++)
+ {
+ for (iz = 0; iz < overlap; iz++)
+ {
+ pmegrid[(ix*pny+iy)*pnz+nz+iz] =
+ pmegrid[(ix*pny+iy)*pnz+iz];
+ }
+ }
+ }
+}
+
+
+/* This has to be a macro to enable full compiler optimization with xlC (and probably others too) */
+#define DO_BSPLINE(order) \
+ for (ithx = 0; (ithx < order); ithx++) \
+ { \
+ index_x = (i0+ithx)*pny*pnz; \
+ valx = qn*thx[ithx]; \
+ \
+ for (ithy = 0; (ithy < order); ithy++) \
+ { \
+ valxy = valx*thy[ithy]; \
+ index_xy = index_x+(j0+ithy)*pnz; \
+ \
+ for (ithz = 0; (ithz < order); ithz++) \
+ { \
+ index_xyz = index_xy+(k0+ithz); \
+ grid[index_xyz] += valxy*thz[ithz]; \
+ } \
+ } \
+ }
+
+
+static void spread_q_bsplines_thread(pmegrid_t *pmegrid,
+ pme_atomcomm_t *atc, splinedata_t *spline,
+ pme_spline_work_t *work)
+{
+
+ /* spread charges from home atoms to local grid */
+ real *grid;
+ pme_overlap_t *ol;
+ int b, i, nn, n, ithx, ithy, ithz, i0, j0, k0;
+ int * idxptr;
+ int order, norder, index_x, index_xy, index_xyz;
+ real valx, valxy, qn;
+ real *thx, *thy, *thz;
+ int localsize, bndsize;
+ int pnx, pny, pnz, ndatatot;
+ int offx, offy, offz;
+
++#if defined PME_SIMD4_SPREAD_GATHER && !defined PME_SIMD4_UNALIGNED
++ real thz_buffer[12], *thz_aligned;
++
++ thz_aligned = gmx_simd4_align_real(thz_buffer);
++#endif
++
+ pnx = pmegrid->s[XX];
+ pny = pmegrid->s[YY];
+ pnz = pmegrid->s[ZZ];
+
+ offx = pmegrid->offset[XX];
+ offy = pmegrid->offset[YY];
+ offz = pmegrid->offset[ZZ];
+
+ ndatatot = pnx*pny*pnz;
+ grid = pmegrid->grid;
+ for (i = 0; i < ndatatot; i++)
+ {
+ grid[i] = 0;
+ }
+
+ order = pmegrid->order;
+
+ for (nn = 0; nn < spline->n; nn++)
+ {
+ n = spline->ind[nn];
+ qn = atc->q[n];
+
+ if (qn != 0)
+ {
+ idxptr = atc->idx[n];
+ norder = nn*order;
+
+ i0 = idxptr[XX] - offx;
+ j0 = idxptr[YY] - offy;
+ k0 = idxptr[ZZ] - offz;
+
+ thx = spline->theta[XX] + norder;
+ thy = spline->theta[YY] + norder;
+ thz = spline->theta[ZZ] + norder;
+
+ switch (order)
+ {
+ case 4:
- #define PME_SPREAD_SSE_ALIGNED
++#ifdef PME_SIMD4_SPREAD_GATHER
++#ifdef PME_SIMD4_UNALIGNED
++#define PME_SPREAD_SIMD4_ORDER4
+#else
- #include "pme_sse_single.h"
++#define PME_SPREAD_SIMD4_ALIGNED
+#define PME_ORDER 4
+#endif
- #ifdef PME_SSE_SPREAD_GATHER
- #define PME_SPREAD_SSE_ALIGNED
++#include "pme_simd4.h"
+#else
+ DO_BSPLINE(4);
+#endif
+ break;
+ case 5:
- #include "pme_sse_single.h"
++#ifdef PME_SIMD4_SPREAD_GATHER
++#define PME_SPREAD_SIMD4_ALIGNED
+#define PME_ORDER 5
- #ifdef PME_SSE_SPREAD_GATHER
++#include "pme_simd4.h"
+#else
+ DO_BSPLINE(5);
+#endif
+ break;
+ default:
+ DO_BSPLINE(order);
+ break;
+ }
+ }
+ }
+}
+
+static void set_grid_alignment(int *pmegrid_nz, int pme_order)
+{
- #ifndef PME_SSE_UNALIGNED
++#ifdef PME_SIMD4_SPREAD_GATHER
+ if (pme_order == 5
- #ifdef PME_SSE_SPREAD_GATHER
- #ifndef PME_SSE_UNALIGNED
++#ifndef PME_SIMD4_UNALIGNED
+ || pme_order == 4
+#endif
+ )
+ {
+ /* Round nz up to a multiple of 4 to ensure alignment */
+ *pmegrid_nz = ((*pmegrid_nz + 3) & ~3);
+ }
+#endif
+}
+
+static void set_gridsize_alignment(int gmx_unused *gridsize, int gmx_unused pme_order)
+{
- snew_aligned(grid->grid, gridsize, 16);
++#ifdef PME_SIMD4_SPREAD_GATHER
++#ifndef PME_SIMD4_UNALIGNED
+ if (pme_order == 4)
+ {
+ /* Add extra elements to ensured aligned operations do not go
+ * beyond the allocated grid size.
+ * Note that for pme_order=5, the pme grid z-size alignment
+ * ensures that we will not go beyond the grid size.
+ */
+ *gridsize += 4;
+ }
+#endif
+#endif
+}
+
+static void pmegrid_init(pmegrid_t *grid,
+ int cx, int cy, int cz,
+ int x0, int y0, int z0,
+ int x1, int y1, int z1,
+ gmx_bool set_alignment,
+ int pme_order,
+ real *ptr)
+{
+ int nz, gridsize;
+
+ grid->ci[XX] = cx;
+ grid->ci[YY] = cy;
+ grid->ci[ZZ] = cz;
+ grid->offset[XX] = x0;
+ grid->offset[YY] = y0;
+ grid->offset[ZZ] = z0;
+ grid->n[XX] = x1 - x0 + pme_order - 1;
+ grid->n[YY] = y1 - y0 + pme_order - 1;
+ grid->n[ZZ] = z1 - z0 + pme_order - 1;
+ copy_ivec(grid->n, grid->s);
+
+ nz = grid->s[ZZ];
+ set_grid_alignment(&nz, pme_order);
+ if (set_alignment)
+ {
+ grid->s[ZZ] = nz;
+ }
+ else if (nz != grid->s[ZZ])
+ {
+ gmx_incons("pmegrid_init call with an unaligned z size");
+ }
+
+ grid->order = pme_order;
+ if (ptr == NULL)
+ {
+ gridsize = grid->s[XX]*grid->s[YY]*grid->s[ZZ];
+ set_gridsize_alignment(&gridsize, pme_order);
- 16);
++ snew_aligned(grid->grid, gridsize, SIMD4_ALIGNMENT);
+ }
+ else
+ {
+ grid->grid = ptr;
+ }
+}
+
+static int div_round_up(int enumerator, int denominator)
+{
+ return (enumerator + denominator - 1)/denominator;
+}
+
+static void make_subgrid_division(const ivec n, int ovl, int nthread,
+ ivec nsub)
+{
+ int gsize_opt, gsize;
+ int nsx, nsy, nsz;
+ char *env;
+
+ gsize_opt = -1;
+ for (nsx = 1; nsx <= nthread; nsx++)
+ {
+ if (nthread % nsx == 0)
+ {
+ for (nsy = 1; nsy <= nthread; nsy++)
+ {
+ if (nsx*nsy <= nthread && nthread % (nsx*nsy) == 0)
+ {
+ nsz = nthread/(nsx*nsy);
+
+ /* Determine the number of grid points per thread */
+ gsize =
+ (div_round_up(n[XX], nsx) + ovl)*
+ (div_round_up(n[YY], nsy) + ovl)*
+ (div_round_up(n[ZZ], nsz) + ovl);
+
+ /* Minimize the number of grids points per thread
+ * and, secondarily, the number of cuts in minor dimensions.
+ */
+ if (gsize_opt == -1 ||
+ gsize < gsize_opt ||
+ (gsize == gsize_opt &&
+ (nsz < nsub[ZZ] || (nsz == nsub[ZZ] && nsy < nsub[YY]))))
+ {
+ nsub[XX] = nsx;
+ nsub[YY] = nsy;
+ nsub[ZZ] = nsz;
+ gsize_opt = gsize;
+ }
+ }
+ }
+ }
+ }
+
+ env = getenv("GMX_PME_THREAD_DIVISION");
+ if (env != NULL)
+ {
+ sscanf(env, "%d %d %d", &nsub[XX], &nsub[YY], &nsub[ZZ]);
+ }
+
+ if (nsub[XX]*nsub[YY]*nsub[ZZ] != nthread)
+ {
+ gmx_fatal(FARGS, "PME grid thread division (%d x %d x %d) does not match the total number of threads (%d)", nsub[XX], nsub[YY], nsub[ZZ], nthread);
+ }
+}
+
+static void pmegrids_init(pmegrids_t *grids,
+ int nx, int ny, int nz, int nz_base,
+ int pme_order,
+ gmx_bool bUseThreads,
+ int nthread,
+ int overlap_x,
+ int overlap_y)
+{
+ ivec n, n_base, g0, g1;
+ int t, x, y, z, d, i, tfac;
+ int max_comm_lines = -1;
+
+ n[XX] = nx - (pme_order - 1);
+ n[YY] = ny - (pme_order - 1);
+ n[ZZ] = nz - (pme_order - 1);
+
+ copy_ivec(n, n_base);
+ n_base[ZZ] = nz_base;
+
+ pmegrid_init(&grids->grid, 0, 0, 0, 0, 0, 0, n[XX], n[YY], n[ZZ], FALSE, pme_order,
+ NULL);
+
+ grids->nthread = nthread;
+
+ make_subgrid_division(n_base, pme_order-1, grids->nthread, grids->nc);
+
+ if (bUseThreads)
+ {
+ ivec nst;
+ int gridsize;
+
+ for (d = 0; d < DIM; d++)
+ {
+ nst[d] = div_round_up(n[d], grids->nc[d]) + pme_order - 1;
+ }
+ set_grid_alignment(&nst[ZZ], pme_order);
+
+ if (debug)
+ {
+ fprintf(debug, "pmegrid thread local division: %d x %d x %d\n",
+ grids->nc[XX], grids->nc[YY], grids->nc[ZZ]);
+ fprintf(debug, "pmegrid %d %d %d max thread pmegrid %d %d %d\n",
+ nx, ny, nz,
+ nst[XX], nst[YY], nst[ZZ]);
+ }
+
+ snew(grids->grid_th, grids->nthread);
+ t = 0;
+ gridsize = nst[XX]*nst[YY]*nst[ZZ];
+ set_gridsize_alignment(&gridsize, pme_order);
+ snew_aligned(grids->grid_all,
+ grids->nthread*gridsize+(grids->nthread+1)*GMX_CACHE_SEP,
- #define ALIGN_HERE GMX_SIMD_WIDTH_HERE
++ SIMD4_ALIGNMENT);
+
+ for (x = 0; x < grids->nc[XX]; x++)
+ {
+ for (y = 0; y < grids->nc[YY]; y++)
+ {
+ for (z = 0; z < grids->nc[ZZ]; z++)
+ {
+ pmegrid_init(&grids->grid_th[t],
+ x, y, z,
+ (n[XX]*(x ))/grids->nc[XX],
+ (n[YY]*(y ))/grids->nc[YY],
+ (n[ZZ]*(z ))/grids->nc[ZZ],
+ (n[XX]*(x+1))/grids->nc[XX],
+ (n[YY]*(y+1))/grids->nc[YY],
+ (n[ZZ]*(z+1))/grids->nc[ZZ],
+ TRUE,
+ pme_order,
+ grids->grid_all+GMX_CACHE_SEP+t*(gridsize+GMX_CACHE_SEP));
+ t++;
+ }
+ }
+ }
+ }
+ else
+ {
+ grids->grid_th = NULL;
+ }
+
+ snew(grids->g2t, DIM);
+ tfac = 1;
+ for (d = DIM-1; d >= 0; d--)
+ {
+ snew(grids->g2t[d], n[d]);
+ t = 0;
+ for (i = 0; i < n[d]; i++)
+ {
+ /* The second check should match the parameters
+ * of the pmegrid_init call above.
+ */
+ while (t + 1 < grids->nc[d] && i >= (n[d]*(t+1))/grids->nc[d])
+ {
+ t++;
+ }
+ grids->g2t[d][i] = t*tfac;
+ }
+
+ tfac *= grids->nc[d];
+
+ switch (d)
+ {
+ case XX: max_comm_lines = overlap_x; break;
+ case YY: max_comm_lines = overlap_y; break;
+ case ZZ: max_comm_lines = pme_order - 1; break;
+ }
+ grids->nthread_comm[d] = 0;
+ while ((n[d]*grids->nthread_comm[d])/grids->nc[d] < max_comm_lines &&
+ grids->nthread_comm[d] < grids->nc[d])
+ {
+ grids->nthread_comm[d]++;
+ }
+ if (debug != NULL)
+ {
+ fprintf(debug, "pmegrid thread grid communication range in %c: %d\n",
+ 'x'+d, grids->nthread_comm[d]);
+ }
+ /* It should be possible to make grids->nthread_comm[d]==grids->nc[d]
+ * work, but this is not a problematic restriction.
+ */
+ if (grids->nc[d] > 1 && grids->nthread_comm[d] > grids->nc[d])
+ {
+ gmx_fatal(FARGS, "Too many threads for PME (%d) compared to the number of grid lines, reduce the number of threads doing PME", grids->nthread);
+ }
+ }
+}
+
+
+static void pmegrids_destroy(pmegrids_t *grids)
+{
+ int t;
+
+ if (grids->grid.grid != NULL)
+ {
+ sfree(grids->grid.grid);
+
+ if (grids->nthread > 0)
+ {
+ for (t = 0; t < grids->nthread; t++)
+ {
+ sfree(grids->grid_th[t].grid);
+ }
+ sfree(grids->grid_th);
+ }
+ }
+}
+
+
+static void realloc_work(pme_work_t *work, int nkx)
+{
++ int simd_width;
++
+ if (nkx > work->nalloc)
+ {
+ work->nalloc = nkx;
+ srenew(work->mhx, work->nalloc);
+ srenew(work->mhy, work->nalloc);
+ srenew(work->mhz, work->nalloc);
+ srenew(work->m2, work->nalloc);
+ /* Allocate an aligned pointer for SIMD operations, including extra
+ * elements at the end for padding.
+ */
+#ifdef PME_SIMD
- /* We can use any alignment, apart from 0, so we use 4 */
- #define ALIGN_HERE 4
++ simd_width = GMX_SIMD_WIDTH_HERE;
+#else
- snew_aligned(work->denom, work->nalloc+ALIGN_HERE, ALIGN_HERE*sizeof(real));
- snew_aligned(work->tmp1, work->nalloc+ALIGN_HERE, ALIGN_HERE*sizeof(real));
- snew_aligned(work->eterm, work->nalloc+ALIGN_HERE, ALIGN_HERE*sizeof(real));
++ /* We can use any alignment, apart from 0, so we use 4 */
++ simd_width = 4;
+#endif
+ sfree_aligned(work->denom);
+ sfree_aligned(work->tmp1);
+ sfree_aligned(work->eterm);
- f_simd = gmx_load1_pr(&f);
++ snew_aligned(work->denom, work->nalloc+simd_width, simd_width*sizeof(real));
++ snew_aligned(work->tmp1, work->nalloc+simd_width, simd_width*sizeof(real));
++ snew_aligned(work->eterm, work->nalloc+simd_width, simd_width*sizeof(real));
+ srenew(work->m2inv, work->nalloc);
+ }
+}
+
+
+static void free_work(pme_work_t *work)
+{
+ sfree(work->mhx);
+ sfree(work->mhy);
+ sfree(work->mhz);
+ sfree(work->m2);
+ sfree_aligned(work->denom);
+ sfree_aligned(work->tmp1);
+ sfree_aligned(work->eterm);
+ sfree(work->m2inv);
+}
+
+
+#ifdef PME_SIMD
+/* Calculate exponentials through SIMD */
+inline static void calc_exponentials(int start, int end, real f, real *d_aligned, real *r_aligned, real *e_aligned)
+{
+ {
+ const gmx_mm_pr two = gmx_set1_pr(2.0);
+ gmx_mm_pr f_simd;
+ gmx_mm_pr lu;
+ gmx_mm_pr tmp_d1, d_inv, tmp_r, tmp_e;
+ int kx;
- #ifdef PME_SSE_SPREAD_GATHER
- #ifdef PME_SSE_UNALIGNED
- #define PME_GATHER_F_SSE_ORDER4
++ f_simd = gmx_set1_pr(f);
+ for (kx = 0; kx < end; kx += GMX_SIMD_WIDTH_HERE)
+ {
+ tmp_d1 = gmx_load_pr(d_aligned+kx);
+ d_inv = gmx_inv_pr(tmp_d1);
+ tmp_r = gmx_load_pr(r_aligned+kx);
+ tmp_r = gmx_exp_pr(tmp_r);
+ tmp_e = gmx_mul_pr(f_simd, d_inv);
+ tmp_e = gmx_mul_pr(tmp_e, tmp_r);
+ gmx_store_pr(e_aligned+kx, tmp_e);
+ }
+ }
+}
+#else
+inline static void calc_exponentials(int start, int end, real f, real *d, real *r, real *e)
+{
+ int kx;
+ for (kx = start; kx < end; kx++)
+ {
+ d[kx] = 1.0/d[kx];
+ }
+ for (kx = start; kx < end; kx++)
+ {
+ r[kx] = exp(r[kx]);
+ }
+ for (kx = start; kx < end; kx++)
+ {
+ e[kx] = f*r[kx]*d[kx];
+ }
+}
+#endif
+
+
+static int solve_pme_yzx(gmx_pme_t pme, t_complex *grid,
+ real ewaldcoeff, real vol,
+ gmx_bool bEnerVir,
+ int nthread, int thread)
+{
+ /* do recip sum over local cells in grid */
+ /* y major, z middle, x minor or continuous */
+ t_complex *p0;
+ int kx, ky, kz, maxkx, maxky, maxkz;
+ int nx, ny, nz, iyz0, iyz1, iyz, iy, iz, kxstart, kxend;
+ real mx, my, mz;
+ real factor = M_PI*M_PI/(ewaldcoeff*ewaldcoeff);
+ real ets2, struct2, vfactor, ets2vf;
+ real d1, d2, energy = 0;
+ real by, bz;
+ real virxx = 0, virxy = 0, virxz = 0, viryy = 0, viryz = 0, virzz = 0;
+ real rxx, ryx, ryy, rzx, rzy, rzz;
+ pme_work_t *work;
+ real *mhx, *mhy, *mhz, *m2, *denom, *tmp1, *eterm, *m2inv;
+ real mhxk, mhyk, mhzk, m2k;
+ real corner_fac;
+ ivec complex_order;
+ ivec local_ndata, local_offset, local_size;
+ real elfac;
+
+ elfac = ONE_4PI_EPS0/pme->epsilon_r;
+
+ nx = pme->nkx;
+ ny = pme->nky;
+ nz = pme->nkz;
+
+ /* Dimensions should be identical for A/B grid, so we just use A here */
+ gmx_parallel_3dfft_complex_limits(pme->pfft_setupA,
+ complex_order,
+ local_ndata,
+ local_offset,
+ local_size);
+
+ rxx = pme->recipbox[XX][XX];
+ ryx = pme->recipbox[YY][XX];
+ ryy = pme->recipbox[YY][YY];
+ rzx = pme->recipbox[ZZ][XX];
+ rzy = pme->recipbox[ZZ][YY];
+ rzz = pme->recipbox[ZZ][ZZ];
+
+ maxkx = (nx+1)/2;
+ maxky = (ny+1)/2;
+ maxkz = nz/2+1;
+
+ work = &pme->work[thread];
+ mhx = work->mhx;
+ mhy = work->mhy;
+ mhz = work->mhz;
+ m2 = work->m2;
+ denom = work->denom;
+ tmp1 = work->tmp1;
+ eterm = work->eterm;
+ m2inv = work->m2inv;
+
+ iyz0 = local_ndata[YY]*local_ndata[ZZ]* thread /nthread;
+ iyz1 = local_ndata[YY]*local_ndata[ZZ]*(thread+1)/nthread;
+
+ for (iyz = iyz0; iyz < iyz1; iyz++)
+ {
+ iy = iyz/local_ndata[ZZ];
+ iz = iyz - iy*local_ndata[ZZ];
+
+ ky = iy + local_offset[YY];
+
+ if (ky < maxky)
+ {
+ my = ky;
+ }
+ else
+ {
+ my = (ky - ny);
+ }
+
+ by = M_PI*vol*pme->bsp_mod[YY][ky];
+
+ kz = iz + local_offset[ZZ];
+
+ mz = kz;
+
+ bz = pme->bsp_mod[ZZ][kz];
+
+ /* 0.5 correction for corner points */
+ corner_fac = 1;
+ if (kz == 0 || kz == (nz+1)/2)
+ {
+ corner_fac = 0.5;
+ }
+
+ p0 = grid + iy*local_size[ZZ]*local_size[XX] + iz*local_size[XX];
+
+ /* We should skip the k-space point (0,0,0) */
+ if (local_offset[XX] > 0 || ky > 0 || kz > 0)
+ {
+ kxstart = local_offset[XX];
+ }
+ else
+ {
+ kxstart = local_offset[XX] + 1;
+ p0++;
+ }
+ kxend = local_offset[XX] + local_ndata[XX];
+
+ if (bEnerVir)
+ {
+ /* More expensive inner loop, especially because of the storage
+ * of the mh elements in array's.
+ * Because x is the minor grid index, all mh elements
+ * depend on kx for triclinic unit cells.
+ */
+
+ /* Two explicit loops to avoid a conditional inside the loop */
+ for (kx = kxstart; kx < maxkx; kx++)
+ {
+ mx = kx;
+
+ mhxk = mx * rxx;
+ mhyk = mx * ryx + my * ryy;
+ mhzk = mx * rzx + my * rzy + mz * rzz;
+ m2k = mhxk*mhxk + mhyk*mhyk + mhzk*mhzk;
+ mhx[kx] = mhxk;
+ mhy[kx] = mhyk;
+ mhz[kx] = mhzk;
+ m2[kx] = m2k;
+ denom[kx] = m2k*bz*by*pme->bsp_mod[XX][kx];
+ tmp1[kx] = -factor*m2k;
+ }
+
+ for (kx = maxkx; kx < kxend; kx++)
+ {
+ mx = (kx - nx);
+
+ mhxk = mx * rxx;
+ mhyk = mx * ryx + my * ryy;
+ mhzk = mx * rzx + my * rzy + mz * rzz;
+ m2k = mhxk*mhxk + mhyk*mhyk + mhzk*mhzk;
+ mhx[kx] = mhxk;
+ mhy[kx] = mhyk;
+ mhz[kx] = mhzk;
+ m2[kx] = m2k;
+ denom[kx] = m2k*bz*by*pme->bsp_mod[XX][kx];
+ tmp1[kx] = -factor*m2k;
+ }
+
+ for (kx = kxstart; kx < kxend; kx++)
+ {
+ m2inv[kx] = 1.0/m2[kx];
+ }
+
+ calc_exponentials(kxstart, kxend, elfac, denom, tmp1, eterm);
+
+ for (kx = kxstart; kx < kxend; kx++, p0++)
+ {
+ d1 = p0->re;
+ d2 = p0->im;
+
+ p0->re = d1*eterm[kx];
+ p0->im = d2*eterm[kx];
+
+ struct2 = 2.0*(d1*d1+d2*d2);
+
+ tmp1[kx] = eterm[kx]*struct2;
+ }
+
+ for (kx = kxstart; kx < kxend; kx++)
+ {
+ ets2 = corner_fac*tmp1[kx];
+ vfactor = (factor*m2[kx] + 1.0)*2.0*m2inv[kx];
+ energy += ets2;
+
+ ets2vf = ets2*vfactor;
+ virxx += ets2vf*mhx[kx]*mhx[kx] - ets2;
+ virxy += ets2vf*mhx[kx]*mhy[kx];
+ virxz += ets2vf*mhx[kx]*mhz[kx];
+ viryy += ets2vf*mhy[kx]*mhy[kx] - ets2;
+ viryz += ets2vf*mhy[kx]*mhz[kx];
+ virzz += ets2vf*mhz[kx]*mhz[kx] - ets2;
+ }
+ }
+ else
+ {
+ /* We don't need to calculate the energy and the virial.
+ * In this case the triclinic overhead is small.
+ */
+
+ /* Two explicit loops to avoid a conditional inside the loop */
+
+ for (kx = kxstart; kx < maxkx; kx++)
+ {
+ mx = kx;
+
+ mhxk = mx * rxx;
+ mhyk = mx * ryx + my * ryy;
+ mhzk = mx * rzx + my * rzy + mz * rzz;
+ m2k = mhxk*mhxk + mhyk*mhyk + mhzk*mhzk;
+ denom[kx] = m2k*bz*by*pme->bsp_mod[XX][kx];
+ tmp1[kx] = -factor*m2k;
+ }
+
+ for (kx = maxkx; kx < kxend; kx++)
+ {
+ mx = (kx - nx);
+
+ mhxk = mx * rxx;
+ mhyk = mx * ryx + my * ryy;
+ mhzk = mx * rzx + my * rzy + mz * rzz;
+ m2k = mhxk*mhxk + mhyk*mhyk + mhzk*mhzk;
+ denom[kx] = m2k*bz*by*pme->bsp_mod[XX][kx];
+ tmp1[kx] = -factor*m2k;
+ }
+
+ calc_exponentials(kxstart, kxend, elfac, denom, tmp1, eterm);
+
+ for (kx = kxstart; kx < kxend; kx++, p0++)
+ {
+ d1 = p0->re;
+ d2 = p0->im;
+
+ p0->re = d1*eterm[kx];
+ p0->im = d2*eterm[kx];
+ }
+ }
+ }
+
+ if (bEnerVir)
+ {
+ /* Update virial with local values.
+ * The virial is symmetric by definition.
+ * this virial seems ok for isotropic scaling, but I'm
+ * experiencing problems on semiisotropic membranes.
+ * IS THAT COMMENT STILL VALID??? (DvdS, 2001/02/07).
+ */
+ work->vir[XX][XX] = 0.25*virxx;
+ work->vir[YY][YY] = 0.25*viryy;
+ work->vir[ZZ][ZZ] = 0.25*virzz;
+ work->vir[XX][YY] = work->vir[YY][XX] = 0.25*virxy;
+ work->vir[XX][ZZ] = work->vir[ZZ][XX] = 0.25*virxz;
+ work->vir[YY][ZZ] = work->vir[ZZ][YY] = 0.25*viryz;
+
+ /* This energy should be corrected for a charged system */
+ work->energy = 0.5*energy;
+ }
+
+ /* Return the loop count */
+ return local_ndata[YY]*local_ndata[XX];
+}
+
+static void get_pme_ener_vir(const gmx_pme_t pme, int nthread,
+ real *mesh_energy, matrix vir)
+{
+ /* This function sums output over threads
+ * and should therefore only be called after thread synchronization.
+ */
+ int thread;
+
+ *mesh_energy = pme->work[0].energy;
+ copy_mat(pme->work[0].vir, vir);
+
+ for (thread = 1; thread < nthread; thread++)
+ {
+ *mesh_energy += pme->work[thread].energy;
+ m_add(vir, pme->work[thread].vir, vir);
+ }
+}
+
+#define DO_FSPLINE(order) \
+ for (ithx = 0; (ithx < order); ithx++) \
+ { \
+ index_x = (i0+ithx)*pny*pnz; \
+ tx = thx[ithx]; \
+ dx = dthx[ithx]; \
+ \
+ for (ithy = 0; (ithy < order); ithy++) \
+ { \
+ index_xy = index_x+(j0+ithy)*pnz; \
+ ty = thy[ithy]; \
+ dy = dthy[ithy]; \
+ fxy1 = fz1 = 0; \
+ \
+ for (ithz = 0; (ithz < order); ithz++) \
+ { \
+ gval = grid[index_xy+(k0+ithz)]; \
+ fxy1 += thz[ithz]*gval; \
+ fz1 += dthz[ithz]*gval; \
+ } \
+ fx += dx*ty*fxy1; \
+ fy += tx*dy*fxy1; \
+ fz += tx*ty*fz1; \
+ } \
+ }
+
+
+static void gather_f_bsplines(gmx_pme_t pme, real *grid,
+ gmx_bool bClearF, pme_atomcomm_t *atc,
+ splinedata_t *spline,
+ real scale)
+{
+ /* sum forces for local particles */
+ int nn, n, ithx, ithy, ithz, i0, j0, k0;
+ int index_x, index_xy;
+ int nx, ny, nz, pnx, pny, pnz;
+ int * idxptr;
+ real tx, ty, dx, dy, qn;
+ real fx, fy, fz, gval;
+ real fxy1, fz1;
+ real *thx, *thy, *thz, *dthx, *dthy, *dthz;
+ int norder;
+ real rxx, ryx, ryy, rzx, rzy, rzz;
+ int order;
+
+ pme_spline_work_t *work;
+
++#if defined PME_SIMD4_SPREAD_GATHER && !defined PME_SIMD4_UNALIGNED
++ real thz_buffer[12], *thz_aligned;
++ real dthz_buffer[12], *dthz_aligned;
++
++ thz_aligned = gmx_simd4_align_real(thz_buffer);
++ dthz_aligned = gmx_simd4_align_real(dthz_buffer);
++#endif
++
+ work = pme->spline_work;
+
+ order = pme->pme_order;
+ thx = spline->theta[XX];
+ thy = spline->theta[YY];
+ thz = spline->theta[ZZ];
+ dthx = spline->dtheta[XX];
+ dthy = spline->dtheta[YY];
+ dthz = spline->dtheta[ZZ];
+ nx = pme->nkx;
+ ny = pme->nky;
+ nz = pme->nkz;
+ pnx = pme->pmegrid_nx;
+ pny = pme->pmegrid_ny;
+ pnz = pme->pmegrid_nz;
+
+ rxx = pme->recipbox[XX][XX];
+ ryx = pme->recipbox[YY][XX];
+ ryy = pme->recipbox[YY][YY];
+ rzx = pme->recipbox[ZZ][XX];
+ rzy = pme->recipbox[ZZ][YY];
+ rzz = pme->recipbox[ZZ][ZZ];
+
+ for (nn = 0; nn < spline->n; nn++)
+ {
+ n = spline->ind[nn];
+ qn = scale*atc->q[n];
+
+ if (bClearF)
+ {
+ atc->f[n][XX] = 0;
+ atc->f[n][YY] = 0;
+ atc->f[n][ZZ] = 0;
+ }
+ if (qn != 0)
+ {
+ fx = 0;
+ fy = 0;
+ fz = 0;
+ idxptr = atc->idx[n];
+ norder = nn*order;
+
+ i0 = idxptr[XX];
+ j0 = idxptr[YY];
+ k0 = idxptr[ZZ];
+
+ /* Pointer arithmetic alert, next six statements */
+ thx = spline->theta[XX] + norder;
+ thy = spline->theta[YY] + norder;
+ thz = spline->theta[ZZ] + norder;
+ dthx = spline->dtheta[XX] + norder;
+ dthy = spline->dtheta[YY] + norder;
+ dthz = spline->dtheta[ZZ] + norder;
+
+ switch (order)
+ {
+ case 4:
- #define PME_GATHER_F_SSE_ALIGNED
++#ifdef PME_SIMD4_SPREAD_GATHER
++#ifdef PME_SIMD4_UNALIGNED
++#define PME_GATHER_F_SIMD4_ORDER4
+#else
- #include "pme_sse_single.h"
++#define PME_GATHER_F_SIMD4_ALIGNED
+#define PME_ORDER 4
+#endif
- #ifdef PME_SSE_SPREAD_GATHER
- #define PME_GATHER_F_SSE_ALIGNED
++#include "pme_simd4.h"
+#else
+ DO_FSPLINE(4);
+#endif
+ break;
+ case 5:
- #include "pme_sse_single.h"
++#ifdef PME_SIMD4_SPREAD_GATHER
++#define PME_GATHER_F_SIMD4_ALIGNED
+#define PME_ORDER 5
- #ifdef PME_SSE_SPREAD_GATHER
- float tmp[8];
- __m128 zero_SSE;
- int of, i;
++#include "pme_simd4.h"
+#else
+ DO_FSPLINE(5);
+#endif
+ break;
+ default:
+ DO_FSPLINE(order);
+ break;
+ }
+
+ atc->f[n][XX] += -qn*( fx*nx*rxx );
+ atc->f[n][YY] += -qn*( fx*nx*ryx + fy*ny*ryy );
+ atc->f[n][ZZ] += -qn*( fx*nx*rzx + fy*ny*rzy + fz*nz*rzz );
+ }
+ }
+ /* Since the energy and not forces are interpolated
+ * the net force might not be exactly zero.
+ * This can be solved by also interpolating F, but
+ * that comes at a cost.
+ * A better hack is to remove the net force every
+ * step, but that must be done at a higher level
+ * since this routine doesn't see all atoms if running
+ * in parallel. Don't know how important it is? EL 990726
+ */
+}
+
+
+static real gather_energy_bsplines(gmx_pme_t pme, real *grid,
+ pme_atomcomm_t *atc)
+{
+ splinedata_t *spline;
+ int n, ithx, ithy, ithz, i0, j0, k0;
+ int index_x, index_xy;
+ int * idxptr;
+ real energy, pot, tx, ty, qn, gval;
+ real *thx, *thy, *thz;
+ int norder;
+ int order;
+
+ spline = &atc->spline[0];
+
+ order = pme->pme_order;
+
+ energy = 0;
+ for (n = 0; (n < atc->n); n++)
+ {
+ qn = atc->q[n];
+
+ if (qn != 0)
+ {
+ idxptr = atc->idx[n];
+ norder = n*order;
+
+ i0 = idxptr[XX];
+ j0 = idxptr[YY];
+ k0 = idxptr[ZZ];
+
+ /* Pointer arithmetic alert, next three statements */
+ thx = spline->theta[XX] + norder;
+ thy = spline->theta[YY] + norder;
+ thz = spline->theta[ZZ] + norder;
+
+ pot = 0;
+ for (ithx = 0; (ithx < order); ithx++)
+ {
+ index_x = (i0+ithx)*pme->pmegrid_ny*pme->pmegrid_nz;
+ tx = thx[ithx];
+
+ for (ithy = 0; (ithy < order); ithy++)
+ {
+ index_xy = index_x+(j0+ithy)*pme->pmegrid_nz;
+ ty = thy[ithy];
+
+ for (ithz = 0; (ithz < order); ithz++)
+ {
+ gval = grid[index_xy+(k0+ithz)];
+ pot += tx*ty*thz[ithz]*gval;
+ }
+
+ }
+ }
+
+ energy += pot*qn;
+ }
+ }
+
+ return energy;
+}
+
+/* Macro to force loop unrolling by fixing order.
+ * This gives a significant performance gain.
+ */
+#define CALC_SPLINE(order) \
+ { \
+ int j, k, l; \
+ real dr, div; \
+ real data[PME_ORDER_MAX]; \
+ real ddata[PME_ORDER_MAX]; \
+ \
+ for (j = 0; (j < DIM); j++) \
+ { \
+ dr = xptr[j]; \
+ \
+ /* dr is relative offset from lower cell limit */ \
+ data[order-1] = 0; \
+ data[1] = dr; \
+ data[0] = 1 - dr; \
+ \
+ for (k = 3; (k < order); k++) \
+ { \
+ div = 1.0/(k - 1.0); \
+ data[k-1] = div*dr*data[k-2]; \
+ for (l = 1; (l < (k-1)); l++) \
+ { \
+ data[k-l-1] = div*((dr+l)*data[k-l-2]+(k-l-dr)* \
+ data[k-l-1]); \
+ } \
+ data[0] = div*(1-dr)*data[0]; \
+ } \
+ /* differentiate */ \
+ ddata[0] = -data[0]; \
+ for (k = 1; (k < order); k++) \
+ { \
+ ddata[k] = data[k-1] - data[k]; \
+ } \
+ \
+ div = 1.0/(order - 1); \
+ data[order-1] = div*dr*data[order-2]; \
+ for (l = 1; (l < (order-1)); l++) \
+ { \
+ data[order-l-1] = div*((dr+l)*data[order-l-2]+ \
+ (order-l-dr)*data[order-l-1]); \
+ } \
+ data[0] = div*(1 - dr)*data[0]; \
+ \
+ for (k = 0; k < order; k++) \
+ { \
+ theta[j][i*order+k] = data[k]; \
+ dtheta[j][i*order+k] = ddata[k]; \
+ } \
+ } \
+ }
+
+void make_bsplines(splinevec theta, splinevec dtheta, int order,
+ rvec fractx[], int nr, int ind[], real charge[],
+ gmx_bool bFreeEnergy)
+{
+ /* construct splines for local atoms */
+ int i, ii;
+ real *xptr;
+
+ for (i = 0; i < nr; i++)
+ {
+ /* With free energy we do not use the charge check.
+ * In most cases this will be more efficient than calling make_bsplines
+ * twice, since usually more than half the particles have charges.
+ */
+ ii = ind[i];
+ if (bFreeEnergy || charge[ii] != 0.0)
+ {
+ xptr = fractx[ii];
+ switch (order)
+ {
+ case 4: CALC_SPLINE(4); break;
+ case 5: CALC_SPLINE(5); break;
+ default: CALC_SPLINE(order); break;
+ }
+ }
+ }
+}
+
+
+void make_dft_mod(real *mod, real *data, int ndata)
+{
+ int i, j;
+ real sc, ss, arg;
+
+ for (i = 0; i < ndata; i++)
+ {
+ sc = ss = 0;
+ for (j = 0; j < ndata; j++)
+ {
+ arg = (2.0*M_PI*i*j)/ndata;
+ sc += data[j]*cos(arg);
+ ss += data[j]*sin(arg);
+ }
+ mod[i] = sc*sc+ss*ss;
+ }
+ for (i = 0; i < ndata; i++)
+ {
+ if (mod[i] < 1e-7)
+ {
+ mod[i] = (mod[i-1]+mod[i+1])*0.5;
+ }
+ }
+}
+
+
+static void make_bspline_moduli(splinevec bsp_mod,
+ int nx, int ny, int nz, int order)
+{
+ int nmax = max(nx, max(ny, nz));
+ real *data, *ddata, *bsp_data;
+ int i, k, l;
+ real div;
+
+ snew(data, order);
+ snew(ddata, order);
+ snew(bsp_data, nmax);
+
+ data[order-1] = 0;
+ data[1] = 0;
+ data[0] = 1;
+
+ for (k = 3; k < order; k++)
+ {
+ div = 1.0/(k-1.0);
+ data[k-1] = 0;
+ for (l = 1; l < (k-1); l++)
+ {
+ data[k-l-1] = div*(l*data[k-l-2]+(k-l)*data[k-l-1]);
+ }
+ data[0] = div*data[0];
+ }
+ /* differentiate */
+ ddata[0] = -data[0];
+ for (k = 1; k < order; k++)
+ {
+ ddata[k] = data[k-1]-data[k];
+ }
+ div = 1.0/(order-1);
+ data[order-1] = 0;
+ for (l = 1; l < (order-1); l++)
+ {
+ data[order-l-1] = div*(l*data[order-l-2]+(order-l)*data[order-l-1]);
+ }
+ data[0] = div*data[0];
+
+ for (i = 0; i < nmax; i++)
+ {
+ bsp_data[i] = 0;
+ }
+ for (i = 1; i <= order; i++)
+ {
+ bsp_data[i] = data[i-1];
+ }
+
+ make_dft_mod(bsp_mod[XX], bsp_data, nx);
+ make_dft_mod(bsp_mod[YY], bsp_data, ny);
+ make_dft_mod(bsp_mod[ZZ], bsp_data, nz);
+
+ sfree(data);
+ sfree(ddata);
+ sfree(bsp_data);
+}
+
+
+/* Return the P3M optimal influence function */
+static double do_p3m_influence(double z, int order)
+{
+ double z2, z4;
+
+ z2 = z*z;
+ z4 = z2*z2;
+
+ /* The formula and most constants can be found in:
+ * Ballenegger et al., JCTC 8, 936 (2012)
+ */
+ switch (order)
+ {
+ case 2:
+ return 1.0 - 2.0*z2/3.0;
+ break;
+ case 3:
+ return 1.0 - z2 + 2.0*z4/15.0;
+ break;
+ case 4:
+ return 1.0 - 4.0*z2/3.0 + 2.0*z4/5.0 + 4.0*z2*z4/315.0;
+ break;
+ case 5:
+ return 1.0 - 5.0*z2/3.0 + 7.0*z4/9.0 - 17.0*z2*z4/189.0 + 2.0*z4*z4/2835.0;
+ break;
+ case 6:
+ return 1.0 - 2.0*z2 + 19.0*z4/15.0 - 256.0*z2*z4/945.0 + 62.0*z4*z4/4725.0 + 4.0*z2*z4*z4/155925.0;
+ break;
+ case 7:
+ return 1.0 - 7.0*z2/3.0 + 28.0*z4/15.0 - 16.0*z2*z4/27.0 + 26.0*z4*z4/405.0 - 2.0*z2*z4*z4/1485.0 + 4.0*z4*z4*z4/6081075.0;
+ case 8:
+ return 1.0 - 8.0*z2/3.0 + 116.0*z4/45.0 - 344.0*z2*z4/315.0 + 914.0*z4*z4/4725.0 - 248.0*z4*z4*z2/22275.0 + 21844.0*z4*z4*z4/212837625.0 - 8.0*z4*z4*z4*z2/638512875.0;
+ break;
+ }
+
+ return 0.0;
+}
+
+/* Calculate the P3M B-spline moduli for one dimension */
+static void make_p3m_bspline_moduli_dim(real *bsp_mod, int n, int order)
+{
+ double zarg, zai, sinzai, infl;
+ int maxk, i;
+
+ if (order > 8)
+ {
+ gmx_fatal(FARGS, "The current P3M code only supports orders up to 8");
+ }
+
+ zarg = M_PI/n;
+
+ maxk = (n + 1)/2;
+
+ for (i = -maxk; i < 0; i++)
+ {
+ zai = zarg*i;
+ sinzai = sin(zai);
+ infl = do_p3m_influence(sinzai, order);
+ bsp_mod[n+i] = infl*infl*pow(sinzai/zai, -2.0*order);
+ }
+ bsp_mod[0] = 1.0;
+ for (i = 1; i < maxk; i++)
+ {
+ zai = zarg*i;
+ sinzai = sin(zai);
+ infl = do_p3m_influence(sinzai, order);
+ bsp_mod[i] = infl*infl*pow(sinzai/zai, -2.0*order);
+ }
+}
+
+/* Calculate the P3M B-spline moduli */
+static void make_p3m_bspline_moduli(splinevec bsp_mod,
+ int nx, int ny, int nz, int order)
+{
+ make_p3m_bspline_moduli_dim(bsp_mod[XX], nx, order);
+ make_p3m_bspline_moduli_dim(bsp_mod[YY], ny, order);
+ make_p3m_bspline_moduli_dim(bsp_mod[ZZ], nz, order);
+}
+
+
+static void setup_coordinate_communication(pme_atomcomm_t *atc)
+{
+ int nslab, n, i;
+ int fw, bw;
+
+ nslab = atc->nslab;
+
+ n = 0;
+ for (i = 1; i <= nslab/2; i++)
+ {
+ fw = (atc->nodeid + i) % nslab;
+ bw = (atc->nodeid - i + nslab) % nslab;
+ if (n < nslab - 1)
+ {
+ atc->node_dest[n] = fw;
+ atc->node_src[n] = bw;
+ n++;
+ }
+ if (n < nslab - 1)
+ {
+ atc->node_dest[n] = bw;
+ atc->node_src[n] = fw;
+ n++;
+ }
+ }
+}
+
+int gmx_pme_destroy(FILE *log, gmx_pme_t *pmedata)
+{
+ int thread;
+
+ if (NULL != log)
+ {
+ fprintf(log, "Destroying PME data structures.\n");
+ }
+
+ sfree((*pmedata)->nnx);
+ sfree((*pmedata)->nny);
+ sfree((*pmedata)->nnz);
+
+ pmegrids_destroy(&(*pmedata)->pmegridA);
+
+ sfree((*pmedata)->fftgridA);
+ sfree((*pmedata)->cfftgridA);
+ gmx_parallel_3dfft_destroy((*pmedata)->pfft_setupA);
+
+ if ((*pmedata)->pmegridB.grid.grid != NULL)
+ {
+ pmegrids_destroy(&(*pmedata)->pmegridB);
+ sfree((*pmedata)->fftgridB);
+ sfree((*pmedata)->cfftgridB);
+ gmx_parallel_3dfft_destroy((*pmedata)->pfft_setupB);
+ }
+ for (thread = 0; thread < (*pmedata)->nthread; thread++)
+ {
+ free_work(&(*pmedata)->work[thread]);
+ }
+ sfree((*pmedata)->work);
+
+ sfree(*pmedata);
+ *pmedata = NULL;
+
+ return 0;
+}
+
+static int mult_up(int n, int f)
+{
+ return ((n + f - 1)/f)*f;
+}
+
+
+static double pme_load_imbalance(gmx_pme_t pme)
+{
+ int nma, nmi;
+ double n1, n2, n3;
+
+ nma = pme->nnodes_major;
+ nmi = pme->nnodes_minor;
+
+ n1 = mult_up(pme->nkx, nma)*mult_up(pme->nky, nmi)*pme->nkz;
+ n2 = mult_up(pme->nkx, nma)*mult_up(pme->nkz, nmi)*pme->nky;
+ n3 = mult_up(pme->nky, nma)*mult_up(pme->nkz, nmi)*pme->nkx;
+
+ /* pme_solve is roughly double the cost of an fft */
+
+ return (n1 + n2 + 3*n3)/(double)(6*pme->nkx*pme->nky*pme->nkz);
+}
+
+static void init_atomcomm(gmx_pme_t pme, pme_atomcomm_t *atc,
+ int dimind, gmx_bool bSpread)
+{
+ int nk, k, s, thread;
+
+ atc->dimind = dimind;
+ atc->nslab = 1;
+ atc->nodeid = 0;
+ atc->pd_nalloc = 0;
+#ifdef GMX_MPI
+ if (pme->nnodes > 1)
+ {
+ atc->mpi_comm = pme->mpi_comm_d[dimind];
+ MPI_Comm_size(atc->mpi_comm, &atc->nslab);
+ MPI_Comm_rank(atc->mpi_comm, &atc->nodeid);
+ }
+ if (debug)
+ {
+ fprintf(debug, "For PME atom communication in dimind %d: nslab %d rank %d\n", atc->dimind, atc->nslab, atc->nodeid);
+ }
+#endif
+
+ atc->bSpread = bSpread;
+ atc->pme_order = pme->pme_order;
+
+ if (atc->nslab > 1)
+ {
+ /* These three allocations are not required for particle decomp. */
+ snew(atc->node_dest, atc->nslab);
+ snew(atc->node_src, atc->nslab);
+ setup_coordinate_communication(atc);
+
+ snew(atc->count_thread, pme->nthread);
+ for (thread = 0; thread < pme->nthread; thread++)
+ {
+ snew(atc->count_thread[thread], atc->nslab);
+ }
+ atc->count = atc->count_thread[0];
+ snew(atc->rcount, atc->nslab);
+ snew(atc->buf_index, atc->nslab);
+ }
+
+ atc->nthread = pme->nthread;
+ if (atc->nthread > 1)
+ {
+ snew(atc->thread_plist, atc->nthread);
+ }
+ snew(atc->spline, atc->nthread);
+ for (thread = 0; thread < atc->nthread; thread++)
+ {
+ if (atc->nthread > 1)
+ {
+ snew(atc->thread_plist[thread].n, atc->nthread+2*GMX_CACHE_SEP);
+ atc->thread_plist[thread].n += GMX_CACHE_SEP;
+ }
+ snew(atc->spline[thread].thread_one, pme->nthread);
+ atc->spline[thread].thread_one[thread] = 1;
+ }
+}
+
+static void
+init_overlap_comm(pme_overlap_t * ol,
+ int norder,
+#ifdef GMX_MPI
+ MPI_Comm comm,
+#endif
+ int nnodes,
+ int nodeid,
+ int ndata,
+ int commplainsize)
+{
+ int lbnd, rbnd, maxlr, b, i;
+ int exten;
+ int nn, nk;
+ pme_grid_comm_t *pgc;
+ gmx_bool bCont;
+ int fft_start, fft_end, send_index1, recv_index1;
+#ifdef GMX_MPI
+ MPI_Status stat;
+
+ ol->mpi_comm = comm;
+#endif
+
+ ol->nnodes = nnodes;
+ ol->nodeid = nodeid;
+
+ /* Linear translation of the PME grid won't affect reciprocal space
+ * calculations, so to optimize we only interpolate "upwards",
+ * which also means we only have to consider overlap in one direction.
+ * I.e., particles on this node might also be spread to grid indices
+ * that belong to higher nodes (modulo nnodes)
+ */
+
+ snew(ol->s2g0, ol->nnodes+1);
+ snew(ol->s2g1, ol->nnodes);
+ if (debug)
+ {
+ fprintf(debug, "PME slab boundaries:");
+ }
+ for (i = 0; i < nnodes; i++)
+ {
+ /* s2g0 the local interpolation grid start.
+ * s2g1 the local interpolation grid end.
+ * Because grid overlap communication only goes forward,
+ * the grid the slabs for fft's should be rounded down.
+ */
+ ol->s2g0[i] = ( i *ndata + 0 )/nnodes;
+ ol->s2g1[i] = ((i+1)*ndata + nnodes-1)/nnodes + norder - 1;
+
+ if (debug)
+ {
+ fprintf(debug, " %3d %3d", ol->s2g0[i], ol->s2g1[i]);
+ }
+ }
+ ol->s2g0[nnodes] = ndata;
+ if (debug)
+ {
+ fprintf(debug, "\n");
+ }
+
+ /* Determine with how many nodes we need to communicate the grid overlap */
+ b = 0;
+ do
+ {
+ b++;
+ bCont = FALSE;
+ for (i = 0; i < nnodes; i++)
+ {
+ if ((i+b < nnodes && ol->s2g1[i] > ol->s2g0[i+b]) ||
+ (i+b >= nnodes && ol->s2g1[i] > ol->s2g0[i+b-nnodes] + ndata))
+ {
+ bCont = TRUE;
+ }
+ }
+ }
+ while (bCont && b < nnodes);
+ ol->noverlap_nodes = b - 1;
+
+ snew(ol->send_id, ol->noverlap_nodes);
+ snew(ol->recv_id, ol->noverlap_nodes);
+ for (b = 0; b < ol->noverlap_nodes; b++)
+ {
+ ol->send_id[b] = (ol->nodeid + (b + 1)) % ol->nnodes;
+ ol->recv_id[b] = (ol->nodeid - (b + 1) + ol->nnodes) % ol->nnodes;
+ }
+ snew(ol->comm_data, ol->noverlap_nodes);
+
+ ol->send_size = 0;
+ for (b = 0; b < ol->noverlap_nodes; b++)
+ {
+ pgc = &ol->comm_data[b];
+ /* Send */
+ fft_start = ol->s2g0[ol->send_id[b]];
+ fft_end = ol->s2g0[ol->send_id[b]+1];
+ if (ol->send_id[b] < nodeid)
+ {
+ fft_start += ndata;
+ fft_end += ndata;
+ }
+ send_index1 = ol->s2g1[nodeid];
+ send_index1 = min(send_index1, fft_end);
+ pgc->send_index0 = fft_start;
+ pgc->send_nindex = max(0, send_index1 - pgc->send_index0);
+ ol->send_size += pgc->send_nindex;
+
+ /* We always start receiving to the first index of our slab */
+ fft_start = ol->s2g0[ol->nodeid];
+ fft_end = ol->s2g0[ol->nodeid+1];
+ recv_index1 = ol->s2g1[ol->recv_id[b]];
+ if (ol->recv_id[b] > nodeid)
+ {
+ recv_index1 -= ndata;
+ }
+ recv_index1 = min(recv_index1, fft_end);
+ pgc->recv_index0 = fft_start;
+ pgc->recv_nindex = max(0, recv_index1 - pgc->recv_index0);
+ }
+
+#ifdef GMX_MPI
+ /* Communicate the buffer sizes to receive */
+ for (b = 0; b < ol->noverlap_nodes; b++)
+ {
+ MPI_Sendrecv(&ol->send_size, 1, MPI_INT, ol->send_id[b], b,
+ &ol->comm_data[b].recv_size, 1, MPI_INT, ol->recv_id[b], b,
+ ol->mpi_comm, &stat);
+ }
+#endif
+
+ /* For non-divisible grid we need pme_order iso pme_order-1 */
+ snew(ol->sendbuf, norder*commplainsize);
+ snew(ol->recvbuf, norder*commplainsize);
+}
+
+static void
+make_gridindex5_to_localindex(int n, int local_start, int local_range,
+ int **global_to_local,
+ real **fraction_shift)
+{
+ int i;
+ int * gtl;
+ real * fsh;
+
+ snew(gtl, 5*n);
+ snew(fsh, 5*n);
+ for (i = 0; (i < 5*n); i++)
+ {
+ /* Determine the global to local grid index */
+ gtl[i] = (i - local_start + n) % n;
+ /* For coordinates that fall within the local grid the fraction
+ * is correct, we don't need to shift it.
+ */
+ fsh[i] = 0;
+ if (local_range < n)
+ {
+ /* Due to rounding issues i could be 1 beyond the lower or
+ * upper boundary of the local grid. Correct the index for this.
+ * If we shift the index, we need to shift the fraction by
+ * the same amount in the other direction to not affect
+ * the weights.
+ * Note that due to this shifting the weights at the end of
+ * the spline might change, but that will only involve values
+ * between zero and values close to the precision of a real,
+ * which is anyhow the accuracy of the whole mesh calculation.
+ */
+ /* With local_range=0 we should not change i=local_start */
+ if (i % n != local_start)
+ {
+ if (gtl[i] == n-1)
+ {
+ gtl[i] = 0;
+ fsh[i] = -1;
+ }
+ else if (gtl[i] == local_range)
+ {
+ gtl[i] = local_range - 1;
+ fsh[i] = 1;
+ }
+ }
+ }
+ }
+
+ *global_to_local = gtl;
+ *fraction_shift = fsh;
+}
+
+static pme_spline_work_t *make_pme_spline_work(int order)
+{
+ pme_spline_work_t *work;
+
- snew_aligned(work, 1, 16);
++#ifdef PME_SIMD4_SPREAD_GATHER
++ real tmp[12], *tmp_aligned;
++ gmx_simd4_pr zero_S;
++ gmx_simd4_pr real_mask_S0, real_mask_S1;
++ int of, i;
++
++ snew_aligned(work, 1, SIMD4_ALIGNMENT);
+
- zero_SSE = _mm_setzero_ps();
++ tmp_aligned = gmx_simd4_align_real(tmp);
+
- * load into 2 SSE float registers.
++ zero_S = gmx_simd4_setzero_pr();
+
+ /* Generate bit masks to mask out the unused grid entries,
+ * as we only operate on order of the 8 grid entries that are
- tmp[i] = (i >= of && i < of+order ? 1 : 0);
++ * load into 2 SIMD registers.
+ */
+ for (of = 0; of < 8-(order-1); of++)
+ {
+ for (i = 0; i < 8; i++)
+ {
- work->mask_SSE0[of] = _mm_loadu_ps(tmp);
- work->mask_SSE1[of] = _mm_loadu_ps(tmp+4);
- work->mask_SSE0[of] = _mm_cmpgt_ps(work->mask_SSE0[of], zero_SSE);
- work->mask_SSE1[of] = _mm_cmpgt_ps(work->mask_SSE1[of], zero_SSE);
++ tmp_aligned[i] = (i >= of && i < of+order ? -1.0 : 1.0);
+ }
++ real_mask_S0 = gmx_simd4_load_pr(tmp_aligned);
++ real_mask_S1 = gmx_simd4_load_pr(tmp_aligned+4);
++ work->mask_S0[of] = gmx_simd4_cmplt_pr(real_mask_S0, zero_S);
++ work->mask_S1[of] = gmx_simd4_cmplt_pr(real_mask_S1, zero_S);
+ }
+#else
+ work = NULL;
+#endif
+
+ return work;
+}
+
+void gmx_pme_check_restrictions(int pme_order,
+ int nkx, int nky, int nkz,
+ int nnodes_major,
+ int nnodes_minor,
+ gmx_bool bUseThreads,
+ gmx_bool bFatal,
+ gmx_bool *bValidSettings)
+{
+ if (pme_order > PME_ORDER_MAX)
+ {
+ if (!bFatal)
+ {
+ *bValidSettings = FALSE;
+ return;
+ }
+ gmx_fatal(FARGS, "pme_order (%d) is larger than the maximum allowed value (%d). Modify and recompile the code if you really need such a high order.",
+ pme_order, PME_ORDER_MAX);
+ }
+
+ if (nkx <= pme_order*(nnodes_major > 1 ? 2 : 1) ||
+ nky <= pme_order*(nnodes_minor > 1 ? 2 : 1) ||
+ nkz <= pme_order)
+ {
+ if (!bFatal)
+ {
+ *bValidSettings = FALSE;
+ return;
+ }
+ gmx_fatal(FARGS, "The PME grid sizes need to be larger than pme_order (%d) and for dimensions with domain decomposition larger than 2*pme_order",
+ pme_order);
+ }
+
+ /* Check for a limitation of the (current) sum_fftgrid_dd code.
+ * We only allow multiple communication pulses in dim 1, not in dim 0.
+ */
+ if (bUseThreads && (nkx < nnodes_major*pme_order &&
+ nkx != nnodes_major*(pme_order - 1)))
+ {
+ if (!bFatal)
+ {
+ *bValidSettings = FALSE;
+ return;
+ }
+ gmx_fatal(FARGS, "The number of PME grid lines per node along x is %g. But when using OpenMP threads, the number of grid lines per node along x should be >= pme_order (%d) or = pmeorder-1. To resolve this issue, use less nodes along x (and possibly more along y and/or z) by specifying -dd manually.",
+ nkx/(double)nnodes_major, pme_order);
+ }
+
+ if (bValidSettings != NULL)
+ {
+ *bValidSettings = TRUE;
+ }
+
+ return;
+}
+
+int gmx_pme_init(gmx_pme_t * pmedata,
+ t_commrec * cr,
+ int nnodes_major,
+ int nnodes_minor,
+ t_inputrec * ir,
+ int homenr,
+ gmx_bool bFreeEnergy,
+ gmx_bool bReproducible,
+ int nthread)
+{
+ gmx_pme_t pme = NULL;
+
+ int use_threads, sum_use_threads;
+ ivec ndata;
+
+ if (debug)
+ {
+ fprintf(debug, "Creating PME data structures.\n");
+ }
+ snew(pme, 1);
+
+ pme->redist_init = FALSE;
+ pme->sum_qgrid_tmp = NULL;
+ pme->sum_qgrid_dd_tmp = NULL;
+ pme->buf_nalloc = 0;
+ pme->redist_buf_nalloc = 0;
+
+ pme->nnodes = 1;
+ pme->bPPnode = TRUE;
+
+ pme->nnodes_major = nnodes_major;
+ pme->nnodes_minor = nnodes_minor;
+
+#ifdef GMX_MPI
+ if (nnodes_major*nnodes_minor > 1)
+ {
+ pme->mpi_comm = cr->mpi_comm_mygroup;
+
+ MPI_Comm_rank(pme->mpi_comm, &pme->nodeid);
+ MPI_Comm_size(pme->mpi_comm, &pme->nnodes);
+ if (pme->nnodes != nnodes_major*nnodes_minor)
+ {
+ gmx_incons("PME node count mismatch");
+ }
+ }
+ else
+ {
+ pme->mpi_comm = MPI_COMM_NULL;
+ }
+#endif
+
+ if (pme->nnodes == 1)
+ {
+#ifdef GMX_MPI
+ pme->mpi_comm_d[0] = MPI_COMM_NULL;
+ pme->mpi_comm_d[1] = MPI_COMM_NULL;
+#endif
+ pme->ndecompdim = 0;
+ pme->nodeid_major = 0;
+ pme->nodeid_minor = 0;
+#ifdef GMX_MPI
+ pme->mpi_comm_d[0] = pme->mpi_comm_d[1] = MPI_COMM_NULL;
+#endif
+ }
+ else
+ {
+ if (nnodes_minor == 1)
+ {
+#ifdef GMX_MPI
+ pme->mpi_comm_d[0] = pme->mpi_comm;
+ pme->mpi_comm_d[1] = MPI_COMM_NULL;
+#endif
+ pme->ndecompdim = 1;
+ pme->nodeid_major = pme->nodeid;
+ pme->nodeid_minor = 0;
+
+ }
+ else if (nnodes_major == 1)
+ {
+#ifdef GMX_MPI
+ pme->mpi_comm_d[0] = MPI_COMM_NULL;
+ pme->mpi_comm_d[1] = pme->mpi_comm;
+#endif
+ pme->ndecompdim = 1;
+ pme->nodeid_major = 0;
+ pme->nodeid_minor = pme->nodeid;
+ }
+ else
+ {
+ if (pme->nnodes % nnodes_major != 0)
+ {
+ gmx_incons("For 2D PME decomposition, #PME nodes must be divisible by the number of nodes in the major dimension");
+ }
+ pme->ndecompdim = 2;
+
+#ifdef GMX_MPI
+ MPI_Comm_split(pme->mpi_comm, pme->nodeid % nnodes_minor,
+ pme->nodeid, &pme->mpi_comm_d[0]); /* My communicator along major dimension */
+ MPI_Comm_split(pme->mpi_comm, pme->nodeid/nnodes_minor,
+ pme->nodeid, &pme->mpi_comm_d[1]); /* My communicator along minor dimension */
+
+ MPI_Comm_rank(pme->mpi_comm_d[0], &pme->nodeid_major);
+ MPI_Comm_size(pme->mpi_comm_d[0], &pme->nnodes_major);
+ MPI_Comm_rank(pme->mpi_comm_d[1], &pme->nodeid_minor);
+ MPI_Comm_size(pme->mpi_comm_d[1], &pme->nnodes_minor);
+#endif
+ }
+ pme->bPPnode = (cr->duty & DUTY_PP);
+ }
+
+ pme->nthread = nthread;
+
+ /* Check if any of the PME MPI ranks uses threads */
+ use_threads = (pme->nthread > 1 ? 1 : 0);
+#ifdef GMX_MPI
+ if (pme->nnodes > 1)
+ {
+ MPI_Allreduce(&use_threads, &sum_use_threads, 1, MPI_INT,
+ MPI_SUM, pme->mpi_comm);
+ }
+ else
+#endif
+ {
+ sum_use_threads = use_threads;
+ }
+ pme->bUseThreads = (sum_use_threads > 0);
+
+ if (ir->ePBC == epbcSCREW)
+ {
+ gmx_fatal(FARGS, "pme does not (yet) work with pbc = screw");
+ }
+
+ pme->bFEP = ((ir->efep != efepNO) && bFreeEnergy);
+ pme->nkx = ir->nkx;
+ pme->nky = ir->nky;
+ pme->nkz = ir->nkz;
+ pme->bP3M = (ir->coulombtype == eelP3M_AD || getenv("GMX_PME_P3M") != NULL);
+ pme->pme_order = ir->pme_order;
+ pme->epsilon_r = ir->epsilon_r;
+
+ /* If we violate restrictions, generate a fatal error here */
+ gmx_pme_check_restrictions(pme->pme_order,
+ pme->nkx, pme->nky, pme->nkz,
+ pme->nnodes_major,
+ pme->nnodes_minor,
+ pme->bUseThreads,
+ TRUE,
+ NULL);
+
+ if (pme->nnodes > 1)
+ {
+ double imbal;
+
+#ifdef GMX_MPI
+ MPI_Type_contiguous(DIM, mpi_type, &(pme->rvec_mpi));
+ MPI_Type_commit(&(pme->rvec_mpi));
+#endif
+
+ /* Note that the charge spreading and force gathering, which usually
+ * takes about the same amount of time as FFT+solve_pme,
+ * is always fully load balanced
+ * (unless the charge distribution is inhomogeneous).
+ */
+
+ imbal = pme_load_imbalance(pme);
+ if (imbal >= 1.2 && pme->nodeid_major == 0 && pme->nodeid_minor == 0)
+ {
+ fprintf(stderr,
+ "\n"
+ "NOTE: The load imbalance in PME FFT and solve is %d%%.\n"
+ " For optimal PME load balancing\n"
+ " PME grid_x (%d) and grid_y (%d) should be divisible by #PME_nodes_x (%d)\n"
+ " and PME grid_y (%d) and grid_z (%d) should be divisible by #PME_nodes_y (%d)\n"
+ "\n",
+ (int)((imbal-1)*100 + 0.5),
+ pme->nkx, pme->nky, pme->nnodes_major,
+ pme->nky, pme->nkz, pme->nnodes_minor);
+ }
+ }
+
+ /* For non-divisible grid we need pme_order iso pme_order-1 */
+ /* In sum_qgrid_dd x overlap is copied in place: take padding into account.
+ * y is always copied through a buffer: we don't need padding in z,
+ * but we do need the overlap in x because of the communication order.
+ */
+ init_overlap_comm(&pme->overlap[0], pme->pme_order,
+#ifdef GMX_MPI
+ pme->mpi_comm_d[0],
+#endif
+ pme->nnodes_major, pme->nodeid_major,
+ pme->nkx,
+ (div_round_up(pme->nky, pme->nnodes_minor)+pme->pme_order)*(pme->nkz+pme->pme_order-1));
+
+ /* Along overlap dim 1 we can send in multiple pulses in sum_fftgrid_dd.
+ * We do this with an offset buffer of equal size, so we need to allocate
+ * extra for the offset. That's what the (+1)*pme->nkz is for.
+ */
+ init_overlap_comm(&pme->overlap[1], pme->pme_order,
+#ifdef GMX_MPI
+ pme->mpi_comm_d[1],
+#endif
+ pme->nnodes_minor, pme->nodeid_minor,
+ pme->nky,
+ (div_round_up(pme->nkx, pme->nnodes_major)+pme->pme_order+1)*pme->nkz);
+
+ /* Double-check for a limitation of the (current) sum_fftgrid_dd code.
+ * Note that gmx_pme_check_restrictions checked for this already.
+ */
+ if (pme->bUseThreads && pme->overlap[0].noverlap_nodes > 1)
+ {
+ gmx_incons("More than one communication pulse required for grid overlap communication along the major dimension while using threads");
+ }
+
+ snew(pme->bsp_mod[XX], pme->nkx);
+ snew(pme->bsp_mod[YY], pme->nky);
+ snew(pme->bsp_mod[ZZ], pme->nkz);
+
+ /* The required size of the interpolation grid, including overlap.
+ * The allocated size (pmegrid_n?) might be slightly larger.
+ */
+ pme->pmegrid_nx = pme->overlap[0].s2g1[pme->nodeid_major] -
+ pme->overlap[0].s2g0[pme->nodeid_major];
+ pme->pmegrid_ny = pme->overlap[1].s2g1[pme->nodeid_minor] -
+ pme->overlap[1].s2g0[pme->nodeid_minor];
+ pme->pmegrid_nz_base = pme->nkz;
+ pme->pmegrid_nz = pme->pmegrid_nz_base + pme->pme_order - 1;
+ set_grid_alignment(&pme->pmegrid_nz, pme->pme_order);
+
+ pme->pmegrid_start_ix = pme->overlap[0].s2g0[pme->nodeid_major];
+ pme->pmegrid_start_iy = pme->overlap[1].s2g0[pme->nodeid_minor];
+ pme->pmegrid_start_iz = 0;
+
+ make_gridindex5_to_localindex(pme->nkx,
+ pme->pmegrid_start_ix,
+ pme->pmegrid_nx - (pme->pme_order-1),
+ &pme->nnx, &pme->fshx);
+ make_gridindex5_to_localindex(pme->nky,
+ pme->pmegrid_start_iy,
+ pme->pmegrid_ny - (pme->pme_order-1),
+ &pme->nny, &pme->fshy);
+ make_gridindex5_to_localindex(pme->nkz,
+ pme->pmegrid_start_iz,
+ pme->pmegrid_nz_base,
+ &pme->nnz, &pme->fshz);
+
+ pmegrids_init(&pme->pmegridA,
+ pme->pmegrid_nx, pme->pmegrid_ny, pme->pmegrid_nz,
+ pme->pmegrid_nz_base,
+ pme->pme_order,
+ pme->bUseThreads,
+ pme->nthread,
+ pme->overlap[0].s2g1[pme->nodeid_major]-pme->overlap[0].s2g0[pme->nodeid_major+1],
+ pme->overlap[1].s2g1[pme->nodeid_minor]-pme->overlap[1].s2g0[pme->nodeid_minor+1]);
+
+ pme->spline_work = make_pme_spline_work(pme->pme_order);
+
+ ndata[0] = pme->nkx;
+ ndata[1] = pme->nky;
+ ndata[2] = pme->nkz;
+
+ /* This routine will allocate the grid data to fit the FFTs */
+ gmx_parallel_3dfft_init(&pme->pfft_setupA, ndata,
+ &pme->fftgridA, &pme->cfftgridA,
+ pme->mpi_comm_d,
+ bReproducible, pme->nthread);
+
+ if (bFreeEnergy)
+ {
+ pmegrids_init(&pme->pmegridB,
+ pme->pmegrid_nx, pme->pmegrid_ny, pme->pmegrid_nz,
+ pme->pmegrid_nz_base,
+ pme->pme_order,
+ pme->bUseThreads,
+ pme->nthread,
+ pme->nkx % pme->nnodes_major != 0,
+ pme->nky % pme->nnodes_minor != 0);
+
+ gmx_parallel_3dfft_init(&pme->pfft_setupB, ndata,
+ &pme->fftgridB, &pme->cfftgridB,
+ pme->mpi_comm_d,
+ bReproducible, pme->nthread);
+ }
+ else
+ {
+ pme->pmegridB.grid.grid = NULL;
+ pme->fftgridB = NULL;
+ pme->cfftgridB = NULL;
+ }
+
+ if (!pme->bP3M)
+ {
+ /* Use plain SPME B-spline interpolation */
+ make_bspline_moduli(pme->bsp_mod, pme->nkx, pme->nky, pme->nkz, pme->pme_order);
+ }
+ else
+ {
+ /* Use the P3M grid-optimized influence function */
+ make_p3m_bspline_moduli(pme->bsp_mod, pme->nkx, pme->nky, pme->nkz, pme->pme_order);
+ }
+
+ /* Use atc[0] for spreading */
+ init_atomcomm(pme, &pme->atc[0], nnodes_major > 1 ? 0 : 1, TRUE);
+ if (pme->ndecompdim >= 2)
+ {
+ init_atomcomm(pme, &pme->atc[1], 1, FALSE);
+ }
+
+ if (pme->nnodes == 1)
+ {
+ pme->atc[0].n = homenr;
+ pme_realloc_atomcomm_things(&pme->atc[0]);
+ }
+
+ {
+ int thread;
+
+ /* Use fft5d, order after FFT is y major, z, x minor */
+
+ snew(pme->work, pme->nthread);
+ for (thread = 0; thread < pme->nthread; thread++)
+ {
+ realloc_work(&pme->work[thread], pme->nkx);
+ }
+ }
+
+ *pmedata = pme;
+
+ return 0;
+}
+
+static void reuse_pmegrids(const pmegrids_t *old, pmegrids_t *new)
+{
+ int d, t;
+
+ for (d = 0; d < DIM; d++)
+ {
+ if (new->grid.n[d] > old->grid.n[d])
+ {
+ return;
+ }
+ }
+
+ sfree_aligned(new->grid.grid);
+ new->grid.grid = old->grid.grid;
+
+ if (new->grid_th != NULL && new->nthread == old->nthread)
+ {
+ sfree_aligned(new->grid_all);
+ for (t = 0; t < new->nthread; t++)
+ {
+ new->grid_th[t].grid = old->grid_th[t].grid;
+ }
+ }
+}
+
+int gmx_pme_reinit(gmx_pme_t * pmedata,
+ t_commrec * cr,
+ gmx_pme_t pme_src,
+ const t_inputrec * ir,
+ ivec grid_size)
+{
+ t_inputrec irc;
+ int homenr;
+ int ret;
+
+ irc = *ir;
+ irc.nkx = grid_size[XX];
+ irc.nky = grid_size[YY];
+ irc.nkz = grid_size[ZZ];
+
+ if (pme_src->nnodes == 1)
+ {
+ homenr = pme_src->atc[0].n;
+ }
+ else
+ {
+ homenr = -1;
+ }
+
+ ret = gmx_pme_init(pmedata, cr, pme_src->nnodes_major, pme_src->nnodes_minor,
+ &irc, homenr, pme_src->bFEP, FALSE, pme_src->nthread);
+
+ if (ret == 0)
+ {
+ /* We can easily reuse the allocated pme grids in pme_src */
+ reuse_pmegrids(&pme_src->pmegridA, &(*pmedata)->pmegridA);
+ /* We would like to reuse the fft grids, but that's harder */
+ }
+
+ return ret;
+}
+
+
+static void copy_local_grid(gmx_pme_t pme,
+ pmegrids_t *pmegrids, int thread, real *fftgrid)
+{
+ ivec local_fft_ndata, local_fft_offset, local_fft_size;
+ int fft_my, fft_mz;
+ int nsx, nsy, nsz;
+ ivec nf;
+ int offx, offy, offz, x, y, z, i0, i0t;
+ int d;
+ pmegrid_t *pmegrid;
+ real *grid_th;
+
+ gmx_parallel_3dfft_real_limits(pme->pfft_setupA,
+ local_fft_ndata,
+ local_fft_offset,
+ local_fft_size);
+ fft_my = local_fft_size[YY];
+ fft_mz = local_fft_size[ZZ];
+
+ pmegrid = &pmegrids->grid_th[thread];
+
+ nsx = pmegrid->s[XX];
+ nsy = pmegrid->s[YY];
+ nsz = pmegrid->s[ZZ];
+
+ for (d = 0; d < DIM; d++)
+ {
+ nf[d] = min(pmegrid->n[d] - (pmegrid->order - 1),
+ local_fft_ndata[d] - pmegrid->offset[d]);
+ }
+
+ offx = pmegrid->offset[XX];
+ offy = pmegrid->offset[YY];
+ offz = pmegrid->offset[ZZ];
+
+ /* Directly copy the non-overlapping parts of the local grids.
+ * This also initializes the full grid.
+ */
+ grid_th = pmegrid->grid;
+ for (x = 0; x < nf[XX]; x++)
+ {
+ for (y = 0; y < nf[YY]; y++)
+ {
+ i0 = ((offx + x)*fft_my + (offy + y))*fft_mz + offz;
+ i0t = (x*nsy + y)*nsz;
+ for (z = 0; z < nf[ZZ]; z++)
+ {
+ fftgrid[i0+z] = grid_th[i0t+z];
+ }
+ }
+ }
+}
+
+static void
+reduce_threadgrid_overlap(gmx_pme_t pme,
+ const pmegrids_t *pmegrids, int thread,
+ real *fftgrid, real *commbuf_x, real *commbuf_y)
+{
+ ivec local_fft_ndata, local_fft_offset, local_fft_size;
+ int fft_nx, fft_ny, fft_nz;
+ int fft_my, fft_mz;
+ int buf_my = -1;
+ int nsx, nsy, nsz;
+ ivec ne;
+ int offx, offy, offz, x, y, z, i0, i0t;
+ int sx, sy, sz, fx, fy, fz, tx1, ty1, tz1, ox, oy, oz;
+ gmx_bool bClearBufX, bClearBufY, bClearBufXY, bClearBuf;
+ gmx_bool bCommX, bCommY;
+ int d;
+ int thread_f;
+ const pmegrid_t *pmegrid, *pmegrid_g, *pmegrid_f;
+ const real *grid_th;
+ real *commbuf = NULL;
+
+ gmx_parallel_3dfft_real_limits(pme->pfft_setupA,
+ local_fft_ndata,
+ local_fft_offset,
+ local_fft_size);
+ fft_nx = local_fft_ndata[XX];
+ fft_ny = local_fft_ndata[YY];
+ fft_nz = local_fft_ndata[ZZ];
+
+ fft_my = local_fft_size[YY];
+ fft_mz = local_fft_size[ZZ];
+
+ /* This routine is called when all thread have finished spreading.
+ * Here each thread sums grid contributions calculated by other threads
+ * to the thread local grid volume.
+ * To minimize the number of grid copying operations,
+ * this routines sums immediately from the pmegrid to the fftgrid.
+ */
+
+ /* Determine which part of the full node grid we should operate on,
+ * this is our thread local part of the full grid.
+ */
+ pmegrid = &pmegrids->grid_th[thread];
+
+ for (d = 0; d < DIM; d++)
+ {
+ ne[d] = min(pmegrid->offset[d]+pmegrid->n[d]-(pmegrid->order-1),
+ local_fft_ndata[d]);
+ }
+
+ offx = pmegrid->offset[XX];
+ offy = pmegrid->offset[YY];
+ offz = pmegrid->offset[ZZ];
+
+
+ bClearBufX = TRUE;
+ bClearBufY = TRUE;
+ bClearBufXY = TRUE;
+
+ /* Now loop over all the thread data blocks that contribute
+ * to the grid region we (our thread) are operating on.
+ */
+ /* Note that ffy_nx/y is equal to the number of grid points
+ * between the first point of our node grid and the one of the next node.
+ */
+ for (sx = 0; sx >= -pmegrids->nthread_comm[XX]; sx--)
+ {
+ fx = pmegrid->ci[XX] + sx;
+ ox = 0;
+ bCommX = FALSE;
+ if (fx < 0)
+ {
+ fx += pmegrids->nc[XX];
+ ox -= fft_nx;
+ bCommX = (pme->nnodes_major > 1);
+ }
+ pmegrid_g = &pmegrids->grid_th[fx*pmegrids->nc[YY]*pmegrids->nc[ZZ]];
+ ox += pmegrid_g->offset[XX];
+ if (!bCommX)
+ {
+ tx1 = min(ox + pmegrid_g->n[XX], ne[XX]);
+ }
+ else
+ {
+ tx1 = min(ox + pmegrid_g->n[XX], pme->pme_order);
+ }
+
+ for (sy = 0; sy >= -pmegrids->nthread_comm[YY]; sy--)
+ {
+ fy = pmegrid->ci[YY] + sy;
+ oy = 0;
+ bCommY = FALSE;
+ if (fy < 0)
+ {
+ fy += pmegrids->nc[YY];
+ oy -= fft_ny;
+ bCommY = (pme->nnodes_minor > 1);
+ }
+ pmegrid_g = &pmegrids->grid_th[fy*pmegrids->nc[ZZ]];
+ oy += pmegrid_g->offset[YY];
+ if (!bCommY)
+ {
+ ty1 = min(oy + pmegrid_g->n[YY], ne[YY]);
+ }
+ else
+ {
+ ty1 = min(oy + pmegrid_g->n[YY], pme->pme_order);
+ }
+
+ for (sz = 0; sz >= -pmegrids->nthread_comm[ZZ]; sz--)
+ {
+ fz = pmegrid->ci[ZZ] + sz;
+ oz = 0;
+ if (fz < 0)
+ {
+ fz += pmegrids->nc[ZZ];
+ oz -= fft_nz;
+ }
+ pmegrid_g = &pmegrids->grid_th[fz];
+ oz += pmegrid_g->offset[ZZ];
+ tz1 = min(oz + pmegrid_g->n[ZZ], ne[ZZ]);
+
+ if (sx == 0 && sy == 0 && sz == 0)
+ {
+ /* We have already added our local contribution
+ * before calling this routine, so skip it here.
+ */
+ continue;
+ }
+
+ thread_f = (fx*pmegrids->nc[YY] + fy)*pmegrids->nc[ZZ] + fz;
+
+ pmegrid_f = &pmegrids->grid_th[thread_f];
+
+ grid_th = pmegrid_f->grid;
+
+ nsx = pmegrid_f->s[XX];
+ nsy = pmegrid_f->s[YY];
+ nsz = pmegrid_f->s[ZZ];
+
+#ifdef DEBUG_PME_REDUCE
+ printf("n%d t%d add %d %2d %2d %2d %2d %2d %2d %2d-%2d %2d-%2d, %2d-%2d %2d-%2d, %2d-%2d %2d-%2d\n",
+ pme->nodeid, thread, thread_f,
+ pme->pmegrid_start_ix,
+ pme->pmegrid_start_iy,
+ pme->pmegrid_start_iz,
+ sx, sy, sz,
+ offx-ox, tx1-ox, offx, tx1,
+ offy-oy, ty1-oy, offy, ty1,
+ offz-oz, tz1-oz, offz, tz1);
+#endif
+
+ if (!(bCommX || bCommY))
+ {
+ /* Copy from the thread local grid to the node grid */
+ for (x = offx; x < tx1; x++)
+ {
+ for (y = offy; y < ty1; y++)
+ {
+ i0 = (x*fft_my + y)*fft_mz;
+ i0t = ((x - ox)*nsy + (y - oy))*nsz - oz;
+ for (z = offz; z < tz1; z++)
+ {
+ fftgrid[i0+z] += grid_th[i0t+z];
+ }
+ }
+ }
+ }
+ else
+ {
+ /* The order of this conditional decides
+ * where the corner volume gets stored with x+y decomp.
+ */
+ if (bCommY)
+ {
+ commbuf = commbuf_y;
+ buf_my = ty1 - offy;
+ if (bCommX)
+ {
+ /* We index commbuf modulo the local grid size */
+ commbuf += buf_my*fft_nx*fft_nz;
+
+ bClearBuf = bClearBufXY;
+ bClearBufXY = FALSE;
+ }
+ else
+ {
+ bClearBuf = bClearBufY;
+ bClearBufY = FALSE;
+ }
+ }
+ else
+ {
+ commbuf = commbuf_x;
+ buf_my = fft_ny;
+ bClearBuf = bClearBufX;
+ bClearBufX = FALSE;
+ }
+
+ /* Copy to the communication buffer */
+ for (x = offx; x < tx1; x++)
+ {
+ for (y = offy; y < ty1; y++)
+ {
+ i0 = (x*buf_my + y)*fft_nz;
+ i0t = ((x - ox)*nsy + (y - oy))*nsz - oz;
+
+ if (bClearBuf)
+ {
+ /* First access of commbuf, initialize it */
+ for (z = offz; z < tz1; z++)
+ {
+ commbuf[i0+z] = grid_th[i0t+z];
+ }
+ }
+ else
+ {
+ for (z = offz; z < tz1; z++)
+ {
+ commbuf[i0+z] += grid_th[i0t+z];
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+}
+
+
+static void sum_fftgrid_dd(gmx_pme_t pme, real *fftgrid)
+{
+ ivec local_fft_ndata, local_fft_offset, local_fft_size;
+ pme_overlap_t *overlap;
+ int send_index0, send_nindex;
+ int recv_nindex;
+#ifdef GMX_MPI
+ MPI_Status stat;
+#endif
+ int send_size_y, recv_size_y;
+ int ipulse, send_id, recv_id, datasize, gridsize, size_yx;
+ real *sendptr, *recvptr;
+ int x, y, z, indg, indb;
+
+ /* Note that this routine is only used for forward communication.
+ * Since the force gathering, unlike the charge spreading,
+ * can be trivially parallelized over the particles,
+ * the backwards process is much simpler and can use the "old"
+ * communication setup.
+ */
+
+ gmx_parallel_3dfft_real_limits(pme->pfft_setupA,
+ local_fft_ndata,
+ local_fft_offset,
+ local_fft_size);
+
+ if (pme->nnodes_minor > 1)
+ {
+ /* Major dimension */
+ overlap = &pme->overlap[1];
+
+ if (pme->nnodes_major > 1)
+ {
+ size_yx = pme->overlap[0].comm_data[0].send_nindex;
+ }
+ else
+ {
+ size_yx = 0;
+ }
+ datasize = (local_fft_ndata[XX] + size_yx)*local_fft_ndata[ZZ];
+
+ send_size_y = overlap->send_size;
+
+ for (ipulse = 0; ipulse < overlap->noverlap_nodes; ipulse++)
+ {
+ send_id = overlap->send_id[ipulse];
+ recv_id = overlap->recv_id[ipulse];
+ send_index0 =
+ overlap->comm_data[ipulse].send_index0 -
+ overlap->comm_data[0].send_index0;
+ send_nindex = overlap->comm_data[ipulse].send_nindex;
+ /* We don't use recv_index0, as we always receive starting at 0 */
+ recv_nindex = overlap->comm_data[ipulse].recv_nindex;
+ recv_size_y = overlap->comm_data[ipulse].recv_size;
+
+ sendptr = overlap->sendbuf + send_index0*local_fft_ndata[ZZ];
+ recvptr = overlap->recvbuf;
+
+#ifdef GMX_MPI
+ MPI_Sendrecv(sendptr, send_size_y*datasize, GMX_MPI_REAL,
+ send_id, ipulse,
+ recvptr, recv_size_y*datasize, GMX_MPI_REAL,
+ recv_id, ipulse,
+ overlap->mpi_comm, &stat);
+#endif
+
+ for (x = 0; x < local_fft_ndata[XX]; x++)
+ {
+ for (y = 0; y < recv_nindex; y++)
+ {
+ indg = (x*local_fft_size[YY] + y)*local_fft_size[ZZ];
+ indb = (x*recv_size_y + y)*local_fft_ndata[ZZ];
+ for (z = 0; z < local_fft_ndata[ZZ]; z++)
+ {
+ fftgrid[indg+z] += recvptr[indb+z];
+ }
+ }
+ }
+
+ if (pme->nnodes_major > 1)
+ {
+ /* Copy from the received buffer to the send buffer for dim 0 */
+ sendptr = pme->overlap[0].sendbuf;
+ for (x = 0; x < size_yx; x++)
+ {
+ for (y = 0; y < recv_nindex; y++)
+ {
+ indg = (x*local_fft_ndata[YY] + y)*local_fft_ndata[ZZ];
+ indb = ((local_fft_ndata[XX] + x)*recv_size_y + y)*local_fft_ndata[ZZ];
+ for (z = 0; z < local_fft_ndata[ZZ]; z++)
+ {
+ sendptr[indg+z] += recvptr[indb+z];
+ }
+ }
+ }
+ }
+ }
+ }
+
+ /* We only support a single pulse here.
+ * This is not a severe limitation, as this code is only used
+ * with OpenMP and with OpenMP the (PME) domains can be larger.
+ */
+ if (pme->nnodes_major > 1)
+ {
+ /* Major dimension */
+ overlap = &pme->overlap[0];
+
+ datasize = local_fft_ndata[YY]*local_fft_ndata[ZZ];
+ gridsize = local_fft_size[YY] *local_fft_size[ZZ];
+
+ ipulse = 0;
+
+ send_id = overlap->send_id[ipulse];
+ recv_id = overlap->recv_id[ipulse];
+ send_nindex = overlap->comm_data[ipulse].send_nindex;
+ /* We don't use recv_index0, as we always receive starting at 0 */
+ recv_nindex = overlap->comm_data[ipulse].recv_nindex;
+
+ sendptr = overlap->sendbuf;
+ recvptr = overlap->recvbuf;
+
+ if (debug != NULL)
+ {
+ fprintf(debug, "PME fftgrid comm %2d x %2d x %2d\n",
+ send_nindex, local_fft_ndata[YY], local_fft_ndata[ZZ]);
+ }
+
+#ifdef GMX_MPI
+ MPI_Sendrecv(sendptr, send_nindex*datasize, GMX_MPI_REAL,
+ send_id, ipulse,
+ recvptr, recv_nindex*datasize, GMX_MPI_REAL,
+ recv_id, ipulse,
+ overlap->mpi_comm, &stat);
+#endif
+
+ for (x = 0; x < recv_nindex; x++)
+ {
+ for (y = 0; y < local_fft_ndata[YY]; y++)
+ {
+ indg = (x*local_fft_size[YY] + y)*local_fft_size[ZZ];
+ indb = (x*local_fft_ndata[YY] + y)*local_fft_ndata[ZZ];
+ for (z = 0; z < local_fft_ndata[ZZ]; z++)
+ {
+ fftgrid[indg+z] += recvptr[indb+z];
+ }
+ }
+ }
+ }
+}
+
+
+static void spread_on_grid(gmx_pme_t pme,
+ pme_atomcomm_t *atc, pmegrids_t *grids,
+ gmx_bool bCalcSplines, gmx_bool bSpread,
+ real *fftgrid)
+{
+ int nthread, thread;
+#ifdef PME_TIME_THREADS
+ gmx_cycles_t c1, c2, c3, ct1a, ct1b, ct1c;
+ static double cs1 = 0, cs2 = 0, cs3 = 0;
+ static double cs1a[6] = {0, 0, 0, 0, 0, 0};
+ static int cnt = 0;
+#endif
+
+ nthread = pme->nthread;
+ assert(nthread > 0);
+
+#ifdef PME_TIME_THREADS
+ c1 = omp_cyc_start();
+#endif
+ if (bCalcSplines)
+ {
+#pragma omp parallel for num_threads(nthread) schedule(static)
+ for (thread = 0; thread < nthread; thread++)
+ {
+ int start, end;
+
+ start = atc->n* thread /nthread;
+ end = atc->n*(thread+1)/nthread;
+
+ /* Compute fftgrid index for all atoms,
+ * with help of some extra variables.
+ */
+ calc_interpolation_idx(pme, atc, start, end, thread);
+ }
+ }
+#ifdef PME_TIME_THREADS
+ c1 = omp_cyc_end(c1);
+ cs1 += (double)c1;
+#endif
+
+#ifdef PME_TIME_THREADS
+ c2 = omp_cyc_start();
+#endif
+#pragma omp parallel for num_threads(nthread) schedule(static)
+ for (thread = 0; thread < nthread; thread++)
+ {
+ splinedata_t *spline;
+ pmegrid_t *grid = NULL;
+
+ /* make local bsplines */
+ if (grids == NULL || !pme->bUseThreads)
+ {
+ spline = &atc->spline[0];
+
+ spline->n = atc->n;
+
+ if (bSpread)
+ {
+ grid = &grids->grid;
+ }
+ }
+ else
+ {
+ spline = &atc->spline[thread];
+
+ if (grids->nthread == 1)
+ {
+ /* One thread, we operate on all charges */
+ spline->n = atc->n;
+ }
+ else
+ {
+ /* Get the indices our thread should operate on */
+ make_thread_local_ind(atc, thread, spline);
+ }
+
+ grid = &grids->grid_th[thread];
+ }
+
+ if (bCalcSplines)
+ {
+ make_bsplines(spline->theta, spline->dtheta, pme->pme_order,
+ atc->fractx, spline->n, spline->ind, atc->q, pme->bFEP);
+ }
+
+ if (bSpread)
+ {
+ /* put local atoms on grid. */
+#ifdef PME_TIME_SPREAD
+ ct1a = omp_cyc_start();
+#endif
+ spread_q_bsplines_thread(grid, atc, spline, pme->spline_work);
+
+ if (pme->bUseThreads)
+ {
+ copy_local_grid(pme, grids, thread, fftgrid);
+ }
+#ifdef PME_TIME_SPREAD
+ ct1a = omp_cyc_end(ct1a);
+ cs1a[thread] += (double)ct1a;
+#endif
+ }
+ }
+#ifdef PME_TIME_THREADS
+ c2 = omp_cyc_end(c2);
+ cs2 += (double)c2;
+#endif
+
+ if (bSpread && pme->bUseThreads)
+ {
+#ifdef PME_TIME_THREADS
+ c3 = omp_cyc_start();
+#endif
+#pragma omp parallel for num_threads(grids->nthread) schedule(static)
+ for (thread = 0; thread < grids->nthread; thread++)
+ {
+ reduce_threadgrid_overlap(pme, grids, thread,
+ fftgrid,
+ pme->overlap[0].sendbuf,
+ pme->overlap[1].sendbuf);
+ }
+#ifdef PME_TIME_THREADS
+ c3 = omp_cyc_end(c3);
+ cs3 += (double)c3;
+#endif
+
+ if (pme->nnodes > 1)
+ {
+ /* Communicate the overlapping part of the fftgrid.
+ * For this communication call we need to check pme->bUseThreads
+ * to have all ranks communicate here, regardless of pme->nthread.
+ */
+ sum_fftgrid_dd(pme, fftgrid);
+ }
+ }
+
+#ifdef PME_TIME_THREADS
+ cnt++;
+ if (cnt % 20 == 0)
+ {
+ printf("idx %.2f spread %.2f red %.2f",
+ cs1*1e-9, cs2*1e-9, cs3*1e-9);
+#ifdef PME_TIME_SPREAD
+ for (thread = 0; thread < nthread; thread++)
+ {
+ printf(" %.2f", cs1a[thread]*1e-9);
+ }
+#endif
+ printf("\n");
+ }
+#endif
+}
+
+
+static void dump_grid(FILE *fp,
+ int sx, int sy, int sz, int nx, int ny, int nz,
+ int my, int mz, const real *g)
+{
+ int x, y, z;
+
+ for (x = 0; x < nx; x++)
+ {
+ for (y = 0; y < ny; y++)
+ {
+ for (z = 0; z < nz; z++)
+ {
+ fprintf(fp, "%2d %2d %2d %6.3f\n",
+ sx+x, sy+y, sz+z, g[(x*my + y)*mz + z]);
+ }
+ }
+ }
+}
+
+static void dump_local_fftgrid(gmx_pme_t pme, const real *fftgrid)
+{
+ ivec local_fft_ndata, local_fft_offset, local_fft_size;
+
+ gmx_parallel_3dfft_real_limits(pme->pfft_setupA,
+ local_fft_ndata,
+ local_fft_offset,
+ local_fft_size);
+
+ dump_grid(stderr,
+ pme->pmegrid_start_ix,
+ pme->pmegrid_start_iy,
+ pme->pmegrid_start_iz,
+ pme->pmegrid_nx-pme->pme_order+1,
+ pme->pmegrid_ny-pme->pme_order+1,
+ pme->pmegrid_nz-pme->pme_order+1,
+ local_fft_size[YY],
+ local_fft_size[ZZ],
+ fftgrid);
+}
+
+
+void gmx_pme_calc_energy(gmx_pme_t pme, int n, rvec *x, real *q, real *V)
+{
+ pme_atomcomm_t *atc;
+ pmegrids_t *grid;
+
+ if (pme->nnodes > 1)
+ {
+ gmx_incons("gmx_pme_calc_energy called in parallel");
+ }
+ if (pme->bFEP > 1)
+ {
+ gmx_incons("gmx_pme_calc_energy with free energy");
+ }
+
+ atc = &pme->atc_energy;
+ atc->nthread = 1;
+ if (atc->spline == NULL)
+ {
+ snew(atc->spline, atc->nthread);
+ }
+ atc->nslab = 1;
+ atc->bSpread = TRUE;
+ atc->pme_order = pme->pme_order;
+ atc->n = n;
+ pme_realloc_atomcomm_things(atc);
+ atc->x = x;
+ atc->q = q;
+
+ /* We only use the A-charges grid */
+ grid = &pme->pmegridA;
+
+ /* Only calculate the spline coefficients, don't actually spread */
+ spread_on_grid(pme, atc, NULL, TRUE, FALSE, pme->fftgridA);
+
+ *V = gather_energy_bsplines(pme, grid->grid.grid, atc);
+}
+
+
+static void reset_pmeonly_counters(gmx_wallcycle_t wcycle,
+ gmx_runtime_t *runtime,
+ t_nrnb *nrnb, t_inputrec *ir,
+ gmx_large_int_t step)
+{
+ /* Reset all the counters related to performance over the run */
+ wallcycle_stop(wcycle, ewcRUN);
+ wallcycle_reset_all(wcycle);
+ init_nrnb(nrnb);
+ if (ir->nsteps >= 0)
+ {
+ /* ir->nsteps is not used here, but we update it for consistency */
+ ir->nsteps -= step - ir->init_step;
+ }
+ ir->init_step = step;
+ wallcycle_start(wcycle, ewcRUN);
+ runtime_start(runtime);
+}
+
+
+static void gmx_pmeonly_switch(int *npmedata, gmx_pme_t **pmedata,
+ ivec grid_size,
+ t_commrec *cr, t_inputrec *ir,
+ gmx_pme_t *pme_ret)
+{
+ int ind;
+ gmx_pme_t pme = NULL;
+
+ ind = 0;
+ while (ind < *npmedata)
+ {
+ pme = (*pmedata)[ind];
+ if (pme->nkx == grid_size[XX] &&
+ pme->nky == grid_size[YY] &&
+ pme->nkz == grid_size[ZZ])
+ {
+ *pme_ret = pme;
+
+ return;
+ }
+
+ ind++;
+ }
+
+ (*npmedata)++;
+ srenew(*pmedata, *npmedata);
+
+ /* Generate a new PME data structure, copying part of the old pointers */
+ gmx_pme_reinit(&((*pmedata)[ind]), cr, pme, ir, grid_size);
+
+ *pme_ret = (*pmedata)[ind];
+}
+
+
+int gmx_pmeonly(gmx_pme_t pme,
+ t_commrec *cr, t_nrnb *nrnb,
+ gmx_wallcycle_t wcycle,
+ gmx_runtime_t *runtime,
+ real ewaldcoeff,
+ t_inputrec *ir)
+{
+ int npmedata;
+ gmx_pme_t *pmedata;
+ gmx_pme_pp_t pme_pp;
+ int ret;
+ int natoms;
+ matrix box;
+ rvec *x_pp = NULL, *f_pp = NULL;
+ real *chargeA = NULL, *chargeB = NULL;
+ real lambda = 0;
+ int maxshift_x = 0, maxshift_y = 0;
+ real energy, dvdlambda;
+ matrix vir;
+ float cycles;
+ int count;
+ gmx_bool bEnerVir;
+ gmx_large_int_t step, step_rel;
+ ivec grid_switch;
+
+ /* This data will only use with PME tuning, i.e. switching PME grids */
+ npmedata = 1;
+ snew(pmedata, npmedata);
+ pmedata[0] = pme;
+
+ pme_pp = gmx_pme_pp_init(cr);
+
+ init_nrnb(nrnb);
+
+ count = 0;
+ do /****** this is a quasi-loop over time steps! */
+ {
+ /* The reason for having a loop here is PME grid tuning/switching */
+ do
+ {
+ /* Domain decomposition */
+ ret = gmx_pme_recv_q_x(pme_pp,
+ &natoms,
+ &chargeA, &chargeB, box, &x_pp, &f_pp,
+ &maxshift_x, &maxshift_y,
+ &pme->bFEP, &lambda,
+ &bEnerVir,
+ &step,
+ grid_switch, &ewaldcoeff);
+
+ if (ret == pmerecvqxSWITCHGRID)
+ {
+ /* Switch the PME grid to grid_switch */
+ gmx_pmeonly_switch(&npmedata, &pmedata, grid_switch, cr, ir, &pme);
+ }
+
+ if (ret == pmerecvqxRESETCOUNTERS)
+ {
+ /* Reset the cycle and flop counters */
+ reset_pmeonly_counters(wcycle, runtime, nrnb, ir, step);
+ }
+ }
+ while (ret == pmerecvqxSWITCHGRID || ret == pmerecvqxRESETCOUNTERS);
+
+ if (ret == pmerecvqxFINISH)
+ {
+ /* We should stop: break out of the loop */
+ break;
+ }
+
+ step_rel = step - ir->init_step;
+
+ if (count == 0)
+ {
+ wallcycle_start(wcycle, ewcRUN);
+ runtime_start(runtime);
+ }
+
+ wallcycle_start(wcycle, ewcPMEMESH);
+
+ dvdlambda = 0;
+ clear_mat(vir);
+ gmx_pme_do(pme, 0, natoms, x_pp, f_pp, chargeA, chargeB, box,
+ cr, maxshift_x, maxshift_y, nrnb, wcycle, vir, ewaldcoeff,
+ &energy, lambda, &dvdlambda,
+ GMX_PME_DO_ALL_F | (bEnerVir ? GMX_PME_CALC_ENER_VIR : 0));
+
+ cycles = wallcycle_stop(wcycle, ewcPMEMESH);
+
+ gmx_pme_send_force_vir_ener(pme_pp,
+ f_pp, vir, energy, dvdlambda,
+ cycles);
+
+ count++;
+ } /***** end of quasi-loop, we stop with the break above */
+ while (TRUE);
+
+ runtime_end(runtime);
+
+ return 0;
+}
+
+int gmx_pme_do(gmx_pme_t pme,
+ int start, int homenr,
+ rvec x[], rvec f[],
+ real *chargeA, real *chargeB,
+ matrix box, t_commrec *cr,
+ int maxshift_x, int maxshift_y,
+ t_nrnb *nrnb, gmx_wallcycle_t wcycle,
+ matrix vir, real ewaldcoeff,
+ real *energy, real lambda,
+ real *dvdlambda, int flags)
+{
+ int q, d, i, j, ntot, npme;
+ int nx, ny, nz;
+ int n_d, local_ny;
+ pme_atomcomm_t *atc = NULL;
+ pmegrids_t *pmegrid = NULL;
+ real *grid = NULL;
+ real *ptr;
+ rvec *x_d, *f_d;
+ real *charge = NULL, *q_d;
+ real energy_AB[2];
+ matrix vir_AB[2];
+ gmx_bool bClearF;
+ gmx_parallel_3dfft_t pfft_setup;
+ real * fftgrid;
+ t_complex * cfftgrid;
+ int thread;
+ const gmx_bool bCalcEnerVir = flags & GMX_PME_CALC_ENER_VIR;
+ const gmx_bool bCalcF = flags & GMX_PME_CALC_F;
+
+ assert(pme->nnodes > 0);
+ assert(pme->nnodes == 1 || pme->ndecompdim > 0);
+
+ if (pme->nnodes > 1)
+ {
+ atc = &pme->atc[0];
+ atc->npd = homenr;
+ if (atc->npd > atc->pd_nalloc)
+ {
+ atc->pd_nalloc = over_alloc_dd(atc->npd);
+ srenew(atc->pd, atc->pd_nalloc);
+ }
+ atc->maxshift = (atc->dimind == 0 ? maxshift_x : maxshift_y);
+ }
+ else
+ {
+ /* This could be necessary for TPI */
+ pme->atc[0].n = homenr;
+ }
+
+ for (q = 0; q < (pme->bFEP ? 2 : 1); q++)
+ {
+ if (q == 0)
+ {
+ pmegrid = &pme->pmegridA;
+ fftgrid = pme->fftgridA;
+ cfftgrid = pme->cfftgridA;
+ pfft_setup = pme->pfft_setupA;
+ charge = chargeA+start;
+ }
+ else
+ {
+ pmegrid = &pme->pmegridB;
+ fftgrid = pme->fftgridB;
+ cfftgrid = pme->cfftgridB;
+ pfft_setup = pme->pfft_setupB;
+ charge = chargeB+start;
+ }
+ grid = pmegrid->grid.grid;
+ /* Unpack structure */
+ if (debug)
+ {
+ fprintf(debug, "PME: nnodes = %d, nodeid = %d\n",
+ cr->nnodes, cr->nodeid);
+ fprintf(debug, "Grid = %p\n", (void*)grid);
+ if (grid == NULL)
+ {
+ gmx_fatal(FARGS, "No grid!");
+ }
+ }
+ where();
+
+ m_inv_ur0(box, pme->recipbox);
+
+ if (pme->nnodes == 1)
+ {
+ atc = &pme->atc[0];
+ if (DOMAINDECOMP(cr))
+ {
+ atc->n = homenr;
+ pme_realloc_atomcomm_things(atc);
+ }
+ atc->x = x;
+ atc->q = charge;
+ atc->f = f;
+ }
+ else
+ {
+ wallcycle_start(wcycle, ewcPME_REDISTXF);
+ for (d = pme->ndecompdim-1; d >= 0; d--)
+ {
+ if (d == pme->ndecompdim-1)
+ {
+ n_d = homenr;
+ x_d = x + start;
+ q_d = charge;
+ }
+ else
+ {
+ n_d = pme->atc[d+1].n;
+ x_d = atc->x;
+ q_d = atc->q;
+ }
+ atc = &pme->atc[d];
+ atc->npd = n_d;
+ if (atc->npd > atc->pd_nalloc)
+ {
+ atc->pd_nalloc = over_alloc_dd(atc->npd);
+ srenew(atc->pd, atc->pd_nalloc);
+ }
+ atc->maxshift = (atc->dimind == 0 ? maxshift_x : maxshift_y);
+ pme_calc_pidx_wrapper(n_d, pme->recipbox, x_d, atc);
+ where();
+
+ /* Redistribute x (only once) and qA or qB */
+ if (DOMAINDECOMP(cr))
+ {
+ dd_pmeredist_x_q(pme, n_d, q == 0, x_d, q_d, atc);
+ }
+ else
+ {
+ pmeredist_pd(pme, TRUE, n_d, q == 0, x_d, q_d, atc);
+ }
+ }
+ where();
+
+ wallcycle_stop(wcycle, ewcPME_REDISTXF);
+ }
+
+ if (debug)
+ {
+ fprintf(debug, "Node= %6d, pme local particles=%6d\n",
+ cr->nodeid, atc->n);
+ }
+
+ if (flags & GMX_PME_SPREAD_Q)
+ {
+ wallcycle_start(wcycle, ewcPME_SPREADGATHER);
+
+ /* Spread the charges on a grid */
+ spread_on_grid(pme, &pme->atc[0], pmegrid, q == 0, TRUE, fftgrid);
+
+ if (q == 0)
+ {
+ inc_nrnb(nrnb, eNR_WEIGHTS, DIM*atc->n);
+ }
+ inc_nrnb(nrnb, eNR_SPREADQBSP,
+ pme->pme_order*pme->pme_order*pme->pme_order*atc->n);
+
+ if (!pme->bUseThreads)
+ {
+ wrap_periodic_pmegrid(pme, grid);
+
+ /* sum contributions to local grid from other nodes */
+#ifdef GMX_MPI
+ if (pme->nnodes > 1)
+ {
+ gmx_sum_qgrid_dd(pme, grid, GMX_SUM_QGRID_FORWARD);
+ where();
+ }
+#endif
+
+ copy_pmegrid_to_fftgrid(pme, grid, fftgrid);
+ }
+
+ wallcycle_stop(wcycle, ewcPME_SPREADGATHER);
+
+ /*
+ dump_local_fftgrid(pme,fftgrid);
+ exit(0);
+ */
+ }
+
+ /* Here we start a large thread parallel region */
+#pragma omp parallel num_threads(pme->nthread) private(thread)
+ {
+ thread = gmx_omp_get_thread_num();
+ if (flags & GMX_PME_SOLVE)
+ {
+ int loop_count;
+
+ /* do 3d-fft */
+ if (thread == 0)
+ {
+ wallcycle_start(wcycle, ewcPME_FFT);
+ }
+ gmx_parallel_3dfft_execute(pfft_setup, GMX_FFT_REAL_TO_COMPLEX,
+ thread, wcycle);
+ if (thread == 0)
+ {
+ wallcycle_stop(wcycle, ewcPME_FFT);
+ }
+ where();
+
+ /* solve in k-space for our local cells */
+ if (thread == 0)
+ {
+ wallcycle_start(wcycle, ewcPME_SOLVE);
+ }
+ loop_count =
+ solve_pme_yzx(pme, cfftgrid, ewaldcoeff,
+ box[XX][XX]*box[YY][YY]*box[ZZ][ZZ],
+ bCalcEnerVir,
+ pme->nthread, thread);
+ if (thread == 0)
+ {
+ wallcycle_stop(wcycle, ewcPME_SOLVE);
+ where();
+ inc_nrnb(nrnb, eNR_SOLVEPME, loop_count);
+ }
+ }
+
+ if (bCalcF)
+ {
+ /* do 3d-invfft */
+ if (thread == 0)
+ {
+ where();
+ wallcycle_start(wcycle, ewcPME_FFT);
+ }
+ gmx_parallel_3dfft_execute(pfft_setup, GMX_FFT_COMPLEX_TO_REAL,
+ thread, wcycle);
+ if (thread == 0)
+ {
+ wallcycle_stop(wcycle, ewcPME_FFT);
+
+ where();
+
+ if (pme->nodeid == 0)
+ {
+ ntot = pme->nkx*pme->nky*pme->nkz;
+ npme = ntot*log((real)ntot)/log(2.0);
+ inc_nrnb(nrnb, eNR_FFT, 2*npme);
+ }
+
+ wallcycle_start(wcycle, ewcPME_SPREADGATHER);
+ }
+
+ copy_fftgrid_to_pmegrid(pme, fftgrid, grid, pme->nthread, thread);
+ }
+ }
+ /* End of thread parallel section.
+ * With MPI we have to synchronize here before gmx_sum_qgrid_dd.
+ */
+
+ if (bCalcF)
+ {
+ /* distribute local grid to all nodes */
+#ifdef GMX_MPI
+ if (pme->nnodes > 1)
+ {
+ gmx_sum_qgrid_dd(pme, grid, GMX_SUM_QGRID_BACKWARD);
+ }
+#endif
+ where();
+
+ unwrap_periodic_pmegrid(pme, grid);
+
+ /* interpolate forces for our local atoms */
+
+ where();
+
+ /* If we are running without parallelization,
+ * atc->f is the actual force array, not a buffer,
+ * therefore we should not clear it.
+ */
+ bClearF = (q == 0 && PAR(cr));
+#pragma omp parallel for num_threads(pme->nthread) schedule(static)
+ for (thread = 0; thread < pme->nthread; thread++)
+ {
+ gather_f_bsplines(pme, grid, bClearF, atc,
+ &atc->spline[thread],
+ pme->bFEP ? (q == 0 ? 1.0-lambda : lambda) : 1.0);
+ }
+
+ where();
+
+ inc_nrnb(nrnb, eNR_GATHERFBSP,
+ pme->pme_order*pme->pme_order*pme->pme_order*pme->atc[0].n);
+ wallcycle_stop(wcycle, ewcPME_SPREADGATHER);
+ }
+
+ if (bCalcEnerVir)
+ {
+ /* This should only be called on the master thread
+ * and after the threads have synchronized.
+ */
+ get_pme_ener_vir(pme, pme->nthread, &energy_AB[q], vir_AB[q]);
+ }
+ } /* of q-loop */
+
+ if (bCalcF && pme->nnodes > 1)
+ {
+ wallcycle_start(wcycle, ewcPME_REDISTXF);
+ for (d = 0; d < pme->ndecompdim; d++)
+ {
+ atc = &pme->atc[d];
+ if (d == pme->ndecompdim - 1)
+ {
+ n_d = homenr;
+ f_d = f + start;
+ }
+ else
+ {
+ n_d = pme->atc[d+1].n;
+ f_d = pme->atc[d+1].f;
+ }
+ if (DOMAINDECOMP(cr))
+ {
+ dd_pmeredist_f(pme, atc, n_d, f_d,
+ d == pme->ndecompdim-1 && pme->bPPnode);
+ }
+ else
+ {
+ pmeredist_pd(pme, FALSE, n_d, TRUE, f_d, NULL, atc);
+ }
+ }
+
+ wallcycle_stop(wcycle, ewcPME_REDISTXF);
+ }
+ where();
+
+ if (bCalcEnerVir)
+ {
+ if (!pme->bFEP)
+ {
+ *energy = energy_AB[0];
+ m_add(vir, vir_AB[0], vir);
+ }
+ else
+ {
+ *energy = (1.0-lambda)*energy_AB[0] + lambda*energy_AB[1];
+ *dvdlambda += energy_AB[1] - energy_AB[0];
+ for (i = 0; i < DIM; i++)
+ {
+ for (j = 0; j < DIM; j++)
+ {
+ vir[i][j] += (1.0-lambda)*vir_AB[0][i][j] +
+ lambda*vir_AB[1][i][j];
+ }
+ }
+ }
+ }
+ else
+ {
+ *energy = 0;
+ }
+
+ if (debug)
+ {
+ fprintf(debug, "PME mesh energy: %g\n", *energy);
+ }
+
+ return 0;
+}
--- /dev/null
- df_history_t df_history;
+/*
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * VERSION 3.2.0
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2004, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * Gallium Rubidium Oxygen Manganese Argon Carbon Silicon
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include "typedefs.h"
+#include "smalloc.h"
+#include "sysstuff.h"
+#include "vec.h"
+#include "statutil.h"
+#include "vcm.h"
+#include "mdebin.h"
+#include "nrnb.h"
+#include "calcmu.h"
+#include "index.h"
+#include "vsite.h"
+#include "update.h"
+#include "ns.h"
+#include "trnio.h"
+#include "xtcio.h"
+#include "mdrun.h"
+#include "md_support.h"
+#include "md_logging.h"
+#include "confio.h"
+#include "network.h"
+#include "pull.h"
+#include "xvgr.h"
+#include "physics.h"
+#include "names.h"
+#include "force.h"
+#include "disre.h"
+#include "orires.h"
+#include "pme.h"
+#include "mdatoms.h"
+#include "repl_ex.h"
+#include "qmmm.h"
+#include "domdec.h"
+#include "domdec_network.h"
+#include "partdec.h"
+#include "topsort.h"
+#include "coulomb.h"
+#include "constr.h"
+#include "shellfc.h"
+#include "compute_io.h"
+#include "mvdata.h"
+#include "checkpoint.h"
+#include "mtop_util.h"
+#include "sighandler.h"
+#include "txtdump.h"
+#include "string2.h"
+#include "pme_loadbal.h"
+#include "bondf.h"
+#include "membed.h"
+#include "types/nlistheuristics.h"
+#include "types/iteratedconstraints.h"
+#include "nbnxn_cuda_data_mgmt.h"
+
+#include "gromacs/utility/gmxmpi.h"
+
+#ifdef GMX_FAHCORE
+#include "corewrap.h"
+#endif
+
+static void reset_all_counters(FILE *fplog, t_commrec *cr,
+ gmx_large_int_t step,
+ gmx_large_int_t *step_rel, t_inputrec *ir,
+ gmx_wallcycle_t wcycle, t_nrnb *nrnb,
+ gmx_runtime_t *runtime,
+ nbnxn_cuda_ptr_t cu_nbv)
+{
+ char sbuf[STEPSTRSIZE];
+
+ /* Reset all the counters related to performance over the run */
+ md_print_warn(cr, fplog, "step %s: resetting all time and cycle counters\n",
+ gmx_step_str(step, sbuf));
+
+ if (cu_nbv)
+ {
+ nbnxn_cuda_reset_timings(cu_nbv);
+ }
+
+ wallcycle_stop(wcycle, ewcRUN);
+ wallcycle_reset_all(wcycle);
+ if (DOMAINDECOMP(cr))
+ {
+ reset_dd_statistics_counters(cr->dd);
+ }
+ init_nrnb(nrnb);
+ ir->init_step += *step_rel;
+ ir->nsteps -= *step_rel;
+ *step_rel = 0;
+ wallcycle_start(wcycle, ewcRUN);
+ runtime_start(runtime);
+ print_date_and_time(fplog, cr->nodeid, "Restarted time", runtime);
+}
+
+double do_md(FILE *fplog, t_commrec *cr, int nfile, const t_filenm fnm[],
+ const output_env_t oenv, gmx_bool bVerbose, gmx_bool bCompact,
+ int nstglobalcomm,
+ gmx_vsite_t *vsite, gmx_constr_t constr,
+ int stepout, t_inputrec *ir,
+ gmx_mtop_t *top_global,
+ t_fcdata *fcd,
+ t_state *state_global,
+ t_mdatoms *mdatoms,
+ t_nrnb *nrnb, gmx_wallcycle_t wcycle,
+ gmx_edsam_t ed, t_forcerec *fr,
+ int repl_ex_nst, int repl_ex_nex, int repl_ex_seed, gmx_membed_t membed,
+ real cpt_period, real max_hours,
+ const char gmx_unused *deviceOptions,
+ unsigned long Flags,
+ gmx_runtime_t *runtime)
+{
+ gmx_mdoutf_t *outf;
+ gmx_large_int_t step, step_rel;
+ double run_time;
+ double t, t0, lam0[efptNR];
+ gmx_bool bGStatEveryStep, bGStat, bCalcVir, bCalcEner;
+ gmx_bool bNS, bNStList, bSimAnn, bStopCM, bRerunMD, bNotLastFrame = FALSE,
+ bFirstStep, bStateFromCP, bStateFromTPX, bInitStep, bLastStep,
+ bBornRadii, bStartingFromCpt;
+ gmx_bool bDoDHDL = FALSE, bDoFEP = FALSE, bDoExpanded = FALSE;
+ gmx_bool do_ene, do_log, do_verbose, bRerunWarnNoV = TRUE,
+ bForceUpdate = FALSE, bCPT;
+ gmx_bool bMasterState;
+ int force_flags, cglo_flags;
+ tensor force_vir, shake_vir, total_vir, tmp_vir, pres;
+ int i, m;
+ t_trxstatus *status;
+ rvec mu_tot;
+ t_vcm *vcm;
+ t_state *bufstate = NULL;
+ matrix *scale_tot, pcoupl_mu, M, ebox;
+ gmx_nlheur_t nlh;
+ t_trxframe rerun_fr;
+ gmx_repl_ex_t repl_ex = NULL;
+ int nchkpt = 1;
+ gmx_localtop_t *top;
+ t_mdebin *mdebin = NULL;
- /* lambda Monte carlo random number generator */
- if (ir->bExpanded)
- {
- mcrng = gmx_rng_init(ir->expandedvals->lmc_seed);
- }
- /* copy the state into df_history */
- copy_df_history(&df_history, &state_global->dfhist);
-
+ t_state *state = NULL;
+ rvec *f_global = NULL;
+ gmx_enerdata_t *enerd;
+ rvec *f = NULL;
+ gmx_global_stat_t gstat;
+ gmx_update_t upd = NULL;
+ t_graph *graph = NULL;
+ globsig_t gs;
+ gmx_rng_t mcrng = NULL;
+ gmx_groups_t *groups;
+ gmx_ekindata_t *ekind, *ekind_save;
+ gmx_shellfc_t shellfc;
+ int count, nconverged = 0;
+ real timestep = 0;
+ double tcount = 0;
+ gmx_bool bConverged = TRUE, bOK, bSumEkinhOld, bExchanged;
+ gmx_bool bAppend;
+ gmx_bool bResetCountersHalfMaxH = FALSE;
+ gmx_bool bVV, bIterativeCase, bFirstIterate, bTemp, bPres, bTrotter;
+ gmx_bool bUpdateDoLR;
+ real dvdl_constr;
+ int a0, a1;
+ rvec *cbuf = NULL;
+ matrix lastbox;
+ real veta_save, scalevir, tracevir;
+ real vetanew = 0;
+ int lamnew = 0;
+ /* for FEP */
+ int nstfep;
+ double cycles;
+ real saved_conserved_quantity = 0;
+ real last_ekin = 0;
+ int iter_i;
+ t_extmass MassQ;
+ int **trotter_seq;
+ char sbuf[STEPSTRSIZE], sbuf2[STEPSTRSIZE];
+ int handled_stop_condition = gmx_stop_cond_none; /* compare to get_stop_condition*/
+ gmx_iterate_t iterate;
+ gmx_large_int_t multisim_nsteps = -1; /* number of steps to do before first multisim
+ simulation stops. If equal to zero, don't
+ communicate any more between multisims.*/
+ /* PME load balancing data for GPU kernels */
+ pme_load_balancing_t pme_loadbal = NULL;
+ double cycles_pmes;
+ gmx_bool bPMETuneTry = FALSE, bPMETuneRunning = FALSE;
+
+#ifdef GMX_FAHCORE
+ /* Temporary addition for FAHCORE checkpointing */
+ int chkpt_ret;
+#endif
+
+ /* Check for special mdrun options */
+ bRerunMD = (Flags & MD_RERUN);
+ bAppend = (Flags & MD_APPENDFILES);
+ if (Flags & MD_RESETCOUNTERSHALFWAY)
+ {
+ if (ir->nsteps > 0)
+ {
+ /* Signal to reset the counters half the simulation steps. */
+ wcycle_set_reset_counters(wcycle, ir->nsteps/2);
+ }
+ /* Signal to reset the counters halfway the simulation time. */
+ bResetCountersHalfMaxH = (max_hours > 0);
+ }
+
+ /* md-vv uses averaged full step velocities for T-control
+ md-vv-avek uses averaged half step velocities for T-control (but full step ekin for P control)
+ md uses averaged half step kinetic energies to determine temperature unless defined otherwise by GMX_EKIN_AVE_VEL; */
+ bVV = EI_VV(ir->eI);
+ if (bVV) /* to store the initial velocities while computing virial */
+ {
+ snew(cbuf, top_global->natoms);
+ }
+ /* all the iteratative cases - only if there are constraints */
+ bIterativeCase = ((IR_NPH_TROTTER(ir) || IR_NPT_TROTTER(ir)) && (constr) && (!bRerunMD));
+ gmx_iterate_init(&iterate, FALSE); /* The default value of iterate->bIterationActive is set to
+ false in this step. The correct value, true or false,
+ is set at each step, as it depends on the frequency of temperature
+ and pressure control.*/
+ bTrotter = (bVV && (IR_NPT_TROTTER(ir) || IR_NPH_TROTTER(ir) || IR_NVT_TROTTER(ir)));
+
+ if (bRerunMD)
+ {
+ /* Since we don't know if the frames read are related in any way,
+ * rebuild the neighborlist at every step.
+ */
+ ir->nstlist = 1;
+ ir->nstcalcenergy = 1;
+ nstglobalcomm = 1;
+ }
+
+ check_ir_old_tpx_versions(cr, fplog, ir, top_global);
+
+ nstglobalcomm = check_nstglobalcomm(fplog, cr, nstglobalcomm, ir);
+ bGStatEveryStep = (nstglobalcomm == 1);
+
+ if (!bGStatEveryStep && ir->nstlist == -1 && fplog != NULL)
+ {
+ fprintf(fplog,
+ "To reduce the energy communication with nstlist = -1\n"
+ "the neighbor list validity should not be checked at every step,\n"
+ "this means that exact integration is not guaranteed.\n"
+ "The neighbor list validity is checked after:\n"
+ " <n.list life time> - 2*std.dev.(n.list life time) steps.\n"
+ "In most cases this will result in exact integration.\n"
+ "This reduces the energy communication by a factor of 2 to 3.\n"
+ "If you want less energy communication, set nstlist > 3.\n\n");
+ }
+
+ if (bRerunMD)
+ {
+ ir->nstxtcout = 0;
+ }
+ groups = &top_global->groups;
+
+ /* Initial values */
+ init_md(fplog, cr, ir, oenv, &t, &t0, state_global->lambda,
+ &(state_global->fep_state), lam0,
+ nrnb, top_global, &upd,
+ nfile, fnm, &outf, &mdebin,
+ force_vir, shake_vir, mu_tot, &bSimAnn, &vcm, Flags);
+
+ clear_mat(total_vir);
+ clear_mat(pres);
+ /* Energy terms and groups */
+ snew(enerd, 1);
+ init_enerdata(top_global->groups.grps[egcENER].nr, ir->fepvals->n_lambda,
+ enerd);
+ if (DOMAINDECOMP(cr))
+ {
+ f = NULL;
+ }
+ else
+ {
+ snew(f, top_global->natoms);
+ }
+
- bDoExpanded = (do_per_step(step, ir->expandedvals->nstexpanded) && (ir->bExpanded) && (step > 0));
+ /* Kinetic energy data */
+ snew(ekind, 1);
+ init_ekindata(fplog, top_global, &(ir->opts), ekind);
+ /* needed for iteration of constraints */
+ snew(ekind_save, 1);
+ init_ekindata(fplog, top_global, &(ir->opts), ekind_save);
+ /* Copy the cos acceleration to the groups struct */
+ ekind->cosacc.cos_accel = ir->cos_accel;
+
+ gstat = global_stat_init(ir);
+ debug_gmx();
+
+ /* Check for polarizable models and flexible constraints */
+ shellfc = init_shell_flexcon(fplog,
+ top_global, n_flexible_constraints(constr),
+ (ir->bContinuation ||
+ (DOMAINDECOMP(cr) && !MASTER(cr))) ?
+ NULL : state_global->x);
+
+ if (DEFORM(*ir))
+ {
+#ifdef GMX_THREAD_MPI
+ tMPI_Thread_mutex_lock(&deform_init_box_mutex);
+#endif
+ set_deform_reference_box(upd,
+ deform_init_init_step_tpx,
+ deform_init_box_tpx);
+#ifdef GMX_THREAD_MPI
+ tMPI_Thread_mutex_unlock(&deform_init_box_mutex);
+#endif
+ }
+
+ {
+ double io = compute_io(ir, top_global->natoms, groups, mdebin->ebin->nener, 1);
+ if ((io > 2000) && MASTER(cr))
+ {
+ fprintf(stderr,
+ "\nWARNING: This run will generate roughly %.0f Mb of data\n\n",
+ io);
+ }
+ }
+
+ if (DOMAINDECOMP(cr))
+ {
+ top = dd_init_local_top(top_global);
+
+ snew(state, 1);
+ dd_init_local_state(cr->dd, state_global, state);
+
+ if (DDMASTER(cr->dd) && ir->nstfout)
+ {
+ snew(f_global, state_global->natoms);
+ }
+ }
+ else
+ {
+ if (PAR(cr))
+ {
+ /* Initialize the particle decomposition and split the topology */
+ top = split_system(fplog, top_global, ir, cr);
+
+ pd_cg_range(cr, &fr->cg0, &fr->hcg);
+ pd_at_range(cr, &a0, &a1);
+ }
+ else
+ {
+ top = gmx_mtop_generate_local_top(top_global, ir);
+
+ a0 = 0;
+ a1 = top_global->natoms;
+ }
+
+ forcerec_set_excl_load(fr, top, cr);
+
+ state = partdec_init_local_state(cr, state_global);
+ f_global = f;
+
+ atoms2md(top_global, ir, 0, NULL, a0, a1-a0, mdatoms);
+
+ if (vsite)
+ {
+ set_vsite_top(vsite, top, mdatoms, cr);
+ }
+
+ if (ir->ePBC != epbcNONE && !fr->bMolPBC)
+ {
+ graph = mk_graph(fplog, &(top->idef), 0, top_global->natoms, FALSE, FALSE);
+ }
+
+ if (shellfc)
+ {
+ make_local_shells(cr, mdatoms, shellfc);
+ }
+
+ setup_bonded_threading(fr, &top->idef);
+
+ if (ir->pull && PAR(cr))
+ {
+ dd_make_local_pull_groups(NULL, ir->pull, mdatoms);
+ }
+ }
+
+ if (DOMAINDECOMP(cr))
+ {
+ /* Distribute the charge groups over the nodes from the master node */
+ dd_partition_system(fplog, ir->init_step, cr, TRUE, 1,
+ state_global, top_global, ir,
+ state, &f, mdatoms, top, fr,
+ vsite, shellfc, constr,
+ nrnb, wcycle, FALSE);
+
+ }
+
+ update_mdatoms(mdatoms, state->lambda[efptMASS]);
+
+ if (opt2bSet("-cpi", nfile, fnm))
+ {
+ bStateFromCP = gmx_fexist_master(opt2fn_master("-cpi", nfile, fnm, cr), cr);
+ }
+ else
+ {
+ bStateFromCP = FALSE;
+ }
+
++ if (ir->bExpanded)
++ {
++ init_expanded_ensemble(bStateFromCP,ir,&mcrng,&state->dfhist);
++ }
++
+ if (MASTER(cr))
+ {
+ if (bStateFromCP)
+ {
+ /* Update mdebin with energy history if appending to output files */
+ if (Flags & MD_APPENDFILES)
+ {
+ restore_energyhistory_from_state(mdebin, &state_global->enerhist);
+ }
+ else
+ {
+ /* We might have read an energy history from checkpoint,
+ * free the allocated memory and reset the counts.
+ */
+ done_energyhistory(&state_global->enerhist);
+ init_energyhistory(&state_global->enerhist);
+ }
+ }
+ /* Set the initial energy history in state by updating once */
+ update_energyhistory(&state_global->enerhist, mdebin);
+ }
+
+ if ((state->flags & (1<<estLD_RNG)) && (Flags & MD_READ_RNG))
+ {
+ /* Set the random state if we read a checkpoint file */
+ set_stochd_state(upd, state);
+ }
+
+ if (state->flags & (1<<estMC_RNG))
+ {
+ set_mc_state(mcrng, state);
+ }
+
+ /* Initialize constraints */
+ if (constr)
+ {
+ if (!DOMAINDECOMP(cr))
+ {
+ set_constraints(constr, top, ir, mdatoms, cr);
+ }
+ }
+
+ if (repl_ex_nst > 0)
+ {
+ /* We need to be sure replica exchange can only occur
+ * when the energies are current */
+ check_nst_param(fplog, cr, "nstcalcenergy", ir->nstcalcenergy,
+ "repl_ex_nst", &repl_ex_nst);
+ /* This check needs to happen before inter-simulation
+ * signals are initialized, too */
+ }
+ if (repl_ex_nst > 0 && MASTER(cr))
+ {
+ repl_ex = init_replica_exchange(fplog, cr->ms, state_global, ir,
+ repl_ex_nst, repl_ex_nex, repl_ex_seed);
+ }
+
+ /* PME tuning is only supported with GPUs or PME nodes and not with rerun.
+ * With perturbed charges with soft-core we should not change the cut-off.
+ */
+ if ((Flags & MD_TUNEPME) &&
+ EEL_PME(fr->eeltype) &&
+ ( (fr->cutoff_scheme == ecutsVERLET && fr->nbv->bUseGPU) || !(cr->duty & DUTY_PME)) &&
+ !(ir->efep != efepNO && mdatoms->nChargePerturbed > 0 && ir->fepvals->bScCoul) &&
+ !bRerunMD)
+ {
+ pme_loadbal_init(&pme_loadbal, ir, state->box, fr->ic, fr->pmedata);
+ cycles_pmes = 0;
+ if (cr->duty & DUTY_PME)
+ {
+ /* Start tuning right away, as we can't measure the load */
+ bPMETuneRunning = TRUE;
+ }
+ else
+ {
+ /* Separate PME nodes, we can measure the PP/PME load balance */
+ bPMETuneTry = TRUE;
+ }
+ }
+
+ if (!ir->bContinuation && !bRerunMD)
+ {
+ if (mdatoms->cFREEZE && (state->flags & (1<<estV)))
+ {
+ /* Set the velocities of frozen particles to zero */
+ for (i = mdatoms->start; i < mdatoms->start+mdatoms->homenr; i++)
+ {
+ for (m = 0; m < DIM; m++)
+ {
+ if (ir->opts.nFreeze[mdatoms->cFREEZE[i]][m])
+ {
+ state->v[i][m] = 0;
+ }
+ }
+ }
+ }
+
+ if (constr)
+ {
+ /* Constrain the initial coordinates and velocities */
+ do_constrain_first(fplog, constr, ir, mdatoms, state,
+ cr, nrnb, fr, top);
+ }
+ if (vsite)
+ {
+ /* Construct the virtual sites for the initial configuration */
+ construct_vsites(vsite, state->x, ir->delta_t, NULL,
+ top->idef.iparams, top->idef.il,
+ fr->ePBC, fr->bMolPBC, graph, cr, state->box);
+ }
+ }
+
+ debug_gmx();
+
+ /* set free energy calculation frequency as the minimum
+ greatest common denominator of nstdhdl, nstexpanded, and repl_ex_nst*/
+ nstfep = ir->fepvals->nstdhdl;
+ if (ir->bExpanded)
+ {
+ nstfep = gmx_greatest_common_divisor(ir->fepvals->nstdhdl, nstfep);
+ }
+ if (repl_ex_nst > 0)
+ {
+ nstfep = gmx_greatest_common_divisor(repl_ex_nst, nstfep);
+ }
+
+ /* I'm assuming we need global communication the first time! MRS */
+ cglo_flags = (CGLO_TEMPERATURE | CGLO_GSTAT
+ | ((ir->comm_mode != ecmNO) ? CGLO_STOPCM : 0)
+ | (bVV ? CGLO_PRESSURE : 0)
+ | (bVV ? CGLO_CONSTRAINT : 0)
+ | (bRerunMD ? CGLO_RERUNMD : 0)
+ | ((Flags & MD_READ_EKIN) ? CGLO_READEKIN : 0));
+
+ bSumEkinhOld = FALSE;
+ compute_globals(fplog, gstat, cr, ir, fr, ekind, state, state_global, mdatoms, nrnb, vcm,
+ NULL, enerd, force_vir, shake_vir, total_vir, pres, mu_tot,
+ constr, NULL, FALSE, state->box,
+ top_global, &bSumEkinhOld, cglo_flags);
+ if (ir->eI == eiVVAK)
+ {
+ /* a second call to get the half step temperature initialized as well */
+ /* we do the same call as above, but turn the pressure off -- internally to
+ compute_globals, this is recognized as a velocity verlet half-step
+ kinetic energy calculation. This minimized excess variables, but
+ perhaps loses some logic?*/
+
+ compute_globals(fplog, gstat, cr, ir, fr, ekind, state, state_global, mdatoms, nrnb, vcm,
+ NULL, enerd, force_vir, shake_vir, total_vir, pres, mu_tot,
+ constr, NULL, FALSE, state->box,
+ top_global, &bSumEkinhOld,
+ cglo_flags &~(CGLO_STOPCM | CGLO_PRESSURE));
+ }
+
+ /* Calculate the initial half step temperature, and save the ekinh_old */
+ if (!(Flags & MD_STARTFROMCPT))
+ {
+ for (i = 0; (i < ir->opts.ngtc); i++)
+ {
+ copy_mat(ekind->tcstat[i].ekinh, ekind->tcstat[i].ekinh_old);
+ }
+ }
+ if (ir->eI != eiVV)
+ {
+ enerd->term[F_TEMP] *= 2; /* result of averages being done over previous and current step,
+ and there is no previous step */
+ }
+
+ /* if using an iterative algorithm, we need to create a working directory for the state. */
+ if (bIterativeCase)
+ {
+ bufstate = init_bufstate(state);
+ }
+
+ /* need to make an initiation call to get the Trotter variables set, as well as other constants for non-trotter
+ temperature control */
+ trotter_seq = init_npt_vars(ir, state, &MassQ, bTrotter);
+
+ if (MASTER(cr))
+ {
+ if (constr && !ir->bContinuation && ir->eConstrAlg == econtLINCS)
+ {
+ fprintf(fplog,
+ "RMS relative constraint deviation after constraining: %.2e\n",
+ constr_rmsd(constr, FALSE));
+ }
+ if (EI_STATE_VELOCITY(ir->eI))
+ {
+ fprintf(fplog, "Initial temperature: %g K\n", enerd->term[F_TEMP]);
+ }
+ if (bRerunMD)
+ {
+ fprintf(stderr, "starting md rerun '%s', reading coordinates from"
+ " input trajectory '%s'\n\n",
+ *(top_global->name), opt2fn("-rerun", nfile, fnm));
+ if (bVerbose)
+ {
+ fprintf(stderr, "Calculated time to finish depends on nsteps from "
+ "run input file,\nwhich may not correspond to the time "
+ "needed to process input trajectory.\n\n");
+ }
+ }
+ else
+ {
+ char tbuf[20];
+ fprintf(stderr, "starting mdrun '%s'\n",
+ *(top_global->name));
+ if (ir->nsteps >= 0)
+ {
+ sprintf(tbuf, "%8.1f", (ir->init_step+ir->nsteps)*ir->delta_t);
+ }
+ else
+ {
+ sprintf(tbuf, "%s", "infinite");
+ }
+ if (ir->init_step > 0)
+ {
+ fprintf(stderr, "%s steps, %s ps (continuing from step %s, %8.1f ps).\n",
+ gmx_step_str(ir->init_step+ir->nsteps, sbuf), tbuf,
+ gmx_step_str(ir->init_step, sbuf2),
+ ir->init_step*ir->delta_t);
+ }
+ else
+ {
+ fprintf(stderr, "%s steps, %s ps.\n",
+ gmx_step_str(ir->nsteps, sbuf), tbuf);
+ }
+ }
+ fprintf(fplog, "\n");
+ }
+
+ /* Set and write start time */
+ runtime_start(runtime);
+ print_date_and_time(fplog, cr->nodeid, "Started mdrun", runtime);
+ wallcycle_start(wcycle, ewcRUN);
+ if (fplog)
+ {
+ fprintf(fplog, "\n");
+ }
+
+ /* safest point to do file checkpointing is here. More general point would be immediately before integrator call */
+#ifdef GMX_FAHCORE
+ chkpt_ret = fcCheckPointParallel( cr->nodeid,
+ NULL, 0);
+ if (chkpt_ret == 0)
+ {
+ gmx_fatal( 3, __FILE__, __LINE__, "Checkpoint error on step %d\n", 0 );
+ }
+#endif
+
+ debug_gmx();
+ /***********************************************************
+ *
+ * Loop over MD steps
+ *
+ ************************************************************/
+
+ /* if rerunMD then read coordinates and velocities from input trajectory */
+ if (bRerunMD)
+ {
+ if (getenv("GMX_FORCE_UPDATE"))
+ {
+ bForceUpdate = TRUE;
+ }
+
+ rerun_fr.natoms = 0;
+ if (MASTER(cr))
+ {
+ bNotLastFrame = read_first_frame(oenv, &status,
+ opt2fn("-rerun", nfile, fnm),
+ &rerun_fr, TRX_NEED_X | TRX_READ_V);
+ if (rerun_fr.natoms != top_global->natoms)
+ {
+ gmx_fatal(FARGS,
+ "Number of atoms in trajectory (%d) does not match the "
+ "run input file (%d)\n",
+ rerun_fr.natoms, top_global->natoms);
+ }
+ if (ir->ePBC != epbcNONE)
+ {
+ if (!rerun_fr.bBox)
+ {
+ gmx_fatal(FARGS, "Rerun trajectory frame step %d time %f does not contain a box, while pbc is used", rerun_fr.step, rerun_fr.time);
+ }
+ if (max_cutoff2(ir->ePBC, rerun_fr.box) < sqr(fr->rlistlong))
+ {
+ gmx_fatal(FARGS, "Rerun trajectory frame step %d time %f has too small box dimensions", rerun_fr.step, rerun_fr.time);
+ }
+ }
+ }
+
+ if (PAR(cr))
+ {
+ rerun_parallel_comm(cr, &rerun_fr, &bNotLastFrame);
+ }
+
+ if (ir->ePBC != epbcNONE)
+ {
+ /* Set the shift vectors.
+ * Necessary here when have a static box different from the tpr box.
+ */
+ calc_shifts(rerun_fr.box, fr->shift_vec);
+ }
+ }
+
+ /* loop over MD steps or if rerunMD to end of input trajectory */
+ bFirstStep = TRUE;
+ /* Skip the first Nose-Hoover integration when we get the state from tpx */
+ bStateFromTPX = !bStateFromCP;
+ bInitStep = bFirstStep && (bStateFromTPX || bVV);
+ bStartingFromCpt = (Flags & MD_STARTFROMCPT) && bInitStep;
+ bLastStep = FALSE;
+ bSumEkinhOld = FALSE;
+ bExchanged = FALSE;
+
+ init_global_signals(&gs, cr, ir, repl_ex_nst);
+
+ step = ir->init_step;
+ step_rel = 0;
+
+ if (ir->nstlist == -1)
+ {
+ init_nlistheuristics(&nlh, bGStatEveryStep, step);
+ }
+
+ if (MULTISIM(cr) && (repl_ex_nst <= 0 ))
+ {
+ /* check how many steps are left in other sims */
+ multisim_nsteps = get_multisim_nsteps(cr, ir->nsteps);
+ }
+
+
+ /* and stop now if we should */
+ bLastStep = (bRerunMD || (ir->nsteps >= 0 && step_rel > ir->nsteps) ||
+ ((multisim_nsteps >= 0) && (step_rel >= multisim_nsteps )));
+ while (!bLastStep || (bRerunMD && bNotLastFrame))
+ {
+
+ wallcycle_start(wcycle, ewcSTEP);
+
+ if (bRerunMD)
+ {
+ if (rerun_fr.bStep)
+ {
+ step = rerun_fr.step;
+ step_rel = step - ir->init_step;
+ }
+ if (rerun_fr.bTime)
+ {
+ t = rerun_fr.time;
+ }
+ else
+ {
+ t = step;
+ }
+ }
+ else
+ {
+ bLastStep = (step_rel == ir->nsteps);
+ t = t0 + step*ir->delta_t;
+ }
+
+ if (ir->efep != efepNO || ir->bSimTemp)
+ {
+ /* find and set the current lambdas. If rerunning, we either read in a state, or a lambda value,
+ requiring different logic. */
+
+ set_current_lambdas(step, ir->fepvals, bRerunMD, &rerun_fr, state_global, state, lam0);
+ bDoDHDL = do_per_step(step, ir->fepvals->nstdhdl);
+ bDoFEP = (do_per_step(step, nstfep) && (ir->efep != efepNO));
- lamnew = ExpandedEnsembleDynamics(fplog, ir, enerd, state, &MassQ, &df_history, step, mcrng, state->v, mdatoms);
++ bDoExpanded = (do_per_step(step, ir->expandedvals->nstexpanded)
++ && (ir->bExpanded) && (step > 0) && (!bStartingFromCpt));
+ }
+
+ if (bSimAnn)
+ {
+ update_annealing_target_temp(&(ir->opts), t);
+ }
+
+ if (bRerunMD)
+ {
+ if (!(DOMAINDECOMP(cr) && !MASTER(cr)))
+ {
+ for (i = 0; i < state_global->natoms; i++)
+ {
+ copy_rvec(rerun_fr.x[i], state_global->x[i]);
+ }
+ if (rerun_fr.bV)
+ {
+ for (i = 0; i < state_global->natoms; i++)
+ {
+ copy_rvec(rerun_fr.v[i], state_global->v[i]);
+ }
+ }
+ else
+ {
+ for (i = 0; i < state_global->natoms; i++)
+ {
+ clear_rvec(state_global->v[i]);
+ }
+ if (bRerunWarnNoV)
+ {
+ fprintf(stderr, "\nWARNING: Some frames do not contain velocities.\n"
+ " Ekin, temperature and pressure are incorrect,\n"
+ " the virial will be incorrect when constraints are present.\n"
+ "\n");
+ bRerunWarnNoV = FALSE;
+ }
+ }
+ }
+ copy_mat(rerun_fr.box, state_global->box);
+ copy_mat(state_global->box, state->box);
+
+ if (vsite && (Flags & MD_RERUN_VSITE))
+ {
+ if (DOMAINDECOMP(cr))
+ {
+ gmx_fatal(FARGS, "Vsite recalculation with -rerun is not implemented for domain decomposition, use particle decomposition");
+ }
+ if (graph)
+ {
+ /* Following is necessary because the graph may get out of sync
+ * with the coordinates if we only have every N'th coordinate set
+ */
+ mk_mshift(fplog, graph, fr->ePBC, state->box, state->x);
+ shift_self(graph, state->box, state->x);
+ }
+ construct_vsites(vsite, state->x, ir->delta_t, state->v,
+ top->idef.iparams, top->idef.il,
+ fr->ePBC, fr->bMolPBC, graph, cr, state->box);
+ if (graph)
+ {
+ unshift_self(graph, state->box, state->x);
+ }
+ }
+ }
+
+ /* Stop Center of Mass motion */
+ bStopCM = (ir->comm_mode != ecmNO && do_per_step(step, ir->nstcomm));
+
+ if (bRerunMD)
+ {
+ /* for rerun MD always do Neighbour Searching */
+ bNS = (bFirstStep || ir->nstlist != 0);
+ bNStList = bNS;
+ }
+ else
+ {
+ /* Determine whether or not to do Neighbour Searching and LR */
+ bNStList = (ir->nstlist > 0 && step % ir->nstlist == 0);
+
+ bNS = (bFirstStep || bExchanged || bNStList || bDoFEP ||
+ (ir->nstlist == -1 && nlh.nabnsb > 0));
+
+ if (bNS && ir->nstlist == -1)
+ {
+ set_nlistheuristics(&nlh, bFirstStep || bExchanged || bDoFEP, step);
+ }
+ }
+
+ /* check whether we should stop because another simulation has
+ stopped. */
+ if (MULTISIM(cr))
+ {
+ if ( (multisim_nsteps >= 0) && (step_rel >= multisim_nsteps) &&
+ (multisim_nsteps != ir->nsteps) )
+ {
+ if (bNS)
+ {
+ if (MASTER(cr))
+ {
+ fprintf(stderr,
+ "Stopping simulation %d because another one has finished\n",
+ cr->ms->sim);
+ }
+ bLastStep = TRUE;
+ gs.sig[eglsCHKPT] = 1;
+ }
+ }
+ }
+
+ /* < 0 means stop at next step, > 0 means stop at next NS step */
+ if ( (gs.set[eglsSTOPCOND] < 0) ||
+ ( (gs.set[eglsSTOPCOND] > 0) && (bNStList || ir->nstlist == 0) ) )
+ {
+ bLastStep = TRUE;
+ }
+
+ /* Determine whether or not to update the Born radii if doing GB */
+ bBornRadii = bFirstStep;
+ if (ir->implicit_solvent && (step % ir->nstgbradii == 0))
+ {
+ bBornRadii = TRUE;
+ }
+
+ do_log = do_per_step(step, ir->nstlog) || bFirstStep || bLastStep;
+ do_verbose = bVerbose &&
+ (step % stepout == 0 || bFirstStep || bLastStep);
+
+ if (bNS && !(bFirstStep && ir->bContinuation && !bRerunMD))
+ {
+ if (bRerunMD)
+ {
+ bMasterState = TRUE;
+ }
+ else
+ {
+ bMasterState = FALSE;
+ /* Correct the new box if it is too skewed */
+ if (DYNAMIC_BOX(*ir))
+ {
+ if (correct_box(fplog, step, state->box, graph))
+ {
+ bMasterState = TRUE;
+ }
+ }
+ if (DOMAINDECOMP(cr) && bMasterState)
+ {
+ dd_collect_state(cr->dd, state, state_global);
+ }
+ }
+
+ if (DOMAINDECOMP(cr))
+ {
+ /* Repartition the domain decomposition */
+ wallcycle_start(wcycle, ewcDOMDEC);
+ dd_partition_system(fplog, step, cr,
+ bMasterState, nstglobalcomm,
+ state_global, top_global, ir,
+ state, &f, mdatoms, top, fr,
+ vsite, shellfc, constr,
+ nrnb, wcycle,
+ do_verbose && !bPMETuneRunning);
+ wallcycle_stop(wcycle, ewcDOMDEC);
+ /* If using an iterative integrator, reallocate space to match the decomposition */
+ }
+ }
+
+ if (MASTER(cr) && do_log)
+ {
+ print_ebin_header(fplog, step, t, state->lambda[efptFEP]); /* can we improve the information printed here? */
+ }
+
+ if (ir->efep != efepNO)
+ {
+ update_mdatoms(mdatoms, state->lambda[efptMASS]);
+ }
+
+ if ((bRerunMD && rerun_fr.bV) || bExchanged)
+ {
+
+ /* We need the kinetic energy at minus the half step for determining
+ * the full step kinetic energy and possibly for T-coupling.*/
+ /* This may not be quite working correctly yet . . . . */
+ compute_globals(fplog, gstat, cr, ir, fr, ekind, state, state_global, mdatoms, nrnb, vcm,
+ wcycle, enerd, NULL, NULL, NULL, NULL, mu_tot,
+ constr, NULL, FALSE, state->box,
+ top_global, &bSumEkinhOld,
+ CGLO_RERUNMD | CGLO_GSTAT | CGLO_TEMPERATURE);
+ }
+ clear_mat(force_vir);
+
+ /* We write a checkpoint at this MD step when:
+ * either at an NS step when we signalled through gs,
+ * or at the last step (but not when we do not want confout),
+ * but never at the first step or with rerun.
+ */
+ bCPT = (((gs.set[eglsCHKPT] && (bNS || ir->nstlist == 0)) ||
+ (bLastStep && (Flags & MD_CONFOUT))) &&
+ step > ir->init_step && !bRerunMD);
+ if (bCPT)
+ {
+ gs.set[eglsCHKPT] = 0;
+ }
+
+ /* Determine the energy and pressure:
+ * at nstcalcenergy steps and at energy output steps (set below).
+ */
+ if (EI_VV(ir->eI) && (!bInitStep))
+ {
+ /* for vv, the first half of the integration actually corresponds
+ to the previous step. bCalcEner is only required to be evaluated on the 'next' step,
+ but the virial needs to be calculated on both the current step and the 'next' step. Future
+ reorganization may be able to get rid of one of the bCalcVir=TRUE steps. */
+
+ bCalcEner = do_per_step(step-1, ir->nstcalcenergy);
+ bCalcVir = bCalcEner ||
+ (ir->epc != epcNO && (do_per_step(step, ir->nstpcouple) || do_per_step(step-1, ir->nstpcouple)));
+ }
+ else
+ {
+ bCalcEner = do_per_step(step, ir->nstcalcenergy);
+ bCalcVir = bCalcEner ||
+ (ir->epc != epcNO && do_per_step(step, ir->nstpcouple));
+ }
+
+ /* Do we need global communication ? */
+ bGStat = (bCalcVir || bCalcEner || bStopCM ||
+ do_per_step(step, nstglobalcomm) || (bVV && IR_NVT_TROTTER(ir) && do_per_step(step-1, nstglobalcomm)) ||
+ (ir->nstlist == -1 && !bRerunMD && step >= nlh.step_nscheck));
+
+ do_ene = (do_per_step(step, ir->nstenergy) || bLastStep);
+
+ if (do_ene || do_log)
+ {
+ bCalcVir = TRUE;
+ bCalcEner = TRUE;
+ bGStat = TRUE;
+ }
+
+ /* these CGLO_ options remain the same throughout the iteration */
+ cglo_flags = ((bRerunMD ? CGLO_RERUNMD : 0) |
+ (bGStat ? CGLO_GSTAT : 0)
+ );
+
+ force_flags = (GMX_FORCE_STATECHANGED |
+ ((DYNAMIC_BOX(*ir) || bRerunMD) ? GMX_FORCE_DYNAMICBOX : 0) |
+ GMX_FORCE_ALLFORCES |
+ GMX_FORCE_SEPLRF |
+ (bCalcVir ? GMX_FORCE_VIRIAL : 0) |
+ (bCalcEner ? GMX_FORCE_ENERGY : 0) |
+ (bDoFEP ? GMX_FORCE_DHDL : 0)
+ );
+
+ if (fr->bTwinRange)
+ {
+ if (do_per_step(step, ir->nstcalclr))
+ {
+ force_flags |= GMX_FORCE_DO_LR;
+ }
+ }
+
+ if (shellfc)
+ {
+ /* Now is the time to relax the shells */
+ count = relax_shell_flexcon(fplog, cr, bVerbose, step,
+ ir, bNS, force_flags,
+ top,
+ constr, enerd, fcd,
+ state, f, force_vir, mdatoms,
+ nrnb, wcycle, graph, groups,
+ shellfc, fr, bBornRadii, t, mu_tot,
+ &bConverged, vsite,
+ outf->fp_field);
+ tcount += count;
+
+ if (bConverged)
+ {
+ nconverged++;
+ }
+ }
+ else
+ {
+ /* The coordinates (x) are shifted (to get whole molecules)
+ * in do_force.
+ * This is parallellized as well, and does communication too.
+ * Check comments in sim_util.c
+ */
+ do_force(fplog, cr, ir, step, nrnb, wcycle, top, groups,
+ state->box, state->x, &state->hist,
+ f, force_vir, mdatoms, enerd, fcd,
+ state->lambda, graph,
+ fr, vsite, mu_tot, t, outf->fp_field, ed, bBornRadii,
+ (bNS ? GMX_FORCE_NS : 0) | force_flags);
+ }
+
+ if (bVV && !bStartingFromCpt && !bRerunMD)
+ /* ############### START FIRST UPDATE HALF-STEP FOR VV METHODS############### */
+ {
+ if (ir->eI == eiVV && bInitStep)
+ {
+ /* if using velocity verlet with full time step Ekin,
+ * take the first half step only to compute the
+ * virial for the first step. From there,
+ * revert back to the initial coordinates
+ * so that the input is actually the initial step.
+ */
+ copy_rvecn(state->v, cbuf, 0, state->natoms); /* should make this better for parallelizing? */
+ }
+ else
+ {
+ /* this is for NHC in the Ekin(t+dt/2) version of vv */
+ trotter_update(ir, step, ekind, enerd, state, total_vir, mdatoms, &MassQ, trotter_seq, ettTSEQ1);
+ }
+
+ /* If we are using twin-range interactions where the long-range component
+ * is only evaluated every nstcalclr>1 steps, we should do a special update
+ * step to combine the long-range forces on these steps.
+ * For nstcalclr=1 this is not done, since the forces would have been added
+ * directly to the short-range forces already.
+ */
+ bUpdateDoLR = (fr->bTwinRange && do_per_step(step, ir->nstcalclr));
+
+ update_coords(fplog, step, ir, mdatoms, state, fr->bMolPBC,
+ f, bUpdateDoLR, fr->f_twin, fcd,
+ ekind, M, upd, bInitStep, etrtVELOCITY1,
+ cr, nrnb, constr, &top->idef);
+
+ if (bIterativeCase && do_per_step(step-1, ir->nstpcouple) && !bInitStep)
+ {
+ gmx_iterate_init(&iterate, TRUE);
+ }
+ /* for iterations, we save these vectors, as we will be self-consistently iterating
+ the calculations */
+
+ /*#### UPDATE EXTENDED VARIABLES IN TROTTER FORMULATION */
+
+ /* save the state */
+ if (iterate.bIterationActive)
+ {
+ copy_coupling_state(state, bufstate, ekind, ekind_save, &(ir->opts));
+ }
+
+ bFirstIterate = TRUE;
+ while (bFirstIterate || iterate.bIterationActive)
+ {
+ if (iterate.bIterationActive)
+ {
+ copy_coupling_state(bufstate, state, ekind_save, ekind, &(ir->opts));
+ if (bFirstIterate && bTrotter)
+ {
+ /* The first time through, we need a decent first estimate
+ of veta(t+dt) to compute the constraints. Do
+ this by computing the box volume part of the
+ trotter integration at this time. Nothing else
+ should be changed by this routine here. If
+ !(first time), we start with the previous value
+ of veta. */
+
+ veta_save = state->veta;
+ trotter_update(ir, step, ekind, enerd, state, total_vir, mdatoms, &MassQ, trotter_seq, ettTSEQ0);
+ vetanew = state->veta;
+ state->veta = veta_save;
+ }
+ }
+
+ bOK = TRUE;
+ if (!bRerunMD || rerun_fr.bV || bForceUpdate) /* Why is rerun_fr.bV here? Unclear. */
+ {
+ update_constraints(fplog, step, NULL, ir, ekind, mdatoms,
+ state, fr->bMolPBC, graph, f,
+ &top->idef, shake_vir,
+ cr, nrnb, wcycle, upd, constr,
+ TRUE, bCalcVir, vetanew);
+
+ if (!bOK)
+ {
+ gmx_fatal(FARGS, "Constraint error: Shake, Lincs or Settle could not solve the constrains");
+ }
+
+ }
+ else if (graph)
+ {
+ /* Need to unshift here if a do_force has been
+ called in the previous step */
+ unshift_self(graph, state->box, state->x);
+ }
+
+ /* if VV, compute the pressure and constraints */
+ /* For VV2, we strictly only need this if using pressure
+ * control, but we really would like to have accurate pressures
+ * printed out.
+ * Think about ways around this in the future?
+ * For now, keep this choice in comments.
+ */
+ /*bPres = (ir->eI==eiVV || IR_NPT_TROTTER(ir)); */
+ /*bTemp = ((ir->eI==eiVV &&(!bInitStep)) || (ir->eI==eiVVAK && IR_NPT_TROTTER(ir)));*/
+ bPres = TRUE;
+ bTemp = ((ir->eI == eiVV && (!bInitStep)) || (ir->eI == eiVVAK));
+ if (bCalcEner && ir->eI == eiVVAK) /*MRS: 7/9/2010 -- this still doesn't fix it?*/
+ {
+ bSumEkinhOld = TRUE;
+ }
+ /* for vv, the first half of the integration actually corresponds to the previous step.
+ So we need information from the last step in the first half of the integration */
+ if (bGStat || do_per_step(step-1, nstglobalcomm))
+ {
+ compute_globals(fplog, gstat, cr, ir, fr, ekind, state, state_global, mdatoms, nrnb, vcm,
+ wcycle, enerd, force_vir, shake_vir, total_vir, pres, mu_tot,
+ constr, NULL, FALSE, state->box,
+ top_global, &bSumEkinhOld,
+ cglo_flags
+ | CGLO_ENERGY
+ | (bTemp ? CGLO_TEMPERATURE : 0)
+ | (bPres ? CGLO_PRESSURE : 0)
+ | (bPres ? CGLO_CONSTRAINT : 0)
+ | ((iterate.bIterationActive) ? CGLO_ITERATE : 0)
+ | (bFirstIterate ? CGLO_FIRSTITERATE : 0)
+ | CGLO_SCALEEKIN
+ );
+ /* explanation of above:
+ a) We compute Ekin at the full time step
+ if 1) we are using the AveVel Ekin, and it's not the
+ initial step, or 2) if we are using AveEkin, but need the full
+ time step kinetic energy for the pressure (always true now, since we want accurate statistics).
+ b) If we are using EkinAveEkin for the kinetic energy for the temperature control, we still feed in
+ EkinAveVel because it's needed for the pressure */
+ }
+ /* temperature scaling and pressure scaling to produce the extended variables at t+dt */
+ if (!bInitStep)
+ {
+ if (bTrotter)
+ {
+ m_add(force_vir, shake_vir, total_vir); /* we need the un-dispersion corrected total vir here */
+ trotter_update(ir, step, ekind, enerd, state, total_vir, mdatoms, &MassQ, trotter_seq, ettTSEQ2);
+ }
+ else
+ {
+ if (bExchanged)
+ {
+
+ /* We need the kinetic energy at minus the half step for determining
+ * the full step kinetic energy and possibly for T-coupling.*/
+ /* This may not be quite working correctly yet . . . . */
+ compute_globals(fplog, gstat, cr, ir, fr, ekind, state, state_global, mdatoms, nrnb, vcm,
+ wcycle, enerd, NULL, NULL, NULL, NULL, mu_tot,
+ constr, NULL, FALSE, state->box,
+ top_global, &bSumEkinhOld,
+ CGLO_RERUNMD | CGLO_GSTAT | CGLO_TEMPERATURE);
+ }
+ }
+ }
+
+ if (iterate.bIterationActive &&
+ done_iterating(cr, fplog, step, &iterate, bFirstIterate,
+ state->veta, &vetanew))
+ {
+ break;
+ }
+ bFirstIterate = FALSE;
+ }
+
+ if (bTrotter && !bInitStep)
+ {
+ copy_mat(shake_vir, state->svir_prev);
+ copy_mat(force_vir, state->fvir_prev);
+ if (IR_NVT_TROTTER(ir) && ir->eI == eiVV)
+ {
+ /* update temperature and kinetic energy now that step is over - this is the v(t+dt) point */
+ enerd->term[F_TEMP] = sum_ekin(&(ir->opts), ekind, NULL, (ir->eI == eiVV), FALSE);
+ enerd->term[F_EKIN] = trace(ekind->ekin);
+ }
+ }
+ /* if it's the initial step, we performed this first step just to get the constraint virial */
+ if (bInitStep && ir->eI == eiVV)
+ {
+ copy_rvecn(cbuf, state->v, 0, state->natoms);
+ }
+ }
+
+ /* MRS -- now done iterating -- compute the conserved quantity */
+ if (bVV)
+ {
+ saved_conserved_quantity = compute_conserved_from_auxiliary(ir, state, &MassQ);
+ if (ir->eI == eiVV)
+ {
+ last_ekin = enerd->term[F_EKIN];
+ }
+ if ((ir->eDispCorr != edispcEnerPres) && (ir->eDispCorr != edispcAllEnerPres))
+ {
+ saved_conserved_quantity -= enerd->term[F_DISPCORR];
+ }
+ /* sum up the foreign energy and dhdl terms for vv. currently done every step so that dhdl is correct in the .edr */
+ if (!bRerunMD)
+ {
+ sum_dhdl(enerd, state->lambda, ir->fepvals);
+ }
+ }
+
+ /* ######## END FIRST UPDATE STEP ############## */
+ /* ######## If doing VV, we now have v(dt) ###### */
+ if (bDoExpanded)
+ {
+ /* perform extended ensemble sampling in lambda - we don't
+ actually move to the new state before outputting
+ statistics, but if performing simulated tempering, we
+ do update the velocities and the tau_t. */
+
- outf, mdebin, ekind, df_history, f, f_global,
++ lamnew = ExpandedEnsembleDynamics(fplog, ir, enerd, state, &MassQ, state->fep_state, &state->dfhist, step, mcrng, state->v, mdatoms);
++ /* history is maintained in state->dfhist, but state_global is what is sent to trajectory and log output */
++ copy_df_history(&state_global->dfhist,&state->dfhist);
+ }
+
+ /* Now we have the energies and forces corresponding to the
+ * coordinates at time t. We must output all of this before
+ * the update.
+ */
+ do_trajectory_writing(fplog, cr, nfile, fnm, step, step_rel, t,
+ ir, state, state_global, top_global, fr, upd,
- &df_history, state->fep_state, ir->nstlog, step);
++ outf, mdebin, ekind, f, f_global,
+ wcycle, mcrng, &nchkpt,
+ bCPT, bRerunMD, bLastStep, (Flags & MD_CONFOUT),
+ bSumEkinhOld);
+
+ /* kludge -- virial is lost with restart for NPT control. Must restart */
+ if (bStartingFromCpt && bVV)
+ {
+ copy_mat(state->svir_prev, shake_vir);
+ copy_mat(state->fvir_prev, force_vir);
+ }
+
+ /* Determine the wallclock run time up till now */
+ run_time = gmx_gettime() - (double)runtime->real;
+
+ /* Check whether everything is still allright */
+ if (((int)gmx_get_stop_condition() > handled_stop_condition)
+#ifdef GMX_THREAD_MPI
+ && MASTER(cr)
+#endif
+ )
+ {
+ /* this is just make gs.sig compatible with the hack
+ of sending signals around by MPI_Reduce with together with
+ other floats */
+ if (gmx_get_stop_condition() == gmx_stop_cond_next_ns)
+ {
+ gs.sig[eglsSTOPCOND] = 1;
+ }
+ if (gmx_get_stop_condition() == gmx_stop_cond_next)
+ {
+ gs.sig[eglsSTOPCOND] = -1;
+ }
+ /* < 0 means stop at next step, > 0 means stop at next NS step */
+ if (fplog)
+ {
+ fprintf(fplog,
+ "\n\nReceived the %s signal, stopping at the next %sstep\n\n",
+ gmx_get_signal_name(),
+ gs.sig[eglsSTOPCOND] == 1 ? "NS " : "");
+ fflush(fplog);
+ }
+ fprintf(stderr,
+ "\n\nReceived the %s signal, stopping at the next %sstep\n\n",
+ gmx_get_signal_name(),
+ gs.sig[eglsSTOPCOND] == 1 ? "NS " : "");
+ fflush(stderr);
+ handled_stop_condition = (int)gmx_get_stop_condition();
+ }
+ else if (MASTER(cr) && (bNS || ir->nstlist <= 0) &&
+ (max_hours > 0 && run_time > max_hours*60.0*60.0*0.99) &&
+ gs.sig[eglsSTOPCOND] == 0 && gs.set[eglsSTOPCOND] == 0)
+ {
+ /* Signal to terminate the run */
+ gs.sig[eglsSTOPCOND] = 1;
+ if (fplog)
+ {
+ fprintf(fplog, "\nStep %s: Run time exceeded %.3f hours, will terminate the run\n", gmx_step_str(step, sbuf), max_hours*0.99);
+ }
+ fprintf(stderr, "\nStep %s: Run time exceeded %.3f hours, will terminate the run\n", gmx_step_str(step, sbuf), max_hours*0.99);
+ }
+
+ if (bResetCountersHalfMaxH && MASTER(cr) &&
+ run_time > max_hours*60.0*60.0*0.495)
+ {
+ gs.sig[eglsRESETCOUNTERS] = 1;
+ }
+
+ if (ir->nstlist == -1 && !bRerunMD)
+ {
+ /* When bGStatEveryStep=FALSE, global_stat is only called
+ * when we check the atom displacements, not at NS steps.
+ * This means that also the bonded interaction count check is not
+ * performed immediately after NS. Therefore a few MD steps could
+ * be performed with missing interactions.
+ * But wrong energies are never written to file,
+ * since energies are only written after global_stat
+ * has been called.
+ */
+ if (step >= nlh.step_nscheck)
+ {
+ nlh.nabnsb = natoms_beyond_ns_buffer(ir, fr, &top->cgs,
+ nlh.scale_tot, state->x);
+ }
+ else
+ {
+ /* This is not necessarily true,
+ * but step_nscheck is determined quite conservatively.
+ */
+ nlh.nabnsb = 0;
+ }
+ }
+
+ /* In parallel we only have to check for checkpointing in steps
+ * where we do global communication,
+ * otherwise the other nodes don't know.
+ */
+ if (MASTER(cr) && ((bGStat || !PAR(cr)) &&
+ cpt_period >= 0 &&
+ (cpt_period == 0 ||
+ run_time >= nchkpt*cpt_period*60.0)) &&
+ gs.set[eglsCHKPT] == 0)
+ {
+ gs.sig[eglsCHKPT] = 1;
+ }
+
+ /* at the start of step, randomize or scale the velocities (trotter done elsewhere) */
+ if (EI_VV(ir->eI))
+ {
+ if (!bInitStep)
+ {
+ update_tcouple(step, ir, state, ekind, upd, &MassQ, mdatoms);
+ }
+ if (ETC_ANDERSEN(ir->etc)) /* keep this outside of update_tcouple because of the extra info required to pass */
+ {
+ gmx_bool bIfRandomize;
+ bIfRandomize = update_randomize_velocities(ir, step, mdatoms, state, upd, &top->idef, constr);
+ /* if we have constraints, we have to remove the kinetic energy parallel to the bonds */
+ if (constr && bIfRandomize)
+ {
+ update_constraints(fplog, step, NULL, ir, ekind, mdatoms,
+ state, fr->bMolPBC, graph, f,
+ &top->idef, tmp_vir,
+ cr, nrnb, wcycle, upd, constr,
+ TRUE, bCalcVir, vetanew);
+ }
+ }
+ }
+
+ if (bIterativeCase && do_per_step(step, ir->nstpcouple))
+ {
+ gmx_iterate_init(&iterate, TRUE);
+ /* for iterations, we save these vectors, as we will be redoing the calculations */
+ copy_coupling_state(state, bufstate, ekind, ekind_save, &(ir->opts));
+ }
+
+ bFirstIterate = TRUE;
+ while (bFirstIterate || iterate.bIterationActive)
+ {
+ /* We now restore these vectors to redo the calculation with improved extended variables */
+ if (iterate.bIterationActive)
+ {
+ copy_coupling_state(bufstate, state, ekind_save, ekind, &(ir->opts));
+ }
+
+ /* We make the decision to break or not -after- the calculation of Ekin and Pressure,
+ so scroll down for that logic */
+
+ /* ######### START SECOND UPDATE STEP ################# */
+ /* Box is changed in update() when we do pressure coupling,
+ * but we should still use the old box for energy corrections and when
+ * writing it to the energy file, so it matches the trajectory files for
+ * the same timestep above. Make a copy in a separate array.
+ */
+ copy_mat(state->box, lastbox);
+
+ bOK = TRUE;
+ dvdl_constr = 0;
+
+ if (!(bRerunMD && !rerun_fr.bV && !bForceUpdate))
+ {
+ wallcycle_start(wcycle, ewcUPDATE);
+ /* UPDATE PRESSURE VARIABLES IN TROTTER FORMULATION WITH CONSTRAINTS */
+ if (bTrotter)
+ {
+ if (iterate.bIterationActive)
+ {
+ if (bFirstIterate)
+ {
+ scalevir = 1;
+ }
+ else
+ {
+ /* we use a new value of scalevir to converge the iterations faster */
+ scalevir = tracevir/trace(shake_vir);
+ }
+ msmul(shake_vir, scalevir, shake_vir);
+ m_add(force_vir, shake_vir, total_vir);
+ clear_mat(shake_vir);
+ }
+ trotter_update(ir, step, ekind, enerd, state, total_vir, mdatoms, &MassQ, trotter_seq, ettTSEQ3);
+ /* We can only do Berendsen coupling after we have summed
+ * the kinetic energy or virial. Since the happens
+ * in global_state after update, we should only do it at
+ * step % nstlist = 1 with bGStatEveryStep=FALSE.
+ */
+ }
+ else
+ {
+ update_tcouple(step, ir, state, ekind, upd, &MassQ, mdatoms);
+ update_pcouple(fplog, step, ir, state, pcoupl_mu, M, bInitStep);
+ }
+
+ if (bVV)
+ {
+ bUpdateDoLR = (fr->bTwinRange && do_per_step(step, ir->nstcalclr));
+
+ /* velocity half-step update */
+ update_coords(fplog, step, ir, mdatoms, state, fr->bMolPBC, f,
+ bUpdateDoLR, fr->f_twin, fcd,
+ ekind, M, upd, FALSE, etrtVELOCITY2,
+ cr, nrnb, constr, &top->idef);
+ }
+
+ /* Above, initialize just copies ekinh into ekin,
+ * it doesn't copy position (for VV),
+ * and entire integrator for MD.
+ */
+
+ if (ir->eI == eiVVAK)
+ {
+ copy_rvecn(state->x, cbuf, 0, state->natoms);
+ }
+ bUpdateDoLR = (fr->bTwinRange && do_per_step(step, ir->nstcalclr));
+
+ update_coords(fplog, step, ir, mdatoms, state, fr->bMolPBC, f,
+ bUpdateDoLR, fr->f_twin, fcd,
+ ekind, M, upd, bInitStep, etrtPOSITION, cr, nrnb, constr, &top->idef);
+ wallcycle_stop(wcycle, ewcUPDATE);
+
+ update_constraints(fplog, step, &dvdl_constr, ir, ekind, mdatoms, state,
+ fr->bMolPBC, graph, f,
+ &top->idef, shake_vir,
+ cr, nrnb, wcycle, upd, constr,
+ FALSE, bCalcVir, state->veta);
+
+ if (ir->eI == eiVVAK)
+ {
+ /* erase F_EKIN and F_TEMP here? */
+ /* just compute the kinetic energy at the half step to perform a trotter step */
+ compute_globals(fplog, gstat, cr, ir, fr, ekind, state, state_global, mdatoms, nrnb, vcm,
+ wcycle, enerd, force_vir, shake_vir, total_vir, pres, mu_tot,
+ constr, NULL, FALSE, lastbox,
+ top_global, &bSumEkinhOld,
+ cglo_flags | CGLO_TEMPERATURE
+ );
+ wallcycle_start(wcycle, ewcUPDATE);
+ trotter_update(ir, step, ekind, enerd, state, total_vir, mdatoms, &MassQ, trotter_seq, ettTSEQ4);
+ /* now we know the scaling, we can compute the positions again again */
+ copy_rvecn(cbuf, state->x, 0, state->natoms);
+
+ bUpdateDoLR = (fr->bTwinRange && do_per_step(step, ir->nstcalclr));
+
+ update_coords(fplog, step, ir, mdatoms, state, fr->bMolPBC, f,
+ bUpdateDoLR, fr->f_twin, fcd,
+ ekind, M, upd, bInitStep, etrtPOSITION, cr, nrnb, constr, &top->idef);
+ wallcycle_stop(wcycle, ewcUPDATE);
+
+ /* do we need an extra constraint here? just need to copy out of state->v to upd->xp? */
+ /* are the small terms in the shake_vir here due
+ * to numerical errors, or are they important
+ * physically? I'm thinking they are just errors, but not completely sure.
+ * For now, will call without actually constraining, constr=NULL*/
+ update_constraints(fplog, step, NULL, ir, ekind, mdatoms,
+ state, fr->bMolPBC, graph, f,
+ &top->idef, tmp_vir,
+ cr, nrnb, wcycle, upd, NULL,
+ FALSE, bCalcVir,
+ state->veta);
+ }
+ if (!bOK)
+ {
+ gmx_fatal(FARGS, "Constraint error: Shake, Lincs or Settle could not solve the constrains");
+ }
+
+ if (fr->bSepDVDL && fplog && do_log)
+ {
+ gmx_print_sepdvdl(fplog, "Constraint dV/dl", 0.0, dvdl_constr);
+ }
+ if (bVV)
+ {
+ /* this factor or 2 correction is necessary
+ because half of the constraint force is removed
+ in the vv step, so we have to double it. See
+ the Redmine issue #1255. It is not yet clear
+ if the factor of 2 is exact, or just a very
+ good approximation, and this will be
+ investigated. The next step is to see if this
+ can be done adding a dhdl contribution from the
+ rattle step, but this is somewhat more
+ complicated with the current code. Will be
+ investigated, hopefully for 4.6.3. However,
+ this current solution is much better than
+ having it completely wrong.
+ */
+ enerd->term[F_DVDL_CONSTR] += 2*dvdl_constr;
+ }
+ else
+ {
+ enerd->term[F_DVDL_CONSTR] += dvdl_constr;
+ }
+ }
+ else if (graph)
+ {
+ /* Need to unshift here */
+ unshift_self(graph, state->box, state->x);
+ }
+
+ if (vsite != NULL)
+ {
+ wallcycle_start(wcycle, ewcVSITECONSTR);
+ if (graph != NULL)
+ {
+ shift_self(graph, state->box, state->x);
+ }
+ construct_vsites(vsite, state->x, ir->delta_t, state->v,
+ top->idef.iparams, top->idef.il,
+ fr->ePBC, fr->bMolPBC, graph, cr, state->box);
+
+ if (graph != NULL)
+ {
+ unshift_self(graph, state->box, state->x);
+ }
+ wallcycle_stop(wcycle, ewcVSITECONSTR);
+ }
+
+ /* ############## IF NOT VV, Calculate globals HERE, also iterate constraints ############ */
+ /* With Leap-Frog we can skip compute_globals at
+ * non-communication steps, but we need to calculate
+ * the kinetic energy one step before communication.
+ */
+ if (bGStat || (!EI_VV(ir->eI) && do_per_step(step+1, nstglobalcomm)))
+ {
+ if (ir->nstlist == -1 && bFirstIterate)
+ {
+ gs.sig[eglsNABNSB] = nlh.nabnsb;
+ }
+ compute_globals(fplog, gstat, cr, ir, fr, ekind, state, state_global, mdatoms, nrnb, vcm,
+ wcycle, enerd, force_vir, shake_vir, total_vir, pres, mu_tot,
+ constr,
+ bFirstIterate ? &gs : NULL,
+ (step_rel % gs.nstms == 0) &&
+ (multisim_nsteps < 0 || (step_rel < multisim_nsteps)),
+ lastbox,
+ top_global, &bSumEkinhOld,
+ cglo_flags
+ | (!EI_VV(ir->eI) || bRerunMD ? CGLO_ENERGY : 0)
+ | (!EI_VV(ir->eI) && bStopCM ? CGLO_STOPCM : 0)
+ | (!EI_VV(ir->eI) ? CGLO_TEMPERATURE : 0)
+ | (!EI_VV(ir->eI) || bRerunMD ? CGLO_PRESSURE : 0)
+ | (iterate.bIterationActive ? CGLO_ITERATE : 0)
+ | (bFirstIterate ? CGLO_FIRSTITERATE : 0)
+ | CGLO_CONSTRAINT
+ );
+ if (ir->nstlist == -1 && bFirstIterate)
+ {
+ nlh.nabnsb = gs.set[eglsNABNSB];
+ gs.set[eglsNABNSB] = 0;
+ }
+ }
+ /* bIterate is set to keep it from eliminating the old ekin kinetic energy terms */
+ /* ############# END CALC EKIN AND PRESSURE ################# */
+
+ /* Note: this is OK, but there are some numerical precision issues with using the convergence of
+ the virial that should probably be addressed eventually. state->veta has better properies,
+ but what we actually need entering the new cycle is the new shake_vir value. Ideally, we could
+ generate the new shake_vir, but test the veta value for convergence. This will take some thought. */
+
+ if (iterate.bIterationActive &&
+ done_iterating(cr, fplog, step, &iterate, bFirstIterate,
+ trace(shake_vir), &tracevir))
+ {
+ break;
+ }
+ bFirstIterate = FALSE;
+ }
+
+ if (!bVV || bRerunMD)
+ {
+ /* sum up the foreign energy and dhdl terms for md and sd. currently done every step so that dhdl is correct in the .edr */
+ sum_dhdl(enerd, state->lambda, ir->fepvals);
+ }
+ update_box(fplog, step, ir, mdatoms, state, f,
+ ir->nstlist == -1 ? &nlh.scale_tot : NULL, pcoupl_mu, nrnb, upd);
+
+ /* ################# END UPDATE STEP 2 ################# */
+ /* #### We now have r(t+dt) and v(t+dt/2) ############# */
+
+ /* The coordinates (x) were unshifted in update */
+ if (!bGStat)
+ {
+ /* We will not sum ekinh_old,
+ * so signal that we still have to do it.
+ */
+ bSumEkinhOld = TRUE;
+ }
+
+ /* ######### BEGIN PREPARING EDR OUTPUT ########### */
+
+ /* use the directly determined last velocity, not actually the averaged half steps */
+ if (bTrotter && ir->eI == eiVV)
+ {
+ enerd->term[F_EKIN] = last_ekin;
+ }
+ enerd->term[F_ETOT] = enerd->term[F_EPOT] + enerd->term[F_EKIN];
+
+ if (bVV)
+ {
+ enerd->term[F_ECONSERVED] = enerd->term[F_ETOT] + saved_conserved_quantity;
+ }
+ else
+ {
+ enerd->term[F_ECONSERVED] = enerd->term[F_ETOT] + compute_conserved_from_auxiliary(ir, state, &MassQ);
+ }
+ /* ######### END PREPARING EDR OUTPUT ########### */
+
+ /* Time for performance */
+ if (((step % stepout) == 0) || bLastStep)
+ {
+ runtime_upd_proc(runtime);
+ }
+
+ /* Output stuff */
+ if (MASTER(cr))
+ {
+ gmx_bool do_dr, do_or;
+
+ if (fplog && do_log && bDoExpanded)
+ {
+ /* only needed if doing expanded ensemble */
+ PrintFreeEnergyInfoToFile(fplog, ir->fepvals, ir->expandedvals, ir->bSimTemp ? ir->simtempvals : NULL,
- /* Have to do this part after outputting the logfile and the edr file */
++ &state_global->dfhist, state->fep_state, ir->nstlog, step);
+ }
+ if (!(bStartingFromCpt && (EI_VV(ir->eI))))
+ {
+ if (bCalcEner)
+ {
+ upd_mdebin(mdebin, bDoDHDL, TRUE,
+ t, mdatoms->tmass, enerd, state,
+ ir->fepvals, ir->expandedvals, lastbox,
+ shake_vir, force_vir, total_vir, pres,
+ ekind, mu_tot, constr);
+ }
+ else
+ {
+ upd_mdebin_step(mdebin);
+ }
+
+ do_dr = do_per_step(step, ir->nstdisreout);
+ do_or = do_per_step(step, ir->nstorireout);
+
+ print_ebin(outf->fp_ene, do_ene, do_dr, do_or, do_log ? fplog : NULL,
+ step, t,
+ eprNORMAL, bCompact, mdebin, fcd, groups, &(ir->opts));
+ }
+ if (ir->ePull != epullNO)
+ {
+ pull_print_output(ir->pull, step, t);
+ }
+
+ if (do_per_step(step, ir->nstlog))
+ {
+ if (fflush(fplog) != 0)
+ {
+ gmx_fatal(FARGS, "Cannot flush logfile - maybe you are out of disk space?");
+ }
+ }
+ }
+ if (bDoExpanded)
+ {
- for (i = 0; i < efptNR; i++)
- {
- state_global->lambda[i] = ir->fepvals->all_lambda[i][lamnew];
- }
++ /* Have to do this part _after_ outputting the logfile and the edr file */
++ /* Gets written into the state at the beginning of next loop*/
+ state->fep_state = lamnew;
+ }
++
+ /* Remaining runtime */
+ if (MULTIMASTER(cr) && (do_verbose || gmx_got_usr_signal()) && !bPMETuneRunning)
+ {
+ if (shellfc)
+ {
+ fprintf(stderr, "\n");
+ }
+ print_time(stderr, runtime, step, ir, cr);
+ }
+
+ /* Replica exchange */
+ bExchanged = FALSE;
+ if ((repl_ex_nst > 0) && (step > 0) && !bLastStep &&
+ do_per_step(step, repl_ex_nst))
+ {
+ bExchanged = replica_exchange(fplog, cr, repl_ex,
+ state_global, enerd,
+ state, step, t);
+
+ if (bExchanged && DOMAINDECOMP(cr))
+ {
+ dd_partition_system(fplog, step, cr, TRUE, 1,
+ state_global, top_global, ir,
+ state, &f, mdatoms, top, fr,
+ vsite, shellfc, constr,
+ nrnb, wcycle, FALSE);
+ }
+ }
+
+ bFirstStep = FALSE;
+ bInitStep = FALSE;
+ bStartingFromCpt = FALSE;
+
+ /* ####### SET VARIABLES FOR NEXT ITERATION IF THEY STILL NEED IT ###### */
+ /* With all integrators, except VV, we need to retain the pressure
+ * at the current step for coupling at the next step.
+ */
+ if ((state->flags & (1<<estPRES_PREV)) &&
+ (bGStatEveryStep ||
+ (ir->nstpcouple > 0 && step % ir->nstpcouple == 0)))
+ {
+ /* Store the pressure in t_state for pressure coupling
+ * at the next MD step.
+ */
+ copy_mat(pres, state->pres_prev);
+ }
+
+ /* ####### END SET VARIABLES FOR NEXT ITERATION ###### */
+
+ if ( (membed != NULL) && (!bLastStep) )
+ {
+ rescale_membed(step_rel, membed, state_global->x);
+ }
+
+ if (bRerunMD)
+ {
+ if (MASTER(cr))
+ {
+ /* read next frame from input trajectory */
+ bNotLastFrame = read_next_frame(oenv, status, &rerun_fr);
+ }
+
+ if (PAR(cr))
+ {
+ rerun_parallel_comm(cr, &rerun_fr, &bNotLastFrame);
+ }
+ }
+
+ if (!bRerunMD || !rerun_fr.bStep)
+ {
+ /* increase the MD step number */
+ step++;
+ step_rel++;
+ }
+
+ cycles = wallcycle_stop(wcycle, ewcSTEP);
+ if (DOMAINDECOMP(cr) && wcycle)
+ {
+ dd_cycles_add(cr->dd, cycles, ddCyclStep);
+ }
+
+ if (bPMETuneRunning || bPMETuneTry)
+ {
+ /* PME grid + cut-off optimization with GPUs or PME nodes */
+
+ /* Count the total cycles over the last steps */
+ cycles_pmes += cycles;
+
+ /* We can only switch cut-off at NS steps */
+ if (step % ir->nstlist == 0)
+ {
+ /* PME grid + cut-off optimization with GPUs or PME nodes */
+ if (bPMETuneTry)
+ {
+ if (DDMASTER(cr->dd))
+ {
+ /* PME node load is too high, start tuning */
+ bPMETuneRunning = (dd_pme_f_ratio(cr->dd) >= 1.05);
+ }
+ dd_bcast(cr->dd, sizeof(gmx_bool), &bPMETuneRunning);
+
+ if (bPMETuneRunning || step_rel > ir->nstlist*50)
+ {
+ bPMETuneTry = FALSE;
+ }
+ }
+ if (bPMETuneRunning)
+ {
+ /* init_step might not be a multiple of nstlist,
+ * but the first cycle is always skipped anyhow.
+ */
+ bPMETuneRunning =
+ pme_load_balance(pme_loadbal, cr,
+ (bVerbose && MASTER(cr)) ? stderr : NULL,
+ fplog,
+ ir, state, cycles_pmes,
+ fr->ic, fr->nbv, &fr->pmedata,
+ step);
+
+ /* Update constants in forcerec/inputrec to keep them in sync with fr->ic */
+ fr->ewaldcoeff = fr->ic->ewaldcoeff;
+ fr->rlist = fr->ic->rlist;
+ fr->rlistlong = fr->ic->rlistlong;
+ fr->rcoulomb = fr->ic->rcoulomb;
+ fr->rvdw = fr->ic->rvdw;
+ }
+ cycles_pmes = 0;
+ }
+ }
+
+ if (step_rel == wcycle_get_reset_counters(wcycle) ||
+ gs.set[eglsRESETCOUNTERS] != 0)
+ {
+ /* Reset all the counters related to performance over the run */
+ reset_all_counters(fplog, cr, step, &step_rel, ir, wcycle, nrnb, runtime,
+ fr->nbv != NULL && fr->nbv->bUseGPU ? fr->nbv->cu_nbv : NULL);
+ wcycle_set_reset_counters(wcycle, -1);
+ if (!(cr->duty & DUTY_PME))
+ {
+ /* Tell our PME node to reset its counters */
+ gmx_pme_send_resetcounters(cr, step);
+ }
+ /* Correct max_hours for the elapsed time */
+ max_hours -= run_time/(60.0*60.0);
+ bResetCountersHalfMaxH = FALSE;
+ gs.set[eglsRESETCOUNTERS] = 0;
+ }
+
+ }
+ /* End of main MD loop */
+ debug_gmx();
+
+ /* Stop the time */
+ runtime_end(runtime);
+
+ if (bRerunMD && MASTER(cr))
+ {
+ close_trj(status);
+ }
+
+ if (!(cr->duty & DUTY_PME))
+ {
+ /* Tell the PME only node to finish */
+ gmx_pme_send_finish(cr);
+ }
+
+ if (MASTER(cr))
+ {
+ if (ir->nstcalcenergy > 0 && !bRerunMD)
+ {
+ print_ebin(outf->fp_ene, FALSE, FALSE, FALSE, fplog, step, t,
+ eprAVER, FALSE, mdebin, fcd, groups, &(ir->opts));
+ }
+ }
+
+ done_mdoutf(outf);
+
+ debug_gmx();
+
+ if (ir->nstlist == -1 && nlh.nns > 0 && fplog)
+ {
+ fprintf(fplog, "Average neighborlist lifetime: %.1f steps, std.dev.: %.1f steps\n", nlh.s1/nlh.nns, sqrt(nlh.s2/nlh.nns - sqr(nlh.s1/nlh.nns)));
+ fprintf(fplog, "Average number of atoms that crossed the half buffer length: %.1f\n\n", nlh.ab/nlh.nns);
+ }
+
+ if (pme_loadbal != NULL)
+ {
+ pme_loadbal_done(pme_loadbal, cr, fplog,
+ fr->nbv != NULL && fr->nbv->bUseGPU);
+ }
+
+ if (shellfc && fplog)
+ {
+ fprintf(fplog, "Fraction of iterations that converged: %.2f %%\n",
+ (nconverged*100.0)/step_rel);
+ fprintf(fplog, "Average number of force evaluations per MD step: %.2f\n\n",
+ tcount/step_rel);
+ }
+
+ if (repl_ex_nst > 0 && MASTER(cr))
+ {
+ print_replica_exchange_statistics(fplog, repl_ex);
+ }
+
+ runtime->nsteps_done = step_rel;
+
+ return 0;
+}
--- /dev/null
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * VERSION 3.2.0
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2004, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * Gallium Rubidium Oxygen Manganese Argon Carbon Silicon
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+#include <signal.h>
+#include <stdlib.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#include <string.h>
+#include <assert.h>
+
+#include "typedefs.h"
+#include "smalloc.h"
+#include "sysstuff.h"
+#include "statutil.h"
+#include "force.h"
+#include "mdrun.h"
+#include "md_logging.h"
+#include "md_support.h"
+#include "network.h"
+#include "pull.h"
+#include "pull_rotation.h"
+#include "names.h"
+#include "disre.h"
+#include "orires.h"
+#include "pme.h"
+#include "mdatoms.h"
+#include "repl_ex.h"
+#include "qmmm.h"
+#include "domdec.h"
+#include "partdec.h"
+#include "coulomb.h"
+#include "constr.h"
+#include "mvdata.h"
+#include "checkpoint.h"
+#include "mtop_util.h"
+#include "sighandler.h"
+#include "tpxio.h"
+#include "txtdump.h"
+#include "gmx_detect_hardware.h"
+#include "gmx_omp_nthreads.h"
+#include "pull_rotation.h"
+#include "calc_verletbuf.h"
+#include "../mdlib/nbnxn_search.h"
+#include "../mdlib/nbnxn_consts.h"
+#include "gmx_fatal_collective.h"
+#include "membed.h"
+#include "macros.h"
+#include "gmx_omp.h"
+#include "gmx_thread_affinity.h"
+
+#include "gromacs/utility/gmxmpi.h"
+
+#ifdef GMX_FAHCORE
+#include "corewrap.h"
+#endif
+
+#include "gpu_utils.h"
+#include "nbnxn_cuda_data_mgmt.h"
+
+typedef struct {
+ gmx_integrator_t *func;
+} gmx_intp_t;
+
+/* The array should match the eI array in include/types/enums.h */
+const gmx_intp_t integrator[eiNR] = { {do_md}, {do_steep}, {do_cg}, {do_md}, {do_md}, {do_nm}, {do_lbfgs}, {do_tpi}, {do_tpi}, {do_md}, {do_md}, {do_md}};
+
+gmx_large_int_t deform_init_init_step_tpx;
+matrix deform_init_box_tpx;
+#ifdef GMX_THREAD_MPI
+tMPI_Thread_mutex_t deform_init_box_mutex = TMPI_THREAD_MUTEX_INITIALIZER;
+#endif
+
+
+#ifdef GMX_THREAD_MPI
+struct mdrunner_arglist
+{
+ gmx_hw_opt_t *hw_opt;
+ FILE *fplog;
+ t_commrec *cr;
+ int nfile;
+ const t_filenm *fnm;
+ output_env_t oenv;
+ gmx_bool bVerbose;
+ gmx_bool bCompact;
+ int nstglobalcomm;
+ ivec ddxyz;
+ int dd_node_order;
+ real rdd;
+ real rconstr;
+ const char *dddlb_opt;
+ real dlb_scale;
+ const char *ddcsx;
+ const char *ddcsy;
+ const char *ddcsz;
+ const char *nbpu_opt;
+ gmx_large_int_t nsteps_cmdline;
+ int nstepout;
+ int resetstep;
+ int nmultisim;
+ int repl_ex_nst;
+ int repl_ex_nex;
+ int repl_ex_seed;
+ real pforce;
+ real cpt_period;
+ real max_hours;
+ const char *deviceOptions;
+ unsigned long Flags;
+ int ret; /* return value */
+};
+
+
+/* The function used for spawning threads. Extracts the mdrunner()
+ arguments from its one argument and calls mdrunner(), after making
+ a commrec. */
+static void mdrunner_start_fn(void *arg)
+{
+ struct mdrunner_arglist *mda = (struct mdrunner_arglist*)arg;
+ struct mdrunner_arglist mc = *mda; /* copy the arg list to make sure
+ that it's thread-local. This doesn't
+ copy pointed-to items, of course,
+ but those are all const. */
+ t_commrec *cr; /* we need a local version of this */
+ FILE *fplog = NULL;
+ t_filenm *fnm;
+
+ fnm = dup_tfn(mc.nfile, mc.fnm);
+
+ cr = reinitialize_commrec_for_this_thread(mc.cr);
+
+ if (MASTER(cr))
+ {
+ fplog = mc.fplog;
+ }
+
+ mda->ret = mdrunner(mc.hw_opt, fplog, cr, mc.nfile, fnm, mc.oenv,
+ mc.bVerbose, mc.bCompact, mc.nstglobalcomm,
+ mc.ddxyz, mc.dd_node_order, mc.rdd,
+ mc.rconstr, mc.dddlb_opt, mc.dlb_scale,
+ mc.ddcsx, mc.ddcsy, mc.ddcsz,
+ mc.nbpu_opt,
+ mc.nsteps_cmdline, mc.nstepout, mc.resetstep,
+ mc.nmultisim, mc.repl_ex_nst, mc.repl_ex_nex, mc.repl_ex_seed, mc.pforce,
+ mc.cpt_period, mc.max_hours, mc.deviceOptions, mc.Flags);
+}
+
+/* called by mdrunner() to start a specific number of threads (including
+ the main thread) for thread-parallel runs. This in turn calls mdrunner()
+ for each thread.
+ All options besides nthreads are the same as for mdrunner(). */
+static t_commrec *mdrunner_start_threads(gmx_hw_opt_t *hw_opt,
+ FILE *fplog, t_commrec *cr, int nfile,
+ const t_filenm fnm[], const output_env_t oenv, gmx_bool bVerbose,
+ gmx_bool bCompact, int nstglobalcomm,
+ ivec ddxyz, int dd_node_order, real rdd, real rconstr,
+ const char *dddlb_opt, real dlb_scale,
+ const char *ddcsx, const char *ddcsy, const char *ddcsz,
+ const char *nbpu_opt,
+ gmx_large_int_t nsteps_cmdline,
+ int nstepout, int resetstep,
+ int nmultisim, int repl_ex_nst, int repl_ex_nex, int repl_ex_seed,
+ real pforce, real cpt_period, real max_hours,
+ const char *deviceOptions, unsigned long Flags)
+{
+ int ret;
+ struct mdrunner_arglist *mda;
+ t_commrec *crn; /* the new commrec */
+ t_filenm *fnmn;
+
+ /* first check whether we even need to start tMPI */
+ if (hw_opt->nthreads_tmpi < 2)
+ {
+ return cr;
+ }
+
+ /* a few small, one-time, almost unavoidable memory leaks: */
+ snew(mda, 1);
+ fnmn = dup_tfn(nfile, fnm);
+
+ /* fill the data structure to pass as void pointer to thread start fn */
+ mda->hw_opt = hw_opt;
+ mda->fplog = fplog;
+ mda->cr = cr;
+ mda->nfile = nfile;
+ mda->fnm = fnmn;
+ mda->oenv = oenv;
+ mda->bVerbose = bVerbose;
+ mda->bCompact = bCompact;
+ mda->nstglobalcomm = nstglobalcomm;
+ mda->ddxyz[XX] = ddxyz[XX];
+ mda->ddxyz[YY] = ddxyz[YY];
+ mda->ddxyz[ZZ] = ddxyz[ZZ];
+ mda->dd_node_order = dd_node_order;
+ mda->rdd = rdd;
+ mda->rconstr = rconstr;
+ mda->dddlb_opt = dddlb_opt;
+ mda->dlb_scale = dlb_scale;
+ mda->ddcsx = ddcsx;
+ mda->ddcsy = ddcsy;
+ mda->ddcsz = ddcsz;
+ mda->nbpu_opt = nbpu_opt;
+ mda->nsteps_cmdline = nsteps_cmdline;
+ mda->nstepout = nstepout;
+ mda->resetstep = resetstep;
+ mda->nmultisim = nmultisim;
+ mda->repl_ex_nst = repl_ex_nst;
+ mda->repl_ex_nex = repl_ex_nex;
+ mda->repl_ex_seed = repl_ex_seed;
+ mda->pforce = pforce;
+ mda->cpt_period = cpt_period;
+ mda->max_hours = max_hours;
+ mda->deviceOptions = deviceOptions;
+ mda->Flags = Flags;
+
+ /* now spawn new threads that start mdrunner_start_fn(), while
+ the main thread returns, we set thread affinity later */
+ ret = tMPI_Init_fn(TRUE, hw_opt->nthreads_tmpi, TMPI_AFFINITY_NONE,
+ mdrunner_start_fn, (void*)(mda) );
+ if (ret != TMPI_SUCCESS)
+ {
+ return NULL;
+ }
+
+ crn = reinitialize_commrec_for_this_thread(cr);
+ return crn;
+}
+
+
+static int get_tmpi_omp_thread_division(const gmx_hw_info_t *hwinfo,
+ const gmx_hw_opt_t *hw_opt,
+ int nthreads_tot,
+ int ngpu)
+{
+ int nthreads_tmpi;
+
+ /* There are no separate PME nodes here, as we ensured in
+ * check_and_update_hw_opt that nthreads_tmpi>0 with PME nodes
+ * and a conditional ensures we would not have ended up here.
+ * Note that separate PME nodes might be switched on later.
+ */
+ if (ngpu > 0)
+ {
+ nthreads_tmpi = ngpu;
+ if (nthreads_tot > 0 && nthreads_tot < nthreads_tmpi)
+ {
+ nthreads_tmpi = nthreads_tot;
+ }
+ }
+ else if (hw_opt->nthreads_omp > 0)
+ {
+ /* Here we could oversubscribe, when we do, we issue a warning later */
+ nthreads_tmpi = max(1, nthreads_tot/hw_opt->nthreads_omp);
+ }
+ else
+ {
+ /* TODO choose nthreads_omp based on hardware topology
+ when we have a hardware topology detection library */
+ /* In general, when running up to 4 threads, OpenMP should be faster.
+ * Note: on AMD Bulldozer we should avoid running OpenMP over two dies.
+ * On Intel>=Nehalem running OpenMP on a single CPU is always faster,
+ * even on two CPUs it's usually faster (but with many OpenMP threads
+ * it could be faster not to use HT, currently we always use HT).
+ * On Nehalem/Westmere we want to avoid running 16 threads over
+ * two CPUs with HT, so we need a limit<16; thus we use 12.
+ * A reasonable limit for Intel Sandy and Ivy bridge,
+ * not knowing the topology, is 16 threads.
+ */
+ const int nthreads_omp_always_faster = 4;
+ const int nthreads_omp_always_faster_Nehalem = 12;
+ const int nthreads_omp_always_faster_SandyBridge = 16;
+ const int first_model_Nehalem = 0x1A;
+ const int first_model_SandyBridge = 0x2A;
+ gmx_bool bIntel_Family6;
+
+ bIntel_Family6 =
+ (gmx_cpuid_vendor(hwinfo->cpuid_info) == GMX_CPUID_VENDOR_INTEL &&
+ gmx_cpuid_family(hwinfo->cpuid_info) == 6);
+
+ if (nthreads_tot <= nthreads_omp_always_faster ||
+ (bIntel_Family6 &&
+ ((gmx_cpuid_model(hwinfo->cpuid_info) >= nthreads_omp_always_faster_Nehalem && nthreads_tot <= nthreads_omp_always_faster_Nehalem) ||
+ (gmx_cpuid_model(hwinfo->cpuid_info) >= nthreads_omp_always_faster_SandyBridge && nthreads_tot <= nthreads_omp_always_faster_SandyBridge))))
+ {
+ /* Use pure OpenMP parallelization */
+ nthreads_tmpi = 1;
+ }
+ else
+ {
+ /* Don't use OpenMP parallelization */
+ nthreads_tmpi = nthreads_tot;
+ }
+ }
+
+ return nthreads_tmpi;
+}
+
+
+/* Get the number of threads to use for thread-MPI based on how many
+ * were requested, which algorithms we're using,
+ * and how many particles there are.
+ * At the point we have already called check_and_update_hw_opt.
+ * Thus all options should be internally consistent and consistent
+ * with the hardware, except that ntmpi could be larger than #GPU.
+ */
+static int get_nthreads_mpi(const gmx_hw_info_t *hwinfo,
+ gmx_hw_opt_t *hw_opt,
+ t_inputrec *inputrec, gmx_mtop_t *mtop,
+ const t_commrec *cr,
+ FILE *fplog)
+{
+ int nthreads_hw, nthreads_tot_max, nthreads_tmpi, nthreads_new, ngpu;
+ int min_atoms_per_mpi_thread;
+ char *env;
+ char sbuf[STRLEN];
+ gmx_bool bCanUseGPU;
+
+ if (hw_opt->nthreads_tmpi > 0)
+ {
+ /* Trivial, return right away */
+ return hw_opt->nthreads_tmpi;
+ }
+
+ nthreads_hw = hwinfo->nthreads_hw_avail;
+
+ /* How many total (#tMPI*#OpenMP) threads can we start? */
+ if (hw_opt->nthreads_tot > 0)
+ {
+ nthreads_tot_max = hw_opt->nthreads_tot;
+ }
+ else
+ {
+ nthreads_tot_max = nthreads_hw;
+ }
+
+ bCanUseGPU = (inputrec->cutoff_scheme == ecutsVERLET && hwinfo->bCanUseGPU);
+ if (bCanUseGPU)
+ {
+ ngpu = hwinfo->gpu_info.ncuda_dev_use;
+ }
+ else
+ {
+ ngpu = 0;
+ }
+
+ nthreads_tmpi =
+ get_tmpi_omp_thread_division(hwinfo, hw_opt, nthreads_tot_max, ngpu);
+
+ if (inputrec->eI == eiNM || EI_TPI(inputrec->eI))
+ {
+ /* Dims/steps are divided over the nodes iso splitting the atoms */
+ min_atoms_per_mpi_thread = 0;
+ }
+ else
+ {
+ if (bCanUseGPU)
+ {
+ min_atoms_per_mpi_thread = MIN_ATOMS_PER_GPU;
+ }
+ else
+ {
+ min_atoms_per_mpi_thread = MIN_ATOMS_PER_MPI_THREAD;
+ }
+ }
+
+ /* Check if an algorithm does not support parallel simulation. */
+ if (nthreads_tmpi != 1 &&
+ ( inputrec->eI == eiLBFGS ||
+ inputrec->coulombtype == eelEWALD ) )
+ {
+ nthreads_tmpi = 1;
+
+ md_print_warn(cr, fplog, "The integration or electrostatics algorithm doesn't support parallel runs. Using a single thread-MPI thread.\n");
+ if (hw_opt->nthreads_tmpi > nthreads_tmpi)
+ {
+ gmx_fatal(FARGS, "You asked for more than 1 thread-MPI thread, but an algorithm doesn't support that");
+ }
+ }
+ else if (mtop->natoms/nthreads_tmpi < min_atoms_per_mpi_thread)
+ {
+ /* the thread number was chosen automatically, but there are too many
+ threads (too few atoms per thread) */
+ nthreads_new = max(1, mtop->natoms/min_atoms_per_mpi_thread);
+
+ /* Avoid partial use of Hyper-Threading */
+ if (gmx_cpuid_x86_smt(hwinfo->cpuid_info) == GMX_CPUID_X86_SMT_ENABLED &&
+ nthreads_new > nthreads_hw/2 && nthreads_new < nthreads_hw)
+ {
+ nthreads_new = nthreads_hw/2;
+ }
+
+ /* Avoid large prime numbers in the thread count */
+ if (nthreads_new >= 6)
+ {
+ /* Use only 6,8,10 with additional factors of 2 */
+ int fac;
+
+ fac = 2;
+ while (3*fac*2 <= nthreads_new)
+ {
+ fac *= 2;
+ }
+
+ nthreads_new = (nthreads_new/fac)*fac;
+ }
+ else
+ {
+ /* Avoid 5 */
+ if (nthreads_new == 5)
+ {
+ nthreads_new = 4;
+ }
+ }
+
+ nthreads_tmpi = nthreads_new;
+
+ fprintf(stderr, "\n");
+ fprintf(stderr, "NOTE: Parallelization is limited by the small number of atoms,\n");
+ fprintf(stderr, " only starting %d thread-MPI threads.\n", nthreads_tmpi);
+ fprintf(stderr, " You can use the -nt and/or -ntmpi option to optimize the number of threads.\n\n");
+ }
+
+ return nthreads_tmpi;
+}
+#endif /* GMX_THREAD_MPI */
+
+
+/* Environment variable for setting nstlist */
+static const char* NSTLIST_ENVVAR = "GMX_NSTLIST";
+/* Try to increase nstlist when using a GPU with nstlist less than this */
+static const int NSTLIST_GPU_ENOUGH = 20;
+/* Increase nstlist until the non-bonded cost increases more than this factor */
+static const float NBNXN_GPU_LIST_OK_FAC = 1.25;
+/* Don't increase nstlist beyond a non-bonded cost increases of this factor */
+static const float NBNXN_GPU_LIST_MAX_FAC = 1.40;
+
+/* Try to increase nstlist when running on a GPU */
+static void increase_nstlist(FILE *fp, t_commrec *cr,
+ t_inputrec *ir, const gmx_mtop_t *mtop, matrix box)
+{
+ char *env;
+ int nstlist_orig, nstlist_prev;
+ verletbuf_list_setup_t ls;
+ real rlist_inc, rlist_ok, rlist_max, rlist_new, rlist_prev;
+ int i;
+ t_state state_tmp;
+ gmx_bool bBox, bDD, bCont;
+ const char *nstl_fmt = "\nFor optimal performance with a GPU nstlist (now %d) should be larger.\nThe optimum depends on your CPU and GPU resources.\nYou might want to try several nstlist values.\n";
+ const char *vbd_err = "Can not increase nstlist for GPU run because verlet-buffer-drift is not set or used";
+ const char *box_err = "Can not increase nstlist for GPU run because the box is too small";
+ const char *dd_err = "Can not increase nstlist for GPU run because of domain decomposition limitations";
+ char buf[STRLEN];
+
+ /* Number of + nstlist alternative values to try when switching */
+ const int nstl[] = { 20, 25, 40, 50 };
+#define NNSTL sizeof(nstl)/sizeof(nstl[0])
+
+ env = getenv(NSTLIST_ENVVAR);
+ if (env == NULL)
+ {
+ if (fp != NULL)
+ {
+ fprintf(fp, nstl_fmt, ir->nstlist);
+ }
+ }
+
+ if (ir->verletbuf_drift == 0)
+ {
+ gmx_fatal(FARGS, "You are using an old tpr file with a GPU, please generate a new tpr file with an up to date version of grompp");
+ }
+
+ if (ir->verletbuf_drift < 0)
+ {
+ if (MASTER(cr))
+ {
+ fprintf(stderr, "%s\n", vbd_err);
+ }
+ if (fp != NULL)
+ {
+ fprintf(fp, "%s\n", vbd_err);
+ }
+
+ return;
+ }
+
+ nstlist_orig = ir->nstlist;
+ if (env != NULL)
+ {
+ sprintf(buf, "Getting nstlist from environment variable GMX_NSTLIST=%s", env);
+ if (MASTER(cr))
+ {
+ fprintf(stderr, "%s\n", buf);
+ }
+ if (fp != NULL)
+ {
+ fprintf(fp, "%s\n", buf);
+ }
+ sscanf(env, "%d", &ir->nstlist);
+ }
+
+ verletbuf_get_list_setup(TRUE, &ls);
+
+ /* Allow rlist to make the list double the size of the cut-off sphere */
+ rlist_inc = nbnxn_get_rlist_effective_inc(NBNXN_GPU_CLUSTER_SIZE, mtop->natoms/det(box));
+ rlist_ok = (max(ir->rvdw, ir->rcoulomb) + rlist_inc)*pow(NBNXN_GPU_LIST_OK_FAC, 1.0/3.0) - rlist_inc;
+ rlist_max = (max(ir->rvdw, ir->rcoulomb) + rlist_inc)*pow(NBNXN_GPU_LIST_MAX_FAC, 1.0/3.0) - rlist_inc;
+ if (debug)
+ {
+ fprintf(debug, "GPU nstlist tuning: rlist_inc %.3f rlist_max %.3f\n",
+ rlist_inc, rlist_max);
+ }
+
+ i = 0;
+ nstlist_prev = nstlist_orig;
+ rlist_prev = ir->rlist;
+ do
+ {
+ if (env == NULL)
+ {
+ ir->nstlist = nstl[i];
+ }
+
+ /* Set the pair-list buffer size in ir */
+ calc_verlet_buffer_size(mtop, det(box), ir, ir->verletbuf_drift, &ls,
+ NULL, &rlist_new);
+
+ /* Does rlist fit in the box? */
+ bBox = (sqr(rlist_new) < max_cutoff2(ir->ePBC, box));
+ bDD = TRUE;
+ if (bBox && DOMAINDECOMP(cr))
+ {
+ /* Check if rlist fits in the domain decomposition */
+ if (inputrec2nboundeddim(ir) < DIM)
+ {
+ gmx_incons("Changing nstlist with domain decomposition and unbounded dimensions is not implemented yet");
+ }
+ copy_mat(box, state_tmp.box);
+ bDD = change_dd_cutoff(cr, &state_tmp, ir, rlist_new);
+ }
+
+ bCont = FALSE;
+
+ if (env == NULL)
+ {
+ if (bBox && bDD && rlist_new <= rlist_max)
+ {
+ /* Increase nstlist */
+ nstlist_prev = ir->nstlist;
+ rlist_prev = rlist_new;
+ bCont = (i+1 < NNSTL && rlist_new < rlist_ok);
+ }
+ else
+ {
+ /* Stick with the previous nstlist */
+ ir->nstlist = nstlist_prev;
+ rlist_new = rlist_prev;
+ bBox = TRUE;
+ bDD = TRUE;
+ }
+ }
+
+ i++;
+ }
+ while (bCont);
+
+ if (!bBox || !bDD)
+ {
+ gmx_warning(!bBox ? box_err : dd_err);
+ if (fp != NULL)
+ {
+ fprintf(fp, "\n%s\n", bBox ? box_err : dd_err);
+ }
+ ir->nstlist = nstlist_orig;
+ }
+ else if (ir->nstlist != nstlist_orig || rlist_new != ir->rlist)
+ {
+ sprintf(buf, "Changing nstlist from %d to %d, rlist from %g to %g",
+ nstlist_orig, ir->nstlist,
+ ir->rlist, rlist_new);
+ if (MASTER(cr))
+ {
+ fprintf(stderr, "%s\n\n", buf);
+ }
+ if (fp != NULL)
+ {
+ fprintf(fp, "%s\n\n", buf);
+ }
+ ir->rlist = rlist_new;
+ ir->rlistlong = rlist_new;
+ }
+}
+
+static void prepare_verlet_scheme(FILE *fplog,
+ const gmx_hw_info_t *hwinfo,
+ t_commrec *cr,
+ t_inputrec *ir,
+ const gmx_mtop_t *mtop,
+ matrix box,
+ gmx_bool *bUseGPU)
+{
+ /* Here we only check for GPU usage on the MPI master process,
+ * as here we don't know how many GPUs we will use yet.
+ * We check for a GPU on all processes later.
+ */
+ *bUseGPU = hwinfo->bCanUseGPU || (getenv("GMX_EMULATE_GPU") != NULL);
+
+ if (ir->verletbuf_drift > 0)
+ {
+ /* Update the Verlet buffer size for the current run setup */
+ verletbuf_list_setup_t ls;
+ real rlist_new;
+
+ /* Here we assume CPU acceleration is on. But as currently
+ * calc_verlet_buffer_size gives the same results for 4x8 and 4x4
+ * and 4x2 gives a larger buffer than 4x4, this is ok.
+ */
+ verletbuf_get_list_setup(*bUseGPU, &ls);
+
+ calc_verlet_buffer_size(mtop, det(box), ir,
+ ir->verletbuf_drift, &ls,
+ NULL, &rlist_new);
+ if (rlist_new != ir->rlist)
+ {
+ if (fplog != NULL)
+ {
+ fprintf(fplog, "\nChanging rlist from %g to %g for non-bonded %dx%d atom kernels\n\n",
+ ir->rlist, rlist_new,
+ ls.cluster_size_i, ls.cluster_size_j);
+ }
+ ir->rlist = rlist_new;
+ ir->rlistlong = rlist_new;
+ }
+ }
+
+ /* With GPU or emulation we should check nstlist for performance */
+ if ((EI_DYNAMICS(ir->eI) &&
+ *bUseGPU &&
+ ir->nstlist < NSTLIST_GPU_ENOUGH) ||
+ getenv(NSTLIST_ENVVAR) != NULL)
+ {
+ /* Choose a better nstlist */
+ increase_nstlist(fplog, cr, ir, mtop, box);
+ }
+}
+
+static void convert_to_verlet_scheme(FILE *fplog,
+ t_inputrec *ir,
+ gmx_mtop_t *mtop, real box_vol)
+{
+ char *conv_mesg = "Converting input file with group cut-off scheme to the Verlet cut-off scheme";
+
+ md_print_warn(NULL, fplog, "%s\n", conv_mesg);
+
+ ir->cutoff_scheme = ecutsVERLET;
+ ir->verletbuf_drift = 0.005;
+
+ if (ir->rcoulomb != ir->rvdw)
+ {
+ gmx_fatal(FARGS, "The VdW and Coulomb cut-offs are different, whereas the Verlet scheme only supports equal cut-offs");
+ }
+
+ if (ir->vdwtype == evdwUSER || EEL_USER(ir->coulombtype))
+ {
+ gmx_fatal(FARGS, "User non-bonded potentials are not (yet) supported with the Verlet scheme");
+ }
+ else if (EVDW_SWITCHED(ir->vdwtype) || EEL_SWITCHED(ir->coulombtype))
+ {
+ md_print_warn(NULL, fplog, "Converting switched or shifted interactions to a shifted potential (without force shift), this will lead to slightly different interaction potentials");
+
+ if (EVDW_SWITCHED(ir->vdwtype))
+ {
+ ir->vdwtype = evdwCUT;
+ }
+ if (EEL_SWITCHED(ir->coulombtype))
+ {
+ if (EEL_FULL(ir->coulombtype))
+ {
+ /* With full electrostatic only PME can be switched */
+ ir->coulombtype = eelPME;
+ }
+ else
+ {
+ md_print_warn(NULL, fplog, "NOTE: Replacing %s electrostatics with reaction-field with epsilon-rf=inf\n", eel_names[ir->coulombtype]);
+ ir->coulombtype = eelRF;
+ ir->epsilon_rf = 0.0;
+ }
+ }
+
+ /* We set the target energy drift to a small number.
+ * Note that this is only for testing. For production the user
+ * should think about this and set the mdp options.
+ */
+ ir->verletbuf_drift = 1e-4;
+ }
+
+ if (inputrec2nboundeddim(ir) != 3)
+ {
+ gmx_fatal(FARGS, "Can only convert old tpr files to the Verlet cut-off scheme with 3D pbc");
+ }
+
+ if (ir->efep != efepNO || ir->implicit_solvent != eisNO)
+ {
+ gmx_fatal(FARGS, "Will not convert old tpr files to the Verlet cut-off scheme with free-energy calculations or implicit solvent");
+ }
+
+ if (EI_DYNAMICS(ir->eI) && !(EI_MD(ir->eI) && ir->etc == etcNO))
+ {
+ verletbuf_list_setup_t ls;
+
+ verletbuf_get_list_setup(FALSE, &ls);
+ calc_verlet_buffer_size(mtop, box_vol, ir, ir->verletbuf_drift, &ls,
+ NULL, &ir->rlist);
+ }
+ else
+ {
+ ir->verletbuf_drift = -1;
+ ir->rlist = 1.05*max(ir->rvdw, ir->rcoulomb);
+ }
+
+ gmx_mtop_remove_chargegroups(mtop);
+}
+
+static void check_and_update_hw_opt(gmx_hw_opt_t *hw_opt,
+ int cutoff_scheme,
+ gmx_bool bIsSimMaster)
+{
+ gmx_omp_nthreads_read_env(&hw_opt->nthreads_omp, bIsSimMaster);
+
+#ifndef GMX_THREAD_MPI
+ if (hw_opt->nthreads_tot > 0)
+ {
+ gmx_fatal(FARGS, "Setting the total number of threads is only supported with thread-MPI and Gromacs was compiled without thread-MPI");
+ }
+ if (hw_opt->nthreads_tmpi > 0)
+ {
+ gmx_fatal(FARGS, "Setting the number of thread-MPI threads is only supported with thread-MPI and Gromacs was compiled without thread-MPI");
+ }
+#endif
+
+#ifndef GMX_OPENMP
+ if (hw_opt->nthreads_omp > 1)
+ {
+ gmx_fatal(FARGS, "More than 1 OpenMP thread requested, but Gromacs was compiled without OpenMP support");
+ }
+ hw_opt->nthreads_omp = 1;
+#endif
+
+ if (hw_opt->nthreads_tot > 0 && hw_opt->nthreads_omp_pme <= 0)
+ {
+ /* We have the same number of OpenMP threads for PP and PME processes,
+ * thus we can perform several consistency checks.
+ */
+ if (hw_opt->nthreads_tmpi > 0 &&
+ hw_opt->nthreads_omp > 0 &&
+ hw_opt->nthreads_tot != hw_opt->nthreads_tmpi*hw_opt->nthreads_omp)
+ {
+ gmx_fatal(FARGS, "The total number of threads requested (%d) does not match the thread-MPI threads (%d) times the OpenMP threads (%d) requested",
+ hw_opt->nthreads_tot, hw_opt->nthreads_tmpi, hw_opt->nthreads_omp);
+ }
+
+ if (hw_opt->nthreads_tmpi > 0 &&
+ hw_opt->nthreads_tot % hw_opt->nthreads_tmpi != 0)
+ {
+ gmx_fatal(FARGS, "The total number of threads requested (%d) is not divisible by the number of thread-MPI threads requested (%d)",
+ hw_opt->nthreads_tot, hw_opt->nthreads_tmpi);
+ }
+
+ if (hw_opt->nthreads_omp > 0 &&
+ hw_opt->nthreads_tot % hw_opt->nthreads_omp != 0)
+ {
+ gmx_fatal(FARGS, "The total number of threads requested (%d) is not divisible by the number of OpenMP threads requested (%d)",
+ hw_opt->nthreads_tot, hw_opt->nthreads_omp);
+ }
+
+ if (hw_opt->nthreads_tmpi > 0 &&
+ hw_opt->nthreads_omp <= 0)
+ {
+ hw_opt->nthreads_omp = hw_opt->nthreads_tot/hw_opt->nthreads_tmpi;
+ }
+ }
+
+#ifndef GMX_OPENMP
+ if (hw_opt->nthreads_omp > 1)
+ {
+ gmx_fatal(FARGS, "OpenMP threads are requested, but Gromacs was compiled without OpenMP support");
+ }
+#endif
+
+ if (cutoff_scheme == ecutsGROUP)
+ {
+ /* We only have OpenMP support for PME only nodes */
+ if (hw_opt->nthreads_omp > 1)
+ {
+ gmx_fatal(FARGS, "OpenMP threads have been requested with cut-off scheme %s, but these are only supported with cut-off scheme %s",
+ ecutscheme_names[cutoff_scheme],
+ ecutscheme_names[ecutsVERLET]);
+ }
+ hw_opt->nthreads_omp = 1;
+ }
+
+ if (hw_opt->nthreads_omp_pme > 0 && hw_opt->nthreads_omp <= 0)
+ {
+ gmx_fatal(FARGS, "You need to specify -ntomp in addition to -ntomp_pme");
+ }
+
+ if (hw_opt->nthreads_tot == 1)
+ {
+ hw_opt->nthreads_tmpi = 1;
+
+ if (hw_opt->nthreads_omp > 1)
+ {
+ gmx_fatal(FARGS, "You requested %d OpenMP threads with %d total threads",
+ hw_opt->nthreads_tmpi, hw_opt->nthreads_tot);
+ }
+ hw_opt->nthreads_omp = 1;
+ }
+
+ if (hw_opt->nthreads_omp_pme <= 0 && hw_opt->nthreads_omp > 0)
+ {
+ hw_opt->nthreads_omp_pme = hw_opt->nthreads_omp;
+ }
+
+ if (debug)
+ {
+ fprintf(debug, "hw_opt: nt %d ntmpi %d ntomp %d ntomp_pme %d gpu_id '%s'\n",
+ hw_opt->nthreads_tot,
+ hw_opt->nthreads_tmpi,
+ hw_opt->nthreads_omp,
+ hw_opt->nthreads_omp_pme,
+ hw_opt->gpu_id != NULL ? hw_opt->gpu_id : "");
+
+ }
+}
+
+
+/* Override the value in inputrec with value passed on the command line (if any) */
+static void override_nsteps_cmdline(FILE *fplog,
+ gmx_large_int_t nsteps_cmdline,
+ t_inputrec *ir,
+ const t_commrec *cr)
+{
+ char sbuf[STEPSTRSIZE];
+
+ assert(ir);
+ assert(cr);
+
+ /* override with anything else than the default -2 */
+ if (nsteps_cmdline > -2)
+ {
+ char stmp[STRLEN];
+
+ ir->nsteps = nsteps_cmdline;
+ if (EI_DYNAMICS(ir->eI))
+ {
+ sprintf(stmp, "Overriding nsteps with value passed on the command line: %s steps, %.3f ps",
+ gmx_step_str(nsteps_cmdline, sbuf),
+ nsteps_cmdline*ir->delta_t);
+ }
+ else
+ {
+ sprintf(stmp, "Overriding nsteps with value passed on the command line: %s steps",
+ gmx_step_str(nsteps_cmdline, sbuf));
+ }
+
+ md_print_warn(cr, fplog, "%s\n", stmp);
+ }
+}
+
+/* Data structure set by SIMMASTER which needs to be passed to all nodes
+ * before the other nodes have read the tpx file and called gmx_detect_hardware.
+ */
+typedef struct {
+ int cutoff_scheme; /* The cutoff scheme from inputrec_t */
+ gmx_bool bUseGPU; /* Use GPU or GPU emulation */
+} master_inf_t;
+
+int mdrunner(gmx_hw_opt_t *hw_opt,
+ FILE *fplog, t_commrec *cr, int nfile,
+ const t_filenm fnm[], const output_env_t oenv, gmx_bool bVerbose,
+ gmx_bool bCompact, int nstglobalcomm,
+ ivec ddxyz, int dd_node_order, real rdd, real rconstr,
+ const char *dddlb_opt, real dlb_scale,
+ const char *ddcsx, const char *ddcsy, const char *ddcsz,
+ const char *nbpu_opt,
+ gmx_large_int_t nsteps_cmdline, int nstepout, int resetstep,
+ int nmultisim, int repl_ex_nst, int repl_ex_nex,
+ int repl_ex_seed, real pforce, real cpt_period, real max_hours,
+ const char *deviceOptions, unsigned long Flags)
+{
+ gmx_bool bForceUseGPU, bTryUseGPU;
+ double nodetime = 0, realtime;
+ t_inputrec *inputrec;
+ t_state *state = NULL;
+ matrix box;
+ gmx_ddbox_t ddbox = {0};
+ int npme_major, npme_minor;
+ real tmpr1, tmpr2;
+ t_nrnb *nrnb;
+ gmx_mtop_t *mtop = NULL;
+ t_mdatoms *mdatoms = NULL;
+ t_forcerec *fr = NULL;
+ t_fcdata *fcd = NULL;
+ real ewaldcoeff = 0;
+ gmx_pme_t *pmedata = NULL;
+ gmx_vsite_t *vsite = NULL;
+ gmx_constr_t constr;
+ int i, m, nChargePerturbed = -1, status, nalloc;
+ char *gro;
+ gmx_wallcycle_t wcycle;
+ gmx_bool bReadRNG, bReadEkin;
+ int list;
+ gmx_runtime_t runtime;
+ int rc;
+ gmx_large_int_t reset_counters;
+ gmx_edsam_t ed = NULL;
+ t_commrec *cr_old = cr;
+ int nthreads_pme = 1;
+ int nthreads_pp = 1;
+ gmx_membed_t membed = NULL;
+ gmx_hw_info_t *hwinfo = NULL;
+ master_inf_t minf = {-1, FALSE};
+
+ /* CAUTION: threads may be started later on in this function, so
+ cr doesn't reflect the final parallel state right now */
+ snew(inputrec, 1);
+ snew(mtop, 1);
+
+ if (Flags & MD_APPENDFILES)
+ {
+ fplog = NULL;
+ }
+
+ bForceUseGPU = (strncmp(nbpu_opt, "gpu", 3) == 0);
+ bTryUseGPU = (strncmp(nbpu_opt, "auto", 4) == 0) || bForceUseGPU;
+
+ /* Detect hardware, gather information. This is an operation that is
+ * global for this process (MPI rank). */
+ hwinfo = gmx_detect_hardware(fplog, cr,
+ bForceUseGPU, bTryUseGPU, hw_opt->gpu_id);
+
+
+ snew(state, 1);
+ if (SIMMASTER(cr))
+ {
+ /* Read (nearly) all data required for the simulation */
+ read_tpx_state(ftp2fn(efTPX, nfile, fnm), inputrec, state, NULL, mtop);
+
+ if (inputrec->cutoff_scheme != ecutsVERLET &&
+ ((Flags & MD_TESTVERLET) || getenv("GMX_VERLET_SCHEME") != NULL))
+ {
+ convert_to_verlet_scheme(fplog, inputrec, mtop, det(state->box));
+ }
+
+
+ minf.cutoff_scheme = inputrec->cutoff_scheme;
+ minf.bUseGPU = FALSE;
+
+ if (inputrec->cutoff_scheme == ecutsVERLET)
+ {
+ prepare_verlet_scheme(fplog, hwinfo, cr,
+ inputrec, mtop, state->box,
+ &minf.bUseGPU);
+ }
+ else if (hwinfo->bCanUseGPU)
+ {
+ md_print_warn(cr, fplog,
+ "NOTE: GPU(s) found, but the current simulation can not use GPUs\n"
+ " To use a GPU, set the mdp option: cutoff-scheme = Verlet\n"
+ " (for quick performance testing you can use the -testverlet option)\n");
+
+ if (bForceUseGPU)
+ {
+ gmx_fatal(FARGS, "GPU requested, but can't be used without cutoff-scheme=Verlet");
+ }
+ }
++#ifdef GMX_IS_BGQ
++ else
++ {
++ md_print_warn(cr, fplog,
++ "NOTE: There is no SIMD implementation of the group scheme kernels on\n"
++ " BlueGene/Q. You will observe better performance from using the\n"
++ " Verlet cut-off scheme.\n");
++ }
++#endif
+ }
+#ifndef GMX_THREAD_MPI
+ if (PAR(cr))
+ {
+ gmx_bcast_sim(sizeof(minf), &minf, cr);
+ }
+#endif
+ if (minf.bUseGPU && cr->npmenodes == -1)
+ {
+ /* Don't automatically use PME-only nodes with GPUs */
+ cr->npmenodes = 0;
+ }
+
+ /* Check for externally set OpenMP affinity and turn off internal
+ * pinning if any is found. We need to do this check early to tell
+ * thread-MPI whether it should do pinning when spawning threads.
+ * TODO: the above no longer holds, we should move these checks down
+ */
+ gmx_omp_check_thread_affinity(fplog, cr, hw_opt);
+
+#ifdef GMX_THREAD_MPI
+ /* With thread-MPI inputrec is only set here on the master thread */
+ if (SIMMASTER(cr))
+#endif
+ {
+ check_and_update_hw_opt(hw_opt, minf.cutoff_scheme, SIMMASTER(cr));
+
+#ifdef GMX_THREAD_MPI
+ /* Early check for externally set process affinity. Can't do over all
+ * MPI processes because hwinfo is not available everywhere, but with
+ * thread-MPI it's needed as pinning might get turned off which needs
+ * to be known before starting thread-MPI. */
+ gmx_check_thread_affinity_set(fplog,
+ NULL,
+ hw_opt, hwinfo->nthreads_hw_avail, FALSE);
+#endif
+
+#ifdef GMX_THREAD_MPI
+ if (cr->npmenodes > 0 && hw_opt->nthreads_tmpi <= 0)
+ {
+ gmx_fatal(FARGS, "You need to explicitly specify the number of MPI threads (-ntmpi) when using separate PME nodes");
+ }
+#endif
+
+ if (hw_opt->nthreads_omp_pme != hw_opt->nthreads_omp &&
+ cr->npmenodes <= 0)
+ {
+ gmx_fatal(FARGS, "You need to explicitly specify the number of PME nodes (-npme) when using different number of OpenMP threads for PP and PME nodes");
+ }
+ }
+
+#ifdef GMX_THREAD_MPI
+ if (SIMMASTER(cr))
+ {
+ /* NOW the threads will be started: */
+ hw_opt->nthreads_tmpi = get_nthreads_mpi(hwinfo,
+ hw_opt,
+ inputrec, mtop,
+ cr, fplog);
+ if (hw_opt->nthreads_tot > 0 && hw_opt->nthreads_omp <= 0)
+ {
+ hw_opt->nthreads_omp = hw_opt->nthreads_tot/hw_opt->nthreads_tmpi;
+ }
+
+ if (hw_opt->nthreads_tmpi > 1)
+ {
+ /* now start the threads. */
+ cr = mdrunner_start_threads(hw_opt, fplog, cr_old, nfile, fnm,
+ oenv, bVerbose, bCompact, nstglobalcomm,
+ ddxyz, dd_node_order, rdd, rconstr,
+ dddlb_opt, dlb_scale, ddcsx, ddcsy, ddcsz,
+ nbpu_opt,
+ nsteps_cmdline, nstepout, resetstep, nmultisim,
+ repl_ex_nst, repl_ex_nex, repl_ex_seed, pforce,
+ cpt_period, max_hours, deviceOptions,
+ Flags);
+ /* the main thread continues here with a new cr. We don't deallocate
+ the old cr because other threads may still be reading it. */
+ if (cr == NULL)
+ {
+ gmx_comm("Failed to spawn threads");
+ }
+ }
+ }
+#endif
+ /* END OF CAUTION: cr is now reliable */
+
+ /* g_membed initialisation *
+ * Because we change the mtop, init_membed is called before the init_parallel *
+ * (in case we ever want to make it run in parallel) */
+ if (opt2bSet("-membed", nfile, fnm))
+ {
+ if (MASTER(cr))
+ {
+ fprintf(stderr, "Initializing membed");
+ }
+ membed = init_membed(fplog, nfile, fnm, mtop, inputrec, state, cr, &cpt_period);
+ }
+
+ if (PAR(cr))
+ {
+ /* now broadcast everything to the non-master nodes/threads: */
+ init_parallel(cr, inputrec, mtop);
+
+ /* This check needs to happen after get_nthreads_mpi() */
+ if (inputrec->cutoff_scheme == ecutsVERLET && (Flags & MD_PARTDEC))
+ {
+ gmx_fatal_collective(FARGS, cr, NULL,
+ "The Verlet cut-off scheme is not supported with particle decomposition.\n"
+ "You can achieve the same effect as particle decomposition by running in parallel using only OpenMP threads.");
+ }
+ }
+ if (fplog != NULL)
+ {
+ pr_inputrec(fplog, 0, "Input Parameters", inputrec, FALSE);
+ }
+
+ /* now make sure the state is initialized and propagated */
+ set_state_entries(state, inputrec, cr->nnodes);
+
+ /* A parallel command line option consistency check that we can
+ only do after any threads have started. */
+ if (!PAR(cr) &&
+ (ddxyz[XX] > 1 || ddxyz[YY] > 1 || ddxyz[ZZ] > 1 || cr->npmenodes > 0))
+ {
+ gmx_fatal(FARGS,
+ "The -dd or -npme option request a parallel simulation, "
+#ifndef GMX_MPI
+ "but %s was compiled without threads or MPI enabled"
+#else
+#ifdef GMX_THREAD_MPI
+ "but the number of threads (option -nt) is 1"
+#else
+ "but %s was not started through mpirun/mpiexec or only one process was requested through mpirun/mpiexec"
+#endif
+#endif
+ , ShortProgram()
+ );
+ }
+
+ if ((Flags & MD_RERUN) &&
+ (EI_ENERGY_MINIMIZATION(inputrec->eI) || eiNM == inputrec->eI))
+ {
+ gmx_fatal(FARGS, "The .mdp file specified an energy mininization or normal mode algorithm, and these are not compatible with mdrun -rerun");
+ }
+
+ if (can_use_allvsall(inputrec, TRUE, cr, fplog) && PAR(cr))
+ {
+ /* Simple neighbour searching and (also?) all-vs-all loops
+ * do not work with domain decomposition. */
+ Flags |= MD_PARTDEC;
+ }
+
+ if (!EEL_PME(inputrec->coulombtype) || (Flags & MD_PARTDEC))
+ {
+ if (cr->npmenodes > 0)
+ {
+ if (!EEL_PME(inputrec->coulombtype))
+ {
+ gmx_fatal_collective(FARGS, cr, NULL,
+ "PME nodes are requested, but the system does not use PME electrostatics");
+ }
+ if (Flags & MD_PARTDEC)
+ {
+ gmx_fatal_collective(FARGS, cr, NULL,
+ "PME nodes are requested, but particle decomposition does not support separate PME nodes");
+ }
+ }
+
+ cr->npmenodes = 0;
+ }
+
+#ifdef GMX_FAHCORE
+ fcRegisterSteps(inputrec->nsteps, inputrec->init_step);
+#endif
+
+ /* NMR restraints must be initialized before load_checkpoint,
+ * since with time averaging the history is added to t_state.
+ * For proper consistency check we therefore need to extend
+ * t_state here.
+ * So the PME-only nodes (if present) will also initialize
+ * the distance restraints.
+ */
+ snew(fcd, 1);
+
+ /* This needs to be called before read_checkpoint to extend the state */
+ init_disres(fplog, mtop, inputrec, cr, Flags & MD_PARTDEC, fcd, state, repl_ex_nst > 0);
+
+ if (gmx_mtop_ftype_count(mtop, F_ORIRES) > 0)
+ {
+ if (PAR(cr) && !(Flags & MD_PARTDEC))
+ {
+ gmx_fatal(FARGS, "Orientation restraints do not work (yet) with domain decomposition, use particle decomposition (mdrun option -pd)");
+ }
+ /* Orientation restraints */
+ if (MASTER(cr))
+ {
+ init_orires(fplog, mtop, state->x, inputrec, cr->ms, &(fcd->orires),
+ state);
+ }
+ }
+
+ if (DEFORM(*inputrec))
+ {
+ /* Store the deform reference box before reading the checkpoint */
+ if (SIMMASTER(cr))
+ {
+ copy_mat(state->box, box);
+ }
+ if (PAR(cr))
+ {
+ gmx_bcast(sizeof(box), box, cr);
+ }
+ /* Because we do not have the update struct available yet
+ * in which the reference values should be stored,
+ * we store them temporarily in static variables.
+ * This should be thread safe, since they are only written once
+ * and with identical values.
+ */
+#ifdef GMX_THREAD_MPI
+ tMPI_Thread_mutex_lock(&deform_init_box_mutex);
+#endif
+ deform_init_init_step_tpx = inputrec->init_step;
+ copy_mat(box, deform_init_box_tpx);
+#ifdef GMX_THREAD_MPI
+ tMPI_Thread_mutex_unlock(&deform_init_box_mutex);
+#endif
+ }
+
+ if (opt2bSet("-cpi", nfile, fnm))
+ {
+ /* Check if checkpoint file exists before doing continuation.
+ * This way we can use identical input options for the first and subsequent runs...
+ */
+ if (gmx_fexist_master(opt2fn_master("-cpi", nfile, fnm, cr), cr) )
+ {
+ load_checkpoint(opt2fn_master("-cpi", nfile, fnm, cr), &fplog,
+ cr, Flags & MD_PARTDEC, ddxyz,
+ inputrec, state, &bReadRNG, &bReadEkin,
+ (Flags & MD_APPENDFILES),
+ (Flags & MD_APPENDFILESSET));
+
+ if (bReadRNG)
+ {
+ Flags |= MD_READ_RNG;
+ }
+ if (bReadEkin)
+ {
+ Flags |= MD_READ_EKIN;
+ }
+ }
+ }
+
+ if (((MASTER(cr) || (Flags & MD_SEPPOT)) && (Flags & MD_APPENDFILES))
+#ifdef GMX_THREAD_MPI
+ /* With thread MPI only the master node/thread exists in mdrun.c,
+ * therefore non-master nodes need to open the "seppot" log file here.
+ */
+ || (!MASTER(cr) && (Flags & MD_SEPPOT))
+#endif
+ )
+ {
+ gmx_log_open(ftp2fn(efLOG, nfile, fnm), cr, !(Flags & MD_SEPPOT),
+ Flags, &fplog);
+ }
+
+ /* override nsteps with value from cmdline */
+ override_nsteps_cmdline(fplog, nsteps_cmdline, inputrec, cr);
+
+ if (SIMMASTER(cr))
+ {
+ copy_mat(state->box, box);
+ }
+
+ if (PAR(cr))
+ {
+ gmx_bcast(sizeof(box), box, cr);
+ }
+
+ /* Essential dynamics */
+ if (opt2bSet("-ei", nfile, fnm))
+ {
+ /* Open input and output files, allocate space for ED data structure */
+ ed = ed_open(mtop->natoms, &state->edsamstate, nfile, fnm, Flags, oenv, cr);
+ }
+
+ if (PAR(cr) && !((Flags & MD_PARTDEC) ||
+ EI_TPI(inputrec->eI) ||
+ inputrec->eI == eiNM))
+ {
+ cr->dd = init_domain_decomposition(fplog, cr, Flags, ddxyz, rdd, rconstr,
+ dddlb_opt, dlb_scale,
+ ddcsx, ddcsy, ddcsz,
+ mtop, inputrec,
+ box, state->x,
+ &ddbox, &npme_major, &npme_minor);
+
+ make_dd_communicators(fplog, cr, dd_node_order);
+
+ /* Set overallocation to avoid frequent reallocation of arrays */
+ set_over_alloc_dd(TRUE);
+ }
+ else
+ {
+ /* PME, if used, is done on all nodes with 1D decomposition */
+ cr->npmenodes = 0;
+ cr->duty = (DUTY_PP | DUTY_PME);
+ npme_major = 1;
+ npme_minor = 1;
+ /* NM and TPI perform single node energy calculations in parallel */
+ if (!(inputrec->eI == eiNM || EI_TPI(inputrec->eI)))
+ {
+ npme_major = cr->nnodes;
+ }
+
+ if (inputrec->ePBC == epbcSCREW)
+ {
+ gmx_fatal(FARGS,
+ "pbc=%s is only implemented with domain decomposition",
+ epbc_names[inputrec->ePBC]);
+ }
+ }
+
+ if (PAR(cr))
+ {
+ /* After possible communicator splitting in make_dd_communicators.
+ * we can set up the intra/inter node communication.
+ */
+ gmx_setup_nodecomm(fplog, cr);
+ }
+
+ /* Initialize per-physical-node MPI process/thread ID and counters. */
+ gmx_init_intranode_counters(cr);
+
+#ifdef GMX_MPI
+ md_print_info(cr, fplog, "Using %d MPI %s\n",
+ cr->nnodes,
+#ifdef GMX_THREAD_MPI
+ cr->nnodes == 1 ? "thread" : "threads"
+#else
+ cr->nnodes == 1 ? "process" : "processes"
+#endif
+ );
+ fflush(stderr);
+#endif
+
+ gmx_omp_nthreads_init(fplog, cr,
+ hwinfo->nthreads_hw_avail,
+ hw_opt->nthreads_omp,
+ hw_opt->nthreads_omp_pme,
+ (cr->duty & DUTY_PP) == 0,
+ inputrec->cutoff_scheme == ecutsVERLET);
+
+ /* check consistency and decide on the number of gpus to use. */
+ gmx_check_hw_runconf_consistency(fplog, hwinfo, cr, hw_opt->nthreads_tmpi,
+ minf.bUseGPU);
+
+ /* getting number of PP/PME threads
+ PME: env variable should be read only on one node to make sure it is
+ identical everywhere;
+ */
+ /* TODO nthreads_pp is only used for pinning threads.
+ * This is a temporary solution until we have a hw topology library.
+ */
+ nthreads_pp = gmx_omp_nthreads_get(emntNonbonded);
+ nthreads_pme = gmx_omp_nthreads_get(emntPME);
+
+ wcycle = wallcycle_init(fplog, resetstep, cr, nthreads_pp, nthreads_pme);
+
+ if (PAR(cr))
+ {
+ /* Master synchronizes its value of reset_counters with all nodes
+ * including PME only nodes */
+ reset_counters = wcycle_get_reset_counters(wcycle);
+ gmx_bcast_sim(sizeof(reset_counters), &reset_counters, cr);
+ wcycle_set_reset_counters(wcycle, reset_counters);
+ }
+
+ snew(nrnb, 1);
+ if (cr->duty & DUTY_PP)
+ {
+ /* For domain decomposition we allocate dynamically
+ * in dd_partition_system.
+ */
+ if (DOMAINDECOMP(cr))
+ {
+ bcast_state_setup(cr, state);
+ }
+ else
+ {
+ if (PAR(cr))
+ {
+ bcast_state(cr, state, TRUE);
+ }
+ }
+
+ /* Initiate forcerecord */
+ fr = mk_forcerec();
+ fr->hwinfo = hwinfo;
+ init_forcerec(fplog, oenv, fr, fcd, inputrec, mtop, cr, box,
+ opt2fn("-table", nfile, fnm),
+ opt2fn("-tabletf", nfile, fnm),
+ opt2fn("-tablep", nfile, fnm),
+ opt2fn("-tableb", nfile, fnm),
+ nbpu_opt,
+ FALSE, pforce);
+
+ /* version for PCA_NOT_READ_NODE (see md.c) */
+ /*init_forcerec(fplog,fr,fcd,inputrec,mtop,cr,box,FALSE,
+ "nofile","nofile","nofile","nofile",FALSE,pforce);
+ */
+ fr->bSepDVDL = ((Flags & MD_SEPPOT) == MD_SEPPOT);
+
+ /* Initialize QM-MM */
+ if (fr->bQMMM)
+ {
+ init_QMMMrec(cr, mtop, inputrec, fr);
+ }
+
+ /* Initialize the mdatoms structure.
+ * mdatoms is not filled with atom data,
+ * as this can not be done now with domain decomposition.
+ */
+ mdatoms = init_mdatoms(fplog, mtop, inputrec->efep != efepNO);
+
+ if (mdatoms->nPerturbed > 0 && inputrec->cutoff_scheme == ecutsVERLET)
+ {
+ gmx_fatal(FARGS, "The Verlet cut-off scheme does not (yet) support free-energy calculations with perturbed atoms, only perturbed interactions. This will be implemented soon. Use the group scheme for now.");
+ }
+
+ /* Initialize the virtual site communication */
+ vsite = init_vsite(mtop, cr, FALSE);
+
+ calc_shifts(box, fr->shift_vec);
+
+ /* With periodic molecules the charge groups should be whole at start up
+ * and the virtual sites should not be far from their proper positions.
+ */
+ if (!inputrec->bContinuation && MASTER(cr) &&
+ !(inputrec->ePBC != epbcNONE && inputrec->bPeriodicMols))
+ {
+ /* Make molecules whole at start of run */
+ if (fr->ePBC != epbcNONE)
+ {
+ do_pbc_first_mtop(fplog, inputrec->ePBC, box, mtop, state->x);
+ }
+ if (vsite)
+ {
+ /* Correct initial vsite positions are required
+ * for the initial distribution in the domain decomposition
+ * and for the initial shell prediction.
+ */
+ construct_vsites_mtop(vsite, mtop, state->x);
+ }
+ }
+
+ if (EEL_PME(fr->eeltype))
+ {
+ ewaldcoeff = fr->ewaldcoeff;
+ pmedata = &fr->pmedata;
+ }
+ else
+ {
+ pmedata = NULL;
+ }
+ }
+ else
+ {
+ /* This is a PME only node */
+
+ /* We don't need the state */
+ done_state(state);
+
+ ewaldcoeff = calc_ewaldcoeff(inputrec->rcoulomb, inputrec->ewald_rtol);
+ snew(pmedata, 1);
+ }
+
+ if (hw_opt->thread_affinity != threadaffOFF)
+ {
+ /* Before setting affinity, check whether the affinity has changed
+ * - which indicates that probably the OpenMP library has changed it
+ * since we first checked).
+ */
+ gmx_check_thread_affinity_set(fplog, cr,
+ hw_opt, hwinfo->nthreads_hw_avail, TRUE);
+
+ /* Set the CPU affinity */
+ gmx_set_thread_affinity(fplog, cr, hw_opt, hwinfo);
+ }
+
+ /* Initiate PME if necessary,
+ * either on all nodes or on dedicated PME nodes only. */
+ if (EEL_PME(inputrec->coulombtype))
+ {
+ if (mdatoms)
+ {
+ nChargePerturbed = mdatoms->nChargePerturbed;
+ }
+ if (cr->npmenodes > 0)
+ {
+ /* The PME only nodes need to know nChargePerturbed */
+ gmx_bcast_sim(sizeof(nChargePerturbed), &nChargePerturbed, cr);
+ }
+
+ if (cr->duty & DUTY_PME)
+ {
+ status = gmx_pme_init(pmedata, cr, npme_major, npme_minor, inputrec,
+ mtop ? mtop->natoms : 0, nChargePerturbed,
+ (Flags & MD_REPRODUCIBLE), nthreads_pme);
+ if (status != 0)
+ {
+ gmx_fatal(FARGS, "Error %d initializing PME", status);
+ }
+ }
+ }
+
+
+ if (integrator[inputrec->eI].func == do_md)
+ {
+ /* Turn on signal handling on all nodes */
+ /*
+ * (A user signal from the PME nodes (if any)
+ * is communicated to the PP nodes.
+ */
+ signal_handler_install();
+ }
+
+ if (cr->duty & DUTY_PP)
+ {
+ if (inputrec->ePull != epullNO)
+ {
+ /* Initialize pull code */
+ init_pull(fplog, inputrec, nfile, fnm, mtop, cr, oenv, inputrec->fepvals->init_lambda,
+ EI_DYNAMICS(inputrec->eI) && MASTER(cr), Flags);
+ }
+
+ if (inputrec->bRot)
+ {
+ /* Initialize enforced rotation code */
+ init_rot(fplog, inputrec, nfile, fnm, cr, state->x, box, mtop, oenv,
+ bVerbose, Flags);
+ }
+
+ constr = init_constraints(fplog, mtop, inputrec, ed, state, cr);
+
+ if (DOMAINDECOMP(cr))
+ {
+ dd_init_bondeds(fplog, cr->dd, mtop, vsite, inputrec,
+ Flags & MD_DDBONDCHECK, fr->cginfo_mb);
+
+ set_dd_parameters(fplog, cr->dd, dlb_scale, inputrec, &ddbox);
+
+ setup_dd_grid(fplog, cr->dd);
+ }
+
+ /* Now do whatever the user wants us to do (how flexible...) */
+ integrator[inputrec->eI].func(fplog, cr, nfile, fnm,
+ oenv, bVerbose, bCompact,
+ nstglobalcomm,
+ vsite, constr,
+ nstepout, inputrec, mtop,
+ fcd, state,
+ mdatoms, nrnb, wcycle, ed, fr,
+ repl_ex_nst, repl_ex_nex, repl_ex_seed,
+ membed,
+ cpt_period, max_hours,
+ deviceOptions,
+ Flags,
+ &runtime);
+
+ if (inputrec->ePull != epullNO)
+ {
+ finish_pull(inputrec->pull);
+ }
+
+ if (inputrec->bRot)
+ {
+ finish_rot(inputrec->rot);
+ }
+
+ }
+ else
+ {
+ /* do PME only */
+ gmx_pmeonly(*pmedata, cr, nrnb, wcycle, &runtime, ewaldcoeff, inputrec);
+ }
+
+ if (EI_DYNAMICS(inputrec->eI) || EI_TPI(inputrec->eI))
+ {
+ /* Some timing stats */
+ if (SIMMASTER(cr))
+ {
+ if (runtime.proc == 0)
+ {
+ runtime.proc = runtime.real;
+ }
+ }
+ else
+ {
+ runtime.real = 0;
+ }
+ }
+
+ wallcycle_stop(wcycle, ewcRUN);
+
+ /* Finish up, write some stuff
+ * if rerunMD, don't write last frame again
+ */
+ finish_run(fplog, cr,
+ inputrec, nrnb, wcycle, &runtime,
+ fr != NULL && fr->nbv != NULL && fr->nbv->bUseGPU ?
+ nbnxn_cuda_get_timings(fr->nbv->cu_nbv) : NULL,
+ EI_DYNAMICS(inputrec->eI) && !MULTISIM(cr));
+
+ if ((cr->duty & DUTY_PP) && fr->nbv != NULL && fr->nbv->bUseGPU)
+ {
+ char gpu_err_str[STRLEN];
+
+ /* free GPU memory and uninitialize GPU (by destroying the context) */
+ nbnxn_cuda_free(fplog, fr->nbv->cu_nbv);
+
+ if (!free_gpu(gpu_err_str))
+ {
+ gmx_warning("On node %d failed to free GPU #%d: %s",
+ cr->nodeid, get_current_gpu_device_id(), gpu_err_str);
+ }
+ }
+
+ if (opt2bSet("-membed", nfile, fnm))
+ {
+ sfree(membed);
+ }
+
+ gmx_hardware_info_free(hwinfo);
+
+ /* Does what it says */
+ print_date_and_time(fplog, cr->nodeid, "Finished mdrun", &runtime);
+
+ /* Close logfile already here if we were appending to it */
+ if (MASTER(cr) && (Flags & MD_APPENDFILES))
+ {
+ gmx_log_close(fplog);
+ }
+
+ rc = (int)gmx_get_stop_condition();
+
+#ifdef GMX_THREAD_MPI
+ /* we need to join all threads. The sub-threads join when they
+ exit this function, but the master thread needs to be told to
+ wait for that. */
+ if (PAR(cr) && MASTER(cr))
+ {
+ tMPI_Finalize();
+ }
+#endif
+
+ return rc;
+}
--- /dev/null
- df_history_t df_history,
+/*
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * VERSION 3.2.0
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2004, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * Gallium Rubidium Oxygen Manganese Argon Carbon Silicon
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include "typedefs.h"
+#include "smalloc.h"
+#include "sysstuff.h"
+#include "vec.h"
+#include "sim_util.h"
+#include "gmx_wallcycle.h"
+#include "mdrun.h"
+#include "confio.h"
+
+void
+do_trajectory_writing(FILE *fplog,
+ t_commrec *cr,
+ int nfile,
+ const t_filenm fnm[],
+ gmx_large_int_t step,
+ gmx_large_int_t step_rel,
+ double t,
+ t_inputrec *ir,
+ t_state *state,
+ t_state *state_global,
+ gmx_mtop_t *top_global,
+ t_forcerec *fr,
+ gmx_update_t upd,
+ gmx_mdoutf_t *outf,
+ t_mdebin *mdebin,
+ gmx_ekindata_t *ekind,
- if (ir->efep != efepNO || ir->bSimTemp)
- {
- state_global->fep_state = state->fep_state; /* MRS: seems kludgy. The code should be
- structured so this isn't necessary.
- Note this reassignment is only necessary
- for single threads.*/
- copy_df_history(&state_global->dfhist, &df_history);
- }
+ rvec *f,
+ rvec *f_global,
+ gmx_wallcycle_t wcycle,
+ gmx_rng_t mcrng,
+ int *nchkpt,
+ gmx_bool bCPT,
+ gmx_bool bRerunMD,
+ gmx_bool bLastStep,
+ gmx_bool bDoConfOut,
+ gmx_bool bSumEkinhOld
+ )
+{
+ int mdof_flags;
+ int n_xtc = -1;
+ rvec *x_xtc = NULL;
+
+ mdof_flags = 0;
+ if (do_per_step(step, ir->nstxout))
+ {
+ mdof_flags |= MDOF_X;
+ }
+ if (do_per_step(step, ir->nstvout))
+ {
+ mdof_flags |= MDOF_V;
+ }
+ if (do_per_step(step, ir->nstfout))
+ {
+ mdof_flags |= MDOF_F;
+ }
+ if (do_per_step(step, ir->nstxtcout))
+ {
+ mdof_flags |= MDOF_XTC;
+ }
+ if (bCPT)
+ {
+ mdof_flags |= MDOF_CPT;
+ }
+ ;
+
+#if defined(GMX_FAHCORE) || defined(GMX_WRITELASTSTEP)
+ if (bLastStep)
+ {
+ /* Enforce writing positions and velocities at end of run */
+ mdof_flags |= (MDOF_X | MDOF_V);
+ }
+#endif
+#ifdef GMX_FAHCORE
+ if (MASTER(cr))
+ {
+ fcReportProgress( ir->nsteps, step );
+ }
+
+ /* sync bCPT and fc record-keeping */
+ if (bCPT && MASTER(cr))
+ {
+ fcRequestCheckPoint();
+ }
+#endif
+
+ if (mdof_flags != 0)
+ {
+ wallcycle_start(wcycle, ewcTRAJ);
+ if (bCPT)
+ {
+ if (state->flags & (1<<estLD_RNG))
+ {
+ get_stochd_state(upd, state);
+ }
+ if (state->flags & (1<<estMC_RNG))
+ {
+ get_mc_state(mcrng, state);
+ }
+ if (MASTER(cr))
+ {
+ if (bSumEkinhOld)
+ {
+ state_global->ekinstate.bUpToDate = FALSE;
+ }
+ else
+ {
+ update_ekinstate(&state_global->ekinstate, ekind);
+ state_global->ekinstate.bUpToDate = TRUE;
+ }
+ update_energyhistory(&state_global->enerhist, mdebin);
+ }
+ }
+ write_traj(fplog, cr, outf, mdof_flags, top_global,
+ step, t, state, state_global, f, f_global, &n_xtc, &x_xtc);
+ if (bCPT)
+ {
+ (*nchkpt)++;
+ bCPT = FALSE;
+ }
+ debug_gmx();
+ if (bLastStep && step_rel == ir->nsteps &&
+ bDoConfOut && MASTER(cr) &&
+ !bRerunMD)
+ {
+ /* x and v have been collected in write_traj,
+ * because a checkpoint file will always be written
+ * at the last step.
+ */
+ fprintf(stderr, "\nWriting final coordinates.\n");
+ if (fr->bMolPBC)
+ {
+ /* Make molecules whole only for confout writing */
+ do_pbc_mtop(fplog, ir->ePBC, state->box, top_global, state_global->x);
+ }
+ write_sto_conf_mtop(ftp2fn(efSTO, nfile, fnm),
+ *top_global->name, top_global,
+ state_global->x, state_global->v,
+ ir->ePBC, state->box);
+ debug_gmx();
+ }
+ wallcycle_stop(wcycle, ewcTRAJ);
+ }
+}
biod.pnpi.spb.ru / alexxy / gromacs.git / commitdiff