# decide on GPU settings based on user-settings and GPU/CUDA detection
include(gmxManageGPU)
+# Detect the architecture the compiler is targetting, detect
+# acceleration possibilities on that hardware, suggest an acceleration
+# to use if none is specified, and populate the cache option for CPU
+# accleration.
+include(gmxDetectTargetArchitecture)
+gmx_detect_target_architecture()
include(gmxDetectAcceleration)
-if(NOT DEFINED GMX_CPU_ACCELERATION)
- if(CMAKE_CROSSCOMPILING)
- if("${CMAKE_SYSTEM_NAME}" MATCHES "BlueGeneQ")
- set(GMX_SUGGESTED_CPU_ACCELERATION "IBM_QPX")
- else()
- set(GMX_SUGGESTED_CPU_ACCELERATION "None")
- endif()
- else(CMAKE_CROSSCOMPILING)
- gmx_detect_acceleration(GMX_SUGGESTED_CPU_ACCELERATION)
- endif(CMAKE_CROSSCOMPILING)
-endif(NOT DEFINED GMX_CPU_ACCELERATION)
+gmx_detect_acceleration(GMX_SUGGESTED_CPU_ACCELERATION)
gmx_option_multichoice(
GMX_CPU_ACCELERATION
# The user should not be able to set this orthogonally to the acceleration
set(GMX_X86_SSE2 1)
set(ACCELERATION_STATUS_MESSAGE
- "Enabling SSE2 Gromacs acceleration, and it will help compiler optimization.")
+ "Enabling SSE2 Gromacs acceleration")
elseif(${GMX_CPU_ACCELERATION} STREQUAL "SSE4.1")
set(GMX_X86_SSE4_1 1)
set(GMX_X86_SSE2 1)
set(ACCELERATION_STATUS_MESSAGE
- "Enabling SSE4.1 Gromacs acceleration, and it will help compiler optimization.")
+ "Enabling SSE4.1 Gromacs acceleration")
elseif(${GMX_CPU_ACCELERATION} STREQUAL "AVX_128_FMA" OR ${GMX_CPU_ACCELERATION} STREQUAL "AVX_256")
set(GMX_CPU_ACCELERATION_X86_AVX_128_FMA 1)
set(GMX_X86_AVX_128_FMA 1)
set(ACCELERATION_STATUS_MESSAGE
- "Enabling 128-bit AVX Gromacs acceleration (with fused-multiply add), and it will help compiler optimization.")
+ "Enabling 128-bit AVX Gromacs acceleration (with fused-multiply add)")
else()
# If we are not doing AVX_128, it must be AVX_256...
set(GMX_CPU_ACCELERATION_X86_AVX_256 1)
set(GMX_X86_AVX_256 1)
set(ACCELERATION_STATUS_MESSAGE
- "Enabling 256-bit AVX Gromacs acceleration, and it will help compiler optimization.")
+ "Enabling 256-bit AVX Gromacs acceleration")
endif()
# Unfortunately gcc-4.5.2 and gcc-4.6.0 has a bug where they use the wrong datatype for the formal
gmx_test_avx_gcc_maskload_bug(${ACCELERATION_C_FLAGS} GMX_X86_AVX_GCC_MASKLOAD_BUG)
elseif(${GMX_CPU_ACCELERATION} STREQUAL "IBM_QPX")
- # Used on BlueGene/Q
- if (CMAKE_C_COMPILER_ID MATCHES "XL")
- GMX_TEST_CFLAG(XLC_BLUEGENEQ_CFLAG "-qarch=qp -qtune=qp" ACCELERATION_C_FLAGS)
- try_compile(TEST_QPX ${CMAKE_BINARY_DIR}
- "${CMAKE_SOURCE_DIR}/cmake/TestQPX.c"
- COMPILE_DEFINITIONS "${ACCELERATION_C_FLAGS}")
- if(NOT TEST_QPX)
- message(FATAL_ERROR "Cannot compile the requested IBM QPX intrinsics.")
- endif()
- endif()
- if (CMAKE_CXX_COMPILER_ID MATCHES "XL")
- GMX_TEST_CXXFLAG(XLC_BLUEGENEQ_CXXFLAG "-qarch=qp -qtune=qp" ACCELERATION_CXX_FLAGS)
- try_compile(TEST_QPX ${CMAKE_BINARY_DIR}
- "${CMAKE_SOURCE_DIR}/cmake/TestQPX.c"
- COMPILE_DEFINITIONS "${ACCELERATION_CXX_FLAGS}")
- if(NOT TEST_QPX)
- message(FATAL_ERROR "Cannot compile the requested IBM QPX intrinsics.")
- endif()
- endif()
+ try_compile(TEST_QPX ${CMAKE_BINARY_DIR}
+ "${CMAKE_SOURCE_DIR}/cmake/TestQPX.c")
if (TEST_QPX)
- message(WARNING "IBM QPX acceleration was selected and could be compiled, but the accelerated kernels are not yet available.")
+ message(WARNING "IBM QPX acceleration was selected. This will work, but SIMD-accelerated kernels are only available for the Verlet cut-off scheme. The plain C kernels that are used for the group cut-off scheme kernels will be slow, so please consider using the Verlet cut-off scheme.")
set(GMX_CPU_ACCELERATION_IBM_QPX 1)
else()
- message(FATAL_ERROR "Cannot compile IBM QPX intrinsics without the XL compiler. If you are compiling for BlueGene/Q, use 'cmake .. -DCMAKE_TOOLCHAIN_FILE=BlueGeneQ-static-XL-C' to set up the tool chain.")
+ message(FATAL_ERROR "Cannot compile the requested IBM QPX intrinsics. If you are compiling for BlueGene/Q with the XL compilers, use 'cmake .. -DCMAKE_TOOLCHAIN_FILE=Platform/BlueGeneQ-static-XL-C' to set up the tool chain.")
endif()
elseif(${GMX_CPU_ACCELERATION} STREQUAL "SPARC64_HPC_ACE")
set(GMX_CPU_ACCELERATION_SPARC64_HPC_ACE 1)
if(HAVE_LIBM)
list(APPEND GMX_EXTRA_LIBRARIES m)
endif(HAVE_LIBM)
+if (${CMAKE_SYSTEM_NAME} MATCHES "BlueGene")
+ check_library_exists(mass_simd atan2f4 "" HAVE_MASS_SIMD)
+ if(HAVE_MASS_SIMD)
+ list(APPEND GMX_EXTRA_LIBRARIES mass_simd)
+ else()
+ message(FATAL_ERROR "Could not link to the SIMD version of the IBM MASS library. Please adjust your CMAKE_PREFIX_PATH to contain it")
+ endif()
+endif()
if(GMX_FAHCORE)
set(COREWRAP_INCLUDE_DIR "${CMAKE_SOURCE_DIR}/../corewrap" CACHE STRING
build \fftw{} \fftwversion{} from source automatically for you (use
\verb+cmake -DGMX_BUILD_OWN_FFTW=ON+), or
\item that you build \fftw{} from the source code.
+Note that the GROMACS-managed download of the FFTW tarball has a
+slight chance of posing a security risk. If you use this option, you
+will see a warning that advises how you can eliminate this risk.
\end{itemize}
If you build \fftw{} from source yourself, get the most recent version
\end{itemize}
The respective '\verb+include+', '\verb+lib+', or '\verb+bin+' is
appended to the path. For each of these variables, a list of paths can
-be specified (on Unix seperated with ":"). Note that these are
+be specified (on Unix separated with ":"). Note that these are
enviroment variables (and not \cmake{} command-line arguments) and in
a '\verb+bash+' shell are used like:
\begin{verbatim}
accordingly. The internal versions are fine for normal use. If you
need to specify a non-standard path to search, use
\verb+-DCMAKE_PREFIX_PATH=/path/to/search+. If you need to specify a
-library with a non-standard name (e.g. ESSL on AIX), then set
-\verb+-DGMX_BLAS_USER=/path/to/reach/lib/libwhatever.a+.
+library with a non-standard name (e.g. ESSL on AIX or BlueGene), then
+set \verb+-DGMX_BLAS_USER=/path/to/reach/lib/libwhatever.a+.
If you are using Intel's \mkl{} for \fft{}, then the \blas{} and
\lapack{} it provides are used automatically. This could be
\subsubsection{BlueGene/P}
-There is currently no native acceleration on this platform, but the
-default plain C kernels will work.
+There is currently no native acceleration on this platform and no
+plans to make one. The default plain C kernels will work.
\subsubsection{BlueGene/Q}
-There is currently no native acceleration on this platform, but the
-default plain C kernels will work. We have accelerated kernels in
-progress for this platform, but they are not quite done yet.
+There is currently native acceleration on this platform for the Verlet
+cut-off scheme. Accelerated kernels for the group cut-off scheme may
+come in the future, but the default plain C kernels will work.
Only static linking with XL compilers is supported by \gromacs{}. Dynamic
linking would be supported by the architecture and \gromacs{}, but has no
above instructions.
mpicc is used for compiling and linking. This can make it awkward to
-attempt to use IBM's optimized BLAS/LAPACK called ESSL. Since mdrun is
-the only part of \gromacs{} that should normally run on the compute
-nodes, and there is nearly no need for linear algebra support for
-mdrun, it is recommended to use the \gromacs{} built-in linear algebra
-routines - it is rare for this to be a bottleneck.
-
+attempt to use IBM's optimized BLAS/LAPACK called ESSL (see the
+section on linear algebra). Since mdrun is the only part of \gromacs{}
+that should normally run on the compute nodes, and there is nearly no
+need for linear algebra support for mdrun, it is recommended to use
+the \gromacs{} built-in linear algebra routines - it is rare for this
+to be a bottleneck.
+
+The recommended configuration is to use
\begin{verbatim}
-cmake .. -DCMAKE_TOOLCHAIN_FILE=BlueGeneQ-static-XL-C \
+cmake .. -DCMAKE_TOOLCHAIN_FILE=Platform/BlueGeneQ-static-XL-CXX \
-DCMAKE_PREFIX_PATH=/your/fftw/installation/prefix \
+ -DGMX_MPI=ON \
-DGMX_BUILD_MDRUN_ONLY=ON
make
make install
\end{verbatim}
+which will build a statically-linked MPI-enabled mdrun for the back
+end. Otherwise, GROMACS default configuration behaviour applies.
+
It is possible to configure and make the remaining \gromacs{} tools
-with the compute node toolchain, but as none of those tools are
-\mpi{}-aware, this would not normally be useful. Instead, these should
-be planned to run on the login node, and a seperate \gromacs{}
-installation performed for that using the login node's toolchain.
+with the compute-node toolchain, but as none of those tools are
+\mpi{}-aware and could then only run on the compute nodes, this
+would not normally be useful. Instead, these should be planned
+to run on the login node, and a separate \gromacs{} installation
+performed for that using the login node's toolchain - not the
+above platform file, or any other compute-node toolchain.
+
+Note that only the MPI build is available for the compute-node
+toolchains. The GROMACS thread-MPI or serial builds are not useful at
+all on BlueGene/Q.
\subsubsection{Fujitsu PRIMEHPC}
from 4.4 through 4.7, and versions 12 and 13 of the Intel compiler.
Under Windows we test both the visual studio compilers and icc,
-We test irregularly on BlueGene/L, BlueGene/P, BlueGene/Q, Cray,
+We test irregularly on BlueGene/Q, Cray,
Fujitsu PRIMEHPC, Google nativeclient and other environments. In
the future we expect ARM to be an important test target too, but this
is currently not included.
"<FLAGS> <CMAKE_${lang}_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
set(CMAKE_${lang}_LINK_EXECUTABLE
"<CMAKE_${lang}_COMPILER> ${BG/Q_${lang}_DEFAULT_EXE_FLAGS}")
+
+ if(CMAKE_BUILD_TYPE STREQUAL "Debug" AND ${compiler_id} STREQUAL "XL")
+ # Work around an unknown compiler bug triggered in
+ # compute_globals(). Using -O0 disables -qhot and this seems
+ # to break the normal OpenMP flag -qsmp unless qualified with
+ # noauto.
+ set(OpenMP_C_FLAGS "-qsmp=noauto" CACHE STRING "Compiler flag for OpenMP parallelization")
+ set(OpenMP_CXX_FLAGS "-qsmp=noauto" CACHE STRING "Compiler flag for OpenMP parallelization")
+ endif()
+
endmacro()
int main(void)
{
- return 0;
+ int i;
+ void *ptr;
+ tMPI_Atomic_t some_atomic;
+ tMPI_Atomic_ptr_t *some_atomic_ptr = NULL;
+ tMPI_Spinlock_t some_spinlock;
+
+ /* Make the compiler actually emit code for these functions, so
+ that things like inability to emit inline assembly get
+ tested. It is not expected that the code below can run. */
+ tMPI_Atomic_memory_barrier();
+ tMPI_Atomic_memory_barrier_acq();
+ tMPI_Atomic_memory_barrier_rel();
+ i = tMPI_Atomic_get(&some_atomic);
+ tMPI_Atomic_set(&some_atomic, 0);
+ ptr = tMPI_Atomic_ptr_get(some_atomic_ptr);
+ tMPI_Atomic_ptr_set(some_atomic_ptr, ptr);
+ tMPI_Atomic_add_return(&some_atomic, 0);
+ tMPI_Atomic_fetch_add(&some_atomic, 0);
+ tMPI_Atomic_cas(&some_atomic, 0, 1);
+ tMPI_Atomic_ptr_cas(some_atomic_ptr, ptr, ptr);
+ tMPI_Atomic_swap(&some_atomic, 0);
+ tMPI_Atomic_ptr_swap(some_atomic_ptr, ptr);
+ tMPI_Spinlock_init(&some_spinlock);
+ tMPI_Spinlock_lock(&some_spinlock);
+ tMPI_Spinlock_trylock(&some_spinlock);
+ tMPI_Spinlock_unlock(&some_spinlock);
+ tMPI_Spinlock_islocked(&some_spinlock);
+ tMPI_Spinlock_wait(&some_spinlock);
+return 0;
}
--- /dev/null
+int main()
+{
+#ifdef __bgq__
+ return 0;
+#else
+#error This compiler is not targetting BlueGene/Q
+#endif
+}
--- /dev/null
+int main()
+{
+#if defined (__i386__) || defined (__x86_64__) || defined (_M_IX86) || defined (_M_X64)
+ return 0;
+#else
+#error This is not x86
+#endif
+}
# - Check the username performing the build, as well as date and time
#
-# GMX_DETECT_ACCELERATION(GMX_SUGGESTED_ACCELERATION)
+# gmx_detect_acceleration(GMX_SUGGESTED_CPU_ACCELERATION)
#
# Try to detect CPU information and suggest an acceleration option
-# (such as SSE/AVX) that fits the current CPU.
+# (such as SSE/AVX) that fits the current CPU. These functions assume
+# that gmx_detect_target_architecture() has already been run, so that
+# things like GMX_IS_X86 are already available.
#
-# GMX_SUGGESTED_ACCELERATION
+# Sets ${GMX_SUGGESTED_CPU_ACCELERATION} in the parent scope if
+# GMX_CPU_ACCELERATION is not set (e.g. by the user, or a previous run
+# of CMake).
#
# we rely on inline asm support for GNU!
include(gmxTestInlineASM)
-macro(gmx_detect_acceleration GMX_SUGGESTED_ACCELERATION)
- IF(NOT DEFINED ${GMX_SUGGESTED_ACCELERATION})
+function(gmx_suggest_x86_acceleration _suggested_acceleration)
gmx_test_inline_asm_gcc_x86(GMX_X86_GCC_INLINE_ASM)
try_run(GMX_CPUID_RUN_ACC GMX_CPUID_COMPILED
${CMAKE_BINARY_DIR}
${CMAKE_SOURCE_DIR}/src/gromacs/gmxlib/gmx_cpuid.c
- COMPILE_DEFINITIONS "@GCC_INLINE_ASM_DEFINE@ -I${CMAKE_SOURCE_DIR}/src/gromacs/legacyheaders/ -DGMX_CPUID_STANDALONE"
+ COMPILE_DEFINITIONS "@GCC_INLINE_ASM_DEFINE@ -I${CMAKE_SOURCE_DIR}/src/gromacs/legacyheaders -DGMX_CPUID_STANDALONE -DGMX_IS_X86"
RUN_OUTPUT_VARIABLE OUTPUT_TMP
COMPILE_OUTPUT_VARIABLE GMX_CPUID_COMPILE_OUTPUT
ARGS "-acceleration")
string(STRIP "@OUTPUT_TMP@" OUTPUT_ACC)
- message(STATUS "Detecting best acceleration for this CPU - @OUTPUT_ACC@")
+ set(${_suggested_acceleration} "@OUTPUT_ACC@" PARENT_SCOPE)
+ message(STATUS "Detected best acceleration for this CPU - @OUTPUT_ACC@")
+endfunction()
- set(${GMX_SUGGESTED_ACCELERATION} "@OUTPUT_ACC@" CACHE INTERNAL "GROMACS CPU-specific acceleration")
-
- ENDIF(NOT DEFINED ${GMX_SUGGESTED_ACCELERATION})
-endmacro(gmx_detect_acceleration GMX_SUGGESTED_ACCELERATION)
+function(gmx_detect_acceleration _suggested_acceleration)
+ if(NOT DEFINED GMX_CPU_ACCELERATION)
+ if(GMX_IS_BGQ)
+ set(${_suggested_acceleration} "IBM_QPX")
+ elseif(GMX_IS_X86)
+ gmx_suggest_x86_acceleration(${_suggested_acceleration})
+ else()
+ set(${_suggested_acceleration} "None")
+ endif()
+ set(${_suggested_acceleration} ${${_suggested_acceleration}} PARENT_SCOPE)
+ endif()
+endfunction()
--- /dev/null
+# - Define function to detect whether the compiler's target
+# - architecture is one for which GROMACS has special treatment
+# - (e.g. kernel acceleration)
+#
+# Sets GMX_IS_X86 or GMX_IS_BGQ if targetting that architecture
+
+function(gmx_detect_target_architecture)
+ try_compile(GMX_IS_X86 ${CMAKE_BINARY_DIR}
+ "${CMAKE_SOURCE_DIR}/cmake/TestX86.c")
+ try_compile(GMX_IS_BGQ ${CMAKE_BINARY_DIR}
+ "${CMAKE_SOURCE_DIR}/cmake/TestBlueGeneQ.c")
+endfunction()
# - with cmake <=2.8.8: compilers that accept "-dumpversion" argument:
# gcc, Intel Compiler (on Linux and Mac OS), Open64, EkoPath, clang
# (and probably other gcc-compatible compilers).
+# - with cmake <=2.8.8: xlC is not supported (it does not take -dumpversion,
+# but fortunately so far GROMACS never needs to know the version number)
#
# C_COMPILER_VERSION - version string of the current C compiler (CMAKE_C_COMPILER)
# CXX_COMPILER_VERSION - version string of the current C++ compiler (CMAKE_CXX_COMPILER)
set(_cc_dumpversion_res 0)
if (DEFINED CMAKE_C_COMPILER_VERSION AND CMAKE_VERSION VERSION_GREATER 2.8.8)
set(_cc_version ${CMAKE_C_COMPILER_VERSION})
+ elseif (CMAKE_C_COMPILER_ID MATCHES "XL")
+ set(_cc_dumpversion_res 1)
else()
execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion
RESULT_VARIABLE _cc_dumpversion_res
set(_cxx_dumpversion_res 0)
if (DEFINED CMAKE_CXX_COMPILER_VERSION AND CMAKE_VERSION VERSION_GREATER 2.8.8)
set(_cxx_version ${CMAKE_CXX_COMPILER_VERSION})
+ elseif (CMAKE_CXX_COMPILER_ID MATCHES "XL")
+ set(_cxx_dumpversion_res 1)
else()
execute_process(COMMAND ${CMAKE_CXX_COMPILER} -dumpversion
RESULT_VARIABLE _cxx_dumpversion_res
# BUILD_FLAGS - [output variable] flags for the compiler
#
macro(get_compiler_info LANGUAGE BUILD_COMPILER BUILD_FLAGS)
- execute_process(COMMAND ${CMAKE_${LANGUAGE}_COMPILER} --version
+ if (CMAKE_C_COMPILER_ID MATCHES "XL")
+ set(_flag_to_query_version "-qversion")
+ else()
+ set(_flag_to_query_version "--version")
+ endif()
+ execute_process(COMMAND ${CMAKE_${LANGUAGE}_COMPILER} ${_flag_to_query_version}
RESULT_VARIABLE _exec_result
OUTPUT_VARIABLE _compiler_version
ERROR_VARIABLE _compiler_version)
- # Try executing just the compiler command --version failed
+
if(_exec_result)
+ # Try executing just the compiler command, since --version failed
execute_process(COMMAND ${CMAKE_${LANGUAGE}_COMPILER}
RESULT_VARIABLE _exec_result
OUTPUT_VARIABLE _compiler_version
# facility to run lots of jobs on small chunks of the machine. You
# certainly need proper MPI to use a whole chunk of the machine that
# the scheduler will allocate.
-set(GMX_THREAD_MPI OFF CACHE BOOL "Thread-MPI generally not compatible with BlueGene, defaulting to disabled!")
-set(GMX_MPI ON CACHE BOOL "MPI is normally required on BlueGene" FORCE)
+set(GMX_THREAD_MPI OFF CACHE BOOL "GROMACS bundled thread-MPI is not supported on BlueGene" FORCE)
+set(GMX_MPI ON CACHE BOOL "MPI is required on BlueGene" FORCE)
# Access to /etc/passwd is not available on the back end of BlueGeneP
# (at least), despite being detected by CMake. This can cause linker
# optimized for sm_35 results in lower performance than with sm_30.
if(CUDA_VERSION VERSION_LESS "4.2")
set(_CUDA_ARCH_STR "-gencode;arch=compute_20,code=sm_20;-gencode;arch=compute_20,code=sm_21;-gencode;arch=compute_20,code=compute_20")
- else()
+ elseif(CUDA_VERSION VERSION_LESS "5.0")
set(_CUDA_ARCH_STR "-gencode;arch=compute_20,code=sm_20;-gencode;arch=compute_20,code=sm_21;-gencode;arch=compute_30,code=sm_30;-gencode;arch=compute_30,code=compute_30")
+ else()
+ set(_CUDA_ARCH_STR "-gencode;arch=compute_20,code=sm_20;-gencode;arch=compute_20,code=sm_21;-gencode;arch=compute_30,code=sm_30;-gencode;arch=compute_35,code=sm_35;-gencode;arch=compute_35,code=compute_35")
endif()
# finally set the damn flags
if(NOT CMAKE_CROSSCOMPILING)
# Get CPU acceleration information
+ set(_compile_definitions "@GCC_INLINE_ASM_DEFINE@ -I${CMAKE_SOURCE_DIR}/src/gromacs/legacyheaders -DGMX_CPUID_STANDALONE -DGMX_IS_X86")
try_run(GMX_CPUID_RUN_VENDOR GMX_CPUID_COMPILED
${CMAKE_BINARY_DIR}
${CMAKE_SOURCE_DIR}/src/gromacs/gmxlib/gmx_cpuid.c
- COMPILE_DEFINITIONS "@GCC_INLINE_ASM_DEFINE@ -I${CMAKE_SOURCE_DIR}/src/gromacs/legacyheaders/ -DGMX_CPUID_STANDALONE"
+ COMPILE_DEFINITIONS ${_compile_definitions}
RUN_OUTPUT_VARIABLE OUTPUT_CPU_VENDOR ARGS "-vendor")
try_run(GMX_CPUID_RUN_BRAND GMX_CPUID_COMPILED
${CMAKE_BINARY_DIR}
${CMAKE_SOURCE_DIR}/src/gromacs/gmxlib/gmx_cpuid.c
- COMPILE_DEFINITIONS "@GCC_INLINE_ASM_DEFINE@ -I${CMAKE_SOURCE_DIR}/src/gromacs/legacyheaders/ -DGMX_CPUID_STANDALONE"
+ COMPILE_DEFINITIONS ${_compile_definitions}
RUN_OUTPUT_VARIABLE OUTPUT_CPU_BRAND ARGS "-brand")
try_run(GMX_CPUID_RUN_FAMILY GMX_CPUID_COMPILED
${CMAKE_BINARY_DIR}
${CMAKE_SOURCE_DIR}/src/gromacs/gmxlib/gmx_cpuid.c
- COMPILE_DEFINITIONS "@GCC_INLINE_ASM_DEFINE@ -I${CMAKE_SOURCE_DIR}/src/gromacs/legacyheaders/ -DGMX_CPUID_STANDALONE"
+ COMPILE_DEFINITIONS ${_compile_definitions}
RUN_OUTPUT_VARIABLE OUTPUT_CPU_FAMILY ARGS "-family")
try_run(GMX_CPUID_RUN_MODEL GMX_CPUID_COMPILED
${CMAKE_BINARY_DIR}
${CMAKE_SOURCE_DIR}/src/gromacs/gmxlib/gmx_cpuid.c
- COMPILE_DEFINITIONS "@GCC_INLINE_ASM_DEFINE@ -I${CMAKE_SOURCE_DIR}/src/gromacs/legacyheaders/ -DGMX_CPUID_STANDALONE"
+ COMPILE_DEFINITIONS ${_compile_definitions}
RUN_OUTPUT_VARIABLE OUTPUT_CPU_MODEL ARGS "-model")
try_run(GMX_CPUID_RUN_STEPPING GMX_CPUID_COMPILED
${CMAKE_BINARY_DIR}
${CMAKE_SOURCE_DIR}/src/gromacs/gmxlib/gmx_cpuid.c
- COMPILE_DEFINITIONS "@GCC_INLINE_ASM_DEFINE@ -I${CMAKE_SOURCE_DIR}/src/gromacs/legacyheaders/ -DGMX_CPUID_STANDALONE"
+ COMPILE_DEFINITIONS ${_compile_definitions}
RUN_OUTPUT_VARIABLE OUTPUT_CPU_STEPPING ARGS "-stepping")
try_run(GMX_CPUID_RUN_FEATURES GMX_CPUID_COMPILED
${CMAKE_BINARY_DIR}
${CMAKE_SOURCE_DIR}/src/gromacs/gmxlib/gmx_cpuid.c
- COMPILE_DEFINITIONS "@GCC_INLINE_ASM_DEFINE@ -I${CMAKE_SOURCE_DIR}/src/gromacs/legacyheaders/ -DGMX_CPUID_STANDALONE"
+ COMPILE_DEFINITIONS ${_compile_definitions}
RUN_OUTPUT_VARIABLE OUTPUT_CPU_FEATURES ARGS "-features")
+ unset(_compile_definitions)
string(STRIP "@OUTPUT_CPU_VENDOR@" OUTPUT_CPU_VENDOR)
string(STRIP "@OUTPUT_CPU_BRAND@" OUTPUT_CPU_BRAND)
# GMX_TEST_INLINE_ASM_GCC_X86(VARIABLE)
#
# VARIABLE will be set to true if GCC x86 inline asm works.
-#
-# Remember to have a cmakedefine for it too...
MACRO(GMX_TEST_INLINE_ASM_GCC_X86 VARIABLE)
IF(NOT DEFINED ${VARIABLE})
/* Use AMD core math library */
#cmakedefine GMX_FFT_ACML
+/* Target platform is x86 or x86_64 */
+#cmakedefine GMX_IS_X86
+
+/* Target platform is BlueGene/Q */
+#cmakedefine GMX_IS_BGQ
+
/* SSE2 instructions available */
#cmakedefine GMX_X86_SSE2
# Machinery for running the external project
set(EXTERNAL_FFTW_VERSION 3.3.2)
-set(EXTERNAL_FFTW_VERSION_MD5_SUM 6977ee770ed68c85698c7168ffa6e178)
include(ExternalProject)
-if(CMAKE_VERSION VERSION_LESS 2.8.3)
- # Can't check MD5 sum with this CMake version
- message(WARNING "GROMACS is downloading FFTW ${EXTERNAL_FFTW_VERSION}, but your version of CMake is too old to allow GROMACS to check that the archive it receives is correct. GROMACS will build and link to FFTW anyway, but there is a security risk if you execute a GROMACS tool that calls this library. Either build your own FFTW, or update CMake to at least version 2.8.3.")
- ExternalProject_add(gmxfftw
- URL "http://www.fftw.org/fftw-${EXTERNAL_FFTW_VERSION}.tar.gz"
- CONFIGURE_COMMAND <SOURCE_DIR>/configure --prefix=<INSTALL_DIR>
- ${GMX_BUILD_OWN_FFTW_SHARED_FLAG} ${GMX_BUILD_OWN_FFTW_OPTIMIZATION_CONFIGURATION}
- ${GMX_BUILD_OWN_FFTW_PREC})
-else()
- ExternalProject_add(gmxfftw
+# TODO in master branch - show this warning only on the first run
+# by using gmx_check_if_changed_result from I21b791ab8e4f3 when
+# that becomes available
+message(WARNING "The GROMACS build will download FFTW ${EXTERNAL_FFTW_VERSION} as requested, but it will not know the file it receives is correct. GROMACS will build and link to FFTW anyway, but there is a possible security risk if you execute a GROMACS tool that calls this library. You can use\nmake gmxfftw\n to do the download and build, and then run\nmd5sum src/contrib/fftw/gmxfftw-prefix/src/fftw-3.3.2.tar.gz\nto see if it matches 6977ee770ed68c85698c7168ffa6e178. If so, everything is OK and you should proceed with the rest of the GROMACS build. Alternatively, you can follow the GROMACS installation instructions to build FFTW yourself.")
+# TODO if/when CMake fixes http://www.cmake.org/Bug/view.php?id=14330
+# (ie. at least version > 2.8.11.2), consider reverting to using an
+# md5sum check to avoid needing the above warning
+ ExternalProject_add(fftwBuild
URL "http://www.fftw.org/fftw-${EXTERNAL_FFTW_VERSION}.tar.gz"
- URL_MD5 ${EXTERNAL_FFTW_VERSION_MD5_SUM}
- CONFIGURE_COMMAND <SOURCE_DIR>/configure --prefix=<INSTALL_DIR>
+ CONFIGURE_COMMAND <SOURCE_DIR>/configure --prefix=<INSTALL_DIR> --libdir=<INSTALL_DIR>/lib
${GMX_BUILD_OWN_FFTW_SHARED_FLAG} ${GMX_BUILD_OWN_FFTW_OPTIMIZATION_CONFIGURATION}
${GMX_BUILD_OWN_FFTW_PREC})
-endif()
-externalproject_get_property(gmxfftw INSTALL_DIR)
-# The dependency that triggers building the gmxfftw target gets made where libmd is constructed
+externalproject_get_property(fftwBuild INSTALL_DIR)
string(TOUPPER "${FFTW}" UPPERFFTW)
string(TOLOWER "${FFTW}" LOWERFFTW)
string(REGEX REPLACE "fftw" "fftw3" FFTW_LIBNAME ${LOWERFFTW})
-set(${UPPERFFTW}_LIBRARIES ${CMAKE_BINARY_PREFIX}/${INSTALL_DIR}/lib/lib${FFTW_LIBNAME}${CMAKE_STATIC_LIBRARY_SUFFIX} PARENT_SCOPE)
+set(${UPPERFFTW}_LIBRARIES ${CMAKE_BINARY_PREFIX}/${INSTALL_DIR}/lib/lib${FFTW_LIBNAME}${CMAKE_STATIC_LIBRARY_SUFFIX})
set(${UPPERFFTW}_INCLUDE_DIRS ${CMAKE_BINARY_PREFIX}/${INSTALL_DIR}/include PARENT_SCOPE)
set(${UPPERFFTW}_FOUND TRUE PARENT_SCOPE)
set(${UPPERFFTW}_HAVE_SIMD TRUE PARENT_SCOPE)
+if(CMAKE_VERSION VERSION_GREATER 2.8.7) #add_library GLOBAL is broken in 2.8.7
+ add_library(gmxfftw STATIC IMPORTED GLOBAL)
+ set_target_properties(gmxfftw PROPERTIES IMPORTED_LOCATION ${${UPPERFFTW}_LIBRARIES})
+ set(${UPPERFFTW}_LIBRARIES gmxfftw PARENT_SCOPE)
+else()
+ # The dependency that triggers building the gmxfftw target gets made where libmd is constructed
+ add_custom_target(gmxfftw)
+ set(${UPPERFFTW}_LIBRARIES ${${UPPERFFTW}_LIBRARIES} PARENT_SCOPE)
+endif()
+add_dependencies(gmxfftw fftwBuild)
+
message(STATUS "The GROMACS-managed build of FFTW 3 will configure with the following optimizations: ${GMX_BUILD_OWN_FFTW_OPTIMIZATION_CONFIGURATION}")
endif ()
if(GMX_BUILD_OWN_FFTW)
+ # Only needed for cmake 2.8.7, otherwise this should be automatic
# This dependency has to be made here rather than the CMakeLists.txt that
# does the FFTW build, because of the order in which
# add_subdirectory() calls are made in the top-level CMakeLists.txt; the
#ifdef GMX_MPI
if ((s == 0 && !(plan->flags&FFT5D_ORDER_YZ)) || (s == 1 && (plan->flags&FFT5D_ORDER_YZ)))
{
- MPI_Alltoall(lout2, N[s]*pM[s]*K[s]*sizeof(t_complex)/sizeof(real), GMX_MPI_REAL, lout3, N[s]*pM[s]*K[s]*sizeof(t_complex)/sizeof(real), GMX_MPI_REAL, cart[s]);
+ MPI_Alltoall((real *)lout2, N[s]*pM[s]*K[s]*sizeof(t_complex)/sizeof(real), GMX_MPI_REAL, (real *)lout3, N[s]*pM[s]*K[s]*sizeof(t_complex)/sizeof(real), GMX_MPI_REAL, cart[s]);
}
else
{
- MPI_Alltoall(lout2, N[s]*M[s]*pK[s]*sizeof(t_complex)/sizeof(real), GMX_MPI_REAL, lout3, N[s]*M[s]*pK[s]*sizeof(t_complex)/sizeof(real), GMX_MPI_REAL, cart[s]);
+ MPI_Alltoall((real *)lout2, N[s]*M[s]*pK[s]*sizeof(t_complex)/sizeof(real), GMX_MPI_REAL, (real *)lout3, N[s]*M[s]*pK[s]*sizeof(t_complex)/sizeof(real), GMX_MPI_REAL, cart[s]);
}
#else
gmx_incons("fft5d MPI call without MPI configuration");
};
FILE *fp, *log;
- int i, i1, i2, j, nf, nrms;
+ int nf, i, i1, i2, j;
+ gmx_large_int_t nrms = 0;
matrix box;
rvec *xtps, *usextps, *x1, **xx = NULL;
}
}
}
- /* Initiate arrays */
- snew(d1, isize);
- snew(d2, isize);
- for (i = 0; (i < isize); i++)
- {
- snew(d1[i], isize);
- snew(d2[i], isize);
- }
if (bReadTraj)
{
else /* !bReadMat */
{
rms = init_mat(nf, method == m_diagonalize);
- nrms = (nf*(nf-1))/2;
+ nrms = ((gmx_large_int_t)nf*((gmx_large_int_t)nf-1))/2;
if (!bRMSdist)
{
fprintf(stderr, "Computing %dx%d RMS deviation matrix\n", nf, nf);
+ /* Initialize work array */
snew(x1, isize);
- for (i1 = 0; (i1 < nf); i1++)
+ for (i1 = 0; i1 < nf; i1++)
{
- for (i2 = i1+1; (i2 < nf); i2++)
+ for (i2 = i1+1; i2 < nf; i2++)
{
for (i = 0; i < isize; i++)
{
rmsd = rmsdev(isize, mass, xx[i2], x1);
set_mat_entry(rms, i1, i2, rmsd);
}
- nrms -= (nf-i1-1);
- fprintf(stderr, "\r# RMSD calculations left: %d ", nrms);
+ nrms -= (gmx_large_int_t) (nf-i1-1);
+ fprintf(stderr, "\r# RMSD calculations left: "gmx_large_int_pfmt" ", nrms);
}
+ sfree(x1);
}
else /* bRMSdist */
{
fprintf(stderr, "Computing %dx%d RMS distance deviation matrix\n", nf, nf);
- for (i1 = 0; (i1 < nf); i1++)
+
+ /* Initiate work arrays */
+ snew(d1, isize);
+ snew(d2, isize);
+ for (i = 0; (i < isize); i++)
+ {
+ snew(d1[i], isize);
+ snew(d2[i], isize);
+ }
+ for (i1 = 0; i1 < nf ; i1++)
{
calc_dist(isize, xx[i1], d1);
for (i2 = i1+1; (i2 < nf); i2++)
set_mat_entry(rms, i1, i2, rms_dist(isize, d1, d2));
}
nrms -= (nf-i1-1);
- fprintf(stderr, "\r# RMSD calculations left: %d ", nrms);
+ fprintf(stderr, "\r# RMSD calculations left: "gmx_large_int_pfmt" ", nrms);
+ }
+ /* Clean up work arrays */
+ for (i = 0; (i < isize); i++)
+ {
+ sfree(d1[i]);
+ sfree(d2[i]);
}
+ sfree(d1);
+ sfree(d2);
}
fprintf(stderr, "\n\n");
}
"files should also be merged. Consecutively, create a [TT].tpr[tt] file (input for [TT]g_membed[tt]) from these files,"
"with the following options included in the [TT].mdp[tt] file.[BR]",
" - [TT]integrator = md[tt][BR]",
- " - [TT]energygrp = Protein[tt] (or other group that you want to insert)[BR]",
+ " - [TT]energygrps = Protein[tt] (or other group that you want to insert)[BR]",
" - [TT]freezegrps = Protein[tt][BR]",
" - [TT]freezedim = Y Y Y[tt][BR]",
" - [TT]energygrp_excl = Protein Protein[tt][BR]",
"Cartesian coordinate. To do that, simply omit the preliminary [TT]trjconv[tt] steps. \n",
"USAGE: \n",
"1. Use [TT]make_ndx[tt] to create a group containing the atoms around which you want the SDF \n",
- "2. [TT]trjconv -s a.tpr -f a.xtc -o b.xtc -center tric -ur compact -pbc none[tt] \n",
+ "2. [TT]trjconv -s a.tpr -f a.xtc -o b.xtc -boxcenter tric -ur compact -pbc none[tt] \n",
"3. [TT]trjconv -s a.tpr -f b.xtc -o c.xtc -fit rot+trans[tt] \n",
"4. run [TT]g_spatial[tt] on the [TT].xtc[tt] output of step #3. \n",
"5. Load [TT]grid.cube[tt] into VMD and view as an isosurface. \n",
for (i = 0; i < isize; i++)
{
d2 = distance2(sx[f][i], sx[ff][i]);
- bin = (int)(sqrt(d2)*invbin);
+ bin = (int)(sqrt(d2)*invbin + 0.5);
if (bin >= nalloc)
{
nallocn = 10*(bin/10) + 11;
gmx_mm_pr rijx_S, rijy_S, rijz_S;
gmx_mm_pr rkjx_S, rkjy_S, rkjz_S;
gmx_mm_pr one_S;
+ gmx_mm_pr min_one_plus_eps_S;
gmx_mm_pr rij_rkj_S;
gmx_mm_pr nrij2_S, nrij_1_S;
gmx_mm_pr nrkj2_S, nrkj_1_S;
- gmx_mm_pr cos_S, sin_S;
+ gmx_mm_pr cos_S, invsin_S;
gmx_mm_pr theta_S;
gmx_mm_pr st_S, sth_S;
gmx_mm_pr cik_S, cii_S, ckk_S;
one_S = gmx_set1_pr(1.0);
+ /* The smallest number > -1 */
+ min_one_plus_eps_S = gmx_set1_pr(-1.0 + 2*GMX_REAL_EPS);
+
/* nbonds is the number of angles times nfa1, here we step UNROLL angles */
for (i = 0; (i < nbonds); i += UNROLL*nfa1)
{
cos_S = gmx_mul_pr(rij_rkj_S, gmx_mul_pr(nrij_1_S, nrkj_1_S));
+ /* To allow for 180 degrees, we take the max of cos and -1 + 1bit,
+ * so we can safely get the 1/sin from 1/sqrt(1 - cos^2).
+ * This also ensures that rounding errors would cause the argument
+ * of gmx_acos_pr to be < -1.
+ * Note that we do not take precautions for cos(0)=1, so the outer
+ * atoms in an angle should not be on top of each other.
+ */
+ cos_S = gmx_max_pr(cos_S, min_one_plus_eps_S);
+
theta_S = gmx_acos_pr(cos_S);
- sin_S = gmx_invsqrt_pr(gmx_max_pr(gmx_sub_pr(one_S, gmx_mul_pr(cos_S, cos_S)),
- gmx_setzero_pr()));
+ invsin_S = gmx_invsqrt_pr(gmx_sub_pr(one_S, gmx_mul_pr(cos_S, cos_S)));
+
st_S = gmx_mul_pr(gmx_mul_pr(k_S, gmx_sub_pr(theta0_S, theta_S)),
- sin_S);
+ invsin_S);
sth_S = gmx_mul_pr(st_S, cos_S);
cik_S = gmx_mul_pr(st_S, gmx_mul_pr(nrij_1_S, nrkj_1_S));
gmx_mm_pr ipr_S;
gmx_mm_pr iprm_S, iprn_S;
gmx_mm_pr nrkj2_S, nrkj_1_S, nrkj_2_S, nrkj_S;
+ gmx_mm_pr toler_S;
gmx_mm_pr p_S, q_S;
- gmx_mm_pr fmin_S = gmx_set1_pr(GMX_FLOAT_MIN);
+ gmx_mm_pr nrkj2_min_S;
+ gmx_mm_pr real_eps_S;
+
+ /* Used to avoid division by zero.
+ * We take into acount that we multiply the result by real_eps_S.
+ */
+ nrkj2_min_S = gmx_set1_pr(GMX_REAL_MIN/(2*GMX_REAL_EPS));
+
+ /* The value of the last significant bit (GMX_REAL_EPS is half of that) */
+ real_eps_S = gmx_set1_pr(2*GMX_REAL_EPS);
for (s = 0; s < UNROLL; s++)
{
/* Avoid division by zero. When zero, the result is multiplied by 0
* anyhow, so the 3 max below do not affect the final result.
*/
- nrkj2_S = gmx_max_pr(nrkj2_S, fmin_S);
+ nrkj2_S = gmx_max_pr(nrkj2_S, nrkj2_min_S);
nrkj_1_S = gmx_invsqrt_pr(nrkj2_S);
nrkj_2_S = gmx_mul_pr(nrkj_1_S, nrkj_1_S);
nrkj_S = gmx_mul_pr(nrkj2_S, nrkj_1_S);
- iprm_S = gmx_max_pr(iprm_S, fmin_S);
- iprn_S = gmx_max_pr(iprn_S, fmin_S);
+ toler_S = gmx_mul_pr(nrkj2_S, real_eps_S);
+
+ /* Here the plain-C code uses a conditional, but we can't do that in SIMD.
+ * So we take a max with the tolerance instead. Since we multiply with
+ * m or n later, the max does not affect the results.
+ */
+ iprm_S = gmx_max_pr(iprm_S, toler_S);
+ iprn_S = gmx_max_pr(iprn_S, toler_S);
*nrkj_m2_S = gmx_mul_pr(nrkj_S, gmx_inv_pr(iprm_S));
*nrkj_n2_S = gmx_mul_pr(nrkj_S, gmx_inv_pr(iprn_S));
/* free energy history variable names */
const char *edfh_names[edfhNR] =
{
- "bEquilibrated", "N_at_state", "Wang-Landau_Histogram", "Wang-Landau-delta", "Weights", "Free Energies", "minvar", "variance",
+ "bEquilibrated", "N_at_state", "Wang-Landau Histogram", "Wang-Landau Delta", "Weights", "Free Energies", "minvar", "variance",
"accumulated_plus", "accumulated_minus", "accumulated_plus_2", "accumulated_minus_2", "Tij", "Tij_empirical"
};
return 0;
}
-static int do_cpt_state(XDR *xd, int fflags, t_state *state,
+static int do_cpt_state(XDR *xd, gmx_bool bRead,
+ int fflags, t_state *state,
gmx_bool bReadRNG, FILE *list)
{
int sflags;
rng_p = NULL;
rngi_p = NULL;
}
+
+ if (bRead) /* we need to allocate space for dfhist if we are reading */
+ {
+ init_df_history(&state->dfhist,state->dfhist.nlambda);
+ }
+
/* We want the MC_RNG the same across all the notes for now -- lambda MC is global */
sflags = state->flags;
sfree(bhost);
sfree(fprog);
- if ((do_cpt_state(gmx_fio_getxdr(fp), state->flags, state, TRUE, NULL) < 0) ||
+ if ((do_cpt_state(gmx_fio_getxdr(fp), FALSE, state->flags, state, TRUE, NULL) < 0) ||
(do_cpt_ekinstate(gmx_fio_getxdr(fp), flags_eks, &state->ekinstate, NULL) < 0) ||
(do_cpt_enerhist(gmx_fio_getxdr(fp), FALSE, flags_enh, &state->enerhist, NULL) < 0) ||
(do_cpt_df_hist(gmx_fio_getxdr(fp), flags_dfh, &state->dfhist, NULL) < 0) ||
cr, bPartDecomp, nppnodes_f, npmenodes_f, dd_nc, dd_nc_f);
}
}
- ret = do_cpt_state(gmx_fio_getxdr(fp), fflags, state, *bReadRNG, NULL);
+ ret = do_cpt_state(gmx_fio_getxdr(fp), TRUE, fflags, state, *bReadRNG, NULL);
*init_fep_state = state->fep_state; /* there should be a better way to do this than setting it here.
Investigate for 5.0. */
if (ret)
&(state->dfhist.nlambda), &state->flags, &flags_eks, &flags_enh, &flags_dfh,
&state->edsamstate.nED, NULL);
ret =
- do_cpt_state(gmx_fio_getxdr(fp), state->flags, state, bReadRNG, NULL);
+ do_cpt_state(gmx_fio_getxdr(fp), TRUE, state->flags, state, bReadRNG, NULL);
if (ret)
{
cp_error();
&state.natoms, &state.ngtc, &state.nnhpres, &state.nhchainlength,
&(state.dfhist.nlambda), &state.flags,
&flags_eks, &flags_enh, &flags_dfh, &state.edsamstate.nED, out);
- ret = do_cpt_state(gmx_fio_getxdr(fp), state.flags, &state, TRUE, out);
+ ret = do_cpt_state(gmx_fio_getxdr(fp), TRUE, state.flags, &state, TRUE, out);
if (ret)
{
cp_error();
if (ret == 0)
{
- init_df_history(&state.dfhist, state.dfhist.nlambda, 0); /* reinitialize state with correct sizes */
- ret = do_cpt_df_hist(gmx_fio_getxdr(fp), flags_dfh, &state.dfhist, out);
+ ret = do_cpt_df_hist(gmx_fio_getxdr(fp),
+ flags_dfh, &state.dfhist, out);
}
if (ret == 0)
/* For convenience, and to enable configure-time invocation, we keep all architectures
* in a single file, but to avoid repeated ifdefs we set the overall architecture here.
*/
-#if defined (__i386__) || defined (__x86_64__) || defined (_M_IX86) || defined (_M_X64)
+#ifdef GMX_IS_X86
/* OK, it is x86, but can we execute cpuid? */
#if defined(GMX_X86_GCC_INLINE_ASM) || ( defined(_MSC_VER) && ( (_MSC_VER > 1500) || (_MSC_VER==1500 & _MSC_FULL_VER >= 150030729)))
# define GMX_CPUID_X86
gmx_hw_opt_t *hw_opt,
const gmx_hw_info_t *hwinfo)
{
- int nth_affinity_set, thread_id_node, thread_id,
+ int nth_affinity_set, thread0_id_node,
nthread_local, nthread_node, nthread_hw_max, nphyscore;
int offset;
const int *locality_order;
}
/* map the current process to cores */
- thread_id_node = 0;
- nthread_node = nthread_local;
+ thread0_id_node = 0;
+ nthread_node = nthread_local;
#ifdef GMX_MPI
if (PAR(cr) || MULTISIM(cr))
{
MPI_Comm_split(MPI_COMM_WORLD, gmx_hostname_num(), cr->rank_intranode,
&comm_intra);
- MPI_Scan(&nthread_local, &thread_id_node, 1, MPI_INT, MPI_SUM, comm_intra);
+ MPI_Scan(&nthread_local, &thread0_id_node, 1, MPI_INT, MPI_SUM, comm_intra);
/* MPI_Scan is inclusive, but here we need exclusive */
- thread_id_node -= nthread_local;
+ thread0_id_node -= nthread_local;
/* Get the total number of threads on this physical node */
MPI_Allreduce(&nthread_local, &nthread_node, 1, MPI_INT, MPI_SUM, comm_intra);
MPI_Comm_free(&comm_intra);
* of threads on which we succeeded.
*/
nth_affinity_set = 0;
-#pragma omp parallel firstprivate(thread_id_node) num_threads(nthread_local) \
- reduction(+:nth_affinity_set)
+#pragma omp parallel num_threads(nthread_local) reduction(+:nth_affinity_set)
{
+ int thread_id, thread_id_node;
int index, core;
gmx_bool setaffinity_ret;
- thread_id = gmx_omp_get_thread_num();
- thread_id_node += thread_id;
- index = offset + thread_id_node*hw_opt->core_pinning_stride;
+ thread_id = gmx_omp_get_thread_num();
+ thread_id_node = thread0_id_node + thread_id;
+ index = offset + thread_id_node*hw_opt->core_pinning_stride;
if (locality_order != NULL)
{
core = locality_order[index];
if (debug)
{
- fprintf(debug, "On rank %2d, thread %2d, core %2d the affinity setting returned %d\n",
- cr->nodeid, gmx_omp_get_thread_num(), core, setaffinity_ret);
+ fprintf(debug, "On rank %2d, thread %2d, index %2d, core %2d the affinity setting returned %d\n",
+ cr->nodeid, gmx_omp_get_thread_num(), index, core, setaffinity_ret);
}
}
#endif
}
+#if defined GMX_LIB_MPI && defined GMX_IS_BGQ
+#include <spi/include/kernel/location.h>
+#endif
int gmx_hostname_num()
{
char mpi_hostname[MPI_MAX_PROCESSOR_NAME], hostnum_str[MPI_MAX_PROCESSOR_NAME];
MPI_Get_processor_name(mpi_hostname, &resultlen);
+#ifdef GMX_IS_BGQ
+ Personality_t personality;
+ Kernel_GetPersonality(&personality, sizeof(personality));
+ /* Each MPI rank has a unique coordinate in a 6-dimensional space
+ (A,B,C,D,E,T), with dimensions A-E corresponding to different
+ physical nodes, and T within each node. Each node has sixteen
+ physical cores, each of which can have up to four hardware
+ threads, so 0 <= T <= 63 (but the maximum value of T depends on
+ the confituration of ranks and OpenMP threads per
+ node). However, T is irrelevant for computing a suitable return
+ value for gmx_hostname_num().
+ */
+ hostnum = personality.Network_Config.Acoord;
+ hostnum *= personality.Network_Config.Bnodes;
+ hostnum += personality.Network_Config.Bcoord;
+ hostnum *= personality.Network_Config.Cnodes;
+ hostnum += personality.Network_Config.Ccoord;
+ hostnum *= personality.Network_Config.Dnodes;
+ hostnum += personality.Network_Config.Dcoord;
+ hostnum *= personality.Network_Config.Enodes;
+ hostnum += personality.Network_Config.Ecoord;
+#else
/* This procedure can only differentiate nodes with host names
* that end on unique numbers.
*/
/* Use only the last 9 decimals, so we don't overflow an int */
hostnum = strtol(hostnum_str + max(0, j-9), NULL, 10);
}
+#endif
if (debug)
{
- fprintf(debug, "In gmx_setup_nodecomm: hostname '%s', hostnum %d\n",
+ fprintf(debug, "In gmx_hostname_num: hostname '%s', hostnum %d\n",
mpi_hostname, hostnum);
+#ifdef GMX_IS_BGQ
+ fprintf(debug,
+ "Torus ID A: %d / %d B: %d / %d C: %d / %d D: %d / %d E: %d / %d\nNode ID T: %d / %d core: %d / %d hardware thread: %d / %d\n",
+ personality.Network_Config.Acoord,
+ personality.Network_Config.Anodes,
+ personality.Network_Config.Bcoord,
+ personality.Network_Config.Bnodes,
+ personality.Network_Config.Ccoord,
+ personality.Network_Config.Cnodes,
+ personality.Network_Config.Dcoord,
+ personality.Network_Config.Dnodes,
+ personality.Network_Config.Ecoord,
+ personality.Network_Config.Enodes,
+ Kernel_ProcessorCoreID(),
+ 16,
+ Kernel_ProcessorID(),
+ 64,
+ Kernel_ProcessorThreadID(),
+ 4);
+#endif
}
return hostnum;
#endif
case GMX_NBKERNEL_ELEC_EWALD:
/* simple cutoff (yes, ewald is done all on direct space for free energy) */
Vcoul[i] = qq[i]*rinvC;
- FscalC[i] = Vcoul[i]*rpinvC;
+ FscalC[i] = Vcoul[i];
break;
case GMX_NBKERNEL_ELEC_REACTIONFIELD:
/* reaction-field */
- Vcoul[i] = qq[i]*(rinvC+krf*rC*rC-crf);
- FscalC[i] = qq[i]*(rinvC*rpinvC-2.0*krf);
+ Vcoul[i] = qq[i]*(rinvC + krf*rC*rC-crf);
+ FscalC[i] = qq[i]*(rinvC - 2.0*krf*rC*rC);
break;
case GMX_NBKERNEL_ELEC_CUBICSPLINETABLE:
VV = Y+epsC*Fp;
FF = Fp+Geps+2.0*Heps2;
Vcoul[i] = qq[i]*VV;
- FscalC[i] = -qq[i]*tabscale*FF*rC*rpinvC;
+ FscalC[i] = -qq[i]*tabscale*FF*rC;
break;
case GMX_NBKERNEL_ELEC_GENERALIZEDBORN:
}
else
{
- Vvdw[i] = Vvdw12*(1.0/12.0)-Vvdw6*(1.0/6.0);
+ Vvdw[i] = Vvdw12*(1.0/12.0) - Vvdw6*(1.0/6.0);
}
- FscalV[i] = (Vvdw12-Vvdw6)*rpinvV;
+ FscalV[i] = Vvdw12 - Vvdw6;
break;
case GMX_NBKERNEL_VDW_BUCKINGHAM:
VV = Y+epsV*Fp;
FF = Fp+Geps+2.0*Heps2;
Vvdw[i] += c6[i]*VV;
- FscalV[i] -= c6[i]*tabscale*FF*rV*rpinvV;
+ FscalV[i] -= c6[i]*tabscale*FF*rV;
/* repulsion */
Y = VFtab[nnn+4];
VV = Y+epsV*Fp;
FF = Fp+Geps+2.0*Heps2;
Vvdw[i] += c12[i]*VV;
- FscalV[i] -= c12[i]*tabscale*FF*rV*rpinvV;
+ FscalV[i] -= c12[i]*tabscale*FF*rV;
break;
case GMX_NBKERNEL_VDW_NONE:
Vvdw[i] = (rV < rvdw) ? Vvdw[i] : 0.0;
}
}
+
+ /* FscalC (and FscalV) now contain: dV/drC * rC
+ * Now we multiply by rC^-p, so it will be: dV/drC * rC^1-p
+ * Further down we first multiply by r^p-2 and then by
+ * the vector r, which in total gives: dV/drC * (r/rC)^1-p
+ */
+ FscalC[i] *= rpinvC;
+ FscalV[i] *= rpinvV;
}
}
zero_history(&state->hist);
zero_ekinstate(&state->ekinstate);
init_energyhistory(&state->enerhist);
- init_df_history(&state->dfhist, nlambda, 0);
+ init_df_history(&state->dfhist,nlambda);
state->ddp_count = 0;
state->ddp_count_cg_gl = 0;
state->cg_gl = NULL;
}
}
-extern void init_df_history(df_history_t *dfhist, int nlambda, real wl_delta)
+void init_df_history(df_history_t *dfhist, int nlambda)
{
int i;
- dfhist->bEquil = 0;
dfhist->nlambda = nlambda;
- dfhist->wl_delta = wl_delta;
- snew(dfhist->sum_weights, dfhist->nlambda);
- snew(dfhist->sum_dg, dfhist->nlambda);
- snew(dfhist->sum_minvar, dfhist->nlambda);
- snew(dfhist->sum_variance, dfhist->nlambda);
- snew(dfhist->n_at_lam, dfhist->nlambda);
- snew(dfhist->wl_histo, dfhist->nlambda);
-
- /* allocate transition matrices here */
- snew(dfhist->Tij, dfhist->nlambda);
- snew(dfhist->Tij_empirical, dfhist->nlambda);
-
- for (i = 0; i < dfhist->nlambda; i++)
- {
- snew(dfhist->Tij[i], dfhist->nlambda);
- snew(dfhist->Tij_empirical[i], dfhist->nlambda);
- }
-
- snew(dfhist->accum_p, dfhist->nlambda);
- snew(dfhist->accum_m, dfhist->nlambda);
- snew(dfhist->accum_p2, dfhist->nlambda);
- snew(dfhist->accum_m2, dfhist->nlambda);
+ dfhist->bEquil = 0;
+ dfhist->wl_delta = 0;
- for (i = 0; i < dfhist->nlambda; i++)
+ if (nlambda > 0)
{
- snew((dfhist->accum_p)[i], dfhist->nlambda);
- snew((dfhist->accum_m)[i], dfhist->nlambda);
- snew((dfhist->accum_p2)[i], dfhist->nlambda);
- snew((dfhist->accum_m2)[i], dfhist->nlambda);
+ snew(dfhist->sum_weights, dfhist->nlambda);
+ snew(dfhist->sum_dg, dfhist->nlambda);
+ snew(dfhist->sum_minvar, dfhist->nlambda);
+ snew(dfhist->sum_variance, dfhist->nlambda);
+ snew(dfhist->n_at_lam, dfhist->nlambda);
+ snew(dfhist->wl_histo, dfhist->nlambda);
+
+ /* allocate transition matrices here */
+ snew(dfhist->Tij, dfhist->nlambda);
+ snew(dfhist->Tij_empirical, dfhist->nlambda);
+
+ /* allocate accumulators for various transition matrix
+ free energy methods here */
+ snew(dfhist->accum_p, dfhist->nlambda);
+ snew(dfhist->accum_m, dfhist->nlambda);
+ snew(dfhist->accum_p2, dfhist->nlambda);
+ snew(dfhist->accum_m2, dfhist->nlambda);
+
+ for (i = 0; i < dfhist->nlambda; i++)
+ {
+ snew(dfhist->Tij[i], dfhist->nlambda);
+ snew(dfhist->Tij_empirical[i], dfhist->nlambda);
+ snew((dfhist->accum_p)[i], dfhist->nlambda);
+ snew((dfhist->accum_m)[i], dfhist->nlambda);
+ snew((dfhist->accum_p2)[i], dfhist->nlambda);
+ snew((dfhist->accum_m2)[i], dfhist->nlambda);
+ }
}
}
{
int i, j;
- init_df_history(df_dest, df_source->nlambda, df_source->wl_delta);
+ /* Currently, there should not be any difference in nlambda between the two,
+ but this is included for completeness for potential later functionality */
df_dest->nlambda = df_source->nlambda;
df_dest->bEquil = df_source->bEquil;
+ df_dest->wl_delta = df_source->wl_delta;
+
for (i = 0; i < df_dest->nlambda; i++)
{
df_dest->sum_weights[i] = df_source->sum_weights[i];
df_dest->sum_variance[i] = df_source->sum_variance[i];
df_dest->n_at_lam[i] = df_source->n_at_lam[i];
df_dest->wl_histo[i] = df_source->wl_histo[i];
- df_dest->accum_p[i] = df_source->accum_p[i];
- df_dest->accum_m[i] = df_source->accum_m[i];
- df_dest->accum_p2[i] = df_source->accum_p2[i];
- df_dest->accum_m2[i] = df_source->accum_m2[i];
}
for (i = 0; i < df_dest->nlambda; i++)
{
for (j = 0; j < df_dest->nlambda; j++)
{
+ df_dest->accum_p[i][j] = df_source->accum_p[i][j];
+ df_dest->accum_m[i][j] = df_source->accum_m[i][j];
+ df_dest->accum_p2[i][j] = df_source->accum_p2[i][j];
+ df_dest->accum_m2[i][j] = df_source->accum_m2[i][j];
df_dest->Tij[i][j] = df_source->Tij[i][j];
df_dest->Tij_empirical[i][j] = df_source->Tij_empirical[i][j];
}
}
}
+
+void done_df_history(df_history_t *dfhist)
+{
+ int i;
+
+ if (dfhist->nlambda > 0)
+ {
+ sfree(dfhist->n_at_lam);
+ sfree(dfhist->wl_histo);
+ sfree(dfhist->sum_weights);
+ sfree(dfhist->sum_dg);
+ sfree(dfhist->sum_minvar);
+ sfree(dfhist->sum_variance);
+
+ for (i=0;i<dfhist->nlambda;i++)
+ {
+ sfree(dfhist->Tij[i]);
+ sfree(dfhist->Tij_empirical[i]);
+ sfree(dfhist->accum_p[i]);
+ sfree(dfhist->accum_m[i]);
+ sfree(dfhist->accum_p2[i]);
+ sfree(dfhist->accum_m2[i]);
+ }
+ }
+ dfhist->bEquil = 0;
+ dfhist->nlambda = 0;
+ dfhist->wl_delta = 0;
+}
sprintf(err_buf, "Can't use postive delta-lambda (%g) with expanded ensemble simulations", fep->delta_lambda);
CHECK(fep->delta_lambda > 0 && (ir->efep == efepEXPANDED));
+ sprintf(err_buf, "Can only use expanded ensemble with md-vv for now; should be supported for other integrators in 5.0");
+ CHECK(!(EI_VV(ir->eI)) && (ir->efep == efepEXPANDED));
+
sprintf(err_buf, "Free-energy not implemented for Ewald");
CHECK(ir->coulombtype == eelEWALD);
parse_n_real(weights, &nweights, &(expand->init_lambda_weights));
if (nweights == 0)
{
- expand->bInit_weights = FALSE;
snew(expand->init_lambda_weights, fep->n_lambda); /* initialize to zero */
}
else if (nweights != fep->n_lambda)
gmx_fatal(FARGS, "Number of weights (%d) is not equal to number of lambda values (%d)",
nweights, fep->n_lambda);
}
- else
- {
- expand->bInit_weights = TRUE;
- }
if ((expand->nstexpanded < 0) && (ir->efep != efepNO))
{
expand->nstexpanded = fep->nstdhdl;
static void convert_pairs_to_pairsQ(t_params *plist,
real fudgeQQ, t_atoms *atoms)
{
- t_param *param;
- int i;
- real v, w;
+ t_param *paramp1,*paramp2,*paramnew;
+ int i,j,p1nr,p2nr,p2newnr;
- /* Copy the pair list to the pairQ list */
- plist[F_LJC14_Q] = plist[F_LJ14];
- /* Empty the pair list */
- plist[F_LJ14].nr = 0;
- plist[F_LJ14].param = NULL;
- param = plist[F_LJC14_Q].param;
- for (i = 0; i < plist[F_LJC14_Q].nr; i++)
+ /* Add the pair list to the pairQ list */
+ p1nr = plist[F_LJ14].nr;
+ p2nr = plist[F_LJC14_Q].nr;
+ p2newnr = p1nr + p2nr;
+ snew(paramnew,p2newnr);
+
+ paramp1 = plist[F_LJ14].param;
+ paramp2 = plist[F_LJC14_Q].param;
+
+ /* Fill in the new F_LJC14_Q array with the old one. NOTE:
+ it may be possible to just ADD the converted F_LJ14 array
+ to the old F_LJC14_Q array, but since we have to create
+ a new sized memory structure, better just to deep copy it all.
+ */
+
+ for (i = 0; i < p2nr; i++)
+ {
+ /* Copy over parameters */
+ for (j=0;j<5;j++) /* entries are 0-4 for F_LJC14_Q */
+ {
+ paramnew[i].c[j] = paramp2[i].c[j];
+ }
+
+ /* copy over atoms */
+ for (j=0;j<2;j++)
+ {
+ paramnew[i].a[j] = paramp2[i].a[j];
+ }
+ }
+
+ for (i = p2nr; i < p2newnr; i++)
{
- v = param[i].c[0];
- w = param[i].c[1];
- param[i].c[0] = fudgeQQ;
- param[i].c[1] = atoms->atom[param[i].a[0]].q;
- param[i].c[2] = atoms->atom[param[i].a[1]].q;
- param[i].c[3] = v;
- param[i].c[4] = w;
+ j = i-p2nr;
+
+ /* Copy over parameters */
+ paramnew[i].c[0] = fudgeQQ;
+ paramnew[i].c[1] = atoms->atom[paramp1[j].a[0]].q;
+ paramnew[i].c[2] = atoms->atom[paramp1[j].a[1]].q;
+ paramnew[i].c[3] = paramp1[j].c[0];
+ paramnew[i].c[4] = paramp1[j].c[1];
+
+ /* copy over atoms */
+ paramnew[i].a[0] = paramp1[j].a[0];
+ paramnew[i].a[1] = paramp1[j].a[1];
}
+
+ /* free the old pairlists */
+ sfree(plist[F_LJC14_Q].param);
+ sfree(plist[F_LJ14].param);
+
+ /* now assign the new data to the F_LJC14_Q structure */
+ plist[F_LJC14_Q].param = paramnew;
+ plist[F_LJC14_Q].nr = p2newnr;
+
+ /* Empty the LJ14 pairlist */
+ plist[F_LJ14].nr = 0;
+ plist[F_LJ14].param = NULL;
}
static void generate_LJCpairsNB(t_molinfo *mol, int nb_funct, t_params *nbp)
--- /dev/null
+/*
+ * This file is part of the GROMACS molecular simulation package.
+ *
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2012, The GROMACS Development Team
+ * Copyright (c) 2012,2013, by the GROMACS development team, led by
+ * David van der Spoel, Berk Hess, Erik Lindahl, and including many
+ * others, as listed in the AUTHORS file in the top-level source
+ * directory and at http://www.gromacs.org.
+ *
+ * GROMACS is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ *
+ * GROMACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with GROMACS; if not, see
+ * http://www.gnu.org/licenses, or write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * If you want to redistribute modifications to GROMACS, please
+ * consider that scientific software is very special. Version
+ * control is crucial - bugs must be traceable. We will be happy to
+ * consider code for inclusion in the official distribution, but
+ * derived work must not be called official GROMACS. Details are found
+ * in the README & COPYING files - if they are missing, get the
+ * official version at http://www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the research papers on the package. Check out http://www.gromacs.org.
+ */
+
+/* The macros in this file are intended to be used for writing
+ * architecture-independent SIMD intrinsics code with a SIMD width of 4.
+ * To support a new architecture, adding macros here should be all
+ * that is needed.
+ *
+ * Note that this file is intended only for SIMD operations that require
+ * a SIMD width of 4. In general gmx_simd_macros.h provides wider hardware
+ * support, more functionality and higher performance, but the SIMD width is
+ * not necessarily equal to 4.
+ */
+
+#ifdef _gmx_simd4_macros_h_
+#error "gmx_simd4_macros.h included twice"
+#else
+#define _gmx_simd4_macros_h_
+
+
+/* The SIMD width here is always 4, since that is the whole point */
+#define GMX_SIMD4_WIDTH 4
+
+
+#if defined GMX_SIMD4_SINGLE || defined GMX_SIMD4_DOUBLE
+/* Precision set before inclusion, honour that request */
+#else
+/* Match precision to the Gromacs real precision */
+#ifdef GMX_DOUBLE
+#define GMX_SIMD4_DOUBLE
+#else
+#define GMX_SIMD4_SINGLE
+#endif
+#endif
+
+#ifdef GMX_SIMD4_DOUBLE
+typedef double gmx_simd4_real;
+#endif
+#ifdef GMX_SIMD4_SINGLE
+typedef float gmx_simd4_real;
+#endif
+
+/* Uncomment the next line, without other SIMD active, for testing plain-C */
+/* #define GMX_SIMD4_REFERENCE_PLAIN_C */
+#ifdef GMX_SIMD4_REFERENCE_PLAIN_C
+/* Plain C SIMD reference implementation, also serves as documentation */
+#define GMX_HAVE_SIMD4_MACROS
+
+/* Include plain-C reference implementation, also serves as documentation */
+#include "gmx_simd4_ref.h"
+
+/* float/double SIMD register type */
+#define gmx_simd4_pr gmx_simd4_ref_pr
+
+/* boolean SIMD register type */
+#define gmx_simd4_pb gmx_simd4_ref_pb
+
+#define gmx_simd4_load_pr gmx_simd4_ref_load_pr
+#define gmx_simd4_load_bb_pr gmx_simd4_ref_load_pr
+#define gmx_simd4_set1_pr gmx_simd4_ref_set1_pr
+#define gmx_simd4_setzero_pr gmx_simd4_ref_setzero_pr
+#define gmx_simd4_store_pr gmx_simd4_ref_store_pr
+
+/* Unaligned load+store are not required,
+ * but they can speed up the PME spread+gather operations.
+ */
+#define GMX_SIMD4_HAVE_UNALIGNED
+#ifdef GMX_SIMD4_HAVE_UNALIGNED
+#define gmx_simd4_loadu_pr gmx_simd4_ref_load_pr
+#define gmx_simd4_storeu_pr gmx_simd4_ref_store_pr
+#endif
+
+#define gmx_simd4_add_pr gmx_simd4_ref_add_pr
+#define gmx_simd4_sub_pr gmx_simd4_ref_sub_pr
+#define gmx_simd4_mul_pr gmx_simd4_ref_mul_pr
+/* For the FMA macros below, aim for c=d in code, so FMA3 uses 1 instruction */
+#define gmx_simd4_madd_pr gmx_simd4_ref_madd_pr
+#define gmx_simd4_nmsub_pr gmx_simd4_ref_nmsub_pr
+
+#define gmx_simd4_dotproduct3 gmx_simd4_ref_dotproduct3
+
+#define gmx_simd4_min_pr gmx_simd4_ref_min_pr
+#define gmx_simd4_max_pr gmx_simd4_ref_max_pr
+
+#define gmx_simd4_blendzero_pr gmx_simd4_ref_blendzero_pr
+
+/* Comparison */
+#define gmx_simd4_cmplt_pr gmx_simd4_ref_cmplt_pr
+
+/* Logical operations on SIMD booleans */
+#define gmx_simd4_and_pb gmx_simd4_ref_and_pb
+#define gmx_simd4_or_pb gmx_simd4_ref_or_pb
+
+/* Returns a single int (0/1) which tells if any of the 4 booleans is True */
+#define gmx_simd4_anytrue_pb gmx_simd4_ref_anytrue_pb
+
+#endif /* GMX_SIMD4_REFERENCE_PLAIN_C */
+
+
+/* The same SIMD macros can be translated to SIMD intrinsics (and compiled
+ * to instructions for) different SIMD width and float precision.
+ *
+ * On x86: The gmx_simd4 prefix is replaced by _mm_ or _mm256_ (SSE or AVX).
+ * The _pr suffix is replaced by _ps or _pd (for single or double precision).
+ * Compiler settings will decide if 128-bit intrinsics will
+ * be translated into SSE or AVX instructions.
+ */
+
+
+#ifdef GMX_X86_SSE2
+/* This is for general x86 SIMD instruction sets that also support SSE2 */
+
+#ifdef GMX_SIMD4_SINGLE
+#define GMX_HAVE_SIMD4_MACROS
+#endif
+
+#ifdef GMX_SIMD4_DOUBLE
+/* Note that here we will use 256-bit SIMD with GMX_X86_AVX_128_FMA.
+ * This is inconsistent naming wise, but should give the best performance.
+ */
+#if defined GMX_X86_AVX_128_FMA || defined GMX_X86_AVX_256
+#define GMX_HAVE_SIMD4_MACROS
+#endif
+#endif
+
+#ifdef GMX_HAVE_SIMD4_MACROS
+
+#if defined GMX_X86_AVX_128_FMA || defined GMX_X86_AVX_256
+
+#include <immintrin.h>
+#ifdef HAVE_X86INTRIN_H
+#include <x86intrin.h> /* FMA */
+#endif
+#ifdef HAVE_INTRIN_H
+#include <intrin.h> /* FMA MSVC */
+#endif
+
+#else
+#ifdef GMX_X86_SSE4_1
+#include <smmintrin.h>
+#else
+/* We only have SSE2 */
+#include <emmintrin.h>
+#endif
+#endif
+
+#ifdef GMX_SIMD4_SINGLE
+
+#define gmx_simd4_pr __m128
+
+#define gmx_simd4_pb __m128
+
+#define gmx_simd4_load_pr _mm_load_ps
+#define gmx_simd4_load_bb_pr _mm_load_ps
+#define gmx_simd4_set1_pr _mm_set1_ps
+#define gmx_simd4_setzero_pr _mm_setzero_ps
+#define gmx_simd4_store_pr _mm_store_ps
+
+/* Some old AMD processors could have problems with unaligned loads+stores */
+#ifndef GMX_FAHCORE
+#define GMX_SIMD4_HAVE_UNALIGNED
+#endif
+#ifdef GMX_SIMD4_HAVE_UNALIGNED
+#define gmx_simd4_loadu_pr _mm_loadu_ps
+#define gmx_simd4_storeu_pr _mm_storeu_ps
+#endif
+
+#define gmx_simd4_add_pr _mm_add_ps
+#define gmx_simd4_sub_pr _mm_sub_ps
+#define gmx_simd4_mul_pr _mm_mul_ps
+
+#ifdef GMX_X86_AVX_128_FMA
+#define gmx_simd4_madd_pr(a, b, c) _mm_macc_ps(a, b, c)
+#define gmx_simd4_nmsub_pr(a, b, c) _mm_nmacc_ps(a, b, c)
+#else
+#define gmx_simd4_madd_pr(a, b, c) _mm_add_ps(c, _mm_mul_ps(a, b))
+#define gmx_simd4_nmsub_pr(a, b, c) _mm_sub_ps(c, _mm_mul_ps(a, b))
+#endif
+
+static inline float gmx_simd4_dotproduct3(__m128 a, __m128 b)
+#ifdef GMX_X86_SSE4_1
+{
+ float dp;
+
+ /* SSE4.1 dot product of components 0,1,2, stored in component 0 */
+ _mm_store_ss(&dp, _mm_dp_ps(a, b, 0x71));
+
+ return dp;
+}
+#else
+{
+ float dp_array[7], *dp;
+
+ /* Generate an aligned pointer */
+ dp = (float *)(((size_t)(dp_array+3)) & (~((size_t)15)));
+
+ _mm_store_ps(dp, _mm_mul_ps(a, b));
+
+ return dp[0] + dp[1] + dp[2];
+}
+#endif
+
+#define gmx_simd4_min_pr _mm_min_ps
+#define gmx_simd4_max_pr _mm_max_ps
+
+#define gmx_simd4_blendzero_pr _mm_and_ps
+
+#define gmx_simd4_cmplt_pr _mm_cmplt_ps
+#define gmx_simd4_and_pb _mm_and_ps
+#define gmx_simd4_or_pb _mm_or_ps
+
+#define gmx_simd4_anytrue_pb _mm_movemask_ps
+
+#endif /* GMX_SIMD4_SINGLE */
+
+
+#ifdef GMX_SIMD4_DOUBLE
+
+#define gmx_simd4_pr __m256d
+
+#define gmx_simd4_pb __m256d
+
+#define gmx_simd4_load_pr _mm256_load_pd
+#define gmx_simd4_load_bb_pr _mm256_load_pd
+#define gmx_simd4_set1_pr _mm256_set1_pd
+#define gmx_simd4_setzero_pr _mm256_setzero_pd
+#define gmx_simd4_store_pr _mm256_store_pd
+
+#define GMX_SIMD4_HAVE_UNALIGNED
+#define gmx_simd4_loadu_pr _mm256_loadu_pd
+#define gmx_simd4_storeu_pr _mm256_storeu_pd
+
+#define gmx_simd4_add_pr _mm256_add_pd
+#define gmx_simd4_sub_pr _mm256_sub_pd
+#define gmx_simd4_mul_pr _mm256_mul_pd
+#ifdef GMX_X86_AVX_128_FMA
+#define gmx_simd4_madd_pr(a, b, c) _mm256_macc_pd(a, b, c)
+#define gmx_simd4_nmsub_pr(a, b, c) _mm256_nmacc_pd(a, b, c)
+#else
+#define gmx_simd4_madd_pr(a, b, c) _mm256_add_pd(c, _mm256_mul_pd(a, b))
+#define gmx_simd4_nmsub_pr(a, b, c) _mm256_sub_pd(c, _mm256_mul_pd(a, b))
+#endif
+#define gmx_simd4_min_pr _mm256_min_pd
+#define gmx_simd4_max_pr _mm256_max_pd
+
+#define gmx_simd4_blendzero_pr _mm256_and_pd
+
+/* Less-than (we use ordered, non-signaling, but that's not required) */
+#define gmx_simd4_cmplt_pr(x, y) _mm256_cmp_pd(x, y, 0x11)
+#define gmx_simd4_and_pb _mm256_and_pd
+#define gmx_simd4_or_pb _mm256_or_pd
+
+#define gmx_simd4_anytrue_pb _mm256_movemask_pd
+
+#endif /* GMX_SIMD4_DOUBLE */
+
+
+#endif /* GMX_HAVE_SIMD4_MACROS */
+
+#endif /* GMX_X86_SSE2 */
+
+#ifdef GMX_CPU_ACCELERATION_IBM_QPX
+/* i.e. BlueGene/Q */
+
+/* This hack works on the compilers that can reach this code. A real
+ solution with broader scope will be proposed in master branch. */
+#define gmx_always_inline __attribute__((always_inline))
+
+#ifdef GMX_SIMD4_SINGLE
+#define GMX_HAVE_SIMD4_MACROS
+#endif
+
+typedef vector4double gmx_simd4_pr;
+typedef vector4double gmx_simd4_pb;
+
+/* The declarations of vec_ld* use non-const pointers, and IBM
+ can't/won't fix this any time soon. So GROMACS has to cast away the
+ const-ness of its pointers before loads. Four-wide SIMD loads
+ sometimes occur from variables of type real, and sometimes from
+ variables of type float (even at double precison), so the correct
+ cast cannot be done easily. The correct cast is necessary because
+ the resulting type determines the alignment assumption of vec_ld*,
+ which is different for float and double. So the loads of
+ always-float variables have to be done with a function that does
+ the correct cast. Since functions cannot be overloaded by type in
+ C, they have to have different names. Thus we have
+ gmx_simd4_load_pr and gmx_simd4_load_bb_pr.
+ */
+
+static gmx_inline gmx_simd4_pr gmx_always_inline gmx_simd4_load_pr(const real *a)
+{
+#ifdef NDEBUG
+ return vec_ld(0, (real *) a);
+#else
+ return vec_lda(0, (real *) a);
+#endif
+}
+
+static gmx_inline gmx_simd4_pr gmx_always_inline gmx_simd4_load_bb_pr(const float *a)
+{
+#ifdef NDEBUG
+ return vec_ld(0, (float *) a);
+#else
+ return vec_lda(0, (float *) a);
+#endif
+}
+
+static gmx_inline gmx_simd4_pr gmx_always_inline gmx_simd4_set1_pr(const real a)
+{
+ return vec_splats(a);
+}
+
+static gmx_inline gmx_simd4_pr gmx_always_inline gmx_simd4_setzero_pr()
+{
+ return vec_splats(0.0);
+}
+
+/* TODO this will not yet work, because the function might be passed a
+ pointer to a float when running in double precision.
+ */
+static gmx_inline void gmx_always_inline gmx_simd4_store_pr(real *a, gmx_simd4_pr b)
+{
+#ifdef NDEBUG
+ vec_st(b, 0, a);
+#else
+ vec_sta(b, 0, a);
+#endif
+}
+
+static gmx_inline gmx_simd4_pr gmx_always_inline gmx_simd4_add_pr(gmx_simd4_pr a, gmx_simd4_pr b)
+{
+ return vec_add(a, b);
+}
+
+static gmx_inline gmx_simd4_pr gmx_always_inline gmx_simd4_sub_pr(gmx_simd4_pr a, gmx_simd4_pr b)
+{
+ return vec_sub(a, b);
+}
+
+static gmx_inline gmx_simd4_pr gmx_always_inline gmx_simd4_mul_pr(gmx_simd4_pr a, gmx_simd4_pr b)
+{
+ return vec_mul(a, b);
+}
+
+static gmx_inline gmx_simd4_pr gmx_always_inline gmx_simd4_madd_pr(gmx_simd4_pr a, gmx_simd4_pr b, gmx_simd4_pr c)
+{
+ return vec_madd(a, b, c);
+}
+
+static gmx_inline gmx_simd4_pr gmx_always_inline gmx_simd4_nmsub_pr(gmx_simd4_pr a, gmx_simd4_pr b, gmx_simd4_pr c)
+{
+ return vec_nmsub(a, b, c);
+}
+
+static gmx_inline gmx_simd4_pr gmx_always_inline gmx_simd4_min_pr(gmx_simd4_pr a, gmx_simd4_pr b)
+{
+ /* Implemented the same way as max, but with the subtraction
+ operands swapped. */
+ return vec_sel(b, a, vec_sub(b, a));
+}
+
+static gmx_inline gmx_simd4_pr gmx_always_inline gmx_simd4_max_pr(gmx_simd4_pr a, gmx_simd4_pr b)
+{
+ return vec_sel(b, a, vec_sub(a, b));
+}
+
+static gmx_inline gmx_simd4_pr gmx_always_inline gmx_simd4_blendzero_pr(gmx_simd4_pr a, gmx_simd4_pr b)
+{
+ return vec_sel(gmx_setzero_pr(), a, b);
+}
+
+static gmx_inline gmx_simd4_pb gmx_always_inline gmx_simd4_cmplt_pr(gmx_simd4_pr a, gmx_simd4_pr b)
+{
+ return vec_cmplt(a, b);
+}
+
+static gmx_inline gmx_simd4_pb gmx_always_inline gmx_simd4_and_pb(gmx_simd4_pb a, gmx_simd4_pb b)
+{
+ return vec_and(a, b);
+}
+
+static gmx_inline gmx_simd4_pb gmx_always_inline gmx_simd4_or_pb(gmx_simd4_pb a, gmx_simd4_pb b)
+{
+ return vec_or(a, b);
+}
+
+static gmx_inline float gmx_always_inline gmx_simd4_dotproduct3(gmx_simd4_pr a, gmx_simd4_pr b)
+{
+ /* The dot product is done solely on the QPX AXU (which is the
+ only available FPU). This is awkward, because pretty much no
+ "horizontal" SIMD-vector operations exist, unlike x86 where
+ SSE4.1 added various kinds of horizontal operations. So we have
+ to make do with shifting vector elements and operating on the
+ results. This makes for lots of data dependency, but the main
+ alternative of storing to memory and reloading is not going to
+ help, either. OpenMP over 2 or 4 hardware threads per core will
+ hide much of the latency from the data dependency. The
+ vec_extract() lets the compiler correctly use a floating-point
+ comparison on the zeroth vector element, which avoids needing
+ memory at all.
+ */
+
+ gmx_simd4_pr dp_shifted_left_0 = vec_mul(a, b);
+ gmx_simd4_pr dp_shifted_left_1 = vec_sldw(dp_shifted_left_0, dp_shifted_left_0, 1);
+ gmx_simd4_pr dp_shifted_left_2 = vec_sldw(dp_shifted_left_0, dp_shifted_left_0, 2);
+ gmx_simd4_pr dp = vec_add(dp_shifted_left_2,
+ vec_add(dp_shifted_left_0, dp_shifted_left_1));
+
+ /* See comment in nbnxn_make_pairlist_part() about how this should
+ be able to return a double on PowerPC. */
+ return (float) vec_extract(dp, 0);
+}
+
+static gmx_inline int gmx_always_inline gmx_simd4_anytrue_pb(gmx_simd4_pb a)
+{
+ return gmx_anytrue_pb(a);
+}
+
+#undef gmx_always_inline
+
+#endif /* GMX_CPU_ACCELERATION_IBM_QPX */
+
+#ifdef GMX_HAVE_SIMD4_MACROS
+/* Generic functions to extract a SIMD4 aligned pointer from a pointer x.
+ * x should have at least GMX_SIMD4_WIDTH=4 elements extra compared
+ * to how many you want to use, to avoid indexing outside the aligned region.
+ */
+
+static gmx_inline gmx_simd4_real *
+gmx_simd4_align_real(const gmx_simd4_real *x)
+{
+ return (gmx_simd4_real *)(((size_t)((x)+GMX_SIMD4_WIDTH)) & (~((size_t)(GMX_SIMD4_WIDTH*sizeof(gmx_simd4_real)-1))));
+}
+#endif
+
+
+#endif /* _gmx_simd4_macros_h_ */
--- /dev/null
+/*
+ * This file is part of the GROMACS molecular simulation package.
+ *
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2012, The GROMACS Development Team
+ * Copyright (c) 2012,2013, by the GROMACS development team, led by
+ * David van der Spoel, Berk Hess, Erik Lindahl, and including many
+ * others, as listed in the AUTHORS file in the top-level source
+ * directory and at http://www.gromacs.org.
+ *
+ * GROMACS is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ *
+ * GROMACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with GROMACS; if not, see
+ * http://www.gnu.org/licenses, or write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * If you want to redistribute modifications to GROMACS, please
+ * consider that scientific software is very special. Version
+ * control is crucial - bugs must be traceable. We will be happy to
+ * consider code for inclusion in the official distribution, but
+ * derived work must not be called official GROMACS. Details are found
+ * in the README & COPYING files - if they are missing, get the
+ * official version at http://www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the research papers on the package. Check out http://www.gromacs.org.
+ */
+
+#ifndef _gmx_simd4_ref_h_
+#define _gmx_simd4_ref_h_
+
+/* This file contains a reference plain-C implementation of 4-wide SIMD.
+ * This code is only useful for testing and documentation.
+ * Either float or double precision is supported through gmx_simd4_real,
+ * which is set in gmx_simd4_macros.h
+ */
+
+
+#include <math.h>
+
+/* float/double SIMD register type */
+typedef struct {
+ gmx_simd4_real r[GMX_SIMD4_WIDTH];
+} gmx_simd4_ref_pr;
+
+/* boolean SIMD register type */
+typedef struct {
+ char r[GMX_SIMD4_WIDTH];
+} gmx_simd4_ref_pb;
+
+
+/* Load GMX_SIMD4_WIDTH reals for memory starting at r */
+static gmx_inline gmx_simd4_ref_pr
+gmx_simd4_ref_load_pr(const gmx_simd4_real *r)
+{
+ gmx_simd4_ref_pr a;
+ int i;
+
+ for (i = 0; i < GMX_SIMD4_WIDTH; i++)
+ {
+ a.r[i] = r[i];
+ }
+
+ return a;
+}
+
+/* Set all SIMD register elements to r */
+static gmx_inline gmx_simd4_ref_pr
+gmx_simd4_ref_set1_pr(gmx_simd4_real r)
+{
+ gmx_simd4_ref_pr a;
+ int i;
+
+ for (i = 0; i < GMX_SIMD4_WIDTH; i++)
+ {
+ a.r[i] = r;
+ }
+
+ return a;
+}
+
+/* Set all SIMD register elements to 0 */
+static gmx_inline gmx_simd4_ref_pr
+gmx_simd4_ref_setzero_pr()
+{
+ gmx_simd4_ref_pr a;
+ int i;
+
+ for (i = 0; i < GMX_SIMD4_WIDTH; i++)
+ {
+ a.r[i] = 0.0;
+ }
+
+ return a;
+}
+
+static gmx_inline void
+gmx_simd4_ref_store_pr(gmx_simd4_real *dest, gmx_simd4_ref_pr src)
+{
+ int i;
+
+ for (i = 0; i < GMX_SIMD4_WIDTH; i++)
+ {
+ dest[i] = src.r[i];
+ }
+}
+
+static gmx_inline gmx_simd4_ref_pr
+gmx_simd4_ref_add_pr(gmx_simd4_ref_pr a, gmx_simd4_ref_pr b)
+{
+ gmx_simd4_ref_pr c;
+ int i;
+
+ for (i = 0; i < GMX_SIMD4_WIDTH; i++)
+ {
+ c.r[i] = a.r[i] + b.r[i];
+ }
+
+ return c;
+}
+
+static gmx_inline gmx_simd4_ref_pr
+gmx_simd4_ref_sub_pr(gmx_simd4_ref_pr a, gmx_simd4_ref_pr b)
+{
+ gmx_simd4_ref_pr c;
+ int i;
+
+ for (i = 0; i < GMX_SIMD4_WIDTH; i++)
+ {
+ c.r[i] = a.r[i] - b.r[i];
+ }
+
+ return c;
+}
+
+static gmx_inline gmx_simd4_ref_pr
+gmx_simd4_ref_mul_pr(gmx_simd4_ref_pr a, gmx_simd4_ref_pr b)
+{
+ gmx_simd4_ref_pr c;
+ int i;
+
+ for (i = 0; i < GMX_SIMD4_WIDTH; i++)
+ {
+ c.r[i] = a.r[i]*b.r[i];
+ }
+
+ return c;
+}
+
+static gmx_inline gmx_simd4_ref_pr
+gmx_simd4_ref_madd_pr(gmx_simd4_ref_pr a, gmx_simd4_ref_pr b, gmx_simd4_ref_pr c)
+{
+ gmx_simd4_ref_pr d;
+ int i;
+
+ for (i = 0; i < GMX_SIMD4_WIDTH; i++)
+ {
+ d.r[i] = a.r[i]*b.r[i] + c.r[i];
+ }
+
+ return d;
+}
+
+static gmx_inline gmx_simd4_ref_pr
+gmx_simd4_ref_nmsub_pr(gmx_simd4_ref_pr a, gmx_simd4_ref_pr b, gmx_simd4_ref_pr c)
+{
+ gmx_simd4_ref_pr d;
+ int i;
+
+ for (i = 0; i < GMX_SIMD4_WIDTH; i++)
+ {
+ d.r[i] = -a.r[i]*b.r[i] + c.r[i];
+ }
+
+ return d;
+}
+
+static gmx_inline gmx_simd4_real
+gmx_simd4_ref_dotproduct3(gmx_simd4_ref_pr a, gmx_simd4_ref_pr b)
+{
+ gmx_simd4_real dp;
+ int i;
+
+ dp = 0.0;
+ for (i = 0; i < 3; i++)
+ {
+ dp += a.r[i]*b.r[i];
+ }
+
+ return dp;
+}
+
+static gmx_inline gmx_simd4_ref_pr
+gmx_simd4_ref_min_pr(gmx_simd4_ref_pr a, gmx_simd4_ref_pr b)
+{
+ gmx_simd4_ref_pr c;
+ int i;
+
+ for (i = 0; i < GMX_SIMD4_WIDTH; i++)
+ {
+ c.r[i] = (a.r[i] <= b.r[i] ? a.r[i] : b.r[i]);
+ }
+
+ return c;
+}
+
+static gmx_inline gmx_simd4_ref_pr
+gmx_simd4_ref_max_pr(gmx_simd4_ref_pr a, gmx_simd4_ref_pr b)
+{
+ gmx_simd4_ref_pr c;
+ int i;
+
+ for (i = 0; i < GMX_SIMD4_WIDTH; i++)
+ {
+ c.r[i] = (a.r[i] >= b.r[i] ? a.r[i] : b.r[i]);
+ }
+
+ return c;
+}
+
+static gmx_inline gmx_simd4_ref_pr
+gmx_simd4_ref_blendzero_pr(gmx_simd4_ref_pr a, gmx_simd4_ref_pb b)
+{
+ gmx_simd4_ref_pr c;
+ int i;
+
+ for (i = 0; i < GMX_SIMD4_WIDTH; i++)
+ {
+ c.r[i] = (b.r[i] ? a.r[i] : 0.0);
+ }
+
+ return c;
+}
+
+/* Comparison */
+static gmx_inline gmx_simd4_ref_pb
+gmx_simd4_ref_cmplt_pr(gmx_simd4_ref_pr a, gmx_simd4_ref_pr b)
+{
+ gmx_simd4_ref_pb c;
+ int i;
+
+ for (i = 0; i < GMX_SIMD4_WIDTH; i++)
+ {
+ c.r[i] = (a.r[i] < b.r[i]);
+ }
+
+ return c;
+}
+
+/* Logical AND on SIMD booleans */
+static gmx_inline gmx_simd4_ref_pb
+gmx_simd4_ref_and_pb(gmx_simd4_ref_pb a, gmx_simd4_ref_pb b)
+{
+ gmx_simd4_ref_pb c;
+ int i;
+
+ for (i = 0; i < GMX_SIMD4_WIDTH; i++)
+ {
+ c.r[i] = (a.r[i] && b.r[i]);
+ }
+
+ return c;
+}
+
+/* Logical OR on SIMD booleans */
+static gmx_inline gmx_simd4_ref_pb
+gmx_simd4_ref_or_pb(gmx_simd4_ref_pb a, gmx_simd4_ref_pb b)
+{
+ gmx_simd4_ref_pb c;
+ int i;
+
+ for (i = 0; i < GMX_SIMD4_WIDTH; i++)
+ {
+ c.r[i] = (a.r[i] || b.r[i]);
+ }
+
+ return c;
+}
+
+/* gmx_anytrue_pb(x) returns if any of the boolean is x is True */
+static gmx_inline int
+gmx_simd4_ref_anytrue_pb(gmx_simd4_ref_pb a)
+{
+ int anytrue;
+ int i;
+
+ anytrue = 0;
+ for (i = 0; i < GMX_SIMD4_WIDTH; i++)
+ {
+ if (a.r[i])
+ {
+ anytrue = 1;
+ }
+ }
+
+ return anytrue;
+}
+
+#endif /* _gmx_simd4_ref_h_ */
#define GMX_HAVE_SIMD_MACROS
/* In general the reference SIMD supports any SIMD width, including 1.
- * For the nbnxn 4xn kernels all widths (2, 4 and 8) are supported.
- * The nbnxn 2xnn kernels are currently not supported.
+ * See types/nb_verlet.h for details
*/
#define GMX_SIMD_REF_WIDTH 4
#define gmx_and_pb gmx_simd_ref_and_pb
#define gmx_or_pb gmx_simd_ref_or_pb
-/* Not required, gmx_anytrue_pb(x) returns if any of the boolean is x is True.
- * If this is not present, define GMX_SIMD_IS_TRUE(real x),
- * which should return x==True, where True is True as defined in SIMD.
- */
-#define GMX_SIMD_HAVE_ANYTRUE
-#ifdef GMX_SIMD_HAVE_ANYTRUE
+/* Returns a single int (0/1) which tells if any of the 4 booleans is True */
#define gmx_anytrue_pb gmx_simd_ref_anytrue_pb
-#else
-/* If we don't have gmx_anytrue_pb, we need to store gmx_mm_pb */
-#define gmx_store_pb gmx_simd_ref_store_pb
-#endif
/* Conversions only used for PME table lookup */
#define gmx_cvttpr_epi32 gmx_simd_ref_cvttpr_epi32
#endif
#endif
+#ifdef GMX_IS_X86
#ifdef GMX_X86_SSE2
/* This is for general x86 SIMD instruction sets that also support SSE2 */
#include "gmx_x86_avx_256.h"
#ifdef GMX_DOUBLE
#include "gmx_math_x86_avx_256_double.h"
-#else
+#else /* GMX_DOUBLE */
#include "gmx_math_x86_avx_256_single.h"
-#endif
-#else
+#endif /* GMX_DOUBLE */
+#else /* GMX_X86_AVX_256 */
#ifdef GMX_X86_AVX_128_FMA
#include "gmx_x86_avx_128_fma.h"
#ifdef GMX_DOUBLE
#include "gmx_math_x86_avx_128_fma_double.h"
-#else
+#else /* GMX_DOUBLE */
#include "gmx_math_x86_avx_128_fma_single.h"
-#endif
-#else
+#endif /* GMX_DOUBLE */
+#else /* GMX_X86_AVX_128_FMA */
#ifdef GMX_X86_SSE4_1
#include "gmx_x86_sse4_1.h"
#ifdef GMX_DOUBLE
#include "gmx_math_x86_sse4_1_double.h"
-#else
+#else /* GMX_DOUBLE */
#include "gmx_math_x86_sse4_1_single.h"
-#endif
-#else
+#endif /* GMX_DOUBLE */
+#else /* GMX_X86_SSE4_1 */
#ifdef GMX_X86_SSE2
#include "gmx_x86_sse2.h"
#ifdef GMX_DOUBLE
#include "gmx_math_x86_sse2_double.h"
-#else
+#else /* GMX_DOUBLE */
#include "gmx_math_x86_sse2_single.h"
-#endif
-#else
+#endif /* GMX_DOUBLE */
+#else /* GMX_X86_SSE2 */
#error No x86 acceleration defined
-#endif
-#endif
-#endif
-#endif
+#endif /* GMX_X86_SSE2 */
+#endif /* GMX_X86_SSE4_1 */
+#endif /* GMX_X86_AVX_128_FMA */
+#endif /* GMX_X86_AVX_256 */
+
/* exp and trigonometric functions are included above */
#define GMX_SIMD_HAVE_EXP
#define GMX_SIMD_HAVE_TRIGONOMETRIC
return _mm_or_ps(_mm_and_ps(a, sign_mask), b);
};
-static gmx_inline gmx_mm_pr gmx_masknot_add_pr(gmx_mm_pb a, gmx_mm_pr b, gmx_mm_pr c) { return _mm_add_ps(b, _mm_andnot_ps(a, c)); };
+static gmx_inline gmx_mm_pr gmx_masknot_add_pr(gmx_mm_pb a, gmx_mm_pr b, gmx_mm_pr c)
+{
+ return _mm_add_ps(b, _mm_andnot_ps(a, c));
+};
-#define GMX_SIMD_HAVE_ANYTRUE
#define gmx_anytrue_pb _mm_movemask_ps
#define gmx_cvttpr_epi32 _mm_cvttps_epi32
return _mm_or_pd(_mm_and_pd(a, sign_mask), b);
};
-static gmx_inline gmx_mm_pr gmx_masknot_add_pr(gmx_mm_pb a, gmx_mm_pr b, gmx_mm_pr c) { return _mm_add_pd(b, _mm_andnot_pd(a, c)); };
+static gmx_inline gmx_mm_pr gmx_masknot_add_pr(gmx_mm_pb a, gmx_mm_pr b, gmx_mm_pr c)
+{
+ return _mm_add_pd(b, _mm_andnot_pd(a, c));
+};
#define gmx_cmplt_pr _mm_cmplt_pd
#define gmx_and_pb _mm_and_pd
#define gmx_or_pb _mm_or_pd
-#define GMX_SIMD_HAVE_ANYTRUE
#define gmx_anytrue_pb _mm_movemask_pd
#define gmx_cvttpr_epi32 _mm_cvttpd_epi32
return _mm256_or_ps(_mm256_and_ps(a, sign_mask), b);
};
-static gmx_inline gmx_mm_pr gmx_masknot_add_pr(gmx_mm_pb a, gmx_mm_pr b, gmx_mm_pr c) { return _mm256_add_ps(b, _mm256_andnot_ps(a, c)); };
+static gmx_inline gmx_mm_pr gmx_masknot_add_pr(gmx_mm_pb a, gmx_mm_pr b, gmx_mm_pr c)
+{
+ return _mm256_add_ps(b, _mm256_andnot_ps(a, c));
+};
/* Less-than (we use ordered, non-signaling, but that's not required) */
#define gmx_cmplt_pr(x, y) _mm256_cmp_ps(x, y, 0x11)
#define gmx_and_pb _mm256_and_ps
#define gmx_or_pb _mm256_or_ps
-#define GMX_SIMD_HAVE_ANYTRUE
#define gmx_anytrue_pb _mm256_movemask_ps
#define gmx_cvttpr_epi32 _mm256_cvttps_epi32
return _mm256_or_pd(_mm256_and_pd(a, sign_mask), b);
};
-static gmx_inline gmx_mm_pr gmx_masknot_add_pr(gmx_mm_pb a, gmx_mm_pr b, gmx_mm_pr c) { return _mm256_add_pd(b, _mm256_andnot_pd(a, c)); };
+static gmx_inline gmx_mm_pr gmx_masknot_add_pr(gmx_mm_pb a, gmx_mm_pr b, gmx_mm_pr c)
+{
+ return _mm256_add_pd(b, _mm256_andnot_pd(a, c));
+};
/* Less-than (we use ordered, non-signaling, but that's not required) */
#define gmx_cmplt_pr(x, y) _mm256_cmp_pd(x, y, 0x11)
#define gmx_and_pb _mm256_and_pd
#define gmx_or_pb _mm256_or_pd
-#define GMX_SIMD_HAVE_ANYTRUE
#define gmx_anytrue_pb _mm256_movemask_pd
#define gmx_cvttpr_epi32 _mm256_cvttpd_epi32
#endif /* GMX_X86_SSE2 */
+#endif /* GMX_IS_X86 */
+
+#ifdef GMX_CPU_ACCELERATION_IBM_QPX
+
+/* This hack works on the compilers that can reach this code. A real
+ solution with broader scope will be proposed in master branch. */
+#define gmx_always_inline __attribute__((always_inline))
+
+/* This is for the A2 core on BlueGene/Q that supports IBM's QPX
+ vector built-in functions */
+#define GMX_HAVE_SIMD_MACROS
+#ifdef __clang__
+#include <qpxmath.h>
+#else
+#include "mass_simd.h"
+#endif
+
+/* No need to version the code by the precision, because the QPX AXU
+ extends to and truncates from double precision for free. */
+
+#define GMX_SIMD_WIDTH_HERE 4
+typedef vector4double gmx_mm_pr;
+typedef vector4double gmx_mm_pb;
+typedef vector4double gmx_epi32;
+#define GMX_SIMD_EPI32_WIDTH 4
+
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_load_pr(const real *a)
+{
+#ifdef NDEBUG
+ return vec_ld(0, (real *) a);
+#else
+ return vec_lda(0, (real *) a);
+#endif
+}
+
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_load1_pr(const real *a)
+{
+ return vec_splats(*a);
+}
+
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_set1_pr(real a)
+{
+ return vec_splats(a);
+}
+
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_setzero_pr()
+{
+ return vec_splats(0.0);
+}
+
+static gmx_inline void gmx_always_inline gmx_store_pr(real *a, gmx_mm_pr b)
+{
+#ifdef NDEBUG
+ vec_st(b, 0, a);
+#else
+ vec_sta(b, 0, a);
+#endif
+}
+
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_add_pr(gmx_mm_pr a, gmx_mm_pr b)
+{
+ return vec_add(a, b);
+}
+
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_sub_pr(gmx_mm_pr a, gmx_mm_pr b)
+{
+ return vec_sub(a, b);
+}
+
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_mul_pr(gmx_mm_pr a, gmx_mm_pr b)
+{
+ return vec_mul(a, b);
+}
+
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_madd_pr(gmx_mm_pr a, gmx_mm_pr b, gmx_mm_pr c)
+{
+ return vec_madd(a, b, c);
+}
+
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_nmsub_pr(gmx_mm_pr a, gmx_mm_pr b, gmx_mm_pr c)
+{
+ return vec_nmsub(a, b, c);
+}
+
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_max_pr(gmx_mm_pr a, gmx_mm_pr b)
+{
+ return vec_sel(b, a, vec_sub(a, b));
+}
+
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_blendzero_pr(gmx_mm_pr a, gmx_mm_pr b)
+{
+ return vec_sel(gmx_setzero_pr(), a, b);
+}
+
+static gmx_inline gmx_mm_pb gmx_always_inline gmx_cmplt_pr(gmx_mm_pr a, gmx_mm_pr b)
+{
+ return vec_cmplt(a, b);
+}
+
+static gmx_inline gmx_mm_pb gmx_always_inline gmx_and_pb(gmx_mm_pb a, gmx_mm_pb b)
+{
+ return vec_and(a, b);
+}
+
+static gmx_inline gmx_mm_pb gmx_always_inline gmx_or_pb(gmx_mm_pb a, gmx_mm_pb b)
+{
+ return vec_or(a, b);
+}
+
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_round_pr(gmx_mm_pr a)
+{
+ return vec_round(a);
+}
+
+#define GMX_SIMD_HAVE_FLOOR
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_floor_pr(gmx_mm_pr a)
+{
+ return vec_floor(a);
+}
+
+#define GMX_SIMD_HAVE_BLENDV
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_blendv_pr(gmx_mm_pr a, gmx_mm_pr b, gmx_mm_pr c)
+{
+ return vec_sel(b, a, gmx_cmplt_pr(gmx_setzero_pr(), c));
+}
+
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_cpsgn_nonneg_pr(gmx_mm_pr a, gmx_mm_pr b)
+{
+ return vec_cpsgn(a, b);
+};
+
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_masknot_add_pr(gmx_mm_pb a, gmx_mm_pr b, gmx_mm_pr c)
+{
+ return vec_add(b, vec_sel(c, gmx_setzero_pr(), a));
+};
+
+static gmx_inline gmx_bool gmx_always_inline
+GMX_SIMD_IS_TRUE(real x)
+{
+ return x >= 0.0;
+}
+
+static gmx_inline gmx_epi32 gmx_always_inline gmx_cvttpr_epi32(gmx_mm_pr a)
+{
+ return vec_ctiwuz(a);
+}
+/* Don't want this, we have floor */
+/* #define gmx_cvtepi32_pr vec_cvtepi32 */
+
+/* A2 core on BG/Q delivers relative error of 2^-14, whereas Power ISA
+ Architecture only promises 2^-8. So probably no need for
+ Newton-Raphson iterates at single or double. */
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_rsqrt_pr(gmx_mm_pr a)
+{
+ return vec_rsqrte(a);
+}
+
+/* A2 core on BG/Q delivers relative error of 2^-14, whereas Power ISA
+ Architecture only promises 2^-5. So probably no need for
+ Newton-Raphson iterates at single or double. */
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_rcp_pr(gmx_mm_pr a)
+{
+ return vec_re(a);
+}
+
+/* Note that here, and below, we use the built-in SLEEF port when
+ compiling on BlueGene/Q with clang */
+
+#define GMX_SIMD_HAVE_EXP
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_exp_pr(gmx_mm_pr a)
+{
+#ifdef __clang__
+#ifndef GMX_DOUBLE
+ return xexpf(a);
+#else
+ return xexp(a);
+#endif
+#else
+#ifndef GMX_DOUBLE
+ return expf4(a);
+#else
+ return expd4(a);
+#endif
+#endif
+}
+
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_sqrt_pr(gmx_mm_pr a)
+{
+#ifdef NDEBUG
+ return vec_swsqrt_nochk(a);
+#else
+ return vec_swsqrt(a);
+#endif
+}
+
+#define GMX_SIMD_HAVE_TRIGONOMETRIC
+static gmx_inline int gmx_always_inline gmx_sincos_pr(gmx_mm_pr a, gmx_mm_pr *b, gmx_mm_pr *c)
+{
+#ifdef __clang__
+#ifndef GMX_DOUBLE
+ xsincosf(a, b, c);
+#else
+ xsincos(a, b, c);
+#endif
+#else
+#ifndef GMX_DOUBLE
+ sincosf4(a, b, c);
+#else
+ sincosd4(a, b, c);
+#endif
+#endif
+ return 1;
+}
+
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_acos_pr(gmx_mm_pr a)
+{
+#ifdef __clang__
+#ifndef GMX_DOUBLE
+ return xacosf(a);
+#else
+ return xacos(a);
+#endif
+#else
+#ifndef GMX_DOUBLE
+ return acosf4(a);
+#else
+ return acosd4(a);
+#endif
+#endif
+}
+
+/* NB The order of parameters here is correct; the
+ documentation of atan2[df]4 in SIMD MASS is wrong. */
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_atan2_pr(gmx_mm_pr a, gmx_mm_pr b)
+{
+#ifdef __clang__
+#ifndef GMX_DOUBLE
+ return xatan2f(a, b);
+#else
+ return xatan2(a, b);
+#endif
+#else
+#ifndef GMX_DOUBLE
+ return atan2f4(a, b);
+#else
+ return atan2d4(a, b);
+#endif
+#endif
+}
+
+static gmx_inline int gmx_always_inline
+gmx_anytrue_pb(gmx_mm_pb a)
+{
+ /* The "anytrue" is done solely on the QPX AXU (which is the only
+ available FPU). This is awkward, because pretty much no
+ "horizontal" SIMD-vector operations exist, unlike x86 where
+ SSE4.1 added various kinds of horizontal operations. So we have
+ to make do with shifting vector elements and operating on the
+ results. This makes for lots of data dependency, but the main
+ alternative of storing to memory and reloading is not going to
+ help, either. OpenMP over 2 or 4 hardware threads per core will
+ hide much of the latency from the data dependency. The
+ vec_extract() lets the compiler correctly use a floating-point
+ comparison on the zeroth vector element, which avoids needing
+ memory at all.
+ */
+ gmx_mm_pb vec_shifted_left_0 = a;
+ gmx_mm_pb vec_shifted_left_1 = vec_sldw(a, a, 1);
+ gmx_mm_pb vec_shifted_left_2 = vec_sldw(a, a, 2);
+ gmx_mm_pb vec_shifted_left_3 = vec_sldw(a, a, 3);
+
+ gmx_mm_pb vec_return = vec_or(vec_or(vec_shifted_left_2, vec_shifted_left_3),
+ vec_or(vec_shifted_left_0, vec_shifted_left_1));
+ return (0.0 < vec_extract(vec_return, 0));
+};
+
+#undef gmx_always_inline
+
+#endif /* GMX_CPU_ACCELERATION_IBM_QPX */
#ifdef GMX_HAVE_SIMD_MACROS
/* Generic functions to extract a SIMD aligned pointer from a pointer x.
#include "gmx_simd_math_single.h"
#endif
+
#endif /* GMX_HAVE_SIMD_MACROS */
#endif /* _gmx_simd_macros_h_ */
return c;
}
-/* Logical AND on SIMD booleans */
-static gmx_inline gmx_simd_ref_pb
+/* Logical AND on SIMD booleans. Can't be static or it can't be a
+ template parameter (at least on XLC for BlueGene/Q) */
+gmx_inline gmx_simd_ref_pb
gmx_simd_ref_and_pb(gmx_simd_ref_pb a, gmx_simd_ref_pb b)
{
gmx_simd_ref_pb c;
return c;
}
-/* Logical OR on SIMD booleans */
-static gmx_inline gmx_simd_ref_pb
+/* Logical OR on SIMD booleans. Can't be static or it can't be a
+ template parameter (at least on XLC for BlueGene/Q) */
+gmx_inline gmx_simd_ref_pb
gmx_simd_ref_or_pb(gmx_simd_ref_pb a, gmx_simd_ref_pb b)
{
gmx_simd_ref_pb c;
return c;
}
-/* Not required, gmx_anytrue_pb(x) returns if any of the boolean is x is True.
- * If this is not present, define GMX_SIMD_IS_TRUE(real x),
- * which should return x==True, where True is True as defined in SIMD.
- */
+/* Returns a single int (0/1) which tells if any of the booleans is True */
static gmx_inline int
gmx_simd_ref_anytrue_pb(gmx_simd_ref_pb a)
{
return anytrue;
}
-/* If we don't have gmx_anytrue_pb, we need to store gmx_mm_pb */
-static gmx_inline void
-gmx_simd_ref_store_pb(real *dest, gmx_simd_ref_pb src)
-{
- int i;
-
- for (i = 0; i < GMX_SIMD_REF_WIDTH; i++)
- {
- dest[i] = src.r[i];
- }
-};
-
/* Conversions only used for PME table lookup */
static gmx_inline gmx_simd_ref_epi32
gmx_simd_ref_cvttpr_epi32(gmx_simd_ref_pr a)
void init_npt_masses(t_inputrec *ir, t_state *state, t_extmass *MassQ, gmx_bool bInit);
+void init_expanded_ensemble(gmx_bool bStateFromCP, t_inputrec *ir, gmx_rng_t *mcrng, df_history_t *dfhist);
+
int ExpandedEnsembleDynamics(FILE *log, t_inputrec *ir, gmx_enerdata_t *enerd,
- t_state *state, t_extmass *MassQ, df_history_t *dfhist,
+ t_state *state, t_extmass *MassQ, int fep_state, df_history_t *dfhist,
gmx_large_int_t step, gmx_rng_t mcrng,
rvec *v, t_mdatoms *mdatoms);
void PrintFreeEnergyInfoToFile(FILE *outfile, t_lambda *fep, t_expanded *expand, t_simtemp *simtemp, df_history_t *dfhist,
- int nlam, int frequency, gmx_large_int_t step);
+ int fep_state, int frequency, gmx_large_int_t step);
void get_mc_state(gmx_rng_t rng, t_state *state);
gmx_mdoutf_t *outf,
t_mdebin *mdebin,
gmx_ekindata_t *ekind,
- df_history_t df_history,
rvec *f,
rvec *f_global,
gmx_wallcycle_t wcycle,
/* return the rank of the node */
int gmx_hostname_num(void);
-/* If the first part of the hostname (up to the first dot) ends with a number, returns this number.
- If the first part of the hostname does not ends in a number (0-9 characters), returns 0.
+/* Ostensibly, returns a integer characteristic of and unique to each
+ physical node in the MPI system. If the first part of the MPI
+ hostname (up to the first dot) ends with a number, returns this
+ number. If the first part of the MPI hostname does not ends in a
+ number (0-9 characters), returns 0.
*/
void gmx_setup_nodecomm(FILE *fplog, t_commrec *cr);
#else
/* older versions of gcc don't support atomic intrinsics */
-
+#ifndef __MIC__
#define tMPI_Atomic_memory_barrier() __asm__ __volatile__("sfence;" : : : "memory")
+#else
+/* MIC is in-order and does not need nor support sfense */
+#define tMPI_Atomic_memory_barrier() __asm__ __volatile__("":::"memory")
+#endif
#define TMPI_ATOMIC_HAVE_NATIVE_FETCH_ADD
static inline int tMPI_Atomic_fetch_add(tMPI_Atomic_t *a, int i)
void done_energyhistory(energyhistory_t * enerhist);
void init_gtc_state(t_state *state, int ngtc, int nnhpres, int nhchainlength);
void init_state(t_state *state, int natoms, int ngtc, int nnhpres, int nhchainlength, int nlambda);
-void init_df_history(df_history_t *dfhist, int nlambda, real wl_delta);
+void init_df_history(df_history_t *dfhist, int nlambda);
+void done_df_history(df_history_t *dfhist);
void copy_df_history(df_history_t * df_dest, df_history_t *df_source);
void copy_blocka(const t_blocka *src, t_blocka *dest);
real wl_ratio; /* ratio between largest and smallest number for freezing the weights */
real init_wl_delta; /* starting delta for wang-landau */
gmx_bool bWLoneovert; /* use one over t convergece for wang-landau when the delta get sufficiently small */
- gmx_bool bInit_weights; /* did we initialize the weights? */
+ gmx_bool bInit_weights; /* did we initialize the weights? TODO: REMOVE FOR 5.0, no longer needed with new logic */
real mc_temp; /* To override the main temperature, or define it if it's not defined */
real *init_lambda_weights; /* user-specified initial weights to start with */
} t_expanded;
/* #define GMX_NBNXN_SIMD_2XNN */
-#ifdef GMX_X86_SSE2
+#if (defined GMX_X86_SSE2) || (defined GMX_CPU_ACCELERATION_IBM_QPX)
/* Use SIMD accelerated nbnxn search and kernels */
#define GMX_NBNXN_SIMD
#ifndef _nbnxn_pairlist_h
#define _nbnxn_pairlist_h
+#ifdef HAVE_CONFIG_H
+# include <config.h>
+#endif
+
#ifdef __cplusplus
extern "C" {
#endif
* is found, all subsequent j-entries in the i-entry also have full masks.
*/
typedef struct {
- int cj; /* The j-cluster */
- unsigned excl; /* The topology exclusion (interaction) bits */
+ int cj; /* The j-cluster */
+ unsigned excl; /* The exclusion (interaction) bits */
+#ifdef GMX_CPU_ACCELERATION_IBM_QPX
+ /* Indices into the arrays of SIMD interaction masks. */
+ char interaction_mask_indices[4];
+#endif
} nbnxn_cj_t;
/* In nbnxn_ci_t the integer shift contains the shift in the lower 7 bits.
*/
unsigned *simd_exclusion_filter1;
unsigned *simd_exclusion_filter2;
-
- int nout; /* The number of force arrays */
- nbnxn_atomdata_output_t *out; /* Output data structures */
- int nalloc; /* Allocation size of all arrays (for x/f *x/fstride) */
- gmx_bool bUseBufferFlags; /* Use the flags or operate on all atoms */
- nbnxn_buffer_flags_t buffer_flags; /* Flags for buffer zeroing+reduc. */
+#ifdef GMX_CPU_ACCELERATION_IBM_QPX
+ real *simd_interaction_array; /* Array of masks needed for exclusions on QPX */
+#endif
+ int nout; /* The number of force arrays */
+ nbnxn_atomdata_output_t *out; /* Output data structures */
+ int nalloc; /* Allocation size of all arrays (for x/f *x/fstride) */
+ gmx_bool bUseBufferFlags; /* Use the flags or operate on all atoms */
+ nbnxn_buffer_flags_t buffer_flags; /* Flags for buffer zeroing+reduc. */
} nbnxn_atomdata_t;
#ifdef __cplusplus
{
int nlambda; /* total number of lambda states - for history*/
- gmx_bool bEquil; /* reached equilibration */
+ gmx_bool bEquil; /* Have we reached equilibration */
int *n_at_lam; /* number of points observed at each lambda */
real *wl_histo; /* histogram for WL flatness determination */
real wl_delta; /* current wang-landau delta */
real **Tij; /* transition matrix */
real **Tij_empirical; /* Empirical transition matrix */
+
} df_history_t;
typedef struct
copy_mat(state_local->fvir_prev, state->fvir_prev);
copy_mat(state_local->pres_prev, state->pres_prev);
-
for (i = 0; i < state_local->ngtc; i++)
{
for (j = 0; j < nh; j++)
}
}
+static void dd_distribute_dfhist(gmx_domdec_t *dd, df_history_t *dfhist)
+{
+ int i;
+ dd_bcast(dd, sizeof(int), &dfhist->bEquil);
+ dd_bcast(dd, sizeof(int), &dfhist->nlambda);
+ dd_bcast(dd, sizeof(real), &dfhist->wl_delta);
+
+ if (dfhist->nlambda > 0)
+ {
+ int nlam = dfhist->nlambda;
+ dd_bcast(dd, sizeof(int)*nlam, dfhist->n_at_lam);
+ dd_bcast(dd, sizeof(real)*nlam, dfhist->wl_histo);
+ dd_bcast(dd, sizeof(real)*nlam, dfhist->sum_weights);
+ dd_bcast(dd, sizeof(real)*nlam, dfhist->sum_dg);
+ dd_bcast(dd, sizeof(real)*nlam, dfhist->sum_minvar);
+ dd_bcast(dd, sizeof(real)*nlam, dfhist->sum_variance);
+
+ for (i = 0; i<nlam; i++) {
+ dd_bcast(dd, sizeof(real)*nlam, dfhist->accum_p[i]);
+ dd_bcast(dd, sizeof(real)*nlam, dfhist->accum_m[i]);
+ dd_bcast(dd, sizeof(real)*nlam, dfhist->accum_p2[i]);
+ dd_bcast(dd, sizeof(real)*nlam, dfhist->accum_m2[i]);
+ dd_bcast(dd, sizeof(real)*nlam, dfhist->Tij[i]);
+ dd_bcast(dd, sizeof(real)*nlam, dfhist->Tij_empirical[i]);
+ }
+ }
+}
+
static void dd_distribute_state(gmx_domdec_t *dd, t_block *cgs,
t_state *state, t_state *state_local,
rvec **f)
copy_mat(state->boxv, state_local->boxv);
copy_mat(state->svir_prev, state_local->svir_prev);
copy_mat(state->fvir_prev, state_local->fvir_prev);
+ copy_df_history(&state_local->dfhist,&state->dfhist);
for (i = 0; i < state_local->ngtc; i++)
{
for (j = 0; j < nh; j++)
dd_bcast(dd, ((state_local->nnhpres*nh)*sizeof(double)), state_local->nhpres_xi);
dd_bcast(dd, ((state_local->nnhpres*nh)*sizeof(double)), state_local->nhpres_vxi);
+ /* communicate df_history -- required for restarting from checkpoint */
+ dd_distribute_dfhist(dd,&state_local->dfhist);
+
if (dd->nat_home > state_local->nalloc)
{
dd_realloc_state(state_local, f, dd->nat_home);
#include "gromacs/utility/gmxmpi.h"
-void GenerateGibbsProbabilities(real *ene, double *p_k, double *pks, int minfep, int maxfep)
+static void init_df_history_weights(df_history_t *dfhist, t_expanded *expand, int nlim)
+{
+ int i;
+ dfhist->wl_delta = expand->init_wl_delta;
+ for (i = 0; i < nlim; i++)
+ {
+ dfhist->sum_weights[i] = expand->init_lambda_weights[i];
+ dfhist->sum_dg[i] = expand->init_lambda_weights[i];
+ }
+}
+
+/* Eventually should contain all the functions needed to initialize expanded ensemble
+ before the md loop starts */
+extern void init_expanded_ensemble(gmx_bool bStateFromCP, t_inputrec *ir, gmx_rng_t *mcrng, df_history_t *dfhist)
+{
+ *mcrng = gmx_rng_init(ir->expandedvals->lmc_seed);
+ if (!bStateFromCP)
+ {
+ init_df_history_weights(dfhist,ir->expandedvals,ir->fepvals->n_lambda);
+ }
+}
+
+static void GenerateGibbsProbabilities(real *ene, double *p_k, double *pks, int minfep, int maxfep)
{
int i;
}
}
-void GenerateWeightedGibbsProbabilities(real *ene, double *p_k, double *pks, int nlim, real *nvals, real delta)
+static void GenerateWeightedGibbsProbabilities(real *ene, double *p_k, double *pks, int nlim, real *nvals, real delta)
{
int i;
{
real maxdiff = 0.000000001;
gmx_bool bSufficientSamples;
- int i, k, n, nz, indexi, indexk, min_n, max_n, nlam, totali;
+ int i, k, n, nz, indexi, indexk, min_n, max_n, totali;
int n0, np1, nm1, nval, min_nvalm, min_nvalp, maxc;
real omega_m1_0, omega_p1_m1, omega_m1_p1, omega_p1_0, clam_osum;
real de, de_function, dr, denom, maxdr;
/* print out the weights to the log, along with current state */
extern void PrintFreeEnergyInfoToFile(FILE *outfile, t_lambda *fep, t_expanded *expand, t_simtemp *simtemp, df_history_t *dfhist,
- int nlam, int frequency, gmx_large_int_t step)
+ int fep_state, int frequency, gmx_large_int_t step)
{
int nlim, i, ifep, jfep;
real dw, dg, dv, dm, Tprint;
{
fprintf(outfile, " %10.5f %10.5f", dfhist->sum_weights[ifep], dw);
}
- if (ifep == nlam)
+ if (ifep == fep_state)
{
fprintf(outfile, " <<\n");
}
}
extern int ExpandedEnsembleDynamics(FILE *log, t_inputrec *ir, gmx_enerdata_t *enerd,
- t_state *state, t_extmass *MassQ, df_history_t *dfhist,
+ t_state *state, t_extmass *MassQ, int fep_state, df_history_t *dfhist,
gmx_large_int_t step, gmx_rng_t mcrng,
rvec *v, t_mdatoms *mdatoms)
+/* Note that the state variable is only needed for simulated tempering, not
+ Hamiltonian expanded ensemble. May be able to remove it after integrator refactoring. */
{
real *pfep_lamee, *scaled_lamee, *weighted_lamee;
double *p_k;
- int i, nlam, nlim, lamnew, totalsamples;
+ int i, nlim, lamnew, totalsamples;
real oneovert, maxscaled = 0, maxweighted = 0;
t_expanded *expand;
t_simtemp *simtemp;
expand = ir->expandedvals;
simtemp = ir->simtempvals;
nlim = ir->fepvals->n_lambda;
- nlam = state->fep_state;
snew(scaled_lamee, nlim);
snew(weighted_lamee, nlim);
snew(pfep_lamee, nlim);
snew(p_k, nlim);
- if (expand->bInit_weights) /* if initialized weights, we need to fill them in */
- {
- dfhist->wl_delta = expand->init_wl_delta; /* MRS -- this would fit better somewhere else? */
- for (i = 0; i < nlim; i++)
- {
- dfhist->sum_weights[i] = expand->init_lambda_weights[i];
- dfhist->sum_dg[i] = expand->init_lambda_weights[i];
- }
- expand->bInit_weights = FALSE;
- }
-
/* update the count at the current lambda*/
- dfhist->n_at_lam[nlam]++;
+ dfhist->n_at_lam[fep_state]++;
/* need to calculate the PV term somewhere, but not needed here? Not until there's a lambda state that's
pressure controlled.*/
{
/* Note -- this assumes no mass changes, since kinetic energy is not added . . . */
scaled_lamee[i] = (enerd->enerpart_lambda[i+1]-enerd->enerpart_lambda[0])/(simtemp->temperatures[i]*BOLTZ)
- + enerd->term[F_EPOT]*(1.0/(simtemp->temperatures[i])- 1.0/(simtemp->temperatures[nlam]))/BOLTZ;
+ + enerd->term[F_EPOT]*(1.0/(simtemp->temperatures[i])- 1.0/(simtemp->temperatures[fep_state]))/BOLTZ;
}
else
{
{
for (i = 0; i < nlim; i++)
{
- scaled_lamee[i] = enerd->term[F_EPOT]*(1.0/simtemp->temperatures[i] - 1.0/simtemp->temperatures[nlam])/BOLTZ;
+ scaled_lamee[i] = enerd->term[F_EPOT]*(1.0/simtemp->temperatures[i] - 1.0/simtemp->temperatures[fep_state])/BOLTZ;
}
}
}
/* update weights - we decide whether or not to actually do this inside */
- bDoneEquilibrating = UpdateWeights(nlim, expand, dfhist, nlam, scaled_lamee, weighted_lamee, step);
+ bDoneEquilibrating = UpdateWeights(nlim, expand, dfhist, fep_state, scaled_lamee, weighted_lamee, step);
if (bDoneEquilibrating)
{
if (log)
}
}
- lamnew = ChooseNewLambda(nlim, expand, dfhist, nlam, weighted_lamee, p_k, mcrng);
+ lamnew = ChooseNewLambda(nlim, expand, dfhist, fep_state, weighted_lamee, p_k, mcrng);
/* if using simulated tempering, we need to adjust the temperatures */
- if (ir->bSimTemp && (lamnew != nlam)) /* only need to change the temperatures if we change the state */
+ if (ir->bSimTemp && (lamnew != fep_state)) /* only need to change the temperatures if we change the state */
{
int i, j, n, d;
real *buf_ngtc;
#endif
}
- /* Analytical Ewald exclusion correction is only an option in the
- * x86 SIMD kernel. This is faster in single precision
- * on Bulldozer and slightly faster on Sandy Bridge.
+ /* Analytical Ewald exclusion correction is only an option in
+ * the SIMD kernel. On BlueGene/Q, this is faster regardless
+ * of precision. In single precision, this is faster on
+ * Bulldozer, and slightly faster on Sandy Bridge.
*/
-#if (defined GMX_X86_AVX_128_FMA || defined GMX_X86_AVX_256) && !defined GMX_DOUBLE
+#if ((defined GMX_X86_AVX_128_FMA || defined GMX_X86_AVX_256) && !defined GMX_DOUBLE) || (defined GMX_CPU_ACCELERATION_IBM_QPX)
*ewald_excl = ewaldexclAnalytical;
#endif
if (getenv("GMX_NBNXN_EWALD_TABLE") != NULL)
}
}
-#endif /* GMX_X86_SSE2 */
+#endif /* GMX_NBNXN_SIMD */
}
#endif
#endif
#endif
-#else /* GMX_X86_SSE2 */
+#else /* GMX_X86_SSE2 */
/* not GMX_X86_SSE2, but other SIMD */
returnvalue = "SIMD";
#endif /* GMX_X86_SSE2 */
-#else /* GMX_NBNXN_SIMD */
+#else /* GMX_NBNXN_SIMD */
returnvalue = "not available";
#endif /* GMX_NBNXN_SIMD */
break;
init_ekinstate(&state->ekinstate, ir);
init_energyhistory(&state->enerhist);
- init_df_history(&state->dfhist, ir->fepvals->n_lambda, ir->expandedvals->init_wl_delta);
+ init_df_history(&state->dfhist, ir->fepvals->n_lambda);
}
{
if (rerun_fr->bLambda)
{
- if (fepvals->delta_lambda != 0)
+ if (fepvals->delta_lambda==0)
{
state_global->lambda[efptFEP] = rerun_fr->lambda;
for (i = 0; i < efptNR; i++)
}
}
}
+ else
+ {
+ if (state->fep_state > 0) {
+ state_global->fep_state = state->fep_state; /* state->fep is the one updated by bExpanded */
+ for (i = 0; i < efptNR; i++)
+ {
+ state_global->lambda[i] = fepvals->all_lambda[i][state_global->fep_state];
+ }
+ }
+ }
}
for (i = 0; i < efptNR; i++)
{
#include "nbnxn_consts.h"
#include "nbnxn_internal.h"
#include "nbnxn_search.h"
-#include "nbnxn_atomdata.h"
#include "gmx_omp_nthreads.h"
/* Default nbnxn allocation routine, allocates NBNXN_MEM_ALIGN byte aligned */
}
}
+#ifdef GMX_NBNXN_SIMD
+static void
+nbnxn_atomdata_init_simple_exclusion_masks(nbnxn_atomdata_t *nbat)
+{
+ int i, j;
+ const int simd_width = GMX_SIMD_WIDTH_HERE;
+ int simd_excl_size;
+ /* Set the diagonal cluster pair exclusion mask setup data.
+ * In the kernel we check 0 < j - i to generate the masks.
+ * Here we store j - i for generating the mask for the first i,
+ * we substract 0.5 to avoid rounding issues.
+ * In the kernel we can subtract 1 to generate the subsequent mask.
+ */
+ int simd_4xn_diag_size;
+ const real simdFalse = -1, simdTrue = 1;
+ real *simd_interaction_array;
+
+ simd_4xn_diag_size = max(NBNXN_CPU_CLUSTER_I_SIZE, simd_width);
+ snew_aligned(nbat->simd_4xn_diagonal_j_minus_i, simd_4xn_diag_size, NBNXN_MEM_ALIGN);
+ for (j = 0; j < simd_4xn_diag_size; j++)
+ {
+ nbat->simd_4xn_diagonal_j_minus_i[j] = j - 0.5;
+ }
+
+ snew_aligned(nbat->simd_2xnn_diagonal_j_minus_i, simd_width, NBNXN_MEM_ALIGN);
+ for (j = 0; j < simd_width/2; j++)
+ {
+ /* The j-cluster size is half the SIMD width */
+ nbat->simd_2xnn_diagonal_j_minus_i[j] = j - 0.5;
+ /* The next half of the SIMD width is for i + 1 */
+ nbat->simd_2xnn_diagonal_j_minus_i[simd_width/2+j] = j - 1 - 0.5;
+ }
+
+ /* We use up to 32 bits for exclusion masking.
+ * The same masks are used for the 4xN and 2x(N+N) kernels.
+ * The masks are read either into epi32 SIMD registers or into
+ * real SIMD registers (together with a cast).
+ * In single precision this means the real and epi32 SIMD registers
+ * are of equal size.
+ * In double precision the epi32 registers can be smaller than
+ * the real registers, so depending on the architecture, we might
+ * need to use two, identical, 32-bit masks per real.
+ */
+ simd_excl_size = NBNXN_CPU_CLUSTER_I_SIZE*simd_width;
+ snew_aligned(nbat->simd_exclusion_filter1, simd_excl_size, NBNXN_MEM_ALIGN);
+ snew_aligned(nbat->simd_exclusion_filter2, simd_excl_size*2, NBNXN_MEM_ALIGN);
+
+ for (j = 0; j < simd_excl_size; j++)
+ {
+ /* Set the consecutive bits for masking pair exclusions */
+ nbat->simd_exclusion_filter1[j] = (1U << j);
+ nbat->simd_exclusion_filter2[j*2 + 0] = (1U << j);
+ nbat->simd_exclusion_filter2[j*2 + 1] = (1U << j);
+ }
+
+#if (defined GMX_CPU_ACCELERATION_IBM_QPX)
+ /* The QPX kernels shouldn't do the bit masking that is done on
+ * x86, because the SIMD units lack bit-wise operations. Instead,
+ * we generate a vector of all 2^4 possible ways an i atom
+ * interacts with its 4 j atoms. Each array entry contains
+ * simd_width signed ints that are read in a single SIMD
+ * load. These ints must contain values that will be interpreted
+ * as true and false when loaded in the SIMD floating-point
+ * registers, ie. any positive or any negative value,
+ * respectively. Each array entry encodes how this i atom will
+ * interact with the 4 j atoms. Matching code exists in
+ * set_ci_top_excls() to generate indices into this array. Those
+ * indices are used in the kernels. */
+
+ simd_excl_size = NBNXN_CPU_CLUSTER_I_SIZE*NBNXN_CPU_CLUSTER_I_SIZE;
+ const int qpx_simd_width = GMX_SIMD_WIDTH_HERE;
+ snew_aligned(simd_interaction_array, simd_excl_size * qpx_simd_width, NBNXN_MEM_ALIGN);
+ for (j = 0; j < simd_excl_size; j++)
+ {
+ int index = j * qpx_simd_width;
+ for (i = 0; i < qpx_simd_width; i++)
+ {
+ simd_interaction_array[index + i] = (j & (1 << i)) ? simdTrue : simdFalse;
+ }
+ }
+ nbat->simd_interaction_array = simd_interaction_array;
+#endif
+}
+#endif
+
/* Initializes an nbnxn_atomdata_t data structure */
void nbnxn_atomdata_init(FILE *fp,
nbnxn_atomdata_t *nbat,
#ifdef GMX_NBNXN_SIMD
if (simple)
{
- /* Set the diagonal cluster pair interaction mask setup data.
- * In the kernel we check 0 < j - i to generate the masks.
- * Here we store j - i for generating the mask for the first i (i=0);
- * we substract 0.5 to avoid rounding issues.
- * In the kernel we can subtract 1 to generate the mask for the next i.
- */
- const int simd_width = GMX_SIMD_WIDTH_HERE;
- int simd_4xn_diag_ind_size, simd_interaction_size, j;
-
- simd_4xn_diag_ind_size = max(NBNXN_CPU_CLUSTER_I_SIZE, simd_width);
- snew_aligned(nbat->simd_4xn_diagonal_j_minus_i,
- simd_4xn_diag_ind_size, NBNXN_MEM_ALIGN);
- for (j = 0; j < simd_4xn_diag_ind_size; j++)
- {
- nbat->simd_4xn_diagonal_j_minus_i[j] = j - 0.5;
- }
-
- snew_aligned(nbat->simd_2xnn_diagonal_j_minus_i,
- simd_width, NBNXN_MEM_ALIGN);
- for (j = 0; j < simd_width/2; j++)
- {
- /* The j-cluster size is half the SIMD width */
- nbat->simd_2xnn_diagonal_j_minus_i[j] = j - 0.5;
- /* The next half of the SIMD width is for i + 1 */
- nbat->simd_2xnn_diagonal_j_minus_i[simd_width/2+j] = j - 1 - 0.5;
- }
-
- /* We use up to 32 bits for exclusion masking.
- * The same masks are used for the 4xN and 2x(N+N) kernels.
- * The masks are read either into epi32 SIMD registers or into
- * real SIMD registers (together with a cast).
- * In single precision this means the real and epi32 SIMD registers
- * are of equal size.
- * In double precision the epi32 registers can be smaller than
- * the real registers, so depending on the architecture, we might
- * need to use two, identical, 32-bit masks per real.
- */
- simd_interaction_size = NBNXN_CPU_CLUSTER_I_SIZE*simd_width;
- snew_aligned(nbat->simd_exclusion_filter1, simd_interaction_size, NBNXN_MEM_ALIGN);
- snew_aligned(nbat->simd_exclusion_filter2, simd_interaction_size*2, NBNXN_MEM_ALIGN);
-
- for (j = 0; j < simd_interaction_size; j++)
- {
- /* Set the consecutive bits for filters pair exclusions masks */
- nbat->simd_exclusion_filter1[j] = (1U << j);
- nbat->simd_exclusion_filter2[j*2 + 0] = (1U << j);
- nbat->simd_exclusion_filter2[j*2 + 1] = (1U << j);
- }
+ nbnxn_atomdata_init_simple_exclusion_masks(nbat);
}
#endif
#include "nbnxn_cuda.h"
#include "nbnxn_cuda_data_mgmt.h"
+#if defined TEXOBJ_SUPPORTED && __CUDA_ARCH__ > 300
+#define USE_TEXOBJ
+#endif
/*! Texture reference for nonbonded parameters; bound to cu_nbparam_t.nbfp*/
-texture<float, 1, cudaReadModeElementType> tex_nbfp;
+texture<float, 1, cudaReadModeElementType> nbfp_texref;
/*! Texture reference for Ewald coulomb force table; bound to cu_nbparam_t.coulomb_tab */
-texture<float, 1, cudaReadModeElementType> tex_coulomb_tab;
+texture<float, 1, cudaReadModeElementType> coulomb_tab_texref;
/* Convenience defines */
#define NCL_PER_SUPERCL (NBNXN_GPU_NCLUSTER_PER_SUPERCLUSTER)
/*! Return the reference to the nbfp texture. */
const struct texture<float, 1, cudaReadModeElementType>& nbnxn_cuda_get_nbfp_texref()
{
- return tex_nbfp;
+ return nbfp_texref;
}
/*! Return the reference to the coulomb_tab. */
const struct texture<float, 1, cudaReadModeElementType>& nbnxn_cuda_get_coulomb_tab_texref()
{
- return tex_coulomb_tab;
+ return coulomb_tab_texref;
}
/*! Set up the cache configuration for the non-bonded kernels,
and the table GPU array. If called with an already allocated table,
it just re-uploads the table.
*/
-static void init_ewald_coulomb_force_table(cu_nbparam_t *nbp)
+static void init_ewald_coulomb_force_table(cu_nbparam_t *nbp,
+ const cuda_dev_info_t *dev_info)
{
float *ftmp, *coul_tab;
int tabsize;
nbp->coulomb_tab = coul_tab;
- cudaChannelFormatDesc cd = cudaCreateChannelDesc<float>();
- stat = cudaBindTexture(NULL, &nbnxn_cuda_get_coulomb_tab_texref(),
- coul_tab, &cd, tabsize*sizeof(*coul_tab));
- CU_RET_ERR(stat, "cudaBindTexture on coul_tab failed");
+#ifdef TEXOBJ_SUPPORTED
+ /* Only device CC >= 3.0 (Kepler and later) support texture objects */
+ if (dev_info->prop.major >= 3)
+ {
+ cudaResourceDesc rd;
+ memset(&rd, 0, sizeof(rd));
+ rd.resType = cudaResourceTypeLinear;
+ rd.res.linear.devPtr = nbp->coulomb_tab;
+ rd.res.linear.desc.f = cudaChannelFormatKindFloat;
+ rd.res.linear.desc.x = 32;
+ rd.res.linear.sizeInBytes = tabsize*sizeof(*coul_tab);
+
+ cudaTextureDesc td;
+ memset(&td, 0, sizeof(td));
+ td.readMode = cudaReadModeElementType;
+ stat = cudaCreateTextureObject(&nbp->coulomb_tab_texobj, &rd, &td, NULL);
+ CU_RET_ERR(stat, "cudaCreateTextureObject on coulomb_tab_texobj failed");
+ }
+ else
+#endif
+ {
+ cudaChannelFormatDesc cd = cudaCreateChannelDesc<float>();
+ stat = cudaBindTexture(NULL, &nbnxn_cuda_get_coulomb_tab_texref(),
+ coul_tab, &cd, tabsize*sizeof(*coul_tab));
+ CU_RET_ERR(stat, "cudaBindTexture on coulomb_tab_texref failed");
+ }
}
cu_copy_H2D(coul_tab, ftmp, tabsize*sizeof(*coul_tab));
nbp->coulomb_tab = NULL;
if (nbp->eeltype == eelCuEWALD_TAB || nbp->eeltype == eelCuEWALD_TAB_TWIN)
{
- init_ewald_coulomb_force_table(nbp);
+ init_ewald_coulomb_force_table(nbp, dev_info);
}
nnbfp = 2*ntypes*ntypes;
CU_RET_ERR(stat, "cudaMalloc failed on nbp->nbfp");
cu_copy_H2D(nbp->nbfp, nbat->nbfp, nnbfp*sizeof(*nbp->nbfp));
- cudaChannelFormatDesc cd = cudaCreateChannelDesc<float>();
- stat = cudaBindTexture(NULL, &nbnxn_cuda_get_nbfp_texref(),
- nbp->nbfp, &cd, nnbfp*sizeof(*nbp->nbfp));
- CU_RET_ERR(stat, "cudaBindTexture on nbfp failed");
+#ifdef TEXOBJ_SUPPORTED
+ /* Only device CC >= 3.0 (Kepler and later) support texture objects */
+ if (dev_info->prop.major >= 3)
+ {
+ cudaResourceDesc rd;
+ memset(&rd, 0, sizeof(rd));
+ rd.resType = cudaResourceTypeLinear;
+ rd.res.linear.devPtr = nbp->nbfp;
+ rd.res.linear.desc.f = cudaChannelFormatKindFloat;
+ rd.res.linear.desc.x = 32;
+ rd.res.linear.sizeInBytes = nnbfp*sizeof(*nbp->nbfp);
+
+ cudaTextureDesc td;
+ memset(&td, 0, sizeof(td));
+ td.readMode = cudaReadModeElementType;
+ stat = cudaCreateTextureObject(&nbp->nbfp_texobj, &rd, &td, NULL);
+ CU_RET_ERR(stat, "cudaCreateTextureObject on nbfp_texobj failed");
+ }
+ else
+#endif
+ {
+ cudaChannelFormatDesc cd = cudaCreateChannelDesc<float>();
+ stat = cudaBindTexture(NULL, &nbnxn_cuda_get_nbfp_texref(),
+ nbp->nbfp, &cd, nnbfp*sizeof(*nbp->nbfp));
+ CU_RET_ERR(stat, "cudaBindTexture on nbfp_texref failed");
+ }
}
/*! Re-generate the GPU Ewald force table, resets rlist, and update the
nbp->eeltype = pick_ewald_kernel_type(ic->rcoulomb != ic->rvdw,
cu_nb->dev_info);
- init_ewald_coulomb_force_table(cu_nb->nbparam);
+ init_ewald_coulomb_force_table(cu_nb->nbparam, cu_nb->dev_info);
}
/*! Initializes the pair list data structure. */
init_plist(nb->plist[eintLocal]);
+ /* set device info, just point it to the right GPU among the detected ones */
+ nb->dev_info = &gpu_info->cuda_dev[get_gpu_device_id(gpu_info, my_gpu_index)];
+
/* local/non-local GPU streams */
stat = cudaStreamCreate(&nb->stream[eintLocal]);
CU_RET_ERR(stat, "cudaStreamCreate on stream[eintLocal] failed");
if (nb->bUseTwoStreams)
{
init_plist(nb->plist[eintNonlocal]);
+
+ /* CUDA stream priority available in the CUDA RT 5.5 API.
+ * Note that the device we're running on does not have to support
+ * priorities, because we are querying the priority range which in this
+ * case will be a single value.
+ */
+#if CUDA_VERSION >= 5500
+ {
+ int highest_priority;
+ stat = cudaDeviceGetStreamPriorityRange(NULL, &highest_priority);
+ CU_RET_ERR(stat, "cudaDeviceGetStreamPriorityRange failed");
+
+ stat = cudaStreamCreateWithPriority(&nb->stream[eintNonlocal],
+ cudaStreamDefault,
+ highest_priority);
+ CU_RET_ERR(stat, "cudaStreamCreateWithPriority on stream[eintNonlocal] failed");
+ }
+#else
stat = cudaStreamCreate(&nb->stream[eintNonlocal]);
CU_RET_ERR(stat, "cudaStreamCreate on stream[eintNonlocal] failed");
+#endif
}
/* init events for sychronization (timing disabled for performance reasons!) */
stat = cudaEventCreateWithFlags(&nb->misc_ops_done, cudaEventDisableTiming);
CU_RET_ERR(stat, "cudaEventCreate on misc_ops_one failed");
- /* set device info, just point it to the right GPU among the detected ones */
- nb->dev_info = &gpu_info->cuda_dev[get_gpu_device_id(gpu_info, my_gpu_index)];
-
/* On GPUs with ECC enabled, cudaStreamSynchronize shows a large overhead
* (which increases with shorter time/step) caused by a known CUDA driver bug.
* To work around the issue we'll use an (admittedly fragile) memory polling
bTMPIAtomics = false;
#endif
-#if defined(i386) || defined(__x86_64__)
+#ifdef GMX_IS_X86
bX86 = true;
#else
bX86 = false;
if (nbparam->eeltype == eelCuEWALD_TAB || nbparam->eeltype == eelCuEWALD_TAB_TWIN)
{
- stat = cudaUnbindTexture(nbnxn_cuda_get_coulomb_tab_texref());
- CU_RET_ERR(stat, "cudaUnbindTexture on coulomb_tab failed");
- cu_free_buffered(nbparam->coulomb_tab, &nbparam->coulomb_tab_size);
+
+#ifdef TEXOBJ_SUPPORTED
+ /* Only device CC >= 3.0 (Kepler and later) support texture objects */
+ if (cu_nb->dev_info->prop.major >= 3)
+ {
+ stat = cudaDestroyTextureObject(nbparam->coulomb_tab_texobj);
+ CU_RET_ERR(stat, "cudaDestroyTextureObject on coulomb_tab_texobj failed");
+ }
+ else
+#endif
+ {
+ stat = cudaUnbindTexture(nbnxn_cuda_get_coulomb_tab_texref());
+ CU_RET_ERR(stat, "cudaUnbindTexture on coulomb_tab_texref failed");
+ }
+ cu_free_buffered(nbparam->coulomb_tab, &nbparam->coulomb_tab_size);
}
stat = cudaEventDestroy(cu_nb->nonlocal_done);
}
}
- stat = cudaUnbindTexture(nbnxn_cuda_get_nbfp_texref());
- CU_RET_ERR(stat, "cudaUnbindTexture on coulomb_tab failed");
+#ifdef TEXOBJ_SUPPORTED
+ /* Only device CC >= 3.0 (Kepler and later) support texture objects */
+ if (cu_nb->dev_info->prop.major >= 3)
+ {
+ stat = cudaDestroyTextureObject(nbparam->nbfp_texobj);
+ CU_RET_ERR(stat, "cudaDestroyTextureObject on nbfp_texobj failed");
+ }
+ else
+#endif
+ {
+ stat = cudaUnbindTexture(nbnxn_cuda_get_nbfp_texref());
+ CU_RET_ERR(stat, "cudaUnbindTexture on nbfp_texref failed");
+ }
cu_free_buffered(nbparam->nbfp);
stat = cudaFree(atdat->shift_vec);
Each thread calculates an i force-component taking one pair of i-j atoms.
*/
+#if __CUDA_ARCH__ >= 350
+__launch_bounds__(64,16)
+#endif
#ifdef PRUNE_NBL
#ifdef CALC_ENERGIES
__global__ void NB_KERNEL_FUNC_NAME(k_nbnxn, _ener_prune)
#endif
/* LJ 6*C6 and 12*C12 */
- c6 = tex1Dfetch(tex_nbfp, 2 * (ntypes * typei + typej));
- c12 = tex1Dfetch(tex_nbfp, 2 * (ntypes * typei + typej) + 1);
+#ifdef USE_TEXOBJ
+ c6 = tex1Dfetch<float>(nbparam.nbfp_texobj, 2 * (ntypes * typei + typej));
+ c12 = tex1Dfetch<float>(nbparam.nbfp_texobj, 2 * (ntypes * typei + typej) + 1);
+#else
+ c6 = tex1Dfetch(nbfp_texref, 2 * (ntypes * typei + typej));
+ c12 = tex1Dfetch(nbfp_texref, 2 * (ntypes * typei + typej) + 1);
+#endif /* USE_TEXOBJ */
+
/* avoid NaN for excluded pairs at r=0 */
r2 += (1.0f - int_bit) * NBNXN_AVOID_SING_R2_INC;
#if defined EL_EWALD_ANA
F_invr += qi * qj_f * (int_bit*inv_r2*inv_r + pmecorrF(beta2*r2)*beta3);
#elif defined EL_EWALD_TAB
- F_invr += qi * qj_f * (int_bit*inv_r2 - interpolate_coulomb_force_r(r2 * inv_r, coulomb_tab_scale)) * inv_r;
+ F_invr += qi * qj_f * (int_bit*inv_r2 -
+#ifdef USE_TEXOBJ
+ interpolate_coulomb_force_r(nbparam.coulomb_tab_texobj, r2 * inv_r, coulomb_tab_scale)
+#else
+ interpolate_coulomb_force_r(r2 * inv_r, coulomb_tab_scale)
+#endif /* USE_TEXOBJ */
+ ) * inv_r;
#endif /* EL_EWALD_ANA/TAB */
#ifdef CALC_ENERGIES
Each thread calculates an i force-component taking one pair of i-j atoms.
*/
+#if __CUDA_ARCH__ >= 350
+__launch_bounds__(64,16)
+#endif
#ifdef PRUNE_NBL
#ifdef CALC_ENERGIES
__global__ void NB_KERNEL_FUNC_NAME(k_nbnxn, _ener_prune_legacy)
typei = atom_types[ai];
/* LJ 6*C6 and 12*C12 */
- c6 = tex1Dfetch(tex_nbfp, 2 * (ntypes * typei + typej));
- c12 = tex1Dfetch(tex_nbfp, 2 * (ntypes * typei + typej) + 1);
+ c6 = tex1Dfetch(nbfp_texref, 2 * (ntypes * typei + typej));
+ c12 = tex1Dfetch(nbfp_texref, 2 * (ntypes * typei + typej) + 1);
/* avoid NaN for excluded pairs at r=0 */
r2 += (1.0f - int_bit) * NBNXN_AVOID_SING_R2_INC;
float fract2 = normalized - index;
float fract1 = 1.0f - fract2;
- return fract1 * tex1Dfetch(tex_coulomb_tab, index)
- + fract2 * tex1Dfetch(tex_coulomb_tab, index + 1);
+ return fract1 * tex1Dfetch(coulomb_tab_texref, index)
+ + fract2 * tex1Dfetch(coulomb_tab_texref, index + 1);
}
+#ifdef TEXOBJ_SUPPORTED
+static inline __device__
+float interpolate_coulomb_force_r(cudaTextureObject_t texobj_coulomb_tab,
+ float r, float scale)
+{
+ float normalized = scale * r;
+ int index = (int) normalized;
+ float fract2 = normalized - index;
+ float fract1 = 1.0f - fract2;
+
+ return fract1 * tex1Dfetch<float>(texobj_coulomb_tab, index) +
+ fract2 * tex1Dfetch<float>(texobj_coulomb_tab, index + 1);
+}
+#endif
+
+
/*! Calculate analytical Ewald correction term. */
static inline __device__
float pmecorrF(float z2)
#include "types/nbnxn_cuda_types_ext.h"
#include "../../gmxlib/cuda_tools/cudautils.cuh"
+/* CUDA versions from 5.0 above support texture objects. */
+#if CUDA_VERSION >= 5000
+#define TEXOBJ_SUPPORTED
+#else /* CUDA_VERSION */
+/* This typedef allows us to define only one version of struct cu_nbparam */
+typedef int cudaTextureObject_t;
+#endif /* CUDA_VERSION */
+
#ifdef __cplusplus
extern "C" {
#endif
/* Non-bonded kernel versions. */
-/* All structs prefixed with "cu_" hold data used in GPU calculations and
- * are passed to the kernels, except cu_timers_t. */
+/* All structs prefixed with "cu_" hold data used in GPU calculations and
+ * are passed to the kernels, except cu_timers_t. */
typedef struct cu_plist cu_plist_t;
typedef struct cu_atomdata cu_atomdata_t;
typedef struct cu_nbparam cu_nbparam_t;
/** Staging area for temporary data. The energies get downloaded here first,
- * before getting added to the CPU-side aggregate values.
+ * before getting added to the CPU-side aggregate values.
*/
struct nb_staging
{
/** Parameters required for the CUDA nonbonded calculations. */
struct cu_nbparam
{
- int eeltype; /**< type of electrostatics */
-
- float epsfac; /**< charge multiplication factor */
- float c_rf, /**< Reaction-field/plain cutoff electrostatics const. */
- two_k_rf; /**< Reaction-field electrostatics constant */
- float ewald_beta; /**< Ewald/PME parameter */
- float sh_ewald; /**< Ewald/PME correction term */
- float rvdw_sq; /**< VdW cut-off */
- float rcoulomb_sq; /**< Coulomb cut-off */
- float rlist_sq; /**< pair-list cut-off */
- float sh_invrc6; /**< LJ potential correction term */
-
- float *nbfp; /**< nonbonded parameter table with C6/C12 pairs */
-
- /* Ewald Coulomb force table data */
- int coulomb_tab_size; /**< table size (s.t. it fits in texture cache) */
- float coulomb_tab_scale; /**< table scale/spacing */
- float *coulomb_tab; /**< pointer to the table in the device memory */
+ int eeltype; /**< type of electrostatics */
+
+ float epsfac; /**< charge multiplication factor */
+ float c_rf; /**< Reaction-field/plain cutoff electrostatics const. */
+ float two_k_rf; /**< Reaction-field electrostatics constant */
+ float ewald_beta; /**< Ewald/PME parameter */
+ float sh_ewald; /**< Ewald/PME correction term */
+ float rvdw_sq; /**< VdW cut-off */
+ float rcoulomb_sq; /**< Coulomb cut-off */
+ float rlist_sq; /**< pair-list cut-off */
+ float sh_invrc6; /**< LJ potential correction term */
+
+ /* Non-bonded parameters - accessed through texture memory */
+ float *nbfp; /**< nonbonded parameter table with C6/C12 pairs */
+ cudaTextureObject_t nbfp_texobj; /**< texture object bound to nbfp */
+
+ /* Ewald Coulomb force table data - accessed through texture memory */
+ int coulomb_tab_size; /**< table size (s.t. it fits in texture cache) */
+ float coulomb_tab_scale; /**< table scale/spacing */
+ float *coulomb_tab; /**< pointer to the table in the device memory */
+ cudaTextureObject_t coulomb_tab_texobj; /**< texture object bound to coulomb_tab */
};
/** Pair list data */
cudaStream_t stream[2]; /**< local and non-local GPU streams */
/** events used for synchronization */
- cudaEvent_t nonlocal_done; /**< event triggered when the non-local non-bonded kernel
- is done (and the local transfer can proceed) */
- cudaEvent_t misc_ops_done; /**< event triggered when the operations that precede the
- main force calculations are done (e.g. buffer 0-ing) */
+ cudaEvent_t nonlocal_done; /**< event triggered when the non-local non-bonded kernel
+ is done (and the local transfer can proceed) */
+ cudaEvent_t misc_ops_done; /**< event triggered when the operations that precede the
+ main force calculations are done (e.g. buffer 0-ing) */
/* NOTE: With current CUDA versions (<=5.0) timing doesn't work with multiple
* concurrent streams, so we won't time if both l/nl work is done on GPUs.
#include "gmx_simd_macros.h"
#endif
-#ifdef __cplusplus
-extern "C" {
+
+/* Bounding box calculations are (currently) always in single precision.
+ * This uses less (cache-)memory and SIMD is faster, at least on x86.
+ */
+#define GMX_SIMD4_SINGLE
+/* Include the 4-wide SIMD macro file */
+#include "gmx_simd4_macros.h"
+/* Check if we have 4-wide SIMD macro support */
+#ifdef GMX_HAVE_SIMD4_MACROS
+#define NBNXN_SEARCH_BB_SIMD4
#endif
-#ifdef GMX_X86_SSE2
-/* Use 4-way SIMD for, always, single precision bounding box calculations */
-#define NBNXN_SEARCH_BB_SSE
+#ifdef __cplusplus
+extern "C" {
#endif
#endif
+#ifdef NBNXN_SEARCH_BB_SIMD4
+/* Memory alignment in bytes as required by SIMD aligned loads/stores */
+#define NBNXN_SEARCH_BB_MEM_ALIGN (GMX_SIMD4_WIDTH*sizeof(float))
+#else
+/* No alignment required, but set it so we can call the same routines */
+#define NBNXN_SEARCH_BB_MEM_ALIGN 32
+#endif
+
+
/* Pair search box lower and upper corner in x,y,z.
- * Store this in 4 iso 3 reals, which is useful with SSE.
- * To avoid complicating the code we also use 4 without SSE.
+ * Store this in 4 iso 3 reals, which is useful with 4-wide SIMD.
+ * To avoid complicating the code we also use 4 without 4-wide SIMD.
*/
#define NNBSBB_C 4
/* Pair search box lower and upper bound in z only. */
int *nsubc; /* The number of sub cells for each super cell */
float *bbcz; /* Bounding boxes in z for the super cells */
nbnxn_bb_t *bb; /* 3D bounding boxes for the sub cells */
- nbnxn_bb_t *bbj; /* 3D j-b.boxes for SSE-double or AVX-single */
+ nbnxn_bb_t *bbj; /* 3D j-bounding boxes for the case where *
+ * the i- and j-cluster sizes are different */
float *pbb; /* 3D b. boxes in xxxx format per super cell */
int *flags; /* Flag for the super cells */
int nc_nalloc; /* Allocation size for the pointers above */
typedef struct nbnxn_x_ci_simd_4xn {
/* The i-cluster coordinates for simple search */
- gmx_mm_pr ix_SSE0, iy_SSE0, iz_SSE0;
- gmx_mm_pr ix_SSE1, iy_SSE1, iz_SSE1;
- gmx_mm_pr ix_SSE2, iy_SSE2, iz_SSE2;
- gmx_mm_pr ix_SSE3, iy_SSE3, iz_SSE3;
+ gmx_mm_pr ix_S0, iy_S0, iz_S0;
+ gmx_mm_pr ix_S1, iy_S1, iz_S1;
+ gmx_mm_pr ix_S2, iy_S2, iz_S2;
+ gmx_mm_pr ix_S3, iy_S3, iz_S3;
} nbnxn_x_ci_simd_4xn_t;
typedef struct nbnxn_x_ci_simd_2xnn {
/* The i-cluster coordinates for simple search */
- gmx_mm_pr ix_SSE0, iy_SSE0, iz_SSE0;
- gmx_mm_pr ix_SSE2, iy_SSE2, iz_SSE2;
+ gmx_mm_pr ix_S0, iy_S0, iz_S0;
+ gmx_mm_pr ix_S2, iy_S2, iz_S2;
} nbnxn_x_ci_simd_2xnn_t;
#endif
#ifdef GMX_SIMD_REFERENCE_PLAIN_C
+/* Set the stride for the lookup of the two LJ parameters from their
+ * (padded) array.
+ * Note that currently only arrays with stride 2 and 4 are available.
+ * Since the reference code does not require alignment, we can always use 2.
+ */
+static const int nbfp_stride = 2;
+
+/* Align a stack-based thread-local working array. */
+static gmx_inline int *
+prepare_table_load_buffer(const int *array)
+{
+ return NULL;
+}
+
#include "nbnxn_kernel_simd_utils_ref.h"
#else /* GMX_SIMD_REFERENCE_PLAIN_C */
/* Include x86 SSE2 compatible SIMD functions */
/* Set the stride for the lookup of the two LJ parameters from their
- (padded) array. Only strides of 2 and 4 are currently supported. */
-#if defined GMX_NBNXN_SIMD_2XNN
-static const int nbfp_stride = 4;
-#elif defined GMX_DOUBLE
+ * (padded) array. We use the minimum supported SIMD memory alignment.
+ */
+#if defined GMX_DOUBLE
static const int nbfp_stride = 2;
#else
static const int nbfp_stride = 4;
static const int nbfp_stride = GMX_SIMD_WIDTH_HERE;
#endif
+/* We use the FDV0 table layout when we can use aligned table loads */
+#if GMX_SIMD_WIDTH_HERE == 4
+#define TAB_FDV0
+#endif
+
+#ifdef GMX_CPU_ACCELERATION_IBM_QPX
+#include "nbnxn_kernel_simd_utils_ibm_qpx.h"
+#endif /* GMX_CPU_ACCELERATION_IBM_QPX */
+
#endif /* GMX_X86_SSE2 */
#endif /* GMX_SIMD_REFERENCE_PLAIN_C */
--- /dev/null
+/*
+ * This file is part of the GROMACS molecular simulation package.
+ *
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2012, The GROMACS Development Team
+ * Copyright (c) 2012,2013, by the GROMACS development team, led by
+ * David van der Spoel, Berk Hess, Erik Lindahl, and including many
+ * others, as listed in the AUTHORS file in the top-level source
+ * directory and at http://www.gromacs.org.
+ *
+ * GROMACS is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ *
+ * GROMACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with GROMACS; if not, see
+ * http://www.gnu.org/licenses, or write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * If you want to redistribute modifications to GROMACS, please
+ * consider that scientific software is very special. Version
+ * control is crucial - bugs must be traceable. We will be happy to
+ * consider code for inclusion in the official distribution, but
+ * derived work must not be called official GROMACS. Details are found
+ * in the README & COPYING files - if they are missing, get the
+ * official version at http://www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the research papers on the package. Check out http://www.gromacs.org.
+ */
+#ifndef _nbnxn_kernel_simd_utils_ibm_qpx_h_
+#define _nbnxn_kernel_simd_utils_ibm_qpx_h_
+
+typedef gmx_mm_pr gmx_exclfilter;
+static const int filter_stride = 1;
+
+/* The 4xn kernel operates on 4-wide i-force registers */
+typedef gmx_mm_pr gmx_mm_pr4;
+
+/* This files contains all functions/macros for the SIMD kernels
+ * which have explicit dependencies on the j-cluster size and/or SIMD-width.
+ * The functionality which depends on the j-cluster size is:
+ * LJ-parameter lookup
+ * force table lookup
+ * energy group pair energy storage
+ */
+
+/* Collect all [0123] elements of the 4 inputs to out[0123], respectively */
+static gmx_inline void
+gmx_transpose_4_ps(gmx_mm_pr a, gmx_mm_pr b, gmx_mm_pr c, gmx_mm_pr d,
+ gmx_mm_pr *out0, gmx_mm_pr *out1,
+ gmx_mm_pr *out2, gmx_mm_pr *out3)
+{
+ /* Prepare control vectors for swizzling. In its third input,
+ vec_perm accepts indices into the effective 8-wide SIMD vector
+ created by concatenating its first two inputs. Those indices
+ map data from the input vectors to the output vector.
+
+ vec_gpci() converts an octal literal of the indices into the
+ correct form for vec_perm() to use. That form is an octal digit
+ in bits 0-2 of the mantissa of each double. */
+ gmx_mm_pr p6420 = vec_gpci(06420);
+ gmx_mm_pr p7531 = vec_gpci(07531);
+
+ /* Four-way swizzle (i.e. transpose) of vectors a = a0a1a2a3, etc. */
+ gmx_mm_pr b2b0a2a0 = vec_perm(a, b, p6420);
+ gmx_mm_pr b3b1a3a1 = vec_perm(a, b, p7531);
+ gmx_mm_pr d2d0c2c0 = vec_perm(c, d, p6420);
+ gmx_mm_pr d3d1c3c1 = vec_perm(c, d, p7531);
+ *out0 = vec_perm(d2d0c2c0, b2b0a2a0, p7531);
+ *out1 = vec_perm(d3d1c3c1, b3b1a3a1, p7531);
+ *out2 = vec_perm(d2d0c2c0, b2b0a2a0, p6420);
+ *out3 = vec_perm(d3d1c3c1, b3b1a3a1, p6420);
+}
+
+/* Collect element 0 and 1 of the 4 inputs to out0 and out1, respectively */
+static gmx_inline void
+gmx_shuffle_4_ps_fil01_to_2_ps(gmx_mm_pr a, gmx_mm_pr b, gmx_mm_pr c, gmx_mm_pr d,
+ gmx_mm_pr *out0, gmx_mm_pr *out1)
+{
+ gmx_mm_pr p6420 = vec_gpci(06420);
+ gmx_mm_pr p7531 = vec_gpci(07531);
+
+ /* Partial four-way swizzle of vectors a = a0a1a2a3, etc. */
+ gmx_mm_pr b2b0a2a0 = vec_perm(a, b, p6420);
+ gmx_mm_pr b3b1a3a1 = vec_perm(a, b, p7531);
+ gmx_mm_pr d2d0c2c0 = vec_perm(c, d, p6420);
+ gmx_mm_pr d3d1c3c1 = vec_perm(c, d, p7531);
+ *out0 = vec_perm(d2d0c2c0, b2b0a2a0, p7531);
+ *out1 = vec_perm(d3d1c3c1, b3b1a3a1, p7531);
+}
+
+/* Collect element 2 of the 4 inputs to out */
+static gmx_inline gmx_mm_pr
+gmx_shuffle_4_ps_fil2_to_1_ps(gmx_mm_pr a, gmx_mm_pr b, gmx_mm_pr c, gmx_mm_pr d)
+{
+ gmx_mm_pr p6420 = vec_gpci(06420);
+
+ /* Partial four-way swizzle of vectors a = a0a1a2a3, etc. */
+ gmx_mm_pr b2b0a2a0 = vec_perm(a, b, p6420);
+ gmx_mm_pr d2d0c2c0 = vec_perm(c, d, p6420);
+ return vec_perm(d2d0c2c0, b2b0a2a0, p6420);
+}
+
+#ifdef TAB_FDV0
+/* Align a stack-based thread-local working array. Table loads on QPX
+ * use the array, but most other implementations do not. */
+static gmx_inline int *
+prepare_table_load_buffer(const int *array)
+{
+ return gmx_simd_align_int(array);
+}
+
+static gmx_inline void
+load_table_f(const real *tab_coul_FDV0, gmx_epi32 ti_S, int *ti,
+ gmx_mm_pr *ctab0_S, gmx_mm_pr *ctab1_S)
+{
+#ifdef NDEBUG
+ /* Just like 256-bit AVX, we need to use memory to get indices
+ into integer registers efficiently. */
+ vec_st(ti_S, 0, ti);
+#else
+ vec_sta(ti_S, 0, ti);
+#endif
+
+ /* Here we load 4 aligned reals, but we need just 2 elements of each */
+ gmx_mm_pr a = gmx_load_pr(tab_coul_FDV0 + ti[0] * nbfp_stride);
+ gmx_mm_pr b = gmx_load_pr(tab_coul_FDV0 + ti[1] * nbfp_stride);
+ gmx_mm_pr c = gmx_load_pr(tab_coul_FDV0 + ti[2] * nbfp_stride);
+ gmx_mm_pr d = gmx_load_pr(tab_coul_FDV0 + ti[3] * nbfp_stride);
+
+ gmx_shuffle_4_ps_fil01_to_2_ps(a, b, c, d, ctab0_S, ctab1_S);
+}
+
+static gmx_inline void
+load_table_f_v(const real *tab_coul_FDV0,
+ gmx_epi32 ti_S, int *ti,
+ gmx_mm_pr *ctab0_S, gmx_mm_pr *ctab1_S,
+ gmx_mm_pr *ctabv_S)
+{
+#ifdef NDEBUG
+ /* Just like 256-bit AVX, we need to use memory to get indices
+ into integer registers efficiently. */
+ vec_st(ti_S, 0, ti);
+#else
+ vec_sta(ti_S, 0, ti);
+#endif
+
+ /* Here we load 4 aligned reals, but we need just 3 elements of each. */
+ gmx_mm_pr a = gmx_load_pr(tab_coul_FDV0 + ti[0] * nbfp_stride);
+ gmx_mm_pr b = gmx_load_pr(tab_coul_FDV0 + ti[1] * nbfp_stride);
+ gmx_mm_pr c = gmx_load_pr(tab_coul_FDV0 + ti[2] * nbfp_stride);
+ gmx_mm_pr d = gmx_load_pr(tab_coul_FDV0 + ti[3] * nbfp_stride);
+
+ gmx_shuffle_4_ps_fil01_to_2_ps(a, b, c, d, ctab0_S, ctab1_S);
+ *ctabv_S = gmx_shuffle_4_ps_fil2_to_1_ps(a, b, c, d);
+}
+#else
+
+/* Not required for BlueGene/Q */
+
+#endif
+
+/* Sum the elements within each input register and store the sums in out.
+ */
+static gmx_inline gmx_mm_pr
+gmx_mm_transpose_sum4_pr(gmx_mm_pr a, gmx_mm_pr b,
+ gmx_mm_pr c, gmx_mm_pr d)
+{
+ gmx_mm_pr a0b0c0d0, a1b1c1d1, a2b2c2d2, a3b3c3d3;
+ gmx_transpose_4_ps(a, b, c, d,
+ &a0b0c0d0,
+ &a1b1c1d1,
+ &a2b2c2d2,
+ &a3b3c3d3);
+ /* Now reduce the transposed vectors */
+ gmx_mm_pr sum01 = gmx_add_pr(a0b0c0d0, a1b1c1d1);
+ gmx_mm_pr sim23 = gmx_add_pr(a2b2c2d2, a3b3c3d3);
+ return gmx_add_pr(sum01, sim23);
+}
+
+#ifdef GMX_DOUBLE
+/* In double precision on x86 it can be faster to first calculate
+ * single precision square roots for two double precision registers at
+ * once and then use double precision Newton-Raphson iteration to
+ * reach full double precision. For QPX, we just wrap the usual
+ * reciprocal square roots.
+ */
+static gmx_inline void
+gmx_mm_invsqrt2_pd(gmx_mm_pr in0, gmx_mm_pr in1,
+ gmx_mm_pr *out0, gmx_mm_pr *out1)
+{
+ *out0 = gmx_invsqrt_pr(in0);
+ *out1 = gmx_invsqrt_pr(in1);
+}
+#endif
+
+static gmx_inline void
+load_lj_pair_params(const real *nbfp, const int *type, int aj,
+ gmx_mm_pr *c6_S, gmx_mm_pr *c12_S)
+{
+ /* Here we load 4 aligned reals, but we need just 2 elemnts of each. */
+ gmx_mm_pr a = gmx_load_pr(nbfp + type[aj+0] * nbfp_stride);
+ gmx_mm_pr b = gmx_load_pr(nbfp + type[aj+1] * nbfp_stride);
+ gmx_mm_pr c = gmx_load_pr(nbfp + type[aj+2] * nbfp_stride);
+ gmx_mm_pr d = gmx_load_pr(nbfp + type[aj+3] * nbfp_stride);
+
+ gmx_shuffle_4_ps_fil01_to_2_ps(a, b, c, d, c6_S, c12_S);
+}
+
+/* Define USE_FUNCTIONS_FOR_QPX to get the static inline functions
+ * that seem to exhaust xlC 12.1 during kernel compilation */
+
+#ifndef USE_FUNCTIONS_FOR_QPX
+
+#define gmx_load_exclusion_filter(a) vec_ldia(0, (int *) a)
+#define gmx_load_interaction_mask_pb(a, b) vec_ld(a, (real *) b)
+
+#else /* USE_FUNCTIONS_FOR_QPX */
+
+static gmx_inline gmx_exclfilter gmx_load_exclusion_filter(const unsigned *a)
+{
+#ifdef NDEBUG
+ return vec_ldia(0, (int *) a);
+#else
+ return vec_ldiaa(0, (int *) a);
+#endif
+}
+
+/* Code for handling loading and applying exclusion masks. Note that
+ parameter a is not treated like an array index; it is naively added
+ to b, so should be in bytes. */
+static gmx_inline gmx_mm_pb gmx_load_interaction_mask_pb(long a, const real *b)
+{
+#ifdef NDEBUG
+ return vec_ld(a, (real *) b);
+#else
+ return vec_lda(a, (real *) b);
+#endif
+}
+
+#endif /* USE_FUNCTIONS_FOR_QPX */
+
+#endif /* _nbnxn_kernel_simd_utils_ibm_qpx_h_ */
#ifndef _nbnxn_kernel_simd_utils_ref_h_
#define _nbnxn_kernel_simd_utils_ref_h_
-typedef gmx_simd_ref_epi32 gmx_simd_ref_exclfilter;
-#define gmx_exclfilter gmx_simd_ref_exclfilter
+typedef gmx_simd_ref_epi32 gmx_simd_ref_exclfilter;
+typedef gmx_simd_ref_exclfilter gmx_exclfilter;
static const int filter_stride = GMX_SIMD_EPI32_WIDTH/GMX_SIMD_WIDTH_HERE;
+/* Set the stride for the lookup of the two LJ parameters from their
+ (padded) array. Only strides of 2 and 4 are currently supported. */
+#if defined GMX_NBNXN_SIMD_2XNN
+static const int nbfp_stride = 4;
+#elif defined GMX_DOUBLE
+static const int nbfp_stride = 2;
+#else
+static const int nbfp_stride = 4;
+#endif
+
#if GMX_SIMD_WIDTH_HERE > 4
/* The 4xn kernel operates on 4-wide i-force registers */
return c;
}
+#else
+
+typedef gmx_simd_ref_pr gmx_simd_ref_pr4;
+
#endif
}
static gmx_inline gmx_simd_ref_exclfilter
-gmx_simd_ref_load_exclusion_filter(const unsigned *src)
+gmx_simd_ref_load_exclusion_filter(const int *src)
{
gmx_simd_ref_exclfilter a;
int i;
* energy group pair energy storage
*/
-#define gmx_exclfilter gmx_epi32
+typedef gmx_epi32 gmx_exclfilter;
static const int filter_stride = GMX_SIMD_EPI32_WIDTH/GMX_SIMD_WIDTH_HERE;
/* Transpose 2 double precision registers */
* energy group pair energy storage
*/
-#define gmx_exclfilter gmx_epi32
+typedef gmx_epi32 gmx_exclfilter;
static const int filter_stride = GMX_SIMD_EPI32_WIDTH/GMX_SIMD_WIDTH_HERE;
/* Collect element 0 and 1 of the 4 inputs to out0 and out1, respectively */
_MM_TRANSPOSE4_PS(in0, in1, in2, in3);
in0 = _mm_add_ps(in0, in1);
in2 = _mm_add_ps(in2, in3);
-
+
return _mm_add_ps(in0, in2);
}
{
__m128 clj_S[UNROLLJ];
int p;
-
+
for (p = 0; p < UNROLLJ; p++)
{
/* Here we load 4 aligned floats, but we need just 2 */
* energy group pair energy storage
*/
-#define gmx_exclfilter gmx_mm_pr
+typedef gmx_mm_pr gmx_exclfilter;
static const int filter_stride = 2;
/* Transpose 2 double precision registers */
in0 = _mm256_hadd_pd(in0, in1);
in2 = _mm256_hadd_pd(in2, in3);
- return _mm256_add_pd(_mm256_permute2f128_pd(in0, in2, 0x20), _mm256_permute2f128_pd(in0, in2, 0x31));
+ return _mm256_add_pd(_mm256_permute2f128_pd(in0, in2, 0x20), _mm256_permute2f128_pd(in0, in2, 0x31));
}
static gmx_inline __m256
__m256 s, ir;
__m256d lu0, lu1;
- s = gmx_2_m128_to_m256(_mm256_cvtpd_ps(in0), _mm256_cvtpd_ps(in1));
+ s = gmx_2_m128_to_m256(_mm256_cvtpd_ps(in0), _mm256_cvtpd_ps(in1));
ir = gmx_mm256_invsqrt_ps_single(s);
lu0 = _mm256_cvtps_pd(_mm256_castps256_ps128(ir));
lu1 = _mm256_cvtps_pd(_mm256_extractf128_ps(ir, 1));
* energy group pair energy storage
*/
-#define gmx_exclfilter gmx_mm_pr
+typedef gmx_mm_pr gmx_exclfilter;
static const int filter_stride = 1;
/* The 4xn kernel operates on 4-wide i-force registers */
#include "maths.h"
#endif
+#ifndef GMX_SIMD_J_UNROLL_SIZE
+#error "Need to define GMX_SIMD_J_UNROLL_SIZE before including the 2xnn kernel common header file"
+#endif
#define SUM_SIMD4(x) (x[0]+x[1]+x[2]+x[3])
ajz = ajy + STRIDE;
#ifdef CHECK_EXCLS
- gmx_load_simd_2xnn_interactions(l_cj[cjind].excl, filter_S0, filter_S2, &interact_S0, &interact_S2);
+ gmx_load_simd_2xnn_interactions(l_cj[cjind].excl,
+ filter_S0, filter_S2,
+ &interact_S0, &interact_S2);
#endif /* CHECK_EXCLS */
/* load j atom coordinates */
#endif /* CALC_ENERGIES */
#ifdef CALC_LJ
- fscal_S0 = gmx_mul_pr(rinvsq_S0,
#ifdef CALC_COULOMB
+ fscal_S0 = gmx_mul_pr(rinvsq_S0,
gmx_add_pr(frcoul_S0,
+ gmx_sub_pr(FrLJ12_S0, FrLJ6_S0)));
#else
+ fscal_S0 = gmx_mul_pr(rinvsq_S0,
(
-#endif
gmx_sub_pr(FrLJ12_S0, FrLJ6_S0)));
+#endif
#else
fscal_S0 = gmx_mul_pr(rinvsq_S0, frcoul_S0);
#endif /* CALC_LJ */
#if defined CALC_LJ && !defined HALF_LJ
- fscal_S2 = gmx_mul_pr(rinvsq_S2,
#ifdef CALC_COULOMB
+ fscal_S2 = gmx_mul_pr(rinvsq_S2,
gmx_add_pr(frcoul_S2,
+ gmx_sub_pr(FrLJ12_S2, FrLJ6_S2)));
#else
+ fscal_S2 = gmx_mul_pr(rinvsq_S2,
(
-#endif
gmx_sub_pr(FrLJ12_S2, FrLJ6_S2)));
+#endif
#else
/* Atom 2 and 3 don't have LJ, so only add Coulomb forces */
fscal_S2 = gmx_mul_pr(rinvsq_S2, frcoul_S2);
#include "maths.h"
#endif
+#ifndef GMX_SIMD_J_UNROLL_SIZE
+#error "Need to define GMX_SIMD_J_UNROLL_SIZE before including the 4xn kernel common header file"
+#endif
+
#define SUM_SIMD4(x) (x[0]+x[1]+x[2]+x[3])
#define UNROLLI NBNXN_CPU_CLUSTER_I_SIZE
gmx_exclfilter filter_S1,
gmx_exclfilter filter_S2,
gmx_exclfilter filter_S3,
+ const char *interaction_mask_indices,
+ real *simd_interaction_array,
gmx_mm_pb *interact_S0,
gmx_mm_pb *interact_S1,
gmx_mm_pb *interact_S2,
gmx_mm_pb *interact_S3)
{
+#ifdef GMX_X86_SSE2
/* Load integer interaction mask */
gmx_exclfilter mask_pr_S = gmx_load1_exclfilter(excl);
*interact_S0 = gmx_checkbitmask_pb(mask_pr_S, filter_S0);
*interact_S1 = gmx_checkbitmask_pb(mask_pr_S, filter_S1);
*interact_S2 = gmx_checkbitmask_pb(mask_pr_S, filter_S2);
*interact_S3 = gmx_checkbitmask_pb(mask_pr_S, filter_S3);
+#endif
+#ifdef GMX_CPU_ACCELERATION_IBM_QPX
+ const int size = GMX_SIMD_WIDTH_HERE * sizeof(real);
+ *interact_S0 = gmx_load_interaction_mask_pb(size*interaction_mask_indices[0], simd_interaction_array);
+ *interact_S1 = gmx_load_interaction_mask_pb(size*interaction_mask_indices[1], simd_interaction_array);
+ *interact_S2 = gmx_load_interaction_mask_pb(size*interaction_mask_indices[2], simd_interaction_array);
+ *interact_S3 = gmx_load_interaction_mask_pb(size*interaction_mask_indices[3], simd_interaction_array);
+#endif
}
/* All functionality defines are set here, except for:
ajz = ajy + STRIDE;
#ifdef CHECK_EXCLS
- gmx_load_simd_4xn_interactions(l_cj[cjind].excl, filter_S0, filter_S1, filter_S2, filter_S3, &interact_S0, &interact_S1, &interact_S2, &interact_S3);
+ gmx_load_simd_4xn_interactions(l_cj[cjind].excl,
+ filter_S0, filter_S1,
+ filter_S2, filter_S3,
+#ifdef GMX_CPU_ACCELERATION_IBM_QPX
+ l_cj[cjind].interaction_mask_indices,
+ nbat->simd_interaction_array,
+#else
+ /* The struct fields do not exist
+ except on BlueGene/Q */
+ NULL,
+ NULL,
+#endif
+ &interact_S0, &interact_S1,
+ &interact_S2, &interact_S3);
#endif /* CHECK_EXCLS */
/* load j atom coordinates */
#endif
#endif
#ifndef ENERGY_GROUPS
- Vvdwtot_S = gmx_add_pr(Vvdwtot_S,
#ifndef HALF_LJ
+ Vvdwtot_S = gmx_add_pr(Vvdwtot_S,
gmx_sum4_pr(VLJ_S0, VLJ_S1, VLJ_S2, VLJ_S3)
+ );
#else
+ Vvdwtot_S = gmx_add_pr(Vvdwtot_S,
gmx_add_pr(VLJ_S0, VLJ_S1)
-#endif
);
+#endif
#else
add_ener_grp(VLJ_S0, vvdwtp[0], egp_jj);
add_ener_grp(VLJ_S1, vvdwtp[1], egp_jj);
#endif /* CALC_ENERGIES */
#ifdef CALC_LJ
- fscal_S0 = gmx_mul_pr(rinvsq_S0,
#ifdef CALC_COULOMB
+ fscal_S0 = gmx_mul_pr(rinvsq_S0,
gmx_add_pr(frcoul_S0,
+ gmx_sub_pr(FrLJ12_S0, FrLJ6_S0)));
#else
+ fscal_S0 = gmx_mul_pr(rinvsq_S0,
(
-#endif
gmx_sub_pr(FrLJ12_S0, FrLJ6_S0)));
- fscal_S1 = gmx_mul_pr(rinvsq_S1,
+#endif
#ifdef CALC_COULOMB
+ fscal_S1 = gmx_mul_pr(rinvsq_S1,
gmx_add_pr(frcoul_S1,
+ gmx_sub_pr(FrLJ12_S1, FrLJ6_S1)));
#else
+ fscal_S1 = gmx_mul_pr(rinvsq_S1,
(
-#endif
gmx_sub_pr(FrLJ12_S1, FrLJ6_S1)));
+#endif
#else
fscal_S0 = gmx_mul_pr(rinvsq_S0, frcoul_S0);
fscal_S1 = gmx_mul_pr(rinvsq_S1, frcoul_S1);
#endif /* CALC_LJ */
#if defined CALC_LJ && !defined HALF_LJ
- fscal_S2 = gmx_mul_pr(rinvsq_S2,
#ifdef CALC_COULOMB
+ fscal_S2 = gmx_mul_pr(rinvsq_S2,
gmx_add_pr(frcoul_S2,
+ gmx_sub_pr(FrLJ12_S2, FrLJ6_S2)));
#else
+ fscal_S2 = gmx_mul_pr(rinvsq_S2,
(
-#endif
gmx_sub_pr(FrLJ12_S2, FrLJ6_S2)));
- fscal_S3 = gmx_mul_pr(rinvsq_S3,
+#endif
#ifdef CALC_COULOMB
+ fscal_S3 = gmx_mul_pr(rinvsq_S3,
gmx_add_pr(frcoul_S3,
+ gmx_sub_pr(FrLJ12_S3, FrLJ6_S3)));
#else
+ fscal_S3 = gmx_mul_pr(rinvsq_S3,
(
-#endif
gmx_sub_pr(FrLJ12_S3, FrLJ6_S3)));
+#endif
#else
/* Atom 2 and 3 don't have LJ, so only add Coulomb forces */
fscal_S2 = gmx_mul_pr(rinvsq_S2, frcoul_S2);
#if UNROLLJ >= 4
/* We use an i-force SIMD register width of 4 */
#if UNROLLJ == 4
-#define gmx_mm_pr4 gmx_mm_pr
-#define gmx_load_pr4 gmx_load_pr
-#define gmx_store_pr4 gmx_store_pr
-#define gmx_add_pr4 gmx_add_pr
+#define gmx_mm_pr4 gmx_mm_pr
+#define gmx_load_pr4 gmx_load_pr
+#define gmx_store_pr4 gmx_store_pr
+#define gmx_add_pr4 gmx_add_pr
#else
/* The pr4 stuff is defined in nbnxn_kernel_simd_utils.h */
#endif
unsigned *exclusion_filter;
gmx_exclfilter filter_S0, filter_S1, filter_S2, filter_S3;
- gmx_mm_pr zero_S = gmx_set1_pr(0);
+ gmx_mm_pr zero_S = gmx_set1_pr(0.0);
- gmx_mm_pr one_S = gmx_set1_pr(1.0);
- gmx_mm_pr iq_S0 = gmx_setzero_pr();
- gmx_mm_pr iq_S1 = gmx_setzero_pr();
- gmx_mm_pr iq_S2 = gmx_setzero_pr();
- gmx_mm_pr iq_S3 = gmx_setzero_pr();
- gmx_mm_pr mrc_3_S;
+ gmx_mm_pr one_S = gmx_set1_pr(1.0);
+ gmx_mm_pr iq_S0 = gmx_setzero_pr();
+ gmx_mm_pr iq_S1 = gmx_setzero_pr();
+ gmx_mm_pr iq_S2 = gmx_setzero_pr();
+ gmx_mm_pr iq_S3 = gmx_setzero_pr();
+ gmx_mm_pr mrc_3_S;
#ifdef CALC_ENERGIES
- gmx_mm_pr hrc_3_S, moh_rc_S;
+ gmx_mm_pr hrc_3_S, moh_rc_S;
#endif
#ifdef CALC_COUL_TAB
#include "nrnb.h"
-#ifdef NBNXN_SEARCH_BB_SSE
-/* We use SSE or AVX-128bit for bounding box calculations */
+#ifdef NBNXN_SEARCH_BB_SIMD4
+/* We use 4-wide SIMD for bounding box calculations */
#ifndef GMX_DOUBLE
-/* Single precision BBs + coordinates, we can also load coordinates using SSE */
-#define NBNXN_SEARCH_SSE_SINGLE
+/* Single precision BBs + coordinates, we can also load coordinates with SIMD */
+#define NBNXN_SEARCH_SIMD4_FLOAT_X_BB
#endif
-/* Include basic SSE2 stuff */
-#include <emmintrin.h>
-
-#if defined NBNXN_SEARCH_SSE_SINGLE && (GPU_NSUBCELL == 4 || GPU_NSUBCELL == 8)
+#if defined NBNXN_SEARCH_SIMD4_FLOAT_X_BB && (GPU_NSUBCELL == 4 || GPU_NSUBCELL == 8)
/* Store bounding boxes with x, y and z coordinates in packs of 4 */
-#define NBNXN_PBB_SSE
+#define NBNXN_PBB_SIMD4
#endif
-/* The width of SSE/AVX128 with single precision for bounding boxes with GPU.
- * Here AVX-256 turns out to be slightly slower than AVX-128.
+/* The packed bounding box coordinate stride is always set to 4.
+ * With AVX we could use 8, but that turns out not to be faster.
*/
#define STRIDE_PBB 4
#define STRIDE_PBB_2LOG 2
-#endif /* NBNXN_SEARCH_BB_SSE */
+#endif /* NBNXN_SEARCH_BB_SIMD4 */
#ifdef GMX_NBNXN_SIMD
#endif /* GMX_NBNXN_SIMD */
-#ifdef NBNXN_SEARCH_BB_SSE
+#ifdef NBNXN_SEARCH_BB_SIMD4
/* Store bounding boxes corners as quadruplets: xxxxyyyyzzzz */
#define NBNXN_BBXXXX
/* Size of bounding box corners quadruplet */
sfree_aligned(grid->bb);
/* This snew also zeros the contents, this avoid possible
- * floating exceptions in SSE with the unused bb elements.
+ * floating exceptions in SIMD with the unused bb elements.
*/
if (grid->bSimple)
{
/* Set the "empty" bounding box to the same as the first one,
* so we don't need to treat special cases in the rest of the code.
*/
-#ifdef NBNXN_SEARCH_BB_SSE
- _mm_store_ps(&bbj[1].lower[0], _mm_load_ps(&bbj[0].lower[0]));
- _mm_store_ps(&bbj[1].upper[0], _mm_load_ps(&bbj[0].upper[0]));
+#ifdef NBNXN_SEARCH_BB_SIMD4
+ gmx_simd4_store_pr(&bbj[1].lower[0], gmx_simd4_load_bb_pr(&bbj[0].lower[0]));
+ gmx_simd4_store_pr(&bbj[1].upper[0], gmx_simd4_load_bb_pr(&bbj[0].upper[0]));
#else
bbj[1] = bbj[0];
#endif
}
-#ifdef NBNXN_SEARCH_BB_SSE
- _mm_store_ps(&bb->lower[0], _mm_min_ps(_mm_load_ps(&bbj[0].lower[0]),
- _mm_load_ps(&bbj[1].lower[0])));
- _mm_store_ps(&bb->upper[0], _mm_max_ps(_mm_load_ps(&bbj[0].upper[0]),
- _mm_load_ps(&bbj[1].upper[0])));
+#ifdef NBNXN_SEARCH_BB_SIMD4
+ gmx_simd4_store_pr(&bb->lower[0],
+ gmx_simd4_min_pr(gmx_simd4_load_bb_pr(&bbj[0].lower[0]),
+ gmx_simd4_load_bb_pr(&bbj[1].lower[0])));
+ gmx_simd4_store_pr(&bb->upper[0],
+ gmx_simd4_max_pr(gmx_simd4_load_bb_pr(&bbj[0].upper[0]),
+ gmx_simd4_load_bb_pr(&bbj[1].upper[0])));
#else
{
int i;
#endif
}
-#ifdef NBNXN_SEARCH_BB_SSE
+#ifdef NBNXN_SEARCH_BB_SIMD4
/* Coordinate order xyz, bb order xxxxyyyyzzzz */
static void calc_bounding_box_xxxx(int na, int stride, const real *x, float *bb)
bb[5*STRIDE_PBB] = R2F_U(zh);
}
-#endif /* NBNXN_SEARCH_BB_SSE */
+#endif /* NBNXN_SEARCH_BB_SIMD4 */
-#ifdef NBNXN_SEARCH_SSE_SINGLE
+#ifdef NBNXN_SEARCH_SIMD4_FLOAT_X_BB
/* Coordinate order xyz?, bb order xyz0 */
-static void calc_bounding_box_sse(int na, const float *x, nbnxn_bb_t *bb)
+static void calc_bounding_box_simd4(int na, const float *x, nbnxn_bb_t *bb)
{
- __m128 bb_0_SSE, bb_1_SSE;
- __m128 x_SSE;
+ gmx_simd4_pr bb_0_S, bb_1_S;
+ gmx_simd4_pr x_S;
int i;
- bb_0_SSE = _mm_load_ps(x);
- bb_1_SSE = bb_0_SSE;
+ bb_0_S = gmx_simd4_load_bb_pr(x);
+ bb_1_S = bb_0_S;
for (i = 1; i < na; i++)
{
- x_SSE = _mm_load_ps(x+i*NNBSBB_C);
- bb_0_SSE = _mm_min_ps(bb_0_SSE, x_SSE);
- bb_1_SSE = _mm_max_ps(bb_1_SSE, x_SSE);
+ x_S = gmx_simd4_load_bb_pr(x+i*NNBSBB_C);
+ bb_0_S = gmx_simd4_min_pr(bb_0_S, x_S);
+ bb_1_S = gmx_simd4_max_pr(bb_1_S, x_S);
}
- _mm_store_ps(&bb->lower[0], bb_0_SSE);
- _mm_store_ps(&bb->upper[0], bb_1_SSE);
+ gmx_simd4_store_pr(&bb->lower[0], bb_0_S);
+ gmx_simd4_store_pr(&bb->upper[0], bb_1_S);
}
/* Coordinate order xyz?, bb order xxxxyyyyzzzz */
-static void calc_bounding_box_xxxx_sse(int na, const float *x,
- nbnxn_bb_t *bb_work_aligned,
- real *bb)
+static void calc_bounding_box_xxxx_simd4(int na, const float *x,
+ nbnxn_bb_t *bb_work_aligned,
+ real *bb)
{
- calc_bounding_box_sse(na, x, bb_work_aligned);
+ calc_bounding_box_simd4(na, x, bb_work_aligned);
bb[0*STRIDE_PBB] = bb_work_aligned->lower[BB_X];
bb[1*STRIDE_PBB] = bb_work_aligned->lower[BB_Y];
bb[5*STRIDE_PBB] = bb_work_aligned->upper[BB_Z];
}
-#endif /* NBNXN_SEARCH_SSE_SINGLE */
+#endif /* NBNXN_SEARCH_SIMD4_FLOAT_X_BB */
/* Combines pairs of consecutive bounding boxes */
nc2 = (grid->cxy_na[i]+3)>>(2+1);
for (c2 = sc2; c2 < sc2+nc2; c2++)
{
-#ifdef NBNXN_SEARCH_BB_SSE
- __m128 min_SSE, max_SSE;
-
- min_SSE = _mm_min_ps(_mm_load_ps(&bb[c2*2+0].lower[0]),
- _mm_load_ps(&bb[c2*2+1].lower[0]));
- max_SSE = _mm_max_ps(_mm_load_ps(&bb[c2*2+0].upper[0]),
- _mm_load_ps(&bb[c2*2+1].upper[0]));
- _mm_store_ps(&grid->bbj[c2].lower[0], min_SSE);
- _mm_store_ps(&grid->bbj[c2].upper[0], max_SSE);
+#ifdef NBNXN_SEARCH_BB_SIMD4
+ gmx_simd4_pr min_S, max_S;
+
+ min_S = gmx_simd4_min_pr(gmx_simd4_load_bb_pr(&bb[c2*2+0].lower[0]),
+ gmx_simd4_load_bb_pr(&bb[c2*2+1].lower[0]));
+ max_S = gmx_simd4_max_pr(gmx_simd4_load_bb_pr(&bb[c2*2+0].upper[0]),
+ gmx_simd4_load_bb_pr(&bb[c2*2+1].upper[0]));
+ gmx_simd4_store_pr(&grid->bbj[c2].lower[0], min_S);
+ gmx_simd4_store_pr(&grid->bbj[c2].upper[0], max_S);
#else
for (j = 0; j < NNBSBB_C; j++)
{
- grid->bbj[c2].lower[j] = min(bb[c2*2].lower[j],
- bb[c2*2].lower[j]);
- grid->bbj[c2].upper[j] = max(bb[c2*2].upper[j],
- bb[c2*2].upper[j]);
+ grid->bbj[c2].lower[j] = min(bb[c2*2+0].lower[j],
+ bb[c2*2+1].lower[j]);
+ grid->bbj[c2].upper[j] = max(bb[c2*2+0].upper[j],
+ bb[c2*2+1].upper[j]);
}
#endif
}
if (((grid->cxy_na[i]+3)>>2) & 1)
{
- /* Copy the last bb for odd bb count in this column */
+ /* The bb count in this column is odd: duplicate the last bb */
for (j = 0; j < NNBSBB_C; j++)
{
grid->bbj[c2].lower[j] = bb[c2*2].lower[j];
else if (!grid->bSimple)
{
/* Store the bounding boxes in a format convenient
- * for SSE calculations: xxxxyyyyzzzz...
+ * for SIMD4 calculations: xxxxyyyyzzzz...
*/
pbb_ptr =
grid->pbb +
((a0-grid->cell0*grid->na_sc)>>(grid->na_c_2log+STRIDE_PBB_2LOG))*NNBSBB_XXXX +
(((a0-grid->cell0*grid->na_sc)>>grid->na_c_2log) & (STRIDE_PBB-1));
-#ifdef NBNXN_SEARCH_SSE_SINGLE
+#ifdef NBNXN_SEARCH_SIMD4_FLOAT_X_BB
if (nbat->XFormat == nbatXYZQ)
{
- calc_bounding_box_xxxx_sse(na, nbat->x+a0*nbat->xstride,
- bb_work_aligned, pbb_ptr);
+ calc_bounding_box_xxxx_simd4(na, nbat->x+a0*nbat->xstride,
+ bb_work_aligned, pbb_ptr);
}
else
#endif
return d2;
}
-#ifdef NBNXN_SEARCH_BB_SSE
+#ifdef NBNXN_SEARCH_BB_SIMD4
-/* SSE code for bb distance for bb format xyz0 */
-static float subc_bb_dist2_sse(int si, const nbnxn_bb_t *bb_i_ci,
- int csj, const nbnxn_bb_t *bb_j_all)
+/* 4-wide SIMD code for bb distance for bb format xyz0 */
+static float subc_bb_dist2_simd4(int si, const nbnxn_bb_t *bb_i_ci,
+ int csj, const nbnxn_bb_t *bb_j_all)
{
- __m128 bb_i_SSE0, bb_i_SSE1;
- __m128 bb_j_SSE0, bb_j_SSE1;
- __m128 dl_SSE;
- __m128 dh_SSE;
- __m128 dm_SSE;
- __m128 dm0_SSE;
- __m128 d2_SSE;
-#ifndef GMX_X86_SSE4_1
- float d2_array[7], *d2_align;
-
- d2_align = (float *)(((size_t)(d2_array+3)) & (~((size_t)15)));
-#else
- float d2;
-#endif
+ gmx_simd4_pr bb_i_S0, bb_i_S1;
+ gmx_simd4_pr bb_j_S0, bb_j_S1;
+ gmx_simd4_pr dl_S;
+ gmx_simd4_pr dh_S;
+ gmx_simd4_pr dm_S;
+ gmx_simd4_pr dm0_S;
- bb_i_SSE0 = _mm_load_ps(&bb_i_ci[si].lower[0]);
- bb_i_SSE1 = _mm_load_ps(&bb_i_ci[si].upper[0]);
- bb_j_SSE0 = _mm_load_ps(&bb_j_all[csj].lower[0]);
- bb_j_SSE1 = _mm_load_ps(&bb_j_all[csj].upper[0]);
+ bb_i_S0 = gmx_simd4_load_bb_pr(&bb_i_ci[si].lower[0]);
+ bb_i_S1 = gmx_simd4_load_bb_pr(&bb_i_ci[si].upper[0]);
+ bb_j_S0 = gmx_simd4_load_bb_pr(&bb_j_all[csj].lower[0]);
+ bb_j_S1 = gmx_simd4_load_bb_pr(&bb_j_all[csj].upper[0]);
- dl_SSE = _mm_sub_ps(bb_i_SSE0, bb_j_SSE1);
- dh_SSE = _mm_sub_ps(bb_j_SSE0, bb_i_SSE1);
+ dl_S = gmx_simd4_sub_pr(bb_i_S0, bb_j_S1);
+ dh_S = gmx_simd4_sub_pr(bb_j_S0, bb_i_S1);
- dm_SSE = _mm_max_ps(dl_SSE, dh_SSE);
- dm0_SSE = _mm_max_ps(dm_SSE, _mm_setzero_ps());
-#ifndef GMX_X86_SSE4_1
- d2_SSE = _mm_mul_ps(dm0_SSE, dm0_SSE);
-
- _mm_store_ps(d2_align, d2_SSE);
-
- return d2_align[0] + d2_align[1] + d2_align[2];
-#else
- /* SSE4.1 dot product of components 0,1,2 */
- d2_SSE = _mm_dp_ps(dm0_SSE, dm0_SSE, 0x71);
+ dm_S = gmx_simd4_max_pr(dl_S, dh_S);
+ dm0_S = gmx_simd4_max_pr(dm_S, gmx_simd4_setzero_pr());
- _mm_store_ss(&d2, d2_SSE);
-
- return d2;
-#endif
+ return gmx_simd4_dotproduct3(dm0_S, dm0_S);
}
/* Calculate bb bounding distances of bb_i[si,...,si+3] and store them in d2 */
-#define SUBC_BB_DIST2_SSE_XXXX_INNER(si, bb_i, d2) \
+#define SUBC_BB_DIST2_SIMD4_XXXX_INNER(si, bb_i, d2) \
{ \
int shi; \
\
- __m128 dx_0, dy_0, dz_0; \
- __m128 dx_1, dy_1, dz_1; \
+ gmx_simd4_pr dx_0, dy_0, dz_0; \
+ gmx_simd4_pr dx_1, dy_1, dz_1; \
\
- __m128 mx, my, mz; \
- __m128 m0x, m0y, m0z; \
+ gmx_simd4_pr mx, my, mz; \
+ gmx_simd4_pr m0x, m0y, m0z; \
\
- __m128 d2x, d2y, d2z; \
- __m128 d2s, d2t; \
+ gmx_simd4_pr d2x, d2y, d2z; \
+ gmx_simd4_pr d2s, d2t; \
\
shi = si*NNBSBB_D*DIM; \
\
- xi_l = _mm_load_ps(bb_i+shi+0*STRIDE_PBB); \
- yi_l = _mm_load_ps(bb_i+shi+1*STRIDE_PBB); \
- zi_l = _mm_load_ps(bb_i+shi+2*STRIDE_PBB); \
- xi_h = _mm_load_ps(bb_i+shi+3*STRIDE_PBB); \
- yi_h = _mm_load_ps(bb_i+shi+4*STRIDE_PBB); \
- zi_h = _mm_load_ps(bb_i+shi+5*STRIDE_PBB); \
+ xi_l = gmx_simd4_load_bb_pr(bb_i+shi+0*STRIDE_PBB); \
+ yi_l = gmx_simd4_load_bb_pr(bb_i+shi+1*STRIDE_PBB); \
+ zi_l = gmx_simd4_load_bb_pr(bb_i+shi+2*STRIDE_PBB); \
+ xi_h = gmx_simd4_load_bb_pr(bb_i+shi+3*STRIDE_PBB); \
+ yi_h = gmx_simd4_load_bb_pr(bb_i+shi+4*STRIDE_PBB); \
+ zi_h = gmx_simd4_load_bb_pr(bb_i+shi+5*STRIDE_PBB); \
\
- dx_0 = _mm_sub_ps(xi_l, xj_h); \
- dy_0 = _mm_sub_ps(yi_l, yj_h); \
- dz_0 = _mm_sub_ps(zi_l, zj_h); \
+ dx_0 = gmx_simd4_sub_pr(xi_l, xj_h); \
+ dy_0 = gmx_simd4_sub_pr(yi_l, yj_h); \
+ dz_0 = gmx_simd4_sub_pr(zi_l, zj_h); \
\
- dx_1 = _mm_sub_ps(xj_l, xi_h); \
- dy_1 = _mm_sub_ps(yj_l, yi_h); \
- dz_1 = _mm_sub_ps(zj_l, zi_h); \
+ dx_1 = gmx_simd4_sub_pr(xj_l, xi_h); \
+ dy_1 = gmx_simd4_sub_pr(yj_l, yi_h); \
+ dz_1 = gmx_simd4_sub_pr(zj_l, zi_h); \
\
- mx = _mm_max_ps(dx_0, dx_1); \
- my = _mm_max_ps(dy_0, dy_1); \
- mz = _mm_max_ps(dz_0, dz_1); \
+ mx = gmx_simd4_max_pr(dx_0, dx_1); \
+ my = gmx_simd4_max_pr(dy_0, dy_1); \
+ mz = gmx_simd4_max_pr(dz_0, dz_1); \
\
- m0x = _mm_max_ps(mx, zero); \
- m0y = _mm_max_ps(my, zero); \
- m0z = _mm_max_ps(mz, zero); \
+ m0x = gmx_simd4_max_pr(mx, zero); \
+ m0y = gmx_simd4_max_pr(my, zero); \
+ m0z = gmx_simd4_max_pr(mz, zero); \
\
- d2x = _mm_mul_ps(m0x, m0x); \
- d2y = _mm_mul_ps(m0y, m0y); \
- d2z = _mm_mul_ps(m0z, m0z); \
+ d2x = gmx_simd4_mul_pr(m0x, m0x); \
+ d2y = gmx_simd4_mul_pr(m0y, m0y); \
+ d2z = gmx_simd4_mul_pr(m0z, m0z); \
\
- d2s = _mm_add_ps(d2x, d2y); \
- d2t = _mm_add_ps(d2s, d2z); \
+ d2s = gmx_simd4_add_pr(d2x, d2y); \
+ d2t = gmx_simd4_add_pr(d2s, d2z); \
\
- _mm_store_ps(d2+si, d2t); \
+ gmx_simd4_store_pr(d2+si, d2t); \
}
-/* SSE code for nsi bb distances for bb format xxxxyyyyzzzz */
-static void subc_bb_dist2_sse_xxxx(const float *bb_j,
- int nsi, const float *bb_i,
- float *d2)
+/* 4-wide SIMD code for nsi bb distances for bb format xxxxyyyyzzzz */
+static void subc_bb_dist2_simd4_xxxx(const float *bb_j,
+ int nsi, const float *bb_i,
+ float *d2)
{
- __m128 xj_l, yj_l, zj_l;
- __m128 xj_h, yj_h, zj_h;
- __m128 xi_l, yi_l, zi_l;
- __m128 xi_h, yi_h, zi_h;
+ gmx_simd4_pr xj_l, yj_l, zj_l;
+ gmx_simd4_pr xj_h, yj_h, zj_h;
+ gmx_simd4_pr xi_l, yi_l, zi_l;
+ gmx_simd4_pr xi_h, yi_h, zi_h;
- __m128 zero;
+ gmx_simd4_pr zero;
- zero = _mm_setzero_ps();
+ zero = gmx_simd4_setzero_pr();
- xj_l = _mm_set1_ps(bb_j[0*STRIDE_PBB]);
- yj_l = _mm_set1_ps(bb_j[1*STRIDE_PBB]);
- zj_l = _mm_set1_ps(bb_j[2*STRIDE_PBB]);
- xj_h = _mm_set1_ps(bb_j[3*STRIDE_PBB]);
- yj_h = _mm_set1_ps(bb_j[4*STRIDE_PBB]);
- zj_h = _mm_set1_ps(bb_j[5*STRIDE_PBB]);
+ xj_l = gmx_simd4_set1_pr(bb_j[0*STRIDE_PBB]);
+ yj_l = gmx_simd4_set1_pr(bb_j[1*STRIDE_PBB]);
+ zj_l = gmx_simd4_set1_pr(bb_j[2*STRIDE_PBB]);
+ xj_h = gmx_simd4_set1_pr(bb_j[3*STRIDE_PBB]);
+ yj_h = gmx_simd4_set1_pr(bb_j[4*STRIDE_PBB]);
+ zj_h = gmx_simd4_set1_pr(bb_j[5*STRIDE_PBB]);
/* Here we "loop" over si (0,STRIDE_PBB) from 0 to nsi with step STRIDE_PBB.
* But as we know the number of iterations is 1 or 2, we unroll manually.
*/
- SUBC_BB_DIST2_SSE_XXXX_INNER(0, bb_i, d2);
+ SUBC_BB_DIST2_SIMD4_XXXX_INNER(0, bb_i, d2);
if (STRIDE_PBB < nsi)
{
- SUBC_BB_DIST2_SSE_XXXX_INNER(STRIDE_PBB, bb_i, d2);
+ SUBC_BB_DIST2_SIMD4_XXXX_INNER(STRIDE_PBB, bb_i, d2);
}
}
-#endif /* NBNXN_SEARCH_BB_SSE */
+#endif /* NBNXN_SEARCH_BB_SIMD4 */
/* Plain C function which determines if any atom pair between two cells
* is within distance sqrt(rl2).
return FALSE;
}
-#ifdef NBNXN_SEARCH_SSE_SINGLE
+#ifdef NBNXN_SEARCH_SIMD4_FLOAT_X_BB
/* When we make seperate single/double precision SIMD vector operation
* include files, this function should be moved there (also using FMA).
*/
-static inline __m128
-gmx_mm_calc_rsq_ps(__m128 x, __m128 y, __m128 z)
+static inline gmx_simd4_pr
+gmx_simd4_calc_rsq_pr(gmx_simd4_pr x, gmx_simd4_pr y, gmx_simd4_pr z)
{
- return _mm_add_ps( _mm_add_ps( _mm_mul_ps(x, x), _mm_mul_ps(y, y) ), _mm_mul_ps(z, z) );
+ return gmx_simd4_add_pr( gmx_simd4_add_pr( gmx_simd4_mul_pr(x, x), gmx_simd4_mul_pr(y, y) ), gmx_simd4_mul_pr(z, z) );
}
#endif
-/* SSE function which determines if any atom pair between two cells,
+/* 4-wide SIMD function which determines if any atom pair between two cells,
* both with 8 atoms, is within distance sqrt(rl2).
- * Not performance critical, so only uses plain SSE.
+ * Using 8-wide AVX is not faster on Intel Sandy Bridge.
*/
-static gmx_bool subc_in_range_sse8(int na_c,
- int si, const real *x_i,
- int csj, int stride, const real *x_j,
- real rl2)
+static gmx_bool subc_in_range_simd4(int na_c,
+ int si, const real *x_i,
+ int csj, int stride, const real *x_j,
+ real rl2)
{
-#ifdef NBNXN_SEARCH_SSE_SINGLE
- __m128 ix_SSE0, iy_SSE0, iz_SSE0;
- __m128 ix_SSE1, iy_SSE1, iz_SSE1;
+#ifdef NBNXN_SEARCH_SIMD4_FLOAT_X_BB
+ gmx_simd4_pr ix_S0, iy_S0, iz_S0;
+ gmx_simd4_pr ix_S1, iy_S1, iz_S1;
- __m128 rc2_SSE;
+ gmx_simd4_pr rc2_S;
- int na_c_sse;
+ int dim_stride;
int j0, j1;
- rc2_SSE = _mm_set1_ps(rl2);
+ rc2_S = gmx_simd4_set1_pr(rl2);
- na_c_sse = NBNXN_GPU_CLUSTER_SIZE/STRIDE_PBB;
- ix_SSE0 = _mm_load_ps(x_i+(si*na_c_sse*DIM+0)*STRIDE_PBB);
- iy_SSE0 = _mm_load_ps(x_i+(si*na_c_sse*DIM+1)*STRIDE_PBB);
- iz_SSE0 = _mm_load_ps(x_i+(si*na_c_sse*DIM+2)*STRIDE_PBB);
- ix_SSE1 = _mm_load_ps(x_i+(si*na_c_sse*DIM+3)*STRIDE_PBB);
- iy_SSE1 = _mm_load_ps(x_i+(si*na_c_sse*DIM+4)*STRIDE_PBB);
- iz_SSE1 = _mm_load_ps(x_i+(si*na_c_sse*DIM+5)*STRIDE_PBB);
+ dim_stride = NBNXN_GPU_CLUSTER_SIZE/STRIDE_PBB*DIM;
+ ix_S0 = gmx_simd4_load_bb_pr(x_i+(si*dim_stride+0)*STRIDE_PBB);
+ iy_S0 = gmx_simd4_load_bb_pr(x_i+(si*dim_stride+1)*STRIDE_PBB);
+ iz_S0 = gmx_simd4_load_bb_pr(x_i+(si*dim_stride+2)*STRIDE_PBB);
+ ix_S1 = gmx_simd4_load_bb_pr(x_i+(si*dim_stride+3)*STRIDE_PBB);
+ iy_S1 = gmx_simd4_load_bb_pr(x_i+(si*dim_stride+4)*STRIDE_PBB);
+ iz_S1 = gmx_simd4_load_bb_pr(x_i+(si*dim_stride+5)*STRIDE_PBB);
/* We loop from the outer to the inner particles to maximize
* the chance that we find a pair in range quickly and return.
j1 = j0 + na_c - 1;
while (j0 < j1)
{
- __m128 jx0_SSE, jy0_SSE, jz0_SSE;
- __m128 jx1_SSE, jy1_SSE, jz1_SSE;
+ gmx_simd4_pr jx0_S, jy0_S, jz0_S;
+ gmx_simd4_pr jx1_S, jy1_S, jz1_S;
- __m128 dx_SSE0, dy_SSE0, dz_SSE0;
- __m128 dx_SSE1, dy_SSE1, dz_SSE1;
- __m128 dx_SSE2, dy_SSE2, dz_SSE2;
- __m128 dx_SSE3, dy_SSE3, dz_SSE3;
+ gmx_simd4_pr dx_S0, dy_S0, dz_S0;
+ gmx_simd4_pr dx_S1, dy_S1, dz_S1;
+ gmx_simd4_pr dx_S2, dy_S2, dz_S2;
+ gmx_simd4_pr dx_S3, dy_S3, dz_S3;
- __m128 rsq_SSE0;
- __m128 rsq_SSE1;
- __m128 rsq_SSE2;
- __m128 rsq_SSE3;
+ gmx_simd4_pr rsq_S0;
+ gmx_simd4_pr rsq_S1;
+ gmx_simd4_pr rsq_S2;
+ gmx_simd4_pr rsq_S3;
- __m128 wco_SSE0;
- __m128 wco_SSE1;
- __m128 wco_SSE2;
- __m128 wco_SSE3;
- __m128 wco_any_SSE01, wco_any_SSE23, wco_any_SSE;
+ gmx_simd4_pb wco_S0;
+ gmx_simd4_pb wco_S1;
+ gmx_simd4_pb wco_S2;
+ gmx_simd4_pb wco_S3;
+ gmx_simd4_pb wco_any_S01, wco_any_S23, wco_any_S;
- jx0_SSE = _mm_load1_ps(x_j+j0*stride+0);
- jy0_SSE = _mm_load1_ps(x_j+j0*stride+1);
- jz0_SSE = _mm_load1_ps(x_j+j0*stride+2);
+ jx0_S = gmx_simd4_set1_pr(x_j[j0*stride+0]);
+ jy0_S = gmx_simd4_set1_pr(x_j[j0*stride+1]);
+ jz0_S = gmx_simd4_set1_pr(x_j[j0*stride+2]);
- jx1_SSE = _mm_load1_ps(x_j+j1*stride+0);
- jy1_SSE = _mm_load1_ps(x_j+j1*stride+1);
- jz1_SSE = _mm_load1_ps(x_j+j1*stride+2);
+ jx1_S = gmx_simd4_set1_pr(x_j[j1*stride+0]);
+ jy1_S = gmx_simd4_set1_pr(x_j[j1*stride+1]);
+ jz1_S = gmx_simd4_set1_pr(x_j[j1*stride+2]);
/* Calculate distance */
- dx_SSE0 = _mm_sub_ps(ix_SSE0, jx0_SSE);
- dy_SSE0 = _mm_sub_ps(iy_SSE0, jy0_SSE);
- dz_SSE0 = _mm_sub_ps(iz_SSE0, jz0_SSE);
- dx_SSE1 = _mm_sub_ps(ix_SSE1, jx0_SSE);
- dy_SSE1 = _mm_sub_ps(iy_SSE1, jy0_SSE);
- dz_SSE1 = _mm_sub_ps(iz_SSE1, jz0_SSE);
- dx_SSE2 = _mm_sub_ps(ix_SSE0, jx1_SSE);
- dy_SSE2 = _mm_sub_ps(iy_SSE0, jy1_SSE);
- dz_SSE2 = _mm_sub_ps(iz_SSE0, jz1_SSE);
- dx_SSE3 = _mm_sub_ps(ix_SSE1, jx1_SSE);
- dy_SSE3 = _mm_sub_ps(iy_SSE1, jy1_SSE);
- dz_SSE3 = _mm_sub_ps(iz_SSE1, jz1_SSE);
+ dx_S0 = gmx_simd4_sub_pr(ix_S0, jx0_S);
+ dy_S0 = gmx_simd4_sub_pr(iy_S0, jy0_S);
+ dz_S0 = gmx_simd4_sub_pr(iz_S0, jz0_S);
+ dx_S1 = gmx_simd4_sub_pr(ix_S1, jx0_S);
+ dy_S1 = gmx_simd4_sub_pr(iy_S1, jy0_S);
+ dz_S1 = gmx_simd4_sub_pr(iz_S1, jz0_S);
+ dx_S2 = gmx_simd4_sub_pr(ix_S0, jx1_S);
+ dy_S2 = gmx_simd4_sub_pr(iy_S0, jy1_S);
+ dz_S2 = gmx_simd4_sub_pr(iz_S0, jz1_S);
+ dx_S3 = gmx_simd4_sub_pr(ix_S1, jx1_S);
+ dy_S3 = gmx_simd4_sub_pr(iy_S1, jy1_S);
+ dz_S3 = gmx_simd4_sub_pr(iz_S1, jz1_S);
/* rsq = dx*dx+dy*dy+dz*dz */
- rsq_SSE0 = gmx_mm_calc_rsq_ps(dx_SSE0, dy_SSE0, dz_SSE0);
- rsq_SSE1 = gmx_mm_calc_rsq_ps(dx_SSE1, dy_SSE1, dz_SSE1);
- rsq_SSE2 = gmx_mm_calc_rsq_ps(dx_SSE2, dy_SSE2, dz_SSE2);
- rsq_SSE3 = gmx_mm_calc_rsq_ps(dx_SSE3, dy_SSE3, dz_SSE3);
+ rsq_S0 = gmx_simd4_calc_rsq_pr(dx_S0, dy_S0, dz_S0);
+ rsq_S1 = gmx_simd4_calc_rsq_pr(dx_S1, dy_S1, dz_S1);
+ rsq_S2 = gmx_simd4_calc_rsq_pr(dx_S2, dy_S2, dz_S2);
+ rsq_S3 = gmx_simd4_calc_rsq_pr(dx_S3, dy_S3, dz_S3);
- wco_SSE0 = _mm_cmplt_ps(rsq_SSE0, rc2_SSE);
- wco_SSE1 = _mm_cmplt_ps(rsq_SSE1, rc2_SSE);
- wco_SSE2 = _mm_cmplt_ps(rsq_SSE2, rc2_SSE);
- wco_SSE3 = _mm_cmplt_ps(rsq_SSE3, rc2_SSE);
+ wco_S0 = gmx_simd4_cmplt_pr(rsq_S0, rc2_S);
+ wco_S1 = gmx_simd4_cmplt_pr(rsq_S1, rc2_S);
+ wco_S2 = gmx_simd4_cmplt_pr(rsq_S2, rc2_S);
+ wco_S3 = gmx_simd4_cmplt_pr(rsq_S3, rc2_S);
- wco_any_SSE01 = _mm_or_ps(wco_SSE0, wco_SSE1);
- wco_any_SSE23 = _mm_or_ps(wco_SSE2, wco_SSE3);
- wco_any_SSE = _mm_or_ps(wco_any_SSE01, wco_any_SSE23);
+ wco_any_S01 = gmx_simd4_or_pb(wco_S0, wco_S1);
+ wco_any_S23 = gmx_simd4_or_pb(wco_S2, wco_S3);
+ wco_any_S = gmx_simd4_or_pb(wco_any_S01, wco_any_S23);
- if (_mm_movemask_ps(wco_any_SSE))
+ if (gmx_simd4_anytrue_pb(wco_any_S))
{
return TRUE;
}
return FALSE;
#else
- /* No SSE */
- gmx_incons("SSE function called without SSE support");
+ /* No SIMD4 */
+ gmx_incons("SIMD4 function called without 4-wide SIMD support");
return TRUE;
#endif
snew(nbl->work, 1);
if (nbl->bSimple)
{
- snew_aligned(nbl->work->bb_ci, 1, NBNXN_MEM_ALIGN);
+ snew_aligned(nbl->work->bb_ci, 1, NBNXN_SEARCH_BB_MEM_ALIGN);
}
else
{
#ifdef NBNXN_BBXXXX
- snew_aligned(nbl->work->pbb_ci, GPU_NSUBCELL/STRIDE_PBB*NNBSBB_XXXX, NBNXN_MEM_ALIGN);
+ snew_aligned(nbl->work->pbb_ci, GPU_NSUBCELL/STRIDE_PBB*NNBSBB_XXXX, NBNXN_SEARCH_BB_MEM_ALIGN);
#else
- snew_aligned(nbl->work->bb_ci, GPU_NSUBCELL, NBNXN_MEM_ALIGN);
+ snew_aligned(nbl->work->bb_ci, GPU_NSUBCELL, NBNXN_SEARCH_BB_MEM_ALIGN);
#endif
}
- snew_aligned(nbl->work->x_ci, NBNXN_NA_SC_MAX*DIM, NBNXN_MEM_ALIGN);
+ snew_aligned(nbl->work->x_ci, NBNXN_NA_SC_MAX*DIM, NBNXN_SEARCH_BB_MEM_ALIGN);
#ifdef GMX_NBNXN_SIMD
snew_aligned(nbl->work->x_ci_simd_4xn, 1, NBNXN_MEM_ALIGN);
snew_aligned(nbl->work->x_ci_simd_2xnn, 1, NBNXN_MEM_ALIGN);
#endif
- snew_aligned(nbl->work->d2, GPU_NSUBCELL, NBNXN_MEM_ALIGN);
+ snew_aligned(nbl->work->d2, GPU_NSUBCELL, NBNXN_SEARCH_BB_MEM_ALIGN);
nbl->work->sort = NULL;
nbl->work->sort_nalloc = 0;
#include "nbnxn_search_simd_2xnn.h"
#endif
-/* Plain C or SSE code for making a pair list of super-cell sci vs scj.
+/* Plain C or SIMD4 code for making a pair list of super-cell sci vs scj.
* Checks bounding box distances and possibly atom pair distances.
*/
static void make_cluster_list_supersub(const nbnxn_grid_t *gridi,
}
#ifdef NBNXN_BBXXXX
- /* Determine all ci1 bb distances in one call with SSE */
- subc_bb_dist2_sse_xxxx(gridj->pbb+(cj>>STRIDE_PBB_2LOG)*NNBSBB_XXXX+(cj & (STRIDE_PBB-1)),
+ /* Determine all ci1 bb distances in one call with SIMD4 */
+ subc_bb_dist2_simd4_xxxx(gridj->pbb+(cj>>STRIDE_PBB_2LOG)*NNBSBB_XXXX+(cj & (STRIDE_PBB-1)),
ci1, pbb_ci, d2l);
*ndistc += na_c*2;
#endif
*ndistc += na_c*na_c;
if (d2 < rbb2 ||
(d2 < rl2 &&
-#ifdef NBNXN_PBB_SSE
- subc_in_range_sse8
+#ifdef NBNXN_PBB_SIMD4
+ subc_in_range_simd4
#else
subc_in_range_x
#endif
{
/* Avoid using function pointers here, as it's slower */
if (
-#ifdef NBNXN_PBB_SSE
- !subc_in_range_sse8
+#ifdef NBNXN_PBB_SIMD4
+ !subc_in_range_simd4
#else
!subc_in_range_x
#endif
ndirect++;
}
}
-#ifdef NBNXN_SEARCH_BB_SSE
+#ifdef NBNXN_SEARCH_BB_SIMD4
else
{
while (cj_ind_first + ndirect <= cj_ind_last &&
inner_e = ge - (se << na_cj_2log);
nbl->cj[found].excl &= ~(1U<<((inner_i<<na_cj_2log) + inner_e));
+/* The next code line is usually not needed. We do not want to version
+ * away the above line, because there is logic that relies on being
+ * able to detect easily whether any exclusions exist. */
+#if (defined GMX_CPU_ACCELERATION_IBM_QPX)
+ nbl->cj[found].interaction_mask_indices[inner_i] &= ~(1U << inner_e);
+#endif
}
}
}
}
}
-#ifdef NBNXN_SEARCH_BB_SSE
+#ifdef NBNXN_SEARCH_BB_SIMD4
/* Copies PBC shifted super-cell packed atom coordinates to working array */
-static void icell_set_x_supersub_sse8(int ci,
- real shx, real shy, real shz,
- int na_c,
- int stride, const real *x,
- nbnxn_list_work_t *work)
+static void icell_set_x_supersub_simd4(int ci,
+ real shx, real shy, real shz,
+ int na_c,
+ int stride, const real *x,
+ nbnxn_list_work_t *work)
{
int si, io, ia, i, j;
real *x_ci;
}
else
{
-#ifdef NBNXN_SEARCH_BB_SSE
- nbs->icell_set_x = icell_set_x_supersub_sse8;
+#ifdef NBNXN_SEARCH_BB_SIMD4
+ nbs->icell_set_x = icell_set_x_supersub_simd4;
#else
nbs->icell_set_x = icell_set_x_supersub;
#endif
extern "C" {
#endif
-
/* Returns the j-cluster size for kernel of type nb_kernel_type */
int nbnxn_kernel_to_cj_size(int nb_kernel_type);
ia = X_IND_CI_SIMD_2XNN(ci);
- x_ci->ix_SSE0 = gmx_set_2real_shift_pr(x + ia + 0*STRIDE_S + 0, shx);
- x_ci->iy_SSE0 = gmx_set_2real_shift_pr(x + ia + 1*STRIDE_S + 0, shy);
- x_ci->iz_SSE0 = gmx_set_2real_shift_pr(x + ia + 2*STRIDE_S + 0, shz);
- x_ci->ix_SSE2 = gmx_set_2real_shift_pr(x + ia + 0*STRIDE_S + 2, shx);
- x_ci->iy_SSE2 = gmx_set_2real_shift_pr(x + ia + 1*STRIDE_S + 2, shy);
- x_ci->iz_SSE2 = gmx_set_2real_shift_pr(x + ia + 2*STRIDE_S + 2, shz);
+ x_ci->ix_S0 = gmx_set_2real_shift_pr(x + ia + 0*STRIDE_S + 0, shx);
+ x_ci->iy_S0 = gmx_set_2real_shift_pr(x + ia + 1*STRIDE_S + 0, shy);
+ x_ci->iz_S0 = gmx_set_2real_shift_pr(x + ia + 2*STRIDE_S + 0, shz);
+ x_ci->ix_S2 = gmx_set_2real_shift_pr(x + ia + 0*STRIDE_S + 2, shx);
+ x_ci->iy_S2 = gmx_set_2real_shift_pr(x + ia + 1*STRIDE_S + 2, shy);
+ x_ci->iz_S2 = gmx_set_2real_shift_pr(x + ia + 2*STRIDE_S + 2, shz);
}
-#ifndef GMX_SIMD_HAVE_ANYTRUE
-/* Fallback function in case gmx_anytrue_pr is not present */
-static gmx_inline gmx_bool
-gmx_anytrue_2xn_pb(gmx_mm_pb bool_S)
-{
- real bools_array[2*GMX_SIMD_WIDTH_HERE], *bools;
- gmx_bool any;
- int s;
-
- bools = gmx_simd_align_real(bools_array);
-
- gmx_store_pb(bools, bool_S);
-
- any = FALSE;
- for (s = 0; s < GMX_SIMD_WIDTH_HERE; s++)
- {
- if (GMX_SIMD_IS_TRUE(s))
- {
- any = TRUE;
- }
- }
-
- return any;
-}
-#endif
-
/* SIMD code for making a pair list of cell ci vs cell cjf-cjl
* for coordinates in packed format.
* Checks bouding box distances and possibly atom pair distances.
const nbnxn_x_ci_simd_2xnn_t *work;
const nbnxn_bb_t *bb_ci;
- gmx_mm_pr jx_SSE, jy_SSE, jz_SSE;
+ gmx_mm_pr jx_S, jy_S, jz_S;
- gmx_mm_pr dx_SSE0, dy_SSE0, dz_SSE0;
- gmx_mm_pr dx_SSE2, dy_SSE2, dz_SSE2;
+ gmx_mm_pr dx_S0, dy_S0, dz_S0;
+ gmx_mm_pr dx_S2, dy_S2, dz_S2;
- gmx_mm_pr rsq_SSE0;
- gmx_mm_pr rsq_SSE2;
+ gmx_mm_pr rsq_S0;
+ gmx_mm_pr rsq_S2;
- gmx_mm_pb wco_SSE0;
- gmx_mm_pb wco_SSE2;
- gmx_mm_pb wco_any_SSE;
+ gmx_mm_pb wco_S0;
+ gmx_mm_pb wco_S2;
+ gmx_mm_pb wco_any_S;
- gmx_mm_pr rc2_SSE;
+ gmx_mm_pr rc2_S;
gmx_bool InRange;
float d2;
bb_ci = nbl->work->bb_ci;
- rc2_SSE = gmx_set1_pr(rl2);
+ rc2_S = gmx_set1_pr(rl2);
InRange = FALSE;
while (!InRange && cjf <= cjl)
{
-#ifdef NBNXN_SEARCH_BB_SSE
- d2 = subc_bb_dist2_sse(0, bb_ci, cjf, gridj->bbj);
+#ifdef NBNXN_SEARCH_BB_SIMD4
+ d2 = subc_bb_dist2_simd4(0, bb_ci, cjf, gridj->bbj);
#else
d2 = subc_bb_dist2(0, bb_ci, cjf, gridj->bbj);
#endif
{
xind_f = X_IND_CJ_SIMD_2XNN(CI_TO_CJ_SIMD_2XNN(gridj->cell0) + cjf);
- jx_SSE = gmx_load_hpr_hilo_pr(x_j+xind_f+0*STRIDE_S);
- jy_SSE = gmx_load_hpr_hilo_pr(x_j+xind_f+1*STRIDE_S);
- jz_SSE = gmx_load_hpr_hilo_pr(x_j+xind_f+2*STRIDE_S);
+ jx_S = gmx_load_hpr_hilo_pr(x_j+xind_f+0*STRIDE_S);
+ jy_S = gmx_load_hpr_hilo_pr(x_j+xind_f+1*STRIDE_S);
+ jz_S = gmx_load_hpr_hilo_pr(x_j+xind_f+2*STRIDE_S);
/* Calculate distance */
- dx_SSE0 = gmx_sub_pr(work->ix_SSE0, jx_SSE);
- dy_SSE0 = gmx_sub_pr(work->iy_SSE0, jy_SSE);
- dz_SSE0 = gmx_sub_pr(work->iz_SSE0, jz_SSE);
- dx_SSE2 = gmx_sub_pr(work->ix_SSE2, jx_SSE);
- dy_SSE2 = gmx_sub_pr(work->iy_SSE2, jy_SSE);
- dz_SSE2 = gmx_sub_pr(work->iz_SSE2, jz_SSE);
+ dx_S0 = gmx_sub_pr(work->ix_S0, jx_S);
+ dy_S0 = gmx_sub_pr(work->iy_S0, jy_S);
+ dz_S0 = gmx_sub_pr(work->iz_S0, jz_S);
+ dx_S2 = gmx_sub_pr(work->ix_S2, jx_S);
+ dy_S2 = gmx_sub_pr(work->iy_S2, jy_S);
+ dz_S2 = gmx_sub_pr(work->iz_S2, jz_S);
/* rsq = dx*dx+dy*dy+dz*dz */
- rsq_SSE0 = gmx_calc_rsq_pr(dx_SSE0, dy_SSE0, dz_SSE0);
- rsq_SSE2 = gmx_calc_rsq_pr(dx_SSE2, dy_SSE2, dz_SSE2);
+ rsq_S0 = gmx_calc_rsq_pr(dx_S0, dy_S0, dz_S0);
+ rsq_S2 = gmx_calc_rsq_pr(dx_S2, dy_S2, dz_S2);
- wco_SSE0 = gmx_cmplt_pr(rsq_SSE0, rc2_SSE);
- wco_SSE2 = gmx_cmplt_pr(rsq_SSE2, rc2_SSE);
+ wco_S0 = gmx_cmplt_pr(rsq_S0, rc2_S);
+ wco_S2 = gmx_cmplt_pr(rsq_S2, rc2_S);
- wco_any_SSE = gmx_or_pb(wco_SSE0, wco_SSE2);
+ wco_any_S = gmx_or_pb(wco_S0, wco_S2);
-#ifdef GMX_SIMD_HAVE_ANYTRUE
- InRange = gmx_anytrue_pb(wco_any_SSE);
-#else
- InRange = gmx_anytrue_2xn_pb(wco_any_SSE);
-#endif
+ InRange = gmx_anytrue_pb(wco_any_S);
*ndistc += 2*GMX_SIMD_WIDTH_HERE;
}
InRange = FALSE;
while (!InRange && cjl > cjf)
{
-#ifdef NBNXN_SEARCH_BB_SSE
- d2 = subc_bb_dist2_sse(0, bb_ci, cjl, gridj->bbj);
+#ifdef NBNXN_SEARCH_BB_SIMD4
+ d2 = subc_bb_dist2_simd4(0, bb_ci, cjl, gridj->bbj);
#else
d2 = subc_bb_dist2(0, bb_ci, cjl, gridj->bbj);
#endif
{
xind_l = X_IND_CJ_SIMD_2XNN(CI_TO_CJ_SIMD_2XNN(gridj->cell0) + cjl);
- jx_SSE = gmx_load_hpr_hilo_pr(x_j+xind_l+0*STRIDE_S);
- jy_SSE = gmx_load_hpr_hilo_pr(x_j+xind_l+1*STRIDE_S);
- jz_SSE = gmx_load_hpr_hilo_pr(x_j+xind_l+2*STRIDE_S);
+ jx_S = gmx_load_hpr_hilo_pr(x_j+xind_l+0*STRIDE_S);
+ jy_S = gmx_load_hpr_hilo_pr(x_j+xind_l+1*STRIDE_S);
+ jz_S = gmx_load_hpr_hilo_pr(x_j+xind_l+2*STRIDE_S);
/* Calculate distance */
- dx_SSE0 = gmx_sub_pr(work->ix_SSE0, jx_SSE);
- dy_SSE0 = gmx_sub_pr(work->iy_SSE0, jy_SSE);
- dz_SSE0 = gmx_sub_pr(work->iz_SSE0, jz_SSE);
- dx_SSE2 = gmx_sub_pr(work->ix_SSE2, jx_SSE);
- dy_SSE2 = gmx_sub_pr(work->iy_SSE2, jy_SSE);
- dz_SSE2 = gmx_sub_pr(work->iz_SSE2, jz_SSE);
+ dx_S0 = gmx_sub_pr(work->ix_S0, jx_S);
+ dy_S0 = gmx_sub_pr(work->iy_S0, jy_S);
+ dz_S0 = gmx_sub_pr(work->iz_S0, jz_S);
+ dx_S2 = gmx_sub_pr(work->ix_S2, jx_S);
+ dy_S2 = gmx_sub_pr(work->iy_S2, jy_S);
+ dz_S2 = gmx_sub_pr(work->iz_S2, jz_S);
/* rsq = dx*dx+dy*dy+dz*dz */
- rsq_SSE0 = gmx_calc_rsq_pr(dx_SSE0, dy_SSE0, dz_SSE0);
- rsq_SSE2 = gmx_calc_rsq_pr(dx_SSE2, dy_SSE2, dz_SSE2);
+ rsq_S0 = gmx_calc_rsq_pr(dx_S0, dy_S0, dz_S0);
+ rsq_S2 = gmx_calc_rsq_pr(dx_S2, dy_S2, dz_S2);
- wco_SSE0 = gmx_cmplt_pr(rsq_SSE0, rc2_SSE);
- wco_SSE2 = gmx_cmplt_pr(rsq_SSE2, rc2_SSE);
+ wco_S0 = gmx_cmplt_pr(rsq_S0, rc2_S);
+ wco_S2 = gmx_cmplt_pr(rsq_S2, rc2_S);
- wco_any_SSE = gmx_or_pb(wco_SSE0, wco_SSE2);
+ wco_any_S = gmx_or_pb(wco_S0, wco_S2);
-#ifdef GMX_SIMD_HAVE_ANYTRUE
- InRange = gmx_anytrue_pb(wco_any_SSE);
-#else
- InRange = gmx_anytrue_2xn_pb(wco_any_SSE);
-#endif
+ InRange = gmx_anytrue_pb(wco_any_S);
*ndistc += 2*GMX_SIMD_WIDTH_HERE;
}
ia = X_IND_CI_SIMD_4XN(ci);
- x_ci->ix_SSE0 = gmx_set1_pr(x[ia + 0*STRIDE_S ] + shx);
- x_ci->iy_SSE0 = gmx_set1_pr(x[ia + 1*STRIDE_S ] + shy);
- x_ci->iz_SSE0 = gmx_set1_pr(x[ia + 2*STRIDE_S ] + shz);
- x_ci->ix_SSE1 = gmx_set1_pr(x[ia + 0*STRIDE_S + 1] + shx);
- x_ci->iy_SSE1 = gmx_set1_pr(x[ia + 1*STRIDE_S + 1] + shy);
- x_ci->iz_SSE1 = gmx_set1_pr(x[ia + 2*STRIDE_S + 1] + shz);
- x_ci->ix_SSE2 = gmx_set1_pr(x[ia + 0*STRIDE_S + 2] + shx);
- x_ci->iy_SSE2 = gmx_set1_pr(x[ia + 1*STRIDE_S + 2] + shy);
- x_ci->iz_SSE2 = gmx_set1_pr(x[ia + 2*STRIDE_S + 2] + shz);
- x_ci->ix_SSE3 = gmx_set1_pr(x[ia + 0*STRIDE_S + 3] + shx);
- x_ci->iy_SSE3 = gmx_set1_pr(x[ia + 1*STRIDE_S + 3] + shy);
- x_ci->iz_SSE3 = gmx_set1_pr(x[ia + 2*STRIDE_S + 3] + shz);
+ x_ci->ix_S0 = gmx_set1_pr(x[ia + 0*STRIDE_S ] + shx);
+ x_ci->iy_S0 = gmx_set1_pr(x[ia + 1*STRIDE_S ] + shy);
+ x_ci->iz_S0 = gmx_set1_pr(x[ia + 2*STRIDE_S ] + shz);
+ x_ci->ix_S1 = gmx_set1_pr(x[ia + 0*STRIDE_S + 1] + shx);
+ x_ci->iy_S1 = gmx_set1_pr(x[ia + 1*STRIDE_S + 1] + shy);
+ x_ci->iz_S1 = gmx_set1_pr(x[ia + 2*STRIDE_S + 1] + shz);
+ x_ci->ix_S2 = gmx_set1_pr(x[ia + 0*STRIDE_S + 2] + shx);
+ x_ci->iy_S2 = gmx_set1_pr(x[ia + 1*STRIDE_S + 2] + shy);
+ x_ci->iz_S2 = gmx_set1_pr(x[ia + 2*STRIDE_S + 2] + shz);
+ x_ci->ix_S3 = gmx_set1_pr(x[ia + 0*STRIDE_S + 3] + shx);
+ x_ci->iy_S3 = gmx_set1_pr(x[ia + 1*STRIDE_S + 3] + shy);
+ x_ci->iz_S3 = gmx_set1_pr(x[ia + 2*STRIDE_S + 3] + shz);
}
-#ifndef GMX_SIMD_HAVE_ANYTRUE
-/* Fallback function in case gmx_anytrue_pr is not present */
-static gmx_inline gmx_bool
-gmx_anytrue_4xn_pb(gmx_mm_pb bool_S)
-{
- real bools_array[2*GMX_SIMD_WIDTH_HERE], *bools;
- gmx_bool any;
- int s;
-
- bools = gmx_simd_align_real(bools_array);
-
- gmx_store_pb(bools, bool_S);
-
- any = FALSE;
- for (s = 0; s < GMX_SIMD_WIDTH_HERE; s++)
- {
- if (GMX_SIMD_IS_TRUE(bools[s]))
- {
- any = TRUE;
- }
- }
-
- return any;
-}
-#endif
-
/* SIMD code for making a pair list of cell ci vs cell cjf-cjl
* for coordinates in packed format.
* Checks bouding box distances and possibly atom pair distances.
const nbnxn_x_ci_simd_4xn_t *work;
const nbnxn_bb_t *bb_ci;
- gmx_mm_pr jx_SSE, jy_SSE, jz_SSE;
+ gmx_mm_pr jx_S, jy_S, jz_S;
- gmx_mm_pr dx_SSE0, dy_SSE0, dz_SSE0;
- gmx_mm_pr dx_SSE1, dy_SSE1, dz_SSE1;
- gmx_mm_pr dx_SSE2, dy_SSE2, dz_SSE2;
- gmx_mm_pr dx_SSE3, dy_SSE3, dz_SSE3;
+ gmx_mm_pr dx_S0, dy_S0, dz_S0;
+ gmx_mm_pr dx_S1, dy_S1, dz_S1;
+ gmx_mm_pr dx_S2, dy_S2, dz_S2;
+ gmx_mm_pr dx_S3, dy_S3, dz_S3;
- gmx_mm_pr rsq_SSE0;
- gmx_mm_pr rsq_SSE1;
- gmx_mm_pr rsq_SSE2;
- gmx_mm_pr rsq_SSE3;
+ gmx_mm_pr rsq_S0;
+ gmx_mm_pr rsq_S1;
+ gmx_mm_pr rsq_S2;
+ gmx_mm_pr rsq_S3;
- gmx_mm_pb wco_SSE0;
- gmx_mm_pb wco_SSE1;
- gmx_mm_pb wco_SSE2;
- gmx_mm_pb wco_SSE3;
- gmx_mm_pb wco_any_SSE01, wco_any_SSE23, wco_any_SSE;
+ gmx_mm_pb wco_S0;
+ gmx_mm_pb wco_S1;
+ gmx_mm_pb wco_S2;
+ gmx_mm_pb wco_S3;
+ gmx_mm_pb wco_any_S01, wco_any_S23, wco_any_S;
- gmx_mm_pr rc2_SSE;
+ gmx_mm_pr rc2_S;
gmx_bool InRange;
float d2;
bb_ci = nbl->work->bb_ci;
- rc2_SSE = gmx_set1_pr(rl2);
+ rc2_S = gmx_set1_pr(rl2);
InRange = FALSE;
while (!InRange && cjf <= cjl)
{
-#ifdef NBNXN_SEARCH_BB_SSE
- d2 = subc_bb_dist2_sse(0, bb_ci, cjf, gridj->bbj);
+#ifdef NBNXN_SEARCH_BB_SIMD4
+ d2 = subc_bb_dist2_simd4(0, bb_ci, cjf, gridj->bbj);
#else
d2 = subc_bb_dist2(0, bb_ci, cjf, gridj->bbj);
#endif
{
xind_f = X_IND_CJ_SIMD_4XN(CI_TO_CJ_SIMD_4XN(gridj->cell0) + cjf);
- jx_SSE = gmx_load_pr(x_j+xind_f+0*STRIDE_S);
- jy_SSE = gmx_load_pr(x_j+xind_f+1*STRIDE_S);
- jz_SSE = gmx_load_pr(x_j+xind_f+2*STRIDE_S);
+ jx_S = gmx_load_pr(x_j+xind_f+0*STRIDE_S);
+ jy_S = gmx_load_pr(x_j+xind_f+1*STRIDE_S);
+ jz_S = gmx_load_pr(x_j+xind_f+2*STRIDE_S);
/* Calculate distance */
- dx_SSE0 = gmx_sub_pr(work->ix_SSE0, jx_SSE);
- dy_SSE0 = gmx_sub_pr(work->iy_SSE0, jy_SSE);
- dz_SSE0 = gmx_sub_pr(work->iz_SSE0, jz_SSE);
- dx_SSE1 = gmx_sub_pr(work->ix_SSE1, jx_SSE);
- dy_SSE1 = gmx_sub_pr(work->iy_SSE1, jy_SSE);
- dz_SSE1 = gmx_sub_pr(work->iz_SSE1, jz_SSE);
- dx_SSE2 = gmx_sub_pr(work->ix_SSE2, jx_SSE);
- dy_SSE2 = gmx_sub_pr(work->iy_SSE2, jy_SSE);
- dz_SSE2 = gmx_sub_pr(work->iz_SSE2, jz_SSE);
- dx_SSE3 = gmx_sub_pr(work->ix_SSE3, jx_SSE);
- dy_SSE3 = gmx_sub_pr(work->iy_SSE3, jy_SSE);
- dz_SSE3 = gmx_sub_pr(work->iz_SSE3, jz_SSE);
+ dx_S0 = gmx_sub_pr(work->ix_S0, jx_S);
+ dy_S0 = gmx_sub_pr(work->iy_S0, jy_S);
+ dz_S0 = gmx_sub_pr(work->iz_S0, jz_S);
+ dx_S1 = gmx_sub_pr(work->ix_S1, jx_S);
+ dy_S1 = gmx_sub_pr(work->iy_S1, jy_S);
+ dz_S1 = gmx_sub_pr(work->iz_S1, jz_S);
+ dx_S2 = gmx_sub_pr(work->ix_S2, jx_S);
+ dy_S2 = gmx_sub_pr(work->iy_S2, jy_S);
+ dz_S2 = gmx_sub_pr(work->iz_S2, jz_S);
+ dx_S3 = gmx_sub_pr(work->ix_S3, jx_S);
+ dy_S3 = gmx_sub_pr(work->iy_S3, jy_S);
+ dz_S3 = gmx_sub_pr(work->iz_S3, jz_S);
/* rsq = dx*dx+dy*dy+dz*dz */
- rsq_SSE0 = gmx_calc_rsq_pr(dx_SSE0, dy_SSE0, dz_SSE0);
- rsq_SSE1 = gmx_calc_rsq_pr(dx_SSE1, dy_SSE1, dz_SSE1);
- rsq_SSE2 = gmx_calc_rsq_pr(dx_SSE2, dy_SSE2, dz_SSE2);
- rsq_SSE3 = gmx_calc_rsq_pr(dx_SSE3, dy_SSE3, dz_SSE3);
-
- wco_SSE0 = gmx_cmplt_pr(rsq_SSE0, rc2_SSE);
- wco_SSE1 = gmx_cmplt_pr(rsq_SSE1, rc2_SSE);
- wco_SSE2 = gmx_cmplt_pr(rsq_SSE2, rc2_SSE);
- wco_SSE3 = gmx_cmplt_pr(rsq_SSE3, rc2_SSE);
-
- wco_any_SSE01 = gmx_or_pb(wco_SSE0, wco_SSE1);
- wco_any_SSE23 = gmx_or_pb(wco_SSE2, wco_SSE3);
- wco_any_SSE = gmx_or_pb(wco_any_SSE01, wco_any_SSE23);
-
-#ifdef GMX_SIMD_HAVE_ANYTRUE
- InRange = gmx_anytrue_pb(wco_any_SSE);
-#else
- InRange = gmx_anytrue_4xn_pb(wco_any_SSE);
-#endif
+ rsq_S0 = gmx_calc_rsq_pr(dx_S0, dy_S0, dz_S0);
+ rsq_S1 = gmx_calc_rsq_pr(dx_S1, dy_S1, dz_S1);
+ rsq_S2 = gmx_calc_rsq_pr(dx_S2, dy_S2, dz_S2);
+ rsq_S3 = gmx_calc_rsq_pr(dx_S3, dy_S3, dz_S3);
+
+ wco_S0 = gmx_cmplt_pr(rsq_S0, rc2_S);
+ wco_S1 = gmx_cmplt_pr(rsq_S1, rc2_S);
+ wco_S2 = gmx_cmplt_pr(rsq_S2, rc2_S);
+ wco_S3 = gmx_cmplt_pr(rsq_S3, rc2_S);
+
+ wco_any_S01 = gmx_or_pb(wco_S0, wco_S1);
+ wco_any_S23 = gmx_or_pb(wco_S2, wco_S3);
+ wco_any_S = gmx_or_pb(wco_any_S01, wco_any_S23);
+
+ InRange = gmx_anytrue_pb(wco_any_S);
*ndistc += 4*GMX_SIMD_WIDTH_HERE;
}
InRange = FALSE;
while (!InRange && cjl > cjf)
{
-#ifdef NBNXN_SEARCH_BB_SSE
- d2 = subc_bb_dist2_sse(0, bb_ci, cjl, gridj->bbj);
+#ifdef NBNXN_SEARCH_BB_SIMD4
+ d2 = subc_bb_dist2_simd4(0, bb_ci, cjl, gridj->bbj);
#else
d2 = subc_bb_dist2(0, bb_ci, cjl, gridj->bbj);
#endif
{
xind_l = X_IND_CJ_SIMD_4XN(CI_TO_CJ_SIMD_4XN(gridj->cell0) + cjl);
- jx_SSE = gmx_load_pr(x_j+xind_l+0*STRIDE_S);
- jy_SSE = gmx_load_pr(x_j+xind_l+1*STRIDE_S);
- jz_SSE = gmx_load_pr(x_j+xind_l+2*STRIDE_S);
+ jx_S = gmx_load_pr(x_j+xind_l+0*STRIDE_S);
+ jy_S = gmx_load_pr(x_j+xind_l+1*STRIDE_S);
+ jz_S = gmx_load_pr(x_j+xind_l+2*STRIDE_S);
/* Calculate distance */
- dx_SSE0 = gmx_sub_pr(work->ix_SSE0, jx_SSE);
- dy_SSE0 = gmx_sub_pr(work->iy_SSE0, jy_SSE);
- dz_SSE0 = gmx_sub_pr(work->iz_SSE0, jz_SSE);
- dx_SSE1 = gmx_sub_pr(work->ix_SSE1, jx_SSE);
- dy_SSE1 = gmx_sub_pr(work->iy_SSE1, jy_SSE);
- dz_SSE1 = gmx_sub_pr(work->iz_SSE1, jz_SSE);
- dx_SSE2 = gmx_sub_pr(work->ix_SSE2, jx_SSE);
- dy_SSE2 = gmx_sub_pr(work->iy_SSE2, jy_SSE);
- dz_SSE2 = gmx_sub_pr(work->iz_SSE2, jz_SSE);
- dx_SSE3 = gmx_sub_pr(work->ix_SSE3, jx_SSE);
- dy_SSE3 = gmx_sub_pr(work->iy_SSE3, jy_SSE);
- dz_SSE3 = gmx_sub_pr(work->iz_SSE3, jz_SSE);
+ dx_S0 = gmx_sub_pr(work->ix_S0, jx_S);
+ dy_S0 = gmx_sub_pr(work->iy_S0, jy_S);
+ dz_S0 = gmx_sub_pr(work->iz_S0, jz_S);
+ dx_S1 = gmx_sub_pr(work->ix_S1, jx_S);
+ dy_S1 = gmx_sub_pr(work->iy_S1, jy_S);
+ dz_S1 = gmx_sub_pr(work->iz_S1, jz_S);
+ dx_S2 = gmx_sub_pr(work->ix_S2, jx_S);
+ dy_S2 = gmx_sub_pr(work->iy_S2, jy_S);
+ dz_S2 = gmx_sub_pr(work->iz_S2, jz_S);
+ dx_S3 = gmx_sub_pr(work->ix_S3, jx_S);
+ dy_S3 = gmx_sub_pr(work->iy_S3, jy_S);
+ dz_S3 = gmx_sub_pr(work->iz_S3, jz_S);
/* rsq = dx*dx+dy*dy+dz*dz */
- rsq_SSE0 = gmx_calc_rsq_pr(dx_SSE0, dy_SSE0, dz_SSE0);
- rsq_SSE1 = gmx_calc_rsq_pr(dx_SSE1, dy_SSE1, dz_SSE1);
- rsq_SSE2 = gmx_calc_rsq_pr(dx_SSE2, dy_SSE2, dz_SSE2);
- rsq_SSE3 = gmx_calc_rsq_pr(dx_SSE3, dy_SSE3, dz_SSE3);
-
- wco_SSE0 = gmx_cmplt_pr(rsq_SSE0, rc2_SSE);
- wco_SSE1 = gmx_cmplt_pr(rsq_SSE1, rc2_SSE);
- wco_SSE2 = gmx_cmplt_pr(rsq_SSE2, rc2_SSE);
- wco_SSE3 = gmx_cmplt_pr(rsq_SSE3, rc2_SSE);
-
- wco_any_SSE01 = gmx_or_pb(wco_SSE0, wco_SSE1);
- wco_any_SSE23 = gmx_or_pb(wco_SSE2, wco_SSE3);
- wco_any_SSE = gmx_or_pb(wco_any_SSE01, wco_any_SSE23);
-
-#ifdef GMX_SIMD_HAVE_ANYTRUE
- InRange = gmx_anytrue_pb(wco_any_SSE);
-#else
- InRange = gmx_anytrue_4xn_pb(wco_any_SSE);
-#endif
+ rsq_S0 = gmx_calc_rsq_pr(dx_S0, dy_S0, dz_S0);
+ rsq_S1 = gmx_calc_rsq_pr(dx_S1, dy_S1, dz_S1);
+ rsq_S2 = gmx_calc_rsq_pr(dx_S2, dy_S2, dz_S2);
+ rsq_S3 = gmx_calc_rsq_pr(dx_S3, dy_S3, dz_S3);
+
+ wco_S0 = gmx_cmplt_pr(rsq_S0, rc2_S);
+ wco_S1 = gmx_cmplt_pr(rsq_S1, rc2_S);
+ wco_S2 = gmx_cmplt_pr(rsq_S2, rc2_S);
+ wco_S3 = gmx_cmplt_pr(rsq_S3, rc2_S);
+
+ wco_any_S01 = gmx_or_pb(wco_S0, wco_S1);
+ wco_any_S23 = gmx_or_pb(wco_S2, wco_S3);
+ wco_any_S = gmx_or_pb(wco_any_S01, wco_any_S23);
+
+ InRange = gmx_anytrue_pb(wco_any_S);
*ndistc += 4*GMX_SIMD_WIDTH_HERE;
}
/* Store cj and the interaction mask */
nbl->cj[nbl->ncj].cj = CI_TO_CJ_SIMD_4XN(gridj->cell0) + cj;
nbl->cj[nbl->ncj].excl = get_imask_simd_4xn(remove_sub_diag, ci, cj);
+#ifdef GMX_CPU_ACCELERATION_IBM_QPX
+ nbl->cj[nbl->ncj].interaction_mask_indices[0] = (nbl->cj[nbl->ncj].excl & 0x000F) >> (0 * 4);
+ nbl->cj[nbl->ncj].interaction_mask_indices[1] = (nbl->cj[nbl->ncj].excl & 0x00F0) >> (1 * 4);
+ nbl->cj[nbl->ncj].interaction_mask_indices[2] = (nbl->cj[nbl->ncj].excl & 0x0F00) >> (2 * 4);
+ nbl->cj[nbl->ncj].interaction_mask_indices[3] = (nbl->cj[nbl->ncj].excl & 0xF000) >> (3 * 4);
+#endif
nbl->ncj++;
}
/* Increase the closing index in i super-cell list */
}
#undef STRIDE_S
-
nlist->gid[nri] = gid;
nlist->shift[nri] = shift;
}
+ else
+ {
+ /* Adding to previous list. First remove possible previous padding */
+ if(nlist->simd_padding_width>1)
+ {
+ while(nlist->nrj>0 && nlist->jjnr[nlist->nrj-1]<0)
+ {
+ nlist->nrj--;
+ }
+ }
+ }
}
static inline void close_i_nblist(t_nblist *nlist)
/* Include the SIMD macro file and then check for support */
#include "gmx_simd_macros.h"
#if defined GMX_HAVE_SIMD_MACROS && defined GMX_SIMD_HAVE_EXP
-/* Turn on SIMD intrinsics for PME solve */
+/* Turn on arbitrary width SIMD intrinsics for PME solve */
#define PME_SIMD
#endif
-/* SIMD spread+gather only in single precision with SSE2 or higher available.
- * We might want to switch to use gmx_simd_macros.h, but this is somewhat
- * complicated, as we use unaligned and/or 4-wide only loads.
+/* Include the 4-wide SIMD macro file */
+#include "gmx_simd4_macros.h"
+/* Check if we have 4-wide SIMD macro support */
+#ifdef GMX_HAVE_SIMD4_MACROS
+/* Do PME spread and gather with 4-wide SIMD.
+ * NOTE: SIMD is only used with PME order 4 and 5 (which are the most common).
*/
-#if defined(GMX_X86_SSE2) && !defined(GMX_DOUBLE)
-#define PME_SSE_SPREAD_GATHER
-#include <emmintrin.h>
-/* Some old AMD processors could have problems with unaligned loads+stores */
-#ifndef GMX_FAHCORE
-#define PME_SSE_UNALIGNED
+#define PME_SIMD4_SPREAD_GATHER
+
+#ifdef GMX_SIMD4_HAVE_UNALIGNED
+/* With PME-order=4 on x86, unaligned load+store is slightly faster
+ * than doubling all SIMD operations when using aligned load+store.
+ */
+#define PME_SIMD4_UNALIGNED
#endif
#endif
#define mpi_type MPI_FLOAT
#endif
-/* GMX_CACHE_SEP should be a multiple of 16 to preserve alignment */
+#ifdef PME_SIMD4_SPREAD_GATHER
+#define SIMD4_ALIGNMENT (GMX_SIMD4_WIDTH*sizeof(real))
+#else
+/* We can use any alignment, apart from 0, so we use 4 reals */
+#define SIMD4_ALIGNMENT (4*sizeof(real))
+#endif
+
+/* GMX_CACHE_SEP should be a multiple of the SIMD and SIMD4 register size
+ * to preserve alignment.
+ */
#define GMX_CACHE_SEP 64
/* We only define a maximum to be able to use local arrays without allocation.
typedef struct {
-#ifdef PME_SSE_SPREAD_GATHER
- /* Masks for SSE aligned spreading and gathering */
- __m128 mask_SSE0[6], mask_SSE1[6];
+#ifdef PME_SIMD4_SPREAD_GATHER
+ /* Masks for 4-wide SIMD aligned spreading and gathering */
+ gmx_simd4_pb mask_S0[6], mask_S1[6];
#else
int dummy; /* C89 requires that struct has at least one member */
#endif
srenew(th[XX], nalloc);
srenew(th[YY], nalloc);
- /* In z we add padding, this is only required for the aligned SSE code */
- srenew(*ptr_z, nalloc+2*padding);
+ /* In z we add padding, this is only required for the aligned SIMD code */
+ sfree_aligned(*ptr_z);
+ snew_aligned(*ptr_z, nalloc+2*padding, SIMD4_ALIGNMENT);
th[ZZ] = *ptr_z + padding;
for (i = 0; i < padding; i++)
int pnx, pny, pnz, ndatatot;
int offx, offy, offz;
+#if defined PME_SIMD4_SPREAD_GATHER && !defined PME_SIMD4_UNALIGNED
+ real thz_buffer[12], *thz_aligned;
+
+ thz_aligned = gmx_simd4_align_real(thz_buffer);
+#endif
+
pnx = pmegrid->s[XX];
pny = pmegrid->s[YY];
pnz = pmegrid->s[ZZ];
switch (order)
{
case 4:
-#ifdef PME_SSE_SPREAD_GATHER
-#ifdef PME_SSE_UNALIGNED
-#define PME_SPREAD_SSE_ORDER4
+#ifdef PME_SIMD4_SPREAD_GATHER
+#ifdef PME_SIMD4_UNALIGNED
+#define PME_SPREAD_SIMD4_ORDER4
#else
-#define PME_SPREAD_SSE_ALIGNED
+#define PME_SPREAD_SIMD4_ALIGNED
#define PME_ORDER 4
#endif
-#include "pme_sse_single.h"
+#include "pme_simd4.h"
#else
DO_BSPLINE(4);
#endif
break;
case 5:
-#ifdef PME_SSE_SPREAD_GATHER
-#define PME_SPREAD_SSE_ALIGNED
+#ifdef PME_SIMD4_SPREAD_GATHER
+#define PME_SPREAD_SIMD4_ALIGNED
#define PME_ORDER 5
-#include "pme_sse_single.h"
+#include "pme_simd4.h"
#else
DO_BSPLINE(5);
#endif
static void set_grid_alignment(int *pmegrid_nz, int pme_order)
{
-#ifdef PME_SSE_SPREAD_GATHER
+#ifdef PME_SIMD4_SPREAD_GATHER
if (pme_order == 5
-#ifndef PME_SSE_UNALIGNED
+#ifndef PME_SIMD4_UNALIGNED
|| pme_order == 4
#endif
)
static void set_gridsize_alignment(int gmx_unused *gridsize, int gmx_unused pme_order)
{
-#ifdef PME_SSE_SPREAD_GATHER
-#ifndef PME_SSE_UNALIGNED
+#ifdef PME_SIMD4_SPREAD_GATHER
+#ifndef PME_SIMD4_UNALIGNED
if (pme_order == 4)
{
/* Add extra elements to ensured aligned operations do not go
{
gridsize = grid->s[XX]*grid->s[YY]*grid->s[ZZ];
set_gridsize_alignment(&gridsize, pme_order);
- snew_aligned(grid->grid, gridsize, 16);
+ snew_aligned(grid->grid, gridsize, SIMD4_ALIGNMENT);
}
else
{
set_gridsize_alignment(&gridsize, pme_order);
snew_aligned(grids->grid_all,
grids->nthread*gridsize+(grids->nthread+1)*GMX_CACHE_SEP,
- 16);
+ SIMD4_ALIGNMENT);
for (x = 0; x < grids->nc[XX]; x++)
{
static void realloc_work(pme_work_t *work, int nkx)
{
+ int simd_width;
+
if (nkx > work->nalloc)
{
work->nalloc = nkx;
* elements at the end for padding.
*/
#ifdef PME_SIMD
-#define ALIGN_HERE GMX_SIMD_WIDTH_HERE
+ simd_width = GMX_SIMD_WIDTH_HERE;
#else
-/* We can use any alignment, apart from 0, so we use 4 */
-#define ALIGN_HERE 4
+ /* We can use any alignment, apart from 0, so we use 4 */
+ simd_width = 4;
#endif
sfree_aligned(work->denom);
sfree_aligned(work->tmp1);
sfree_aligned(work->eterm);
- snew_aligned(work->denom, work->nalloc+ALIGN_HERE, ALIGN_HERE*sizeof(real));
- snew_aligned(work->tmp1, work->nalloc+ALIGN_HERE, ALIGN_HERE*sizeof(real));
- snew_aligned(work->eterm, work->nalloc+ALIGN_HERE, ALIGN_HERE*sizeof(real));
+ snew_aligned(work->denom, work->nalloc+simd_width, simd_width*sizeof(real));
+ snew_aligned(work->tmp1, work->nalloc+simd_width, simd_width*sizeof(real));
+ snew_aligned(work->eterm, work->nalloc+simd_width, simd_width*sizeof(real));
srenew(work->m2inv, work->nalloc);
}
}
gmx_mm_pr lu;
gmx_mm_pr tmp_d1, d_inv, tmp_r, tmp_e;
int kx;
- f_simd = gmx_load1_pr(&f);
+ f_simd = gmx_set1_pr(f);
for (kx = 0; kx < end; kx += GMX_SIMD_WIDTH_HERE)
{
tmp_d1 = gmx_load_pr(d_aligned+kx);
pme_spline_work_t *work;
+#if defined PME_SIMD4_SPREAD_GATHER && !defined PME_SIMD4_UNALIGNED
+ real thz_buffer[12], *thz_aligned;
+ real dthz_buffer[12], *dthz_aligned;
+
+ thz_aligned = gmx_simd4_align_real(thz_buffer);
+ dthz_aligned = gmx_simd4_align_real(dthz_buffer);
+#endif
+
work = pme->spline_work;
order = pme->pme_order;
switch (order)
{
case 4:
-#ifdef PME_SSE_SPREAD_GATHER
-#ifdef PME_SSE_UNALIGNED
-#define PME_GATHER_F_SSE_ORDER4
+#ifdef PME_SIMD4_SPREAD_GATHER
+#ifdef PME_SIMD4_UNALIGNED
+#define PME_GATHER_F_SIMD4_ORDER4
#else
-#define PME_GATHER_F_SSE_ALIGNED
+#define PME_GATHER_F_SIMD4_ALIGNED
#define PME_ORDER 4
#endif
-#include "pme_sse_single.h"
+#include "pme_simd4.h"
#else
DO_FSPLINE(4);
#endif
break;
case 5:
-#ifdef PME_SSE_SPREAD_GATHER
-#define PME_GATHER_F_SSE_ALIGNED
+#ifdef PME_SIMD4_SPREAD_GATHER
+#define PME_GATHER_F_SIMD4_ALIGNED
#define PME_ORDER 5
-#include "pme_sse_single.h"
+#include "pme_simd4.h"
#else
DO_FSPLINE(5);
#endif
{
pme_spline_work_t *work;
-#ifdef PME_SSE_SPREAD_GATHER
- float tmp[8];
- __m128 zero_SSE;
- int of, i;
+#ifdef PME_SIMD4_SPREAD_GATHER
+ real tmp[12], *tmp_aligned;
+ gmx_simd4_pr zero_S;
+ gmx_simd4_pr real_mask_S0, real_mask_S1;
+ int of, i;
+
+ snew_aligned(work, 1, SIMD4_ALIGNMENT);
- snew_aligned(work, 1, 16);
+ tmp_aligned = gmx_simd4_align_real(tmp);
- zero_SSE = _mm_setzero_ps();
+ zero_S = gmx_simd4_setzero_pr();
/* Generate bit masks to mask out the unused grid entries,
* as we only operate on order of the 8 grid entries that are
- * load into 2 SSE float registers.
+ * load into 2 SIMD registers.
*/
for (of = 0; of < 8-(order-1); of++)
{
for (i = 0; i < 8; i++)
{
- tmp[i] = (i >= of && i < of+order ? 1 : 0);
+ tmp_aligned[i] = (i >= of && i < of+order ? -1.0 : 1.0);
}
- work->mask_SSE0[of] = _mm_loadu_ps(tmp);
- work->mask_SSE1[of] = _mm_loadu_ps(tmp+4);
- work->mask_SSE0[of] = _mm_cmpgt_ps(work->mask_SSE0[of], zero_SSE);
- work->mask_SSE1[of] = _mm_cmpgt_ps(work->mask_SSE1[of], zero_SSE);
+ real_mask_S0 = gmx_simd4_load_pr(tmp_aligned);
+ real_mask_S1 = gmx_simd4_load_pr(tmp_aligned+4);
+ work->mask_S0[of] = gmx_simd4_cmplt_pr(real_mask_S0, zero_S);
+ work->mask_S1[of] = gmx_simd4_cmplt_pr(real_mask_S1, zero_S);
}
#else
work = NULL;
--- /dev/null
+/*
+ * This file is part of the GROMACS molecular simulation package.
+ *
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2004, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+ * Copyright (c) 2012,2013, by the GROMACS development team, led by
+ * David van der Spoel, Berk Hess, Erik Lindahl, and including many
+ * others, as listed in the AUTHORS file in the top-level source
+ * directory and at http://www.gromacs.org.
+ *
+ * GROMACS is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ *
+ * GROMACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with GROMACS; if not, see
+ * http://www.gnu.org/licenses, or write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * If you want to redistribute modifications to GROMACS, please
+ * consider that scientific software is very special. Version
+ * control is crucial - bugs must be traceable. We will be happy to
+ * consider code for inclusion in the official distribution, but
+ * derived work must not be called official GROMACS. Details are found
+ * in the README & COPYING files - if they are missing, get the
+ * official version at http://www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the research papers on the package. Check out http://www.gromacs.org.
+ */
+
+/* This include file has code between ifdef's to make sure
+ * that this performance sensitive code is inlined
+ * and to remove conditionals and variable loop bounds at compile time.
+ */
+
+#ifdef PME_SPREAD_SIMD4_ORDER4
+/* Spread one charge with pme_order=4 with unaligned SIMD4 load+store.
+ * This code does not assume any memory alignment for the grid.
+ */
+{
+ gmx_simd4_pr ty_S0, ty_S1, ty_S2, ty_S3;
+ gmx_simd4_pr tz_S;
+ gmx_simd4_pr vx_S;
+ gmx_simd4_pr vx_tz_S;
+ gmx_simd4_pr sum_S0, sum_S1, sum_S2, sum_S3;
+ gmx_simd4_pr gri_S0, gri_S1, gri_S2, gri_S3;
+
+ ty_S0 = gmx_simd4_set1_pr(thy[0]);
+ ty_S1 = gmx_simd4_set1_pr(thy[1]);
+ ty_S2 = gmx_simd4_set1_pr(thy[2]);
+ ty_S3 = gmx_simd4_set1_pr(thy[3]);
+
+ /* With order 4 the z-spline is actually aligned */
+ tz_S = gmx_simd4_load_pr(thz);
+
+ for (ithx = 0; (ithx < 4); ithx++)
+ {
+ index_x = (i0+ithx)*pny*pnz;
+ valx = qn*thx[ithx];
+
+ vx_S = gmx_simd4_set1_pr(valx);
+
+ vx_tz_S = gmx_simd4_mul_pr(vx_S, tz_S);
+
+ gri_S0 = gmx_simd4_loadu_pr(grid+index_x+(j0+0)*pnz+k0);
+ gri_S1 = gmx_simd4_loadu_pr(grid+index_x+(j0+1)*pnz+k0);
+ gri_S2 = gmx_simd4_loadu_pr(grid+index_x+(j0+2)*pnz+k0);
+ gri_S3 = gmx_simd4_loadu_pr(grid+index_x+(j0+3)*pnz+k0);
+
+ sum_S0 = gmx_simd4_madd_pr(vx_tz_S, ty_S0, gri_S0);
+ sum_S1 = gmx_simd4_madd_pr(vx_tz_S, ty_S1, gri_S1);
+ sum_S2 = gmx_simd4_madd_pr(vx_tz_S, ty_S2, gri_S2);
+ sum_S3 = gmx_simd4_madd_pr(vx_tz_S, ty_S3, gri_S3);
+
+ gmx_simd4_storeu_pr(grid+index_x+(j0+0)*pnz+k0, sum_S0);
+ gmx_simd4_storeu_pr(grid+index_x+(j0+1)*pnz+k0, sum_S1);
+ gmx_simd4_storeu_pr(grid+index_x+(j0+2)*pnz+k0, sum_S2);
+ gmx_simd4_storeu_pr(grid+index_x+(j0+3)*pnz+k0, sum_S3);
+ }
+}
+#undef PME_SPREAD_SIMD4_ORDER4
+#endif
+
+
+#ifdef PME_GATHER_F_SIMD4_ORDER4
+/* Gather for one charge with pme_order=4 with unaligned SIMD4 load+store.
+ * This code does not assume any memory alignment for the grid.
+ */
+{
+ real fx_tmp[4], fy_tmp[4], fz_tmp[4];
+
+ gmx_simd4_pr fx_S, fy_S, fz_S;
+
+ gmx_simd4_pr tx_S, ty_S, tz_S;
+ gmx_simd4_pr dx_S, dy_S, dz_S;
+
+ gmx_simd4_pr gval_S;
+
+ gmx_simd4_pr fxy1_S;
+ gmx_simd4_pr fz1_S;
+
+ fx_S = gmx_simd4_setzero_pr();
+ fy_S = gmx_simd4_setzero_pr();
+ fz_S = gmx_simd4_setzero_pr();
+
+ /* With order 4 the z-spline is actually aligned */
+ tz_S = gmx_simd4_load_pr(thz);
+ dz_S = gmx_simd4_load_pr(dthz);
+
+ for (ithx = 0; (ithx < 4); ithx++)
+ {
+ index_x = (i0+ithx)*pny*pnz;
+ tx_S = gmx_simd4_set1_pr(thx[ithx]);
+ dx_S = gmx_simd4_set1_pr(dthx[ithx]);
+
+ for (ithy = 0; (ithy < 4); ithy++)
+ {
+ index_xy = index_x+(j0+ithy)*pnz;
+ ty_S = gmx_simd4_set1_pr(thy[ithy]);
+ dy_S = gmx_simd4_set1_pr(dthy[ithy]);
+
+ gval_S = gmx_simd4_loadu_pr(grid+index_xy+k0);
+
+ fxy1_S = gmx_simd4_mul_pr(tz_S, gval_S);
+ fz1_S = gmx_simd4_mul_pr(dz_S, gval_S);
+
+ fx_S = gmx_simd4_madd_pr(gmx_simd4_mul_pr(dx_S, ty_S), fxy1_S, fx_S);
+ fy_S = gmx_simd4_madd_pr(gmx_simd4_mul_pr(tx_S, dy_S), fxy1_S, fy_S);
+ fz_S = gmx_simd4_madd_pr(gmx_simd4_mul_pr(tx_S, ty_S), fz1_S, fz_S);
+ }
+ }
+
+ gmx_simd4_storeu_pr(fx_tmp, fx_S);
+ gmx_simd4_storeu_pr(fy_tmp, fy_S);
+ gmx_simd4_storeu_pr(fz_tmp, fz_S);
+
+ fx += fx_tmp[0]+fx_tmp[1]+fx_tmp[2]+fx_tmp[3];
+ fy += fy_tmp[0]+fy_tmp[1]+fy_tmp[2]+fy_tmp[3];
+ fz += fz_tmp[0]+fz_tmp[1]+fz_tmp[2]+fz_tmp[3];
+}
+#undef PME_GATHER_F_SIMD4_ORDER4
+#endif
+
+
+#ifdef PME_SPREAD_SIMD4_ALIGNED
+/* This code assumes that the grid is allocated 4-real aligned
+ * and that pnz is a multiple of 4.
+ * This code supports pme_order <= 5.
+ */
+{
+ int offset;
+ int index;
+ gmx_simd4_pr ty_S0, ty_S1, ty_S2, ty_S3, ty_S4;
+ gmx_simd4_pr tz_S0;
+ gmx_simd4_pr tz_S1;
+ gmx_simd4_pr vx_S;
+ gmx_simd4_pr vx_tz_S0;
+ gmx_simd4_pr vx_tz_S1;
+ gmx_simd4_pr sum_S00, sum_S01, sum_S02, sum_S03, sum_S04;
+ gmx_simd4_pr sum_S10, sum_S11, sum_S12, sum_S13, sum_S14;
+ gmx_simd4_pr gri_S00, gri_S01, gri_S02, gri_S03, gri_S04;
+ gmx_simd4_pr gri_S10, gri_S11, gri_S12, gri_S13, gri_S14;
+
+ offset = k0 & 3;
+
+ ty_S0 = gmx_simd4_set1_pr(thy[0]);
+ ty_S1 = gmx_simd4_set1_pr(thy[1]);
+ ty_S2 = gmx_simd4_set1_pr(thy[2]);
+ ty_S3 = gmx_simd4_set1_pr(thy[3]);
+#if PME_ORDER == 5
+ ty_S4 = gmx_simd4_set1_pr(thy[4]);
+#endif
+
+#ifdef GMX_SIMD4_HAVE_UNALIGNED
+ tz_S0 = gmx_simd4_loadu_pr(thz-offset);
+ tz_S1 = gmx_simd4_loadu_pr(thz-offset+4);
+#else
+ {
+ int i;
+ /* Copy thz to an aligned buffer (unused buffer parts are masked) */
+ for (i = 0; i < PME_ORDER; i++)
+ {
+ thz_aligned[offset+i] = thz[i];
+ }
+ tz_S0 = gmx_simd4_load_pr(thz_aligned);
+ tz_S1 = gmx_simd4_load_pr(thz_aligned+4);
+ }
+#endif
+ tz_S0 = gmx_simd4_blendzero_pr(tz_S0, work->mask_S0[offset]);
+ tz_S1 = gmx_simd4_blendzero_pr(tz_S1, work->mask_S1[offset]);
+
+ for (ithx = 0; (ithx < PME_ORDER); ithx++)
+ {
+ index = (i0+ithx)*pny*pnz + j0*pnz + k0 - offset;
+ valx = qn*thx[ithx];
+
+ vx_S = gmx_simd4_set1_pr(valx);
+
+ vx_tz_S0 = gmx_simd4_mul_pr(vx_S, tz_S0);
+ vx_tz_S1 = gmx_simd4_mul_pr(vx_S, tz_S1);
+
+ gri_S00 = gmx_simd4_load_pr(grid+index+0*pnz);
+ gri_S01 = gmx_simd4_load_pr(grid+index+1*pnz);
+ gri_S02 = gmx_simd4_load_pr(grid+index+2*pnz);
+ gri_S03 = gmx_simd4_load_pr(grid+index+3*pnz);
+#if PME_ORDER == 5
+ gri_S04 = gmx_simd4_load_pr(grid+index+4*pnz);
+#endif
+ gri_S10 = gmx_simd4_load_pr(grid+index+0*pnz+4);
+ gri_S11 = gmx_simd4_load_pr(grid+index+1*pnz+4);
+ gri_S12 = gmx_simd4_load_pr(grid+index+2*pnz+4);
+ gri_S13 = gmx_simd4_load_pr(grid+index+3*pnz+4);
+#if PME_ORDER == 5
+ gri_S14 = gmx_simd4_load_pr(grid+index+4*pnz+4);
+#endif
+
+ sum_S00 = gmx_simd4_madd_pr(vx_tz_S0, ty_S0, gri_S00);
+ sum_S01 = gmx_simd4_madd_pr(vx_tz_S0, ty_S1, gri_S01);
+ sum_S02 = gmx_simd4_madd_pr(vx_tz_S0, ty_S2, gri_S02);
+ sum_S03 = gmx_simd4_madd_pr(vx_tz_S0, ty_S3, gri_S03);
+#if PME_ORDER == 5
+ sum_S04 = gmx_simd4_madd_pr(vx_tz_S0, ty_S4, gri_S04);
+#endif
+ sum_S10 = gmx_simd4_madd_pr(vx_tz_S1, ty_S0, gri_S10);
+ sum_S11 = gmx_simd4_madd_pr(vx_tz_S1, ty_S1, gri_S11);
+ sum_S12 = gmx_simd4_madd_pr(vx_tz_S1, ty_S2, gri_S12);
+ sum_S13 = gmx_simd4_madd_pr(vx_tz_S1, ty_S3, gri_S13);
+#if PME_ORDER == 5
+ sum_S14 = gmx_simd4_madd_pr(vx_tz_S1, ty_S4, gri_S14);
+#endif
+
+ gmx_simd4_store_pr(grid+index+0*pnz, sum_S00);
+ gmx_simd4_store_pr(grid+index+1*pnz, sum_S01);
+ gmx_simd4_store_pr(grid+index+2*pnz, sum_S02);
+ gmx_simd4_store_pr(grid+index+3*pnz, sum_S03);
+#if PME_ORDER == 5
+ gmx_simd4_store_pr(grid+index+4*pnz, sum_S04);
+#endif
+ gmx_simd4_store_pr(grid+index+0*pnz+4, sum_S10);
+ gmx_simd4_store_pr(grid+index+1*pnz+4, sum_S11);
+ gmx_simd4_store_pr(grid+index+2*pnz+4, sum_S12);
+ gmx_simd4_store_pr(grid+index+3*pnz+4, sum_S13);
+#if PME_ORDER == 5
+ gmx_simd4_store_pr(grid+index+4*pnz+4, sum_S14);
+#endif
+ }
+}
+#undef PME_ORDER
+#undef PME_SPREAD_SIMD4_ALIGNED
+#endif
+
+
+#ifdef PME_GATHER_F_SIMD4_ALIGNED
+/* This code assumes that the grid is allocated 4-real aligned
+ * and that pnz is a multiple of 4.
+ * This code supports pme_order <= 5.
+ */
+{
+ int offset;
+
+ real fx_tmp[4], fy_tmp[4], fz_tmp[4];
+
+ gmx_simd4_pr fx_S, fy_S, fz_S;
+
+ gmx_simd4_pr tx_S, ty_S, tz_S0, tz_S1;
+ gmx_simd4_pr dx_S, dy_S, dz_S0, dz_S1;
+
+ gmx_simd4_pr gval_S0;
+ gmx_simd4_pr gval_S1;
+
+ gmx_simd4_pr fxy1_S0;
+ gmx_simd4_pr fz1_S0;
+ gmx_simd4_pr fxy1_S1;
+ gmx_simd4_pr fz1_S1;
+ gmx_simd4_pr fxy1_S;
+ gmx_simd4_pr fz1_S;
+
+ offset = k0 & 3;
+
+ fx_S = gmx_simd4_setzero_pr();
+ fy_S = gmx_simd4_setzero_pr();
+ fz_S = gmx_simd4_setzero_pr();
+
+#ifdef GMX_SIMD4_HAVE_UNALIGNED
+ tz_S0 = gmx_simd4_loadu_pr(thz-offset);
+ tz_S1 = gmx_simd4_loadu_pr(thz-offset+4);
+ dz_S0 = gmx_simd4_loadu_pr(dthz-offset);
+ dz_S1 = gmx_simd4_loadu_pr(dthz-offset+4);
+#else
+ {
+ int i;
+ /* Copy (d)thz to an aligned buffer (unused buffer parts are masked) */
+ for (i = 0; i < PME_ORDER; i++)
+ {
+ thz_aligned[offset+i] = thz[i];
+ dthz_aligned[offset+i] = dthz[i];
+ }
+ tz_S0 = gmx_simd4_load_pr(thz_aligned);
+ tz_S1 = gmx_simd4_load_pr(thz_aligned+4);
+ dz_S0 = gmx_simd4_load_pr(dthz_aligned);
+ dz_S1 = gmx_simd4_load_pr(dthz_aligned+4);
+ }
+#endif
+ tz_S0 = gmx_simd4_blendzero_pr(tz_S0, work->mask_S0[offset]);
+ dz_S0 = gmx_simd4_blendzero_pr(dz_S0, work->mask_S0[offset]);
+ tz_S1 = gmx_simd4_blendzero_pr(tz_S1, work->mask_S1[offset]);
+ dz_S1 = gmx_simd4_blendzero_pr(dz_S1, work->mask_S1[offset]);
+
+ for (ithx = 0; (ithx < PME_ORDER); ithx++)
+ {
+ index_x = (i0+ithx)*pny*pnz;
+ tx_S = gmx_simd4_set1_pr(thx[ithx]);
+ dx_S = gmx_simd4_set1_pr(dthx[ithx]);
+
+ for (ithy = 0; (ithy < PME_ORDER); ithy++)
+ {
+ index_xy = index_x+(j0+ithy)*pnz;
+ ty_S = gmx_simd4_set1_pr(thy[ithy]);
+ dy_S = gmx_simd4_set1_pr(dthy[ithy]);
+
+ gval_S0 = gmx_simd4_load_pr(grid+index_xy+k0-offset);
+ gval_S1 = gmx_simd4_load_pr(grid+index_xy+k0-offset+4);
+
+ fxy1_S0 = gmx_simd4_mul_pr(tz_S0, gval_S0);
+ fz1_S0 = gmx_simd4_mul_pr(dz_S0, gval_S0);
+ fxy1_S1 = gmx_simd4_mul_pr(tz_S1, gval_S1);
+ fz1_S1 = gmx_simd4_mul_pr(dz_S1, gval_S1);
+
+ fxy1_S = gmx_simd4_add_pr(fxy1_S0, fxy1_S1);
+ fz1_S = gmx_simd4_add_pr(fz1_S0, fz1_S1);
+
+ fx_S = gmx_simd4_madd_pr(gmx_simd4_mul_pr(dx_S, ty_S), fxy1_S, fx_S);
+ fy_S = gmx_simd4_madd_pr(gmx_simd4_mul_pr(tx_S, dy_S), fxy1_S, fy_S);
+ fz_S = gmx_simd4_madd_pr(gmx_simd4_mul_pr(tx_S, ty_S), fz1_S, fz_S);
+ }
+ }
+
+ gmx_simd4_store_pr(fx_tmp, fx_S);
+ gmx_simd4_store_pr(fy_tmp, fy_S);
+ gmx_simd4_store_pr(fz_tmp, fz_S);
+
+ fx += fx_tmp[0]+fx_tmp[1]+fx_tmp[2]+fx_tmp[3];
+ fy += fy_tmp[0]+fy_tmp[1]+fy_tmp[2]+fy_tmp[3];
+ fz += fz_tmp[0]+fz_tmp[1]+fz_tmp[2]+fz_tmp[3];
+}
+#undef PME_ORDER
+#undef PME_GATHER_F_SIMD4_ALIGNED
+#endif
+++ /dev/null
-/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
- *
- *
- * This source code is part of
- *
- * G R O M A C S
- *
- * GROningen MAchine for Chemical Simulations
- *
- * VERSION 4.5
- * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
- * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
- * Copyright (c) 2001-2004, The GROMACS development team,
- * check out http://www.gromacs.org for more information.
-
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
- * of the License, or (at your option) any later version.
- *
- * If you want to redistribute modifications, please consider that
- * scientific software is very special. Version control is crucial -
- * bugs must be traceable. We will be happy to consider code for
- * inclusion in the official distribution, but derived work must not
- * be called official GROMACS. Details are found in the README & COPYING
- * files - if they are missing, get the official version at www.gromacs.org.
- *
- * To help us fund GROMACS development, we humbly ask that you cite
- * the papers on the package - you can find them in the top README file.
- *
- * For more info, check our website at http://www.gromacs.org
- *
- * And Hey:
- * GROwing Monsters And Cloning Shrimps
- */
-
-/* This include file has code between ifdef's to make sure
- * that this performance sensitive code is inlined
- * and to remove conditionals and variable loop bounds at compile time.
- */
-
-#ifdef PME_SPREAD_SSE_ORDER4
-/* This code does not assume any memory alignment.
- * This code only works for pme_order = 4.
- */
-{
- __m128 ty_SSE0, ty_SSE1, ty_SSE2, ty_SSE3;
- __m128 tz_SSE;
- __m128 vx_SSE;
- __m128 vx_tz_SSE;
- __m128 sum_SSE0, sum_SSE1, sum_SSE2, sum_SSE3;
- __m128 gri_SSE0, gri_SSE1, gri_SSE2, gri_SSE3;
-
- ty_SSE0 = _mm_load1_ps(&thy[0]);
- ty_SSE1 = _mm_load1_ps(&thy[1]);
- ty_SSE2 = _mm_load1_ps(&thy[2]);
- ty_SSE3 = _mm_load1_ps(&thy[3]);
-
- tz_SSE = _mm_loadu_ps(thz);
-
- for (ithx = 0; (ithx < 4); ithx++)
- {
- index_x = (i0+ithx)*pny*pnz;
- valx = qn*thx[ithx];
-
- vx_SSE = _mm_load1_ps(&valx);
-
- vx_tz_SSE = _mm_mul_ps(vx_SSE, tz_SSE);
-
- gri_SSE0 = _mm_loadu_ps(grid+index_x+(j0+0)*pnz+k0);
- gri_SSE1 = _mm_loadu_ps(grid+index_x+(j0+1)*pnz+k0);
- gri_SSE2 = _mm_loadu_ps(grid+index_x+(j0+2)*pnz+k0);
- gri_SSE3 = _mm_loadu_ps(grid+index_x+(j0+3)*pnz+k0);
-
- sum_SSE0 = _mm_add_ps(gri_SSE0, _mm_mul_ps(vx_tz_SSE, ty_SSE0));
- sum_SSE1 = _mm_add_ps(gri_SSE1, _mm_mul_ps(vx_tz_SSE, ty_SSE1));
- sum_SSE2 = _mm_add_ps(gri_SSE2, _mm_mul_ps(vx_tz_SSE, ty_SSE2));
- sum_SSE3 = _mm_add_ps(gri_SSE3, _mm_mul_ps(vx_tz_SSE, ty_SSE3));
-
- _mm_storeu_ps(grid+index_x+(j0+0)*pnz+k0, sum_SSE0);
- _mm_storeu_ps(grid+index_x+(j0+1)*pnz+k0, sum_SSE1);
- _mm_storeu_ps(grid+index_x+(j0+2)*pnz+k0, sum_SSE2);
- _mm_storeu_ps(grid+index_x+(j0+3)*pnz+k0, sum_SSE3);
- }
-}
-#undef PME_SPREAD_SSE_ORDER4
-#endif
-
-
-#ifdef PME_GATHER_F_SSE_ORDER4
-/* This code does not assume any memory alignment.
- * This code only works for pme_order = 4.
- */
-{
- float fx_tmp[4], fy_tmp[4], fz_tmp[4];
-
- __m128 fx_SSE, fy_SSE, fz_SSE;
-
- __m128 tx_SSE, ty_SSE, tz_SSE;
- __m128 dx_SSE, dy_SSE, dz_SSE;
-
- __m128 gval_SSE;
-
- __m128 fxy1_SSE;
- __m128 fz1_SSE;
-
- fx_SSE = _mm_setzero_ps();
- fy_SSE = _mm_setzero_ps();
- fz_SSE = _mm_setzero_ps();
-
- tz_SSE = _mm_loadu_ps(thz);
- dz_SSE = _mm_loadu_ps(dthz);
-
- for (ithx = 0; (ithx < 4); ithx++)
- {
- index_x = (i0+ithx)*pny*pnz;
- tx_SSE = _mm_load1_ps(thx+ithx);
- dx_SSE = _mm_load1_ps(dthx+ithx);
-
- for (ithy = 0; (ithy < 4); ithy++)
- {
- index_xy = index_x+(j0+ithy)*pnz;
- ty_SSE = _mm_load1_ps(thy+ithy);
- dy_SSE = _mm_load1_ps(dthy+ithy);
-
- gval_SSE = _mm_loadu_ps(grid+index_xy+k0);
-
- fxy1_SSE = _mm_mul_ps(tz_SSE, gval_SSE);
- fz1_SSE = _mm_mul_ps(dz_SSE, gval_SSE);
-
- fx_SSE = _mm_add_ps(fx_SSE, _mm_mul_ps(_mm_mul_ps(dx_SSE, ty_SSE), fxy1_SSE));
- fy_SSE = _mm_add_ps(fy_SSE, _mm_mul_ps(_mm_mul_ps(tx_SSE, dy_SSE), fxy1_SSE));
- fz_SSE = _mm_add_ps(fz_SSE, _mm_mul_ps(_mm_mul_ps(tx_SSE, ty_SSE), fz1_SSE));
- }
- }
-
- _mm_storeu_ps(fx_tmp, fx_SSE);
- _mm_storeu_ps(fy_tmp, fy_SSE);
- _mm_storeu_ps(fz_tmp, fz_SSE);
-
- fx += fx_tmp[0]+fx_tmp[1]+fx_tmp[2]+fx_tmp[3];
- fy += fy_tmp[0]+fy_tmp[1]+fy_tmp[2]+fy_tmp[3];
- fz += fz_tmp[0]+fz_tmp[1]+fz_tmp[2]+fz_tmp[3];
-}
-#undef PME_GATHER_F_SSE_ORDER4
-#endif
-
-
-#ifdef PME_SPREAD_SSE_ALIGNED
-/* This code assumes that the grid is allocated 16 bit aligned
- * and that pnz is a multiple of 4.
- * This code supports pme_order <= 5.
- */
-{
- int offset;
- int index;
- __m128 ty_SSE0, ty_SSE1, ty_SSE2, ty_SSE3, ty_SSE4;
- __m128 tz_SSE0;
- __m128 tz_SSE1;
- __m128 vx_SSE;
- __m128 vx_tz_SSE0;
- __m128 vx_tz_SSE1;
- __m128 sum_SSE00, sum_SSE01, sum_SSE02, sum_SSE03, sum_SSE04;
- __m128 sum_SSE10, sum_SSE11, sum_SSE12, sum_SSE13, sum_SSE14;
- __m128 gri_SSE00, gri_SSE01, gri_SSE02, gri_SSE03, gri_SSE04;
- __m128 gri_SSE10, gri_SSE11, gri_SSE12, gri_SSE13, gri_SSE14;
-
- offset = k0 & 3;
-
- ty_SSE0 = _mm_load1_ps(&thy[0]);
- ty_SSE1 = _mm_load1_ps(&thy[1]);
- ty_SSE2 = _mm_load1_ps(&thy[2]);
- ty_SSE3 = _mm_load1_ps(&thy[3]);
-#if PME_ORDER == 5
- ty_SSE4 = _mm_load1_ps(&thy[4]);
-#endif
-
- tz_SSE0 = _mm_loadu_ps(thz-offset);
- tz_SSE1 = _mm_loadu_ps(thz-offset+4);
- tz_SSE0 = _mm_and_ps(tz_SSE0, work->mask_SSE0[offset]);
- tz_SSE1 = _mm_and_ps(tz_SSE1, work->mask_SSE1[offset]);
-
- for (ithx = 0; (ithx < PME_ORDER); ithx++)
- {
- index = (i0+ithx)*pny*pnz + j0*pnz + k0 - offset;
- valx = qn*thx[ithx];
-
- vx_SSE = _mm_load1_ps(&valx);
-
- vx_tz_SSE0 = _mm_mul_ps(vx_SSE, tz_SSE0);
- vx_tz_SSE1 = _mm_mul_ps(vx_SSE, tz_SSE1);
-
- gri_SSE00 = _mm_load_ps(grid+index+0*pnz);
- gri_SSE01 = _mm_load_ps(grid+index+1*pnz);
- gri_SSE02 = _mm_load_ps(grid+index+2*pnz);
- gri_SSE03 = _mm_load_ps(grid+index+3*pnz);
-#if PME_ORDER == 5
- gri_SSE04 = _mm_load_ps(grid+index+4*pnz);
-#endif
- gri_SSE10 = _mm_load_ps(grid+index+0*pnz+4);
- gri_SSE11 = _mm_load_ps(grid+index+1*pnz+4);
- gri_SSE12 = _mm_load_ps(grid+index+2*pnz+4);
- gri_SSE13 = _mm_load_ps(grid+index+3*pnz+4);
-#if PME_ORDER == 5
- gri_SSE14 = _mm_load_ps(grid+index+4*pnz+4);
-#endif
-
- sum_SSE00 = _mm_add_ps(gri_SSE00, _mm_mul_ps(vx_tz_SSE0, ty_SSE0));
- sum_SSE01 = _mm_add_ps(gri_SSE01, _mm_mul_ps(vx_tz_SSE0, ty_SSE1));
- sum_SSE02 = _mm_add_ps(gri_SSE02, _mm_mul_ps(vx_tz_SSE0, ty_SSE2));
- sum_SSE03 = _mm_add_ps(gri_SSE03, _mm_mul_ps(vx_tz_SSE0, ty_SSE3));
-#if PME_ORDER == 5
- sum_SSE04 = _mm_add_ps(gri_SSE04, _mm_mul_ps(vx_tz_SSE0, ty_SSE4));
-#endif
- sum_SSE10 = _mm_add_ps(gri_SSE10, _mm_mul_ps(vx_tz_SSE1, ty_SSE0));
- sum_SSE11 = _mm_add_ps(gri_SSE11, _mm_mul_ps(vx_tz_SSE1, ty_SSE1));
- sum_SSE12 = _mm_add_ps(gri_SSE12, _mm_mul_ps(vx_tz_SSE1, ty_SSE2));
- sum_SSE13 = _mm_add_ps(gri_SSE13, _mm_mul_ps(vx_tz_SSE1, ty_SSE3));
-#if PME_ORDER == 5
- sum_SSE14 = _mm_add_ps(gri_SSE14, _mm_mul_ps(vx_tz_SSE1, ty_SSE4));
-#endif
-
- _mm_store_ps(grid+index+0*pnz, sum_SSE00);
- _mm_store_ps(grid+index+1*pnz, sum_SSE01);
- _mm_store_ps(grid+index+2*pnz, sum_SSE02);
- _mm_store_ps(grid+index+3*pnz, sum_SSE03);
-#if PME_ORDER == 5
- _mm_store_ps(grid+index+4*pnz, sum_SSE04);
-#endif
- _mm_store_ps(grid+index+0*pnz+4, sum_SSE10);
- _mm_store_ps(grid+index+1*pnz+4, sum_SSE11);
- _mm_store_ps(grid+index+2*pnz+4, sum_SSE12);
- _mm_store_ps(grid+index+3*pnz+4, sum_SSE13);
-#if PME_ORDER == 5
- _mm_store_ps(grid+index+4*pnz+4, sum_SSE14);
-#endif
- }
-}
-#undef PME_ORDER
-#undef PME_SPREAD_SSE_ALIGNED
-#endif
-
-
-#ifdef PME_GATHER_F_SSE_ALIGNED
-/* This code assumes that the grid is allocated 16 bit aligned
- * and that pnz is a multiple of 4.
- * This code supports pme_order <= 5.
- */
-{
- int offset;
-
- float fx_tmp[4], fy_tmp[4], fz_tmp[4];
-
- __m128 fx_SSE, fy_SSE, fz_SSE;
-
- __m128 tx_SSE, ty_SSE, tz_SSE0, tz_SSE1;
- __m128 dx_SSE, dy_SSE, dz_SSE0, dz_SSE1;
-
- __m128 gval_SSE0;
- __m128 gval_SSE1;
-
- __m128 fxy1_SSE0;
- __m128 fz1_SSE0;
- __m128 fxy1_SSE1;
- __m128 fz1_SSE1;
- __m128 fxy1_SSE;
- __m128 fz1_SSE;
-
- offset = k0 & 3;
-
- fx_SSE = _mm_setzero_ps();
- fy_SSE = _mm_setzero_ps();
- fz_SSE = _mm_setzero_ps();
-
- tz_SSE0 = _mm_loadu_ps(thz-offset);
- dz_SSE0 = _mm_loadu_ps(dthz-offset);
- tz_SSE1 = _mm_loadu_ps(thz-offset+4);
- dz_SSE1 = _mm_loadu_ps(dthz-offset+4);
- tz_SSE0 = _mm_and_ps(tz_SSE0, work->mask_SSE0[offset]);
- dz_SSE0 = _mm_and_ps(dz_SSE0, work->mask_SSE0[offset]);
- tz_SSE1 = _mm_and_ps(tz_SSE1, work->mask_SSE1[offset]);
- dz_SSE1 = _mm_and_ps(dz_SSE1, work->mask_SSE1[offset]);
-
- for (ithx = 0; (ithx < PME_ORDER); ithx++)
- {
- index_x = (i0+ithx)*pny*pnz;
- tx_SSE = _mm_load1_ps(thx+ithx);
- dx_SSE = _mm_load1_ps(dthx+ithx);
-
- for (ithy = 0; (ithy < PME_ORDER); ithy++)
- {
- index_xy = index_x+(j0+ithy)*pnz;
- ty_SSE = _mm_load1_ps(thy+ithy);
- dy_SSE = _mm_load1_ps(dthy+ithy);
-
- gval_SSE0 = _mm_load_ps(grid+index_xy+k0-offset);
- gval_SSE1 = _mm_load_ps(grid+index_xy+k0-offset+4);
-
- fxy1_SSE0 = _mm_mul_ps(tz_SSE0, gval_SSE0);
- fz1_SSE0 = _mm_mul_ps(dz_SSE0, gval_SSE0);
- fxy1_SSE1 = _mm_mul_ps(tz_SSE1, gval_SSE1);
- fz1_SSE1 = _mm_mul_ps(dz_SSE1, gval_SSE1);
-
- fxy1_SSE = _mm_add_ps(fxy1_SSE0, fxy1_SSE1);
- fz1_SSE = _mm_add_ps(fz1_SSE0, fz1_SSE1);
-
- fx_SSE = _mm_add_ps(fx_SSE, _mm_mul_ps(_mm_mul_ps(dx_SSE, ty_SSE), fxy1_SSE));
- fy_SSE = _mm_add_ps(fy_SSE, _mm_mul_ps(_mm_mul_ps(tx_SSE, dy_SSE), fxy1_SSE));
- fz_SSE = _mm_add_ps(fz_SSE, _mm_mul_ps(_mm_mul_ps(tx_SSE, ty_SSE), fz1_SSE));
- }
- }
-
- _mm_store_ps(fx_tmp, fx_SSE);
- _mm_store_ps(fy_tmp, fy_SSE);
- _mm_store_ps(fz_tmp, fz_SSE);
-
- fx += fx_tmp[0]+fx_tmp[1]+fx_tmp[2]+fx_tmp[3];
- fy += fy_tmp[0]+fy_tmp[1]+fy_tmp[2]+fy_tmp[3];
- fz += fz_tmp[0]+fz_tmp[1]+fz_tmp[2]+fz_tmp[3];
-}
-#undef PME_ORDER
-#undef PME_GATHER_F_SSE_ALIGNED
-#endif
int nchkpt = 1;
gmx_localtop_t *top;
t_mdebin *mdebin = NULL;
- df_history_t df_history;
t_state *state = NULL;
rvec *f_global = NULL;
gmx_enerdata_t *enerd;
snew(f, top_global->natoms);
}
- /* lambda Monte carlo random number generator */
- if (ir->bExpanded)
- {
- mcrng = gmx_rng_init(ir->expandedvals->lmc_seed);
- }
- /* copy the state into df_history */
- copy_df_history(&df_history, &state_global->dfhist);
-
/* Kinetic energy data */
snew(ekind, 1);
init_ekindata(fplog, top_global, &(ir->opts), ekind);
bStateFromCP = FALSE;
}
+ if (ir->bExpanded)
+ {
+ init_expanded_ensemble(bStateFromCP,ir,&mcrng,&state->dfhist);
+ }
+
if (MASTER(cr))
{
if (bStateFromCP)
set_current_lambdas(step, ir->fepvals, bRerunMD, &rerun_fr, state_global, state, lam0);
bDoDHDL = do_per_step(step, ir->fepvals->nstdhdl);
bDoFEP = (do_per_step(step, nstfep) && (ir->efep != efepNO));
- bDoExpanded = (do_per_step(step, ir->expandedvals->nstexpanded) && (ir->bExpanded) && (step > 0));
+ bDoExpanded = (do_per_step(step, ir->expandedvals->nstexpanded)
+ && (ir->bExpanded) && (step > 0) && (!bStartingFromCpt));
}
if (bSimAnn)
statistics, but if performing simulated tempering, we
do update the velocities and the tau_t. */
- lamnew = ExpandedEnsembleDynamics(fplog, ir, enerd, state, &MassQ, &df_history, step, mcrng, state->v, mdatoms);
+ lamnew = ExpandedEnsembleDynamics(fplog, ir, enerd, state, &MassQ, state->fep_state, &state->dfhist, step, mcrng, state->v, mdatoms);
+ /* history is maintained in state->dfhist, but state_global is what is sent to trajectory and log output */
+ copy_df_history(&state_global->dfhist,&state->dfhist);
}
/* Now we have the energies and forces corresponding to the
*/
do_trajectory_writing(fplog, cr, nfile, fnm, step, step_rel, t,
ir, state, state_global, top_global, fr, upd,
- outf, mdebin, ekind, df_history, f, f_global,
+ outf, mdebin, ekind, f, f_global,
wcycle, mcrng, &nchkpt,
bCPT, bRerunMD, bLastStep, (Flags & MD_CONFOUT),
bSumEkinhOld);
{
/* only needed if doing expanded ensemble */
PrintFreeEnergyInfoToFile(fplog, ir->fepvals, ir->expandedvals, ir->bSimTemp ? ir->simtempvals : NULL,
- &df_history, state->fep_state, ir->nstlog, step);
+ &state_global->dfhist, state->fep_state, ir->nstlog, step);
}
if (!(bStartingFromCpt && (EI_VV(ir->eI))))
{
}
if (bDoExpanded)
{
- /* Have to do this part after outputting the logfile and the edr file */
+ /* Have to do this part _after_ outputting the logfile and the edr file */
+ /* Gets written into the state at the beginning of next loop*/
state->fep_state = lamnew;
- for (i = 0; i < efptNR; i++)
- {
- state_global->lambda[i] = ir->fepvals->all_lambda[i][lamnew];
- }
}
+
/* Remaining runtime */
if (MULTIMASTER(cr) && (do_verbose || gmx_got_usr_signal()) && !bPMETuneRunning)
{
gmx_fatal(FARGS, "GPU requested, but can't be used without cutoff-scheme=Verlet");
}
}
+#ifdef GMX_IS_BGQ
+ else
+ {
+ md_print_warn(cr, fplog,
+ "NOTE: There is no SIMD implementation of the group scheme kernels on\n"
+ " BlueGene/Q. You will observe better performance from using the\n"
+ " Verlet cut-off scheme.\n");
+ }
+#endif
}
#ifndef GMX_THREAD_MPI
if (PAR(cr))
gmx_mdoutf_t *outf,
t_mdebin *mdebin,
gmx_ekindata_t *ekind,
- df_history_t df_history,
rvec *f,
rvec *f_global,
gmx_wallcycle_t wcycle,
state_global->ekinstate.bUpToDate = TRUE;
}
update_energyhistory(&state_global->enerhist, mdebin);
- if (ir->efep != efepNO || ir->bSimTemp)
- {
- state_global->fep_state = state->fep_state; /* MRS: seems kludgy. The code should be
- structured so this isn't necessary.
- Note this reassignment is only necessary
- for single threads.*/
- copy_df_history(&state_global->dfhist, &df_history);
- }
}
}
write_traj(fplog, cr, outf, mdof_flags, top_global,
biod.pnpi.spb.ru / alexxy / gromacs.git / commitdiff