message(STATUS "Detecting best acceleration for this CPU")
# Get CPU acceleration information
- set(_compile_definitions "@GCC_INLINE_ASM_DEFINE@ -I${CMAKE_SOURCE_DIR}/include -DGMX_CPUID_STANDALONE")
++ set(_compile_definitions "@GCC_INLINE_ASM_DEFINE@ -I${CMAKE_SOURCE_DIR}/src/gromacs/legacyheaders -DGMX_CPUID_STANDALONE")
+ if(GMX_TARGET_X86)
+ set(_compile_definitions "${_compile_definitions} -DGMX_TARGET_X86")
+ endif()
try_run(GMX_CPUID_RUN_ACC GMX_CPUID_COMPILED
${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}/src/gmxlib/gmx_cpuid.c
+ ${CMAKE_SOURCE_DIR}/src/gromacs/gmxlib/gmx_cpuid.c
- COMPILE_DEFINITIONS "@GCC_INLINE_ASM_DEFINE@ -I${CMAKE_SOURCE_DIR}/src/gromacs/legacyheaders -DGMX_CPUID_STANDALONE -DGMX_IS_X86"
+ COMPILE_DEFINITIONS ${_compile_definitions}
RUN_OUTPUT_VARIABLE OUTPUT_TMP
COMPILE_OUTPUT_VARIABLE GMX_CPUID_COMPILE_OUTPUT
ARGS "-acceleration")
if(NOT CMAKE_CROSSCOMPILING)
# Get CPU acceleration information
- set(_compile_definitions "@GCC_INLINE_ASM_DEFINE@ -I${CMAKE_SOURCE_DIR}/src/gromacs/legacyheaders -DGMX_CPUID_STANDALONE -DGMX_IS_X86")
- set(_compile_definitions "@GCC_INLINE_ASM_DEFINE@ -I${CMAKE_SOURCE_DIR}/include -DGMX_CPUID_STANDALONE")
++ set(_compile_definitions "@GCC_INLINE_ASM_DEFINE@ -I${CMAKE_SOURCE_DIR}/src/gromacs/legacyheaders -DGMX_CPUID_STANDALONE")
+ if(GMX_TARGET_X86)
+ set(_compile_definitions "${_compile_definitions} -DGMX_TARGET_X86")
+ endif()
try_run(GMX_CPUID_RUN_VENDOR GMX_CPUID_COMPILED
${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}/src/gmxlib/gmx_cpuid.c
+ ${CMAKE_SOURCE_DIR}/src/gromacs/gmxlib/gmx_cpuid.c
COMPILE_DEFINITIONS ${_compile_definitions}
RUN_OUTPUT_VARIABLE OUTPUT_CPU_VENDOR ARGS "-vendor")
try_run(GMX_CPUID_RUN_BRAND GMX_CPUID_COMPILED
--- /dev/null
- #ifdef GMX_IS_X86
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This file is part of GROMACS.
+ * Copyright (c) 2012-
+ *
+ * Written by the Gromacs development team under coordination of
+ * David van der Spoel, Berk Hess, and Erik Lindahl.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the research papers on the package. Check out http://www.gromacs.org
+ *
+ * And Hey:
+ * Gnomes, ROck Monsters And Chili Sauce
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#ifdef HAVE_SCHED_H
+#define _GNU_SOURCE
+#include <sched.h>
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+#ifdef _MSC_VER
+/* MSVC definition for __cpuid() */
+#include <intrin.h>
+/* sysinfo functions */
+#include <windows.h>
+#endif
+#ifdef HAVE_UNISTD_H
+/* sysconf() definition */
+#include <unistd.h>
+#endif
+
+#include "gmx_cpuid.h"
+
+
+
+/* For convenience, and to enable configure-time invocation, we keep all architectures
+ * in a single file, but to avoid repeated ifdefs we set the overall architecture here.
+ */
- FILE * log)
++#ifdef GMX_TARGET_X86
+/* OK, it is x86, but can we execute cpuid? */
+#if defined(GMX_X86_GCC_INLINE_ASM) || ( defined(_MSC_VER) && ( (_MSC_VER > 1500) || (_MSC_VER==1500 & _MSC_FULL_VER >= 150030729)))
+# define GMX_CPUID_X86
+#endif
+#endif
+
+/* Global constant character strings corresponding to our enumerated types */
+const char *
+gmx_cpuid_vendor_string[GMX_CPUID_NVENDORS] =
+{
+ "CannotDetect",
+ "Unknown",
+ "GenuineIntel",
+ "AuthenticAMD",
+ "Fujitsu",
+ "IBM"
+};
+
+const char *
+gmx_cpuid_vendor_string_alternative[GMX_CPUID_NVENDORS] =
+{
+ "CannotDetect",
+ "Unknown",
+ "GenuineIntel",
+ "AuthenticAMD",
+ "Fujitsu",
+ "ibm" /* Used on BlueGene/Q */
+};
+
+const char *
+gmx_cpuid_feature_string[GMX_CPUID_NFEATURES] =
+{
+ "CannotDetect",
+ "aes",
+ "apic",
+ "avx",
+ "avx2",
+ "clfsh",
+ "cmov",
+ "cx8",
+ "cx16",
+ "f16c",
+ "fma",
+ "fma4",
+ "htt",
+ "lahf_lm",
+ "misalignsse",
+ "mmx",
+ "msr",
+ "nonstop_tsc",
+ "pcid",
+ "pclmuldq",
+ "pdcm",
+ "pdpe1gb",
+ "popcnt",
+ "pse",
+ "rdrnd",
+ "rdtscp",
+ "sse2",
+ "sse3",
+ "sse4a",
+ "sse4.1",
+ "sse4.2",
+ "ssse3",
+ "tdt",
+ "x2apic",
+ "xop"
+};
+
+const char *
+gmx_cpuid_acceleration_string[GMX_CPUID_NACCELERATIONS] =
+{
+ "CannotDetect",
+ "None",
+ "SSE2",
+ "SSE4.1",
+ "AVX_128_FMA",
+ "AVX_256",
+ "Sparc64 HPC-ACE",
+ "IBM_QPX"
+};
+
+/* Max length of brand string */
+#define GMX_CPUID_BRAND_MAXLEN 256
+
+
+/* Contents of the abstract datatype */
+struct gmx_cpuid
+{
+ enum gmx_cpuid_vendor vendor;
+ char brand[GMX_CPUID_BRAND_MAXLEN];
+ int family;
+ int model;
+ int stepping;
+ /* Not using gmx_bool here, since this file must be possible to compile without simple.h */
+ char feature[GMX_CPUID_NFEATURES];
+
+ /* Basic CPU topology information. For x86 this is a bit complicated since the topology differs between
+ * operating systems and sometimes even settings. For most other architectures you can likely just check
+ * the documentation and then write static information to these arrays rather than detecting on-the-fly.
+ */
+ int have_cpu_topology;
+ int nproc; /* total number of logical processors from OS */
+ int npackages;
+ int ncores_per_package;
+ int nhwthreads_per_core;
+ int * package_id;
+ int * core_id; /* Local core id in each package */
+ int * hwthread_id; /* Local hwthread id in each core */
+ int * locality_order; /* Processor indices sorted in locality order */
+};
+
+
+/* Simple routines to access the data structure. The initialization routine is
+ * further down since that needs to call other static routines in this file.
+ */
+enum gmx_cpuid_vendor
+gmx_cpuid_vendor (gmx_cpuid_t cpuid)
+{
+ return cpuid->vendor;
+}
+
+
+const char *
+gmx_cpuid_brand (gmx_cpuid_t cpuid)
+{
+ return cpuid->brand;
+}
+
+int
+gmx_cpuid_family (gmx_cpuid_t cpuid)
+{
+ return cpuid->family;
+}
+
+int
+gmx_cpuid_model (gmx_cpuid_t cpuid)
+{
+ return cpuid->model;
+}
+
+int
+gmx_cpuid_stepping (gmx_cpuid_t cpuid)
+{
+ return cpuid->stepping;
+}
+
+int
+gmx_cpuid_feature (gmx_cpuid_t cpuid,
+ enum gmx_cpuid_feature feature)
+{
+ return (cpuid->feature[feature] != 0);
+}
+
+
+
+
+/* What type of acceleration was compiled in, if any?
+ * This is set from Cmake. Note that the SSE2 and SSE4_1 macros are set for
+ * AVX too, so it is important that they appear last in the list.
+ */
+#ifdef GMX_X86_AVX_256
+static const
+enum gmx_cpuid_acceleration
+ compiled_acc = GMX_CPUID_ACCELERATION_X86_AVX_256;
+#elif defined GMX_X86_AVX_128_FMA
+static const
+enum gmx_cpuid_acceleration
+ compiled_acc = GMX_CPUID_ACCELERATION_X86_AVX_128_FMA;
+#elif defined GMX_X86_SSE4_1
+static const
+enum gmx_cpuid_acceleration
+ compiled_acc = GMX_CPUID_ACCELERATION_X86_SSE4_1;
+#elif defined GMX_X86_SSE2
+static const
+enum gmx_cpuid_acceleration
+ compiled_acc = GMX_CPUID_ACCELERATION_X86_SSE2;
+#elif defined GMX_CPU_ACCELERATION_SPARC64_HPC_ACE
+static const
+enum gmx_cpuid_acceleration
+ compiled_acc = GMX_CPUID_ACCELERATION_SPARC64_HPC_ACE;
+#elif defined GMX_CPU_ACCELERATION_IBM_QPX
+static const
+enum gmx_cpuid_acceleration
+ compiled_acc = GMX_CPUID_ACCELERATION_IBM_QPX;
+#else
+static const
+enum gmx_cpuid_acceleration
+ compiled_acc = GMX_CPUID_ACCELERATION_NONE;
+#endif
+
+
+#ifdef GMX_CPUID_X86
+
+/* Execute CPUID on x86 class CPUs. level sets function to exec, and the
+ * contents of register output is returned. See Intel/AMD docs for details.
+ *
+ * This version supports extended information where we can also have an input
+ * value in the ecx register. This is ignored for most levels, but some of them
+ * (e.g. level 0xB on Intel) use it.
+ */
+static int
+execute_x86cpuid(unsigned int level,
+ unsigned int ecxval,
+ unsigned int * eax,
+ unsigned int * ebx,
+ unsigned int * ecx,
+ unsigned int * edx)
+{
+ int rc = 0;
+
+ /* Currently CPUID is only supported (1) if we can use an instruction on MSVC, or (2)
+ * if the compiler handles GNU-style inline assembly.
+ */
+
+#if (defined _MSC_VER)
+ int CPUInfo[4];
+
+#if (_MSC_VER > 1500) || (_MSC_VER == 1500 & _MSC_FULL_VER >= 150030729)
+ /* MSVC 9.0 SP1 or later */
+ __cpuidex(CPUInfo, level, ecxval);
+ rc = 0;
+#else
+ __cpuid(CPUInfo, level);
+ /* Set an error code if the user wanted a non-zero ecxval, since we did not have cpuidex */
+ rc = (ecxval > 0) ? -1 : 0;
+#endif
+ *eax = CPUInfo[0];
+ *ebx = CPUInfo[1];
+ *ecx = CPUInfo[2];
+ *edx = CPUInfo[3];
+
+#elif (defined GMX_X86_GCC_INLINE_ASM)
+ /* for now this means GMX_X86_GCC_INLINE_ASM should be defined,
+ * but there might be more options added in the future.
+ */
+ *eax = level;
+ *ecx = ecxval;
+ *ebx = 0;
+ *edx = 0;
+#if defined(__i386__) && defined(__PIC__)
+ /* Avoid clobbering the global offset table in 32-bit pic code (ebx register) */
+ __asm__ __volatile__ ("xchgl %%ebx, %1 \n\t"
+ "cpuid \n\t"
+ "xchgl %%ebx, %1 \n\t"
+ : "+a" (*eax), "+r" (*ebx), "+c" (*ecx), "+d" (*edx));
+#else
+ /* i386 without PIC, or x86-64. Things are easy and we can clobber any reg we want :-) */
+ __asm__ __volatile__ ("cpuid \n\t"
+ : "+a" (*eax), "+b" (*ebx), "+c" (*ecx), "+d" (*edx));
+#endif
+ rc = 0;
+#else
+ /* Death and horror!
+ * Apparently this is an x86 platform where we don't know how to call cpuid.
+ *
+ * This is REALLY bad, since we will lose all Gromacs acceleration.
+ */
+ *eax = 0;
+ *ebx = 0;
+ *ecx = 0;
+ *edx = 0;
+
+ rc = -1;
+#endif
+ return rc;
+}
+
+
+/* Identify CPU features common to Intel & AMD - mainly brand string,
+ * version and some features. Vendor has already been detected outside this.
+ */
+static int
+cpuid_check_common_x86(gmx_cpuid_t cpuid)
+{
+ int fn, max_stdfn, max_extfn;
+ unsigned int eax, ebx, ecx, edx;
+ char str[GMX_CPUID_BRAND_MAXLEN];
+ char * p;
+
+ /* Find largest standard/extended function input value */
+ execute_x86cpuid(0x0, 0, &eax, &ebx, &ecx, &edx);
+ max_stdfn = eax;
+ execute_x86cpuid(0x80000000, 0, &eax, &ebx, &ecx, &edx);
+ max_extfn = eax;
+
+ p = str;
+ if (max_extfn >= 0x80000005)
+ {
+ /* Get CPU brand string */
+ for (fn = 0x80000002; fn < 0x80000005; fn++)
+ {
+ execute_x86cpuid(fn, 0, &eax, &ebx, &ecx, &edx);
+ memcpy(p, &eax, 4);
+ memcpy(p+4, &ebx, 4);
+ memcpy(p+8, &ecx, 4);
+ memcpy(p+12, &edx, 4);
+ p += 16;
+ }
+ *p = '\0';
+
+ /* Remove empty initial space */
+ p = str;
+ while (isspace(*(p)))
+ {
+ p++;
+ }
+ strncpy(cpuid->brand, p, GMX_CPUID_BRAND_MAXLEN);
+ }
+ else
+ {
+ strncpy(cpuid->brand, "Unknown CPU brand", GMX_CPUID_BRAND_MAXLEN);
+ }
+
+ /* Find basic CPU properties */
+ if (max_stdfn >= 1)
+ {
+ execute_x86cpuid(0x1, 0, &eax, &ebx, &ecx, &edx);
+
+ cpuid->family = ((eax & 0x0FF00000) >> 20) + ((eax & 0x00000F00) >> 8);
+ /* Note that extended model should be shifted left 4, so only shift right 12 iso 16. */
+ cpuid->model = ((eax & 0x000F0000) >> 12) + ((eax & 0x000000F0) >> 4);
+ cpuid->stepping = (eax & 0x0000000F);
+
+ /* Feature flags common to AMD and intel */
+ cpuid->feature[GMX_CPUID_FEATURE_X86_SSE3] = (ecx & (1 << 0)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_PCLMULDQ] = (ecx & (1 << 1)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_SSSE3] = (ecx & (1 << 9)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_FMA] = (ecx & (1 << 12)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_CX16] = (ecx & (1 << 13)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_SSE4_1] = (ecx & (1 << 19)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_SSE4_2] = (ecx & (1 << 20)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_POPCNT] = (ecx & (1 << 23)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_AES] = (ecx & (1 << 25)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_AVX] = (ecx & (1 << 28)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_F16C] = (ecx & (1 << 29)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_RDRND] = (ecx & (1 << 30)) != 0;
+
+ cpuid->feature[GMX_CPUID_FEATURE_X86_PSE] = (edx & (1 << 3)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_MSR] = (edx & (1 << 5)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_CX8] = (edx & (1 << 8)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_APIC] = (edx & (1 << 9)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_CMOV] = (edx & (1 << 15)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_CLFSH] = (edx & (1 << 19)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_MMX] = (edx & (1 << 23)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_SSE2] = (edx & (1 << 26)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_HTT] = (edx & (1 << 28)) != 0;
+ }
+ else
+ {
+ cpuid->family = -1;
+ cpuid->model = -1;
+ cpuid->stepping = -1;
+ }
+
+ if (max_extfn >= 0x80000001)
+ {
+ execute_x86cpuid(0x80000001, 0, &eax, &ebx, &ecx, &edx);
+ cpuid->feature[GMX_CPUID_FEATURE_X86_LAHF_LM] = (ecx & (1 << 0)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_PDPE1GB] = (edx & (1 << 26)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_RDTSCP] = (edx & (1 << 27)) != 0;
+ }
+
+ if (max_extfn >= 0x80000007)
+ {
+ execute_x86cpuid(0x80000007, 0, &eax, &ebx, &ecx, &edx);
+ cpuid->feature[GMX_CPUID_FEATURE_X86_NONSTOP_TSC] = (edx & (1 << 8)) != 0;
+ }
+ return 0;
+}
+
+/* This routine returns the number of unique different elements found in the array,
+ * and renumbers these starting from 0. For example, the array {0,1,2,8,9,10,8,9,10,0,1,2}
+ * will be rewritten to {0,1,2,3,4,5,3,4,5,0,1,2}, and it returns 6 for the
+ * number of unique elements.
+ */
+static int
+cpuid_renumber_elements(int *data, int n)
+{
+ int *unique;
+ int i, j, nunique, found;
+
+ unique = malloc(sizeof(int)*n);
+
+ nunique = 0;
+ for (i = 0; i < n; i++)
+ {
+ for (j = 0, found = 0; j < nunique && !found; j++)
+ {
+ found = (data[i] == unique[j]);
+ }
+ if (!found)
+ {
+ /* Insert in sorted order! */
+ for (j = nunique++; j > 0 && unique[j-1] > data[i]; j--)
+ {
+ unique[j] = unique[j-1];
+ }
+ unique[j] = data[i];
+ }
+ }
+ /* renumber */
+ for (i = 0; i < n; i++)
+ {
+ for (j = 0; j < nunique; j++)
+ {
+ if (data[i] == unique[j])
+ {
+ data[i] = j;
+ }
+ }
+ }
+ return nunique;
+}
+
+/* APIC IDs, or everything you wanted to know about your x86 cores but were afraid to ask...
+ *
+ * Raw APIC IDs are unfortunately somewhat dirty. For technical reasons they are assigned
+ * in power-of-2 chunks, and even then there are no guarantees about specific numbers - all
+ * we know is that the part for each thread/core/package is unique, and how many bits are
+ * reserved for that part.
+ * This routine does internal renumbering so we get continuous indices, and also
+ * decodes the actual number of packages,cores-per-package and hwthreads-per-core.
+ * Returns: 0 on success, non-zero on failure.
+ */
+static int
+cpuid_x86_decode_apic_id(gmx_cpuid_t cpuid, int *apic_id, int core_bits, int hwthread_bits)
+{
+ int i, idx;
+ int hwthread_mask, core_mask_after_shift;
+
+ cpuid->hwthread_id = malloc(sizeof(int)*cpuid->nproc);
+ cpuid->core_id = malloc(sizeof(int)*cpuid->nproc);
+ cpuid->package_id = malloc(sizeof(int)*cpuid->nproc);
+ cpuid->locality_order = malloc(sizeof(int)*cpuid->nproc);
+
+ hwthread_mask = (1 << hwthread_bits) - 1;
+ core_mask_after_shift = (1 << core_bits) - 1;
+
+ for (i = 0; i < cpuid->nproc; i++)
+ {
+ cpuid->hwthread_id[i] = apic_id[i] & hwthread_mask;
+ cpuid->core_id[i] = (apic_id[i] >> hwthread_bits) & core_mask_after_shift;
+ cpuid->package_id[i] = apic_id[i] >> (core_bits + hwthread_bits);
+ }
+
+ cpuid->npackages = cpuid_renumber_elements(cpuid->package_id, cpuid->nproc);
+ cpuid->ncores_per_package = cpuid_renumber_elements(cpuid->core_id, cpuid->nproc);
+ cpuid->nhwthreads_per_core = cpuid_renumber_elements(cpuid->hwthread_id, cpuid->nproc);
+
+ /* now check for consistency */
+ if ( (cpuid->npackages * cpuid->ncores_per_package *
+ cpuid->nhwthreads_per_core) != cpuid->nproc )
+ {
+ /* the packages/cores-per-package/hwthreads-per-core counts are
+ inconsistent. */
+ return -1;
+ }
+
+ /* Create a locality order array, i.e. first all resources in package0, which in turn
+ * are sorted so we first have all resources in core0, where threads are sorted in order, etc.
+ */
+
+ for (i = 0; i < cpuid->nproc; i++)
+ {
+ idx = (cpuid->package_id[i]*cpuid->ncores_per_package + cpuid->core_id[i])*cpuid->nhwthreads_per_core + cpuid->hwthread_id[i];
+ cpuid->locality_order[idx] = i;
+ }
+ return 0;
+}
+
+
+/* Detection of AMD-specific CPU features */
+static int
+cpuid_check_amd_x86(gmx_cpuid_t cpuid)
+{
+ int max_stdfn, max_extfn, ret;
+ unsigned int eax, ebx, ecx, edx;
+ int hwthread_bits, core_bits;
+ int * apic_id;
+
+ cpuid_check_common_x86(cpuid);
+
+ execute_x86cpuid(0x0, 0, &eax, &ebx, &ecx, &edx);
+ max_stdfn = eax;
+
+ execute_x86cpuid(0x80000000, 0, &eax, &ebx, &ecx, &edx);
+ max_extfn = eax;
+
+ if (max_extfn >= 0x80000001)
+ {
+ execute_x86cpuid(0x80000001, 0, &eax, &ebx, &ecx, &edx);
+
+ cpuid->feature[GMX_CPUID_FEATURE_X86_SSE4A] = (ecx & (1 << 6)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_MISALIGNSSE] = (ecx & (1 << 7)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_XOP] = (ecx & (1 << 11)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_FMA4] = (ecx & (1 << 16)) != 0;
+ }
+
+ /* Query APIC information on AMD */
+ if (max_extfn >= 0x80000008)
+ {
+#if (defined HAVE_SCHED_H && defined HAVE_SCHED_SETAFFINITY && defined HAVE_SYSCONF && defined __linux__)
+ /* Linux */
+ unsigned int i;
+ cpu_set_t cpuset, save_cpuset;
+ cpuid->nproc = sysconf(_SC_NPROCESSORS_ONLN);
+ apic_id = malloc(sizeof(int)*cpuid->nproc);
+ sched_getaffinity(0, sizeof(cpu_set_t), &save_cpuset);
+ /* Get APIC id from each core */
+ CPU_ZERO(&cpuset);
+ for (i = 0; i < cpuid->nproc; i++)
+ {
+ CPU_SET(i, &cpuset);
+ sched_setaffinity(0, sizeof(cpu_set_t), &cpuset);
+ execute_x86cpuid(0x1, 0, &eax, &ebx, &ecx, &edx);
+ apic_id[i] = ebx >> 24;
+ CPU_CLR(i, &cpuset);
+ }
+ /* Reset affinity to the value it had when calling this routine */
+ sched_setaffinity(0, sizeof(cpu_set_t), &save_cpuset);
+#define CPUID_HAVE_APIC
+#elif defined GMX_NATIVE_WINDOWS
+ /* Windows */
+ DWORD_PTR i;
+ SYSTEM_INFO sysinfo;
+ unsigned int save_affinity, affinity;
+ GetSystemInfo( &sysinfo );
+ cpuid->nproc = sysinfo.dwNumberOfProcessors;
+ apic_id = malloc(sizeof(int)*cpuid->nproc);
+ /* Get previous affinity mask */
+ save_affinity = SetThreadAffinityMask(GetCurrentThread(), 1);
+ for (i = 0; i < cpuid->nproc; i++)
+ {
+ SetThreadAffinityMask(GetCurrentThread(), (((DWORD_PTR)1)<<i));
+ Sleep(0);
+ execute_x86cpuid(0x1, 0, &eax, &ebx, &ecx, &edx);
+ apic_id[i] = ebx >> 24;
+ }
+ SetThreadAffinityMask(GetCurrentThread(), save_affinity);
+#define CPUID_HAVE_APIC
+#endif
+#ifdef CPUID_HAVE_APIC
+ /* AMD does not support SMT yet - there are no hwthread bits in apic ID */
+ hwthread_bits = 0;
+ /* Get number of core bits in apic ID - try modern extended method first */
+ execute_x86cpuid(0x80000008, 0, &eax, &ebx, &ecx, &edx);
+ core_bits = (ecx >> 12) & 0xf;
+ if (core_bits == 0)
+ {
+ /* Legacy method for old single/dual core AMD CPUs */
+ int i = ecx & 0xF;
+ for (core_bits = 0; (i>>core_bits) > 0; core_bits++)
+ {
+ ;
+ }
+ }
+ ret = cpuid_x86_decode_apic_id(cpuid, apic_id, core_bits,
+ hwthread_bits);
+ cpuid->have_cpu_topology = (ret == 0);
+#endif
+ }
+ return 0;
+}
+
+/* Detection of Intel-specific CPU features */
+static int
+cpuid_check_intel_x86(gmx_cpuid_t cpuid)
+{
+ unsigned int max_stdfn, max_extfn, ret;
+ unsigned int eax, ebx, ecx, edx;
+ unsigned int max_logical_cores, max_physical_cores;
+ int hwthread_bits, core_bits;
+ int * apic_id;
+
+ cpuid_check_common_x86(cpuid);
+
+ execute_x86cpuid(0x0, 0, &eax, &ebx, &ecx, &edx);
+ max_stdfn = eax;
+
+ execute_x86cpuid(0x80000000, 0, &eax, &ebx, &ecx, &edx);
+ max_extfn = eax;
+
+ if (max_stdfn >= 1)
+ {
+ execute_x86cpuid(0x1, 0, &eax, &ebx, &ecx, &edx);
+ cpuid->feature[GMX_CPUID_FEATURE_X86_PDCM] = (ecx & (1 << 15)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_PCID] = (ecx & (1 << 17)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_X2APIC] = (ecx & (1 << 21)) != 0;
+ cpuid->feature[GMX_CPUID_FEATURE_X86_TDT] = (ecx & (1 << 24)) != 0;
+ }
+
+ if (max_stdfn >= 7)
+ {
+ execute_x86cpuid(0x7, 0, &eax, &ebx, &ecx, &edx);
+ cpuid->feature[GMX_CPUID_FEATURE_X86_AVX2] = (ebx & (1 << 5)) != 0;
+ }
+
+ /* Check whether Hyper-Threading is enabled, not only supported */
+ if (cpuid->feature[GMX_CPUID_FEATURE_X86_HTT] && max_stdfn >= 4)
+ {
+ execute_x86cpuid(0x1, 0, &eax, &ebx, &ecx, &edx);
+ max_logical_cores = (ebx >> 16) & 0x0FF;
+ execute_x86cpuid(0x4, 0, &eax, &ebx, &ecx, &edx);
+ max_physical_cores = ((eax >> 26) & 0x3F) + 1;
+
+ /* Clear HTT flag if we only have 1 logical core per physical */
+ if (max_logical_cores/max_physical_cores < 2)
+ {
+ cpuid->feature[GMX_CPUID_FEATURE_X86_HTT] = 0;
+ }
+ }
+
+ if (max_stdfn >= 0xB)
+ {
+ /* Query x2 APIC information from cores */
+#if (defined HAVE_SCHED_H && defined HAVE_SCHED_SETAFFINITY && defined HAVE_SYSCONF && defined __linux__)
+ /* Linux */
+ unsigned int i;
+ cpu_set_t cpuset, save_cpuset;
+ cpuid->nproc = sysconf(_SC_NPROCESSORS_ONLN);
+ apic_id = malloc(sizeof(int)*cpuid->nproc);
+ sched_getaffinity(0, sizeof(cpu_set_t), &save_cpuset);
+ /* Get x2APIC ID from each hardware thread */
+ CPU_ZERO(&cpuset);
+ for (i = 0; i < cpuid->nproc; i++)
+ {
+ CPU_SET(i, &cpuset);
+ sched_setaffinity(0, sizeof(cpu_set_t), &cpuset);
+ execute_x86cpuid(0xB, 0, &eax, &ebx, &ecx, &edx);
+ apic_id[i] = edx;
+ CPU_CLR(i, &cpuset);
+ }
+ /* Reset affinity to the value it had when calling this routine */
+ sched_setaffinity(0, sizeof(cpu_set_t), &save_cpuset);
+#define CPUID_HAVE_APIC
+#elif defined GMX_NATIVE_WINDOWS
+ /* Windows */
+ DWORD_PTR i;
+ SYSTEM_INFO sysinfo;
+ unsigned int save_affinity, affinity;
+ GetSystemInfo( &sysinfo );
+ cpuid->nproc = sysinfo.dwNumberOfProcessors;
+ apic_id = malloc(sizeof(int)*cpuid->nproc);
+ /* Get previous affinity mask */
+ save_affinity = SetThreadAffinityMask(GetCurrentThread(), 1);
+ for (i = 0; i < cpuid->nproc; i++)
+ {
+ SetThreadAffinityMask(GetCurrentThread(), (((DWORD_PTR)1)<<i));
+ Sleep(0);
+ execute_x86cpuid(0xB, 0, &eax, &ebx, &ecx, &edx);
+ apic_id[i] = edx;
+ }
+ SetThreadAffinityMask(GetCurrentThread(), save_affinity);
+#define CPUID_HAVE_APIC
+#endif
+#ifdef CPUID_HAVE_APIC
+ execute_x86cpuid(0xB, 0, &eax, &ebx, &ecx, &edx);
+ hwthread_bits = eax & 0x1F;
+ execute_x86cpuid(0xB, 1, &eax, &ebx, &ecx, &edx);
+ core_bits = (eax & 0x1F) - hwthread_bits;
+ ret = cpuid_x86_decode_apic_id(cpuid, apic_id, core_bits,
+ hwthread_bits);
+ cpuid->have_cpu_topology = (ret == 0);
+#endif
+ }
+ return 0;
+}
+#endif /* GMX_CPUID_X86 */
+
+
+
+
+static void
+chomp_substring_before_colon(const char *in, char *s, int maxlength)
+{
+ char *p;
+ strncpy(s,in,maxlength);
+ p = strchr(s,':');
+ if(p!=NULL)
+ {
+ *p='\0';
+ while(isspace(*(--p)) && (p>=s))
+ {
+ *p='\0';
+ }
+ }
+ else
+ {
+ *s='\0';
+ }
+}
+
+static void
+chomp_substring_after_colon(const char *in, char *s, int maxlength)
+{
+ char *p;
+ if( (p = strchr(in,':'))!=NULL)
+ {
+ p++;
+ while(isspace(*p)) p++;
+ strncpy(s,p,maxlength);
+ p = s+strlen(s);
+ while(isspace(*(--p)) && (p>=s))
+ {
+ *p='\0';
+ }
+ }
+ else
+ {
+ *s='\0';
+ }
+}
+
+/* Try to find the vendor of the current CPU, so we know what specific
+ * detection routine to call.
+ */
+static enum gmx_cpuid_vendor
+cpuid_check_vendor(void)
+{
+ enum gmx_cpuid_vendor i, vendor;
+ /* Register data used on x86 */
+ unsigned int eax, ebx, ecx, edx;
+ char vendorstring[13];
+ FILE * fp;
+ char buffer[255],before_colon[255], after_colon[255];
+
+ /* Set default first */
+ vendor = GMX_CPUID_VENDOR_UNKNOWN;
+
+#ifdef GMX_CPUID_X86
+ execute_x86cpuid(0x0, 0, &eax, &ebx, &ecx, &edx);
+
+ memcpy(vendorstring, &ebx, 4);
+ memcpy(vendorstring+4, &edx, 4);
+ memcpy(vendorstring+8, &ecx, 4);
+
+ vendorstring[12] = '\0';
+
+ for (i = GMX_CPUID_VENDOR_UNKNOWN; i < GMX_CPUID_NVENDORS; i++)
+ {
+ if (!strncmp(vendorstring, gmx_cpuid_vendor_string[i], 12))
+ {
+ vendor = i;
+ }
+ }
+#elif defined(__linux__) || defined(__linux)
+ /* General Linux. Try to get CPU vendor from /proc/cpuinfo */
+ if( (fp = fopen("/proc/cpuinfo","r")) != NULL)
+ {
+ while( (vendor == GMX_CPUID_VENDOR_UNKNOWN) && (fgets(buffer,sizeof(buffer),fp) != NULL))
+ {
+ chomp_substring_before_colon(buffer,before_colon,sizeof(before_colon));
+ /* Intel/AMD use "vendor_id", IBM "vendor"(?) or "model". Fujitsu "manufacture". Add others if you have them! */
+ if( !strcmp(before_colon,"vendor_id")
+ || !strcmp(before_colon,"vendor")
+ || !strcmp(before_colon,"manufacture")
+ || !strcmp(before_colon,"model"))
+ {
+ chomp_substring_after_colon(buffer,after_colon,sizeof(after_colon));
+ for(i=GMX_CPUID_VENDOR_UNKNOWN; i<GMX_CPUID_NVENDORS; i++)
+ {
+ /* Be liberal and accept if we find the vendor
+ * string (or alternative string) anywhere. Using
+ * strcasestr() would be non-portable. */
+ if(strstr(after_colon,gmx_cpuid_vendor_string[i])
+ || strstr(after_colon,gmx_cpuid_vendor_string_alternative[i]))
+ {
+ vendor = i;
+ }
+ }
+ }
+ }
+ }
+ fclose(fp);
+#endif
+
+ return vendor;
+}
+
+
+
+int
+gmx_cpuid_topology(gmx_cpuid_t cpuid,
+ int * nprocessors,
+ int * npackages,
+ int * ncores_per_package,
+ int * nhwthreads_per_core,
+ const int ** package_id,
+ const int ** core_id,
+ const int ** hwthread_id,
+ const int ** locality_order)
+{
+ int rc;
+
+ if (cpuid->have_cpu_topology)
+ {
+ *nprocessors = cpuid->nproc;
+ *npackages = cpuid->npackages;
+ *ncores_per_package = cpuid->ncores_per_package;
+ *nhwthreads_per_core = cpuid->nhwthreads_per_core;
+ *package_id = cpuid->package_id;
+ *core_id = cpuid->core_id;
+ *hwthread_id = cpuid->hwthread_id;
+ *locality_order = cpuid->locality_order;
+ rc = 0;
+ }
+ else
+ {
+ rc = -1;
+ }
+ return rc;
+}
+
+
+enum gmx_cpuid_x86_smt
+gmx_cpuid_x86_smt(gmx_cpuid_t cpuid)
+{
+ enum gmx_cpuid_x86_smt rc;
+
+ if (cpuid->have_cpu_topology)
+ {
+ rc = (cpuid->nhwthreads_per_core > 1) ? GMX_CPUID_X86_SMT_ENABLED : GMX_CPUID_X86_SMT_DISABLED;
+ }
+ else if (cpuid->vendor == GMX_CPUID_VENDOR_AMD || gmx_cpuid_feature(cpuid, GMX_CPUID_FEATURE_X86_HTT) == 0)
+ {
+ rc = GMX_CPUID_X86_SMT_DISABLED;
+ }
+ else
+ {
+ rc = GMX_CPUID_X86_SMT_CANNOTDETECT;
+ }
+ return rc;
+}
+
+
+int
+gmx_cpuid_init (gmx_cpuid_t * pcpuid)
+{
+ gmx_cpuid_t cpuid;
+ int i;
+ FILE * fp;
+ char buffer[255],buffer2[255];
+ int found_brand;
+
+ cpuid = malloc(sizeof(*cpuid));
+
+ *pcpuid = cpuid;
+
+ for (i = 0; i < GMX_CPUID_NFEATURES; i++)
+ {
+ cpuid->feature[i] = 0;
+ }
+
+ cpuid->have_cpu_topology = 0;
+ cpuid->nproc = 0;
+ cpuid->npackages = 0;
+ cpuid->ncores_per_package = 0;
+ cpuid->nhwthreads_per_core = 0;
+ cpuid->package_id = NULL;
+ cpuid->core_id = NULL;
+ cpuid->hwthread_id = NULL;
+ cpuid->locality_order = NULL;
+
+ cpuid->vendor = cpuid_check_vendor();
+
+ switch (cpuid->vendor)
+ {
+#ifdef GMX_CPUID_X86
+ case GMX_CPUID_VENDOR_INTEL:
+ cpuid_check_intel_x86(cpuid);
+ break;
+ case GMX_CPUID_VENDOR_AMD:
+ cpuid_check_amd_x86(cpuid);
+ break;
+#endif
+ default:
+ /* Default value */
+ strncpy(cpuid->brand,"Unknown CPU brand",GMX_CPUID_BRAND_MAXLEN);
+#if defined(__linux__) || defined(__linux)
+ /* General Linux. Try to get CPU type from /proc/cpuinfo */
+ if( (fp = fopen("/proc/cpuinfo","r")) != NULL)
+ {
+ found_brand = 0;
+ while( (found_brand==0) && (fgets(buffer,sizeof(buffer),fp) !=NULL))
+ {
+ chomp_substring_before_colon(buffer,buffer2,sizeof(buffer2));
+ /* Intel uses "model name", Fujitsu and IBM "cpu". */
+ if( !strcmp(buffer2,"model name") || !strcmp(buffer2,"cpu"))
+ {
+ chomp_substring_after_colon(buffer,cpuid->brand,GMX_CPUID_BRAND_MAXLEN);
+ found_brand = 1;
+ }
+ }
+ }
+ fclose(fp);
+#endif
+ cpuid->family = 0;
+ cpuid->model = 0;
+ cpuid->stepping = 0;
+
+ for(i=0; i<GMX_CPUID_NFEATURES; i++)
+ {
+ cpuid->feature[i]=0;
+ }
+ cpuid->feature[GMX_CPUID_FEATURE_CANNOTDETECT] = 1;
+ break;
+ }
+ return 0;
+}
+
+
+
+void
+gmx_cpuid_done (gmx_cpuid_t cpuid)
+{
+ free(cpuid);
+}
+
+
+int
+gmx_cpuid_formatstring (gmx_cpuid_t cpuid,
+ char * str,
+ int n)
+{
+ int c;
+ int i;
+ enum gmx_cpuid_feature feature;
+
+#ifdef _MSC_VER
+ _snprintf(str, n,
+ "Vendor: %s\n"
+ "Brand: %s\n"
+ "Family: %2d Model: %2d Stepping: %2d\n"
+ "Features:",
+ gmx_cpuid_vendor_string[gmx_cpuid_vendor(cpuid)],
+ gmx_cpuid_brand(cpuid),
+ gmx_cpuid_family(cpuid), gmx_cpuid_model(cpuid), gmx_cpuid_stepping(cpuid));
+#else
+ snprintf(str, n,
+ "Vendor: %s\n"
+ "Brand: %s\n"
+ "Family: %2d Model: %2d Stepping: %2d\n"
+ "Features:",
+ gmx_cpuid_vendor_string[gmx_cpuid_vendor(cpuid)],
+ gmx_cpuid_brand(cpuid),
+ gmx_cpuid_family(cpuid), gmx_cpuid_model(cpuid), gmx_cpuid_stepping(cpuid));
+#endif
+
+ str[n-1] = '\0';
+ c = strlen(str);
+ n -= c;
+ str += c;
+
+ for (feature = GMX_CPUID_FEATURE_CANNOTDETECT; feature < GMX_CPUID_NFEATURES; feature++)
+ {
+ if (gmx_cpuid_feature(cpuid, feature) == 1)
+ {
+#ifdef _MSC_VER
+ _snprintf(str, n, " %s", gmx_cpuid_feature_string[feature]);
+#else
+ snprintf(str, n, " %s", gmx_cpuid_feature_string[feature]);
+#endif
+ str[n-1] = '\0';
+ c = strlen(str);
+ n -= c;
+ str += c;
+ }
+ }
+#ifdef _MSC_VER
+ _snprintf(str, n, "\n");
+#else
+ snprintf(str, n, "\n");
+#endif
+ str[n-1] = '\0';
+
+ return 0;
+}
+
+
+
+enum gmx_cpuid_acceleration
+gmx_cpuid_acceleration_suggest (gmx_cpuid_t cpuid)
+{
+ enum gmx_cpuid_acceleration tmpacc;
+
+ tmpacc = GMX_CPUID_ACCELERATION_NONE;
+
+ if (gmx_cpuid_vendor(cpuid) == GMX_CPUID_VENDOR_INTEL)
+ {
+ if (gmx_cpuid_feature(cpuid, GMX_CPUID_FEATURE_X86_AVX))
+ {
+ tmpacc = GMX_CPUID_ACCELERATION_X86_AVX_256;
+ }
+ else if (gmx_cpuid_feature(cpuid, GMX_CPUID_FEATURE_X86_SSE4_1))
+ {
+ tmpacc = GMX_CPUID_ACCELERATION_X86_SSE4_1;
+ }
+ else if (gmx_cpuid_feature(cpuid, GMX_CPUID_FEATURE_X86_SSE2))
+ {
+ tmpacc = GMX_CPUID_ACCELERATION_X86_SSE2;
+ }
+ }
+ else if (gmx_cpuid_vendor(cpuid) == GMX_CPUID_VENDOR_AMD)
+ {
+ if (gmx_cpuid_feature(cpuid, GMX_CPUID_FEATURE_X86_AVX))
+ {
+ tmpacc = GMX_CPUID_ACCELERATION_X86_AVX_128_FMA;
+ }
+ else if (gmx_cpuid_feature(cpuid, GMX_CPUID_FEATURE_X86_SSE4_1))
+ {
+ tmpacc = GMX_CPUID_ACCELERATION_X86_SSE4_1;
+ }
+ else if (gmx_cpuid_feature(cpuid, GMX_CPUID_FEATURE_X86_SSE2))
+ {
+ tmpacc = GMX_CPUID_ACCELERATION_X86_SSE2;
+ }
+ }
+ else if(gmx_cpuid_vendor(cpuid)==GMX_CPUID_VENDOR_FUJITSU)
+ {
+ if(strstr(gmx_cpuid_brand(cpuid),"SPARC64"))
+ {
+ tmpacc = GMX_CPUID_ACCELERATION_SPARC64_HPC_ACE;
+ }
+ }
+ else if(gmx_cpuid_vendor(cpuid)==GMX_CPUID_VENDOR_IBM)
+ {
+ if(strstr(gmx_cpuid_brand(cpuid),"A2"))
+ {
+ tmpacc = GMX_CPUID_ACCELERATION_IBM_QPX;
+ }
+ }
+ return tmpacc;
+}
+
+
+
+int
+gmx_cpuid_acceleration_check(gmx_cpuid_t cpuid,
- printf("Compiled acceleration: %s (Gromacs could use %s on this machine, which is better)\n",
- gmx_cpuid_acceleration_string[compiled_acc],
- gmx_cpuid_acceleration_string[acc]);
++ FILE * log,
++ int print_to_stderr)
+{
+ int rc;
+ char str[1024];
+ enum gmx_cpuid_acceleration acc;
+
+ acc = gmx_cpuid_acceleration_suggest(cpuid);
+
+ rc = (acc != compiled_acc);
+
+ gmx_cpuid_formatstring(cpuid, str, 1023);
+ str[1023] = '\0';
+
+ if (log != NULL)
+ {
+ fprintf(log,
+ "\nDetecting CPU-specific acceleration.\nPresent hardware specification:\n"
+ "%s"
+ "Acceleration most likely to fit this hardware: %s\n"
+ "Acceleration selected at GROMACS compile time: %s\n\n",
+ str,
+ gmx_cpuid_acceleration_string[acc],
+ gmx_cpuid_acceleration_string[compiled_acc]);
+ }
+
+ if (rc != 0)
+ {
+ if (log != NULL)
+ {
+ fprintf(log, "\nBinary not matching hardware - you might be losing performance.\n"
+ "Acceleration most likely to fit this hardware: %s\n"
+ "Acceleration selected at GROMACS compile time: %s\n\n",
+ gmx_cpuid_acceleration_string[acc],
+ gmx_cpuid_acceleration_string[compiled_acc]);
+ }
++ if (print_to_stderr)
++ {
++ fprintf(stderr, "Compiled acceleration: %s (Gromacs could use %s on this machine, which is better)\n",
++ gmx_cpuid_acceleration_string[compiled_acc],
++ gmx_cpuid_acceleration_string[acc]);
++ }
+ }
+ return rc;
+}
+
+
+#ifdef GMX_CPUID_STANDALONE
+/* Stand-alone program to enable queries of CPU features from Cmake.
+ * Note that you need to check inline ASM capabilities before compiling and set
+ * -DGMX_X86_GCC_INLINE_ASM for the cpuid instruction to work...
+ */
+int
+main(int argc, char **argv)
+{
+ gmx_cpuid_t cpuid;
+ enum gmx_cpuid_acceleration acc;
+ int i, cnt;
+
+ if (argc < 2)
+ {
+ fprintf(stdout,
+ "Usage:\n\n%s [flags]\n\n"
+ "Available flags:\n"
+ "-vendor Print CPU vendor.\n"
+ "-brand Print CPU brand string.\n"
+ "-family Print CPU family version.\n"
+ "-model Print CPU model version.\n"
+ "-stepping Print CPU stepping version.\n"
+ "-features Print CPU feature flags.\n"
+ "-acceleration Print suggested GROMACS acceleration.\n",
+ argv[0]);
+ exit(0);
+ }
+
+ gmx_cpuid_init(&cpuid);
+
+ if (!strncmp(argv[1], "-vendor", 3))
+ {
+ printf("%s\n", gmx_cpuid_vendor_string[cpuid->vendor]);
+ }
+ else if (!strncmp(argv[1], "-brand", 3))
+ {
+ printf("%s\n", cpuid->brand);
+ }
+ else if (!strncmp(argv[1], "-family", 3))
+ {
+ printf("%d\n", cpuid->family);
+ }
+ else if (!strncmp(argv[1], "-model", 3))
+ {
+ printf("%d\n", cpuid->model);
+ }
+ else if (!strncmp(argv[1], "-stepping", 3))
+ {
+ printf("%d\n", cpuid->stepping);
+ }
+ else if (!strncmp(argv[1], "-features", 3))
+ {
+ cnt = 0;
+ for (i = 0; i < GMX_CPUID_NFEATURES; i++)
+ {
+ if (cpuid->feature[i] == 1)
+ {
+ if (cnt++ > 0)
+ {
+ printf(" ");
+ }
+ printf("%s", gmx_cpuid_feature_string[i]);
+ }
+ }
+ printf("\n");
+ }
+ else if (!strncmp(argv[1], "-acceleration", 3))
+ {
+ acc = gmx_cpuid_acceleration_suggest(cpuid);
+ fprintf(stdout, "%s\n", gmx_cpuid_acceleration_string[acc]);
+ }
+
+ gmx_cpuid_done(cpuid);
+
+
+ return 0;
+}
+
+#endif
--- /dev/null
- /* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
++/*
++ * This file is part of the GROMACS molecular simulation package.
+ *
++ * Copyright (c) 2012,2013, by the GROMACS development team, led by
++ * David van der Spoel, Berk Hess, Erik Lindahl, and including many
++ * others, as listed in the AUTHORS file in the top-level source
++ * directory and at http://www.gromacs.org.
+ *
- * This file is part of GROMACS.
- * Copyright (c) 2012-
- *
- * Written by the Gromacs development team under coordination of
- * David van der Spoel, Berk Hess, and Erik Lindahl.
- *
- * This library is free software; you can redistribute it and/or
++ * GROMACS is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
- * as published by the Free Software Foundation; either version 2
++ * as published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ *
- * To help us fund GROMACS development, we humbly ask that you cite
- * the research papers on the package. Check out http://www.gromacs.org
++ * GROMACS is distributed in the hope that it will be useful,
++ * but WITHOUT ANY WARRANTY; without even the implied warranty of
++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
++ * Lesser General Public License for more details.
++ *
++ * You should have received a copy of the GNU Lesser General Public
++ * License along with GROMACS; if not, see
++ * http://www.gnu.org/licenses, or write to the Free Software Foundation,
++ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
- * And Hey:
- * GROup of MAchos and Cynical Suckers
++ * If you want to redistribute modifications to GROMACS, please
++ * consider that scientific software is very special. Version
++ * control is crucial - bugs must be traceable. We will be happy to
++ * consider code for inclusion in the official distribution, but
++ * derived work must not be called official GROMACS. Details are found
++ * in the README & COPYING files - if they are missing, get the
++ * official version at http://www.gromacs.org.
++ *
++ * To help us fund GROMACS development, we humbly ask that you cite
++ * the research papers on the package. Check out http://www.gromacs.org.
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdlib.h>
+#include <assert.h>
+#include <string.h>
+
+#include "types/enums.h"
+#include "types/hw_info.h"
+#include "types/commrec.h"
+#include "gmx_fatal.h"
+#include "gmx_fatal_collective.h"
+#include "smalloc.h"
+#include "gpu_utils.h"
+#include "statutil.h"
+#include "gmx_detect_hardware.h"
+#include "main.h"
+#include "md_logging.h"
+
+#include "thread_mpi/threads.h"
+
- #ifdef HAVE_UNISTD_H
- #include <unistd.h>
- #endif
+#if ((defined(WIN32) || defined( _WIN32 ) || defined(WIN64) || defined( _WIN64 )) && !(defined (__CYGWIN__) || defined (__CYGWIN32__)))
+#include "windows.h"
+#endif
+
- /* Although we can't have more than 10 GPU different ID-s passed by the user as
- * the id-s are assumed to be represented by single digits, as multiple
- * processes can share a GPU, we can end up with more than 10 IDs.
- * To account for potential extreme cases we'll set the limit to a pretty
- * ridiculous number. */
- static unsigned int max_gpu_ids_user = 64;
++#ifdef GMX_GPU
++const gmx_bool bGPUBinary = TRUE;
++#else
++const gmx_bool bGPUBinary = FALSE;
++#endif
+
+static const char * invalid_gpuid_hint =
+ "A delimiter-free sequence of valid numeric IDs of available GPUs is expected.";
+
+/* The globally shared hwinfo structure. */
+static gmx_hw_info_t *hwinfo_g;
+/* A reference counter for the hwinfo structure */
+static int n_hwinfo = 0;
+/* A lock to protect the hwinfo structure */
+static tMPI_Thread_mutex_t hw_info_lock = TMPI_THREAD_MUTEX_INITIALIZER;
+
+
+/* FW decl. */
- static void limit_num_gpus_used(gmx_hw_info_t *hwinfo, int count);
++static void limit_num_gpus_used(gmx_gpu_opt_t *gpu_opt, int count);
++static int gmx_count_gpu_dev_unique(const gmx_gpu_info_t *gpu_info,
++ const gmx_gpu_opt_t *gpu_opt);
+
- static void sprint_gpus(char *sbuf, const gmx_gpu_info_t *gpu_info)
++static void sprint_gpus(char *sbuf, const gmx_gpu_info_t *gpu_info, gmx_bool bPrintAll)
+{
+ int i, ndev;
+ char stmp[STRLEN];
+
+ ndev = gpu_info->ncuda_dev;
+
+ sbuf[0] = '\0';
+ for (i = 0; i < ndev; i++)
+ {
+ get_gpu_device_info_string(stmp, gpu_info, i);
+ strcat(sbuf, " ");
+ strcat(sbuf, stmp);
+ if (i < ndev - 1)
+ {
+ strcat(sbuf, "\n");
+ }
+ }
+}
+
+static void print_gpu_detection_stats(FILE *fplog,
+ const gmx_gpu_info_t *gpu_info,
+ const t_commrec *cr)
+{
+ char onhost[266], stmp[STRLEN];
+ int ngpu;
+
++ if (!gpu_info->bDetectGPUs)
++ {
++ /* We skipped the detection, so don't print detection stats */
++ return;
++ }
++
+ ngpu = gpu_info->ncuda_dev;
+
+#if defined GMX_MPI && !defined GMX_THREAD_MPI
+ /* We only print the detection on one, of possibly multiple, nodes */
+ strncpy(onhost, " on host ", 10);
+ gmx_gethostname(onhost+9, 256);
+#else
+ /* We detect all relevant GPUs */
+ strncpy(onhost, "", 1);
+#endif
+
+ if (ngpu > 0)
+ {
- sprint_gpus(stmp, gpu_info);
++ sprint_gpus(stmp, gpu_info, TRUE);
+ md_print_warn(cr, fplog, "%d GPU%s detected%s:\n%s\n",
+ ngpu, (ngpu > 1) ? "s" : "", onhost, stmp);
+ }
+ else
+ {
+ md_print_warn(cr, fplog, "No GPUs detected%s\n", onhost);
+ }
+}
+
+static void print_gpu_use_stats(FILE *fplog,
+ const gmx_gpu_info_t *gpu_info,
++ const gmx_gpu_opt_t *gpu_opt,
+ const t_commrec *cr)
+{
+ char sbuf[STRLEN], stmp[STRLEN];
- int i, ngpu, ngpu_all;
++ int i, ngpu_comp, ngpu_use;
+
- ngpu = gpu_info->ncuda_dev_use;
- ngpu_all = gpu_info->ncuda_dev;
++ ngpu_comp = gpu_info->ncuda_dev_compatible;
++ ngpu_use = gpu_opt->ncuda_dev_use;
+
- /* Issue note if GPUs are available but not used */
- if (ngpu_all > 0 && ngpu < 1)
++ /* Issue a note if GPUs are available but not used */
++ if (ngpu_comp > 0 && ngpu_use < 1)
+ {
+ sprintf(sbuf,
+ "%d compatible GPU%s detected in the system, but none will be used.\n"
+ "Consider trying GPU acceleration with the Verlet scheme!",
- ngpu_all, (ngpu_all > 1) ? "s" : "");
++ ngpu_comp, (ngpu_comp > 1) ? "s" : "");
+ }
+ else
+ {
- sprintf(sbuf, "%d GPU%s %sselected for this run: ",
- ngpu, (ngpu > 1) ? "s" : "",
- gpu_info->bUserSet ? "user-" : "auto-");
- for (i = 0; i < ngpu; i++)
++ int ngpu_use_uniq;
++
++ ngpu_use_uniq = gmx_count_gpu_dev_unique(gpu_info, gpu_opt);
++
++ sprintf(sbuf, "%d GPU%s %sselected for this run.\n"
++ "Mapping of GPU%s to the %d PP rank%s in this node: ",
++ ngpu_use_uniq, (ngpu_use_uniq > 1) ? "s" : "",
++ gpu_opt->bUserSet ? "user-" : "auto-",
++ (ngpu_use > 1) ? "s" : "",
++ cr->nrank_pp_intranode,
++ (cr->nrank_pp_intranode > 1) ? "s" : "");
++
++ for (i = 0; i < ngpu_use; i++)
+ {
- sprintf(stmp, "#%d", get_gpu_device_id(gpu_info, i));
- if (i < ngpu - 1)
++ sprintf(stmp, "#%d", get_gpu_device_id(gpu_info, gpu_opt, i));
++ if (i < ngpu_use - 1)
+ {
+ strcat(stmp, ", ");
+ }
+ strcat(sbuf, stmp);
+ }
+ }
+ md_print_info(cr, fplog, "%s\n\n", sbuf);
+}
+
+/* Parse a "plain" GPU ID string which contains a sequence of digits corresponding
+ * to GPU IDs; the order will indicate the process/tMPI thread - GPU assignment. */
- static void parse_gpu_id_plain_string(const char *idstr, int *nid, int *idlist)
++static void parse_gpu_id_plain_string(const char *idstr, int *nid, int **idlist)
+{
- int i;
- size_t len_idstr;
++ int i;
+
- len_idstr = strlen(idstr);
++ *nid = strlen(idstr);
+
- if (len_idstr > max_gpu_ids_user)
- {
- gmx_fatal(FARGS, "%d GPU IDs provided, but only at most %d are supported",
- len_idstr, max_gpu_ids_user);
- }
-
- *nid = len_idstr;
++ snew(*idlist, *nid);
+
+ for (i = 0; i < *nid; i++)
+ {
+ if (idstr[i] < '0' || idstr[i] > '9')
+ {
+ gmx_fatal(FARGS, "Invalid character in GPU ID string: '%c'\n%s\n",
+ idstr[i], invalid_gpuid_hint);
+ }
- idlist[i] = idstr[i] - '0';
++ (*idlist)[i] = idstr[i] - '0';
+ }
+}
+
- void gmx_check_hw_runconf_consistency(FILE *fplog, gmx_hw_info_t *hwinfo,
- const t_commrec *cr, int ntmpi_requested,
- gmx_bool bUseGPU)
++static void parse_gpu_id_csv_string(const char *idstr, int *nid, int *idlist)
+{
- int npppn, ntmpi_pp, ngpu;
- char sbuf[STRLEN], th_or_proc[STRLEN], th_or_proc_plural[STRLEN], pernode[STRLEN];
- char gpu_plural[2];
- gmx_bool bGPUBin, btMPI, bMPI, bMaxMpiThreadsSet, bNthreadsAuto, bEmulateGPU;
- int ret;
- static tMPI_Thread_mutex_t cons_lock = TMPI_THREAD_MUTEX_INITIALIZER;
++ /* XXX implement cvs format to support more than 10 different GPUs in a box. */
++ gmx_incons("Not implemented yet");
++}
+
++void gmx_check_hw_runconf_consistency(FILE *fplog,
++ const gmx_hw_info_t *hwinfo,
++ const t_commrec *cr,
++ const gmx_hw_opt_t *hw_opt,
++ gmx_bool bUseGPU)
++{
++ int npppn, ntmpi_pp;
++ char sbuf[STRLEN], th_or_proc[STRLEN], th_or_proc_plural[STRLEN], pernode[STRLEN];
++ gmx_bool btMPI, bMPI, bMaxMpiThreadsSet, bNthreadsAuto, bEmulateGPU;
+
+ assert(hwinfo);
+ assert(cr);
+
+ /* Below we only do consistency checks for PP and GPUs,
+ * this is irrelevant for PME only nodes, so in that case we return
+ * here.
+ */
+ if (!(cr->duty & DUTY_PP))
+ {
+ return;
+ }
+
- /* We run this function only once, but must make sure that all threads
- that are alive run this function, so they get consistent data. We
- achieve this by mutual exclusion and returning if the structure is
- already properly checked & set */
- ret = tMPI_Thread_mutex_lock(&cons_lock);
- if (ret != 0)
++ btMPI = bMPI = FALSE;
++ bNthreadsAuto = FALSE;
++#if defined(GMX_THREAD_MPI)
++ btMPI = TRUE;
++ bNthreadsAuto = (hw_opt->nthreads_tmpi < 1);
++#elif defined(GMX_LIB_MPI)
++ bMPI = TRUE;
++#endif
++
++ /* GPU emulation detection is done later, but we need here as well
++ * -- uncool, but there's no elegant workaround */
++ bEmulateGPU = (getenv("GMX_EMULATE_GPU") != NULL);
++ bMaxMpiThreadsSet = (getenv("GMX_MAX_MPI_THREADS") != NULL);
++
++ /* check the acceleration mdrun is compiled with against hardware
++ capabilities */
++ /* TODO: Here we assume homogeneous hardware which is not necessarily
++ the case! Might not hurt to add an extra check over MPI. */
++ gmx_cpuid_acceleration_check(hwinfo->cpuid_info, fplog, SIMMASTER(cr));
++
++ /* NOTE: this print is only for and on one physical node */
++ print_gpu_detection_stats(fplog, &hwinfo->gpu_info, cr);
++
++ if (hwinfo->gpu_info.ncuda_dev_compatible > 0)
+ {
- gmx_fatal(FARGS, "Error locking cons mutex: %s", strerror(errno));
++ /* NOTE: this print is only for and on one physical node */
++ print_gpu_use_stats(fplog, &hwinfo->gpu_info, &hw_opt->gpu_opt, cr);
+ }
+
- if (!hwinfo->bConsistencyChecked)
++ /* Need to ensure that we have enough GPUs:
++ * - need one GPU per PP node
++ * - no GPU oversubscription with tMPI
++ * */
++ /* number of PP processes per node */
++ npppn = cr->nrank_pp_intranode;
++
++ pernode[0] = '\0';
++ th_or_proc_plural[0] = '\0';
++ if (btMPI)
+ {
- btMPI = bMPI = FALSE;
- bNthreadsAuto = FALSE;
- #if defined(GMX_THREAD_MPI)
- btMPI = TRUE;
- bNthreadsAuto = (ntmpi_requested < 1);
- #elif defined(GMX_LIB_MPI)
- bMPI = TRUE;
- #endif
++ sprintf(th_or_proc, "thread-MPI thread");
++ if (npppn > 1)
++ {
++ sprintf(th_or_proc_plural, "s");
++ }
++ }
++ else if (bMPI)
++ {
++ sprintf(th_or_proc, "MPI process");
++ if (npppn > 1)
++ {
++ sprintf(th_or_proc_plural, "es");
++ }
++ sprintf(pernode, " per node");
++ }
++ else
++ {
++ /* neither MPI nor tMPI */
++ sprintf(th_or_proc, "process");
++ }
+
- #ifdef GMX_GPU
- bGPUBin = TRUE;
- #else
- bGPUBin = FALSE;
- #endif
++ if (bUseGPU && hwinfo->gpu_info.ncuda_dev_compatible > 0 &&
++ !bEmulateGPU)
++ {
++ int ngpu_comp, ngpu_use;
++ char gpu_comp_plural[2], gpu_use_plural[2];
++
++ ngpu_comp = hwinfo->gpu_info.ncuda_dev_compatible;
++ ngpu_use = hw_opt->gpu_opt.ncuda_dev_use;
++
++ sprintf(gpu_comp_plural, "%s", (ngpu_comp> 1) ? "s" : "");
++ sprintf(gpu_use_plural, "%s", (ngpu_use > 1) ? "s" : "");
+
- /* GPU emulation detection is done later, but we need here as well
- * -- uncool, but there's no elegant workaround */
- bEmulateGPU = (getenv("GMX_EMULATE_GPU") != NULL);
- bMaxMpiThreadsSet = (getenv("GMX_MAX_MPI_THREADS") != NULL);
-
- /* check the acceleration mdrun is compiled with against hardware
- capabilities */
- /* TODO: Here we assume homogeneous hardware which is not necessarily
- the case! Might not hurt to add an extra check over MPI. */
- gmx_cpuid_acceleration_check(hwinfo->cpuid_info, fplog);
-
- /* Need to ensure that we have enough GPUs:
- * - need one GPU per PP node
- * - no GPU oversubscription with tMPI
- * => keep on the GPU support, otherwise turn off (or bail if forced)
- * */
- /* number of PP processes per node */
- npppn = cr->nrank_pp_intranode;
-
- pernode[0] = '\0';
- th_or_proc_plural[0] = '\0';
- if (btMPI)
++ /* number of tMPI threads auto-adjusted */
++ if (btMPI && bNthreadsAuto)
+ {
- sprintf(th_or_proc, "thread-MPI thread");
- if (npppn > 1)
++ if (hw_opt->gpu_opt.bUserSet && npppn < ngpu_use)
+ {
- sprintf(th_or_proc_plural, "s");
++ /* The user manually provided more GPUs than threads we
++ could automatically start. */
++ gmx_fatal(FARGS,
++ "%d GPU%s provided, but only %d PP thread-MPI thread%s coud be started.\n"
++ "%s requires one PP tread-MPI thread per GPU; use fewer GPUs%s.",
++ ngpu_use, gpu_use_plural,
++ npppn, th_or_proc_plural,
++ ShortProgram(), bMaxMpiThreadsSet ? "\nor allow more threads to be used" : "");
+ }
- }
- else if (bMPI)
- {
- sprintf(th_or_proc, "MPI process");
- if (npppn > 1)
++
++ if (!hw_opt->gpu_opt.bUserSet && npppn < ngpu_comp)
+ {
- sprintf(th_or_proc_plural, "es");
++ /* There are more GPUs than tMPI threads; we have
++ limited the number GPUs used. */
++ md_print_warn(cr, fplog,
++ "NOTE: %d GPU%s were detected, but only %d PP thread-MPI thread%s can be started.\n"
++ " %s can use one GPU per PP tread-MPI thread, so only %d GPU%s will be used.%s\n",
++ ngpu_comp, gpu_comp_plural,
++ npppn, th_or_proc_plural,
++ ShortProgram(), npppn,
++ npppn > 1 ? "s" : "",
++ bMaxMpiThreadsSet ? "\n Also, you can allow more threads to be used by increasing GMX_MAX_MPI_THREADS" : "");
+ }
- sprintf(pernode, " per node");
- }
- else
- {
- /* neither MPI nor tMPI */
- sprintf(th_or_proc, "process");
+ }
+
- if (bGPUBin)
++ if (hw_opt->gpu_opt.bUserSet)
+ {
- print_gpu_detection_stats(fplog, &hwinfo->gpu_info, cr);
++ if (ngpu_use != npppn)
++ {
++ gmx_fatal(FARGS,
++ "Incorrect launch configuration: mismatching number of PP %s%s and GPUs%s.\n"
++ "%s was started with %d PP %s%s%s, but you provided %d GPU%s.",
++ th_or_proc, btMPI ? "s" : "es", pernode,
++ ShortProgram(), npppn, th_or_proc,
++ th_or_proc_plural, pernode,
++ ngpu_use, gpu_use_plural);
++ }
+ }
-
- if (bUseGPU && hwinfo->bCanUseGPU && !bEmulateGPU)
++ else
+ {
- ngpu = hwinfo->gpu_info.ncuda_dev_use;
- sprintf(gpu_plural, "%s", (ngpu > 1) ? "s" : "");
-
- /* number of tMPI threads atuo-adjusted */
- if (btMPI && bNthreadsAuto)
++ if (ngpu_comp > npppn)
+ {
- if (npppn < ngpu)
- {
- if (hwinfo->gpu_info.bUserSet)
- {
- /* The user manually provided more GPUs than threads we
- could automatically start. */
- gmx_fatal(FARGS,
- "%d GPU%s provided, but only %d PP thread-MPI thread%s coud be started.\n"
- "%s requires one PP tread-MPI thread per GPU; use fewer GPUs%s.",
- ngpu, gpu_plural, npppn, th_or_proc_plural,
- ShortProgram(), bMaxMpiThreadsSet ? "\nor allow more threads to be used" : "");
- }
- else
- {
- /* There are more GPUs than tMPI threads; we have to
- limit the number GPUs used. */
- md_print_warn(cr, fplog,
- "NOTE: %d GPU%s were detected, but only %d PP thread-MPI thread%s can be started.\n"
- " %s can use one GPU per PP tread-MPI thread, so only %d GPU%s will be used.%s\n",
- ngpu, gpu_plural, npppn,
- th_or_proc_plural,
- ShortProgram(), npppn,
- npppn > 1 ? "s" : "",
- bMaxMpiThreadsSet ? "\n Also, you can allow more threads to be used by increasing GMX_MAX_MPI_THREADS" : "");
-
- if (cr->rank_pp_intranode == 0)
- {
- limit_num_gpus_used(hwinfo, npppn);
- ngpu = hwinfo->gpu_info.ncuda_dev_use;
- sprintf(gpu_plural, "%s", (ngpu > 1) ? "s" : "");
- }
- }
- }
++ md_print_warn(cr, fplog,
++ "NOTE: potentially sub-optimal launch configuration, %s started with less\n"
++ " PP %s%s%s than GPU%s available.\n"
++ " Each PP %s can use only one GPU, %d GPU%s%s will be used.\n",
++ ShortProgram(), th_or_proc,
++ th_or_proc_plural, pernode, gpu_comp_plural,
++ th_or_proc, npppn, gpu_use_plural, pernode);
+ }
+
- if (ngpu != npppn)
++ if (ngpu_use != npppn)
+ {
- if (hwinfo->gpu_info.bUserSet)
++ /* Avoid duplicate error messages.
++ * Unfortunately we can only do this at the physical node
++ * level, since the hardware setup and MPI process count
++ * might differ between physical nodes.
++ */
++ if (cr->rank_pp_intranode == 0)
+ {
+ gmx_fatal(FARGS,
+ "Incorrect launch configuration: mismatching number of PP %s%s and GPUs%s.\n"
- "%s was started with %d PP %s%s%s, but you provided %d GPU%s.",
++ "%s was started with %d PP %s%s%s, but only %d GPU%s were detected.",
+ th_or_proc, btMPI ? "s" : "es", pernode,
+ ShortProgram(), npppn, th_or_proc,
- th_or_proc_plural, pernode, ngpu, gpu_plural);
- }
- else
- {
- if (ngpu > npppn)
- {
- md_print_warn(cr, fplog,
- "NOTE: potentially sub-optimal launch configuration, %s started with less\n"
- " PP %s%s%s than GPU%s available.\n"
- " Each PP %s can use only one GPU, %d GPU%s%s will be used.\n",
- ShortProgram(), th_or_proc,
- th_or_proc_plural, pernode, gpu_plural,
- th_or_proc, npppn, gpu_plural, pernode);
-
- if (bMPI || (btMPI && cr->rank_pp_intranode == 0))
- {
- limit_num_gpus_used(hwinfo, npppn);
- ngpu = hwinfo->gpu_info.ncuda_dev_use;
- sprintf(gpu_plural, "%s", (ngpu > 1) ? "s" : "");
- }
- }
- else
- {
- /* Avoid duplicate error messages.
- * Unfortunately we can only do this at the physical node
- * level, since the hardware setup and MPI process count
- * might be differ over physical nodes.
- */
- if (cr->rank_pp_intranode == 0)
- {
- gmx_fatal(FARGS,
- "Incorrect launch configuration: mismatching number of PP %s%s and GPUs%s.\n"
- "%s was started with %d PP %s%s%s, but only %d GPU%s were detected.",
- th_or_proc, btMPI ? "s" : "es", pernode,
- ShortProgram(), npppn, th_or_proc,
- th_or_proc_plural, pernode, ngpu,
- gpu_plural);
- }
- }
++ th_or_proc_plural, pernode,
++ ngpu_use, gpu_use_plural);
+ }
+ }
++ }
+
- {
- int same_count;
++ {
++ int same_count;
+
- same_count = gmx_count_gpu_dev_shared(&(hwinfo->gpu_info));
++ same_count = gmx_count_gpu_dev_shared(&hw_opt->gpu_opt);
+
- if (btMPI && same_count > 0)
- {
- gmx_fatal(FARGS,
- "Invalid GPU assignment: can't share a GPU among multiple thread-MPI threads.\n"
- "Use MPI if you are sure that you want to assign GPU to multiple threads.");
- }
-
- if (same_count > 0)
- {
- md_print_warn(cr, fplog,
- "NOTE: Potentially sub-optimal launch configuration: you assigned %s to\n"
- " multiple %s%s; this should be avoided as it can cause\n"
- " performance loss.\n",
- same_count > 1 ? "GPUs" : "a GPU", th_or_proc, btMPI ? "s" : "es");
- }
++ if (same_count > 0)
++ {
++ md_print_info(cr, fplog,
++ "NOTE: You assigned %s to multiple %s%s.\n",
++ same_count > 1 ? "GPUs" : "a GPU", th_or_proc, btMPI ? "s" : "es");
+ }
- print_gpu_use_stats(fplog, &hwinfo->gpu_info, cr);
+ }
- hwinfo->bConsistencyChecked = TRUE;
- }
-
- ret = tMPI_Thread_mutex_unlock(&cons_lock);
- if (ret != 0)
- {
- gmx_fatal(FARGS, "Error unlocking cons mutex: %s", strerror(errno));
+ }
+
+#ifdef GMX_MPI
+ if (PAR(cr))
+ {
+ /* Avoid other ranks to continue after
+ inconsistency */
+ MPI_Barrier(cr->mpi_comm_mygroup);
+ }
+#endif
+
+}
+
- int gmx_count_gpu_dev_shared(const gmx_gpu_info_t *gpu_info)
++/* Return 0 if none of the GPU (per node) are shared among PP ranks.
++ *
++ * Sharing GPUs among multiple PP ranks is possible when the user passes
++ * GPU IDs. Here we check for sharing and return a non-zero value when
++ * this is detected. Note that the return value represents the number of
++ * PP rank pairs that share a device.
++ */
++int gmx_count_gpu_dev_shared(const gmx_gpu_opt_t *gpu_opt)
+{
+ int same_count = 0;
- int ngpu = gpu_info->ncuda_dev_use;
++ int ngpu = gpu_opt->ncuda_dev_use;
+
- if (gpu_info->bUserSet)
++ if (gpu_opt->bUserSet)
+ {
+ int i, j;
+
+ for (i = 0; i < ngpu - 1; i++)
+ {
+ for (j = i + 1; j < ngpu; j++)
+ {
- same_count += (gpu_info->cuda_dev_use[i] ==
- gpu_info->cuda_dev_use[j]);
++ same_count += (gpu_opt->cuda_dev_use[i] ==
++ gpu_opt->cuda_dev_use[j]);
+ }
+ }
+ }
+
+ return same_count;
+}
+
++/* Count and return the number of unique GPUs (per node) selected.
++ *
++ * As sharing GPUs among multiple PP ranks is possible when the user passes
++ * GPU IDs, the number of GPUs user (per node) can be different from the
++ * number of GPU IDs selected.
++ */
++static int gmx_count_gpu_dev_unique(const gmx_gpu_info_t *gpu_info,
++ const gmx_gpu_opt_t *gpu_opt)
++{
++ int i, uniq_count, ngpu;
++ int *uniq_ids;
++
++ assert(gpu_info);
++ assert(gpu_opt);
++
++ ngpu = gpu_info->ncuda_dev;
++ uniq_count = 0;
++
++ snew(uniq_ids, ngpu);
++
++ /* Each element in uniq_ids will be set to 0 or 1. The n-th element set
++ * to 1 indicates that the respective GPU was selected to be used. */
++ for (i = 0; i < gpu_opt->ncuda_dev_use; i++)
++ {
++ uniq_ids[get_gpu_device_id(gpu_info, gpu_opt, i)] = 1;
++ }
++ /* Count the devices used. */
++ for (i = 0; i < ngpu; i++)
++ {
++ uniq_count += uniq_ids[i];
++ }
++
++ sfree(uniq_ids);
++
++ return uniq_count;
++}
++
+
+/* Return the number of hardware threads supported by the current CPU.
+ * We assume that this is equal with the number of CPUs reported to be
+ * online by the OS at the time of the call.
+ */
- static int get_nthreads_hw_avail(FILE gmx_unused *fplog, const t_commrec gmx_unused *cr)
++static int get_nthreads_hw_avail(FILE *fplog, const t_commrec *cr)
+{
+ int ret = 0;
+
+#if ((defined(WIN32) || defined( _WIN32 ) || defined(WIN64) || defined( _WIN64 )) && !(defined (__CYGWIN__) || defined (__CYGWIN32__)))
+ /* Windows */
+ SYSTEM_INFO sysinfo;
+ GetSystemInfo( &sysinfo );
+ ret = sysinfo.dwNumberOfProcessors;
+#elif defined HAVE_SYSCONF
+ /* We are probably on Unix.
+ * Now check if we have the argument to use before executing the call
+ */
+#if defined(_SC_NPROCESSORS_ONLN)
+ ret = sysconf(_SC_NPROCESSORS_ONLN);
+#elif defined(_SC_NPROC_ONLN)
+ ret = sysconf(_SC_NPROC_ONLN);
+#elif defined(_SC_NPROCESSORS_CONF)
+ ret = sysconf(_SC_NPROCESSORS_CONF);
+#elif defined(_SC_NPROC_CONF)
+ ret = sysconf(_SC_NPROC_CONF);
+#endif /* End of check for sysconf argument values */
+
+#else
+ /* Neither windows nor Unix. No fscking idea how many CPUs we have! */
+ ret = -1;
+#endif
+
+ if (debug)
+ {
+ fprintf(debug, "Detected %d processors, will use this as the number "
+ "of supported hardware threads.\n", ret);
+ }
+
+#ifdef GMX_OMPENMP
+ if (ret != gmx_omp_get_num_procs())
+ {
+ md_print_warn(cr, fplog,
+ "Number of CPUs detected (%d) does not match the number reported by OpenMP (%d).\n"
+ "Consider setting the launch configuration manually!",
+ ret, gmx_omp_get_num_procs());
+ }
+#endif
+
+ return ret;
+}
+
++static void gmx_detect_gpus(FILE *fplog, const t_commrec *cr,
++ gmx_gpu_info_t *gpu_info)
++{
++#ifdef GMX_LIB_MPI
++ int rank_world;
++ MPI_Comm physicalnode_comm;
++#endif
++ int rank_local;
++
++ /* Under certain circumstances MPI ranks on the same physical node
++ * can not simultaneously access the same GPU(s). Therefore we run
++ * the detection only on one MPI rank per node and broadcast the info.
++ * Note that with thread-MPI only a single thread runs this code.
++ *
++ * TODO: We should also do CPU hardware detection only once on each
++ * physical node and broadcast it, instead of do it on every MPI rank.
++ */
++#ifdef GMX_LIB_MPI
++ /* A split of MPI_COMM_WORLD over physical nodes is only required here,
++ * so we create and destroy it locally.
++ */
++ MPI_Comm_rank(MPI_COMM_WORLD, &rank_world);
++ MPI_Comm_split(MPI_COMM_WORLD, gmx_physicalnode_id_hash(),
++ rank_world, &physicalnode_comm);
++ MPI_Comm_rank(physicalnode_comm, &rank_local);
++#else
++ /* Here there should be only one process, check this */
++ assert(cr->nnodes == 1 && cr->sim_nodeid == 0);
++
++ rank_local = 0;
++#endif
++
++ if (rank_local == 0)
++ {
++ char detection_error[STRLEN], sbuf[STRLEN];
++
++ if (detect_cuda_gpus(&hwinfo_g->gpu_info, detection_error) != 0)
++ {
++ if (detection_error != NULL && detection_error[0] != '\0')
++ {
++ sprintf(sbuf, ":\n %s\n", detection_error);
++ }
++ else
++ {
++ sprintf(sbuf, ".");
++ }
++ md_print_warn(cr, fplog,
++ "NOTE: Error occurred during GPU detection%s"
++ " Can not use GPU acceleration, will fall back to CPU kernels.\n",
++ sbuf);
++ }
++ }
++
++#ifdef GMX_LIB_MPI
++ /* Broadcast the GPU info to the other ranks within this node */
++ MPI_Bcast(&hwinfo_g->gpu_info.ncuda_dev, 1, MPI_INT, 0, physicalnode_comm);
++
++ if (hwinfo_g->gpu_info.ncuda_dev > 0)
++ {
++ int cuda_dev_size;
++
++ cuda_dev_size = hwinfo_g->gpu_info.ncuda_dev*sizeof_cuda_dev_info();
++
++ if (rank_local > 0)
++ {
++ hwinfo_g->gpu_info.cuda_dev =
++ (cuda_dev_info_ptr_t)malloc(cuda_dev_size);
++ }
++ MPI_Bcast(hwinfo_g->gpu_info.cuda_dev, cuda_dev_size, MPI_BYTE,
++ 0, physicalnode_comm);
++ MPI_Bcast(&hwinfo_g->gpu_info.ncuda_dev_compatible, 1, MPI_INT,
++ 0, physicalnode_comm);
++ }
++
++ MPI_Comm_free(&physicalnode_comm);
++#endif
++}
++
+gmx_hw_info_t *gmx_detect_hardware(FILE *fplog, const t_commrec *cr,
- gmx_bool bForceUseGPU, gmx_bool bTryUseGPU,
- const char *gpu_id)
++ gmx_bool bDetectGPUs)
+{
- int i;
- const char *env;
- char sbuf[STRLEN], stmp[STRLEN];
+ gmx_hw_info_t *hw;
- gmx_gpu_info_t gpuinfo_auto, gpuinfo_user;
- gmx_bool bGPUBin;
+ int ret;
+
+ /* make sure no one else is doing the same thing */
+ ret = tMPI_Thread_mutex_lock(&hw_info_lock);
+ if (ret != 0)
+ {
+ gmx_fatal(FARGS, "Error locking hwinfo mutex: %s", strerror(errno));
+ }
+
+ /* only initialize the hwinfo structure if it is not already initalized */
+ if (n_hwinfo == 0)
+ {
+ snew(hwinfo_g, 1);
- hwinfo_g->bConsistencyChecked = FALSE;
+
+ /* detect CPUID info; no fuss, we don't detect system-wide
+ * -- sloppy, but that's it for now */
+ if (gmx_cpuid_init(&hwinfo_g->cpuid_info) != 0)
+ {
+ gmx_fatal_collective(FARGS, cr, NULL, "CPUID detection failed!");
+ }
+
+ /* detect number of hardware threads */
+ hwinfo_g->nthreads_hw_avail = get_nthreads_hw_avail(fplog, cr);
+
+ /* detect GPUs */
- hwinfo_g->gpu_info.ncuda_dev_use = 0;
- hwinfo_g->gpu_info.cuda_dev_use = NULL;
- hwinfo_g->gpu_info.ncuda_dev = 0;
- hwinfo_g->gpu_info.cuda_dev = NULL;
-
- #ifdef GMX_GPU
- bGPUBin = TRUE;
- #else
- bGPUBin = FALSE;
- #endif
-
- /* Bail if binary is not compiled with GPU acceleration, but this is either
- * explicitly (-nb gpu) or implicitly (gpu ID passed) requested. */
- if (bForceUseGPU && !bGPUBin)
++ hwinfo_g->gpu_info.ncuda_dev = 0;
++ hwinfo_g->gpu_info.cuda_dev = NULL;
++ hwinfo_g->gpu_info.ncuda_dev_compatible = 0;
++
++ /* Run the detection if the binary was compiled with GPU support
++ * and we requested detection.
++ */
++ hwinfo_g->gpu_info.bDetectGPUs =
++ (bGPUBinary && bDetectGPUs &&
++ getenv("GMX_DISABLE_GPU_DETECTION") == NULL);
++ if (hwinfo_g->gpu_info.bDetectGPUs)
+ {
- gmx_fatal(FARGS, "GPU acceleration requested, but %s was compiled without GPU support!", ShortProgram());
- }
- if (gpu_id != NULL && !bGPUBin)
- {
- gmx_fatal(FARGS, "GPU ID string set, but %s was compiled without GPU support!", ShortProgram());
++ gmx_detect_gpus(fplog, cr, &hwinfo_g->gpu_info);
+ }
++ }
++ /* increase the reference counter */
++ n_hwinfo++;
+
- /* run the detection if the binary was compiled with GPU support */
- if (bGPUBin && getenv("GMX_DISABLE_GPU_DETECTION") == NULL)
- {
- char detection_error[STRLEN];
++ ret = tMPI_Thread_mutex_unlock(&hw_info_lock);
++ if (ret != 0)
++ {
++ gmx_fatal(FARGS, "Error unlocking hwinfo mutex: %s", strerror(errno));
++ }
+
- if (detect_cuda_gpus(&hwinfo_g->gpu_info, detection_error) != 0)
- {
- if (detection_error != NULL && detection_error[0] != '\0')
- {
- sprintf(sbuf, ":\n %s\n", detection_error);
- }
- else
- {
- sprintf(sbuf, ".");
- }
- md_print_warn(cr, fplog,
- "NOTE: Error occurred during GPU detection%s"
- " Can not use GPU acceleration, will fall back to CPU kernels.\n",
- sbuf);
- }
- }
++ return hwinfo_g;
++}
+
- if (bForceUseGPU || bTryUseGPU)
++void gmx_parse_gpu_ids(gmx_gpu_opt_t *gpu_opt)
++{
++ char *env;
++
++ if (gpu_opt->gpu_id != NULL && !bGPUBinary)
++ {
++ gmx_fatal(FARGS, "GPU ID string set, but %s was compiled without GPU support!", ShortProgram());
++ }
++
++ env = getenv("GMX_GPU_ID");
++ if (env != NULL && gpu_opt->gpu_id != NULL)
++ {
++ gmx_fatal(FARGS, "GMX_GPU_ID and -gpu_id can not be used at the same time");
++ }
++ if (env == NULL)
++ {
++ env = gpu_opt->gpu_id;
++ }
++
++ /* parse GPU IDs if the user passed any */
++ if (env != NULL)
++ {
++ parse_gpu_id_plain_string(env,
++ &gpu_opt->ncuda_dev_use,
++ &gpu_opt->cuda_dev_use);
++
++ if (gpu_opt->ncuda_dev_use == 0)
+ {
- env = getenv("GMX_GPU_ID");
- if (env != NULL && gpu_id != NULL)
- {
- gmx_fatal(FARGS, "GMX_GPU_ID and -gpu_id can not be used at the same time");
- }
- if (env == NULL)
- {
- env = gpu_id;
- }
++ gmx_fatal(FARGS, "Empty GPU ID string encountered.\n%s\n",
++ invalid_gpuid_hint);
++ }
+
- /* parse GPU IDs if the user passed any */
- if (env != NULL)
- {
- int *gpuid, *checkres;
- int nid, res;
++ gpu_opt->bUserSet = TRUE;
++ }
++}
+
- snew(gpuid, max_gpu_ids_user);
- snew(checkres, max_gpu_ids_user);
++void gmx_select_gpu_ids(FILE *fplog, const t_commrec *cr,
++ const gmx_gpu_info_t *gpu_info,
++ gmx_bool bForceUseGPU,
++ gmx_gpu_opt_t *gpu_opt)
++{
++ int i;
++ const char *env;
++ char sbuf[STRLEN], stmp[STRLEN];
+
- parse_gpu_id_plain_string(env, &nid, gpuid);
++ /* Bail if binary is not compiled with GPU acceleration, but this is either
++ * explicitly (-nb gpu) or implicitly (gpu ID passed) requested. */
++ if (bForceUseGPU && !bGPUBinary)
++ {
++ gmx_fatal(FARGS, "GPU acceleration requested, but %s was compiled without GPU support!", ShortProgram());
++ }
+
- if (nid == 0)
- {
- gmx_fatal(FARGS, "Empty GPU ID string encountered.\n%s\n",
- invalid_gpuid_hint);
- }
++ if (gpu_opt->bUserSet)
++ {
++ /* Check the GPU IDs passed by the user.
++ * (GPU IDs have been parsed by gmx_parse_gpu_ids before)
++ */
++ int *checkres;
++ int res;
+
- res = check_select_cuda_gpus(checkres, &hwinfo_g->gpu_info,
- gpuid, nid);
++ snew(checkres, gpu_opt->ncuda_dev_use);
+
- if (!res)
- {
- print_gpu_detection_stats(fplog, &hwinfo_g->gpu_info, cr);
-
- sprintf(sbuf, "Some of the requested GPUs do not exist, behave strangely, or are not compatible:\n");
- for (i = 0; i < nid; i++)
- {
- if (checkres[i] != egpuCompatible)
- {
- sprintf(stmp, " GPU #%d: %s\n",
- gpuid[i], gpu_detect_res_str[checkres[i]]);
- strcat(sbuf, stmp);
- }
- }
- gmx_fatal(FARGS, "%s", sbuf);
- }
++ res = check_selected_cuda_gpus(checkres, gpu_info, gpu_opt);
+
- hwinfo_g->gpu_info.bUserSet = TRUE;
++ if (!res)
++ {
++ print_gpu_detection_stats(fplog, gpu_info, cr);
+
- sfree(gpuid);
- sfree(checkres);
- }
- else
++ sprintf(sbuf, "Some of the requested GPUs do not exist, behave strangely, or are not compatible:\n");
++ for (i = 0; i < gpu_opt->ncuda_dev_use; i++)
+ {
- pick_compatible_gpus(&hwinfo_g->gpu_info);
- hwinfo_g->gpu_info.bUserSet = FALSE;
++ if (checkres[i] != egpuCompatible)
++ {
++ sprintf(stmp, " GPU #%d: %s\n",
++ gpu_opt->cuda_dev_use[i],
++ gpu_detect_res_str[checkres[i]]);
++ strcat(sbuf, stmp);
++ }
+ }
++ gmx_fatal(FARGS, "%s", sbuf);
++ }
+
- /* decide whether we can use GPU */
- hwinfo_g->bCanUseGPU = (hwinfo_g->gpu_info.ncuda_dev_use > 0);
- if (!hwinfo_g->bCanUseGPU && bForceUseGPU)
- {
- gmx_fatal(FARGS, "GPU acceleration requested, but no compatible GPUs were detected.");
- }
++ sfree(checkres);
++ }
++ else
++ {
++ pick_compatible_gpus(&hwinfo_g->gpu_info, gpu_opt);
++
++ if (gpu_opt->ncuda_dev_use > cr->nrank_pp_intranode)
++ {
++ /* We picked more GPUs than we can use: limit the number.
++ * We print detailed messages about this later in
++ * gmx_check_hw_runconf_consistency.
++ */
++ limit_num_gpus_used(gpu_opt, cr->nrank_pp_intranode);
+ }
++
++ gpu_opt->bUserSet = FALSE;
+ }
- /* increase the reference counter */
- n_hwinfo++;
+
- ret = tMPI_Thread_mutex_unlock(&hw_info_lock);
- if (ret != 0)
++ /* If the user asked for a GPU, check whether we have a GPU */
++ if (bForceUseGPU && gpu_info->ncuda_dev_compatible == 0)
+ {
- gmx_fatal(FARGS, "Error unlocking hwinfo mutex: %s", strerror(errno));
++ gmx_fatal(FARGS, "GPU acceleration requested, but no compatible GPUs were detected.");
+ }
-
- return hwinfo_g;
+}
+
- static void limit_num_gpus_used(gmx_hw_info_t *hwinfo, int count)
++static void limit_num_gpus_used(gmx_gpu_opt_t *gpu_opt, int count)
+{
+ int ndev_use;
+
- assert(hwinfo);
++ assert(gpu_opt);
+
- ndev_use = hwinfo->gpu_info.ncuda_dev_use;
++ ndev_use = gpu_opt->ncuda_dev_use;
+
+ if (count > ndev_use)
+ {
+ /* won't increase the # of GPUs */
+ return;
+ }
+
+ if (count < 1)
+ {
+ char sbuf[STRLEN];
+ sprintf(sbuf, "Limiting the number of GPUs to <1 doesn't make sense (detected %d, %d requested)!",
+ ndev_use, count);
+ gmx_incons(sbuf);
+ }
+
+ /* TODO: improve this implementation: either sort GPUs or remove the weakest here */
- hwinfo->gpu_info.ncuda_dev_use = count;
++ gpu_opt->ncuda_dev_use = count;
+}
+
+void gmx_hardware_info_free(gmx_hw_info_t *hwinfo)
+{
+ int ret;
+
+ ret = tMPI_Thread_mutex_lock(&hw_info_lock);
+ if (ret != 0)
+ {
+ gmx_fatal(FARGS, "Error locking hwinfo mutex: %s", strerror(errno));
+ }
+
+ /* decrease the reference counter */
+ n_hwinfo--;
+
+
+ if (hwinfo != hwinfo_g)
+ {
+ gmx_incons("hwinfo < hwinfo_g");
+ }
+
+ if (n_hwinfo < 0)
+ {
+ gmx_incons("n_hwinfo < 0");
+ }
+
+ if (n_hwinfo == 0)
+ {
+ gmx_cpuid_done(hwinfo_g->cpuid_info);
+ free_gpu_info(&hwinfo_g->gpu_info);
+ sfree(hwinfo_g);
+ }
+
+ ret = tMPI_Thread_mutex_unlock(&hw_info_lock);
+ if (ret != 0)
+ {
+ gmx_fatal(FARGS, "Error unlocking hwinfo mutex: %s", strerror(errno));
+ }
+}
--- /dev/null
- #include "mdrun.h"
+/*
+ * This file is part of the GROMACS molecular simulation package.
+ *
+ * Copyright (c) 2012, by the GROMACS development team, led by
+ * David van der Spoel, Berk Hess, Erik Lindahl, and including many
+ * others, as listed in the AUTHORS file in the top-level source
+ * directory and at http://www.gromacs.org.
+ *
+ * GROMACS is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ *
+ * GROMACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with GROMACS; if not, see
+ * http://www.gnu.org/licenses, or write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * If you want to redistribute modifications to GROMACS, please
+ * consider that scientific software is very special. Version
+ * control is crucial - bugs must be traceable. We will be happy to
+ * consider code for inclusion in the official distribution, but
+ * derived work must not be called official GROMACS. Details are found
+ * in the README & COPYING files - if they are missing, get the
+ * official version at http://www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the research papers on the package. Check out http://www.gromacs.org.
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+#if defined(HAVE_SCHED_H) && defined(HAVE_SCHED_GETAFFINITY)
+#define _GNU_SOURCE
+#include <sched.h>
+#include <sys/syscall.h>
+#endif
+#include <string.h>
+#include <errno.h>
+#include <assert.h>
+#include <stdio.h>
+#include "typedefs.h"
+#include "types/commrec.h"
+#include "types/hw_info.h"
+#include "gmx_cpuid.h"
+#include "gmx_omp.h"
+#include "gmx_omp_nthreads.h"
+#include "md_logging.h"
+#include "statutil.h"
+#include "gmx_thread_affinity.h"
+
+#include "thread_mpi/threads.h"
+
+
+static int
+get_thread_affinity_layout(FILE *fplog,
+ const t_commrec *cr,
+ const gmx_hw_info_t * hwinfo,
+ int nthreads,
+ int pin_offset, int * pin_stride,
+ const int **locality_order)
+{
+ int nhwthreads, npkg, ncores, nhwthreads_per_core, rc;
+ const int * pkg_id;
+ const int * core_id;
+ const int * hwthread_id;
+ gmx_bool bPickPinStride;
+
+ if (pin_offset < 0)
+ {
+ gmx_fatal(FARGS, "Negative thread pinning offset requested");
+ }
+ if (*pin_stride < 0)
+ {
+ gmx_fatal(FARGS, "Negative thread pinning stride requested");
+ }
+
+ rc = gmx_cpuid_topology(hwinfo->cpuid_info, &nhwthreads, &npkg, &ncores,
+ &nhwthreads_per_core,
+ &pkg_id, &core_id, &hwthread_id, locality_order);
+
+ if (rc != 0)
+ {
+ /* topology information not available or invalid, ignore it */
+ nhwthreads = hwinfo->nthreads_hw_avail;
+ *locality_order = NULL;
+
+ if (nhwthreads <= 0)
+ {
+ /* We don't know anything about the hardware, don't pin */
+ md_print_warn(cr, fplog,
+ "NOTE: We don't know how many logical cores we have, will not pin threads");
+
+ return -1;
+ }
+ }
+
+ if (nthreads > nhwthreads)
+ {
+ /* We are oversubscribing, don't pin */
+ md_print_warn(NULL, fplog,
+ "WARNING: Oversubscribing the CPU, will not pin threads");
+
+ return -1;
+ }
+
+ if (pin_offset + nthreads > nhwthreads)
+ {
+ /* We are oversubscribing, don't pin */
+ md_print_warn(NULL, fplog,
+ "WARNING: The requested pin offset is too large for the available logical cores,\n"
+ " will not pin threads");
+
+ return -1;
+ }
+
+
+ /* do we need to choose the pinning stride? */
+ bPickPinStride = (*pin_stride == 0);
+
+ if (bPickPinStride)
+ {
+ if (rc == 0 && pin_offset + nthreads*nhwthreads_per_core <= nhwthreads)
+ {
+ /* Put one thread on each physical core */
+ *pin_stride = nhwthreads_per_core;
+ }
+ else
+ {
+ /* We don't know if we have SMT, and if we do, we don't know
+ * if hw threads in the same physical core are consecutive.
+ * Without SMT the pinning layout should not matter too much.
+ * so we assume a consecutive layout and maximally spread out"
+ * the threads at equal threads per core.
+ * Note that IBM is the major non-x86 case with cpuid support
+ * and probably threads are already pinned by the queuing system,
+ * so we wouldn't end up here in the first place.
+ */
+ *pin_stride = (nhwthreads - pin_offset)/nthreads;
+ }
+ }
+ else
+ {
+ /* Check the placement of the thread with the largest index to make sure
+ * that the offset & stride doesn't cause pinning beyond the last hardware thread. */
+ if (pin_offset + (nthreads-1)*(*pin_stride) >= nhwthreads)
+ {
+ /* We are oversubscribing, don't pin */
+ md_print_warn(NULL, fplog,
+ "WARNING: The requested pinning stride is too large for the available logical cores,\n"
+ " will not pin threads");
+
+ return -1;
+ }
+ }
+
+ if (fplog != NULL)
+ {
+ fprintf(fplog, "Pinning threads with a%s logical core stride of %d\n",
+ bPickPinStride ? "n auto-selected" : " user-specified",
+ *pin_stride);
+ }
+
+ return 0;
+}
+
+/* Set CPU affinity. Can be important for performance.
+ On some systems (e.g. Cray) CPU Affinity is set by default.
+ But default assigning doesn't work (well) with only some ranks
+ having threads. This causes very low performance.
+ External tools have cumbersome syntax for setting affinity
+ in the case that only some ranks have threads.
+ Thus it is important that GROMACS sets the affinity internally
+ if only PME is using threads.
+ */
+void
+gmx_set_thread_affinity(FILE *fplog,
+ const t_commrec *cr,
+ gmx_hw_opt_t *hw_opt,
+ const gmx_hw_info_t *hwinfo)
+{
+ int nth_affinity_set, thread0_id_node,
+ nthread_local, nthread_node, nthread_hw_max, nphyscore;
+ int offset;
+ const int *locality_order;
+ int rc;
+
+ if (hw_opt->thread_affinity == threadaffOFF)
+ {
+ /* Nothing to do */
+ return;
+ }
+
+ /* If the tMPI thread affinity setting is not supported encourage the user
+ * to report it as it's either a bug or an exotic platform which we might
+ * want to support. */
+ if (tMPI_Thread_setaffinity_support() != TMPI_SETAFFINITY_SUPPORT_YES)
+ {
+ /* we know Mac OS doesn't support setting thread affinity, so there's
+ no point in warning the user in that case. In any other case
+ the user might be able to do something about it. */
+#ifndef __APPLE__
+ md_print_warn(NULL, fplog,
+ "Can not set thread affinities on the current platform. On NUMA systems this\n"
+ "can cause performance degradation. If you think your platform should support\n"
+ "setting affinities, contact the GROMACS developers.");
+#endif /* __APPLE__ */
+ return;
+ }
+
+ /* threads on this MPI process or TMPI thread */
+ if (cr->duty & DUTY_PP)
+ {
+ nthread_local = gmx_omp_nthreads_get(emntNonbonded);
+ }
+ else
+ {
+ nthread_local = gmx_omp_nthreads_get(emntPME);
+ }
+
+ /* map the current process to cores */
+ thread0_id_node = 0;
+ nthread_node = nthread_local;
+#ifdef GMX_MPI
+ if (PAR(cr) || MULTISIM(cr))
+ {
+ /* We need to determine a scan of the thread counts in this
+ * compute node.
+ */
+ MPI_Comm comm_intra;
+
+ MPI_Comm_split(MPI_COMM_WORLD, gmx_hostname_num(), cr->rank_intranode,
+ &comm_intra);
+ MPI_Scan(&nthread_local, &thread0_id_node, 1, MPI_INT, MPI_SUM, comm_intra);
+ /* MPI_Scan is inclusive, but here we need exclusive */
+ thread0_id_node -= nthread_local;
+ /* Get the total number of threads on this physical node */
+ MPI_Allreduce(&nthread_local, &nthread_node, 1, MPI_INT, MPI_SUM, comm_intra);
+ MPI_Comm_free(&comm_intra);
+ }
+#endif
+
+ if (hw_opt->thread_affinity == threadaffAUTO &&
+ nthread_node != hwinfo->nthreads_hw_avail)
+ {
+ if (nthread_node > 1 && nthread_node < hwinfo->nthreads_hw_avail)
+ {
+ md_print_warn(cr, fplog,
+ "NOTE: The number of threads is not equal to the number of (logical) cores\n"
+ " and the -pin option is set to auto: will not pin thread to cores.\n"
+ " This can lead to significant performance degradation.\n"
+ " Consider using -pin on (and -pinoffset in case you run multiple jobs).\n");
+ }
+
+ return;
+ }
+
+ offset = 0;
+ if (hw_opt->core_pinning_offset != 0)
+ {
+ offset = hw_opt->core_pinning_offset;
+ md_print_info(cr, fplog, "Applying core pinning offset %d\n", offset);
+ }
+
+ rc = get_thread_affinity_layout(fplog, cr, hwinfo,
+ nthread_node,
+ offset, &hw_opt->core_pinning_stride,
+ &locality_order);
+
+ if (rc != 0)
+ {
+ /* Incompatible layout, don't pin, warning was already issued */
+ return;
+ }
+
+ /* Set the per-thread affinity. In order to be able to check the success
+ * of affinity settings, we will set nth_affinity_set to 1 on threads
+ * where the affinity setting succeded and to 0 where it failed.
+ * Reducing these 0/1 values over the threads will give the total number
+ * of threads on which we succeeded.
+ */
+ nth_affinity_set = 0;
+#pragma omp parallel num_threads(nthread_local) reduction(+:nth_affinity_set)
+ {
+ int thread_id, thread_id_node;
+ int index, core;
+ gmx_bool setaffinity_ret;
+
+ thread_id = gmx_omp_get_thread_num();
+ thread_id_node = thread0_id_node + thread_id;
+ index = offset + thread_id_node*hw_opt->core_pinning_stride;
+ if (locality_order != NULL)
+ {
+ core = locality_order[index];
+ }
+ else
+ {
+ core = index;
+ }
+
+ setaffinity_ret = tMPI_Thread_setaffinity_single(tMPI_Thread_self(), core);
+
+ /* store the per-thread success-values of the setaffinity */
+ nth_affinity_set = (setaffinity_ret == 0);
+
+ if (debug)
+ {
+ fprintf(debug, "On rank %2d, thread %2d, index %2d, core %2d the affinity setting returned %d\n",
+ cr->nodeid, gmx_omp_get_thread_num(), index, core, setaffinity_ret);
+ }
+ }
+
+ if (nth_affinity_set > nthread_local)
+ {
+ char msg[STRLEN];
+
+ sprintf(msg, "Looks like we have set affinity for more threads than "
+ "we have (%d > %d)!\n", nth_affinity_set, nthread_local);
+ gmx_incons(msg);
+ }
+ else
+ {
+ /* check & warn if some threads failed to set their affinities */
+ if (nth_affinity_set != nthread_local)
+ {
+ char sbuf1[STRLEN], sbuf2[STRLEN];
+
+ /* sbuf1 contains rank info, while sbuf2 OpenMP thread info */
+ sbuf1[0] = sbuf2[0] = '\0';
+ /* Only add rank info if we have more than one rank. */
+ if (cr->nnodes > 1)
+ {
+#ifdef GMX_MPI
+#ifdef GMX_THREAD_MPI
+ sprintf(sbuf1, "In tMPI thread #%d: ", cr->nodeid);
+#else /* GMX_LIB_MPI */
+ sprintf(sbuf1, "In MPI process #%d: ", cr->nodeid);
+#endif
+#endif /* GMX_MPI */
+ }
+
+ if (nthread_local > 1)
+ {
+ sprintf(sbuf2, "for %d/%d thread%s ",
+ nthread_local - nth_affinity_set, nthread_local,
+ nthread_local > 1 ? "s" : "");
+ }
+
+ md_print_warn(NULL, fplog,
+ "WARNING: %sAffinity setting %sfailed.\n"
+ " This can cause performance degradation! If you think your setting are\n"
+ " correct, contact the GROMACS developers.",
+ sbuf1, sbuf2);
+ }
+ }
+ return;
+}
+
+/* Check the process affinity mask and if it is found to be non-zero,
+ * will honor it and disable mdrun internal affinity setting.
+ * Note that this will only work on Linux as we use a GNU feature.
+ */
+void
+gmx_check_thread_affinity_set(FILE *fplog, const t_commrec *cr,
+ gmx_hw_opt_t *hw_opt, int ncpus,
+ gmx_bool bAfterOpenmpInit)
+{
+#ifdef HAVE_SCHED_GETAFFINITY
+ cpu_set_t mask_current;
+ int i, ret, cpu_count, cpu_set;
+ gmx_bool bAllSet;
+
+ assert(hw_opt);
+ if (hw_opt->thread_affinity == threadaffOFF)
+ {
+ /* internal affinity setting is off, don't bother checking process affinity */
+ return;
+ }
+
+ CPU_ZERO(&mask_current);
+ if ((ret = sched_getaffinity(0, sizeof(cpu_set_t), &mask_current)) != 0)
+ {
+ /* failed to query affinity mask, will just return */
+ if (debug)
+ {
+ fprintf(debug, "Failed to query affinity mask (error %d)", ret);
+ }
+ return;
+ }
+
+ /* Before proceeding with the actual check, make sure that the number of
+ * detected CPUs is >= the CPUs in the current set.
+ * We need to check for CPU_COUNT as it was added only in glibc 2.6. */
+#ifdef CPU_COUNT
+ if (ncpus < CPU_COUNT(&mask_current))
+ {
+ if (debug)
+ {
+ fprintf(debug, "%d CPUs detected, but %d was returned by CPU_COUNT",
+ ncpus, CPU_COUNT(&mask_current));
+ }
+ return;
+ }
+#endif /* CPU_COUNT */
+
+ bAllSet = TRUE;
+ for (i = 0; (i < ncpus && i < CPU_SETSIZE); i++)
+ {
+ bAllSet = bAllSet && (CPU_ISSET(i, &mask_current) != 0);
+ }
+
+ if (!bAllSet)
+ {
+ if (hw_opt->thread_affinity == threadaffAUTO)
+ {
+ if (!bAfterOpenmpInit)
+ {
+ md_print_warn(cr, fplog,
+ "Non-default thread affinity set, disabling internal thread affinity");
+ }
+ else
+ {
+ md_print_warn(cr, fplog,
+ "Non-default thread affinity set probably by the OpenMP library,\n"
+ "disabling internal thread affinity");
+ }
+ hw_opt->thread_affinity = threadaffOFF;
+ }
+ else
+ {
+ /* Only warn once, at the last check (bAfterOpenmpInit==TRUE) */
+ if (bAfterOpenmpInit)
+ {
+ md_print_warn(cr, fplog,
+ "Overriding thread affinity set outside %s\n",
+ ShortProgram());
+ }
+ }
+
+ if (debug)
+ {
+ fprintf(debug, "Non-default affinity mask found\n");
+ }
+ }
+ else
+ {
+ if (debug)
+ {
+ fprintf(debug, "Default affinity mask found\n");
+ }
+ }
+#endif /* HAVE_SCHED_GETAFFINITY */
+}
--- /dev/null
- gmx_bool init_gpu(int mygpu, char *result_str, const gmx_gpu_info_t *gpu_info)
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2010,2012 The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * Gallium Rubidium Oxygen Manganese Argon Carbon Silicon
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+
+#include "smalloc.h"
+#include "string2.h"
+#include "types/hw_info.h"
+
+#include "gpu_utils.h"
+#include "../cuda_tools/cudautils.cuh"
+#include "memtestG80_core.h"
+
+
+#define QUICK_MEM 250 /*!< Amount of memory to be used in quick memtest. */
+#define QUICK_TESTS MOD_20_32BIT | LOGIC_4_ITER_SHMEM | RANDOM_BLOCKS /*!< Bit flag with type of tests
+ to run in quick memtest. */
+#define QUICK_ITER 3 /*!< Number of iterations in quick memtest. */
+
+#define FULL_TESTS 0x3FFF /*!< Bitflag with all test set on for full memetest. */
+#define FULL_ITER 25 /*!< Number of iterations in full memtest. */
+
+#define TIMED_TESTS MOD_20_32BIT | LOGIC_4_ITER_SHMEM | RANDOM_BLOCKS /*!< Bit flag with type of tests to
+ run in time constrained memtest. */
+
+static int cuda_max_device_count = 32; /*! Max number of devices supported by CUDA (for consistency checking).
+ In reality it 16 with CUDA <=v5.0, but let's stay on the safe side. */
+
+/*! Dummy kernel used for sanity checking. */
+__global__ void k_dummy_test(){}
+
+
+/*! Bit-flags which refer to memtestG80 test types and are used in do_memtest to specify which tests to run. */
+enum memtest_G80_test_types {
+ MOVING_INVERSIONS_10 = 0x1,
+ MOVING_INVERSIONS_RAND = 0x2,
+ WALKING_8BIT_M86 = 0x4,
+ WALKING_0_8BIT = 0x8,
+ WALKING_1_8BIT = 0x10,
+ WALKING_0_32BIT = 0x20,
+ WALKING_1_32BIT = 0x40,
+ RANDOM_BLOCKS = 0x80,
+ MOD_20_32BIT = 0x100,
+ LOGIC_1_ITER = 0x200,
+ LOGIC_4_ITER = 0x400,
+ LOGIC_1_ITER_SHMEM = 0x800,
+ LOGIC_4_ITER_SHMEM = 0x1000
+};
+
+
+/*!
+ * \brief Runs GPU sanity checks.
+ *
+ * Runs a series of checks to determine that the given GPU and underlying CUDA
+ * driver/runtime functions properly.
+ * Returns properties of a device with given ID or the one that has
+ * already been initialized earlier in the case if of \dev_id == -1.
+ *
+ * \param[in] dev_id the device ID of the GPU or -1 if the device has already been initialized
+ * \param[out] dev_prop pointer to the structure in which the device properties will be returned
+ * \returns 0 if the device looks OK
+ *
+ * TODO: introduce errors codes and handle errors more smoothly.
+ */
+static int do_sanity_checks(int dev_id, cudaDeviceProp *dev_prop)
+{
+ cudaError_t cu_err;
+ int dev_count, id;
+
+ cu_err = cudaGetDeviceCount(&dev_count);
+ if (cu_err != cudaSuccess)
+ {
+ fprintf(stderr, "Error %d while querying device count: %s\n", cu_err,
+ cudaGetErrorString(cu_err));
+ return -1;
+ }
+
+ /* no CUDA compatible device at all */
+ if (dev_count == 0)
+ return -1;
+
+ /* things might go horribly wrong if cudart is not compatible with the driver */
+ if (dev_count < 0 || dev_count > cuda_max_device_count)
+ return -1;
+
+ if (dev_id == -1) /* device already selected let's not destroy the context */
+ {
+ cu_err = cudaGetDevice(&id);
+ if (cu_err != cudaSuccess)
+ {
+ fprintf(stderr, "Error %d while querying device id: %s\n", cu_err,
+ cudaGetErrorString(cu_err));
+ return -1;
+ }
+ }
+ else
+ {
+ id = dev_id;
+ if (id > dev_count - 1) /* pfff there's no such device */
+ {
+ fprintf(stderr, "The requested device with id %d does not seem to exist (device count=%d)\n",
+ dev_id, dev_count);
+ return -1;
+ }
+ }
+
+ memset(dev_prop, 0, sizeof(cudaDeviceProp));
+ cu_err = cudaGetDeviceProperties(dev_prop, id);
+ if (cu_err != cudaSuccess)
+ {
+ fprintf(stderr, "Error %d while querying device properties: %s\n", cu_err,
+ cudaGetErrorString(cu_err));
+ return -1;
+ }
+
+ /* both major & minor is 9999 if no CUDA capable devices are present */
+ if (dev_prop->major == 9999 && dev_prop->minor == 9999)
+ return -1;
+ /* we don't care about emulation mode */
+ if (dev_prop->major == 0)
+ return -1;
+
+ if (id != -1)
+ {
+ cu_err = cudaSetDevice(id);
+ if (cu_err != cudaSuccess)
+ {
+ fprintf(stderr, "Error %d while switching to device #%d: %s\n",
+ cu_err, id, cudaGetErrorString(cu_err));
+ return -1;
+ }
+ }
+
+ /* try to execute a dummy kernel */
+ k_dummy_test<<<1, 512>>>();
+ if (cudaThreadSynchronize() != cudaSuccess)
+ {
+ return -1;
+ }
+
+ /* destroy context if we created one */
+ if (id != -1)
+ {
+#if CUDA_VERSION < 4000
+ cu_err = cudaThreadExit();
+ CU_RET_ERR(cu_err, "cudaThreadExit failed");
+#else
+ cu_err = cudaDeviceReset();
+ CU_RET_ERR(cu_err, "cudaDeviceReset failed");
+#endif
+ }
+
+ return 0;
+}
+
+
+/*!
+ * \brief Runs a set of memory tests specified by the given bit-flags.
+ * Tries to allocate and do the test on \p megs Mb memory or
+ * the greatest amount that can be allocated (>10Mb).
+ * In case if an error is detected it stops without finishing the remaining
+ * steps/iterations and returns greater then zero value.
+ * In case of other errors (e.g. kernel launch errors, device querying errors)
+ * -1 is returned.
+ *
+ * \param[in] which_tests variable with bit-flags of the requested tests
+ * \param[in] megs amount of memory that will be tested in MB
+ * \param[in] iter number of iterations
+ * \returns 0 if no error was detected, otherwise >0
+ */
+static int do_memtest(unsigned int which_tests, int megs, int iter)
+{
+ memtestState tester;
+ int i;
+ uint err_count; //, err_iter;
+
+ // no parameter check as this fn won't be called externally
+
+ // let's try to allocate the mem
+ while (!tester.allocate(megs) && (megs - 10 > 0))
+ { megs -= 10; tester.deallocate(); }
+
+ if (megs <= 10)
+ {
+ fprintf(stderr, "Unable to allocate GPU memory!\n");
+ return -1;
+ }
+
+ // clear the first 18 bits
+ which_tests &= 0x3FFF;
+ for (i = 0; i < iter; i++)
+ {
+ // Moving Inversions (ones and zeros)
+ if ((MOVING_INVERSIONS_10 & which_tests) == MOVING_INVERSIONS_10)
+ {
+ tester.gpuMovingInversionsOnesZeros(err_count);
+ if (err_count > 0)
+ return MOVING_INVERSIONS_10;
+ }
+ // Moving Inversions (random)
+ if ((MOVING_INVERSIONS_RAND & which_tests) == MOVING_INVERSIONS_RAND)
+ {
+ tester.gpuMovingInversionsRandom(err_count);
+ if (err_count > 0)
+ return MOVING_INVERSIONS_RAND;
+ }
+ // Memtest86 Walking 8-bit
+ if ((WALKING_8BIT_M86 & which_tests) == WALKING_8BIT_M86)
+ {
+ for (uint shift = 0; shift < 8; shift++)
+ {
+ tester.gpuWalking8BitM86(err_count, shift);
+ if (err_count > 0)
+ return WALKING_8BIT_M86;
+ }
+ }
+ // True Walking zeros (8-bit)
+ if ((WALKING_0_8BIT & which_tests) == WALKING_0_8BIT)
+ {
+ for (uint shift = 0; shift < 8; shift++)
+ {
+ tester.gpuWalking8Bit(err_count, false, shift);
+ if (err_count > 0)
+ return WALKING_0_8BIT;
+ }
+ }
+ // True Walking ones (8-bit)
+ if ((WALKING_1_8BIT & which_tests) == WALKING_1_8BIT)
+ {
+ for (uint shift = 0; shift < 8; shift++)
+ {
+ tester.gpuWalking8Bit(err_count, true, shift);
+ if (err_count > 0)
+ return WALKING_1_8BIT;
+ }
+ }
+ // Memtest86 Walking zeros (32-bit)
+ if ((WALKING_0_32BIT & which_tests) == WALKING_0_32BIT)
+ {
+ for (uint shift = 0; shift < 32; shift++)
+ {
+ tester.gpuWalking32Bit(err_count, false, shift);
+ if (err_count > 0)
+ return WALKING_0_32BIT;
+ }
+ }
+ // Memtest86 Walking ones (32-bit)
+ if ((WALKING_1_32BIT & which_tests) == WALKING_1_32BIT)
+ {
+ for (uint shift = 0; shift < 32; shift++)
+ {
+ tester.gpuWalking32Bit(err_count, true, shift);
+ if (err_count > 0)
+ return WALKING_1_32BIT;
+ }
+ }
+ // Random blocks
+ if ((RANDOM_BLOCKS & which_tests) == RANDOM_BLOCKS)
+ {
+ tester.gpuRandomBlocks(err_count,rand());
+ if (err_count > 0)
+ return RANDOM_BLOCKS;
+
+ }
+
+ // Memtest86 Modulo-20
+ if ((MOD_20_32BIT & which_tests) == MOD_20_32BIT)
+ {
+ for (uint shift = 0; shift < 20; shift++)
+ {
+ tester.gpuModuloX(err_count, shift, rand(), 20, 2);
+ if (err_count > 0)
+ return MOD_20_32BIT;
+ }
+ }
+ // Logic (one iteration)
+ if ((LOGIC_1_ITER & which_tests) == LOGIC_1_ITER)
+ {
+ tester.gpuShortLCG0(err_count,1);
+ if (err_count > 0)
+ return LOGIC_1_ITER;
+ }
+ // Logic (4 iterations)
+ if ((LOGIC_4_ITER & which_tests) == LOGIC_4_ITER)
+ {
+ tester.gpuShortLCG0(err_count,4);
+ if (err_count > 0)
+ return LOGIC_4_ITER;
+
+ }
+ // Logic (shared memory, one iteration)
+ if ((LOGIC_1_ITER_SHMEM & which_tests) == LOGIC_1_ITER_SHMEM)
+ {
+ tester.gpuShortLCG0Shmem(err_count,1);
+ if (err_count > 0)
+ return LOGIC_1_ITER_SHMEM;
+ }
+ // Logic (shared-memory, 4 iterations)
+ if ((LOGIC_4_ITER_SHMEM & which_tests) == LOGIC_4_ITER_SHMEM)
+ {
+ tester.gpuShortLCG0Shmem(err_count,4);
+ if (err_count > 0)
+ return LOGIC_4_ITER_SHMEM;
+ }
+ }
+
+ tester.deallocate();
+ return err_count;
+}
+
+/*! \brief Runs a quick memory test and returns 0 in case if no error is detected.
+ * If an error is detected it stops before completing the test and returns a
+ * value greater then 0. In case of other errors (e.g. kernel launch errors,
+ * device querying errors) -1 is returned.
+ *
+ * \param[in] dev_id the device id of the GPU or -1 if the device has already been selected
+ * \returns 0 if no error was detected, otherwise >0
+ */
+int do_quick_memtest(int dev_id)
+{
+ cudaDeviceProp dev_prop;
+ int devmem, res, time=0;
+
+ if (debug) { time = getTimeMilliseconds(); }
+
+ if (do_sanity_checks(dev_id, &dev_prop) != 0)
+ {
+ // something went wrong
+ return -1;
+ }
+
+ if (debug)
+ {
+ devmem = dev_prop.totalGlobalMem/(1024*1024); // in MiB
+ fprintf(debug, ">> Running QUICK memtests on %d MiB (out of total %d MiB), %d iterations\n",
+ QUICK_MEM, devmem, QUICK_ITER);
+ }
+
+ res = do_memtest(QUICK_TESTS, QUICK_MEM, QUICK_ITER);
+
+ if (debug)
+ {
+ fprintf(debug, "Q-RES = %d\n", res);
+ fprintf(debug, "Q-runtime: %d ms\n", getTimeMilliseconds() - time);
+ }
+
+ /* destroy context only if we created it */
+ if (dev_id !=-1) cudaThreadExit();
+ return res;
+}
+
+/*! \brief Runs a full memory test and returns 0 in case if no error is detected.
+ * If an error is detected it stops before completing the test and returns a
+ * value greater then 0. In case of other errors (e.g. kernel launch errors,
+ * device querying errors) -1 is returned.
+ *
+ * \param[in] dev_id the device id of the GPU or -1 if the device has already been selected
+ * \returns 0 if no error was detected, otherwise >0
+ */
+
+int do_full_memtest(int dev_id)
+{
+ cudaDeviceProp dev_prop;
+ int devmem, res, time=0;
+
+ if (debug) { time = getTimeMilliseconds(); }
+
+ if (do_sanity_checks(dev_id, &dev_prop) != 0)
+ {
+ // something went wrong
+ return -1;
+ }
+
+ devmem = dev_prop.totalGlobalMem/(1024*1024); // in MiB
+
+ if (debug)
+ {
+ fprintf(debug, ">> Running FULL memtests on %d MiB (out of total %d MiB), %d iterations\n",
+ devmem, devmem, FULL_ITER);
+ }
+
+ /* do all test on the entire memory */
+ res = do_memtest(FULL_TESTS, devmem, FULL_ITER);
+
+ if (debug)
+ {
+ fprintf(debug, "F-RES = %d\n", res);
+ fprintf(debug, "F-runtime: %d ms\n", getTimeMilliseconds() - time);
+ }
+
+ /* destroy context only if we created it */
+ if (dev_id != -1) cudaThreadExit();
+ return res;
+}
+
+/*! \brief Runs a time constrained memory test and returns 0 in case if no error is detected.
+ * If an error is detected it stops before completing the test and returns a value greater
+ * than zero. In case of other errors (e.g. kernel launch errors, device querying errors) -1
+ * is returned. Note, that test iterations are not interrupted therefor the total runtime of
+ * the test will always be multipple of one iteration's runtime.
+ *
+ * \param[in] dev_id the device id of the GPU or -1 if the device has laredy been selected
+ * \param[in] time_constr the time limit of the testing
+ * \returns 0 if no error was detected, otherwise >0
+ */
+int do_timed_memtest(int dev_id, int time_constr)
+{
+ cudaDeviceProp dev_prop;
+ int devmem, res=0, time=0, startt;
+
+ if (debug) { time = getTimeMilliseconds(); }
+
+ time_constr *= 1000; /* convert to ms for convenience */
+ startt = getTimeMilliseconds();
+
+ if (do_sanity_checks(dev_id, &dev_prop) != 0)
+ {
+ // something went wrong
+ return -1;
+ }
+
+ devmem = dev_prop.totalGlobalMem/(1024*1024); // in MiB
+
+ if (debug)
+ {
+ fprintf(debug, ">> Running time constrained memtests on %d MiB (out of total %d MiB), time limit of %d s \n",
+ devmem, devmem, time_constr);
+ }
+
+ /* do the TIMED_TESTS set, one step at a time on the entire memory
+ that can be allocated, and stop when the given time is exceeded */
+ while ( ((int)getTimeMilliseconds() - startt) < time_constr)
+ {
+ res = do_memtest(TIMED_TESTS, devmem, 1);
+ if (res != 0) break;
+ }
+
+ if (debug)
+ {
+ fprintf(debug, "T-RES = %d\n", res);
+ fprintf(debug, "T-runtime: %d ms\n", getTimeMilliseconds() - time);
+ }
+
+ /* destroy context only if we created it */
+ if (dev_id != -1) cudaThreadExit();
+ return res;
+}
+
+/*! \brief Initializes the GPU with the given index.
+ *
+ * The varible \mygpu is the index of the GPU to initialize in the
+ * gpu_info.cuda_dev array.
+ *
+ * \param[in] mygpu index of the GPU to initialize
+ * \param[out] result_str the message related to the error that occurred
+ * during the initialization (if there was any).
+ * \param[in] gpu_info GPU info of all detected devices in the system.
++ * \param[in] gpu_opt options for using the GPUs in gpu_info
+ * \returns true if no error occurs during initialization.
+ */
- if (mygpu < 0 || mygpu >= gpu_info->ncuda_dev_use)
++gmx_bool init_gpu(int mygpu, char *result_str,
++ const gmx_gpu_info_t *gpu_info,
++ const gmx_gpu_opt_t *gpu_opt)
+{
+ cudaError_t stat;
+ char sbuf[STRLEN];
+ int gpuid;
+
+ assert(gpu_info);
+ assert(result_str);
+
- gpu_info->ncuda_dev_use, gpu_info->bUserSet ? "user" : "auto", mygpu);
++ if (mygpu < 0 || mygpu >= gpu_opt->ncuda_dev_use)
+ {
+ sprintf(sbuf, "Trying to initialize an inexistent GPU: "
+ "there are %d %s-selected GPU(s), but #%d was requested.",
- gpuid = gpu_info->cuda_dev[gpu_info->cuda_dev_use[mygpu]].id;
++ gpu_opt->ncuda_dev_use, gpu_opt->bUserSet ? "user" : "auto", mygpu);
+ gmx_incons(sbuf);
+ }
+
- * Given the list of GPUs available in the system the it checks each gpu in
- * gpu_info->cuda_dev and puts the the indices (into gpu_info->cuda_dev) of
- * the compatible ones into cuda_dev_use with this marking the respective
- * GPUs as "available for use."
++ gpuid = gpu_info->cuda_dev[gpu_opt->cuda_dev_use[mygpu]].id;
+
+ stat = cudaSetDevice(gpuid);
+ strncpy(result_str, cudaGetErrorString(stat), STRLEN);
+
+ if (debug)
+ {
+ fprintf(stderr, "Initialized GPU ID #%d: %s\n", gpuid, gpu_info->cuda_dev[gpuid].prop.name);
+ }
+
+ return (stat == cudaSuccess);
+}
+
+/*! \brief Frees up the CUDA GPU used by the active context at the time of calling.
+ *
+ * The context is explicitly destroyed and therefore all data uploaded to the GPU
+ * is lost. This should only be called when none of this data is required anymore.
+ *
+ * \param[out] result_str the message related to the error that occurred
+ * during the initialization (if there was any).
+ * \returns true if no error occurs during the freeing.
+ */
+gmx_bool free_gpu(char *result_str)
+{
+ cudaError_t stat;
+
+ assert(result_str);
+
+ if (debug)
+ {
+ int gpuid;
+ stat = cudaGetDevice(&gpuid);
+ CU_RET_ERR(stat, "cudaGetDevice failed");
+ fprintf(stderr, "Cleaning up context on GPU ID #%d\n", gpuid);
+ }
+
+#if CUDA_VERSION < 4000
+ stat = cudaThreadExit();
+#else
+ stat = cudaDeviceReset();
+#endif
+ strncpy(result_str, cudaGetErrorString(stat), STRLEN);
+
+ return (stat == cudaSuccess);
+}
+
+/*! \brief Returns true if the gpu characterized by the device properties is
+ * supported by the native gpu acceleration.
+ *
+ * \param[in] dev_prop the CUDA device properties of the gpus to test.
+ * \returns true if the GPU properties passed indicate a compatible
+ * GPU, otherwise false.
+ */
+static bool is_gmx_supported_gpu(const cudaDeviceProp *dev_prop)
+{
+ return (dev_prop->major >= 2);
+}
+
+/*! \brief Helper function that checks whether a given GPU status indicates compatible GPU.
+ *
+ * \param[in] stat GPU status.
+ * \returns true if the provided status is egpuCompatible, otherwise false.
+ */
+static bool is_compatible_gpu(int stat)
+{
+ return (stat == egpuCompatible);
+}
+
+/*! \brief Checks if a GPU with a given ID is supported by the native GROMACS acceleration.
+ *
+ * Returns a status value which indicates compatibility or one of the following
+ * errors: incompatibility, insistence, or insanity (=unexpected behavior).
+ * It also returns the respective device's properties in \dev_prop (if applicable).
+ *
+ * \param[in] dev_id the ID of the GPU to check.
+ * \param[out] dev_prop the CUDA device properties of the device checked.
+ * \returns the status of the requested device
+ */
+static int is_gmx_supported_gpu_id(int dev_id, cudaDeviceProp *dev_prop)
+{
+ cudaError_t stat;
+ int ndev;
+
+ stat = cudaGetDeviceCount(&ndev);
+ if (stat != cudaSuccess)
+ {
+ return egpuInsane;
+ }
+
+ if (dev_id > ndev - 1)
+ {
+ return egpuNonexistent;
+ }
+
+ /* TODO: currently we do not make a distinction between the type of errors
+ * that can appear during sanity checks. This needs to be improved, e.g if
+ * the dummy test kernel fails to execute with a "device busy message" we
+ * should appropriately report that the device is busy instead of insane.
+ */
+ if (do_sanity_checks(dev_id, dev_prop) == 0)
+ {
+ if (is_gmx_supported_gpu(dev_prop))
+ {
+ return egpuCompatible;
+ }
+ else
+ {
+ return egpuIncompatible;
+ }
+ }
+ else
+ {
+ return egpuInsane;
+ }
+}
+
+
+/*! \brief Detect all NVIDIA GPUs in the system.
+ *
+ * Will detect every NVIDIA GPU supported by the device driver in use. Also
+ * check for the compatibility of each and fill the gpu_info->cuda_dev array
+ * with the required information on each the device: ID, device properties,
+ * status.
+ *
+ * \param[in] gpu_info pointer to structure holding GPU information.
+ * \param[out] err_str The error message of any CUDA API error that caused
+ * the detection to fail (if there was any). The memory
+ * the pointer points to should be managed externally.
+ * \returns non-zero if the detection encountered a failure, zero otherwise.
+ */
+int detect_cuda_gpus(gmx_gpu_info_t *gpu_info, char *err_str)
+{
+ int i, ndev, checkres, retval;
+ cudaError_t stat;
+ cudaDeviceProp prop;
+ cuda_dev_info_t *devs;
+
+ assert(gpu_info);
+ assert(err_str);
+
++ gpu_info->ncuda_dev_compatible = 0;
++
+ ndev = 0;
+ devs = NULL;
+
+ stat = cudaGetDeviceCount(&ndev);
+ if (stat != cudaSuccess)
+ {
+ const char *s;
+
+ /* cudaGetDeviceCount failed which means that there is something
+ * wrong with the machine: driver-runtime mismatch, all GPUs being
+ * busy in exclusive mode, or some other condition which should
+ * result in us issuing a warning a falling back to CPUs. */
+ retval = -1;
+ s = cudaGetErrorString(stat);
+ strncpy(err_str, s, STRLEN*sizeof(err_str[0]));
+ }
+ else
+ {
+ snew(devs, ndev);
+ for (i = 0; i < ndev; i++)
+ {
+ checkres = is_gmx_supported_gpu_id(i, &prop);
+
+ devs[i].id = i;
+ devs[i].prop = prop;
+ devs[i].stat = checkres;
++
++ if (checkres == egpuCompatible)
++ {
++ gpu_info->ncuda_dev_compatible++;
++ }
+ }
+ retval = 0;
+ }
+
+ gpu_info->ncuda_dev = ndev;
+ gpu_info->cuda_dev = devs;
+
+ return retval;
+}
+
+/*! \brief Select the GPUs compatible with the native GROMACS acceleration.
+ *
+ * This function selects the compatible gpus and initializes
+ * gpu_info->cuda_dev_use and gpu_info->ncuda_dev_use.
+ *
- * \param[in] gpu_info pointer to structure holding GPU information
++ * Given the list of GPUs available in the system check each device in
++ * gpu_info->cuda_dev and place the indices of the compatible GPUs into
++ * cuda_dev_use with this marking the respective GPUs as "available for use."
+ * Note that \detect_cuda_gpus must have been called before.
+ *
- void pick_compatible_gpus(gmx_gpu_info_t *gpu_info)
++ * \param[in] gpu_info pointer to structure holding GPU information
++ * \param[in,out] gpu_opt pointer to structure holding GPU options
+ */
- gpu_info->ncuda_dev_use = ncompat;
- snew(gpu_info->cuda_dev_use, ncompat);
- memcpy(gpu_info->cuda_dev_use, compat, ncompat*sizeof(*compat));
++void pick_compatible_gpus(const gmx_gpu_info_t *gpu_info,
++ gmx_gpu_opt_t *gpu_opt)
+{
+ int i, ncompat;
+ int *compat;
+
+ assert(gpu_info);
+ /* cuda_dev/ncuda_dev have to be either NULL/0 or not (NULL/0) */
+ assert((gpu_info->ncuda_dev != 0 ? 0 : 1) ^ (gpu_info->cuda_dev == NULL ? 0 : 1));
+
+ snew(compat, gpu_info->ncuda_dev);
+ ncompat = 0;
+ for (i = 0; i < gpu_info->ncuda_dev; i++)
+ {
+ if (is_compatible_gpu(gpu_info->cuda_dev[i].stat))
+ {
+ ncompat++;
+ compat[ncompat - 1] = i;
+ }
+ }
+
- * Given the a list of GPU devide IDs in \requested_devs, check for the
- * existence and compatibility of the respective GPUs and fill in \gpu_info
- * with the collected information. Also provide the caller with an array with
++ gpu_opt->ncuda_dev_use = ncompat;
++ snew(gpu_opt->cuda_dev_use, ncompat);
++ memcpy(gpu_opt->cuda_dev_use, compat, ncompat*sizeof(*compat));
+ sfree(compat);
+}
+
+/*! \brief Check the existence/compatibility of a set of GPUs specified by their device IDs.
+ *
- * \param[in] requested_devs array of requested device IDs
- * \param[in] count number of IDs in \requested_devs
- * \returns TRUE if every requested GPU is compatible
++ * Given the a list of gpu->ncuda_dev_use GPU device IDs stored in
++ * gpu_opt->cuda_dev_use check the existence and compatibility
++ * of the respective GPUs. Also provide the caller with an array containing
+ * the result of checks in \checkres.
+ *
+ * \param[out] checkres check result for each ID passed in \requested_devs
+ * \param[in] gpu_info pointer to structure holding GPU information
- gmx_bool check_select_cuda_gpus(int *checkres, gmx_gpu_info_t *gpu_info,
- const int *requested_devs, int count)
++ * \param[out] gpu_opt pointer to structure holding GPU options
++ * \returns TRUE if every the requested GPUs are compatible
+ */
- assert(requested_devs);
- assert(count >= 0);
++gmx_bool check_selected_cuda_gpus(int *checkres,
++ const gmx_gpu_info_t *gpu_info,
++ gmx_gpu_opt_t *gpu_opt)
+{
+ int i, id;
+ bool bAllOk;
+
+ assert(checkres);
+ assert(gpu_info);
- if (count == 0)
++ assert(gpu_opt->ncuda_dev_use >= 0);
+
- gpu_info->ncuda_dev_use = count;
- snew(gpu_info->cuda_dev_use, count);
++ if (gpu_opt->ncuda_dev_use == 0)
+ {
+ return TRUE;
+ }
+
++ assert(gpu_opt->cuda_dev_use);
++
+ /* we will assume that all GPUs requested are valid IDs,
+ otherwise we'll bail anyways */
- for (i = 0; i < count; i++)
+
+ bAllOk = true;
- id = requested_devs[i];
++ for (i = 0; i < gpu_opt->ncuda_dev_use; i++)
+ {
- gpu_info->cuda_dev_use[i] = id;
++ id = gpu_opt->cuda_dev_use[i];
+
+ /* devices are stored in increasing order of IDs in cuda_dev */
- sfree(gpu_info->cuda_dev_use);
++ gpu_opt->cuda_dev_use[i] = id;
+
+ checkres[i] = (id >= gpu_info->ncuda_dev) ?
+ egpuNonexistent : gpu_info->cuda_dev[id].stat;
+
+ bAllOk = bAllOk && is_compatible_gpu(checkres[i]);
+ }
+
+ return bAllOk;
+}
+
+/*! \brief Frees the cuda_dev and cuda_dev_use array fields of \gpu_info.
+ *
+ * \param[in] gpu_info pointer to structure holding GPU information
+ */
+void free_gpu_info(const gmx_gpu_info_t *gpu_info)
+{
+ if (gpu_info == NULL)
+ {
+ return;
+ }
+
- int get_gpu_device_id(const gmx_gpu_info_t *gpu_info, int idx)
+ sfree(gpu_info->cuda_dev);
+}
+
+/*! \brief Formats and returns a device information string for a given GPU.
+ *
+ * Given an index *directly* into the array of available GPUs (cuda_dev)
+ * returns a formatted info string for the respective GPU which includes
+ * ID, name, compute capability, and detection status.
+ *
+ * \param[out] s pointer to output string (has to be allocated externally)
+ * \param[in] gpu_info pointer to structure holding GPU information
+ * \param[in] index an index *directly* into the array of available GPUs
+ */
+void get_gpu_device_info_string(char *s, const gmx_gpu_info_t *gpu_info, int index)
+{
+ assert(s);
+ assert(gpu_info);
+
+ if (index < 0 && index >= gpu_info->ncuda_dev)
+ {
+ return;
+ }
+
+ cuda_dev_info_t *dinfo = &gpu_info->cuda_dev[index];
+
+ bool bGpuExists =
+ dinfo->stat == egpuCompatible ||
+ dinfo->stat == egpuIncompatible;
+
+ if (!bGpuExists)
+ {
+ sprintf(s, "#%d: %s, stat: %s",
+ dinfo->id, "N/A",
+ gpu_detect_res_str[dinfo->stat]);
+ }
+ else
+ {
+ sprintf(s, "#%d: NVIDIA %s, compute cap.: %d.%d, ECC: %3s, stat: %s",
+ dinfo->id, dinfo->prop.name,
+ dinfo->prop.major, dinfo->prop.minor,
+ dinfo->prop.ECCEnabled ? "yes" : " no",
+ gpu_detect_res_str[dinfo->stat]);
+ }
+}
+
+/*! \brief Returns the device ID of the GPU with a given index into the array of used GPUs.
+ *
+ * Getter function which, given an index into the array of GPUs in use
+ * (cuda_dev_use) -- typically a tMPI/MPI rank --, returns the device ID of the
+ * respective CUDA GPU.
+ *
+ * \param[in] gpu_info pointer to structure holding GPU information
++ * \param[in] gpu_opt pointer to structure holding GPU options
+ * \param[in] idx index into the array of used GPUs
+ * \returns device ID of the requested GPU
+ */
- if (idx < 0 && idx >= gpu_info->ncuda_dev_use)
++int get_gpu_device_id(const gmx_gpu_info_t *gpu_info,
++ const gmx_gpu_opt_t *gpu_opt,
++ int idx)
+{
+ assert(gpu_info);
- return gpu_info->cuda_dev[gpu_info->cuda_dev_use[idx]].id;
++ assert(gpu_opt);
++ if (idx < 0 && idx >= gpu_opt->ncuda_dev_use)
+ {
+ return -1;
+ }
+
++ return gpu_info->cuda_dev[gpu_opt->cuda_dev_use[idx]].id;
+}
+
+/*! \brief Returns the device ID of the GPU currently in use.
+ *
+ * The GPU used is the one that is active at the time of the call in the active context.
+ *
+ * \param[in] gpu_info pointer to structure holding GPU information
+ * \returns device ID of the GPU in use at the time of the call
+ */
+int get_current_gpu_device_id(void)
+{
+ int gpuid;
+ CU_RET_ERR(cudaGetDevice(&gpuid), "cudaGetDevice failed");
+
+ return gpuid;
+}
++
++/*! \brief Returns the size of the cuda_dev_info struct.
++ *
++ * The size of cuda_dev_info can be used for allocation and communication.
++ *
++ * \returns size in bytes of cuda_dev_info
++ */
++size_t sizeof_cuda_dev_info(void)
++{
++ return sizeof(cuda_dev_info);
++}
--- /dev/null
- #if defined GMX_LIB_MPI && defined GMX_IS_BGQ
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * VERSION 3.2.0
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2004, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * GROningen Mixture of Alchemy and Childrens' Stories
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <string.h>
+#include "gmx_fatal.h"
+#include "main.h"
+#include "smalloc.h"
+#include "network.h"
+#include "copyrite.h"
+#include "statutil.h"
+#include <ctype.h>
+#include "macros.h"
++#include "string2.h"
+
+#include "gromacs/utility/gmxmpi.h"
+
+
+/* The source code in this file should be thread-safe.
+ Please keep it that way. */
+
+gmx_bool gmx_mpi_initialized(void)
+{
+ int n;
+#ifndef GMX_MPI
+ return 0;
+#else
+ MPI_Initialized(&n);
+
+ return n;
+#endif
+}
+
+void gmx_fill_commrec_from_mpi(t_commrec *cr)
+{
+#ifndef GMX_MPI
+ gmx_call("gmx_fill_commrec_from_mpi");
+#else
+ if (!gmx_mpi_initialized())
+ {
+ gmx_comm("MPI has not been initialized properly");
+ }
+
+ cr->nnodes = gmx_node_num();
+ cr->nodeid = gmx_node_rank();
+ cr->sim_nodeid = cr->nodeid;
+ cr->mpi_comm_mysim = MPI_COMM_WORLD;
+ cr->mpi_comm_mygroup = MPI_COMM_WORLD;
+
+#endif
+}
+
+t_commrec *init_commrec()
+{
+ t_commrec *cr;
+
+ snew(cr, 1);
+
+#ifdef GMX_LIB_MPI
+ gmx_fill_commrec_from_mpi(cr);
+#else
+ cr->mpi_comm_mysim = NULL;
+ cr->mpi_comm_mygroup = NULL;
+ cr->nnodes = 1;
+ cr->sim_nodeid = 0;
+ cr->nodeid = cr->sim_nodeid;
+#endif
+
+ // TODO cr->duty should not be initialized here
+ cr->duty = (DUTY_PP | DUTY_PME);
+
+#if defined GMX_LIB_MPI && !defined MPI_IN_PLACE_EXISTS
+ /* initialize the MPI_IN_PLACE replacement buffers */
+ snew(cr->mpb, 1);
+ cr->mpb->ibuf = NULL;
+ cr->mpb->libuf = NULL;
+ cr->mpb->fbuf = NULL;
+ cr->mpb->dbuf = NULL;
+ cr->mpb->ibuf_alloc = 0;
+ cr->mpb->libuf_alloc = 0;
+ cr->mpb->fbuf_alloc = 0;
+ cr->mpb->dbuf_alloc = 0;
+#endif
+
+ return cr;
+}
+
+t_commrec *reinitialize_commrec_for_this_thread(const t_commrec *cro)
+{
+#ifdef GMX_THREAD_MPI
+ t_commrec *cr;
+
+ /* make a thread-specific commrec */
+ snew(cr, 1);
+ /* now copy the whole thing, so settings like the number of PME nodes
+ get propagated. */
+ *cr = *cro;
+
+ /* and we start setting our own thread-specific values for things */
+ gmx_fill_commrec_from_mpi(cr);
+
+ // TODO cr->duty should not be initialized here
+ cr->duty = (DUTY_PP | DUTY_PME);
+
+ return cr;
+#else
+ return NULL;
+#endif
+}
+
+int gmx_node_num(void)
+{
+#ifndef GMX_MPI
+ return 1;
+#else
+ int i;
+ (void) MPI_Comm_size(MPI_COMM_WORLD, &i);
+ return i;
+#endif
+}
+
+int gmx_node_rank(void)
+{
+#ifndef GMX_MPI
+ return 0;
+#else
+ int i;
+ (void) MPI_Comm_rank(MPI_COMM_WORLD, &i);
+ return i;
+#endif
+}
+
- #ifdef GMX_IS_BGQ
++#if defined GMX_LIB_MPI && defined GMX_TARGET_BGQ
+#include <spi/include/kernel/location.h>
+#endif
+
++int gmx_physicalnode_id_hash(void)
++{
++ int hash_int;
++
++#ifndef GMX_LIB_MPI
++ /* We have a single physical node */
++ hash_int = 0;
++#else
++ int resultlen;
++ char mpi_hostname[MPI_MAX_PROCESSOR_NAME];
++
++ /* This procedure can only differentiate nodes with different names.
++ * Architectures where different physical nodes have identical names,
++ * such as IBM Blue Gene, should use an architecture specific solution.
++ */
++ MPI_Get_processor_name(mpi_hostname, &resultlen);
++
++ /* The string hash function returns an unsigned int. We cast to an int.
++ * Negative numbers are converted to positive by setting the sign bit to 0.
++ * This makes the hash one bit smaller.
++ * A 63-bit hash (with 64-bit int) should be enough for unique node hashes,
++ * even on a million node machine. 31 bits might not be enough though!
++ */
++ hash_int =
++ (int)gmx_string_fullhash_func(mpi_hostname, gmx_string_hash_init);
++ if (hash_int < 0)
++ {
++ hash_int -= INT_MIN;
++ }
++#endif
++
++ return hash_int;
++}
++
++/* TODO: this function should be fully replaced by gmx_physicalnode_id_hash */
+int gmx_hostname_num()
+{
+#ifndef GMX_MPI
+ return 0;
+#else
+#ifdef GMX_THREAD_MPI
+ /* thread-MPI currently puts the thread number in the process name,
+ * we might want to change this, as this is inconsistent with what
+ * most MPI implementations would do when running on a single node.
+ */
+ return 0;
+#else
+ int resultlen, hostnum, i, j;
+ char mpi_hostname[MPI_MAX_PROCESSOR_NAME], hostnum_str[MPI_MAX_PROCESSOR_NAME];
+
+ MPI_Get_processor_name(mpi_hostname, &resultlen);
- #ifdef GMX_IS_BGQ
++#ifdef GMX_TARGET_BGQ
+ Personality_t personality;
+ Kernel_GetPersonality(&personality, sizeof(personality));
+ /* Each MPI rank has a unique coordinate in a 6-dimensional space
+ (A,B,C,D,E,T), with dimensions A-E corresponding to different
+ physical nodes, and T within each node. Each node has sixteen
+ physical cores, each of which can have up to four hardware
+ threads, so 0 <= T <= 63 (but the maximum value of T depends on
+ the confituration of ranks and OpenMP threads per
+ node). However, T is irrelevant for computing a suitable return
+ value for gmx_hostname_num().
+ */
+ hostnum = personality.Network_Config.Acoord;
+ hostnum *= personality.Network_Config.Bnodes;
+ hostnum += personality.Network_Config.Bcoord;
+ hostnum *= personality.Network_Config.Cnodes;
+ hostnum += personality.Network_Config.Ccoord;
+ hostnum *= personality.Network_Config.Dnodes;
+ hostnum += personality.Network_Config.Dcoord;
+ hostnum *= personality.Network_Config.Enodes;
+ hostnum += personality.Network_Config.Ecoord;
+#else
+ /* This procedure can only differentiate nodes with host names
+ * that end on unique numbers.
+ */
+ i = 0;
+ j = 0;
+ /* Only parse the host name up to the first dot */
+ while (i < resultlen && mpi_hostname[i] != '.')
+ {
+ if (isdigit(mpi_hostname[i]))
+ {
+ hostnum_str[j++] = mpi_hostname[i];
+ }
+ i++;
+ }
+ hostnum_str[j] = '\0';
+ if (j == 0)
+ {
+ hostnum = 0;
+ }
+ else
+ {
+ /* Use only the last 9 decimals, so we don't overflow an int */
+ hostnum = strtol(hostnum_str + max(0, j-9), NULL, 10);
+ }
+#endif
+
+ if (debug)
+ {
+ fprintf(debug, "In gmx_hostname_num: hostname '%s', hostnum %d\n",
+ mpi_hostname, hostnum);
++#ifdef GMX_TARGET_BGQ
+ fprintf(debug,
+ "Torus ID A: %d / %d B: %d / %d C: %d / %d D: %d / %d E: %d / %d\nNode ID T: %d / %d core: %d / %d hardware thread: %d / %d\n",
+ personality.Network_Config.Acoord,
+ personality.Network_Config.Anodes,
+ personality.Network_Config.Bcoord,
+ personality.Network_Config.Bnodes,
+ personality.Network_Config.Ccoord,
+ personality.Network_Config.Cnodes,
+ personality.Network_Config.Dcoord,
+ personality.Network_Config.Dnodes,
+ personality.Network_Config.Ecoord,
+ personality.Network_Config.Enodes,
+ Kernel_ProcessorCoreID(),
+ 16,
+ Kernel_ProcessorID(),
+ 64,
+ Kernel_ProcessorThreadID(),
+ 4);
+#endif
+ }
+ return hostnum;
+#endif
+#endif
+}
+
+void gmx_setup_nodecomm(FILE gmx_unused *fplog, t_commrec *cr)
+{
+ gmx_nodecomm_t *nc;
+ int n, rank, hostnum, ng, ni;
+
+ /* Many MPI implementations do not optimize MPI_Allreduce
+ * (and probably also other global communication calls)
+ * for multi-core nodes connected by a network.
+ * We can optimize such communication by using one MPI call
+ * within each node and one between the nodes.
+ * For MVAPICH2 and Intel MPI this reduces the time for
+ * the global_stat communication by 25%
+ * for 2x2-core 3 GHz Woodcrest connected by mixed DDR/SDR Infiniband.
+ * B. Hess, November 2007
+ */
+
+ nc = &cr->nc;
+
+ nc->bUse = FALSE;
+#ifndef GMX_THREAD_MPI
+#ifdef GMX_MPI
+ MPI_Comm_size(cr->mpi_comm_mygroup, &n);
+ MPI_Comm_rank(cr->mpi_comm_mygroup, &rank);
+
+ hostnum = gmx_hostname_num();
+
+ if (debug)
+ {
+ fprintf(debug, "In gmx_setup_nodecomm: splitting communicator of size %d\n", n);
+ }
+
+
+ /* The intra-node communicator, split on node number */
+ MPI_Comm_split(cr->mpi_comm_mygroup, hostnum, rank, &nc->comm_intra);
+ MPI_Comm_rank(nc->comm_intra, &nc->rank_intra);
+ if (debug)
+ {
+ fprintf(debug, "In gmx_setup_nodecomm: node rank %d rank_intra %d\n",
+ rank, nc->rank_intra);
+ }
+ /* The inter-node communicator, split on rank_intra.
+ * We actually only need the one for rank=0,
+ * but it is easier to create them all.
+ */
+ MPI_Comm_split(cr->mpi_comm_mygroup, nc->rank_intra, rank, &nc->comm_inter);
+ /* Check if this really created two step communication */
+ MPI_Comm_size(nc->comm_inter, &ng);
+ MPI_Comm_size(nc->comm_intra, &ni);
+ if (debug)
+ {
+ fprintf(debug, "In gmx_setup_nodecomm: groups %d, my group size %d\n",
+ ng, ni);
+ }
+
+ if (getenv("GMX_NO_NODECOMM") == NULL &&
+ ((ng > 1 && ng < n) || (ni > 1 && ni < n)))
+ {
+ nc->bUse = TRUE;
+ if (fplog)
+ {
+ fprintf(fplog, "Using two step summing over %d groups of on average %.1f processes\n\n",
+ ng, (real)n/(real)ng);
+ }
+ if (nc->rank_intra > 0)
+ {
+ MPI_Comm_free(&nc->comm_inter);
+ }
+ }
+ else
+ {
+ /* One group or all processes in a separate group, use normal summing */
+ MPI_Comm_free(&nc->comm_inter);
+ MPI_Comm_free(&nc->comm_intra);
+ if (debug)
+ {
+ fprintf(debug, "In gmx_setup_nodecomm: not unsing separate inter- and intra-node communicators.\n");
+ }
+ }
+#endif
+#else
+ /* tMPI runs only on a single node so just use the nodeid */
+ nc->rank_intra = cr->nodeid;
+#endif
+}
+
+void gmx_init_intranode_counters(t_commrec *cr)
+{
+ /* counters for PP+PME and PP-only processes on my physical node */
+ int nrank_intranode, rank_intranode;
+ int nrank_pp_intranode, rank_pp_intranode;
+ /* thread-MPI is not initialized when not running in parallel */
+#if defined GMX_MPI && !defined GMX_THREAD_MPI
+ int nrank_world, rank_world;
+ int i, mynum, *num, *num_s, *num_pp, *num_pp_s;
+
+ MPI_Comm_size(MPI_COMM_WORLD, &nrank_world);
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank_world);
+
+ /* Get the node number from the hostname to identify the nodes */
+ mynum = gmx_hostname_num();
+
+ /* We can't rely on MPI_IN_PLACE, so we need send and receive buffers */
+ snew(num, nrank_world);
+ snew(num_s, nrank_world);
+ snew(num_pp, nrank_world);
+ snew(num_pp_s, nrank_world);
+
+ num_s[rank_world] = mynum;
+ num_pp_s[rank_world] = (cr->duty & DUTY_PP) ? mynum : -1;
+
+ MPI_Allreduce(num_s, num, nrank_world, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+ MPI_Allreduce(num_pp_s, num_pp, nrank_world, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+
+ nrank_intranode = 0;
+ rank_intranode = 0;
+ nrank_pp_intranode = 0;
+ rank_pp_intranode = 0;
+ for (i = 0; i < nrank_world; i++)
+ {
+ if (num[i] == mynum)
+ {
+ nrank_intranode++;
+ if (i < rank_world)
+ {
+ rank_intranode++;
+ }
+ }
+ if ((cr->duty & DUTY_PP) && num_pp[i] == mynum)
+ {
+ nrank_pp_intranode++;
+ if (i < rank_world)
+ {
+ rank_pp_intranode++;
+ }
+ }
+ }
+ sfree(num);
+ sfree(num_s);
+ sfree(num_pp);
+ sfree(num_pp_s);
+#else
+ /* Serial or thread-MPI code: we run within a single physical node */
+ nrank_intranode = cr->nnodes;
+ rank_intranode = cr->sim_nodeid;
+ nrank_pp_intranode = cr->nnodes - cr->npmenodes;
+ rank_pp_intranode = cr->nodeid;
+#endif
+
+ if (debug)
+ {
+ char sbuf[STRLEN];
+ if (cr->duty & DUTY_PP && cr->duty & DUTY_PME)
+ {
+ sprintf(sbuf, "PP+PME");
+ }
+ else
+ {
+ sprintf(sbuf, "%s", cr->duty & DUTY_PP ? "PP" : "PME");
+ }
+ fprintf(debug, "On %3s node %d: nrank_intranode=%d, rank_intranode=%d, "
+ "nrank_pp_intranode=%d, rank_pp_intranode=%d\n",
+ sbuf, cr->sim_nodeid,
+ nrank_intranode, rank_intranode,
+ nrank_pp_intranode, rank_pp_intranode);
+ }
+
+ cr->nrank_intranode = nrank_intranode;
+ cr->rank_intranode = rank_intranode;
+ cr->nrank_pp_intranode = nrank_pp_intranode;
+ cr->rank_pp_intranode = rank_pp_intranode;
+}
+
+
+void gmx_barrier(const t_commrec *cr)
+{
+#ifndef GMX_MPI
+ gmx_call("gmx_barrier");
+#else
+ MPI_Barrier(cr->mpi_comm_mygroup);
+#endif
+}
+
+void gmx_abort(int gmx_unused noderank, int gmx_unused nnodes, int gmx_unused errorno)
+{
+#ifndef GMX_MPI
+ gmx_call("gmx_abort");
+#else
+#ifdef GMX_THREAD_MPI
+ fprintf(stderr, "Halting program %s\n", ShortProgram());
+ gmx_thanx(stderr);
+ exit(1);
+#else
+ if (nnodes > 1)
+ {
+ fprintf(stderr, "Halting parallel program %s on CPU %d out of %d\n",
+ ShortProgram(), noderank, nnodes);
+ }
+ else
+ {
+ fprintf(stderr, "Halting program %s\n", ShortProgram());
+ }
+
+ gmx_thanx(stderr);
+ MPI_Abort(MPI_COMM_WORLD, errorno);
+ exit(1);
+#endif
+#endif
+}
+
+void gmx_bcast(int nbytes, void *b, const t_commrec *cr)
+{
+#ifndef GMX_MPI
+ gmx_call("gmx_bast");
+#else
+ MPI_Bcast(b, nbytes, MPI_BYTE, MASTERRANK(cr), cr->mpi_comm_mygroup);
+#endif
+}
+
+void gmx_bcast_sim(int nbytes, void *b, const t_commrec *cr)
+{
+#ifndef GMX_MPI
+ gmx_call("gmx_bast");
+#else
+ MPI_Bcast(b, nbytes, MPI_BYTE, MASTERRANK(cr), cr->mpi_comm_mysim);
+#endif
+}
+
+void gmx_sumd(int nr, double r[], const t_commrec *cr)
+{
+#ifndef GMX_MPI
+ gmx_call("gmx_sumd");
+#else
+#if defined(MPI_IN_PLACE_EXISTS) || defined(GMX_THREAD_MPI)
+ if (cr->nc.bUse)
+ {
+ if (cr->nc.rank_intra == 0)
+ {
+ /* Use two step summing. */
+ MPI_Reduce(MPI_IN_PLACE, r, nr, MPI_DOUBLE, MPI_SUM, 0,
+ cr->nc.comm_intra);
+ /* Sum the roots of the internal (intra) buffers. */
+ MPI_Allreduce(MPI_IN_PLACE, r, nr, MPI_DOUBLE, MPI_SUM,
+ cr->nc.comm_inter);
+ }
+ else
+ {
+ /* This is here because of the silly MPI specification
+ that MPI_IN_PLACE should be put in sendbuf instead of recvbuf */
+ MPI_Reduce(r, NULL, nr, MPI_DOUBLE, MPI_SUM, 0, cr->nc.comm_intra);
+ }
+ MPI_Bcast(r, nr, MPI_DOUBLE, 0, cr->nc.comm_intra);
+ }
+ else
+ {
+ MPI_Allreduce(MPI_IN_PLACE, r, nr, MPI_DOUBLE, MPI_SUM,
+ cr->mpi_comm_mygroup);
+ }
+#else
+ int i;
+
+ if (nr > cr->mpb->dbuf_alloc)
+ {
+ cr->mpb->dbuf_alloc = nr;
+ srenew(cr->mpb->dbuf, cr->mpb->dbuf_alloc);
+ }
+ if (cr->nc.bUse)
+ {
+ /* Use two step summing */
+ MPI_Allreduce(r, cr->mpb->dbuf, nr, MPI_DOUBLE, MPI_SUM, cr->nc.comm_intra);
+ if (cr->nc.rank_intra == 0)
+ {
+ /* Sum with the buffers reversed */
+ MPI_Allreduce(cr->mpb->dbuf, r, nr, MPI_DOUBLE, MPI_SUM,
+ cr->nc.comm_inter);
+ }
+ MPI_Bcast(r, nr, MPI_DOUBLE, 0, cr->nc.comm_intra);
+ }
+ else
+ {
+ MPI_Allreduce(r, cr->mpb->dbuf, nr, MPI_DOUBLE, MPI_SUM,
+ cr->mpi_comm_mygroup);
+ for (i = 0; i < nr; i++)
+ {
+ r[i] = cr->mpb->dbuf[i];
+ }
+ }
+#endif
+#endif
+}
+
+void gmx_sumf(int nr, float r[], const t_commrec *cr)
+{
+#ifndef GMX_MPI
+ gmx_call("gmx_sumf");
+#else
+#if defined(MPI_IN_PLACE_EXISTS) || defined(GMX_THREAD_MPI)
+ if (cr->nc.bUse)
+ {
+ /* Use two step summing. */
+ if (cr->nc.rank_intra == 0)
+ {
+ MPI_Reduce(MPI_IN_PLACE, r, nr, MPI_FLOAT, MPI_SUM, 0,
+ cr->nc.comm_intra);
+ /* Sum the roots of the internal (intra) buffers */
+ MPI_Allreduce(MPI_IN_PLACE, r, nr, MPI_FLOAT, MPI_SUM,
+ cr->nc.comm_inter);
+ }
+ else
+ {
+ /* This is here because of the silly MPI specification
+ that MPI_IN_PLACE should be put in sendbuf instead of recvbuf */
+ MPI_Reduce(r, NULL, nr, MPI_FLOAT, MPI_SUM, 0, cr->nc.comm_intra);
+ }
+ MPI_Bcast(r, nr, MPI_FLOAT, 0, cr->nc.comm_intra);
+ }
+ else
+ {
+ MPI_Allreduce(MPI_IN_PLACE, r, nr, MPI_FLOAT, MPI_SUM, cr->mpi_comm_mygroup);
+ }
+#else
+ int i;
+
+ if (nr > cr->mpb->fbuf_alloc)
+ {
+ cr->mpb->fbuf_alloc = nr;
+ srenew(cr->mpb->fbuf, cr->mpb->fbuf_alloc);
+ }
+ if (cr->nc.bUse)
+ {
+ /* Use two step summing */
+ MPI_Allreduce(r, cr->mpb->fbuf, nr, MPI_FLOAT, MPI_SUM, cr->nc.comm_intra);
+ if (cr->nc.rank_intra == 0)
+ {
+ /* Sum with the buffers reversed */
+ MPI_Allreduce(cr->mpb->fbuf, r, nr, MPI_FLOAT, MPI_SUM,
+ cr->nc.comm_inter);
+ }
+ MPI_Bcast(r, nr, MPI_FLOAT, 0, cr->nc.comm_intra);
+ }
+ else
+ {
+ MPI_Allreduce(r, cr->mpb->fbuf, nr, MPI_FLOAT, MPI_SUM,
+ cr->mpi_comm_mygroup);
+ for (i = 0; i < nr; i++)
+ {
+ r[i] = cr->mpb->fbuf[i];
+ }
+ }
+#endif
+#endif
+}
+
+void gmx_sumi(int nr, int r[], const t_commrec *cr)
+{
+#ifndef GMX_MPI
+ gmx_call("gmx_sumi");
+#else
+#if defined(MPI_IN_PLACE_EXISTS) || defined(GMX_THREAD_MPI)
+ if (cr->nc.bUse)
+ {
+ /* Use two step summing */
+ if (cr->nc.rank_intra == 0)
+ {
+ MPI_Reduce(MPI_IN_PLACE, r, nr, MPI_INT, MPI_SUM, 0, cr->nc.comm_intra);
+ /* Sum with the buffers reversed */
+ MPI_Allreduce(MPI_IN_PLACE, r, nr, MPI_INT, MPI_SUM, cr->nc.comm_inter);
+ }
+ else
+ {
+ /* This is here because of the silly MPI specification
+ that MPI_IN_PLACE should be put in sendbuf instead of recvbuf */
+ MPI_Reduce(r, NULL, nr, MPI_INT, MPI_SUM, 0, cr->nc.comm_intra);
+ }
+ MPI_Bcast(r, nr, MPI_INT, 0, cr->nc.comm_intra);
+ }
+ else
+ {
+ MPI_Allreduce(MPI_IN_PLACE, r, nr, MPI_INT, MPI_SUM, cr->mpi_comm_mygroup);
+ }
+#else
+ int i;
+
+ if (nr > cr->mpb->ibuf_alloc)
+ {
+ cr->mpb->ibuf_alloc = nr;
+ srenew(cr->mpb->ibuf, cr->mpb->ibuf_alloc);
+ }
+ if (cr->nc.bUse)
+ {
+ /* Use two step summing */
+ MPI_Allreduce(r, cr->mpb->ibuf, nr, MPI_INT, MPI_SUM, cr->nc.comm_intra);
+ if (cr->nc.rank_intra == 0)
+ {
+ /* Sum with the buffers reversed */
+ MPI_Allreduce(cr->mpb->ibuf, r, nr, MPI_INT, MPI_SUM, cr->nc.comm_inter);
+ }
+ MPI_Bcast(r, nr, MPI_INT, 0, cr->nc.comm_intra);
+ }
+ else
+ {
+ MPI_Allreduce(r, cr->mpb->ibuf, nr, MPI_INT, MPI_SUM, cr->mpi_comm_mygroup);
+ for (i = 0; i < nr; i++)
+ {
+ r[i] = cr->mpb->ibuf[i];
+ }
+ }
+#endif
+#endif
+}
+
+void gmx_sumli(int nr, gmx_large_int_t r[], const t_commrec *cr)
+{
+#ifndef GMX_MPI
+ gmx_call("gmx_sumli");
+#else
+#if defined(MPI_IN_PLACE_EXISTS) || defined(GMX_THREAD_MPI)
+ if (cr->nc.bUse)
+ {
+ /* Use two step summing */
+ if (cr->nc.rank_intra == 0)
+ {
+ MPI_Reduce(MPI_IN_PLACE, r, nr, GMX_MPI_LARGE_INT, MPI_SUM, 0,
+ cr->nc.comm_intra);
+ /* Sum with the buffers reversed */
+ MPI_Allreduce(MPI_IN_PLACE, r, nr, GMX_MPI_LARGE_INT, MPI_SUM,
+ cr->nc.comm_inter);
+ }
+ else
+ {
+ /* This is here because of the silly MPI specification
+ that MPI_IN_PLACE should be put in sendbuf instead of recvbuf */
+ MPI_Reduce(r, NULL, nr, GMX_MPI_LARGE_INT, MPI_SUM, 0, cr->nc.comm_intra);
+ }
+ MPI_Bcast(r, nr, GMX_MPI_LARGE_INT, 0, cr->nc.comm_intra);
+ }
+ else
+ {
+ MPI_Allreduce(MPI_IN_PLACE, r, nr, GMX_MPI_LARGE_INT, MPI_SUM, cr->mpi_comm_mygroup);
+ }
+#else
+ int i;
+
+ if (nr > cr->mpb->libuf_alloc)
+ {
+ cr->mpb->libuf_alloc = nr;
+ srenew(cr->mpb->libuf, cr->mpb->libuf_alloc);
+ }
+ if (cr->nc.bUse)
+ {
+ /* Use two step summing */
+ MPI_Allreduce(r, cr->mpb->libuf, nr, GMX_MPI_LARGE_INT, MPI_SUM,
+ cr->nc.comm_intra);
+ if (cr->nc.rank_intra == 0)
+ {
+ /* Sum with the buffers reversed */
+ MPI_Allreduce(cr->mpb->libuf, r, nr, GMX_MPI_LARGE_INT, MPI_SUM,
+ cr->nc.comm_inter);
+ }
+ MPI_Bcast(r, nr, GMX_MPI_LARGE_INT, 0, cr->nc.comm_intra);
+ }
+ else
+ {
+ MPI_Allreduce(r, cr->mpb->libuf, nr, GMX_MPI_LARGE_INT, MPI_SUM,
+ cr->mpi_comm_mygroup);
+ for (i = 0; i < nr; i++)
+ {
+ r[i] = cr->mpb->libuf[i];
+ }
+ }
+#endif
+#endif
+}
+
+
+
+#ifdef GMX_MPI
+void gmx_sumd_comm(int nr, double r[], MPI_Comm mpi_comm)
+{
+#if defined(MPI_IN_PLACE_EXISTS) || defined(GMX_THREAD_MPI)
+ MPI_Allreduce(MPI_IN_PLACE, r, nr, MPI_DOUBLE, MPI_SUM, mpi_comm);
+#else
+ /* this function is only used in code that is not performance critical,
+ (during setup, when comm_rec is not the appropriate communication
+ structure), so this isn't as bad as it looks. */
+ double *buf;
+ int i;
+
+ snew(buf, nr);
+ MPI_Allreduce(r, buf, nr, MPI_DOUBLE, MPI_SUM, mpi_comm);
+ for (i = 0; i < nr; i++)
+ {
+ r[i] = buf[i];
+ }
+ sfree(buf);
+#endif
+}
+#endif
+
+#ifdef GMX_MPI
+void gmx_sumf_comm(int nr, float r[], MPI_Comm mpi_comm)
+{
+#if defined(MPI_IN_PLACE_EXISTS) || defined(GMX_THREAD_MPI)
+ MPI_Allreduce(MPI_IN_PLACE, r, nr, MPI_FLOAT, MPI_SUM, mpi_comm);
+#else
+ /* this function is only used in code that is not performance critical,
+ (during setup, when comm_rec is not the appropriate communication
+ structure), so this isn't as bad as it looks. */
+ float *buf;
+ int i;
+
+ snew(buf, nr);
+ MPI_Allreduce(r, buf, nr, MPI_FLOAT, MPI_SUM, mpi_comm);
+ for (i = 0; i < nr; i++)
+ {
+ r[i] = buf[i];
+ }
+ sfree(buf);
+#endif
+}
+#endif
+
+void gmx_sumd_sim(int nr, double r[], const gmx_multisim_t *ms)
+{
+#ifndef GMX_MPI
+ gmx_call("gmx_sumd_sim");
+#else
+ gmx_sumd_comm(nr, r, ms->mpi_comm_masters);
+#endif
+}
+
+void gmx_sumf_sim(int nr, float r[], const gmx_multisim_t *ms)
+{
+#ifndef GMX_MPI
+ gmx_call("gmx_sumf_sim");
+#else
+ gmx_sumf_comm(nr, r, ms->mpi_comm_masters);
+#endif
+}
+
+void gmx_sumi_sim(int nr, int r[], const gmx_multisim_t *ms)
+{
+#ifndef GMX_MPI
+ gmx_call("gmx_sumi_sim");
+#else
+#if defined(MPI_IN_PLACE_EXISTS) || defined(GMX_THREAD_MPI)
+ MPI_Allreduce(MPI_IN_PLACE, r, nr, MPI_INT, MPI_SUM, ms->mpi_comm_masters);
+#else
+ /* this is thread-unsafe, but it will do for now: */
+ int i;
+
+ if (nr > ms->mpb->ibuf_alloc)
+ {
+ ms->mpb->ibuf_alloc = nr;
+ srenew(ms->mpb->ibuf, ms->mpb->ibuf_alloc);
+ }
+ MPI_Allreduce(r, ms->mpb->ibuf, nr, MPI_INT, MPI_SUM, ms->mpi_comm_masters);
+ for (i = 0; i < nr; i++)
+ {
+ r[i] = ms->mpb->ibuf[i];
+ }
+#endif
+#endif
+}
+
+void gmx_sumli_sim(int nr, gmx_large_int_t r[], const gmx_multisim_t *ms)
+{
+#ifndef GMX_MPI
+ gmx_call("gmx_sumli_sim");
+#else
+#if defined(MPI_IN_PLACE_EXISTS) || defined(GMX_THREAD_MPI)
+ MPI_Allreduce(MPI_IN_PLACE, r, nr, GMX_MPI_LARGE_INT, MPI_SUM,
+ ms->mpi_comm_masters);
+#else
+ /* this is thread-unsafe, but it will do for now: */
+ int i;
+
+ if (nr > ms->mpb->libuf_alloc)
+ {
+ ms->mpb->libuf_alloc = nr;
+ srenew(ms->mpb->libuf, ms->mpb->libuf_alloc);
+ }
+ MPI_Allreduce(r, ms->mpb->libuf, nr, GMX_MPI_LARGE_INT, MPI_SUM,
+ ms->mpi_comm_masters);
+ for (i = 0; i < nr; i++)
+ {
+ r[i] = ms->mpb->libuf[i];
+ }
+#endif
+#endif
+}
--- /dev/null
+/*
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * VERSION 3.2.0
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2004, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * GROningen Mixture of Alchemy and Childrens' Stories
+ */
+/* This file is completely threadsafe - keep it that way! */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+#include "gromacs/utility/gmx_header_config.h"
+
+#include <stdio.h>
+#include <ctype.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <time.h>
+
+#ifdef HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#ifdef HAVE_PWD_H
+#include <pwd.h>
+#endif
+#include <time.h>
+#include <assert.h>
+
+#include "typedefs.h"
+#include "smalloc.h"
+#include "gmx_fatal.h"
+#include "macros.h"
+#include "string2.h"
+#include "gromacs/fileio/futil.h"
+
+int continuing(char *s)
+{
+ int sl;
+ assert(s);
+
+ rtrim(s);
+ sl = strlen(s);
+ if ((sl > 0) && (s[sl-1] == CONTINUE))
+ {
+ s[sl-1] = 0;
+ return TRUE;
+ }
+ else
+ {
+ return FALSE;
+ }
+}
+
+
+
+char *fgets2(char *line, int n, FILE *stream)
+{
+ char *c;
+ if (fgets(line, n, stream) == NULL)
+ {
+ return NULL;
+ }
+ if ((c = strchr(line, '\n')) != NULL)
+ {
+ *c = '\0';
+ }
+ else
+ {
+ /* A line not ending in a newline can only occur at the end of a file,
+ * or because of n being too small.
+ * Since both cases occur very infrequently, we can check for EOF.
+ */
+ if (!gmx_eof(stream))
+ {
+ gmx_fatal(FARGS, "An input file contains a line longer than %d characters, while the buffer passed to fgets2 has size %d. The line starts with: '%20.20s'", n, n, line);
+ }
+ }
+ if ((c = strchr(line, '\r')) != NULL)
+ {
+ *c = '\0';
+ }
+
+ return line;
+}
+
+void strip_comment (char *line)
+{
+ char *c;
+
+ if (!line)
+ {
+ return;
+ }
+
+ /* search for a comment mark and replace it by a zero */
+ if ((c = strchr(line, COMMENTSIGN)) != NULL)
+ {
+ (*c) = 0;
+ }
+}
+
+void upstring (char *str)
+{
+ int i;
+
+ for (i = 0; (i < (int)strlen(str)); i++)
+ {
+ str[i] = toupper(str[i]);
+ }
+}
+
+void ltrim (char *str)
+{
+ char *tr;
+ int i, c;
+
+ if (NULL == str)
+ {
+ return;
+ }
+
+ c = 0;
+ while (('\0' != str[c]) && isspace(str[c]))
+ {
+ c++;
+ }
+ if (c > 0)
+ {
+ for (i = c; ('\0' != str[i]); i++)
+ {
+ str[i-c] = str[i];
+ }
+ str[i-c] = '\0';
+ }
+}
+
+void rtrim (char *str)
+{
+ int nul;
+
+ if (NULL == str)
+ {
+ return;
+ }
+
+ nul = strlen(str)-1;
+ while ((nul > 0) && ((str[nul] == ' ') || (str[nul] == '\t')) )
+ {
+ str[nul] = '\0';
+ nul--;
+ }
+}
+
+void trim (char *str)
+{
+ ltrim (str);
+ rtrim (str);
+}
+
+char *
+gmx_ctime_r(const time_t *clock, char *buf, int n)
+{
+ char tmpbuf[STRLEN];
+
+#ifdef GMX_NATIVE_WINDOWS
+ /* Windows */
+ ctime_s( tmpbuf, STRLEN, clock );
+#elif (defined(__sun))
+ /*Solaris*/
+ ctime_r(clock, tmpbuf, n);
+#else
+ ctime_r(clock, tmpbuf);
+#endif
+ strncpy(buf, tmpbuf, n-1);
+ buf[n-1] = '\0';
+
+ return buf;
+}
+
+void nice_header (FILE *out, const char *fn)
+{
+ const char *unk = "onbekend";
+ time_t clock;
+ const char *user = unk;
+ int gh;
+#ifdef HAVE_PWD_H
+ uid_t uid;
+#else
+ int uid;
+#endif
+ char buf[256] = "";
+ char timebuf[STRLEN];
+#ifdef HAVE_PWD_H
+ struct passwd *pw;
+#endif
+
+ /* Print a nice header above the file */
+ time(&clock);
+ fprintf (out, "%c\n", COMMENTSIGN);
+ fprintf (out, "%c\tFile '%s' was generated\n", COMMENTSIGN, fn ? fn : unk);
+
+#ifdef HAVE_PWD_H
+ uid = getuid();
+ pw = getpwuid(uid);
+ gh = gethostname(buf, 255);
+ /* pw returns null on error (e.g. compute nodes lack /etc/passwd) */
+ user = pw ? pw->pw_name : unk;
+#else
+ uid = 0;
+ gh = -1;
+#endif
+
+ gmx_ctime_r(&clock, timebuf, STRLEN);
+ fprintf (out, "%c\tBy user: %s (%d)\n", COMMENTSIGN,
+ user ? user : unk, (int) uid);
+ fprintf(out, "%c\tOn host: %s\n", COMMENTSIGN, (gh == 0) ? buf : unk);
+
+ fprintf (out, "%c\tAt date: %s", COMMENTSIGN, timebuf);
+ fprintf (out, "%c\n", COMMENTSIGN);
+}
+
+
+int gmx_strcasecmp_min(const char *str1, const char *str2)
+{
+ char ch1, ch2;
+
+ do
+ {
+ do
+ {
+ ch1 = toupper(*(str1++));
+ }
+ while ((ch1 == '-') || (ch1 == '_'));
+ do
+ {
+ ch2 = toupper(*(str2++));
+ }
+ while ((ch2 == '-') || (ch2 == '_'));
+
+ if (ch1 != ch2)
+ {
+ return (ch1-ch2);
+ }
+ }
+ while (ch1);
+ return 0;
+}
+
+int gmx_strncasecmp_min(const char *str1, const char *str2, int n)
+{
+ char ch1, ch2;
+ char *stri1, *stri2;
+
+ stri1 = (char *)str1;
+ stri2 = (char *)str2;
+ do
+ {
+ do
+ {
+ ch1 = toupper(*(str1++));
+ }
+ while ((ch1 == '-') || (ch1 == '_'));
+ do
+ {
+ ch2 = toupper(*(str2++));
+ }
+ while ((ch2 == '-') || (ch2 == '_'));
+
+ if (ch1 != ch2)
+ {
+ return (ch1-ch2);
+ }
+ }
+ while (ch1 && (str1-stri1 < n) && (str2-stri2 < n));
+ return 0;
+}
+
+int gmx_strcasecmp(const char *str1, const char *str2)
+{
+ char ch1, ch2;
+
+ do
+ {
+ ch1 = toupper(*(str1++));
+ ch2 = toupper(*(str2++));
+ if (ch1 != ch2)
+ {
+ return (ch1-ch2);
+ }
+ }
+ while (ch1);
+ return 0;
+}
+
+int gmx_strncasecmp(const char *str1, const char *str2, int n)
+{
+ char ch1, ch2;
+
+ if (n == 0)
+ {
+ return 0;
+ }
+
+ do
+ {
+ ch1 = toupper(*(str1++));
+ ch2 = toupper(*(str2++));
+ if (ch1 != ch2)
+ {
+ return (ch1-ch2);
+ }
+ n--;
+ }
+ while (ch1 && n);
+ return 0;
+}
+
+char *gmx_strdup(const char *src)
+{
+ char *dest;
+
+ snew(dest, strlen(src)+1);
+ strcpy(dest, src);
+
+ return dest;
+}
+
+char *
+gmx_strndup(const char *src, int n)
+{
+ int len;
+ char *dest;
+
+ len = strlen(src);
+ if (len > n)
+ {
+ len = n;
+ }
+ snew(dest, len+1);
+ strncpy(dest, src, len);
+ dest[len] = 0;
+ return dest;
+}
+
+/* Magic hash init number for Dan J. Bernsteins algorithm.
+ * Do NOT use any other value unless you really know what you are doing.
+ */
+const unsigned int
+ gmx_string_hash_init = 5381;
+
+
++unsigned int
++gmx_string_fullhash_func(const char *s, unsigned int hash_init)
++{
++ int c;
++
++ while ((c = (*s++)) != '\0')
++ {
++ hash_init = ((hash_init << 5) + hash_init) ^ c; /* (hash * 33) xor c */
++ }
++ return hash_init;
++}
++
+unsigned int
+gmx_string_hash_func(const char *s, unsigned int hash_init)
+{
+ int c;
+
+ while ((c = toupper(*s++)) != '\0')
+ {
+ if (isalnum(c))
+ {
+ hash_init = ((hash_init << 5) + hash_init) ^ c; /* (hash * 33) xor c */
+ }
+ }
+ return hash_init;
+}
+
+int
+gmx_wcmatch(const char *pattern, const char *str)
+{
+ while (*pattern)
+ {
+ if (*pattern == '*')
+ {
+ /* Skip multiple wildcards in a sequence */
+ while (*pattern == '*' || *pattern == '?')
+ {
+ ++pattern;
+ /* For ?, we need to check that there are characters left
+ * in str. */
+ if (*pattern == '?')
+ {
+ if (*str == 0)
+ {
+ return GMX_NO_WCMATCH;
+ }
+ else
+ {
+ ++str;
+ }
+ }
+ }
+ /* If the pattern ends after the star, we have a match */
+ if (*pattern == 0)
+ {
+ return 0;
+ }
+ /* Match the rest against each possible suffix of str */
+ while (*str)
+ {
+ /* Only do the recursive call if the first character
+ * matches. We don't have to worry about wildcards here,
+ * since we have processed them above. */
+ if (*pattern == *str)
+ {
+ int rc;
+ /* Match the suffix, and return if a match or an error */
+ rc = gmx_wcmatch(pattern, str);
+ if (rc != GMX_NO_WCMATCH)
+ {
+ return rc;
+ }
+ }
+ ++str;
+ }
+ /* If no suffix of str matches, we don't have a match */
+ return GMX_NO_WCMATCH;
+ }
+ else if ((*pattern == '?' && *str != 0) || *pattern == *str)
+ {
+ ++str;
+ }
+ else
+ {
+ return GMX_NO_WCMATCH;
+ }
+ ++pattern;
+ }
+ /* When the pattern runs out, we have a match if the string has ended. */
+ return (*str == 0) ? 0 : GMX_NO_WCMATCH;
+}
+
+char *wrap_lines(const char *buf, int line_width, int indent, gmx_bool bIndentFirst)
+{
+ char *b2;
+ int i, i0, i2, j, b2len, lspace = 0, l2space = 0;
+ gmx_bool bFirst, bFitsOnLine;
+
+ /* characters are copied from buf to b2 with possible spaces changed
+ * into newlines and extra space added for indentation.
+ * i indexes buf (source buffer) and i2 indexes b2 (destination buffer)
+ * i0 points to the beginning of the current line (in buf, source)
+ * lspace and l2space point to the last space on the current line
+ * bFirst is set to prevent indentation of first line
+ * bFitsOnLine says if the first space occurred before line_width, if
+ * that is not the case, we have a word longer than line_width which
+ * will also not fit on the next line, so we might as well keep it on
+ * the current line (where it also won't fit, but looks better)
+ */
+
+ b2 = NULL;
+ b2len = strlen(buf)+1+indent;
+ snew(b2, b2len);
+ i0 = i2 = 0;
+ if (bIndentFirst)
+ {
+ for (i2 = 0; (i2 < indent); i2++)
+ {
+ b2[i2] = ' ';
+ }
+ }
+ bFirst = TRUE;
+ do
+ {
+ l2space = -1;
+ /* find the last space before end of line */
+ for (i = i0; ((i-i0 < line_width) || (l2space == -1)) && (buf[i]); i++)
+ {
+ b2[i2++] = buf[i];
+ /* remember the position of a space */
+ if (buf[i] == ' ')
+ {
+ lspace = i;
+ l2space = i2-1;
+ }
+ /* if we have a newline before the line is full, reset counters */
+ if (buf[i] == '\n' && buf[i+1])
+ {
+ i0 = i+1;
+ b2len += indent;
+ srenew(b2, b2len);
+ /* add indentation after the newline */
+ for (j = 0; (j < indent); j++)
+ {
+ b2[i2++] = ' ';
+ }
+ }
+ }
+ /* If we are at the last newline, copy it */
+ if (buf[i] == '\n' && !buf[i+1])
+ {
+ b2[i2++] = buf[i++];
+ }
+ /* if we're not at the end of the string */
+ if (buf[i])
+ {
+ /* check if one word does not fit on the line */
+ bFitsOnLine = (i-i0 <= line_width);
+ /* reset line counters to just after the space */
+ i0 = lspace+1;
+ i2 = l2space+1;
+ /* if the words fit on the line, and we're beyond the indentation part */
+ if ( (bFitsOnLine) && (l2space >= indent) )
+ {
+ /* start a new line */
+ b2[l2space] = '\n';
+ /* and add indentation */
+ if (indent)
+ {
+ if (bFirst)
+ {
+ line_width -= indent;
+ bFirst = FALSE;
+ }
+ b2len += indent;
+ srenew(b2, b2len);
+ for (j = 0; (j < indent); j++)
+ {
+ b2[i2++] = ' ';
+ }
+ /* no extra spaces after indent; */
+ while (buf[i0] == ' ')
+ {
+ i0++;
+ }
+ }
+ }
+ }
+ }
+ while (buf[i]);
+ b2[i2] = '\0';
+
+ return b2;
+}
+
+char **split(char sep, const char *str)
+{
+ char **ptr = NULL;
+ int n, nn, nptr = 0;
+
+ if (str == NULL)
+ {
+ return NULL;
+ }
+ nn = strlen(str);
+ for (n = 0; (n < nn); n++)
+ {
+ if (str[n] == sep)
+ {
+ nptr++;
+ }
+ }
+ snew(ptr, nptr+2);
+ nptr = 0;
+ while (*str != '\0')
+ {
+ while ((*str != '\0') && (*str == sep))
+ {
+ str++;
+ }
+ if (*str != '\0')
+ {
+ snew(ptr[nptr], 1+strlen(str));
+ n = 0;
+ while ((*str != '\0') && (*str != sep))
+ {
+ ptr[nptr][n] = *str;
+ str++;
+ n++;
+ }
+ ptr[nptr][n] = '\0';
+ nptr++;
+ }
+ }
+ ptr[nptr] = NULL;
+
+ return ptr;
+}
+
+
+gmx_large_int_t
+str_to_large_int_t(const char *str, char **endptr)
+{
+ int sign = 1;
+ gmx_large_int_t val = 0;
+ char ch;
+ const char *p;
+
+ p = str;
+ if (p == NULL)
+ {
+ *endptr = NULL;
+ return 0;
+ }
+
+ /* Strip off initial white space */
+ while (isspace(*p))
+ {
+ p++;
+ }
+ /* Conform to ISO C99 - return original pointer if string does not contain a number */
+ if (*str == '\0')
+ {
+ *endptr = (char *)str;
+ }
+
+ if (*p == '-')
+ {
+ p++;
+ sign *= -1;
+ }
+
+ while ( ((ch = *p) != '\0') && isdigit(ch) )
+ {
+ /* Important to add sign here, so we dont overflow in final multiplication */
+ ch = (ch-'0')*sign;
+ val = val*10 + ch;
+ if (ch != val%10)
+ {
+ /* Some sort of overflow has occured, set endptr to original string */
+ *endptr = (char *)str;
+ errno = ERANGE;
+ return(0);
+ }
+ p++;
+ }
+
+ *endptr = (char *)p;
+
+ return val;
+}
+
+char *gmx_strsep(char **stringp, const char *delim)
+{
+ char *ret;
+ int len = strlen(delim);
+ int i, j = 0;
+ int found = 0;
+
+ if (!*stringp)
+ {
+ return NULL;
+ }
+ ret = *stringp;
+ do
+ {
+ if ( (*stringp)[j] == '\0')
+ {
+ found = 1;
+ *stringp = NULL;
+ break;
+ }
+ for (i = 0; i < len; i++)
+ {
+ if ( (*stringp)[j] == delim[i])
+ {
+ (*stringp)[j] = '\0';
+ *stringp = *stringp+j+1;
+ found = 1;
+ break;
+ }
+ }
+ j++;
+ }
+ while (!found);
+
+ return ret;
+}
--- /dev/null
- ddCyclStep, ddCyclPPduringPME, ddCyclF, ddCyclPME, ddCyclNr
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This file is part of Gromacs Copyright (c) 1991-2008
+ * David van der Spoel, Erik Lindahl, Berk Hess, University of Groningen.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the research papers on the package. Check out http://www.gromacs.org
+ *
+ * And Hey:
+ * Gnomes, ROck Monsters And Chili Sauce
+ */
+
+#ifndef _domdec_h
+#define _domdec_h
+
+#include "typedefs.h"
+#include "types/commrec.h"
+#include "vsite.h"
+#include "genborn.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int ddglatnr(gmx_domdec_t *dd, int i);
+/* Returns the global topology atom number belonging to local atom index i.
+ * This function is intended for writing ascii output
+ * and returns atom numbers starting at 1.
+ * When dd=NULL returns i+1.
+ */
+
+t_block *dd_charge_groups_global(gmx_domdec_t *dd);
+/* Return a block struct for the charge groups of the whole system */
+
+gmx_bool dd_filled_nsgrid_home(gmx_domdec_t *dd);
+/* Is the ns grid already filled with the home particles? */
+
+void dd_store_state(gmx_domdec_t *dd, t_state *state);
+/* Store the global cg indices of the home cgs in state,
+ * so it can be reset, even after a new DD partitioning.
+ */
+
+gmx_domdec_zones_t *domdec_zones(gmx_domdec_t *dd);
+
+void dd_get_ns_ranges(gmx_domdec_t *dd, int icg,
+ int *jcg0, int *jcg1, ivec shift0, ivec shift1);
+
+int dd_natoms_vsite(gmx_domdec_t *dd);
+
+void dd_get_constraint_range(gmx_domdec_t *dd,
+ int *at_start, int *at_end);
+
+real dd_cutoff_mbody(gmx_domdec_t *dd);
+
+real dd_cutoff_twobody(gmx_domdec_t *dd);
+
+void get_pme_nnodes(const gmx_domdec_t *dd,
+ int *npmenodes_x, int *npmenodes_y);
+/* Get the number of PME nodes along x and y, can be called with dd=NULL */
+
+gmx_bool gmx_pmeonlynode(t_commrec *cr, int nodeid);
+/* Return if nodeid in cr->mpi_comm_mysim is a PME-only node */
+
+void get_pme_ddnodes(t_commrec *cr, int pmenodeid,
+ int *nmy_ddnodes, int **my_ddnodes, int *node_peer);
+/* Returns the set of DD nodes that communicate with pme node cr->nodeid */
+
+int dd_pme_maxshift_x(gmx_domdec_t *dd);
+/* Returns the maximum shift for coordinate communication in PME, dim x */
+
+int dd_pme_maxshift_y(gmx_domdec_t *dd);
+/* Returns the maximum shift for coordinate communication in PME, dim y */
+
+void make_dd_communicators(FILE *fplog, t_commrec *cr, int dd_node_order);
+
+gmx_domdec_t *
+init_domain_decomposition(FILE *fplog,
+ t_commrec *cr,
+ unsigned long Flags,
+ ivec nc,
+ real comm_distance_min, real rconstr,
+ const char *dlb_opt, real dlb_scale,
+ const char *sizex, const char *sizey, const char *sizez,
+ gmx_mtop_t *mtop, t_inputrec *ir,
+ matrix box, rvec *x,
+ gmx_ddbox_t *ddbox,
+ int *npme_x, int *npme_y);
+
+void dd_init_bondeds(FILE *fplog,
+ gmx_domdec_t *dd, gmx_mtop_t *mtop,
+ gmx_vsite_t *vsite,
+ t_inputrec *ir, gmx_bool bBCheck, cginfo_mb_t *cginfo_mb);
+/* Initialize data structures for bonded interactions */
+
+gmx_bool dd_bonded_molpbc(gmx_domdec_t *dd, int ePBC);
+/* Returns if we need to do pbc for calculating bonded interactions */
+
+void set_dd_parameters(FILE *fplog, gmx_domdec_t *dd, real dlb_scale,
+ t_inputrec *ir,
+ gmx_ddbox_t *ddbox);
+/* Set DD grid dimensions and limits,
+ * should be called after calling dd_init_bondeds.
+ */
+
+gmx_bool change_dd_cutoff(t_commrec *cr, t_state *state, t_inputrec *ir,
+ real cutoff_req );
+/* Change the DD non-bonded communication cut-off.
+ * This could fail when trying to increase the cut-off,
+ * then FALSE will be returned and the cut-off is not modified.
+ */
+
+void change_dd_dlb_cutoff_limit(t_commrec *cr);
+/* Domain boundary changes due to the DD dynamic load balancing can limit
+ * the cut-off distance that can be set in change_dd_cutoff. This function
+ * limits the DLB such that using the currently set cut-off should still be
+ * possible after subsequently setting a shorter cut-off with change_dd_cutoff.
+ */
+
++void dd_setup_dlb_resource_sharing(t_commrec *cr,
++ const gmx_hw_info_t *hwinfo,
++ const gmx_hw_opt_t *hw_opt);
++/* When domains (PP MPI ranks) share a GPU, the individual GPU wait times
++ * are meaningless, as it depends on the order in which tasks on the same
++ * GPU finish. Therefore there wait times need to be averaged over the ranks
++ * sharing the same GPU. This function sets up the communication for that.
++ */
++
+void setup_dd_grid(FILE *fplog, gmx_domdec_t *dd);
+
+void dd_collect_vec(gmx_domdec_t *dd,
+ t_state *state_local, rvec *lv, rvec *v);
+
+void dd_collect_state(gmx_domdec_t *dd,
+ t_state *state_local, t_state *state);
+
+enum {
++ ddCyclStep, ddCyclPPduringPME, ddCyclF, ddCyclWaitGPU, ddCyclPME, ddCyclNr
+};
+
+void dd_cycles_add(gmx_domdec_t *dd, float cycles, int ddCycl);
+/* Add the wallcycle count to the DD counter */
+
+void dd_force_flop_start(gmx_domdec_t *dd, t_nrnb *nrnb);
+/* Start the force flop count */
+
+void dd_force_flop_stop(gmx_domdec_t *dd, t_nrnb *nrnb);
+/* Stop the force flop count */
+
+float dd_pme_f_ratio(gmx_domdec_t *dd);
+/* Return the PME/PP force load ratio, or -1 if nothing was measured.
+ * Should only be called on the DD master node.
+ */
+
+void dd_move_x(gmx_domdec_t *dd, matrix box, rvec x[]);
+/* Communicate the coordinates to the neighboring cells and do pbc. */
+
+void dd_move_f(gmx_domdec_t *dd, rvec f[], rvec *fshift);
+/* Sum the forces over the neighboring cells.
+ * When fshift!=NULL the shift forces are updated to obtain
+ * the correct virial from the single sum including f.
+ */
+
+void dd_atom_spread_real(gmx_domdec_t *dd, real v[]);
+/* Communicate a real for each atom to the neighboring cells. */
+
+void dd_atom_sum_real(gmx_domdec_t *dd, real v[]);
+/* Sum the contributions to a real for each atom over the neighboring cells. */
+
+void dd_partition_system(FILE *fplog,
+ gmx_large_int_t step,
+ t_commrec *cr,
+ gmx_bool bMasterState,
+ int nstglobalcomm,
+ t_state *state_global,
+ gmx_mtop_t *top_global,
+ t_inputrec *ir,
+ t_state *state_local,
+ rvec **f,
+ t_mdatoms *mdatoms,
+ gmx_localtop_t *top_local,
+ t_forcerec *fr,
+ gmx_vsite_t *vsite,
+ gmx_shellfc_t shellfc,
+ gmx_constr_t constr,
+ t_nrnb *nrnb,
+ gmx_wallcycle_t wcycle,
+ gmx_bool bVerbose);
+/* Partition the system over the nodes.
+ * step is only used for printing error messages.
+ * If bMasterState==TRUE then state_global from the master node is used,
+ * else state_local is redistributed between the nodes.
+ * When f!=NULL, *f will be reallocated to the size of state_local.
+ */
+
+void reset_dd_statistics_counters(gmx_domdec_t *dd);
+/* Reset all the statistics and counters for total run counting */
+
+void print_dd_statistics(t_commrec *cr, t_inputrec *ir, FILE *fplog);
+
+/* In domdec_con.c */
+
+void dd_move_f_vsites(gmx_domdec_t *dd, rvec *f, rvec *fshift);
+
+void dd_clear_f_vsites(gmx_domdec_t *dd, rvec *f);
+
+void dd_move_x_constraints(gmx_domdec_t *dd, matrix box,
+ rvec *x0, rvec *x1);
+/* Move x0 and also x1 if x1!=NULL */
+
+void dd_move_x_vsites(gmx_domdec_t *dd, matrix box, rvec *x);
+
+int *dd_constraints_nlocalatoms(gmx_domdec_t *dd);
+
+void dd_clear_local_constraint_indices(gmx_domdec_t *dd);
+
+void dd_clear_local_vsite_indices(gmx_domdec_t *dd);
+
+int dd_make_local_vsites(gmx_domdec_t *dd, int at_start, t_ilist *lil);
+
+int dd_make_local_constraints(gmx_domdec_t *dd, int at_start,
+ const gmx_mtop_t *mtop,
+ const int *cginfo,
+ gmx_constr_t constr, int nrec,
+ t_ilist *il_local);
+
+void init_domdec_constraints(gmx_domdec_t *dd,
+ gmx_mtop_t *mtop);
+
+void init_domdec_vsites(gmx_domdec_t *dd, int n_intercg_vsite);
+
+
+/* In domdec_top.c */
+
+void dd_print_missing_interactions(FILE *fplog, t_commrec *cr,
+ int local_count, gmx_mtop_t *top_global, t_state *state_local);
+
+void dd_make_reverse_top(FILE *fplog,
+ gmx_domdec_t *dd, gmx_mtop_t *mtop,
+ gmx_vsite_t *vsite,
+ t_inputrec *ir, gmx_bool bBCheck);
+
+void dd_make_local_cgs(gmx_domdec_t *dd, t_block *lcgs);
+
+void dd_make_local_top(gmx_domdec_t *dd, gmx_domdec_zones_t *zones,
+ int npbcdim, matrix box,
+ rvec cellsize_min, ivec npulse,
+ t_forcerec *fr,
+ rvec *cgcm_or_x,
+ gmx_vsite_t *vsite,
+ gmx_mtop_t *top, gmx_localtop_t *ltop);
+
+void dd_sort_local_top(gmx_domdec_t *dd, t_mdatoms *mdatoms,
+ gmx_localtop_t *ltop);
+/* Sort ltop->ilist when we are doing free energy. */
+
+gmx_localtop_t *dd_init_local_top(gmx_mtop_t *top_global);
+
+void dd_init_local_state(gmx_domdec_t *dd,
+ t_state *state_global, t_state *local_state);
+
+t_blocka *make_charge_group_links(gmx_mtop_t *mtop, gmx_domdec_t *dd,
+ cginfo_mb_t *cginfo_mb);
+
+void dd_bonded_cg_distance(FILE *fplog, gmx_mtop_t *mtop,
+ t_inputrec *ir, rvec *x, matrix box,
+ gmx_bool bBCheck,
+ real *r_2b, real *r_mb);
+
+void write_dd_pdb(const char *fn, gmx_large_int_t step, const char *title,
+ gmx_mtop_t *mtop,
+ t_commrec *cr,
+ int natoms, rvec x[], matrix box);
+/* Dump a pdb file with the current DD home + communicated atoms.
+ * When natoms=-1, dump all known atoms.
+ */
+
+
+/* In domdec_setup.c */
+
+real comm_box_frac(ivec dd_nc, real cutoff, gmx_ddbox_t *ddbox);
+/* Returns the volume fraction of the system that is communicated */
+
+real dd_choose_grid(FILE *fplog,
+ t_commrec *cr, gmx_domdec_t *dd, t_inputrec *ir,
+ gmx_mtop_t *mtop, matrix box, gmx_ddbox_t *ddbox,
+ gmx_bool bDynLoadBal, real dlb_scale,
+ real cellsize_limit, real cutoff_dd,
+ gmx_bool bInterCGBondeds);
+/* Determines the optimal DD cell setup dd->nc and possibly npmenodes
+ * for the system.
+ * On the master node returns the actual cellsize limit used.
+ */
+
+
+/* In domdec_box.c */
+
+void set_ddbox(gmx_domdec_t *dd, gmx_bool bMasterState, t_commrec *cr_sum,
+ t_inputrec *ir, matrix box,
+ gmx_bool bCalcUnboundedSize, t_block *cgs, rvec *x,
+ gmx_ddbox_t *ddbox);
+
+void set_ddbox_cr(t_commrec *cr, ivec *dd_nc,
+ t_inputrec *ir, matrix box, t_block *cgs, rvec *x,
+ gmx_ddbox_t *ddbox);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _domdec_h */
--- /dev/null
- void init_interaction_const(FILE *fp,
- interaction_const_t **interaction_const,
- const t_forcerec *fr,
- real rtab);
- /* Initializes the interaction constant data structure. Currently it
- * uses forcerec as input.
- */
-
+/*
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * VERSION 3.2.0
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2004, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * Gromacs Runs On Most of All Computer Systems
+ */
+
+#ifndef _force_h
+#define _force_h
+
+
+#include "typedefs.h"
+#include "types/force_flags.h"
+#include "pbc.h"
+#include "network.h"
+#include "tgroup.h"
+#include "vsite.h"
+#include "genborn.h"
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void gmx_print_sepdvdl(FILE *fplog, const char *s, real v, real dvdlambda);
+
+void calc_vir(int nxf, rvec x[], rvec f[], tensor vir,
+ gmx_bool bScrewPBC, matrix box);
+/* Calculate virial for nxf atoms, and add it to vir */
+
+void f_calc_vir(int i0, int i1, rvec x[], rvec f[], tensor vir,
+ t_graph *g, rvec shift_vec[]);
+/* Calculate virial taking periodicity into account */
+
+real RF_excl_correction(const t_forcerec *fr, t_graph *g,
+ const t_mdatoms *mdatoms, const t_blocka *excl,
+ rvec x[], rvec f[], rvec *fshift, const t_pbc *pbc,
+ real lambda, real *dvdlambda);
+/* Calculate the reaction-field energy correction for this node:
+ * epsfac q_i q_j (k_rf r_ij^2 - c_rf)
+ * and force correction for all excluded pairs, including self pairs.
+ */
+
+void calc_rffac(FILE *fplog, int eel, real eps_r, real eps_rf,
+ real Rc, real Temp,
+ real zsq, matrix box,
+ real *kappa, real *krf, real *crf);
+/* Determine the reaction-field constants */
+
+void init_generalized_rf(FILE *fplog,
+ const gmx_mtop_t *mtop, const t_inputrec *ir,
+ t_forcerec *fr);
+/* Initialize the generalized reaction field parameters */
+
+
+/* In wall.c */
+void make_wall_tables(FILE *fplog, const output_env_t oenv,
+ const t_inputrec *ir, const char *tabfn,
+ const gmx_groups_t *groups,
+ t_forcerec *fr);
+
+real do_walls(t_inputrec *ir, t_forcerec *fr, matrix box, t_mdatoms *md,
+ rvec x[], rvec f[], real lambda, real Vlj[], t_nrnb *nrnb);
+
+t_forcerec *mk_forcerec(void);
+
+#define GMX_MAKETABLES_FORCEUSER (1<<0)
+#define GMX_MAKETABLES_14ONLY (1<<1)
+
+t_forcetable make_tables(FILE *fp, const output_env_t oenv,
+ const t_forcerec *fr, gmx_bool bVerbose,
+ const char *fn, real rtab, int flags);
+/* Return tables for inner loops. When bVerbose the tables are printed
+ * to .xvg files
+ */
+
+bondedtable_t make_bonded_table(FILE *fplog, char *fn, int angle);
+/* Return a table for bonded interactions,
+ * angle should be: bonds 0, angles 1, dihedrals 2
+ */
+
+/* Return a table for GB calculations */
+t_forcetable make_gb_table(const output_env_t oenv,
+ const t_forcerec *fr);
+
+/* Read a table for AdResS Thermo Force calculations */
+extern t_forcetable make_atf_table(FILE *out, const output_env_t oenv,
+ const t_forcerec *fr,
+ const char *fn,
+ matrix box);
+
+void pr_forcerec(FILE *fplog, t_forcerec *fr);
+
+void
+forcerec_set_ranges(t_forcerec *fr,
+ int ncg_home, int ncg_force,
+ int natoms_force,
+ int natoms_force_constr, int natoms_f_novirsum);
+/* Set the number of cg's and atoms for the force calculation */
+
+gmx_bool can_use_allvsall(const t_inputrec *ir,
+ gmx_bool bPrintNote, t_commrec *cr, FILE *fp);
+/* Returns if we can use all-vs-all loops.
+ * If bPrintNote==TRUE, prints a note, if necessary, to stderr
+ * and fp (if !=NULL) on the master node.
+ */
+
+gmx_bool uses_simple_tables(int cutoff_scheme,
+ nonbonded_verlet_t *nbv,
+ int group);
+/* Returns whether simple tables (i.e. not for use with GPUs) are used
+ * with the type of kernel indicated.
+ */
+
+void init_interaction_const_tables(FILE *fp,
+ interaction_const_t *ic,
+ gmx_bool bSimpleTable,
+ real rtab);
+/* Initializes the tables in the interaction constant data structure.
+ * Setting verlet_kernel_type to -1 always initializes tables for
+ * use with group kernels.
+ */
+
+void init_forcerec(FILE *fplog,
+ const output_env_t oenv,
+ t_forcerec *fr,
+ t_fcdata *fcd,
+ const t_inputrec *ir,
+ const gmx_mtop_t *mtop,
+ const t_commrec *cr,
+ matrix box,
+ const char *tabfn,
+ const char *tabafn,
+ const char *tabpfn,
+ const char *tabbfn,
+ const char *nbpu_opt,
+ gmx_bool bNoSolvOpt,
+ real print_force);
+/* The Force rec struct must be created with mk_forcerec
+ * The gmx_booleans have the following meaning:
+ * bSetQ: Copy the charges [ only necessary when they change ]
+ * bMolEpot: Use the free energy stuff per molecule
+ * print_force >= 0: print forces for atoms with force >= print_force
+ */
+
+void forcerec_set_excl_load(t_forcerec *fr,
+ const gmx_localtop_t *top, const t_commrec *cr);
+/* Set the exclusion load for the local exclusions and possibly threads */
+
+void init_enerdata(int ngener, int n_lambda, gmx_enerdata_t *enerd);
+/* Intializes the energy storage struct */
+
+void destroy_enerdata(gmx_enerdata_t *enerd);
+/* Free all memory associated with enerd */
+
+void reset_foreign_enerdata(gmx_enerdata_t *enerd);
+/* Resets only the foreign energy data */
+
+void reset_enerdata(t_forcerec *fr, gmx_bool bNS,
+ gmx_enerdata_t *enerd,
+ gmx_bool bMaster);
+/* Resets the energy data, if bNS=TRUE also zeros the long-range part */
+
+void sum_epot(gmx_grppairener_t *grpp, real *epot);
+/* Locally sum the non-bonded potential energy terms */
+
+void sum_dhdl(gmx_enerdata_t *enerd, real *lambda, t_lambda *fepvals);
+/* Sum the free energy contributions */
+
+void update_forcerec(t_forcerec *fr, matrix box);
+/* Updates parameters in the forcerec that are time dependent */
+
+/* Compute the average C6 and C12 params for LJ corrections */
+void set_avcsixtwelve(FILE *fplog, t_forcerec *fr,
+ const gmx_mtop_t *mtop);
+
+extern void do_force(FILE *log, t_commrec *cr,
+ t_inputrec *inputrec,
+ gmx_large_int_t step, t_nrnb *nrnb, gmx_wallcycle_t wcycle,
+ gmx_localtop_t *top,
+ gmx_groups_t *groups,
+ matrix box, rvec x[], history_t *hist,
+ rvec f[],
+ tensor vir_force,
+ t_mdatoms *mdatoms,
+ gmx_enerdata_t *enerd, t_fcdata *fcd,
+ real *lambda, t_graph *graph,
+ t_forcerec *fr,
+ gmx_vsite_t *vsite, rvec mu_tot,
+ double t, FILE *field, gmx_edsam_t ed,
+ gmx_bool bBornRadii,
+ int flags);
+
+/* Communicate coordinates (if parallel).
+ * Do neighbor searching (if necessary).
+ * Calculate forces.
+ * Communicate forces (if parallel).
+ * Spread forces for vsites (if present).
+ *
+ * f is always required.
+ */
+
+void ns(FILE *fplog,
+ t_forcerec *fr,
+ matrix box,
+ gmx_groups_t *groups,
+ gmx_localtop_t *top,
+ t_mdatoms *md,
+ t_commrec *cr,
+ t_nrnb *nrnb,
+ gmx_bool bFillGrid,
+ gmx_bool bDoLongRangeNS);
+/* Call the neighborsearcher */
+
+extern void do_force_lowlevel(FILE *fplog,
+ gmx_large_int_t step,
+ t_forcerec *fr,
+ t_inputrec *ir,
+ t_idef *idef,
+ t_commrec *cr,
+ t_nrnb *nrnb,
+ gmx_wallcycle_t wcycle,
+ t_mdatoms *md,
+ rvec x[],
+ history_t *hist,
+ rvec f_shortrange[],
+ rvec f_longrange[],
+ gmx_enerdata_t *enerd,
+ t_fcdata *fcd,
+ gmx_localtop_t *top,
+ gmx_genborn_t *born,
+ t_atomtypes *atype,
+ gmx_bool bBornRadii,
+ matrix box,
+ t_lambda *fepvals,
+ real *lambda,
+ t_graph *graph,
+ t_blocka *excl,
+ rvec mu_tot[2],
+ int flags,
+ float *cycles_pme);
+/* Call all the force routines */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _force_h */
--- /dev/null
- * if it is non-NULL, and print a warning in stdout if we don't have a match.
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This file is part of GROMACS.
+ * Copyright (c) 2012-
+ *
+ * Written by the Gromacs development team under coordination of
+ * David van der Spoel, Berk Hess, and Erik Lindahl.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the research papers on the package. Check out http://www.gromacs.org
+ *
+ * And Hey:
+ * Gnomes, ROck Monsters And Chili Sauce
+ */
+#ifndef GMX_CPUID_H_
+#define GMX_CPUID_H_
+
+#include <stdio.h>
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#if 0
+} /* fixes auto-indentation problems */
+#endif
+
+
+/* Currently identifiable CPU Vendors */
+enum gmx_cpuid_vendor
+{
+ GMX_CPUID_VENDOR_CANNOTDETECT, /* Should only be used if something fails */
+ GMX_CPUID_VENDOR_UNKNOWN,
+ GMX_CPUID_VENDOR_INTEL,
+ GMX_CPUID_VENDOR_AMD,
+ GMX_CPUID_VENDOR_FUJITSU,
+ GMX_CPUID_VENDOR_IBM,
+ GMX_CPUID_NVENDORS
+};
+
+
+/* CPU feature/property list, to be used as indices into the feature array of the
+ * gmxcpuid_t data structure.
+ *
+ * To facilitate looking things up, we keep this list alphabetical.
+ * The list is NOT exhaustive - we have basically added stuff that might be
+ * useful in an application like Gromacs.
+ *
+ * AMD and Intel tend to share most architectural elements, and even if the
+ * flags might have to be detected in different ways (different cpuid registers),
+ * once the flag is present the functions should be identical. Unfortunately the
+ * trend right now (2012) seems to be that they are diverging. This means that
+ * we need to use specific flags to the compiler to maximize performance, and
+ * then the binaries might not be portable between Intel and AMD as they were
+ * before when we only needed to check for SSE and/or SSE2 support in Gromacs.
+ */
+enum gmx_cpuid_feature
+{
+ GMX_CPUID_FEATURE_CANNOTDETECT, /* Flag set if we could not detect on this CPU */
+ GMX_CPUID_FEATURE_X86_AES, /* x86 advanced encryption standard accel. */
+ GMX_CPUID_FEATURE_X86_APIC, /* APIC support */
+ GMX_CPUID_FEATURE_X86_AVX, /* Advanced vector extensions */
+ GMX_CPUID_FEATURE_X86_AVX2, /* AVX2 including gather support (not used yet) */
+ GMX_CPUID_FEATURE_X86_CLFSH, /* Supports CLFLUSH instruction */
+ GMX_CPUID_FEATURE_X86_CMOV, /* Conditional move insn support */
+ GMX_CPUID_FEATURE_X86_CX8, /* Supports CMPXCHG8B (8-byte compare-exchange) */
+ GMX_CPUID_FEATURE_X86_CX16, /* Supports CMPXCHG16B (16-byte compare-exchg) */
+ GMX_CPUID_FEATURE_X86_F16C, /* Supports 16-bit FP conversion instructions */
+ GMX_CPUID_FEATURE_X86_FMA, /* Fused-multiply add support (mainly for AVX) */
+ GMX_CPUID_FEATURE_X86_FMA4, /* 4-operand FMA, only on AMD for now */
+ GMX_CPUID_FEATURE_X86_HTT, /* Hyper-Threading supported */
+ GMX_CPUID_FEATURE_X86_LAHF_LM, /* LAHF/SAHF support in 64 bits */
+ GMX_CPUID_FEATURE_X86_MISALIGNSSE, /* Support for misaligned SSE data instructions */
+ GMX_CPUID_FEATURE_X86_MMX, /* MMX registers and instructions */
+ GMX_CPUID_FEATURE_X86_MSR, /* Supports Intel model-specific-registers */
+ GMX_CPUID_FEATURE_X86_NONSTOP_TSC, /* Invariant TSC (constant rate in ACPI states) */
+ GMX_CPUID_FEATURE_X86_PCID, /* Process context identifier support */
+ GMX_CPUID_FEATURE_X86_PCLMULDQ, /* Carry-less 64-bit multiplication supported */
+ GMX_CPUID_FEATURE_X86_PDCM, /* Perfmon and Debug Capability */
+ GMX_CPUID_FEATURE_X86_PDPE1GB, /* Support for 1GB pages */
+ GMX_CPUID_FEATURE_X86_POPCNT, /* Supports the POPCNT (population count) insn */
+ GMX_CPUID_FEATURE_X86_PSE, /* Supports 4MB-pages (page size extension) */
+ GMX_CPUID_FEATURE_X86_RDRND, /* RDRAND high-quality hardware random numbers */
+ GMX_CPUID_FEATURE_X86_RDTSCP, /* Serializing rdtscp instruction available */
+ GMX_CPUID_FEATURE_X86_SSE2, /* SSE 2 */
+ GMX_CPUID_FEATURE_X86_SSE3, /* SSE 3 */
+ GMX_CPUID_FEATURE_X86_SSE4A, /* SSE 4A */
+ GMX_CPUID_FEATURE_X86_SSE4_1, /* SSE 4.1 */
+ GMX_CPUID_FEATURE_X86_SSE4_2, /* SSE 4.2 */
+ GMX_CPUID_FEATURE_X86_SSSE3, /* Supplemental SSE3 */
+ GMX_CPUID_FEATURE_X86_TDT, /* TSC deadline timer */
+ GMX_CPUID_FEATURE_X86_X2APIC, /* Extended xAPIC Support */
+ GMX_CPUID_FEATURE_X86_XOP, /* AMD extended instructions, only AMD for now */
+ GMX_CPUID_NFEATURES
+};
+
+
+/* Currently supported acceleration instruction sets, intrinsics or other similar combinations
+ * in Gromacs. There is not always a 1-to-1 correspondence with feature flags; on some AMD
+ * hardware we prefer to use 128bit AVX instructions (although 256-bit ones could be executed),
+ * and we still haven't written the AVX2 kernels.
+ */
+enum gmx_cpuid_acceleration
+{
+ GMX_CPUID_ACCELERATION_CANNOTDETECT, /* Should only be used if something fails */
+ GMX_CPUID_ACCELERATION_NONE,
+ GMX_CPUID_ACCELERATION_X86_SSE2,
+ GMX_CPUID_ACCELERATION_X86_SSE4_1,
+ GMX_CPUID_ACCELERATION_X86_AVX_128_FMA,
+ GMX_CPUID_ACCELERATION_X86_AVX_256,
+ GMX_CPUID_ACCELERATION_SPARC64_HPC_ACE,
+ GMX_CPUID_ACCELERATION_IBM_QPX,
+ GMX_CPUID_NACCELERATIONS
+};
+
+/* Text strings corresponding to CPU vendors */
+extern const char *
+gmx_cpuid_vendor_string[GMX_CPUID_NVENDORS];
+
+/* Text strings for CPU feature indices */
+extern const char *
+gmx_cpuid_feature_string[GMX_CPUID_NFEATURES];
+
+/* Text strings for Gromacs acceleration/instruction sets */
+extern const char *
+gmx_cpuid_acceleration_string[GMX_CPUID_NACCELERATIONS];
+
+
+/* Abstract data type with CPU detection information. Set by gmx_cpuid_init(). */
+typedef struct gmx_cpuid *
+ gmx_cpuid_t;
+
+
+/* Fill the data structure by using CPU detection instructions.
+ * Return 0 on success, 1 if something bad happened.
+ */
+int
+gmx_cpuid_init (gmx_cpuid_t * cpuid);
+
+
+/* Return the vendor id as enumerated type. Use gmx_cpuid_vendor_string[]
+ * to get the corresponding text string.
+ */
+enum gmx_cpuid_vendor
+gmx_cpuid_vendor (gmx_cpuid_t cpuid);
+
+
+/* Return a constant pointer to the processor brand string. */
+const char *
+gmx_cpuid_brand (gmx_cpuid_t cpuid);
+
+
+/* Return processor family version. For a chip of version 1.2.3, this is 1 */
+int
+gmx_cpuid_family (gmx_cpuid_t cpuid);
+
+/* Return processor model version, For a chip of version 1.2.3, this is 2. */
+int
+gmx_cpuid_model (gmx_cpuid_t cpuid);
+
+/* Return processor stepping version, For a chip of version 1.2.3, this is 3. */
+int
+gmx_cpuid_stepping (gmx_cpuid_t cpuid);
+
+
+/* Check whether a particular CPUID feature is set.
+ * Returns 0 if flag "feature" is not set, 1 if the flag is set. We cannot use
+ * gmx_bool here since this file must be possible to compile without simple.h.
+ */
+int
+gmx_cpuid_feature (gmx_cpuid_t cpuid,
+ enum gmx_cpuid_feature feature);
+
+
+/* Return pointers to cpu topology information.
+ *
+ * Important: CPU topology requires more OS support than most other
+ * functions in this file, including support for thread pinning to hardware.
+ * This means it will not work on some platforms, including e.g. Mac OS X.
+ * Thus, it is IMPERATIVE that you check the return value from this routine
+ * before doing anything with the information. It is only if the return
+ * value is zero that the data is valid.
+ *
+ * For the returned values we have:
+ * - nprocessors Total number of logical processors reported by OS
+ * - npackages Usually number of CPU sockets
+ * - ncores_per_package Number of cores in each package
+ * - nhwthreads_per_core Number of hardware threads per core; 2 for hyperthreading.
+ * - package_id Array with the package index for each logical cpu
+ * - core_id Array with local core index for each logical cpu
+ * - hwthread_id Array with local hwthread index for each logical cpu
+ * - locality_order Array with logical cpu numbers, sorted in order
+ * of physical and logical locality in the system.
+ *
+ * All arrays are of length nprocessors.
+ */
+int
+gmx_cpuid_topology(gmx_cpuid_t cpuid,
+ int * nprocessors,
+ int * npackages,
+ int * ncores_per_package,
+ int * nhwthreads_per_core,
+ const int ** package_id,
+ const int ** core_id,
+ const int ** hwthread_id,
+ const int ** locality_order);
+
+/* Enumerated values for x86 SMT enabled-status. Note that this does not refer
+ * to Hyper-Threading support (that is the flag GMX_CPUID_FEATURE_X86_HTT), but
+ * whether Hyper-Threading is _enabled_ and _used_ in bios right now.
+ */
+enum gmx_cpuid_x86_smt
+{
+ GMX_CPUID_X86_SMT_CANNOTDETECT,
+ GMX_CPUID_X86_SMT_DISABLED,
+ GMX_CPUID_X86_SMT_ENABLED
+};
+
+/* Returns the status of x86 SMT support. IMPORTANT: There are non-zero
+ * return values for this routine that still do not indicate supported and
+ * enabled smt/Hyper-Threading. You need to carefully check the return value
+ * against the enumerated type values to see what you are getting.
+ *
+ * Long-term, this functionality will move to a new hardware topology detection
+ * layer, but that will require a lot of new code and a working interface to the
+ * hwloc library. Surprisingly, there is no simple way to find out that
+ * Hyper-Threading is actually turned on without fully enumerating and checking
+ * all the cores, which we presently can only do on Linux. This means a couple
+ * of things:
+ *
+ * 1) If you want to know whether your CPU _supports_ Hyper-Threading in the
+ * first place, check the GMX_CPUID_FEATURE_X86_HTT flag instead!
+ * 2) There are several scenarios where this routine will say that it cannot
+ * detect whether SMT is enabled and used right now.
+ * 3) If you need support on non-Linux x86, you have to write it :-)
+ * 4) Don't invest too much efforts, since this will be replaced with
+ * full hardware topology detection in the future.
+ * 5) Don't worry if the detection does not work. It is not a catastrophe, but
+ * but we get slightly better performance on x86 if we use Hyper-Threading
+ * cores in direct space, but not reciprocal space.
+ *
+ * Since this routine presently only supports Hyper-Threading we say X86_SMT
+ * in order not to give the impression we can detect any SMT. We haven't
+ * even tested the performance on other SMT implementations, so it is not
+ * obvious we shouldn't use SMT there.
+ *
+ * Note that you can get more complete topology information from
+ * gmx_cpuid_topology(), although that requires slightly more OS support.
+ */
+enum gmx_cpuid_x86_smt
+gmx_cpuid_x86_smt(gmx_cpuid_t cpuid);
+
+
+/* Formats a text string (up to n characters) from the data structure.
+ * The output will have max 80 chars between newline characters.
+ */
+int
+gmx_cpuid_formatstring (gmx_cpuid_t cpuid,
+ char * s,
+ int n);
+
+
+/* Suggests a suitable gromacs acceleration based on the support in the
+ * hardware.
+ */
+enum gmx_cpuid_acceleration
+gmx_cpuid_acceleration_suggest (gmx_cpuid_t cpuid);
+
+
+/* Check if this binary was compiled with the same acceleration as we
+ * would suggest for the current hardware. Always print stats to the log file
- FILE * log);
++ * if it is non-NULL, and if we don't have a match, print a warning in log
++ * (if non-NULL) and if print_to_stderr!=0 also to stderr.
+ */
+int
+gmx_cpuid_acceleration_check (gmx_cpuid_t cpuid,
++ FILE * log,
++ int print_to_stderr);
+
+
+/* Release resources used by data structure. Note that the pointer to the
+ * CPU brand string will no longer be valid once this routine has been called.
+ */
+void
+gmx_cpuid_done (gmx_cpuid_t cpuid);
+
+
+
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif /* GMX_CPUID_H_ */
--- /dev/null
- /* the init and consistency functions depend on commrec that may not be
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This file is part of GROMACS.
+ * Copyright (c) 2012-
+ *
+ * Written by the Gromacs development team under coordination of
+ * David van der Spoel, Berk Hess, and Erik Lindahl.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the research papers on the package. Check out http://www.gromacs.org
+ *
+ * And Hey:
+ * GROup of MAchos and Cynical Suckers
+ */
+
+#ifndef GMX_HARDWARE_DETECT_H
+#define GMX_HARDWARE_DETECT_H
+
+#include "types/hw_info.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#if 0
+} /* fixes auto-indentation problems */
+#endif
+
- gmx_bool bForceUseGPU, gmx_bool bTryUseGPU,
- const char *gpu_id);
++/* the init and consistency functions depend on commrec that may not be
+ consistent in cuda because MPI types don't exist there. */
+#ifndef __CUDACC__
+#include "types/commrec.h"
+/* return a pointer to a global hwinfo structure. */
+gmx_hw_info_t *gmx_detect_hardware(FILE *fplog, const t_commrec *cr,
- /* Check the thread count + GPU assignment. This function must
- either be run by all threads that persist (i.e. all tmpi threads),
- or be run before they are created. */
- void gmx_check_hw_runconf_consistency(FILE *fplog, gmx_hw_info_t *hwinfo,
- const t_commrec *cr, int ntmpi_requsted,
- gmx_bool bUseGPU);
++ gmx_bool bDetectGPUs);
+
+void gmx_hardware_info_free(gmx_hw_info_t *hwinfo);
+
- hwinfo = the hwinfo struct
++void gmx_parse_gpu_ids(gmx_gpu_opt_t *gpu_opt);
++
++void gmx_select_gpu_ids(FILE *fplog, const t_commrec *cr,
++ const gmx_gpu_info_t *gpu_info,
++ gmx_bool bForceUseGPU,
++ gmx_gpu_opt_t *gpu_opt);
++
++/* Check the consistency of hw_opt with hwinfo.
++ This function should be called once on each MPI rank. */
++void gmx_check_hw_runconf_consistency(FILE *fplog,
++ const gmx_hw_info_t *hwinfo,
++ const t_commrec *cr,
++ const gmx_hw_opt_t *hw_opt,
++ gmx_bool bUseGPU);
+#endif
+
+
+/* Check whether a GPU is shared among ranks, and return the number of shared
+ gpus
+
- int gmx_count_gpu_dev_shared(const gmx_gpu_info_t *gpu_info);
++ gpu_opt = the gpu options struct
+
+ returns: The number of GPUs shared among ranks, or 0 */
++int gmx_count_gpu_dev_shared(const gmx_gpu_opt_t *gpu_opt);
+
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif /* GMX_HARDWARE_DETECT_H */
--- /dev/null
- #ifdef GMX_IS_X86
+/*
+ * This file is part of the GROMACS molecular simulation package.
+ *
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2012, The GROMACS Development Team
+ * Copyright (c) 2012, by the GROMACS development team, led by
+ * David van der Spoel, Berk Hess, Erik Lindahl, and including many
+ * others, as listed in the AUTHORS file in the top-level source
+ * directory and at http://www.gromacs.org.
+ *
+ * GROMACS is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ *
+ * GROMACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with GROMACS; if not, see
+ * http://www.gnu.org/licenses, or write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * If you want to redistribute modifications to GROMACS, please
+ * consider that scientific software is very special. Version
+ * control is crucial - bugs must be traceable. We will be happy to
+ * consider code for inclusion in the official distribution, but
+ * derived work must not be called official GROMACS. Details are found
+ * in the README & COPYING files - if they are missing, get the
+ * official version at http://www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the research papers on the package. Check out http://www.gromacs.org.
+ */
+
+/* The macros in this file are intended to be used for writing
+ * architecture-independent SIMD intrinsics code.
+ * To support a new architecture, adding macros here should be (nearly)
+ * all that is needed.
+ */
+
+#ifdef _gmx_simd_macros_h_
+#error "gmx_simd_macros.h included twice"
+#else
+#define _gmx_simd_macros_h_
+
+/* NOTE: SSE2 acceleration does not include floor or blendv */
+
+
+/* Uncomment the next line, without other SIMD active, for testing plain-C */
+/* #define GMX_SIMD_REFERENCE_PLAIN_C */
+#ifdef GMX_SIMD_REFERENCE_PLAIN_C
+/* Plain C SIMD reference implementation, also serves as documentation */
+#define GMX_HAVE_SIMD_MACROS
+
+/* In general the reference SIMD supports any SIMD width, including 1.
+ * See types/nb_verlet.h for details
+ */
+#define GMX_SIMD_REF_WIDTH 4
+
+/* Include plain-C reference implementation, also serves as documentation */
+#include "gmx_simd_ref.h"
+
+#define GMX_SIMD_WIDTH_HERE GMX_SIMD_REF_WIDTH
+
+/* float/double SIMD register type */
+#define gmx_mm_pr gmx_simd_ref_pr
+
+/* boolean SIMD register type */
+#define gmx_mm_pb gmx_simd_ref_pb
+
+/* integer SIMD register type, only for table indexing and exclusion masks */
+#define gmx_epi32 gmx_simd_ref_epi32
+#define GMX_SIMD_EPI32_WIDTH GMX_SIMD_REF_EPI32_WIDTH
+
+/* Load GMX_SIMD_WIDTH_HERE reals for memory starting at r */
+#define gmx_load_pr gmx_simd_ref_load_pr
+/* Set all SIMD register elements to *r */
+#define gmx_load1_pr gmx_simd_ref_load1_pr
+#define gmx_set1_pr gmx_simd_ref_set1_pr
+#define gmx_setzero_pr gmx_simd_ref_setzero_pr
+#define gmx_store_pr gmx_simd_ref_store_pr
+
+#define gmx_add_pr gmx_simd_ref_add_pr
+#define gmx_sub_pr gmx_simd_ref_sub_pr
+#define gmx_mul_pr gmx_simd_ref_mul_pr
+/* For the FMA macros below, aim for c=d in code, so FMA3 uses 1 instruction */
+#define gmx_madd_pr gmx_simd_ref_madd_pr
+#define gmx_nmsub_pr gmx_simd_ref_nmsub_pr
+
+#define gmx_max_pr gmx_simd_ref_max_pr
+#define gmx_blendzero_pr gmx_simd_ref_blendzero_pr
+
+#define gmx_round_pr gmx_simd_ref_round_pr
+
+/* Not required, only used to speed up the nbnxn tabulated PME kernels */
+#define GMX_SIMD_HAVE_FLOOR
+#ifdef GMX_SIMD_HAVE_FLOOR
+#define gmx_floor_pr gmx_simd_ref_floor_pr
+#endif
+
+/* Not required, only used when blendv is faster than comparison */
+#define GMX_SIMD_HAVE_BLENDV
+#ifdef GMX_SIMD_HAVE_BLENDV
+#define gmx_blendv_pr gmx_simd_ref_blendv_pr
+#endif
+
+/* Copy the sign of a to b, assumes b >= 0 for efficiency */
+#define gmx_cpsgn_nonneg_pr gmx_simd_ref_cpsgn_nonneg_pr
+
+/* Very specific operation required in the non-bonded kernels */
+#define gmx_masknot_add_pr gmx_simd_ref_masknot_add_pr
+
+/* Comparison */
+#define gmx_cmplt_pr gmx_simd_ref_cmplt_pr
+
+/* Logical operations on SIMD booleans */
+#define gmx_and_pb gmx_simd_ref_and_pb
+#define gmx_or_pb gmx_simd_ref_or_pb
+
+/* Returns a single int (0/1) which tells if any of the 4 booleans is True */
+#define gmx_anytrue_pb gmx_simd_ref_anytrue_pb
+
+/* Conversions only used for PME table lookup */
+#define gmx_cvttpr_epi32 gmx_simd_ref_cvttpr_epi32
+#define gmx_cvtepi32_pr gmx_simd_ref_cvtepi32_pr
+
+/* These two function only need to be approximate, Newton-Raphson iteration
+ * is used for full accuracy in gmx_invsqrt_pr and gmx_inv_pr.
+ */
+#define gmx_rsqrt_pr gmx_simd_ref_rsqrt_pr
+#define gmx_rcp_pr gmx_simd_ref_rcp_pr
+
+/* sqrt+inv+sin+cos+acos+atan2 are used for bonded potentials, exp for PME */
+#define GMX_SIMD_HAVE_EXP
+#ifdef GMX_SIMD_HAVE_EXP
+#define gmx_exp_pr gmx_simd_ref_exp_pr
+#endif
+#define GMX_SIMD_HAVE_TRIGONOMETRIC
+#ifdef GMX_SIMD_HAVE_TRIGONOMETRIC
+#define gmx_sqrt_pr gmx_simd_ref_sqrt_pr
+#define gmx_sincos_pr gmx_simd_ref_sincos_pr
+#define gmx_acos_pr gmx_simd_ref_acos_pr
+#define gmx_atan2_pr gmx_simd_ref_atan2_pr
+#endif
+
+#endif /* GMX_SIMD_REFERENCE_PLAIN_C */
+
+
+/* The same SIMD macros can be translated to SIMD intrinsics (and compiled
+ * to instructions for) different SIMD width and float precision.
+ *
+ * On x86: The gmx_ prefix is replaced by _mm_ or _mm256_ (SSE or AVX).
+ * The _pr suffix is replaced by _ps or _pd (for single or double precision).
+ * Compiler settings will decide if 128-bit intrinsics will
+ * be translated into SSE or AVX instructions.
+ */
+
+
+#ifdef GMX_USE_HALF_WIDTH_SIMD_HERE
+#if defined GMX_X86_AVX_256
+/* We have half SIMD width support, continue */
+#else
+#error "half SIMD width intrinsics are not supported"
+#endif
+#endif
+
- #endif /* GMX_IS_X86 */
++#ifdef GMX_TARGET_X86
+
+#ifdef GMX_X86_SSE2
+/* This is for general x86 SIMD instruction sets that also support SSE2 */
+#define GMX_HAVE_SIMD_MACROS
+
+/* Include the highest supported x86 SIMD intrisics + math functions */
+#ifdef GMX_X86_AVX_256
+#include "gmx_x86_avx_256.h"
+#ifdef GMX_DOUBLE
+#include "gmx_math_x86_avx_256_double.h"
+#else /* GMX_DOUBLE */
+#include "gmx_math_x86_avx_256_single.h"
+#endif /* GMX_DOUBLE */
+#else /* GMX_X86_AVX_256 */
+#ifdef GMX_X86_AVX_128_FMA
+#include "gmx_x86_avx_128_fma.h"
+#ifdef GMX_DOUBLE
+#include "gmx_math_x86_avx_128_fma_double.h"
+#else /* GMX_DOUBLE */
+#include "gmx_math_x86_avx_128_fma_single.h"
+#endif /* GMX_DOUBLE */
+#else /* GMX_X86_AVX_128_FMA */
+#ifdef GMX_X86_SSE4_1
+#include "gmx_x86_sse4_1.h"
+#ifdef GMX_DOUBLE
+#include "gmx_math_x86_sse4_1_double.h"
+#else /* GMX_DOUBLE */
+#include "gmx_math_x86_sse4_1_single.h"
+#endif /* GMX_DOUBLE */
+#else /* GMX_X86_SSE4_1 */
+#ifdef GMX_X86_SSE2
+#include "gmx_x86_sse2.h"
+#ifdef GMX_DOUBLE
+#include "gmx_math_x86_sse2_double.h"
+#else /* GMX_DOUBLE */
+#include "gmx_math_x86_sse2_single.h"
+#endif /* GMX_DOUBLE */
+#else /* GMX_X86_SSE2 */
+#error No x86 acceleration defined
+#endif /* GMX_X86_SSE2 */
+#endif /* GMX_X86_SSE4_1 */
+#endif /* GMX_X86_AVX_128_FMA */
+#endif /* GMX_X86_AVX_256 */
+
+/* exp and trigonometric functions are included above */
+#define GMX_SIMD_HAVE_EXP
+#define GMX_SIMD_HAVE_TRIGONOMETRIC
+
+#if !defined GMX_X86_AVX_256 || defined GMX_USE_HALF_WIDTH_SIMD_HERE
+
+#ifndef GMX_DOUBLE
+
+#define GMX_SIMD_WIDTH_HERE 4
+
+#define gmx_mm_pr __m128
+
+#define gmx_mm_pb __m128
+
+#define gmx_epi32 __m128i
+#define GMX_SIMD_EPI32_WIDTH 4
+
+#define gmx_load_pr _mm_load_ps
+#define gmx_load1_pr _mm_load1_ps
+#define gmx_set1_pr _mm_set1_ps
+#define gmx_setzero_pr _mm_setzero_ps
+#define gmx_store_pr _mm_store_ps
+
+#define gmx_add_pr _mm_add_ps
+#define gmx_sub_pr _mm_sub_ps
+#define gmx_mul_pr _mm_mul_ps
+#ifdef GMX_X86_AVX_128_FMA
+#define GMX_SIMD_HAVE_FMA
+#define gmx_madd_pr(a, b, c) _mm_macc_ps(a, b, c)
+#define gmx_nmsub_pr(a, b, c) _mm_nmacc_ps(a, b, c)
+#else
+#define gmx_madd_pr(a, b, c) _mm_add_ps(c, _mm_mul_ps(a, b))
+#define gmx_nmsub_pr(a, b, c) _mm_sub_ps(c, _mm_mul_ps(a, b))
+#endif
+#define gmx_max_pr _mm_max_ps
+#define gmx_blendzero_pr _mm_and_ps
+
+#define gmx_cmplt_pr _mm_cmplt_ps
+#define gmx_and_pb _mm_and_ps
+#define gmx_or_pb _mm_or_ps
+
+#ifdef GMX_X86_SSE4_1
+#define gmx_round_pr(x) _mm_round_ps(x, 0x0)
+#define GMX_SIMD_HAVE_FLOOR
+#define gmx_floor_pr _mm_floor_ps
+#else
+#define gmx_round_pr(x) _mm_cvtepi32_ps(_mm_cvtps_epi32(x))
+#endif
+
+#ifdef GMX_X86_SSE4_1
+#define GMX_SIMD_HAVE_BLENDV
+#define gmx_blendv_pr _mm_blendv_ps
+#endif
+
+static gmx_inline gmx_mm_pr gmx_cpsgn_nonneg_pr(gmx_mm_pr a, gmx_mm_pr b)
+{
+ /* The value -0.0 has only the sign-bit set */
+ gmx_mm_pr sign_mask = _mm_set1_ps(-0.0);
+ return _mm_or_ps(_mm_and_ps(a, sign_mask), b);
+};
+
+static gmx_inline gmx_mm_pr gmx_masknot_add_pr(gmx_mm_pb a, gmx_mm_pr b, gmx_mm_pr c)
+{
+ return _mm_add_ps(b, _mm_andnot_ps(a, c));
+};
+
+#define gmx_anytrue_pb _mm_movemask_ps
+
+#define gmx_cvttpr_epi32 _mm_cvttps_epi32
+#define gmx_cvtepi32_pr _mm_cvtepi32_ps
+
+#define gmx_rsqrt_pr _mm_rsqrt_ps
+#define gmx_rcp_pr _mm_rcp_ps
+
+#define gmx_exp_pr gmx_mm_exp_ps
+#define gmx_sqrt_pr gmx_mm_sqrt_ps
+#define gmx_sincos_pr gmx_mm_sincos_ps
+#define gmx_acos_pr gmx_mm_acos_ps
+#define gmx_atan2_pr gmx_mm_atan2_ps
+
+#else /* ifndef GMX_DOUBLE */
+
+#define GMX_SIMD_WIDTH_HERE 2
+
+#define gmx_mm_pr __m128d
+
+#define gmx_mm_pb __m128d
+
+#define gmx_epi32 __m128i
+#define GMX_SIMD_EPI32_WIDTH 4
+
+#define gmx_load_pr _mm_load_pd
+#define gmx_load1_pr _mm_load1_pd
+#define gmx_set1_pr _mm_set1_pd
+#define gmx_setzero_pr _mm_setzero_pd
+#define gmx_store_pr _mm_store_pd
+
+#define gmx_add_pr _mm_add_pd
+#define gmx_sub_pr _mm_sub_pd
+#define gmx_mul_pr _mm_mul_pd
+#ifdef GMX_X86_AVX_128_FMA
+#define GMX_SIMD_HAVE_FMA
+#define gmx_madd_pr(a, b, c) _mm_macc_pd(a, b, c)
+#define gmx_nmsub_pr(a, b, c) _mm_nmacc_pd(a, b, c)
+#else
+#define gmx_madd_pr(a, b, c) _mm_add_pd(c, _mm_mul_pd(a, b))
+#define gmx_nmsub_pr(a, b, c) _mm_sub_pd(c, _mm_mul_pd(a, b))
+#endif
+#define gmx_max_pr _mm_max_pd
+#define gmx_blendzero_pr _mm_and_pd
+
+#ifdef GMX_X86_SSE4_1
+#define gmx_round_pr(x) _mm_round_pd(x, 0x0)
+#define GMX_SIMD_HAVE_FLOOR
+#define gmx_floor_pr _mm_floor_pd
+#else
+#define gmx_round_pr(x) _mm_cvtepi32_pd(_mm_cvtpd_epi32(x))
+/* gmx_floor_pr is not used in code for pre-SSE4_1 hardware */
+#endif
+
+#ifdef GMX_X86_SSE4_1
+#define GMX_SIMD_HAVE_BLENDV
+#define gmx_blendv_pr _mm_blendv_pd
+#endif
+
+static gmx_inline gmx_mm_pr gmx_cpsgn_nonneg_pr(gmx_mm_pr a, gmx_mm_pr b)
+{
+ gmx_mm_pr sign_mask = _mm_set1_pd(-0.0);
+ return _mm_or_pd(_mm_and_pd(a, sign_mask), b);
+};
+
+static gmx_inline gmx_mm_pr gmx_masknot_add_pr(gmx_mm_pb a, gmx_mm_pr b, gmx_mm_pr c)
+{
+ return _mm_add_pd(b, _mm_andnot_pd(a, c));
+};
+
+#define gmx_cmplt_pr _mm_cmplt_pd
+
+#define gmx_and_pb _mm_and_pd
+#define gmx_or_pb _mm_or_pd
+
+#define gmx_anytrue_pb _mm_movemask_pd
+
+#define gmx_cvttpr_epi32 _mm_cvttpd_epi32
+#define gmx_cvtepi32_pr _mm_cvtepi32_pd
+
+#define gmx_rsqrt_pr(r) _mm_cvtps_pd(_mm_rsqrt_ps(_mm_cvtpd_ps(r)))
+#define gmx_rcp_pr(r) _mm_cvtps_pd(_mm_rcp_ps(_mm_cvtpd_ps(r)))
+
+#define gmx_exp_pr gmx_mm_exp_pd
+#define gmx_sqrt_pr gmx_mm_sqrt_pd
+#define gmx_sincos_pr gmx_mm_sincos_pd
+#define gmx_acos_pr gmx_mm_acos_pd
+#define gmx_atan2_pr gmx_mm_atan2_pd
+
+#endif /* ifndef GMX_DOUBLE */
+
+#else
+/* We have GMX_X86_AVX_256 and not GMX_USE_HALF_WIDTH_SIMD_HERE,
+ * so we use 256-bit SIMD.
+ */
+
+#ifndef GMX_DOUBLE
+
+#define GMX_SIMD_WIDTH_HERE 8
+
+#define gmx_mm_pr __m256
+
+#define gmx_mm_pb __m256
+
+#define gmx_epi32 __m256i
+#define GMX_SIMD_EPI32_WIDTH 8
+
+#define gmx_load_pr _mm256_load_ps
+#define gmx_load1_pr(x) _mm256_set1_ps((x)[0])
+#define gmx_set1_pr _mm256_set1_ps
+#define gmx_setzero_pr _mm256_setzero_ps
+#define gmx_store_pr _mm256_store_ps
+
+#define gmx_add_pr _mm256_add_ps
+#define gmx_sub_pr _mm256_sub_ps
+#define gmx_mul_pr _mm256_mul_ps
+#define gmx_madd_pr(a, b, c) _mm256_add_ps(c, _mm256_mul_ps(a, b))
+#define gmx_nmsub_pr(a, b, c) _mm256_sub_ps(c, _mm256_mul_ps(a, b))
+#define gmx_max_pr _mm256_max_ps
+#define gmx_blendzero_pr _mm256_and_ps
+
+#define gmx_round_pr(x) _mm256_round_ps(x, 0x0)
+#define GMX_SIMD_HAVE_FLOOR
+#define gmx_floor_pr _mm256_floor_ps
+
+#define GMX_SIMD_HAVE_BLENDV
+#define gmx_blendv_pr _mm256_blendv_ps
+
+static gmx_inline gmx_mm_pr gmx_cpsgn_nonneg_pr(gmx_mm_pr a, gmx_mm_pr b)
+{
+ gmx_mm_pr sign_mask = _mm256_set1_ps(-0.0);
+ return _mm256_or_ps(_mm256_and_ps(a, sign_mask), b);
+};
+
+static gmx_inline gmx_mm_pr gmx_masknot_add_pr(gmx_mm_pb a, gmx_mm_pr b, gmx_mm_pr c)
+{
+ return _mm256_add_ps(b, _mm256_andnot_ps(a, c));
+};
+
+/* Less-than (we use ordered, non-signaling, but that's not required) */
+#define gmx_cmplt_pr(x, y) _mm256_cmp_ps(x, y, 0x11)
+#define gmx_and_pb _mm256_and_ps
+#define gmx_or_pb _mm256_or_ps
+
+#define gmx_anytrue_pb _mm256_movemask_ps
+
+#define gmx_cvttpr_epi32 _mm256_cvttps_epi32
+
+#define gmx_rsqrt_pr _mm256_rsqrt_ps
+#define gmx_rcp_pr _mm256_rcp_ps
+
+#define gmx_exp_pr gmx_mm256_exp_ps
+#define gmx_sqrt_pr gmx_mm256_sqrt_ps
+#define gmx_sincos_pr gmx_mm256_sincos_ps
+#define gmx_acos_pr gmx_mm256_acos_ps
+#define gmx_atan2_pr gmx_mm256_atan2_ps
+
+#else /* ifndef GMX_DOUBLE */
+
+#define GMX_SIMD_WIDTH_HERE 4
+
+#define gmx_mm_pr __m256d
+
+#define gmx_mm_pb __m256d
+
+/* We use 128-bit integer registers because of missing 256-bit operations */
+#define gmx_epi32 __m128i
+#define GMX_SIMD_EPI32_WIDTH 4
+
+#define gmx_load_pr _mm256_load_pd
+#define gmx_load1_pr(x) _mm256_set1_pd((x)[0])
+#define gmx_set1_pr _mm256_set1_pd
+#define gmx_setzero_pr _mm256_setzero_pd
+#define gmx_store_pr _mm256_store_pd
+
+#define gmx_add_pr _mm256_add_pd
+#define gmx_sub_pr _mm256_sub_pd
+#define gmx_mul_pr _mm256_mul_pd
+#define gmx_madd_pr(a, b, c) _mm256_add_pd(c, _mm256_mul_pd(a, b))
+#define gmx_nmsub_pr(a, b, c) _mm256_sub_pd(c, _mm256_mul_pd(a, b))
+#define gmx_max_pr _mm256_max_pd
+#define gmx_blendzero_pr _mm256_and_pd
+
+#define gmx_round_pr(x) _mm256_round_pd(x, 0x0)
+#define GMX_SIMD_HAVE_FLOOR
+#define gmx_floor_pr _mm256_floor_pd
+
+#define GMX_SIMD_HAVE_BLENDV
+#define gmx_blendv_pr _mm256_blendv_pd
+
+static gmx_inline gmx_mm_pr gmx_cpsgn_nonneg_pr(gmx_mm_pr a, gmx_mm_pr b)
+{
+ gmx_mm_pr sign_mask = _mm256_set1_pd(-0.0);
+ return _mm256_or_pd(_mm256_and_pd(a, sign_mask), b);
+};
+
+static gmx_inline gmx_mm_pr gmx_masknot_add_pr(gmx_mm_pb a, gmx_mm_pr b, gmx_mm_pr c)
+{
+ return _mm256_add_pd(b, _mm256_andnot_pd(a, c));
+};
+
+/* Less-than (we use ordered, non-signaling, but that's not required) */
+#define gmx_cmplt_pr(x, y) _mm256_cmp_pd(x, y, 0x11)
+
+#define gmx_and_pb _mm256_and_pd
+#define gmx_or_pb _mm256_or_pd
+
+#define gmx_anytrue_pb _mm256_movemask_pd
+
+#define gmx_cvttpr_epi32 _mm256_cvttpd_epi32
+
+#define gmx_rsqrt_pr(r) _mm256_cvtps_pd(_mm_rsqrt_ps(_mm256_cvtpd_ps(r)))
+#define gmx_rcp_pr(r) _mm256_cvtps_pd(_mm_rcp_ps(_mm256_cvtpd_ps(r)))
+
+#define gmx_exp_pr gmx_mm256_exp_pd
+#define gmx_sqrt_pr gmx_mm256_sqrt_pd
+#define gmx_sincos_pr gmx_mm256_sincos_pd
+#define gmx_acos_pr gmx_mm256_acos_pd
+#define gmx_atan2_pr gmx_mm256_atan2_pd
+
+#endif /* ifndef GMX_DOUBLE */
+
+#endif /* 128- or 256-bit x86 SIMD */
+
+#endif /* GMX_X86_SSE2 */
+
++#endif /* GMX_TARGET_X86 */
+
+#ifdef GMX_CPU_ACCELERATION_IBM_QPX
+
+/* This hack works on the compilers that can reach this code. A real
+ solution with broader scope will be proposed in master branch. */
+#define gmx_always_inline __attribute__((always_inline))
+
+/* This is for the A2 core on BlueGene/Q that supports IBM's QPX
+ vector built-in functions */
+#define GMX_HAVE_SIMD_MACROS
+#ifdef __clang__
+#include <qpxmath.h>
+#else
+#include "mass_simd.h"
+#endif
+
+/* No need to version the code by the precision, because the QPX AXU
+ extends to and truncates from double precision for free. */
+
+#define GMX_SIMD_WIDTH_HERE 4
+typedef vector4double gmx_mm_pr;
+typedef vector4double gmx_mm_pb;
+typedef vector4double gmx_epi32;
+#define GMX_SIMD_EPI32_WIDTH 4
+
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_load_pr(const real *a)
+{
+#ifdef NDEBUG
+ return vec_ld(0, (real *) a);
+#else
+ return vec_lda(0, (real *) a);
+#endif
+}
+
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_load1_pr(const real *a)
+{
+ return vec_splats(*a);
+}
+
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_set1_pr(real a)
+{
+ return vec_splats(a);
+}
+
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_setzero_pr()
+{
+ return vec_splats(0.0);
+}
+
+static gmx_inline void gmx_always_inline gmx_store_pr(real *a, gmx_mm_pr b)
+{
+#ifdef NDEBUG
+ vec_st(b, 0, a);
+#else
+ vec_sta(b, 0, a);
+#endif
+}
+
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_add_pr(gmx_mm_pr a, gmx_mm_pr b)
+{
+ return vec_add(a, b);
+}
+
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_sub_pr(gmx_mm_pr a, gmx_mm_pr b)
+{
+ return vec_sub(a, b);
+}
+
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_mul_pr(gmx_mm_pr a, gmx_mm_pr b)
+{
+ return vec_mul(a, b);
+}
+
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_madd_pr(gmx_mm_pr a, gmx_mm_pr b, gmx_mm_pr c)
+{
+ return vec_madd(a, b, c);
+}
+
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_nmsub_pr(gmx_mm_pr a, gmx_mm_pr b, gmx_mm_pr c)
+{
+ return vec_nmsub(a, b, c);
+}
+
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_max_pr(gmx_mm_pr a, gmx_mm_pr b)
+{
+ return vec_sel(b, a, vec_sub(a, b));
+}
+
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_blendzero_pr(gmx_mm_pr a, gmx_mm_pr b)
+{
+ return vec_sel(gmx_setzero_pr(), a, b);
+}
+
+static gmx_inline gmx_mm_pb gmx_always_inline gmx_cmplt_pr(gmx_mm_pr a, gmx_mm_pr b)
+{
+ return vec_cmplt(a, b);
+}
+
+static gmx_inline gmx_mm_pb gmx_always_inline gmx_and_pb(gmx_mm_pb a, gmx_mm_pb b)
+{
+ return vec_and(a, b);
+}
+
+static gmx_inline gmx_mm_pb gmx_always_inline gmx_or_pb(gmx_mm_pb a, gmx_mm_pb b)
+{
+ return vec_or(a, b);
+}
+
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_round_pr(gmx_mm_pr a)
+{
+ return vec_round(a);
+}
+
+#define GMX_SIMD_HAVE_FLOOR
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_floor_pr(gmx_mm_pr a)
+{
+ return vec_floor(a);
+}
+
+#define GMX_SIMD_HAVE_BLENDV
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_blendv_pr(gmx_mm_pr a, gmx_mm_pr b, gmx_mm_pr c)
+{
+ return vec_sel(b, a, gmx_cmplt_pr(gmx_setzero_pr(), c));
+}
+
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_cpsgn_nonneg_pr(gmx_mm_pr a, gmx_mm_pr b)
+{
+ return vec_cpsgn(a, b);
+};
+
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_masknot_add_pr(gmx_mm_pb a, gmx_mm_pr b, gmx_mm_pr c)
+{
+ return vec_add(b, vec_sel(c, gmx_setzero_pr(), a));
+};
+
+static gmx_inline gmx_bool gmx_always_inline
+GMX_SIMD_IS_TRUE(real x)
+{
+ return x >= 0.0;
+}
+
+static gmx_inline gmx_epi32 gmx_always_inline gmx_cvttpr_epi32(gmx_mm_pr a)
+{
+ return vec_ctiwuz(a);
+}
+/* Don't want this, we have floor */
+/* #define gmx_cvtepi32_pr vec_cvtepi32 */
+
+/* A2 core on BG/Q delivers relative error of 2^-14, whereas Power ISA
+ Architecture only promises 2^-8. So probably no need for
+ Newton-Raphson iterates at single or double. */
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_rsqrt_pr(gmx_mm_pr a)
+{
+ return vec_rsqrte(a);
+}
+
+/* A2 core on BG/Q delivers relative error of 2^-14, whereas Power ISA
+ Architecture only promises 2^-5. So probably no need for
+ Newton-Raphson iterates at single or double. */
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_rcp_pr(gmx_mm_pr a)
+{
+ return vec_re(a);
+}
+
+/* Note that here, and below, we use the built-in SLEEF port when
+ compiling on BlueGene/Q with clang */
+
+#define GMX_SIMD_HAVE_EXP
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_exp_pr(gmx_mm_pr a)
+{
+#ifdef __clang__
+#ifndef GMX_DOUBLE
+ return xexpf(a);
+#else
+ return xexp(a);
+#endif
+#else
+#ifndef GMX_DOUBLE
+ return expf4(a);
+#else
+ return expd4(a);
+#endif
+#endif
+}
+
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_sqrt_pr(gmx_mm_pr a)
+{
+#ifdef NDEBUG
+ return vec_swsqrt_nochk(a);
+#else
+ return vec_swsqrt(a);
+#endif
+}
+
+#define GMX_SIMD_HAVE_TRIGONOMETRIC
+static gmx_inline int gmx_always_inline gmx_sincos_pr(gmx_mm_pr a, gmx_mm_pr *b, gmx_mm_pr *c)
+{
+#ifdef __clang__
+#ifndef GMX_DOUBLE
+ xsincosf(a, b, c);
+#else
+ xsincos(a, b, c);
+#endif
+#else
+#ifndef GMX_DOUBLE
+ sincosf4(a, b, c);
+#else
+ sincosd4(a, b, c);
+#endif
+#endif
+ return 1;
+}
+
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_acos_pr(gmx_mm_pr a)
+{
+#ifdef __clang__
+#ifndef GMX_DOUBLE
+ return xacosf(a);
+#else
+ return xacos(a);
+#endif
+#else
+#ifndef GMX_DOUBLE
+ return acosf4(a);
+#else
+ return acosd4(a);
+#endif
+#endif
+}
+
+/* NB The order of parameters here is correct; the
+ documentation of atan2[df]4 in SIMD MASS is wrong. */
+static gmx_inline gmx_mm_pr gmx_always_inline gmx_atan2_pr(gmx_mm_pr a, gmx_mm_pr b)
+{
+#ifdef __clang__
+#ifndef GMX_DOUBLE
+ return xatan2f(a, b);
+#else
+ return xatan2(a, b);
+#endif
+#else
+#ifndef GMX_DOUBLE
+ return atan2f4(a, b);
+#else
+ return atan2d4(a, b);
+#endif
+#endif
+}
+
+static gmx_inline int gmx_always_inline
+gmx_anytrue_pb(gmx_mm_pb a)
+{
+ /* The "anytrue" is done solely on the QPX AXU (which is the only
+ available FPU). This is awkward, because pretty much no
+ "horizontal" SIMD-vector operations exist, unlike x86 where
+ SSE4.1 added various kinds of horizontal operations. So we have
+ to make do with shifting vector elements and operating on the
+ results. This makes for lots of data dependency, but the main
+ alternative of storing to memory and reloading is not going to
+ help, either. OpenMP over 2 or 4 hardware threads per core will
+ hide much of the latency from the data dependency. The
+ vec_extract() lets the compiler correctly use a floating-point
+ comparison on the zeroth vector element, which avoids needing
+ memory at all.
+ */
+ gmx_mm_pb vec_shifted_left_0 = a;
+ gmx_mm_pb vec_shifted_left_1 = vec_sldw(a, a, 1);
+ gmx_mm_pb vec_shifted_left_2 = vec_sldw(a, a, 2);
+ gmx_mm_pb vec_shifted_left_3 = vec_sldw(a, a, 3);
+
+ gmx_mm_pb vec_return = vec_or(vec_or(vec_shifted_left_2, vec_shifted_left_3),
+ vec_or(vec_shifted_left_0, vec_shifted_left_1));
+ return (0.0 < vec_extract(vec_return, 0));
+};
+
+#undef gmx_always_inline
+
+#endif /* GMX_CPU_ACCELERATION_IBM_QPX */
+
+#ifdef GMX_HAVE_SIMD_MACROS
+/* Generic functions to extract a SIMD aligned pointer from a pointer x.
+ * x should have at least GMX_SIMD_WIDTH_HERE elements extra compared
+ * to how many you want to use, to avoid indexing outside the aligned region.
+ */
+
+static gmx_inline real *
+gmx_simd_align_real(const real *x)
+{
+ return (real *)(((size_t)((x)+GMX_SIMD_WIDTH_HERE)) & (~((size_t)(GMX_SIMD_WIDTH_HERE*sizeof(real)-1))));
+}
+
+static gmx_inline int *
+gmx_simd_align_int(const int *x)
+{
+ return (int *)(((size_t)((x)+GMX_SIMD_WIDTH_HERE)) & (~((size_t)(GMX_SIMD_WIDTH_HERE*sizeof(int )-1))));
+}
+
+
+/* Include the math functions which only need the above macros,
+ * generally these are the ones that don't need masking operations.
+ */
+#ifdef GMX_DOUBLE
+#include "gmx_simd_math_double.h"
+#else
+#include "gmx_simd_math_single.h"
+#endif
+
+
+#endif /* GMX_HAVE_SIMD_MACROS */
+
+#endif /* _gmx_simd_macros_h_ */
--- /dev/null
- /* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
++/*
++ * This file is part of the GROMACS molecular simulation package.
+ *
- *
- * This source code is part of
- *
- * G R O M A C S
- *
- * GROningen MAchine for Chemical Simulations
- *
- * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2010, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
-
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
++ * Copyright (c) 2012,2013, by the GROMACS development team, led by
++ * David van der Spoel, Berk Hess, Erik Lindahl, and including many
++ * others, as listed in the AUTHORS file in the top-level source
++ * directory and at http://www.gromacs.org.
++ *
++ * GROMACS is free software; you can redistribute it and/or
++ * modify it under the terms of the GNU Lesser General Public License
++ * as published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ *
- * If you want to redistribute modifications, please consider that
- * scientific software is very special. Version control is crucial -
- * bugs must be traceable. We will be happy to consider code for
- * inclusion in the official distribution, but derived work must not
- * be called official GROMACS. Details are found in the README & COPYING
- * files - if they are missing, get the official version at www.gromacs.org.
++ * GROMACS is distributed in the hope that it will be useful,
++ * but WITHOUT ANY WARRANTY; without even the implied warranty of
++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
++ * Lesser General Public License for more details.
+ *
- * To help us fund GROMACS development, we humbly ask that you cite
- * the papers on the package - you can find them in the top README file.
++ * You should have received a copy of the GNU Lesser General Public
++ * License along with GROMACS; if not, see
++ * http://www.gnu.org/licenses, or write to the Free Software Foundation,
++ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
- * For more info, check our website at http://www.gromacs.org
++ * If you want to redistribute modifications to GROMACS, please
++ * consider that scientific software is very special. Version
++ * control is crucial - bugs must be traceable. We will be happy to
++ * consider code for inclusion in the official distribution, but
++ * derived work must not be called official GROMACS. Details are found
++ * in the README & COPYING files - if they are missing, get the
++ * official version at http://www.gromacs.org.
+ *
- * And Hey:
- * Gallium Rubidium Oxygen Manganese Argon Carbon Silicon
++ * To help us fund GROMACS development, we humbly ask that you cite
++ * the research papers on the package. Check out http://www.gromacs.org.
+ */
+
+#ifndef _GPU_UTILS_H_
+#define _GPU_UTILS_H_
+
+#include "types/simple.h"
+#include "types/hw_info.h"
+
+#ifdef GMX_GPU
+#define FUNC_TERM_INT ;
+#define FUNC_TERM_VOID ;
+#define FUNC_QUALIFIER
+#else
+#define FUNC_TERM_INT {return -1; }
+#define FUNC_TERM_VOID {}
+#define FUNC_QUALIFIER static
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+FUNC_QUALIFIER
+int do_quick_memtest(int gmx_unused dev_id) FUNC_TERM_INT
+
+FUNC_QUALIFIER
+int do_full_memtest(int gmx_unused dev_id) FUNC_TERM_INT
+
+FUNC_QUALIFIER
+int do_timed_memtest(int gmx_unused dev_id, int gmx_unused time_limit) FUNC_TERM_INT
+
+FUNC_QUALIFIER
+int detect_cuda_gpus(gmx_gpu_info_t gmx_unused *gpu_info, char gmx_unused *err_str) FUNC_TERM_INT
+
+FUNC_QUALIFIER
- void pick_compatible_gpus(gmx_gpu_info_t gmx_unused *gpu_info) FUNC_TERM_VOID
++void pick_compatible_gpus(const gmx_gpu_info_t gmx_unused *gpu_info,
++ gmx_gpu_opt_t gmx_unused *gpu_opt) FUNC_TERM_VOID
+
+FUNC_QUALIFIER
- gmx_bool check_select_cuda_gpus(int gmx_unused *checkres, gmx_gpu_info_t gmx_unused *gpu_info,
- const int gmx_unused *requested_devs, int gmx_unused count) FUNC_TERM_INT
++gmx_bool check_selected_cuda_gpus(int gmx_unused *checkres,
++ const gmx_gpu_info_t gmx_unused *gpu_info,
++ gmx_gpu_opt_t gmx_unused *gpu_opt) FUNC_TERM_INT
+
+FUNC_QUALIFIER
+void free_gpu_info(const gmx_gpu_info_t gmx_unused *gpu_info) FUNC_TERM_VOID
+
+FUNC_QUALIFIER
- gmx_bool init_gpu(int gmx_unused mygpu, char gmx_unused *result_str, const gmx_gpu_info_t gmx_unused *gpu_info) FUNC_TERM_INT
++gmx_bool init_gpu(int gmx_unused mygpu, char gmx_unused *result_str,
++ const gmx_gpu_info_t gmx_unused *gpu_info,
++ const gmx_gpu_opt_t gmx_unused *gpu_opt) FUNC_TERM_INT
+
+FUNC_QUALIFIER
+gmx_bool free_gpu(char gmx_unused *result_str) FUNC_TERM_INT
+
+/*! \brief Returns the device ID of the GPU currently in use.*/
+FUNC_QUALIFIER
+int get_current_gpu_device_id(void) FUNC_TERM_INT
+
+FUNC_QUALIFIER
- int get_gpu_device_id(const gmx_gpu_info_t gmx_unused *gpu_info, int gmx_unused index) FUNC_TERM_INT
++int get_gpu_device_id(const gmx_gpu_info_t gmx_unused *gpu_info,
++ const gmx_gpu_opt_t gmx_unused *gpu_opt,
++ int index) FUNC_TERM_INT
+
+FUNC_QUALIFIER
+void get_gpu_device_info_string(char gmx_unused *s, const gmx_gpu_info_t gmx_unused *gpu_info, int gmx_unused index) FUNC_TERM_VOID
+
++FUNC_QUALIFIER
++size_t sizeof_cuda_dev_info(void) FUNC_TERM_INT
++
+#ifdef __cplusplus
+}
+#endif
+
+#undef FUNC_TERM_INT
+#undef FUNC_TERM_VOID
+#undef FUNC_QUALIFIER
+
+#endif /* _GPU_UTILS_H_ */
--- /dev/null
- /* The options for the thread affinity setting, default: auto */
- enum {
- threadaffSEL, threadaffAUTO, threadaffON, threadaffOFF, threadaffNR
- };
-
- typedef struct {
- int nthreads_tot; /* Total number of threads requested (TMPI) */
- int nthreads_tmpi; /* Number of TMPI threads requested */
- int nthreads_omp; /* Number of OpenMP threads requested */
- int nthreads_omp_pme; /* As nthreads_omp, but for PME only nodes */
- int thread_affinity; /* Thread affinity switch, see enum above */
- int core_pinning_stride; /* Logical core pinning stride */
- int core_pinning_offset; /* Logical core pinning offset */
- char *gpu_id; /* GPU id's to use, each specified as chars */
- } gmx_hw_opt_t;
-
+/*
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * VERSION 3.2.0
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2004, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * Gromacs Runs On Most of All Computer Systems
+ */
+
+#ifndef _mdrun_h
+#define _mdrun_h
+
+#include <stdio.h>
+#include <time.h>
+#include "typedefs.h"
+#include "network.h"
+#include "sim_util.h"
+#include "tgroup.h"
+#include "../fileio/filenm.h"
+#include "mshift.h"
+#include "edsam.h"
+#include "mdebin.h"
+#include "vcm.h"
+#include "vsite.h"
+#include "pull.h"
+#include "update.h"
+#include "types/membedt.h"
+#include "types/globsig.h"
+
+
+#ifdef GMX_THREAD_MPI
+#include "thread_mpi/threads.h"
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define MD_POLARISE (1<<2)
+#define MD_RERUN (1<<4)
+#define MD_RERUN_VSITE (1<<5)
+#define MD_SEPPOT (1<<7)
+#define MD_PARTDEC (1<<9)
+#define MD_DDBONDCHECK (1<<10)
+#define MD_DDBONDCOMM (1<<11)
+#define MD_CONFOUT (1<<12)
+#define MD_REPRODUCIBLE (1<<13)
+#define MD_READ_RNG (1<<14)
+#define MD_APPENDFILES (1<<15)
+#define MD_APPENDFILESSET (1<<21)
+#define MD_KEEPANDNUMCPT (1<<16)
+#define MD_READ_EKIN (1<<17)
+#define MD_STARTFROMCPT (1<<18)
+#define MD_RESETCOUNTERSHALFWAY (1<<19)
+#define MD_TUNEPME (1<<20)
+#define MD_TESTVERLET (1<<22)
+
+/* The options for the domain decomposition MPI task ordering */
+enum {
+ ddnoSEL, ddnoINTERLEAVE, ddnoPP_PME, ddnoCARTESIAN, ddnoNR
+};
+
+/* Variables for temporary use with the deform option,
+ * used in runner.c and md.c.
+ * (These variables should be stored in the tpx file.)
+ */
+extern gmx_large_int_t deform_init_init_step_tpx;
+extern matrix deform_init_box_tpx;
+#ifdef GMX_THREAD_MPI
+extern tMPI_Thread_mutex_t deform_init_box_mutex;
+
+/* The minimum number of atoms per tMPI thread. With fewer atoms than this,
+ * the number of threads will get lowered.
+ */
+#define MIN_ATOMS_PER_MPI_THREAD 90
+#define MIN_ATOMS_PER_GPU 900
+#endif
+
+
+typedef double gmx_integrator_t (FILE *log, t_commrec *cr,
+ int nfile, const t_filenm fnm[],
+ const output_env_t oenv, gmx_bool bVerbose,
+ gmx_bool bCompact, int nstglobalcomm,
+ gmx_vsite_t *vsite, gmx_constr_t constr,
+ int stepout,
+ t_inputrec *inputrec,
+ gmx_mtop_t *mtop, t_fcdata *fcd,
+ t_state *state,
+ t_mdatoms *mdatoms,
+ t_nrnb *nrnb, gmx_wallcycle_t wcycle,
+ gmx_edsam_t ed,
+ t_forcerec *fr,
+ int repl_ex_nst, int repl_ex_nex, int repl_ex_seed,
+ gmx_membed_t membed,
+ real cpt_period, real max_hours,
+ const char *deviceOptions,
+ unsigned long Flags,
+ gmx_walltime_accounting_t walltime_accounting);
+
+/* ROUTINES from md.c */
+
+gmx_integrator_t do_md;
+
+
+/* ROUTINES from minimize.c */
+
+gmx_integrator_t do_steep;
+/* Do steepest descents EM */
+
+gmx_integrator_t do_cg;
+/* Do conjugate gradient EM */
+
+gmx_integrator_t do_lbfgs;
+/* Do conjugate gradient L-BFGS */
+
+gmx_integrator_t do_nm;
+/* Do normal mode analysis */
+
+/* ROUTINES from tpi.c */
+
+gmx_integrator_t do_tpi;
+/* Do test particle insertion */
+
+void init_npt_masses(t_inputrec *ir, t_state *state, t_extmass *MassQ, gmx_bool bInit);
+
+void init_expanded_ensemble(gmx_bool bStateFromCP, t_inputrec *ir, gmx_rng_t *mcrng, df_history_t *dfhist);
+
+int ExpandedEnsembleDynamics(FILE *log, t_inputrec *ir, gmx_enerdata_t *enerd,
+ t_state *state, t_extmass *MassQ, int fep_state, df_history_t *dfhist,
+ gmx_large_int_t step, gmx_rng_t mcrng,
+ rvec *v, t_mdatoms *mdatoms);
+
+void PrintFreeEnergyInfoToFile(FILE *outfile, t_lambda *fep, t_expanded *expand, t_simtemp *simtemp, df_history_t *dfhist,
+ int fep_state, int frequency, gmx_large_int_t step);
+
+void get_mc_state(gmx_rng_t rng, t_state *state);
+
+void set_mc_state(gmx_rng_t rng, t_state *state);
+
+/* check the version */
+void check_ir_old_tpx_versions(t_commrec *cr, FILE *fplog,
+ t_inputrec *ir, gmx_mtop_t *mtop);
+
+/* Allocate and initialize node-local state entries. */
+void set_state_entries(t_state *state, const t_inputrec *ir, int nnodes);
+
+/* Broadcast the data for a simulation, and allocate node-specific settings
+ such as rng generators. */
+void init_parallel(t_commrec *cr, t_inputrec *inputrec,
+ gmx_mtop_t *mtop);
+
+int mdrunner(gmx_hw_opt_t *hw_opt,
+ FILE *fplog, t_commrec *cr, int nfile,
+ const t_filenm fnm[], const output_env_t oenv, gmx_bool bVerbose,
+ gmx_bool bCompact, int nstglobalcomm, ivec ddxyz, int dd_node_order,
+ real rdd, real rconstr, const char *dddlb_opt, real dlb_scale,
+ const char *ddcsx, const char *ddcsy, const char *ddcsz,
+ const char *nbpu_opt,
+ gmx_large_int_t nsteps_cmdline, int nstepout, int resetstep,
+ int nmultisim, int repl_ex_nst, int repl_ex_nex,
+ int repl_ex_seed, real pforce, real cpt_period, real max_hours,
+ const char *deviceOptions, unsigned long Flags);
+/* Driver routine, that calls the different methods */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _mdrun_h */
--- /dev/null
- /* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
++/*
++ * This file is part of the GROMACS molecular simulation package.
+ *
- *
- * This source code is part of
- *
- * G R O M A C S
- *
- * GROningen MAchine for Chemical Simulations
- *
- * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2012, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
++ * Copyright (c) 2012,2013, by the GROMACS development team, led by
++ * David van der Spoel, Berk Hess, Erik Lindahl, and including many
++ * others, as listed in the AUTHORS file in the top-level source
++ * directory and at http://www.gromacs.org.
+ *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
++ * GROMACS is free software; you can redistribute it and/or
++ * modify it under the terms of the GNU Lesser General Public License
++ * as published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ *
- * If you want to redistribute modifications, please consider that
- * scientific software is very special. Version control is crucial -
- * bugs must be traceable. We will be happy to consider code for
- * inclusion in the official distribution, but derived work must not
- * be called official GROMACS. Details are found in the README & COPYING
- * files - if they are missing, get the official version at www.gromacs.org.
++ * GROMACS is distributed in the hope that it will be useful,
++ * but WITHOUT ANY WARRANTY; without even the implied warranty of
++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
++ * Lesser General Public License for more details.
+ *
- * To help us fund GROMACS development, we humbly ask that you cite
- * the papers on the package - you can find them in the top README file.
++ * You should have received a copy of the GNU Lesser General Public
++ * License along with GROMACS; if not, see
++ * http://www.gnu.org/licenses, or write to the Free Software Foundation,
++ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
- * For more info, check our website at http://www.gromacs.org
++ * If you want to redistribute modifications to GROMACS, please
++ * consider that scientific software is very special. Version
++ * control is crucial - bugs must be traceable. We will be happy to
++ * consider code for inclusion in the official distribution, but
++ * derived work must not be called official GROMACS. Details are found
++ * in the README & COPYING files - if they are missing, get the
++ * official version at http://www.gromacs.org.
+ *
- * And Hey:
- * Gallium Rubidium Oxygen Manganese Argon Carbon Silicon
++ * To help us fund GROMACS development, we humbly ask that you cite
++ * the research papers on the package. Check out http://www.gromacs.org.
+ */
+
+#ifndef NBNXN_CUDA_DATA_MGMT_H
+#define NBNXN_CUDA_DATA_MGMT_H
+
+#include "types/simple.h"
+#include "types/interaction_const.h"
+#include "types/nbnxn_cuda_types_ext.h"
+#include "types/hw_info.h"
+
+#ifdef GMX_GPU
+#define FUNC_TERM ;
+#define FUNC_QUALIFIER
+#else
+#define FUNC_TERM {}
+#define FUNC_QUALIFIER static
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*! Initializes the data structures related to CUDA nonbonded calculations. */
+FUNC_QUALIFIER
- void nbnxn_cuda_init(FILE gmx_unused *fplog,
- nbnxn_cuda_ptr_t gmx_unused *p_cu_nb,
- const gmx_gpu_info_t gmx_unused *gpu_info, int gmx_unused my_gpu_index,
++void nbnxn_cuda_init(FILE gmx_unused *fplog,
++ nbnxn_cuda_ptr_t gmx_unused *p_cu_nb,
++ const gmx_gpu_info_t gmx_unused *gpu_info,
++ const gmx_gpu_opt_t gmx_unused *gpu_opt,
++ int gmx_unused my_gpu_index,
+ /* true of both local and non-local are don on GPU */
- gmx_bool gmx_unused bLocalAndNonlocal) FUNC_TERM
++ gmx_bool gmx_unused bLocalAndNonlocal) FUNC_TERM
+
+/*! Initializes simulation constant data. */
+FUNC_QUALIFIER
+void nbnxn_cuda_init_const(nbnxn_cuda_ptr_t gmx_unused cu_nb,
+ const interaction_const_t gmx_unused *ic,
+ const nonbonded_verlet_group_t gmx_unused *nbv_group) FUNC_TERM
+
+/*! Initializes pair-list data for GPU, called at every pair search step. */
+FUNC_QUALIFIER
+void nbnxn_cuda_init_pairlist(nbnxn_cuda_ptr_t gmx_unused cu_nb,
+ const nbnxn_pairlist_t gmx_unused *h_nblist,
+ int gmx_unused iloc) FUNC_TERM
+
+/*! Initializes atom-data on the GPU, called at every pair search step. */
+FUNC_QUALIFIER
+void nbnxn_cuda_init_atomdata(nbnxn_cuda_ptr_t gmx_unused cu_nb,
+ const nbnxn_atomdata_t gmx_unused *atomdata) FUNC_TERM
+
+/*! \brief Update parameters during PP-PME load balancing. */
+FUNC_QUALIFIER
+void nbnxn_cuda_pme_loadbal_update_param(nbnxn_cuda_ptr_t gmx_unused cu_nb,
+ const interaction_const_t gmx_unused *ic) FUNC_TERM
+
+/*! Uploads shift vector to the GPU if the box is dynamic (otherwise just returns). */
+FUNC_QUALIFIER
+void nbnxn_cuda_upload_shiftvec(nbnxn_cuda_ptr_t gmx_unused cu_nb,
+ const nbnxn_atomdata_t gmx_unused *nbatom) FUNC_TERM
+
+/*! Clears GPU outputs: nonbonded force, shift force and energy. */
+FUNC_QUALIFIER
+void nbnxn_cuda_clear_outputs(nbnxn_cuda_ptr_t gmx_unused cu_nb,
+ int gmx_unused flags) FUNC_TERM
+
+/*! Frees all GPU resources used for the nonbonded calculations. */
+FUNC_QUALIFIER
+void nbnxn_cuda_free(FILE gmx_unused *fplog,
+ nbnxn_cuda_ptr_t gmx_unused cu_nb) FUNC_TERM
+
+/*! Returns the GPU timings structure or NULL if GPU is not used or timing is off. */
+FUNC_QUALIFIER
+wallclock_gpu_t * nbnxn_cuda_get_timings(nbnxn_cuda_ptr_t gmx_unused cu_nb)
+#ifdef GMX_GPU
+;
+#else
+{
+ return NULL;
+}
+#endif
+
+/*! Resets nonbonded GPU timings. */
+FUNC_QUALIFIER
+void nbnxn_cuda_reset_timings(nbnxn_cuda_ptr_t gmx_unused cu_nb) FUNC_TERM
+
+/*! Calculates the minimum size of proximity lists to improve SM load balance
+ with CUDA non-bonded kernels. */
+FUNC_QUALIFIER
+int nbnxn_cuda_min_ci_balanced(nbnxn_cuda_ptr_t gmx_unused cu_nb)
+#ifdef GMX_GPU
+;
+#else
+{
+ return -1;
+}
+#endif
+
+/*! Returns if analytical Ewald CUDA kernels are used. */
+FUNC_QUALIFIER
+gmx_bool nbnxn_cuda_is_kernel_ewald_analytical(const nbnxn_cuda_ptr_t gmx_unused cu_nb)
+#ifdef GMX_GPU
+;
+#else
+{
+ return FALSE;
+}
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#undef FUNC_TERM
+#undef FUNC_QUALIFIER
+
+#endif /* NBNXN_CUDA_DATA_MGMT_H */
--- /dev/null
+/*
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * VERSION 3.2.0
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2004, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * Gromacs Runs On Most of All Computer Systems
+ */
+
+#ifndef _network_h
+#define _network_h
+
+
+/*
+ * This module defines the interface of the actual communication routines.
+ */
+
+#include <stdio.h>
+
+#include "types/simple.h"
+#include "types/commrec.h"
+#include "typedefs.h"
+#include "main.h"
+#include "gmx_fatal.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+t_commrec *init_commrec(void);
+/* Allocate, initialize and return the commrec. */
+
+t_commrec *reinitialize_commrec_for_this_thread(const t_commrec *cro);
+/* Initialize communication records for thread-parallel simulations.
+ Must be called on all threads before any communication takes place by
+ the individual threads. Copies the original commrec to
+ thread-local versions (a small memory leak results because we don't
+ deallocate the old shared version). */
+
+void gmx_fill_commrec_from_mpi(t_commrec *cr);
+/* Continues t_commrec construction */
+
+int gmx_node_num(void);
+/* return the number of nodes in the ring */
+
+int gmx_node_rank(void);
+/* return the rank of the node */
+
++int gmx_physicalnode_id_hash(void);
++/* Return a non-negative hash that is, hopefully, unique for each physical node.
++ * This hash is useful for determining hardware locality.
++ */
++
+int gmx_hostname_num(void);
+/* Ostensibly, returns a integer characteristic of and unique to each
+ physical node in the MPI system. If the first part of the MPI
+ hostname (up to the first dot) ends with a number, returns this
+ number. If the first part of the MPI hostname does not ends in a
+ number (0-9 characters), returns 0.
+ */
+
+void gmx_setup_nodecomm(FILE *fplog, t_commrec *cr);
+/* Sets up fast global communication for clusters with multi-core nodes */
+
+void gmx_init_intranode_counters(t_commrec *cr);
+/* Initializes intra-physical-node MPI process/thread counts and ID. */
+
+gmx_bool gmx_mpi_initialized(void);
+/* return TRUE when MPI_Init has been called.
+ * return FALSE when MPI_Init has not been called OR
+ * when GROMACS was compiled without MPI support.
+ */
+
+void gmx_barrier(const t_commrec *cr);
+/* Wait till all processes in cr->mpi_comm_mygroup have reached the barrier */
+
+void gmx_bcast(int nbytes, void *b, const t_commrec *cr);
+/* Broadcast nbytes bytes from the master to cr->mpi_comm_mygroup */
+
+void gmx_bcast_sim(int nbytes, void *b, const t_commrec *cr);
+/* Broadcast nbytes bytes from the sim master to cr->mpi_comm_mysim */
+
+void gmx_sumi(int nr, int r[], const t_commrec *cr);
+/* Calculate the global sum of an array of ints */
+
+void gmx_sumli(int nr, gmx_large_int_t r[], const t_commrec *cr);
+/* Calculate the global sum of an array of large ints */
+
+void gmx_sumf(int nr, float r[], const t_commrec *cr);
+/* Calculate the global sum of an array of floats */
+
+void gmx_sumd(int nr, double r[], const t_commrec *cr);
+/* Calculate the global sum of an array of doubles */
+
+void gmx_sumf_comm(int nr, float r[], MPI_Comm mpi_comm);
+/* Calculate the global sum of an array of floats */
+
+void gmx_sumd_comm(int nr, double r[], MPI_Comm mpi_comm);
+/* Calculate the global sum of an array of doubles */
+
+void gmx_sumi_sim(int nr, int r[], const gmx_multisim_t *ms);
+/* Calculate the sum over the simulations of an array of ints */
+
+void gmx_sumli_sim(int nr, gmx_large_int_t r[], const gmx_multisim_t *ms);
+/* Calculate the sum over the simulations of an array of large ints */
+
+void gmx_sumf_sim(int nr, float r[], const gmx_multisim_t *ms);
+/* Calculate the sum over the simulations of an array of floats */
+
+void gmx_sumd_sim(int nr, double r[], const gmx_multisim_t *ms);
+/* Calculate the sum over the simulations of an array of doubles */
+
+void gmx_abort(int nodeid, int nnodes, int errorno);
+/* Abort the parallel run */
+
+#ifdef GMX_DOUBLE
+#define gmx_sum_comm gmx_sumd_comm
+#define gmx_sum gmx_sumd
+#define gmx_sum_sim gmx_sumd_sim
+#else
+#define gmx_sum_comm gmx_sumf_comm
+#define gmx_sum gmx_sumf
+#define gmx_sum_sim gmx_sumf_sim
+#endif
+
+#ifdef DEBUG_GMX
+#define debug_gmx() do { FILE *fp = debug ? debug : stderr; \
+ if (bDebugMode()) { fprintf(fp, "NODEID=%d, %s %d\n", gmx_mpi_initialized() ? gmx_node_rank() : -1, __FILE__, __LINE__); } fflush(fp); } while (0)
+#else
+#define debug_gmx()
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif /* _network_h */
--- /dev/null
- * This source code is part of
- *
- * G R O M A C S
- *
- * GROningen MAchine for Chemical Simulations
- *
- * VERSION 3.2.0
- * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+/*
++ * This file is part of the GROMACS molecular simulation package.
+ *
-
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2004, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
- * If you want to redistribute modifications, please consider that
- * scientific software is very special. Version control is crucial -
- * bugs must be traceable. We will be happy to consider code for
- * inclusion in the official distribution, but derived work must not
- * be called official GROMACS. Details are found in the README & COPYING
- * files - if they are missing, get the official version at www.gromacs.org.
++ * Copyright (c) 2012,2013, by the GROMACS development team, led by
++ * David van der Spoel, Berk Hess, Erik Lindahl, and including many
++ * others, as listed in the AUTHORS file in the top-level source
++ * directory and at http://www.gromacs.org.
++ *
++ * GROMACS is free software; you can redistribute it and/or
++ * modify it under the terms of the GNU Lesser General Public License
++ * as published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ *
- * To help us fund GROMACS development, we humbly ask that you cite
- * the papers on the package - you can find them in the top README file.
++ * GROMACS is distributed in the hope that it will be useful,
++ * but WITHOUT ANY WARRANTY; without even the implied warranty of
++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
++ * Lesser General Public License for more details.
+ *
- * For more info, check our website at http://www.gromacs.org
++ * You should have received a copy of the GNU Lesser General Public
++ * License along with GROMACS; if not, see
++ * http://www.gnu.org/licenses, or write to the Free Software Foundation,
++ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
- * And Hey:
- * Gromacs Runs On Most of All Computer Systems
++ * If you want to redistribute modifications to GROMACS, please
++ * consider that scientific software is very special. Version
++ * control is crucial - bugs must be traceable. We will be happy to
++ * consider code for inclusion in the official distribution, but
++ * derived work must not be called official GROMACS. Details are found
++ * in the README & COPYING files - if they are missing, get the
++ * official version at http://www.gromacs.org.
+ *
- * This routine only uses characters for which isalnum(c) is true,
- * and all characters are converted to upper case.
++ * To help us fund GROMACS development, we humbly ask that you cite
++ * the research papers on the package. Check out http://www.gromacs.org.
+ */
+/*! \file
+ * \brief Generic string handling functions.
+ */
+#ifndef _string2_h
+#define _string2_h
+
+/*
+ *
+ * string2.h
+ * David van der Spoel
+ *
+ */
+
+
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include <time.h>
+#include <errno.h>
+#include "../utility/gmx_header_config.h"
+
+#include "types/simple.h"
+
+/* Suppress Cygwin compiler warnings from using newlib version of
+ * ctype.h */
+#ifdef GMX_CYGWIN
+#undef isdigit
+#undef isstring
+#undef isspace
+#undef isalnum
+#undef isalpha
+#undef ispunct
+#undef isxdigit
+#undef isupper
+#undef islower
+#undef toupper
+#undef tolower
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#if 0
+}
+#endif
+
+/** Continuation character. */
+#define CONTINUE '\\'
+/** Comment sign to use. */
+#define COMMENTSIGN ';'
+
+/*! \brief
+ * Strip trailing spaces and if s ends with a ::CONTINUE remove that too.
+ *
+ * \returns TRUE if s ends with a CONTINUE, FALSE otherwise.
+ */
+int continuing(char *s);
+
+/*! \brief
+ * Reads a line from a stream.
+ *
+ * This routine reads a string from stream of max length n
+ * and zero terminated, without newlines.
+ * \p s should be long enough (>= \p n)
+ */
+char *fgets2(char *s, int n, FILE *stream);
+
+/** Remove portion of a line after a ::COMMENTSIGN. */
+void strip_comment(char *line);
+
+/** Make a string uppercase. */
+void upstring(char *str);
+
+/** Remove leading whitespace from a string. */
+void ltrim(char *str);
+
+/** Remove trailing whitespace from a string. */
+void rtrim(char *str);
+
+/** Remove leading and trailing whitespace from a string. */
+void trim(char *str);
+
+/** Portable version of ctime_r. */
+char *gmx_ctime_r(const time_t *clock, char *buf, int n);
+
+/** Prints creation time stamp and user information into a file as comments. */
+void nice_header(FILE *out, const char *fn);
+
+/** Version of gmx_strcasecmp() that also ignores '-' and '_'. */
+int gmx_strcasecmp_min(const char *str1, const char *str2);
+/** Version of gmx_strncasecmp() that also ignores '-' and '_'. */
+int gmx_strncasecmp_min(const char *str1, const char *str2, int n);
+
+/** Case-insensitive strcmp(). */
+int gmx_strcasecmp(const char *str1, const char *str2);
+/** Case-insensitive strncmp(). */
+int gmx_strncasecmp(const char *str1, const char *str2, int n);
+
+/** Creates a duplicate of \p src. */
+char *gmx_strdup(const char *src);
+/** Duplicates first \p n characters of \p src. */
+char *gmx_strndup(const char *src, int n);
+
+/*! \brief
+ * Pattern matching with wildcards.
+ *
+ * \param[in] pattern Pattern to match against.
+ * \param[in] str String to match.
+ * \returns 0 on match, GMX_NO_WCMATCH if there is no match.
+ *
+ * Matches \p str against \p pattern, which may contain * and ? wildcards.
+ * All other characters are matched literally.
+ * Currently, it is not possible to match literal * or ?.
+ */
+int gmx_wcmatch(const char *pattern, const char *str);
+
+/** Magic hash initialization number from Dan J. Bernstein. */
+extern const unsigned int
+ gmx_string_hash_init;
+
+/*! \brief
+ * Return a hash of the string according to Dan J. Bernsteins algorithm.
+ *
+ * \param[in] s String to calculate hash for.
+ * \param[in] hash_init Initial (or previous) hash value.
+ * \returns Updated hash value (hash_init combined with string hash).
+ *
+ * On the first invocation for a new string, use the constant
+ * gmx_string_hash_init for the second argument. If you want to create a hash
+ * corresponding to several concatenated strings, provide the returned hash
+ * value as hash_init for the second string, etc.
+ */
+unsigned int
++gmx_string_fullhash_func(const char *s, unsigned int hash_init);
++
++/*! \brief
++ * Return a hash of the string according to Dan J. Bernsteins algorithm.
++ *
++ * \param[in] s String to calculate hash for.
++ * \param[in] hash_init Initial (or previous) hash value.
++ * \returns Updated hash value (hash_init combined with string hash).
++ *
++ * Identical to gmx_string_fullhash_func, except that
++ * this routine only uses characters for which isalnum(c) is true,
++ * and all characters are converted to upper case.
++ */
++unsigned int
+gmx_string_hash_func(const char *s, unsigned int hash_init);
+
+/** Return value for gmx_wcmatch() when there is no match. */
+#define GMX_NO_WCMATCH 1
+
+/** Our implementation of strsep, the thread-safe replacement for strtok. */
+char *gmx_strsep(char **stringp, const char *delim);
+
+/*! \brief
+ * Wraps lines, optionally indenting lines.
+ *
+ * Wraps lines at \p linewidth, indenting all following lines by \p indent
+ * spaces. A temp buffer is allocated and returned, which can be disposed of
+ * if no longer needed.
+ * If \p bIndentFirst is FALSE, then the first line will not be indented, only
+ * the lines that are created due to wapping.
+ */
+char *wrap_lines(const char *buf, int line_width, int indent,
+ gmx_bool bIndentFirst);
+
+/** Implementation of the well-known Perl function split. */
+char **split(char sep, const char *str);
+
+/*! \brief
+ * Convert a string to gmx_large_int_t.
+ *
+ * This method works as the standard library function strtol(), except that it
+ * does not support different bases.
+ *
+ * \attention
+ * The following differences are present from the standard behavior:
+ * - \p endptr cannot be NULL.
+ * - If an overflow occurs, returns zero and \p *endptr will equal \p str.
+ * errno is still set to ERANGE.
+ */
+gmx_large_int_t str_to_large_int_t(const char *str, char **endptr);
+
+#ifdef GMX_NATIVE_WINDOWS
+#define snprintf _snprintf
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _string2_h */
--- /dev/null
+/*
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * VERSION 3.2.0
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2004, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * GRoups of Organic Molecules in ACtion for Science
+ */
+
+#include "ns.h"
+#include "genborn.h"
+#include "qmmmrec.h"
+#include "idef.h"
+#include "nb_verlet.h"
+#include "interaction_const.h"
+#include "hw_info.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#if 0
+} /* fixes auto-indentation problems */
+#endif
+
+/* Abstract type for PME that is defined only in the routine that use them. */
+typedef struct gmx_pme *gmx_pme_t;
+
+
+
+/* Structure describing the data in a single table */
+typedef struct
+{
+ enum gmx_table_interaction interaction; /* Types of interactions stored in this table */
+ enum gmx_table_format format; /* Interpolation type and data format */
+
+ real r; /* range of the table */
+ int n; /* n+1 is the number of table points */
+ real scale; /* distance (nm) between two table points */
+ real scale_exp; /* distance for exponential part of VdW table, not always used */
+ real * data; /* the actual table data */
+
+ /* Some information about the table layout. This can also be derived from the interpolation
+ * type and the table interactions, but it is convenient to have here for sanity checks, and it makes it
+ * much easier to access the tables in the nonbonded kernels when we can set the data from variables.
+ * It is always true that stride = formatsize*ninteractions
+ */
+ int formatsize; /* Number of fp variables for each table point (1 for F, 2 for VF, 4 for YFGH, etc.) */
+ int ninteractions; /* Number of interactions in table, 1 for coul-only, 3 for coul+rep+disp. */
+ int stride; /* Distance to next table point (number of fp variables per table point in total) */
+} t_forcetable;
+
+typedef struct
+{
+ t_forcetable table_elec;
+ t_forcetable table_vdw;
+ t_forcetable table_elec_vdw;
+
+ /* The actual neighbor lists, short and long range, see enum above
+ * for definition of neighborlist indices.
+ */
+ t_nblist nlist_sr[eNL_NR];
+ t_nblist nlist_lr[eNL_NR];
+} t_nblists;
+
+/* macros for the cginfo data in forcerec */
+/* The maximum cg size in cginfo is 63
+ * because we only have space for 6 bits in cginfo,
+ * this cg size entry is actually only read with domain decomposition.
+ * But there is a smaller limit due to the t_excl data structure
+ * which is defined in nblist.h.
+ */
+#define SET_CGINFO_GID(cgi, gid) (cgi) = (((cgi) & ~65535) | (gid) )
+#define GET_CGINFO_GID(cgi) ( (cgi) & 65535)
+#define SET_CGINFO_EXCL_INTRA(cgi) (cgi) = ((cgi) | (1<<16))
+#define GET_CGINFO_EXCL_INTRA(cgi) ( (cgi) & (1<<16))
+#define SET_CGINFO_EXCL_INTER(cgi) (cgi) = ((cgi) | (1<<17))
+#define GET_CGINFO_EXCL_INTER(cgi) ( (cgi) & (1<<17))
+#define SET_CGINFO_SOLOPT(cgi, opt) (cgi) = (((cgi) & ~(3<<18)) | ((opt)<<18))
+#define GET_CGINFO_SOLOPT(cgi) (((cgi)>>18) & 3)
+#define SET_CGINFO_CONSTR(cgi) (cgi) = ((cgi) | (1<<20))
+#define GET_CGINFO_CONSTR(cgi) ( (cgi) & (1<<20))
+#define SET_CGINFO_SETTLE(cgi) (cgi) = ((cgi) | (1<<21))
+#define GET_CGINFO_SETTLE(cgi) ( (cgi) & (1<<21))
+/* This bit is only used with bBondComm in the domain decomposition */
+#define SET_CGINFO_BOND_INTER(cgi) (cgi) = ((cgi) | (1<<22))
+#define GET_CGINFO_BOND_INTER(cgi) ( (cgi) & (1<<22))
+#define SET_CGINFO_HAS_VDW(cgi) (cgi) = ((cgi) | (1<<23))
+#define GET_CGINFO_HAS_VDW(cgi) ( (cgi) & (1<<23))
+#define SET_CGINFO_HAS_Q(cgi) (cgi) = ((cgi) | (1<<24))
+#define GET_CGINFO_HAS_Q(cgi) ( (cgi) & (1<<24))
+#define SET_CGINFO_NATOMS(cgi, opt) (cgi) = (((cgi) & ~(63<<25)) | ((opt)<<25))
+#define GET_CGINFO_NATOMS(cgi) (((cgi)>>25) & 63)
+
+
+/* Value to be used in mdrun for an infinite cut-off.
+ * Since we need to compare with the cut-off squared,
+ * this value should be slighlty smaller than sqrt(GMX_FLOAT_MAX).
+ */
+#define GMX_CUTOFF_INF 1E+18
+
+/* enums for the neighborlist type */
+enum {
+ enbvdwNONE, enbvdwLJ, enbvdwBHAM, enbvdwTAB, enbvdwNR
+};
+/* OOR is "one over r" -- standard coul */
+enum {
+ enbcoulNONE, enbcoulOOR, enbcoulRF, enbcoulTAB, enbcoulGB, enbcoulFEWALD, enbcoulNR
+};
+
+enum {
+ egCOULSR, egLJSR, egBHAMSR, egCOULLR, egLJLR, egBHAMLR,
+ egCOUL14, egLJ14, egGB, egNR
+};
+
+typedef struct {
+ int nener; /* The number of energy group pairs */
+ real *ener[egNR]; /* Energy terms for each pair of groups */
+} gmx_grppairener_t;
+
+typedef struct {
+ real term[F_NRE]; /* The energies for all different interaction types */
+ gmx_grppairener_t grpp;
+ double dvdl_lin[efptNR]; /* Contributions to dvdl with linear lam-dependence */
+ double dvdl_nonlin[efptNR]; /* Idem, but non-linear dependence */
+ int n_lambda;
+ int fep_state; /*current fep state -- just for printing */
+ double *enerpart_lambda; /* Partial energy for lambda and flambda[] */
+ real foreign_term[F_NRE]; /* alternate array for storing foreign lambda energies */
+ gmx_grppairener_t foreign_grpp; /* alternate array for storing foreign lambda energies */
+} gmx_enerdata_t;
+/* The idea is that dvdl terms with linear lambda dependence will be added
+ * automatically to enerpart_lambda. Terms with non-linear lambda dependence
+ * should explicitly determine the energies at foreign lambda points
+ * when n_lambda > 0.
+ */
+
+typedef struct {
+ int cg_start;
+ int cg_end;
+ int cg_mod;
+ int *cginfo;
+} cginfo_mb_t;
+
+
+/* ewald table type */
+typedef struct ewald_tab *ewald_tab_t;
+
+typedef struct {
+ rvec *f;
+ int f_nalloc;
+ unsigned red_mask; /* Mask for marking which parts of f are filled */
+ rvec *fshift;
+ real ener[F_NRE];
+ gmx_grppairener_t grpp;
+ real Vcorr;
+ real dvdl[efptNR];
+ tensor vir;
+} f_thread_t;
+
+typedef struct {
+ interaction_const_t *ic;
+
+ /* Domain Decomposition */
+ gmx_bool bDomDec;
+
+ /* PBC stuff */
+ int ePBC;
+ gmx_bool bMolPBC;
+ int rc_scaling;
+ rvec posres_com;
+ rvec posres_comB;
+
+ const gmx_hw_info_t *hwinfo;
++ const gmx_gpu_opt_t *gpu_opt;
+ gmx_bool use_cpu_acceleration;
+
+ /* Interaction for calculated in kernels. In many cases this is similar to
+ * the electrostatics settings in the inputrecord, but the difference is that
+ * these variables always specify the actual interaction in the kernel - if
+ * we are tabulating reaction-field the inputrec will say reaction-field, but
+ * the kernel interaction will say cubic-spline-table. To be safe we also
+ * have a kernel-specific setting for the modifiers - if the interaction is
+ * tabulated we already included the inputrec modification there, so the kernel
+ * modification setting will say 'none' in that case.
+ */
+ int nbkernel_elec_interaction;
+ int nbkernel_vdw_interaction;
+ int nbkernel_elec_modifier;
+ int nbkernel_vdw_modifier;
+
+ /* Use special N*N kernels? */
+ gmx_bool bAllvsAll;
+ /* Private work data */
+ void *AllvsAll_work;
+ void *AllvsAll_workgb;
+
+ /* Cut-Off stuff.
+ * Infinite cut-off's will be GMX_CUTOFF_INF (unlike in t_inputrec: 0).
+ */
+ real rlist, rlistlong;
+
+ /* Dielectric constant resp. multiplication factor for charges */
+ real zsquare, temp;
+ real epsilon_r, epsilon_rf, epsfac;
+
+ /* Constants for reaction fields */
+ real kappa, k_rf, c_rf;
+
+ /* Charge sum and dipole for topology A/B ([0]/[1]) for Ewald corrections */
+ double qsum[2];
+ double q2sum[2];
+ rvec mu_tot[2];
+
+ /* Dispersion correction stuff */
+ int eDispCorr;
+
+ /* The shift of the shift or user potentials */
+ real enershiftsix;
+ real enershifttwelve;
+ /* Integrated differces for energy and virial with cut-off functions */
+ real enerdiffsix;
+ real enerdifftwelve;
+ real virdiffsix;
+ real virdifftwelve;
+ /* Constant for long range dispersion correction (average dispersion)
+ * for topology A/B ([0]/[1]) */
+ real avcsix[2];
+ /* Constant for long range repulsion term. Relative difference of about
+ * 0.1 percent with 0.8 nm cutoffs. But hey, it's cheap anyway...
+ */
+ real avctwelve[2];
+
+ /* Fudge factors */
+ real fudgeQQ;
+
+ /* Table stuff */
+ gmx_bool bcoultab;
+ gmx_bool bvdwtab;
+ /* The normal tables are in the nblists struct(s) below */
+ t_forcetable tab14; /* for 1-4 interactions only */
+
+ /* PPPM & Shifting stuff */
+ int coulomb_modifier;
+ real rcoulomb_switch, rcoulomb;
+ real *phi;
+
+ /* VdW stuff */
+ int vdw_modifier;
+ double reppow;
+ real rvdw_switch, rvdw;
+ real bham_b_max;
+
+ /* Free energy */
+ int efep;
+ real sc_alphavdw;
+ real sc_alphacoul;
+ int sc_power;
+ real sc_r_power;
+ real sc_sigma6_def;
+ real sc_sigma6_min;
+ gmx_bool bSepDVDL;
+
+ /* NS Stuff */
+ int eeltype;
+ int vdwtype;
+ int cg0, hcg;
+ /* solvent_opt contains the enum for the most common solvent
+ * in the system, which will be optimized.
+ * It can be set to esolNO to disable all water optimization */
+ int solvent_opt;
+ int nWatMol;
+ gmx_bool bGrid;
+ gmx_bool bExcl_IntraCGAll_InterCGNone;
+ cginfo_mb_t *cginfo_mb;
+ int *cginfo;
+ rvec *cg_cm;
+ int cg_nalloc;
+ rvec *shift_vec;
+
+ /* The neighborlists including tables */
+ int nnblists;
+ int *gid2nblists;
+ t_nblists *nblists;
+
+ int cutoff_scheme; /* group- or Verlet-style cutoff */
+ gmx_bool bNonbonded; /* true if nonbonded calculations are *not* turned off */
+ nonbonded_verlet_t *nbv;
+
+ /* The wall tables (if used) */
+ int nwall;
+ t_forcetable **wall_tab;
+
+ /* The number of charge groups participating in do_force_lowlevel */
+ int ncg_force;
+ /* The number of atoms participating in do_force_lowlevel */
+ int natoms_force;
+ /* The number of atoms participating in force and constraints */
+ int natoms_force_constr;
+ /* The allocation size of vectors of size natoms_force */
+ int nalloc_force;
+
+ /* Twin Range stuff, f_twin has size natoms_force */
+ gmx_bool bTwinRange;
+ int nlr;
+ rvec *f_twin;
+
+ /* Forces that should not enter into the virial summation:
+ * PPPM/PME/Ewald/posres
+ */
+ gmx_bool bF_NoVirSum;
+ int f_novirsum_n;
+ int f_novirsum_nalloc;
+ rvec *f_novirsum_alloc;
+ /* Pointer that points to f_novirsum_alloc when pressure is calcaluted,
+ * points to the normal force vectors wen pressure is not requested.
+ */
+ rvec *f_novirsum;
+
+ /* Long-range forces and virial for PPPM/PME/Ewald */
+ gmx_pme_t pmedata;
+ tensor vir_el_recip;
+
+ /* PME/Ewald stuff */
+ gmx_bool bEwald;
+ real ewaldcoeff;
+ ewald_tab_t ewald_table;
+
+ /* Virial Stuff */
+ rvec *fshift;
+ rvec vir_diag_posres;
+ dvec vir_wall_z;
+
+ /* Non bonded Parameter lists */
+ int ntype; /* Number of atom types */
+ gmx_bool bBHAM;
+ real *nbfp;
+
+ /* Energy group pair flags */
+ int *egp_flags;
+
+ /* Shell molecular dynamics flexible constraints */
+ real fc_stepsize;
+
+ /* Generalized born implicit solvent */
+ gmx_bool bGB;
+ /* Generalized born stuff */
+ real gb_epsilon_solvent;
+ /* Table data for GB */
+ t_forcetable gbtab;
+ /* VdW radius for each atomtype (dim is thus ntype) */
+ real *atype_radius;
+ /* Effective radius (derived from effective volume) for each type */
+ real *atype_vol;
+ /* Implicit solvent - surface tension for each atomtype */
+ real *atype_surftens;
+ /* Implicit solvent - radius for GB calculation */
+ real *atype_gb_radius;
+ /* Implicit solvent - overlap for HCT model */
+ real *atype_S_hct;
+ /* Generalized born interaction data */
+ gmx_genborn_t *born;
+
+ /* Table scale for GB */
+ real gbtabscale;
+ /* Table range for GB */
+ real gbtabr;
+ /* GB neighborlists (the sr list will contain for each atom all other atoms
+ * (for use in the SA calculation) and the lr list will contain
+ * for each atom all atoms 1-4 or greater (for use in the GB calculation)
+ */
+ t_nblist gblist_sr;
+ t_nblist gblist_lr;
+ t_nblist gblist;
+
+ /* Inverse square root of the Born radii for implicit solvent */
+ real *invsqrta;
+ /* Derivatives of the potential with respect to the Born radii */
+ real *dvda;
+ /* Derivatives of the Born radii with respect to coordinates */
+ real *dadx;
+ real *dadx_rawptr;
+ int nalloc_dadx; /* Allocated size of dadx */
+
+ /* If > 0 signals Test Particle Insertion,
+ * the value is the number of atoms of the molecule to insert
+ * Only the energy difference due to the addition of the last molecule
+ * should be calculated.
+ */
+ gmx_bool n_tpi;
+
+ /* Neighbor searching stuff */
+ gmx_ns_t ns;
+
+ /* QMMM stuff */
+ gmx_bool bQMMM;
+ t_QMMMrec *qr;
+
+ /* QM-MM neighborlists */
+ t_nblist QMMMlist;
+
+ /* Limit for printing large forces, negative is don't print */
+ real print_force;
+
+ /* coarse load balancing time measurement */
+ double t_fnbf;
+ double t_wait;
+ int timesteps;
+
+ /* parameter needed for AdResS simulation */
+ int adress_type;
+ gmx_bool badress_tf_full_box;
+ real adress_const_wf;
+ real adress_ex_width;
+ real adress_hy_width;
+ int adress_icor;
+ int adress_site;
+ rvec adress_refs;
+ int n_adress_tf_grps;
+ int * adress_tf_table_index;
+ int *adress_group_explicit;
+ t_forcetable * atf_tabs;
+ real adress_ex_forcecap;
+ gmx_bool adress_do_hybridpairs;
+
+ /* User determined parameters, copied from the inputrec */
+ int userint1;
+ int userint2;
+ int userint3;
+ int userint4;
+ real userreal1;
+ real userreal2;
+ real userreal3;
+ real userreal4;
+
+ /* Thread local force and energy data */
+ /* FIXME move to bonded_thread_data_t */
+ int nthreads;
+ int red_ashift;
+ int red_nblock;
+ f_thread_t *f_t;
+
+ /* Exclusion load distribution over the threads */
+ int *excl_load;
+} t_forcerec;
+
+/* Important: Starting with Gromacs-4.6, the values of c6 and c12 in the nbfp array have
+ * been scaled by 6.0 or 12.0 to save flops in the kernels. We have corrected this everywhere
+ * in the code, but beware if you are using these macros externally.
+ */
+#define C6(nbfp, ntp, ai, aj) (nbfp)[2*((ntp)*(ai)+(aj))]
+#define C12(nbfp, ntp, ai, aj) (nbfp)[2*((ntp)*(ai)+(aj))+1]
+#define BHAMC(nbfp, ntp, ai, aj) (nbfp)[3*((ntp)*(ai)+(aj))]
+#define BHAMA(nbfp, ntp, ai, aj) (nbfp)[3*((ntp)*(ai)+(aj))+1]
+#define BHAMB(nbfp, ntp, ai, aj) (nbfp)[3*((ntp)*(ai)+(aj))+2]
+
+#ifdef __cplusplus
+}
+#endif
--- /dev/null
- gmx_bool bUserSet; /* true if the GPUs in cuda_dev_use are manually provided by the user */
-
- int ncuda_dev_use; /* number of devices selected to be used */
- int *cuda_dev_use; /* index of the devices selected to be used */
- int ncuda_dev; /* total number of devices detected */
- cuda_dev_info_ptr_t cuda_dev; /* devices detected in the system (per node) */
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This file is part of GROMACS.
+ * Copyright (c) 2012-
+ *
+ * Written by the Gromacs development team under coordination of
+ * David van der Spoel, Berk Hess, and Erik Lindahl.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the research papers on the package. Check out http://www.gromacs.org
+ *
+ * And Hey:
+ * Gromacs Runs On Most of All Computer Systems
+ */
+
+#ifndef HWINFO_H
+#define HWINFO_H
+
+#include "simple.h"
+#include "nbnxn_cuda_types_ext.h"
+#include "../gmx_cpuid.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#if 0
+} /* fixes auto-indentation problems */
+#endif
+
+/* Possible results of the GPU detection/check.
+ *
+ * The egpuInsane value means that during the sanity checks an error
+ * occurred that indicates malfunctioning of the device, driver, or
+ * incompatible driver/runtime. */
+typedef enum
+{
+ egpuCompatible = 0, egpuNonexistent, egpuIncompatible, egpuInsane
+} e_gpu_detect_res_t;
+
+/* Textual names of the GPU detection/check results (see e_gpu_detect_res_t). */
+static const char * const gpu_detect_res_str[] =
+{
+ "compatible", "inexistent", "incompatible", "insane"
+};
+
+/* GPU device information -- for now with only CUDA devices.
+ * The gmx_hardware_detect module initializes it. */
+typedef struct
+{
- gmx_bool bCanUseGPU; /* True if compatible GPUs are detected during hardware detection */
++ gmx_bool bDetectGPUs; /* Did we try to detect GPUs? */
++ int ncuda_dev; /* total number of devices detected */
++ cuda_dev_info_ptr_t cuda_dev; /* devices detected in the system (per node) */
++ int ncuda_dev_compatible; /* number of compatible GPUs */
+} gmx_gpu_info_t;
+
+/* Hardware information structure with CPU and GPU information.
+ * It is initialized by gmx_detect_hardware().
+ * NOTE: this structure may only contain structures that are globally valid
+ * (i.e. must be able to be shared among all threads) */
+typedef struct
+{
- gmx_bool bConsistencyChecked; /* whether
- gmx_check_hw_runconf_consistency()
- has been run with this hw_info */
+ gmx_gpu_info_t gpu_info; /* Information about GPUs detected in the system */
+
+ gmx_cpuid_t cpuid_info; /* CPUID information about CPU detected;
+ NOTE: this will only detect the CPU thread 0 of the
+ current process runs on. */
+ int nthreads_hw_avail; /* Number of hardware threads available; this number
+ is based on the number of CPUs reported as available
+ by the OS at the time of detection. */
+} gmx_hw_info_t;
+
++
++/* The options for the thread affinity setting, default: auto */
++enum {
++ threadaffSEL, threadaffAUTO, threadaffON, threadaffOFF, threadaffNR
++};
++
++/* GPU device selection information -- for now with only CUDA devices */
++typedef struct
++{
++ char *gpu_id; /* GPU id's to use, each specified as chars */
++ gmx_bool bUserSet; /* true if the GPUs in cuda_dev_use are manually provided by the user */
++
++ int ncuda_dev_use; /* number of device (IDs) selected to be used */
++ int *cuda_dev_use; /* device index list providing GPU to PP rank mapping, GPUs can be listed multiple times when ranks share them */
++} gmx_gpu_opt_t;
++
++/* Threading and GPU options, can be set automatically or by the user */
++typedef struct {
++ int nthreads_tot; /* Total number of threads requested (TMPI) */
++ int nthreads_tmpi; /* Number of TMPI threads requested */
++ int nthreads_omp; /* Number of OpenMP threads requested */
++ int nthreads_omp_pme; /* As nthreads_omp, but for PME only nodes */
++ int thread_affinity; /* Thread affinity switch, see enum above */
++ int core_pinning_stride; /* Logical core pinning stride */
++ int core_pinning_offset; /* Logical core pinning offset */
++
++ gmx_gpu_opt_t gpu_opt; /* The GPU options */
++} gmx_hw_opt_t;
++
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* HWINFO_H */
--- /dev/null
- * to get rid of possible high counts due to other soures,
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This file is part of Gromacs Copyright (c) 1991-2008
+ * David van der Spoel, Erik Lindahl, Berk Hess, University of Groningen.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the research papers on the package. Check out http://www.gromacs.org
+ *
+ * And Hey:
+ * Gnomes, ROck Monsters And Chili Sauce
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdio.h>
+#include <time.h>
+#include <math.h>
+#include <string.h>
+#include <stdlib.h>
+#include "typedefs.h"
+#include "smalloc.h"
+#include "gmx_fatal.h"
+#include "gmx_fatal_collective.h"
+#include "vec.h"
+#include "domdec.h"
+#include "domdec_network.h"
+#include "nrnb.h"
+#include "pbc.h"
+#include "chargegroup.h"
+#include "constr.h"
+#include "mdatoms.h"
+#include "names.h"
+#include "force.h"
+#include "pme.h"
+#include "pull.h"
+#include "pull_rotation.h"
+#include "mdrun.h"
+#include "nsgrid.h"
+#include "shellfc.h"
+#include "mtop_util.h"
+#include "gmx_ga2la.h"
+#include "gmx_sort.h"
+#include "macros.h"
+#include "nbnxn_search.h"
+#include "bondf.h"
+#include "gmx_omp_nthreads.h"
++#include "gpu_utils.h"
+
+#include "gromacs/fileio/futil.h"
+#include "gromacs/fileio/gmxfio.h"
+#include "gromacs/fileio/pdbio.h"
+#include "gromacs/timing/wallcycle.h"
+#include "gromacs/utility/gmxmpi.h"
+
+#define DDRANK(dd, rank) (rank)
+#define DDMASTERRANK(dd) (dd->masterrank)
+
+typedef struct gmx_domdec_master
+{
+ /* The cell boundaries */
+ real **cell_x;
+ /* The global charge group division */
+ int *ncg; /* Number of home charge groups for each node */
+ int *index; /* Index of nnodes+1 into cg */
+ int *cg; /* Global charge group index */
+ int *nat; /* Number of home atoms for each node. */
+ int *ibuf; /* Buffer for communication */
+ rvec *vbuf; /* Buffer for state scattering and gathering */
+} gmx_domdec_master_t;
+
+typedef struct
+{
+ /* The numbers of charge groups to send and receive for each cell
+ * that requires communication, the last entry contains the total
+ * number of atoms that needs to be communicated.
+ */
+ int nsend[DD_MAXIZONE+2];
+ int nrecv[DD_MAXIZONE+2];
+ /* The charge groups to send */
+ int *index;
+ int nalloc;
+ /* The atom range for non-in-place communication */
+ int cell2at0[DD_MAXIZONE];
+ int cell2at1[DD_MAXIZONE];
+} gmx_domdec_ind_t;
+
+typedef struct
+{
+ int np; /* Number of grid pulses in this dimension */
+ int np_dlb; /* For dlb, for use with edlbAUTO */
+ gmx_domdec_ind_t *ind; /* The indices to communicate, size np */
+ int np_nalloc;
+ gmx_bool bInPlace; /* Can we communicate in place? */
+} gmx_domdec_comm_dim_t;
+
+typedef struct
+{
+ gmx_bool *bCellMin; /* Temp. var.: is this cell size at the limit */
+ real *cell_f; /* State var.: cell boundaries, box relative */
+ real *old_cell_f; /* Temp. var.: old cell size */
+ real *cell_f_max0; /* State var.: max lower boundary, incl neighbors */
+ real *cell_f_min1; /* State var.: min upper boundary, incl neighbors */
+ real *bound_min; /* Temp. var.: lower limit for cell boundary */
+ real *bound_max; /* Temp. var.: upper limit for cell boundary */
+ gmx_bool bLimited; /* State var.: is DLB limited in this dim and row */
+ real *buf_ncd; /* Temp. var. */
+} gmx_domdec_root_t;
+
+#define DD_NLOAD_MAX 9
+
+/* Here floats are accurate enough, since these variables
+ * only influence the load balancing, not the actual MD results.
+ */
+typedef struct
+{
+ int nload;
+ float *load;
+ float sum;
+ float max;
+ float sum_m;
+ float cvol_min;
+ float mdf;
+ float pme;
+ int flags;
+} gmx_domdec_load_t;
+
+typedef struct
+{
+ int nsc;
+ int ind_gl;
+ int ind;
+} gmx_cgsort_t;
+
+typedef struct
+{
+ gmx_cgsort_t *sort;
+ gmx_cgsort_t *sort2;
+ int sort_nalloc;
+ gmx_cgsort_t *sort_new;
+ int sort_new_nalloc;
+ int *ibuf;
+ int ibuf_nalloc;
+} gmx_domdec_sort_t;
+
+typedef struct
+{
+ rvec *v;
+ int nalloc;
+} vec_rvec_t;
+
+/* This enum determines the order of the coordinates.
+ * ddnatHOME and ddnatZONE should be first and second,
+ * the others can be ordered as wanted.
+ */
+enum {
+ ddnatHOME, ddnatZONE, ddnatVSITE, ddnatCON, ddnatNR
+};
+
+enum {
+ edlbAUTO, edlbNO, edlbYES, edlbNR
+};
+const char *edlb_names[edlbNR] = { "auto", "no", "yes" };
+
+typedef struct
+{
+ int dim; /* The dimension */
+ gmx_bool dim_match; /* Tells if DD and PME dims match */
+ int nslab; /* The number of PME slabs in this dimension */
+ real *slb_dim_f; /* Cell sizes for determining the PME comm. with SLB */
+ int *pp_min; /* The minimum pp node location, size nslab */
+ int *pp_max; /* The maximum pp node location,size nslab */
+ int maxshift; /* The maximum shift for coordinate redistribution in PME */
+} gmx_ddpme_t;
+
+typedef struct
+{
+ real min0; /* The minimum bottom of this zone */
+ real max1; /* The maximum top of this zone */
+ real min1; /* The minimum top of this zone */
+ real mch0; /* The maximum bottom communicaton height for this zone */
+ real mch1; /* The maximum top communicaton height for this zone */
+ real p1_0; /* The bottom value of the first cell in this zone */
+ real p1_1; /* The top value of the first cell in this zone */
+} gmx_ddzone_t;
+
+typedef struct
+{
+ gmx_domdec_ind_t ind;
+ int *ibuf;
+ int ibuf_nalloc;
+ vec_rvec_t vbuf;
+ int nsend;
+ int nat;
+ int nsend_zone;
+} dd_comm_setup_work_t;
+
+typedef struct gmx_domdec_comm
+{
+ /* All arrays are indexed with 0 to dd->ndim (not Cartesian indexing),
+ * unless stated otherwise.
+ */
+
+ /* The number of decomposition dimensions for PME, 0: no PME */
+ int npmedecompdim;
+ /* The number of nodes doing PME (PP/PME or only PME) */
+ int npmenodes;
+ int npmenodes_x;
+ int npmenodes_y;
+ /* The communication setup including the PME only nodes */
+ gmx_bool bCartesianPP_PME;
+ ivec ntot;
+ int cartpmedim;
+ int *pmenodes; /* size npmenodes */
+ int *ddindex2simnodeid; /* size npmenodes, only with bCartesianPP
+ * but with bCartesianPP_PME */
+ gmx_ddpme_t ddpme[2];
+
+ /* The DD particle-particle nodes only */
+ gmx_bool bCartesianPP;
+ int *ddindex2ddnodeid; /* size npmenode, only with bCartesianPP_PME */
+
+ /* The global charge groups */
+ t_block cgs_gl;
+
+ /* Should we sort the cgs */
+ int nstSortCG;
+ gmx_domdec_sort_t *sort;
+
+ /* Are there charge groups? */
+ gmx_bool bCGs;
+
+ /* Are there bonded and multi-body interactions between charge groups? */
+ gmx_bool bInterCGBondeds;
+ gmx_bool bInterCGMultiBody;
+
+ /* Data for the optional bonded interaction atom communication range */
+ gmx_bool bBondComm;
+ t_blocka *cglink;
+ char *bLocalCG;
+
+ /* The DLB option */
+ int eDLB;
+ /* Are we actually using DLB? */
+ gmx_bool bDynLoadBal;
+
+ /* Cell sizes for static load balancing, first index cartesian */
+ real **slb_frac;
+
+ /* The width of the communicated boundaries */
+ real cutoff_mbody;
+ real cutoff;
+ /* The minimum cell size (including triclinic correction) */
+ rvec cellsize_min;
+ /* For dlb, for use with edlbAUTO */
+ rvec cellsize_min_dlb;
+ /* The lower limit for the DD cell size with DLB */
+ real cellsize_limit;
+ /* Effectively no NB cut-off limit with DLB for systems without PBC? */
+ gmx_bool bVacDLBNoLimit;
+
+ /* With PME load balancing we set limits on DLB */
+ gmx_bool bPMELoadBalDLBLimits;
+ /* DLB needs to take into account that we want to allow this maximum
+ * cut-off (for PME load balancing), this could limit cell boundaries.
+ */
+ real PMELoadBal_max_cutoff;
+
+ /* tric_dir is only stored here because dd_get_ns_ranges needs it */
+ ivec tric_dir;
+ /* box0 and box_size are required with dim's without pbc and -gcom */
+ rvec box0;
+ rvec box_size;
+
+ /* The cell boundaries */
+ rvec cell_x0;
+ rvec cell_x1;
+
+ /* The old location of the cell boundaries, to check cg displacements */
+ rvec old_cell_x0;
+ rvec old_cell_x1;
+
+ /* The communication setup and charge group boundaries for the zones */
+ gmx_domdec_zones_t zones;
+
+ /* The zone limits for DD dimensions 1 and 2 (not 0), determined from
+ * cell boundaries of neighboring cells for dynamic load balancing.
+ */
+ gmx_ddzone_t zone_d1[2];
+ gmx_ddzone_t zone_d2[2][2];
+
+ /* The coordinate/force communication setup and indices */
+ gmx_domdec_comm_dim_t cd[DIM];
+ /* The maximum number of cells to communicate with in one dimension */
+ int maxpulse;
+
+ /* Which cg distribution is stored on the master node */
+ int master_cg_ddp_count;
+
+ /* The number of cg's received from the direct neighbors */
+ int zone_ncg1[DD_MAXZONE];
+
+ /* The atom counts, the range for each type t is nat[t-1] <= at < nat[t] */
+ int nat[ddnatNR];
+
+ /* Array for signalling if atoms have moved to another domain */
+ int *moved;
+ int moved_nalloc;
+
+ /* Communication buffer for general use */
+ int *buf_int;
+ int nalloc_int;
+
+ /* Communication buffer for general use */
+ vec_rvec_t vbuf;
+
+ /* Temporary storage for thread parallel communication setup */
+ int nth;
+ dd_comm_setup_work_t *dth;
+
+ /* Communication buffers only used with multiple grid pulses */
+ int *buf_int2;
+ int nalloc_int2;
+ vec_rvec_t vbuf2;
+
+ /* Communication buffers for local redistribution */
+ int **cggl_flag;
+ int cggl_flag_nalloc[DIM*2];
+ rvec **cgcm_state;
+ int cgcm_state_nalloc[DIM*2];
+
+ /* Cell sizes for dynamic load balancing */
+ gmx_domdec_root_t **root;
+ real *cell_f_row;
+ real cell_f0[DIM];
+ real cell_f1[DIM];
+ real cell_f_max0[DIM];
+ real cell_f_min1[DIM];
+
+ /* Stuff for load communication */
+ gmx_bool bRecordLoad;
+ gmx_domdec_load_t *load;
++ int nrank_gpu_shared;
+#ifdef GMX_MPI
+ MPI_Comm *mpi_comm_load;
++ MPI_Comm mpi_comm_gpu_shared;
+#endif
+
+ /* Maximum DLB scaling per load balancing step in percent */
+ int dlb_scale_lim;
+
+ /* Cycle counters */
+ float cycl[ddCyclNr];
+ int cycl_n[ddCyclNr];
+ float cycl_max[ddCyclNr];
+ /* Flop counter (0=no,1=yes,2=with (eFlop-1)*5% noise */
+ int eFlop;
+ double flop;
+ int flop_n;
+ /* Have often have did we have load measurements */
+ int n_load_have;
+ /* Have often have we collected the load measurements */
+ int n_load_collect;
+
+ /* Statistics */
+ double sum_nat[ddnatNR-ddnatZONE];
+ int ndecomp;
+ int nload;
+ double load_step;
+ double load_sum;
+ double load_max;
+ ivec load_lim;
+ double load_mdf;
+ double load_pme;
+
+ /* The last partition step */
+ gmx_large_int_t partition_step;
+
+ /* Debugging */
+ int nstDDDump;
+ int nstDDDumpGrid;
+ int DD_debug;
+} gmx_domdec_comm_t;
+
+/* The size per charge group of the cggl_flag buffer in gmx_domdec_comm_t */
+#define DD_CGIBS 2
+
+/* The flags for the cggl_flag buffer in gmx_domdec_comm_t */
+#define DD_FLAG_NRCG 65535
+#define DD_FLAG_FW(d) (1<<(16+(d)*2))
+#define DD_FLAG_BW(d) (1<<(16+(d)*2+1))
+
+/* Zone permutation required to obtain consecutive charge groups
+ * for neighbor searching.
+ */
+static const int zone_perm[3][4] = { {0, 0, 0, 0}, {1, 0, 0, 0}, {3, 0, 1, 2} };
+
+/* dd_zo and dd_zp3/dd_zp2 are set up such that i zones with non-zero
+ * components see only j zones with that component 0.
+ */
+
+/* The DD zone order */
+static const ivec dd_zo[DD_MAXZONE] =
+{{0, 0, 0}, {1, 0, 0}, {1, 1, 0}, {0, 1, 0}, {0, 1, 1}, {0, 0, 1}, {1, 0, 1}, {1, 1, 1}};
+
+/* The 3D setup */
+#define dd_z3n 8
+#define dd_zp3n 4
+static const ivec dd_zp3[dd_zp3n] = {{0, 0, 8}, {1, 3, 6}, {2, 5, 6}, {3, 5, 7}};
+
+/* The 2D setup */
+#define dd_z2n 4
+#define dd_zp2n 2
+static const ivec dd_zp2[dd_zp2n] = {{0, 0, 4}, {1, 3, 4}};
+
+/* The 1D setup */
+#define dd_z1n 2
+#define dd_zp1n 1
+static const ivec dd_zp1[dd_zp1n] = {{0, 0, 2}};
+
+/* Factors used to avoid problems due to rounding issues */
+#define DD_CELL_MARGIN 1.0001
+#define DD_CELL_MARGIN2 1.00005
+/* Factor to account for pressure scaling during nstlist steps */
+#define DD_PRES_SCALE_MARGIN 1.02
+
+/* Allowed performance loss before we DLB or warn */
+#define DD_PERF_LOSS 0.05
+
+#define DD_CELL_F_SIZE(dd, di) ((dd)->nc[(dd)->dim[(di)]]+1+(di)*2+1+(di))
+
+/* Use separate MPI send and receive commands
+ * when nnodes <= GMX_DD_NNODES_SENDRECV.
+ * This saves memory (and some copying for small nnodes).
+ * For high parallelization scatter and gather calls are used.
+ */
+#define GMX_DD_NNODES_SENDRECV 4
+
+
+/*
+ #define dd_index(n,i) ((((i)[ZZ]*(n)[YY] + (i)[YY])*(n)[XX]) + (i)[XX])
+
+ static void index2xyz(ivec nc,int ind,ivec xyz)
+ {
+ xyz[XX] = ind % nc[XX];
+ xyz[YY] = (ind / nc[XX]) % nc[YY];
+ xyz[ZZ] = ind / (nc[YY]*nc[XX]);
+ }
+ */
+
+/* This order is required to minimize the coordinate communication in PME
+ * which uses decomposition in the x direction.
+ */
+#define dd_index(n, i) ((((i)[XX]*(n)[YY] + (i)[YY])*(n)[ZZ]) + (i)[ZZ])
+
+static void ddindex2xyz(ivec nc, int ind, ivec xyz)
+{
+ xyz[XX] = ind / (nc[YY]*nc[ZZ]);
+ xyz[YY] = (ind / nc[ZZ]) % nc[YY];
+ xyz[ZZ] = ind % nc[ZZ];
+}
+
+static int ddcoord2ddnodeid(gmx_domdec_t *dd, ivec c)
+{
+ int ddindex;
+ int ddnodeid = -1;
+
+ ddindex = dd_index(dd->nc, c);
+ if (dd->comm->bCartesianPP_PME)
+ {
+ ddnodeid = dd->comm->ddindex2ddnodeid[ddindex];
+ }
+ else if (dd->comm->bCartesianPP)
+ {
+#ifdef GMX_MPI
+ MPI_Cart_rank(dd->mpi_comm_all, c, &ddnodeid);
+#endif
+ }
+ else
+ {
+ ddnodeid = ddindex;
+ }
+
+ return ddnodeid;
+}
+
+static gmx_bool dynamic_dd_box(gmx_ddbox_t *ddbox, t_inputrec *ir)
+{
+ return (ddbox->nboundeddim < DIM || DYNAMIC_BOX(*ir));
+}
+
+int ddglatnr(gmx_domdec_t *dd, int i)
+{
+ int atnr;
+
+ if (dd == NULL)
+ {
+ atnr = i + 1;
+ }
+ else
+ {
+ if (i >= dd->comm->nat[ddnatNR-1])
+ {
+ gmx_fatal(FARGS, "glatnr called with %d, which is larger than the local number of atoms (%d)", i, dd->comm->nat[ddnatNR-1]);
+ }
+ atnr = dd->gatindex[i] + 1;
+ }
+
+ return atnr;
+}
+
+t_block *dd_charge_groups_global(gmx_domdec_t *dd)
+{
+ return &dd->comm->cgs_gl;
+}
+
+static void vec_rvec_init(vec_rvec_t *v)
+{
+ v->nalloc = 0;
+ v->v = NULL;
+}
+
+static void vec_rvec_check_alloc(vec_rvec_t *v, int n)
+{
+ if (n > v->nalloc)
+ {
+ v->nalloc = over_alloc_dd(n);
+ srenew(v->v, v->nalloc);
+ }
+}
+
+void dd_store_state(gmx_domdec_t *dd, t_state *state)
+{
+ int i;
+
+ if (state->ddp_count != dd->ddp_count)
+ {
+ gmx_incons("The state does not the domain decomposition state");
+ }
+
+ state->ncg_gl = dd->ncg_home;
+ if (state->ncg_gl > state->cg_gl_nalloc)
+ {
+ state->cg_gl_nalloc = over_alloc_dd(state->ncg_gl);
+ srenew(state->cg_gl, state->cg_gl_nalloc);
+ }
+ for (i = 0; i < state->ncg_gl; i++)
+ {
+ state->cg_gl[i] = dd->index_gl[i];
+ }
+
+ state->ddp_count_cg_gl = dd->ddp_count;
+}
+
+gmx_domdec_zones_t *domdec_zones(gmx_domdec_t *dd)
+{
+ return &dd->comm->zones;
+}
+
+void dd_get_ns_ranges(gmx_domdec_t *dd, int icg,
+ int *jcg0, int *jcg1, ivec shift0, ivec shift1)
+{
+ gmx_domdec_zones_t *zones;
+ int izone, d, dim;
+
+ zones = &dd->comm->zones;
+
+ izone = 0;
+ while (icg >= zones->izone[izone].cg1)
+ {
+ izone++;
+ }
+
+ if (izone == 0)
+ {
+ *jcg0 = icg;
+ }
+ else if (izone < zones->nizone)
+ {
+ *jcg0 = zones->izone[izone].jcg0;
+ }
+ else
+ {
+ gmx_fatal(FARGS, "DD icg %d out of range: izone (%d) >= nizone (%d)",
+ icg, izone, zones->nizone);
+ }
+
+ *jcg1 = zones->izone[izone].jcg1;
+
+ for (d = 0; d < dd->ndim; d++)
+ {
+ dim = dd->dim[d];
+ shift0[dim] = zones->izone[izone].shift0[dim];
+ shift1[dim] = zones->izone[izone].shift1[dim];
+ if (dd->comm->tric_dir[dim] || (dd->bGridJump && d > 0))
+ {
+ /* A conservative approach, this can be optimized */
+ shift0[dim] -= 1;
+ shift1[dim] += 1;
+ }
+ }
+}
+
+int dd_natoms_vsite(gmx_domdec_t *dd)
+{
+ return dd->comm->nat[ddnatVSITE];
+}
+
+void dd_get_constraint_range(gmx_domdec_t *dd, int *at_start, int *at_end)
+{
+ *at_start = dd->comm->nat[ddnatCON-1];
+ *at_end = dd->comm->nat[ddnatCON];
+}
+
+void dd_move_x(gmx_domdec_t *dd, matrix box, rvec x[])
+{
+ int nzone, nat_tot, n, d, p, i, j, at0, at1, zone;
+ int *index, *cgindex;
+ gmx_domdec_comm_t *comm;
+ gmx_domdec_comm_dim_t *cd;
+ gmx_domdec_ind_t *ind;
+ rvec shift = {0, 0, 0}, *buf, *rbuf;
+ gmx_bool bPBC, bScrew;
+
+ comm = dd->comm;
+
+ cgindex = dd->cgindex;
+
+ buf = comm->vbuf.v;
+
+ nzone = 1;
+ nat_tot = dd->nat_home;
+ for (d = 0; d < dd->ndim; d++)
+ {
+ bPBC = (dd->ci[dd->dim[d]] == 0);
+ bScrew = (bPBC && dd->bScrewPBC && dd->dim[d] == XX);
+ if (bPBC)
+ {
+ copy_rvec(box[dd->dim[d]], shift);
+ }
+ cd = &comm->cd[d];
+ for (p = 0; p < cd->np; p++)
+ {
+ ind = &cd->ind[p];
+ index = ind->index;
+ n = 0;
+ if (!bPBC)
+ {
+ for (i = 0; i < ind->nsend[nzone]; i++)
+ {
+ at0 = cgindex[index[i]];
+ at1 = cgindex[index[i]+1];
+ for (j = at0; j < at1; j++)
+ {
+ copy_rvec(x[j], buf[n]);
+ n++;
+ }
+ }
+ }
+ else if (!bScrew)
+ {
+ for (i = 0; i < ind->nsend[nzone]; i++)
+ {
+ at0 = cgindex[index[i]];
+ at1 = cgindex[index[i]+1];
+ for (j = at0; j < at1; j++)
+ {
+ /* We need to shift the coordinates */
+ rvec_add(x[j], shift, buf[n]);
+ n++;
+ }
+ }
+ }
+ else
+ {
+ for (i = 0; i < ind->nsend[nzone]; i++)
+ {
+ at0 = cgindex[index[i]];
+ at1 = cgindex[index[i]+1];
+ for (j = at0; j < at1; j++)
+ {
+ /* Shift x */
+ buf[n][XX] = x[j][XX] + shift[XX];
+ /* Rotate y and z.
+ * This operation requires a special shift force
+ * treatment, which is performed in calc_vir.
+ */
+ buf[n][YY] = box[YY][YY] - x[j][YY];
+ buf[n][ZZ] = box[ZZ][ZZ] - x[j][ZZ];
+ n++;
+ }
+ }
+ }
+
+ if (cd->bInPlace)
+ {
+ rbuf = x + nat_tot;
+ }
+ else
+ {
+ rbuf = comm->vbuf2.v;
+ }
+ /* Send and receive the coordinates */
+ dd_sendrecv_rvec(dd, d, dddirBackward,
+ buf, ind->nsend[nzone+1],
+ rbuf, ind->nrecv[nzone+1]);
+ if (!cd->bInPlace)
+ {
+ j = 0;
+ for (zone = 0; zone < nzone; zone++)
+ {
+ for (i = ind->cell2at0[zone]; i < ind->cell2at1[zone]; i++)
+ {
+ copy_rvec(rbuf[j], x[i]);
+ j++;
+ }
+ }
+ }
+ nat_tot += ind->nrecv[nzone+1];
+ }
+ nzone += nzone;
+ }
+}
+
+void dd_move_f(gmx_domdec_t *dd, rvec f[], rvec *fshift)
+{
+ int nzone, nat_tot, n, d, p, i, j, at0, at1, zone;
+ int *index, *cgindex;
+ gmx_domdec_comm_t *comm;
+ gmx_domdec_comm_dim_t *cd;
+ gmx_domdec_ind_t *ind;
+ rvec *buf, *sbuf;
+ ivec vis;
+ int is;
+ gmx_bool bPBC, bScrew;
+
+ comm = dd->comm;
+
+ cgindex = dd->cgindex;
+
+ buf = comm->vbuf.v;
+
+ n = 0;
+ nzone = comm->zones.n/2;
+ nat_tot = dd->nat_tot;
+ for (d = dd->ndim-1; d >= 0; d--)
+ {
+ bPBC = (dd->ci[dd->dim[d]] == 0);
+ bScrew = (bPBC && dd->bScrewPBC && dd->dim[d] == XX);
+ if (fshift == NULL && !bScrew)
+ {
+ bPBC = FALSE;
+ }
+ /* Determine which shift vector we need */
+ clear_ivec(vis);
+ vis[dd->dim[d]] = 1;
+ is = IVEC2IS(vis);
+
+ cd = &comm->cd[d];
+ for (p = cd->np-1; p >= 0; p--)
+ {
+ ind = &cd->ind[p];
+ nat_tot -= ind->nrecv[nzone+1];
+ if (cd->bInPlace)
+ {
+ sbuf = f + nat_tot;
+ }
+ else
+ {
+ sbuf = comm->vbuf2.v;
+ j = 0;
+ for (zone = 0; zone < nzone; zone++)
+ {
+ for (i = ind->cell2at0[zone]; i < ind->cell2at1[zone]; i++)
+ {
+ copy_rvec(f[i], sbuf[j]);
+ j++;
+ }
+ }
+ }
+ /* Communicate the forces */
+ dd_sendrecv_rvec(dd, d, dddirForward,
+ sbuf, ind->nrecv[nzone+1],
+ buf, ind->nsend[nzone+1]);
+ index = ind->index;
+ /* Add the received forces */
+ n = 0;
+ if (!bPBC)
+ {
+ for (i = 0; i < ind->nsend[nzone]; i++)
+ {
+ at0 = cgindex[index[i]];
+ at1 = cgindex[index[i]+1];
+ for (j = at0; j < at1; j++)
+ {
+ rvec_inc(f[j], buf[n]);
+ n++;
+ }
+ }
+ }
+ else if (!bScrew)
+ {
+ for (i = 0; i < ind->nsend[nzone]; i++)
+ {
+ at0 = cgindex[index[i]];
+ at1 = cgindex[index[i]+1];
+ for (j = at0; j < at1; j++)
+ {
+ rvec_inc(f[j], buf[n]);
+ /* Add this force to the shift force */
+ rvec_inc(fshift[is], buf[n]);
+ n++;
+ }
+ }
+ }
+ else
+ {
+ for (i = 0; i < ind->nsend[nzone]; i++)
+ {
+ at0 = cgindex[index[i]];
+ at1 = cgindex[index[i]+1];
+ for (j = at0; j < at1; j++)
+ {
+ /* Rotate the force */
+ f[j][XX] += buf[n][XX];
+ f[j][YY] -= buf[n][YY];
+ f[j][ZZ] -= buf[n][ZZ];
+ if (fshift)
+ {
+ /* Add this force to the shift force */
+ rvec_inc(fshift[is], buf[n]);
+ }
+ n++;
+ }
+ }
+ }
+ }
+ nzone /= 2;
+ }
+}
+
+void dd_atom_spread_real(gmx_domdec_t *dd, real v[])
+{
+ int nzone, nat_tot, n, d, p, i, j, at0, at1, zone;
+ int *index, *cgindex;
+ gmx_domdec_comm_t *comm;
+ gmx_domdec_comm_dim_t *cd;
+ gmx_domdec_ind_t *ind;
+ real *buf, *rbuf;
+
+ comm = dd->comm;
+
+ cgindex = dd->cgindex;
+
+ buf = &comm->vbuf.v[0][0];
+
+ nzone = 1;
+ nat_tot = dd->nat_home;
+ for (d = 0; d < dd->ndim; d++)
+ {
+ cd = &comm->cd[d];
+ for (p = 0; p < cd->np; p++)
+ {
+ ind = &cd->ind[p];
+ index = ind->index;
+ n = 0;
+ for (i = 0; i < ind->nsend[nzone]; i++)
+ {
+ at0 = cgindex[index[i]];
+ at1 = cgindex[index[i]+1];
+ for (j = at0; j < at1; j++)
+ {
+ buf[n] = v[j];
+ n++;
+ }
+ }
+
+ if (cd->bInPlace)
+ {
+ rbuf = v + nat_tot;
+ }
+ else
+ {
+ rbuf = &comm->vbuf2.v[0][0];
+ }
+ /* Send and receive the coordinates */
+ dd_sendrecv_real(dd, d, dddirBackward,
+ buf, ind->nsend[nzone+1],
+ rbuf, ind->nrecv[nzone+1]);
+ if (!cd->bInPlace)
+ {
+ j = 0;
+ for (zone = 0; zone < nzone; zone++)
+ {
+ for (i = ind->cell2at0[zone]; i < ind->cell2at1[zone]; i++)
+ {
+ v[i] = rbuf[j];
+ j++;
+ }
+ }
+ }
+ nat_tot += ind->nrecv[nzone+1];
+ }
+ nzone += nzone;
+ }
+}
+
+void dd_atom_sum_real(gmx_domdec_t *dd, real v[])
+{
+ int nzone, nat_tot, n, d, p, i, j, at0, at1, zone;
+ int *index, *cgindex;
+ gmx_domdec_comm_t *comm;
+ gmx_domdec_comm_dim_t *cd;
+ gmx_domdec_ind_t *ind;
+ real *buf, *sbuf;
+
+ comm = dd->comm;
+
+ cgindex = dd->cgindex;
+
+ buf = &comm->vbuf.v[0][0];
+
+ n = 0;
+ nzone = comm->zones.n/2;
+ nat_tot = dd->nat_tot;
+ for (d = dd->ndim-1; d >= 0; d--)
+ {
+ cd = &comm->cd[d];
+ for (p = cd->np-1; p >= 0; p--)
+ {
+ ind = &cd->ind[p];
+ nat_tot -= ind->nrecv[nzone+1];
+ if (cd->bInPlace)
+ {
+ sbuf = v + nat_tot;
+ }
+ else
+ {
+ sbuf = &comm->vbuf2.v[0][0];
+ j = 0;
+ for (zone = 0; zone < nzone; zone++)
+ {
+ for (i = ind->cell2at0[zone]; i < ind->cell2at1[zone]; i++)
+ {
+ sbuf[j] = v[i];
+ j++;
+ }
+ }
+ }
+ /* Communicate the forces */
+ dd_sendrecv_real(dd, d, dddirForward,
+ sbuf, ind->nrecv[nzone+1],
+ buf, ind->nsend[nzone+1]);
+ index = ind->index;
+ /* Add the received forces */
+ n = 0;
+ for (i = 0; i < ind->nsend[nzone]; i++)
+ {
+ at0 = cgindex[index[i]];
+ at1 = cgindex[index[i]+1];
+ for (j = at0; j < at1; j++)
+ {
+ v[j] += buf[n];
+ n++;
+ }
+ }
+ }
+ nzone /= 2;
+ }
+}
+
+static void print_ddzone(FILE *fp, int d, int i, int j, gmx_ddzone_t *zone)
+{
+ fprintf(fp, "zone d0 %d d1 %d d2 %d min0 %6.3f max1 %6.3f mch0 %6.3f mch1 %6.3f p1_0 %6.3f p1_1 %6.3f\n",
+ d, i, j,
+ zone->min0, zone->max1,
+ zone->mch0, zone->mch0,
+ zone->p1_0, zone->p1_1);
+}
+
+
+#define DDZONECOMM_MAXZONE 5
+#define DDZONECOMM_BUFSIZE 3
+
+static void dd_sendrecv_ddzone(const gmx_domdec_t *dd,
+ int ddimind, int direction,
+ gmx_ddzone_t *buf_s, int n_s,
+ gmx_ddzone_t *buf_r, int n_r)
+{
+#define ZBS DDZONECOMM_BUFSIZE
+ rvec vbuf_s[DDZONECOMM_MAXZONE*ZBS];
+ rvec vbuf_r[DDZONECOMM_MAXZONE*ZBS];
+ int i;
+
+ for (i = 0; i < n_s; i++)
+ {
+ vbuf_s[i*ZBS ][0] = buf_s[i].min0;
+ vbuf_s[i*ZBS ][1] = buf_s[i].max1;
+ vbuf_s[i*ZBS ][2] = buf_s[i].min1;
+ vbuf_s[i*ZBS+1][0] = buf_s[i].mch0;
+ vbuf_s[i*ZBS+1][1] = buf_s[i].mch1;
+ vbuf_s[i*ZBS+1][2] = 0;
+ vbuf_s[i*ZBS+2][0] = buf_s[i].p1_0;
+ vbuf_s[i*ZBS+2][1] = buf_s[i].p1_1;
+ vbuf_s[i*ZBS+2][2] = 0;
+ }
+
+ dd_sendrecv_rvec(dd, ddimind, direction,
+ vbuf_s, n_s*ZBS,
+ vbuf_r, n_r*ZBS);
+
+ for (i = 0; i < n_r; i++)
+ {
+ buf_r[i].min0 = vbuf_r[i*ZBS ][0];
+ buf_r[i].max1 = vbuf_r[i*ZBS ][1];
+ buf_r[i].min1 = vbuf_r[i*ZBS ][2];
+ buf_r[i].mch0 = vbuf_r[i*ZBS+1][0];
+ buf_r[i].mch1 = vbuf_r[i*ZBS+1][1];
+ buf_r[i].p1_0 = vbuf_r[i*ZBS+2][0];
+ buf_r[i].p1_1 = vbuf_r[i*ZBS+2][1];
+ }
+
+#undef ZBS
+}
+
+static void dd_move_cellx(gmx_domdec_t *dd, gmx_ddbox_t *ddbox,
+ rvec cell_ns_x0, rvec cell_ns_x1)
+{
+ int d, d1, dim, dim1, pos, buf_size, i, j, k, p, npulse, npulse_min;
+ gmx_ddzone_t *zp;
+ gmx_ddzone_t buf_s[DDZONECOMM_MAXZONE];
+ gmx_ddzone_t buf_r[DDZONECOMM_MAXZONE];
+ gmx_ddzone_t buf_e[DDZONECOMM_MAXZONE];
+ rvec extr_s[2], extr_r[2];
+ rvec dh;
+ real dist_d, c = 0, det;
+ gmx_domdec_comm_t *comm;
+ gmx_bool bPBC, bUse;
+
+ comm = dd->comm;
+
+ for (d = 1; d < dd->ndim; d++)
+ {
+ dim = dd->dim[d];
+ zp = (d == 1) ? &comm->zone_d1[0] : &comm->zone_d2[0][0];
+ zp->min0 = cell_ns_x0[dim];
+ zp->max1 = cell_ns_x1[dim];
+ zp->min1 = cell_ns_x1[dim];
+ zp->mch0 = cell_ns_x0[dim];
+ zp->mch1 = cell_ns_x1[dim];
+ zp->p1_0 = cell_ns_x0[dim];
+ zp->p1_1 = cell_ns_x1[dim];
+ }
+
+ for (d = dd->ndim-2; d >= 0; d--)
+ {
+ dim = dd->dim[d];
+ bPBC = (dim < ddbox->npbcdim);
+
+ /* Use an rvec to store two reals */
+ extr_s[d][0] = comm->cell_f0[d+1];
+ extr_s[d][1] = comm->cell_f1[d+1];
+ extr_s[d][2] = comm->cell_f1[d+1];
+
+ pos = 0;
+ /* Store the extremes in the backward sending buffer,
+ * so the get updated separately from the forward communication.
+ */
+ for (d1 = d; d1 < dd->ndim-1; d1++)
+ {
+ /* We invert the order to be able to use the same loop for buf_e */
+ buf_s[pos].min0 = extr_s[d1][1];
+ buf_s[pos].max1 = extr_s[d1][0];
+ buf_s[pos].min1 = extr_s[d1][2];
+ buf_s[pos].mch0 = 0;
+ buf_s[pos].mch1 = 0;
+ /* Store the cell corner of the dimension we communicate along */
+ buf_s[pos].p1_0 = comm->cell_x0[dim];
+ buf_s[pos].p1_1 = 0;
+ pos++;
+ }
+
+ buf_s[pos] = (dd->ndim == 2) ? comm->zone_d1[0] : comm->zone_d2[0][0];
+ pos++;
+
+ if (dd->ndim == 3 && d == 0)
+ {
+ buf_s[pos] = comm->zone_d2[0][1];
+ pos++;
+ buf_s[pos] = comm->zone_d1[0];
+ pos++;
+ }
+
+ /* We only need to communicate the extremes
+ * in the forward direction
+ */
+ npulse = comm->cd[d].np;
+ if (bPBC)
+ {
+ /* Take the minimum to avoid double communication */
+ npulse_min = min(npulse, dd->nc[dim]-1-npulse);
+ }
+ else
+ {
+ /* Without PBC we should really not communicate over
+ * the boundaries, but implementing that complicates
+ * the communication setup and therefore we simply
+ * do all communication, but ignore some data.
+ */
+ npulse_min = npulse;
+ }
+ for (p = 0; p < npulse_min; p++)
+ {
+ /* Communicate the extremes forward */
+ bUse = (bPBC || dd->ci[dim] > 0);
+
+ dd_sendrecv_rvec(dd, d, dddirForward,
+ extr_s+d, dd->ndim-d-1,
+ extr_r+d, dd->ndim-d-1);
+
+ if (bUse)
+ {
+ for (d1 = d; d1 < dd->ndim-1; d1++)
+ {
+ extr_s[d1][0] = max(extr_s[d1][0], extr_r[d1][0]);
+ extr_s[d1][1] = min(extr_s[d1][1], extr_r[d1][1]);
+ extr_s[d1][2] = min(extr_s[d1][2], extr_r[d1][2]);
+ }
+ }
+ }
+
+ buf_size = pos;
+ for (p = 0; p < npulse; p++)
+ {
+ /* Communicate all the zone information backward */
+ bUse = (bPBC || dd->ci[dim] < dd->nc[dim] - 1);
+
+ dd_sendrecv_ddzone(dd, d, dddirBackward,
+ buf_s, buf_size,
+ buf_r, buf_size);
+
+ clear_rvec(dh);
+ if (p > 0)
+ {
+ for (d1 = d+1; d1 < dd->ndim; d1++)
+ {
+ /* Determine the decrease of maximum required
+ * communication height along d1 due to the distance along d,
+ * this avoids a lot of useless atom communication.
+ */
+ dist_d = comm->cell_x1[dim] - buf_r[0].p1_0;
+
+ if (ddbox->tric_dir[dim])
+ {
+ /* c is the off-diagonal coupling between the cell planes
+ * along directions d and d1.
+ */
+ c = ddbox->v[dim][dd->dim[d1]][dim];
+ }
+ else
+ {
+ c = 0;
+ }
+ det = (1 + c*c)*comm->cutoff*comm->cutoff - dist_d*dist_d;
+ if (det > 0)
+ {
+ dh[d1] = comm->cutoff - (c*dist_d + sqrt(det))/(1 + c*c);
+ }
+ else
+ {
+ /* A negative value signals out of range */
+ dh[d1] = -1;
+ }
+ }
+ }
+
+ /* Accumulate the extremes over all pulses */
+ for (i = 0; i < buf_size; i++)
+ {
+ if (p == 0)
+ {
+ buf_e[i] = buf_r[i];
+ }
+ else
+ {
+ if (bUse)
+ {
+ buf_e[i].min0 = min(buf_e[i].min0, buf_r[i].min0);
+ buf_e[i].max1 = max(buf_e[i].max1, buf_r[i].max1);
+ buf_e[i].min1 = min(buf_e[i].min1, buf_r[i].min1);
+ }
+
+ if (dd->ndim == 3 && d == 0 && i == buf_size - 1)
+ {
+ d1 = 1;
+ }
+ else
+ {
+ d1 = d + 1;
+ }
+ if (bUse && dh[d1] >= 0)
+ {
+ buf_e[i].mch0 = max(buf_e[i].mch0, buf_r[i].mch0-dh[d1]);
+ buf_e[i].mch1 = max(buf_e[i].mch1, buf_r[i].mch1-dh[d1]);
+ }
+ }
+ /* Copy the received buffer to the send buffer,
+ * to pass the data through with the next pulse.
+ */
+ buf_s[i] = buf_r[i];
+ }
+ if (((bPBC || dd->ci[dim]+npulse < dd->nc[dim]) && p == npulse-1) ||
+ (!bPBC && dd->ci[dim]+1+p == dd->nc[dim]-1))
+ {
+ /* Store the extremes */
+ pos = 0;
+
+ for (d1 = d; d1 < dd->ndim-1; d1++)
+ {
+ extr_s[d1][1] = min(extr_s[d1][1], buf_e[pos].min0);
+ extr_s[d1][0] = max(extr_s[d1][0], buf_e[pos].max1);
+ extr_s[d1][2] = min(extr_s[d1][2], buf_e[pos].min1);
+ pos++;
+ }
+
+ if (d == 1 || (d == 0 && dd->ndim == 3))
+ {
+ for (i = d; i < 2; i++)
+ {
+ comm->zone_d2[1-d][i] = buf_e[pos];
+ pos++;
+ }
+ }
+ if (d == 0)
+ {
+ comm->zone_d1[1] = buf_e[pos];
+ pos++;
+ }
+ }
+ }
+ }
+
+ if (dd->ndim >= 2)
+ {
+ dim = dd->dim[1];
+ for (i = 0; i < 2; i++)
+ {
+ if (debug)
+ {
+ print_ddzone(debug, 1, i, 0, &comm->zone_d1[i]);
+ }
+ cell_ns_x0[dim] = min(cell_ns_x0[dim], comm->zone_d1[i].min0);
+ cell_ns_x1[dim] = max(cell_ns_x1[dim], comm->zone_d1[i].max1);
+ }
+ }
+ if (dd->ndim >= 3)
+ {
+ dim = dd->dim[2];
+ for (i = 0; i < 2; i++)
+ {
+ for (j = 0; j < 2; j++)
+ {
+ if (debug)
+ {
+ print_ddzone(debug, 2, i, j, &comm->zone_d2[i][j]);
+ }
+ cell_ns_x0[dim] = min(cell_ns_x0[dim], comm->zone_d2[i][j].min0);
+ cell_ns_x1[dim] = max(cell_ns_x1[dim], comm->zone_d2[i][j].max1);
+ }
+ }
+ }
+ for (d = 1; d < dd->ndim; d++)
+ {
+ comm->cell_f_max0[d] = extr_s[d-1][0];
+ comm->cell_f_min1[d] = extr_s[d-1][1];
+ if (debug)
+ {
+ fprintf(debug, "Cell fraction d %d, max0 %f, min1 %f\n",
+ d, comm->cell_f_max0[d], comm->cell_f_min1[d]);
+ }
+ }
+}
+
+static void dd_collect_cg(gmx_domdec_t *dd,
+ t_state *state_local)
+{
+ gmx_domdec_master_t *ma = NULL;
+ int buf2[2], *ibuf, i, ncg_home = 0, *cg = NULL, nat_home = 0;
+ t_block *cgs_gl;
+
+ if (state_local->ddp_count == dd->comm->master_cg_ddp_count)
+ {
+ /* The master has the correct distribution */
+ return;
+ }
+
+ if (state_local->ddp_count == dd->ddp_count)
+ {
+ ncg_home = dd->ncg_home;
+ cg = dd->index_gl;
+ nat_home = dd->nat_home;
+ }
+ else if (state_local->ddp_count_cg_gl == state_local->ddp_count)
+ {
+ cgs_gl = &dd->comm->cgs_gl;
+
+ ncg_home = state_local->ncg_gl;
+ cg = state_local->cg_gl;
+ nat_home = 0;
+ for (i = 0; i < ncg_home; i++)
+ {
+ nat_home += cgs_gl->index[cg[i]+1] - cgs_gl->index[cg[i]];
+ }
+ }
+ else
+ {
+ gmx_incons("Attempted to collect a vector for a state for which the charge group distribution is unknown");
+ }
+
+ buf2[0] = dd->ncg_home;
+ buf2[1] = dd->nat_home;
+ if (DDMASTER(dd))
+ {
+ ma = dd->ma;
+ ibuf = ma->ibuf;
+ }
+ else
+ {
+ ibuf = NULL;
+ }
+ /* Collect the charge group and atom counts on the master */
+ dd_gather(dd, 2*sizeof(int), buf2, ibuf);
+
+ if (DDMASTER(dd))
+ {
+ ma->index[0] = 0;
+ for (i = 0; i < dd->nnodes; i++)
+ {
+ ma->ncg[i] = ma->ibuf[2*i];
+ ma->nat[i] = ma->ibuf[2*i+1];
+ ma->index[i+1] = ma->index[i] + ma->ncg[i];
+
+ }
+ /* Make byte counts and indices */
+ for (i = 0; i < dd->nnodes; i++)
+ {
+ ma->ibuf[i] = ma->ncg[i]*sizeof(int);
+ ma->ibuf[dd->nnodes+i] = ma->index[i]*sizeof(int);
+ }
+ if (debug)
+ {
+ fprintf(debug, "Initial charge group distribution: ");
+ for (i = 0; i < dd->nnodes; i++)
+ {
+ fprintf(debug, " %d", ma->ncg[i]);
+ }
+ fprintf(debug, "\n");
+ }
+ }
+
+ /* Collect the charge group indices on the master */
+ dd_gatherv(dd,
+ dd->ncg_home*sizeof(int), dd->index_gl,
+ DDMASTER(dd) ? ma->ibuf : NULL,
+ DDMASTER(dd) ? ma->ibuf+dd->nnodes : NULL,
+ DDMASTER(dd) ? ma->cg : NULL);
+
+ dd->comm->master_cg_ddp_count = state_local->ddp_count;
+}
+
+static void dd_collect_vec_sendrecv(gmx_domdec_t *dd,
+ rvec *lv, rvec *v)
+{
+ gmx_domdec_master_t *ma;
+ int n, i, c, a, nalloc = 0;
+ rvec *buf = NULL;
+ t_block *cgs_gl;
+
+ ma = dd->ma;
+
+ if (!DDMASTER(dd))
+ {
+#ifdef GMX_MPI
+ MPI_Send(lv, dd->nat_home*sizeof(rvec), MPI_BYTE, DDMASTERRANK(dd),
+ dd->rank, dd->mpi_comm_all);
+#endif
+ }
+ else
+ {
+ /* Copy the master coordinates to the global array */
+ cgs_gl = &dd->comm->cgs_gl;
+
+ n = DDMASTERRANK(dd);
+ a = 0;
+ for (i = ma->index[n]; i < ma->index[n+1]; i++)
+ {
+ for (c = cgs_gl->index[ma->cg[i]]; c < cgs_gl->index[ma->cg[i]+1]; c++)
+ {
+ copy_rvec(lv[a++], v[c]);
+ }
+ }
+
+ for (n = 0; n < dd->nnodes; n++)
+ {
+ if (n != dd->rank)
+ {
+ if (ma->nat[n] > nalloc)
+ {
+ nalloc = over_alloc_dd(ma->nat[n]);
+ srenew(buf, nalloc);
+ }
+#ifdef GMX_MPI
+ MPI_Recv(buf, ma->nat[n]*sizeof(rvec), MPI_BYTE, DDRANK(dd, n),
+ n, dd->mpi_comm_all, MPI_STATUS_IGNORE);
+#endif
+ a = 0;
+ for (i = ma->index[n]; i < ma->index[n+1]; i++)
+ {
+ for (c = cgs_gl->index[ma->cg[i]]; c < cgs_gl->index[ma->cg[i]+1]; c++)
+ {
+ copy_rvec(buf[a++], v[c]);
+ }
+ }
+ }
+ }
+ sfree(buf);
+ }
+}
+
+static void get_commbuffer_counts(gmx_domdec_t *dd,
+ int **counts, int **disps)
+{
+ gmx_domdec_master_t *ma;
+ int n;
+
+ ma = dd->ma;
+
+ /* Make the rvec count and displacment arrays */
+ *counts = ma->ibuf;
+ *disps = ma->ibuf + dd->nnodes;
+ for (n = 0; n < dd->nnodes; n++)
+ {
+ (*counts)[n] = ma->nat[n]*sizeof(rvec);
+ (*disps)[n] = (n == 0 ? 0 : (*disps)[n-1] + (*counts)[n-1]);
+ }
+}
+
+static void dd_collect_vec_gatherv(gmx_domdec_t *dd,
+ rvec *lv, rvec *v)
+{
+ gmx_domdec_master_t *ma;
+ int *rcounts = NULL, *disps = NULL;
+ int n, i, c, a;
+ rvec *buf = NULL;
+ t_block *cgs_gl;
+
+ ma = dd->ma;
+
+ if (DDMASTER(dd))
+ {
+ get_commbuffer_counts(dd, &rcounts, &disps);
+
+ buf = ma->vbuf;
+ }
+
+ dd_gatherv(dd, dd->nat_home*sizeof(rvec), lv, rcounts, disps, buf);
+
+ if (DDMASTER(dd))
+ {
+ cgs_gl = &dd->comm->cgs_gl;
+
+ a = 0;
+ for (n = 0; n < dd->nnodes; n++)
+ {
+ for (i = ma->index[n]; i < ma->index[n+1]; i++)
+ {
+ for (c = cgs_gl->index[ma->cg[i]]; c < cgs_gl->index[ma->cg[i]+1]; c++)
+ {
+ copy_rvec(buf[a++], v[c]);
+ }
+ }
+ }
+ }
+}
+
+void dd_collect_vec(gmx_domdec_t *dd,
+ t_state *state_local, rvec *lv, rvec *v)
+{
+ gmx_domdec_master_t *ma;
+ int n, i, c, a, nalloc = 0;
+ rvec *buf = NULL;
+
+ dd_collect_cg(dd, state_local);
+
+ if (dd->nnodes <= GMX_DD_NNODES_SENDRECV)
+ {
+ dd_collect_vec_sendrecv(dd, lv, v);
+ }
+ else
+ {
+ dd_collect_vec_gatherv(dd, lv, v);
+ }
+}
+
+
+void dd_collect_state(gmx_domdec_t *dd,
+ t_state *state_local, t_state *state)
+{
+ int est, i, j, nh;
+
+ nh = state->nhchainlength;
+
+ if (DDMASTER(dd))
+ {
+ for (i = 0; i < efptNR; i++)
+ {
+ state->lambda[i] = state_local->lambda[i];
+ }
+ state->fep_state = state_local->fep_state;
+ state->veta = state_local->veta;
+ state->vol0 = state_local->vol0;
+ copy_mat(state_local->box, state->box);
+ copy_mat(state_local->boxv, state->boxv);
+ copy_mat(state_local->svir_prev, state->svir_prev);
+ copy_mat(state_local->fvir_prev, state->fvir_prev);
+ copy_mat(state_local->pres_prev, state->pres_prev);
+
+ for (i = 0; i < state_local->ngtc; i++)
+ {
+ for (j = 0; j < nh; j++)
+ {
+ state->nosehoover_xi[i*nh+j] = state_local->nosehoover_xi[i*nh+j];
+ state->nosehoover_vxi[i*nh+j] = state_local->nosehoover_vxi[i*nh+j];
+ }
+ state->therm_integral[i] = state_local->therm_integral[i];
+ }
+ for (i = 0; i < state_local->nnhpres; i++)
+ {
+ for (j = 0; j < nh; j++)
+ {
+ state->nhpres_xi[i*nh+j] = state_local->nhpres_xi[i*nh+j];
+ state->nhpres_vxi[i*nh+j] = state_local->nhpres_vxi[i*nh+j];
+ }
+ }
+ }
+ for (est = 0; est < estNR; est++)
+ {
+ if (EST_DISTR(est) && (state_local->flags & (1<<est)))
+ {
+ switch (est)
+ {
+ case estX:
+ dd_collect_vec(dd, state_local, state_local->x, state->x);
+ break;
+ case estV:
+ dd_collect_vec(dd, state_local, state_local->v, state->v);
+ break;
+ case estSDX:
+ dd_collect_vec(dd, state_local, state_local->sd_X, state->sd_X);
+ break;
+ case estCGP:
+ dd_collect_vec(dd, state_local, state_local->cg_p, state->cg_p);
+ break;
+ case estLD_RNG:
+ if (state->nrngi == 1)
+ {
+ if (DDMASTER(dd))
+ {
+ for (i = 0; i < state_local->nrng; i++)
+ {
+ state->ld_rng[i] = state_local->ld_rng[i];
+ }
+ }
+ }
+ else
+ {
+ dd_gather(dd, state_local->nrng*sizeof(state->ld_rng[0]),
+ state_local->ld_rng, state->ld_rng);
+ }
+ break;
+ case estLD_RNGI:
+ if (state->nrngi == 1)
+ {
+ if (DDMASTER(dd))
+ {
+ state->ld_rngi[0] = state_local->ld_rngi[0];
+ }
+ }
+ else
+ {
+ dd_gather(dd, sizeof(state->ld_rngi[0]),
+ state_local->ld_rngi, state->ld_rngi);
+ }
+ break;
+ case estDISRE_INITF:
+ case estDISRE_RM3TAV:
+ case estORIRE_INITF:
+ case estORIRE_DTAV:
+ break;
+ default:
+ gmx_incons("Unknown state entry encountered in dd_collect_state");
+ }
+ }
+ }
+}
+
+static void dd_realloc_state(t_state *state, rvec **f, int nalloc)
+{
+ int est;
+
+ if (debug)
+ {
+ fprintf(debug, "Reallocating state: currently %d, required %d, allocating %d\n", state->nalloc, nalloc, over_alloc_dd(nalloc));
+ }
+
+ state->nalloc = over_alloc_dd(nalloc);
+
+ for (est = 0; est < estNR; est++)
+ {
+ if (EST_DISTR(est) && (state->flags & (1<<est)))
+ {
+ switch (est)
+ {
+ case estX:
+ srenew(state->x, state->nalloc);
+ break;
+ case estV:
+ srenew(state->v, state->nalloc);
+ break;
+ case estSDX:
+ srenew(state->sd_X, state->nalloc);
+ break;
+ case estCGP:
+ srenew(state->cg_p, state->nalloc);
+ break;
+ case estLD_RNG:
+ case estLD_RNGI:
+ case estDISRE_INITF:
+ case estDISRE_RM3TAV:
+ case estORIRE_INITF:
+ case estORIRE_DTAV:
+ /* No reallocation required */
+ break;
+ default:
+ gmx_incons("Unknown state entry encountered in dd_realloc_state");
+ }
+ }
+ }
+
+ if (f != NULL)
+ {
+ srenew(*f, state->nalloc);
+ }
+}
+
+static void dd_check_alloc_ncg(t_forcerec *fr, t_state *state, rvec **f,
+ int nalloc)
+{
+ if (nalloc > fr->cg_nalloc)
+ {
+ if (debug)
+ {
+ fprintf(debug, "Reallocating forcerec: currently %d, required %d, allocating %d\n", fr->cg_nalloc, nalloc, over_alloc_dd(nalloc));
+ }
+ fr->cg_nalloc = over_alloc_dd(nalloc);
+ srenew(fr->cginfo, fr->cg_nalloc);
+ if (fr->cutoff_scheme == ecutsGROUP)
+ {
+ srenew(fr->cg_cm, fr->cg_nalloc);
+ }
+ }
+ if (fr->cutoff_scheme == ecutsVERLET && nalloc > state->nalloc)
+ {
+ /* We don't use charge groups, we use x in state to set up
+ * the atom communication.
+ */
+ dd_realloc_state(state, f, nalloc);
+ }
+}
+
+static void dd_distribute_vec_sendrecv(gmx_domdec_t *dd, t_block *cgs,
+ rvec *v, rvec *lv)
+{
+ gmx_domdec_master_t *ma;
+ int n, i, c, a, nalloc = 0;
+ rvec *buf = NULL;
+
+ if (DDMASTER(dd))
+ {
+ ma = dd->ma;
+
+ for (n = 0; n < dd->nnodes; n++)
+ {
+ if (n != dd->rank)
+ {
+ if (ma->nat[n] > nalloc)
+ {
+ nalloc = over_alloc_dd(ma->nat[n]);
+ srenew(buf, nalloc);
+ }
+ /* Use lv as a temporary buffer */
+ a = 0;
+ for (i = ma->index[n]; i < ma->index[n+1]; i++)
+ {
+ for (c = cgs->index[ma->cg[i]]; c < cgs->index[ma->cg[i]+1]; c++)
+ {
+ copy_rvec(v[c], buf[a++]);
+ }
+ }
+ if (a != ma->nat[n])
+ {
+ gmx_fatal(FARGS, "Internal error a (%d) != nat (%d)",
+ a, ma->nat[n]);
+ }
+
+#ifdef GMX_MPI
+ MPI_Send(buf, ma->nat[n]*sizeof(rvec), MPI_BYTE,
+ DDRANK(dd, n), n, dd->mpi_comm_all);
+#endif
+ }
+ }
+ sfree(buf);
+ n = DDMASTERRANK(dd);
+ a = 0;
+ for (i = ma->index[n]; i < ma->index[n+1]; i++)
+ {
+ for (c = cgs->index[ma->cg[i]]; c < cgs->index[ma->cg[i]+1]; c++)
+ {
+ copy_rvec(v[c], lv[a++]);
+ }
+ }
+ }
+ else
+ {
+#ifdef GMX_MPI
+ MPI_Recv(lv, dd->nat_home*sizeof(rvec), MPI_BYTE, DDMASTERRANK(dd),
+ MPI_ANY_TAG, dd->mpi_comm_all, MPI_STATUS_IGNORE);
+#endif
+ }
+}
+
+static void dd_distribute_vec_scatterv(gmx_domdec_t *dd, t_block *cgs,
+ rvec *v, rvec *lv)
+{
+ gmx_domdec_master_t *ma;
+ int *scounts = NULL, *disps = NULL;
+ int n, i, c, a, nalloc = 0;
+ rvec *buf = NULL;
+
+ if (DDMASTER(dd))
+ {
+ ma = dd->ma;
+
+ get_commbuffer_counts(dd, &scounts, &disps);
+
+ buf = ma->vbuf;
+ a = 0;
+ for (n = 0; n < dd->nnodes; n++)
+ {
+ for (i = ma->index[n]; i < ma->index[n+1]; i++)
+ {
+ for (c = cgs->index[ma->cg[i]]; c < cgs->index[ma->cg[i]+1]; c++)
+ {
+ copy_rvec(v[c], buf[a++]);
+ }
+ }
+ }
+ }
+
+ dd_scatterv(dd, scounts, disps, buf, dd->nat_home*sizeof(rvec), lv);
+}
+
+static void dd_distribute_vec(gmx_domdec_t *dd, t_block *cgs, rvec *v, rvec *lv)
+{
+ if (dd->nnodes <= GMX_DD_NNODES_SENDRECV)
+ {
+ dd_distribute_vec_sendrecv(dd, cgs, v, lv);
+ }
+ else
+ {
+ dd_distribute_vec_scatterv(dd, cgs, v, lv);
+ }
+}
+
+static void dd_distribute_dfhist(gmx_domdec_t *dd, df_history_t *dfhist)
+{
+ int i;
+ dd_bcast(dd, sizeof(int), &dfhist->bEquil);
+ dd_bcast(dd, sizeof(int), &dfhist->nlambda);
+ dd_bcast(dd, sizeof(real), &dfhist->wl_delta);
+
+ if (dfhist->nlambda > 0)
+ {
+ int nlam = dfhist->nlambda;
+ dd_bcast(dd, sizeof(int)*nlam, dfhist->n_at_lam);
+ dd_bcast(dd, sizeof(real)*nlam, dfhist->wl_histo);
+ dd_bcast(dd, sizeof(real)*nlam, dfhist->sum_weights);
+ dd_bcast(dd, sizeof(real)*nlam, dfhist->sum_dg);
+ dd_bcast(dd, sizeof(real)*nlam, dfhist->sum_minvar);
+ dd_bcast(dd, sizeof(real)*nlam, dfhist->sum_variance);
+
+ for (i = 0; i<nlam; i++) {
+ dd_bcast(dd, sizeof(real)*nlam, dfhist->accum_p[i]);
+ dd_bcast(dd, sizeof(real)*nlam, dfhist->accum_m[i]);
+ dd_bcast(dd, sizeof(real)*nlam, dfhist->accum_p2[i]);
+ dd_bcast(dd, sizeof(real)*nlam, dfhist->accum_m2[i]);
+ dd_bcast(dd, sizeof(real)*nlam, dfhist->Tij[i]);
+ dd_bcast(dd, sizeof(real)*nlam, dfhist->Tij_empirical[i]);
+ }
+ }
+}
+
+static void dd_distribute_state(gmx_domdec_t *dd, t_block *cgs,
+ t_state *state, t_state *state_local,
+ rvec **f)
+{
+ int i, j, nh;
+
+ nh = state->nhchainlength;
+
+ if (DDMASTER(dd))
+ {
+ for (i = 0; i < efptNR; i++)
+ {
+ state_local->lambda[i] = state->lambda[i];
+ }
+ state_local->fep_state = state->fep_state;
+ state_local->veta = state->veta;
+ state_local->vol0 = state->vol0;
+ copy_mat(state->box, state_local->box);
+ copy_mat(state->box_rel, state_local->box_rel);
+ copy_mat(state->boxv, state_local->boxv);
+ copy_mat(state->svir_prev, state_local->svir_prev);
+ copy_mat(state->fvir_prev, state_local->fvir_prev);
+ copy_df_history(&state_local->dfhist,&state->dfhist);
+ for (i = 0; i < state_local->ngtc; i++)
+ {
+ for (j = 0; j < nh; j++)
+ {
+ state_local->nosehoover_xi[i*nh+j] = state->nosehoover_xi[i*nh+j];
+ state_local->nosehoover_vxi[i*nh+j] = state->nosehoover_vxi[i*nh+j];
+ }
+ state_local->therm_integral[i] = state->therm_integral[i];
+ }
+ for (i = 0; i < state_local->nnhpres; i++)
+ {
+ for (j = 0; j < nh; j++)
+ {
+ state_local->nhpres_xi[i*nh+j] = state->nhpres_xi[i*nh+j];
+ state_local->nhpres_vxi[i*nh+j] = state->nhpres_vxi[i*nh+j];
+ }
+ }
+ }
+ dd_bcast(dd, ((efptNR)*sizeof(real)), state_local->lambda);
+ dd_bcast(dd, sizeof(int), &state_local->fep_state);
+ dd_bcast(dd, sizeof(real), &state_local->veta);
+ dd_bcast(dd, sizeof(real), &state_local->vol0);
+ dd_bcast(dd, sizeof(state_local->box), state_local->box);
+ dd_bcast(dd, sizeof(state_local->box_rel), state_local->box_rel);
+ dd_bcast(dd, sizeof(state_local->boxv), state_local->boxv);
+ dd_bcast(dd, sizeof(state_local->svir_prev), state_local->svir_prev);
+ dd_bcast(dd, sizeof(state_local->fvir_prev), state_local->fvir_prev);
+ dd_bcast(dd, ((state_local->ngtc*nh)*sizeof(double)), state_local->nosehoover_xi);
+ dd_bcast(dd, ((state_local->ngtc*nh)*sizeof(double)), state_local->nosehoover_vxi);
+ dd_bcast(dd, state_local->ngtc*sizeof(double), state_local->therm_integral);
+ dd_bcast(dd, ((state_local->nnhpres*nh)*sizeof(double)), state_local->nhpres_xi);
+ dd_bcast(dd, ((state_local->nnhpres*nh)*sizeof(double)), state_local->nhpres_vxi);
+
+ /* communicate df_history -- required for restarting from checkpoint */
+ dd_distribute_dfhist(dd,&state_local->dfhist);
+
+ if (dd->nat_home > state_local->nalloc)
+ {
+ dd_realloc_state(state_local, f, dd->nat_home);
+ }
+ for (i = 0; i < estNR; i++)
+ {
+ if (EST_DISTR(i) && (state_local->flags & (1<<i)))
+ {
+ switch (i)
+ {
+ case estX:
+ dd_distribute_vec(dd, cgs, state->x, state_local->x);
+ break;
+ case estV:
+ dd_distribute_vec(dd, cgs, state->v, state_local->v);
+ break;
+ case estSDX:
+ dd_distribute_vec(dd, cgs, state->sd_X, state_local->sd_X);
+ break;
+ case estCGP:
+ dd_distribute_vec(dd, cgs, state->cg_p, state_local->cg_p);
+ break;
+ case estLD_RNG:
+ if (state->nrngi == 1)
+ {
+ dd_bcastc(dd,
+ state_local->nrng*sizeof(state_local->ld_rng[0]),
+ state->ld_rng, state_local->ld_rng);
+ }
+ else
+ {
+ dd_scatter(dd,
+ state_local->nrng*sizeof(state_local->ld_rng[0]),
+ state->ld_rng, state_local->ld_rng);
+ }
+ break;
+ case estLD_RNGI:
+ if (state->nrngi == 1)
+ {
+ dd_bcastc(dd, sizeof(state_local->ld_rngi[0]),
+ state->ld_rngi, state_local->ld_rngi);
+ }
+ else
+ {
+ dd_scatter(dd, sizeof(state_local->ld_rngi[0]),
+ state->ld_rngi, state_local->ld_rngi);
+ }
+ break;
+ case estDISRE_INITF:
+ case estDISRE_RM3TAV:
+ case estORIRE_INITF:
+ case estORIRE_DTAV:
+ /* Not implemented yet */
+ break;
+ default:
+ gmx_incons("Unknown state entry encountered in dd_distribute_state");
+ }
+ }
+ }
+}
+
+static char dim2char(int dim)
+{
+ char c = '?';
+
+ switch (dim)
+ {
+ case XX: c = 'X'; break;
+ case YY: c = 'Y'; break;
+ case ZZ: c = 'Z'; break;
+ default: gmx_fatal(FARGS, "Unknown dim %d", dim);
+ }
+
+ return c;
+}
+
+static void write_dd_grid_pdb(const char *fn, gmx_large_int_t step,
+ gmx_domdec_t *dd, matrix box, gmx_ddbox_t *ddbox)
+{
+ rvec grid_s[2], *grid_r = NULL, cx, r;
+ char fname[STRLEN], format[STRLEN], buf[22];
+ FILE *out;
+ int a, i, d, z, y, x;
+ matrix tric;
+ real vol;
+
+ copy_rvec(dd->comm->cell_x0, grid_s[0]);
+ copy_rvec(dd->comm->cell_x1, grid_s[1]);
+
+ if (DDMASTER(dd))
+ {
+ snew(grid_r, 2*dd->nnodes);
+ }
+
+ dd_gather(dd, 2*sizeof(rvec), grid_s[0], DDMASTER(dd) ? grid_r[0] : NULL);
+
+ if (DDMASTER(dd))
+ {
+ for (d = 0; d < DIM; d++)
+ {
+ for (i = 0; i < DIM; i++)
+ {
+ if (d == i)
+ {
+ tric[d][i] = 1;
+ }
+ else
+ {
+ if (d < ddbox->npbcdim && dd->nc[d] > 1)
+ {
+ tric[d][i] = box[i][d]/box[i][i];
+ }
+ else
+ {
+ tric[d][i] = 0;
+ }
+ }
+ }
+ }
+ sprintf(fname, "%s_%s.pdb", fn, gmx_step_str(step, buf));
+ sprintf(format, "%s%s\n", get_pdbformat(), "%6.2f%6.2f");
+ out = gmx_fio_fopen(fname, "w");
+ gmx_write_pdb_box(out, dd->bScrewPBC ? epbcSCREW : epbcXYZ, box);
+ a = 1;
+ for (i = 0; i < dd->nnodes; i++)
+ {
+ vol = dd->nnodes/(box[XX][XX]*box[YY][YY]*box[ZZ][ZZ]);
+ for (d = 0; d < DIM; d++)
+ {
+ vol *= grid_r[i*2+1][d] - grid_r[i*2][d];
+ }
+ for (z = 0; z < 2; z++)
+ {
+ for (y = 0; y < 2; y++)
+ {
+ for (x = 0; x < 2; x++)
+ {
+ cx[XX] = grid_r[i*2+x][XX];
+ cx[YY] = grid_r[i*2+y][YY];
+ cx[ZZ] = grid_r[i*2+z][ZZ];
+ mvmul(tric, cx, r);
+ fprintf(out, format, "ATOM", a++, "CA", "GLY", ' ', 1+i,
+ ' ', 10*r[XX], 10*r[YY], 10*r[ZZ], 1.0, vol);
+ }
+ }
+ }
+ for (d = 0; d < DIM; d++)
+ {
+ for (x = 0; x < 4; x++)
+ {
+ switch (d)
+ {
+ case 0: y = 1 + i*8 + 2*x; break;
+ case 1: y = 1 + i*8 + 2*x - (x % 2); break;
+ case 2: y = 1 + i*8 + x; break;
+ }
+ fprintf(out, "%6s%5d%5d\n", "CONECT", y, y+(1<<d));
+ }
+ }
+ }
+ gmx_fio_fclose(out);
+ sfree(grid_r);
+ }
+}
+
+void write_dd_pdb(const char *fn, gmx_large_int_t step, const char *title,
+ gmx_mtop_t *mtop, t_commrec *cr,
+ int natoms, rvec x[], matrix box)
+{
+ char fname[STRLEN], format[STRLEN], format4[STRLEN], buf[22];
+ FILE *out;
+ int i, ii, resnr, c;
+ char *atomname, *resname;
+ real b;
+ gmx_domdec_t *dd;
+
+ dd = cr->dd;
+ if (natoms == -1)
+ {
+ natoms = dd->comm->nat[ddnatVSITE];
+ }
+
+ sprintf(fname, "%s_%s_n%d.pdb", fn, gmx_step_str(step, buf), cr->sim_nodeid);
+
+ sprintf(format, "%s%s\n", get_pdbformat(), "%6.2f%6.2f");
+ sprintf(format4, "%s%s\n", get_pdbformat4(), "%6.2f%6.2f");
+
+ out = gmx_fio_fopen(fname, "w");
+
+ fprintf(out, "TITLE %s\n", title);
+ gmx_write_pdb_box(out, dd->bScrewPBC ? epbcSCREW : epbcXYZ, box);
+ for (i = 0; i < natoms; i++)
+ {
+ ii = dd->gatindex[i];
+ gmx_mtop_atominfo_global(mtop, ii, &atomname, &resnr, &resname);
+ if (i < dd->comm->nat[ddnatZONE])
+ {
+ c = 0;
+ while (i >= dd->cgindex[dd->comm->zones.cg_range[c+1]])
+ {
+ c++;
+ }
+ b = c;
+ }
+ else if (i < dd->comm->nat[ddnatVSITE])
+ {
+ b = dd->comm->zones.n;
+ }
+ else
+ {
+ b = dd->comm->zones.n + 1;
+ }
+ fprintf(out, strlen(atomname) < 4 ? format : format4,
+ "ATOM", (ii+1)%100000,
+ atomname, resname, ' ', resnr%10000, ' ',
+ 10*x[i][XX], 10*x[i][YY], 10*x[i][ZZ], 1.0, b);
+ }
+ fprintf(out, "TER\n");
+
+ gmx_fio_fclose(out);
+}
+
+real dd_cutoff_mbody(gmx_domdec_t *dd)
+{
+ gmx_domdec_comm_t *comm;
+ int di;
+ real r;
+
+ comm = dd->comm;
+
+ r = -1;
+ if (comm->bInterCGBondeds)
+ {
+ if (comm->cutoff_mbody > 0)
+ {
+ r = comm->cutoff_mbody;
+ }
+ else
+ {
+ /* cutoff_mbody=0 means we do not have DLB */
+ r = comm->cellsize_min[dd->dim[0]];
+ for (di = 1; di < dd->ndim; di++)
+ {
+ r = min(r, comm->cellsize_min[dd->dim[di]]);
+ }
+ if (comm->bBondComm)
+ {
+ r = max(r, comm->cutoff_mbody);
+ }
+ else
+ {
+ r = min(r, comm->cutoff);
+ }
+ }
+ }
+
+ return r;
+}
+
+real dd_cutoff_twobody(gmx_domdec_t *dd)
+{
+ real r_mb;
+
+ r_mb = dd_cutoff_mbody(dd);
+
+ return max(dd->comm->cutoff, r_mb);
+}
+
+
+static void dd_cart_coord2pmecoord(gmx_domdec_t *dd, ivec coord, ivec coord_pme)
+{
+ int nc, ntot;
+
+ nc = dd->nc[dd->comm->cartpmedim];
+ ntot = dd->comm->ntot[dd->comm->cartpmedim];
+ copy_ivec(coord, coord_pme);
+ coord_pme[dd->comm->cartpmedim] =
+ nc + (coord[dd->comm->cartpmedim]*(ntot - nc) + (ntot - nc)/2)/nc;
+}
+
+static int low_ddindex2pmeindex(int ndd, int npme, int ddindex)
+{
+ /* Here we assign a PME node to communicate with this DD node
+ * by assuming that the major index of both is x.
+ * We add cr->npmenodes/2 to obtain an even distribution.
+ */
+ return (ddindex*npme + npme/2)/ndd;
+}
+
+static int ddindex2pmeindex(const gmx_domdec_t *dd, int ddindex)
+{
+ return low_ddindex2pmeindex(dd->nnodes, dd->comm->npmenodes, ddindex);
+}
+
+static int cr_ddindex2pmeindex(const t_commrec *cr, int ddindex)
+{
+ return low_ddindex2pmeindex(cr->dd->nnodes, cr->npmenodes, ddindex);
+}
+
+static int *dd_pmenodes(t_commrec *cr)
+{
+ int *pmenodes;
+ int n, i, p0, p1;
+
+ snew(pmenodes, cr->npmenodes);
+ n = 0;
+ for (i = 0; i < cr->dd->nnodes; i++)
+ {
+ p0 = cr_ddindex2pmeindex(cr, i);
+ p1 = cr_ddindex2pmeindex(cr, i+1);
+ if (i+1 == cr->dd->nnodes || p1 > p0)
+ {
+ if (debug)
+ {
+ fprintf(debug, "pmenode[%d] = %d\n", n, i+1+n);
+ }
+ pmenodes[n] = i + 1 + n;
+ n++;
+ }
+ }
+
+ return pmenodes;
+}
+
+static int gmx_ddcoord2pmeindex(t_commrec *cr, int x, int y, int z)
+{
+ gmx_domdec_t *dd;
+ ivec coords, coords_pme, nc;
+ int slab;
+
+ dd = cr->dd;
+ /*
+ if (dd->comm->bCartesian) {
+ gmx_ddindex2xyz(dd->nc,ddindex,coords);
+ dd_coords2pmecoords(dd,coords,coords_pme);
+ copy_ivec(dd->ntot,nc);
+ nc[dd->cartpmedim] -= dd->nc[dd->cartpmedim];
+ coords_pme[dd->cartpmedim] -= dd->nc[dd->cartpmedim];
+
+ slab = (coords_pme[XX]*nc[YY] + coords_pme[YY])*nc[ZZ] + coords_pme[ZZ];
+ } else {
+ slab = (ddindex*cr->npmenodes + cr->npmenodes/2)/dd->nnodes;
+ }
+ */
+ coords[XX] = x;
+ coords[YY] = y;
+ coords[ZZ] = z;
+ slab = ddindex2pmeindex(dd, dd_index(dd->nc, coords));
+
+ return slab;
+}
+
+static int ddcoord2simnodeid(t_commrec *cr, int x, int y, int z)
+{
+ gmx_domdec_comm_t *comm;
+ ivec coords;
+ int ddindex, nodeid = -1;
+
+ comm = cr->dd->comm;
+
+ coords[XX] = x;
+ coords[YY] = y;
+ coords[ZZ] = z;
+ if (comm->bCartesianPP_PME)
+ {
+#ifdef GMX_MPI
+ MPI_Cart_rank(cr->mpi_comm_mysim, coords, &nodeid);
+#endif
+ }
+ else
+ {
+ ddindex = dd_index(cr->dd->nc, coords);
+ if (comm->bCartesianPP)
+ {
+ nodeid = comm->ddindex2simnodeid[ddindex];
+ }
+ else
+ {
+ if (comm->pmenodes)
+ {
+ nodeid = ddindex + gmx_ddcoord2pmeindex(cr, x, y, z);
+ }
+ else
+ {
+ nodeid = ddindex;
+ }
+ }
+ }
+
+ return nodeid;
+}
+
+static int dd_simnode2pmenode(t_commrec *cr, int sim_nodeid)
+{
+ gmx_domdec_t *dd;
+ gmx_domdec_comm_t *comm;
+ ivec coord, coord_pme;
+ int i;
+ int pmenode = -1;
+
+ dd = cr->dd;
+ comm = dd->comm;
+
+ /* This assumes a uniform x domain decomposition grid cell size */
+ if (comm->bCartesianPP_PME)
+ {
+#ifdef GMX_MPI
+ MPI_Cart_coords(cr->mpi_comm_mysim, sim_nodeid, DIM, coord);
+ if (coord[comm->cartpmedim] < dd->nc[comm->cartpmedim])
+ {
+ /* This is a PP node */
+ dd_cart_coord2pmecoord(dd, coord, coord_pme);
+ MPI_Cart_rank(cr->mpi_comm_mysim, coord_pme, &pmenode);
+ }
+#endif
+ }
+ else if (comm->bCartesianPP)
+ {
+ if (sim_nodeid < dd->nnodes)
+ {
+ pmenode = dd->nnodes + ddindex2pmeindex(dd, sim_nodeid);
+ }
+ }
+ else
+ {
+ /* This assumes DD cells with identical x coordinates
+ * are numbered sequentially.
+ */
+ if (dd->comm->pmenodes == NULL)
+ {
+ if (sim_nodeid < dd->nnodes)
+ {
+ /* The DD index equals the nodeid */
+ pmenode = dd->nnodes + ddindex2pmeindex(dd, sim_nodeid);
+ }
+ }
+ else
+ {
+ i = 0;
+ while (sim_nodeid > dd->comm->pmenodes[i])
+ {
+ i++;
+ }
+ if (sim_nodeid < dd->comm->pmenodes[i])
+ {
+ pmenode = dd->comm->pmenodes[i];
+ }
+ }
+ }
+
+ return pmenode;
+}
+
+void get_pme_nnodes(const gmx_domdec_t *dd,
+ int *npmenodes_x, int *npmenodes_y)
+{
+ if (dd != NULL)
+ {
+ *npmenodes_x = dd->comm->npmenodes_x;
+ *npmenodes_y = dd->comm->npmenodes_y;
+ }
+ else
+ {
+ *npmenodes_x = 1;
+ *npmenodes_y = 1;
+ }
+}
+
+gmx_bool gmx_pmeonlynode(t_commrec *cr, int sim_nodeid)
+{
+ gmx_bool bPMEOnlyNode;
+
+ if (DOMAINDECOMP(cr))
+ {
+ bPMEOnlyNode = (dd_simnode2pmenode(cr, sim_nodeid) == -1);
+ }
+ else
+ {
+ bPMEOnlyNode = FALSE;
+ }
+
+ return bPMEOnlyNode;
+}
+
+void get_pme_ddnodes(t_commrec *cr, int pmenodeid,
+ int *nmy_ddnodes, int **my_ddnodes, int *node_peer)
+{
+ gmx_domdec_t *dd;
+ int x, y, z;
+ ivec coord, coord_pme;
+
+ dd = cr->dd;
+
+ snew(*my_ddnodes, (dd->nnodes+cr->npmenodes-1)/cr->npmenodes);
+
+ *nmy_ddnodes = 0;
+ for (x = 0; x < dd->nc[XX]; x++)
+ {
+ for (y = 0; y < dd->nc[YY]; y++)
+ {
+ for (z = 0; z < dd->nc[ZZ]; z++)
+ {
+ if (dd->comm->bCartesianPP_PME)
+ {
+ coord[XX] = x;
+ coord[YY] = y;
+ coord[ZZ] = z;
+ dd_cart_coord2pmecoord(dd, coord, coord_pme);
+ if (dd->ci[XX] == coord_pme[XX] &&
+ dd->ci[YY] == coord_pme[YY] &&
+ dd->ci[ZZ] == coord_pme[ZZ])
+ {
+ (*my_ddnodes)[(*nmy_ddnodes)++] = ddcoord2simnodeid(cr, x, y, z);
+ }
+ }
+ else
+ {
+ /* The slab corresponds to the nodeid in the PME group */
+ if (gmx_ddcoord2pmeindex(cr, x, y, z) == pmenodeid)
+ {
+ (*my_ddnodes)[(*nmy_ddnodes)++] = ddcoord2simnodeid(cr, x, y, z);
+ }
+ }
+ }
+ }
+ }
+
+ /* The last PP-only node is the peer node */
+ *node_peer = (*my_ddnodes)[*nmy_ddnodes-1];
+
+ if (debug)
+ {
+ fprintf(debug, "Receive coordinates from PP nodes:");
+ for (x = 0; x < *nmy_ddnodes; x++)
+ {
+ fprintf(debug, " %d", (*my_ddnodes)[x]);
+ }
+ fprintf(debug, "\n");
+ }
+}
+
+static gmx_bool receive_vir_ener(t_commrec *cr)
+{
+ gmx_domdec_comm_t *comm;
+ int pmenode, coords[DIM], rank;
+ gmx_bool bReceive;
+
+ bReceive = TRUE;
+ if (cr->npmenodes < cr->dd->nnodes)
+ {
+ comm = cr->dd->comm;
+ if (comm->bCartesianPP_PME)
+ {
+ pmenode = dd_simnode2pmenode(cr, cr->sim_nodeid);
+#ifdef GMX_MPI
+ MPI_Cart_coords(cr->mpi_comm_mysim, cr->sim_nodeid, DIM, coords);
+ coords[comm->cartpmedim]++;
+ if (coords[comm->cartpmedim] < cr->dd->nc[comm->cartpmedim])
+ {
+ MPI_Cart_rank(cr->mpi_comm_mysim, coords, &rank);
+ if (dd_simnode2pmenode(cr, rank) == pmenode)
+ {
+ /* This is not the last PP node for pmenode */
+ bReceive = FALSE;
+ }
+ }
+#endif
+ }
+ else
+ {
+ pmenode = dd_simnode2pmenode(cr, cr->sim_nodeid);
+ if (cr->sim_nodeid+1 < cr->nnodes &&
+ dd_simnode2pmenode(cr, cr->sim_nodeid+1) == pmenode)
+ {
+ /* This is not the last PP node for pmenode */
+ bReceive = FALSE;
+ }
+ }
+ }
+
+ return bReceive;
+}
+
+static void set_zones_ncg_home(gmx_domdec_t *dd)
+{
+ gmx_domdec_zones_t *zones;
+ int i;
+
+ zones = &dd->comm->zones;
+
+ zones->cg_range[0] = 0;
+ for (i = 1; i < zones->n+1; i++)
+ {
+ zones->cg_range[i] = dd->ncg_home;
+ }
+ /* zone_ncg1[0] should always be equal to ncg_home */
+ dd->comm->zone_ncg1[0] = dd->ncg_home;
+}
+
+static void rebuild_cgindex(gmx_domdec_t *dd,
+ const int *gcgs_index, t_state *state)
+{
+ int nat, i, *ind, *dd_cg_gl, *cgindex, cg_gl;
+
+ ind = state->cg_gl;
+ dd_cg_gl = dd->index_gl;
+ cgindex = dd->cgindex;
+ nat = 0;
+ cgindex[0] = nat;
+ for (i = 0; i < state->ncg_gl; i++)
+ {
+ cgindex[i] = nat;
+ cg_gl = ind[i];
+ dd_cg_gl[i] = cg_gl;
+ nat += gcgs_index[cg_gl+1] - gcgs_index[cg_gl];
+ }
+ cgindex[i] = nat;
+
+ dd->ncg_home = state->ncg_gl;
+ dd->nat_home = nat;
+
+ set_zones_ncg_home(dd);
+}
+
+static int ddcginfo(const cginfo_mb_t *cginfo_mb, int cg)
+{
+ while (cg >= cginfo_mb->cg_end)
+ {
+ cginfo_mb++;
+ }
+
+ return cginfo_mb->cginfo[(cg - cginfo_mb->cg_start) % cginfo_mb->cg_mod];
+}
+
+static void dd_set_cginfo(int *index_gl, int cg0, int cg1,
+ t_forcerec *fr, char *bLocalCG)
+{
+ cginfo_mb_t *cginfo_mb;
+ int *cginfo;
+ int cg;
+
+ if (fr != NULL)
+ {
+ cginfo_mb = fr->cginfo_mb;
+ cginfo = fr->cginfo;
+
+ for (cg = cg0; cg < cg1; cg++)
+ {
+ cginfo[cg] = ddcginfo(cginfo_mb, index_gl[cg]);
+ }
+ }
+
+ if (bLocalCG != NULL)
+ {
+ for (cg = cg0; cg < cg1; cg++)
+ {
+ bLocalCG[index_gl[cg]] = TRUE;
+ }
+ }
+}
+
+static void make_dd_indices(gmx_domdec_t *dd,
+ const int *gcgs_index, int cg_start)
+{
+ int nzone, zone, zone1, cg0, cg1, cg1_p1, cg, cg_gl, a, a_gl;
+ int *zone2cg, *zone_ncg1, *index_gl, *gatindex;
+ gmx_ga2la_t *ga2la;
+ char *bLocalCG;
+ gmx_bool bCGs;
+
+ bLocalCG = dd->comm->bLocalCG;
+
+ if (dd->nat_tot > dd->gatindex_nalloc)
+ {
+ dd->gatindex_nalloc = over_alloc_dd(dd->nat_tot);
+ srenew(dd->gatindex, dd->gatindex_nalloc);
+ }
+
+ nzone = dd->comm->zones.n;
+ zone2cg = dd->comm->zones.cg_range;
+ zone_ncg1 = dd->comm->zone_ncg1;
+ index_gl = dd->index_gl;
+ gatindex = dd->gatindex;
+ bCGs = dd->comm->bCGs;
+
+ if (zone2cg[1] != dd->ncg_home)
+ {
+ gmx_incons("dd->ncg_zone is not up to date");
+ }
+
+ /* Make the local to global and global to local atom index */
+ a = dd->cgindex[cg_start];
+ for (zone = 0; zone < nzone; zone++)
+ {
+ if (zone == 0)
+ {
+ cg0 = cg_start;
+ }
+ else
+ {
+ cg0 = zone2cg[zone];
+ }
+ cg1 = zone2cg[zone+1];
+ cg1_p1 = cg0 + zone_ncg1[zone];
+
+ for (cg = cg0; cg < cg1; cg++)
+ {
+ zone1 = zone;
+ if (cg >= cg1_p1)
+ {
+ /* Signal that this cg is from more than one pulse away */
+ zone1 += nzone;
+ }
+ cg_gl = index_gl[cg];
+ if (bCGs)
+ {
+ for (a_gl = gcgs_index[cg_gl]; a_gl < gcgs_index[cg_gl+1]; a_gl++)
+ {
+ gatindex[a] = a_gl;
+ ga2la_set(dd->ga2la, a_gl, a, zone1);
+ a++;
+ }
+ }
+ else
+ {
+ gatindex[a] = cg_gl;
+ ga2la_set(dd->ga2la, cg_gl, a, zone1);
+ a++;
+ }
+ }
+ }
+}
+
+static int check_bLocalCG(gmx_domdec_t *dd, int ncg_sys, const char *bLocalCG,
+ const char *where)
+{
+ int ncg, i, ngl, nerr;
+
+ nerr = 0;
+ if (bLocalCG == NULL)
+ {
+ return nerr;
+ }
+ for (i = 0; i < dd->ncg_tot; i++)
+ {
+ if (!bLocalCG[dd->index_gl[i]])
+ {
+ fprintf(stderr,
+ "DD node %d, %s: cg %d, global cg %d is not marked in bLocalCG (ncg_home %d)\n", dd->rank, where, i+1, dd->index_gl[i]+1, dd->ncg_home);
+ nerr++;
+ }
+ }
+ ngl = 0;
+ for (i = 0; i < ncg_sys; i++)
+ {
+ if (bLocalCG[i])
+ {
+ ngl++;
+ }
+ }
+ if (ngl != dd->ncg_tot)
+ {
+ fprintf(stderr, "DD node %d, %s: In bLocalCG %d cgs are marked as local, whereas there are %d\n", dd->rank, where, ngl, dd->ncg_tot);
+ nerr++;
+ }
+
+ return nerr;
+}
+
+static void check_index_consistency(gmx_domdec_t *dd,
+ int natoms_sys, int ncg_sys,
+ const char *where)
+{
+ int nerr, ngl, i, a, cell;
+ int *have;
+
+ nerr = 0;
+
+ if (dd->comm->DD_debug > 1)
+ {
+ snew(have, natoms_sys);
+ for (a = 0; a < dd->nat_tot; a++)
+ {
+ if (have[dd->gatindex[a]] > 0)
+ {
+ fprintf(stderr, "DD node %d: global atom %d occurs twice: index %d and %d\n", dd->rank, dd->gatindex[a]+1, have[dd->gatindex[a]], a+1);
+ }
+ else
+ {
+ have[dd->gatindex[a]] = a + 1;
+ }
+ }
+ sfree(have);
+ }
+
+ snew(have, dd->nat_tot);
+
+ ngl = 0;
+ for (i = 0; i < natoms_sys; i++)
+ {
+ if (ga2la_get(dd->ga2la, i, &a, &cell))
+ {
+ if (a >= dd->nat_tot)
+ {
+ fprintf(stderr, "DD node %d: global atom %d marked as local atom %d, which is larger than nat_tot (%d)\n", dd->rank, i+1, a+1, dd->nat_tot);
+ nerr++;
+ }
+ else
+ {
+ have[a] = 1;
+ if (dd->gatindex[a] != i)
+ {
+ fprintf(stderr, "DD node %d: global atom %d marked as local atom %d, which has global atom index %d\n", dd->rank, i+1, a+1, dd->gatindex[a]+1);
+ nerr++;
+ }
+ }
+ ngl++;
+ }
+ }
+ if (ngl != dd->nat_tot)
+ {
+ fprintf(stderr,
+ "DD node %d, %s: %d global atom indices, %d local atoms\n",
+ dd->rank, where, ngl, dd->nat_tot);
+ }
+ for (a = 0; a < dd->nat_tot; a++)
+ {
+ if (have[a] == 0)
+ {
+ fprintf(stderr,
+ "DD node %d, %s: local atom %d, global %d has no global index\n",
+ dd->rank, where, a+1, dd->gatindex[a]+1);
+ }
+ }
+ sfree(have);
+
+ nerr += check_bLocalCG(dd, ncg_sys, dd->comm->bLocalCG, where);
+
+ if (nerr > 0)
+ {
+ gmx_fatal(FARGS, "DD node %d, %s: %d atom/cg index inconsistencies",
+ dd->rank, where, nerr);
+ }
+}
+
+static void clear_dd_indices(gmx_domdec_t *dd, int cg_start, int a_start)
+{
+ int i;
+ char *bLocalCG;
+
+ if (a_start == 0)
+ {
+ /* Clear the whole list without searching */
+ ga2la_clear(dd->ga2la);
+ }
+ else
+ {
+ for (i = a_start; i < dd->nat_tot; i++)
+ {
+ ga2la_del(dd->ga2la, dd->gatindex[i]);
+ }
+ }
+
+ bLocalCG = dd->comm->bLocalCG;
+ if (bLocalCG)
+ {
+ for (i = cg_start; i < dd->ncg_tot; i++)
+ {
+ bLocalCG[dd->index_gl[i]] = FALSE;
+ }
+ }
+
+ dd_clear_local_vsite_indices(dd);
+
+ if (dd->constraints)
+ {
+ dd_clear_local_constraint_indices(dd);
+ }
+}
+
+/* This function should be used for moving the domain boudaries during DLB,
+ * for obtaining the minimum cell size. It checks the initially set limit
+ * comm->cellsize_min, for bonded and initial non-bonded cut-offs,
+ * and, possibly, a longer cut-off limit set for PME load balancing.
+ */
+static real cellsize_min_dlb(gmx_domdec_comm_t *comm, int dim_ind, int dim)
+{
+ real cellsize_min;
+
+ cellsize_min = comm->cellsize_min[dim];
+
+ if (!comm->bVacDLBNoLimit)
+ {
+ /* The cut-off might have changed, e.g. by PME load balacning,
+ * from the value used to set comm->cellsize_min, so check it.
+ */
+ cellsize_min = max(cellsize_min, comm->cutoff/comm->cd[dim_ind].np_dlb);
+
+ if (comm->bPMELoadBalDLBLimits)
+ {
+ /* Check for the cut-off limit set by the PME load balancing */
+ cellsize_min = max(cellsize_min, comm->PMELoadBal_max_cutoff/comm->cd[dim_ind].np_dlb);
+ }
+ }
+
+ return cellsize_min;
+}
+
+static real grid_jump_limit(gmx_domdec_comm_t *comm, real cutoff,
+ int dim_ind)
+{
+ real grid_jump_limit;
+
+ /* The distance between the boundaries of cells at distance
+ * x+-1,y+-1 or y+-1,z+-1 is limited by the cut-off restrictions
+ * and by the fact that cells should not be shifted by more than
+ * half their size, such that cg's only shift by one cell
+ * at redecomposition.
+ */
+ grid_jump_limit = comm->cellsize_limit;
+ if (!comm->bVacDLBNoLimit)
+ {
+ if (comm->bPMELoadBalDLBLimits)
+ {
+ cutoff = max(cutoff, comm->PMELoadBal_max_cutoff);
+ }
+ grid_jump_limit = max(grid_jump_limit,
+ cutoff/comm->cd[dim_ind].np);
+ }
+
+ return grid_jump_limit;
+}
+
+static gmx_bool check_grid_jump(gmx_large_int_t step,
+ gmx_domdec_t *dd,
+ real cutoff,
+ gmx_ddbox_t *ddbox,
+ gmx_bool bFatal)
+{
+ gmx_domdec_comm_t *comm;
+ int d, dim;
+ real limit, bfac;
+ gmx_bool bInvalid;
+
+ bInvalid = FALSE;
+
+ comm = dd->comm;
+
+ for (d = 1; d < dd->ndim; d++)
+ {
+ dim = dd->dim[d];
+ limit = grid_jump_limit(comm, cutoff, d);
+ bfac = ddbox->box_size[dim];
+ if (ddbox->tric_dir[dim])
+ {
+ bfac *= ddbox->skew_fac[dim];
+ }
+ if ((comm->cell_f1[d] - comm->cell_f_max0[d])*bfac < limit ||
+ (comm->cell_f0[d] - comm->cell_f_min1[d])*bfac > -limit)
+ {
+ bInvalid = TRUE;
+
+ if (bFatal)
+ {
+ char buf[22];
+
+ /* This error should never be triggered under normal
+ * circumstances, but you never know ...
+ */
+ gmx_fatal(FARGS, "Step %s: The domain decomposition grid has shifted too much in the %c-direction around cell %d %d %d. This should not have happened. Running with less nodes might avoid this issue.",
+ gmx_step_str(step, buf),
+ dim2char(dim), dd->ci[XX], dd->ci[YY], dd->ci[ZZ]);
+ }
+ }
+ }
+
+ return bInvalid;
+}
+
+static int dd_load_count(gmx_domdec_comm_t *comm)
+{
+ return (comm->eFlop ? comm->flop_n : comm->cycl_n[ddCyclF]);
+}
+
+static float dd_force_load(gmx_domdec_comm_t *comm)
+{
+ float load;
+
+ if (comm->eFlop)
+ {
+ load = comm->flop;
+ if (comm->eFlop > 1)
+ {
+ load *= 1.0 + (comm->eFlop - 1)*(0.1*rand()/RAND_MAX - 0.05);
+ }
+ }
+ else
+ {
+ load = comm->cycl[ddCyclF];
+ if (comm->cycl_n[ddCyclF] > 1)
+ {
+ /* Subtract the maximum of the last n cycle counts
++ * to get rid of possible high counts due to other sources,
+ * for instance system activity, that would otherwise
+ * affect the dynamic load balancing.
+ */
+ load -= comm->cycl_max[ddCyclF];
+ }
++
++#ifdef GMX_MPI
++ if (comm->cycl_n[ddCyclWaitGPU] && comm->nrank_gpu_shared > 1)
++ {
++ float gpu_wait, gpu_wait_sum;
++
++ gpu_wait = comm->cycl[ddCyclWaitGPU];
++ if (comm->cycl_n[ddCyclF] > 1)
++ {
++ /* We should remove the WaitGPU time of the same MD step
++ * as the one with the maximum F time, since the F time
++ * and the wait time are not independent.
++ * Furthermore, the step for the max F time should be chosen
++ * the same on all ranks that share the same GPU.
++ * But to keep the code simple, we remove the average instead.
++ * The main reason for artificially long times at some steps
++ * is spurious CPU activity or MPI time, so we don't expect
++ * that changes in the GPU wait time matter a lot here.
++ */
++ gpu_wait *= (comm->cycl_n[ddCyclF] - 1)/(float)comm->cycl_n[ddCyclF];
++ }
++ /* Sum the wait times over the ranks that share the same GPU */
++ MPI_Allreduce(&gpu_wait, &gpu_wait_sum, 1, MPI_FLOAT, MPI_SUM,
++ comm->mpi_comm_gpu_shared);
++ /* Replace the wait time by the average over the ranks */
++ load += -gpu_wait + gpu_wait_sum/comm->nrank_gpu_shared;
++ }
++#endif
+ }
+
+ return load;
+}
+
+static void set_slb_pme_dim_f(gmx_domdec_t *dd, int dim, real **dim_f)
+{
+ gmx_domdec_comm_t *comm;
+ int i;
+
+ comm = dd->comm;
+
+ snew(*dim_f, dd->nc[dim]+1);
+ (*dim_f)[0] = 0;
+ for (i = 1; i < dd->nc[dim]; i++)
+ {
+ if (comm->slb_frac[dim])
+ {
+ (*dim_f)[i] = (*dim_f)[i-1] + comm->slb_frac[dim][i-1];
+ }
+ else
+ {
+ (*dim_f)[i] = (real)i/(real)dd->nc[dim];
+ }
+ }
+ (*dim_f)[dd->nc[dim]] = 1;
+}
+
+static void init_ddpme(gmx_domdec_t *dd, gmx_ddpme_t *ddpme, int dimind)
+{
+ int pmeindex, slab, nso, i;
+ ivec xyz;
+
+ if (dimind == 0 && dd->dim[0] == YY && dd->comm->npmenodes_x == 1)
+ {
+ ddpme->dim = YY;
+ }
+ else
+ {
+ ddpme->dim = dimind;
+ }
+ ddpme->dim_match = (ddpme->dim == dd->dim[dimind]);
+
+ ddpme->nslab = (ddpme->dim == 0 ?
+ dd->comm->npmenodes_x :
+ dd->comm->npmenodes_y);
+
+ if (ddpme->nslab <= 1)
+ {
+ return;
+ }
+
+ nso = dd->comm->npmenodes/ddpme->nslab;
+ /* Determine for each PME slab the PP location range for dimension dim */
+ snew(ddpme->pp_min, ddpme->nslab);
+ snew(ddpme->pp_max, ddpme->nslab);
+ for (slab = 0; slab < ddpme->nslab; slab++)
+ {
+ ddpme->pp_min[slab] = dd->nc[dd->dim[dimind]] - 1;
+ ddpme->pp_max[slab] = 0;
+ }
+ for (i = 0; i < dd->nnodes; i++)
+ {
+ ddindex2xyz(dd->nc, i, xyz);
+ /* For y only use our y/z slab.
+ * This assumes that the PME x grid size matches the DD grid size.
+ */
+ if (dimind == 0 || xyz[XX] == dd->ci[XX])
+ {
+ pmeindex = ddindex2pmeindex(dd, i);
+ if (dimind == 0)
+ {
+ slab = pmeindex/nso;
+ }
+ else
+ {
+ slab = pmeindex % ddpme->nslab;
+ }
+ ddpme->pp_min[slab] = min(ddpme->pp_min[slab], xyz[dimind]);
+ ddpme->pp_max[slab] = max(ddpme->pp_max[slab], xyz[dimind]);
+ }
+ }
+
+ set_slb_pme_dim_f(dd, ddpme->dim, &ddpme->slb_dim_f);
+}
+
+int dd_pme_maxshift_x(gmx_domdec_t *dd)
+{
+ if (dd->comm->ddpme[0].dim == XX)
+ {
+ return dd->comm->ddpme[0].maxshift;
+ }
+ else
+ {
+ return 0;
+ }
+}
+
+int dd_pme_maxshift_y(gmx_domdec_t *dd)
+{
+ if (dd->comm->ddpme[0].dim == YY)
+ {
+ return dd->comm->ddpme[0].maxshift;
+ }
+ else if (dd->comm->npmedecompdim >= 2 && dd->comm->ddpme[1].dim == YY)
+ {
+ return dd->comm->ddpme[1].maxshift;
+ }
+ else
+ {
+ return 0;
+ }
+}
+
+static void set_pme_maxshift(gmx_domdec_t *dd, gmx_ddpme_t *ddpme,
+ gmx_bool bUniform, gmx_ddbox_t *ddbox, real *cell_f)
+{
+ gmx_domdec_comm_t *comm;
+ int nc, ns, s;
+ int *xmin, *xmax;
+ real range, pme_boundary;
+ int sh;
+
+ comm = dd->comm;
+ nc = dd->nc[ddpme->dim];
+ ns = ddpme->nslab;
+
+ if (!ddpme->dim_match)
+ {
+ /* PP decomposition is not along dim: the worst situation */
+ sh = ns/2;
+ }
+ else if (ns <= 3 || (bUniform && ns == nc))
+ {
+ /* The optimal situation */
+ sh = 1;
+ }
+ else
+ {
+ /* We need to check for all pme nodes which nodes they
+ * could possibly need to communicate with.
+ */
+ xmin = ddpme->pp_min;
+ xmax = ddpme->pp_max;
+ /* Allow for atoms to be maximally 2/3 times the cut-off
+ * out of their DD cell. This is a reasonable balance between
+ * between performance and support for most charge-group/cut-off
+ * combinations.
+ */
+ range = 2.0/3.0*comm->cutoff/ddbox->box_size[ddpme->dim];
+ /* Avoid extra communication when we are exactly at a boundary */
+ range *= 0.999;
+
+ sh = 1;
+ for (s = 0; s < ns; s++)
+ {
+ /* PME slab s spreads atoms between box frac. s/ns and (s+1)/ns */
+ pme_boundary = (real)s/ns;
+ while (sh+1 < ns &&
+ ((s-(sh+1) >= 0 &&
+ cell_f[xmax[s-(sh+1) ]+1] + range > pme_boundary) ||
+ (s-(sh+1) < 0 &&
+ cell_f[xmax[s-(sh+1)+ns]+1] - 1 + range > pme_boundary)))
+ {
+ sh++;
+ }
+ pme_boundary = (real)(s+1)/ns;
+ while (sh+1 < ns &&
+ ((s+(sh+1) < ns &&
+ cell_f[xmin[s+(sh+1) ] ] - range < pme_boundary) ||
+ (s+(sh+1) >= ns &&
+ cell_f[xmin[s+(sh+1)-ns] ] + 1 - range < pme_boundary)))
+ {
+ sh++;
+ }
+ }
+ }
+
+ ddpme->maxshift = sh;
+
+ if (debug)
+ {
+ fprintf(debug, "PME slab communication range for dim %d is %d\n",
+ ddpme->dim, ddpme->maxshift);
+ }
+}
+
+static void check_box_size(gmx_domdec_t *dd, gmx_ddbox_t *ddbox)
+{
+ int d, dim;
+
+ for (d = 0; d < dd->ndim; d++)
+ {
+ dim = dd->dim[d];
+ if (dim < ddbox->nboundeddim &&
+ ddbox->box_size[dim]*ddbox->skew_fac[dim] <
+ dd->nc[dim]*dd->comm->cellsize_limit*DD_CELL_MARGIN)
+ {
+ gmx_fatal(FARGS, "The %c-size of the box (%f) times the triclinic skew factor (%f) is smaller than the number of DD cells (%d) times the smallest allowed cell size (%f)\n",
+ dim2char(dim), ddbox->box_size[dim], ddbox->skew_fac[dim],
+ dd->nc[dim], dd->comm->cellsize_limit);
+ }
+ }
+}
+
+static void set_dd_cell_sizes_slb(gmx_domdec_t *dd, gmx_ddbox_t *ddbox,
+ gmx_bool bMaster, ivec npulse)
+{
+ gmx_domdec_comm_t *comm;
+ int d, j;
+ rvec cellsize_min;
+ real *cell_x, cell_dx, cellsize;
+
+ comm = dd->comm;
+
+ for (d = 0; d < DIM; d++)
+ {
+ cellsize_min[d] = ddbox->box_size[d]*ddbox->skew_fac[d];
+ npulse[d] = 1;
+ if (dd->nc[d] == 1 || comm->slb_frac[d] == NULL)
+ {
+ /* Uniform grid */
+ cell_dx = ddbox->box_size[d]/dd->nc[d];
+ if (bMaster)
+ {
+ for (j = 0; j < dd->nc[d]+1; j++)
+ {
+ dd->ma->cell_x[d][j] = ddbox->box0[d] + j*cell_dx;
+ }
+ }
+ else
+ {
+ comm->cell_x0[d] = ddbox->box0[d] + (dd->ci[d] )*cell_dx;
+ comm->cell_x1[d] = ddbox->box0[d] + (dd->ci[d]+1)*cell_dx;
+ }
+ cellsize = cell_dx*ddbox->skew_fac[d];
+ while (cellsize*npulse[d] < comm->cutoff && npulse[d] < dd->nc[d]-1)
+ {
+ npulse[d]++;
+ }
+ cellsize_min[d] = cellsize;
+ }
+ else
+ {
+ /* Statically load balanced grid */
+ /* Also when we are not doing a master distribution we determine
+ * all cell borders in a loop to obtain identical values
+ * to the master distribution case and to determine npulse.
+ */
+ if (bMaster)
+ {
+ cell_x = dd->ma->cell_x[d];
+ }
+ else
+ {
+ snew(cell_x, dd->nc[d]+1);
+ }
+ cell_x[0] = ddbox->box0[d];
+ for (j = 0; j < dd->nc[d]; j++)
+ {
+ cell_dx = ddbox->box_size[d]*comm->slb_frac[d][j];
+ cell_x[j+1] = cell_x[j] + cell_dx;
+ cellsize = cell_dx*ddbox->skew_fac[d];
+ while (cellsize*npulse[d] < comm->cutoff &&
+ npulse[d] < dd->nc[d]-1)
+ {
+ npulse[d]++;
+ }
+ cellsize_min[d] = min(cellsize_min[d], cellsize);
+ }
+ if (!bMaster)
+ {
+ comm->cell_x0[d] = cell_x[dd->ci[d]];
+ comm->cell_x1[d] = cell_x[dd->ci[d]+1];
+ sfree(cell_x);
+ }
+ }
+ /* The following limitation is to avoid that a cell would receive
+ * some of its own home charge groups back over the periodic boundary.
+ * Double charge groups cause trouble with the global indices.
+ */
+ if (d < ddbox->npbcdim &&
+ dd->nc[d] > 1 && npulse[d] >= dd->nc[d])
+ {
+ gmx_fatal_collective(FARGS, NULL, dd,
+ "The box size in direction %c (%f) times the triclinic skew factor (%f) is too small for a cut-off of %f with %d domain decomposition cells, use 1 or more than %d %s or increase the box size in this direction",
+ dim2char(d), ddbox->box_size[d], ddbox->skew_fac[d],
+ comm->cutoff,
+ dd->nc[d], dd->nc[d],
+ dd->nnodes > dd->nc[d] ? "cells" : "processors");
+ }
+ }
+
+ if (!comm->bDynLoadBal)
+ {
+ copy_rvec(cellsize_min, comm->cellsize_min);
+ }
+
+ for (d = 0; d < comm->npmedecompdim; d++)
+ {
+ set_pme_maxshift(dd, &comm->ddpme[d],
+ comm->slb_frac[dd->dim[d]] == NULL, ddbox,
+ comm->ddpme[d].slb_dim_f);
+ }
+}
+
+
+static void dd_cell_sizes_dlb_root_enforce_limits(gmx_domdec_t *dd,
+ int d, int dim, gmx_domdec_root_t *root,
+ gmx_ddbox_t *ddbox,
+ gmx_bool bUniform, gmx_large_int_t step, real cellsize_limit_f, int range[])
+{
+ gmx_domdec_comm_t *comm;
+ int ncd, i, j, nmin, nmin_old;
+ gmx_bool bLimLo, bLimHi;
+ real *cell_size;
+ real fac, halfway, cellsize_limit_f_i, region_size;
+ gmx_bool bPBC, bLastHi = FALSE;
+ int nrange[] = {range[0], range[1]};
+
+ region_size = root->cell_f[range[1]]-root->cell_f[range[0]];
+
+ comm = dd->comm;
+
+ ncd = dd->nc[dim];
+
+ bPBC = (dim < ddbox->npbcdim);
+
+ cell_size = root->buf_ncd;
+
+ if (debug)
+ {
+ fprintf(debug, "enforce_limits: %d %d\n", range[0], range[1]);
+ }
+
+ /* First we need to check if the scaling does not make cells
+ * smaller than the smallest allowed size.
+ * We need to do this iteratively, since if a cell is too small,
+ * it needs to be enlarged, which makes all the other cells smaller,
+ * which could in turn make another cell smaller than allowed.
+ */
+ for (i = range[0]; i < range[1]; i++)
+ {
+ root->bCellMin[i] = FALSE;
+ }
+ nmin = 0;
+ do
+ {
+ nmin_old = nmin;
+ /* We need the total for normalization */
+ fac = 0;
+ for (i = range[0]; i < range[1]; i++)
+ {
+ if (root->bCellMin[i] == FALSE)
+ {
+ fac += cell_size[i];
+ }
+ }
+ fac = ( region_size - nmin*cellsize_limit_f)/fac; /* substracting cells already set to cellsize_limit_f */
+ /* Determine the cell boundaries */
+ for (i = range[0]; i < range[1]; i++)
+ {
+ if (root->bCellMin[i] == FALSE)
+ {
+ cell_size[i] *= fac;
+ if (!bPBC && (i == 0 || i == dd->nc[dim] -1))
+ {
+ cellsize_limit_f_i = 0;
+ }
+ else
+ {
+ cellsize_limit_f_i = cellsize_limit_f;
+ }
+ if (cell_size[i] < cellsize_limit_f_i)
+ {
+ root->bCellMin[i] = TRUE;
+ cell_size[i] = cellsize_limit_f_i;
+ nmin++;
+ }
+ }
+ root->cell_f[i+1] = root->cell_f[i] + cell_size[i];
+ }
+ }
+ while (nmin > nmin_old);
+
+ i = range[1]-1;
+ cell_size[i] = root->cell_f[i+1] - root->cell_f[i];
+ /* For this check we should not use DD_CELL_MARGIN,
+ * but a slightly smaller factor,
+ * since rounding could get use below the limit.
+ */
+ if (bPBC && cell_size[i] < cellsize_limit_f*DD_CELL_MARGIN2/DD_CELL_MARGIN)
+ {
+ char buf[22];
+ gmx_fatal(FARGS, "Step %s: the dynamic load balancing could not balance dimension %c: box size %f, triclinic skew factor %f, #cells %d, minimum cell size %f\n",
+ gmx_step_str(step, buf),
+ dim2char(dim), ddbox->box_size[dim], ddbox->skew_fac[dim],
+ ncd, comm->cellsize_min[dim]);
+ }
+
+ root->bLimited = (nmin > 0) || (range[0] > 0) || (range[1] < ncd);
+
+ if (!bUniform)
+ {
+ /* Check if the boundary did not displace more than halfway
+ * each of the cells it bounds, as this could cause problems,
+ * especially when the differences between cell sizes are large.
+ * If changes are applied, they will not make cells smaller
+ * than the cut-off, as we check all the boundaries which
+ * might be affected by a change and if the old state was ok,
+ * the cells will at most be shrunk back to their old size.
+ */
+ for (i = range[0]+1; i < range[1]; i++)
+ {
+ halfway = 0.5*(root->old_cell_f[i] + root->old_cell_f[i-1]);
+ if (root->cell_f[i] < halfway)
+ {
+ root->cell_f[i] = halfway;
+ /* Check if the change also causes shifts of the next boundaries */
+ for (j = i+1; j < range[1]; j++)
+ {
+ if (root->cell_f[j] < root->cell_f[j-1] + cellsize_limit_f)
+ {
+ root->cell_f[j] = root->cell_f[j-1] + cellsize_limit_f;
+ }
+ }
+ }
+ halfway = 0.5*(root->old_cell_f[i] + root->old_cell_f[i+1]);
+ if (root->cell_f[i] > halfway)
+ {
+ root->cell_f[i] = halfway;
+ /* Check if the change also causes shifts of the next boundaries */
+ for (j = i-1; j >= range[0]+1; j--)
+ {
+ if (root->cell_f[j] > root->cell_f[j+1] - cellsize_limit_f)
+ {
+ root->cell_f[j] = root->cell_f[j+1] - cellsize_limit_f;
+ }
+ }
+ }
+ }
+ }
+
+ /* nrange is defined as [lower, upper) range for new call to enforce_limits */
+ /* find highest violation of LimLo (a) and the following violation of LimHi (thus the lowest following) (b)
+ * then call enforce_limits for (oldb,a), (a,b). In the next step: (b,nexta). oldb and nexta can be the boundaries.
+ * for a and b nrange is used */
+ if (d > 0)
+ {
+ /* Take care of the staggering of the cell boundaries */
+ if (bUniform)
+ {
+ for (i = range[0]; i < range[1]; i++)
+ {
+ root->cell_f_max0[i] = root->cell_f[i];
+ root->cell_f_min1[i] = root->cell_f[i+1];
+ }
+ }
+ else
+ {
+ for (i = range[0]+1; i < range[1]; i++)
+ {
+ bLimLo = (root->cell_f[i] < root->bound_min[i]);
+ bLimHi = (root->cell_f[i] > root->bound_max[i]);
+ if (bLimLo && bLimHi)
+ {
+ /* Both limits violated, try the best we can */
+ /* For this case we split the original range (range) in two parts and care about the other limitiations in the next iteration. */
+ root->cell_f[i] = 0.5*(root->bound_min[i] + root->bound_max[i]);
+ nrange[0] = range[0];
+ nrange[1] = i;
+ dd_cell_sizes_dlb_root_enforce_limits(dd, d, dim, root, ddbox, bUniform, step, cellsize_limit_f, nrange);
+
+ nrange[0] = i;
+ nrange[1] = range[1];
+ dd_cell_sizes_dlb_root_enforce_limits(dd, d, dim, root, ddbox, bUniform, step, cellsize_limit_f, nrange);
+
+ return;
+ }
+ else if (bLimLo)
+ {
+ /* root->cell_f[i] = root->bound_min[i]; */
+ nrange[1] = i; /* only store violation location. There could be a LimLo violation following with an higher index */
+ bLastHi = FALSE;
+ }
+ else if (bLimHi && !bLastHi)
+ {
+ bLastHi = TRUE;
+ if (nrange[1] < range[1]) /* found a LimLo before */
+ {
+ root->cell_f[nrange[1]] = root->bound_min[nrange[1]];
+ dd_cell_sizes_dlb_root_enforce_limits(dd, d, dim, root, ddbox, bUniform, step, cellsize_limit_f, nrange);
+ nrange[0] = nrange[1];
+ }
+ root->cell_f[i] = root->bound_max[i];
+ nrange[1] = i;
+ dd_cell_sizes_dlb_root_enforce_limits(dd, d, dim, root, ddbox, bUniform, step, cellsize_limit_f, nrange);
+ nrange[0] = i;
+ nrange[1] = range[1];
+ }
+ }
+ if (nrange[1] < range[1]) /* found last a LimLo */
+ {
+ root->cell_f[nrange[1]] = root->bound_min[nrange[1]];
+ dd_cell_sizes_dlb_root_enforce_limits(dd, d, dim, root, ddbox, bUniform, step, cellsize_limit_f, nrange);
+ nrange[0] = nrange[1];
+ nrange[1] = range[1];
+ dd_cell_sizes_dlb_root_enforce_limits(dd, d, dim, root, ddbox, bUniform, step, cellsize_limit_f, nrange);
+ }
+ else if (nrange[0] > range[0]) /* found at least one LimHi */
+ {
+ dd_cell_sizes_dlb_root_enforce_limits(dd, d, dim, root, ddbox, bUniform, step, cellsize_limit_f, nrange);
+ }
+ }
+ }
+}
+
+
+static void set_dd_cell_sizes_dlb_root(gmx_domdec_t *dd,
+ int d, int dim, gmx_domdec_root_t *root,
+ gmx_ddbox_t *ddbox, gmx_bool bDynamicBox,
+ gmx_bool bUniform, gmx_large_int_t step)
+{
+ gmx_domdec_comm_t *comm;
+ int ncd, d1, i, j, pos;
+ real *cell_size;
+ real load_aver, load_i, imbalance, change, change_max, sc;
+ real cellsize_limit_f, dist_min_f, dist_min_f_hard, space;
+ real change_limit;
+ real relax = 0.5;
+ gmx_bool bPBC;
+ int range[] = { 0, 0 };
+
+ comm = dd->comm;
+
+ /* Convert the maximum change from the input percentage to a fraction */
+ change_limit = comm->dlb_scale_lim*0.01;
+
+ ncd = dd->nc[dim];
+
+ bPBC = (dim < ddbox->npbcdim);
+
+ cell_size = root->buf_ncd;
+
+ /* Store the original boundaries */
+ for (i = 0; i < ncd+1; i++)
+ {
+ root->old_cell_f[i] = root->cell_f[i];
+ }
+ if (bUniform)
+ {
+ for (i = 0; i < ncd; i++)
+ {
+ cell_size[i] = 1.0/ncd;
+ }
+ }
+ else if (dd_load_count(comm))
+ {
+ load_aver = comm->load[d].sum_m/ncd;
+ change_max = 0;
+ for (i = 0; i < ncd; i++)
+ {
+ /* Determine the relative imbalance of cell i */
+ load_i = comm->load[d].load[i*comm->load[d].nload+2];
+ imbalance = (load_i - load_aver)/(load_aver > 0 ? load_aver : 1);
+ /* Determine the change of the cell size using underrelaxation */
+ change = -relax*imbalance;
+ change_max = max(change_max, max(change, -change));
+ }
+ /* Limit the amount of scaling.
+ * We need to use the same rescaling for all cells in one row,
+ * otherwise the load balancing might not converge.
+ */
+ sc = relax;
+ if (change_max > change_limit)
+ {
+ sc *= change_limit/change_max;
+ }
+ for (i = 0; i < ncd; i++)
+ {
+ /* Determine the relative imbalance of cell i */
+ load_i = comm->load[d].load[i*comm->load[d].nload+2];
+ imbalance = (load_i - load_aver)/(load_aver > 0 ? load_aver : 1);
+ /* Determine the change of the cell size using underrelaxation */
+ change = -sc*imbalance;
+ cell_size[i] = (root->cell_f[i+1]-root->cell_f[i])*(1 + change);
+ }
+ }
+
+ cellsize_limit_f = cellsize_min_dlb(comm, d, dim)/ddbox->box_size[dim];
+ cellsize_limit_f *= DD_CELL_MARGIN;
+ dist_min_f_hard = grid_jump_limit(comm, comm->cutoff, d)/ddbox->box_size[dim];
+ dist_min_f = dist_min_f_hard * DD_CELL_MARGIN;
+ if (ddbox->tric_dir[dim])
+ {
+ cellsize_limit_f /= ddbox->skew_fac[dim];
+ dist_min_f /= ddbox->skew_fac[dim];
+ }
+ if (bDynamicBox && d > 0)
+ {
+ dist_min_f *= DD_PRES_SCALE_MARGIN;
+ }
+ if (d > 0 && !bUniform)
+ {
+ /* Make sure that the grid is not shifted too much */
+ for (i = 1; i < ncd; i++)
+ {
+ if (root->cell_f_min1[i] - root->cell_f_max0[i-1] < 2 * dist_min_f_hard)
+ {
+ gmx_incons("Inconsistent DD boundary staggering limits!");
+ }
+ root->bound_min[i] = root->cell_f_max0[i-1] + dist_min_f;
+ space = root->cell_f[i] - (root->cell_f_max0[i-1] + dist_min_f);
+ if (space > 0)
+ {
+ root->bound_min[i] += 0.5*space;
+ }
+ root->bound_max[i] = root->cell_f_min1[i] - dist_min_f;
+ space = root->cell_f[i] - (root->cell_f_min1[i] - dist_min_f);
+ if (space < 0)
+ {
+ root->bound_max[i] += 0.5*space;
+ }
+ if (debug)
+ {
+ fprintf(debug,
+ "dim %d boundary %d %.3f < %.3f < %.3f < %.3f < %.3f\n",
+ d, i,
+ root->cell_f_max0[i-1] + dist_min_f,
+ root->bound_min[i], root->cell_f[i], root->bound_max[i],
+ root->cell_f_min1[i] - dist_min_f);
+ }
+ }
+ }
+ range[1] = ncd;
+ root->cell_f[0] = 0;
+ root->cell_f[ncd] = 1;
+ dd_cell_sizes_dlb_root_enforce_limits(dd, d, dim, root, ddbox, bUniform, step, cellsize_limit_f, range);
+
+
+ /* After the checks above, the cells should obey the cut-off
+ * restrictions, but it does not hurt to check.
+ */
+ for (i = 0; i < ncd; i++)
+ {
+ if (debug)
+ {
+ fprintf(debug, "Relative bounds dim %d cell %d: %f %f\n",
+ dim, i, root->cell_f[i], root->cell_f[i+1]);
+ }
+
+ if ((bPBC || (i != 0 && i != dd->nc[dim]-1)) &&
+ root->cell_f[i+1] - root->cell_f[i] <
+ cellsize_limit_f/DD_CELL_MARGIN)
+ {
+ char buf[22];
+ fprintf(stderr,
+ "\nWARNING step %s: direction %c, cell %d too small: %f\n",
+ gmx_step_str(step, buf), dim2char(dim), i,
+ (root->cell_f[i+1] - root->cell_f[i])
+ *ddbox->box_size[dim]*ddbox->skew_fac[dim]);
+ }
+ }
+
+ pos = ncd + 1;
+ /* Store the cell boundaries of the lower dimensions at the end */
+ for (d1 = 0; d1 < d; d1++)
+ {
+ root->cell_f[pos++] = comm->cell_f0[d1];
+ root->cell_f[pos++] = comm->cell_f1[d1];
+ }
+
+ if (d < comm->npmedecompdim)
+ {
+ /* The master determines the maximum shift for
+ * the coordinate communication between separate PME nodes.
+ */
+ set_pme_maxshift(dd, &comm->ddpme[d], bUniform, ddbox, root->cell_f);
+ }
+ root->cell_f[pos++] = comm->ddpme[0].maxshift;
+ if (d >= 1)
+ {
+ root->cell_f[pos++] = comm->ddpme[1].maxshift;
+ }
+}
+
+static void relative_to_absolute_cell_bounds(gmx_domdec_t *dd,
+ gmx_ddbox_t *ddbox, int dimind)
+{
+ gmx_domdec_comm_t *comm;
+ int dim;
+
+ comm = dd->comm;
+
+ /* Set the cell dimensions */
+ dim = dd->dim[dimind];
+ comm->cell_x0[dim] = comm->cell_f0[dimind]*ddbox->box_size[dim];
+ comm->cell_x1[dim] = comm->cell_f1[dimind]*ddbox->box_size[dim];
+ if (dim >= ddbox->nboundeddim)
+ {
+ comm->cell_x0[dim] += ddbox->box0[dim];
+ comm->cell_x1[dim] += ddbox->box0[dim];
+ }
+}
+
+static void distribute_dd_cell_sizes_dlb(gmx_domdec_t *dd,
+ int d, int dim, real *cell_f_row,
+ gmx_ddbox_t *ddbox)
+{
+ gmx_domdec_comm_t *comm;
+ int d1, dim1, pos;
+
+ comm = dd->comm;
+
+#ifdef GMX_MPI
+ /* Each node would only need to know two fractions,
+ * but it is probably cheaper to broadcast the whole array.
+ */
+ MPI_Bcast(cell_f_row, DD_CELL_F_SIZE(dd, d)*sizeof(real), MPI_BYTE,
+ 0, comm->mpi_comm_load[d]);
+#endif
+ /* Copy the fractions for this dimension from the buffer */
+ comm->cell_f0[d] = cell_f_row[dd->ci[dim] ];
+ comm->cell_f1[d] = cell_f_row[dd->ci[dim]+1];
+ /* The whole array was communicated, so set the buffer position */
+ pos = dd->nc[dim] + 1;
+ for (d1 = 0; d1 <= d; d1++)
+ {
+ if (d1 < d)
+ {
+ /* Copy the cell fractions of the lower dimensions */
+ comm->cell_f0[d1] = cell_f_row[pos++];
+ comm->cell_f1[d1] = cell_f_row[pos++];
+ }
+ relative_to_absolute_cell_bounds(dd, ddbox, d1);
+ }
+ /* Convert the communicated shift from float to int */
+ comm->ddpme[0].maxshift = (int)(cell_f_row[pos++] + 0.5);
+ if (d >= 1)
+ {
+ comm->ddpme[1].maxshift = (int)(cell_f_row[pos++] + 0.5);
+ }
+}
+
+static void set_dd_cell_sizes_dlb_change(gmx_domdec_t *dd,
+ gmx_ddbox_t *ddbox, gmx_bool bDynamicBox,
+ gmx_bool bUniform, gmx_large_int_t step)
+{
+ gmx_domdec_comm_t *comm;
+ int d, dim, d1;
+ gmx_bool bRowMember, bRowRoot;
+ real *cell_f_row;
+
+ comm = dd->comm;
+
+ for (d = 0; d < dd->ndim; d++)
+ {
+ dim = dd->dim[d];
+ bRowMember = TRUE;
+ bRowRoot = TRUE;
+ for (d1 = d; d1 < dd->ndim; d1++)
+ {
+ if (dd->ci[dd->dim[d1]] > 0)
+ {
+ if (d1 > d)
+ {
+ bRowMember = FALSE;
+ }
+ bRowRoot = FALSE;
+ }
+ }
+ if (bRowMember)
+ {
+ if (bRowRoot)
+ {
+ set_dd_cell_sizes_dlb_root(dd, d, dim, comm->root[d],
+ ddbox, bDynamicBox, bUniform, step);
+ cell_f_row = comm->root[d]->cell_f;
+ }
+ else
+ {
+ cell_f_row = comm->cell_f_row;
+ }
+ distribute_dd_cell_sizes_dlb(dd, d, dim, cell_f_row, ddbox);
+ }
+ }
+}
+
+static void set_dd_cell_sizes_dlb_nochange(gmx_domdec_t *dd, gmx_ddbox_t *ddbox)
+{
+ int d;
+
+ /* This function assumes the box is static and should therefore
+ * not be called when the box has changed since the last
+ * call to dd_partition_system.
+ */
+ for (d = 0; d < dd->ndim; d++)
+ {
+ relative_to_absolute_cell_bounds(dd, ddbox, d);
+ }
+}
+
+
+
+static void set_dd_cell_sizes_dlb(gmx_domdec_t *dd,
+ gmx_ddbox_t *ddbox, gmx_bool bDynamicBox,
+ gmx_bool bUniform, gmx_bool bDoDLB, gmx_large_int_t step,
+ gmx_wallcycle_t wcycle)
+{
+ gmx_domdec_comm_t *comm;
+ int dim;
+
+ comm = dd->comm;
+
+ if (bDoDLB)
+ {
+ wallcycle_start(wcycle, ewcDDCOMMBOUND);
+ set_dd_cell_sizes_dlb_change(dd, ddbox, bDynamicBox, bUniform, step);
+ wallcycle_stop(wcycle, ewcDDCOMMBOUND);
+ }
+ else if (bDynamicBox)
+ {
+ set_dd_cell_sizes_dlb_nochange(dd, ddbox);
+ }
+
+ /* Set the dimensions for which no DD is used */
+ for (dim = 0; dim < DIM; dim++)
+ {
+ if (dd->nc[dim] == 1)
+ {
+ comm->cell_x0[dim] = 0;
+ comm->cell_x1[dim] = ddbox->box_size[dim];
+ if (dim >= ddbox->nboundeddim)
+ {
+ comm->cell_x0[dim] += ddbox->box0[dim];
+ comm->cell_x1[dim] += ddbox->box0[dim];
+ }
+ }
+ }
+}
+
+static void realloc_comm_ind(gmx_domdec_t *dd, ivec npulse)
+{
+ int d, np, i;
+ gmx_domdec_comm_dim_t *cd;
+
+ for (d = 0; d < dd->ndim; d++)
+ {
+ cd = &dd->comm->cd[d];
+ np = npulse[dd->dim[d]];
+ if (np > cd->np_nalloc)
+ {
+ if (debug)
+ {
+ fprintf(debug, "(Re)allocing cd for %c to %d pulses\n",
+ dim2char(dd->dim[d]), np);
+ }
+ if (DDMASTER(dd) && cd->np_nalloc > 0)
+ {
+ fprintf(stderr, "\nIncreasing the number of cell to communicate in dimension %c to %d for the first time\n", dim2char(dd->dim[d]), np);
+ }
+ srenew(cd->ind, np);
+ for (i = cd->np_nalloc; i < np; i++)
+ {
+ cd->ind[i].index = NULL;
+ cd->ind[i].nalloc = 0;
+ }
+ cd->np_nalloc = np;
+ }
+ cd->np = np;
+ }
+}
+
+
+static void set_dd_cell_sizes(gmx_domdec_t *dd,
+ gmx_ddbox_t *ddbox, gmx_bool bDynamicBox,
+ gmx_bool bUniform, gmx_bool bDoDLB, gmx_large_int_t step,
+ gmx_wallcycle_t wcycle)
+{
+ gmx_domdec_comm_t *comm;
+ int d;
+ ivec npulse;
+
+ comm = dd->comm;
+
+ /* Copy the old cell boundaries for the cg displacement check */
+ copy_rvec(comm->cell_x0, comm->old_cell_x0);
+ copy_rvec(comm->cell_x1, comm->old_cell_x1);
+
+ if (comm->bDynLoadBal)
+ {
+ if (DDMASTER(dd))
+ {
+ check_box_size(dd, ddbox);
+ }
+ set_dd_cell_sizes_dlb(dd, ddbox, bDynamicBox, bUniform, bDoDLB, step, wcycle);
+ }
+ else
+ {
+ set_dd_cell_sizes_slb(dd, ddbox, FALSE, npulse);
+ realloc_comm_ind(dd, npulse);
+ }
+
+ if (debug)
+ {
+ for (d = 0; d < DIM; d++)
+ {
+ fprintf(debug, "cell_x[%d] %f - %f skew_fac %f\n",
+ d, comm->cell_x0[d], comm->cell_x1[d], ddbox->skew_fac[d]);
+ }
+ }
+}
+
+static void comm_dd_ns_cell_sizes(gmx_domdec_t *dd,
+ gmx_ddbox_t *ddbox,
+ rvec cell_ns_x0, rvec cell_ns_x1,
+ gmx_large_int_t step)
+{
+ gmx_domdec_comm_t *comm;
+ int dim_ind, dim;
+
+ comm = dd->comm;
+
+ for (dim_ind = 0; dim_ind < dd->ndim; dim_ind++)
+ {
+ dim = dd->dim[dim_ind];
+
+ /* Without PBC we don't have restrictions on the outer cells */
+ if (!(dim >= ddbox->npbcdim &&
+ (dd->ci[dim] == 0 || dd->ci[dim] == dd->nc[dim] - 1)) &&
+ comm->bDynLoadBal &&
+ (comm->cell_x1[dim] - comm->cell_x0[dim])*ddbox->skew_fac[dim] <
+ comm->cellsize_min[dim])
+ {
+ char buf[22];
+ gmx_fatal(FARGS, "Step %s: The %c-size (%f) times the triclinic skew factor (%f) is smaller than the smallest allowed cell size (%f) for domain decomposition grid cell %d %d %d",
+ gmx_step_str(step, buf), dim2char(dim),
+ comm->cell_x1[dim] - comm->cell_x0[dim],
+ ddbox->skew_fac[dim],
+ dd->comm->cellsize_min[dim],
+ dd->ci[XX], dd->ci[YY], dd->ci[ZZ]);
+ }
+ }
+
+ if ((dd->bGridJump && dd->ndim > 1) || ddbox->nboundeddim < DIM)
+ {
+ /* Communicate the boundaries and update cell_ns_x0/1 */
+ dd_move_cellx(dd, ddbox, cell_ns_x0, cell_ns_x1);
+ if (dd->bGridJump && dd->ndim > 1)
+ {
+ check_grid_jump(step, dd, dd->comm->cutoff, ddbox, TRUE);
+ }
+ }
+}
+
+static void make_tric_corr_matrix(int npbcdim, matrix box, matrix tcm)
+{
+ if (YY < npbcdim)
+ {
+ tcm[YY][XX] = -box[YY][XX]/box[YY][YY];
+ }
+ else
+ {
+ tcm[YY][XX] = 0;
+ }
+ if (ZZ < npbcdim)
+ {
+ tcm[ZZ][XX] = -(box[ZZ][YY]*tcm[YY][XX] + box[ZZ][XX])/box[ZZ][ZZ];
+ tcm[ZZ][YY] = -box[ZZ][YY]/box[ZZ][ZZ];
+ }
+ else
+ {
+ tcm[ZZ][XX] = 0;
+ tcm[ZZ][YY] = 0;
+ }
+}
+
+static void check_screw_box(matrix box)
+{
+ /* Mathematical limitation */
+ if (box[YY][XX] != 0 || box[ZZ][XX] != 0)
+ {
+ gmx_fatal(FARGS, "With screw pbc the unit cell can not have non-zero off-diagonal x-components");
+ }
+
+ /* Limitation due to the asymmetry of the eighth shell method */
+ if (box[ZZ][YY] != 0)
+ {
+ gmx_fatal(FARGS, "pbc=screw with non-zero box_zy is not supported");
+ }
+}
+
+static void distribute_cg(FILE *fplog, gmx_large_int_t step,
+ matrix box, ivec tric_dir, t_block *cgs, rvec pos[],
+ gmx_domdec_t *dd)
+{
+ gmx_domdec_master_t *ma;
+ int **tmp_ind = NULL, *tmp_nalloc = NULL;
+ int i, icg, j, k, k0, k1, d, npbcdim;
+ matrix tcm;
+ rvec box_size, cg_cm;
+ ivec ind;
+ real nrcg, inv_ncg, pos_d;
+ atom_id *cgindex;
+ gmx_bool bUnbounded, bScrew;
+
+ ma = dd->ma;
+
+ if (tmp_ind == NULL)
+ {
+ snew(tmp_nalloc, dd->nnodes);
+ snew(tmp_ind, dd->nnodes);
+ for (i = 0; i < dd->nnodes; i++)
+ {
+ tmp_nalloc[i] = over_alloc_large(cgs->nr/dd->nnodes+1);
+ snew(tmp_ind[i], tmp_nalloc[i]);
+ }
+ }
+
+ /* Clear the count */
+ for (i = 0; i < dd->nnodes; i++)
+ {
+ ma->ncg[i] = 0;
+ ma->nat[i] = 0;
+ }
+
+ make_tric_corr_matrix(dd->npbcdim, box, tcm);
+
+ cgindex = cgs->index;
+
+ /* Compute the center of geometry for all charge groups */
+ for (icg = 0; icg < cgs->nr; icg++)
+ {
+ k0 = cgindex[icg];
+ k1 = cgindex[icg+1];
+ nrcg = k1 - k0;
+ if (nrcg == 1)
+ {
+ copy_rvec(pos[k0], cg_cm);
+ }
+ else
+ {
+ inv_ncg = 1.0/nrcg;
+
+ clear_rvec(cg_cm);
+ for (k = k0; (k < k1); k++)
+ {
+ rvec_inc(cg_cm, pos[k]);
+ }
+ for (d = 0; (d < DIM); d++)
+ {
+ cg_cm[d] *= inv_ncg;
+ }
+ }
+ /* Put the charge group in the box and determine the cell index */
+ for (d = DIM-1; d >= 0; d--)
+ {
+ pos_d = cg_cm[d];
+ if (d < dd->npbcdim)
+ {
+ bScrew = (dd->bScrewPBC && d == XX);
+ if (tric_dir[d] && dd->nc[d] > 1)
+ {
+ /* Use triclinic coordintates for this dimension */
+ for (j = d+1; j < DIM; j++)
+ {
+ pos_d += cg_cm[j]*tcm[j][d];
+ }
+ }
+ while (pos_d >= box[d][d])
+ {
+ pos_d -= box[d][d];
+ rvec_dec(cg_cm, box[d]);
+ if (bScrew)
+ {
+ cg_cm[YY] = box[YY][YY] - cg_cm[YY];
+ cg_cm[ZZ] = box[ZZ][ZZ] - cg_cm[ZZ];
+ }
+ for (k = k0; (k < k1); k++)
+ {
+ rvec_dec(pos[k], box[d]);
+ if (bScrew)
+ {
+ pos[k][YY] = box[YY][YY] - pos[k][YY];
+ pos[k][ZZ] = box[ZZ][ZZ] - pos[k][ZZ];
+ }
+ }
+ }
+ while (pos_d < 0)
+ {
+ pos_d += box[d][d];
+ rvec_inc(cg_cm, box[d]);
+ if (bScrew)
+ {
+ cg_cm[YY] = box[YY][YY] - cg_cm[YY];
+ cg_cm[ZZ] = box[ZZ][ZZ] - cg_cm[ZZ];
+ }
+ for (k = k0; (k < k1); k++)
+ {
+ rvec_inc(pos[k], box[d]);
+ if (bScrew)
+ {
+ pos[k][YY] = box[YY][YY] - pos[k][YY];
+ pos[k][ZZ] = box[ZZ][ZZ] - pos[k][ZZ];
+ }
+ }
+ }
+ }
+ /* This could be done more efficiently */
+ ind[d] = 0;
+ while (ind[d]+1 < dd->nc[d] && pos_d >= ma->cell_x[d][ind[d]+1])
+ {
+ ind[d]++;
+ }
+ }
+ i = dd_index(dd->nc, ind);
+ if (ma->ncg[i] == tmp_nalloc[i])
+ {
+ tmp_nalloc[i] = over_alloc_large(ma->ncg[i]+1);
+ srenew(tmp_ind[i], tmp_nalloc[i]);
+ }
+ tmp_ind[i][ma->ncg[i]] = icg;
+ ma->ncg[i]++;
+ ma->nat[i] += cgindex[icg+1] - cgindex[icg];
+ }
+
+ k1 = 0;
+ for (i = 0; i < dd->nnodes; i++)
+ {
+ ma->index[i] = k1;
+ for (k = 0; k < ma->ncg[i]; k++)
+ {
+ ma->cg[k1++] = tmp_ind[i][k];
+ }
+ }
+ ma->index[dd->nnodes] = k1;
+
+ for (i = 0; i < dd->nnodes; i++)
+ {
+ sfree(tmp_ind[i]);
+ }
+ sfree(tmp_ind);
+ sfree(tmp_nalloc);
+
+ if (fplog)
+ {
+ char buf[22];
+ fprintf(fplog, "Charge group distribution at step %s:",
+ gmx_step_str(step, buf));
+ for (i = 0; i < dd->nnodes; i++)
+ {
+ fprintf(fplog, " %d", ma->ncg[i]);
+ }
+ fprintf(fplog, "\n");
+ }
+}
+
+static void get_cg_distribution(FILE *fplog, gmx_large_int_t step, gmx_domdec_t *dd,
+ t_block *cgs, matrix box, gmx_ddbox_t *ddbox,
+ rvec pos[])
+{
+ gmx_domdec_master_t *ma = NULL;
+ ivec npulse;
+ int i, cg_gl;
+ int *ibuf, buf2[2] = { 0, 0 };
+ gmx_bool bMaster = DDMASTER(dd);
+ if (bMaster)
+ {
+ ma = dd->ma;
+
+ if (dd->bScrewPBC)
+ {
+ check_screw_box(box);
+ }
+
+ set_dd_cell_sizes_slb(dd, ddbox, TRUE, npulse);
+
+ distribute_cg(fplog, step, box, ddbox->tric_dir, cgs, pos, dd);
+ for (i = 0; i < dd->nnodes; i++)
+ {
+ ma->ibuf[2*i] = ma->ncg[i];
+ ma->ibuf[2*i+1] = ma->nat[i];
+ }
+ ibuf = ma->ibuf;
+ }
+ else
+ {
+ ibuf = NULL;
+ }
+ dd_scatter(dd, 2*sizeof(int), ibuf, buf2);
+
+ dd->ncg_home = buf2[0];
+ dd->nat_home = buf2[1];
+ dd->ncg_tot = dd->ncg_home;
+ dd->nat_tot = dd->nat_home;
+ if (dd->ncg_home > dd->cg_nalloc || dd->cg_nalloc == 0)
+ {
+ dd->cg_nalloc = over_alloc_dd(dd->ncg_home);
+ srenew(dd->index_gl, dd->cg_nalloc);
+ srenew(dd->cgindex, dd->cg_nalloc+1);
+ }
+ if (bMaster)
+ {
+ for (i = 0; i < dd->nnodes; i++)
+ {
+ ma->ibuf[i] = ma->ncg[i]*sizeof(int);
+ ma->ibuf[dd->nnodes+i] = ma->index[i]*sizeof(int);
+ }
+ }
+
+ dd_scatterv(dd,
+ DDMASTER(dd) ? ma->ibuf : NULL,
+ DDMASTER(dd) ? ma->ibuf+dd->nnodes : NULL,
+ DDMASTER(dd) ? ma->cg : NULL,
+ dd->ncg_home*sizeof(int), dd->index_gl);
+
+ /* Determine the home charge group sizes */
+ dd->cgindex[0] = 0;
+ for (i = 0; i < dd->ncg_home; i++)
+ {
+ cg_gl = dd->index_gl[i];
+ dd->cgindex[i+1] =
+ dd->cgindex[i] + cgs->index[cg_gl+1] - cgs->index[cg_gl];
+ }
+
+ if (debug)
+ {
+ fprintf(debug, "Home charge groups:\n");
+ for (i = 0; i < dd->ncg_home; i++)
+ {
+ fprintf(debug, " %d", dd->index_gl[i]);
+ if (i % 10 == 9)
+ {
+ fprintf(debug, "\n");
+ }
+ }
+ fprintf(debug, "\n");
+ }
+}
+
+static int compact_and_copy_vec_at(int ncg, int *move,
+ int *cgindex,
+ int nvec, int vec,
+ rvec *src, gmx_domdec_comm_t *comm,
+ gmx_bool bCompact)
+{
+ int m, icg, i, i0, i1, nrcg;
+ int home_pos;
+ int pos_vec[DIM*2];
+
+ home_pos = 0;
+
+ for (m = 0; m < DIM*2; m++)
+ {
+ pos_vec[m] = 0;
+ }
+
+ i0 = 0;
+ for (icg = 0; icg < ncg; icg++)
+ {
+ i1 = cgindex[icg+1];
+ m = move[icg];
+ if (m == -1)
+ {
+ if (bCompact)
+ {
+ /* Compact the home array in place */
+ for (i = i0; i < i1; i++)
+ {
+ copy_rvec(src[i], src[home_pos++]);
+ }
+ }
+ }
+ else
+ {
+ /* Copy to the communication buffer */
+ nrcg = i1 - i0;
+ pos_vec[m] += 1 + vec*nrcg;
+ for (i = i0; i < i1; i++)
+ {
+ copy_rvec(src[i], comm->cgcm_state[m][pos_vec[m]++]);
+ }
+ pos_vec[m] += (nvec - vec - 1)*nrcg;
+ }
+ if (!bCompact)
+ {
+ home_pos += i1 - i0;
+ }
+ i0 = i1;
+ }
+
+ return home_pos;
+}
+
+static int compact_and_copy_vec_cg(int ncg, int *move,
+ int *cgindex,
+ int nvec, rvec *src, gmx_domdec_comm_t *comm,
+ gmx_bool bCompact)
+{
+ int m, icg, i0, i1, nrcg;
+ int home_pos;
+ int pos_vec[DIM*2];
+
+ home_pos = 0;
+
+ for (m = 0; m < DIM*2; m++)
+ {
+ pos_vec[m] = 0;
+ }
+
+ i0 = 0;
+ for (icg = 0; icg < ncg; icg++)
+ {
+ i1 = cgindex[icg+1];
+ m = move[icg];
+ if (m == -1)
+ {
+ if (bCompact)
+ {
+ /* Compact the home array in place */
+ copy_rvec(src[icg], src[home_pos++]);
+ }
+ }
+ else
+ {
+ nrcg = i1 - i0;
+ /* Copy to the communication buffer */
+ copy_rvec(src[icg], comm->cgcm_state[m][pos_vec[m]]);
+ pos_vec[m] += 1 + nrcg*nvec;
+ }
+ i0 = i1;
+ }
+ if (!bCompact)
+ {
+ home_pos = ncg;
+ }
+
+ return home_pos;
+}
+
+static int compact_ind(int ncg, int *move,
+ int *index_gl, int *cgindex,
+ int *gatindex,
+ gmx_ga2la_t ga2la, char *bLocalCG,
+ int *cginfo)
+{
+ int cg, nat, a0, a1, a, a_gl;
+ int home_pos;
+
+ home_pos = 0;
+ nat = 0;
+ for (cg = 0; cg < ncg; cg++)
+ {
+ a0 = cgindex[cg];
+ a1 = cgindex[cg+1];
+ if (move[cg] == -1)
+ {
+ /* Compact the home arrays in place.
+ * Anything that can be done here avoids access to global arrays.
+ */
+ cgindex[home_pos] = nat;
+ for (a = a0; a < a1; a++)
+ {
+ a_gl = gatindex[a];
+ gatindex[nat] = a_gl;
+ /* The cell number stays 0, so we don't need to set it */
+ ga2la_change_la(ga2la, a_gl, nat);
+ nat++;
+ }
+ index_gl[home_pos] = index_gl[cg];
+ cginfo[home_pos] = cginfo[cg];
+ /* The charge group remains local, so bLocalCG does not change */
+ home_pos++;
+ }
+ else
+ {
+ /* Clear the global indices */
+ for (a = a0; a < a1; a++)
+ {
+ ga2la_del(ga2la, gatindex[a]);
+ }
+ if (bLocalCG)
+ {
+ bLocalCG[index_gl[cg]] = FALSE;
+ }
+ }
+ }
+ cgindex[home_pos] = nat;
+
+ return home_pos;
+}
+
+static void clear_and_mark_ind(int ncg, int *move,
+ int *index_gl, int *cgindex, int *gatindex,
+ gmx_ga2la_t ga2la, char *bLocalCG,
+ int *cell_index)
+{
+ int cg, a0, a1, a;
+
+ for (cg = 0; cg < ncg; cg++)
+ {
+ if (move[cg] >= 0)
+ {
+ a0 = cgindex[cg];
+ a1 = cgindex[cg+1];
+ /* Clear the global indices */
+ for (a = a0; a < a1; a++)
+ {
+ ga2la_del(ga2la, gatindex[a]);
+ }
+ if (bLocalCG)
+ {
+ bLocalCG[index_gl[cg]] = FALSE;
+ }
+ /* Signal that this cg has moved using the ns cell index.
+ * Here we set it to -1. fill_grid will change it
+ * from -1 to NSGRID_SIGNAL_MOVED_FAC*grid->ncells.
+ */
+ cell_index[cg] = -1;
+ }
+ }
+}
+
+static void print_cg_move(FILE *fplog,
+ gmx_domdec_t *dd,
+ gmx_large_int_t step, int cg, int dim, int dir,
+ gmx_bool bHaveLimitdAndCMOld, real limitd,
+ rvec cm_old, rvec cm_new, real pos_d)
+{
+ gmx_domdec_comm_t *comm;
+ char buf[22];
+
+ comm = dd->comm;
+
+ fprintf(fplog, "\nStep %s:\n", gmx_step_str(step, buf));
+ if (bHaveLimitdAndCMOld)
+ {
+ fprintf(fplog, "The charge group starting at atom %d moved more than the distance allowed by the domain decomposition (%f) in direction %c\n",
+ ddglatnr(dd, dd->cgindex[cg]), limitd, dim2char(dim));
+ }
+ else
+ {
+ fprintf(fplog, "The charge group starting at atom %d moved than the distance allowed by the domain decomposition in direction %c\n",
+ ddglatnr(dd, dd->cgindex[cg]), dim2char(dim));
+ }
+ fprintf(fplog, "distance out of cell %f\n",
+ dir == 1 ? pos_d - comm->cell_x1[dim] : pos_d - comm->cell_x0[dim]);
+ if (bHaveLimitdAndCMOld)
+ {
+ fprintf(fplog, "Old coordinates: %8.3f %8.3f %8.3f\n",
+ cm_old[XX], cm_old[YY], cm_old[ZZ]);
+ }
+ fprintf(fplog, "New coordinates: %8.3f %8.3f %8.3f\n",
+ cm_new[XX], cm_new[YY], cm_new[ZZ]);
+ fprintf(fplog, "Old cell boundaries in direction %c: %8.3f %8.3f\n",
+ dim2char(dim),
+ comm->old_cell_x0[dim], comm->old_cell_x1[dim]);
+ fprintf(fplog, "New cell boundaries in direction %c: %8.3f %8.3f\n",
+ dim2char(dim),
+ comm->cell_x0[dim], comm->cell_x1[dim]);
+}
+
+static void cg_move_error(FILE *fplog,
+ gmx_domdec_t *dd,
+ gmx_large_int_t step, int cg, int dim, int dir,
+ gmx_bool bHaveLimitdAndCMOld, real limitd,
+ rvec cm_old, rvec cm_new, real pos_d)
+{
+ if (fplog)
+ {
+ print_cg_move(fplog, dd, step, cg, dim, dir,
+ bHaveLimitdAndCMOld, limitd, cm_old, cm_new, pos_d);
+ }
+ print_cg_move(stderr, dd, step, cg, dim, dir,
+ bHaveLimitdAndCMOld, limitd, cm_old, cm_new, pos_d);
+ gmx_fatal(FARGS,
+ "A charge group moved too far between two domain decomposition steps\n"
+ "This usually means that your system is not well equilibrated");
+}
+
+static void rotate_state_atom(t_state *state, int a)
+{
+ int est;
+
+ for (est = 0; est < estNR; est++)
+ {
+ if (EST_DISTR(est) && (state->flags & (1<<est)))
+ {
+ switch (est)
+ {
+ case estX:
+ /* Rotate the complete state; for a rectangular box only */
+ state->x[a][YY] = state->box[YY][YY] - state->x[a][YY];
+ state->x[a][ZZ] = state->box[ZZ][ZZ] - state->x[a][ZZ];
+ break;
+ case estV:
+ state->v[a][YY] = -state->v[a][YY];
+ state->v[a][ZZ] = -state->v[a][ZZ];
+ break;
+ case estSDX:
+ state->sd_X[a][YY] = -state->sd_X[a][YY];
+ state->sd_X[a][ZZ] = -state->sd_X[a][ZZ];
+ break;
+ case estCGP:
+ state->cg_p[a][YY] = -state->cg_p[a][YY];
+ state->cg_p[a][ZZ] = -state->cg_p[a][ZZ];
+ break;
+ case estDISRE_INITF:
+ case estDISRE_RM3TAV:
+ case estORIRE_INITF:
+ case estORIRE_DTAV:
+ /* These are distances, so not affected by rotation */
+ break;
+ default:
+ gmx_incons("Unknown state entry encountered in rotate_state_atom");
+ }
+ }
+ }
+}
+
+static int *get_moved(gmx_domdec_comm_t *comm, int natoms)
+{
+ if (natoms > comm->moved_nalloc)
+ {
+ /* Contents should be preserved here */
+ comm->moved_nalloc = over_alloc_dd(natoms);
+ srenew(comm->moved, comm->moved_nalloc);
+ }
+
+ return comm->moved;
+}
+
+static void calc_cg_move(FILE *fplog, gmx_large_int_t step,
+ gmx_domdec_t *dd,
+ t_state *state,
+ ivec tric_dir, matrix tcm,
+ rvec cell_x0, rvec cell_x1,
+ rvec limitd, rvec limit0, rvec limit1,
+ const int *cgindex,
+ int cg_start, int cg_end,
+ rvec *cg_cm,
+ int *move)
+{
+ int npbcdim;
+ int c, i, cg, k, k0, k1, d, dim, dim2, dir, d2, d3, d4, cell_d;
+ int mc, cdd, nrcg, ncg_recv, nat_recv, nvs, nvr, nvec, vec;
+ int flag;
+ gmx_bool bScrew;
+ ivec dev;
+ real inv_ncg, pos_d;
+ rvec cm_new;
+
+ npbcdim = dd->npbcdim;
+
+ for (cg = cg_start; cg < cg_end; cg++)
+ {
+ k0 = cgindex[cg];
+ k1 = cgindex[cg+1];
+ nrcg = k1 - k0;
+ if (nrcg == 1)
+ {
+ copy_rvec(state->x[k0], cm_new);
+ }
+ else
+ {
+ inv_ncg = 1.0/nrcg;
+
+ clear_rvec(cm_new);
+ for (k = k0; (k < k1); k++)
+ {
+ rvec_inc(cm_new, state->x[k]);
+ }
+ for (d = 0; (d < DIM); d++)
+ {
+ cm_new[d] = inv_ncg*cm_new[d];
+ }
+ }
+
+ clear_ivec(dev);
+ /* Do pbc and check DD cell boundary crossings */
+ for (d = DIM-1; d >= 0; d--)
+ {
+ if (dd->nc[d] > 1)
+ {
+ bScrew = (dd->bScrewPBC && d == XX);
+ /* Determine the location of this cg in lattice coordinates */
+ pos_d = cm_new[d];
+ if (tric_dir[d])
+ {
+ for (d2 = d+1; d2 < DIM; d2++)
+ {
+ pos_d += cm_new[d2]*tcm[d2][d];
+ }
+ }
+ /* Put the charge group in the triclinic unit-cell */
+ if (pos_d >= cell_x1[d])
+ {
+ if (pos_d >= limit1[d])
+ {
+ cg_move_error(fplog, dd, step, cg, d, 1, TRUE, limitd[d],
+ cg_cm[cg], cm_new, pos_d);
+ }
+ dev[d] = 1;
+ if (dd->ci[d] == dd->nc[d] - 1)
+ {
+ rvec_dec(cm_new, state->box[d]);
+ if (bScrew)
+ {
+ cm_new[YY] = state->box[YY][YY] - cm_new[YY];
+ cm_new[ZZ] = state->box[ZZ][ZZ] - cm_new[ZZ];
+ }
+ for (k = k0; (k < k1); k++)
+ {
+ rvec_dec(state->x[k], state->box[d]);
+ if (bScrew)
+ {
+ rotate_state_atom(state, k);
+ }
+ }
+ }
+ }
+ else if (pos_d < cell_x0[d])
+ {
+ if (pos_d < limit0[d])
+ {
+ cg_move_error(fplog, dd, step, cg, d, -1, TRUE, limitd[d],
+ cg_cm[cg], cm_new, pos_d);
+ }
+ dev[d] = -1;
+ if (dd->ci[d] == 0)
+ {
+ rvec_inc(cm_new, state->box[d]);
+ if (bScrew)
+ {
+ cm_new[YY] = state->box[YY][YY] - cm_new[YY];
+ cm_new[ZZ] = state->box[ZZ][ZZ] - cm_new[ZZ];
+ }
+ for (k = k0; (k < k1); k++)
+ {
+ rvec_inc(state->x[k], state->box[d]);
+ if (bScrew)
+ {
+ rotate_state_atom(state, k);
+ }
+ }
+ }
+ }
+ }
+ else if (d < npbcdim)
+ {
+ /* Put the charge group in the rectangular unit-cell */
+ while (cm_new[d] >= state->box[d][d])
+ {
+ rvec_dec(cm_new, state->box[d]);
+ for (k = k0; (k < k1); k++)
+ {
+ rvec_dec(state->x[k], state->box[d]);
+ }
+ }
+ while (cm_new[d] < 0)
+ {
+ rvec_inc(cm_new, state->box[d]);
+ for (k = k0; (k < k1); k++)
+ {
+ rvec_inc(state->x[k], state->box[d]);
+ }
+ }
+ }
+ }
+
+ copy_rvec(cm_new, cg_cm[cg]);
+
+ /* Determine where this cg should go */
+ flag = 0;
+ mc = -1;
+ for (d = 0; d < dd->ndim; d++)
+ {
+ dim = dd->dim[d];
+ if (dev[dim] == 1)
+ {
+ flag |= DD_FLAG_FW(d);
+ if (mc == -1)
+ {
+ mc = d*2;
+ }
+ }
+ else if (dev[dim] == -1)
+ {
+ flag |= DD_FLAG_BW(d);
+ if (mc == -1)
+ {
+ if (dd->nc[dim] > 2)
+ {
+ mc = d*2 + 1;
+ }
+ else
+ {
+ mc = d*2;
+ }
+ }
+ }
+ }
+ /* Temporarily store the flag in move */
+ move[cg] = mc + flag;
+ }
+}
+
+static void dd_redistribute_cg(FILE *fplog, gmx_large_int_t step,
+ gmx_domdec_t *dd, ivec tric_dir,
+ t_state *state, rvec **f,
+ t_forcerec *fr,
+ gmx_bool bCompact,
+ t_nrnb *nrnb,
+ int *ncg_stay_home,
+ int *ncg_moved)
+{
+ int *move;
+ int npbcdim;
+ int ncg[DIM*2], nat[DIM*2];
+ int c, i, cg, k, k0, k1, d, dim, dim2, dir, d2, d3, d4, cell_d;
+ int mc, cdd, nrcg, ncg_recv, nat_recv, nvs, nvr, nvec, vec;
+ int sbuf[2], rbuf[2];
+ int home_pos_cg, home_pos_at, buf_pos;
+ int flag;
+ gmx_bool bV = FALSE, bSDX = FALSE, bCGP = FALSE;
+ gmx_bool bScrew;
+ ivec dev;
+ real inv_ncg, pos_d;
+ matrix tcm;
+ rvec *cg_cm = NULL, cell_x0, cell_x1, limitd, limit0, limit1, cm_new;
+ atom_id *cgindex;
+ cginfo_mb_t *cginfo_mb;
+ gmx_domdec_comm_t *comm;
+ int *moved;
+ int nthread, thread;
+
+ if (dd->bScrewPBC)
+ {
+ check_screw_box(state->box);
+ }
+
+ comm = dd->comm;
+ if (fr->cutoff_scheme == ecutsGROUP)
+ {
+ cg_cm = fr->cg_cm;
+ }
+
+ for (i = 0; i < estNR; i++)
+ {
+ if (EST_DISTR(i))
+ {
+ switch (i)
+ {
+ case estX: /* Always present */ break;
+ case estV: bV = (state->flags & (1<<i)); break;
+ case estSDX: bSDX = (state->flags & (1<<i)); break;
+ case estCGP: bCGP = (state->flags & (1<<i)); break;
+ case estLD_RNG:
+ case estLD_RNGI:
+ case estDISRE_INITF:
+ case estDISRE_RM3TAV:
+ case estORIRE_INITF:
+ case estORIRE_DTAV:
+ /* No processing required */
+ break;
+ default:
+ gmx_incons("Unknown state entry encountered in dd_redistribute_cg");
+ }
+ }
+ }
+
+ if (dd->ncg_tot > comm->nalloc_int)
+ {
+ comm->nalloc_int = over_alloc_dd(dd->ncg_tot);
+ srenew(comm->buf_int, comm->nalloc_int);
+ }
+ move = comm->buf_int;
+
+ /* Clear the count */
+ for (c = 0; c < dd->ndim*2; c++)
+ {
+ ncg[c] = 0;
+ nat[c] = 0;
+ }
+
+ npbcdim = dd->npbcdim;
+
+ for (d = 0; (d < DIM); d++)
+ {
+ limitd[d] = dd->comm->cellsize_min[d];
+ if (d >= npbcdim && dd->ci[d] == 0)
+ {
+ cell_x0[d] = -GMX_FLOAT_MAX;
+ }
+ else
+ {
+ cell_x0[d] = comm->cell_x0[d];
+ }
+ if (d >= npbcdim && dd->ci[d] == dd->nc[d] - 1)
+ {
+ cell_x1[d] = GMX_FLOAT_MAX;
+ }
+ else
+ {
+ cell_x1[d] = comm->cell_x1[d];
+ }
+ if (d < npbcdim)
+ {
+ limit0[d] = comm->old_cell_x0[d] - limitd[d];
+ limit1[d] = comm->old_cell_x1[d] + limitd[d];
+ }
+ else
+ {
+ /* We check after communication if a charge group moved
+ * more than one cell. Set the pre-comm check limit to float_max.
+ */
+ limit0[d] = -GMX_FLOAT_MAX;
+ limit1[d] = GMX_FLOAT_MAX;
+ }
+ }
+
+ make_tric_corr_matrix(npbcdim, state->box, tcm);
+
+ cgindex = dd->cgindex;
+
+ nthread = gmx_omp_nthreads_get(emntDomdec);
+
+ /* Compute the center of geometry for all home charge groups
+ * and put them in the box and determine where they should go.
+ */
+#pragma omp parallel for num_threads(nthread) schedule(static)
+ for (thread = 0; thread < nthread; thread++)
+ {
+ calc_cg_move(fplog, step, dd, state, tric_dir, tcm,
+ cell_x0, cell_x1, limitd, limit0, limit1,
+ cgindex,
+ ( thread *dd->ncg_home)/nthread,
+ ((thread+1)*dd->ncg_home)/nthread,
+ fr->cutoff_scheme == ecutsGROUP ? cg_cm : state->x,
+ move);
+ }
+
+ for (cg = 0; cg < dd->ncg_home; cg++)
+ {
+ if (move[cg] >= 0)
+ {
+ mc = move[cg];
+ flag = mc & ~DD_FLAG_NRCG;
+ mc = mc & DD_FLAG_NRCG;
+ move[cg] = mc;
+
+ if (ncg[mc]+1 > comm->cggl_flag_nalloc[mc])
+ {
+ comm->cggl_flag_nalloc[mc] = over_alloc_dd(ncg[mc]+1);
+ srenew(comm->cggl_flag[mc], comm->cggl_flag_nalloc[mc]*DD_CGIBS);
+ }
+ comm->cggl_flag[mc][ncg[mc]*DD_CGIBS ] = dd->index_gl[cg];
+ /* We store the cg size in the lower 16 bits
+ * and the place where the charge group should go
+ * in the next 6 bits. This saves some communication volume.
+ */
+ nrcg = cgindex[cg+1] - cgindex[cg];
+ comm->cggl_flag[mc][ncg[mc]*DD_CGIBS+1] = nrcg | flag;
+ ncg[mc] += 1;
+ nat[mc] += nrcg;
+ }
+ }
+
+ inc_nrnb(nrnb, eNR_CGCM, dd->nat_home);
+ inc_nrnb(nrnb, eNR_RESETX, dd->ncg_home);
+
+ *ncg_moved = 0;
+ for (i = 0; i < dd->ndim*2; i++)
+ {
+ *ncg_moved += ncg[i];
+ }
+
+ nvec = 1;
+ if (bV)
+ {
+ nvec++;
+ }
+ if (bSDX)
+ {
+ nvec++;
+ }
+ if (bCGP)
+ {
+ nvec++;
+ }
+
+ /* Make sure the communication buffers are large enough */
+ for (mc = 0; mc < dd->ndim*2; mc++)
+ {
+ nvr = ncg[mc] + nat[mc]*nvec;
+ if (nvr > comm->cgcm_state_nalloc[mc])
+ {
+ comm->cgcm_state_nalloc[mc] = over_alloc_dd(nvr);
+ srenew(comm->cgcm_state[mc], comm->cgcm_state_nalloc[mc]);
+ }
+ }
+
+ switch (fr->cutoff_scheme)
+ {
+ case ecutsGROUP:
+ /* Recalculating cg_cm might be cheaper than communicating,
+ * but that could give rise to rounding issues.
+ */
+ home_pos_cg =
+ compact_and_copy_vec_cg(dd->ncg_home, move, cgindex,
+ nvec, cg_cm, comm, bCompact);
+ break;
+ case ecutsVERLET:
+ /* Without charge groups we send the moved atom coordinates
+ * over twice. This is so the code below can be used without
+ * many conditionals for both for with and without charge groups.
+ */
+ home_pos_cg =
+ compact_and_copy_vec_cg(dd->ncg_home, move, cgindex,
+ nvec, state->x, comm, FALSE);
+ if (bCompact)
+ {
+ home_pos_cg -= *ncg_moved;
+ }
+ break;
+ default:
+ gmx_incons("unimplemented");
+ home_pos_cg = 0;
+ }
+
+ vec = 0;
+ home_pos_at =
+ compact_and_copy_vec_at(dd->ncg_home, move, cgindex,
+ nvec, vec++, state->x, comm, bCompact);
+ if (bV)
+ {
+ compact_and_copy_vec_at(dd->ncg_home, move, cgindex,
+ nvec, vec++, state->v, comm, bCompact);
+ }
+ if (bSDX)
+ {
+ compact_and_copy_vec_at(dd->ncg_home, move, cgindex,
+ nvec, vec++, state->sd_X, comm, bCompact);
+ }
+ if (bCGP)
+ {
+ compact_and_copy_vec_at(dd->ncg_home, move, cgindex,
+ nvec, vec++, state->cg_p, comm, bCompact);
+ }
+
+ if (bCompact)
+ {
+ compact_ind(dd->ncg_home, move,
+ dd->index_gl, dd->cgindex, dd->gatindex,
+ dd->ga2la, comm->bLocalCG,
+ fr->cginfo);
+ }
+ else
+ {
+ if (fr->cutoff_scheme == ecutsVERLET)
+ {
+ moved = get_moved(comm, dd->ncg_home);
+
+ for (k = 0; k < dd->ncg_home; k++)
+ {
+ moved[k] = 0;
+ }
+ }
+ else
+ {
+ moved = fr->ns.grid->cell_index;
+ }
+
+ clear_and_mark_ind(dd->ncg_home, move,
+ dd->index_gl, dd->cgindex, dd->gatindex,
+ dd->ga2la, comm->bLocalCG,
+ moved);
+ }
+
+ cginfo_mb = fr->cginfo_mb;
+
+ *ncg_stay_home = home_pos_cg;
+ for (d = 0; d < dd->ndim; d++)
+ {
+ dim = dd->dim[d];
+ ncg_recv = 0;
+ nat_recv = 0;
+ nvr = 0;
+ for (dir = 0; dir < (dd->nc[dim] == 2 ? 1 : 2); dir++)
+ {
+ cdd = d*2 + dir;
+ /* Communicate the cg and atom counts */
+ sbuf[0] = ncg[cdd];
+ sbuf[1] = nat[cdd];
+ if (debug)
+ {
+ fprintf(debug, "Sending ddim %d dir %d: ncg %d nat %d\n",
+ d, dir, sbuf[0], sbuf[1]);
+ }
+ dd_sendrecv_int(dd, d, dir, sbuf, 2, rbuf, 2);
+
+ if ((ncg_recv+rbuf[0])*DD_CGIBS > comm->nalloc_int)
+ {
+ comm->nalloc_int = over_alloc_dd((ncg_recv+rbuf[0])*DD_CGIBS);
+ srenew(comm->buf_int, comm->nalloc_int);
+ }
+
+ /* Communicate the charge group indices, sizes and flags */
+ dd_sendrecv_int(dd, d, dir,
+ comm->cggl_flag[cdd], sbuf[0]*DD_CGIBS,
+ comm->buf_int+ncg_recv*DD_CGIBS, rbuf[0]*DD_CGIBS);
+
+ nvs = ncg[cdd] + nat[cdd]*nvec;
+ i = rbuf[0] + rbuf[1] *nvec;
+ vec_rvec_check_alloc(&comm->vbuf, nvr+i);
+
+ /* Communicate cgcm and state */
+ dd_sendrecv_rvec(dd, d, dir,
+ comm->cgcm_state[cdd], nvs,
+ comm->vbuf.v+nvr, i);
+ ncg_recv += rbuf[0];
+ nat_recv += rbuf[1];
+ nvr += i;
+ }
+
+ /* Process the received charge groups */
+ buf_pos = 0;
+ for (cg = 0; cg < ncg_recv; cg++)
+ {
+ flag = comm->buf_int[cg*DD_CGIBS+1];
+
+ if (dim >= npbcdim && dd->nc[dim] > 2)
+ {
+ /* No pbc in this dim and more than one domain boundary.
+ * We do a separate check if a charge group didn't move too far.
+ */
+ if (((flag & DD_FLAG_FW(d)) &&
+ comm->vbuf.v[buf_pos][dim] > cell_x1[dim]) ||
+ ((flag & DD_FLAG_BW(d)) &&
+ comm->vbuf.v[buf_pos][dim] < cell_x0[dim]))
+ {
+ cg_move_error(fplog, dd, step, cg, dim,
+ (flag & DD_FLAG_FW(d)) ? 1 : 0,
+ FALSE, 0,
+ comm->vbuf.v[buf_pos],
+ comm->vbuf.v[buf_pos],
+ comm->vbuf.v[buf_pos][dim]);
+ }
+ }
+
+ mc = -1;
+ if (d < dd->ndim-1)
+ {
+ /* Check which direction this cg should go */
+ for (d2 = d+1; (d2 < dd->ndim && mc == -1); d2++)
+ {
+ if (dd->bGridJump)
+ {
+ /* The cell boundaries for dimension d2 are not equal
+ * for each cell row of the lower dimension(s),
+ * therefore we might need to redetermine where
+ * this cg should go.
+ */
+ dim2 = dd->dim[d2];
+ /* If this cg crosses the box boundary in dimension d2
+ * we can use the communicated flag, so we do not
+ * have to worry about pbc.
+ */
+ if (!((dd->ci[dim2] == dd->nc[dim2]-1 &&
+ (flag & DD_FLAG_FW(d2))) ||
+ (dd->ci[dim2] == 0 &&
+ (flag & DD_FLAG_BW(d2)))))
+ {
+ /* Clear the two flags for this dimension */
+ flag &= ~(DD_FLAG_FW(d2) | DD_FLAG_BW(d2));
+ /* Determine the location of this cg
+ * in lattice coordinates
+ */
+ pos_d = comm->vbuf.v[buf_pos][dim2];
+ if (tric_dir[dim2])
+ {
+ for (d3 = dim2+1; d3 < DIM; d3++)
+ {
+ pos_d +=
+ comm->vbuf.v[buf_pos][d3]*tcm[d3][dim2];
+ }
+ }
+ /* Check of we are not at the box edge.
+ * pbc is only handled in the first step above,
+ * but this check could move over pbc while
+ * the first step did not due to different rounding.
+ */
+ if (pos_d >= cell_x1[dim2] &&
+ dd->ci[dim2] != dd->nc[dim2]-1)
+ {
+ flag |= DD_FLAG_FW(d2);
+ }
+ else if (pos_d < cell_x0[dim2] &&
+ dd->ci[dim2] != 0)
+ {
+ flag |= DD_FLAG_BW(d2);
+ }
+ comm->buf_int[cg*DD_CGIBS+1] = flag;
+ }
+ }
+ /* Set to which neighboring cell this cg should go */
+ if (flag & DD_FLAG_FW(d2))
+ {
+ mc = d2*2;
+ }
+ else if (flag & DD_FLAG_BW(d2))
+ {
+ if (dd->nc[dd->dim[d2]] > 2)
+ {
+ mc = d2*2+1;
+ }
+ else
+ {
+ mc = d2*2;
+ }
+ }
+ }
+ }
+
+ nrcg = flag & DD_FLAG_NRCG;
+ if (mc == -1)
+ {
+ if (home_pos_cg+1 > dd->cg_nalloc)
+ {
+ dd->cg_nalloc = over_alloc_dd(home_pos_cg+1);
+ srenew(dd->index_gl, dd->cg_nalloc);
+ srenew(dd->cgindex, dd->cg_nalloc+1);
+ }
+ /* Set the global charge group index and size */
+ dd->index_gl[home_pos_cg] = comm->buf_int[cg*DD_CGIBS];
+ dd->cgindex[home_pos_cg+1] = dd->cgindex[home_pos_cg] + nrcg;
+ /* Copy the state from the buffer */
+ dd_check_alloc_ncg(fr, state, f, home_pos_cg+1);
+ if (fr->cutoff_scheme == ecutsGROUP)
+ {
+ cg_cm = fr->cg_cm;
+ copy_rvec(comm->vbuf.v[buf_pos], cg_cm[home_pos_cg]);
+ }
+ buf_pos++;
+
+ /* Set the cginfo */
+ fr->cginfo[home_pos_cg] = ddcginfo(cginfo_mb,
+ dd->index_gl[home_pos_cg]);
+ if (comm->bLocalCG)
+ {
+ comm->bLocalCG[dd->index_gl[home_pos_cg]] = TRUE;
+ }
+
+ if (home_pos_at+nrcg > state->nalloc)
+ {
+ dd_realloc_state(state, f, home_pos_at+nrcg);
+ }
+ for (i = 0; i < nrcg; i++)
+ {
+ copy_rvec(comm->vbuf.v[buf_pos++],
+ state->x[home_pos_at+i]);
+ }
+ if (bV)
+ {
+ for (i = 0; i < nrcg; i++)
+ {
+ copy_rvec(comm->vbuf.v[buf_pos++],
+ state->v[home_pos_at+i]);
+ }
+ }
+ if (bSDX)
+ {
+ for (i = 0; i < nrcg; i++)
+ {
+ copy_rvec(comm->vbuf.v[buf_pos++],
+ state->sd_X[home_pos_at+i]);
+ }
+ }
+ if (bCGP)
+ {
+ for (i = 0; i < nrcg; i++)
+ {
+ copy_rvec(comm->vbuf.v[buf_pos++],
+ state->cg_p[home_pos_at+i]);
+ }
+ }
+ home_pos_cg += 1;
+ home_pos_at += nrcg;
+ }
+ else
+ {
+ /* Reallocate the buffers if necessary */
+ if (ncg[mc]+1 > comm->cggl_flag_nalloc[mc])
+ {
+ comm->cggl_flag_nalloc[mc] = over_alloc_dd(ncg[mc]+1);
+ srenew(comm->cggl_flag[mc], comm->cggl_flag_nalloc[mc]*DD_CGIBS);
+ }
+ nvr = ncg[mc] + nat[mc]*nvec;
+ if (nvr + 1 + nrcg*nvec > comm->cgcm_state_nalloc[mc])
+ {
+ comm->cgcm_state_nalloc[mc] = over_alloc_dd(nvr + 1 + nrcg*nvec);
+ srenew(comm->cgcm_state[mc], comm->cgcm_state_nalloc[mc]);
+ }
+ /* Copy from the receive to the send buffers */
+ memcpy(comm->cggl_flag[mc] + ncg[mc]*DD_CGIBS,
+ comm->buf_int + cg*DD_CGIBS,
+ DD_CGIBS*sizeof(int));
+ memcpy(comm->cgcm_state[mc][nvr],
+ comm->vbuf.v[buf_pos],
+ (1+nrcg*nvec)*sizeof(rvec));
+ buf_pos += 1 + nrcg*nvec;
+ ncg[mc] += 1;
+ nat[mc] += nrcg;
+ }
+ }
+ }
+
+ /* With sorting (!bCompact) the indices are now only partially up to date
+ * and ncg_home and nat_home are not the real count, since there are
+ * "holes" in the arrays for the charge groups that moved to neighbors.
+ */
+ if (fr->cutoff_scheme == ecutsVERLET)
+ {
+ moved = get_moved(comm, home_pos_cg);
+
+ for (i = dd->ncg_home; i < home_pos_cg; i++)
+ {
+ moved[i] = 0;
+ }
+ }
+ dd->ncg_home = home_pos_cg;
+ dd->nat_home = home_pos_at;
+
+ if (debug)
+ {
+ fprintf(debug,
+ "Finished repartitioning: cgs moved out %d, new home %d\n",
+ *ncg_moved, dd->ncg_home-*ncg_moved);
+
+ }
+}
+
+void dd_cycles_add(gmx_domdec_t *dd, float cycles, int ddCycl)
+{
+ dd->comm->cycl[ddCycl] += cycles;
+ dd->comm->cycl_n[ddCycl]++;
+ if (cycles > dd->comm->cycl_max[ddCycl])
+ {
+ dd->comm->cycl_max[ddCycl] = cycles;
+ }
+}
+
+static double force_flop_count(t_nrnb *nrnb)
+{
+ int i;
+ double sum;
+ const char *name;
+
+ sum = 0;
+ for (i = 0; i < eNR_NBKERNEL_FREE_ENERGY; i++)
+ {
+ /* To get closer to the real timings, we half the count
+ * for the normal loops and again half it for water loops.
+ */
+ name = nrnb_str(i);
+ if (strstr(name, "W3") != NULL || strstr(name, "W4") != NULL)
+ {
+ sum += nrnb->n[i]*0.25*cost_nrnb(i);
+ }
+ else
+ {
+ sum += nrnb->n[i]*0.50*cost_nrnb(i);
+ }
+ }
+ for (i = eNR_NBKERNEL_FREE_ENERGY; i <= eNR_NB14; i++)
+ {
+ name = nrnb_str(i);
+ if (strstr(name, "W3") != NULL || strstr(name, "W4") != NULL)
+ {
+ sum += nrnb->n[i]*cost_nrnb(i);
+ }
+ }
+ for (i = eNR_BONDS; i <= eNR_WALLS; i++)
+ {
+ sum += nrnb->n[i]*cost_nrnb(i);
+ }
+
+ return sum;
+}
+
+void dd_force_flop_start(gmx_domdec_t *dd, t_nrnb *nrnb)
+{
+ if (dd->comm->eFlop)
+ {
+ dd->comm->flop -= force_flop_count(nrnb);
+ }
+}
+void dd_force_flop_stop(gmx_domdec_t *dd, t_nrnb *nrnb)
+{
+ if (dd->comm->eFlop)
+ {
+ dd->comm->flop += force_flop_count(nrnb);
+ dd->comm->flop_n++;
+ }
+}
+
+static void clear_dd_cycle_counts(gmx_domdec_t *dd)
+{
+ int i;
+
+ for (i = 0; i < ddCyclNr; i++)
+ {
+ dd->comm->cycl[i] = 0;
+ dd->comm->cycl_n[i] = 0;
+ dd->comm->cycl_max[i] = 0;
+ }
+ dd->comm->flop = 0;
+ dd->comm->flop_n = 0;
+}
+
+static void get_load_distribution(gmx_domdec_t *dd, gmx_wallcycle_t wcycle)
+{
+ gmx_domdec_comm_t *comm;
+ gmx_domdec_load_t *load;
+ gmx_domdec_root_t *root = NULL;
+ int d, dim, cid, i, pos;
+ float cell_frac = 0, sbuf[DD_NLOAD_MAX];
+ gmx_bool bSepPME;
+
+ if (debug)
+ {
+ fprintf(debug, "get_load_distribution start\n");
+ }
+
+ wallcycle_start(wcycle, ewcDDCOMMLOAD);
+
+ comm = dd->comm;
+
+ bSepPME = (dd->pme_nodeid >= 0);
+
+ for (d = dd->ndim-1; d >= 0; d--)
+ {
+ dim = dd->dim[d];
+ /* Check if we participate in the communication in this dimension */
+ if (d == dd->ndim-1 ||
+ (dd->ci[dd->dim[d+1]] == 0 && dd->ci[dd->dim[dd->ndim-1]] == 0))
+ {
+ load = &comm->load[d];
+ if (dd->bGridJump)
+ {
+ cell_frac = comm->cell_f1[d] - comm->cell_f0[d];
+ }
+ pos = 0;
+ if (d == dd->ndim-1)
+ {
+ sbuf[pos++] = dd_force_load(comm);
+ sbuf[pos++] = sbuf[0];
+ if (dd->bGridJump)
+ {
+ sbuf[pos++] = sbuf[0];
+ sbuf[pos++] = cell_frac;
+ if (d > 0)
+ {
+ sbuf[pos++] = comm->cell_f_max0[d];
+ sbuf[pos++] = comm->cell_f_min1[d];
+ }
+ }
+ if (bSepPME)
+ {
+ sbuf[pos++] = comm->cycl[ddCyclPPduringPME];
+ sbuf[pos++] = comm->cycl[ddCyclPME];
+ }
+ }
+ else
+ {
+ sbuf[pos++] = comm->load[d+1].sum;
+ sbuf[pos++] = comm->load[d+1].max;
+ if (dd->bGridJump)
+ {
+ sbuf[pos++] = comm->load[d+1].sum_m;
+ sbuf[pos++] = comm->load[d+1].cvol_min*cell_frac;
+ sbuf[pos++] = comm->load[d+1].flags;
+ if (d > 0)
+ {
+ sbuf[pos++] = comm->cell_f_max0[d];
+ sbuf[pos++] = comm->cell_f_min1[d];
+ }
+ }
+ if (bSepPME)
+ {
+ sbuf[pos++] = comm->load[d+1].mdf;
+ sbuf[pos++] = comm->load[d+1].pme;
+ }
+ }
+ load->nload = pos;
+ /* Communicate a row in DD direction d.
+ * The communicators are setup such that the root always has rank 0.
+ */
+#ifdef GMX_MPI
+ MPI_Gather(sbuf, load->nload*sizeof(float), MPI_BYTE,
+ load->load, load->nload*sizeof(float), MPI_BYTE,
+ 0, comm->mpi_comm_load[d]);
+#endif
+ if (dd->ci[dim] == dd->master_ci[dim])
+ {
+ /* We are the root, process this row */
+ if (comm->bDynLoadBal)
+ {
+ root = comm->root[d];
+ }
+ load->sum = 0;
+ load->max = 0;
+ load->sum_m = 0;
+ load->cvol_min = 1;
+ load->flags = 0;
+ load->mdf = 0;
+ load->pme = 0;
+ pos = 0;
+ for (i = 0; i < dd->nc[dim]; i++)
+ {
+ load->sum += load->load[pos++];
+ load->max = max(load->max, load->load[pos]);
+ pos++;
+ if (dd->bGridJump)
+ {
+ if (root->bLimited)
+ {
+ /* This direction could not be load balanced properly,
+ * therefore we need to use the maximum iso the average load.
+ */
+ load->sum_m = max(load->sum_m, load->load[pos]);
+ }
+ else
+ {
+ load->sum_m += load->load[pos];
+ }
+ pos++;
+ load->cvol_min = min(load->cvol_min, load->load[pos]);
+ pos++;
+ if (d < dd->ndim-1)
+ {
+ load->flags = (int)(load->load[pos++] + 0.5);
+ }
+ if (d > 0)
+ {
+ root->cell_f_max0[i] = load->load[pos++];
+ root->cell_f_min1[i] = load->load[pos++];
+ }
+ }
+ if (bSepPME)
+ {
+ load->mdf = max(load->mdf, load->load[pos]);
+ pos++;
+ load->pme = max(load->pme, load->load[pos]);
+ pos++;
+ }
+ }
+ if (comm->bDynLoadBal && root->bLimited)
+ {
+ load->sum_m *= dd->nc[dim];
+ load->flags |= (1<<d);
+ }
+ }
+ }
+ }
+
+ if (DDMASTER(dd))
+ {
+ comm->nload += dd_load_count(comm);
+ comm->load_step += comm->cycl[ddCyclStep];
+ comm->load_sum += comm->load[0].sum;
+ comm->load_max += comm->load[0].max;
+ if (comm->bDynLoadBal)
+ {
+ for (d = 0; d < dd->ndim; d++)
+ {
+ if (comm->load[0].flags & (1<<d))
+ {
+ comm->load_lim[d]++;
+ }
+ }
+ }
+ if (bSepPME)
+ {
+ comm->load_mdf += comm->load[0].mdf;
+ comm->load_pme += comm->load[0].pme;
+ }
+ }
+
+ wallcycle_stop(wcycle, ewcDDCOMMLOAD);
+
+ if (debug)
+ {
+ fprintf(debug, "get_load_distribution finished\n");
+ }
+}
+
+static float dd_force_imb_perf_loss(gmx_domdec_t *dd)
+{
+ /* Return the relative performance loss on the total run time
+ * due to the force calculation load imbalance.
+ */
+ if (dd->comm->nload > 0)
+ {
+ return
+ (dd->comm->load_max*dd->nnodes - dd->comm->load_sum)/
+ (dd->comm->load_step*dd->nnodes);
+ }
+ else
+ {
+ return 0;
+ }
+}
+
+static void print_dd_load_av(FILE *fplog, gmx_domdec_t *dd)
+{
+ char buf[STRLEN];
+ int npp, npme, nnodes, d, limp;
+ float imbal, pme_f_ratio, lossf, lossp = 0;
+ gmx_bool bLim;
+ gmx_domdec_comm_t *comm;
+
+ comm = dd->comm;
+ if (DDMASTER(dd) && comm->nload > 0)
+ {
+ npp = dd->nnodes;
+ npme = (dd->pme_nodeid >= 0) ? comm->npmenodes : 0;
+ nnodes = npp + npme;
+ imbal = comm->load_max*npp/comm->load_sum - 1;
+ lossf = dd_force_imb_perf_loss(dd);
+ sprintf(buf, " Average load imbalance: %.1f %%\n", imbal*100);
+ fprintf(fplog, "%s", buf);
+ fprintf(stderr, "\n");
+ fprintf(stderr, "%s", buf);
+ sprintf(buf, " Part of the total run time spent waiting due to load imbalance: %.1f %%\n", lossf*100);
+ fprintf(fplog, "%s", buf);
+ fprintf(stderr, "%s", buf);
+ bLim = FALSE;
+ if (comm->bDynLoadBal)
+ {
+ sprintf(buf, " Steps where the load balancing was limited by -rdd, -rcon and/or -dds:");
+ for (d = 0; d < dd->ndim; d++)
+ {
+ limp = (200*comm->load_lim[d]+1)/(2*comm->nload);
+ sprintf(buf+strlen(buf), " %c %d %%", dim2char(dd->dim[d]), limp);
+ if (limp >= 50)
+ {
+ bLim = TRUE;
+ }
+ }
+ sprintf(buf+strlen(buf), "\n");
+ fprintf(fplog, "%s", buf);
+ fprintf(stderr, "%s", buf);
+ }
+ if (npme > 0)
+ {
+ pme_f_ratio = comm->load_pme/comm->load_mdf;
+ lossp = (comm->load_pme -comm->load_mdf)/comm->load_step;
+ if (lossp <= 0)
+ {
+ lossp *= (float)npme/(float)nnodes;
+ }
+ else
+ {
+ lossp *= (float)npp/(float)nnodes;
+ }
+ sprintf(buf, " Average PME mesh/force load: %5.3f\n", pme_f_ratio);
+ fprintf(fplog, "%s", buf);
+ fprintf(stderr, "%s", buf);
+ sprintf(buf, " Part of the total run time spent waiting due to PP/PME imbalance: %.1f %%\n", fabs(lossp)*100);
+ fprintf(fplog, "%s", buf);
+ fprintf(stderr, "%s", buf);
+ }
+ fprintf(fplog, "\n");
+ fprintf(stderr, "\n");
+
+ if (lossf >= DD_PERF_LOSS)
+ {
+ sprintf(buf,
+ "NOTE: %.1f %% of the available CPU time was lost due to load imbalance\n"
+ " in the domain decomposition.\n", lossf*100);
+ if (!comm->bDynLoadBal)
+ {
+ sprintf(buf+strlen(buf), " You might want to use dynamic load balancing (option -dlb.)\n");
+ }
+ else if (bLim)
+ {
+ sprintf(buf+strlen(buf), " You might want to decrease the cell size limit (options -rdd, -rcon and/or -dds).\n");
+ }
+ fprintf(fplog, "%s\n", buf);
+ fprintf(stderr, "%s\n", buf);
+ }
+ if (npme > 0 && fabs(lossp) >= DD_PERF_LOSS)
+ {
+ sprintf(buf,
+ "NOTE: %.1f %% performance was lost because the PME nodes\n"
+ " had %s work to do than the PP nodes.\n"
+ " You might want to %s the number of PME nodes\n"
+ " or %s the cut-off and the grid spacing.\n",
+ fabs(lossp*100),
+ (lossp < 0) ? "less" : "more",
+ (lossp < 0) ? "decrease" : "increase",
+ (lossp < 0) ? "decrease" : "increase");
+ fprintf(fplog, "%s\n", buf);
+ fprintf(stderr, "%s\n", buf);
+ }
+ }
+}
+
+static float dd_vol_min(gmx_domdec_t *dd)
+{
+ return dd->comm->load[0].cvol_min*dd->nnodes;
+}
+
+static gmx_bool dd_load_flags(gmx_domdec_t *dd)
+{
+ return dd->comm->load[0].flags;
+}
+
+static float dd_f_imbal(gmx_domdec_t *dd)
+{
+ return dd->comm->load[0].max*dd->nnodes/dd->comm->load[0].sum - 1;
+}
+
+float dd_pme_f_ratio(gmx_domdec_t *dd)
+{
+ if (dd->comm->cycl_n[ddCyclPME] > 0)
+ {
+ return dd->comm->load[0].pme/dd->comm->load[0].mdf;
+ }
+ else
+ {
+ return -1.0;
+ }
+}
+
+static void dd_print_load(FILE *fplog, gmx_domdec_t *dd, gmx_large_int_t step)
+{
+ int flags, d;
+ char buf[22];
+
+ flags = dd_load_flags(dd);
+ if (flags)
+ {
+ fprintf(fplog,
+ "DD load balancing is limited by minimum cell size in dimension");
+ for (d = 0; d < dd->ndim; d++)
+ {
+ if (flags & (1<<d))
+ {
+ fprintf(fplog, " %c", dim2char(dd->dim[d]));
+ }
+ }
+ fprintf(fplog, "\n");
+ }
+ fprintf(fplog, "DD step %s", gmx_step_str(step, buf));
+ if (dd->comm->bDynLoadBal)
+ {
+ fprintf(fplog, " vol min/aver %5.3f%c",
+ dd_vol_min(dd), flags ? '!' : ' ');
+ }
+ fprintf(fplog, " load imb.: force %4.1f%%", dd_f_imbal(dd)*100);
+ if (dd->comm->cycl_n[ddCyclPME])
+ {
+ fprintf(fplog, " pme mesh/force %5.3f", dd_pme_f_ratio(dd));
+ }
+ fprintf(fplog, "\n\n");
+}
+
+static void dd_print_load_verbose(gmx_domdec_t *dd)
+{
+ if (dd->comm->bDynLoadBal)
+ {
+ fprintf(stderr, "vol %4.2f%c ",
+ dd_vol_min(dd), dd_load_flags(dd) ? '!' : ' ');
+ }
+ fprintf(stderr, "imb F %2d%% ", (int)(dd_f_imbal(dd)*100+0.5));
+ if (dd->comm->cycl_n[ddCyclPME])
+ {
+ fprintf(stderr, "pme/F %4.2f ", dd_pme_f_ratio(dd));
+ }
+}
+
+#ifdef GMX_MPI
+static void make_load_communicator(gmx_domdec_t *dd, int dim_ind, ivec loc)
+{
+ MPI_Comm c_row;
+ int dim, i, rank;
+ ivec loc_c;
+ gmx_domdec_root_t *root;
+ gmx_bool bPartOfGroup = FALSE;
+
+ dim = dd->dim[dim_ind];
+ copy_ivec(loc, loc_c);
+ for (i = 0; i < dd->nc[dim]; i++)
+ {
+ loc_c[dim] = i;
+ rank = dd_index(dd->nc, loc_c);
+ if (rank == dd->rank)
+ {
+ /* This process is part of the group */
+ bPartOfGroup = TRUE;
+ }
+ }
+ MPI_Comm_split(dd->mpi_comm_all, bPartOfGroup ? 0 : MPI_UNDEFINED, dd->rank,
+ &c_row);
+ if (bPartOfGroup)
+ {
+ dd->comm->mpi_comm_load[dim_ind] = c_row;
+ if (dd->comm->eDLB != edlbNO)
+ {
+ if (dd->ci[dim] == dd->master_ci[dim])
+ {
+ /* This is the root process of this row */
+ snew(dd->comm->root[dim_ind], 1);
+ root = dd->comm->root[dim_ind];
+ snew(root->cell_f, DD_CELL_F_SIZE(dd, dim_ind));
+ snew(root->old_cell_f, dd->nc[dim]+1);
+ snew(root->bCellMin, dd->nc[dim]);
+ if (dim_ind > 0)
+ {
+ snew(root->cell_f_max0, dd->nc[dim]);
+ snew(root->cell_f_min1, dd->nc[dim]);
+ snew(root->bound_min, dd->nc[dim]);
+ snew(root->bound_max, dd->nc[dim]);
+ }
+ snew(root->buf_ncd, dd->nc[dim]);
+ }
+ else
+ {
+ /* This is not a root process, we only need to receive cell_f */
+ snew(dd->comm->cell_f_row, DD_CELL_F_SIZE(dd, dim_ind));
+ }
+ }
+ if (dd->ci[dim] == dd->master_ci[dim])
+ {
+ snew(dd->comm->load[dim_ind].load, dd->nc[dim]*DD_NLOAD_MAX);
+ }
+ }
+}
+#endif
+
++void dd_setup_dlb_resource_sharing(t_commrec *cr,
++ const gmx_hw_info_t *hwinfo,
++ const gmx_hw_opt_t *hw_opt)
++{
++#ifdef GMX_MPI
++ int physicalnode_id_hash;
++ int gpu_id;
++ gmx_domdec_t *dd;
++ MPI_Comm mpi_comm_pp_physicalnode;
++
++ if (!(cr->duty & DUTY_PP) ||
++ hw_opt->gpu_opt.ncuda_dev_use == 0)
++ {
++ /* Only PP nodes (currently) use GPUs.
++ * If we don't have GPUs, there are no resources to share.
++ */
++ return;
++ }
++
++ physicalnode_id_hash = gmx_physicalnode_id_hash();
++
++ gpu_id = get_gpu_device_id(&hwinfo->gpu_info, &hw_opt->gpu_opt, cr->nodeid);
++
++ dd = cr->dd;
++
++ if (debug)
++ {
++ fprintf(debug, "dd_setup_dd_dlb_gpu_sharing:\n");
++ fprintf(debug, "DD PP rank %d physical node hash %d gpu_id %d\n",
++ dd->rank, physicalnode_id_hash, gpu_id);
++ }
++ /* Split the PP communicator over the physical nodes */
++ /* TODO: See if we should store this (before), as it's also used for
++ * for the nodecomm summution.
++ */
++ MPI_Comm_split(dd->mpi_comm_all, physicalnode_id_hash, dd->rank,
++ &mpi_comm_pp_physicalnode);
++ MPI_Comm_split(mpi_comm_pp_physicalnode, gpu_id, dd->rank,
++ &dd->comm->mpi_comm_gpu_shared);
++ MPI_Comm_free(&mpi_comm_pp_physicalnode);
++ MPI_Comm_size(dd->comm->mpi_comm_gpu_shared, &dd->comm->nrank_gpu_shared);
++
++ if (debug)
++ {
++ fprintf(debug, "nrank_gpu_shared %d\n", dd->comm->nrank_gpu_shared);
++ }
++
++ /* Note that some ranks could share a GPU, while others don't */
++
++ if (dd->comm->nrank_gpu_shared == 1)
++ {
++ MPI_Comm_free(&dd->comm->mpi_comm_gpu_shared);
++ }
++#endif
++}
++
+static void make_load_communicators(gmx_domdec_t *dd)
+{
+#ifdef GMX_MPI
+ int dim0, dim1, i, j;
+ ivec loc;
+
+ if (debug)
+ {
+ fprintf(debug, "Making load communicators\n");
+ }
+
+ snew(dd->comm->load, dd->ndim);
+ snew(dd->comm->mpi_comm_load, dd->ndim);
+
+ clear_ivec(loc);
+ make_load_communicator(dd, 0, loc);
+ if (dd->ndim > 1)
+ {
+ dim0 = dd->dim[0];
+ for (i = 0; i < dd->nc[dim0]; i++)
+ {
+ loc[dim0] = i;
+ make_load_communicator(dd, 1, loc);
+ }
+ }
+ if (dd->ndim > 2)
+ {
+ dim0 = dd->dim[0];
+ for (i = 0; i < dd->nc[dim0]; i++)
+ {
+ loc[dim0] = i;
+ dim1 = dd->dim[1];
+ for (j = 0; j < dd->nc[dim1]; j++)
+ {
+ loc[dim1] = j;
+ make_load_communicator(dd, 2, loc);
+ }
+ }
+ }
+
+ if (debug)
+ {
+ fprintf(debug, "Finished making load communicators\n");
+ }
+#endif
+}
+
+void setup_dd_grid(FILE *fplog, gmx_domdec_t *dd)
+{
+ gmx_bool bZYX;
+ int d, dim, i, j, m;
+ ivec tmp, s;
+ int nzone, nzonep;
+ ivec dd_zp[DD_MAXIZONE];
+ gmx_domdec_zones_t *zones;
+ gmx_domdec_ns_ranges_t *izone;
+
+ for (d = 0; d < dd->ndim; d++)
+ {
+ dim = dd->dim[d];
+ copy_ivec(dd->ci, tmp);
+ tmp[dim] = (tmp[dim] + 1) % dd->nc[dim];
+ dd->neighbor[d][0] = ddcoord2ddnodeid(dd, tmp);
+ copy_ivec(dd->ci, tmp);
+ tmp[dim] = (tmp[dim] - 1 + dd->nc[dim]) % dd->nc[dim];
+ dd->neighbor[d][1] = ddcoord2ddnodeid(dd, tmp);
+ if (debug)
+ {
+ fprintf(debug, "DD rank %d neighbor ranks in dir %d are + %d - %d\n",
+ dd->rank, dim,
+ dd->neighbor[d][0],
+ dd->neighbor[d][1]);
+ }
+ }
+
+ if (fplog)
+ {
+ fprintf(fplog, "\nMaking %dD domain decomposition grid %d x %d x %d, home cell index %d %d %d\n\n",
+ dd->ndim,
+ dd->nc[XX], dd->nc[YY], dd->nc[ZZ],
+ dd->ci[XX], dd->ci[YY], dd->ci[ZZ]);
+ }
+ switch (dd->ndim)
+ {
+ case 3:
+ nzone = dd_z3n;
+ nzonep = dd_zp3n;
+ for (i = 0; i < nzonep; i++)
+ {
+ copy_ivec(dd_zp3[i], dd_zp[i]);
+ }
+ break;
+ case 2:
+ nzone = dd_z2n;
+ nzonep = dd_zp2n;
+ for (i = 0; i < nzonep; i++)
+ {
+ copy_ivec(dd_zp2[i], dd_zp[i]);
+ }
+ break;
+ case 1:
+ nzone = dd_z1n;
+ nzonep = dd_zp1n;
+ for (i = 0; i < nzonep; i++)
+ {
+ copy_ivec(dd_zp1[i], dd_zp[i]);
+ }
+ break;
+ default:
+ gmx_fatal(FARGS, "Can only do 1, 2 or 3D domain decomposition");
+ nzone = 0;
+ nzonep = 0;
+ }
+
+ zones = &dd->comm->zones;
+
+ for (i = 0; i < nzone; i++)
+ {
+ m = 0;
+ clear_ivec(zones->shift[i]);
+ for (d = 0; d < dd->ndim; d++)
+ {
+ zones->shift[i][dd->dim[d]] = dd_zo[i][m++];
+ }
+ }
+
+ zones->n = nzone;
+ for (i = 0; i < nzone; i++)
+ {
+ for (d = 0; d < DIM; d++)
+ {
+ s[d] = dd->ci[d] - zones->shift[i][d];
+ if (s[d] < 0)
+ {
+ s[d] += dd->nc[d];
+ }
+ else if (s[d] >= dd->nc[d])
+ {
+ s[d] -= dd->nc[d];
+ }
+ }
+ }
+ zones->nizone = nzonep;
+ for (i = 0; i < zones->nizone; i++)
+ {
+ if (dd_zp[i][0] != i)
+ {
+ gmx_fatal(FARGS, "Internal inconsistency in the dd grid setup");
+ }
+ izone = &zones->izone[i];
+ izone->j0 = dd_zp[i][1];
+ izone->j1 = dd_zp[i][2];
+ for (dim = 0; dim < DIM; dim++)
+ {
+ if (dd->nc[dim] == 1)
+ {
+ /* All shifts should be allowed */
+ izone->shift0[dim] = -1;
+ izone->shift1[dim] = 1;
+ }
+ else
+ {
+ /*
+ izone->shift0[d] = 0;
+ izone->shift1[d] = 0;
+ for(j=izone->j0; j<izone->j1; j++) {
+ if (dd->shift[j][d] > dd->shift[i][d])
+ izone->shift0[d] = -1;
+ if (dd->shift[j][d] < dd->shift[i][d])
+ izone->shift1[d] = 1;
+ }
+ */
+
+ int shift_diff;
+
+ /* Assume the shift are not more than 1 cell */
+ izone->shift0[dim] = 1;
+ izone->shift1[dim] = -1;
+ for (j = izone->j0; j < izone->j1; j++)
+ {
+ shift_diff = zones->shift[j][dim] - zones->shift[i][dim];
+ if (shift_diff < izone->shift0[dim])
+ {
+ izone->shift0[dim] = shift_diff;
+ }
+ if (shift_diff > izone->shift1[dim])
+ {
+ izone->shift1[dim] = shift_diff;
+ }
+ }
+ }
+ }
+ }
+
+ if (dd->comm->eDLB != edlbNO)
+ {
+ snew(dd->comm->root, dd->ndim);
+ }
+
+ if (dd->comm->bRecordLoad)
+ {
+ make_load_communicators(dd);
+ }
+}
+
+static void make_pp_communicator(FILE *fplog, t_commrec *cr, int reorder)
+{
+ gmx_domdec_t *dd;
+ gmx_domdec_comm_t *comm;
+ int i, rank, *buf;
+ ivec periods;
+#ifdef GMX_MPI
+ MPI_Comm comm_cart;
+#endif
+
+ dd = cr->dd;
+ comm = dd->comm;
+
+#ifdef GMX_MPI
+ if (comm->bCartesianPP)
+ {
+ /* Set up cartesian communication for the particle-particle part */
+ if (fplog)
+ {
+ fprintf(fplog, "Will use a Cartesian communicator: %d x %d x %d\n",
+ dd->nc[XX], dd->nc[YY], dd->nc[ZZ]);
+ }
+
+ for (i = 0; i < DIM; i++)
+ {
+ periods[i] = TRUE;
+ }
+ MPI_Cart_create(cr->mpi_comm_mygroup, DIM, dd->nc, periods, reorder,
+ &comm_cart);
+ /* We overwrite the old communicator with the new cartesian one */
+ cr->mpi_comm_mygroup = comm_cart;
+ }
+
+ dd->mpi_comm_all = cr->mpi_comm_mygroup;
+ MPI_Comm_rank(dd->mpi_comm_all, &dd->rank);
+
+ if (comm->bCartesianPP_PME)
+ {
+ /* Since we want to use the original cartesian setup for sim,
+ * and not the one after split, we need to make an index.
+ */
+ snew(comm->ddindex2ddnodeid, dd->nnodes);
+ comm->ddindex2ddnodeid[dd_index(dd->nc, dd->ci)] = dd->rank;
+ gmx_sumi(dd->nnodes, comm->ddindex2ddnodeid, cr);
+ /* Get the rank of the DD master,
+ * above we made sure that the master node is a PP node.
+ */
+ if (MASTER(cr))
+ {
+ rank = dd->rank;
+ }
+ else
+ {
+ rank = 0;
+ }
+ MPI_Allreduce(&rank, &dd->masterrank, 1, MPI_INT, MPI_SUM, dd->mpi_comm_all);
+ }
+ else if (comm->bCartesianPP)
+ {
+ if (cr->npmenodes == 0)
+ {
+ /* The PP communicator is also
+ * the communicator for this simulation
+ */
+ cr->mpi_comm_mysim = cr->mpi_comm_mygroup;
+ }
+ cr->nodeid = dd->rank;
+
+ MPI_Cart_coords(dd->mpi_comm_all, dd->rank, DIM, dd->ci);
+
+ /* We need to make an index to go from the coordinates
+ * to the nodeid of this simulation.
+ */
+ snew(comm->ddindex2simnodeid, dd->nnodes);
+ snew(buf, dd->nnodes);
+ if (cr->duty & DUTY_PP)
+ {
+ buf[dd_index(dd->nc, dd->ci)] = cr->sim_nodeid;
+ }
+ /* Communicate the ddindex to simulation nodeid index */
+ MPI_Allreduce(buf, comm->ddindex2simnodeid, dd->nnodes, MPI_INT, MPI_SUM,
+ cr->mpi_comm_mysim);
+ sfree(buf);
+
+ /* Determine the master coordinates and rank.
+ * The DD master should be the same node as the master of this sim.
+ */
+ for (i = 0; i < dd->nnodes; i++)
+ {
+ if (comm->ddindex2simnodeid[i] == 0)
+ {
+ ddindex2xyz(dd->nc, i, dd->master_ci);
+ MPI_Cart_rank(dd->mpi_comm_all, dd->master_ci, &dd->masterrank);
+ }
+ }
+ if (debug)
+ {
+ fprintf(debug, "The master rank is %d\n", dd->masterrank);
+ }
+ }
+ else
+ {
+ /* No Cartesian communicators */
+ /* We use the rank in dd->comm->all as DD index */
+ ddindex2xyz(dd->nc, dd->rank, dd->ci);
+ /* The simulation master nodeid is 0, so the DD master rank is also 0 */
+ dd->masterrank = 0;
+ clear_ivec(dd->master_ci);
+ }
+#endif
+
+ if (fplog)
+ {
+ fprintf(fplog,
+ "Domain decomposition nodeid %d, coordinates %d %d %d\n\n",
+ dd->rank, dd->ci[XX], dd->ci[YY], dd->ci[ZZ]);
+ }
+ if (debug)
+ {
+ fprintf(debug,
+ "Domain decomposition nodeid %d, coordinates %d %d %d\n\n",
+ dd->rank, dd->ci[XX], dd->ci[YY], dd->ci[ZZ]);
+ }
+}
+
+static void receive_ddindex2simnodeid(t_commrec *cr)
+{
+ gmx_domdec_t *dd;
+
+ gmx_domdec_comm_t *comm;
+ int *buf;
+
+ dd = cr->dd;
+ comm = dd->comm;
+
+#ifdef GMX_MPI
+ if (!comm->bCartesianPP_PME && comm->bCartesianPP)
+ {
+ snew(comm->ddindex2simnodeid, dd->nnodes);
+ snew(buf, dd->nnodes);
+ if (cr->duty & DUTY_PP)
+ {
+ buf[dd_index(dd->nc, dd->ci)] = cr->sim_nodeid;
+ }
+#ifdef GMX_MPI
+ /* Communicate the ddindex to simulation nodeid index */
+ MPI_Allreduce(buf, comm->ddindex2simnodeid, dd->nnodes, MPI_INT, MPI_SUM,
+ cr->mpi_comm_mysim);
+#endif
+ sfree(buf);
+ }
+#endif
+}
+
+static gmx_domdec_master_t *init_gmx_domdec_master_t(gmx_domdec_t *dd,
+ int ncg, int natoms)
+{
+ gmx_domdec_master_t *ma;
+ int i;
+
+ snew(ma, 1);
+
+ snew(ma->ncg, dd->nnodes);
+ snew(ma->index, dd->nnodes+1);
+ snew(ma->cg, ncg);
+ snew(ma->nat, dd->nnodes);
+ snew(ma->ibuf, dd->nnodes*2);
+ snew(ma->cell_x, DIM);
+ for (i = 0; i < DIM; i++)
+ {
+ snew(ma->cell_x[i], dd->nc[i]+1);
+ }
+
+ if (dd->nnodes <= GMX_DD_NNODES_SENDRECV)
+ {
+ ma->vbuf = NULL;
+ }
+ else
+ {
+ snew(ma->vbuf, natoms);
+ }
+
+ return ma;
+}
+
+static void split_communicator(FILE *fplog, t_commrec *cr, int dd_node_order,
+ int reorder)
+{
+ gmx_domdec_t *dd;
+ gmx_domdec_comm_t *comm;
+ int i, rank;
+ gmx_bool bDiv[DIM];
+ ivec periods;
+#ifdef GMX_MPI
+ MPI_Comm comm_cart;
+#endif
+
+ dd = cr->dd;
+ comm = dd->comm;
+
+ if (comm->bCartesianPP)
+ {
+ for (i = 1; i < DIM; i++)
+ {
+ bDiv[i] = ((cr->npmenodes*dd->nc[i]) % (dd->nnodes) == 0);
+ }
+ if (bDiv[YY] || bDiv[ZZ])
+ {
+ comm->bCartesianPP_PME = TRUE;
+ /* If we have 2D PME decomposition, which is always in x+y,
+ * we stack the PME only nodes in z.
+ * Otherwise we choose the direction that provides the thinnest slab
+ * of PME only nodes as this will have the least effect
+ * on the PP communication.
+ * But for the PME communication the opposite might be better.
+ */
+ if (bDiv[ZZ] && (comm->npmenodes_y > 1 ||
+ !bDiv[YY] ||
+ dd->nc[YY] > dd->nc[ZZ]))
+ {
+ comm->cartpmedim = ZZ;
+ }
+ else
+ {
+ comm->cartpmedim = YY;
+ }
+ comm->ntot[comm->cartpmedim]
+ += (cr->npmenodes*dd->nc[comm->cartpmedim])/dd->nnodes;
+ }
+ else if (fplog)
+ {
+ fprintf(fplog, "#pmenodes (%d) is not a multiple of nx*ny (%d*%d) or nx*nz (%d*%d)\n", cr->npmenodes, dd->nc[XX], dd->nc[YY], dd->nc[XX], dd->nc[ZZ]);
+ fprintf(fplog,
+ "Will not use a Cartesian communicator for PP <-> PME\n\n");
+ }
+ }
+
+#ifdef GMX_MPI
+ if (comm->bCartesianPP_PME)
+ {
+ if (fplog)
+ {
+ fprintf(fplog, "Will use a Cartesian communicator for PP <-> PME: %d x %d x %d\n", comm->ntot[XX], comm->ntot[YY], comm->ntot[ZZ]);
+ }
+
+ for (i = 0; i < DIM; i++)
+ {
+ periods[i] = TRUE;
+ }
+ MPI_Cart_create(cr->mpi_comm_mysim, DIM, comm->ntot, periods, reorder,
+ &comm_cart);
+
+ MPI_Comm_rank(comm_cart, &rank);
+ if (MASTERNODE(cr) && rank != 0)
+ {
+ gmx_fatal(FARGS, "MPI rank 0 was renumbered by MPI_Cart_create, we do not allow this");
+ }
+
+ /* With this assigment we loose the link to the original communicator
+ * which will usually be MPI_COMM_WORLD, unless have multisim.
+ */
+ cr->mpi_comm_mysim = comm_cart;
+ cr->sim_nodeid = rank;
+
+ MPI_Cart_coords(cr->mpi_comm_mysim, cr->sim_nodeid, DIM, dd->ci);
+
+ if (fplog)
+ {
+ fprintf(fplog, "Cartesian nodeid %d, coordinates %d %d %d\n\n",
+ cr->sim_nodeid, dd->ci[XX], dd->ci[YY], dd->ci[ZZ]);
+ }
+
+ if (dd->ci[comm->cartpmedim] < dd->nc[comm->cartpmedim])
+ {
+ cr->duty = DUTY_PP;
+ }
+ if (cr->npmenodes == 0 ||
+ dd->ci[comm->cartpmedim] >= dd->nc[comm->cartpmedim])
+ {
+ cr->duty = DUTY_PME;
+ }
+
+ /* Split the sim communicator into PP and PME only nodes */
+ MPI_Comm_split(cr->mpi_comm_mysim,
+ cr->duty,
+ dd_index(comm->ntot, dd->ci),
+ &cr->mpi_comm_mygroup);
+ }
+ else
+ {
+ switch (dd_node_order)
+ {
+ case ddnoPP_PME:
+ if (fplog)
+ {
+ fprintf(fplog, "Order of the nodes: PP first, PME last\n");
+ }
+ break;
+ case ddnoINTERLEAVE:
+ /* Interleave the PP-only and PME-only nodes,
+ * as on clusters with dual-core machines this will double
+ * the communication bandwidth of the PME processes
+ * and thus speed up the PP <-> PME and inter PME communication.
+ */
+ if (fplog)
+ {
+ fprintf(fplog, "Interleaving PP and PME nodes\n");
+ }
+ comm->pmenodes = dd_pmenodes(cr);
+ break;
+ case ddnoCARTESIAN:
+ break;
+ default:
+ gmx_fatal(FARGS, "Unknown dd_node_order=%d", dd_node_order);
+ }
+
+ if (dd_simnode2pmenode(cr, cr->sim_nodeid) == -1)
+ {
+ cr->duty = DUTY_PME;
+ }
+ else
+ {
+ cr->duty = DUTY_PP;
+ }
+
+ /* Split the sim communicator into PP and PME only nodes */
+ MPI_Comm_split(cr->mpi_comm_mysim,
+ cr->duty,
+ cr->nodeid,
+ &cr->mpi_comm_mygroup);
+ MPI_Comm_rank(cr->mpi_comm_mygroup, &cr->nodeid);
+ }
+#endif
+
+ if (fplog)
+ {
+ fprintf(fplog, "This is a %s only node\n\n",
+ (cr->duty & DUTY_PP) ? "particle-particle" : "PME-mesh");
+ }
+}
+
+void make_dd_communicators(FILE *fplog, t_commrec *cr, int dd_node_order)
+{
+ gmx_domdec_t *dd;
+ gmx_domdec_comm_t *comm;
+ int CartReorder;
+
+ dd = cr->dd;
+ comm = dd->comm;
+
+ copy_ivec(dd->nc, comm->ntot);
+
+ comm->bCartesianPP = (dd_node_order == ddnoCARTESIAN);
+ comm->bCartesianPP_PME = FALSE;
+
+ /* Reorder the nodes by default. This might change the MPI ranks.
+ * Real reordering is only supported on very few architectures,
+ * Blue Gene is one of them.
+ */
+ CartReorder = (getenv("GMX_NO_CART_REORDER") == NULL);
+
+ if (cr->npmenodes > 0)
+ {
+ /* Split the communicator into a PP and PME part */
+ split_communicator(fplog, cr, dd_node_order, CartReorder);
+ if (comm->bCartesianPP_PME)
+ {
+ /* We (possibly) reordered the nodes in split_communicator,
+ * so it is no longer required in make_pp_communicator.
+ */
+ CartReorder = FALSE;
+ }
+ }
+ else
+ {
+ /* All nodes do PP and PME */
+#ifdef GMX_MPI
+ /* We do not require separate communicators */
+ cr->mpi_comm_mygroup = cr->mpi_comm_mysim;
+#endif
+ }
+
+ if (cr->duty & DUTY_PP)
+ {
+ /* Copy or make a new PP communicator */
+ make_pp_communicator(fplog, cr, CartReorder);
+ }
+ else
+ {
+ receive_ddindex2simnodeid(cr);
+ }
+
+ if (!(cr->duty & DUTY_PME))
+ {
+ /* Set up the commnuication to our PME node */
+ dd->pme_nodeid = dd_simnode2pmenode(cr, cr->sim_nodeid);
+ dd->pme_receive_vir_ener = receive_vir_ener(cr);
+ if (debug)
+ {
+ fprintf(debug, "My pme_nodeid %d receive ener %d\n",
+ dd->pme_nodeid, dd->pme_receive_vir_ener);
+ }
+ }
+ else
+ {
+ dd->pme_nodeid = -1;
+ }
+
+ if (DDMASTER(dd))
+ {
+ dd->ma = init_gmx_domdec_master_t(dd,
+ comm->cgs_gl.nr,
+ comm->cgs_gl.index[comm->cgs_gl.nr]);
+ }
+}
+
+static real *get_slb_frac(FILE *fplog, const char *dir, int nc, const char *size_string)
+{
+ real *slb_frac, tot;
+ int i, n;
+ double dbl;
+
+ slb_frac = NULL;
+ if (nc > 1 && size_string != NULL)
+ {
+ if (fplog)
+ {
+ fprintf(fplog, "Using static load balancing for the %s direction\n",
+ dir);
+ }
+ snew(slb_frac, nc);
+ tot = 0;
+ for (i = 0; i < nc; i++)
+ {
+ dbl = 0;
+ sscanf(size_string, "%lf%n", &dbl, &n);
+ if (dbl == 0)
+ {
+ gmx_fatal(FARGS, "Incorrect or not enough DD cell size entries for direction %s: '%s'", dir, size_string);
+ }
+ slb_frac[i] = dbl;
+ size_string += n;
+ tot += slb_frac[i];
+ }
+ /* Normalize */
+ if (fplog)
+ {
+ fprintf(fplog, "Relative cell sizes:");
+ }
+ for (i = 0; i < nc; i++)
+ {
+ slb_frac[i] /= tot;
+ if (fplog)
+ {
+ fprintf(fplog, " %5.3f", slb_frac[i]);
+ }
+ }
+ if (fplog)
+ {
+ fprintf(fplog, "\n");
+ }
+ }
+
+ return slb_frac;
+}
+
+static int multi_body_bondeds_count(gmx_mtop_t *mtop)
+{
+ int n, nmol, ftype;
+ gmx_mtop_ilistloop_t iloop;
+ t_ilist *il;
+
+ n = 0;
+ iloop = gmx_mtop_ilistloop_init(mtop);
+ while (gmx_mtop_ilistloop_next(iloop, &il, &nmol))
+ {
+ for (ftype = 0; ftype < F_NRE; ftype++)
+ {
+ if ((interaction_function[ftype].flags & IF_BOND) &&
+ NRAL(ftype) > 2)
+ {
+ n += nmol*il[ftype].nr/(1 + NRAL(ftype));
+ }
+ }
+ }
+
+ return n;
+}
+
+static int dd_nst_env(FILE *fplog, const char *env_var, int def)
+{
+ char *val;
+ int nst;
+
+ nst = def;
+ val = getenv(env_var);
+ if (val)
+ {
+ if (sscanf(val, "%d", &nst) <= 0)
+ {
+ nst = 1;
+ }
+ if (fplog)
+ {
+ fprintf(fplog, "Found env.var. %s = %s, using value %d\n",
+ env_var, val, nst);
+ }
+ }
+
+ return nst;
+}
+
+static void dd_warning(t_commrec *cr, FILE *fplog, const char *warn_string)
+{
+ if (MASTER(cr))
+ {
+ fprintf(stderr, "\n%s\n", warn_string);
+ }
+ if (fplog)
+ {
+ fprintf(fplog, "\n%s\n", warn_string);
+ }
+}
+
+static void check_dd_restrictions(t_commrec *cr, gmx_domdec_t *dd,
+ t_inputrec *ir, FILE *fplog)
+{
+ if (ir->ePBC == epbcSCREW &&
+ (dd->nc[XX] == 1 || dd->nc[YY] > 1 || dd->nc[ZZ] > 1))
+ {
+ gmx_fatal(FARGS, "With pbc=%s can only do domain decomposition in the x-direction", epbc_names[ir->ePBC]);
+ }
+
+ if (ir->ns_type == ensSIMPLE)
+ {
+ gmx_fatal(FARGS, "Domain decomposition does not support simple neighbor searching, use grid searching or use particle decomposition");
+ }
+
+ if (ir->nstlist == 0)
+ {
+ gmx_fatal(FARGS, "Domain decomposition does not work with nstlist=0");
+ }
+
+ if (ir->comm_mode == ecmANGULAR && ir->ePBC != epbcNONE)
+ {
+ dd_warning(cr, fplog, "comm-mode angular will give incorrect results when the comm group partially crosses a periodic boundary");
+ }
+}
+
+static real average_cellsize_min(gmx_domdec_t *dd, gmx_ddbox_t *ddbox)
+{
+ int di, d;
+ real r;
+
+ r = ddbox->box_size[XX];
+ for (di = 0; di < dd->ndim; di++)
+ {
+ d = dd->dim[di];
+ /* Check using the initial average cell size */
+ r = min(r, ddbox->box_size[d]*ddbox->skew_fac[d]/dd->nc[d]);
+ }
+
+ return r;
+}
+
+static int check_dlb_support(FILE *fplog, t_commrec *cr,
+ const char *dlb_opt, gmx_bool bRecordLoad,
+ unsigned long Flags, t_inputrec *ir)
+{
+ gmx_domdec_t *dd;
+ int eDLB = -1;
+ char buf[STRLEN];
+
+ switch (dlb_opt[0])
+ {
+ case 'a': eDLB = edlbAUTO; break;
+ case 'n': eDLB = edlbNO; break;
+ case 'y': eDLB = edlbYES; break;
+ default: gmx_incons("Unknown dlb_opt");
+ }
+
+ if (Flags & MD_RERUN)
+ {
+ return edlbNO;
+ }
+
+ if (!EI_DYNAMICS(ir->eI))
+ {
+ if (eDLB == edlbYES)
+ {
+ sprintf(buf, "NOTE: dynamic load balancing is only supported with dynamics, not with integrator '%s'\n", EI(ir->eI));
+ dd_warning(cr, fplog, buf);
+ }
+
+ return edlbNO;
+ }
+
+ if (!bRecordLoad)
+ {
+ dd_warning(cr, fplog, "NOTE: Cycle counting is not supported on this architecture, will not use dynamic load balancing\n");
+
+ return edlbNO;
+ }
+
+ if (Flags & MD_REPRODUCIBLE)
+ {
+ switch (eDLB)
+ {
+ case edlbNO:
+ break;
+ case edlbAUTO:
+ dd_warning(cr, fplog, "NOTE: reproducibility requested, will not use dynamic load balancing\n");
+ eDLB = edlbNO;
+ break;
+ case edlbYES:
+ dd_warning(cr, fplog, "WARNING: reproducibility requested with dynamic load balancing, the simulation will NOT be binary reproducible\n");
+ break;
+ default:
+ gmx_fatal(FARGS, "Death horror: undefined case (%d) for load balancing choice", eDLB);
+ break;
+ }
+ }
+
+ return eDLB;
+}
+
+static void set_dd_dim(FILE *fplog, gmx_domdec_t *dd)
+{
+ int dim;
+
+ dd->ndim = 0;
+ if (getenv("GMX_DD_ORDER_ZYX") != NULL)
+ {
+ /* Decomposition order z,y,x */
+ if (fplog)
+ {
+ fprintf(fplog, "Using domain decomposition order z, y, x\n");
+ }
+ for (dim = DIM-1; dim >= 0; dim--)
+ {
+ if (dd->nc[dim] > 1)
+ {
+ dd->dim[dd->ndim++] = dim;
+ }
+ }
+ }
+ else
+ {
+ /* Decomposition order x,y,z */
+ for (dim = 0; dim < DIM; dim++)
+ {
+ if (dd->nc[dim] > 1)
+ {
+ dd->dim[dd->ndim++] = dim;
+ }
+ }
+ }
+}
+
+static gmx_domdec_comm_t *init_dd_comm()
+{
+ gmx_domdec_comm_t *comm;
+ int i;
+
+ snew(comm, 1);
+ snew(comm->cggl_flag, DIM*2);
+ snew(comm->cgcm_state, DIM*2);
+ for (i = 0; i < DIM*2; i++)
+ {
+ comm->cggl_flag_nalloc[i] = 0;
+ comm->cgcm_state_nalloc[i] = 0;
+ }
+
+ comm->nalloc_int = 0;
+ comm->buf_int = NULL;
+
+ vec_rvec_init(&comm->vbuf);
+
+ comm->n_load_have = 0;
+ comm->n_load_collect = 0;
+
+ for (i = 0; i < ddnatNR-ddnatZONE; i++)
+ {
+ comm->sum_nat[i] = 0;
+ }
+ comm->ndecomp = 0;
+ comm->nload = 0;
+ comm->load_step = 0;
+ comm->load_sum = 0;
+ comm->load_max = 0;
+ clear_ivec(comm->load_lim);
+ comm->load_mdf = 0;
+ comm->load_pme = 0;
+
+ return comm;
+}
+
+gmx_domdec_t *init_domain_decomposition(FILE *fplog, t_commrec *cr,
+ unsigned long Flags,
+ ivec nc,
+ real comm_distance_min, real rconstr,
+ const char *dlb_opt, real dlb_scale,
+ const char *sizex, const char *sizey, const char *sizez,
+ gmx_mtop_t *mtop, t_inputrec *ir,
+ matrix box, rvec *x,
+ gmx_ddbox_t *ddbox,
+ int *npme_x, int *npme_y)
+{
+ gmx_domdec_t *dd;
+ gmx_domdec_comm_t *comm;
+ int recload;
+ int d, i, j;
+ real r_2b, r_mb, r_bonded = -1, r_bonded_limit = -1, limit, acs;
+ gmx_bool bC;
+ char buf[STRLEN];
+
+ if (fplog)
+ {
+ fprintf(fplog,
+ "\nInitializing Domain Decomposition on %d nodes\n", cr->nnodes);
+ }
+
+ snew(dd, 1);
+
+ dd->comm = init_dd_comm();
+ comm = dd->comm;
+ snew(comm->cggl_flag, DIM*2);
+ snew(comm->cgcm_state, DIM*2);
+
+ dd->npbcdim = ePBC2npbcdim(ir->ePBC);
+ dd->bScrewPBC = (ir->ePBC == epbcSCREW);
+
+ dd->bSendRecv2 = dd_nst_env(fplog, "GMX_DD_SENDRECV2", 0);
+ comm->dlb_scale_lim = dd_nst_env(fplog, "GMX_DLB_MAX", 10);
+ comm->eFlop = dd_nst_env(fplog, "GMX_DLB_FLOP", 0);
+ recload = dd_nst_env(fplog, "GMX_DD_LOAD", 1);
+ comm->nstSortCG = dd_nst_env(fplog, "GMX_DD_SORT", 1);
+ comm->nstDDDump = dd_nst_env(fplog, "GMX_DD_DUMP", 0);
+ comm->nstDDDumpGrid = dd_nst_env(fplog, "GMX_DD_DUMP_GRID", 0);
+ comm->DD_debug = dd_nst_env(fplog, "GMX_DD_DEBUG", 0);
+
+ dd->pme_recv_f_alloc = 0;
+ dd->pme_recv_f_buf = NULL;
+
+ if (dd->bSendRecv2 && fplog)
+ {
+ fprintf(fplog, "Will use two sequential MPI_Sendrecv calls instead of two simultaneous non-blocking MPI_Irecv and MPI_Isend pairs for constraint and vsite communication\n");
+ }
+ if (comm->eFlop)
+ {
+ if (fplog)
+ {
+ fprintf(fplog, "Will load balance based on FLOP count\n");
+ }
+ if (comm->eFlop > 1)
+ {
+ srand(1+cr->nodeid);
+ }
+ comm->bRecordLoad = TRUE;
+ }
+ else
+ {
+ comm->bRecordLoad = (wallcycle_have_counter() && recload > 0);
+
+ }
+
++ /* Initialize to GPU share count to 0, might change later */
++ comm->nrank_gpu_shared = 0;
++
+ comm->eDLB = check_dlb_support(fplog, cr, dlb_opt, comm->bRecordLoad, Flags, ir);
+
+ comm->bDynLoadBal = (comm->eDLB == edlbYES);
+ if (fplog)
+ {
+ fprintf(fplog, "Dynamic load balancing: %s\n", edlb_names[comm->eDLB]);
+ }
+ dd->bGridJump = comm->bDynLoadBal;
+ comm->bPMELoadBalDLBLimits = FALSE;
+
+ if (comm->nstSortCG)
+ {
+ if (fplog)
+ {
+ if (comm->nstSortCG == 1)
+ {
+ fprintf(fplog, "Will sort the charge groups at every domain (re)decomposition\n");
+ }
+ else
+ {
+ fprintf(fplog, "Will sort the charge groups every %d steps\n",
+ comm->nstSortCG);
+ }
+ }
+ snew(comm->sort, 1);
+ }
+ else
+ {
+ if (fplog)
+ {
+ fprintf(fplog, "Will not sort the charge groups\n");
+ }
+ }
+
+ comm->bCGs = (ncg_mtop(mtop) < mtop->natoms);
+
+ comm->bInterCGBondeds = (ncg_mtop(mtop) > mtop->mols.nr);
+ if (comm->bInterCGBondeds)
+ {
+ comm->bInterCGMultiBody = (multi_body_bondeds_count(mtop) > 0);
+ }
+ else
+ {
+ comm->bInterCGMultiBody = FALSE;
+ }
+
+ dd->bInterCGcons = inter_charge_group_constraints(mtop);
+ dd->bInterCGsettles = inter_charge_group_settles(mtop);
+
+ if (ir->rlistlong == 0)
+ {
+ /* Set the cut-off to some very large value,
+ * so we don't need if statements everywhere in the code.
+ * We use sqrt, since the cut-off is squared in some places.
+ */
+ comm->cutoff = GMX_CUTOFF_INF;
+ }
+ else
+ {
+ comm->cutoff = ir->rlistlong;
+ }
+ comm->cutoff_mbody = 0;
+
+ comm->cellsize_limit = 0;
+ comm->bBondComm = FALSE;
+
+ if (comm->bInterCGBondeds)
+ {
+ if (comm_distance_min > 0)
+ {
+ comm->cutoff_mbody = comm_distance_min;
+ if (Flags & MD_DDBONDCOMM)
+ {
+ comm->bBondComm = (comm->cutoff_mbody > comm->cutoff);
+ }
+ else
+ {
+ comm->cutoff = max(comm->cutoff, comm->cutoff_mbody);
+ }
+ r_bonded_limit = comm->cutoff_mbody;
+ }
+ else if (ir->bPeriodicMols)
+ {
+ /* Can not easily determine the required cut-off */
+ dd_warning(cr, fplog, "NOTE: Periodic molecules are present in this system. Because of this, the domain decomposition algorithm cannot easily determine the minimum cell size that it requires for treating bonded interactions. Instead, domain decomposition will assume that half the non-bonded cut-off will be a suitable lower bound.\n");
+ comm->cutoff_mbody = comm->cutoff/2;
+ r_bonded_limit = comm->cutoff_mbody;
+ }
+ else
+ {
+ if (MASTER(cr))
+ {
+ dd_bonded_cg_distance(fplog, mtop, ir, x, box,
+ Flags & MD_DDBONDCHECK, &r_2b, &r_mb);
+ }
+ gmx_bcast(sizeof(r_2b), &r_2b, cr);
+ gmx_bcast(sizeof(r_mb), &r_mb, cr);
+
+ /* We use an initial margin of 10% for the minimum cell size,
+ * except when we are just below the non-bonded cut-off.
+ */
+ if (Flags & MD_DDBONDCOMM)
+ {
+ if (max(r_2b, r_mb) > comm->cutoff)
+ {
+ r_bonded = max(r_2b, r_mb);
+ r_bonded_limit = 1.1*r_bonded;
+ comm->bBondComm = TRUE;
+ }
+ else
+ {
+ r_bonded = r_mb;
+ r_bonded_limit = min(1.1*r_bonded, comm->cutoff);
+ }
+ /* We determine cutoff_mbody later */
+ }
+ else
+ {
+ /* No special bonded communication,
+ * simply increase the DD cut-off.
+ */
+ r_bonded_limit = 1.1*max(r_2b, r_mb);
+ comm->cutoff_mbody = r_bonded_limit;
+ comm->cutoff = max(comm->cutoff, comm->cutoff_mbody);
+ }
+ }
+ comm->cellsize_limit = max(comm->cellsize_limit, r_bonded_limit);
+ if (fplog)
+ {
+ fprintf(fplog,
+ "Minimum cell size due to bonded interactions: %.3f nm\n",
+ comm->cellsize_limit);
+ }
+ }
+
+ if (dd->bInterCGcons && rconstr <= 0)
+ {
+ /* There is a cell size limit due to the constraints (P-LINCS) */
+ rconstr = constr_r_max(fplog, mtop, ir);
+ if (fplog)
+ {
+ fprintf(fplog,
+ "Estimated maximum distance required for P-LINCS: %.3f nm\n",
+ rconstr);
+ if (rconstr > comm->cellsize_limit)
+ {
+ fprintf(fplog, "This distance will limit the DD cell size, you can override this with -rcon\n");
+ }
+ }
+ }
+ else if (rconstr > 0 && fplog)
+ {
+ /* Here we do not check for dd->bInterCGcons,
+ * because one can also set a cell size limit for virtual sites only
+ * and at this point we don't know yet if there are intercg v-sites.
+ */
+ fprintf(fplog,
+ "User supplied maximum distance required for P-LINCS: %.3f nm\n",
+ rconstr);
+ }
+ comm->cellsize_limit = max(comm->cellsize_limit, rconstr);
+
+ comm->cgs_gl = gmx_mtop_global_cgs(mtop);
+
+ if (nc[XX] > 0)
+ {
+ copy_ivec(nc, dd->nc);
+ set_dd_dim(fplog, dd);
+ set_ddbox_cr(cr, &dd->nc, ir, box, &comm->cgs_gl, x, ddbox);
+
+ if (cr->npmenodes == -1)
+ {
+ cr->npmenodes = 0;
+ }
+ acs = average_cellsize_min(dd, ddbox);
+ if (acs < comm->cellsize_limit)
+ {
+ if (fplog)
+ {
+ fprintf(fplog, "ERROR: The initial cell size (%f) is smaller than the cell size limit (%f)\n", acs, comm->cellsize_limit);
+ }
+ gmx_fatal_collective(FARGS, cr, NULL,
+ "The initial cell size (%f) is smaller than the cell size limit (%f), change options -dd, -rdd or -rcon, see the log file for details",
+ acs, comm->cellsize_limit);
+ }
+ }
+ else
+ {
+ set_ddbox_cr(cr, NULL, ir, box, &comm->cgs_gl, x, ddbox);
+
+ /* We need to choose the optimal DD grid and possibly PME nodes */
+ limit = dd_choose_grid(fplog, cr, dd, ir, mtop, box, ddbox,
+ comm->eDLB != edlbNO, dlb_scale,
+ comm->cellsize_limit, comm->cutoff,
+ comm->bInterCGBondeds);
+
+ if (dd->nc[XX] == 0)
+ {
+ bC = (dd->bInterCGcons && rconstr > r_bonded_limit);
+ sprintf(buf, "Change the number of nodes or mdrun option %s%s%s",
+ !bC ? "-rdd" : "-rcon",
+ comm->eDLB != edlbNO ? " or -dds" : "",
+ bC ? " or your LINCS settings" : "");
+
+ gmx_fatal_collective(FARGS, cr, NULL,
+ "There is no domain decomposition for %d nodes that is compatible with the given box and a minimum cell size of %g nm\n"
+ "%s\n"
+ "Look in the log file for details on the domain decomposition",
+ cr->nnodes-cr->npmenodes, limit, buf);
+ }
+ set_dd_dim(fplog, dd);
+ }
+
+ if (fplog)
+ {
+ fprintf(fplog,
+ "Domain decomposition grid %d x %d x %d, separate PME nodes %d\n",
+ dd->nc[XX], dd->nc[YY], dd->nc[ZZ], cr->npmenodes);
+ }
+
+ dd->nnodes = dd->nc[XX]*dd->nc[YY]*dd->nc[ZZ];
+ if (cr->nnodes - dd->nnodes != cr->npmenodes)
+ {
+ gmx_fatal_collective(FARGS, cr, NULL,
+ "The size of the domain decomposition grid (%d) does not match the number of nodes (%d). The total number of nodes is %d",
+ dd->nnodes, cr->nnodes - cr->npmenodes, cr->nnodes);
+ }
+ if (cr->npmenodes > dd->nnodes)
+ {
+ gmx_fatal_collective(FARGS, cr, NULL,
+ "The number of separate PME nodes (%d) is larger than the number of PP nodes (%d), this is not supported.", cr->npmenodes, dd->nnodes);
+ }
+ if (cr->npmenodes > 0)
+ {
+ comm->npmenodes = cr->npmenodes;
+ }
+ else
+ {
+ comm->npmenodes = dd->nnodes;
+ }
+
+ if (EEL_PME(ir->coulombtype))
+ {
+ /* The following choices should match those
+ * in comm_cost_est in domdec_setup.c.
+ * Note that here the checks have to take into account
+ * that the decomposition might occur in a different order than xyz
+ * (for instance through the env.var. GMX_DD_ORDER_ZYX),
+ * in which case they will not match those in comm_cost_est,
+ * but since that is mainly for testing purposes that's fine.
+ */
+ if (dd->ndim >= 2 && dd->dim[0] == XX && dd->dim[1] == YY &&
+ comm->npmenodes > dd->nc[XX] && comm->npmenodes % dd->nc[XX] == 0 &&
+ getenv("GMX_PMEONEDD") == NULL)
+ {
+ comm->npmedecompdim = 2;
+ comm->npmenodes_x = dd->nc[XX];
+ comm->npmenodes_y = comm->npmenodes/comm->npmenodes_x;
+ }
+ else
+ {
+ /* In case nc is 1 in both x and y we could still choose to
+ * decompose pme in y instead of x, but we use x for simplicity.
+ */
+ comm->npmedecompdim = 1;
+ if (dd->dim[0] == YY)
+ {
+ comm->npmenodes_x = 1;
+ comm->npmenodes_y = comm->npmenodes;
+ }
+ else
+ {
+ comm->npmenodes_x = comm->npmenodes;
+ comm->npmenodes_y = 1;
+ }
+ }
+ if (fplog)
+ {
+ fprintf(fplog, "PME domain decomposition: %d x %d x %d\n",
+ comm->npmenodes_x, comm->npmenodes_y, 1);
+ }
+ }
+ else
+ {
+ comm->npmedecompdim = 0;
+ comm->npmenodes_x = 0;
+ comm->npmenodes_y = 0;
+ }
+
+ /* Technically we don't need both of these,
+ * but it simplifies code not having to recalculate it.
+ */
+ *npme_x = comm->npmenodes_x;
+ *npme_y = comm->npmenodes_y;
+
+ snew(comm->slb_frac, DIM);
+ if (comm->eDLB == edlbNO)
+ {
+ comm->slb_frac[XX] = get_slb_frac(fplog, "x", dd->nc[XX], sizex);
+ comm->slb_frac[YY] = get_slb_frac(fplog, "y", dd->nc[YY], sizey);
+ comm->slb_frac[ZZ] = get_slb_frac(fplog, "z", dd->nc[ZZ], sizez);
+ }
+
+ if (comm->bInterCGBondeds && comm->cutoff_mbody == 0)
+ {
+ if (comm->bBondComm || comm->eDLB != edlbNO)
+ {
+ /* Set the bonded communication distance to halfway
+ * the minimum and the maximum,
+ * since the extra communication cost is nearly zero.
+ */
+ acs = average_cellsize_min(dd, ddbox);
+ comm->cutoff_mbody = 0.5*(r_bonded + acs);
+ if (comm->eDLB != edlbNO)
+ {
+ /* Check if this does not limit the scaling */
+ comm->cutoff_mbody = min(comm->cutoff_mbody, dlb_scale*acs);
+ }
+ if (!comm->bBondComm)
+ {
+ /* Without bBondComm do not go beyond the n.b. cut-off */
+ comm->cutoff_mbody = min(comm->cutoff_mbody, comm->cutoff);
+ if (comm->cellsize_limit >= comm->cutoff)
+ {
+ /* We don't loose a lot of efficieny
+ * when increasing it to the n.b. cut-off.
+ * It can even be slightly faster, because we need
+ * less checks for the communication setup.
+ */
+ comm->cutoff_mbody = comm->cutoff;
+ }
+ }
+ /* Check if we did not end up below our original limit */
+ comm->cutoff_mbody = max(comm->cutoff_mbody, r_bonded_limit);
+
+ if (comm->cutoff_mbody > comm->cellsize_limit)
+ {
+ comm->cellsize_limit = comm->cutoff_mbody;
+ }
+ }
+ /* Without DLB and cutoff_mbody<cutoff, cutoff_mbody is dynamic */
+ }
+
+ if (debug)
+ {
+ fprintf(debug, "Bonded atom communication beyond the cut-off: %d\n"
+ "cellsize limit %f\n",
+ comm->bBondComm, comm->cellsize_limit);
+ }
+
+ if (MASTER(cr))
+ {
+ check_dd_restrictions(cr, dd, ir, fplog);
+ }
+
+ comm->partition_step = INT_MIN;
+ dd->ddp_count = 0;
+
+ clear_dd_cycle_counts(dd);
+
+ return dd;
+}
+
+static void set_dlb_limits(gmx_domdec_t *dd)
+
+{
+ int d;
+
+ for (d = 0; d < dd->ndim; d++)
+ {
+ dd->comm->cd[d].np = dd->comm->cd[d].np_dlb;
+ dd->comm->cellsize_min[dd->dim[d]] =
+ dd->comm->cellsize_min_dlb[dd->dim[d]];
+ }
+}
+
+
+static void turn_on_dlb(FILE *fplog, t_commrec *cr, gmx_large_int_t step)
+{
+ gmx_domdec_t *dd;
+ gmx_domdec_comm_t *comm;
+ real cellsize_min;
+ int d, nc, i;
+ char buf[STRLEN];
+
+ dd = cr->dd;
+ comm = dd->comm;
+
+ if (fplog)
+ {
+ fprintf(fplog, "At step %s the performance loss due to force load imbalance is %.1f %%\n", gmx_step_str(step, buf), dd_force_imb_perf_loss(dd)*100);
+ }
+
+ cellsize_min = comm->cellsize_min[dd->dim[0]];
+ for (d = 1; d < dd->ndim; d++)
+ {
+ cellsize_min = min(cellsize_min, comm->cellsize_min[dd->dim[d]]);
+ }
+
+ if (cellsize_min < comm->cellsize_limit*1.05)
+ {
+ dd_warning(cr, fplog, "NOTE: the minimum cell size is smaller than 1.05 times the cell size limit, will not turn on dynamic load balancing\n");
+
+ /* Change DLB from "auto" to "no". */
+ comm->eDLB = edlbNO;
+
+ return;
+ }
+
+ dd_warning(cr, fplog, "NOTE: Turning on dynamic load balancing\n");
+ comm->bDynLoadBal = TRUE;
+ dd->bGridJump = TRUE;
+
+ set_dlb_limits(dd);
+
+ /* We can set the required cell size info here,
+ * so we do not need to communicate this.
+ * The grid is completely uniform.
+ */
+ for (d = 0; d < dd->ndim; d++)
+ {
+ if (comm->root[d])
+ {
+ comm->load[d].sum_m = comm->load[d].sum;
+
+ nc = dd->nc[dd->dim[d]];
+ for (i = 0; i < nc; i++)
+ {
+ comm->root[d]->cell_f[i] = i/(real)nc;
+ if (d > 0)
+ {
+ comm->root[d]->cell_f_max0[i] = i /(real)nc;
+ comm->root[d]->cell_f_min1[i] = (i+1)/(real)nc;
+ }
+ }
+ comm->root[d]->cell_f[nc] = 1.0;
+ }
+ }
+}
+
+static char *init_bLocalCG(gmx_mtop_t *mtop)
+{
+ int ncg, cg;
+ char *bLocalCG;
+
+ ncg = ncg_mtop(mtop);
+ snew(bLocalCG, ncg);
+ for (cg = 0; cg < ncg; cg++)
+ {
+ bLocalCG[cg] = FALSE;
+ }
+
+ return bLocalCG;
+}
+
+void dd_init_bondeds(FILE *fplog,
+ gmx_domdec_t *dd, gmx_mtop_t *mtop,
+ gmx_vsite_t *vsite,
+ t_inputrec *ir, gmx_bool bBCheck, cginfo_mb_t *cginfo_mb)
+{
+ gmx_domdec_comm_t *comm;
+ gmx_bool bBondComm;
+ int d;
+
+ dd_make_reverse_top(fplog, dd, mtop, vsite, ir, bBCheck);
+
+ comm = dd->comm;
+
+ if (comm->bBondComm)
+ {
+ /* Communicate atoms beyond the cut-off for bonded interactions */
+ comm = dd->comm;
+
+ comm->cglink = make_charge_group_links(mtop, dd, cginfo_mb);
+
+ comm->bLocalCG = init_bLocalCG(mtop);
+ }
+ else
+ {
+ /* Only communicate atoms based on cut-off */
+ comm->cglink = NULL;
+ comm->bLocalCG = NULL;
+ }
+}
+
+static void print_dd_settings(FILE *fplog, gmx_domdec_t *dd,
+ t_inputrec *ir,
+ gmx_bool bDynLoadBal, real dlb_scale,
+ gmx_ddbox_t *ddbox)
+{
+ gmx_domdec_comm_t *comm;
+ int d;
+ ivec np;
+ real limit, shrink;
+ char buf[64];
+
+ if (fplog == NULL)
+ {
+ return;
+ }
+
+ comm = dd->comm;
+
+ if (bDynLoadBal)
+ {
+ fprintf(fplog, "The maximum number of communication pulses is:");
+ for (d = 0; d < dd->ndim; d++)
+ {
+ fprintf(fplog, " %c %d", dim2char(dd->dim[d]), comm->cd[d].np_dlb);
+ }
+ fprintf(fplog, "\n");
+ fprintf(fplog, "The minimum size for domain decomposition cells is %.3f nm\n", comm->cellsize_limit);
+ fprintf(fplog, "The requested allowed shrink of DD cells (option -dds) is: %.2f\n", dlb_scale);
+ fprintf(fplog, "The allowed shrink of domain decomposition cells is:");
+ for (d = 0; d < DIM; d++)
+ {
+ if (dd->nc[d] > 1)
+ {
+ if (d >= ddbox->npbcdim && dd->nc[d] == 2)
+ {
+ shrink = 0;
+ }
+ else
+ {
+ shrink =
+ comm->cellsize_min_dlb[d]/
+ (ddbox->box_size[d]*ddbox->skew_fac[d]/dd->nc[d]);
+ }
+ fprintf(fplog, " %c %.2f", dim2char(d), shrink);
+ }
+ }
+ fprintf(fplog, "\n");
+ }
+ else
+ {
+ set_dd_cell_sizes_slb(dd, ddbox, FALSE, np);
+ fprintf(fplog, "The initial number of communication pulses is:");
+ for (d = 0; d < dd->ndim; d++)
+ {
+ fprintf(fplog, " %c %d", dim2char(dd->dim[d]), np[dd->dim[d]]);
+ }
+ fprintf(fplog, "\n");
+ fprintf(fplog, "The initial domain decomposition cell size is:");
+ for (d = 0; d < DIM; d++)
+ {
+ if (dd->nc[d] > 1)
+ {
+ fprintf(fplog, " %c %.2f nm",
+ dim2char(d), dd->comm->cellsize_min[d]);
+ }
+ }
+ fprintf(fplog, "\n\n");
+ }
+
+ if (comm->bInterCGBondeds || dd->vsite_comm || dd->constraint_comm)
+ {
+ fprintf(fplog, "The maximum allowed distance for charge groups involved in interactions is:\n");
+ fprintf(fplog, "%40s %-7s %6.3f nm\n",
+ "non-bonded interactions", "", comm->cutoff);
+
+ if (bDynLoadBal)
+ {
+ limit = dd->comm->cellsize_limit;
+ }
+ else
+ {
+ if (dynamic_dd_box(ddbox, ir))
+ {
+ fprintf(fplog, "(the following are initial values, they could change due to box deformation)\n");
+ }
+ limit = dd->comm->cellsize_min[XX];
+ for (d = 1; d < DIM; d++)
+ {
+ limit = min(limit, dd->comm->cellsize_min[d]);
+ }
+ }
+
+ if (comm->bInterCGBondeds)
+ {
+ fprintf(fplog, "%40s %-7s %6.3f nm\n",
+ "two-body bonded interactions", "(-rdd)",
+ max(comm->cutoff, comm->cutoff_mbody));
+ fprintf(fplog, "%40s %-7s %6.3f nm\n",
+ "multi-body bonded interactions", "(-rdd)",
+ (comm->bBondComm || dd->bGridJump) ? comm->cutoff_mbody : min(comm->cutoff, limit));
+ }
+ if (dd->vsite_comm)
+ {
+ fprintf(fplog, "%40s %-7s %6.3f nm\n",
+ "virtual site constructions", "(-rcon)", limit);
+ }
+ if (dd->constraint_comm)
+ {
+ sprintf(buf, "atoms separated by up to %d constraints",
+ 1+ir->nProjOrder);
+ fprintf(fplog, "%40s %-7s %6.3f nm\n",
+ buf, "(-rcon)", limit);
+ }
+ fprintf(fplog, "\n");
+ }
+
+ fflush(fplog);
+}
+
+static void set_cell_limits_dlb(gmx_domdec_t *dd,
+ real dlb_scale,
+ const t_inputrec *ir,
+ const gmx_ddbox_t *ddbox)
+{
+ gmx_domdec_comm_t *comm;
+ int d, dim, npulse, npulse_d_max, npulse_d;
+ gmx_bool bNoCutOff;
+
+ comm = dd->comm;
+
+ bNoCutOff = (ir->rvdw == 0 || ir->rcoulomb == 0);
+
+ /* Determine the maximum number of comm. pulses in one dimension */
+
+ comm->cellsize_limit = max(comm->cellsize_limit, comm->cutoff_mbody);
+
+ /* Determine the maximum required number of grid pulses */
+ if (comm->cellsize_limit >= comm->cutoff)
+ {
+ /* Only a single pulse is required */
+ npulse = 1;
+ }
+ else if (!bNoCutOff && comm->cellsize_limit > 0)
+ {
+ /* We round down slightly here to avoid overhead due to the latency
+ * of extra communication calls when the cut-off
+ * would be only slightly longer than the cell size.
+ * Later cellsize_limit is redetermined,
+ * so we can not miss interactions due to this rounding.
+ */
+ npulse = (int)(0.96 + comm->cutoff/comm->cellsize_limit);
+ }
+ else
+ {
+ /* There is no cell size limit */
+ npulse = max(dd->nc[XX]-1, max(dd->nc[YY]-1, dd->nc[ZZ]-1));
+ }
+
+ if (!bNoCutOff && npulse > 1)
+ {
+ /* See if we can do with less pulses, based on dlb_scale */
+ npulse_d_max = 0;
+ for (d = 0; d < dd->ndim; d++)
+ {
+ dim = dd->dim[d];
+ npulse_d = (int)(1 + dd->nc[dim]*comm->cutoff
+ /(ddbox->box_size[dim]*ddbox->skew_fac[dim]*dlb_scale));
+ npulse_d_max = max(npulse_d_max, npulse_d);
+ }
+ npulse = min(npulse, npulse_d_max);
+ }
+
+ /* This env var can override npulse */
+ d = dd_nst_env(debug, "GMX_DD_NPULSE", 0);
+ if (d > 0)
+ {
+ npulse = d;
+ }
+
+ comm->maxpulse = 1;
+ comm->bVacDLBNoLimit = (ir->ePBC == epbcNONE);
+ for (d = 0; d < dd->ndim; d++)
+ {
+ comm->cd[d].np_dlb = min(npulse, dd->nc[dd->dim[d]]-1);
+ comm->cd[d].np_nalloc = comm->cd[d].np_dlb;
+ snew(comm->cd[d].ind, comm->cd[d].np_nalloc);
+ comm->maxpulse = max(comm->maxpulse, comm->cd[d].np_dlb);
+ if (comm->cd[d].np_dlb < dd->nc[dd->dim[d]]-1)
+ {
+ comm->bVacDLBNoLimit = FALSE;
+ }
+ }
+
+ /* cellsize_limit is set for LINCS in init_domain_decomposition */
+ if (!comm->bVacDLBNoLimit)
+ {
+ comm->cellsize_limit = max(comm->cellsize_limit,
+ comm->cutoff/comm->maxpulse);
+ }
+ comm->cellsize_limit = max(comm->cellsize_limit, comm->cutoff_mbody);
+ /* Set the minimum cell size for each DD dimension */
+ for (d = 0; d < dd->ndim; d++)
+ {
+ if (comm->bVacDLBNoLimit ||
+ comm->cd[d].np_dlb*comm->cellsize_limit >= comm->cutoff)
+ {
+ comm->cellsize_min_dlb[dd->dim[d]] = comm->cellsize_limit;
+ }
+ else
+ {
+ comm->cellsize_min_dlb[dd->dim[d]] =
+ comm->cutoff/comm->cd[d].np_dlb;
+ }
+ }
+ if (comm->cutoff_mbody <= 0)
+ {
+ comm->cutoff_mbody = min(comm->cutoff, comm->cellsize_limit);
+ }
+ if (comm->bDynLoadBal)
+ {
+ set_dlb_limits(dd);
+ }
+}
+
+gmx_bool dd_bonded_molpbc(gmx_domdec_t *dd, int ePBC)
+{
+ /* If each molecule is a single charge group
+ * or we use domain decomposition for each periodic dimension,
+ * we do not need to take pbc into account for the bonded interactions.
+ */
+ return (ePBC != epbcNONE && dd->comm->bInterCGBondeds &&
+ !(dd->nc[XX] > 1 &&
+ dd->nc[YY] > 1 &&
+ (dd->nc[ZZ] > 1 || ePBC == epbcXY)));
+}
+
+void set_dd_parameters(FILE *fplog, gmx_domdec_t *dd, real dlb_scale,
+ t_inputrec *ir, gmx_ddbox_t *ddbox)
+{
+ gmx_domdec_comm_t *comm;
+ int natoms_tot;
+ real vol_frac;
+
+ comm = dd->comm;
+
+ /* Initialize the thread data.
+ * This can not be done in init_domain_decomposition,
+ * as the numbers of threads is determined later.
+ */
+ comm->nth = gmx_omp_nthreads_get(emntDomdec);
+ if (comm->nth > 1)
+ {
+ snew(comm->dth, comm->nth);
+ }
+
+ if (EEL_PME(ir->coulombtype))
+ {
+ init_ddpme(dd, &comm->ddpme[0], 0);
+ if (comm->npmedecompdim >= 2)
+ {
+ init_ddpme(dd, &comm->ddpme[1], 1);
+ }
+ }
+ else
+ {
+ comm->npmenodes = 0;
+ if (dd->pme_nodeid >= 0)
+ {
+ gmx_fatal_collective(FARGS, NULL, dd,
+ "Can not have separate PME nodes without PME electrostatics");
+ }
+ }
+
+ if (debug)
+ {
+ fprintf(debug, "The DD cut-off is %f\n", comm->cutoff);
+ }
+ if (comm->eDLB != edlbNO)
+ {
+ set_cell_limits_dlb(dd, dlb_scale, ir, ddbox);
+ }
+
+ print_dd_settings(fplog, dd, ir, comm->bDynLoadBal, dlb_scale, ddbox);
+ if (comm->eDLB == edlbAUTO)
+ {
+ if (fplog)
+ {
+ fprintf(fplog, "When dynamic load balancing gets turned on, these settings will change to:\n");
+ }
+ print_dd_settings(fplog, dd, ir, TRUE, dlb_scale, ddbox);
+ }
+
+ if (ir->ePBC == epbcNONE)
+ {
+ vol_frac = 1 - 1/(double)dd->nnodes;
+ }
+ else
+ {
+ vol_frac =
+ (1 + comm_box_frac(dd->nc, comm->cutoff, ddbox))/(double)dd->nnodes;
+ }
+ if (debug)
+ {
+ fprintf(debug, "Volume fraction for all DD zones: %f\n", vol_frac);
+ }
+ natoms_tot = comm->cgs_gl.index[comm->cgs_gl.nr];
+
+ dd->ga2la = ga2la_init(natoms_tot, vol_frac*natoms_tot);
+}
+
+static gmx_bool test_dd_cutoff(t_commrec *cr,
+ t_state *state, t_inputrec *ir,
+ real cutoff_req)
+{
+ gmx_domdec_t *dd;
+ gmx_ddbox_t ddbox;
+ int d, dim, np;
+ real inv_cell_size;
+ int LocallyLimited;
+
+ dd = cr->dd;
+
+ set_ddbox(dd, FALSE, cr, ir, state->box,
+ TRUE, &dd->comm->cgs_gl, state->x, &ddbox);
+
+ LocallyLimited = 0;
+
+ for (d = 0; d < dd->ndim; d++)
+ {
+ dim = dd->dim[d];
+
+ inv_cell_size = DD_CELL_MARGIN*dd->nc[dim]/ddbox.box_size[dim];
+ if (dynamic_dd_box(&ddbox, ir))
+ {
+ inv_cell_size *= DD_PRES_SCALE_MARGIN;
+ }
+
+ np = 1 + (int)(cutoff_req*inv_cell_size*ddbox.skew_fac[dim]);
+
+ if (dd->comm->eDLB != edlbNO && dim < ddbox.npbcdim &&
+ dd->comm->cd[d].np_dlb > 0)
+ {
+ if (np > dd->comm->cd[d].np_dlb)
+ {
+ return FALSE;
+ }
+
+ /* If a current local cell size is smaller than the requested
+ * cut-off, we could still fix it, but this gets very complicated.
+ * Without fixing here, we might actually need more checks.
+ */
+ if ((dd->comm->cell_x1[dim] - dd->comm->cell_x0[dim])*ddbox.skew_fac[dim]*dd->comm->cd[d].np_dlb < cutoff_req)
+ {
+ LocallyLimited = 1;
+ }
+ }
+ }
+
+ if (dd->comm->eDLB != edlbNO)
+ {
+ /* If DLB is not active yet, we don't need to check the grid jumps.
+ * Actually we shouldn't, because then the grid jump data is not set.
+ */
+ if (dd->comm->bDynLoadBal &&
+ check_grid_jump(0, dd, cutoff_req, &ddbox, FALSE))
+ {
+ LocallyLimited = 1;
+ }
+
+ gmx_sumi(1, &LocallyLimited, cr);
+
+ if (LocallyLimited > 0)
+ {
+ return FALSE;
+ }
+ }
+
+ return TRUE;
+}
+
+gmx_bool change_dd_cutoff(t_commrec *cr, t_state *state, t_inputrec *ir,
+ real cutoff_req)
+{
+ gmx_bool bCutoffAllowed;
+
+ bCutoffAllowed = test_dd_cutoff(cr, state, ir, cutoff_req);
+
+ if (bCutoffAllowed)
+ {
+ cr->dd->comm->cutoff = cutoff_req;
+ }
+
+ return bCutoffAllowed;
+}
+
+void change_dd_dlb_cutoff_limit(t_commrec *cr)
+{
+ gmx_domdec_comm_t *comm;
+
+ comm = cr->dd->comm;
+
+ /* Turn on the DLB limiting (might have been on already) */
+ comm->bPMELoadBalDLBLimits = TRUE;
+
+ /* Change the cut-off limit */
+ comm->PMELoadBal_max_cutoff = comm->cutoff;
+}
+
+static void merge_cg_buffers(int ncell,
+ gmx_domdec_comm_dim_t *cd, int pulse,
+ int *ncg_cell,
+ int *index_gl, int *recv_i,
+ rvec *cg_cm, rvec *recv_vr,
+ int *cgindex,
+ cginfo_mb_t *cginfo_mb, int *cginfo)
+{
+ gmx_domdec_ind_t *ind, *ind_p;
+ int p, cell, c, cg, cg0, cg1, cg_gl, nat;
+ int shift, shift_at;
+
+ ind = &cd->ind[pulse];
+
+ /* First correct the already stored data */
+ shift = ind->nrecv[ncell];
+ for (cell = ncell-1; cell >= 0; cell--)
+ {
+ shift -= ind->nrecv[cell];
+ if (shift > 0)
+ {
+ /* Move the cg's present from previous grid pulses */
+ cg0 = ncg_cell[ncell+cell];
+ cg1 = ncg_cell[ncell+cell+1];
+ cgindex[cg1+shift] = cgindex[cg1];
+ for (cg = cg1-1; cg >= cg0; cg--)
+ {
+ index_gl[cg+shift] = index_gl[cg];
+ copy_rvec(cg_cm[cg], cg_cm[cg+shift]);
+ cgindex[cg+shift] = cgindex[cg];
+ cginfo[cg+shift] = cginfo[cg];
+ }
+ /* Correct the already stored send indices for the shift */
+ for (p = 1; p <= pulse; p++)
+ {
+ ind_p = &cd->ind[p];
+ cg0 = 0;
+ for (c = 0; c < cell; c++)
+ {
+ cg0 += ind_p->nsend[c];
+ }
+ cg1 = cg0 + ind_p->nsend[cell];
+ for (cg = cg0; cg < cg1; cg++)
+ {
+ ind_p->index[cg] += shift;
+ }
+ }
+ }
+ }
+
+ /* Merge in the communicated buffers */
+ shift = 0;
+ shift_at = 0;
+ cg0 = 0;
+ for (cell = 0; cell < ncell; cell++)
+ {
+ cg1 = ncg_cell[ncell+cell+1] + shift;
+ if (shift_at > 0)
+ {
+ /* Correct the old cg indices */
+ for (cg = ncg_cell[ncell+cell]; cg < cg1; cg++)
+ {
+ cgindex[cg+1] += shift_at;
+ }
+ }
+ for (cg = 0; cg < ind->nrecv[cell]; cg++)
+ {
+ /* Copy this charge group from the buffer */
+ index_gl[cg1] = recv_i[cg0];
+ copy_rvec(recv_vr[cg0], cg_cm[cg1]);
+ /* Add it to the cgindex */
+ cg_gl = index_gl[cg1];
+ cginfo[cg1] = ddcginfo(cginfo_mb, cg_gl);
+ nat = GET_CGINFO_NATOMS(cginfo[cg1]);
+ cgindex[cg1+1] = cgindex[cg1] + nat;
+ cg0++;
+ cg1++;
+ shift_at += nat;
+ }
+ shift += ind->nrecv[cell];
+ ncg_cell[ncell+cell+1] = cg1;
+ }
+}
+
+static void make_cell2at_index(gmx_domdec_comm_dim_t *cd,
+ int nzone, int cg0, const int *cgindex)
+{
+ int cg, zone, p;
+
+ /* Store the atom block boundaries for easy copying of communication buffers
+ */
+ cg = cg0;
+ for (zone = 0; zone < nzone; zone++)
+ {
+ for (p = 0; p < cd->np; p++)
+ {
+ cd->ind[p].cell2at0[zone] = cgindex[cg];
+ cg += cd->ind[p].nrecv[zone];
+ cd->ind[p].cell2at1[zone] = cgindex[cg];
+ }
+ }
+}
+
+static gmx_bool missing_link(t_blocka *link, int cg_gl, char *bLocalCG)
+{
+ int i;
+ gmx_bool bMiss;
+
+ bMiss = FALSE;
+ for (i = link->index[cg_gl]; i < link->index[cg_gl+1]; i++)
+ {
+ if (!bLocalCG[link->a[i]])
+ {
+ bMiss = TRUE;
+ }
+ }
+
+ return bMiss;
+}
+
+/* Domain corners for communication, a maximum of 4 i-zones see a j domain */
+typedef struct {
+ real c[DIM][4]; /* the corners for the non-bonded communication */
+ real cr0; /* corner for rounding */
+ real cr1[4]; /* corners for rounding */
+ real bc[DIM]; /* corners for bounded communication */
+ real bcr1; /* corner for rounding for bonded communication */
+} dd_corners_t;
+
+/* Determine the corners of the domain(s) we are communicating with */
+static void
+set_dd_corners(const gmx_domdec_t *dd,
+ int dim0, int dim1, int dim2,
+ gmx_bool bDistMB,
+ dd_corners_t *c)
+{
+ const gmx_domdec_comm_t *comm;
+ const gmx_domdec_zones_t *zones;
+ int i, j;
+
+ comm = dd->comm;
+
+ zones = &comm->zones;
+
+ /* Keep the compiler happy */
+ c->cr0 = 0;
+ c->bcr1 = 0;
+
+ /* The first dimension is equal for all cells */
+ c->c[0][0] = comm->cell_x0[dim0];
+ if (bDistMB)
+ {
+ c->bc[0] = c->c[0][0];
+ }
+ if (dd->ndim >= 2)
+ {
+ dim1 = dd->dim[1];
+ /* This cell row is only seen from the first row */
+ c->c[1][0] = comm->cell_x0[dim1];
+ /* All rows can see this row */
+ c->c[1][1] = comm->cell_x0[dim1];
+ if (dd->bGridJump)
+ {
+ c->c[1][1] = max(comm->cell_x0[dim1], comm->zone_d1[1].mch0);
+ if (bDistMB)
+ {
+ /* For the multi-body distance we need the maximum */
+ c->bc[1] = max(comm->cell_x0[dim1], comm->zone_d1[1].p1_0);
+ }
+ }
+ /* Set the upper-right corner for rounding */
+ c->cr0 = comm->cell_x1[dim0];
+
+ if (dd->ndim >= 3)
+ {
+ dim2 = dd->dim[2];
+ for (j = 0; j < 4; j++)
+ {
+ c->c[2][j] = comm->cell_x0[dim2];
+ }
+ if (dd->bGridJump)
+ {
+ /* Use the maximum of the i-cells that see a j-cell */
+ for (i = 0; i < zones->nizone; i++)
+ {
+ for (j = zones->izone[i].j0; j < zones->izone[i].j1; j++)
+ {
+ if (j >= 4)
+ {
+ c->c[2][j-4] =
+ max(c->c[2][j-4],
+ comm->zone_d2[zones->shift[i][dim0]][zones->shift[i][dim1]].mch0);
+ }
+ }
+ }
+ if (bDistMB)
+ {
+ /* For the multi-body distance we need the maximum */
+ c->bc[2] = comm->cell_x0[dim2];
+ for (i = 0; i < 2; i++)
+ {
+ for (j = 0; j < 2; j++)
+ {
+ c->bc[2] = max(c->bc[2], comm->zone_d2[i][j].p1_0);
+ }
+ }
+ }
+ }
+
+ /* Set the upper-right corner for rounding */
+ /* Cell (0,0,0) and cell (1,0,0) can see cell 4 (0,1,1)
+ * Only cell (0,0,0) can see cell 7 (1,1,1)
+ */
+ c->cr1[0] = comm->cell_x1[dim1];
+ c->cr1[3] = comm->cell_x1[dim1];
+ if (dd->bGridJump)
+ {
+ c->cr1[0] = max(comm->cell_x1[dim1], comm->zone_d1[1].mch1);
+ if (bDistMB)
+ {
+ /* For the multi-body distance we need the maximum */
+ c->bcr1 = max(comm->cell_x1[dim1], comm->zone_d1[1].p1_1);
+ }
+ }
+ }
+ }
+}
+
+/* Determine which cg's we need to send in this pulse from this zone */
+static void
+get_zone_pulse_cgs(gmx_domdec_t *dd,
+ int zonei, int zone,
+ int cg0, int cg1,
+ const int *index_gl,
+ const int *cgindex,
+ int dim, int dim_ind,
+ int dim0, int dim1, int dim2,
+ real r_comm2, real r_bcomm2,
+ matrix box,
+ ivec tric_dist,
+ rvec *normal,
+ real skew_fac2_d, real skew_fac_01,
+ rvec *v_d, rvec *v_0, rvec *v_1,
+ const dd_corners_t *c,
+ rvec sf2_round,
+ gmx_bool bDistBonded,
+ gmx_bool bBondComm,
+ gmx_bool bDist2B,
+ gmx_bool bDistMB,
+ rvec *cg_cm,
+ int *cginfo,
+ gmx_domdec_ind_t *ind,
+ int **ibuf, int *ibuf_nalloc,
+ vec_rvec_t *vbuf,
+ int *nsend_ptr,
+ int *nat_ptr,
+ int *nsend_z_ptr)
+{
+ gmx_domdec_comm_t *comm;
+ gmx_bool bScrew;
+ gmx_bool bDistMB_pulse;
+ int cg, i;
+ real r2, rb2, r, tric_sh;
+ rvec rn, rb;
+ int dimd;
+ int nsend_z, nsend, nat;
+
+ comm = dd->comm;
+
+ bScrew = (dd->bScrewPBC && dim == XX);
+
+ bDistMB_pulse = (bDistMB && bDistBonded);
+
+ nsend_z = 0;
+ nsend = *nsend_ptr;
+ nat = *nat_ptr;
+
+ for (cg = cg0; cg < cg1; cg++)
+ {
+ r2 = 0;
+ rb2 = 0;
+ if (tric_dist[dim_ind] == 0)
+ {
+ /* Rectangular direction, easy */
+ r = cg_cm[cg][dim] - c->c[dim_ind][zone];
+ if (r > 0)
+ {
+ r2 += r*r;
+ }
+ if (bDistMB_pulse)
+ {
+ r = cg_cm[cg][dim] - c->bc[dim_ind];
+ if (r > 0)
+ {
+ rb2 += r*r;
+ }
+ }
+ /* Rounding gives at most a 16% reduction
+ * in communicated atoms
+ */
+ if (dim_ind >= 1 && (zonei == 1 || zonei == 2))
+ {
+ r = cg_cm[cg][dim0] - c->cr0;
+ /* This is the first dimension, so always r >= 0 */
+ r2 += r*r;
+ if (bDistMB_pulse)
+ {
+ rb2 += r*r;
+ }
+ }
+ if (dim_ind == 2 && (zonei == 2 || zonei == 3))
+ {
+ r = cg_cm[cg][dim1] - c->cr1[zone];
+ if (r > 0)
+ {
+ r2 += r*r;
+ }
+ if (bDistMB_pulse)
+ {
+ r = cg_cm[cg][dim1] - c->bcr1;
+ if (r > 0)
+ {
+ rb2 += r*r;
+ }
+ }
+ }
+ }
+ else
+ {
+ /* Triclinic direction, more complicated */
+ clear_rvec(rn);
+ clear_rvec(rb);
+ /* Rounding, conservative as the skew_fac multiplication
+ * will slightly underestimate the distance.
+ */
+ if (dim_ind >= 1 && (zonei == 1 || zonei == 2))
+ {
+ rn[dim0] = cg_cm[cg][dim0] - c->cr0;
+ for (i = dim0+1; i < DIM; i++)
+ {
+ rn[dim0] -= cg_cm[cg][i]*v_0[i][dim0];
+ }
+ r2 = rn[dim0]*rn[dim0]*sf2_round[dim0];
+ if (bDistMB_pulse)
+ {
+ rb[dim0] = rn[dim0];
+ rb2 = r2;
+ }
+ /* Take care that the cell planes along dim0 might not
+ * be orthogonal to those along dim1 and dim2.
+ */
+ for (i = 1; i <= dim_ind; i++)
+ {
+ dimd = dd->dim[i];
+ if (normal[dim0][dimd] > 0)
+ {
+ rn[dimd] -= rn[dim0]*normal[dim0][dimd];
+ if (bDistMB_pulse)
+ {
+ rb[dimd] -= rb[dim0]*normal[dim0][dimd];
+ }
+ }
+ }
+ }
+ if (dim_ind == 2 && (zonei == 2 || zonei == 3))
+ {
+ rn[dim1] += cg_cm[cg][dim1] - c->cr1[zone];
+ tric_sh = 0;
+ for (i = dim1+1; i < DIM; i++)
+ {
+ tric_sh -= cg_cm[cg][i]*v_1[i][dim1];
+ }
+ rn[dim1] += tric_sh;
+ if (rn[dim1] > 0)
+ {
+ r2 += rn[dim1]*rn[dim1]*sf2_round[dim1];
+ /* Take care of coupling of the distances
+ * to the planes along dim0 and dim1 through dim2.
+ */
+ r2 -= rn[dim0]*rn[dim1]*skew_fac_01;
+ /* Take care that the cell planes along dim1
+ * might not be orthogonal to that along dim2.
+ */
+ if (normal[dim1][dim2] > 0)
+ {
+ rn[dim2] -= rn[dim1]*normal[dim1][dim2];
+ }
+ }
+ if (bDistMB_pulse)
+ {
+ rb[dim1] +=
+ cg_cm[cg][dim1] - c->bcr1 + tric_sh;
+ if (rb[dim1] > 0)
+ {
+ rb2 += rb[dim1]*rb[dim1]*sf2_round[dim1];
+ /* Take care of coupling of the distances
+ * to the planes along dim0 and dim1 through dim2.
+ */
+ rb2 -= rb[dim0]*rb[dim1]*skew_fac_01;
+ /* Take care that the cell planes along dim1
+ * might not be orthogonal to that along dim2.
+ */
+ if (normal[dim1][dim2] > 0)
+ {
+ rb[dim2] -= rb[dim1]*normal[dim1][dim2];
+ }
+ }
+ }
+ }
+ /* The distance along the communication direction */
+ rn[dim] += cg_cm[cg][dim] - c->c[dim_ind][zone];
+ tric_sh = 0;
+ for (i = dim+1; i < DIM; i++)
+ {
+ tric_sh -= cg_cm[cg][i]*v_d[i][dim];
+ }
+ rn[dim] += tric_sh;
+ if (rn[dim] > 0)
+ {
+ r2 += rn[dim]*rn[dim]*skew_fac2_d;
+ /* Take care of coupling of the distances
+ * to the planes along dim0 and dim1 through dim2.
+ */
+ if (dim_ind == 1 && zonei == 1)
+ {
+ r2 -= rn[dim0]*rn[dim]*skew_fac_01;
+ }
+ }
+ if (bDistMB_pulse)
+ {
+ clear_rvec(rb);
+ rb[dim] += cg_cm[cg][dim] - c->bc[dim_ind] + tric_sh;
+ if (rb[dim] > 0)
+ {
+ rb2 += rb[dim]*rb[dim]*skew_fac2_d;
+ /* Take care of coupling of the distances
+ * to the planes along dim0 and dim1 through dim2.
+ */
+ if (dim_ind == 1 && zonei == 1)
+ {
+ rb2 -= rb[dim0]*rb[dim]*skew_fac_01;
+ }
+ }
+ }
+ }
+
+ if (r2 < r_comm2 ||
+ (bDistBonded &&
+ ((bDistMB && rb2 < r_bcomm2) ||
+ (bDist2B && r2 < r_bcomm2)) &&
+ (!bBondComm ||
+ (GET_CGINFO_BOND_INTER(cginfo[cg]) &&
+ missing_link(comm->cglink, index_gl[cg],
+ comm->bLocalCG)))))
+ {
+ /* Make an index to the local charge groups */
+ if (nsend+1 > ind->nalloc)
+ {
+ ind->nalloc = over_alloc_large(nsend+1);
+ srenew(ind->index, ind->nalloc);
+ }
+ if (nsend+1 > *ibuf_nalloc)
+ {
+ *ibuf_nalloc = over_alloc_large(nsend+1);
+ srenew(*ibuf, *ibuf_nalloc);
+ }
+ ind->index[nsend] = cg;
+ (*ibuf)[nsend] = index_gl[cg];
+ nsend_z++;
+ vec_rvec_check_alloc(vbuf, nsend+1);
+
+ if (dd->ci[dim] == 0)
+ {
+ /* Correct cg_cm for pbc */
+ rvec_add(cg_cm[cg], box[dim], vbuf->v[nsend]);
+ if (bScrew)
+ {
+ vbuf->v[nsend][YY] = box[YY][YY] - vbuf->v[nsend][YY];
+ vbuf->v[nsend][ZZ] = box[ZZ][ZZ] - vbuf->v[nsend][ZZ];
+ }
+ }
+ else
+ {
+ copy_rvec(cg_cm[cg], vbuf->v[nsend]);
+ }
+ nsend++;
+ nat += cgindex[cg+1] - cgindex[cg];
+ }
+ }
+
+ *nsend_ptr = nsend;
+ *nat_ptr = nat;
+ *nsend_z_ptr = nsend_z;
+}
+
+static void setup_dd_communication(gmx_domdec_t *dd,
+ matrix box, gmx_ddbox_t *ddbox,
+ t_forcerec *fr, t_state *state, rvec **f)
+{
+ int dim_ind, dim, dim0, dim1, dim2, dimd, p, nat_tot;
+ int nzone, nzone_send, zone, zonei, cg0, cg1;
+ int c, i, j, cg, cg_gl, nrcg;
+ int *zone_cg_range, pos_cg, *index_gl, *cgindex, *recv_i;
+ gmx_domdec_comm_t *comm;
+ gmx_domdec_zones_t *zones;
+ gmx_domdec_comm_dim_t *cd;
+ gmx_domdec_ind_t *ind;
+ cginfo_mb_t *cginfo_mb;
+ gmx_bool bBondComm, bDist2B, bDistMB, bDistBonded;
+ real r_mb, r_comm2, r_scomm2, r_bcomm2, r_0, r_1, r2inc, inv_ncg;
+ dd_corners_t corners;
+ ivec tric_dist;
+ rvec *cg_cm, *normal, *v_d, *v_0 = NULL, *v_1 = NULL, *recv_vr;
+ real skew_fac2_d, skew_fac_01;
+ rvec sf2_round;
+ int nsend, nat;
+ int th;
+
+ if (debug)
+ {
+ fprintf(debug, "Setting up DD communication\n");
+ }
+
+ comm = dd->comm;
+
+ switch (fr->cutoff_scheme)
+ {
+ case ecutsGROUP:
+ cg_cm = fr->cg_cm;
+ break;
+ case ecutsVERLET:
+ cg_cm = state->x;
+ break;
+ default:
+ gmx_incons("unimplemented");
+ cg_cm = NULL;
+ }
+
+ for (dim_ind = 0; dim_ind < dd->ndim; dim_ind++)
+ {
+ dim = dd->dim[dim_ind];
+
+ /* Check if we need to use triclinic distances */
+ tric_dist[dim_ind] = 0;
+ for (i = 0; i <= dim_ind; i++)
+ {
+ if (ddbox->tric_dir[dd->dim[i]])
+ {
+ tric_dist[dim_ind] = 1;
+ }
+ }
+ }
+
+ bBondComm = comm->bBondComm;
+
+ /* Do we need to determine extra distances for multi-body bondeds? */
+ bDistMB = (comm->bInterCGMultiBody && dd->bGridJump && dd->ndim > 1);
+
+ /* Do we need to determine extra distances for only two-body bondeds? */
+ bDist2B = (bBondComm && !bDistMB);
+
+ r_comm2 = sqr(comm->cutoff);
+ r_bcomm2 = sqr(comm->cutoff_mbody);
+
+ if (debug)
+ {
+ fprintf(debug, "bBondComm %d, r_bc %f\n", bBondComm, sqrt(r_bcomm2));
+ }
+
+ zones = &comm->zones;
+
+ dim0 = dd->dim[0];
+ dim1 = (dd->ndim >= 2 ? dd->dim[1] : -1);
+ dim2 = (dd->ndim >= 3 ? dd->dim[2] : -1);
+
+ set_dd_corners(dd, dim0, dim1, dim2, bDistMB, &corners);
+
+ /* Triclinic stuff */
+ normal = ddbox->normal;
+ skew_fac_01 = 0;
+ if (dd->ndim >= 2)
+ {
+ v_0 = ddbox->v[dim0];
+ if (ddbox->tric_dir[dim0] && ddbox->tric_dir[dim1])
+ {
+ /* Determine the coupling coefficient for the distances
+ * to the cell planes along dim0 and dim1 through dim2.
+ * This is required for correct rounding.
+ */
+ skew_fac_01 =
+ ddbox->v[dim0][dim1+1][dim0]*ddbox->v[dim1][dim1+1][dim1];
+ if (debug)
+ {
+ fprintf(debug, "\nskew_fac_01 %f\n", skew_fac_01);
+ }
+ }
+ }
+ if (dd->ndim >= 3)
+ {
+ v_1 = ddbox->v[dim1];
+ }
+
+ zone_cg_range = zones->cg_range;
+ index_gl = dd->index_gl;
+ cgindex = dd->cgindex;
+ cginfo_mb = fr->cginfo_mb;
+
+ zone_cg_range[0] = 0;
+ zone_cg_range[1] = dd->ncg_home;
+ comm->zone_ncg1[0] = dd->ncg_home;
+ pos_cg = dd->ncg_home;
+
+ nat_tot = dd->nat_home;
+ nzone = 1;
+ for (dim_ind = 0; dim_ind < dd->ndim; dim_ind++)
+ {
+ dim = dd->dim[dim_ind];
+ cd = &comm->cd[dim_ind];
+
+ if (dim >= ddbox->npbcdim && dd->ci[dim] == 0)
+ {
+ /* No pbc in this dimension, the first node should not comm. */
+ nzone_send = 0;
+ }
+ else
+ {
+ nzone_send = nzone;
+ }
+
+ v_d = ddbox->v[dim];
+ skew_fac2_d = sqr(ddbox->skew_fac[dim]);
+
+ cd->bInPlace = TRUE;
+ for (p = 0; p < cd->np; p++)
+ {
+ /* Only atoms communicated in the first pulse are used
+ * for multi-body bonded interactions or for bBondComm.
+ */
+ bDistBonded = ((bDistMB || bDist2B) && p == 0);
+
+ ind = &cd->ind[p];
+ nsend = 0;
+ nat = 0;
+ for (zone = 0; zone < nzone_send; zone++)
+ {
+ if (tric_dist[dim_ind] && dim_ind > 0)
+ {
+ /* Determine slightly more optimized skew_fac's
+ * for rounding.
+ * This reduces the number of communicated atoms
+ * by about 10% for 3D DD of rhombic dodecahedra.
+ */
+ for (dimd = 0; dimd < dim; dimd++)
+ {
+ sf2_round[dimd] = 1;
+ if (ddbox->tric_dir[dimd])
+ {
+ for (i = dd->dim[dimd]+1; i < DIM; i++)
+ {
+ /* If we are shifted in dimension i
+ * and the cell plane is tilted forward
+ * in dimension i, skip this coupling.
+ */
+ if (!(zones->shift[nzone+zone][i] &&
+ ddbox->v[dimd][i][dimd] >= 0))
+ {
+ sf2_round[dimd] +=
+ sqr(ddbox->v[dimd][i][dimd]);
+ }
+ }
+ sf2_round[dimd] = 1/sf2_round[dimd];
+ }
+ }
+ }
+
+ zonei = zone_perm[dim_ind][zone];
+ if (p == 0)
+ {
+ /* Here we permutate the zones to obtain a convenient order
+ * for neighbor searching
+ */
+ cg0 = zone_cg_range[zonei];
+ cg1 = zone_cg_range[zonei+1];
+ }
+ else
+ {
+ /* Look only at the cg's received in the previous grid pulse
+ */
+ cg1 = zone_cg_range[nzone+zone+1];
+ cg0 = cg1 - cd->ind[p-1].nrecv[zone];
+ }
+
+#pragma omp parallel for num_threads(comm->nth) schedule(static)
+ for (th = 0; th < comm->nth; th++)
+ {
+ gmx_domdec_ind_t *ind_p;
+ int **ibuf_p, *ibuf_nalloc_p;
+ vec_rvec_t *vbuf_p;
+ int *nsend_p, *nat_p;
+ int *nsend_zone_p;
+ int cg0_th, cg1_th;
+
+ if (th == 0)
+ {
+ /* Thread 0 writes in the comm buffers */
+ ind_p = ind;
+ ibuf_p = &comm->buf_int;
+ ibuf_nalloc_p = &comm->nalloc_int;
+ vbuf_p = &comm->vbuf;
+ nsend_p = &nsend;
+ nat_p = &nat;
+ nsend_zone_p = &ind->nsend[zone];
+ }
+ else
+ {
+ /* Other threads write into temp buffers */
+ ind_p = &comm->dth[th].ind;
+ ibuf_p = &comm->dth[th].ibuf;
+ ibuf_nalloc_p = &comm->dth[th].ibuf_nalloc;
+ vbuf_p = &comm->dth[th].vbuf;
+ nsend_p = &comm->dth[th].nsend;
+ nat_p = &comm->dth[th].nat;
+ nsend_zone_p = &comm->dth[th].nsend_zone;
+
+ comm->dth[th].nsend = 0;
+ comm->dth[th].nat = 0;
+ comm->dth[th].nsend_zone = 0;
+ }
+
+ if (comm->nth == 1)
+ {
+ cg0_th = cg0;
+ cg1_th = cg1;
+ }
+ else
+ {
+ cg0_th = cg0 + ((cg1 - cg0)* th )/comm->nth;
+ cg1_th = cg0 + ((cg1 - cg0)*(th+1))/comm->nth;
+ }
+
+ /* Get the cg's for this pulse in this zone */
+ get_zone_pulse_cgs(dd, zonei, zone, cg0_th, cg1_th,
+ index_gl, cgindex,
+ dim, dim_ind, dim0, dim1, dim2,
+ r_comm2, r_bcomm2,
+ box, tric_dist,
+ normal, skew_fac2_d, skew_fac_01,
+ v_d, v_0, v_1, &corners, sf2_round,
+ bDistBonded, bBondComm,
+ bDist2B, bDistMB,
+ cg_cm, fr->cginfo,
+ ind_p,
+ ibuf_p, ibuf_nalloc_p,
+ vbuf_p,
+ nsend_p, nat_p,
+ nsend_zone_p);
+ }
+
+ /* Append data of threads>=1 to the communication buffers */
+ for (th = 1; th < comm->nth; th++)
+ {
+ dd_comm_setup_work_t *dth;
+ int i, ns1;
+
+ dth = &comm->dth[th];
+
+ ns1 = nsend + dth->nsend_zone;
+ if (ns1 > ind->nalloc)
+ {
+ ind->nalloc = over_alloc_dd(ns1);
+ srenew(ind->index, ind->nalloc);
+ }
+ if (ns1 > comm->nalloc_int)
+ {
+ comm->nalloc_int = over_alloc_dd(ns1);
+ srenew(comm->buf_int, comm->nalloc_int);
+ }
+ if (ns1 > comm->vbuf.nalloc)
+ {
+ comm->vbuf.nalloc = over_alloc_dd(ns1);
+ srenew(comm->vbuf.v, comm->vbuf.nalloc);
+ }
+
+ for (i = 0; i < dth->nsend_zone; i++)
+ {
+ ind->index[nsend] = dth->ind.index[i];
+ comm->buf_int[nsend] = dth->ibuf[i];
+ copy_rvec(dth->vbuf.v[i],
+ comm->vbuf.v[nsend]);
+ nsend++;
+ }
+ nat += dth->nat;
+ ind->nsend[zone] += dth->nsend_zone;
+ }
+ }
+ /* Clear the counts in case we do not have pbc */
+ for (zone = nzone_send; zone < nzone; zone++)
+ {
+ ind->nsend[zone] = 0;
+ }
+ ind->nsend[nzone] = nsend;
+ ind->nsend[nzone+1] = nat;
+ /* Communicate the number of cg's and atoms to receive */
+ dd_sendrecv_int(dd, dim_ind, dddirBackward,
+ ind->nsend, nzone+2,
+ ind->nrecv, nzone+2);
+
+ /* The rvec buffer is also required for atom buffers of size nsend
+ * in dd_move_x and dd_move_f.
+ */
+ vec_rvec_check_alloc(&comm->vbuf, ind->nsend[nzone+1]);
+
+ if (p > 0)
+ {
+ /* We can receive in place if only the last zone is not empty */
+ for (zone = 0; zone < nzone-1; zone++)
+ {
+ if (ind->nrecv[zone] > 0)
+ {
+ cd->bInPlace = FALSE;
+ }
+ }
+ if (!cd->bInPlace)
+ {
+ /* The int buffer is only required here for the cg indices */
+ if (ind->nrecv[nzone] > comm->nalloc_int2)
+ {
+ comm->nalloc_int2 = over_alloc_dd(ind->nrecv[nzone]);
+ srenew(comm->buf_int2, comm->nalloc_int2);
+ }
+ /* The rvec buffer is also required for atom buffers
+ * of size nrecv in dd_move_x and dd_move_f.
+ */
+ i = max(cd->ind[0].nrecv[nzone+1], ind->nrecv[nzone+1]);
+ vec_rvec_check_alloc(&comm->vbuf2, i);
+ }
+ }
+
+ /* Make space for the global cg indices */
+ if (pos_cg + ind->nrecv[nzone] > dd->cg_nalloc
+ || dd->cg_nalloc == 0)
+ {
+ dd->cg_nalloc = over_alloc_dd(pos_cg + ind->nrecv[nzone]);
+ srenew(index_gl, dd->cg_nalloc);
+ srenew(cgindex, dd->cg_nalloc+1);
+ }
+ /* Communicate the global cg indices */
+ if (cd->bInPlace)
+ {
+ recv_i = index_gl + pos_cg;
+ }
+ else
+ {
+ recv_i = comm->buf_int2;
+ }
+ dd_sendrecv_int(dd, dim_ind, dddirBackward,
+ comm->buf_int, nsend,
+ recv_i, ind->nrecv[nzone]);
+
+ /* Make space for cg_cm */
+ dd_check_alloc_ncg(fr, state, f, pos_cg + ind->nrecv[nzone]);
+ if (fr->cutoff_scheme == ecutsGROUP)
+ {
+ cg_cm = fr->cg_cm;
+ }
+ else
+ {
+ cg_cm = state->x;
+ }
+ /* Communicate cg_cm */
+ if (cd->bInPlace)
+ {
+ recv_vr = cg_cm + pos_cg;
+ }
+ else
+ {
+ recv_vr = comm->vbuf2.v;
+ }
+ dd_sendrecv_rvec(dd, dim_ind, dddirBackward,
+ comm->vbuf.v, nsend,
+ recv_vr, ind->nrecv[nzone]);
+
+ /* Make the charge group index */
+ if (cd->bInPlace)
+ {
+ zone = (p == 0 ? 0 : nzone - 1);
+ while (zone < nzone)
+ {
+ for (cg = 0; cg < ind->nrecv[zone]; cg++)
+ {
+ cg_gl = index_gl[pos_cg];
+ fr->cginfo[pos_cg] = ddcginfo(cginfo_mb, cg_gl);
+ nrcg = GET_CGINFO_NATOMS(fr->cginfo[pos_cg]);
+ cgindex[pos_cg+1] = cgindex[pos_cg] + nrcg;
+ if (bBondComm)
+ {
+ /* Update the charge group presence,
+ * so we can use it in the next pass of the loop.
+ */
+ comm->bLocalCG[cg_gl] = TRUE;
+ }
+ pos_cg++;
+ }
+ if (p == 0)
+ {
+ comm->zone_ncg1[nzone+zone] = ind->nrecv[zone];
+ }
+ zone++;
+ zone_cg_range[nzone+zone] = pos_cg;
+ }
+ }
+ else
+ {
+ /* This part of the code is never executed with bBondComm. */
+ merge_cg_buffers(nzone, cd, p, zone_cg_range,
+ index_gl, recv_i, cg_cm, recv_vr,
+ cgindex, fr->cginfo_mb, fr->cginfo);
+ pos_cg += ind->nrecv[nzone];
+ }
+ nat_tot += ind->nrecv[nzone+1];
+ }
+ if (!cd->bInPlace)
+ {
+ /* Store the atom block for easy copying of communication buffers */
+ make_cell2at_index(cd, nzone, zone_cg_range[nzone], cgindex);
+ }
+ nzone += nzone;
+ }
+ dd->index_gl = index_gl;
+ dd->cgindex = cgindex;
+
+ dd->ncg_tot = zone_cg_range[zones->n];
+ dd->nat_tot = nat_tot;
+ comm->nat[ddnatHOME] = dd->nat_home;
+ for (i = ddnatZONE; i < ddnatNR; i++)
+ {
+ comm->nat[i] = dd->nat_tot;
+ }
+
+ if (!bBondComm)
+ {
+ /* We don't need to update cginfo, since that was alrady done above.
+ * So we pass NULL for the forcerec.
+ */
+ dd_set_cginfo(dd->index_gl, dd->ncg_home, dd->ncg_tot,
+ NULL, comm->bLocalCG);
+ }
+
+ if (debug)
+ {
+ fprintf(debug, "Finished setting up DD communication, zones:");
+ for (c = 0; c < zones->n; c++)
+ {
+ fprintf(debug, " %d", zones->cg_range[c+1]-zones->cg_range[c]);
+ }
+ fprintf(debug, "\n");
+ }
+}
+
+static void set_cg_boundaries(gmx_domdec_zones_t *zones)
+{
+ int c;
+
+ for (c = 0; c < zones->nizone; c++)
+ {
+ zones->izone[c].cg1 = zones->cg_range[c+1];
+ zones->izone[c].jcg0 = zones->cg_range[zones->izone[c].j0];
+ zones->izone[c].jcg1 = zones->cg_range[zones->izone[c].j1];
+ }
+}
+
+static void set_zones_size(gmx_domdec_t *dd,
+ matrix box, const gmx_ddbox_t *ddbox,
+ int zone_start, int zone_end)
+{
+ gmx_domdec_comm_t *comm;
+ gmx_domdec_zones_t *zones;
+ gmx_bool bDistMB;
+ int z, zi, zj0, zj1, d, dim;
+ real rcs, rcmbs;
+ int i, j;
+ real size_j, add_tric;
+ real vol;
+
+ comm = dd->comm;
+
+ zones = &comm->zones;
+
+ /* Do we need to determine extra distances for multi-body bondeds? */
+ bDistMB = (comm->bInterCGMultiBody && dd->bGridJump && dd->ndim > 1);
+
+ for (z = zone_start; z < zone_end; z++)
+ {
+ /* Copy cell limits to zone limits.
+ * Valid for non-DD dims and non-shifted dims.
+ */
+ copy_rvec(comm->cell_x0, zones->size[z].x0);
+ copy_rvec(comm->cell_x1, zones->size[z].x1);
+ }
+
+ for (d = 0; d < dd->ndim; d++)
+ {
+ dim = dd->dim[d];
+
+ for (z = 0; z < zones->n; z++)
+ {
+ /* With a staggered grid we have different sizes
+ * for non-shifted dimensions.
+ */
+ if (dd->bGridJump && zones->shift[z][dim] == 0)
+ {
+ if (d == 1)
+ {
+ zones->size[z].x0[dim] = comm->zone_d1[zones->shift[z][dd->dim[d-1]]].min0;
+ zones->size[z].x1[dim] = comm->zone_d1[zones->shift[z][dd->dim[d-1]]].max1;
+ }
+ else if (d == 2)
+ {
+ zones->size[z].x0[dim] = comm->zone_d2[zones->shift[z][dd->dim[d-2]]][zones->shift[z][dd->dim[d-1]]].min0;
+ zones->size[z].x1[dim] = comm->zone_d2[zones->shift[z][dd->dim[d-2]]][zones->shift[z][dd->dim[d-1]]].max1;
+ }
+ }
+ }
+
+ rcs = comm->cutoff;
+ rcmbs = comm->cutoff_mbody;
+ if (ddbox->tric_dir[dim])
+ {
+ rcs /= ddbox->skew_fac[dim];
+ rcmbs /= ddbox->skew_fac[dim];
+ }
+
+ /* Set the lower limit for the shifted zone dimensions */
+ for (z = zone_start; z < zone_end; z++)
+ {
+ if (zones->shift[z][dim] > 0)
+ {
+ dim = dd->dim[d];
+ if (!dd->bGridJump || d == 0)
+ {
+ zones->size[z].x0[dim] = comm->cell_x1[dim];
+ zones->size[z].x1[dim] = comm->cell_x1[dim] + rcs;
+ }
+ else
+ {
+ /* Here we take the lower limit of the zone from
+ * the lowest domain of the zone below.
+ */
+ if (z < 4)
+ {
+ zones->size[z].x0[dim] =
+ comm->zone_d1[zones->shift[z][dd->dim[d-1]]].min1;
+ }
+ else
+ {
+ if (d == 1)
+ {
+ zones->size[z].x0[dim] =
+ zones->size[zone_perm[2][z-4]].x0[dim];
+ }
+ else
+ {
+ zones->size[z].x0[dim] =
+ comm->zone_d2[zones->shift[z][dd->dim[d-2]]][zones->shift[z][dd->dim[d-1]]].min1;
+ }
+ }
+ /* A temporary limit, is updated below */
+ zones->size[z].x1[dim] = zones->size[z].x0[dim];
+
+ if (bDistMB)
+ {
+ for (zi = 0; zi < zones->nizone; zi++)
+ {
+ if (zones->shift[zi][dim] == 0)
+ {
+ /* This takes the whole zone into account.
+ * With multiple pulses this will lead
+ * to a larger zone then strictly necessary.
+ */
+ zones->size[z].x1[dim] = max(zones->size[z].x1[dim],
+ zones->size[zi].x1[dim]+rcmbs);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ /* Loop over the i-zones to set the upper limit of each
+ * j-zone they see.
+ */
+ for (zi = 0; zi < zones->nizone; zi++)
+ {
+ if (zones->shift[zi][dim] == 0)
+ {
+ for (z = zones->izone[zi].j0; z < zones->izone[zi].j1; z++)
+ {
+ if (zones->shift[z][dim] > 0)
+ {
+ zones->size[z].x1[dim] = max(zones->size[z].x1[dim],
+ zones->size[zi].x1[dim]+rcs);
+ }
+ }
+ }
+ }
+ }
+
+ for (z = zone_start; z < zone_end; z++)
+ {
+ /* Initialization only required to keep the compiler happy */
+ rvec corner_min = {0, 0, 0}, corner_max = {0, 0, 0}, corner;
+ int nc, c;
+
+ /* To determine the bounding box for a zone we need to find
+ * the extreme corners of 4, 2 or 1 corners.
+ */
+ nc = 1 << (ddbox->npbcdim - 1);
+
+ for (c = 0; c < nc; c++)
+ {
+ /* Set up a zone corner at x=0, ignoring trilinic couplings */
+ corner[XX] = 0;
+ if ((c & 1) == 0)
+ {
+ corner[YY] = zones->size[z].x0[YY];
+ }
+ else
+ {
+ corner[YY] = zones->size[z].x1[YY];
+ }
+ if ((c & 2) == 0)
+ {
+ corner[ZZ] = zones->size[z].x0[ZZ];
+ }
+ else
+ {
+ corner[ZZ] = zones->size[z].x1[ZZ];
+ }
+ if (dd->ndim == 1 && box[ZZ][YY] != 0)
+ {
+ /* With 1D domain decomposition the cg's are not in
+ * the triclinic box, but triclinic x-y and rectangular y-z.
+ * Shift y back, so it will later end up at 0.
+ */
+ corner[YY] -= corner[ZZ]*box[ZZ][YY]/box[ZZ][ZZ];
+ }
+ /* Apply the triclinic couplings */
+ for (i = YY; i < ddbox->npbcdim; i++)
+ {
+ for (j = XX; j < i; j++)
+ {
+ corner[j] += corner[i]*box[i][j]/box[i][i];
+ }
+ }
+ if (c == 0)
+ {
+ copy_rvec(corner, corner_min);
+ copy_rvec(corner, corner_max);
+ }
+ else
+ {
+ for (i = 0; i < DIM; i++)
+ {
+ corner_min[i] = min(corner_min[i], corner[i]);
+ corner_max[i] = max(corner_max[i], corner[i]);
+ }
+ }
+ }
+ /* Copy the extreme cornes without offset along x */
+ for (i = 0; i < DIM; i++)
+ {
+ zones->size[z].bb_x0[i] = corner_min[i];
+ zones->size[z].bb_x1[i] = corner_max[i];
+ }
+ /* Add the offset along x */
+ zones->size[z].bb_x0[XX] += zones->size[z].x0[XX];
+ zones->size[z].bb_x1[XX] += zones->size[z].x1[XX];
+ }
+
+ if (zone_start == 0)
+ {
+ vol = 1;
+ for (dim = 0; dim < DIM; dim++)
+ {
+ vol *= zones->size[0].x1[dim] - zones->size[0].x0[dim];
+ }
+ zones->dens_zone0 = (zones->cg_range[1] - zones->cg_range[0])/vol;
+ }
+
+ if (debug)
+ {
+ for (z = zone_start; z < zone_end; z++)
+ {
+ fprintf(debug, "zone %d %6.3f - %6.3f %6.3f - %6.3f %6.3f - %6.3f\n",
+ z,
+ zones->size[z].x0[XX], zones->size[z].x1[XX],
+ zones->size[z].x0[YY], zones->size[z].x1[YY],
+ zones->size[z].x0[ZZ], zones->size[z].x1[ZZ]);
+ fprintf(debug, "zone %d bb %6.3f - %6.3f %6.3f - %6.3f %6.3f - %6.3f\n",
+ z,
+ zones->size[z].bb_x0[XX], zones->size[z].bb_x1[XX],
+ zones->size[z].bb_x0[YY], zones->size[z].bb_x1[YY],
+ zones->size[z].bb_x0[ZZ], zones->size[z].bb_x1[ZZ]);
+ }
+ }
+}
+
+static int comp_cgsort(const void *a, const void *b)
+{
+ int comp;
+
+ gmx_cgsort_t *cga, *cgb;
+ cga = (gmx_cgsort_t *)a;
+ cgb = (gmx_cgsort_t *)b;
+
+ comp = cga->nsc - cgb->nsc;
+ if (comp == 0)
+ {
+ comp = cga->ind_gl - cgb->ind_gl;
+ }
+
+ return comp;
+}
+
+static void order_int_cg(int n, const gmx_cgsort_t *sort,
+ int *a, int *buf)
+{
+ int i;
+
+ /* Order the data */
+ for (i = 0; i < n; i++)
+ {
+ buf[i] = a[sort[i].ind];
+ }
+
+ /* Copy back to the original array */
+ for (i = 0; i < n; i++)
+ {
+ a[i] = buf[i];
+ }
+}
+
+static void order_vec_cg(int n, const gmx_cgsort_t *sort,
+ rvec *v, rvec *buf)
+{
+ int i;
+
+ /* Order the data */
+ for (i = 0; i < n; i++)
+ {
+ copy_rvec(v[sort[i].ind], buf[i]);
+ }
+
+ /* Copy back to the original array */
+ for (i = 0; i < n; i++)
+ {
+ copy_rvec(buf[i], v[i]);
+ }
+}
+
+static void order_vec_atom(int ncg, const int *cgindex, const gmx_cgsort_t *sort,
+ rvec *v, rvec *buf)
+{
+ int a, atot, cg, cg0, cg1, i;
+
+ if (cgindex == NULL)
+ {
+ /* Avoid the useless loop of the atoms within a cg */
+ order_vec_cg(ncg, sort, v, buf);
+
+ return;
+ }
+
+ /* Order the data */
+ a = 0;
+ for (cg = 0; cg < ncg; cg++)
+ {
+ cg0 = cgindex[sort[cg].ind];
+ cg1 = cgindex[sort[cg].ind+1];
+ for (i = cg0; i < cg1; i++)
+ {
+ copy_rvec(v[i], buf[a]);
+ a++;
+ }
+ }
+ atot = a;
+
+ /* Copy back to the original array */
+ for (a = 0; a < atot; a++)
+ {
+ copy_rvec(buf[a], v[a]);
+ }
+}
+
+static void ordered_sort(int nsort2, gmx_cgsort_t *sort2,
+ int nsort_new, gmx_cgsort_t *sort_new,
+ gmx_cgsort_t *sort1)
+{
+ int i1, i2, i_new;
+
+ /* The new indices are not very ordered, so we qsort them */
+ qsort_threadsafe(sort_new, nsort_new, sizeof(sort_new[0]), comp_cgsort);
+
+ /* sort2 is already ordered, so now we can merge the two arrays */
+ i1 = 0;
+ i2 = 0;
+ i_new = 0;
+ while (i2 < nsort2 || i_new < nsort_new)
+ {
+ if (i2 == nsort2)
+ {
+ sort1[i1++] = sort_new[i_new++];
+ }
+ else if (i_new == nsort_new)
+ {
+ sort1[i1++] = sort2[i2++];
+ }
+ else if (sort2[i2].nsc < sort_new[i_new].nsc ||
+ (sort2[i2].nsc == sort_new[i_new].nsc &&
+ sort2[i2].ind_gl < sort_new[i_new].ind_gl))
+ {
+ sort1[i1++] = sort2[i2++];
+ }
+ else
+ {
+ sort1[i1++] = sort_new[i_new++];
+ }
+ }
+}
+
+static int dd_sort_order(gmx_domdec_t *dd, t_forcerec *fr, int ncg_home_old)
+{
+ gmx_domdec_sort_t *sort;
+ gmx_cgsort_t *cgsort, *sort_i;
+ int ncg_new, nsort2, nsort_new, i, *a, moved, *ibuf;
+ int sort_last, sort_skip;
+
+ sort = dd->comm->sort;
+
+ a = fr->ns.grid->cell_index;
+
+ moved = NSGRID_SIGNAL_MOVED_FAC*fr->ns.grid->ncells;
+
+ if (ncg_home_old >= 0)
+ {
+ /* The charge groups that remained in the same ns grid cell
+ * are completely ordered. So we can sort efficiently by sorting
+ * the charge groups that did move into the stationary list.
+ */
+ ncg_new = 0;
+ nsort2 = 0;
+ nsort_new = 0;
+ for (i = 0; i < dd->ncg_home; i++)
+ {
+ /* Check if this cg did not move to another node */
+ if (a[i] < moved)
+ {
+ if (i >= ncg_home_old || a[i] != sort->sort[i].nsc)
+ {
+ /* This cg is new on this node or moved ns grid cell */
+ if (nsort_new >= sort->sort_new_nalloc)
+ {
+ sort->sort_new_nalloc = over_alloc_dd(nsort_new+1);
+ srenew(sort->sort_new, sort->sort_new_nalloc);
+ }
+ sort_i = &(sort->sort_new[nsort_new++]);
+ }
+ else
+ {
+ /* This cg did not move */
+ sort_i = &(sort->sort2[nsort2++]);
+ }
+ /* Sort on the ns grid cell indices
+ * and the global topology index.
+ * index_gl is irrelevant with cell ns,
+ * but we set it here anyhow to avoid a conditional.
+ */
+ sort_i->nsc = a[i];
+ sort_i->ind_gl = dd->index_gl[i];
+ sort_i->ind = i;
+ ncg_new++;
+ }
+ }
+ if (debug)
+ {
+ fprintf(debug, "ordered sort cgs: stationary %d moved %d\n",
+ nsort2, nsort_new);
+ }
+ /* Sort efficiently */
+ ordered_sort(nsort2, sort->sort2, nsort_new, sort->sort_new,
+ sort->sort);
+ }
+ else
+ {
+ cgsort = sort->sort;
+ ncg_new = 0;
+ for (i = 0; i < dd->ncg_home; i++)
+ {
+ /* Sort on the ns grid cell indices
+ * and the global topology index
+ */
+ cgsort[i].nsc = a[i];
+ cgsort[i].ind_gl = dd->index_gl[i];
+ cgsort[i].ind = i;
+ if (cgsort[i].nsc < moved)
+ {
+ ncg_new++;
+ }
+ }
+ if (debug)
+ {
+ fprintf(debug, "qsort cgs: %d new home %d\n", dd->ncg_home, ncg_new);
+ }
+ /* Determine the order of the charge groups using qsort */
+ qsort_threadsafe(cgsort, dd->ncg_home, sizeof(cgsort[0]), comp_cgsort);
+ }
+
+ return ncg_new;
+}
+
+static int dd_sort_order_nbnxn(gmx_domdec_t *dd, t_forcerec *fr)
+{
+ gmx_cgsort_t *sort;
+ int ncg_new, i, *a, na;
+
+ sort = dd->comm->sort->sort;
+
+ nbnxn_get_atomorder(fr->nbv->nbs, &a, &na);
+
+ ncg_new = 0;
+ for (i = 0; i < na; i++)
+ {
+ if (a[i] >= 0)
+ {
+ sort[ncg_new].ind = a[i];
+ ncg_new++;
+ }
+ }
+
+ return ncg_new;
+}
+
+static void dd_sort_state(gmx_domdec_t *dd, rvec *cgcm, t_forcerec *fr, t_state *state,
+ int ncg_home_old)
+{
+ gmx_domdec_sort_t *sort;
+ gmx_cgsort_t *cgsort, *sort_i;
+ int *cgindex;
+ int ncg_new, i, *ibuf, cgsize;
+ rvec *vbuf;
+
+ sort = dd->comm->sort;
+
+ if (dd->ncg_home > sort->sort_nalloc)
+ {
+ sort->sort_nalloc = over_alloc_dd(dd->ncg_home);
+ srenew(sort->sort, sort->sort_nalloc);
+ srenew(sort->sort2, sort->sort_nalloc);
+ }
+ cgsort = sort->sort;
+
+ switch (fr->cutoff_scheme)
+ {
+ case ecutsGROUP:
+ ncg_new = dd_sort_order(dd, fr, ncg_home_old);
+ break;
+ case ecutsVERLET:
+ ncg_new = dd_sort_order_nbnxn(dd, fr);
+ break;
+ default:
+ gmx_incons("unimplemented");
+ ncg_new = 0;
+ }
+
+ /* We alloc with the old size, since cgindex is still old */
+ vec_rvec_check_alloc(&dd->comm->vbuf, dd->cgindex[dd->ncg_home]);
+ vbuf = dd->comm->vbuf.v;
+
+ if (dd->comm->bCGs)
+ {
+ cgindex = dd->cgindex;
+ }
+ else
+ {
+ cgindex = NULL;
+ }
+
+ /* Remove the charge groups which are no longer at home here */
+ dd->ncg_home = ncg_new;
+ if (debug)
+ {
+ fprintf(debug, "Set the new home charge group count to %d\n",
+ dd->ncg_home);
+ }
+
+ /* Reorder the state */
+ for (i = 0; i < estNR; i++)
+ {
+ if (EST_DISTR(i) && (state->flags & (1<<i)))
+ {
+ switch (i)
+ {
+ case estX:
+ order_vec_atom(dd->ncg_home, cgindex, cgsort, state->x, vbuf);
+ break;
+ case estV:
+ order_vec_atom(dd->ncg_home, cgindex, cgsort, state->v, vbuf);
+ break;
+ case estSDX:
+ order_vec_atom(dd->ncg_home, cgindex, cgsort, state->sd_X, vbuf);
+ break;
+ case estCGP:
+ order_vec_atom(dd->ncg_home, cgindex, cgsort, state->cg_p, vbuf);
+ break;
+ case estLD_RNG:
+ case estLD_RNGI:
+ case estDISRE_INITF:
+ case estDISRE_RM3TAV:
+ case estORIRE_INITF:
+ case estORIRE_DTAV:
+ /* No ordering required */
+ break;
+ default:
+ gmx_incons("Unknown state entry encountered in dd_sort_state");
+ break;
+ }
+ }
+ }
+ if (fr->cutoff_scheme == ecutsGROUP)
+ {
+ /* Reorder cgcm */
+ order_vec_cg(dd->ncg_home, cgsort, cgcm, vbuf);
+ }
+
+ if (dd->ncg_home+1 > sort->ibuf_nalloc)
+ {
+ sort->ibuf_nalloc = over_alloc_dd(dd->ncg_home+1);
+ srenew(sort->ibuf, sort->ibuf_nalloc);
+ }
+ ibuf = sort->ibuf;
+ /* Reorder the global cg index */
+ order_int_cg(dd->ncg_home, cgsort, dd->index_gl, ibuf);
+ /* Reorder the cginfo */
+ order_int_cg(dd->ncg_home, cgsort, fr->cginfo, ibuf);
+ /* Rebuild the local cg index */
+ if (dd->comm->bCGs)
+ {
+ ibuf[0] = 0;
+ for (i = 0; i < dd->ncg_home; i++)
+ {
+ cgsize = dd->cgindex[cgsort[i].ind+1] - dd->cgindex[cgsort[i].ind];
+ ibuf[i+1] = ibuf[i] + cgsize;
+ }
+ for (i = 0; i < dd->ncg_home+1; i++)
+ {
+ dd->cgindex[i] = ibuf[i];
+ }
+ }
+ else
+ {
+ for (i = 0; i < dd->ncg_home+1; i++)
+ {
+ dd->cgindex[i] = i;
+ }
+ }
+ /* Set the home atom number */
+ dd->nat_home = dd->cgindex[dd->ncg_home];
+
+ if (fr->cutoff_scheme == ecutsVERLET)
+ {
+ /* The atoms are now exactly in grid order, update the grid order */
+ nbnxn_set_atomorder(fr->nbv->nbs);
+ }
+ else
+ {
+ /* Copy the sorted ns cell indices back to the ns grid struct */
+ for (i = 0; i < dd->ncg_home; i++)
+ {
+ fr->ns.grid->cell_index[i] = cgsort[i].nsc;
+ }
+ fr->ns.grid->nr = dd->ncg_home;
+ }
+}
+
+static void add_dd_statistics(gmx_domdec_t *dd)
+{
+ gmx_domdec_comm_t *comm;
+ int ddnat;
+
+ comm = dd->comm;
+
+ for (ddnat = ddnatZONE; ddnat < ddnatNR; ddnat++)
+ {
+ comm->sum_nat[ddnat-ddnatZONE] +=
+ comm->nat[ddnat] - comm->nat[ddnat-1];
+ }
+ comm->ndecomp++;
+}
+
+void reset_dd_statistics_counters(gmx_domdec_t *dd)
+{
+ gmx_domdec_comm_t *comm;
+ int ddnat;
+
+ comm = dd->comm;
+
+ /* Reset all the statistics and counters for total run counting */
+ for (ddnat = ddnatZONE; ddnat < ddnatNR; ddnat++)
+ {
+ comm->sum_nat[ddnat-ddnatZONE] = 0;
+ }
+ comm->ndecomp = 0;
+ comm->nload = 0;
+ comm->load_step = 0;
+ comm->load_sum = 0;
+ comm->load_max = 0;
+ clear_ivec(comm->load_lim);
+ comm->load_mdf = 0;
+ comm->load_pme = 0;
+}
+
+void print_dd_statistics(t_commrec *cr, t_inputrec *ir, FILE *fplog)
+{
+ gmx_domdec_comm_t *comm;
+ int ddnat;
+ double av;
+
+ comm = cr->dd->comm;
+
+ gmx_sumd(ddnatNR-ddnatZONE, comm->sum_nat, cr);
+
+ if (fplog == NULL)
+ {
+ return;
+ }
+
+ fprintf(fplog, "\n D O M A I N D E C O M P O S I T I O N S T A T I S T I C S\n\n");
+
+ for (ddnat = ddnatZONE; ddnat < ddnatNR; ddnat++)
+ {
+ av = comm->sum_nat[ddnat-ddnatZONE]/comm->ndecomp;
+ switch (ddnat)
+ {
+ case ddnatZONE:
+ fprintf(fplog,
+ " av. #atoms communicated per step for force: %d x %.1f\n",
+ 2, av);
+ break;
+ case ddnatVSITE:
+ if (cr->dd->vsite_comm)
+ {
+ fprintf(fplog,
+ " av. #atoms communicated per step for vsites: %d x %.1f\n",
+ (EEL_PME(ir->coulombtype) || ir->coulombtype == eelEWALD) ? 3 : 2,
+ av);
+ }
+ break;
+ case ddnatCON:
+ if (cr->dd->constraint_comm)
+ {
+ fprintf(fplog,
+ " av. #atoms communicated per step for LINCS: %d x %.1f\n",
+ 1 + ir->nLincsIter, av);
+ }
+ break;
+ default:
+ gmx_incons(" Unknown type for DD statistics");
+ }
+ }
+ fprintf(fplog, "\n");
+
+ if (comm->bRecordLoad && EI_DYNAMICS(ir->eI))
+ {
+ print_dd_load_av(fplog, cr->dd);
+ }
+}
+
+void dd_partition_system(FILE *fplog,
+ gmx_large_int_t step,
+ t_commrec *cr,
+ gmx_bool bMasterState,
+ int nstglobalcomm,
+ t_state *state_global,
+ gmx_mtop_t *top_global,
+ t_inputrec *ir,
+ t_state *state_local,
+ rvec **f,
+ t_mdatoms *mdatoms,
+ gmx_localtop_t *top_local,
+ t_forcerec *fr,
+ gmx_vsite_t *vsite,
+ gmx_shellfc_t shellfc,
+ gmx_constr_t constr,
+ t_nrnb *nrnb,
+ gmx_wallcycle_t wcycle,
+ gmx_bool bVerbose)
+{
+ gmx_domdec_t *dd;
+ gmx_domdec_comm_t *comm;
+ gmx_ddbox_t ddbox = {0};
+ t_block *cgs_gl;
+ gmx_large_int_t step_pcoupl;
+ rvec cell_ns_x0, cell_ns_x1;
+ int i, j, n, ncgindex_set, ncg_home_old = -1, ncg_moved, nat_f_novirsum;
+ gmx_bool bBoxChanged, bNStGlobalComm, bDoDLB, bCheckDLB, bTurnOnDLB, bLogLoad;
+ gmx_bool bRedist, bSortCG, bResortAll;
+ ivec ncells_old = {0, 0, 0}, ncells_new = {0, 0, 0}, np;
+ real grid_density;
+ char sbuf[22];
+
+ dd = cr->dd;
+ comm = dd->comm;
+
+ bBoxChanged = (bMasterState || DEFORM(*ir));
+ if (ir->epc != epcNO)
+ {
+ /* With nstpcouple > 1 pressure coupling happens.
+ * one step after calculating the pressure.
+ * Box scaling happens at the end of the MD step,
+ * after the DD partitioning.
+ * We therefore have to do DLB in the first partitioning
+ * after an MD step where P-coupling occured.
+ * We need to determine the last step in which p-coupling occurred.
+ * MRS -- need to validate this for vv?
+ */
+ n = ir->nstpcouple;
+ if (n == 1)
+ {
+ step_pcoupl = step - 1;
+ }
+ else
+ {
+ step_pcoupl = ((step - 1)/n)*n + 1;
+ }
+ if (step_pcoupl >= comm->partition_step)
+ {
+ bBoxChanged = TRUE;
+ }
+ }
+
+ bNStGlobalComm = (step % nstglobalcomm == 0);
+
+ if (!comm->bDynLoadBal)
+ {
+ bDoDLB = FALSE;
+ }
+ else
+ {
+ /* Should we do dynamic load balacing this step?
+ * Since it requires (possibly expensive) global communication,
+ * we might want to do DLB less frequently.
+ */
+ if (bBoxChanged || ir->epc != epcNO)
+ {
+ bDoDLB = bBoxChanged;
+ }
+ else
+ {
+ bDoDLB = bNStGlobalComm;
+ }
+ }
+
+ /* Check if we have recorded loads on the nodes */
+ if (comm->bRecordLoad && dd_load_count(comm))
+ {
+ if (comm->eDLB == edlbAUTO && !comm->bDynLoadBal)
+ {
+ /* Check if we should use DLB at the second partitioning
+ * and every 100 partitionings,
+ * so the extra communication cost is negligible.
+ */
+ n = max(100, nstglobalcomm);
+ bCheckDLB = (comm->n_load_collect == 0 ||
+ comm->n_load_have % n == n-1);
+ }
+ else
+ {
+ bCheckDLB = FALSE;
+ }
+
+ /* Print load every nstlog, first and last step to the log file */
+ bLogLoad = ((ir->nstlog > 0 && step % ir->nstlog == 0) ||
+ comm->n_load_collect == 0 ||
+ (ir->nsteps >= 0 &&
+ (step + ir->nstlist > ir->init_step + ir->nsteps)));
+
+ /* Avoid extra communication due to verbose screen output
+ * when nstglobalcomm is set.
+ */
+ if (bDoDLB || bLogLoad || bCheckDLB ||
+ (bVerbose && (ir->nstlist == 0 || nstglobalcomm <= ir->nstlist)))
+ {
+ get_load_distribution(dd, wcycle);
+ if (DDMASTER(dd))
+ {
+ if (bLogLoad)
+ {
+ dd_print_load(fplog, dd, step-1);
+ }
+ if (bVerbose)
+ {
+ dd_print_load_verbose(dd);
+ }
+ }
+ comm->n_load_collect++;
+
+ if (bCheckDLB)
+ {
+ /* Since the timings are node dependent, the master decides */
+ if (DDMASTER(dd))
+ {
+ bTurnOnDLB =
+ (dd_force_imb_perf_loss(dd) >= DD_PERF_LOSS);
+ if (debug)
+ {
+ fprintf(debug, "step %s, imb loss %f\n",
+ gmx_step_str(step, sbuf),
+ dd_force_imb_perf_loss(dd));
+ }
+ }
+ dd_bcast(dd, sizeof(bTurnOnDLB), &bTurnOnDLB);
+ if (bTurnOnDLB)
+ {
+ turn_on_dlb(fplog, cr, step);
+ bDoDLB = TRUE;
+ }
+ }
+ }
+ comm->n_load_have++;
+ }
+
+ cgs_gl = &comm->cgs_gl;
+
+ bRedist = FALSE;
+ if (bMasterState)
+ {
+ /* Clear the old state */
+ clear_dd_indices(dd, 0, 0);
+ ncgindex_set = 0;
+
+ set_ddbox(dd, bMasterState, cr, ir, state_global->box,
+ TRUE, cgs_gl, state_global->x, &ddbox);
+
+ get_cg_distribution(fplog, step, dd, cgs_gl,
+ state_global->box, &ddbox, state_global->x);
+
+ dd_distribute_state(dd, cgs_gl,
+ state_global, state_local, f);
+
+ dd_make_local_cgs(dd, &top_local->cgs);
+
+ /* Ensure that we have space for the new distribution */
+ dd_check_alloc_ncg(fr, state_local, f, dd->ncg_home);
+
+ if (fr->cutoff_scheme == ecutsGROUP)
+ {
+ calc_cgcm(fplog, 0, dd->ncg_home,
+ &top_local->cgs, state_local->x, fr->cg_cm);
+ }
+
+ inc_nrnb(nrnb, eNR_CGCM, dd->nat_home);
+
+ dd_set_cginfo(dd->index_gl, 0, dd->ncg_home, fr, comm->bLocalCG);
+ }
+ else if (state_local->ddp_count != dd->ddp_count)
+ {
+ if (state_local->ddp_count > dd->ddp_count)
+ {
+ gmx_fatal(FARGS, "Internal inconsistency state_local->ddp_count (%d) > dd->ddp_count (%d)", state_local->ddp_count, dd->ddp_count);
+ }
+
+ if (state_local->ddp_count_cg_gl != state_local->ddp_count)
+ {
+ gmx_fatal(FARGS, "Internal inconsistency state_local->ddp_count_cg_gl (%d) != state_local->ddp_count (%d)", state_local->ddp_count_cg_gl, state_local->ddp_count);
+ }
+
+ /* Clear the old state */
+ clear_dd_indices(dd, 0, 0);
+
+ /* Build the new indices */
+ rebuild_cgindex(dd, cgs_gl->index, state_local);
+ make_dd_indices(dd, cgs_gl->index, 0);
+ ncgindex_set = dd->ncg_home;
+
+ if (fr->cutoff_scheme == ecutsGROUP)
+ {
+ /* Redetermine the cg COMs */
+ calc_cgcm(fplog, 0, dd->ncg_home,
+ &top_local->cgs, state_local->x, fr->cg_cm);
+ }
+
+ inc_nrnb(nrnb, eNR_CGCM, dd->nat_home);
+
+ dd_set_cginfo(dd->index_gl, 0, dd->ncg_home, fr, comm->bLocalCG);
+
+ set_ddbox(dd, bMasterState, cr, ir, state_local->box,
+ TRUE, &top_local->cgs, state_local->x, &ddbox);
+
+ bRedist = comm->bDynLoadBal;
+ }
+ else
+ {
+ /* We have the full state, only redistribute the cgs */
+
+ /* Clear the non-home indices */
+ clear_dd_indices(dd, dd->ncg_home, dd->nat_home);
+ ncgindex_set = 0;
+
+ /* Avoid global communication for dim's without pbc and -gcom */
+ if (!bNStGlobalComm)
+ {
+ copy_rvec(comm->box0, ddbox.box0 );
+ copy_rvec(comm->box_size, ddbox.box_size);
+ }
+ set_ddbox(dd, bMasterState, cr, ir, state_local->box,
+ bNStGlobalComm, &top_local->cgs, state_local->x, &ddbox);
+
+ bBoxChanged = TRUE;
+ bRedist = TRUE;
+ }
+ /* For dim's without pbc and -gcom */
+ copy_rvec(ddbox.box0, comm->box0 );
+ copy_rvec(ddbox.box_size, comm->box_size);
+
+ set_dd_cell_sizes(dd, &ddbox, dynamic_dd_box(&ddbox, ir), bMasterState, bDoDLB,
+ step, wcycle);
+
+ if (comm->nstDDDumpGrid > 0 && step % comm->nstDDDumpGrid == 0)
+ {
+ write_dd_grid_pdb("dd_grid", step, dd, state_local->box, &ddbox);
+ }
+
+ /* Check if we should sort the charge groups */
+ if (comm->nstSortCG > 0)
+ {
+ bSortCG = (bMasterState ||
+ (bRedist && (step % comm->nstSortCG == 0)));
+ }
+ else
+ {
+ bSortCG = FALSE;
+ }
+
+ ncg_home_old = dd->ncg_home;
+
+ ncg_moved = 0;
+ if (bRedist)
+ {
+ wallcycle_sub_start(wcycle, ewcsDD_REDIST);
+
+ dd_redistribute_cg(fplog, step, dd, ddbox.tric_dir,
+ state_local, f, fr,
+ !bSortCG, nrnb, &ncgindex_set, &ncg_moved);
+
+ wallcycle_sub_stop(wcycle, ewcsDD_REDIST);
+ }
+
+ get_nsgrid_boundaries(ddbox.nboundeddim, state_local->box,
+ dd, &ddbox,
+ &comm->cell_x0, &comm->cell_x1,
+ dd->ncg_home, fr->cg_cm,
+ cell_ns_x0, cell_ns_x1, &grid_density);
+
+ if (bBoxChanged)
+ {
+ comm_dd_ns_cell_sizes(dd, &ddbox, cell_ns_x0, cell_ns_x1, step);
+ }
+
+ switch (fr->cutoff_scheme)
+ {
+ case ecutsGROUP:
+ copy_ivec(fr->ns.grid->n, ncells_old);
+ grid_first(fplog, fr->ns.grid, dd, &ddbox,
+ state_local->box, cell_ns_x0, cell_ns_x1,
+ fr->rlistlong, grid_density);
+ break;
+ case ecutsVERLET:
+ nbnxn_get_ncells(fr->nbv->nbs, &ncells_old[XX], &ncells_old[YY]);
+ break;
+ default:
+ gmx_incons("unimplemented");
+ }
+ /* We need to store tric_dir for dd_get_ns_ranges called from ns.c */
+ copy_ivec(ddbox.tric_dir, comm->tric_dir);
+
+ if (bSortCG)
+ {
+ wallcycle_sub_start(wcycle, ewcsDD_GRID);
+
+ /* Sort the state on charge group position.
+ * This enables exact restarts from this step.
+ * It also improves performance by about 15% with larger numbers
+ * of atoms per node.
+ */
+
+ /* Fill the ns grid with the home cell,
+ * so we can sort with the indices.
+ */
+ set_zones_ncg_home(dd);
+
+ switch (fr->cutoff_scheme)
+ {
+ case ecutsVERLET:
+ set_zones_size(dd, state_local->box, &ddbox, 0, 1);
+
+ nbnxn_put_on_grid(fr->nbv->nbs, fr->ePBC, state_local->box,
+ 0,
+ comm->zones.size[0].bb_x0,
+ comm->zones.size[0].bb_x1,
+ 0, dd->ncg_home,
+ comm->zones.dens_zone0,
+ fr->cginfo,
+ state_local->x,
+ ncg_moved, bRedist ? comm->moved : NULL,
+ fr->nbv->grp[eintLocal].kernel_type,
+ fr->nbv->grp[eintLocal].nbat);
+
+ nbnxn_get_ncells(fr->nbv->nbs, &ncells_new[XX], &ncells_new[YY]);
+ break;
+ case ecutsGROUP:
+ fill_grid(&comm->zones, fr->ns.grid, dd->ncg_home,
+ 0, dd->ncg_home, fr->cg_cm);
+
+ copy_ivec(fr->ns.grid->n, ncells_new);
+ break;
+ default:
+ gmx_incons("unimplemented");
+ }
+
+ bResortAll = bMasterState;
+
+ /* Check if we can user the old order and ns grid cell indices
+ * of the charge groups to sort the charge groups efficiently.
+ */
+ if (ncells_new[XX] != ncells_old[XX] ||
+ ncells_new[YY] != ncells_old[YY] ||
+ ncells_new[ZZ] != ncells_old[ZZ])
+ {
+ bResortAll = TRUE;
+ }
+
+ if (debug)
+ {
+ fprintf(debug, "Step %s, sorting the %d home charge groups\n",
+ gmx_step_str(step, sbuf), dd->ncg_home);
+ }
+ dd_sort_state(dd, fr->cg_cm, fr, state_local,
+ bResortAll ? -1 : ncg_home_old);
+ /* Rebuild all the indices */
+ ga2la_clear(dd->ga2la);
+ ncgindex_set = 0;
+
+ wallcycle_sub_stop(wcycle, ewcsDD_GRID);
+ }
+
+ wallcycle_sub_start(wcycle, ewcsDD_SETUPCOMM);
+
+ /* Setup up the communication and communicate the coordinates */
+ setup_dd_communication(dd, state_local->box, &ddbox, fr, state_local, f);
+
+ /* Set the indices */
+ make_dd_indices(dd, cgs_gl->index, ncgindex_set);
+
+ /* Set the charge group boundaries for neighbor searching */
+ set_cg_boundaries(&comm->zones);
+
+ if (fr->cutoff_scheme == ecutsVERLET)
+ {
+ set_zones_size(dd, state_local->box, &ddbox,
+ bSortCG ? 1 : 0, comm->zones.n);
+ }
+
+ wallcycle_sub_stop(wcycle, ewcsDD_SETUPCOMM);
+
+ /*
+ write_dd_pdb("dd_home",step,"dump",top_global,cr,
+ -1,state_local->x,state_local->box);
+ */
+
+ wallcycle_sub_start(wcycle, ewcsDD_MAKETOP);
+
+ /* Extract a local topology from the global topology */
+ for (i = 0; i < dd->ndim; i++)
+ {
+ np[dd->dim[i]] = comm->cd[i].np;
+ }
+ dd_make_local_top(dd, &comm->zones, dd->npbcdim, state_local->box,
+ comm->cellsize_min, np,
+ fr,
+ fr->cutoff_scheme == ecutsGROUP ? fr->cg_cm : state_local->x,
+ vsite, top_global, top_local);
+
+ wallcycle_sub_stop(wcycle, ewcsDD_MAKETOP);
+
+ wallcycle_sub_start(wcycle, ewcsDD_MAKECONSTR);
+
+ /* Set up the special atom communication */
+ n = comm->nat[ddnatZONE];
+ for (i = ddnatZONE+1; i < ddnatNR; i++)
+ {
+ switch (i)
+ {
+ case ddnatVSITE:
+ if (vsite && vsite->n_intercg_vsite)
+ {
+ n = dd_make_local_vsites(dd, n, top_local->idef.il);
+ }
+ break;
+ case ddnatCON:
+ if (dd->bInterCGcons || dd->bInterCGsettles)
+ {
+ /* Only for inter-cg constraints we need special code */
+ n = dd_make_local_constraints(dd, n, top_global, fr->cginfo,
+ constr, ir->nProjOrder,
+ top_local->idef.il);
+ }
+ break;
+ default:
+ gmx_incons("Unknown special atom type setup");
+ }
+ comm->nat[i] = n;
+ }
+
+ wallcycle_sub_stop(wcycle, ewcsDD_MAKECONSTR);
+
+ wallcycle_sub_start(wcycle, ewcsDD_TOPOTHER);
+
+ /* Make space for the extra coordinates for virtual site
+ * or constraint communication.
+ */
+ state_local->natoms = comm->nat[ddnatNR-1];
+ if (state_local->natoms > state_local->nalloc)
+ {
+ dd_realloc_state(state_local, f, state_local->natoms);
+ }
+
+ if (fr->bF_NoVirSum)
+ {
+ if (vsite && vsite->n_intercg_vsite)
+ {
+ nat_f_novirsum = comm->nat[ddnatVSITE];
+ }
+ else
+ {
+ if (EEL_FULL(ir->coulombtype) && dd->n_intercg_excl > 0)
+ {
+ nat_f_novirsum = dd->nat_tot;
+ }
+ else
+ {
+ nat_f_novirsum = dd->nat_home;
+ }
+ }
+ }
+ else
+ {
+ nat_f_novirsum = 0;
+ }
+
+ /* Set the number of atoms required for the force calculation.
+ * Forces need to be constrained when using a twin-range setup
+ * or with energy minimization. For simple simulations we could
+ * avoid some allocation, zeroing and copying, but this is
+ * probably not worth the complications ande checking.
+ */
+ forcerec_set_ranges(fr, dd->ncg_home, dd->ncg_tot,
+ dd->nat_tot, comm->nat[ddnatCON], nat_f_novirsum);
+
+ /* We make the all mdatoms up to nat_tot_con.
+ * We could save some work by only setting invmass
+ * between nat_tot and nat_tot_con.
+ */
+ /* This call also sets the new number of home particles to dd->nat_home */
+ atoms2md(top_global, ir,
+ comm->nat[ddnatCON], dd->gatindex, 0, dd->nat_home, mdatoms);
+
+ /* Now we have the charges we can sort the FE interactions */
+ dd_sort_local_top(dd, mdatoms, top_local);
+
+ if (vsite != NULL)
+ {
+ /* Now we have updated mdatoms, we can do the last vsite bookkeeping */
+ split_vsites_over_threads(top_local->idef.il, mdatoms, FALSE, vsite);
+ }
+
+ if (shellfc)
+ {
+ /* Make the local shell stuff, currently no communication is done */
+ make_local_shells(cr, mdatoms, shellfc);
+ }
+
+ if (ir->implicit_solvent)
+ {
+ make_local_gb(cr, fr->born, ir->gb_algorithm);
+ }
+
+ setup_bonded_threading(fr, &top_local->idef);
+
+ if (!(cr->duty & DUTY_PME))
+ {
+ /* Send the charges to our PME only node */
+ gmx_pme_send_q(cr, mdatoms->nChargePerturbed,
+ mdatoms->chargeA, mdatoms->chargeB,
+ dd_pme_maxshift_x(dd), dd_pme_maxshift_y(dd));
+ }
+
+ if (constr)
+ {
+ set_constraints(constr, top_local, ir, mdatoms, cr);
+ }
+
+ if (ir->ePull != epullNO)
+ {
+ /* Update the local pull groups */
+ dd_make_local_pull_groups(dd, ir->pull, mdatoms);
+ }
+
+ if (ir->bRot)
+ {
+ /* Update the local rotation groups */
+ dd_make_local_rotation_groups(dd, ir->rot);
+ }
+
+
+ add_dd_statistics(dd);
+
+ /* Make sure we only count the cycles for this DD partitioning */
+ clear_dd_cycle_counts(dd);
+
+ /* Because the order of the atoms might have changed since
+ * the last vsite construction, we need to communicate the constructing
+ * atom coordinates again (for spreading the forces this MD step).
+ */
+ dd_move_x_vsites(dd, state_local->box, state_local->x);
+
+ wallcycle_sub_stop(wcycle, ewcsDD_TOPOTHER);
+
+ if (comm->nstDDDump > 0 && step % comm->nstDDDump == 0)
+ {
+ dd_move_x(dd, state_local->box, state_local->x);
+ write_dd_pdb("dd_dump", step, "dump", top_global, cr,
+ -1, state_local->x, state_local->box);
+ }
+
+ /* Store the partitioning step */
+ comm->partition_step = step;
+
+ /* Increase the DD partitioning counter */
+ dd->ddp_count++;
+ /* The state currently matches this DD partitioning count, store it */
+ state_local->ddp_count = dd->ddp_count;
+ if (bMasterState)
+ {
+ /* The DD master node knows the complete cg distribution,
+ * store the count so we can possibly skip the cg info communication.
+ */
+ comm->master_cg_ddp_count = (bSortCG ? 0 : dd->ddp_count);
+ }
+
+ if (comm->DD_debug > 0)
+ {
+ /* Set the env var GMX_DD_DEBUG if you suspect corrupted indices */
+ check_index_consistency(dd, top_global->natoms, ncg_mtop(top_global),
+ "after partitioning");
+ }
+}
--- /dev/null
- gmx_bool *bEmulateGPU)
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * VERSION 3.2.0
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2004, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * GROwing Monsters And Cloning Shrimps
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <math.h>
+#include <string.h>
+#include <assert.h>
+#include "sysstuff.h"
+#include "typedefs.h"
+#include "vec.h"
+#include "maths.h"
+#include "macros.h"
+#include "smalloc.h"
+#include "macros.h"
+#include "gmx_fatal.h"
+#include "gmx_fatal_collective.h"
+#include "physics.h"
+#include "force.h"
+#include "tables.h"
+#include "nonbonded.h"
+#include "invblock.h"
+#include "names.h"
+#include "network.h"
+#include "pbc.h"
+#include "ns.h"
+#include "mshift.h"
+#include "txtdump.h"
+#include "coulomb.h"
+#include "md_support.h"
+#include "md_logging.h"
+#include "domdec.h"
+#include "partdec.h"
+#include "qmmm.h"
+#include "copyrite.h"
+#include "mtop_util.h"
+#include "nbnxn_search.h"
+#include "nbnxn_atomdata.h"
+#include "nbnxn_consts.h"
+#include "statutil.h"
+#include "gmx_omp_nthreads.h"
+#include "gmx_detect_hardware.h"
+
+#ifdef _MSC_VER
+/* MSVC definition for __cpuid() */
+#include <intrin.h>
+#endif
+
+#include "types/nbnxn_cuda_types_ext.h"
+#include "gpu_utils.h"
+#include "nbnxn_cuda_data_mgmt.h"
+#include "pmalloc_cuda.h"
+
+t_forcerec *mk_forcerec(void)
+{
+ t_forcerec *fr;
+
+ snew(fr, 1);
+
+ return fr;
+}
+
+#ifdef DEBUG
+static void pr_nbfp(FILE *fp, real *nbfp, gmx_bool bBHAM, int atnr)
+{
+ int i, j;
+
+ for (i = 0; (i < atnr); i++)
+ {
+ for (j = 0; (j < atnr); j++)
+ {
+ fprintf(fp, "%2d - %2d", i, j);
+ if (bBHAM)
+ {
+ fprintf(fp, " a=%10g, b=%10g, c=%10g\n", BHAMA(nbfp, atnr, i, j),
+ BHAMB(nbfp, atnr, i, j), BHAMC(nbfp, atnr, i, j)/6.0);
+ }
+ else
+ {
+ fprintf(fp, " c6=%10g, c12=%10g\n", C6(nbfp, atnr, i, j)/6.0,
+ C12(nbfp, atnr, i, j)/12.0);
+ }
+ }
+ }
+}
+#endif
+
+static real *mk_nbfp(const gmx_ffparams_t *idef, gmx_bool bBHAM)
+{
+ real *nbfp;
+ int i, j, k, atnr;
+
+ atnr = idef->atnr;
+ if (bBHAM)
+ {
+ snew(nbfp, 3*atnr*atnr);
+ for (i = k = 0; (i < atnr); i++)
+ {
+ for (j = 0; (j < atnr); j++, k++)
+ {
+ BHAMA(nbfp, atnr, i, j) = idef->iparams[k].bham.a;
+ BHAMB(nbfp, atnr, i, j) = idef->iparams[k].bham.b;
+ /* nbfp now includes the 6.0 derivative prefactor */
+ BHAMC(nbfp, atnr, i, j) = idef->iparams[k].bham.c*6.0;
+ }
+ }
+ }
+ else
+ {
+ snew(nbfp, 2*atnr*atnr);
+ for (i = k = 0; (i < atnr); i++)
+ {
+ for (j = 0; (j < atnr); j++, k++)
+ {
+ /* nbfp now includes the 6.0/12.0 derivative prefactors */
+ C6(nbfp, atnr, i, j) = idef->iparams[k].lj.c6*6.0;
+ C12(nbfp, atnr, i, j) = idef->iparams[k].lj.c12*12.0;
+ }
+ }
+ }
+
+ return nbfp;
+}
+
+/* This routine sets fr->solvent_opt to the most common solvent in the
+ * system, e.g. esolSPC or esolTIP4P. It will also mark each charge group in
+ * the fr->solvent_type array with the correct type (or esolNO).
+ *
+ * Charge groups that fulfill the conditions but are not identical to the
+ * most common one will be marked as esolNO in the solvent_type array.
+ *
+ * TIP3p is identical to SPC for these purposes, so we call it
+ * SPC in the arrays (Apologies to Bill Jorgensen ;-)
+ *
+ * NOTE: QM particle should not
+ * become an optimized solvent. Not even if there is only one charge
+ * group in the Qm
+ */
+
+typedef struct
+{
+ int model;
+ int count;
+ int vdwtype[4];
+ real charge[4];
+} solvent_parameters_t;
+
+static void
+check_solvent_cg(const gmx_moltype_t *molt,
+ int cg0,
+ int nmol,
+ const unsigned char *qm_grpnr,
+ const t_grps *qm_grps,
+ t_forcerec * fr,
+ int *n_solvent_parameters,
+ solvent_parameters_t **solvent_parameters_p,
+ int cginfo,
+ int *cg_sp)
+{
+ const t_blocka * excl;
+ t_atom *atom;
+ int j, k;
+ int j0, j1, nj;
+ gmx_bool perturbed;
+ gmx_bool has_vdw[4];
+ gmx_bool match;
+ real tmp_charge[4];
+ int tmp_vdwtype[4];
+ int tjA;
+ gmx_bool qm;
+ solvent_parameters_t *solvent_parameters;
+
+ /* We use a list with parameters for each solvent type.
+ * Every time we discover a new molecule that fulfills the basic
+ * conditions for a solvent we compare with the previous entries
+ * in these lists. If the parameters are the same we just increment
+ * the counter for that type, and otherwise we create a new type
+ * based on the current molecule.
+ *
+ * Once we've finished going through all molecules we check which
+ * solvent is most common, and mark all those molecules while we
+ * clear the flag on all others.
+ */
+
+ solvent_parameters = *solvent_parameters_p;
+
+ /* Mark the cg first as non optimized */
+ *cg_sp = -1;
+
+ /* Check if this cg has no exclusions with atoms in other charge groups
+ * and all atoms inside the charge group excluded.
+ * We only have 3 or 4 atom solvent loops.
+ */
+ if (GET_CGINFO_EXCL_INTER(cginfo) ||
+ !GET_CGINFO_EXCL_INTRA(cginfo))
+ {
+ return;
+ }
+
+ /* Get the indices of the first atom in this charge group */
+ j0 = molt->cgs.index[cg0];
+ j1 = molt->cgs.index[cg0+1];
+
+ /* Number of atoms in our molecule */
+ nj = j1 - j0;
+
+ if (debug)
+ {
+ fprintf(debug,
+ "Moltype '%s': there are %d atoms in this charge group\n",
+ *molt->name, nj);
+ }
+
+ /* Check if it could be an SPC (3 atoms) or TIP4p (4) water,
+ * otherwise skip it.
+ */
+ if (nj < 3 || nj > 4)
+ {
+ return;
+ }
+
+ /* Check if we are doing QM on this group */
+ qm = FALSE;
+ if (qm_grpnr != NULL)
+ {
+ for (j = j0; j < j1 && !qm; j++)
+ {
+ qm = (qm_grpnr[j] < qm_grps->nr - 1);
+ }
+ }
+ /* Cannot use solvent optimization with QM */
+ if (qm)
+ {
+ return;
+ }
+
+ atom = molt->atoms.atom;
+
+ /* Still looks like a solvent, time to check parameters */
+
+ /* If it is perturbed (free energy) we can't use the solvent loops,
+ * so then we just skip to the next molecule.
+ */
+ perturbed = FALSE;
+
+ for (j = j0; j < j1 && !perturbed; j++)
+ {
+ perturbed = PERTURBED(atom[j]);
+ }
+
+ if (perturbed)
+ {
+ return;
+ }
+
+ /* Now it's only a question if the VdW and charge parameters
+ * are OK. Before doing the check we compare and see if they are
+ * identical to a possible previous solvent type.
+ * First we assign the current types and charges.
+ */
+ for (j = 0; j < nj; j++)
+ {
+ tmp_vdwtype[j] = atom[j0+j].type;
+ tmp_charge[j] = atom[j0+j].q;
+ }
+
+ /* Does it match any previous solvent type? */
+ for (k = 0; k < *n_solvent_parameters; k++)
+ {
+ match = TRUE;
+
+
+ /* We can only match SPC with 3 atoms and TIP4p with 4 atoms */
+ if ( (solvent_parameters[k].model == esolSPC && nj != 3) ||
+ (solvent_parameters[k].model == esolTIP4P && nj != 4) )
+ {
+ match = FALSE;
+ }
+
+ /* Check that types & charges match for all atoms in molecule */
+ for (j = 0; j < nj && match == TRUE; j++)
+ {
+ if (tmp_vdwtype[j] != solvent_parameters[k].vdwtype[j])
+ {
+ match = FALSE;
+ }
+ if (tmp_charge[j] != solvent_parameters[k].charge[j])
+ {
+ match = FALSE;
+ }
+ }
+ if (match == TRUE)
+ {
+ /* Congratulations! We have a matched solvent.
+ * Flag it with this type for later processing.
+ */
+ *cg_sp = k;
+ solvent_parameters[k].count += nmol;
+
+ /* We are done with this charge group */
+ return;
+ }
+ }
+
+ /* If we get here, we have a tentative new solvent type.
+ * Before we add it we must check that it fulfills the requirements
+ * of the solvent optimized loops. First determine which atoms have
+ * VdW interactions.
+ */
+ for (j = 0; j < nj; j++)
+ {
+ has_vdw[j] = FALSE;
+ tjA = tmp_vdwtype[j];
+
+ /* Go through all other tpes and see if any have non-zero
+ * VdW parameters when combined with this one.
+ */
+ for (k = 0; k < fr->ntype && (has_vdw[j] == FALSE); k++)
+ {
+ /* We already checked that the atoms weren't perturbed,
+ * so we only need to check state A now.
+ */
+ if (fr->bBHAM)
+ {
+ has_vdw[j] = (has_vdw[j] ||
+ (BHAMA(fr->nbfp, fr->ntype, tjA, k) != 0.0) ||
+ (BHAMB(fr->nbfp, fr->ntype, tjA, k) != 0.0) ||
+ (BHAMC(fr->nbfp, fr->ntype, tjA, k) != 0.0));
+ }
+ else
+ {
+ /* Standard LJ */
+ has_vdw[j] = (has_vdw[j] ||
+ (C6(fr->nbfp, fr->ntype, tjA, k) != 0.0) ||
+ (C12(fr->nbfp, fr->ntype, tjA, k) != 0.0));
+ }
+ }
+ }
+
+ /* Now we know all we need to make the final check and assignment. */
+ if (nj == 3)
+ {
+ /* So, is it an SPC?
+ * For this we require thatn all atoms have charge,
+ * the charges on atom 2 & 3 should be the same, and only
+ * atom 1 might have VdW.
+ */
+ if (has_vdw[1] == FALSE &&
+ has_vdw[2] == FALSE &&
+ tmp_charge[0] != 0 &&
+ tmp_charge[1] != 0 &&
+ tmp_charge[2] == tmp_charge[1])
+ {
+ srenew(solvent_parameters, *n_solvent_parameters+1);
+ solvent_parameters[*n_solvent_parameters].model = esolSPC;
+ solvent_parameters[*n_solvent_parameters].count = nmol;
+ for (k = 0; k < 3; k++)
+ {
+ solvent_parameters[*n_solvent_parameters].vdwtype[k] = tmp_vdwtype[k];
+ solvent_parameters[*n_solvent_parameters].charge[k] = tmp_charge[k];
+ }
+
+ *cg_sp = *n_solvent_parameters;
+ (*n_solvent_parameters)++;
+ }
+ }
+ else if (nj == 4)
+ {
+ /* Or could it be a TIP4P?
+ * For this we require thatn atoms 2,3,4 have charge, but not atom 1.
+ * Only atom 1 mght have VdW.
+ */
+ if (has_vdw[1] == FALSE &&
+ has_vdw[2] == FALSE &&
+ has_vdw[3] == FALSE &&
+ tmp_charge[0] == 0 &&
+ tmp_charge[1] != 0 &&
+ tmp_charge[2] == tmp_charge[1] &&
+ tmp_charge[3] != 0)
+ {
+ srenew(solvent_parameters, *n_solvent_parameters+1);
+ solvent_parameters[*n_solvent_parameters].model = esolTIP4P;
+ solvent_parameters[*n_solvent_parameters].count = nmol;
+ for (k = 0; k < 4; k++)
+ {
+ solvent_parameters[*n_solvent_parameters].vdwtype[k] = tmp_vdwtype[k];
+ solvent_parameters[*n_solvent_parameters].charge[k] = tmp_charge[k];
+ }
+
+ *cg_sp = *n_solvent_parameters;
+ (*n_solvent_parameters)++;
+ }
+ }
+
+ *solvent_parameters_p = solvent_parameters;
+}
+
+static void
+check_solvent(FILE * fp,
+ const gmx_mtop_t * mtop,
+ t_forcerec * fr,
+ cginfo_mb_t *cginfo_mb)
+{
+ const t_block * cgs;
+ const t_block * mols;
+ const gmx_moltype_t *molt;
+ int mb, mol, cg_mol, at_offset, cg_offset, am, cgm, i, nmol_ch, nmol;
+ int n_solvent_parameters;
+ solvent_parameters_t *solvent_parameters;
+ int **cg_sp;
+ int bestsp, bestsol;
+
+ if (debug)
+ {
+ fprintf(debug, "Going to determine what solvent types we have.\n");
+ }
+
+ mols = &mtop->mols;
+
+ n_solvent_parameters = 0;
+ solvent_parameters = NULL;
+ /* Allocate temporary array for solvent type */
+ snew(cg_sp, mtop->nmolblock);
+
+ cg_offset = 0;
+ at_offset = 0;
+ for (mb = 0; mb < mtop->nmolblock; mb++)
+ {
+ molt = &mtop->moltype[mtop->molblock[mb].type];
+ cgs = &molt->cgs;
+ /* Here we have to loop over all individual molecules
+ * because we need to check for QMMM particles.
+ */
+ snew(cg_sp[mb], cginfo_mb[mb].cg_mod);
+ nmol_ch = cginfo_mb[mb].cg_mod/cgs->nr;
+ nmol = mtop->molblock[mb].nmol/nmol_ch;
+ for (mol = 0; mol < nmol_ch; mol++)
+ {
+ cgm = mol*cgs->nr;
+ am = mol*cgs->index[cgs->nr];
+ for (cg_mol = 0; cg_mol < cgs->nr; cg_mol++)
+ {
+ check_solvent_cg(molt, cg_mol, nmol,
+ mtop->groups.grpnr[egcQMMM] ?
+ mtop->groups.grpnr[egcQMMM]+at_offset+am : 0,
+ &mtop->groups.grps[egcQMMM],
+ fr,
+ &n_solvent_parameters, &solvent_parameters,
+ cginfo_mb[mb].cginfo[cgm+cg_mol],
+ &cg_sp[mb][cgm+cg_mol]);
+ }
+ }
+ cg_offset += cgs->nr;
+ at_offset += cgs->index[cgs->nr];
+ }
+
+ /* Puh! We finished going through all charge groups.
+ * Now find the most common solvent model.
+ */
+
+ /* Most common solvent this far */
+ bestsp = -2;
+ for (i = 0; i < n_solvent_parameters; i++)
+ {
+ if (bestsp == -2 ||
+ solvent_parameters[i].count > solvent_parameters[bestsp].count)
+ {
+ bestsp = i;
+ }
+ }
+
+ if (bestsp >= 0)
+ {
+ bestsol = solvent_parameters[bestsp].model;
+ }
+ else
+ {
+ bestsol = esolNO;
+ }
+
+#ifdef DISABLE_WATER_NLIST
+ bestsol = esolNO;
+#endif
+
+ fr->nWatMol = 0;
+ for (mb = 0; mb < mtop->nmolblock; mb++)
+ {
+ cgs = &mtop->moltype[mtop->molblock[mb].type].cgs;
+ nmol = (mtop->molblock[mb].nmol*cgs->nr)/cginfo_mb[mb].cg_mod;
+ for (i = 0; i < cginfo_mb[mb].cg_mod; i++)
+ {
+ if (cg_sp[mb][i] == bestsp)
+ {
+ SET_CGINFO_SOLOPT(cginfo_mb[mb].cginfo[i], bestsol);
+ fr->nWatMol += nmol;
+ }
+ else
+ {
+ SET_CGINFO_SOLOPT(cginfo_mb[mb].cginfo[i], esolNO);
+ }
+ }
+ sfree(cg_sp[mb]);
+ }
+ sfree(cg_sp);
+
+ if (bestsol != esolNO && fp != NULL)
+ {
+ fprintf(fp, "\nEnabling %s-like water optimization for %d molecules.\n\n",
+ esol_names[bestsol],
+ solvent_parameters[bestsp].count);
+ }
+
+ sfree(solvent_parameters);
+ fr->solvent_opt = bestsol;
+}
+
+enum {
+ acNONE = 0, acCONSTRAINT, acSETTLE
+};
+
+static cginfo_mb_t *init_cginfo_mb(FILE *fplog, const gmx_mtop_t *mtop,
+ t_forcerec *fr, gmx_bool bNoSolvOpt,
+ gmx_bool *bExcl_IntraCGAll_InterCGNone)
+{
+ const t_block *cgs;
+ const t_blocka *excl;
+ const gmx_moltype_t *molt;
+ const gmx_molblock_t *molb;
+ cginfo_mb_t *cginfo_mb;
+ gmx_bool *type_VDW;
+ int *cginfo;
+ int cg_offset, a_offset, cgm, am;
+ int mb, m, ncg_tot, cg, a0, a1, gid, ai, j, aj, excl_nalloc;
+ int *a_con;
+ int ftype;
+ int ia;
+ gmx_bool bId, *bExcl, bExclIntraAll, bExclInter, bHaveVDW, bHaveQ;
+
+ ncg_tot = ncg_mtop(mtop);
+ snew(cginfo_mb, mtop->nmolblock);
+
+ snew(type_VDW, fr->ntype);
+ for (ai = 0; ai < fr->ntype; ai++)
+ {
+ type_VDW[ai] = FALSE;
+ for (j = 0; j < fr->ntype; j++)
+ {
+ type_VDW[ai] = type_VDW[ai] ||
+ fr->bBHAM ||
+ C6(fr->nbfp, fr->ntype, ai, j) != 0 ||
+ C12(fr->nbfp, fr->ntype, ai, j) != 0;
+ }
+ }
+
+ *bExcl_IntraCGAll_InterCGNone = TRUE;
+
+ excl_nalloc = 10;
+ snew(bExcl, excl_nalloc);
+ cg_offset = 0;
+ a_offset = 0;
+ for (mb = 0; mb < mtop->nmolblock; mb++)
+ {
+ molb = &mtop->molblock[mb];
+ molt = &mtop->moltype[molb->type];
+ cgs = &molt->cgs;
+ excl = &molt->excls;
+
+ /* Check if the cginfo is identical for all molecules in this block.
+ * If so, we only need an array of the size of one molecule.
+ * Otherwise we make an array of #mol times #cgs per molecule.
+ */
+ bId = TRUE;
+ am = 0;
+ for (m = 0; m < molb->nmol; m++)
+ {
+ am = m*cgs->index[cgs->nr];
+ for (cg = 0; cg < cgs->nr; cg++)
+ {
+ a0 = cgs->index[cg];
+ a1 = cgs->index[cg+1];
+ if (ggrpnr(&mtop->groups, egcENER, a_offset+am+a0) !=
+ ggrpnr(&mtop->groups, egcENER, a_offset +a0))
+ {
+ bId = FALSE;
+ }
+ if (mtop->groups.grpnr[egcQMMM] != NULL)
+ {
+ for (ai = a0; ai < a1; ai++)
+ {
+ if (mtop->groups.grpnr[egcQMMM][a_offset+am+ai] !=
+ mtop->groups.grpnr[egcQMMM][a_offset +ai])
+ {
+ bId = FALSE;
+ }
+ }
+ }
+ }
+ }
+
+ cginfo_mb[mb].cg_start = cg_offset;
+ cginfo_mb[mb].cg_end = cg_offset + molb->nmol*cgs->nr;
+ cginfo_mb[mb].cg_mod = (bId ? 1 : molb->nmol)*cgs->nr;
+ snew(cginfo_mb[mb].cginfo, cginfo_mb[mb].cg_mod);
+ cginfo = cginfo_mb[mb].cginfo;
+
+ /* Set constraints flags for constrained atoms */
+ snew(a_con, molt->atoms.nr);
+ for (ftype = 0; ftype < F_NRE; ftype++)
+ {
+ if (interaction_function[ftype].flags & IF_CONSTRAINT)
+ {
+ int nral;
+
+ nral = NRAL(ftype);
+ for (ia = 0; ia < molt->ilist[ftype].nr; ia += 1+nral)
+ {
+ int a;
+
+ for (a = 0; a < nral; a++)
+ {
+ a_con[molt->ilist[ftype].iatoms[ia+1+a]] =
+ (ftype == F_SETTLE ? acSETTLE : acCONSTRAINT);
+ }
+ }
+ }
+ }
+
+ for (m = 0; m < (bId ? 1 : molb->nmol); m++)
+ {
+ cgm = m*cgs->nr;
+ am = m*cgs->index[cgs->nr];
+ for (cg = 0; cg < cgs->nr; cg++)
+ {
+ a0 = cgs->index[cg];
+ a1 = cgs->index[cg+1];
+
+ /* Store the energy group in cginfo */
+ gid = ggrpnr(&mtop->groups, egcENER, a_offset+am+a0);
+ SET_CGINFO_GID(cginfo[cgm+cg], gid);
+
+ /* Check the intra/inter charge group exclusions */
+ if (a1-a0 > excl_nalloc)
+ {
+ excl_nalloc = a1 - a0;
+ srenew(bExcl, excl_nalloc);
+ }
+ /* bExclIntraAll: all intra cg interactions excluded
+ * bExclInter: any inter cg interactions excluded
+ */
+ bExclIntraAll = TRUE;
+ bExclInter = FALSE;
+ bHaveVDW = FALSE;
+ bHaveQ = FALSE;
+ for (ai = a0; ai < a1; ai++)
+ {
+ /* Check VDW and electrostatic interactions */
+ bHaveVDW = bHaveVDW || (type_VDW[molt->atoms.atom[ai].type] ||
+ type_VDW[molt->atoms.atom[ai].typeB]);
+ bHaveQ = bHaveQ || (molt->atoms.atom[ai].q != 0 ||
+ molt->atoms.atom[ai].qB != 0);
+
+ /* Clear the exclusion list for atom ai */
+ for (aj = a0; aj < a1; aj++)
+ {
+ bExcl[aj-a0] = FALSE;
+ }
+ /* Loop over all the exclusions of atom ai */
+ for (j = excl->index[ai]; j < excl->index[ai+1]; j++)
+ {
+ aj = excl->a[j];
+ if (aj < a0 || aj >= a1)
+ {
+ bExclInter = TRUE;
+ }
+ else
+ {
+ bExcl[aj-a0] = TRUE;
+ }
+ }
+ /* Check if ai excludes a0 to a1 */
+ for (aj = a0; aj < a1; aj++)
+ {
+ if (!bExcl[aj-a0])
+ {
+ bExclIntraAll = FALSE;
+ }
+ }
+
+ switch (a_con[ai])
+ {
+ case acCONSTRAINT:
+ SET_CGINFO_CONSTR(cginfo[cgm+cg]);
+ break;
+ case acSETTLE:
+ SET_CGINFO_SETTLE(cginfo[cgm+cg]);
+ break;
+ default:
+ break;
+ }
+ }
+ if (bExclIntraAll)
+ {
+ SET_CGINFO_EXCL_INTRA(cginfo[cgm+cg]);
+ }
+ if (bExclInter)
+ {
+ SET_CGINFO_EXCL_INTER(cginfo[cgm+cg]);
+ }
+ if (a1 - a0 > MAX_CHARGEGROUP_SIZE)
+ {
+ /* The size in cginfo is currently only read with DD */
+ gmx_fatal(FARGS, "A charge group has size %d which is larger than the limit of %d atoms", a1-a0, MAX_CHARGEGROUP_SIZE);
+ }
+ if (bHaveVDW)
+ {
+ SET_CGINFO_HAS_VDW(cginfo[cgm+cg]);
+ }
+ if (bHaveQ)
+ {
+ SET_CGINFO_HAS_Q(cginfo[cgm+cg]);
+ }
+ /* Store the charge group size */
+ SET_CGINFO_NATOMS(cginfo[cgm+cg], a1-a0);
+
+ if (!bExclIntraAll || bExclInter)
+ {
+ *bExcl_IntraCGAll_InterCGNone = FALSE;
+ }
+ }
+ }
+
+ sfree(a_con);
+
+ cg_offset += molb->nmol*cgs->nr;
+ a_offset += molb->nmol*cgs->index[cgs->nr];
+ }
+ sfree(bExcl);
+
+ /* the solvent optimizer is called after the QM is initialized,
+ * because we don't want to have the QM subsystemto become an
+ * optimized solvent
+ */
+
+ check_solvent(fplog, mtop, fr, cginfo_mb);
+
+ if (getenv("GMX_NO_SOLV_OPT"))
+ {
+ if (fplog)
+ {
+ fprintf(fplog, "Found environment variable GMX_NO_SOLV_OPT.\n"
+ "Disabling all solvent optimization\n");
+ }
+ fr->solvent_opt = esolNO;
+ }
+ if (bNoSolvOpt)
+ {
+ fr->solvent_opt = esolNO;
+ }
+ if (!fr->solvent_opt)
+ {
+ for (mb = 0; mb < mtop->nmolblock; mb++)
+ {
+ for (cg = 0; cg < cginfo_mb[mb].cg_mod; cg++)
+ {
+ SET_CGINFO_SOLOPT(cginfo_mb[mb].cginfo[cg], esolNO);
+ }
+ }
+ }
+
+ return cginfo_mb;
+}
+
+static int *cginfo_expand(int nmb, cginfo_mb_t *cgi_mb)
+{
+ int ncg, mb, cg;
+ int *cginfo;
+
+ ncg = cgi_mb[nmb-1].cg_end;
+ snew(cginfo, ncg);
+ mb = 0;
+ for (cg = 0; cg < ncg; cg++)
+ {
+ while (cg >= cgi_mb[mb].cg_end)
+ {
+ mb++;
+ }
+ cginfo[cg] =
+ cgi_mb[mb].cginfo[(cg - cgi_mb[mb].cg_start) % cgi_mb[mb].cg_mod];
+ }
+
+ return cginfo;
+}
+
+static void set_chargesum(FILE *log, t_forcerec *fr, const gmx_mtop_t *mtop)
+{
+ double qsum, q2sum, q;
+ int mb, nmol, i;
+ const t_atoms *atoms;
+
+ qsum = 0;
+ q2sum = 0;
+ for (mb = 0; mb < mtop->nmolblock; mb++)
+ {
+ nmol = mtop->molblock[mb].nmol;
+ atoms = &mtop->moltype[mtop->molblock[mb].type].atoms;
+ for (i = 0; i < atoms->nr; i++)
+ {
+ q = atoms->atom[i].q;
+ qsum += nmol*q;
+ q2sum += nmol*q*q;
+ }
+ }
+ fr->qsum[0] = qsum;
+ fr->q2sum[0] = q2sum;
+ if (fr->efep != efepNO)
+ {
+ qsum = 0;
+ q2sum = 0;
+ for (mb = 0; mb < mtop->nmolblock; mb++)
+ {
+ nmol = mtop->molblock[mb].nmol;
+ atoms = &mtop->moltype[mtop->molblock[mb].type].atoms;
+ for (i = 0; i < atoms->nr; i++)
+ {
+ q = atoms->atom[i].qB;
+ qsum += nmol*q;
+ q2sum += nmol*q*q;
+ }
+ fr->qsum[1] = qsum;
+ fr->q2sum[1] = q2sum;
+ }
+ }
+ else
+ {
+ fr->qsum[1] = fr->qsum[0];
+ fr->q2sum[1] = fr->q2sum[0];
+ }
+ if (log)
+ {
+ if (fr->efep == efepNO)
+ {
+ fprintf(log, "System total charge: %.3f\n", fr->qsum[0]);
+ }
+ else
+ {
+ fprintf(log, "System total charge, top. A: %.3f top. B: %.3f\n",
+ fr->qsum[0], fr->qsum[1]);
+ }
+ }
+}
+
+void update_forcerec(t_forcerec *fr, matrix box)
+{
+ if (fr->eeltype == eelGRF)
+ {
+ calc_rffac(NULL, fr->eeltype, fr->epsilon_r, fr->epsilon_rf,
+ fr->rcoulomb, fr->temp, fr->zsquare, box,
+ &fr->kappa, &fr->k_rf, &fr->c_rf);
+ }
+}
+
+void set_avcsixtwelve(FILE *fplog, t_forcerec *fr, const gmx_mtop_t *mtop)
+{
+ const t_atoms *atoms, *atoms_tpi;
+ const t_blocka *excl;
+ int mb, nmol, nmolc, i, j, tpi, tpj, j1, j2, k, n, nexcl, q;
+#if (defined SIZEOF_LONG_LONG_INT) && (SIZEOF_LONG_LONG_INT >= 8)
+ long long int npair, npair_ij, tmpi, tmpj;
+#else
+ double npair, npair_ij, tmpi, tmpj;
+#endif
+ double csix, ctwelve;
+ int ntp, *typecount;
+ gmx_bool bBHAM;
+ real *nbfp;
+
+ ntp = fr->ntype;
+ bBHAM = fr->bBHAM;
+ nbfp = fr->nbfp;
+
+ for (q = 0; q < (fr->efep == efepNO ? 1 : 2); q++)
+ {
+ csix = 0;
+ ctwelve = 0;
+ npair = 0;
+ nexcl = 0;
+ if (!fr->n_tpi)
+ {
+ /* Count the types so we avoid natoms^2 operations */
+ snew(typecount, ntp);
+ for (mb = 0; mb < mtop->nmolblock; mb++)
+ {
+ nmol = mtop->molblock[mb].nmol;
+ atoms = &mtop->moltype[mtop->molblock[mb].type].atoms;
+ for (i = 0; i < atoms->nr; i++)
+ {
+ if (q == 0)
+ {
+ tpi = atoms->atom[i].type;
+ }
+ else
+ {
+ tpi = atoms->atom[i].typeB;
+ }
+ typecount[tpi] += nmol;
+ }
+ }
+ for (tpi = 0; tpi < ntp; tpi++)
+ {
+ for (tpj = tpi; tpj < ntp; tpj++)
+ {
+ tmpi = typecount[tpi];
+ tmpj = typecount[tpj];
+ if (tpi != tpj)
+ {
+ npair_ij = tmpi*tmpj;
+ }
+ else
+ {
+ npair_ij = tmpi*(tmpi - 1)/2;
+ }
+ if (bBHAM)
+ {
+ /* nbfp now includes the 6.0 derivative prefactor */
+ csix += npair_ij*BHAMC(nbfp, ntp, tpi, tpj)/6.0;
+ }
+ else
+ {
+ /* nbfp now includes the 6.0/12.0 derivative prefactors */
+ csix += npair_ij* C6(nbfp, ntp, tpi, tpj)/6.0;
+ ctwelve += npair_ij* C12(nbfp, ntp, tpi, tpj)/12.0;
+ }
+ npair += npair_ij;
+ }
+ }
+ sfree(typecount);
+ /* Subtract the excluded pairs.
+ * The main reason for substracting exclusions is that in some cases
+ * some combinations might never occur and the parameters could have
+ * any value. These unused values should not influence the dispersion
+ * correction.
+ */
+ for (mb = 0; mb < mtop->nmolblock; mb++)
+ {
+ nmol = mtop->molblock[mb].nmol;
+ atoms = &mtop->moltype[mtop->molblock[mb].type].atoms;
+ excl = &mtop->moltype[mtop->molblock[mb].type].excls;
+ for (i = 0; (i < atoms->nr); i++)
+ {
+ if (q == 0)
+ {
+ tpi = atoms->atom[i].type;
+ }
+ else
+ {
+ tpi = atoms->atom[i].typeB;
+ }
+ j1 = excl->index[i];
+ j2 = excl->index[i+1];
+ for (j = j1; j < j2; j++)
+ {
+ k = excl->a[j];
+ if (k > i)
+ {
+ if (q == 0)
+ {
+ tpj = atoms->atom[k].type;
+ }
+ else
+ {
+ tpj = atoms->atom[k].typeB;
+ }
+ if (bBHAM)
+ {
+ /* nbfp now includes the 6.0 derivative prefactor */
+ csix -= nmol*BHAMC(nbfp, ntp, tpi, tpj)/6.0;
+ }
+ else
+ {
+ /* nbfp now includes the 6.0/12.0 derivative prefactors */
+ csix -= nmol*C6 (nbfp, ntp, tpi, tpj)/6.0;
+ ctwelve -= nmol*C12(nbfp, ntp, tpi, tpj)/12.0;
+ }
+ nexcl += nmol;
+ }
+ }
+ }
+ }
+ }
+ else
+ {
+ /* Only correct for the interaction of the test particle
+ * with the rest of the system.
+ */
+ atoms_tpi =
+ &mtop->moltype[mtop->molblock[mtop->nmolblock-1].type].atoms;
+
+ npair = 0;
+ for (mb = 0; mb < mtop->nmolblock; mb++)
+ {
+ nmol = mtop->molblock[mb].nmol;
+ atoms = &mtop->moltype[mtop->molblock[mb].type].atoms;
+ for (j = 0; j < atoms->nr; j++)
+ {
+ nmolc = nmol;
+ /* Remove the interaction of the test charge group
+ * with itself.
+ */
+ if (mb == mtop->nmolblock-1)
+ {
+ nmolc--;
+
+ if (mb == 0 && nmol == 1)
+ {
+ gmx_fatal(FARGS, "Old format tpr with TPI, please generate a new tpr file");
+ }
+ }
+ if (q == 0)
+ {
+ tpj = atoms->atom[j].type;
+ }
+ else
+ {
+ tpj = atoms->atom[j].typeB;
+ }
+ for (i = 0; i < fr->n_tpi; i++)
+ {
+ if (q == 0)
+ {
+ tpi = atoms_tpi->atom[i].type;
+ }
+ else
+ {
+ tpi = atoms_tpi->atom[i].typeB;
+ }
+ if (bBHAM)
+ {
+ /* nbfp now includes the 6.0 derivative prefactor */
+ csix += nmolc*BHAMC(nbfp, ntp, tpi, tpj)/6.0;
+ }
+ else
+ {
+ /* nbfp now includes the 6.0/12.0 derivative prefactors */
+ csix += nmolc*C6 (nbfp, ntp, tpi, tpj)/6.0;
+ ctwelve += nmolc*C12(nbfp, ntp, tpi, tpj)/12.0;
+ }
+ npair += nmolc;
+ }
+ }
+ }
+ }
+ if (npair - nexcl <= 0 && fplog)
+ {
+ fprintf(fplog, "\nWARNING: There are no atom pairs for dispersion correction\n\n");
+ csix = 0;
+ ctwelve = 0;
+ }
+ else
+ {
+ csix /= npair - nexcl;
+ ctwelve /= npair - nexcl;
+ }
+ if (debug)
+ {
+ fprintf(debug, "Counted %d exclusions\n", nexcl);
+ fprintf(debug, "Average C6 parameter is: %10g\n", (double)csix);
+ fprintf(debug, "Average C12 parameter is: %10g\n", (double)ctwelve);
+ }
+ fr->avcsix[q] = csix;
+ fr->avctwelve[q] = ctwelve;
+ }
+ if (fplog != NULL)
+ {
+ if (fr->eDispCorr == edispcAllEner ||
+ fr->eDispCorr == edispcAllEnerPres)
+ {
+ fprintf(fplog, "Long Range LJ corr.: <C6> %10.4e, <C12> %10.4e\n",
+ fr->avcsix[0], fr->avctwelve[0]);
+ }
+ else
+ {
+ fprintf(fplog, "Long Range LJ corr.: <C6> %10.4e\n", fr->avcsix[0]);
+ }
+ }
+}
+
+
+static void set_bham_b_max(FILE *fplog, t_forcerec *fr,
+ const gmx_mtop_t *mtop)
+{
+ const t_atoms *at1, *at2;
+ int mt1, mt2, i, j, tpi, tpj, ntypes;
+ real b, bmin;
+ real *nbfp;
+
+ if (fplog)
+ {
+ fprintf(fplog, "Determining largest Buckingham b parameter for table\n");
+ }
+ nbfp = fr->nbfp;
+ ntypes = fr->ntype;
+
+ bmin = -1;
+ fr->bham_b_max = 0;
+ for (mt1 = 0; mt1 < mtop->nmoltype; mt1++)
+ {
+ at1 = &mtop->moltype[mt1].atoms;
+ for (i = 0; (i < at1->nr); i++)
+ {
+ tpi = at1->atom[i].type;
+ if (tpi >= ntypes)
+ {
+ gmx_fatal(FARGS, "Atomtype[%d] = %d, maximum = %d", i, tpi, ntypes);
+ }
+
+ for (mt2 = mt1; mt2 < mtop->nmoltype; mt2++)
+ {
+ at2 = &mtop->moltype[mt2].atoms;
+ for (j = 0; (j < at2->nr); j++)
+ {
+ tpj = at2->atom[j].type;
+ if (tpj >= ntypes)
+ {
+ gmx_fatal(FARGS, "Atomtype[%d] = %d, maximum = %d", j, tpj, ntypes);
+ }
+ b = BHAMB(nbfp, ntypes, tpi, tpj);
+ if (b > fr->bham_b_max)
+ {
+ fr->bham_b_max = b;
+ }
+ if ((b < bmin) || (bmin == -1))
+ {
+ bmin = b;
+ }
+ }
+ }
+ }
+ }
+ if (fplog)
+ {
+ fprintf(fplog, "Buckingham b parameters, min: %g, max: %g\n",
+ bmin, fr->bham_b_max);
+ }
+}
+
+static void make_nbf_tables(FILE *fp, const output_env_t oenv,
+ t_forcerec *fr, real rtab,
+ const t_commrec *cr,
+ const char *tabfn, char *eg1, char *eg2,
+ t_nblists *nbl)
+{
+ char buf[STRLEN];
+ int i, j;
+
+ if (tabfn == NULL)
+ {
+ if (debug)
+ {
+ fprintf(debug, "No table file name passed, can not read table, can not do non-bonded interactions\n");
+ }
+ return;
+ }
+
+ sprintf(buf, "%s", tabfn);
+ if (eg1 && eg2)
+ {
+ /* Append the two energy group names */
+ sprintf(buf + strlen(tabfn) - strlen(ftp2ext(efXVG)) - 1, "_%s_%s.%s",
+ eg1, eg2, ftp2ext(efXVG));
+ }
+ nbl->table_elec_vdw = make_tables(fp, oenv, fr, MASTER(cr), buf, rtab, 0);
+ /* Copy the contents of the table to separate coulomb and LJ tables too,
+ * to improve cache performance.
+ */
+ /* For performance reasons we want
+ * the table data to be aligned to 16-byte. The pointers could be freed
+ * but currently aren't.
+ */
+ nbl->table_elec.interaction = GMX_TABLE_INTERACTION_ELEC;
+ nbl->table_elec.format = nbl->table_elec_vdw.format;
+ nbl->table_elec.r = nbl->table_elec_vdw.r;
+ nbl->table_elec.n = nbl->table_elec_vdw.n;
+ nbl->table_elec.scale = nbl->table_elec_vdw.scale;
+ nbl->table_elec.scale_exp = nbl->table_elec_vdw.scale_exp;
+ nbl->table_elec.formatsize = nbl->table_elec_vdw.formatsize;
+ nbl->table_elec.ninteractions = 1;
+ nbl->table_elec.stride = nbl->table_elec.formatsize * nbl->table_elec.ninteractions;
+ snew_aligned(nbl->table_elec.data, nbl->table_elec.stride*(nbl->table_elec.n+1), 32);
+
+ nbl->table_vdw.interaction = GMX_TABLE_INTERACTION_VDWREP_VDWDISP;
+ nbl->table_vdw.format = nbl->table_elec_vdw.format;
+ nbl->table_vdw.r = nbl->table_elec_vdw.r;
+ nbl->table_vdw.n = nbl->table_elec_vdw.n;
+ nbl->table_vdw.scale = nbl->table_elec_vdw.scale;
+ nbl->table_vdw.scale_exp = nbl->table_elec_vdw.scale_exp;
+ nbl->table_vdw.formatsize = nbl->table_elec_vdw.formatsize;
+ nbl->table_vdw.ninteractions = 2;
+ nbl->table_vdw.stride = nbl->table_vdw.formatsize * nbl->table_vdw.ninteractions;
+ snew_aligned(nbl->table_vdw.data, nbl->table_vdw.stride*(nbl->table_vdw.n+1), 32);
+
+ for (i = 0; i <= nbl->table_elec_vdw.n; i++)
+ {
+ for (j = 0; j < 4; j++)
+ {
+ nbl->table_elec.data[4*i+j] = nbl->table_elec_vdw.data[12*i+j];
+ }
+ for (j = 0; j < 8; j++)
+ {
+ nbl->table_vdw.data[8*i+j] = nbl->table_elec_vdw.data[12*i+4+j];
+ }
+ }
+}
+
+static void count_tables(int ftype1, int ftype2, const gmx_mtop_t *mtop,
+ int *ncount, int **count)
+{
+ const gmx_moltype_t *molt;
+ const t_ilist *il;
+ int mt, ftype, stride, i, j, tabnr;
+
+ for (mt = 0; mt < mtop->nmoltype; mt++)
+ {
+ molt = &mtop->moltype[mt];
+ for (ftype = 0; ftype < F_NRE; ftype++)
+ {
+ if (ftype == ftype1 || ftype == ftype2)
+ {
+ il = &molt->ilist[ftype];
+ stride = 1 + NRAL(ftype);
+ for (i = 0; i < il->nr; i += stride)
+ {
+ tabnr = mtop->ffparams.iparams[il->iatoms[i]].tab.table;
+ if (tabnr < 0)
+ {
+ gmx_fatal(FARGS, "A bonded table number is smaller than 0: %d\n", tabnr);
+ }
+ if (tabnr >= *ncount)
+ {
+ srenew(*count, tabnr+1);
+ for (j = *ncount; j < tabnr+1; j++)
+ {
+ (*count)[j] = 0;
+ }
+ *ncount = tabnr+1;
+ }
+ (*count)[tabnr]++;
+ }
+ }
+ }
+ }
+}
+
+static bondedtable_t *make_bonded_tables(FILE *fplog,
+ int ftype1, int ftype2,
+ const gmx_mtop_t *mtop,
+ const char *basefn, const char *tabext)
+{
+ int i, ncount, *count;
+ char tabfn[STRLEN];
+ bondedtable_t *tab;
+
+ tab = NULL;
+
+ ncount = 0;
+ count = NULL;
+ count_tables(ftype1, ftype2, mtop, &ncount, &count);
+
+ if (ncount > 0)
+ {
+ snew(tab, ncount);
+ for (i = 0; i < ncount; i++)
+ {
+ if (count[i] > 0)
+ {
+ sprintf(tabfn, "%s", basefn);
+ sprintf(tabfn + strlen(basefn) - strlen(ftp2ext(efXVG)) - 1, "_%s%d.%s",
+ tabext, i, ftp2ext(efXVG));
+ tab[i] = make_bonded_table(fplog, tabfn, NRAL(ftype1)-2);
+ }
+ }
+ sfree(count);
+ }
+
+ return tab;
+}
+
+void forcerec_set_ranges(t_forcerec *fr,
+ int ncg_home, int ncg_force,
+ int natoms_force,
+ int natoms_force_constr, int natoms_f_novirsum)
+{
+ fr->cg0 = 0;
+ fr->hcg = ncg_home;
+
+ /* fr->ncg_force is unused in the standard code,
+ * but it can be useful for modified code dealing with charge groups.
+ */
+ fr->ncg_force = ncg_force;
+ fr->natoms_force = natoms_force;
+ fr->natoms_force_constr = natoms_force_constr;
+
+ if (fr->natoms_force_constr > fr->nalloc_force)
+ {
+ fr->nalloc_force = over_alloc_dd(fr->natoms_force_constr);
+
+ if (fr->bTwinRange)
+ {
+ srenew(fr->f_twin, fr->nalloc_force);
+ }
+ }
+
+ if (fr->bF_NoVirSum)
+ {
+ fr->f_novirsum_n = natoms_f_novirsum;
+ if (fr->f_novirsum_n > fr->f_novirsum_nalloc)
+ {
+ fr->f_novirsum_nalloc = over_alloc_dd(fr->f_novirsum_n);
+ srenew(fr->f_novirsum_alloc, fr->f_novirsum_nalloc);
+ }
+ }
+ else
+ {
+ fr->f_novirsum_n = 0;
+ }
+}
+
+static real cutoff_inf(real cutoff)
+{
+ if (cutoff == 0)
+ {
+ cutoff = GMX_CUTOFF_INF;
+ }
+
+ return cutoff;
+}
+
+static void make_adress_tf_tables(FILE *fp, const output_env_t oenv,
+ t_forcerec *fr, const t_inputrec *ir,
+ const char *tabfn, const gmx_mtop_t *mtop,
+ matrix box)
+{
+ char buf[STRLEN];
+ int i, j;
+
+ if (tabfn == NULL)
+ {
+ gmx_fatal(FARGS, "No thermoforce table file given. Use -tabletf to specify a file\n");
+ return;
+ }
+
+ snew(fr->atf_tabs, ir->adress->n_tf_grps);
+
+ sprintf(buf, "%s", tabfn);
+ for (i = 0; i < ir->adress->n_tf_grps; i++)
+ {
+ j = ir->adress->tf_table_index[i]; /* get energy group index */
+ sprintf(buf + strlen(tabfn) - strlen(ftp2ext(efXVG)) - 1, "tf_%s.%s",
+ *(mtop->groups.grpname[mtop->groups.grps[egcENER].nm_ind[j]]), ftp2ext(efXVG));
+ if (fp)
+ {
+ fprintf(fp, "loading tf table for energygrp index %d from %s\n", ir->adress->tf_table_index[i], buf);
+ }
+ fr->atf_tabs[i] = make_atf_table(fp, oenv, fr, buf, box);
+ }
+
+}
+
+gmx_bool can_use_allvsall(const t_inputrec *ir, gmx_bool bPrintNote, t_commrec *cr, FILE *fp)
+{
+ gmx_bool bAllvsAll;
+
+ bAllvsAll =
+ (
+ ir->rlist == 0 &&
+ ir->rcoulomb == 0 &&
+ ir->rvdw == 0 &&
+ ir->ePBC == epbcNONE &&
+ ir->vdwtype == evdwCUT &&
+ ir->coulombtype == eelCUT &&
+ ir->efep == efepNO &&
+ (ir->implicit_solvent == eisNO ||
+ (ir->implicit_solvent == eisGBSA && (ir->gb_algorithm == egbSTILL ||
+ ir->gb_algorithm == egbHCT ||
+ ir->gb_algorithm == egbOBC))) &&
+ getenv("GMX_NO_ALLVSALL") == NULL
+ );
+
+ if (bAllvsAll && ir->opts.ngener > 1)
+ {
+ const char *note = "NOTE: Can not use all-vs-all force loops, because there are multiple energy monitor groups; you might get significantly higher performance when using only a single energy monitor group.\n";
+
+ if (bPrintNote)
+ {
+ if (MASTER(cr))
+ {
+ fprintf(stderr, "\n%s\n", note);
+ }
+ if (fp != NULL)
+ {
+ fprintf(fp, "\n%s\n", note);
+ }
+ }
+ bAllvsAll = FALSE;
+ }
+
+ if (bAllvsAll && fp && MASTER(cr))
+ {
+ fprintf(fp, "\nUsing accelerated all-vs-all kernels.\n\n");
+ }
+
+ return bAllvsAll;
+}
+
+
+static void init_forcerec_f_threads(t_forcerec *fr, int nenergrp)
+{
+ int t, i;
+
+ /* These thread local data structures are used for bondeds only */
+ fr->nthreads = gmx_omp_nthreads_get(emntBonded);
+
+ if (fr->nthreads > 1)
+ {
+ snew(fr->f_t, fr->nthreads);
+ /* Thread 0 uses the global force and energy arrays */
+ for (t = 1; t < fr->nthreads; t++)
+ {
+ fr->f_t[t].f = NULL;
+ fr->f_t[t].f_nalloc = 0;
+ snew(fr->f_t[t].fshift, SHIFTS);
+ fr->f_t[t].grpp.nener = nenergrp*nenergrp;
+ for (i = 0; i < egNR; i++)
+ {
+ snew(fr->f_t[t].grpp.ener[i], fr->f_t[t].grpp.nener);
+ }
+ }
+ }
+}
+
+
+static void pick_nbnxn_kernel_cpu(const t_inputrec gmx_unused *ir,
+ int *kernel_type,
+ int *ewald_excl)
+{
+ *kernel_type = nbnxnk4x4_PlainC;
+ *ewald_excl = ewaldexclTable;
+
+#ifdef GMX_NBNXN_SIMD
+ {
+#ifdef GMX_NBNXN_SIMD_4XN
+ *kernel_type = nbnxnk4xN_SIMD_4xN;
+#endif
+#ifdef GMX_NBNXN_SIMD_2XNN
+ /* We expect the 2xNN kernels to be faster in most cases */
+ *kernel_type = nbnxnk4xN_SIMD_2xNN;
+#endif
+
+#if defined GMX_NBNXN_SIMD_4XN && defined GMX_X86_AVX_256
+ if (EEL_RF(ir->coulombtype) || ir->coulombtype == eelCUT)
+ {
+ /* The raw pair rate of the 4x8 kernel is higher than 2x(4+4),
+ * 10% with HT, 50% without HT, but extra zeros interactions
+ * can compensate. As we currently don't detect the actual use
+ * of HT, switch to 4x8 to avoid a potential performance hit.
+ */
+ *kernel_type = nbnxnk4xN_SIMD_4xN;
+ }
+#endif
+ if (getenv("GMX_NBNXN_SIMD_4XN") != NULL)
+ {
+#ifdef GMX_NBNXN_SIMD_4XN
+ *kernel_type = nbnxnk4xN_SIMD_4xN;
+#else
+ gmx_fatal(FARGS, "SIMD 4xN kernels requested, but Gromacs has been compiled without support for these kernels");
+#endif
+ }
+ if (getenv("GMX_NBNXN_SIMD_2XNN") != NULL)
+ {
+#ifdef GMX_NBNXN_SIMD_2XNN
+ *kernel_type = nbnxnk4xN_SIMD_2xNN;
+#else
+ gmx_fatal(FARGS, "SIMD 2x(N+N) kernels requested, but Gromacs has been compiled without support for these kernels");
+#endif
+ }
+
+ /* Analytical Ewald exclusion correction is only an option in
+ * the SIMD kernel. On BlueGene/Q, this is faster regardless
+ * of precision. In single precision, this is faster on
+ * Bulldozer, and slightly faster on Sandy Bridge.
+ */
+#if ((defined GMX_X86_AVX_128_FMA || defined GMX_X86_AVX_256) && !defined GMX_DOUBLE) || (defined GMX_CPU_ACCELERATION_IBM_QPX)
+ *ewald_excl = ewaldexclAnalytical;
+#endif
+ if (getenv("GMX_NBNXN_EWALD_TABLE") != NULL)
+ {
+ *ewald_excl = ewaldexclTable;
+ }
+ if (getenv("GMX_NBNXN_EWALD_ANALYTICAL") != NULL)
+ {
+ *ewald_excl = ewaldexclAnalytical;
+ }
+
+ }
+#endif /* GMX_NBNXN_SIMD */
+}
+
+
+const char *lookup_nbnxn_kernel_name(int kernel_type)
+{
+ const char *returnvalue = NULL;
+ switch (kernel_type)
+ {
+ case nbnxnkNotSet:
+ returnvalue = "not set";
+ break;
+ case nbnxnk4x4_PlainC:
+ returnvalue = "plain C";
+ break;
+ case nbnxnk4xN_SIMD_4xN:
+ case nbnxnk4xN_SIMD_2xNN:
+#ifdef GMX_NBNXN_SIMD
+#ifdef GMX_X86_SSE2
+ /* We have x86 SSE2 compatible SIMD */
+#ifdef GMX_X86_AVX_128_FMA
+ returnvalue = "AVX-128-FMA";
+#else
+#if defined GMX_X86_AVX_256 || defined __AVX__
+ /* x86 SIMD intrinsics can be converted to SSE or AVX depending
+ * on compiler flags. As we use nearly identical intrinsics,
+ * compiling for AVX without an AVX macros effectively results
+ * in AVX kernels.
+ * For gcc we check for __AVX__
+ * At least a check for icc should be added (if there is a macro)
+ */
+#if defined GMX_X86_AVX_256 && !defined GMX_NBNXN_HALF_WIDTH_SIMD
+ returnvalue = "AVX-256";
+#else
+ returnvalue = "AVX-128";
+#endif
+#else
+#ifdef GMX_X86_SSE4_1
+ returnvalue = "SSE4.1";
+#else
+ returnvalue = "SSE2";
+#endif
+#endif
+#endif
+#else /* GMX_X86_SSE2 */
+ /* not GMX_X86_SSE2, but other SIMD */
+ returnvalue = "SIMD";
+#endif /* GMX_X86_SSE2 */
+#else /* GMX_NBNXN_SIMD */
+ returnvalue = "not available";
+#endif /* GMX_NBNXN_SIMD */
+ break;
+ case nbnxnk8x8x8_CUDA: returnvalue = "CUDA"; break;
+ case nbnxnk8x8x8_PlainC: returnvalue = "plain C"; break;
+
+ case nbnxnkNR:
+ default:
+ gmx_fatal(FARGS, "Illegal kernel type selected");
+ returnvalue = NULL;
+ break;
+ }
+ return returnvalue;
+};
+
+static void pick_nbnxn_kernel(FILE *fp,
+ const t_commrec *cr,
+ gmx_bool use_cpu_acceleration,
+ gmx_bool bUseGPU,
+ gmx_bool bEmulateGPU,
+ const t_inputrec *ir,
+ int *kernel_type,
+ int *ewald_excl,
+ gmx_bool bDoNonbonded)
+{
+ assert(kernel_type);
+
+ *kernel_type = nbnxnkNotSet;
+ *ewald_excl = ewaldexclTable;
+
+ if (bEmulateGPU)
+ {
+ *kernel_type = nbnxnk8x8x8_PlainC;
+
+ if (bDoNonbonded)
+ {
+ md_print_warn(cr, fp, "Emulating a GPU run on the CPU (slow)");
+ }
+ }
+ else if (bUseGPU)
+ {
+ *kernel_type = nbnxnk8x8x8_CUDA;
+ }
+
+ if (*kernel_type == nbnxnkNotSet)
+ {
+ if (use_cpu_acceleration)
+ {
+ pick_nbnxn_kernel_cpu(ir, kernel_type, ewald_excl);
+ }
+ else
+ {
+ *kernel_type = nbnxnk4x4_PlainC;
+ }
+ }
+
+ if (bDoNonbonded && fp != NULL)
+ {
+ fprintf(fp, "\nUsing %s %dx%d non-bonded kernels\n\n",
+ lookup_nbnxn_kernel_name(*kernel_type),
+ nbnxn_kernel_pairlist_simple(*kernel_type) ? NBNXN_CPU_CLUSTER_I_SIZE : NBNXN_GPU_CLUSTER_SIZE,
+ nbnxn_kernel_to_cj_size(*kernel_type));
+ }
+}
+
+static void pick_nbnxn_resources(const t_commrec *cr,
+ const gmx_hw_info_t *hwinfo,
+ gmx_bool bDoNonbonded,
+ gmx_bool *bUseGPU,
- (!bDoNonbonded && hwinfo->bCanUseGPU));
++ gmx_bool *bEmulateGPU,
++ const gmx_gpu_opt_t *gpu_opt)
+{
+ gmx_bool bEmulateGPUEnvVarSet;
+ char gpu_err_str[STRLEN];
+
+ *bUseGPU = FALSE;
+
+ bEmulateGPUEnvVarSet = (getenv("GMX_EMULATE_GPU") != NULL);
+
+ /* Run GPU emulation mode if GMX_EMULATE_GPU is defined. Because
+ * GPUs (currently) only handle non-bonded calculations, we will
+ * automatically switch to emulation if non-bonded calculations are
+ * turned off via GMX_NO_NONBONDED - this is the simple and elegant
+ * way to turn off GPU initialization, data movement, and cleanup.
+ *
+ * GPU emulation can be useful to assess the performance one can expect by
+ * adding GPU(s) to the machine. The conditional below allows this even
+ * if mdrun is compiled without GPU acceleration support.
+ * Note that you should freezing the system as otherwise it will explode.
+ */
+ *bEmulateGPU = (bEmulateGPUEnvVarSet ||
- if (hwinfo->bCanUseGPU && !(*bEmulateGPU))
++ (!bDoNonbonded &&
++ gpu_opt->ncuda_dev_use > 0));
+
+ /* Enable GPU mode when GPUs are available or no GPU emulation is requested.
+ */
- if (!init_gpu(cr->rank_pp_intranode, gpu_err_str, &hwinfo->gpu_info))
++ if (gpu_opt->ncuda_dev_use > 0 && !(*bEmulateGPU))
+ {
+ /* Each PP node will use the intra-node id-th device from the
+ * list of detected/selected GPUs. */
- get_gpu_device_id(&hwinfo->gpu_info, cr->rank_pp_intranode),
++ if (!init_gpu(cr->rank_pp_intranode, gpu_err_str,
++ &hwinfo->gpu_info, gpu_opt))
+ {
+ /* At this point the init should never fail as we made sure that
+ * we have all the GPUs we need. If it still does, we'll bail. */
+ gmx_fatal(FARGS, "On node %d failed to initialize GPU #%d: %s",
+ cr->nodeid,
- void init_interaction_const(FILE *fp,
- interaction_const_t **interaction_const,
- const t_forcerec *fr,
- real rtab)
++ get_gpu_device_id(&hwinfo->gpu_info, gpu_opt,
++ cr->rank_pp_intranode),
+ gpu_err_str);
+ }
+
+ /* Here we actually turn on hardware GPU acceleration */
+ *bUseGPU = TRUE;
+ }
+}
+
+gmx_bool uses_simple_tables(int cutoff_scheme,
+ nonbonded_verlet_t *nbv,
+ int group)
+{
+ gmx_bool bUsesSimpleTables = TRUE;
+ int grp_index;
+
+ switch (cutoff_scheme)
+ {
+ case ecutsGROUP:
+ bUsesSimpleTables = TRUE;
+ break;
+ case ecutsVERLET:
+ assert(NULL != nbv && NULL != nbv->grp);
+ grp_index = (group < 0) ? 0 : (nbv->ngrp - 1);
+ bUsesSimpleTables = nbnxn_kernel_pairlist_simple(nbv->grp[grp_index].kernel_type);
+ break;
+ default:
+ gmx_incons("unimplemented");
+ }
+ return bUsesSimpleTables;
+}
+
+static void init_ewald_f_table(interaction_const_t *ic,
+ gmx_bool bUsesSimpleTables,
+ real rtab)
+{
+ real maxr;
+
+ if (bUsesSimpleTables)
+ {
+ /* With a spacing of 0.0005 we are at the force summation accuracy
+ * for the SSE kernels for "normal" atomistic simulations.
+ */
+ ic->tabq_scale = ewald_spline3_table_scale(ic->ewaldcoeff,
+ ic->rcoulomb);
+
+ maxr = (rtab > ic->rcoulomb) ? rtab : ic->rcoulomb;
+ ic->tabq_size = (int)(maxr*ic->tabq_scale) + 2;
+ }
+ else
+ {
+ ic->tabq_size = GPU_EWALD_COULOMB_FORCE_TABLE_SIZE;
+ /* Subtract 2 iso 1 to avoid access out of range due to rounding */
+ ic->tabq_scale = (ic->tabq_size - 2)/ic->rcoulomb;
+ }
+
+ sfree_aligned(ic->tabq_coul_FDV0);
+ sfree_aligned(ic->tabq_coul_F);
+ sfree_aligned(ic->tabq_coul_V);
+
+ /* Create the original table data in FDV0 */
+ snew_aligned(ic->tabq_coul_FDV0, ic->tabq_size*4, 32);
+ snew_aligned(ic->tabq_coul_F, ic->tabq_size, 32);
+ snew_aligned(ic->tabq_coul_V, ic->tabq_size, 32);
+ table_spline3_fill_ewald_lr(ic->tabq_coul_F, ic->tabq_coul_V, ic->tabq_coul_FDV0,
+ ic->tabq_size, 1/ic->tabq_scale, ic->ewaldcoeff);
+}
+
+void init_interaction_const_tables(FILE *fp,
+ interaction_const_t *ic,
+ gmx_bool bUsesSimpleTables,
+ real rtab)
+{
+ real spacing;
+
+ if (ic->eeltype == eelEWALD || EEL_PME(ic->eeltype))
+ {
+ init_ewald_f_table(ic, bUsesSimpleTables, rtab);
+
+ if (fp != NULL)
+ {
+ fprintf(fp, "Initialized non-bonded Ewald correction tables, spacing: %.2e size: %d\n\n",
+ 1/ic->tabq_scale, ic->tabq_size);
+ }
+ }
+}
+
- &bEmulateGPU);
++static void init_interaction_const(FILE *fp,
++ const t_commrec *cr,
++ interaction_const_t **interaction_const,
++ const t_forcerec *fr,
++ real rtab)
+{
+ interaction_const_t *ic;
+ gmx_bool bUsesSimpleTables = TRUE;
+
+ snew(ic, 1);
+
+ /* Just allocate something so we can free it */
+ snew_aligned(ic->tabq_coul_FDV0, 16, 32);
+ snew_aligned(ic->tabq_coul_F, 16, 32);
+ snew_aligned(ic->tabq_coul_V, 16, 32);
+
+ ic->rlist = fr->rlist;
+ ic->rlistlong = fr->rlistlong;
+
+ /* Lennard-Jones */
+ ic->rvdw = fr->rvdw;
+ if (fr->vdw_modifier == eintmodPOTSHIFT)
+ {
+ ic->sh_invrc6 = pow(ic->rvdw, -6.0);
+ }
+ else
+ {
+ ic->sh_invrc6 = 0;
+ }
+
+ /* Electrostatics */
+ ic->eeltype = fr->eeltype;
+ ic->rcoulomb = fr->rcoulomb;
+ ic->epsilon_r = fr->epsilon_r;
+ ic->epsfac = fr->epsfac;
+
+ /* Ewald */
+ ic->ewaldcoeff = fr->ewaldcoeff;
+ if (fr->coulomb_modifier == eintmodPOTSHIFT)
+ {
+ ic->sh_ewald = gmx_erfc(ic->ewaldcoeff*ic->rcoulomb);
+ }
+ else
+ {
+ ic->sh_ewald = 0;
+ }
+
+ /* Reaction-field */
+ if (EEL_RF(ic->eeltype))
+ {
+ ic->epsilon_rf = fr->epsilon_rf;
+ ic->k_rf = fr->k_rf;
+ ic->c_rf = fr->c_rf;
+ }
+ else
+ {
+ /* For plain cut-off we might use the reaction-field kernels */
+ ic->epsilon_rf = ic->epsilon_r;
+ ic->k_rf = 0;
+ if (fr->coulomb_modifier == eintmodPOTSHIFT)
+ {
+ ic->c_rf = 1/ic->rcoulomb;
+ }
+ else
+ {
+ ic->c_rf = 0;
+ }
+ }
+
+ if (fp != NULL)
+ {
+ fprintf(fp, "Potential shift: LJ r^-12: %.3f r^-6 %.3f",
+ sqr(ic->sh_invrc6), ic->sh_invrc6);
+ if (ic->eeltype == eelCUT)
+ {
+ fprintf(fp, ", Coulomb %.3f", ic->c_rf);
+ }
+ else if (EEL_PME(ic->eeltype))
+ {
+ fprintf(fp, ", Ewald %.3e", ic->sh_ewald);
+ }
+ fprintf(fp, "\n");
+ }
+
+ *interaction_const = ic;
+
+ if (fr->nbv != NULL && fr->nbv->bUseGPU)
+ {
+ nbnxn_cuda_init_const(fr->nbv->cu_nbv, ic, fr->nbv->grp);
++
++ /* With tMPI + GPUs some ranks may be sharing GPU(s) and therefore
++ * also sharing texture references. To keep the code simple, we don't
++ * treat texture references as shared resources, but this means that
++ * the coulomb_tab and nbfp texture refs will get updated by multiple threads.
++ * Hence, to ensure that the non-bonded kernels don't start before all
++ * texture binding operations are finished, we need to wait for all ranks
++ * to arrive here before continuing.
++ *
++ * Note that we could omit this barrier if GPUs are not shared (or
++ * texture objects are used), but as this is initialization code, there
++ * is not point in complicating things.
++ */
++#ifdef GMX_THREAD_MPI
++ if (PAR(cr))
++ {
++ gmx_barrier(cr);
++ }
++#endif /* GMX_THREAD_MPI */
+ }
+
+ bUsesSimpleTables = uses_simple_tables(fr->cutoff_scheme, fr->nbv, -1);
+ init_interaction_const_tables(fp, ic, bUsesSimpleTables, rtab);
+}
+
+static void init_nb_verlet(FILE *fp,
+ nonbonded_verlet_t **nb_verlet,
+ const t_inputrec *ir,
+ const t_forcerec *fr,
+ const t_commrec *cr,
+ const char *nbpu_opt)
+{
+ nonbonded_verlet_t *nbv;
+ int i;
+ char *env;
+ gmx_bool bEmulateGPU, bHybridGPURun = FALSE;
+
+ nbnxn_alloc_t *nb_alloc;
+ nbnxn_free_t *nb_free;
+
+ snew(nbv, 1);
+
+ pick_nbnxn_resources(cr, fr->hwinfo,
+ fr->bNonbonded,
+ &nbv->bUseGPU,
- &fr->hwinfo->gpu_info, cr->rank_pp_intranode,
++ &bEmulateGPU,
++ fr->gpu_opt);
+
+ nbv->nbs = NULL;
+
+ nbv->ngrp = (DOMAINDECOMP(cr) ? 2 : 1);
+ for (i = 0; i < nbv->ngrp; i++)
+ {
+ nbv->grp[i].nbl_lists.nnbl = 0;
+ nbv->grp[i].nbat = NULL;
+ nbv->grp[i].kernel_type = nbnxnkNotSet;
+
+ if (i == 0) /* local */
+ {
+ pick_nbnxn_kernel(fp, cr, fr->use_cpu_acceleration,
+ nbv->bUseGPU, bEmulateGPU, ir,
+ &nbv->grp[i].kernel_type,
+ &nbv->grp[i].ewald_excl,
+ fr->bNonbonded);
+ }
+ else /* non-local */
+ {
+ if (nbpu_opt != NULL && strcmp(nbpu_opt, "gpu_cpu") == 0)
+ {
+ /* Use GPU for local, select a CPU kernel for non-local */
+ pick_nbnxn_kernel(fp, cr, fr->use_cpu_acceleration,
+ FALSE, FALSE, ir,
+ &nbv->grp[i].kernel_type,
+ &nbv->grp[i].ewald_excl,
+ fr->bNonbonded);
+
+ bHybridGPURun = TRUE;
+ }
+ else
+ {
+ /* Use the same kernel for local and non-local interactions */
+ nbv->grp[i].kernel_type = nbv->grp[0].kernel_type;
+ nbv->grp[i].ewald_excl = nbv->grp[0].ewald_excl;
+ }
+ }
+ }
+
+ if (nbv->bUseGPU)
+ {
+ /* init the NxN GPU data; the last argument tells whether we'll have
+ * both local and non-local NB calculation on GPU */
+ nbnxn_cuda_init(fp, &nbv->cu_nbv,
- fr->hwinfo = gmx_detect_hardware(fp, cr, FALSE, FALSE, NULL);
++ &fr->hwinfo->gpu_info, fr->gpu_opt,
++ cr->rank_pp_intranode,
+ (nbv->ngrp > 1) && !bHybridGPURun);
+
+ if ((env = getenv("GMX_NB_MIN_CI")) != NULL)
+ {
+ char *end;
+
+ nbv->min_ci_balanced = strtol(env, &end, 10);
+ if (!end || (*end != 0) || nbv->min_ci_balanced <= 0)
+ {
+ gmx_fatal(FARGS, "Invalid value passed in GMX_NB_MIN_CI=%s, positive integer required", env);
+ }
+
+ if (debug)
+ {
+ fprintf(debug, "Neighbor-list balancing parameter: %d (passed as env. var.)\n",
+ nbv->min_ci_balanced);
+ }
+ }
+ else
+ {
+ nbv->min_ci_balanced = nbnxn_cuda_min_ci_balanced(nbv->cu_nbv);
+ if (debug)
+ {
+ fprintf(debug, "Neighbor-list balancing parameter: %d (auto-adjusted to the number of GPU multi-processors)\n",
+ nbv->min_ci_balanced);
+ }
+ }
+ }
+ else
+ {
+ nbv->min_ci_balanced = 0;
+ }
+
+ *nb_verlet = nbv;
+
+ nbnxn_init_search(&nbv->nbs,
+ DOMAINDECOMP(cr) ? &cr->dd->nc : NULL,
+ DOMAINDECOMP(cr) ? domdec_zones(cr->dd) : NULL,
+ gmx_omp_nthreads_get(emntNonbonded));
+
+ for (i = 0; i < nbv->ngrp; i++)
+ {
+ if (nbv->grp[0].kernel_type == nbnxnk8x8x8_CUDA)
+ {
+ nb_alloc = &pmalloc;
+ nb_free = &pfree;
+ }
+ else
+ {
+ nb_alloc = NULL;
+ nb_free = NULL;
+ }
+
+ nbnxn_init_pairlist_set(&nbv->grp[i].nbl_lists,
+ nbnxn_kernel_pairlist_simple(nbv->grp[i].kernel_type),
+ /* 8x8x8 "non-simple" lists are ATM always combined */
+ !nbnxn_kernel_pairlist_simple(nbv->grp[i].kernel_type),
+ nb_alloc, nb_free);
+
+ if (i == 0 ||
+ nbv->grp[0].kernel_type != nbv->grp[i].kernel_type)
+ {
+ snew(nbv->grp[i].nbat, 1);
+ nbnxn_atomdata_init(fp,
+ nbv->grp[i].nbat,
+ nbv->grp[i].kernel_type,
+ fr->ntype, fr->nbfp,
+ ir->opts.ngener,
+ nbnxn_kernel_pairlist_simple(nbv->grp[i].kernel_type) ? gmx_omp_nthreads_get(emntNonbonded) : 1,
+ nb_alloc, nb_free);
+ }
+ else
+ {
+ nbv->grp[i].nbat = nbv->grp[0].nbat;
+ }
+ }
+}
+
+void init_forcerec(FILE *fp,
+ const output_env_t oenv,
+ t_forcerec *fr,
+ t_fcdata *fcd,
+ const t_inputrec *ir,
+ const gmx_mtop_t *mtop,
+ const t_commrec *cr,
+ matrix box,
+ const char *tabfn,
+ const char *tabafn,
+ const char *tabpfn,
+ const char *tabbfn,
+ const char *nbpu_opt,
+ gmx_bool bNoSolvOpt,
+ real print_force)
+{
+ int i, j, m, natoms, ngrp, negp_pp, negptable, egi, egj;
+ real rtab;
+ char *env;
+ double dbl;
+ const t_block *cgs;
+ gmx_bool bGenericKernelOnly;
+ gmx_bool bTab, bSep14tab, bNormalnblists;
+ t_nblists *nbl;
+ int *nm_ind, egp_flags;
+
+ if (fr->hwinfo == NULL)
+ {
+ /* Detect hardware, gather information.
+ * In mdrun, hwinfo has already been set before calling init_forcerec.
+ * Here we ignore GPUs, as tools will not use them anyhow.
+ */
- init_interaction_const(fp, &fr->ic, fr, rtab);
++ fr->hwinfo = gmx_detect_hardware(fp, cr, FALSE);
+ }
+
+ /* By default we turn acceleration on, but it might be turned off further down... */
+ fr->use_cpu_acceleration = TRUE;
+
+ fr->bDomDec = DOMAINDECOMP(cr);
+
+ natoms = mtop->natoms;
+
+ if (check_box(ir->ePBC, box))
+ {
+ gmx_fatal(FARGS, check_box(ir->ePBC, box));
+ }
+
+ /* Test particle insertion ? */
+ if (EI_TPI(ir->eI))
+ {
+ /* Set to the size of the molecule to be inserted (the last one) */
+ /* Because of old style topologies, we have to use the last cg
+ * instead of the last molecule type.
+ */
+ cgs = &mtop->moltype[mtop->molblock[mtop->nmolblock-1].type].cgs;
+ fr->n_tpi = cgs->index[cgs->nr] - cgs->index[cgs->nr-1];
+ if (fr->n_tpi != mtop->mols.index[mtop->mols.nr] - mtop->mols.index[mtop->mols.nr-1])
+ {
+ gmx_fatal(FARGS, "The molecule to insert can not consist of multiple charge groups.\nMake it a single charge group.");
+ }
+ }
+ else
+ {
+ fr->n_tpi = 0;
+ }
+
+ /* Copy AdResS parameters */
+ if (ir->bAdress)
+ {
+ fr->adress_type = ir->adress->type;
+ fr->adress_const_wf = ir->adress->const_wf;
+ fr->adress_ex_width = ir->adress->ex_width;
+ fr->adress_hy_width = ir->adress->hy_width;
+ fr->adress_icor = ir->adress->icor;
+ fr->adress_site = ir->adress->site;
+ fr->adress_ex_forcecap = ir->adress->ex_forcecap;
+ fr->adress_do_hybridpairs = ir->adress->do_hybridpairs;
+
+
+ snew(fr->adress_group_explicit, ir->adress->n_energy_grps);
+ for (i = 0; i < ir->adress->n_energy_grps; i++)
+ {
+ fr->adress_group_explicit[i] = ir->adress->group_explicit[i];
+ }
+
+ fr->n_adress_tf_grps = ir->adress->n_tf_grps;
+ snew(fr->adress_tf_table_index, fr->n_adress_tf_grps);
+ for (i = 0; i < fr->n_adress_tf_grps; i++)
+ {
+ fr->adress_tf_table_index[i] = ir->adress->tf_table_index[i];
+ }
+ copy_rvec(ir->adress->refs, fr->adress_refs);
+ }
+ else
+ {
+ fr->adress_type = eAdressOff;
+ fr->adress_do_hybridpairs = FALSE;
+ }
+
+ /* Copy the user determined parameters */
+ fr->userint1 = ir->userint1;
+ fr->userint2 = ir->userint2;
+ fr->userint3 = ir->userint3;
+ fr->userint4 = ir->userint4;
+ fr->userreal1 = ir->userreal1;
+ fr->userreal2 = ir->userreal2;
+ fr->userreal3 = ir->userreal3;
+ fr->userreal4 = ir->userreal4;
+
+ /* Shell stuff */
+ fr->fc_stepsize = ir->fc_stepsize;
+
+ /* Free energy */
+ fr->efep = ir->efep;
+ fr->sc_alphavdw = ir->fepvals->sc_alpha;
+ if (ir->fepvals->bScCoul)
+ {
+ fr->sc_alphacoul = ir->fepvals->sc_alpha;
+ fr->sc_sigma6_min = pow(ir->fepvals->sc_sigma_min, 6);
+ }
+ else
+ {
+ fr->sc_alphacoul = 0;
+ fr->sc_sigma6_min = 0; /* only needed when bScCoul is on */
+ }
+ fr->sc_power = ir->fepvals->sc_power;
+ fr->sc_r_power = ir->fepvals->sc_r_power;
+ fr->sc_sigma6_def = pow(ir->fepvals->sc_sigma, 6);
+
+ env = getenv("GMX_SCSIGMA_MIN");
+ if (env != NULL)
+ {
+ dbl = 0;
+ sscanf(env, "%lf", &dbl);
+ fr->sc_sigma6_min = pow(dbl, 6);
+ if (fp)
+ {
+ fprintf(fp, "Setting the minimum soft core sigma to %g nm\n", dbl);
+ }
+ }
+
+ fr->bNonbonded = TRUE;
+ if (getenv("GMX_NO_NONBONDED") != NULL)
+ {
+ /* turn off non-bonded calculations */
+ fr->bNonbonded = FALSE;
+ md_print_warn(cr, fp,
+ "Found environment variable GMX_NO_NONBONDED.\n"
+ "Disabling nonbonded calculations.\n");
+ }
+
+ bGenericKernelOnly = FALSE;
+
+ /* We now check in the NS code whether a particular combination of interactions
+ * can be used with water optimization, and disable it if that is not the case.
+ */
+
+ if (getenv("GMX_NB_GENERIC") != NULL)
+ {
+ if (fp != NULL)
+ {
+ fprintf(fp,
+ "Found environment variable GMX_NB_GENERIC.\n"
+ "Disabling all interaction-specific nonbonded kernels, will only\n"
+ "use the slow generic ones in src/gmxlib/nonbonded/nb_generic.c\n\n");
+ }
+ bGenericKernelOnly = TRUE;
+ }
+
+ if (bGenericKernelOnly == TRUE)
+ {
+ bNoSolvOpt = TRUE;
+ }
+
+ if ( (getenv("GMX_DISABLE_CPU_ACCELERATION") != NULL) || (getenv("GMX_NOOPTIMIZEDKERNELS") != NULL) )
+ {
+ fr->use_cpu_acceleration = FALSE;
+ if (fp != NULL)
+ {
+ fprintf(fp,
+ "\nFound environment variable GMX_DISABLE_CPU_ACCELERATION.\n"
+ "Disabling all CPU architecture-specific (e.g. SSE2/SSE4/AVX) routines.\n\n");
+ }
+ }
+
+ fr->bBHAM = (mtop->ffparams.functype[0] == F_BHAM);
+
+ /* Check if we can/should do all-vs-all kernels */
+ fr->bAllvsAll = can_use_allvsall(ir, FALSE, NULL, NULL);
+ fr->AllvsAll_work = NULL;
+ fr->AllvsAll_workgb = NULL;
+
+ /* All-vs-all kernels have not been implemented in 4.6, and
+ * the SIMD group kernels are also buggy in this case. Non-accelerated
+ * group kernels are OK. See Redmine #1249. */
+ if (fr->bAllvsAll)
+ {
+ fr->bAllvsAll = FALSE;
+ fr->use_cpu_acceleration = FALSE;
+ if (fp != NULL)
+ {
+ fprintf(fp,
+ "\nYour simulation settings would have triggered the efficient all-vs-all\n"
+ "kernels in GROMACS 4.5, but these have not been implemented in GROMACS\n"
+ "4.6. Also, we can't use the accelerated SIMD kernels here because\n"
+ "of an unfixed bug. The reference C kernels are correct, though, so\n"
+ "we are proceeding by disabling all CPU architecture-specific\n"
+ "(e.g. SSE2/SSE4/AVX) routines. If performance is important, please\n"
+ "use GROMACS 4.5.7 or try cutoff-scheme = Verlet.\n\n");
+ }
+ }
+
+ /* Neighbour searching stuff */
+ fr->cutoff_scheme = ir->cutoff_scheme;
+ fr->bGrid = (ir->ns_type == ensGRID);
+ fr->ePBC = ir->ePBC;
+
+ /* Determine if we will do PBC for distances in bonded interactions */
+ if (fr->ePBC == epbcNONE)
+ {
+ fr->bMolPBC = FALSE;
+ }
+ else
+ {
+ if (!DOMAINDECOMP(cr))
+ {
+ /* The group cut-off scheme and SHAKE assume charge groups
+ * are whole, but not using molpbc is faster in most cases.
+ */
+ if (fr->cutoff_scheme == ecutsGROUP ||
+ (ir->eConstrAlg == econtSHAKE &&
+ (gmx_mtop_ftype_count(mtop, F_CONSTR) > 0 ||
+ gmx_mtop_ftype_count(mtop, F_CONSTRNC) > 0)))
+ {
+ fr->bMolPBC = ir->bPeriodicMols;
+ }
+ else
+ {
+ fr->bMolPBC = TRUE;
+ if (getenv("GMX_USE_GRAPH") != NULL)
+ {
+ fr->bMolPBC = FALSE;
+ if (fp)
+ {
+ fprintf(fp, "\nGMX_MOLPBC is set, using the graph for bonded interactions\n\n");
+ }
+ }
+ }
+ }
+ else
+ {
+ fr->bMolPBC = dd_bonded_molpbc(cr->dd, fr->ePBC);
+ }
+ }
+ fr->bGB = (ir->implicit_solvent == eisGBSA);
+
+ fr->rc_scaling = ir->refcoord_scaling;
+ copy_rvec(ir->posres_com, fr->posres_com);
+ copy_rvec(ir->posres_comB, fr->posres_comB);
+ fr->rlist = cutoff_inf(ir->rlist);
+ fr->rlistlong = cutoff_inf(ir->rlistlong);
+ fr->eeltype = ir->coulombtype;
+ fr->vdwtype = ir->vdwtype;
+
+ fr->coulomb_modifier = ir->coulomb_modifier;
+ fr->vdw_modifier = ir->vdw_modifier;
+
+ /* Electrostatics: Translate from interaction-setting-in-mdp-file to kernel interaction format */
+ switch (fr->eeltype)
+ {
+ case eelCUT:
+ fr->nbkernel_elec_interaction = (fr->bGB) ? GMX_NBKERNEL_ELEC_GENERALIZEDBORN : GMX_NBKERNEL_ELEC_COULOMB;
+ break;
+
+ case eelRF:
+ case eelGRF:
+ case eelRF_NEC:
+ fr->nbkernel_elec_interaction = GMX_NBKERNEL_ELEC_REACTIONFIELD;
+ break;
+
+ case eelRF_ZERO:
+ fr->nbkernel_elec_interaction = GMX_NBKERNEL_ELEC_REACTIONFIELD;
+ fr->coulomb_modifier = eintmodEXACTCUTOFF;
+ break;
+
+ case eelSWITCH:
+ case eelSHIFT:
+ case eelUSER:
+ case eelENCADSHIFT:
+ case eelPMESWITCH:
+ case eelPMEUSER:
+ case eelPMEUSERSWITCH:
+ fr->nbkernel_elec_interaction = GMX_NBKERNEL_ELEC_CUBICSPLINETABLE;
+ break;
+
+ case eelPME:
+ case eelEWALD:
+ fr->nbkernel_elec_interaction = GMX_NBKERNEL_ELEC_EWALD;
+ break;
+
+ default:
+ gmx_fatal(FARGS, "Unsupported electrostatic interaction: %s", eel_names[fr->eeltype]);
+ break;
+ }
+
+ /* Vdw: Translate from mdp settings to kernel format */
+ switch (fr->vdwtype)
+ {
+ case evdwCUT:
+ if (fr->bBHAM)
+ {
+ fr->nbkernel_vdw_interaction = GMX_NBKERNEL_VDW_BUCKINGHAM;
+ }
+ else
+ {
+ fr->nbkernel_vdw_interaction = GMX_NBKERNEL_VDW_LENNARDJONES;
+ }
+ break;
+
+ case evdwSWITCH:
+ case evdwSHIFT:
+ case evdwUSER:
+ case evdwENCADSHIFT:
+ fr->nbkernel_vdw_interaction = GMX_NBKERNEL_VDW_CUBICSPLINETABLE;
+ break;
+
+ default:
+ gmx_fatal(FARGS, "Unsupported vdw interaction: %s", evdw_names[fr->vdwtype]);
+ break;
+ }
+
+ /* These start out identical to ir, but might be altered if we e.g. tabulate the interaction in the kernel */
+ fr->nbkernel_elec_modifier = fr->coulomb_modifier;
+ fr->nbkernel_vdw_modifier = fr->vdw_modifier;
+
+ fr->bTwinRange = fr->rlistlong > fr->rlist;
+ fr->bEwald = (EEL_PME(fr->eeltype) || fr->eeltype == eelEWALD);
+
+ fr->reppow = mtop->ffparams.reppow;
+
+ if (ir->cutoff_scheme == ecutsGROUP)
+ {
+ fr->bvdwtab = (fr->vdwtype != evdwCUT ||
+ !gmx_within_tol(fr->reppow, 12.0, 10*GMX_DOUBLE_EPS));
+ /* We have special kernels for standard Ewald and PME, but the pme-switch ones are tabulated above */
+ fr->bcoultab = !(fr->eeltype == eelCUT ||
+ fr->eeltype == eelEWALD ||
+ fr->eeltype == eelPME ||
+ fr->eeltype == eelRF ||
+ fr->eeltype == eelRF_ZERO);
+
+ /* If the user absolutely wants different switch/shift settings for coul/vdw, it is likely
+ * going to be faster to tabulate the interaction than calling the generic kernel.
+ */
+ if (fr->nbkernel_elec_modifier == eintmodPOTSWITCH && fr->nbkernel_vdw_modifier == eintmodPOTSWITCH)
+ {
+ if ((fr->rcoulomb_switch != fr->rvdw_switch) || (fr->rcoulomb != fr->rvdw))
+ {
+ fr->bcoultab = TRUE;
+ }
+ }
+ else if ((fr->nbkernel_elec_modifier == eintmodPOTSHIFT && fr->nbkernel_vdw_modifier == eintmodPOTSHIFT) ||
+ ((fr->nbkernel_elec_interaction == GMX_NBKERNEL_ELEC_REACTIONFIELD &&
+ fr->nbkernel_elec_modifier == eintmodEXACTCUTOFF &&
+ (fr->nbkernel_vdw_modifier == eintmodPOTSWITCH || fr->nbkernel_vdw_modifier == eintmodPOTSHIFT))))
+ {
+ if (fr->rcoulomb != fr->rvdw)
+ {
+ fr->bcoultab = TRUE;
+ }
+ }
+
+ if (getenv("GMX_REQUIRE_TABLES"))
+ {
+ fr->bvdwtab = TRUE;
+ fr->bcoultab = TRUE;
+ }
+
+ if (fp)
+ {
+ fprintf(fp, "Table routines are used for coulomb: %s\n", bool_names[fr->bcoultab]);
+ fprintf(fp, "Table routines are used for vdw: %s\n", bool_names[fr->bvdwtab ]);
+ }
+
+ if (fr->bvdwtab == TRUE)
+ {
+ fr->nbkernel_vdw_interaction = GMX_NBKERNEL_VDW_CUBICSPLINETABLE;
+ fr->nbkernel_vdw_modifier = eintmodNONE;
+ }
+ if (fr->bcoultab == TRUE)
+ {
+ fr->nbkernel_elec_interaction = GMX_NBKERNEL_ELEC_CUBICSPLINETABLE;
+ fr->nbkernel_elec_modifier = eintmodNONE;
+ }
+ }
+
+ if (ir->cutoff_scheme == ecutsVERLET)
+ {
+ if (!gmx_within_tol(fr->reppow, 12.0, 10*GMX_DOUBLE_EPS))
+ {
+ gmx_fatal(FARGS, "Cut-off scheme %S only supports LJ repulsion power 12", ecutscheme_names[ir->cutoff_scheme]);
+ }
+ fr->bvdwtab = FALSE;
+ fr->bcoultab = FALSE;
+ }
+
+ /* Tables are used for direct ewald sum */
+ if (fr->bEwald)
+ {
+ if (EEL_PME(ir->coulombtype))
+ {
+ if (fp)
+ {
+ fprintf(fp, "Will do PME sum in reciprocal space.\n");
+ }
+ if (ir->coulombtype == eelP3M_AD)
+ {
+ please_cite(fp, "Hockney1988");
+ please_cite(fp, "Ballenegger2012");
+ }
+ else
+ {
+ please_cite(fp, "Essmann95a");
+ }
+
+ if (ir->ewald_geometry == eewg3DC)
+ {
+ if (fp)
+ {
+ fprintf(fp, "Using the Ewald3DC correction for systems with a slab geometry.\n");
+ }
+ please_cite(fp, "In-Chul99a");
+ }
+ }
+ fr->ewaldcoeff = calc_ewaldcoeff(ir->rcoulomb, ir->ewald_rtol);
+ init_ewald_tab(&(fr->ewald_table), ir, fp);
+ if (fp)
+ {
+ fprintf(fp, "Using a Gaussian width (1/beta) of %g nm for Ewald\n",
+ 1/fr->ewaldcoeff);
+ }
+ }
+
+ /* Electrostatics */
+ fr->epsilon_r = ir->epsilon_r;
+ fr->epsilon_rf = ir->epsilon_rf;
+ fr->fudgeQQ = mtop->ffparams.fudgeQQ;
+ fr->rcoulomb_switch = ir->rcoulomb_switch;
+ fr->rcoulomb = cutoff_inf(ir->rcoulomb);
+
+ /* Parameters for generalized RF */
+ fr->zsquare = 0.0;
+ fr->temp = 0.0;
+
+ if (fr->eeltype == eelGRF)
+ {
+ init_generalized_rf(fp, mtop, ir, fr);
+ }
+
+ fr->bF_NoVirSum = (EEL_FULL(fr->eeltype) ||
+ gmx_mtop_ftype_count(mtop, F_POSRES) > 0 ||
+ gmx_mtop_ftype_count(mtop, F_FBPOSRES) > 0 ||
+ IR_ELEC_FIELD(*ir) ||
+ (fr->adress_icor != eAdressICOff)
+ );
+
+ if (fr->cutoff_scheme == ecutsGROUP &&
+ ncg_mtop(mtop) > fr->cg_nalloc && !DOMAINDECOMP(cr))
+ {
+ /* Count the total number of charge groups */
+ fr->cg_nalloc = ncg_mtop(mtop);
+ srenew(fr->cg_cm, fr->cg_nalloc);
+ }
+ if (fr->shift_vec == NULL)
+ {
+ snew(fr->shift_vec, SHIFTS);
+ }
+
+ if (fr->fshift == NULL)
+ {
+ snew(fr->fshift, SHIFTS);
+ }
+
+ if (fr->nbfp == NULL)
+ {
+ fr->ntype = mtop->ffparams.atnr;
+ fr->nbfp = mk_nbfp(&mtop->ffparams, fr->bBHAM);
+ }
+
+ /* Copy the energy group exclusions */
+ fr->egp_flags = ir->opts.egp_flags;
+
+ /* Van der Waals stuff */
+ fr->rvdw = cutoff_inf(ir->rvdw);
+ fr->rvdw_switch = ir->rvdw_switch;
+ if ((fr->vdwtype != evdwCUT) && (fr->vdwtype != evdwUSER) && !fr->bBHAM)
+ {
+ if (fr->rvdw_switch >= fr->rvdw)
+ {
+ gmx_fatal(FARGS, "rvdw_switch (%f) must be < rvdw (%f)",
+ fr->rvdw_switch, fr->rvdw);
+ }
+ if (fp)
+ {
+ fprintf(fp, "Using %s Lennard-Jones, switch between %g and %g nm\n",
+ (fr->eeltype == eelSWITCH) ? "switched" : "shifted",
+ fr->rvdw_switch, fr->rvdw);
+ }
+ }
+
+ if (fr->bBHAM && (fr->vdwtype == evdwSHIFT || fr->vdwtype == evdwSWITCH))
+ {
+ gmx_fatal(FARGS, "Switch/shift interaction not supported with Buckingham");
+ }
+
+ if (fp)
+ {
+ fprintf(fp, "Cut-off's: NS: %g Coulomb: %g %s: %g\n",
+ fr->rlist, fr->rcoulomb, fr->bBHAM ? "BHAM" : "LJ", fr->rvdw);
+ }
+
+ fr->eDispCorr = ir->eDispCorr;
+ if (ir->eDispCorr != edispcNO)
+ {
+ set_avcsixtwelve(fp, fr, mtop);
+ }
+
+ if (fr->bBHAM)
+ {
+ set_bham_b_max(fp, fr, mtop);
+ }
+
+ fr->gb_epsilon_solvent = ir->gb_epsilon_solvent;
+
+ /* Copy the GBSA data (radius, volume and surftens for each
+ * atomtype) from the topology atomtype section to forcerec.
+ */
+ snew(fr->atype_radius, fr->ntype);
+ snew(fr->atype_vol, fr->ntype);
+ snew(fr->atype_surftens, fr->ntype);
+ snew(fr->atype_gb_radius, fr->ntype);
+ snew(fr->atype_S_hct, fr->ntype);
+
+ if (mtop->atomtypes.nr > 0)
+ {
+ for (i = 0; i < fr->ntype; i++)
+ {
+ fr->atype_radius[i] = mtop->atomtypes.radius[i];
+ }
+ for (i = 0; i < fr->ntype; i++)
+ {
+ fr->atype_vol[i] = mtop->atomtypes.vol[i];
+ }
+ for (i = 0; i < fr->ntype; i++)
+ {
+ fr->atype_surftens[i] = mtop->atomtypes.surftens[i];
+ }
+ for (i = 0; i < fr->ntype; i++)
+ {
+ fr->atype_gb_radius[i] = mtop->atomtypes.gb_radius[i];
+ }
+ for (i = 0; i < fr->ntype; i++)
+ {
+ fr->atype_S_hct[i] = mtop->atomtypes.S_hct[i];
+ }
+ }
+
+ /* Generate the GB table if needed */
+ if (fr->bGB)
+ {
+#ifdef GMX_DOUBLE
+ fr->gbtabscale = 2000;
+#else
+ fr->gbtabscale = 500;
+#endif
+
+ fr->gbtabr = 100;
+ fr->gbtab = make_gb_table(oenv, fr);
+
+ init_gb(&fr->born, cr, fr, ir, mtop, ir->gb_algorithm);
+
+ /* Copy local gb data (for dd, this is done in dd_partition_system) */
+ if (!DOMAINDECOMP(cr))
+ {
+ make_local_gb(cr, fr->born, ir->gb_algorithm);
+ }
+ }
+
+ /* Set the charge scaling */
+ if (fr->epsilon_r != 0)
+ {
+ fr->epsfac = ONE_4PI_EPS0/fr->epsilon_r;
+ }
+ else
+ {
+ /* eps = 0 is infinite dieletric: no coulomb interactions */
+ fr->epsfac = 0;
+ }
+
+ /* Reaction field constants */
+ if (EEL_RF(fr->eeltype))
+ {
+ calc_rffac(fp, fr->eeltype, fr->epsilon_r, fr->epsilon_rf,
+ fr->rcoulomb, fr->temp, fr->zsquare, box,
+ &fr->kappa, &fr->k_rf, &fr->c_rf);
+ }
+
+ set_chargesum(fp, fr, mtop);
+
+ /* if we are using LR electrostatics, and they are tabulated,
+ * the tables will contain modified coulomb interactions.
+ * Since we want to use the non-shifted ones for 1-4
+ * coulombic interactions, we must have an extra set of tables.
+ */
+
+ /* Construct tables.
+ * A little unnecessary to make both vdw and coul tables sometimes,
+ * but what the heck... */
+
+ bTab = fr->bcoultab || fr->bvdwtab || fr->bEwald;
+
+ bSep14tab = ((!bTab || fr->eeltype != eelCUT || fr->vdwtype != evdwCUT ||
+ fr->bBHAM || fr->bEwald) &&
+ (gmx_mtop_ftype_count(mtop, F_LJ14) > 0 ||
+ gmx_mtop_ftype_count(mtop, F_LJC14_Q) > 0 ||
+ gmx_mtop_ftype_count(mtop, F_LJC_PAIRS_NB) > 0));
+
+ negp_pp = ir->opts.ngener - ir->nwall;
+ negptable = 0;
+ if (!bTab)
+ {
+ bNormalnblists = TRUE;
+ fr->nnblists = 1;
+ }
+ else
+ {
+ bNormalnblists = (ir->eDispCorr != edispcNO);
+ for (egi = 0; egi < negp_pp; egi++)
+ {
+ for (egj = egi; egj < negp_pp; egj++)
+ {
+ egp_flags = ir->opts.egp_flags[GID(egi, egj, ir->opts.ngener)];
+ if (!(egp_flags & EGP_EXCL))
+ {
+ if (egp_flags & EGP_TABLE)
+ {
+ negptable++;
+ }
+ else
+ {
+ bNormalnblists = TRUE;
+ }
+ }
+ }
+ }
+ if (bNormalnblists)
+ {
+ fr->nnblists = negptable + 1;
+ }
+ else
+ {
+ fr->nnblists = negptable;
+ }
+ if (fr->nnblists > 1)
+ {
+ snew(fr->gid2nblists, ir->opts.ngener*ir->opts.ngener);
+ }
+ }
+
+ if (ir->adress)
+ {
+ fr->nnblists *= 2;
+ }
+
+ snew(fr->nblists, fr->nnblists);
+
+ /* This code automatically gives table length tabext without cut-off's,
+ * in that case grompp should already have checked that we do not need
+ * normal tables and we only generate tables for 1-4 interactions.
+ */
+ rtab = ir->rlistlong + ir->tabext;
+
+ if (bTab)
+ {
+ /* make tables for ordinary interactions */
+ if (bNormalnblists)
+ {
+ make_nbf_tables(fp, oenv, fr, rtab, cr, tabfn, NULL, NULL, &fr->nblists[0]);
+ if (ir->adress)
+ {
+ make_nbf_tables(fp, oenv, fr, rtab, cr, tabfn, NULL, NULL, &fr->nblists[fr->nnblists/2]);
+ }
+ if (!bSep14tab)
+ {
+ fr->tab14 = fr->nblists[0].table_elec_vdw;
+ }
+ m = 1;
+ }
+ else
+ {
+ m = 0;
+ }
+ if (negptable > 0)
+ {
+ /* Read the special tables for certain energy group pairs */
+ nm_ind = mtop->groups.grps[egcENER].nm_ind;
+ for (egi = 0; egi < negp_pp; egi++)
+ {
+ for (egj = egi; egj < negp_pp; egj++)
+ {
+ egp_flags = ir->opts.egp_flags[GID(egi, egj, ir->opts.ngener)];
+ if ((egp_flags & EGP_TABLE) && !(egp_flags & EGP_EXCL))
+ {
+ nbl = &(fr->nblists[m]);
+ if (fr->nnblists > 1)
+ {
+ fr->gid2nblists[GID(egi, egj, ir->opts.ngener)] = m;
+ }
+ /* Read the table file with the two energy groups names appended */
+ make_nbf_tables(fp, oenv, fr, rtab, cr, tabfn,
+ *mtop->groups.grpname[nm_ind[egi]],
+ *mtop->groups.grpname[nm_ind[egj]],
+ &fr->nblists[m]);
+ if (ir->adress)
+ {
+ make_nbf_tables(fp, oenv, fr, rtab, cr, tabfn,
+ *mtop->groups.grpname[nm_ind[egi]],
+ *mtop->groups.grpname[nm_ind[egj]],
+ &fr->nblists[fr->nnblists/2+m]);
+ }
+ m++;
+ }
+ else if (fr->nnblists > 1)
+ {
+ fr->gid2nblists[GID(egi, egj, ir->opts.ngener)] = 0;
+ }
+ }
+ }
+ }
+ }
+ if (bSep14tab)
+ {
+ /* generate extra tables with plain Coulomb for 1-4 interactions only */
+ fr->tab14 = make_tables(fp, oenv, fr, MASTER(cr), tabpfn, rtab,
+ GMX_MAKETABLES_14ONLY);
+ }
+
+ /* Read AdResS Thermo Force table if needed */
+ if (fr->adress_icor == eAdressICThermoForce)
+ {
+ /* old todo replace */
+
+ if (ir->adress->n_tf_grps > 0)
+ {
+ make_adress_tf_tables(fp, oenv, fr, ir, tabfn, mtop, box);
+
+ }
+ else
+ {
+ /* load the default table */
+ snew(fr->atf_tabs, 1);
+ fr->atf_tabs[DEFAULT_TF_TABLE] = make_atf_table(fp, oenv, fr, tabafn, box);
+ }
+ }
+
+ /* Wall stuff */
+ fr->nwall = ir->nwall;
+ if (ir->nwall && ir->wall_type == ewtTABLE)
+ {
+ make_wall_tables(fp, oenv, ir, tabfn, &mtop->groups, fr);
+ }
+
+ if (fcd && tabbfn)
+ {
+ fcd->bondtab = make_bonded_tables(fp,
+ F_TABBONDS, F_TABBONDSNC,
+ mtop, tabbfn, "b");
+ fcd->angletab = make_bonded_tables(fp,
+ F_TABANGLES, -1,
+ mtop, tabbfn, "a");
+ fcd->dihtab = make_bonded_tables(fp,
+ F_TABDIHS, -1,
+ mtop, tabbfn, "d");
+ }
+ else
+ {
+ if (debug)
+ {
+ fprintf(debug, "No fcdata or table file name passed, can not read table, can not do bonded interactions\n");
+ }
+ }
+
+ /* QM/MM initialization if requested
+ */
+ if (ir->bQMMM)
+ {
+ fprintf(stderr, "QM/MM calculation requested.\n");
+ }
+
+ fr->bQMMM = ir->bQMMM;
+ fr->qr = mk_QMMMrec();
+
+ /* Set all the static charge group info */
+ fr->cginfo_mb = init_cginfo_mb(fp, mtop, fr, bNoSolvOpt,
+ &fr->bExcl_IntraCGAll_InterCGNone);
+ if (DOMAINDECOMP(cr))
+ {
+ fr->cginfo = NULL;
+ }
+ else
+ {
+ fr->cginfo = cginfo_expand(mtop->nmolblock, fr->cginfo_mb);
+ }
+
+ if (!DOMAINDECOMP(cr))
+ {
+ /* When using particle decomposition, the effect of the second argument,
+ * which sets fr->hcg, is corrected later in do_md and init_em.
+ */
+ forcerec_set_ranges(fr, ncg_mtop(mtop), ncg_mtop(mtop),
+ mtop->natoms, mtop->natoms, mtop->natoms);
+ }
+
+ fr->print_force = print_force;
+
+
+ /* coarse load balancing vars */
+ fr->t_fnbf = 0.;
+ fr->t_wait = 0.;
+ fr->timesteps = 0;
+
+ /* Initialize neighbor search */
+ init_ns(fp, cr, &fr->ns, fr, mtop);
+
+ if (cr->duty & DUTY_PP)
+ {
+ gmx_nonbonded_setup(fr, bGenericKernelOnly);
+ /*
+ if (ir->bAdress)
+ {
+ gmx_setup_adress_kernels(fp,bGenericKernelOnly);
+ }
+ */
+ }
+
+ /* Initialize the thread working data for bonded interactions */
+ init_forcerec_f_threads(fr, mtop->groups.grps[egcENER].nr);
+
+ snew(fr->excl_load, fr->nthreads+1);
+
+ if (fr->cutoff_scheme == ecutsVERLET)
+ {
+ if (ir->rcoulomb != ir->rvdw)
+ {
+ gmx_fatal(FARGS, "With Verlet lists rcoulomb and rvdw should be identical");
+ }
+
+ init_nb_verlet(fp, &fr->nbv, ir, fr, cr, nbpu_opt);
+ }
+
+ /* fr->ic is used both by verlet and group kernels (to some extent) now */
++ init_interaction_const(fp, cr, &fr->ic, fr, rtab);
++
+ if (ir->eDispCorr != edispcNO)
+ {
+ calc_enervirdiff(fp, ir->eDispCorr, fr);
+ }
+}
+
+#define pr_real(fp, r) fprintf(fp, "%s: %e\n",#r, r)
+#define pr_int(fp, i) fprintf((fp), "%s: %d\n",#i, i)
+#define pr_bool(fp, b) fprintf((fp), "%s: %s\n",#b, bool_names[b])
+
+void pr_forcerec(FILE *fp, t_forcerec *fr)
+{
+ int i;
+
+ pr_real(fp, fr->rlist);
+ pr_real(fp, fr->rcoulomb);
+ pr_real(fp, fr->fudgeQQ);
+ pr_bool(fp, fr->bGrid);
+ pr_bool(fp, fr->bTwinRange);
+ /*pr_int(fp,fr->cg0);
+ pr_int(fp,fr->hcg);*/
+ for (i = 0; i < fr->nnblists; i++)
+ {
+ pr_int(fp, fr->nblists[i].table_elec_vdw.n);
+ }
+ pr_real(fp, fr->rcoulomb_switch);
+ pr_real(fp, fr->rcoulomb);
+
+ fflush(fp);
+}
+
+void forcerec_set_excl_load(t_forcerec *fr,
+ const gmx_localtop_t *top, const t_commrec *cr)
+{
+ const int *ind, *a;
+ int t, i, j, ntot, n, ntarget;
+
+ if (cr != NULL && PARTDECOMP(cr))
+ {
+ /* No OpenMP with particle decomposition */
+ pd_at_range(cr,
+ &fr->excl_load[0],
+ &fr->excl_load[1]);
+
+ return;
+ }
+
+ ind = top->excls.index;
+ a = top->excls.a;
+
+ ntot = 0;
+ for (i = 0; i < top->excls.nr; i++)
+ {
+ for (j = ind[i]; j < ind[i+1]; j++)
+ {
+ if (a[j] > i)
+ {
+ ntot++;
+ }
+ }
+ }
+
+ fr->excl_load[0] = 0;
+ n = 0;
+ i = 0;
+ for (t = 1; t <= fr->nthreads; t++)
+ {
+ ntarget = (ntot*t)/fr->nthreads;
+ while (i < top->excls.nr && n < ntarget)
+ {
+ for (j = ind[i]; j < ind[i+1]; j++)
+ {
+ if (a[j] > i)
+ {
+ n++;
+ }
+ }
+ i++;
+ }
+ fr->excl_load[t] = i;
+ }
+}
--- /dev/null
- const gmx_gpu_info_t *gpu_info, int my_gpu_index,
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2012, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * Gallium Rubidium Oxygen Manganese Argon Carbon Silicon
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <assert.h>
+
+#include <cuda.h>
+
+#include "gmx_fatal.h"
+#include "smalloc.h"
+#include "tables.h"
+#include "typedefs.h"
+#include "types/nb_verlet.h"
+#include "types/interaction_const.h"
+#include "types/force_flags.h"
+#include "../nbnxn_consts.h"
+#include "gmx_detect_hardware.h"
+
+#include "nbnxn_cuda_types.h"
+#include "../../gmxlib/cuda_tools/cudautils.cuh"
+#include "nbnxn_cuda_data_mgmt.h"
+#include "pmalloc_cuda.h"
+#include "gpu_utils.h"
+
+static bool bUseCudaEventBlockingSync = false; /* makes the CPU thread block */
+
+/* This is a heuristically determined parameter for the Fermi architecture for
+ * the minimum size of ci lists by multiplying this constant with the # of
+ * multiprocessors on the current device.
+ */
+static unsigned int gpu_min_ci_balanced_factor = 40;
+
+/* Functions from nbnxn_cuda.cu */
+extern void nbnxn_cuda_set_cacheconfig(cuda_dev_info_t *devinfo);
+extern const struct texture<float, 1, cudaReadModeElementType>& nbnxn_cuda_get_nbfp_texref();
+extern const struct texture<float, 1, cudaReadModeElementType>& nbnxn_cuda_get_coulomb_tab_texref();
+
+/* We should actually be using md_print_warn in md_logging.c,
+ * but we can't include mpi.h in CUDA code.
+ */
+static void md_print_warn(FILE *fplog,
+ const char *fmt, ...)
+{
+ va_list ap;
+
+ if (fplog != NULL)
+ {
+ /* We should only print to stderr on the master node,
+ * in most cases fplog is only set on the master node, so this works.
+ */
+ va_start(ap, fmt);
+ fprintf(stderr, "\n");
+ vfprintf(stderr, fmt, ap);
+ fprintf(stderr, "\n");
+ va_end(ap);
+
+ va_start(ap, fmt);
+ fprintf(fplog, "\n");
+ vfprintf(fplog, fmt, ap);
+ fprintf(fplog, "\n");
+ va_end(ap);
+ }
+}
+
+
+/* Fw. decl. */
+static void nbnxn_cuda_clear_e_fshift(nbnxn_cuda_ptr_t cu_nb);
+
+
+/*! Tabulates the Ewald Coulomb force and initializes the size/scale
+ and the table GPU array. If called with an already allocated table,
+ it just re-uploads the table.
+ */
+static void init_ewald_coulomb_force_table(cu_nbparam_t *nbp,
+ const cuda_dev_info_t *dev_info)
+{
+ float *ftmp, *coul_tab;
+ int tabsize;
+ double tabscale;
+ cudaError_t stat;
+
+ tabsize = GPU_EWALD_COULOMB_FORCE_TABLE_SIZE;
+ /* Subtract 2 iso 1 to avoid access out of range due to rounding */
+ tabscale = (tabsize - 2) / sqrt(nbp->rcoulomb_sq);
+
+ pmalloc((void**)&ftmp, tabsize*sizeof(*ftmp));
+
+ table_spline3_fill_ewald_lr(ftmp, NULL, NULL, tabsize,
+ 1/tabscale, nbp->ewald_beta);
+
+ /* If the table pointer == NULL the table is generated the first time =>
+ the array pointer will be saved to nbparam and the texture is bound.
+ */
+ coul_tab = nbp->coulomb_tab;
+ if (coul_tab == NULL)
+ {
+ stat = cudaMalloc((void **)&coul_tab, tabsize*sizeof(*coul_tab));
+ CU_RET_ERR(stat, "cudaMalloc failed on coul_tab");
+
+ nbp->coulomb_tab = coul_tab;
+
+#ifdef TEXOBJ_SUPPORTED
+ /* Only device CC >= 3.0 (Kepler and later) support texture objects */
+ if (dev_info->prop.major >= 3)
+ {
+ cudaResourceDesc rd;
+ memset(&rd, 0, sizeof(rd));
+ rd.resType = cudaResourceTypeLinear;
+ rd.res.linear.devPtr = nbp->coulomb_tab;
+ rd.res.linear.desc.f = cudaChannelFormatKindFloat;
+ rd.res.linear.desc.x = 32;
+ rd.res.linear.sizeInBytes = tabsize*sizeof(*coul_tab);
+
+ cudaTextureDesc td;
+ memset(&td, 0, sizeof(td));
+ td.readMode = cudaReadModeElementType;
+ stat = cudaCreateTextureObject(&nbp->coulomb_tab_texobj, &rd, &td, NULL);
+ CU_RET_ERR(stat, "cudaCreateTextureObject on coulomb_tab_texobj failed");
+ }
+ else
+#endif
+ {
+ cudaChannelFormatDesc cd = cudaCreateChannelDesc<float>();
+ stat = cudaBindTexture(NULL, &nbnxn_cuda_get_coulomb_tab_texref(),
+ coul_tab, &cd, tabsize*sizeof(*coul_tab));
+ CU_RET_ERR(stat, "cudaBindTexture on coulomb_tab_texref failed");
+ }
+ }
+
+ cu_copy_H2D(coul_tab, ftmp, tabsize*sizeof(*coul_tab));
+
+ nbp->coulomb_tab_size = tabsize;
+ nbp->coulomb_tab_scale = tabscale;
+
+ pfree(ftmp);
+}
+
+
+/*! Initializes the atomdata structure first time, it only gets filled at
+ pair-search. */
+static void init_atomdata_first(cu_atomdata_t *ad, int ntypes)
+{
+ cudaError_t stat;
+
+ ad->ntypes = ntypes;
+ stat = cudaMalloc((void**)&ad->shift_vec, SHIFTS*sizeof(*ad->shift_vec));
+ CU_RET_ERR(stat, "cudaMalloc failed on ad->shift_vec");
+ ad->bShiftVecUploaded = false;
+
+ stat = cudaMalloc((void**)&ad->fshift, SHIFTS*sizeof(*ad->fshift));
+ CU_RET_ERR(stat, "cudaMalloc failed on ad->fshift");
+
+ stat = cudaMalloc((void**)&ad->e_lj, sizeof(*ad->e_lj));
+ CU_RET_ERR(stat, "cudaMalloc failed on ad->e_lj");
+ stat = cudaMalloc((void**)&ad->e_el, sizeof(*ad->e_el));
+ CU_RET_ERR(stat, "cudaMalloc failed on ad->e_el");
+
+ /* initialize to NULL poiters to data that is not allocated here and will
+ need reallocation in nbnxn_cuda_init_atomdata */
+ ad->xq = NULL;
+ ad->f = NULL;
+
+ /* size -1 indicates that the respective array hasn't been initialized yet */
+ ad->natoms = -1;
+ ad->nalloc = -1;
+}
+
+/*! Selects the Ewald kernel type, analytical on SM 3.0 and later, tabulated on
+ earlier GPUs, single or twin cut-off. */
+static int pick_ewald_kernel_type(bool bTwinCut,
+ const cuda_dev_info_t *dev_info)
+{
+ bool bUseAnalyticalEwald, bForceAnalyticalEwald, bForceTabulatedEwald;
+ int kernel_type;
+
+ /* Benchmarking/development environment variables to force the use of
+ analytical or tabulated Ewald kernel. */
+ bForceAnalyticalEwald = (getenv("GMX_CUDA_NB_ANA_EWALD") != NULL);
+ bForceTabulatedEwald = (getenv("GMX_CUDA_NB_TAB_EWALD") != NULL);
+
+ if (bForceAnalyticalEwald && bForceTabulatedEwald)
+ {
+ gmx_incons("Both analytical and tabulated Ewald CUDA non-bonded kernels "
+ "requested through environment variables.");
+ }
+
+ /* By default, on SM 3.0 and later use analytical Ewald, on earlier tabulated. */
+ if ((dev_info->prop.major >= 3 || bForceAnalyticalEwald) && !bForceTabulatedEwald)
+ {
+ bUseAnalyticalEwald = true;
+
+ if (debug)
+ {
+ fprintf(debug, "Using analytical Ewald CUDA kernels\n");
+ }
+ }
+ else
+ {
+ bUseAnalyticalEwald = false;
+
+ if (debug)
+ {
+ fprintf(debug, "Using tabulated Ewald CUDA kernels\n");
+ }
+ }
+
+ /* Use twin cut-off kernels if requested by bTwinCut or the env. var.
+ forces it (use it for debugging/benchmarking only). */
+ if (!bTwinCut && (getenv("GMX_CUDA_NB_EWALD_TWINCUT") == NULL))
+ {
+ kernel_type = bUseAnalyticalEwald ? eelCuEWALD_ANA : eelCuEWALD_TAB;
+ }
+ else
+ {
+ kernel_type = bUseAnalyticalEwald ? eelCuEWALD_ANA_TWIN : eelCuEWALD_TAB_TWIN;
+ }
+
+ return kernel_type;
+}
+
+
+/*! Initializes the nonbonded parameter data structure. */
+static void init_nbparam(cu_nbparam_t *nbp,
+ const interaction_const_t *ic,
+ const nbnxn_atomdata_t *nbat,
+ const cuda_dev_info_t *dev_info)
+{
+ cudaError_t stat;
+ int ntypes, nnbfp;
+
+ ntypes = nbat->ntype;
+
+ nbp->ewald_beta = ic->ewaldcoeff;
+ nbp->sh_ewald = ic->sh_ewald;
+ nbp->epsfac = ic->epsfac;
+ nbp->two_k_rf = 2.0 * ic->k_rf;
+ nbp->c_rf = ic->c_rf;
+ nbp->rvdw_sq = ic->rvdw * ic->rvdw;
+ nbp->rcoulomb_sq= ic->rcoulomb * ic->rcoulomb;
+ nbp->rlist_sq = ic->rlist * ic->rlist;
+ nbp->sh_invrc6 = ic->sh_invrc6;
+
+ if (ic->eeltype == eelCUT)
+ {
+ nbp->eeltype = eelCuCUT;
+ }
+ else if (EEL_RF(ic->eeltype))
+ {
+ nbp->eeltype = eelCuRF;
+ }
+ else if ((EEL_PME(ic->eeltype) || ic->eeltype==eelEWALD))
+ {
+ /* Initially rcoulomb == rvdw, so it's surely not twin cut-off. */
+ nbp->eeltype = pick_ewald_kernel_type(false, dev_info);
+ }
+ else
+ {
+ /* Shouldn't happen, as this is checked when choosing Verlet-scheme */
+ gmx_incons("The requested electrostatics type is not implemented in the CUDA GPU accelerated kernels!");
+ }
+
+ /* generate table for PME */
+ nbp->coulomb_tab = NULL;
+ if (nbp->eeltype == eelCuEWALD_TAB || nbp->eeltype == eelCuEWALD_TAB_TWIN)
+ {
+ init_ewald_coulomb_force_table(nbp, dev_info);
+ }
+
+ nnbfp = 2*ntypes*ntypes;
+ stat = cudaMalloc((void **)&nbp->nbfp, nnbfp*sizeof(*nbp->nbfp));
+ CU_RET_ERR(stat, "cudaMalloc failed on nbp->nbfp");
+ cu_copy_H2D(nbp->nbfp, nbat->nbfp, nnbfp*sizeof(*nbp->nbfp));
+
+#ifdef TEXOBJ_SUPPORTED
+ /* Only device CC >= 3.0 (Kepler and later) support texture objects */
+ if (dev_info->prop.major >= 3)
+ {
+ cudaResourceDesc rd;
+ memset(&rd, 0, sizeof(rd));
+ rd.resType = cudaResourceTypeLinear;
+ rd.res.linear.devPtr = nbp->nbfp;
+ rd.res.linear.desc.f = cudaChannelFormatKindFloat;
+ rd.res.linear.desc.x = 32;
+ rd.res.linear.sizeInBytes = nnbfp*sizeof(*nbp->nbfp);
+
+ cudaTextureDesc td;
+ memset(&td, 0, sizeof(td));
+ td.readMode = cudaReadModeElementType;
+ stat = cudaCreateTextureObject(&nbp->nbfp_texobj, &rd, &td, NULL);
+ CU_RET_ERR(stat, "cudaCreateTextureObject on nbfp_texobj failed");
+ }
+ else
+#endif
+ {
+ cudaChannelFormatDesc cd = cudaCreateChannelDesc<float>();
+ stat = cudaBindTexture(NULL, &nbnxn_cuda_get_nbfp_texref(),
+ nbp->nbfp, &cd, nnbfp*sizeof(*nbp->nbfp));
+ CU_RET_ERR(stat, "cudaBindTexture on nbfp_texref failed");
+ }
+}
+
+/*! Re-generate the GPU Ewald force table, resets rlist, and update the
+ * electrostatic type switching to twin cut-off (or back) if needed. */
+void nbnxn_cuda_pme_loadbal_update_param(nbnxn_cuda_ptr_t cu_nb,
+ const interaction_const_t *ic)
+{
+ cu_nbparam_t *nbp = cu_nb->nbparam;
+
+ nbp->rlist_sq = ic->rlist * ic->rlist;
+ nbp->rcoulomb_sq = ic->rcoulomb * ic->rcoulomb;
+ nbp->ewald_beta = ic->ewaldcoeff;
+
+ nbp->eeltype = pick_ewald_kernel_type(ic->rcoulomb != ic->rvdw,
+ cu_nb->dev_info);
+
+ init_ewald_coulomb_force_table(cu_nb->nbparam, cu_nb->dev_info);
+}
+
+/*! Initializes the pair list data structure. */
+static void init_plist(cu_plist_t *pl)
+{
+ /* initialize to NULL pointers to data that is not allocated here and will
+ need reallocation in nbnxn_cuda_init_pairlist */
+ pl->sci = NULL;
+ pl->cj4 = NULL;
+ pl->excl = NULL;
+
+ /* size -1 indicates that the respective array hasn't been initialized yet */
+ pl->na_c = -1;
+ pl->nsci = -1;
+ pl->sci_nalloc = -1;
+ pl->ncj4 = -1;
+ pl->cj4_nalloc = -1;
+ pl->nexcl = -1;
+ pl->excl_nalloc = -1;
+ pl->bDoPrune = false;
+}
+
+/*! Initializes the timer data structure. */
+static void init_timers(cu_timers_t *t, bool bUseTwoStreams)
+{
+ cudaError_t stat;
+ int eventflags = ( bUseCudaEventBlockingSync ? cudaEventBlockingSync: cudaEventDefault );
+
+ stat = cudaEventCreateWithFlags(&(t->start_atdat), eventflags);
+ CU_RET_ERR(stat, "cudaEventCreate on start_atdat failed");
+ stat = cudaEventCreateWithFlags(&(t->stop_atdat), eventflags);
+ CU_RET_ERR(stat, "cudaEventCreate on stop_atdat failed");
+
+ /* The non-local counters/stream (second in the array) are needed only with DD. */
+ for (int i = 0; i <= (bUseTwoStreams ? 1 : 0); i++)
+ {
+ stat = cudaEventCreateWithFlags(&(t->start_nb_k[i]), eventflags);
+ CU_RET_ERR(stat, "cudaEventCreate on start_nb_k failed");
+ stat = cudaEventCreateWithFlags(&(t->stop_nb_k[i]), eventflags);
+ CU_RET_ERR(stat, "cudaEventCreate on stop_nb_k failed");
+
+
+ stat = cudaEventCreateWithFlags(&(t->start_pl_h2d[i]), eventflags);
+ CU_RET_ERR(stat, "cudaEventCreate on start_pl_h2d failed");
+ stat = cudaEventCreateWithFlags(&(t->stop_pl_h2d[i]), eventflags);
+ CU_RET_ERR(stat, "cudaEventCreate on stop_pl_h2d failed");
+
+ stat = cudaEventCreateWithFlags(&(t->start_nb_h2d[i]), eventflags);
+ CU_RET_ERR(stat, "cudaEventCreate on start_nb_h2d failed");
+ stat = cudaEventCreateWithFlags(&(t->stop_nb_h2d[i]), eventflags);
+ CU_RET_ERR(stat, "cudaEventCreate on stop_nb_h2d failed");
+
+ stat = cudaEventCreateWithFlags(&(t->start_nb_d2h[i]), eventflags);
+ CU_RET_ERR(stat, "cudaEventCreate on start_nb_d2h failed");
+ stat = cudaEventCreateWithFlags(&(t->stop_nb_d2h[i]), eventflags);
+ CU_RET_ERR(stat, "cudaEventCreate on stop_nb_d2h failed");
+ }
+}
+
+/*! Initializes the timings data structure. */
+static void init_timings(wallclock_gpu_t *t)
+{
+ int i, j;
+
+ t->nb_h2d_t = 0.0;
+ t->nb_d2h_t = 0.0;
+ t->nb_c = 0;
+ t->pl_h2d_t = 0.0;
+ t->pl_h2d_c = 0;
+ for (i = 0; i < 2; i++)
+ {
+ for(j = 0; j < 2; j++)
+ {
+ t->ktime[i][j].t = 0.0;
+ t->ktime[i][j].c = 0;
+ }
+ }
+}
+
+/* Decide which kernel version to use (default or legacy) based on:
+ * - CUDA version used for compilation
+ * - non-bonded kernel selector environment variables
+ * - GPU architecture version
+ */
+static int pick_nbnxn_kernel_version(FILE *fplog,
+ cuda_dev_info_t *devinfo)
+{
+ bool bForceLegacyKernel, bForceDefaultKernel, bCUDA40, bCUDA32;
+ char sbuf[STRLEN];
+ int kver;
+
+ /* Legacy kernel (former k2), kept for backward compatibility as it is
+ faster than the default with CUDA 3.2/4.0 on Fermi (not on Kepler). */
+ bForceLegacyKernel = (getenv("GMX_CUDA_NB_LEGACY") != NULL);
+ /* default kernel (former k3). */
+ bForceDefaultKernel = (getenv("GMX_CUDA_NB_DEFAULT") != NULL);
+
+ if ((unsigned)(bForceLegacyKernel + bForceDefaultKernel) > 1)
+ {
+ gmx_fatal(FARGS, "Multiple CUDA non-bonded kernels requested; to manually pick a kernel set only one \n"
+ "of the following environment variables: \n"
+ "GMX_CUDA_NB_DEFAULT, GMX_CUDA_NB_LEGACY");
+ }
+
+ bCUDA32 = bCUDA40 = false;
+#if CUDA_VERSION == 3200
+ bCUDA32 = true;
+ sprintf(sbuf, "3.2");
+#elif CUDA_VERSION == 4000
+ bCUDA40 = true;
+ sprintf(sbuf, "4.0");
+#endif
+
+ /* default is default ;) */
+ kver = eNbnxnCuKDefault;
+
+ /* Consider switching to legacy kernels only on Fermi */
+ if (devinfo->prop.major < 3 && (bCUDA32 || bCUDA40))
+ {
+ /* use legacy kernel unless something else is forced by an env. var */
+ if (bForceDefaultKernel)
+ {
+ md_print_warn(fplog,
+ "NOTE: CUDA %s compilation detected; with this compiler version the legacy\n"
+ " non-bonded kernels perform best. However, the default kernels were\n"
+ " selected by the GMX_CUDA_NB_DEFAULT environment variable.\n"
+ " For best performance upgrade your CUDA toolkit.\n",
+ sbuf);
+ }
+ else
+ {
+ kver = eNbnxnCuKLegacy;
+ }
+ }
+ else
+ {
+ /* issue note if the non-default kernel is forced by an env. var */
+ if (bForceLegacyKernel)
+ {
+ md_print_warn(fplog,
+ "NOTE: Legacy non-bonded CUDA kernels selected by the GMX_CUDA_NB_LEGACY\n"
+ " env. var. Consider using using the default kernels which should be faster!\n");
+
+ kver = eNbnxnCuKLegacy;
+ }
+ }
+
+ return kver;
+}
+
+void nbnxn_cuda_init(FILE *fplog,
+ nbnxn_cuda_ptr_t *p_cu_nb,
- nb->dev_info = &gpu_info->cuda_dev[get_gpu_device_id(gpu_info, my_gpu_index)];
++ const gmx_gpu_info_t *gpu_info,
++ const gmx_gpu_opt_t *gpu_opt,
++ int my_gpu_index,
+ gmx_bool bLocalAndNonlocal)
+{
+ cudaError_t stat;
+ nbnxn_cuda_ptr_t nb;
+ char sbuf[STRLEN];
+ bool bStreamSync, bNoStreamSync, bTMPIAtomics, bX86, bOldDriver;
+ int cuda_drv_ver;
+
+ assert(gpu_info);
+
+ if (p_cu_nb == NULL) return;
+
+ snew(nb, 1);
+ snew(nb->atdat, 1);
+ snew(nb->nbparam, 1);
+ snew(nb->plist[eintLocal], 1);
+ if (bLocalAndNonlocal)
+ {
+ snew(nb->plist[eintNonlocal], 1);
+ }
+
+ nb->bUseTwoStreams = bLocalAndNonlocal;
+
+ snew(nb->timers, 1);
+ snew(nb->timings, 1);
+
+ /* init nbst */
+ pmalloc((void**)&nb->nbst.e_lj, sizeof(*nb->nbst.e_lj));
+ pmalloc((void**)&nb->nbst.e_el, sizeof(*nb->nbst.e_el));
+ pmalloc((void**)&nb->nbst.fshift, SHIFTS * sizeof(*nb->nbst.fshift));
+
+ init_plist(nb->plist[eintLocal]);
+
+ /* set device info, just point it to the right GPU among the detected ones */
- #ifdef GMX_IS_X86
++ nb->dev_info = &gpu_info->cuda_dev[get_gpu_device_id(gpu_info, gpu_opt, my_gpu_index)];
+
+ /* local/non-local GPU streams */
+ stat = cudaStreamCreate(&nb->stream[eintLocal]);
+ CU_RET_ERR(stat, "cudaStreamCreate on stream[eintLocal] failed");
+ if (nb->bUseTwoStreams)
+ {
+ init_plist(nb->plist[eintNonlocal]);
+
+ /* CUDA stream priority available in the CUDA RT 5.5 API.
+ * Note that the device we're running on does not have to support
+ * priorities, because we are querying the priority range which in this
+ * case will be a single value.
+ */
+#if CUDA_VERSION >= 5500
+ {
+ int highest_priority;
+ stat = cudaDeviceGetStreamPriorityRange(NULL, &highest_priority);
+ CU_RET_ERR(stat, "cudaDeviceGetStreamPriorityRange failed");
+
+ stat = cudaStreamCreateWithPriority(&nb->stream[eintNonlocal],
+ cudaStreamDefault,
+ highest_priority);
+ CU_RET_ERR(stat, "cudaStreamCreateWithPriority on stream[eintNonlocal] failed");
+ }
+#else
+ stat = cudaStreamCreate(&nb->stream[eintNonlocal]);
+ CU_RET_ERR(stat, "cudaStreamCreate on stream[eintNonlocal] failed");
+#endif
+ }
+
+ /* init events for sychronization (timing disabled for performance reasons!) */
+ stat = cudaEventCreateWithFlags(&nb->nonlocal_done, cudaEventDisableTiming);
+ CU_RET_ERR(stat, "cudaEventCreate on nonlocal_done failed");
+ stat = cudaEventCreateWithFlags(&nb->misc_ops_done, cudaEventDisableTiming);
+ CU_RET_ERR(stat, "cudaEventCreate on misc_ops_one failed");
+
+ /* On GPUs with ECC enabled, cudaStreamSynchronize shows a large overhead
+ * (which increases with shorter time/step) caused by a known CUDA driver bug.
+ * To work around the issue we'll use an (admittedly fragile) memory polling
+ * waiting to preserve performance. This requires support for atomic
+ * operations and only works on x86/x86_64.
+ * With polling wait event-timing also needs to be disabled.
+ *
+ * The overhead is greatly reduced in API v5.0 drivers and the improvement
+ $ is independent of runtime version. Hence, with API v5.0 drivers and later
+ * we won't switch to polling.
+ *
+ * NOTE: Unfortunately, this is known to fail when GPUs are shared by (t)MPI,
+ * ranks so we will also disable it in that case.
+ */
+
+ bStreamSync = getenv("GMX_CUDA_STREAMSYNC") != NULL;
+ bNoStreamSync = getenv("GMX_NO_CUDA_STREAMSYNC") != NULL;
+
+#ifdef TMPI_ATOMICS
+ bTMPIAtomics = true;
+#else
+ bTMPIAtomics = false;
+#endif
+
- (gmx_count_gpu_dev_shared(gpu_info) < 1));
++#ifdef GMX_TARGET_X86
+ bX86 = true;
+#else
+ bX86 = false;
+#endif
+
+ if (bStreamSync && bNoStreamSync)
+ {
+ gmx_fatal(FARGS, "Conflicting environment variables: both GMX_CUDA_STREAMSYNC and GMX_NO_CUDA_STREAMSYNC defined");
+ }
+
+ stat = cudaDriverGetVersion(&cuda_drv_ver);
+ CU_RET_ERR(stat, "cudaDriverGetVersion failed");
+
+ bOldDriver = (cuda_drv_ver < 5000);
+
+ if ((nb->dev_info->prop.ECCEnabled == 1) && bOldDriver)
+ {
+ /* Polling wait should be used instead of cudaStreamSynchronize only if:
+ * - ECC is ON & driver is old (checked above),
+ * - we're on x86/x86_64,
+ * - atomics are available, and
+ * - GPUs are not being shared.
+ */
+ bool bShouldUsePollSync = (bX86 && bTMPIAtomics &&
++ (gmx_count_gpu_dev_shared(gpu_opt) < 1));
+
+ if (bStreamSync)
+ {
+ nb->bUseStreamSync = true;
+
+ /* only warn if polling should be used */
+ if (bShouldUsePollSync)
+ {
+ md_print_warn(fplog,
+ "NOTE: Using a GPU with ECC enabled and CUDA driver API version <5.0, but\n"
+ " cudaStreamSynchronize waiting is forced by the GMX_CUDA_STREAMSYNC env. var.\n");
+ }
+ }
+ else
+ {
+ nb->bUseStreamSync = !bShouldUsePollSync;
+
+ if (bShouldUsePollSync)
+ {
+ md_print_warn(fplog,
+ "NOTE: Using a GPU with ECC enabled and CUDA driver API version <5.0, known to\n"
+ " cause performance loss. Switching to the alternative polling GPU wait.\n"
+ " If you encounter issues, switch back to standard GPU waiting by setting\n"
+ " the GMX_CUDA_STREAMSYNC environment variable.\n");
+ }
+ else
+ {
+ /* Tell the user that the ECC+old driver combination can be bad */
+ sprintf(sbuf,
+ "NOTE: Using a GPU with ECC enabled and CUDA driver API version <5.0.\n"
+ " A known bug in this driver version can cause performance loss.\n"
+ " However, the polling wait workaround can not be used because\n%s\n"
+ " Consider updating the driver or turning ECC off.",
+ (bX86 && bTMPIAtomics) ?
+ " GPU(s) are being oversubscribed." :
+ " atomic operations are not supported by the platform/CPU+compiler.");
+ md_print_warn(fplog, sbuf);
+ }
+ }
+ }
+ else
+ {
+ if (bNoStreamSync)
+ {
+ nb->bUseStreamSync = false;
+
+ md_print_warn(fplog,
+ "NOTE: Polling wait for GPU synchronization requested by GMX_NO_CUDA_STREAMSYNC\n");
+ }
+ else
+ {
+ /* no/off ECC, cudaStreamSynchronize not turned off by env. var. */
+ nb->bUseStreamSync = true;
+ }
+ }
+
+ /* CUDA timing disabled as event timers don't work:
+ - with multiple streams = domain-decomposition;
+ - with the polling waiting hack (without cudaStreamSynchronize);
+ - when turned off by GMX_DISABLE_CUDA_TIMING.
+ */
+ nb->bDoTime = (!nb->bUseTwoStreams && nb->bUseStreamSync &&
+ (getenv("GMX_DISABLE_CUDA_TIMING") == NULL));
+
+ if (nb->bDoTime)
+ {
+ init_timers(nb->timers, nb->bUseTwoStreams);
+ init_timings(nb->timings);
+ }
+
+ /* set the kernel type for the current GPU */
+ nb->kernel_ver = pick_nbnxn_kernel_version(fplog, nb->dev_info);
+ /* pick L1 cache configuration */
+ nbnxn_cuda_set_cacheconfig(nb->dev_info);
+
+ *p_cu_nb = nb;
+
+ if (debug)
+ {
+ fprintf(debug, "Initialized CUDA data structures.\n");
+ }
+}
+
+void nbnxn_cuda_init_const(nbnxn_cuda_ptr_t cu_nb,
+ const interaction_const_t *ic,
+ const nonbonded_verlet_group_t *nbv_group)
+{
+ init_atomdata_first(cu_nb->atdat, nbv_group[0].nbat->ntype);
+ init_nbparam(cu_nb->nbparam, ic, nbv_group[0].nbat, cu_nb->dev_info);
+
+ /* clear energy and shift force outputs */
+ nbnxn_cuda_clear_e_fshift(cu_nb);
+}
+
+void nbnxn_cuda_init_pairlist(nbnxn_cuda_ptr_t cu_nb,
+ const nbnxn_pairlist_t *h_plist,
+ int iloc)
+{
+ char sbuf[STRLEN];
+ cudaError_t stat;
+ bool bDoTime = cu_nb->bDoTime;
+ cudaStream_t stream = cu_nb->stream[iloc];
+ cu_plist_t *d_plist = cu_nb->plist[iloc];
+
+ if (d_plist->na_c < 0)
+ {
+ d_plist->na_c = h_plist->na_ci;
+ }
+ else
+ {
+ if (d_plist->na_c != h_plist->na_ci)
+ {
+ sprintf(sbuf, "In cu_init_plist: the #atoms per cell has changed (from %d to %d)",
+ d_plist->na_c, h_plist->na_ci);
+ gmx_incons(sbuf);
+ }
+ }
+
+ if (bDoTime)
+ {
+ stat = cudaEventRecord(cu_nb->timers->start_pl_h2d[iloc], stream);
+ CU_RET_ERR(stat, "cudaEventRecord failed");
+ }
+
+ cu_realloc_buffered((void **)&d_plist->sci, h_plist->sci, sizeof(*d_plist->sci),
+ &d_plist->nsci, &d_plist->sci_nalloc,
+ h_plist->nsci,
+ stream, true);
+
+ cu_realloc_buffered((void **)&d_plist->cj4, h_plist->cj4, sizeof(*d_plist->cj4),
+ &d_plist->ncj4, &d_plist->cj4_nalloc,
+ h_plist->ncj4,
+ stream, true);
+
+ cu_realloc_buffered((void **)&d_plist->excl, h_plist->excl, sizeof(*d_plist->excl),
+ &d_plist->nexcl, &d_plist->excl_nalloc,
+ h_plist->nexcl,
+ stream, true);
+
+ if (bDoTime)
+ {
+ stat = cudaEventRecord(cu_nb->timers->stop_pl_h2d[iloc], stream);
+ CU_RET_ERR(stat, "cudaEventRecord failed");
+ }
+
+ /* need to prune the pair list during the next step */
+ d_plist->bDoPrune = true;
+}
+
+void nbnxn_cuda_upload_shiftvec(nbnxn_cuda_ptr_t cu_nb,
+ const nbnxn_atomdata_t *nbatom)
+{
+ cu_atomdata_t *adat = cu_nb->atdat;
+ cudaStream_t ls = cu_nb->stream[eintLocal];
+
+ /* only if we have a dynamic box */
+ if (nbatom->bDynamicBox || !adat->bShiftVecUploaded)
+ {
+ cu_copy_H2D_async(adat->shift_vec, nbatom->shift_vec,
+ SHIFTS * sizeof(*adat->shift_vec), ls);
+ adat->bShiftVecUploaded = true;
+ }
+}
+
+/*! Clears the first natoms_clear elements of the GPU nonbonded force output array. */
+static void nbnxn_cuda_clear_f(nbnxn_cuda_ptr_t cu_nb, int natoms_clear)
+{
+ cudaError_t stat;
+ cu_atomdata_t *adat = cu_nb->atdat;
+ cudaStream_t ls = cu_nb->stream[eintLocal];
+
+ stat = cudaMemsetAsync(adat->f, 0, natoms_clear * sizeof(*adat->f), ls);
+ CU_RET_ERR(stat, "cudaMemsetAsync on f falied");
+}
+
+/*! Clears nonbonded shift force output array and energy outputs on the GPU. */
+static void nbnxn_cuda_clear_e_fshift(nbnxn_cuda_ptr_t cu_nb)
+{
+ cudaError_t stat;
+ cu_atomdata_t *adat = cu_nb->atdat;
+ cudaStream_t ls = cu_nb->stream[eintLocal];
+
+ stat = cudaMemsetAsync(adat->fshift, 0, SHIFTS * sizeof(*adat->fshift), ls);
+ CU_RET_ERR(stat, "cudaMemsetAsync on fshift falied");
+ stat = cudaMemsetAsync(adat->e_lj, 0, sizeof(*adat->e_lj), ls);
+ CU_RET_ERR(stat, "cudaMemsetAsync on e_lj falied");
+ stat = cudaMemsetAsync(adat->e_el, 0, sizeof(*adat->e_el), ls);
+ CU_RET_ERR(stat, "cudaMemsetAsync on e_el falied");
+}
+
+void nbnxn_cuda_clear_outputs(nbnxn_cuda_ptr_t cu_nb, int flags)
+{
+ nbnxn_cuda_clear_f(cu_nb, cu_nb->atdat->natoms);
+ /* clear shift force array and energies if the outputs were
+ used in the current step */
+ if (flags & GMX_FORCE_VIRIAL)
+ {
+ nbnxn_cuda_clear_e_fshift(cu_nb);
+ }
+}
+
+void nbnxn_cuda_init_atomdata(nbnxn_cuda_ptr_t cu_nb,
+ const nbnxn_atomdata_t *nbat)
+{
+ cudaError_t stat;
+ int nalloc, natoms;
+ bool realloced;
+ bool bDoTime = cu_nb->bDoTime;
+ cu_timers_t *timers = cu_nb->timers;
+ cu_atomdata_t *d_atdat = cu_nb->atdat;
+ cudaStream_t ls = cu_nb->stream[eintLocal];
+
+ natoms = nbat->natoms;
+ realloced = false;
+
+ if (bDoTime)
+ {
+ /* time async copy */
+ stat = cudaEventRecord(timers->start_atdat, ls);
+ CU_RET_ERR(stat, "cudaEventRecord failed");
+ }
+
+ /* need to reallocate if we have to copy more atoms than the amount of space
+ available and only allocate if we haven't initialized yet, i.e d_atdat->natoms == -1 */
+ if (natoms > d_atdat->nalloc)
+ {
+ nalloc = over_alloc_small(natoms);
+
+ /* free up first if the arrays have already been initialized */
+ if (d_atdat->nalloc != -1)
+ {
+ cu_free_buffered(d_atdat->f, &d_atdat->natoms, &d_atdat->nalloc);
+ cu_free_buffered(d_atdat->xq);
+ cu_free_buffered(d_atdat->atom_types);
+ }
+
+ stat = cudaMalloc((void **)&d_atdat->f, nalloc*sizeof(*d_atdat->f));
+ CU_RET_ERR(stat, "cudaMalloc failed on d_atdat->f");
+ stat = cudaMalloc((void **)&d_atdat->xq, nalloc*sizeof(*d_atdat->xq));
+ CU_RET_ERR(stat, "cudaMalloc failed on d_atdat->xq");
+
+ stat = cudaMalloc((void **)&d_atdat->atom_types, nalloc*sizeof(*d_atdat->atom_types));
+ CU_RET_ERR(stat, "cudaMalloc failed on d_atdat->atom_types");
+
+ d_atdat->nalloc = nalloc;
+ realloced = true;
+ }
+
+ d_atdat->natoms = natoms;
+ d_atdat->natoms_local = nbat->natoms_local;
+
+ /* need to clear GPU f output if realloc happened */
+ if (realloced)
+ {
+ nbnxn_cuda_clear_f(cu_nb, nalloc);
+ }
+
+ cu_copy_H2D_async(d_atdat->atom_types, nbat->type,
+ natoms*sizeof(*d_atdat->atom_types), ls);
+
+ if (bDoTime)
+ {
+ stat = cudaEventRecord(timers->stop_atdat, ls);
+ CU_RET_ERR(stat, "cudaEventRecord failed");
+ }
+}
+
+void nbnxn_cuda_free(FILE *fplog, nbnxn_cuda_ptr_t cu_nb)
+{
+ cudaError_t stat;
+ cu_atomdata_t *atdat;
+ cu_nbparam_t *nbparam;
+ cu_plist_t *plist, *plist_nl;
+ cu_timers_t *timers;
+
+ if (cu_nb == NULL) return;
+
+ atdat = cu_nb->atdat;
+ nbparam = cu_nb->nbparam;
+ plist = cu_nb->plist[eintLocal];
+ plist_nl = cu_nb->plist[eintNonlocal];
+ timers = cu_nb->timers;
+
+ if (nbparam->eeltype == eelCuEWALD_TAB || nbparam->eeltype == eelCuEWALD_TAB_TWIN)
+ {
+
+#ifdef TEXOBJ_SUPPORTED
+ /* Only device CC >= 3.0 (Kepler and later) support texture objects */
+ if (cu_nb->dev_info->prop.major >= 3)
+ {
+ stat = cudaDestroyTextureObject(nbparam->coulomb_tab_texobj);
+ CU_RET_ERR(stat, "cudaDestroyTextureObject on coulomb_tab_texobj failed");
+ }
+ else
+#endif
+ {
+ stat = cudaUnbindTexture(nbnxn_cuda_get_coulomb_tab_texref());
+ CU_RET_ERR(stat, "cudaUnbindTexture on coulomb_tab_texref failed");
+ }
+ cu_free_buffered(nbparam->coulomb_tab, &nbparam->coulomb_tab_size);
+ }
+
+ stat = cudaEventDestroy(cu_nb->nonlocal_done);
+ CU_RET_ERR(stat, "cudaEventDestroy failed on timers->nonlocal_done");
+ stat = cudaEventDestroy(cu_nb->misc_ops_done);
+ CU_RET_ERR(stat, "cudaEventDestroy failed on timers->misc_ops_done");
+
+ if (cu_nb->bDoTime)
+ {
+ stat = cudaEventDestroy(timers->start_atdat);
+ CU_RET_ERR(stat, "cudaEventDestroy failed on timers->start_atdat");
+ stat = cudaEventDestroy(timers->stop_atdat);
+ CU_RET_ERR(stat, "cudaEventDestroy failed on timers->stop_atdat");
+
+ /* The non-local counters/stream (second in the array) are needed only with DD. */
+ for (int i = 0; i <= (cu_nb->bUseTwoStreams ? 1 : 0); i++)
+ {
+ stat = cudaEventDestroy(timers->start_nb_k[i]);
+ CU_RET_ERR(stat, "cudaEventDestroy failed on timers->start_nb_k");
+ stat = cudaEventDestroy(timers->stop_nb_k[i]);
+ CU_RET_ERR(stat, "cudaEventDestroy failed on timers->stop_nb_k");
+
+ stat = cudaEventDestroy(timers->start_pl_h2d[i]);
+ CU_RET_ERR(stat, "cudaEventDestroy failed on timers->start_pl_h2d");
+ stat = cudaEventDestroy(timers->stop_pl_h2d[i]);
+ CU_RET_ERR(stat, "cudaEventDestroy failed on timers->stop_pl_h2d");
+
+ stat = cudaStreamDestroy(cu_nb->stream[i]);
+ CU_RET_ERR(stat, "cudaStreamDestroy failed on stream");
+
+ stat = cudaEventDestroy(timers->start_nb_h2d[i]);
+ CU_RET_ERR(stat, "cudaEventDestroy failed on timers->start_nb_h2d");
+ stat = cudaEventDestroy(timers->stop_nb_h2d[i]);
+ CU_RET_ERR(stat, "cudaEventDestroy failed on timers->stop_nb_h2d");
+
+ stat = cudaEventDestroy(timers->start_nb_d2h[i]);
+ CU_RET_ERR(stat, "cudaEventDestroy failed on timers->start_nb_d2h");
+ stat = cudaEventDestroy(timers->stop_nb_d2h[i]);
+ CU_RET_ERR(stat, "cudaEventDestroy failed on timers->stop_nb_d2h");
+ }
+ }
+
+#ifdef TEXOBJ_SUPPORTED
+ /* Only device CC >= 3.0 (Kepler and later) support texture objects */
+ if (cu_nb->dev_info->prop.major >= 3)
+ {
+ stat = cudaDestroyTextureObject(nbparam->nbfp_texobj);
+ CU_RET_ERR(stat, "cudaDestroyTextureObject on nbfp_texobj failed");
+ }
+ else
+#endif
+ {
+ stat = cudaUnbindTexture(nbnxn_cuda_get_nbfp_texref());
+ CU_RET_ERR(stat, "cudaUnbindTexture on nbfp_texref failed");
+ }
+ cu_free_buffered(nbparam->nbfp);
+
+ stat = cudaFree(atdat->shift_vec);
+ CU_RET_ERR(stat, "cudaFree failed on atdat->shift_vec");
+ stat = cudaFree(atdat->fshift);
+ CU_RET_ERR(stat, "cudaFree failed on atdat->fshift");
+
+ stat = cudaFree(atdat->e_lj);
+ CU_RET_ERR(stat, "cudaFree failed on atdat->e_lj");
+ stat = cudaFree(atdat->e_el);
+ CU_RET_ERR(stat, "cudaFree failed on atdat->e_el");
+
+ cu_free_buffered(atdat->f, &atdat->natoms, &atdat->nalloc);
+ cu_free_buffered(atdat->xq);
+ cu_free_buffered(atdat->atom_types, &atdat->ntypes);
+
+ cu_free_buffered(plist->sci, &plist->nsci, &plist->sci_nalloc);
+ cu_free_buffered(plist->cj4, &plist->ncj4, &plist->cj4_nalloc);
+ cu_free_buffered(plist->excl, &plist->nexcl, &plist->excl_nalloc);
+ if (cu_nb->bUseTwoStreams)
+ {
+ cu_free_buffered(plist_nl->sci, &plist_nl->nsci, &plist_nl->sci_nalloc);
+ cu_free_buffered(plist_nl->cj4, &plist_nl->ncj4, &plist_nl->cj4_nalloc);
+ cu_free_buffered(plist_nl->excl, &plist_nl->nexcl, &plist->excl_nalloc);
+ }
+
+ sfree(atdat);
+ sfree(nbparam);
+ sfree(plist);
+ if (cu_nb->bUseTwoStreams)
+ {
+ sfree(plist_nl);
+ }
+ sfree(timers);
+ sfree(cu_nb->timings);
+ sfree(cu_nb);
+
+ if (debug)
+ {
+ fprintf(debug, "Cleaned up CUDA data structures.\n");
+ }
+}
+
+void cu_synchstream_atdat(nbnxn_cuda_ptr_t cu_nb, int iloc)
+{
+ cudaError_t stat;
+ cudaStream_t stream = cu_nb->stream[iloc];
+
+ stat = cudaStreamWaitEvent(stream, cu_nb->timers->stop_atdat, 0);
+ CU_RET_ERR(stat, "cudaStreamWaitEvent failed");
+}
+
+wallclock_gpu_t * nbnxn_cuda_get_timings(nbnxn_cuda_ptr_t cu_nb)
+{
+ return (cu_nb != NULL && cu_nb->bDoTime) ? cu_nb->timings : NULL;
+}
+
+void nbnxn_cuda_reset_timings(nbnxn_cuda_ptr_t cu_nb)
+{
+ if (cu_nb->bDoTime)
+ {
+ init_timings(cu_nb->timings);
+ }
+}
+
+int nbnxn_cuda_min_ci_balanced(nbnxn_cuda_ptr_t cu_nb)
+{
+ return cu_nb != NULL ?
+ gpu_min_ci_balanced_factor*cu_nb->dev_info->prop.multiProcessorCount : 0;
+
+}
+
+gmx_bool nbnxn_cuda_is_kernel_ewald_analytical(const nbnxn_cuda_ptr_t cu_nb)
+{
+ return ((cu_nb->nbparam->eeltype == eelCuEWALD_ANA) ||
+ (cu_nb->nbparam->eeltype == eelCuEWALD_ANA_TWIN));
+}
--- /dev/null
- /* With Ewald type electrostatics we the forces for excluded atom pairs
- * should not contribute to the virial sum. The exclusion forces
- * are not calculate in the energy kernels, but are in _noener.
- */
- if (!((force_flags & GMX_FORCE_ENERGY) ||
- (EEL_FULL(ic->eeltype) && (force_flags & GMX_FORCE_VIRIAL))))
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include "typedefs.h"
+
+#ifdef {0}
+
+{1}
+#include "gmx_simd_macros.h"
+#include "gmx_simd_vec.h"
+{2}
+#define GMX_SIMD_J_UNROLL_SIZE {3}
+#include "{4}"
+#include "../nbnxn_kernel_common.h"
+#include "gmx_omp_nthreads.h"
+#include "types/force_flags.h"
+
+/*! \brief Kinds of electrostatic treatments in SIMD Verlet kernels
+ */
+enum {{
+ coultRF, coultTAB, coultTAB_TWIN, coultEWALD, coultEWALD_TWIN, coultNR
+}};
+
+/* Declare and define the kernel function pointer lookup tables. */
+static p_nbk_func_ener p_nbk_ener[coultNR][ljcrNR] =
+{7}
+static p_nbk_func_ener p_nbk_energrp[coultNR][ljcrNR] =
+{8}
+static p_nbk_func_noener p_nbk_noener[coultNR][ljcrNR] =
+{9}
+
+static void
+reduce_group_energies(int ng, int ng_2log,
+ const real *VSvdw, const real *VSc,
+ real *Vvdw, real *Vc)
+{{
+ const int unrollj = GMX_SIMD_WIDTH_HERE/GMX_SIMD_J_UNROLL_SIZE;
+ const int unrollj_half = unrollj/2;
+ int ng_p2, i, j, j0, j1, c, s;
+
+ ng_p2 = (1<<ng_2log);
+
+ /* The size of the x86 SIMD energy group buffer array is:
+ * ng*ng*ng_p2*unrollj_half*simd_width
+ */
+ for (i = 0; i < ng; i++)
+ {{
+ for (j = 0; j < ng; j++)
+ {{
+ Vvdw[i*ng+j] = 0;
+ Vc[i*ng+j] = 0;
+ }}
+
+ for (j1 = 0; j1 < ng; j1++)
+ {{
+ for (j0 = 0; j0 < ng; j0++)
+ {{
+ c = ((i*ng + j1)*ng_p2 + j0)*unrollj_half*unrollj;
+ for (s = 0; s < unrollj_half; s++)
+ {{
+ Vvdw[i*ng+j0] += VSvdw[c+0];
+ Vvdw[i*ng+j1] += VSvdw[c+1];
+ Vc [i*ng+j0] += VSc [c+0];
+ Vc [i*ng+j1] += VSc [c+1];
+ c += unrollj + 2;
+ }}
+ }}
+ }}
+ }}
+}}
+
+#else /* {0} */
+
+#include "gmx_fatal.h"
+
+#endif /* {0} */
+
+void
+{5}(nbnxn_pairlist_set_t gmx_unused *nbl_list,
+{6}const nbnxn_atomdata_t gmx_unused *nbat,
+{6}const interaction_const_t gmx_unused *ic,
+{6}int gmx_unused ewald_excl,
+{6}rvec gmx_unused *shift_vec,
+{6}int gmx_unused force_flags,
+{6}int gmx_unused clearF,
+{6}real gmx_unused *fshift,
+{6}real gmx_unused *Vc,
+{6}real gmx_unused *Vvdw)
+#ifdef {0}
+{{
+ int nnbl;
+ nbnxn_pairlist_t **nbl;
+ int coult;
+ int nb;
+
+ nnbl = nbl_list->nnbl;
+ nbl = nbl_list->nbl;
+
+ if (EEL_RF(ic->eeltype) || ic->eeltype == eelCUT)
+ {{
+ coult = coultRF;
+ }}
+ else
+ {{
+ if (ewald_excl == ewaldexclTable)
+ {{
+ if (ic->rcoulomb == ic->rvdw)
+ {{
+ coult = coultTAB;
+ }}
+ else
+ {{
+ coult = coultTAB_TWIN;
+ }}
+ }}
+ else
+ {{
+ if (ic->rcoulomb == ic->rvdw)
+ {{
+ coult = coultEWALD;
+ }}
+ else
+ {{
+ coult = coultEWALD_TWIN;
+ }}
+ }}
+ }}
+
+#pragma omp parallel for schedule(static) num_threads(gmx_omp_nthreads_get(emntNonbonded))
+ for (nb = 0; nb < nnbl; nb++)
+ {{
+ nbnxn_atomdata_output_t *out;
+ real *fshift_p;
+
+ out = &nbat->out[nb];
+
+ if (clearF == enbvClearFYes)
+ {{
+ clear_f(nbat, nb, out->f);
+ }}
+
+ if ((force_flags & GMX_FORCE_VIRIAL) && nnbl == 1)
+ {{
+ fshift_p = fshift;
+ }}
+ else
+ {{
+ fshift_p = out->fshift;
+
+ if (clearF == enbvClearFYes)
+ {{
+ clear_fshift(fshift_p);
+ }}
+ }}
+
- else if (out->nV == 1 || !(force_flags & GMX_FORCE_ENERGY))
++ if (!(force_flags & GMX_FORCE_ENERGY))
+ {{
+ /* Don't calculate energies */
+ p_nbk_noener[coult][nbat->comb_rule](nbl[nb], nbat,
+ ic,
+ shift_vec,
+ out->f,
+ fshift_p);
+ }}
++ else if (out->nV == 1)
+ {{
+ /* No energy groups */
+ out->Vvdw[0] = 0;
+ out->Vc[0] = 0;
+
+ p_nbk_ener[coult][nbat->comb_rule](nbl[nb], nbat,
+ ic,
+ shift_vec,
+ out->f,
+ fshift_p,
+ out->Vvdw,
+ out->Vc);
+ }}
+ else
+ {{
+ /* Calculate energy group contributions */
+ int i;
+
+ for (i = 0; i < out->nVS; i++)
+ {{
+ out->VSvdw[i] = 0;
+ }}
+ for (i = 0; i < out->nVS; i++)
+ {{
+ out->VSc[i] = 0;
+ }}
+
+ p_nbk_energrp[coult][nbat->comb_rule](nbl[nb], nbat,
+ ic,
+ shift_vec,
+ out->f,
+ fshift_p,
+ out->VSvdw,
+ out->VSc);
+
+ reduce_group_energies(nbat->nenergrp, nbat->neg_2log,
+ out->VSvdw, out->VSc,
+ out->Vvdw, out->Vc);
+ }}
+ }}
+
+ if (force_flags & GMX_FORCE_ENERGY)
+ {{
+ reduce_energies_over_lists(nbat, nnbl, Vvdw, Vc);
+ }}
+}}
+#else
+{{
+ gmx_incons("{5} called when such kernels "
+ " are not enabled.");
+}}
+#endif
+#undef GMX_SIMD_J_UNROLL_SIZE
--- /dev/null
- /* With Ewald type electrostatics we the forces for excluded atom pairs
- * should not contribute to the virial sum. The exclusion forces
- * are not calculate in the energy kernels, but are in _noener.
- */
- if (!((force_flags & GMX_FORCE_ENERGY) ||
- (EEL_FULL(ic->eeltype) && (force_flags & GMX_FORCE_VIRIAL))))
+/*
+ * This file is part of the GROMACS molecular simulation package.
+ *
+ * Copyright (c) 2012,2013, by the GROMACS development team, led by
+ * David van der Spoel, Berk Hess, Erik Lindahl, and including many
+ * others, as listed in the AUTHORS file in the top-level source
+ * directory and at http://www.gromacs.org.
+ *
+ * GROMACS is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ *
+ * GROMACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with GROMACS; if not, see
+ * http://www.gnu.org/licenses, or write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * If you want to redistribute modifications to GROMACS, please
+ * consider that scientific software is very special. Version
+ * control is crucial - bugs must be traceable. We will be happy to
+ * consider code for inclusion in the official distribution, but
+ * derived work must not be called official GROMACS. Details are found
+ * in the README & COPYING files - if they are missing, get the
+ * official version at http://www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the research papers on the package. Check out http://www.gromacs.org.
+ */
+/*
+ * Note: this file was generated by the Verlet kernel generator for
+ * kernel type 2xnn.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include "typedefs.h"
+
+#ifdef GMX_NBNXN_SIMD_2XNN
+
+/* Include the full-width SIMD macros */
+
+#include "gmx_simd_macros.h"
+#include "gmx_simd_vec.h"
+#if !(GMX_SIMD_WIDTH_HERE == 8 || GMX_SIMD_WIDTH_HERE == 16)
+#error "unsupported SIMD width"
+#endif
+
+#define GMX_SIMD_J_UNROLL_SIZE 2
+#include "nbnxn_kernel_simd_2xnn.h"
+#include "../nbnxn_kernel_common.h"
+#include "gmx_omp_nthreads.h"
+#include "types/force_flags.h"
+
+/*! \brief Kinds of electrostatic treatments in SIMD Verlet kernels
+ */
+enum {
+ coultRF, coultTAB, coultTAB_TWIN, coultEWALD, coultEWALD_TWIN, coultNR
+};
+
+/* Declare and define the kernel function pointer lookup tables. */
+static p_nbk_func_ener p_nbk_ener[coultNR][ljcrNR] =
+{
+ {
+ nbnxn_kernel_simd_2xnn_rf_comb_geom_ener,
+ nbnxn_kernel_simd_2xnn_rf_comb_lb_ener,
+ nbnxn_kernel_simd_2xnn_rf_comb_none_ener,
+ },
+ {
+ nbnxn_kernel_simd_2xnn_tab_comb_geom_ener,
+ nbnxn_kernel_simd_2xnn_tab_comb_lb_ener,
+ nbnxn_kernel_simd_2xnn_tab_comb_none_ener,
+ },
+ {
+ nbnxn_kernel_simd_2xnn_tab_twin_comb_geom_ener,
+ nbnxn_kernel_simd_2xnn_tab_twin_comb_lb_ener,
+ nbnxn_kernel_simd_2xnn_tab_twin_comb_none_ener,
+ },
+ {
+ nbnxn_kernel_simd_2xnn_ewald_comb_geom_ener,
+ nbnxn_kernel_simd_2xnn_ewald_comb_lb_ener,
+ nbnxn_kernel_simd_2xnn_ewald_comb_none_ener,
+ },
+ {
+ nbnxn_kernel_simd_2xnn_ewald_twin_comb_geom_ener,
+ nbnxn_kernel_simd_2xnn_ewald_twin_comb_lb_ener,
+ nbnxn_kernel_simd_2xnn_ewald_twin_comb_none_ener,
+ },
+};
+
+static p_nbk_func_ener p_nbk_energrp[coultNR][ljcrNR] =
+{
+ {
+ nbnxn_kernel_simd_2xnn_rf_comb_geom_energrp,
+ nbnxn_kernel_simd_2xnn_rf_comb_lb_energrp,
+ nbnxn_kernel_simd_2xnn_rf_comb_none_energrp,
+ },
+ {
+ nbnxn_kernel_simd_2xnn_tab_comb_geom_energrp,
+ nbnxn_kernel_simd_2xnn_tab_comb_lb_energrp,
+ nbnxn_kernel_simd_2xnn_tab_comb_none_energrp,
+ },
+ {
+ nbnxn_kernel_simd_2xnn_tab_twin_comb_geom_energrp,
+ nbnxn_kernel_simd_2xnn_tab_twin_comb_lb_energrp,
+ nbnxn_kernel_simd_2xnn_tab_twin_comb_none_energrp,
+ },
+ {
+ nbnxn_kernel_simd_2xnn_ewald_comb_geom_energrp,
+ nbnxn_kernel_simd_2xnn_ewald_comb_lb_energrp,
+ nbnxn_kernel_simd_2xnn_ewald_comb_none_energrp,
+ },
+ {
+ nbnxn_kernel_simd_2xnn_ewald_twin_comb_geom_energrp,
+ nbnxn_kernel_simd_2xnn_ewald_twin_comb_lb_energrp,
+ nbnxn_kernel_simd_2xnn_ewald_twin_comb_none_energrp,
+ },
+};
+
+static p_nbk_func_noener p_nbk_noener[coultNR][ljcrNR] =
+{
+ {
+ nbnxn_kernel_simd_2xnn_rf_comb_geom_noener,
+ nbnxn_kernel_simd_2xnn_rf_comb_lb_noener,
+ nbnxn_kernel_simd_2xnn_rf_comb_none_noener,
+ },
+ {
+ nbnxn_kernel_simd_2xnn_tab_comb_geom_noener,
+ nbnxn_kernel_simd_2xnn_tab_comb_lb_noener,
+ nbnxn_kernel_simd_2xnn_tab_comb_none_noener,
+ },
+ {
+ nbnxn_kernel_simd_2xnn_tab_twin_comb_geom_noener,
+ nbnxn_kernel_simd_2xnn_tab_twin_comb_lb_noener,
+ nbnxn_kernel_simd_2xnn_tab_twin_comb_none_noener,
+ },
+ {
+ nbnxn_kernel_simd_2xnn_ewald_comb_geom_noener,
+ nbnxn_kernel_simd_2xnn_ewald_comb_lb_noener,
+ nbnxn_kernel_simd_2xnn_ewald_comb_none_noener,
+ },
+ {
+ nbnxn_kernel_simd_2xnn_ewald_twin_comb_geom_noener,
+ nbnxn_kernel_simd_2xnn_ewald_twin_comb_lb_noener,
+ nbnxn_kernel_simd_2xnn_ewald_twin_comb_none_noener,
+ },
+};
+
+
+static void
+reduce_group_energies(int ng, int ng_2log,
+ const real *VSvdw, const real *VSc,
+ real *Vvdw, real *Vc)
+{
+ const int unrollj = GMX_SIMD_WIDTH_HERE/GMX_SIMD_J_UNROLL_SIZE;
+ const int unrollj_half = unrollj/2;
+ int ng_p2, i, j, j0, j1, c, s;
+
+ ng_p2 = (1<<ng_2log);
+
+ /* The size of the x86 SIMD energy group buffer array is:
+ * ng*ng*ng_p2*unrollj_half*simd_width
+ */
+ for (i = 0; i < ng; i++)
+ {
+ for (j = 0; j < ng; j++)
+ {
+ Vvdw[i*ng+j] = 0;
+ Vc[i*ng+j] = 0;
+ }
+
+ for (j1 = 0; j1 < ng; j1++)
+ {
+ for (j0 = 0; j0 < ng; j0++)
+ {
+ c = ((i*ng + j1)*ng_p2 + j0)*unrollj_half*unrollj;
+ for (s = 0; s < unrollj_half; s++)
+ {
+ Vvdw[i*ng+j0] += VSvdw[c+0];
+ Vvdw[i*ng+j1] += VSvdw[c+1];
+ Vc [i*ng+j0] += VSc [c+0];
+ Vc [i*ng+j1] += VSc [c+1];
+ c += unrollj + 2;
+ }
+ }
+ }
+ }
+}
+
+#else /* GMX_NBNXN_SIMD_2XNN */
+
+#include "gmx_fatal.h"
+
+#endif /* GMX_NBNXN_SIMD_2XNN */
+
+void
+nbnxn_kernel_simd_2xnn(nbnxn_pairlist_set_t gmx_unused *nbl_list,
+ const nbnxn_atomdata_t gmx_unused *nbat,
+ const interaction_const_t gmx_unused *ic,
+ int gmx_unused ewald_excl,
+ rvec gmx_unused *shift_vec,
+ int gmx_unused force_flags,
+ int gmx_unused clearF,
+ real gmx_unused *fshift,
+ real gmx_unused *Vc,
+ real gmx_unused *Vvdw)
+#ifdef GMX_NBNXN_SIMD_2XNN
+{
+ int nnbl;
+ nbnxn_pairlist_t **nbl;
+ int coult;
+ int nb;
+
+ nnbl = nbl_list->nnbl;
+ nbl = nbl_list->nbl;
+
+ if (EEL_RF(ic->eeltype) || ic->eeltype == eelCUT)
+ {
+ coult = coultRF;
+ }
+ else
+ {
+ if (ewald_excl == ewaldexclTable)
+ {
+ if (ic->rcoulomb == ic->rvdw)
+ {
+ coult = coultTAB;
+ }
+ else
+ {
+ coult = coultTAB_TWIN;
+ }
+ }
+ else
+ {
+ if (ic->rcoulomb == ic->rvdw)
+ {
+ coult = coultEWALD;
+ }
+ else
+ {
+ coult = coultEWALD_TWIN;
+ }
+ }
+ }
+
+#pragma omp parallel for schedule(static) num_threads(gmx_omp_nthreads_get(emntNonbonded))
+ for (nb = 0; nb < nnbl; nb++)
+ {
+ nbnxn_atomdata_output_t *out;
+ real *fshift_p;
+
+ out = &nbat->out[nb];
+
+ if (clearF == enbvClearFYes)
+ {
+ clear_f(nbat, nb, out->f);
+ }
+
+ if ((force_flags & GMX_FORCE_VIRIAL) && nnbl == 1)
+ {
+ fshift_p = fshift;
+ }
+ else
+ {
+ fshift_p = out->fshift;
+
+ if (clearF == enbvClearFYes)
+ {
+ clear_fshift(fshift_p);
+ }
+ }
+
- else if (out->nV == 1 || !(force_flags & GMX_FORCE_ENERGY))
++ if (!(force_flags & GMX_FORCE_ENERGY))
+ {
+ /* Don't calculate energies */
+ p_nbk_noener[coult][nbat->comb_rule](nbl[nb], nbat,
+ ic,
+ shift_vec,
+ out->f,
+ fshift_p);
+ }
++ else if (out->nV == 1)
+ {
+ /* No energy groups */
+ out->Vvdw[0] = 0;
+ out->Vc[0] = 0;
+
+ p_nbk_ener[coult][nbat->comb_rule](nbl[nb], nbat,
+ ic,
+ shift_vec,
+ out->f,
+ fshift_p,
+ out->Vvdw,
+ out->Vc);
+ }
+ else
+ {
+ /* Calculate energy group contributions */
+ int i;
+
+ for (i = 0; i < out->nVS; i++)
+ {
+ out->VSvdw[i] = 0;
+ }
+ for (i = 0; i < out->nVS; i++)
+ {
+ out->VSc[i] = 0;
+ }
+
+ p_nbk_energrp[coult][nbat->comb_rule](nbl[nb], nbat,
+ ic,
+ shift_vec,
+ out->f,
+ fshift_p,
+ out->VSvdw,
+ out->VSc);
+
+ reduce_group_energies(nbat->nenergrp, nbat->neg_2log,
+ out->VSvdw, out->VSc,
+ out->Vvdw, out->Vc);
+ }
+ }
+
+ if (force_flags & GMX_FORCE_ENERGY)
+ {
+ reduce_energies_over_lists(nbat, nnbl, Vvdw, Vc);
+ }
+}
+#else
+{
+ gmx_incons("nbnxn_kernel_simd_2xnn called when such kernels "
+ " are not enabled.");
+}
+#endif
+#undef GMX_SIMD_J_UNROLL_SIZE
--- /dev/null
- /* With Ewald type electrostatics we the forces for excluded atom pairs
- * should not contribute to the virial sum. The exclusion forces
- * are not calculate in the energy kernels, but are in _noener.
- */
- if (!((force_flags & GMX_FORCE_ENERGY) ||
- (EEL_FULL(ic->eeltype) && (force_flags & GMX_FORCE_VIRIAL))))
+/*
+ * This file is part of the GROMACS molecular simulation package.
+ *
+ * Copyright (c) 2012,2013, by the GROMACS development team, led by
+ * David van der Spoel, Berk Hess, Erik Lindahl, and including many
+ * others, as listed in the AUTHORS file in the top-level source
+ * directory and at http://www.gromacs.org.
+ *
+ * GROMACS is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ *
+ * GROMACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with GROMACS; if not, see
+ * http://www.gnu.org/licenses, or write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * If you want to redistribute modifications to GROMACS, please
+ * consider that scientific software is very special. Version
+ * control is crucial - bugs must be traceable. We will be happy to
+ * consider code for inclusion in the official distribution, but
+ * derived work must not be called official GROMACS. Details are found
+ * in the README & COPYING files - if they are missing, get the
+ * official version at http://www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the research papers on the package. Check out http://www.gromacs.org.
+ */
+/*
+ * Note: this file was generated by the Verlet kernel generator for
+ * kernel type 4xn.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include "typedefs.h"
+
+#ifdef GMX_NBNXN_SIMD_4XN
+
+#ifdef GMX_NBNXN_HALF_WIDTH_SIMD
+#define GMX_USE_HALF_WIDTH_SIMD_HERE
+#endif
+
+#include "gmx_simd_macros.h"
+#include "gmx_simd_vec.h"
+#if !(GMX_SIMD_WIDTH_HERE == 2 || GMX_SIMD_WIDTH_HERE == 4 || GMX_SIMD_WIDTH_HERE == 8)
+#error "unsupported SIMD width"
+#endif
+
+#define GMX_SIMD_J_UNROLL_SIZE 1
+#include "nbnxn_kernel_simd_4xn.h"
+#include "../nbnxn_kernel_common.h"
+#include "gmx_omp_nthreads.h"
+#include "types/force_flags.h"
+
+/*! \brief Kinds of electrostatic treatments in SIMD Verlet kernels
+ */
+enum {
+ coultRF, coultTAB, coultTAB_TWIN, coultEWALD, coultEWALD_TWIN, coultNR
+};
+
+/* Declare and define the kernel function pointer lookup tables. */
+static p_nbk_func_ener p_nbk_ener[coultNR][ljcrNR] =
+{
+ {
+ nbnxn_kernel_simd_4xn_rf_comb_geom_ener,
+ nbnxn_kernel_simd_4xn_rf_comb_lb_ener,
+ nbnxn_kernel_simd_4xn_rf_comb_none_ener,
+ },
+ {
+ nbnxn_kernel_simd_4xn_tab_comb_geom_ener,
+ nbnxn_kernel_simd_4xn_tab_comb_lb_ener,
+ nbnxn_kernel_simd_4xn_tab_comb_none_ener,
+ },
+ {
+ nbnxn_kernel_simd_4xn_tab_twin_comb_geom_ener,
+ nbnxn_kernel_simd_4xn_tab_twin_comb_lb_ener,
+ nbnxn_kernel_simd_4xn_tab_twin_comb_none_ener,
+ },
+ {
+ nbnxn_kernel_simd_4xn_ewald_comb_geom_ener,
+ nbnxn_kernel_simd_4xn_ewald_comb_lb_ener,
+ nbnxn_kernel_simd_4xn_ewald_comb_none_ener,
+ },
+ {
+ nbnxn_kernel_simd_4xn_ewald_twin_comb_geom_ener,
+ nbnxn_kernel_simd_4xn_ewald_twin_comb_lb_ener,
+ nbnxn_kernel_simd_4xn_ewald_twin_comb_none_ener,
+ },
+};
+
+static p_nbk_func_ener p_nbk_energrp[coultNR][ljcrNR] =
+{
+ {
+ nbnxn_kernel_simd_4xn_rf_comb_geom_energrp,
+ nbnxn_kernel_simd_4xn_rf_comb_lb_energrp,
+ nbnxn_kernel_simd_4xn_rf_comb_none_energrp,
+ },
+ {
+ nbnxn_kernel_simd_4xn_tab_comb_geom_energrp,
+ nbnxn_kernel_simd_4xn_tab_comb_lb_energrp,
+ nbnxn_kernel_simd_4xn_tab_comb_none_energrp,
+ },
+ {
+ nbnxn_kernel_simd_4xn_tab_twin_comb_geom_energrp,
+ nbnxn_kernel_simd_4xn_tab_twin_comb_lb_energrp,
+ nbnxn_kernel_simd_4xn_tab_twin_comb_none_energrp,
+ },
+ {
+ nbnxn_kernel_simd_4xn_ewald_comb_geom_energrp,
+ nbnxn_kernel_simd_4xn_ewald_comb_lb_energrp,
+ nbnxn_kernel_simd_4xn_ewald_comb_none_energrp,
+ },
+ {
+ nbnxn_kernel_simd_4xn_ewald_twin_comb_geom_energrp,
+ nbnxn_kernel_simd_4xn_ewald_twin_comb_lb_energrp,
+ nbnxn_kernel_simd_4xn_ewald_twin_comb_none_energrp,
+ },
+};
+
+static p_nbk_func_noener p_nbk_noener[coultNR][ljcrNR] =
+{
+ {
+ nbnxn_kernel_simd_4xn_rf_comb_geom_noener,
+ nbnxn_kernel_simd_4xn_rf_comb_lb_noener,
+ nbnxn_kernel_simd_4xn_rf_comb_none_noener,
+ },
+ {
+ nbnxn_kernel_simd_4xn_tab_comb_geom_noener,
+ nbnxn_kernel_simd_4xn_tab_comb_lb_noener,
+ nbnxn_kernel_simd_4xn_tab_comb_none_noener,
+ },
+ {
+ nbnxn_kernel_simd_4xn_tab_twin_comb_geom_noener,
+ nbnxn_kernel_simd_4xn_tab_twin_comb_lb_noener,
+ nbnxn_kernel_simd_4xn_tab_twin_comb_none_noener,
+ },
+ {
+ nbnxn_kernel_simd_4xn_ewald_comb_geom_noener,
+ nbnxn_kernel_simd_4xn_ewald_comb_lb_noener,
+ nbnxn_kernel_simd_4xn_ewald_comb_none_noener,
+ },
+ {
+ nbnxn_kernel_simd_4xn_ewald_twin_comb_geom_noener,
+ nbnxn_kernel_simd_4xn_ewald_twin_comb_lb_noener,
+ nbnxn_kernel_simd_4xn_ewald_twin_comb_none_noener,
+ },
+};
+
+
+static void
+reduce_group_energies(int ng, int ng_2log,
+ const real *VSvdw, const real *VSc,
+ real *Vvdw, real *Vc)
+{
+ const int unrollj = GMX_SIMD_WIDTH_HERE/GMX_SIMD_J_UNROLL_SIZE;
+ const int unrollj_half = unrollj/2;
+ int ng_p2, i, j, j0, j1, c, s;
+
+ ng_p2 = (1<<ng_2log);
+
+ /* The size of the x86 SIMD energy group buffer array is:
+ * ng*ng*ng_p2*unrollj_half*simd_width
+ */
+ for (i = 0; i < ng; i++)
+ {
+ for (j = 0; j < ng; j++)
+ {
+ Vvdw[i*ng+j] = 0;
+ Vc[i*ng+j] = 0;
+ }
+
+ for (j1 = 0; j1 < ng; j1++)
+ {
+ for (j0 = 0; j0 < ng; j0++)
+ {
+ c = ((i*ng + j1)*ng_p2 + j0)*unrollj_half*unrollj;
+ for (s = 0; s < unrollj_half; s++)
+ {
+ Vvdw[i*ng+j0] += VSvdw[c+0];
+ Vvdw[i*ng+j1] += VSvdw[c+1];
+ Vc [i*ng+j0] += VSc [c+0];
+ Vc [i*ng+j1] += VSc [c+1];
+ c += unrollj + 2;
+ }
+ }
+ }
+ }
+}
+
+#else /* GMX_NBNXN_SIMD_4XN */
+
+#include "gmx_fatal.h"
+
+#endif /* GMX_NBNXN_SIMD_4XN */
+
+void
+nbnxn_kernel_simd_4xn(nbnxn_pairlist_set_t gmx_unused *nbl_list,
+ const nbnxn_atomdata_t gmx_unused *nbat,
+ const interaction_const_t gmx_unused *ic,
+ int gmx_unused ewald_excl,
+ rvec gmx_unused *shift_vec,
+ int gmx_unused force_flags,
+ int gmx_unused clearF,
+ real gmx_unused *fshift,
+ real gmx_unused *Vc,
+ real gmx_unused *Vvdw)
+#ifdef GMX_NBNXN_SIMD_4XN
+{
+ int nnbl;
+ nbnxn_pairlist_t **nbl;
+ int coult;
+ int nb;
+
+ nnbl = nbl_list->nnbl;
+ nbl = nbl_list->nbl;
+
+ if (EEL_RF(ic->eeltype) || ic->eeltype == eelCUT)
+ {
+ coult = coultRF;
+ }
+ else
+ {
+ if (ewald_excl == ewaldexclTable)
+ {
+ if (ic->rcoulomb == ic->rvdw)
+ {
+ coult = coultTAB;
+ }
+ else
+ {
+ coult = coultTAB_TWIN;
+ }
+ }
+ else
+ {
+ if (ic->rcoulomb == ic->rvdw)
+ {
+ coult = coultEWALD;
+ }
+ else
+ {
+ coult = coultEWALD_TWIN;
+ }
+ }
+ }
+
+#pragma omp parallel for schedule(static) num_threads(gmx_omp_nthreads_get(emntNonbonded))
+ for (nb = 0; nb < nnbl; nb++)
+ {
+ nbnxn_atomdata_output_t *out;
+ real *fshift_p;
+
+ out = &nbat->out[nb];
+
+ if (clearF == enbvClearFYes)
+ {
+ clear_f(nbat, nb, out->f);
+ }
+
+ if ((force_flags & GMX_FORCE_VIRIAL) && nnbl == 1)
+ {
+ fshift_p = fshift;
+ }
+ else
+ {
+ fshift_p = out->fshift;
+
+ if (clearF == enbvClearFYes)
+ {
+ clear_fshift(fshift_p);
+ }
+ }
+
- else if (out->nV == 1 || !(force_flags & GMX_FORCE_ENERGY))
++ if (!(force_flags & GMX_FORCE_ENERGY))
+ {
+ /* Don't calculate energies */
+ p_nbk_noener[coult][nbat->comb_rule](nbl[nb], nbat,
+ ic,
+ shift_vec,
+ out->f,
+ fshift_p);
+ }
++ else if (out->nV == 1)
+ {
+ /* No energy groups */
+ out->Vvdw[0] = 0;
+ out->Vc[0] = 0;
+
+ p_nbk_ener[coult][nbat->comb_rule](nbl[nb], nbat,
+ ic,
+ shift_vec,
+ out->f,
+ fshift_p,
+ out->Vvdw,
+ out->Vc);
+ }
+ else
+ {
+ /* Calculate energy group contributions */
+ int i;
+
+ for (i = 0; i < out->nVS; i++)
+ {
+ out->VSvdw[i] = 0;
+ }
+ for (i = 0; i < out->nVS; i++)
+ {
+ out->VSc[i] = 0;
+ }
+
+ p_nbk_energrp[coult][nbat->comb_rule](nbl[nb], nbat,
+ ic,
+ shift_vec,
+ out->f,
+ fshift_p,
+ out->VSvdw,
+ out->VSc);
+
+ reduce_group_energies(nbat->nenergrp, nbat->neg_2log,
+ out->VSvdw, out->VSc,
+ out->Vvdw, out->Vc);
+ }
+ }
+
+ if (force_flags & GMX_FORCE_ENERGY)
+ {
+ reduce_energies_over_lists(nbat, nnbl, Vvdw, Vc);
+ }
+}
+#else
+{
+ gmx_incons("nbnxn_kernel_simd_4xn called when such kernels "
+ " are not enabled.");
+}
+#endif
+#undef GMX_SIMD_J_UNROLL_SIZE
--- /dev/null
- if (zi < 0 || zi > n_per_h*SORT_GRID_OVERSIZE)
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2012, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <math.h>
+#include <string.h>
+#include "sysstuff.h"
+#include "smalloc.h"
+#include "macros.h"
+#include "maths.h"
+#include "vec.h"
+#include "pbc.h"
+#include "nbnxn_consts.h"
+/* nbnxn_internal.h included gmx_simd_macros.h */
+#include "nbnxn_internal.h"
+#ifdef GMX_NBNXN_SIMD
+#include "gmx_simd_vec.h"
+#endif
+#include "nbnxn_atomdata.h"
+#include "nbnxn_search.h"
+#include "gmx_omp_nthreads.h"
+#include "nrnb.h"
+
+#include "gromacs/fileio/gmxfio.h"
+
+#ifdef NBNXN_SEARCH_BB_SIMD4
+/* We use 4-wide SIMD for bounding box calculations */
+
+#ifndef GMX_DOUBLE
+/* Single precision BBs + coordinates, we can also load coordinates with SIMD */
+#define NBNXN_SEARCH_SIMD4_FLOAT_X_BB
+#endif
+
+#if defined NBNXN_SEARCH_SIMD4_FLOAT_X_BB && (GPU_NSUBCELL == 4 || GPU_NSUBCELL == 8)
+/* Store bounding boxes with x, y and z coordinates in packs of 4 */
+#define NBNXN_PBB_SIMD4
+#endif
+
+/* The packed bounding box coordinate stride is always set to 4.
+ * With AVX we could use 8, but that turns out not to be faster.
+ */
+#define STRIDE_PBB 4
+#define STRIDE_PBB_2LOG 2
+
+#endif /* NBNXN_SEARCH_BB_SIMD4 */
+
+#ifdef GMX_NBNXN_SIMD
+
+/* The functions below are macros as they are performance sensitive */
+
+/* 4x4 list, pack=4: no complex conversion required */
+/* i-cluster to j-cluster conversion */
+#define CI_TO_CJ_J4(ci) (ci)
+/* cluster index to coordinate array index conversion */
+#define X_IND_CI_J4(ci) ((ci)*STRIDE_P4)
+#define X_IND_CJ_J4(cj) ((cj)*STRIDE_P4)
+
+/* 4x2 list, pack=4: j-cluster size is half the packing width */
+/* i-cluster to j-cluster conversion */
+#define CI_TO_CJ_J2(ci) ((ci)<<1)
+/* cluster index to coordinate array index conversion */
+#define X_IND_CI_J2(ci) ((ci)*STRIDE_P4)
+#define X_IND_CJ_J2(cj) (((cj)>>1)*STRIDE_P4 + ((cj) & 1)*(PACK_X4>>1))
+
+/* 4x8 list, pack=8: i-cluster size is half the packing width */
+/* i-cluster to j-cluster conversion */
+#define CI_TO_CJ_J8(ci) ((ci)>>1)
+/* cluster index to coordinate array index conversion */
+#define X_IND_CI_J8(ci) (((ci)>>1)*STRIDE_P8 + ((ci) & 1)*(PACK_X8>>1))
+#define X_IND_CJ_J8(cj) ((cj)*STRIDE_P8)
+
+/* The j-cluster size is matched to the SIMD width */
+#if GMX_SIMD_WIDTH_HERE == 2
+#define CI_TO_CJ_SIMD_4XN(ci) CI_TO_CJ_J2(ci)
+#define X_IND_CI_SIMD_4XN(ci) X_IND_CI_J2(ci)
+#define X_IND_CJ_SIMD_4XN(cj) X_IND_CJ_J2(cj)
+#else
+#if GMX_SIMD_WIDTH_HERE == 4
+#define CI_TO_CJ_SIMD_4XN(ci) CI_TO_CJ_J4(ci)
+#define X_IND_CI_SIMD_4XN(ci) X_IND_CI_J4(ci)
+#define X_IND_CJ_SIMD_4XN(cj) X_IND_CJ_J4(cj)
+#else
+#if GMX_SIMD_WIDTH_HERE == 8
+#define CI_TO_CJ_SIMD_4XN(ci) CI_TO_CJ_J8(ci)
+#define X_IND_CI_SIMD_4XN(ci) X_IND_CI_J8(ci)
+#define X_IND_CJ_SIMD_4XN(cj) X_IND_CJ_J8(cj)
+/* Half SIMD with j-cluster size */
+#define CI_TO_CJ_SIMD_2XNN(ci) CI_TO_CJ_J4(ci)
+#define X_IND_CI_SIMD_2XNN(ci) X_IND_CI_J4(ci)
+#define X_IND_CJ_SIMD_2XNN(cj) X_IND_CJ_J4(cj)
+#else
+#if GMX_SIMD_WIDTH_HERE == 16
+#define CI_TO_CJ_SIMD_2XNN(ci) CI_TO_CJ_J8(ci)
+#define X_IND_CI_SIMD_2XNN(ci) X_IND_CI_J8(ci)
+#define X_IND_CJ_SIMD_2XNN(cj) X_IND_CJ_J8(cj)
+#else
+#error "unsupported GMX_NBNXN_SIMD_WIDTH"
+#endif
+#endif
+#endif
+#endif
+
+#endif /* GMX_NBNXN_SIMD */
+
+
+#ifdef NBNXN_SEARCH_BB_SIMD4
+/* Store bounding boxes corners as quadruplets: xxxxyyyyzzzz */
+#define NBNXN_BBXXXX
+/* Size of bounding box corners quadruplet */
+#define NNBSBB_XXXX (NNBSBB_D*DIM*STRIDE_PBB)
+#endif
+
+/* We shift the i-particles backward for PBC.
+ * This leads to more conditionals than shifting forward.
+ * We do this to get more balanced pair lists.
+ */
+#define NBNXN_SHIFT_BACKWARD
+
+
+/* This define is a lazy way to avoid interdependence of the grid
+ * and searching data structures.
+ */
+#define NBNXN_NA_SC_MAX (GPU_NSUBCELL*NBNXN_GPU_CLUSTER_SIZE)
+
+
+static void nbs_cycle_clear(nbnxn_cycle_t *cc)
+{
+ int i;
+
+ for (i = 0; i < enbsCCnr; i++)
+ {
+ cc[i].count = 0;
+ cc[i].c = 0;
+ }
+}
+
+static double Mcyc_av(const nbnxn_cycle_t *cc)
+{
+ return (double)cc->c*1e-6/cc->count;
+}
+
+static void nbs_cycle_print(FILE *fp, const nbnxn_search_t nbs)
+{
+ int n;
+ int t;
+
+ fprintf(fp, "\n");
+ fprintf(fp, "ns %4d grid %4.1f search %4.1f red.f %5.3f",
+ nbs->cc[enbsCCgrid].count,
+ Mcyc_av(&nbs->cc[enbsCCgrid]),
+ Mcyc_av(&nbs->cc[enbsCCsearch]),
+ Mcyc_av(&nbs->cc[enbsCCreducef]));
+
+ if (nbs->nthread_max > 1)
+ {
+ if (nbs->cc[enbsCCcombine].count > 0)
+ {
+ fprintf(fp, " comb %5.2f",
+ Mcyc_av(&nbs->cc[enbsCCcombine]));
+ }
+ fprintf(fp, " s. th");
+ for (t = 0; t < nbs->nthread_max; t++)
+ {
+ fprintf(fp, " %4.1f",
+ Mcyc_av(&nbs->work[t].cc[enbsCCsearch]));
+ }
+ }
+ fprintf(fp, "\n");
+}
+
+static void nbnxn_grid_init(nbnxn_grid_t * grid)
+{
+ grid->cxy_na = NULL;
+ grid->cxy_ind = NULL;
+ grid->cxy_nalloc = 0;
+ grid->bb = NULL;
+ grid->bbj = NULL;
+ grid->nc_nalloc = 0;
+}
+
+static int get_2log(int n)
+{
+ int log2;
+
+ log2 = 0;
+ while ((1<<log2) < n)
+ {
+ log2++;
+ }
+ if ((1<<log2) != n)
+ {
+ gmx_fatal(FARGS, "nbnxn na_c (%d) is not a power of 2", n);
+ }
+
+ return log2;
+}
+
+static int nbnxn_kernel_to_ci_size(int nb_kernel_type)
+{
+ switch (nb_kernel_type)
+ {
+ case nbnxnk4x4_PlainC:
+ case nbnxnk4xN_SIMD_4xN:
+ case nbnxnk4xN_SIMD_2xNN:
+ return NBNXN_CPU_CLUSTER_I_SIZE;
+ case nbnxnk8x8x8_CUDA:
+ case nbnxnk8x8x8_PlainC:
+ /* The cluster size for super/sub lists is only set here.
+ * Any value should work for the pair-search and atomdata code.
+ * The kernels, of course, might require a particular value.
+ */
+ return NBNXN_GPU_CLUSTER_SIZE;
+ default:
+ gmx_incons("unknown kernel type");
+ }
+
+ return 0;
+}
+
+int nbnxn_kernel_to_cj_size(int nb_kernel_type)
+{
+ int nbnxn_simd_width = 0;
+ int cj_size = 0;
+
+#ifdef GMX_NBNXN_SIMD
+ nbnxn_simd_width = GMX_SIMD_WIDTH_HERE;
+#endif
+
+ switch (nb_kernel_type)
+ {
+ case nbnxnk4x4_PlainC:
+ cj_size = NBNXN_CPU_CLUSTER_I_SIZE;
+ break;
+ case nbnxnk4xN_SIMD_4xN:
+ cj_size = nbnxn_simd_width;
+ break;
+ case nbnxnk4xN_SIMD_2xNN:
+ cj_size = nbnxn_simd_width/2;
+ break;
+ case nbnxnk8x8x8_CUDA:
+ case nbnxnk8x8x8_PlainC:
+ cj_size = nbnxn_kernel_to_ci_size(nb_kernel_type);
+ break;
+ default:
+ gmx_incons("unknown kernel type");
+ }
+
+ return cj_size;
+}
+
+static int ci_to_cj(int na_cj_2log, int ci)
+{
+ switch (na_cj_2log)
+ {
+ case 2: return ci; break;
+ case 1: return (ci<<1); break;
+ case 3: return (ci>>1); break;
+ }
+
+ return 0;
+}
+
+gmx_bool nbnxn_kernel_pairlist_simple(int nb_kernel_type)
+{
+ if (nb_kernel_type == nbnxnkNotSet)
+ {
+ gmx_fatal(FARGS, "Non-bonded kernel type not set for Verlet-style pair-list.");
+ }
+
+ switch (nb_kernel_type)
+ {
+ case nbnxnk8x8x8_CUDA:
+ case nbnxnk8x8x8_PlainC:
+ return FALSE;
+
+ case nbnxnk4x4_PlainC:
+ case nbnxnk4xN_SIMD_4xN:
+ case nbnxnk4xN_SIMD_2xNN:
+ return TRUE;
+
+ default:
+ gmx_incons("Invalid nonbonded kernel type passed!");
+ return FALSE;
+ }
+}
+
+void nbnxn_init_search(nbnxn_search_t * nbs_ptr,
+ ivec *n_dd_cells,
+ gmx_domdec_zones_t *zones,
+ int nthread_max)
+{
+ nbnxn_search_t nbs;
+ int d, g, t;
+
+ snew(nbs, 1);
+ *nbs_ptr = nbs;
+
+ nbs->DomDec = (n_dd_cells != NULL);
+
+ clear_ivec(nbs->dd_dim);
+ nbs->ngrid = 1;
+ if (nbs->DomDec)
+ {
+ nbs->zones = zones;
+
+ for (d = 0; d < DIM; d++)
+ {
+ if ((*n_dd_cells)[d] > 1)
+ {
+ nbs->dd_dim[d] = 1;
+ /* Each grid matches a DD zone */
+ nbs->ngrid *= 2;
+ }
+ }
+ }
+
+ snew(nbs->grid, nbs->ngrid);
+ for (g = 0; g < nbs->ngrid; g++)
+ {
+ nbnxn_grid_init(&nbs->grid[g]);
+ }
+ nbs->cell = NULL;
+ nbs->cell_nalloc = 0;
+ nbs->a = NULL;
+ nbs->a_nalloc = 0;
+
+ nbs->nthread_max = nthread_max;
+
+ /* Initialize the work data structures for each thread */
+ snew(nbs->work, nbs->nthread_max);
+ for (t = 0; t < nbs->nthread_max; t++)
+ {
+ nbs->work[t].cxy_na = NULL;
+ nbs->work[t].cxy_na_nalloc = 0;
+ nbs->work[t].sort_work = NULL;
+ nbs->work[t].sort_work_nalloc = 0;
+ }
+
+ /* Initialize detailed nbsearch cycle counting */
+ nbs->print_cycles = (getenv("GMX_NBNXN_CYCLE") != 0);
+ nbs->search_count = 0;
+ nbs_cycle_clear(nbs->cc);
+ for (t = 0; t < nbs->nthread_max; t++)
+ {
+ nbs_cycle_clear(nbs->work[t].cc);
+ }
+}
+
+static real grid_atom_density(int n, rvec corner0, rvec corner1)
+{
+ rvec size;
+
+ rvec_sub(corner1, corner0, size);
+
+ return n/(size[XX]*size[YY]*size[ZZ]);
+}
+
+static int set_grid_size_xy(const nbnxn_search_t nbs,
+ nbnxn_grid_t *grid,
+ int dd_zone,
+ int n, rvec corner0, rvec corner1,
+ real atom_density)
+{
+ rvec size;
+ int na_c;
+ real adens, tlen, tlen_x, tlen_y, nc_max;
+ int t;
+
+ rvec_sub(corner1, corner0, size);
+
+ if (n > grid->na_sc)
+ {
+ /* target cell length */
+ if (grid->bSimple)
+ {
+ /* To minimize the zero interactions, we should make
+ * the largest of the i/j cell cubic.
+ */
+ na_c = max(grid->na_c, grid->na_cj);
+
+ /* Approximately cubic cells */
+ tlen = pow(na_c/atom_density, 1.0/3.0);
+ tlen_x = tlen;
+ tlen_y = tlen;
+ }
+ else
+ {
+ /* Approximately cubic sub cells */
+ tlen = pow(grid->na_c/atom_density, 1.0/3.0);
+ tlen_x = tlen*GPU_NSUBCELL_X;
+ tlen_y = tlen*GPU_NSUBCELL_Y;
+ }
+ /* We round ncx and ncy down, because we get less cell pairs
+ * in the nbsist when the fixed cell dimensions (x,y) are
+ * larger than the variable one (z) than the other way around.
+ */
+ grid->ncx = max(1, (int)(size[XX]/tlen_x));
+ grid->ncy = max(1, (int)(size[YY]/tlen_y));
+ }
+ else
+ {
+ grid->ncx = 1;
+ grid->ncy = 1;
+ }
+
+ grid->sx = size[XX]/grid->ncx;
+ grid->sy = size[YY]/grid->ncy;
+ grid->inv_sx = 1/grid->sx;
+ grid->inv_sy = 1/grid->sy;
+
+ if (dd_zone > 0)
+ {
+ /* This is a non-home zone, add an extra row of cells
+ * for particles communicated for bonded interactions.
+ * These can be beyond the cut-off. It doesn't matter where
+ * they end up on the grid, but for performance it's better
+ * if they don't end up in cells that can be within cut-off range.
+ */
+ grid->ncx++;
+ grid->ncy++;
+ }
+
+ /* We need one additional cell entry for particles moved by DD */
+ if (grid->ncx*grid->ncy+1 > grid->cxy_nalloc)
+ {
+ grid->cxy_nalloc = over_alloc_large(grid->ncx*grid->ncy+1);
+ srenew(grid->cxy_na, grid->cxy_nalloc);
+ srenew(grid->cxy_ind, grid->cxy_nalloc+1);
+ }
+ for (t = 0; t < nbs->nthread_max; t++)
+ {
+ if (grid->ncx*grid->ncy+1 > nbs->work[t].cxy_na_nalloc)
+ {
+ nbs->work[t].cxy_na_nalloc = over_alloc_large(grid->ncx*grid->ncy+1);
+ srenew(nbs->work[t].cxy_na, nbs->work[t].cxy_na_nalloc);
+ }
+ }
+
+ /* Worst case scenario of 1 atom in each last cell */
+ if (grid->na_cj <= grid->na_c)
+ {
+ nc_max = n/grid->na_sc + grid->ncx*grid->ncy;
+ }
+ else
+ {
+ nc_max = n/grid->na_sc + grid->ncx*grid->ncy*grid->na_cj/grid->na_c;
+ }
+
+ if (nc_max > grid->nc_nalloc)
+ {
+ grid->nc_nalloc = over_alloc_large(nc_max);
+ srenew(grid->nsubc, grid->nc_nalloc);
+ srenew(grid->bbcz, grid->nc_nalloc*NNBSBB_D);
+
+ sfree_aligned(grid->bb);
+ /* This snew also zeros the contents, this avoid possible
+ * floating exceptions in SIMD with the unused bb elements.
+ */
+ if (grid->bSimple)
+ {
+ snew_aligned(grid->bb, grid->nc_nalloc, 16);
+ }
+ else
+ {
+#ifdef NBNXN_BBXXXX
+ int pbb_nalloc;
+
+ pbb_nalloc = grid->nc_nalloc*GPU_NSUBCELL/STRIDE_PBB*NNBSBB_XXXX;
+ snew_aligned(grid->pbb, pbb_nalloc, 16);
+#else
+ snew_aligned(grid->bb, grid->nc_nalloc*GPU_NSUBCELL, 16);
+#endif
+ }
+
+ if (grid->bSimple)
+ {
+ if (grid->na_cj == grid->na_c)
+ {
+ grid->bbj = grid->bb;
+ }
+ else
+ {
+ sfree_aligned(grid->bbj);
+ snew_aligned(grid->bbj, grid->nc_nalloc*grid->na_c/grid->na_cj, 16);
+ }
+ }
+
+ srenew(grid->flags, grid->nc_nalloc);
+ }
+
+ copy_rvec(corner0, grid->c0);
+ copy_rvec(corner1, grid->c1);
+
+ return nc_max;
+}
+
+/* We need to sort paricles in grid columns on z-coordinate.
+ * As particle are very often distributed homogeneously, we a sorting
+ * algorithm similar to pigeonhole sort. We multiply the z-coordinate
+ * by a factor, cast to an int and try to store in that hole. If the hole
+ * is full, we move this or another particle. A second pass is needed to make
+ * contiguous elements. SORT_GRID_OVERSIZE is the ratio of holes to particles.
+ * 4 is the optimal value for homogeneous particle distribution and allows
+ * for an O(#particles) sort up till distributions were all particles are
+ * concentrated in 1/4 of the space. No NlogN fallback is implemented,
+ * as it can be expensive to detect imhomogeneous particle distributions.
+ * SGSF is the maximum ratio of holes used, in the worst case all particles
+ * end up in the last hole and we need #particles extra holes at the end.
+ */
+#define SORT_GRID_OVERSIZE 4
+#define SGSF (SORT_GRID_OVERSIZE + 1)
+
+/* Sort particle index a on coordinates x along dim.
+ * Backwards tells if we want decreasing iso increasing coordinates.
+ * h0 is the minimum of the coordinate range.
+ * invh is the 1/length of the sorting range.
+ * n_per_h (>=n) is the expected average number of particles per 1/invh
+ * sort is the sorting work array.
+ * sort should have a size of at least n_per_h*SORT_GRID_OVERSIZE + n,
+ * or easier, allocate at least n*SGSF elements.
+ */
+static void sort_atoms(int dim, gmx_bool Backwards,
++ int dd_zone,
+ int *a, int n, rvec *x,
+ real h0, real invh, int n_per_h,
+ int *sort)
+{
+ int nsort, i, c;
+ int zi, zim, zi_min, zi_max;
+ int cp, tmp;
+
+ if (n <= 1)
+ {
+ /* Nothing to do */
+ return;
+ }
+
+#ifndef NDEBUG
+ if (n > n_per_h)
+ {
+ gmx_incons("n > n_per_h");
+ }
+#endif
+
+ /* Transform the inverse range height into the inverse hole height */
+ invh *= n_per_h*SORT_GRID_OVERSIZE;
+
+ /* Set nsort to the maximum possible number of holes used.
+ * In worst case all n elements end up in the last bin.
+ */
+ nsort = n_per_h*SORT_GRID_OVERSIZE + n;
+
+ /* Determine the index range used, so we can limit it for the second pass */
+ zi_min = INT_MAX;
+ zi_max = -1;
+
+ /* Sort the particles using a simple index sort */
+ for (i = 0; i < n; i++)
+ {
+ /* The cast takes care of float-point rounding effects below zero.
+ * This code assumes particles are less than 1/SORT_GRID_OVERSIZE
+ * times the box height out of the box.
+ */
+ zi = (int)((x[a[i]][dim] - h0)*invh);
+
+#ifndef NDEBUG
+ /* As we can have rounding effect, we use > iso >= here */
- sort_atoms(ZZ, FALSE,
++ if (zi < 0 || (dd_zone == 0 && zi > n_per_h*SORT_GRID_OVERSIZE))
+ {
+ gmx_fatal(FARGS, "(int)((x[%d][%c]=%f - %f)*%f) = %d, not in 0 - %d*%d\n",
+ a[i], 'x'+dim, x[a[i]][dim], h0, invh, zi,
+ n_per_h, SORT_GRID_OVERSIZE);
+ }
+#endif
+
++ /* In a non-local domain, particles communcated for bonded interactions
++ * can be far beyond the grid size, which is set by the non-bonded
++ * cut-off distance. We sort such particles into the last cell.
++ */
++ if (zi > n_per_h*SORT_GRID_OVERSIZE)
++ {
++ zi = n_per_h*SORT_GRID_OVERSIZE;
++ }
++
+ /* Ideally this particle should go in sort cell zi,
+ * but that might already be in use,
+ * in that case find the first empty cell higher up
+ */
+ if (sort[zi] < 0)
+ {
+ sort[zi] = a[i];
+ zi_min = min(zi_min, zi);
+ zi_max = max(zi_max, zi);
+ }
+ else
+ {
+ /* We have multiple atoms in the same sorting slot.
+ * Sort on real z for minimal bounding box size.
+ * There is an extra check for identical z to ensure
+ * well-defined output order, independent of input order
+ * to ensure binary reproducibility after restarts.
+ */
+ while (sort[zi] >= 0 && ( x[a[i]][dim] > x[sort[zi]][dim] ||
+ (x[a[i]][dim] == x[sort[zi]][dim] &&
+ a[i] > sort[zi])))
+ {
+ zi++;
+ }
+
+ if (sort[zi] >= 0)
+ {
+ /* Shift all elements by one slot until we find an empty slot */
+ cp = sort[zi];
+ zim = zi + 1;
+ while (sort[zim] >= 0)
+ {
+ tmp = sort[zim];
+ sort[zim] = cp;
+ cp = tmp;
+ zim++;
+ }
+ sort[zim] = cp;
+ zi_max = max(zi_max, zim);
+ }
+ sort[zi] = a[i];
+ zi_max = max(zi_max, zi);
+ }
+ }
+
+ c = 0;
+ if (!Backwards)
+ {
+ for (zi = 0; zi < nsort; zi++)
+ {
+ if (sort[zi] >= 0)
+ {
+ a[c++] = sort[zi];
+ sort[zi] = -1;
+ }
+ }
+ }
+ else
+ {
+ for (zi = zi_max; zi >= zi_min; zi--)
+ {
+ if (sort[zi] >= 0)
+ {
+ a[c++] = sort[zi];
+ sort[zi] = -1;
+ }
+ }
+ }
+ if (c < n)
+ {
+ gmx_incons("Lost particles while sorting");
+ }
+}
+
+#ifdef GMX_DOUBLE
+#define R2F_D(x) ((float)((x) >= 0 ? ((1-GMX_FLOAT_EPS)*(x)) : ((1+GMX_FLOAT_EPS)*(x))))
+#define R2F_U(x) ((float)((x) >= 0 ? ((1+GMX_FLOAT_EPS)*(x)) : ((1-GMX_FLOAT_EPS)*(x))))
+#else
+#define R2F_D(x) (x)
+#define R2F_U(x) (x)
+#endif
+
+/* Coordinate order x,y,z, bb order xyz0 */
+static void calc_bounding_box(int na, int stride, const real *x, nbnxn_bb_t *bb)
+{
+ int i, j;
+ real xl, xh, yl, yh, zl, zh;
+
+ i = 0;
+ xl = x[i+XX];
+ xh = x[i+XX];
+ yl = x[i+YY];
+ yh = x[i+YY];
+ zl = x[i+ZZ];
+ zh = x[i+ZZ];
+ i += stride;
+ for (j = 1; j < na; j++)
+ {
+ xl = min(xl, x[i+XX]);
+ xh = max(xh, x[i+XX]);
+ yl = min(yl, x[i+YY]);
+ yh = max(yh, x[i+YY]);
+ zl = min(zl, x[i+ZZ]);
+ zh = max(zh, x[i+ZZ]);
+ i += stride;
+ }
+ /* Note: possible double to float conversion here */
+ bb->lower[BB_X] = R2F_D(xl);
+ bb->lower[BB_Y] = R2F_D(yl);
+ bb->lower[BB_Z] = R2F_D(zl);
+ bb->upper[BB_X] = R2F_U(xh);
+ bb->upper[BB_Y] = R2F_U(yh);
+ bb->upper[BB_Z] = R2F_U(zh);
+}
+
+/* Packed coordinates, bb order xyz0 */
+static void calc_bounding_box_x_x4(int na, const real *x, nbnxn_bb_t *bb)
+{
+ int j;
+ real xl, xh, yl, yh, zl, zh;
+
+ xl = x[XX*PACK_X4];
+ xh = x[XX*PACK_X4];
+ yl = x[YY*PACK_X4];
+ yh = x[YY*PACK_X4];
+ zl = x[ZZ*PACK_X4];
+ zh = x[ZZ*PACK_X4];
+ for (j = 1; j < na; j++)
+ {
+ xl = min(xl, x[j+XX*PACK_X4]);
+ xh = max(xh, x[j+XX*PACK_X4]);
+ yl = min(yl, x[j+YY*PACK_X4]);
+ yh = max(yh, x[j+YY*PACK_X4]);
+ zl = min(zl, x[j+ZZ*PACK_X4]);
+ zh = max(zh, x[j+ZZ*PACK_X4]);
+ }
+ /* Note: possible double to float conversion here */
+ bb->lower[BB_X] = R2F_D(xl);
+ bb->lower[BB_Y] = R2F_D(yl);
+ bb->lower[BB_Z] = R2F_D(zl);
+ bb->upper[BB_X] = R2F_U(xh);
+ bb->upper[BB_Y] = R2F_U(yh);
+ bb->upper[BB_Z] = R2F_U(zh);
+}
+
+/* Packed coordinates, bb order xyz0 */
+static void calc_bounding_box_x_x8(int na, const real *x, nbnxn_bb_t *bb)
+{
+ int j;
+ real xl, xh, yl, yh, zl, zh;
+
+ xl = x[XX*PACK_X8];
+ xh = x[XX*PACK_X8];
+ yl = x[YY*PACK_X8];
+ yh = x[YY*PACK_X8];
+ zl = x[ZZ*PACK_X8];
+ zh = x[ZZ*PACK_X8];
+ for (j = 1; j < na; j++)
+ {
+ xl = min(xl, x[j+XX*PACK_X8]);
+ xh = max(xh, x[j+XX*PACK_X8]);
+ yl = min(yl, x[j+YY*PACK_X8]);
+ yh = max(yh, x[j+YY*PACK_X8]);
+ zl = min(zl, x[j+ZZ*PACK_X8]);
+ zh = max(zh, x[j+ZZ*PACK_X8]);
+ }
+ /* Note: possible double to float conversion here */
+ bb->lower[BB_X] = R2F_D(xl);
+ bb->lower[BB_Y] = R2F_D(yl);
+ bb->lower[BB_Z] = R2F_D(zl);
+ bb->upper[BB_X] = R2F_U(xh);
+ bb->upper[BB_Y] = R2F_U(yh);
+ bb->upper[BB_Z] = R2F_U(zh);
+}
+
+/* Packed coordinates, bb order xyz0 */
+static void calc_bounding_box_x_x4_halves(int na, const real *x,
+ nbnxn_bb_t *bb, nbnxn_bb_t *bbj)
+{
+ calc_bounding_box_x_x4(min(na, 2), x, bbj);
+
+ if (na > 2)
+ {
+ calc_bounding_box_x_x4(min(na-2, 2), x+(PACK_X4>>1), bbj+1);
+ }
+ else
+ {
+ /* Set the "empty" bounding box to the same as the first one,
+ * so we don't need to treat special cases in the rest of the code.
+ */
+#ifdef NBNXN_SEARCH_BB_SIMD4
+ gmx_simd4_store_pr(&bbj[1].lower[0], gmx_simd4_load_bb_pr(&bbj[0].lower[0]));
+ gmx_simd4_store_pr(&bbj[1].upper[0], gmx_simd4_load_bb_pr(&bbj[0].upper[0]));
+#else
+ bbj[1] = bbj[0];
+#endif
+ }
+
+#ifdef NBNXN_SEARCH_BB_SIMD4
+ gmx_simd4_store_pr(&bb->lower[0],
+ gmx_simd4_min_pr(gmx_simd4_load_bb_pr(&bbj[0].lower[0]),
+ gmx_simd4_load_bb_pr(&bbj[1].lower[0])));
+ gmx_simd4_store_pr(&bb->upper[0],
+ gmx_simd4_max_pr(gmx_simd4_load_bb_pr(&bbj[0].upper[0]),
+ gmx_simd4_load_bb_pr(&bbj[1].upper[0])));
+#else
+ {
+ int i;
+
+ for (i = 0; i < NNBSBB_C; i++)
+ {
+ bb->lower[i] = min(bbj[0].lower[i], bbj[1].lower[i]);
+ bb->upper[i] = max(bbj[0].upper[i], bbj[1].upper[i]);
+ }
+ }
+#endif
+}
+
+#ifdef NBNXN_SEARCH_BB_SIMD4
+
+/* Coordinate order xyz, bb order xxxxyyyyzzzz */
+static void calc_bounding_box_xxxx(int na, int stride, const real *x, float *bb)
+{
+ int i, j;
+ real xl, xh, yl, yh, zl, zh;
+
+ i = 0;
+ xl = x[i+XX];
+ xh = x[i+XX];
+ yl = x[i+YY];
+ yh = x[i+YY];
+ zl = x[i+ZZ];
+ zh = x[i+ZZ];
+ i += stride;
+ for (j = 1; j < na; j++)
+ {
+ xl = min(xl, x[i+XX]);
+ xh = max(xh, x[i+XX]);
+ yl = min(yl, x[i+YY]);
+ yh = max(yh, x[i+YY]);
+ zl = min(zl, x[i+ZZ]);
+ zh = max(zh, x[i+ZZ]);
+ i += stride;
+ }
+ /* Note: possible double to float conversion here */
+ bb[0*STRIDE_PBB] = R2F_D(xl);
+ bb[1*STRIDE_PBB] = R2F_D(yl);
+ bb[2*STRIDE_PBB] = R2F_D(zl);
+ bb[3*STRIDE_PBB] = R2F_U(xh);
+ bb[4*STRIDE_PBB] = R2F_U(yh);
+ bb[5*STRIDE_PBB] = R2F_U(zh);
+}
+
+#endif /* NBNXN_SEARCH_BB_SIMD4 */
+
+#ifdef NBNXN_SEARCH_SIMD4_FLOAT_X_BB
+
+/* Coordinate order xyz?, bb order xyz0 */
+static void calc_bounding_box_simd4(int na, const float *x, nbnxn_bb_t *bb)
+{
+ gmx_simd4_pr bb_0_S, bb_1_S;
+ gmx_simd4_pr x_S;
+
+ int i;
+
+ bb_0_S = gmx_simd4_load_bb_pr(x);
+ bb_1_S = bb_0_S;
+
+ for (i = 1; i < na; i++)
+ {
+ x_S = gmx_simd4_load_bb_pr(x+i*NNBSBB_C);
+ bb_0_S = gmx_simd4_min_pr(bb_0_S, x_S);
+ bb_1_S = gmx_simd4_max_pr(bb_1_S, x_S);
+ }
+
+ gmx_simd4_store_pr(&bb->lower[0], bb_0_S);
+ gmx_simd4_store_pr(&bb->upper[0], bb_1_S);
+}
+
+/* Coordinate order xyz?, bb order xxxxyyyyzzzz */
+static void calc_bounding_box_xxxx_simd4(int na, const float *x,
+ nbnxn_bb_t *bb_work_aligned,
+ real *bb)
+{
+ calc_bounding_box_simd4(na, x, bb_work_aligned);
+
+ bb[0*STRIDE_PBB] = bb_work_aligned->lower[BB_X];
+ bb[1*STRIDE_PBB] = bb_work_aligned->lower[BB_Y];
+ bb[2*STRIDE_PBB] = bb_work_aligned->lower[BB_Z];
+ bb[3*STRIDE_PBB] = bb_work_aligned->upper[BB_X];
+ bb[4*STRIDE_PBB] = bb_work_aligned->upper[BB_Y];
+ bb[5*STRIDE_PBB] = bb_work_aligned->upper[BB_Z];
+}
+
+#endif /* NBNXN_SEARCH_SIMD4_FLOAT_X_BB */
+
+
+/* Combines pairs of consecutive bounding boxes */
+static void combine_bounding_box_pairs(nbnxn_grid_t *grid, const nbnxn_bb_t *bb)
+{
+ int i, j, sc2, nc2, c2;
+
+ for (i = 0; i < grid->ncx*grid->ncy; i++)
+ {
+ /* Starting bb in a column is expected to be 2-aligned */
+ sc2 = grid->cxy_ind[i]>>1;
+ /* For odd numbers skip the last bb here */
+ nc2 = (grid->cxy_na[i]+3)>>(2+1);
+ for (c2 = sc2; c2 < sc2+nc2; c2++)
+ {
+#ifdef NBNXN_SEARCH_BB_SIMD4
+ gmx_simd4_pr min_S, max_S;
+
+ min_S = gmx_simd4_min_pr(gmx_simd4_load_bb_pr(&bb[c2*2+0].lower[0]),
+ gmx_simd4_load_bb_pr(&bb[c2*2+1].lower[0]));
+ max_S = gmx_simd4_max_pr(gmx_simd4_load_bb_pr(&bb[c2*2+0].upper[0]),
+ gmx_simd4_load_bb_pr(&bb[c2*2+1].upper[0]));
+ gmx_simd4_store_pr(&grid->bbj[c2].lower[0], min_S);
+ gmx_simd4_store_pr(&grid->bbj[c2].upper[0], max_S);
+#else
+ for (j = 0; j < NNBSBB_C; j++)
+ {
+ grid->bbj[c2].lower[j] = min(bb[c2*2+0].lower[j],
+ bb[c2*2+1].lower[j]);
+ grid->bbj[c2].upper[j] = max(bb[c2*2+0].upper[j],
+ bb[c2*2+1].upper[j]);
+ }
+#endif
+ }
+ if (((grid->cxy_na[i]+3)>>2) & 1)
+ {
+ /* The bb count in this column is odd: duplicate the last bb */
+ for (j = 0; j < NNBSBB_C; j++)
+ {
+ grid->bbj[c2].lower[j] = bb[c2*2].lower[j];
+ grid->bbj[c2].upper[j] = bb[c2*2].upper[j];
+ }
+ }
+ }
+}
+
+
+/* Prints the average bb size, used for debug output */
+static void print_bbsizes_simple(FILE *fp,
+ const nbnxn_search_t nbs,
+ const nbnxn_grid_t *grid)
+{
+ int c, d;
+ dvec ba;
+
+ clear_dvec(ba);
+ for (c = 0; c < grid->nc; c++)
+ {
+ for (d = 0; d < DIM; d++)
+ {
+ ba[d] += grid->bb[c].upper[d] - grid->bb[c].lower[d];
+ }
+ }
+ dsvmul(1.0/grid->nc, ba, ba);
+
+ fprintf(fp, "ns bb: %4.2f %4.2f %4.2f %4.2f %4.2f %4.2f rel %4.2f %4.2f %4.2f\n",
+ nbs->box[XX][XX]/grid->ncx,
+ nbs->box[YY][YY]/grid->ncy,
+ nbs->box[ZZ][ZZ]*grid->ncx*grid->ncy/grid->nc,
+ ba[XX], ba[YY], ba[ZZ],
+ ba[XX]*grid->ncx/nbs->box[XX][XX],
+ ba[YY]*grid->ncy/nbs->box[YY][YY],
+ ba[ZZ]*grid->nc/(grid->ncx*grid->ncy*nbs->box[ZZ][ZZ]));
+}
+
+/* Prints the average bb size, used for debug output */
+static void print_bbsizes_supersub(FILE *fp,
+ const nbnxn_search_t nbs,
+ const nbnxn_grid_t *grid)
+{
+ int ns, c, s;
+ dvec ba;
+
+ clear_dvec(ba);
+ ns = 0;
+ for (c = 0; c < grid->nc; c++)
+ {
+#ifdef NBNXN_BBXXXX
+ for (s = 0; s < grid->nsubc[c]; s += STRIDE_PBB)
+ {
+ int cs_w, i, d;
+
+ cs_w = (c*GPU_NSUBCELL + s)/STRIDE_PBB;
+ for (i = 0; i < STRIDE_PBB; i++)
+ {
+ for (d = 0; d < DIM; d++)
+ {
+ ba[d] +=
+ grid->pbb[cs_w*NNBSBB_XXXX+(DIM+d)*STRIDE_PBB+i] -
+ grid->pbb[cs_w*NNBSBB_XXXX+ d *STRIDE_PBB+i];
+ }
+ }
+ }
+#else
+ for (s = 0; s < grid->nsubc[c]; s++)
+ {
+ int cs, d;
+
+ cs = c*GPU_NSUBCELL + s;
+ for (d = 0; d < DIM; d++)
+ {
+ ba[d] += grid->bb[cs].upper[d] - grid->bb[cs].lower[d];
+ }
+ }
+#endif
+ ns += grid->nsubc[c];
+ }
+ dsvmul(1.0/ns, ba, ba);
+
+ fprintf(fp, "ns bb: %4.2f %4.2f %4.2f %4.2f %4.2f %4.2f rel %4.2f %4.2f %4.2f\n",
+ nbs->box[XX][XX]/(grid->ncx*GPU_NSUBCELL_X),
+ nbs->box[YY][YY]/(grid->ncy*GPU_NSUBCELL_Y),
+ nbs->box[ZZ][ZZ]*grid->ncx*grid->ncy/(grid->nc*GPU_NSUBCELL_Z),
+ ba[XX], ba[YY], ba[ZZ],
+ ba[XX]*grid->ncx*GPU_NSUBCELL_X/nbs->box[XX][XX],
+ ba[YY]*grid->ncy*GPU_NSUBCELL_Y/nbs->box[YY][YY],
+ ba[ZZ]*grid->nc*GPU_NSUBCELL_Z/(grid->ncx*grid->ncy*nbs->box[ZZ][ZZ]));
+}
+
+/* Potentially sorts atoms on LJ coefficients !=0 and ==0.
+ * Also sets interaction flags.
+ */
+void sort_on_lj(int na_c,
+ int a0, int a1, const int *atinfo,
+ int *order,
+ int *flags)
+{
+ int subc, s, a, n1, n2, a_lj_max, i, j;
+ int sort1[NBNXN_NA_SC_MAX/GPU_NSUBCELL];
+ int sort2[NBNXN_NA_SC_MAX/GPU_NSUBCELL];
+ gmx_bool haveQ;
+
+ *flags = 0;
+
+ subc = 0;
+ for (s = a0; s < a1; s += na_c)
+ {
+ /* Make lists for this (sub-)cell on atoms with and without LJ */
+ n1 = 0;
+ n2 = 0;
+ haveQ = FALSE;
+ a_lj_max = -1;
+ for (a = s; a < min(s+na_c, a1); a++)
+ {
+ haveQ = haveQ || GET_CGINFO_HAS_Q(atinfo[order[a]]);
+
+ if (GET_CGINFO_HAS_VDW(atinfo[order[a]]))
+ {
+ sort1[n1++] = order[a];
+ a_lj_max = a;
+ }
+ else
+ {
+ sort2[n2++] = order[a];
+ }
+ }
+
+ /* If we don't have atom with LJ, there's nothing to sort */
+ if (n1 > 0)
+ {
+ *flags |= NBNXN_CI_DO_LJ(subc);
+
+ if (2*n1 <= na_c)
+ {
+ /* Only sort when strictly necessary. Ordering particles
+ * Ordering particles can lead to less accurate summation
+ * due to rounding, both for LJ and Coulomb interactions.
+ */
+ if (2*(a_lj_max - s) >= na_c)
+ {
+ for (i = 0; i < n1; i++)
+ {
+ order[a0+i] = sort1[i];
+ }
+ for (j = 0; j < n2; j++)
+ {
+ order[a0+n1+j] = sort2[j];
+ }
+ }
+
+ *flags |= NBNXN_CI_HALF_LJ(subc);
+ }
+ }
+ if (haveQ)
+ {
+ *flags |= NBNXN_CI_DO_COUL(subc);
+ }
+ subc++;
+ }
+}
+
+/* Fill a pair search cell with atoms.
+ * Potentially sorts atoms and sets the interaction flags.
+ */
+void fill_cell(const nbnxn_search_t nbs,
+ nbnxn_grid_t *grid,
+ nbnxn_atomdata_t *nbat,
+ int a0, int a1,
+ const int *atinfo,
+ rvec *x,
+ int sx, int sy, int sz,
+ nbnxn_bb_t gmx_unused *bb_work_aligned)
+{
+ int na, a;
+ size_t offset;
+ nbnxn_bb_t *bb_ptr;
+#ifdef NBNXN_BBXXXX
+ float *pbb_ptr;
+#endif
+
+ na = a1 - a0;
+
+ if (grid->bSimple)
+ {
+ sort_on_lj(grid->na_c, a0, a1, atinfo, nbs->a,
+ grid->flags+(a0>>grid->na_c_2log)-grid->cell0);
+ }
+
+ /* Now we have sorted the atoms, set the cell indices */
+ for (a = a0; a < a1; a++)
+ {
+ nbs->cell[nbs->a[a]] = a;
+ }
+
+ copy_rvec_to_nbat_real(nbs->a+a0, a1-a0, grid->na_c, x,
+ nbat->XFormat, nbat->x, a0,
+ sx, sy, sz);
+
+ if (nbat->XFormat == nbatX4)
+ {
+ /* Store the bounding boxes as xyz.xyz. */
+ offset = (a0 - grid->cell0*grid->na_sc) >> grid->na_c_2log;
+ bb_ptr = grid->bb + offset;
+
+#if defined GMX_NBNXN_SIMD && GMX_SIMD_WIDTH_HERE == 2
+ if (2*grid->na_cj == grid->na_c)
+ {
+ calc_bounding_box_x_x4_halves(na, nbat->x+X4_IND_A(a0), bb_ptr,
+ grid->bbj+offset*2);
+ }
+ else
+#endif
+ {
+ calc_bounding_box_x_x4(na, nbat->x+X4_IND_A(a0), bb_ptr);
+ }
+ }
+ else if (nbat->XFormat == nbatX8)
+ {
+ /* Store the bounding boxes as xyz.xyz. */
+ offset = (a0 - grid->cell0*grid->na_sc) >> grid->na_c_2log;
+ bb_ptr = grid->bb + offset;
+
+ calc_bounding_box_x_x8(na, nbat->x+X8_IND_A(a0), bb_ptr);
+ }
+#ifdef NBNXN_BBXXXX
+ else if (!grid->bSimple)
+ {
+ /* Store the bounding boxes in a format convenient
+ * for SIMD4 calculations: xxxxyyyyzzzz...
+ */
+ pbb_ptr =
+ grid->pbb +
+ ((a0-grid->cell0*grid->na_sc)>>(grid->na_c_2log+STRIDE_PBB_2LOG))*NNBSBB_XXXX +
+ (((a0-grid->cell0*grid->na_sc)>>grid->na_c_2log) & (STRIDE_PBB-1));
+
+#ifdef NBNXN_SEARCH_SIMD4_FLOAT_X_BB
+ if (nbat->XFormat == nbatXYZQ)
+ {
+ calc_bounding_box_xxxx_simd4(na, nbat->x+a0*nbat->xstride,
+ bb_work_aligned, pbb_ptr);
+ }
+ else
+#endif
+ {
+ calc_bounding_box_xxxx(na, nbat->xstride, nbat->x+a0*nbat->xstride,
+ pbb_ptr);
+ }
+ if (gmx_debug_at)
+ {
+ fprintf(debug, "%2d %2d %2d bb %5.2f %5.2f %5.2f %5.2f %5.2f %5.2f\n",
+ sx, sy, sz,
+ pbb_ptr[0*STRIDE_PBB], pbb_ptr[3*STRIDE_PBB],
+ pbb_ptr[1*STRIDE_PBB], pbb_ptr[4*STRIDE_PBB],
+ pbb_ptr[2*STRIDE_PBB], pbb_ptr[5*STRIDE_PBB]);
+ }
+ }
+#endif
+ else
+ {
+ /* Store the bounding boxes as xyz.xyz. */
+ bb_ptr = grid->bb+((a0-grid->cell0*grid->na_sc)>>grid->na_c_2log);
+
+ calc_bounding_box(na, nbat->xstride, nbat->x+a0*nbat->xstride,
+ bb_ptr);
+
+ if (gmx_debug_at)
+ {
+ int bbo;
+ bbo = (a0 - grid->cell0*grid->na_sc)/grid->na_c;
+ fprintf(debug, "%2d %2d %2d bb %5.2f %5.2f %5.2f %5.2f %5.2f %5.2f\n",
+ sx, sy, sz,
+ grid->bb[bbo].lower[BB_X],
+ grid->bb[bbo].lower[BB_Y],
+ grid->bb[bbo].lower[BB_Z],
+ grid->bb[bbo].upper[BB_X],
+ grid->bb[bbo].upper[BB_Y],
+ grid->bb[bbo].upper[BB_Z]);
+ }
+ }
+}
+
+/* Spatially sort the atoms within one grid column */
+static void sort_columns_simple(const nbnxn_search_t nbs,
+ int dd_zone,
+ nbnxn_grid_t *grid,
+ int a0, int a1,
+ const int *atinfo,
+ rvec *x,
+ nbnxn_atomdata_t *nbat,
+ int cxy_start, int cxy_end,
+ int *sort_work)
+{
+ int cxy;
+ int cx, cy, cz, ncz, cfilled, c;
+ int na, ash, ind, a;
+ int na_c, ash_c;
+
+ if (debug)
+ {
+ fprintf(debug, "cell0 %d sorting columns %d - %d, atoms %d - %d\n",
+ grid->cell0, cxy_start, cxy_end, a0, a1);
+ }
+
+ /* Sort the atoms within each x,y column in 3 dimensions */
+ for (cxy = cxy_start; cxy < cxy_end; cxy++)
+ {
+ cx = cxy/grid->ncy;
+ cy = cxy - cx*grid->ncy;
+
+ na = grid->cxy_na[cxy];
+ ncz = grid->cxy_ind[cxy+1] - grid->cxy_ind[cxy];
+ ash = (grid->cell0 + grid->cxy_ind[cxy])*grid->na_sc;
+
+ /* Sort the atoms within each x,y column on z coordinate */
- sort_atoms(ZZ, FALSE,
++ sort_atoms(ZZ, FALSE, dd_zone,
+ nbs->a+ash, na, x,
+ grid->c0[ZZ],
+ 1.0/nbs->box[ZZ][ZZ], ncz*grid->na_sc,
+ sort_work);
+
+ /* Fill the ncz cells in this column */
+ cfilled = grid->cxy_ind[cxy];
+ for (cz = 0; cz < ncz; cz++)
+ {
+ c = grid->cxy_ind[cxy] + cz;
+
+ ash_c = ash + cz*grid->na_sc;
+ na_c = min(grid->na_sc, na-(ash_c-ash));
+
+ fill_cell(nbs, grid, nbat,
+ ash_c, ash_c+na_c, atinfo, x,
+ grid->na_sc*cx + (dd_zone >> 2),
+ grid->na_sc*cy + (dd_zone & 3),
+ grid->na_sc*cz,
+ NULL);
+
+ /* This copy to bbcz is not really necessary.
+ * But it allows to use the same grid search code
+ * for the simple and supersub cell setups.
+ */
+ if (na_c > 0)
+ {
+ cfilled = c;
+ }
+ grid->bbcz[c*NNBSBB_D ] = grid->bb[cfilled].lower[BB_Z];
+ grid->bbcz[c*NNBSBB_D+1] = grid->bb[cfilled].upper[BB_Z];
+ }
+
+ /* Set the unused atom indices to -1 */
+ for (ind = na; ind < ncz*grid->na_sc; ind++)
+ {
+ nbs->a[ash+ind] = -1;
+ }
+ }
+}
+
+/* Spatially sort the atoms within one grid column */
+static void sort_columns_supersub(const nbnxn_search_t nbs,
+ int dd_zone,
+ nbnxn_grid_t *grid,
+ int a0, int a1,
+ const int *atinfo,
+ rvec *x,
+ nbnxn_atomdata_t *nbat,
+ int cxy_start, int cxy_end,
+ int *sort_work)
+{
+ int cxy;
+ int cx, cy, cz = -1, c = -1, ncz;
+ int na, ash, na_c, ind, a;
+ int subdiv_z, sub_z, na_z, ash_z;
+ int subdiv_y, sub_y, na_y, ash_y;
+ int subdiv_x, sub_x, na_x, ash_x;
+
+ /* cppcheck-suppress unassignedVariable */
+ nbnxn_bb_t bb_work_array[2], *bb_work_aligned;
+
+ bb_work_aligned = (nbnxn_bb_t *)(((size_t)(bb_work_array+1)) & (~((size_t)15)));
+
+ if (debug)
+ {
+ fprintf(debug, "cell0 %d sorting columns %d - %d, atoms %d - %d\n",
+ grid->cell0, cxy_start, cxy_end, a0, a1);
+ }
+
+ subdiv_x = grid->na_c;
+ subdiv_y = GPU_NSUBCELL_X*subdiv_x;
+ subdiv_z = GPU_NSUBCELL_Y*subdiv_y;
+
+ /* Sort the atoms within each x,y column in 3 dimensions */
+ for (cxy = cxy_start; cxy < cxy_end; cxy++)
+ {
+ cx = cxy/grid->ncy;
+ cy = cxy - cx*grid->ncy;
+
+ na = grid->cxy_na[cxy];
+ ncz = grid->cxy_ind[cxy+1] - grid->cxy_ind[cxy];
+ ash = (grid->cell0 + grid->cxy_ind[cxy])*grid->na_sc;
+
+ /* Sort the atoms within each x,y column on z coordinate */
- sort_atoms(YY, (sub_z & 1),
++ sort_atoms(ZZ, FALSE, dd_zone,
+ nbs->a+ash, na, x,
+ grid->c0[ZZ],
+ 1.0/nbs->box[ZZ][ZZ], ncz*grid->na_sc,
+ sort_work);
+
+ /* This loop goes over the supercells and subcells along z at once */
+ for (sub_z = 0; sub_z < ncz*GPU_NSUBCELL_Z; sub_z++)
+ {
+ ash_z = ash + sub_z*subdiv_z;
+ na_z = min(subdiv_z, na-(ash_z-ash));
+
+ /* We have already sorted on z */
+
+ if (sub_z % GPU_NSUBCELL_Z == 0)
+ {
+ cz = sub_z/GPU_NSUBCELL_Z;
+ c = grid->cxy_ind[cxy] + cz;
+
+ /* The number of atoms in this supercell */
+ na_c = min(grid->na_sc, na-(ash_z-ash));
+
+ grid->nsubc[c] = min(GPU_NSUBCELL, (na_c+grid->na_c-1)/grid->na_c);
+
+ /* Store the z-boundaries of the super cell */
+ grid->bbcz[c*NNBSBB_D ] = x[nbs->a[ash_z]][ZZ];
+ grid->bbcz[c*NNBSBB_D+1] = x[nbs->a[ash_z+na_c-1]][ZZ];
+ }
+
+#if GPU_NSUBCELL_Y > 1
+ /* Sort the atoms along y */
- sort_atoms(XX, ((cz*GPU_NSUBCELL_Y + sub_y) & 1),
++ sort_atoms(YY, (sub_z & 1), dd_zone,
+ nbs->a+ash_z, na_z, x,
+ grid->c0[YY]+cy*grid->sy,
+ grid->inv_sy, subdiv_z,
+ sort_work);
+#endif
+
+ for (sub_y = 0; sub_y < GPU_NSUBCELL_Y; sub_y++)
+ {
+ ash_y = ash_z + sub_y*subdiv_y;
+ na_y = min(subdiv_y, na-(ash_y-ash));
+
+#if GPU_NSUBCELL_X > 1
+ /* Sort the atoms along x */
++ sort_atoms(XX, ((cz*GPU_NSUBCELL_Y + sub_y) & 1), dd_zone,
+ nbs->a+ash_y, na_y, x,
+ grid->c0[XX]+cx*grid->sx,
+ grid->inv_sx, subdiv_y,
+ sort_work);
+#endif
+
+ for (sub_x = 0; sub_x < GPU_NSUBCELL_X; sub_x++)
+ {
+ ash_x = ash_y + sub_x*subdiv_x;
+ na_x = min(subdiv_x, na-(ash_x-ash));
+
+ fill_cell(nbs, grid, nbat,
+ ash_x, ash_x+na_x, atinfo, x,
+ grid->na_c*(cx*GPU_NSUBCELL_X+sub_x) + (dd_zone >> 2),
+ grid->na_c*(cy*GPU_NSUBCELL_Y+sub_y) + (dd_zone & 3),
+ grid->na_c*sub_z,
+ bb_work_aligned);
+ }
+ }
+ }
+
+ /* Set the unused atom indices to -1 */
+ for (ind = na; ind < ncz*grid->na_sc; ind++)
+ {
+ nbs->a[ash+ind] = -1;
+ }
+ }
+}
+
+/* Determine in which grid column atoms should go */
+static void calc_column_indices(nbnxn_grid_t *grid,
+ int a0, int a1,
+ rvec *x,
+ int dd_zone, const int *move,
+ int thread, int nthread,
+ int *cell,
+ int *cxy_na)
+{
+ int n0, n1, i;
+ int cx, cy;
+
+ /* We add one extra cell for particles which moved during DD */
+ for (i = 0; i < grid->ncx*grid->ncy+1; i++)
+ {
+ cxy_na[i] = 0;
+ }
+
+ n0 = a0 + (int)((thread+0)*(a1 - a0))/nthread;
+ n1 = a0 + (int)((thread+1)*(a1 - a0))/nthread;
+ if (dd_zone == 0)
+ {
+ /* Home zone */
+ for (i = n0; i < n1; i++)
+ {
+ if (move == NULL || move[i] >= 0)
+ {
+ /* We need to be careful with rounding,
+ * particles might be a few bits outside the local zone.
+ * The int cast takes care of the lower bound,
+ * we will explicitly take care of the upper bound.
+ */
+ cx = (int)((x[i][XX] - grid->c0[XX])*grid->inv_sx);
+ cy = (int)((x[i][YY] - grid->c0[YY])*grid->inv_sy);
+
+#ifndef NDEBUG
+ if (cx < 0 || cx > grid->ncx ||
+ cy < 0 || cy > grid->ncy)
+ {
+ gmx_fatal(FARGS,
+ "grid cell cx %d cy %d out of range (max %d %d)\n"
+ "atom %f %f %f, grid->c0 %f %f",
+ cx, cy, grid->ncx, grid->ncy,
+ x[i][XX], x[i][YY], x[i][ZZ], grid->c0[XX], grid->c0[YY]);
+ }
+#endif
+ /* Take care of potential rouding issues */
+ cx = min(cx, grid->ncx - 1);
+ cy = min(cy, grid->ncy - 1);
+
+ /* For the moment cell will contain only the, grid local,
+ * x and y indices, not z.
+ */
+ cell[i] = cx*grid->ncy + cy;
+ }
+ else
+ {
+ /* Put this moved particle after the end of the grid,
+ * so we can process it later without using conditionals.
+ */
+ cell[i] = grid->ncx*grid->ncy;
+ }
+
+ cxy_na[cell[i]]++;
+ }
+ }
+ else
+ {
+ /* Non-home zone */
+ for (i = n0; i < n1; i++)
+ {
+ cx = (int)((x[i][XX] - grid->c0[XX])*grid->inv_sx);
+ cy = (int)((x[i][YY] - grid->c0[YY])*grid->inv_sy);
+
+ /* For non-home zones there could be particles outside
+ * the non-bonded cut-off range, which have been communicated
+ * for bonded interactions only. For the result it doesn't
+ * matter where these end up on the grid. For performance
+ * we put them in an extra row at the border.
+ */
+ cx = max(cx, 0);
+ cx = min(cx, grid->ncx - 1);
+ cy = max(cy, 0);
+ cy = min(cy, grid->ncy - 1);
+
+ /* For the moment cell will contain only the, grid local,
+ * x and y indices, not z.
+ */
+ cell[i] = cx*grid->ncy + cy;
+
+ cxy_na[cell[i]]++;
+ }
+ }
+}
+
+/* Determine in which grid cells the atoms should go */
+static void calc_cell_indices(const nbnxn_search_t nbs,
+ int dd_zone,
+ nbnxn_grid_t *grid,
+ int a0, int a1,
+ const int *atinfo,
+ rvec *x,
+ const int *move,
+ nbnxn_atomdata_t *nbat)
+{
+ int n0, n1, i;
+ int cx, cy, cxy, ncz_max, ncz;
+ int nthread, thread;
+ int *cxy_na, cxy_na_i;
+
+ nthread = gmx_omp_nthreads_get(emntPairsearch);
+
+#pragma omp parallel for num_threads(nthread) schedule(static)
+ for (thread = 0; thread < nthread; thread++)
+ {
+ calc_column_indices(grid, a0, a1, x, dd_zone, move, thread, nthread,
+ nbs->cell, nbs->work[thread].cxy_na);
+ }
+
+ /* Make the cell index as a function of x and y */
+ ncz_max = 0;
+ ncz = 0;
+ grid->cxy_ind[0] = 0;
+ for (i = 0; i < grid->ncx*grid->ncy+1; i++)
+ {
+ /* We set ncz_max at the beginning of the loop iso at the end
+ * to skip i=grid->ncx*grid->ncy which are moved particles
+ * that do not need to be ordered on the grid.
+ */
+ if (ncz > ncz_max)
+ {
+ ncz_max = ncz;
+ }
+ cxy_na_i = nbs->work[0].cxy_na[i];
+ for (thread = 1; thread < nthread; thread++)
+ {
+ cxy_na_i += nbs->work[thread].cxy_na[i];
+ }
+ ncz = (cxy_na_i + grid->na_sc - 1)/grid->na_sc;
+ if (nbat->XFormat == nbatX8)
+ {
+ /* Make the number of cell a multiple of 2 */
+ ncz = (ncz + 1) & ~1;
+ }
+ grid->cxy_ind[i+1] = grid->cxy_ind[i] + ncz;
+ /* Clear cxy_na, so we can reuse the array below */
+ grid->cxy_na[i] = 0;
+ }
+ grid->nc = grid->cxy_ind[grid->ncx*grid->ncy] - grid->cxy_ind[0];
+
+ nbat->natoms = (grid->cell0 + grid->nc)*grid->na_sc;
+
+ if (debug)
+ {
+ fprintf(debug, "ns na_sc %d na_c %d super-cells: %d x %d y %d z %.1f maxz %d\n",
+ grid->na_sc, grid->na_c, grid->nc,
+ grid->ncx, grid->ncy, grid->nc/((double)(grid->ncx*grid->ncy)),
+ ncz_max);
+ if (gmx_debug_at)
+ {
+ i = 0;
+ for (cy = 0; cy < grid->ncy; cy++)
+ {
+ for (cx = 0; cx < grid->ncx; cx++)
+ {
+ fprintf(debug, " %2d", grid->cxy_ind[i+1]-grid->cxy_ind[i]);
+ i++;
+ }
+ fprintf(debug, "\n");
+ }
+ }
+ }
+
+ /* Make sure the work array for sorting is large enough */
+ if (ncz_max*grid->na_sc*SGSF > nbs->work[0].sort_work_nalloc)
+ {
+ for (thread = 0; thread < nbs->nthread_max; thread++)
+ {
+ nbs->work[thread].sort_work_nalloc =
+ over_alloc_large(ncz_max*grid->na_sc*SGSF);
+ srenew(nbs->work[thread].sort_work,
+ nbs->work[thread].sort_work_nalloc);
+ /* When not in use, all elements should be -1 */
+ for (i = 0; i < nbs->work[thread].sort_work_nalloc; i++)
+ {
+ nbs->work[thread].sort_work[i] = -1;
+ }
+ }
+ }
+
+ /* Now we know the dimensions we can fill the grid.
+ * This is the first, unsorted fill. We sort the columns after this.
+ */
+ for (i = a0; i < a1; i++)
+ {
+ /* At this point nbs->cell contains the local grid x,y indices */
+ cxy = nbs->cell[i];
+ nbs->a[(grid->cell0 + grid->cxy_ind[cxy])*grid->na_sc + grid->cxy_na[cxy]++] = i;
+ }
+
+ if (dd_zone == 0)
+ {
+ /* Set the cell indices for the moved particles */
+ n0 = grid->nc*grid->na_sc;
+ n1 = grid->nc*grid->na_sc+grid->cxy_na[grid->ncx*grid->ncy];
+ if (dd_zone == 0)
+ {
+ for (i = n0; i < n1; i++)
+ {
+ nbs->cell[nbs->a[i]] = i;
+ }
+ }
+ }
+
+ /* Sort the super-cell columns along z into the sub-cells. */
+#pragma omp parallel for num_threads(nbs->nthread_max) schedule(static)
+ for (thread = 0; thread < nbs->nthread_max; thread++)
+ {
+ if (grid->bSimple)
+ {
+ sort_columns_simple(nbs, dd_zone, grid, a0, a1, atinfo, x, nbat,
+ ((thread+0)*grid->ncx*grid->ncy)/nthread,
+ ((thread+1)*grid->ncx*grid->ncy)/nthread,
+ nbs->work[thread].sort_work);
+ }
+ else
+ {
+ sort_columns_supersub(nbs, dd_zone, grid, a0, a1, atinfo, x, nbat,
+ ((thread+0)*grid->ncx*grid->ncy)/nthread,
+ ((thread+1)*grid->ncx*grid->ncy)/nthread,
+ nbs->work[thread].sort_work);
+ }
+ }
+
+ if (grid->bSimple && nbat->XFormat == nbatX8)
+ {
+ combine_bounding_box_pairs(grid, grid->bb);
+ }
+
+ if (!grid->bSimple)
+ {
+ grid->nsubc_tot = 0;
+ for (i = 0; i < grid->nc; i++)
+ {
+ grid->nsubc_tot += grid->nsubc[i];
+ }
+ }
+
+ if (debug)
+ {
+ if (grid->bSimple)
+ {
+ print_bbsizes_simple(debug, nbs, grid);
+ }
+ else
+ {
+ fprintf(debug, "ns non-zero sub-cells: %d average atoms %.2f\n",
+ grid->nsubc_tot, (a1-a0)/(double)grid->nsubc_tot);
+
+ print_bbsizes_supersub(debug, nbs, grid);
+ }
+ }
+}
+
+static void init_buffer_flags(nbnxn_buffer_flags_t *flags,
+ int natoms)
+{
+ int b;
+
+ flags->nflag = (natoms + NBNXN_BUFFERFLAG_SIZE - 1)/NBNXN_BUFFERFLAG_SIZE;
+ if (flags->nflag > flags->flag_nalloc)
+ {
+ flags->flag_nalloc = over_alloc_large(flags->nflag);
+ srenew(flags->flag, flags->flag_nalloc);
+ }
+ for (b = 0; b < flags->nflag; b++)
+ {
+ flags->flag[b] = 0;
+ }
+}
+
+/* Sets up a grid and puts the atoms on the grid.
+ * This function only operates on one domain of the domain decompostion.
+ * Note that without domain decomposition there is only one domain.
+ */
+void nbnxn_put_on_grid(nbnxn_search_t nbs,
+ int ePBC, matrix box,
+ int dd_zone,
+ rvec corner0, rvec corner1,
+ int a0, int a1,
+ real atom_density,
+ const int *atinfo,
+ rvec *x,
+ int nmoved, int *move,
+ int nb_kernel_type,
+ nbnxn_atomdata_t *nbat)
+{
+ nbnxn_grid_t *grid;
+ int n;
+ int nc_max_grid, nc_max;
+
+ grid = &nbs->grid[dd_zone];
+
+ nbs_cycle_start(&nbs->cc[enbsCCgrid]);
+
+ grid->bSimple = nbnxn_kernel_pairlist_simple(nb_kernel_type);
+
+ grid->na_c = nbnxn_kernel_to_ci_size(nb_kernel_type);
+ grid->na_cj = nbnxn_kernel_to_cj_size(nb_kernel_type);
+ grid->na_sc = (grid->bSimple ? 1 : GPU_NSUBCELL)*grid->na_c;
+ grid->na_c_2log = get_2log(grid->na_c);
+
+ nbat->na_c = grid->na_c;
+
+ if (dd_zone == 0)
+ {
+ grid->cell0 = 0;
+ }
+ else
+ {
+ grid->cell0 =
+ (nbs->grid[dd_zone-1].cell0 + nbs->grid[dd_zone-1].nc)*
+ nbs->grid[dd_zone-1].na_sc/grid->na_sc;
+ }
+
+ n = a1 - a0;
+
+ if (dd_zone == 0)
+ {
+ nbs->ePBC = ePBC;
+ copy_mat(box, nbs->box);
+
+ if (atom_density >= 0)
+ {
+ grid->atom_density = atom_density;
+ }
+ else
+ {
+ grid->atom_density = grid_atom_density(n-nmoved, corner0, corner1);
+ }
+
+ grid->cell0 = 0;
+
+ nbs->natoms_local = a1 - nmoved;
+ /* We assume that nbnxn_put_on_grid is called first
+ * for the local atoms (dd_zone=0).
+ */
+ nbs->natoms_nonlocal = a1 - nmoved;
+ }
+ else
+ {
+ nbs->natoms_nonlocal = max(nbs->natoms_nonlocal, a1);
+ }
+
+ nc_max_grid = set_grid_size_xy(nbs, grid,
+ dd_zone, n-nmoved, corner0, corner1,
+ nbs->grid[0].atom_density);
+
+ nc_max = grid->cell0 + nc_max_grid;
+
+ if (a1 > nbs->cell_nalloc)
+ {
+ nbs->cell_nalloc = over_alloc_large(a1);
+ srenew(nbs->cell, nbs->cell_nalloc);
+ }
+
+ /* To avoid conditionals we store the moved particles at the end of a,
+ * make sure we have enough space.
+ */
+ if (nc_max*grid->na_sc + nmoved > nbs->a_nalloc)
+ {
+ nbs->a_nalloc = over_alloc_large(nc_max*grid->na_sc + nmoved);
+ srenew(nbs->a, nbs->a_nalloc);
+ }
+
+ /* We need padding up to a multiple of the buffer flag size: simply add */
+ if (nc_max*grid->na_sc + NBNXN_BUFFERFLAG_SIZE > nbat->nalloc)
+ {
+ nbnxn_atomdata_realloc(nbat, nc_max*grid->na_sc+NBNXN_BUFFERFLAG_SIZE);
+ }
+
+ calc_cell_indices(nbs, dd_zone, grid, a0, a1, atinfo, x, move, nbat);
+
+ if (dd_zone == 0)
+ {
+ nbat->natoms_local = nbat->natoms;
+ }
+
+ nbs_cycle_stop(&nbs->cc[enbsCCgrid]);
+}
+
+/* Calls nbnxn_put_on_grid for all non-local domains */
+void nbnxn_put_on_grid_nonlocal(nbnxn_search_t nbs,
+ const gmx_domdec_zones_t *zones,
+ const int *atinfo,
+ rvec *x,
+ int nb_kernel_type,
+ nbnxn_atomdata_t *nbat)
+{
+ int zone, d;
+ rvec c0, c1;
+
+ for (zone = 1; zone < zones->n; zone++)
+ {
+ for (d = 0; d < DIM; d++)
+ {
+ c0[d] = zones->size[zone].bb_x0[d];
+ c1[d] = zones->size[zone].bb_x1[d];
+ }
+
+ nbnxn_put_on_grid(nbs, nbs->ePBC, NULL,
+ zone, c0, c1,
+ zones->cg_range[zone],
+ zones->cg_range[zone+1],
+ -1,
+ atinfo,
+ x,
+ 0, NULL,
+ nb_kernel_type,
+ nbat);
+ }
+}
+
+/* Add simple grid type information to the local super/sub grid */
+void nbnxn_grid_add_simple(nbnxn_search_t nbs,
+ nbnxn_atomdata_t *nbat)
+{
+ nbnxn_grid_t *grid;
+ float *bbcz;
+ nbnxn_bb_t *bb;
+ int ncd, sc;
+
+ grid = &nbs->grid[0];
+
+ if (grid->bSimple)
+ {
+ gmx_incons("nbnxn_grid_simple called with a simple grid");
+ }
+
+ ncd = grid->na_sc/NBNXN_CPU_CLUSTER_I_SIZE;
+
+ if (grid->nc*ncd > grid->nc_nalloc_simple)
+ {
+ grid->nc_nalloc_simple = over_alloc_large(grid->nc*ncd);
+ srenew(grid->bbcz_simple, grid->nc_nalloc_simple*NNBSBB_D);
+ srenew(grid->bb_simple, grid->nc_nalloc_simple);
+ srenew(grid->flags_simple, grid->nc_nalloc_simple);
+ if (nbat->XFormat)
+ {
+ sfree_aligned(grid->bbj);
+ snew_aligned(grid->bbj, grid->nc_nalloc_simple/2, 16);
+ }
+ }
+
+ bbcz = grid->bbcz_simple;
+ bb = grid->bb_simple;
+
+#pragma omp parallel for num_threads(gmx_omp_nthreads_get(emntPairsearch)) schedule(static)
+ for (sc = 0; sc < grid->nc; sc++)
+ {
+ int c, tx, na;
+
+ for (c = 0; c < ncd; c++)
+ {
+ tx = sc*ncd + c;
+
+ na = NBNXN_CPU_CLUSTER_I_SIZE;
+ while (na > 0 &&
+ nbat->type[tx*NBNXN_CPU_CLUSTER_I_SIZE+na-1] == nbat->ntype-1)
+ {
+ na--;
+ }
+
+ if (na > 0)
+ {
+ switch (nbat->XFormat)
+ {
+ case nbatX4:
+ /* PACK_X4==NBNXN_CPU_CLUSTER_I_SIZE, so this is simple */
+ calc_bounding_box_x_x4(na, nbat->x+tx*STRIDE_P4,
+ bb+tx);
+ break;
+ case nbatX8:
+ /* PACK_X8>NBNXN_CPU_CLUSTER_I_SIZE, more complicated */
+ calc_bounding_box_x_x8(na, nbat->x+X8_IND_A(tx*NBNXN_CPU_CLUSTER_I_SIZE),
+ bb+tx);
+ break;
+ default:
+ calc_bounding_box(na, nbat->xstride,
+ nbat->x+tx*NBNXN_CPU_CLUSTER_I_SIZE*nbat->xstride,
+ bb+tx);
+ break;
+ }
+ bbcz[tx*NNBSBB_D+0] = bb[tx].lower[BB_Z];
+ bbcz[tx*NNBSBB_D+1] = bb[tx].upper[BB_Z];
+
+ /* No interaction optimization yet here */
+ grid->flags_simple[tx] = NBNXN_CI_DO_LJ(0) | NBNXN_CI_DO_COUL(0);
+ }
+ else
+ {
+ grid->flags_simple[tx] = 0;
+ }
+ }
+ }
+
+ if (grid->bSimple && nbat->XFormat == nbatX8)
+ {
+ combine_bounding_box_pairs(grid, grid->bb_simple);
+ }
+}
+
+void nbnxn_get_ncells(nbnxn_search_t nbs, int *ncx, int *ncy)
+{
+ *ncx = nbs->grid[0].ncx;
+ *ncy = nbs->grid[0].ncy;
+}
+
+void nbnxn_get_atomorder(nbnxn_search_t nbs, int **a, int *n)
+{
+ const nbnxn_grid_t *grid;
+
+ grid = &nbs->grid[0];
+
+ /* Return the atom order for the home cell (index 0) */
+ *a = nbs->a;
+
+ *n = grid->cxy_ind[grid->ncx*grid->ncy]*grid->na_sc;
+}
+
+void nbnxn_set_atomorder(nbnxn_search_t nbs)
+{
+ nbnxn_grid_t *grid;
+ int ao, cx, cy, cxy, cz, j;
+
+ /* Set the atom order for the home cell (index 0) */
+ grid = &nbs->grid[0];
+
+ ao = 0;
+ for (cx = 0; cx < grid->ncx; cx++)
+ {
+ for (cy = 0; cy < grid->ncy; cy++)
+ {
+ cxy = cx*grid->ncy + cy;
+ j = grid->cxy_ind[cxy]*grid->na_sc;
+ for (cz = 0; cz < grid->cxy_na[cxy]; cz++)
+ {
+ nbs->a[j] = ao;
+ nbs->cell[ao] = j;
+ ao++;
+ j++;
+ }
+ }
+ }
+}
+
+/* Determines the cell range along one dimension that
+ * the bounding box b0 - b1 sees.
+ */
+static void get_cell_range(real b0, real b1,
+ int nc, real c0, real s, real invs,
+ real d2, real r2, int *cf, int *cl)
+{
+ *cf = max((int)((b0 - c0)*invs), 0);
+
+ while (*cf > 0 && d2 + sqr((b0 - c0) - (*cf-1+1)*s) < r2)
+ {
+ (*cf)--;
+ }
+
+ *cl = min((int)((b1 - c0)*invs), nc-1);
+ while (*cl < nc-1 && d2 + sqr((*cl+1)*s - (b1 - c0)) < r2)
+ {
+ (*cl)++;
+ }
+}
+
+/* Reference code calculating the distance^2 between two bounding boxes */
+static float box_dist2(float bx0, float bx1, float by0,
+ float by1, float bz0, float bz1,
+ const nbnxn_bb_t *bb)
+{
+ float d2;
+ float dl, dh, dm, dm0;
+
+ d2 = 0;
+
+ dl = bx0 - bb->upper[BB_X];
+ dh = bb->lower[BB_X] - bx1;
+ dm = max(dl, dh);
+ dm0 = max(dm, 0);
+ d2 += dm0*dm0;
+
+ dl = by0 - bb->upper[BB_Y];
+ dh = bb->lower[BB_Y] - by1;
+ dm = max(dl, dh);
+ dm0 = max(dm, 0);
+ d2 += dm0*dm0;
+
+ dl = bz0 - bb->upper[BB_Z];
+ dh = bb->lower[BB_Z] - bz1;
+ dm = max(dl, dh);
+ dm0 = max(dm, 0);
+ d2 += dm0*dm0;
+
+ return d2;
+}
+
+/* Plain C code calculating the distance^2 between two bounding boxes */
+static float subc_bb_dist2(int si, const nbnxn_bb_t *bb_i_ci,
+ int csj, const nbnxn_bb_t *bb_j_all)
+{
+ const nbnxn_bb_t *bb_i, *bb_j;
+ float d2;
+ float dl, dh, dm, dm0;
+
+ bb_i = bb_i_ci + si;
+ bb_j = bb_j_all + csj;
+
+ d2 = 0;
+
+ dl = bb_i->lower[BB_X] - bb_j->upper[BB_X];
+ dh = bb_j->lower[BB_X] - bb_i->upper[BB_X];
+ dm = max(dl, dh);
+ dm0 = max(dm, 0);
+ d2 += dm0*dm0;
+
+ dl = bb_i->lower[BB_Y] - bb_j->upper[BB_Y];
+ dh = bb_j->lower[BB_Y] - bb_i->upper[BB_Y];
+ dm = max(dl, dh);
+ dm0 = max(dm, 0);
+ d2 += dm0*dm0;
+
+ dl = bb_i->lower[BB_Z] - bb_j->upper[BB_Z];
+ dh = bb_j->lower[BB_Z] - bb_i->upper[BB_Z];
+ dm = max(dl, dh);
+ dm0 = max(dm, 0);
+ d2 += dm0*dm0;
+
+ return d2;
+}
+
+#ifdef NBNXN_SEARCH_BB_SIMD4
+
+/* 4-wide SIMD code for bb distance for bb format xyz0 */
+static float subc_bb_dist2_simd4(int si, const nbnxn_bb_t *bb_i_ci,
+ int csj, const nbnxn_bb_t *bb_j_all)
+{
+ gmx_simd4_pr bb_i_S0, bb_i_S1;
+ gmx_simd4_pr bb_j_S0, bb_j_S1;
+ gmx_simd4_pr dl_S;
+ gmx_simd4_pr dh_S;
+ gmx_simd4_pr dm_S;
+ gmx_simd4_pr dm0_S;
+
+ bb_i_S0 = gmx_simd4_load_bb_pr(&bb_i_ci[si].lower[0]);
+ bb_i_S1 = gmx_simd4_load_bb_pr(&bb_i_ci[si].upper[0]);
+ bb_j_S0 = gmx_simd4_load_bb_pr(&bb_j_all[csj].lower[0]);
+ bb_j_S1 = gmx_simd4_load_bb_pr(&bb_j_all[csj].upper[0]);
+
+ dl_S = gmx_simd4_sub_pr(bb_i_S0, bb_j_S1);
+ dh_S = gmx_simd4_sub_pr(bb_j_S0, bb_i_S1);
+
+ dm_S = gmx_simd4_max_pr(dl_S, dh_S);
+ dm0_S = gmx_simd4_max_pr(dm_S, gmx_simd4_setzero_pr());
+
+ return gmx_simd4_dotproduct3(dm0_S, dm0_S);
+}
+
+/* Calculate bb bounding distances of bb_i[si,...,si+3] and store them in d2 */
+#define SUBC_BB_DIST2_SIMD4_XXXX_INNER(si, bb_i, d2) \
+ { \
+ int shi; \
+ \
+ gmx_simd4_pr dx_0, dy_0, dz_0; \
+ gmx_simd4_pr dx_1, dy_1, dz_1; \
+ \
+ gmx_simd4_pr mx, my, mz; \
+ gmx_simd4_pr m0x, m0y, m0z; \
+ \
+ gmx_simd4_pr d2x, d2y, d2z; \
+ gmx_simd4_pr d2s, d2t; \
+ \
+ shi = si*NNBSBB_D*DIM; \
+ \
+ xi_l = gmx_simd4_load_bb_pr(bb_i+shi+0*STRIDE_PBB); \
+ yi_l = gmx_simd4_load_bb_pr(bb_i+shi+1*STRIDE_PBB); \
+ zi_l = gmx_simd4_load_bb_pr(bb_i+shi+2*STRIDE_PBB); \
+ xi_h = gmx_simd4_load_bb_pr(bb_i+shi+3*STRIDE_PBB); \
+ yi_h = gmx_simd4_load_bb_pr(bb_i+shi+4*STRIDE_PBB); \
+ zi_h = gmx_simd4_load_bb_pr(bb_i+shi+5*STRIDE_PBB); \
+ \
+ dx_0 = gmx_simd4_sub_pr(xi_l, xj_h); \
+ dy_0 = gmx_simd4_sub_pr(yi_l, yj_h); \
+ dz_0 = gmx_simd4_sub_pr(zi_l, zj_h); \
+ \
+ dx_1 = gmx_simd4_sub_pr(xj_l, xi_h); \
+ dy_1 = gmx_simd4_sub_pr(yj_l, yi_h); \
+ dz_1 = gmx_simd4_sub_pr(zj_l, zi_h); \
+ \
+ mx = gmx_simd4_max_pr(dx_0, dx_1); \
+ my = gmx_simd4_max_pr(dy_0, dy_1); \
+ mz = gmx_simd4_max_pr(dz_0, dz_1); \
+ \
+ m0x = gmx_simd4_max_pr(mx, zero); \
+ m0y = gmx_simd4_max_pr(my, zero); \
+ m0z = gmx_simd4_max_pr(mz, zero); \
+ \
+ d2x = gmx_simd4_mul_pr(m0x, m0x); \
+ d2y = gmx_simd4_mul_pr(m0y, m0y); \
+ d2z = gmx_simd4_mul_pr(m0z, m0z); \
+ \
+ d2s = gmx_simd4_add_pr(d2x, d2y); \
+ d2t = gmx_simd4_add_pr(d2s, d2z); \
+ \
+ gmx_simd4_store_pr(d2+si, d2t); \
+ }
+
+/* 4-wide SIMD code for nsi bb distances for bb format xxxxyyyyzzzz */
+static void subc_bb_dist2_simd4_xxxx(const float *bb_j,
+ int nsi, const float *bb_i,
+ float *d2)
+{
+ gmx_simd4_pr xj_l, yj_l, zj_l;
+ gmx_simd4_pr xj_h, yj_h, zj_h;
+ gmx_simd4_pr xi_l, yi_l, zi_l;
+ gmx_simd4_pr xi_h, yi_h, zi_h;
+
+ gmx_simd4_pr zero;
+
+ zero = gmx_simd4_setzero_pr();
+
+ xj_l = gmx_simd4_set1_pr(bb_j[0*STRIDE_PBB]);
+ yj_l = gmx_simd4_set1_pr(bb_j[1*STRIDE_PBB]);
+ zj_l = gmx_simd4_set1_pr(bb_j[2*STRIDE_PBB]);
+ xj_h = gmx_simd4_set1_pr(bb_j[3*STRIDE_PBB]);
+ yj_h = gmx_simd4_set1_pr(bb_j[4*STRIDE_PBB]);
+ zj_h = gmx_simd4_set1_pr(bb_j[5*STRIDE_PBB]);
+
+ /* Here we "loop" over si (0,STRIDE_PBB) from 0 to nsi with step STRIDE_PBB.
+ * But as we know the number of iterations is 1 or 2, we unroll manually.
+ */
+ SUBC_BB_DIST2_SIMD4_XXXX_INNER(0, bb_i, d2);
+ if (STRIDE_PBB < nsi)
+ {
+ SUBC_BB_DIST2_SIMD4_XXXX_INNER(STRIDE_PBB, bb_i, d2);
+ }
+}
+
+#endif /* NBNXN_SEARCH_BB_SIMD4 */
+
+/* Plain C function which determines if any atom pair between two cells
+ * is within distance sqrt(rl2).
+ */
+static gmx_bool subc_in_range_x(int na_c,
+ int si, const real *x_i,
+ int csj, int stride, const real *x_j,
+ real rl2)
+{
+ int i, j, i0, j0;
+ real d2;
+
+ for (i = 0; i < na_c; i++)
+ {
+ i0 = (si*na_c + i)*DIM;
+ for (j = 0; j < na_c; j++)
+ {
+ j0 = (csj*na_c + j)*stride;
+
+ d2 = sqr(x_i[i0 ] - x_j[j0 ]) +
+ sqr(x_i[i0+1] - x_j[j0+1]) +
+ sqr(x_i[i0+2] - x_j[j0+2]);
+
+ if (d2 < rl2)
+ {
+ return TRUE;
+ }
+ }
+ }
+
+ return FALSE;
+}
+
+#ifdef NBNXN_SEARCH_SIMD4_FLOAT_X_BB
+/* When we make seperate single/double precision SIMD vector operation
+ * include files, this function should be moved there (also using FMA).
+ */
+static inline gmx_simd4_pr
+gmx_simd4_calc_rsq_pr(gmx_simd4_pr x, gmx_simd4_pr y, gmx_simd4_pr z)
+{
+ return gmx_simd4_add_pr( gmx_simd4_add_pr( gmx_simd4_mul_pr(x, x), gmx_simd4_mul_pr(y, y) ), gmx_simd4_mul_pr(z, z) );
+}
+
+/* 4-wide SIMD function which determines if any atom pair between two cells,
+ * both with 8 atoms, is within distance sqrt(rl2).
+ * Using 8-wide AVX is not faster on Intel Sandy Bridge.
+ */
+static gmx_bool subc_in_range_simd4(int na_c,
+ int si, const real *x_i,
+ int csj, int stride, const real *x_j,
+ real rl2)
+{
+ gmx_simd4_pr ix_S0, iy_S0, iz_S0;
+ gmx_simd4_pr ix_S1, iy_S1, iz_S1;
+
+ gmx_simd4_pr rc2_S;
+
+ int dim_stride;
+ int j0, j1;
+
+ rc2_S = gmx_simd4_set1_pr(rl2);
+
+ dim_stride = NBNXN_GPU_CLUSTER_SIZE/STRIDE_PBB*DIM;
+ ix_S0 = gmx_simd4_load_bb_pr(x_i+(si*dim_stride+0)*STRIDE_PBB);
+ iy_S0 = gmx_simd4_load_bb_pr(x_i+(si*dim_stride+1)*STRIDE_PBB);
+ iz_S0 = gmx_simd4_load_bb_pr(x_i+(si*dim_stride+2)*STRIDE_PBB);
+ ix_S1 = gmx_simd4_load_bb_pr(x_i+(si*dim_stride+3)*STRIDE_PBB);
+ iy_S1 = gmx_simd4_load_bb_pr(x_i+(si*dim_stride+4)*STRIDE_PBB);
+ iz_S1 = gmx_simd4_load_bb_pr(x_i+(si*dim_stride+5)*STRIDE_PBB);
+
+ /* We loop from the outer to the inner particles to maximize
+ * the chance that we find a pair in range quickly and return.
+ */
+ j0 = csj*na_c;
+ j1 = j0 + na_c - 1;
+ while (j0 < j1)
+ {
+ gmx_simd4_pr jx0_S, jy0_S, jz0_S;
+ gmx_simd4_pr jx1_S, jy1_S, jz1_S;
+
+ gmx_simd4_pr dx_S0, dy_S0, dz_S0;
+ gmx_simd4_pr dx_S1, dy_S1, dz_S1;
+ gmx_simd4_pr dx_S2, dy_S2, dz_S2;
+ gmx_simd4_pr dx_S3, dy_S3, dz_S3;
+
+ gmx_simd4_pr rsq_S0;
+ gmx_simd4_pr rsq_S1;
+ gmx_simd4_pr rsq_S2;
+ gmx_simd4_pr rsq_S3;
+
+ gmx_simd4_pb wco_S0;
+ gmx_simd4_pb wco_S1;
+ gmx_simd4_pb wco_S2;
+ gmx_simd4_pb wco_S3;
+ gmx_simd4_pb wco_any_S01, wco_any_S23, wco_any_S;
+
+ jx0_S = gmx_simd4_set1_pr(x_j[j0*stride+0]);
+ jy0_S = gmx_simd4_set1_pr(x_j[j0*stride+1]);
+ jz0_S = gmx_simd4_set1_pr(x_j[j0*stride+2]);
+
+ jx1_S = gmx_simd4_set1_pr(x_j[j1*stride+0]);
+ jy1_S = gmx_simd4_set1_pr(x_j[j1*stride+1]);
+ jz1_S = gmx_simd4_set1_pr(x_j[j1*stride+2]);
+
+ /* Calculate distance */
+ dx_S0 = gmx_simd4_sub_pr(ix_S0, jx0_S);
+ dy_S0 = gmx_simd4_sub_pr(iy_S0, jy0_S);
+ dz_S0 = gmx_simd4_sub_pr(iz_S0, jz0_S);
+ dx_S1 = gmx_simd4_sub_pr(ix_S1, jx0_S);
+ dy_S1 = gmx_simd4_sub_pr(iy_S1, jy0_S);
+ dz_S1 = gmx_simd4_sub_pr(iz_S1, jz0_S);
+ dx_S2 = gmx_simd4_sub_pr(ix_S0, jx1_S);
+ dy_S2 = gmx_simd4_sub_pr(iy_S0, jy1_S);
+ dz_S2 = gmx_simd4_sub_pr(iz_S0, jz1_S);
+ dx_S3 = gmx_simd4_sub_pr(ix_S1, jx1_S);
+ dy_S3 = gmx_simd4_sub_pr(iy_S1, jy1_S);
+ dz_S3 = gmx_simd4_sub_pr(iz_S1, jz1_S);
+
+ /* rsq = dx*dx+dy*dy+dz*dz */
+ rsq_S0 = gmx_simd4_calc_rsq_pr(dx_S0, dy_S0, dz_S0);
+ rsq_S1 = gmx_simd4_calc_rsq_pr(dx_S1, dy_S1, dz_S1);
+ rsq_S2 = gmx_simd4_calc_rsq_pr(dx_S2, dy_S2, dz_S2);
+ rsq_S3 = gmx_simd4_calc_rsq_pr(dx_S3, dy_S3, dz_S3);
+
+ wco_S0 = gmx_simd4_cmplt_pr(rsq_S0, rc2_S);
+ wco_S1 = gmx_simd4_cmplt_pr(rsq_S1, rc2_S);
+ wco_S2 = gmx_simd4_cmplt_pr(rsq_S2, rc2_S);
+ wco_S3 = gmx_simd4_cmplt_pr(rsq_S3, rc2_S);
+
+ wco_any_S01 = gmx_simd4_or_pb(wco_S0, wco_S1);
+ wco_any_S23 = gmx_simd4_or_pb(wco_S2, wco_S3);
+ wco_any_S = gmx_simd4_or_pb(wco_any_S01, wco_any_S23);
+
+ if (gmx_simd4_anytrue_pb(wco_any_S))
+ {
+ return TRUE;
+ }
+
+ j0++;
+ j1--;
+ }
+ return FALSE;
+
+}
+#endif
+
+
+/* Returns the j sub-cell for index cj_ind */
+static int nbl_cj(const nbnxn_pairlist_t *nbl, int cj_ind)
+{
+ return nbl->cj4[cj_ind >> NBNXN_GPU_JGROUP_SIZE_2LOG].cj[cj_ind & (NBNXN_GPU_JGROUP_SIZE - 1)];
+}
+
+/* Returns the i-interaction mask of the j sub-cell for index cj_ind */
+static unsigned nbl_imask0(const nbnxn_pairlist_t *nbl, int cj_ind)
+{
+ return nbl->cj4[cj_ind >> NBNXN_GPU_JGROUP_SIZE_2LOG].imei[0].imask;
+}
+
+/* Ensures there is enough space for extra extra exclusion masks */
+static void check_excl_space(nbnxn_pairlist_t *nbl, int extra)
+{
+ if (nbl->nexcl+extra > nbl->excl_nalloc)
+ {
+ nbl->excl_nalloc = over_alloc_small(nbl->nexcl+extra);
+ nbnxn_realloc_void((void **)&nbl->excl,
+ nbl->nexcl*sizeof(*nbl->excl),
+ nbl->excl_nalloc*sizeof(*nbl->excl),
+ nbl->alloc, nbl->free);
+ }
+}
+
+/* Ensures there is enough space for ncell extra j-cells in the list */
+static void check_subcell_list_space_simple(nbnxn_pairlist_t *nbl,
+ int ncell)
+{
+ int cj_max;
+
+ cj_max = nbl->ncj + ncell;
+
+ if (cj_max > nbl->cj_nalloc)
+ {
+ nbl->cj_nalloc = over_alloc_small(cj_max);
+ nbnxn_realloc_void((void **)&nbl->cj,
+ nbl->ncj*sizeof(*nbl->cj),
+ nbl->cj_nalloc*sizeof(*nbl->cj),
+ nbl->alloc, nbl->free);
+ }
+}
+
+/* Ensures there is enough space for ncell extra j-subcells in the list */
+static void check_subcell_list_space_supersub(nbnxn_pairlist_t *nbl,
+ int nsupercell)
+{
+ int ncj4_max, j4, j, w, t;
+
+#define NWARP 2
+#define WARP_SIZE 32
+
+ /* We can have maximally nsupercell*GPU_NSUBCELL sj lists */
+ /* We can store 4 j-subcell - i-supercell pairs in one struct.
+ * since we round down, we need one extra entry.
+ */
+ ncj4_max = ((nbl->work->cj_ind + nsupercell*GPU_NSUBCELL + NBNXN_GPU_JGROUP_SIZE - 1) >> NBNXN_GPU_JGROUP_SIZE_2LOG);
+
+ if (ncj4_max > nbl->cj4_nalloc)
+ {
+ nbl->cj4_nalloc = over_alloc_small(ncj4_max);
+ nbnxn_realloc_void((void **)&nbl->cj4,
+ nbl->work->cj4_init*sizeof(*nbl->cj4),
+ nbl->cj4_nalloc*sizeof(*nbl->cj4),
+ nbl->alloc, nbl->free);
+ }
+
+ if (ncj4_max > nbl->work->cj4_init)
+ {
+ for (j4 = nbl->work->cj4_init; j4 < ncj4_max; j4++)
+ {
+ /* No i-subcells and no excl's in the list initially */
+ for (w = 0; w < NWARP; w++)
+ {
+ nbl->cj4[j4].imei[w].imask = 0U;
+ nbl->cj4[j4].imei[w].excl_ind = 0;
+
+ }
+ }
+ nbl->work->cj4_init = ncj4_max;
+ }
+}
+
+/* Set all excl masks for one GPU warp no exclusions */
+static void set_no_excls(nbnxn_excl_t *excl)
+{
+ int t;
+
+ for (t = 0; t < WARP_SIZE; t++)
+ {
+ /* Turn all interaction bits on */
+ excl->pair[t] = NBNXN_INTERACTION_MASK_ALL;
+ }
+}
+
+/* Initializes a single nbnxn_pairlist_t data structure */
+static void nbnxn_init_pairlist(nbnxn_pairlist_t *nbl,
+ gmx_bool bSimple,
+ nbnxn_alloc_t *alloc,
+ nbnxn_free_t *free)
+{
+ if (alloc == NULL)
+ {
+ nbl->alloc = nbnxn_alloc_aligned;
+ }
+ else
+ {
+ nbl->alloc = alloc;
+ }
+ if (free == NULL)
+ {
+ nbl->free = nbnxn_free_aligned;
+ }
+ else
+ {
+ nbl->free = free;
+ }
+
+ nbl->bSimple = bSimple;
+ nbl->na_sc = 0;
+ nbl->na_ci = 0;
+ nbl->na_cj = 0;
+ nbl->nci = 0;
+ nbl->ci = NULL;
+ nbl->ci_nalloc = 0;
+ nbl->ncj = 0;
+ nbl->cj = NULL;
+ nbl->cj_nalloc = 0;
+ nbl->ncj4 = 0;
+ /* We need one element extra in sj, so alloc initially with 1 */
+ nbl->cj4_nalloc = 0;
+ nbl->cj4 = NULL;
+ nbl->nci_tot = 0;
+
+ if (!nbl->bSimple)
+ {
+ nbl->excl = NULL;
+ nbl->excl_nalloc = 0;
+ nbl->nexcl = 0;
+ check_excl_space(nbl, 1);
+ nbl->nexcl = 1;
+ set_no_excls(&nbl->excl[0]);
+ }
+
+ snew(nbl->work, 1);
+ if (nbl->bSimple)
+ {
+ snew_aligned(nbl->work->bb_ci, 1, NBNXN_SEARCH_BB_MEM_ALIGN);
+ }
+ else
+ {
+#ifdef NBNXN_BBXXXX
+ snew_aligned(nbl->work->pbb_ci, GPU_NSUBCELL/STRIDE_PBB*NNBSBB_XXXX, NBNXN_SEARCH_BB_MEM_ALIGN);
+#else
+ snew_aligned(nbl->work->bb_ci, GPU_NSUBCELL, NBNXN_SEARCH_BB_MEM_ALIGN);
+#endif
+ }
+ snew_aligned(nbl->work->x_ci, NBNXN_NA_SC_MAX*DIM, NBNXN_SEARCH_BB_MEM_ALIGN);
+#ifdef GMX_NBNXN_SIMD
+ snew_aligned(nbl->work->x_ci_simd_4xn, 1, NBNXN_MEM_ALIGN);
+ snew_aligned(nbl->work->x_ci_simd_2xnn, 1, NBNXN_MEM_ALIGN);
+#endif
+ snew_aligned(nbl->work->d2, GPU_NSUBCELL, NBNXN_SEARCH_BB_MEM_ALIGN);
+
+ nbl->work->sort = NULL;
+ nbl->work->sort_nalloc = 0;
+ nbl->work->sci_sort = NULL;
+ nbl->work->sci_sort_nalloc = 0;
+}
+
+void nbnxn_init_pairlist_set(nbnxn_pairlist_set_t *nbl_list,
+ gmx_bool bSimple, gmx_bool bCombined,
+ nbnxn_alloc_t *alloc,
+ nbnxn_free_t *free)
+{
+ int i;
+
+ nbl_list->bSimple = bSimple;
+ nbl_list->bCombined = bCombined;
+
+ nbl_list->nnbl = gmx_omp_nthreads_get(emntNonbonded);
+
+ if (!nbl_list->bCombined &&
+ nbl_list->nnbl > NBNXN_BUFFERFLAG_MAX_THREADS)
+ {
+ gmx_fatal(FARGS, "%d OpenMP threads were requested. Since the non-bonded force buffer reduction is prohibitively slow with more than %d threads, we do not allow this. Use %d or less OpenMP threads.",
+ nbl_list->nnbl, NBNXN_BUFFERFLAG_MAX_THREADS, NBNXN_BUFFERFLAG_MAX_THREADS);
+ }
+
+ snew(nbl_list->nbl, nbl_list->nnbl);
+ /* Execute in order to avoid memory interleaving between threads */
+#pragma omp parallel for num_threads(nbl_list->nnbl) schedule(static)
+ for (i = 0; i < nbl_list->nnbl; i++)
+ {
+ /* Allocate the nblist data structure locally on each thread
+ * to optimize memory access for NUMA architectures.
+ */
+ snew(nbl_list->nbl[i], 1);
+
+ /* Only list 0 is used on the GPU, use normal allocation for i>0 */
+ if (i == 0)
+ {
+ nbnxn_init_pairlist(nbl_list->nbl[i], nbl_list->bSimple, alloc, free);
+ }
+ else
+ {
+ nbnxn_init_pairlist(nbl_list->nbl[i], nbl_list->bSimple, NULL, NULL);
+ }
+ }
+}
+
+/* Print statistics of a pair list, used for debug output */
+static void print_nblist_statistics_simple(FILE *fp, const nbnxn_pairlist_t *nbl,
+ const nbnxn_search_t nbs, real rl)
+{
+ const nbnxn_grid_t *grid;
+ int cs[SHIFTS];
+ int s, i, j;
+ int npexcl;
+
+ /* This code only produces correct statistics with domain decomposition */
+ grid = &nbs->grid[0];
+
+ fprintf(fp, "nbl nci %d ncj %d\n",
+ nbl->nci, nbl->ncj);
+ fprintf(fp, "nbl na_sc %d rl %g ncp %d per cell %.1f atoms %.1f ratio %.2f\n",
+ nbl->na_sc, rl, nbl->ncj, nbl->ncj/(double)grid->nc,
+ nbl->ncj/(double)grid->nc*grid->na_sc,
+ nbl->ncj/(double)grid->nc*grid->na_sc/(0.5*4.0/3.0*M_PI*rl*rl*rl*grid->nc*grid->na_sc/det(nbs->box)));
+
+ fprintf(fp, "nbl average j cell list length %.1f\n",
+ 0.25*nbl->ncj/(double)nbl->nci);
+
+ for (s = 0; s < SHIFTS; s++)
+ {
+ cs[s] = 0;
+ }
+ npexcl = 0;
+ for (i = 0; i < nbl->nci; i++)
+ {
+ cs[nbl->ci[i].shift & NBNXN_CI_SHIFT] +=
+ nbl->ci[i].cj_ind_end - nbl->ci[i].cj_ind_start;
+
+ j = nbl->ci[i].cj_ind_start;
+ while (j < nbl->ci[i].cj_ind_end &&
+ nbl->cj[j].excl != NBNXN_INTERACTION_MASK_ALL)
+ {
+ npexcl++;
+ j++;
+ }
+ }
+ fprintf(fp, "nbl cell pairs, total: %d excl: %d %.1f%%\n",
+ nbl->ncj, npexcl, 100*npexcl/(double)nbl->ncj);
+ for (s = 0; s < SHIFTS; s++)
+ {
+ if (cs[s] > 0)
+ {
+ fprintf(fp, "nbl shift %2d ncj %3d\n", s, cs[s]);
+ }
+ }
+}
+
+/* Print statistics of a pair lists, used for debug output */
+static void print_nblist_statistics_supersub(FILE *fp, const nbnxn_pairlist_t *nbl,
+ const nbnxn_search_t nbs, real rl)
+{
+ const nbnxn_grid_t *grid;
+ int i, j4, j, si, b;
+ int c[GPU_NSUBCELL+1];
+
+ /* This code only produces correct statistics with domain decomposition */
+ grid = &nbs->grid[0];
+
+ fprintf(fp, "nbl nsci %d ncj4 %d nsi %d excl4 %d\n",
+ nbl->nsci, nbl->ncj4, nbl->nci_tot, nbl->nexcl);
+ fprintf(fp, "nbl na_c %d rl %g ncp %d per cell %.1f atoms %.1f ratio %.2f\n",
+ nbl->na_ci, rl, nbl->nci_tot, nbl->nci_tot/(double)grid->nsubc_tot,
+ nbl->nci_tot/(double)grid->nsubc_tot*grid->na_c,
+ nbl->nci_tot/(double)grid->nsubc_tot*grid->na_c/(0.5*4.0/3.0*M_PI*rl*rl*rl*grid->nsubc_tot*grid->na_c/det(nbs->box)));
+
+ fprintf(fp, "nbl average j super cell list length %.1f\n",
+ 0.25*nbl->ncj4/(double)nbl->nsci);
+ fprintf(fp, "nbl average i sub cell list length %.1f\n",
+ nbl->nci_tot/((double)nbl->ncj4));
+
+ for (si = 0; si <= GPU_NSUBCELL; si++)
+ {
+ c[si] = 0;
+ }
+ for (i = 0; i < nbl->nsci; i++)
+ {
+ for (j4 = nbl->sci[i].cj4_ind_start; j4 < nbl->sci[i].cj4_ind_end; j4++)
+ {
+ for (j = 0; j < NBNXN_GPU_JGROUP_SIZE; j++)
+ {
+ b = 0;
+ for (si = 0; si < GPU_NSUBCELL; si++)
+ {
+ if (nbl->cj4[j4].imei[0].imask & (1U << (j*GPU_NSUBCELL + si)))
+ {
+ b++;
+ }
+ }
+ c[b]++;
+ }
+ }
+ }
+ for (b = 0; b <= GPU_NSUBCELL; b++)
+ {
+ fprintf(fp, "nbl j-list #i-subcell %d %7d %4.1f\n",
+ b, c[b], 100.0*c[b]/(double)(nbl->ncj4*NBNXN_GPU_JGROUP_SIZE));
+ }
+}
+
+/* Returns a pointer to the exclusion mask for cj4-unit cj4, warp warp */
+static void low_get_nbl_exclusions(nbnxn_pairlist_t *nbl, int cj4,
+ int warp, nbnxn_excl_t **excl)
+{
+ if (nbl->cj4[cj4].imei[warp].excl_ind == 0)
+ {
+ /* No exclusions set, make a new list entry */
+ nbl->cj4[cj4].imei[warp].excl_ind = nbl->nexcl;
+ nbl->nexcl++;
+ *excl = &nbl->excl[nbl->cj4[cj4].imei[warp].excl_ind];
+ set_no_excls(*excl);
+ }
+ else
+ {
+ /* We already have some exclusions, new ones can be added to the list */
+ *excl = &nbl->excl[nbl->cj4[cj4].imei[warp].excl_ind];
+ }
+}
+
+/* Returns a pointer to the exclusion mask for cj4-unit cj4, warp warp,
+ * allocates extra memory, if necessary.
+ */
+static void get_nbl_exclusions_1(nbnxn_pairlist_t *nbl, int cj4,
+ int warp, nbnxn_excl_t **excl)
+{
+ if (nbl->cj4[cj4].imei[warp].excl_ind == 0)
+ {
+ /* We need to make a new list entry, check if we have space */
+ check_excl_space(nbl, 1);
+ }
+ low_get_nbl_exclusions(nbl, cj4, warp, excl);
+}
+
+/* Returns pointers to the exclusion mask for cj4-unit cj4 for both warps,
+ * allocates extra memory, if necessary.
+ */
+static void get_nbl_exclusions_2(nbnxn_pairlist_t *nbl, int cj4,
+ nbnxn_excl_t **excl_w0,
+ nbnxn_excl_t **excl_w1)
+{
+ /* Check for space we might need */
+ check_excl_space(nbl, 2);
+
+ low_get_nbl_exclusions(nbl, cj4, 0, excl_w0);
+ low_get_nbl_exclusions(nbl, cj4, 1, excl_w1);
+}
+
+/* Sets the self exclusions i=j and pair exclusions i>j */
+static void set_self_and_newton_excls_supersub(nbnxn_pairlist_t *nbl,
+ int cj4_ind, int sj_offset,
+ int si)
+{
+ nbnxn_excl_t *excl[2];
+ int ei, ej, w;
+
+ /* Here we only set the set self and double pair exclusions */
+
+ get_nbl_exclusions_2(nbl, cj4_ind, &excl[0], &excl[1]);
+
+ /* Only minor < major bits set */
+ for (ej = 0; ej < nbl->na_ci; ej++)
+ {
+ w = (ej>>2);
+ for (ei = ej; ei < nbl->na_ci; ei++)
+ {
+ excl[w]->pair[(ej & (NBNXN_GPU_JGROUP_SIZE-1))*nbl->na_ci + ei] &=
+ ~(1U << (sj_offset*GPU_NSUBCELL + si));
+ }
+ }
+}
+
+/* Returns a diagonal or off-diagonal interaction mask for plain C lists */
+static unsigned int get_imask(gmx_bool rdiag, int ci, int cj)
+{
+ return (rdiag && ci == cj ? NBNXN_INTERACTION_MASK_DIAG : NBNXN_INTERACTION_MASK_ALL);
+}
+
+/* Returns a diagonal or off-diagonal interaction mask for cj-size=2 */
+static unsigned int get_imask_simd_j2(gmx_bool rdiag, int ci, int cj)
+{
+ return (rdiag && ci*2 == cj ? NBNXN_INTERACTION_MASK_DIAG_J2_0 :
+ (rdiag && ci*2+1 == cj ? NBNXN_INTERACTION_MASK_DIAG_J2_1 :
+ NBNXN_INTERACTION_MASK_ALL));
+}
+
+/* Returns a diagonal or off-diagonal interaction mask for cj-size=4 */
+static unsigned int get_imask_simd_j4(gmx_bool rdiag, int ci, int cj)
+{
+ return (rdiag && ci == cj ? NBNXN_INTERACTION_MASK_DIAG : NBNXN_INTERACTION_MASK_ALL);
+}
+
+/* Returns a diagonal or off-diagonal interaction mask for cj-size=8 */
+static unsigned int get_imask_simd_j8(gmx_bool rdiag, int ci, int cj)
+{
+ return (rdiag && ci == cj*2 ? NBNXN_INTERACTION_MASK_DIAG_J8_0 :
+ (rdiag && ci == cj*2+1 ? NBNXN_INTERACTION_MASK_DIAG_J8_1 :
+ NBNXN_INTERACTION_MASK_ALL));
+}
+
+#ifdef GMX_NBNXN_SIMD
+#if GMX_SIMD_WIDTH_HERE == 2
+#define get_imask_simd_4xn get_imask_simd_j2
+#endif
+#if GMX_SIMD_WIDTH_HERE == 4
+#define get_imask_simd_4xn get_imask_simd_j4
+#endif
+#if GMX_SIMD_WIDTH_HERE == 8
+#define get_imask_simd_4xn get_imask_simd_j8
+#define get_imask_simd_2xnn get_imask_simd_j4
+#endif
+#if GMX_SIMD_WIDTH_HERE == 16
+#define get_imask_simd_2xnn get_imask_simd_j8
+#endif
+#endif
+
+/* Plain C code for making a pair list of cell ci vs cell cjf-cjl.
+ * Checks bounding box distances and possibly atom pair distances.
+ */
+static void make_cluster_list_simple(const nbnxn_grid_t *gridj,
+ nbnxn_pairlist_t *nbl,
+ int ci, int cjf, int cjl,
+ gmx_bool remove_sub_diag,
+ const real *x_j,
+ real rl2, float rbb2,
+ int *ndistc)
+{
+ const nbnxn_list_work_t *work;
+
+ const nbnxn_bb_t *bb_ci;
+ const real *x_ci;
+
+ gmx_bool InRange;
+ real d2;
+ int cjf_gl, cjl_gl, cj;
+
+ work = nbl->work;
+
+ bb_ci = nbl->work->bb_ci;
+ x_ci = nbl->work->x_ci;
+
+ InRange = FALSE;
+ while (!InRange && cjf <= cjl)
+ {
+ d2 = subc_bb_dist2(0, bb_ci, cjf, gridj->bb);
+ *ndistc += 2;
+
+ /* Check if the distance is within the distance where
+ * we use only the bounding box distance rbb,
+ * or within the cut-off and there is at least one atom pair
+ * within the cut-off.
+ */
+ if (d2 < rbb2)
+ {
+ InRange = TRUE;
+ }
+ else if (d2 < rl2)
+ {
+ int i, j;
+
+ cjf_gl = gridj->cell0 + cjf;
+ for (i = 0; i < NBNXN_CPU_CLUSTER_I_SIZE && !InRange; i++)
+ {
+ for (j = 0; j < NBNXN_CPU_CLUSTER_I_SIZE; j++)
+ {
+ InRange = InRange ||
+ (sqr(x_ci[i*STRIDE_XYZ+XX] - x_j[(cjf_gl*NBNXN_CPU_CLUSTER_I_SIZE+j)*STRIDE_XYZ+XX]) +
+ sqr(x_ci[i*STRIDE_XYZ+YY] - x_j[(cjf_gl*NBNXN_CPU_CLUSTER_I_SIZE+j)*STRIDE_XYZ+YY]) +
+ sqr(x_ci[i*STRIDE_XYZ+ZZ] - x_j[(cjf_gl*NBNXN_CPU_CLUSTER_I_SIZE+j)*STRIDE_XYZ+ZZ]) < rl2);
+ }
+ }
+ *ndistc += NBNXN_CPU_CLUSTER_I_SIZE*NBNXN_CPU_CLUSTER_I_SIZE;
+ }
+ if (!InRange)
+ {
+ cjf++;
+ }
+ }
+ if (!InRange)
+ {
+ return;
+ }
+
+ InRange = FALSE;
+ while (!InRange && cjl > cjf)
+ {
+ d2 = subc_bb_dist2(0, bb_ci, cjl, gridj->bb);
+ *ndistc += 2;
+
+ /* Check if the distance is within the distance where
+ * we use only the bounding box distance rbb,
+ * or within the cut-off and there is at least one atom pair
+ * within the cut-off.
+ */
+ if (d2 < rbb2)
+ {
+ InRange = TRUE;
+ }
+ else if (d2 < rl2)
+ {
+ int i, j;
+
+ cjl_gl = gridj->cell0 + cjl;
+ for (i = 0; i < NBNXN_CPU_CLUSTER_I_SIZE && !InRange; i++)
+ {
+ for (j = 0; j < NBNXN_CPU_CLUSTER_I_SIZE; j++)
+ {
+ InRange = InRange ||
+ (sqr(x_ci[i*STRIDE_XYZ+XX] - x_j[(cjl_gl*NBNXN_CPU_CLUSTER_I_SIZE+j)*STRIDE_XYZ+XX]) +
+ sqr(x_ci[i*STRIDE_XYZ+YY] - x_j[(cjl_gl*NBNXN_CPU_CLUSTER_I_SIZE+j)*STRIDE_XYZ+YY]) +
+ sqr(x_ci[i*STRIDE_XYZ+ZZ] - x_j[(cjl_gl*NBNXN_CPU_CLUSTER_I_SIZE+j)*STRIDE_XYZ+ZZ]) < rl2);
+ }
+ }
+ *ndistc += NBNXN_CPU_CLUSTER_I_SIZE*NBNXN_CPU_CLUSTER_I_SIZE;
+ }
+ if (!InRange)
+ {
+ cjl--;
+ }
+ }
+
+ if (cjf <= cjl)
+ {
+ for (cj = cjf; cj <= cjl; cj++)
+ {
+ /* Store cj and the interaction mask */
+ nbl->cj[nbl->ncj].cj = gridj->cell0 + cj;
+ nbl->cj[nbl->ncj].excl = get_imask(remove_sub_diag, ci, cj);
+ nbl->ncj++;
+ }
+ /* Increase the closing index in i super-cell list */
+ nbl->ci[nbl->nci].cj_ind_end = nbl->ncj;
+ }
+}
+
+#ifdef GMX_NBNXN_SIMD_4XN
+#include "nbnxn_search_simd_4xn.h"
+#endif
+#ifdef GMX_NBNXN_SIMD_2XNN
+#include "nbnxn_search_simd_2xnn.h"
+#endif
+
+/* Plain C or SIMD4 code for making a pair list of super-cell sci vs scj.
+ * Checks bounding box distances and possibly atom pair distances.
+ */
+static void make_cluster_list_supersub(const nbnxn_grid_t *gridi,
+ const nbnxn_grid_t *gridj,
+ nbnxn_pairlist_t *nbl,
+ int sci, int scj,
+ gmx_bool sci_equals_scj,
+ int stride, const real *x,
+ real rl2, float rbb2,
+ int *ndistc)
+{
+ int na_c;
+ int npair;
+ int cjo, ci1, ci, cj, cj_gl;
+ int cj4_ind, cj_offset;
+ unsigned imask;
+ nbnxn_cj4_t *cj4;
+#ifdef NBNXN_BBXXXX
+ const float *pbb_ci;
+#else
+ const nbnxn_bb_t *bb_ci;
+#endif
+ const real *x_ci;
+ float *d2l, d2;
+ int w;
+#define PRUNE_LIST_CPU_ONE
+#ifdef PRUNE_LIST_CPU_ONE
+ int ci_last = -1;
+#endif
+
+ d2l = nbl->work->d2;
+
+#ifdef NBNXN_BBXXXX
+ pbb_ci = nbl->work->pbb_ci;
+#else
+ bb_ci = nbl->work->bb_ci;
+#endif
+ x_ci = nbl->work->x_ci;
+
+ na_c = gridj->na_c;
+
+ for (cjo = 0; cjo < gridj->nsubc[scj]; cjo++)
+ {
+ cj4_ind = (nbl->work->cj_ind >> NBNXN_GPU_JGROUP_SIZE_2LOG);
+ cj_offset = nbl->work->cj_ind - cj4_ind*NBNXN_GPU_JGROUP_SIZE;
+ cj4 = &nbl->cj4[cj4_ind];
+
+ cj = scj*GPU_NSUBCELL + cjo;
+
+ cj_gl = gridj->cell0*GPU_NSUBCELL + cj;
+
+ /* Initialize this j-subcell i-subcell list */
+ cj4->cj[cj_offset] = cj_gl;
+ imask = 0;
+
+ if (sci_equals_scj)
+ {
+ ci1 = cjo + 1;
+ }
+ else
+ {
+ ci1 = gridi->nsubc[sci];
+ }
+
+#ifdef NBNXN_BBXXXX
+ /* Determine all ci1 bb distances in one call with SIMD4 */
+ subc_bb_dist2_simd4_xxxx(gridj->pbb+(cj>>STRIDE_PBB_2LOG)*NNBSBB_XXXX+(cj & (STRIDE_PBB-1)),
+ ci1, pbb_ci, d2l);
+ *ndistc += na_c*2;
+#endif
+
+ npair = 0;
+ /* We use a fixed upper-bound instead of ci1 to help optimization */
+ for (ci = 0; ci < GPU_NSUBCELL; ci++)
+ {
+ if (ci == ci1)
+ {
+ break;
+ }
+
+#ifndef NBNXN_BBXXXX
+ /* Determine the bb distance between ci and cj */
+ d2l[ci] = subc_bb_dist2(ci, bb_ci, cj, gridj->bb);
+ *ndistc += 2;
+#endif
+ d2 = d2l[ci];
+
+#ifdef PRUNE_LIST_CPU_ALL
+ /* Check if the distance is within the distance where
+ * we use only the bounding box distance rbb,
+ * or within the cut-off and there is at least one atom pair
+ * within the cut-off. This check is very costly.
+ */
+ *ndistc += na_c*na_c;
+ if (d2 < rbb2 ||
+ (d2 < rl2 &&
+#ifdef NBNXN_PBB_SIMD4
+ subc_in_range_simd4
+#else
+ subc_in_range_x
+#endif
+ (na_c, ci, x_ci, cj_gl, stride, x, rl2)))
+#else
+ /* Check if the distance between the two bounding boxes
+ * in within the pair-list cut-off.
+ */
+ if (d2 < rl2)
+#endif
+ {
+ /* Flag this i-subcell to be taken into account */
+ imask |= (1U << (cj_offset*GPU_NSUBCELL+ci));
+
+#ifdef PRUNE_LIST_CPU_ONE
+ ci_last = ci;
+#endif
+
+ npair++;
+ }
+ }
+
+#ifdef PRUNE_LIST_CPU_ONE
+ /* If we only found 1 pair, check if any atoms are actually
+ * within the cut-off, so we could get rid of it.
+ */
+ if (npair == 1 && d2l[ci_last] >= rbb2)
+ {
+ /* Avoid using function pointers here, as it's slower */
+ if (
+#ifdef NBNXN_PBB_SIMD4
+ !subc_in_range_simd4
+#else
+ !subc_in_range_x
+#endif
+ (na_c, ci_last, x_ci, cj_gl, stride, x, rl2))
+ {
+ imask &= ~(1U << (cj_offset*GPU_NSUBCELL+ci_last));
+ npair--;
+ }
+ }
+#endif
+
+ if (npair > 0)
+ {
+ /* We have a useful sj entry, close it now */
+
+ /* Set the exclucions for the ci== sj entry.
+ * Here we don't bother to check if this entry is actually flagged,
+ * as it will nearly always be in the list.
+ */
+ if (sci_equals_scj)
+ {
+ set_self_and_newton_excls_supersub(nbl, cj4_ind, cj_offset, cjo);
+ }
+
+ /* Copy the cluster interaction mask to the list */
+ for (w = 0; w < NWARP; w++)
+ {
+ cj4->imei[w].imask |= imask;
+ }
+
+ nbl->work->cj_ind++;
+
+ /* Keep the count */
+ nbl->nci_tot += npair;
+
+ /* Increase the closing index in i super-cell list */
+ nbl->sci[nbl->nsci].cj4_ind_end =
+ ((nbl->work->cj_ind+NBNXN_GPU_JGROUP_SIZE-1) >> NBNXN_GPU_JGROUP_SIZE_2LOG);
+ }
+ }
+}
+
+/* Set all atom-pair exclusions from the topology stored in excl
+ * as masks in the pair-list for simple list i-entry nbl_ci
+ */
+static void set_ci_top_excls(const nbnxn_search_t nbs,
+ nbnxn_pairlist_t *nbl,
+ gmx_bool diagRemoved,
+ int na_ci_2log,
+ int na_cj_2log,
+ const nbnxn_ci_t *nbl_ci,
+ const t_blocka *excl)
+{
+ const int *cell;
+ int ci;
+ int cj_ind_first, cj_ind_last;
+ int cj_first, cj_last;
+ int ndirect;
+ int i, ai, aj, si, eind, ge, se;
+ int found, cj_ind_0, cj_ind_1, cj_ind_m;
+ int cj_m;
+ gmx_bool Found_si;
+ int si_ind;
+ nbnxn_excl_t *nbl_excl;
+ int inner_i, inner_e;
+
+ cell = nbs->cell;
+
+ if (nbl_ci->cj_ind_end == nbl_ci->cj_ind_start)
+ {
+ /* Empty list */
+ return;
+ }
+
+ ci = nbl_ci->ci;
+
+ cj_ind_first = nbl_ci->cj_ind_start;
+ cj_ind_last = nbl->ncj - 1;
+
+ cj_first = nbl->cj[cj_ind_first].cj;
+ cj_last = nbl->cj[cj_ind_last].cj;
+
+ /* Determine how many contiguous j-cells we have starting
+ * from the first i-cell. This number can be used to directly
+ * calculate j-cell indices for excluded atoms.
+ */
+ ndirect = 0;
+ if (na_ci_2log == na_cj_2log)
+ {
+ while (cj_ind_first + ndirect <= cj_ind_last &&
+ nbl->cj[cj_ind_first+ndirect].cj == ci + ndirect)
+ {
+ ndirect++;
+ }
+ }
+#ifdef NBNXN_SEARCH_BB_SIMD4
+ else
+ {
+ while (cj_ind_first + ndirect <= cj_ind_last &&
+ nbl->cj[cj_ind_first+ndirect].cj == ci_to_cj(na_cj_2log, ci) + ndirect)
+ {
+ ndirect++;
+ }
+ }
+#endif
+
+ /* Loop over the atoms in the i super-cell */
+ for (i = 0; i < nbl->na_sc; i++)
+ {
+ ai = nbs->a[ci*nbl->na_sc+i];
+ if (ai >= 0)
+ {
+ si = (i>>na_ci_2log);
+
+ /* Loop over the topology-based exclusions for this i-atom */
+ for (eind = excl->index[ai]; eind < excl->index[ai+1]; eind++)
+ {
+ aj = excl->a[eind];
+
+ if (aj == ai)
+ {
+ /* The self exclusion are already set, save some time */
+ continue;
+ }
+
+ ge = cell[aj];
+
+ /* Without shifts we only calculate interactions j>i
+ * for one-way pair-lists.
+ */
+ if (diagRemoved && ge <= ci*nbl->na_sc + i)
+ {
+ continue;
+ }
+
+ se = (ge >> na_cj_2log);
+
+ /* Could the cluster se be in our list? */
+ if (se >= cj_first && se <= cj_last)
+ {
+ if (se < cj_first + ndirect)
+ {
+ /* We can calculate cj_ind directly from se */
+ found = cj_ind_first + se - cj_first;
+ }
+ else
+ {
+ /* Search for se using bisection */
+ found = -1;
+ cj_ind_0 = cj_ind_first + ndirect;
+ cj_ind_1 = cj_ind_last + 1;
+ while (found == -1 && cj_ind_0 < cj_ind_1)
+ {
+ cj_ind_m = (cj_ind_0 + cj_ind_1)>>1;
+
+ cj_m = nbl->cj[cj_ind_m].cj;
+
+ if (se == cj_m)
+ {
+ found = cj_ind_m;
+ }
+ else if (se < cj_m)
+ {
+ cj_ind_1 = cj_ind_m;
+ }
+ else
+ {
+ cj_ind_0 = cj_ind_m + 1;
+ }
+ }
+ }
+
+ if (found >= 0)
+ {
+ inner_i = i - (si << na_ci_2log);
+ inner_e = ge - (se << na_cj_2log);
+
+ nbl->cj[found].excl &= ~(1U<<((inner_i<<na_cj_2log) + inner_e));
+/* The next code line is usually not needed. We do not want to version
+ * away the above line, because there is logic that relies on being
+ * able to detect easily whether any exclusions exist. */
+#if (defined GMX_CPU_ACCELERATION_IBM_QPX)
+ nbl->cj[found].interaction_mask_indices[inner_i] &= ~(1U << inner_e);
+#endif
+ }
+ }
+ }
+ }
+ }
+}
+
+/* Set all atom-pair exclusions from the topology stored in excl
+ * as masks in the pair-list for i-super-cell entry nbl_sci
+ */
+static void set_sci_top_excls(const nbnxn_search_t nbs,
+ nbnxn_pairlist_t *nbl,
+ gmx_bool diagRemoved,
+ int na_c_2log,
+ const nbnxn_sci_t *nbl_sci,
+ const t_blocka *excl)
+{
+ const int *cell;
+ int na_c;
+ int sci;
+ int cj_ind_first, cj_ind_last;
+ int cj_first, cj_last;
+ int ndirect;
+ int i, ai, aj, si, eind, ge, se;
+ int found, cj_ind_0, cj_ind_1, cj_ind_m;
+ int cj_m;
+ gmx_bool Found_si;
+ int si_ind;
+ nbnxn_excl_t *nbl_excl;
+ int inner_i, inner_e, w;
+
+ cell = nbs->cell;
+
+ na_c = nbl->na_ci;
+
+ if (nbl_sci->cj4_ind_end == nbl_sci->cj4_ind_start)
+ {
+ /* Empty list */
+ return;
+ }
+
+ sci = nbl_sci->sci;
+
+ cj_ind_first = nbl_sci->cj4_ind_start*NBNXN_GPU_JGROUP_SIZE;
+ cj_ind_last = nbl->work->cj_ind - 1;
+
+ cj_first = nbl->cj4[nbl_sci->cj4_ind_start].cj[0];
+ cj_last = nbl_cj(nbl, cj_ind_last);
+
+ /* Determine how many contiguous j-clusters we have starting
+ * from the first i-cluster. This number can be used to directly
+ * calculate j-cluster indices for excluded atoms.
+ */
+ ndirect = 0;
+ while (cj_ind_first + ndirect <= cj_ind_last &&
+ nbl_cj(nbl, cj_ind_first+ndirect) == sci*GPU_NSUBCELL + ndirect)
+ {
+ ndirect++;
+ }
+
+ /* Loop over the atoms in the i super-cell */
+ for (i = 0; i < nbl->na_sc; i++)
+ {
+ ai = nbs->a[sci*nbl->na_sc+i];
+ if (ai >= 0)
+ {
+ si = (i>>na_c_2log);
+
+ /* Loop over the topology-based exclusions for this i-atom */
+ for (eind = excl->index[ai]; eind < excl->index[ai+1]; eind++)
+ {
+ aj = excl->a[eind];
+
+ if (aj == ai)
+ {
+ /* The self exclusion are already set, save some time */
+ continue;
+ }
+
+ ge = cell[aj];
+
+ /* Without shifts we only calculate interactions j>i
+ * for one-way pair-lists.
+ */
+ if (diagRemoved && ge <= sci*nbl->na_sc + i)
+ {
+ continue;
+ }
+
+ se = ge>>na_c_2log;
+ /* Could the cluster se be in our list? */
+ if (se >= cj_first && se <= cj_last)
+ {
+ if (se < cj_first + ndirect)
+ {
+ /* We can calculate cj_ind directly from se */
+ found = cj_ind_first + se - cj_first;
+ }
+ else
+ {
+ /* Search for se using bisection */
+ found = -1;
+ cj_ind_0 = cj_ind_first + ndirect;
+ cj_ind_1 = cj_ind_last + 1;
+ while (found == -1 && cj_ind_0 < cj_ind_1)
+ {
+ cj_ind_m = (cj_ind_0 + cj_ind_1)>>1;
+
+ cj_m = nbl_cj(nbl, cj_ind_m);
+
+ if (se == cj_m)
+ {
+ found = cj_ind_m;
+ }
+ else if (se < cj_m)
+ {
+ cj_ind_1 = cj_ind_m;
+ }
+ else
+ {
+ cj_ind_0 = cj_ind_m + 1;
+ }
+ }
+ }
+
+ if (found >= 0)
+ {
+ inner_i = i - si*na_c;
+ inner_e = ge - se*na_c;
+
+/* Macro for getting the index of atom a within a cluster */
+#define AMODCJ4(a) ((a) & (NBNXN_GPU_JGROUP_SIZE - 1))
+/* Macro for converting an atom number to a cluster number */
+#define A2CJ4(a) ((a) >> NBNXN_GPU_JGROUP_SIZE_2LOG)
+/* Macro for getting the index of an i-atom within a warp */
+#define AMODWI(a) ((a) & (NBNXN_GPU_CLUSTER_SIZE/2 - 1))
+
+ if (nbl_imask0(nbl, found) & (1U << (AMODCJ4(found)*GPU_NSUBCELL + si)))
+ {
+ w = (inner_e >> 2);
+
+ get_nbl_exclusions_1(nbl, A2CJ4(found), w, &nbl_excl);
+
+ nbl_excl->pair[AMODWI(inner_e)*nbl->na_ci+inner_i] &=
+ ~(1U << (AMODCJ4(found)*GPU_NSUBCELL + si));
+ }
+
+#undef AMODCJ4
+#undef A2CJ4
+#undef AMODWI
+ }
+ }
+ }
+ }
+ }
+}
+
+/* Reallocate the simple ci list for at least n entries */
+static void nb_realloc_ci(nbnxn_pairlist_t *nbl, int n)
+{
+ nbl->ci_nalloc = over_alloc_small(n);
+ nbnxn_realloc_void((void **)&nbl->ci,
+ nbl->nci*sizeof(*nbl->ci),
+ nbl->ci_nalloc*sizeof(*nbl->ci),
+ nbl->alloc, nbl->free);
+}
+
+/* Reallocate the super-cell sci list for at least n entries */
+static void nb_realloc_sci(nbnxn_pairlist_t *nbl, int n)
+{
+ nbl->sci_nalloc = over_alloc_small(n);
+ nbnxn_realloc_void((void **)&nbl->sci,
+ nbl->nsci*sizeof(*nbl->sci),
+ nbl->sci_nalloc*sizeof(*nbl->sci),
+ nbl->alloc, nbl->free);
+}
+
+/* Make a new ci entry at index nbl->nci */
+static void new_ci_entry(nbnxn_pairlist_t *nbl, int ci, int shift, int flags)
+{
+ if (nbl->nci + 1 > nbl->ci_nalloc)
+ {
+ nb_realloc_ci(nbl, nbl->nci+1);
+ }
+ nbl->ci[nbl->nci].ci = ci;
+ nbl->ci[nbl->nci].shift = shift;
+ /* Store the interaction flags along with the shift */
+ nbl->ci[nbl->nci].shift |= flags;
+ nbl->ci[nbl->nci].cj_ind_start = nbl->ncj;
+ nbl->ci[nbl->nci].cj_ind_end = nbl->ncj;
+}
+
+/* Make a new sci entry at index nbl->nsci */
+static void new_sci_entry(nbnxn_pairlist_t *nbl, int sci, int shift)
+{
+ if (nbl->nsci + 1 > nbl->sci_nalloc)
+ {
+ nb_realloc_sci(nbl, nbl->nsci+1);
+ }
+ nbl->sci[nbl->nsci].sci = sci;
+ nbl->sci[nbl->nsci].shift = shift;
+ nbl->sci[nbl->nsci].cj4_ind_start = nbl->ncj4;
+ nbl->sci[nbl->nsci].cj4_ind_end = nbl->ncj4;
+}
+
+/* Sort the simple j-list cj on exclusions.
+ * Entries with exclusions will all be sorted to the beginning of the list.
+ */
+static void sort_cj_excl(nbnxn_cj_t *cj, int ncj,
+ nbnxn_list_work_t *work)
+{
+ int jnew, j;
+
+ if (ncj > work->cj_nalloc)
+ {
+ work->cj_nalloc = over_alloc_large(ncj);
+ srenew(work->cj, work->cj_nalloc);
+ }
+
+ /* Make a list of the j-cells involving exclusions */
+ jnew = 0;
+ for (j = 0; j < ncj; j++)
+ {
+ if (cj[j].excl != NBNXN_INTERACTION_MASK_ALL)
+ {
+ work->cj[jnew++] = cj[j];
+ }
+ }
+ /* Check if there are exclusions at all or not just the first entry */
+ if (!((jnew == 0) ||
+ (jnew == 1 && cj[0].excl != NBNXN_INTERACTION_MASK_ALL)))
+ {
+ for (j = 0; j < ncj; j++)
+ {
+ if (cj[j].excl == NBNXN_INTERACTION_MASK_ALL)
+ {
+ work->cj[jnew++] = cj[j];
+ }
+ }
+ for (j = 0; j < ncj; j++)
+ {
+ cj[j] = work->cj[j];
+ }
+ }
+}
+
+/* Close this simple list i entry */
+static void close_ci_entry_simple(nbnxn_pairlist_t *nbl)
+{
+ int jlen;
+
+ /* All content of the new ci entry have already been filled correctly,
+ * we only need to increase the count here (for non empty lists).
+ */
+ jlen = nbl->ci[nbl->nci].cj_ind_end - nbl->ci[nbl->nci].cj_ind_start;
+ if (jlen > 0)
+ {
+ sort_cj_excl(nbl->cj+nbl->ci[nbl->nci].cj_ind_start, jlen, nbl->work);
+
+ /* The counts below are used for non-bonded pair/flop counts
+ * and should therefore match the available kernel setups.
+ */
+ if (!(nbl->ci[nbl->nci].shift & NBNXN_CI_DO_COUL(0)))
+ {
+ nbl->work->ncj_noq += jlen;
+ }
+ else if ((nbl->ci[nbl->nci].shift & NBNXN_CI_HALF_LJ(0)) ||
+ !(nbl->ci[nbl->nci].shift & NBNXN_CI_DO_LJ(0)))
+ {
+ nbl->work->ncj_hlj += jlen;
+ }
+
+ nbl->nci++;
+ }
+}
+
+/* Split sci entry for load balancing on the GPU.
+ * Splitting ensures we have enough lists to fully utilize the whole GPU.
+ * With progBal we generate progressively smaller lists, which improves
+ * load balancing. As we only know the current count on our own thread,
+ * we will need to estimate the current total amount of i-entries.
+ * As the lists get concatenated later, this estimate depends
+ * both on nthread and our own thread index.
+ */
+static void split_sci_entry(nbnxn_pairlist_t *nbl,
+ int nsp_max_av, gmx_bool progBal, int nc_bal,
+ int thread, int nthread)
+{
+ int nsci_est;
+ int nsp_max;
+ int cj4_start, cj4_end, j4len, cj4;
+ int sci;
+ int nsp, nsp_sci, nsp_cj4, nsp_cj4_e, nsp_cj4_p;
+ int p;
+
+ if (progBal)
+ {
+ /* Estimate the total numbers of ci's of the nblist combined
+ * over all threads using the target number of ci's.
+ */
+ nsci_est = nc_bal*thread/nthread + nbl->nsci;
+
+ /* The first ci blocks should be larger, to avoid overhead.
+ * The last ci blocks should be smaller, to improve load balancing.
+ */
+ nsp_max = max(1,
+ nsp_max_av*nc_bal*3/(2*(nsci_est - 1 + nc_bal)));
+ }
+ else
+ {
+ nsp_max = nsp_max_av;
+ }
+
+ cj4_start = nbl->sci[nbl->nsci-1].cj4_ind_start;
+ cj4_end = nbl->sci[nbl->nsci-1].cj4_ind_end;
+ j4len = cj4_end - cj4_start;
+
+ if (j4len > 1 && j4len*GPU_NSUBCELL*NBNXN_GPU_JGROUP_SIZE > nsp_max)
+ {
+ /* Remove the last ci entry and process the cj4's again */
+ nbl->nsci -= 1;
+
+ sci = nbl->nsci;
+ nsp = 0;
+ nsp_sci = 0;
+ nsp_cj4_e = 0;
+ nsp_cj4 = 0;
+ for (cj4 = cj4_start; cj4 < cj4_end; cj4++)
+ {
+ nsp_cj4_p = nsp_cj4;
+ /* Count the number of cluster pairs in this cj4 group */
+ nsp_cj4 = 0;
+ for (p = 0; p < GPU_NSUBCELL*NBNXN_GPU_JGROUP_SIZE; p++)
+ {
+ nsp_cj4 += (nbl->cj4[cj4].imei[0].imask >> p) & 1;
+ }
+
+ if (nsp_cj4 > 0 && nsp + nsp_cj4 > nsp_max)
+ {
+ /* Split the list at cj4 */
+ nbl->sci[sci].cj4_ind_end = cj4;
+ /* Create a new sci entry */
+ sci++;
+ nbl->nsci++;
+ if (nbl->nsci+1 > nbl->sci_nalloc)
+ {
+ nb_realloc_sci(nbl, nbl->nsci+1);
+ }
+ nbl->sci[sci].sci = nbl->sci[nbl->nsci-1].sci;
+ nbl->sci[sci].shift = nbl->sci[nbl->nsci-1].shift;
+ nbl->sci[sci].cj4_ind_start = cj4;
+ nsp_sci = nsp;
+ nsp_cj4_e = nsp_cj4_p;
+ nsp = 0;
+ }
+ nsp += nsp_cj4;
+ }
+
+ /* Put the remaining cj4's in the last sci entry */
+ nbl->sci[sci].cj4_ind_end = cj4_end;
+
+ /* Possibly balance out the last two sci's
+ * by moving the last cj4 of the second last sci.
+ */
+ if (nsp_sci - nsp_cj4_e >= nsp + nsp_cj4_e)
+ {
+ nbl->sci[sci-1].cj4_ind_end--;
+ nbl->sci[sci].cj4_ind_start--;
+ }
+
+ nbl->nsci++;
+ }
+}
+
+/* Clost this super/sub list i entry */
+static void close_ci_entry_supersub(nbnxn_pairlist_t *nbl,
+ int nsp_max_av,
+ gmx_bool progBal, int nc_bal,
+ int thread, int nthread)
+{
+ int j4len, tlen;
+ int nb, b;
+
+ /* All content of the new ci entry have already been filled correctly,
+ * we only need to increase the count here (for non empty lists).
+ */
+ j4len = nbl->sci[nbl->nsci].cj4_ind_end - nbl->sci[nbl->nsci].cj4_ind_start;
+ if (j4len > 0)
+ {
+ /* We can only have complete blocks of 4 j-entries in a list,
+ * so round the count up before closing.
+ */
+ nbl->ncj4 = ((nbl->work->cj_ind + NBNXN_GPU_JGROUP_SIZE - 1) >> NBNXN_GPU_JGROUP_SIZE_2LOG);
+ nbl->work->cj_ind = nbl->ncj4*NBNXN_GPU_JGROUP_SIZE;
+
+ nbl->nsci++;
+
+ if (nsp_max_av > 0)
+ {
+ /* Measure the size of the new entry and potentially split it */
+ split_sci_entry(nbl, nsp_max_av, progBal, nc_bal, thread, nthread);
+ }
+ }
+}
+
+/* Syncs the working array before adding another grid pair to the list */
+static void sync_work(nbnxn_pairlist_t *nbl)
+{
+ if (!nbl->bSimple)
+ {
+ nbl->work->cj_ind = nbl->ncj4*NBNXN_GPU_JGROUP_SIZE;
+ nbl->work->cj4_init = nbl->ncj4;
+ }
+}
+
+/* Clears an nbnxn_pairlist_t data structure */
+static void clear_pairlist(nbnxn_pairlist_t *nbl)
+{
+ nbl->nci = 0;
+ nbl->nsci = 0;
+ nbl->ncj = 0;
+ nbl->ncj4 = 0;
+ nbl->nci_tot = 0;
+ nbl->nexcl = 1;
+
+ nbl->work->ncj_noq = 0;
+ nbl->work->ncj_hlj = 0;
+}
+
+/* Sets a simple list i-cell bounding box, including PBC shift */
+static gmx_inline void set_icell_bb_simple(const nbnxn_bb_t *bb, int ci,
+ real shx, real shy, real shz,
+ nbnxn_bb_t *bb_ci)
+{
+ bb_ci->lower[BB_X] = bb[ci].lower[BB_X] + shx;
+ bb_ci->lower[BB_Y] = bb[ci].lower[BB_Y] + shy;
+ bb_ci->lower[BB_Z] = bb[ci].lower[BB_Z] + shz;
+ bb_ci->upper[BB_X] = bb[ci].upper[BB_X] + shx;
+ bb_ci->upper[BB_Y] = bb[ci].upper[BB_Y] + shy;
+ bb_ci->upper[BB_Z] = bb[ci].upper[BB_Z] + shz;
+}
+
+#ifdef NBNXN_BBXXXX
+/* Sets a super-cell and sub cell bounding boxes, including PBC shift */
+static void set_icell_bbxxxx_supersub(const float *bb, int ci,
+ real shx, real shy, real shz,
+ float *bb_ci)
+{
+ int ia, m, i;
+
+ ia = ci*(GPU_NSUBCELL>>STRIDE_PBB_2LOG)*NNBSBB_XXXX;
+ for (m = 0; m < (GPU_NSUBCELL>>STRIDE_PBB_2LOG)*NNBSBB_XXXX; m += NNBSBB_XXXX)
+ {
+ for (i = 0; i < STRIDE_PBB; i++)
+ {
+ bb_ci[m+0*STRIDE_PBB+i] = bb[ia+m+0*STRIDE_PBB+i] + shx;
+ bb_ci[m+1*STRIDE_PBB+i] = bb[ia+m+1*STRIDE_PBB+i] + shy;
+ bb_ci[m+2*STRIDE_PBB+i] = bb[ia+m+2*STRIDE_PBB+i] + shz;
+ bb_ci[m+3*STRIDE_PBB+i] = bb[ia+m+3*STRIDE_PBB+i] + shx;
+ bb_ci[m+4*STRIDE_PBB+i] = bb[ia+m+4*STRIDE_PBB+i] + shy;
+ bb_ci[m+5*STRIDE_PBB+i] = bb[ia+m+5*STRIDE_PBB+i] + shz;
+ }
+ }
+}
+#endif
+
+/* Sets a super-cell and sub cell bounding boxes, including PBC shift */
+static void set_icell_bb_supersub(const nbnxn_bb_t *bb, int ci,
+ real shx, real shy, real shz,
+ nbnxn_bb_t *bb_ci)
+{
+ int i;
+
+ for (i = 0; i < GPU_NSUBCELL; i++)
+ {
+ set_icell_bb_simple(bb, ci*GPU_NSUBCELL+i,
+ shx, shy, shz,
+ &bb_ci[i]);
+ }
+}
+
+/* Copies PBC shifted i-cell atom coordinates x,y,z to working array */
+static void icell_set_x_simple(int ci,
+ real shx, real shy, real shz,
+ int gmx_unused na_c,
+ int stride, const real *x,
+ nbnxn_list_work_t *work)
+{
+ int ia, i;
+
+ ia = ci*NBNXN_CPU_CLUSTER_I_SIZE;
+
+ for (i = 0; i < NBNXN_CPU_CLUSTER_I_SIZE; i++)
+ {
+ work->x_ci[i*STRIDE_XYZ+XX] = x[(ia+i)*stride+XX] + shx;
+ work->x_ci[i*STRIDE_XYZ+YY] = x[(ia+i)*stride+YY] + shy;
+ work->x_ci[i*STRIDE_XYZ+ZZ] = x[(ia+i)*stride+ZZ] + shz;
+ }
+}
+
+/* Copies PBC shifted super-cell atom coordinates x,y,z to working array */
+static void icell_set_x_supersub(int ci,
+ real shx, real shy, real shz,
+ int na_c,
+ int stride, const real *x,
+ nbnxn_list_work_t *work)
+{
+ int ia, i;
+ real *x_ci;
+
+ x_ci = work->x_ci;
+
+ ia = ci*GPU_NSUBCELL*na_c;
+ for (i = 0; i < GPU_NSUBCELL*na_c; i++)
+ {
+ x_ci[i*DIM + XX] = x[(ia+i)*stride + XX] + shx;
+ x_ci[i*DIM + YY] = x[(ia+i)*stride + YY] + shy;
+ x_ci[i*DIM + ZZ] = x[(ia+i)*stride + ZZ] + shz;
+ }
+}
+
+#ifdef NBNXN_SEARCH_BB_SIMD4
+/* Copies PBC shifted super-cell packed atom coordinates to working array */
+static void icell_set_x_supersub_simd4(int ci,
+ real shx, real shy, real shz,
+ int na_c,
+ int stride, const real *x,
+ nbnxn_list_work_t *work)
+{
+ int si, io, ia, i, j;
+ real *x_ci;
+
+ x_ci = work->x_ci;
+
+ for (si = 0; si < GPU_NSUBCELL; si++)
+ {
+ for (i = 0; i < na_c; i += STRIDE_PBB)
+ {
+ io = si*na_c + i;
+ ia = ci*GPU_NSUBCELL*na_c + io;
+ for (j = 0; j < STRIDE_PBB; j++)
+ {
+ x_ci[io*DIM + j + XX*STRIDE_PBB] = x[(ia+j)*stride+XX] + shx;
+ x_ci[io*DIM + j + YY*STRIDE_PBB] = x[(ia+j)*stride+YY] + shy;
+ x_ci[io*DIM + j + ZZ*STRIDE_PBB] = x[(ia+j)*stride+ZZ] + shz;
+ }
+ }
+ }
+}
+#endif
+
+static real nbnxn_rlist_inc_nonloc_fac = 0.6;
+
+/* Due to the cluster size the effective pair-list is longer than
+ * that of a simple atom pair-list. This function gives the extra distance.
+ */
+real nbnxn_get_rlist_effective_inc(int cluster_size, real atom_density)
+{
+ return ((0.5 + nbnxn_rlist_inc_nonloc_fac)*sqr(((cluster_size) - 1.0)/(cluster_size))*pow((cluster_size)/(atom_density), 1.0/3.0));
+}
+
+/* Estimates the interaction volume^2 for non-local interactions */
+static real nonlocal_vol2(const gmx_domdec_zones_t *zones, rvec ls, real r)
+{
+ int z, d;
+ real cl, ca, za;
+ real vold_est;
+ real vol2_est_tot;
+
+ vol2_est_tot = 0;
+
+ /* Here we simply add up the volumes of 1, 2 or 3 1D decomposition
+ * not home interaction volume^2. As these volumes are not additive,
+ * this is an overestimate, but it would only be significant in the limit
+ * of small cells, where we anyhow need to split the lists into
+ * as small parts as possible.
+ */
+
+ for (z = 0; z < zones->n; z++)
+ {
+ if (zones->shift[z][XX] + zones->shift[z][YY] + zones->shift[z][ZZ] == 1)
+ {
+ cl = 0;
+ ca = 1;
+ za = 1;
+ for (d = 0; d < DIM; d++)
+ {
+ if (zones->shift[z][d] == 0)
+ {
+ cl += 0.5*ls[d];
+ ca *= ls[d];
+ za *= zones->size[z].x1[d] - zones->size[z].x0[d];
+ }
+ }
+
+ /* 4 octants of a sphere */
+ vold_est = 0.25*M_PI*r*r*r*r;
+ /* 4 quarter pie slices on the edges */
+ vold_est += 4*cl*M_PI/6.0*r*r*r;
+ /* One rectangular volume on a face */
+ vold_est += ca*0.5*r*r;
+
+ vol2_est_tot += vold_est*za;
+ }
+ }
+
+ return vol2_est_tot;
+}
+
+/* Estimates the average size of a full j-list for super/sub setup */
+static int get_nsubpair_max(const nbnxn_search_t nbs,
+ int iloc,
+ real rlist,
+ int min_ci_balanced)
+{
+ const nbnxn_grid_t *grid;
+ rvec ls;
+ real xy_diag2, r_eff_sup, vol_est, nsp_est, nsp_est_nl;
+ int nsubpair_max;
+
+ grid = &nbs->grid[0];
+
+ ls[XX] = (grid->c1[XX] - grid->c0[XX])/(grid->ncx*GPU_NSUBCELL_X);
+ ls[YY] = (grid->c1[YY] - grid->c0[YY])/(grid->ncy*GPU_NSUBCELL_Y);
+ ls[ZZ] = (grid->c1[ZZ] - grid->c0[ZZ])*grid->ncx*grid->ncy/(grid->nc*GPU_NSUBCELL_Z);
+
+ /* The average squared length of the diagonal of a sub cell */
+ xy_diag2 = ls[XX]*ls[XX] + ls[YY]*ls[YY] + ls[ZZ]*ls[ZZ];
+
+ /* The formulas below are a heuristic estimate of the average nsj per si*/
+ r_eff_sup = rlist + nbnxn_rlist_inc_nonloc_fac*sqr((grid->na_c - 1.0)/grid->na_c)*sqrt(xy_diag2/3);
+
+ if (!nbs->DomDec || nbs->zones->n == 1)
+ {
+ nsp_est_nl = 0;
+ }
+ else
+ {
+ nsp_est_nl =
+ sqr(grid->atom_density/grid->na_c)*
+ nonlocal_vol2(nbs->zones, ls, r_eff_sup);
+ }
+
+ if (LOCAL_I(iloc))
+ {
+ /* Sub-cell interacts with itself */
+ vol_est = ls[XX]*ls[YY]*ls[ZZ];
+ /* 6/2 rectangular volume on the faces */
+ vol_est += (ls[XX]*ls[YY] + ls[XX]*ls[ZZ] + ls[YY]*ls[ZZ])*r_eff_sup;
+ /* 12/2 quarter pie slices on the edges */
+ vol_est += 2*(ls[XX] + ls[YY] + ls[ZZ])*0.25*M_PI*sqr(r_eff_sup);
+ /* 4 octants of a sphere */
+ vol_est += 0.5*4.0/3.0*M_PI*pow(r_eff_sup, 3);
+
+ nsp_est = grid->nsubc_tot*vol_est*grid->atom_density/grid->na_c;
+
+ /* Subtract the non-local pair count */
+ nsp_est -= nsp_est_nl;
+
+ if (debug)
+ {
+ fprintf(debug, "nsp_est local %5.1f non-local %5.1f\n",
+ nsp_est, nsp_est_nl);
+ }
+ }
+ else
+ {
+ nsp_est = nsp_est_nl;
+ }
+
+ if (min_ci_balanced <= 0 || grid->nc >= min_ci_balanced || grid->nc == 0)
+ {
+ /* We don't need to worry */
+ nsubpair_max = -1;
+ }
+ else
+ {
+ /* Thus the (average) maximum j-list size should be as follows */
+ nsubpair_max = max(1, (int)(nsp_est/min_ci_balanced+0.5));
+
+ /* Since the target value is a maximum (this avoids high outliers,
+ * which lead to load imbalance), not average, we add half the
+ * number of pairs in a cj4 block to get the average about right.
+ */
+ nsubpair_max += GPU_NSUBCELL*NBNXN_GPU_JGROUP_SIZE/2;
+ }
+
+ if (debug)
+ {
+ fprintf(debug, "nbl nsp estimate %.1f, nsubpair_max %d\n",
+ nsp_est, nsubpair_max);
+ }
+
+ return nsubpair_max;
+}
+
+/* Debug list print function */
+static void print_nblist_ci_cj(FILE *fp, const nbnxn_pairlist_t *nbl)
+{
+ int i, j;
+
+ for (i = 0; i < nbl->nci; i++)
+ {
+ fprintf(fp, "ci %4d shift %2d ncj %3d\n",
+ nbl->ci[i].ci, nbl->ci[i].shift,
+ nbl->ci[i].cj_ind_end - nbl->ci[i].cj_ind_start);
+
+ for (j = nbl->ci[i].cj_ind_start; j < nbl->ci[i].cj_ind_end; j++)
+ {
+ fprintf(fp, " cj %5d imask %x\n",
+ nbl->cj[j].cj,
+ nbl->cj[j].excl);
+ }
+ }
+}
+
+/* Debug list print function */
+static void print_nblist_sci_cj(FILE *fp, const nbnxn_pairlist_t *nbl)
+{
+ int i, j4, j, ncp, si;
+
+ for (i = 0; i < nbl->nsci; i++)
+ {
+ fprintf(fp, "ci %4d shift %2d ncj4 %2d\n",
+ nbl->sci[i].sci, nbl->sci[i].shift,
+ nbl->sci[i].cj4_ind_end - nbl->sci[i].cj4_ind_start);
+
+ ncp = 0;
+ for (j4 = nbl->sci[i].cj4_ind_start; j4 < nbl->sci[i].cj4_ind_end; j4++)
+ {
+ for (j = 0; j < NBNXN_GPU_JGROUP_SIZE; j++)
+ {
+ fprintf(fp, " sj %5d imask %x\n",
+ nbl->cj4[j4].cj[j],
+ nbl->cj4[j4].imei[0].imask);
+ for (si = 0; si < GPU_NSUBCELL; si++)
+ {
+ if (nbl->cj4[j4].imei[0].imask & (1U << (j*GPU_NSUBCELL + si)))
+ {
+ ncp++;
+ }
+ }
+ }
+ }
+ fprintf(fp, "ci %4d shift %2d ncj4 %2d ncp %3d\n",
+ nbl->sci[i].sci, nbl->sci[i].shift,
+ nbl->sci[i].cj4_ind_end - nbl->sci[i].cj4_ind_start,
+ ncp);
+ }
+}
+
+/* Combine pair lists *nbl generated on multiple threads nblc */
+static void combine_nblists(int nnbl, nbnxn_pairlist_t **nbl,
+ nbnxn_pairlist_t *nblc)
+{
+ int nsci, ncj4, nexcl;
+ int n, i;
+
+ if (nblc->bSimple)
+ {
+ gmx_incons("combine_nblists does not support simple lists");
+ }
+
+ nsci = nblc->nsci;
+ ncj4 = nblc->ncj4;
+ nexcl = nblc->nexcl;
+ for (i = 0; i < nnbl; i++)
+ {
+ nsci += nbl[i]->nsci;
+ ncj4 += nbl[i]->ncj4;
+ nexcl += nbl[i]->nexcl;
+ }
+
+ if (nsci > nblc->sci_nalloc)
+ {
+ nb_realloc_sci(nblc, nsci);
+ }
+ if (ncj4 > nblc->cj4_nalloc)
+ {
+ nblc->cj4_nalloc = over_alloc_small(ncj4);
+ nbnxn_realloc_void((void **)&nblc->cj4,
+ nblc->ncj4*sizeof(*nblc->cj4),
+ nblc->cj4_nalloc*sizeof(*nblc->cj4),
+ nblc->alloc, nblc->free);
+ }
+ if (nexcl > nblc->excl_nalloc)
+ {
+ nblc->excl_nalloc = over_alloc_small(nexcl);
+ nbnxn_realloc_void((void **)&nblc->excl,
+ nblc->nexcl*sizeof(*nblc->excl),
+ nblc->excl_nalloc*sizeof(*nblc->excl),
+ nblc->alloc, nblc->free);
+ }
+
+ /* Each thread should copy its own data to the combined arrays,
+ * as otherwise data will go back and forth between different caches.
+ */
+#pragma omp parallel for num_threads(gmx_omp_nthreads_get(emntPairsearch)) schedule(static)
+ for (n = 0; n < nnbl; n++)
+ {
+ int sci_offset;
+ int cj4_offset;
+ int ci_offset;
+ int excl_offset;
+ int i, j4;
+ const nbnxn_pairlist_t *nbli;
+
+ /* Determine the offset in the combined data for our thread */
+ sci_offset = nblc->nsci;
+ cj4_offset = nblc->ncj4;
+ ci_offset = nblc->nci_tot;
+ excl_offset = nblc->nexcl;
+
+ for (i = 0; i < n; i++)
+ {
+ sci_offset += nbl[i]->nsci;
+ cj4_offset += nbl[i]->ncj4;
+ ci_offset += nbl[i]->nci_tot;
+ excl_offset += nbl[i]->nexcl;
+ }
+
+ nbli = nbl[n];
+
+ for (i = 0; i < nbli->nsci; i++)
+ {
+ nblc->sci[sci_offset+i] = nbli->sci[i];
+ nblc->sci[sci_offset+i].cj4_ind_start += cj4_offset;
+ nblc->sci[sci_offset+i].cj4_ind_end += cj4_offset;
+ }
+
+ for (j4 = 0; j4 < nbli->ncj4; j4++)
+ {
+ nblc->cj4[cj4_offset+j4] = nbli->cj4[j4];
+ nblc->cj4[cj4_offset+j4].imei[0].excl_ind += excl_offset;
+ nblc->cj4[cj4_offset+j4].imei[1].excl_ind += excl_offset;
+ }
+
+ for (j4 = 0; j4 < nbli->nexcl; j4++)
+ {
+ nblc->excl[excl_offset+j4] = nbli->excl[j4];
+ }
+ }
+
+ for (n = 0; n < nnbl; n++)
+ {
+ nblc->nsci += nbl[n]->nsci;
+ nblc->ncj4 += nbl[n]->ncj4;
+ nblc->nci_tot += nbl[n]->nci_tot;
+ nblc->nexcl += nbl[n]->nexcl;
+ }
+}
+
+/* Returns the next ci to be processes by our thread */
+static gmx_bool next_ci(const nbnxn_grid_t *grid,
+ int conv,
+ int nth, int ci_block,
+ int *ci_x, int *ci_y,
+ int *ci_b, int *ci)
+{
+ (*ci_b)++;
+ (*ci)++;
+
+ if (*ci_b == ci_block)
+ {
+ /* Jump to the next block assigned to this task */
+ *ci += (nth - 1)*ci_block;
+ *ci_b = 0;
+ }
+
+ if (*ci >= grid->nc*conv)
+ {
+ return FALSE;
+ }
+
+ while (*ci >= grid->cxy_ind[*ci_x*grid->ncy + *ci_y + 1]*conv)
+ {
+ *ci_y += 1;
+ if (*ci_y == grid->ncy)
+ {
+ *ci_x += 1;
+ *ci_y = 0;
+ }
+ }
+
+ return TRUE;
+}
+
+/* Returns the distance^2 for which we put cell pairs in the list
+ * without checking atom pair distances. This is usually < rlist^2.
+ */
+static float boundingbox_only_distance2(const nbnxn_grid_t *gridi,
+ const nbnxn_grid_t *gridj,
+ real rlist,
+ gmx_bool simple)
+{
+ /* If the distance between two sub-cell bounding boxes is less
+ * than this distance, do not check the distance between
+ * all particle pairs in the sub-cell, since then it is likely
+ * that the box pair has atom pairs within the cut-off.
+ * We use the nblist cut-off minus 0.5 times the average x/y diagonal
+ * spacing of the sub-cells. Around 40% of the checked pairs are pruned.
+ * Using more than 0.5 gains at most 0.5%.
+ * If forces are calculated more than twice, the performance gain
+ * in the force calculation outweighs the cost of checking.
+ * Note that with subcell lists, the atom-pair distance check
+ * is only performed when only 1 out of 8 sub-cells in within range,
+ * this is because the GPU is much faster than the cpu.
+ */
+ real bbx, bby;
+ real rbb2;
+
+ bbx = 0.5*(gridi->sx + gridj->sx);
+ bby = 0.5*(gridi->sy + gridj->sy);
+ if (!simple)
+ {
+ bbx /= GPU_NSUBCELL_X;
+ bby /= GPU_NSUBCELL_Y;
+ }
+
+ rbb2 = sqr(max(0, rlist - 0.5*sqrt(bbx*bbx + bby*bby)));
+
+#ifndef GMX_DOUBLE
+ return rbb2;
+#else
+ return (float)((1+GMX_FLOAT_EPS)*rbb2);
+#endif
+}
+
+static int get_ci_block_size(const nbnxn_grid_t *gridi,
+ gmx_bool bDomDec, int nth)
+{
+ const int ci_block_enum = 5;
+ const int ci_block_denom = 11;
+ const int ci_block_min_atoms = 16;
+ int ci_block;
+
+ /* Here we decide how to distribute the blocks over the threads.
+ * We use prime numbers to try to avoid that the grid size becomes
+ * a multiple of the number of threads, which would lead to some
+ * threads getting "inner" pairs and others getting boundary pairs,
+ * which in turns will lead to load imbalance between threads.
+ * Set the block size as 5/11/ntask times the average number of cells
+ * in a y,z slab. This should ensure a quite uniform distribution
+ * of the grid parts of the different thread along all three grid
+ * zone boundaries with 3D domain decomposition. At the same time
+ * the blocks will not become too small.
+ */
+ ci_block = (gridi->nc*ci_block_enum)/(ci_block_denom*gridi->ncx*nth);
+
+ /* Ensure the blocks are not too small: avoids cache invalidation */
+ if (ci_block*gridi->na_sc < ci_block_min_atoms)
+ {
+ ci_block = (ci_block_min_atoms + gridi->na_sc - 1)/gridi->na_sc;
+ }
+
+ /* Without domain decomposition
+ * or with less than 3 blocks per task, divide in nth blocks.
+ */
+ if (!bDomDec || ci_block*3*nth > gridi->nc)
+ {
+ ci_block = (gridi->nc + nth - 1)/nth;
+ }
+
+ return ci_block;
+}
+
+/* Generates the part of pair-list nbl assigned to our thread */
+static void nbnxn_make_pairlist_part(const nbnxn_search_t nbs,
+ const nbnxn_grid_t *gridi,
+ const nbnxn_grid_t *gridj,
+ nbnxn_search_work_t *work,
+ const nbnxn_atomdata_t *nbat,
+ const t_blocka *excl,
+ real rlist,
+ int nb_kernel_type,
+ int ci_block,
+ gmx_bool bFBufferFlag,
+ int nsubpair_max,
+ gmx_bool progBal,
+ int min_ci_balanced,
+ int th, int nth,
+ nbnxn_pairlist_t *nbl)
+{
+ int na_cj_2log;
+ matrix box;
+ real rl2;
+ float rbb2;
+ int d;
+ int ci_b, ci, ci_x, ci_y, ci_xy, cj;
+ ivec shp;
+ int tx, ty, tz;
+ int shift;
+ gmx_bool bMakeList;
+ real shx, shy, shz;
+ int conv_i, cell0_i;
+ const nbnxn_bb_t *bb_i = NULL;
+#ifdef NBNXN_BBXXXX
+ const float *pbb_i = NULL;
+#endif
+ const float *bbcz_i, *bbcz_j;
+ const int *flags_i;
+ real bx0, bx1, by0, by1, bz0, bz1;
+ real bz1_frac;
+ real d2cx, d2z, d2z_cx, d2z_cy, d2zx, d2zxy, d2xy;
+ int cxf, cxl, cyf, cyf_x, cyl;
+ int cx, cy;
+ int c0, c1, cs, cf, cl;
+ int ndistc;
+ int ncpcheck;
+ int gridi_flag_shift = 0, gridj_flag_shift = 0;
+ unsigned *gridj_flag = NULL;
+ int ncj_old_i, ncj_old_j;
+
+ nbs_cycle_start(&work->cc[enbsCCsearch]);
+
+ if (gridj->bSimple != nbl->bSimple)
+ {
+ gmx_incons("Grid incompatible with pair-list");
+ }
+
+ sync_work(nbl);
+ nbl->na_sc = gridj->na_sc;
+ nbl->na_ci = gridj->na_c;
+ nbl->na_cj = nbnxn_kernel_to_cj_size(nb_kernel_type);
+ na_cj_2log = get_2log(nbl->na_cj);
+
+ nbl->rlist = rlist;
+
+ if (bFBufferFlag)
+ {
+ /* Determine conversion of clusters to flag blocks */
+ gridi_flag_shift = 0;
+ while ((nbl->na_ci<<gridi_flag_shift) < NBNXN_BUFFERFLAG_SIZE)
+ {
+ gridi_flag_shift++;
+ }
+ gridj_flag_shift = 0;
+ while ((nbl->na_cj<<gridj_flag_shift) < NBNXN_BUFFERFLAG_SIZE)
+ {
+ gridj_flag_shift++;
+ }
+
+ gridj_flag = work->buffer_flags.flag;
+ }
+
+ copy_mat(nbs->box, box);
+
+ rl2 = nbl->rlist*nbl->rlist;
+
+ rbb2 = boundingbox_only_distance2(gridi, gridj, nbl->rlist, nbl->bSimple);
+
+ if (debug)
+ {
+ fprintf(debug, "nbl bounding box only distance %f\n", sqrt(rbb2));
+ }
+
+ /* Set the shift range */
+ for (d = 0; d < DIM; d++)
+ {
+ /* Check if we need periodicity shifts.
+ * Without PBC or with domain decomposition we don't need them.
+ */
+ if (d >= ePBC2npbcdim(nbs->ePBC) || nbs->dd_dim[d])
+ {
+ shp[d] = 0;
+ }
+ else
+ {
+ if (d == XX &&
+ box[XX][XX] - fabs(box[YY][XX]) - fabs(box[ZZ][XX]) < sqrt(rl2))
+ {
+ shp[d] = 2;
+ }
+ else
+ {
+ shp[d] = 1;
+ }
+ }
+ }
+
+ if (nbl->bSimple && !gridi->bSimple)
+ {
+ conv_i = gridi->na_sc/gridj->na_sc;
+ bb_i = gridi->bb_simple;
+ bbcz_i = gridi->bbcz_simple;
+ flags_i = gridi->flags_simple;
+ }
+ else
+ {
+ conv_i = 1;
+#ifdef NBNXN_BBXXXX
+ if (gridi->bSimple)
+ {
+ bb_i = gridi->bb;
+ }
+ else
+ {
+ pbb_i = gridi->pbb;
+ }
+#else
+ /* We use the normal bounding box format for both grid types */
+ bb_i = gridi->bb;
+#endif
+ bbcz_i = gridi->bbcz;
+ flags_i = gridi->flags;
+ }
+ cell0_i = gridi->cell0*conv_i;
+
+ bbcz_j = gridj->bbcz;
+
+ if (conv_i != 1)
+ {
+ /* Blocks of the conversion factor - 1 give a large repeat count
+ * combined with a small block size. This should result in good
+ * load balancing for both small and large domains.
+ */
+ ci_block = conv_i - 1;
+ }
+ if (debug)
+ {
+ fprintf(debug, "nbl nc_i %d col.av. %.1f ci_block %d\n",
+ gridi->nc, gridi->nc/(double)(gridi->ncx*gridi->ncy), ci_block);
+ }
+
+ ndistc = 0;
+ ncpcheck = 0;
+
+ /* Initially ci_b and ci to 1 before where we want them to start,
+ * as they will both be incremented in next_ci.
+ */
+ ci_b = -1;
+ ci = th*ci_block - 1;
+ ci_x = 0;
+ ci_y = 0;
+ while (next_ci(gridi, conv_i, nth, ci_block, &ci_x, &ci_y, &ci_b, &ci))
+ {
+ if (nbl->bSimple && flags_i[ci] == 0)
+ {
+ continue;
+ }
+
+ ncj_old_i = nbl->ncj;
+
+ d2cx = 0;
+ if (gridj != gridi && shp[XX] == 0)
+ {
+ if (nbl->bSimple)
+ {
+ bx1 = bb_i[ci].upper[BB_X];
+ }
+ else
+ {
+ bx1 = gridi->c0[XX] + (ci_x+1)*gridi->sx;
+ }
+ if (bx1 < gridj->c0[XX])
+ {
+ d2cx = sqr(gridj->c0[XX] - bx1);
+
+ if (d2cx >= rl2)
+ {
+ continue;
+ }
+ }
+ }
+
+ ci_xy = ci_x*gridi->ncy + ci_y;
+
+ /* Loop over shift vectors in three dimensions */
+ for (tz = -shp[ZZ]; tz <= shp[ZZ]; tz++)
+ {
+ shz = tz*box[ZZ][ZZ];
+
+ bz0 = bbcz_i[ci*NNBSBB_D ] + shz;
+ bz1 = bbcz_i[ci*NNBSBB_D+1] + shz;
+
+ if (tz == 0)
+ {
+ d2z = 0;
+ }
+ else if (tz < 0)
+ {
+ d2z = sqr(bz1);
+ }
+ else
+ {
+ d2z = sqr(bz0 - box[ZZ][ZZ]);
+ }
+
+ d2z_cx = d2z + d2cx;
+
+ if (d2z_cx >= rl2)
+ {
+ continue;
+ }
+
+ bz1_frac =
+ bz1/((real)(gridi->cxy_ind[ci_xy+1] - gridi->cxy_ind[ci_xy]));
+ if (bz1_frac < 0)
+ {
+ bz1_frac = 0;
+ }
+ /* The check with bz1_frac close to or larger than 1 comes later */
+
+ for (ty = -shp[YY]; ty <= shp[YY]; ty++)
+ {
+ shy = ty*box[YY][YY] + tz*box[ZZ][YY];
+
+ if (nbl->bSimple)
+ {
+ by0 = bb_i[ci].lower[BB_Y] + shy;
+ by1 = bb_i[ci].upper[BB_Y] + shy;
+ }
+ else
+ {
+ by0 = gridi->c0[YY] + (ci_y )*gridi->sy + shy;
+ by1 = gridi->c0[YY] + (ci_y+1)*gridi->sy + shy;
+ }
+
+ get_cell_range(by0, by1,
+ gridj->ncy, gridj->c0[YY], gridj->sy, gridj->inv_sy,
+ d2z_cx, rl2,
+ &cyf, &cyl);
+
+ if (cyf > cyl)
+ {
+ continue;
+ }
+
+ d2z_cy = d2z;
+ if (by1 < gridj->c0[YY])
+ {
+ d2z_cy += sqr(gridj->c0[YY] - by1);
+ }
+ else if (by0 > gridj->c1[YY])
+ {
+ d2z_cy += sqr(by0 - gridj->c1[YY]);
+ }
+
+ for (tx = -shp[XX]; tx <= shp[XX]; tx++)
+ {
+ shift = XYZ2IS(tx, ty, tz);
+
+#ifdef NBNXN_SHIFT_BACKWARD
+ if (gridi == gridj && shift > CENTRAL)
+ {
+ continue;
+ }
+#endif
+
+ shx = tx*box[XX][XX] + ty*box[YY][XX] + tz*box[ZZ][XX];
+
+ if (nbl->bSimple)
+ {
+ bx0 = bb_i[ci].lower[BB_X] + shx;
+ bx1 = bb_i[ci].upper[BB_X] + shx;
+ }
+ else
+ {
+ bx0 = gridi->c0[XX] + (ci_x )*gridi->sx + shx;
+ bx1 = gridi->c0[XX] + (ci_x+1)*gridi->sx + shx;
+ }
+
+ get_cell_range(bx0, bx1,
+ gridj->ncx, gridj->c0[XX], gridj->sx, gridj->inv_sx,
+ d2z_cy, rl2,
+ &cxf, &cxl);
+
+ if (cxf > cxl)
+ {
+ continue;
+ }
+
+ if (nbl->bSimple)
+ {
+ new_ci_entry(nbl, cell0_i+ci, shift, flags_i[ci]);
+ }
+ else
+ {
+ new_sci_entry(nbl, cell0_i+ci, shift);
+ }
+
+#ifndef NBNXN_SHIFT_BACKWARD
+ if (cxf < ci_x)
+#else
+ if (shift == CENTRAL && gridi == gridj &&
+ cxf < ci_x)
+#endif
+ {
+ /* Leave the pairs with i > j.
+ * x is the major index, so skip half of it.
+ */
+ cxf = ci_x;
+ }
+
+ if (nbl->bSimple)
+ {
+ set_icell_bb_simple(bb_i, ci, shx, shy, shz,
+ nbl->work->bb_ci);
+ }
+ else
+ {
+#ifdef NBNXN_BBXXXX
+ set_icell_bbxxxx_supersub(pbb_i, ci, shx, shy, shz,
+ nbl->work->pbb_ci);
+#else
+ set_icell_bb_supersub(bb_i, ci, shx, shy, shz,
+ nbl->work->bb_ci);
+#endif
+ }
+
+ nbs->icell_set_x(cell0_i+ci, shx, shy, shz,
+ gridi->na_c, nbat->xstride, nbat->x,
+ nbl->work);
+
+ for (cx = cxf; cx <= cxl; cx++)
+ {
+ d2zx = d2z;
+ if (gridj->c0[XX] + cx*gridj->sx > bx1)
+ {
+ d2zx += sqr(gridj->c0[XX] + cx*gridj->sx - bx1);
+ }
+ else if (gridj->c0[XX] + (cx+1)*gridj->sx < bx0)
+ {
+ d2zx += sqr(gridj->c0[XX] + (cx+1)*gridj->sx - bx0);
+ }
+
+#ifndef NBNXN_SHIFT_BACKWARD
+ if (gridi == gridj &&
+ cx == 0 && cyf < ci_y)
+#else
+ if (gridi == gridj &&
+ cx == 0 && shift == CENTRAL && cyf < ci_y)
+#endif
+ {
+ /* Leave the pairs with i > j.
+ * Skip half of y when i and j have the same x.
+ */
+ cyf_x = ci_y;
+ }
+ else
+ {
+ cyf_x = cyf;
+ }
+
+ for (cy = cyf_x; cy <= cyl; cy++)
+ {
+ c0 = gridj->cxy_ind[cx*gridj->ncy+cy];
+ c1 = gridj->cxy_ind[cx*gridj->ncy+cy+1];
+#ifdef NBNXN_SHIFT_BACKWARD
+ if (gridi == gridj &&
+ shift == CENTRAL && c0 < ci)
+ {
+ c0 = ci;
+ }
+#endif
+
+ d2zxy = d2zx;
+ if (gridj->c0[YY] + cy*gridj->sy > by1)
+ {
+ d2zxy += sqr(gridj->c0[YY] + cy*gridj->sy - by1);
+ }
+ else if (gridj->c0[YY] + (cy+1)*gridj->sy < by0)
+ {
+ d2zxy += sqr(gridj->c0[YY] + (cy+1)*gridj->sy - by0);
+ }
+ if (c1 > c0 && d2zxy < rl2)
+ {
+ cs = c0 + (int)(bz1_frac*(c1 - c0));
+ if (cs >= c1)
+ {
+ cs = c1 - 1;
+ }
+
+ d2xy = d2zxy - d2z;
+
+ /* Find the lowest cell that can possibly
+ * be within range.
+ */
+ cf = cs;
+ while (cf > c0 &&
+ (bbcz_j[cf*NNBSBB_D+1] >= bz0 ||
+ d2xy + sqr(bbcz_j[cf*NNBSBB_D+1] - bz0) < rl2))
+ {
+ cf--;
+ }
+
+ /* Find the highest cell that can possibly
+ * be within range.
+ */
+ cl = cs;
+ while (cl < c1-1 &&
+ (bbcz_j[cl*NNBSBB_D] <= bz1 ||
+ d2xy + sqr(bbcz_j[cl*NNBSBB_D] - bz1) < rl2))
+ {
+ cl++;
+ }
+
+#ifdef NBNXN_REFCODE
+ {
+ /* Simple reference code, for debugging,
+ * overrides the more complex code above.
+ */
+ int k;
+ cf = c1;
+ cl = -1;
+ for (k = c0; k < c1; k++)
+ {
+ if (box_dist2(bx0, bx1, by0, by1, bz0, bz1, bb+k) < rl2 &&
+ k < cf)
+ {
+ cf = k;
+ }
+ if (box_dist2(bx0, bx1, by0, by1, bz0, bz1, bb+k) < rl2 &&
+ k > cl)
+ {
+ cl = k;
+ }
+ }
+ }
+#endif
+
+ if (gridi == gridj)
+ {
+ /* We want each atom/cell pair only once,
+ * only use cj >= ci.
+ */
+#ifndef NBNXN_SHIFT_BACKWARD
+ cf = max(cf, ci);
+#else
+ if (shift == CENTRAL)
+ {
+ cf = max(cf, ci);
+ }
+#endif
+ }
+
+ if (cf <= cl)
+ {
+ /* For f buffer flags with simple lists */
+ ncj_old_j = nbl->ncj;
+
+ switch (nb_kernel_type)
+ {
+ case nbnxnk4x4_PlainC:
+ check_subcell_list_space_simple(nbl, cl-cf+1);
+
+ make_cluster_list_simple(gridj,
+ nbl, ci, cf, cl,
+ (gridi == gridj && shift == CENTRAL),
+ nbat->x,
+ rl2, rbb2,
+ &ndistc);
+ break;
+#ifdef GMX_NBNXN_SIMD_4XN
+ case nbnxnk4xN_SIMD_4xN:
+ check_subcell_list_space_simple(nbl, ci_to_cj(na_cj_2log, cl-cf)+2);
+ make_cluster_list_simd_4xn(gridj,
+ nbl, ci, cf, cl,
+ (gridi == gridj && shift == CENTRAL),
+ nbat->x,
+ rl2, rbb2,
+ &ndistc);
+ break;
+#endif
+#ifdef GMX_NBNXN_SIMD_2XNN
+ case nbnxnk4xN_SIMD_2xNN:
+ check_subcell_list_space_simple(nbl, ci_to_cj(na_cj_2log, cl-cf)+2);
+ make_cluster_list_simd_2xnn(gridj,
+ nbl, ci, cf, cl,
+ (gridi == gridj && shift == CENTRAL),
+ nbat->x,
+ rl2, rbb2,
+ &ndistc);
+ break;
+#endif
+ case nbnxnk8x8x8_PlainC:
+ case nbnxnk8x8x8_CUDA:
+ check_subcell_list_space_supersub(nbl, cl-cf+1);
+ for (cj = cf; cj <= cl; cj++)
+ {
+ make_cluster_list_supersub(gridi, gridj,
+ nbl, ci, cj,
+ (gridi == gridj && shift == CENTRAL && ci == cj),
+ nbat->xstride, nbat->x,
+ rl2, rbb2,
+ &ndistc);
+ }
+ break;
+ }
+ ncpcheck += cl - cf + 1;
+
+ if (bFBufferFlag && nbl->ncj > ncj_old_j)
+ {
+ int cbf, cbl, cb;
+
+ cbf = nbl->cj[ncj_old_j].cj >> gridj_flag_shift;
+ cbl = nbl->cj[nbl->ncj-1].cj >> gridj_flag_shift;
+ for (cb = cbf; cb <= cbl; cb++)
+ {
+ gridj_flag[cb] = 1U<<th;
+ }
+ }
+ }
+ }
+ }
+ }
+
+ /* Set the exclusions for this ci list */
+ if (nbl->bSimple)
+ {
+ set_ci_top_excls(nbs,
+ nbl,
+ shift == CENTRAL && gridi == gridj,
+ gridj->na_c_2log,
+ na_cj_2log,
+ &(nbl->ci[nbl->nci]),
+ excl);
+ }
+ else
+ {
+ set_sci_top_excls(nbs,
+ nbl,
+ shift == CENTRAL && gridi == gridj,
+ gridj->na_c_2log,
+ &(nbl->sci[nbl->nsci]),
+ excl);
+ }
+
+ /* Close this ci list */
+ if (nbl->bSimple)
+ {
+ close_ci_entry_simple(nbl);
+ }
+ else
+ {
+ close_ci_entry_supersub(nbl,
+ nsubpair_max,
+ progBal, min_ci_balanced,
+ th, nth);
+ }
+ }
+ }
+ }
+
+ if (bFBufferFlag && nbl->ncj > ncj_old_i)
+ {
+ work->buffer_flags.flag[(gridi->cell0+ci)>>gridi_flag_shift] = 1U<<th;
+ }
+ }
+
+ work->ndistc = ndistc;
+
+ nbs_cycle_stop(&work->cc[enbsCCsearch]);
+
+ if (debug)
+ {
+ fprintf(debug, "number of distance checks %d\n", ndistc);
+ fprintf(debug, "ncpcheck %s %d\n", gridi == gridj ? "local" : "non-local",
+ ncpcheck);
+
+ if (nbl->bSimple)
+ {
+ print_nblist_statistics_simple(debug, nbl, nbs, rlist);
+ }
+ else
+ {
+ print_nblist_statistics_supersub(debug, nbl, nbs, rlist);
+ }
+
+ }
+}
+
+static void reduce_buffer_flags(const nbnxn_search_t nbs,
+ int nsrc,
+ const nbnxn_buffer_flags_t *dest)
+{
+ int s, b;
+ const unsigned *flag;
+
+ for (s = 0; s < nsrc; s++)
+ {
+ flag = nbs->work[s].buffer_flags.flag;
+
+ for (b = 0; b < dest->nflag; b++)
+ {
+ dest->flag[b] |= flag[b];
+ }
+ }
+}
+
+static void print_reduction_cost(const nbnxn_buffer_flags_t *flags, int nout)
+{
+ int nelem, nkeep, ncopy, nred, b, c, out;
+
+ nelem = 0;
+ nkeep = 0;
+ ncopy = 0;
+ nred = 0;
+ for (b = 0; b < flags->nflag; b++)
+ {
+ if (flags->flag[b] == 1)
+ {
+ /* Only flag 0 is set, no copy of reduction required */
+ nelem++;
+ nkeep++;
+ }
+ else if (flags->flag[b] > 0)
+ {
+ c = 0;
+ for (out = 0; out < nout; out++)
+ {
+ if (flags->flag[b] & (1U<<out))
+ {
+ c++;
+ }
+ }
+ nelem += c;
+ if (c == 1)
+ {
+ ncopy++;
+ }
+ else
+ {
+ nred += c;
+ }
+ }
+ }
+
+ fprintf(debug, "nbnxn reduction: #flag %d #list %d elem %4.2f, keep %4.2f copy %4.2f red %4.2f\n",
+ flags->nflag, nout,
+ nelem/(double)(flags->nflag),
+ nkeep/(double)(flags->nflag),
+ ncopy/(double)(flags->nflag),
+ nred/(double)(flags->nflag));
+}
+
+/* Perform a count (linear) sort to sort the smaller lists to the end.
+ * This avoids load imbalance on the GPU, as large lists will be
+ * scheduled and executed first and the smaller lists later.
+ * Load balancing between multi-processors only happens at the end
+ * and there smaller lists lead to more effective load balancing.
+ * The sorting is done on the cj4 count, not on the actual pair counts.
+ * Not only does this make the sort faster, but it also results in
+ * better load balancing than using a list sorted on exact load.
+ * This function swaps the pointer in the pair list to avoid a copy operation.
+ */
+static void sort_sci(nbnxn_pairlist_t *nbl)
+{
+ nbnxn_list_work_t *work;
+ int m, i, s, s0, s1;
+ nbnxn_sci_t *sci_sort;
+
+ if (nbl->ncj4 <= nbl->nsci)
+ {
+ /* nsci = 0 or all sci have size 1, sorting won't change the order */
+ return;
+ }
+
+ work = nbl->work;
+
+ /* We will distinguish differences up to double the average */
+ m = (2*nbl->ncj4)/nbl->nsci;
+
+ if (m + 1 > work->sort_nalloc)
+ {
+ work->sort_nalloc = over_alloc_large(m + 1);
+ srenew(work->sort, work->sort_nalloc);
+ }
+
+ if (work->sci_sort_nalloc != nbl->sci_nalloc)
+ {
+ work->sci_sort_nalloc = nbl->sci_nalloc;
+ nbnxn_realloc_void((void **)&work->sci_sort,
+ 0,
+ work->sci_sort_nalloc*sizeof(*work->sci_sort),
+ nbl->alloc, nbl->free);
+ }
+
+ /* Count the entries of each size */
+ for (i = 0; i <= m; i++)
+ {
+ work->sort[i] = 0;
+ }
+ for (s = 0; s < nbl->nsci; s++)
+ {
+ i = min(m, nbl->sci[s].cj4_ind_end - nbl->sci[s].cj4_ind_start);
+ work->sort[i]++;
+ }
+ /* Calculate the offset for each count */
+ s0 = work->sort[m];
+ work->sort[m] = 0;
+ for (i = m - 1; i >= 0; i--)
+ {
+ s1 = work->sort[i];
+ work->sort[i] = work->sort[i + 1] + s0;
+ s0 = s1;
+ }
+
+ /* Sort entries directly into place */
+ sci_sort = work->sci_sort;
+ for (s = 0; s < nbl->nsci; s++)
+ {
+ i = min(m, nbl->sci[s].cj4_ind_end - nbl->sci[s].cj4_ind_start);
+ sci_sort[work->sort[i]++] = nbl->sci[s];
+ }
+
+ /* Swap the sci pointers so we use the new, sorted list */
+ work->sci_sort = nbl->sci;
+ nbl->sci = sci_sort;
+}
+
+/* Make a local or non-local pair-list, depending on iloc */
+void nbnxn_make_pairlist(const nbnxn_search_t nbs,
+ nbnxn_atomdata_t *nbat,
+ const t_blocka *excl,
+ real rlist,
+ int min_ci_balanced,
+ nbnxn_pairlist_set_t *nbl_list,
+ int iloc,
+ int nb_kernel_type,
+ t_nrnb *nrnb)
+{
+ nbnxn_grid_t *gridi, *gridj;
+ gmx_bool bGPUCPU;
+ int nzi, zi, zj0, zj1, zj;
+ int nsubpair_max;
+ int th;
+ int nnbl;
+ nbnxn_pairlist_t **nbl;
+ int ci_block;
+ gmx_bool CombineNBLists;
+ gmx_bool progBal;
+ int np_tot, np_noq, np_hlj, nap;
+
+ /* Check if we are running hybrid GPU + CPU nbnxn mode */
+ bGPUCPU = (!nbs->grid[0].bSimple && nbl_list->bSimple);
+
+ nnbl = nbl_list->nnbl;
+ nbl = nbl_list->nbl;
+ CombineNBLists = nbl_list->bCombined;
+
+ if (debug)
+ {
+ fprintf(debug, "ns making %d nblists\n", nnbl);
+ }
+
+ nbat->bUseBufferFlags = (nbat->nout > 1);
+ /* We should re-init the flags before making the first list */
+ if (nbat->bUseBufferFlags && (LOCAL_I(iloc) || bGPUCPU))
+ {
+ init_buffer_flags(&nbat->buffer_flags, nbat->natoms);
+ }
+
+ if (nbl_list->bSimple)
+ {
+ switch (nb_kernel_type)
+ {
+#ifdef GMX_NBNXN_SIMD_4XN
+ case nbnxnk4xN_SIMD_4xN:
+ nbs->icell_set_x = icell_set_x_simd_4xn;
+ break;
+#endif
+#ifdef GMX_NBNXN_SIMD_2XNN
+ case nbnxnk4xN_SIMD_2xNN:
+ nbs->icell_set_x = icell_set_x_simd_2xnn;
+ break;
+#endif
+ default:
+ nbs->icell_set_x = icell_set_x_simple;
+ break;
+ }
+ }
+ else
+ {
+#ifdef NBNXN_SEARCH_BB_SIMD4
+ nbs->icell_set_x = icell_set_x_supersub_simd4;
+#else
+ nbs->icell_set_x = icell_set_x_supersub;
+#endif
+ }
+
+ if (LOCAL_I(iloc))
+ {
+ /* Only zone (grid) 0 vs 0 */
+ nzi = 1;
+ zj0 = 0;
+ zj1 = 1;
+ }
+ else
+ {
+ nzi = nbs->zones->nizone;
+ }
+
+ if (!nbl_list->bSimple && min_ci_balanced > 0)
+ {
+ nsubpair_max = get_nsubpair_max(nbs, iloc, rlist, min_ci_balanced);
+ }
+ else
+ {
+ nsubpair_max = 0;
+ }
+
+ /* Clear all pair-lists */
+ for (th = 0; th < nnbl; th++)
+ {
+ clear_pairlist(nbl[th]);
+ }
+
+ for (zi = 0; zi < nzi; zi++)
+ {
+ gridi = &nbs->grid[zi];
+
+ if (NONLOCAL_I(iloc))
+ {
+ zj0 = nbs->zones->izone[zi].j0;
+ zj1 = nbs->zones->izone[zi].j1;
+ if (zi == 0)
+ {
+ zj0++;
+ }
+ }
+ for (zj = zj0; zj < zj1; zj++)
+ {
+ gridj = &nbs->grid[zj];
+
+ if (debug)
+ {
+ fprintf(debug, "ns search grid %d vs %d\n", zi, zj);
+ }
+
+ nbs_cycle_start(&nbs->cc[enbsCCsearch]);
+
+ if (nbl[0]->bSimple && !gridi->bSimple)
+ {
+ /* Hybrid list, determine blocking later */
+ ci_block = 0;
+ }
+ else
+ {
+ ci_block = get_ci_block_size(gridi, nbs->DomDec, nnbl);
+ }
+
+#pragma omp parallel for num_threads(nnbl) schedule(static)
+ for (th = 0; th < nnbl; th++)
+ {
+ /* Re-init the thread-local work flag data before making
+ * the first list (not an elegant conditional).
+ */
+ if (nbat->bUseBufferFlags && ((zi == 0 && zj == 0) ||
+ (bGPUCPU && zi == 0 && zj == 1)))
+ {
+ init_buffer_flags(&nbs->work[th].buffer_flags, nbat->natoms);
+ }
+
+ if (CombineNBLists && th > 0)
+ {
+ clear_pairlist(nbl[th]);
+ }
+
+ /* With GPU: generate progressively smaller lists for
+ * load balancing for local only or non-local with 2 zones.
+ */
+ progBal = (LOCAL_I(iloc) || nbs->zones->n <= 2);
+
+ /* Divide the i super cell equally over the nblists */
+ nbnxn_make_pairlist_part(nbs, gridi, gridj,
+ &nbs->work[th], nbat, excl,
+ rlist,
+ nb_kernel_type,
+ ci_block,
+ nbat->bUseBufferFlags,
+ nsubpair_max,
+ progBal, min_ci_balanced,
+ th, nnbl,
+ nbl[th]);
+ }
+ nbs_cycle_stop(&nbs->cc[enbsCCsearch]);
+
+ np_tot = 0;
+ np_noq = 0;
+ np_hlj = 0;
+ for (th = 0; th < nnbl; th++)
+ {
+ inc_nrnb(nrnb, eNR_NBNXN_DIST2, nbs->work[th].ndistc);
+
+ if (nbl_list->bSimple)
+ {
+ np_tot += nbl[th]->ncj;
+ np_noq += nbl[th]->work->ncj_noq;
+ np_hlj += nbl[th]->work->ncj_hlj;
+ }
+ else
+ {
+ /* This count ignores potential subsequent pair pruning */
+ np_tot += nbl[th]->nci_tot;
+ }
+ }
+ nap = nbl[0]->na_ci*nbl[0]->na_cj;
+ nbl_list->natpair_ljq = (np_tot - np_noq)*nap - np_hlj*nap/2;
+ nbl_list->natpair_lj = np_noq*nap;
+ nbl_list->natpair_q = np_hlj*nap/2;
+
+ if (CombineNBLists && nnbl > 1)
+ {
+ nbs_cycle_start(&nbs->cc[enbsCCcombine]);
+
+ combine_nblists(nnbl-1, nbl+1, nbl[0]);
+
+ nbs_cycle_stop(&nbs->cc[enbsCCcombine]);
+ }
+ }
+ }
+
+ if (!nbl_list->bSimple)
+ {
+ /* Sort the entries on size, large ones first */
+ if (CombineNBLists || nnbl == 1)
+ {
+ sort_sci(nbl[0]);
+ }
+ else
+ {
+#pragma omp parallel for num_threads(nnbl) schedule(static)
+ for (th = 0; th < nnbl; th++)
+ {
+ sort_sci(nbl[th]);
+ }
+ }
+ }
+
+ if (nbat->bUseBufferFlags)
+ {
+ reduce_buffer_flags(nbs, nnbl, &nbat->buffer_flags);
+ }
+
+ /* Special performance logging stuff (env.var. GMX_NBNXN_CYCLE) */
+ if (LOCAL_I(iloc))
+ {
+ nbs->search_count++;
+ }
+ if (nbs->print_cycles &&
+ (!nbs->DomDec || (nbs->DomDec && !LOCAL_I(iloc))) &&
+ nbs->search_count % 100 == 0)
+ {
+ nbs_cycle_print(stderr, nbs);
+ }
+
+ if (debug && (CombineNBLists && nnbl > 1))
+ {
+ if (nbl[0]->bSimple)
+ {
+ print_nblist_statistics_simple(debug, nbl[0], nbs, rlist);
+ }
+ else
+ {
+ print_nblist_statistics_supersub(debug, nbl[0], nbs, rlist);
+ }
+ }
+
+ if (debug)
+ {
+ if (gmx_debug_at)
+ {
+ if (nbl[0]->bSimple)
+ {
+ print_nblist_ci_cj(debug, nbl[0]);
+ }
+ else
+ {
+ print_nblist_sci_cj(debug, nbl[0]);
+ }
+ }
+
+ if (nbat->bUseBufferFlags)
+ {
+ print_reduction_cost(&nbat->buffer_flags, nnbl);
+ }
+ }
+}
--- /dev/null
- float cycles_pme, cycles_force;
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * VERSION 3.2.0
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2004, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * GROwing Monsters And Cloning Shrimps
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdio.h>
+#ifdef HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif
+#include <math.h>
+#include "typedefs.h"
+#include "string2.h"
+#include "smalloc.h"
+#include "names.h"
+#include "mvdata.h"
+#include "txtdump.h"
+#include "pbc.h"
+#include "chargegroup.h"
+#include "vec.h"
+#include "nrnb.h"
+#include "mshift.h"
+#include "mdrun.h"
+#include "sim_util.h"
+#include "update.h"
+#include "physics.h"
+#include "main.h"
+#include "mdatoms.h"
+#include "force.h"
+#include "bondf.h"
+#include "pme.h"
+#include "disre.h"
+#include "orires.h"
+#include "network.h"
+#include "calcmu.h"
+#include "constr.h"
+#include "xvgr.h"
+#include "copyrite.h"
+#include "pull_rotation.h"
+#include "gmx_random.h"
+#include "domdec.h"
+#include "partdec.h"
+#include "genborn.h"
+#include "nbnxn_atomdata.h"
+#include "nbnxn_search.h"
+#include "nbnxn_kernels/nbnxn_kernel_ref.h"
+#include "nbnxn_kernels/simd_4xn/nbnxn_kernel_simd_4xn.h"
+#include "nbnxn_kernels/simd_2xnn/nbnxn_kernel_simd_2xnn.h"
+#include "nbnxn_kernels/nbnxn_kernel_gpu_ref.h"
+
+#include "gromacs/timing/wallcycle.h"
+#include "gromacs/timing/walltime_accounting.h"
+#include "gromacs/utility/gmxmpi.h"
+
+#include "adress.h"
+#include "qmmm.h"
+
+#include "nbnxn_cuda_data_mgmt.h"
+#include "nbnxn_cuda/nbnxn_cuda.h"
+
+void print_time(FILE *out,
+ gmx_walltime_accounting_t walltime_accounting,
+ gmx_large_int_t step,
+ t_inputrec *ir,
+ t_commrec gmx_unused *cr)
+{
+ time_t finish;
+ char timebuf[STRLEN];
+ double dt, elapsed_seconds, time_per_step;
+ char buf[48];
+
+#ifndef GMX_THREAD_MPI
+ if (!PAR(cr))
+#endif
+ {
+ fprintf(out, "\r");
+ }
+ fprintf(out, "step %s", gmx_step_str(step, buf));
+ if ((step >= ir->nstlist))
+ {
+ double seconds_since_epoch = gmx_gettime();
+ elapsed_seconds = seconds_since_epoch - walltime_accounting_get_start_time_stamp(walltime_accounting);
+ time_per_step = elapsed_seconds/(step - ir->init_step + 1);
+ dt = (ir->nsteps + ir->init_step - step) * time_per_step;
+
+ if (ir->nsteps >= 0)
+ {
+ if (dt >= 300)
+ {
+ finish = (time_t) (seconds_since_epoch + dt);
+ gmx_ctime_r(&finish, timebuf, STRLEN);
+ sprintf(buf, "%s", timebuf);
+ buf[strlen(buf)-1] = '\0';
+ fprintf(out, ", will finish %s", buf);
+ }
+ else
+ {
+ fprintf(out, ", remaining wall clock time: %5d s ", (int)dt);
+ }
+ }
+ else
+ {
+ fprintf(out, " performance: %.1f ns/day ",
+ ir->delta_t/1000*24*60*60/time_per_step);
+ }
+ }
+#ifndef GMX_THREAD_MPI
+ if (PAR(cr))
+ {
+ fprintf(out, "\n");
+ }
+#endif
+
+ fflush(out);
+}
+
+void print_date_and_time(FILE *fplog, int nodeid, const char *title,
+ const gmx_walltime_accounting_t walltime_accounting)
+{
+ int i;
+ char timebuf[STRLEN];
+ char time_string[STRLEN];
+ time_t tmptime;
+
+ if (fplog)
+ {
+ if (walltime_accounting != NULL)
+ {
+ tmptime = (time_t) walltime_accounting_get_start_time_stamp(walltime_accounting);
+ gmx_ctime_r(&tmptime, timebuf, STRLEN);
+ }
+ else
+ {
+ tmptime = (time_t) gmx_gettime();
+ gmx_ctime_r(&tmptime, timebuf, STRLEN);
+ }
+ for (i = 0; timebuf[i] >= ' '; i++)
+ {
+ time_string[i] = timebuf[i];
+ }
+ time_string[i] = '\0';
+
+ fprintf(fplog, "%s on node %d %s\n", title, nodeid, time_string);
+ }
+}
+
+static void sum_forces(int start, int end, rvec f[], rvec flr[])
+{
+ int i;
+
+ if (gmx_debug_at)
+ {
+ pr_rvecs(debug, 0, "fsr", f+start, end-start);
+ pr_rvecs(debug, 0, "flr", flr+start, end-start);
+ }
+ for (i = start; (i < end); i++)
+ {
+ rvec_inc(f[i], flr[i]);
+ }
+}
+
+/*
+ * calc_f_el calculates forces due to an electric field.
+ *
+ * force is kJ mol^-1 nm^-1 = e * kJ mol^-1 nm^-1 / e
+ *
+ * Et[] contains the parameters for the time dependent
+ * part of the field (not yet used).
+ * Ex[] contains the parameters for
+ * the spatial dependent part of the field. You can have cool periodic
+ * fields in principle, but only a constant field is supported
+ * now.
+ * The function should return the energy due to the electric field
+ * (if any) but for now returns 0.
+ *
+ * WARNING:
+ * There can be problems with the virial.
+ * Since the field is not self-consistent this is unavoidable.
+ * For neutral molecules the virial is correct within this approximation.
+ * For neutral systems with many charged molecules the error is small.
+ * But for systems with a net charge or a few charged molecules
+ * the error can be significant when the field is high.
+ * Solution: implement a self-consitent electric field into PME.
+ */
+static void calc_f_el(FILE *fp, int start, int homenr,
+ real charge[], rvec f[],
+ t_cosines Ex[], t_cosines Et[], double t)
+{
+ rvec Ext;
+ real t0;
+ int i, m;
+
+ for (m = 0; (m < DIM); m++)
+ {
+ if (Et[m].n > 0)
+ {
+ if (Et[m].n == 3)
+ {
+ t0 = Et[m].a[1];
+ Ext[m] = cos(Et[m].a[0]*(t-t0))*exp(-sqr(t-t0)/(2.0*sqr(Et[m].a[2])));
+ }
+ else
+ {
+ Ext[m] = cos(Et[m].a[0]*t);
+ }
+ }
+ else
+ {
+ Ext[m] = 1.0;
+ }
+ if (Ex[m].n > 0)
+ {
+ /* Convert the field strength from V/nm to MD-units */
+ Ext[m] *= Ex[m].a[0]*FIELDFAC;
+ for (i = start; (i < start+homenr); i++)
+ {
+ f[i][m] += charge[i]*Ext[m];
+ }
+ }
+ else
+ {
+ Ext[m] = 0;
+ }
+ }
+ if (fp != NULL)
+ {
+ fprintf(fp, "%10g %10g %10g %10g #FIELD\n", t,
+ Ext[XX]/FIELDFAC, Ext[YY]/FIELDFAC, Ext[ZZ]/FIELDFAC);
+ }
+}
+
+static void calc_virial(int start, int homenr, rvec x[], rvec f[],
+ tensor vir_part, t_graph *graph, matrix box,
+ t_nrnb *nrnb, const t_forcerec *fr, int ePBC)
+{
+ int i, j;
+ tensor virtest;
+
+ /* The short-range virial from surrounding boxes */
+ clear_mat(vir_part);
+ calc_vir(SHIFTS, fr->shift_vec, fr->fshift, vir_part, ePBC == epbcSCREW, box);
+ inc_nrnb(nrnb, eNR_VIRIAL, SHIFTS);
+
+ /* Calculate partial virial, for local atoms only, based on short range.
+ * Total virial is computed in global_stat, called from do_md
+ */
+ f_calc_vir(start, start+homenr, x, f, vir_part, graph, box);
+ inc_nrnb(nrnb, eNR_VIRIAL, homenr);
+
+ /* Add position restraint contribution */
+ for (i = 0; i < DIM; i++)
+ {
+ vir_part[i][i] += fr->vir_diag_posres[i];
+ }
+
+ /* Add wall contribution */
+ for (i = 0; i < DIM; i++)
+ {
+ vir_part[i][ZZ] += fr->vir_wall_z[i];
+ }
+
+ if (debug)
+ {
+ pr_rvecs(debug, 0, "vir_part", vir_part, DIM);
+ }
+}
+
+static void posres_wrapper(FILE *fplog,
+ int flags,
+ gmx_bool bSepDVDL,
+ t_inputrec *ir,
+ t_nrnb *nrnb,
+ gmx_localtop_t *top,
+ matrix box, rvec x[],
+ gmx_enerdata_t *enerd,
+ real *lambda,
+ t_forcerec *fr)
+{
+ t_pbc pbc;
+ real v, dvdl;
+ int i;
+
+ /* Position restraints always require full pbc */
+ set_pbc(&pbc, ir->ePBC, box);
+ dvdl = 0;
+ v = posres(top->idef.il[F_POSRES].nr, top->idef.il[F_POSRES].iatoms,
+ top->idef.iparams_posres,
+ (const rvec*)x, fr->f_novirsum, fr->vir_diag_posres,
+ ir->ePBC == epbcNONE ? NULL : &pbc,
+ lambda[efptRESTRAINT], &dvdl,
+ fr->rc_scaling, fr->ePBC, fr->posres_com, fr->posres_comB);
+ if (bSepDVDL)
+ {
+ gmx_print_sepdvdl(fplog, interaction_function[F_POSRES].longname, v, dvdl);
+ }
+ enerd->term[F_POSRES] += v;
+ /* If just the force constant changes, the FEP term is linear,
+ * but if k changes, it is not.
+ */
+ enerd->dvdl_nonlin[efptRESTRAINT] += dvdl;
+ inc_nrnb(nrnb, eNR_POSRES, top->idef.il[F_POSRES].nr/2);
+
+ if ((ir->fepvals->n_lambda > 0) && (flags & GMX_FORCE_DHDL))
+ {
+ for (i = 0; i < enerd->n_lambda; i++)
+ {
+ real dvdl_dum, lambda_dum;
+
+ lambda_dum = (i == 0 ? lambda[efptRESTRAINT] : ir->fepvals->all_lambda[efptRESTRAINT][i-1]);
+ v = posres(top->idef.il[F_POSRES].nr, top->idef.il[F_POSRES].iatoms,
+ top->idef.iparams_posres,
+ (const rvec*)x, NULL, NULL,
+ ir->ePBC == epbcNONE ? NULL : &pbc, lambda_dum, &dvdl,
+ fr->rc_scaling, fr->ePBC, fr->posres_com, fr->posres_comB);
+ enerd->enerpart_lambda[i] += v;
+ }
+ }
+}
+
+static void pull_potential_wrapper(FILE *fplog,
+ gmx_bool bSepDVDL,
+ t_commrec *cr,
+ t_inputrec *ir,
+ matrix box, rvec x[],
+ rvec f[],
+ tensor vir_force,
+ t_mdatoms *mdatoms,
+ gmx_enerdata_t *enerd,
+ real *lambda,
+ double t)
+{
+ t_pbc pbc;
+ real dvdl;
+
+ /* Calculate the center of mass forces, this requires communication,
+ * which is why pull_potential is called close to other communication.
+ * The virial contribution is calculated directly,
+ * which is why we call pull_potential after calc_virial.
+ */
+ set_pbc(&pbc, ir->ePBC, box);
+ dvdl = 0;
+ enerd->term[F_COM_PULL] +=
+ pull_potential(ir->ePull, ir->pull, mdatoms, &pbc,
+ cr, t, lambda[efptRESTRAINT], x, f, vir_force, &dvdl);
+ if (bSepDVDL)
+ {
+ gmx_print_sepdvdl(fplog, "Com pull", enerd->term[F_COM_PULL], dvdl);
+ }
+ enerd->dvdl_lin[efptRESTRAINT] += dvdl;
+}
+
+static void pme_receive_force_ener(FILE *fplog,
+ gmx_bool bSepDVDL,
+ t_commrec *cr,
+ gmx_wallcycle_t wcycle,
+ gmx_enerdata_t *enerd,
+ t_forcerec *fr)
+{
+ real e, v, dvdl;
+ float cycles_ppdpme, cycles_seppme;
+
+ cycles_ppdpme = wallcycle_stop(wcycle, ewcPPDURINGPME);
+ dd_cycles_add(cr->dd, cycles_ppdpme, ddCyclPPduringPME);
+
+ /* In case of node-splitting, the PP nodes receive the long-range
+ * forces, virial and energy from the PME nodes here.
+ */
+ wallcycle_start(wcycle, ewcPP_PMEWAITRECVF);
+ dvdl = 0;
+ gmx_pme_receive_f(cr, fr->f_novirsum, fr->vir_el_recip, &e, &dvdl,
+ &cycles_seppme);
+ if (bSepDVDL)
+ {
+ gmx_print_sepdvdl(fplog, "PME mesh", e, dvdl);
+ }
+ enerd->term[F_COUL_RECIP] += e;
+ enerd->dvdl_lin[efptCOUL] += dvdl;
+ if (wcycle)
+ {
+ dd_cycles_add(cr->dd, cycles_seppme, ddCyclPME);
+ }
+ wallcycle_stop(wcycle, ewcPP_PMEWAITRECVF);
+}
+
+static void print_large_forces(FILE *fp, t_mdatoms *md, t_commrec *cr,
+ gmx_large_int_t step, real pforce, rvec *x, rvec *f)
+{
+ int i;
+ real pf2, fn2;
+ char buf[STEPSTRSIZE];
+
+ pf2 = sqr(pforce);
+ for (i = md->start; i < md->start+md->homenr; i++)
+ {
+ fn2 = norm2(f[i]);
+ /* We also catch NAN, if the compiler does not optimize this away. */
+ if (fn2 >= pf2 || fn2 != fn2)
+ {
+ fprintf(fp, "step %s atom %6d x %8.3f %8.3f %8.3f force %12.5e\n",
+ gmx_step_str(step, buf),
+ ddglatnr(cr->dd, i), x[i][XX], x[i][YY], x[i][ZZ], sqrt(fn2));
+ }
+ }
+}
+
+static void post_process_forces(t_commrec *cr,
+ gmx_large_int_t step,
+ t_nrnb *nrnb, gmx_wallcycle_t wcycle,
+ gmx_localtop_t *top,
+ matrix box, rvec x[],
+ rvec f[],
+ tensor vir_force,
+ t_mdatoms *mdatoms,
+ t_graph *graph,
+ t_forcerec *fr, gmx_vsite_t *vsite,
+ int flags)
+{
+ if (fr->bF_NoVirSum)
+ {
+ if (vsite)
+ {
+ /* Spread the mesh force on virtual sites to the other particles...
+ * This is parallellized. MPI communication is performed
+ * if the constructing atoms aren't local.
+ */
+ wallcycle_start(wcycle, ewcVSITESPREAD);
+ spread_vsite_f(vsite, x, fr->f_novirsum, NULL,
+ (flags & GMX_FORCE_VIRIAL), fr->vir_el_recip,
+ nrnb,
+ &top->idef, fr->ePBC, fr->bMolPBC, graph, box, cr);
+ wallcycle_stop(wcycle, ewcVSITESPREAD);
+ }
+ if (flags & GMX_FORCE_VIRIAL)
+ {
+ /* Now add the forces, this is local */
+ if (fr->bDomDec)
+ {
+ sum_forces(0, fr->f_novirsum_n, f, fr->f_novirsum);
+ }
+ else
+ {
+ sum_forces(mdatoms->start, mdatoms->start+mdatoms->homenr,
+ f, fr->f_novirsum);
+ }
+ if (EEL_FULL(fr->eeltype))
+ {
+ /* Add the mesh contribution to the virial */
+ m_add(vir_force, fr->vir_el_recip, vir_force);
+ }
+ if (debug)
+ {
+ pr_rvecs(debug, 0, "vir_force", vir_force, DIM);
+ }
+ }
+ }
+
+ if (fr->print_force >= 0)
+ {
+ print_large_forces(stderr, mdatoms, cr, step, fr->print_force, x, f);
+ }
+}
+
+static void do_nb_verlet(t_forcerec *fr,
+ interaction_const_t *ic,
+ gmx_enerdata_t *enerd,
+ int flags, int ilocality,
+ int clearF,
+ t_nrnb *nrnb,
+ gmx_wallcycle_t wcycle)
+{
+ int nnbl, kernel_type, enr_nbnxn_kernel_ljc, enr_nbnxn_kernel_lj;
+ char *env;
+ nonbonded_verlet_group_t *nbvg;
+ gmx_bool bCUDA;
+
+ if (!(flags & GMX_FORCE_NONBONDED))
+ {
+ /* skip non-bonded calculation */
+ return;
+ }
+
+ nbvg = &fr->nbv->grp[ilocality];
+
+ /* CUDA kernel launch overhead is already timed separately */
+ if (fr->cutoff_scheme != ecutsVERLET)
+ {
+ gmx_incons("Invalid cut-off scheme passed!");
+ }
+
+ bCUDA = (nbvg->kernel_type == nbnxnk8x8x8_CUDA);
+
+ if (!bCUDA)
+ {
+ wallcycle_sub_start(wcycle, ewcsNONBONDED);
+ }
+ switch (nbvg->kernel_type)
+ {
+ case nbnxnk4x4_PlainC:
+ nbnxn_kernel_ref(&nbvg->nbl_lists,
+ nbvg->nbat, ic,
+ fr->shift_vec,
+ flags,
+ clearF,
+ fr->fshift[0],
+ enerd->grpp.ener[egCOULSR],
+ fr->bBHAM ?
+ enerd->grpp.ener[egBHAMSR] :
+ enerd->grpp.ener[egLJSR]);
+ break;
+
+ case nbnxnk4xN_SIMD_4xN:
+ nbnxn_kernel_simd_4xn(&nbvg->nbl_lists,
+ nbvg->nbat, ic,
+ nbvg->ewald_excl,
+ fr->shift_vec,
+ flags,
+ clearF,
+ fr->fshift[0],
+ enerd->grpp.ener[egCOULSR],
+ fr->bBHAM ?
+ enerd->grpp.ener[egBHAMSR] :
+ enerd->grpp.ener[egLJSR]);
+ break;
+ case nbnxnk4xN_SIMD_2xNN:
+ nbnxn_kernel_simd_2xnn(&nbvg->nbl_lists,
+ nbvg->nbat, ic,
+ nbvg->ewald_excl,
+ fr->shift_vec,
+ flags,
+ clearF,
+ fr->fshift[0],
+ enerd->grpp.ener[egCOULSR],
+ fr->bBHAM ?
+ enerd->grpp.ener[egBHAMSR] :
+ enerd->grpp.ener[egLJSR]);
+ break;
+
+ case nbnxnk8x8x8_CUDA:
+ nbnxn_cuda_launch_kernel(fr->nbv->cu_nbv, nbvg->nbat, flags, ilocality);
+ break;
+
+ case nbnxnk8x8x8_PlainC:
+ nbnxn_kernel_gpu_ref(nbvg->nbl_lists.nbl[0],
+ nbvg->nbat, ic,
+ fr->shift_vec,
+ flags,
+ clearF,
+ nbvg->nbat->out[0].f,
+ fr->fshift[0],
+ enerd->grpp.ener[egCOULSR],
+ fr->bBHAM ?
+ enerd->grpp.ener[egBHAMSR] :
+ enerd->grpp.ener[egLJSR]);
+ break;
+
+ default:
+ gmx_incons("Invalid nonbonded kernel type passed!");
+
+ }
+ if (!bCUDA)
+ {
+ wallcycle_sub_stop(wcycle, ewcsNONBONDED);
+ }
+
+ if (EEL_RF(ic->eeltype) || ic->eeltype == eelCUT)
+ {
+ enr_nbnxn_kernel_ljc = eNR_NBNXN_LJ_RF;
+ }
+ else if ((!bCUDA && nbvg->ewald_excl == ewaldexclAnalytical) ||
+ (bCUDA && nbnxn_cuda_is_kernel_ewald_analytical(fr->nbv->cu_nbv)))
+ {
+ enr_nbnxn_kernel_ljc = eNR_NBNXN_LJ_EWALD;
+ }
+ else
+ {
+ enr_nbnxn_kernel_ljc = eNR_NBNXN_LJ_TAB;
+ }
+ enr_nbnxn_kernel_lj = eNR_NBNXN_LJ;
+ if (flags & GMX_FORCE_ENERGY)
+ {
+ /* In eNR_??? the nbnxn F+E kernels are always the F kernel + 1 */
+ enr_nbnxn_kernel_ljc += 1;
+ enr_nbnxn_kernel_lj += 1;
+ }
+
+ inc_nrnb(nrnb, enr_nbnxn_kernel_ljc,
+ nbvg->nbl_lists.natpair_ljq);
+ inc_nrnb(nrnb, enr_nbnxn_kernel_lj,
+ nbvg->nbl_lists.natpair_lj);
+ inc_nrnb(nrnb, enr_nbnxn_kernel_ljc-eNR_NBNXN_LJ_RF+eNR_NBNXN_RF,
+ nbvg->nbl_lists.natpair_q);
+}
+
+void do_force_cutsVERLET(FILE *fplog, t_commrec *cr,
+ t_inputrec *inputrec,
+ gmx_large_int_t step, t_nrnb *nrnb, gmx_wallcycle_t wcycle,
+ gmx_localtop_t *top,
+ gmx_groups_t gmx_unused *groups,
+ matrix box, rvec x[], history_t *hist,
+ rvec f[],
+ tensor vir_force,
+ t_mdatoms *mdatoms,
+ gmx_enerdata_t *enerd, t_fcdata *fcd,
+ real *lambda, t_graph *graph,
+ t_forcerec *fr, interaction_const_t *ic,
+ gmx_vsite_t *vsite, rvec mu_tot,
+ double t, FILE *field, gmx_edsam_t ed,
+ gmx_bool bBornRadii,
+ int flags)
+{
+ int cg0, cg1, i, j;
+ int start, homenr;
+ int nb_kernel_type;
+ double mu[2*DIM];
+ gmx_bool bSepDVDL, bStateChanged, bNS, bFillGrid, bCalcCGCM, bBS;
+ gmx_bool bDoLongRange, bDoForces, bSepLRF, bUseGPU, bUseOrEmulGPU;
+ gmx_bool bDiffKernels = FALSE;
+ matrix boxs;
+ rvec vzero, box_diag;
+ real e, v, dvdl;
- cycles_force = 0;
- nbv = fr->nbv;
- nb_kernel_type = fr->nbv->grp[0].kernel_type;
++ float cycles_pme, cycles_force, cycles_wait_gpu;
+ nonbonded_verlet_t *nbv;
+
- cycles_force += wallcycle_stop(wcycle, ewcWAIT_GPU_NB_NL);
++ cycles_force = 0;
++ cycles_wait_gpu = 0;
++ nbv = fr->nbv;
++ nb_kernel_type = fr->nbv->grp[0].kernel_type;
+
+ start = mdatoms->start;
+ homenr = mdatoms->homenr;
+
+ bSepDVDL = (fr->bSepDVDL && do_per_step(step, inputrec->nstlog));
+
+ clear_mat(vir_force);
+
+ cg0 = 0;
+ if (DOMAINDECOMP(cr))
+ {
+ cg1 = cr->dd->ncg_tot;
+ }
+ else
+ {
+ cg1 = top->cgs.nr;
+ }
+ if (fr->n_tpi > 0)
+ {
+ cg1--;
+ }
+
+ bStateChanged = (flags & GMX_FORCE_STATECHANGED);
+ bNS = (flags & GMX_FORCE_NS) && (fr->bAllvsAll == FALSE);
+ bFillGrid = (bNS && bStateChanged);
+ bCalcCGCM = (bFillGrid && !DOMAINDECOMP(cr));
+ bDoLongRange = (fr->bTwinRange && bNS && (flags & GMX_FORCE_DO_LR));
+ bDoForces = (flags & GMX_FORCE_FORCES);
+ bSepLRF = (bDoLongRange && bDoForces && (flags & GMX_FORCE_SEPLRF));
+ bUseGPU = fr->nbv->bUseGPU;
+ bUseOrEmulGPU = bUseGPU || (nbv->grp[0].kernel_type == nbnxnk8x8x8_PlainC);
+
+ if (bStateChanged)
+ {
+ update_forcerec(fr, box);
+
+ if (NEED_MUTOT(*inputrec))
+ {
+ /* Calculate total (local) dipole moment in a temporary common array.
+ * This makes it possible to sum them over nodes faster.
+ */
+ calc_mu(start, homenr,
+ x, mdatoms->chargeA, mdatoms->chargeB, mdatoms->nChargePerturbed,
+ mu, mu+DIM);
+ }
+ }
+
+ if (fr->ePBC != epbcNONE)
+ {
+ /* Compute shift vectors every step,
+ * because of pressure coupling or box deformation!
+ */
+ if ((flags & GMX_FORCE_DYNAMICBOX) && bStateChanged)
+ {
+ calc_shifts(box, fr->shift_vec);
+ }
+
+ if (bCalcCGCM)
+ {
+ put_atoms_in_box_omp(fr->ePBC, box, homenr, x);
+ inc_nrnb(nrnb, eNR_SHIFTX, homenr);
+ }
+ else if (EI_ENERGY_MINIMIZATION(inputrec->eI) && graph)
+ {
+ unshift_self(graph, box, x);
+ }
+ }
+
+ nbnxn_atomdata_copy_shiftvec(flags & GMX_FORCE_DYNAMICBOX,
+ fr->shift_vec, nbv->grp[0].nbat);
+
+#ifdef GMX_MPI
+ if (!(cr->duty & DUTY_PME))
+ {
+ /* Send particle coordinates to the pme nodes.
+ * Since this is only implemented for domain decomposition
+ * and domain decomposition does not use the graph,
+ * we do not need to worry about shifting.
+ */
+
+ wallcycle_start(wcycle, ewcPP_PMESENDX);
+
+ bBS = (inputrec->nwall == 2);
+ if (bBS)
+ {
+ copy_mat(box, boxs);
+ svmul(inputrec->wall_ewald_zfac, boxs[ZZ], boxs[ZZ]);
+ }
+
+ gmx_pme_send_x(cr, bBS ? boxs : box, x,
+ mdatoms->nChargePerturbed, lambda[efptCOUL],
+ (flags & (GMX_FORCE_VIRIAL | GMX_FORCE_ENERGY)), step);
+
+ wallcycle_stop(wcycle, ewcPP_PMESENDX);
+ }
+#endif /* GMX_MPI */
+
+ /* do gridding for pair search */
+ if (bNS)
+ {
+ if (graph && bStateChanged)
+ {
+ /* Calculate intramolecular shift vectors to make molecules whole */
+ mk_mshift(fplog, graph, fr->ePBC, box, x);
+ }
+
+ clear_rvec(vzero);
+ box_diag[XX] = box[XX][XX];
+ box_diag[YY] = box[YY][YY];
+ box_diag[ZZ] = box[ZZ][ZZ];
+
+ wallcycle_start(wcycle, ewcNS);
+ if (!fr->bDomDec)
+ {
+ wallcycle_sub_start(wcycle, ewcsNBS_GRID_LOCAL);
+ nbnxn_put_on_grid(nbv->nbs, fr->ePBC, box,
+ 0, vzero, box_diag,
+ 0, mdatoms->homenr, -1, fr->cginfo, x,
+ 0, NULL,
+ nbv->grp[eintLocal].kernel_type,
+ nbv->grp[eintLocal].nbat);
+ wallcycle_sub_stop(wcycle, ewcsNBS_GRID_LOCAL);
+ }
+ else
+ {
+ wallcycle_sub_start(wcycle, ewcsNBS_GRID_NONLOCAL);
+ nbnxn_put_on_grid_nonlocal(nbv->nbs, domdec_zones(cr->dd),
+ fr->cginfo, x,
+ nbv->grp[eintNonlocal].kernel_type,
+ nbv->grp[eintNonlocal].nbat);
+ wallcycle_sub_stop(wcycle, ewcsNBS_GRID_NONLOCAL);
+ }
+
+ if (nbv->ngrp == 1 ||
+ nbv->grp[eintNonlocal].nbat == nbv->grp[eintLocal].nbat)
+ {
+ nbnxn_atomdata_set(nbv->grp[eintLocal].nbat, eatAll,
+ nbv->nbs, mdatoms, fr->cginfo);
+ }
+ else
+ {
+ nbnxn_atomdata_set(nbv->grp[eintLocal].nbat, eatLocal,
+ nbv->nbs, mdatoms, fr->cginfo);
+ nbnxn_atomdata_set(nbv->grp[eintNonlocal].nbat, eatAll,
+ nbv->nbs, mdatoms, fr->cginfo);
+ }
+ wallcycle_stop(wcycle, ewcNS);
+ }
+
+ /* initialize the GPU atom data and copy shift vector */
+ if (bUseGPU)
+ {
+ if (bNS)
+ {
+ wallcycle_start_nocount(wcycle, ewcLAUNCH_GPU_NB);
+ nbnxn_cuda_init_atomdata(nbv->cu_nbv, nbv->grp[eintLocal].nbat);
+ wallcycle_stop(wcycle, ewcLAUNCH_GPU_NB);
+ }
+
+ wallcycle_start_nocount(wcycle, ewcLAUNCH_GPU_NB);
+ nbnxn_cuda_upload_shiftvec(nbv->cu_nbv, nbv->grp[eintLocal].nbat);
+ wallcycle_stop(wcycle, ewcLAUNCH_GPU_NB);
+ }
+
+ /* do local pair search */
+ if (bNS)
+ {
+ wallcycle_start_nocount(wcycle, ewcNS);
+ wallcycle_sub_start(wcycle, ewcsNBS_SEARCH_LOCAL);
+ nbnxn_make_pairlist(nbv->nbs, nbv->grp[eintLocal].nbat,
+ &top->excls,
+ ic->rlist,
+ nbv->min_ci_balanced,
+ &nbv->grp[eintLocal].nbl_lists,
+ eintLocal,
+ nbv->grp[eintLocal].kernel_type,
+ nrnb);
+ wallcycle_sub_stop(wcycle, ewcsNBS_SEARCH_LOCAL);
+
+ if (bUseGPU)
+ {
+ /* initialize local pair-list on the GPU */
+ nbnxn_cuda_init_pairlist(nbv->cu_nbv,
+ nbv->grp[eintLocal].nbl_lists.nbl[0],
+ eintLocal);
+ }
+ wallcycle_stop(wcycle, ewcNS);
+ }
+ else
+ {
+ wallcycle_start(wcycle, ewcNB_XF_BUF_OPS);
+ wallcycle_sub_start(wcycle, ewcsNB_X_BUF_OPS);
+ nbnxn_atomdata_copy_x_to_nbat_x(nbv->nbs, eatLocal, FALSE, x,
+ nbv->grp[eintLocal].nbat);
+ wallcycle_sub_stop(wcycle, ewcsNB_X_BUF_OPS);
+ wallcycle_stop(wcycle, ewcNB_XF_BUF_OPS);
+ }
+
+ if (bUseGPU)
+ {
+ wallcycle_start(wcycle, ewcLAUNCH_GPU_NB);
+ /* launch local nonbonded F on GPU */
+ do_nb_verlet(fr, ic, enerd, flags, eintLocal, enbvClearFNo,
+ nrnb, wcycle);
+ wallcycle_stop(wcycle, ewcLAUNCH_GPU_NB);
+ }
+
+ /* Communicate coordinates and sum dipole if necessary +
+ do non-local pair search */
+ if (DOMAINDECOMP(cr))
+ {
+ bDiffKernels = (nbv->grp[eintNonlocal].kernel_type !=
+ nbv->grp[eintLocal].kernel_type);
+
+ if (bDiffKernels)
+ {
+ /* With GPU+CPU non-bonded calculations we need to copy
+ * the local coordinates to the non-local nbat struct
+ * (in CPU format) as the non-local kernel call also
+ * calculates the local - non-local interactions.
+ */
+ wallcycle_start(wcycle, ewcNB_XF_BUF_OPS);
+ wallcycle_sub_start(wcycle, ewcsNB_X_BUF_OPS);
+ nbnxn_atomdata_copy_x_to_nbat_x(nbv->nbs, eatLocal, TRUE, x,
+ nbv->grp[eintNonlocal].nbat);
+ wallcycle_sub_stop(wcycle, ewcsNB_X_BUF_OPS);
+ wallcycle_stop(wcycle, ewcNB_XF_BUF_OPS);
+ }
+
+ if (bNS)
+ {
+ wallcycle_start_nocount(wcycle, ewcNS);
+ wallcycle_sub_start(wcycle, ewcsNBS_SEARCH_NONLOCAL);
+
+ if (bDiffKernels)
+ {
+ nbnxn_grid_add_simple(nbv->nbs, nbv->grp[eintNonlocal].nbat);
+ }
+
+ nbnxn_make_pairlist(nbv->nbs, nbv->grp[eintNonlocal].nbat,
+ &top->excls,
+ ic->rlist,
+ nbv->min_ci_balanced,
+ &nbv->grp[eintNonlocal].nbl_lists,
+ eintNonlocal,
+ nbv->grp[eintNonlocal].kernel_type,
+ nrnb);
+
+ wallcycle_sub_stop(wcycle, ewcsNBS_SEARCH_NONLOCAL);
+
+ if (nbv->grp[eintNonlocal].kernel_type == nbnxnk8x8x8_CUDA)
+ {
+ /* initialize non-local pair-list on the GPU */
+ nbnxn_cuda_init_pairlist(nbv->cu_nbv,
+ nbv->grp[eintNonlocal].nbl_lists.nbl[0],
+ eintNonlocal);
+ }
+ wallcycle_stop(wcycle, ewcNS);
+ }
+ else
+ {
+ wallcycle_start(wcycle, ewcMOVEX);
+ dd_move_x(cr->dd, box, x);
+
+ /* When we don't need the total dipole we sum it in global_stat */
+ if (bStateChanged && NEED_MUTOT(*inputrec))
+ {
+ gmx_sumd(2*DIM, mu, cr);
+ }
+ wallcycle_stop(wcycle, ewcMOVEX);
+
+ wallcycle_start(wcycle, ewcNB_XF_BUF_OPS);
+ wallcycle_sub_start(wcycle, ewcsNB_X_BUF_OPS);
+ nbnxn_atomdata_copy_x_to_nbat_x(nbv->nbs, eatNonlocal, FALSE, x,
+ nbv->grp[eintNonlocal].nbat);
+ wallcycle_sub_stop(wcycle, ewcsNB_X_BUF_OPS);
+ cycles_force += wallcycle_stop(wcycle, ewcNB_XF_BUF_OPS);
+ }
+
+ if (bUseGPU && !bDiffKernels)
+ {
+ wallcycle_start(wcycle, ewcLAUNCH_GPU_NB);
+ /* launch non-local nonbonded F on GPU */
+ do_nb_verlet(fr, ic, enerd, flags, eintNonlocal, enbvClearFNo,
+ nrnb, wcycle);
+ cycles_force += wallcycle_stop(wcycle, ewcLAUNCH_GPU_NB);
+ }
+ }
+
+ if (bUseGPU)
+ {
+ /* launch D2H copy-back F */
+ wallcycle_start_nocount(wcycle, ewcLAUNCH_GPU_NB);
+ if (DOMAINDECOMP(cr) && !bDiffKernels)
+ {
+ nbnxn_cuda_launch_cpyback(nbv->cu_nbv, nbv->grp[eintNonlocal].nbat,
+ flags, eatNonlocal);
+ }
+ nbnxn_cuda_launch_cpyback(nbv->cu_nbv, nbv->grp[eintLocal].nbat,
+ flags, eatLocal);
+ cycles_force += wallcycle_stop(wcycle, ewcLAUNCH_GPU_NB);
+ }
+
+ if (bStateChanged && NEED_MUTOT(*inputrec))
+ {
+ if (PAR(cr))
+ {
+ gmx_sumd(2*DIM, mu, cr);
+ }
+
+ for (i = 0; i < 2; i++)
+ {
+ for (j = 0; j < DIM; j++)
+ {
+ fr->mu_tot[i][j] = mu[i*DIM + j];
+ }
+ }
+ }
+ if (fr->efep == efepNO)
+ {
+ copy_rvec(fr->mu_tot[0], mu_tot);
+ }
+ else
+ {
+ for (j = 0; j < DIM; j++)
+ {
+ mu_tot[j] =
+ (1.0 - lambda[efptCOUL])*fr->mu_tot[0][j] +
+ lambda[efptCOUL]*fr->mu_tot[1][j];
+ }
+ }
+
+ /* Reset energies */
+ reset_enerdata(fr, bNS, enerd, MASTER(cr));
+ clear_rvecs(SHIFTS, fr->fshift);
+
+ if (DOMAINDECOMP(cr))
+ {
+ if (!(cr->duty & DUTY_PME))
+ {
+ wallcycle_start(wcycle, ewcPPDURINGPME);
+ dd_force_flop_start(cr->dd, nrnb);
+ }
+ }
+
+ if (inputrec->bRot)
+ {
+ /* Enforced rotation has its own cycle counter that starts after the collective
+ * coordinates have been communicated. It is added to ddCyclF to allow
+ * for proper load-balancing */
+ wallcycle_start(wcycle, ewcROT);
+ do_rotation(cr, inputrec, box, x, t, step, wcycle, bNS);
+ wallcycle_stop(wcycle, ewcROT);
+ }
+
+ /* Start the force cycle counter.
+ * This counter is stopped in do_forcelow_level.
+ * No parallel communication should occur while this counter is running,
+ * since that will interfere with the dynamic load balancing.
+ */
+ wallcycle_start(wcycle, ewcFORCE);
+ if (bDoForces)
+ {
+ /* Reset forces for which the virial is calculated separately:
+ * PME/Ewald forces if necessary */
+ if (fr->bF_NoVirSum)
+ {
+ if (flags & GMX_FORCE_VIRIAL)
+ {
+ fr->f_novirsum = fr->f_novirsum_alloc;
+ if (fr->bDomDec)
+ {
+ clear_rvecs(fr->f_novirsum_n, fr->f_novirsum);
+ }
+ else
+ {
+ clear_rvecs(homenr, fr->f_novirsum+start);
+ }
+ }
+ else
+ {
+ /* We are not calculating the pressure so we do not need
+ * a separate array for forces that do not contribute
+ * to the pressure.
+ */
+ fr->f_novirsum = f;
+ }
+ }
+
+ /* Clear the short- and long-range forces */
+ clear_rvecs(fr->natoms_force_constr, f);
+ if (bSepLRF && do_per_step(step, inputrec->nstcalclr))
+ {
+ clear_rvecs(fr->natoms_force_constr, fr->f_twin);
+ }
+
+ clear_rvec(fr->vir_diag_posres);
+ }
+
+ if (inputrec->ePull == epullCONSTRAINT)
+ {
+ clear_pull_forces(inputrec->pull);
+ }
+
+ /* We calculate the non-bonded forces, when done on the CPU, here.
+ * We do this before calling do_force_lowlevel, as in there bondeds
+ * forces are calculated before PME, which does communication.
+ * With this order, non-bonded and bonded force calculation imbalance
+ * can be balanced out by the domain decomposition load balancing.
+ */
+
+ if (!bUseOrEmulGPU)
+ {
+ /* Maybe we should move this into do_force_lowlevel */
+ do_nb_verlet(fr, ic, enerd, flags, eintLocal, enbvClearFYes,
+ nrnb, wcycle);
+ }
+
+ if (!bUseOrEmulGPU || bDiffKernels)
+ {
+ int aloc;
+
+ if (DOMAINDECOMP(cr))
+ {
+ do_nb_verlet(fr, ic, enerd, flags, eintNonlocal,
+ bDiffKernels ? enbvClearFYes : enbvClearFNo,
+ nrnb, wcycle);
+ }
+
+ if (!bUseOrEmulGPU)
+ {
+ aloc = eintLocal;
+ }
+ else
+ {
+ aloc = eintNonlocal;
+ }
+
+ /* Add all the non-bonded force to the normal force array.
+ * This can be split into a local a non-local part when overlapping
+ * communication with calculation with domain decomposition.
+ */
+ cycles_force += wallcycle_stop(wcycle, ewcFORCE);
+ wallcycle_start(wcycle, ewcNB_XF_BUF_OPS);
+ wallcycle_sub_start(wcycle, ewcsNB_F_BUF_OPS);
+ nbnxn_atomdata_add_nbat_f_to_f(nbv->nbs, eatAll, nbv->grp[aloc].nbat, f);
+ wallcycle_sub_stop(wcycle, ewcsNB_F_BUF_OPS);
+ cycles_force += wallcycle_stop(wcycle, ewcNB_XF_BUF_OPS);
+ wallcycle_start_nocount(wcycle, ewcFORCE);
+
+ /* if there are multiple fshift output buffers reduce them */
+ if ((flags & GMX_FORCE_VIRIAL) &&
+ nbv->grp[aloc].nbl_lists.nnbl > 1)
+ {
+ nbnxn_atomdata_add_nbat_fshift_to_fshift(nbv->grp[aloc].nbat,
+ fr->fshift);
+ }
+ }
+
+ /* update QMMMrec, if necessary */
+ if (fr->bQMMM)
+ {
+ update_QMMMrec(cr, fr, x, mdatoms, box, top);
+ }
+
+ if ((flags & GMX_FORCE_BONDED) && top->idef.il[F_POSRES].nr > 0)
+ {
+ posres_wrapper(fplog, flags, bSepDVDL, inputrec, nrnb, top, box, x,
+ enerd, lambda, fr);
+ }
+
+ /* Compute the bonded and non-bonded energies and optionally forces */
+ do_force_lowlevel(fplog, step, fr, inputrec, &(top->idef),
+ cr, nrnb, wcycle, mdatoms,
+ x, hist, f, bSepLRF ? fr->f_twin : f, enerd, fcd, top, fr->born,
+ &(top->atomtypes), bBornRadii, box,
+ inputrec->fepvals, lambda, graph, &(top->excls), fr->mu_tot,
+ flags, &cycles_pme);
+
+ if (bSepLRF)
+ {
+ if (do_per_step(step, inputrec->nstcalclr))
+ {
+ /* Add the long range forces to the short range forces */
+ for (i = 0; i < fr->natoms_force_constr; i++)
+ {
+ rvec_add(fr->f_twin[i], f[i], f[i]);
+ }
+ }
+ }
+
+ cycles_force += wallcycle_stop(wcycle, ewcFORCE);
+
+ if (ed)
+ {
+ do_flood(cr, inputrec, x, f, ed, box, step, bNS);
+ }
+
+ if (bUseOrEmulGPU && !bDiffKernels)
+ {
+ /* wait for non-local forces (or calculate in emulation mode) */
+ if (DOMAINDECOMP(cr))
+ {
+ if (bUseGPU)
+ {
++ float cycles_tmp;
++
+ wallcycle_start(wcycle, ewcWAIT_GPU_NB_NL);
+ nbnxn_cuda_wait_gpu(nbv->cu_nbv,
+ nbv->grp[eintNonlocal].nbat,
+ flags, eatNonlocal,
+ enerd->grpp.ener[egLJSR], enerd->grpp.ener[egCOULSR],
+ fr->fshift);
- wallcycle_stop(wcycle, ewcWAIT_GPU_NB_L);
++ cycles_tmp = wallcycle_stop(wcycle, ewcWAIT_GPU_NB_NL);
++ cycles_wait_gpu += cycles_tmp;
++ cycles_force += cycles_tmp;
+ }
+ else
+ {
+ wallcycle_start_nocount(wcycle, ewcFORCE);
+ do_nb_verlet(fr, ic, enerd, flags, eintNonlocal, enbvClearFYes,
+ nrnb, wcycle);
+ cycles_force += wallcycle_stop(wcycle, ewcFORCE);
+ }
+ wallcycle_start(wcycle, ewcNB_XF_BUF_OPS);
+ wallcycle_sub_start(wcycle, ewcsNB_F_BUF_OPS);
+ /* skip the reduction if there was no non-local work to do */
+ if (nbv->grp[eintLocal].nbl_lists.nbl[0]->nsci > 0)
+ {
+ nbnxn_atomdata_add_nbat_f_to_f(nbv->nbs, eatNonlocal,
+ nbv->grp[eintNonlocal].nbat, f);
+ }
+ wallcycle_sub_stop(wcycle, ewcsNB_F_BUF_OPS);
+ cycles_force += wallcycle_stop(wcycle, ewcNB_XF_BUF_OPS);
+ }
+ }
+
+ if (bDoForces)
+ {
+ /* Communicate the forces */
+ if (PAR(cr))
+ {
+ wallcycle_start(wcycle, ewcMOVEF);
+ if (DOMAINDECOMP(cr))
+ {
+ dd_move_f(cr->dd, f, fr->fshift);
+ /* Do we need to communicate the separate force array
+ * for terms that do not contribute to the single sum virial?
+ * Position restraints and electric fields do not introduce
+ * inter-cg forces, only full electrostatics methods do.
+ * When we do not calculate the virial, fr->f_novirsum = f,
+ * so we have already communicated these forces.
+ */
+ if (EEL_FULL(fr->eeltype) && cr->dd->n_intercg_excl &&
+ (flags & GMX_FORCE_VIRIAL))
+ {
+ dd_move_f(cr->dd, fr->f_novirsum, NULL);
+ }
+ if (bSepLRF)
+ {
+ /* We should not update the shift forces here,
+ * since f_twin is already included in f.
+ */
+ dd_move_f(cr->dd, fr->f_twin, NULL);
+ }
+ }
+ wallcycle_stop(wcycle, ewcMOVEF);
+ }
+ }
+
+ if (bUseOrEmulGPU)
+ {
+ /* wait for local forces (or calculate in emulation mode) */
+ if (bUseGPU)
+ {
+ wallcycle_start(wcycle, ewcWAIT_GPU_NB_L);
+ nbnxn_cuda_wait_gpu(nbv->cu_nbv,
+ nbv->grp[eintLocal].nbat,
+ flags, eatLocal,
+ enerd->grpp.ener[egLJSR], enerd->grpp.ener[egCOULSR],
+ fr->fshift);
++ cycles_wait_gpu += wallcycle_stop(wcycle, ewcWAIT_GPU_NB_L);
+
+ /* now clear the GPU outputs while we finish the step on the CPU */
+
+ wallcycle_start_nocount(wcycle, ewcLAUNCH_GPU_NB);
+ nbnxn_cuda_clear_outputs(nbv->cu_nbv, flags);
+ wallcycle_stop(wcycle, ewcLAUNCH_GPU_NB);
+ }
+ else
+ {
+ wallcycle_start_nocount(wcycle, ewcFORCE);
+ do_nb_verlet(fr, ic, enerd, flags, eintLocal,
+ DOMAINDECOMP(cr) ? enbvClearFNo : enbvClearFYes,
+ nrnb, wcycle);
+ wallcycle_stop(wcycle, ewcFORCE);
+ }
+ wallcycle_start(wcycle, ewcNB_XF_BUF_OPS);
+ wallcycle_sub_start(wcycle, ewcsNB_F_BUF_OPS);
+ if (nbv->grp[eintLocal].nbl_lists.nbl[0]->nsci > 0)
+ {
+ /* skip the reduction if there was no non-local work to do */
+ nbnxn_atomdata_add_nbat_f_to_f(nbv->nbs, eatLocal,
+ nbv->grp[eintLocal].nbat, f);
+ }
+ wallcycle_sub_stop(wcycle, ewcsNB_F_BUF_OPS);
+ wallcycle_stop(wcycle, ewcNB_XF_BUF_OPS);
+ }
+
+ if (DOMAINDECOMP(cr))
+ {
+ dd_force_flop_stop(cr->dd, nrnb);
+ if (wcycle)
+ {
+ dd_cycles_add(cr->dd, cycles_force-cycles_pme, ddCyclF);
++ if (bUseGPU)
++ {
++ dd_cycles_add(cr->dd, cycles_wait_gpu, ddCyclWaitGPU);
++ }
+ }
+ }
+
+ if (bDoForces)
+ {
+ if (IR_ELEC_FIELD(*inputrec))
+ {
+ /* Compute forces due to electric field */
+ calc_f_el(MASTER(cr) ? field : NULL,
+ start, homenr, mdatoms->chargeA, fr->f_novirsum,
+ inputrec->ex, inputrec->et, t);
+ }
+
+ /* If we have NoVirSum forces, but we do not calculate the virial,
+ * we sum fr->f_novirum=f later.
+ */
+ if (vsite && !(fr->bF_NoVirSum && !(flags & GMX_FORCE_VIRIAL)))
+ {
+ wallcycle_start(wcycle, ewcVSITESPREAD);
+ spread_vsite_f(vsite, x, f, fr->fshift, FALSE, NULL, nrnb,
+ &top->idef, fr->ePBC, fr->bMolPBC, graph, box, cr);
+ wallcycle_stop(wcycle, ewcVSITESPREAD);
+
+ if (bSepLRF)
+ {
+ wallcycle_start(wcycle, ewcVSITESPREAD);
+ spread_vsite_f(vsite, x, fr->f_twin, NULL, FALSE, NULL,
+ nrnb,
+ &top->idef, fr->ePBC, fr->bMolPBC, graph, box, cr);
+ wallcycle_stop(wcycle, ewcVSITESPREAD);
+ }
+ }
+
+ if (flags & GMX_FORCE_VIRIAL)
+ {
+ /* Calculation of the virial must be done after vsites! */
+ calc_virial(mdatoms->start, mdatoms->homenr, x, f,
+ vir_force, graph, box, nrnb, fr, inputrec->ePBC);
+ }
+ }
+
+ if (inputrec->ePull == epullUMBRELLA || inputrec->ePull == epullCONST_F)
+ {
+ pull_potential_wrapper(fplog, bSepDVDL, cr, inputrec, box, x,
+ f, vir_force, mdatoms, enerd, lambda, t);
+ }
+
+ /* Add the forces from enforced rotation potentials (if any) */
+ if (inputrec->bRot)
+ {
+ wallcycle_start(wcycle, ewcROTadd);
+ enerd->term[F_COM_PULL] += add_rot_forces(inputrec->rot, f, cr, step, t);
+ wallcycle_stop(wcycle, ewcROTadd);
+ }
+
+ if (PAR(cr) && !(cr->duty & DUTY_PME))
+ {
+ /* In case of node-splitting, the PP nodes receive the long-range
+ * forces, virial and energy from the PME nodes here.
+ */
+ pme_receive_force_ener(fplog, bSepDVDL, cr, wcycle, enerd, fr);
+ }
+
+ if (bDoForces)
+ {
+ post_process_forces(cr, step, nrnb, wcycle,
+ top, box, x, f, vir_force, mdatoms, graph, fr, vsite,
+ flags);
+ }
+
+ /* Sum the potential energy terms from group contributions */
+ sum_epot(&(enerd->grpp), enerd->term);
+}
+
+void do_force_cutsGROUP(FILE *fplog, t_commrec *cr,
+ t_inputrec *inputrec,
+ gmx_large_int_t step, t_nrnb *nrnb, gmx_wallcycle_t wcycle,
+ gmx_localtop_t *top,
+ gmx_groups_t *groups,
+ matrix box, rvec x[], history_t *hist,
+ rvec f[],
+ tensor vir_force,
+ t_mdatoms *mdatoms,
+ gmx_enerdata_t *enerd, t_fcdata *fcd,
+ real *lambda, t_graph *graph,
+ t_forcerec *fr, gmx_vsite_t *vsite, rvec mu_tot,
+ double t, FILE *field, gmx_edsam_t ed,
+ gmx_bool bBornRadii,
+ int flags)
+{
+ int cg0, cg1, i, j;
+ int start, homenr;
+ double mu[2*DIM];
+ gmx_bool bSepDVDL, bStateChanged, bNS, bFillGrid, bCalcCGCM, bBS;
+ gmx_bool bDoLongRangeNS, bDoForces, bDoPotential, bSepLRF;
+ gmx_bool bDoAdressWF;
+ matrix boxs;
+ rvec vzero, box_diag;
+ real e, v, dvdlambda[efptNR];
+ t_pbc pbc;
+ float cycles_pme, cycles_force;
+
+ start = mdatoms->start;
+ homenr = mdatoms->homenr;
+
+ bSepDVDL = (fr->bSepDVDL && do_per_step(step, inputrec->nstlog));
+
+ clear_mat(vir_force);
+
+ if (PARTDECOMP(cr))
+ {
+ pd_cg_range(cr, &cg0, &cg1);
+ }
+ else
+ {
+ cg0 = 0;
+ if (DOMAINDECOMP(cr))
+ {
+ cg1 = cr->dd->ncg_tot;
+ }
+ else
+ {
+ cg1 = top->cgs.nr;
+ }
+ if (fr->n_tpi > 0)
+ {
+ cg1--;
+ }
+ }
+
+ bStateChanged = (flags & GMX_FORCE_STATECHANGED);
+ bNS = (flags & GMX_FORCE_NS) && (fr->bAllvsAll == FALSE);
+ /* Should we update the long-range neighborlists at this step? */
+ bDoLongRangeNS = fr->bTwinRange && bNS;
+ /* Should we perform the long-range nonbonded evaluation inside the neighborsearching? */
+ bFillGrid = (bNS && bStateChanged);
+ bCalcCGCM = (bFillGrid && !DOMAINDECOMP(cr));
+ bDoForces = (flags & GMX_FORCE_FORCES);
+ bDoPotential = (flags & GMX_FORCE_ENERGY);
+ bSepLRF = ((inputrec->nstcalclr > 1) && bDoForces &&
+ (flags & GMX_FORCE_SEPLRF) && (flags & GMX_FORCE_DO_LR));
+
+ /* should probably move this to the forcerec since it doesn't change */
+ bDoAdressWF = ((fr->adress_type != eAdressOff));
+
+ if (bStateChanged)
+ {
+ update_forcerec(fr, box);
+
+ if (NEED_MUTOT(*inputrec))
+ {
+ /* Calculate total (local) dipole moment in a temporary common array.
+ * This makes it possible to sum them over nodes faster.
+ */
+ calc_mu(start, homenr,
+ x, mdatoms->chargeA, mdatoms->chargeB, mdatoms->nChargePerturbed,
+ mu, mu+DIM);
+ }
+ }
+
+ if (fr->ePBC != epbcNONE)
+ {
+ /* Compute shift vectors every step,
+ * because of pressure coupling or box deformation!
+ */
+ if ((flags & GMX_FORCE_DYNAMICBOX) && bStateChanged)
+ {
+ calc_shifts(box, fr->shift_vec);
+ }
+
+ if (bCalcCGCM)
+ {
+ put_charge_groups_in_box(fplog, cg0, cg1, fr->ePBC, box,
+ &(top->cgs), x, fr->cg_cm);
+ inc_nrnb(nrnb, eNR_CGCM, homenr);
+ inc_nrnb(nrnb, eNR_RESETX, cg1-cg0);
+ }
+ else if (EI_ENERGY_MINIMIZATION(inputrec->eI) && graph)
+ {
+ unshift_self(graph, box, x);
+ }
+ }
+ else if (bCalcCGCM)
+ {
+ calc_cgcm(fplog, cg0, cg1, &(top->cgs), x, fr->cg_cm);
+ inc_nrnb(nrnb, eNR_CGCM, homenr);
+ }
+
+ if (bCalcCGCM)
+ {
+ if (PAR(cr))
+ {
+ move_cgcm(fplog, cr, fr->cg_cm);
+ }
+ if (gmx_debug_at)
+ {
+ pr_rvecs(debug, 0, "cgcm", fr->cg_cm, top->cgs.nr);
+ }
+ }
+
+#ifdef GMX_MPI
+ if (!(cr->duty & DUTY_PME))
+ {
+ /* Send particle coordinates to the pme nodes.
+ * Since this is only implemented for domain decomposition
+ * and domain decomposition does not use the graph,
+ * we do not need to worry about shifting.
+ */
+
+ wallcycle_start(wcycle, ewcPP_PMESENDX);
+
+ bBS = (inputrec->nwall == 2);
+ if (bBS)
+ {
+ copy_mat(box, boxs);
+ svmul(inputrec->wall_ewald_zfac, boxs[ZZ], boxs[ZZ]);
+ }
+
+ gmx_pme_send_x(cr, bBS ? boxs : box, x,
+ mdatoms->nChargePerturbed, lambda[efptCOUL],
+ (flags & (GMX_FORCE_VIRIAL | GMX_FORCE_ENERGY)), step);
+
+ wallcycle_stop(wcycle, ewcPP_PMESENDX);
+ }
+#endif /* GMX_MPI */
+
+ /* Communicate coordinates and sum dipole if necessary */
+ if (PAR(cr))
+ {
+ wallcycle_start(wcycle, ewcMOVEX);
+ if (DOMAINDECOMP(cr))
+ {
+ dd_move_x(cr->dd, box, x);
+ }
+ else
+ {
+ move_x(cr, x, nrnb);
+ }
+ wallcycle_stop(wcycle, ewcMOVEX);
+ }
+
+ /* update adress weight beforehand */
+ if (bStateChanged && bDoAdressWF)
+ {
+ /* need pbc for adress weight calculation with pbc_dx */
+ set_pbc(&pbc, inputrec->ePBC, box);
+ if (fr->adress_site == eAdressSITEcog)
+ {
+ update_adress_weights_cog(top->idef.iparams, top->idef.il, x, fr, mdatoms,
+ inputrec->ePBC == epbcNONE ? NULL : &pbc);
+ }
+ else if (fr->adress_site == eAdressSITEcom)
+ {
+ update_adress_weights_com(fplog, cg0, cg1, &(top->cgs), x, fr, mdatoms,
+ inputrec->ePBC == epbcNONE ? NULL : &pbc);
+ }
+ else if (fr->adress_site == eAdressSITEatomatom)
+ {
+ update_adress_weights_atom_per_atom(cg0, cg1, &(top->cgs), x, fr, mdatoms,
+ inputrec->ePBC == epbcNONE ? NULL : &pbc);
+ }
+ else
+ {
+ update_adress_weights_atom(cg0, cg1, &(top->cgs), x, fr, mdatoms,
+ inputrec->ePBC == epbcNONE ? NULL : &pbc);
+ }
+ }
+
+ if (NEED_MUTOT(*inputrec))
+ {
+
+ if (bStateChanged)
+ {
+ if (PAR(cr))
+ {
+ gmx_sumd(2*DIM, mu, cr);
+ }
+ for (i = 0; i < 2; i++)
+ {
+ for (j = 0; j < DIM; j++)
+ {
+ fr->mu_tot[i][j] = mu[i*DIM + j];
+ }
+ }
+ }
+ if (fr->efep == efepNO)
+ {
+ copy_rvec(fr->mu_tot[0], mu_tot);
+ }
+ else
+ {
+ for (j = 0; j < DIM; j++)
+ {
+ mu_tot[j] =
+ (1.0 - lambda[efptCOUL])*fr->mu_tot[0][j] + lambda[efptCOUL]*fr->mu_tot[1][j];
+ }
+ }
+ }
+
+ /* Reset energies */
+ reset_enerdata(fr, bNS, enerd, MASTER(cr));
+ clear_rvecs(SHIFTS, fr->fshift);
+
+ if (bNS)
+ {
+ wallcycle_start(wcycle, ewcNS);
+
+ if (graph && bStateChanged)
+ {
+ /* Calculate intramolecular shift vectors to make molecules whole */
+ mk_mshift(fplog, graph, fr->ePBC, box, x);
+ }
+
+ /* Do the actual neighbour searching */
+ ns(fplog, fr, box,
+ groups, top, mdatoms,
+ cr, nrnb, bFillGrid,
+ bDoLongRangeNS);
+
+ wallcycle_stop(wcycle, ewcNS);
+ }
+
+ if (inputrec->implicit_solvent && bNS)
+ {
+ make_gb_nblist(cr, inputrec->gb_algorithm,
+ x, box, fr, &top->idef, graph, fr->born);
+ }
+
+ if (DOMAINDECOMP(cr))
+ {
+ if (!(cr->duty & DUTY_PME))
+ {
+ wallcycle_start(wcycle, ewcPPDURINGPME);
+ dd_force_flop_start(cr->dd, nrnb);
+ }
+ }
+
+ if (inputrec->bRot)
+ {
+ /* Enforced rotation has its own cycle counter that starts after the collective
+ * coordinates have been communicated. It is added to ddCyclF to allow
+ * for proper load-balancing */
+ wallcycle_start(wcycle, ewcROT);
+ do_rotation(cr, inputrec, box, x, t, step, wcycle, bNS);
+ wallcycle_stop(wcycle, ewcROT);
+ }
+
+ /* Start the force cycle counter.
+ * This counter is stopped in do_forcelow_level.
+ * No parallel communication should occur while this counter is running,
+ * since that will interfere with the dynamic load balancing.
+ */
+ wallcycle_start(wcycle, ewcFORCE);
+
+ if (bDoForces)
+ {
+ /* Reset forces for which the virial is calculated separately:
+ * PME/Ewald forces if necessary */
+ if (fr->bF_NoVirSum)
+ {
+ if (flags & GMX_FORCE_VIRIAL)
+ {
+ fr->f_novirsum = fr->f_novirsum_alloc;
+ if (fr->bDomDec)
+ {
+ clear_rvecs(fr->f_novirsum_n, fr->f_novirsum);
+ }
+ else
+ {
+ clear_rvecs(homenr, fr->f_novirsum+start);
+ }
+ }
+ else
+ {
+ /* We are not calculating the pressure so we do not need
+ * a separate array for forces that do not contribute
+ * to the pressure.
+ */
+ fr->f_novirsum = f;
+ }
+ }
+
+ /* Clear the short- and long-range forces */
+ clear_rvecs(fr->natoms_force_constr, f);
+ if (bSepLRF && do_per_step(step, inputrec->nstcalclr))
+ {
+ clear_rvecs(fr->natoms_force_constr, fr->f_twin);
+ }
+
+ clear_rvec(fr->vir_diag_posres);
+ }
+ if (inputrec->ePull == epullCONSTRAINT)
+ {
+ clear_pull_forces(inputrec->pull);
+ }
+
+ /* update QMMMrec, if necessary */
+ if (fr->bQMMM)
+ {
+ update_QMMMrec(cr, fr, x, mdatoms, box, top);
+ }
+
+ if ((flags & GMX_FORCE_BONDED) && top->idef.il[F_POSRES].nr > 0)
+ {
+ posres_wrapper(fplog, flags, bSepDVDL, inputrec, nrnb, top, box, x,
+ enerd, lambda, fr);
+ }
+
+ if ((flags & GMX_FORCE_BONDED) && top->idef.il[F_FBPOSRES].nr > 0)
+ {
+ /* Flat-bottomed position restraints always require full pbc */
+ if (!(bStateChanged && bDoAdressWF))
+ {
+ set_pbc(&pbc, inputrec->ePBC, box);
+ }
+ v = fbposres(top->idef.il[F_FBPOSRES].nr, top->idef.il[F_FBPOSRES].iatoms,
+ top->idef.iparams_fbposres,
+ (const rvec*)x, fr->f_novirsum, fr->vir_diag_posres,
+ inputrec->ePBC == epbcNONE ? NULL : &pbc,
+ fr->rc_scaling, fr->ePBC, fr->posres_com);
+ enerd->term[F_FBPOSRES] += v;
+ inc_nrnb(nrnb, eNR_FBPOSRES, top->idef.il[F_FBPOSRES].nr/2);
+ }
+
+ /* Compute the bonded and non-bonded energies and optionally forces */
+ do_force_lowlevel(fplog, step, fr, inputrec, &(top->idef),
+ cr, nrnb, wcycle, mdatoms,
+ x, hist, f, bSepLRF ? fr->f_twin : f, enerd, fcd, top, fr->born,
+ &(top->atomtypes), bBornRadii, box,
+ inputrec->fepvals, lambda,
+ graph, &(top->excls), fr->mu_tot,
+ flags,
+ &cycles_pme);
+
+ if (bSepLRF)
+ {
+ if (do_per_step(step, inputrec->nstcalclr))
+ {
+ /* Add the long range forces to the short range forces */
+ for (i = 0; i < fr->natoms_force_constr; i++)
+ {
+ rvec_add(fr->f_twin[i], f[i], f[i]);
+ }
+ }
+ }
+
+ cycles_force = wallcycle_stop(wcycle, ewcFORCE);
+
+ if (ed)
+ {
+ do_flood(cr, inputrec, x, f, ed, box, step, bNS);
+ }
+
+ if (DOMAINDECOMP(cr))
+ {
+ dd_force_flop_stop(cr->dd, nrnb);
+ if (wcycle)
+ {
+ dd_cycles_add(cr->dd, cycles_force-cycles_pme, ddCyclF);
+ }
+ }
+
+ if (bDoForces)
+ {
+ if (IR_ELEC_FIELD(*inputrec))
+ {
+ /* Compute forces due to electric field */
+ calc_f_el(MASTER(cr) ? field : NULL,
+ start, homenr, mdatoms->chargeA, fr->f_novirsum,
+ inputrec->ex, inputrec->et, t);
+ }
+
+ if (bDoAdressWF && fr->adress_icor == eAdressICThermoForce)
+ {
+ /* Compute thermodynamic force in hybrid AdResS region */
+ adress_thermo_force(start, homenr, &(top->cgs), x, fr->f_novirsum, fr, mdatoms,
+ inputrec->ePBC == epbcNONE ? NULL : &pbc);
+ }
+
+ /* Communicate the forces */
+ if (PAR(cr))
+ {
+ wallcycle_start(wcycle, ewcMOVEF);
+ if (DOMAINDECOMP(cr))
+ {
+ dd_move_f(cr->dd, f, fr->fshift);
+ /* Do we need to communicate the separate force array
+ * for terms that do not contribute to the single sum virial?
+ * Position restraints and electric fields do not introduce
+ * inter-cg forces, only full electrostatics methods do.
+ * When we do not calculate the virial, fr->f_novirsum = f,
+ * so we have already communicated these forces.
+ */
+ if (EEL_FULL(fr->eeltype) && cr->dd->n_intercg_excl &&
+ (flags & GMX_FORCE_VIRIAL))
+ {
+ dd_move_f(cr->dd, fr->f_novirsum, NULL);
+ }
+ if (bSepLRF)
+ {
+ /* We should not update the shift forces here,
+ * since f_twin is already included in f.
+ */
+ dd_move_f(cr->dd, fr->f_twin, NULL);
+ }
+ }
+ else
+ {
+ pd_move_f(cr, f, nrnb);
+ if (bSepLRF)
+ {
+ pd_move_f(cr, fr->f_twin, nrnb);
+ }
+ }
+ wallcycle_stop(wcycle, ewcMOVEF);
+ }
+
+ /* If we have NoVirSum forces, but we do not calculate the virial,
+ * we sum fr->f_novirum=f later.
+ */
+ if (vsite && !(fr->bF_NoVirSum && !(flags & GMX_FORCE_VIRIAL)))
+ {
+ wallcycle_start(wcycle, ewcVSITESPREAD);
+ spread_vsite_f(vsite, x, f, fr->fshift, FALSE, NULL, nrnb,
+ &top->idef, fr->ePBC, fr->bMolPBC, graph, box, cr);
+ wallcycle_stop(wcycle, ewcVSITESPREAD);
+
+ if (bSepLRF)
+ {
+ wallcycle_start(wcycle, ewcVSITESPREAD);
+ spread_vsite_f(vsite, x, fr->f_twin, NULL, FALSE, NULL,
+ nrnb,
+ &top->idef, fr->ePBC, fr->bMolPBC, graph, box, cr);
+ wallcycle_stop(wcycle, ewcVSITESPREAD);
+ }
+ }
+
+ if (flags & GMX_FORCE_VIRIAL)
+ {
+ /* Calculation of the virial must be done after vsites! */
+ calc_virial(mdatoms->start, mdatoms->homenr, x, f,
+ vir_force, graph, box, nrnb, fr, inputrec->ePBC);
+ }
+ }
+
+ if (inputrec->ePull == epullUMBRELLA || inputrec->ePull == epullCONST_F)
+ {
+ pull_potential_wrapper(fplog, bSepDVDL, cr, inputrec, box, x,
+ f, vir_force, mdatoms, enerd, lambda, t);
+ }
+
+ /* Add the forces from enforced rotation potentials (if any) */
+ if (inputrec->bRot)
+ {
+ wallcycle_start(wcycle, ewcROTadd);
+ enerd->term[F_COM_PULL] += add_rot_forces(inputrec->rot, f, cr, step, t);
+ wallcycle_stop(wcycle, ewcROTadd);
+ }
+
+ if (PAR(cr) && !(cr->duty & DUTY_PME))
+ {
+ /* In case of node-splitting, the PP nodes receive the long-range
+ * forces, virial and energy from the PME nodes here.
+ */
+ pme_receive_force_ener(fplog, bSepDVDL, cr, wcycle, enerd, fr);
+ }
+
+ if (bDoForces)
+ {
+ post_process_forces(cr, step, nrnb, wcycle,
+ top, box, x, f, vir_force, mdatoms, graph, fr, vsite,
+ flags);
+ }
+
+ /* Sum the potential energy terms from group contributions */
+ sum_epot(&(enerd->grpp), enerd->term);
+}
+
+void do_force(FILE *fplog, t_commrec *cr,
+ t_inputrec *inputrec,
+ gmx_large_int_t step, t_nrnb *nrnb, gmx_wallcycle_t wcycle,
+ gmx_localtop_t *top,
+ gmx_groups_t *groups,
+ matrix box, rvec x[], history_t *hist,
+ rvec f[],
+ tensor vir_force,
+ t_mdatoms *mdatoms,
+ gmx_enerdata_t *enerd, t_fcdata *fcd,
+ real *lambda, t_graph *graph,
+ t_forcerec *fr,
+ gmx_vsite_t *vsite, rvec mu_tot,
+ double t, FILE *field, gmx_edsam_t ed,
+ gmx_bool bBornRadii,
+ int flags)
+{
+ /* modify force flag if not doing nonbonded */
+ if (!fr->bNonbonded)
+ {
+ flags &= ~GMX_FORCE_NONBONDED;
+ }
+
+ switch (inputrec->cutoff_scheme)
+ {
+ case ecutsVERLET:
+ do_force_cutsVERLET(fplog, cr, inputrec,
+ step, nrnb, wcycle,
+ top,
+ groups,
+ box, x, hist,
+ f, vir_force,
+ mdatoms,
+ enerd, fcd,
+ lambda, graph,
+ fr, fr->ic,
+ vsite, mu_tot,
+ t, field, ed,
+ bBornRadii,
+ flags);
+ break;
+ case ecutsGROUP:
+ do_force_cutsGROUP(fplog, cr, inputrec,
+ step, nrnb, wcycle,
+ top,
+ groups,
+ box, x, hist,
+ f, vir_force,
+ mdatoms,
+ enerd, fcd,
+ lambda, graph,
+ fr, vsite, mu_tot,
+ t, field, ed,
+ bBornRadii,
+ flags);
+ break;
+ default:
+ gmx_incons("Invalid cut-off scheme passed!");
+ }
+}
+
+
+void do_constrain_first(FILE *fplog, gmx_constr_t constr,
+ t_inputrec *ir, t_mdatoms *md,
+ t_state *state, t_commrec *cr, t_nrnb *nrnb,
+ t_forcerec *fr, gmx_localtop_t *top)
+{
+ int i, m, start, end;
+ gmx_large_int_t step;
+ real dt = ir->delta_t;
+ real dvdl_dum;
+ rvec *savex;
+
+ snew(savex, state->natoms);
+
+ start = md->start;
+ end = md->homenr + start;
+
+ if (debug)
+ {
+ fprintf(debug, "vcm: start=%d, homenr=%d, end=%d\n",
+ start, md->homenr, end);
+ }
+ /* Do a first constrain to reset particles... */
+ step = ir->init_step;
+ if (fplog)
+ {
+ char buf[STEPSTRSIZE];
+ fprintf(fplog, "\nConstraining the starting coordinates (step %s)\n",
+ gmx_step_str(step, buf));
+ }
+ dvdl_dum = 0;
+
+ /* constrain the current position */
+ constrain(NULL, TRUE, FALSE, constr, &(top->idef),
+ ir, NULL, cr, step, 0, md,
+ state->x, state->x, NULL,
+ fr->bMolPBC, state->box,
+ state->lambda[efptBONDED], &dvdl_dum,
+ NULL, NULL, nrnb, econqCoord,
+ ir->epc == epcMTTK, state->veta, state->veta);
+ if (EI_VV(ir->eI))
+ {
+ /* constrain the inital velocity, and save it */
+ /* also may be useful if we need the ekin from the halfstep for velocity verlet */
+ /* might not yet treat veta correctly */
+ constrain(NULL, TRUE, FALSE, constr, &(top->idef),
+ ir, NULL, cr, step, 0, md,
+ state->x, state->v, state->v,
+ fr->bMolPBC, state->box,
+ state->lambda[efptBONDED], &dvdl_dum,
+ NULL, NULL, nrnb, econqVeloc,
+ ir->epc == epcMTTK, state->veta, state->veta);
+ }
+ /* constrain the inital velocities at t-dt/2 */
+ if (EI_STATE_VELOCITY(ir->eI) && ir->eI != eiVV)
+ {
+ for (i = start; (i < end); i++)
+ {
+ for (m = 0; (m < DIM); m++)
+ {
+ /* Reverse the velocity */
+ state->v[i][m] = -state->v[i][m];
+ /* Store the position at t-dt in buf */
+ savex[i][m] = state->x[i][m] + dt*state->v[i][m];
+ }
+ }
+ /* Shake the positions at t=-dt with the positions at t=0
+ * as reference coordinates.
+ */
+ if (fplog)
+ {
+ char buf[STEPSTRSIZE];
+ fprintf(fplog, "\nConstraining the coordinates at t0-dt (step %s)\n",
+ gmx_step_str(step, buf));
+ }
+ dvdl_dum = 0;
+ constrain(NULL, TRUE, FALSE, constr, &(top->idef),
+ ir, NULL, cr, step, -1, md,
+ state->x, savex, NULL,
+ fr->bMolPBC, state->box,
+ state->lambda[efptBONDED], &dvdl_dum,
+ state->v, NULL, nrnb, econqCoord,
+ ir->epc == epcMTTK, state->veta, state->veta);
+
+ for (i = start; i < end; i++)
+ {
+ for (m = 0; m < DIM; m++)
+ {
+ /* Re-reverse the velocities */
+ state->v[i][m] = -state->v[i][m];
+ }
+ }
+ }
+ sfree(savex);
+}
+
+void calc_enervirdiff(FILE *fplog, int eDispCorr, t_forcerec *fr)
+{
+ double eners[2], virs[2], enersum, virsum, y0, f, g, h;
+ double r0, r1, r, rc3, rc9, ea, eb, ec, pa, pb, pc, pd;
+ double invscale, invscale2, invscale3;
+ int ri0, ri1, ri, i, offstart, offset;
+ real scale, *vdwtab, tabfactor, tmp;
+
+ fr->enershiftsix = 0;
+ fr->enershifttwelve = 0;
+ fr->enerdiffsix = 0;
+ fr->enerdifftwelve = 0;
+ fr->virdiffsix = 0;
+ fr->virdifftwelve = 0;
+
+ if (eDispCorr != edispcNO)
+ {
+ for (i = 0; i < 2; i++)
+ {
+ eners[i] = 0;
+ virs[i] = 0;
+ }
+ if ((fr->vdwtype == evdwSWITCH) || (fr->vdwtype == evdwSHIFT))
+ {
+ if (fr->rvdw_switch == 0)
+ {
+ gmx_fatal(FARGS,
+ "With dispersion correction rvdw-switch can not be zero "
+ "for vdw-type = %s", evdw_names[fr->vdwtype]);
+ }
+
+ scale = fr->nblists[0].table_elec_vdw.scale;
+ vdwtab = fr->nblists[0].table_vdw.data;
+
+ /* Round the cut-offs to exact table values for precision */
+ ri0 = floor(fr->rvdw_switch*scale);
+ ri1 = ceil(fr->rvdw*scale);
+ r0 = ri0/scale;
+ r1 = ri1/scale;
+ rc3 = r0*r0*r0;
+ rc9 = rc3*rc3*rc3;
+
+ if (fr->vdwtype == evdwSHIFT)
+ {
+ /* Determine the constant energy shift below rvdw_switch.
+ * Table has a scale factor since we have scaled it down to compensate
+ * for scaling-up c6/c12 with the derivative factors to save flops in analytical kernels.
+ */
+ fr->enershiftsix = (real)(-1.0/(rc3*rc3)) - 6.0*vdwtab[8*ri0];
+ fr->enershifttwelve = (real)( 1.0/(rc9*rc3)) - 12.0*vdwtab[8*ri0 + 4];
+ }
+ /* Add the constant part from 0 to rvdw_switch.
+ * This integration from 0 to rvdw_switch overcounts the number
+ * of interactions by 1, as it also counts the self interaction.
+ * We will correct for this later.
+ */
+ eners[0] += 4.0*M_PI*fr->enershiftsix*rc3/3.0;
+ eners[1] += 4.0*M_PI*fr->enershifttwelve*rc3/3.0;
+
+ invscale = 1.0/(scale);
+ invscale2 = invscale*invscale;
+ invscale3 = invscale*invscale2;
+
+ /* following summation derived from cubic spline definition,
+ Numerical Recipies in C, second edition, p. 113-116. Exact
+ for the cubic spline. We first calculate the negative of
+ the energy from rvdw to rvdw_switch, assuming that g(r)=1,
+ and then add the more standard, abrupt cutoff correction to
+ that result, yielding the long-range correction for a
+ switched function. We perform both the pressure and energy
+ loops at the same time for simplicity, as the computational
+ cost is low. */
+
+ for (i = 0; i < 2; i++)
+ {
+ enersum = 0.0; virsum = 0.0;
+ if (i == 0)
+ {
+ offstart = 0;
+ /* Since the dispersion table has been scaled down a factor 6.0 and the repulsion
+ * a factor 12.0 to compensate for the c6/c12 parameters inside nbfp[] being scaled
+ * up (to save flops in kernels), we need to correct for this.
+ */
+ tabfactor = 6.0;
+ }
+ else
+ {
+ offstart = 4;
+ tabfactor = 12.0;
+ }
+ for (ri = ri0; ri < ri1; ri++)
+ {
+ r = ri*invscale;
+ ea = invscale3;
+ eb = 2.0*invscale2*r;
+ ec = invscale*r*r;
+
+ pa = invscale3;
+ pb = 3.0*invscale2*r;
+ pc = 3.0*invscale*r*r;
+ pd = r*r*r;
+
+ /* this "8" is from the packing in the vdwtab array - perhaps should be #define'ed? */
+ offset = 8*ri + offstart;
+ y0 = vdwtab[offset];
+ f = vdwtab[offset+1];
+ g = vdwtab[offset+2];
+ h = vdwtab[offset+3];
+
+ enersum += y0*(ea/3 + eb/2 + ec) + f*(ea/4 + eb/3 + ec/2) + g*(ea/5 + eb/4 + ec/3) + h*(ea/6 + eb/5 + ec/4);
+ virsum += f*(pa/4 + pb/3 + pc/2 + pd) + 2*g*(pa/5 + pb/4 + pc/3 + pd/2) + 3*h*(pa/6 + pb/5 + pc/4 + pd/3);
+ }
+
+ enersum *= 4.0*M_PI*tabfactor;
+ virsum *= 4.0*M_PI*tabfactor;
+ eners[i] -= enersum;
+ virs[i] -= virsum;
+ }
+
+ /* now add the correction for rvdw_switch to infinity */
+ eners[0] += -4.0*M_PI/(3.0*rc3);
+ eners[1] += 4.0*M_PI/(9.0*rc9);
+ virs[0] += 8.0*M_PI/rc3;
+ virs[1] += -16.0*M_PI/(3.0*rc9);
+ }
+ else if ((fr->vdwtype == evdwCUT) || (fr->vdwtype == evdwUSER))
+ {
+ if (fr->vdwtype == evdwUSER && fplog)
+ {
+ fprintf(fplog,
+ "WARNING: using dispersion correction with user tables\n");
+ }
+ rc3 = fr->rvdw*fr->rvdw*fr->rvdw;
+ rc9 = rc3*rc3*rc3;
+ /* Contribution beyond the cut-off */
+ eners[0] += -4.0*M_PI/(3.0*rc3);
+ eners[1] += 4.0*M_PI/(9.0*rc9);
+ if (fr->vdw_modifier == eintmodPOTSHIFT)
+ {
+ /* Contribution within the cut-off */
+ eners[0] += -4.0*M_PI/(3.0*rc3);
+ eners[1] += 4.0*M_PI/(3.0*rc9);
+ }
+ /* Contribution beyond the cut-off */
+ virs[0] += 8.0*M_PI/rc3;
+ virs[1] += -16.0*M_PI/(3.0*rc9);
+ }
+ else
+ {
+ gmx_fatal(FARGS,
+ "Dispersion correction is not implemented for vdw-type = %s",
+ evdw_names[fr->vdwtype]);
+ }
+ fr->enerdiffsix = eners[0];
+ fr->enerdifftwelve = eners[1];
+ /* The 0.5 is due to the Gromacs definition of the virial */
+ fr->virdiffsix = 0.5*virs[0];
+ fr->virdifftwelve = 0.5*virs[1];
+ }
+}
+
+void calc_dispcorr(FILE *fplog, t_inputrec *ir, t_forcerec *fr,
+ gmx_large_int_t step, int natoms,
+ matrix box, real lambda, tensor pres, tensor virial,
+ real *prescorr, real *enercorr, real *dvdlcorr)
+{
+ gmx_bool bCorrAll, bCorrPres;
+ real dvdlambda, invvol, dens, ninter, avcsix, avctwelve, enerdiff, svir = 0, spres = 0;
+ int m;
+
+ *prescorr = 0;
+ *enercorr = 0;
+ *dvdlcorr = 0;
+
+ clear_mat(virial);
+ clear_mat(pres);
+
+ if (ir->eDispCorr != edispcNO)
+ {
+ bCorrAll = (ir->eDispCorr == edispcAllEner ||
+ ir->eDispCorr == edispcAllEnerPres);
+ bCorrPres = (ir->eDispCorr == edispcEnerPres ||
+ ir->eDispCorr == edispcAllEnerPres);
+
+ invvol = 1/det(box);
+ if (fr->n_tpi)
+ {
+ /* Only correct for the interactions with the inserted molecule */
+ dens = (natoms - fr->n_tpi)*invvol;
+ ninter = fr->n_tpi;
+ }
+ else
+ {
+ dens = natoms*invvol;
+ ninter = 0.5*natoms;
+ }
+
+ if (ir->efep == efepNO)
+ {
+ avcsix = fr->avcsix[0];
+ avctwelve = fr->avctwelve[0];
+ }
+ else
+ {
+ avcsix = (1 - lambda)*fr->avcsix[0] + lambda*fr->avcsix[1];
+ avctwelve = (1 - lambda)*fr->avctwelve[0] + lambda*fr->avctwelve[1];
+ }
+
+ enerdiff = ninter*(dens*fr->enerdiffsix - fr->enershiftsix);
+ *enercorr += avcsix*enerdiff;
+ dvdlambda = 0.0;
+ if (ir->efep != efepNO)
+ {
+ dvdlambda += (fr->avcsix[1] - fr->avcsix[0])*enerdiff;
+ }
+ if (bCorrAll)
+ {
+ enerdiff = ninter*(dens*fr->enerdifftwelve - fr->enershifttwelve);
+ *enercorr += avctwelve*enerdiff;
+ if (fr->efep != efepNO)
+ {
+ dvdlambda += (fr->avctwelve[1] - fr->avctwelve[0])*enerdiff;
+ }
+ }
+
+ if (bCorrPres)
+ {
+ svir = ninter*dens*avcsix*fr->virdiffsix/3.0;
+ if (ir->eDispCorr == edispcAllEnerPres)
+ {
+ svir += ninter*dens*avctwelve*fr->virdifftwelve/3.0;
+ }
+ /* The factor 2 is because of the Gromacs virial definition */
+ spres = -2.0*invvol*svir*PRESFAC;
+
+ for (m = 0; m < DIM; m++)
+ {
+ virial[m][m] += svir;
+ pres[m][m] += spres;
+ }
+ *prescorr += spres;
+ }
+
+ /* Can't currently control when it prints, for now, just print when degugging */
+ if (debug)
+ {
+ if (bCorrAll)
+ {
+ fprintf(debug, "Long Range LJ corr.: <C6> %10.4e, <C12> %10.4e\n",
+ avcsix, avctwelve);
+ }
+ if (bCorrPres)
+ {
+ fprintf(debug,
+ "Long Range LJ corr.: Epot %10g, Pres: %10g, Vir: %10g\n",
+ *enercorr, spres, svir);
+ }
+ else
+ {
+ fprintf(debug, "Long Range LJ corr.: Epot %10g\n", *enercorr);
+ }
+ }
+
+ if (fr->bSepDVDL && do_per_step(step, ir->nstlog))
+ {
+ gmx_print_sepdvdl(fplog, "Dispersion correction", *enercorr, dvdlambda);
+ }
+ if (fr->efep != efepNO)
+ {
+ *dvdlcorr += dvdlambda;
+ }
+ }
+}
+
+void do_pbc_first(FILE *fplog, matrix box, t_forcerec *fr,
+ t_graph *graph, rvec x[])
+{
+ if (fplog)
+ {
+ fprintf(fplog, "Removing pbc first time\n");
+ }
+ calc_shifts(box, fr->shift_vec);
+ if (graph)
+ {
+ mk_mshift(fplog, graph, fr->ePBC, box, x);
+ if (gmx_debug_at)
+ {
+ p_graph(debug, "do_pbc_first 1", graph);
+ }
+ shift_self(graph, box, x);
+ /* By doing an extra mk_mshift the molecules that are broken
+ * because they were e.g. imported from another software
+ * will be made whole again. Such are the healing powers
+ * of GROMACS.
+ */
+ mk_mshift(fplog, graph, fr->ePBC, box, x);
+ if (gmx_debug_at)
+ {
+ p_graph(debug, "do_pbc_first 2", graph);
+ }
+ }
+ if (fplog)
+ {
+ fprintf(fplog, "Done rmpbc\n");
+ }
+}
+
+static void low_do_pbc_mtop(FILE *fplog, int ePBC, matrix box,
+ gmx_mtop_t *mtop, rvec x[],
+ gmx_bool bFirst)
+{
+ t_graph *graph;
+ int mb, as, mol;
+ gmx_molblock_t *molb;
+
+ if (bFirst && fplog)
+ {
+ fprintf(fplog, "Removing pbc first time\n");
+ }
+
+ snew(graph, 1);
+ as = 0;
+ for (mb = 0; mb < mtop->nmolblock; mb++)
+ {
+ molb = &mtop->molblock[mb];
+ if (molb->natoms_mol == 1 ||
+ (!bFirst && mtop->moltype[molb->type].cgs.nr == 1))
+ {
+ /* Just one atom or charge group in the molecule, no PBC required */
+ as += molb->nmol*molb->natoms_mol;
+ }
+ else
+ {
+ /* Pass NULL iso fplog to avoid graph prints for each molecule type */
+ mk_graph_ilist(NULL, mtop->moltype[molb->type].ilist,
+ 0, molb->natoms_mol, FALSE, FALSE, graph);
+
+ for (mol = 0; mol < molb->nmol; mol++)
+ {
+ mk_mshift(fplog, graph, ePBC, box, x+as);
+
+ shift_self(graph, box, x+as);
+ /* The molecule is whole now.
+ * We don't need the second mk_mshift call as in do_pbc_first,
+ * since we no longer need this graph.
+ */
+
+ as += molb->natoms_mol;
+ }
+ done_graph(graph);
+ }
+ }
+ sfree(graph);
+}
+
+void do_pbc_first_mtop(FILE *fplog, int ePBC, matrix box,
+ gmx_mtop_t *mtop, rvec x[])
+{
+ low_do_pbc_mtop(fplog, ePBC, box, mtop, x, TRUE);
+}
+
+void do_pbc_mtop(FILE *fplog, int ePBC, matrix box,
+ gmx_mtop_t *mtop, rvec x[])
+{
+ low_do_pbc_mtop(fplog, ePBC, box, mtop, x, FALSE);
+}
+
+void finish_run(FILE *fplog, t_commrec *cr,
+ t_inputrec *inputrec,
+ t_nrnb nrnb[], gmx_wallcycle_t wcycle,
+ gmx_walltime_accounting_t walltime_accounting,
+ wallclock_gpu_t *gputimes,
+ gmx_bool bWriteStat)
+{
+ int i, j;
+ t_nrnb *nrnb_tot = NULL;
+ real delta_t;
+ double nbfs, mflop;
+ double elapsed_time,
+ elapsed_time_over_all_ranks,
+ elapsed_time_over_all_threads,
+ elapsed_time_over_all_threads_over_all_ranks;
+ wallcycle_sum(cr, wcycle);
+
+ if (cr->nnodes > 1)
+ {
+ snew(nrnb_tot, 1);
+#ifdef GMX_MPI
+ MPI_Allreduce(nrnb->n, nrnb_tot->n, eNRNB, MPI_DOUBLE, MPI_SUM,
+ cr->mpi_comm_mysim);
+#endif
+ }
+ else
+ {
+ nrnb_tot = nrnb;
+ }
+
+ elapsed_time = walltime_accounting_get_elapsed_time(walltime_accounting);
+ elapsed_time_over_all_ranks = elapsed_time;
+ elapsed_time_over_all_threads = walltime_accounting_get_elapsed_time_over_all_threads(walltime_accounting);
+ elapsed_time_over_all_threads_over_all_ranks = elapsed_time_over_all_threads;
+#ifdef GMX_MPI
+ if (cr->nnodes > 1)
+ {
+ /* reduce elapsed_time over all MPI ranks in the current simulation */
+ MPI_Allreduce(&elapsed_time,
+ &elapsed_time_over_all_ranks,
+ 1, MPI_DOUBLE, MPI_SUM,
+ cr->mpi_comm_mysim);
+ elapsed_time_over_all_ranks /= cr->nnodes;
+ /* Reduce elapsed_time_over_all_threads over all MPI ranks in the
+ * current simulation. */
+ MPI_Allreduce(&elapsed_time_over_all_threads,
+ &elapsed_time_over_all_threads_over_all_ranks,
+ 1, MPI_DOUBLE, MPI_SUM,
+ cr->mpi_comm_mysim);
+ }
+#endif
+
+ if (SIMMASTER(cr))
+ {
+ print_flop(fplog, nrnb_tot, &nbfs, &mflop);
+ }
+ if (cr->nnodes > 1)
+ {
+ sfree(nrnb_tot);
+ }
+
+ if ((cr->duty & DUTY_PP) && DOMAINDECOMP(cr))
+ {
+ print_dd_statistics(cr, inputrec, fplog);
+ }
+
+#ifdef GMX_MPI
+ if (PARTDECOMP(cr))
+ {
+ if (MASTER(cr))
+ {
+ t_nrnb *nrnb_all;
+ int s;
+ MPI_Status stat;
+
+ snew(nrnb_all, cr->nnodes);
+ nrnb_all[0] = *nrnb;
+ for (s = 1; s < cr->nnodes; s++)
+ {
+ MPI_Recv(nrnb_all[s].n, eNRNB, MPI_DOUBLE, s, 0,
+ cr->mpi_comm_mysim, &stat);
+ }
+ pr_load(fplog, cr, nrnb_all);
+ sfree(nrnb_all);
+ }
+ else
+ {
+ MPI_Send(nrnb->n, eNRNB, MPI_DOUBLE, MASTERRANK(cr), 0,
+ cr->mpi_comm_mysim);
+ }
+ }
+#endif
+
+ if (SIMMASTER(cr))
+ {
+ wallcycle_print(fplog, cr->nnodes, cr->npmenodes,
+ elapsed_time_over_all_ranks,
+ wcycle, gputimes);
+
+ if (EI_DYNAMICS(inputrec->eI))
+ {
+ delta_t = inputrec->delta_t;
+ }
+ else
+ {
+ delta_t = 0;
+ }
+
+ if (fplog)
+ {
+ print_perf(fplog, elapsed_time_over_all_threads_over_all_ranks,
+ elapsed_time_over_all_ranks,
+ walltime_accounting_get_nsteps_done(walltime_accounting),
+ delta_t, nbfs, mflop);
+ }
+ if (bWriteStat)
+ {
+ print_perf(stderr, elapsed_time_over_all_threads_over_all_ranks,
+ elapsed_time_over_all_ranks,
+ walltime_accounting_get_nsteps_done(walltime_accounting),
+ delta_t, nbfs, mflop);
+ }
+ }
+}
+
+extern void initialize_lambdas(FILE *fplog, t_inputrec *ir, int *fep_state, real *lambda, double *lam0)
+{
+ /* this function works, but could probably use a logic rewrite to keep all the different
+ types of efep straight. */
+
+ int i;
+ t_lambda *fep = ir->fepvals;
+
+ if ((ir->efep == efepNO) && (ir->bSimTemp == FALSE))
+ {
+ for (i = 0; i < efptNR; i++)
+ {
+ lambda[i] = 0.0;
+ if (lam0)
+ {
+ lam0[i] = 0.0;
+ }
+ }
+ return;
+ }
+ else
+ {
+ *fep_state = fep->init_fep_state; /* this might overwrite the checkpoint
+ if checkpoint is set -- a kludge is in for now
+ to prevent this.*/
+ for (i = 0; i < efptNR; i++)
+ {
+ /* overwrite lambda state with init_lambda for now for backwards compatibility */
+ if (fep->init_lambda >= 0) /* if it's -1, it was never initializd */
+ {
+ lambda[i] = fep->init_lambda;
+ if (lam0)
+ {
+ lam0[i] = lambda[i];
+ }
+ }
+ else
+ {
+ lambda[i] = fep->all_lambda[i][*fep_state];
+ if (lam0)
+ {
+ lam0[i] = lambda[i];
+ }
+ }
+ }
+ if (ir->bSimTemp)
+ {
+ /* need to rescale control temperatures to match current state */
+ for (i = 0; i < ir->opts.ngtc; i++)
+ {
+ if (ir->opts.ref_t[i] > 0)
+ {
+ ir->opts.ref_t[i] = ir->simtempvals->temperatures[*fep_state];
+ }
+ }
+ }
+ }
+
+ /* Send to the log the information on the current lambdas */
+ if (fplog != NULL)
+ {
+ fprintf(fplog, "Initial vector of lambda components:[ ");
+ for (i = 0; i < efptNR; i++)
+ {
+ fprintf(fplog, "%10.4f ", lambda[i]);
+ }
+ fprintf(fplog, "]\n");
+ }
+ return;
+}
+
+
+void init_md(FILE *fplog,
+ t_commrec *cr, t_inputrec *ir, const output_env_t oenv,
+ double *t, double *t0,
+ real *lambda, int *fep_state, double *lam0,
+ t_nrnb *nrnb, gmx_mtop_t *mtop,
+ gmx_update_t *upd,
+ int nfile, const t_filenm fnm[],
+ gmx_mdoutf_t **outf, t_mdebin **mdebin,
+ tensor force_vir, tensor shake_vir, rvec mu_tot,
+ gmx_bool *bSimAnn, t_vcm **vcm, unsigned long Flags)
+{
+ int i, j, n;
+ real tmpt, mod;
+
+ /* Initial values */
+ *t = *t0 = ir->init_t;
+
+ *bSimAnn = FALSE;
+ for (i = 0; i < ir->opts.ngtc; i++)
+ {
+ /* set bSimAnn if any group is being annealed */
+ if (ir->opts.annealing[i] != eannNO)
+ {
+ *bSimAnn = TRUE;
+ }
+ }
+ if (*bSimAnn)
+ {
+ update_annealing_target_temp(&(ir->opts), ir->init_t);
+ }
+
+ /* Initialize lambda variables */
+ initialize_lambdas(fplog, ir, fep_state, lambda, lam0);
+
+ if (upd)
+ {
+ *upd = init_update(ir);
+ }
+
+
+ if (vcm != NULL)
+ {
+ *vcm = init_vcm(fplog, &mtop->groups, ir);
+ }
+
+ if (EI_DYNAMICS(ir->eI) && !(Flags & MD_APPENDFILES))
+ {
+ if (ir->etc == etcBERENDSEN)
+ {
+ please_cite(fplog, "Berendsen84a");
+ }
+ if (ir->etc == etcVRESCALE)
+ {
+ please_cite(fplog, "Bussi2007a");
+ }
+ }
+
+ init_nrnb(nrnb);
+
+ if (nfile != -1)
+ {
+ *outf = init_mdoutf(nfile, fnm, Flags, cr, ir, oenv);
+
+ *mdebin = init_mdebin((Flags & MD_APPENDFILES) ? NULL : (*outf)->fp_ene,
+ mtop, ir, (*outf)->fp_dhdl);
+ }
+
+ if (ir->bAdress)
+ {
+ please_cite(fplog, "Fritsch12");
+ please_cite(fplog, "Junghans10");
+ }
+ /* Initiate variables */
+ clear_mat(force_vir);
+ clear_mat(shake_vir);
+ clear_rvec(mu_tot);
+
+ debug_gmx();
+}
--- /dev/null
- "excluding PME-only processes/threads. With thread-MPI the number",
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * VERSION 3.2.0
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2004, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * Gallium Rubidium Oxygen Manganese Argon Carbon Silicon
+ */
+#include "mdrun_main.h"
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdio.h>
+
+#include "gromacs/legacyheaders/checkpoint.h"
+#include "gromacs/legacyheaders/copyrite.h"
+#include "gromacs/fileio/filenm.h"
+#include "gromacs/legacyheaders/gmx_fatal.h"
+#include "gromacs/legacyheaders/macros.h"
+#include "gromacs/legacyheaders/main.h"
+#include "gromacs/legacyheaders/mdrun.h"
+#include "gromacs/legacyheaders/network.h"
+#include "gromacs/legacyheaders/readinp.h"
+#include "gromacs/legacyheaders/statutil.h"
+#include "gromacs/legacyheaders/typedefs.h"
+
+int gmx_mdrun(int argc, char *argv[])
+{
+ const char *desc[] = {
+ "The [TT]mdrun[tt] program is the main computational chemistry engine",
+ "within GROMACS. Obviously, it performs Molecular Dynamics simulations,",
+ "but it can also perform Stochastic Dynamics, Energy Minimization,",
+ "test particle insertion or (re)calculation of energies.",
+ "Normal mode analysis is another option. In this case [TT]mdrun[tt]",
+ "builds a Hessian matrix from single conformation.",
+ "For usual Normal Modes-like calculations, make sure that",
+ "the structure provided is properly energy-minimized.",
+ "The generated matrix can be diagonalized by [TT]g_nmeig[tt].[PAR]",
+ "The [TT]mdrun[tt] program reads the run input file ([TT]-s[tt])",
+ "and distributes the topology over nodes if needed.",
+ "[TT]mdrun[tt] produces at least four output files.",
+ "A single log file ([TT]-g[tt]) is written, unless the option",
+ "[TT]-seppot[tt] is used, in which case each node writes a log file.",
+ "The trajectory file ([TT]-o[tt]), contains coordinates, velocities and",
+ "optionally forces.",
+ "The structure file ([TT]-c[tt]) contains the coordinates and",
+ "velocities of the last step.",
+ "The energy file ([TT]-e[tt]) contains energies, the temperature,",
+ "pressure, etc, a lot of these things are also printed in the log file.",
+ "Optionally coordinates can be written to a compressed trajectory file",
+ "([TT]-x[tt]).[PAR]",
+ "The option [TT]-dhdl[tt] is only used when free energy calculation is",
+ "turned on.[PAR]",
+ "A simulation can be run in parallel using two different parallelization",
+ "schemes: MPI parallelization and/or OpenMP thread parallelization.",
+ "The MPI parallelization uses multiple processes when [TT]mdrun[tt] is",
+ "compiled with a normal MPI library or threads when [TT]mdrun[tt] is",
+ "compiled with the GROMACS built-in thread-MPI library. OpenMP threads",
+ "are supported when mdrun is compiled with OpenMP. Full OpenMP support",
+ "is only available with the Verlet cut-off scheme, with the (older)",
+ "group scheme only PME-only processes can use OpenMP parallelization.",
+ "In all cases [TT]mdrun[tt] will by default try to use all the available",
+ "hardware resources. With a normal MPI library only the options",
+ "[TT]-ntomp[tt] (with the Verlet cut-off scheme) and [TT]-ntomp_pme[tt],",
+ "for PME-only processes, can be used to control the number of threads.",
+ "With thread-MPI there are additional options [TT]-nt[tt], which sets",
+ "the total number of threads, and [TT]-ntmpi[tt], which sets the number",
+ "of thread-MPI threads.",
+ "Note that using combined MPI+OpenMP parallelization is almost always",
+ "slower than single parallelization, except at the scaling limit, where",
+ "especially OpenMP parallelization of PME reduces the communication cost.",
+ "OpenMP-only parallelization is much faster than MPI-only parallelization",
+ "on a single CPU(-die). Since we currently don't have proper hardware",
+ "topology detection, [TT]mdrun[tt] compiled with thread-MPI will only",
+ "automatically use OpenMP-only parallelization when you use up to 4",
+ "threads, up to 12 threads with Intel Nehalem/Westmere, or up to 16",
+ "threads with Intel Sandy Bridge or newer CPUs. Otherwise MPI-only",
+ "parallelization is used (except with GPUs, see below).",
+ "[PAR]",
+ "To quickly test the performance of the new Verlet cut-off scheme",
+ "with old [TT].tpr[tt] files, either on CPUs or CPUs+GPUs, you can use",
+ "the [TT]-testverlet[tt] option. This should not be used for production,",
+ "since it can slightly modify potentials and it will remove charge groups",
+ "making analysis difficult, as the [TT].tpr[tt] file will still contain",
+ "charge groups. For production simulations it is highly recommended",
+ "to specify [TT]cutoff-scheme = Verlet[tt] in the [TT].mdp[tt] file.",
+ "[PAR]",
+ "With GPUs (only supported with the Verlet cut-off scheme), the number",
+ "of GPUs should match the number of MPI processes or MPI threads,",
- "When you want to use a subset of the available GPUs, you can use",
- "the [TT]-gpu_id[tt] option, where GPU id's are passed as a string,",
- "e.g. 02 for using GPUs 0 and 2. When you want different GPU id's",
- "on different nodes of a compute cluster, use the GMX_GPU_ID environment",
- "variable instead. The format for GMX_GPU_ID is identical to ",
- "[TT]-gpu_id[tt], but an environment variable can have different values",
- "on different nodes of a cluster.",
++ "excluding PME-only processes/threads. With thread-MPI, unless set on the command line, the number",
+ "of MPI threads will automatically be set to the number of GPUs detected.",
- gmx_hw_opt_t hw_opt = {0, 0, 0, 0, threadaffSEL, 0, 0, NULL};
++ "To use a subset of the available GPUs, or to manually provide a mapping of",
++ "GPUs to PP ranks, you can use the [TT]-gpu_id[tt] option. The argument of [TT]-gpu_id[tt] is",
++ "a string of digits (without delimiter) representing device id-s of the GPUs to be used.",
++ "For example, \"[TT]02[tt]\" specifies using GPUs 0 and 2 in the first and second PP ranks per compute node",
++ "respectively. To select different sets of GPU-s",
++ "on different nodes of a compute cluster, use the [TT]GMX_GPU_ID[tt] environment",
++ "variable instead. The format for [TT]GMX_GPU_ID[tt] is identical to ",
++ "[TT]-gpu_id[tt], with the difference that an environment variable can have",
++ "different values on different compute nodes. Multiple MPI ranks on each node",
++ "can share GPUs. This is accomplished by specifying the id(s) of the GPU(s)",
++ "multiple times, e.g. \"[TT]0011[tt]\" for four ranks sharing two GPUs in this node.",
++ "This works within a single simulation, or a multi-simulation, with any form of MPI.",
+ "[PAR]",
+ "When using PME with separate PME nodes or with a GPU, the two major",
+ "compute tasks, the non-bonded force calculation and the PME calculation",
+ "run on different compute resources. If this load is not balanced,",
+ "some of the resources will be idle part of time. With the Verlet",
+ "cut-off scheme this load is automatically balanced when the PME load",
+ "is too high (but not when it is too low). This is done by scaling",
+ "the Coulomb cut-off and PME grid spacing by the same amount. In the first",
+ "few hundred steps different settings are tried and the fastest is chosen",
+ "for the rest of the simulation. This does not affect the accuracy of",
+ "the results, but it does affect the decomposition of the Coulomb energy",
+ "into particle and mesh contributions. The auto-tuning can be turned off",
+ "with the option [TT]-notunepme[tt].",
+ "[PAR]",
+ "[TT]mdrun[tt] pins (sets affinity of) threads to specific cores,",
+ "when all (logical) cores on a compute node are used by [TT]mdrun[tt],",
+ "even when no multi-threading is used,",
+ "as this usually results in significantly better performance.",
+ "If the queuing systems or the OpenMP library pinned threads, we honor",
+ "this and don't pin again, even though the layout may be sub-optimal.",
+ "If you want to have [TT]mdrun[tt] override an already set thread affinity",
+ "or pin threads when using less cores, use [TT]-pin on[tt].",
+ "With SMT (simultaneous multithreading), e.g. Intel Hyper-Threading,",
+ "there are multiple logical cores per physical core.",
+ "The option [TT]-pinstride[tt] sets the stride in logical cores for",
+ "pinning consecutive threads. Without SMT, 1 is usually the best choice.",
+ "With Intel Hyper-Threading 2 is best when using half or less of the",
+ "logical cores, 1 otherwise. The default value of 0 do exactly that:",
+ "it minimizes the threads per logical core, to optimize performance.",
+ "If you want to run multiple mdrun jobs on the same physical node,"
+ "you should set [TT]-pinstride[tt] to 1 when using all logical cores.",
+ "When running multiple mdrun (or other) simulations on the same physical",
+ "node, some simulations need to start pinning from a non-zero core",
+ "to avoid overloading cores; with [TT]-pinoffset[tt] you can specify",
+ "the offset in logical cores for pinning.",
+ "[PAR]",
+ "When [TT]mdrun[tt] is started using MPI with more than 1 process",
+ "or with thread-MPI with more than 1 thread, MPI parallelization is used.",
+ "By default domain decomposition is used, unless the [TT]-pd[tt]",
+ "option is set, which selects particle decomposition.",
+ "[PAR]",
+ "With domain decomposition, the spatial decomposition can be set",
+ "with option [TT]-dd[tt]. By default [TT]mdrun[tt] selects a good decomposition.",
+ "The user only needs to change this when the system is very inhomogeneous.",
+ "Dynamic load balancing is set with the option [TT]-dlb[tt],",
+ "which can give a significant performance improvement,",
+ "especially for inhomogeneous systems. The only disadvantage of",
+ "dynamic load balancing is that runs are no longer binary reproducible,",
+ "but in most cases this is not important.",
+ "By default the dynamic load balancing is automatically turned on",
+ "when the measured performance loss due to load imbalance is 5% or more.",
+ "At low parallelization these are the only important options",
+ "for domain decomposition.",
+ "At high parallelization the options in the next two sections",
+ "could be important for increasing the performace.",
+ "[PAR]",
+ "When PME is used with domain decomposition, separate nodes can",
+ "be assigned to do only the PME mesh calculation;",
+ "this is computationally more efficient starting at about 12 nodes.",
+ "The number of PME nodes is set with option [TT]-npme[tt],",
+ "this can not be more than half of the nodes.",
+ "By default [TT]mdrun[tt] makes a guess for the number of PME",
+ "nodes when the number of nodes is larger than 11 or performance wise",
+ "not compatible with the PME grid x dimension.",
+ "But the user should optimize npme. Performance statistics on this issue",
+ "are written at the end of the log file.",
+ "For good load balancing at high parallelization, the PME grid x and y",
+ "dimensions should be divisible by the number of PME nodes",
+ "(the simulation will run correctly also when this is not the case).",
+ "[PAR]",
+ "This section lists all options that affect the domain decomposition.",
+ "[PAR]",
+ "Option [TT]-rdd[tt] can be used to set the required maximum distance",
+ "for inter charge-group bonded interactions.",
+ "Communication for two-body bonded interactions below the non-bonded",
+ "cut-off distance always comes for free with the non-bonded communication.",
+ "Atoms beyond the non-bonded cut-off are only communicated when they have",
+ "missing bonded interactions; this means that the extra cost is minor",
+ "and nearly indepedent of the value of [TT]-rdd[tt].",
+ "With dynamic load balancing option [TT]-rdd[tt] also sets",
+ "the lower limit for the domain decomposition cell sizes.",
+ "By default [TT]-rdd[tt] is determined by [TT]mdrun[tt] based on",
+ "the initial coordinates. The chosen value will be a balance",
+ "between interaction range and communication cost.",
+ "[PAR]",
+ "When inter charge-group bonded interactions are beyond",
+ "the bonded cut-off distance, [TT]mdrun[tt] terminates with an error message.",
+ "For pair interactions and tabulated bonds",
+ "that do not generate exclusions, this check can be turned off",
+ "with the option [TT]-noddcheck[tt].",
+ "[PAR]",
+ "When constraints are present, option [TT]-rcon[tt] influences",
+ "the cell size limit as well.",
+ "Atoms connected by NC constraints, where NC is the LINCS order plus 1,",
+ "should not be beyond the smallest cell size. A error message is",
+ "generated when this happens and the user should change the decomposition",
+ "or decrease the LINCS order and increase the number of LINCS iterations.",
+ "By default [TT]mdrun[tt] estimates the minimum cell size required for P-LINCS",
+ "in a conservative fashion. For high parallelization it can be useful",
+ "to set the distance required for P-LINCS with the option [TT]-rcon[tt].",
+ "[PAR]",
+ "The [TT]-dds[tt] option sets the minimum allowed x, y and/or z scaling",
+ "of the cells with dynamic load balancing. [TT]mdrun[tt] will ensure that",
+ "the cells can scale down by at least this factor. This option is used",
+ "for the automated spatial decomposition (when not using [TT]-dd[tt])",
+ "as well as for determining the number of grid pulses, which in turn",
+ "sets the minimum allowed cell size. Under certain circumstances",
+ "the value of [TT]-dds[tt] might need to be adjusted to account for",
+ "high or low spatial inhomogeneity of the system.",
+ "[PAR]",
+ "The option [TT]-gcom[tt] can be used to only do global communication",
+ "every n steps.",
+ "This can improve performance for highly parallel simulations",
+ "where this global communication step becomes the bottleneck.",
+ "For a global thermostat and/or barostat the temperature",
+ "and/or pressure will also only be updated every [TT]-gcom[tt] steps.",
+ "By default it is set to the minimum of nstcalcenergy and nstlist.[PAR]",
+ "With [TT]-rerun[tt] an input trajectory can be given for which ",
+ "forces and energies will be (re)calculated. Neighbor searching will be",
+ "performed for every frame, unless [TT]nstlist[tt] is zero",
+ "(see the [TT].mdp[tt] file).[PAR]",
+ "ED (essential dynamics) sampling and/or additional flooding potentials",
+ "are switched on by using the [TT]-ei[tt] flag followed by an [TT].edi[tt]",
+ "file. The [TT].edi[tt] file can be produced with the [TT]make_edi[tt] tool",
+ "or by using options in the essdyn menu of the WHAT IF program.",
+ "[TT]mdrun[tt] produces a [TT].xvg[tt] output file that",
+ "contains projections of positions, velocities and forces onto selected",
+ "eigenvectors.[PAR]",
+ "When user-defined potential functions have been selected in the",
+ "[TT].mdp[tt] file the [TT]-table[tt] option is used to pass [TT]mdrun[tt]",
+ "a formatted table with potential functions. The file is read from",
+ "either the current directory or from the [TT]GMXLIB[tt] directory.",
+ "A number of pre-formatted tables are presented in the [TT]GMXLIB[tt] dir,",
+ "for 6-8, 6-9, 6-10, 6-11, 6-12 Lennard-Jones potentials with",
+ "normal Coulomb.",
+ "When pair interactions are present, a separate table for pair interaction",
+ "functions is read using the [TT]-tablep[tt] option.[PAR]",
+ "When tabulated bonded functions are present in the topology,",
+ "interaction functions are read using the [TT]-tableb[tt] option.",
+ "For each different tabulated interaction type the table file name is",
+ "modified in a different way: before the file extension an underscore is",
+ "appended, then a 'b' for bonds, an 'a' for angles or a 'd' for dihedrals",
+ "and finally the table number of the interaction type.[PAR]",
+ "The options [TT]-px[tt] and [TT]-pf[tt] are used for writing pull COM",
+ "coordinates and forces when pulling is selected",
+ "in the [TT].mdp[tt] file.[PAR]",
+ "With [TT]-multi[tt] or [TT]-multidir[tt], multiple systems can be ",
+ "simulated in parallel.",
+ "As many input files/directories are required as the number of systems. ",
+ "The [TT]-multidir[tt] option takes a list of directories (one for each ",
+ "system) and runs in each of them, using the input/output file names, ",
+ "such as specified by e.g. the [TT]-s[tt] option, relative to these ",
+ "directories.",
+ "With [TT]-multi[tt], the system number is appended to the run input ",
+ "and each output filename, for instance [TT]topol.tpr[tt] becomes",
+ "[TT]topol0.tpr[tt], [TT]topol1.tpr[tt] etc.",
+ "The number of nodes per system is the total number of nodes",
+ "divided by the number of systems.",
+ "One use of this option is for NMR refinement: when distance",
+ "or orientation restraints are present these can be ensemble averaged",
+ "over all the systems.[PAR]",
+ "With [TT]-replex[tt] replica exchange is attempted every given number",
+ "of steps. The number of replicas is set with the [TT]-multi[tt] or ",
+ "[TT]-multidir[tt] option, described above.",
+ "All run input files should use a different coupling temperature,",
+ "the order of the files is not important. The random seed is set with",
+ "[TT]-reseed[tt]. The velocities are scaled and neighbor searching",
+ "is performed after every exchange.[PAR]",
+ "Finally some experimental algorithms can be tested when the",
+ "appropriate options have been given. Currently under",
+ "investigation are: polarizability.",
+ "[PAR]",
+ "The option [TT]-membed[tt] does what used to be g_membed, i.e. embed",
+ "a protein into a membrane. The data file should contain the options",
+ "that where passed to g_membed before. The [TT]-mn[tt] and [TT]-mp[tt]",
+ "both apply to this as well.",
+ "[PAR]",
+ "The option [TT]-pforce[tt] is useful when you suspect a simulation",
+ "crashes due to too large forces. With this option coordinates and",
+ "forces of atoms with a force larger than a certain value will",
+ "be printed to stderr.",
+ "[PAR]",
+ "Checkpoints containing the complete state of the system are written",
+ "at regular intervals (option [TT]-cpt[tt]) to the file [TT]-cpo[tt],",
+ "unless option [TT]-cpt[tt] is set to -1.",
+ "The previous checkpoint is backed up to [TT]state_prev.cpt[tt] to",
+ "make sure that a recent state of the system is always available,",
+ "even when the simulation is terminated while writing a checkpoint.",
+ "With [TT]-cpnum[tt] all checkpoint files are kept and appended",
+ "with the step number.",
+ "A simulation can be continued by reading the full state from file",
+ "with option [TT]-cpi[tt]. This option is intelligent in the way that",
+ "if no checkpoint file is found, Gromacs just assumes a normal run and",
+ "starts from the first step of the [TT].tpr[tt] file. By default the output",
+ "will be appending to the existing output files. The checkpoint file",
+ "contains checksums of all output files, such that you will never",
+ "loose data when some output files are modified, corrupt or removed.",
+ "There are three scenarios with [TT]-cpi[tt]:[PAR]",
+ "[TT]*[tt] no files with matching names are present: new output files are written[PAR]",
+ "[TT]*[tt] all files are present with names and checksums matching those stored",
+ "in the checkpoint file: files are appended[PAR]",
+ "[TT]*[tt] otherwise no files are modified and a fatal error is generated[PAR]",
+ "With [TT]-noappend[tt] new output files are opened and the simulation",
+ "part number is added to all output file names.",
+ "Note that in all cases the checkpoint file itself is not renamed",
+ "and will be overwritten, unless its name does not match",
+ "the [TT]-cpo[tt] option.",
+ "[PAR]",
+ "With checkpointing the output is appended to previously written",
+ "output files, unless [TT]-noappend[tt] is used or none of the previous",
+ "output files are present (except for the checkpoint file).",
+ "The integrity of the files to be appended is verified using checksums",
+ "which are stored in the checkpoint file. This ensures that output can",
+ "not be mixed up or corrupted due to file appending. When only some",
+ "of the previous output files are present, a fatal error is generated",
+ "and no old output files are modified and no new output files are opened.",
+ "The result with appending will be the same as from a single run.",
+ "The contents will be binary identical, unless you use a different number",
+ "of nodes or dynamic load balancing or the FFT library uses optimizations",
+ "through timing.",
+ "[PAR]",
+ "With option [TT]-maxh[tt] a simulation is terminated and a checkpoint",
+ "file is written at the first neighbor search step where the run time",
+ "exceeds [TT]-maxh[tt]*0.99 hours.",
+ "[PAR]",
+ "When [TT]mdrun[tt] receives a TERM signal, it will set nsteps to the current",
+ "step plus one. When [TT]mdrun[tt] receives an INT signal (e.g. when ctrl+C is",
+ "pressed), it will stop after the next neighbor search step ",
+ "(with nstlist=0 at the next step).",
+ "In both cases all the usual output will be written to file.",
+ "When running with MPI, a signal to one of the [TT]mdrun[tt] processes",
+ "is sufficient, this signal should not be sent to mpirun or",
+ "the [TT]mdrun[tt] process that is the parent of the others.",
+ "[PAR]",
+ "When [TT]mdrun[tt] is started with MPI, it does not run niced by default."
+ };
+ t_commrec *cr;
+ t_filenm fnm[] = {
+ { efTPX, NULL, NULL, ffREAD },
+ { efTRN, "-o", NULL, ffWRITE },
+ { efXTC, "-x", NULL, ffOPTWR },
+ { efCPT, "-cpi", NULL, ffOPTRD },
+ { efCPT, "-cpo", NULL, ffOPTWR },
+ { efSTO, "-c", "confout", ffWRITE },
+ { efEDR, "-e", "ener", ffWRITE },
+ { efLOG, "-g", "md", ffWRITE },
+ { efXVG, "-dhdl", "dhdl", ffOPTWR },
+ { efXVG, "-field", "field", ffOPTWR },
+ { efXVG, "-table", "table", ffOPTRD },
+ { efXVG, "-tabletf", "tabletf", ffOPTRD },
+ { efXVG, "-tablep", "tablep", ffOPTRD },
+ { efXVG, "-tableb", "table", ffOPTRD },
+ { efTRX, "-rerun", "rerun", ffOPTRD },
+ { efXVG, "-tpi", "tpi", ffOPTWR },
+ { efXVG, "-tpid", "tpidist", ffOPTWR },
+ { efEDI, "-ei", "sam", ffOPTRD },
+ { efXVG, "-eo", "edsam", ffOPTWR },
+ { efXVG, "-devout", "deviatie", ffOPTWR },
+ { efXVG, "-runav", "runaver", ffOPTWR },
+ { efXVG, "-px", "pullx", ffOPTWR },
+ { efXVG, "-pf", "pullf", ffOPTWR },
+ { efXVG, "-ro", "rotation", ffOPTWR },
+ { efLOG, "-ra", "rotangles", ffOPTWR },
+ { efLOG, "-rs", "rotslabs", ffOPTWR },
+ { efLOG, "-rt", "rottorque", ffOPTWR },
+ { efMTX, "-mtx", "nm", ffOPTWR },
+ { efNDX, "-dn", "dipole", ffOPTWR },
+ { efRND, "-multidir", NULL, ffOPTRDMULT},
+ { efDAT, "-membed", "membed", ffOPTRD },
+ { efTOP, "-mp", "membed", ffOPTRD },
+ { efNDX, "-mn", "membed", ffOPTRD }
+ };
+#define NFILE asize(fnm)
+
+ /* Command line options ! */
+ gmx_bool bPartDec = FALSE;
+ gmx_bool bDDBondCheck = TRUE;
+ gmx_bool bDDBondComm = TRUE;
+ gmx_bool bTunePME = TRUE;
+ gmx_bool bTestVerlet = FALSE;
+ gmx_bool bVerbose = FALSE;
+ gmx_bool bCompact = TRUE;
+ gmx_bool bSepPot = FALSE;
+ gmx_bool bRerunVSite = FALSE;
+ gmx_bool bConfout = TRUE;
+ gmx_bool bReproducible = FALSE;
+
+ int npme = -1;
+ int nmultisim = 0;
+ int nstglobalcomm = -1;
+ int repl_ex_nst = 0;
+ int repl_ex_seed = -1;
+ int repl_ex_nex = 0;
+ int nstepout = 100;
+ int resetstep = -1;
+ gmx_large_int_t nsteps = -2; /* the value -2 means that the mdp option will be used */
+
+ rvec realddxyz = {0, 0, 0};
+ const char *ddno_opt[ddnoNR+1] =
+ { NULL, "interleave", "pp_pme", "cartesian", NULL };
+ const char *dddlb_opt[] =
+ { NULL, "auto", "no", "yes", NULL };
+ const char *thread_aff_opt[threadaffNR+1] =
+ { NULL, "auto", "on", "off", NULL };
+ const char *nbpu_opt[] =
+ { NULL, "auto", "cpu", "gpu", "gpu_cpu", NULL };
+ real rdd = 0.0, rconstr = 0.0, dlb_scale = 0.8, pforce = -1;
+ char *ddcsx = NULL, *ddcsy = NULL, *ddcsz = NULL;
+ real cpt_period = 15.0, max_hours = -1;
+ gmx_bool bAppendFiles = TRUE;
+ gmx_bool bKeepAndNumCPT = FALSE;
+ gmx_bool bResetCountersHalfWay = FALSE;
+ output_env_t oenv = NULL;
+ const char *deviceOptions = "";
+
- { "-gpu_id", FALSE, etSTR, {&hw_opt.gpu_id},
- "List of GPU id's to use" },
++ /* Non transparent initialization of a complex gmx_hw_opt_t struct.
++ * But unfortunately we are not allowed to call a function here,
++ * since declarations follow below.
++ */
++ gmx_hw_opt_t hw_opt = { 0, 0, 0, 0, threadaffSEL, 0, 0,
++ { NULL, FALSE, 0, NULL } };
+
+ t_pargs pa[] = {
+
+ { "-pd", FALSE, etBOOL, {&bPartDec},
+ "Use particle decompostion" },
+ { "-dd", FALSE, etRVEC, {&realddxyz},
+ "Domain decomposition grid, 0 is optimize" },
+ { "-ddorder", FALSE, etENUM, {ddno_opt},
+ "DD node order" },
+ { "-npme", FALSE, etINT, {&npme},
+ "Number of separate nodes to be used for PME, -1 is guess" },
+ { "-nt", FALSE, etINT, {&hw_opt.nthreads_tot},
+ "Total number of threads to start (0 is guess)" },
+ { "-ntmpi", FALSE, etINT, {&hw_opt.nthreads_tmpi},
+ "Number of thread-MPI threads to start (0 is guess)" },
+ { "-ntomp", FALSE, etINT, {&hw_opt.nthreads_omp},
+ "Number of OpenMP threads per MPI process/thread to start (0 is guess)" },
+ { "-ntomp_pme", FALSE, etINT, {&hw_opt.nthreads_omp_pme},
+ "Number of OpenMP threads per MPI process/thread to start (0 is -ntomp)" },
+ { "-pin", FALSE, etENUM, {thread_aff_opt},
+ "Fix threads (or processes) to specific cores" },
+ { "-pinoffset", FALSE, etINT, {&hw_opt.core_pinning_offset},
+ "The starting logical core number for pinning to cores; used to avoid pinning threads from different mdrun instances to the same core" },
+ { "-pinstride", FALSE, etINT, {&hw_opt.core_pinning_stride},
+ "Pinning distance in logical cores for threads, use 0 to minimize the number of threads per physical core" },
++ { "-gpu_id", FALSE, etSTR, {&hw_opt.gpu_opt.gpu_id},
++ "List of GPU device id-s to use, specifies the per-node PP rank to GPU mapping" },
+ { "-ddcheck", FALSE, etBOOL, {&bDDBondCheck},
+ "Check for all bonded interactions with DD" },
+ { "-ddbondcomm", FALSE, etBOOL, {&bDDBondComm},
+ "HIDDENUse special bonded atom communication when [TT]-rdd[tt] > cut-off" },
+ { "-rdd", FALSE, etREAL, {&rdd},
+ "The maximum distance for bonded interactions with DD (nm), 0 is determine from initial coordinates" },
+ { "-rcon", FALSE, etREAL, {&rconstr},
+ "Maximum distance for P-LINCS (nm), 0 is estimate" },
+ { "-dlb", FALSE, etENUM, {dddlb_opt},
+ "Dynamic load balancing (with DD)" },
+ { "-dds", FALSE, etREAL, {&dlb_scale},
+ "Minimum allowed dlb scaling of the DD cell size" },
+ { "-ddcsx", FALSE, etSTR, {&ddcsx},
+ "HIDDENThe DD cell sizes in x" },
+ { "-ddcsy", FALSE, etSTR, {&ddcsy},
+ "HIDDENThe DD cell sizes in y" },
+ { "-ddcsz", FALSE, etSTR, {&ddcsz},
+ "HIDDENThe DD cell sizes in z" },
+ { "-gcom", FALSE, etINT, {&nstglobalcomm},
+ "Global communication frequency" },
+ { "-nb", FALSE, etENUM, {&nbpu_opt},
+ "Calculate non-bonded interactions on" },
+ { "-tunepme", FALSE, etBOOL, {&bTunePME},
+ "Optimize PME load between PP/PME nodes or GPU/CPU" },
+ { "-testverlet", FALSE, etBOOL, {&bTestVerlet},
+ "Test the Verlet non-bonded scheme" },
+ { "-v", FALSE, etBOOL, {&bVerbose},
+ "Be loud and noisy" },
+ { "-compact", FALSE, etBOOL, {&bCompact},
+ "Write a compact log file" },
+ { "-seppot", FALSE, etBOOL, {&bSepPot},
+ "Write separate V and dVdl terms for each interaction type and node to the log file(s)" },
+ { "-pforce", FALSE, etREAL, {&pforce},
+ "Print all forces larger than this (kJ/mol nm)" },
+ { "-reprod", FALSE, etBOOL, {&bReproducible},
+ "Try to avoid optimizations that affect binary reproducibility" },
+ { "-cpt", FALSE, etREAL, {&cpt_period},
+ "Checkpoint interval (minutes)" },
+ { "-cpnum", FALSE, etBOOL, {&bKeepAndNumCPT},
+ "Keep and number checkpoint files" },
+ { "-append", FALSE, etBOOL, {&bAppendFiles},
+ "Append to previous output files when continuing from checkpoint instead of adding the simulation part number to all file names" },
+ { "-nsteps", FALSE, etGMX_LARGE_INT, {&nsteps},
+ "Run this number of steps, overrides .mdp file option" },
+ { "-maxh", FALSE, etREAL, {&max_hours},
+ "Terminate after 0.99 times this time (hours)" },
+ { "-multi", FALSE, etINT, {&nmultisim},
+ "Do multiple simulations in parallel" },
+ { "-replex", FALSE, etINT, {&repl_ex_nst},
+ "Attempt replica exchange periodically with this period (steps)" },
+ { "-nex", FALSE, etINT, {&repl_ex_nex},
+ "Number of random exchanges to carry out each exchange interval (N^3 is one suggestion). -nex zero or not specified gives neighbor replica exchange." },
+ { "-reseed", FALSE, etINT, {&repl_ex_seed},
+ "Seed for replica exchange, -1 is generate a seed" },
+ { "-rerunvsite", FALSE, etBOOL, {&bRerunVSite},
+ "HIDDENRecalculate virtual site coordinates with [TT]-rerun[tt]" },
+ { "-confout", FALSE, etBOOL, {&bConfout},
+ "HIDDENWrite the last configuration with [TT]-c[tt] and force checkpointing at the last step" },
+ { "-stepout", FALSE, etINT, {&nstepout},
+ "HIDDENFrequency of writing the remaining wall clock time for the run" },
+ { "-resetstep", FALSE, etINT, {&resetstep},
+ "HIDDENReset cycle counters after these many time steps" },
+ { "-resethway", FALSE, etBOOL, {&bResetCountersHalfWay},
+ "HIDDENReset the cycle counters after half the number of steps or halfway [TT]-maxh[tt]" }
+ };
+ unsigned long Flags, PCA_Flags;
+ ivec ddxyz;
+ int dd_node_order;
+ gmx_bool bAddPart;
+ FILE *fplog, *fpmulti;
+ int sim_part, sim_part_fn;
+ const char *part_suffix = ".part";
+ char suffix[STRLEN];
+ int rc;
+ char **multidir = NULL;
+
+
+ cr = init_commrec();
+
+ PCA_Flags = (PCA_CAN_SET_DEFFNM | (MASTER(cr) ? 0 : PCA_QUIET));
+
+ /* Comment this in to do fexist calls only on master
+ * works not with rerun or tables at the moment
+ * also comment out the version of init_forcerec in md.c
+ * with NULL instead of opt2fn
+ */
+ /*
+ if (!MASTER(cr))
+ {
+ PCA_Flags |= PCA_NOT_READ_NODE;
+ }
+ */
+
+ if (!parse_common_args(&argc, argv, PCA_Flags, NFILE, fnm, asize(pa), pa,
+ asize(desc), desc, 0, NULL, &oenv))
+ {
+ return 0;
+ }
+
+
+ /* we set these early because they might be used in init_multisystem()
+ Note that there is the potential for npme>nnodes until the number of
+ threads is set later on, if there's thread parallelization. That shouldn't
+ lead to problems. */
+ dd_node_order = nenum(ddno_opt);
+ cr->npmenodes = npme;
+
+ hw_opt.thread_affinity = nenum(thread_aff_opt);
+
+ /* now check the -multi and -multidir option */
+ if (opt2bSet("-multidir", NFILE, fnm))
+ {
+ if (nmultisim > 0)
+ {
+ gmx_fatal(FARGS, "mdrun -multi and -multidir options are mutually exclusive.");
+ }
+ nmultisim = opt2fns(&multidir, "-multidir", NFILE, fnm);
+ }
+
+
+ if (repl_ex_nst != 0 && nmultisim < 2)
+ {
+ gmx_fatal(FARGS, "Need at least two replicas for replica exchange (option -multi)");
+ }
+
+ if (repl_ex_nex < 0)
+ {
+ gmx_fatal(FARGS, "Replica exchange number of exchanges needs to be positive");
+ }
+
+ if (nmultisim > 1)
+ {
+#ifndef GMX_THREAD_MPI
+ gmx_bool bParFn = (multidir == NULL);
+ init_multisystem(cr, nmultisim, multidir, NFILE, fnm, bParFn);
+#else
+ gmx_fatal(FARGS, "mdrun -multi is not supported with the thread library.Please compile GROMACS with MPI support");
+#endif
+ }
+
+ bAddPart = !bAppendFiles;
+
+ /* Check if there is ANY checkpoint file available */
+ sim_part = 1;
+ sim_part_fn = sim_part;
+ if (opt2bSet("-cpi", NFILE, fnm))
+ {
+ if (bSepPot && bAppendFiles)
+ {
+ gmx_fatal(FARGS, "Output file appending is not supported with -seppot");
+ }
+
+ bAppendFiles =
+ read_checkpoint_simulation_part(opt2fn_master("-cpi", NFILE,
+ fnm, cr),
+ &sim_part_fn, NULL, cr,
+ bAppendFiles, NFILE, fnm,
+ part_suffix, &bAddPart);
+ if (sim_part_fn == 0 && MULTIMASTER(cr))
+ {
+ fprintf(stdout, "No previous checkpoint file present, assuming this is a new run.\n");
+ }
+ else
+ {
+ sim_part = sim_part_fn + 1;
+ }
+
+ if (MULTISIM(cr) && MASTER(cr))
+ {
+ if (MULTIMASTER(cr))
+ {
+ /* Log file is not yet available, so if there's a
+ * problem we can only write to stderr. */
+ fpmulti = stderr;
+ }
+ else
+ {
+ fpmulti = NULL;
+ }
+ check_multi_int(fpmulti, cr->ms, sim_part, "simulation part", TRUE);
+ }
+ }
+ else
+ {
+ bAppendFiles = FALSE;
+ }
+
+ if (!bAppendFiles)
+ {
+ sim_part_fn = sim_part;
+ }
+
+ if (bAddPart)
+ {
+ /* Rename all output files (except checkpoint files) */
+ /* create new part name first (zero-filled) */
+ sprintf(suffix, "%s%04d", part_suffix, sim_part_fn);
+
+ add_suffix_to_output_names(fnm, NFILE, suffix);
+ if (MULTIMASTER(cr))
+ {
+ fprintf(stdout, "Checkpoint file is from part %d, new output files will be suffixed '%s'.\n", sim_part-1, suffix);
+ }
+ }
+
+ Flags = opt2bSet("-rerun", NFILE, fnm) ? MD_RERUN : 0;
+ Flags = Flags | (bSepPot ? MD_SEPPOT : 0);
+ Flags = Flags | (bPartDec ? MD_PARTDEC : 0);
+ Flags = Flags | (bDDBondCheck ? MD_DDBONDCHECK : 0);
+ Flags = Flags | (bDDBondComm ? MD_DDBONDCOMM : 0);
+ Flags = Flags | (bTunePME ? MD_TUNEPME : 0);
+ Flags = Flags | (bTestVerlet ? MD_TESTVERLET : 0);
+ Flags = Flags | (bConfout ? MD_CONFOUT : 0);
+ Flags = Flags | (bRerunVSite ? MD_RERUN_VSITE : 0);
+ Flags = Flags | (bReproducible ? MD_REPRODUCIBLE : 0);
+ Flags = Flags | (bAppendFiles ? MD_APPENDFILES : 0);
+ Flags = Flags | (opt2parg_bSet("-append", asize(pa), pa) ? MD_APPENDFILESSET : 0);
+ Flags = Flags | (bKeepAndNumCPT ? MD_KEEPANDNUMCPT : 0);
+ Flags = Flags | (sim_part > 1 ? MD_STARTFROMCPT : 0);
+ Flags = Flags | (bResetCountersHalfWay ? MD_RESETCOUNTERSHALFWAY : 0);
+
+
+ /* We postpone opening the log file if we are appending, so we can
+ first truncate the old log file and append to the correct position
+ there instead. */
+ if ((MASTER(cr) || bSepPot) && !bAppendFiles)
+ {
+ gmx_log_open(ftp2fn(efLOG, NFILE, fnm), cr,
+ !bSepPot, Flags & MD_APPENDFILES, &fplog);
+ please_cite(fplog, "Hess2008b");
+ please_cite(fplog, "Spoel2005a");
+ please_cite(fplog, "Lindahl2001a");
+ please_cite(fplog, "Berendsen95a");
+ }
+ else if (!MASTER(cr) && bSepPot)
+ {
+ gmx_log_open(ftp2fn(efLOG, NFILE, fnm), cr, !bSepPot, Flags, &fplog);
+ }
+ else
+ {
+ fplog = NULL;
+ }
+
+ ddxyz[XX] = (int)(realddxyz[XX] + 0.5);
+ ddxyz[YY] = (int)(realddxyz[YY] + 0.5);
+ ddxyz[ZZ] = (int)(realddxyz[ZZ] + 0.5);
+
+ rc = mdrunner(&hw_opt, fplog, cr, NFILE, fnm, oenv, bVerbose, bCompact,
+ nstglobalcomm, ddxyz, dd_node_order, rdd, rconstr,
+ dddlb_opt[0], dlb_scale, ddcsx, ddcsy, ddcsz,
+ nbpu_opt[0],
+ nsteps, nstepout, resetstep,
+ nmultisim, repl_ex_nst, repl_ex_nex, repl_ex_seed,
+ pforce, cpt_period, max_hours, deviceOptions, Flags);
+
+ /* Log file has to be closed in mdrunner if we are appending to it
+ (fplog not set here) */
+ if (MASTER(cr) && !bAppendFiles)
+ {
+ gmx_log_close(fplog);
+ }
+
+ return rc;
+}
--- /dev/null
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * VERSION 4.6.0
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2011, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * Gallium Rubidium Oxygen Manganese Argon Carbon Silicon
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include "smalloc.h"
+#include "network.h"
+#include "calcgrid.h"
+#include "pme.h"
+#include "vec.h"
+#include "domdec.h"
+#include "nbnxn_cuda_data_mgmt.h"
+#include "force.h"
+#include "macros.h"
+#include "md_logging.h"
+#include "pme_loadbal.h"
+
+/* Parameters and setting for one PP-PME setup */
+typedef struct {
+ real rcut_coulomb; /* Coulomb cut-off */
+ real rlist; /* pair-list cut-off */
+ real rlistlong; /* LR pair-list cut-off */
+ int nstcalclr; /* frequency of evaluating long-range forces for group scheme */
+ real spacing; /* (largest) PME grid spacing */
+ ivec grid; /* the PME grid dimensions */
+ real grid_efficiency; /* ineffiency factor for non-uniform grids <= 1 */
+ real ewaldcoeff; /* the Ewald coefficient */
+ gmx_pme_t pmedata; /* the data structure used in the PME code */
+
+ int count; /* number of times this setup has been timed */
+ double cycles; /* the fastest time for this setup in cycles */
+} pme_setup_t;
+
+/* In the initial scan, step by grids that are at least a factor 0.8 coarser */
+#define PME_LB_GRID_SCALE_FAC 0.8
+/* In the initial scan, try to skip grids with uneven x/y/z spacing,
+ * checking if the "efficiency" is more than 5% worse than the previous grid.
+ */
+#define PME_LB_GRID_EFFICIENCY_REL_FAC 1.05
+/* Rerun up till 12% slower setups than the fastest up till now */
+#define PME_LB_SLOW_FAC 1.12
+/* If setups get more than 2% faster, do another round to avoid
+ * choosing a slower setup due to acceleration or fluctuations.
+ */
+#define PME_LB_ACCEL_TOL 1.02
+
+enum {
+ epmelblimNO, epmelblimBOX, epmelblimDD, epmelblimPMEGRID, epmelblimNR
+};
+
+const char *pmelblim_str[epmelblimNR] =
+{ "no", "box size", "domain decompostion", "PME grid restriction" };
+
+struct pme_load_balancing {
+ int nstage; /* the current maximum number of stages */
+
+ real cut_spacing; /* the minimum cutoff / PME grid spacing ratio */
+ real rcut_vdw; /* Vdw cutoff (does not change) */
+ real rcut_coulomb_start; /* Initial electrostatics cutoff */
+ int nstcalclr_start; /* Initial electrostatics cutoff */
+ real rbuf_coulomb; /* the pairlist buffer size */
+ real rbuf_vdw; /* the pairlist buffer size */
+ matrix box_start; /* the initial simulation box */
+ int n; /* the count of setup as well as the allocation size */
+ pme_setup_t *setup; /* the PME+cutoff setups */
+ int cur; /* the current setup */
+ int fastest; /* fastest setup up till now */
+ int start; /* start of setup range to consider in stage>0 */
+ int end; /* end of setup range to consider in stage>0 */
+ int elimited; /* was the balancing limited, uses enum above */
+ int cutoff_scheme; /* Verlet or group cut-offs */
+
+ int stage; /* the current stage */
+};
+
+void pme_loadbal_init(pme_load_balancing_t *pme_lb_p,
+ const t_inputrec *ir, matrix box,
+ const interaction_const_t *ic,
+ gmx_pme_t pmedata)
+{
+ pme_load_balancing_t pme_lb;
+ real spm, sp;
+ int d;
+
+ snew(pme_lb, 1);
+
+ /* Any number of stages >= 2 is supported */
+ pme_lb->nstage = 2;
+
+ pme_lb->cutoff_scheme = ir->cutoff_scheme;
+
+ if (pme_lb->cutoff_scheme == ecutsVERLET)
+ {
+ pme_lb->rbuf_coulomb = ic->rlist - ic->rcoulomb;
+ pme_lb->rbuf_vdw = pme_lb->rbuf_coulomb;
+ }
+ else
+ {
+ if (ic->rcoulomb > ic->rlist)
+ {
+ pme_lb->rbuf_coulomb = ic->rlistlong - ic->rcoulomb;
+ }
+ else
+ {
+ pme_lb->rbuf_coulomb = ic->rlist - ic->rcoulomb;
+ }
+ if (ic->rvdw > ic->rlist)
+ {
+ pme_lb->rbuf_vdw = ic->rlistlong - ic->rvdw;
+ }
+ else
+ {
+ pme_lb->rbuf_vdw = ic->rlist - ic->rvdw;
+ }
+ }
+
+ copy_mat(box, pme_lb->box_start);
+ if (ir->ePBC == epbcXY && ir->nwall == 2)
+ {
+ svmul(ir->wall_ewald_zfac, pme_lb->box_start[ZZ], pme_lb->box_start[ZZ]);
+ }
+
+ pme_lb->n = 1;
+ snew(pme_lb->setup, pme_lb->n);
+
+ pme_lb->rcut_vdw = ic->rvdw;
+ pme_lb->rcut_coulomb_start = ir->rcoulomb;
+ pme_lb->nstcalclr_start = ir->nstcalclr;
+
+ pme_lb->cur = 0;
+ pme_lb->setup[0].rcut_coulomb = ic->rcoulomb;
+ pme_lb->setup[0].rlist = ic->rlist;
+ pme_lb->setup[0].rlistlong = ic->rlistlong;
+ pme_lb->setup[0].nstcalclr = ir->nstcalclr;
+ pme_lb->setup[0].grid[XX] = ir->nkx;
+ pme_lb->setup[0].grid[YY] = ir->nky;
+ pme_lb->setup[0].grid[ZZ] = ir->nkz;
+ pme_lb->setup[0].ewaldcoeff = ic->ewaldcoeff;
+
+ pme_lb->setup[0].pmedata = pmedata;
+
+ spm = 0;
+ for (d = 0; d < DIM; d++)
+ {
+ sp = norm(pme_lb->box_start[d])/pme_lb->setup[0].grid[d];
+ if (sp > spm)
+ {
+ spm = sp;
+ }
+ }
+ pme_lb->setup[0].spacing = spm;
+
+ if (ir->fourier_spacing > 0)
+ {
+ pme_lb->cut_spacing = ir->rcoulomb/ir->fourier_spacing;
+ }
+ else
+ {
+ pme_lb->cut_spacing = ir->rcoulomb/pme_lb->setup[0].spacing;
+ }
+
+ pme_lb->stage = 0;
+
+ pme_lb->fastest = 0;
+ pme_lb->start = 0;
+ pme_lb->end = 0;
+ pme_lb->elimited = epmelblimNO;
+
+ *pme_lb_p = pme_lb;
+}
+
+static gmx_bool pme_loadbal_increase_cutoff(pme_load_balancing_t pme_lb,
+ int pme_order,
+ const gmx_domdec_t *dd)
+{
+ pme_setup_t *set;
+ int npmenodes_x, npmenodes_y;
+ real fac, sp;
+ real tmpr_coulomb, tmpr_vdw;
+ int d;
+ gmx_bool grid_ok;
+
+ /* Try to add a new setup with next larger cut-off to the list */
+ pme_lb->n++;
+ srenew(pme_lb->setup, pme_lb->n);
+ set = &pme_lb->setup[pme_lb->n-1];
+ set->pmedata = NULL;
+
+ get_pme_nnodes(dd, &npmenodes_x, &npmenodes_y);
+
+ fac = 1;
+ do
+ {
+ /* Avoid infinite while loop, which can occur at the minimum grid size.
+ * Note that in practice load balancing will stop before this point.
+ * The factor 2.1 allows for the extreme case in which only grids
+ * of powers of 2 are allowed (the current code supports more grids).
+ */
+ if (fac > 2.1)
+ {
+ pme_lb->n--;
+
+ return FALSE;
+ }
+
+ fac *= 1.01;
+ clear_ivec(set->grid);
+ sp = calc_grid(NULL, pme_lb->box_start,
+ fac*pme_lb->setup[pme_lb->cur].spacing,
+ &set->grid[XX],
+ &set->grid[YY],
+ &set->grid[ZZ]);
+
+ /* As here we can't easily check if one of the PME nodes
+ * uses threading, we do a conservative grid check.
+ * This means we can't use pme_order or less grid lines
+ * per PME node along x, which is not a strong restriction.
+ */
+ gmx_pme_check_restrictions(pme_order,
+ set->grid[XX], set->grid[YY], set->grid[ZZ],
+ npmenodes_x, npmenodes_y,
+ TRUE,
+ FALSE,
+ &grid_ok);
+ }
+ while (sp <= 1.001*pme_lb->setup[pme_lb->cur].spacing || !grid_ok);
+
+ set->rcut_coulomb = pme_lb->cut_spacing*sp;
+
+ if (pme_lb->cutoff_scheme == ecutsVERLET)
+ {
+ set->rlist = set->rcut_coulomb + pme_lb->rbuf_coulomb;
+ /* We dont use LR lists with Verlet, but this avoids if-statements in further checks */
+ set->rlistlong = set->rlist;
+ }
+ else
+ {
+ tmpr_coulomb = set->rcut_coulomb + pme_lb->rbuf_coulomb;
+ tmpr_vdw = pme_lb->rcut_vdw + pme_lb->rbuf_vdw;
+ set->rlist = min(tmpr_coulomb, tmpr_vdw);
+ set->rlistlong = max(tmpr_coulomb, tmpr_vdw);
+
+ /* Set the long-range update frequency */
+ if (set->rlist == set->rlistlong)
+ {
+ /* No long-range interactions if the short-/long-range cutoffs are identical */
+ set->nstcalclr = 0;
+ }
+ else if (pme_lb->nstcalclr_start == 0 || pme_lb->nstcalclr_start == 1)
+ {
+ /* We were not doing long-range before, but now we are since rlist!=rlistlong */
+ set->nstcalclr = 1;
+ }
+ else
+ {
+ /* We were already doing long-range interactions from the start */
+ if (pme_lb->rcut_vdw > pme_lb->rcut_coulomb_start)
+ {
+ /* We were originally doing long-range VdW-only interactions.
+ * If rvdw is still longer than rcoulomb we keep the original nstcalclr,
+ * but if the coulomb cutoff has become longer we should update the long-range
+ * part every step.
+ */
+ set->nstcalclr = (tmpr_vdw > tmpr_coulomb) ? pme_lb->nstcalclr_start : 1;
+ }
+ else
+ {
+ /* We were not doing any long-range interaction from the start,
+ * since it is not possible to do twin-range coulomb for the PME interaction.
+ */
+ set->nstcalclr = 1;
+ }
+ }
+ }
+
+ set->spacing = sp;
+ /* The grid efficiency is the size wrt a grid with uniform x/y/z spacing */
+ set->grid_efficiency = 1;
+ for (d = 0; d < DIM; d++)
+ {
+ set->grid_efficiency *= (set->grid[d]*sp)/norm(pme_lb->box_start[d]);
+ }
+ /* The Ewald coefficient is inversly proportional to the cut-off */
+ set->ewaldcoeff =
+ pme_lb->setup[0].ewaldcoeff*pme_lb->setup[0].rcut_coulomb/set->rcut_coulomb;
+
+ set->count = 0;
+ set->cycles = 0;
+
+ if (debug)
+ {
+ fprintf(debug, "PME loadbal: grid %d %d %d, coulomb cutoff %f\n",
+ set->grid[XX], set->grid[YY], set->grid[ZZ], set->rcut_coulomb);
+ }
+ return TRUE;
+}
+
+static void print_grid(FILE *fp_err, FILE *fp_log,
+ const char *pre,
+ const char *desc,
+ const pme_setup_t *set,
+ double cycles)
+{
+ char buf[STRLEN], buft[STRLEN];
+
+ if (cycles >= 0)
+ {
+ sprintf(buft, ": %.1f M-cycles", cycles*1e-6);
+ }
+ else
+ {
+ buft[0] = '\0';
+ }
+ sprintf(buf, "%-11s%10s pme grid %d %d %d, coulomb cutoff %.3f%s",
+ pre,
+ desc, set->grid[XX], set->grid[YY], set->grid[ZZ], set->rcut_coulomb,
+ buft);
+ if (fp_err != NULL)
+ {
+ fprintf(fp_err, "\r%s\n", buf);
+ }
+ if (fp_log != NULL)
+ {
+ fprintf(fp_log, "%s\n", buf);
+ }
+}
+
+static int pme_loadbal_end(pme_load_balancing_t pme_lb)
+{
+ /* In the initial stage only n is set; end is not set yet */
+ if (pme_lb->end > 0)
+ {
+ return pme_lb->end;
+ }
+ else
+ {
+ return pme_lb->n;
+ }
+}
+
+static void print_loadbal_limited(FILE *fp_err, FILE *fp_log,
+ gmx_large_int_t step,
+ pme_load_balancing_t pme_lb)
+{
+ char buf[STRLEN], sbuf[22];
+
+ sprintf(buf, "step %4s: the %s limits the PME load balancing to a coulomb cut-off of %.3f",
+ gmx_step_str(step, sbuf),
+ pmelblim_str[pme_lb->elimited],
+ pme_lb->setup[pme_loadbal_end(pme_lb)-1].rcut_coulomb);
+ if (fp_err != NULL)
+ {
+ fprintf(fp_err, "\r%s\n", buf);
+ }
+ if (fp_log != NULL)
+ {
+ fprintf(fp_log, "%s\n", buf);
+ }
+}
+
+static void switch_to_stage1(pme_load_balancing_t pme_lb)
+{
+ pme_lb->start = 0;
+ while (pme_lb->start+1 < pme_lb->n &&
+ (pme_lb->setup[pme_lb->start].count == 0 ||
+ pme_lb->setup[pme_lb->start].cycles >
+ pme_lb->setup[pme_lb->fastest].cycles*PME_LB_SLOW_FAC))
+ {
+ pme_lb->start++;
+ }
+ while (pme_lb->start > 0 && pme_lb->setup[pme_lb->start-1].cycles == 0)
+ {
+ pme_lb->start--;
+ }
+
+ pme_lb->end = pme_lb->n;
+ if (pme_lb->setup[pme_lb->end-1].count > 0 &&
+ pme_lb->setup[pme_lb->end-1].cycles >
+ pme_lb->setup[pme_lb->fastest].cycles*PME_LB_SLOW_FAC)
+ {
+ pme_lb->end--;
+ }
+
+ pme_lb->stage = 1;
+
+ /* Next we want to choose setup pme_lb->start, but as we will increase
+ * pme_ln->cur by one right after returning, we subtract 1 here.
+ */
+ pme_lb->cur = pme_lb->start - 1;
+}
+
+gmx_bool pme_load_balance(pme_load_balancing_t pme_lb,
+ t_commrec *cr,
+ FILE *fp_err,
+ FILE *fp_log,
+ t_inputrec *ir,
+ t_state *state,
+ double cycles,
+ interaction_const_t *ic,
+ nonbonded_verlet_t *nbv,
+ gmx_pme_t *pmedata,
+ gmx_large_int_t step)
+{
+ gmx_bool OK;
+ pme_setup_t *set;
+ double cycles_fast;
+ char buf[STRLEN], sbuf[22];
+ real rtab;
+ gmx_bool bUsesSimpleTables = TRUE;
+
+ if (pme_lb->stage == pme_lb->nstage)
+ {
+ return FALSE;
+ }
+
+ if (PAR(cr))
+ {
+ gmx_sumd(1, &cycles, cr);
+ cycles /= cr->nnodes;
+ }
+
+ set = &pme_lb->setup[pme_lb->cur];
+ set->count++;
+
+ rtab = ir->rlistlong + ir->tabext;
+
+ if (set->count % 2 == 1)
+ {
+ /* Skip the first cycle, because the first step after a switch
+ * is much slower due to allocation and/or caching effects.
+ */
+ return TRUE;
+ }
+
+ sprintf(buf, "step %4s: ", gmx_step_str(step, sbuf));
+ print_grid(fp_err, fp_log, buf, "timed with", set, cycles);
+
+ if (set->count <= 2)
+ {
+ set->cycles = cycles;
+ }
+ else
+ {
+ if (cycles*PME_LB_ACCEL_TOL < set->cycles &&
+ pme_lb->stage == pme_lb->nstage - 1)
+ {
+ /* The performance went up a lot (due to e.g. DD load balancing).
+ * Add a stage, keep the minima, but rescan all setups.
+ */
+ pme_lb->nstage++;
+
+ if (debug)
+ {
+ fprintf(debug, "The performance for grid %d %d %d went from %.3f to %.1f M-cycles, this is more than %f\n"
+ "Increased the number stages to %d"
+ " and ignoring the previous performance\n",
+ set->grid[XX], set->grid[YY], set->grid[ZZ],
+ cycles*1e-6, set->cycles*1e-6, PME_LB_ACCEL_TOL,
+ pme_lb->nstage);
+ }
+ }
+ set->cycles = min(set->cycles, cycles);
+ }
+
+ if (set->cycles < pme_lb->setup[pme_lb->fastest].cycles)
+ {
+ pme_lb->fastest = pme_lb->cur;
+
+ if (DOMAINDECOMP(cr))
+ {
+ /* We found a new fastest setting, ensure that with subsequent
+ * shorter cut-off's the dynamic load balancing does not make
+ * the use of the current cut-off impossible. This solution is
+ * a trade-off, as the PME load balancing and DD domain size
+ * load balancing can interact in complex ways.
+ * With the Verlet kernels, DD load imbalance will usually be
+ * mainly due to bonded interaction imbalance, which will often
+ * quickly push the domain boundaries beyond the limit for the
+ * optimal, PME load balanced, cut-off. But it could be that
+ * better overal performance can be obtained with a slightly
+ * shorter cut-off and better DD load balancing.
+ */
+ change_dd_dlb_cutoff_limit(cr);
+ }
+ }
+ cycles_fast = pme_lb->setup[pme_lb->fastest].cycles;
+
+ /* Check in stage 0 if we should stop scanning grids.
+ * Stop when the time is more than SLOW_FAC longer than the fastest.
+ */
+ if (pme_lb->stage == 0 && pme_lb->cur > 0 &&
+ cycles > pme_lb->setup[pme_lb->fastest].cycles*PME_LB_SLOW_FAC)
+ {
+ pme_lb->n = pme_lb->cur + 1;
+ /* Done with scanning, go to stage 1 */
+ switch_to_stage1(pme_lb);
+ }
+
+ if (pme_lb->stage == 0)
+ {
+ int gridsize_start;
+
+ gridsize_start = set->grid[XX]*set->grid[YY]*set->grid[ZZ];
+
+ do
+ {
+ if (pme_lb->cur+1 < pme_lb->n)
+ {
+ /* We had already generated the next setup */
+ OK = TRUE;
+ }
+ else
+ {
+ /* Find the next setup */
+ OK = pme_loadbal_increase_cutoff(pme_lb, ir->pme_order, cr->dd);
+
+ if (!OK)
+ {
+ pme_lb->elimited = epmelblimPMEGRID;
+ }
+ }
+
+ if (OK && ir->ePBC != epbcNONE)
+ {
+ OK = (sqr(pme_lb->setup[pme_lb->cur+1].rlistlong)
+ <= max_cutoff2(ir->ePBC, state->box));
+ if (!OK)
+ {
+ pme_lb->elimited = epmelblimBOX;
+ }
+ }
+
+ if (OK)
+ {
+ pme_lb->cur++;
+
+ if (DOMAINDECOMP(cr))
+ {
+ OK = change_dd_cutoff(cr, state, ir,
+ pme_lb->setup[pme_lb->cur].rlistlong);
+ if (!OK)
+ {
+ /* Failed: do not use this setup */
+ pme_lb->cur--;
+ pme_lb->elimited = epmelblimDD;
+ }
+ }
+ }
+ if (!OK)
+ {
+ /* We hit the upper limit for the cut-off,
+ * the setup should not go further than cur.
+ */
+ pme_lb->n = pme_lb->cur + 1;
+ print_loadbal_limited(fp_err, fp_log, step, pme_lb);
+ /* Switch to the next stage */
+ switch_to_stage1(pme_lb);
+ }
+ }
+ while (OK &&
+ !(pme_lb->setup[pme_lb->cur].grid[XX]*
+ pme_lb->setup[pme_lb->cur].grid[YY]*
+ pme_lb->setup[pme_lb->cur].grid[ZZ] <
+ gridsize_start*PME_LB_GRID_SCALE_FAC
+ &&
+ pme_lb->setup[pme_lb->cur].grid_efficiency <
+ pme_lb->setup[pme_lb->cur-1].grid_efficiency*PME_LB_GRID_EFFICIENCY_REL_FAC));
+ }
+
+ if (pme_lb->stage > 0 && pme_lb->end == 1)
+ {
+ pme_lb->cur = 0;
+ pme_lb->stage = pme_lb->nstage;
+ }
+ else if (pme_lb->stage > 0 && pme_lb->end > 1)
+ {
+ /* If stage = nstage-1:
+ * scan over all setups, rerunning only those setups
+ * which are not much slower than the fastest
+ * else:
+ * use the next setup
+ */
+ do
+ {
+ pme_lb->cur++;
+ if (pme_lb->cur == pme_lb->end)
+ {
+ pme_lb->stage++;
+ pme_lb->cur = pme_lb->start;
+ }
+ }
+ while (pme_lb->stage == pme_lb->nstage - 1 &&
+ pme_lb->setup[pme_lb->cur].count > 0 &&
+ pme_lb->setup[pme_lb->cur].cycles > cycles_fast*PME_LB_SLOW_FAC);
+
+ if (pme_lb->stage == pme_lb->nstage)
+ {
+ /* We are done optimizing, use the fastest setup we found */
+ pme_lb->cur = pme_lb->fastest;
+ }
+ }
+
+ if (DOMAINDECOMP(cr) && pme_lb->stage > 0)
+ {
+ OK = change_dd_cutoff(cr, state, ir, pme_lb->setup[pme_lb->cur].rlistlong);
+ if (!OK)
+ {
+ /* Failsafe solution */
+ if (pme_lb->cur > 1 && pme_lb->stage == pme_lb->nstage)
+ {
+ pme_lb->stage--;
+ }
+ pme_lb->fastest = 0;
+ pme_lb->start = 0;
+ pme_lb->end = pme_lb->cur;
+ pme_lb->cur = pme_lb->start;
+ pme_lb->elimited = epmelblimDD;
+ print_loadbal_limited(fp_err, fp_log, step, pme_lb);
+ }
+ }
+
+ /* Change the Coulomb cut-off and the PME grid */
+
+ set = &pme_lb->setup[pme_lb->cur];
+
+ ic->rcoulomb = set->rcut_coulomb;
+ ic->rlist = set->rlist;
+ ic->rlistlong = set->rlistlong;
+ ir->nstcalclr = set->nstcalclr;
+ ic->ewaldcoeff = set->ewaldcoeff;
+
+ bUsesSimpleTables = uses_simple_tables(ir->cutoff_scheme, nbv, 0);
+ if (pme_lb->cutoff_scheme == ecutsVERLET &&
+ nbv->grp[0].kernel_type == nbnxnk8x8x8_CUDA)
+ {
+ nbnxn_cuda_pme_loadbal_update_param(nbv->cu_nbv, ic);
++
++ /* With tMPI + GPUs some ranks may be sharing GPU(s) and therefore
++ * also sharing texture references. To keep the code simple, we don't
++ * treat texture references as shared resources, but this means that
++ * the coulomb_tab texture ref will get updated by multiple threads.
++ * Hence, to ensure that the non-bonded kernels don't start before all
++ * texture binding operations are finished, we need to wait for all ranks
++ * to arrive here before continuing.
++ *
++ * Note that we could omit this barrier if GPUs are not shared (or
++ * texture objects are used), but as this is initialization code, there
++ * is not point in complicating things.
++ */
++#ifdef GMX_THREAD_MPI
++ if (PAR(cr))
++ {
++ gmx_barrier(cr);
++ }
++#endif /* GMX_THREAD_MPI */
+ }
+ else
+ {
+ init_interaction_const_tables(NULL, ic, bUsesSimpleTables,
+ rtab);
+ }
+
+ if (pme_lb->cutoff_scheme == ecutsVERLET && nbv->ngrp > 1)
+ {
+ init_interaction_const_tables(NULL, ic, bUsesSimpleTables,
+ rtab);
+ }
+
+ if (cr->duty & DUTY_PME)
+ {
+ if (pme_lb->setup[pme_lb->cur].pmedata == NULL)
+ {
+ /* Generate a new PME data structure,
+ * copying part of the old pointers.
+ */
+ gmx_pme_reinit(&set->pmedata,
+ cr, pme_lb->setup[0].pmedata, ir,
+ set->grid);
+ }
+ *pmedata = set->pmedata;
+ }
+ else
+ {
+ /* Tell our PME-only node to switch grid */
+ gmx_pme_send_switchgrid(cr, set->grid, set->ewaldcoeff);
+ }
+
+ if (debug)
+ {
+ print_grid(NULL, debug, "", "switched to", set, -1);
+ }
+
+ if (pme_lb->stage == pme_lb->nstage)
+ {
+ print_grid(fp_err, fp_log, "", "optimal", set, -1);
+ }
+
+ return TRUE;
+}
+
+void restart_pme_loadbal(pme_load_balancing_t pme_lb, int n)
+{
+ pme_lb->nstage += n;
+}
+
+static int pme_grid_points(const pme_setup_t *setup)
+{
+ return setup->grid[XX]*setup->grid[YY]*setup->grid[ZZ];
+}
+
+static real pme_loadbal_rlist(const pme_setup_t *setup)
+{
+ /* With the group cut-off scheme we can have twin-range either
+ * for Coulomb or for VdW, so we need a check here.
+ * With the Verlet cut-off scheme rlist=rlistlong.
+ */
+ if (setup->rcut_coulomb > setup->rlist)
+ {
+ return setup->rlistlong;
+ }
+ else
+ {
+ return setup->rlist;
+ }
+}
+
+static void print_pme_loadbal_setting(FILE *fplog,
+ char *name,
+ const pme_setup_t *setup)
+{
+ fprintf(fplog,
+ " %-7s %6.3f nm %6.3f nm %3d %3d %3d %5.3f nm %5.3f nm\n",
+ name,
+ setup->rcut_coulomb, pme_loadbal_rlist(setup),
+ setup->grid[XX], setup->grid[YY], setup->grid[ZZ],
+ setup->spacing, 1/setup->ewaldcoeff);
+}
+
+static void print_pme_loadbal_settings(pme_load_balancing_t pme_lb,
+ t_commrec *cr,
+ FILE *fplog,
+ gmx_bool bNonBondedOnGPU)
+{
+ double pp_ratio, grid_ratio;
+
+ pp_ratio = pow(pme_loadbal_rlist(&pme_lb->setup[pme_lb->cur])/pme_loadbal_rlist(&pme_lb->setup[0]), 3.0);
+ grid_ratio = pme_grid_points(&pme_lb->setup[pme_lb->cur])/
+ (double)pme_grid_points(&pme_lb->setup[0]);
+
+ fprintf(fplog, "\n");
+ fprintf(fplog, " P P - P M E L O A D B A L A N C I N G\n");
+ fprintf(fplog, "\n");
+ /* Here we only warn when the optimal setting is the last one */
+ if (pme_lb->elimited != epmelblimNO &&
+ pme_lb->cur == pme_loadbal_end(pme_lb)-1)
+ {
+ fprintf(fplog, " NOTE: The PP/PME load balancing was limited by the %s,\n",
+ pmelblim_str[pme_lb->elimited]);
+ fprintf(fplog, " you might not have reached a good load balance.\n");
+ if (pme_lb->elimited == epmelblimDD)
+ {
+ fprintf(fplog, " Try different mdrun -dd settings or lower the -dds value.\n");
+ }
+ fprintf(fplog, "\n");
+ }
+ fprintf(fplog, " PP/PME load balancing changed the cut-off and PME settings:\n");
+ fprintf(fplog, " particle-particle PME\n");
+ fprintf(fplog, " rcoulomb rlist grid spacing 1/beta\n");
+ print_pme_loadbal_setting(fplog, "initial", &pme_lb->setup[0]);
+ print_pme_loadbal_setting(fplog, "final", &pme_lb->setup[pme_lb->cur]);
+ fprintf(fplog, " cost-ratio %4.2f %4.2f\n",
+ pp_ratio, grid_ratio);
+ fprintf(fplog, " (note that these numbers concern only part of the total PP and PME load)\n");
+
+ if (pp_ratio > 1.5 && !bNonBondedOnGPU)
+ {
+ md_print_warn(cr, fplog,
+ "NOTE: PME load balancing increased the non-bonded workload by more than 50%%.\n"
+ " For better performance use (more) PME nodes (mdrun -npme),\n"
+ " or in case you are beyond the scaling limit, use less nodes in total.\n");
+ }
+ else
+ {
+ fprintf(fplog, "\n");
+ }
+}
+
+void pme_loadbal_done(pme_load_balancing_t pme_lb,
+ t_commrec *cr, FILE *fplog,
+ gmx_bool bNonBondedOnGPU)
+{
+ if (fplog != NULL && (pme_lb->cur > 0 || pme_lb->elimited != epmelblimNO))
+ {
+ print_pme_loadbal_settings(pme_lb, cr, fplog, bNonBondedOnGPU);
+ }
+
+ /* TODO: Here we should free all pointers in pme_lb,
+ * but as it contains pme data structures,
+ * we need to first make pme.c free all data.
+ */
+}
--- /dev/null
- gmx_hw_opt_t *hw_opt;
+/* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
+ *
+ *
+ * This source code is part of
+ *
+ * G R O M A C S
+ *
+ * GROningen MAchine for Chemical Simulations
+ *
+ * VERSION 3.2.0
+ * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
+ * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
+ * Copyright (c) 2001-2004, The GROMACS development team,
+ * check out http://www.gromacs.org for more information.
+
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * If you want to redistribute modifications, please consider that
+ * scientific software is very special. Version control is crucial -
+ * bugs must be traceable. We will be happy to consider code for
+ * inclusion in the official distribution, but derived work must not
+ * be called official GROMACS. Details are found in the README & COPYING
+ * files - if they are missing, get the official version at www.gromacs.org.
+ *
+ * To help us fund GROMACS development, we humbly ask that you cite
+ * the papers on the package - you can find them in the top README file.
+ *
+ * For more info, check our website at http://www.gromacs.org
+ *
+ * And Hey:
+ * Gallium Rubidium Oxygen Manganese Argon Carbon Silicon
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+#include <signal.h>
+#include <stdlib.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#include <string.h>
+#include <assert.h>
+
+#include "typedefs.h"
+#include "smalloc.h"
+#include "sysstuff.h"
+#include "statutil.h"
+#include "force.h"
+#include "mdrun.h"
+#include "md_logging.h"
+#include "md_support.h"
+#include "network.h"
+#include "pull.h"
+#include "pull_rotation.h"
+#include "names.h"
+#include "disre.h"
+#include "orires.h"
+#include "pme.h"
+#include "mdatoms.h"
+#include "repl_ex.h"
+#include "qmmm.h"
+#include "domdec.h"
+#include "partdec.h"
+#include "coulomb.h"
+#include "constr.h"
+#include "mvdata.h"
+#include "checkpoint.h"
+#include "mtop_util.h"
+#include "sighandler.h"
+#include "txtdump.h"
+#include "gmx_detect_hardware.h"
+#include "gmx_omp_nthreads.h"
+#include "pull_rotation.h"
+#include "calc_verletbuf.h"
+#include "gmx_fatal_collective.h"
+#include "membed.h"
+#include "macros.h"
+#include "gmx_omp.h"
+#include "gmx_thread_affinity.h"
+
+#include "gromacs/fileio/tpxio.h"
+#include "gromacs/mdlib/nbnxn_search.h"
+#include "gromacs/mdlib/nbnxn_consts.h"
+#include "gromacs/timing/wallcycle.h"
+#include "gromacs/utility/gmxmpi.h"
+
+#ifdef GMX_FAHCORE
+#include "corewrap.h"
+#endif
+
+#include "gpu_utils.h"
+#include "nbnxn_cuda_data_mgmt.h"
+
+typedef struct {
+ gmx_integrator_t *func;
+} gmx_intp_t;
+
+/* The array should match the eI array in include/types/enums.h */
+const gmx_intp_t integrator[eiNR] = { {do_md}, {do_steep}, {do_cg}, {do_md}, {do_md}, {do_nm}, {do_lbfgs}, {do_tpi}, {do_tpi}, {do_md}, {do_md}, {do_md}};
+
+gmx_large_int_t deform_init_init_step_tpx;
+matrix deform_init_box_tpx;
+#ifdef GMX_THREAD_MPI
+tMPI_Thread_mutex_t deform_init_box_mutex = TMPI_THREAD_MUTEX_INITIALIZER;
+#endif
+
+
+#ifdef GMX_THREAD_MPI
+struct mdrunner_arglist
+{
- mda->ret = mdrunner(mc.hw_opt, fplog, cr, mc.nfile, fnm, mc.oenv,
++ gmx_hw_opt_t hw_opt;
+ FILE *fplog;
+ t_commrec *cr;
+ int nfile;
+ const t_filenm *fnm;
+ output_env_t oenv;
+ gmx_bool bVerbose;
+ gmx_bool bCompact;
+ int nstglobalcomm;
+ ivec ddxyz;
+ int dd_node_order;
+ real rdd;
+ real rconstr;
+ const char *dddlb_opt;
+ real dlb_scale;
+ const char *ddcsx;
+ const char *ddcsy;
+ const char *ddcsz;
+ const char *nbpu_opt;
+ gmx_large_int_t nsteps_cmdline;
+ int nstepout;
+ int resetstep;
+ int nmultisim;
+ int repl_ex_nst;
+ int repl_ex_nex;
+ int repl_ex_seed;
+ real pforce;
+ real cpt_period;
+ real max_hours;
+ const char *deviceOptions;
+ unsigned long Flags;
+ int ret; /* return value */
+};
+
+
+/* The function used for spawning threads. Extracts the mdrunner()
+ arguments from its one argument and calls mdrunner(), after making
+ a commrec. */
+static void mdrunner_start_fn(void *arg)
+{
+ struct mdrunner_arglist *mda = (struct mdrunner_arglist*)arg;
+ struct mdrunner_arglist mc = *mda; /* copy the arg list to make sure
+ that it's thread-local. This doesn't
+ copy pointed-to items, of course,
+ but those are all const. */
+ t_commrec *cr; /* we need a local version of this */
+ FILE *fplog = NULL;
+ t_filenm *fnm;
+
+ fnm = dup_tfn(mc.nfile, mc.fnm);
+
+ cr = reinitialize_commrec_for_this_thread(mc.cr);
+
+ if (MASTER(cr))
+ {
+ fplog = mc.fplog;
+ }
+
- mda->hw_opt = hw_opt;
++ mda->ret = mdrunner(&mc.hw_opt, fplog, cr, mc.nfile, fnm, mc.oenv,
+ mc.bVerbose, mc.bCompact, mc.nstglobalcomm,
+ mc.ddxyz, mc.dd_node_order, mc.rdd,
+ mc.rconstr, mc.dddlb_opt, mc.dlb_scale,
+ mc.ddcsx, mc.ddcsy, mc.ddcsz,
+ mc.nbpu_opt,
+ mc.nsteps_cmdline, mc.nstepout, mc.resetstep,
+ mc.nmultisim, mc.repl_ex_nst, mc.repl_ex_nex, mc.repl_ex_seed, mc.pforce,
+ mc.cpt_period, mc.max_hours, mc.deviceOptions, mc.Flags);
+}
+
+/* called by mdrunner() to start a specific number of threads (including
+ the main thread) for thread-parallel runs. This in turn calls mdrunner()
+ for each thread.
+ All options besides nthreads are the same as for mdrunner(). */
+static t_commrec *mdrunner_start_threads(gmx_hw_opt_t *hw_opt,
+ FILE *fplog, t_commrec *cr, int nfile,
+ const t_filenm fnm[], const output_env_t oenv, gmx_bool bVerbose,
+ gmx_bool bCompact, int nstglobalcomm,
+ ivec ddxyz, int dd_node_order, real rdd, real rconstr,
+ const char *dddlb_opt, real dlb_scale,
+ const char *ddcsx, const char *ddcsy, const char *ddcsz,
+ const char *nbpu_opt,
+ gmx_large_int_t nsteps_cmdline,
+ int nstepout, int resetstep,
+ int nmultisim, int repl_ex_nst, int repl_ex_nex, int repl_ex_seed,
+ real pforce, real cpt_period, real max_hours,
+ const char *deviceOptions, unsigned long Flags)
+{
+ int ret;
+ struct mdrunner_arglist *mda;
+ t_commrec *crn; /* the new commrec */
+ t_filenm *fnmn;
+
+ /* first check whether we even need to start tMPI */
+ if (hw_opt->nthreads_tmpi < 2)
+ {
+ return cr;
+ }
+
+ /* a few small, one-time, almost unavoidable memory leaks: */
+ snew(mda, 1);
+ fnmn = dup_tfn(nfile, fnm);
+
+ /* fill the data structure to pass as void pointer to thread start fn */
- bCanUseGPU = (inputrec->cutoff_scheme == ecutsVERLET && hwinfo->bCanUseGPU);
++ /* hw_opt contains pointers, which should all be NULL at this stage */
++ mda->hw_opt = *hw_opt;
+ mda->fplog = fplog;
+ mda->cr = cr;
+ mda->nfile = nfile;
+ mda->fnm = fnmn;
+ mda->oenv = oenv;
+ mda->bVerbose = bVerbose;
+ mda->bCompact = bCompact;
+ mda->nstglobalcomm = nstglobalcomm;
+ mda->ddxyz[XX] = ddxyz[XX];
+ mda->ddxyz[YY] = ddxyz[YY];
+ mda->ddxyz[ZZ] = ddxyz[ZZ];
+ mda->dd_node_order = dd_node_order;
+ mda->rdd = rdd;
+ mda->rconstr = rconstr;
+ mda->dddlb_opt = dddlb_opt;
+ mda->dlb_scale = dlb_scale;
+ mda->ddcsx = ddcsx;
+ mda->ddcsy = ddcsy;
+ mda->ddcsz = ddcsz;
+ mda->nbpu_opt = nbpu_opt;
+ mda->nsteps_cmdline = nsteps_cmdline;
+ mda->nstepout = nstepout;
+ mda->resetstep = resetstep;
+ mda->nmultisim = nmultisim;
+ mda->repl_ex_nst = repl_ex_nst;
+ mda->repl_ex_nex = repl_ex_nex;
+ mda->repl_ex_seed = repl_ex_seed;
+ mda->pforce = pforce;
+ mda->cpt_period = cpt_period;
+ mda->max_hours = max_hours;
+ mda->deviceOptions = deviceOptions;
+ mda->Flags = Flags;
+
+ /* now spawn new threads that start mdrunner_start_fn(), while
+ the main thread returns, we set thread affinity later */
+ ret = tMPI_Init_fn(TRUE, hw_opt->nthreads_tmpi, TMPI_AFFINITY_NONE,
+ mdrunner_start_fn, (void*)(mda) );
+ if (ret != TMPI_SUCCESS)
+ {
+ return NULL;
+ }
+
+ crn = reinitialize_commrec_for_this_thread(cr);
+ return crn;
+}
+
+
+static int get_tmpi_omp_thread_division(const gmx_hw_info_t *hwinfo,
+ const gmx_hw_opt_t *hw_opt,
+ int nthreads_tot,
+ int ngpu)
+{
+ int nthreads_tmpi;
+
+ /* There are no separate PME nodes here, as we ensured in
+ * check_and_update_hw_opt that nthreads_tmpi>0 with PME nodes
+ * and a conditional ensures we would not have ended up here.
+ * Note that separate PME nodes might be switched on later.
+ */
+ if (ngpu > 0)
+ {
+ nthreads_tmpi = ngpu;
+ if (nthreads_tot > 0 && nthreads_tot < nthreads_tmpi)
+ {
+ nthreads_tmpi = nthreads_tot;
+ }
+ }
+ else if (hw_opt->nthreads_omp > 0)
+ {
+ /* Here we could oversubscribe, when we do, we issue a warning later */
+ nthreads_tmpi = max(1, nthreads_tot/hw_opt->nthreads_omp);
+ }
+ else
+ {
+ /* TODO choose nthreads_omp based on hardware topology
+ when we have a hardware topology detection library */
+ /* In general, when running up to 4 threads, OpenMP should be faster.
+ * Note: on AMD Bulldozer we should avoid running OpenMP over two dies.
+ * On Intel>=Nehalem running OpenMP on a single CPU is always faster,
+ * even on two CPUs it's usually faster (but with many OpenMP threads
+ * it could be faster not to use HT, currently we always use HT).
+ * On Nehalem/Westmere we want to avoid running 16 threads over
+ * two CPUs with HT, so we need a limit<16; thus we use 12.
+ * A reasonable limit for Intel Sandy and Ivy bridge,
+ * not knowing the topology, is 16 threads.
+ */
+ const int nthreads_omp_always_faster = 4;
+ const int nthreads_omp_always_faster_Nehalem = 12;
+ const int nthreads_omp_always_faster_SandyBridge = 16;
+ const int first_model_Nehalem = 0x1A;
+ const int first_model_SandyBridge = 0x2A;
+ gmx_bool bIntel_Family6;
+
+ bIntel_Family6 =
+ (gmx_cpuid_vendor(hwinfo->cpuid_info) == GMX_CPUID_VENDOR_INTEL &&
+ gmx_cpuid_family(hwinfo->cpuid_info) == 6);
+
+ if (nthreads_tot <= nthreads_omp_always_faster ||
+ (bIntel_Family6 &&
+ ((gmx_cpuid_model(hwinfo->cpuid_info) >= nthreads_omp_always_faster_Nehalem && nthreads_tot <= nthreads_omp_always_faster_Nehalem) ||
+ (gmx_cpuid_model(hwinfo->cpuid_info) >= nthreads_omp_always_faster_SandyBridge && nthreads_tot <= nthreads_omp_always_faster_SandyBridge))))
+ {
+ /* Use pure OpenMP parallelization */
+ nthreads_tmpi = 1;
+ }
+ else
+ {
+ /* Don't use OpenMP parallelization */
+ nthreads_tmpi = nthreads_tot;
+ }
+ }
+
+ return nthreads_tmpi;
+}
+
+
+/* Get the number of threads to use for thread-MPI based on how many
+ * were requested, which algorithms we're using,
+ * and how many particles there are.
+ * At the point we have already called check_and_update_hw_opt.
+ * Thus all options should be internally consistent and consistent
+ * with the hardware, except that ntmpi could be larger than #GPU.
+ */
+static int get_nthreads_mpi(const gmx_hw_info_t *hwinfo,
+ gmx_hw_opt_t *hw_opt,
+ t_inputrec *inputrec, gmx_mtop_t *mtop,
+ const t_commrec *cr,
+ FILE *fplog)
+{
+ int nthreads_hw, nthreads_tot_max, nthreads_tmpi, nthreads_new, ngpu;
+ int min_atoms_per_mpi_thread;
+ char *env;
+ char sbuf[STRLEN];
+ gmx_bool bCanUseGPU;
+
+ if (hw_opt->nthreads_tmpi > 0)
+ {
+ /* Trivial, return right away */
+ return hw_opt->nthreads_tmpi;
+ }
+
+ nthreads_hw = hwinfo->nthreads_hw_avail;
+
+ /* How many total (#tMPI*#OpenMP) threads can we start? */
+ if (hw_opt->nthreads_tot > 0)
+ {
+ nthreads_tot_max = hw_opt->nthreads_tot;
+ }
+ else
+ {
+ nthreads_tot_max = nthreads_hw;
+ }
+
- ngpu = hwinfo->gpu_info.ncuda_dev_use;
++ bCanUseGPU = (inputrec->cutoff_scheme == ecutsVERLET &&
++ hwinfo->gpu_info.ncuda_dev_compatible > 0);
+ if (bCanUseGPU)
+ {
- gmx_bool *bUseGPU)
++ ngpu = hwinfo->gpu_info.ncuda_dev_compatible;
+ }
+ else
+ {
+ ngpu = 0;
+ }
+
++ if (inputrec->cutoff_scheme == ecutsGROUP)
++ {
++ /* We checked this before, but it doesn't hurt to do it once more */
++ assert(hw_opt->nthreads_omp == 1);
++ }
++
+ nthreads_tmpi =
+ get_tmpi_omp_thread_division(hwinfo, hw_opt, nthreads_tot_max, ngpu);
+
+ if (inputrec->eI == eiNM || EI_TPI(inputrec->eI))
+ {
+ /* Dims/steps are divided over the nodes iso splitting the atoms */
+ min_atoms_per_mpi_thread = 0;
+ }
+ else
+ {
+ if (bCanUseGPU)
+ {
+ min_atoms_per_mpi_thread = MIN_ATOMS_PER_GPU;
+ }
+ else
+ {
+ min_atoms_per_mpi_thread = MIN_ATOMS_PER_MPI_THREAD;
+ }
+ }
+
+ /* Check if an algorithm does not support parallel simulation. */
+ if (nthreads_tmpi != 1 &&
+ ( inputrec->eI == eiLBFGS ||
+ inputrec->coulombtype == eelEWALD ) )
+ {
+ nthreads_tmpi = 1;
+
+ md_print_warn(cr, fplog, "The integration or electrostatics algorithm doesn't support parallel runs. Using a single thread-MPI thread.\n");
+ if (hw_opt->nthreads_tmpi > nthreads_tmpi)
+ {
+ gmx_fatal(FARGS, "You asked for more than 1 thread-MPI thread, but an algorithm doesn't support that");
+ }
+ }
+ else if (mtop->natoms/nthreads_tmpi < min_atoms_per_mpi_thread)
+ {
+ /* the thread number was chosen automatically, but there are too many
+ threads (too few atoms per thread) */
+ nthreads_new = max(1, mtop->natoms/min_atoms_per_mpi_thread);
+
+ /* Avoid partial use of Hyper-Threading */
+ if (gmx_cpuid_x86_smt(hwinfo->cpuid_info) == GMX_CPUID_X86_SMT_ENABLED &&
+ nthreads_new > nthreads_hw/2 && nthreads_new < nthreads_hw)
+ {
+ nthreads_new = nthreads_hw/2;
+ }
+
+ /* Avoid large prime numbers in the thread count */
+ if (nthreads_new >= 6)
+ {
+ /* Use only 6,8,10 with additional factors of 2 */
+ int fac;
+
+ fac = 2;
+ while (3*fac*2 <= nthreads_new)
+ {
+ fac *= 2;
+ }
+
+ nthreads_new = (nthreads_new/fac)*fac;
+ }
+ else
+ {
+ /* Avoid 5 */
+ if (nthreads_new == 5)
+ {
+ nthreads_new = 4;
+ }
+ }
+
+ nthreads_tmpi = nthreads_new;
+
+ fprintf(stderr, "\n");
+ fprintf(stderr, "NOTE: Parallelization is limited by the small number of atoms,\n");
+ fprintf(stderr, " only starting %d thread-MPI threads.\n", nthreads_tmpi);
+ fprintf(stderr, " You can use the -nt and/or -ntmpi option to optimize the number of threads.\n\n");
+ }
+
+ return nthreads_tmpi;
+}
+#endif /* GMX_THREAD_MPI */
+
+
+/* Environment variable for setting nstlist */
+static const char* NSTLIST_ENVVAR = "GMX_NSTLIST";
+/* Try to increase nstlist when using a GPU with nstlist less than this */
+static const int NSTLIST_GPU_ENOUGH = 20;
+/* Increase nstlist until the non-bonded cost increases more than this factor */
+static const float NBNXN_GPU_LIST_OK_FAC = 1.20;
+/* Don't increase nstlist beyond a non-bonded cost increases of this factor.
+ * A standard (protein+)water system at 300K with PME ewald_rtol=1e-5
+ * needs 1.28 at rcoulomb=0.9 and 1.24 at rcoulomb=1.0 to get to nstlist=40.
+ */
+static const float NBNXN_GPU_LIST_MAX_FAC = 1.30;
+
+/* Try to increase nstlist when running on a GPU */
+static void increase_nstlist(FILE *fp, t_commrec *cr,
+ t_inputrec *ir, const gmx_mtop_t *mtop, matrix box)
+{
+ char *env;
+ int nstlist_orig, nstlist_prev;
+ verletbuf_list_setup_t ls;
+ real rlist_nstlist10, rlist_inc, rlist_ok, rlist_max;
+ real rlist_new, rlist_prev;
+ int i;
+ t_state state_tmp;
+ gmx_bool bBox, bDD, bCont;
+ const char *nstl_fmt = "\nFor optimal performance with a GPU nstlist (now %d) should be larger.\nThe optimum depends on your CPU and GPU resources.\nYou might want to try several nstlist values.\n";
+ const char *vbd_err = "Can not increase nstlist for GPU run because verlet-buffer-drift is not set or used";
+ const char *box_err = "Can not increase nstlist for GPU run because the box is too small";
+ const char *dd_err = "Can not increase nstlist for GPU run because of domain decomposition limitations";
+ char buf[STRLEN];
+
+ /* Number of + nstlist alternative values to try when switching */
+ const int nstl[] = { 20, 25, 40 };
+#define NNSTL sizeof(nstl)/sizeof(nstl[0])
+
+ env = getenv(NSTLIST_ENVVAR);
+ if (env == NULL)
+ {
+ if (fp != NULL)
+ {
+ fprintf(fp, nstl_fmt, ir->nstlist);
+ }
+ }
+
+ if (ir->verletbuf_drift == 0)
+ {
+ gmx_fatal(FARGS, "You are using an old tpr file with a GPU, please generate a new tpr file with an up to date version of grompp");
+ }
+
+ if (ir->verletbuf_drift < 0)
+ {
+ if (MASTER(cr))
+ {
+ fprintf(stderr, "%s\n", vbd_err);
+ }
+ if (fp != NULL)
+ {
+ fprintf(fp, "%s\n", vbd_err);
+ }
+
+ return;
+ }
+
+ nstlist_orig = ir->nstlist;
+ if (env != NULL)
+ {
+ sprintf(buf, "Getting nstlist from environment variable GMX_NSTLIST=%s", env);
+ if (MASTER(cr))
+ {
+ fprintf(stderr, "%s\n", buf);
+ }
+ if (fp != NULL)
+ {
+ fprintf(fp, "%s\n", buf);
+ }
+ sscanf(env, "%d", &ir->nstlist);
+ }
+
+ verletbuf_get_list_setup(TRUE, &ls);
+
+ /* Allow rlist to make the list a given factor larger than the list
+ * would be with nstlist=10.
+ */
+ nstlist_prev = ir->nstlist;
+ ir->nstlist = 10;
+ calc_verlet_buffer_size(mtop, det(box), ir, ir->verletbuf_drift, &ls,
+ NULL, &rlist_nstlist10);
+ ir->nstlist = nstlist_prev;
+
+ /* Determine the pair list size increase due to zero interactions */
+ rlist_inc = nbnxn_get_rlist_effective_inc(NBNXN_GPU_CLUSTER_SIZE, mtop->natoms/det(box));
+ rlist_ok = (rlist_nstlist10 + rlist_inc)*pow(NBNXN_GPU_LIST_OK_FAC, 1.0/3.0) - rlist_inc;
+ rlist_max = (rlist_nstlist10 + rlist_inc)*pow(NBNXN_GPU_LIST_MAX_FAC, 1.0/3.0) - rlist_inc;
+ if (debug)
+ {
+ fprintf(debug, "GPU nstlist tuning: rlist_inc %.3f rlist_max %.3f\n",
+ rlist_inc, rlist_max);
+ }
+
+ i = 0;
+ nstlist_prev = nstlist_orig;
+ rlist_prev = ir->rlist;
+ do
+ {
+ if (env == NULL)
+ {
+ ir->nstlist = nstl[i];
+ }
+
+ /* Set the pair-list buffer size in ir */
+ calc_verlet_buffer_size(mtop, det(box), ir, ir->verletbuf_drift, &ls,
+ NULL, &rlist_new);
+
+ /* Does rlist fit in the box? */
+ bBox = (sqr(rlist_new) < max_cutoff2(ir->ePBC, box));
+ bDD = TRUE;
+ if (bBox && DOMAINDECOMP(cr))
+ {
+ /* Check if rlist fits in the domain decomposition */
+ if (inputrec2nboundeddim(ir) < DIM)
+ {
+ gmx_incons("Changing nstlist with domain decomposition and unbounded dimensions is not implemented yet");
+ }
+ copy_mat(box, state_tmp.box);
+ bDD = change_dd_cutoff(cr, &state_tmp, ir, rlist_new);
+ }
+
+ bCont = FALSE;
+
+ if (env == NULL)
+ {
+ if (bBox && bDD && rlist_new <= rlist_max)
+ {
+ /* Increase nstlist */
+ nstlist_prev = ir->nstlist;
+ rlist_prev = rlist_new;
+ bCont = (i+1 < NNSTL && rlist_new < rlist_ok);
+ }
+ else
+ {
+ /* Stick with the previous nstlist */
+ ir->nstlist = nstlist_prev;
+ rlist_new = rlist_prev;
+ bBox = TRUE;
+ bDD = TRUE;
+ }
+ }
+
+ i++;
+ }
+ while (bCont);
+
+ if (!bBox || !bDD)
+ {
+ gmx_warning(!bBox ? box_err : dd_err);
+ if (fp != NULL)
+ {
+ fprintf(fp, "\n%s\n", bBox ? box_err : dd_err);
+ }
+ ir->nstlist = nstlist_orig;
+ }
+ else if (ir->nstlist != nstlist_orig || rlist_new != ir->rlist)
+ {
+ sprintf(buf, "Changing nstlist from %d to %d, rlist from %g to %g",
+ nstlist_orig, ir->nstlist,
+ ir->rlist, rlist_new);
+ if (MASTER(cr))
+ {
+ fprintf(stderr, "%s\n\n", buf);
+ }
+ if (fp != NULL)
+ {
+ fprintf(fp, "%s\n\n", buf);
+ }
+ ir->rlist = rlist_new;
+ ir->rlistlong = rlist_new;
+ }
+}
+
+static void prepare_verlet_scheme(FILE *fplog,
+ const gmx_hw_info_t *hwinfo,
+ t_commrec *cr,
+ t_inputrec *ir,
+ const gmx_mtop_t *mtop,
+ matrix box,
- /* Here we only check for GPU usage on the MPI master process,
- * as here we don't know how many GPUs we will use yet.
- * We check for a GPU on all processes later.
- */
- *bUseGPU = hwinfo->bCanUseGPU || (getenv("GMX_EMULATE_GPU") != NULL);
-
++ gmx_bool bUseGPU)
+{
- verletbuf_get_list_setup(*bUseGPU, &ls);
+ if (ir->verletbuf_drift > 0)
+ {
+ /* Update the Verlet buffer size for the current run setup */
+ verletbuf_list_setup_t ls;
+ real rlist_new;
+
+ /* Here we assume CPU acceleration is on. But as currently
+ * calc_verlet_buffer_size gives the same results for 4x8 and 4x4
+ * and 4x2 gives a larger buffer than 4x4, this is ok.
+ */
- *bUseGPU &&
++ verletbuf_get_list_setup(bUseGPU, &ls);
+
+ calc_verlet_buffer_size(mtop, det(box), ir,
+ ir->verletbuf_drift, &ls,
+ NULL, &rlist_new);
+ if (rlist_new != ir->rlist)
+ {
+ if (fplog != NULL)
+ {
+ fprintf(fplog, "\nChanging rlist from %g to %g for non-bonded %dx%d atom kernels\n\n",
+ ir->rlist, rlist_new,
+ ls.cluster_size_i, ls.cluster_size_j);
+ }
+ ir->rlist = rlist_new;
+ ir->rlistlong = rlist_new;
+ }
+ }
+
+ /* With GPU or emulation we should check nstlist for performance */
+ if ((EI_DYNAMICS(ir->eI) &&
- static void check_and_update_hw_opt(gmx_hw_opt_t *hw_opt,
- int cutoff_scheme,
- gmx_bool bIsSimMaster)
++ bUseGPU &&
+ ir->nstlist < NSTLIST_GPU_ENOUGH) ||
+ getenv(NSTLIST_ENVVAR) != NULL)
+ {
+ /* Choose a better nstlist */
+ increase_nstlist(fplog, cr, ir, mtop, box);
+ }
+}
+
+static void convert_to_verlet_scheme(FILE *fplog,
+ t_inputrec *ir,
+ gmx_mtop_t *mtop, real box_vol)
+{
+ char *conv_mesg = "Converting input file with group cut-off scheme to the Verlet cut-off scheme";
+
+ md_print_warn(NULL, fplog, "%s\n", conv_mesg);
+
+ ir->cutoff_scheme = ecutsVERLET;
+ ir->verletbuf_drift = 0.005;
+
+ if (ir->rcoulomb != ir->rvdw)
+ {
+ gmx_fatal(FARGS, "The VdW and Coulomb cut-offs are different, whereas the Verlet scheme only supports equal cut-offs");
+ }
+
+ if (ir->vdwtype == evdwUSER || EEL_USER(ir->coulombtype))
+ {
+ gmx_fatal(FARGS, "User non-bonded potentials are not (yet) supported with the Verlet scheme");
+ }
+ else if (EVDW_SWITCHED(ir->vdwtype) || EEL_SWITCHED(ir->coulombtype))
+ {
+ md_print_warn(NULL, fplog, "Converting switched or shifted interactions to a shifted potential (without force shift), this will lead to slightly different interaction potentials");
+
+ if (EVDW_SWITCHED(ir->vdwtype))
+ {
+ ir->vdwtype = evdwCUT;
+ }
+ if (EEL_SWITCHED(ir->coulombtype))
+ {
+ if (EEL_FULL(ir->coulombtype))
+ {
+ /* With full electrostatic only PME can be switched */
+ ir->coulombtype = eelPME;
+ }
+ else
+ {
+ md_print_warn(NULL, fplog, "NOTE: Replacing %s electrostatics with reaction-field with epsilon-rf=inf\n", eel_names[ir->coulombtype]);
+ ir->coulombtype = eelRF;
+ ir->epsilon_rf = 0.0;
+ }
+ }
+
+ /* We set the target energy drift to a small number.
+ * Note that this is only for testing. For production the user
+ * should think about this and set the mdp options.
+ */
+ ir->verletbuf_drift = 1e-4;
+ }
+
+ if (inputrec2nboundeddim(ir) != 3)
+ {
+ gmx_fatal(FARGS, "Can only convert old tpr files to the Verlet cut-off scheme with 3D pbc");
+ }
+
+ if (ir->efep != efepNO || ir->implicit_solvent != eisNO)
+ {
+ gmx_fatal(FARGS, "Will not convert old tpr files to the Verlet cut-off scheme with free-energy calculations or implicit solvent");
+ }
+
+ if (EI_DYNAMICS(ir->eI) && !(EI_MD(ir->eI) && ir->etc == etcNO))
+ {
+ verletbuf_list_setup_t ls;
+
+ verletbuf_get_list_setup(FALSE, &ls);
+ calc_verlet_buffer_size(mtop, box_vol, ir, ir->verletbuf_drift, &ls,
+ NULL, &ir->rlist);
+ }
+ else
+ {
+ ir->verletbuf_drift = -1;
+ ir->rlist = 1.05*max(ir->rvdw, ir->rcoulomb);
+ }
+
+ gmx_mtop_remove_chargegroups(mtop);
+}
+
- if (cutoff_scheme == ecutsGROUP)
- {
- /* We only have OpenMP support for PME only nodes */
- if (hw_opt->nthreads_omp > 1)
- {
- gmx_fatal(FARGS, "OpenMP threads have been requested with cut-off scheme %s, but these are only supported with cut-off scheme %s",
- ecutscheme_names[cutoff_scheme],
- ecutscheme_names[ecutsVERLET]);
- }
- hw_opt->nthreads_omp = 1;
- }
-
++static void print_hw_opt(FILE *fp, const gmx_hw_opt_t *hw_opt)
++{
++ fprintf(fp, "hw_opt: nt %d ntmpi %d ntomp %d ntomp_pme %d gpu_id '%s'\n",
++ hw_opt->nthreads_tot,
++ hw_opt->nthreads_tmpi,
++ hw_opt->nthreads_omp,
++ hw_opt->nthreads_omp_pme,
++ hw_opt->gpu_opt.gpu_id != NULL ? hw_opt->gpu_opt.gpu_id : "");
++}
++
++/* Checks we can do when we don't (yet) know the cut-off scheme */
++static void check_and_update_hw_opt_1(gmx_hw_opt_t *hw_opt,
++ gmx_bool bIsSimMaster)
+{
+ gmx_omp_nthreads_read_env(&hw_opt->nthreads_omp, bIsSimMaster);
+
+#ifndef GMX_THREAD_MPI
+ if (hw_opt->nthreads_tot > 0)
+ {
+ gmx_fatal(FARGS, "Setting the total number of threads is only supported with thread-MPI and Gromacs was compiled without thread-MPI");
+ }
+ if (hw_opt->nthreads_tmpi > 0)
+ {
+ gmx_fatal(FARGS, "Setting the number of thread-MPI threads is only supported with thread-MPI and Gromacs was compiled without thread-MPI");
+ }
+#endif
+
+#ifndef GMX_OPENMP
+ if (hw_opt->nthreads_omp > 1)
+ {
+ gmx_fatal(FARGS, "More than 1 OpenMP thread requested, but Gromacs was compiled without OpenMP support");
+ }
+ hw_opt->nthreads_omp = 1;
+#endif
+
+ if (hw_opt->nthreads_tot > 0 && hw_opt->nthreads_omp_pme <= 0)
+ {
+ /* We have the same number of OpenMP threads for PP and PME processes,
+ * thus we can perform several consistency checks.
+ */
+ if (hw_opt->nthreads_tmpi > 0 &&
+ hw_opt->nthreads_omp > 0 &&
+ hw_opt->nthreads_tot != hw_opt->nthreads_tmpi*hw_opt->nthreads_omp)
+ {
+ gmx_fatal(FARGS, "The total number of threads requested (%d) does not match the thread-MPI threads (%d) times the OpenMP threads (%d) requested",
+ hw_opt->nthreads_tot, hw_opt->nthreads_tmpi, hw_opt->nthreads_omp);
+ }
+
+ if (hw_opt->nthreads_tmpi > 0 &&
+ hw_opt->nthreads_tot % hw_opt->nthreads_tmpi != 0)
+ {
+ gmx_fatal(FARGS, "The total number of threads requested (%d) is not divisible by the number of thread-MPI threads requested (%d)",
+ hw_opt->nthreads_tot, hw_opt->nthreads_tmpi);
+ }
+
+ if (hw_opt->nthreads_omp > 0 &&
+ hw_opt->nthreads_tot % hw_opt->nthreads_omp != 0)
+ {
+ gmx_fatal(FARGS, "The total number of threads requested (%d) is not divisible by the number of OpenMP threads requested (%d)",
+ hw_opt->nthreads_tot, hw_opt->nthreads_omp);
+ }
+
+ if (hw_opt->nthreads_tmpi > 0 &&
+ hw_opt->nthreads_omp <= 0)
+ {
+ hw_opt->nthreads_omp = hw_opt->nthreads_tot/hw_opt->nthreads_tmpi;
+ }
+ }
+
+#ifndef GMX_OPENMP
+ if (hw_opt->nthreads_omp > 1)
+ {
+ gmx_fatal(FARGS, "OpenMP threads are requested, but Gromacs was compiled without OpenMP support");
+ }
+#endif
+
- fprintf(debug, "hw_opt: nt %d ntmpi %d ntomp %d ntomp_pme %d gpu_id '%s'\n",
- hw_opt->nthreads_tot,
- hw_opt->nthreads_tmpi,
- hw_opt->nthreads_omp,
- hw_opt->nthreads_omp_pme,
- hw_opt->gpu_id != NULL ? hw_opt->gpu_id : "");
+ if (hw_opt->nthreads_omp_pme > 0 && hw_opt->nthreads_omp <= 0)
+ {
+ gmx_fatal(FARGS, "You need to specify -ntomp in addition to -ntomp_pme");
+ }
+
+ if (hw_opt->nthreads_tot == 1)
+ {
+ hw_opt->nthreads_tmpi = 1;
+
+ if (hw_opt->nthreads_omp > 1)
+ {
+ gmx_fatal(FARGS, "You requested %d OpenMP threads with %d total threads",
+ hw_opt->nthreads_tmpi, hw_opt->nthreads_tot);
+ }
+ hw_opt->nthreads_omp = 1;
+ }
+
+ if (hw_opt->nthreads_omp_pme <= 0 && hw_opt->nthreads_omp > 0)
+ {
+ hw_opt->nthreads_omp_pme = hw_opt->nthreads_omp;
+ }
+
++ /* Parse GPU IDs, if provided.
++ * We check consistency with the tMPI thread count later.
++ */
++ gmx_parse_gpu_ids(&hw_opt->gpu_opt);
++
++#ifdef GMX_THREAD_MPI
++ if (hw_opt->gpu_opt.ncuda_dev_use > 0 && hw_opt->nthreads_tmpi == 0)
++ {
++ /* Set the number of MPI threads equal to the number of GPUs */
++ hw_opt->nthreads_tmpi = hw_opt->gpu_opt.ncuda_dev_use;
++
++ if (hw_opt->nthreads_tot > 0 &&
++ hw_opt->nthreads_tmpi > hw_opt->nthreads_tot)
++ {
++ /* We have more GPUs than total threads requested.
++ * We choose to (later) generate a mismatch error,
++ * instead of launching more threads than requested.
++ */
++ hw_opt->nthreads_tmpi = hw_opt->nthreads_tot;
++ }
++ }
++#endif
++
+ if (debug)
+ {
- /* Data structure set by SIMMASTER which needs to be passed to all nodes
- * before the other nodes have read the tpx file and called gmx_detect_hardware.
++ print_hw_opt(debug, hw_opt);
++ }
++}
+
++/* Checks we can do when we know the cut-off scheme */
++static void check_and_update_hw_opt_2(gmx_hw_opt_t *hw_opt,
++ int cutoff_scheme)
++{
++ if (cutoff_scheme == ecutsGROUP)
++ {
++ /* We only have OpenMP support for PME only nodes */
++ if (hw_opt->nthreads_omp > 1)
++ {
++ gmx_fatal(FARGS, "OpenMP threads have been requested with cut-off scheme %s, but these are only supported with cut-off scheme %s",
++ ecutscheme_names[cutoff_scheme],
++ ecutscheme_names[ecutsVERLET]);
++ }
++ hw_opt->nthreads_omp = 1;
++ }
++
++ if (hw_opt->nthreads_omp_pme <= 0 && hw_opt->nthreads_omp > 0)
++ {
++ hw_opt->nthreads_omp_pme = hw_opt->nthreads_omp;
++ }
++
++ if (debug)
++ {
++ print_hw_opt(debug, hw_opt);
+ }
+}
+
+
+/* Override the value in inputrec with value passed on the command line (if any) */
+static void override_nsteps_cmdline(FILE *fplog,
+ gmx_large_int_t nsteps_cmdline,
+ t_inputrec *ir,
+ const t_commrec *cr)
+{
+ char sbuf[STEPSTRSIZE];
+
+ assert(ir);
+ assert(cr);
+
+ /* override with anything else than the default -2 */
+ if (nsteps_cmdline > -2)
+ {
+ char stmp[STRLEN];
+
+ ir->nsteps = nsteps_cmdline;
+ if (EI_DYNAMICS(ir->eI))
+ {
+ sprintf(stmp, "Overriding nsteps with value passed on the command line: %s steps, %.3f ps",
+ gmx_step_str(nsteps_cmdline, sbuf),
+ nsteps_cmdline*ir->delta_t);
+ }
+ else
+ {
+ sprintf(stmp, "Overriding nsteps with value passed on the command line: %s steps",
+ gmx_step_str(nsteps_cmdline, sbuf));
+ }
+
+ md_print_warn(cr, fplog, "%s\n", stmp);
+ }
+}
+
- typedef struct {
- int cutoff_scheme; /* The cutoff scheme from inputrec_t */
- gmx_bool bUseGPU; /* Use GPU or GPU emulation */
- } master_inf_t;
++/* Frees GPU memory and destroys the CUDA context.
++ *
++ * Note that this function needs to be called even if GPUs are not used
++ * in this run because the PME ranks have no knowledge of whether GPUs
++ * are used or not, but all ranks need to enter the barrier below.
+ */
- master_inf_t minf = {-1, FALSE};
++static void free_gpu_resources(FILE *fplog,
++ const t_forcerec *fr,
++ const t_commrec *cr)
++{
++ gmx_bool bIsPPrankUsingGPU;
++ char gpu_err_str[STRLEN];
++
++ bIsPPrankUsingGPU = (cr->duty & DUTY_PP) && fr->nbv != NULL && fr->nbv->bUseGPU;
++
++ if (bIsPPrankUsingGPU)
++ {
++ /* free nbnxn data in GPU memory */
++ nbnxn_cuda_free(fplog, fr->nbv->cu_nbv);
++
++ /* With tMPI we need to wait for all ranks to finish deallocation before
++ * destroying the context in free_gpu() as some ranks may be sharing
++ * GPU and context.
++ * Note: as only PP ranks need to free GPU resources, so it is safe to
++ * not call the barrier on PME ranks.
++ */
++#ifdef GMX_THREAD_MPI
++ if (PAR(cr))
++ {
++ gmx_barrier(cr);
++ }
++#endif /* GMX_THREAD_MPI */
++
++ /* uninitialize GPU (by destroying the context) */
++ if (!free_gpu(gpu_err_str))
++ {
++ gmx_warning("On node %d failed to free GPU #%d: %s",
++ cr->nodeid, get_current_gpu_device_id(), gpu_err_str);
++ }
++ }
++}
+
+int mdrunner(gmx_hw_opt_t *hw_opt,
+ FILE *fplog, t_commrec *cr, int nfile,
+ const t_filenm fnm[], const output_env_t oenv, gmx_bool bVerbose,
+ gmx_bool bCompact, int nstglobalcomm,
+ ivec ddxyz, int dd_node_order, real rdd, real rconstr,
+ const char *dddlb_opt, real dlb_scale,
+ const char *ddcsx, const char *ddcsy, const char *ddcsz,
+ const char *nbpu_opt,
+ gmx_large_int_t nsteps_cmdline, int nstepout, int resetstep,
+ int nmultisim, int repl_ex_nst, int repl_ex_nex,
+ int repl_ex_seed, real pforce, real cpt_period, real max_hours,
+ const char *deviceOptions, unsigned long Flags)
+{
+ gmx_bool bForceUseGPU, bTryUseGPU;
+ double nodetime = 0, realtime;
+ t_inputrec *inputrec;
+ t_state *state = NULL;
+ matrix box;
+ gmx_ddbox_t ddbox = {0};
+ int npme_major, npme_minor;
+ real tmpr1, tmpr2;
+ t_nrnb *nrnb;
+ gmx_mtop_t *mtop = NULL;
+ t_mdatoms *mdatoms = NULL;
+ t_forcerec *fr = NULL;
+ t_fcdata *fcd = NULL;
+ real ewaldcoeff = 0;
+ gmx_pme_t *pmedata = NULL;
+ gmx_vsite_t *vsite = NULL;
+ gmx_constr_t constr;
+ int i, m, nChargePerturbed = -1, status, nalloc;
+ char *gro;
+ gmx_wallcycle_t wcycle;
+ gmx_bool bReadRNG, bReadEkin;
+ int list;
+ gmx_walltime_accounting_t walltime_accounting = NULL;
+ int rc;
+ gmx_large_int_t reset_counters;
+ gmx_edsam_t ed = NULL;
+ t_commrec *cr_old = cr;
+ int nthreads_pme = 1;
+ int nthreads_pp = 1;
+ gmx_membed_t membed = NULL;
+ gmx_hw_info_t *hwinfo = NULL;
- hwinfo = gmx_detect_hardware(fplog, cr,
- bForceUseGPU, bTryUseGPU, hw_opt->gpu_id);
++ /* The master rank decides early on bUseGPU and broadcasts this later */
++ gmx_bool bUseGPU = FALSE;
+
+ /* CAUTION: threads may be started later on in this function, so
+ cr doesn't reflect the final parallel state right now */
+ snew(inputrec, 1);
+ snew(mtop, 1);
+
+ if (Flags & MD_APPENDFILES)
+ {
+ fplog = NULL;
+ }
+
+ bForceUseGPU = (strncmp(nbpu_opt, "gpu", 3) == 0);
+ bTryUseGPU = (strncmp(nbpu_opt, "auto", 4) == 0) || bForceUseGPU;
+
+ /* Detect hardware, gather information. This is an operation that is
+ * global for this process (MPI rank). */
-
- minf.cutoff_scheme = inputrec->cutoff_scheme;
- minf.bUseGPU = FALSE;
-
++ hwinfo = gmx_detect_hardware(fplog, cr, bTryUseGPU);
+
+
+ snew(state, 1);
+ if (SIMMASTER(cr))
+ {
+ /* Read (nearly) all data required for the simulation */
+ read_tpx_state(ftp2fn(efTPX, nfile, fnm), inputrec, state, NULL, mtop);
+
+ if (inputrec->cutoff_scheme != ecutsVERLET &&
+ ((Flags & MD_TESTVERLET) || getenv("GMX_VERLET_SCHEME") != NULL))
+ {
+ convert_to_verlet_scheme(fplog, inputrec, mtop, det(state->box));
+ }
+
- &minf.bUseGPU);
+ if (inputrec->cutoff_scheme == ecutsVERLET)
+ {
++ /* Here the master rank decides if all ranks will use GPUs */
++ bUseGPU = (hwinfo->gpu_info.ncuda_dev_compatible > 0 ||
++ getenv("GMX_EMULATE_GPU") != NULL);
++
+ prepare_verlet_scheme(fplog, hwinfo, cr,
+ inputrec, mtop, state->box,
- else if (hwinfo->bCanUseGPU)
++ bUseGPU);
+ }
- #ifdef GMX_IS_BGQ
++ else if (hwinfo->gpu_info.ncuda_dev_compatible > 0)
+ {
+ md_print_warn(cr, fplog,
+ "NOTE: GPU(s) found, but the current simulation can not use GPUs\n"
+ " To use a GPU, set the mdp option: cutoff-scheme = Verlet\n"
+ " (for quick performance testing you can use the -testverlet option)\n");
+
+ if (bForceUseGPU)
+ {
+ gmx_fatal(FARGS, "GPU requested, but can't be used without cutoff-scheme=Verlet");
+ }
+ }
- #ifndef GMX_THREAD_MPI
- if (PAR(cr))
- {
- gmx_bcast_sim(sizeof(minf), &minf, cr);
- }
- #endif
- if (minf.bUseGPU && cr->npmenodes == -1)
- {
- /* Don't automatically use PME-only nodes with GPUs */
- cr->npmenodes = 0;
- }
++#ifdef GMX_TARGET_BGQ
+ else
+ {
+ md_print_warn(cr, fplog,
+ "NOTE: There is no SIMD implementation of the group scheme kernels on\n"
+ " BlueGene/Q. You will observe better performance from using the\n"
+ " Verlet cut-off scheme.\n");
+ }
+#endif
+ }
- #ifdef GMX_THREAD_MPI
- /* With thread-MPI inputrec is only set here on the master thread */
+
+ /* Check for externally set OpenMP affinity and turn off internal
+ * pinning if any is found. We need to do this check early to tell
+ * thread-MPI whether it should do pinning when spawning threads.
+ * TODO: the above no longer holds, we should move these checks down
+ */
+ gmx_omp_check_thread_affinity(fplog, cr, hw_opt);
+
- #endif
++ /* Check and update the hardware options for internal consistency */
++ check_and_update_hw_opt_1(hw_opt, SIMMASTER(cr));
++
+ if (SIMMASTER(cr))
- check_and_update_hw_opt(hw_opt, minf.cutoff_scheme, SIMMASTER(cr));
-
+ {
- /* Early check for externally set process affinity. Can't do over all
- * MPI processes because hwinfo is not available everywhere, but with
- * thread-MPI it's needed as pinning might get turned off which needs
- * to be known before starting thread-MPI. */
+#ifdef GMX_THREAD_MPI
- /* check consistency and decide on the number of gpus to use. */
- gmx_check_hw_runconf_consistency(fplog, hwinfo, cr, hw_opt->nthreads_tmpi,
- minf.bUseGPU);
++ /* Early check for externally set process affinity.
++ * With thread-MPI this is needed as pinning might get turned off,
++ * which needs to be known before starting thread-MPI.
++ * With thread-MPI hw_opt is processed here on the master rank
++ * and passed to the other ranks later, so we only do this on master.
++ */
+ gmx_check_thread_affinity_set(fplog,
+ NULL,
+ hw_opt, hwinfo->nthreads_hw_avail, FALSE);
+#endif
+
+#ifdef GMX_THREAD_MPI
+ if (cr->npmenodes > 0 && hw_opt->nthreads_tmpi <= 0)
+ {
+ gmx_fatal(FARGS, "You need to explicitly specify the number of MPI threads (-ntmpi) when using separate PME nodes");
+ }
+#endif
+
+ if (hw_opt->nthreads_omp_pme != hw_opt->nthreads_omp &&
+ cr->npmenodes <= 0)
+ {
+ gmx_fatal(FARGS, "You need to explicitly specify the number of PME nodes (-npme) when using different number of OpenMP threads for PP and PME nodes");
+ }
+ }
+
+#ifdef GMX_THREAD_MPI
+ if (SIMMASTER(cr))
+ {
++ /* Since the master knows the cut-off scheme, update hw_opt for this.
++ * This is done later for normal MPI and also once more with tMPI
++ * for all tMPI ranks.
++ */
++ check_and_update_hw_opt_2(hw_opt, inputrec->cutoff_scheme);
++
+ /* NOW the threads will be started: */
+ hw_opt->nthreads_tmpi = get_nthreads_mpi(hwinfo,
+ hw_opt,
+ inputrec, mtop,
+ cr, fplog);
+ if (hw_opt->nthreads_tot > 0 && hw_opt->nthreads_omp <= 0)
+ {
+ hw_opt->nthreads_omp = hw_opt->nthreads_tot/hw_opt->nthreads_tmpi;
+ }
+
+ if (hw_opt->nthreads_tmpi > 1)
+ {
+ /* now start the threads. */
+ cr = mdrunner_start_threads(hw_opt, fplog, cr_old, nfile, fnm,
+ oenv, bVerbose, bCompact, nstglobalcomm,
+ ddxyz, dd_node_order, rdd, rconstr,
+ dddlb_opt, dlb_scale, ddcsx, ddcsy, ddcsz,
+ nbpu_opt,
+ nsteps_cmdline, nstepout, resetstep, nmultisim,
+ repl_ex_nst, repl_ex_nex, repl_ex_seed, pforce,
+ cpt_period, max_hours, deviceOptions,
+ Flags);
+ /* the main thread continues here with a new cr. We don't deallocate
+ the old cr because other threads may still be reading it. */
+ if (cr == NULL)
+ {
+ gmx_comm("Failed to spawn threads");
+ }
+ }
+ }
+#endif
+ /* END OF CAUTION: cr is now reliable */
+
+ /* g_membed initialisation *
+ * Because we change the mtop, init_membed is called before the init_parallel *
+ * (in case we ever want to make it run in parallel) */
+ if (opt2bSet("-membed", nfile, fnm))
+ {
+ if (MASTER(cr))
+ {
+ fprintf(stderr, "Initializing membed");
+ }
+ membed = init_membed(fplog, nfile, fnm, mtop, inputrec, state, cr, &cpt_period);
+ }
+
+ if (PAR(cr))
+ {
+ /* now broadcast everything to the non-master nodes/threads: */
+ init_parallel(cr, inputrec, mtop);
+
+ /* This check needs to happen after get_nthreads_mpi() */
+ if (inputrec->cutoff_scheme == ecutsVERLET && (Flags & MD_PARTDEC))
+ {
+ gmx_fatal_collective(FARGS, cr, NULL,
+ "The Verlet cut-off scheme is not supported with particle decomposition.\n"
+ "You can achieve the same effect as particle decomposition by running in parallel using only OpenMP threads.");
+ }
+ }
+ if (fplog != NULL)
+ {
+ pr_inputrec(fplog, 0, "Input Parameters", inputrec, FALSE);
+ }
+
+ /* now make sure the state is initialized and propagated */
+ set_state_entries(state, inputrec, cr->nnodes);
+
+ /* A parallel command line option consistency check that we can
+ only do after any threads have started. */
+ if (!PAR(cr) &&
+ (ddxyz[XX] > 1 || ddxyz[YY] > 1 || ddxyz[ZZ] > 1 || cr->npmenodes > 0))
+ {
+ gmx_fatal(FARGS,
+ "The -dd or -npme option request a parallel simulation, "
+#ifndef GMX_MPI
+ "but %s was compiled without threads or MPI enabled"
+#else
+#ifdef GMX_THREAD_MPI
+ "but the number of threads (option -nt) is 1"
+#else
+ "but %s was not started through mpirun/mpiexec or only one process was requested through mpirun/mpiexec"
+#endif
+#endif
+ , ShortProgram()
+ );
+ }
+
+ if ((Flags & MD_RERUN) &&
+ (EI_ENERGY_MINIMIZATION(inputrec->eI) || eiNM == inputrec->eI))
+ {
+ gmx_fatal(FARGS, "The .mdp file specified an energy mininization or normal mode algorithm, and these are not compatible with mdrun -rerun");
+ }
+
+ if (can_use_allvsall(inputrec, TRUE, cr, fplog) && PAR(cr))
+ {
+ /* Simple neighbour searching and (also?) all-vs-all loops
+ * do not work with domain decomposition. */
+ Flags |= MD_PARTDEC;
+ }
+
+ if (!EEL_PME(inputrec->coulombtype) || (Flags & MD_PARTDEC))
+ {
+ if (cr->npmenodes > 0)
+ {
+ if (!EEL_PME(inputrec->coulombtype))
+ {
+ gmx_fatal_collective(FARGS, cr, NULL,
+ "PME nodes are requested, but the system does not use PME electrostatics");
+ }
+ if (Flags & MD_PARTDEC)
+ {
+ gmx_fatal_collective(FARGS, cr, NULL,
+ "PME nodes are requested, but particle decomposition does not support separate PME nodes");
+ }
+ }
+
+ cr->npmenodes = 0;
+ }
+
+#ifdef GMX_FAHCORE
+ fcRegisterSteps(inputrec->nsteps, inputrec->init_step);
+#endif
+
+ /* NMR restraints must be initialized before load_checkpoint,
+ * since with time averaging the history is added to t_state.
+ * For proper consistency check we therefore need to extend
+ * t_state here.
+ * So the PME-only nodes (if present) will also initialize
+ * the distance restraints.
+ */
+ snew(fcd, 1);
+
+ /* This needs to be called before read_checkpoint to extend the state */
+ init_disres(fplog, mtop, inputrec, cr, Flags & MD_PARTDEC, fcd, state, repl_ex_nst > 0);
+
+ if (gmx_mtop_ftype_count(mtop, F_ORIRES) > 0)
+ {
+ if (PAR(cr) && !(Flags & MD_PARTDEC))
+ {
+ gmx_fatal(FARGS, "Orientation restraints do not work (yet) with domain decomposition, use particle decomposition (mdrun option -pd)");
+ }
+ /* Orientation restraints */
+ if (MASTER(cr))
+ {
+ init_orires(fplog, mtop, state->x, inputrec, cr->ms, &(fcd->orires),
+ state);
+ }
+ }
+
+ if (DEFORM(*inputrec))
+ {
+ /* Store the deform reference box before reading the checkpoint */
+ if (SIMMASTER(cr))
+ {
+ copy_mat(state->box, box);
+ }
+ if (PAR(cr))
+ {
+ gmx_bcast(sizeof(box), box, cr);
+ }
+ /* Because we do not have the update struct available yet
+ * in which the reference values should be stored,
+ * we store them temporarily in static variables.
+ * This should be thread safe, since they are only written once
+ * and with identical values.
+ */
+#ifdef GMX_THREAD_MPI
+ tMPI_Thread_mutex_lock(&deform_init_box_mutex);
+#endif
+ deform_init_init_step_tpx = inputrec->init_step;
+ copy_mat(box, deform_init_box_tpx);
+#ifdef GMX_THREAD_MPI
+ tMPI_Thread_mutex_unlock(&deform_init_box_mutex);
+#endif
+ }
+
+ if (opt2bSet("-cpi", nfile, fnm))
+ {
+ /* Check if checkpoint file exists before doing continuation.
+ * This way we can use identical input options for the first and subsequent runs...
+ */
+ if (gmx_fexist_master(opt2fn_master("-cpi", nfile, fnm, cr), cr) )
+ {
+ load_checkpoint(opt2fn_master("-cpi", nfile, fnm, cr), &fplog,
+ cr, Flags & MD_PARTDEC, ddxyz,
+ inputrec, state, &bReadRNG, &bReadEkin,
+ (Flags & MD_APPENDFILES),
+ (Flags & MD_APPENDFILESSET));
+
+ if (bReadRNG)
+ {
+ Flags |= MD_READ_RNG;
+ }
+ if (bReadEkin)
+ {
+ Flags |= MD_READ_EKIN;
+ }
+ }
+ }
+
+ if (((MASTER(cr) || (Flags & MD_SEPPOT)) && (Flags & MD_APPENDFILES))
+#ifdef GMX_THREAD_MPI
+ /* With thread MPI only the master node/thread exists in mdrun.c,
+ * therefore non-master nodes need to open the "seppot" log file here.
+ */
+ || (!MASTER(cr) && (Flags & MD_SEPPOT))
+#endif
+ )
+ {
+ gmx_log_open(ftp2fn(efLOG, nfile, fnm), cr, !(Flags & MD_SEPPOT),
+ Flags, &fplog);
+ }
+
+ /* override nsteps with value from cmdline */
+ override_nsteps_cmdline(fplog, nsteps_cmdline, inputrec, cr);
+
+ if (SIMMASTER(cr))
+ {
+ copy_mat(state->box, box);
+ }
+
+ if (PAR(cr))
+ {
+ gmx_bcast(sizeof(box), box, cr);
+ }
+
+ /* Essential dynamics */
+ if (opt2bSet("-ei", nfile, fnm))
+ {
+ /* Open input and output files, allocate space for ED data structure */
+ ed = ed_open(mtop->natoms, &state->edsamstate, nfile, fnm, Flags, oenv, cr);
+ }
+
+ if (PAR(cr) && !((Flags & MD_PARTDEC) ||
+ EI_TPI(inputrec->eI) ||
+ inputrec->eI == eiNM))
+ {
+ cr->dd = init_domain_decomposition(fplog, cr, Flags, ddxyz, rdd, rconstr,
+ dddlb_opt, dlb_scale,
+ ddcsx, ddcsy, ddcsz,
+ mtop, inputrec,
+ box, state->x,
+ &ddbox, &npme_major, &npme_minor);
+
+ make_dd_communicators(fplog, cr, dd_node_order);
+
+ /* Set overallocation to avoid frequent reallocation of arrays */
+ set_over_alloc_dd(TRUE);
+ }
+ else
+ {
+ /* PME, if used, is done on all nodes with 1D decomposition */
+ cr->npmenodes = 0;
+ cr->duty = (DUTY_PP | DUTY_PME);
+ npme_major = 1;
+ npme_minor = 1;
+ /* NM and TPI perform single node energy calculations in parallel */
+ if (!(inputrec->eI == eiNM || EI_TPI(inputrec->eI)))
+ {
+ npme_major = cr->nnodes;
+ }
+
+ if (inputrec->ePBC == epbcSCREW)
+ {
+ gmx_fatal(FARGS,
+ "pbc=%s is only implemented with domain decomposition",
+ epbc_names[inputrec->ePBC]);
+ }
+ }
+
+ if (PAR(cr))
+ {
+ /* After possible communicator splitting in make_dd_communicators.
+ * we can set up the intra/inter node communication.
+ */
+ gmx_setup_nodecomm(fplog, cr);
+ }
+
+ /* Initialize per-physical-node MPI process/thread ID and counters. */
+ gmx_init_intranode_counters(cr);
+
+#ifdef GMX_MPI
+ md_print_info(cr, fplog, "Using %d MPI %s\n",
+ cr->nnodes,
+#ifdef GMX_THREAD_MPI
+ cr->nnodes == 1 ? "thread" : "threads"
+#else
+ cr->nnodes == 1 ? "process" : "processes"
+#endif
+ );
+ fflush(stderr);
+#endif
+
++ /* Check and update hw_opt for the cut-off scheme */
++ check_and_update_hw_opt_2(hw_opt, inputrec->cutoff_scheme);
++
+ gmx_omp_nthreads_init(fplog, cr,
+ hwinfo->nthreads_hw_avail,
+ hw_opt->nthreads_omp,
+ hw_opt->nthreads_omp_pme,
+ (cr->duty & DUTY_PP) == 0,
+ inputrec->cutoff_scheme == ecutsVERLET);
+
- fr = mk_forcerec();
- fr->hwinfo = hwinfo;
++ if (PAR(cr))
++ {
++ /* The master rank decided on the use of GPUs,
++ * broadcast this information to all ranks.
++ */
++ gmx_bcast_sim(sizeof(bUseGPU), &bUseGPU, cr);
++ }
++
++ if (bUseGPU)
++ {
++ if (cr->npmenodes == -1)
++ {
++ /* Don't automatically use PME-only nodes with GPUs */
++ cr->npmenodes = 0;
++ }
++
++ /* Select GPU id's to use */
++ gmx_select_gpu_ids(fplog, cr, &hwinfo->gpu_info, bForceUseGPU,
++ &hw_opt->gpu_opt);
++ }
++
++ /* check consistency of CPU acceleration and number of GPUs selected */
++ gmx_check_hw_runconf_consistency(fplog, hwinfo, cr, hw_opt, bUseGPU);
++
++ if (DOMAINDECOMP(cr))
++ {
++ /* When we share GPUs over ranks, we need to know this for the DLB */
++ dd_setup_dlb_resource_sharing(cr, hwinfo, hw_opt);
++ }
+
+ /* getting number of PP/PME threads
+ PME: env variable should be read only on one node to make sure it is
+ identical everywhere;
+ */
+ /* TODO nthreads_pp is only used for pinning threads.
+ * This is a temporary solution until we have a hw topology library.
+ */
+ nthreads_pp = gmx_omp_nthreads_get(emntNonbonded);
+ nthreads_pme = gmx_omp_nthreads_get(emntPME);
+
+ wcycle = wallcycle_init(fplog, resetstep, cr, nthreads_pp, nthreads_pme);
+
+ if (PAR(cr))
+ {
+ /* Master synchronizes its value of reset_counters with all nodes
+ * including PME only nodes */
+ reset_counters = wcycle_get_reset_counters(wcycle);
+ gmx_bcast_sim(sizeof(reset_counters), &reset_counters, cr);
+ wcycle_set_reset_counters(wcycle, reset_counters);
+ }
+
+ snew(nrnb, 1);
+ if (cr->duty & DUTY_PP)
+ {
+ /* For domain decomposition we allocate dynamically
+ * in dd_partition_system.
+ */
+ if (DOMAINDECOMP(cr))
+ {
+ bcast_state_setup(cr, state);
+ }
+ else
+ {
+ if (PAR(cr))
+ {
+ bcast_state(cr, state, TRUE);
+ }
+ }
+
+ /* Initiate forcerecord */
- FALSE, pforce);
++ fr = mk_forcerec();
++ fr->hwinfo = hwinfo;
++ fr->gpu_opt = &hw_opt->gpu_opt;
+ init_forcerec(fplog, oenv, fr, fcd, inputrec, mtop, cr, box,
+ opt2fn("-table", nfile, fnm),
+ opt2fn("-tabletf", nfile, fnm),
+ opt2fn("-tablep", nfile, fnm),
+ opt2fn("-tableb", nfile, fnm),
+ nbpu_opt,
- if ((cr->duty & DUTY_PP) && fr->nbv != NULL && fr->nbv->bUseGPU)
- {
- char gpu_err_str[STRLEN];
-
- /* free GPU memory and uninitialize GPU (by destroying the context) */
- nbnxn_cuda_free(fplog, fr->nbv->cu_nbv);
++ FALSE,
++ pforce);
+
+ /* version for PCA_NOT_READ_NODE (see md.c) */
+ /*init_forcerec(fplog,fr,fcd,inputrec,mtop,cr,box,FALSE,
+ "nofile","nofile","nofile","nofile",FALSE,pforce);
+ */
+ fr->bSepDVDL = ((Flags & MD_SEPPOT) == MD_SEPPOT);
+
+ /* Initialize QM-MM */
+ if (fr->bQMMM)
+ {
+ init_QMMMrec(cr, mtop, inputrec, fr);
+ }
+
+ /* Initialize the mdatoms structure.
+ * mdatoms is not filled with atom data,
+ * as this can not be done now with domain decomposition.
+ */
+ mdatoms = init_mdatoms(fplog, mtop, inputrec->efep != efepNO);
+
+ if (mdatoms->nPerturbed > 0 && inputrec->cutoff_scheme == ecutsVERLET)
+ {
+ gmx_fatal(FARGS, "The Verlet cut-off scheme does not (yet) support free-energy calculations with perturbed atoms, only perturbed interactions. This will be implemented soon. Use the group scheme for now.");
+ }
+
+ /* Initialize the virtual site communication */
+ vsite = init_vsite(mtop, cr, FALSE);
+
+ calc_shifts(box, fr->shift_vec);
+
+ /* With periodic molecules the charge groups should be whole at start up
+ * and the virtual sites should not be far from their proper positions.
+ */
+ if (!inputrec->bContinuation && MASTER(cr) &&
+ !(inputrec->ePBC != epbcNONE && inputrec->bPeriodicMols))
+ {
+ /* Make molecules whole at start of run */
+ if (fr->ePBC != epbcNONE)
+ {
+ do_pbc_first_mtop(fplog, inputrec->ePBC, box, mtop, state->x);
+ }
+ if (vsite)
+ {
+ /* Correct initial vsite positions are required
+ * for the initial distribution in the domain decomposition
+ * and for the initial shell prediction.
+ */
+ construct_vsites_mtop(vsite, mtop, state->x);
+ }
+ }
+
+ if (EEL_PME(fr->eeltype))
+ {
+ ewaldcoeff = fr->ewaldcoeff;
+ pmedata = &fr->pmedata;
+ }
+ else
+ {
+ pmedata = NULL;
+ }
+ }
+ else
+ {
+ /* This is a PME only node */
+
+ /* We don't need the state */
+ done_state(state);
+
+ ewaldcoeff = calc_ewaldcoeff(inputrec->rcoulomb, inputrec->ewald_rtol);
+ snew(pmedata, 1);
+ }
+
+ if (hw_opt->thread_affinity != threadaffOFF)
+ {
+ /* Before setting affinity, check whether the affinity has changed
+ * - which indicates that probably the OpenMP library has changed it
+ * since we first checked).
+ */
+ gmx_check_thread_affinity_set(fplog, cr,
+ hw_opt, hwinfo->nthreads_hw_avail, TRUE);
+
+ /* Set the CPU affinity */
+ gmx_set_thread_affinity(fplog, cr, hw_opt, hwinfo);
+ }
+
+ /* Initiate PME if necessary,
+ * either on all nodes or on dedicated PME nodes only. */
+ if (EEL_PME(inputrec->coulombtype))
+ {
+ if (mdatoms)
+ {
+ nChargePerturbed = mdatoms->nChargePerturbed;
+ }
+ if (cr->npmenodes > 0)
+ {
+ /* The PME only nodes need to know nChargePerturbed */
+ gmx_bcast_sim(sizeof(nChargePerturbed), &nChargePerturbed, cr);
+ }
+
+ if (cr->duty & DUTY_PME)
+ {
+ status = gmx_pme_init(pmedata, cr, npme_major, npme_minor, inputrec,
+ mtop ? mtop->natoms : 0, nChargePerturbed,
+ (Flags & MD_REPRODUCIBLE), nthreads_pme);
+ if (status != 0)
+ {
+ gmx_fatal(FARGS, "Error %d initializing PME", status);
+ }
+ }
+ }
+
+
+ if (integrator[inputrec->eI].func == do_md)
+ {
+ /* Turn on signal handling on all nodes */
+ /*
+ * (A user signal from the PME nodes (if any)
+ * is communicated to the PP nodes.
+ */
+ signal_handler_install();
+ }
+
+ if (cr->duty & DUTY_PP)
+ {
+ /* Assumes uniform use of the number of OpenMP threads */
+ walltime_accounting = walltime_accounting_init(gmx_omp_nthreads_get(emntDefault));
+
+ if (inputrec->ePull != epullNO)
+ {
+ /* Initialize pull code */
+ init_pull(fplog, inputrec, nfile, fnm, mtop, cr, oenv, inputrec->fepvals->init_lambda,
+ EI_DYNAMICS(inputrec->eI) && MASTER(cr), Flags);
+ }
+
+ if (inputrec->bRot)
+ {
+ /* Initialize enforced rotation code */
+ init_rot(fplog, inputrec, nfile, fnm, cr, state->x, box, mtop, oenv,
+ bVerbose, Flags);
+ }
+
+ constr = init_constraints(fplog, mtop, inputrec, ed, state, cr);
+
+ if (DOMAINDECOMP(cr))
+ {
+ dd_init_bondeds(fplog, cr->dd, mtop, vsite, inputrec,
+ Flags & MD_DDBONDCHECK, fr->cginfo_mb);
+
+ set_dd_parameters(fplog, cr->dd, dlb_scale, inputrec, &ddbox);
+
+ setup_dd_grid(fplog, cr->dd);
+ }
+
+ /* Now do whatever the user wants us to do (how flexible...) */
+ integrator[inputrec->eI].func(fplog, cr, nfile, fnm,
+ oenv, bVerbose, bCompact,
+ nstglobalcomm,
+ vsite, constr,
+ nstepout, inputrec, mtop,
+ fcd, state,
+ mdatoms, nrnb, wcycle, ed, fr,
+ repl_ex_nst, repl_ex_nex, repl_ex_seed,
+ membed,
+ cpt_period, max_hours,
+ deviceOptions,
+ Flags,
+ walltime_accounting);
+
+ if (inputrec->ePull != epullNO)
+ {
+ finish_pull(inputrec->pull);
+ }
+
+ if (inputrec->bRot)
+ {
+ finish_rot(inputrec->rot);
+ }
+
+ }
+ else
+ {
+ /* do PME only */
+ walltime_accounting = walltime_accounting_init(gmx_omp_nthreads_get(emntPME));
+ gmx_pmeonly(*pmedata, cr, nrnb, wcycle, walltime_accounting, ewaldcoeff, inputrec);
+ }
+
+ wallcycle_stop(wcycle, ewcRUN);
+
+ /* Finish up, write some stuff
+ * if rerunMD, don't write last frame again
+ */
+ finish_run(fplog, cr,
+ inputrec, nrnb, wcycle, walltime_accounting,
+ fr != NULL && fr->nbv != NULL && fr->nbv->bUseGPU ?
+ nbnxn_cuda_get_timings(fr->nbv->cu_nbv) : NULL,
+ EI_DYNAMICS(inputrec->eI) && !MULTISIM(cr));
+
- if (!free_gpu(gpu_err_str))
- {
- gmx_warning("On node %d failed to free GPU #%d: %s",
- cr->nodeid, get_current_gpu_device_id(), gpu_err_str);
- }
- }
+
++ /* Free GPU memory and context */
++ free_gpu_resources(fplog, fr, cr);
+
+ if (opt2bSet("-membed", nfile, fnm))
+ {
+ sfree(membed);
+ }
+
+ gmx_hardware_info_free(hwinfo);
+
+ /* Does what it says */
+ print_date_and_time(fplog, cr->nodeid, "Finished mdrun", walltime_accounting);
+ walltime_accounting_destroy(walltime_accounting);
+
+ /* Close logfile already here if we were appending to it */
+ if (MASTER(cr) && (Flags & MD_APPENDFILES))
+ {
+ gmx_log_close(fplog);
+ }
+
+ rc = (int)gmx_get_stop_condition();
+
+#ifdef GMX_THREAD_MPI
+ /* we need to join all threads. The sub-threads join when they
+ exit this function, but the master thread needs to be told to
+ wait for that. */
+ if (PAR(cr) && MASTER(cr))
+ {
+ tMPI_Finalize();
+ }
+#endif
+
+ return rc;
+}
biod.pnpi.spb.ru / alexxy / gromacs.git / commitdiff