fix internal/external OpenMP thread affinity clash

[alexxy/gromacs.git] / src / gmxlib / gmx_omp.c

diff --git a/src/gmxlib/gmx_omp.c b/src/gmxlib/gmx_omp.c
index e5c1f4540b4c0ab2d330f7c362f53a6a938cfa75..29622e9a7e307a9a33490528ba145c33d0d0a531 100644 (file)
--- a/src/gmxlib/gmx_omp.c
+++ b/src/gmxlib/gmx_omp.c
@@ -30,6 +30,12 @@
 #include <omp.h>
 #endif
 
+#include <stdio.h>
+
+#include "md_logging.h"
+#include "gmx_fatal.h"
+#include "statutil.h"
+#include "string2.h"
 #include "gmx_omp.h"
 
 int gmx_omp_get_max_threads(void)
@@ -67,3 +73,107 @@ void gmx_omp_set_num_threads(int num_threads)
     return;
 #endif
 }
+
+/*!
+ * Thread affinity set by the OpenMP library can conflict with the GROMACS
+ * internal affinity setting.
+ *
+ * While GNU OpenMP does not set affinity by default, the Intel OpenMP library
+ * does. This conflicts with the internal affinity (especially thread-MPI)
+ * setting, results in incorrectly locked threads, and causes dreadful performance.
+ *
+ * The KMP_AFFINITY environment variable is used by Intel, GOMP_CPU_AFFINITY
+ * by the GNU compilers (Intel also honors it well). If any of the variables
+ * is set, we honor it, disable the internal pinning, and warn the user.
+ * When using Intel OpenMP, we will disable affinity if the user did not set it
+ * anually through one of the aforementioned environment variables.
+ *
+ * Note that the Intel OpenMP affinity disabling iwll only take effect if this
+ * function is called before the OpenMP library gets initialized which happens
+ * when the first call is made into a compilation unit that contains OpenMP
+ * pragmas.
+ */
+void gmx_omp_check_thread_affinity(FILE *fplog, const t_commrec *cr,
+                                   gmx_hw_opt_t *hw_opt)
+{
+    gmx_bool bKmpAffinitySet, bGompCpuAffinitySet;
+    char *kmp_env, *gomp_env;
+
+    /* no need to worry if internal thread pinning is turned off */
+    if (!hw_opt->bThreadPinning)
+    {
+        return;
+    }
+
+#if defined(GMX_OPENMP)
+
+    /* We assume that the affinity setting is available on all platforms
+     * gcc supports. Even if this is not the case (e.g. Mac OS) the user
+     * will only get a warning.*/
+    bGompCpuAffinitySet = FALSE;
+    gomp_env = NULL;
+#if defined(__GNUC__)
+    gomp_env = getenv("GOMP_CPU_AFFINITY");
+    bGompCpuAffinitySet = (gomp_env != NULL);
+#endif /* __GNUC__ */
+
+    bKmpAffinitySet = FALSE;
+#if defined(__INTEL_COMPILER)
+    kmp_env = getenv("KMP_AFFINITY");
+    bKmpAffinitySet = (kmp_env != NULL);
+
+    /* disable Intel OpenMP affinity if neither KMP_AFFINITY nor
+     * GOMP_CPU_AFFINITY is set (Intel uses the GNU env. var as well) */
+    if (!bKmpAffinitySet && !bGompCpuAffinitySet)
+    {
+        int retval;
+
+#ifdef _MSC_VER
+        /* Windows not POSIX */
+        retval = _putenv_s("KMP_AFFINITY", "disabled");
+#else
+        /* POSIX */
+        retval = setenv("KMP_AFFINITY", "disabled", 0);
+#endif /* _MSC_VER */
+
+        if (debug)
+        {
+            fprintf(debug, "Disabling Intel OpenMP affinity by setting the KMP_AFFINITY=disabled env. var.\n");
+        }
+
+        if (retval != 0)
+        {
+            gmx_warning("Disabling Intel OpenMp affinity setting failed!");
+        }
+    }
+
+    /* turn off internal pinning KMP_AFFINITY != "disabled" */
+    if (bKmpAffinitySet && (gmx_strncasecmp(kmp_env, "disabled", 8) != 0))
+    {
+        md_print_warn(cr, fplog, "WARNING: KMP_AFFINITY set, will turn off %s internal affinity\n"
+                      "         setting as the two can conflict and cause performance degradation.\n"
+                      "         To keep using the %s internal affinity setting, set the\n"
+                      "         KMP_AFFINITY=disabled environment variable.",
+                      ShortProgram(), ShortProgram());
+
+        hw_opt->bThreadPinning = FALSE;
+    }
+#endif /* __INTEL_COMPILER */
+
+#if defined(__INTEL_COMPILER) || defined(__GNUC__)
+    /* turn off internal pinning f GOMP_CPU_AFFINITY is set & non-empty */
+    if (bGompCpuAffinitySet && gomp_env != NULL && gomp_env != '\0')
+    {
+        md_print_warn(cr, fplog,
+                      "WARNING: GOMP_CPU_AFFINITY set, will turn off %s internal affinity\n"
+                      "         setting as the two can conflict and cause performance degradation.\n"
+                      "         To keep using the %s internal affinity setting, unset the\n"
+                      "         GOMP_CPU_AFFINITY environment variable.",
+                      ShortProgram(), ShortProgram());
+
+        hw_opt->bThreadPinning = FALSE;
+    }
+#endif /* __INTEL_COMPILER || __GNUC__ */
+
+#endif /* GMX_OPENMP */
+}