/*
* This file is part of the GROMACS molecular simulation package.
*
- * Copyright (c) 2012,2013, by the GROMACS development team, led by
+ * Copyright (c) 2012,2013,2014, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
* To help us fund GROMACS development, we humbly ask that you cite
* the research papers on the package. Check out http://www.gromacs.org.
*/
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-#if defined(HAVE_SCHED_H) && defined(HAVE_SCHED_GETAFFINITY)
-#define _GNU_SOURCE
-#include <sched.h>
-#include <sys/syscall.h>
-#endif
-#include <string.h>
-#include <errno.h>
+#include "gmxpre.h"
+
+#include "gromacs/legacyheaders/gmx_thread_affinity.h"
+
+#include "config.h"
+
#include <assert.h>
+#include <errno.h>
#include <stdio.h>
+#include <string.h>
-#include "typedefs.h"
-#include "types/commrec.h"
-#include "types/hw_info.h"
-#include "copyrite.h"
-#include "gmx_cpuid.h"
-#include "gmx_omp_nthreads.h"
-#include "md_logging.h"
-#include "gmx_thread_affinity.h"
+#ifdef HAVE_SCHED_AFFINITY
+# include <sched.h>
+# include <sys/syscall.h>
+#endif
#include "thread_mpi/threads.h"
+
+#include "gromacs/legacyheaders/copyrite.h"
+#include "gromacs/legacyheaders/gmx_cpuid.h"
+#include "gromacs/legacyheaders/gmx_omp_nthreads.h"
+#include "gromacs/legacyheaders/md_logging.h"
+#include "gromacs/legacyheaders/typedefs.h"
+#include "gromacs/legacyheaders/types/commrec.h"
+#include "gromacs/legacyheaders/types/hw_info.h"
+#include "gromacs/utility/basenetwork.h"
+#include "gromacs/utility/cstringutil.h"
+#include "gromacs/utility/fatalerror.h"
#include "gromacs/utility/gmxomp.h"
+#include "gromacs/utility/smalloc.h"
static int
get_thread_affinity_layout(FILE *fplog,
*/
MPI_Comm comm_intra;
- MPI_Comm_split(MPI_COMM_WORLD, gmx_hostname_num(), cr->rank_intranode,
+ MPI_Comm_split(MPI_COMM_WORLD,
+ gmx_physicalnode_id_hash(), cr->rank_intranode,
&comm_intra);
MPI_Scan(&nthread_local, &thread0_id_node, 1, MPI_INT, MPI_SUM, comm_intra);
/* MPI_Scan is inclusive, but here we need exclusive */
* Note that this will only work on Linux as we use a GNU feature.
*/
void
-gmx_check_thread_affinity_set(FILE gmx_unused *fplog,
- const t_commrec gmx_unused *cr,
- gmx_hw_opt_t gmx_unused *hw_opt,
- int gmx_unused ncpus,
- gmx_bool gmx_unused bAfterOpenmpInit)
+gmx_check_thread_affinity_set(FILE *fplog,
+ const t_commrec *cr,
+ gmx_hw_opt_t *hw_opt,
+ int gmx_unused nthreads_hw_avail,
+ gmx_bool bAfterOpenmpInit)
{
-#ifdef HAVE_SCHED_GETAFFINITY
+#ifdef HAVE_SCHED_AFFINITY
cpu_set_t mask_current;
int i, ret, cpu_count, cpu_set;
gmx_bool bAllSet;
+#endif
assert(hw_opt);
+ if (!bAfterOpenmpInit)
+ {
+ /* Check for externally set OpenMP affinity and turn off internal
+ * pinning if any is found. We need to do this check early to tell
+ * thread-MPI whether it should do pinning when spawning threads.
+ * TODO: the above no longer holds, we should move these checks later
+ */
+ if (hw_opt->thread_affinity != threadaffOFF)
+ {
+ char *message;
+ if (!gmx_omp_check_thread_affinity(&message))
+ {
+ /* TODO: with -pin auto we should only warn when using all cores */
+ md_print_warn(cr, fplog, "%s", message);
+ sfree(message);
+ hw_opt->thread_affinity = threadaffOFF;
+ }
+ }
+
+ /* With thread-MPI this is needed as pinning might get turned off,
+ * which needs to be known before starting thread-MPI.
+ * With thread-MPI hw_opt is processed here on the master rank
+ * and passed to the other ranks later, so we only do this on master.
+ */
+ if (!SIMMASTER(cr))
+ {
+ return;
+ }
+#ifndef GMX_THREAD_MPI
+ return;
+#endif
+ }
+
+#ifdef HAVE_SCHED_GETAFFINITY
if (hw_opt->thread_affinity == threadaffOFF)
{
/* internal affinity setting is off, don't bother checking process affinity */
* detected CPUs is >= the CPUs in the current set.
* We need to check for CPU_COUNT as it was added only in glibc 2.6. */
#ifdef CPU_COUNT
- if (ncpus < CPU_COUNT(&mask_current))
+ if (nthreads_hw_avail < CPU_COUNT(&mask_current))
{
if (debug)
{
- fprintf(debug, "%d CPUs detected, but %d was returned by CPU_COUNT",
- ncpus, CPU_COUNT(&mask_current));
+ fprintf(debug, "%d hardware threads detected, but %d was returned by CPU_COUNT",
+ nthreads_hw_avail, CPU_COUNT(&mask_current));
}
return;
}
#endif /* CPU_COUNT */
bAllSet = TRUE;
- for (i = 0; (i < ncpus && i < CPU_SETSIZE); i++)
+ for (i = 0; (i < nthreads_hw_avail && i < CPU_SETSIZE); i++)
{
bAllSet = bAllSet && (CPU_ISSET(i, &mask_current) != 0);
}
fprintf(debug, "Default affinity mask found\n");
}
}
-#endif /* HAVE_SCHED_GETAFFINITY */
+#endif /* HAVE_SCHED_AFFINITY */
}
biod.pnpi.spb.ru / alexxy / gromacs.git / blobdiff