From: Szilard Pall Date: Wed, 31 Oct 2012 20:19:53 +0000 (+0100) Subject: detect externally set CPU affinity X-Git-Url: http://biod.pnpi.spb.ru/gitweb/?a=commitdiff_plain;h=a6e118187b6c84799224186265083575b572fe64;p=alexxy%2Fgromacs.git detect externally set CPU affinity Try to preserve externally set CPU affinity instead of silently overriding it. If CPU affinity is set externally, e.g. by taskset or some OpenMP library we don't check for, disable internal CPU affinity and warn the user. This only works on Linux as it uses the sched.h CPU set interface which is GNU feature. Change-Id: Ie5f292fb1f29e07a3275429688c5d79398564979 --- diff --git a/CMakeLists.txt b/CMakeLists.txt index 2a608f1f82..c413e515e9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -408,6 +408,7 @@ check_function_exists(lstat HAVE_LSTAT) check_function_exists(sigaction HAVE_SIGACTION) check_function_exists(sysconf HAVE_SYSCONF) check_function_exists(sched_setaffinity HAVE_SCHED_SETAFFINITY) +check_function_exists(sched_getaffinity HAVE_SCHED_GETAFFINITY) include(CheckLibraryExists) check_library_exists(m sqrt "" HAVE_LIBM) diff --git a/src/config.h.cmakein b/src/config.h.cmakein index 8277706b2c..ea0d65d47f 100644 --- a/src/config.h.cmakein +++ b/src/config.h.cmakein @@ -359,6 +359,9 @@ /* Define to 1 if you have the sysconf() function */ #cmakedefine HAVE_SYSCONF +/* Define to 1 if you have the sched_getaffinity() function */ +#cmakedefine HAVE_SCHED_GETAFFINITY + /* Define to 1 if you have the sched_setaffinity() function */ #cmakedefine HAVE_SCHED_SETAFFINITY diff --git a/src/kernel/runner.c b/src/kernel/runner.c index b7549d9a44..e4d2403dc9 100644 --- a/src/kernel/runner.c +++ b/src/kernel/runner.c @@ -36,7 +36,7 @@ #ifdef HAVE_CONFIG_H #include #endif -#ifdef __linux +#if defined(HAVE_SCHED_H) && (defined(HAVE_SCHED_GETAFFINITY) || defined(HAVE_SCHED_SETAFFINITY)) #define _GNU_SOURCE #include #include @@ -770,6 +770,89 @@ static void convert_to_verlet_scheme(FILE *fplog, gmx_mtop_remove_chargegroups(mtop); } +/* Check the process affinity mask and if it is found to be non-zero, + * will honor it and disable mdrun internal affinity setting. + * This function should be called first before the OpenMP library gets + * initialized with the last argument FALSE (which will detect affinity + * set by external tools like taskset), and later, after the OpenMP + * initialization, with the last argument TRUE to detect affinity changes + * made by the OpenMP library. + * + * Note that this will only work on Linux as we use a GNU feature. */ +static void check_cpu_affinity_set(FILE *fplog, const t_commrec *cr, + gmx_hw_opt_t *hw_opt, int ncpus, + gmx_bool bAfterOpenmpInit) +{ +#ifdef HAVE_SCHED_GETAFFINITY + cpu_set_t mask_current; + int i, ret, cpu_count, cpu_set; + gmx_bool bAllSet; + + assert(hw_opt); + if (!hw_opt->bThreadPinning) + { + /* internal affinity setting is off, don't bother checking process affinity */ + return; + } + + CPU_ZERO(&mask_current); + if ((ret = sched_getaffinity(0, sizeof(cpu_set_t), &mask_current)) != 0) + { + /* failed to query affinity mask, will just return */ + if (debug) + { + fprintf(debug, "Failed to query affinity mask (error %d)", ret); + } + return; + } + + /* Before proceeding with the actual check, make sure that the number of + * detected CPUs is >= the CPUs in the current set. */ + if (ncpus < CPU_COUNT(&mask_current)) + { + if (debug) + { + fprintf(debug, "%d CPUs detected, but %d was returned by CPU_COUNT", + ncpus, CPU_COUNT(&mask_current)); + } + return; + } + + bAllSet = TRUE; + for (i = 0; (i < ncpus && i < CPU_SETSIZE); i++) + { + bAllSet = bAllSet && (CPU_ISSET(i, &mask_current) != 0); + } + + if (!bAllSet) + { + if (!bAfterOpenmpInit) + { + md_print_warn(cr, fplog, + "Non-default process affinity set, disabling internal affinity"); + } + else + { + md_print_warn(cr, fplog, + "Non-default process affinity set probably by the OpenMP library, " + "disabling internal affinity"); + } + hw_opt->bThreadPinning = FALSE; + + if (debug) + { + fprintf(debug, "Non-default affinity mask found\n"); + } + } + else + { + if (debug) + { + fprintf(debug, "Default affinity mask found\n"); + } + } +#endif /* HAVE_SCHED_GETAFFINITY */ +} /* Set CPU affinity. Can be important for performance. On some systems (e.g. Cray) CPU Affinity is set by default. @@ -798,7 +881,7 @@ static void set_cpu_affinity(FILE *fplog, #endif #ifdef GMX_OPENMP /* TODO: actually we could do this even without OpenMP?! */ -#ifdef __linux /* TODO: only linux? why not everywhere if sched_setaffinity is available */ +#ifdef HAVE_SCHED_SETAFFINITY if (hw_opt->bThreadPinning) { int thread, nthread_local, nthread_node, nthread_hw_max, nphyscore; @@ -914,7 +997,7 @@ static void set_cpu_affinity(FILE *fplog, sched_setaffinity((pid_t) syscall (SYS_gettid), sizeof(cpu_set_t), &mask); } } -#endif /* __linux */ +#endif /* HAVE_SCHED_SETAFFINITY */ #endif /* GMX_OPENMP */ } @@ -1172,6 +1255,12 @@ int mdrunner(gmx_hw_opt_t *hw_opt, cr->npmenodes = 0; } + /* Check for externally set OpenMP affinity and turn off internal + * pinning if any is found. We need to do this check early to tell + * thread-MPI whether it should do pinning when spawning threads. + */ + gmx_omp_check_thread_affinity(fplog, cr, hw_opt); + #ifdef GMX_THREAD_MPI /* With thread-MPI inputrec is only set here on the master thread */ if (SIMMASTER(cr)) @@ -1179,6 +1268,16 @@ int mdrunner(gmx_hw_opt_t *hw_opt, { check_and_update_hw_opt(hw_opt,minf.cutoff_scheme); +#ifdef GMX_THREAD_MPI + /* Early check for externally set process affinity. Can't do over all + * MPI processes because hwinfo is not available everywhere, but with + * thread-MPI it's needed as pinning might get turned off which needs + * to be known before starting thread-MPI. */ + check_cpu_affinity_set(fplog, + NULL, + hw_opt, hwinfo->nthreads_hw_avail, FALSE); +#endif + #ifdef GMX_THREAD_MPI if (cr->npmenodes > 0 && hw_opt->nthreads_tmpi <= 0) { @@ -1193,12 +1292,6 @@ int mdrunner(gmx_hw_opt_t *hw_opt, } } - /* Check for externally set OpenMP affinity and turn off internal - * pinning if any is found. We need to do this check early to tell - * thread-MPI whether it should do pinning when spawning threads. - */ - gmx_omp_check_thread_affinity(fplog, cr, hw_opt); - #ifdef GMX_THREAD_MPI if (SIMMASTER(cr)) { @@ -1286,6 +1379,10 @@ int mdrunner(gmx_hw_opt_t *hw_opt, gmx_detect_hardware(fplog, hwinfo, cr, bForceUseGPU, bTryUseGPU, hw_opt->gpu_id); } + + /* Now do the affinity check with MPI/no-MPI (done earlier with thread-MPI). */ + check_cpu_affinity_set(fplog, cr, + hw_opt, hwinfo->nthreads_hw_avail, FALSE); #endif /* now make sure the state is initialized and propagated */ @@ -1645,6 +1742,11 @@ int mdrunner(gmx_hw_opt_t *hw_opt, snew(pmedata,1); } + /* Before setting affinity, check whether the affinity has changed + * - which indicates that probably the OpenMP library has changed it since + * we first checked). */ + check_cpu_affinity_set(fplog, cr, hw_opt, hwinfo->nthreads_hw_avail, TRUE); + /* Set the CPU affinity */ set_cpu_affinity(fplog,cr,hw_opt,nthreads_pme,hwinfo,inputrec);