1 /* -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
4 * This file is part of GROMACS.
7 * Written by the Gromacs development team under coordination of
8 * David van der Spoel, Berk Hess, and Erik Lindahl.
10 * This library is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU Lesser General Public License
12 * as published by the Free Software Foundation; either version 2
13 * of the License, or (at your option) any later version.
15 * To help us fund GROMACS development, we humbly ask that you cite
16 * the research papers on the package. Check out http://www.gromacs.org
19 * GROup of MAchos and Cynical Suckers
29 #include "types/enums.h"
30 #include "types/hw_info.h"
31 #include "types/commrec.h"
32 #include "gmx_fatal.h"
33 #include "gmx_fatal_collective.h"
35 #include "gpu_utils.h"
37 #include "gmx_detect_hardware.h"
39 #include "md_logging.h"
41 #if ((defined(WIN32) || defined( _WIN32 ) || defined(WIN64) || defined( _WIN64 )) && !(defined (__CYGWIN__) || defined (__CYGWIN32__)))
45 /* Although we can't have more than 10 GPU different ID-s passed by the user as
46 * the id-s are assumed to be represented by single digits, as multiple
47 * processes can share a GPU, we can end up with more than 10 IDs.
48 * To account for potential extreme cases we'll set the limit to a pretty
49 * ridiculous number. */
50 static unsigned int max_gpu_ids_user = 64;
52 static const char* invalid_gpuid_hint =
53 "A delimiter-free sequence of valid numeric IDs of available GPUs is expected.";
56 void limit_num_gpus_used(gmx_hw_info_t *hwinfo, int count);
58 static void sprint_gpus(char *sbuf, const gmx_gpu_info_t *gpu_info, gmx_bool bPrintAll)
63 ndev = gpu_info->ncuda_dev;
66 for (i = 0; i < ndev; i++)
68 get_gpu_device_info_string(stmp, gpu_info, i);
78 static void print_gpu_detection_stats(FILE *fplog,
79 const gmx_gpu_info_t *gpu_info,
82 char onhost[266],stmp[STRLEN];
85 ngpu = gpu_info->ncuda_dev;
87 #if defined GMX_MPI && !defined GMX_THREAD_MPI
88 /* We only print the detection on one, of possibly multiple, nodes */
89 strncpy(onhost," on host ",10);
90 gmx_gethostname(onhost+9,256);
92 /* We detect all relevant GPUs */
98 sprint_gpus(stmp, gpu_info, TRUE);
99 md_print_warn(cr, fplog, "%d GPU%s detected%s:\n%s\n",
100 ngpu, (ngpu > 1) ? "s" : "", onhost, stmp);
104 md_print_warn(cr, fplog, "No GPUs detected%s\n", onhost);
108 static void print_gpu_use_stats(FILE *fplog,
109 const gmx_gpu_info_t *gpu_info,
112 char sbuf[STRLEN], stmp[STRLEN];
113 int i, ngpu, ngpu_all;
115 ngpu = gpu_info->ncuda_dev_use;
116 ngpu_all = gpu_info->ncuda_dev;
118 /* Issue note if GPUs are available but not used */
119 if (ngpu_all > 0 && ngpu < 1)
122 "%d compatible GPU%s detected in the system, but none will be used.\n"
123 "Consider trying GPU acceleration with the Verlet scheme!",
124 ngpu_all, (ngpu_all > 1) ? "s" : "");
128 sprintf(sbuf, "%d GPU%s %sselected for this run: ",
129 ngpu, (ngpu > 1) ? "s" : "",
130 gpu_info->bUserSet ? "user-" : "auto-");
131 for (i = 0; i < ngpu; i++)
133 sprintf(stmp, "#%d", get_gpu_device_id(gpu_info, i));
141 md_print_info(cr, fplog, "%s\n\n", sbuf);
144 /* Parse a "plain" GPU ID string which contains a sequence of digits corresponding
145 * to GPU IDs; the order will indicate the process/tMPI thread - GPU assignment. */
146 static void parse_gpu_id_plain_string(const char *idstr, int *nid, int *idlist)
151 len_idstr = strlen(idstr);
153 if (len_idstr > max_gpu_ids_user)
155 gmx_fatal(FARGS,"%d GPU IDs provided, but only at most %d are supported",
156 len_idstr, max_gpu_ids_user);
161 for (i = 0; i < *nid; i++)
163 if (idstr[i] < '0' || idstr[i] > '9')
165 gmx_fatal(FARGS, "Invalid character in GPU ID string: '%c'\n%s\n",
166 idstr[i], invalid_gpuid_hint);
168 idlist[i] = idstr[i] - '0';
172 static void parse_gpu_id_csv_string(const char *idstr, int *nid, int *idlist)
174 /* XXX implement cvs format to support more than 10 different GPUs in a box. */
175 gmx_incons("Not implemented yet");
178 void gmx_check_hw_runconf_consistency(FILE *fplog, gmx_hw_info_t *hwinfo,
179 const t_commrec *cr, int ntmpi_requested,
182 int npppn, ntmpi_pp, ngpu;
183 char sbuf[STRLEN], th_or_proc[STRLEN], th_or_proc_plural[STRLEN], pernode[STRLEN];
185 gmx_bool bGPUBin, btMPI, bMPI, bMaxMpiThreadsSet, bNthreadsAuto, bEmulateGPU;
190 btMPI = bMPI = FALSE;
191 bNthreadsAuto = FALSE;
192 #if defined(GMX_THREAD_MPI)
194 bNthreadsAuto = (ntmpi_requested < 1);
195 #elif defined(GMX_LIB_MPI)
205 /* GPU emulation detection is done later, but we need here as well
206 * -- uncool, but there's no elegant workaround */
207 bEmulateGPU = (getenv("GMX_EMULATE_GPU") != NULL);
208 bMaxMpiThreadsSet = (getenv("GMX_MAX_MPI_THREADS") != NULL);
212 /* check the acceleration mdrun is compiled with against hardware capabilities */
213 /* TODO: Here we assume homogeneous hardware which is not necessarily the case!
214 * Might not hurt to add an extra check over MPI. */
215 gmx_cpuid_acceleration_check(hwinfo->cpuid_info, fplog);
218 /* Below we only do consistency checks for PP and GPUs,
219 * this is irrelevant for PME only nodes, so in that case we return here.
221 if (!(cr->duty & DUTY_PP))
226 /* Need to ensure that we have enough GPUs:
227 * - need one GPU per PP node
228 * - no GPU oversubscription with tMPI
229 * => keep on the GPU support, otherwise turn off (or bail if forced)
231 /* number of PP processes per node */
232 npppn = cr->nrank_pp_intranode;
235 th_or_proc_plural[0] = '\0';
238 sprintf(th_or_proc, "thread-MPI thread");
241 sprintf(th_or_proc_plural, "s");
246 sprintf(th_or_proc, "MPI process");
249 sprintf(th_or_proc_plural, "es");
251 sprintf(pernode, " per node");
255 /* neither MPI nor tMPI */
256 sprintf(th_or_proc, "process");
261 print_gpu_detection_stats(fplog, &hwinfo->gpu_info, cr);
264 if (bUseGPU && hwinfo->bCanUseGPU && !bEmulateGPU)
266 ngpu = hwinfo->gpu_info.ncuda_dev_use;
267 sprintf(gpu_plural, "%s", (ngpu > 1) ? "s" : "");
269 /* number of tMPI threads atuo-adjusted */
270 if (btMPI && bNthreadsAuto && SIMMASTER(cr))
274 if (hwinfo->gpu_info.bUserSet)
276 /* The user manually provided more GPUs than threads we could
277 * automatically start. */
279 "%d GPU%s provided, but only %d PP thread-MPI thread%s coud be started.\n"
280 "%s requires one PP tread-MPI thread per GPU; use fewer GPUs%s.",
281 ngpu, gpu_plural, npppn, th_or_proc_plural,
282 ShortProgram(), bMaxMpiThreadsSet ? "\nor allow more threads to be used" : "");
286 /* There are more GPUs than tMPI threads; we have to limit the number GPUs used. */
287 md_print_warn(cr,fplog,
288 "NOTE: %d GPU%s were detected, but only %d PP thread-MPI thread%s can be started.\n"
289 " %s can use one GPU per PP tread-MPI thread, so only %d GPU%s will be used.%s\n",
290 ngpu, gpu_plural, npppn, th_or_proc_plural,
291 ShortProgram(), npppn, npppn > 1 ? "s" : "",
292 bMaxMpiThreadsSet ? "\n Also, you can allow more threads to be used by increasing GMX_MAX_MPI_THREADS" : "");
294 if (cr->rank_pp_intranode == 0)
296 limit_num_gpus_used(hwinfo, npppn);
297 ngpu = hwinfo->gpu_info.ncuda_dev_use;
298 sprintf(gpu_plural, "%s", (ngpu > 1) ? "s" : "");
306 if (hwinfo->gpu_info.bUserSet)
309 "Incorrect launch configuration: mismatching number of PP %s%s and GPUs%s.\n"
310 "%s was started with %d PP %s%s%s, but you provided %d GPU%s.",
311 th_or_proc, btMPI ? "s" : "es" , pernode,
312 ShortProgram(), npppn, th_or_proc, th_or_proc_plural, pernode, ngpu, gpu_plural);
318 md_print_warn(cr,fplog,
319 "NOTE: potentially sub-optimal launch configuration, %s started with less\n"
320 " PP %s%s%s than GPU%s available.\n"
321 " Each PP %s can use only one GPU, %d GPU%s%s will be used.\n",
323 th_or_proc, th_or_proc_plural, pernode, gpu_plural,
324 th_or_proc, npppn, gpu_plural, pernode);
326 if (bMPI || (btMPI && cr->rank_pp_intranode == 0))
328 limit_num_gpus_used(hwinfo, npppn);
329 ngpu = hwinfo->gpu_info.ncuda_dev_use;
330 sprintf(gpu_plural, "%s", (ngpu > 1) ? "s" : "");
335 /* Avoid duplicate error messages.
336 * Unfortunately we can only do this at the physical node
337 * level, since the hardware setup and MPI process count
338 * might be differ over physical nodes.
340 if (cr->rank_pp_intranode == 0)
343 "Incorrect launch configuration: mismatching number of PP %s%s and GPUs%s.\n"
344 "%s was started with %d PP %s%s%s, but only %d GPU%s were detected.",
345 th_or_proc, btMPI ? "s" : "es" , pernode,
346 ShortProgram(), npppn, th_or_proc, th_or_proc_plural, pernode, ngpu, gpu_plural);
351 /* Avoid other ranks to continue after inconsistency */
352 MPI_Barrier(cr->mpi_comm_mygroup);
359 hwinfo->gpu_info.bDevShare = FALSE;
360 if (hwinfo->gpu_info.bUserSet && (cr->rank_pp_intranode == 0))
362 int i, j, same_count;
363 gmx_bool bSomeSame, bAllDifferent;
365 same_count = 0; /* number of GPUs shared among ranks */
367 bAllDifferent = TRUE;
369 for (i = 0; i < ngpu - 1; i++)
371 for (j = i + 1; j < ngpu; j++)
373 bSomeSame |= hwinfo->gpu_info.cuda_dev_use[i] == hwinfo->gpu_info.cuda_dev_use[j];
374 bAllDifferent &= hwinfo->gpu_info.cuda_dev_use[i] != hwinfo->gpu_info.cuda_dev_use[j];
375 same_count += hwinfo->gpu_info.cuda_dev_use[i] == hwinfo->gpu_info.cuda_dev_use[j];
379 /* store the number of shared/oversubscribed GPUs */
380 hwinfo->gpu_info.bDevShare = bSomeSame;
382 if (btMPI && !bAllDifferent)
385 "Invalid GPU assignment: can't share a GPU among multiple thread-MPI threads.\n"
386 "Use MPI if you are sure that you want to assign GPU to multiple threads.");
391 md_print_warn(cr,fplog,
392 "NOTE: Potentially sub-optimal launch configuration: you assigned %s to\n"
393 " multiple %s%s; this should be avoided as it can cause\n"
394 " performance loss.\n",
395 same_count > 1 ? "GPUs" : "a GPU", th_or_proc, btMPI ? "s" : "es");
398 print_gpu_use_stats(fplog, &hwinfo->gpu_info, cr);
402 /* Return the number of hardware threads supported by the current CPU.
403 * We assume that this is equal with the number of CPUs reported to be
404 * online by the OS at the time of the call.
406 static int get_nthreads_hw_avail(FILE *fplog, const t_commrec *cr)
410 #if ((defined(WIN32) || defined( _WIN32 ) || defined(WIN64) || defined( _WIN64 )) && !(defined (__CYGWIN__) || defined (__CYGWIN32__)))
413 GetSystemInfo( &sysinfo );
414 ret = sysinfo.dwNumberOfProcessors;
415 #elif defined HAVE_SYSCONF
416 /* We are probably on Unix.
417 * Now check if we have the argument to use before executing the call
419 #if defined(_SC_NPROCESSORS_ONLN)
420 ret = sysconf(_SC_NPROCESSORS_ONLN);
421 #elif defined(_SC_NPROC_ONLN)
422 ret = sysconf(_SC_NPROC_ONLN);
423 #elif defined(_SC_NPROCESSORS_CONF)
424 ret = sysconf(_SC_NPROCESSORS_CONF);
425 #elif defined(_SC_NPROC_CONF)
426 ret = sysconf(_SC_NPROC_CONF);
427 #endif /* End of check for sysconf argument values */
430 /* Neither windows nor Unix. No fscking idea how many CPUs we have! */
436 fprintf(debug, "Detected %d processors, will use this as the number "
437 "of supported hardware threads.\n", ret);
441 if (ret != gmx_omp_get_num_procs())
443 md_print_warn(cr, fplog,
444 "Number of CPUs detected (%d) does not match the number reported by OpenMP (%d).\n"
445 "Consider setting the launch configuration manually!",
446 ret, gmx_omp_get_num_procs());
453 void gmx_detect_hardware(FILE *fplog, gmx_hw_info_t *hwinfo,
455 gmx_bool bForceUseGPU, gmx_bool bTryUseGPU,
460 char sbuf[STRLEN], stmp[STRLEN];
462 gmx_gpu_info_t gpuinfo_auto, gpuinfo_user;
467 /* detect CPUID info; no fuss, we don't detect system-wide
468 * -- sloppy, but that's it for now */
469 if (gmx_cpuid_init(&hwinfo->cpuid_info) != 0)
471 gmx_fatal_collective(FARGS, cr, NULL, "CPUID detection failed!");
474 /* detect number of hardware threads */
475 hwinfo->nthreads_hw_avail = get_nthreads_hw_avail(fplog, cr);
478 hwinfo->gpu_info.ncuda_dev_use = 0;
479 hwinfo->gpu_info.cuda_dev_use = NULL;
480 hwinfo->gpu_info.ncuda_dev = 0;
481 hwinfo->gpu_info.cuda_dev = NULL;
489 /* Bail if binary is not compiled with GPU acceleration, but this is either
490 * explicitly (-nb gpu) or implicitly (gpu ID passed) requested. */
491 if (bForceUseGPU && !bGPUBin)
493 gmx_fatal(FARGS, "GPU acceleration requested, but %s was compiled without GPU support!", ShortProgram());
495 if (gpu_id != NULL && !bGPUBin)
497 gmx_fatal(FARGS, "GPU ID string set, but %s was compiled without GPU support!", ShortProgram());
500 /* run the detection if the binary was compiled with GPU support */
501 if (bGPUBin && getenv("GMX_DISABLE_GPU_DETECTION")==NULL)
503 char detection_error[STRLEN];
505 if (detect_cuda_gpus(&hwinfo->gpu_info, detection_error) != 0)
507 if (detection_error != NULL && detection_error[0] != '\0')
509 sprintf(sbuf, ":\n %s\n", detection_error);
515 md_print_warn(cr, fplog,
516 "NOTE: Error occurred during GPU detection%s"
517 " Can not use GPU acceleration, will fall back to CPU kernels.\n",
522 if (bForceUseGPU || bTryUseGPU)
524 env = getenv("GMX_GPU_ID");
525 if (env != NULL && gpu_id != NULL)
527 gmx_fatal(FARGS,"GMX_GPU_ID and -gpu_id can not be used at the same time");
534 /* parse GPU IDs if the user passed any */
537 int *gpuid, *checkres;
540 snew(gpuid, max_gpu_ids_user);
541 snew(checkres, max_gpu_ids_user);
543 parse_gpu_id_plain_string(env, &nid, gpuid);
547 gmx_fatal(FARGS, "Empty GPU ID string encountered.\n%s\n", invalid_gpuid_hint);
550 res = check_select_cuda_gpus(checkres, &hwinfo->gpu_info, gpuid, nid);
554 print_gpu_detection_stats(fplog, &hwinfo->gpu_info, cr);
556 sprintf(sbuf, "Some of the requested GPUs do not exist, behave strangely, or are not compatible:\n");
557 for (i = 0; i < nid; i++)
559 if (checkres[i] != egpuCompatible)
561 sprintf(stmp, " GPU #%d: %s\n",
562 gpuid[i], gpu_detect_res_str[checkres[i]]);
566 gmx_fatal(FARGS, "%s", sbuf);
569 hwinfo->gpu_info.bUserSet = TRUE;
576 pick_compatible_gpus(&hwinfo->gpu_info);
577 hwinfo->gpu_info.bUserSet = FALSE;
580 /* decide whether we can use GPU */
581 hwinfo->bCanUseGPU = (hwinfo->gpu_info.ncuda_dev_use > 0);
582 if (!hwinfo->bCanUseGPU && bForceUseGPU)
584 gmx_fatal(FARGS, "GPU acceleration requested, but no compatible GPUs were detected.");
589 void limit_num_gpus_used(gmx_hw_info_t *hwinfo, int count)
595 ndev_use = hwinfo->gpu_info.ncuda_dev_use;
597 if (count > ndev_use)
599 /* won't increase the # of GPUs */
606 sprintf(sbuf, "Limiting the number of GPUs to <1 doesn't make sense (detected %d, %d requested)!",
611 /* TODO: improve this implementation: either sort GPUs or remove the weakest here */
612 hwinfo->gpu_info.ncuda_dev_use = count;
615 void gmx_hardware_info_free(gmx_hw_info_t *hwinfo)
619 gmx_cpuid_done(hwinfo->cpuid_info);
620 free_gpu_info(&hwinfo->gpu_info);