+void dd_setup_dlb_resource_sharing(t_commrec *cr,
+ const gmx_hw_info_t *hwinfo,
+ const gmx_hw_opt_t *hw_opt)
+{
+#ifdef GMX_MPI
+ int physicalnode_id_hash;
+ int gpu_id;
+ gmx_domdec_t *dd;
+ MPI_Comm mpi_comm_pp_physicalnode;
+
+ if (!(cr->duty & DUTY_PP) ||
+ hw_opt->gpu_opt.ncuda_dev_use == 0)
+ {
+ /* Only PP nodes (currently) use GPUs.
+ * If we don't have GPUs, there are no resources to share.
+ */
+ return;
+ }
+
+ physicalnode_id_hash = gmx_physicalnode_id_hash();
+
+ gpu_id = get_gpu_device_id(&hwinfo->gpu_info, &hw_opt->gpu_opt, cr->nodeid);
+
+ dd = cr->dd;
+
+ if (debug)
+ {
+ fprintf(debug, "dd_setup_dd_dlb_gpu_sharing:\n");
+ fprintf(debug, "DD PP rank %d physical node hash %d gpu_id %d\n",
+ dd->rank, physicalnode_id_hash, gpu_id);
+ }
+ /* Split the PP communicator over the physical nodes */
+ /* TODO: See if we should store this (before), as it's also used for
+ * for the nodecomm summution.
+ */
+ MPI_Comm_split(dd->mpi_comm_all, physicalnode_id_hash, dd->rank,
+ &mpi_comm_pp_physicalnode);
+ MPI_Comm_split(mpi_comm_pp_physicalnode, gpu_id, dd->rank,
+ &dd->comm->mpi_comm_gpu_shared);
+ MPI_Comm_free(&mpi_comm_pp_physicalnode);
+ MPI_Comm_size(dd->comm->mpi_comm_gpu_shared, &dd->comm->nrank_gpu_shared);
+
+ if (debug)
+ {
+ fprintf(debug, "nrank_gpu_shared %d\n", dd->comm->nrank_gpu_shared);
+ }
+
+ /* Note that some ranks could share a GPU, while others don't */
+
+ if (dd->comm->nrank_gpu_shared == 1)
+ {
+ MPI_Comm_free(&dd->comm->mpi_comm_gpu_shared);
+ }
+#endif
+}
+