Introduced MPI initialization counter

Normal GROMACS code needs to initialize and finalize MPI. Code that
uses GROMACS as a client (e.g. external code, possible GROMACS test
code) needs GROMACS to work normally and not make erroneous calls to
the MPI library. For example, this would allow a single test binary
to call MPI-enabled mdrun repeatedly to test a range of test cases.

Change-Id: I546518923561a13e3973f90182fa31129ab1e6ae

diff --git a/src/gromacs/utility/init.cpp b/src/gromacs/utility/init.cpp
index 7aee502cf6cc6a17d92e721e202fca36841361df..973a73cd59bbe88ab10151129ccd6ddc889efc56 100644 (file)
--- a/src/gromacs/utility/init.cpp
+++ b/src/gromacs/utility/init.cpp
@@ -54,12 +54,20 @@
 #include "gromacs/legacyheaders/network.h"
 #include "gromacs/legacyheaders/smalloc.h"
 #include "gromacs/legacyheaders/types/commrec.h"
-
 #include "gromacs/utility/programinfo.h"
+#include "gromacs/utility/gmxassert.h"
 
 namespace gmx
 {
 
+namespace
+{
+#ifdef GMX_LIB_MPI
+    //! Maintains global counter of attempts to initialize MPI
+    int g_initializationCounter = 0;
+#endif
+}
+
 #ifdef GMX_LIB_MPI
 namespace
 {
@@ -88,17 +96,35 @@ void broadcastArguments(const t_commrec *cr, int *argc, char ***argv)
     }
 }
 
-} // namespace
+}   // namespace
 #endif
 
 ProgramInfo &init(const char *realBinaryName, int *argc, char ***argv)
 {
 #ifdef GMX_LIB_MPI
+    int isInitialized = 0, isFinalized = 0;
+    MPI_Finalized(&isFinalized);
+    GMX_RELEASE_ASSERT(!isFinalized, "Invalid attempt to initialize MPI after finalization");
+    MPI_Initialized(&isInitialized);
+    if (isInitialized)
+    {
+        if (0 == g_initializationCounter)
+        {
+            // Some other code has already initialized MPI, so bump the counter so that
+            // we know not to finalize MPI ourselves later.
+            g_initializationCounter++;
+        }
+    }
+    else
+    {
 #ifdef GMX_FAHCORE
-    (void) fah_MPI_Init(argc, argv);
+        (void) fah_MPI_Init(argc, argv);
 #else
-    (void) MPI_Init(argc, argv);
+        (void) MPI_Init(argc, argv);
 #endif
+    }
+    // Bump the counter to record this initialization event
+    g_initializationCounter++;
 
     // TODO: Rewrite this to not use t_commrec once there is clarity on
     // the approach for MPI in C++ code.
@@ -123,33 +149,24 @@ ProgramInfo &init(int *argc, char ***argv)
 
 void finalize()
 {
-#ifndef GMX_MPI
-    /* Compiled without MPI, no MPI finalizing needed */
-    return;
-#else
-    int finalized;
+#ifdef GMX_LIB_MPI
+    GMX_RELEASE_ASSERT(0 < g_initializationCounter, "Excess attempt to finalize MPI");
+    // Bump the counter to record this finalization event
+    g_initializationCounter--;
 
-    if (!gmx_mpi_initialized())
+    if (0 == g_initializationCounter)
     {
-        return;
+        /* We sync the processes here to try to avoid problems
+         * with buggy MPI implementations that could cause
+         * unfinished processes to terminate.
+         */
+        MPI_Barrier(MPI_COMM_WORLD);
+
+        /* Apparently certain mpich implementations cause problems
+         * with MPI_Finalize. In that case comment out MPI_Finalize.
+         */
+        MPI_Finalize();
     }
-    /* just as a check; we don't want to finalize twice */
-    MPI_Finalized(&finalized);
-    if (finalized)
-    {
-        return;
-    }
-
-    /* We sync the processes here to try to avoid problems
-     * with buggy MPI implementations that could cause
-     * unfinished processes to terminate.
-     */
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    /* Apparently certain mpich implementations cause problems
-     * with MPI_Finalize. In that case comment out MPI_Finalize.
-     */
-    MPI_Finalize();
 #endif
 }
 

diff --git a/src/gromacs/utility/init.h b/src/gromacs/utility/init.h
index 90d6de8d1d72aba72bd80dcfa8379d39da9feb96..09c7ebe29df3bd0d282d261fc2b57b9b7aaa4d38 100644 (file)
--- a/src/gromacs/utility/init.h
+++ b/src/gromacs/utility/init.h
@@ -36,6 +36,26 @@
  * \brief
  * Declares functions for initializing the GROMACS library.
  *
+ * Initialization for the GROMACS library.
+ *
+ * Currently, only MPI initialization/finalization management is
+ * required, and only if external MPI support is enabled.
+ *
+ * If MPI is already initialized, we should not call MPI_Init() or
+ * MPI_Finalize(). This management object permits GROMACS test code to
+ * nest calls to functions that might normally implement a stand-alone
+ * MPI-using tool. It also permits GROMACS code to be called from code
+ * that has already initialized MPI and needs that environment to work
+ * and persist after GROMACS code returns (e.g. GROMACS tests,
+ * external libraries that call GROMACS code).
+ *
+ * It does so by maintaining a counter of the number of MPI
+ * initializations, and only calling MPI_Init() or MPI_Finalize when
+ * it is safe (ie. when the counter is at zero).
+ *
+ * Thread-MPI initialization and finalization for mdrun is all managed
+ * in runner.c.
+ *
  * \author Teemu Murtola <teemu.murtola@gmail.com>
  * \inpublicapi
  * \ingroup module_utility
@@ -43,15 +63,13 @@
 #ifndef GMX_UTILITY_INIT_H
 #define GMX_UTILITY_INIT_H
 
-// Forward declaration is not sufficient for MSVC if the return value is
-// ignored...
+// Forward declaration of class ProgramInfo is not sufficient for MSVC if
+// the return value of init() is ignored(!)
 #include "programinfo.h"
 
 namespace gmx
 {
 
-class ProgramInfo;
-
 /*! \brief
  * Initializes the Gromacs library with explicit binary name.
  *
@@ -64,7 +82,14 @@ class ProgramInfo;
  * This overload is provided for cases where the program may be invoked
  * through a symlink, and it is necessary to know the real name of the
  * binary.
- * See init(int *, char ***) for more information on the general behavior.
+ *
+ * Currently, this is tailored for use in command-line/standalone applications.
+ * Some additional thought may be required to make it generally usable.
+ *
+ * The command line arguments are communicated so that they can be
+ * parsed on each processor.
+ * Arguments are the number of command line arguments, and a pointer to the
+ * array of argument strings. Both are allowed to be NULL.
  *
  * Does not throw. Terminates the program on out-of-memory error.
  */
@@ -76,25 +101,17 @@ ProgramInfo &init(const char *realBinaryName, int *argc, char ***argv);
  * \param[in] argv  argv array passed to main().
  * \returns   Reference to initialized program information object.
  *
- * Currently, this is tailored for use in command-line/standalone applications.
- * Some additional thought may be required to make it generally usable.
- * Always calls MPI_Init() if Gromacs is compiled with MPI support.
- *
- * The command line arguments are communicated so that they can be
- * parsed on each processor.
- * Arguments are the number of command line arguments, and a pointer to the
- * array of argument strings. Both are allowed to be NULL.
- *
  * Does not throw. Terminates the program on out-of-memory error.
  */
 ProgramInfo &init(int *argc, char ***argv);
 /*! \brief
  * Deinitializes the Gromacs library.
  *
- * Currently always calls MPI_Finalize() if Gromacs is compiled with MPI
- * support, unless MPI has already been finalized.
- * Thus, it is not possible to reinitialize Gromacs after calling this
- * function.
+ * Decrements the initialization counter, and calls MPI_Finalize()
+ * if Gromacs is compiled with MPI support and the counter has
+ * reached zero. In that case, it is not possible to reinitialize
+ * Gromacs after calling this function. Instead, call init at a
+ * higher level, and note that calls to init can be nested safely.
  */
 void finalize();