Merge branch 'release-4-5-patches' into release-4-6

[alexxy/gromacs.git] / src / mdlib / domdec.c

diff --git a/src/mdlib/domdec.c b/src/mdlib/domdec.c
index 285e66b42e46787a44f95a7e0a19e34194e9c33f..472778af04349f2ac47f6e16b70e96bbbc5df8b5 100644 (file)
--- a/src/mdlib/domdec.c
+++ b/src/mdlib/domdec.c
@@ -41,6 +41,7 @@
 #include "force.h"
 #include "pme.h"
 #include "pull.h"
+#include "pull_rotation.h"
 #include "gmx_wallcycle.h"
 #include "mdrun.h"
 #include "nsgrid.h"
@@ -53,7 +54,7 @@
 #ifdef GMX_LIB_MPI
 #include <mpi.h>
 #endif
-#ifdef GMX_THREADS
+#ifdef GMX_THREAD_MPI
 #include "tmpi.h"
 #endif
 
@@ -1453,7 +1454,10 @@ void dd_collect_state(gmx_domdec_t *dd,
 
     if (DDMASTER(dd))
     {
-        state->lambda = state_local->lambda;
+        for (i=0;i<efptNR;i++) {
+            state->lambda[i] = state_local->lambda[i];
+        }
+        state->fep_state = state_local->fep_state;
         state->veta = state_local->veta;
         state->vol0 = state_local->vol0;
         copy_mat(state_local->box,state->box);
@@ -1481,7 +1485,7 @@ void dd_collect_state(gmx_domdec_t *dd,
     }
     for(est=0; est<estNR; est++)
     {
-        if (EST_DISTR(est) && state_local->flags & (1<<est))
+        if (EST_DISTR(est) && (state_local->flags & (1<<est)))
         {
             switch (est) {
             case estX:
@@ -1563,7 +1567,7 @@ static void dd_realloc_state(t_state *state,rvec **f,int nalloc)
     
     for(est=0; est<estNR; est++)
     {
-        if (EST_DISTR(est) && state->flags & (1<<est))
+        if (EST_DISTR(est) && (state->flags & (1<<est)))
         {
             switch(est) {
             case estX:
@@ -1706,13 +1710,17 @@ static void dd_distribute_state(gmx_domdec_t *dd,t_block *cgs,
                                 t_state *state,t_state *state_local,
                                 rvec **f)
 {
-    int  i,j,ngtch,ngtcp,nh;
+    int  i,j,nh;
 
     nh = state->nhchainlength;
 
     if (DDMASTER(dd))
     {
-        state_local->lambda = state->lambda;
+        for(i=0;i<efptNR;i++)
+        {
+            state_local->lambda[i] = state->lambda[i];
+        }
+        state_local->fep_state = state->fep_state;
         state_local->veta   = state->veta;
         state_local->vol0   = state->vol0;
         copy_mat(state->box,state_local->box);
@@ -1736,7 +1744,8 @@ static void dd_distribute_state(gmx_domdec_t *dd,t_block *cgs,
             }
         }
     }
-    dd_bcast(dd,sizeof(real),&state_local->lambda);
+    dd_bcast(dd,((efptNR)*sizeof(real)),state_local->lambda);
+    dd_bcast(dd,sizeof(int),&state_local->fep_state);
     dd_bcast(dd,sizeof(real),&state_local->veta);
     dd_bcast(dd,sizeof(real),&state_local->vol0);
     dd_bcast(dd,sizeof(state_local->box),state_local->box);
@@ -1756,7 +1765,7 @@ static void dd_distribute_state(gmx_domdec_t *dd,t_block *cgs,
     }
     for(i=0; i<estNR; i++)
     {
-        if (EST_DISTR(i) && state_local->flags & (1<<i))
+        if (EST_DISTR(i) && (state_local->flags & (1<<i)))
         {
             switch (i) {
             case estX:
@@ -1857,7 +1866,7 @@ static void write_dd_grid_pdb(const char *fn,gmx_large_int_t step,
                 }
                 else
                 {
-                    if (dd->nc[d] > 1 && d < ddbox->npbcdim)
+                    if (d < ddbox->npbcdim && dd->nc[d] > 1)
                     {
                         tric[d][i] = box[i][d]/box[i][i];
                     }
@@ -3813,8 +3822,8 @@ static void get_cg_distribution(FILE *fplog,gmx_large_int_t step,gmx_domdec_t *d
     ivec npulse;
     int  i,cg_gl;
     int  *ibuf,buf2[2] = { 0, 0 };
-    
-    if (DDMASTER(dd))
+    gmx_bool bMaster = DDMASTER(dd);
+    if (bMaster)
     {
         ma = dd->ma;
         
@@ -3849,7 +3858,7 @@ static void get_cg_distribution(FILE *fplog,gmx_large_int_t step,gmx_domdec_t *d
         srenew(dd->index_gl,dd->cg_nalloc);
         srenew(dd->cgindex,dd->cg_nalloc+1);
     }
-    if (DDMASTER(dd))
+    if (bMaster)
     {
         for(i=0; i<dd->nnodes; i++)
         {
@@ -4132,7 +4141,7 @@ static void rotate_state_atom(t_state *state,int a)
 
     for(est=0; est<estNR; est++)
     {
-        if (EST_DISTR(est) && state->flags & (1<<est)) {
+        if (EST_DISTR(est) && (state->flags & (1<<est))) {
             switch (est) {
             case estX:
                 /* Rotate the complete state; for a rectangular box only */
@@ -5198,34 +5207,30 @@ static void dd_print_load_verbose(gmx_domdec_t *dd)
 }
 
 #ifdef GMX_MPI
-static void make_load_communicator(gmx_domdec_t *dd,MPI_Group g_all,
-                                   int dim_ind,ivec loc)
+static void make_load_communicator(gmx_domdec_t *dd, int dim_ind,ivec loc)
 {
-    MPI_Group g_row;
     MPI_Comm  c_row;
-    int  dim,i,*rank;
+    int  dim, i, rank;
     ivec loc_c;
     gmx_domdec_root_t *root;
+    gmx_bool bPartOfGroup = FALSE;
     
     dim = dd->dim[dim_ind];
     copy_ivec(loc,loc_c);
-    snew(rank,dd->nc[dim]);
     for(i=0; i<dd->nc[dim]; i++)
     {
         loc_c[dim] = i;
-        rank[i] = dd_index(dd->nc,loc_c);
+        rank = dd_index(dd->nc,loc_c);
+        if (rank == dd->rank)
+        {
+            /* This process is part of the group */
+            bPartOfGroup = TRUE;
+        }
     }
-    /* Here we create a new group, that does not necessarily
-     * include our process. But MPI_Comm_create needs to be
-     * called by all the processes in the original communicator.
-     * Calling MPI_Group_free afterwards gives errors, so I assume
-     * also the group is needed by all processes. (B. Hess)
-     */
-    MPI_Group_incl(g_all,dd->nc[dim],rank,&g_row);
-    MPI_Comm_create(dd->mpi_comm_all,g_row,&c_row);
-    if (c_row != MPI_COMM_NULL)
+    MPI_Comm_split(dd->mpi_comm_all, bPartOfGroup?0:MPI_UNDEFINED, dd->rank,
+                   &c_row);
+    if (bPartOfGroup)
     {
-        /* This process is part of the group */
         dd->comm->mpi_comm_load[dim_ind] = c_row;
         if (dd->comm->eDLB != edlbNO)
         {
@@ -5257,32 +5262,28 @@ static void make_load_communicator(gmx_domdec_t *dd,MPI_Group g_all,
             snew(dd->comm->load[dim_ind].load,dd->nc[dim]*DD_NLOAD_MAX);
         }
     }
-    sfree(rank);
 }
 #endif
 
 static void make_load_communicators(gmx_domdec_t *dd)
 {
 #ifdef GMX_MPI
-  MPI_Group g_all;
   int  dim0,dim1,i,j;
   ivec loc;
 
   if (debug)
     fprintf(debug,"Making load communicators\n");
 
-  MPI_Comm_group(dd->mpi_comm_all,&g_all);
-  
   snew(dd->comm->load,dd->ndim);
   snew(dd->comm->mpi_comm_load,dd->ndim);
   
   clear_ivec(loc);
-  make_load_communicator(dd,g_all,0,loc);
+  make_load_communicator(dd,0,loc);
   if (dd->ndim > 1) {
     dim0 = dd->dim[0];
     for(i=0; i<dd->nc[dim0]; i++) {
       loc[dim0] = i;
-      make_load_communicator(dd,g_all,1,loc);
+      make_load_communicator(dd,1,loc);
     }
   }
   if (dd->ndim > 2) {
@@ -5292,13 +5293,11 @@ static void make_load_communicators(gmx_domdec_t *dd)
       dim1 = dd->dim[1];
       for(j=0; j<dd->nc[dim1]; j++) {
          loc[dim1] = j;
-         make_load_communicator(dd,g_all,2,loc);
+         make_load_communicator(dd,2,loc);
       }
     }
   }
 
-  MPI_Group_free(&g_all);
-
   if (debug)
     fprintf(debug,"Finished making load communicators\n");
 #endif
@@ -7991,7 +7990,7 @@ static void dd_sort_state(gmx_domdec_t *dd,int ePBC,
     /* Reorder the state */
     for(i=0; i<estNR; i++)
     {
-        if (EST_DISTR(i) && state->flags & (1<<i))
+        if (EST_DISTR(i) && (state->flags & (1<<i)))
         {
             switch (i)
             {
@@ -8600,6 +8599,13 @@ void dd_partition_system(FILE            *fplog,
         /* Update the local pull groups */
         dd_make_local_pull_groups(dd,ir->pull,mdatoms);
     }
+    
+    if (ir->bRot)
+    {
+        /* Update the local rotation groups */
+        dd_make_local_rotation_groups(dd,ir->rot);
+    }
+
 
     add_dd_statistics(dd);