group|comm: add initial support for group sentinel values

This commit modifies ompi's process list group object to support a
sentinel value for non-existant ompi_proc_t objects. The sentinel was
chosen to be the negative of the opal_process_name_t of the associated
ompi_proc_t. This takes advantage of the fact that on most (all?)
systems the top bit of a user-space pointer is never set. If this
changes then a new sentinel will be needed.

In addition this commit modifies the way ompi_mpi_comm_world is
initialized to fill in the group with sentinel values if the number of
processes exceeds the new add_procs behavior cutoff.

Signed-off-by: Nathan Hjelm <[email protected]>

Author: hjelmn

ompi/communicator/comm_cid.c

@@ -139,54 +139,8 @@ static opal_mutex_t ompi_cid_lock;
 static opal_list_t ompi_registered_comms;
 
 
-/* This variable is zero (false) if all processes in MPI_COMM_WORLD
- * did not require MPI_THREAD_MULTIPLE support, and is 1 (true) as
- * soon as at least one process requested support for THREAD_MULTIPLE */
-static int ompi_comm_world_thread_level_mult=0;
-
-
 int ompi_comm_cid_init (void)
 {
-#if OMPI_ENABLE_THREAD_MULTIPLE
-    ompi_proc_t **procs, *thisproc;
-    uint8_t thread_level;
-    uint8_t *tlpointer;
-    int ret;
-    size_t i, size, numprocs;
-
-    /** Note that the following call only returns processes
-     * with the same jobid. This is on purpose, since
-     * we switch for the dynamic communicators anyway
-     * to the original (slower) cid allocation algorithm.
-     */
-    procs = ompi_proc_world ( &numprocs );
-
-    for ( i=0; i<numprocs; i++ ) {
-        thisproc = procs[i];
-
-        OPAL_MODEX_RECV_STRING(ret, "MPI_THREAD_LEVEL",
-                               &thisproc->super.proc_name,
-                               (uint8_t**)&tlpointer, &size);
-        if (OMPI_SUCCESS == ret) {
-            thread_level = *((uint8_t *) tlpointer);
-            if ( OMPI_THREADLEVEL_IS_MULTIPLE (thread_level) ) {
-                ompi_comm_world_thread_level_mult = 1;
-                break;
-            }
-        } else if (OMPI_ERR_NOT_IMPLEMENTED == ret) {
-            if (ompi_mpi_thread_multiple) {
-                ompi_comm_world_thread_level_mult = 1;
-            }
-            break;
-        } else {
-            return ret;
-        }
-    }
-    free(procs);
-#else
-    ompi_comm_world_thread_level_mult = 0; // silence compiler warning if not used
-#endif
-
     return OMPI_SUCCESS;
 }
 

ompi/communicator/comm_init.c

@@ -13,7 +13,7 @@
  * Copyright (c) 2006-2010 University of Houston. All rights reserved.
  * Copyright (c) 2007-2012 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2009      Sun Microsystems, Inc. All rights reserved.
- * Copyright (c) 2012-2014 Los Alamos National Security, LLC.
+ * Copyright (c) 2012-2015 Los Alamos National Security, LLC.
  *                         All rights reserved.
  * Copyright (c) 2011-2013 Inria.  All rights reserved.
  * Copyright (c) 2011-2013 Universite Bordeaux 1
@@ -102,12 +102,25 @@ int ompi_comm_init(void)
     OBJ_CONSTRUCT(&ompi_mpi_comm_world, ompi_communicator_t);
     assert(ompi_mpi_comm_world.comm.c_f_to_c_index == 0);
     group = OBJ_NEW(ompi_group_t);
-    group->grp_proc_pointers = ompi_proc_world(&size);
-    group->grp_proc_count    = (int)size;
+
+    size = ompi_process_info.num_procs;
+    group->grp_proc_pointers = (ompi_proc_t **) calloc (size, sizeof (ompi_proc_t *));
+    group->grp_proc_count = size;
+
+    for (size_t i = 0 ; i < size ; ++i) {
+        opal_process_name_t name = {.vpid = i, .jobid = OMPI_PROC_MY_NAME->jobid};
+        group->grp_proc_pointers[i] = (ompi_proc_t *) ompi_proc_lookup (name);
+        if (NULL == group->grp_proc_pointers[i]) {
+            /* set sentinel value */
+            group->grp_proc_pointers[i] = (ompi_proc_t *)(-*((intptr_t *) &name));
+        } else {
+            OBJ_RETAIN (ompi_proc_local_proc);
+        }
+    }
+
     OMPI_GROUP_SET_INTRINSIC (group);
     OMPI_GROUP_SET_DENSE (group);
     ompi_set_group_rank(group, ompi_proc_local());
-    ompi_group_increment_proc_count (group);
 
     ompi_mpi_comm_world.comm.c_contextid    = 0;
     ompi_mpi_comm_world.comm.c_id_start_index = 4;

ompi/group/group.c

@@ -568,3 +568,23 @@ int ompi_group_compare(ompi_group_t *group1,
 
     return return_value;
 }
+
+bool ompi_group_have_remote_peers (ompi_group_t *group)
+{
+    for (size_t i = 0 ; i < group->grp_proc_count ; ++i) {
+        ompi_proc_t *proc = NULL;
+#if OMPI_GROUP_SPARSE
+        proc = ompi_group_peer_lookup (group, i);
+#else
+        if ((intptr_t) group->grp_proc_pointers[i] < 0) {
+            return true;
+        }
+        proc = group->grp_proc_pointers[i];
+#endif
+        if (!OPAL_PROC_ON_LOCAL_NODE(proc->super.proc_flags)) {
+            return true;
+        }
+    }
+
+    return false;
+}

ompi/group/group.h

@@ -14,7 +14,7 @@
  * Copyright (c) 2007-2012 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2009      Sun Microsystems, Inc. All rights reserved.
  * Copyright (c) 2012      Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ * Copyright (c) 2013-2015 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * $COPYRIGHT$
  *
@@ -339,10 +339,38 @@ static inline struct ompi_proc_t* ompi_group_peer_lookup(ompi_group_t *group, in
 #if OMPI_GROUP_SPARSE
     return ompi_group_get_proc_ptr (group, peer_id);
 #else
+    if (OPAL_UNLIKELY((intptr_t) group->grp_proc_pointers[peer_id] < 0)) {
+        intptr_t sentinel = -(intptr_t) group->grp_proc_pointers[peer_id];
+        /* replace sentinel value with an actual ompi_proc_t */
+        group->grp_proc_pointers[peer_id] =
+            (ompi_proc_t *) ompi_proc_for_name (*((opal_process_name_t *) &sentinel));
+        OBJ_RETAIN(group->grp_proc_pointers[peer_id]);
+    }
     return group->grp_proc_pointers[peer_id];
 #endif
 }
 
+static inline struct ompi_proc_t *ompi_group_peer_lookup_existing (ompi_group_t *group, int peer_id)
+{
+#if OPAL_ENABLE_DEBUG
+    if (peer_id >= group->grp_proc_count) {
+        opal_output(0, "ompi_group_peer_lookup_existing: invalid peer index (%d)", peer_id);
+        return (struct ompi_proc_t *) NULL;
+    }
+#endif
+#if OMPI_GROUP_SPARSE
+    return ompi_group_get_proc_ptr (group, peer_id);
+#else
+    if (OPAL_UNLIKELY((intptr_t) group->grp_proc_pointers[peer_id] < 0)) {
+        return NULL;
+    }
+
+    return group->grp_proc_pointers[peer_id];
+#endif
+}
+
+bool ompi_group_have_remote_peers (ompi_group_t *group);
+
 /**
  *  Function to print the group info
  */

ompi/group/group_init.c

@@ -210,14 +210,13 @@ ompi_group_t *ompi_group_allocate_bmap(int orig_group_size , int group_size)
  */
 void ompi_group_increment_proc_count(ompi_group_t *group)
 {
-    int proc;
     ompi_proc_t * proc_pointer;
-    for (proc = 0; proc < group->grp_proc_count; proc++) {
-        proc_pointer = ompi_group_peer_lookup(group,proc);
-        OBJ_RETAIN(proc_pointer);
+    for (int proc = 0 ; proc < group->grp_proc_count ; ++proc) {
+	proc_pointer = ompi_group_peer_lookup_existing (group, proc);
+	if (proc_pointer) {
+	    OBJ_RETAIN(proc_pointer);
+	}
     }
-
-    return;
 }
 
 /*
@@ -226,14 +225,13 @@ void ompi_group_increment_proc_count(ompi_group_t *group)
 
 void ompi_group_decrement_proc_count(ompi_group_t *group)
 {
-    int proc;
     ompi_proc_t * proc_pointer;
-    for (proc = 0; proc < group->grp_proc_count; proc++) {
-        proc_pointer = ompi_group_peer_lookup(group,proc);
-        OBJ_RELEASE(proc_pointer);
+    for (int proc = 0 ; proc < group->grp_proc_count ; ++proc) {
+	proc_pointer = ompi_group_peer_lookup_existing (group, proc);
+	if (proc_pointer) {
+	    OBJ_RELEASE(proc_pointer);
+	}
     }
-
-    return;
 }
 
 /*

ompi/group/group_set_rank.c

@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -10,6 +11,8 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2006-2007 University of Houston. All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -38,12 +41,10 @@ void ompi_set_group_rank(ompi_group_t *group, struct ompi_proc_t *proc_pointer)
         for (proc = 0; proc < group->grp_proc_count; proc++) {
             /* check and see if this proc pointer matches proc_pointer
              */
-            if (ompi_group_peer_lookup(group,proc) == proc_pointer) {
+	    if (ompi_group_peer_lookup_existing (group, proc) == proc_pointer) {
                 group->grp_my_rank = proc;
-            }
+		break;
+	    }
         }                       /* end proc loop */
     }
-
-    /* return */
-    return;
 }