Merge pull request #4371 from rhc54/topic/xvr

Updates to support cross-version operations with OMPI v2.x

Author: Ralph Castain

opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v12/bfrop_v12.c

@@ -421,7 +421,11 @@ int pmix12_v2_to_v1_datatype(pmix_data_type_t v2type)
             v1type = 6;
             break;
 
-        case 22:
+        case 39:
+            /* data arrays must be converted to info arrays */
+            v1type = 22;
+            break;
+
         case 23:
         case 24:
         case 25:
@@ -494,6 +498,10 @@ pmix_status_t pmix12_bfrop_get_data_type(pmix_buffer_t *buffer, pmix_data_type_t
     pmix_status_t rc;
 
     rc = pmix12_bfrop_unpack_datatype(buffer, &v1type, &n, PMIX_INT);
+    if (UINT16_MAX < v1type) {
+        *type = 0;
+        return PMIX_ERR_UNKNOWN_DATA_TYPE;
+    }
     if (PMIX_SUCCESS == rc) {
         *type = pmix12_v1_to_v2_datatype(v1type);
     }

opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v12/pack.c

@@ -86,6 +86,9 @@ pmix_status_t pmix12_bfrop_pack_buffer(pmix_buffer_t *buffer,
         case PMIX_PERSIST:
             v1type = PMIX_INT;
             break;
+        case PMIX_INFO_ARRAY:
+            v1type = 22;
+            break;
         default:
             v1type = type;
     }
@@ -96,6 +99,11 @@ pmix_status_t pmix12_bfrop_pack_buffer(pmix_buffer_t *buffer,
             return rc;
         }
     }
+    /* if it is an info array, we have to set the type back
+     * so the pack routine will get the correct function */
+    if (PMIX_INFO_ARRAY == type) {
+        v1type = PMIX_INFO_ARRAY;
+    }
 
     /* Lookup the pack function for this type and call it */
 
@@ -436,6 +444,7 @@ static pmix_status_t pack_val(pmix_buffer_t *buffer,
 {
     pmix_status_t ret;
     pmix_info_array_t array;
+    int rank;
 
     switch (p->type) {
     case PMIX_BOOL:
@@ -529,7 +538,7 @@ static pmix_status_t pack_val(pmix_buffer_t *buffer,
         }
         break;
     case PMIX_INFO_ARRAY:
-        if (PMIX_SUCCESS != (ret = pmix12_bfrop_pack_buffer(buffer, p->data.array, 1, PMIX_INFO_ARRAY))) {
+        if (PMIX_SUCCESS != (ret = pmix12_bfrop_pack_buffer(buffer, &p->data.array, 1, PMIX_INFO_ARRAY))) {
             return ret;
         }
         break;
@@ -550,6 +559,14 @@ static pmix_status_t pack_val(pmix_buffer_t *buffer,
         }
         break;
 
+    case PMIX_PROC_RANK:
+        /* must convert this to an int */
+        rank = p->data.rank;
+        if (PMIX_SUCCESS != (ret = pmix12_bfrop_pack_buffer(buffer, &rank, 1, PMIX_INT))) {
+            return ret;
+        }
+        break;
+
     default:
         pmix_output(0, "PACK-PMIX-VALUE: UNSUPPORTED TYPE %d", (int)p->type);
         return PMIX_ERROR;

opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v12/unpack.c

@@ -188,7 +188,8 @@ pmix_status_t pmix12_bfrop_unpack_buffer(pmix_buffer_t *buffer, void *dst, int32
         }
         /* if the data types don't match, then return an error */
         if (v1type != local_type) {
-            pmix_output(0, "PMIX bfrop:unpack: got type %d when expecting type %d", local_type, v1type);
+            pmix_output_verbose(1, pmix_bfrops_base_framework.framework_output,
+                                "PMIX bfrop:unpack: got type %d when expecting type %d", local_type, v1type);
             return PMIX_ERR_PACK_MISMATCH;
         }
     }

opal/mca/pmix/pmix3x/pmix/src/mca/gds/hash/gds_hash.c

@@ -1157,7 +1157,6 @@ static pmix_status_t hash_store_modex(struct pmix_nspace_t *nspace,
     pmix_buffer_t pbkt;
     pmix_proc_t proc;
     pmix_kval_t *kv;
-    pmix_peer_t *peer;
 
     pmix_output_verbose(2, pmix_gds_base_framework.framework_output,
                         "[%s:%d] gds:hash:store_modex for nspace %s",
@@ -1184,26 +1183,15 @@ static pmix_status_t hash_store_modex(struct pmix_nspace_t *nspace,
      * REMOTE/GLOBAL data. The byte object contains
      * the rank followed by pmix_kval_t's. The list of callbacks
      * contains all local participants. */
-    peer = NULL;
-    PMIX_LIST_FOREACH(scd, cbs, pmix_server_caddy_t) {
-        if (scd->peer->nptr == ns) {
-            peer = scd->peer;
-            break;
-        }
-    }
-    if (NULL == peer) {
-        /* we can ignore this one */
-        return PMIX_SUCCESS;
-    }
 
     /* setup the byte object for unpacking */
     PMIX_CONSTRUCT(&pbkt, pmix_buffer_t);
     /* the next step unfortunately NULLs the byte object's
      * entries, so we need to ensure we restore them! */
-    PMIX_LOAD_BUFFER(peer, &pbkt, bo->bytes, bo->size);
+    PMIX_LOAD_BUFFER(pmix_globals.mypeer, &pbkt, bo->bytes, bo->size);
     /* unload the proc that provided this data */
     cnt = 1;
-    PMIX_BFROPS_UNPACK(rc, peer, &pbkt, &proc, &cnt, PMIX_PROC);
+    PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, &pbkt, &proc, &cnt, PMIX_PROC);
     if (PMIX_SUCCESS != rc) {
         PMIX_ERROR_LOG(rc);
         bo->bytes = pbkt.base_ptr;
@@ -1215,7 +1203,7 @@ static pmix_status_t hash_store_modex(struct pmix_nspace_t *nspace,
     /* unpack the remaining values until we hit the end of the buffer */
     cnt = 1;
     kv = PMIX_NEW(pmix_kval_t);
-    PMIX_BFROPS_UNPACK(rc, peer, &pbkt, kv, &cnt, PMIX_KVAL);
+    PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, &pbkt, kv, &cnt, PMIX_KVAL);
     while (PMIX_SUCCESS == rc) {
         /* store this in the hash table */
         if (PMIX_SUCCESS != (rc = pmix_hash_store(&trk->remote, proc.rank, kv))) {
@@ -1230,7 +1218,7 @@ static pmix_status_t hash_store_modex(struct pmix_nspace_t *nspace,
         /* continue along */
         kv = PMIX_NEW(pmix_kval_t);
         cnt = 1;
-        PMIX_BFROPS_UNPACK(rc, peer, &pbkt, kv, &cnt, PMIX_KVAL);
+        PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, &pbkt, kv, &cnt, PMIX_KVAL);
     }
     PMIX_RELEASE(kv);  // maintain accounting
     if (PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER != rc) {

opal/mca/pmix/pmix3x/pmix/src/server/pmix_server.c

@@ -563,6 +563,16 @@ void pmix_server_execute_collective(int sd, short args, void *cbdata)
              * each participant. */
             peer = pmix_globals.mypeer;
         } else {
+            /* in some error situations, the list of local callbacks can
+             * be empty - if that happens, we just need to call the fence
+             * function to prevent others from hanging */
+            if (0 == pmix_list_get_size(&trk->local_cbs)) {
+                pmix_host_server.fence_nb(trk->pcs, trk->npcs,
+                                          trk->info, trk->ninfo,
+                                          data, sz, trk->modexcbfunc, trk);
+                PMIX_RELEASE(tcd);
+                return;
+            }
             /* since all procs are the same, just use the first proc's module */
             cd = (pmix_server_caddy_t*)pmix_list_get_first(&trk->local_cbs);
             peer = cd->peer;
@@ -1676,8 +1686,6 @@ static void _mdxcbfunc(int sd, short argc, void *cbdata)
     }
     if (PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER == rc) {
         rc = PMIX_SUCCESS;
-    } else if (PMIX_SUCCESS != rc) {
-        PMIX_ERROR_LOG(rc);
     }
 
   finish_collective:

orte/orted/pmix/pmix_server.c

@@ -148,6 +148,14 @@ void pmix_server_register_params(void)
                                   OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_ALL,
                                   &orte_pmix_server_globals.wait_for_server);
 
+    /* whether or not to support legacy usock connections as well as tcp */
+    orte_pmix_server_globals.legacy = false;
+    (void) mca_base_var_register ("orte", "pmix", NULL, "server_usock_connections",
+                                  "Whether or not to support legacy usock connections",
+                                  MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
+                                  OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_ALL,
+                                  &orte_pmix_server_globals.legacy);
+
     /* whether or not to drop a session-level tool rendezvous point */
     orte_pmix_server_globals.session_server = false;
     (void) mca_base_var_register ("orte", "pmix", NULL, "session_server",
@@ -250,12 +258,14 @@ int pmix_server_init(void)
     kv->type = OPAL_STRING;
     kv->data.string = opal_os_path(false, orte_process_info.jobfam_session_dir, NULL);
     opal_list_append(&info, &kv->super);
-    /* use only one listener */
-    kv = OBJ_NEW(opal_value_t);
-    kv->key = strdup(OPAL_PMIX_SINGLE_LISTENER);
-    kv->type = OPAL_BOOL;
-    kv->data.flag = true;
-    opal_list_append(&info, &kv->super);
+    if (!orte_pmix_server_globals.legacy) {
+        /* use only one listener */
+        kv = OBJ_NEW(opal_value_t);
+        kv->key = strdup(OPAL_PMIX_SINGLE_LISTENER);
+        kv->type = OPAL_BOOL;
+        kv->data.flag = true;
+        opal_list_append(&info, &kv->super);
+    }
     /* tell the server to use its own internal monitoring */
     kv = OBJ_NEW(opal_value_t);
     kv->key = strdup(OPAL_PMIX_SERVER_ENABLE_MONITORING);

orte/orted/pmix/pmix_server_internal.h

@@ -265,6 +265,7 @@ typedef struct {
     bool pubsub_init;
     bool session_server;
     bool system_server;
+    bool legacy;
 } pmix_server_globals_t;
 
 extern pmix_server_globals_t orte_pmix_server_globals;