Re: RDMAoE verbs questions

["Pavel Shamis (Pasha)" <pashash-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org>]

[-- Attachment #1: Type: text/plain, Size: 869 bytes --]

Or,
The patch is attached.

Regards,
Pasha.

Or Gerlitz wrote:
> Jeff Squyres wrote:
>> Here's one thread:  
>> http://www.open-mpi.org/community/lists/devel/2009/11/7063.php
> Jeff, looking on the threads you have sent, I didn't find a way to 
> download the patch in a form which can be applied on a source tree, is 
> there a way to do it through this archive? are these patches available 
> from some git tree @mellanox or elsewhere? does anyone have the email 
> address of Vasily Philipov (/vasily_at_[hidden]/), if yes, can you op 
> Pasha please ask him to send me or better, this list the proposed 
> patch, many thanks.
>
> Or
>
> -- 
> To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
> the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
>


[-- Attachment #2: rdmaoe.patch --]
[-- Type: text/x-patch, Size: 6393 bytes --]

diff -r 16b0d6d73529 ompi/config/ompi_check_openib.m4
--- a/ompi/config/ompi_check_openib.m4	Tue Nov 03 20:00:16 2009 -0800
+++ b/ompi/config/ompi_check_openib.m4	Sun Nov 15 14:58:37 2009 +0200
@@ -13,7 +13,7 @@
 # Copyright (c) 2006-2008 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2006-2007 Los Alamos National Security, LLC.  All rights
 #                         reserved.
-# Copyright (c) 2006-2008 Mellanox Technologies. All rights reserved.
+# Copyright (c) 2006-2009 Mellanox Technologies. All rights reserved.
 # $COPYRIGHT$
 # 
 # Additional copyrights may follow
@@ -204,6 +204,21 @@
                        [$1_have_ibcm=1
                        $1_LIBS="-libcm $$1_LIBS"])])
            fi
+		   
+           # Check support for RDMAoE devices
+           $1_have_rdmaoe=0
+           AC_CHECK_DECLS([RDMA_TRANSPORT_RDMAOE],
+                          [$1_have_rdmaoe=1], [],
+                          [#include <infiniband/verbs.h>])
+
+           AC_MSG_CHECKING([if RDMAoE support is enabled])
+           if test "1" = "$$1_have_rdmaoe"; then
+                AC_DEFINE_UNQUOTED([OMPI_HAVE_RDMAOE], [$$1_have_rdmaoe], [Enable RDMAoE support])
+                AC_MSG_RESULT([yes])
+           else
+                AC_MSG_RESULT([no])
+           fi
+
           ])
 
     # Check to see if <infiniband/driver.h> works.  It is known to
diff -r 16b0d6d73529 ompi/mca/btl/openib/btl_openib.c
--- a/ompi/mca/btl/openib/btl_openib.c	Tue Nov 03 20:00:16 2009 -0800
+++ b/ompi/mca/btl/openib/btl_openib.c	Sun Nov 15 14:58:37 2009 +0200
@@ -354,6 +354,13 @@
         }
 #endif
 
+#ifdef OMPI_HAVE_RDMAOE
+        if(RDMA_TRANSPORT_RDMAOE == (openib_btl->ib_port_attr.transport) &&
+                OPAL_PROC_ON_LOCAL_NODE(ompi_proc->proc_flags)) {
+            continue;
+        }
+#endif
+
         if(NULL == (ib_proc = mca_btl_openib_proc_create(ompi_proc))) {
             return OMPI_ERR_OUT_OF_RESOURCE;
         }
diff -r 16b0d6d73529 ompi/mca/btl/openib/connect/base.h
--- a/ompi/mca/btl/openib/connect/base.h	Tue Nov 03 20:00:16 2009 -0800
+++ b/ompi/mca/btl/openib/connect/base.h	Sun Nov 15 14:58:37 2009 +0200
@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2007-2008 Cisco Systems, Inc.  All rights reserved.
  *
+ * Copyright (c) 2009      Mellanox Technologies.  All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -13,6 +14,17 @@
 
 #include "connect/connect.h"
 
+#ifdef OMPI_HAVE_RDMAOE
+#define BTL_OPENIB_CONNECT_BASE_CHECK_IF_NOT_IB(btl)                       \
+        (((IBV_TRANSPORT_IB != ((btl)->device->ib_dev->transport_type)) || \
+        (RDMA_TRANSPORT_RDMAOE == ((btl)->ib_port_attr.transport))) ?      \
+        true : false)
+#else
+#define BTL_OPENIB_CONNECT_BASE_CHECK_IF_NOT_IB(btl)                       \
+        ((IBV_TRANSPORT_IB != ((btl)->device->ib_dev->transport_type)) ?   \
+        true : false)
+#endif
+
 BEGIN_C_DECLS
 
 /*
diff -r 16b0d6d73529 ompi/mca/btl/openib/connect/btl_openib_connect_ibcm.c
--- a/ompi/mca/btl/openib/connect/btl_openib_connect_ibcm.c	Tue Nov 03 20:00:16 2009 -0800
+++ b/ompi/mca/btl/openib/connect/btl_openib_connect_ibcm.c	Sun Nov 15 14:58:37 2009 +0200
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2007-2009 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2008      Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2008-2009 Mellanox Technologies. All rights reserved.
  *
  * $COPYRIGHT$
  * 
@@ -653,7 +653,7 @@
        we're in an old version of OFED that is IB only (i.e., no
        iWarp), so we can safely assume that we can use this CPC. */
 #if defined(HAVE_STRUCT_IBV_DEVICE_TRANSPORT_TYPE)
-    if (IBV_TRANSPORT_IB != btl->device->ib_dev->transport_type) {
+    if (BTL_OPENIB_CONNECT_BASE_CHECK_IF_NOT_IB(btl)) {
         BTL_VERBOSE(("ibcm CPC only supported on InfiniBand; skipped on %s:%d",
                      ibv_get_device_name(btl->device->ib_dev),
                      openib_btl->port_num));
diff -r 16b0d6d73529 ompi/mca/btl/openib/connect/btl_openib_connect_oob.c
--- a/ompi/mca/btl/openib/connect/btl_openib_connect_oob.c	Tue Nov 03 20:00:16 2009 -0800
+++ b/ompi/mca/btl/openib/connect/btl_openib_connect_oob.c	Sun Nov 15 14:58:37 2009 +0200
@@ -12,7 +12,7 @@
  * Copyright (c) 2006-2009 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2006      Los Alamos National Security, LLC.  All rights
  *                         reserved. 
- * Copyright (c) 2008      Mellanox Technologies.  All rights reserved.
+ * Copyright (c) 2008-2009 Mellanox Technologies.  All rights reserved.
  *
  * $COPYRIGHT$
  * 
@@ -120,7 +120,7 @@
        transport_type member, then we must be < OFED v1.2, and
        therefore we must be IB. */   
 #if defined(HAVE_STRUCT_IBV_DEVICE_TRANSPORT_TYPE)
-    if (IBV_TRANSPORT_IB != btl->device->ib_dev->transport_type) {
+    if (BTL_OPENIB_CONNECT_BASE_CHECK_IF_NOT_IB(btl)) {
         opal_output_verbose(5, mca_btl_base_output,
                             "openib BTL: oob CPC only supported on InfiniBand; skipped on  %s:%d",
                             ibv_get_device_name(btl->device->ib_dev),
diff -r 16b0d6d73529 ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.c
--- a/ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.c	Tue Nov 03 20:00:16 2009 -0800
+++ b/ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.c	Sun Nov 15 14:58:37 2009 +0200
@@ -1,7 +1,7 @@
 /*
  * Copyright (c) 2007-2009 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2007-2008 Chelsio, Inc. All rights reserved.
- * Copyright (c) 2008      Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2008-2009 Mellanox Technologies. All rights reserved.
  * Copyright (c) 2009      Sandia National Laboratories. All rights reserved.
  *
  * $COPYRIGHT$
@@ -1158,10 +1158,15 @@
  */
 static int rdmacm_destroy_dummy_qp(id_context_t *context)
 {
-    if (NULL != context->id->qp) {
-        ibv_destroy_qp(context->id->qp);
-        context->id->qp = NULL;
+    /* We need to check id pointer because of retransmitions.
+                                 Maybe the reject was already done. */
+    if (NULL != context->id) {
+	    if (NULL != context->id->qp) {
+            ibv_destroy_qp(context->id->qp);
+            context->id->qp = NULL;
+        }
     }
+
     if (NULL != context->contents->dummy_cq) {
         ibv_destroy_cq(context->contents->dummy_cq);
     }