From mboxrd@z Thu Jan 1 00:00:00 1970 From: "Pavel Shamis (Pasha)" Subject: Re: RDMAoE verbs questions Date: Wed, 25 Nov 2009 22:13:06 +0200 Message-ID: <4B0D8FD2.9000702@dev.mellanox.co.il> References: <209ECE49-3AB9-4E62-B825-54E474321FA6@cisco.com> <4B0CDBEC.1030203@voltaire.com> <10904370-1D66-4013-BB3F-0F4B5EE581EE@cisco.com> <4B0D4660.5060701@voltaire.com> Reply-To: pasha-LDSdmyG8hGV8YrgS2mwiifqBs+8SCbDb@public.gmane.org Mime-Version: 1.0 Content-Type: multipart/mixed; boundary="------------090800050003090107040905" Return-path: In-Reply-To: <4B0D4660.5060701-smomgflXvOZWk0Htik3J/w@public.gmane.org> Sender: linux-rdma-owner-u79uwXL29TY76Z2rM5mHXA@public.gmane.org To: Or Gerlitz Cc: Jeff Squyres , linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org List-Id: linux-rdma@vger.kernel.org This is a multi-part message in MIME format. --------------090800050003090107040905 Content-Type: text/plain; charset=ISO-8859-1; format=flowed Content-Transfer-Encoding: 7bit Or, The patch is attached. Regards, Pasha. Or Gerlitz wrote: > Jeff Squyres wrote: >> Here's one thread: >> http://www.open-mpi.org/community/lists/devel/2009/11/7063.php > Jeff, looking on the threads you have sent, I didn't find a way to > download the patch in a form which can be applied on a source tree, is > there a way to do it through this archive? are these patches available > from some git tree @mellanox or elsewhere? does anyone have the email > address of Vasily Philipov (/vasily_at_[hidden]/), if yes, can you op > Pasha please ask him to send me or better, this list the proposed > patch, many thanks. > > Or > > -- > To unsubscribe from this list: send the line "unsubscribe linux-rdma" in > the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org > More majordomo info at http://vger.kernel.org/majordomo-info.html > --------------090800050003090107040905 Content-Type: text/x-patch; name="rdmaoe.patch" Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="rdmaoe.patch" diff -r 16b0d6d73529 ompi/config/ompi_check_openib.m4 --- a/ompi/config/ompi_check_openib.m4 Tue Nov 03 20:00:16 2009 -0800 +++ b/ompi/config/ompi_check_openib.m4 Sun Nov 15 14:58:37 2009 +0200 @@ -13,7 +13,7 @@ # Copyright (c) 2006-2008 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2006-2007 Los Alamos National Security, LLC. All rights # reserved. -# Copyright (c) 2006-2008 Mellanox Technologies. All rights reserved. +# Copyright (c) 2006-2009 Mellanox Technologies. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -204,6 +204,21 @@ [$1_have_ibcm=1 $1_LIBS="-libcm $$1_LIBS"])]) fi + + # Check support for RDMAoE devices + $1_have_rdmaoe=0 + AC_CHECK_DECLS([RDMA_TRANSPORT_RDMAOE], + [$1_have_rdmaoe=1], [], + [#include ]) + + AC_MSG_CHECKING([if RDMAoE support is enabled]) + if test "1" = "$$1_have_rdmaoe"; then + AC_DEFINE_UNQUOTED([OMPI_HAVE_RDMAOE], [$$1_have_rdmaoe], [Enable RDMAoE support]) + AC_MSG_RESULT([yes]) + else + AC_MSG_RESULT([no]) + fi + ]) # Check to see if works. It is known to diff -r 16b0d6d73529 ompi/mca/btl/openib/btl_openib.c --- a/ompi/mca/btl/openib/btl_openib.c Tue Nov 03 20:00:16 2009 -0800 +++ b/ompi/mca/btl/openib/btl_openib.c Sun Nov 15 14:58:37 2009 +0200 @@ -354,6 +354,13 @@ } #endif +#ifdef OMPI_HAVE_RDMAOE + if(RDMA_TRANSPORT_RDMAOE == (openib_btl->ib_port_attr.transport) && + OPAL_PROC_ON_LOCAL_NODE(ompi_proc->proc_flags)) { + continue; + } +#endif + if(NULL == (ib_proc = mca_btl_openib_proc_create(ompi_proc))) { return OMPI_ERR_OUT_OF_RESOURCE; } diff -r 16b0d6d73529 ompi/mca/btl/openib/connect/base.h --- a/ompi/mca/btl/openib/connect/base.h Tue Nov 03 20:00:16 2009 -0800 +++ b/ompi/mca/btl/openib/connect/base.h Sun Nov 15 14:58:37 2009 +0200 @@ -1,6 +1,7 @@ /* * Copyright (c) 2007-2008 Cisco Systems, Inc. All rights reserved. * + * Copyright (c) 2009 Mellanox Technologies. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -13,6 +14,17 @@ #include "connect/connect.h" +#ifdef OMPI_HAVE_RDMAOE +#define BTL_OPENIB_CONNECT_BASE_CHECK_IF_NOT_IB(btl) \ + (((IBV_TRANSPORT_IB != ((btl)->device->ib_dev->transport_type)) || \ + (RDMA_TRANSPORT_RDMAOE == ((btl)->ib_port_attr.transport))) ? \ + true : false) +#else +#define BTL_OPENIB_CONNECT_BASE_CHECK_IF_NOT_IB(btl) \ + ((IBV_TRANSPORT_IB != ((btl)->device->ib_dev->transport_type)) ? \ + true : false) +#endif + BEGIN_C_DECLS /* diff -r 16b0d6d73529 ompi/mca/btl/openib/connect/btl_openib_connect_ibcm.c --- a/ompi/mca/btl/openib/connect/btl_openib_connect_ibcm.c Tue Nov 03 20:00:16 2009 -0800 +++ b/ompi/mca/btl/openib/connect/btl_openib_connect_ibcm.c Sun Nov 15 14:58:37 2009 +0200 @@ -1,6 +1,6 @@ /* * Copyright (c) 2007-2009 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2008 Mellanox Technologies. All rights reserved. + * Copyright (c) 2008-2009 Mellanox Technologies. All rights reserved. * * $COPYRIGHT$ * @@ -653,7 +653,7 @@ we're in an old version of OFED that is IB only (i.e., no iWarp), so we can safely assume that we can use this CPC. */ #if defined(HAVE_STRUCT_IBV_DEVICE_TRANSPORT_TYPE) - if (IBV_TRANSPORT_IB != btl->device->ib_dev->transport_type) { + if (BTL_OPENIB_CONNECT_BASE_CHECK_IF_NOT_IB(btl)) { BTL_VERBOSE(("ibcm CPC only supported on InfiniBand; skipped on %s:%d", ibv_get_device_name(btl->device->ib_dev), openib_btl->port_num)); diff -r 16b0d6d73529 ompi/mca/btl/openib/connect/btl_openib_connect_oob.c --- a/ompi/mca/btl/openib/connect/btl_openib_connect_oob.c Tue Nov 03 20:00:16 2009 -0800 +++ b/ompi/mca/btl/openib/connect/btl_openib_connect_oob.c Sun Nov 15 14:58:37 2009 +0200 @@ -12,7 +12,7 @@ * Copyright (c) 2006-2009 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2006 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2008 Mellanox Technologies. All rights reserved. + * Copyright (c) 2008-2009 Mellanox Technologies. All rights reserved. * * $COPYRIGHT$ * @@ -120,7 +120,7 @@ transport_type member, then we must be < OFED v1.2, and therefore we must be IB. */ #if defined(HAVE_STRUCT_IBV_DEVICE_TRANSPORT_TYPE) - if (IBV_TRANSPORT_IB != btl->device->ib_dev->transport_type) { + if (BTL_OPENIB_CONNECT_BASE_CHECK_IF_NOT_IB(btl)) { opal_output_verbose(5, mca_btl_base_output, "openib BTL: oob CPC only supported on InfiniBand; skipped on %s:%d", ibv_get_device_name(btl->device->ib_dev), diff -r 16b0d6d73529 ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.c --- a/ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.c Tue Nov 03 20:00:16 2009 -0800 +++ b/ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.c Sun Nov 15 14:58:37 2009 +0200 @@ -1,7 +1,7 @@ /* * Copyright (c) 2007-2009 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2007-2008 Chelsio, Inc. All rights reserved. - * Copyright (c) 2008 Mellanox Technologies. All rights reserved. + * Copyright (c) 2008-2009 Mellanox Technologies. All rights reserved. * Copyright (c) 2009 Sandia National Laboratories. All rights reserved. * * $COPYRIGHT$ @@ -1158,10 +1158,15 @@ */ static int rdmacm_destroy_dummy_qp(id_context_t *context) { - if (NULL != context->id->qp) { - ibv_destroy_qp(context->id->qp); - context->id->qp = NULL; + /* We need to check id pointer because of retransmitions. + Maybe the reject was already done. */ + if (NULL != context->id) { + if (NULL != context->id->qp) { + ibv_destroy_qp(context->id->qp); + context->id->qp = NULL; + } } + if (NULL != context->contents->dummy_cq) { ibv_destroy_cq(context->contents->dummy_cq); } --------------090800050003090107040905-- -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org More majordomo info at http://vger.kernel.org/majordomo-info.html