* Re: [PATCH 31/37] librdmacm: provide abstracted verb calls
From: Steve Wise @ 2010-04-07 19:39 UTC (permalink / raw)
To: Sean Hefty; +Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA
In-Reply-To: <27A4CFF94530491F88C8E528428FF93E-Zpru7NauK7drdx17CPfAsdBPR1lH4CV8@public.gmane.org>
Sean Hefty wrote:
> Provide abstractions to the verb calls to simplify the user
> interface for more casual verbs consumers. Users still have
> access to the full range of verbs functionality by calling
> verbs directly.
>
> Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
> ---
>
> Makefile.am | 5 -
> include/rdma/rdma_verbs.h | 287 +++++++++++++++++++++++++++++++++++++++++++++
> 2 files changed, 290 insertions(+), 2 deletions(-)
>
> diff --git a/Makefile.am b/Makefile.am
> index 8d86045..8aef24a 100644
> --- a/Makefile.am
> +++ b/Makefile.am
> @@ -31,7 +31,8 @@ librdmacmincludedir = $(includedir)/rdma $(includedir)/infiniband
>
> librdmacminclude_HEADERS = include/rdma/rdma_cma_abi.h \
> include/rdma/rdma_cma.h \
> - include/infiniband/ib.h
> + include/infiniband/ib.h \
> + include/rdma/rdma_verbs.h
>
> man_MANS = \
> man/rdma_accept.3 \
> @@ -69,7 +70,7 @@ man_MANS = \
> man/rdma_cm.7
>
> EXTRA_DIST = include/rdma/rdma_cma_abi.h include/rdma/rdma_cma.h \
> - include/infiniband/ib.h \
> + include/infiniband/ib.h include/rdma/rdma_verbs.h \
> src/cma.h src/librdmacm.map librdmacm.spec.in $(man_MANS)
>
> dist-hook: librdmacm.spec
> diff --git a/include/rdma/rdma_verbs.h b/include/rdma/rdma_verbs.h
> new file mode 100644
> index 0000000..05964c1
> --- /dev/null
> +++ b/include/rdma/rdma_verbs.h
> @@ -0,0 +1,287 @@
> +/*
> + * Copyright (c) 2010 Intel Corporation. All rights reserved.
> + *
> + * This software is available to you under a choice of one of two
> + * licenses. You may choose to be licensed under the terms of the GNU
> + * General Public License (GPL) Version 2, available from the file
> + * COPYING in the main directory of this source tree, or the
> + * OpenIB.org BSD license below:
> + *
> + * Redistribution and use in source and binary forms, with or
> + * without modification, are permitted provided that the following
> + * conditions are met:
> + *
> + * - Redistributions of source code must retain the above
> + * copyright notice, this list of conditions and the following
> + * disclaimer.
> + *
> + * - Redistributions in binary form must reproduce the above
> + * copyright notice, this list of conditions and the following
> + * disclaimer in the documentation and/or other materials
> + * provided with the distribution.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
> + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
> + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
> + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
> + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
> + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
> + * SOFTWARE.
> + */
> +
> +#if !defined(RDMA_VERBS_H)
> +#define RDMA_VERBS_H
> +
> +#include <assert.h>
> +#include <infiniband/verbs.h>
> +#include <rdma/rdma_cma.h>
> +
> +#ifdef __cplusplus
> +extern "C" {
> +#endif
> +
> +/*
> + * Memory registration helpers.
> + */
> +static inline struct ibv_mr *
> +rdma_reg_msgs(struct rdma_cm_id *id, void *addr, size_t length)
> +{
> + return ibv_reg_mr(id->qp->pd, addr, length, IBV_ACCESS_LOCAL_WRITE);
> +}
> +
> +static inline struct ibv_mr *
> +rdma_reg_read(struct rdma_cm_id *id, void *addr, size_t length)
> +{
> + return ibv_reg_mr(id->qp->pd, addr, length, IBV_ACCESS_LOCAL_WRITE|
> + IBV_ACCESS_REMOTE_READ);
> +}
> +
> +static inline struct ibv_mr *
> +rdma_reg_write(struct rdma_cm_id *id, void *addr, size_t length)
> +{
> + return ibv_reg_mr(id->qp->pd, addr, length, IBV_ACCESS_LOCAL_WRITE |
> + IBV_ACCESS_REMOTE_WRITE);
> +}
> +
> +static inline int
> +rdma_dereg_mr(struct ibv_mr *mr)
> +{
> + return ibv_dereg_mr(mr);
> +}
> +
> +
> +/*
> + * Vectored send, receive, and RDMA operations.
> + * Support multiple scatter-gather entries.
> + */
> +static inline int
> +rdma_post_recvv(struct rdma_cm_id *id, void *context, struct ibv_sge *sgl,
> + int nsge)
> +{
> + struct ibv_recv_wr wr, *bad;
> +
> + wr.wr_id = (uintptr_t) context;
> + wr.next = NULL;
> + wr.sg_list = sgl;
> + wr.num_sge = nsge;
> +
> + return ibv_post_recv(id->qp, &wr, &bad);
> +}
> +
> +static inline int
> +rdma_post_sendv(struct rdma_cm_id *id, void *context, struct ibv_sge *sgl,
> + int nsge, int flags)
> +{
> + struct ibv_send_wr wr, *bad;
> +
> + wr.wr_id = (uintptr_t) context;
> + wr.next = NULL;
> + wr.sg_list = sgl;
> + wr.num_sge = nsge;
> + wr.opcode = IBV_WR_SEND;
> + wr.send_flags = flags;
> +
> + return ibv_post_send(id->qp, &wr, &bad);
> +}
> +
> +static inline int
> +rdma_post_readv(struct rdma_cm_id *id, void *context, struct ibv_sge *sgl,
> + int nsge, int flags, uint64_t remote_addr, uint32_t rkey)
> +{
> + struct ibv_send_wr wr, *bad;
> +
> + wr.wr_id = (uintptr_t) context;
> + wr.next = NULL;
> + wr.sg_list = sgl;
> + wr.num_sge = nsge;
> + wr.opcode = IBV_WR_RDMA_READ;
> + wr.send_flags = flags;
> + wr.wr.rdma.remote_addr = remote_addr;
> + wr.wr.rdma.rkey = rkey;
> +
> + return ibv_post_send(id->qp, &wr, &bad);
> +}
> +
> +static inline int
> +rdma_post_writev(struct rdma_cm_id *id, void *context, struct ibv_sge *sgl,
> + int nsge, int flags, uint64_t remote_addr, uint32_t rkey)
> +{
> + struct ibv_send_wr wr, *bad;
> +
> + wr.wr_id = (uintptr_t) context;
> + wr.next = NULL;
> + wr.sg_list = sgl;
> + wr.num_sge = nsge;
> + wr.opcode = IBV_WR_RDMA_WRITE;
> + wr.send_flags = flags;
> + wr.wr.rdma.remote_addr = remote_addr;
> + wr.wr.rdma.rkey = rkey;
> +
> + return ibv_post_send(id->qp, &wr, &bad);
> +}
> +
> +/*
> + * Simple send, receive, and RDMA calls.
> + */
> +static inline int
> +rdma_post_recv(struct rdma_cm_id *id, void *context, void *addr,
> + size_t length, struct ibv_mr *mr)
> +{
> + struct ibv_sge sge;
> +
> + assert((addr >= mr->addr) && ((addr + length) <= (mr->addr + mr->length)));
> + sge.addr = (uint64_t) addr;
> + sge.length = (uint32_t) length;
> + sge.lkey = mr->lkey;
> +
> + return rdma_post_recvv(id, context, &sge, 1);
> +}
> +
> +static inline int
> +rdma_post_send(struct rdma_cm_id *id, void *context, void *addr,
> + size_t length, struct ibv_mr *mr, int flags)
> +{
> + struct ibv_sge sge;
> +
> + sge.addr = (uint64_t) addr;
> + sge.length = (uint32_t) length;
> + sge.lkey = mr ? mr->lkey : 0;
> +
> + return rdma_post_sendv(id, context, &sge, 1, flags);
> +}
> +
> +static inline int
> +rdma_post_read(struct rdma_cm_id *id, void *context, void *addr,
> + size_t length, struct ibv_mr *mr, int flags,
> + uint64_t remote_addr, uint32_t rkey)
> +{
> + struct ibv_sge sge;
> +
> + sge.addr = (uint64_t) addr;
> + sge.length = (uint32_t) length;
> + sge.lkey = mr->lkey;
> +
> + return rdma_post_readv(id, context, &sge, 1, flags, remote_addr, rkey);
> +}
> +
> +static inline int
> +rdma_post_write(struct rdma_cm_id *id, void *context, void *addr,
> + size_t length, struct ibv_mr *mr, int flags,
> + uint64_t remote_addr, uint32_t rkey)
> +{
> + struct ibv_sge sge;
> +
> + sge.addr = (uint64_t) addr;
> + sge.length = (uint32_t) length;
> + sge.lkey = mr ? mr->lkey : 0;
> +
> + return rdma_post_writev(id, context, &sge, 1, flags, remote_addr, rkey);
> +}
> +
> +static inline int
> +rdma_post_ud_send(struct rdma_cm_id *id, void *context, void *addr,
> + size_t length, struct ibv_mr *mr, int flags,
> + struct ibv_ah *ah, uint32_t remote_qpn)
> +{
> + struct ibv_send_wr wr, *bad;
> + struct ibv_sge sge;
> +
> + sge.addr = (uint64_t) addr;
> + sge.length = (uint32_t) length;
> + sge.lkey = mr ? mr->lkey : 0;
> +
> + wr.wr_id = (uintptr_t) context;
> + wr.next = NULL;
> + wr.sg_list = &sge;
> + wr.num_sge = 1;
> + wr.opcode = IBV_WR_SEND;
> + wr.send_flags = flags;
> + wr.wr.ud.ah = ah;
> + wr.wr.ud.remote_qpn = remote_qpn;
> + wr.wr.ud.remote_qkey = RDMA_UDP_QKEY;
> +
> + return ibv_post_send(id->qp, &wr, &bad);
> +}
> +
> +static inline int
> +rdma_get_send_comp(struct rdma_cm_id *id, struct ibv_wc *wc)
> +{
> + struct ibv_cq *cq;
> + void *context;
> + int ret;
> +
> + ret = ibv_poll_cq(id->send_cq, 1, wc);
> + if (ret)
> + return ret;
> +
> + ret = ibv_req_notify_cq(id->send_cq, 0);
> + if (ret)
> + return ret;
> +
> + ret = ibv_poll_cq(id->send_cq, 1, wc);
> + if (ret)
> + return ret;
> +
> + ret = ibv_get_cq_event(id->send_cq_channel, &cq, &context);
> + if (ret)
> + return ret;
>
This doesn't look correct. If the send isn't complete by the time the
2nd ibv_poll_cq() completes, then this function will return without
having filled in the wc. Or am I missing something? Shouldn't the
ibv_get_cq_event() be the first thing this function does? The same
issue/question exists for rdma_get_recv_comp().
Steve.
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* Re: Ummunotify: progress at last!
From: Roland Dreier @ 2010-04-07 19:37 UTC (permalink / raw)
To: Jason Gunthorpe; +Cc: Jeff Squyres, Linux RDMA List, Brad Benton
In-Reply-To: <20100323201124.GK29129-ePGOBjL8dl3ta4EC/59zMFaTQe2KTcn/@public.gmane.org>
> No, there is no mmap. Like this:
>
> u64 my_counter = 0;
>
> ibv_set_mmu_counter(verbs, &my_counter);
> [..]
> while (my_counter != last_my_counter) {
> last_my_counter = my_counter;
> ibv_get_mmu_notifications(verbs, ...); // <- I am a memory barrier as well
> }
>
> The kernel 'syscall' ibv_set_mmu_counter would bind the given verbs to
> the 8 byte counter you specified without having to the mmap thing. As
> I understand it this is what perfevents does.
I was trying to look at how perf events handles this, and AFAICT it
looks like kernel/perf_event.c just supports mmap(). Can you expand on
what you meant here?
(I was trying to figure out how one would handle the case where
userspace gives us a counter in highmem -- doing kmap_atomic() seems to
be to only option but then I'm not sure if I want to deal with that...)
--
Roland Dreier <rolandd-FYB4Gu1CFyUAvxtiuMwx3w@public.gmane.org> || For corporate legal information go to:
http://www.cisco.com/web/about/doing_business/legal/cri/index.html
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* Re: [PATCH 22/37] librdmacm: add new call to create id
From: Jason Gunthorpe @ 2010-04-07 19:34 UTC (permalink / raw)
To: Sean Hefty; +Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA
In-Reply-To: <F818D641D3E64DA197E789362F5F0222-Zpru7NauK7drdx17CPfAsdBPR1lH4CV8@public.gmane.org>
On Wed, Apr 07, 2010 at 10:12:44AM -0700, Sean Hefty wrote:
> + * The rdma_cm_id will be set to use synchronous operations (connect,
> + * listen, and get_request). To convert to synchronous operation, the
^^^^^^^^^
asynchronous?
Jason
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* Re: [PATCH v2 13/51] IB/qib: Add qib_driver.c
From: Roland Dreier @ 2010-04-07 19:33 UTC (permalink / raw)
To: Steve Wise; +Cc: Ralph Campbell, linux-rdma-u79uwXL29TY76Z2rM5mHXA
In-Reply-To: <4BBCDBD6.8010809-7bPotxP6k4+P2YhJcF5u+vpXobYPEAuW@public.gmane.org>
> Where can I find information on trace events? Something in Documentation/*?
Yep, Documentation/trace/events.txt.
Also look at eg include/trace/events/ext4.h and how the trace_ext4_{xxx}
functions are used in fs/ext4/*.
It's much better than anything you're going to hack up for a single
driver to use.
- R.
--
Roland Dreier <rolandd-FYB4Gu1CFyUAvxtiuMwx3w@public.gmane.org> || For corporate legal information go to:
http://www.cisco.com/web/about/doing_business/legal/cri/index.html
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* Re: [PATCH 26/37] librdmacm: set src_addr in rdma_getaddrinfo
From: Jason Gunthorpe @ 2010-04-07 19:28 UTC (permalink / raw)
To: Sean Hefty; +Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA
In-Reply-To: <B7B5C575DF7D4C51AB84AFFAC7A5D5C5-Zpru7NauK7drdx17CPfAsdBPR1lH4CV8@public.gmane.org>
On Wed, Apr 07, 2010 at 10:12:43AM -0700, Sean Hefty wrote:
> RDMA requires the user to allocate hardware resources before
> establishing a connection. To support this, the user must know
> the source address that the connection will use to reach the
> remote endpoint. Modify rdma_getaddrinfo to determine an
> appropriate source address based on the specified destination,
> when a source address is not given.
I haven't looked through everything you posted to make a suggestion
here, but this bothers me..
The resources should be allocated after the rdma_bind syscall, prior to
listen/accept or connect, IMHO.
How does tha rai->ai_src_addr get used to allocate resources anyhow?
Jason
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* Re: [PATCH] Dimension port order file support
From: Dale Purdy @ 2010-04-07 19:27 UTC (permalink / raw)
To: Sasha Khapyorsky; +Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA
In-Reply-To: <20100324133221.GK4808@me>
[-- Attachment #1: Type: TEXT/PLAIN, Size: 3544 bytes --]
On Wed, 24 Mar 2010, Sasha Khapyorsky wrote:
> Hi Dale,
>
> On 18:06 Wed 03 Mar , Dale Purdy wrote:
>>
>> Provide a means to specify on a per switch basis the mapping (order)
>> between switch ports and dimensions for Dimension Order Routing. This
>> allows the DOR routing engine to be used when the cabling is not
>> properly aligned for DOR, either initially, or for an upgrade.
>
> Nice stuff.
>
> Is this something useful with ! '-R dor'?
>
I'm not using the dimn_ports array in anything but DOR, but I do think
it could be useful for some of the other routing engines.
>>
>> Signed-off-by: Dale Purdy <purdy-sJ/iWh9BUns@public.gmane.org>
>
> The patch itself is broken somehow - it has double space at start of
> non-changed line (it is fixable with sed -e 's/^ / /', so don't resend
> patch only for this).
>
Yes I see - odd. The original patch file didn't have this - must have
happened when loading it into mail. Hopefully my updated patch will
be ok.
> Some more minor comments are below.
...
>
>> +static int set_dimn_ports(void *ctx, uint64_t guid, char *p)
>> +{
>> + osm_ucast_mgr_t *m = ctx;
>> + osm_node_t *node = osm_get_node_by_guid(m->p_subn, cl_hton64(guid));
>> + osm_switch_t *sw;
>> + uint8_t *dimn_ports = NULL;
>> + uint8_t port;
>> + uint *ports = NULL;
>
> 'uint' is not something standard (we had some build compatibility issues
> with 'uint' in infiniband-diags in the past), so what about 'unsigned
> int'?
>
ok, fixed.
>> + const int bpw = sizeof(*ports)*8;
>> + int words;
>> + int i = 1; /* port 0 maps to port 0 */
>> +
>> + if (!node || !(sw = node->sw)) {
>> + OSM_LOG(m->p_log, OSM_LOG_DEBUG,
>> + "switch with guid 0x%016" PRIx64 " is not found\n",
>> + guid);
>> + return 0;
>> + }
>> +
>> + if (sw->dimn_ports) {
>> + OSM_LOG(m->p_log, OSM_LOG_DEBUG,
>> + "switch with guid 0x%016" PRIx64 " already listed\n",
>> + guid);
>
> It is GIUD double listed case, right? Wouldn't OSM_LOG_VERBOSE be more
> appropriate?
>
fixed.
>> + while ((*p != '\0') && (*p != '#')) {
>> + char *e;
>> +
>> + port = strtoul(p, &e, 0);
>> + if ((p == e) || (port == 0) || (port >= sw->num_ports) ||
>> + !osm_node_get_physp_ptr(node, port)) {
>> + OSM_LOG(m->p_log, OSM_LOG_DEBUG,
>> + "bad port %d specified for guid 0x%016" PRIx64 "\n",
>> + port, guid);
>> + free(dimn_ports);
>> + free(ports);
>
> Ditto.
>
fixed.
>> + return 0;
>> + }
>> +
>> + if (ports[port/bpw] & (1u << (port%bpw))) {
>> + OSM_LOG(m->p_log, OSM_LOG_DEBUG,
>> + "port %d already specified for guid 0x%016" PRIx64 "\n",
>> + port, guid);
>
> Ditto.
>
fixed.
>> + cl_qmap_apply_func(p_sw_guid_tbl, free_dimn_ports, NULL);
>> + if (p_mgr->p_subn->opt.dimn_ports_file) {
>> + OSM_LOG(p_mgr->p_log, OSM_LOG_DEBUG,
>> + "Fetching dimension ports file \'%s\'\n",
>> + p_mgr->p_subn->opt.dimn_ports_file);
>> + if (parse_node_map(p_mgr->p_subn->opt.dimn_ports_file,
>> + set_dimn_ports, p_mgr)) {
>> + OSM_LOG(p_mgr->p_log, OSM_LOG_ERROR, "ERR 3A05: "
>> + "cannot parse dimn_ports_file \'%s\'\n",
>> + p_mgr->p_subn->opt.dimn_ports_file);
>> + }
>> + }
>> +
>
> Hmm, if it is DOR only it can be done under 'if (is_dor)' (to save
> cycles of other REs). Otherwise (generic usability)
> ucast_mgr_setup_all_switches() seems as more appropriate place to have
> such setup, no?
>
moved to ucast_mgr_setup_all_switches() as you suggested.
> And what about adding:
>
> if (sw->dimn_ports)
> free(dimn_ports);
>
> in osm_switch_delete()?
>
fixed.
New patch attached.
Dale
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: Type: TEXT/x-diff; name=dimn_ports.patch, Size: 14624 bytes --]
Dimension port order file support (V2)
Provide a means to specify on a per switch basis the mapping (order)
between switch ports and dimensions for Dimension Order Routing. This
allows the DOR routing engine to be used when the cabling is not
properly aligned for DOR, either initially, or for an upgrade.
Signed-off-by: Dale Purdy <purdy@sgi.com>
---
opensm/include/opensm/osm_subnet.h | 1 +
opensm/include/opensm/osm_switch.h | 30 +++++++++
opensm/man/opensm.8.in | 31 ++++++++--
opensm/opensm/main.c | 13 ++++-
opensm/opensm/osm_subnet.c | 7 ++
opensm/opensm/osm_switch.c | 4 +-
opensm/opensm/osm_ucast_mgr.c | 116 +++++++++++++++++++++++++++++++++++-
7 files changed, 192 insertions(+), 10 deletions(-)
diff --git a/opensm/include/opensm/osm_subnet.h b/opensm/include/opensm/osm_subnet.h
index 3970e98..e4e298e 100644
--- a/opensm/include/opensm/osm_subnet.h
+++ b/opensm/include/opensm/osm_subnet.h
@@ -186,6 +186,7 @@ typedef struct osm_subn_opt {
uint16_t console_port;
char *port_prof_ignore_file;
char *hop_weights_file;
+ char *dimn_ports_file;
boolean_t port_profile_switch_nodes;
boolean_t sweep_on_trap;
char *routing_engine_names;
diff --git a/opensm/include/opensm/osm_switch.h b/opensm/include/opensm/osm_switch.h
index cb6e5ac..1c6807e 100644
--- a/opensm/include/opensm/osm_switch.h
+++ b/opensm/include/opensm/osm_switch.h
@@ -100,6 +100,7 @@ typedef struct osm_switch {
uint16_t num_hops;
uint8_t **hops;
osm_port_profile_t *p_prof;
+ uint8_t *dimn_ports;
uint8_t *lft;
uint8_t *new_lft;
uint16_t lft_size;
@@ -871,6 +872,35 @@ static inline uint8_t osm_switch_get_mft_max_position(IN osm_switch_t * p_sw)
* RETURN VALUE
*/
+/****f* OpenSM: Switch/osm_switch_get_dimn_port
+* NAME
+* osm_switch_get_dimn_port
+*
+* DESCRIPTION
+* Get the routing ordered port
+*
+* SYNOPSIS
+*/
+static inline uint8_t osm_switch_get_dimn_port(IN const osm_switch_t * p_sw,
+ IN uint8_t port_num)
+{
+ CL_ASSERT(p_sw);
+ if (p_sw->dimn_ports == NULL)
+ return port_num;
+ return p_sw->dimn_ports[port_num];
+}
+/*
+* PARAMETERS
+* p_sw
+* [in] Pointer to the switch object.
+*
+* port_num
+* [in] Port number in the switch
+*
+* RETURN VALUES
+* Returns the port number ordered for routing purposes.
+*/
+
/****f* OpenSM: Switch/osm_switch_recommend_path
* NAME
* osm_switch_recommend_path
diff --git a/opensm/man/opensm.8.in b/opensm/man/opensm.8.in
index 7aca8f9..9053611 100644
--- a/opensm/man/opensm.8.in
+++ b/opensm/man/opensm.8.in
@@ -37,6 +37,7 @@ opensm \- InfiniBand subnet manager and administration (SM/SA)
[\-console-port <port>]
[\-i(gnore-guids) <equalize-ignore-guids-file>]
[\-w | \-\-hop_weights_file <path to file>]
+[\-O | \-\-dimn_ports_file <path to file>]
[\-f <log file path> | \-\-log_file <log file path> ]
[\-L | \-\-log_limit <size in MB>] [\-e(rase_log_file)]
[\-P(config) <partition config file> ]
@@ -273,6 +274,16 @@ factor of 1. Lines starting with # are comments. Weights affect only the
output route from the port, so many useful configurations will require weights
to be specified in pairs.
.TP
+\fB\-O\fR, \fB\-\-dimn_ports_file\fR <path to file>
+This option provides a mapping between hypercube dimensions and ports
+on a per switch basis for the DOR routing engine. The file consists
+of lines containing a switch node GUID (specified as a 64 bit hex
+number, with leading 0x) followed by a list of non-zero port numbers,
+separated by spaces, one switch per line. The order for the port
+numbers is in one to one correspondence to the dimensions. Ports not
+listed on a line are assigned to the remaining dimensions, in port
+order. Anything after a # is a comment.
+.TP
\fB\-x\fR, \fB\-\-honor_guid2lid\fR
This option forces OpenSM to honor the guid2lid file,
when it comes out of Standby state, if such file exists
@@ -969,17 +980,20 @@ algorithm and so uses shortest paths. Instead of spreading traffic
out across different paths with the same shortest distance, it chooses
among the available shortest paths based on an ordering of dimensions.
Each port must be consistently cabled to represent a hypercube
-dimension or a mesh dimension. Paths are grown from a destination
-back to a source using the lowest dimension (port) of available paths
-at each step. This provides the ordering necessary to avoid deadlock.
+dimension or a mesh dimension. Alternatively, the -O option can be
+used to assign a custom mapping between the ports on a given switch,
+and the associated dimension. Paths are grown from a destination back
+to a source using the lowest dimension (port) of available paths at
+each step. This provides the ordering necessary to avoid deadlock.
When there are multiple links between any two switches, they still
represent only one dimension and traffic is balanced across them
unless port equalization is turned off. In the case of hypercubes,
the same port must be used throughout the fabric to represent the
-hypercube dimension and match on both ends of the cable. In the case
-of meshes, the dimension should consistently use the same pair of
-ports, one port on one end of the cable, and the other port on the
-other end, continuing along the mesh dimension.
+hypercube dimension and match on both ends of the cable, or the -O
+option used to accomplish the alignment. In the case of meshes, the
+dimension should consistently use the same pair of ports, one port on
+one end of the cable, and the other port on the other end, continuing
+along the mesh dimension, or the -O option used as an override.
Use '-R dor' option to activate the DOR algorithm.
@@ -1111,3 +1125,6 @@ Thomas Sodring
.TP
Ira Weiny
.RI < weiny2@llnl.gov >
+.TP
+Dale Purdy
+.RI < purdy@sgi.com >
diff --git a/opensm/opensm/main.c b/opensm/opensm/main.c
index f9a33af..0093aa7 100644
--- a/opensm/opensm/main.c
+++ b/opensm/opensm/main.c
@@ -275,6 +275,10 @@ static void show_usage(void)
" This option provides the means to define a weighting\n"
" factor per port for customizing the least weight\n"
" hops for the routing.\n\n");
+ printf("--dimn_ports_file, -O <path to file>\n"
+ " This option provides the means to define a mapping\n"
+ " between ports and dimension (Order) for controlling\n"
+ " Dimension Order Routing (DOR).\n\n");
printf("--honor_guid2lid, -x\n"
" This option forces OpenSM to honor the guid2lid file,\n"
" when it comes out of Standby state, if such file exists\n"
@@ -543,7 +547,7 @@ int main(int argc, char *argv[])
char *conf_template = NULL, *config_file = NULL;
uint32_t val;
const char *const short_option =
- "F:c:i:w:f:ed:D:g:l:L:s:t:a:u:m:X:R:zM:U:S:P:Y:ANBIQvVhoryxp:n:q:k:C:G:H:";
+ "F:c:i:w:O:f:ed:D:g:l:L:s:t:a:u:m:X:R:zM:U:S:P:Y:ANBIQvVhoryxp:n:q:k:C:G:H:";
/*
In the array below, the 2nd parameter specifies the number
@@ -560,6 +564,7 @@ int main(int argc, char *argv[])
{"guid", 1, NULL, 'g'},
{"ignore_guids", 1, NULL, 'i'},
{"hop_weights_file", 1, NULL, 'w'},
+ {"dimn_ports_file", 1, NULL, 'O'},
{"lmc", 1, NULL, 'l'},
{"sweep", 1, NULL, 's'},
{"timeout", 1, NULL, 't'},
@@ -696,6 +701,12 @@ int main(int argc, char *argv[])
opt.hop_weights_file);
break;
+ case 'O':
+ opt.dimn_ports_file = optarg;
+ printf(" Dimension Ports File = %s\n",
+ opt.dimn_ports_file);
+ break;
+
case 'g':
/*
Specifies port guid with which to bind.
diff --git a/opensm/opensm/osm_subnet.c b/opensm/opensm/osm_subnet.c
index e4126bc..e3f118b 100644
--- a/opensm/opensm/osm_subnet.c
+++ b/opensm/opensm/osm_subnet.c
@@ -324,6 +324,7 @@ static const opt_rec_t opt_tbl[] = {
{ "force_heavy_sweep", OPT_OFFSET(force_heavy_sweep), opts_parse_boolean, NULL, 1 },
{ "port_prof_ignore_file", OPT_OFFSET(port_prof_ignore_file), opts_parse_charp, NULL, 0 },
{ "hop_weights_file", OPT_OFFSET(hop_weights_file), opts_parse_charp, NULL, 0 },
+ { "dimn_ports_file", OPT_OFFSET(dimn_ports_file), opts_parse_charp, NULL, 0 },
{ "port_profile_switch_nodes", OPT_OFFSET(port_profile_switch_nodes), opts_parse_boolean, NULL, 1 },
{ "sweep_on_trap", OPT_OFFSET(sweep_on_trap), opts_parse_boolean, NULL, 1 },
{ "routing_engine", OPT_OFFSET(routing_engine_names), opts_parse_charp, NULL, 0 },
@@ -742,6 +743,7 @@ void osm_subn_set_default_opt(IN osm_subn_opt_t * p_opt)
p_opt->accum_log_file = TRUE;
p_opt->port_prof_ignore_file = NULL;
p_opt->hop_weights_file = NULL;
+ p_opt->dimn_ports_file = NULL;
p_opt->port_profile_switch_nodes = FALSE;
p_opt->sweep_on_trap = TRUE;
p_opt->use_ucast_cache = FALSE;
@@ -1385,6 +1387,11 @@ int osm_subn_output_conf(FILE *out, IN osm_subn_opt_t * p_opts)
p_opts->hop_weights_file ? p_opts->hop_weights_file : null_str);
fprintf(out,
+ "# The file holding non-default port order per switch for DOR routing \n"
+ "dimn_ports_file %s\n\n",
+ p_opts->dimn_ports_file ? p_opts->dimn_ports_file : null_str);
+
+ fprintf(out,
"# Routing engine\n"
"# Multiple routing engines can be specified separated by\n"
"# commas so that specific ordering of routing algorithms will\n"
diff --git a/opensm/opensm/osm_switch.c b/opensm/opensm/osm_switch.c
index 1cd8bfc..20fa098 100644
--- a/opensm/opensm/osm_switch.c
+++ b/opensm/opensm/osm_switch.c
@@ -78,6 +78,8 @@ void osm_switch_delete(IN OUT osm_switch_t ** pp_sw)
osm_mcast_tbl_destroy(&p_sw->mcast_tbl);
if (p_sw->p_prof)
free(p_sw->p_prof);
+ if (p_sw->dimn_ports)
+ free(p_sw->dimn_ports);
if (p_sw->lft)
free(p_sw->lft);
if (p_sw->new_lft)
@@ -341,7 +343,7 @@ uint8_t osm_switch_recommend_path(IN const osm_switch_t * p_sw,
/* port number starts with one and num_ports is 1 + num phys ports */
for (i = start_from; i < start_from + num_ports; i++) {
- port_num = i%num_ports;
+ port_num = osm_switch_get_dimn_port(p_sw, i%num_ports);
if (!port_num ||
osm_switch_get_hop_count(p_sw, base_lid, port_num) !=
least_hops)
diff --git a/opensm/opensm/osm_ucast_mgr.c b/opensm/opensm/osm_ucast_mgr.c
index 7ec58b5..e94370f 100644
--- a/opensm/opensm/osm_ucast_mgr.c
+++ b/opensm/opensm/osm_ucast_mgr.c
@@ -46,6 +46,7 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
+#include <ctype.h>
#include <iba/ib_types.h>
#include <complib/cl_qmap.h>
#include <complib/cl_debug.h>
@@ -488,6 +489,102 @@ static void set_default_hop_wf(cl_map_item_t * p_map_item, void *ctx)
}
}
+static int set_dimn_ports(void *ctx, uint64_t guid, char *p)
+{
+ osm_subn_t *p_subn = ctx;
+ osm_node_t *node = osm_get_node_by_guid(p_subn, cl_hton64(guid));
+ osm_switch_t *sw;
+ uint8_t *dimn_ports = NULL;
+ uint8_t port;
+ unsigned int *ports = NULL;
+ const int bpw = sizeof(*ports)*8;
+ int words;
+ int i = 1; /* port 0 maps to port 0 */
+
+ if (!node || !(sw = node->sw)) {
+ OSM_LOG(&p_subn->p_osm->log, OSM_LOG_VERBOSE,
+ "switch with guid 0x%016" PRIx64 " is not found\n",
+ guid);
+ return 0;
+ }
+
+ if (sw->dimn_ports) {
+ OSM_LOG(&p_subn->p_osm->log, OSM_LOG_VERBOSE,
+ "switch with guid 0x%016" PRIx64 " already listed\n",
+ guid);
+ return 0;
+ }
+
+ dimn_ports = malloc(sizeof(*dimn_ports)*sw->num_ports);
+ if (!dimn_ports) {
+ OSM_LOG(&p_subn->p_osm->log, OSM_LOG_ERROR,
+ "ERR 3A07: cannot allocate memory for dimn_ports\n");
+ return -1;
+ }
+ memset(dimn_ports, 0, sizeof(*dimn_ports)*sw->num_ports);
+
+ /* the ports array is for record keeping of which ports have
+ * been seen */
+ words = (sw->num_ports + bpw - 1)/bpw;
+ ports = malloc(words*sizeof(*ports));
+ if (!ports) {
+ OSM_LOG(&p_subn->p_osm->log, OSM_LOG_ERROR,
+ "ERR 3A08: cannot allocate memory for ports\n");
+ return -1;
+ }
+ memset(ports, 0, words*sizeof(*ports));
+
+ while ((*p != '\0') && (*p != '#')) {
+ char *e;
+
+ port = strtoul(p, &e, 0);
+ if ((p == e) || (port == 0) || (port >= sw->num_ports) ||
+ !osm_node_get_physp_ptr(node, port)) {
+ OSM_LOG(&p_subn->p_osm->log, OSM_LOG_VERBOSE,
+ "bad port %d specified for guid 0x%016" PRIx64 "\n",
+ port, guid);
+ free(dimn_ports);
+ free(ports);
+ return 0;
+ }
+
+ if (ports[port/bpw] & (1u << (port%bpw))) {
+ OSM_LOG(&p_subn->p_osm->log, OSM_LOG_VERBOSE,
+ "port %d already specified for guid 0x%016" PRIx64 "\n",
+ port, guid);
+ free(dimn_ports);
+ free(ports);
+ return 0;
+ }
+
+ ports[port/bpw] |= (1u << (port%bpw));
+ dimn_ports[i++] = port;
+
+ p = e;
+ while (isspace(*p)) {
+ p++;
+ }
+ }
+
+ if (i > 1) {
+ for (port = 1; port < sw->num_ports; port++) {
+ /* fill out the rest of the dimn_ports array
+ * in sequence using the remaining unspecified
+ * ports.
+ */
+ if (!(ports[port/bpw] & (1u << (port%bpw)))) {
+ dimn_ports[i++] = port;
+ }
+ }
+ sw->dimn_ports = dimn_ports;
+ } else {
+ free(dimn_ports);
+ }
+
+ free(ports);
+ return 0;
+}
+
int osm_ucast_mgr_build_lid_matrices(IN osm_ucast_mgr_t * p_mgr)
{
uint32_t i;
@@ -579,7 +676,7 @@ static int ucast_mgr_setup_all_switches(osm_subn_t * p_subn)
for (p_sw = (osm_switch_t *) cl_qmap_head(&p_subn->sw_guid_tbl);
p_sw != (osm_switch_t *) cl_qmap_end(&p_subn->sw_guid_tbl);
- p_sw = (osm_switch_t *) cl_qmap_next(&p_sw->map_item))
+ p_sw = (osm_switch_t *) cl_qmap_next(&p_sw->map_item)) {
if (osm_switch_prepare_path_rebuild(p_sw, lids)) {
OSM_LOG(&p_subn->p_osm->log, OSM_LOG_ERROR, "ERR 3A0B: "
"cannot setup switch 0x%016" PRIx64 "\n",
@@ -587,6 +684,23 @@ static int ucast_mgr_setup_all_switches(osm_subn_t * p_subn)
(p_sw->p_node)));
return -1;
}
+ if (p_sw->dimn_ports) {
+ free(p_sw->dimn_ports);
+ p_sw->dimn_ports = NULL;
+ }
+ }
+
+ if (p_subn->opt.dimn_ports_file) {
+ OSM_LOG(&p_subn->p_osm->log, OSM_LOG_DEBUG,
+ "Fetching dimension ports file \'%s\'\n",
+ p_subn->opt.dimn_ports_file);
+ if (parse_node_map(p_subn->opt.dimn_ports_file,
+ set_dimn_ports, p_subn)) {
+ OSM_LOG(&p_subn->p_osm->log, OSM_LOG_ERROR, "ERR 3A05: "
+ "cannot parse dimn_ports_file \'%s\'\n",
+ p_subn->opt.dimn_ports_file);
+ }
+ }
return 0;
}
--
1.7.0
^ permalink raw reply related
* Re: [PATCH v2 13/51] IB/qib: Add qib_driver.c
From: Steve Wise @ 2010-04-07 19:24 UTC (permalink / raw)
To: Roland Dreier; +Cc: Ralph Campbell, linux-rdma-u79uwXL29TY76Z2rM5mHXA
In-Reply-To: <adafx3743n6.fsf-BjVyx320WGW9gfZ95n9DRSW4+XlvGpQz@public.gmane.org>
Roland Dreier wrote:
> > +unsigned qib_debug;
> > +module_param_named(debug, qib_debug, uint, S_IWUSR | S_IRUGO);
> > +MODULE_PARM_DESC(debug, "mask for debug prints");
>
> Did you look at using trace events for this stuff? That gives you
> extremely low overhead when tracing is turned off (dynamic patching to
> NOP out the tracing when it's disabled) and also very fine-grained (per
> trace site) control over what gets printed; plus you get dumping of the
> trace buffer on crash, etc.
>
> - R.
>
Where can I find information on trace events? Something in Documentation/*?
Thanks,
Steve.
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* Re: [PATCH 11/37] librdmacm: add zalloc call
From: Jason Gunthorpe @ 2010-04-07 19:19 UTC (permalink / raw)
To: Sean Hefty; +Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA
In-Reply-To: <53A8AB949BCE4311B58A4881EC549E90-Zpru7NauK7drdx17CPfAsdBPR1lH4CV8@public.gmane.org>
On Wed, Apr 07, 2010 at 10:12:44AM -0700, Sean Hefty wrote:
> Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
>
> src/cma.c | 6 ++----
> src/cma.h | 10 ++++++++++
> 2 files changed, 12 insertions(+), 4 deletions(-)
>
> diff --git a/src/cma.c b/src/cma.c
> index a85448b..4025aeb 100644
> +++ b/src/cma.c
> @@ -342,11 +342,10 @@ static struct cma_id_private *ucma_alloc_id(struct rdma_event_channel *channel,
> {
> struct cma_id_private *id_priv;
>
> - id_priv = malloc(sizeof *id_priv);
> + id_priv = zalloc(sizeof *id_priv);
> if (!id_priv)
> return NULL;
Maybe use calloc(,1 sizeof(x)); instead?
Jason
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* [PATCH 9/37] librdmacm: add support for PF_IB multicast addresses
From: Sean Hefty @ 2010-04-07 18:13 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Allow the user to specify PF_IB addresses when joining
multicast groups. PF_IB addresses contain MGIDs directly.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
include/rdma/rdma_cma_abi.h | 12 +++++++++++-
src/cma.c | 44 +++++++++++++++++++++++++++++++------------
2 files changed, 43 insertions(+), 13 deletions(-)
diff --git a/include/rdma/rdma_cma_abi.h b/include/rdma/rdma_cma_abi.h
index 4a7a55d..c3981e6 100644
--- a/include/rdma/rdma_cma_abi.h
+++ b/include/rdma/rdma_cma_abi.h
@@ -68,7 +68,8 @@ enum {
UCMA_CMD_MIGRATE_ID,
UCMA_CMD_QUERY,
UCMA_CMD_BIND,
- UCMA_CMD_RESOLVE_ADDR
+ UCMA_CMD_RESOLVE_ADDR,
+ UCMA_CMD_JOIN_MCAST
};
struct ucma_abi_cmd_hdr {
@@ -243,6 +244,15 @@ struct ucma_abi_join_ip_mcast {
__u32 id;
};
+struct ucma_abi_join_mcast {
+ __u64 response; /* rdma_ucma_create_id_resp */
+ __u64 uid;
+ __u32 id;
+ __u16 addr_size;
+ __u16 reserved;
+ struct sockaddr_storage addr;
+};
+
struct ucma_abi_get_event {
__u64 response;
};
diff --git a/src/cma.c b/src/cma.c
index e22e1b4..c83d9d2 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -1244,21 +1244,16 @@ int rdma_disconnect(struct rdma_cm_id *id)
return 0;
}
-int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
- void *context)
+static int rdma_join_multicast2(struct rdma_cm_id *id, struct sockaddr *addr,
+ socklen_t addrlen, void *context)
{
- struct ucma_abi_join_ip_mcast *cmd;
struct ucma_abi_create_id_resp *resp;
struct cma_id_private *id_priv;
struct cma_multicast *mc, **pos;
void *msg;
- int ret, size, addrlen;
+ int ret, size;
id_priv = container_of(id, struct cma_id_private, id);
- addrlen = ucma_addrlen(addr);
- if (!addrlen)
- return ERR(EINVAL);
-
mc = malloc(sizeof *mc);
if (!mc)
return ERR(ENOMEM);
@@ -1277,10 +1272,23 @@ int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
id_priv->mc_list = mc;
pthread_mutex_unlock(&id_priv->mut);
- CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_JOIN_IP_MCAST, size);
- cmd->id = id_priv->handle;
- memcpy(&cmd->addr, addr, addrlen);
- cmd->uid = (uintptr_t) mc;
+ if (af_ib_support) {
+ struct ucma_abi_join_mcast *cmd;
+
+ CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_JOIN_MCAST, size);
+ cmd->id = id_priv->handle;
+ memcpy(&cmd->addr, addr, addrlen);
+ cmd->addr_size = addrlen;
+ cmd->uid = (uintptr_t) mc;
+ cmd->reserved = 0;
+ } else {
+ struct ucma_abi_join_ip_mcast *cmd;
+
+ CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_JOIN_IP_MCAST, size);
+ cmd->id = id_priv->handle;
+ memcpy(&cmd->addr, addr, addrlen);
+ cmd->uid = (uintptr_t) mc;
+ }
ret = write(id->channel->fd, msg, size);
if (ret != size) {
@@ -1303,6 +1311,18 @@ err1:
return ret;
}
+int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
+ void *context)
+{
+ int addrlen;
+
+ addrlen = ucma_addrlen(addr);
+ if (!addrlen)
+ return ERR(EINVAL);
+
+ return rdma_join_multicast2(id, addr, addrlen, context);
+}
+
int rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr)
{
struct ucma_abi_destroy_id *cmd;
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 0/37] librdmacm: add support for AF_IB
From: Sean Hefty @ 2010-04-07 18:13 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
The following patch series adds several enhancements to the librdmacm
intended to simplify using RDMA devices and address scalability issues.
Major changes include:
* Adding support for AF_IB.
* The addition of a new API: rdma_getaddrinfo. This call provides
functionality similar to getaddrinfo for RDMA devices. In addition
to resolving names to addresses, it can also resolve route and
connection data. rdma_getaddrinfo can return addresses using
AF_INET, AF_INET6, and AF_IB.
* Add support for IB ACM. IB ACM defines a socket based protocol to
an IB address and route resolution service. One implementation of that
service is provided separately, but anyone can implement the service
provided that they adhere to the IB ACM communication protocol.
Use of IB ACM is not required.
* Support synchronous operation for library calls. Users can control
whether an rdma_cm_id operates asynchronously or synchronously based on
the rdma_event_channel parameter. Use of synchronous operations
reduces the amount of application code required to use the librdmacm.
* Allow the library to abstract RDMA resource creation for simpler RDMA
applications. The library can now allocate PDs, CQs, and QPs for the
user, if not provided.
* Provide a set of helper verbs calls for posting work requests and
checking for completions. These are simple wrappers around libibverbs
calls.
This patch series is also available through my git tree at:
git://git.openfabrics.org/~shefty/librdmacm.git af_ib
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* Re: [PATCH] [RFC] ummunotify: Userspace support for MMU notifications
From: Randy Dunlap @ 2010-04-07 17:38 UTC (permalink / raw)
To: Eric B Munson
Cc: linux-kernel-u79uwXL29TY76Z2rM5mHXA,
linux-rdma-u79uwXL29TY76Z2rM5mHXA, rolandd-FYB4Gu1CFyUAvxtiuMwx3w,
peterz-wEGCiKHe2LqWVfeAwA7xHQ, pavel-+ZI9xUNit7I,
mingo-X9Un+BFzKDI
In-Reply-To: <1270643429-20688-1-git-send-email-ebmunson-r/Jw6+rmf7HQT0dZR+AlfA@public.gmane.org>
On Wed, 7 Apr 2010 13:30:29 +0100 Eric B Munson wrote:
> Signed-off-by: Roland Dreier <rolandd <at> cisco.com>
Use unobfuscated @.
> Signed-off-by: Eric B Munson <ebmunson-r/Jw6+rmf7HQT0dZR+AlfA@public.gmane.org>
>
> ---
>
> Changes since v3:
> - Fixed replaced [get|put] user with copy_[from|to]_user to fix x86
> builds
> ---
> Documentation/Makefile | 3 +-
> drivers/char/Kconfig | 12 +
> drivers/char/Makefile | 1 +
> drivers/char/ummunotify.c | 567 +++++++++++++++++++++++++++++++++++++++++++++
> 4 files changed, 582 insertions(+), 1 deletions(-)
> create mode 100644 drivers/char/ummunotify.c
>
> diff --git a/Documentation/Makefile b/Documentation/Makefile
> index 6fc7ea1..27ba76a 100644
> --- a/Documentation/Makefile
> +++ b/Documentation/Makefile
> @@ -1,3 +1,4 @@
> obj-m := DocBook/ accounting/ auxdisplay/ connector/ \
> filesystems/ filesystems/configfs/ ia64/ laptops/ networking/ \
> - pcmcia/ spi/ timers/ video4linux/ vm/ watchdog/src/
> + pcmcia/ spi/ timers/ video4linux/ vm/ ummunotify/ \
> + watchdog/src/
What is this change to Documentation/Makefile for?
Is there some file that should be added in Documentation/ummunotify/ ?
> diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig
> index 3141dd3..cf26019 100644
> --- a/drivers/char/Kconfig
> +++ b/drivers/char/Kconfig
> @@ -1111,6 +1111,18 @@ config DEVPORT
> depends on ISA || PCI
> default y
>
> +config UMMUNOTIFY
> + tristate "Userspace MMU notifications"
> + select MMU_NOTIFIER
> + help
> + The ummunotify (userspace MMU notification) driver creates a
> + character device that can be used by userspace libraries to
> + get notifications when an application's memory mapping
> + changed. This is used, for example, by RDMA libraries to
> + improve the reliability of memory registration caching, since
> + the kernel's MMU notifications can be used to know precisely
> + when to shoot down a cached registration.
> +
> source "drivers/s390/char/Kconfig"
>
> endmenu
> diff --git a/drivers/char/Makefile b/drivers/char/Makefile
> index f957edf..521e5de 100644
> --- a/drivers/char/Makefile
> +++ b/drivers/char/Makefile
> @@ -97,6 +97,7 @@ obj-$(CONFIG_NSC_GPIO) += nsc_gpio.o
> obj-$(CONFIG_CS5535_GPIO) += cs5535_gpio.o
> obj-$(CONFIG_GPIO_TB0219) += tb0219.o
> obj-$(CONFIG_TELCLOCK) += tlclk.o
> +obj-$(CONFIG_UMMUNOTIFY) += ummunotify.o
>
> obj-$(CONFIG_MWAVE) += mwave/
> obj-$(CONFIG_AGP) += agp/
---
~Randy
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* Re: Fork safe clarification
From: Roland Dreier @ 2010-04-07 17:13 UTC (permalink / raw)
To: Matthew Small; +Cc: linux-rdma
In-Reply-To: <s2pbb8be6ca1004010709hf5d0fe3cr2a76f28eaab97373-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org>
> Are PDs, QPs and CQs created before a fork shared by the parent and child
> after fork() has returned (ie. both can submit WRs, poll CQ, etc.)?
no, QPs and CQs are accessible only in the parent. The child can still
use the uverbs file descriptor to do things, but libibverbs will
probably get very confused in this case. More userspace development
would probably be required to make this really work. Since the PD is
attached to the FD, it could be shared.
> What about MRs registered before the fork? Even though the child doesn't
> have access to the parent's memory, can he sill submit WRs on a QP with an
> MR created before the fork?
yes.
> What if the MR pages in the above scenario are accessible in both parent and
> child (shared memory)? Are there complications with registering shared
> memory?
shouldn't make a difference.
> In general, are pointers returned by libibverbs pointer to user/process
> address space (as ibv_mr pointers must be) or kernel space (eg. if an
> unrelated process had another process's QP pointer, lkey, and a virtual
> address could it post (almost certainly unsafely) a WR to the other
> process's QP?
Not sure I understand this. All the pointers from libibverbs are of
course userspace pointers. What could a userspace process do with a
kernel pointer? Processes own all their resources and can't access
other resources.
- R.
--
Roland Dreier <rolandd-FYB4Gu1CFyUAvxtiuMwx3w@public.gmane.org> || For corporate legal information go to:
http://www.cisco.com/web/about/doing_business/legal/cri/index.html
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* [PATCH 13/37] librdmacm: allow pd parameter to be optional
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Allow the user to create a QP using rdma_create_qp without
specifying a PD. If a PD is not given, a default PD will be
used instead. This simplifies the user interface.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
include/rdma/rdma_cma.h | 4 +++-
src/cma.c | 24 +++++++++++++++++++-----
2 files changed, 22 insertions(+), 6 deletions(-)
diff --git a/include/rdma/rdma_cma.h b/include/rdma/rdma_cma.h
index 83418c3..ccf6cd4 100644
--- a/include/rdma/rdma_cma.h
+++ b/include/rdma/rdma_cma.h
@@ -279,7 +279,7 @@ int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms);
/**
* rdma_create_qp - Allocate a QP.
* @id: RDMA identifier.
- * @pd: protection domain for the QP.
+ * @pd: Optional protection domain for the QP.
* @qp_init_attr: initial QP attributes.
* Description:
* Allocate a QP associated with the specified rdma_cm_id and transition it
@@ -291,6 +291,8 @@ int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms);
* librdmacm through their states. After being allocated, the QP will be
* ready to handle posting of receives. If the QP is unconnected, it will
* be ready to post sends.
+ * If pd is NULL, then the QP will be allocated using a default protection
+ * domain associated with the underlying RDMA device.
* See also:
* rdma_bind_addr, rdma_resolve_addr, rdma_destroy_qp, ibv_create_qp,
* ibv_modify_qp
diff --git a/src/cma.c b/src/cma.c
index c7a3a7b..0587ab3 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -95,6 +95,7 @@ do { \
struct cma_device {
struct ibv_context *verbs;
+ struct ibv_pd *pd;
uint64_t guid;
int port_cnt;
uint8_t max_initiator_depth;
@@ -144,9 +145,11 @@ int af_ib_support;
static void ucma_cleanup(void)
{
if (cma_dev_cnt) {
- while (cma_dev_cnt)
- ibv_close_device(cma_dev_array[--cma_dev_cnt].verbs);
-
+ while (cma_dev_cnt--) {
+ ibv_dealloc_pd(cma_dev_array[cma_dev_cnt].pd);
+ ibv_close_device(cma_dev_array[cma_dev_cnt].verbs);
+ }
+
free(cma_dev_array);
cma_dev_cnt = 0;
}
@@ -224,6 +227,13 @@ static int ucma_init(void)
goto err3;
}
+ cma_dev->pd = ibv_alloc_pd(cma_dev->verbs);
+ if (!cma_dev->pd) {
+ ibv_close_device(cma_dev->verbs);
+ ret = ERR(ENOMEM);
+ goto err3;
+ }
+
i++;
ret = ibv_query_device(cma_dev->verbs, &attr);
if (ret) {
@@ -242,8 +252,10 @@ static int ucma_init(void)
return 0;
err3:
- while (i--)
+ while (i--) {
+ ibv_dealloc_pd(cma_dev_array[i].pd);
ibv_close_device(cma_dev_array[i].verbs);
+ }
free(cma_dev_array);
err2:
ibv_free_device_list(dev_list);
@@ -1021,7 +1033,9 @@ int rdma_create_qp(struct rdma_cm_id *id, struct ibv_pd *pd,
int ret;
id_priv = container_of(id, struct cma_id_private, id);
- if (id->verbs != pd->context)
+ if (!pd)
+ pd = id_priv->cma_dev->pd;
+ else if (id->verbs != pd->context)
return ERR(EINVAL);
qp = ibv_create_qp(pd, qp_init_attr);
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 21/37] librdmacm: specify qp_type when creating id
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
To support AF_IB / PS_IB, we need to specify the qp type when
creating the rdma_cm_id. The kernel requires this in order
to select the correct type of operation to perform (e.g. SIDR
versus REQ).
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
include/rdma/rdma_cma_abi.h | 3 ++-
src/cma.c | 18 +++++++++++++++---
2 files changed, 17 insertions(+), 4 deletions(-)
diff --git a/include/rdma/rdma_cma_abi.h b/include/rdma/rdma_cma_abi.h
index c3981e6..bd4ca0f 100644
--- a/include/rdma/rdma_cma_abi.h
+++ b/include/rdma/rdma_cma_abi.h
@@ -82,7 +82,8 @@ struct ucma_abi_create_id {
__u64 uid;
__u64 response;
__u16 ps;
- __u8 reserved[6];
+ __u8 qp_type;
+ __u8 reserved[5];
};
struct ucma_abi_create_id_resp {
diff --git a/src/cma.c b/src/cma.c
index 9de33d4..e31fb8a 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -390,9 +390,9 @@ err: ucma_free_id(id_priv);
return NULL;
}
-int rdma_create_id(struct rdma_event_channel *channel,
- struct rdma_cm_id **id, void *context,
- enum rdma_port_space ps)
+static int rdma_create_id2(struct rdma_event_channel *channel,
+ struct rdma_cm_id **id, void *context,
+ enum rdma_port_space ps, enum ibv_qp_type qp_type)
{
struct ucma_abi_create_id_resp *resp;
struct ucma_abi_create_id *cmd;
@@ -411,6 +411,7 @@ int rdma_create_id(struct rdma_event_channel *channel,
CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_CREATE_ID, size);
cmd->uid = (uintptr_t) id_priv;
cmd->ps = ps;
+ cmd->qp_type = qp_type;
ret = write(id_priv->id.channel->fd, msg, size);
if (ret != size)
@@ -426,6 +427,17 @@ err: ucma_free_id(id_priv);
return ret;
}
+int rdma_create_id(struct rdma_event_channel *channel,
+ struct rdma_cm_id **id, void *context,
+ enum rdma_port_space ps)
+{
+ enum ibv_qp_type qp_type;
+
+ qp_type = (ps == RDMA_PS_IPOIB || ps == RDMA_PS_UDP) ?
+ IBV_QPT_UD : IBV_QPT_RC;
+ return rdma_create_id2(channel, id, context, ps, qp_type);
+}
+
static int ucma_destroy_kern_id(int fd, uint32_t handle)
{
struct ucma_abi_destroy_id_resp *resp;
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 8/37] librdmacm: add support for PF_IB to resolve_addr
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Allow user to specify PF_IB addresses to rdma_resolve_addr.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
include/rdma/rdma_cma_abi.h | 13 ++++++++++++-
src/cma.c | 44 +++++++++++++++++++++++++++++++++++++------
2 files changed, 50 insertions(+), 7 deletions(-)
diff --git a/include/rdma/rdma_cma_abi.h b/include/rdma/rdma_cma_abi.h
index 8add397..4a7a55d 100644
--- a/include/rdma/rdma_cma_abi.h
+++ b/include/rdma/rdma_cma_abi.h
@@ -67,7 +67,8 @@ enum {
UCMA_CMD_LEAVE_MCAST,
UCMA_CMD_MIGRATE_ID,
UCMA_CMD_QUERY,
- UCMA_CMD_BIND
+ UCMA_CMD_BIND,
+ UCMA_CMD_RESOLVE_ADDR
};
struct ucma_abi_cmd_hdr {
@@ -117,6 +118,16 @@ struct ucma_abi_resolve_ip {
__u32 timeout_ms;
};
+struct ucma_abi_resolve_addr {
+ __u32 id;
+ __u32 timeout_ms;
+ __u16 src_size;
+ __u16 dst_size;
+ __u32 reserved;
+ struct sockaddr_storage src_addr;
+ struct sockaddr_storage dst_addr;
+};
+
struct ucma_abi_resolve_route {
__u32 id;
__u32 timeout_ms;
diff --git a/src/cma.c b/src/cma.c
index be61333..e22e1b4 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -721,31 +721,63 @@ int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
return ucma_query_route(id);
}
+static int rdma_resolve_addr2(struct rdma_cm_id *id, struct sockaddr *src_addr,
+ socklen_t src_len, struct sockaddr *dst_addr,
+ socklen_t dst_len, int timeout_ms)
+{
+ struct ucma_abi_resolve_addr *cmd;
+ struct cma_id_private *id_priv;
+ void *msg;
+ int ret, size;
+
+ CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_RESOLVE_ADDR, size);
+ id_priv = container_of(id, struct cma_id_private, id);
+ cmd->id = id_priv->handle;
+ if ((cmd->src_size = src_len))
+ memcpy(&cmd->src_addr, src_addr, src_len);
+ memcpy(&cmd->dst_addr, dst_addr, dst_len);
+ cmd->dst_size = dst_len;
+ cmd->timeout_ms = timeout_ms;
+ cmd->reserved = 0;
+
+ ret = write(id->channel->fd, msg, size);
+ if (ret != size)
+ return (ret >= 0) ? ERR(ENODATA) : -1;
+
+ memcpy(&id->route.addr.dst_addr, dst_addr, dst_len);
+ return 0;
+}
+
int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
struct sockaddr *dst_addr, int timeout_ms)
{
struct ucma_abi_resolve_ip *cmd;
struct cma_id_private *id_priv;
void *msg;
- int ret, size, daddrlen;
+ int ret, size, dst_len, src_len;
- daddrlen = ucma_addrlen(dst_addr);
- if (!daddrlen)
+ dst_len = ucma_addrlen(dst_addr);
+ if (!dst_len)
return ERR(EINVAL);
+ src_len = ucma_addrlen(src_addr);
+ if (af_ib_support)
+ return rdma_resolve_addr2(id, src_addr, src_len, dst_addr,
+ dst_len, timeout_ms);
+
CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_RESOLVE_IP, size);
id_priv = container_of(id, struct cma_id_private, id);
cmd->id = id_priv->handle;
if (src_addr)
- memcpy(&cmd->src_addr, src_addr, ucma_addrlen(src_addr));
- memcpy(&cmd->dst_addr, dst_addr, daddrlen);
+ memcpy(&cmd->src_addr, src_addr, src_len);
+ memcpy(&cmd->dst_addr, dst_addr, dst_len);
cmd->timeout_ms = timeout_ms;
ret = write(id->channel->fd, msg, size);
if (ret != size)
return (ret >= 0) ? ERR(ENODATA) : -1;
- memcpy(&id->route.addr.dst_addr, dst_addr, daddrlen);
+ memcpy(&id->route.addr.dst_addr, dst_addr, dst_len);
return 0;
}
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 12/37] librdmacm: support synchronous rdma_cm_id's
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Allow the user to specify NULL as the rdma_event_channel in
order to indicate that the rdma_cm_id should process all requests
synchronously.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
include/rdma/rdma_cma.h | 1 +
src/cma.c | 93 +++++++++++++++++++++++++++++++++++++++--------
2 files changed, 78 insertions(+), 16 deletions(-)
diff --git a/include/rdma/rdma_cma.h b/include/rdma/rdma_cma.h
index a071a9b..83418c3 100644
--- a/include/rdma/rdma_cma.h
+++ b/include/rdma/rdma_cma.h
@@ -114,6 +114,7 @@ struct rdma_cm_id {
struct rdma_route route;
enum rdma_port_space ps;
uint8_t port_num;
+ struct rdma_cm_event *event;
};
struct rdma_conn_param {
diff --git a/src/cma.c b/src/cma.c
index 4025aeb..c7a3a7b 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -106,6 +106,7 @@ struct cma_id_private {
struct cma_device *cma_dev;
int events_completed;
int connect_error;
+ int sync;
pthread_cond_t cond;
pthread_mutex_t mut;
uint32_t handle;
@@ -333,6 +334,9 @@ static void ucma_free_id(struct cma_id_private *id_priv)
pthread_mutex_destroy(&id_priv->mut);
if (id_priv->id.route.path_rec)
free(id_priv->id.route.path_rec);
+
+ if (id_priv->sync)
+ rdma_destroy_event_channel(id_priv->id.channel);
free(id_priv);
}
@@ -348,7 +352,16 @@ static struct cma_id_private *ucma_alloc_id(struct rdma_event_channel *channel,
id_priv->id.context = context;
id_priv->id.ps = ps;
- id_priv->id.channel = channel;
+
+ if (!channel) {
+ id_priv->id.channel = rdma_create_event_channel();
+ if (!id_priv->id.channel)
+ goto err;
+ id_priv->sync = 1;
+ } else {
+ id_priv->id.channel = channel;
+ }
+
pthread_mutex_init(&id_priv->mut, NULL);
if (pthread_cond_init(&id_priv->cond, NULL))
goto err;
@@ -381,7 +394,7 @@ int rdma_create_id(struct rdma_event_channel *channel,
cmd->uid = (uintptr_t) id_priv;
cmd->ps = ps;
- ret = write(channel->fd, msg, size);
+ ret = write(id_priv->id.channel->fd, msg, size);
if (ret != size)
goto err;
@@ -424,6 +437,9 @@ int rdma_destroy_id(struct rdma_cm_id *id)
if (ret < 0)
return ret;
+ if (id_priv->id.event)
+ rdma_ack_cm_event(id_priv->id.event);
+
pthread_mutex_lock(&id_priv->mut);
while (id_priv->events_completed < ret)
pthread_cond_wait(&id_priv->cond, &id_priv->mut);
@@ -694,6 +710,25 @@ int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
return ucma_query_route(id);
}
+static int ucma_complete(struct cma_id_private *id_priv)
+{
+ int ret;
+
+ if (!id_priv->sync)
+ return 0;
+
+ if (id_priv->id.event) {
+ rdma_ack_cm_event(id_priv->id.event);
+ id_priv->id.event = NULL;
+ }
+
+ ret = rdma_get_cm_event(id_priv->id.channel, &id_priv->id.event);
+ if (ret)
+ return ret;
+
+ return id_priv->id.event->status;
+}
+
static int rdma_resolve_addr2(struct rdma_cm_id *id, struct sockaddr *src_addr,
socklen_t src_len, struct sockaddr *dst_addr,
socklen_t dst_len, int timeout_ms)
@@ -718,7 +753,7 @@ static int rdma_resolve_addr2(struct rdma_cm_id *id, struct sockaddr *src_addr,
return (ret >= 0) ? ERR(ENODATA) : -1;
memcpy(&id->route.addr.dst_addr, dst_addr, dst_len);
- return 0;
+ return ucma_complete(id_priv);
}
int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
@@ -751,7 +786,7 @@ int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
return (ret >= 0) ? ERR(ENODATA) : -1;
memcpy(&id->route.addr.dst_addr, dst_addr, dst_len);
- return 0;
+ return ucma_complete(id_priv);
}
int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)
@@ -770,7 +805,7 @@ int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)
if (ret != size)
return (ret >= 0) ? ERR(ENODATA) : -1;
- return 0;
+ return ucma_complete(id_priv);
}
static int ucma_is_ud_ps(enum rdma_port_space ps)
@@ -1074,7 +1109,7 @@ int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
if (ret != size)
return (ret >= 0) ? ERR(ENODATA) : -1;
- return 0;
+ return ucma_complete(id_priv);
}
int rdma_listen(struct rdma_cm_id *id, int backlog)
@@ -1139,7 +1174,7 @@ int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
return (ret >= 0) ? ERR(ENODATA) : -1;
}
- return 0;
+ return ucma_complete(id_priv);
}
int rdma_reject(struct rdma_cm_id *id, const void *private_data,
@@ -1214,7 +1249,7 @@ int rdma_disconnect(struct rdma_cm_id *id)
if (ret != size)
return (ret >= 0) ? ERR(ENODATA) : -1;
- return 0;
+ return ucma_complete(id_priv);
}
static int rdma_join_multicast2(struct rdma_cm_id *id, struct sockaddr *addr,
@@ -1271,7 +1306,8 @@ static int rdma_join_multicast2(struct rdma_cm_id *id, struct sockaddr *addr,
VALGRIND_MAKE_MEM_DEFINED(resp, sizeof *resp);
mc->handle = resp->id;
- return 0;
+ return ucma_complete(id_priv);
+
err2:
pthread_mutex_lock(&id_priv->mut);
for (pos = &id_priv->mc_list; *pos != mc; pos = &(*pos)->next)
@@ -1443,21 +1479,28 @@ static int ucma_process_conn_req(struct cma_event *evt,
if (!id_priv) {
ucma_destroy_kern_id(evt->id_priv->id.channel->fd, handle);
ret = ERR(ENOMEM);
- goto err;
+ goto err1;
}
evt->event.listen_id = &evt->id_priv->id;
evt->event.id = &id_priv->id;
id_priv->handle = handle;
- ret = ucma_query_req_info(&id_priv->id);
- if (ret) {
- rdma_destroy_id(&id_priv->id);
- goto err;
+ if (evt->id_priv->sync) {
+ ret = rdma_migrate_id(&id_priv->id, NULL);
+ if (ret)
+ goto err2;
}
+ ret = ucma_query_req_info(&id_priv->id);
+ if (ret)
+ goto err2;
+
return 0;
-err:
+
+err2:
+ rdma_destroy_id(&id_priv->id);
+err1:
ucma_complete_event(evt->id_priv);
return ret;
}
@@ -1728,9 +1771,18 @@ int rdma_migrate_id(struct rdma_cm_id *id, struct rdma_event_channel *channel)
struct ucma_abi_migrate_id *cmd;
struct cma_id_private *id_priv;
void *msg;
- int ret, size;
+ int ret, size, sync;
id_priv = container_of(id, struct cma_id_private, id);
+ if (id_priv->sync && !channel)
+ return ERR(EINVAL);
+
+ if ((sync = (channel == NULL))) {
+ channel = rdma_create_event_channel();
+ if (!channel)
+ return ERR(ENOMEM);
+ }
+
CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_MIGRATE_ID, size);
cmd->id = id_priv->handle;
cmd->fd = id->channel->fd;
@@ -1741,6 +1793,14 @@ int rdma_migrate_id(struct rdma_cm_id *id, struct rdma_event_channel *channel)
VALGRIND_MAKE_MEM_DEFINED(resp, sizeof *resp);
+ if (id_priv->sync) {
+ if (id->event) {
+ rdma_ack_cm_event(id->event);
+ id->event = NULL;
+ }
+ rdma_destroy_event_channel(id->channel);
+ }
+
/*
* Eventually if we want to support migrating channels while events are
* being processed on the current channel, we need to block here while
@@ -1749,6 +1809,7 @@ int rdma_migrate_id(struct rdma_cm_id *id, struct rdma_event_channel *channel)
* channel after this call returns.
*/
pthread_mutex_lock(&id_priv->mut);
+ id_priv->sync = sync;
id->channel = channel;
while (id_priv->events_completed < resp->events_reported)
pthread_cond_wait(&id_priv->cond, &id_priv->mut);
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 10/37] librdmacm: move common definitions to internal header file
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
Makefile.am | 2 +-
src/cma.c | 28 +---------------------
src/cma.h | 74 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 76 insertions(+), 28 deletions(-)
diff --git a/Makefile.am b/Makefile.am
index 2898ad9..c9be437 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -70,7 +70,7 @@ man_MANS = \
EXTRA_DIST = include/rdma/rdma_cma_abi.h include/rdma/rdma_cma.h \
include/infiniband/ib.h \
- src/librdmacm.map librdmacm.spec.in $(man_MANS)
+ src/cma.h src/librdmacm.map librdmacm.spec.in $(man_MANS)
dist-hook: librdmacm.spec
cp librdmacm.spec $(distdir)
diff --git a/src/cma.c b/src/cma.c
index c83d9d2..a85448b 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -50,39 +50,13 @@
#include <byteswap.h>
#include <stddef.h>
+#include "cma.h"
#include <infiniband/driver.h>
#include <infiniband/marshall.h>
#include <rdma/rdma_cma.h>
#include <rdma/rdma_cma_abi.h>
#include <infiniband/ib.h>
-#ifdef INCLUDE_VALGRIND
-# include <valgrind/memcheck.h>
-# ifndef VALGRIND_MAKE_MEM_DEFINED
-# warning "Valgrind requested, but VALGRIND_MAKE_MEM_DEFINED undefined"
-# endif
-#endif
-
-#ifndef VALGRIND_MAKE_MEM_DEFINED
-# define VALGRIND_MAKE_MEM_DEFINED(addr,len)
-#endif
-
-#define PFX "librdmacm: "
-
-#if __BYTE_ORDER == __LITTLE_ENDIAN
-static inline uint64_t htonll(uint64_t x) { return bswap_64(x); }
-static inline uint64_t ntohll(uint64_t x) { return bswap_64(x); }
-#else
-static inline uint64_t htonll(uint64_t x) { return x; }
-static inline uint64_t ntohll(uint64_t x) { return x; }
-#endif
-
-static inline int ERR(int err)
-{
- errno = err;
- return -1;
-}
-
#define CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, type, size) \
do { \
struct ucma_abi_cmd_hdr *hdr; \
diff --git a/src/cma.h b/src/cma.h
new file mode 100644
index 0000000..1c0ab8b
--- /dev/null
+++ b/src/cma.h
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2005-2010 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#if !defined(CMA_H)
+#define CMA_H
+
+#if HAVE_CONFIG_H
+# include <config.h>
+#endif /* HAVE_CONFIG_H */
+
+#include <stdlib.h>
+#include <errno.h>
+#include <endian.h>
+#include <byteswap.h>
+
+#ifdef INCLUDE_VALGRIND
+# include <valgrind/memcheck.h>
+# ifndef VALGRIND_MAKE_MEM_DEFINED
+# warning "Valgrind requested, but VALGRIND_MAKE_MEM_DEFINED undefined"
+# endif
+#endif
+
+#ifndef VALGRIND_MAKE_MEM_DEFINED
+# define VALGRIND_MAKE_MEM_DEFINED(addr,len)
+#endif
+
+#define PFX "librdmacm: "
+
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+static inline uint64_t htonll(uint64_t x) { return bswap_64(x); }
+static inline uint64_t ntohll(uint64_t x) { return bswap_64(x); }
+#else
+static inline uint64_t htonll(uint64_t x) { return x; }
+static inline uint64_t ntohll(uint64_t x) { return x; }
+#endif
+
+static inline int ERR(int err)
+{
+ errno = err;
+ return -1;
+}
+
+#endif /* CMA_H */
+
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 5/37] librdmacm: replace query_route call with separate queries
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
To support other address families and multiple path records,
replace the query_route call with specific query calls to obtain
only the desired information.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
src/cma.c | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++--------
1 files changed, 60 insertions(+), 9 deletions(-)
diff --git a/src/cma.c b/src/cma.c
index 2a70d20..c57d166 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -161,6 +161,7 @@ static struct cma_device *cma_dev_array;
static int cma_dev_cnt;
static pthread_mutex_t mut = PTHREAD_MUTEX_INITIALIZER;
static int abi_ver = RDMA_USER_CM_MAX_ABI_VERSION;
+int af_ib_support;
#define container_of(ptr, type, field) \
((type *) ((void *)ptr - offsetof(type, field)))
@@ -627,7 +628,7 @@ static int ucma_query_route(struct rdma_cm_id *id)
struct cma_id_private *id_priv;
void *msg;
int ret, size, i;
-
+
CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_QUERY_ROUTE, size);
id_priv = container_of(id, struct cma_id_private, id);
cmd->id = id_priv->handle;
@@ -1060,7 +1061,10 @@ int rdma_listen(struct rdma_cm_id *id, int backlog)
if (ret != size)
return (ret >= 0) ? ERR(ENODATA) : -1;
- return ucma_query_route(id);
+ if (af_ib_support)
+ return ucma_query_addr(id);
+ else
+ return ucma_query_route(id);
}
int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
@@ -1326,6 +1330,57 @@ int rdma_ack_cm_event(struct rdma_cm_event *event)
return 0;
}
+static void ucma_process_addr_resolved(struct cma_event *evt)
+{
+ if (af_ib_support) {
+ evt->event.status = ucma_query_addr(&evt->id_priv->id);
+ if (!evt->event.status &&
+ evt->id_priv->id.verbs->device->transport_type == IBV_TRANSPORT_IB)
+ evt->event.status = ucma_query_gid(&evt->id_priv->id);
+ } else {
+ evt->event.status = ucma_query_route(&evt->id_priv->id);
+ }
+
+ if (evt->event.status)
+ evt->event.event = RDMA_CM_EVENT_ADDR_ERROR;
+}
+
+static void ucma_process_route_resolved(struct cma_event *evt)
+{
+ if (evt->id_priv->id.verbs->device->transport_type != IBV_TRANSPORT_IB)
+ return;
+
+ if (af_ib_support)
+ evt->event.status = ucma_query_path(&evt->id_priv->id);
+ else
+ evt->event.status = ucma_query_route(&evt->id_priv->id);
+
+ if (evt->event.status)
+ evt->event.event = RDMA_CM_EVENT_ROUTE_ERROR;
+}
+
+static int ucma_query_req_info(struct rdma_cm_id *id)
+{
+ int ret;
+
+ if (!af_ib_support)
+ return ucma_query_route(id);
+
+ ret = ucma_query_addr(id);
+ if (ret)
+ return ret;
+
+ ret = ucma_query_gid(id);
+ if (ret)
+ return ret;
+
+ ret = ucma_query_path(id);
+ if (ret)
+ return ret;
+
+ return 0;
+}
+
static int ucma_process_conn_req(struct cma_event *evt,
uint32_t handle)
{
@@ -1344,7 +1399,7 @@ static int ucma_process_conn_req(struct cma_event *evt,
evt->event.id = &id_priv->id;
id_priv->handle = handle;
- ret = ucma_query_route(&id_priv->id);
+ ret = ucma_query_req_info(&id_priv->id);
if (ret) {
rdma_destroy_id(&id_priv->id);
goto err;
@@ -1473,14 +1528,10 @@ retry:
switch (resp->event) {
case RDMA_CM_EVENT_ADDR_RESOLVED:
- evt->event.status = ucma_query_route(&evt->id_priv->id);
- if (evt->event.status)
- evt->event.event = RDMA_CM_EVENT_ADDR_ERROR;
+ ucma_process_addr_resolved(evt);
break;
case RDMA_CM_EVENT_ROUTE_RESOLVED:
- evt->event.status = ucma_query_route(&evt->id_priv->id);
- if (evt->event.status)
- evt->event.event = RDMA_CM_EVENT_ROUTE_ERROR;
+ ucma_process_route_resolved(evt);
break;
case RDMA_CM_EVENT_CONNECT_REQUEST:
evt->id_priv = (void *) (uintptr_t) resp->uid;
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 7/37] librdmacm: add support to bind using AF_IB
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Allow rdma_bind_addr to accept AF_IB addresses.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
include/rdma/rdma_cma_abi.h | 10 +++++++++-
src/cma.c | 25 +++++++++++++++++++++++++
2 files changed, 34 insertions(+), 1 deletions(-)
diff --git a/include/rdma/rdma_cma_abi.h b/include/rdma/rdma_cma_abi.h
index 0ef9564..8add397 100644
--- a/include/rdma/rdma_cma_abi.h
+++ b/include/rdma/rdma_cma_abi.h
@@ -66,7 +66,8 @@ enum {
UCMA_CMD_JOIN_IP_MCAST,
UCMA_CMD_LEAVE_MCAST,
UCMA_CMD_MIGRATE_ID,
- UCMA_CMD_QUERY
+ UCMA_CMD_QUERY,
+ UCMA_CMD_BIND
};
struct ucma_abi_cmd_hdr {
@@ -102,6 +103,13 @@ struct ucma_abi_bind_ip {
__u32 id;
};
+struct ucma_abi_bind {
+ __u32 id;
+ __u16 addr_size;
+ __u16 reserved;
+ struct sockaddr_storage addr;
+};
+
struct ucma_abi_resolve_ip {
struct sockaddr_in6 src_addr;
struct sockaddr_in6 dst_addr;
diff --git a/src/cma.c b/src/cma.c
index e85ef2d..be61333 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -673,6 +673,28 @@ static int ucma_query_route(struct rdma_cm_id *id)
return 0;
}
+static int rdma_bind_addr2(struct rdma_cm_id *id, struct sockaddr *addr,
+ socklen_t addrlen)
+{
+ struct ucma_abi_bind *cmd;
+ struct cma_id_private *id_priv;
+ void *msg;
+ int ret, size;
+
+ CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_BIND, size);
+ id_priv = container_of(id, struct cma_id_private, id);
+ cmd->id = id_priv->handle;
+ cmd->addr_size = addrlen;
+ cmd->reserved = 0;
+ memcpy(&cmd->addr, addr, addrlen);
+
+ ret = write(id->channel->fd, msg, size);
+ if (ret != size)
+ return (ret >= 0) ? ERR(ENODATA) : -1;
+
+ return ucma_query_addr(id);
+}
+
int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
{
struct ucma_abi_bind_ip *cmd;
@@ -684,6 +706,9 @@ int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
if (!addrlen)
return ERR(EINVAL);
+ if (af_ib_support)
+ return rdma_bind_addr2(id, addr, addrlen);
+
CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_BIND_IP, size);
id_priv = container_of(id, struct cma_id_private, id);
cmd->id = id_priv->handle;
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 1/37] librdmacm: name changes to indicate only IP addresses supported
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Several commands to the kernel RDMA CM only support IP addresses
because of limitations in the structure definition. Update
the library to match the name changes in the kernel and indicate
that only IP addresses can be used with the current commands.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
include/rdma/rdma_cma_abi.h | 12 ++++++------
src/cma.c | 12 ++++++------
2 files changed, 12 insertions(+), 12 deletions(-)
diff --git a/include/rdma/rdma_cma_abi.h b/include/rdma/rdma_cma_abi.h
index 1a3a9c2..e51a372 100644
--- a/include/rdma/rdma_cma_abi.h
+++ b/include/rdma/rdma_cma_abi.h
@@ -48,8 +48,8 @@
enum {
UCMA_CMD_CREATE_ID,
UCMA_CMD_DESTROY_ID,
- UCMA_CMD_BIND_ADDR,
- UCMA_CMD_RESOLVE_ADDR,
+ UCMA_CMD_BIND_IP,
+ UCMA_CMD_RESOLVE_IP,
UCMA_CMD_RESOLVE_ROUTE,
UCMA_CMD_QUERY_ROUTE,
UCMA_CMD_CONNECT,
@@ -62,7 +62,7 @@ enum {
UCMA_CMD_GET_OPTION,
UCMA_CMD_SET_OPTION,
UCMA_CMD_NOTIFY,
- UCMA_CMD_JOIN_MCAST,
+ UCMA_CMD_JOIN_IP_MCAST,
UCMA_CMD_LEAVE_MCAST,
UCMA_CMD_MIGRATE_ID
};
@@ -94,13 +94,13 @@ struct ucma_abi_destroy_id_resp {
__u32 events_reported;
};
-struct ucma_abi_bind_addr {
+struct ucma_abi_bind_ip {
__u64 response;
struct sockaddr_in6 addr;
__u32 id;
};
-struct ucma_abi_resolve_addr {
+struct ucma_abi_resolve_ip {
struct sockaddr_in6 src_addr;
struct sockaddr_in6 dst_addr;
__u32 id;
@@ -192,7 +192,7 @@ struct ucma_abi_notify {
__u32 event;
};
-struct ucma_abi_join_mcast {
+struct ucma_abi_join_ip_mcast {
__u64 response; /* ucma_abi_create_id_resp */
__u64 uid;
struct sockaddr_in6 addr;
diff --git a/src/cma.c b/src/cma.c
index 59e89dd..b5f71d0 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -525,7 +525,7 @@ static int ucma_query_route(struct rdma_cm_id *id)
int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
{
- struct ucma_abi_bind_addr *cmd;
+ struct ucma_abi_bind_ip *cmd;
struct cma_id_private *id_priv;
void *msg;
int ret, size, addrlen;
@@ -534,7 +534,7 @@ int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
if (!addrlen)
return ERR(EINVAL);
- CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_BIND_ADDR, size);
+ CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_BIND_IP, size);
id_priv = container_of(id, struct cma_id_private, id);
cmd->id = id_priv->handle;
memcpy(&cmd->addr, addr, addrlen);
@@ -549,7 +549,7 @@ int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
struct sockaddr *dst_addr, int timeout_ms)
{
- struct ucma_abi_resolve_addr *cmd;
+ struct ucma_abi_resolve_ip *cmd;
struct cma_id_private *id_priv;
void *msg;
int ret, size, daddrlen;
@@ -558,7 +558,7 @@ int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
if (!daddrlen)
return ERR(EINVAL);
- CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_RESOLVE_ADDR, size);
+ CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_RESOLVE_IP, size);
id_priv = container_of(id, struct cma_id_private, id);
cmd->id = id_priv->handle;
if (src_addr)
@@ -1037,7 +1037,7 @@ int rdma_disconnect(struct rdma_cm_id *id)
int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
void *context)
{
- struct ucma_abi_join_mcast *cmd;
+ struct ucma_abi_join_ip_mcast *cmd;
struct ucma_abi_create_id_resp *resp;
struct cma_id_private *id_priv;
struct cma_multicast *mc, **pos;
@@ -1067,7 +1067,7 @@ int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
id_priv->mc_list = mc;
pthread_mutex_unlock(&id_priv->mut);
- CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_JOIN_MCAST, size);
+ CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_JOIN_IP_MCAST, size);
cmd->id = id_priv->handle;
memcpy(&cmd->addr, addr, addrlen);
cmd->uid = (uintptr_t) mc;
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 6/37] librdmacm: add support for sockaddr_ib length
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Enhance ucma_addrlen to return the correct address size
for AF_IB.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
src/cma.c | 2 ++
1 files changed, 2 insertions(+), 0 deletions(-)
diff --git a/src/cma.c b/src/cma.c
index c57d166..e85ef2d 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -470,6 +470,8 @@ static int ucma_addrlen(struct sockaddr *addr)
return sizeof(struct sockaddr_in);
case PF_INET6:
return sizeof(struct sockaddr_in6);
+ case PF_IB:
+ return sizeof(struct sockaddr_ib);
default:
return 0;
}
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 4/37] librdmacm: add support to query GIDs
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Support query GID ABI to obtain GID information separately from
path record data and sa_family addressing.
This patch also adds the definition for sockaddr_ib for userspace.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
Makefile.am | 6 ++-
include/infiniband/ib.h | 97 +++++++++++++++++++++++++++++++++++++++++++
include/rdma/rdma_cma.h | 4 +-
include/rdma/rdma_cma_abi.h | 3 +
src/cma.c | 32 ++++++++++++++
5 files changed, 137 insertions(+), 5 deletions(-)
diff --git a/Makefile.am b/Makefile.am
index 290cbc3..2898ad9 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -27,10 +27,11 @@ examples_udaddy_LDADD = $(top_builddir)/src/librdmacm.la
examples_mckey_SOURCES = examples/mckey.c
examples_mckey_LDADD = $(top_builddir)/src/librdmacm.la
-librdmacmincludedir = $(includedir)/rdma
+librdmacmincludedir = $(includedir)/rdma $(includedir)/infiniband
librdmacminclude_HEADERS = include/rdma/rdma_cma_abi.h \
- include/rdma/rdma_cma.h
+ include/rdma/rdma_cma.h \
+ include/infiniband/ib.h
man_MANS = \
man/rdma_accept.3 \
@@ -68,6 +69,7 @@ man_MANS = \
man/rdma_cm.7
EXTRA_DIST = include/rdma/rdma_cma_abi.h include/rdma/rdma_cma.h \
+ include/infiniband/ib.h \
src/librdmacm.map librdmacm.spec.in $(man_MANS)
dist-hook: librdmacm.spec
diff --git a/include/infiniband/ib.h b/include/infiniband/ib.h
new file mode 100644
index 0000000..3a97322
--- /dev/null
+++ b/include/infiniband/ib.h
@@ -0,0 +1,97 @@
+/*
+ * Copyright (c) 2010 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#if !defined(_RDMA_IB_H)
+#define _RDMA_IB_H
+
+#include <linux/types.h>
+#include <string.h>
+
+#ifndef AF_IB
+#define AF_IB 27
+#endif
+#ifndef PF_IB
+#define PF_IB AF_IB
+#endif
+
+struct ib_addr {
+ union {
+ __u8 uib_addr8[16];
+ __be16 uib_addr16[8];
+ __be32 uib_addr32[4];
+ __be64 uib_addr64[2];
+ } ib_u;
+#define sib_addr8 ib_u.uib_addr8
+#define sib_addr16 ib_u.uib_addr16
+#define sib_addr32 ib_u.uib_addr32
+#define sib_addr64 ib_u.uib_addr64
+#define sib_raw ib_u.uib_addr8
+#define sib_subnet_prefix ib_u.uib_addr64[0]
+#define sib_interface_id ib_u.uib_addr64[1]
+};
+
+static inline int ib_addr_any(const struct ib_addr *a)
+{
+ return ((a->sib_addr64[0] | a->sib_addr64[1]) == 0);
+}
+
+static inline int ib_addr_loopback(const struct ib_addr *a)
+{
+ return ((a->sib_addr32[0] | a->sib_addr32[1] |
+ a->sib_addr32[2] | (a->sib_addr32[3] ^ htonl(1))) == 0);
+}
+
+static inline void ib_addr_set(struct ib_addr *addr,
+ __be32 w1, __be32 w2, __be32 w3, __be32 w4)
+{
+ addr->sib_addr32[0] = w1;
+ addr->sib_addr32[1] = w2;
+ addr->sib_addr32[2] = w3;
+ addr->sib_addr32[3] = w4;
+}
+
+static inline int ib_addr_cmp(const struct ib_addr *a1, const struct ib_addr *a2)
+{
+ return memcmp(a1, a2, sizeof(struct ib_addr));
+}
+
+struct sockaddr_ib {
+ unsigned short int sib_family; /* AF_IB */
+ __be16 sib_pkey;
+ __be32 sib_flowinfo;
+ struct ib_addr sib_addr;
+ __be64 sib_sid;
+ __be64 sib_sid_mask;
+ __u64 sib_scope_id;
+};
+
+#endif /* _RDMA_IB_H */
diff --git a/include/rdma/rdma_cma.h b/include/rdma/rdma_cma.h
index 534489d..a071a9b 100644
--- a/include/rdma/rdma_cma.h
+++ b/include/rdma/rdma_cma.h
@@ -78,7 +78,7 @@ enum rdma_port_space {
*/
#define RDMA_UDP_QKEY 0x01234567
-struct ib_addr {
+struct rdma_ib_addr {
union ibv_gid sgid;
union ibv_gid dgid;
uint16_t pkey;
@@ -92,7 +92,7 @@ struct rdma_addr {
uint8_t dst_pad[sizeof(struct sockaddr_storage) -
sizeof(struct sockaddr)];
union {
- struct ib_addr ibaddr;
+ struct rdma_ib_addr ibaddr;
} addr;
};
diff --git a/include/rdma/rdma_cma_abi.h b/include/rdma/rdma_cma_abi.h
index 6c83fe8..0ef9564 100644
--- a/include/rdma/rdma_cma_abi.h
+++ b/include/rdma/rdma_cma_abi.h
@@ -116,7 +116,8 @@ struct ucma_abi_resolve_route {
enum {
UCMA_QUERY_ADDR,
- UCMA_QUERY_PATH
+ UCMA_QUERY_PATH,
+ UCMA_QUERY_GID
};
struct ucma_abi_query {
diff --git a/src/cma.c b/src/cma.c
index c3c6b73..2a70d20 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -54,6 +54,7 @@
#include <infiniband/marshall.h>
#include <rdma/rdma_cma.h>
#include <rdma/rdma_cma_abi.h>
+#include <infiniband/ib.h>
#ifdef INCLUDE_VALGRIND
# include <valgrind/memcheck.h>
@@ -506,6 +507,37 @@ static int ucma_query_addr(struct rdma_cm_id *id)
return 0;
}
+static int ucma_query_gid(struct rdma_cm_id *id)
+{
+ struct ucma_abi_query_addr_resp *resp;
+ struct ucma_abi_query *cmd;
+ struct cma_id_private *id_priv;
+ struct sockaddr_ib *sib;
+ void *msg;
+ int ret, size;
+
+ CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_QUERY, size);
+ id_priv = container_of(id, struct cma_id_private, id);
+ cmd->id = id_priv->handle;
+ cmd->option = UCMA_QUERY_GID;
+
+ ret = write(id->channel->fd, msg, size);
+ if (ret != size)
+ return (ret >= 0) ? ERR(ENODATA) : -1;
+
+ VALGRIND_MAKE_MEM_DEFINED(resp, sizeof *resp);
+
+ sib = (struct sockaddr_ib *) &resp->src_addr;
+ memcpy(id->route.addr.addr.ibaddr.sgid.raw, sib->sib_addr.sib_raw,
+ sizeof id->route.addr.addr.ibaddr.sgid);
+
+ sib = (struct sockaddr_ib *) &resp->dst_addr;
+ memcpy(id->route.addr.addr.ibaddr.dgid.raw, sib->sib_addr.sib_raw,
+ sizeof id->route.addr.addr.ibaddr.dgid);
+
+ return 0;
+}
+
static void ucma_convert_path(struct ib_path_data *path_data,
struct ibv_sa_path_rec *sa_path)
{
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 15/37] librdmacm: allow user to specify max RDMA resources
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Allow the user to indicate that the library should select the
maximum RDMA read values available that should be used when
establishing a connection. The library selects the maximum
based on local hardware limitations and connection request
data.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
include/rdma/rdma_cma.h | 5 +++
src/cma.c | 83 +++++++++++++++++++++++++++++++----------------
src/cma.h | 2 +
3 files changed, 62 insertions(+), 28 deletions(-)
diff --git a/include/rdma/rdma_cma.h b/include/rdma/rdma_cma.h
index d8cbb91..f50b4dd 100644
--- a/include/rdma/rdma_cma.h
+++ b/include/rdma/rdma_cma.h
@@ -121,6 +121,11 @@ struct rdma_cm_id {
struct ibv_cq *recv_cq;
};
+enum {
+ RDMA_MAX_RESP_RES = 0xFF,
+ RDMA_MAX_INIT_DEPTH = 0xFF
+};
+
struct rdma_conn_param {
const void *private_data;
uint8_t private_data_len;
diff --git a/src/cma.c b/src/cma.c
index 805aca3..b8d57a5 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -112,6 +112,8 @@ struct cma_id_private {
pthread_mutex_t mut;
uint32_t handle;
struct cma_multicast *mc_list;
+ uint8_t initiator_depth;
+ uint8_t responder_resources;
};
struct cma_multicast {
@@ -850,8 +852,7 @@ static int rdma_init_qp_attr(struct rdma_cm_id *id, struct ibv_qp_attr *qp_attr,
return 0;
}
-static int ucma_modify_qp_rtr(struct rdma_cm_id *id,
- struct rdma_conn_param *conn_param)
+static int ucma_modify_qp_rtr(struct rdma_cm_id *id, uint8_t resp_res)
{
struct ibv_qp_attr qp_attr;
int qp_attr_mask, ret;
@@ -874,13 +875,12 @@ static int ucma_modify_qp_rtr(struct rdma_cm_id *id,
if (ret)
return ret;
- if (conn_param)
- qp_attr.max_dest_rd_atomic = conn_param->responder_resources;
+ if (resp_res != RDMA_MAX_RESP_RES)
+ qp_attr.max_dest_rd_atomic = resp_res;
return ibv_modify_qp(id->qp, &qp_attr, qp_attr_mask);
}
-static int ucma_modify_qp_rts(struct rdma_cm_id *id,
- struct rdma_conn_param *conn_param)
+static int ucma_modify_qp_rts(struct rdma_cm_id *id, uint8_t init_depth)
{
struct ibv_qp_attr qp_attr;
int qp_attr_mask, ret;
@@ -890,8 +890,8 @@ static int ucma_modify_qp_rts(struct rdma_cm_id *id,
if (ret)
return ret;
- if (conn_param)
- qp_attr.max_rd_atomic = conn_param->initiator_depth;
+ if (init_depth != RDMA_MAX_INIT_DEPTH)
+ qp_attr.max_rd_atomic = init_depth;
return ibv_modify_qp(id->qp, &qp_attr, qp_attr_mask);
}
@@ -1128,28 +1128,31 @@ void rdma_destroy_qp(struct rdma_cm_id *id)
}
static int ucma_valid_param(struct cma_id_private *id_priv,
- struct rdma_conn_param *conn_param)
+ struct rdma_conn_param *param)
{
if (id_priv->id.ps != RDMA_PS_TCP)
return 0;
- if ((conn_param->responder_resources >
- id_priv->cma_dev->max_responder_resources) ||
- (conn_param->initiator_depth >
- id_priv->cma_dev->max_initiator_depth))
+ if ((param->responder_resources != RDMA_MAX_RESP_RES) &&
+ (param->responder_resources > id_priv->cma_dev->max_responder_resources))
+ return ERR(EINVAL);
+
+ if ((param->initiator_depth != RDMA_MAX_INIT_DEPTH) &&
+ (param->initiator_depth > id_priv->cma_dev->max_initiator_depth))
return ERR(EINVAL);
return 0;
}
-static void ucma_copy_conn_param_to_kern(struct ucma_abi_conn_param *dst,
+static void ucma_copy_conn_param_to_kern(struct cma_id_private *id_priv,
+ struct ucma_abi_conn_param *dst,
struct rdma_conn_param *src,
uint32_t qp_num, uint8_t srq)
{
dst->qp_num = qp_num;
dst->srq = srq;
- dst->responder_resources = src->responder_resources;
- dst->initiator_depth = src->initiator_depth;
+ dst->responder_resources = id_priv->responder_resources;
+ dst->initiator_depth = id_priv->initiator_depth;
dst->flow_control = src->flow_control;
dst->retry_count = src->retry_count;
dst->rnr_retry_count = src->rnr_retry_count;
@@ -1174,15 +1177,24 @@ int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
if (ret)
return ret;
+ if (conn_param->initiator_depth != RDMA_MAX_INIT_DEPTH)
+ id_priv->initiator_depth = conn_param->initiator_depth;
+ else
+ id_priv->initiator_depth = id_priv->cma_dev->max_initiator_depth;
+ if (conn_param->responder_resources != RDMA_MAX_RESP_RES)
+ id_priv->responder_resources = conn_param->responder_resources;
+ else
+ id_priv->responder_resources = id_priv->cma_dev->max_responder_resources;
+
CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_CONNECT, size);
cmd->id = id_priv->handle;
if (id->qp)
- ucma_copy_conn_param_to_kern(&cmd->conn_param, conn_param,
- id->qp->qp_num,
+ ucma_copy_conn_param_to_kern(id_priv, &cmd->conn_param,
+ conn_param, id->qp->qp_num,
(id->qp->srq != NULL));
else
- ucma_copy_conn_param_to_kern(&cmd->conn_param, conn_param,
- conn_param->qp_num,
+ ucma_copy_conn_param_to_kern(id_priv, &cmd->conn_param,
+ conn_param, conn_param->qp_num,
conn_param->srq);
ret = write(id->channel->fd, msg, size);
@@ -1226,12 +1238,25 @@ int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
if (ret)
return ret;
+ if (conn_param->initiator_depth == RDMA_MAX_INIT_DEPTH) {
+ id_priv->initiator_depth = min(id_priv->initiator_depth,
+ id_priv->cma_dev->max_initiator_depth);
+ } else {
+ id_priv->initiator_depth = conn_param->initiator_depth;
+ }
+ if (conn_param->responder_resources == RDMA_MAX_RESP_RES) {
+ id_priv->responder_resources = min(id_priv->responder_resources,
+ id_priv->cma_dev->max_responder_resources);
+ } else {
+ id_priv->responder_resources = conn_param->responder_resources;
+ }
+
if (!ucma_is_ud_ps(id->ps)) {
- ret = ucma_modify_qp_rtr(id, conn_param);
+ ret = ucma_modify_qp_rtr(id, id_priv->responder_resources);
if (ret)
return ret;
- ret = ucma_modify_qp_rts(id, conn_param);
+ ret = ucma_modify_qp_rts(id, id_priv->initiator_depth);
if (ret)
return ret;
}
@@ -1240,12 +1265,12 @@ int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
cmd->id = id_priv->handle;
cmd->uid = (uintptr_t) id_priv;
if (id->qp)
- ucma_copy_conn_param_to_kern(&cmd->conn_param, conn_param,
- id->qp->qp_num,
+ ucma_copy_conn_param_to_kern(id_priv, &cmd->conn_param,
+ conn_param, id->qp->qp_num,
(id->qp->srq != NULL));
else
- ucma_copy_conn_param_to_kern(&cmd->conn_param, conn_param,
- conn_param->qp_num,
+ ucma_copy_conn_param_to_kern(id_priv, &cmd->conn_param,
+ conn_param, conn_param->qp_num,
conn_param->srq);
ret = write(id->channel->fd, msg, size);
@@ -1565,6 +1590,8 @@ static int ucma_process_conn_req(struct cma_event *evt,
evt->event.listen_id = &evt->id_priv->id;
evt->event.id = &id_priv->id;
id_priv->handle = handle;
+ id_priv->initiator_depth = evt->event.param.conn.initiator_depth;
+ id_priv->responder_resources = evt->event.param.conn.responder_resources;
if (evt->id_priv->sync) {
ret = rdma_migrate_id(&id_priv->id, NULL);
@@ -1591,11 +1618,11 @@ static int ucma_process_conn_resp(struct cma_id_private *id_priv)
void *msg;
int ret, size;
- ret = ucma_modify_qp_rtr(&id_priv->id, NULL);
+ ret = ucma_modify_qp_rtr(&id_priv->id, RDMA_MAX_RESP_RES);
if (ret)
goto err;
- ret = ucma_modify_qp_rts(&id_priv->id, NULL);
+ ret = ucma_modify_qp_rts(&id_priv->id, RDMA_MAX_INIT_DEPTH);
if (ret)
goto err;
diff --git a/src/cma.h b/src/cma.h
index fcfb1f7..92e771e 100644
--- a/src/cma.h
+++ b/src/cma.h
@@ -65,6 +65,8 @@ static inline uint64_t htonll(uint64_t x) { return x; }
static inline uint64_t ntohll(uint64_t x) { return x; }
#endif
+#define min(a, b) (a < b ? a : b)
+
static inline int ERR(int err)
{
errno = err;
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 17/37] librdmacm: expose ucma_init to other internal modules
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Remove static property from ucma_init and expose its
definition in cma.h. The address resolution module will
need access to this function.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
src/cma.c | 14 +++++++++-----
src/cma.h | 2 ++
2 files changed, 11 insertions(+), 5 deletions(-)
diff --git a/src/cma.c b/src/cma.c
index 6ef4b96..8aa7b05 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -188,13 +188,17 @@ static int check_abi_version(void)
return 0;
}
-static int ucma_init(void)
+int ucma_init(void)
{
struct ibv_device **dev_list = NULL;
struct cma_device *cma_dev;
struct ibv_device_attr attr;
int i, ret, dev_cnt;
+ /* Quick check without lock to see if we're already initialized */
+ if (cma_dev_cnt)
+ return 0;
+
pthread_mutex_lock(&mut);
if (cma_dev_cnt) {
pthread_mutex_unlock(&mut);
@@ -271,7 +275,7 @@ struct ibv_context **rdma_get_devices(int *num_devices)
struct ibv_context **devs = NULL;
int i;
- if (!cma_dev_cnt && ucma_init())
+ if (ucma_init())
goto out;
devs = malloc(sizeof *devs * (cma_dev_cnt + 1));
@@ -301,7 +305,7 @@ struct rdma_event_channel *rdma_create_event_channel(void)
{
struct rdma_event_channel *channel;
- if (!cma_dev_cnt && ucma_init())
+ if (ucma_init())
return NULL;
channel = malloc(sizeof *channel);
@@ -396,7 +400,7 @@ int rdma_create_id(struct rdma_event_channel *channel,
void *msg;
int ret, size;
- ret = cma_dev_cnt ? 0 : ucma_init();
+ ret = ucma_init();
if (ret)
return ret;
@@ -1712,7 +1716,7 @@ int rdma_get_cm_event(struct rdma_event_channel *channel,
void *msg;
int ret, size;
- ret = cma_dev_cnt ? 0 : ucma_init();
+ ret = ucma_init();
if (ret)
return ret;
diff --git a/src/cma.h b/src/cma.h
index 92e771e..06ca38c 100644
--- a/src/cma.h
+++ b/src/cma.h
@@ -82,5 +82,7 @@ static inline void *zalloc(size_t size)
return buf;
}
+int ucma_init();
+
#endif /* CMA_H */
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 2/37] librdmacm: support querying AF_IB addresses
From: Sean Hefty @ 2010-04-07 17:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
The current query route command returns path record data and address
information. The latter is restricted to sizeof(sockaddr_in6). In
order to support AF_IB, modify the library to use the new query addr
command, which supports larger address sizes and avoids querying for
path records data when none are available.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
include/rdma/rdma_cma_abi.h | 22 +++++++++++++++++++---
src/cma.c | 35 ++++++++++++++++++++++++++++++++++-
2 files changed, 53 insertions(+), 4 deletions(-)
diff --git a/include/rdma/rdma_cma_abi.h b/include/rdma/rdma_cma_abi.h
index e51a372..5c736fb 100644
--- a/include/rdma/rdma_cma_abi.h
+++ b/include/rdma/rdma_cma_abi.h
@@ -64,7 +64,8 @@ enum {
UCMA_CMD_NOTIFY,
UCMA_CMD_JOIN_IP_MCAST,
UCMA_CMD_LEAVE_MCAST,
- UCMA_CMD_MIGRATE_ID
+ UCMA_CMD_MIGRATE_ID,
+ UCMA_CMD_QUERY
};
struct ucma_abi_cmd_hdr {
@@ -112,10 +113,14 @@ struct ucma_abi_resolve_route {
__u32 timeout_ms;
};
-struct ucma_abi_query_route {
+enum {
+ UCMA_QUERY_ADDR
+};
+
+struct ucma_abi_query {
__u64 response;
__u32 id;
- __u32 reserved;
+ __u32 option;
};
struct ucma_abi_query_route_resp {
@@ -128,6 +133,17 @@ struct ucma_abi_query_route_resp {
__u8 reserved[3];
};
+struct ucma_abi_query_addr_resp {
+ __u64 node_guid;
+ __u8 port_num;
+ __u8 reserved;
+ __u16 pkey;
+ __u16 src_size;
+ __u16 dst_size;
+ struct sockaddr_storage src_addr;
+ struct sockaddr_storage dst_addr;
+};
+
struct ucma_abi_conn_param {
__u32 qp_num;
__u32 reserved;
diff --git a/src/cma.c b/src/cma.c
index b5f71d0..2aef594 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -473,10 +473,43 @@ static int ucma_addrlen(struct sockaddr *addr)
}
}
+static int ucma_query_addr(struct rdma_cm_id *id)
+{
+ struct ucma_abi_query_addr_resp *resp;
+ struct ucma_abi_query *cmd;
+ struct cma_id_private *id_priv;
+ void *msg;
+ int ret, size;
+
+ CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_QUERY, size);
+ id_priv = container_of(id, struct cma_id_private, id);
+ cmd->id = id_priv->handle;
+ cmd->option = UCMA_QUERY_ADDR;
+
+ ret = write(id->channel->fd, msg, size);
+ if (ret != size)
+ return (ret >= 0) ? ERR(ENODATA) : -1;
+
+ VALGRIND_MAKE_MEM_DEFINED(resp, sizeof *resp);
+
+ memcpy(&id->route.addr.src_addr, &resp->src_addr, resp->src_size);
+ memcpy(&id->route.addr.dst_addr, &resp->dst_addr, resp->dst_size);
+
+ if (!id_priv->cma_dev && resp->node_guid) {
+ ret = ucma_get_device(id_priv, resp->node_guid);
+ if (ret)
+ return ret;
+ id->port_num = resp->port_num;
+ id->route.addr.addr.ibaddr.pkey = resp->pkey;
+ }
+
+ return 0;
+}
+
static int ucma_query_route(struct rdma_cm_id *id)
{
struct ucma_abi_query_route_resp *resp;
- struct ucma_abi_query_route *cmd;
+ struct ucma_abi_query *cmd;
struct cma_id_private *id_priv;
void *msg;
int ret, size, i;
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox