[RFC 0/3] RDMA: Add netlink infrastructure

[RFC 0/3] RDMA: Add netlink infrastructure

[Roland Dreier]

From: Roland Dreier <roland-BHEL68pLQRGGvPXPguhicg@public.gmane.org>

Hi,

Here are the highlights of a patch series adding support for getting
information on active RDMA connections via netlink.  I would really
appreciate it if netlink experts could take a look and make sure we
are doing this the right way and using a suitably extensible
structure.

I'm by no means particularly knowledgeable about netlink, so although
the RDMA side of things looks OK to me, I hope we can get some level
of review of the netlink side of things; the changelog for patch 2/3
tries to explain the messages we're using.

The full series based on 2.6.39-rc7 is also available at:

    git://git.kernel.org/pub/scm/linux/kernel/git/roland/infiniband.git netlink

Thanks!
  Roland
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[RFC 1/3] RDMA: Add netlink infrastructure

[Roland Dreier]

From: Roland Dreier <roland@purestorage.com>

[Dave please do not apply even if this ends up in netdev patchwork!]

Add basic RDMA netlink infrastructure that allows for registration of
RDMA clients for which data is to be exported and supplies message
construction callbacks.

NOT-Signed-off-by: Nir Muchtar <nirm@voltaire.com>

[ Rename and reorganize a few things.  - Roland ]

NOT-Signed-off-by: Roland Dreier <roland@purestorage.com>
---
 drivers/infiniband/core/Makefile  |    2 +-
 drivers/infiniband/core/device.c  |   11 ++
 drivers/infiniband/core/netlink.c |  190 +++++++++++++++++++++++++++++++++++++
 include/linux/netlink.h           |    1 +
 include/rdma/rdma_netlink.h       |   64 +++++++++++++
 5 files changed, 267 insertions(+), 1 deletions(-)
 create mode 100644 drivers/infiniband/core/netlink.c
 create mode 100644 include/rdma/rdma_netlink.h

diff --git a/drivers/infiniband/core/Makefile b/drivers/infiniband/core/Makefile
index cb1ab3e..c8bbaef 100644
--- a/drivers/infiniband/core/Makefile
+++ b/drivers/infiniband/core/Makefile
@@ -8,7 +8,7 @@ obj-$(CONFIG_INFINIBAND_USER_ACCESS) +=	ib_uverbs.o ib_ucm.o \
 					$(user_access-y)
 
 ib_core-y :=			packer.o ud_header.o verbs.o sysfs.o \
-				device.o fmr_pool.o cache.o
+				device.o fmr_pool.o cache.o netlink.o
 ib_core-$(CONFIG_INFINIBAND_USER_MEM) += umem.o
 
 ib_mad-y :=			mad.o smi.o agent.o mad_rmpp.o
diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c
index 46a7a3f..635c338 100644
--- a/drivers/infiniband/core/device.c
+++ b/drivers/infiniband/core/device.c
@@ -38,6 +38,7 @@
 #include <linux/slab.h>
 #include <linux/init.h>
 #include <linux/mutex.h>
+#include <rdma/rdma_netlink.h>
 
 #include "core_priv.h"
 
@@ -736,8 +737,17 @@ static int __init ib_core_init(void)
 		goto err_sysfs;
 	}
 
+	ret = ibnl_init();
+	if (ret) {
+		printk(KERN_WARNING "Couldn't init IB netlink interface\n");
+		goto err_cache;
+	}
+
 	return 0;
 
+err_cache:
+	ib_cache_cleanup();
+
 err_sysfs:
 	ib_sysfs_cleanup();
 
@@ -748,6 +758,7 @@ err:
 
 static void __exit ib_core_cleanup(void)
 {
+	ibnl_cleanup();
 	ib_cache_cleanup();
 	ib_sysfs_cleanup();
 	/* Make sure that any pending umem accounting work is done. */
diff --git a/drivers/infiniband/core/netlink.c b/drivers/infiniband/core/netlink.c
new file mode 100644
index 0000000..361e29f
--- /dev/null
+++ b/drivers/infiniband/core/netlink.c
@@ -0,0 +1,190 @@
+/*
+ * Copyright (c) 2010 Voltaire Inc.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#define pr_fmt(fmt) "%s:%s: " fmt, KBUILD_MODNAME, __func__
+
+#include <net/netlink.h>
+#include <net/net_namespace.h>
+#include <net/sock.h>
+#include <rdma/rdma_netlink.h>
+
+struct ibnl_client {
+	struct list_head		list;
+	int				index;
+	int				nops;
+	const struct ibnl_client_cbs   *cb_table;
+};
+
+static DEFINE_MUTEX(ibnl_mutex);
+static struct sock *nls;
+static LIST_HEAD(client_list);
+
+int ibnl_add_client(int index, int nops,
+		    const struct ibnl_client_cbs cb_table[])
+{
+	struct ibnl_client *cur;
+	struct ibnl_client *nl_client;
+
+	nl_client = kmalloc(sizeof *nl_client, GFP_KERNEL);
+	if (!nl_client)
+		return -ENOMEM;
+
+	nl_client->index	= index;
+	nl_client->nops		= nops;
+	nl_client->cb_table	= cb_table;
+
+	mutex_lock(&ibnl_mutex);
+
+	list_for_each_entry(cur, &client_list, list) {
+		if (cur->index == index) {
+			pr_warn("Client for %d already exists\n", index);
+			mutex_unlock(&ibnl_mutex);
+			kfree(nl_client);
+			return -EINVAL;
+		}
+	}
+
+	list_add_tail(&nl_client->list, &client_list);
+
+	mutex_unlock(&ibnl_mutex);
+
+	return 0;
+}
+EXPORT_SYMBOL(ibnl_add_client);
+
+int ibnl_remove_client(int index)
+{
+	struct ibnl_client *cur, *next;
+
+	mutex_lock(&ibnl_mutex);
+	list_for_each_entry_safe(cur, next, &client_list, list) {
+		if (cur->index == index) {
+			list_del(&(cur->list));
+			mutex_unlock(&ibnl_mutex);
+			kfree(cur);
+			return 0;
+		}
+	}
+	pr_warn("Can't remove callback for client idx %d. Not found\n", index);
+	mutex_unlock(&ibnl_mutex);
+
+	return -EINVAL;
+}
+EXPORT_SYMBOL(ibnl_remove_client);
+
+void *ibnl_put_msg(struct sk_buff *skb, struct nlmsghdr **nlh, int seq,
+		   int len, int client, int op)
+{
+	unsigned char *prev_tail;
+
+	prev_tail = skb_tail_pointer(skb);
+	*nlh = NLMSG_NEW(skb, 0, seq, RDMA_NL_GET_TYPE(client, op),
+			len, NLM_F_MULTI);
+	(*nlh)->nlmsg_len = skb_tail_pointer(skb) - prev_tail;
+	return NLMSG_DATA(*nlh);
+
+nlmsg_failure:
+	nlmsg_trim(skb, prev_tail);
+	return NULL;
+}
+EXPORT_SYMBOL(ibnl_put_msg);
+
+int ibnl_put_attr(struct sk_buff *skb, struct nlmsghdr *nlh,
+		  int len, void *data, int type)
+{
+	unsigned char *prev_tail;
+
+	prev_tail = skb_tail_pointer(skb);
+	NLA_PUT(skb, type, len, data);
+	nlh->nlmsg_len += skb_tail_pointer(skb) - prev_tail;
+	return 0;
+
+nla_put_failure:
+	nlmsg_trim(skb, prev_tail - nlh->nlmsg_len);
+	return -EMSGSIZE;
+}
+EXPORT_SYMBOL(ibnl_put_attr);
+
+static int ibnl_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh)
+{
+	struct ibnl_client *client;
+	int type = nlh->nlmsg_type;
+	int index = RDMA_NL_GET_CLIENT(type);
+	int op = RDMA_NL_GET_OP(type);
+
+	list_for_each_entry(client, &client_list, list) {
+		if (client->index == index) {
+			if (op < 0 || op >= client->nops ||
+			    !client->cb_table[RDMA_NL_GET_OP(op)].dump)
+				return -EINVAL;
+			return netlink_dump_start(nls, skb, nlh,
+						  client->cb_table[op].dump,
+						  NULL);
+		}
+	}
+
+	pr_info("Index %d wasn't found in client list\n", index);
+	return -EINVAL;
+}
+
+static void ibnl_rcv(struct sk_buff *skb)
+{
+	mutex_lock(&ibnl_mutex);
+	netlink_rcv_skb(skb, &ibnl_rcv_msg);
+	mutex_unlock(&ibnl_mutex);
+}
+
+int ibnl_init(void)
+{
+	nls = netlink_kernel_create(&init_net, NETLINK_INFINIBAND, 0, ibnl_rcv,
+				    NULL, THIS_MODULE);
+	if (!nls) {
+		pr_warn("Failed to create netlink socket\n");
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+void ibnl_cleanup(void)
+{
+	struct ibnl_client *cur, *next;
+
+	mutex_lock(&ibnl_mutex);
+	list_for_each_entry_safe(cur, next, &client_list, list) {
+		list_del(&(cur->list));
+		kfree(cur);
+	}
+	mutex_unlock(&ibnl_mutex);
+
+	netlink_kernel_release(nls);
+}
diff --git a/include/linux/netlink.h b/include/linux/netlink.h
index 4c4ac3f..b1e3e59 100644
--- a/include/linux/netlink.h
+++ b/include/linux/netlink.h
@@ -24,6 +24,7 @@
 /* leave room for NETLINK_DM (DM Events) */
 #define NETLINK_SCSITRANSPORT	18	/* SCSI Transports */
 #define NETLINK_ECRYPTFS	19
+#define NETLINK_INFINIBAND	20
 
 #define MAX_LINKS 32		
 
diff --git a/include/rdma/rdma_netlink.h b/include/rdma/rdma_netlink.h
new file mode 100644
index 0000000..c983a19
--- /dev/null
+++ b/include/rdma/rdma_netlink.h
@@ -0,0 +1,64 @@
+#ifndef _RDMA_NETLINK_H
+#define _RDMA_NETLINK_H
+
+#define RDMA_NL_GET_CLIENT(type) ((type & (((1 << 6) - 1) << 10)) >> 10)
+#define RDMA_NL_GET_OP(type) (type & ((1 << 10) - 1))
+#define RDMA_NL_GET_TYPE(client, op) ((client << 10) + op)
+
+#ifdef __KERNEL__
+
+#include <linux/netlink.h>
+
+struct ibnl_client_cbs {
+	int (*dump)(struct sk_buff *skb, struct netlink_callback *nlcb);
+};
+
+int ibnl_init(void);
+void ibnl_cleanup(void);
+
+/**
+ * Add a a client to the list of IB netlink exporters.
+ * @index: Index of the added client
+ * @nops: Number of supported ops by the added client.
+ * @cb_table: A table for op->callback
+ *
+ * Returns 0 on success or a negative error code.
+ */
+int ibnl_add_client(int index, int nops,
+		    const struct ibnl_client_cbs cb_table[]);
+
+/**
+ * Remove a client from IB netlink.
+ * @index: Index of the removed IB client.
+ *
+ * Returns 0 on success or a negative error code.
+ */
+int ibnl_remove_client(int index);
+
+/**
+ * Put a new message in a supplied skb.
+ * @skb: The netlink skb.
+ * @nlh: Pointer to put the header of the new netlink message.
+ * @seq: The message sequence number.
+ * @len: The requested message length to allocate.
+ * @client: Calling IB netlink client.
+ * @op: message content op.
+ * Returns the allocated buffer on success and NULL on failure.
+ */
+void *ibnl_put_msg(struct sk_buff *skb, struct nlmsghdr **nlh, int seq,
+		   int len, int client, int op);
+/**
+ * Put a new attribute in a supplied skb.
+ * @skb: The netlink skb.
+ * @nlh: Header of the netlink message to append the attribute to.
+ * @len: The length of the attribute data.
+ * @data: The attribute data to put.
+ * @type: The attribute type.
+ * Returns the 0 and a negative error code on failure.
+ */
+int ibnl_put_attr(struct sk_buff *skb, struct nlmsghdr *nlh,
+		  int len, void *data, int type);
+
+#endif /* __KERNEL__ */
+
+#endif /* _RDMA_NETLINK_H */
-- 
1.7.4.1

Re: [RFC 1/3] RDMA: Add netlink infrastructure

[Joe Perches]

On Fri, 2011-05-13 at 09:18 -0700, Roland Dreier wrote:
> From: Roland Dreier <roland-BHEL68pLQRGGvPXPguhicg@public.gmane.org>
> [Dave please do not apply even if this ends up in netdev patchwork!]

Just trivia:

> diff --git a/drivers/infiniband/core/netlink.c b/drivers/infiniband/core/netlink.c
[]
> +#define pr_fmt(fmt) "%s:%s: " fmt, KBUILD_MODNAME, __func__

Using #define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
generally produces smaller overall object size, especially
at 64 bit.

For instance, here's the size of netlink.o at 32 bit:

$ size drivers/infiniband/core/netlink.o.*
   text	   data	    bss	    dec	    hex	filename
   2663	    153	    736	   3552	    de0	drivers/infiniband/core/netlink.o.old
   2640	    153	    736	   3529	    dc9	drivers/infiniband/core/netlink.o.new

Also, I rarely find __func__ useful in message output.
It may be more useful for active development/debugging.

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [RFC 1/3] RDMA: Add netlink infrastructure

[Bart Van Assche]

On Fri, May 13, 2011 at 6:44 PM, Joe Perches <joe-6d6DIl74uiNBDgjK7y7TUQ@public.gmane.org> wrote:
> On Fri, 2011-05-13 at 09:18 -0700, Roland Dreier wrote:
>> From: Roland Dreier <roland-BHEL68pLQRGGvPXPguhicg@public.gmane.org>
>> [Dave please do not apply even if this ends up in netdev patchwork!]
>
> Just trivia:
>
>> diff --git a/drivers/infiniband/core/netlink.c b/drivers/infiniband/core/netlink.c
> []
>> +#define pr_fmt(fmt) "%s:%s: " fmt, KBUILD_MODNAME, __func__
>
> Using #define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
> generally produces smaller overall object size, especially
> at 64 bit.
>
> For instance, here's the size of netlink.o at 32 bit:
>
> $ size drivers/infiniband/core/netlink.o.*
>   text    data     bss     dec     hex filename
>   2663     153     736    3552     de0 drivers/infiniband/core/netlink.o.old
>   2640     153     736    3529     dc9 drivers/infiniband/core/netlink.o.new
>
> Also, I rarely find __func__ useful in message output.
> It may be more useful for active development/debugging.

A recent dynamic debug patch made it possible to enable/disable at
runtime whether or not the function name (and more) should be included
in the output. See also http://lwn.net/Articles/434833/ for more
information.

Bart.
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [RFC 1/3] RDMA: Add netlink infrastructure

[Joe Perches]

On Fri, 2011-05-13 at 19:18 +0200, Bart Van Assche wrote:
> On Fri, May 13, 2011 at 6:44 PM, Joe Perches <joe-6d6DIl74uiNBDgjK7y7TUQ@public.gmane.org> wrote:
> > On Fri, 2011-05-13 at 09:18 -0700, Roland Dreier wrote:
> >> From: Roland Dreier <roland-BHEL68pLQRGGvPXPguhicg@public.gmane.org>
> >> [Dave please do not apply even if this ends up in netdev patchwork!]
> >> diff --git a/drivers/infiniband/core/netlink.c b/drivers/infiniband/core/netlink.c
> > []
> >> +#define pr_fmt(fmt) "%s:%s: " fmt, KBUILD_MODNAME, __func__
> > Using #define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
> > generally produces smaller overall object size, especially
> > at 64 bit.
> > For instance, here's the size of netlink.o at 32 bit:
> > $ size drivers/infiniband/core/netlink.o.*
> >   text    data     bss     dec     hex filename
> >   2663     153     736    3552     de0 drivers/infiniband/core/netlink.o.old
> >   2640     153     736    3529     dc9 drivers/infiniband/core/netlink.o.new
> > Also, I rarely find __func__ useful in message output.
> > It may be more useful for active development/debugging.
> A recent dynamic debug patch made it possible to enable/disable at
> runtime whether or not the function name (and more) should be included
> in the output. See also http://lwn.net/Articles/434833/ for more
> information.

One long term goal for me is a generic run-time mechanism
to prefix all pr_<level> uses not just the <foo>_dbg ones
with or without module or function name.

It will also allow the removal of all the uses of
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

New #define pr_fmt lines need to be added to the current
files that use pr_<level> without a prefix so it could
take awhile.

Eventually.

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [RFC 1/3] RDMA: Add netlink infrastructure

[Bart Van Assche]

On Fri, May 13, 2011 at 7:36 PM, Joe Perches <joe@perches.com> wrote:
> On Fri, 2011-05-13 at 19:18 +0200, Bart Van Assche wrote:
>> A recent dynamic debug patch made it possible to enable/disable at
>> runtime whether or not the function name (and more) should be included
>> in the output. See also http://lwn.net/Articles/434833/ for more
>> information.
>
> One long term goal for me is a generic run-time mechanism
> to prefix all pr_<level> uses not just the <foo>_dbg ones
> with or without module or function name.
>
> It will also allow the removal of all the uses of
> #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
>
> New #define pr_fmt lines need to be added to the current
> files that use pr_<level> without a prefix so it could
> take awhile.

Something like the +m flag documented in
http://git.kernel.org/?p=linux/kernel/git/torvalds/linux-2.6.git;a=blob;f=Documentation/dynamic-debug-howto.txt;hb=HEAD
?

Bart.

Re: [RFC 1/3] RDMA: Add netlink infrastructure

[Joe Perches]

On Fri, 2011-05-13 at 20:12 +0200, Bart Van Assche wrote:
> On Fri, May 13, 2011 at 7:36 PM, Joe Perches <joe-6d6DIl74uiNBDgjK7y7TUQ@public.gmane.org> wrote:
> > On Fri, 2011-05-13 at 19:18 +0200, Bart Van Assche wrote:
> > One long term goal for me is a generic run-time mechanism
> > to prefix all pr_<level> uses not just the <foo>_dbg ones
> > with or without module or function name.
> > It will also allow the removal of all the uses of
> > #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
> > New #define pr_fmt lines need to be added to the current
> > files that use pr_<level> without a prefix so it could
> > take awhile.
> Something like the +m flag documented in
> http://git.kernel.org/?p=linux/kernel/git/torvalds/linux-2.6.git;a=blob;f=Documentation/dynamic-debug-howto.txt;hb=HEAD
> ?

Yes, akin to that, but using /proc not debugfs.

Basically, just per module on/off prefixing of
module name and optionally function name via a
CONFIG_PR_LEVEL_STORE_FUNCTION_NAME.

I'd like to remove all current specific uses of
__func__ in pr_<level> and use per module
/proc controls to add them back as requested.

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

RE: [RFC 1/3] RDMA: Add netlink infrastructure

[Hefty, Sean]

> +#define NETLINK_INFINIBAND	20

Would NETLINK_RDMA be better?
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [RFC 1/3] RDMA: Add netlink infrastructure

[Roland Dreier]

On Fri, May 13, 2011 at 10:19 AM, Hefty, Sean <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org> wrote:
>> +#define NETLINK_INFINIBAND   20
>
> Would NETLINK_RDMA be better?

Yes, I tried to change all the externally visible examples of IB to RDMA, but I
missed that one (I figure internal kernel APIs can be cleaned up
later).  I'll update my tree.

 - R.
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[RFC 2/3] RDMA/cma: Add support for netlink statistics export

[Roland Dreier]

From: Nir Muchtar <nirm-smomgflXvOZWk0Htik3J/w@public.gmane.org>

[Dave please do not apply even if this ends up in netdev patchwork!]

Add callbacks and data types for statistics export of all current
devices/ids.  The schema for RDMA CM is a series of netlink messages.
Each one contains an rdma_cm_stat struct.  Additionally, two netlink
attributes are created for the addresses for each message (if
applicable).

Their types used are:
RDMA_NL_RDMA_CM_ATTR_SRC_ADDR (The source address for this ID)
RDMA_NL_RDMA_CM_ATTR_DST_ADDR (The destination address for this ID)
sockaddr_* structs are encapsulated within these attributes.

In other words, every transaction contains a series of messages like:

-------message 1-------
struct rdma_cm_id_stats {
       __u32 qp_num;
       __u32 bound_dev_if;
       __u32 port_space;
       __s32 pid;
       __u8 cm_state;
       __u8 node_type;
       __u8 port_num;
       __u8 reserved;
}
RDMA_NL_RDMA_CM_ATTR_SRC_ADDR attribute - contains the source address
RDMA_NL_RDMA_CM_ATTR_DST_ADDR attribute - contains the destination address
-------end 1-------
-------message 2-------
struct rdma_cm_id_stats
RDMA_NL_RDMA_CM_ATTR_SRC_ADDR attribute
RDMA_NL_RDMA_CM_ATTR_DST_ADDR attribute
-------end 2-------

NOT-Signed-off-by: Nir Muchtar <nirm-smomgflXvOZWk0Htik3J/w@public.gmane.org>
NOT-Signed-off-by: Roland Dreier <roland-BHEL68pLQRGGvPXPguhicg@public.gmane.org>
---
 drivers/infiniband/core/cma.c |   98 +++++++++++++++++++++++++++++++++++++++++
 include/rdma/rdma_netlink.h   |   28 ++++++++++++
 2 files changed, 126 insertions(+), 0 deletions(-)

diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c
index 05b55e4..d4701a8 100644
--- a/drivers/infiniband/core/cma.c
+++ b/drivers/infiniband/core/cma.c
@@ -47,6 +47,7 @@
 
 #include <rdma/rdma_cm.h>
 #include <rdma/rdma_cm_ib.h>
+#include <rdma/rdma_netlink.h>
 #include <rdma/ib_cache.h>
 #include <rdma/ib_cm.h>
 #include <rdma/ib_sa.h>
@@ -3247,6 +3248,98 @@ static void cma_remove_one(struct ib_device *device)
 	kfree(cma_dev);
 }
 
+static int cma_get_id_stats(struct sk_buff *skb, struct netlink_callback *cb)
+{
+	struct nlmsghdr *nlh;
+	struct rdma_cm_id_stats *id_stats;
+	struct rdma_id_private *id_priv;
+	struct rdma_cm_id *id = NULL;
+	struct cma_device *cma_dev;
+	int i_dev = 0, i_id = 0;
+
+	/*
+	 * We export all of the IDs as a sequence of messages.  Each
+	 * ID gets its own netlink message.
+	 */
+	mutex_lock(&lock);
+
+	list_for_each_entry(cma_dev, &dev_list, list) {
+		if (i_dev < cb->args[0]) {
+			i_dev++;
+			continue;
+		}
+
+		i_id = 0;
+		list_for_each_entry(id_priv, &cma_dev->id_list, list) {
+			if (i_id < cb->args[1]) {
+				i_id++;
+				continue;
+			}
+
+			id_stats = ibnl_put_msg(skb, &nlh, cb->nlh->nlmsg_seq,
+						sizeof *id_stats, RDMA_NL_RDMA_CM,
+						RDMA_NL_RDMA_CM_ID_STATS);
+			if (!id_stats)
+				goto out;
+
+			memset(id_stats, 0, sizeof *id_stats);
+			id = &id_priv->id;
+			id_stats->node_type = id->route.addr.dev_addr.dev_type;
+			id_stats->port_num = id->port_num;
+			id_stats->bound_dev_if =
+				id->route.addr.dev_addr.bound_dev_if;
+
+			if (id->route.addr.src_addr.ss_family == AF_INET) {
+				if (ibnl_put_attr(skb, nlh,
+						  sizeof(struct sockaddr_in),
+						  &id->route.addr.src_addr,
+						  RDMA_NL_RDMA_CM_ATTR_SRC_ADDR)) {
+					goto out;
+				}
+				if (ibnl_put_attr(skb, nlh,
+						  sizeof(struct sockaddr_in),
+						  &id->route.addr.dst_addr,
+						  RDMA_NL_RDMA_CM_ATTR_DST_ADDR)) {
+					goto out;
+				}
+			} else if (id->route.addr.src_addr.ss_family == AF_INET6) {
+				if (ibnl_put_attr(skb, nlh,
+						  sizeof(struct sockaddr_in6),
+						  &id->route.addr.src_addr,
+						  RDMA_NL_RDMA_CM_ATTR_SRC_ADDR)) {
+					goto out;
+				}
+				if (ibnl_put_attr(skb, nlh,
+						  sizeof(struct sockaddr_in6),
+						  &id->route.addr.dst_addr,
+						  RDMA_NL_RDMA_CM_ATTR_DST_ADDR)) {
+					goto out;
+				}
+			}
+
+			id_stats->port_space = id->ps;
+			id_stats->cm_state = id_priv->state;
+			id_stats->qp_num = id_priv->qp_num;
+
+			i_id++;
+		}
+
+		cb->args[1] = 0;
+		i_dev++;
+	}
+
+out:
+	mutex_unlock(&lock);
+	cb->args[0] = i_dev;
+	cb->args[1] = i_id;
+
+	return skb->len;
+}
+
+static const struct ibnl_client_cbs cma_cb_table[] = {
+	[RDMA_NL_RDMA_CM_ID_STATS] = { .dump = cma_get_id_stats },
+};
+
 static int __init cma_init(void)
 {
 	int ret;
@@ -3262,6 +3355,10 @@ static int __init cma_init(void)
 	ret = ib_register_client(&cma_client);
 	if (ret)
 		goto err;
+
+	if (ibnl_add_client(RDMA_NL_RDMA_CM, RDMA_NL_RDMA_CM_NUM_OPS, cma_cb_table))
+		printk(KERN_WARNING "RDMA CMA: failed to add netlink callback\n");
+
 	return 0;
 
 err:
@@ -3274,6 +3371,7 @@ err:
 
 static void __exit cma_cleanup(void)
 {
+	ibnl_remove_client(RDMA_NL_RDMA_CM);
 	ib_unregister_client(&cma_client);
 	unregister_netdevice_notifier(&cma_nb);
 	rdma_addr_unregister_client(&addr_client);
diff --git a/include/rdma/rdma_netlink.h b/include/rdma/rdma_netlink.h
index c983a19..fa318af 100644
--- a/include/rdma/rdma_netlink.h
+++ b/include/rdma/rdma_netlink.h
@@ -1,10 +1,38 @@
 #ifndef _RDMA_NETLINK_H
 #define _RDMA_NETLINK_H
 
+#include <linux/types.h>
+
+enum {
+	RDMA_NL_RDMA_CM = 1
+};
+
 #define RDMA_NL_GET_CLIENT(type) ((type & (((1 << 6) - 1) << 10)) >> 10)
 #define RDMA_NL_GET_OP(type) (type & ((1 << 10) - 1))
 #define RDMA_NL_GET_TYPE(client, op) ((client << 10) + op)
 
+enum {
+	RDMA_NL_RDMA_CM_ID_STATS = 0,
+	RDMA_NL_RDMA_CM_NUM_OPS
+};
+
+enum {
+	RDMA_NL_RDMA_CM_ATTR_SRC_ADDR = 1,
+	RDMA_NL_RDMA_CM_ATTR_DST_ADDR,
+	RDMA_NL_RDMA_CM_NUM_ATTR,
+};
+
+struct rdma_cm_id_stats {
+	__u32	qp_num;
+	__u32	bound_dev_if;
+	__u32	port_space;
+	__s32	pid;
+	__u8	cm_state;
+	__u8	node_type;
+	__u8	port_num;
+	__u8	reserved;
+};
+
 #ifdef __KERNEL__
 
 #include <linux/netlink.h>
-- 
1.7.4.1

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

RE: [RFC 2/3] RDMA/cma: Add support for netlink statistics export

[Hefty, Sean]

> +struct rdma_cm_id_stats {
> +	__u32	qp_num;
> +	__u32	bound_dev_if;
> +	__u32	port_space;
> +	__s32	pid;
> +	__u8	cm_state;
> +	__u8	node_type;
> +	__u8	port_num;
> +	__u8	reserved;
> +};

We may also want to add qp_type

Re: [RFC 2/3] RDMA/cma: Add support for netlink statistics export

[Or Gerlitz]

Hefty, Sean wrote:
>> +struct rdma_cm_id_stats {
>> +	__u32	qp_num;
>> +	__u32	bound_dev_if;
>> +	__u32	port_space;
>> +	__s32	pid;
>> +	__u8	cm_state;
>> +	__u8	node_type;
>> +	__u8	port_num;
>> +	__u8	reserved;
>> +};
> 
> We may also want to add qp_type

Sean,

Isn't the port space enough here? specifically, what qp type buys us 
over port space?

Or.
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

RE: [RFC 2/3] RDMA/cma: Add support for netlink statistics export

[Hefty, Sean]

> Isn't the port space enough here? specifically, what qp type buys us
> over port space?

That assumes a 1:1 relationship between the port space and QP type, or more specifically, that the IB SID range is divided based on QP type.  This is outside of the architecture.  If we look at UC or XRC, it's not clear what port space those map to, but I believe it's desirable for the rdma_cm to support those.  Note that the AF_IB patch set ended up requiring that the user specify the qp type for these reasons.

- Sean
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [RFC 2/3] RDMA/cma: Add support for netlink statistics export

[Or Gerlitz]

Hefty, Sean <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org> wrote:

> That assumes a 1:1 relationship between the port space and QP type, or more
> specifically, that the IB SID range is divided based on QP type.  This is outside of the
> architecture.  If we look at UC or XRC, it's not clear what port space those map to, but I
> believe it's desirable for the rdma_cm to support those.

makes sense, so we can add the qp type here.

Or.
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [RFC 2/3] RDMA/cma: Add support for netlink statistics export

[Roland Dreier]

On Fri, May 13, 2011 at 10:21 AM, Hefty, Sean <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org> wrote:
>> +struct rdma_cm_id_stats {
>> +     __u32   qp_num;
>> +     __u32   bound_dev_if;
>> +     __u32   port_space;
>> +     __s32   pid;
>> +     __u8    cm_state;
>> +     __u8    node_type;
>> +     __u8    port_num;
>> +     __u8    reserved;
>> +};
>
> We may also want to add qp_type

Is there an easy way to get the qp_type from a struct rdma_cm_id?

ie what code needs to go into cma_get_id_stats() to handle this?

Thanks,
  Roland
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

RE: [RFC 2/3] RDMA/cma: Add support for netlink statistics export

[Hefty, Sean]

> Is there an easy way to get the qp_type from a struct rdma_cm_id?
> 
> ie what code needs to go into cma_get_id_stats() to handle this?

With the current code, you'd need to map from the port space:

static inline enum ib_qp_type cma_get_qp_type(struct rdma_cm_id *id)
{
	if (id->ps == RDMA_PS_IPOIB || id->ps == RDMA_PS_UDP)
		return IB_QPT_RC;
	else
		return IB_QPT_RC;
}

	..
	id_stats->qp_type = cma_get_qp_type(id);
	..

One of the patches in the af_ib patch set adds the qp_type to struct rdma_cm_id.  I'm guessing that patch will also be needed for xrc.

- Sean

Re: [RFC 2/3] RDMA/cma: Add support for netlink statistics export

[Roland Dreier]

On Thu, May 19, 2011 at 11:35 AM, Hefty, Sean <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org> wrote:
> One of the patches in the af_ib patch set adds the qp_type to struct rdma_cm_id.  I'm guessing that patch will also be needed for xrc.

Would it make sense to pull that patch at least in?

Sorry I haven't had time to really think about AF_IB in general but
maybe I can at least merge the netlink stuff this cycle?

 - R.
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [RFC 2/3] RDMA/cma: Add support for netlink statistics export

[Roland Dreier]

On Thu, May 19, 2011 at 11:49 AM, Roland Dreier <roland-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org> wrote:
> On Thu, May 19, 2011 at 11:35 AM, Hefty, Sean <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org> wrote:
>> One of the patches in the af_ib patch set adds the qp_type to struct rdma_cm_id.  I'm guessing that patch will also be needed for xrc.
>
> Would it make sense to pull that patch at least in?
>
> Sorry I haven't had time to really think about AF_IB in general but
> maybe I can at least merge the netlink stuff this cycle?

And actually looking at https://patchwork.kernel.org/patch/90204/
it looks reasonable to merge now... or I could just use
cma_is_ud_ps() in the netlink stuff.

I'm inclined to just take your patch now if that makes sense to you
(it looks pretty independent of everything else)

Is that the latest version in patchwork?

Thanks,
 - R.
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

RE: [RFC 2/3] RDMA/cma: Add support for netlink statistics export

[Hefty, Sean]

> And actually looking at https://patchwork.kernel.org/patch/90204/
> it looks reasonable to merge now... or I could just use
> cma_is_ud_ps() in the netlink stuff.
> 
> I'm inclined to just take your patch now if that makes sense to you
> (it looks pretty independent of everything else)
> 
> Is that the latest version in patchwork?

The only difference that I see between that patch and what's in my tree are the line numbers.  The patch looks safe enough to me to go in separately from anything else.  Thanks

- Sean
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[RFC 3/3] RDMA/cma: Save PID of ID's owner

[Roland Dreier]

From: Nir Muchtar <nirm@voltaire.com>

[Dave please do not apply even if this ends up in netdev patchwork!]

Save the PID associated with an RDMA CM ID for reporting via netlink.

NOT-Signed-off-by: Nir Muchtar <nirm@voltaire.com>
NOT-Signed-off-by: Roland Dreier <roland@purestorage.com>
---
 drivers/infiniband/core/cma.c |    6 ++++++
 1 files changed, 6 insertions(+), 0 deletions(-)

diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c
index d4701a8..1e25434 100644
--- a/drivers/infiniband/core/cma.c
+++ b/drivers/infiniband/core/cma.c
@@ -133,6 +133,7 @@ struct rdma_id_private {
 	u32			seq_num;
 	u32			qkey;
 	u32			qp_num;
+	pid_t			owner;
 	u8			srq;
 	u8			tos;
 };
@@ -423,6 +424,7 @@ struct rdma_cm_id *rdma_create_id(rdma_cm_event_handler event_handler,
 	if (!id_priv)
 		return ERR_PTR(-ENOMEM);
 
+	id_priv->owner = task_pid_nr(current);
 	id_priv->state = RDMA_CM_IDLE;
 	id_priv->id.context = context;
 	id_priv->id.event_handler = event_handler;
@@ -2678,6 +2680,9 @@ int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
 	int ret;
 
 	id_priv = container_of(id, struct rdma_id_private, id);
+
+	id_priv->owner = task_pid_nr(current);
+
 	if (!cma_comp(id_priv, RDMA_CM_CONNECT))
 		return -EINVAL;
 
@@ -3320,6 +3325,7 @@ static int cma_get_id_stats(struct sk_buff *skb, struct netlink_callback *cb)
 			id_stats->port_space = id->ps;
 			id_stats->cm_state = id_priv->state;
 			id_stats->qp_num = id_priv->qp_num;
+			id_stats->pid = id_priv->owner;
 
 			i_id++;
 		}
-- 
1.7.4.1