public inbox for linux-rdma@vger.kernel.org
 help / color / mirror / Atom feed
* First byte of private_data transfered with rdma_connect() is always 0
@ 2014-12-16 14:32 Fabian Holler
       [not found] ` <20141216143242.GB4401-99BIx50xQYGELgA04lAiVw@public.gmane.org>
  0 siblings, 1 reply; 2+ messages in thread
From: Fabian Holler @ 2014-12-16 14:32 UTC (permalink / raw)
  To: linux-rdma-u79uwXL29TY76Z2rM5mHXA

[-- Attachment #1: Type: text/plain, Size: 2235 bytes --]

Hello,

we are using the conn_param->private_data field to transfer data with the
rdma_connect() call to the server.
When it's done on a RDMA_PS_IB rdma_cm_id the first Byte of the
private_data, that is received by the server is _always_ 0.
When a RDMA_PS_TCP rdma_cm_id is used, the data is received correctly on the
server.

We are using:
- kernel 3.14.13
- Mellanox Technologies MT26428 HCA
- mlx4_0 driver

I attached a simple client and server module to reproduce the behaviour.
Can somebody have a look? Is there a problem in our modules? Or is it a bug?

----------
Connection establishment via GID (RDMA_PS_IB):
client:
    # insmod client.ko gid_addr=fe80:0000:0000:0000:0002:c903:0010:c0f5
    [ 7328.586773] private_dataffff88022a263c50: 57 48 41 5a 20 55 50 20 53 45 52 56 45 52 3f 00  WHAZ UP SERVER?.
    
server:
    [ 1658.208238] private_dataffff8800b93e3bec: 00 48 41 5a 20 55 50 20 53 45 52 56 45 52 3f 00  .HAZ UP SERVER?.
    [ 1658.208239] private_dataffff8800b93e3bfc: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00  ................
    [ 1658.208241] private_dataffff8800b93e3c0c: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00  ................
    [ 1658.208242] private_dataffff8800b93e3c1c: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00  ................
    [ 1658.208244] private_dataffff8800b93e3c2c: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00  ................
    [ 1658.208245] private_dataffff8800b93e3c3c: 00 00 00 00 00 00 00 00 00 00 00 00              ............


Connection establishment via IPv4 address (RDMA_PS_TCP):
client:
    # insmod client.ko ip_addr=10.50.100.62
    [ 7179.219773] private_dataffff88022a263c50: 57 48 41 5a 20 55 50 20 53 45 # 52 56 45 52 3f 00  WHAZ UP SERVER?.
    
server:
    [ 1508.840508] private_dataffff8800b8d25b90: 57 48 41 5a 20 55 50 20 53 45 52 56 45 52 3f 00  WHAZ UP SERVER?.
    [ 1508.840509] private_dataffff8800b8d25ba0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00  ................
    [ 1508.840511] private_dataffff8800b8d25bb0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00  ................
    [ 1508.840512] private_dataffff8800b8d25bc0: 00 00 00 00 00 00 00 00                          ........
----------


thanks

Fabian

[-- Attachment #2: client.c --]
[-- Type: text/x-csrc, Size: 4860 bytes --]

#include <linux/module.h>
#include <rdma/rdma_cm.h>
#include <rdma/ib.h>
#include <linux/inet.h>
#include <linux/wait.h>

MODULE_LICENSE("GPL");

static char *ip_addr;
module_param(ip_addr, charp, 0444);

static char *gid_addr;
module_param(gid_addr, charp, 0444);

#define SERVER_PORT 1234

#define my_printk(level, format, arg...) printk(level KBUILD_MODNAME \
		" %s(), %d: " \
		format, __func__, __LINE__, ## arg)
#define LOG(format, arg...) my_printk(KERN_INFO, format, ## arg)

struct con {
	struct rdma_cm_id	*cm_id;
	enum rdma_cm_event_type	ev;
	wait_queue_head_t	wait_q;
};

static struct con con;

#define XX(a) case (a): return #a
static inline const char *rdma_event_str(enum rdma_cm_event_type event)
{
	switch (event) {
	XX(RDMA_CM_EVENT_ADDR_RESOLVED);
	XX(RDMA_CM_EVENT_ADDR_ERROR);
	XX(RDMA_CM_EVENT_ROUTE_RESOLVED);
	XX(RDMA_CM_EVENT_ROUTE_ERROR);
	XX(RDMA_CM_EVENT_CONNECT_REQUEST);
	XX(RDMA_CM_EVENT_CONNECT_RESPONSE);
	XX(RDMA_CM_EVENT_CONNECT_ERROR);
	XX(RDMA_CM_EVENT_UNREACHABLE);
	XX(RDMA_CM_EVENT_REJECTED);
	XX(RDMA_CM_EVENT_ESTABLISHED);
	XX(RDMA_CM_EVENT_DISCONNECTED);
	XX(RDMA_CM_EVENT_DEVICE_REMOVAL);
	XX(RDMA_CM_EVENT_MULTICAST_JOIN);
	XX(RDMA_CM_EVENT_MULTICAST_ERROR);
	XX(RDMA_CM_EVENT_ADDR_CHANGE);
	XX(RDMA_CM_EVENT_TIMEWAIT_EXIT);
	default: return "RDMA_CM_UNKNOWN";
	}
}

static int rdma_cm_ev_handler(struct rdma_cm_id *cm_id,
			      struct rdma_cm_event *event)
{
	struct con *con = cm_id->context;

	LOG("CM event %s, error %d\n", rdma_event_str(event->event),
	     event->status);

	con->ev = event->event;
	wake_up_interruptible(&con->wait_q);
	return 0;
}

static int str_gid_to_sockaddr(const char *gid, struct sockaddr_ib *dst)
{
	int ret;

	ret = in6_pton(gid, strlen(gid),
		       dst->sib_addr.sib_raw, '\n', NULL);
	if (ret == 0)
		return -EINVAL;

	dst->sib_family = AF_IB;
	dst->sib_sid = cpu_to_be64(RDMA_IB_IP_PS_IB | SERVER_PORT);
	dst->sib_sid_mask = cpu_to_be64(0xffffffffffffffffULL);
	dst->sib_pkey = cpu_to_be16(0xffff);
	LOG("Converted %s to binary GID %pI6\n", gid, dst->sib_addr.sib_raw);
	return 0;
}

static int str_ip_to_sockaddr(const char *ipaddr, struct sockaddr_storage *dst)
{
	int ret;
	u8 ip4[4];

	ret = in6_pton(ipaddr, strlen(ipaddr),
		       ((struct sockaddr_in6 *)dst)->sin6_addr.s6_addr,
		       '\n', NULL);
	if (ret == 1) {
		dst->ss_family = AF_INET6;
		((struct sockaddr_in6 *)dst)->sin6_port =
						htons(SERVER_PORT);
		LOG("Converted %s to binary IPv6 %pI6\n", ipaddr,
		    ((struct sockaddr_in6 *)dst)->sin6_addr.s6_addr);
		return 0;
	}

	ret = in4_pton(ipaddr, strlen(ipaddr), ip4, '\n', NULL);
	if (ret == 0)
		return -EINVAL;

	memcpy(&((struct sockaddr_in *)dst)->sin_addr.s_addr, ip4,
	       sizeof(((struct sockaddr_in *)dst)->sin_addr.s_addr));
	dst->ss_family = AF_INET;
	((struct sockaddr_in *)dst)->sin_port = htons(SERVER_PORT);
	LOG("Converted %s to binary IPv4 %pI4\n", ipaddr,
	    &((struct sockaddr_in *)dst)->sin_addr.s_addr);
	return 0;
}

static int fill_sockaddr(struct sockaddr_storage *s)
{
	if (gid_addr != NULL && strlen(gid_addr) > 0)
		return str_gid_to_sockaddr(gid_addr, (struct sockaddr_ib *)s);
	else if (ip_addr != NULL && strlen(ip_addr) > 0)
		return str_ip_to_sockaddr(ip_addr, s);
	return -EINVAL;
}

static int __init in(void)
{
	int ret;
	struct sockaddr_storage addr;
	struct rdma_conn_param conn_param;
	const char msg[] = "WHAZ UP SERVER?";

	con.ev = RDMA_CM_EVENT_DEVICE_REMOVAL;

	init_waitqueue_head(&con.wait_q);
	ret = fill_sockaddr(&addr);
	if (ret)
		return ret;

	if (addr.ss_family == AF_IB)
		con.cm_id = rdma_create_id(rdma_cm_ev_handler, &con, RDMA_PS_IB,
					    IB_QPT_RC);
	else
		con.cm_id = rdma_create_id(rdma_cm_ev_handler, &con,
					   RDMA_PS_TCP, IB_QPT_RC);
	if (IS_ERR(con.cm_id)) {
		ret = PTR_ERR(con.cm_id);
		return ret;
	}

	ret = rdma_resolve_addr(con.cm_id, NULL, (struct sockaddr *)&addr,
				1000);
	if (ret) {
		LOG("resolving addr failed\n");
		goto cm_id;
	}

	ret = wait_event_interruptible(con.wait_q, con.ev ==
				       RDMA_CM_EVENT_ADDR_RESOLVED);
	if (ret < 0)
		goto cm_id;

	ret = rdma_resolve_route(con.cm_id, 1000);
	if (ret) {
		LOG("resolving route failed\n");
		goto cm_id;
	}
	ret = wait_event_interruptible(con.wait_q, con.ev ==
				       RDMA_CM_EVENT_ROUTE_RESOLVED);
	if (ret < 0)
		goto cm_id;

	memset(&conn_param, 0, sizeof(conn_param));
	conn_param.private_data		= &msg;
	conn_param.private_data_len	= sizeof(msg);
	conn_param.retry_count = 7;
	print_hex_dump(KERN_INFO, "private_data", DUMP_PREFIX_ADDRESS, 16, 1,
		       conn_param.private_data, conn_param.private_data_len,
		       true);
	ret = rdma_connect(con.cm_id, &conn_param);
	if (ret) {
		LOG("Connect failed: %d\n", ret);
		goto cm_id;
	}

	rdma_disconnect(con.cm_id);
cm_id:
	rdma_destroy_id(con.cm_id);
	return ret;
}

static void __exit out(void)
{
}
module_init(in);
module_exit(out);

[-- Attachment #3: server.c --]
[-- Type: text/x-csrc, Size: 3391 bytes --]

#include <linux/module.h>
#include <rdma/rdma_cm.h>
#include <rdma/ib.h>
#include <linux/inet.h>


#define SERVER_PORT 1234

#define my_printk(level, format, arg...) printk(level KBUILD_MODNAME \
		" %s(), %d: " \
		format, __func__, __LINE__, ## arg)
#define LOG(format, arg...) my_printk(KERN_INFO, format, ## arg)

static const char msg[] = "WHAZ UP SERVER?";
static struct rdma_cm_id *cm_ip;
static struct rdma_cm_id *cm_ib;

#define XX(a) case (a): return #a
static inline const char *rdma_event_str(enum rdma_cm_event_type event)
{
	switch (event) {
	XX(RDMA_CM_EVENT_ADDR_RESOLVED);
	XX(RDMA_CM_EVENT_ADDR_ERROR);
	XX(RDMA_CM_EVENT_ROUTE_RESOLVED);
	XX(RDMA_CM_EVENT_ROUTE_ERROR);
	XX(RDMA_CM_EVENT_CONNECT_REQUEST);
	XX(RDMA_CM_EVENT_CONNECT_RESPONSE);
	XX(RDMA_CM_EVENT_CONNECT_ERROR);
	XX(RDMA_CM_EVENT_UNREACHABLE);
	XX(RDMA_CM_EVENT_REJECTED);
	XX(RDMA_CM_EVENT_ESTABLISHED);
	XX(RDMA_CM_EVENT_DISCONNECTED);
	XX(RDMA_CM_EVENT_DEVICE_REMOVAL);
	XX(RDMA_CM_EVENT_MULTICAST_JOIN);
	XX(RDMA_CM_EVENT_MULTICAST_ERROR);
	XX(RDMA_CM_EVENT_ADDR_CHANGE);
	XX(RDMA_CM_EVENT_TIMEWAIT_EXIT);
	default: return "RDMA_CM_UNKNOWN";
	}
}

static int rdma_cm_ev_handler(struct rdma_cm_id *cm_id,
			      struct rdma_cm_event *event)
{
	LOG("CM event %s, error %d\n", rdma_event_str(event->event),
	    event->status);

	switch (event->event) {
	case RDMA_CM_EVENT_CONNECT_REQUEST:
		print_hex_dump(KERN_INFO, "private_data", DUMP_PREFIX_ADDRESS,
			       16, 1, event->param.conn.private_data,
			       event->param.conn.private_data_len, true);
		return rdma_reject(cm_id, NULL, 0);

		return 0;
	default:
		return 0;
	}
}

static int ibtrs_srv_cm_init(struct rdma_cm_id **cm_id, struct sockaddr *addr,
			     enum rdma_port_space ps)
{
	int ret;

	*cm_id = rdma_create_id(rdma_cm_ev_handler, NULL, ps, IB_QPT_RC);
	if (IS_ERR(*cm_id)) {
		ret = PTR_ERR(*cm_id);
		goto err_out;
	}

	ret = rdma_bind_addr(*cm_id, addr);
	if (ret)
		goto err_cm;
	ret = rdma_listen(*cm_id, 64);
	if (ret) {
		LOG("rdma_listen() return %d\n", ret);
		goto err_cm;
	}

	switch (addr->sa_family) {
	case AF_INET:
		LOG("listening on port %u\n",
		    ((struct sockaddr_in *)addr)->sin_port);
		break;
	case AF_INET6:
		LOG("listening on port %u\n",
		    ((struct sockaddr_in6 *)addr)->sin6_port);
		break;
	case AF_IB:
		LOG("listening on service id 0x%016llx\n",
		    be64_to_cpu(rdma_get_service_id(*cm_id, addr)));
		break;
	default:
		LOG("listening on address family %u\n", addr->sa_family);
	}

	return 0;

err_cm:
	rdma_destroy_id(*cm_id);
err_out:
	return ret;
}

static int gogo(void)
{
	int ret;

	struct sockaddr_in6 sin = {
		.sin6_family = AF_INET6,
		.sin6_addr = IN6ADDR_ANY_INIT,
		.sin6_port = htons(SERVER_PORT),
	};
	struct sockaddr_ib sib = {
		.sib_family = AF_IB,
		.sib_addr.sib_subnet_prefix = 0ULL,
		.sib_addr.sib_interface_id = 0ULL,
		.sib_sid = cpu_to_be64(RDMA_IB_IP_PS_IB | SERVER_PORT),
		.sib_sid_mask = cpu_to_be64(0xffffffffffffffffULL),
		.sib_pkey = cpu_to_be16(0xffff),
	};

	ret = ibtrs_srv_cm_init(&cm_ip, (struct sockaddr *)&sin, RDMA_PS_TCP);
	if (ret)
		return ret;

	return ibtrs_srv_cm_init(&cm_ib, (struct sockaddr *)&sib, RDMA_PS_IB);
}

static int __init in(void)
{
	return gogo();
}

static void __exit out(void)
{
	rdma_disconnect(cm_ip);
	rdma_disconnect(cm_ib);
	rdma_destroy_id(cm_ip);
	rdma_destroy_id(cm_ib);
	return;
}
module_init(in);
module_exit(out);

^ permalink raw reply	[flat|nested] 2+ messages in thread
end of thread, other threads:[~2014-12-16 15:02 UTC | newest]

Thread overview: 2+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2014-12-16 14:32 First byte of private_data transfered with rdma_connect() is always 0 Fabian Holler
     [not found] ` <20141216143242.GB4401-99BIx50xQYGELgA04lAiVw@public.gmane.org>
2014-12-16 15:02   ` Jack Wang

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox