NVMe over Fabrics support for nvme-cli

NVMe over Fabrics support for nvme-cli

[Christoph Hellwig]

Hi Keith,

the series below adds support for the discovery, connect, connect-all and
disconnect commands to nvme-cli, which are useful to support the NVMe over
Fabrics host code.  Most of the work is from Ming with various bits and
pieces from me.  I also did the final rebase to the latest master tree.

[PATCH 01/11] Makefile: introduce OBJS variable

[Christoph Hellwig]

Instead of listing all the objects twice.

Signed-off-by: Christoph Hellwig <hch at lst.de>
---
 Makefile | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 9260ae4..72f5732 100644
--- a/Makefile
+++ b/Makefile
@@ -31,8 +31,10 @@ override CFLAGS += -DNVME_VERSION='"$(NVME_VERSION)"'
 
 NVME_DPKG_VERSION=1~`lsb_release -sc`
 
-nvme: nvme.c ./linux/nvme.h argconfig.o suffix.o nvme-print.o nvme-ioctl.o nvme-lightnvm.o NVME-VERSION-FILE
-	$(CC) $(CPPFLAGS) $(CFLAGS) nvme.c $(LDFLAGS) -o $(NVME) argconfig.o suffix.o nvme-print.o nvme-ioctl.o nvme-lightnvm.o
+OBJS := argconfig.o suffix.o nvme-print.o nvme-ioctl.o nvme-lightnvm.o
+
+nvme: nvme.c ./linux/nvme.h $(OBJS) NVME-VERSION-FILE
+	$(CC) $(CPPFLAGS) $(CFLAGS) nvme.c $(LDFLAGS) -o $(NVME) $(OBJS)
 
 nvme-ioctl.o: nvme-ioctl.c nvme-ioctl.h
 	$(CC) $(CPPFLAGS) $(CFLAGS) -c nvme-ioctl.c
-- 
2.1.4

[PATCH 02/11] Support Get Log Page extended Number of Dwords

[Christoph Hellwig]

From: Ming Lin <ming.l@ssi.samsung.com>

This supports 32 bits(rather than 12 bits) extended Number of Dwords
in nvme_get_log().

Signed-off-by: Ming Lin <ming.l at ssi.samsung.com>
Reviewed-by: Christoph Hellwig <hch at lst.de>
Reviewed-and-tested-by: Sagi Grimberg <sagi at grimberg.me>
Tested-by: Armen Baloyan <armenx.baloyan at intel.com>
---
 nvme-ioctl.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/nvme-ioctl.c b/nvme-ioctl.c
index 849d3b6..0510837 100644
--- a/nvme-ioctl.c
+++ b/nvme-ioctl.c
@@ -362,8 +362,13 @@ int nvme_get_log(int fd, __u32 nsid, __u8 log_id, __u32 data_len, void *data)
 		.addr		= (__u64)(uintptr_t) data,
 		.data_len	= data_len,
 	};
+	__u32 numd = (data_len >> 2) - 1;
+	__le16 numdu, numdl;
 
-	cmd.cdw10 = log_id | ((data_len >> 2) - 1) << 16;
+	numdu = htole16((numd & 0xFFFF0000) >> 16);
+	numdl = htole16(numd & 0x0000FFFF);
+	cmd.cdw10 = log_id | (numdl << 16);
+	cmd.cdw11 = numdu;
 
 	return nvme_submit_admin_passthru(fd, &cmd);
 }
-- 
2.1.4

[PATCH 03/11] Add NVMeoF definition to nvme.h

[Christoph Hellwig]

From: Ming Lin <ming.l@ssi.samsung.com>

Signed-off-by: Ming Lin <ming.l at ssi.samsung.com>
Reviewed-by: Christoph Hellwig <hch at lst.de>
Reviewed-and-tested-by: Sagi Grimberg <sagi at grimberg.me>
Tested-by: Armen Baloyan <armenx.baloyan at intel.com>
---
 linux/nvme.h | 103 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 103 insertions(+)

diff --git a/linux/nvme.h b/linux/nvme.h
index 7220b43..d05304b 100644
--- a/linux/nvme.h
+++ b/linux/nvme.h
@@ -17,6 +17,71 @@
 
 #include <linux/types.h>
 
+/* NQN names in commands fields specified one size */
+#define NVMF_NQN_FIELD_LEN	256
+
+/* However the max length of a qualified name is another size */
+#define NVMF_NQN_SIZE		223
+#define NVMF_TRSVCID_SIZE	32
+#define NVMF_TRADDR_SIZE	256
+#define NVMF_TSAS_SIZE		256
+
+#define NVME_DISC_SUBSYS_NAME	"nqn.2014-08.org.nvmexpress.discovery"
+
+enum nvme_subsys_type {
+	NVME_NQN_DISC	= 1,		/* Discovery type target subsystem */
+	NVME_NQN_NVME	= 2,		/* NVME type target subsystem */
+};
+
+/* Transport Type codes for Discovery Log Page entry TRTYPE field */
+enum {
+	NVMF_TRTYPE_RDMA	= 1,	/* RDMA */
+	NVMF_TRTYPE_FC		= 2,	/* Fibre Channel */
+	NVMF_TRTYPE_LOOP	= 254,  /* Reserved for host usage */
+};
+
+/* Transport Requirements codes for Discovery Log Page entry TREQ field */
+enum {
+	NVMF_TREQ_NOT_SPECIFIED	= 0,	/* Not specified */
+	NVMF_TREQ_REQUIRED	= 1,	/* Required */
+	NVMF_TREQ_NOT_REQUIRED	= 2,	/* Not Required */
+};
+
+/* RDMA QP Service Type codes for Discovery Log Page entry TSAS
+ * RDMA_QPTYPE field
+ */
+enum {
+	NVMF_RDMA_QPTYPE_CONNECTED	= 0, /* Reliable Connected */
+	NVMF_RDMA_QPTYPE_DATAGRAM	= 1, /* Reliable Datagram */
+};
+
+/* RDMA QP Service Type codes for Discovery Log Page entry TSAS
+ * RDMA_QPTYPE field
+ */
+enum {
+	NVMF_RDMA_PRTYPE_NOT_SPECIFIED	= 0, /* No Provider Specified */
+	NVMF_RDMA_PRTYPE_IB		= 1, /* InfiniBand */
+	NVMF_RDMA_PRTYPE_ROCE		= 2, /* InfiniBand RoCE */
+	NVMF_RDMA_PRTYPE_ROCEV2		= 3, /* InfiniBand RoCEV2 */
+	NVMF_RDMA_PRTYPE_IWARP		= 4, /* IWARP */
+};
+
+/* RDMA Connection Management Service Type codes for Discovery Log Page
+ * entry TSAS RDMA_CMS field
+ */
+enum {
+	NVMF_RDMA_CMS_RDMA_CM	= 0, /* Sockets based enpoint addressing */
+};
+
+/* Address Family codes for Discovery Log Page entry ADRFAM field */
+enum {
+	NVMF_ADDR_FAMILY_PCI	= 0,	/* PCIe */
+	NVMF_ADDR_FAMILY_IP4	= 1,	/* IP4 */
+	NVMF_ADDR_FAMILY_IP6	= 2,	/* IP6 */
+	NVMF_ADDR_FAMILY_IB	= 3,	/* InfiniBand */
+	NVMF_ADDR_FAMILY_FC	= 4,	/* Fibre Channel */
+};
+
 struct nvme_error_log_page {
 	__u64	error_count;
 	__u16	sqid;
@@ -397,6 +462,7 @@ enum {
 	NVME_LOG_ERROR		= 0x01,
 	NVME_LOG_SMART		= 0x02,
 	NVME_LOG_FW_SLOT	= 0x03,
+	NVME_LOG_DISC		= 0x70,
 	NVME_LOG_RESERVATION	= 0x80,
 	NVME_FWACT_REPL		= (0 << 3),
 	NVME_FWACT_REPL_ACTV	= (1 << 3),
@@ -520,6 +586,43 @@ struct nvme_bar {
 	__u32			cmbsz;	/* Controller Memory Buffer Size */
 };
 
+#define NVME_CNTLID_DYNAMIC	0xFFFF
+
+/* Discovery log page entry */
+struct nvmf_disc_rsp_page_entry {
+	__u8	trtype;
+	__u8	adrfam;
+	__u8	nqntype;
+	__u8	treq;
+	__le16	portid;
+	__le16	cntlid;
+	__u8	resv8[24];
+	char	trsvcid[NVMF_TRSVCID_SIZE];
+	__u8	resv64[192];
+	char	subnqn[NVMF_NQN_FIELD_LEN];
+	char	traddr[NVMF_TRADDR_SIZE];
+	union tsas {
+		char		common[NVMF_TSAS_SIZE];
+		struct rdma {
+			__u8	qptype;
+			__u8	prtype;
+			__u8	cms;
+			__u8	resv3[5];
+			__u16	pkey;
+			__u8	resv10[246];
+		} rdma;
+	} tsas;
+};
+
+/* Discovery log page header */
+struct nvmf_disc_rsp_page_hdr {
+	__le64	genctr;
+	__le64	numrec;
+	__le16	recfmt;
+	__u8	resv14[1006];
+	struct nvmf_disc_rsp_page_entry entries[0];
+};
+
 #define nvme_admin_cmd nvme_passthru_cmd
 
 #define NVME_VS(major, minor) (((major) << 16) | ((minor) << 8))
-- 
2.1.4

[PATCH 04/11] Add macro helpers to common.h

[Christoph Hellwig]

From: Ming Lin <ming.l@ssi.samsung.com>

Signed-off-by: Ming Lin <ming.l at ssi.samsung.com>
Reviewed-by: Christoph Hellwig <hch at lst.de>
Reviewed-and-tested-by: Sagi Grimberg <sagi at grimberg.me>
Tested-by: Armen Baloyan <armenx.baloyan at intel.com>
---
 common.h | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 common.h

diff --git a/common.h b/common.h
new file mode 100644
index 0000000..639186d
--- /dev/null
+++ b/common.h
@@ -0,0 +1,9 @@
+#ifndef _COMMON_H
+#define _COMMON_H
+
+#define offsetof(x, y)	__builtin_offsetof(x, y)
+
+#define __round_mask(x, y) ((__typeof__(x))((y)-1))
+#define round_up(x, y) ((((x)-1) | __round_mask(x, y))+1)
+
+#endif
-- 
2.1.4

[PATCH 05/11] Adds nvme_discovery_log()

[Christoph Hellwig]

From: Ming Lin <ming.l@ssi.samsung.com>

Signed-off-by: Ming Lin <ming.l at ssi.samsung.com>
Reviewed-by: Christoph Hellwig <hch at lst.de>
Reviewed-and-tested-by: Sagi Grimberg <sagi at grimberg.me>
Tested-by: Armen Baloyan <armenx.baloyan at intel.com>
---
 nvme-ioctl.c | 5 +++++
 nvme-ioctl.h | 1 +
 2 files changed, 6 insertions(+)

diff --git a/nvme-ioctl.c b/nvme-ioctl.c
index 0510837..2a26ca2 100644
--- a/nvme-ioctl.c
+++ b/nvme-ioctl.c
@@ -396,6 +396,11 @@ int nvme_intel_smart_log(int fd, __u32 nsid,
 			intel_smart_log);
 }
 
+int nvme_discovery_log(int fd, struct nvmf_disc_rsp_page_hdr *log, __u32 size)
+{
+	return nvme_get_log(fd, 0, NVME_LOG_DISC, size, log);
+}
+
 int nvme_feature(int fd, __u8 opcode, __u32 nsid, __u32 cdw10, __u32 cdw11,
 		 __u32 data_len, void *data, __u32 *result)
 {
diff --git a/nvme-ioctl.h b/nvme-ioctl.h
index 77c0bc8..5485b64 100644
--- a/nvme-ioctl.h
+++ b/nvme-ioctl.h
@@ -83,6 +83,7 @@ int nvme_error_log(int fd, __u32 nsid, int entries,
 int nvme_smart_log(int fd, __u32 nsid, struct nvme_smart_log *smart_log);
 int nvme_intel_smart_log(int fd, __u32 nsid,
 			 struct nvme_additional_smart_log *intel_smart_log);
+int nvme_discovery_log(int fd, struct nvmf_disc_rsp_page_hdr *log, __u32 size);
 
 int nvme_feature(int fd, __u8 opcode, __u32 nsid, __u32 cdw10,
 		 __u32 cdw11, __u32 data_len, void *data, __u32 *result);
-- 
2.1.4

[PATCH 06/11] Adds parser library

[Christoph Hellwig]

From: Ming Lin <ming.l@ssi.samsung.com>

This is imported from the Linux kernel 4.5.

Signed-off-by: Ming Lin <ming.l at ssi.samsung.com>
Reviewed-by: Christoph Hellwig <hch at lst.de>
Reviewed-and-tested-by: Sagi Grimberg <sagi at grimberg.me>
Tested-by: Armen Baloyan <armenx.baloyan at intel.com>
---
 Makefile |   5 +-
 parser.c | 284 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 parser.h |  34 ++++++++
 3 files changed, 322 insertions(+), 1 deletion(-)
 create mode 100644 parser.c
 create mode 100644 parser.h

diff --git a/Makefile b/Makefile
index 72f5732..c125756 100644
--- a/Makefile
+++ b/Makefile
@@ -31,7 +31,7 @@ override CFLAGS += -DNVME_VERSION='"$(NVME_VERSION)"'
 
 NVME_DPKG_VERSION=1~`lsb_release -sc`
 
-OBJS := argconfig.o suffix.o nvme-print.o nvme-ioctl.o nvme-lightnvm.o
+OBJS := argconfig.o suffix.o parser.o nvme-print.o nvme-ioctl.o nvme-lightnvm.o
 
 nvme: nvme.c ./linux/nvme.h $(OBJS) NVME-VERSION-FILE
 	$(CC) $(CPPFLAGS) $(CFLAGS) nvme.c $(LDFLAGS) -o $(NVME) $(OBJS)
@@ -39,6 +39,9 @@ nvme: nvme.c ./linux/nvme.h $(OBJS) NVME-VERSION-FILE
 nvme-ioctl.o: nvme-ioctl.c nvme-ioctl.h
 	$(CC) $(CPPFLAGS) $(CFLAGS) -c nvme-ioctl.c
 
+parser.o: parser.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) -c parser.c
+
 argconfig.o: $(SRC)/argconfig.c $(SRC)/argconfig.h $(SRC)/suffix.h
 	$(CC) $(CPPFLAGS) $(CFLAGS) -c $(SRC)/argconfig.c
 
diff --git a/parser.c b/parser.c
new file mode 100644
index 0000000..2dd0922
--- /dev/null
+++ b/parser.c
@@ -0,0 +1,284 @@
+/*
+ * lib/parser.c - simple parser for mount, etc. options.
+ *
+ * This source code is licensed under the GNU General Public License,
+ * Version 2.  See the file COPYING for more details.
+ */
+
+#include <ctype.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stdbool.h>
+#include <errno.h>
+#include <limits.h>
+#include <sys/types.h>
+#include "parser.h"
+
+/**
+ * match_one: - Determines if a string matches a simple pattern
+ * @s: the string to examine for presence of the pattern
+ * @p: the string containing the pattern
+ * @args: array of %MAX_OPT_ARGS &substring_t elements. Used to return match
+ * locations.
+ *
+ * Description: Determines if the pattern @p is present in string @s. Can only
+ * match extremely simple token=arg style patterns. If the pattern is found,
+ * the location(s) of the arguments will be returned in the @args array.
+ */
+static int match_one(char *s, const char *p, substring_t args[])
+{
+	char *meta;
+	int argc = 0;
+
+	if (!p)
+		return 1;
+
+	while(1) {
+		int len = -1;
+		meta = strchr(p, '%');
+		if (!meta)
+			return strcmp(p, s) == 0;
+
+		if (strncmp(p, s, meta-p))
+			return 0;
+
+		s += meta - p;
+		p = meta + 1;
+
+		if (isdigit(*p))
+			len = strtoul(p, (char **) &p, 10);
+		else if (*p == '%') {
+			if (*s++ != '%')
+				return 0;
+			p++;
+			continue;
+		}
+
+		if (argc >= MAX_OPT_ARGS)
+			return 0;
+
+		args[argc].from = s;
+		switch (*p++) {
+		case 's': {
+			size_t str_len = strlen(s);
+
+			if (str_len == 0)
+				return 0;
+			if (len == -1 || len > str_len)
+				len = str_len;
+			args[argc].to = s + len;
+			break;
+		}
+		case 'd':
+			strtol(s, &args[argc].to, 0);
+			goto num;
+		case 'u':
+			strtoul(s, &args[argc].to, 0);
+			goto num;
+		case 'o':
+			strtoul(s, &args[argc].to, 8);
+			goto num;
+		case 'x':
+			strtoul(s, &args[argc].to, 16);
+		num:
+			if (args[argc].to == args[argc].from)
+				return 0;
+			break;
+		default:
+			return 0;
+		}
+		s = args[argc].to;
+		argc++;
+	}
+}
+
+/**
+ * match_token: - Find a token (and optional args) in a string
+ * @s: the string to examine for token/argument pairs
+ * @table: match_table_t describing the set of allowed option tokens and the
+ * arguments that may be associated with them. Must be terminated with a
+ * &struct match_token whose pattern is set to the NULL pointer.
+ * @args: array of %MAX_OPT_ARGS &substring_t elements. Used to return match
+ * locations.
+ *
+ * Description: Detects which if any of a set of token strings has been passed
+ * to it. Tokens can include up to MAX_OPT_ARGS instances of basic c-style
+ * format identifiers which will be taken into account when matching the
+ * tokens, and whose locations will be returned in the @args array.
+ */
+int match_token(char *s, const match_table_t table, substring_t args[])
+{
+	const struct match_token *p;
+
+	for (p = table; !match_one(s, p->pattern, args) ; p++)
+		;
+
+	return p->token;
+}
+
+/**
+ * match_number: scan a number in the given base from a substring_t
+ * @s: substring to be scanned
+ * @result: resulting integer on success
+ * @base: base to use when converting string
+ *
+ * Description: Given a &substring_t and a base, attempts to parse the substring
+ * as a number in that base. On success, sets @result to the integer represented
+ * by the string and returns 0. Returns -ENOMEM, -EINVAL, or -ERANGE on failure.
+ */
+static int match_number(substring_t *s, int *result, int base)
+{
+	char *endp;
+	char *buf;
+	int ret;
+	long val;
+	size_t len = s->to - s->from;
+
+	buf = malloc(len + 1);
+	if (!buf)
+		return -ENOMEM;
+	memcpy(buf, s->from, len);
+	buf[len] = '\0';
+
+	ret = 0;
+	val = strtol(buf, &endp, base);
+	if (endp == buf)
+		ret = -EINVAL;
+	else if (val < (long)INT_MIN || val > (long)INT_MAX)
+		ret = -ERANGE;
+	else
+		*result = (int) val;
+	free(buf);
+	return ret;
+}
+
+/**
+ * match_int: - scan a decimal representation of an integer from a substring_t
+ * @s: substring_t to be scanned
+ * @result: resulting integer on success
+ *
+ * Description: Attempts to parse the &substring_t @s as a decimal integer. On
+ * success, sets @result to the integer represented by the string and returns 0.
+ * Returns -ENOMEM, -EINVAL, or -ERANGE on failure.
+ */
+int match_int(substring_t *s, int *result)
+{
+	return match_number(s, result, 0);
+}
+
+/**
+ * match_octal: - scan an octal representation of an integer from a substring_t
+ * @s: substring_t to be scanned
+ * @result: resulting integer on success
+ *
+ * Description: Attempts to parse the &substring_t @s as an octal integer. On
+ * success, sets @result to the integer represented by the string and returns
+ * 0. Returns -ENOMEM, -EINVAL, or -ERANGE on failure.
+ */
+int match_octal(substring_t *s, int *result)
+{
+	return match_number(s, result, 8);
+}
+
+/**
+ * match_hex: - scan a hex representation of an integer from a substring_t
+ * @s: substring_t to be scanned
+ * @result: resulting integer on success
+ *
+ * Description: Attempts to parse the &substring_t @s as a hexadecimal integer.
+ * On success, sets @result to the integer represented by the string and
+ * returns 0. Returns -ENOMEM, -EINVAL, or -ERANGE on failure.
+ */
+int match_hex(substring_t *s, int *result)
+{
+	return match_number(s, result, 16);
+}
+
+/**
+ * match_wildcard: - parse if a string matches given wildcard pattern
+ * @pattern: wildcard pattern
+ * @str: the string to be parsed
+ *
+ * Description: Parse the string @str to check if matches wildcard
+ * pattern @pattern. The pattern may contain two type wildcardes:
+ *   '*' - matches zero or more characters
+ *   '?' - matches one character
+ * If it's matched, return true, else return false.
+ */
+bool match_wildcard(const char *pattern, const char *str)
+{
+	const char *s = str;
+	const char *p = pattern;
+	bool star = false;
+
+	while (*s) {
+		switch (*p) {
+		case '?':
+			s++;
+			p++;
+			break;
+		case '*':
+			star = true;
+			str = s;
+			if (!*++p)
+				return true;
+			pattern = p;
+			break;
+		default:
+			if (*s == *p) {
+				s++;
+				p++;
+			} else {
+				if (!star)
+					return false;
+				str++;
+				s = str;
+				p = pattern;
+			}
+			break;
+		}
+	}
+
+	if (*p == '*')
+		++p;
+	return !*p;
+}
+
+/**
+ * match_strlcpy: - Copy the characters from a substring_t to a sized buffer
+ * @dest: where to copy to
+ * @src: &substring_t to copy
+ * @size: size of destination buffer
+ *
+ * Description: Copy the characters in &substring_t @src to the
+ * c-style string @dest.  Copy no more than @size - 1 characters, plus
+ * the terminating NUL.  Return length of @src.
+ */
+size_t match_strlcpy(char *dest, const substring_t *src, size_t size)
+{
+	size_t ret = src->to - src->from;
+
+	if (size) {
+		size_t len = ret >= size ? size - 1 : ret;
+		memcpy(dest, src->from, len);
+		dest[len] = '\0';
+	}
+	return ret;
+}
+
+/**
+ * match_strdup: - allocate a new string with the contents of a substring_t
+ * @s: &substring_t to copy
+ *
+ * Description: Allocates and returns a string filled with the contents of
+ * the &substring_t @s. The caller is responsible for freeing the returned
+ * string with free().
+ */
+char *match_strdup(const substring_t *s)
+{
+	size_t sz = s->to - s->from + 1;
+	char *p = malloc(sz);
+	if (p)
+		match_strlcpy(p, s, sz);
+	return p;
+}
diff --git a/parser.h b/parser.h
new file mode 100644
index 0000000..39d5b79
--- /dev/null
+++ b/parser.h
@@ -0,0 +1,34 @@
+/*
+ * linux/include/linux/parser.h
+ *
+ * Header for lib/parser.c
+ * Intended use of these functions is parsing filesystem argument lists,
+ * but could potentially be used anywhere else that simple option=arg
+ * parsing is required.
+ */
+
+
+/* associates an integer enumerator with a pattern string. */
+struct match_token {
+	int token;
+	const char *pattern;
+};
+
+typedef struct match_token match_table_t[];
+
+/* Maximum number of arguments that match_token will find in a pattern */
+enum {MAX_OPT_ARGS = 3};
+
+/* Describe the location within a string of a substring */
+typedef struct {
+	char *from;
+	char *to;
+} substring_t;
+
+int match_token(char *, const match_table_t table, substring_t args[]);
+int match_int(substring_t *, int *result);
+int match_octal(substring_t *, int *result);
+int match_hex(substring_t *, int *result);
+bool match_wildcard(const char *pattern, const char *str);
+size_t match_strlcpy(char *, const substring_t *, size_t);
+char *match_strdup(const substring_t *);
-- 
2.1.4

[PATCH 07/11] Implement discover command

[Christoph Hellwig]

From: Ming Lin <ming.l@ssi.samsung.com>

Example output:

root at host:~# nvme discover -t rdma -a 192.168.2.2 -p 1023
Discovery Log Number of Records 1, Generation counter 3
=====Discovery Log Entry 0======
trtype:  1
adrfam:  1
nqntype: 2
treq:    0
portid:  1
trsvcid: 1023

subnqn:  nqn.testiqn
traddr:  192.168.2.2

rdma_prtype: 0
rdma_qptype: 0
rdma_cms:    0
rdma_pkey: 0x0000

Signed-off-by: Ming Lin <ming.l at ssi.samsung.com>
Tested-by: Armen Baloyan <armenx.baloyan at intel.com>
Tested-by: Sagi Grimberg <sagi at grimberg.me>
Reviewed-by: Jay Freyensee <james.p.freyensee at intel.com>
---
 Makefile  |   6 +-
 fabrics.c | 393 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 fabrics.h |   6 +
 nvme.c    |   8 ++
 4 files changed, 412 insertions(+), 1 deletion(-)
 create mode 100644 fabrics.c
 create mode 100644 fabrics.h

diff --git a/Makefile b/Makefile
index c125756..e43c65d 100644
--- a/Makefile
+++ b/Makefile
@@ -31,7 +31,8 @@ override CFLAGS += -DNVME_VERSION='"$(NVME_VERSION)"'
 
 NVME_DPKG_VERSION=1~`lsb_release -sc`
 
-OBJS := argconfig.o suffix.o parser.o nvme-print.o nvme-ioctl.o nvme-lightnvm.o
+OBJS := argconfig.o suffix.o parser.o nvme-print.o nvme-ioctl.o \
+	nvme-lightnvm.o fabrics.o
 
 nvme: nvme.c ./linux/nvme.h $(OBJS) NVME-VERSION-FILE
 	$(CC) $(CPPFLAGS) $(CFLAGS) nvme.c $(LDFLAGS) -o $(NVME) $(OBJS)
@@ -39,6 +40,9 @@ nvme: nvme.c ./linux/nvme.h $(OBJS) NVME-VERSION-FILE
 nvme-ioctl.o: nvme-ioctl.c nvme-ioctl.h
 	$(CC) $(CPPFLAGS) $(CFLAGS) -c nvme-ioctl.c
 
+fabrics.o: fabrics.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) -c fabrics.c
+
 parser.o: parser.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) -c parser.c
 
diff --git a/fabrics.c b/fabrics.c
new file mode 100644
index 0000000..ad79017
--- /dev/null
+++ b/fabrics.c
@@ -0,0 +1,393 @@
+/*
+ * Copyright (C) 2016 Intel Corporation. All rights reserved.
+ * Copyright (c) 2016 HGST, a Western Digital Company.
+ * Copyright (c) 2016 Samsung Electronics Co., Ltd.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License version
+ * 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ *
+ * This file implements the discovery controller feature of NVMe over
+ * Fabrics specification standard.
+ */
+
+#include <errno.h>
+#include <getopt.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <unistd.h>
+#include <sys/ioctl.h>
+#include <libudev.h>
+
+#include <linux/types.h>
+
+#include "parser.h"
+#include "nvme-ioctl.h"
+#include "fabrics.h"
+
+#include "linux/nvme.h"
+#include "src/argconfig.h"
+
+#include "common.h"
+
+struct config {
+	char *nqn;
+	char *transport;
+	char *traddr;
+	char *trsvcid;
+} cfg = { 0 };
+
+#define BUF_SIZE		4096
+#define PATH_NVME_FABRICS	"/dev/nvme-fabrics"
+
+enum {
+	OPT_INSTANCE,
+	OPT_CNTLID,
+	OPT_ERR
+};
+
+static const match_table_t opt_tokens = {
+	{ OPT_INSTANCE,		"instance=%d"	},
+	{ OPT_CNTLID,		"cntlid=%d"	},
+	{ OPT_ERR,		NULL		},
+};
+
+static int add_ctrl(const char *argstr)
+{
+	substring_t args[MAX_OPT_ARGS];
+	char buf[BUF_SIZE], *options, *p;
+	size_t len = strlen(argstr);
+	int token, ret, fd;
+
+	fd = open(PATH_NVME_FABRICS, O_RDWR);
+	if (fd < 0) {
+		fprintf(stderr, "Failed to open %s: %s\n",
+			 PATH_NVME_FABRICS, strerror(errno));
+		ret = -errno;
+		goto out;
+	}
+
+	if (write(fd, argstr, len) != len) {
+		fprintf(stderr, "Failed to write to %s: %s\n",
+			 PATH_NVME_FABRICS, strerror(errno));
+		ret = -errno;
+		goto out_close;
+	}
+
+	len = read(fd, buf, BUF_SIZE);
+	if (len < 0) {
+		fprintf(stderr, "Failed to read from %s: %s\n",
+			 PATH_NVME_FABRICS, strerror(errno));
+		ret = -errno;
+		goto out_close;
+	}
+
+	buf[len] = '\0';
+	options = buf;
+	while ((p = strsep(&options, ",\n")) != NULL) {
+		if (!*p)
+			continue;
+
+		token = match_token(p, opt_tokens, args);
+		switch (token) {
+		case OPT_INSTANCE:
+			if (match_int(args, &token))
+				goto out_fail;
+			ret = token;
+			goto out_close;
+		default:
+			/* ignore */
+			break;
+		}
+	}
+
+out_fail:
+	fprintf(stderr, "Failed to parse ctrl info for \"%s\"\n", argstr);
+	ret = -EINVAL;
+out_close:
+	close(fd);
+out:
+	return ret;
+}
+
+static int remove_ctrl_by_path(char *sysfs_path)
+{
+	int ret, fd;
+
+	fd = open(sysfs_path, O_WRONLY);
+	if (fd < 0) {
+		ret = errno;
+		goto out;
+	}
+
+	if (write(fd, "1", 1) != 1) {
+		ret = errno;
+		goto out_close;
+	}
+
+	ret = 0;
+out_close:
+	close(fd);
+out:
+	return ret;
+}
+
+static int remove_ctrl(int instance)
+{
+	char *sysfs_path;
+	int ret;
+
+	if (asprintf(&sysfs_path, "/sys/class/nvme/nvme%d/delete_controller",
+			instance) < 0) {
+		ret = errno;
+		goto out;
+	}
+
+	ret = remove_ctrl_by_path(sysfs_path);
+	free(sysfs_path);
+out:
+	return ret;
+}
+
+enum {
+	DISC_OK,
+	DISC_NO_LOG,
+	DISC_GET_NUMRECS,
+	DISC_GET_LOG,
+	DISC_NOT_EQUAL,
+};
+
+static int nvmf_get_log_page_discovery(const char *dev_path,
+		struct nvmf_disc_rsp_page_hdr **logp, int *numrec)
+{
+	struct nvmf_disc_rsp_page_hdr *log;
+	unsigned log_size = 0;
+	unsigned long genctr;
+	int error, fd;
+
+	fd = open(dev_path, O_RDWR);
+	if (fd < 0) {
+		error = -errno;
+		goto out;
+	}
+
+	/* first get_log_page we just need numrec entry from discovery hdr.
+	 * host supplies its desired bytes via dwords, per NVMe spec.
+	 */
+	log_size = round_up((offsetof(struct nvmf_disc_rsp_page_hdr, numrec) +
+			    sizeof(log->numrec)), sizeof(__u32));
+
+	/*
+	 * Issue first get log page w/numdl small enough to retrieve numrec.
+	 * We just want to know how many records to retrieve.
+	 */
+	log = calloc(1, log_size);
+	if (!log) {
+		error = -ENOMEM;
+		goto out_close;
+	}
+
+	error = nvme_discovery_log(fd, log, log_size);
+	if (error) {
+		error = DISC_GET_NUMRECS;
+		goto out_free_log;
+	}
+
+	/* check numrec limits */
+	*numrec = le64toh(log->numrec);
+	genctr = le64toh(log->genctr);
+	free(log);
+
+	if (*numrec == 0) {
+		error = DISC_NO_LOG;
+		goto out_close;
+	}
+
+	/* we are actually retrieving the entire discovery tables
+	 * for the second get_log_page(), per
+	 * NVMe spec so no need to round_up(), or there is something
+	 * seriously wrong with the standard
+	 */
+	log_size = sizeof(struct nvmf_disc_rsp_page_hdr) +
+			sizeof(struct nvmf_disc_rsp_page_entry) * *numrec;
+
+	/* allocate discovery log pages based on page_hdr->numrec */
+	log = calloc(1, log_size);
+	if (!log) {
+		error = -ENOMEM;
+		goto out_close;
+	}
+
+	/*
+	 * issue new get_log_page w/numdl+numdh set to get all records,
+	 * up to MAX_DISC_LOGS.
+	 */
+	error = nvme_discovery_log(fd, log, log_size);
+	if (error) {
+		error = DISC_GET_LOG;
+		goto out_free_log;
+	}
+
+	if (*numrec != le32toh(log->numrec) || genctr != le64toh(log->genctr)) {
+		error = DISC_NOT_EQUAL;
+		goto out_free_log;
+	}
+
+	/* needs to be freed by the caller */
+	*logp = log;
+	goto out_close;
+
+	error = DISC_OK;
+out_free_log:
+	free(log);
+out_close:
+	close(fd);
+out:
+	return error;
+}
+
+static void print_discovery_log(struct nvmf_disc_rsp_page_hdr *log, int numrec)
+{
+	int i;
+
+	printf("Discovery Log Number of Records %d, Generation counter %lld\n",
+		numrec, log->genctr);
+
+	for (i = 0; i < numrec; i++) {
+		struct nvmf_disc_rsp_page_entry *e = &log->entries[i];
+
+		printf("=====Discovery Log Entry %d======\n", i);
+		printf("trtype:  %d\n", e->trtype);
+		printf("adrfam:  %d\n", e->adrfam);
+		printf("nqntype: %d\n", e->nqntype);
+		printf("treq:    %d\n", e->treq);
+		printf("portid:  %d\n", e->portid);
+		printf("trsvcid: %s\n", e->trsvcid);
+		printf("subnqn:  %s\n", e->subnqn);
+		printf("traddr:  %s\n", e->traddr);
+
+		switch (e->trtype) {
+		case NVMF_TRTYPE_RDMA:
+			printf("rdma_prtype: %d\n", e->tsas.rdma.prtype);
+			printf("rdma_qptype: %d\n", e->tsas.rdma.qptype);
+			printf("rdma_cms:    %d\n", e->tsas.rdma.cms);
+			printf("rdma_pkey: 0x%04x\n", e->tsas.rdma.pkey);
+			break;
+		}
+	}
+}
+
+static int build_options(char *argstr, int max_len)
+{
+	int len;
+
+	if (!cfg.transport) {
+		fprintf(stderr, "need a transport (-t) argument\n");
+		return -EINVAL;
+	}
+
+	if (strncmp(cfg.transport, "loop", 4)) {
+		if (!cfg.traddr) {
+			fprintf(stderr, "need a address (-a) argument\n");
+			return -EINVAL;
+		}
+	}
+
+	len = snprintf(argstr, max_len, "nqn=%s", cfg.nqn);
+	if (len < 0)
+		return -EINVAL;
+	argstr += len;
+	max_len -= len;
+
+	len = snprintf(argstr, max_len, ",transport=%s", cfg.transport);
+	if (len < 0)
+		return -EINVAL;
+	argstr += len;
+	max_len -= len;
+
+	if (cfg.traddr) {
+		len = snprintf(argstr, max_len, ",traddr=%s", cfg.traddr);
+		if (len < 0)
+			return -EINVAL;
+		argstr += len;
+		max_len -= len;
+	}
+
+	if (cfg.trsvcid) {
+		len = snprintf(argstr, max_len, ",trsvcid=%s", cfg.trsvcid);
+		if (len < 0)
+			return -EINVAL;
+		argstr += len;
+		max_len -= len;
+	}
+
+	return 0;
+}
+
+int discover(const char *desc, int argc, char **argv)
+{
+	char argstr[BUF_SIZE];
+	struct nvmf_disc_rsp_page_hdr *log = NULL;
+	char *dev_name;
+	int instance, numrec = 0, ret;
+	const struct argconfig_commandline_options command_line_options[] = {
+		{"transport", 't', "LIST", CFG_STRING, &cfg.transport, required_argument,
+			"transport type" },
+		{"traddr", 'a', "LIST", CFG_STRING, &cfg.traddr, required_argument,
+			"transport address" },
+		{"trsvcid", 's', "LIST", CFG_STRING, &cfg.trsvcid, required_argument,
+			"transport service id (e.g. IP port)" },
+		{0},
+	};
+
+	argconfig_parse(argc, argv, desc, command_line_options, &cfg, sizeof(cfg));
+
+	cfg.nqn = NVME_DISC_SUBSYS_NAME;
+
+	ret = build_options(argstr, BUF_SIZE);
+	if (ret)
+		return ret;
+
+	instance = add_ctrl(argstr);
+	if (instance < 0)
+		return instance;
+
+	if (asprintf(&dev_name, "/dev/nvme%d", instance) < 0)
+		return errno;
+	ret = nvmf_get_log_page_discovery(dev_name, &log, &numrec);
+	free(dev_name);
+	remove_ctrl(instance);
+
+	switch (ret) {
+	case DISC_OK:
+		print_discovery_log(log, numrec);
+		break;
+	case DISC_GET_NUMRECS:
+		fprintf(stderr, "Get number of discovery log entries failed.\n");
+		break;
+	case DISC_GET_LOG:
+		fprintf(stderr, "Get discovery log entries failed.\n");
+		break;
+	case DISC_NO_LOG:
+		fprintf(stderr, "No discovery log entries to fetch.\n");
+		break;
+	case DISC_NOT_EQUAL:
+		fprintf(stderr, "Numrec values of last two get dicovery log page not equal\n");
+		break;
+	default:
+		fprintf(stderr, "Get dicovery log page failed: %d\n", ret);
+		break;
+	}
+
+	return ret;
+}
diff --git a/fabrics.h b/fabrics.h
new file mode 100644
index 0000000..c80b016
--- /dev/null
+++ b/fabrics.h
@@ -0,0 +1,6 @@
+#ifndef _DISCOVER_H
+#define _DISCOVER_H
+
+extern int discover(const char *desc, int argc, char **argv);
+
+#endif
diff --git a/nvme.c b/nvme.c
index 72a85a4..2dd2f3a 100644
--- a/nvme.c
+++ b/nvme.c
@@ -54,6 +54,8 @@
 #include "src/argconfig.h"
 #include "src/suffix.h"
 
+#include "fabrics.h"
+
 #define array_len(x) ((size_t)(sizeof(x) / sizeof(x[0])))
 #define min(x, y) (x) > (y) ? (y) : (x)
 #define max(x, y) (x) > (y) ? (x) : (y)
@@ -112,6 +114,7 @@ static const char nvme_version_string[] = NVME_VERSION;
 	ENTRY(LNVM_FACTORY, "lnvm-factory", "Reset device to factory state", lnvm_factory_init) \
 	ENTRY(LNVM_BBTBL_GET, "lnvm-diag-bbtbl", "Diagnose bad block table", lnvm_get_bbtbl) \
 	ENTRY(LNVM_BBTBL_SET, "lnvm-diag-set-bbtbl", "Update bad block table", lnvm_set_bbtbl) \
+	ENTRY(DISCOVER, "discover", "Discover NVMeoF subsystems", discover_cmd) \
 	ENTRY(VERSION, "version", "Shows the program version", version) \
 	ENTRY(HELP, "help", "Display this help", help)
 
@@ -2837,6 +2840,11 @@ static int lnvm_set_bbtbl(int argc, char **argv)
 				 cfg.value);
 }
 
+static int discover_cmd(int argc, char **argv)
+{
+	const char *desc = "Send command to discovery service.";
+	return discover(desc, argc, argv);
+}
 
 static void usage()
 {
-- 
2.1.4

[PATCH 08/11] Add option to save discovery raw log to a file

[Christoph Hellwig]

From: Ming Lin <ming.l@ssi.samsung.com>

root at host:~# nvme discover -t rdma -a 192.168.2.2 -p 1023 -r disc.raw
Discovery log is saved to disc.raw

Signed-off-by: Ming Lin <ming.l at ssi.samsung.com>
Tested-by: Armen Baloyan <armenx.baloyan at intel.com>
Tested-by: Sagi Grimberg <sagi at grimberg.me>
---
 fabrics.c | 30 +++++++++++++++++++++++++++++-
 1 file changed, 29 insertions(+), 1 deletion(-)

diff --git a/fabrics.c b/fabrics.c
index ad79017..50dfcd8 100644
--- a/fabrics.c
+++ b/fabrics.c
@@ -45,6 +45,7 @@ struct config {
 	char *transport;
 	char *traddr;
 	char *trsvcid;
+	char *raw;
 } cfg = { 0 };
 
 #define BUF_SIZE		4096
@@ -287,6 +288,28 @@ static void print_discovery_log(struct nvmf_disc_rsp_page_hdr *log, int numrec)
 	}
 }
 
+static void save_discovery_log(struct nvmf_disc_rsp_page_hdr *log, int numrec)
+{
+	int fd;
+	int len, ret;
+
+	fd = open(cfg.raw, O_CREAT|O_RDWR|O_TRUNC, S_IRUSR|S_IWUSR);
+	if (fd < 0) {
+		fprintf(stderr, "failed to open %s: %s\n", cfg.raw, strerror(errno));
+		return;
+	}
+
+	len = sizeof(struct nvmf_disc_rsp_page_hdr) +
+			numrec * sizeof(struct nvmf_disc_rsp_page_entry);
+	ret = write(fd, log, len);
+	if (ret < 0)
+		fprintf(stderr, "failed to write to %s: %s\n", cfg.raw, strerror(errno));
+	else
+		printf("Discovery log is saved to %s\n", cfg.raw);
+
+	close(fd);
+}
+
 static int build_options(char *argstr, int max_len)
 {
 	int len;
@@ -347,6 +370,8 @@ int discover(const char *desc, int argc, char **argv)
 			"transport address" },
 		{"trsvcid", 's', "LIST", CFG_STRING, &cfg.trsvcid, required_argument,
 			"transport service id (e.g. IP port)" },
+		{"raw", 'r', "LIST", CFG_STRING, &cfg.raw, required_argument,
+			"raw" },
 		{0},
 	};
 
@@ -370,7 +395,10 @@ int discover(const char *desc, int argc, char **argv)
 
 	switch (ret) {
 	case DISC_OK:
-		print_discovery_log(log, numrec);
+		if (cfg.raw)
+			save_discovery_log(log, numrec);
+		else
+			print_discovery_log(log, numrec);
 		break;
 	case DISC_GET_NUMRECS:
 		fprintf(stderr, "Get number of discovery log entries failed.\n");
-- 
2.1.4

[PATCH 09/11] Implement connect command

[Christoph Hellwig]

From: Ming Lin <ming.l@ssi.samsung.com>

Example usage:

root at host: nvme connect -t rdma -a 192.168.2.2 -p 1023 -n nqn.testiqn

Signed-off-by: Ming Lin <ming.l at ssi.samsung.com>
Tested-by: Armen Baloyan <armenx.baloyan at intel.com>
Tested-by: Sagi Grimberg <sagi at grimberg.me>
Reviewed-by: Jay Freyensee <james.p.freyensee at intel.com>
---
 fabrics.c | 35 ++++++++++++++++++++++++++++++++++-
 fabrics.h |  1 +
 nvme.c    |  7 +++++++
 3 files changed, 42 insertions(+), 1 deletion(-)

diff --git a/fabrics.c b/fabrics.c
index 50dfcd8..84dde2b 100644
--- a/fabrics.c
+++ b/fabrics.c
@@ -371,7 +371,7 @@ int discover(const char *desc, int argc, char **argv)
 		{"trsvcid", 's', "LIST", CFG_STRING, &cfg.trsvcid, required_argument,
 			"transport service id (e.g. IP port)" },
 		{"raw", 'r', "LIST", CFG_STRING, &cfg.raw, required_argument,
-			"raw" },
+			"raw output file" },
 		{0},
 	};
 
@@ -419,3 +419,36 @@ int discover(const char *desc, int argc, char **argv)
 
 	return ret;
 }
+
+int connect(const char *desc, int argc, char **argv)
+{
+	char argstr[BUF_SIZE];
+	int instance, ret;
+	const struct argconfig_commandline_options command_line_options[] = {
+		{"transport", 't', "LIST", CFG_STRING, &cfg.transport, required_argument,
+			"transport type" },
+		{"nqn", 'n', "LIST", CFG_STRING, &cfg.nqn, required_argument,
+			"nqn name" },
+		{"traddr", 'a', "LIST", CFG_STRING, &cfg.traddr, required_argument,
+			"transport address" },
+		{"trsvcid", 's', "LIST", CFG_STRING, &cfg.trsvcid, required_argument,
+			"transport service id (e.g. IP port)" },
+		{0},
+	};
+
+	argconfig_parse(argc, argv, desc, command_line_options, &cfg, sizeof(cfg));
+
+	ret = build_options(argstr, BUF_SIZE);
+	if (ret)
+		return ret;
+
+	if (!cfg.nqn) {
+		fprintf(stderr, "need a -n argument\n");
+		return -EINVAL;
+	}
+
+	instance = add_ctrl(argstr);
+	if (instance < 0)
+		return instance;
+	return 0;
+}
diff --git a/fabrics.h b/fabrics.h
index c80b016..43072a1 100644
--- a/fabrics.h
+++ b/fabrics.h
@@ -2,5 +2,6 @@
 #define _DISCOVER_H
 
 extern int discover(const char *desc, int argc, char **argv);
+extern int connect(const char *desc, int argc, char **argv);
 
 #endif
diff --git a/nvme.c b/nvme.c
index 2dd2f3a..902d84b 100644
--- a/nvme.c
+++ b/nvme.c
@@ -115,6 +115,7 @@ static const char nvme_version_string[] = NVME_VERSION;
 	ENTRY(LNVM_BBTBL_GET, "lnvm-diag-bbtbl", "Diagnose bad block table", lnvm_get_bbtbl) \
 	ENTRY(LNVM_BBTBL_SET, "lnvm-diag-set-bbtbl", "Update bad block table", lnvm_set_bbtbl) \
 	ENTRY(DISCOVER, "discover", "Discover NVMeoF subsystems", discover_cmd) \
+	ENTRY(CONNECT, "connect", "Connect to NVMeoF subsystem", connect_cmd) \
 	ENTRY(VERSION, "version", "Shows the program version", version) \
 	ENTRY(HELP, "help", "Display this help", help)
 
@@ -2846,6 +2847,12 @@ static int discover_cmd(int argc, char **argv)
 	return discover(desc, argc, argv);
 }
 
+static int connect_cmd(int argc, char **argv)
+{
+	const char *desc = "Connect to NVMeoF subsystem";
+	return connect(desc, argc, argv);
+}
+
 static void usage()
 {
 	printf("usage: nvme <command> [<device>] [<args>]\n");
-- 
2.1.4

[PATCH 10/11] Implement disconnect command

[Christoph Hellwig]

From: Ming Lin <ming.l@ssi.samsung.com>

Example usage:

Disconnect by NQN,
nvme disconnect -n nqn.testiqn

or by device name,
nvme disconnect -d nvme0

Signed-off-by: Ming Lin <ming.l at ssi.samsung.com>
Tested-by: Armen Baloyan <armenx.baloyan at intel.com>
Tested-by: Sagi Grimberg <sagi at grimberg.me>
Reviewed-by: Jay Freyensee <james.p.freyensee at intel.com>
---
 fabrics.c | 114 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 fabrics.h |   1 +
 nvme.c    |   7 ++++
 3 files changed, 122 insertions(+)

diff --git a/fabrics.c b/fabrics.c
index 84dde2b..0a60258 100644
--- a/fabrics.c
+++ b/fabrics.c
@@ -46,6 +46,7 @@ struct config {
 	char *traddr;
 	char *trsvcid;
 	char *raw;
+	char *device;
 } cfg = { 0 };
 
 #define BUF_SIZE		4096
@@ -452,3 +453,116 @@ int connect(const char *desc, int argc, char **argv)
 		return instance;
 	return 0;
 }
+
+static int disconnect_subsys(struct udev_enumerate *enumerate, char *nqn)
+{
+	struct udev_list_entry *list_entry;
+	const char *subsysnqn;
+	char *sysfs_path;
+	int ret = 1;
+
+	udev_list_entry_foreach(list_entry, udev_enumerate_get_list_entry(enumerate)) {
+		struct udev_device *device;
+
+		device = udev_device_new_from_syspath(udev_enumerate_get_udev(enumerate),
+						udev_list_entry_get_name(list_entry));
+		if (device != NULL) {
+			subsysnqn = udev_device_get_sysattr_value(device, "subsysnqn");
+			if (subsysnqn && !strcmp(subsysnqn, nqn)) {
+				if (asprintf(&sysfs_path, "%s/delete_controller",
+						udev_device_get_syspath(device)) < 0) {
+					ret = errno;
+					udev_device_unref(device);
+					break;
+				}
+				udev_device_unref(device);
+				ret = remove_ctrl_by_path(sysfs_path);
+				free(sysfs_path);
+				break;
+			}
+			udev_device_unref(device);
+		}
+	}
+
+	return ret;
+}
+
+static int disconnect_by_nqn(char *nqn)
+{
+	struct udev *udev;
+	struct udev_enumerate *udev_enumerate;
+	int ret;
+
+	if (strlen(nqn) > NVMF_NQN_SIZE) {
+		ret = -EINVAL;
+		goto exit;
+	}
+
+	udev = udev_new();
+	if (!udev) {
+		fprintf(stderr, "failed to create udev\n");
+		ret = -ENOMEM;
+		goto exit;
+	}
+
+	udev_enumerate = udev_enumerate_new(udev);
+	if (udev_enumerate == NULL) {
+		ret = -ENOMEM;
+		goto free_udev;
+	}
+
+	udev_enumerate_add_match_subsystem(udev_enumerate, "nvme");
+	udev_enumerate_scan_devices(udev_enumerate);
+	ret = disconnect_subsys(udev_enumerate, nqn);
+	udev_enumerate_unref(udev_enumerate);
+
+free_udev:
+	udev_unref(udev);
+exit:
+	return ret;
+}
+
+static int disconnect_by_device(char *device)
+{
+	int instance;
+	int ret;
+
+	ret = sscanf(device, "nvme%d", &instance);
+	if (ret < 0)
+		return ret;
+
+	return remove_ctrl(instance);
+}
+
+int disconnect(const char *desc, int argc, char **argv)
+{
+	const char *nqn = "nqn name";
+	const char *device = "nvme device";
+	int ret = 0;
+
+	const struct argconfig_commandline_options command_line_options[] = {
+		{"nqn", 'n', "LIST", CFG_STRING, &cfg.nqn, required_argument, nqn},
+		{"device", 'd', "LIST", CFG_STRING, &cfg.device, required_argument, device},
+		{0},
+	};
+
+	argconfig_parse(argc, argv, desc, command_line_options, &cfg, sizeof(cfg));
+	if (!cfg.nqn && !cfg.device) {
+		fprintf(stderr, "need a -n or -d argument\n");
+		return -EINVAL;
+	}
+
+	if (cfg.nqn) {
+		ret = disconnect_by_nqn(cfg.nqn);
+		if (ret)
+			fprintf(stderr, "Failed to disconnect by NQN: %s\n", cfg.nqn);
+	}
+
+	if (cfg.device) {
+		ret = disconnect_by_device(cfg.device);
+		if (ret)
+			fprintf(stderr, "Failed to disconnect by device name: %s\n", cfg.device);
+	}
+
+	return ret;
+}
diff --git a/fabrics.h b/fabrics.h
index 43072a1..7e3eb19 100644
--- a/fabrics.h
+++ b/fabrics.h
@@ -3,5 +3,6 @@
 
 extern int discover(const char *desc, int argc, char **argv);
 extern int connect(const char *desc, int argc, char **argv);
+extern int disconnect(const char *desc, int argc, char **argv);
 
 #endif
diff --git a/nvme.c b/nvme.c
index 902d84b..e403022 100644
--- a/nvme.c
+++ b/nvme.c
@@ -116,6 +116,7 @@ static const char nvme_version_string[] = NVME_VERSION;
 	ENTRY(LNVM_BBTBL_SET, "lnvm-diag-set-bbtbl", "Update bad block table", lnvm_set_bbtbl) \
 	ENTRY(DISCOVER, "discover", "Discover NVMeoF subsystems", discover_cmd) \
 	ENTRY(CONNECT, "connect", "Connect to NVMeoF subsystem", connect_cmd) \
+	ENTRY(DISCONNECT, "disconnect", "Disconnect from NVMeoF subsystem", disconnect_cmd) \
 	ENTRY(VERSION, "version", "Shows the program version", version) \
 	ENTRY(HELP, "help", "Display this help", help)
 
@@ -2853,6 +2854,12 @@ static int connect_cmd(int argc, char **argv)
 	return connect(desc, argc, argv);
 }
 
+static int disconnect_cmd(int argc, char **argv)
+{
+	const char *desc = "Disconnect from NVMeoF subsystem";
+	return disconnect(desc, argc, argv);
+}
+
 static void usage()
 {
 	printf("usage: nvme <command> [<device>] [<args>]\n");
-- 
2.1.4

[PATCH 11/11] implement the connect-all command

[Christoph Hellwig]

Connects to all controllers pointed to by the discovery service.
Includes following referrals.

Signed-off-by: Christoph Hellwig <hch at lst.de>
Reviewed-by: Sagi Grimberg <sagi at grimberg.me>
Reviewed-by: Ming Lin <ming.l at samsung.com>
Tested-by: Ming Lin <ming.l at samsung.com>
---
 fabrics.c | 126 +++++++++++++++++++++++++++++++++++++++++++++++++++-----------
 fabrics.h |   2 +-
 nvme.c    |   9 ++++-
 3 files changed, 113 insertions(+), 24 deletions(-)

diff --git a/fabrics.c b/fabrics.c
index 0a60258..c643598 100644
--- a/fabrics.c
+++ b/fabrics.c
@@ -64,6 +64,8 @@ static const match_table_t opt_tokens = {
 	{ OPT_ERR,		NULL		},
 };
 
+static int do_discover(char *argstr, bool connect);
+
 static int add_ctrl(const char *argstr)
 {
 	substring_t args[MAX_OPT_ARGS];
@@ -358,31 +360,82 @@ static int build_options(char *argstr, int max_len)
 	return 0;
 }
 
-int discover(const char *desc, int argc, char **argv)
+static int connect_ctrl(struct nvmf_disc_rsp_page_entry *e)
 {
-	char argstr[BUF_SIZE];
-	struct nvmf_disc_rsp_page_hdr *log = NULL;
-	char *dev_name;
-	int instance, numrec = 0, ret;
-	const struct argconfig_commandline_options command_line_options[] = {
-		{"transport", 't', "LIST", CFG_STRING, &cfg.transport, required_argument,
-			"transport type" },
-		{"traddr", 'a', "LIST", CFG_STRING, &cfg.traddr, required_argument,
-			"transport address" },
-		{"trsvcid", 's', "LIST", CFG_STRING, &cfg.trsvcid, required_argument,
-			"transport service id (e.g. IP port)" },
-		{"raw", 'r', "LIST", CFG_STRING, &cfg.raw, required_argument,
-			"raw output file" },
-		{0},
-	};
+	char argstr[BUF_SIZE], *p = argstr;
+	bool discover = false;
+	int len;
 
-	argconfig_parse(argc, argv, desc, command_line_options, &cfg, sizeof(cfg));
+	switch (e->nqntype) {
+	case NVME_NQN_DISC:
+		discover = true;
+	case NVME_NQN_NVME:
+		break;
+	default:
+		fprintf(stderr, "skipping unsupported NQNtype %d\n",
+			 e->nqntype);
+		return -EINVAL;
+	}
 
-	cfg.nqn = NVME_DISC_SUBSYS_NAME;
+	len = sprintf(p, "nqn=%s", e->subnqn);
+	if (len < 0)
+		return -EINVAL;
+	p += len;
 
-	ret = build_options(argstr, BUF_SIZE);
-	if (ret)
-		return ret;
+	switch (e->trtype) {
+	case NVMF_TRTYPE_LOOP: /* loop */
+		len = sprintf(p, ",transport=loop");
+		if (len < 0)
+			return -EINVAL;
+		p += len;
+		/* we can safely ignore the rest of the entries */
+		break;
+	case NVMF_TRTYPE_RDMA:
+		if (e->adrfam != NVMF_ADDR_FAMILY_IP4) {
+			fprintf(stderr, "skipping unsupported adrfam\n");
+			return -EINVAL;
+		}
+
+		len = sprintf(p, ",transport=rdma");
+		if (len < 0)
+			return -EINVAL;
+		p += len;
+
+		len = sprintf(p, ",traddr=%s", e->traddr);
+		if (len < 0)
+			return -EINVAL;
+		p += len;
+
+		len = sprintf(p, ",trsvcid=%s", e->trsvcid);
+		if (len < 0)
+			return -EINVAL;
+		p += len;
+		break;
+	default:
+		fprintf(stderr, "skipping unsupported transport %d\n",
+				 e->trtype);
+		return -EINVAL;
+	}
+
+	if (discover)
+		return do_discover(argstr, true);
+	else
+		return add_ctrl(argstr);
+}
+
+static void connect_ctrls(struct nvmf_disc_rsp_page_hdr *log, int numrec)
+{
+	int i;
+
+	for (i = 0; i < numrec; i++)
+		connect_ctrl(&log->entries[i]);
+}
+
+static int do_discover(char *argstr, bool connect)
+{
+	struct nvmf_disc_rsp_page_hdr *log = NULL;
+	char *dev_name;
+	int instance, numrec = 0, ret;
 
 	instance = add_ctrl(argstr);
 	if (instance < 0)
@@ -396,7 +449,9 @@ int discover(const char *desc, int argc, char **argv)
 
 	switch (ret) {
 	case DISC_OK:
-		if (cfg.raw)
+		if (connect)
+			connect_ctrls(log, numrec);
+		else if (cfg.raw)
 			save_discovery_log(log, numrec);
 		else
 			print_discovery_log(log, numrec);
@@ -421,6 +476,33 @@ int discover(const char *desc, int argc, char **argv)
 	return ret;
 }
 
+int discover(const char *desc, int argc, char **argv, bool connect)
+{
+	char argstr[BUF_SIZE];
+	int ret;
+	const struct argconfig_commandline_options command_line_options[] = {
+		{"transport", 't', "LIST", CFG_STRING, &cfg.transport, required_argument,
+			"transport type" },
+		{"traddr", 'a', "LIST", CFG_STRING, &cfg.traddr, required_argument,
+			"transport address" },
+		{"trsvcid", 's', "LIST", CFG_STRING, &cfg.trsvcid, required_argument,
+			"transport service id (e.g. IP port)" },
+		{"raw", 'r', "LIST", CFG_STRING, &cfg.raw, required_argument,
+			"raw output file" },
+		{0},
+	};
+
+	argconfig_parse(argc, argv, desc, command_line_options, &cfg, sizeof(cfg));
+
+	cfg.nqn = NVME_DISC_SUBSYS_NAME;
+
+	ret = build_options(argstr, BUF_SIZE);
+	if (ret)
+		return ret;
+
+	return do_discover(argstr, connect);
+}
+
 int connect(const char *desc, int argc, char **argv)
 {
 	char argstr[BUF_SIZE];
diff --git a/fabrics.h b/fabrics.h
index 7e3eb19..91aec90 100644
--- a/fabrics.h
+++ b/fabrics.h
@@ -1,7 +1,7 @@
 #ifndef _DISCOVER_H
 #define _DISCOVER_H
 
-extern int discover(const char *desc, int argc, char **argv);
+extern int discover(const char *desc, int argc, char **argv, bool connect);
 extern int connect(const char *desc, int argc, char **argv);
 extern int disconnect(const char *desc, int argc, char **argv);
 
diff --git a/nvme.c b/nvme.c
index e403022..06799a1 100644
--- a/nvme.c
+++ b/nvme.c
@@ -115,6 +115,7 @@ static const char nvme_version_string[] = NVME_VERSION;
 	ENTRY(LNVM_BBTBL_GET, "lnvm-diag-bbtbl", "Diagnose bad block table", lnvm_get_bbtbl) \
 	ENTRY(LNVM_BBTBL_SET, "lnvm-diag-set-bbtbl", "Update bad block table", lnvm_set_bbtbl) \
 	ENTRY(DISCOVER, "discover", "Discover NVMeoF subsystems", discover_cmd) \
+	ENTRY(CONNECT_ALL, "connect-all", "Discover and Connect to NVMeoF subsystems", connect_all_cmd) \
 	ENTRY(CONNECT, "connect", "Connect to NVMeoF subsystem", connect_cmd) \
 	ENTRY(DISCONNECT, "disconnect", "Disconnect from NVMeoF subsystem", disconnect_cmd) \
 	ENTRY(VERSION, "version", "Shows the program version", version) \
@@ -2845,7 +2846,13 @@ static int lnvm_set_bbtbl(int argc, char **argv)
 static int discover_cmd(int argc, char **argv)
 {
 	const char *desc = "Send command to discovery service.";
-	return discover(desc, argc, argv);
+	return discover(desc, argc, argv, false);
+}
+
+static int connect_all_cmd(int argc, char **argv)
+{
+	const char *desc = "Discover NVMeoF subsystems and connect to them";
+	return discover(desc, argc, argv, true);
 }
 
 static int connect_cmd(int argc, char **argv)
-- 
2.1.4

NVMe over Fabrics support for nvme-cli

[Keith Busch]

On Tue, Jun 07, 2016@05:19:16PM +0200, Christoph Hellwig wrote:
> Hi Keith,
> 
> the series below adds support for the discovery, connect, connect-all and
> disconnect commands to nvme-cli, which are useful to support the NVMe over
> Fabrics host code.  Most of the work is from Ming with various bits and
> pieces from me.  I also did the final rebase to the latest master tree.

Thanks, applied.

Happy to get fabrics in now as I have new features I'd like to add very
soon, and didn't want you to have to rebase again. The new features are
vendor extentions and json output. I'll post them to the linux-nvme list
first in case anyone is interested in reviewing.

NVMe over Fabrics support for nvme-cli

[Keith Busch]

On Tue, Jun 07, 2016@02:42:48PM -0400, Keith Busch wrote:
> Happy to get fabrics in now as I have new features I'd like to add very
> soon, and didn't want you to have to rebase again. The new features are
> vendor extentions and json output. I'll post them to the linux-nvme list
> first in case anyone is interested in reviewing.

Not quite ready to my latest, but I pushed it to a different branch here:

  https://github.com/linux-nvme/nvme-cli/tree/experimental

It adds a "--output-format" option that takes normal, json, or binary types.

The main thing though, is support for plugin extensions, and I made LightNVM
and Intel built-in extensions with specifics to their devices.

It also adds support for dynamically loaded plug-ins. This is so a
3rd party or vendor can extend the program without having their stuff
in-tree. It accomplishes this using the dynamic linking loader (dlopen,
dlsym, and friends) to open and register symbols.

nvme-cli plugins

[Keith Busch]

On Wed, Jun 08, 2016@03:07:54PM +0200, Christoph Hellwig wrote:
> On Tue, Jun 07, 2016@06:39:09PM -0400, Keith Busch wrote:
> > It also adds support for dynamically loaded plug-ins. This is so a
> > 3rd party or vendor can extend the program without having their stuff
> > in-tree. It accomplishes this using the dynamic linking loader (dlopen,
> > dlsym, and friends) to open and register symbols.
> 
> I'm really worried about dlopen()able symbols.  They mean you now
> expose an ABI, and once in distros will probably cause all kinds of
> vendors to ask for ABI stability.

I'm happy to drop it for now, but would like something like this since
the demand exists. If we get the right ABI, maitaining it shouldn't much
trouble; 'fio' does dynamic loading for ioengines, and nvme-cli is a
small project in comparison.

I would like to get it right the first time, so no particular rush yet.

NVMe over Fabrics support for nvme-cli

[Steve Wise]

> 
> Hi Keith,
> 
> the series below adds support for the discovery, connect, connect-all and
> disconnect commands to nvme-cli, which are useful to support the NVMe over
> Fabrics host code.  Most of the work is from Ming with various bits and
> pieces from me.  I also did the final rebase to the latest master tree.

Hey Christoph, looks like the man page needs to be updated.

Also, connect-all doesn't seem to work for me.  Am I doing something wrong?

[root at stevo1 nvme-cli]# nvme discover -t rdma -a 10.0.1.14 -s 4420
Discovery Log Number of Records 7, Generation counter 51
=====Discovery Log Entry 0======
trtype:  1
adrfam:  0
nqntype: 2
treq:    0
portid:  1
trsvcid: 4420
subnqn:  test-ram0
traddr:  10.0.1.14
rdma_prtype: 0
rdma_qptype: 0
rdma_cms:    0
rdma_pkey: 0x0000
=====Discovery Log Entry 1======
trtype:  1
adrfam:  0
nqntype: 2
treq:    0
portid:  1
trsvcid: 4420
subnqn:  test-ram1
traddr:  10.0.1.14
rdma_prtype: 0
rdma_qptype: 0
rdma_cms:    0
rdma_pkey: 0x0000
=====Discovery Log Entry 2======
trtype:  1
adrfam:  0
nqntype: 2
treq:    0
portid:  1
trsvcid: 4420
subnqn:  test-ram2
traddr:  10.0.1.14
rdma_prtype: 0
rdma_qptype: 0
rdma_cms:    0
rdma_pkey: 0x0000
=====Discovery Log Entry 3======
trtype:  1
adrfam:  0
nqntype: 2
treq:    0
portid:  1
trsvcid: 4420
subnqn:  test-ram3
traddr:  10.0.1.14
rdma_prtype: 0
rdma_qptype: 0
rdma_cms:    0
rdma_pkey: 0x0000
=====Discovery Log Entry 4======
trtype:  1
adrfam:  0
nqntype: 2
treq:    0
portid:  1
trsvcid: 4420
subnqn:  test-hynix0
traddr:  10.0.1.14
rdma_prtype: 0
rdma_qptype: 0
rdma_cms:    0
rdma_pkey: 0x0000
=====Discovery Log Entry 5======
trtype:  1
adrfam:  0
nqntype: 2
treq:    0
portid:  1
trsvcid: 4420
subnqn:  test-nullb0
traddr:  10.0.1.14
rdma_prtype: 0
rdma_qptype: 0
rdma_cms:    0
rdma_pkey: 0x0000
=====Discovery Log Entry 6======
trtype:  1
adrfam:  0
nqntype: 2
treq:    0
portid:  1
trsvcid: 4420
subnqn:  test-nullb1
traddr:  10.0.1.14
rdma_prtype: 0
rdma_qptype: 0
rdma_cms:    0
rdma_pkey: 0x0000
[root at stevo1 nvme-cli]# nvme connect-all -t rdma -a 10.0.1.14
skipping unsupported adrfam
skipping unsupported adrfam
skipping unsupported adrfam
skipping unsupported adrfam
skipping unsupported adrfam
skipping unsupported adrfam
skipping unsupported adrfam

NVMe over Fabrics support for nvme-cli

['Christoph Hellwig']

On Tue, Jun 07, 2016@03:08:15PM -0500, Steve Wise wrote:
> > 
> > Hi Keith,
> > 
> > the series below adds support for the discovery, connect, connect-all and
> > disconnect commands to nvme-cli, which are useful to support the NVMe over
> > Fabrics host code.  Most of the work is from Ming with various bits and
> > pieces from me.  I also did the final rebase to the latest master tree.
> 
> Hey Christoph, looks like the man page needs to be updated.

Or rather added..

> Also, connect-all doesn't seem to work for me.  Am I doing something wrong?

Yes!

> 
> [root at stevo1 nvme-cli]# nvme discover -t rdma -a 10.0.1.14 -s 4420
> Discovery Log Number of Records 7, Generation counter 51
> =====Discovery Log Entry 0======
> trtype:  1
> adrfam:  0

adrfam = 0 means you didn't set the address family on the target.
I'll also add checking for a valid adrfam on the target to the TODO list.

NVMe over Fabrics support for nvme-cli

[Ming Lin]

On Tue, Jun 7, 2016@1:25 PM, Christoph Hellwig <hch@lst.de> wrote:
> On Tue, Jun 07, 2016@03:08:15PM -0500, Steve Wise wrote:
>> >
>> > Hi Keith,
>> >
>> > the series below adds support for the discovery, connect, connect-all and
>> > disconnect commands to nvme-cli, which are useful to support the NVMe over
>> > Fabrics host code.  Most of the work is from Ming with various bits and
>> > pieces from me.  I also did the final rebase to the latest master tree.
>>
>> Hey Christoph, looks like the man page needs to be updated.
>
> Or rather added..
>
>> Also, connect-all doesn't seem to work for me.  Am I doing something wrong?
>
> Yes!
>
>>
>> [root at stevo1 nvme-cli]# nvme discover -t rdma -a 10.0.1.14 -s 4420
>> Discovery Log Number of Records 7, Generation counter 51
>> =====Discovery Log Entry 0======
>> trtype:  1
>> adrfam:  0
>
> adrfam = 0 means you didn't set the address family on the target.
> I'll also add checking for a valid adrfam on the target to the TODO list.

I'll send a patch soon.

NVMe over Fabrics support for nvme-cli

[Steve Wise]

> > Hey Christoph, looks like the man page needs to be updated.
> 
> Or rather added..
> 
> > Also, connect-all doesn't seem to work for me.  Am I doing something wrong?
> 
> Yes!
> 
> >
> > [root at stevo1 nvme-cli]# nvme discover -t rdma -a 10.0.1.14 -s 4420
> > Discovery Log Number of Records 7, Generation counter 51
> > =====Discovery Log Entry 0======
> > trtype:  1
> > adrfam:  0
> 
> adrfam = 0 means you didn't set the address family on the target.
> I'll also add checking for a valid adrfam on the target to the TODO list.

I updated my json input file to add "ipv4" to the "adrfam" entries, and
connect-all works like a champ.

Thanks,

Steve.