* [PATCH v1 net-next] ip6_tunnel: Adding support of mapping rules for MAP-E tunnel
From: Felix Jia @ 2018-11-20 1:53 UTC (permalink / raw)
To: David S . Miller
Cc: Blair Steven, netdev, Felix Jia, Sheena Mira-ato, Masakazu Asama
From: Blair Steven <blair.steven@alliedtelesis.co.nz>
Mapping of Addresses and Ports with Encapsulation (MAP-E) is defined in
RFC7597, and is an IPv6 transition technology providing interoperability
between IPv4 and IPv6 networks.
MAP-E uses the encapsulation mode described in RFC2473 (IPv6 Tunneling)
to transport IPv4 and IPv6 packets over an IPv6 network. It requires a
list rules for mapping between IPv4 prefix/shared addresses and IPv6
addresses.
This patch also support the mapping rules define in the draft3 version of
the RFC.
Co-developed-by: Felix Jia <felix.jia@alliedtelesis.co.nz>
Co-developed-by: Sheena Mira-ato <sheena.mira-ato@alliedtelesis.co.nz>
Co-developed-by: Masakazu Asama <masakazu.asama@gmail.com>
Signed-off-by: Blair Steven <blair.steven@alliedtelesis.co.nz>
Signed-off-by: Felix Jia <felix.jia@alliedtelesis.co.nz>
Signed-off-by: Sheena Mira-ato <sheena.mira-ato@alliedtelesis.co.nz>
Signed-off-by: Masakazu Asama <masakazu.asama@gmail.com>
---
include/net/ip6_tunnel.h | 18 ++
include/uapi/linux/if_tunnel.h | 18 ++
net/ipv6/ip6_tunnel.c | 491 ++++++++++++++++++++++++++++++++-
3 files changed, 525 insertions(+), 2 deletions(-)
diff --git a/include/net/ip6_tunnel.h b/include/net/ip6_tunnel.h
index 69b4bcf880c9..daa159185228 100644
--- a/include/net/ip6_tunnel.h
+++ b/include/net/ip6_tunnel.h
@@ -18,6 +18,16 @@
/* determine capability on a per-packet basis */
#define IP6_TNL_F_CAP_PER_PACKET 0x40000
+struct ip6_tnl_rule {
+ u8 version;
+ struct in6_addr ipv6_subnet;
+ u8 ipv6_prefixlen;
+ struct in_addr ipv4_subnet;
+ u8 ipv4_prefixlen;
+ u8 ea_length;
+ u8 psid_offset;
+};
+
struct __ip6_tnl_parm {
char name[IFNAMSIZ]; /* name of tunnel device */
int link; /* ifindex of underlying L2 interface */
@@ -40,6 +50,13 @@ struct __ip6_tnl_parm {
__u8 erspan_ver; /* ERSPAN version */
__u8 dir; /* direction */
__u16 hwid; /* hwid */
+ __u8 rule_action;
+ struct ip6_tnl_rule rule;
+};
+
+struct ip6_rule_list {
+ struct list_head list;
+ struct ip6_tnl_rule data;
};
/* IPv6 tunnel */
@@ -63,6 +80,7 @@ struct ip6_tnl {
int encap_hlen; /* Encap header length (FOU,GUE) */
struct ip_tunnel_encap encap;
int mlink;
+ struct ip6_rule_list rules;
};
struct ip6_tnl_encap_ops {
diff --git a/include/uapi/linux/if_tunnel.h b/include/uapi/linux/if_tunnel.h
index 1b3d148c4560..7cb09c8c4d8a 100644
--- a/include/uapi/linux/if_tunnel.h
+++ b/include/uapi/linux/if_tunnel.h
@@ -77,10 +77,28 @@ enum {
IFLA_IPTUN_ENCAP_DPORT,
IFLA_IPTUN_COLLECT_METADATA,
IFLA_IPTUN_FWMARK,
+ IFLA_IPTUN_RULE_VERSION,
+ IFLA_IPTUN_RULE_ACTION,
+ IFLA_IPTUN_RULE_IPV6_PREFIX,
+ IFLA_IPTUN_RULE_IPV6_PREFIXLEN,
+ IFLA_IPTUN_RULE_IPV4_PREFIX,
+ IFLA_IPTUN_RULE_IPV4_PREFIXLEN,
+ IFLA_IPTUN_RULE_EA_LENGTH,
+ IFLA_IPTUN_RULE_PSID_OFFSET,
__IFLA_IPTUN_MAX,
};
#define IFLA_IPTUN_MAX (__IFLA_IPTUN_MAX - 1)
+enum map_rule_versions {
+ MAP_VERSION_RFC,
+ MAP_VERSION_DRAFT3,
+};
+
+enum tunnel_rule_actions {
+ TUNNEL_RULE_ACTION_ADD = 1,
+ TUNNEL_RULE_ACTION_DELETE = 2,
+};
+
enum tunnel_encap_types {
TUNNEL_ENCAP_NONE,
TUNNEL_ENCAP_FOU,
diff --git a/net/ipv6/ip6_tunnel.c b/net/ipv6/ip6_tunnel.c
index a9d06d4dd057..e13dc5bd09b5 100644
--- a/net/ipv6/ip6_tunnel.c
+++ b/net/ipv6/ip6_tunnel.c
@@ -20,6 +20,8 @@
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+#include <linux/list.h>
+#include <linux/list_sort.h>
#include <linux/module.h>
#include <linux/capability.h>
#include <linux/errno.h>
@@ -32,6 +34,7 @@
#include <linux/net.h>
#include <linux/in6.h>
#include <linux/netdevice.h>
+#include <linux/inetdevice.h>
#include <linux/if_arp.h>
#include <linux/icmpv6.h>
#include <linux/init.h>
@@ -124,6 +127,228 @@ static struct net_device_stats *ip6_get_stats(struct net_device *dev)
return &dev->stats;
}
+int
+ip6_get_addrport(struct iphdr *iph, __be32 *saddr4, __be32 *daddr4,
+ __be16 *sport4, __be16 *dport4, __u8 *proto, int *icmperr)
+{
+ u8 *ptr;
+ struct iphdr *icmpiph = NULL;
+ struct tcphdr *tcph, *icmptcph;
+ struct udphdr *udph, *icmpudph;
+ struct icmphdr *icmph, *icmpicmph;
+
+ *icmperr = 0;
+ *saddr4 = iph->saddr;
+ *daddr4 = iph->daddr;
+ ptr = (u8 *)iph;
+ ptr += iph->ihl * 4;
+ switch (iph->protocol) {
+ case IPPROTO_TCP:
+ *proto = IPPROTO_TCP;
+ tcph = (struct tcphdr *)ptr;
+ *sport4 = tcph->source;
+ *dport4 = tcph->dest;
+ break;
+ case IPPROTO_UDP:
+ *proto = IPPROTO_UDP;
+ udph = (struct udphdr *)ptr;
+ *sport4 = udph->source;
+ *dport4 = udph->dest;
+ break;
+ case IPPROTO_ICMP:
+ *proto = IPPROTO_ICMP;
+ icmph = (struct icmphdr *)ptr;
+ switch (icmph->type) {
+ case ICMP_DEST_UNREACH:
+ case ICMP_SOURCE_QUENCH:
+ case ICMP_TIME_EXCEEDED:
+ case ICMP_PARAMETERPROB:
+ *icmperr = 1;
+ ptr = (u8 *)icmph;
+ ptr += sizeof(struct icmphdr);
+ icmpiph = (struct iphdr *)ptr;
+ *saddr4 = icmpiph->saddr;
+ *daddr4 = icmpiph->daddr;
+ if (ntohs(iph->tot_len) < icmpiph->ihl * 4 + 12)
+ return -1;
+ ptr += icmpiph->ihl * 4;
+ switch (icmpiph->protocol) {
+ case IPPROTO_TCP:
+ *proto = IPPROTO_TCP;
+ icmptcph = (struct tcphdr *)ptr;
+ *sport4 = icmptcph->source;
+ *dport4 = icmptcph->dest;
+ break;
+ case IPPROTO_UDP:
+ *proto = IPPROTO_UDP;
+ icmpudph = (struct udphdr *)ptr;
+ *sport4 = icmpudph->source;
+ *dport4 = icmpudph->dest;
+ break;
+ case IPPROTO_ICMP:
+ *proto = IPPROTO_ICMP;
+ icmpicmph = (struct icmphdr *)ptr;
+ *sport4 = icmpicmph->un.echo.id;
+ *dport4 = icmpicmph->un.echo.id;
+ break;
+ default:
+ return -1;
+ }
+ break;
+ default:
+ *sport4 = icmph->un.echo.id;
+ *dport4 = icmph->un.echo.id;
+ }
+ break;
+ default:
+ return -1;
+ }
+
+ return 0;
+}
+
+int
+ip6_mape_gen_addr6(struct ip6_tnl_rule *mr, struct in6_addr *addr6,
+ __be32 addr4, __be16 port4)
+{
+ int i, pbw0, pbi0, pbi1;
+ __u32 addr[4];
+ __u32 psid = 0;
+ __u32 mask = 0;
+ __u32 a = ntohl(addr4);
+ __u16 p = ntohs(port4);
+ int psid_prefix_length = 0;
+ int psid_mask;
+ __u32 id0 = 0;
+ __u32 id1 = 0;
+
+ /* If we have no rule then we can return immediately */
+ if (!mr)
+ return -1;
+
+ psid_prefix_length = mr->ea_length - (32 - mr->ipv4_prefixlen);
+
+ /* In some configurations the PSID can overlap the IPv4 address
+ * segment
+ */
+ if (psid_prefix_length < 0)
+ a &= 0xffffffff << (psid_prefix_length * -1);
+
+ /* Find the PSID from the port by shifting the non-PSID bits out */
+ if (psid_prefix_length > 0) {
+ mask = 0xffffffff >> (32 - psid_prefix_length);
+ psid =
+ (p >> (16 - psid_prefix_length - mr->psid_offset)) & mask;
+ }
+
+ /* We start with the IPv6 subnet provided for this rule */
+ for (i = 0; i < 4; ++i)
+ addr[i] = ntohl(mr->ipv6_subnet.s6_addr32[i]);
+
+ /* We then copy in the IPv4 host portion - pbw0 is the starting
+ * 32 bit chunk, pbi0 is the offset into that chunk
+ */
+ if (mr->ipv4_prefixlen < 32) {
+ pbw0 = mr->ipv6_prefixlen >> 5;
+ pbi0 = mr->ipv6_prefixlen & 0x1f;
+ addr[pbw0] |= (a << mr->ipv4_prefixlen) >> pbi0;
+ pbi1 = pbi0 - mr->ipv4_prefixlen;
+ if (pbi1 > 0)
+ addr[pbw0 + 1] |= a << (32 - pbi1);
+ }
+
+ /* The PSID may need to be inserted into the IPv6 address within the
+ * top 64 bits. Again: pbw0 is the chunk, pbi0 is the offset.
+ */
+ if ((psid_prefix_length - mr->psid_offset) > 0) {
+ psid_mask = mask;
+ pbw0 = (mr->ipv6_prefixlen + (32 - mr->ipv4_prefixlen)) >> 5;
+ pbi0 = (mr->ipv6_prefixlen + (32 - mr->ipv4_prefixlen)) & 0x1f;
+
+ pbi1 = pbi0 - (32 - psid_prefix_length);
+
+ if (pbi1 > 0) {
+ addr[pbw0] |= (psid >> pbi1);
+ addr[pbw0 + 1] |= (psid & psid_mask) << (32 - pbi1);
+ } else {
+ addr[pbw0] |= psid << (32 - pbi0 - psid_prefix_length);
+ }
+ }
+
+ /* The format of the interface id changes depending on version */
+ if (mr->version == MAP_VERSION_RFC) {
+ addr[2] |= (a >> 16);
+ addr[3] |= (a << 16);
+ addr[3] |= psid;
+ } else if (mr->version == MAP_VERSION_DRAFT3) {
+ id0 |= a >> 8;
+ id1 |= (a & 0xFF) << 24;
+ id1 |= psid << 8;
+ addr[2] |= id0;
+ addr[3] |= id1;
+ } else {
+ return -1;
+ }
+
+ /* Copy the formed address back into network order */
+ for (i = 0; i < 4; ++i)
+ addr6->s6_addr32[i] = htonl(addr[i]);
+
+ return 0;
+}
+
+static struct ip6_tnl_rule *ip6_tnl_rule_find(struct net_device *dev,
+ __be32 _dst)
+{
+ struct ip6_tnl *t = netdev_priv(dev);
+ struct ip6_rule_list *pos = NULL;
+ u32 dst = ntohl(_dst);
+
+ list_for_each_entry(pos, &t->rules.list, list) {
+ int mask =
+ 0xFFFFFFFF ^ ((1 << (32 - pos->data.ipv4_prefixlen)) - 1);
+ if ((dst & mask) == ntohl(pos->data.ipv4_subnet.s_addr))
+ return &pos->data;
+ }
+ return NULL;
+}
+
+static void
+ip6_tnl_mape_dst(struct net_device *dev, struct sk_buff *skb,
+ struct flowi6 *fl6)
+{
+ struct ip6_tnl *t = netdev_priv(dev);
+ struct iphdr *iph;
+ __be32 saddr4, daddr4, addr;
+ __be16 sport4, dport4, port;
+ __u8 proto;
+ int icmperr;
+ struct ip6_tnl_rule *mr = NULL;
+
+ /* If we have no rules then packets always go to the tunnel dst */
+ if (list_empty(&t->rules.list))
+ return;
+
+ /* Find the src / dst address / port (or ID for ICMP) */
+ iph = ip_hdr(skb);
+ ip6_get_addrport(iph, &saddr4, &daddr4, &sport4, &dport4, &proto,
+ &icmperr);
+
+ if (icmperr) {
+ mr = ip6_tnl_rule_find(dev, saddr4);
+ addr = saddr4;
+ port = sport4;
+ } else {
+ mr = ip6_tnl_rule_find(dev, daddr4);
+ addr = daddr4;
+ port = dport4;
+ }
+
+ /* Generate an destination IPv6 address from the internal packet */
+ if (mr)
+ ip6_mape_gen_addr6(mr, &fl6->daddr, addr, port);
+}
+
/**
* ip6_tnl_lookup - fetch tunnel matching the end-point addresses
* @remote: the address of the tunnel exit-point
@@ -207,6 +432,33 @@ ip6_tnl_bucket(struct ip6_tnl_net *ip6n, const struct __ip6_tnl_parm *p)
return &ip6n->tnls[prio][h];
}
+/**
+ * ip6_tnl_bucket_r_any - get head of list matching given tunnel parameters
+ * @p: parameters containing tunnel end-points
+ *
+ * Description:
+ * ip6_tnl_bucket_r_any() returns the head of the list matching the
+ * &struct in6_addr entry laddr in @p.
+ *
+ * Return: head of IPv6 tunnel list
+ **/
+
+static struct ip6_tnl __rcu **
+ip6_tnl_bucket_r_any(struct ip6_tnl_net *ip6n, const struct __ip6_tnl_parm *p)
+{
+ const struct in6_addr *local = &p->laddr;
+ unsigned int h = 0;
+ int prio = 0;
+ struct in6_addr any;
+
+ if (!ipv6_addr_any(local)) {
+ memset(&any, 0, sizeof(any));
+ prio = 1;
+ h = HASH(&any, local);
+ }
+ return &ip6n->tnls[prio][h];
+}
+
/**
* ip6_tnl_link - add tunnel to hash table
* @t: tunnel to be added
@@ -215,7 +467,12 @@ ip6_tnl_bucket(struct ip6_tnl_net *ip6n, const struct __ip6_tnl_parm *p)
static void
ip6_tnl_link(struct ip6_tnl_net *ip6n, struct ip6_tnl *t)
{
- struct ip6_tnl __rcu **tp = ip6_tnl_bucket(ip6n, &t->parms);
+ struct ip6_tnl __rcu **tp;
+
+ if (!list_empty(&t->rules.list))
+ tp = ip6_tnl_bucket_r_any(ip6n, &t->parms);
+ else
+ tp = ip6_tnl_bucket(ip6n, &t->parms);
if (t->parms.collect_md)
rcu_assign_pointer(ip6n->collect_md_tun, t);
@@ -234,6 +491,15 @@ ip6_tnl_unlink(struct ip6_tnl_net *ip6n, struct ip6_tnl *t)
struct ip6_tnl __rcu **tp;
struct ip6_tnl *iter;
+ for (tp = ip6_tnl_bucket_r_any(ip6n, &t->parms);
+ (iter = rtnl_dereference(*tp)) != NULL;
+ tp = &iter->next) {
+ if (t == iter) {
+ rcu_assign_pointer(*tp, t->next);
+ return;
+ }
+ }
+
if (t->parms.collect_md)
rcu_assign_pointer(ip6n->collect_md_tun, NULL);
@@ -352,6 +618,16 @@ static struct ip6_tnl *ip6_tnl_locate(struct net *net,
struct ip6_tnl *t;
struct ip6_tnl_net *ip6n = net_generic(net, ip6_tnl_net_id);
+ for (tp = ip6_tnl_bucket_r_any(ip6n, p);
+ (t = rtnl_dereference(*tp)) != NULL;
+ tp = &t->next) {
+ if (ipv6_addr_equal(local, &t->parms.laddr)) {
+ if (create)
+ return ERR_PTR(-EEXIST);
+ return t;
+ }
+ }
+
for (tp = ip6_tnl_bucket(ip6n, p);
(t = rtnl_dereference(*tp)) != NULL;
tp = &t->next) {
@@ -880,6 +1156,64 @@ static const struct tnl_ptk_info tpi_v4 = {
.proto = htons(ETH_P_IP),
};
+static int ip6_tnl_ipv4_addr_match(struct ip6_tnl *t, __be32 addr)
+{
+ struct in_device *ind;
+ int ret = 0;
+
+ ind = in_dev_get(t->dev);
+ if (!ind)
+ return 0;
+
+ for_ifa(ind) {
+ if (ifa->ifa_address == addr) {
+ ret = 1;
+ break;
+ }
+ }
+ endfor_ifa(ind);
+ in_dev_put(ind);
+
+ return ret;
+}
+
+static int ip6_tnl_map_rule_validate(struct ip6_tnl *t, struct sk_buff *skb)
+{
+ struct ipv6hdr *ipv6h;
+ struct iphdr *iph;
+ __be32 src, dst;
+ __be16 sport, dport;
+ __u8 proto;
+ int icmperr;
+ struct ip6_tnl_rule *mr = NULL;
+ struct in6_addr src6 = { };
+
+ if (likely(list_empty(&t->rules.list)))
+ return 1;
+
+ iph = ipip_hdr(skb);
+ ip6_get_addrport(iph, &src, &dst, &sport, &dport, &proto, &icmperr);
+
+ /* Check if the destination of the packet is for the MAP-E tunnel. */
+ if (!ip6_tnl_ipv4_addr_match(t, dst))
+ return 0;
+
+ /* if the packet's destination is for the tunnel, we need to check
+ * if the source ipv4 address matches any map rules. If there is
+ * no matching rule, just return success (the packet is from br).
+ * Otherwise, we check if the source ipv6 address is valid.
+ */
+ mr = ip6_tnl_rule_find(t->dev, src);
+ if (mr) {
+ ip6_mape_gen_addr6(mr, &src6, src, sport);
+ ipv6h = ipv6_hdr(skb);
+ if (!ipv6_addr_equal(&src6, &ipv6h->saddr))
+ return 0;
+ }
+
+ return 1;
+}
+
static int ipxip6_rcv(struct sk_buff *skb, u8 ipproto,
const struct tnl_ptk_info *tpi,
int (*dscp_ecn_decapsulate)(const struct ip6_tnl *t,
@@ -905,6 +1239,8 @@ static int ipxip6_rcv(struct sk_buff *skb, u8 ipproto,
goto drop;
if (iptunnel_pull_header(skb, 0, tpi->proto, false))
goto drop;
+ if (!ip6_tnl_map_rule_validate(t, skb))
+ goto drop;
if (t->parms.collect_md) {
tun_dst = ipv6_tun_rx_dst(skb, 0, 0, 0);
if (!tun_dst)
@@ -1092,6 +1428,12 @@ int ip6_tnl_xmit(struct sk_buff *skb, struct net_device *dev, __u8 dsfield,
use_cache = true;
}
+ /* In MAP-E mode we need to always do a route lookup */
+ if (!list_empty(&t->rules.list)) {
+ use_cache = false;
+ ip6_tnl_mape_dst(dev, skb, fl6);
+ }
+
if (use_cache)
dst = dst_cache_get(&t->dst_cache);
@@ -1863,6 +2205,8 @@ ip6_tnl_dev_init_gen(struct net_device *dev)
dev->min_mtu = ETH_MIN_MTU;
dev->max_mtu = IP6_MAX_MTU - dev->hard_header_len;
+ INIT_LIST_HEAD(&t->rules.list);
+
return 0;
destroy_dst:
@@ -1968,6 +2312,95 @@ static void ip6_tnl_netlink_parms(struct nlattr *data[],
if (data[IFLA_IPTUN_FWMARK])
parms->fwmark = nla_get_u32(data[IFLA_IPTUN_FWMARK]);
+
+ if (data[IFLA_IPTUN_RULE_ACTION])
+ parms->rule_action = nla_get_u8(data[IFLA_IPTUN_RULE_ACTION]);
+
+ if (data[IFLA_IPTUN_RULE_VERSION])
+ parms->rule.version = nla_get_u8(data[IFLA_IPTUN_RULE_VERSION]);
+
+ if (data[IFLA_IPTUN_RULE_IPV6_PREFIX])
+ parms->rule.ipv6_subnet =
+ nla_get_in6_addr(data[IFLA_IPTUN_RULE_IPV6_PREFIX]);
+
+ if (data[IFLA_IPTUN_RULE_IPV6_PREFIXLEN])
+ parms->rule.ipv6_prefixlen =
+ nla_get_u8(data[IFLA_IPTUN_RULE_IPV6_PREFIXLEN]);
+
+ if (data[IFLA_IPTUN_RULE_IPV4_PREFIX])
+ parms->rule.ipv4_subnet.s_addr =
+ nla_get_in_addr(data[IFLA_IPTUN_RULE_IPV4_PREFIX]);
+
+ if (data[IFLA_IPTUN_RULE_IPV4_PREFIXLEN])
+ parms->rule.ipv4_prefixlen =
+ nla_get_u8(data[IFLA_IPTUN_RULE_IPV4_PREFIXLEN]);
+
+ if (data[IFLA_IPTUN_RULE_EA_LENGTH])
+ parms->rule.ea_length =
+ nla_get_u8(data[IFLA_IPTUN_RULE_EA_LENGTH]);
+
+ if (data[IFLA_IPTUN_RULE_PSID_OFFSET])
+ parms->rule.psid_offset =
+ nla_get_u8(data[IFLA_IPTUN_RULE_PSID_OFFSET]);
+}
+
+static int ip6_tnl_rule_cmp(void *unused, struct list_head *_a,
+ struct list_head *_b)
+{
+ struct ip6_rule_list *a = list_entry(_a, struct ip6_rule_list, list);
+ struct ip6_rule_list *b = list_entry(_b, struct ip6_rule_list, list);
+ int diff;
+
+ diff = b->data.ipv4_prefixlen - a->data.ipv4_prefixlen;
+ if (!diff)
+ diff = b->data.ipv4_subnet.s_addr - a->data.ipv4_subnet.s_addr;
+
+ if (!diff)
+ diff = b->data.ipv6_prefixlen - a->data.ipv6_prefixlen;
+
+ if (!diff)
+ diff =
+ memcmp(&b->data.ipv6_subnet, &a->data.ipv6_subnet,
+ sizeof(b->data.ipv6_subnet));
+
+ if (!diff)
+ diff = b->data.ea_length - a->data.ea_length;
+
+ if (!diff)
+ diff = b->data.psid_offset - a->data.psid_offset;
+
+ return diff;
+}
+
+static void ip6_tnl_add_rule(struct net_device *dev,
+ struct __ip6_tnl_parm *new_cfg)
+{
+ struct ip6_tnl *t = netdev_priv(dev);
+ struct ip6_rule_list *new_rule = kmalloc(sizeof(t->rules), GFP_KERNEL);
+
+ if (!new_rule)
+ return;
+
+ memcpy(&new_rule->data, &new_cfg->rule, sizeof(new_rule->data));
+ INIT_LIST_HEAD(&new_rule->list);
+ list_add(&new_rule->list, &t->rules.list);
+ list_sort(NULL, &t->rules.list, ip6_tnl_rule_cmp);
+}
+
+static void ip6_tnl_delete_rule(struct net_device *dev,
+ struct __ip6_tnl_parm *_old_rule)
+{
+ struct ip6_tnl *t = netdev_priv(dev);
+ struct ip6_rule_list *pos = NULL;
+ struct ip6_rule_list old_rule = { .data = _old_rule->rule };
+ struct ip6_rule_list *iter = NULL;
+
+ list_for_each_entry_safe(pos, iter, &t->rules.list, list) {
+ if (ip6_tnl_rule_cmp(NULL, &pos->list, &old_rule.list) == 0) {
+ list_del(&pos->list);
+ break;
+ }
+ }
}
static bool ip6_tnl_netlink_encap_parms(struct nlattr *data[],
@@ -2035,6 +2468,16 @@ static int ip6_tnl_newlink(struct net *src_net, struct net_device *dev,
err = ip6_tnl_create2(dev);
if (!err && tb[IFLA_MTU])
ip6_tnl_change_mtu(dev, nla_get_u32(tb[IFLA_MTU]));
+ if (!err && data[IFLA_IPTUN_RULE_ACTION]) {
+ switch (nla_get_u8(data[IFLA_IPTUN_RULE_ACTION])) {
+ case TUNNEL_RULE_ACTION_ADD:
+ ip6_tnl_add_rule(dev, &nt->parms);
+ break;
+ case TUNNEL_RULE_ACTION_DELETE:
+ ip6_tnl_delete_rule(dev, &nt->parms);
+ break;
+ }
+ }
return err;
}
@@ -2048,6 +2491,7 @@ static int ip6_tnl_changelink(struct net_device *dev, struct nlattr *tb[],
struct net *net = t->net;
struct ip6_tnl_net *ip6n = net_generic(net, ip6_tnl_net_id);
struct ip_tunnel_encap ipencap;
+ int err;
if (dev == ip6n->fb_tnl_dev)
return -EINVAL;
@@ -2069,13 +2513,32 @@ static int ip6_tnl_changelink(struct net_device *dev, struct nlattr *tb[],
} else
t = netdev_priv(dev);
- return ip6_tnl_update(t, &p);
+ err = ip6_tnl_update(t, &p);
+ if (!err && data[IFLA_IPTUN_RULE_ACTION]) {
+ switch (nla_get_u8(data[IFLA_IPTUN_RULE_ACTION])) {
+ case TUNNEL_RULE_ACTION_ADD:
+ ip6_tnl_add_rule(dev, &p);
+ break;
+ case TUNNEL_RULE_ACTION_DELETE:
+ ip6_tnl_delete_rule(dev, &p);
+ break;
+ }
+ }
+
+ return err;
}
static void ip6_tnl_dellink(struct net_device *dev, struct list_head *head)
{
struct net *net = dev_net(dev);
struct ip6_tnl_net *ip6n = net_generic(net, ip6_tnl_net_id);
+ struct ip6_tnl *t = netdev_priv(dev);
+ struct ip6_rule_list *pos = NULL;
+ struct ip6_rule_list *iter = NULL;
+
+ list_for_each_entry_safe(pos, iter, &t->rules.list, list) {
+ list_del(&pos->list);
+ }
if (dev != ip6n->fb_tnl_dev)
unregister_netdevice_queue(dev, head);
@@ -2112,6 +2575,22 @@ static size_t ip6_tnl_get_size(const struct net_device *dev)
nla_total_size(0) +
/* IFLA_IPTUN_FWMARK */
nla_total_size(4) +
+ /* IFLA_IPTUN_RULE_VERSION */
+ nla_total_size(1) +
+ /* IFLA_IPTUN_RULE_ACTION */
+ nla_total_size(1) +
+ /* IFLA_IPTUN_RULE_IPV6_PREFIXLEN */
+ nla_total_size(1) +
+ /* IFLA_IPTUN_RULE_IPV4_PREFIXLEN */
+ nla_total_size(1) +
+ /* IFLA_IPTUN_RULE_EA_LENGTH */
+ nla_total_size(1) +
+ /* IFLA_IPTUN_RULE_PSID_OFFSET */
+ nla_total_size(1) +
+ /* IFLA_IPTUN_RULE_IPV6_PREFIX */
+ nla_total_size(sizeof(struct in6_addr)) +
+ /* IFLA_IPTUN_RULE_IPV4_PREFIX */
+ nla_total_size(sizeof(struct in6_addr)) +
0;
}
@@ -2170,6 +2649,14 @@ static const struct nla_policy ip6_tnl_policy[IFLA_IPTUN_MAX + 1] = {
[IFLA_IPTUN_ENCAP_DPORT] = { .type = NLA_U16 },
[IFLA_IPTUN_COLLECT_METADATA] = { .type = NLA_FLAG },
[IFLA_IPTUN_FWMARK] = { .type = NLA_U32 },
+ [IFLA_IPTUN_RULE_VERSION] = { .type = NLA_U8 },
+ [IFLA_IPTUN_RULE_ACTION] = { .type = NLA_U8 },
+ [IFLA_IPTUN_RULE_IPV6_PREFIXLEN] = { .type = NLA_U8 },
+ [IFLA_IPTUN_RULE_IPV4_PREFIXLEN] = { .type = NLA_U8 },
+ [IFLA_IPTUN_RULE_EA_LENGTH] = { .type = NLA_U8 },
+ [IFLA_IPTUN_RULE_PSID_OFFSET] = { .type = NLA_U8 },
+ [IFLA_IPTUN_RULE_IPV6_PREFIX] = { .len = sizeof(struct in6_addr) },
+ [IFLA_IPTUN_RULE_IPV4_PREFIX] = { .len = sizeof(struct in_addr) },
};
static struct rtnl_link_ops ip6_link_ops __read_mostly = {
--
2.19.1
^ permalink raw reply related
* Re: [PATCH v3 00/21] octeontx2-af: NPC MCAM support and FLR handling
From: David Miller @ 2018-11-20 1:56 UTC (permalink / raw)
To: sunil.kovvuri; +Cc: netdev, arnd, linux-soc, sgoutham
In-Reply-To: <1542624463-5868-1-git-send-email-sunil.kovvuri@gmail.com>
From: sunil.kovvuri@gmail.com
Date: Mon, 19 Nov 2018 16:17:22 +0530
> This patchset is a continuation to earlier submitted three patch
> series to add a new driver for Marvell's OcteonTX2 SOC's
> Resource virtualization unit (RVU) admin function driver.
...
Series applied, thanks.
^ permalink raw reply
* Re: [PATCH net-next 00/18] selftests: Add tests for VXLAN at an 802.1d bridge
From: David Miller @ 2018-11-20 2:00 UTC (permalink / raw)
To: idosch; +Cc: netdev, linux-kselftest, shuah, jiri, petrm, roopa, mlxsw
In-Reply-To: <20181119161006.5405-1-idosch@mellanox.com>
From: Ido Schimmel <idosch@mellanox.com>
Date: Mon, 19 Nov 2018 16:11:06 +0000
> This patchset adds several tests for VXLAN attached to an 802.1d bridge
> and fixes a related bug.
So I'm still applying this as-is to net-next, but I still expect patch #1 to
be submitted to net.
Thanks.
^ permalink raw reply
* Re: [PATCH net] ipv6: fix a dst leak when removing its exception
From: David Miller @ 2018-11-20 2:16 UTC (permalink / raw)
To: lucien.xin; +Cc: dsa, netdev
In-Reply-To: <CADvbK_dBExLLRRtft4tF5_gUeshOKoOT5mxnC6+zjpjOoHC8uw@mail.gmail.com>
From: Xin Long <lucien.xin@gmail.com>
Date: Thu, 15 Nov 2018 16:23:38 +0900
> The attachment is the ip6_dst.sh with IPVS.
>
> # sh ip6_dst.sh
Maybe a selftests candidate?
^ permalink raw reply
* Re: [PATCH net] ipv6: fix a dst leak when removing its exception
From: David Ahern @ 2018-11-20 2:19 UTC (permalink / raw)
To: David Miller, lucien.xin; +Cc: netdev
In-Reply-To: <20181119.181617.1403625899925070224.davem@davemloft.net>
On 11/19/18 7:16 PM, David Miller wrote:
> From: Xin Long <lucien.xin@gmail.com>
> Date: Thu, 15 Nov 2018 16:23:38 +0900
>
>> The attachment is the ip6_dst.sh with IPVS.
>>
>> # sh ip6_dst.sh
>
> Maybe a selftests candidate?
>
That script was not a reliable reproducer for me.
I created a much simpler one that shows the problem every time - and
proves the change. It needs some adjustments to work as a selftest.
Specifically, it needs to check dmesg for net_device reference counts
after the rcu delay. I'll add to the to-do list.
^ permalink raw reply
* Re: [RFCv3 PATCH 1/6] uacce: Add documents for WarpDrive/uacce
From: Kenneth Lee @ 2018-11-20 2:30 UTC (permalink / raw)
To: Leon Romanovsky
Cc: Tim Sell, linux-doc, Alexander Shishkin, Zaibo Xu, zhangfei.gao,
linuxarm, haojian.zhuang, Christoph Lameter, Hao Fang,
Gavin Schenk, RDMA mailing list, Vinod Koul, Jason Gunthorpe,
Doug Ledford, Uwe Kleine-König, David Kershner, Kenneth Lee,
Johan Hovold, Cyrille Pitchen, Sagar Dharia
In-Reply-To: <20181119104801.GF8268@mtr-leonro.mtl.com>
On Mon, Nov 19, 2018 at 12:48:01PM +0200, Leon Romanovsky wrote:
> Date: Mon, 19 Nov 2018 12:48:01 +0200
> From: Leon Romanovsky <leon@kernel.org>
> To: Kenneth Lee <liguozhu@hisilicon.com>
> CC: Tim Sell <timothy.sell@unisys.com>, linux-doc@vger.kernel.org,
> Alexander Shishkin <alexander.shishkin@linux.intel.com>, Zaibo Xu
> <xuzaibo@huawei.com>, zhangfei.gao@foxmail.com, linuxarm@huawei.com,
> haojian.zhuang@linaro.org, Christoph Lameter <cl@linux.com>, Hao Fang
> <fanghao11@huawei.com>, Gavin Schenk <g.schenk@eckelmann.de>, RDMA mailing
> list <linux-rdma@vger.kernel.org>, Vinod Koul <vkoul@kernel.org>, Jason
> Gunthorpe <jgg@ziepe.ca>, Doug Ledford <dledford@redhat.com>, Uwe
> Kleine-König <u.kleine-koenig@pengutronix.de>, David Kershner
> <david.kershner@unisys.com>, Kenneth Lee <nek.in.cn@gmail.com>, Johan
> Hovold <johan@kernel.org>, Cyrille Pitchen
> <cyrille.pitchen@free-electrons.com>, Sagar Dharia
> <sdharia@codeaurora.org>, Jens Axboe <axboe@kernel.dk>,
> guodong.xu@linaro.org, linux-netdev <netdev@vger.kernel.org>, Randy Dunlap
> <rdunlap@infradead.org>, linux-kernel@vger.kernel.org, Zhou Wang
> <wangzhou1@hisilicon.com>, linux-crypto@vger.kernel.org, Philippe
> Ombredanne <pombredanne@nexb.com>, Sanyog Kale <sanyog.r.kale@intel.com>,
> "David S. Miller" <davem@davemloft.net>,
> linux-accelerators@lists.ozlabs.org, Jerome Glisse <jglisse@redhat.com>
> Subject: Re: [RFCv3 PATCH 1/6] uacce: Add documents for WarpDrive/uacce
> User-Agent: Mutt/1.10.1 (2018-07-13)
> Message-ID: <20181119104801.GF8268@mtr-leonro.mtl.com>
>
> On Mon, Nov 19, 2018 at 05:19:10PM +0800, Kenneth Lee wrote:
> > On Mon, Nov 19, 2018 at 05:14:05PM +0800, Kenneth Lee wrote:
> > > Date: Mon, 19 Nov 2018 17:14:05 +0800
> > > From: Kenneth Lee <liguozhu@hisilicon.com>
> > > To: Leon Romanovsky <leon@kernel.org>
> > > CC: Tim Sell <timothy.sell@unisys.com>, linux-doc@vger.kernel.org,
> > > Alexander Shishkin <alexander.shishkin@linux.intel.com>, Zaibo Xu
> > > <xuzaibo@huawei.com>, zhangfei.gao@foxmail.com, linuxarm@huawei.com,
> > > haojian.zhuang@linaro.org, Christoph Lameter <cl@linux.com>, Hao Fang
> > > <fanghao11@huawei.com>, Gavin Schenk <g.schenk@eckelmann.de>, RDMA mailing
> > > list <linux-rdma@vger.kernel.org>, Vinod Koul <vkoul@kernel.org>, Jason
> > > Gunthorpe <jgg@ziepe.ca>, Doug Ledford <dledford@redhat.com>, Uwe
> > > Kleine-König <u.kleine-koenig@pengutronix.de>, David Kershner
> > > <david.kershner@unisys.com>, Kenneth Lee <nek.in.cn@gmail.com>, Johan
> > > Hovold <johan@kernel.org>, Cyrille Pitchen
> > > <cyrille.pitchen@free-electrons.com>, Sagar Dharia
> > > <sdharia@codeaurora.org>, Jens Axboe <axboe@kernel.dk>,
> > > guodong.xu@linaro.org, linux-netdev <netdev@vger.kernel.org>, Randy Dunlap
> > > <rdunlap@infradead.org>, linux-kernel@vger.kernel.org, Zhou Wang
> > > <wangzhou1@hisilicon.com>, linux-crypto@vger.kernel.org, Philippe
> > > Ombredanne <pombredanne@nexb.com>, Sanyog Kale <sanyog.r.kale@intel.com>,
> > > "David S. Miller" <davem@davemloft.net>,
> > > linux-accelerators@lists.ozlabs.org
> > > Subject: Re: [RFCv3 PATCH 1/6] uacce: Add documents for WarpDrive/uacce
> > > User-Agent: Mutt/1.5.21 (2010-09-15)
> > > Message-ID: <20181119091405.GE157308@Turing-Arch-b>
> > >
> > > On Thu, Nov 15, 2018 at 04:54:55PM +0200, Leon Romanovsky wrote:
> > > > Date: Thu, 15 Nov 2018 16:54:55 +0200
> > > > From: Leon Romanovsky <leon@kernel.org>
> > > > To: Kenneth Lee <liguozhu@hisilicon.com>
> > > > CC: Kenneth Lee <nek.in.cn@gmail.com>, Tim Sell <timothy.sell@unisys.com>,
> > > > linux-doc@vger.kernel.org, Alexander Shishkin
> > > > <alexander.shishkin@linux.intel.com>, Zaibo Xu <xuzaibo@huawei.com>,
> > > > zhangfei.gao@foxmail.com, linuxarm@huawei.com, haojian.zhuang@linaro.org,
> > > > Christoph Lameter <cl@linux.com>, Hao Fang <fanghao11@huawei.com>, Gavin
> > > > Schenk <g.schenk@eckelmann.de>, RDMA mailing list
> > > > <linux-rdma@vger.kernel.org>, Zhou Wang <wangzhou1@hisilicon.com>, Jason
> > > > Gunthorpe <jgg@ziepe.ca>, Doug Ledford <dledford@redhat.com>, Uwe
> > > > Kleine-König <u.kleine-koenig@pengutronix.de>, David Kershner
> > > > <david.kershner@unisys.com>, Johan Hovold <johan@kernel.org>, Cyrille
> > > > Pitchen <cyrille.pitchen@free-electrons.com>, Sagar Dharia
> > > > <sdharia@codeaurora.org>, Jens Axboe <axboe@kernel.dk>,
> > > > guodong.xu@linaro.org, linux-netdev <netdev@vger.kernel.org>, Randy Dunlap
> > > > <rdunlap@infradead.org>, linux-kernel@vger.kernel.org, Vinod Koul
> > > > <vkoul@kernel.org>, linux-crypto@vger.kernel.org, Philippe Ombredanne
> > > > <pombredanne@nexb.com>, Sanyog Kale <sanyog.r.kale@intel.com>, "David S.
> > > > Miller" <davem@davemloft.net>, linux-accelerators@lists.ozlabs.org
> > > > Subject: Re: [RFCv3 PATCH 1/6] uacce: Add documents for WarpDrive/uacce
> > > > User-Agent: Mutt/1.10.1 (2018-07-13)
> > > > Message-ID: <20181115145455.GN3759@mtr-leonro.mtl.com>
> > > >
> > > > On Thu, Nov 15, 2018 at 04:51:09PM +0800, Kenneth Lee wrote:
> > > > > On Wed, Nov 14, 2018 at 06:00:17PM +0200, Leon Romanovsky wrote:
> > > > > > Date: Wed, 14 Nov 2018 18:00:17 +0200
> > > > > > From: Leon Romanovsky <leon@kernel.org>
> > > > > > To: Kenneth Lee <nek.in.cn@gmail.com>
> > > > > > CC: Tim Sell <timothy.sell@unisys.com>, linux-doc@vger.kernel.org,
> > > > > > Alexander Shishkin <alexander.shishkin@linux.intel.com>, Zaibo Xu
> > > > > > <xuzaibo@huawei.com>, zhangfei.gao@foxmail.com, linuxarm@huawei.com,
> > > > > > haojian.zhuang@linaro.org, Christoph Lameter <cl@linux.com>, Hao Fang
> > > > > > <fanghao11@huawei.com>, Gavin Schenk <g.schenk@eckelmann.de>, RDMA mailing
> > > > > > list <linux-rdma@vger.kernel.org>, Zhou Wang <wangzhou1@hisilicon.com>,
> > > > > > Jason Gunthorpe <jgg@ziepe.ca>, Doug Ledford <dledford@redhat.com>, Uwe
> > > > > > Kleine-König <u.kleine-koenig@pengutronix.de>, David Kershner
> > > > > > <david.kershner@unisys.com>, Johan Hovold <johan@kernel.org>, Cyrille
> > > > > > Pitchen <cyrille.pitchen@free-electrons.com>, Sagar Dharia
> > > > > > <sdharia@codeaurora.org>, Jens Axboe <axboe@kernel.dk>,
> > > > > > guodong.xu@linaro.org, linux-netdev <netdev@vger.kernel.org>, Randy Dunlap
> > > > > > <rdunlap@infradead.org>, linux-kernel@vger.kernel.org, Vinod Koul
> > > > > > <vkoul@kernel.org>, linux-crypto@vger.kernel.org, Philippe Ombredanne
> > > > > > <pombredanne@nexb.com>, Sanyog Kale <sanyog.r.kale@intel.com>, Kenneth Lee
> > > > > > <liguozhu@hisilicon.com>, "David S. Miller" <davem@davemloft.net>,
> > > > > > linux-accelerators@lists.ozlabs.org
> > > > > > Subject: Re: [RFCv3 PATCH 1/6] uacce: Add documents for WarpDrive/uacce
> > > > > > User-Agent: Mutt/1.10.1 (2018-07-13)
> > > > > > Message-ID: <20181114160017.GI3759@mtr-leonro.mtl.com>
> > > > > >
> > > > > > On Wed, Nov 14, 2018 at 10:58:09AM +0800, Kenneth Lee wrote:
> > > > > > >
> > > > > > > 在 2018/11/13 上午8:23, Leon Romanovsky 写道:
> > > > > > > > On Mon, Nov 12, 2018 at 03:58:02PM +0800, Kenneth Lee wrote:
> > > > > > > > > From: Kenneth Lee <liguozhu@hisilicon.com>
> > > > > > > > >
> > > > > > > > > WarpDrive is a general accelerator framework for the user application to
> > > > > > > > > access the hardware without going through the kernel in data path.
> > > > > > > > >
> > > > > > > > > The kernel component to provide kernel facility to driver for expose the
> > > > > > > > > user interface is called uacce. It a short name for
> > > > > > > > > "Unified/User-space-access-intended Accelerator Framework".
> > > > > > > > >
> > > > > > > > > This patch add document to explain how it works.
> > > > > > > > + RDMA and netdev folks
> > > > > > > >
> > > > > > > > Sorry, to be late in the game, I don't see other patches, but from
> > > > > > > > the description below it seems like you are reinventing RDMA verbs
> > > > > > > > model. I have hard time to see the differences in the proposed
> > > > > > > > framework to already implemented in drivers/infiniband/* for the kernel
> > > > > > > > space and for the https://github.com/linux-rdma/rdma-core/ for the user
> > > > > > > > space parts.
> > > > > > >
> > > > > > > Thanks Leon,
> > > > > > >
> > > > > > > Yes, we tried to solve similar problem in RDMA. We also learned a lot from
> > > > > > > the exist code of RDMA. But we we have to make a new one because we cannot
> > > > > > > register accelerators such as AI operation, encryption or compression to the
> > > > > > > RDMA framework:)
> > > > > >
> > > > > > Assuming that you did everything right and still failed to use RDMA
> > > > > > framework, you was supposed to fix it and not to reinvent new exactly
> > > > > > same one. It is how we develop kernel, by reusing existing code.
> > > > >
> > > > > Yes, but we don't force other system such as NIC or GPU into RDMA, do we?
> > > >
> > > > You don't introduce new NIC or GPU, but proposing another interface to
> > > > directly access HW memory and bypass kernel for the data path. This is
> > > > whole idea of RDMA and this is why it is already present in the kernel.
> > > >
> > > > Various hardware devices are supported in our stack allow a ton of crazy
> > > > stuff, including GPUs interconnections and NIC functionalities.
> > >
> > > Yes. We don't want to invent new wheel. That is why we did it behind VFIO in RFC
> > > v1 and v2. But finally we were persuaded by Mr. Jerome Glisse that VFIO was not
> > > a good place to solve the problem.
>
> I saw a couple of his responses, he constantly said to you that you are
> reinventing the wheel.
> https://lore.kernel.org/lkml/20180904150019.GA4024@redhat.com/
>
No. I think he asked me did not create trouble in VFIO but just use common
interface from dma_buf and iommu itself. That is exactly what I am doing.
> > >
> > > And currently, as you see, IB is bound with devices doing RDMA. The register
> > > function, ib_register_device() hint that it is a netdev (get_netdev() callback), it know
> > > about gid, pkey, and Memory Window. IB is not simply a address space management
> > > framework. And verbs to IB are not transparent. If we start to add
> > > compression/decompression, AI (RNN, CNN stuff) operations, and encryption/decryption
> > > to the verbs set. It will become very complexity. Or maybe I misunderstand the
> > > IB idea? But I don't see compression hardware is integrated in the mainline
> > > Kernel. Could you directly point out which one I can used as a reference?
> > >
>
> I strongly advise you to read the code, not all drivers are implementing
> gids, pkeys and get_netdev() callback.
>
> Yes, you are misunderstanding drivers/infiniband subsystem. We have
> plenty options to expose APIs to the user space applications, starting
> from standard verbs API and ending with private objects which are
> understandable by specific device/driver.
>
> IB stack provides secure FD to access device, by creating context,
> after that you can send direct commands to the FW (see mlx5 DEVX
> or hfi1) in sane way.
>
> So actually, you will need to register your device, declare your own
> set of objects (similar to mlx5 include/uapi/rdma/mlx5_user_ioctl_*.h).
>
> In regards to reference of compression hardware, I don't have.
> But there is an example of how T10-DIF can be implemented in verbs
> layer:
> https://www.openfabrics.org/images/2018workshop/presentations/307_TOved_T10-DIFOffload.pdf
> Or IPsec crypto:
> https://www.spinics.net/lists/linux-rdma/msg48906.html
>
OK. I will spend some time on it first. But according to current discussion,
Don't you think I should avoid all these complexities but simply use SVM/SVA on
iommu or let the user application use the kernel-allocated VMA and page? It
does not create anything new. Just a new user of IOMMU and its SVM/SVA
capability.
> > > >
> > > > >
> > > > > I assume you would not agree to register a zip accelerator to infiniband? :)
> > > >
> > > > "infiniband" name in the "drivers/infiniband/" is legacy one and the
> > > > current code supports IB, RoCE, iWARP and OmniPath as a transport layers.
> > > > For a lone time, we wanted to rename that folder to be "drivers/rdma",
> > > > but didn't find enough brave men/women to do it, due to backport mess
> > > > for such move.
> > > >
> > > > The addition of zip accelerator to RDMA is possible and depends on how
> > > > you will model such new functionality - new driver, or maybe new ULP.
> > > >
> > > > >
> > > > > Further, I don't think it is wise to break an exist system (RDMA) to fulfill a
> > > > > totally new scenario. The better choice is to let them run in parallel for some
> > > > > time and try to merge them accordingly.
> > > >
> > > > Awesome, so please run your code out-of-tree for now and once you are ready
> > > > for submission let's try to merge it.
> > >
> > > Yes, yes. We know trust need time to gain. But the fact is that there is no
> > > accelerator user driver can be added to mainline kernel. We should raise the
> > > topic time to time. So to help the communication to fix the gap, right?
> > >
> > > We are also opened to cooperate with IB to do it within the IB framework. But
> > > please let me know where to start. I feel it is quite wired to make a
> > > ib_register_device for a zip or RSA accelerator.
>
> Most of ib_ prefixes in drivers/infinband/ are legacy names. You can
> rename them to be rdma_register_device() if it helps.
>
> So from implementation point of view, as I wrote above.
> Create minimal driver to register, expose MR to user space, add your own
> objects and capabilities through our new KABI and implement user space part
> in github.com/linux-rdma/rdma-core.
I don't think it is just a name. But anyway, let me spend some time to try the
possibility.
>
> > >
> > > >
> > > > >
> > > > > >
> > > > > > >
> > > > > > > Another problem we tried to address is the way to pin the memory for dma
> > > > > > > operation. The RDMA way to pin the memory cannot avoid the page lost due to
> > > > > > > copy-on-write operation during the memory is used by the device. This may
> > > > > > > not be important to RDMA library. But it is important to accelerator.
> > > > > >
> > > > > > Such support exists in drivers/infiniband/ from late 2014 and
> > > > > > it is called ODP (on demand paging).
> > > > >
> > > > > I reviewed ODP and I think it is a solution bound to infiniband. It is part of
> > > > > MR semantics and required a infiniband specific hook
> > > > > (ucontext->invalidate_range()). And the hook requires the device to be able to
> > > > > stop using the page for a while for the copying. It is ok for infiniband
> > > > > (actually, only mlx5 uses it). I don't think most accelerators can support
> > > > > this mode. But WarpDrive works fully on top of IOMMU interface, it has no this
> > > > > limitation.
> > > >
> > > > 1. It has nothing to do with infiniband.
> > >
> > > But it must be a ib_dev first.
>
> It is just a name.
>
> > >
> > > > 2. MR and uncontext are verbs semantics and needed to ensure that host
> > > > memory exposed to user is properly protected from security point of view.
> > > > 3. "stop using the page for a while for the copying" - I'm not fully
> > > > understand this claim, maybe this article will help you to better
> > > > describe : https://lwn.net/Articles/753027/
> > >
> > > This topic was being discussed in RFCv2. The key problem here is that:
> > >
> > > The device need to hold the memory for its own calculation, but the CPU/software
> > > want to stop it for a while for synchronizing with disk or COW.
> > >
> > > If the hardware support SVM/SVA (Shared Virtual Memory/Address), it is easy, the
> > > device share page table with CPU, the device will raise a page fault when the
> > > CPU downgrade the PTE to read-only.
> > >
> > > If the hardware cannot share page table with the CPU, we then need to have
> > > some way to change the device page table. This is what happen in ODP. It
> > > invalidates the page table in device upon mmu_notifier call back. But this cannot
> > > solve the COW problem: if the user process A share a page P with device, and A
> > > forks a new process B, and it continue to write to the page. By COW, the
> > > process B will keep the page P, while A will get a new page P'. But you have
> > > no way to let the device know it should use P' rather than P.
>
> I didn't hear about such issue and we supported fork for a long time.
>
> > >
> > > This may be OK for RDMA application. Because RDMA is a big thing and we can ask
> > > the programmer to avoid the situation. But for a accelerator, I don't think we
> > > can ask a programmer to care for this when use a zlib.
> > >
> > > In WarpDrive/uacce, we make this simple. If you support IOMMU and it support
> > > SVM/SVA. Everything will be fine just like ODP implicit mode. And you don't need
> > > to write any code for that. Because it has been done by IOMMU framework. If it
> > > dose not, you have to use the kernel allocated memory which has the same IOVA as
> > > the VA in user space. So we can still maintain a unify address space among the
> > > devices and the applicatin.
> > >
> > > > 4. mlx5 supports ODP not because of being partially IB device,
> > > > but because HW performance oriented implementation is not an easy task.
> > > >
> > > > >
> > > > > >
> > > > > > >
> > > > > > > Hope this can help the understanding.
> > > > > >
> > > > > > Yes, it helped me a lot.
> > > > > > Now, I'm more than before convinced that this whole patchset shouldn't
> > > > > > exist in the first place.
> > > > >
> > > > > Then maybe you can tell me how I can register my accelerator to the user space?
> > > >
> > > > Write kernel driver and write user space part of it.
> > > > https://github.com/linux-rdma/rdma-core/
> > > >
> > > > I have no doubts that your colleagues who wrote and maintain
> > > > drivers/infiniband/hw/hns driver know best how to do it.
> > > > They did it very successfully.
> > > >
> > > > Thanks
> > > >
> > > > >
> > > > > >
> > > > > > To be clear, NAK.
> > > > > >
> > > > > > Thanks
> > > > > >
> > > > > > >
> > > > > > > Cheers
> > > > > > >
> > > > > > > >
> > > > > > > > Hard NAK from RDMA side.
> > > > > > > >
> > > > > > > > Thanks
> > > > > > > >
> > > > > > > > > Signed-off-by: Kenneth Lee <liguozhu@hisilicon.com>
> > > > > > > > > ---
> > > > > > > > > Documentation/warpdrive/warpdrive.rst | 260 +++++++
> > > > > > > > > Documentation/warpdrive/wd-arch.svg | 764 ++++++++++++++++++++
> > > > > > > > > Documentation/warpdrive/wd.svg | 526 ++++++++++++++
> > > > > > > > > Documentation/warpdrive/wd_q_addr_space.svg | 359 +++++++++
> > > > > > > > > 4 files changed, 1909 insertions(+)
> > > > > > > > > create mode 100644 Documentation/warpdrive/warpdrive.rst
> > > > > > > > > create mode 100644 Documentation/warpdrive/wd-arch.svg
> > > > > > > > > create mode 100644 Documentation/warpdrive/wd.svg
> > > > > > > > > create mode 100644 Documentation/warpdrive/wd_q_addr_space.svg
> > > > > > > > >
> > > > > > > > > diff --git a/Documentation/warpdrive/warpdrive.rst b/Documentation/warpdrive/warpdrive.rst
> > > > > > > > > new file mode 100644
> > > > > > > > > index 000000000000..ef84d3a2d462
> > > > > > > > > --- /dev/null
> > > > > > > > > +++ b/Documentation/warpdrive/warpdrive.rst
> > > > > > > > > @@ -0,0 +1,260 @@
> > > > > > > > > +Introduction of WarpDrive
> > > > > > > > > +=========================
> > > > > > > > > +
> > > > > > > > > +*WarpDrive* is a general accelerator framework for the user application to
> > > > > > > > > +access the hardware without going through the kernel in data path.
> > > > > > > > > +
> > > > > > > > > +It can be used as the quick channel for accelerators, network adaptors or
> > > > > > > > > +other hardware for application in user space.
> > > > > > > > > +
> > > > > > > > > +This may make some implementation simpler. E.g. you can reuse most of the
> > > > > > > > > +*netdev* driver in kernel and just share some ring buffer to the user space
> > > > > > > > > +driver for *DPDK* [4] or *ODP* [5]. Or you can combine the RSA accelerator with
> > > > > > > > > +the *netdev* in the user space as a https reversed proxy, etc.
> > > > > > > > > +
> > > > > > > > > +*WarpDrive* takes the hardware accelerator as a heterogeneous processor which
> > > > > > > > > +can share particular load from the CPU:
> > > > > > > > > +
> > > > > > > > > +.. image:: wd.svg
> > > > > > > > > + :alt: WarpDrive Concept
> > > > > > > > > +
> > > > > > > > > +The virtual concept, queue, is used to manage the requests sent to the
> > > > > > > > > +accelerator. The application send requests to the queue by writing to some
> > > > > > > > > +particular address, while the hardware takes the requests directly from the
> > > > > > > > > +address and send feedback accordingly.
> > > > > > > > > +
> > > > > > > > > +The format of the queue may differ from hardware to hardware. But the
> > > > > > > > > +application need not to make any system call for the communication.
> > > > > > > > > +
> > > > > > > > > +*WarpDrive* tries to create a shared virtual address space for all involved
> > > > > > > > > +accelerators. Within this space, the requests sent to queue can refer to any
> > > > > > > > > +virtual address, which will be valid to the application and all involved
> > > > > > > > > +accelerators.
> > > > > > > > > +
> > > > > > > > > +The name *WarpDrive* is simply a cool and general name meaning the framework
> > > > > > > > > +makes the application faster. It includes general user library, kernel
> > > > > > > > > +management module and drivers for the hardware. In kernel, the management
> > > > > > > > > +module is called *uacce*, meaning "Unified/User-space-access-intended
> > > > > > > > > +Accelerator Framework".
> > > > > > > > > +
> > > > > > > > > +
> > > > > > > > > +How does it work
> > > > > > > > > +================
> > > > > > > > > +
> > > > > > > > > +*WarpDrive* uses *mmap* and *IOMMU* to play the trick.
> > > > > > > > > +
> > > > > > > > > +*Uacce* creates a chrdev for the device registered to it. A "queue" will be
> > > > > > > > > +created when the chrdev is opened. The application access the queue by mmap
> > > > > > > > > +different address region of the queue file.
> > > > > > > > > +
> > > > > > > > > +The following figure demonstrated the queue file address space:
> > > > > > > > > +
> > > > > > > > > +.. image:: wd_q_addr_space.svg
> > > > > > > > > + :alt: WarpDrive Queue Address Space
> > > > > > > > > +
> > > > > > > > > +The first region of the space, device region, is used for the application to
> > > > > > > > > +write request or read answer to or from the hardware.
> > > > > > > > > +
> > > > > > > > > +Normally, there can be three types of device regions mmio and memory regions.
> > > > > > > > > +It is recommended to use common memory for request/answer descriptors and use
> > > > > > > > > +the mmio space for device notification, such as doorbell. But of course, this
> > > > > > > > > +is all up to the interface designer.
> > > > > > > > > +
> > > > > > > > > +There can be two types of device memory regions, kernel-only and user-shared.
> > > > > > > > > +This will be explained in the "kernel APIs" section.
> > > > > > > > > +
> > > > > > > > > +The Static Share Virtual Memory region is necessary only when the device IOMMU
> > > > > > > > > +does not support "Share Virtual Memory". This will be explained after the
> > > > > > > > > +*IOMMU* idea.
> > > > > > > > > +
> > > > > > > > > +
> > > > > > > > > +Architecture
> > > > > > > > > +------------
> > > > > > > > > +
> > > > > > > > > +The full *WarpDrive* architecture is represented in the following class
> > > > > > > > > +diagram:
> > > > > > > > > +
> > > > > > > > > +.. image:: wd-arch.svg
> > > > > > > > > + :alt: WarpDrive Architecture
> > > > > > > > > +
> > > > > > > > > +
> > > > > > > > > +The user API
> > > > > > > > > +------------
> > > > > > > > > +
> > > > > > > > > +We adopt a polling style interface in the user space: ::
> > > > > > > > > +
> > > > > > > > > + int wd_request_queue(struct wd_queue *q);
> > > > > > > > > + void wd_release_queue(struct wd_queue *q);
> > > > > > > > > +
> > > > > > > > > + int wd_send(struct wd_queue *q, void *req);
> > > > > > > > > + int wd_recv(struct wd_queue *q, void **req);
> > > > > > > > > + int wd_recv_sync(struct wd_queue *q, void **req);
> > > > > > > > > + void wd_flush(struct wd_queue *q);
> > > > > > > > > +
> > > > > > > > > +wd_recv_sync() is a wrapper to its non-sync version. It will trapped into
> > > > > > > > > +kernel and waits until the queue become available.
> > > > > > > > > +
> > > > > > > > > +If the queue do not support SVA/SVM. The following helper function
> > > > > > > > > +can be used to create Static Virtual Share Memory: ::
> > > > > > > > > +
> > > > > > > > > + void *wd_preserve_share_memory(struct wd_queue *q, size_t size);
> > > > > > > > > +
> > > > > > > > > +The user API is not mandatory. It is simply a suggestion and hint what the
> > > > > > > > > +kernel interface is supposed to support.
> > > > > > > > > +
> > > > > > > > > +
> > > > > > > > > +The user driver
> > > > > > > > > +---------------
> > > > > > > > > +
> > > > > > > > > +The queue file mmap space will need a user driver to wrap the communication
> > > > > > > > > +protocol. *UACCE* provides some attributes in sysfs for the user driver to
> > > > > > > > > +match the right accelerator accordingly.
> > > > > > > > > +
> > > > > > > > > +The *UACCE* device attribute is under the following directory:
> > > > > > > > > +
> > > > > > > > > +/sys/class/uacce/<dev-name>/params
> > > > > > > > > +
> > > > > > > > > +The following attributes is supported:
> > > > > > > > > +
> > > > > > > > > +nr_queue_remained (ro)
> > > > > > > > > + number of queue remained
> > > > > > > > > +
> > > > > > > > > +api_version (ro)
> > > > > > > > > + a string to identify the queue mmap space format and its version
> > > > > > > > > +
> > > > > > > > > +device_attr (ro)
> > > > > > > > > + attributes of the device, see UACCE_DEV_xxx flag defined in uacce.h
> > > > > > > > > +
> > > > > > > > > +numa_node (ro)
> > > > > > > > > + id of numa node
> > > > > > > > > +
> > > > > > > > > +priority (rw)
> > > > > > > > > + Priority or the device, bigger is higher
> > > > > > > > > +
> > > > > > > > > +(This is not yet implemented in RFC version)
> > > > > > > > > +
> > > > > > > > > +
> > > > > > > > > +The kernel API
> > > > > > > > > +--------------
> > > > > > > > > +
> > > > > > > > > +The *uacce* kernel API is defined in uacce.h. If the hardware support SVM/SVA,
> > > > > > > > > +The driver need only the following API functions: ::
> > > > > > > > > +
> > > > > > > > > + int uacce_register(uacce);
> > > > > > > > > + void uacce_unregister(uacce);
> > > > > > > > > + void uacce_wake_up(q);
> > > > > > > > > +
> > > > > > > > > +*uacce_wake_up* is used to notify the process who epoll() on the queue file.
> > > > > > > > > +
> > > > > > > > > +According to the IOMMU capability, *uacce* categories the devices as follow:
> > > > > > > > > +
> > > > > > > > > +UACCE_DEV_NOIOMMU
> > > > > > > > > + The device has no IOMMU. The user process cannot use VA on the hardware
> > > > > > > > > + This mode is not recommended.
> > > > > > > > > +
> > > > > > > > > +UACCE_DEV_SVA (UACCE_DEV_PASID | UACCE_DEV_FAULT_FROM_DEV)
> > > > > > > > > + The device has IOMMU which can share the same page table with user
> > > > > > > > > + process
> > > > > > > > > +
> > > > > > > > > +UACCE_DEV_SHARE_DOMAIN
> > > > > > > > > + The device has IOMMU which has no multiple page table and device page
> > > > > > > > > + fault support
> > > > > > > > > +
> > > > > > > > > +If the device works in mode other than UACCE_DEV_NOIOMMU, *uacce* will set its
> > > > > > > > > +IOMMU to IOMMU_DOMAIN_UNMANAGED. So the driver must not use any kernel
> > > > > > > > > +DMA API but the following ones from *uacce* instead: ::
> > > > > > > > > +
> > > > > > > > > + uacce_dma_map(q, va, size, prot);
> > > > > > > > > + uacce_dma_unmap(q, va, size, prot);
> > > > > > > > > +
> > > > > > > > > +*uacce_dma_map/unmap* is valid only for UACCE_DEV_SVA device. It creates a
> > > > > > > > > +particular PASID and page table for the kernel in the IOMMU (Not yet
> > > > > > > > > +implemented in the RFC)
> > > > > > > > > +
> > > > > > > > > +For the UACCE_DEV_SHARE_DOMAIN device, uacce_dma_map/unmap is not valid.
> > > > > > > > > +*Uacce* call back start_queue only when the DUS and DKO region is mmapped. The
> > > > > > > > > +accelerator driver must use those dma buffer, via uacce_queue->qfrs[], on
> > > > > > > > > +start_queue call back. The size of the queue file region is defined by
> > > > > > > > > +uacce->ops->qf_pg_start[].
> > > > > > > > > +
> > > > > > > > > +We have to do it this way because most of current IOMMU cannot support the
> > > > > > > > > +kernel and user virtual address at the same time. So we have to let them both
> > > > > > > > > +share the same user virtual address space.
> > > > > > > > > +
> > > > > > > > > +If the device have to support kernel and user at the same time, both kernel
> > > > > > > > > +and the user should use these DMA API. This is not convenient. A better
> > > > > > > > > +solution is to change the future DMA/IOMMU design to let them separate the
> > > > > > > > > +address space between the user and kernel space. But it is not going to be in
> > > > > > > > > +a short time.
> > > > > > > > > +
> > > > > > > > > +
> > > > > > > > > +Multiple processes support
> > > > > > > > > +==========================
> > > > > > > > > +
> > > > > > > > > +In the latest mainline kernel (4.19) when this document is written, the IOMMU
> > > > > > > > > +subsystem do not support multiple process page tables yet.
> > > > > > > > > +
> > > > > > > > > +Most IOMMU hardware implementation support multi-process with the concept
> > > > > > > > > +of PASID. But they may use different name, e.g. it is call sub-stream-id in
> > > > > > > > > +SMMU of ARM. With PASID or similar design, multi page table can be added to
> > > > > > > > > +the IOMMU and referred by its PASID.
> > > > > > > > > +
> > > > > > > > > +*JPB* has a patchset to enable this[1]_. We have tested it with our hardware
> > > > > > > > > +(which is known as *D06*). It works well. *WarpDrive* rely on them to support
> > > > > > > > > +UACCE_DEV_SVA. If it is not enabled, *WarpDrive* can still work. But it
> > > > > > > > > +support only one process, the device will be set to UACCE_DEV_SHARE_DOMAIN
> > > > > > > > > +even it is set to UACCE_DEV_SVA initially.
> > > > > > > > > +
> > > > > > > > > +Static Share Virtual Memory is mainly used by UACCE_DEV_SHARE_DOMAIN device.
> > > > > > > > > +
> > > > > > > > > +
> > > > > > > > > +Legacy Mode Support
> > > > > > > > > +===================
> > > > > > > > > +For the hardware without IOMMU, WarpDrive can still work, the only problem is
> > > > > > > > > +VA cannot be used in the device. The driver should adopt another strategy for
> > > > > > > > > +the shared memory. It is only for testing, and not recommended.
> > > > > > > > > +
> > > > > > > > > +
> > > > > > > > > +The Folk Scenario
> > > > > > > > > +=================
> > > > > > > > > +For a process with allocated queues and shared memory, what happen if it forks
> > > > > > > > > +a child?
> > > > > > > > > +
> > > > > > > > > +The fd of the queue will be duplicated on folk, so the child can send request
> > > > > > > > > +to the same queue as its parent. But the requests which is sent from processes
> > > > > > > > > +except for the one who open the queue will be blocked.
> > > > > > > > > +
> > > > > > > > > +It is recommended to add O_CLOEXEC to the queue file.
> > > > > > > > > +
> > > > > > > > > +The queue mmap space has a VM_DONTCOPY in its VMA. So the child will lost all
> > > > > > > > > +those VMAs.
> > > > > > > > > +
> > > > > > > > > +This is why *WarpDrive* does not adopt the mode used in *VFIO* and *InfiniBand*.
> > > > > > > > > +Both solutions can set any user pointer for hardware sharing. But they cannot
> > > > > > > > > +support fork when the dma is in process. Or the "Copy-On-Write" procedure will
> > > > > > > > > +make the parent process lost its physical pages.
> > > > > > > > > +
> > > > > > > > > +
> > > > > > > > > +The Sample Code
> > > > > > > > > +===============
> > > > > > > > > +There is a sample user land implementation with a simple driver for Hisilicon
> > > > > > > > > +Hi1620 ZIP Accelerator.
> > > > > > > > > +
> > > > > > > > > +To test, do the following in samples/warpdrive (for the case of PC host): ::
> > > > > > > > > + ./autogen.sh
> > > > > > > > > + ./conf.sh # or simply ./configure if you build on target system
> > > > > > > > > + make
> > > > > > > > > +
> > > > > > > > > +Then you can get test_hisi_zip in the test subdirectory. Copy it to the target
> > > > > > > > > +system and make sure the hisi_zip driver is enabled (the major and minor of
> > > > > > > > > +the uacce chrdev can be gotten from the dmesg or sysfs), and run: ::
> > > > > > > > > + mknod /dev/ua1 c <major> <minior>
> > > > > > > > > + test/test_hisi_zip -z < data > data.zip
> > > > > > > > > + test/test_hisi_zip -g < data > data.gzip
> > > > > > > > > +
> > > > > > > > > +
> > > > > > > > > +References
> > > > > > > > > +==========
> > > > > > > > > +.. [1] https://patchwork.kernel.org/patch/10394851/
> > > > > > > > > +
> > > > > > > > > +.. vim: tw=78
> > > > > [...]
> > > > > > > > > --
> > > > > > > > > 2.17.1
> > > > > > > > >
> >
> > I don't know if Mr. Jerome Glisse in the list. I think I should cc him for my
> > respectation to his help on last RFC.
> >
> > - Kenneth
^ permalink raw reply
* Re: [PATCH net 0/2] qed: Fix Queue Manager getters
From: David Miller @ 2018-11-20 2:38 UTC (permalink / raw)
To: denis.bolotin; +Cc: netdev, ariel.elior
In-Reply-To: <20181119142831.8035-1-denis.bolotin@cavium.com>
From: Denis Bolotin <denis.bolotin@cavium.com>
Date: Mon, 19 Nov 2018 16:28:29 +0200
> This patch series fixes various queue manager getter functions. It is
> important to make sure the getter's caller will receive a valid queue even
> in error case to prevent more serious bugs.
> Please consider applying to net.
Series applied, thanks.
^ permalink raw reply
* Re: [PATCH net-next 00/14] gred: add offload support
From: David Miller @ 2018-11-20 2:54 UTC (permalink / raw)
To: jakub.kicinski; +Cc: oss-drivers, netdev
In-Reply-To: <20181119232150.13038-1-jakub.kicinski@netronome.com>
From: Jakub Kicinski <jakub.kicinski@netronome.com>
Date: Mon, 19 Nov 2018 15:21:36 -0800
> This series adds support for GRED offload in the nfp driver. So
> far we have only supported the RED Qdisc offload, but we need a
> way to differentiate traffic types e.g. based on DSCP marking.
>
> It may seem like PRIO+RED is a good match for this job, however,
> (a) we don't need strict priority behaviour of PRIO, and (b) PRIO
> uses the legacy way of mapping ToS fields to bands, which is quite
> awkward and limitting.
>
> The less commonly used GRED Qdisc is a better much for the scenario,
> it allows multiple sets of RED parameters and queue lengths to be
> maintained with a single FIFO queue. This is exactly how nfp offload
> behaves. We use a trivial u32 classifier to assign packets to virtual
> queues.
>
> There is also the minor advantage that GRED can't have its child
> changed, therefore limitting ways in which the configuration of SW
> path can diverge from HW offload.
>
> Last patch of the series adds support for (G)RED in non-ECN mode,
> where packets are dropped instead of marked.
Series applied, thanks Jakub.
^ permalink raw reply
* Re: [PATCH net-next v3 1/6] net: phy: mscc: migrate to phy_select/restore_page functions
From: Quentin Schulz @ 2018-11-20 13:20 UTC (permalink / raw)
To: Andreas Schwab
Cc: Andrew Lunn, Alexandre Belloni, davem, f.fainelli, allan.nielsen,
linux-kernel, netdev, thomas.petazzoni, linux-riscv
In-Reply-To: <mvmefbgm0zc.fsf@suse.de>
[-- Attachment #1: Type: text/plain, Size: 1649 bytes --]
Hi Andreas,
On Tue, Nov 20, 2018 at 12:39:51PM +0100, Andreas Schwab wrote:
> On Nov 19 2018, Andrew Lunn <andrew@lunn.ch> wrote:
>
> > I still don't see why that would cause a hang.
>
> [ 996.370000] [<ffffffe0011d7324>] mutex_lock+0x22/0x2a
> [ 996.380000] [<ffffffe001039ba0>] mdiobus_read+0x36/0x60
> [ 996.380000] [<ffffffe00103b82c>] vsc85xx_config_init+0x4c/0x1e2
>
> vsc85xx_config_init is calling mdiobus_read while holding the mdio_lock.
>
Thanks for reporting the bug. However, it could be nice the next time
you report a bug to give as much info as you can, such as the device
you're using. Architecture of your platform (or the board itself if it's
a DT in the Linux kernel), the MAC and the PHY IPs (the Microsemi PHY
driver now supports 6 different PHYs that do not work the same way so I
can't guess the one you're using), they all matter and help to
understand your bug. Also, if you could give a full bootlog of the
kernel and how it's possible to reproduce your issue (.config of your
kernel, on which commit on which branch you built your kernel).
Otherwise, you might not have any answer or considerably slow down the
bug identifying and fixing processes, which is annoying to you,
the users, the developers and the maintainers: a lot of upset people :)
Could you try something please? I'm pretty sure I found an issue (I
don't know if it's fixing your issue or not, but it might be).
Please replace in vsc85xx_default_config() the phy_read and phy_write by
respectively __phy_read and __phy_write (note the __ in front).
Let me know if you need help with this.
Thanks,
Quentin
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply
* Re: [PATCH net] net/ibmnvic: Fix deadlock problem in reset
From: David Miller @ 2018-11-20 2:56 UTC (permalink / raw)
To: julietk; +Cc: netdev, linuxppc-dev, tlfalcon, mwb, tyreld
In-Reply-To: <20181119215727.22197.97260.stgit@ltcalpine2-lp22.aus.stglabs.ibm.com>
From: Juliet Kim <julietk@linux.vnet.ibm.com>
Date: Mon, 19 Nov 2018 15:59:22 -0600
> This patch changes to use rtnl_lock only during a reset to avoid
> deadlock that could occur when a thread operating close is holding
> rtnl_lock and waiting for reset_lock acquired by another thread,
> which is waiting for rtnl_lock in order to set the number of tx/rx
> queues during a reset.
>
> Also, we now setting the number of tx/rx queues during a soft reset
> for failover or LPM events.
>
> Signed-off-by: Juliet Kim <julietk@linux.vnet.ibm.com>
Applied.
^ permalink raw reply
* [PATCH v2 02/14] datagram: open-code copy_page_to_iter
From: Sagi Grimberg @ 2018-11-20 3:00 UTC (permalink / raw)
To: linux-nvme
Cc: linux-block, netdev, David S. Miller, Keith Busch,
Christoph Hellwig
In-Reply-To: <20181120030019.31738-1-sagi@grimberg.me>
This will be useful to consolidate skb_copy_and_hash_datagram_iter and
skb_copy_and_csum_datagram to a single code path.
Signed-off-by: Sagi Grimberg <sagi@grimberg.me>
---
net/core/datagram.c | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/net/core/datagram.c b/net/core/datagram.c
index 57f3a6fcfc1e..abe642181b64 100644
--- a/net/core/datagram.c
+++ b/net/core/datagram.c
@@ -445,11 +445,14 @@ int skb_copy_datagram_iter(const struct sk_buff *skb, int offset,
end = start + skb_frag_size(frag);
if ((copy = end - offset) > 0) {
+ struct page *page = skb_frag_page(frag);
+ u8 *vaddr = kmap(page);
+
if (copy > len)
copy = len;
- n = copy_page_to_iter(skb_frag_page(frag),
- frag->page_offset + offset -
- start, copy, to);
+ n = copy_to_iter(vaddr + frag->page_offset +
+ offset - start, copy, to);
+ kunmap(page);
offset += n;
if (n != copy)
goto short_copy;
--
2.17.1
^ permalink raw reply related
* [PATCH v2 00/14] TCP transport binding for NVMe over Fabrics
From: Sagi Grimberg @ 2018-11-20 3:00 UTC (permalink / raw)
To: linux-nvme
Cc: linux-block, netdev, David S. Miller, Keith Busch,
Christoph Hellwig
Changes from v1:
- unified skb_copy_datagram_iter and skb_copy_and_csum_datagram (and the
new skb_hash_and_copy_datagram_iter) to a single code path
- removed nvmet modparam budgets (made them a define set to their default
values)
- fixed nvme-tcp host chained r2t transfers reported off-list
- made .install_queue callout return nvme status code
- Added some review tags
- rebased on top of nvme-4.21 branch (nvme tree) + sqflow disable patches
This patch set implements the NVMe over Fabrics TCP host and the target
drivers. Now NVMe over Fabrics can run on every Ethernet port in the world.
The implementation conforms to NVMe over Fabrics 1.1 specification (which
will include already publicly available NVMe/TCP transport binding, TP 8000).
The host driver hooks into the NVMe host stack and implements the TCP
transport binding for NVMe over Fabrics. The NVMe over Fabrics TCP host
driver is responsible for establishing a NVMe/TCP connection, TCP event
and error handling and data-plane messaging and stream processing.
The target driver hooks into the NVMe target core stack and implements
the TCP transport binding. The NVMe over Fabrics target driver is
responsible for accepting and establishing NVMe/TCP connections, TCP
event and error handling, and data-plane messaging and stream processing.
The implementation of both the host and target are fairly simple and
straight-forward. Every NVMe queue is backed by a TCP socket that provides
us reliable, in-order delivery of fabrics capsules and/or data.
All NVMe queues are sharded over a private bound workqueue such that we
always have a single context handling the byte stream and we don't need
to worry about any locking/serialization. In addition, close attention
was paid to a completely non-blocking data plane to minimize context
switching and/or unforced scheduling.
I piggybacked nvme-cli patches to the set for completeness.
Also, @netdev mailing list is cc'd as this patch set contains generic
helpers for online digest calculation (patches 1-3).
The patchset structure:
- patches 1-6 are prep to add a helper for digest calculation online with data placement
- patches 7-11 are preparatory patches for NVMe/TCP
- patches 12-14 implements NVMe/TCP
- patches 15-17 are nvme-cli additions for NVMe/TCP
Thanks to the members of the Fabrics Linux Driver team that helped development,
testing and benchmarking this work.
Gitweb code is available at:
git://git.infradead.org/nvme.git nvme-tcp
Sagi Grimberg (14):
ath6kl: add ath6kl_ prefix to crypto_type
datagram: open-code copy_page_to_iter
iov_iter: pass void csum pointer to csum_and_copy_to_iter
net/datagram: consolidate datagram copy to iter helpers
iov_iter: introduce hash_and_copy_to_iter helper
datagram: introduce skb_copy_and_hash_datagram_iter helper
nvme-core: add work elements to struct nvme_ctrl
nvmet: Add install_queue callout
nvmet: allow configfs tcp trtype configuration
nvme-fabrics: allow user passing header digest
nvme-fabrics: allow user passing data digest
nvme-tcp: Add protocol header
nvmet-tcp: add NVMe over TCP target driver
nvme-tcp: add NVMe over TCP host driver
drivers/net/wireless/ath/ath6kl/cfg80211.c | 2 +-
drivers/net/wireless/ath/ath6kl/common.h | 2 +-
drivers/net/wireless/ath/ath6kl/wmi.c | 6 +-
drivers/net/wireless/ath/ath6kl/wmi.h | 6 +-
drivers/nvme/host/Kconfig | 15 +
drivers/nvme/host/Makefile | 3 +
drivers/nvme/host/fabrics.c | 10 +
drivers/nvme/host/fabrics.h | 4 +
drivers/nvme/host/fc.c | 18 +-
drivers/nvme/host/nvme.h | 2 +
drivers/nvme/host/rdma.c | 19 +-
drivers/nvme/host/tcp.c | 2306 ++++++++++++++++++++
drivers/nvme/target/Kconfig | 10 +
drivers/nvme/target/Makefile | 2 +
drivers/nvme/target/configfs.c | 1 +
drivers/nvme/target/fabrics-cmd.c | 9 +
drivers/nvme/target/nvmet.h | 1 +
drivers/nvme/target/tcp.c | 1741 +++++++++++++++
include/linux/nvme-tcp.h | 189 ++
include/linux/nvme.h | 1 +
include/linux/skbuff.h | 3 +
include/linux/uio.h | 5 +-
lib/iov_iter.c | 19 +-
net/core/datagram.c | 158 +-
24 files changed, 4406 insertions(+), 126 deletions(-)
create mode 100644 drivers/nvme/host/tcp.c
create mode 100644 drivers/nvme/target/tcp.c
create mode 100644 include/linux/nvme-tcp.h
--
2.17.1
^ permalink raw reply
* [PATCH v2 03/14] iov_iter: pass void csum pointer to csum_and_copy_to_iter
From: Sagi Grimberg @ 2018-11-20 3:00 UTC (permalink / raw)
To: linux-nvme
Cc: linux-block, netdev, David S. Miller, Keith Busch,
Christoph Hellwig
In-Reply-To: <20181120030019.31738-1-sagi@grimberg.me>
The single caller to csum_and_copy_to_iter is skb_copy_and_csum_datagram
and we are trying to unite its logic with skb_copy_datagram_iter by passing
a callback to the copy function that we want to apply. Thus, we need
to make the checksum pointer private to the function.
Signed-off-by: Sagi Grimberg <sagi@grimberg.me>
---
include/linux/uio.h | 2 +-
lib/iov_iter.c | 3 ++-
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/include/linux/uio.h b/include/linux/uio.h
index 55ce99ddb912..41d1f8d3313d 100644
--- a/include/linux/uio.h
+++ b/include/linux/uio.h
@@ -266,7 +266,7 @@ static inline void iov_iter_reexpand(struct iov_iter *i, size_t count)
{
i->count = count;
}
-size_t csum_and_copy_to_iter(const void *addr, size_t bytes, __wsum *csum, struct iov_iter *i);
+size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *csump, struct iov_iter *i);
size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum, struct iov_iter *i);
bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum, struct iov_iter *i);
diff --git a/lib/iov_iter.c b/lib/iov_iter.c
index 7ebccb5c1637..db93531ca3e3 100644
--- a/lib/iov_iter.c
+++ b/lib/iov_iter.c
@@ -1432,10 +1432,11 @@ bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum,
}
EXPORT_SYMBOL(csum_and_copy_from_iter_full);
-size_t csum_and_copy_to_iter(const void *addr, size_t bytes, __wsum *csum,
+size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *csump,
struct iov_iter *i)
{
const char *from = addr;
+ __wsum *csum = csump;
__wsum sum, next;
size_t off = 0;
sum = *csum;
--
2.17.1
^ permalink raw reply related
* [PATCH v2 01/14] ath6kl: add ath6kl_ prefix to crypto_type
From: Sagi Grimberg @ 2018-11-20 3:00 UTC (permalink / raw)
To: linux-nvme
Cc: linux-block, netdev, David S. Miller, Keith Busch,
Christoph Hellwig
In-Reply-To: <20181120030019.31738-1-sagi@grimberg.me>
From: Sagi Grimberg <sagi@lightbitslabs.com>
Prevent a namespace conflict as in following patches as skbuff.h will
include the crypto API.
Cc: Kalle Valo <kvalo@codeaurora.org>
Signed-off-by: Sagi Grimberg <sagi@lightbitslabs.com>
---
drivers/net/wireless/ath/ath6kl/cfg80211.c | 2 +-
drivers/net/wireless/ath/ath6kl/common.h | 2 +-
drivers/net/wireless/ath/ath6kl/wmi.c | 6 +++---
drivers/net/wireless/ath/ath6kl/wmi.h | 6 +++---
4 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/drivers/net/wireless/ath/ath6kl/cfg80211.c b/drivers/net/wireless/ath/ath6kl/cfg80211.c
index e121187f371f..fa049c4ae315 100644
--- a/drivers/net/wireless/ath/ath6kl/cfg80211.c
+++ b/drivers/net/wireless/ath/ath6kl/cfg80211.c
@@ -1322,7 +1322,7 @@ static int ath6kl_cfg80211_set_default_key(struct wiphy *wiphy,
struct ath6kl_vif *vif = netdev_priv(ndev);
struct ath6kl_key *key = NULL;
u8 key_usage;
- enum crypto_type key_type = NONE_CRYPT;
+ enum ath6kl_crypto_type key_type = NONE_CRYPT;
ath6kl_dbg(ATH6KL_DBG_WLAN_CFG, "%s: index %d\n", __func__, key_index);
diff --git a/drivers/net/wireless/ath/ath6kl/common.h b/drivers/net/wireless/ath/ath6kl/common.h
index 4f82e8632d37..d6e5234f67a1 100644
--- a/drivers/net/wireless/ath/ath6kl/common.h
+++ b/drivers/net/wireless/ath/ath6kl/common.h
@@ -67,7 +67,7 @@ struct ath6kl_llc_snap_hdr {
__be16 eth_type;
} __packed;
-enum crypto_type {
+enum ath6kl_crypto_type {
NONE_CRYPT = 0x01,
WEP_CRYPT = 0x02,
TKIP_CRYPT = 0x04,
diff --git a/drivers/net/wireless/ath/ath6kl/wmi.c b/drivers/net/wireless/ath/ath6kl/wmi.c
index 777acc564ac9..9d7ac1ab2d02 100644
--- a/drivers/net/wireless/ath/ath6kl/wmi.c
+++ b/drivers/net/wireless/ath/ath6kl/wmi.c
@@ -1849,9 +1849,9 @@ int ath6kl_wmi_connect_cmd(struct wmi *wmi, u8 if_idx,
enum network_type nw_type,
enum dot11_auth_mode dot11_auth_mode,
enum auth_mode auth_mode,
- enum crypto_type pairwise_crypto,
+ enum ath6kl_crypto_type pairwise_crypto,
u8 pairwise_crypto_len,
- enum crypto_type group_crypto,
+ enum ath6kl_crypto_type group_crypto,
u8 group_crypto_len, int ssid_len, u8 *ssid,
u8 *bssid, u16 channel, u32 ctrl_flags,
u8 nw_subtype)
@@ -2301,7 +2301,7 @@ int ath6kl_wmi_disctimeout_cmd(struct wmi *wmi, u8 if_idx, u8 timeout)
}
int ath6kl_wmi_addkey_cmd(struct wmi *wmi, u8 if_idx, u8 key_index,
- enum crypto_type key_type,
+ enum ath6kl_crypto_type key_type,
u8 key_usage, u8 key_len,
u8 *key_rsc, unsigned int key_rsc_len,
u8 *key_material,
diff --git a/drivers/net/wireless/ath/ath6kl/wmi.h b/drivers/net/wireless/ath/ath6kl/wmi.h
index a60bb49fe920..784940ba4c90 100644
--- a/drivers/net/wireless/ath/ath6kl/wmi.h
+++ b/drivers/net/wireless/ath/ath6kl/wmi.h
@@ -2556,9 +2556,9 @@ int ath6kl_wmi_connect_cmd(struct wmi *wmi, u8 if_idx,
enum network_type nw_type,
enum dot11_auth_mode dot11_auth_mode,
enum auth_mode auth_mode,
- enum crypto_type pairwise_crypto,
+ enum ath6kl_crypto_type pairwise_crypto,
u8 pairwise_crypto_len,
- enum crypto_type group_crypto,
+ enum ath6kl_crypto_type group_crypto,
u8 group_crypto_len, int ssid_len, u8 *ssid,
u8 *bssid, u16 channel, u32 ctrl_flags,
u8 nw_subtype);
@@ -2610,7 +2610,7 @@ int ath6kl_wmi_config_debug_module_cmd(struct wmi *wmi, u32 valid, u32 config);
int ath6kl_wmi_get_stats_cmd(struct wmi *wmi, u8 if_idx);
int ath6kl_wmi_addkey_cmd(struct wmi *wmi, u8 if_idx, u8 key_index,
- enum crypto_type key_type,
+ enum ath6kl_crypto_type key_type,
u8 key_usage, u8 key_len,
u8 *key_rsc, unsigned int key_rsc_len,
u8 *key_material,
--
2.17.1
^ permalink raw reply related
* [PATCH v2 04/14] net/datagram: consolidate datagram copy to iter helpers
From: Sagi Grimberg @ 2018-11-20 3:00 UTC (permalink / raw)
To: linux-nvme
Cc: linux-block, netdev, David S. Miller, Keith Busch,
Christoph Hellwig
In-Reply-To: <20181120030019.31738-1-sagi@grimberg.me>
skb_copy_datagram_iter and skb_copy_and_csum_datagram are essentialy
the same but with a couple of differences: The first is the copy
operation used which either a simple copy or a csum_and_copy, and the
second are the behavior on the "short copy" path where simply copy
needs to return the number of bytes successfully copied while csum_and_copy
needs to fault immediately as the checksum is partial.
Introduce __skb_datagram_iter that additionally accepts:
1. copy operation function pointer
2. private data that goes with the copy operation
3. fault_short flag to indicate the action on short copy
Suggested-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Sagi Grimberg <sagi@grimberg.me>
---
net/core/datagram.c | 136 ++++++++++++++------------------------------
1 file changed, 42 insertions(+), 94 deletions(-)
diff --git a/net/core/datagram.c b/net/core/datagram.c
index abe642181b64..382543302ae5 100644
--- a/net/core/datagram.c
+++ b/net/core/datagram.c
@@ -408,27 +408,20 @@ int skb_kill_datagram(struct sock *sk, struct sk_buff *skb, unsigned int flags)
}
EXPORT_SYMBOL(skb_kill_datagram);
-/**
- * skb_copy_datagram_iter - Copy a datagram to an iovec iterator.
- * @skb: buffer to copy
- * @offset: offset in the buffer to start copying from
- * @to: iovec iterator to copy to
- * @len: amount of data to copy from buffer to iovec
- */
-int skb_copy_datagram_iter(const struct sk_buff *skb, int offset,
- struct iov_iter *to, int len)
+int __skb_datagram_iter(const struct sk_buff *skb, int offset,
+ struct iov_iter *to, int len, bool fault_short,
+ size_t (*cb)(const void *, size_t, void *, struct iov_iter *),
+ void *data)
{
int start = skb_headlen(skb);
int i, copy = start - offset, start_off = offset, n;
struct sk_buff *frag_iter;
- trace_skb_copy_datagram_iovec(skb, len);
-
/* Copy header. */
if (copy > 0) {
if (copy > len)
copy = len;
- n = copy_to_iter(skb->data + offset, copy, to);
+ n = cb(skb->data + offset, copy, data, to);
offset += n;
if (n != copy)
goto short_copy;
@@ -450,8 +443,8 @@ int skb_copy_datagram_iter(const struct sk_buff *skb, int offset,
if (copy > len)
copy = len;
- n = copy_to_iter(vaddr + frag->page_offset +
- offset - start, copy, to);
+ n = cb(vaddr + frag->page_offset +
+ offset - start, copy, data, to);
kunmap(page);
offset += n;
if (n != copy)
@@ -471,8 +464,8 @@ int skb_copy_datagram_iter(const struct sk_buff *skb, int offset,
if ((copy = end - offset) > 0) {
if (copy > len)
copy = len;
- if (skb_copy_datagram_iter(frag_iter, offset - start,
- to, copy))
+ if (__skb_datagram_iter(frag_iter, offset - start,
+ to, copy, short_copy, cb, data))
goto fault;
if ((len -= copy) == 0)
return 0;
@@ -493,11 +486,32 @@ int skb_copy_datagram_iter(const struct sk_buff *skb, int offset,
return -EFAULT;
short_copy:
- if (iov_iter_count(to))
+ if (fault_short || iov_iter_count(to))
goto fault;
return 0;
}
+
+static size_t simple_copy_to_iter(const void *addr, size_t bytes,
+ void *data __always_unused, struct iov_iter *i)
+{
+ return copy_to_iter(addr, bytes, i);
+}
+
+/**
+ * skb_copy_datagram_iter - Copy a datagram to an iovec iterator.
+ * @skb: buffer to copy
+ * @offset: offset in the buffer to start copying from
+ * @to: iovec iterator to copy to
+ * @len: amount of data to copy from buffer to iovec
+ */
+int skb_copy_datagram_iter(const struct sk_buff *skb, int offset,
+ struct iov_iter *to, int len)
+{
+ trace_skb_copy_datagram_iovec(skb, len);
+ return __skb_datagram_iter(skb, offset, to, len, false,
+ simple_copy_to_iter, NULL);
+}
EXPORT_SYMBOL(skb_copy_datagram_iter);
/**
@@ -648,87 +662,21 @@ int zerocopy_sg_from_iter(struct sk_buff *skb, struct iov_iter *from)
}
EXPORT_SYMBOL(zerocopy_sg_from_iter);
+/**
+ * skb_copy_and_csum_datagram_iter - Copy datagram to an iovec iterator
+ * and update a checksum.
+ * @skb: buffer to copy
+ * @offset: offset in the buffer to start copying from
+ * @to: iovec iterator to copy to
+ * @len: amount of data to copy from buffer to iovec
+ * @csump: checksum pointer
+ */
static int skb_copy_and_csum_datagram(const struct sk_buff *skb, int offset,
struct iov_iter *to, int len,
__wsum *csump)
{
- int start = skb_headlen(skb);
- int i, copy = start - offset, start_off = offset;
- struct sk_buff *frag_iter;
- int pos = 0;
- int n;
-
- /* Copy header. */
- if (copy > 0) {
- if (copy > len)
- copy = len;
- n = csum_and_copy_to_iter(skb->data + offset, copy, csump, to);
- offset += n;
- if (n != copy)
- goto fault;
- if ((len -= copy) == 0)
- return 0;
- pos = copy;
- }
-
- for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
- int end;
- const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
-
- WARN_ON(start > offset + len);
-
- end = start + skb_frag_size(frag);
- if ((copy = end - offset) > 0) {
- __wsum csum2 = 0;
- struct page *page = skb_frag_page(frag);
- u8 *vaddr = kmap(page);
-
- if (copy > len)
- copy = len;
- n = csum_and_copy_to_iter(vaddr + frag->page_offset +
- offset - start, copy,
- &csum2, to);
- kunmap(page);
- offset += n;
- if (n != copy)
- goto fault;
- *csump = csum_block_add(*csump, csum2, pos);
- if (!(len -= copy))
- return 0;
- pos += copy;
- }
- start = end;
- }
-
- skb_walk_frags(skb, frag_iter) {
- int end;
-
- WARN_ON(start > offset + len);
-
- end = start + frag_iter->len;
- if ((copy = end - offset) > 0) {
- __wsum csum2 = 0;
- if (copy > len)
- copy = len;
- if (skb_copy_and_csum_datagram(frag_iter,
- offset - start,
- to, copy,
- &csum2))
- goto fault;
- *csump = csum_block_add(*csump, csum2, pos);
- if ((len -= copy) == 0)
- return 0;
- offset += copy;
- pos += copy;
- }
- start = end;
- }
- if (!len)
- return 0;
-
-fault:
- iov_iter_revert(to, offset - start_off);
- return -EFAULT;
+ return __skb_datagram_iter(skb, offset, to, len, true,
+ csum_and_copy_to_iter, csump);
}
__sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len)
--
2.17.1
^ permalink raw reply related
* [PATCH v2 04/14] datagram: consolidate datagram copy to iter helpers
From: Sagi Grimberg @ 2018-11-20 3:00 UTC (permalink / raw)
To: linux-nvme
Cc: linux-block, netdev, David S. Miller, Keith Busch,
Christoph Hellwig
In-Reply-To: <20181120030019.31738-1-sagi@grimberg.me>
skb_copy_datagram_iter and skb_copy_and_csum_datagram are essentialy
the same but with a couple of differences: The first is the copy
operation used which either a simple copy or a csum_and_copy, and the
second are the behavior on the "short copy" path where simply copy
needs to return the number of bytes successfully copied while csum_and_copy
needs to fault immediately as the checksum is partial.
Introduce __skb_datagram_iter that additionally accepts:
1. copy operation function pointer
2. private data that goes with the copy operation
3. fault_short flag to indicate the action on short copy
Suggested-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Sagi Grimberg <sagi@grimberg.me>
---
net/core/datagram.c | 136 ++++++++++++++------------------------------
1 file changed, 42 insertions(+), 94 deletions(-)
diff --git a/net/core/datagram.c b/net/core/datagram.c
index abe642181b64..382543302ae5 100644
--- a/net/core/datagram.c
+++ b/net/core/datagram.c
@@ -408,27 +408,20 @@ int skb_kill_datagram(struct sock *sk, struct sk_buff *skb, unsigned int flags)
}
EXPORT_SYMBOL(skb_kill_datagram);
-/**
- * skb_copy_datagram_iter - Copy a datagram to an iovec iterator.
- * @skb: buffer to copy
- * @offset: offset in the buffer to start copying from
- * @to: iovec iterator to copy to
- * @len: amount of data to copy from buffer to iovec
- */
-int skb_copy_datagram_iter(const struct sk_buff *skb, int offset,
- struct iov_iter *to, int len)
+int __skb_datagram_iter(const struct sk_buff *skb, int offset,
+ struct iov_iter *to, int len, bool fault_short,
+ size_t (*cb)(const void *, size_t, void *, struct iov_iter *),
+ void *data)
{
int start = skb_headlen(skb);
int i, copy = start - offset, start_off = offset, n;
struct sk_buff *frag_iter;
- trace_skb_copy_datagram_iovec(skb, len);
-
/* Copy header. */
if (copy > 0) {
if (copy > len)
copy = len;
- n = copy_to_iter(skb->data + offset, copy, to);
+ n = cb(skb->data + offset, copy, data, to);
offset += n;
if (n != copy)
goto short_copy;
@@ -450,8 +443,8 @@ int skb_copy_datagram_iter(const struct sk_buff *skb, int offset,
if (copy > len)
copy = len;
- n = copy_to_iter(vaddr + frag->page_offset +
- offset - start, copy, to);
+ n = cb(vaddr + frag->page_offset +
+ offset - start, copy, data, to);
kunmap(page);
offset += n;
if (n != copy)
@@ -471,8 +464,8 @@ int skb_copy_datagram_iter(const struct sk_buff *skb, int offset,
if ((copy = end - offset) > 0) {
if (copy > len)
copy = len;
- if (skb_copy_datagram_iter(frag_iter, offset - start,
- to, copy))
+ if (__skb_datagram_iter(frag_iter, offset - start,
+ to, copy, short_copy, cb, data))
goto fault;
if ((len -= copy) == 0)
return 0;
@@ -493,11 +486,32 @@ int skb_copy_datagram_iter(const struct sk_buff *skb, int offset,
return -EFAULT;
short_copy:
- if (iov_iter_count(to))
+ if (fault_short || iov_iter_count(to))
goto fault;
return 0;
}
+
+static size_t simple_copy_to_iter(const void *addr, size_t bytes,
+ void *data __always_unused, struct iov_iter *i)
+{
+ return copy_to_iter(addr, bytes, i);
+}
+
+/**
+ * skb_copy_datagram_iter - Copy a datagram to an iovec iterator.
+ * @skb: buffer to copy
+ * @offset: offset in the buffer to start copying from
+ * @to: iovec iterator to copy to
+ * @len: amount of data to copy from buffer to iovec
+ */
+int skb_copy_datagram_iter(const struct sk_buff *skb, int offset,
+ struct iov_iter *to, int len)
+{
+ trace_skb_copy_datagram_iovec(skb, len);
+ return __skb_datagram_iter(skb, offset, to, len, false,
+ simple_copy_to_iter, NULL);
+}
EXPORT_SYMBOL(skb_copy_datagram_iter);
/**
@@ -648,87 +662,21 @@ int zerocopy_sg_from_iter(struct sk_buff *skb, struct iov_iter *from)
}
EXPORT_SYMBOL(zerocopy_sg_from_iter);
+/**
+ * skb_copy_and_csum_datagram_iter - Copy datagram to an iovec iterator
+ * and update a checksum.
+ * @skb: buffer to copy
+ * @offset: offset in the buffer to start copying from
+ * @to: iovec iterator to copy to
+ * @len: amount of data to copy from buffer to iovec
+ * @csump: checksum pointer
+ */
static int skb_copy_and_csum_datagram(const struct sk_buff *skb, int offset,
struct iov_iter *to, int len,
__wsum *csump)
{
- int start = skb_headlen(skb);
- int i, copy = start - offset, start_off = offset;
- struct sk_buff *frag_iter;
- int pos = 0;
- int n;
-
- /* Copy header. */
- if (copy > 0) {
- if (copy > len)
- copy = len;
- n = csum_and_copy_to_iter(skb->data + offset, copy, csump, to);
- offset += n;
- if (n != copy)
- goto fault;
- if ((len -= copy) == 0)
- return 0;
- pos = copy;
- }
-
- for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
- int end;
- const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
-
- WARN_ON(start > offset + len);
-
- end = start + skb_frag_size(frag);
- if ((copy = end - offset) > 0) {
- __wsum csum2 = 0;
- struct page *page = skb_frag_page(frag);
- u8 *vaddr = kmap(page);
-
- if (copy > len)
- copy = len;
- n = csum_and_copy_to_iter(vaddr + frag->page_offset +
- offset - start, copy,
- &csum2, to);
- kunmap(page);
- offset += n;
- if (n != copy)
- goto fault;
- *csump = csum_block_add(*csump, csum2, pos);
- if (!(len -= copy))
- return 0;
- pos += copy;
- }
- start = end;
- }
-
- skb_walk_frags(skb, frag_iter) {
- int end;
-
- WARN_ON(start > offset + len);
-
- end = start + frag_iter->len;
- if ((copy = end - offset) > 0) {
- __wsum csum2 = 0;
- if (copy > len)
- copy = len;
- if (skb_copy_and_csum_datagram(frag_iter,
- offset - start,
- to, copy,
- &csum2))
- goto fault;
- *csump = csum_block_add(*csump, csum2, pos);
- if ((len -= copy) == 0)
- return 0;
- offset += copy;
- pos += copy;
- }
- start = end;
- }
- if (!len)
- return 0;
-
-fault:
- iov_iter_revert(to, offset - start_off);
- return -EFAULT;
+ return __skb_datagram_iter(skb, offset, to, len, true,
+ csum_and_copy_to_iter, csump);
}
__sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len)
--
2.17.1
^ permalink raw reply related
* [PATCH v2 07/14] nvme-core: add work elements to struct nvme_ctrl
From: Sagi Grimberg @ 2018-11-20 3:00 UTC (permalink / raw)
To: linux-nvme
Cc: linux-block, netdev, David S. Miller, Keith Busch,
Christoph Hellwig
In-Reply-To: <20181120030019.31738-1-sagi@grimberg.me>
From: Sagi Grimberg <sagi@lightbitslabs.com>
connect_work and err_work will be reused by nvme-tcp so
share those in nvme_ctrl for rdma and fc to share.
Reviewed-by: Max Gurtovoy <maxg@mellanox.com>
Signed-off-by: Sagi Grimberg <sagi@lightbitslabs.com>
---
drivers/nvme/host/fc.c | 18 ++++++++----------
drivers/nvme/host/nvme.h | 2 ++
drivers/nvme/host/rdma.c | 19 ++++++++-----------
3 files changed, 18 insertions(+), 21 deletions(-)
diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c
index 83131e42b336..16812e427e17 100644
--- a/drivers/nvme/host/fc.c
+++ b/drivers/nvme/host/fc.c
@@ -159,8 +159,6 @@ struct nvme_fc_ctrl {
struct blk_mq_tag_set admin_tag_set;
struct blk_mq_tag_set tag_set;
- struct delayed_work connect_work;
-
struct kref ref;
u32 flags;
u32 iocnt;
@@ -547,7 +545,7 @@ nvme_fc_resume_controller(struct nvme_fc_ctrl *ctrl)
"NVME-FC{%d}: connectivity re-established. "
"Attempting reconnect\n", ctrl->cnum);
- queue_delayed_work(nvme_wq, &ctrl->connect_work, 0);
+ queue_delayed_work(nvme_wq, &ctrl->ctrl.connect_work, 0);
break;
case NVME_CTRL_RESETTING:
@@ -2815,7 +2813,7 @@ nvme_fc_delete_ctrl(struct nvme_ctrl *nctrl)
{
struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl);
- cancel_delayed_work_sync(&ctrl->connect_work);
+ cancel_delayed_work_sync(&ctrl->ctrl.connect_work);
/*
* kill the association on the link side. this will block
* waiting for io to terminate
@@ -2850,7 +2848,7 @@ nvme_fc_reconnect_or_delete(struct nvme_fc_ctrl *ctrl, int status)
else if (time_after(jiffies + recon_delay, rport->dev_loss_end))
recon_delay = rport->dev_loss_end - jiffies;
- queue_delayed_work(nvme_wq, &ctrl->connect_work, recon_delay);
+ queue_delayed_work(nvme_wq, &ctrl->ctrl.connect_work, recon_delay);
} else {
if (portptr->port_state == FC_OBJSTATE_ONLINE)
dev_warn(ctrl->ctrl.device,
@@ -2918,7 +2916,7 @@ nvme_fc_connect_ctrl_work(struct work_struct *work)
struct nvme_fc_ctrl *ctrl =
container_of(to_delayed_work(work),
- struct nvme_fc_ctrl, connect_work);
+ struct nvme_fc_ctrl, ctrl.connect_work);
ret = nvme_fc_create_association(ctrl);
if (ret)
@@ -3015,7 +3013,7 @@ nvme_fc_init_ctrl(struct device *dev, struct nvmf_ctrl_options *opts,
kref_init(&ctrl->ref);
INIT_WORK(&ctrl->ctrl.reset_work, nvme_fc_reset_ctrl_work);
- INIT_DELAYED_WORK(&ctrl->connect_work, nvme_fc_connect_ctrl_work);
+ INIT_DELAYED_WORK(&ctrl->ctrl.connect_work, nvme_fc_connect_ctrl_work);
spin_lock_init(&ctrl->lock);
/* io queue count */
@@ -3086,7 +3084,7 @@ nvme_fc_init_ctrl(struct device *dev, struct nvmf_ctrl_options *opts,
nvme_get_ctrl(&ctrl->ctrl);
- if (!queue_delayed_work(nvme_wq, &ctrl->connect_work, 0)) {
+ if (!queue_delayed_work(nvme_wq, &ctrl->ctrl.connect_work, 0)) {
nvme_put_ctrl(&ctrl->ctrl);
dev_err(ctrl->ctrl.device,
"NVME-FC{%d}: failed to schedule initial connect\n",
@@ -3094,7 +3092,7 @@ nvme_fc_init_ctrl(struct device *dev, struct nvmf_ctrl_options *opts,
goto fail_ctrl;
}
- flush_delayed_work(&ctrl->connect_work);
+ flush_delayed_work(&ctrl->ctrl.connect_work);
dev_info(ctrl->ctrl.device,
"NVME-FC{%d}: new ctrl: NQN \"%s\"\n",
@@ -3105,7 +3103,7 @@ nvme_fc_init_ctrl(struct device *dev, struct nvmf_ctrl_options *opts,
fail_ctrl:
nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_DELETING);
cancel_work_sync(&ctrl->ctrl.reset_work);
- cancel_delayed_work_sync(&ctrl->connect_work);
+ cancel_delayed_work_sync(&ctrl->ctrl.connect_work);
ctrl->ctrl.opts = NULL;
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index 27663ce3044e..031195e5d7d3 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -240,6 +240,8 @@ struct nvme_ctrl {
u16 maxcmd;
int nr_reconnects;
struct nvmf_ctrl_options *opts;
+ struct delayed_work connect_work;
+ struct work_struct err_work;
};
struct nvme_subsystem {
diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c
index 4468d672ced9..779c2c043242 100644
--- a/drivers/nvme/host/rdma.c
+++ b/drivers/nvme/host/rdma.c
@@ -101,12 +101,9 @@ struct nvme_rdma_ctrl {
/* other member variables */
struct blk_mq_tag_set tag_set;
- struct work_struct err_work;
struct nvme_rdma_qe async_event_sqe;
- struct delayed_work reconnect_work;
-
struct list_head list;
struct blk_mq_tag_set admin_tag_set;
@@ -910,8 +907,8 @@ static void nvme_rdma_stop_ctrl(struct nvme_ctrl *nctrl)
{
struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
- cancel_work_sync(&ctrl->err_work);
- cancel_delayed_work_sync(&ctrl->reconnect_work);
+ cancel_work_sync(&ctrl->ctrl.err_work);
+ cancel_delayed_work_sync(&ctrl->ctrl.connect_work);
}
static void nvme_rdma_free_ctrl(struct nvme_ctrl *nctrl)
@@ -943,7 +940,7 @@ static void nvme_rdma_reconnect_or_remove(struct nvme_rdma_ctrl *ctrl)
if (nvmf_should_reconnect(&ctrl->ctrl)) {
dev_info(ctrl->ctrl.device, "Reconnecting in %d seconds...\n",
ctrl->ctrl.opts->reconnect_delay);
- queue_delayed_work(nvme_wq, &ctrl->reconnect_work,
+ queue_delayed_work(nvme_wq, &ctrl->ctrl.connect_work,
ctrl->ctrl.opts->reconnect_delay * HZ);
} else {
nvme_delete_ctrl(&ctrl->ctrl);
@@ -1015,7 +1012,7 @@ static int nvme_rdma_setup_ctrl(struct nvme_rdma_ctrl *ctrl, bool new)
static void nvme_rdma_reconnect_ctrl_work(struct work_struct *work)
{
struct nvme_rdma_ctrl *ctrl = container_of(to_delayed_work(work),
- struct nvme_rdma_ctrl, reconnect_work);
+ struct nvme_rdma_ctrl, ctrl.connect_work);
++ctrl->ctrl.nr_reconnects;
@@ -1038,7 +1035,7 @@ static void nvme_rdma_reconnect_ctrl_work(struct work_struct *work)
static void nvme_rdma_error_recovery_work(struct work_struct *work)
{
struct nvme_rdma_ctrl *ctrl = container_of(work,
- struct nvme_rdma_ctrl, err_work);
+ struct nvme_rdma_ctrl, ctrl.err_work);
nvme_stop_keep_alive(&ctrl->ctrl);
nvme_rdma_teardown_io_queues(ctrl, false);
@@ -1059,7 +1056,7 @@ static void nvme_rdma_error_recovery(struct nvme_rdma_ctrl *ctrl)
if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_RESETTING))
return;
- queue_work(nvme_wq, &ctrl->err_work);
+ queue_work(nvme_wq, &ctrl->ctrl.err_work);
}
static void nvme_rdma_wr_error(struct ib_cq *cq, struct ib_wc *wc,
@@ -1932,9 +1929,9 @@ static struct nvme_ctrl *nvme_rdma_create_ctrl(struct device *dev,
goto out_free_ctrl;
}
- INIT_DELAYED_WORK(&ctrl->reconnect_work,
+ INIT_DELAYED_WORK(&ctrl->ctrl.connect_work,
nvme_rdma_reconnect_ctrl_work);
- INIT_WORK(&ctrl->err_work, nvme_rdma_error_recovery_work);
+ INIT_WORK(&ctrl->ctrl.err_work, nvme_rdma_error_recovery_work);
INIT_WORK(&ctrl->ctrl.reset_work, nvme_rdma_reset_ctrl_work);
ctrl->ctrl.queue_count = opts->nr_io_queues + 1; /* +1 for admin queue */
--
2.17.1
^ permalink raw reply related
* [PATCH v2 05/14] iov_iter: introduce hash_and_copy_to_iter helper
From: Sagi Grimberg @ 2018-11-20 3:00 UTC (permalink / raw)
To: linux-nvme
Cc: linux-block, netdev, David S. Miller, Keith Busch,
Christoph Hellwig
In-Reply-To: <20181120030019.31738-1-sagi@grimberg.me>
From: Sagi Grimberg <sagi@lightbitslabs.com>
Allow consumers that want to use iov iterator helpers and also update
a predefined hash calculation online when copying data. This is useful
when copying incoming network buffers to a local iterator and calculate
a digest on the incoming stream. nvme-tcp host driver that will be
introduced in following patches is the first consumer via
skb_copy_and_hash_datagram_iter.
Signed-off-by: Sagi Grimberg <sagi@lightbitslabs.com>
---
include/linux/uio.h | 3 +++
lib/iov_iter.c | 16 ++++++++++++++++
2 files changed, 19 insertions(+)
diff --git a/include/linux/uio.h b/include/linux/uio.h
index 41d1f8d3313d..ecf584f6b82d 100644
--- a/include/linux/uio.h
+++ b/include/linux/uio.h
@@ -11,6 +11,7 @@
#include <linux/kernel.h>
#include <linux/thread_info.h>
+#include <crypto/hash.h>
#include <uapi/linux/uio.h>
struct page;
@@ -269,6 +270,8 @@ static inline void iov_iter_reexpand(struct iov_iter *i, size_t count)
size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *csump, struct iov_iter *i);
size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum, struct iov_iter *i);
bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum, struct iov_iter *i);
+size_t hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp,
+ struct iov_iter *i);
int import_iovec(int type, const struct iovec __user * uvector,
unsigned nr_segs, unsigned fast_segs,
diff --git a/lib/iov_iter.c b/lib/iov_iter.c
index db93531ca3e3..8a5f7b2ae346 100644
--- a/lib/iov_iter.c
+++ b/lib/iov_iter.c
@@ -6,6 +6,7 @@
#include <linux/vmalloc.h>
#include <linux/splice.h>
#include <net/checksum.h>
+#include <linux/scatterlist.h>
#define PIPE_PARANOIA /* for now */
@@ -1475,6 +1476,21 @@ size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *csump,
}
EXPORT_SYMBOL(csum_and_copy_to_iter);
+size_t hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp,
+ struct iov_iter *i)
+{
+ struct ahash_request *hash = hashp;
+ struct scatterlist sg;
+ size_t copied;
+
+ copied = copy_to_iter(addr, bytes, i);
+ sg_init_one(&sg, addr, copied);
+ ahash_request_set_crypt(hash, &sg, NULL, copied);
+ crypto_ahash_update(hash);
+ return copied;
+}
+EXPORT_SYMBOL(hash_and_copy_to_iter);
+
int iov_iter_npages(const struct iov_iter *i, int maxpages)
{
size_t size = i->count;
--
2.17.1
^ permalink raw reply related
* [PATCH v2 06/14] datagram: introduce skb_copy_and_hash_datagram_iter helper
From: Sagi Grimberg @ 2018-11-20 3:00 UTC (permalink / raw)
To: linux-nvme
Cc: linux-block, netdev, David S. Miller, Keith Busch,
Christoph Hellwig
In-Reply-To: <20181120030019.31738-1-sagi@grimberg.me>
Introduce a helper to copy datagram into an iovec iterator
but also update a predefined hash. This is useful for
consumers of skb_copy_datagram_iter to also support inflight
data digest without having to finish to copy and only then
traverse the iovec and calculate the digest hash.
Signed-off-by: Sagi Grimberg <sagi@grimberg.me>
---
include/linux/skbuff.h | 3 +++
net/core/datagram.c | 17 +++++++++++++++++
2 files changed, 20 insertions(+)
diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
index 0ba687454267..b0b8d5653f0d 100644
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -3309,6 +3309,9 @@ static inline int skb_copy_datagram_msg(const struct sk_buff *from, int offset,
}
int skb_copy_and_csum_datagram_msg(struct sk_buff *skb, int hlen,
struct msghdr *msg);
+int skb_copy_and_hash_datagram_iter(const struct sk_buff *skb, int offset,
+ struct iov_iter *to, int len,
+ struct ahash_request *hash);
int skb_copy_datagram_from_iter(struct sk_buff *skb, int offset,
struct iov_iter *from, int len);
int zerocopy_sg_from_iter(struct sk_buff *skb, struct iov_iter *frm);
diff --git a/net/core/datagram.c b/net/core/datagram.c
index 382543302ae5..e6a4fc845f72 100644
--- a/net/core/datagram.c
+++ b/net/core/datagram.c
@@ -492,6 +492,23 @@ int __skb_datagram_iter(const struct sk_buff *skb, int offset,
return 0;
}
+/**
+ * skb_copy_and_hash_datagram_iter - Copy datagram to an iovec iterator
+ * and update a hash.
+ * @skb: buffer to copy
+ * @offset: offset in the buffer to start copying from
+ * @to: iovec iterator to copy to
+ * @len: amount of data to copy from buffer to iovec
+ * @hash: hash request to update
+ */
+int skb_copy_and_hash_datagram_iter(const struct sk_buff *skb, int offset,
+ struct iov_iter *to, int len,
+ struct ahash_request *hash)
+{
+ return __skb_datagram_iter(skb, offset, to, len, true,
+ hash_and_copy_to_iter, hash);
+}
+
static size_t simple_copy_to_iter(const void *addr, size_t bytes,
void *data __always_unused, struct iov_iter *i)
{
--
2.17.1
^ permalink raw reply related
* [PATCH v2 08/14] nvmet: Add install_queue callout
From: Sagi Grimberg @ 2018-11-20 3:00 UTC (permalink / raw)
To: linux-nvme
Cc: linux-block, netdev, David S. Miller, Keith Busch,
Christoph Hellwig
In-Reply-To: <20181120030019.31738-1-sagi@grimberg.me>
From: Sagi Grimberg <sagi@lightbitslabs.com>
nvmet-tcp will implement it to allocate queue commands which
are only known at nvmf connect time (sq size).
Signed-off-by: Sagi Grimberg <sagi@lightbitslabs.com>
---
drivers/nvme/target/fabrics-cmd.c | 9 +++++++++
drivers/nvme/target/nvmet.h | 1 +
2 files changed, 10 insertions(+)
diff --git a/drivers/nvme/target/fabrics-cmd.c b/drivers/nvme/target/fabrics-cmd.c
index 328ae46d8344..86ebf191b035 100644
--- a/drivers/nvme/target/fabrics-cmd.c
+++ b/drivers/nvme/target/fabrics-cmd.c
@@ -121,6 +121,15 @@ static u16 nvmet_install_queue(struct nvmet_ctrl *ctrl, struct nvmet_req *req)
req->rsp->sq_head = cpu_to_le16(0xffff);
}
+ if (ctrl->ops->install_queue) {
+ u16 ret = ctrl->ops->install_queue(req->sq);
+ if (ret) {
+ pr_err("failed to install queue %d cntlid %d ret %x\n",
+ qid, ret, ctrl->cntlid);
+ return ret;
+ }
+ }
+
return 0;
}
diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h
index 547108c41ce9..957eb7edb902 100644
--- a/drivers/nvme/target/nvmet.h
+++ b/drivers/nvme/target/nvmet.h
@@ -279,6 +279,7 @@ struct nvmet_fabrics_ops {
void (*delete_ctrl)(struct nvmet_ctrl *ctrl);
void (*disc_traddr)(struct nvmet_req *req,
struct nvmet_port *port, char *traddr);
+ u16 (*install_queue)(struct nvmet_sq *nvme_sq);
};
#define NVMET_MAX_INLINE_BIOVEC 8
--
2.17.1
^ permalink raw reply related
* [PATCH v2 11/14] nvme-fabrics: allow user passing data digest
From: Sagi Grimberg @ 2018-11-20 3:00 UTC (permalink / raw)
To: linux-nvme
Cc: linux-block, netdev, David S. Miller, Keith Busch,
Christoph Hellwig
In-Reply-To: <20181120030019.31738-1-sagi@grimberg.me>
From: Sagi Grimberg <sagi@lightbitslabs.com>
Data digest is a nvme-tcp specific feature, but nothing prevents other
transports reusing the concept so do not associate with tcp transport
solely.
Signed-off-by: Sagi Grimberg <sagi@lightbitslabs.com>
---
drivers/nvme/host/fabrics.c | 5 +++++
drivers/nvme/host/fabrics.h | 2 ++
2 files changed, 7 insertions(+)
diff --git a/drivers/nvme/host/fabrics.c b/drivers/nvme/host/fabrics.c
index 4272f8a95db3..9c62c6838b76 100644
--- a/drivers/nvme/host/fabrics.c
+++ b/drivers/nvme/host/fabrics.c
@@ -615,6 +615,7 @@ static const match_table_t opt_tokens = {
{ NVMF_OPT_DUP_CONNECT, "duplicate_connect" },
{ NVMF_OPT_DISABLE_SQFLOW, "disable_sqflow" },
{ NVMF_OPT_HDR_DIGEST, "hdr_digest" },
+ { NVMF_OPT_DATA_DIGEST, "data_digest" },
{ NVMF_OPT_ERR, NULL }
};
@@ -635,6 +636,7 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts,
opts->kato = NVME_DEFAULT_KATO;
opts->duplicate_connect = false;
opts->hdr_digest = false;
+ opts->data_digest = false;
options = o = kstrdup(buf, GFP_KERNEL);
if (!options)
@@ -832,6 +834,9 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts,
case NVMF_OPT_HDR_DIGEST:
opts->hdr_digest = true;
break;
+ case NVMF_OPT_DATA_DIGEST:
+ opts->data_digest = true;
+ break;
default:
pr_warn("unknown parameter or missing value '%s' in ctrl creation request\n",
p);
diff --git a/drivers/nvme/host/fabrics.h b/drivers/nvme/host/fabrics.h
index a6127f1a9e8e..524a02a67817 100644
--- a/drivers/nvme/host/fabrics.h
+++ b/drivers/nvme/host/fabrics.h
@@ -60,6 +60,7 @@ enum {
NVMF_OPT_DUP_CONNECT = 1 << 13,
NVMF_OPT_DISABLE_SQFLOW = 1 << 14,
NVMF_OPT_HDR_DIGEST = 1 << 15,
+ NVMF_OPT_DATA_DIGEST = 1 << 16,
};
/**
@@ -105,6 +106,7 @@ struct nvmf_ctrl_options {
int max_reconnects;
bool disable_sqflow;
bool hdr_digest;
+ bool data_digest;
};
/*
--
2.17.1
^ permalink raw reply related
* [PATCH v2 10/14] nvme-fabrics: allow user passing header digest
From: Sagi Grimberg @ 2018-11-20 3:00 UTC (permalink / raw)
To: linux-nvme
Cc: linux-block, netdev, David S. Miller, Keith Busch,
Christoph Hellwig
In-Reply-To: <20181120030019.31738-1-sagi@grimberg.me>
From: Sagi Grimberg <sagi@lightbitslabs.com>
Header digest is a nvme-tcp specific feature, but nothing prevents other
transports reusing the concept so do not associate with tcp transport
solely.
Signed-off-by: Sagi Grimberg <sagi@lightbitslabs.com>
---
drivers/nvme/host/fabrics.c | 5 +++++
drivers/nvme/host/fabrics.h | 2 ++
2 files changed, 7 insertions(+)
diff --git a/drivers/nvme/host/fabrics.c b/drivers/nvme/host/fabrics.c
index 10074ac7731b..4272f8a95db3 100644
--- a/drivers/nvme/host/fabrics.c
+++ b/drivers/nvme/host/fabrics.c
@@ -614,6 +614,7 @@ static const match_table_t opt_tokens = {
{ NVMF_OPT_HOST_ID, "hostid=%s" },
{ NVMF_OPT_DUP_CONNECT, "duplicate_connect" },
{ NVMF_OPT_DISABLE_SQFLOW, "disable_sqflow" },
+ { NVMF_OPT_HDR_DIGEST, "hdr_digest" },
{ NVMF_OPT_ERR, NULL }
};
@@ -633,6 +634,7 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts,
opts->reconnect_delay = NVMF_DEF_RECONNECT_DELAY;
opts->kato = NVME_DEFAULT_KATO;
opts->duplicate_connect = false;
+ opts->hdr_digest = false;
options = o = kstrdup(buf, GFP_KERNEL);
if (!options)
@@ -827,6 +829,9 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts,
case NVMF_OPT_DISABLE_SQFLOW:
opts->disable_sqflow = true;
break;
+ case NVMF_OPT_HDR_DIGEST:
+ opts->hdr_digest = true;
+ break;
default:
pr_warn("unknown parameter or missing value '%s' in ctrl creation request\n",
p);
diff --git a/drivers/nvme/host/fabrics.h b/drivers/nvme/host/fabrics.h
index ecd9a006a091..a6127f1a9e8e 100644
--- a/drivers/nvme/host/fabrics.h
+++ b/drivers/nvme/host/fabrics.h
@@ -59,6 +59,7 @@ enum {
NVMF_OPT_HOST_ID = 1 << 12,
NVMF_OPT_DUP_CONNECT = 1 << 13,
NVMF_OPT_DISABLE_SQFLOW = 1 << 14,
+ NVMF_OPT_HDR_DIGEST = 1 << 15,
};
/**
@@ -103,6 +104,7 @@ struct nvmf_ctrl_options {
struct nvmf_host *host;
int max_reconnects;
bool disable_sqflow;
+ bool hdr_digest;
};
/*
--
2.17.1
^ permalink raw reply related
* [PATCH v2 09/14] nvmet: allow configfs tcp trtype configuration
From: Sagi Grimberg @ 2018-11-20 3:00 UTC (permalink / raw)
To: linux-nvme
Cc: linux-block, netdev, David S. Miller, Keith Busch,
Christoph Hellwig
In-Reply-To: <20181120030019.31738-1-sagi@grimberg.me>
From: Sagi Grimberg <sagi@lightbitslabs.com>
Reviewed-by: Max Gurtovoy <maxg@mellanox.com>
Signed-off-by: Sagi Grimberg <sagi@lightbitslabs.com>
---
drivers/nvme/target/configfs.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c
index db2cb64be7ba..618bbd006544 100644
--- a/drivers/nvme/target/configfs.c
+++ b/drivers/nvme/target/configfs.c
@@ -34,6 +34,7 @@ static const struct nvmet_transport_name {
} nvmet_transport_names[] = {
{ NVMF_TRTYPE_RDMA, "rdma" },
{ NVMF_TRTYPE_FC, "fc" },
+ { NVMF_TRTYPE_TCP, "tcp" },
{ NVMF_TRTYPE_LOOP, "loop" },
};
--
2.17.1
^ permalink raw reply related
* [PATCH nvme-cli v2 16/14] fabrics: add tcp port tsas decoding
From: Sagi Grimberg @ 2018-11-20 3:00 UTC (permalink / raw)
To: linux-nvme
Cc: linux-block, netdev, David S. Miller, Keith Busch,
Christoph Hellwig
In-Reply-To: <20181120030019.31738-1-sagi@grimberg.me>
From: Sagi Grimberg <sagi@lightbitslabs.com>
tcp tsas include sectype indication for unsecured/tls ports.
Signed-off-by: Sagi Grimberg <sagi@lightbitslabs.com>
---
fabrics.c | 14 ++++++++++++++
linux/nvme.h | 10 ++++++++++
2 files changed, 24 insertions(+)
diff --git a/fabrics.c b/fabrics.c
index b4fac97253ba..3df273894632 100644
--- a/fabrics.c
+++ b/fabrics.c
@@ -141,6 +141,16 @@ static inline const char *treq_str(__u8 treq)
return arg_str(treqs, ARRAY_SIZE(treqs), treq);
}
+static const char * const sectypes[] = {
+ [NVMF_TCP_SECTYPE_NONE] = "none",
+ [NVMF_TCP_SECTYPE_TLS] = "tls",
+};
+
+static inline const char *sectype_str(__u8 sectype)
+{
+ return arg_str(sectypes, ARRAY_SIZE(sectypes), sectype);
+}
+
static const char * const prtypes[] = {
[NVMF_RDMA_PRTYPE_NOT_SPECIFIED] = "not specified",
[NVMF_RDMA_PRTYPE_IB] = "infiniband",
@@ -450,6 +460,10 @@ static void print_discovery_log(struct nvmf_disc_rsp_page_hdr *log, int numrec)
printf("rdma_pkey: 0x%04x\n",
e->tsas.rdma.pkey);
break;
+ case NVMF_TRTYPE_TCP:
+ printf("sectype: %s\n",
+ sectype_str(e->tsas.tcp.sectype));
+ break;
}
}
}
diff --git a/linux/nvme.h b/linux/nvme.h
index 7a600c791877..68000eb8c1dc 100644
--- a/linux/nvme.h
+++ b/linux/nvme.h
@@ -91,6 +91,13 @@ enum {
NVMF_RDMA_CMS_RDMA_CM = 1, /* Sockets based endpoint addressing */
};
+/* TCP port security type for Discovery Log Page entry TSAS
+ */
+enum {
+ NVMF_TCP_SECTYPE_NONE = 0, /* No Security */
+ NVMF_TCP_SECTYPE_TLS = 1, /* Transport Layer Security */
+};
+
#define NVME_AQ_DEPTH 32
#define NVME_NR_AEN_COMMANDS 1
#define NVME_AQ_BLK_MQ_DEPTH (NVME_AQ_DEPTH - NVME_NR_AEN_COMMANDS)
@@ -1184,6 +1191,9 @@ struct nvmf_disc_rsp_page_entry {
__u16 pkey;
__u8 resv10[246];
} rdma;
+ struct tcp {
+ __u8 sectype;
+ } tcp;
} tsas;
};
--
2.17.1
^ permalink raw reply related
* [PATCH v2 13/14] nvmet-tcp: add NVMe over TCP target driver
From: Sagi Grimberg @ 2018-11-20 3:00 UTC (permalink / raw)
To: linux-nvme
Cc: linux-block, netdev, David S. Miller, Keith Busch,
Christoph Hellwig
In-Reply-To: <20181120030019.31738-1-sagi@grimberg.me>
From: Sagi Grimberg <sagi@lightbitslabs.com>
This patch implements the TCP transport driver for the NVMe over Fabrics
target stack. This allows exporting NVMe over Fabrics functionality over
good old TCP/IP.
The driver implements the TP 8000 of how nvme over fabrics capsules and
data are encapsulated in nvme-tcp pdus and exchaged on top of a TCP byte
stream. nvme-tcp header and data digest are supported as well.
Signed-off-by: Sagi Grimberg <sagi@lightbitslabs.com>
Signed-off-by: Roy Shterman <roys@lightbitslabs.com>
Signed-off-by: Solganik Alexander <sashas@lightbitslabs.com>
---
drivers/nvme/target/Kconfig | 10 +
drivers/nvme/target/Makefile | 2 +
drivers/nvme/target/tcp.c | 1741 ++++++++++++++++++++++++++++++++++
include/linux/nvme-tcp.h | 1 +
4 files changed, 1754 insertions(+)
create mode 100644 drivers/nvme/target/tcp.c
diff --git a/drivers/nvme/target/Kconfig b/drivers/nvme/target/Kconfig
index 3c7b61ddb0d1..d94f25cde019 100644
--- a/drivers/nvme/target/Kconfig
+++ b/drivers/nvme/target/Kconfig
@@ -60,3 +60,13 @@ config NVME_TARGET_FCLOOP
to test NVMe-FC transport interfaces.
If unsure, say N.
+
+config NVME_TARGET_TCP
+ tristate "NVMe over Fabrics TCP target support"
+ depends on INET
+ depends on NVME_TARGET
+ help
+ This enables the NVMe TCP target support, which allows exporting NVMe
+ devices over TCP.
+
+ If unsure, say N.
diff --git a/drivers/nvme/target/Makefile b/drivers/nvme/target/Makefile
index 8118c93391c6..8c3ad0fb6860 100644
--- a/drivers/nvme/target/Makefile
+++ b/drivers/nvme/target/Makefile
@@ -5,6 +5,7 @@ obj-$(CONFIG_NVME_TARGET_LOOP) += nvme-loop.o
obj-$(CONFIG_NVME_TARGET_RDMA) += nvmet-rdma.o
obj-$(CONFIG_NVME_TARGET_FC) += nvmet-fc.o
obj-$(CONFIG_NVME_TARGET_FCLOOP) += nvme-fcloop.o
+obj-$(CONFIG_NVME_TARGET_TCP) += nvmet-tcp.o
nvmet-y += core.o configfs.o admin-cmd.o fabrics-cmd.o \
discovery.o io-cmd-file.o io-cmd-bdev.o
@@ -12,3 +13,4 @@ nvme-loop-y += loop.o
nvmet-rdma-y += rdma.o
nvmet-fc-y += fc.o
nvme-fcloop-y += fcloop.o
+nvmet-tcp-y += tcp.o
diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c
new file mode 100644
index 000000000000..61311e518072
--- /dev/null
+++ b/drivers/nvme/target/tcp.c
@@ -0,0 +1,1741 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * NVMe over Fabrics TCP target.
+ * Copyright (c) 2018 LightBits Labs. All rights reserved.
+ */
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <linux/err.h>
+#include <linux/nvme-tcp.h>
+#include <net/sock.h>
+#include <net/tcp.h>
+#include <linux/inet.h>
+#include <linux/llist.h>
+#include <crypto/hash.h>
+
+#include "nvmet.h"
+
+#define NVMET_TCP_DEF_INLINE_DATA_SIZE (4 * PAGE_SIZE)
+
+#define NVMET_TCP_RECV_BUDGET 8
+#define NVMET_TCP_SEND_BUDGET 8
+#define NVMET_TCP_IO_WORK_BUDGET 64
+
+enum nvmet_tcp_send_state {
+ NVMET_TCP_SEND_DATA_PDU = 0,
+ NVMET_TCP_SEND_DATA,
+ NVMET_TCP_SEND_R2T,
+ NVMET_TCP_SEND_DDGST,
+ NVMET_TCP_SEND_RESPONSE
+};
+
+enum nvmet_tcp_recv_state {
+ NVMET_TCP_RECV_PDU,
+ NVMET_TCP_RECV_DATA,
+ NVMET_TCP_RECV_DDGST,
+ NVMET_TCP_RECV_ERR,
+};
+
+struct nvmet_tcp_send_ctx {
+ u32 offset;
+ struct scatterlist *cur_sg;
+ enum nvmet_tcp_send_state state;
+};
+
+enum {
+ NVMET_TCP_F_INIT_FAILED = (1 << 0),
+};
+
+struct nvmet_tcp_cmd {
+ struct nvmet_tcp_queue *queue;
+ struct nvmet_req req;
+
+ struct nvme_tcp_cmd_pdu *cmd_pdu;
+ struct nvme_tcp_rsp_pdu *rsp_pdu;
+ struct nvme_tcp_data_pdu *data_pdu;
+ struct nvme_tcp_r2t_pdu *r2t_pdu;
+
+ u32 rbytes_done;
+ u32 wbytes_done;
+
+ u32 pdu_len;
+ u32 pdu_recv;
+ int sg_idx;
+ int nr_mapped;
+ struct msghdr recv_msg;
+ struct kvec *iov;
+ u32 flags;
+
+ struct list_head entry;
+ struct llist_node lentry;
+ struct nvmet_tcp_send_ctx snd;
+ __le32 exp_ddgst;
+ __le32 recv_ddgst;
+};
+
+enum nvmet_tcp_queue_state {
+ NVMET_TCP_Q_CONNECTING,
+ NVMET_TCP_Q_LIVE,
+ NVMET_TCP_Q_DISCONNECTING,
+};
+
+struct nvmet_tcp_recv_ctx {
+ union nvme_tcp_pdu pdu;
+ int offset;
+ int left;
+ enum nvmet_tcp_recv_state state;
+ struct nvmet_tcp_cmd *cmd;
+};
+
+struct nvmet_tcp_queue {
+ struct socket *sock;
+ struct nvmet_tcp_port *port;
+
+ struct nvmet_tcp_cmd *cmds;
+ unsigned nr_cmds;
+ struct list_head free_list;
+ struct llist_head resp_list;
+ struct list_head resp_send_list;
+ int send_list_len;
+
+ spinlock_t state_lock;
+ enum nvmet_tcp_queue_state state;
+ struct nvmet_cq nvme_cq;
+ struct nvmet_sq nvme_sq;
+
+ struct sockaddr_storage sockaddr;
+ struct sockaddr_storage sockaddr_peer;
+ struct work_struct release_work;
+ struct work_struct io_work;
+
+ int idx;
+ int cpu;
+
+ struct list_head queue_list;
+ struct nvmet_tcp_cmd *snd_cmd;
+ struct nvmet_tcp_recv_ctx rcv;
+
+ bool hdr_digest;
+ bool data_digest;
+ struct ahash_request *snd_hash;
+ struct ahash_request *rcv_hash;
+
+ struct nvmet_tcp_cmd connect;
+
+ struct page_frag_cache pf_cache;
+
+ void (*old_data_ready)(struct sock *);
+ void (*old_state_change)(struct sock *);
+ void (*old_write_space)(struct sock *);
+};
+
+struct nvmet_tcp_port {
+ struct socket *sock;
+ struct work_struct accept_work;
+ struct nvmet_port *nport;
+ struct sockaddr_storage addr;
+ int last_cpu;
+ void (*old_data_ready) (struct sock *);
+};
+
+static DEFINE_IDA(nvmet_tcp_queue_ida);
+static LIST_HEAD(nvmet_tcp_queue_list);
+static DEFINE_MUTEX(nvmet_tcp_queue_mutex);
+
+static struct workqueue_struct *nvmet_tcp_wq;
+static struct nvmet_fabrics_ops nvmet_tcp_ops;
+static void nvmet_tcp_free_cmd(struct nvmet_tcp_cmd *c);
+static void nvmet_tcp_finish_cmd(struct nvmet_tcp_cmd *cmd);
+
+static inline u16 nvmet_tcp_cmd_id(struct nvmet_tcp_queue *queue,
+ struct nvmet_tcp_cmd *cmd)
+{
+ return cmd - queue->cmds;
+}
+
+static inline bool nvmet_tcp_has_data_in(struct nvmet_tcp_cmd *cmd)
+{
+ return nvme_is_write(cmd->req.cmd) &&
+ cmd->rbytes_done < cmd->req.transfer_len;
+}
+
+static inline bool nvmet_tcp_need_data_in(struct nvmet_tcp_cmd *cmd)
+{
+ return nvmet_tcp_has_data_in(cmd) && !cmd->req.rsp->status;
+}
+
+static inline bool nvmet_tcp_need_data_out(struct nvmet_tcp_cmd *cmd)
+{
+ return !nvme_is_write(cmd->req.cmd) &&
+ cmd->req.transfer_len > 0 &&
+ !cmd->req.rsp->status;
+}
+
+static inline bool nvmet_tcp_has_inline_data(struct nvmet_tcp_cmd *cmd)
+{
+ return nvme_is_write(cmd->req.cmd) && cmd->pdu_len &&
+ !cmd->rbytes_done;
+}
+
+static inline struct nvmet_tcp_cmd *
+nvmet_tcp_get_cmd(struct nvmet_tcp_queue *queue)
+{
+ struct nvmet_tcp_cmd *cmd;
+
+ cmd = list_first_entry_or_null(&queue->free_list,
+ struct nvmet_tcp_cmd, entry);
+ if (!cmd)
+ return NULL;
+ list_del_init(&cmd->entry);
+
+ cmd->rbytes_done = cmd->wbytes_done = 0;
+ cmd->pdu_len = 0;
+ cmd->pdu_recv = 0;
+ cmd->iov = NULL;
+ cmd->flags = 0;
+ return cmd;
+}
+
+static inline void nvmet_tcp_put_cmd(struct nvmet_tcp_cmd *cmd)
+{
+ if (unlikely(cmd == &cmd->queue->connect))
+ return;
+
+ list_add_tail(&cmd->entry, &cmd->queue->free_list);
+}
+
+static inline u8 nvmet_tcp_hdgst_len(struct nvmet_tcp_queue *queue)
+{
+ return queue->hdr_digest ? NVME_TCP_DIGEST_LENGTH : 0;
+}
+
+static inline u8 nvmet_tcp_ddgst_len(struct nvmet_tcp_queue *queue)
+{
+ return queue->data_digest ? NVME_TCP_DIGEST_LENGTH : 0;
+}
+
+static inline void nvmet_tcp_hdgst(struct ahash_request *hash,
+ void *pdu, size_t len)
+{
+ struct scatterlist sg;
+
+ sg_init_one(&sg, pdu, len);
+ ahash_request_set_crypt(hash, &sg, pdu + len, len);
+ crypto_ahash_digest(hash);
+}
+
+static int nvmet_tcp_verify_hdgst(struct nvmet_tcp_queue *queue,
+ void *pdu, size_t len)
+{
+ struct nvme_tcp_hdr *hdr = pdu;
+ __le32 recv_digest;
+ __le32 exp_digest;
+
+ if (unlikely(!(hdr->flags & NVME_TCP_F_HDGST))) {
+ pr_err("queue %d: header digest enabled but pdu without digest\n",
+ queue->idx);
+ return -EPROTO;
+ }
+
+ recv_digest = *(__le32 *)(pdu + hdr->hlen);
+ nvmet_tcp_hdgst(queue->rcv_hash, pdu, len);
+ exp_digest = *(__le32 *)(pdu + hdr->hlen);
+ if (recv_digest != exp_digest) {
+ pr_err("queue %d: header digest error: recv %#x expected %#x\n",
+ queue->idx, le32_to_cpu(recv_digest),
+ le32_to_cpu(exp_digest));
+ return -EPROTO;
+ }
+
+ return 0;
+}
+
+static int nvmet_tcp_check_ddgst(struct nvmet_tcp_queue *queue, void *pdu)
+{
+ struct nvme_tcp_hdr *hdr = pdu;
+ u32 len;
+
+ len = le32_to_cpu(hdr->plen) - hdr->hlen -
+ (hdr->flags & NVME_TCP_F_HDGST ? nvmet_tcp_hdgst_len(queue) : 0);
+
+ if (unlikely(len && !(hdr->flags & NVME_TCP_F_DDGST))) {
+ pr_err("queue %d: data digest flag is cleared\n", queue->idx);
+ return -EPROTO;
+ }
+
+ return 0;
+}
+
+static void nvmet_tcp_unmap_pdu_iovec(struct nvmet_tcp_cmd *cmd)
+{
+ struct scatterlist *sg;
+ int i;
+
+ sg = &cmd->req.sg[cmd->sg_idx];
+
+ for (i = 0; i < cmd->nr_mapped; i++)
+ kunmap(sg_page(&sg[i]));
+}
+
+static void nvmet_tcp_map_pdu_iovec(struct nvmet_tcp_cmd *cmd)
+{
+ struct kvec *iov = cmd->iov;
+ struct scatterlist *sg;
+ u32 length, offset, sg_offset;
+
+ length = cmd->pdu_len;
+ cmd->nr_mapped = DIV_ROUND_UP(length, PAGE_SIZE);
+ offset = cmd->rbytes_done;
+ cmd->sg_idx = DIV_ROUND_UP(offset, PAGE_SIZE);
+ sg_offset = offset % PAGE_SIZE;
+ sg = &cmd->req.sg[cmd->sg_idx];
+
+ while (length) {
+ u32 iov_len = min_t(u32, length, sg->length - sg_offset);
+
+ iov->iov_base = kmap(sg_page(sg)) + sg->offset + sg_offset;
+ iov->iov_len = iov_len;
+
+ length -= iov_len;
+ sg = sg_next(sg);
+ iov++;
+ }
+
+ iov_iter_kvec(&cmd->recv_msg.msg_iter, READ, cmd->iov,
+ cmd->nr_mapped, cmd->pdu_len);
+}
+
+static void nvmet_tcp_fatal_error(struct nvmet_tcp_queue *queue)
+{
+ queue->rcv.state = NVMET_TCP_RECV_ERR;
+ if (queue->nvme_sq.ctrl)
+ nvmet_ctrl_fatal_error(queue->nvme_sq.ctrl);
+ else
+ kernel_sock_shutdown(queue->sock, SHUT_RDWR);
+}
+
+static int nvmet_tcp_map_data(struct nvmet_tcp_cmd *cmd)
+{
+ struct nvme_sgl_desc *sgl = &cmd->req.cmd->common.dptr.sgl;
+ u32 len = le32_to_cpu(sgl->length);
+
+ if (!cmd->req.data_len)
+ return 0;
+
+ if (sgl->type == ((NVME_SGL_FMT_DATA_DESC << 4) |
+ NVME_SGL_FMT_OFFSET)) {
+ if (!nvme_is_write(cmd->req.cmd))
+ return NVME_SC_INVALID_FIELD | NVME_SC_DNR;
+
+ if (len > cmd->req.port->inline_data_size)
+ return NVME_SC_SGL_INVALID_OFFSET | NVME_SC_DNR;
+ cmd->pdu_len = len;
+ }
+ cmd->req.transfer_len += len;
+
+ cmd->req.sg = sgl_alloc(len, GFP_KERNEL, &cmd->req.sg_cnt);
+ if (!cmd->req.sg)
+ return NVME_SC_INTERNAL;
+ cmd->snd.cur_sg = cmd->req.sg;
+
+ if (nvmet_tcp_has_data_in(cmd)) {
+ cmd->iov = kmalloc_array(cmd->req.sg_cnt,
+ sizeof(*cmd->iov), GFP_KERNEL);
+ if (!cmd->iov)
+ goto err;
+ }
+
+ return 0;
+err:
+ sgl_free(cmd->req.sg);
+ return NVME_SC_INTERNAL;
+}
+
+static void nvmet_tcp_ddgst(struct ahash_request *hash,
+ struct nvmet_tcp_cmd *cmd)
+{
+ ahash_request_set_crypt(hash, cmd->req.sg,
+ (void *)&cmd->exp_ddgst, cmd->req.transfer_len);
+ crypto_ahash_digest(hash);
+}
+
+static void nvmet_setup_c2h_data_pdu(struct nvmet_tcp_cmd *cmd)
+{
+ struct nvme_tcp_data_pdu *pdu = cmd->data_pdu;
+ struct nvmet_tcp_queue *queue = cmd->queue;
+ u8 hdgst = nvmet_tcp_hdgst_len(cmd->queue);
+ u8 ddgst = nvmet_tcp_ddgst_len(cmd->queue);
+
+ cmd->snd.offset = 0;
+ cmd->snd.state = NVMET_TCP_SEND_DATA_PDU;
+
+ pdu->hdr.type = nvme_tcp_c2h_data;
+ pdu->hdr.flags = NVME_TCP_F_DATA_LAST;
+ pdu->hdr.hlen = sizeof(*pdu);
+ pdu->hdr.pdo = pdu->hdr.hlen + hdgst;
+ pdu->hdr.plen =
+ cpu_to_le32(pdu->hdr.hlen + hdgst + cmd->req.transfer_len + ddgst);
+ pdu->command_id = cmd->req.rsp->command_id;
+ pdu->data_length = cpu_to_le32(cmd->req.transfer_len);
+ pdu->data_offset = cpu_to_le32(cmd->wbytes_done);
+
+ if (queue->data_digest) {
+ pdu->hdr.flags |= NVME_TCP_F_DDGST;
+ nvmet_tcp_ddgst(queue->snd_hash, cmd);
+ }
+
+ if (cmd->queue->hdr_digest) {
+ pdu->hdr.flags |= NVME_TCP_F_HDGST;
+ nvmet_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
+ }
+}
+
+static void nvmet_setup_r2t_pdu(struct nvmet_tcp_cmd *cmd)
+{
+ struct nvme_tcp_r2t_pdu *pdu = cmd->r2t_pdu;
+ struct nvmet_tcp_queue *queue = cmd->queue;
+ u8 hdgst = nvmet_tcp_hdgst_len(cmd->queue);
+
+ cmd->snd.offset = 0;
+ cmd->snd.state = NVMET_TCP_SEND_R2T;
+
+ pdu->hdr.type = nvme_tcp_r2t;
+ pdu->hdr.flags = 0;
+ pdu->hdr.hlen = sizeof(*pdu);
+ pdu->hdr.pdo = 0;
+ pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst);
+
+ pdu->command_id = cmd->req.cmd->common.command_id;
+ pdu->ttag = nvmet_tcp_cmd_id(cmd->queue, cmd);
+ pdu->r2t_length = cpu_to_le32(cmd->req.transfer_len - cmd->rbytes_done);
+ pdu->r2t_offset = cpu_to_le32(cmd->rbytes_done);
+ if (cmd->queue->hdr_digest) {
+ pdu->hdr.flags |= NVME_TCP_F_HDGST;
+ nvmet_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
+ }
+}
+
+static void nvmet_setup_response_pdu(struct nvmet_tcp_cmd *cmd)
+{
+ struct nvme_tcp_rsp_pdu *pdu = cmd->rsp_pdu;
+ struct nvmet_tcp_queue *queue = cmd->queue;
+ u8 hdgst = nvmet_tcp_hdgst_len(cmd->queue);
+
+ cmd->snd.offset = 0;
+ cmd->snd.state = NVMET_TCP_SEND_RESPONSE;
+
+ pdu->hdr.type = nvme_tcp_rsp;
+ pdu->hdr.flags = 0;
+ pdu->hdr.hlen = sizeof(*pdu);
+ pdu->hdr.pdo = 0;
+ pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst);
+ if (cmd->queue->hdr_digest) {
+ pdu->hdr.flags |= NVME_TCP_F_HDGST;
+ nvmet_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
+ }
+}
+
+static struct nvmet_tcp_cmd *nvmet_tcp_reverse_list(struct nvmet_tcp_queue *queue, struct llist_node *node)
+{
+ struct nvmet_tcp_cmd *cmd;
+
+ while (node) {
+ struct nvmet_tcp_cmd *cmd = container_of(node, struct nvmet_tcp_cmd, lentry);
+
+ list_add(&cmd->entry, &queue->resp_send_list);
+ node = node->next;
+ queue->send_list_len++;
+ }
+
+ cmd = list_first_entry(&queue->resp_send_list, struct nvmet_tcp_cmd, entry);
+ return cmd;
+}
+
+static struct nvmet_tcp_cmd *nvmet_tcp_fetch_send_command(struct nvmet_tcp_queue *queue)
+{
+ struct llist_node *node;
+
+ queue->snd_cmd = list_first_entry_or_null(&queue->resp_send_list,
+ struct nvmet_tcp_cmd, entry);
+ if (!queue->snd_cmd) {
+ node = llist_del_all(&queue->resp_list);
+ if (!node)
+ return NULL;
+ queue->snd_cmd = nvmet_tcp_reverse_list(queue, node);
+ }
+
+ list_del_init(&queue->snd_cmd->entry);
+ queue->send_list_len--;
+
+ if (nvmet_tcp_need_data_out(queue->snd_cmd))
+ nvmet_setup_c2h_data_pdu(queue->snd_cmd);
+ else if (nvmet_tcp_need_data_in(queue->snd_cmd))
+ nvmet_setup_r2t_pdu(queue->snd_cmd);
+ else
+ nvmet_setup_response_pdu(queue->snd_cmd);
+
+ return queue->snd_cmd;
+}
+
+static void nvmet_tcp_queue_response(struct nvmet_req *req)
+{
+ struct nvmet_tcp_cmd *cmd =
+ container_of(req, struct nvmet_tcp_cmd, req);
+ struct nvmet_tcp_queue *queue = cmd->queue;
+
+ llist_add(&cmd->lentry, &queue->resp_list);
+ queue_work_on(cmd->queue->cpu, nvmet_tcp_wq, &cmd->queue->io_work);
+}
+
+static int nvmet_try_send_data_pdu(struct nvmet_tcp_cmd *cmd)
+{
+ struct nvmet_tcp_send_ctx *snd = &cmd->snd;
+ u8 hdgst = nvmet_tcp_hdgst_len(cmd->queue);
+ int left = sizeof(*cmd->data_pdu) - snd->offset + hdgst;
+ int ret;
+
+ ret = kernel_sendpage(cmd->queue->sock, virt_to_page(cmd->data_pdu),
+ offset_in_page(cmd->data_pdu) + snd->offset,
+ left, MSG_DONTWAIT | MSG_MORE);
+ if (ret <= 0)
+ return ret;
+
+ snd->offset += ret;
+ left -= ret;
+
+ if (left)
+ return -EAGAIN;
+
+ snd->state = NVMET_TCP_SEND_DATA;
+ snd->offset = 0;
+ return 1;
+}
+
+static int nvmet_try_send_data(struct nvmet_tcp_cmd *cmd)
+{
+ struct nvmet_tcp_send_ctx *snd = &cmd->snd;
+ struct nvmet_tcp_queue *queue = cmd->queue;
+ int ret;
+
+ while (snd->cur_sg) {
+ struct page *page = sg_page(snd->cur_sg);
+ u32 left = snd->cur_sg->length - snd->offset;
+
+ ret = kernel_sendpage(cmd->queue->sock, page, snd->offset,
+ left, MSG_DONTWAIT | MSG_MORE);
+ if (ret <= 0)
+ return ret;
+
+ snd->offset += ret;
+ cmd->wbytes_done += ret;
+
+ /* Done with sg?*/
+ if (snd->offset == snd->cur_sg->length) {
+ snd->cur_sg = sg_next(snd->cur_sg);
+ snd->offset = 0;
+ }
+ }
+
+ if (queue->data_digest) {
+ cmd->snd.state = NVMET_TCP_SEND_DDGST;
+ snd->offset = 0;
+ } else {
+ nvmet_setup_response_pdu(cmd);
+ }
+ return 1;
+
+}
+
+static int nvmet_try_send_response(struct nvmet_tcp_cmd *cmd, bool last_in_batch)
+{
+ struct nvmet_tcp_send_ctx *snd = &cmd->snd;
+ u8 hdgst = nvmet_tcp_hdgst_len(cmd->queue);
+ int left = sizeof(*cmd->rsp_pdu) - snd->offset + hdgst;
+ int flags = MSG_DONTWAIT;
+ int ret;
+
+ if (!last_in_batch && cmd->queue->send_list_len)
+ flags |= MSG_MORE;
+ else
+ flags |= MSG_EOR;
+
+ ret = kernel_sendpage(cmd->queue->sock, virt_to_page(cmd->rsp_pdu),
+ offset_in_page(cmd->rsp_pdu) + snd->offset, left, flags);
+ if (ret <= 0)
+ return ret;
+ snd->offset += ret;
+ left -= ret;
+
+ if (left)
+ return -EAGAIN;
+
+ kfree(cmd->iov);
+ sgl_free(cmd->req.sg);
+ cmd->queue->snd_cmd = NULL;
+ nvmet_tcp_put_cmd(cmd);
+ return 1;
+}
+
+static int nvmet_try_send_r2t(struct nvmet_tcp_cmd *cmd, bool last_in_batch)
+{
+ struct nvmet_tcp_send_ctx *snd = &cmd->snd;
+ u8 hdgst = nvmet_tcp_hdgst_len(cmd->queue);
+ int left = sizeof(*cmd->r2t_pdu) - snd->offset + hdgst;
+ int flags = MSG_DONTWAIT;
+ int ret;
+
+ if (!last_in_batch && cmd->queue->send_list_len)
+ flags |= MSG_MORE;
+ else
+ flags |= MSG_EOR;
+
+ ret = kernel_sendpage(cmd->queue->sock, virt_to_page(cmd->r2t_pdu),
+ offset_in_page(cmd->r2t_pdu) + snd->offset, left, flags);
+ if (ret <= 0)
+ return ret;
+ snd->offset += ret;
+ left -= ret;
+
+ if (left)
+ return -EAGAIN;
+
+ cmd->queue->snd_cmd = NULL;
+ return 1;
+}
+
+static int nvmet_try_send_ddgst(struct nvmet_tcp_cmd *cmd)
+{
+ struct nvmet_tcp_queue *queue = cmd->queue;
+ struct msghdr msg = { .msg_flags = MSG_DONTWAIT };
+ struct kvec iov = {
+ .iov_base = &cmd->exp_ddgst + cmd->snd.offset,
+ .iov_len = NVME_TCP_DIGEST_LENGTH - cmd->snd.offset
+ };
+ int ret;
+
+ ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len);
+ if (unlikely(ret <= 0))
+ return ret;
+
+ cmd->snd.offset += ret;
+ nvmet_setup_response_pdu(cmd);
+ return 1;
+}
+
+static int nvmet_tcp_try_send_one(struct nvmet_tcp_queue *queue,
+ bool last_in_batch)
+{
+ struct nvmet_tcp_cmd *cmd = queue->snd_cmd;
+ int ret = 0;
+
+ if (!cmd || queue->state == NVMET_TCP_Q_DISCONNECTING) {
+ cmd = nvmet_tcp_fetch_send_command(queue);
+ if (unlikely(!cmd))
+ return 0;
+ }
+
+ if (cmd->snd.state == NVMET_TCP_SEND_DATA_PDU) {
+ ret = nvmet_try_send_data_pdu(cmd);
+ if (ret <= 0)
+ goto done_send;
+ }
+
+ if (cmd->snd.state == NVMET_TCP_SEND_DATA) {
+ ret = nvmet_try_send_data(cmd);
+ if (ret <= 0)
+ goto done_send;
+ }
+
+ if (cmd->snd.state == NVMET_TCP_SEND_DDGST) {
+ ret = nvmet_try_send_ddgst(cmd);
+ if (ret <= 0)
+ goto done_send;
+ }
+
+ if (cmd->snd.state == NVMET_TCP_SEND_R2T) {
+ ret = nvmet_try_send_r2t(cmd, last_in_batch);
+ if (ret <= 0)
+ goto done_send;
+ }
+
+ if (cmd->snd.state == NVMET_TCP_SEND_RESPONSE)
+ ret = nvmet_try_send_response(cmd, last_in_batch);
+
+done_send:
+ if (ret < 0) {
+ if (ret == -EAGAIN)
+ return 0;
+ return ret;
+ }
+
+ return 1;
+}
+
+static int nvmet_tcp_try_send(struct nvmet_tcp_queue *queue,
+ int budget, int *sends)
+{
+ int i, ret = 0;
+
+ for (i = 0; i < budget; i++) {
+ ret = nvmet_tcp_try_send_one(queue, i == budget - 1);
+ if (ret <= 0)
+ break;
+ (*sends)++;
+ }
+
+ return ret;
+}
+
+static void nvmet_prepare_receive_pdu(struct nvmet_tcp_queue *queue)
+{
+ struct nvmet_tcp_recv_ctx *rcv = &queue->rcv;
+
+ rcv->offset = 0;
+ rcv->left = sizeof(struct nvme_tcp_hdr);
+ rcv->cmd = NULL;
+ rcv->state = NVMET_TCP_RECV_PDU;
+}
+
+static void nvmet_tcp_free_crypto(struct nvmet_tcp_queue *queue)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(queue->rcv_hash);
+
+ ahash_request_free(queue->rcv_hash);
+ ahash_request_free(queue->snd_hash);
+ crypto_free_ahash(tfm);
+}
+
+static int nvmet_tcp_alloc_crypto(struct nvmet_tcp_queue *queue)
+{
+ struct crypto_ahash *tfm;
+
+ tfm = crypto_alloc_ahash("crc32c", 0, CRYPTO_ALG_ASYNC);
+ if (IS_ERR(tfm))
+ return PTR_ERR(tfm);
+
+ queue->snd_hash = ahash_request_alloc(tfm, GFP_KERNEL);
+ if (!queue->snd_hash)
+ goto free_tfm;
+ ahash_request_set_callback(queue->snd_hash, 0, NULL, NULL);
+
+ queue->rcv_hash = ahash_request_alloc(tfm, GFP_KERNEL);
+ if (!queue->rcv_hash)
+ goto free_snd_hash;
+ ahash_request_set_callback(queue->rcv_hash, 0, NULL, NULL);
+
+ return 0;
+free_snd_hash:
+ ahash_request_free(queue->snd_hash);
+free_tfm:
+ crypto_free_ahash(tfm);
+ return -ENOMEM;
+}
+
+
+static int nvmet_tcp_handle_icreq(struct nvmet_tcp_queue *queue)
+{
+ struct nvme_tcp_icreq_pdu *icreq = &queue->rcv.pdu.icreq;
+ struct nvme_tcp_icresp_pdu *icresp = &queue->rcv.pdu.icresp;
+ struct msghdr msg = {};
+ struct kvec iov;
+ int ret;
+
+ if (le32_to_cpu(icreq->hdr.plen) != sizeof(struct nvme_tcp_icreq_pdu)) {
+ pr_err("bad nvme-tcp pdu length (%d)\n",
+ le32_to_cpu(icreq->hdr.plen));
+ nvmet_tcp_fatal_error(queue);
+ }
+
+ if (icreq->pfv != NVME_TCP_PFV_1_0) {
+ pr_err("queue %d: bad pfv %d\n", queue->idx, icreq->pfv);
+ return -EINVAL;
+ }
+
+ queue->hdr_digest = !!(icreq->digest & NVME_TCP_HDR_DIGEST_ENABLE);
+ queue->data_digest = !!(icreq->digest & NVME_TCP_DATA_DIGEST_ENABLE);
+ if (queue->hdr_digest || queue->data_digest) {
+ ret = nvmet_tcp_alloc_crypto(queue);
+ if (ret)
+ return ret;
+ }
+
+ if (icreq->hpda != 0) {
+ pr_err("queue %d: unsupported hpda %d\n", queue->idx,
+ icreq->hpda);
+ ret = -EPROTO;
+ goto free_crypto;
+ }
+
+ memset(icresp, 0, sizeof(*icresp));
+ icresp->hdr.type = nvme_tcp_icresp;
+ icresp->hdr.hlen = sizeof(*icresp);
+ icresp->hdr.pdo = 0;
+ icresp->hdr.plen = cpu_to_le32(icresp->hdr.hlen);
+ icresp->pfv = cpu_to_le16(NVME_TCP_PFV_1_0);
+ icresp->maxdata = 0xffff; /* FIXME: support r2t */
+ icresp->cpda = 0;
+ if (queue->hdr_digest)
+ icresp->digest |= NVME_TCP_HDR_DIGEST_ENABLE;
+ if (queue->data_digest)
+ icresp->digest |= NVME_TCP_DATA_DIGEST_ENABLE;
+
+ iov.iov_base = icresp;
+ iov.iov_len = sizeof(*icresp);
+ ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len);
+ if (ret < 0)
+ goto free_crypto;
+
+ queue->state = NVMET_TCP_Q_LIVE;
+ nvmet_prepare_receive_pdu(queue);
+ return 0;
+free_crypto:
+ if (queue->hdr_digest || queue->data_digest)
+ nvmet_tcp_free_crypto(queue);
+ return ret;
+}
+
+static void nvmet_tcp_handle_req_failure(struct nvmet_tcp_queue *queue,
+ struct nvmet_tcp_cmd *cmd, struct nvmet_req *req)
+{
+ int ret;
+
+ /* recover the expected data transfer length */
+ req->data_len = le32_to_cpu(req->cmd->common.dptr.sgl.length);
+
+ if (!nvme_is_write(cmd->req.cmd) ||
+ req->data_len > cmd->req.port->inline_data_size) {
+ nvmet_prepare_receive_pdu(queue);
+ return;
+ }
+
+ ret = nvmet_tcp_map_data(cmd);
+ if (unlikely(ret)) {
+ pr_err("queue %d: failed to map data\n", queue->idx);
+ nvmet_tcp_fatal_error(queue);
+ return;
+ }
+
+ queue->rcv.state = NVMET_TCP_RECV_DATA;
+ nvmet_tcp_map_pdu_iovec(cmd);
+ cmd->flags |= NVMET_TCP_F_INIT_FAILED;
+}
+
+static int nvmet_tcp_handle_h2c_data_pdu(struct nvmet_tcp_queue *queue)
+{
+ struct nvmet_tcp_recv_ctx *rcv = &queue->rcv;
+ struct nvme_tcp_data_pdu *data = &rcv->pdu.data;
+ struct nvmet_tcp_cmd *cmd;
+
+ cmd = &queue->cmds[data->ttag];
+
+ if (le32_to_cpu(data->data_offset) != cmd->rbytes_done) {
+ pr_err("queue %d ttag %u unexpected data offset %u (expected %u)\n",
+ queue->idx, data->ttag, le32_to_cpu(data->data_offset),
+ cmd->rbytes_done);
+ /* FIXME: use path and transport errors */
+ nvmet_req_complete(&cmd->req,
+ NVME_SC_INVALID_FIELD | NVME_SC_DNR);
+ return -EPROTO;
+ }
+
+ cmd->pdu_len = le32_to_cpu(data->data_length);
+ cmd->pdu_recv = 0;
+ nvmet_tcp_map_pdu_iovec(cmd);
+ rcv->cmd = cmd;
+ rcv->state = NVMET_TCP_RECV_DATA;
+
+ return 0;
+}
+
+static int nvmet_tcp_done_recv_pdu(struct nvmet_tcp_queue *queue)
+{
+ struct nvmet_tcp_recv_ctx *rcv = &queue->rcv;
+ struct nvme_tcp_hdr *hdr = &rcv->pdu.cmd.hdr;
+ struct nvme_command *nvme_cmd = &rcv->pdu.cmd.cmd;
+ struct nvmet_req *req;
+ int ret;
+
+ if (unlikely(queue->state == NVMET_TCP_Q_CONNECTING)) {
+ if (hdr->type != nvme_tcp_icreq) {
+ pr_err("unexpected pdu type (%d) before icreq\n",
+ hdr->type);
+ nvmet_tcp_fatal_error(queue);
+ return -EPROTO;
+ }
+ return nvmet_tcp_handle_icreq(queue);
+ }
+
+ if (hdr->type == nvme_tcp_h2c_data) {
+ ret = nvmet_tcp_handle_h2c_data_pdu(queue);
+ if(unlikely(ret))
+ return ret;
+ return 0;
+ }
+
+ rcv->cmd = nvmet_tcp_get_cmd(queue);
+ if (unlikely(!rcv->cmd)) {
+ /* This should never happen */
+ pr_err("queue %d: failed get command nr_cmds: %d, send_list_len: %d, opcode: %d",
+ queue->idx, queue->nr_cmds, queue->send_list_len, nvme_cmd->common.opcode);
+ nvmet_tcp_fatal_error(queue);
+ return -ENOMEM;
+ }
+
+ req = &rcv->cmd->req;
+ memcpy(req->cmd, nvme_cmd, sizeof(*nvme_cmd));
+
+ if (unlikely(!nvmet_req_init(req, &queue->nvme_cq,
+ &queue->nvme_sq, &nvmet_tcp_ops))) {
+ pr_err("failed cmd %p id %d opcode %d, data_len: %d\n",
+ req->cmd, req->cmd->common.command_id,
+ req->cmd->common.opcode,
+ le32_to_cpu(req->cmd->common.dptr.sgl.length));
+
+ nvmet_tcp_handle_req_failure(queue, rcv->cmd, req);
+ return -EAGAIN;
+ }
+
+ ret = nvmet_tcp_map_data(rcv->cmd);
+ if (unlikely(ret)) {
+ pr_err("queue %d: failed to map data\n", queue->idx);
+ if (nvmet_tcp_has_inline_data(rcv->cmd))
+ nvmet_tcp_fatal_error(queue);
+ else
+ nvmet_req_complete(req, ret);
+ ret = -EAGAIN;
+ goto out;
+ }
+
+ if (nvmet_tcp_need_data_in(rcv->cmd)) {
+ if (nvmet_tcp_has_inline_data(rcv->cmd)) {
+ rcv->state = NVMET_TCP_RECV_DATA;
+ nvmet_tcp_map_pdu_iovec(rcv->cmd);
+ return 0;
+ } else {
+ /* send back R2T */
+ nvmet_tcp_queue_response(&rcv->cmd->req);
+ goto out;
+ }
+ }
+
+ nvmet_req_execute(&rcv->cmd->req);
+out:
+ nvmet_prepare_receive_pdu(queue);
+ return ret;
+}
+
+static const u8 nvme_tcp_pdu_sizes[] = {
+ [nvme_tcp_icreq] = sizeof(struct nvme_tcp_icreq_pdu),
+ [nvme_tcp_cmd] = sizeof(struct nvme_tcp_cmd_pdu),
+ [nvme_tcp_h2c_data] = sizeof(struct nvme_tcp_data_pdu),
+};
+
+static inline u8 nvmet_tcp_pdu_size(u8 type)
+{
+ size_t idx = type;
+
+ return (idx < ARRAY_SIZE(nvme_tcp_pdu_sizes) && nvme_tcp_pdu_sizes[idx]) ?
+ nvme_tcp_pdu_sizes[idx] : 0;
+}
+
+static inline bool nvmet_tcp_pdu_valid(u8 type)
+{
+ switch (type) {
+ case nvme_tcp_icreq:
+ case nvme_tcp_cmd:
+ case nvme_tcp_h2c_data:
+ /* fallthru */
+ return true;
+ }
+
+ return false;
+}
+
+static int nvmet_tcp_try_recv_pdu(struct nvmet_tcp_queue *queue)
+{
+ struct nvmet_tcp_recv_ctx *rcv = &queue->rcv;
+ struct nvme_tcp_hdr *hdr = &rcv->pdu.cmd.hdr;
+ int len;
+ struct kvec iov;
+ struct msghdr msg = { .msg_flags = MSG_DONTWAIT };
+
+recv:
+ iov.iov_base = (void *)&rcv->pdu + rcv->offset;
+ iov.iov_len = rcv->left;
+ len = kernel_recvmsg(queue->sock, &msg, &iov, 1,
+ iov.iov_len, msg.msg_flags);
+ if (unlikely(len < 0))
+ return len;
+
+ rcv->offset += len;
+ rcv->left -= len;
+ if (rcv->left) {
+ return -EAGAIN;
+ } else if (rcv->offset == sizeof(struct nvme_tcp_hdr)) {
+ u8 hdgst = nvmet_tcp_hdgst_len(queue);
+
+ if (unlikely(!nvmet_tcp_pdu_valid(hdr->type))) {
+ pr_err("unexpected pdu type %d\n", hdr->type);
+ nvmet_tcp_fatal_error(queue);
+ return -EIO;
+ }
+
+ if (unlikely(hdr->hlen != nvmet_tcp_pdu_size(hdr->type))) {
+ pr_err("pdu type %d bad hlen %d\n", hdr->type, hdr->hlen);
+ return -EIO;
+ }
+
+ rcv->left = hdr->hlen - rcv->offset + hdgst;
+ goto recv;
+ }
+
+ if (queue->hdr_digest &&
+ nvmet_tcp_verify_hdgst(queue, &rcv->pdu, rcv->offset)) {
+ nvmet_tcp_fatal_error(queue); /* fatal */
+ return -EPROTO;
+ }
+
+ if (queue->data_digest &&
+ nvmet_tcp_check_ddgst(queue, &rcv->pdu)) {
+ nvmet_tcp_fatal_error(queue); /* fatal */
+ return -EPROTO;
+ }
+
+ return nvmet_tcp_done_recv_pdu(queue);
+}
+
+static void nvmet_tcp_prep_recv_ddgst(struct nvmet_tcp_cmd *cmd)
+{
+ struct nvmet_tcp_queue *queue = cmd->queue;
+
+ nvmet_tcp_ddgst(queue->rcv_hash, cmd);
+ queue->rcv.offset = 0;
+ queue->rcv.left = NVME_TCP_DIGEST_LENGTH;
+ queue->rcv.state = NVMET_TCP_RECV_DDGST;
+}
+
+static int nvmet_tcp_try_recv_data(struct nvmet_tcp_queue *queue)
+{
+ struct nvmet_tcp_cmd *cmd = queue->rcv.cmd;
+ int ret;
+
+ while (msg_data_left(&cmd->recv_msg)) {
+ ret = sock_recvmsg(cmd->queue->sock, &cmd->recv_msg,
+ cmd->recv_msg.msg_flags);
+ if (ret <= 0)
+ return ret;
+
+ cmd->pdu_recv += ret;
+ cmd->rbytes_done += ret;
+ }
+
+ nvmet_tcp_unmap_pdu_iovec(cmd);
+
+ if (!(cmd->flags & NVMET_TCP_F_INIT_FAILED) &&
+ cmd->rbytes_done == cmd->req.transfer_len) {
+ if (queue->data_digest) {
+ nvmet_tcp_prep_recv_ddgst(cmd);
+ return 0;
+ } else {
+ nvmet_req_execute(&cmd->req);
+ }
+ }
+
+ nvmet_prepare_receive_pdu(queue);
+ return 0;
+}
+
+static int nvmet_tcp_try_recv_ddgst(struct nvmet_tcp_queue *queue)
+{
+ struct nvmet_tcp_recv_ctx *rcv = &queue->rcv;
+ struct nvmet_tcp_cmd *cmd = rcv->cmd;
+ int ret;
+ struct msghdr msg = { .msg_flags = MSG_DONTWAIT };
+ struct kvec iov = {
+ .iov_base = (void *)&cmd->recv_ddgst + rcv->offset,
+ .iov_len = rcv->left
+ };
+
+ ret = kernel_recvmsg(queue->sock, &msg, &iov, 1,
+ iov.iov_len, msg.msg_flags);
+ if (unlikely(ret < 0))
+ return ret;
+
+ rcv->offset += ret;
+ rcv->left -= ret;
+ if (rcv->left)
+ return -EAGAIN;
+
+ if (queue->data_digest && cmd->exp_ddgst != cmd->recv_ddgst) {
+ pr_err("queue %d: cmd %d pdu (%d) data digest error: recv %#x expected %#x\n",
+ queue->idx, cmd->req.cmd->common.command_id, rcv->pdu.cmd.hdr.type,
+ le32_to_cpu(cmd->recv_ddgst), le32_to_cpu(cmd->exp_ddgst));
+ nvmet_tcp_finish_cmd(cmd);
+ nvmet_tcp_fatal_error(queue);
+ ret = -EPROTO;
+ goto out;
+ }
+
+ if (!(cmd->flags & NVMET_TCP_F_INIT_FAILED) &&
+ cmd->rbytes_done == cmd->req.transfer_len)
+ nvmet_req_execute(&cmd->req);
+ ret = 0;
+out:
+ nvmet_prepare_receive_pdu(queue);
+ return ret;
+}
+
+static int nvmet_tcp_try_recv_one(struct nvmet_tcp_queue *queue)
+{
+ struct nvmet_tcp_recv_ctx *rcv = &queue->rcv;
+ int result;
+
+ if (unlikely(rcv->state == NVMET_TCP_RECV_ERR))
+ return 0;
+
+ if (rcv->state == NVMET_TCP_RECV_PDU) {
+ result = nvmet_tcp_try_recv_pdu(queue);
+ if (result != 0)
+ goto done_recv;
+ }
+
+ if (rcv->state == NVMET_TCP_RECV_DATA) {
+ result = nvmet_tcp_try_recv_data(queue);
+ if (result != 0)
+ goto done_recv;
+ }
+
+ if (rcv->state == NVMET_TCP_RECV_DDGST) {
+ result = nvmet_tcp_try_recv_ddgst(queue);
+ if (result != 0)
+ goto done_recv;
+ }
+
+done_recv:
+ if (result < 0) {
+ if (result == -EAGAIN)
+ return 0;
+ return result;
+ }
+ return 1;
+}
+
+static int nvmet_tcp_try_recv(struct nvmet_tcp_queue *queue,
+ int budget, int *recvs)
+{
+ int i, ret = 0;
+
+ for (i = 0; i < budget; i++) {
+ ret = nvmet_tcp_try_recv_one(queue);
+ if (ret <= 0)
+ break;
+ (*recvs)++;
+ }
+
+ return ret;
+}
+
+static void nvmet_tcp_schedule_release_queue(struct nvmet_tcp_queue *queue)
+{
+ spin_lock(&queue->state_lock);
+ if (queue->state == NVMET_TCP_Q_DISCONNECTING)
+ goto out;
+
+ queue->state = NVMET_TCP_Q_DISCONNECTING;
+ schedule_work(&queue->release_work);
+out:
+ spin_unlock(&queue->state_lock);
+}
+
+static void nvmet_tcp_io_work(struct work_struct *w)
+{
+ struct nvmet_tcp_queue *queue =
+ container_of(w, struct nvmet_tcp_queue, io_work);
+ bool pending;
+ int ret, ops = 0;
+
+ do {
+ pending = false;
+
+ ret = nvmet_tcp_try_recv(queue, NVMET_TCP_RECV_BUDGET, &ops);
+ if (ret > 0) {
+ pending = true;
+ } else if (ret < 0) {
+ if (ret == -EPIPE || ret == -ECONNRESET)
+ kernel_sock_shutdown(queue->sock, SHUT_RDWR);
+ else
+ nvmet_tcp_fatal_error(queue);
+ return;
+ }
+
+ ret = nvmet_tcp_try_send(queue, NVMET_TCP_SEND_BUDGET, &ops);
+ if (ret > 0) {
+ /* transmitted message/data */
+ pending = true;
+ } else if (ret < 0) {
+ if (ret == -EPIPE || ret == -ECONNRESET)
+ kernel_sock_shutdown(queue->sock, SHUT_RDWR);
+ else
+ nvmet_tcp_fatal_error(queue);
+ return;
+ }
+
+ } while (pending && ops < NVMET_TCP_IO_WORK_BUDGET);
+
+ /*
+ * We exahusted our budget, requeue our selves
+ */
+ if (pending)
+ queue_work_on(queue->cpu, nvmet_tcp_wq, &queue->io_work);
+}
+
+static int nvmet_tcp_alloc_cmd(struct nvmet_tcp_queue *queue,
+ struct nvmet_tcp_cmd *c)
+{
+ u8 hdgst = nvmet_tcp_hdgst_len(queue);
+
+ c->queue = queue;
+ c->req.port = queue->port->nport;
+
+ c->cmd_pdu = page_frag_alloc(&queue->pf_cache,
+ sizeof(*c->cmd_pdu) + hdgst, GFP_KERNEL | __GFP_ZERO);
+ if (!c->cmd_pdu)
+ return -ENOMEM;
+ c->req.cmd = &c->cmd_pdu->cmd;
+
+ c->rsp_pdu = page_frag_alloc(&queue->pf_cache,
+ sizeof(*c->rsp_pdu) + hdgst, GFP_KERNEL | __GFP_ZERO);
+ if (!c->rsp_pdu)
+ goto out_free_cmd;
+ c->req.rsp = &c->rsp_pdu->cqe;
+
+ c->data_pdu = page_frag_alloc(&queue->pf_cache,
+ sizeof(*c->data_pdu) + hdgst, GFP_KERNEL | __GFP_ZERO);
+ if (!c->data_pdu)
+ goto out_free_rsp;
+
+ c->r2t_pdu = page_frag_alloc(&queue->pf_cache,
+ sizeof(*c->r2t_pdu) + hdgst, GFP_KERNEL | __GFP_ZERO);
+ if (!c->r2t_pdu)
+ goto out_free_data;
+
+ c->recv_msg.msg_flags = MSG_DONTWAIT | MSG_NOSIGNAL;
+
+ list_add_tail(&c->entry, &queue->free_list);
+
+ return 0;
+out_free_data:
+ page_frag_free(c->data_pdu);
+out_free_rsp:
+ page_frag_free(c->rsp_pdu);
+out_free_cmd:
+ page_frag_free(c->cmd_pdu);
+ return -ENOMEM;
+}
+
+static void nvmet_tcp_free_cmd(struct nvmet_tcp_cmd *c)
+{
+ page_frag_free(c->r2t_pdu);
+ page_frag_free(c->data_pdu);
+ page_frag_free(c->rsp_pdu);
+ page_frag_free(c->cmd_pdu);
+}
+
+static int nvmet_tcp_alloc_cmds(struct nvmet_tcp_queue *queue)
+{
+ struct nvmet_tcp_cmd *cmds;
+ int i, ret = -EINVAL, nr_cmds = queue->nr_cmds;
+
+ cmds = kcalloc(nr_cmds, sizeof(struct nvmet_tcp_cmd), GFP_KERNEL);
+ if (!cmds)
+ goto out;
+
+ for (i = 0; i < nr_cmds; i++) {
+ ret = nvmet_tcp_alloc_cmd(queue, cmds + i);
+ if (ret)
+ goto out_free;
+ }
+
+ queue->cmds = cmds;
+
+ return 0;
+out_free:
+ while (--i >= 0)
+ nvmet_tcp_free_cmd(cmds + i);
+ kfree(cmds);
+out:
+ return ret;
+}
+
+static void nvmet_tcp_free_cmds(struct nvmet_tcp_queue *queue)
+{
+ struct nvmet_tcp_cmd *cmds = queue->cmds;
+ int i;
+
+ for (i = 0; i < queue->nr_cmds; i++)
+ nvmet_tcp_free_cmd(cmds + i);
+
+ nvmet_tcp_free_cmd(&queue->connect);
+ kfree(cmds);
+}
+
+static void nvmet_tcp_restore_socket_callbacks(struct nvmet_tcp_queue *queue)
+{
+ struct socket *sock = queue->sock;
+
+ write_lock_bh(&sock->sk->sk_callback_lock);
+ sock->sk->sk_data_ready = queue->old_data_ready;
+ sock->sk->sk_state_change = queue->old_state_change;
+ sock->sk->sk_write_space = queue->old_write_space;
+ sock->sk->sk_user_data = NULL;
+ write_unlock_bh(&sock->sk->sk_callback_lock);
+}
+
+static void nvmet_tcp_finish_cmd(struct nvmet_tcp_cmd *cmd)
+{
+ nvmet_req_uninit(&cmd->req);
+ nvmet_tcp_unmap_pdu_iovec(cmd);
+ sgl_free(cmd->req.sg);
+}
+
+static void nvmet_tcp_uninit_data_in_cmds(struct nvmet_tcp_queue *queue)
+{
+ struct nvmet_tcp_cmd *cmd = queue->cmds;
+ int i;
+
+ for (i = 0; i < queue->nr_cmds; i++, cmd++) {
+ if (nvmet_tcp_need_data_in(cmd))
+ nvmet_tcp_finish_cmd(cmd);
+ }
+
+ if (!queue->nr_cmds && nvmet_tcp_need_data_in(&queue->connect)) {
+ /* failed in connect */
+ nvmet_tcp_finish_cmd(&queue->connect);
+ }
+}
+
+static void nvmet_tcp_release_queue_work(struct work_struct *w)
+{
+ struct nvmet_tcp_queue *queue =
+ container_of(w, struct nvmet_tcp_queue, release_work);
+
+ mutex_lock(&nvmet_tcp_queue_mutex);
+ list_del_init(&queue->queue_list);
+ mutex_unlock(&nvmet_tcp_queue_mutex);
+
+ nvmet_tcp_restore_socket_callbacks(queue);
+ flush_work(&queue->io_work);
+
+ nvmet_tcp_uninit_data_in_cmds(queue);
+ nvmet_sq_destroy(&queue->nvme_sq);
+ cancel_work_sync(&queue->io_work);
+ sock_release(queue->sock);
+ nvmet_tcp_free_cmds(queue);
+ if (queue->hdr_digest || queue->data_digest)
+ nvmet_tcp_free_crypto(queue);
+ ida_simple_remove(&nvmet_tcp_queue_ida, queue->idx);
+
+ kfree(queue);
+}
+
+static void nvmet_tcp_data_ready(struct sock *sk)
+{
+ struct nvmet_tcp_queue *queue;
+
+ read_lock_bh(&sk->sk_callback_lock);
+ queue = sk->sk_user_data;
+ if (!queue)
+ goto out;
+
+ queue_work_on(queue->cpu, nvmet_tcp_wq, &queue->io_work);
+out:
+ read_unlock_bh(&sk->sk_callback_lock);
+}
+
+static void nvmet_tcp_write_space(struct sock *sk)
+{
+ struct nvmet_tcp_queue *queue;
+
+ read_lock_bh(&sk->sk_callback_lock);
+ queue = sk->sk_user_data;
+ if (!queue)
+ goto out;
+
+ if (unlikely(queue->state == NVMET_TCP_Q_CONNECTING)) {
+ queue->old_write_space(sk);
+ goto out;
+ }
+
+ if (sk_stream_is_writeable(sk)) {
+ clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
+ queue_work_on(queue->cpu, nvmet_tcp_wq, &queue->io_work);
+ }
+out:
+ read_unlock_bh(&sk->sk_callback_lock);
+}
+
+static void nvmet_tcp_state_change(struct sock *sk)
+{
+ struct nvmet_tcp_queue *queue;
+
+ write_lock_bh(&sk->sk_callback_lock);
+ queue = sk->sk_user_data;
+ if (!queue)
+ goto done;
+
+ switch (sk->sk_state) {
+ case TCP_FIN_WAIT1:
+ case TCP_CLOSE_WAIT:
+ case TCP_CLOSE:
+ /* FALLTHRU */
+ sk->sk_user_data = NULL;
+ nvmet_tcp_schedule_release_queue(queue);
+ break;
+ default:
+ pr_warn("queue %d unhandled state %d\n", queue->idx, sk->sk_state);
+ }
+done:
+ write_unlock_bh(&sk->sk_callback_lock);
+}
+
+static int nvmet_tcp_set_queue_sock(struct nvmet_tcp_queue *queue)
+{
+ struct socket *sock = queue->sock;
+ struct linger sol = { .l_onoff = 1, .l_linger = 0 };
+ int ret;
+
+ ret = kernel_getsockname(sock,
+ (struct sockaddr *)&queue->sockaddr);
+ if (ret < 0)
+ return ret;
+
+ ret = kernel_getpeername(sock,
+ (struct sockaddr *)&queue->sockaddr_peer);
+ if (ret < 0)
+ return ret;
+
+ /*
+ * Cleanup whatever is sitting in the TCP transmit queue on socket
+ * close. This is done to prevent stale data from being sent should
+ * the network connection be restored before TCP times out.
+ */
+ ret = kernel_setsockopt(sock, SOL_SOCKET, SO_LINGER,
+ (char *)&sol, sizeof(sol));
+ if (ret)
+ return ret;
+
+ write_lock_bh(&sock->sk->sk_callback_lock);
+ sock->sk->sk_user_data = queue;
+ queue->old_data_ready = sock->sk->sk_data_ready;
+ sock->sk->sk_data_ready = nvmet_tcp_data_ready;
+ queue->old_state_change = sock->sk->sk_state_change;
+ sock->sk->sk_state_change = nvmet_tcp_state_change;
+ queue->old_write_space = sock->sk->sk_write_space;
+ sock->sk->sk_write_space = nvmet_tcp_write_space;
+ write_unlock_bh(&sock->sk->sk_callback_lock);
+
+ return 0;
+}
+
+static int nvmet_tcp_alloc_queue(struct nvmet_tcp_port *port,
+ struct socket *newsock)
+{
+ struct nvmet_tcp_queue *queue;
+ int ret;
+
+ queue = kzalloc(sizeof(*queue), GFP_KERNEL);
+ if (!queue)
+ return -ENOMEM;
+
+ INIT_WORK(&queue->release_work, nvmet_tcp_release_queue_work);
+ INIT_WORK(&queue->io_work, nvmet_tcp_io_work);
+ queue->sock = newsock;
+ queue->port = port;
+ queue->nr_cmds = 0;
+ spin_lock_init(&queue->state_lock);
+ queue->state = NVMET_TCP_Q_CONNECTING;
+ INIT_LIST_HEAD(&queue->free_list);
+ init_llist_head(&queue->resp_list);
+ INIT_LIST_HEAD(&queue->resp_send_list);
+
+ queue->idx = ida_simple_get(&nvmet_tcp_queue_ida, 0, 0, GFP_KERNEL);
+ if (queue->idx < 0) {
+ ret = queue->idx;
+ goto out_free_queue;
+ }
+
+ ret = nvmet_tcp_alloc_cmd(queue, &queue->connect);
+ if (ret)
+ goto out_ida_remove;
+
+ ret = nvmet_sq_init(&queue->nvme_sq);
+ if (ret)
+ goto out_ida_remove;
+
+ port->last_cpu = cpumask_next_wrap(port->last_cpu,
+ cpu_online_mask, -1, false);
+ queue->cpu = port->last_cpu;
+ nvmet_prepare_receive_pdu(queue);
+
+ mutex_lock(&nvmet_tcp_queue_mutex);
+ list_add_tail(&queue->queue_list, &nvmet_tcp_queue_list);
+ mutex_unlock(&nvmet_tcp_queue_mutex);
+
+ ret = nvmet_tcp_set_queue_sock(queue);
+ if (ret)
+ goto out_destroy_sq;
+
+ queue_work_on(queue->cpu, nvmet_tcp_wq, &queue->io_work);
+
+ return 0;
+out_destroy_sq:
+ mutex_lock(&nvmet_tcp_queue_mutex);
+ list_del_init(&queue->queue_list);
+ mutex_unlock(&nvmet_tcp_queue_mutex);
+ nvmet_sq_destroy(&queue->nvme_sq);
+out_ida_remove:
+ ida_simple_remove(&nvmet_tcp_queue_ida, queue->idx);
+out_free_queue:
+ kfree(queue);
+ return ret;
+}
+
+static void nvmet_tcp_accept_work(struct work_struct *w)
+{
+ struct nvmet_tcp_port *port =
+ container_of(w, struct nvmet_tcp_port, accept_work);
+ struct socket *newsock;
+ int ret;
+
+ while (true) {
+ ret = kernel_accept(port->sock, &newsock, O_NONBLOCK);
+ if (ret < 0) {
+ if (ret != -EAGAIN)
+ pr_warn("failed to accept err=%d\n", ret);
+ return;
+ }
+ ret = nvmet_tcp_alloc_queue(port, newsock);
+ if (ret) {
+ pr_err("failed to allocate queue\n");
+ sock_release(newsock);
+ }
+ }
+}
+
+static void nvmet_tcp_listen_data_ready(struct sock *sk)
+{
+ struct nvmet_tcp_port *port;
+
+ read_lock_bh(&sk->sk_callback_lock);
+ port = sk->sk_user_data;
+ if (!port)
+ goto out;
+
+ if (sk->sk_state == TCP_LISTEN)
+ schedule_work(&port->accept_work);
+out:
+ read_unlock_bh(&sk->sk_callback_lock);
+}
+
+static int nvmet_tcp_add_port(struct nvmet_port *nport)
+{
+ struct nvmet_tcp_port *port;
+ __kernel_sa_family_t af;
+ int opt, ret;
+
+ port = kzalloc(sizeof(*port), GFP_KERNEL);
+ if (!port)
+ return -ENOMEM;
+
+ switch (nport->disc_addr.adrfam) {
+ case NVMF_ADDR_FAMILY_IP4:
+ af = AF_INET;
+ break;
+ case NVMF_ADDR_FAMILY_IP6:
+ af = AF_INET6;
+ break;
+ default:
+ pr_err("address family %d not supported\n",
+ nport->disc_addr.adrfam);
+ ret = -EINVAL;
+ goto err_port;
+ }
+
+ ret = inet_pton_with_scope(&init_net, af, nport->disc_addr.traddr,
+ nport->disc_addr.trsvcid, &port->addr);
+ if (ret) {
+ pr_err("malformed ip/port passed: %s:%s\n",
+ nport->disc_addr.traddr, nport->disc_addr.trsvcid);
+ goto err_port;
+ }
+
+ port->nport = nport;
+ port->last_cpu = -1;
+ INIT_WORK(&port->accept_work, nvmet_tcp_accept_work);
+ if (port->nport->inline_data_size < 0)
+ port->nport->inline_data_size = NVMET_TCP_DEF_INLINE_DATA_SIZE;
+
+ ret = sock_create(port->addr.ss_family, SOCK_STREAM,
+ IPPROTO_TCP, &port->sock);
+ if (ret) {
+ pr_err("failed to create a socket\n");
+ goto err_port;
+ }
+
+ port->sock->sk->sk_user_data = port;
+ port->old_data_ready = port->sock->sk->sk_data_ready;
+ port->sock->sk->sk_data_ready = nvmet_tcp_listen_data_ready;
+
+ opt = 1;
+ ret = kernel_setsockopt(port->sock, IPPROTO_TCP,
+ TCP_NODELAY, (char *)&opt, sizeof(opt));
+ if (ret) {
+ pr_err("failed to set TCP_NODELAY sock opt %d\n", ret);
+ goto err_sock;
+ }
+
+ ret = kernel_setsockopt(port->sock, SOL_SOCKET, SO_REUSEADDR,
+ (char *)&opt, sizeof(opt));
+ if (ret) {
+ pr_err("failed to set SO_REUSEADDR sock opt %d\n", ret);
+ goto err_sock;
+ }
+
+ ret = kernel_bind(port->sock, (struct sockaddr *)&port->addr,
+ sizeof(port->addr));
+ if (ret) {
+ pr_err("failed to bind port socket %d\n", ret);
+ goto err_sock;
+ }
+
+ ret = kernel_listen(port->sock, 128);
+ if (ret) {
+ pr_err("failed to listen %d on port sock\n", ret);
+ goto err_sock;
+ }
+
+ nport->priv = port;
+ pr_info("enabling port %d (%pISpc)\n",
+ le16_to_cpu(nport->disc_addr.portid), &port->addr);
+
+ return 0;
+
+err_sock:
+ sock_release(port->sock);
+err_port:
+ kfree(port);
+ return ret;
+}
+
+static void nvmet_tcp_remove_port(struct nvmet_port *nport)
+{
+ struct nvmet_tcp_port *port = nport->priv;
+
+ write_lock_bh(&port->sock->sk->sk_callback_lock);
+ port->sock->sk->sk_data_ready = port->old_data_ready;
+ port->sock->sk->sk_user_data = NULL;
+ write_unlock_bh(&port->sock->sk->sk_callback_lock);
+ cancel_work_sync(&port->accept_work);
+
+ sock_release(port->sock);
+ kfree(port);
+}
+
+static void nvmet_tcp_delete_ctrl(struct nvmet_ctrl *ctrl)
+{
+ struct nvmet_tcp_queue *queue;
+
+ mutex_lock(&nvmet_tcp_queue_mutex);
+ list_for_each_entry(queue, &nvmet_tcp_queue_list, queue_list)
+ if (queue->nvme_sq.ctrl == ctrl)
+ kernel_sock_shutdown(queue->sock, SHUT_RDWR);
+ mutex_unlock(&nvmet_tcp_queue_mutex);
+}
+
+static u16 nvmet_tcp_install_queue(struct nvmet_sq *sq)
+{
+ struct nvmet_tcp_queue *queue =
+ container_of(sq, struct nvmet_tcp_queue, nvme_sq);
+ int ret;
+
+ if (sq->qid == 0) {
+ /* Let inflight controller teardown complete */
+ flush_scheduled_work();
+ }
+
+ queue->nr_cmds = sq->size * 2;
+ if (nvmet_tcp_alloc_cmds(queue))
+ return NVME_SC_INTERNAL;
+ return 0;
+}
+
+static void nvmet_tcp_disc_port_addr(struct nvmet_req *req,
+ struct nvmet_port *nport, char *traddr)
+{
+ struct nvmet_tcp_port *port = nport->priv;
+
+ if (inet_addr_is_any((struct sockaddr *)&port->addr)) {
+ struct nvmet_tcp_cmd *cmd =
+ container_of(req, struct nvmet_tcp_cmd, req);
+ struct nvmet_tcp_queue *queue = cmd->queue;
+
+ sprintf(traddr, "%pISc", (struct sockaddr *)&queue->sockaddr);
+ } else {
+ memcpy(traddr, nport->disc_addr.traddr, NVMF_TRADDR_SIZE);
+ }
+}
+
+static struct nvmet_fabrics_ops nvmet_tcp_ops = {
+ .owner = THIS_MODULE,
+ .type = NVMF_TRTYPE_TCP,
+ .msdbd = 1,
+ .has_keyed_sgls = 0,
+ .add_port = nvmet_tcp_add_port,
+ .remove_port = nvmet_tcp_remove_port,
+ .queue_response = nvmet_tcp_queue_response,
+ .delete_ctrl = nvmet_tcp_delete_ctrl,
+ .install_queue = nvmet_tcp_install_queue,
+ .disc_traddr = nvmet_tcp_disc_port_addr,
+};
+
+static int __init nvmet_tcp_init(void)
+{
+ int ret;
+
+ nvmet_tcp_wq = alloc_workqueue("nvmet_tcp_wq", WQ_HIGHPRI, 0);
+ if (!nvmet_tcp_wq)
+ return -ENOMEM;
+
+ ret = nvmet_register_transport(&nvmet_tcp_ops);
+ if (ret)
+ goto err;
+
+ return 0;
+err:
+ destroy_workqueue(nvmet_tcp_wq);
+ return ret;
+}
+
+static void __exit nvmet_tcp_exit(void)
+{
+ struct nvmet_tcp_queue *queue;
+
+ nvmet_unregister_transport(&nvmet_tcp_ops);
+
+ flush_scheduled_work();
+ mutex_lock(&nvmet_tcp_queue_mutex);
+ list_for_each_entry(queue, &nvmet_tcp_queue_list, queue_list)
+ kernel_sock_shutdown(queue->sock, SHUT_RDWR);
+ mutex_unlock(&nvmet_tcp_queue_mutex);
+ flush_scheduled_work();
+
+ destroy_workqueue(nvmet_tcp_wq);
+}
+
+module_init(nvmet_tcp_init);
+module_exit(nvmet_tcp_exit);
+
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("nvmet-transport-3"); /* 3 == NVMF_TRTYPE_TCP */
diff --git a/include/linux/nvme-tcp.h b/include/linux/nvme-tcp.h
index 33c8afaf63bd..685780d1ed04 100644
--- a/include/linux/nvme-tcp.h
+++ b/include/linux/nvme-tcp.h
@@ -11,6 +11,7 @@
#define NVME_TCP_DISC_PORT 8009
#define NVME_TCP_ADMIN_CCSZ SZ_8K
+#define NVME_TCP_DIGEST_LENGTH 4
enum nvme_tcp_pfv {
NVME_TCP_PFV_1_0 = 0x0,
--
2.17.1
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox