* [PATCH iproute2 net-next] iplink: vxlan: gpe support
2016-02-11 10:35 [PATCH net-next 0/2] NSH and VxLAN-GPE Brian Russell
@ 2016-02-11 10:35 ` Brian Russell
2016-02-11 10:35 ` [PATCH net-next 1/2] nsh: encapsulation module Brian Russell
2016-02-11 10:35 ` [PATCH net-next 2/2] vxlan: support GPE/NSH Brian Russell
2 siblings, 0 replies; 8+ messages in thread
From: Brian Russell @ 2016-02-11 10:35 UTC (permalink / raw)
To: netdev
Add VXLAN GPE support to ip link
Signed-off-by: Brian Russell <brussell@brocade.com>
---
include/linux/if_link.h | 1 +
ip/iplink_vxlan.c | 21 ++++++++++++++++++---
man/man8/ip-link.8.in | 12 +++++++++++-
3 files changed, 30 insertions(+), 4 deletions(-)
diff --git a/include/linux/if_link.h b/include/linux/if_link.h
index 0331c72..759ea6b 100644
--- a/include/linux/if_link.h
+++ b/include/linux/if_link.h
@@ -454,6 +454,7 @@ enum {
IFLA_VXLAN_GBP,
IFLA_VXLAN_REMCSUM_NOPARTIAL,
IFLA_VXLAN_COLLECT_METADATA,
+ IFLA_VXLAN_GPE,
__IFLA_VXLAN_MAX
};
#define IFLA_VXLAN_MAX (__IFLA_VXLAN_MAX - 1)
diff --git a/ip/iplink_vxlan.c b/ip/iplink_vxlan.c
index ede8482..df749fa 100644
--- a/ip/iplink_vxlan.c
+++ b/ip/iplink_vxlan.c
@@ -31,7 +31,7 @@ static void print_explain(FILE *f)
fprintf(f, " [ ageing SECONDS ] [ maxaddress NUMBER ]\n");
fprintf(f, " [ [no]udpcsum ] [ [no]udp6zerocsumtx ] [ [no]udp6zerocsumrx ]\n");
fprintf(f, " [ [no]remcsumtx ] [ [no]remcsumrx ]\n");
- fprintf(f, " [ [no]external ] [ gbp ]\n");
+ fprintf(f, " [ [no]external ] [ gbp | gpe ]\n");
fprintf(f, "\n");
fprintf(f, "Where: VNI := 0-16777215\n");
fprintf(f, " ADDR := { IP_ADDRESS | any }\n");
@@ -74,6 +74,7 @@ static int vxlan_parse_opt(struct link_util *lu, int argc, char **argv,
__u8 remcsumrx = 0;
__u8 metadata = 0;
__u8 gbp = 0;
+ __u8 gpe = 0;
int dst_port_set = 0;
struct ifla_vxlan_port_range range = { 0, 0 };
@@ -217,6 +218,8 @@ static int vxlan_parse_opt(struct link_util *lu, int argc, char **argv,
metadata = 0;
} else if (!matches(*argv, "gbp")) {
gbp = 1;
+ } else if (!matches(*argv, "gpe")) {
+ gpe = 1;
} else if (matches(*argv, "help") == 0) {
explain();
return -1;
@@ -249,8 +252,9 @@ static int vxlan_parse_opt(struct link_util *lu, int argc, char **argv,
fprintf(stderr, "vxlan: destination port not specified\n"
"Will use Linux kernel default (non-standard value)\n");
fprintf(stderr,
- "Use 'dstport 4789' to get the IANA assigned value\n"
- "Use 'dstport 0' to get default and quiet this message\n");
+ "Use 'dstport %s' to get the IANA assigned value\n"
+ "Use 'dstport 0' to get default and quiet this message\n",
+ gpe ? "4790" : "4789");
}
addattr32(n, 1024, IFLA_VXLAN_ID, vni);
@@ -296,9 +300,17 @@ static int vxlan_parse_opt(struct link_util *lu, int argc, char **argv,
if (dstport)
addattr16(n, 1024, IFLA_VXLAN_PORT, htons(dstport));
+ if (gbp && gpe) {
+ fprintf(stderr,
+ "vxlan: both gbp and gpe cannot be specified\n");
+ return -1;
+ }
+
if (gbp)
addattr_l(n, 1024, IFLA_VXLAN_GBP, NULL, 0);
+ if (gpe)
+ addattr_l(n, 1024, IFLA_VXLAN_GPE, NULL, 0);
return 0;
}
@@ -445,6 +457,9 @@ static void vxlan_print_opt(struct link_util *lu, FILE *f, struct rtattr *tb[])
if (tb[IFLA_VXLAN_GBP])
fputs("gbp ", f);
+
+ if (tb[IFLA_VXLAN_GPE])
+ fputs("gpe ", f);
}
static void vxlan_print_help(struct link_util *lu, int argc, char **argv,
diff --git a/man/man8/ip-link.8.in b/man/man8/ip-link.8.in
index 4d32343..2b92e81 100644
--- a/man/man8/ip-link.8.in
+++ b/man/man8/ip-link.8.in
@@ -422,7 +422,7 @@ the following additional arguments are supported:
] [
.BI maxaddress " NUMBER "
] [
-.B gbp
+.BR gbp " | " gpe
]
.in +8
@@ -554,6 +554,16 @@ Example:
.in -4
+.sp
+.B gpe
+- enables the Generic Protocol extension (VXLAN-GPE). This parameter cannot
+be specified with the
+.B gbp
+parameter. Sending packets with encapsulations other than ethernet requires
+that the
+.B remote
+parameter be specified.
+
.in -8
.TP
--
2.1.4
^ permalink raw reply related [flat|nested] 8+ messages in thread* [PATCH net-next 1/2] nsh: encapsulation module
2016-02-11 10:35 [PATCH net-next 0/2] NSH and VxLAN-GPE Brian Russell
2016-02-11 10:35 ` [PATCH iproute2 net-next] iplink: vxlan: gpe support Brian Russell
@ 2016-02-11 10:35 ` Brian Russell
2016-02-11 16:17 ` Robert Shearman
2016-02-11 16:39 ` David Miller
2016-02-11 10:35 ` [PATCH net-next 2/2] vxlan: support GPE/NSH Brian Russell
2 siblings, 2 replies; 8+ messages in thread
From: Brian Russell @ 2016-02-11 10:35 UTC (permalink / raw)
To: netdev
Support encap/decap of Network Service Header (NSH) as defined in
https://tools.ietf.org/html/draft-ietf-sfc-nsh-01
Includes support for Type 1 and Type 2 metadata and a simple registration
for listeners to see decapsulated packets based on the Type/Class.
Signed-off-by: Brian Russell <brussell@brocade.com>
---
include/net/nsh.h | 158 ++++++++++++++++++
include/uapi/linux/if_ether.h | 1 +
net/ipv4/Kconfig | 10 ++
net/ipv4/Makefile | 1 +
net/ipv4/nsh.c | 362 ++++++++++++++++++++++++++++++++++++++++++
5 files changed, 532 insertions(+)
create mode 100644 include/net/nsh.h
create mode 100644 net/ipv4/nsh.c
diff --git a/include/net/nsh.h b/include/net/nsh.h
new file mode 100644
index 0000000..7a5fb95
--- /dev/null
+++ b/include/net/nsh.h
@@ -0,0 +1,158 @@
+/*
+ * Network Service Header (NSH) inserted onto encapsulated packets
+ * or frames to realize service function paths.
+ * NSH also provides a mechanism for metadata exchange along the
+ * instantiated service path.
+ *
+ * https://tools.ietf.org/html/draft-ietf-sfc-nsh-01
+ *
+ * Copyright (c) 2015 by Brocade Communications Systems, Inc.
+ * All rights reserved.
+ */
+#ifndef __NET_NSH_H
+#define __NET_NSH_H
+
+#include <linux/types.h>
+#include <linux/skbuff.h>
+
+/*
+ * NSH Base Header + Service Path Header
+ *
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * |Ver|O|C|R|R|R|R|R|R| Length | MD Type | Next Protocol |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | Service Path ID | Service Index |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ *
+ * Ver - Version, set to 0
+ * O - Indicates payload is OAM.
+ * C - Indicates critical metadata TLV is present (must be 0 for MD type 1).
+ * Length - total header length in 4-byte words.
+ * MD Type - Metadata type
+ * Type 1 - 4 mandatory 4 byte context headers.
+ * Type 2 - 0 or more var length context headers.
+ * Next Protocol - protocol type of original packet.
+ * Service Path ID (SPI) - identifies a service path. Participating nodes
+ * MUST use this identifier for Service Function
+ * Path selection.
+ * Service Index (SI) - provides location within the SFP.
+ */
+#define NSH_BF_VER0 0
+#define NSH_BF_VER_MASK 0xc0
+#define NSH_BF_OAM BIT(5)
+#define NSH_BF_CRIT BIT(4)
+#define NSH_N_SPI (1u << 24)
+#define NSH_SPI_MASK ((NSH_N_SPI-1) << 8)
+#define NSH_N_SI (1u << 8)
+#define NSH_SI_MASK (NSH_N_SI-1)
+
+#define NSH_MD_TYPE_1 1
+#define NSH_MD_TYPE_2 2
+
+#define NSH_NEXT_PROTO_IPv4 1
+#define NSH_NEXT_PROTO_IPv6 2
+#define NSH_NEXT_PROTO_ETH 3
+
+#define NSH_LEN_TYPE_1 6
+#define NSH_LEN_TYPE_2_MIN 2
+
+struct nsh_base {
+ __u8 base_flags;
+ __u8 length;
+ __u8 md_type;
+ __u8 next_proto;
+};
+
+struct nsh_header {
+ struct nsh_base base;
+ __be32 sp_header;
+};
+
+/*
+ * When the Base Header specifies MD Type 1, four 4-byte Context Headers
+ * MUST be added immediately following the Service Path Header. Thus length
+ * in the base header is set to 6.
+ * Context Headers that carry no metadata MUST be set to zero.
+ */
+#define NSH_MD_TYPE_1_NUM_HDRS 4
+
+struct nsh_md_type_1 {
+ __be32 ctx_hdr1;
+ __be32 ctx_hdr2;
+ __be32 ctx_hdr3;
+ __be32 ctx_hdr4;
+};
+
+/*
+ * When the Base Header specifies MD Type 2, zero or more variable
+ * length Context Headers follow the Service Path Header.
+ *
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | TLV Class |C| Type |R|R|R| Len |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | Variable Metadata |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ *
+ * TLV Class - Scope of class (e.g. may be vendor or standards body).
+ * Type - Specific type of information within the scope of given class.
+ * C bit (MSB) indicates criticality. When set, receiver must process.
+ * Len - Length of variable metadata in 4-byte words.
+ */
+#define NSH_TYPE_CRIT BIT(7)
+
+struct nsh_md_type_2 {
+ __be16 tlv_class;
+ __u8 tlv_type;
+ __u8 length;
+};
+
+/*
+ * Context header for encap/decap.
+ */
+#define NSH_MD_CLASS_TYPE_1 USHRT_MAX
+#define NSH_MD_TYPE_TYPE_1 U8_MAX
+#define NSH_MD_LEN_TYPE_1 4
+
+struct nsh_metadata {
+ u_short class;
+ u_char crit;
+ u_char type;
+ u_int len; /* 4 byte words */
+ void *data;
+};
+
+/*
+ * Parse NSH header and notify registered listeners about any metadata.
+ */
+int nsh_decap(struct sk_buff *skb,
+ u_int *spi,
+ u_char *si,
+ u_char *np);
+
+/*
+ * Add NSH header.
+ */
+int nsh_encap(struct sk_buff *skb,
+ u_int spi,
+ u_char si,
+ u_char np,
+ u_int num_ctx_hdrs,
+ struct nsh_metadata *ctx_hdrs);
+
+
+/* Register hooks to be informed of nsh metadata of specified class */
+struct nsh_listener {
+ struct list_head list;
+ u_short class;
+ u_char max_ctx_hdrs;
+ int (*notify)(struct sk_buff *skb,
+ u_int service_path_id,
+ u_char service_index,
+ u_char next_proto,
+ struct nsh_metadata *ctx_hdrs,
+ u_int num_ctx_hdrs);
+};
+
+int nsh_register_listener(struct nsh_listener *listener);
+int nsh_unregister_listener(struct nsh_listener *listener);
+#endif /* __NET_NSH_H */
diff --git a/include/uapi/linux/if_ether.h b/include/uapi/linux/if_ether.h
index ea9221b..eb512b1 100644
--- a/include/uapi/linux/if_ether.h
+++ b/include/uapi/linux/if_ether.h
@@ -91,6 +91,7 @@
#define ETH_P_TDLS 0x890D /* TDLS */
#define ETH_P_FIP 0x8914 /* FCoE Initialization Protocol */
#define ETH_P_80221 0x8917 /* IEEE 802.21 Media Independent Handover Protocol */
+#define ETH_P_NSH 0x894F /* Network Service Header */
#define ETH_P_LOOPBACK 0x9000 /* Ethernet loopback packet, per IEEE 802.3 */
#define ETH_P_QINQ1 0x9100 /* deprecated QinQ VLAN [ NOT AN OFFICIALLY REGISTERED ID ] */
#define ETH_P_QINQ2 0x9200 /* deprecated QinQ VLAN [ NOT AN OFFICIALLY REGISTERED ID ] */
diff --git a/net/ipv4/Kconfig b/net/ipv4/Kconfig
index 7758247..37c8c23 100644
--- a/net/ipv4/Kconfig
+++ b/net/ipv4/Kconfig
@@ -212,6 +212,16 @@ config NET_IPGRE_BROADCAST
Network), but can be distributed all over the Internet. If you want
to do that, say Y here and to "IP multicast routing" below.
+config NET_NSH
+ tristate 'Network Servive Header Encapsulation'
+ help
+ Network Service Header (NSH) inserted onto
+ encapsulated packets or frames to realize service function paths.
+ NSH also provides a mechanism for metadata exchange along the
+ instantiated service path.
+
+ To compile it as a module, choose M here. If unsure, say N.
+
config IP_MROUTE
bool "IP: multicast routing"
depends on IP_MULTICAST
diff --git a/net/ipv4/Makefile b/net/ipv4/Makefile
index 62c049b..46d65f8 100644
--- a/net/ipv4/Makefile
+++ b/net/ipv4/Makefile
@@ -24,6 +24,7 @@ gre-y := gre_demux.o
obj-$(CONFIG_NET_FOU) += fou.o
obj-$(CONFIG_NET_IPGRE_DEMUX) += gre.o
obj-$(CONFIG_NET_IPGRE) += ip_gre.o
+obj-$(CONFIG_NET_NSH) += nsh.o
obj-$(CONFIG_NET_UDP_TUNNEL) += udp_tunnel.o
obj-$(CONFIG_NET_IPVTI) += ip_vti.o
obj-$(CONFIG_SYN_COOKIES) += syncookies.o
diff --git a/net/ipv4/nsh.c b/net/ipv4/nsh.c
new file mode 100644
index 0000000..70e5ef0
--- /dev/null
+++ b/net/ipv4/nsh.c
@@ -0,0 +1,362 @@
+/*
+ * Network Service Header (NSH) inserted onto encapsulated packets
+ * or frames to realize service function paths.
+ * NSH also provides a mechanism for metadata exchange along the
+ * instantiated service path.
+ *
+ * https://tools.ietf.org/html/draft-ietf-sfc-nsh-01
+ *
+ * Copyright (c) 2015 by Brocade Communications Systems, Inc.
+ * All rights reserved.
+ */
+#include <linux/module.h>
+#include <net/nsh.h>
+
+static struct list_head nsh_listeners;
+static DEFINE_MUTEX(nsh_listener_mutex);
+static struct nsh_metadata *decap_ctx_hdrs;
+static u_char limit_ctx_hdrs = 10;
+module_param_named(nsh_hdrs, limit_ctx_hdrs, byte, 0444);
+MODULE_PARM_DESC(nsh_hdrs, "Maximum NSH metadata headers per packet");
+
+int nsh_register_listener(struct nsh_listener *listener)
+{
+ if (listener->max_ctx_hdrs > limit_ctx_hdrs)
+ return -ENOMEM;
+
+ mutex_lock(&nsh_listener_mutex);
+ list_add(&listener->list, &nsh_listeners);
+ mutex_unlock(&nsh_listener_mutex);
+ return 0;
+}
+EXPORT_SYMBOL(nsh_register_listener);
+
+int nsh_unregister_listener(struct nsh_listener *listener)
+{
+ mutex_lock(&nsh_listener_mutex);
+ list_del(&listener->list);
+ mutex_unlock(&nsh_listener_mutex);
+ return 0;
+}
+EXPORT_SYMBOL(nsh_unregister_listener);
+
+static int
+notify_listeners(struct sk_buff *skb,
+ u_int service_path_id,
+ u_char service_index,
+ u_char next_proto,
+ struct nsh_metadata *ctx_hdrs,
+ u_int num_ctx_hdrs)
+{
+ struct nsh_listener *listener;
+ int i, err = 0;
+
+ mutex_lock(&nsh_listener_mutex);
+ list_for_each_entry(listener, &nsh_listeners, list) {
+ for (i = 0; i < num_ctx_hdrs; i++)
+ if (listener->class == ctx_hdrs[i].class) {
+ err = listener->notify(skb,
+ service_path_id,
+ service_index,
+ next_proto,
+ ctx_hdrs,
+ num_ctx_hdrs);
+ if (err < 0) {
+ mutex_unlock(&nsh_listener_mutex);
+ return err;
+ }
+ break;
+ }
+ }
+ mutex_unlock(&nsh_listener_mutex);
+ return 0;
+}
+
+static int
+type_1_decap(struct sk_buff *skb,
+ struct nsh_md_type_1 *md,
+ u_int max_ctx_hdrs,
+ struct nsh_metadata *ctx_hdrs,
+ u_int *num_ctx_hdrs)
+{
+ int i;
+ u32 *data = &md->ctx_hdr1;
+
+ if (max_ctx_hdrs == 0)
+ return -ENOMEM;
+
+ ctx_hdrs[0].class = NSH_MD_CLASS_TYPE_1;
+ ctx_hdrs[0].type = NSH_MD_TYPE_TYPE_1;
+ ctx_hdrs[0].len = NSH_MD_LEN_TYPE_1;
+ ctx_hdrs[0].data = data;
+
+ for (i = 0; i < NSH_MD_TYPE_1_NUM_HDRS; i++, data++)
+ *data = ntohl(*data);
+
+ *num_ctx_hdrs = 1;
+
+ return 0;
+}
+
+static int
+type_2_decap(struct sk_buff *skb,
+ struct nsh_md_type_2 *md,
+ u_int md_len,
+ u_int max_ctx_hdrs,
+ struct nsh_metadata *ctx_hdrs,
+ u_int *num_ctx_hdrs)
+{
+ u32 *data;
+ int i = 0, j;
+
+ while (md_len > 0) {
+ if (i > max_ctx_hdrs)
+ return -ENOMEM;
+
+ ctx_hdrs[i].class = ntohs(md->tlv_class);
+ ctx_hdrs[i].type = md->tlv_type;
+ if (ctx_hdrs[i].type & NSH_TYPE_CRIT) {
+ ctx_hdrs[i].type &= ~NSH_TYPE_CRIT;
+ ctx_hdrs[i].crit = 1;
+ }
+ ctx_hdrs[i].len = md->length;
+
+ data = (u32 *) ++md;
+ md_len--;
+
+ ctx_hdrs[i].data = data;
+
+ for (j = 0; j < ctx_hdrs[i].len; j++)
+ data[j] = ntohl(data[j]);
+
+ md = (struct nsh_md_type_2 *)&data[j];
+ md_len -= j;
+ i++;
+ }
+ *num_ctx_hdrs = i;
+
+ return 0;
+}
+
+/* Parse NSH header.
+ *
+ * No additional memory is allocated. Context header data is pointed
+ * to in the buffer payload. Context headers and skb are passed to anyone
+ * who has registered interest in the class(es) of metadata received.
+ *
+ * Returns the total number of 4 byte words in the NSH headers, <0 on failure.
+ */
+int nsh_decap(struct sk_buff *skb,
+ u_int *spi,
+ u_char *si,
+ u_char *np)
+{
+ struct nsh_header *nsh = (struct nsh_header *)skb->data;
+ struct nsh_base *base = &nsh->base;
+ u_int max_ctx_hdrs = limit_ctx_hdrs;
+ u_int num_ctx_hdrs;
+ u_int service_path_id;
+ u_char service_index;
+ u_char next_proto;
+ u32 sph;
+ u_char md_type;
+ u_char hdrlen; /* 4 byte words */
+ u_int len; /* bytes */
+ int err;
+
+ hdrlen = base->length;
+ len = hdrlen * sizeof(u32);
+
+ if (unlikely(!pskb_may_pull(skb, len)))
+ return -ENOMEM;
+
+ skb_pull_rcsum(skb, len);
+
+ if (((base->base_flags & NSH_BF_VER_MASK) >> 6) != NSH_BF_VER0)
+ return -EINVAL;
+
+ next_proto = base->next_proto;
+
+ switch (next_proto) {
+ case NSH_NEXT_PROTO_IPv4:
+ skb->protocol = htons(ETH_P_IP);
+ break;
+ case NSH_NEXT_PROTO_IPv6:
+ skb->protocol = htons(ETH_P_IPV6);
+ break;
+ case NSH_NEXT_PROTO_ETH:
+ skb->protocol = htons(ETH_P_TEB);
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ if (np)
+ *np = next_proto;
+
+ md_type = base->md_type;
+
+ switch (md_type) {
+ case NSH_MD_TYPE_1:
+ if (hdrlen != NSH_LEN_TYPE_1)
+ return -EINVAL;
+ err = type_1_decap(skb, (struct nsh_md_type_1 *) ++nsh,
+ max_ctx_hdrs, decap_ctx_hdrs, &num_ctx_hdrs);
+ break;
+ case NSH_MD_TYPE_2:
+ if (hdrlen < NSH_LEN_TYPE_2_MIN)
+ return -EINVAL;
+ err = type_2_decap(skb, (struct nsh_md_type_2 *) ++nsh,
+ hdrlen - NSH_LEN_TYPE_2_MIN,
+ max_ctx_hdrs, decap_ctx_hdrs, &num_ctx_hdrs);
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ if (err < 0)
+ return err;
+
+ sph = ntohl(nsh->sp_header);
+ service_path_id = (sph & NSH_SPI_MASK) >> 8;
+ service_index = sph & NSH_SI_MASK;
+
+ if (spi)
+ *spi = service_path_id;
+ if (si)
+ *si = service_index;
+
+ err = notify_listeners(skb, service_path_id,
+ service_index, next_proto,
+ decap_ctx_hdrs, num_ctx_hdrs);
+ if (err < 0)
+ return err;
+
+ return hdrlen;
+}
+EXPORT_SYMBOL_GPL(nsh_decap);
+
+static void
+type_1_encap(u32 *data_out,
+ struct nsh_metadata *ctx_hdrs)
+{
+ int i;
+ u32 *data_in = (u32 *)ctx_hdrs[0].data;
+
+ for (i = 0; i < NSH_MD_TYPE_1_NUM_HDRS; i++)
+ data_out[i] = htonl(data_in[i]);
+}
+
+static void
+type_2_encap(struct nsh_md_type_2 *md,
+ u_int num_ctx_hdrs,
+ struct nsh_metadata *ctx_hdrs)
+{
+ int i, j;
+ u32 *data_in, *data_out;
+
+ for (i = 0; i < num_ctx_hdrs; i++) {
+ md->tlv_class = htons(ctx_hdrs[i].class);
+ md->tlv_type = ctx_hdrs[i].type;
+ if (ctx_hdrs[i].crit)
+ md->tlv_type |= NSH_TYPE_CRIT;
+ md->length = ctx_hdrs[i].len;
+
+ data_out = (u32 *) ++md;
+ data_in = (u32 *)ctx_hdrs[i].data;
+
+ for (j = 0; j < ctx_hdrs[i].len; j++)
+ data_out[j] = htonl(data_in[j]);
+
+ md = (struct nsh_md_type_2 *)&data_out[j];
+ }
+}
+
+/* Add NSH header.
+ */
+int nsh_encap(struct sk_buff *skb,
+ u_int spi,
+ u_char si,
+ u_char np,
+ u_int num_ctx_hdrs,
+ struct nsh_metadata *ctx_hdrs)
+{
+ bool has_t1 = false, has_t2 = false;
+ bool has_crit = false;
+ u_int headroom = sizeof(struct nsh_header);
+ struct nsh_header *nsh;
+ struct nsh_base *base;
+ int i;
+ int err;
+
+ if (np != NSH_NEXT_PROTO_IPv4 &&
+ np != NSH_NEXT_PROTO_IPv6 &&
+ np != NSH_NEXT_PROTO_ETH)
+ return -EINVAL;
+
+ if (spi >= NSH_N_SPI)
+ return -EINVAL;
+
+ for (i = 0; i < num_ctx_hdrs; i++) {
+ if (ctx_hdrs[i].class == NSH_MD_CLASS_TYPE_1) {
+ if (num_ctx_hdrs != 1)
+ return -EINVAL;
+ headroom += NSH_MD_LEN_TYPE_1 * sizeof(u32);
+ has_t1 |= true;
+ } else {
+ headroom += ctx_hdrs[i].len * sizeof(u32) +
+ sizeof(struct nsh_md_type_2);
+ has_t2 |= true;
+ has_crit |= ctx_hdrs[i].type & NSH_TYPE_CRIT;
+ }
+
+ if (has_t1 && has_t2)
+ return -EINVAL;
+ }
+
+ err = skb_cow_head(skb, headroom);
+ if (err)
+ return err;
+
+ nsh = (struct nsh_header *)__skb_push(skb, headroom);
+
+ base = &nsh->base;
+ base->base_flags = has_crit ? NSH_BF_CRIT : 0; /* Ver 0, OAM 0 */
+ base->length = headroom / sizeof(u32);
+ base->md_type = has_t1 ? NSH_MD_TYPE_1 : NSH_MD_TYPE_2;
+ base->next_proto = np;
+
+ nsh->sp_header = htonl((spi << 8) | si);
+
+ if (has_t1)
+ type_1_encap((u32 *) ++nsh, ctx_hdrs);
+ else
+ type_2_encap((struct nsh_md_type_2 *) ++nsh, num_ctx_hdrs,
+ ctx_hdrs);
+ return 0;
+}
+EXPORT_SYMBOL_GPL(nsh_encap);
+
+static int __init nsh_init(void)
+{
+ INIT_LIST_HEAD(&nsh_listeners);
+
+ decap_ctx_hdrs = kmalloc_array(limit_ctx_hdrs, sizeof(*decap_ctx_hdrs),
+ GFP_KERNEL);
+ if (!decap_ctx_hdrs)
+ return -ENOMEM;
+
+ return 0;
+}
+
+static void __exit nsh_exit(void)
+{
+ kfree(decap_ctx_hdrs);
+}
+
+module_init(nsh_init);
+module_exit(nsh_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Brian Russell <brussell@brocade.com>");
+MODULE_DESCRIPTION("Network Service Header Encap/Decap");
--
2.1.4
^ permalink raw reply related [flat|nested] 8+ messages in thread* Re: [PATCH net-next 1/2] nsh: encapsulation module
2016-02-11 10:35 ` [PATCH net-next 1/2] nsh: encapsulation module Brian Russell
@ 2016-02-11 16:17 ` Robert Shearman
2016-02-11 16:39 ` Brian Russell
2016-02-11 16:39 ` David Miller
1 sibling, 1 reply; 8+ messages in thread
From: Robert Shearman @ 2016-02-11 16:17 UTC (permalink / raw)
To: Brian Russell, netdev
On 11/02/16 11:35, Brian Russell wrote:
...
> diff --git a/include/net/nsh.h b/include/net/nsh.h
> new file mode 100644
> index 0000000..7a5fb95
> --- /dev/null
> +++ b/include/net/nsh.h
> @@ -0,0 +1,158 @@
> +/*
> + * Network Service Header (NSH) inserted onto encapsulated packets
> + * or frames to realize service function paths.
> + * NSH also provides a mechanism for metadata exchange along the
> + * instantiated service path.
> + *
> + * https://tools.ietf.org/html/draft-ietf-sfc-nsh-01
> + *
> + * Copyright (c) 2015 by Brocade Communications Systems, Inc.
> + * All rights reserved.
> + */
All rights reserved isn't really compatible with the GPL :-)
...
> diff --git a/net/ipv4/nsh.c b/net/ipv4/nsh.c
> new file mode 100644
> index 0000000..70e5ef0
> --- /dev/null
> +++ b/net/ipv4/nsh.c
> @@ -0,0 +1,362 @@
> +/*
> + * Network Service Header (NSH) inserted onto encapsulated packets
> + * or frames to realize service function paths.
> + * NSH also provides a mechanism for metadata exchange along the
> + * instantiated service path.
> + *
> + * https://tools.ietf.org/html/draft-ietf-sfc-nsh-01
> + *
> + * Copyright (c) 2015 by Brocade Communications Systems, Inc.
> + * All rights reserved.
> + */
Ditto.
Thanks,
Rob
^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [PATCH net-next 1/2] nsh: encapsulation module
2016-02-11 16:17 ` Robert Shearman
@ 2016-02-11 16:39 ` Brian Russell
0 siblings, 0 replies; 8+ messages in thread
From: Brian Russell @ 2016-02-11 16:39 UTC (permalink / raw)
To: Robert Shearman, netdev
On 11/02/16 16:17, Robert Shearman wrote:
> On 11/02/16 11:35, Brian Russell wrote:
> ...
>> diff --git a/include/net/nsh.h b/include/net/nsh.h
>> new file mode 100644
>> index 0000000..7a5fb95
>> --- /dev/null
>> +++ b/include/net/nsh.h
>> @@ -0,0 +1,158 @@
>> +/*
>> + * Network Service Header (NSH) inserted onto encapsulated packets
>> + * or frames to realize service function paths.
>> + * NSH also provides a mechanism for metadata exchange along the
>> + * instantiated service path.
>> + *
>> + * https://tools.ietf.org/html/draft-ietf-sfc-nsh-01
>> + *
>> + * Copyright (c) 2015 by Brocade Communications Systems, Inc.
>> + * All rights reserved.
>> + */
>
> All rights reserved isn't really compatible with the GPL :-)
>
Oops. Will fix that up in next version.
Thanks,
Brian
> ...
>> diff --git a/net/ipv4/nsh.c b/net/ipv4/nsh.c
>> new file mode 100644
>> index 0000000..70e5ef0
>> --- /dev/null
>> +++ b/net/ipv4/nsh.c
>> @@ -0,0 +1,362 @@
>> +/*
>> + * Network Service Header (NSH) inserted onto encapsulated packets
>> + * or frames to realize service function paths.
>> + * NSH also provides a mechanism for metadata exchange along the
>> + * instantiated service path.
>> + *
>> + * https://tools.ietf.org/html/draft-ietf-sfc-nsh-01
>> + *
>> + * Copyright (c) 2015 by Brocade Communications Systems, Inc.
>> + * All rights reserved.
>> + */
>
> Ditto.
>
> Thanks,
> Rob
>
^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [PATCH net-next 1/2] nsh: encapsulation module
2016-02-11 10:35 ` [PATCH net-next 1/2] nsh: encapsulation module Brian Russell
2016-02-11 16:17 ` Robert Shearman
@ 2016-02-11 16:39 ` David Miller
2016-02-11 19:54 ` Brian Russell
1 sibling, 1 reply; 8+ messages in thread
From: David Miller @ 2016-02-11 16:39 UTC (permalink / raw)
To: brussell; +Cc: netdev
From: Brian Russell <brussell@brocade.com>
Date: Thu, 11 Feb 2016 10:35:13 +0000
> +++ b/include/net/nsh.h
> +struct nsh_metadata {
> + u_short class;
> + u_char crit;
> + u_char type;
> + u_int len; /* 4 byte words */
> + void *data;
Do not use these shorthands, spell out "unsigned short" etc. explicitly.
But in these cases you should be using fixed sized types like "u16".
Also, in a header file such as this one which is private to the kernel
you should be using the variants of these types without the leading underscores.
^ permalink raw reply [flat|nested] 8+ messages in thread* Re: [PATCH net-next 1/2] nsh: encapsulation module
2016-02-11 16:39 ` David Miller
@ 2016-02-11 19:54 ` Brian Russell
0 siblings, 0 replies; 8+ messages in thread
From: Brian Russell @ 2016-02-11 19:54 UTC (permalink / raw)
To: David Miller; +Cc: netdev
On 11/02/16 16:39, David Miller wrote:
> From: Brian Russell <brussell@brocade.com>
> Date: Thu, 11 Feb 2016 10:35:13 +0000
>
>> +++ b/include/net/nsh.h
>> +struct nsh_metadata {
>> + u_short class;
>> + u_char crit;
>> + u_char type;
>> + u_int len; /* 4 byte words */
>> + void *data;
>
> Do not use these shorthands, spell out "unsigned short" etc. explicitly.
>
> But in these cases you should be using fixed sized types like "u16".
>
> Also, in a header file such as this one which is private to the kernel
> you should be using the variants of these types without the leading underscores.
>
Ok, will fix those up.
Thanks,
Brian
^ permalink raw reply [flat|nested] 8+ messages in thread
* [PATCH net-next 2/2] vxlan: support GPE/NSH
2016-02-11 10:35 [PATCH net-next 0/2] NSH and VxLAN-GPE Brian Russell
2016-02-11 10:35 ` [PATCH iproute2 net-next] iplink: vxlan: gpe support Brian Russell
2016-02-11 10:35 ` [PATCH net-next 1/2] nsh: encapsulation module Brian Russell
@ 2016-02-11 10:35 ` Brian Russell
2 siblings, 0 replies; 8+ messages in thread
From: Brian Russell @ 2016-02-11 10:35 UTC (permalink / raw)
To: netdev
Support the Generic Protocol Extension to VxLAN which extends VxLAN to
allow multi-protocol encapsulation. IPv4, IPv6, MPLS unicast and
NSH encapsulated packets can be sent and received in addition to ethernet
frames. As defined in:
https://tools.ietf.org/html/draft-ietf-nvo3-vxlan-gpe-01
Signed-off-by: Brian Russell <brussell@brocade.com>
---
drivers/net/vxlan.c | 139 +++++++++++++++++++++++++++++++++++++++----
include/net/vxlan.h | 40 ++++++++++++-
include/uapi/linux/if_link.h | 1 +
3 files changed, 166 insertions(+), 14 deletions(-)
diff --git a/drivers/net/vxlan.c b/drivers/net/vxlan.c
index 65f5247..92a4cdc 100644
--- a/drivers/net/vxlan.c
+++ b/drivers/net/vxlan.c
@@ -50,6 +50,7 @@
#include <net/ip6_checksum.h>
#endif
#include <net/dst_metadata.h>
+#include <net/nsh.h>
#define VXLAN_VERSION "0.1"
@@ -1168,14 +1169,7 @@ static void vxlan_rcv(struct vxlan_sock *vs, struct sk_buff *skb,
if (!vxlan)
goto drop;
- skb_reset_mac_header(skb);
skb_scrub_packet(skb, !net_eq(vxlan->net, dev_net(vxlan->dev)));
- skb->protocol = eth_type_trans(skb, vxlan->dev);
- skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
-
- /* Ignore packet loops (and multicast echo) */
- if (ether_addr_equal(eth_hdr(skb)->h_source, vxlan->dev->dev_addr))
- goto drop;
/* Get data from the outer IP header */
if (vxlan_get_sk_family(vs) == AF_INET) {
@@ -1195,13 +1189,57 @@ static void vxlan_rcv(struct vxlan_sock *vs, struct sk_buff *skb,
tun_dst = NULL;
}
+ switch (md->gpe_np) {
+ case VXLAN_GPE_NP_IPv4:
+ skb->protocol = htons(ETH_P_IP);
+ goto skip_l2;
+#if IS_ENABLED(CONFIG_IPV6)
+ case VXLAN_GPE_NP_IPv6:
+ skb->protocol = htons(ETH_P_IPV6);
+ goto skip_l2;
+#endif
+#if IS_ENABLED(CONFIG_MPLS)
+ case VXLAN_GPE_NP_MPLS:
+ skb->protocol = htons(ETH_P_MPLS_UC);
+ goto skip_l2;
+#endif
+#if IS_ENABLED(CONFIG_NET_NSH)
+ case VXLAN_GPE_NP_NSH:
+ {
+ u_char next_proto;
+
+ if (nsh_decap(skb, NULL, NULL, &next_proto) < 0)
+ goto drop;
+
+ if (next_proto != NSH_NEXT_PROTO_ETH)
+ goto skip_l2;
+ }
+ break;
+#endif
+ case VXLAN_GPE_NP_ETH:
+ /* GPE with next proto eth is equivalent to vanilla vxlan. */
+ default:
+ break;
+ }
+
+ skb_reset_mac_header(skb);
+ skb->protocol = eth_type_trans(skb, vxlan->dev);
+ skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
+
+ /* Ignore packet loops (and multicast echo) */
+ if (ether_addr_equal(eth_hdr(skb)->h_source, vxlan->dev->dev_addr))
+ goto drop;
+
if ((vxlan->flags & VXLAN_F_LEARN) &&
vxlan_snoop(skb->dev, &saddr, eth_hdr(skb)->h_source))
goto drop;
+skip_l2:
skb_reset_network_header(skb);
+
/* In flow-based mode, GBP is carried in dst_metadata */
- if (!(vs->flags & VXLAN_F_COLLECT_METADATA))
+ if (!(vs->flags & VXLAN_F_COLLECT_METADATA) &&
+ !(vs->flags & VXLAN_F_GPE))
skb->mark = md->gbp;
if (oip6)
@@ -1252,6 +1290,10 @@ static int vxlan_udp_encap_recv(struct sock *sk, struct sk_buff *skb)
struct vxlan_metadata _md;
struct vxlan_metadata *md = &_md;
+ vs = rcu_dereference_sk_user_data(sk);
+ if (!vs)
+ goto drop;
+
/* Need Vxlan and inner Ethernet header to be present */
if (!pskb_may_pull(skb, VXLAN_HLEN))
goto error;
@@ -1267,14 +1309,13 @@ static int vxlan_udp_encap_recv(struct sock *sk, struct sk_buff *skb)
goto bad_flags;
}
- if (iptunnel_pull_header(skb, VXLAN_HLEN, htons(ETH_P_TEB)))
+ /* If GPE, protocol will be set once next proto examined. */
+ if (iptunnel_pull_header(skb, VXLAN_HLEN,
+ vs->flags & VXLAN_F_GPE ?
+ htons(ETH_P_IP) : htons(ETH_P_TEB)))
goto drop;
vxh = (struct vxlanhdr *)(udp_hdr(skb) + 1);
- vs = rcu_dereference_sk_user_data(sk);
- if (!vs)
- goto drop;
-
if ((flags & VXLAN_HF_RCO) && (vs->flags & VXLAN_F_REMCSUM_RX)) {
vxh = vxlan_remcsum(skb, vxh, sizeof(struct vxlanhdr), vni,
!!(vs->flags & VXLAN_F_REMCSUM_NOPARTIAL));
@@ -1318,6 +1359,16 @@ static int vxlan_udp_encap_recv(struct sock *sk, struct sk_buff *skb)
flags &= ~VXLAN_GBP_USED_BITS;
}
+ if (vs->flags & VXLAN_F_GPE) {
+ /* Next protocol is required */
+ if (!(flags & VXLAN_HF_GPE_NP))
+ goto bad_flags;
+
+ md->gpe_np = flags & VXLAN_GPE_NP_MASK;
+
+ flags &= ~VXLAN_GPE_USED_BITS;
+ }
+
if (flags || vni & ~VXLAN_VNI_MASK) {
/* If there are any unprocessed flags remaining treat
* this as a malformed packet. This behavior diverges from
@@ -1664,6 +1715,37 @@ static bool route_shortcircuit(struct net_device *dev, struct sk_buff *skb)
return false;
}
+static void vxlan_build_gpe_hdr(struct vxlanhdr *vxh, __be16 proto)
+{
+ u32 next_proto;
+
+ switch (proto) {
+#if IS_ENABLED(CONFIG_NET_NSH)
+ case htons(ETH_P_NSH):
+ next_proto = VXLAN_GPE_NP_NSH;
+ break;
+#endif
+ case htons(ETH_P_IP):
+ next_proto = VXLAN_GPE_NP_IPv4;
+ break;
+#if IS_ENABLED(CONFIG_IPV6)
+ case htons(ETH_P_IPV6):
+ next_proto = VXLAN_GPE_NP_IPv6;
+ break;
+#endif
+#if IS_ENABLED(CONFIG_MPLS)
+ case htons(ETH_P_MPLS_UC):
+ next_proto = VXLAN_GPE_NP_MPLS;
+ break;
+#endif
+ default:
+ next_proto = VXLAN_GPE_NP_ETH;
+ break;
+ }
+
+ vxh->vx_flags |= htonl(VXLAN_HF_GPE_NP | next_proto);
+}
+
static void vxlan_build_gbp_hdr(struct vxlanhdr *vxh, u32 vxflags,
struct vxlan_metadata *md)
{
@@ -1750,6 +1832,9 @@ static int vxlan_build_skb(struct sk_buff *skb, struct dst_entry *dst,
if (vxflags & VXLAN_F_GBP)
vxlan_build_gbp_hdr(vxh, vxflags, md);
+ if (vxflags & VXLAN_F_GPE)
+ vxlan_build_gpe_hdr(vxh, skb->protocol);
+
skb_set_inner_protocol(skb, htons(ETH_P_TEB));
return 0;
}
@@ -2073,6 +2158,26 @@ static netdev_tx_t vxlan_xmit(struct sk_buff *skb, struct net_device *dev)
struct vxlan_rdst *rdst, *fdst = NULL;
struct vxlan_fdb *f;
+ if (vxlan->flags & VXLAN_F_GPE) {
+ switch (skb->protocol) {
+#if IS_ENABLED(CONFIG_NET_NSH)
+ case htons(ETH_P_NSH):
+#endif
+#if IS_ENABLED(CONFIG_IPV6)
+ case htons(ETH_P_IPV6):
+#endif
+#if IS_ENABLED(CONFIG_MPLS)
+ case htons(ETH_P_MPLS_UC):
+#endif
+ case htons(ETH_P_IP):
+ vxlan_xmit_one(skb, dev, &vxlan->default_dst, false);
+ return NETDEV_TX_OK;
+ default:
+ /* Assume L2 and look for FDB entry */
+ break;
+ }
+ }
+
info = skb_tunnel_info(skb);
skb_reset_mac_header(skb);
@@ -2475,6 +2580,7 @@ static const struct nla_policy vxlan_policy[IFLA_VXLAN_MAX + 1] = {
[IFLA_VXLAN_REMCSUM_RX] = { .type = NLA_U8 },
[IFLA_VXLAN_GBP] = { .type = NLA_FLAG, },
[IFLA_VXLAN_REMCSUM_NOPARTIAL] = { .type = NLA_FLAG },
+ [IFLA_VXLAN_GPE] = { .type = NLA_FLAG, },
};
static int vxlan_validate(struct nlattr *tb[], struct nlattr *data[])
@@ -2895,6 +3001,9 @@ static int vxlan_newlink(struct net *src_net, struct net_device *dev,
if (data[IFLA_VXLAN_REMCSUM_NOPARTIAL])
conf.flags |= VXLAN_F_REMCSUM_NOPARTIAL;
+ if (data[IFLA_VXLAN_GPE])
+ conf.flags |= VXLAN_F_GPE;
+
err = vxlan_dev_configure(src_net, dev, &conf);
switch (err) {
case -ENODEV:
@@ -3037,6 +3146,10 @@ static int vxlan_fill_info(struct sk_buff *skb, const struct net_device *dev)
nla_put_flag(skb, IFLA_VXLAN_REMCSUM_NOPARTIAL))
goto nla_put_failure;
+ if (vxlan->flags & VXLAN_F_GPE &&
+ nla_put_flag(skb, IFLA_VXLAN_GPE))
+ goto nla_put_failure;
+
return 0;
nla_put_failure:
diff --git a/include/net/vxlan.h b/include/net/vxlan.h
index 25bd919..7886296 100644
--- a/include/net/vxlan.h
+++ b/include/net/vxlan.h
@@ -121,8 +121,44 @@ struct vxlanhdr_gbp {
struct vxlan_metadata {
u32 gbp;
+ u8 gpe_np;
};
+/*
+ * VXLAN Generic Protocol Extension:
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * |R|R|Ver|I|P|R|O| Reserved |Next Protocol |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | VXLAN Network Identifier (VNI) | Reserved |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ *
+ * Ver Version, initially 0
+ * I = 1 VXLAN Network Identifier (VNI) present
+ * P = 1 Next Protocol field is present
+ * O = 1 OAM
+ * Next Protocol Indicates the protocol header immediately following
+ * the VXLAN GPE header.
+ *
+ * https://tools.ietf.org/html/draft-ietf-nvo3-vxlan-gpe-01
+ *
+ * Use struct vxlanhdr above with some extra defines:
+ */
+#define VXLAN_HF_GPE_OAM BIT(25) /* GPE OAM bit */
+#define VXLAN_HF_GPE_NP BIT(26) /* GPE protocol bit */
+
+#define VXLAN_GPE_NP_MASK (0xFF)
+
+#define VXLAN_GPE_NP_IPv4 0x1
+#define VXLAN_GPE_NP_IPv6 0x2
+#define VXLAN_GPE_NP_ETH 0x3
+#define VXLAN_GPE_NP_NSH 0x4
+#define VXLAN_GPE_NP_MPLS 0x5
+
+#define VXLAN_GPE_USED_BITS (VXLAN_HF_GPE_NP | \
+ VXLAN_HF_GPE_OAM | \
+ VXLAN_GPE_NP_MASK)
+
+
/* per UDP socket information */
struct vxlan_sock {
struct hlist_node hlist;
@@ -204,6 +240,7 @@ struct vxlan_dev {
#define VXLAN_F_GBP 0x800
#define VXLAN_F_REMCSUM_NOPARTIAL 0x1000
#define VXLAN_F_COLLECT_METADATA 0x2000
+#define VXLAN_F_GPE 0x4000
/* Flags that are used in the receive path. These flags must match in
* order for a socket to be shareable
@@ -212,7 +249,8 @@ struct vxlan_dev {
VXLAN_F_UDP_ZERO_CSUM6_RX | \
VXLAN_F_REMCSUM_RX | \
VXLAN_F_REMCSUM_NOPARTIAL | \
- VXLAN_F_COLLECT_METADATA)
+ VXLAN_F_COLLECT_METADATA | \
+ VXLAN_F_GPE)
struct net_device *vxlan_dev_create(struct net *net, const char *name,
u8 name_assign_type, struct vxlan_config *conf);
diff --git a/include/uapi/linux/if_link.h b/include/uapi/linux/if_link.h
index d452cea..e8d74a5 100644
--- a/include/uapi/linux/if_link.h
+++ b/include/uapi/linux/if_link.h
@@ -456,6 +456,7 @@ enum {
IFLA_VXLAN_GBP,
IFLA_VXLAN_REMCSUM_NOPARTIAL,
IFLA_VXLAN_COLLECT_METADATA,
+ IFLA_VXLAN_GPE,
__IFLA_VXLAN_MAX
};
#define IFLA_VXLAN_MAX (__IFLA_VXLAN_MAX - 1)
--
2.1.4
^ permalink raw reply related [flat|nested] 8+ messages in thread