[PATCH RFC net-next 0/2] XDP multi buffer helpers

[PATCH RFC net-next 0/2] XDP multi buffer helpers

[sameehj]

From: Sameeh Jubran <sameehj@amazon.com>

This series is based on the series that Lorenzo sent [0].

This series simply adds new bpf helpers for xdp mb support as well as
introduces a sample program that uses them.

[0] - [RFC net-next 00/22] Introduce mb bit in xdp_buff/xdp_frame

Sameeh Jubran (2):
  xdp: helpers: add multibuffer support
  samples/bpf: add bpf program that uses xdp mb helpers

 include/uapi/linux/bpf.h       |  13 +++
 net/core/filter.c              |  60 ++++++++++++++
 samples/bpf/Makefile           |   3 +
 samples/bpf/xdp_mb_kern.c      |  66 ++++++++++++++++
 samples/bpf/xdp_mb_user.c      | 174 +++++++++++++++++++++++++++++++++++++++++
 tools/include/uapi/linux/bpf.h |  13 +++
 6 files changed, 329 insertions(+)
 create mode 100644 samples/bpf/xdp_mb_kern.c
 create mode 100644 samples/bpf/xdp_mb_user.c

-- 
2.16.6

[PATCH RFC net-next 1/2] xdp: helpers: add multibuffer support

[sameehj]

From: Sameeh Jubran <sameehj@amazon.com>

The implementation is based on this [0] draft by Jesper D. Brouer.

Provided two helpers:

* bpf_xdp_get_frag()
* bpf_xdp_get_frag_count()

[0] xdp mb design - https://github.com/xdp-project/xdp-project/blob/master/areas/core/xdp-multi-buffer01-design.org
Signed-off-by: Sameeh Jubran <sameehj@amazon.com>
---
 include/uapi/linux/bpf.h       | 13 +++++++++
 net/core/filter.c              | 60 ++++++++++++++++++++++++++++++++++++++++++
 tools/include/uapi/linux/bpf.h | 13 +++++++++
 3 files changed, 86 insertions(+)

diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 5e3863899..3484e481a 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -3320,6 +3320,17 @@ union bpf_attr {
  *		A non-negative value equal to or less than *size* on success,
  *		or a negative error in case of failure.
  *
+ * int bpf_xdp_get_frag(struct xdp_buff *xdp_md, u32 frag_index, u32 *size, u32 *offset)
+ * 	Description
+ *		Get the offset from containing page and size of a given frag.
+ * 	Return
+ * 		0 on success, or a negative error in case of failure.
+ *
+ * int bpf_xdp_get_frag_count(struct xdp_buff *xdp_md)
+ * 	Description
+ *		Get the total number of frags for a given packet.
+ * 	Return
+ * 		The number of frags
  */
 #define __BPF_FUNC_MAPPER(FN)		\
 	FN(unspec),			\
@@ -3464,6 +3475,8 @@ union bpf_attr {
 	FN(skc_to_tcp_request_sock),	\
 	FN(skc_to_udp6_sock),		\
 	FN(get_task_stack),		\
+	FN(xdp_get_frag),		\
+	FN(xdp_get_frag_count),		\
 	/* */
 
 /* integer value in 'imm' field of BPF_CALL instruction selects which helper
diff --git a/net/core/filter.c b/net/core/filter.c
index bdd2382e6..93e790d7b 100644
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -3452,6 +3452,62 @@ static const struct bpf_func_proto bpf_xdp_adjust_head_proto = {
 	.arg2_type	= ARG_ANYTHING,
 };
 
+static inline bool __bpf_xdp_has_frags(struct  xdp_buff *xdp)
+{
+	return xdp->mb != 0;
+}
+
+BPF_CALL_1(bpf_xdp_get_frag_count, struct  xdp_buff*, xdp)
+{
+	return __bpf_xdp_has_frags(xdp) ?
+		((struct skb_shared_info *)xdp->data_end)->nr_frags : 0;
+}
+
+const struct bpf_func_proto bpf_xdp_get_frag_count_proto = {
+	.func		= bpf_xdp_get_frag_count,
+	.gpl_only	= false,
+	.ret_type	= RET_INTEGER,
+	.arg1_type	= ARG_PTR_TO_CTX,
+};
+
+BPF_CALL_4(bpf_xdp_get_frag, struct  xdp_buff*, xdp, u32, frag_index,
+	   u32*, size, u32*, offset)
+{
+	skb_frag_t *frags;
+	u32 frag_size;
+	u16 nr_frags;
+	struct skb_shared_info *skb_info;
+
+	if (!__bpf_xdp_has_frags(xdp))
+		return -EINVAL;
+
+	skb_info = xdp_data_hard_end(xdp);
+	frags = skb_info->frags;
+	nr_frags = skb_info->nr_frags;
+
+	if (frag_index >= nr_frags)
+		return -EINVAL;
+
+	frag_size = frags[frag_index].bv_len;
+
+	if (size)
+		memcpy(size, &frag_size, sizeof(frag_size));
+	if (offset)
+		memcpy(offset, &frags[frag_index].bv_offset, sizeof(frags[frag_index].bv_offset));
+
+	return 0;
+}
+
+const struct bpf_func_proto bpf_xdp_get_frag_proto = {
+	.func		= bpf_xdp_get_frag,
+	.gpl_only	= false,
+	.ret_type	= RET_INTEGER,
+	.arg1_type	= ARG_PTR_TO_CTX,
+	.arg2_type	= ARG_ANYTHING,
+	.arg3_type	= ARG_PTR_TO_INT,
+	.arg4_type	= ARG_PTR_TO_INT,
+};
+
 BPF_CALL_2(bpf_xdp_adjust_tail, struct xdp_buff *, xdp, int, offset)
 {
 	void *data_hard_end = xdp_data_hard_end(xdp); /* use xdp->frame_sz */
@@ -6475,6 +6531,10 @@ xdp_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 		return &bpf_xdp_redirect_map_proto;
 	case BPF_FUNC_xdp_adjust_tail:
 		return &bpf_xdp_adjust_tail_proto;
+	case BPF_FUNC_xdp_get_frag_count:
+		return &bpf_xdp_get_frag_count_proto;
+	case BPF_FUNC_xdp_get_frag:
+		return &bpf_xdp_get_frag_proto;
 	case BPF_FUNC_fib_lookup:
 		return &bpf_xdp_fib_lookup_proto;
 #ifdef CONFIG_INET
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index 5e3863899..3484e481a 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -3320,6 +3320,17 @@ union bpf_attr {
  *		A non-negative value equal to or less than *size* on success,
  *		or a negative error in case of failure.
  *
+ * int bpf_xdp_get_frag(struct xdp_buff *xdp_md, u32 frag_index, u32 *size, u32 *offset)
+ * 	Description
+ *		Get the offset from containing page and size of a given frag.
+ * 	Return
+ * 		0 on success, or a negative error in case of failure.
+ *
+ * int bpf_xdp_get_frag_count(struct xdp_buff *xdp_md)
+ * 	Description
+ *		Get the total number of frags for a given packet.
+ * 	Return
+ * 		The number of frags
  */
 #define __BPF_FUNC_MAPPER(FN)		\
 	FN(unspec),			\
@@ -3464,6 +3475,8 @@ union bpf_attr {
 	FN(skc_to_tcp_request_sock),	\
 	FN(skc_to_udp6_sock),		\
 	FN(get_task_stack),		\
+	FN(xdp_get_frag),		\
+	FN(xdp_get_frag_count),		\
 	/* */
 
 /* integer value in 'imm' field of BPF_CALL instruction selects which helper
-- 
2.16.6

Re: [PATCH RFC net-next 1/2] xdp: helpers: add multibuffer support

[David Miller]

From: <sameehj@amazon.com>
Date: Mon, 27 Jul 2020 12:56:52 +0000

> diff --git a/net/core/filter.c b/net/core/filter.c
> index bdd2382e6..93e790d7b 100644
> --- a/net/core/filter.c
> +++ b/net/core/filter.c
> @@ -3452,6 +3452,62 @@ static const struct bpf_func_proto bpf_xdp_adjust_head_proto = {
>  	.arg2_type	= ARG_ANYTHING,
>  };
>  
> +static inline bool __bpf_xdp_has_frags(struct  xdp_buff *xdp)
> +{
> +	return xdp->mb != 0;
> +}

Please don't use the inline keyword in foo.c files, let the compiler decide.

Thank you.

[PATCH RFC net-next 2/2] samples/bpf: add bpf program that uses xdp mb helpers

[sameehj]

From: Sameeh Jubran <sameehj@amazon.com>

The bpf program returns XDP_PASS for every packet and calculates the
total number of bytes in its linear and paged parts.

The program is executed with:
./xdp_mb [if name]

and has the following output format:
[if index]: [rx packet count] pkt/sec, [number of bytes] bytes/sec

Signed-off-by: Shay Agroskin <shayagr@amazon.com>
Signed-off-by: Sameeh Jubran <sameehj@amazon.com>
---
 samples/bpf/Makefile      |   3 +
 samples/bpf/xdp_mb_kern.c |  66 ++++++++++++++++++
 samples/bpf/xdp_mb_user.c | 174 ++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 243 insertions(+)
 create mode 100644 samples/bpf/xdp_mb_kern.c
 create mode 100644 samples/bpf/xdp_mb_user.c

diff --git a/samples/bpf/Makefile b/samples/bpf/Makefile
index f87ee0207..fd21d99e5 100644
--- a/samples/bpf/Makefile
+++ b/samples/bpf/Makefile
@@ -53,6 +53,7 @@ tprogs-y += task_fd_query
 tprogs-y += xdp_sample_pkts
 tprogs-y += ibumad
 tprogs-y += hbm
+tprogs-y += xdp_mb
 
 # Libbpf dependencies
 LIBBPF = $(TOOLS_PATH)/lib/bpf/libbpf.a
@@ -109,6 +110,7 @@ task_fd_query-objs := bpf_load.o task_fd_query_user.o $(TRACE_HELPERS)
 xdp_sample_pkts-objs := xdp_sample_pkts_user.o $(TRACE_HELPERS)
 ibumad-objs := bpf_load.o ibumad_user.o $(TRACE_HELPERS)
 hbm-objs := bpf_load.o hbm.o $(CGROUP_HELPERS)
+xdp_mb-objs := xdp_mb_user.o
 
 # Tell kbuild to always build the programs
 always-y := $(tprogs-y)
@@ -170,6 +172,7 @@ always-y += ibumad_kern.o
 always-y += hbm_out_kern.o
 always-y += hbm_edt_kern.o
 always-y += xdpsock_kern.o
+always-y += xdp_mb_kern.o
 
 ifeq ($(ARCH), arm)
 # Strip all except -D__LINUX_ARM_ARCH__ option needed to handle linux
diff --git a/samples/bpf/xdp_mb_kern.c b/samples/bpf/xdp_mb_kern.c
new file mode 100644
index 000000000..1dfd61b34
--- /dev/null
+++ b/samples/bpf/xdp_mb_kern.c
@@ -0,0 +1,66 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/*
+ * Copyright 2020 Amazon.com, Inc. or its affiliates. All rights reserved.
+ */
+#define KBUILD_MODNAME "foo"
+#include <uapi/linux/bpf.h>
+#include <linux/in.h>
+#include <linux/if_ether.h>
+#include <linux/if_packet.h>
+#include <linux/if_vlan.h>
+#include <linux/ip.h>
+#include <linux/ipv6.h>
+#include <bpf/bpf_helpers.h>
+
+#define MAX_FRAGS 5
+
+/* count RX packets */
+struct {
+	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
+	__type(key, u32);
+	__type(value, long);
+	__uint(max_entries, 1);
+} rxcnt SEC(".maps");
+
+/* count total number of bytes */
+struct {
+	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
+	__type(key, u32);
+	__type(value, long);
+	__uint(max_entries, 1);
+} total_bytes SEC(".maps");
+
+SEC("xdp_mb")
+int xdp_mb_prog(struct xdp_md *ctx)
+{
+	void *data_end = (void *)(long)ctx->data_end;
+	void *data = (void *)(long)ctx->data;
+	u32 frag_offset = 0;
+	u32 frag_size = 0;
+	u32 key = 0;
+	u32 frag_nr;
+	long *value;
+	int total;
+	int i;
+
+	value = bpf_map_lookup_elem(&rxcnt, &key);
+	if (value)
+		*value += 1;
+
+	value = bpf_map_lookup_elem(&total_bytes, &key);
+	if (value) {
+		total = data_end - data;
+
+		frag_nr = bpf_xdp_get_frag_count(ctx);
+		for (i = 0; i < MAX_FRAGS && i < frag_nr; i++) {
+			bpf_xdp_get_frag(ctx, i, &frag_size, &frag_offset);
+			total += frag_size;
+		}
+
+		*value += total;
+	}
+
+	return XDP_PASS;
+}
+
+char _license[] SEC("license") = "GPL";
diff --git a/samples/bpf/xdp_mb_user.c b/samples/bpf/xdp_mb_user.c
new file mode 100644
index 000000000..ff122d1e6
--- /dev/null
+++ b/samples/bpf/xdp_mb_user.c
@@ -0,0 +1,174 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/*
+ * Copyright 2020 Amazon.com, Inc. or its affiliates. All rights reserved.
+ */
+#include <linux/bpf.h>
+#include <linux/if_link.h>
+#include <assert.h>
+#include <errno.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <libgen.h>
+#include <sys/resource.h>
+#include <net/if.h>
+
+#include "bpf_util.h"
+#include <bpf/bpf.h>
+#include <bpf/libbpf.h>
+
+static int ifindex;
+static __u32 xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST | XDP_FLAGS_DRV_MODE;
+static __u32 prog_id;
+
+static int rxcnt_fd, total_bytes_fd;
+
+static void int_exit(int sig)
+{
+	__u32 curr_prog_id = 0;
+
+	if (bpf_get_link_xdp_id(ifindex, &curr_prog_id, xdp_flags)) {
+		printf("bpf_get_link_xdp_id failed\n");
+		exit(1);
+	}
+	if (prog_id == curr_prog_id)
+		bpf_set_link_xdp_fd(ifindex, -1, xdp_flags);
+	else if (!curr_prog_id)
+		printf("couldn't find a prog id on a given interface\n");
+	else
+		printf("program on interface changed, not removing\n");
+	exit(0);
+}
+
+/* count total packets and bytes per second */
+static void poll_stats(int interval)
+{
+	unsigned int nr_cpus = bpf_num_possible_cpus();
+	__u64 rx_cnt[nr_cpus], rx_cnt_prev[nr_cpus];
+	__u64 total_bytes[nr_cpus], total_bytes_prev[nr_cpus];
+	int i;
+
+	memset(rx_cnt_prev, 0, sizeof(rx_cnt_prev));
+	memset(total_bytes_prev, 0, sizeof(total_bytes_prev));
+
+	while (1) {
+		__u64 sum = 0, mb_sum = 0;
+		__u32 key = 0;
+
+		sleep(interval);
+
+		/* fetch rx cnt */
+		assert(bpf_map_lookup_elem(rxcnt_fd, &key, rx_cnt) == 0);
+		for (i = 0; i < nr_cpus; i++)
+			sum += (rx_cnt[i] - rx_cnt_prev[i]);
+		memcpy(rx_cnt_prev, rx_cnt, sizeof(rx_cnt));
+
+		/* count total bytes of packets */
+		assert(bpf_map_lookup_elem(total_bytes_fd, &key, total_bytes) == 0);
+		for (i = 0; i < nr_cpus; i++)
+			mb_sum += (total_bytes[i] - total_bytes_prev[i]);
+		memcpy(total_bytes_prev, total_bytes, sizeof(total_bytes));
+
+		if (sum)
+			printf("ifindex %i: %10llu pkt/s, %10llu bytes/s\n",
+			       ifindex, sum / interval, mb_sum / interval);
+	}
+}
+
+static void usage(const char *prog)
+{
+	fprintf(stderr,
+		"usage: %s [OPTS] IFACE\n\n"
+		"OPTS:\n"
+		"    -F    force loading prog\n",
+		prog);
+}
+
+int main(int argc, char **argv)
+{
+	struct rlimit r = {RLIM_INFINITY, RLIM_INFINITY};
+	struct bpf_prog_load_attr prog_load_attr = {
+		.prog_type	= BPF_PROG_TYPE_XDP,
+	};
+	int prog_fd, opt;
+	struct bpf_prog_info info = {};
+	__u32 info_len = sizeof(info);
+	const char *optstr = "F";
+	struct bpf_program *prog;
+	struct bpf_object *obj;
+	char filename[256];
+	int err;
+
+	while ((opt = getopt(argc, argv, optstr)) != -1) {
+		switch (opt) {
+		case 'F':
+			xdp_flags &= ~XDP_FLAGS_UPDATE_IF_NOEXIST;
+			break;
+		default:
+			usage(basename(argv[0]));
+			return 1;
+		}
+	}
+
+
+	if (optind == argc) {
+		usage(basename(argv[0]));
+		return 1;
+	}
+
+	if (setrlimit(RLIMIT_MEMLOCK, &r)) {
+		perror("setrlimit(RLIMIT_MEMLOCK)");
+		return 1;
+	}
+
+	ifindex = if_nametoindex(argv[optind]);
+	if (!ifindex) {
+		perror("if_nametoindex");
+		return 1;
+	}
+
+	snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]);
+	prog_load_attr.file = filename;
+
+	if (bpf_prog_load_xattr(&prog_load_attr, &obj, &prog_fd))
+		return 1;
+
+	prog = bpf_program__next(NULL, obj);
+	if (!prog) {
+		printf("finding a prog in obj file failed\n");
+		return 1;
+	}
+
+	if (!prog_fd) {
+		printf("bpf_prog_load_xattr: %s\n", strerror(errno));
+		return 1;
+	}
+
+	rxcnt_fd = bpf_object__find_map_fd_by_name(obj, "rxcnt");
+	total_bytes_fd = bpf_object__find_map_fd_by_name(obj, "total_bytes");
+	if (rxcnt_fd < 0 || total_bytes_fd < 0) {
+		printf("bpf_object__find_map_fd_by_name failed\n");
+		return 1;
+	}
+
+	if (bpf_set_link_xdp_fd(ifindex, prog_fd, xdp_flags) < 0) {
+		printf("ERROR: link set xdp fd failed on %d\n", ifindex);
+		return 1;
+	}
+
+	err = bpf_obj_get_info_by_fd(prog_fd, &info, &info_len);
+	if (err) {
+		printf("can't get prog info - %s\n", strerror(errno));
+		return err;
+	}
+	prog_id = info.id;
+
+	signal(SIGINT, int_exit);
+	signal(SIGTERM, int_exit);
+
+	poll_stats(1);
+
+	return 0;
+}
-- 
2.16.6

Re: [PATCH RFC net-next 0/2] XDP multi buffer helpers

[Maciej Fijalkowski]

On Mon, Jul 27, 2020 at 12:56:51PM +0000, sameehj@amazon.com wrote:
> From: Sameeh Jubran <sameehj@amazon.com>
> 
> This series is based on the series that Lorenzo sent [0].

What is your final design for multi buffer support in XDP?
Why don't you provide a single RFC that is fully functional but instead
you're sending a bunch of separate RFCs?

IMHO it's a weird strategy. Not sure what others think about.

> 
> This series simply adds new bpf helpers for xdp mb support as well as
> introduces a sample program that uses them.
> 
> [0] - [RFC net-next 00/22] Introduce mb bit in xdp_buff/xdp_frame

Direct link wouldn't hurt I guess :) Please also include all the previous
discussions that took place on mailing list around this topic. This will
make reviewers life easier I suppose. As I asked above, I'm not sure
what's your final design for this feature.

> 
> Sameeh Jubran (2):
>   xdp: helpers: add multibuffer support
>   samples/bpf: add bpf program that uses xdp mb helpers
> 
>  include/uapi/linux/bpf.h       |  13 +++
>  net/core/filter.c              |  60 ++++++++++++++
>  samples/bpf/Makefile           |   3 +
>  samples/bpf/xdp_mb_kern.c      |  66 ++++++++++++++++
>  samples/bpf/xdp_mb_user.c      | 174 +++++++++++++++++++++++++++++++++++++++++
>  tools/include/uapi/linux/bpf.h |  13 +++
>  6 files changed, 329 insertions(+)
>  create mode 100644 samples/bpf/xdp_mb_kern.c
>  create mode 100644 samples/bpf/xdp_mb_user.c
> 
> -- 
> 2.16.6
> 

Re: [PATCH RFC net-next 0/2] XDP multi buffer helpers

[Lorenzo Bianconi]

[-- Attachment #1: Type: text/plain, Size: 2303 bytes --]

> On Mon, Jul 27, 2020 at 12:56:51PM +0000, sameehj@amazon.com wrote:
> > From: Sameeh Jubran <sameehj@amazon.com>
> > 
> > This series is based on the series that Lorenzo sent [0].
> 
> What is your final design for multi buffer support in XDP?
> Why don't you provide a single RFC that is fully functional but instead
> you're sending a bunch of separate RFCs?

Hi Maciej,

IMO the buffer layout is defined now (we will use the trailing part of the
"linear" area of the xdp_buff to store references for subsequent buffers like
we already do for skb).
What is not defined yet are the metadata (e.g. number of frames, total length, ..)
we want to pass to the bpf layer. This is the main reason why I sent this series
[1] as RFC, I want to collect feedbacks about this approach. For the moment (at
least in my series) mb in xdp_buff is just used to indicate if this is a
multi-buff xdp_buff.

> 
> IMHO it's a weird strategy. Not sure what others think about.
> 

we did not coordinate, sorry for the noise.

Regards,
Lorenzo

[1] https://patchwork.ozlabs.org/project/netdev/cover/cover.1595503780.git.lorenzo@kernel.org/

> > 
> > This series simply adds new bpf helpers for xdp mb support as well as
> > introduces a sample program that uses them.
> > 
> > [0] - [RFC net-next 00/22] Introduce mb bit in xdp_buff/xdp_frame
> 
> Direct link wouldn't hurt I guess :) Please also include all the previous
> discussions that took place on mailing list around this topic. This will
> make reviewers life easier I suppose. As I asked above, I'm not sure
> what's your final design for this feature.
> 
> > 
> > Sameeh Jubran (2):
> >   xdp: helpers: add multibuffer support
> >   samples/bpf: add bpf program that uses xdp mb helpers
> > 
> >  include/uapi/linux/bpf.h       |  13 +++
> >  net/core/filter.c              |  60 ++++++++++++++
> >  samples/bpf/Makefile           |   3 +
> >  samples/bpf/xdp_mb_kern.c      |  66 ++++++++++++++++
> >  samples/bpf/xdp_mb_user.c      | 174 +++++++++++++++++++++++++++++++++++++++++
> >  tools/include/uapi/linux/bpf.h |  13 +++
> >  6 files changed, 329 insertions(+)
> >  create mode 100644 samples/bpf/xdp_mb_kern.c
> >  create mode 100644 samples/bpf/xdp_mb_user.c
> > 
> > -- 
> > 2.16.6
> > 
> 

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 228 bytes --]

Re: [PATCH RFC net-next 0/2] XDP multi buffer helpers

[Jakub Kicinski]

On Tue, 28 Jul 2020 14:33:27 +0200 Maciej Fijalkowski wrote:
> On Mon, Jul 27, 2020 at 12:56:51PM +0000, sameehj@amazon.com wrote:
> > From: Sameeh Jubran <sameehj@amazon.com>
> > 
> > This series is based on the series that Lorenzo sent [0].  
> 
> What is your final design for multi buffer support in XDP?
> Why don't you provide a single RFC that is fully functional but instead
> you're sending a bunch of separate RFCs?

+1

are we expecting a new attachment type?