[PATCH v2 0/4] libnetfilter_queue: gso handling support

[PATCH v2 0/4] libnetfilter_queue: gso handling support

[Florian Westphal]

Hi,

this is V2 of the gso paches for libnetfilter_queue.

Changes since V1:
- refresh nfnetlink_queue.h with nf-next copy instead of removing it
- don't install internal.h on "make install"
- avoid magic number in example program

Thanks,
Florian

[PATCH 1/4] build: don't install internal.h

[Florian Westphal]

---
 src/Makefile.am |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/src/Makefile.am b/src/Makefile.am
index d694794..6e5d503 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -24,7 +24,7 @@ include ${top_srcdir}/Make_global.am
 
 lib_LTLIBRARIES = libnetfilter_queue.la
 
-include_HEADERS = internal.h
+noinst_HEADERS = internal.h
 
 libnetfilter_queue_la_LDFLAGS = -Wc,-nostartfiles -lnfnetlink \
 				-version-info $(LIBVERSION)
-- 
1.7.8.6

[PATCH 2/4] examples/nf-queue: handle recv error, use larger buffer

[Florian Westphal]

We ask for 0xffff copy size, so we need a buffer that can
hold 0xffff, plus a few more bytes to allow for netlink attributes.

Also, turn off ENOBUFS.

Signed-off-by: Florian Westphal <fw@strlen.de>
---
 examples/nf-queue.c |   37 ++++++++++++++++++++++++-------------
 1 files changed, 24 insertions(+), 13 deletions(-)

diff --git a/examples/nf-queue.c b/examples/nf-queue.c
index 7adac21..6641a24 100644
--- a/examples/nf-queue.c
+++ b/examples/nf-queue.c
@@ -1,3 +1,4 @@
+#include <errno.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
@@ -82,7 +83,9 @@ static int queue_cb(const struct nlmsghdr *nlh, void *data)
 
 int main(int argc, char *argv[])
 {
-	char buf[MNL_SOCKET_BUFFER_SIZE];
+	char *buf;
+	/* largest possible packet payload, plus netlink data overhead: */
+	size_t sizeof_buf = 0xffff + (MNL_SOCKET_BUFFER_SIZE/2);
 	struct nlmsghdr *nlh;
 	int ret;
 	unsigned int portid, queue_num;
@@ -105,6 +108,12 @@ int main(int argc, char *argv[])
 	}
 	portid = mnl_socket_get_portid(nl);
 
+	buf = malloc(sizeof_buf);
+	if (!buf) {
+		perror("allocate receive buffer");
+		exit(EXIT_FAILURE);
+	}
+
 	nlh = nfq_hdr_put(buf, NFQNL_MSG_CONFIG, 0);
 	nfq_nlmsg_cfg_put_cmd(nlh, AF_INET, NFQNL_CFG_CMD_PF_UNBIND);
 
@@ -137,23 +146,25 @@ int main(int argc, char *argv[])
 		exit(EXIT_FAILURE);
 	}
 
-	ret = mnl_socket_recvfrom(nl, buf, sizeof(buf));
-	if (ret == -1) {
-		perror("mnl_socket_recvfrom");
-		exit(EXIT_FAILURE);
-	}
-	while (ret > 0) {
-		ret = mnl_cb_run(buf, ret, 0, portid, queue_cb, NULL);
-		if (ret < 0){
-			perror("mnl_cb_run");
-			exit(EXIT_FAILURE);
-		}
+	/* ENOBUFS is signalled to userspace when packets were lost
+	 * on kernel side.  In most cases, userspace isn't interested
+	 * in this information, so turn it off.
+	 */
+	ret = 1;
+	mnl_socket_setsockopt(nl, NETLINK_NO_ENOBUFS, &ret, sizeof(int));
 
-		ret = mnl_socket_recvfrom(nl, buf, sizeof(buf));
+	for (;;) {
+		ret = mnl_socket_recvfrom(nl, buf, sizeof_buf);
 		if (ret == -1) {
 			perror("mnl_socket_recvfrom");
 			exit(EXIT_FAILURE);
 		}
+
+		ret = mnl_cb_run(buf, ret, 0, portid, queue_cb, NULL);
+		if (ret < 0){
+			perror("mnl_cb_run");
+			exit(EXIT_FAILURE);
+		}
 	}
 
 	mnl_socket_close(nl);
-- 
1.7.8.6

[PATCH 3/4] src: add new GSO handling capabilities

[Florian Westphal]

allows userspace to ask for large gso packets via nfqueue.

Signed-off-by: Florian Westphal <fw@strlen.de>
---
 include/libnetfilter_queue/linux_nfnetlink_queue.h |   13 +++++++++++-
 include/linux/netfilter/nfnetlink_queue.h          |   21 +++++++++++++++----
 src/libnetfilter_queue.c                           |    6 +++++
 src/nlmsg.c                                        |    2 +
 4 files changed, 36 insertions(+), 6 deletions(-)

diff --git a/include/libnetfilter_queue/linux_nfnetlink_queue.h b/include/libnetfilter_queue/linux_nfnetlink_queue.h
index 58c8ca5..81a485b 100644
--- a/include/libnetfilter_queue/linux_nfnetlink_queue.h
+++ b/include/libnetfilter_queue/linux_nfnetlink_queue.h
@@ -45,6 +45,10 @@ enum nfqnl_attr_type {
 	NFQA_IFINDEX_PHYSOUTDEV,	/* u_int32_t ifindex */
 	NFQA_HWADDR,			/* nfqnl_msg_packet_hw */
 	NFQA_PAYLOAD,			/* opaque data payload */
+	NFQA_CT,			/* nf_conntrack_netlink.h */
+	NFQA_CT_INFO,			/* enum ip_conntrack_info */
+	NFQA_CAP_LEN,                   /* __u32 length of captured packet */
+	NFQA_SKB_INFO,                  /* __u32 skb meta information */
 
 	__NFQA_MAX
 };
@@ -96,6 +100,13 @@ enum nfqnl_attr_config {
 /* Flags/options for NFQA_CFG_FLAGS */
 #define NFQA_CFG_F_FAIL_OPEN		(1 << 0)
 #define NFQA_CFG_F_CONNTRACK		(1 << 1)
-#define NFQA_CFG_F_MAX			(1 << 2)
+#define NFQA_CFG_F_GSO			(1 << 2)
+#define NFQA_CFG_F_MAX			(1 << 3)
+
+/* flags for NFQA_SKB_INFO */
+/* packet appears to have wrong checksums, but they are ok */
+#define NFQA_SKB_CSUMNOTREADY (1 << 0)
+/* packet is GSO (i.e., exceeds device mtu) */
+#define NFQA_SKB_GSO (1 << 1)
 
 #endif /* _NFNETLINK_QUEUE_H */
diff --git a/include/linux/netfilter/nfnetlink_queue.h b/include/linux/netfilter/nfnetlink_queue.h
index da44b33..a2308ae 100644
--- a/include/linux/netfilter/nfnetlink_queue.h
+++ b/include/linux/netfilter/nfnetlink_queue.h
@@ -44,6 +44,8 @@ enum nfqnl_attr_type {
 	NFQA_PAYLOAD,			/* opaque data payload */
 	NFQA_CT,			/* nf_conntrack_netlink.h */
 	NFQA_CT_INFO,			/* enum ip_conntrack_info */
+	NFQA_CAP_LEN,			/* __u32 length of captured packet */
+	NFQA_SKB_INFO,			/* __u32 skb meta information */
 
 	__NFQA_MAX
 };
@@ -80,19 +82,28 @@ struct nfqnl_msg_config_params {
 	__u8	copy_mode;	/* enum nfqnl_config_mode */
 } __attribute__ ((packed));
 
-enum nfqnl_flags {
-	NFQNL_F_NONE		= 0,
-	NFQNL_F_CONNTRACK	= (1 << 0),
-};
 
 enum nfqnl_attr_config {
 	NFQA_CFG_UNSPEC,
 	NFQA_CFG_CMD,			/* nfqnl_msg_config_cmd */
 	NFQA_CFG_PARAMS,		/* nfqnl_msg_config_params */
 	NFQA_CFG_QUEUE_MAXLEN,		/* __u32 */
-	NFQA_CFG_FLAGS,			/* __u32 */
+	NFQA_CFG_MASK,			/* identify which flags to change */
+	NFQA_CFG_FLAGS,			/* value of these flags (__u32) */
 	__NFQA_CFG_MAX
 };
 #define NFQA_CFG_MAX (__NFQA_CFG_MAX-1)
 
+/* Flags for NFQA_CFG_FLAGS */
+#define NFQA_CFG_F_FAIL_OPEN			(1 << 0)
+#define NFQA_CFG_F_CONNTRACK			(1 << 1)
+#define NFQA_CFG_F_GSO				(1 << 2)
+#define NFQA_CFG_F_MAX				(1 << 3)
+
+/* flags for NFQA_SKB_INFO */
+/* packet appears to have wrong checksums, but they are ok */
+#define NFQA_SKB_CSUMNOTREADY (1 << 0)
+/* packet is GSO (i.e., exceeds device mtu) */
+#define NFQA_SKB_GSO (1 << 1)
+
 #endif /* _NFNETLINK_QUEUE_H */
diff --git a/src/libnetfilter_queue.c b/src/libnetfilter_queue.c
index 2894ccd..b933a2a 100644
--- a/src/libnetfilter_queue.c
+++ b/src/libnetfilter_queue.c
@@ -639,6 +639,12 @@ int nfq_set_mode(struct nfq_q_handle *qh,
  * - NFQA_CFG_F_CONNTRACK (requires Linux kernel >= 3.6): the kernel will
  *   include the Connection Tracking system information.
  *
+ * - NFQA_CFG_F_GSO (requires Linux kernel >= 3.10): the kernel will
+ *   not normalize offload packets, i.e. your application will need to
+ *   be able to handle packets larger than the mtu (up to 64k) and will
+ *   need to check the NFQA_SKB_INFO attribute to determine when ip/tcp
+ *   checksums are valid even if they appear to be invalid.
+ *
  * Here's a little code snippet to show how to use this API:
  * \verbatim
 	uint32_t flags = NFQA_CFG_F_FAIL_OPEN;
diff --git a/src/nlmsg.c b/src/nlmsg.c
index 6c4a139..e7a30e0 100644
--- a/src/nlmsg.c
+++ b/src/nlmsg.c
@@ -132,6 +132,8 @@ static int nfq_pkt_parse_attr_cb(const struct nlattr *attr, void *data)
 	case NFQA_IFINDEX_OUTDEV:
 	case NFQA_IFINDEX_PHYSINDEV:
 	case NFQA_IFINDEX_PHYSOUTDEV:
+	case NFQA_CAP_LEN:
+	case NFQA_SKB_INFO:
 		if (mnl_attr_validate(attr, MNL_TYPE_U32) < 0)
 			return MNL_CB_ERROR;
 		break;
-- 
1.7.8.6

[PATCH 4/4] examples/nf-queue: receive large gso packets

[Florian Westphal]

Signed-off-by: Florian Westphal <fw@strlen.de>
---
 examples/nf-queue.c |   29 +++++++++++++++++++++++++++--
 1 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/examples/nf-queue.c b/examples/nf-queue.c
index 6641a24..1f465ad 100644
--- a/examples/nf-queue.c
+++ b/examples/nf-queue.c
@@ -51,7 +51,7 @@ static int queue_cb(const struct nlmsghdr *nlh, void *data)
 {
 	struct nfqnl_msg_packet_hdr *ph = NULL;
 	struct nlattr *attr[NFQA_MAX+1] = {};
-	uint32_t id = 0;
+	uint32_t id = 0, skbinfo;
 	struct nfgenmsg *nfg;
 	uint16_t plen;
 
@@ -72,10 +72,32 @@ static int queue_cb(const struct nlmsghdr *nlh, void *data)
 	plen = mnl_attr_get_payload_len(attr[NFQA_PAYLOAD]);
 	/* void *payload = mnl_attr_get_payload(attr[NFQA_PAYLOAD]); */
 
+	skbinfo = attr[NFQA_SKB_INFO] ? ntohl(mnl_attr_get_u32(attr[NFQA_SKB_INFO])) : 0;
+
+	if (attr[NFQA_CAP_LEN]) {
+		uint32_t orig_len = ntohl(mnl_attr_get_u32(attr[NFQA_CAP_LEN]));
+		if (orig_len != plen)
+			printf("truncated ");
+	}
+
+	if (skbinfo & NFQA_SKB_GSO)
+		printf("GSO ");
+
 	id = ntohl(ph->packet_id);
-	printf("packet received (id=%u hw=0x%04x hook=%u, payload len %u)\n",
+	printf("packet received (id=%u hw=0x%04x hook=%u, payload len %u",
 		id, ntohs(ph->hw_protocol), ph->hook, plen);
 
+	/*
+	 * ip/tcp checksums are not yet valid, e.g. due to GRO/GSO.
+	 * The application should behave as if the checksums are correct.
+	 *
+	 * If these packets are later forwarded/sent out, the checksums will
+	 * be corrected by kernel/hardware.
+	 */
+	if (skbinfo & NFQA_SKB_CSUMNOTREADY)
+		printf(", checksum not ready");
+	puts(")");
+
 	nfq_send_verdict(ntohs(nfg->res_id), id);
 
 	return MNL_CB_OK;
@@ -141,6 +163,9 @@ int main(int argc, char *argv[])
 	nlh = nfq_hdr_put(buf, NFQNL_MSG_CONFIG, queue_num);
 	nfq_nlmsg_cfg_put_params(nlh, NFQNL_COPY_PACKET, 0xffff);
 
+	mnl_attr_put_u32(nlh, NFQA_CFG_FLAGS, htonl(NFQA_CFG_F_GSO));
+	mnl_attr_put_u32(nlh, NFQA_CFG_MASK, htonl(NFQA_CFG_F_GSO));
+
 	if (mnl_socket_sendto(nl, nlh, nlh->nlmsg_len) < 0) {
 		perror("mnl_socket_send");
 		exit(EXIT_FAILURE);
-- 
1.7.8.6

[PATCH 0/4] libnetfilter_queue: gso handling support

[Florian Westphal]

Hello,

these are the userspace changes for gso-avoidance in nf_queue.
These are really just updates to header files to get the new flag
and attribute, plus a small change to the validation function.

Two major changes:
- the example program
- partial revert of the libmnl API change

The latter needs double-checking that everything is consistent
now.

I'll push these changes to -next branch after a couple of days
[ unless there are issues that need fixing, of course ].

Cheers,
Florian

[PATCH 2/4] examples/nf-queue: handle recv error, use larger buffer

[Florian Westphal]

We ask for 0xffff copy size, so we need a buffer that can
hold 0xffff, plus a few more bytes to allow for netlink attributes.

Also, turn off/handle ENOBUFS.

Signed-off-by: Florian Westphal <fw@strlen.de>
---
 examples/nf-queue.c |   38 +++++++++++++++++++++++++-------------
 1 files changed, 25 insertions(+), 13 deletions(-)

diff --git a/examples/nf-queue.c b/examples/nf-queue.c
index 7adac21..57ba483 100644
--- a/examples/nf-queue.c
+++ b/examples/nf-queue.c
@@ -1,3 +1,4 @@
+#include <errno.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
@@ -82,7 +83,8 @@ static int queue_cb(const struct nlmsghdr *nlh, void *data)
 
 int main(int argc, char *argv[])
 {
-	char buf[MNL_SOCKET_BUFFER_SIZE];
+	char *buf;
+	size_t sizeof_buf = 0xffff + 2084;
 	struct nlmsghdr *nlh;
 	int ret;
 	unsigned int portid, queue_num;
@@ -105,6 +107,12 @@ int main(int argc, char *argv[])
 	}
 	portid = mnl_socket_get_portid(nl);
 
+	buf = malloc(sizeof_buf);
+	if (!buf) {
+		perror("allocate receive buffer");
+		exit(EXIT_FAILURE);
+	}
+
 	nlh = nfq_hdr_put(buf, NFQNL_MSG_CONFIG, 0);
 	nfq_nlmsg_cfg_put_cmd(nlh, AF_INET, NFQNL_CFG_CMD_PF_UNBIND);
 
@@ -137,23 +145,27 @@ int main(int argc, char *argv[])
 		exit(EXIT_FAILURE);
 	}
 
-	ret = mnl_socket_recvfrom(nl, buf, sizeof(buf));
-	if (ret == -1) {
-		perror("mnl_socket_recvfrom");
-		exit(EXIT_FAILURE);
-	}
-	while (ret > 0) {
-		ret = mnl_cb_run(buf, ret, 0, portid, queue_cb, NULL);
-		if (ret < 0){
-			perror("mnl_cb_run");
-			exit(EXIT_FAILURE);
-		}
+	/* ENOBUFS is signalled to userspace when packets were lost
+	 * on kernel side.  In most cases, userspace isn't interested
+	 * in this information, so turn it off.
+	 */
+	ret = 1;
+	mnl_socket_setsockopt(nl, NETLINK_NO_ENOBUFS, &ret, sizeof(int));
 
-		ret = mnl_socket_recvfrom(nl, buf, sizeof(buf));
+	for (;;) {
+		ret = mnl_socket_recvfrom(nl, buf, sizeof_buf);
 		if (ret == -1) {
+			if (errno == ENOBUFS) /* messages were lost */
+				continue;
 			perror("mnl_socket_recvfrom");
 			exit(EXIT_FAILURE);
 		}
+
+		ret = mnl_cb_run(buf, ret, 0, portid, queue_cb, NULL);
+		if (ret < 0){
+			perror("mnl_cb_run");
+			exit(EXIT_FAILURE);
+		}
 	}
 
 	mnl_socket_close(nl);
-- 
1.7.8.6

Re: [PATCH 2/4] examples/nf-queue: handle recv error, use larger buffer

[Pablo Neira Ayuso]

On Thu, Apr 25, 2013 at 12:43:29PM +0200, Florian Westphal wrote:
> We ask for 0xffff copy size, so we need a buffer that can
> hold 0xffff, plus a few more bytes to allow for netlink attributes.
> 
> Also, turn off/handle ENOBUFS.
> 
> Signed-off-by: Florian Westphal <fw@strlen.de>
> ---
>  examples/nf-queue.c |   38 +++++++++++++++++++++++++-------------
>  1 files changed, 25 insertions(+), 13 deletions(-)
> 
> diff --git a/examples/nf-queue.c b/examples/nf-queue.c
> index 7adac21..57ba483 100644
> --- a/examples/nf-queue.c
> +++ b/examples/nf-queue.c
> @@ -1,3 +1,4 @@
> +#include <errno.h>
>  #include <stdio.h>
>  #include <stdlib.h>
>  #include <unistd.h>
> @@ -82,7 +83,8 @@ static int queue_cb(const struct nlmsghdr *nlh, void *data)
>  
>  int main(int argc, char *argv[])
>  {
> -	char buf[MNL_SOCKET_BUFFER_SIZE];
> +	char *buf;
> +	size_t sizeof_buf = 0xffff + 2084;

I think users will appreciate a comment to explain why those black
magic numbers are there ;-). Probably using MNL_SOCKET_BUFFER_SIZE/2
instead of 2084.

>  	struct nlmsghdr *nlh;
>  	int ret;
>  	unsigned int portid, queue_num;
> @@ -105,6 +107,12 @@ int main(int argc, char *argv[])
>  	}
>  	portid = mnl_socket_get_portid(nl);
>  
> +	buf = malloc(sizeof_buf);
> +	if (!buf) {
> +		perror("allocate receive buffer");
> +		exit(EXIT_FAILURE);
> +	}
> +
>  	nlh = nfq_hdr_put(buf, NFQNL_MSG_CONFIG, 0);
>  	nfq_nlmsg_cfg_put_cmd(nlh, AF_INET, NFQNL_CFG_CMD_PF_UNBIND);
>  
> @@ -137,23 +145,27 @@ int main(int argc, char *argv[])
>  		exit(EXIT_FAILURE);
>  	}
>  
> -	ret = mnl_socket_recvfrom(nl, buf, sizeof(buf));
> -	if (ret == -1) {
> -		perror("mnl_socket_recvfrom");
> -		exit(EXIT_FAILURE);
> -	}
> -	while (ret > 0) {
> -		ret = mnl_cb_run(buf, ret, 0, portid, queue_cb, NULL);
> -		if (ret < 0){
> -			perror("mnl_cb_run");
> -			exit(EXIT_FAILURE);
> -		}
> +	/* ENOBUFS is signalled to userspace when packets were lost
> +	 * on kernel side.  In most cases, userspace isn't interested
> +	 * in this information, so turn it off.
> +	 */
> +	ret = 1;
> +	mnl_socket_setsockopt(nl, NETLINK_NO_ENOBUFS, &ret, sizeof(int));
>  
> -		ret = mnl_socket_recvfrom(nl, buf, sizeof(buf));
> +	for (;;) {
> +		ret = mnl_socket_recvfrom(nl, buf, sizeof_buf);
>  		if (ret == -1) {
> +			if (errno == ENOBUFS) /* messages were lost */

Hm, you disabled ENOBUFS errors, right?

> +				continue;
>  			perror("mnl_socket_recvfrom");
>  			exit(EXIT_FAILURE);
>  		}
> +
> +		ret = mnl_cb_run(buf, ret, 0, portid, queue_cb, NULL);
> +		if (ret < 0){
> +			perror("mnl_cb_run");
> +			exit(EXIT_FAILURE);
> +		}
>  	}
>  
>  	mnl_socket_close(nl);
> -- 
> 1.7.8.6
> 
> --
> To unsubscribe from this list: send the line "unsubscribe netfilter-devel" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH 2/4] examples/nf-queue: handle recv error, use larger buffer

[Florian Westphal]

Pablo Neira Ayuso <pablo@netfilter.org> wrote:
> On Thu, Apr 25, 2013 at 12:43:29PM +0200, Florian Westphal wrote:
> > We ask for 0xffff copy size, so we need a buffer that can
> > hold 0xffff, plus a few more bytes to allow for netlink attributes.
> > 
> > Also, turn off/handle ENOBUFS.
> > 
> > Signed-off-by: Florian Westphal <fw@strlen.de>
> > ---
> >  examples/nf-queue.c |   38 +++++++++++++++++++++++++-------------
> >  1 files changed, 25 insertions(+), 13 deletions(-)
> > 
> > diff --git a/examples/nf-queue.c b/examples/nf-queue.c
> > index 7adac21..57ba483 100644
> > --- a/examples/nf-queue.c
> > +++ b/examples/nf-queue.c
> > @@ -1,3 +1,4 @@
> > +#include <errno.h>
> >  #include <stdio.h>
> >  #include <stdlib.h>
> >  #include <unistd.h>
> > @@ -82,7 +83,8 @@ static int queue_cb(const struct nlmsghdr *nlh, void *data)
> >  
> >  int main(int argc, char *argv[])
> >  {
> > -	char buf[MNL_SOCKET_BUFFER_SIZE];
> > +	char *buf;
> > +	size_t sizeof_buf = 0xffff + 2084;
> 
> I think users will appreciate a comment to explain why those black
> magic numbers are there ;-). Probably using MNL_SOCKET_BUFFER_SIZE/2
> instead of 2084.

Good point :-)
I'll do that and add a comment explaining this, e.g.
"largest possible packet payload, plus netlink data overhead"

> > +	/* ENOBUFS is signalled to userspace when packets were lost
> > +	 * on kernel side.  In most cases, userspace isn't interested
> > +	 * in this information, so turn it off.
> > +	 */
> > +	ret = 1;
> > +	mnl_socket_setsockopt(nl, NETLINK_NO_ENOBUFS, &ret, sizeof(int));
> >  
> > -		ret = mnl_socket_recvfrom(nl, buf, sizeof(buf));
> > +	for (;;) {
> > +		ret = mnl_socket_recvfrom(nl, buf, sizeof_buf);
> >  		if (ret == -1) {
> > +			if (errno == ENOBUFS) /* messages were lost */
> 
> Hm, you disabled ENOBUFS errors, right?

True. I'll remove the check.