* [PATCHv4 iproute2 1/2] lib/libnetlink: re malloc buff if size is not enough
From: Hangbin Liu @ 2017-09-28 13:33 UTC (permalink / raw)
To: netdev; +Cc: Stephen Hemminger, Michal Kubecek, Phil Sutter, Hangbin Liu
In-Reply-To: <1506605626-1744-1-git-send-email-haliu@redhat.com>
From: Hangbin Liu <liuhangbin@gmail.com>
With commit 72b365e8e0fd ("libnetlink: Double the dump buffer size")
we doubled the buffer size to support more VFs. But the VFs number is
increasing all the time. Some customers even use more than 200 VFs now.
We could not double it everytime when the buffer is not enough. Let's just
not hard code the buffer size and malloc the correct number when running.
Introduce function rtnl_recvmsg() to always return a newly allocated buffer.
The caller need to free it after using.
Signed-off-by: Hangbin Liu <liuhangbin@gmail.com>
Signed-off-by: Phil Sutter <phil@nwl.cc>
---
lib/libnetlink.c | 114 ++++++++++++++++++++++++++++++++++++++-----------------
1 file changed, 80 insertions(+), 34 deletions(-)
diff --git a/lib/libnetlink.c b/lib/libnetlink.c
index be7ac86..1847c0b 100644
--- a/lib/libnetlink.c
+++ b/lib/libnetlink.c
@@ -402,6 +402,64 @@ static void rtnl_dump_error(const struct rtnl_handle *rth,
}
}
+static int __rtnl_recvmsg(int fd, struct msghdr *msg, int flags)
+{
+ int len;
+
+ do {
+ len = recvmsg(fd, msg, flags);
+ } while (len < 0 && (errno == EINTR || errno == EAGAIN));
+
+ if (len < 0) {
+ fprintf(stderr, "netlink receive error %s (%d)\n",
+ strerror(errno), errno);
+ return -errno;
+ }
+
+ if (len == 0) {
+ fprintf(stderr, "EOF on netlink\n");
+ return -ENODATA;
+ }
+
+ return len;
+}
+
+static int rtnl_recvmsg(int fd, struct msghdr *msg, char **answer)
+{
+ struct iovec *iov = msg->msg_iov;
+ char *buf;
+ int len;
+
+ iov->iov_base = NULL;
+ iov->iov_len = 0;
+
+ len = __rtnl_recvmsg(fd, msg, MSG_PEEK | MSG_TRUNC);
+ if (len < 0)
+ return len;
+
+ buf = malloc(len);
+ if (!buf) {
+ fprintf(stderr, "malloc error: not enough buffer\n");
+ return -ENOMEM;
+ }
+
+ iov->iov_base = buf;
+ iov->iov_len = len;
+
+ len = __rtnl_recvmsg(fd, msg, 0);
+ if (len < 0) {
+ free(buf);
+ return len;
+ }
+
+ if (answer)
+ *answer = buf;
+ else
+ free(buf);
+
+ return len;
+}
+
int rtnl_dump_filter_l(struct rtnl_handle *rth,
const struct rtnl_dump_filter_arg *arg)
{
@@ -413,31 +471,18 @@ int rtnl_dump_filter_l(struct rtnl_handle *rth,
.msg_iov = &iov,
.msg_iovlen = 1,
};
- char buf[32768];
+ char *buf;
int dump_intr = 0;
- iov.iov_base = buf;
while (1) {
int status;
const struct rtnl_dump_filter_arg *a;
int found_done = 0;
int msglen = 0;
- iov.iov_len = sizeof(buf);
- status = recvmsg(rth->fd, &msg, 0);
-
- if (status < 0) {
- if (errno == EINTR || errno == EAGAIN)
- continue;
- fprintf(stderr, "netlink receive error %s (%d)\n",
- strerror(errno), errno);
- return -1;
- }
-
- if (status == 0) {
- fprintf(stderr, "EOF on netlink\n");
- return -1;
- }
+ status = rtnl_recvmsg(rth->fd, &msg, &buf);
+ if (status < 0)
+ return status;
if (rth->dump_fp)
fwrite(buf, 1, NLMSG_ALIGN(status), rth->dump_fp);
@@ -462,8 +507,10 @@ int rtnl_dump_filter_l(struct rtnl_handle *rth,
if (h->nlmsg_type == NLMSG_DONE) {
err = rtnl_dump_done(h);
- if (err < 0)
+ if (err < 0) {
+ free(buf);
return -1;
+ }
found_done = 1;
break; /* process next filter */
@@ -471,19 +518,23 @@ int rtnl_dump_filter_l(struct rtnl_handle *rth,
if (h->nlmsg_type == NLMSG_ERROR) {
rtnl_dump_error(rth, h);
+ free(buf);
return -1;
}
if (!rth->dump_fp) {
err = a->filter(&nladdr, h, a->arg1);
- if (err < 0)
+ if (err < 0) {
+ free(buf);
return err;
+ }
}
skip_it:
h = NLMSG_NEXT(h, msglen);
}
}
+ free(buf);
if (found_done) {
if (dump_intr)
@@ -543,7 +594,7 @@ static int __rtnl_talk(struct rtnl_handle *rtnl, struct nlmsghdr *n,
.msg_iov = &iov,
.msg_iovlen = 1,
};
- char buf[32768] = {};
+ char *buf;
n->nlmsg_seq = seq = ++rtnl->seq;
@@ -556,22 +607,12 @@ static int __rtnl_talk(struct rtnl_handle *rtnl, struct nlmsghdr *n,
return -1;
}
- iov.iov_base = buf;
while (1) {
- iov.iov_len = sizeof(buf);
- status = recvmsg(rtnl->fd, &msg, 0);
+ status = rtnl_recvmsg(rtnl->fd, &msg, &buf);
+
+ if (status < 0)
+ return status;
- if (status < 0) {
- if (errno == EINTR || errno == EAGAIN)
- continue;
- fprintf(stderr, "netlink receive error %s (%d)\n",
- strerror(errno), errno);
- return -1;
- }
- if (status == 0) {
- fprintf(stderr, "EOF on netlink\n");
- return -1;
- }
if (msg.msg_namelen != sizeof(nladdr)) {
fprintf(stderr,
"sender address length == %d\n",
@@ -585,6 +626,7 @@ static int __rtnl_talk(struct rtnl_handle *rtnl, struct nlmsghdr *n,
if (l < 0 || len > status) {
if (msg.msg_flags & MSG_TRUNC) {
fprintf(stderr, "Truncated message\n");
+ free(buf);
return -1;
}
fprintf(stderr,
@@ -611,6 +653,7 @@ static int __rtnl_talk(struct rtnl_handle *rtnl, struct nlmsghdr *n,
if (answer)
memcpy(answer, h,
MIN(maxlen, h->nlmsg_len));
+ free(buf);
return 0;
}
@@ -619,12 +662,14 @@ static int __rtnl_talk(struct rtnl_handle *rtnl, struct nlmsghdr *n,
rtnl_talk_error(h, err, errfn);
errno = -err->error;
+ free(buf);
return -1;
}
if (answer) {
memcpy(answer, h,
MIN(maxlen, h->nlmsg_len));
+ free(buf);
return 0;
}
@@ -633,6 +678,7 @@ static int __rtnl_talk(struct rtnl_handle *rtnl, struct nlmsghdr *n,
status -= NLMSG_ALIGN(len);
h = (struct nlmsghdr *)((char *)h + NLMSG_ALIGN(len));
}
+ free(buf);
if (msg.msg_flags & MSG_TRUNC) {
fprintf(stderr, "Message truncated\n");
--
2.5.5
^ permalink raw reply related
* [PATCHv4 iproute2 0/2] libnetlink: malloc correct buff at run time
From: Hangbin Liu @ 2017-09-28 13:33 UTC (permalink / raw)
To: netdev; +Cc: Stephen Hemminger, Michal Kubecek, Phil Sutter, Hangbin Liu
From: Hangbin Liu <liuhangbin@gmail.com>
With commit 72b365e8e0fd ("libnetlink: Double the dump buffer size") and
460c03f3f3cc ("iplink: double the buffer size also in iplink_get()"), we
extend the buffer size to avoid truncated message with large numbers of
VFs. But just as Michal said, this is not future-proof since the NIC
number is increasing. We have customer even has 220+ VFs now.
This is not make sense to hard code the buffer and increase it all the time.
So let's just malloc the correct buff size at run time.
Tested with most ip cmds and all look good.
---
v3 -> v4:
* rtnl_recvmsg():
* As Michal suggested, use zero iov len at the first time to avoid
copy same data from kernel to userspace.
* As Stephen suggested, remove loops via goto to make the logic more clear.
* With Phil's help, add __rtnl_recvmsg() to reduce duplicate code.
v2 -> v3:
* rtnl_recvmsg():
* free buf before each return.
* return errno when recvmsg failed.
v1 -> v2 by Phil:
* rtnl_recvmsg():
* Rename output buffer pointer arg to 'answer'.
* Use realloc() and make sure old buffer is freed on error.
* Always return a newly allocated buffer for caller to free.
* Retry on EINTR or EAGAIN so caller doesn't have to.
* Return well-known negative error codes instead of just -1 on error.
* Simplify goto label names.
* If no answer pointer was passed, just free the buffer.
* rtnl_dump_filter_l():
* Don't retry if rtnl_recvmsg() returns 0 as this can't happen
anymore.
* Free buffer returned by rtnl_recvmsg().
* __rtnl_talk():
* Don't retry if rtnl_recvmsg() returns 0 as this can't happen
anymore.
* Free buffer returned by rtnl_recvmsg().
* Return a newly allocated buffer for callers to free.
* genl_ctrl_resolve_family()
* Replace 'ghdr + GENL_HDRLEN' to 'answer + NLMSG_LENGTH(GENL_HDRLEN)'
* tc_action_gd()
* Call print_action() only if cmd == RTM_GETACTION
* Change callers of rtnl_talk*() to always free the answer buffer if
they passed one.
* Drop extra request buffer space in callers if only used for holding
output data.
* Drop initialization of answer pointer if not necessary.
* Change callers to pass NULL instead of answer pointer if they don't
use it afterwards.
Hangbin Liu (2):
lib/libnetlink: re malloc buff if size is not enough
lib/libnetlink: update rtnl_talk to support malloc buff at run time
bridge/fdb.c | 2 +-
bridge/link.c | 2 +-
bridge/mdb.c | 2 +-
bridge/vlan.c | 2 +-
genl/ctrl.c | 19 +++++---
include/libnetlink.h | 6 +--
ip/ipaddress.c | 4 +-
ip/ipaddrlabel.c | 4 +-
ip/ipfou.c | 4 +-
ip/ipila.c | 4 +-
ip/ipl2tp.c | 8 +--
ip/iplink.c | 38 +++++++--------
ip/iplink_vrf.c | 44 ++++++++---------
ip/ipmacsec.c | 2 +-
ip/ipneigh.c | 2 +-
ip/ipnetns.c | 23 +++++----
ip/ipntable.c | 2 +-
ip/iproute.c | 26 ++++++----
ip/iprule.c | 6 +--
ip/ipseg6.c | 8 +--
ip/iptoken.c | 2 +-
ip/link_gre.c | 11 +++--
ip/link_gre6.c | 11 +++--
ip/link_ip6tnl.c | 11 +++--
ip/link_iptnl.c | 10 ++--
ip/link_vti.c | 11 +++--
ip/link_vti6.c | 11 +++--
ip/tcp_metrics.c | 8 +--
ip/xfrm_policy.c | 25 +++++-----
ip/xfrm_state.c | 30 ++++++------
lib/libgenl.c | 9 +++-
lib/libnetlink.c | 134 ++++++++++++++++++++++++++++++++++-----------------
misc/ss.c | 2 +-
tc/m_action.c | 12 ++---
tc/tc_class.c | 2 +-
tc/tc_filter.c | 8 +--
tc/tc_qdisc.c | 2 +-
37 files changed, 298 insertions(+), 209 deletions(-)
--
2.5.5
^ permalink raw reply
* Re: [PATCH net-next] net: ipv4: remove fib_weight
From: David Ahern @ 2017-09-28 13:25 UTC (permalink / raw)
To: David Miller; +Cc: netdev
In-Reply-To: <20170927.215101.2052729160923454927.davem@davemloft.net>
On 9/27/17 10:51 PM, David Miller wrote:
> From: David Ahern <dsahern@gmail.com>
> Date: Wed, 27 Sep 2017 19:08:00 -0700
>
>> fib_weight in fib_info is set but not used. Remove it and the
>> helpers for setting it.
>>
>> Signed-off-by: David Ahern <dsahern@gmail.com>
>
> Hmmm, I wonder then what Peter intended in commit
> 0e884c78ee19e902f300ed147083c28a0c6302f0 ("ipv4: L3 hash-based
> multipath") because that's where this came from.
>
You can see that fib_weight is set but not used in his patch either.
Perhaps it was an early idea that was replaced by the per-nexthop
nh_upper_bound and that code was not removed.
^ permalink raw reply
* RE: [bug][tipc] Too short struct tipc_subscr passed to tipc_subscrb_rcv_cb()
From: Jon Maloy @ 2017-09-28 13:23 UTC (permalink / raw)
To: Alexander Potapenko, David Miller, Ying Xue; +Cc: Networking, syzkaller
In-Reply-To: <CAG_fn=WegzttX52Jq8n52dOxHww8Prgdwnh5_D9kfWVy-nQTTQ@mail.gmail.com>
> -----Original Message-----
> From: Alexander Potapenko [mailto:glider@google.com]
> Sent: Wednesday, September 27, 2017 14:16
> To: David Miller <davem@davemloft.net>; Ying Xue
> <ying.xue@windriver.com>; Jon Maloy <jon.maloy@ericsson.com>
> Cc: Networking <netdev@vger.kernel.org>; syzkaller
> <syzkaller@googlegroups.com>
> Subject: [bug][tipc] Too short struct tipc_subscr passed to
> tipc_subscrb_rcv_cb()
>
> Hi all,
>
> I've spotted the following bug in TIPC server code while fuzzing it with
> KMSAN and syzkaller:
>
> ==========================================================
> ========
> BUG: KMSAN: use of uninitialized memory in
> tipc_subscrb_rcv_cb+0x14a/0xc20
> CPU: 2 PID: 29 Comm: kworker/u8:1 Not tainted 4.13.0+ #3150 Hardware
> name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS Bochs 01/01/2011
> Workqueue: tipc_rcv tipc_recv_work
> Call Trace:
> __dump_stack lib/dump_stack.c:16
> dump_stack+0x172/0x1c0 lib/dump_stack.c:52
> kmsan_report+0x145/0x3d0 mm/kmsan/kmsan.c:943
> __msan_warning_32+0x66/0xb0 mm/kmsan/kmsan_instr.c:477 htohl
> net/tipc/subscr.c:66
> tipc_subscrb_rcv_cb+0x14a/0xc20 net/tipc/subscr.c:333
> tipc_receive_from_sock+0x369/0x540 net/tipc/server.c:273
> tipc_recv_work+0xbe/0x1c0 net/tipc/server.c:546
> process_one_work+0xf9e/0x1ad0 kernel/workqueue.c:2097
> worker_thread+0xefd/0x2090 kernel/workqueue.c:2231
> kthread+0x499/0x620 kernel/kthread.c:232
> ret_from_fork+0x22/0x30 arch/x86/entry/entry_64.S:425
> origin:
> save_stack_trace+0x16/0x20 arch/x86/kernel/stacktrace.c:59
> kmsan_save_stack_with_flags mm/kmsan/kmsan.c:302
> kmsan_internal_poison_shadow+0xb3/0x1b0 mm/kmsan/kmsan.c:198
> kmsan_kmalloc+0x80/0xe0 mm/kmsan/kmsan.c:337
> kmem_cache_alloc+0x1e2/0x220 mm/slub.c:2756
> tipc_receive_from_sock+0xff/0x540 net/tipc/server.c:257
> tipc_recv_work+0xbe/0x1c0 net/tipc/server.c:546
> process_one_work+0xf9e/0x1ad0 kernel/workqueue.c:2097
> worker_thread+0xefd/0x2090 kernel/workqueue.c:2231
> kthread+0x499/0x620 kernel/kthread.c:232
> ret_from_fork+0x22/0x30 arch/x86/entry/entry_64.S:425
> ==========================================================
> ========
> (see the reproducer attached).
>
> Here the user sends 16 bytes via sendmsg(), which the kernel attempts to
> treat as a struct tipc_subscr. Because the actual size of a struct is bigger, we
> end up touching uninitialized fields.
>
> What is the best way to fix this?
>
> I think it's too late to check for |len| in tipc_subscrb_rcv_cb()
> (http://elixir.free-
> electrons.com/linux/latest/source/net/tipc/subscr.c#L320),
> because we'll probably need to call tipc_subscrp_cancel(s, subscriber) in
> order to cancel the subscription, which is a bad idea since |s| is invalid.
> We could check that ret >= sizeof(struct tipc_subscr) in
> tipc_receive_from_sock()
> (http://elixir.free-electrons.com/linux/v4.8/source/net/tipc/server.c#L273),
> but this function is agnostic to the type of the data received by the callback,
> so we'd better not hardcode the size there.
I don't think that is a problem. This server was built to be more generic than what
we really need (maybe because we also had a configuration server at some moment)
but you can safely assume that an incoming message should have the size of a
tipc_subscr, and nothing else.
>
> On a related note, something is wrong with the tipc_receive_from_sock()'s
> return value.
> First, the user appears to receive a positive value (16 in our case) regardless
> of what tipc_receive_from_sock() returns.
> Second, the branch at
> http://elixir.free-electrons.com/linux/v4.8/source/net/tipc/server.c#L288
> is never taken.
I believe the test at the beginning of the loop
while (test_bit(CF_CONNECTED, &con->flags)) {
e = list_entry(con->outqueue.next, struct outqueue_entry,
list);
if ((struct list_head *) e == &con->outqueue)
break;
.....
is incorrect somehow. I was troubleshooting this code myself the last day, and noted that the loop always seems to take an extra loop after the queue is already empty, of course attempting to send crazy values.
I myself am changing this to
while (test_bit(CF_CONNECTED, &con->flags) && !list_empty(&con->outqueue)) {
e = list_first_entry(con->outqueue, struct outqueue_entry, list);
.....
I am also doing some more changes to this, which will be sent out to the tipc-discussion list in the near future.
BR
///jon
>
> WBR,
>
> Alexander Potapenko
> Software Engineer
>
> Google Germany GmbH
> Erika-Mann-Straße, 33
> 80636 München
>
> Geschäftsführer: Paul Manicle, Halimah DeLaine Prado Registergericht und -
> nummer: Hamburg, HRB 86891 Sitz der Gesellschaft: Hamburg
^ permalink raw reply
* Re: [PATCH v4 4/9] em28xx: fix em28xx_dvb_init for KASAN
From: Andrey Ryabinin @ 2017-09-28 13:09 UTC (permalink / raw)
To: Arnd Bergmann
Cc: David Laight, Mauro Carvalho Chehab, Jiri Pirko, Arend van Spriel,
Kalle Valo, David S. Miller, Alexander Potapenko, Dmitry Vyukov,
Masahiro Yamada, Michal Marek, Andrew Morton, Kees Cook,
Geert Uytterhoeven, Greg Kroah-Hartman,
linux-media@vger.kernel.org, linux-kernel@vger.kernel.org,
netdev@vger.kernel.org, "linux-wireless@v
In-Reply-To: <CAK8P3a2C7DBTfQZvRi-QQfrfm1GXktFcXQRmXmzpF4SCa+BADA@mail.gmail.com>
On 09/27/2017 04:26 PM, Arnd Bergmann wrote:
> On Tue, Sep 26, 2017 at 9:49 AM, Andrey Ryabinin
> <aryabinin@virtuozzo.com> wrote:
>>
>>
>> On 09/26/2017 09:47 AM, Arnd Bergmann wrote:
>>> On Mon, Sep 25, 2017 at 11:32 PM, Arnd Bergmann <arnd@arndb.de> wrote:
>
>>> + ret = __builtin_strlen(q);
>>
>>
>> I think this is not correct. Fortified strlen called here on purpose. If sizeof q is known at compile time
>> and 'q' contains not-null fortified strlen() will panic.
>
> Ok, got it.
>
>>> if (size) {
>>> size_t len = (ret >= size) ? size - 1 : ret;
>>> if (__builtin_constant_p(len) && len >= p_size)
>>>
>>> The problem is apparently that the fortified strlcpy calls the fortified strlen,
>>> which in turn calls strnlen and that ends up calling the extern '__real_strnlen'
>>> that gcc cannot reduce to a constant expression for a constant input.
>>
>>
>> Per my observation, it's the code like this:
>> if ()
>> fortify_panic(__func__);
>>
>>
>> somehow prevent gcc to merge several "struct i2c_board_info info;" into one stack slot.
>> With the hack bellow, stack usage reduced to ~1,6K:
>
> 1.6k is also what I see with my patch, or any other approach I tried
> that changes
> string.h. With the split up em28xx_dvb_init() function (and without
> changes to string.h),
> I got down to a few hundred bytes for the largest handler.
>
>> ---
>> include/linux/string.h | 4 ----
>> 1 file changed, 4 deletions(-)
>>
>> diff --git a/include/linux/string.h b/include/linux/string.h
>> index 54d21783e18d..9a96ff3ebf94 100644
>> --- a/include/linux/string.h
>> +++ b/include/linux/string.h
>> @@ -261,8 +261,6 @@ __FORTIFY_INLINE __kernel_size_t strlen(const char *p)
>> if (p_size == (size_t)-1)
>> return __builtin_strlen(p);
>> ret = strnlen(p, p_size);
>> - if (p_size <= ret)
>> - fortify_panic(__func__);
>> return ret;
>> }
>>
>> @@ -271,8 +269,6 @@ __FORTIFY_INLINE __kernel_size_t strnlen(const char *p, __kernel_size_t maxlen)
>> {
>> size_t p_size = __builtin_object_size(p, 0);
>> __kernel_size_t ret = __real_strnlen(p, maxlen < p_size ? maxlen : p_size);
>> - if (p_size <= ret && maxlen != ret)
>> - fortify_panic(__func__);
>> return ret;
>
> I've reduced it further to this change:
>
> --- a/include/linux/string.h
> +++ b/include/linux/string.h
> @@ -227,7 +227,7 @@ static inline const char *kbasename(const char *path)
> #define __FORTIFY_INLINE extern __always_inline __attribute__((gnu_inline))
> #define __RENAME(x) __asm__(#x)
>
> -void fortify_panic(const char *name) __noreturn __cold;
> +void fortify_panic(const char *name) __cold;
> void __read_overflow(void) __compiletime_error("detected read beyond
> size of object passed as 1st parameter");
> void __read_overflow2(void) __compiletime_error("detected read beyond
> size of object passed as 2nd parameter");
> void __read_overflow3(void) __compiletime_error("detected read beyond
> size of object passed as 3rd parameter");
>
> I don't immediately see why the __noreturn changes the behavior here, any idea?
>
At first I thought that this somehow might be related to __asan_handle_no_return(). GCC calls it
before noreturn function. So I made patch to remove generation of these calls (we don't need them in the kernel anyway)
but it didn't help. It must be something else than.
>>> Not sure if that change is the best fix, but it seems to address the problem in
>>> this driver and probably leads to better code in other places as well.
>>>
>>
>> Probably it would be better to solve this on the strlcpy side, but I haven't found the way to do this right.
>> Alternative solutions:
>>
>> - use memcpy() instead of strlcpy(). All source strings are smaller than I2C_NAME_SIZE, so we could
>> do something like this - memcpy(info.type, "si2168", sizeof("si2168"));
>> Also this should be faster.
>
> This would be very similar to the patch I posted at the start of this
> thread to use strncpy(), right?
Sure.
> I was hoping that changing strlcpy() here could also improve other
> users that might run into
> the same situation, but stay below the 2048-byte stack frame limit.
>
>> - Move code under different "case:" in the switch(dev->model) to the separate function should help as well.
>> But it might be harder to backport into stables.
>
> Agreed, I posted this in earlier versions of the patch series, see
> https://patchwork.kernel.org/patch/9601025/
>
> The new patch was a result of me trying to come up with a less
> invasive version to
> make it easier to backport, since I would like to backport the last
> patch in the series
> that depends on all the earlier ones.
>
> Arnd
>
^ permalink raw reply
* [net-next PATCH 5/5] samples/bpf: add cpumap sample program xdp_redirect_cpu
From: Jesper Dangaard Brouer @ 2017-09-28 12:57 UTC (permalink / raw)
To: netdev
Cc: jakub.kicinski, Michael S. Tsirkin, Jason Wang, mchan,
John Fastabend, peter.waskiewicz.jr, Jesper Dangaard Brouer,
Daniel Borkmann, Alexei Starovoitov, Andy Gospodarek
In-Reply-To: <150660339205.2808.7084136789768233829.stgit@firesoul>
This sample program show how to use cpumap and the associated
tracepoints.
It provides command line stats, which shows how the XDP-RX process,
cpumap-enqueue and cpumap kthread dequeue is cooperating on a per CPU
basis. It also utilize the xdp_exception and xdp_redirect_err
transpoints to allow users quickly to identify setup issues.
One issue with ixgbe driver is that the driver reset the link when
loading XDP. This reset the procfs smp_affinity settings. Thus,
after loading the program, these must be reconfigured. The easiest
workaround it to reduce the RX-queue to e.g. two via:
# ethtool --set-channels ixgbe1 combined 2
And then add CPUs above 0 and 1, like:
# xdp_redirect_cpu --dev ixgbe1 --prog 2 --cpu 2 --cpu 3 --cpu 4
Another issue with ixgbe is that the page recycle mechanism is tied to
the RX-ring size. And the default setting of 512 elements is too
small. This is the same issue with regular devmap XDP_REDIRECT.
To overcome this I've been using 1024 rx-ring size:
# ethtool -G ixgbe1 rx 1024 tx 1024
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
---
samples/bpf/Makefile | 4
samples/bpf/xdp_redirect_cpu_kern.c | 640 +++++++++++++++++++++++++++++++++++
samples/bpf/xdp_redirect_cpu_user.c | 639 +++++++++++++++++++++++++++++++++++
3 files changed, 1283 insertions(+)
create mode 100644 samples/bpf/xdp_redirect_cpu_kern.c
create mode 100644 samples/bpf/xdp_redirect_cpu_user.c
diff --git a/samples/bpf/Makefile b/samples/bpf/Makefile
index ebc2ad69b62c..52c4dab2c153 100644
--- a/samples/bpf/Makefile
+++ b/samples/bpf/Makefile
@@ -39,6 +39,7 @@ hostprogs-y += per_socket_stats_example
hostprogs-y += load_sock_ops
hostprogs-y += xdp_redirect
hostprogs-y += xdp_redirect_map
+hostprogs-y += xdp_redirect_cpu
hostprogs-y += xdp_monitor
hostprogs-y += syscall_tp
@@ -84,6 +85,7 @@ test_map_in_map-objs := bpf_load.o $(LIBBPF) test_map_in_map_user.o
per_socket_stats_example-objs := $(LIBBPF) cookie_uid_helper_example.o
xdp_redirect-objs := bpf_load.o $(LIBBPF) xdp_redirect_user.o
xdp_redirect_map-objs := bpf_load.o $(LIBBPF) xdp_redirect_map_user.o
+xdp_redirect_cpu-objs := bpf_load.o $(LIBBPF) xdp_redirect_cpu_user.o
xdp_monitor-objs := bpf_load.o $(LIBBPF) xdp_monitor_user.o
syscall_tp-objs := bpf_load.o $(LIBBPF) syscall_tp_user.o
@@ -129,6 +131,7 @@ always += tcp_iw_kern.o
always += tcp_clamp_kern.o
always += xdp_redirect_kern.o
always += xdp_redirect_map_kern.o
+always += xdp_redirect_cpu_kern.o
always += xdp_monitor_kern.o
always += syscall_tp_kern.o
@@ -169,6 +172,7 @@ HOSTLOADLIBES_xdp_tx_iptunnel += -lelf
HOSTLOADLIBES_test_map_in_map += -lelf
HOSTLOADLIBES_xdp_redirect += -lelf
HOSTLOADLIBES_xdp_redirect_map += -lelf
+HOSTLOADLIBES_xdp_redirect_cpu += -lelf
HOSTLOADLIBES_xdp_monitor += -lelf
HOSTLOADLIBES_syscall_tp += -lelf
diff --git a/samples/bpf/xdp_redirect_cpu_kern.c b/samples/bpf/xdp_redirect_cpu_kern.c
new file mode 100644
index 000000000000..7403e7841a88
--- /dev/null
+++ b/samples/bpf/xdp_redirect_cpu_kern.c
@@ -0,0 +1,640 @@
+/* XDP redirect to CPUs via cpumap (BPF_MAP_TYPE_CPUMAP)
+ *
+ * GPLv2, Copyright(c) 2017 Jesper Dangaard Brouer, Red Hat, Inc.
+ */
+#include <uapi/linux/if_ether.h>
+#include <uapi/linux/if_packet.h>
+#include <uapi/linux/if_vlan.h>
+#include <uapi/linux/ip.h>
+#include <uapi/linux/ipv6.h>
+#include <uapi/linux/in.h>
+#include <uapi/linux/tcp.h>
+#include <uapi/linux/udp.h>
+
+#include <uapi/linux/bpf.h>
+#include "bpf_helpers.h"
+
+#define MAX_CPUS 12 /* WARNING - sync with _user.c */
+
+/* Special map type that can XDP_REDIRECT frames to another CPU */
+struct bpf_map_def SEC("maps") cpu_map = {
+ .type = BPF_MAP_TYPE_CPUMAP,
+ .key_size = sizeof(u32),
+ .value_size = sizeof(u32),
+ .max_entries = MAX_CPUS,
+};
+
+/* Common stats data record to keep userspace more simple */
+struct datarec {
+ __u64 processed;
+ __u64 dropped;
+ __u64 issue;
+};
+
+/* Count RX packets, as XDP bpf_prog doesn't get direct TX-success
+ * feedback. Redirect TX errors can be caught via a tracepoint.
+ */
+struct bpf_map_def SEC("maps") rx_cnt = {
+ .type = BPF_MAP_TYPE_PERCPU_ARRAY,
+ .key_size = sizeof(u32),
+ .value_size = sizeof(struct datarec),
+ .max_entries = 1,
+};
+
+/* Used by trace point */
+struct bpf_map_def SEC("maps") redirect_err_cnt = {
+ .type = BPF_MAP_TYPE_PERCPU_ARRAY,
+ .key_size = sizeof(u32),
+ .value_size = sizeof(struct datarec),
+ .max_entries = 2,
+ /* TODO: have entries for all possible errno's */
+};
+
+/* Used by trace point */
+struct bpf_map_def SEC("maps") cpumap_enqueue_cnt = {
+ .type = BPF_MAP_TYPE_PERCPU_ARRAY,
+ .key_size = sizeof(u32),
+ .value_size = sizeof(struct datarec),
+ .max_entries = MAX_CPUS,
+};
+
+/* Used by trace point */
+struct bpf_map_def SEC("maps") cpumap_kthread_cnt = {
+ .type = BPF_MAP_TYPE_PERCPU_ARRAY,
+ .key_size = sizeof(u32),
+ .value_size = sizeof(struct datarec),
+ .max_entries = 1,
+};
+
+/* Set of maps controlling available CPU, and for iterating through
+ * selectable redirect CPUs.
+ */
+struct bpf_map_def SEC("maps") cpus_available = {
+ .type = BPF_MAP_TYPE_ARRAY,
+ .key_size = sizeof(u32),
+ .value_size = sizeof(u32),
+ .max_entries = MAX_CPUS,
+};
+struct bpf_map_def SEC("maps") cpus_count = {
+ .type = BPF_MAP_TYPE_ARRAY,
+ .key_size = sizeof(u32),
+ .value_size = sizeof(u32),
+ .max_entries = 1,
+};
+struct bpf_map_def SEC("maps") cpus_iterator = {
+ .type = BPF_MAP_TYPE_PERCPU_ARRAY,
+ .key_size = sizeof(u32),
+ .value_size = sizeof(u32),
+ .max_entries = 1,
+};
+
+/* Used by trace point */
+struct bpf_map_def SEC("maps") exception_cnt = {
+ .type = BPF_MAP_TYPE_PERCPU_ARRAY,
+ .key_size = sizeof(u32),
+ .value_size = sizeof(struct datarec),
+ .max_entries = 1,
+};
+
+/* Helper parse functions */
+
+/* Parse Ethernet layer 2, extract network layer 3 offset and protocol
+ *
+ * Returns false on error and non-supported ether-type
+ */
+struct vlan_hdr {
+ __be16 h_vlan_TCI;
+ __be16 h_vlan_encapsulated_proto;
+};
+
+static __always_inline
+bool parse_eth(struct ethhdr *eth, void *data_end,
+ u16 *eth_proto, u64 *l3_offset)
+{
+ u16 eth_type;
+ u64 offset;
+
+ offset = sizeof(*eth);
+ if ((void *)eth + offset > data_end)
+ return false;
+
+ eth_type = eth->h_proto;
+
+ /* Skip non 802.3 Ethertypes */
+ if (unlikely(ntohs(eth_type) < ETH_P_802_3_MIN))
+ return false;
+
+ /* Handle VLAN tagged packet */
+ if (eth_type == htons(ETH_P_8021Q) || eth_type == htons(ETH_P_8021AD)) {
+ struct vlan_hdr *vlan_hdr;
+
+ vlan_hdr = (void *)eth + offset;
+ offset += sizeof(*vlan_hdr);
+ if ((void *)eth + offset > data_end)
+ return false;
+ eth_type = vlan_hdr->h_vlan_encapsulated_proto;
+ }
+ /* TODO: Handle double VLAN tagged packet */
+
+ *eth_proto = ntohs(eth_type);
+ *l3_offset = offset;
+ return true;
+}
+
+static __always_inline
+u16 get_dest_port_ipv4_udp(struct xdp_md *ctx, u64 nh_off)
+{
+ void *data_end = (void *)(long)ctx->data_end;
+ void *data = (void *)(long)ctx->data;
+ struct iphdr *iph = data + nh_off;
+ struct udphdr *udph;
+ u16 dport;
+
+ if (iph + 1 > data_end)
+ return 0;
+ if (!(iph->protocol == IPPROTO_UDP))
+ return 0;
+
+ udph = (void *)(iph + 1);
+ if (udph + 1 > data_end)
+ return 0;
+
+ dport = ntohs(udph->dest);
+ return dport;
+}
+
+static __always_inline
+int get_proto_ipv4(struct xdp_md *ctx, u64 nh_off)
+{
+ void *data_end = (void *)(long)ctx->data_end;
+ void *data = (void *)(long)ctx->data;
+ struct iphdr *iph = data + nh_off;
+
+ if (iph + 1 > data_end)
+ return 0;
+ return iph->protocol;
+}
+
+static __always_inline
+int get_proto_ipv6(struct xdp_md *ctx, u64 nh_off)
+{
+ void *data_end = (void *)(long)ctx->data_end;
+ void *data = (void *)(long)ctx->data;
+ struct ipv6hdr *ip6h = data + nh_off;
+
+ if (ip6h + 1 > data_end)
+ return 0;
+ return ip6h->nexthdr;
+}
+
+SEC("xdp_cpu_map0")
+int xdp_prognum0_no_touch(struct xdp_md *ctx)
+{
+ void *data_end = (void *)(long)ctx->data_end;
+ void *data = (void *)(long)ctx->data;
+ struct datarec* rec;
+ u32 cpu_dest;
+ u32 key = 0;
+
+ /* Only use first entry in cpus_available */
+ u32 *cpu_selected;
+ cpu_selected = bpf_map_lookup_elem(&cpus_available, &key);
+ if (!cpu_selected)
+ return XDP_ABORTED;
+ cpu_dest = *cpu_selected;
+
+ /* Count RX packet in map */
+ rec = bpf_map_lookup_elem(&rx_cnt, &key);
+ if (rec)
+ rec->processed++;
+
+ return bpf_redirect_map(&cpu_map, cpu_dest, 0);
+}
+
+SEC("xdp_cpu_map1_touch_data")
+int xdp_prognum1_touch_data(struct xdp_md *ctx)
+{
+ void *data_end = (void *)(long)ctx->data_end;
+ void *data = (void *)(long)ctx->data;
+ struct ethhdr *eth = data;
+ volatile u16 eth_type;
+ struct datarec* rec;
+ u32 cpu_dest;
+ u32 key = 0;
+
+ /* Only use first entry in cpus_available */
+ u32 *cpu_selected;
+ cpu_selected = bpf_map_lookup_elem(&cpus_available, &key);
+ if (!cpu_selected)
+ return XDP_ABORTED;
+ cpu_dest = *cpu_selected;
+
+ /* Validate packet length is minimum Eth header size */
+ if (eth + 1 > data_end) {
+ return XDP_ABORTED;
+ }
+
+ /* Count RX packet in map */
+ rec = bpf_map_lookup_elem(&rx_cnt, &key);
+ if (!rec)
+ return XDP_ABORTED;
+ rec->processed++;
+
+ /* Read packet data, and use it (drop non 802.3 Ethertypes) */
+ eth_type = eth->h_proto;
+ if (ntohs(eth_type) < ETH_P_802_3_MIN) {
+ rec->dropped++;
+ return XDP_DROP;
+ }
+
+ return bpf_redirect_map(&cpu_map, cpu_dest, 0);
+}
+
+SEC("xdp_cpu_map2_round_robin")
+int xdp_prognum2_round_robin(struct xdp_md *ctx)
+{
+ void *data_end = (void *)(long)ctx->data_end;
+ void *data = (void *)(long)ctx->data;
+ struct ethhdr *eth = data;
+ struct datarec* rec;
+ u32 cpu_dest;
+ u32 *cpu_lookup;
+ u32 key0 = 0;
+
+ u32 *cpu_selected;
+ u32 *cpu_iterator;
+ u32 *cpu_max;
+ u32 cpu_idx;
+
+ cpu_max = bpf_map_lookup_elem(&cpus_count, &key0);
+ if (!cpu_max)
+ return XDP_ABORTED;
+
+ cpu_iterator = bpf_map_lookup_elem(&cpus_iterator, &key0);
+ if (!cpu_iterator)
+ return XDP_ABORTED;
+ cpu_idx = *cpu_iterator;
+
+ *cpu_iterator += 1;
+ if (*cpu_iterator == *cpu_max)
+ *cpu_iterator = 0;
+
+ cpu_selected = bpf_map_lookup_elem(&cpus_available, &cpu_idx);
+ if (!cpu_selected)
+ return XDP_ABORTED;
+ cpu_dest = *cpu_selected;
+
+ /* Count RX packet in map */
+ rec = bpf_map_lookup_elem(&rx_cnt, &key0);
+ if (!rec)
+ return XDP_ABORTED;
+ rec->processed++;
+
+ /* Check cpu_dest is valid */
+ cpu_lookup = bpf_map_lookup_elem(&cpu_map, &cpu_dest);
+ if (!cpu_lookup) {
+ rec->issue++;
+ return XDP_DROP;
+ }
+
+ if (cpu_dest >= MAX_CPUS )
+ return XDP_ABORTED;
+
+ return bpf_redirect_map(&cpu_map, cpu_dest, 0);
+}
+
+SEC("xdp_cpu_map3_proto_separate")
+int xdp_prognum3_proto_separate(struct xdp_md *ctx)
+{
+ void *data_end = (void *)(long)ctx->data_end;
+ void *data = (void *)(long)ctx->data;
+ struct ethhdr *eth = data;
+ u8 ip_proto = IPPROTO_UDP;
+ struct datarec* rec;
+ u16 eth_proto = 0;
+ u64 l3_offset = 0;
+ u32 cpu_dest = 0;
+ u32 cpu_idx = 0;
+ u32 *cpu_lookup;
+ u32 key = 0;
+
+ /* Count RX packet in map */
+ rec = bpf_map_lookup_elem(&rx_cnt, &key);
+ if (!rec)
+ return XDP_ABORTED;
+ rec->processed++;
+
+ if (!(parse_eth(eth, data_end, ð_proto, &l3_offset))) {
+ return XDP_PASS; /* Just skip */
+ }
+
+ /* Extract L4 protocol */
+ switch (eth_proto) {
+ case ETH_P_IP:
+ ip_proto = get_proto_ipv4(ctx, l3_offset);
+ break;
+ case ETH_P_IPV6:
+ ip_proto = get_proto_ipv6(ctx, l3_offset);
+ break;
+ case ETH_P_ARP:
+ cpu_idx = 0; /* ARP packet handled on separate CPU */
+ break;
+ default:
+ cpu_idx = 0;
+ }
+
+ /* Choose CPU based on L4 protocol */
+ switch (ip_proto) {
+ case IPPROTO_ICMP:
+ case IPPROTO_ICMPV6:
+ cpu_idx = 2;
+ break;
+ case IPPROTO_TCP:
+ cpu_idx = 0;
+ break;
+ case IPPROTO_UDP:
+ cpu_idx = 1;
+ break;
+ default:
+ cpu_idx = 0;
+ }
+
+ cpu_lookup = bpf_map_lookup_elem(&cpus_available, &cpu_idx);
+ if (!cpu_lookup)
+ return XDP_ABORTED;
+ cpu_dest = *cpu_lookup;
+
+ if (cpu_dest >= MAX_CPUS )
+ return XDP_ABORTED;
+
+ /* Check cpu_dest is valid */
+ cpu_lookup = bpf_map_lookup_elem(&cpu_map, &cpu_dest);
+ if (!cpu_lookup) {
+ rec->issue++;
+ return XDP_DROP;
+ }
+
+ return bpf_redirect_map(&cpu_map, cpu_dest, 0);
+}
+
+SEC("xdp_cpu_map4_ddos_filter_pktgen")
+int xdp_prognum4_ddos_filter_pktgen(struct xdp_md *ctx)
+{
+ void *data_end = (void *)(long)ctx->data_end;
+ void *data = (void *)(long)ctx->data;
+ struct ethhdr *eth = data;
+ u8 ip_proto = IPPROTO_UDP;
+ struct datarec* rec;
+ u16 eth_proto = 0;
+ u64 l3_offset = 0;
+ u32 cpu_dest = 0;
+ u32 cpu_idx = 0;
+ u16 dest_port;
+ u32 *cpu_lookup;
+ u32 key = 0;
+
+ /* Count RX packet in map */
+ rec = bpf_map_lookup_elem(&rx_cnt, &key);
+ if (!rec)
+ return XDP_ABORTED;
+ rec->processed++;
+
+ if (!(parse_eth(eth, data_end, ð_proto, &l3_offset))) {
+ return XDP_PASS; /* Just skip */
+ }
+
+ /* Extract L4 protocol */
+ switch (eth_proto) {
+ case ETH_P_IP:
+ ip_proto = get_proto_ipv4(ctx, l3_offset);
+ break;
+ case ETH_P_IPV6:
+ ip_proto = get_proto_ipv6(ctx, l3_offset);
+ break;
+ case ETH_P_ARP:
+ cpu_idx = 0; /* ARP packet handled on separate CPU */
+ break;
+ default:
+ cpu_idx = 0;
+ }
+
+ /* Choose CPU based on L4 protocol */
+ switch (ip_proto) {
+ case IPPROTO_ICMP:
+ case IPPROTO_ICMPV6:
+ cpu_idx = 2;
+ break;
+ case IPPROTO_TCP:
+ cpu_idx = 0;
+ break;
+ case IPPROTO_UDP:
+ cpu_idx = 1;
+ /* DDoS filter UDP port 9 (pktgen) */
+ dest_port = get_dest_port_ipv4_udp(ctx, l3_offset);
+ if (dest_port == 9) {
+ if (rec)
+ rec->dropped++;
+ return XDP_DROP;
+ }
+ break;
+ default:
+ cpu_idx = 0;
+ }
+
+ cpu_lookup = bpf_map_lookup_elem(&cpus_available, &cpu_idx);
+ if (!cpu_lookup)
+ return XDP_ABORTED;
+ cpu_dest = *cpu_lookup;
+
+ if (cpu_dest >= MAX_CPUS )
+ return XDP_ABORTED;
+
+ /* Check cpu_dest is valid */
+ cpu_lookup = bpf_map_lookup_elem(&cpu_map, &cpu_dest);
+ if (!cpu_lookup) {
+ rec->issue++;
+ return XDP_DROP;
+ }
+
+ if (cpu_dest >= MAX_CPUS )
+ return XDP_ABORTED;
+
+ return bpf_redirect_map(&cpu_map, cpu_dest, 0);
+}
+
+
+char _license[] SEC("license") = "GPL";
+
+/*** Trace point code ***/
+
+/* Tracepoint format: /sys/kernel/debug/tracing/events/xdp/xdp_redirect/format
+ * Code in: kernel/include/trace/events/xdp.h
+ */
+struct xdp_redirect_ctx {
+ unsigned short common_type; // offset:0; size:2; signed:0;
+ unsigned char common_flags; // offset:2; size:1; signed:0;
+ unsigned char common_preempt_count;// offset:3; size:1; signed:0;
+ int common_pid; // offset:4; size:4; signed:1;
+
+ int prog_id; // offset:8; size:4; signed:1;
+ u32 act; // offset:12 size:4; signed:0;
+ int ifindex; // offset:16 size:4; signed:1;
+ int err; // offset:20 size:4; signed:1;
+ int to_ifindex; // offset:24 size:4; signed:1;
+ u32 map_id; // offset:28 size:4; signed:0;
+ int map_index; // offset:32 size:4; signed:1;
+}; // offset:36
+
+enum {
+ XDP_REDIRECT_SUCCESS = 0,
+ XDP_REDIRECT_ERROR = 1
+};
+
+static __always_inline
+int xdp_redirect_collect_stat(struct xdp_redirect_ctx *ctx)
+{
+ u32 key = XDP_REDIRECT_ERROR;
+ struct datarec *rec;
+ int err = ctx->err;
+
+ if (!err)
+ key = XDP_REDIRECT_SUCCESS;
+
+ rec = bpf_map_lookup_elem(&redirect_err_cnt, &key);
+ if (!rec)
+ return 0;
+ rec->dropped += 1;
+
+ return 0; /* Indicate event was filtered (no further processing)*/
+ /*
+ * Returning 1 here would allow e.g. a perf-record tracepoint
+ * to see and record these events, but it doesn't work well
+ * in-practice as stopping perf-record also unload this
+ * bpf_prog. Plus, there is additional overhead of doing so.
+ */
+}
+
+SEC("tracepoint/xdp/xdp_redirect_err")
+int trace_xdp_redirect_err(struct xdp_redirect_ctx *ctx)
+{
+ return xdp_redirect_collect_stat(ctx);
+}
+
+
+SEC("tracepoint/xdp/xdp_redirect_map_err")
+int trace_xdp_redirect_map_err(struct xdp_redirect_ctx *ctx)
+{
+ return xdp_redirect_collect_stat(ctx);
+}
+
+/* Tracepoint format: /sys/kernel/debug/tracing/events/xdp/xdp_exception/format
+ * Code in: kernel/include/trace/events/xdp.h
+ */
+struct xdp_exception_ctx {
+ unsigned short common_type; // offset:0; size:2; signed:0;
+ unsigned char common_flags; // offset:2; size:1; signed:0;
+ unsigned char common_preempt_count;// offset:3; size:1; signed:0;
+ int common_pid; // offset:4; size:4; signed:1;
+
+ int prog_id; // offset:8; size:4; signed:1;
+ u32 act; // offset:12; size:4; signed:0;
+ int ifindex; // offset:16; size:4; signed:1;
+};
+
+SEC("tracepoint/xdp/xdp_exception")
+int trace_xdp_exception(struct xdp_exception_ctx *ctx)
+{
+ struct datarec *rec;
+ u32 key = 0;
+
+ rec = bpf_map_lookup_elem(&exception_cnt, &key);
+ if (!rec)
+ return 1;
+ rec->dropped += 1;
+
+ return 0;
+}
+
+/* Tracepoint: /sys/kernel/debug/tracing/events/xdp/xdp_cpumap_enqueue/format
+ * Code in: kernel/include/trace/events/xdp.h
+ */
+struct cpumap_enqueue_ctx {
+ unsigned short common_type; // offset:0; size:2; signed:0;
+ unsigned char common_flags; // offset:2; size:1; signed:0;
+ unsigned char common_preempt_count;// offset:3; size:1; signed:0;
+ int common_pid; // offset:4; size:4; signed:1;
+
+ int map_id; // offset:8; size:4; signed:1;
+ u32 act; // offset:12; size:4; signed:0;
+ int cpu; // offset:16; size:4; signed:1;
+ unsigned int drops; // offset:20; size:4; signed:0;
+ unsigned int processed; // offset:24; size:4; signed:0;
+ int to_cpu; // offset:28; size:4; signed:1;
+};
+
+SEC("tracepoint/xdp/xdp_cpumap_enqueue")
+int trace_xdp_cpumap_enqueue(struct cpumap_enqueue_ctx *ctx)
+{
+ u32 to_cpu = ctx->to_cpu;
+ struct datarec *rec;
+
+ if (to_cpu >= MAX_CPUS)
+ return 1;
+
+ rec = bpf_map_lookup_elem(&cpumap_enqueue_cnt, &to_cpu);
+ if (!rec)
+ return 0;
+ rec->processed += ctx->processed;
+ rec->dropped += ctx->drops;
+
+ /* Detect misconfig. Redirect to "same" CPU, makes no sense
+ * and indicate user of cpumap have not done proper IRQ RXq
+ * setup.
+ */
+ if (ctx->cpu == ctx->to_cpu)
+ rec->issue += ctx->processed;
+
+ /* Keep seperate map for feedback loop */
+ // have map that boolean mark drops, and RX side can clean
+ // this, indicating it have got the notification. TODO, should
+ // this also contain a (k)timestamp.
+
+ return 0;
+}
+
+/* Tracepoint: /sys/kernel/debug/tracing/events/xdp/xdp_cpumap_kthread/format
+ * Code in: kernel/include/trace/events/xdp.h
+ */
+struct cpumap_kthread_ctx {
+ unsigned short common_type; // offset:0; size:2; signed:0;
+ unsigned char common_flags; // offset:2; size:1; signed:0;
+ unsigned char common_preempt_count;// offset:3; size:1; signed:0;
+ int common_pid; // offset:4; size:4; signed:1;
+
+ int map_id; // offset:8; size:4; signed:1;
+ u32 act; // offset:12; size:4; signed:0;
+ int cpu; // offset:16; size:4; signed:1;
+ unsigned int drops; // offset:20; size:4; signed:0;
+ unsigned int processed; // offset:24; size:4; signed:0;
+ int time_limit; // offset:28; size:4; signed:1;
+};
+
+SEC("tracepoint/xdp/xdp_cpumap_kthread")
+int trace_xdp_cpumap_kthread(struct cpumap_kthread_ctx *ctx)
+{
+ struct datarec *rec;
+ u32 key = 0;
+
+ rec = bpf_map_lookup_elem(&cpumap_kthread_cnt, &key);
+ if (!rec)
+ return 0;
+ rec->processed += ctx->processed;
+ rec->dropped += ctx->drops;
+
+ /* Detect when time limit was exceeded, but queue was not-empty */
+ if (ctx->processed > 0 && ctx->time_limit)
+ rec->issue++;
+
+ return 0;
+}
+
diff --git a/samples/bpf/xdp_redirect_cpu_user.c b/samples/bpf/xdp_redirect_cpu_user.c
new file mode 100644
index 000000000000..c2c971ab7078
--- /dev/null
+++ b/samples/bpf/xdp_redirect_cpu_user.c
@@ -0,0 +1,639 @@
+/* GPLv2 Copyright(c) 2017 Jesper Dangaard Brouer, Red Hat, Inc.
+ */
+static const char *__doc__=
+ " XDP redirect with a CPU-map type \"BPF_MAP_TYPE_CPUMAP\"";
+
+#include <errno.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdbool.h>
+#include <string.h>
+#include <unistd.h>
+#include <locale.h>
+#include <sys/resource.h>
+#include <getopt.h>
+#include <net/if.h>
+#include <time.h>
+
+#include <arpa/inet.h>
+#include <linux/if_link.h>
+
+#define MAX_CPUS 12 /* WARNING - sync with _kern.c */
+
+/* How many xdp_progs are defined in _kern.c */
+#define MAX_PROG 5
+
+/* Wanted to get rid of bpf_load.h and fake-"libbpf.h" (and instead
+ * use bpf/libbpf.h), but cannot as (currently) needed for XDP
+ * attaching to a device via set_link_xdp_fd()
+ */
+#include "libbpf.h"
+#include "bpf_load.h"
+
+#include "bpf_util.h"
+
+static int ifindex = -1;
+static char ifname_buf[IF_NAMESIZE];
+static char *ifname = NULL;
+static __u32 xdp_flags = 0;
+
+/* Exit return codes */
+#define EXIT_OK 0
+#define EXIT_FAIL 1
+#define EXIT_FAIL_OPTION 2
+#define EXIT_FAIL_XDP 3
+#define EXIT_FAIL_BPF 4
+#define EXIT_FAIL_MEM 5
+
+static const struct option long_options[] = {
+ {"help", no_argument, NULL, 'h' },
+ {"dev", required_argument, NULL, 'd' },
+ {"skb-mode", no_argument, NULL, 'S' },
+ {"debug", no_argument, NULL, 'D' },
+ {"sec", required_argument, NULL, 's' },
+ {"prognum", required_argument, NULL, 'p' },
+ {"qsize", required_argument, NULL, 'q' },
+ {"cpu", required_argument, NULL, 'c' },
+ {"no-separators",no_argument, NULL, 'z' },
+ {0, 0, NULL, 0 }
+};
+
+static void int_exit(int sig)
+{
+ fprintf(stderr,
+ "Interrupted: Removing XDP program on ifindex:%d device:%s\n",
+ ifindex, ifname);
+ if (ifindex > -1)
+ set_link_xdp_fd(ifindex, -1, xdp_flags);
+ exit(EXIT_OK);
+}
+
+static void usage(char *argv[])
+{
+ int i;
+ printf("\nDOCUMENTATION:\n%s\n", __doc__);
+ printf("\n");
+ printf(" Usage: %s (options-see-below)\n",
+ argv[0]);
+ printf(" Listing options:\n");
+ for (i = 0; long_options[i].name != 0; i++) {
+ printf(" --%-12s", long_options[i].name);
+ if (long_options[i].flag != NULL)
+ printf(" flag (internal value:%d)",
+ *long_options[i].flag);
+ else
+ printf(" short-option: -%c",
+ long_options[i].val);
+ printf("\n");
+ }
+ printf("\n");
+}
+
+/* gettime returns the current time of day in nanoseconds.
+ * Cost: clock_gettime (ns) => 26ns (CLOCK_MONOTONIC)
+ * clock_gettime (ns) => 9ns (CLOCK_MONOTONIC_COARSE)
+ */
+#define NANOSEC_PER_SEC 1000000000 /* 10^9 */
+static __u64 gettime(void)
+{
+ struct timespec t;
+ int res;
+
+ res = clock_gettime(CLOCK_MONOTONIC, &t);
+ if (res < 0) {
+ fprintf(stderr, "Error with gettimeofday! (%i)\n", res);
+ exit(EXIT_FAIL);
+ }
+ return (__u64) t.tv_sec * NANOSEC_PER_SEC + t.tv_nsec;
+}
+
+/* Common stats data record shared with _kern.c */
+struct datarec {
+ __u64 processed;
+ __u64 dropped;
+ __u64 issue;
+};
+struct record {
+ __u64 timestamp;
+ struct datarec total;
+ struct datarec *cpu;
+};
+struct stats_record {
+ struct record rx_cnt;
+ struct record redir_err;
+ struct record kthread;
+ struct record exception;
+ struct record enq[MAX_CPUS];
+};
+
+static bool map_collect_percpu(int fd, __u32 key, struct record* rec)
+{
+ /* For percpu maps, userspace gets a value per possible CPU */
+ unsigned int nr_cpus = bpf_num_possible_cpus();
+ struct datarec values[nr_cpus];
+ __u64 sum_processed = 0;
+ __u64 sum_dropped = 0;
+ __u64 sum_issue = 0;
+ int i;
+
+ if ((bpf_map_lookup_elem(fd, &key, values)) != 0) {
+ fprintf(stderr,
+ "ERR: bpf_map_lookup_elem failed key:0x%X\n", key);
+ return false;
+ }
+ /* Get time as close as possible to reading map contents */
+ rec->timestamp = gettime();
+
+ /* Record and sum values from each CPU */
+ for (i = 0; i < nr_cpus; i++) {
+ rec->cpu[i].processed = values[i].processed;
+ sum_processed += values[i].processed;
+ rec->cpu[i].dropped = values[i].dropped;
+ sum_dropped += values[i].dropped;
+ rec->cpu[i].issue = values[i].issue;
+ sum_issue += values[i].issue;
+ }
+ rec->total.processed = sum_processed;
+ rec->total.dropped = sum_dropped;
+ rec->total.issue = sum_issue;
+ return true;
+}
+
+static struct datarec *alloc_record_per_cpu(void)
+{
+ unsigned int nr_cpus = bpf_num_possible_cpus();
+ struct datarec *array;
+ size_t size;
+
+ size = sizeof(struct datarec) * nr_cpus;
+ array = malloc(size);
+ memset(array, 0, size);
+ if (!array) {
+ fprintf(stderr, "Mem alloc error (nr_cpus:%u)\n", nr_cpus);
+ exit(EXIT_FAIL_MEM);
+ }
+ return array;
+}
+
+static struct stats_record* alloc_stats_record(void)
+{
+ struct stats_record* rec;
+ int i;
+
+ rec = malloc(sizeof(*rec));
+ memset(rec, 0, sizeof(*rec));
+ if (!rec) {
+ fprintf(stderr, "Mem alloc error\n");
+ exit(EXIT_FAIL_MEM);
+ }
+ rec->rx_cnt.cpu = alloc_record_per_cpu();
+ rec->redir_err.cpu = alloc_record_per_cpu();
+ rec->kthread.cpu = alloc_record_per_cpu();
+ rec->exception.cpu = alloc_record_per_cpu();
+ for (i = 0; i < MAX_CPUS; i++)
+ rec->enq[i].cpu = alloc_record_per_cpu();
+
+ return rec;
+}
+
+static void free_stats_record(struct stats_record* r)
+{
+ int i;
+
+ for (i = 0; i < MAX_CPUS; i++)
+ free(r->enq[i].cpu);
+ free(r->exception.cpu);
+ free(r->kthread.cpu);
+ free(r->redir_err.cpu);
+ free(r->rx_cnt.cpu);
+ free(r);
+}
+
+static double calc_period(struct record *r, struct record *p)
+{
+ double period_ = 0;
+ __u64 period = 0;
+
+ period = r->timestamp - p->timestamp;
+ if (period > 0) {
+ period_ = ((double) period / NANOSEC_PER_SEC);
+ }
+ return period_;
+}
+
+static __u64 calc_pps(struct datarec *r, struct datarec *p, double period_)
+{
+ __u64 packets = 0;
+ __u64 pps = 0;
+
+ if (period_ > 0) {
+ packets = r->processed - p->processed;
+ pps = packets / period_;
+ }
+ return pps;
+}
+
+static __u64 calc_drop_pps(struct datarec *r, struct datarec *p, double period_)
+{
+ __u64 packets = 0;
+ __u64 pps = 0;
+
+ if (period_ > 0) {
+ packets = r->dropped - p->dropped;
+ pps = packets / period_;
+ }
+ return pps;
+}
+
+static __u64 calc_errs_pps(struct datarec *r,
+ struct datarec *p, double period_)
+{
+ __u64 packets = 0;
+ __u64 pps = 0;
+
+ if (period_ > 0) {
+ packets = r->issue - p->issue;
+ pps = packets / period_;
+ }
+ return pps;
+}
+
+static void stats_print(struct stats_record *stats_rec,
+ struct stats_record *stats_prev,
+ int prog_num)
+{
+ unsigned int nr_cpus = bpf_num_possible_cpus();
+ double pps = 0, drop = 0, err = 0;
+ struct record *rec, *prev;
+ int to_cpu;
+ double t;
+ int i;
+
+ /* Header */
+ printf("Running XDP/eBPF prog_num:%d\n", prog_num);
+ printf("%-15s %-7s %-14s %-11s %-9s\n",
+ "XDP-cpumap", "CPU:to", "pps", "drop-pps", "extra-info");
+
+ /* XDP rx_cnt */
+ {
+ char * fmt_rx = "%-15s %-7d %'-14.0f %'-11.0f %'-10.0f %s\n";
+ char * fm2_rx = "%-15s %-7s %'-14.0f %'-11.0f\n";
+ char *errstr = "";
+
+ rec = &stats_rec->rx_cnt;
+ prev = &stats_prev->rx_cnt;
+ t = calc_period(rec, prev);
+ for (i = 0; i < nr_cpus; i++) {
+ struct datarec *r = &rec->cpu[i];
+ struct datarec *p = &prev->cpu[i];
+ pps = calc_pps(r, p, t);
+ drop = calc_drop_pps(r, p, t);
+ err = calc_errs_pps(r, p, t);
+ if (err > 0)
+ errstr = "cpu-dest/err";
+ if (pps > 0)
+ printf(fmt_rx, "XDP-RX",
+ i, pps, drop, err, errstr);
+ }
+ pps = calc_pps(&rec->total, &prev->total, t);
+ drop = calc_drop_pps(&rec->total, &prev->total, t);
+ err = calc_errs_pps(&rec->total, &prev->total, t);
+ printf(fm2_rx, "XDP-RX", "total", pps, drop);
+ }
+
+ /* cpumap enqueue stats */
+ for (to_cpu = 0; to_cpu < MAX_CPUS; to_cpu++) {
+ char *fmt="%-15s %3d:%-3d %'-14.0f %'-11.0f %'-10.0f %s\n";
+ char *fm2="%-15s %3s:%-3d %'-14.0f %'-11.0f %'-10.0f %s\n";
+ char *errstr = "";
+
+ rec = &stats_rec->enq[to_cpu];
+ prev = &stats_prev->enq[to_cpu];
+ t = calc_period(rec, prev);
+ for (i = 0; i < nr_cpus; i++) {
+ struct datarec *r = &rec->cpu[i];
+ struct datarec *p = &prev->cpu[i];
+ pps = calc_pps(r, p, t);
+ drop = calc_drop_pps(r, p, t);
+ err = calc_errs_pps(r, p, t);
+ if (err > 0)
+ errstr = "same-cpu/pps";
+ if (pps > 0)
+ printf(fmt, "cpumap-enqueue",
+ i, to_cpu, pps, drop, err, errstr);
+ }
+ pps = calc_pps(&rec->total, &prev->total, t);
+ if (pps > 0) {
+ drop = calc_drop_pps(&rec->total, &prev->total, t);
+ err = calc_errs_pps(&rec->total, &prev->total, t);
+ printf(fm2, "cpumap-enqueue",
+ "sum", to_cpu, pps, drop, err, errstr);
+ }
+ }
+
+ /* cpumap kthread stats */
+ {
+ char *fmt_k = "%-15s %-7d %'-14.0f %'-11.0f %-10.0f %s\n";
+ char *fm2_k = "%-15s %-7s %'-14.0f %'-11.0f %-10.0f %s\n";
+ char *errstr = "";
+ rec = &stats_rec->kthread;
+ prev = &stats_prev->kthread;
+ t = calc_period(rec, prev);
+ for (i = 0; i < nr_cpus; i++) {
+ struct datarec *r = &rec->cpu[i];
+ struct datarec *p = &prev->cpu[i];
+ pps = calc_pps(r, p, t);
+ drop = calc_drop_pps(r, p, t);
+ err = calc_errs_pps(r, p, t);
+ if (err > 0)
+ errstr = "time_exceed";
+ if (pps > 0)
+ printf(fmt_k, "cpumap_kthread",
+ i, pps, drop, err, errstr);
+ }
+ pps = calc_pps(&rec->total, &prev->total, t);
+ drop = calc_drop_pps(&rec->total, &prev->total, t);
+ printf(fm2_k, "cpumap_kthread", "total", pps, drop);
+ }
+
+ /* XDP redirect err tracepoints (very unlikely) */
+ {
+ char *fmt_err = "%-15s %-7d %'-14.0f %'-11.0f\n";
+ char *fm2_err = "%-15s %-7s %'-14.0f %'-11.0f\n";
+ rec = &stats_rec->redir_err;
+ prev = &stats_prev->redir_err;
+ t = calc_period(rec, prev);
+ for (i = 0; i < nr_cpus; i++) {
+ struct datarec *r = &rec->cpu[i];
+ struct datarec *p = &prev->cpu[i];
+ pps = calc_pps(r, p, t);
+ drop = calc_drop_pps(r, p, t);
+ if (pps > 0)
+ printf(fmt_err, "redirect_err", i, pps, drop);
+ }
+ pps = calc_pps(&rec->total, &prev->total, t);
+ drop = calc_drop_pps(&rec->total, &prev->total, t);
+ printf(fm2_err, "redirect_err", "total", pps, drop);
+ }
+
+ /* XDP general exception tracepoints */
+ {
+ char *fmt_err = "%-15s %-7d %'-14.0f %'-11.0f\n";
+ char *fm2_err = "%-15s %-7s %'-14.0f %'-11.0f\n";
+ rec = &stats_rec->exception;
+ prev = &stats_prev->exception;
+ t = calc_period(rec, prev);
+ for (i = 0; i < nr_cpus; i++) {
+ struct datarec *r = &rec->cpu[i];
+ struct datarec *p = &prev->cpu[i];
+ pps = calc_pps(r, p, t);
+ drop = calc_drop_pps(r, p, t);
+ if (pps > 0)
+ printf(fmt_err, "xdp_exception", i, pps, drop);
+ }
+ pps = calc_pps(&rec->total, &prev->total, t);
+ drop = calc_drop_pps(&rec->total, &prev->total, t);
+ printf(fm2_err, "xdp_exception", "total", pps, drop);
+ }
+
+ printf("\n");
+ fflush(stdout);
+}
+
+static void stats_collect(struct stats_record *rec)
+{
+ int fd, i;
+
+ fd = map_fd[1]; /* map: rx_cnt */
+ map_collect_percpu(fd, 0, &rec->rx_cnt);
+
+ fd = map_fd[2]; /* map: redirect_err_cnt */
+ map_collect_percpu(fd, 1, &rec->redir_err);
+
+ fd = map_fd[3]; /* map: cpumap_enqueue_cnt */
+ for (i = 0; i < MAX_CPUS; i++) {
+ map_collect_percpu(fd, i, &rec->enq[i]);
+ }
+
+ fd = map_fd[4]; /* map: cpumap_kthread_cnt */
+ map_collect_percpu(fd, 0, &rec->kthread);
+
+ fd = map_fd[8]; /* map: exception_cnt */
+ map_collect_percpu(fd, 0, &rec->exception);
+}
+
+
+/* Pointer swap trick */
+static inline void swap(struct stats_record **a, struct stats_record **b)
+{
+ struct stats_record *tmp;
+
+ tmp = *a;
+ *a = *b;
+ *b = tmp;
+}
+
+static void stats_poll(int interval, bool use_separators, int prog_num)
+{
+ struct stats_record *record, *prev;
+
+ record = alloc_stats_record();
+ prev = alloc_stats_record();
+ stats_collect(record);
+
+ /* Trick to pretty printf with thousands separators use %' */
+ if (use_separators)
+ setlocale(LC_NUMERIC, "en_US");
+
+ while (1) {
+ swap(&prev, &record);
+ stats_collect(record);
+ stats_print(record, prev, prog_num);
+ sleep(interval);
+ }
+
+ free_stats_record(record);
+ free_stats_record(prev);
+}
+
+static int create_cpu_entry(__u32 cpu, __u32 queue_size,
+ __u32 avail_idx, bool new)
+{
+ __u32 curr_cpus_count;
+ __u32 key = 0;
+ int ret;
+
+ /* Add a CPU entry to cpumap, as this allocate a cpu entry in
+ * the kernel for the cpu.
+ */
+ ret = bpf_map_update_elem(map_fd[0], &cpu, &queue_size, 0);
+ if (ret) {
+ fprintf(stderr, "Create CPU entry failed\n");
+ exit(EXIT_FAIL_BPF);
+ }
+
+ /* Inform bpf_prog's that a new CPU is available to select
+ * from via some control maps.
+ */
+ /* map_fd[5] = cpus_available */
+ ret = bpf_map_update_elem(map_fd[5], &avail_idx, &cpu, 0);
+ if (ret) {
+ fprintf(stderr, "Add to avail CPUs failed\n");
+ exit(EXIT_FAIL_BPF);
+ }
+
+ /* When not replacing/updating existing entry, bump the count */
+ /* map_fd[6] = cpus_count */
+ if (new) {
+ ret = bpf_map_lookup_elem(map_fd[6], &key, &curr_cpus_count);
+ if (ret) {
+ fprintf(stderr, "Failed reading curr cpus_count \n");
+ exit(EXIT_FAIL_BPF);
+ }
+ curr_cpus_count++;
+ ret = bpf_map_update_elem(map_fd[6], &key, &curr_cpus_count, 0);
+ if (ret) {
+ fprintf(stderr, "Failed write curr cpus_count \n");
+ exit(EXIT_FAIL_BPF);
+ }
+ }
+ /* map_fd[7] = cpus_iterator */
+ printf("%s CPU:%u as idx:%u cpus_count:%u\n",
+ new ? "Add-new":"Replace", cpu, avail_idx, curr_cpus_count);
+
+ return 0;
+}
+
+int main(int argc, char **argv)
+{
+ struct rlimit r = {10 * 1024*1024, RLIM_INFINITY};
+ bool use_separators = true;
+ char filename[256];
+ bool debug = false;
+ int added_cpus = 0;
+ int longindex = 0;
+ int interval = 2;
+ int prog_num = 0;
+ int add_cpu = -1;
+ __u32 qsize;
+ int opt;
+
+ /* Notice: choosing he queue size is very important with the
+ * ixgbe driver, because it's driver page recycling trick is
+ * dependend on pages being returned quickly. The number of
+ * out-standing packets in the system must be less-than 2x
+ * RX-ring size.
+ */
+ qsize = 128+64;
+
+ snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]);
+
+ if (setrlimit(RLIMIT_MEMLOCK, &r)) {
+ perror("setrlimit(RLIMIT_MEMLOCK)");
+ return 1;
+ }
+
+ if (load_bpf_file(filename)) {
+ fprintf(stderr, "ERR in load_bpf_file(): %s", bpf_log_buf);
+ return EXIT_FAIL;
+ }
+
+ if (!prog_fd[0]) {
+ fprintf(stderr, "ERR: load_bpf_file: %s\n", strerror(errno));
+ return EXIT_FAIL;
+ }
+
+ /* Parse commands line args */
+ while ((opt = getopt_long(argc, argv, "hSd:",
+ long_options, &longindex)) != -1) {
+ switch (opt) {
+ case 'd':
+ if (strlen(optarg) >= IF_NAMESIZE) {
+ fprintf(stderr, "ERR: --dev name too long\n");
+ goto error;
+ }
+ ifname = (char *)&ifname_buf;
+ strncpy(ifname, optarg, IF_NAMESIZE);
+ ifindex = if_nametoindex(ifname);
+ if (ifindex == 0) {
+ fprintf(stderr,
+ "ERR: --dev name unknown err(%d):%s\n",
+ errno, strerror(errno));
+ goto error;
+ }
+ break;
+ case 's':
+ interval = atoi(optarg);
+ break;
+ case 'S':
+ xdp_flags |= XDP_FLAGS_SKB_MODE;
+ break;
+ case 'D':
+ debug = true;
+ break;
+ case 'z':
+ use_separators = false;
+ break;
+ case 'p':
+ /* Selecting eBPF prog to load */
+ prog_num = atoi(optarg);
+ if (prog_num < 0 || prog_num >= MAX_PROG) {
+ fprintf(stderr,
+ "--prognum too large err(%d):%s\n",
+ errno, strerror(errno));
+ goto error;
+ }
+ break;
+ case 'c':
+ /* Add multiple CPUs */
+ add_cpu = strtoul(optarg, NULL, 0);
+ if (add_cpu > MAX_CPUS) {
+ fprintf(stderr,
+ "--cpu nr too large for cpumap err(%d):%s\n",
+ errno, strerror(errno));
+ goto error;
+ }
+ create_cpu_entry(add_cpu, qsize, added_cpus, true);
+ added_cpus++;
+ break;
+ case 'q':
+ qsize = atoi(optarg);
+ break;
+ case 'h':
+ error:
+ default:
+ usage(argv);
+ return EXIT_FAIL_OPTION;
+ }
+ }
+ /* Required option */
+ if (ifindex == -1) {
+ fprintf(stderr, "ERR: required option --dev missing\n");
+ usage(argv);
+ return EXIT_FAIL_OPTION;
+ }
+ /* Required option */
+ if (add_cpu == -1) {
+ fprintf(stderr, "ERR: required option --cpu missing\n");
+ fprintf(stderr, " Specify multiple --cpu option to add more\n");
+ usage(argv);
+ return EXIT_FAIL_OPTION;
+ }
+
+ /* Remove XDP program when program is interrupted */
+ signal(SIGINT, int_exit);
+
+ if (set_link_xdp_fd(ifindex, prog_fd[prog_num], xdp_flags) < 0) {
+ fprintf(stderr, "link set xdp fd failed\n");
+ return EXIT_FAIL_XDP;
+ }
+
+ if (debug) {
+ printf("Debug-mode reading trace pipe (fix #define DEBUG)\n");
+ read_trace_pipe();
+ }
+
+ stats_poll(interval, use_separators, prog_num);
+ return EXIT_OK;
+}
^ permalink raw reply related
* [net-next PATCH 4/5] bpf: cpumap add tracepoints
From: Jesper Dangaard Brouer @ 2017-09-28 12:57 UTC (permalink / raw)
To: netdev
Cc: jakub.kicinski, Michael S. Tsirkin, Jason Wang, mchan,
John Fastabend, peter.waskiewicz.jr, Jesper Dangaard Brouer,
Daniel Borkmann, Alexei Starovoitov, Andy Gospodarek
In-Reply-To: <150660339205.2808.7084136789768233829.stgit@firesoul>
This adds two tracepoint to the cpumap. One for the enqueue side
trace_xdp_cpumap_enqueue() and one for the kthread dequeue side
trace_xdp_cpumap_kthread().
To mitigate the tracepoint overhead, these are invoked during the
enqueue/dequeue bulking phases, thus amortizing the cost.
The obvious use-cases are for debugging and monitoring. The
non-intuitive use-case is using these as a feedback loop to know the
system load. One can imagine auto-scaling by reducing, adding or
activating more worker CPUs on demand.
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
---
include/trace/events/xdp.h | 70 ++++++++++++++++++++++++++++++++++++++++++++
kernel/bpf/cpumap.c | 18 +++++++++--
2 files changed, 85 insertions(+), 3 deletions(-)
diff --git a/include/trace/events/xdp.h b/include/trace/events/xdp.h
index eb2ece96c1a2..bc48c13892c4 100644
--- a/include/trace/events/xdp.h
+++ b/include/trace/events/xdp.h
@@ -150,6 +150,76 @@ DEFINE_EVENT_PRINT(xdp_redirect_template, xdp_redirect_map_err,
trace_xdp_redirect_map_err(dev, xdp, devmap_ifindex(fwd, map), \
err, map, idx)
+TRACE_EVENT(xdp_cpumap_kthread,
+
+ TP_PROTO(int map_id, unsigned int processed, unsigned int drops,
+ int time_limit),
+
+ TP_ARGS(map_id, processed, drops, time_limit),
+
+ TP_STRUCT__entry(
+ __field(int, map_id)
+ __field(u32, act)
+ __field(int, cpu)
+ __field(unsigned int, drops)
+ __field(unsigned int, processed)
+ __field(int, time_limit)
+ ),
+
+ TP_fast_assign(
+ __entry->map_id = map_id;
+ __entry->act = XDP_REDIRECT;
+ __entry->cpu = smp_processor_id();
+ __entry->drops = drops;
+ __entry->processed = processed;
+ __entry->time_limit = time_limit;
+ ),
+
+ TP_printk("kthread"
+ " cpu=%d map_id=%d action=%s"
+ " processed=%u drops=%u"
+ " time_limit=%d",
+ __entry->cpu, __entry->map_id,
+ __print_symbolic(__entry->act, __XDP_ACT_SYM_TAB),
+ __entry->processed, __entry->drops,
+ __entry->time_limit)
+);
+
+TRACE_EVENT(xdp_cpumap_enqueue,
+
+ TP_PROTO(int map_id, unsigned int processed, unsigned int drops,
+ int to_cpu),
+
+ TP_ARGS(map_id, processed, drops, to_cpu),
+
+ TP_STRUCT__entry(
+ __field(int, map_id)
+ __field(u32, act)
+ __field(int, cpu)
+ __field(unsigned int, drops)
+ __field(unsigned int, processed)
+ __field(int, to_cpu)
+ ),
+
+ TP_fast_assign(
+ __entry->map_id = map_id;
+ __entry->act = XDP_REDIRECT;
+ __entry->cpu = smp_processor_id();
+ __entry->drops = drops;
+ __entry->processed = processed;
+ __entry->to_cpu = to_cpu;
+ ),
+
+ TP_printk("enqueue"
+ " cpu=%d map_id=%d action=%s"
+ " processed=%u drops=%u"
+ " to_cpu=%d",
+ __entry->cpu, __entry->map_id,
+ __print_symbolic(__entry->act, __XDP_ACT_SYM_TAB),
+ __entry->processed, __entry->drops,
+ __entry->to_cpu)
+);
+
#endif /* _TRACE_XDP_H */
#include <trace/define_trace.h>
diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c
index 352cc071c9cc..3b0288e4e998 100644
--- a/kernel/bpf/cpumap.c
+++ b/kernel/bpf/cpumap.c
@@ -23,6 +23,7 @@
#include <linux/sched.h>
#include <linux/workqueue.h>
#include <linux/kthread.h>
+#include <trace/events/xdp.h>
#include <linux/netdevice.h> /* netif_receive_skb */
#include <linux/etherdevice.h> /* eth_type_trans */
@@ -294,6 +295,9 @@ static int cpu_map_kthread_run(void *data)
if (++processed == 8)
break;
}
+ /* Feedback loop via tracepoint */
+ trace_xdp_cpumap_kthread(rcpu->map_id, processed, drops,
+ time_after_eq(jiffies, time_limit));
local_bh_enable();
__set_current_state(TASK_INTERRUPTIBLE);
@@ -331,7 +335,10 @@ struct bpf_cpu_map_entry *__cpu_map_entry_alloc(u32 qsize, u32 cpu, int map_id)
err = ptr_ring_init(rcpu->queue, qsize, gfp);
if (err)
goto fail;
- rcpu->qsize = qsize;
+
+ rcpu->cpu = cpu;
+ rcpu->map_id = map_id;
+ rcpu->qsize = qsize;
/* Setup kthread */
rcpu->kthread = kthread_create_on_node(cpu_map_kthread_run, rcpu, numa,
@@ -555,6 +562,8 @@ const struct bpf_map_ops cpu_map_ops = {
static int bq_flush_to_queue(struct bpf_cpu_map_entry *rcpu,
struct xdp_bulk_queue *bq)
{
+ unsigned int processed = 0, drops = 0;
+ const int to_cpu = rcpu->cpu;
struct ptr_ring *q;
int i;
@@ -570,13 +579,16 @@ static int bq_flush_to_queue(struct bpf_cpu_map_entry *rcpu,
err = __ptr_ring_produce(q, xdp_pkt);
if (err) {
- /* Free xdp_pkt */
- page_frag_free(xdp_pkt);
+ drops++;
+ page_frag_free(xdp_pkt); /* Free xdp_pkt */
}
+ processed++;
}
bq->count = 0;
spin_unlock(&q->producer_lock);
+ /* Feedback loop via tracepoints */
+ trace_xdp_cpumap_enqueue(rcpu->map_id, processed, drops, to_cpu);
return 0;
}
^ permalink raw reply related
* [net-next PATCH 3/5] bpf: cpumap xdp_buff to skb conversion and allocation
From: Jesper Dangaard Brouer @ 2017-09-28 12:57 UTC (permalink / raw)
To: netdev
Cc: jakub.kicinski, Michael S. Tsirkin, Jason Wang, mchan,
John Fastabend, peter.waskiewicz.jr, Jesper Dangaard Brouer,
Daniel Borkmann, Alexei Starovoitov, Andy Gospodarek
In-Reply-To: <150660339205.2808.7084136789768233829.stgit@firesoul>
This patch makes cpumap functional, by adding SKB allocation and
invoking the network stack on the dequeuing CPU.
For constructing the SKB on the remote CPU, the xdp_buff in converted
into a struct xdp_pkt, and it mapped into the top headroom of the
packet, to avoid allocating separate mem. For now, struct xdp_pkt is
just a cpumap internal data structure, with info carried between
enqueue to dequeue.
If a driver doesn't have enough headroom it is simply dropped, with
return code -EOVERFLOW. This will be picked up the xdp tracepoint
infrastructure, to allow users to catch this.
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
---
kernel/bpf/cpumap.c | 153 ++++++++++++++++++++++++++++++++++++++++++++-------
1 file changed, 132 insertions(+), 21 deletions(-)
diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c
index ce2490ad860d..352cc071c9cc 100644
--- a/kernel/bpf/cpumap.c
+++ b/kernel/bpf/cpumap.c
@@ -24,6 +24,9 @@
#include <linux/workqueue.h>
#include <linux/kthread.h>
+#include <linux/netdevice.h> /* netif_receive_skb */
+#include <linux/etherdevice.h> /* eth_type_trans */
+
/*
* General idea: XDP packets getting XDP redirected to another CPU,
* will maximum be stored/queued for one driver ->poll() call. It is
@@ -160,20 +163,139 @@ static void cpu_map_kthread_stop(struct work_struct *work)
kthread_stop(rcpu->kthread); /* calls put_cpu_map_entry */
}
+/* For now, xdp_pkt is a cpumap internal data structure, with info
+ * carried between enqueue to dequeue. It is mapped into the top
+ * headroom of the packet, to avoid allocating separate mem.
+ */
+struct xdp_pkt {
+ void *data;
+ u16 len;
+ u16 headroom;
+ struct net_device *dev_rx;
+};
+
+/* Convert xdp_buff to xdp_pkt */
+static struct xdp_pkt *convert_to_xdp_pkt(struct xdp_buff *xdp)
+{
+ struct xdp_pkt *xdp_pkt;
+ int headroom;
+
+ /* Assure headroom is available for storing info */
+ headroom = xdp->data - xdp->data_hard_start;
+ if (headroom < sizeof(*xdp_pkt))
+ return NULL;
+
+ /* Store info in top of packet */
+ xdp_pkt = xdp->data_hard_start;
+
+ xdp_pkt->data = xdp->data;
+ xdp_pkt->len = xdp->data_end - xdp->data;
+ xdp_pkt->headroom = headroom - sizeof(*xdp_pkt);
+
+ return xdp_pkt;
+}
+
+static struct sk_buff *cpu_map_build_skb(struct bpf_cpu_map_entry *rcpu,
+ struct xdp_pkt *xdp_pkt)
+{
+ unsigned int frame_size;
+ void *pkt_data_start;
+ struct sk_buff *skb;
+
+ /* build_skb need to place skb_shared_info after SKB end, and
+ * also want to know the memory "truesize". Thus, need to
+ * know the memory frame size backing xdp_buff.
+ *
+ * XDP was designed to have PAGE_SIZE frames, but this
+ * assumption is not longer true with ixgbe and i40e. It
+ * would be preferred to set frame_size to 2048 or 4096
+ * depending on the driver.
+ * frame_size = 2048;
+ * frame_len = frame_size - sizeof(*xdp_pkt);
+ *
+ * Instead, with info avail, skb_shared_info in placed after
+ * packet len. This, unfortunately fakes the truesize.
+ * Another disadvantage of this approach, the skb_shared_info
+ * is not at a fixed memory location, with mixed length
+ * packets, which is bad for cache-line hotness.
+ */
+ frame_size = SKB_DATA_ALIGN(xdp_pkt->len) + xdp_pkt->headroom +
+ SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
+
+ pkt_data_start = xdp_pkt->data - xdp_pkt->headroom;
+ skb = build_skb(pkt_data_start, frame_size);
+ if (!skb)
+ return NULL;
+
+ skb_reserve(skb, xdp_pkt->headroom);
+ __skb_put(skb, xdp_pkt->len);
+
+ /* Essential SKB info: protocol and skb->dev */
+ skb->protocol = eth_type_trans(skb, xdp_pkt->dev_rx);
+
+ /* Optional SKB info, currently missing:
+ * - HW checksum info (skb->ip_summed)
+ * - HW RX hash (skb_set_hash)
+ * - RX ring dev queue index (skb_record_rx_queue)
+ */
+
+ return skb;
+}
+
static int cpu_map_kthread_run(void *data)
{
+ const unsigned long busy_poll_jiffies = usecs_to_jiffies(2000);
+ unsigned long time_limit = jiffies + busy_poll_jiffies;
struct bpf_cpu_map_entry *rcpu = data;
+ unsigned int empty_cnt = 0;
set_current_state(TASK_INTERRUPTIBLE);
while (!kthread_should_stop()) {
+ unsigned int processed = 0, drops = 0;
struct xdp_pkt *xdp_pkt;
- schedule();
- /* Do work */
- while ((xdp_pkt = ptr_ring_consume(rcpu->queue))) {
- /* For now just "refcnt-free" */
- page_frag_free(xdp_pkt);
+ /* Release CPU reschedule checks */
+ if ((time_after_eq(jiffies, time_limit) || empty_cnt > 25) &&
+ __ptr_ring_empty(rcpu->queue)) {
+ empty_cnt++;
+ schedule();
+ time_limit = jiffies + busy_poll_jiffies;
+ WARN_ON(smp_processor_id() != rcpu->cpu);
+ } else {
+ cond_resched();
}
+
+ /* Process packets in rcpu->queue */
+ local_bh_disable();
+ /*
+ * The bpf_cpu_map_entry is single consumer, with this
+ * kthread CPU pinned. Lockless access to ptr_ring
+ * consume side valid as no-resize allowed of queue.
+ */
+ while ((xdp_pkt = __ptr_ring_consume(rcpu->queue))) {
+ struct sk_buff *skb;
+ int ret;
+
+ /* Allow busy polling again */
+ empty_cnt = 0;
+
+ skb = cpu_map_build_skb(rcpu, xdp_pkt);
+ if (!skb) {
+ page_frag_free(xdp_pkt);
+ continue;
+ }
+
+ /* Inject into network stack */
+ ret = netif_receive_skb(skb);
+ if (ret == NET_RX_DROP)
+ drops++;
+
+ /* Limit BH-disable period */
+ if (++processed == 8)
+ break;
+ }
+ local_bh_enable();
+
__set_current_state(TASK_INTERRUPTIBLE);
}
put_cpu_map_entry(rcpu);
@@ -458,13 +580,6 @@ static int bq_flush_to_queue(struct bpf_cpu_map_entry *rcpu,
return 0;
}
-/* Notice: Will change in later patch */
-struct xdp_pkt {
- void *data;
- u16 len;
- u16 headroom;
-};
-
/* Runs under RCU-read-side, plus in softirq under NAPI protection.
* Thus, safe percpu variable access.
*/
@@ -492,17 +607,13 @@ int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp,
struct net_device *dev_rx)
{
struct xdp_pkt *xdp_pkt;
- int headroom;
- /* Convert xdp_buff to xdp_pkt */
- headroom = xdp->data - xdp->data_hard_start;
- if (headroom < sizeof(*xdp_pkt))
+ xdp_pkt = convert_to_xdp_pkt(xdp);
+ if (!xdp_pkt)
return -EOVERFLOW;
- xdp_pkt = xdp->data_hard_start;
- xdp_pkt->data = xdp->data;
- xdp_pkt->len = xdp->data_end - xdp->data;
- xdp_pkt->headroom = headroom - sizeof(*xdp_pkt);
- /* For now this is just used as a void pointer to data_hard_start */
+
+ /* Info needed when constructing SKB on remote CPU */
+ xdp_pkt->dev_rx = dev_rx;
bq_enqueue(rcpu, xdp_pkt);
return 0;
^ permalink raw reply related
* [net-next PATCH 2/5] bpf: XDP_REDIRECT enable use of cpumap
From: Jesper Dangaard Brouer @ 2017-09-28 12:57 UTC (permalink / raw)
To: netdev
Cc: jakub.kicinski, Michael S. Tsirkin, Jason Wang, mchan,
John Fastabend, peter.waskiewicz.jr, Jesper Dangaard Brouer,
Daniel Borkmann, Alexei Starovoitov, Andy Gospodarek
In-Reply-To: <150660339205.2808.7084136789768233829.stgit@firesoul>
This patch connects cpumap to the xdp_do_redirect_map infrastructure.
Still no SKB allocation are done yet. The XDP frames are transferred
to the other CPU, but they are simply refcnt decremented on the remote
CPU. This served as a good benchmark for measuring the overhead of
remote refcnt decrement. If driver page recycle cache is not
efficient then this, exposes a bottleneck in the page allocator.
A shout-out to MST's ptr_ring, which is the secret behind is being so
efficient to transfer memory pointers between CPUs, without constantly
bouncing cache-lines between CPUs.
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
---
include/linux/bpf.h | 7 +++++
include/trace/events/xdp.h | 10 +++++--
kernel/bpf/cpumap.c | 5 ++-
kernel/bpf/verifier.c | 3 +-
net/core/filter.c | 65 +++++++++++++++++++++++++++++++++++++++-----
5 files changed, 77 insertions(+), 13 deletions(-)
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 2b672c50f160..7f70b03e7426 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -317,6 +317,13 @@ struct net_device *__dev_map_lookup_elem(struct bpf_map *map, u32 key);
void __dev_map_insert_ctx(struct bpf_map *map, u32 index);
void __dev_map_flush(struct bpf_map *map);
+struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key);
+void __cpu_map_insert_ctx(struct bpf_map *map, u32 index);
+void __cpu_map_flush(struct bpf_map *map);
+struct xdp_buff;
+int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp,
+ struct net_device *dev_rx);
+
/* Return map's numa specified by userspace */
static inline int bpf_map_attr_numa_node(const union bpf_attr *attr)
{
diff --git a/include/trace/events/xdp.h b/include/trace/events/xdp.h
index 4e16c43fba10..eb2ece96c1a2 100644
--- a/include/trace/events/xdp.h
+++ b/include/trace/events/xdp.h
@@ -136,12 +136,18 @@ DEFINE_EVENT_PRINT(xdp_redirect_template, xdp_redirect_map_err,
__entry->map_id, __entry->map_index)
);
+#define devmap_ifindex(fwd, map) \
+ (!fwd ? 0 : \
+ (!map ? 0 : \
+ ((map->map_type == BPF_MAP_TYPE_DEVMAP) ? \
+ ((struct net_device *)fwd)->ifindex : 0)))
+
#define _trace_xdp_redirect_map(dev, xdp, fwd, map, idx) \
- trace_xdp_redirect_map(dev, xdp, fwd ? fwd->ifindex : 0, \
+ trace_xdp_redirect_map(dev, xdp, devmap_ifindex(fwd, map), \
0, map, idx)
#define _trace_xdp_redirect_map_err(dev, xdp, fwd, map, idx, err) \
- trace_xdp_redirect_map_err(dev, xdp, fwd ? fwd->ifindex : 0, \
+ trace_xdp_redirect_map_err(dev, xdp, devmap_ifindex(fwd, map), \
err, map, idx)
#endif /* _TRACE_XDP_H */
diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c
index f0948af82e65..ce2490ad860d 100644
--- a/kernel/bpf/cpumap.c
+++ b/kernel/bpf/cpumap.c
@@ -488,7 +488,8 @@ static int bq_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_pkt *xdp_pkt)
return 0;
}
-int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp)
+int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp,
+ struct net_device *dev_rx)
{
struct xdp_pkt *xdp_pkt;
int headroom;
@@ -500,7 +501,7 @@ int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp)
xdp_pkt = xdp->data_hard_start;
xdp_pkt->data = xdp->data;
xdp_pkt->len = xdp->data_end - xdp->data;
- xdp_pkt->headroom = headroom;
+ xdp_pkt->headroom = headroom - sizeof(*xdp_pkt);
/* For now this is just used as a void pointer to data_hard_start */
bq_enqueue(rcpu, xdp_pkt);
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index f849eca36052..a712c7431c2d 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -1589,7 +1589,8 @@ static int check_map_func_compatibility(struct bpf_map *map, int func_id)
goto error;
break;
case BPF_FUNC_redirect_map:
- if (map->map_type != BPF_MAP_TYPE_DEVMAP)
+ if (map->map_type != BPF_MAP_TYPE_DEVMAP &&
+ map->map_type != BPF_MAP_TYPE_CPUMAP)
goto error;
break;
case BPF_FUNC_sk_redirect_map:
diff --git a/net/core/filter.c b/net/core/filter.c
index 9b6e7e84aafd..37fe9e631ee4 100644
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -2521,10 +2521,37 @@ static int __bpf_tx_xdp(struct net_device *dev,
err = dev->netdev_ops->ndo_xdp_xmit(dev, xdp);
if (err)
return err;
- if (map)
+ dev->netdev_ops->ndo_xdp_flush(dev);
+ return 0;
+}
+
+static int __bpf_tx_xdp_map(struct net_device *dev_rx, void *fwd,
+ struct bpf_map *map,
+ struct xdp_buff *xdp,
+ u32 index)
+{
+ int err;
+
+ if (map->map_type == BPF_MAP_TYPE_DEVMAP) {
+ struct net_device *dev = fwd;
+
+ if (!dev->netdev_ops->ndo_xdp_xmit) {
+ return -EOPNOTSUPP;
+ }
+
+ err = dev->netdev_ops->ndo_xdp_xmit(dev, xdp);
+ if (err)
+ return err;
__dev_map_insert_ctx(map, index);
- else
- dev->netdev_ops->ndo_xdp_flush(dev);
+
+ } else if (map->map_type == BPF_MAP_TYPE_CPUMAP) {
+ struct bpf_cpu_map_entry *rcpu = fwd;
+
+ err = cpu_map_enqueue(rcpu, xdp, dev_rx);
+ if (err)
+ return err;
+ __cpu_map_insert_ctx(map, index);
+ }
return 0;
}
@@ -2534,11 +2561,33 @@ void xdp_do_flush_map(void)
struct bpf_map *map = ri->map_to_flush;
ri->map_to_flush = NULL;
- if (map)
- __dev_map_flush(map);
+ if (map) {
+ switch (map->map_type) {
+ case BPF_MAP_TYPE_DEVMAP:
+ __dev_map_flush(map);
+ break;
+ case BPF_MAP_TYPE_CPUMAP:
+ __cpu_map_flush(map);
+ break;
+ default:
+ break;
+ }
+ }
}
EXPORT_SYMBOL_GPL(xdp_do_flush_map);
+static void *__xdp_map_lookup_elem(struct bpf_map *map, u32 index)
+{
+ switch (map->map_type) {
+ case BPF_MAP_TYPE_DEVMAP:
+ return __dev_map_lookup_elem(map, index);
+ case BPF_MAP_TYPE_CPUMAP:
+ return __cpu_map_lookup_elem(map, index);
+ default:
+ return NULL;
+ }
+}
+
static inline bool xdp_map_invalid(const struct bpf_prog *xdp_prog,
unsigned long aux)
{
@@ -2551,8 +2600,8 @@ static int xdp_do_redirect_map(struct net_device *dev, struct xdp_buff *xdp,
struct redirect_info *ri = this_cpu_ptr(&redirect_info);
unsigned long map_owner = ri->map_owner;
struct bpf_map *map = ri->map;
- struct net_device *fwd = NULL;
u32 index = ri->ifindex;
+ void *fwd = NULL;
int err;
ri->ifindex = 0;
@@ -2565,7 +2614,7 @@ static int xdp_do_redirect_map(struct net_device *dev, struct xdp_buff *xdp,
goto err;
}
- fwd = __dev_map_lookup_elem(map, index);
+ fwd = __xdp_map_lookup_elem(map, index);
if (!fwd) {
err = -EINVAL;
goto err;
@@ -2573,7 +2622,7 @@ static int xdp_do_redirect_map(struct net_device *dev, struct xdp_buff *xdp,
if (ri->map_to_flush && ri->map_to_flush != map)
xdp_do_flush_map();
- err = __bpf_tx_xdp(fwd, map, xdp, index);
+ err = __bpf_tx_xdp_map(dev, fwd, map, xdp, index);
if (unlikely(err))
goto err;
^ permalink raw reply related
* [net-next PATCH 1/5] bpf: introduce new bpf cpu map type BPF_MAP_TYPE_CPUMAP
From: Jesper Dangaard Brouer @ 2017-09-28 12:57 UTC (permalink / raw)
To: netdev
Cc: jakub.kicinski, Michael S. Tsirkin, Jason Wang, mchan,
John Fastabend, peter.waskiewicz.jr, Jesper Dangaard Brouer,
Daniel Borkmann, Alexei Starovoitov, Andy Gospodarek
In-Reply-To: <150660339205.2808.7084136789768233829.stgit@firesoul>
The 'cpumap' is primary used as a backend map for XDP BPF helper
call bpf_redirect_map() and XDP_REDIRECT action, like 'devmap'.
This patch implement the main part of the map. It is not connected to
the XDP redirect system yet, and no SKB allocation are done yet.
The main concern in this patch is to ensure the datapath can run
without any locking. This adds complexity to the setup and tear-down
procedure, which assumptions are extra carefully documented in the
code comments.
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
---
include/linux/bpf_types.h | 1
include/uapi/linux/bpf.h | 1
kernel/bpf/Makefile | 1
kernel/bpf/cpumap.c | 547 ++++++++++++++++++++++++++++++++++++++++
kernel/bpf/syscall.c | 8 +
tools/include/uapi/linux/bpf.h | 1
6 files changed, 558 insertions(+), 1 deletion(-)
create mode 100644 kernel/bpf/cpumap.c
diff --git a/include/linux/bpf_types.h b/include/linux/bpf_types.h
index 6f1a567667b8..814c1081a4a9 100644
--- a/include/linux/bpf_types.h
+++ b/include/linux/bpf_types.h
@@ -41,4 +41,5 @@ BPF_MAP_TYPE(BPF_MAP_TYPE_DEVMAP, dev_map_ops)
#ifdef CONFIG_STREAM_PARSER
BPF_MAP_TYPE(BPF_MAP_TYPE_SOCKMAP, sock_map_ops)
#endif
+BPF_MAP_TYPE(BPF_MAP_TYPE_CPUMAP, cpu_map_ops)
#endif
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index e43491ac4823..f14e15702533 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -111,6 +111,7 @@ enum bpf_map_type {
BPF_MAP_TYPE_HASH_OF_MAPS,
BPF_MAP_TYPE_DEVMAP,
BPF_MAP_TYPE_SOCKMAP,
+ BPF_MAP_TYPE_CPUMAP,
};
enum bpf_prog_type {
diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile
index 897daa005b23..dba0bd33a43c 100644
--- a/kernel/bpf/Makefile
+++ b/kernel/bpf/Makefile
@@ -4,6 +4,7 @@ obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o
obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o
ifeq ($(CONFIG_NET),y)
obj-$(CONFIG_BPF_SYSCALL) += devmap.o
+obj-$(CONFIG_BPF_SYSCALL) += cpumap.o
ifeq ($(CONFIG_STREAM_PARSER),y)
obj-$(CONFIG_BPF_SYSCALL) += sockmap.o
endif
diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c
new file mode 100644
index 000000000000..f0948af82e65
--- /dev/null
+++ b/kernel/bpf/cpumap.c
@@ -0,0 +1,547 @@
+/* bpf/cpumap.c
+ *
+ * Copyright (c) 2017 Jesper Dangaard Brouer, Red Hat Inc.
+ * Released under terms in GPL version 2. See COPYING.
+ */
+
+/* The 'cpumap' is primary used as a backend map for XDP BPF helper
+ * call bpf_redirect_map() and XDP_REDIRECT action, like 'devmap'.
+ *
+ * Unlike devmap which redirect XDP frames out another NIC device,
+ * this map type redirect raw XDP frames to another CPU. The remote
+ * CPU will do SKB-allocation and call the normal network stack.
+ *
+ * This is a scalability and isolation mechanism, that allow
+ * separating the early driver network XDP layer, from the rest of the
+ * netstack, and assigning dedicated CPUs for this stage. This
+ * basically allows for 10G wirespeed pre-filtering via bpf.
+ */
+#include <linux/bpf.h>
+#include <linux/filter.h>
+#include <linux/ptr_ring.h>
+
+#include <linux/sched.h>
+#include <linux/workqueue.h>
+#include <linux/kthread.h>
+
+/*
+ * General idea: XDP packets getting XDP redirected to another CPU,
+ * will maximum be stored/queued for one driver ->poll() call. It is
+ * guaranteed that setting flush bit and flush operation happen on
+ * same CPU. Thus, cpu_map_flush operation can deduct via this_cpu_ptr()
+ * which queue in bpf_cpu_map_entry contains packets.
+ */
+
+#define CPU_MAP_BULK_SIZE 8 /* 8 == one cacheline on 64-bit archs */
+struct xdp_bulk_queue {
+ void *q[CPU_MAP_BULK_SIZE];
+ unsigned int count;
+};
+
+/* Struct for every remote "destination" CPU in map */
+struct bpf_cpu_map_entry {
+ u32 cpu; /* kthread CPU and map index */
+ int map_id; /* Back reference to map */
+ u32 qsize; /* Redundant queue size for map lookup */
+
+ /* XDP can run multiple RX-ring queues, need __percpu enqueue store */
+ struct xdp_bulk_queue __percpu *bulkq;
+
+ /* Queue with potential multi-producers, and single-consumer kthread */
+ struct ptr_ring *queue;
+ struct task_struct *kthread;
+ struct work_struct kthread_stop_wq;
+
+ atomic_t refcnt; /* Control when this struct can be free'ed */
+ struct rcu_head rcu;
+};
+
+struct bpf_cpu_map {
+ struct bpf_map map;
+ /* Below members specific for map type */
+ struct bpf_cpu_map_entry **cpu_map;
+ unsigned long __percpu *flush_needed;
+};
+
+static int bq_flush_to_queue(struct bpf_cpu_map_entry *rcpu,
+ struct xdp_bulk_queue *bq);
+
+static u64 cpu_map_bitmap_size(const union bpf_attr *attr)
+{
+ return BITS_TO_LONGS(attr->max_entries) * sizeof(unsigned long);
+}
+
+static struct bpf_map *cpu_map_alloc(union bpf_attr *attr)
+{
+ struct bpf_cpu_map *cmap;
+ u64 cost;
+ int err;
+
+ /* check sanity of attributes */
+ if (attr->max_entries == 0 || attr->key_size != 4 ||
+ attr->value_size != 4 || attr->map_flags & ~BPF_F_NUMA_NODE)
+ return ERR_PTR(-EINVAL);
+
+ cmap = kzalloc(sizeof(*cmap), GFP_USER);
+ if (!cmap)
+ return ERR_PTR(-ENOMEM);
+
+ /* mandatory map attributes */
+ cmap->map.map_type = attr->map_type;
+ cmap->map.key_size = attr->key_size;
+ cmap->map.value_size = attr->value_size;
+ cmap->map.max_entries = attr->max_entries;
+ cmap->map.map_flags = attr->map_flags;
+ cmap->map.numa_node = bpf_map_attr_numa_node(attr);
+
+ /* make sure page count doesn't overflow */
+ cost = (u64) cmap->map.max_entries * sizeof(struct bpf_cpu_map_entry *);
+ cost += cpu_map_bitmap_size(attr) * num_possible_cpus();
+ if (cost >= U32_MAX - PAGE_SIZE)
+ goto free_cmap;
+ cmap->map.pages = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT;
+
+ /* if map size is larger than memlock limit, reject it early */
+ err = bpf_map_precharge_memlock(cmap->map.pages);
+ if (err)
+ goto free_cmap;
+
+ /* A per cpu bitfield with a bit per possible CPU in map */
+ cmap->flush_needed = __alloc_percpu(cpu_map_bitmap_size(attr),
+ __alignof__(unsigned long));
+ if (!cmap->flush_needed)
+ goto free_cmap;
+
+ /* Alloc array for possible remote "destination" CPUs */
+ cmap->cpu_map = bpf_map_area_alloc(cmap->map.max_entries *
+ sizeof(struct bpf_cpu_map_entry *),
+ cmap->map.numa_node);
+ if (!cmap->cpu_map)
+ goto free_cmap;
+
+ return &cmap->map;
+free_cmap:
+ free_percpu(cmap->flush_needed);
+ kfree(cmap);
+ return ERR_PTR(-ENOMEM);
+}
+
+void __cpu_map_queue_destructor(void *ptr)
+{
+ /* For now, just catch this as an error */
+ if (!ptr)
+ return;
+ pr_err("ERROR: %s() cpu_map queue was not empty\n", __func__);
+ page_frag_free(ptr);
+}
+
+static void put_cpu_map_entry(struct bpf_cpu_map_entry *rcpu)
+{
+ if (atomic_dec_and_test(&rcpu->refcnt)) {
+ /* The queue should be empty at this point */
+ ptr_ring_cleanup(rcpu->queue, __cpu_map_queue_destructor);
+ kfree(rcpu->queue);
+ kfree(rcpu);
+ }
+}
+
+static void get_cpu_map_entry(struct bpf_cpu_map_entry *rcpu)
+{
+ atomic_inc(&rcpu->refcnt);
+}
+
+/* called from workqueue, to workaround syscall using preempt_disable */
+static void cpu_map_kthread_stop(struct work_struct *work)
+{
+ struct bpf_cpu_map_entry *rcpu;
+
+ rcpu = container_of(work, struct bpf_cpu_map_entry, kthread_stop_wq);
+ synchronize_rcu(); /* wait for flush in __cpu_map_entry_free() */
+ kthread_stop(rcpu->kthread); /* calls put_cpu_map_entry */
+}
+
+static int cpu_map_kthread_run(void *data)
+{
+ struct bpf_cpu_map_entry *rcpu = data;
+
+ set_current_state(TASK_INTERRUPTIBLE);
+ while (!kthread_should_stop()) {
+ struct xdp_pkt *xdp_pkt;
+
+ schedule();
+ /* Do work */
+ while ((xdp_pkt = ptr_ring_consume(rcpu->queue))) {
+ /* For now just "refcnt-free" */
+ page_frag_free(xdp_pkt);
+ }
+ __set_current_state(TASK_INTERRUPTIBLE);
+ }
+ put_cpu_map_entry(rcpu);
+
+ __set_current_state(TASK_RUNNING);
+ return 0;
+}
+
+struct bpf_cpu_map_entry *__cpu_map_entry_alloc(u32 qsize, u32 cpu, int map_id)
+{
+ gfp_t gfp = GFP_ATOMIC|__GFP_NOWARN;
+ struct bpf_cpu_map_entry *rcpu;
+ int numa, err;
+
+ /* Have map->numa_node, but choose node of redirect target CPU */
+ numa = cpu_to_node(cpu);
+
+ rcpu = kzalloc_node(sizeof(*rcpu), gfp, numa);
+ if (!rcpu)
+ return NULL;
+
+ /* Alloc percpu bulkq */
+ rcpu->bulkq = __alloc_percpu_gfp(sizeof(*rcpu->bulkq),
+ sizeof(void *), gfp);
+ if (!rcpu->bulkq)
+ goto fail;
+
+ /* Alloc queue */
+ rcpu->queue = kzalloc_node(sizeof(*rcpu->queue), gfp, numa);
+ if (!rcpu->queue)
+ goto fail;
+
+ err = ptr_ring_init(rcpu->queue, qsize, gfp);
+ if (err)
+ goto fail;
+ rcpu->qsize = qsize;
+
+ /* Setup kthread */
+ rcpu->kthread = kthread_create_on_node(cpu_map_kthread_run, rcpu, numa,
+ "cpumap/%d/map:%d", cpu, map_id);
+ if (IS_ERR(rcpu->kthread))
+ goto fail;
+
+ /* Make sure kthread runs on a single CPU */
+ kthread_bind(rcpu->kthread, cpu);
+ wake_up_process(rcpu->kthread);
+
+ get_cpu_map_entry(rcpu); /* 1-refcnt for being in cmap->cpu_map[] */
+ get_cpu_map_entry(rcpu); /* 1-refcnt for kthread */
+
+ return rcpu;
+
+fail: /* Hint: free API detect NULL values */
+ free_percpu(rcpu->bulkq);
+ kfree(rcpu->queue);
+ kfree(rcpu);
+ return NULL;
+}
+
+void __cpu_map_entry_free(struct rcu_head *rcu)
+{
+ struct bpf_cpu_map_entry *rcpu;
+ int cpu;
+
+ /* This cpu_map_entry have been disconnected from map and one
+ * RCU graze-period have elapsed. Thus, XDP cannot queue any
+ * new packets and cannot change/set flush_needed that can
+ * find this entry.
+ */
+ rcpu = container_of(rcu, struct bpf_cpu_map_entry, rcu);
+
+ /* Flush remaining packets in percpu bulkq */
+ for_each_online_cpu(cpu) {
+ struct xdp_bulk_queue *bq = per_cpu_ptr(rcpu->bulkq, cpu);
+
+ /* No concurrent bq_enqueue can run at this point */
+ bq_flush_to_queue(rcpu, bq);
+ }
+ free_percpu(rcpu->bulkq);
+ /* Cannot kthread_stop() here, last put free rcpu resources */
+ put_cpu_map_entry(rcpu);
+}
+
+/*
+ * After xchg pointer to bpf_cpu_map_entry, use the call_rcu() to
+ * ensure any driver rcu critical sections have completed, but this
+ * does not guarantee a flush has happened yet. Because driver side
+ * rcu_read_lock/unlock only protects the running XDP program. The
+ * atomic xchg and NULL-ptr check in __cpu_map_flush() makes sure a
+ * pending flush op doesn't fail.
+ *
+ * The bpf_cpu_map_entry is still used by the kthread, and there can
+ * still be pending packets (in queue and percpu bulkq). A refcnt
+ * makes sure to last user (kthread_stop vs. call_rcu) free memory
+ * resources.
+ *
+ * The rcu callback __cpu_map_entry_free flush remaining packets in
+ * percpu bulkq to queue. Due to caller map_delete_elem() disable
+ * preemption, cannot call kthread_stop() to make sure queue is empty.
+ * Instead a work_queue is started for stopping kthread,
+ * cpu_map_kthread_stop, which waits for an RCU graze period before
+ * stopping kthread, emptying the queue.
+ */
+void __cpu_map_entry_replace(struct bpf_cpu_map *cmap,
+ u32 key_cpu, struct bpf_cpu_map_entry *rcpu)
+{
+ struct bpf_cpu_map_entry *old_rcpu;
+
+ old_rcpu = xchg(&cmap->cpu_map[key_cpu], rcpu);
+ if (old_rcpu) {
+ call_rcu(&old_rcpu->rcu, __cpu_map_entry_free);
+ INIT_WORK(&old_rcpu->kthread_stop_wq, cpu_map_kthread_stop);
+ schedule_work(&old_rcpu->kthread_stop_wq);
+ }
+}
+
+int cpu_map_delete_elem(struct bpf_map *map, void *key)
+{
+ struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
+ u32 key_cpu = *(u32 *)key;
+
+ if (key_cpu >= map->max_entries)
+ return -EINVAL;
+
+ /* notice caller map_delete_elem() use preempt_disable() */
+ __cpu_map_entry_replace(cmap, key_cpu, NULL);
+ return 0;
+}
+
+int cpu_map_update_elem(struct bpf_map *map, void *key, void *value,
+ u64 map_flags)
+{
+ struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
+ struct bpf_cpu_map_entry *rcpu;
+
+ /* Array index key correspond to CPU number */
+ u32 key_cpu = *(u32 *)key;
+ /* Q: The value could be the queue size? */
+ u32 qsize = *(u32 *)value;
+
+ if (unlikely(map_flags > BPF_EXIST))
+ return -EINVAL;
+ if (unlikely(key_cpu >= cmap->map.max_entries))
+ return -E2BIG;
+ if (unlikely(map_flags == BPF_NOEXIST))
+ return -EEXIST;
+ if (unlikely(qsize > 16384)) /* sanity limit on qsize */
+ return -EOVERFLOW;
+
+ if (qsize == 0) {
+ rcpu = NULL; /* Same as deleting */
+ } else {
+ /* Updating qsize cause re-allocation of bpf_cpu_map_entry */
+ rcpu = __cpu_map_entry_alloc(qsize, key_cpu, map->id);
+ if (!rcpu)
+ return -ENOMEM;
+ }
+ rcu_read_lock();
+ __cpu_map_entry_replace(cmap, key_cpu, rcpu);
+ rcu_read_unlock();
+ return 0;
+}
+
+void cpu_map_free(struct bpf_map *map)
+{
+ struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
+ int cpu;
+ u32 i;
+
+ /* At this point bpf_prog->aux->refcnt == 0 and this map->refcnt == 0,
+ * so the bpf programs (can be more than one that used this map) were
+ * disconnected from events. Wait for outstanding critical sections in
+ * these programs to complete. The rcu critical section only guarantees
+ * no further "XDP/bpf-side" reads against bpf_cpu_map->cpu_map.
+ * It does __not__ ensure pending flush operations (if any) are
+ * complete.
+ */
+ synchronize_rcu();
+
+ /* To ensure all pending flush operations have completed wait for flush
+ * bitmap to indicate all flush_needed bits to be zero on _all_ cpus.
+ * Because the above synchronize_rcu() ensures the map is disconnected
+ * from the program we can assume no new bits will be set.
+ */
+ for_each_online_cpu(cpu) {
+ unsigned long *bitmap = per_cpu_ptr(cmap->flush_needed, cpu);
+
+ while (!bitmap_empty(bitmap, cmap->map.max_entries))
+ cond_resched();
+ }
+
+ /* For cpu_map the remote CPUs can still be using the entries
+ * (struct bpf_cpu_map_entry).
+ */
+ for (i = 0; i < cmap->map.max_entries; i++) {
+ struct bpf_cpu_map_entry *rcpu;
+
+ rcpu = READ_ONCE(cmap->cpu_map[i]);
+ if (!rcpu)
+ continue;
+
+ /* bq flush and cleanup happens after RCU graze-period */
+ __cpu_map_entry_replace(cmap, i, NULL); /* call_rcu */
+ }
+ free_percpu(cmap->flush_needed);
+ bpf_map_area_free(cmap->cpu_map);
+ kfree(cmap);
+}
+
+struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key)
+{
+ struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
+ struct bpf_cpu_map_entry *rcpu;
+
+ if (key >= map->max_entries)
+ return NULL;
+
+ rcpu = READ_ONCE(cmap->cpu_map[key]);
+ return rcpu;
+}
+
+static void *cpu_map_lookup_elem(struct bpf_map *map, void *key)
+{
+ struct bpf_cpu_map_entry *rcpu =
+ __cpu_map_lookup_elem(map, *(u32 *)key);
+
+ return rcpu ? &rcpu->qsize : NULL;
+}
+
+static int cpu_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
+{
+ struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
+ u32 index = key ? *(u32 *)key : U32_MAX;
+ u32 *next = next_key;
+
+ if (index >= cmap->map.max_entries) {
+ *next = 0;
+ return 0;
+ }
+
+ if (index == cmap->map.max_entries - 1)
+ return -ENOENT;
+ *next = index + 1;
+ return 0;
+}
+
+const struct bpf_map_ops cpu_map_ops = {
+ .map_alloc = cpu_map_alloc,
+ .map_free = cpu_map_free,
+ .map_delete_elem = cpu_map_delete_elem,
+ .map_update_elem = cpu_map_update_elem,
+ .map_lookup_elem = cpu_map_lookup_elem,
+ .map_get_next_key = cpu_map_get_next_key,
+};
+
+
+static int bq_flush_to_queue(struct bpf_cpu_map_entry *rcpu,
+ struct xdp_bulk_queue *bq)
+{
+ struct ptr_ring *q;
+ int i;
+
+ if (unlikely(!bq->count))
+ return 0;
+
+ q = rcpu->queue;
+ spin_lock(&q->producer_lock);
+
+ for (i = 0; i < bq->count; i++) {
+ void *xdp_pkt = bq->q[i];
+ int err;
+
+ err = __ptr_ring_produce(q, xdp_pkt);
+ if (err) {
+ /* Free xdp_pkt */
+ page_frag_free(xdp_pkt);
+ }
+ }
+ bq->count = 0;
+ spin_unlock(&q->producer_lock);
+
+ return 0;
+}
+
+/* Notice: Will change in later patch */
+struct xdp_pkt {
+ void *data;
+ u16 len;
+ u16 headroom;
+};
+
+/* Runs under RCU-read-side, plus in softirq under NAPI protection.
+ * Thus, safe percpu variable access.
+ */
+static int bq_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_pkt *xdp_pkt)
+{
+ struct xdp_bulk_queue *bq = this_cpu_ptr(rcpu->bulkq);
+
+ if (unlikely(bq->count == CPU_MAP_BULK_SIZE)) {
+ bq_flush_to_queue(rcpu, bq);
+ }
+ /* Notice, xdp_buff/page MUST be queued here, long enough for
+ * driver to code invoking us to finished, due to driver
+ * (e.g. ixgbe) recycle tricks based on page-refcnt.
+ *
+ * Thus, incoming xdp_pkt is always queued here (else we race
+ * with another CPU on page-refcnt and remaining driver code).
+ * Queue time is very short, as driver will invoke flush
+ * operation, when completing napi->poll call.
+ */
+ bq->q[bq->count++] = xdp_pkt;
+ return 0;
+}
+
+int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp)
+{
+ struct xdp_pkt *xdp_pkt;
+ int headroom;
+
+ /* Convert xdp_buff to xdp_pkt */
+ headroom = xdp->data - xdp->data_hard_start;
+ if (headroom < sizeof(*xdp_pkt))
+ return -EOVERFLOW;
+ xdp_pkt = xdp->data_hard_start;
+ xdp_pkt->data = xdp->data;
+ xdp_pkt->len = xdp->data_end - xdp->data;
+ xdp_pkt->headroom = headroom;
+ /* For now this is just used as a void pointer to data_hard_start */
+
+ bq_enqueue(rcpu, xdp_pkt);
+ return 0;
+}
+
+void __cpu_map_insert_ctx(struct bpf_map *map, u32 bit)
+{
+ struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
+ unsigned long *bitmap = this_cpu_ptr(cmap->flush_needed);
+
+ __set_bit(bit, bitmap);
+}
+
+void __cpu_map_flush(struct bpf_map *map)
+{
+ struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
+ unsigned long *bitmap = this_cpu_ptr(cmap->flush_needed);
+ u32 bit;
+
+ /* The napi->poll softirq makes sure __cpu_map_insert_ctx()
+ * and __cpu_map_flush() happen on same CPU. Thus, the percpu
+ * bitmap indicate which percpu bulkq have packets.
+ */
+ for_each_set_bit(bit, bitmap, map->max_entries) {
+ struct bpf_cpu_map_entry *rcpu = READ_ONCE(cmap->cpu_map[bit]);
+ struct xdp_bulk_queue *bq;
+
+ /* This is possible if entry is removed by user space
+ * between xdp redirect and flush op.
+ */
+ if (unlikely(!rcpu))
+ continue;
+
+ __clear_bit(bit, bitmap);
+
+ /* Flush all frames in bulkq to real queue */
+ bq = this_cpu_ptr(rcpu->bulkq);
+ bq_flush_to_queue(rcpu, bq);
+
+ /* If already running, costs spin_lock_irqsave + smb_mb */
+ wake_up_process(rcpu->kthread);
+ }
+}
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 25d074920a00..68fe3f51e1a0 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -562,6 +562,12 @@ static int map_update_elem(union bpf_attr *attr)
if (copy_from_user(value, uvalue, value_size) != 0)
goto free_value;
+ /* Need to create a kthread, thus must support schedule */
+ if (map->map_type == BPF_MAP_TYPE_CPUMAP) {
+ err = map->ops->map_update_elem(map, key, value, attr->flags);
+ goto out;
+ }
+
/* must increment bpf_prog_active to avoid kprobe+bpf triggering from
* inside bpf map update or delete otherwise deadlocks are possible
*/
@@ -592,7 +598,7 @@ static int map_update_elem(union bpf_attr *attr)
}
__this_cpu_dec(bpf_prog_active);
preempt_enable();
-
+out:
if (!err)
trace_bpf_map_update_elem(map, ufd, key, value);
free_value:
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index e43491ac4823..f14e15702533 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -111,6 +111,7 @@ enum bpf_map_type {
BPF_MAP_TYPE_HASH_OF_MAPS,
BPF_MAP_TYPE_DEVMAP,
BPF_MAP_TYPE_SOCKMAP,
+ BPF_MAP_TYPE_CPUMAP,
};
enum bpf_prog_type {
^ permalink raw reply related
* [net-next PATCH 0/5] New bpf cpumap type for XDP_REDIRECT
From: Jesper Dangaard Brouer @ 2017-09-28 12:57 UTC (permalink / raw)
To: netdev
Cc: jakub.kicinski, Michael S. Tsirkin, Jason Wang, mchan,
John Fastabend, peter.waskiewicz.jr, Jesper Dangaard Brouer,
Daniel Borkmann, Alexei Starovoitov, Andy Gospodarek
Introducing a new way to redirect XDP frames. Notice how no driver
changes are necessary given the design of XDP_REDIRECT.
This redirect map type is called 'cpumap', as it allows redirection
XDP frames to remote CPUs. The remote CPU will do the SKB allocation
and start the network stack invocation on that CPU.
This is a scalability and isolation mechanism, that allow separating
the early driver network XDP layer, from the rest of the netstack, and
assigning dedicated CPUs for this stage. The sysadm control/configure
the RX-CPU to NIC-RX queue (as usual) via procfs smp_affinity and how
many queues are configured via ethtool --set-channels. Benchmarks
show that a single CPU can handle approx 11Mpps. Thus, only assigning
two NIC RX-queues (and two CPUs) is sufficient for handling 10Gbit/s
wirespeed smallest packet 14.88Mpps. Reducing the number of queues
have the advantage that more packets being "bulk" available per hard
interrupt[1].
[1] https://www.netdevconf.org/2.1/papers/BusyPollingNextGen.pdf
Use-cases:
1. End-host based pre-filtering for DDoS mitigation. This is fast
enough to allow software to see and filter all packets wirespeed.
Thus, no packets getting silently dropped by hardware.
2. Given NIC HW unevenly distributes packets across RX queue, this
mechanism can be used for redistribution load across CPUs. This
usually happens when HW is unaware of a new protocol. This
resembles RPS (Receive Packet Steering), just faster, but with more
responsibility placed on the BPF program for correct steering.
3. Auto-scaling or power saving via only activating the appropriate
number of remote CPUs for handling the current load. The cpumap
tracepoints can function as a feedback loop for this purpose.
Patchset based on net-next at:
commit 14a0d032f4ec ("Merge branch 'mlxsw-pass-gact'")
---
Jesper Dangaard Brouer (5):
bpf: introduce new bpf cpu map type BPF_MAP_TYPE_CPUMAP
bpf: XDP_REDIRECT enable use of cpumap
bpf: cpumap xdp_buff to skb conversion and allocation
bpf: cpumap add tracepoints
samples/bpf: add cpumap sample program xdp_redirect_cpu
include/linux/bpf.h | 7
include/linux/bpf_types.h | 1
include/trace/events/xdp.h | 80 ++++
include/uapi/linux/bpf.h | 1
kernel/bpf/Makefile | 1
kernel/bpf/cpumap.c | 671 +++++++++++++++++++++++++++++++++++
kernel/bpf/syscall.c | 8
kernel/bpf/verifier.c | 3
net/core/filter.c | 65 +++
samples/bpf/Makefile | 4
samples/bpf/xdp_redirect_cpu_kern.c | 640 +++++++++++++++++++++++++++++++++
samples/bpf/xdp_redirect_cpu_user.c | 639 +++++++++++++++++++++++++++++++++
tools/include/uapi/linux/bpf.h | 1
13 files changed, 2109 insertions(+), 12 deletions(-)
create mode 100644 kernel/bpf/cpumap.c
create mode 100644 samples/bpf/xdp_redirect_cpu_kern.c
create mode 100644 samples/bpf/xdp_redirect_cpu_user.c
^ permalink raw reply
* [PATCH ethtool 1/2] ethtool.8: Fix formatting of advertise bitmask
From: Gal Pressman @ 2017-09-28 12:08 UTC (permalink / raw)
To: netdev, John W . Linville; +Cc: Tariq Toukan, Pavel Belous, Gal Pressman
In-Reply-To: <1506600533-10365-1-git-send-email-galp@mellanox.com>
Fields should be separated with a tab instead of spaces.
Also, remove an accidental ' character from 2500baseX Full mode.
Fixes: 64dfc5e2f046 ("ethtool: Add support for 2500baseT/5000baseT link modes")
Signed-off-by: Gal Pressman <galp@mellanox.com>
Reviewed-by: Tariq Toukan <tariqt@mellanox.com>
---
ethtool.8.in | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/ethtool.8.in b/ethtool.8.in
index 7224b04..de2abc8 100644
--- a/ethtool.8.in
+++ b/ethtool.8.in
@@ -590,9 +590,9 @@ lB l lB.
0x020 1000baseT Full
0x20000 1000baseKX Full
0x20000000000 1000baseX Full
-0x800000000000 2500baseT Full
-0x8000 2500baseX Full (not supported by IEEE standards)'
-0x1000000000000 5000baseT Full
+0x800000000000 2500baseT Full
+0x8000 2500baseX Full (not supported by IEEE standards)
+0x1000000000000 5000baseT Full
0x1000 10000baseT Full
0x40000 10000baseKX4 Full
0x80000 10000baseKR Full
--
2.7.4
^ permalink raw reply related
* [PATCH ethtool 0/2] Documentation fixes and additions for advertise bitmask
From: Gal Pressman @ 2017-09-28 12:08 UTC (permalink / raw)
To: netdev, John W . Linville; +Cc: Tariq Toukan, Pavel Belous, Gal Pressman
Hi John,
The next two patches will fix bad formatting in the advertise modes bitmask and
document four missing 56000 modes.
Regards,
Gal.
Gal Pressman (2):
ethtool.8: Fix formatting of advertise bitmask
ethtool.8: Document 56000 advertise link modes
ethtool.8.in | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
--
2.7.4
^ permalink raw reply
* [PATCH ethtool 2/2] ethtool.8: Document 56000 advertise link modes
From: Gal Pressman @ 2017-09-28 12:08 UTC (permalink / raw)
To: netdev, John W . Linville; +Cc: Tariq Toukan, Pavel Belous, Gal Pressman
In-Reply-To: <1506600533-10365-1-git-send-email-galp@mellanox.com>
Add the following advertise modes to the manual:
56000baseKR4 Full
56000baseCR4 Full
56000baseSR4 Full
56000baseLR4 Full
Signed-off-by: Gal Pressman <galp@mellanox.com>
Reviewed-by: Tariq Toukan <tariqt@mellanox.com>
---
ethtool.8.in | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/ethtool.8.in b/ethtool.8.in
index de2abc8..1654c73 100644
--- a/ethtool.8.in
+++ b/ethtool.8.in
@@ -613,6 +613,10 @@ lB l lB.
0x400000000 50000baseCR2 Full
0x800000000 50000baseKR2 Full
0x10000000000 50000baseSR2 Full
+0x8000000 56000baseKR4 Full
+0x10000000 56000baseCR4 Full
+0x20000000 56000baseSR4 Full
+0x40000000 56000baseLR4 Full
0x1000000000 100000baseKR4 Full
0x2000000000 100000baseSR4 Full
0x4000000000 100000baseCR4 Full
--
2.7.4
^ permalink raw reply related
* EBPF-triggered WARNING at mm/percpu.c:1361 in v4-14-rc2
From: Mark Rutland @ 2017-09-28 11:27 UTC (permalink / raw)
To: linux-kernel, linux-mm, netdev, syzkaller
Cc: Daniel Borkmann, David S. Miller, Alexei Starovoitov, Tejun Heo,
Christoph Lameter
Hi,
While fuzzing v4.14-rc2 with Syzkaller, I found it was possible to trigger the
warning at mm/percpu.c:1361, on both arm64 and x86_64. This appears to require
increasing RLIMIT_MEMLOCK, so to the best of my knowledge this cannot be
triggered by an unprivileged user.
I've included example splats for both x86_64 and arm64, along with a C
reproducer, inline below.
It looks like dev_map_alloc() requests a percpu alloction of 32776 bytes, which
is larger than the maximum supported allocation size of 32768 bytes.
I wonder if it would make more sense to pr_warn() for sizes that are too
large, so that callers don't have to roll their own checks against
PCPU_MIN_UNIT_SIZE?
e.g. something like:
----
diff --git a/mm/percpu.c b/mm/percpu.c
index 59d44d6..f731c45 100644
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -1355,8 +1355,13 @@ static void __percpu *pcpu_alloc(size_t size, size_t align, bool reserved,
bits = size >> PCPU_MIN_ALLOC_SHIFT;
bit_align = align >> PCPU_MIN_ALLOC_SHIFT;
- if (unlikely(!size || size > PCPU_MIN_UNIT_SIZE || align > PAGE_SIZE ||
- !is_power_of_2(align))) {
+ if (unlikely(size > PCPU_MIN_UNIT_SIZE)) {
+ pr_warn("cannot allocate pcpu chunk of size %zu (max %zu)\n",
+ size, PCPU_MIN_UNIT_SIZE);
+ return NULL;
+ }
+
+ if (unlikely(!size || align > PAGE_SIZE || !is_power_of_2(align))) {
WARN(true, "illegal size (%zu) or align (%zu) for percpu allocation\n",
size, align);
return NULL;
----
Thanks,
Mark.
Example splat(x86_64)
----
[ 138.144185] illegal size (32776) or align (8) for percpu allocation
[ 138.150452] ------------[ cut here ]------------
[ 138.155074] WARNING: CPU: 1 PID: 2223 at mm/percpu.c:1361 pcpu_alloc+0x7c/0x5f0
[ 138.162369] Modules linked in:
[ 138.165423] CPU: 1 PID: 2223 Comm: repro Not tainted 4.14.0-rc2 #3
[ 138.171593] Hardware name: LENOVO 7484A3G/LENOVO, BIOS 5CKT54AUS 09/07/2009
[ 138.178543] task: ffff881b73069980 task.stack: ffffa36f40f90000
[ 138.184455] RIP: 0010:pcpu_alloc+0x7c/0x5f0
[ 138.188633] RSP: 0018:ffffa36f40f93e00 EFLAGS: 00010286
[ 138.193853] RAX: 0000000000000037 RBX: 0000000000000000 RCX: 0000000000000000
[ 138.200974] RDX: ffff881b7ec94a40 RSI: ffff881b7ec8cbb8 RDI: ffff881b7ec8cbb8
[ 138.208097] RBP: ffffa36f40f93e68 R08: 0000000000000001 R09: 00000000000002c4
[ 138.215219] R10: 0000562a577047f0 R11: ffffffffa10ad7cd R12: ffff881b73216cc0
[ 138.222343] R13: 0000000000000014 R14: 00007ffebeed0900 R15: ffffffffffffffea
[ 138.229463] FS: 00007fef84a15700(0000) GS:ffff881b7ec80000(0000) knlGS:0000000000000000
[ 138.237538] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 138.243274] CR2: 00007fef84497ba0 CR3: 00000001b3235000 CR4: 00000000000406e0
[ 138.250397] Call Trace:
[ 138.252844] __alloc_percpu+0x10/0x20
[ 138.256508] dev_map_alloc+0x122/0x1b0
[ 138.260255] SyS_bpf+0x8f9/0x10b0
[ 138.263570] ? security_task_setrlimit+0x3e/0x60
[ 138.268184] ? do_prlimit+0xa6/0x1f0
[ 138.271760] entry_SYSCALL_64_fastpath+0x13/0x94
[ 138.276372] RIP: 0033:0x7fef84546259
[ 138.279946] RSP: 002b:00007ffebeed09b8 EFLAGS: 00000206 ORIG_RAX: 0000000000000141
[ 138.287503] RAX: ffffffffffffffda RBX: 0000000000000000 RCX: 00007fef84546259
[ 138.294627] RDX: 0000000000000014 RSI: 00007ffebeed09d0 RDI: 0000000000000000
[ 138.301749] RBP: 0000562a57704780 R08: 00007fef84810cb0 R09: 00007ffebeed0ae8
[ 138.308874] R10: 0000562a577047f0 R11: 0000000000000206 R12: 0000562a577045d0
[ 138.315997] R13: 00007ffebeed0ae0 R14: 0000000000000000 R15: 0000000000000000
[ 138.323122] Code: fe 00 10 00 00 77 10 48 8b 4d b8 48 89 c8 48 83 e8 01 48 85 c1 74 1e 48 8b 55 b8 48 8b 75 c0 48 c7 c7 90 5e be a0 e8 40 88 f3 ff <0f> ff 45 31 ed e9 5e 02 00 00 4c 8b 6d c0 49 89 cc 49 c1 ec 02
[ 138.341953] ---[ end trace b6e380365bfb8a36 ]---
----
Example splat (arm64)
----
[ 17.287365] illegal size (32776) or align (8) for percpu allocation
[ 17.295347] ------------[ cut here ]------------
[ 17.297191] WARNING: CPU: 1 PID: 1440 at mm/percpu.c:1361 pcpu_alloc+0x120/0x9f0
[ 17.307723] Kernel panic - not syncing: panic_on_warn set ...
[ 17.307723]
[ 17.311755] CPU: 1 PID: 1440 Comm: repro Not tainted 4.14.0-rc2-00001-gd7ad33d #115
[ 17.320675] Hardware name: linux,dummy-virt (DT)
[ 17.323858] Call trace:
[ 17.325246] [<ffff200008094e98>] dump_backtrace+0x0/0x558
[ 17.332538] [<ffff200008095410>] show_stack+0x20/0x30
[ 17.340391] [<ffff20000a312628>] dump_stack+0x128/0x1a0
[ 17.342081] [<ffff20000815e330>] panic+0x250/0x518
[ 17.344096] [<ffff20000815e074>] __warn+0x2a4/0x310
[ 17.345654] [<ffff20000a310984>] report_bug+0x1d4/0x290
[ 17.348652] [<ffff2000080957c8>] bug_handler.part.1+0x40/0xf8
[ 17.356873] [<ffff2000080958cc>] bug_handler+0x4c/0x88
[ 17.360543] [<ffff20000808640c>] brk_handler+0x1c4/0x360
[ 17.365076] [<ffff200008081b68>] do_debug_exception+0x118/0x398
[ 17.368297] Exception stack(0xffff80001c82b930 to 0xffff80001c82ba70)
[ 17.372981] b920: 0000000000000037 0000000000000000
[ 17.380137] b940: bec1e481d6136f00 dfff200000000000 1fffe40001cbd30c dfff200000000000
[ 17.384902] b960: dfff200000000000 0000000000000000 ffff80001ce6c050 1ffff000039cd809
[ 17.392527] b980: ffff80001ce6c048 ffff80001ce6c068 1ffff000039cd80c 1ffff000039cd80e
[ 17.396935] b9a0: 1ffff000039cd80d ffff20000e1485a0 0000000000000000 0000000000000000
[ 17.404665] b9c0: ffff20000da58140 0000000000000000 00000000014000c0 0000000000000008
[ 17.407064] b9e0: 0000000000000004 000000000000800b 1ffff000039057b9 ffff80001c82bdcc
[ 17.415067] ba00: 0000000000000000 1ffff000039c0b14 1ffff000039c0b12 ffff80001c82ba70
[ 17.419137] ba20: ffff20000850c880 ffff80001c82ba70 ffff20000850c880 0000000080000145
[ 17.426052] ba40: 0000000000000008 ffff20000ae60b88 0001000000000000 00000000f4f4f404
[ 17.437346] ba60: ffff80001c82ba70 ffff20000850c880
[ 17.445759] [<ffff200008083ef0>] el1_dbg+0x18/0x74
[ 17.448272] [<ffff20000850c880>] pcpu_alloc+0x120/0x9f0
[ 17.456523] [<ffff20000850d1c8>] __alloc_percpu+0x30/0x40
[ 17.458412] [<ffff200008425d2c>] dev_map_alloc+0x58c/0x8d8
[ 17.462917] [<ffff2000083f0634>] SyS_bpf+0x86c/0x2d58
[ 17.468126] Exception stack(0xffff80001c82bec0 to 0xffff80001c82c000)
[ 17.470108] bec0: 0000000000000000 0000ffffe78f4898 0000000000000014 0000000000000000
[ 17.474655] bee0: 0000000000000000 0000ffffe78f49f0 0000001000000000 0000001000000000
[ 17.482196] bf00: 0000000000000118 0003ffffffffffff 0101010101010101 0000001000000000
[ 17.486928] bf20: 0000ffffb6468030 0000000000000000 0000ffffb6468028 000000000000071c
[ 17.498389] bf40: 0000ffffb63b8a00 0000aaaaba991028 0000000000000000 0000aaaaba9808f0
[ 17.502871] bf60: 0000000000000000 0000aaaaba980730 0000000000000000 0000000000000000
[ 17.505879] bf80: 0000000000000000 0000000000000000 0000000000000000 0000000000000000
[ 17.514090] bfa0: 0000000000000000 0000ffffe78f4870 0000aaaaba9808e0 0000ffffe78f4870
[ 17.520813] bfc0: 0000ffffb63b8a24 0000000080000000 0000000000000000 0000000000000118
[ 17.523888] bfe0: 0000000000000000 0000000000000000 0000000000000000 0000000000000000
[ 17.532537] [<ffff2000080846f0>] el0_svc_naked+0x24/0x28
[ 17.540427] SMP: stopping secondary CPUs
[ 17.566662] Kernel Offset: disabled
[ 17.567498] CPU features: 0x002082
[ 17.568238] Memory Limit: none
[ 17.568958] Rebooting in 86400 seconds..
----
C reproducer
----
#include <stdint.h>
#include <sys/resource.h>
#include <sys/syscall.h>
#include <unistd.h>
#include <linux/bpf.h>
/*
* Debian Stretch's headers are too old to contain a number of interesting
* values, so manually define them to keep things legible...
*/
struct LOCALDEF_bpf_attr {
uint32_t map_type;
uint32_t key_size;
uint32_t value_size;
uint32_t max_entries;
uint32_t map_flags;
};
#define LOCALDEF_BPF_MAP_TYPE_DEVMAP 0xe
int main(int argc, char *argv[])
{
struct rlimit rlimit = {
.rlim_cur = 8 << 20,
.rlim_max = 8 << 20,
};
setrlimit(RLIMIT_MEMLOCK, &rlimit);
struct LOCALDEF_bpf_attr attr = {
.map_type = LOCALDEF_BPF_MAP_TYPE_DEVMAP,
.key_size = 4,
.value_size = 4,
.max_entries = 0x40001,
.map_flags = 0,
};
syscall(__NR_bpf, BPF_MAP_CREATE, &attr, sizeof(attr));
return 0;
}
----
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply related
* Re: [PATCH net] ipv6: fix net.ipv6.conf.all interface DAD handlers
From: Erik Kline @ 2017-09-28 11:22 UTC (permalink / raw)
To: David Miller; +Cc: mcroce, netdev, linux-doc
In-Reply-To: <CAAedzxp4jdzycLXrXcdbpgnMCK0LJoaTiBV4m6=giV_T+WLkwA@mail.gmail.com>
[-- Attachment #1: Type: text/plain, Size: 952 bytes --]
On 28 September 2017 at 13:47, Erik Kline <ek@google.com> wrote:
>> Erik, please review.
>
> I apologize for the delay. I see that you've already applied this, and
> it's mostly LGTM except I have one thing I'm not seeing clearly.
>
> The documentation accept_dad now claims:
>
> DAD operation and mode on a given interface will be selected according
> to the maximum value of conf/{all,interface}/accept_dad.
>
> but I'm try to square this with my reading of the changes to
> addrconf_dad_begin(). I think setting all.accept_dad to 0 but
> ifname.accept_dad to non-0 still results in the short-circuit call to
> addrconf_dad_completed().
>
> Am I just not seeing (thinking) straight?
Upon further reflection, doesn't the whole premise of this change
means that it's no longer possible to selectively disable these
features if they are set on "all"? Or are we saying that this mode is
only support with "default" enable + "ifname" disable?
[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4835 bytes --]
^ permalink raw reply
* Re: [PATCH] xfrm: don't call xfrm_policy_cache_flush under xfrm_state_lock
From: Steffen Klassert @ 2017-09-28 11:12 UTC (permalink / raw)
To: Florian Westphal
Cc: Artem Savkov, Herbert Xu, David S . Miller, netdev, linux-kernel
In-Reply-To: <20170927123103.GB14971@breakpoint.cc>
On Wed, Sep 27, 2017 at 02:31:03PM +0200, Florian Westphal wrote:
> Artem Savkov <asavkov@redhat.com> wrote:
> > I might be wrong but it doesn't look like xfrm_state_lock is required
> > for xfrm_policy_cache_flush and calling it under this lock triggers both
> > "sleeping function called from invalid context" and "possible circular
> > locking dependency detected" warnings on flush.
> >
> > Fixes: ec30d78c14a8 xfrm: add xdst pcpu cache
> > Signed-off-by: Artem Savkov <asavkov@redhat.com>
>
> You're right, its not needed (and wrong).
>
> Acked-by: Florian Westphal <fw@strlen.de>
Applied to the ipsec tree, thanks everyone!
^ permalink raw reply
* Good day
From: Adams @ 2017-09-28 10:51 UTC (permalink / raw)
[-- Attachment #1: Type: text/plain, Size: 1 bytes --]
[-- Attachment #2: My dear.JPG --]
[-- Type: image/jpeg, Size: 161604 bytes --]
^ permalink raw reply
* Re: [PATCH/RFC net-next] ravb: RX checksum offload
From: Simon Horman @ 2017-09-28 10:49 UTC (permalink / raw)
To: Sergei Shtylyov; +Cc: David Miller, Magnus Damm, netdev, linux-renesas-soc
In-Reply-To: <9dcbc418-128b-2eff-970f-b586b9a3cee6@cogentembedded.com>
On Wed, Sep 13, 2017 at 08:54:00PM +0300, Sergei Shtylyov wrote:
> Hello!
>
> On 09/12/2017 04:04 PM, Simon Horman wrote:
>
> >Add support for RX checksum offload. This is enabled by default and
> >may be disabled and re-enabled using ethtool:
> >
> > # ethtool -K eth0 rx off
> > # ethtool -K eth0 rx on
> >
> >The RAVB provides a simple checksumming scheme which appears to be
> >completely compatible with CHECKSUM_COMPLETE: a 1's complement sum of
>
> Hm, the gen2/3 manuals say calculation doesn't involve bit inversion...
Yes, I believe that matches my observation of the values supplied by
the hardware. Empirically this appears to be what the kernel expects.
> >all packet data after the L2 header is appended to packet data; this may
> >be trivially read by the driver and used to update the skb accordingly.
> >
> >In terms of performance throughput is close to gigabit line-rate both with
> >and without RX checksum offload enabled. Perf output, however, appears to
> >indicate that significantly less time is spent in do_csum(). This is as
> >expected.
>
> [...]
>
> >By inspection this also appears to be compatible with the ravb found
> >on R-Car Gen 2 SoCs, however, this patch is currently untested on such
> >hardware.
>
> I probably won't be able to test it on gen2 too...
>
> >Signed-off-by: Simon Horman <horms+renesas@verge.net.au>
>
> I'm generally OK with the patch but have some questions/comments below...
Thanks, I will try to address them.
> >---
> > drivers/net/ethernet/renesas/ravb_main.c | 58 +++++++++++++++++++++++++++++++-
> > 1 file changed, 57 insertions(+), 1 deletion(-)
> >
> >diff --git a/drivers/net/ethernet/renesas/ravb_main.c b/drivers/net/ethernet/renesas/ravb_main.c
> >index fdf30bfa403b..7c6438cd7de7 100644
> >--- a/drivers/net/ethernet/renesas/ravb_main.c
> >+++ b/drivers/net/ethernet/renesas/ravb_main.c
> [...]
> >@@ -1842,6 +1859,41 @@ static int ravb_do_ioctl(struct net_device *ndev, struct ifreq *req, int cmd)
> > return phy_mii_ioctl(phydev, req, cmd);
> > }
> >+static void ravb_set_rx_csum(struct net_device *ndev, bool enable)
> >+{
> >+ struct ravb_private *priv = netdev_priv(ndev);
> >+ unsigned long flags;
> >+
> >+ spin_lock_irqsave(&priv->lock, flags);
> >+
> >+ /* Disable TX and RX */
> >+ ravb_rcv_snd_disable(ndev);
> >+
> >+ /* Modify RX Checksum setting */
> >+ if (enable)
> >+ ravb_modify(ndev, ECMR, 0, ECMR_RCSC);
>
> Please use ECMR_RCSC as the 3rd argument too to conform the common driver
> style.
>
> >+ else
> >+ ravb_modify(ndev, ECMR, ECMR_RCSC, 0);
>
> This *if* can easily be folded into a single ravb_modify() call...
Thanks, something like this?
ravb_modify(ndev, ECMR, ECMR_RCSC, enable ? ECMR_RCSC : 0);
> [...]
> >@@ -2004,6 +2057,9 @@ static int ravb_probe(struct platform_device *pdev)
> > if (!ndev)
> > return -ENOMEM;
> >+ ndev->features |= NETIF_F_RXCSUM;
> >+ ndev->hw_features |= ndev->features;
>
> Hum, both fields are 0 before this? Then why not use '=' instead of '|='?
> Even if not, why not just use the same value as both the rvalues?
I don't feel strongly about this, how about?
ndev->features = NETIF_F_RXCSUM;
ndev->hw_features = NETIF_F_RXCSUM;
^ permalink raw reply
* Re: [PATCH v2] netlink: do not proceed if dump's start() errs
From: Jason A. Donenfeld @ 2017-09-28 10:40 UTC (permalink / raw)
To: davem, johannes.berg, netdev, linux-kernel; +Cc: Johannes Berg
In-Reply-To: <20170927224144.1749-1-Jason@zx2c4.com>
On Thu, Sep 28, 2017 at 12:41:44AM +0200, Jason A. Donenfeld wrote:
> diff --git a/net/netlink/af_netlink.c b/net/netlink/af_netlink.c
> index 327807731b44..94c11cf0459d 100644
> --- a/net/netlink/af_netlink.c
> +++ b/net/netlink/af_netlink.c
> @@ -2270,10 +2270,13 @@ int __netlink_dump_start(struct sock *ssk, struct sk_buff *skb,
>
> mutex_unlock(nlk->cb_mutex);
>
> + ret = 0;
> if (cb->start)
> - cb->start(cb);
> + ret = cb->start(cb);
> +
> + if (!ret)
> + ret = netlink_dump(sk);
>
> - ret = netlink_dump(sk);
> sock_put(sk);
>
> if (ret)
FYI, using this horrific hack to currently work around bug:
#define KERNEL_VERSION_THAT_HAS_NETLINK_START_FIX KERNEL_VERSION(4, 14, 0) /* Hopefully! */
static int get(struct sk_buff *skb, struct netlink_callback *cb)
{
#if LINUX_VERSION_CODE < KERNEL_VERSION_THAT_HAS_NETLINK_START_FIX
/* https://marc.info/?l=linux-netdev&m=150655213004221&w=2 */
if (!cb->args[0]) {
ret = get_start(cb);
if (ret)
return ret;
}
#endif
...
}
static const struct genl_ops genl_ops[] = {
{
.cmd = CMD_GET,
#if LINUX_VERSION_CODE >= KERNEL_VERSION_THAT_HAS_NETLINK_START_FIX
/* https://marc.info/?l=linux-netdev&m=150655213004221&w=2 */
.start = get_start,
#endif
.dumpit = get,
...
}
}
Gross.
^ permalink raw reply
* Re: [PATCH net-next] ipv6: do lazy dst->__use update when per cpu dst is available
From: Paolo Abeni @ 2017-09-28 10:34 UTC (permalink / raw)
To: Eric Dumazet, Wei Wang
Cc: Martin KaFai Lau, Linux Kernel Network Developers,
David S. Miller, Hannes Frederic Sowa
In-Reply-To: <CANn89iJ0t8Vpukak3WEOA1x4M4hk-Z9hDkcSqG9+7qLsbuJQZg@mail.gmail.com>
On Wed, 2017-09-27 at 11:03 -0700, Eric Dumazet wrote:
> > > diff --git a/net/ipv6/route.c b/net/ipv6/route.c
> >
> > Hi Paolo,
> >
> > Eric and I discussed about this issue recently as well :).
> >
> > What about the following change:
> >
> > diff --git a/include/net/dst.h b/include/net/dst.h
> > index 93568bd0a352..33e1d86bcef6 100644
> > --- a/include/net/dst.h
> > +++ b/include/net/dst.h
> > @@ -258,14 +258,18 @@ static inline void dst_hold(struct dst_entry *dst)
> > static inline void dst_use(struct dst_entry *dst, unsigned long time)
> > {
> > dst_hold(dst);
> > - dst->__use++;
> > - dst->lastuse = time;
> > + if (dst->lastuse != time) {
> > + dst->__use++;
> > + dst->lastuse = time;
> > + }
> > }
> >
> > static inline void dst_use_noref(struct dst_entry *dst, unsigned long time)
> > {
> > - dst->__use++;
> > - dst->lastuse = time;
> > + if (dst->lastuse != time) {
> > + dst->__use++;
> > + dst->lastuse = time;
> > + }
> > }
> >
> > static inline struct dst_entry *dst_clone(struct dst_entry *dst)
> > diff --git a/net/ipv6/route.c b/net/ipv6/route.c
> > index 26cc9f483b6d..e195f093add3 100644
> > --- a/net/ipv6/route.c
> > +++ b/net/ipv6/route.c
> > @@ -1170,8 +1170,7 @@ struct rt6_info *ip6_pol_route(struct net *net,
> > struct fib6_table *table,
> >
> > struct rt6_info *pcpu_rt;
> >
> > - rt->dst.lastuse = jiffies;
> > - rt->dst.__use++;
> > + dst_use_noref(rt, jiffies);
> > pcpu_rt = rt6_get_pcpu_route(rt);
> >
> > if (pcpu_rt) {
> >
> >
> > This way we always only update dst->__use and dst->lastuse at most
> > once per jiffy. And we don't really need to update pcpu and then do
> > the copy over from pcpu_rt to rt operation.
> >
> > Another thing is that I don't really see any places making use of
> > dst->__use. So maybe we can also get rid of this dst->__use field?
> >
> > Thanks.
> > Wei
>
> Paolo, given we are very close to send Wei awesome work about IPv6
> routing cache,
> could we ask you to wait few days before doing the same work from your side ?
Ok, no problem - thanks instead. I'll wait for it.
> Main issue is the rwlock, and we are converting it to full RCU.
>
> You are sending patches that are making our job very difficult IMO.
On my side I have only another small change in this area, I'll
eventually try to rebase it later, if still relevant.
Or I can share it now, if you are interested.
Cheers,
Paolo
^ permalink raw reply
* [PATCH v4 2/2] ip_tunnel: add mpls over gre encapsulation
From: Amine Kherbouche @ 2017-09-28 9:34 UTC (permalink / raw)
To: netdev, xeb, roopa; +Cc: amine.kherbouche, equinox
In-Reply-To: <cover.1506590878.git.amine.kherbouche@6wind.com>
This commit introduces the MPLSoGRE support (RFC 4023), using ip tunnel
API.
Encap:
- Add a new iptunnel type mpls.
- Share tx path: gre type mpls loaded from skb->protocol.
Decap:
- pull gre hdr and call mpls_forward().
Signed-off-by: Amine Kherbouche <amine.kherbouche@6wind.com>
Acked-by: Roopa Prabhu <roopa@cumulusnetworks.com>
---
include/net/gre.h | 1 +
include/uapi/linux/if_tunnel.h | 1 +
net/ipv4/gre_demux.c | 27 +++++++++++++++++++++++++++
net/ipv4/ip_gre.c | 3 +++
net/ipv6/ip6_gre.c | 3 +++
net/mpls/af_mpls.c | 36 ++++++++++++++++++++++++++++++++++++
6 files changed, 71 insertions(+)
diff --git a/include/net/gre.h b/include/net/gre.h
index d25d836..aa3c4d3 100644
--- a/include/net/gre.h
+++ b/include/net/gre.h
@@ -35,6 +35,7 @@ struct net_device *gretap_fb_dev_create(struct net *net, const char *name,
u8 name_assign_type);
int gre_parse_header(struct sk_buff *skb, struct tnl_ptk_info *tpi,
bool *csum_err, __be16 proto, int nhs);
+int mpls_gre_rcv(struct sk_buff *skb, int gre_hdr_len);
static inline int gre_calc_hlen(__be16 o_flags)
{
diff --git a/include/uapi/linux/if_tunnel.h b/include/uapi/linux/if_tunnel.h
index 2e52088..a2f48c0 100644
--- a/include/uapi/linux/if_tunnel.h
+++ b/include/uapi/linux/if_tunnel.h
@@ -84,6 +84,7 @@ enum tunnel_encap_types {
TUNNEL_ENCAP_NONE,
TUNNEL_ENCAP_FOU,
TUNNEL_ENCAP_GUE,
+ TUNNEL_ENCAP_MPLS,
};
#define TUNNEL_ENCAP_FLAG_CSUM (1<<0)
diff --git a/net/ipv4/gre_demux.c b/net/ipv4/gre_demux.c
index b798862..40484a3 100644
--- a/net/ipv4/gre_demux.c
+++ b/net/ipv4/gre_demux.c
@@ -23,6 +23,9 @@
#include <linux/netdevice.h>
#include <linux/if_tunnel.h>
#include <linux/spinlock.h>
+#if IS_ENABLED(CONFIG_MPLS)
+#include <linux/mpls.h>
+#endif
#include <net/protocol.h>
#include <net/gre.h>
@@ -122,6 +125,30 @@ int gre_parse_header(struct sk_buff *skb, struct tnl_ptk_info *tpi,
}
EXPORT_SYMBOL(gre_parse_header);
+#if IS_ENABLED(CONFIG_MPLS)
+int mpls_gre_rcv(struct sk_buff *skb, int gre_hdr_len)
+{
+ if (unlikely(!pskb_may_pull(skb, gre_hdr_len)))
+ goto drop;
+
+ /* Pop GRE hdr and reset the skb */
+ skb_pull(skb, gre_hdr_len);
+ skb_reset_network_header(skb);
+
+ return mpls_forward(skb, skb->dev, NULL, NULL);
+drop:
+ kfree_skb(skb);
+ return NET_RX_DROP;
+}
+#else
+int mpls_gre_rcv(struct sk_buff *skb, int gre_hdr_len)
+{
+ kfree_skb(skb);
+ return NET_RX_DROP;
+}
+#endif
+EXPORT_SYMBOL(mpls_gre_rcv);
+
static int gre_rcv(struct sk_buff *skb)
{
const struct gre_protocol *proto;
diff --git a/net/ipv4/ip_gre.c b/net/ipv4/ip_gre.c
index 9cee986..7a50e4f 100644
--- a/net/ipv4/ip_gre.c
+++ b/net/ipv4/ip_gre.c
@@ -412,6 +412,9 @@ static int gre_rcv(struct sk_buff *skb)
return 0;
}
+ if (unlikely(tpi.proto == htons(ETH_P_MPLS_UC)))
+ return mpls_gre_rcv(skb, hdr_len);
+
if (ipgre_rcv(skb, &tpi, hdr_len) == PACKET_RCVD)
return 0;
diff --git a/net/ipv6/ip6_gre.c b/net/ipv6/ip6_gre.c
index c82d41e..440efb1 100644
--- a/net/ipv6/ip6_gre.c
+++ b/net/ipv6/ip6_gre.c
@@ -476,6 +476,9 @@ static int gre_rcv(struct sk_buff *skb)
if (hdr_len < 0)
goto drop;
+ if (unlikely(tpi.proto == htons(ETH_P_MPLS_UC)))
+ return mpls_gre_rcv(skb, hdr_len);
+
if (iptunnel_pull_header(skb, hdr_len, tpi.proto, false))
goto drop;
diff --git a/net/mpls/af_mpls.c b/net/mpls/af_mpls.c
index 36ea2ad..4274243 100644
--- a/net/mpls/af_mpls.c
+++ b/net/mpls/af_mpls.c
@@ -16,6 +16,7 @@
#include <net/arp.h>
#include <net/ip_fib.h>
#include <net/netevent.h>
+#include <net/ip_tunnels.h>
#include <net/netns/generic.h>
#if IS_ENABLED(CONFIG_IPV6)
#include <net/ipv6.h>
@@ -39,6 +40,36 @@ static int one = 1;
static int label_limit = (1 << 20) - 1;
static int ttl_max = 255;
+#if IS_ENABLED(CONFIG_NET_IP_TUNNEL)
+size_t ipgre_mpls_encap_hlen(struct ip_tunnel_encap *e)
+{
+ return sizeof(struct mpls_shim_hdr);
+}
+
+static const struct ip_tunnel_encap_ops mpls_iptun_ops = {
+ .encap_hlen = ipgre_mpls_encap_hlen,
+};
+
+static int ipgre_tunnel_encap_add_mpls_ops(void)
+{
+ return ip_tunnel_encap_add_ops(&mpls_iptun_ops, TUNNEL_ENCAP_MPLS);
+}
+
+static void ipgre_tunnel_encap_del_mpls_ops(void)
+{
+ ip_tunnel_encap_del_ops(&mpls_iptun_ops, TUNNEL_ENCAP_MPLS);
+}
+#else
+static int ipgre_tunnel_encap_add_mpls_ops(void)
+{
+ return 0;
+}
+
+static void ipgre_tunnel_encap_del_mpls_ops(void)
+{
+}
+#endif
+
static void rtmsg_lfib(int event, u32 label, struct mpls_route *rt,
struct nlmsghdr *nlh, struct net *net, u32 portid,
unsigned int nlm_flags);
@@ -2486,6 +2517,10 @@ static int __init mpls_init(void)
0);
rtnl_register(PF_MPLS, RTM_GETNETCONF, mpls_netconf_get_devconf,
mpls_netconf_dump_devconf, 0);
+ err = ipgre_tunnel_encap_add_mpls_ops();
+ if (err)
+ pr_err("Can't add mpls over gre tunnel ops\n");
+
err = 0;
out:
return err;
@@ -2503,6 +2538,7 @@ static void __exit mpls_exit(void)
dev_remove_pack(&mpls_packet_type);
unregister_netdevice_notifier(&mpls_dev_notifier);
unregister_pernet_subsys(&mpls_net_ops);
+ ipgre_tunnel_encap_del_mpls_ops();
}
module_exit(mpls_exit);
--
2.1.4
^ permalink raw reply related
* [PATCH v4 1/2] expose stack entry function
From: Amine Kherbouche @ 2017-09-28 9:34 UTC (permalink / raw)
To: netdev, xeb, roopa; +Cc: amine.kherbouche, equinox
In-Reply-To: <cover.1506590878.git.amine.kherbouche@6wind.com>
Exporting mpls_forward() function to be able to be called from elsewhere
such as MPLSoverGRE in the next commit.
Signed-off-by: Amine Kherbouche <amine.kherbouche@6wind.com>
Acked-by: Roopa Prabhu <roopa@cumulusnetworks.com>
---
include/linux/mpls.h | 4 ++++
net/mpls/af_mpls.c | 5 +++--
2 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/include/linux/mpls.h b/include/linux/mpls.h
index 384fb22..b0579ab 100644
--- a/include/linux/mpls.h
+++ b/include/linux/mpls.h
@@ -2,10 +2,14 @@
#define _LINUX_MPLS_H
#include <uapi/linux/mpls.h>
+#include <linux/netdevice.h>
#define MPLS_TTL_MASK (MPLS_LS_TTL_MASK >> MPLS_LS_TTL_SHIFT)
#define MPLS_BOS_MASK (MPLS_LS_S_MASK >> MPLS_LS_S_SHIFT)
#define MPLS_TC_MASK (MPLS_LS_TC_MASK >> MPLS_LS_TC_SHIFT)
#define MPLS_LABEL_MASK (MPLS_LS_LABEL_MASK >> MPLS_LS_LABEL_SHIFT)
+int mpls_forward(struct sk_buff *skb, struct net_device *dev,
+ struct packet_type *pt, struct net_device *orig_dev);
+
#endif /* _LINUX_MPLS_H */
diff --git a/net/mpls/af_mpls.c b/net/mpls/af_mpls.c
index c5b9ce4..36ea2ad 100644
--- a/net/mpls/af_mpls.c
+++ b/net/mpls/af_mpls.c
@@ -307,8 +307,8 @@ static bool mpls_egress(struct net *net, struct mpls_route *rt,
return success;
}
-static int mpls_forward(struct sk_buff *skb, struct net_device *dev,
- struct packet_type *pt, struct net_device *orig_dev)
+int mpls_forward(struct sk_buff *skb, struct net_device *dev,
+ struct packet_type *pt, struct net_device *orig_dev)
{
struct net *net = dev_net(dev);
struct mpls_shim_hdr *hdr;
@@ -442,6 +442,7 @@ static int mpls_forward(struct sk_buff *skb, struct net_device *dev,
kfree_skb(skb);
return NET_RX_DROP;
}
+EXPORT_SYMBOL(mpls_forward);
static struct packet_type mpls_packet_type __read_mostly = {
.type = cpu_to_be16(ETH_P_MPLS_UC),
--
2.1.4
^ permalink raw reply related
* [PATCH v4 0/2] Introduce MPLS over GRE
From: Amine Kherbouche @ 2017-09-28 9:34 UTC (permalink / raw)
To: netdev, xeb, roopa; +Cc: amine.kherbouche, equinox
This series introduces the MPLS over GRE encapsulation (RFC 4023).
Various applications of MPLS make use of label stacks with multiple
entries. In some cases, it is possible to replace the top label of
the stack with an IP-based encapsulation, thereby, it is possible for
two LSRs that are adjacent on an LSP to be separated by an IP network,
even if that IP network does not provide MPLS.
Changes in v4:
- Bring back mpls_forward() function exportation patch.
- Move back mpls_gre_rcv() to gre module file and wrap it under
ifdef.
Changes in v3:
- remove mpls_forward() function exportation patch.
- wrap efficiently mpls iptunnel add/del functions and dependent
function/structure.
- move mpls_gre_rcv to af_mpls.c file and export it.
- remove unnecessary functions.
Changes in v2:
- wrap ip tunnel functions under ifdef in mpls file.
- fix indentation.
- check return code.
An example of configuration:
node1 LER1 LER2 node2
+-----+ +------+ +------+ +-----+
| | | | | | | |
| | | |p3 GRE tunnel p4| | | |
| |p1 p2| +-------------------+ |p5 p6| |
| +-------------+ +-------------------+ +------------+| |
| |10.100.0.0/24| | | |10.200.0.0/24| |
| |fd00:100::/64| | 10.125.0.0/24 | |fd00:200::/64| |
| | | | fd00:125::/64 | | | |
| | | | | | | |
| | | | | | | |
| | | | | | | |
| | | | | | | |
+-----+ +------+ +------+ +-----+
### node1 ###
ip link set p1 up
ip addr add 10.100.0.1/24 dev p1
### LER1 ###
ip link set p2 up
ip addr add 10.100.0.2/24 dev p2
ip link set p3 up
ip addr add 10.125.0.1/24 dev p3
modprobe mpls_router
sysctl -w net.mpls.conf.p2.input=1
sysctl -w net.mpls.conf.p3.input=1
sysctl -w net.mpls.platform_labels=1000
ip link add gre1 type gre ttl 64 local 10.125.0.1 remote 10.125.0.2 dev p3
ip link set dev gre1 up
ip -M route add 111 as 222 dev gre1
ip -M route add 555 as 666 via inet 10.100.0.1 dev p2
### LER2 ###
ip link set p5 up
ip addr add 10.200.0.2/24 dev p5
ip link set p4 up
ip addr add 10.125.0.2/24 dev p4
modprobe mpls_router
sysctl -w net.mpls.conf.p4.input=1
sysctl -w net.mpls.conf.p5.input=1
sysctl -w net.mpls.platform_labels=1000
ip link add gre1 type gre ttl 64 local 10.125.0.2 remote 10.125.0.1 dev p4
ip link set dev gre1 up
ip -M route add 444 as 555 dev gre1
ip -M route add 222 as 333 via inet 10.200.0.1 dev p5
### node2 ###
ip link set p6 up
ip addr add 10.200.0.1/24 dev p6
Now using this scapy to forge and send packets from the port p1 of node1:
p = Ether(src='de:ed:01:0c:41:09', dst='de:ed:01:2f:3b:ba')
p /= MPLS(s=1, ttl=64, label=111)/Raw(load='\xde')
sendp(p, iface="p1", count=20, inter=0.1)
Amine Kherbouche (2):
expose stack entry function
ip_tunnel: add mpls over gre encapsulation
include/linux/mpls.h | 4 ++++
include/net/gre.h | 1 +
include/uapi/linux/if_tunnel.h | 1 +
net/ipv4/gre_demux.c | 27 +++++++++++++++++++++++++++
net/ipv4/ip_gre.c | 3 +++
net/ipv6/ip6_gre.c | 3 +++
net/mpls/af_mpls.c | 41 +++++++++++++++++++++++++++++++++++++++--
7 files changed, 78 insertions(+), 2 deletions(-)
--
2.1.4
^ permalink raw reply
* [PATCH v3] tun: bail out from tun_get_user() if the skb is empty
From: Alexander Potapenko @ 2017-09-28 9:32 UTC (permalink / raw)
To: davem, edumazet; +Cc: dvyukov, syzkaller, netdev, linux-kernel
KMSAN (https://github.com/google/kmsan) reported accessing uninitialized
skb->data[0] in the case the skb is empty (i.e. skb->len is 0):
================================================
BUG: KMSAN: use of uninitialized memory in tun_get_user+0x19ba/0x3770
CPU: 0 PID: 3051 Comm: probe Not tainted 4.13.0+ #3140
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS Bochs 01/01/2011
Call Trace:
...
__msan_warning_32+0x66/0xb0 mm/kmsan/kmsan_instr.c:477
tun_get_user+0x19ba/0x3770 drivers/net/tun.c:1301
tun_chr_write_iter+0x19f/0x300 drivers/net/tun.c:1365
call_write_iter ./include/linux/fs.h:1743
new_sync_write fs/read_write.c:457
__vfs_write+0x6c3/0x7f0 fs/read_write.c:470
vfs_write+0x3e4/0x770 fs/read_write.c:518
SYSC_write+0x12f/0x2b0 fs/read_write.c:565
SyS_write+0x55/0x80 fs/read_write.c:557
do_syscall_64+0x242/0x330 arch/x86/entry/common.c:284
entry_SYSCALL64_slow_path+0x25/0x25 arch/x86/entry/entry_64.S:245
...
origin:
...
kmsan_poison_shadow+0x6e/0xc0 mm/kmsan/kmsan.c:211
slab_alloc_node mm/slub.c:2732
__kmalloc_node_track_caller+0x351/0x370 mm/slub.c:4351
__kmalloc_reserve net/core/skbuff.c:138
__alloc_skb+0x26a/0x810 net/core/skbuff.c:231
alloc_skb ./include/linux/skbuff.h:903
alloc_skb_with_frags+0x1d7/0xc80 net/core/skbuff.c:4756
sock_alloc_send_pskb+0xabf/0xfe0 net/core/sock.c:2037
tun_alloc_skb drivers/net/tun.c:1144
tun_get_user+0x9a8/0x3770 drivers/net/tun.c:1274
tun_chr_write_iter+0x19f/0x300 drivers/net/tun.c:1365
call_write_iter ./include/linux/fs.h:1743
new_sync_write fs/read_write.c:457
__vfs_write+0x6c3/0x7f0 fs/read_write.c:470
vfs_write+0x3e4/0x770 fs/read_write.c:518
SYSC_write+0x12f/0x2b0 fs/read_write.c:565
SyS_write+0x55/0x80 fs/read_write.c:557
do_syscall_64+0x242/0x330 arch/x86/entry/common.c:284
return_from_SYSCALL_64+0x0/0x6a arch/x86/entry/entry_64.S:245
================================================
Make sure tun_get_user() doesn't touch skb->data[0] unless there is
actual data.
C reproducer below:
==========================
// autogenerated by syzkaller (http://github.com/google/syzkaller)
#define _GNU_SOURCE
#include <fcntl.h>
#include <linux/if_tun.h>
#include <netinet/ip.h>
#include <net/if.h>
#include <string.h>
#include <sys/ioctl.h>
int main()
{
int sock = socket(PF_INET, SOCK_STREAM, IPPROTO_IP);
int tun_fd = open("/dev/net/tun", O_RDWR);
struct ifreq req;
memset(&req, 0, sizeof(struct ifreq));
strcpy((char*)&req.ifr_name, "gre0");
req.ifr_flags = IFF_UP | IFF_MULTICAST;
ioctl(tun_fd, TUNSETIFF, &req);
ioctl(sock, SIOCSIFFLAGS, "gre0");
write(tun_fd, "hi", 0);
return 0;
}
==========================
Signed-off-by: Alexander Potapenko <glider@google.com>
---
v3: per request by Eric Dumazet introduce ip_version to avoid copy/paste
v2: free the skb
---
drivers/net/tun.c | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/drivers/net/tun.c b/drivers/net/tun.c
index 3c9985f29950..5ce580f413b9 100644
--- a/drivers/net/tun.c
+++ b/drivers/net/tun.c
@@ -1496,11 +1496,13 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,
switch (tun->flags & TUN_TYPE_MASK) {
case IFF_TUN:
if (tun->flags & IFF_NO_PI) {
- switch (skb->data[0] & 0xf0) {
- case 0x40:
+ u8 ip_version = skb->len ? (skb->data[0] >> 4) : 0;
+
+ switch (ip_version) {
+ case 4:
pi.proto = htons(ETH_P_IP);
break;
- case 0x60:
+ case 6:
pi.proto = htons(ETH_P_IPV6);
break;
default:
--
2.14.2.822.g60be5d43e6-goog
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox