* Re: [bpf-next v3 09/12] bpf: Split out some helper functions
From: Krzesimir Nowak @ 2019-07-08 16:40 UTC (permalink / raw)
To: linux-kernel
Cc: Alban Crequy, Iago López Galeiras, Alexei Starovoitov,
Daniel Borkmann, Martin KaFai Lau, Song Liu, Yonghong Song,
David S. Miller, Jakub Kicinski, Jesper Dangaard Brouer,
John Fastabend, Stanislav Fomichev, netdev, bpf, xdp-newbies
In-Reply-To: <20190708163121.18477-10-krzesimir@kinvolk.io>
On Mon, Jul 8, 2019 at 6:31 PM Krzesimir Nowak <krzesimir@kinvolk.io> wrote:
>
> The moved functions are generally useful for implementing
> bpf_prog_test_run for other types of BPF programs - they don't have
> any network-specific stuff in them, so I can use them in a test run
> implementation for perf event BPF program too.
>
> Signed-off-by: Krzesimir Nowak <krzesimir@kinvolk.io>
This patch has some whitespace issues (indentation with spaces instead
of tabs in some places), will fix them for v4. Sorry about that.
> ---
> include/linux/bpf.h | 28 +++++
> kernel/bpf/Makefile | 1 +
> kernel/bpf/test_run.c | 212 ++++++++++++++++++++++++++++++++++
> net/bpf/test_run.c | 263 +++++++++++-------------------------------
> 4 files changed, 308 insertions(+), 196 deletions(-)
> create mode 100644 kernel/bpf/test_run.c
>
> diff --git a/include/linux/bpf.h b/include/linux/bpf.h
> index 18f4cc2c6acd..28db8ba57bc3 100644
> --- a/include/linux/bpf.h
> +++ b/include/linux/bpf.h
> @@ -1143,4 +1143,32 @@ static inline u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type,
> }
> #endif /* CONFIG_INET */
>
> +/* Helper functions for bpf_prog_test_run implementations */
> +typedef u32 bpf_prog_run_helper_t(struct bpf_prog *prog, void *ctx,
> + void *private_data);
> +
> +enum bpf_test_run_flags {
> + BPF_TEST_RUN_PLAIN = 0,
> + BPF_TEST_RUN_SETUP_CGROUP_STORAGE = 1 << 0,
> +};
> +
> +int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat, u32 flags,
> + u32 *retval, u32 *duration);
> +
> +int bpf_test_run_cb(struct bpf_prog *prog, void *ctx, u32 repeat, u32 flags,
> + bpf_prog_run_helper_t run_prog, void *private_data,
> + u32 *retval, u32 *duration);
> +
> +int bpf_test_finish(union bpf_attr __user *uattr, u32 retval, u32 duration);
> +
> +void *bpf_receive_ctx(const union bpf_attr *kattr, u32 max_size);
> +
> +int bpf_send_ctx(const union bpf_attr *kattr, union bpf_attr __user *uattr,
> + const void *data, u32 size);
> +
> +void *bpf_receive_data(const union bpf_attr *kattr, u32 max_size);
> +
> +int bpf_send_data(const union bpf_attr *kattr, union bpf_attr __user *uattr,
> + const void *data, u32 size);
> +
> #endif /* _LINUX_BPF_H */
> diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile
> index 29d781061cd5..570fd40288f4 100644
> --- a/kernel/bpf/Makefile
> +++ b/kernel/bpf/Makefile
> @@ -22,3 +22,4 @@ obj-$(CONFIG_CGROUP_BPF) += cgroup.o
> ifeq ($(CONFIG_INET),y)
> obj-$(CONFIG_BPF_SYSCALL) += reuseport_array.o
> endif
> +obj-$(CONFIG_BPF_SYSCALL) += test_run.o
> diff --git a/kernel/bpf/test_run.c b/kernel/bpf/test_run.c
> new file mode 100644
> index 000000000000..0481373da8be
> --- /dev/null
> +++ b/kernel/bpf/test_run.c
> @@ -0,0 +1,212 @@
> +// SPDX-License-Identifier: GPL-2.0-only
> +/* Copyright (c) 2017 Facebook
> + * Copyright (c) 2019 Tigera, Inc
> + */
> +
> +#include <asm/div64.h>
> +
> +#include <linux/bpf-cgroup.h>
> +#include <linux/bpf.h>
> +#include <linux/err.h>
> +#include <linux/errno.h>
> +#include <linux/filter.h>
> +#include <linux/gfp.h>
> +#include <linux/kernel.h>
> +#include <linux/limits.h>
> +#include <linux/preempt.h>
> +#include <linux/rcupdate.h>
> +#include <linux/sched.h>
> +#include <linux/sched/signal.h>
> +#include <linux/slab.h>
> +#include <linux/timekeeping.h>
> +#include <linux/uaccess.h>
> +
> +static void teardown_cgroup_storage(struct bpf_cgroup_storage **storage)
> +{
> + enum bpf_cgroup_storage_type stype;
> +
> + if (!storage)
> + return;
> + for_each_cgroup_storage_type(stype)
> + bpf_cgroup_storage_free(storage[stype]);
> + kfree(storage);
> +}
> +
> +static struct bpf_cgroup_storage **setup_cgroup_storage(struct bpf_prog *prog)
> +{
> + enum bpf_cgroup_storage_type stype;
> + struct bpf_cgroup_storage **storage;
> + size_t size = MAX_BPF_CGROUP_STORAGE_TYPE;
> +
> + size *= sizeof(struct bpf_cgroup_storage *);
> + storage = kzalloc(size, GFP_KERNEL);
> + for_each_cgroup_storage_type(stype) {
> + storage[stype] = bpf_cgroup_storage_alloc(prog, stype);
> + if (IS_ERR(storage[stype])) {
> + storage[stype] = NULL;
> + teardown_cgroup_storage(storage);
> + return ERR_PTR(-ENOMEM);
> + }
> + }
> + return storage;
> +}
> +
> +static u32 run_bpf_prog(struct bpf_prog *prog, void *ctx, void *private_data)
> +{
> + return BPF_PROG_RUN(prog, ctx);
> +}
> +
> +int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat, u32 flags,
> + u32 *retval, u32 *duration)
> +{
> + return bpf_test_run_cb(prog, ctx, repeat, flags, run_bpf_prog, NULL,
> + retval, duration);
> +}
> +
> +int bpf_test_run_cb(struct bpf_prog *prog, void *ctx, u32 repeat, u32 flags,
> + bpf_prog_run_helper_t run_prog, void *private_data,
> + u32 *retval, u32 *duration)
> +{
> + struct bpf_cgroup_storage **storage = NULL;
> + u64 time_start, time_spent = 0;
> + int ret = 0;
> + u32 i;
> +
> + if (flags & BPF_TEST_RUN_SETUP_CGROUP_STORAGE) {
> + storage = setup_cgroup_storage(prog);
> + if (IS_ERR(storage))
> + return PTR_ERR(storage);
> + }
> +
> + if (!repeat)
> + repeat = 1;
> +
> + rcu_read_lock();
> + preempt_disable();
> + time_start = ktime_get_ns();
> + for (i = 0; i < repeat; i++) {
> + if (storage)
> + bpf_cgroup_storage_set(storage);
> + *retval = run_prog(prog, ctx, private_data);
> +
> + if (signal_pending(current)) {
> + preempt_enable();
> + rcu_read_unlock();
> + teardown_cgroup_storage(storage);
> + return -EINTR;
> + }
> +
> + if (need_resched()) {
> + time_spent += ktime_get_ns() - time_start;
> + preempt_enable();
> + rcu_read_unlock();
> +
> + cond_resched();
> +
> + rcu_read_lock();
> + preempt_disable();
> + time_start = ktime_get_ns();
> + }
> + }
> + time_spent += ktime_get_ns() - time_start;
> + preempt_enable();
> + rcu_read_unlock();
> +
> + do_div(time_spent, repeat);
> + *duration = time_spent > U32_MAX ? U32_MAX : (u32)time_spent;
> +
> + teardown_cgroup_storage(storage);
> +
> + return ret;
> +}
> +
> +int bpf_test_finish(union bpf_attr __user *uattr, u32 retval, u32 duration)
> +{
> + if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval)))
> + return -EFAULT;
> + if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration)))
> + return -EFAULT;
> + return 0;
> +}
> +
> +static void *bpf_receive_mem(u64 in, u32 in_size, u32 max_size)
> +{
> + void __user *data_in = u64_to_user_ptr(in);
> + void *data;
> + int err;
> +
> + if (!data_in && in_size)
> + return ERR_PTR(-EINVAL);
> + data = kzalloc(max_size, GFP_USER);
> + if (!data)
> + return ERR_PTR(-ENOMEM);
> +
> + if (data_in) {
> + err = bpf_check_uarg_tail_zero(data_in, max_size, in_size);
> + if (err) {
> + kfree(data);
> + return ERR_PTR(err);
> + }
> +
> + in_size = min_t(u32, max_size, in_size);
> + if (copy_from_user(data, data_in, in_size)) {
> + kfree(data);
> + return ERR_PTR(-EFAULT);
> + }
> + }
> + return data;
> +}
> +
> +static int bpf_send_mem(u64 out, u32 out_size, u32 *out_size_write,
> + const void *data, u32 data_size)
> +{
> + void __user *data_out = u64_to_user_ptr(out);
> + int err = -EFAULT;
> + u32 copy_size = data_size;
> +
> + if (!data_out && out_size)
> + return -EINVAL;
> +
> + if (!data || !data_out)
> + return 0;
> +
> + if (copy_size > out_size) {
> + copy_size = out_size;
> + err = -ENOSPC;
> + }
> +
> + if (copy_to_user(data_out, data, copy_size))
> + goto out;
> + if (copy_to_user(out_size_write, &data_size, sizeof(data_size)))
> + goto out;
> + if (err != -ENOSPC)
> + err = 0;
> +out:
> + return err;
> +}
> +
> +void *bpf_receive_data(const union bpf_attr *kattr, u32 max_size)
> +{
> + return bpf_receive_mem(kattr->test.data_in, kattr->test.data_size_in,
> + max_size);
> +}
> +
> +int bpf_send_data(const union bpf_attr *kattr, union bpf_attr __user *uattr,
> + const void *data, u32 size)
> +{
> + return bpf_send_mem(kattr->test.data_out, kattr->test.data_size_out,
> + &uattr->test.data_size_out, data, size);
> +}
> +
> +void *bpf_receive_ctx(const union bpf_attr *kattr, u32 max_size)
> +{
> + return bpf_receive_mem(kattr->test.ctx_in, kattr->test.ctx_size_in,
> + max_size);
> +}
> +
> +int bpf_send_ctx(const union bpf_attr *kattr, union bpf_attr __user *uattr,
> + const void *data, u32 size)
> +{
> + return bpf_send_mem(kattr->test.ctx_out, kattr->test.ctx_size_out,
> + &uattr->test.ctx_size_out, data, size);
> +}
> diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c
> index 80e6f3a6864d..fe6b7b1af0cc 100644
> --- a/net/bpf/test_run.c
> +++ b/net/bpf/test_run.c
> @@ -14,97 +14,6 @@
> #define CREATE_TRACE_POINTS
> #include <trace/events/bpf_test_run.h>
>
> -static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat,
> - u32 *retval, u32 *time)
> -{
> - struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = { NULL };
> - enum bpf_cgroup_storage_type stype;
> - u64 time_start, time_spent = 0;
> - int ret = 0;
> - u32 i;
> -
> - for_each_cgroup_storage_type(stype) {
> - storage[stype] = bpf_cgroup_storage_alloc(prog, stype);
> - if (IS_ERR(storage[stype])) {
> - storage[stype] = NULL;
> - for_each_cgroup_storage_type(stype)
> - bpf_cgroup_storage_free(storage[stype]);
> - return -ENOMEM;
> - }
> - }
> -
> - if (!repeat)
> - repeat = 1;
> -
> - rcu_read_lock();
> - preempt_disable();
> - time_start = ktime_get_ns();
> - for (i = 0; i < repeat; i++) {
> - bpf_cgroup_storage_set(storage);
> - *retval = BPF_PROG_RUN(prog, ctx);
> -
> - if (signal_pending(current)) {
> - ret = -EINTR;
> - break;
> - }
> -
> - if (need_resched()) {
> - time_spent += ktime_get_ns() - time_start;
> - preempt_enable();
> - rcu_read_unlock();
> -
> - cond_resched();
> -
> - rcu_read_lock();
> - preempt_disable();
> - time_start = ktime_get_ns();
> - }
> - }
> - time_spent += ktime_get_ns() - time_start;
> - preempt_enable();
> - rcu_read_unlock();
> -
> - do_div(time_spent, repeat);
> - *time = time_spent > U32_MAX ? U32_MAX : (u32)time_spent;
> -
> - for_each_cgroup_storage_type(stype)
> - bpf_cgroup_storage_free(storage[stype]);
> -
> - return ret;
> -}
> -
> -static int bpf_test_finish(const union bpf_attr *kattr,
> - union bpf_attr __user *uattr, const void *data,
> - u32 size, u32 retval, u32 duration)
> -{
> - void __user *data_out = u64_to_user_ptr(kattr->test.data_out);
> - int err = -EFAULT;
> - u32 copy_size = size;
> -
> - /* Clamp copy if the user has provided a size hint, but copy the full
> - * buffer if not to retain old behaviour.
> - */
> - if (kattr->test.data_size_out &&
> - copy_size > kattr->test.data_size_out) {
> - copy_size = kattr->test.data_size_out;
> - err = -ENOSPC;
> - }
> -
> - if (data_out && copy_to_user(data_out, data, copy_size))
> - goto out;
> - if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size)))
> - goto out;
> - if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval)))
> - goto out;
> - if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration)))
> - goto out;
> - if (err != -ENOSPC)
> - err = 0;
> -out:
> - trace_bpf_test_finish(&err);
> - return err;
> -}
> -
> static void *bpf_test_init(const union bpf_attr *kattr, u32 size,
> u32 headroom, u32 tailroom)
> {
> @@ -125,63 +34,6 @@ static void *bpf_test_init(const union bpf_attr *kattr, u32 size,
> return data;
> }
>
> -static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size)
> -{
> - void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in);
> - void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out);
> - u32 size = kattr->test.ctx_size_in;
> - void *data;
> - int err;
> -
> - if (!data_in && !data_out)
> - return NULL;
> -
> - data = kzalloc(max_size, GFP_USER);
> - if (!data)
> - return ERR_PTR(-ENOMEM);
> -
> - if (data_in) {
> - err = bpf_check_uarg_tail_zero(data_in, max_size, size);
> - if (err) {
> - kfree(data);
> - return ERR_PTR(err);
> - }
> -
> - size = min_t(u32, max_size, size);
> - if (copy_from_user(data, data_in, size)) {
> - kfree(data);
> - return ERR_PTR(-EFAULT);
> - }
> - }
> - return data;
> -}
> -
> -static int bpf_ctx_finish(const union bpf_attr *kattr,
> - union bpf_attr __user *uattr, const void *data,
> - u32 size)
> -{
> - void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out);
> - int err = -EFAULT;
> - u32 copy_size = size;
> -
> - if (!data || !data_out)
> - return 0;
> -
> - if (copy_size > kattr->test.ctx_size_out) {
> - copy_size = kattr->test.ctx_size_out;
> - err = -ENOSPC;
> - }
> -
> - if (copy_to_user(data_out, data, copy_size))
> - goto out;
> - if (copy_to_user(&uattr->test.ctx_size_out, &size, sizeof(size)))
> - goto out;
> - if (err != -ENOSPC)
> - err = 0;
> -out:
> - return err;
> -}
> -
> /**
> * range_is_zero - test whether buffer is initialized
> * @buf: buffer to check
> @@ -238,6 +90,36 @@ static void convert_skb_to___skb(struct sk_buff *skb, struct __sk_buff *__skb)
> memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN);
> }
>
> +static int bpf_net_prog_test_run_finish(const union bpf_attr *kattr,
> + union bpf_attr __user *uattr,
> + const void *data, u32 data_size,
> + const void *ctx, u32 ctx_size,
> + u32 retval, u32 duration)
> +{
> + int ret;
> + union bpf_attr fixed_kattr;
> + const union bpf_attr *kattr_ptr = kattr;
> +
> + /* Clamp copy (in bpf_send_mem) if the user has provided a
> + * size hint, but copy the full buffer if not to retain old
> + * behaviour.
> + */
> + if (!kattr->test.data_size_out && kattr->test.data_out) {
> + fixed_kattr = *kattr;
> + fixed_kattr.test.data_size_out = U32_MAX;
> + kattr_ptr = &fixed_kattr;
> + }
> +
> + ret = bpf_send_data(kattr_ptr, uattr, data, data_size);
> + if (!ret) {
> + ret = bpf_test_finish(uattr, retval, duration);
> + if (!ret && ctx)
> + ret = bpf_send_ctx(kattr_ptr, uattr, ctx, ctx_size);
> + }
> + trace_bpf_test_finish(&ret);
> + return ret;
> +}
> +
> int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
> union bpf_attr __user *uattr)
> {
> @@ -257,7 +139,7 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
> if (IS_ERR(data))
> return PTR_ERR(data);
>
> - ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff));
> + ctx = bpf_receive_ctx(kattr, sizeof(struct __sk_buff));
> if (IS_ERR(ctx)) {
> kfree(data);
> return PTR_ERR(ctx);
> @@ -307,7 +189,8 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
> ret = convert___skb_to_skb(skb, ctx);
> if (ret)
> goto out;
> - ret = bpf_test_run(prog, skb, repeat, &retval, &duration);
> + ret = bpf_test_run(prog, skb, repeat, BPF_TEST_RUN_SETUP_CGROUP_STORAGE,
> + &retval, &duration);
> if (ret)
> goto out;
> if (!is_l2) {
> @@ -327,10 +210,9 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
> /* bpf program can never convert linear skb to non-linear */
> if (WARN_ON_ONCE(skb_is_nonlinear(skb)))
> size = skb_headlen(skb);
> - ret = bpf_test_finish(kattr, uattr, skb->data, size, retval, duration);
> - if (!ret)
> - ret = bpf_ctx_finish(kattr, uattr, ctx,
> - sizeof(struct __sk_buff));
> + ret = bpf_net_prog_test_run_finish(kattr, uattr, skb->data, size,
> + ctx, sizeof(struct __sk_buff),
> + retval, duration);
> out:
> kfree_skb(skb);
> bpf_sk_storage_free(sk);
> @@ -365,32 +247,48 @@ int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,
> rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0);
> xdp.rxq = &rxqueue->xdp_rxq;
>
> - ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration);
> + ret = bpf_test_run(prog, &xdp, repeat,
> + BPF_TEST_RUN_SETUP_CGROUP_STORAGE,
> + &retval, &duration);
> if (ret)
> goto out;
> if (xdp.data != data + XDP_PACKET_HEADROOM + NET_IP_ALIGN ||
> xdp.data_end != xdp.data + size)
> size = xdp.data_end - xdp.data;
> - ret = bpf_test_finish(kattr, uattr, xdp.data, size, retval, duration);
> + ret = bpf_net_prog_test_run_finish(kattr, uattr, xdp.data, size,
> + NULL, 0, retval, duration);
> out:
> kfree(data);
> return ret;
> }
>
> +struct bpf_flow_dissect_run_data {
> + __be16 proto;
> + int nhoff;
> + int hlen;
> +};
> +
> +static u32 bpf_flow_dissect_run(struct bpf_prog *prog, void *ctx,
> + void *private_data)
> +{
> + struct bpf_flow_dissect_run_data *data = private_data;
> +
> + return bpf_flow_dissect(prog, ctx, data->proto, data->nhoff, data->hlen);
> +}
> +
> int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
> const union bpf_attr *kattr,
> union bpf_attr __user *uattr)
> {
> + struct bpf_flow_dissect_run_data run_data = {};
> u32 size = kattr->test.data_size_in;
> struct bpf_flow_dissector ctx = {};
> u32 repeat = kattr->test.repeat;
> struct bpf_flow_keys flow_keys;
> - u64 time_start, time_spent = 0;
> const struct ethhdr *eth;
> u32 retval, duration;
> void *data;
> int ret;
> - u32 i;
>
> if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR)
> return -EINVAL;
> @@ -407,49 +305,22 @@ int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
>
> eth = (struct ethhdr *)data;
>
> - if (!repeat)
> - repeat = 1;
> -
> ctx.flow_keys = &flow_keys;
> ctx.data = data;
> ctx.data_end = (__u8 *)data + size;
>
> - rcu_read_lock();
> - preempt_disable();
> - time_start = ktime_get_ns();
> - for (i = 0; i < repeat; i++) {
> - retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN,
> - size);
> -
> - if (signal_pending(current)) {
> - preempt_enable();
> - rcu_read_unlock();
> -
> - ret = -EINTR;
> - goto out;
> - }
> -
> - if (need_resched()) {
> - time_spent += ktime_get_ns() - time_start;
> - preempt_enable();
> - rcu_read_unlock();
> -
> - cond_resched();
> -
> - rcu_read_lock();
> - preempt_disable();
> - time_start = ktime_get_ns();
> - }
> - }
> - time_spent += ktime_get_ns() - time_start;
> - preempt_enable();
> - rcu_read_unlock();
> -
> - do_div(time_spent, repeat);
> - duration = time_spent > U32_MAX ? U32_MAX : (u32)time_spent;
> + run_data.proto = eth->h_proto;
> + run_data.nhoff = ETH_HLEN;
> + run_data.hlen = size;
> + ret = bpf_test_run_cb(prog, &ctx, repeat, BPF_TEST_RUN_PLAIN,
> + bpf_flow_dissect_run, &run_data,
> + &retval, &duration);
> + if (!ret)
> + goto out;
>
> - ret = bpf_test_finish(kattr, uattr, &flow_keys, sizeof(flow_keys),
> - retval, duration);
> + ret = bpf_net_prog_test_run_finish(kattr, uattr, &flow_keys,
> + sizeof(flow_keys), NULL, 0,
> + retval, duration);
>
> out:
> kfree(data);
> --
> 2.20.1
>
--
Kinvolk GmbH | Adalbertstr.6a, 10999 Berlin | tel: +491755589364
Geschäftsführer/Directors: Alban Crequy, Chris Kühl, Iago López Galeiras
Registergericht/Court of registration: Amtsgericht Charlottenburg
Registernummer/Registration number: HRB 171414 B
Ust-ID-Nummer/VAT ID number: DE302207000
^ permalink raw reply
* Re: [PATCH 1/1] tools/dtrace: initial implementation of DTrace
From: Kris Van Hees @ 2019-07-08 16:38 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Kris Van Hees, netdev, bpf, dtrace-devel, linux-kernel, rostedt,
mhiramat, acme, ast, daniel, Chris Mason
In-Reply-To: <20190704130509.GO3402@hirez.programming.kicks-ass.net>
On Thu, Jul 04, 2019 at 03:05:09PM +0200, Peter Zijlstra wrote:
> On Wed, Jul 03, 2019 at 08:14:30PM -0700, Kris Van Hees wrote:
> > +int dt_bpf_attach(int event_id, int bpf_fd)
> > +{
> > + int event_fd;
> > + int rc;
> > + struct perf_event_attr attr = {};
> > +
> > + attr.type = PERF_TYPE_TRACEPOINT;
> > + attr.sample_type = PERF_SAMPLE_RAW;
> > + attr.sample_period = 1;
> > + attr.wakeup_events = 1;
> > + attr.config = event_id;
> > +
> > + /* Register the event (based on its id), and obtain a fd. */
> > + event_fd = perf_event_open(&attr, -1, 0, -1, 0);
> > + if (event_fd < 0) {
> > + perror("sys_perf_event_open");
> > + return -1;
> > + }
> > +
> > + /* Enable the probe. */
> > + rc = ioctl(event_fd, PERF_EVENT_IOC_ENABLE, 0);
>
> AFAICT you didn't use attr.disabled = 1, so this IOC_ENABLE is
> completely superfluous.
Oh yes, good point (and the same applies to the dt_buffer.c code where I set
up the events that own each buffer - no point in doing an explicit enable there
eiteher).
Thanks for catching this!
> > + if (rc < 0) {
> > + perror("PERF_EVENT_IOC_ENABLE");
> > + return -1;
> > + }
> > +
> > + /* Associate the BPF program with the event. */
> > + rc = ioctl(event_fd, PERF_EVENT_IOC_SET_BPF, bpf_fd);
> > + if (rc < 0) {
> > + perror("PERF_EVENT_IOC_SET_BPF");
> > + return -1;
> > + }
> > +
> > + return 0;
> > +}
^ permalink raw reply
* [bpf-next v3 00/12] Test the 32bit narrow reads
From: Krzesimir Nowak @ 2019-07-08 16:31 UTC (permalink / raw)
To: linux-kernel
Cc: Alban Crequy, Iago López Galeiras, Alexei Starovoitov,
Daniel Borkmann, Martin KaFai Lau, Song Liu, Yonghong Song,
David S. Miller, Jakub Kicinski, Jesper Dangaard Brouer,
John Fastabend, Stanislav Fomichev, netdev, bpf, xdp-newbies,
Krzesimir Nowak
These patches try to test the fix made in commit e2f7fc0ac695 ("bpf:
fix undefined behavior in narrow load handling"). The problem existed
in the generated BPF bytecode that was doing a 32bit narrow read of a
64bit field, so to test it the code would need to be executed.
Currently the only such field exists in BPF_PROG_TYPE_PERF_EVENT,
which was not supported by bpf_prog_test_run().
I'm sending these patches to bpf-next now as they introduce a new
feature. But maybe some of those patches could go to the bpf branch?
There is a bit of yak shaving to do for the test to be run:
1. Print why the program could not be run (patch 1).
2. Some fixes for errno clobbering (patches 2 and 3).
3. Using bpf_prog_test_run_xattr, so I can pass ctx_in stuff too
(patch 4).
4. Adding ctx stuff and data out size override to struct bpf_test, and
use them for the perf event tests (patches 5 and 6).
5. Some tools headers syncing (patches 7 and 8).
6. Split out some useful functions for bpf_prog_test_run
implementation out of the net/bpf/test_run.c (patch 9)
7. Implement bpf_prog_test_run for perf event programs and test it
(patches 10 and 11).
The last point is where I'm least sure how things should be done
properly:
1. There is a bunch of stuff to prepare for the
bpf_perf_prog_read_value to work, and that stuff is very hacky. I
would welcome some hints about how to set up the perf_event and
perf_sample_data structs in a way that is a bit more future-proof
than just setting some fields in a specific way, so some other code
won't use some other fields (like setting event.oncpu to -1 to
avoid event.pmu to be used for reading the value of the event).
2. The tests try to see if the test run for perf event sets up the
context properly, so they verify the struct pt_regs contents. They
way it is currently written Works For Me, but surely it won't work
on other arch. So what would be the way forward? Just put the test
case inside #ifdef __x86_64__?
3. Another thing in tests - I'm trying to make sure that the
bpf_perf_prog_read_value helper works as it seems to be the only
bpf perf helper that takes the ctx as a parameter. Is that enough
or I should test other helpers too?
About the test itself - I'm not sure if it will work on a big endian
machine. I think it should, but I don't have anything handy here to
verify it.
Krzesimir Nowak (12):
selftests/bpf: Print a message when tester could not run a program
selftests/bpf: Avoid a clobbering of errno
selftests/bpf: Avoid another case of errno clobbering
selftests/bpf: Use bpf_prog_test_run_xattr
selftests/bpf: Allow passing more information to BPF prog test run
selftests/bpf: Make sure that preexisting tests for perf event work
tools headers: Adopt compiletime_assert from kernel sources
tools headers: Sync struct bpf_perf_event_data
bpf: Split out some helper functions
bpf: Implement bpf_prog_test_run for perf event programs
selftests/bpf: Add tests for bpf_prog_test_run for perf events progs
selftests/bpf: Test correctness of narrow 32bit read on 64bit field
include/linux/bpf.h | 28 ++
kernel/bpf/Makefile | 1 +
kernel/bpf/test_run.c | 212 ++++++++++++++
kernel/trace/bpf_trace.c | 60 ++++
net/bpf/test_run.c | 263 +++++-------------
tools/include/linux/compiler.h | 28 ++
tools/include/uapi/linux/bpf_perf_event.h | 1 +
tools/testing/selftests/bpf/test_verifier.c | 197 ++++++++++++-
.../selftests/bpf/verifier/perf_event_run.c | 96 +++++++
.../bpf/verifier/perf_event_sample_period.c | 4 +
.../testing/selftests/bpf/verifier/var_off.c | 21 ++
11 files changed, 700 insertions(+), 211 deletions(-)
create mode 100644 kernel/bpf/test_run.c
create mode 100644 tools/testing/selftests/bpf/verifier/perf_event_run.c
--
2.20.1
^ permalink raw reply
* [bpf-next v3 02/12] selftests/bpf: Avoid a clobbering of errno
From: Krzesimir Nowak @ 2019-07-08 16:31 UTC (permalink / raw)
To: linux-kernel
Cc: Alban Crequy, Iago López Galeiras, Alexei Starovoitov,
Daniel Borkmann, Martin KaFai Lau, Song Liu, Yonghong Song,
David S. Miller, Jakub Kicinski, Jesper Dangaard Brouer,
John Fastabend, Stanislav Fomichev, netdev, bpf, xdp-newbies,
Krzesimir Nowak
In-Reply-To: <20190708163121.18477-1-krzesimir@kinvolk.io>
Save errno right after bpf_prog_test_run returns, so we later check
the error code actually set by bpf_prog_test_run, not by some libcap
function.
Changes since v1:
- Fix the "Fixes:" tag to mention actual commit that introduced the
bug
Changes since v2:
- Move the declaration so it fits the reverse christmas tree style.
Cc: Daniel Borkmann <daniel@iogearbox.net>
Fixes: 832c6f2c29ec ("bpf: test make sure to run unpriv test cases in test_verifier")
Signed-off-by: Krzesimir Nowak <krzesimir@kinvolk.io>
---
tools/testing/selftests/bpf/test_verifier.c | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/tools/testing/selftests/bpf/test_verifier.c b/tools/testing/selftests/bpf/test_verifier.c
index b8d065623ead..3fe126e0083b 100644
--- a/tools/testing/selftests/bpf/test_verifier.c
+++ b/tools/testing/selftests/bpf/test_verifier.c
@@ -823,16 +823,18 @@ static int do_prog_test_run(int fd_prog, bool unpriv, uint32_t expected_val,
__u8 tmp[TEST_DATA_LEN << 2];
__u32 size_tmp = sizeof(tmp);
uint32_t retval;
+ int saved_errno;
int err;
if (unpriv)
set_admin(true);
err = bpf_prog_test_run(fd_prog, 1, data, size_data,
tmp, &size_tmp, &retval, NULL);
+ saved_errno = errno;
if (unpriv)
set_admin(false);
if (err) {
- switch (errno) {
+ switch (saved_errno) {
case 524/*ENOTSUPP*/:
printf("Did not run the program (not supported) ");
return 0;
--
2.20.1
^ permalink raw reply related
* [bpf-next v3 01/12] selftests/bpf: Print a message when tester could not run a program
From: Krzesimir Nowak @ 2019-07-08 16:31 UTC (permalink / raw)
To: linux-kernel
Cc: Alban Crequy, Iago López Galeiras, Alexei Starovoitov,
Daniel Borkmann, Martin KaFai Lau, Song Liu, Yonghong Song,
David S. Miller, Jakub Kicinski, Jesper Dangaard Brouer,
John Fastabend, Stanislav Fomichev, netdev, bpf, xdp-newbies,
Krzesimir Nowak
In-Reply-To: <20190708163121.18477-1-krzesimir@kinvolk.io>
This prints a message when the error is about program type being not
supported by the test runner or because of permissions problem. This
is to see if the program we expected to run was actually executed.
The messages are open-coded because strerror(ENOTSUPP) returns
"Unknown error 524".
Changes since v2:
- Also print "FAIL" on an unexpected bpf_prog_test_run error, so there
is a corresponding "FAIL" message for each failed test.
Signed-off-by: Krzesimir Nowak <krzesimir@kinvolk.io>
---
tools/testing/selftests/bpf/test_verifier.c | 17 +++++++++++++----
1 file changed, 13 insertions(+), 4 deletions(-)
diff --git a/tools/testing/selftests/bpf/test_verifier.c b/tools/testing/selftests/bpf/test_verifier.c
index c5514daf8865..b8d065623ead 100644
--- a/tools/testing/selftests/bpf/test_verifier.c
+++ b/tools/testing/selftests/bpf/test_verifier.c
@@ -831,11 +831,20 @@ static int do_prog_test_run(int fd_prog, bool unpriv, uint32_t expected_val,
tmp, &size_tmp, &retval, NULL);
if (unpriv)
set_admin(false);
- if (err && errno != 524/*ENOTSUPP*/ && errno != EPERM) {
- printf("Unexpected bpf_prog_test_run error ");
- return err;
+ if (err) {
+ switch (errno) {
+ case 524/*ENOTSUPP*/:
+ printf("Did not run the program (not supported) ");
+ return 0;
+ case EPERM:
+ printf("Did not run the program (no permission) ");
+ return 0;
+ default:
+ printf("FAIL: Unexpected bpf_prog_test_run error (%s) ", strerror(saved_errno));
+ return err;
+ }
}
- if (!err && retval != expected_val &&
+ if (retval != expected_val &&
expected_val != POINTER_VALUE) {
printf("FAIL retval %d != %d ", retval, expected_val);
return 1;
--
2.20.1
^ permalink raw reply related
* [bpf-next v3 03/12] selftests/bpf: Avoid another case of errno clobbering
From: Krzesimir Nowak @ 2019-07-08 16:31 UTC (permalink / raw)
To: linux-kernel
Cc: Alban Crequy, Iago López Galeiras, Alexei Starovoitov,
Daniel Borkmann, Martin KaFai Lau, Song Liu, Yonghong Song,
David S. Miller, Jakub Kicinski, Jesper Dangaard Brouer,
John Fastabend, Stanislav Fomichev, netdev, bpf, xdp-newbies,
Krzesimir Nowak
In-Reply-To: <20190708163121.18477-1-krzesimir@kinvolk.io>
Commit 8184d44c9a57 ("selftests/bpf: skip verifier tests for
unsupported program types") added a check for an unsupported program
type. The function doing it changes errno, so test_verifier should
save it before calling it if test_verifier wants to print a reason why
verifying a BPF program of a supported type failed.
Changes since v2:
- Move the declaration to fit the reverse christmas tree style.
Fixes: 8184d44c9a57 ("selftests/bpf: skip verifier tests for unsupported program types")
Cc: Stanislav Fomichev <sdf@google.com>
Signed-off-by: Krzesimir Nowak <krzesimir@kinvolk.io>
---
tools/testing/selftests/bpf/test_verifier.c | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/tools/testing/selftests/bpf/test_verifier.c b/tools/testing/selftests/bpf/test_verifier.c
index 3fe126e0083b..c7541f572932 100644
--- a/tools/testing/selftests/bpf/test_verifier.c
+++ b/tools/testing/selftests/bpf/test_verifier.c
@@ -864,6 +864,7 @@ static void do_test_single(struct bpf_test *test, bool unpriv,
int run_errs, run_successes;
int map_fds[MAX_NR_MAPS];
const char *expected_err;
+ int saved_errno;
int fixup_skips;
__u32 pflags;
int i, err;
@@ -894,6 +895,7 @@ static void do_test_single(struct bpf_test *test, bool unpriv,
pflags |= BPF_F_ANY_ALIGNMENT;
fd_prog = bpf_verify_program(prog_type, prog, prog_len, pflags,
"GPL", 0, bpf_vlog, sizeof(bpf_vlog), 4);
+ saved_errno = errno;
if (fd_prog < 0 && !bpf_probe_prog_type(prog_type, 0)) {
printf("SKIP (unsupported program type %d)\n", prog_type);
skips++;
@@ -910,7 +912,7 @@ static void do_test_single(struct bpf_test *test, bool unpriv,
if (expected_ret == ACCEPT) {
if (fd_prog < 0) {
printf("FAIL\nFailed to load prog '%s'!\n",
- strerror(errno));
+ strerror(saved_errno));
goto fail_log;
}
#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
--
2.20.1
^ permalink raw reply related
* [bpf-next v3 04/12] selftests/bpf: Use bpf_prog_test_run_xattr
From: Krzesimir Nowak @ 2019-07-08 16:31 UTC (permalink / raw)
To: linux-kernel
Cc: Alban Crequy, Iago López Galeiras, Alexei Starovoitov,
Daniel Borkmann, Martin KaFai Lau, Song Liu, Yonghong Song,
David S. Miller, Jakub Kicinski, Jesper Dangaard Brouer,
John Fastabend, Stanislav Fomichev, netdev, bpf, xdp-newbies,
Krzesimir Nowak
In-Reply-To: <20190708163121.18477-1-krzesimir@kinvolk.io>
The bpf_prog_test_run_xattr function gives more options to set up a
test run of a BPF program than the bpf_prog_test_run function.
We will need this extra flexibility to pass ctx data later.
Signed-off-by: Krzesimir Nowak <krzesimir@kinvolk.io>
---
tools/testing/selftests/bpf/test_verifier.c | 16 +++++++++++-----
1 file changed, 11 insertions(+), 5 deletions(-)
diff --git a/tools/testing/selftests/bpf/test_verifier.c b/tools/testing/selftests/bpf/test_verifier.c
index c7541f572932..1640ba9f12c1 100644
--- a/tools/testing/selftests/bpf/test_verifier.c
+++ b/tools/testing/selftests/bpf/test_verifier.c
@@ -822,14 +822,20 @@ static int do_prog_test_run(int fd_prog, bool unpriv, uint32_t expected_val,
{
__u8 tmp[TEST_DATA_LEN << 2];
__u32 size_tmp = sizeof(tmp);
- uint32_t retval;
int saved_errno;
int err;
+ struct bpf_prog_test_run_attr attr = {
+ .prog_fd = fd_prog,
+ .repeat = 1,
+ .data_in = data,
+ .data_size_in = size_data,
+ .data_out = tmp,
+ .data_size_out = size_tmp,
+ };
if (unpriv)
set_admin(true);
- err = bpf_prog_test_run(fd_prog, 1, data, size_data,
- tmp, &size_tmp, &retval, NULL);
+ err = bpf_prog_test_run_xattr(&attr);
saved_errno = errno;
if (unpriv)
set_admin(false);
@@ -846,9 +852,9 @@ static int do_prog_test_run(int fd_prog, bool unpriv, uint32_t expected_val,
return err;
}
}
- if (retval != expected_val &&
+ if (attr.retval != expected_val &&
expected_val != POINTER_VALUE) {
- printf("FAIL retval %d != %d ", retval, expected_val);
+ printf("FAIL retval %d != %d ", attr.retval, expected_val);
return 1;
}
--
2.20.1
^ permalink raw reply related
* [bpf-next v3 06/12] selftests/bpf: Make sure that preexisting tests for perf event work
From: Krzesimir Nowak @ 2019-07-08 16:31 UTC (permalink / raw)
To: linux-kernel
Cc: Alban Crequy, Iago López Galeiras, Alexei Starovoitov,
Daniel Borkmann, Martin KaFai Lau, Song Liu, Yonghong Song,
David S. Miller, Jakub Kicinski, Jesper Dangaard Brouer,
John Fastabend, Stanislav Fomichev, netdev, bpf, xdp-newbies,
Krzesimir Nowak
In-Reply-To: <20190708163121.18477-1-krzesimir@kinvolk.io>
We are going to introduce a test run implementation for perf event in
a later commit and it will not allow passing any data out or ctx out
to it, and requires their sizes to be specified to zero. To avoid test
failures when the feature is introduced, override the data out size to
zero. That will also cause NULL buffer to be sent to the kernel.
Signed-off-by: Krzesimir Nowak <krzesimir@kinvolk.io>
---
.../testing/selftests/bpf/verifier/perf_event_sample_period.c | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/tools/testing/selftests/bpf/verifier/perf_event_sample_period.c b/tools/testing/selftests/bpf/verifier/perf_event_sample_period.c
index 471c1a5950d8..19f5d824b275 100644
--- a/tools/testing/selftests/bpf/verifier/perf_event_sample_period.c
+++ b/tools/testing/selftests/bpf/verifier/perf_event_sample_period.c
@@ -13,6 +13,7 @@
},
.result = ACCEPT,
.prog_type = BPF_PROG_TYPE_PERF_EVENT,
+ .override_data_out_len = true,
},
{
"check bpf_perf_event_data->sample_period half load permitted",
@@ -29,6 +30,7 @@
},
.result = ACCEPT,
.prog_type = BPF_PROG_TYPE_PERF_EVENT,
+ .override_data_out_len = true,
},
{
"check bpf_perf_event_data->sample_period word load permitted",
@@ -45,6 +47,7 @@
},
.result = ACCEPT,
.prog_type = BPF_PROG_TYPE_PERF_EVENT,
+ .override_data_out_len = true,
},
{
"check bpf_perf_event_data->sample_period dword load permitted",
@@ -56,4 +59,5 @@
},
.result = ACCEPT,
.prog_type = BPF_PROG_TYPE_PERF_EVENT,
+ .override_data_out_len = true,
},
--
2.20.1
^ permalink raw reply related
* [bpf-next v3 05/12] selftests/bpf: Allow passing more information to BPF prog test run
From: Krzesimir Nowak @ 2019-07-08 16:31 UTC (permalink / raw)
To: linux-kernel
Cc: Alban Crequy, Iago López Galeiras, Alexei Starovoitov,
Daniel Borkmann, Martin KaFai Lau, Song Liu, Yonghong Song,
David S. Miller, Jakub Kicinski, Jesper Dangaard Brouer,
John Fastabend, Stanislav Fomichev, netdev, bpf, xdp-newbies,
Krzesimir Nowak
In-Reply-To: <20190708163121.18477-1-krzesimir@kinvolk.io>
The test case can now specify a custom length of the data member,
context data and its length, which will be passed to
bpf_prog_test_run_xattr. For backward compatilibity, if the data
length is 0 (which is what will happen when the field is left
unspecified in the designated initializer of a struct), then the
length passed to the bpf_prog_test_run_xattr is TEST_DATA_LEN.
Also for backward compatilibity, if context data length is 0, NULL is
passed as a context to bpf_prog_test_run_xattr. This is to avoid
breaking other tests, where context data being NULL and context data
length being 0 is handled differently from the case where context data
is not NULL and context data length is 0.
Custom lengths still can't be greater than hardcoded 64 bytes for data
and 192 for context data.
192 for context data was picked to allow passing struct
bpf_perf_event_data as a context for perf event programs. The struct
is quite large, because it contains struct pt_regs.
Test runs for perf event programs will not allow the copying the data
back to data_out buffer, so they require data_out_size to be zero and
data_out to be NULL. Since test_verifier hardcodes it, make it
possible to override the size. Overriding the size to zero will cause
the buffer to be NULL.
Changes since v2:
- Allow overriding the data out size and buffer.
Signed-off-by: Krzesimir Nowak <krzesimir@kinvolk.io>
---
tools/testing/selftests/bpf/test_verifier.c | 105 +++++++++++++++++---
1 file changed, 93 insertions(+), 12 deletions(-)
diff --git a/tools/testing/selftests/bpf/test_verifier.c b/tools/testing/selftests/bpf/test_verifier.c
index 1640ba9f12c1..6f124cc4ee34 100644
--- a/tools/testing/selftests/bpf/test_verifier.c
+++ b/tools/testing/selftests/bpf/test_verifier.c
@@ -54,6 +54,7 @@
#define MAX_TEST_RUNS 8
#define POINTER_VALUE 0xcafe4all
#define TEST_DATA_LEN 64
+#define TEST_CTX_LEN 192
#define F_NEEDS_EFFICIENT_UNALIGNED_ACCESS (1 << 0)
#define F_LOAD_WITH_STRICT_ALIGNMENT (1 << 1)
@@ -96,7 +97,12 @@ struct bpf_test {
enum bpf_prog_type prog_type;
uint8_t flags;
__u8 data[TEST_DATA_LEN];
+ __u32 data_len;
+ __u8 ctx[TEST_CTX_LEN];
+ __u32 ctx_len;
void (*fill_helper)(struct bpf_test *self);
+ bool override_data_out_len;
+ __u32 overridden_data_out_len;
uint8_t runs;
struct {
uint32_t retval, retval_unpriv;
@@ -104,6 +110,9 @@ struct bpf_test {
__u8 data[TEST_DATA_LEN];
__u64 data64[TEST_DATA_LEN / 8];
};
+ __u32 data_len;
+ __u8 ctx[TEST_CTX_LEN];
+ __u32 ctx_len;
} retvals[MAX_TEST_RUNS];
};
@@ -818,21 +827,35 @@ static int set_admin(bool admin)
}
static int do_prog_test_run(int fd_prog, bool unpriv, uint32_t expected_val,
- void *data, size_t size_data)
+ void *data, size_t size_data, void *ctx,
+ size_t size_ctx, u32 *overridden_data_out_size)
{
- __u8 tmp[TEST_DATA_LEN << 2];
- __u32 size_tmp = sizeof(tmp);
- int saved_errno;
- int err;
struct bpf_prog_test_run_attr attr = {
.prog_fd = fd_prog,
.repeat = 1,
.data_in = data,
.data_size_in = size_data,
- .data_out = tmp,
- .data_size_out = size_tmp,
+ .ctx_in = ctx,
+ .ctx_size_in = size_ctx,
};
+ __u8 tmp[TEST_DATA_LEN << 2];
+ __u32 size_tmp = sizeof(tmp);
+ __u32 size_buf = size_tmp;
+ __u8 *buf = tmp;
+ int saved_errno;
+ int err;
+ if (overridden_data_out_size)
+ size_buf = *overridden_data_out_size;
+ if (size_buf > size_tmp) {
+ printf("FAIL: out data size (%d) greater than a buffer size (%d) ",
+ size_buf, size_tmp);
+ return -EINVAL;
+ }
+ if (!size_buf)
+ buf = NULL;
+ attr.data_size_out = size_buf;
+ attr.data_out = buf;
if (unpriv)
set_admin(true);
err = bpf_prog_test_run_xattr(&attr);
@@ -956,13 +979,45 @@ static void do_test_single(struct bpf_test *test, bool unpriv,
if (!alignment_prevented_execution && fd_prog >= 0) {
uint32_t expected_val;
int i;
+ __u32 size_data;
+ __u32 size_ctx;
+ bool bad_size;
+ void *ctx;
+ __u32 *overridden_data_out_size;
if (!test->runs) {
+ if (test->data_len > 0)
+ size_data = test->data_len;
+ else
+ size_data = sizeof(test->data);
+ if (test->override_data_out_len)
+ overridden_data_out_size = &test->overridden_data_out_len;
+ else
+ overridden_data_out_size = NULL;
+ size_ctx = test->ctx_len;
+ bad_size = false;
expected_val = unpriv && test->retval_unpriv ?
test->retval_unpriv : test->retval;
- err = do_prog_test_run(fd_prog, unpriv, expected_val,
- test->data, sizeof(test->data));
+ if (size_data > sizeof(test->data)) {
+ printf("FAIL: data size (%u) greater than TEST_DATA_LEN (%lu) ", size_data, sizeof(test->data));
+ bad_size = true;
+ }
+ if (size_ctx > sizeof(test->ctx)) {
+ printf("FAIL: ctx size (%u) greater than TEST_CTX_LEN (%lu) ", size_ctx, sizeof(test->ctx));
+ bad_size = true;
+ }
+ if (size_ctx)
+ ctx = test->ctx;
+ else
+ ctx = NULL;
+ if (bad_size)
+ err = 1;
+ else
+ err = do_prog_test_run(fd_prog, unpriv, expected_val,
+ test->data, size_data,
+ ctx, size_ctx,
+ overridden_data_out_size);
if (err)
run_errs++;
else
@@ -970,14 +1025,40 @@ static void do_test_single(struct bpf_test *test, bool unpriv,
}
for (i = 0; i < test->runs; i++) {
+ if (test->retvals[i].data_len > 0)
+ size_data = test->retvals[i].data_len;
+ else
+ size_data = sizeof(test->retvals[i].data);
+ if (test->override_data_out_len)
+ overridden_data_out_size = &test->overridden_data_out_len;
+ else
+ overridden_data_out_size = NULL;
+ size_ctx = test->retvals[i].ctx_len;
+ bad_size = false;
if (unpriv && test->retvals[i].retval_unpriv)
expected_val = test->retvals[i].retval_unpriv;
else
expected_val = test->retvals[i].retval;
- err = do_prog_test_run(fd_prog, unpriv, expected_val,
- test->retvals[i].data,
- sizeof(test->retvals[i].data));
+ if (size_data > sizeof(test->retvals[i].data)) {
+ printf("FAIL: data size (%u) at run %i greater than TEST_DATA_LEN (%lu) ", size_data, i + 1, sizeof(test->retvals[i].data));
+ bad_size = true;
+ }
+ if (size_ctx > sizeof(test->retvals[i].ctx)) {
+ printf("FAIL: ctx size (%u) at run %i greater than TEST_CTX_LEN (%lu) ", size_ctx, i + 1, sizeof(test->retvals[i].ctx));
+ bad_size = true;
+ }
+ if (size_ctx)
+ ctx = test->retvals[i].ctx;
+ else
+ ctx = NULL;
+ if (bad_size)
+ err = 1;
+ else
+ err = do_prog_test_run(fd_prog, unpriv, expected_val,
+ test->retvals[i].data, size_data,
+ ctx, size_ctx,
+ overridden_data_out_size);
if (err) {
printf("(run %d/%d) ", i + 1, test->runs);
run_errs++;
--
2.20.1
^ permalink raw reply related
* [bpf-next v3 07/12] tools headers: Adopt compiletime_assert from kernel sources
From: Krzesimir Nowak @ 2019-07-08 16:31 UTC (permalink / raw)
To: linux-kernel
Cc: Alban Crequy, Iago López Galeiras, Alexei Starovoitov,
Daniel Borkmann, Martin KaFai Lau, Song Liu, Yonghong Song,
David S. Miller, Jakub Kicinski, Jesper Dangaard Brouer,
John Fastabend, Stanislav Fomichev, netdev, bpf, xdp-newbies,
Krzesimir Nowak
In-Reply-To: <20190708163121.18477-1-krzesimir@kinvolk.io>
This will come in handy to verify that the hardcoded size of the
context data in bpf_test struct is high enough to hold some struct.
Signed-off-by: Krzesimir Nowak <krzesimir@kinvolk.io>
---
tools/include/linux/compiler.h | 28 ++++++++++++++++++++++++++++
1 file changed, 28 insertions(+)
diff --git a/tools/include/linux/compiler.h b/tools/include/linux/compiler.h
index 1827c2f973f9..b4e97751000a 100644
--- a/tools/include/linux/compiler.h
+++ b/tools/include/linux/compiler.h
@@ -172,4 +172,32 @@ static __always_inline void __write_once_size(volatile void *p, void *res, int s
# define __fallthrough
#endif
+
+#ifdef __OPTIMIZE__
+# define __compiletime_assert(condition, msg, prefix, suffix) \
+ do { \
+ extern void prefix ## suffix(void) __compiletime_error(msg); \
+ if (!(condition)) \
+ prefix ## suffix(); \
+ } while (0)
+#else
+# define __compiletime_assert(condition, msg, prefix, suffix) do { } while (0)
+#endif
+
+#define _compiletime_assert(condition, msg, prefix, suffix) \
+ __compiletime_assert(condition, msg, prefix, suffix)
+
+/**
+ * compiletime_assert - break build and emit msg if condition is false
+ * @condition: a compile-time constant condition to check
+ * @msg: a message to emit if condition is false
+ *
+ * In tradition of POSIX assert, this macro will break the build if the
+ * supplied condition is *false*, emitting the supplied error message if the
+ * compiler has support to do so.
+ */
+#define compiletime_assert(condition, msg) \
+ _compiletime_assert(condition, msg, __compiletime_assert_, __LINE__)
+
+
#endif /* _TOOLS_LINUX_COMPILER_H */
--
2.20.1
^ permalink raw reply related
* [bpf-next v3 08/12] tools headers: Sync struct bpf_perf_event_data
From: Krzesimir Nowak @ 2019-07-08 16:31 UTC (permalink / raw)
To: linux-kernel
Cc: Alban Crequy, Iago López Galeiras, Alexei Starovoitov,
Daniel Borkmann, Martin KaFai Lau, Song Liu, Yonghong Song,
David S. Miller, Jakub Kicinski, Jesper Dangaard Brouer,
John Fastabend, Stanislav Fomichev, netdev, bpf, xdp-newbies,
Krzesimir Nowak
In-Reply-To: <20190708163121.18477-1-krzesimir@kinvolk.io>
struct bpf_perf_event_data in kernel headers has the addr field, which
is missing in the tools version of the struct. This will be important
for the bpf prog test run implementation for perf events as it will
expect data to be an instance of struct bpf_perf_event_data, so the
size of the data needs to match sizeof(bpf_perf_event_data).
Signed-off-by: Krzesimir Nowak <krzesimir@kinvolk.io>
---
tools/include/uapi/linux/bpf_perf_event.h | 1 +
1 file changed, 1 insertion(+)
diff --git a/tools/include/uapi/linux/bpf_perf_event.h b/tools/include/uapi/linux/bpf_perf_event.h
index 8f95303f9d80..eb1b9d21250c 100644
--- a/tools/include/uapi/linux/bpf_perf_event.h
+++ b/tools/include/uapi/linux/bpf_perf_event.h
@@ -13,6 +13,7 @@
struct bpf_perf_event_data {
bpf_user_pt_regs_t regs;
__u64 sample_period;
+ __u64 addr;
};
#endif /* _UAPI__LINUX_BPF_PERF_EVENT_H__ */
--
2.20.1
^ permalink raw reply related
* [bpf-next v3 12/12] selftests/bpf: Test correctness of narrow 32bit read on 64bit field
From: Krzesimir Nowak @ 2019-07-08 16:31 UTC (permalink / raw)
To: linux-kernel
Cc: Alban Crequy, Iago López Galeiras, Alexei Starovoitov,
Daniel Borkmann, Martin KaFai Lau, Song Liu, Yonghong Song,
David S. Miller, Jakub Kicinski, Jesper Dangaard Brouer,
John Fastabend, Stanislav Fomichev, netdev, bpf, xdp-newbies,
Krzesimir Nowak
In-Reply-To: <20190708163121.18477-1-krzesimir@kinvolk.io>
Test the correctness of the 32bit narrow reads by reading both halves
of the 64 bit field and doing a binary or on them to see if we get the
original value.
It succeeds as it should, but with the commit e2f7fc0ac695 ("bpf: fix
undefined behavior in narrow load handling") reverted, the test fails
with a following message:
> $ sudo ./test_verifier
> ...
> #967/p 32bit loads of a 64bit field (both least and most significant words) FAIL retval -1985229329 != 0
> verification time 17 usec
> stack depth 0
> processed 8 insns (limit 1000000) max_states_per_insn 0 total_states 0 peak_states 0 mark_read 0
> ...
> Summary: 1519 PASSED, 0 SKIPPED, 1 FAILED
Signed-off-by: Krzesimir Nowak <krzesimir@kinvolk.io>
---
tools/testing/selftests/bpf/test_verifier.c | 19 +++++++++++++++++
.../testing/selftests/bpf/verifier/var_off.c | 21 +++++++++++++++++++
2 files changed, 40 insertions(+)
diff --git a/tools/testing/selftests/bpf/test_verifier.c b/tools/testing/selftests/bpf/test_verifier.c
index 484ea8842b06..2a20280a4a44 100644
--- a/tools/testing/selftests/bpf/test_verifier.c
+++ b/tools/testing/selftests/bpf/test_verifier.c
@@ -24,6 +24,7 @@
#include <sys/capability.h>
+#include <linux/compiler.h>
#include <linux/unistd.h>
#include <linux/filter.h>
#include <linux/bpf_perf_event.h>
@@ -343,6 +344,24 @@ static void bpf_fill_perf_event_test_run_check(struct bpf_test *self)
self->fill_insns = NULL;
}
+static void bpf_fill_32bit_loads(struct bpf_test *self)
+{
+ compiletime_assert(
+ sizeof(struct bpf_perf_event_data) <= TEST_CTX_LEN,
+ "buffer for ctx is too short to fit struct bpf_perf_event_data");
+ compiletime_assert(
+ sizeof(struct bpf_perf_event_value) <= TEST_DATA_LEN,
+ "buffer for data is too short to fit struct bpf_perf_event_value");
+
+ struct bpf_perf_event_data ctx = {
+ .sample_period = 0x0123456789abcdef,
+ };
+
+ memcpy(self->ctx, &ctx, sizeof(ctx));
+ free(self->fill_insns);
+ self->fill_insns = NULL;
+}
+
/* BPF_SK_LOOKUP contains 13 instructions, if you need to fix up maps */
#define BPF_SK_LOOKUP(func) \
/* struct bpf_sock_tuple tuple = {} */ \
diff --git a/tools/testing/selftests/bpf/verifier/var_off.c b/tools/testing/selftests/bpf/verifier/var_off.c
index 8504ac937809..3f8bee0a50ef 100644
--- a/tools/testing/selftests/bpf/verifier/var_off.c
+++ b/tools/testing/selftests/bpf/verifier/var_off.c
@@ -246,3 +246,24 @@
.result = ACCEPT,
.prog_type = BPF_PROG_TYPE_LWT_IN,
},
+{
+ "32bit loads of a 64bit field (both least and most significant words)",
+ .insns = {
+ BPF_LDX_MEM(BPF_W, BPF_REG_4, BPF_REG_1, offsetof(struct bpf_perf_event_data, sample_period)),
+ BPF_LDX_MEM(BPF_W, BPF_REG_5, BPF_REG_1, offsetof(struct bpf_perf_event_data, sample_period) + 4),
+ BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_1, offsetof(struct bpf_perf_event_data, sample_period)),
+ BPF_ALU64_IMM(BPF_LSH, BPF_REG_5, 32),
+ BPF_ALU64_REG(BPF_OR, BPF_REG_4, BPF_REG_5),
+ BPF_ALU64_REG(BPF_XOR, BPF_REG_4, BPF_REG_6),
+ BPF_MOV64_REG(BPF_REG_0, BPF_REG_4),
+ BPF_EXIT_INSN(),
+ },
+ .result = ACCEPT,
+ .prog_type = BPF_PROG_TYPE_PERF_EVENT,
+ .ctx = { 0, },
+ .ctx_len = sizeof(struct bpf_perf_event_data),
+ .data = { 0, },
+ .data_len = sizeof(struct bpf_perf_event_value),
+ .fill_helper = bpf_fill_32bit_loads,
+ .override_data_out_len = true,
+},
--
2.20.1
^ permalink raw reply related
* [bpf-next v3 11/12] selftests/bpf: Add tests for bpf_prog_test_run for perf events progs
From: Krzesimir Nowak @ 2019-07-08 16:31 UTC (permalink / raw)
To: linux-kernel
Cc: Alban Crequy, Iago López Galeiras, Alexei Starovoitov,
Daniel Borkmann, Martin KaFai Lau, Song Liu, Yonghong Song,
David S. Miller, Jakub Kicinski, Jesper Dangaard Brouer,
John Fastabend, Stanislav Fomichev, netdev, bpf, xdp-newbies,
Krzesimir Nowak
In-Reply-To: <20190708163121.18477-1-krzesimir@kinvolk.io>
The tests check if ctx and data are correctly prepared from ctx_in and
data_in, so accessing the ctx and using the bpf_perf_prog_read_value
work as expected.
Signed-off-by: Krzesimir Nowak <krzesimir@kinvolk.io>
---
tools/testing/selftests/bpf/test_verifier.c | 48 ++++++++++
.../selftests/bpf/verifier/perf_event_run.c | 96 +++++++++++++++++++
2 files changed, 144 insertions(+)
create mode 100644 tools/testing/selftests/bpf/verifier/perf_event_run.c
diff --git a/tools/testing/selftests/bpf/test_verifier.c b/tools/testing/selftests/bpf/test_verifier.c
index 6f124cc4ee34..484ea8842b06 100644
--- a/tools/testing/selftests/bpf/test_verifier.c
+++ b/tools/testing/selftests/bpf/test_verifier.c
@@ -295,6 +295,54 @@ static void bpf_fill_scale(struct bpf_test *self)
}
}
+static void bpf_fill_perf_event_test_run_check(struct bpf_test *self)
+{
+ compiletime_assert(
+ sizeof(struct bpf_perf_event_data) <= TEST_CTX_LEN,
+ "buffer for ctx is too short to fit struct bpf_perf_event_data");
+ compiletime_assert(
+ sizeof(struct bpf_perf_event_value) <= TEST_DATA_LEN,
+ "buffer for data is too short to fit struct bpf_perf_event_value");
+
+ struct bpf_perf_event_data ctx = {
+ .regs = (bpf_user_pt_regs_t) {
+ .r15 = 1,
+ .r14 = 2,
+ .r13 = 3,
+ .r12 = 4,
+ .rbp = 5,
+ .rbx = 6,
+ .r11 = 7,
+ .r10 = 8,
+ .r9 = 9,
+ .r8 = 10,
+ .rax = 11,
+ .rcx = 12,
+ .rdx = 13,
+ .rsi = 14,
+ .rdi = 15,
+ .orig_rax = 16,
+ .rip = 17,
+ .cs = 18,
+ .eflags = 19,
+ .rsp = 20,
+ .ss = 21,
+ },
+ .sample_period = 1,
+ .addr = 2,
+ };
+ struct bpf_perf_event_value data = {
+ .counter = 1,
+ .enabled = 2,
+ .running = 3,
+ };
+
+ memcpy(self->ctx, &ctx, sizeof(ctx));
+ memcpy(self->data, &data, sizeof(data));
+ free(self->fill_insns);
+ self->fill_insns = NULL;
+}
+
/* BPF_SK_LOOKUP contains 13 instructions, if you need to fix up maps */
#define BPF_SK_LOOKUP(func) \
/* struct bpf_sock_tuple tuple = {} */ \
diff --git a/tools/testing/selftests/bpf/verifier/perf_event_run.c b/tools/testing/selftests/bpf/verifier/perf_event_run.c
new file mode 100644
index 000000000000..3f877458a7f8
--- /dev/null
+++ b/tools/testing/selftests/bpf/verifier/perf_event_run.c
@@ -0,0 +1,96 @@
+#define PER_LOAD_AND_CHECK_PTREG(PT_REG_FIELD, VALUE) \
+ PER_LOAD_AND_CHECK_CTX(offsetof(bpf_user_pt_regs_t, PT_REG_FIELD), VALUE)
+#define PER_LOAD_AND_CHECK_EVENT(PED_FIELD, VALUE) \
+ PER_LOAD_AND_CHECK_CTX(offsetof(struct bpf_perf_event_data, PED_FIELD), VALUE)
+#define PER_LOAD_AND_CHECK_CTX(OFFSET, VALUE) \
+ PER_LOAD_AND_CHECK_64(BPF_REG_4, BPF_REG_1, OFFSET, VALUE)
+#define PER_LOAD_AND_CHECK_VALUE(PEV_FIELD, VALUE) \
+ PER_LOAD_AND_CHECK_64(BPF_REG_7, BPF_REG_6, offsetof(struct bpf_perf_event_value, PEV_FIELD), VALUE)
+#define PER_LOAD_AND_CHECK_64(DST, SRC, OFFSET, VALUE) \
+ BPF_LDX_MEM(BPF_DW, DST, SRC, OFFSET), \
+ BPF_JMP_IMM(BPF_JEQ, DST, VALUE, 2), \
+ BPF_MOV64_IMM(BPF_REG_0, VALUE), \
+ BPF_EXIT_INSN()
+
+{
+ "check if regs contain expected values",
+ .insns = {
+ PER_LOAD_AND_CHECK_PTREG(r15, 1),
+ PER_LOAD_AND_CHECK_PTREG(r14, 2),
+ PER_LOAD_AND_CHECK_PTREG(r13, 3),
+ PER_LOAD_AND_CHECK_PTREG(r12, 4),
+ PER_LOAD_AND_CHECK_PTREG(rbp, 5),
+ PER_LOAD_AND_CHECK_PTREG(rbx, 6),
+ PER_LOAD_AND_CHECK_PTREG(r11, 7),
+ PER_LOAD_AND_CHECK_PTREG(r10, 8),
+ PER_LOAD_AND_CHECK_PTREG(r9, 9),
+ PER_LOAD_AND_CHECK_PTREG(r8, 10),
+ PER_LOAD_AND_CHECK_PTREG(rax, 11),
+ PER_LOAD_AND_CHECK_PTREG(rcx, 12),
+ PER_LOAD_AND_CHECK_PTREG(rdx, 13),
+ PER_LOAD_AND_CHECK_PTREG(rsi, 14),
+ PER_LOAD_AND_CHECK_PTREG(rdi, 15),
+ PER_LOAD_AND_CHECK_PTREG(orig_rax, 16),
+ PER_LOAD_AND_CHECK_PTREG(rip, 17),
+ PER_LOAD_AND_CHECK_PTREG(cs, 18),
+ PER_LOAD_AND_CHECK_PTREG(eflags, 19),
+ PER_LOAD_AND_CHECK_PTREG(rsp, 20),
+ PER_LOAD_AND_CHECK_PTREG(ss, 21),
+ BPF_MOV64_IMM(BPF_REG_0, 0),
+ BPF_EXIT_INSN(),
+ },
+ .result = ACCEPT,
+ .prog_type = BPF_PROG_TYPE_PERF_EVENT,
+ .ctx_len = sizeof(struct bpf_perf_event_data),
+ .data_len = sizeof(struct bpf_perf_event_value),
+ .fill_helper = bpf_fill_perf_event_test_run_check,
+ .override_data_out_len = true,
+},
+{
+ "check if sample period and addr contain expected values",
+ .insns = {
+ PER_LOAD_AND_CHECK_EVENT(sample_period, 1),
+ PER_LOAD_AND_CHECK_EVENT(addr, 2),
+ BPF_MOV64_IMM(BPF_REG_0, 0),
+ BPF_EXIT_INSN(),
+ },
+ .result = ACCEPT,
+ .prog_type = BPF_PROG_TYPE_PERF_EVENT,
+ .ctx_len = sizeof(struct bpf_perf_event_data),
+ .data_len = sizeof(struct bpf_perf_event_value),
+ .fill_helper = bpf_fill_perf_event_test_run_check,
+ .override_data_out_len = true,
+},
+{
+ "check if bpf_perf_prog_read_value returns expected data",
+ .insns = {
+ // allocate space for a struct bpf_perf_event_value
+ BPF_MOV64_REG(BPF_REG_6, BPF_REG_10),
+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_6, -(int)sizeof(struct bpf_perf_event_value)),
+ // prepare parameters for bpf_perf_prog_read_value(ctx, struct bpf_perf_event_value*, u32)
+ // BPF_REG_1 already contains the context
+ BPF_MOV64_REG(BPF_REG_2, BPF_REG_6),
+ BPF_MOV64_IMM(BPF_REG_3, sizeof(struct bpf_perf_event_value)),
+ BPF_EMIT_CALL(BPF_FUNC_perf_prog_read_value),
+ // check the return value
+ BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 1),
+ BPF_EXIT_INSN(),
+ // check if the fields match the expected values
+ PER_LOAD_AND_CHECK_VALUE(counter, 1),
+ PER_LOAD_AND_CHECK_VALUE(enabled, 2),
+ PER_LOAD_AND_CHECK_VALUE(running, 3),
+ BPF_MOV64_IMM(BPF_REG_0, 0),
+ BPF_EXIT_INSN(),
+ },
+ .result = ACCEPT,
+ .prog_type = BPF_PROG_TYPE_PERF_EVENT,
+ .ctx_len = sizeof(struct bpf_perf_event_data),
+ .data_len = sizeof(struct bpf_perf_event_value),
+ .fill_helper = bpf_fill_perf_event_test_run_check,
+ .override_data_out_len = true,
+},
+#undef PER_LOAD_AND_CHECK_64
+#undef PER_LOAD_AND_CHECK_VALUE
+#undef PER_LOAD_AND_CHECK_CTX
+#undef PER_LOAD_AND_CHECK_EVENT
+#undef PER_LOAD_AND_CHECK_PTREG
--
2.20.1
^ permalink raw reply related
* [bpf-next v3 09/12] bpf: Split out some helper functions
From: Krzesimir Nowak @ 2019-07-08 16:31 UTC (permalink / raw)
To: linux-kernel
Cc: Alban Crequy, Iago López Galeiras, Alexei Starovoitov,
Daniel Borkmann, Martin KaFai Lau, Song Liu, Yonghong Song,
David S. Miller, Jakub Kicinski, Jesper Dangaard Brouer,
John Fastabend, Stanislav Fomichev, netdev, bpf, xdp-newbies,
Krzesimir Nowak
In-Reply-To: <20190708163121.18477-1-krzesimir@kinvolk.io>
The moved functions are generally useful for implementing
bpf_prog_test_run for other types of BPF programs - they don't have
any network-specific stuff in them, so I can use them in a test run
implementation for perf event BPF program too.
Signed-off-by: Krzesimir Nowak <krzesimir@kinvolk.io>
---
include/linux/bpf.h | 28 +++++
kernel/bpf/Makefile | 1 +
kernel/bpf/test_run.c | 212 ++++++++++++++++++++++++++++++++++
net/bpf/test_run.c | 263 +++++++++++-------------------------------
4 files changed, 308 insertions(+), 196 deletions(-)
create mode 100644 kernel/bpf/test_run.c
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 18f4cc2c6acd..28db8ba57bc3 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -1143,4 +1143,32 @@ static inline u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type,
}
#endif /* CONFIG_INET */
+/* Helper functions for bpf_prog_test_run implementations */
+typedef u32 bpf_prog_run_helper_t(struct bpf_prog *prog, void *ctx,
+ void *private_data);
+
+enum bpf_test_run_flags {
+ BPF_TEST_RUN_PLAIN = 0,
+ BPF_TEST_RUN_SETUP_CGROUP_STORAGE = 1 << 0,
+};
+
+int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat, u32 flags,
+ u32 *retval, u32 *duration);
+
+int bpf_test_run_cb(struct bpf_prog *prog, void *ctx, u32 repeat, u32 flags,
+ bpf_prog_run_helper_t run_prog, void *private_data,
+ u32 *retval, u32 *duration);
+
+int bpf_test_finish(union bpf_attr __user *uattr, u32 retval, u32 duration);
+
+void *bpf_receive_ctx(const union bpf_attr *kattr, u32 max_size);
+
+int bpf_send_ctx(const union bpf_attr *kattr, union bpf_attr __user *uattr,
+ const void *data, u32 size);
+
+void *bpf_receive_data(const union bpf_attr *kattr, u32 max_size);
+
+int bpf_send_data(const union bpf_attr *kattr, union bpf_attr __user *uattr,
+ const void *data, u32 size);
+
#endif /* _LINUX_BPF_H */
diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile
index 29d781061cd5..570fd40288f4 100644
--- a/kernel/bpf/Makefile
+++ b/kernel/bpf/Makefile
@@ -22,3 +22,4 @@ obj-$(CONFIG_CGROUP_BPF) += cgroup.o
ifeq ($(CONFIG_INET),y)
obj-$(CONFIG_BPF_SYSCALL) += reuseport_array.o
endif
+obj-$(CONFIG_BPF_SYSCALL) += test_run.o
diff --git a/kernel/bpf/test_run.c b/kernel/bpf/test_run.c
new file mode 100644
index 000000000000..0481373da8be
--- /dev/null
+++ b/kernel/bpf/test_run.c
@@ -0,0 +1,212 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (c) 2017 Facebook
+ * Copyright (c) 2019 Tigera, Inc
+ */
+
+#include <asm/div64.h>
+
+#include <linux/bpf-cgroup.h>
+#include <linux/bpf.h>
+#include <linux/err.h>
+#include <linux/errno.h>
+#include <linux/filter.h>
+#include <linux/gfp.h>
+#include <linux/kernel.h>
+#include <linux/limits.h>
+#include <linux/preempt.h>
+#include <linux/rcupdate.h>
+#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/slab.h>
+#include <linux/timekeeping.h>
+#include <linux/uaccess.h>
+
+static void teardown_cgroup_storage(struct bpf_cgroup_storage **storage)
+{
+ enum bpf_cgroup_storage_type stype;
+
+ if (!storage)
+ return;
+ for_each_cgroup_storage_type(stype)
+ bpf_cgroup_storage_free(storage[stype]);
+ kfree(storage);
+}
+
+static struct bpf_cgroup_storage **setup_cgroup_storage(struct bpf_prog *prog)
+{
+ enum bpf_cgroup_storage_type stype;
+ struct bpf_cgroup_storage **storage;
+ size_t size = MAX_BPF_CGROUP_STORAGE_TYPE;
+
+ size *= sizeof(struct bpf_cgroup_storage *);
+ storage = kzalloc(size, GFP_KERNEL);
+ for_each_cgroup_storage_type(stype) {
+ storage[stype] = bpf_cgroup_storage_alloc(prog, stype);
+ if (IS_ERR(storage[stype])) {
+ storage[stype] = NULL;
+ teardown_cgroup_storage(storage);
+ return ERR_PTR(-ENOMEM);
+ }
+ }
+ return storage;
+}
+
+static u32 run_bpf_prog(struct bpf_prog *prog, void *ctx, void *private_data)
+{
+ return BPF_PROG_RUN(prog, ctx);
+}
+
+int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat, u32 flags,
+ u32 *retval, u32 *duration)
+{
+ return bpf_test_run_cb(prog, ctx, repeat, flags, run_bpf_prog, NULL,
+ retval, duration);
+}
+
+int bpf_test_run_cb(struct bpf_prog *prog, void *ctx, u32 repeat, u32 flags,
+ bpf_prog_run_helper_t run_prog, void *private_data,
+ u32 *retval, u32 *duration)
+{
+ struct bpf_cgroup_storage **storage = NULL;
+ u64 time_start, time_spent = 0;
+ int ret = 0;
+ u32 i;
+
+ if (flags & BPF_TEST_RUN_SETUP_CGROUP_STORAGE) {
+ storage = setup_cgroup_storage(prog);
+ if (IS_ERR(storage))
+ return PTR_ERR(storage);
+ }
+
+ if (!repeat)
+ repeat = 1;
+
+ rcu_read_lock();
+ preempt_disable();
+ time_start = ktime_get_ns();
+ for (i = 0; i < repeat; i++) {
+ if (storage)
+ bpf_cgroup_storage_set(storage);
+ *retval = run_prog(prog, ctx, private_data);
+
+ if (signal_pending(current)) {
+ preempt_enable();
+ rcu_read_unlock();
+ teardown_cgroup_storage(storage);
+ return -EINTR;
+ }
+
+ if (need_resched()) {
+ time_spent += ktime_get_ns() - time_start;
+ preempt_enable();
+ rcu_read_unlock();
+
+ cond_resched();
+
+ rcu_read_lock();
+ preempt_disable();
+ time_start = ktime_get_ns();
+ }
+ }
+ time_spent += ktime_get_ns() - time_start;
+ preempt_enable();
+ rcu_read_unlock();
+
+ do_div(time_spent, repeat);
+ *duration = time_spent > U32_MAX ? U32_MAX : (u32)time_spent;
+
+ teardown_cgroup_storage(storage);
+
+ return ret;
+}
+
+int bpf_test_finish(union bpf_attr __user *uattr, u32 retval, u32 duration)
+{
+ if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval)))
+ return -EFAULT;
+ if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration)))
+ return -EFAULT;
+ return 0;
+}
+
+static void *bpf_receive_mem(u64 in, u32 in_size, u32 max_size)
+{
+ void __user *data_in = u64_to_user_ptr(in);
+ void *data;
+ int err;
+
+ if (!data_in && in_size)
+ return ERR_PTR(-EINVAL);
+ data = kzalloc(max_size, GFP_USER);
+ if (!data)
+ return ERR_PTR(-ENOMEM);
+
+ if (data_in) {
+ err = bpf_check_uarg_tail_zero(data_in, max_size, in_size);
+ if (err) {
+ kfree(data);
+ return ERR_PTR(err);
+ }
+
+ in_size = min_t(u32, max_size, in_size);
+ if (copy_from_user(data, data_in, in_size)) {
+ kfree(data);
+ return ERR_PTR(-EFAULT);
+ }
+ }
+ return data;
+}
+
+static int bpf_send_mem(u64 out, u32 out_size, u32 *out_size_write,
+ const void *data, u32 data_size)
+{
+ void __user *data_out = u64_to_user_ptr(out);
+ int err = -EFAULT;
+ u32 copy_size = data_size;
+
+ if (!data_out && out_size)
+ return -EINVAL;
+
+ if (!data || !data_out)
+ return 0;
+
+ if (copy_size > out_size) {
+ copy_size = out_size;
+ err = -ENOSPC;
+ }
+
+ if (copy_to_user(data_out, data, copy_size))
+ goto out;
+ if (copy_to_user(out_size_write, &data_size, sizeof(data_size)))
+ goto out;
+ if (err != -ENOSPC)
+ err = 0;
+out:
+ return err;
+}
+
+void *bpf_receive_data(const union bpf_attr *kattr, u32 max_size)
+{
+ return bpf_receive_mem(kattr->test.data_in, kattr->test.data_size_in,
+ max_size);
+}
+
+int bpf_send_data(const union bpf_attr *kattr, union bpf_attr __user *uattr,
+ const void *data, u32 size)
+{
+ return bpf_send_mem(kattr->test.data_out, kattr->test.data_size_out,
+ &uattr->test.data_size_out, data, size);
+}
+
+void *bpf_receive_ctx(const union bpf_attr *kattr, u32 max_size)
+{
+ return bpf_receive_mem(kattr->test.ctx_in, kattr->test.ctx_size_in,
+ max_size);
+}
+
+int bpf_send_ctx(const union bpf_attr *kattr, union bpf_attr __user *uattr,
+ const void *data, u32 size)
+{
+ return bpf_send_mem(kattr->test.ctx_out, kattr->test.ctx_size_out,
+ &uattr->test.ctx_size_out, data, size);
+}
diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c
index 80e6f3a6864d..fe6b7b1af0cc 100644
--- a/net/bpf/test_run.c
+++ b/net/bpf/test_run.c
@@ -14,97 +14,6 @@
#define CREATE_TRACE_POINTS
#include <trace/events/bpf_test_run.h>
-static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat,
- u32 *retval, u32 *time)
-{
- struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = { NULL };
- enum bpf_cgroup_storage_type stype;
- u64 time_start, time_spent = 0;
- int ret = 0;
- u32 i;
-
- for_each_cgroup_storage_type(stype) {
- storage[stype] = bpf_cgroup_storage_alloc(prog, stype);
- if (IS_ERR(storage[stype])) {
- storage[stype] = NULL;
- for_each_cgroup_storage_type(stype)
- bpf_cgroup_storage_free(storage[stype]);
- return -ENOMEM;
- }
- }
-
- if (!repeat)
- repeat = 1;
-
- rcu_read_lock();
- preempt_disable();
- time_start = ktime_get_ns();
- for (i = 0; i < repeat; i++) {
- bpf_cgroup_storage_set(storage);
- *retval = BPF_PROG_RUN(prog, ctx);
-
- if (signal_pending(current)) {
- ret = -EINTR;
- break;
- }
-
- if (need_resched()) {
- time_spent += ktime_get_ns() - time_start;
- preempt_enable();
- rcu_read_unlock();
-
- cond_resched();
-
- rcu_read_lock();
- preempt_disable();
- time_start = ktime_get_ns();
- }
- }
- time_spent += ktime_get_ns() - time_start;
- preempt_enable();
- rcu_read_unlock();
-
- do_div(time_spent, repeat);
- *time = time_spent > U32_MAX ? U32_MAX : (u32)time_spent;
-
- for_each_cgroup_storage_type(stype)
- bpf_cgroup_storage_free(storage[stype]);
-
- return ret;
-}
-
-static int bpf_test_finish(const union bpf_attr *kattr,
- union bpf_attr __user *uattr, const void *data,
- u32 size, u32 retval, u32 duration)
-{
- void __user *data_out = u64_to_user_ptr(kattr->test.data_out);
- int err = -EFAULT;
- u32 copy_size = size;
-
- /* Clamp copy if the user has provided a size hint, but copy the full
- * buffer if not to retain old behaviour.
- */
- if (kattr->test.data_size_out &&
- copy_size > kattr->test.data_size_out) {
- copy_size = kattr->test.data_size_out;
- err = -ENOSPC;
- }
-
- if (data_out && copy_to_user(data_out, data, copy_size))
- goto out;
- if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size)))
- goto out;
- if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval)))
- goto out;
- if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration)))
- goto out;
- if (err != -ENOSPC)
- err = 0;
-out:
- trace_bpf_test_finish(&err);
- return err;
-}
-
static void *bpf_test_init(const union bpf_attr *kattr, u32 size,
u32 headroom, u32 tailroom)
{
@@ -125,63 +34,6 @@ static void *bpf_test_init(const union bpf_attr *kattr, u32 size,
return data;
}
-static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size)
-{
- void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in);
- void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out);
- u32 size = kattr->test.ctx_size_in;
- void *data;
- int err;
-
- if (!data_in && !data_out)
- return NULL;
-
- data = kzalloc(max_size, GFP_USER);
- if (!data)
- return ERR_PTR(-ENOMEM);
-
- if (data_in) {
- err = bpf_check_uarg_tail_zero(data_in, max_size, size);
- if (err) {
- kfree(data);
- return ERR_PTR(err);
- }
-
- size = min_t(u32, max_size, size);
- if (copy_from_user(data, data_in, size)) {
- kfree(data);
- return ERR_PTR(-EFAULT);
- }
- }
- return data;
-}
-
-static int bpf_ctx_finish(const union bpf_attr *kattr,
- union bpf_attr __user *uattr, const void *data,
- u32 size)
-{
- void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out);
- int err = -EFAULT;
- u32 copy_size = size;
-
- if (!data || !data_out)
- return 0;
-
- if (copy_size > kattr->test.ctx_size_out) {
- copy_size = kattr->test.ctx_size_out;
- err = -ENOSPC;
- }
-
- if (copy_to_user(data_out, data, copy_size))
- goto out;
- if (copy_to_user(&uattr->test.ctx_size_out, &size, sizeof(size)))
- goto out;
- if (err != -ENOSPC)
- err = 0;
-out:
- return err;
-}
-
/**
* range_is_zero - test whether buffer is initialized
* @buf: buffer to check
@@ -238,6 +90,36 @@ static void convert_skb_to___skb(struct sk_buff *skb, struct __sk_buff *__skb)
memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN);
}
+static int bpf_net_prog_test_run_finish(const union bpf_attr *kattr,
+ union bpf_attr __user *uattr,
+ const void *data, u32 data_size,
+ const void *ctx, u32 ctx_size,
+ u32 retval, u32 duration)
+{
+ int ret;
+ union bpf_attr fixed_kattr;
+ const union bpf_attr *kattr_ptr = kattr;
+
+ /* Clamp copy (in bpf_send_mem) if the user has provided a
+ * size hint, but copy the full buffer if not to retain old
+ * behaviour.
+ */
+ if (!kattr->test.data_size_out && kattr->test.data_out) {
+ fixed_kattr = *kattr;
+ fixed_kattr.test.data_size_out = U32_MAX;
+ kattr_ptr = &fixed_kattr;
+ }
+
+ ret = bpf_send_data(kattr_ptr, uattr, data, data_size);
+ if (!ret) {
+ ret = bpf_test_finish(uattr, retval, duration);
+ if (!ret && ctx)
+ ret = bpf_send_ctx(kattr_ptr, uattr, ctx, ctx_size);
+ }
+ trace_bpf_test_finish(&ret);
+ return ret;
+}
+
int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
union bpf_attr __user *uattr)
{
@@ -257,7 +139,7 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
if (IS_ERR(data))
return PTR_ERR(data);
- ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff));
+ ctx = bpf_receive_ctx(kattr, sizeof(struct __sk_buff));
if (IS_ERR(ctx)) {
kfree(data);
return PTR_ERR(ctx);
@@ -307,7 +189,8 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
ret = convert___skb_to_skb(skb, ctx);
if (ret)
goto out;
- ret = bpf_test_run(prog, skb, repeat, &retval, &duration);
+ ret = bpf_test_run(prog, skb, repeat, BPF_TEST_RUN_SETUP_CGROUP_STORAGE,
+ &retval, &duration);
if (ret)
goto out;
if (!is_l2) {
@@ -327,10 +210,9 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
/* bpf program can never convert linear skb to non-linear */
if (WARN_ON_ONCE(skb_is_nonlinear(skb)))
size = skb_headlen(skb);
- ret = bpf_test_finish(kattr, uattr, skb->data, size, retval, duration);
- if (!ret)
- ret = bpf_ctx_finish(kattr, uattr, ctx,
- sizeof(struct __sk_buff));
+ ret = bpf_net_prog_test_run_finish(kattr, uattr, skb->data, size,
+ ctx, sizeof(struct __sk_buff),
+ retval, duration);
out:
kfree_skb(skb);
bpf_sk_storage_free(sk);
@@ -365,32 +247,48 @@ int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,
rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0);
xdp.rxq = &rxqueue->xdp_rxq;
- ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration);
+ ret = bpf_test_run(prog, &xdp, repeat,
+ BPF_TEST_RUN_SETUP_CGROUP_STORAGE,
+ &retval, &duration);
if (ret)
goto out;
if (xdp.data != data + XDP_PACKET_HEADROOM + NET_IP_ALIGN ||
xdp.data_end != xdp.data + size)
size = xdp.data_end - xdp.data;
- ret = bpf_test_finish(kattr, uattr, xdp.data, size, retval, duration);
+ ret = bpf_net_prog_test_run_finish(kattr, uattr, xdp.data, size,
+ NULL, 0, retval, duration);
out:
kfree(data);
return ret;
}
+struct bpf_flow_dissect_run_data {
+ __be16 proto;
+ int nhoff;
+ int hlen;
+};
+
+static u32 bpf_flow_dissect_run(struct bpf_prog *prog, void *ctx,
+ void *private_data)
+{
+ struct bpf_flow_dissect_run_data *data = private_data;
+
+ return bpf_flow_dissect(prog, ctx, data->proto, data->nhoff, data->hlen);
+}
+
int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
const union bpf_attr *kattr,
union bpf_attr __user *uattr)
{
+ struct bpf_flow_dissect_run_data run_data = {};
u32 size = kattr->test.data_size_in;
struct bpf_flow_dissector ctx = {};
u32 repeat = kattr->test.repeat;
struct bpf_flow_keys flow_keys;
- u64 time_start, time_spent = 0;
const struct ethhdr *eth;
u32 retval, duration;
void *data;
int ret;
- u32 i;
if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR)
return -EINVAL;
@@ -407,49 +305,22 @@ int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
eth = (struct ethhdr *)data;
- if (!repeat)
- repeat = 1;
-
ctx.flow_keys = &flow_keys;
ctx.data = data;
ctx.data_end = (__u8 *)data + size;
- rcu_read_lock();
- preempt_disable();
- time_start = ktime_get_ns();
- for (i = 0; i < repeat; i++) {
- retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN,
- size);
-
- if (signal_pending(current)) {
- preempt_enable();
- rcu_read_unlock();
-
- ret = -EINTR;
- goto out;
- }
-
- if (need_resched()) {
- time_spent += ktime_get_ns() - time_start;
- preempt_enable();
- rcu_read_unlock();
-
- cond_resched();
-
- rcu_read_lock();
- preempt_disable();
- time_start = ktime_get_ns();
- }
- }
- time_spent += ktime_get_ns() - time_start;
- preempt_enable();
- rcu_read_unlock();
-
- do_div(time_spent, repeat);
- duration = time_spent > U32_MAX ? U32_MAX : (u32)time_spent;
+ run_data.proto = eth->h_proto;
+ run_data.nhoff = ETH_HLEN;
+ run_data.hlen = size;
+ ret = bpf_test_run_cb(prog, &ctx, repeat, BPF_TEST_RUN_PLAIN,
+ bpf_flow_dissect_run, &run_data,
+ &retval, &duration);
+ if (!ret)
+ goto out;
- ret = bpf_test_finish(kattr, uattr, &flow_keys, sizeof(flow_keys),
- retval, duration);
+ ret = bpf_net_prog_test_run_finish(kattr, uattr, &flow_keys,
+ sizeof(flow_keys), NULL, 0,
+ retval, duration);
out:
kfree(data);
--
2.20.1
^ permalink raw reply related
* [bpf-next v3 10/12] bpf: Implement bpf_prog_test_run for perf event programs
From: Krzesimir Nowak @ 2019-07-08 16:31 UTC (permalink / raw)
To: linux-kernel
Cc: Alban Crequy, Iago López Galeiras, Alexei Starovoitov,
Daniel Borkmann, Martin KaFai Lau, Song Liu, Yonghong Song,
David S. Miller, Jakub Kicinski, Jesper Dangaard Brouer,
John Fastabend, Stanislav Fomichev, netdev, bpf, xdp-newbies,
Krzesimir Nowak
In-Reply-To: <20190708163121.18477-1-krzesimir@kinvolk.io>
As an input, test run for perf event program takes struct
bpf_perf_event_data as ctx_in and struct bpf_perf_event_value as
data_in. For an output, it basically ignores ctx_out and data_out.
The implementation sets an instance of struct bpf_perf_event_data_kern
in such a way that the BPF program reading data from context will
receive what we passed to the bpf prog test run in ctx_in. Also BPF
program can call bpf_perf_prog_read_value to receive what was passed
in data_in.
Changes since v2:
- drop the changes in perf event verifier test - they are not needed
anymore after reworked ctx size handling
Signed-off-by: Krzesimir Nowak <krzesimir@kinvolk.io>
---
kernel/trace/bpf_trace.c | 60 ++++++++++++++++++++++++++++++++++++++++
1 file changed, 60 insertions(+)
diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c
index ca1255d14576..b870fc2314d0 100644
--- a/kernel/trace/bpf_trace.c
+++ b/kernel/trace/bpf_trace.c
@@ -19,6 +19,8 @@
#include "trace_probe.h"
#include "trace.h"
+#include <trace/events/bpf_test_run.h>
+
#define bpf_event_rcu_dereference(p) \
rcu_dereference_protected(p, lockdep_is_held(&bpf_event_mutex))
@@ -1160,7 +1162,65 @@ const struct bpf_verifier_ops perf_event_verifier_ops = {
.convert_ctx_access = pe_prog_convert_ctx_access,
};
+static int pe_prog_test_run(struct bpf_prog *prog,
+ const union bpf_attr *kattr,
+ union bpf_attr __user *uattr)
+{
+ struct bpf_perf_event_data_kern real_ctx = {0, };
+ struct perf_sample_data sample_data = {0, };
+ struct bpf_perf_event_data *fake_ctx;
+ struct bpf_perf_event_value *value;
+ struct perf_event event = {0, };
+ u32 retval = 0, duration = 0;
+ int err;
+
+ if (kattr->test.data_size_out || kattr->test.data_out)
+ return -EINVAL;
+ if (kattr->test.ctx_size_out || kattr->test.ctx_out)
+ return -EINVAL;
+
+ fake_ctx = bpf_receive_ctx(kattr, sizeof(struct bpf_perf_event_data));
+ if (IS_ERR(fake_ctx))
+ return PTR_ERR(fake_ctx);
+
+ value = bpf_receive_data(kattr, sizeof(struct bpf_perf_event_value));
+ if (IS_ERR(value)) {
+ kfree(fake_ctx);
+ return PTR_ERR(value);
+ }
+
+ real_ctx.regs = &fake_ctx->regs;
+ real_ctx.data = &sample_data;
+ real_ctx.event = &event;
+ perf_sample_data_init(&sample_data, fake_ctx->addr,
+ fake_ctx->sample_period);
+ event.cpu = smp_processor_id();
+ event.oncpu = -1;
+ event.state = PERF_EVENT_STATE_OFF;
+ local64_set(&event.count, value->counter);
+ event.total_time_enabled = value->enabled;
+ event.total_time_running = value->running;
+ /* make self as a leader - it is used only for checking the
+ * state field
+ */
+ event.group_leader = &event;
+ err = bpf_test_run(prog, &real_ctx, kattr->test.repeat,
+ BPF_TEST_RUN_PLAIN, &retval, &duration);
+ if (err) {
+ kfree(value);
+ kfree(fake_ctx);
+ return err;
+ }
+
+ err = bpf_test_finish(uattr, retval, duration);
+ trace_bpf_test_finish(&err);
+ kfree(value);
+ kfree(fake_ctx);
+ return err;
+}
+
const struct bpf_prog_ops perf_event_prog_ops = {
+ .test_run = pe_prog_test_run,
};
static DEFINE_MUTEX(bpf_event_mutex);
--
2.20.1
^ permalink raw reply related
* [PATCH] sunrpc/cache: remove the exporting of cache_seq_next
From: Denis Efremov @ 2019-07-08 16:14 UTC (permalink / raw)
To: Trond Myklebust
Cc: Denis Efremov, J. Bruce Fields, Chuck Lever, Anna Schumaker,
David S. Miller, linux-nfs, netdev, linux-kernel
The function cache_seq_next is declared static and marked
EXPORT_SYMBOL_GPL, which is at best an odd combination. Because the
function is not used outside of the net/sunrpc/cache.c file it is
defined in, this commit removes the EXPORT_SYMBOL_GPL() marking.
Fixes: d48cf356a130 ("SUNRPC: Remove non-RCU protected lookup")
Signed-off-by: Denis Efremov <efremov@linux.com>
---
net/sunrpc/cache.c | 1 -
1 file changed, 1 deletion(-)
diff --git a/net/sunrpc/cache.c b/net/sunrpc/cache.c
index 66fbb9d2fba7..6f1528f271ee 100644
--- a/net/sunrpc/cache.c
+++ b/net/sunrpc/cache.c
@@ -1375,7 +1375,6 @@ static void *cache_seq_next(struct seq_file *m, void *p, loff_t *pos)
hlist_first_rcu(&cd->hash_table[hash])),
struct cache_head, cache_list);
}
-EXPORT_SYMBOL_GPL(cache_seq_next);
void *cache_seq_start_rcu(struct seq_file *m, loff_t *pos)
__acquires(RCU)
--
2.21.0
^ permalink raw reply related
* Re: [PATCH bpf-next] selftests/bpf: make verifier loop tests arch independent
From: Stanislav Fomichev @ 2019-07-08 16:13 UTC (permalink / raw)
To: Y Song
Cc: Stanislav Fomichev, netdev, bpf, David Miller, Alexei Starovoitov,
Daniel Borkmann, Ilya Leoshkevich
In-Reply-To: <CAH3MdRWePmAZNRfGNcBdjKAJ+D33=4Vgg1STYC3khNps8AmaHQ@mail.gmail.com>
On 07/03, Y Song wrote:
> On Wed, Jul 3, 2019 at 1:51 PM Stanislav Fomichev <sdf@google.com> wrote:
> >
> > Take the first x bytes of pt_regs for scalability tests, there is
> > no real reason we need x86 specific rax.
> >
> > Signed-off-by: Stanislav Fomichev <sdf@google.com>
> > ---
> > tools/testing/selftests/bpf/progs/loop1.c | 3 ++-
> > tools/testing/selftests/bpf/progs/loop2.c | 3 ++-
> > tools/testing/selftests/bpf/progs/loop3.c | 3 ++-
> > 3 files changed, 6 insertions(+), 3 deletions(-)
> >
> > diff --git a/tools/testing/selftests/bpf/progs/loop1.c b/tools/testing/selftests/bpf/progs/loop1.c
> > index dea395af9ea9..d530c61d2517 100644
> > --- a/tools/testing/selftests/bpf/progs/loop1.c
> > +++ b/tools/testing/selftests/bpf/progs/loop1.c
> > @@ -14,11 +14,12 @@ SEC("raw_tracepoint/kfree_skb")
> > int nested_loops(volatile struct pt_regs* ctx)
> > {
> > int i, j, sum = 0, m;
> > + volatile int *any_reg = (volatile int *)ctx;
> >
> > for (j = 0; j < 300; j++)
> > for (i = 0; i < j; i++) {
> > if (j & 1)
> > - m = ctx->rax;
> > + m = *any_reg;
>
> I agree. ctx->rax here is only to generate some operations, which
> cannot be optimized away by the compiler. dereferencing a volatile
> pointee may just serve that purpose.
>
> Comparing the byte code generated with ctx->rax and *any_reg, they are
> slightly different. Using *any_reg is slighly worse, but this should
> be still okay for the test.
>
> > else
> > m = j;
> > sum += i * m;
> > diff --git a/tools/testing/selftests/bpf/progs/loop2.c b/tools/testing/selftests/bpf/progs/loop2.c
> > index 0637bd8e8bcf..91bb89d901e3 100644
> > --- a/tools/testing/selftests/bpf/progs/loop2.c
> > +++ b/tools/testing/selftests/bpf/progs/loop2.c
> > @@ -14,9 +14,10 @@ SEC("raw_tracepoint/consume_skb")
> > int while_true(volatile struct pt_regs* ctx)
> > {
> > int i = 0;
> > + volatile int *any_reg = (volatile int *)ctx;
> >
> > while (true) {
> > - if (ctx->rax & 1)
> > + if (*any_reg & 1)
> > i += 3;
> > else
> > i += 7;
> > diff --git a/tools/testing/selftests/bpf/progs/loop3.c b/tools/testing/selftests/bpf/progs/loop3.c
> > index 30a0f6cba080..3a7f12d7186c 100644
> > --- a/tools/testing/selftests/bpf/progs/loop3.c
> > +++ b/tools/testing/selftests/bpf/progs/loop3.c
> > @@ -14,9 +14,10 @@ SEC("raw_tracepoint/consume_skb")
> > int while_true(volatile struct pt_regs* ctx)
> > {
> > __u64 i = 0, sum = 0;
> > + volatile __u64 *any_reg = (volatile __u64 *)ctx;
> > do {
> > i++;
> > - sum += ctx->rax;
> > + sum += *any_reg;
> > } while (i < 0x100000000ULL);
> > return sum;
> > }
> > --
> > 2.22.0.410.gd8fdbe21b5-goog
>
> Ilya Leoshkevich (iii@linux.ibm.com, cc'ed) has another patch set
> trying to solve this problem by introducing s360 arch register access
> macros. I guess for now that patch set is not needed any more?
Oh, I missed them. Do they fix the tests for other (non-s360) arches as
well? I was trying to fix the issue by not depending on any arch
specific stuff because the test really doesn't care :-)
^ permalink raw reply
* Re: [RFC PATCH net-next 0/6] tc-taprio offload for SJA1105 DSA
From: Vivien Didelot @ 2019-07-08 16:10 UTC (permalink / raw)
To: Vladimir Oltean
Cc: Andrew Lunn, Florian Fainelli, David S. Miller,
Vinicius Costa Gomes, vedang.patel, Richard Cochran, weifeng.voon,
jiri, m-karicheri2, Jose.Abreu, ilias.apalodimas, netdev
In-Reply-To: <CA+h21hoZ-ZgweMEDSBjANVhkVTNDONA+YkSz5y6TAJWByHHzDg@mail.gmail.com>
On Sun, 7 Jul 2019 23:28:24 +0300, Vladimir Oltean <olteanv@gmail.com> wrote:
> I thought the dsa_port_notify functions are just to be
> called from switchdev. I'm still not sure I fully understand, but I'll
> try to switch to that in v2 and see what happens.
dsa_port_notify is used to forward an event occuring on a given port, to
all switch chips of a fabric. For example when programming a VLAN on a port,
all ports interconnecting switches together must to programmed too. Even
though mainly switchdev objects or attributes are notified at the moment,
a DSA notification is not specific to switchdev.
Thanks,
Vivien
^ permalink raw reply
* Re: linux-next: manual merge of the mlx5-next tree with the rdma tree
From: Jason Gunthorpe @ 2019-07-08 16:09 UTC (permalink / raw)
To: Stephen Rothwell
Cc: Leon Romanovsky, Doug Ledford, Linux Next Mailing List,
Linux Kernel Mailing List, Yishai Hadas, David Miller, Networking
In-Reply-To: <20190708132837.5ccb36ed@canb.auug.org.au>
On Mon, Jul 08, 2019 at 01:28:37PM +1000, Stephen Rothwell wrote:
> Hi all,
>
> On Thu, 4 Jul 2019 12:47:38 +1000 Stephen Rothwell <sfr@canb.auug.org.au> wrote:
> >
> > Hi all,
> >
> > Today's linux-next merge of the mlx5-next tree got a conflict in:
> >
> > drivers/infiniband/hw/mlx5/cq.c
> >
> > between commit:
> >
> > e39afe3d6dbd ("RDMA: Convert CQ allocations to be under core responsibility")
> >
> > from the rdma tree and commit:
> >
> > 38164b771947 ("net/mlx5: mlx5_core_create_cq() enhancements")
> >
> > from the mlx5-next tree.
> >
> > I fixed it up (see below) and can carry the fix as necessary. This
> > is now fixed as far as linux-next is concerned, but any non trivial
> > conflicts should be mentioned to your upstream maintainer when your tree
> > is submitted for merging. You may also want to consider cooperating
> > with the maintainer of the conflicting tree to minimise any particularly
> > complex conflicts.
> >
>
> This is now a conflict between the net-next tree and the rdma tree.
You'll see the mlx5-next merge with rdma tomorrow that will take care
of this
Thanks,
Jason
^ permalink raw reply
* RE: [PATCH net-next v3 3/3] net: stmmac: Introducing support for Page Pool
From: Jose Abreu @ 2019-07-08 16:08 UTC (permalink / raw)
To: Ilias Apalodimas, Jose Abreu
Cc: linux-kernel@vger.kernel.org, netdev@vger.kernel.org,
linux-stm32@st-md-mailman.stormreply.com,
linux-arm-kernel@lists.infradead.org, Joao Pinto,
David S . Miller, Giuseppe Cavallaro, Alexandre Torgue,
Jesper Dangaard Brouer, Arnd Bergmann
In-Reply-To: <20190705152453.GA24683@apalos>
From: Ilias Apalodimas <ilias.apalodimas@linaro.org> | Date: Fri, Jul
05, 2019 at 16:24:53
> Well ideally we'd like to get the change in before the merge window ourselves,
> since we dont want to remove->re-add the same function in stable kernels. If
> that doesn't go in i am fine fixing it in the next merge window i guess, since
> it offers substantial speedups
I think the series is marked as "Changes Requested" in patchwork. What's
the status of this ?
---
Thanks,
Jose Miguel Abreu
^ permalink raw reply
* WARNING in mark_chain_precision
From: syzbot @ 2019-07-08 16:07 UTC (permalink / raw)
To: aaron.f.brown, ast, bpf, daniel, davem, hawk, intel-wired-lan,
jakub.kicinski, jeffrey.t.kirsher, john.fastabend, kafai,
linux-kernel, netdev, sasha.neftin, songliubraving,
syzkaller-bugs, xdp-newbies, yhs
Hello,
syzbot found the following crash on:
HEAD commit: a51df9f8 gve: fix -ENOMEM null check on a page allocation
git tree: net-next
console output: https://syzkaller.appspot.com/x/log.txt?x=17e64325a00000
kernel config: https://syzkaller.appspot.com/x/.config?x=6bb3e6e7997c14f9
dashboard link: https://syzkaller.appspot.com/bug?extid=f21251a7468cd46efc60
compiler: gcc (GCC) 9.0.0 20181231 (experimental)
syz repro: https://syzkaller.appspot.com/x/repro.syz?x=114f842da00000
C reproducer: https://syzkaller.appspot.com/x/repro.c?x=1630a5aba00000
The bug was bisected to:
commit 55fdbeaa2db8b271db767240fba24a60bd232528
Author: Sasha Neftin <sasha.neftin@intel.com>
Date: Mon Jan 7 14:40:17 2019 +0000
igc: Remove unused code
bisection log: https://syzkaller.appspot.com/x/bisect.txt?x=15c205b9a00000
final crash: https://syzkaller.appspot.com/x/report.txt?x=17c205b9a00000
console output: https://syzkaller.appspot.com/x/log.txt?x=13c205b9a00000
IMPORTANT: if you fix the bug, please add the following tag to the commit:
Reported-by: syzbot+f21251a7468cd46efc60@syzkaller.appspotmail.com
Fixes: 55fdbeaa2db8 ("igc: Remove unused code")
------------[ cut here ]------------
verifier backtracking bug
WARNING: CPU: 0 PID: 8846 at kernel/bpf/verifier.c:1755
mark_chain_precision+0x15c2/0x18e0 kernel/bpf/verifier.c:1755
Kernel panic - not syncing: panic_on_warn set ...
CPU: 0 PID: 8846 Comm: syz-executor835 Not tainted 5.2.0-rc6+ #56
Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS
Google 01/01/2011
Call Trace:
__dump_stack lib/dump_stack.c:77 [inline]
dump_stack+0x172/0x1f0 lib/dump_stack.c:113
panic+0x2cb/0x744 kernel/panic.c:219
__warn.cold+0x20/0x4d kernel/panic.c:576
report_bug+0x263/0x2b0 lib/bug.c:186
fixup_bug arch/x86/kernel/traps.c:179 [inline]
fixup_bug arch/x86/kernel/traps.c:174 [inline]
do_error_trap+0x11b/0x200 arch/x86/kernel/traps.c:272
do_invalid_op+0x37/0x50 arch/x86/kernel/traps.c:291
invalid_op+0x14/0x20 arch/x86/entry/entry_64.S:986
RIP: 0010:mark_chain_precision+0x15c2/0x18e0 kernel/bpf/verifier.c:1755
Code: e9 55 f2 ff ff 48 89 df e8 4b 0a 2c 00 e9 3a f3 ff ff e8 61 cb f2 ff
48 c7 c7 e0 43 91 87 c6 05 40 2b 1f 08 01 e8 1c 03 c5 ff <0f> 0b 41 be f2
ff ff ff e9 eb f7 ff ff e8 3c cb f2 ff 45 31 f6 e9
RSP: 0018:ffff8880a01ef378 EFLAGS: 00010282
RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000000000000
RDX: 0000000000000000 RSI: ffffffff815adb06 RDI: ffffed101403de61
RBP: ffff8880a01ef4d0 R08: ffff88808e26c400 R09: 0000000000000000
R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000000
R13: ffff8880a14e8440 R14: 0000000000000001 R15: dffffc0000000000
check_cond_jmp_op+0xcce/0x3c20 kernel/bpf/verifier.c:5793
do_check+0x61cf/0x8930 kernel/bpf/verifier.c:7684
bpf_check+0x6f99/0x9950 kernel/bpf/verifier.c:9195
bpf_prog_load+0xec8/0x1670 kernel/bpf/syscall.c:1690
__do_sys_bpf+0xa20/0x42c0 kernel/bpf/syscall.c:2830
__se_sys_bpf kernel/bpf/syscall.c:2789 [inline]
__x64_sys_bpf+0x73/0xb0 kernel/bpf/syscall.c:2789
do_syscall_64+0xfd/0x680 arch/x86/entry/common.c:301
entry_SYSCALL_64_after_hwframe+0x49/0xbe
RIP: 0033:0x440369
Code: 18 89 d0 c3 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 00 48 89 f8 48 89 f7
48 89 d6 48 89 ca 4d 89 c2 4d 89 c8 4c 8b 4c 24 08 0f 05 <48> 3d 01 f0 ff
ff 0f 83 fb 13 fc ff c3 66 2e 0f 1f 84 00 00 00 00
RSP: 002b:00007ffccb952af8 EFLAGS: 00000246 ORIG_RAX: 0000000000000141
RAX: ffffffffffffffda RBX: 00000000004002c8 RCX: 0000000000440369
RDX: 0000000000000048 RSI: 0000000020000200 RDI: 0000000000000005
RBP: 00000000006ca018 R08: 0000000000000000 R09: 0000000000000000
R10: 00000000ffffffff R11: 0000000000000246 R12: 0000000000401bf0
R13: 0000000000401c80 R14: 0000000000000000 R15: 0000000000000000
Kernel Offset: disabled
Rebooting in 86400 seconds..
---
This bug is generated by a bot. It may contain errors.
See https://goo.gl/tpsmEJ for more information about syzbot.
syzbot engineers can be reached at syzkaller@googlegroups.com.
syzbot will keep track of this bug report. See:
https://goo.gl/tpsmEJ#status for how to communicate with syzbot.
For information about bisection process see: https://goo.gl/tpsmEJ#bisection
syzbot can test patches for this bug, for details see:
https://goo.gl/tpsmEJ#testing-patches
^ permalink raw reply
* [PATCH net-next,v3 11/11] netfilter: nf_tables: add hardware offload support
From: Pablo Neira Ayuso @ 2019-07-08 16:06 UTC (permalink / raw)
To: netdev
Cc: davem, thomas.lendacky, f.fainelli, ariel.elior, michael.chan,
madalin.bucur, yisen.zhuang, salil.mehta, jeffrey.t.kirsher,
tariqt, saeedm, jiri, idosch, jakub.kicinski, peppe.cavallaro,
grygorii.strashko, andrew, vivien.didelot, alexandre.torgue,
joabreu, linux-net-drivers, ogerlitz, Manish.Chopra,
marcelo.leitner, mkubecek, venkatkumar.duvvuru, maxime.chevallier,
cphealy, netfilter-devel
In-Reply-To: <20190708160614.2226-1-pablo@netfilter.org>
This patch adds hardware offload support for nftables through the
existing netdev_ops->ndo_setup_tc() interface, the TC_SETUP_CLSFLOWER
classifier and the flow rule API. This hardware offload support is
available for the NFPROTO_NETDEV family and the ingress hook.
Each nftables expression has a new ->offload interface, that is used to
populate the flow rule object that is attached to the transaction
object.
There is a new per-table NFT_TABLE_F_HW flag, that is set on to offload
an entire table, including all of its chains.
This patch supports for basic metadata (layer 3 and 4 protocol numbers),
5-tuple payload matching and the accept/drop actions; this also includes
basechain hardware offload only.
Signed-off-by: Pablo Neira Ayuso <pablo@netfilter.org>
---
v3: no changes.
include/net/netfilter/nf_tables.h | 13 ++
include/net/netfilter/nf_tables_offload.h | 76 +++++++++
include/uapi/linux/netfilter/nf_tables.h | 2 +
net/netfilter/Makefile | 2 +-
net/netfilter/nf_tables_api.c | 22 ++-
net/netfilter/nf_tables_offload.c | 261 ++++++++++++++++++++++++++++++
net/netfilter/nft_cmp.c | 53 ++++++
net/netfilter/nft_immediate.c | 31 ++++
net/netfilter/nft_meta.c | 27 ++++
net/netfilter/nft_payload.c | 187 +++++++++++++++++++++
10 files changed, 671 insertions(+), 3 deletions(-)
create mode 100644 include/net/netfilter/nf_tables_offload.h
create mode 100644 net/netfilter/nf_tables_offload.c
diff --git a/include/net/netfilter/nf_tables.h b/include/net/netfilter/nf_tables.h
index 9e8493aad49d..4f8cb85d18ce 100644
--- a/include/net/netfilter/nf_tables.h
+++ b/include/net/netfilter/nf_tables.h
@@ -161,6 +161,7 @@ struct nft_ctx {
const struct nlattr * const *nla;
u32 portid;
u32 seq;
+ u16 flags;
u8 family;
u8 level;
bool report;
@@ -735,6 +736,9 @@ enum nft_trans_phase {
NFT_TRANS_RELEASE
};
+struct nft_flow_rule;
+struct nft_offload_ctx;
+
/**
* struct nft_expr_ops - nf_tables expression operations
*
@@ -777,6 +781,10 @@ struct nft_expr_ops {
const struct nft_data **data);
bool (*gc)(struct net *net,
const struct nft_expr *expr);
+ int (*offload)(struct nft_offload_ctx *ctx,
+ struct nft_flow_rule *flow,
+ const struct nft_expr *expr);
+ u32 offload_flags;
const struct nft_expr_type *type;
void *data;
};
@@ -942,6 +950,7 @@ struct nft_stats {
* @stats: per-cpu chain stats
* @chain: the chain
* @dev_name: device name that this base chain is attached to (if any)
+ * @cb_list: list of flow block callbacks (for hardware offload)
*/
struct nft_base_chain {
struct nf_hook_ops ops;
@@ -951,6 +960,7 @@ struct nft_base_chain {
struct nft_stats __percpu *stats;
struct nft_chain chain;
char dev_name[IFNAMSIZ];
+ struct list_head cb_list;
};
static inline struct nft_base_chain *nft_base_chain(const struct nft_chain *chain)
@@ -1322,11 +1332,14 @@ struct nft_trans {
struct nft_trans_rule {
struct nft_rule *rule;
+ struct nft_flow_rule *flow;
u32 rule_id;
};
#define nft_trans_rule(trans) \
(((struct nft_trans_rule *)trans->data)->rule)
+#define nft_trans_flow_rule(trans) \
+ (((struct nft_trans_rule *)trans->data)->flow)
#define nft_trans_rule_id(trans) \
(((struct nft_trans_rule *)trans->data)->rule_id)
diff --git a/include/net/netfilter/nf_tables_offload.h b/include/net/netfilter/nf_tables_offload.h
new file mode 100644
index 000000000000..3196663a10e3
--- /dev/null
+++ b/include/net/netfilter/nf_tables_offload.h
@@ -0,0 +1,76 @@
+#ifndef _NET_NF_TABLES_OFFLOAD_H
+#define _NET_NF_TABLES_OFFLOAD_H
+
+#include <net/flow_offload.h>
+#include <net/netfilter/nf_tables.h>
+
+struct nft_offload_reg {
+ u32 key;
+ u32 len;
+ u32 base_offset;
+ u32 offset;
+ struct nft_data mask;
+};
+
+enum nft_offload_dep_type {
+ NFT_OFFLOAD_DEP_UNSPEC = 0,
+ NFT_OFFLOAD_DEP_NETWORK,
+ NFT_OFFLOAD_DEP_TRANSPORT,
+};
+
+struct nft_offload_ctx {
+ struct {
+ enum nft_offload_dep_type type;
+ __be16 l3num;
+ u8 protonum;
+ } dep;
+ unsigned int num_actions;
+ struct nft_offload_reg regs[NFT_REG32_15 + 1];
+};
+
+void nft_offload_set_dependency(struct nft_offload_ctx *ctx,
+ enum nft_offload_dep_type type);
+void nft_offload_update_dependency(struct nft_offload_ctx *ctx,
+ const void *data, u32 len);
+
+struct nft_flow_key {
+ struct flow_dissector_key_basic basic;
+ union {
+ struct flow_dissector_key_ipv4_addrs ipv4;
+ struct flow_dissector_key_ipv6_addrs ipv6;
+ };
+ struct flow_dissector_key_ports tp;
+ struct flow_dissector_key_ip ip;
+ struct flow_dissector_key_vlan vlan;
+ struct flow_dissector_key_eth_addrs eth_addrs;
+} __aligned(BITS_PER_LONG / 8); /* Ensure that we can do comparisons as longs. */
+
+struct nft_flow_match {
+ struct flow_dissector dissector;
+ struct nft_flow_key key;
+ struct nft_flow_key mask;
+};
+
+struct nft_flow_rule {
+ __be16 proto;
+ struct nft_flow_match match;
+ struct flow_rule *rule;
+};
+
+#define NFT_OFFLOAD_F_ACTION (1 << 0)
+
+struct nft_rule;
+struct nft_flow_rule *nft_flow_rule_create(const struct nft_rule *rule);
+void nft_flow_rule_destroy(struct nft_flow_rule *flow);
+int nft_flow_rule_offload_commit(struct net *net);
+
+#define NFT_OFFLOAD_MATCH(__key, __base, __field, __len, __reg) \
+ (__reg)->base_offset = \
+ offsetof(struct nft_flow_key, __base); \
+ (__reg)->offset = \
+ offsetof(struct nft_flow_key, __base.__field); \
+ (__reg)->len = __len; \
+ (__reg)->key = __key; \
+ memset(&(__reg)->mask, 0xff, (__reg)->len);
+
+#endif
diff --git a/include/uapi/linux/netfilter/nf_tables.h b/include/uapi/linux/netfilter/nf_tables.h
index c6c8ec5c7c00..1d076140dc5d 100644
--- a/include/uapi/linux/netfilter/nf_tables.h
+++ b/include/uapi/linux/netfilter/nf_tables.h
@@ -158,9 +158,11 @@ enum nft_hook_attributes {
* enum nft_table_flags - nf_tables table flags
*
* @NFT_TABLE_F_DORMANT: this table is not active
+ * @NFT_TABLE_F_HW_OFFLOAD: enable hardware offload
*/
enum nft_table_flags {
NFT_TABLE_F_DORMANT = 0x1,
+ NFT_TABLE_F_HW_OFFLOAD = 0x2,
};
/**
diff --git a/net/netfilter/Makefile b/net/netfilter/Makefile
index 72cca6b48960..46cb1d34e750 100644
--- a/net/netfilter/Makefile
+++ b/net/netfilter/Makefile
@@ -78,7 +78,7 @@ nf_tables-objs := nf_tables_core.o nf_tables_api.o nft_chain_filter.o \
nf_tables_trace.o nft_immediate.o nft_cmp.o nft_range.o \
nft_bitwise.o nft_byteorder.o nft_payload.o nft_lookup.o \
nft_dynset.o nft_meta.o nft_rt.o nft_exthdr.o \
- nft_chain_route.o
+ nft_chain_route.o nf_tables_offload.o
nf_tables_set-objs := nf_tables_set_core.o \
nft_set_hash.o nft_set_bitmap.o nft_set_rbtree.o
diff --git a/net/netfilter/nf_tables_api.c b/net/netfilter/nf_tables_api.c
index cae5c46e2dd4..11843f8656e5 100644
--- a/net/netfilter/nf_tables_api.c
+++ b/net/netfilter/nf_tables_api.c
@@ -18,6 +18,7 @@
#include <net/netfilter/nf_flow_table.h>
#include <net/netfilter/nf_tables_core.h>
#include <net/netfilter/nf_tables.h>
+#include <net/netfilter/nf_tables_offload.h>
#include <net/net_namespace.h>
#include <net/sock.h>
@@ -97,6 +98,7 @@ static void nft_ctx_init(struct nft_ctx *ctx,
ctx->nla = nla;
ctx->portid = NETLINK_CB(skb).portid;
ctx->report = nlmsg_report(nlh);
+ ctx->flags = nlh->nlmsg_flags;
ctx->seq = nlh->nlmsg_seq;
}
@@ -896,7 +898,7 @@ static int nf_tables_newtable(struct net *net, struct sock *nlsk,
if (nla[NFTA_TABLE_FLAGS]) {
flags = ntohl(nla_get_be32(nla[NFTA_TABLE_FLAGS]));
- if (flags & ~NFT_TABLE_F_DORMANT)
+ if (flags & ~(NFT_TABLE_F_DORMANT | NFT_TABLE_F_HW_OFFLOAD))
return -EINVAL;
}
@@ -1659,6 +1661,7 @@ static int nf_tables_addchain(struct nft_ctx *ctx, u8 family, u8 genmask,
chain->flags |= NFT_BASE_CHAIN;
basechain->policy = NF_ACCEPT;
+ INIT_LIST_HEAD(&basechain->cb_list);
} else {
chain = kzalloc(sizeof(*chain), GFP_KERNEL);
if (chain == NULL)
@@ -2638,6 +2641,7 @@ static int nf_tables_newrule(struct net *net, struct sock *nlsk,
u8 genmask = nft_genmask_next(net);
struct nft_expr_info *info = NULL;
int family = nfmsg->nfgen_family;
+ struct nft_flow_rule *flow;
struct nft_table *table;
struct nft_chain *chain;
struct nft_rule *rule, *old_rule = NULL;
@@ -2784,7 +2788,8 @@ static int nf_tables_newrule(struct net *net, struct sock *nlsk,
list_add_tail_rcu(&rule->list, &old_rule->list);
} else {
- if (nft_trans_rule_add(&ctx, NFT_MSG_NEWRULE, rule) == NULL) {
+ trans = nft_trans_rule_add(&ctx, NFT_MSG_NEWRULE, rule);
+ if (!trans) {
err = -ENOMEM;
goto err2;
}
@@ -2807,6 +2812,14 @@ static int nf_tables_newrule(struct net *net, struct sock *nlsk,
if (net->nft.validate_state == NFT_VALIDATE_DO)
return nft_table_validate(net, table);
+ if (table->flags & NFT_TABLE_F_HW_OFFLOAD) {
+ flow = nft_flow_rule_create(rule);
+ if (IS_ERR(flow))
+ return PTR_ERR(flow);
+
+ nft_trans_flow_rule(trans) = flow;
+ }
+
return 0;
err2:
nf_tables_rule_release(&ctx, rule);
@@ -6604,6 +6617,7 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb)
struct nft_trans_elem *te;
struct nft_chain *chain;
struct nft_table *table;
+ int err;
if (list_empty(&net->nft.commit_list)) {
mutex_unlock(&net->nft.commit_mutex);
@@ -6614,6 +6628,10 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb)
if (nf_tables_validate(net) < 0)
return -EAGAIN;
+ err = nft_flow_rule_offload_commit(net);
+ if (err < 0)
+ return err;
+
/* 1. Allocate space for next generation rules_gen_X[] */
list_for_each_entry_safe(trans, next, &net->nft.commit_list, list) {
int ret;
diff --git a/net/netfilter/nf_tables_offload.c b/net/netfilter/nf_tables_offload.c
new file mode 100644
index 000000000000..e17b67d22940
--- /dev/null
+++ b/net/netfilter/nf_tables_offload.c
@@ -0,0 +1,261 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/netfilter.h>
+#include <net/flow_offload.h>
+#include <net/netfilter/nf_tables.h>
+#include <net/netfilter/nf_tables_offload.h>
+#include <net/pkt_cls.h>
+
+static struct nft_flow_rule *nft_flow_rule_alloc(int num_actions)
+{
+ struct nft_flow_rule *flow;
+
+ flow = kzalloc(sizeof(struct nft_flow_rule), GFP_KERNEL);
+ if (!flow)
+ return NULL;
+
+ flow->rule = flow_rule_alloc(num_actions);
+ if (!flow->rule) {
+ kfree(flow);
+ return NULL;
+ }
+
+ flow->rule->match.dissector = &flow->match.dissector;
+ flow->rule->match.mask = &flow->match.mask;
+ flow->rule->match.key = &flow->match.key;
+
+ return flow;
+}
+
+struct nft_flow_rule *nft_flow_rule_create(const struct nft_rule *rule)
+{
+ struct nft_offload_ctx ctx = {
+ .dep = {
+ .type = NFT_OFFLOAD_DEP_UNSPEC,
+ },
+ };
+ struct nft_flow_rule *flow;
+ int num_actions = 0, err;
+ struct nft_expr *expr;
+
+ expr = nft_expr_first(rule);
+ while (expr->ops && expr != nft_expr_last(rule)) {
+ if (expr->ops->offload_flags & NFT_OFFLOAD_F_ACTION)
+ num_actions++;
+
+ expr = nft_expr_next(expr);
+ }
+
+ flow = nft_flow_rule_alloc(num_actions);
+ if (!flow)
+ return ERR_PTR(-ENOMEM);
+
+ expr = nft_expr_first(rule);
+ while (expr->ops && expr != nft_expr_last(rule)) {
+ if (!expr->ops->offload) {
+ err = -EOPNOTSUPP;
+ goto err_out;
+ }
+ err = expr->ops->offload(&ctx, flow, expr);
+ if (err < 0)
+ goto err_out;
+
+ expr = nft_expr_next(expr);
+ }
+ flow->proto = ctx.dep.l3num;
+
+ return flow;
+err_out:
+ nft_flow_rule_destroy(flow);
+
+ return ERR_PTR(err);
+}
+
+void nft_flow_rule_destroy(struct nft_flow_rule *flow)
+{
+ kfree(flow->rule);
+ kfree(flow);
+}
+
+void nft_offload_set_dependency(struct nft_offload_ctx *ctx,
+ enum nft_offload_dep_type type)
+{
+ ctx->dep.type = type;
+}
+
+void nft_offload_update_dependency(struct nft_offload_ctx *ctx,
+ const void *data, u32 len)
+{
+ switch (ctx->dep.type) {
+ case NFT_OFFLOAD_DEP_NETWORK:
+ WARN_ON(len != sizeof(__u16));
+ memcpy(&ctx->dep.l3num, data, sizeof(__u16));
+ break;
+ case NFT_OFFLOAD_DEP_TRANSPORT:
+ WARN_ON(len != sizeof(__u8));
+ memcpy(&ctx->dep.protonum, data, sizeof(__u8));
+ break;
+ default:
+ break;
+ }
+ ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;
+}
+
+static void nft_flow_offload_common_init(struct tc_cls_common_offload *common,
+ __be16 proto,
+ struct netlink_ext_ack *extack)
+{
+ common->protocol = proto;
+ common->extack = extack;
+}
+
+static int nft_setup_cb_call(struct nft_base_chain *basechain,
+ enum tc_setup_type type, void *type_data)
+{
+ struct flow_block_cb *block_cb;
+ int err;
+
+ list_for_each_entry(block_cb, &basechain->cb_list, list) {
+ err = block_cb->cb(type, type_data, block_cb->cb_priv);
+ if (err < 0)
+ return err;
+ }
+ return 0;
+}
+
+static int nft_flow_offload_rule(struct nft_trans *trans,
+ enum tc_fl_command command)
+{
+ struct nft_flow_rule *flow = nft_trans_flow_rule(trans);
+ struct nft_rule *rule = nft_trans_rule(trans);
+ struct tc_cls_flower_offload cls_flower = {};
+ struct nft_base_chain *basechain;
+ struct netlink_ext_ack extack;
+ __be16 proto = ETH_P_ALL;
+
+ if (!nft_is_base_chain(trans->ctx.chain))
+ return -EOPNOTSUPP;
+
+ basechain = nft_base_chain(trans->ctx.chain);
+
+ if (flow)
+ proto = flow->proto;
+
+ nft_flow_offload_common_init(&cls_flower.common, proto, &extack);
+ cls_flower.command = command;
+ cls_flower.cookie = (unsigned long) rule;
+ if (flow)
+ cls_flower.rule = flow->rule;
+
+ return nft_setup_cb_call(basechain, TC_SETUP_CLSFLOWER, &cls_flower);
+}
+
+static int nft_flow_offload_bind(struct flow_block_offload *bo,
+ struct nft_base_chain *basechain)
+{
+ struct flow_block_cb *block_cb;
+
+ list_for_each_entry(block_cb, &bo->cb_list, driver_list)
+ list_add(&block_cb->list, &basechain->cb_list);
+
+ list_splice(&bo->cb_list, bo->driver_block_list);
+ return 0;
+}
+
+static int nft_flow_offload_unbind(struct flow_block_offload *bo,
+ struct nft_base_chain *basechain)
+{
+ struct flow_block_cb *block_cb, *next;
+
+ list_for_each_entry_safe(block_cb, next, &bo->cb_list, driver_list) {
+ list_del(&block_cb->list);
+ flow_block_cb_free(block_cb);
+ }
+
+ return 0;
+}
+
+static int nft_flow_offload_chain(struct nft_trans *trans,
+ enum flow_block_command cmd)
+{
+ struct nft_chain *chain = trans->ctx.chain;
+ struct netlink_ext_ack extack = {};
+ struct flow_block_offload bo = {};
+ struct nft_base_chain *basechain;
+ struct net_device *dev;
+ int err;
+
+ if (!nft_is_base_chain(chain))
+ return -EOPNOTSUPP;
+
+ basechain = nft_base_chain(chain);
+ dev = basechain->ops.dev;
+ if (!dev || !dev->netdev_ops->ndo_setup_tc)
+ return -EOPNOTSUPP;
+
+ /* Only default policy to accept is supported for now. */
+ if (cmd == FLOW_BLOCK_BIND &&
+ nft_trans_chain_policy(trans) != -1 &&
+ nft_trans_chain_policy(trans) != NF_ACCEPT)
+ return -EOPNOTSUPP;
+
+ bo.command = cmd;
+ bo.binder_type = FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS;
+ bo.extack = &extack;
+ INIT_LIST_HEAD(&bo.cb_list);
+
+ err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo);
+ if (err < 0)
+ return err;
+
+ switch (cmd) {
+ case FLOW_BLOCK_BIND:
+ err = nft_flow_offload_bind(&bo, basechain);
+ break;
+ case FLOW_BLOCK_UNBIND:
+ err = nft_flow_offload_unbind(&bo, basechain);
+ break;
+ }
+
+ return err;
+}
+
+int nft_flow_rule_offload_commit(struct net *net)
+{
+ struct nft_trans *trans;
+ int err = 0;
+
+ list_for_each_entry(trans, &net->nft.commit_list, list) {
+ if (trans->ctx.family != NFPROTO_NETDEV ||
+ !(trans->ctx.table->flags & NFT_TABLE_F_HW_OFFLOAD))
+ continue;
+
+ switch (trans->msg_type) {
+ case NFT_MSG_NEWCHAIN:
+ err = nft_flow_offload_chain(trans, FLOW_BLOCK_BIND);
+ break;
+ case NFT_MSG_DELCHAIN:
+ err = nft_flow_offload_chain(trans, FLOW_BLOCK_UNBIND);
+ break;
+ case NFT_MSG_NEWRULE:
+ if (trans->ctx.flags & NLM_F_REPLACE ||
+ !(trans->ctx.flags & NLM_F_APPEND))
+ return -EOPNOTSUPP;
+
+ err = nft_flow_offload_rule(trans,
+ TC_CLSFLOWER_REPLACE);
+ nft_flow_rule_destroy(nft_trans_flow_rule(trans));
+ break;
+ case NFT_MSG_DELRULE:
+ err = nft_flow_offload_rule(trans,
+ TC_CLSFLOWER_DESTROY);
+ break;
+ }
+
+ if (err)
+ return err;
+ }
+
+ return err;
+}
diff --git a/net/netfilter/nft_cmp.c b/net/netfilter/nft_cmp.c
index 411c0cf741e3..bd173b1824c6 100644
--- a/net/netfilter/nft_cmp.c
+++ b/net/netfilter/nft_cmp.c
@@ -12,6 +12,7 @@
#include <linux/netfilter.h>
#include <linux/netfilter/nf_tables.h>
#include <net/netfilter/nf_tables_core.h>
+#include <net/netfilter/nf_tables_offload.h>
#include <net/netfilter/nf_tables.h>
struct nft_cmp_expr {
@@ -107,12 +108,44 @@ static int nft_cmp_dump(struct sk_buff *skb, const struct nft_expr *expr)
return -1;
}
+static int __nft_cmp_offload(struct nft_offload_ctx *ctx,
+ struct nft_flow_rule *flow,
+ const struct nft_cmp_expr *priv)
+{
+ struct nft_offload_reg *reg = &ctx->regs[priv->sreg];
+ u8 *mask = (u8 *)&flow->match.mask;
+ u8 *key = (u8 *)&flow->match.key;
+
+ if (priv->op != NFT_CMP_EQ)
+ return -EOPNOTSUPP;
+
+ memcpy(key + reg->offset, &priv->data, priv->len);
+ memcpy(mask + reg->offset, ®->mask, priv->len);
+
+ flow->match.dissector.used_keys |= BIT(reg->key);
+ flow->match.dissector.offset[reg->key] = reg->base_offset;
+
+ nft_offload_update_dependency(ctx, &priv->data, priv->len);
+
+ return 0;
+}
+
+static int nft_cmp_offload(struct nft_offload_ctx *ctx,
+ struct nft_flow_rule *flow,
+ const struct nft_expr *expr)
+{
+ const struct nft_cmp_expr *priv = nft_expr_priv(expr);
+
+ return __nft_cmp_offload(ctx, flow, priv);
+}
+
static const struct nft_expr_ops nft_cmp_ops = {
.type = &nft_cmp_type,
.size = NFT_EXPR_SIZE(sizeof(struct nft_cmp_expr)),
.eval = nft_cmp_eval,
.init = nft_cmp_init,
.dump = nft_cmp_dump,
+ .offload = nft_cmp_offload,
};
static int nft_cmp_fast_init(const struct nft_ctx *ctx,
@@ -143,6 +176,25 @@ static int nft_cmp_fast_init(const struct nft_ctx *ctx,
return 0;
}
+static int nft_cmp_fast_offload(struct nft_offload_ctx *ctx,
+ struct nft_flow_rule *flow,
+ const struct nft_expr *expr)
+{
+ const struct nft_cmp_fast_expr *priv = nft_expr_priv(expr);
+ struct nft_cmp_expr cmp = {
+ .data = {
+ .data = {
+ [0] = priv->data,
+ },
+ },
+ .sreg = priv->sreg,
+ .len = priv->len / BITS_PER_BYTE,
+ .op = NFT_CMP_EQ,
+ };
+
+ return __nft_cmp_offload(ctx, flow, &cmp);
+}
+
static int nft_cmp_fast_dump(struct sk_buff *skb, const struct nft_expr *expr)
{
const struct nft_cmp_fast_expr *priv = nft_expr_priv(expr);
@@ -169,6 +221,7 @@ const struct nft_expr_ops nft_cmp_fast_ops = {
.eval = NULL, /* inlined */
.init = nft_cmp_fast_init,
.dump = nft_cmp_fast_dump,
+ .offload = nft_cmp_fast_offload,
};
static const struct nft_expr_ops *
diff --git a/net/netfilter/nft_immediate.c b/net/netfilter/nft_immediate.c
index cb8547f97220..ca2ae4b95a8d 100644
--- a/net/netfilter/nft_immediate.c
+++ b/net/netfilter/nft_immediate.c
@@ -13,6 +13,7 @@
#include <linux/netfilter/nf_tables.h>
#include <net/netfilter/nf_tables_core.h>
#include <net/netfilter/nf_tables.h>
+#include <net/netfilter/nf_tables_offload.h>
void nft_immediate_eval(const struct nft_expr *expr,
struct nft_regs *regs,
@@ -124,6 +125,34 @@ static int nft_immediate_validate(const struct nft_ctx *ctx,
return 0;
}
+static int nft_immediate_offload(struct nft_offload_ctx *ctx,
+ struct nft_flow_rule *flow,
+ const struct nft_expr *expr)
+{
+ const struct nft_immediate_expr *priv = nft_expr_priv(expr);
+ struct flow_action_entry *entry;
+ const struct nft_data *data;
+
+ if (priv->dreg != NFT_REG_VERDICT)
+ return -EOPNOTSUPP;
+
+ entry = &flow->rule->action.entries[ctx->num_actions++];
+
+ data = &priv->data;
+ switch (data->verdict.code) {
+ case NF_ACCEPT:
+ entry->id = FLOW_ACTION_ACCEPT;
+ break;
+ case NF_DROP:
+ entry->id = FLOW_ACTION_DROP;
+ break;
+ default:
+ return -EOPNOTSUPP;
+ }
+
+ return 0;
+}
+
static const struct nft_expr_ops nft_imm_ops = {
.type = &nft_imm_type,
.size = NFT_EXPR_SIZE(sizeof(struct nft_immediate_expr)),
@@ -133,6 +162,8 @@ static const struct nft_expr_ops nft_imm_ops = {
.deactivate = nft_immediate_deactivate,
.dump = nft_immediate_dump,
.validate = nft_immediate_validate,
+ .offload = nft_immediate_offload,
+ .offload_flags = NFT_OFFLOAD_F_ACTION,
};
struct nft_expr_type nft_imm_type __read_mostly = {
diff --git a/net/netfilter/nft_meta.c b/net/netfilter/nft_meta.c
index a54329b8634a..e92f365aca35 100644
--- a/net/netfilter/nft_meta.c
+++ b/net/netfilter/nft_meta.c
@@ -21,6 +21,7 @@
#include <net/tcp_states.h> /* for TCP_TIME_WAIT */
#include <net/netfilter/nf_tables.h>
#include <net/netfilter/nf_tables_core.h>
+#include <net/netfilter/nf_tables_offload.h>
#include <uapi/linux/netfilter_bridge.h> /* NF_BR_PRE_ROUTING */
@@ -515,6 +516,31 @@ static void nft_meta_set_destroy(const struct nft_ctx *ctx,
static_branch_dec(&nft_trace_enabled);
}
+static int nft_meta_get_offload(struct nft_offload_ctx *ctx,
+ struct nft_flow_rule *flow,
+ const struct nft_expr *expr)
+{
+ const struct nft_meta *priv = nft_expr_priv(expr);
+ struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
+
+ switch (priv->key) {
+ case NFT_META_PROTOCOL:
+ NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_BASIC, basic, n_proto,
+ sizeof(__u16), reg);
+ nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_NETWORK);
+ break;
+ case NFT_META_L4PROTO:
+ NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_BASIC, basic, ip_proto,
+ sizeof(__u8), reg);
+ nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_TRANSPORT);
+ break;
+ default:
+ return -EOPNOTSUPP;
+ }
+
+ return 0;
+}
+
static const struct nft_expr_ops nft_meta_get_ops = {
.type = &nft_meta_type,
.size = NFT_EXPR_SIZE(sizeof(struct nft_meta)),
@@ -522,6 +548,7 @@ static const struct nft_expr_ops nft_meta_get_ops = {
.init = nft_meta_get_init,
.dump = nft_meta_get_dump,
.validate = nft_meta_get_validate,
+ .offload = nft_meta_get_offload,
};
static const struct nft_expr_ops nft_meta_set_ops = {
diff --git a/net/netfilter/nft_payload.c b/net/netfilter/nft_payload.c
index 1260f78a034d..22a80eb60222 100644
--- a/net/netfilter/nft_payload.c
+++ b/net/netfilter/nft_payload.c
@@ -15,10 +15,13 @@
#include <linux/netfilter/nf_tables.h>
#include <net/netfilter/nf_tables_core.h>
#include <net/netfilter/nf_tables.h>
+#include <net/netfilter/nf_tables_offload.h>
/* For layer 4 checksum field offset. */
#include <linux/tcp.h>
#include <linux/udp.h>
#include <linux/icmpv6.h>
+#include <linux/ip.h>
+#include <linux/ipv6.h>
/* add vlan header into the user buffer for if tag was removed by offloads */
static bool
@@ -150,12 +153,195 @@ static int nft_payload_dump(struct sk_buff *skb, const struct nft_expr *expr)
return -1;
}
+static int nft_payload_offload_ll(struct nft_offload_ctx *ctx,
+ struct nft_flow_rule *flow,
+ const struct nft_payload *priv)
+{
+ struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
+
+ switch (priv->offset) {
+ case offsetof(struct ethhdr, h_source):
+ NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_ETH_ADDRS, eth_addrs,
+ src, ETH_ALEN, reg);
+ break;
+ case offsetof(struct ethhdr, h_dest):
+ NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_ETH_ADDRS, eth_addrs,
+ dst, ETH_ALEN, reg);
+ break;
+ }
+
+ return 0;
+}
+
+static int nft_payload_offload_ip(struct nft_offload_ctx *ctx,
+ struct nft_flow_rule *flow,
+ const struct nft_payload *priv)
+{
+ struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
+
+ switch (priv->offset) {
+ case offsetof(struct iphdr, saddr):
+ NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4, src,
+ sizeof(struct in_addr), reg);
+ break;
+ case offsetof(struct iphdr, daddr):
+ NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4, dst,
+ sizeof(struct in_addr), reg);
+ break;
+ case offsetof(struct iphdr, protocol):
+ NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_BASIC, basic, ip_proto,
+ sizeof(__u8), reg);
+ nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_TRANSPORT);
+ break;
+ default:
+ return -EOPNOTSUPP;
+ }
+
+ return 0;
+}
+
+static int nft_payload_offload_ip6(struct nft_offload_ctx *ctx,
+ struct nft_flow_rule *flow,
+ const struct nft_payload *priv)
+{
+ struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
+
+ switch (priv->offset) {
+ case offsetof(struct ipv6hdr, saddr):
+ NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6, src,
+ sizeof(struct in6_addr), reg);
+ break;
+ case offsetof(struct ipv6hdr, daddr):
+ NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6, dst,
+ sizeof(struct in6_addr), reg);
+ break;
+ case offsetof(struct ipv6hdr, nexthdr):
+ NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_BASIC, basic, ip_proto,
+ sizeof(__u8), reg);
+ nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_TRANSPORT);
+ break;
+ default:
+ return -EOPNOTSUPP;
+ }
+
+ return 0;
+}
+
+static int nft_payload_offload_nh(struct nft_offload_ctx *ctx,
+ struct nft_flow_rule *flow,
+ const struct nft_payload *priv)
+{
+ int err;
+
+ switch (ctx->dep.l3num) {
+ case htons(ETH_P_IP):
+ err = nft_payload_offload_ip(ctx, flow, priv);
+ break;
+ case htons(ETH_P_IPV6):
+ err = nft_payload_offload_ip6(ctx, flow, priv);
+ break;
+ default:
+ return -EOPNOTSUPP;
+ }
+
+ return err;
+}
+
+static int nft_payload_offload_tcp(struct nft_offload_ctx *ctx,
+ struct nft_flow_rule *flow,
+ const struct nft_payload *priv)
+{
+ struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
+
+ switch (priv->offset) {
+ case offsetof(struct tcphdr, source):
+ NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, src,
+ sizeof(__be16), reg);
+ break;
+ case offsetof(struct tcphdr, dest):
+ NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, dst,
+ sizeof(__be16), reg);
+ break;
+ default:
+ return -EOPNOTSUPP;
+ }
+
+ return 0;
+}
+
+static int nft_payload_offload_udp(struct nft_offload_ctx *ctx,
+ struct nft_flow_rule *flow,
+ const struct nft_payload *priv)
+{
+ struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
+
+ switch (priv->offset) {
+ case offsetof(struct udphdr, source):
+ NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, src,
+ sizeof(__be16), reg);
+ break;
+ case offsetof(struct udphdr, dest):
+ NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, dst,
+ sizeof(__be16), reg);
+ break;
+ default:
+ return -EOPNOTSUPP;
+ }
+
+ return 0;
+}
+
+static int nft_payload_offload_th(struct nft_offload_ctx *ctx,
+ struct nft_flow_rule *flow,
+ const struct nft_payload *priv)
+{
+ int err;
+
+ switch (ctx->dep.protonum) {
+ case IPPROTO_TCP:
+ err = nft_payload_offload_tcp(ctx, flow, priv);
+ break;
+ case IPPROTO_UDP:
+ err = nft_payload_offload_udp(ctx, flow, priv);
+ break;
+ default:
+ return -EOPNOTSUPP;
+ }
+
+ return err;
+}
+
+static int nft_payload_offload(struct nft_offload_ctx *ctx,
+ struct nft_flow_rule *flow,
+ const struct nft_expr *expr)
+{
+ const struct nft_payload *priv = nft_expr_priv(expr);
+ int err;
+
+ switch (priv->base) {
+ case NFT_PAYLOAD_LL_HEADER:
+ err = nft_payload_offload_ll(ctx, flow, priv);
+ break;
+ case NFT_PAYLOAD_NETWORK_HEADER:
+ err = nft_payload_offload_nh(ctx, flow, priv);
+ break;
+ case NFT_PAYLOAD_TRANSPORT_HEADER:
+ err = nft_payload_offload_th(ctx, flow, priv);
+ break;
+ default:
+ err = -EOPNOTSUPP;
+ break;
+ }
+ return err;
+}
+
static const struct nft_expr_ops nft_payload_ops = {
.type = &nft_payload_type,
.size = NFT_EXPR_SIZE(sizeof(struct nft_payload)),
.eval = nft_payload_eval,
.init = nft_payload_init,
.dump = nft_payload_dump,
+ .offload = nft_payload_offload,
};
const struct nft_expr_ops nft_payload_fast_ops = {
@@ -164,6 +350,7 @@ const struct nft_expr_ops nft_payload_fast_ops = {
.eval = nft_payload_eval,
.init = nft_payload_init,
.dump = nft_payload_dump,
+ .offload = nft_payload_offload,
};
static inline void nft_csum_replace(__sum16 *sum, __wsum fsum, __wsum tsum)
--
2.11.0
^ permalink raw reply related
* [PATCH net-next,v3 08/11] drivers: net: use flow block API
From: Pablo Neira Ayuso @ 2019-07-08 16:06 UTC (permalink / raw)
To: netdev
Cc: davem, thomas.lendacky, f.fainelli, ariel.elior, michael.chan,
madalin.bucur, yisen.zhuang, salil.mehta, jeffrey.t.kirsher,
tariqt, saeedm, jiri, idosch, jakub.kicinski, peppe.cavallaro,
grygorii.strashko, andrew, vivien.didelot, alexandre.torgue,
joabreu, linux-net-drivers, ogerlitz, Manish.Chopra,
marcelo.leitner, mkubecek, venkatkumar.duvvuru, maxime.chevallier,
cphealy, netfilter-devel
In-Reply-To: <20190708160614.2226-1-pablo@netfilter.org>
This patch updates flow_block_cb_setup_simple() to use the flow block API.
Several drivers are also adjusted to use it.
This patch introduces the per-driver list of flow blocks to account for
blocks that are already in use.
Remove tc_block_offload alias.
Signed-off-by: Pablo Neira Ayuso <pablo@netfilter.org>
---
v3: incorrect list name variables in several drivers - Jakub Kicinski.
drivers/net/ethernet/broadcom/bnxt/bnxt.c | 5 +-
drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.c | 5 +-
drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c | 5 +-
drivers/net/ethernet/intel/i40e/i40e_main.c | 5 +-
drivers/net/ethernet/intel/iavf/iavf_main.c | 5 +-
drivers/net/ethernet/intel/igb/igb_main.c | 5 +-
drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 5 +-
drivers/net/ethernet/mellanox/mlx5/core/en_main.c | 5 +-
drivers/net/ethernet/mellanox/mlx5/core/en_rep.c | 40 +++++++---
drivers/net/ethernet/mellanox/mlxsw/spectrum.c | 91 +++++++++++++---------
drivers/net/ethernet/mscc/ocelot_ace.h | 4 +-
drivers/net/ethernet/mscc/ocelot_flower.c | 45 ++++++-----
drivers/net/ethernet/mscc/ocelot_tc.c | 33 +++++---
drivers/net/ethernet/netronome/nfp/abm/cls.c | 7 +-
drivers/net/ethernet/netronome/nfp/abm/main.h | 2 +-
drivers/net/ethernet/netronome/nfp/bpf/main.c | 5 +-
.../net/ethernet/netronome/nfp/flower/offload.c | 65 +++++++++++-----
drivers/net/ethernet/qlogic/qede/qede_main.c | 5 +-
drivers/net/ethernet/stmicro/stmmac/stmmac_main.c | 5 +-
drivers/net/netdevsim/netdev.c | 5 +-
include/net/flow_offload.h | 3 +-
include/net/pkt_cls.h | 2 -
net/core/flow_offload.c | 19 ++++-
net/dsa/slave.c | 22 +++++-
net/sched/cls_api.c | 14 ++--
25 files changed, 275 insertions(+), 132 deletions(-)
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
index d2f8c3ed6c73..e6447addf0d6 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
@@ -9847,6 +9847,8 @@ static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
}
}
+static LIST_HEAD(bnxt_block_cb_list);
+
static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type,
void *type_data)
{
@@ -9854,7 +9856,8 @@ static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type,
switch (type) {
case TC_SETUP_BLOCK:
- return flow_block_cb_setup_simple(type_data, NULL,
+ return flow_block_cb_setup_simple(type_data,
+ &bnxt_block_cb_list,
bnxt_setup_tc_block_cb,
bp, bp, true);
case TC_SETUP_QDISC_MQPRIO: {
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.c
index 89398ff011d4..f9bf7d7250ab 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.c
@@ -161,6 +161,8 @@ static int bnxt_vf_rep_setup_tc_block_cb(enum tc_setup_type type,
}
}
+static LIST_HEAD(bnxt_vf_block_cb_list);
+
static int bnxt_vf_rep_setup_tc(struct net_device *dev, enum tc_setup_type type,
void *type_data)
{
@@ -168,7 +170,8 @@ static int bnxt_vf_rep_setup_tc(struct net_device *dev, enum tc_setup_type type,
switch (type) {
case TC_SETUP_BLOCK:
- return flow_block_cb_setup_simple(type_data, NULL,
+ return flow_block_cb_setup_simple(type_data,
+ &bnxt_vf_block_cb_list,
bnxt_vf_rep_setup_tc_block_cb,
vf_rep, vf_rep, true);
default:
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
index 9a486282a32e..fdc8ca4f8891 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
@@ -3190,6 +3190,8 @@ static int cxgb_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
}
}
+static LIST_HEAD(cxgb_block_cb_list);
+
static int cxgb_setup_tc(struct net_device *dev, enum tc_setup_type type,
void *type_data)
{
@@ -3197,7 +3199,8 @@ static int cxgb_setup_tc(struct net_device *dev, enum tc_setup_type type,
switch (type) {
case TC_SETUP_BLOCK:
- return flow_block_cb_setup_simple(type_data, NULL,
+ return flow_block_cb_setup_simple(type_data,
+ &cxgb_block_cb_list,
cxgb_setup_tc_block_cb,
pi, dev, true);
default:
diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c
index 52f0f14d4207..7be1080680f5 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_main.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_main.c
@@ -8177,6 +8177,8 @@ static int i40e_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
}
}
+static LIST_HEAD(i40e_block_cb_list);
+
static int __i40e_setup_tc(struct net_device *netdev, enum tc_setup_type type,
void *type_data)
{
@@ -8186,7 +8188,8 @@ static int __i40e_setup_tc(struct net_device *netdev, enum tc_setup_type type,
case TC_SETUP_QDISC_MQPRIO:
return i40e_setup_tc(netdev, type_data);
case TC_SETUP_BLOCK:
- return flow_block_cb_setup_simple(type_data, NULL,
+ return flow_block_cb_setup_simple(type_data,
+ &i40e_block_cb_list,
i40e_setup_tc_block_cb,
np, np, true);
default:
diff --git a/drivers/net/ethernet/intel/iavf/iavf_main.c b/drivers/net/ethernet/intel/iavf/iavf_main.c
index fd0e2bcc75e5..05eca6f2e890 100644
--- a/drivers/net/ethernet/intel/iavf/iavf_main.c
+++ b/drivers/net/ethernet/intel/iavf/iavf_main.c
@@ -3113,6 +3113,8 @@ static int iavf_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
}
}
+static LIST_HEAD(iavf_block_cb_list);
+
/**
* iavf_setup_tc - configure multiple traffic classes
* @netdev: network interface device structure
@@ -3133,7 +3135,8 @@ static int iavf_setup_tc(struct net_device *netdev, enum tc_setup_type type,
case TC_SETUP_QDISC_MQPRIO:
return __iavf_setup_tc(netdev, type_data);
case TC_SETUP_BLOCK:
- return flow_block_cb_setup_simple(type_data, NULL,
+ return flow_block_cb_setup_simple(type_data,
+ &iavf_block_cb_list,
iavf_setup_tc_block_cb,
adapter, adapter, true);
default:
diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c
index 836f9e1a136c..00e8186e2c59 100644
--- a/drivers/net/ethernet/intel/igb/igb_main.c
+++ b/drivers/net/ethernet/intel/igb/igb_main.c
@@ -2806,6 +2806,8 @@ static int igb_offload_txtime(struct igb_adapter *adapter,
return 0;
}
+static LIST_HEAD(igb_block_cb_list);
+
static int igb_setup_tc(struct net_device *dev, enum tc_setup_type type,
void *type_data)
{
@@ -2815,7 +2817,8 @@ static int igb_setup_tc(struct net_device *dev, enum tc_setup_type type,
case TC_SETUP_QDISC_CBS:
return igb_offload_cbs(adapter, type_data);
case TC_SETUP_BLOCK:
- return flow_block_cb_setup_simple(type_data, NULL,
+ return flow_block_cb_setup_simple(type_data,
+ &igb_block_cb_list,
igb_setup_tc_block_cb,
adapter, adapter, true);
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
index b098f5be9c0d..cbaf712d6529 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
@@ -9614,6 +9614,8 @@ static int ixgbe_setup_tc_mqprio(struct net_device *dev,
return ixgbe_setup_tc(dev, mqprio->num_tc);
}
+static LIST_HEAD(ixgbe_block_cb_list);
+
static int __ixgbe_setup_tc(struct net_device *dev, enum tc_setup_type type,
void *type_data)
{
@@ -9621,7 +9623,8 @@ static int __ixgbe_setup_tc(struct net_device *dev, enum tc_setup_type type,
switch (type) {
case TC_SETUP_BLOCK:
- return flow_block_cb_setup_simple(type_data, NULL,
+ return flow_block_cb_setup_simple(type_data,
+ &ixgbe_block_cb_list,
ixgbe_setup_tc_block_cb,
adapter, adapter, true);
case TC_SETUP_QDISC_MQPRIO:
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
index 395890fea358..10ee7c96d542 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -3459,6 +3459,8 @@ static int mlx5e_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
}
#endif
+static LIST_HEAD(mlx5e_block_cb_list);
+
static int mlx5e_setup_tc(struct net_device *dev, enum tc_setup_type type,
void *type_data)
{
@@ -3467,7 +3469,8 @@ static int mlx5e_setup_tc(struct net_device *dev, enum tc_setup_type type,
switch (type) {
#ifdef CONFIG_MLX5_ESWITCH
case TC_SETUP_BLOCK:
- return flow_block_cb_setup_simple(type_data, NULL,
+ return flow_block_cb_setup_simple(type_data,
+ &mlx5e_block_cb_list,
mlx5e_setup_tc_block_cb,
priv, priv, true);
#endif
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
index f5c430973b35..19133b9e121a 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
@@ -692,17 +692,29 @@ static int mlx5e_rep_indr_setup_block_cb(enum tc_setup_type type,
}
}
+static void mlx5e_rep_indr_tc_block_unbind(void *cb_priv)
+{
+ struct mlx5e_rep_indr_block_priv *indr_priv = cb_priv;
+
+ list_del(&indr_priv->list);
+ kfree(indr_priv);
+}
+
+static LIST_HEAD(mlx5e_block_cb_list);
+
static int
mlx5e_rep_indr_setup_tc_block(struct net_device *netdev,
struct mlx5e_rep_priv *rpriv,
- struct tc_block_offload *f)
+ struct flow_block_offload *f)
{
struct mlx5e_rep_indr_block_priv *indr_priv;
- int err = 0;
+ struct flow_block_cb *block_cb;
if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
return -EOPNOTSUPP;
+ f->driver_block_list = &mlx5e_block_cb_list;
+
switch (f->command) {
case FLOW_BLOCK_BIND:
indr_priv = mlx5e_rep_indr_block_priv_lookup(rpriv, netdev);
@@ -718,26 +730,30 @@ mlx5e_rep_indr_setup_tc_block(struct net_device *netdev,
list_add(&indr_priv->list,
&rpriv->uplink_priv.tc_indr_block_priv_list);
- err = tcf_block_cb_register(f->block,
- mlx5e_rep_indr_setup_block_cb,
- indr_priv, indr_priv, f->extack);
- if (err) {
+ block_cb = flow_block_cb_alloc(f->net,
+ mlx5e_rep_indr_setup_block_cb,
+ indr_priv, indr_priv,
+ mlx5e_rep_indr_tc_block_unbind);
+ if (IS_ERR(block_cb)) {
list_del(&indr_priv->list);
kfree(indr_priv);
+ return PTR_ERR(block_cb);
}
+ flow_block_cb_add(block_cb, f);
- return err;
+ return 0;
case FLOW_BLOCK_UNBIND:
indr_priv = mlx5e_rep_indr_block_priv_lookup(rpriv, netdev);
if (!indr_priv)
return -ENOENT;
- tcf_block_cb_unregister(f->block,
- mlx5e_rep_indr_setup_block_cb,
- indr_priv);
- list_del(&indr_priv->list);
- kfree(indr_priv);
+ block_cb = flow_block_cb_lookup(f->net, &mlx5e_block_cb_list,
+ mlx5e_rep_indr_setup_block_cb,
+ indr_priv);
+ if (!block_cb)
+ return -ENOENT;
+ flow_block_cb_remove(block_cb, f);
return 0;
default:
return -EOPNOTSUPP;
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
index a178d082f061..90a4241551ee 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
@@ -1585,33 +1585,46 @@ static int mlxsw_sp_setup_tc_block_cb_flower(enum tc_setup_type type,
}
}
+static void mlxsw_sp_tc_block_flower_release(void *cb_priv)
+{
+ struct mlxsw_sp_acl_block *acl_block = cb_priv;
+
+ mlxsw_sp_acl_block_destroy(acl_block);
+}
+
+static LIST_HEAD(mlxsw_sp_block_cb_list);
+
static int
mlxsw_sp_setup_tc_block_flower_bind(struct mlxsw_sp_port *mlxsw_sp_port,
- struct tcf_block *block, bool ingress,
- struct netlink_ext_ack *extack)
+ struct flow_block_offload *f, bool ingress)
{
struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
struct mlxsw_sp_acl_block *acl_block;
- struct tcf_block_cb *block_cb;
+ struct flow_block_cb *block_cb;
+ bool register_block = false;
int err;
- block_cb = tcf_block_cb_lookup(block, mlxsw_sp_setup_tc_block_cb_flower,
- mlxsw_sp);
+ block_cb = flow_block_cb_lookup(f->net, &mlxsw_sp_block_cb_list,
+ mlxsw_sp_setup_tc_block_cb_flower,
+ mlxsw_sp);
if (!block_cb) {
- acl_block = mlxsw_sp_acl_block_create(mlxsw_sp, block->net);
+ acl_block = mlxsw_sp_acl_block_create(mlxsw_sp, f->net);
if (!acl_block)
return -ENOMEM;
- block_cb = __tcf_block_cb_register(block,
- mlxsw_sp_setup_tc_block_cb_flower,
- mlxsw_sp, acl_block, extack);
+ block_cb = flow_block_cb_alloc(f->net,
+ mlxsw_sp_setup_tc_block_cb_flower,
+ mlxsw_sp, acl_block,
+ mlxsw_sp_tc_block_flower_release);
if (IS_ERR(block_cb)) {
+ mlxsw_sp_acl_block_destroy(acl_block);
err = PTR_ERR(block_cb);
goto err_cb_register;
}
+ register_block = true;
} else {
- acl_block = tcf_block_cb_priv(block_cb);
+ acl_block = flow_block_cb_priv(block_cb);
}
- tcf_block_cb_incref(block_cb);
+ flow_block_cb_incref(block_cb);
err = mlxsw_sp_acl_block_bind(mlxsw_sp, acl_block,
mlxsw_sp_port, ingress);
if (err)
@@ -1622,28 +1635,30 @@ mlxsw_sp_setup_tc_block_flower_bind(struct mlxsw_sp_port *mlxsw_sp_port,
else
mlxsw_sp_port->eg_acl_block = acl_block;
+ if (register_block)
+ flow_block_cb_add(block_cb, f);
+
return 0;
err_block_bind:
- if (!tcf_block_cb_decref(block_cb)) {
- __tcf_block_cb_unregister(block, block_cb);
+ if (!flow_block_cb_decref(block_cb))
+ flow_block_cb_free(block_cb);
err_cb_register:
- mlxsw_sp_acl_block_destroy(acl_block);
- }
return err;
}
static void
mlxsw_sp_setup_tc_block_flower_unbind(struct mlxsw_sp_port *mlxsw_sp_port,
- struct tcf_block *block, bool ingress)
+ struct flow_block_offload *f, bool ingress)
{
struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
struct mlxsw_sp_acl_block *acl_block;
- struct tcf_block_cb *block_cb;
+ struct flow_block_cb *block_cb;
int err;
- block_cb = tcf_block_cb_lookup(block, mlxsw_sp_setup_tc_block_cb_flower,
- mlxsw_sp);
+ block_cb = flow_block_cb_lookup(f->net, &mlxsw_sp_block_cb_list,
+ mlxsw_sp_setup_tc_block_cb_flower,
+ mlxsw_sp);
if (!block_cb)
return;
@@ -1652,18 +1667,17 @@ mlxsw_sp_setup_tc_block_flower_unbind(struct mlxsw_sp_port *mlxsw_sp_port,
else
mlxsw_sp_port->eg_acl_block = NULL;
- acl_block = tcf_block_cb_priv(block_cb);
+ acl_block = flow_block_cb_priv(block_cb);
err = mlxsw_sp_acl_block_unbind(mlxsw_sp, acl_block,
mlxsw_sp_port, ingress);
- if (!err && !tcf_block_cb_decref(block_cb)) {
- __tcf_block_cb_unregister(block, block_cb);
- mlxsw_sp_acl_block_destroy(acl_block);
- }
+ if (!err && !flow_block_cb_decref(block_cb))
+ flow_block_cb_remove(block_cb, f);
}
static int mlxsw_sp_setup_tc_block(struct mlxsw_sp_port *mlxsw_sp_port,
- struct tc_block_offload *f)
+ struct flow_block_offload *f)
{
+ struct flow_block_cb *block_cb;
tc_setup_cb_t *cb;
bool ingress;
int err;
@@ -1678,24 +1692,31 @@ static int mlxsw_sp_setup_tc_block(struct mlxsw_sp_port *mlxsw_sp_port,
return -EOPNOTSUPP;
}
+ f->driver_block_list = &mlxsw_sp_block_cb_list;
+
switch (f->command) {
case FLOW_BLOCK_BIND:
- err = tcf_block_cb_register(f->block, cb, mlxsw_sp_port,
- mlxsw_sp_port, f->extack);
- if (err)
- return err;
- err = mlxsw_sp_setup_tc_block_flower_bind(mlxsw_sp_port,
- f->block, ingress,
- f->extack);
+ block_cb = flow_block_cb_alloc(f->net, cb, mlxsw_sp_port,
+ mlxsw_sp_port, NULL);
+ if (IS_ERR(block_cb))
+ return PTR_ERR(block_cb);
+ err = mlxsw_sp_setup_tc_block_flower_bind(mlxsw_sp_port, f,
+ ingress);
if (err) {
- tcf_block_cb_unregister(f->block, cb, mlxsw_sp_port);
+ flow_block_cb_free(block_cb);
return err;
}
+ flow_block_cb_add(block_cb, f);
return 0;
case FLOW_BLOCK_UNBIND:
mlxsw_sp_setup_tc_block_flower_unbind(mlxsw_sp_port,
- f->block, ingress);
- tcf_block_cb_unregister(f->block, cb, mlxsw_sp_port);
+ f, ingress);
+ block_cb = flow_block_cb_lookup(f->net, &mlxsw_sp_block_cb_list,
+ cb, mlxsw_sp_port);
+ if (!block_cb)
+ return -ENOENT;
+
+ flow_block_cb_remove(block_cb, f);
return 0;
default:
return -EOPNOTSUPP;
diff --git a/drivers/net/ethernet/mscc/ocelot_ace.h b/drivers/net/ethernet/mscc/ocelot_ace.h
index d621683643e1..e98944c87259 100644
--- a/drivers/net/ethernet/mscc/ocelot_ace.h
+++ b/drivers/net/ethernet/mscc/ocelot_ace.h
@@ -225,8 +225,8 @@ int ocelot_ace_init(struct ocelot *ocelot);
void ocelot_ace_deinit(void);
int ocelot_setup_tc_block_flower_bind(struct ocelot_port *port,
- struct tc_block_offload *f);
+ struct flow_block_offload *f);
void ocelot_setup_tc_block_flower_unbind(struct ocelot_port *port,
- struct tc_block_offload *f);
+ struct flow_block_offload *f);
#endif /* _MSCC_OCELOT_ACE_H_ */
diff --git a/drivers/net/ethernet/mscc/ocelot_flower.c b/drivers/net/ethernet/mscc/ocelot_flower.c
index b682f08a93b4..afe79af035cd 100644
--- a/drivers/net/ethernet/mscc/ocelot_flower.c
+++ b/drivers/net/ethernet/mscc/ocelot_flower.c
@@ -299,36 +299,44 @@ static void ocelot_port_block_destroy(struct ocelot_port_block *block)
kfree(block);
}
+static void ocelot_tc_block_unbind(void *cb_priv)
+{
+ struct ocelot_port_block *port_block = cb_priv;
+
+ ocelot_port_block_destroy(port_block);
+}
+
int ocelot_setup_tc_block_flower_bind(struct ocelot_port *port,
- struct tc_block_offload *f)
+ struct flow_block_offload *f)
{
struct ocelot_port_block *port_block;
- struct tcf_block_cb *block_cb;
+ struct flow_block_cb *block_cb;
int ret;
if (f->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS)
return -EOPNOTSUPP;
- block_cb = tcf_block_cb_lookup(f->block,
- ocelot_setup_tc_block_cb_flower, port);
+ block_cb = flow_block_cb_lookup(f->net, f->driver_block_list,
+ ocelot_setup_tc_block_cb_flower, port);
if (!block_cb) {
port_block = ocelot_port_block_create(port);
if (!port_block)
return -ENOMEM;
- block_cb =
- __tcf_block_cb_register(f->block,
- ocelot_setup_tc_block_cb_flower,
- port, port_block, f->extack);
+ block_cb = flow_block_cb_alloc(f->net,
+ ocelot_setup_tc_block_cb_flower,
+ port, port_block,
+ ocelot_tc_block_unbind);
if (IS_ERR(block_cb)) {
ret = PTR_ERR(block_cb);
goto err_cb_register;
}
+ flow_block_cb_add(block_cb, f);
} else {
- port_block = tcf_block_cb_priv(block_cb);
+ port_block = flow_block_cb_priv(block_cb);
}
- tcf_block_cb_incref(block_cb);
+ flow_block_cb_incref(block_cb);
return 0;
err_cb_register:
@@ -338,20 +346,15 @@ int ocelot_setup_tc_block_flower_bind(struct ocelot_port *port,
}
void ocelot_setup_tc_block_flower_unbind(struct ocelot_port *port,
- struct tc_block_offload *f)
+ struct flow_block_offload *f)
{
- struct ocelot_port_block *port_block;
- struct tcf_block_cb *block_cb;
+ struct flow_block_cb *block_cb;
- block_cb = tcf_block_cb_lookup(f->block,
- ocelot_setup_tc_block_cb_flower, port);
+ block_cb = flow_block_cb_lookup(f->net, f->driver_block_list,
+ ocelot_setup_tc_block_cb_flower, port);
if (!block_cb)
return;
- port_block = tcf_block_cb_priv(block_cb);
- if (!tcf_block_cb_decref(block_cb)) {
- tcf_block_cb_unregister(f->block,
- ocelot_setup_tc_block_cb_flower, port);
- ocelot_port_block_destroy(port_block);
- }
+ if (!flow_block_cb_decref(block_cb))
+ flow_block_cb_remove(block_cb, f);
}
diff --git a/drivers/net/ethernet/mscc/ocelot_tc.c b/drivers/net/ethernet/mscc/ocelot_tc.c
index 58a0b5f8850c..9bca2cc77e6c 100644
--- a/drivers/net/ethernet/mscc/ocelot_tc.c
+++ b/drivers/net/ethernet/mscc/ocelot_tc.c
@@ -128,35 +128,50 @@ static int ocelot_setup_tc_block_cb_eg(enum tc_setup_type type,
cb_priv, false);
}
+static LIST_HEAD(ocelot_block_cb_list);
+
static int ocelot_setup_tc_block(struct ocelot_port *port,
- struct tc_block_offload *f)
+ struct flow_block_offload *f)
{
+ struct flow_block_cb *block_cb;
tc_setup_cb_t *cb;
- int ret;
+ int err;
netdev_dbg(port->dev, "tc_block command %d, binder_type %d\n",
f->command, f->binder_type);
if (f->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS) {
cb = ocelot_setup_tc_block_cb_ig;
- port->tc.block_shared = tcf_block_shared(f->block);
+ port->tc.block_shared = f->block_shared;
} else if (f->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS) {
cb = ocelot_setup_tc_block_cb_eg;
} else {
return -EOPNOTSUPP;
}
+ f->driver_block_list = &ocelot_block_cb_list;
+
switch (f->command) {
case FLOW_BLOCK_BIND:
- ret = tcf_block_cb_register(f->block, cb, port,
- port, f->extack);
- if (ret)
- return ret;
+ block_cb = flow_block_cb_alloc(f->net, cb, port, port, NULL);
+ if (IS_ERR(block_cb))
+ return PTR_ERR(block_cb);
- return ocelot_setup_tc_block_flower_bind(port, f);
+ err = ocelot_setup_tc_block_flower_bind(port, f);
+ if (err < 0) {
+ flow_block_cb_free(block_cb);
+ return err;
+ }
+ flow_block_cb_add(block_cb, f);
+ return 0;
case FLOW_BLOCK_UNBIND:
+ block_cb = flow_block_cb_lookup(f->net, &ocelot_block_cb_list,
+ cb, port);
+ if (!block_cb)
+ return -ENOENT;
+
ocelot_setup_tc_block_flower_unbind(port, f);
- tcf_block_cb_unregister(f->block, cb, port);
+ flow_block_cb_remove(block_cb, f);
return 0;
default:
return -EOPNOTSUPP;
diff --git a/drivers/net/ethernet/netronome/nfp/abm/cls.c b/drivers/net/ethernet/netronome/nfp/abm/cls.c
index 29fb45734962..23ebddfb9532 100644
--- a/drivers/net/ethernet/netronome/nfp/abm/cls.c
+++ b/drivers/net/ethernet/netronome/nfp/abm/cls.c
@@ -262,9 +262,12 @@ static int nfp_abm_setup_tc_block_cb(enum tc_setup_type type,
}
}
+static LIST_HEAD(nfp_abm_block_cb_list);
+
int nfp_abm_setup_cls_block(struct net_device *netdev, struct nfp_repr *repr,
- struct tc_block_offload *f)
+ struct flow_block_offload *f)
{
- return flow_block_cb_setup_simple(f, NULL, nfp_abm_setup_tc_block_cb,
+ return flow_block_cb_setup_simple(f, &nfp_abm_block_cb_list,
+ nfp_abm_setup_tc_block_cb,
repr, repr, true);
}
diff --git a/drivers/net/ethernet/netronome/nfp/abm/main.h b/drivers/net/ethernet/netronome/nfp/abm/main.h
index 49749c60885e..48746c9c6224 100644
--- a/drivers/net/ethernet/netronome/nfp/abm/main.h
+++ b/drivers/net/ethernet/netronome/nfp/abm/main.h
@@ -247,7 +247,7 @@ int nfp_abm_setup_tc_mq(struct net_device *netdev, struct nfp_abm_link *alink,
int nfp_abm_setup_tc_gred(struct net_device *netdev, struct nfp_abm_link *alink,
struct tc_gred_qopt_offload *opt);
int nfp_abm_setup_cls_block(struct net_device *netdev, struct nfp_repr *repr,
- struct tc_block_offload *opt);
+ struct flow_block_offload *opt);
int nfp_abm_ctrl_read_params(struct nfp_abm_link *alink);
int nfp_abm_ctrl_find_addrs(struct nfp_abm *abm);
diff --git a/drivers/net/ethernet/netronome/nfp/bpf/main.c b/drivers/net/ethernet/netronome/nfp/bpf/main.c
index 0c93c84a188a..7549547c4ef0 100644
--- a/drivers/net/ethernet/netronome/nfp/bpf/main.c
+++ b/drivers/net/ethernet/netronome/nfp/bpf/main.c
@@ -160,6 +160,8 @@ static int nfp_bpf_setup_tc_block_cb(enum tc_setup_type type,
return 0;
}
+static LIST_HEAD(nfp_bfp_block_cb_list);
+
static int nfp_bpf_setup_tc(struct nfp_app *app, struct net_device *netdev,
enum tc_setup_type type, void *type_data)
{
@@ -167,7 +169,8 @@ static int nfp_bpf_setup_tc(struct nfp_app *app, struct net_device *netdev,
switch (type) {
case TC_SETUP_BLOCK:
- return flow_block_cb_setup_simple(type_data, NULL,
+ return flow_block_cb_setup_simple(type_data,
+ &nfp_bfp_block_cb_list,
nfp_bpf_setup_tc_block_cb,
nn, nn, true);
default:
diff --git a/drivers/net/ethernet/netronome/nfp/flower/offload.c b/drivers/net/ethernet/netronome/nfp/flower/offload.c
index 46041e509150..9ce61b6e8583 100644
--- a/drivers/net/ethernet/netronome/nfp/flower/offload.c
+++ b/drivers/net/ethernet/netronome/nfp/flower/offload.c
@@ -1302,27 +1302,40 @@ static int nfp_flower_setup_tc_block_cb(enum tc_setup_type type,
}
}
+static LIST_HEAD(nfp_block_cb_list);
+
static int nfp_flower_setup_tc_block(struct net_device *netdev,
- struct tc_block_offload *f)
+ struct flow_block_offload *f)
{
struct nfp_repr *repr = netdev_priv(netdev);
struct nfp_flower_repr_priv *repr_priv;
+ struct flow_block_cb *block_cb;
if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
return -EOPNOTSUPP;
repr_priv = repr->app_priv;
- repr_priv->block_shared = tcf_block_shared(f->block);
+ repr_priv->block_shared = f->block_shared;
+ f->driver_block_list = &nfp_block_cb_list;
switch (f->command) {
case FLOW_BLOCK_BIND:
- return tcf_block_cb_register(f->block,
- nfp_flower_setup_tc_block_cb,
- repr, repr, f->extack);
+ block_cb = flow_block_cb_alloc(f->net,
+ nfp_flower_setup_tc_block_cb,
+ repr, repr, NULL);
+ if (IS_ERR(block_cb))
+ return PTR_ERR(block_cb);
+
+ flow_block_cb_add(block_cb, f);
+ return 0;
case FLOW_BLOCK_UNBIND:
- tcf_block_cb_unregister(f->block,
- nfp_flower_setup_tc_block_cb,
- repr);
+ block_cb = flow_block_cb_lookup(f->net, &nfp_block_cb_list,
+ nfp_flower_setup_tc_block_cb,
+ repr);
+ if (!block_cb)
+ return -ENOENT;
+
+ flow_block_cb_remove(block_cb, f);
return 0;
default:
return -EOPNOTSUPP;
@@ -1381,13 +1394,21 @@ static int nfp_flower_setup_indr_block_cb(enum tc_setup_type type,
}
}
+static void nfp_flower_setup_indr_tc_release(void *cb_priv)
+{
+ struct nfp_flower_indr_block_cb_priv *priv = cb_priv;
+
+ list_del(&priv->list);
+ kfree(priv);
+}
+
static int
nfp_flower_setup_indr_tc_block(struct net_device *netdev, struct nfp_app *app,
- struct tc_block_offload *f)
+ struct flow_block_offload *f)
{
struct nfp_flower_indr_block_cb_priv *cb_priv;
struct nfp_flower_priv *priv = app->priv;
- int err;
+ struct flow_block_cb *block_cb;
if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS &&
!(f->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS &&
@@ -1404,26 +1425,30 @@ nfp_flower_setup_indr_tc_block(struct net_device *netdev, struct nfp_app *app,
cb_priv->app = app;
list_add(&cb_priv->list, &priv->indr_block_cb_priv);
- err = tcf_block_cb_register(f->block,
- nfp_flower_setup_indr_block_cb,
- cb_priv, cb_priv, f->extack);
- if (err) {
+ block_cb = flow_block_cb_alloc(f->net,
+ nfp_flower_setup_indr_block_cb,
+ cb_priv, cb_priv,
+ nfp_flower_setup_indr_tc_release);
+ if (IS_ERR(block_cb)) {
list_del(&cb_priv->list);
kfree(cb_priv);
+ return PTR_ERR(block_cb);
}
- return err;
+ flow_block_cb_add(block_cb, f);
+ return 0;
case FLOW_BLOCK_UNBIND:
cb_priv = nfp_flower_indr_block_cb_priv_lookup(app, netdev);
if (!cb_priv)
return -ENOENT;
- tcf_block_cb_unregister(f->block,
- nfp_flower_setup_indr_block_cb,
- cb_priv);
- list_del(&cb_priv->list);
- kfree(cb_priv);
+ block_cb = flow_block_cb_lookup(f->net, &nfp_block_cb_list,
+ nfp_flower_setup_indr_block_cb,
+ cb_priv);
+ if (!block_cb)
+ return -ENOENT;
+ flow_block_cb_remove(block_cb, f);
return 0;
default:
return -EOPNOTSUPP;
diff --git a/drivers/net/ethernet/qlogic/qede/qede_main.c b/drivers/net/ethernet/qlogic/qede/qede_main.c
index cba97ed3dd56..1be593a6e20d 100644
--- a/drivers/net/ethernet/qlogic/qede/qede_main.c
+++ b/drivers/net/ethernet/qlogic/qede/qede_main.c
@@ -579,6 +579,8 @@ static int qede_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
}
}
+static LIST_HEAD(qede_block_cb_list);
+
static int
qede_setup_tc_offload(struct net_device *dev, enum tc_setup_type type,
void *type_data)
@@ -588,7 +590,8 @@ qede_setup_tc_offload(struct net_device *dev, enum tc_setup_type type,
switch (type) {
case TC_SETUP_BLOCK:
- return flow_block_cb_setup_simple(type_data, NULL,
+ return flow_block_cb_setup_simple(type_data,
+ &qede_block_cb_list,
qede_setup_tc_block_cb,
edev, edev, true);
case TC_SETUP_QDISC_MQPRIO:
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
index 0040f10bbef9..cc3bfb59c8df 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
@@ -3853,6 +3853,8 @@ static int stmmac_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
return ret;
}
+static LIST_HEAD(stmmac_block_cb_list);
+
static int stmmac_setup_tc(struct net_device *ndev, enum tc_setup_type type,
void *type_data)
{
@@ -3860,7 +3862,8 @@ static int stmmac_setup_tc(struct net_device *ndev, enum tc_setup_type type,
switch (type) {
case TC_SETUP_BLOCK:
- return flow_block_cb_setup_simple(type_data, NULL,
+ return flow_block_cb_setup_simple(type_data,
+ &stmmac_block_cb_list,
stmmac_setup_tc_block_cb,
priv, priv, true);
case TC_SETUP_QDISC_CBS:
diff --git a/drivers/net/netdevsim/netdev.c b/drivers/net/netdevsim/netdev.c
index 920dc79e9dc9..0740940f41b1 100644
--- a/drivers/net/netdevsim/netdev.c
+++ b/drivers/net/netdevsim/netdev.c
@@ -203,6 +203,8 @@ static int nsim_set_vf_link_state(struct net_device *dev, int vf, int state)
return 0;
}
+static LIST_HEAD(nsim_block_cb_list);
+
static int
nsim_setup_tc(struct net_device *dev, enum tc_setup_type type, void *type_data)
{
@@ -210,7 +212,8 @@ nsim_setup_tc(struct net_device *dev, enum tc_setup_type type, void *type_data)
switch (type) {
case TC_SETUP_BLOCK:
- return flow_block_cb_setup_simple(type_data, NULL,
+ return flow_block_cb_setup_simple(type_data,
+ &nsim_block_cb_list,
nsim_setup_tc_block_cb,
ns, ns, true);
default:
diff --git a/include/net/flow_offload.h b/include/net/flow_offload.h
index 342e6aef7e3a..7977508ec406 100644
--- a/include/net/flow_offload.h
+++ b/include/net/flow_offload.h
@@ -244,13 +244,12 @@ enum flow_block_binder_type {
FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS,
};
-struct tcf_block;
struct netlink_ext_ack;
struct flow_block_offload {
enum flow_block_command command;
enum flow_block_binder_type binder_type;
- struct tcf_block *block;
+ bool block_shared;
struct net *net;
struct list_head cb_list;
struct list_head *driver_block_list;
diff --git a/include/net/pkt_cls.h b/include/net/pkt_cls.h
index e4499526fde8..9cf606b88526 100644
--- a/include/net/pkt_cls.h
+++ b/include/net/pkt_cls.h
@@ -26,8 +26,6 @@ struct tcf_walker {
int register_tcf_proto_ops(struct tcf_proto_ops *ops);
int unregister_tcf_proto_ops(struct tcf_proto_ops *ops);
-#define tc_block_offload flow_block_offload
-
struct tcf_block_ext_info {
enum flow_block_binder_type binder_type;
tcf_chain_head_change_t *chain_head_change;
diff --git a/net/core/flow_offload.c b/net/core/flow_offload.c
index 916ff32e53e7..4bea343d52a0 100644
--- a/net/core/flow_offload.c
+++ b/net/core/flow_offload.c
@@ -2,7 +2,6 @@
#include <linux/kernel.h>
#include <linux/slab.h>
#include <net/flow_offload.h>
-#include <net/pkt_cls.h>
struct flow_rule *flow_rule_alloc(unsigned int num_actions)
{
@@ -235,6 +234,8 @@ int flow_block_cb_setup_simple(struct flow_block_offload *f,
tc_setup_cb_t *cb, void *cb_ident, void *cb_priv,
bool ingress_only)
{
+ struct flow_block_cb *block_cb;
+
if (ingress_only &&
f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
return -EOPNOTSUPP;
@@ -243,10 +244,20 @@ int flow_block_cb_setup_simple(struct flow_block_offload *f,
switch (f->command) {
case FLOW_BLOCK_BIND:
- return tcf_block_cb_register(f->block, cb, cb_ident, cb_priv,
- f->extack);
+ block_cb = flow_block_cb_alloc(f->net, cb, cb_ident,
+ cb_priv, NULL);
+ if (IS_ERR(block_cb))
+ return PTR_ERR(block_cb);
+
+ flow_block_cb_add(block_cb, f);
+ return 0;
case FLOW_BLOCK_UNBIND:
- tcf_block_cb_unregister(f->block, cb, cb_ident);
+ block_cb = flow_block_cb_lookup(f->net, driver_block_list,
+ cb, cb_ident);
+ if (!block_cb)
+ return -ENOENT;
+
+ flow_block_cb_remove(block_cb, f);
return 0;
default:
return -EOPNOTSUPP;
diff --git a/net/dsa/slave.c b/net/dsa/slave.c
index 9b5e202c255e..d2e74d08c1c4 100644
--- a/net/dsa/slave.c
+++ b/net/dsa/slave.c
@@ -942,9 +942,12 @@ static int dsa_slave_setup_tc_block_cb_eg(enum tc_setup_type type,
return dsa_slave_setup_tc_block_cb(type, type_data, cb_priv, false);
}
+static LIST_HEAD(dsa_slave_block_cb_list);
+
static int dsa_slave_setup_tc_block(struct net_device *dev,
- struct tc_block_offload *f)
+ struct flow_block_offload *f)
{
+ struct flow_block_cb *block_cb;
tc_setup_cb_t *cb;
if (f->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
@@ -954,11 +957,24 @@ static int dsa_slave_setup_tc_block(struct net_device *dev,
else
return -EOPNOTSUPP;
+ f->driver_block_list = &dsa_slave_block_cb_list;
+
switch (f->command) {
case FLOW_BLOCK_BIND:
- return tcf_block_cb_register(f->block, cb, dev, dev, f->extack);
+ block_cb = flow_block_cb_alloc(f->net, cb, dev, dev, NULL);
+ if (IS_ERR(block_cb))
+ return PTR_ERR(block_cb);
+
+ flow_block_cb_add(block_cb, f);
+ return 0;
case FLOW_BLOCK_UNBIND:
- tcf_block_cb_unregister(f->block, cb, dev);
+ block_cb = flow_block_cb_lookup(f->net,
+ &dsa_slave_block_cb_list,
+ cb, dev);
+ if (!block_cb)
+ return -ENOENT;
+
+ flow_block_cb_remove(block_cb, f);
return 0;
default:
return -EOPNOTSUPP;
diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c
index 728714d8f601..cc87f6a2191f 100644
--- a/net/sched/cls_api.c
+++ b/net/sched/cls_api.c
@@ -679,11 +679,11 @@ static void tc_indr_block_ing_cmd(struct tc_indr_block_dev *indr_dev,
struct tc_indr_block_cb *indr_block_cb,
enum flow_block_command command)
{
- struct tc_block_offload bo = {
+ struct flow_block_offload bo = {
.command = command,
.binder_type = FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS,
.net = dev_net(indr_dev->dev),
- .block = indr_dev->block,
+ .block_shared = tcf_block_shared(indr_dev->block),
};
INIT_LIST_HEAD(&bo.cb_list);
@@ -770,11 +770,11 @@ static void tc_indr_block_call(struct tcf_block *block, struct net_device *dev,
{
struct tc_indr_block_cb *indr_block_cb;
struct tc_indr_block_dev *indr_dev;
- struct tc_block_offload bo = {
+ struct flow_block_offload bo = {
.command = command,
.binder_type = ei->binder_type,
.net = dev_net(dev),
- .block = block,
+ .block_shared = tcf_block_shared(block),
.extack = extack,
};
INIT_LIST_HEAD(&bo.cb_list);
@@ -803,13 +803,13 @@ static int tcf_block_offload_cmd(struct tcf_block *block,
enum flow_block_command command,
struct netlink_ext_ack *extack)
{
- struct tc_block_offload bo = {};
+ struct flow_block_offload bo = {};
int err;
bo.net = dev_net(dev);
bo.command = command;
bo.binder_type = ei->binder_type;
- bo.block = block;
+ bo.block_shared = tcf_block_shared(block);
bo.extack = extack;
INIT_LIST_HEAD(&bo.cb_list);
@@ -3246,7 +3246,7 @@ EXPORT_SYMBOL(tcf_exts_dump_stats);
int tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type,
void *type_data, bool err_stop)
{
- struct tcf_block_cb *block_cb;
+ struct flow_block_cb *block_cb;
int ok_count = 0;
int err;
--
2.11.0
^ permalink raw reply related
* [PATCH net-next,v3 10/11] net: flow_offload: add flow_block_cb_is_busy() and use it
From: Pablo Neira Ayuso @ 2019-07-08 16:06 UTC (permalink / raw)
To: netdev
Cc: davem, thomas.lendacky, f.fainelli, ariel.elior, michael.chan,
madalin.bucur, yisen.zhuang, salil.mehta, jeffrey.t.kirsher,
tariqt, saeedm, jiri, idosch, jakub.kicinski, peppe.cavallaro,
grygorii.strashko, andrew, vivien.didelot, alexandre.torgue,
joabreu, linux-net-drivers, ogerlitz, Manish.Chopra,
marcelo.leitner, mkubecek, venkatkumar.duvvuru, maxime.chevallier,
cphealy, netfilter-devel
In-Reply-To: <20190708160614.2226-1-pablo@netfilter.org>
This patch adds a function to check if flow block callback is already in
use. Call this new function from flow_block_cb_setup_simple() and from
drivers.
Signed-off-by: Pablo Neira Ayuso <pablo@netfilter.org>
---
v3: formerly known as "net: flow_offload: don't allow subsystem to reuse blocks"
add flow_block_cb_is_busy() helper. Call it per driver to make it easier
to remove this whenever the first driver client support for multiple
subsystem offloads.
drivers/net/ethernet/mellanox/mlx5/core/en_rep.c | 4 ++++
drivers/net/ethernet/mellanox/mlxsw/spectrum.c | 4 ++++
drivers/net/ethernet/mscc/ocelot_tc.c | 3 +++
drivers/net/ethernet/netronome/nfp/flower/offload.c | 4 ++++
include/net/flow_offload.h | 3 +++
net/core/flow_offload.c | 18 ++++++++++++++++++
net/dsa/slave.c | 3 +++
7 files changed, 39 insertions(+)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
index 19133b9e121a..e303149053e4 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
@@ -721,6 +721,10 @@ mlx5e_rep_indr_setup_tc_block(struct net_device *netdev,
if (indr_priv)
return -EEXIST;
+ if (flow_block_cb_is_busy(mlx5e_rep_indr_setup_block_cb,
+ indr_priv, &mlx5e_block_cb_list))
+ return -EBUSY;
+
indr_priv = kmalloc(sizeof(*indr_priv), GFP_KERNEL);
if (!indr_priv)
return -ENOMEM;
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
index 90a4241551ee..f5b4d5eaa35a 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
@@ -1696,6 +1696,10 @@ static int mlxsw_sp_setup_tc_block(struct mlxsw_sp_port *mlxsw_sp_port,
switch (f->command) {
case FLOW_BLOCK_BIND:
+ if (flow_block_cb_is_busy(cb, mlxsw_sp_port,
+ &mlxsw_sp_block_cb_list))
+ return -EBUSY;
+
block_cb = flow_block_cb_alloc(f->net, cb, mlxsw_sp_port,
mlxsw_sp_port, NULL);
if (IS_ERR(block_cb))
diff --git a/drivers/net/ethernet/mscc/ocelot_tc.c b/drivers/net/ethernet/mscc/ocelot_tc.c
index 9bca2cc77e6c..e7d2473129f0 100644
--- a/drivers/net/ethernet/mscc/ocelot_tc.c
+++ b/drivers/net/ethernet/mscc/ocelot_tc.c
@@ -153,6 +153,9 @@ static int ocelot_setup_tc_block(struct ocelot_port *port,
switch (f->command) {
case FLOW_BLOCK_BIND:
+ if (flow_block_cb_is_busy(cb, port, &ocelot_block_cb_list))
+ return -EBUSY;
+
block_cb = flow_block_cb_alloc(f->net, cb, port, port, NULL);
if (IS_ERR(block_cb))
return PTR_ERR(block_cb);
diff --git a/drivers/net/ethernet/netronome/nfp/flower/offload.c b/drivers/net/ethernet/netronome/nfp/flower/offload.c
index 9ce61b6e8583..4bdbd322fddf 100644
--- a/drivers/net/ethernet/netronome/nfp/flower/offload.c
+++ b/drivers/net/ethernet/netronome/nfp/flower/offload.c
@@ -1320,6 +1320,10 @@ static int nfp_flower_setup_tc_block(struct net_device *netdev,
switch (f->command) {
case FLOW_BLOCK_BIND:
+ if (flow_block_cb_is_busy(nfp_flower_setup_tc_block_cb, repr,
+ &nfp_block_cb_list))
+ return -EBUSY;
+
block_cb = flow_block_cb_alloc(f->net,
nfp_flower_setup_tc_block_cb,
repr, repr, NULL);
diff --git a/include/net/flow_offload.h b/include/net/flow_offload.h
index 7977508ec406..1751b31ea7e2 100644
--- a/include/net/flow_offload.h
+++ b/include/net/flow_offload.h
@@ -292,6 +292,9 @@ static inline void flow_block_cb_remove(struct flow_block_cb *block_cb,
list_move(&block_cb->driver_list, &offload->cb_list);
}
+bool flow_block_cb_is_busy(tc_setup_cb_t *cb, void *cb_ident,
+ struct list_head *driver_block_list);
+
int flow_block_cb_setup_simple(struct flow_block_offload *f,
struct list_head *driver_list, tc_setup_cb_t *cb,
void *cb_ident, void *cb_priv, bool ingress_only);
diff --git a/net/core/flow_offload.c b/net/core/flow_offload.c
index 4bea343d52a0..39219513181e 100644
--- a/net/core/flow_offload.c
+++ b/net/core/flow_offload.c
@@ -229,6 +229,21 @@ unsigned int flow_block_cb_decref(struct flow_block_cb *block_cb)
}
EXPORT_SYMBOL(flow_block_cb_decref);
+bool flow_block_cb_is_busy(tc_setup_cb_t *cb, void *cb_ident,
+ struct list_head *driver_block_list)
+{
+ struct flow_block_cb *block_cb;
+
+ list_for_each_entry(block_cb, driver_block_list, driver_list) {
+ if (block_cb->cb == cb &&
+ block_cb->cb_ident == cb_ident)
+ return true;
+ }
+
+ return false;
+}
+EXPORT_SYMBOL(flow_block_cb_is_busy);
+
int flow_block_cb_setup_simple(struct flow_block_offload *f,
struct list_head *driver_block_list,
tc_setup_cb_t *cb, void *cb_ident, void *cb_priv,
@@ -244,6 +259,9 @@ int flow_block_cb_setup_simple(struct flow_block_offload *f,
switch (f->command) {
case FLOW_BLOCK_BIND:
+ if (flow_block_cb_is_busy(cb, cb_ident, driver_block_list))
+ return -EBUSY;
+
block_cb = flow_block_cb_alloc(f->net, cb, cb_ident,
cb_priv, NULL);
if (IS_ERR(block_cb))
diff --git a/net/dsa/slave.c b/net/dsa/slave.c
index d2e74d08c1c4..335580cfa486 100644
--- a/net/dsa/slave.c
+++ b/net/dsa/slave.c
@@ -961,6 +961,9 @@ static int dsa_slave_setup_tc_block(struct net_device *dev,
switch (f->command) {
case FLOW_BLOCK_BIND:
+ if (flow_block_cb_is_busy(cb, dev, &dsa_slave_block_cb_list))
+ return -EBUSY;
+
block_cb = flow_block_cb_alloc(f->net, cb, dev, dev, NULL);
if (IS_ERR(block_cb))
return PTR_ERR(block_cb);
--
2.11.0
^ permalink raw reply related
* [PATCH net-next,v3 09/11] net: sched: remove tcf block API
From: Pablo Neira Ayuso @ 2019-07-08 16:06 UTC (permalink / raw)
To: netdev
Cc: davem, thomas.lendacky, f.fainelli, ariel.elior, michael.chan,
madalin.bucur, yisen.zhuang, salil.mehta, jeffrey.t.kirsher,
tariqt, saeedm, jiri, idosch, jakub.kicinski, peppe.cavallaro,
grygorii.strashko, andrew, vivien.didelot, alexandre.torgue,
joabreu, linux-net-drivers, ogerlitz, Manish.Chopra,
marcelo.leitner, mkubecek, venkatkumar.duvvuru, maxime.chevallier,
cphealy, netfilter-devel
In-Reply-To: <20190708160614.2226-1-pablo@netfilter.org>
Unused, now replaced by flow block API.
Signed-off-by: Pablo Neira Ayuso <pablo@netfilter.org>
---
v3: remove tcf_block_cb_priv(), tcf_block_cb_lookup(), tcf_block_cb_incref()
tcf_block_cb_decref() too.
Formely known as "net: sched: remove tcf_block_cb_{register,unregister}()"
include/net/pkt_cls.h | 69 ------------------------------------
net/sched/cls_api.c | 98 ---------------------------------------------------
2 files changed, 167 deletions(-)
diff --git a/include/net/pkt_cls.h b/include/net/pkt_cls.h
index 9cf606b88526..17c388090c3c 100644
--- a/include/net/pkt_cls.h
+++ b/include/net/pkt_cls.h
@@ -66,22 +66,6 @@ static inline struct Qdisc *tcf_block_q(struct tcf_block *block)
return block->q;
}
-void *tcf_block_cb_priv(struct tcf_block_cb *block_cb);
-struct tcf_block_cb *tcf_block_cb_lookup(struct tcf_block *block,
- tc_setup_cb_t *cb, void *cb_ident);
-void tcf_block_cb_incref(struct tcf_block_cb *block_cb);
-unsigned int tcf_block_cb_decref(struct tcf_block_cb *block_cb);
-struct tcf_block_cb *__tcf_block_cb_register(struct tcf_block *block,
- tc_setup_cb_t *cb, void *cb_ident,
- void *cb_priv,
- struct netlink_ext_ack *extack);
-int tcf_block_cb_register(struct tcf_block *block,
- tc_setup_cb_t *cb, void *cb_ident,
- void *cb_priv, struct netlink_ext_ack *extack);
-void __tcf_block_cb_unregister(struct tcf_block *block,
- struct tcf_block_cb *block_cb);
-void tcf_block_cb_unregister(struct tcf_block *block,
- tc_setup_cb_t *cb, void *cb_ident);
int __tc_indr_block_cb_register(struct net_device *dev, void *cb_priv,
tc_indr_block_bind_cb_t *cb, void *cb_ident);
int tc_indr_block_cb_register(struct net_device *dev, void *cb_priv,
@@ -145,59 +129,6 @@ void tc_setup_cb_block_unregister(struct tcf_block *block, tc_setup_cb_t *cb,
}
static inline
-void *tcf_block_cb_priv(struct tcf_block_cb *block_cb)
-{
- return NULL;
-}
-
-static inline
-struct tcf_block_cb *tcf_block_cb_lookup(struct tcf_block *block,
- tc_setup_cb_t *cb, void *cb_ident)
-{
- return NULL;
-}
-
-static inline
-void tcf_block_cb_incref(struct tcf_block_cb *block_cb)
-{
-}
-
-static inline
-unsigned int tcf_block_cb_decref(struct tcf_block_cb *block_cb)
-{
- return 0;
-}
-
-static inline
-struct tcf_block_cb *__tcf_block_cb_register(struct tcf_block *block,
- tc_setup_cb_t *cb, void *cb_ident,
- void *cb_priv,
- struct netlink_ext_ack *extack)
-{
- return NULL;
-}
-
-static inline
-int tcf_block_cb_register(struct tcf_block *block,
- tc_setup_cb_t *cb, void *cb_ident,
- void *cb_priv, struct netlink_ext_ack *extack)
-{
- return 0;
-}
-
-static inline
-void __tcf_block_cb_unregister(struct tcf_block *block,
- struct tcf_block_cb *block_cb)
-{
-}
-
-static inline
-void tcf_block_cb_unregister(struct tcf_block *block,
- tc_setup_cb_t *cb, void *cb_ident)
-{
-}
-
-static inline
int __tc_indr_block_cb_register(struct net_device *dev, void *cb_priv,
tc_indr_block_bind_cb_t *cb, void *cb_ident)
{
diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c
index cc87f6a2191f..ed568c534675 100644
--- a/net/sched/cls_api.c
+++ b/net/sched/cls_api.c
@@ -1512,43 +1512,6 @@ void tcf_block_put(struct tcf_block *block)
EXPORT_SYMBOL(tcf_block_put);
-struct tcf_block_cb {
- struct list_head list;
- tc_setup_cb_t *cb;
- void *cb_ident;
- void *cb_priv;
- unsigned int refcnt;
-};
-
-void *tcf_block_cb_priv(struct tcf_block_cb *block_cb)
-{
- return block_cb->cb_priv;
-}
-EXPORT_SYMBOL(tcf_block_cb_priv);
-
-struct tcf_block_cb *tcf_block_cb_lookup(struct tcf_block *block,
- tc_setup_cb_t *cb, void *cb_ident)
-{ struct tcf_block_cb *block_cb;
-
- list_for_each_entry(block_cb, &block->cb_list, list)
- if (block_cb->cb == cb && block_cb->cb_ident == cb_ident)
- return block_cb;
- return NULL;
-}
-EXPORT_SYMBOL(tcf_block_cb_lookup);
-
-void tcf_block_cb_incref(struct tcf_block_cb *block_cb)
-{
- block_cb->refcnt++;
-}
-EXPORT_SYMBOL(tcf_block_cb_incref);
-
-unsigned int tcf_block_cb_decref(struct tcf_block_cb *block_cb)
-{
- return --block_cb->refcnt;
-}
-EXPORT_SYMBOL(tcf_block_cb_decref);
-
static int
tcf_block_playback_offloads(struct tcf_block *block, tc_setup_cb_t *cb,
void *cb_priv, bool add, bool offload_in_use,
@@ -1590,67 +1553,6 @@ tcf_block_playback_offloads(struct tcf_block *block, tc_setup_cb_t *cb,
return err;
}
-struct tcf_block_cb *__tcf_block_cb_register(struct tcf_block *block,
- tc_setup_cb_t *cb, void *cb_ident,
- void *cb_priv,
- struct netlink_ext_ack *extack)
-{
- struct tcf_block_cb *block_cb;
- int err;
-
- /* Replay any already present rules */
- err = tcf_block_playback_offloads(block, cb, cb_priv, true,
- tcf_block_offload_in_use(block),
- extack);
- if (err)
- return ERR_PTR(err);
-
- block_cb = kzalloc(sizeof(*block_cb), GFP_KERNEL);
- if (!block_cb)
- return ERR_PTR(-ENOMEM);
- block_cb->cb = cb;
- block_cb->cb_ident = cb_ident;
- block_cb->cb_priv = cb_priv;
- list_add(&block_cb->list, &block->cb_list);
- return block_cb;
-}
-EXPORT_SYMBOL(__tcf_block_cb_register);
-
-int tcf_block_cb_register(struct tcf_block *block,
- tc_setup_cb_t *cb, void *cb_ident,
- void *cb_priv, struct netlink_ext_ack *extack)
-{
- struct tcf_block_cb *block_cb;
-
- block_cb = __tcf_block_cb_register(block, cb, cb_ident, cb_priv,
- extack);
- return PTR_ERR_OR_ZERO(block_cb);
-}
-EXPORT_SYMBOL(tcf_block_cb_register);
-
-void __tcf_block_cb_unregister(struct tcf_block *block,
- struct tcf_block_cb *block_cb)
-{
- tcf_block_playback_offloads(block, block_cb->cb, block_cb->cb_priv,
- false, tcf_block_offload_in_use(block),
- NULL);
- list_del(&block_cb->list);
- kfree(block_cb);
-}
-EXPORT_SYMBOL(__tcf_block_cb_unregister);
-
-void tcf_block_cb_unregister(struct tcf_block *block,
- tc_setup_cb_t *cb, void *cb_ident)
-{
- struct tcf_block_cb *block_cb;
-
- block_cb = tcf_block_cb_lookup(block, cb, cb_ident);
- if (!block_cb)
- return;
- __tcf_block_cb_unregister(block, block_cb);
-}
-EXPORT_SYMBOL(tcf_block_cb_unregister);
-
static int tcf_block_bind(struct tcf_block *block,
struct flow_block_offload *bo)
{
--
2.11.0
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox