* [net-next V5 PATCH 2/5] bpf: XDP_REDIRECT enable use of cpumap
From: Jesper Dangaard Brouer @ 2017-10-06 16:12 UTC (permalink / raw)
To: netdev
Cc: jakub.kicinski, Michael S. Tsirkin, pavel.odintsov, Jason Wang,
mchan, John Fastabend, peter.waskiewicz.jr,
Jesper Dangaard Brouer, Daniel Borkmann, Alexei Starovoitov,
Andy Gospodarek
In-Reply-To: <150730632837.22839.11804085686478888137.stgit@firesoul>
This patch connects cpumap to the xdp_do_redirect_map infrastructure.
Still no SKB allocation are done yet. The XDP frames are transferred
to the other CPU, but they are simply refcnt decremented on the remote
CPU. This served as a good benchmark for measuring the overhead of
remote refcnt decrement. If driver page recycle cache is not
efficient then this, exposes a bottleneck in the page allocator.
A shout-out to MST's ptr_ring, which is the secret behind is being so
efficient to transfer memory pointers between CPUs, without constantly
bouncing cache-lines between CPUs.
V3: Handle !CONFIG_BPF_SYSCALL pointed out by kbuild test robot.
V4: Make Generic-XDP aware of cpumap type, but don't allow redirect yet,
as implementation require a separate upstream discussion.
V5:
- Fix a maybe-uninitialized pointed out by kbuild test robot.
- Restrict bpf-prog side access to cpumap, open when use-cases appear
- Implement cpu_map_enqueue() as a more simple void pointer enqueue
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
---
include/linux/bpf.h | 31 +++++++++-
include/trace/events/xdp.h | 10 +++
kernel/bpf/cpumap.c | 22 +++++++
kernel/bpf/verifier.c | 8 ++-
net/core/filter.c | 140 +++++++++++++++++++++++++++++++++++---------
5 files changed, 177 insertions(+), 34 deletions(-)
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 252f4bc9eb25..36ec83343c82 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -320,6 +320,13 @@ struct net_device *__dev_map_lookup_elem(struct bpf_map *map, u32 key);
void __dev_map_insert_ctx(struct bpf_map *map, u32 index);
void __dev_map_flush(struct bpf_map *map);
+struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key);
+void __cpu_map_insert_ctx(struct bpf_map *map, u32 index);
+void __cpu_map_flush(struct bpf_map *map);
+struct xdp_buff;
+int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp,
+ struct net_device *dev_rx);
+
/* Return map's numa specified by userspace */
static inline int bpf_map_attr_numa_node(const union bpf_attr *attr)
{
@@ -327,7 +334,7 @@ static inline int bpf_map_attr_numa_node(const union bpf_attr *attr)
attr->numa_node : NUMA_NO_NODE;
}
-#else
+#else /* !CONFIG_BPF_SYSCALL */
static inline struct bpf_prog *bpf_prog_get(u32 ufd)
{
return ERR_PTR(-EOPNOTSUPP);
@@ -385,6 +392,28 @@ static inline void __dev_map_insert_ctx(struct bpf_map *map, u32 index)
static inline void __dev_map_flush(struct bpf_map *map)
{
}
+
+static inline
+struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key)
+{
+ return NULL;
+}
+
+static inline void __cpu_map_insert_ctx(struct bpf_map *map, u32 index)
+{
+}
+
+static inline void __cpu_map_flush(struct bpf_map *map)
+{
+}
+
+struct xdp_buff;
+static inline int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu,
+ struct xdp_buff *xdp,
+ struct net_device *dev_rx)
+{
+ return 0;
+}
#endif /* CONFIG_BPF_SYSCALL */
#if defined(CONFIG_STREAM_PARSER) && defined(CONFIG_BPF_SYSCALL)
diff --git a/include/trace/events/xdp.h b/include/trace/events/xdp.h
index 4e16c43fba10..eb2ece96c1a2 100644
--- a/include/trace/events/xdp.h
+++ b/include/trace/events/xdp.h
@@ -136,12 +136,18 @@ DEFINE_EVENT_PRINT(xdp_redirect_template, xdp_redirect_map_err,
__entry->map_id, __entry->map_index)
);
+#define devmap_ifindex(fwd, map) \
+ (!fwd ? 0 : \
+ (!map ? 0 : \
+ ((map->map_type == BPF_MAP_TYPE_DEVMAP) ? \
+ ((struct net_device *)fwd)->ifindex : 0)))
+
#define _trace_xdp_redirect_map(dev, xdp, fwd, map, idx) \
- trace_xdp_redirect_map(dev, xdp, fwd ? fwd->ifindex : 0, \
+ trace_xdp_redirect_map(dev, xdp, devmap_ifindex(fwd, map), \
0, map, idx)
#define _trace_xdp_redirect_map_err(dev, xdp, fwd, map, idx, err) \
- trace_xdp_redirect_map_err(dev, xdp, fwd ? fwd->ifindex : 0, \
+ trace_xdp_redirect_map_err(dev, xdp, devmap_ifindex(fwd, map), \
err, map, idx)
#endif /* _TRACE_XDP_H */
diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c
index d3e921620097..1499eae96d30 100644
--- a/kernel/bpf/cpumap.c
+++ b/kernel/bpf/cpumap.c
@@ -480,7 +480,7 @@ struct xdp_pkt {
/* Runs under RCU-read-side, plus in softirq under NAPI protection.
* Thus, safe percpu variable access.
*/
-int bq_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_pkt *xdp_pkt)
+static int bq_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_pkt *xdp_pkt)
{
struct xdp_bulk_queue *bq = this_cpu_ptr(rcpu->bulkq);
@@ -500,6 +500,26 @@ int bq_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_pkt *xdp_pkt)
return 0;
}
+int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp,
+ struct net_device *dev_rx)
+{
+ struct xdp_pkt *xdp_pkt;
+ int headroom;
+
+ /* For now this is just used as a void pointer to data_hard_start.
+ * Followup patch will generalize this.
+ */
+ xdp_pkt = xdp->data_hard_start;
+
+ /* Fake writing into xdp_pkt->data to measure overhead */
+ headroom = xdp->data - xdp->data_hard_start;
+ if (headroom < sizeof(*xdp_pkt))
+ xdp_pkt->data = xdp->data;
+
+ bq_enqueue(rcpu, xdp_pkt);
+ return 0;
+}
+
void __cpu_map_insert_ctx(struct bpf_map *map, u32 bit)
{
struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 4cf9b72c59a0..34e3ebbc221f 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -1572,6 +1572,11 @@ static int check_map_func_compatibility(struct bpf_map *map, int func_id)
if (func_id != BPF_FUNC_redirect_map)
goto error;
break;
+ /* Restrict bpf side of cpumap, open when use-cases appear */
+ case BPF_MAP_TYPE_CPUMAP:
+ if (func_id != BPF_FUNC_redirect_map)
+ goto error;
+ break;
case BPF_MAP_TYPE_ARRAY_OF_MAPS:
case BPF_MAP_TYPE_HASH_OF_MAPS:
if (func_id != BPF_FUNC_map_lookup_elem)
@@ -1608,7 +1613,8 @@ static int check_map_func_compatibility(struct bpf_map *map, int func_id)
goto error;
break;
case BPF_FUNC_redirect_map:
- if (map->map_type != BPF_MAP_TYPE_DEVMAP)
+ if (map->map_type != BPF_MAP_TYPE_DEVMAP &&
+ map->map_type != BPF_MAP_TYPE_CPUMAP)
goto error;
break;
case BPF_FUNC_sk_redirect_map:
diff --git a/net/core/filter.c b/net/core/filter.c
index 9b6e7e84aafd..c02e772309d5 100644
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -2521,10 +2521,36 @@ static int __bpf_tx_xdp(struct net_device *dev,
err = dev->netdev_ops->ndo_xdp_xmit(dev, xdp);
if (err)
return err;
- if (map)
+ dev->netdev_ops->ndo_xdp_flush(dev);
+ return 0;
+}
+
+static int __bpf_tx_xdp_map(struct net_device *dev_rx, void *fwd,
+ struct bpf_map *map,
+ struct xdp_buff *xdp,
+ u32 index)
+{
+ int err;
+
+ if (map->map_type == BPF_MAP_TYPE_DEVMAP) {
+ struct net_device *dev = fwd;
+
+ if (!dev->netdev_ops->ndo_xdp_xmit)
+ return -EOPNOTSUPP;
+
+ err = dev->netdev_ops->ndo_xdp_xmit(dev, xdp);
+ if (err)
+ return err;
__dev_map_insert_ctx(map, index);
- else
- dev->netdev_ops->ndo_xdp_flush(dev);
+
+ } else if (map->map_type == BPF_MAP_TYPE_CPUMAP) {
+ struct bpf_cpu_map_entry *rcpu = fwd;
+
+ err = cpu_map_enqueue(rcpu, xdp, dev_rx);
+ if (err)
+ return err;
+ __cpu_map_insert_ctx(map, index);
+ }
return 0;
}
@@ -2534,11 +2560,33 @@ void xdp_do_flush_map(void)
struct bpf_map *map = ri->map_to_flush;
ri->map_to_flush = NULL;
- if (map)
- __dev_map_flush(map);
+ if (map) {
+ switch (map->map_type) {
+ case BPF_MAP_TYPE_DEVMAP:
+ __dev_map_flush(map);
+ break;
+ case BPF_MAP_TYPE_CPUMAP:
+ __cpu_map_flush(map);
+ break;
+ default:
+ break;
+ }
+ }
}
EXPORT_SYMBOL_GPL(xdp_do_flush_map);
+static void *__xdp_map_lookup_elem(struct bpf_map *map, u32 index)
+{
+ switch (map->map_type) {
+ case BPF_MAP_TYPE_DEVMAP:
+ return __dev_map_lookup_elem(map, index);
+ case BPF_MAP_TYPE_CPUMAP:
+ return __cpu_map_lookup_elem(map, index);
+ default:
+ return NULL;
+ }
+}
+
static inline bool xdp_map_invalid(const struct bpf_prog *xdp_prog,
unsigned long aux)
{
@@ -2551,8 +2599,8 @@ static int xdp_do_redirect_map(struct net_device *dev, struct xdp_buff *xdp,
struct redirect_info *ri = this_cpu_ptr(&redirect_info);
unsigned long map_owner = ri->map_owner;
struct bpf_map *map = ri->map;
- struct net_device *fwd = NULL;
u32 index = ri->ifindex;
+ void *fwd = NULL;
int err;
ri->ifindex = 0;
@@ -2565,7 +2613,7 @@ static int xdp_do_redirect_map(struct net_device *dev, struct xdp_buff *xdp,
goto err;
}
- fwd = __dev_map_lookup_elem(map, index);
+ fwd = __xdp_map_lookup_elem(map, index);
if (!fwd) {
err = -EINVAL;
goto err;
@@ -2573,7 +2621,7 @@ static int xdp_do_redirect_map(struct net_device *dev, struct xdp_buff *xdp,
if (ri->map_to_flush && ri->map_to_flush != map)
xdp_do_flush_map();
- err = __bpf_tx_xdp(fwd, map, xdp, index);
+ err = __bpf_tx_xdp_map(dev, fwd, map, xdp, index);
if (unlikely(err))
goto err;
@@ -2615,54 +2663,88 @@ int xdp_do_redirect(struct net_device *dev, struct xdp_buff *xdp,
}
EXPORT_SYMBOL_GPL(xdp_do_redirect);
-int xdp_do_generic_redirect(struct net_device *dev, struct sk_buff *skb,
- struct bpf_prog *xdp_prog)
+static int __xdp_generic_ok_fwd_dev(struct sk_buff *skb, struct net_device *fwd)
+{
+ unsigned int len;
+
+ if (unlikely(!(fwd->flags & IFF_UP)))
+ return -ENETDOWN;
+
+ len = fwd->mtu + fwd->hard_header_len + VLAN_HLEN;
+ if (skb->len > len)
+ return -EMSGSIZE;
+
+ return 0;
+}
+
+int xdp_do_generic_redirect_map(struct net_device *dev, struct sk_buff *skb,
+ struct bpf_prog *xdp_prog)
{
struct redirect_info *ri = this_cpu_ptr(&redirect_info);
unsigned long map_owner = ri->map_owner;
struct bpf_map *map = ri->map;
struct net_device *fwd = NULL;
u32 index = ri->ifindex;
- unsigned int len;
int err = 0;
ri->ifindex = 0;
ri->map = NULL;
ri->map_owner = 0;
- if (map) {
- if (unlikely(xdp_map_invalid(xdp_prog, map_owner))) {
- err = -EFAULT;
- map = NULL;
- goto err;
- }
- fwd = __dev_map_lookup_elem(map, index);
- } else {
- fwd = dev_get_by_index_rcu(dev_net(dev), index);
+ if (unlikely(xdp_map_invalid(xdp_prog, map_owner))) {
+ err = -EFAULT;
+ map = NULL;
+ goto err;
}
+ fwd = __xdp_map_lookup_elem(map, index);
if (unlikely(!fwd)) {
err = -EINVAL;
goto err;
}
- if (unlikely(!(fwd->flags & IFF_UP))) {
- err = -ENETDOWN;
+ if (map->map_type == BPF_MAP_TYPE_DEVMAP) {
+ if (unlikely((err = __xdp_generic_ok_fwd_dev(skb, fwd))))
+ goto err;
+ skb->dev = fwd;
+ } else {
+ /* TODO: Handle BPF_MAP_TYPE_CPUMAP */
+ err = -EBADRQC;
goto err;
}
- len = fwd->mtu + fwd->hard_header_len + VLAN_HLEN;
- if (skb->len > len) {
- err = -EMSGSIZE;
+ _trace_xdp_redirect_map(dev, xdp_prog, fwd, map, index);
+ return 0;
+err:
+ _trace_xdp_redirect_map_err(dev, xdp_prog, fwd, map, index, err);
+ return err;
+}
+
+int xdp_do_generic_redirect(struct net_device *dev, struct sk_buff *skb,
+ struct bpf_prog *xdp_prog)
+{
+ struct redirect_info *ri = this_cpu_ptr(&redirect_info);
+ u32 index = ri->ifindex;
+ struct net_device *fwd;
+ int err = 0;
+
+ if (ri->map)
+ return xdp_do_generic_redirect_map(dev, skb, xdp_prog);
+
+ ri->ifindex = 0;
+ fwd = dev_get_by_index_rcu(dev_net(dev), index);
+ if (unlikely(!fwd)) {
+ err = -EINVAL;
goto err;
}
+ if (unlikely((err = __xdp_generic_ok_fwd_dev(skb, fwd))))
+ goto err;
+
skb->dev = fwd;
- map ? _trace_xdp_redirect_map(dev, xdp_prog, fwd, map, index)
- : _trace_xdp_redirect(dev, xdp_prog, index);
+ _trace_xdp_redirect(dev, xdp_prog, index);
return 0;
err:
- map ? _trace_xdp_redirect_map_err(dev, xdp_prog, fwd, map, index, err)
- : _trace_xdp_redirect_err(dev, xdp_prog, index, err);
+ _trace_xdp_redirect_err(dev, xdp_prog, index, err);
return err;
}
EXPORT_SYMBOL_GPL(xdp_do_generic_redirect);
^ permalink raw reply related
* [net-next V5 PATCH 1/5] bpf: introduce new bpf cpu map type BPF_MAP_TYPE_CPUMAP
From: Jesper Dangaard Brouer @ 2017-10-06 16:12 UTC (permalink / raw)
To: netdev
Cc: jakub.kicinski, Michael S. Tsirkin, pavel.odintsov, Jason Wang,
mchan, John Fastabend, peter.waskiewicz.jr,
Jesper Dangaard Brouer, Daniel Borkmann, Alexei Starovoitov,
Andy Gospodarek
In-Reply-To: <150730632837.22839.11804085686478888137.stgit@firesoul>
The 'cpumap' is primary used as a backend map for XDP BPF helper
call bpf_redirect_map() and XDP_REDIRECT action, like 'devmap'.
This patch implement the main part of the map. It is not connected to
the XDP redirect system yet, and no SKB allocation are done yet.
The main concern in this patch is to ensure the datapath can run
without any locking. This adds complexity to the setup and tear-down
procedure, which assumptions are extra carefully documented in the
code comments.
V2:
- make sure array isn't larger than NR_CPUS
- make sure CPUs added is a valid possible CPU
V3: fix nitpicks from Jakub Kicinski <kubakici@wp.pl>
V5:
- Restrict map allocation to root / CAP_SYS_ADMIN
- WARN_ON_ONCE if queue is not empty on tear-down
- Return -EPERM on memlock limit instead of -ENOMEM
- Error code in __cpu_map_entry_alloc() also handle ptr_ring_cleanup()
- Moved cpu_map_enqueue() to next patch
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
---
include/linux/bpf_types.h | 1
include/uapi/linux/bpf.h | 1
kernel/bpf/Makefile | 1
kernel/bpf/cpumap.c | 540 ++++++++++++++++++++++++++++++++++++++++
kernel/bpf/syscall.c | 8 +
tools/include/uapi/linux/bpf.h | 1
6 files changed, 551 insertions(+), 1 deletion(-)
create mode 100644 kernel/bpf/cpumap.c
diff --git a/include/linux/bpf_types.h b/include/linux/bpf_types.h
index 6f1a567667b8..814c1081a4a9 100644
--- a/include/linux/bpf_types.h
+++ b/include/linux/bpf_types.h
@@ -41,4 +41,5 @@ BPF_MAP_TYPE(BPF_MAP_TYPE_DEVMAP, dev_map_ops)
#ifdef CONFIG_STREAM_PARSER
BPF_MAP_TYPE(BPF_MAP_TYPE_SOCKMAP, sock_map_ops)
#endif
+BPF_MAP_TYPE(BPF_MAP_TYPE_CPUMAP, cpu_map_ops)
#endif
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 6d2137b4cf38..03f8e2827a95 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -111,6 +111,7 @@ enum bpf_map_type {
BPF_MAP_TYPE_HASH_OF_MAPS,
BPF_MAP_TYPE_DEVMAP,
BPF_MAP_TYPE_SOCKMAP,
+ BPF_MAP_TYPE_CPUMAP,
};
enum bpf_prog_type {
diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile
index 897daa005b23..dba0bd33a43c 100644
--- a/kernel/bpf/Makefile
+++ b/kernel/bpf/Makefile
@@ -4,6 +4,7 @@ obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o
obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o
ifeq ($(CONFIG_NET),y)
obj-$(CONFIG_BPF_SYSCALL) += devmap.o
+obj-$(CONFIG_BPF_SYSCALL) += cpumap.o
ifeq ($(CONFIG_STREAM_PARSER),y)
obj-$(CONFIG_BPF_SYSCALL) += sockmap.o
endif
diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c
new file mode 100644
index 000000000000..d3e921620097
--- /dev/null
+++ b/kernel/bpf/cpumap.c
@@ -0,0 +1,540 @@
+/* bpf/cpumap.c
+ *
+ * Copyright (c) 2017 Jesper Dangaard Brouer, Red Hat Inc.
+ * Released under terms in GPL version 2. See COPYING.
+ */
+
+/* The 'cpumap' is primary used as a backend map for XDP BPF helper
+ * call bpf_redirect_map() and XDP_REDIRECT action, like 'devmap'.
+ *
+ * Unlike devmap which redirect XDP frames out another NIC device,
+ * this map type redirect raw XDP frames to another CPU. The remote
+ * CPU will do SKB-allocation and call the normal network stack.
+ *
+ * This is a scalability and isolation mechanism, that allow
+ * separating the early driver network XDP layer, from the rest of the
+ * netstack, and assigning dedicated CPUs for this stage. This
+ * basically allows for 10G wirespeed pre-filtering via bpf.
+ */
+#include <linux/bpf.h>
+#include <linux/filter.h>
+#include <linux/ptr_ring.h>
+
+#include <linux/sched.h>
+#include <linux/workqueue.h>
+#include <linux/kthread.h>
+#include <linux/capability.h>
+
+/* General idea: XDP packets getting XDP redirected to another CPU,
+ * will maximum be stored/queued for one driver ->poll() call. It is
+ * guaranteed that setting flush bit and flush operation happen on
+ * same CPU. Thus, cpu_map_flush operation can deduct via this_cpu_ptr()
+ * which queue in bpf_cpu_map_entry contains packets.
+ */
+
+#define CPU_MAP_BULK_SIZE 8 /* 8 == one cacheline on 64-bit archs */
+struct xdp_bulk_queue {
+ void *q[CPU_MAP_BULK_SIZE];
+ unsigned int count;
+};
+
+/* Struct for every remote "destination" CPU in map */
+struct bpf_cpu_map_entry {
+ u32 cpu; /* kthread CPU and map index */
+ int map_id; /* Back reference to map */
+ u32 qsize; /* Redundant queue size for map lookup */
+
+ /* XDP can run multiple RX-ring queues, need __percpu enqueue store */
+ struct xdp_bulk_queue __percpu *bulkq;
+
+ /* Queue with potential multi-producers, and single-consumer kthread */
+ struct ptr_ring *queue;
+ struct task_struct *kthread;
+ struct work_struct kthread_stop_wq;
+
+ atomic_t refcnt; /* Control when this struct can be free'ed */
+ struct rcu_head rcu;
+};
+
+struct bpf_cpu_map {
+ struct bpf_map map;
+ /* Below members specific for map type */
+ struct bpf_cpu_map_entry **cpu_map;
+ unsigned long __percpu *flush_needed;
+};
+
+static int bq_flush_to_queue(struct bpf_cpu_map_entry *rcpu,
+ struct xdp_bulk_queue *bq);
+
+static u64 cpu_map_bitmap_size(const union bpf_attr *attr)
+{
+ return BITS_TO_LONGS(attr->max_entries) * sizeof(unsigned long);
+}
+
+static struct bpf_map *cpu_map_alloc(union bpf_attr *attr)
+{
+ struct bpf_cpu_map *cmap;
+ int err = -ENOMEM;
+ u64 cost;
+
+ if (!capable(CAP_SYS_ADMIN))
+ return ERR_PTR(-EPERM);
+
+ /* check sanity of attributes */
+ if (attr->max_entries == 0 || attr->key_size != 4 ||
+ attr->value_size != 4 || attr->map_flags & ~BPF_F_NUMA_NODE)
+ return ERR_PTR(-EINVAL);
+
+ cmap = kzalloc(sizeof(*cmap), GFP_USER);
+ if (!cmap)
+ return ERR_PTR(-ENOMEM);
+
+ /* mandatory map attributes */
+ cmap->map.map_type = attr->map_type;
+ cmap->map.key_size = attr->key_size;
+ cmap->map.value_size = attr->value_size;
+ cmap->map.max_entries = attr->max_entries;
+ cmap->map.map_flags = attr->map_flags;
+ cmap->map.numa_node = bpf_map_attr_numa_node(attr);
+
+ /* Pre-limit array size based on NR_CPUS, not final CPU check */
+ if (cmap->map.max_entries > NR_CPUS)
+ return ERR_PTR(-E2BIG);
+
+ /* make sure page count doesn't overflow */
+ cost = (u64) cmap->map.max_entries * sizeof(struct bpf_cpu_map_entry *);
+ cost += cpu_map_bitmap_size(attr) * num_possible_cpus();
+ if (cost >= U32_MAX - PAGE_SIZE)
+ goto free_cmap;
+ cmap->map.pages = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT;
+
+ /* Notice returns -EPERM on if map size is larger than memlock limit */
+ err = bpf_map_precharge_memlock(cmap->map.pages);
+ if (err)
+ goto free_cmap;
+
+ /* A per cpu bitfield with a bit per possible CPU in map */
+ cmap->flush_needed = __alloc_percpu(cpu_map_bitmap_size(attr),
+ __alignof__(unsigned long));
+ if (!cmap->flush_needed)
+ goto free_cmap;
+
+ /* Alloc array for possible remote "destination" CPUs */
+ cmap->cpu_map = bpf_map_area_alloc(cmap->map.max_entries *
+ sizeof(struct bpf_cpu_map_entry *),
+ cmap->map.numa_node);
+ if (!cmap->cpu_map)
+ goto free_cmap;
+
+ return &cmap->map;
+free_cmap:
+ free_percpu(cmap->flush_needed);
+ kfree(cmap);
+ return ERR_PTR(err);
+}
+
+void __cpu_map_queue_destructor(void *ptr)
+{
+ /* The tear-down procedure should have made sure that queue is
+ * empty. See __cpu_map_entry_replace() and work-queue
+ * invoked cpu_map_kthread_stop(). Catch any broken behaviour
+ * gracefully and warn once.
+ */
+ if (WARN_ON_ONCE(ptr))
+ page_frag_free(ptr);
+}
+
+static void put_cpu_map_entry(struct bpf_cpu_map_entry *rcpu)
+{
+ if (atomic_dec_and_test(&rcpu->refcnt)) {
+ /* The queue should be empty at this point */
+ ptr_ring_cleanup(rcpu->queue, __cpu_map_queue_destructor);
+ kfree(rcpu->queue);
+ kfree(rcpu);
+ }
+}
+
+static void get_cpu_map_entry(struct bpf_cpu_map_entry *rcpu)
+{
+ atomic_inc(&rcpu->refcnt);
+}
+
+/* called from workqueue, to workaround syscall using preempt_disable */
+static void cpu_map_kthread_stop(struct work_struct *work)
+{
+ struct bpf_cpu_map_entry *rcpu;
+
+ rcpu = container_of(work, struct bpf_cpu_map_entry, kthread_stop_wq);
+ synchronize_rcu(); /* wait for flush in __cpu_map_entry_free() */
+ kthread_stop(rcpu->kthread); /* calls put_cpu_map_entry */
+}
+
+static int cpu_map_kthread_run(void *data)
+{
+ struct bpf_cpu_map_entry *rcpu = data;
+
+ set_current_state(TASK_INTERRUPTIBLE);
+ while (!kthread_should_stop()) {
+ struct xdp_pkt *xdp_pkt;
+
+ schedule();
+ /* Do work */
+ while ((xdp_pkt = ptr_ring_consume(rcpu->queue))) {
+ /* For now just "refcnt-free" */
+ page_frag_free(xdp_pkt);
+ }
+ __set_current_state(TASK_INTERRUPTIBLE);
+ }
+ put_cpu_map_entry(rcpu);
+
+ __set_current_state(TASK_RUNNING);
+ return 0;
+}
+
+struct bpf_cpu_map_entry *__cpu_map_entry_alloc(u32 qsize, u32 cpu, int map_id)
+{
+ gfp_t gfp = GFP_ATOMIC|__GFP_NOWARN;
+ struct bpf_cpu_map_entry *rcpu;
+ int numa, err;
+
+ /* Have map->numa_node, but choose node of redirect target CPU */
+ numa = cpu_to_node(cpu);
+
+ rcpu = kzalloc_node(sizeof(*rcpu), gfp, numa);
+ if (!rcpu)
+ return NULL;
+
+ /* Alloc percpu bulkq */
+ rcpu->bulkq = __alloc_percpu_gfp(sizeof(*rcpu->bulkq),
+ sizeof(void *), gfp);
+ if (!rcpu->bulkq)
+ goto fail;
+
+ /* Alloc queue */
+ rcpu->queue = kzalloc_node(sizeof(*rcpu->queue), gfp, numa);
+ if (!rcpu->queue)
+ goto fail;
+
+ err = ptr_ring_init(rcpu->queue, qsize, gfp);
+ if (err)
+ goto fail;
+ rcpu->qsize = qsize;
+
+ /* Setup kthread */
+ rcpu->kthread = kthread_create_on_node(cpu_map_kthread_run, rcpu, numa,
+ "cpumap/%d/map:%d", cpu, map_id);
+ if (IS_ERR(rcpu->kthread))
+ goto fail;
+
+ /* Make sure kthread runs on a single CPU */
+ kthread_bind(rcpu->kthread, cpu);
+ wake_up_process(rcpu->kthread);
+
+ get_cpu_map_entry(rcpu); /* 1-refcnt for being in cmap->cpu_map[] */
+ get_cpu_map_entry(rcpu); /* 1-refcnt for kthread */
+
+ return rcpu;
+
+fail: /* Hint: free API detect NULL values */
+ free_percpu(rcpu->bulkq);
+ ptr_ring_cleanup(rcpu->queue, NULL);
+ kfree(rcpu->queue);
+ kfree(rcpu);
+ return NULL;
+}
+
+void __cpu_map_entry_free(struct rcu_head *rcu)
+{
+ struct bpf_cpu_map_entry *rcpu;
+ int cpu;
+
+ /* This cpu_map_entry have been disconnected from map and one
+ * RCU graze-period have elapsed. Thus, XDP cannot queue any
+ * new packets and cannot change/set flush_needed that can
+ * find this entry.
+ */
+ rcpu = container_of(rcu, struct bpf_cpu_map_entry, rcu);
+
+ /* Flush remaining packets in percpu bulkq */
+ for_each_online_cpu(cpu) {
+ struct xdp_bulk_queue *bq = per_cpu_ptr(rcpu->bulkq, cpu);
+
+ /* No concurrent bq_enqueue can run at this point */
+ bq_flush_to_queue(rcpu, bq);
+ }
+ free_percpu(rcpu->bulkq);
+ /* Cannot kthread_stop() here, last put free rcpu resources */
+ put_cpu_map_entry(rcpu);
+}
+
+/* After xchg pointer to bpf_cpu_map_entry, use the call_rcu() to
+ * ensure any driver rcu critical sections have completed, but this
+ * does not guarantee a flush has happened yet. Because driver side
+ * rcu_read_lock/unlock only protects the running XDP program. The
+ * atomic xchg and NULL-ptr check in __cpu_map_flush() makes sure a
+ * pending flush op doesn't fail.
+ *
+ * The bpf_cpu_map_entry is still used by the kthread, and there can
+ * still be pending packets (in queue and percpu bulkq). A refcnt
+ * makes sure to last user (kthread_stop vs. call_rcu) free memory
+ * resources.
+ *
+ * The rcu callback __cpu_map_entry_free flush remaining packets in
+ * percpu bulkq to queue. Due to caller map_delete_elem() disable
+ * preemption, cannot call kthread_stop() to make sure queue is empty.
+ * Instead a work_queue is started for stopping kthread,
+ * cpu_map_kthread_stop, which waits for an RCU graze period before
+ * stopping kthread, emptying the queue.
+ */
+void __cpu_map_entry_replace(struct bpf_cpu_map *cmap,
+ u32 key_cpu, struct bpf_cpu_map_entry *rcpu)
+{
+ struct bpf_cpu_map_entry *old_rcpu;
+
+ old_rcpu = xchg(&cmap->cpu_map[key_cpu], rcpu);
+ if (old_rcpu) {
+ call_rcu(&old_rcpu->rcu, __cpu_map_entry_free);
+ INIT_WORK(&old_rcpu->kthread_stop_wq, cpu_map_kthread_stop);
+ schedule_work(&old_rcpu->kthread_stop_wq);
+ }
+}
+
+int cpu_map_delete_elem(struct bpf_map *map, void *key)
+{
+ struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
+ u32 key_cpu = *(u32 *)key;
+
+ if (key_cpu >= map->max_entries)
+ return -EINVAL;
+
+ /* notice caller map_delete_elem() use preempt_disable() */
+ __cpu_map_entry_replace(cmap, key_cpu, NULL);
+ return 0;
+}
+
+int cpu_map_update_elem(struct bpf_map *map, void *key, void *value,
+ u64 map_flags)
+{
+ struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
+ struct bpf_cpu_map_entry *rcpu;
+
+ /* Array index key correspond to CPU number */
+ u32 key_cpu = *(u32 *)key;
+ /* Value is the queue size */
+ u32 qsize = *(u32 *)value;
+
+ /* Make sure CPU is a valid possible cpu */
+ if (!cpu_possible(key_cpu))
+ return -ENODEV;
+
+ if (unlikely(map_flags > BPF_EXIST))
+ return -EINVAL;
+ if (unlikely(key_cpu >= cmap->map.max_entries))
+ return -E2BIG;
+ if (unlikely(map_flags == BPF_NOEXIST))
+ return -EEXIST;
+ if (unlikely(qsize > 16384)) /* sanity limit on qsize */
+ return -EOVERFLOW;
+
+ if (qsize == 0) {
+ rcpu = NULL; /* Same as deleting */
+ } else {
+ /* Updating qsize cause re-allocation of bpf_cpu_map_entry */
+ rcpu = __cpu_map_entry_alloc(qsize, key_cpu, map->id);
+ if (!rcpu)
+ return -ENOMEM;
+ }
+ rcu_read_lock();
+ __cpu_map_entry_replace(cmap, key_cpu, rcpu);
+ rcu_read_unlock();
+ return 0;
+}
+
+void cpu_map_free(struct bpf_map *map)
+{
+ struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
+ int cpu;
+ u32 i;
+
+ /* At this point bpf_prog->aux->refcnt == 0 and this map->refcnt == 0,
+ * so the bpf programs (can be more than one that used this map) were
+ * disconnected from events. Wait for outstanding critical sections in
+ * these programs to complete. The rcu critical section only guarantees
+ * no further "XDP/bpf-side" reads against bpf_cpu_map->cpu_map.
+ * It does __not__ ensure pending flush operations (if any) are
+ * complete.
+ */
+ synchronize_rcu();
+
+ /* To ensure all pending flush operations have completed wait for flush
+ * bitmap to indicate all flush_needed bits to be zero on _all_ cpus.
+ * Because the above synchronize_rcu() ensures the map is disconnected
+ * from the program we can assume no new bits will be set.
+ */
+ for_each_online_cpu(cpu) {
+ unsigned long *bitmap = per_cpu_ptr(cmap->flush_needed, cpu);
+
+ while (!bitmap_empty(bitmap, cmap->map.max_entries))
+ cond_resched();
+ }
+
+ /* For cpu_map the remote CPUs can still be using the entries
+ * (struct bpf_cpu_map_entry).
+ */
+ for (i = 0; i < cmap->map.max_entries; i++) {
+ struct bpf_cpu_map_entry *rcpu;
+
+ rcpu = READ_ONCE(cmap->cpu_map[i]);
+ if (!rcpu)
+ continue;
+
+ /* bq flush and cleanup happens after RCU graze-period */
+ __cpu_map_entry_replace(cmap, i, NULL); /* call_rcu */
+ }
+ free_percpu(cmap->flush_needed);
+ bpf_map_area_free(cmap->cpu_map);
+ kfree(cmap);
+}
+
+struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key)
+{
+ struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
+ struct bpf_cpu_map_entry *rcpu;
+
+ if (key >= map->max_entries)
+ return NULL;
+
+ rcpu = READ_ONCE(cmap->cpu_map[key]);
+ return rcpu;
+}
+
+static void *cpu_map_lookup_elem(struct bpf_map *map, void *key)
+{
+ struct bpf_cpu_map_entry *rcpu =
+ __cpu_map_lookup_elem(map, *(u32 *)key);
+
+ return rcpu ? &rcpu->qsize : NULL;
+}
+
+static int cpu_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
+{
+ struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
+ u32 index = key ? *(u32 *)key : U32_MAX;
+ u32 *next = next_key;
+
+ if (index >= cmap->map.max_entries) {
+ *next = 0;
+ return 0;
+ }
+
+ if (index == cmap->map.max_entries - 1)
+ return -ENOENT;
+ *next = index + 1;
+ return 0;
+}
+
+const struct bpf_map_ops cpu_map_ops = {
+ .map_alloc = cpu_map_alloc,
+ .map_free = cpu_map_free,
+ .map_delete_elem = cpu_map_delete_elem,
+ .map_update_elem = cpu_map_update_elem,
+ .map_lookup_elem = cpu_map_lookup_elem,
+ .map_get_next_key = cpu_map_get_next_key,
+};
+
+static int bq_flush_to_queue(struct bpf_cpu_map_entry *rcpu,
+ struct xdp_bulk_queue *bq)
+{
+ struct ptr_ring *q;
+ int i;
+
+ if (unlikely(!bq->count))
+ return 0;
+
+ q = rcpu->queue;
+ spin_lock(&q->producer_lock);
+
+ for (i = 0; i < bq->count; i++) {
+ void *xdp_pkt = bq->q[i];
+ int err;
+
+ err = __ptr_ring_produce(q, xdp_pkt);
+ if (err) {
+ /* Free xdp_pkt */
+ page_frag_free(xdp_pkt);
+ }
+ }
+ bq->count = 0;
+ spin_unlock(&q->producer_lock);
+
+ return 0;
+}
+
+/* Notice: Will change in later patch */
+struct xdp_pkt {
+ void *data;
+ u16 len;
+ u16 headroom;
+};
+
+/* Runs under RCU-read-side, plus in softirq under NAPI protection.
+ * Thus, safe percpu variable access.
+ */
+int bq_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_pkt *xdp_pkt)
+{
+ struct xdp_bulk_queue *bq = this_cpu_ptr(rcpu->bulkq);
+
+ if (unlikely(bq->count == CPU_MAP_BULK_SIZE))
+ bq_flush_to_queue(rcpu, bq);
+
+ /* Notice, xdp_buff/page MUST be queued here, long enough for
+ * driver to code invoking us to finished, due to driver
+ * (e.g. ixgbe) recycle tricks based on page-refcnt.
+ *
+ * Thus, incoming xdp_pkt is always queued here (else we race
+ * with another CPU on page-refcnt and remaining driver code).
+ * Queue time is very short, as driver will invoke flush
+ * operation, when completing napi->poll call.
+ */
+ bq->q[bq->count++] = xdp_pkt;
+ return 0;
+}
+
+void __cpu_map_insert_ctx(struct bpf_map *map, u32 bit)
+{
+ struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
+ unsigned long *bitmap = this_cpu_ptr(cmap->flush_needed);
+
+ __set_bit(bit, bitmap);
+}
+
+void __cpu_map_flush(struct bpf_map *map)
+{
+ struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
+ unsigned long *bitmap = this_cpu_ptr(cmap->flush_needed);
+ u32 bit;
+
+ /* The napi->poll softirq makes sure __cpu_map_insert_ctx()
+ * and __cpu_map_flush() happen on same CPU. Thus, the percpu
+ * bitmap indicate which percpu bulkq have packets.
+ */
+ for_each_set_bit(bit, bitmap, map->max_entries) {
+ struct bpf_cpu_map_entry *rcpu = READ_ONCE(cmap->cpu_map[bit]);
+ struct xdp_bulk_queue *bq;
+
+ /* This is possible if entry is removed by user space
+ * between xdp redirect and flush op.
+ */
+ if (unlikely(!rcpu))
+ continue;
+
+ __clear_bit(bit, bitmap);
+
+ /* Flush all frames in bulkq to real queue */
+ bq = this_cpu_ptr(rcpu->bulkq);
+ bq_flush_to_queue(rcpu, bq);
+
+ /* If already running, costs spin_lock_irqsave + smb_mb */
+ wake_up_process(rcpu->kthread);
+ }
+}
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index b927da66f653..641bdb0df020 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -593,6 +593,12 @@ static int map_update_elem(union bpf_attr *attr)
if (copy_from_user(value, uvalue, value_size) != 0)
goto free_value;
+ /* Need to create a kthread, thus must support schedule */
+ if (map->map_type == BPF_MAP_TYPE_CPUMAP) {
+ err = map->ops->map_update_elem(map, key, value, attr->flags);
+ goto out;
+ }
+
/* must increment bpf_prog_active to avoid kprobe+bpf triggering from
* inside bpf map update or delete otherwise deadlocks are possible
*/
@@ -623,7 +629,7 @@ static int map_update_elem(union bpf_attr *attr)
}
__this_cpu_dec(bpf_prog_active);
preempt_enable();
-
+out:
if (!err)
trace_bpf_map_update_elem(map, ufd, key, value);
free_value:
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index 6d2137b4cf38..03f8e2827a95 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -111,6 +111,7 @@ enum bpf_map_type {
BPF_MAP_TYPE_HASH_OF_MAPS,
BPF_MAP_TYPE_DEVMAP,
BPF_MAP_TYPE_SOCKMAP,
+ BPF_MAP_TYPE_CPUMAP,
};
enum bpf_prog_type {
^ permalink raw reply related
* [net-next V5 PATCH 0/5] New bpf cpumap type for XDP_REDIRECT
From: Jesper Dangaard Brouer @ 2017-10-06 16:12 UTC (permalink / raw)
To: netdev
Cc: jakub.kicinski, Michael S. Tsirkin, pavel.odintsov, Jason Wang,
mchan, John Fastabend, peter.waskiewicz.jr,
Jesper Dangaard Brouer, Daniel Borkmann, Alexei Starovoitov,
Andy Gospodarek
Introducing a new way to redirect XDP frames. Notice how no driver
changes are necessary given the design of XDP_REDIRECT.
This redirect map type is called 'cpumap', as it allows redirection
XDP frames to remote CPUs. The remote CPU will do the SKB allocation
and start the network stack invocation on that CPU.
This is a scalability and isolation mechanism, that allow separating
the early driver network XDP layer, from the rest of the netstack, and
assigning dedicated CPUs for this stage. The sysadm control/configure
the RX-CPU to NIC-RX queue (as usual) via procfs smp_affinity and how
many queues are configured via ethtool --set-channels. Benchmarks
show that a single CPU can handle approx 11Mpps. Thus, only assigning
two NIC RX-queues (and two CPUs) is sufficient for handling 10Gbit/s
wirespeed smallest packet 14.88Mpps. Reducing the number of queues
have the advantage that more packets being "bulk" available per hard
interrupt[1].
[1] https://www.netdevconf.org/2.1/papers/BusyPollingNextGen.pdf
Use-cases:
1. End-host based pre-filtering for DDoS mitigation. This is fast
enough to allow software to see and filter all packets wirespeed.
Thus, no packets getting silently dropped by hardware.
2. Given NIC HW unevenly distributes packets across RX queue, this
mechanism can be used for redistribution load across CPUs. This
usually happens when HW is unaware of a new protocol. This
resembles RPS (Receive Packet Steering), just faster, but with more
responsibility placed on the BPF program for correct steering.
3. Auto-scaling or power saving via only activating the appropriate
number of remote CPUs for handling the current load. The cpumap
tracepoints can function as a feedback loop for this purpose.
See individual patches for patchset-version changes.
Patchset V5 based on net-next at:
cc71b7b07119 ("net/ipv6: remove unused err variable on icmpv6_push_pending_frames")
---
Jesper Dangaard Brouer (5):
bpf: introduce new bpf cpu map type BPF_MAP_TYPE_CPUMAP
bpf: XDP_REDIRECT enable use of cpumap
bpf: cpumap xdp_buff to skb conversion and allocation
bpf: cpumap add tracepoints
samples/bpf: add cpumap sample program xdp_redirect_cpu
include/linux/bpf.h | 31 ++
include/linux/bpf_types.h | 1
include/linux/netdevice.h | 1
include/trace/events/xdp.h | 80 ++++
include/uapi/linux/bpf.h | 1
kernel/bpf/Makefile | 1
kernel/bpf/cpumap.c | 681 +++++++++++++++++++++++++++++++++++
kernel/bpf/syscall.c | 8
kernel/bpf/verifier.c | 8
net/core/dev.c | 27 +
net/core/filter.c | 140 ++++++-
samples/bpf/Makefile | 4
samples/bpf/xdp_redirect_cpu_kern.c | 609 +++++++++++++++++++++++++++++++
samples/bpf/xdp_redirect_cpu_user.c | 673 +++++++++++++++++++++++++++++++++++
tools/include/uapi/linux/bpf.h | 1
15 files changed, 2232 insertions(+), 34 deletions(-)
create mode 100644 kernel/bpf/cpumap.c
create mode 100644 samples/bpf/xdp_redirect_cpu_kern.c
create mode 100644 samples/bpf/xdp_redirect_cpu_user.c
^ permalink raw reply
* [PATCH] netfilter: xt_bpf: Fix XT_BPF_MODE_FD_PINNED mode of 'xt_bpf_info_v1'
From: Shmulik Ladkani @ 2017-10-06 16:02 UTC (permalink / raw)
To: Pablo Neira Ayuso, netfilter-devel
Cc: Willem de Bruijn, netdev, Daniel Borkmann, shmulik,
Rafael Buchbinder, Shmulik Ladkani
From: Shmulik Ladkani <shmulik.ladkani@gmail.com>
Commit 2c16d6033264 ("netfilter: xt_bpf: support ebpf") introduced
support for attaching an eBPF object by an fd, with the
'bpf_mt_check_v1' ABI expecting the '.fd' to be specified upon each
IPT_SO_SET_REPLACE call.
However this breaks subsequent iptables calls:
# iptables -A INPUT -m bpf --object-pinned /sys/fs/bpf/xxx -j ACCEPT
# iptables -A INPUT -s 5.6.7.8 -j ACCEPT
iptables: Invalid argument. Run `dmesg' for more information.
That's because iptables works by loading exising rules using
IPT_SO_GET_ENTRIES to userspace, then issuing IPT_SO_SET_REPLACE with
the replacement set.
However, the loaded 'xt_bpf_info_v1' has an arbitrary '.fd' number
(from the initial "iptables -m bpf" invocation) - so when 2nd invocation
occurs, userspace passes a bogus fd number, which leads to
'bpf_mt_check_v1' to fail.
One suggested solution [1] was to hack iptables userspace, to perform a
"entries fixup" immediatley after IPT_SO_GET_ENTRIES, by opening a new,
process-local fd per every 'xt_bpf_info_v1' entry seen.
However, in [2] both Pablo Neira Ayuso and Willem de Bruijn suggested to
depricate the xt_bpf_info_v1 ABI dealing with pinned ebpf objects.
This fix changes the XT_BPF_MODE_FD_PINNED behavior to ignore the given
'.fd' and instead perform an in-kernel lookup for the bpf object given
the provided '.path'.
It also defines an alias for the XT_BPF_MODE_FD_PINNED mode, named
XT_BPF_MODE_PATH_PINNED, to better reflect the fact that the user is
expected to provide the path of the pinned object.
Existing XT_BPF_MODE_FD_ELF behavior (non-pinned fd mode) is preserved.
References: [1] https://marc.info/?l=netfilter-devel&m=150564724607440&w=2
[2] https://marc.info/?l=netfilter-devel&m=150575727129880&w=2
Cc: Pablo Neira Ayuso <pablo@netfilter.org>
Cc: Willem de Bruijn <willemb@google.com>
Reported-by: Rafael Buchbinder <rafi@rbk.ms>
Signed-off-by: Shmulik Ladkani <shmulik.ladkani@gmail.com>
---
include/uapi/linux/netfilter/xt_bpf.h | 1 +
kernel/bpf/inode.c | 1 +
net/netfilter/xt_bpf.c | 22 ++++++++++++++++++++--
3 files changed, 22 insertions(+), 2 deletions(-)
diff --git a/include/uapi/linux/netfilter/xt_bpf.h b/include/uapi/linux/netfilter/xt_bpf.h
index b97725af2ac0..da161b56c79e 100644
--- a/include/uapi/linux/netfilter/xt_bpf.h
+++ b/include/uapi/linux/netfilter/xt_bpf.h
@@ -23,6 +23,7 @@ enum xt_bpf_modes {
XT_BPF_MODE_FD_PINNED,
XT_BPF_MODE_FD_ELF,
};
+#define XT_BPF_MODE_PATH_PINNED XT_BPF_MODE_FD_PINNED
struct xt_bpf_info_v1 {
__u16 mode;
diff --git a/kernel/bpf/inode.c b/kernel/bpf/inode.c
index e833ed914358..be1dde967208 100644
--- a/kernel/bpf/inode.c
+++ b/kernel/bpf/inode.c
@@ -363,6 +363,7 @@ int bpf_obj_get_user(const char __user *pathname)
putname(pname);
return ret;
}
+EXPORT_SYMBOL_GPL(bpf_obj_get_user);
static void bpf_evict_inode(struct inode *inode)
{
diff --git a/net/netfilter/xt_bpf.c b/net/netfilter/xt_bpf.c
index 38986a95216c..29123934887b 100644
--- a/net/netfilter/xt_bpf.c
+++ b/net/netfilter/xt_bpf.c
@@ -8,6 +8,7 @@
*/
#include <linux/module.h>
+#include <linux/syscalls.h>
#include <linux/skbuff.h>
#include <linux/filter.h>
#include <linux/bpf.h>
@@ -49,6 +50,22 @@ static int __bpf_mt_check_fd(int fd, struct bpf_prog **ret)
return 0;
}
+static int __bpf_mt_check_path(const char *path, struct bpf_prog **ret)
+{
+ mm_segment_t oldfs = get_fs();
+ int retval, fd;
+
+ set_fs(KERNEL_DS);
+ fd = bpf_obj_get_user(path);
+ set_fs(oldfs);
+ if (fd < 0)
+ return fd;
+
+ retval = __bpf_mt_check_fd(fd, ret);
+ sys_close(fd);
+ return retval;
+}
+
static int bpf_mt_check(const struct xt_mtchk_param *par)
{
struct xt_bpf_info *info = par->matchinfo;
@@ -66,9 +83,10 @@ static int bpf_mt_check_v1(const struct xt_mtchk_param *par)
return __bpf_mt_check_bytecode(info->bpf_program,
info->bpf_program_num_elem,
&info->filter);
- else if (info->mode == XT_BPF_MODE_FD_PINNED ||
- info->mode == XT_BPF_MODE_FD_ELF)
+ else if (info->mode == XT_BPF_MODE_FD_ELF)
return __bpf_mt_check_fd(info->fd, &info->filter);
+ else if (info->mode == XT_BPF_MODE_PATH_PINNED)
+ return __bpf_mt_check_path(info->path, &info->filter);
else
return -EINVAL;
}
--
2.14.2
^ permalink raw reply related
* Re: [net-next V4 PATCH 1/5] bpf: introduce new bpf cpu map type BPF_MAP_TYPE_CPUMAP
From: Jesper Dangaard Brouer @ 2017-10-06 15:58 UTC (permalink / raw)
To: Daniel Borkmann
Cc: netdev, jakub.kicinski, Michael S. Tsirkin, pavel.odintsov,
Jason Wang, mchan, John Fastabend, peter.waskiewicz.jr,
Daniel Borkmann, Alexei Starovoitov, Andy Gospodarek, brouer
In-Reply-To: <59D798B8.8090101@iogearbox.net>
On Fri, 06 Oct 2017 16:52:40 +0200 Daniel Borkmann <daniel@iogearbox.net> wrote:
> On 10/06/2017 12:50 PM, Jesper Dangaard Brouer wrote:
> > On Thu, 05 Oct 2017 11:40:15 +0200
> > Daniel Borkmann <daniel@iogearbox.net> wrote:
> >> On 10/04/2017 02:03 PM, Jesper Dangaard Brouer wrote:
> >> [...]
> [...]
> >>> + /* Updating qsize cause re-allocation of bpf_cpu_map_entry */
> >>> + rcpu = __cpu_map_entry_alloc(qsize, key_cpu, map->id);
> >>> + if (!rcpu)
> >>> + return -ENOMEM;
> >>> + }
> >>> + rcu_read_lock();
> >>> + __cpu_map_entry_replace(cmap, key_cpu, rcpu);
> >>> + rcu_read_unlock();
> >>> + return 0;
> >>
> >> You need to update verifier such that this function cannot be called
> >> out of an BPF program,
> >
> > In the example BPF program, I do a lookup into the map, but only to
> > verify that an entry exist (I don't look at the value). I would like
> > to support such usage.
>
> Ok, put comment below.
>
> >> otherwise it would be possible under full RCU
> >> read context, which is explicitly avoided here and also it would otherwise
> >> be allowed for other maps of different type as well, which needs to
> >> be avoided.
> >
> > Sorry, I don't follow this.
>
> What I meant is that check_map_func_compatibility() should check for
> BPF_MAP_TYPE_CPUMAP and only allow func_id of BPF_FUNC_redirect_map
> and BPF_FUNC_map_lookup_elem to be used, which I haven't seen the set
> restricting it to. Some of your later patches do this for the helper
> BPF_FUNC_redirect_map but the important point is that map updates
> wouldn't be done out of the BPF program itself, but rather from user
> space control path given they can't be done under full RCU read lock
> context if I read this correctly (which the programs run in though).
Okay, I choose to restrict bpf_prog side in check_map_func_compatibility()
as you describe. And I changed the user program to keep track of valid
entries via secondary map. We can always add/allow lookup later if users
request this.
I'll send out V5 shortly...
--
Best regards,
Jesper Dangaard Brouer
MSc.CS, Principal Kernel Engineer at Red Hat
LinkedIn: http://www.linkedin.com/in/brouer
^ permalink raw reply
* [PATCH v2] gso: fix payload length when gso_size is zero
From: Alexey Kodanev @ 2017-10-06 16:02 UTC (permalink / raw)
To: netdev; +Cc: Steffen Klassert, Alexander Duyck, David Miller, Alexey Kodanev
When gso_size reset to zero for the tail segment in skb_segment(), later
in ipv6_gso_segment(), __skb_udp_tunnel_segment() and gre_gso_segment()
we will get incorrect results (payload length, pcsum) for that segment.
inet_gso_segment() already has a check for gso_size before calculating
payload.
The issue was found with LTP vxlan & gre tests over ixgbe NIC.
Fixes: 07b26c9454a2 ("gso: Support partial splitting at the frag_list pointer")
Signed-off-by: Alexey Kodanev <alexey.kodanev@oracle.com>
---
v2: also added skb_is_gso to gre_gso_segment() and __skb_udp_tunnel_segment()
net/ipv4/gre_offload.c | 2 +-
net/ipv4/udp_offload.c | 2 +-
net/ipv6/ip6_offload.c | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/net/ipv4/gre_offload.c b/net/ipv4/gre_offload.c
index d5cac99..8c72034 100644
--- a/net/ipv4/gre_offload.c
+++ b/net/ipv4/gre_offload.c
@@ -98,7 +98,7 @@ static struct sk_buff *gre_gso_segment(struct sk_buff *skb,
greh = (struct gre_base_hdr *)skb_transport_header(skb);
pcsum = (__sum16 *)(greh + 1);
- if (gso_partial) {
+ if (gso_partial && skb_is_gso(skb)) {
unsigned int partial_adj;
/* Adjust checksum to account for the fact that
diff --git a/net/ipv4/udp_offload.c b/net/ipv4/udp_offload.c
index 0932c85..6401574 100644
--- a/net/ipv4/udp_offload.c
+++ b/net/ipv4/udp_offload.c
@@ -122,7 +122,7 @@ static struct sk_buff *__skb_udp_tunnel_segment(struct sk_buff *skb,
* will be using a length value equal to only one MSS sized
* segment instead of the entire frame.
*/
- if (gso_partial) {
+ if (gso_partial && skb_is_gso(skb)) {
uh->len = htons(skb_shinfo(skb)->gso_size +
SKB_GSO_CB(skb)->data_offset +
skb->head - (unsigned char *)uh);
diff --git a/net/ipv6/ip6_offload.c b/net/ipv6/ip6_offload.c
index cdb3728..4a87f94 100644
--- a/net/ipv6/ip6_offload.c
+++ b/net/ipv6/ip6_offload.c
@@ -105,7 +105,7 @@ static struct sk_buff *ipv6_gso_segment(struct sk_buff *skb,
for (skb = segs; skb; skb = skb->next) {
ipv6h = (struct ipv6hdr *)(skb_mac_header(skb) + nhoff);
- if (gso_partial)
+ if (gso_partial && skb_is_gso(skb))
payload_len = skb_shinfo(skb)->gso_size +
SKB_GSO_CB(skb)->data_offset +
skb->head - (unsigned char *)(ipv6h + 1);
--
1.8.3.1
^ permalink raw reply related
* [PATCH iproute2 v2 3/3] ss: add AF_VSOCK support
From: Stefan Hajnoczi @ 2017-10-06 15:48 UTC (permalink / raw)
To: netdev; +Cc: Stephen Hemminger, Jorgen Hansen, Dexuan Cui, Stefan Hajnoczi
In-Reply-To: <20171006154841.10495-1-stefanha@redhat.com>
The AF_VSOCK address family is a host<->guest communications channel
supported by VMware, KVM, and Hyper-V. Initial VMware support was
released in Linux 3.9 in 2013 and transports for other hypervisors were
added later.
AF_VSOCK addresses are <u32 cid, u32 port> tuples. The 32-bit cid
integer is comparable to an IP address. AF_VSOCK ports work like
TCP/UDP ports.
Both SOCK_STREAM and SOCK_DGRAM socket types are available.
This patch adds AF_VSOCK support to ss(8) so that sockets can be
observed.
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
---
misc/ss.c | 184 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
man/man8/ss.8 | 8 ++-
2 files changed, 188 insertions(+), 4 deletions(-)
diff --git a/misc/ss.c b/misc/ss.c
index 005e781d..8b563bd4 100644
--- a/misc/ss.c
+++ b/misc/ss.c
@@ -44,6 +44,7 @@
#include <linux/packet_diag.h>
#include <linux/netlink_diag.h>
#include <linux/sctp.h>
+#include <linux/vm_sockets_diag.h>
#define MAGIC_SEQ 123456
@@ -126,6 +127,8 @@ enum {
PACKET_R_DB,
NETLINK_DB,
SCTP_DB,
+ VSOCK_ST_DB,
+ VSOCK_DG_DB,
MAX_DB
};
@@ -134,6 +137,7 @@ enum {
#define ALL_DB ((1<<MAX_DB)-1)
#define INET_L4_DBM ((1<<TCP_DB)|(1<<UDP_DB)|(1<<DCCP_DB)|(1<<SCTP_DB))
#define INET_DBM (INET_L4_DBM | (1<<RAW_DB))
+#define VSOCK_DBM ((1<<VSOCK_ST_DB)|(1<<VSOCK_DG_DB))
enum {
SS_UNKNOWN,
@@ -222,6 +226,14 @@ static const struct filter default_dbs[MAX_DB] = {
.states = SS_CONN,
.families = FAMILY_MASK(AF_INET) | FAMILY_MASK(AF_INET6),
},
+ [VSOCK_ST_DB] = {
+ .states = SS_CONN,
+ .families = FAMILY_MASK(AF_VSOCK),
+ },
+ [VSOCK_DG_DB] = {
+ .states = SS_CONN,
+ .families = FAMILY_MASK(AF_VSOCK),
+ },
};
static const struct filter default_afs[AF_MAX] = {
@@ -245,6 +257,10 @@ static const struct filter default_afs[AF_MAX] = {
.dbs = (1 << NETLINK_DB),
.states = (1 << SS_CLOSE),
},
+ [AF_VSOCK] = {
+ .dbs = VSOCK_DBM,
+ .states = SS_CONN,
+ },
};
static int do_default = 1;
@@ -283,6 +299,8 @@ static void filter_default_dbs(struct filter *f)
filter_db_set(f, PACKET_DG_DB);
filter_db_set(f, NETLINK_DB);
filter_db_set(f, SCTP_DB);
+ filter_db_set(f, VSOCK_ST_DB);
+ filter_db_set(f, VSOCK_DG_DB);
}
static void filter_states_set(struct filter *f, int states)
@@ -791,6 +809,18 @@ static const char *proto_name(int protocol)
return "???";
}
+static const char *vsock_netid_name(int type)
+{
+ switch (type) {
+ case SOCK_STREAM:
+ return "v_str";
+ case SOCK_DGRAM:
+ return "v_dgr";
+ default:
+ return "???";
+ }
+}
+
static void sock_state_print(struct sockstat *s)
{
const char *sock_name;
@@ -823,6 +853,9 @@ static void sock_state_print(struct sockstat *s)
case AF_NETLINK:
sock_name = "nl";
break;
+ case AF_VSOCK:
+ sock_name = vsock_netid_name(s->type);
+ break;
default:
sock_name = "unknown";
}
@@ -1149,6 +1182,8 @@ static int run_ssfilter(struct ssfilter *f, struct sockstat *s)
return s->lport == 0 && s->local.data[0] == 0;
if (s->local.family == AF_NETLINK)
return s->lport < 0;
+ if (s->local.family == AF_VSOCK)
+ return s->lport > 1023;
return is_ephemeral(s->lport);
}
@@ -1524,6 +1559,15 @@ void *parse_devcond(char *name)
return res;
}
+static void vsock_set_inet_prefix(inet_prefix *a, __u32 cid)
+{
+ *a = (inet_prefix){
+ .bytelen = sizeof(cid),
+ .family = AF_VSOCK,
+ };
+ memcpy(a->data, &cid, sizeof(cid));
+}
+
void *parse_hostcond(char *addr, bool is_port)
{
char *port = NULL;
@@ -1598,6 +1642,37 @@ void *parse_hostcond(char *addr, bool is_port)
goto out;
}
+ if (fam == AF_VSOCK || strncmp(addr, "vsock:", 6) == 0) {
+ __u32 cid = ~(__u32)0;
+
+ a.addr.family = AF_VSOCK;
+ if (strncmp(addr, "vsock:", 6) == 0)
+ addr += 6;
+
+ if (is_port)
+ port = addr;
+ else {
+ port = strchr(addr, ':');
+ if (port) {
+ *port = '\0';
+ port++;
+ }
+ }
+
+ if (port && strcmp(port, "*") &&
+ get_u32((__u32 *)&a.port, port, 0))
+ return NULL;
+
+ if (addr[0] && strcmp(addr, "*")) {
+ a.addr.bitlen = 32;
+ if (get_u32(&cid, addr, 0))
+ return NULL;
+ }
+ vsock_set_inet_prefix(&a.addr, cid);
+ fam = AF_VSOCK;
+ goto out;
+ }
+
if (fam == AF_INET || !strncmp(addr, "inet:", 5)) {
fam = AF_INET;
if (!strncmp(addr, "inet:", 5))
@@ -3674,6 +3749,88 @@ static int netlink_show(struct filter *f)
return 0;
}
+static bool vsock_type_skip(struct sockstat *s, struct filter *f)
+{
+ if (s->type == SOCK_STREAM && !(f->dbs & (1 << VSOCK_ST_DB)))
+ return true;
+ if (s->type == SOCK_DGRAM && !(f->dbs & (1 << VSOCK_DG_DB)))
+ return true;
+ return false;
+}
+
+static void vsock_addr_print(inet_prefix *a, __u32 port)
+{
+ char cid_str[sizeof("4294967295")];
+ char port_str[sizeof("4294967295")];
+ __u32 cid;
+
+ memcpy(&cid, a->data, sizeof(cid));
+
+ if (cid == ~(__u32)0)
+ snprintf(cid_str, sizeof(cid_str), "*");
+ else
+ snprintf(cid_str, sizeof(cid_str), "%u", cid);
+
+ if (port == ~(__u32)0)
+ snprintf(port_str, sizeof(port_str), "*");
+ else
+ snprintf(port_str, sizeof(port_str), "%u", port);
+
+ sock_addr_print(cid_str, ":", port_str, NULL);
+}
+
+static void vsock_stats_print(struct sockstat *s, struct filter *f)
+{
+ sock_state_print(s);
+
+ vsock_addr_print(&s->local, s->lport);
+ vsock_addr_print(&s->remote, s->rport);
+
+ proc_ctx_print(s);
+
+ printf("\n");
+}
+
+static int vsock_show_sock(const struct sockaddr_nl *addr,
+ struct nlmsghdr *nlh, void *arg)
+{
+ struct filter *f = (struct filter *)arg;
+ struct vsock_diag_msg *r = NLMSG_DATA(nlh);
+ struct sockstat stat = {
+ .type = r->vdiag_type,
+ .lport = r->vdiag_src_port,
+ .rport = r->vdiag_dst_port,
+ .state = r->vdiag_state,
+ .ino = r->vdiag_ino,
+ };
+
+ vsock_set_inet_prefix(&stat.local, r->vdiag_src_cid);
+ vsock_set_inet_prefix(&stat.remote, r->vdiag_dst_cid);
+
+ if (vsock_type_skip(&stat, f))
+ return 0;
+
+ if (f->f && run_ssfilter(f->f, &stat) == 0)
+ return 0;
+
+ vsock_stats_print(&stat, f);
+
+ return 0;
+}
+
+static int vsock_show(struct filter *f)
+{
+ DIAG_REQUEST(req, struct vsock_diag_req r);
+
+ if (!filter_af_get(f, AF_VSOCK))
+ return 0;
+
+ req.r.sdiag_family = AF_VSOCK;
+ req.r.vdiag_states = f->states;
+
+ return handle_netlink_request(f, &req.nlh, sizeof(req), vsock_show_sock);
+}
+
struct sock_diag_msg {
__u8 sdiag_family;
};
@@ -3694,6 +3851,8 @@ static int generic_show_sock(const struct sockaddr_nl *addr,
return packet_show_sock(addr, nlh, arg);
case AF_NETLINK:
return netlink_show_sock(addr, nlh, arg);
+ case AF_VSOCK:
+ return vsock_show_sock(addr, nlh, arg);
default:
return -1;
}
@@ -3921,14 +4080,15 @@ static void _usage(FILE *dest)
" -d, --dccp display only DCCP sockets\n"
" -w, --raw display only RAW sockets\n"
" -x, --unix display only Unix domain sockets\n"
+" --vsock display only vsock sockets\n"
" -f, --family=FAMILY display sockets of type FAMILY\n"
-" FAMILY := {inet|inet6|link|unix|netlink|help}\n"
+" FAMILY := {inet|inet6|link|unix|netlink|vsock|help}\n"
"\n"
" -K, --kill forcibly close sockets, display what was closed\n"
" -H, --no-header Suppress header line\n"
"\n"
" -A, --query=QUERY, --socket=QUERY\n"
-" QUERY := {all|inet|tcp|udp|raw|unix|unix_dgram|unix_stream|unix_seqpacket|packet|netlink}[,QUERY]\n"
+" QUERY := {all|inet|tcp|udp|raw|unix|unix_dgram|unix_stream|unix_seqpacket|packet|netlink|vsock_stream|vsock_dgram}[,QUERY]\n"
"\n"
" -D, --diag=FILE Dump raw information about TCP sockets to FILE\n"
" -F, --filter=FILE read filter information from FILE\n"
@@ -4001,6 +4161,9 @@ static int scan_state(const char *state)
exit(-1);
}
+/* Values 'v' and 'V' are already used so a non-character is used */
+#define OPT_VSOCK 256
+
static const struct option long_opts[] = {
{ "numeric", 0, 0, 'n' },
{ "resolve", 0, 0, 'r' },
@@ -4017,6 +4180,7 @@ static const struct option long_opts[] = {
{ "udp", 0, 0, 'u' },
{ "raw", 0, 0, 'w' },
{ "unix", 0, 0, 'x' },
+ { "vsock", 0, 0, OPT_VSOCK },
{ "all", 0, 0, 'a' },
{ "listening", 0, 0, 'l' },
{ "ipv4", 0, 0, '4' },
@@ -4102,6 +4266,9 @@ int main(int argc, char *argv[])
case 'x':
filter_af_set(¤t_filter, AF_UNIX);
break;
+ case OPT_VSOCK:
+ filter_af_set(¤t_filter, AF_VSOCK);
+ break;
case 'a':
state_filter = SS_ALL;
break;
@@ -4128,6 +4295,8 @@ int main(int argc, char *argv[])
filter_af_set(¤t_filter, AF_UNIX);
else if (strcmp(optarg, "netlink") == 0)
filter_af_set(¤t_filter, AF_NETLINK);
+ else if (strcmp(optarg, "vsock") == 0)
+ filter_af_set(¤t_filter, AF_VSOCK);
else if (strcmp(optarg, "help") == 0)
help();
else {
@@ -4193,6 +4362,15 @@ int main(int argc, char *argv[])
filter_db_set(¤t_filter, PACKET_DG_DB);
} else if (strcmp(p, "netlink") == 0) {
filter_db_set(¤t_filter, NETLINK_DB);
+ } else if (strcmp(p, "vsock") == 0) {
+ filter_db_set(¤t_filter, VSOCK_ST_DB);
+ filter_db_set(¤t_filter, VSOCK_DG_DB);
+ } else if (strcmp(p, "vsock_stream") == 0 ||
+ strcmp(p, "v_str") == 0) {
+ filter_db_set(¤t_filter, VSOCK_ST_DB);
+ } else if (strcmp(p, "vsock_dgram") == 0 ||
+ strcmp(p, "v_dgr") == 0) {
+ filter_db_set(¤t_filter, VSOCK_DG_DB);
} else {
fprintf(stderr, "ss: \"%s\" is illegal socket table id\n", p);
usage();
@@ -4408,6 +4586,8 @@ int main(int argc, char *argv[])
dccp_show(¤t_filter);
if (current_filter.dbs & (1<<SCTP_DB))
sctp_show(¤t_filter);
+ if (current_filter.dbs & VSOCK_DBM)
+ vsock_show(¤t_filter);
if (show_users || show_proc_ctx || show_sock_ctx)
user_ent_destroy();
diff --git a/man/man8/ss.8 b/man/man8/ss.8
index 3bec97f0..3af509e9 100644
--- a/man/man8/ss.8
+++ b/man/man8/ss.8
@@ -125,14 +125,18 @@ Display Unix domain sockets (alias for -f unix).
.B \-S, \-\-sctp
Display SCTP sockets.
.TP
+.B \-\-vsock
+Display vsock sockets (alias for -f vsock).
+.TP
.B \-f FAMILY, \-\-family=FAMILY
Display sockets of type FAMILY.
-Currently the following families are supported: unix, inet, inet6, link, netlink.
+Currently the following families are supported: unix, inet, inet6, link, netlink, vsock.
.TP
.B \-A QUERY, \-\-query=QUERY, \-\-socket=QUERY
List of socket tables to dump, separated by commas. The following identifiers
are understood: all, inet, tcp, udp, raw, unix, packet, netlink, unix_dgram,
-unix_stream, unix_seqpacket, packet_raw, packet_dgram, dccp, sctp.
+unix_stream, unix_seqpacket, packet_raw, packet_dgram, dccp, sctp,
+vsock_stream, vsock_dgram.
.TP
.B \-D FILE, \-\-diag=FILE
Do not display anything, just dump raw information about TCP sockets to FILE after applying filters. If FILE is - stdout is used.
--
2.13.6
^ permalink raw reply related
* [PATCH iproute2 v2 2/3] include: add <linux/vm_sockets_diag.h>
From: Stefan Hajnoczi @ 2017-10-06 15:48 UTC (permalink / raw)
To: netdev; +Cc: Stephen Hemminger, Jorgen Hansen, Dexuan Cui, Stefan Hajnoczi
In-Reply-To: <20171006154841.10495-1-stefanha@redhat.com>
This new Linux header file defines the sock_diag interface used by
AF_VSOCK. This new header file was merged in net-next in commit
413a4317aca7d6367d57a5971b0c461f03851207 ("VSOCK: add sock_diag
interface").
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
---
include/linux/vm_sockets_diag.h | 33 +++++++++++++++++++++++++++++++++
1 file changed, 33 insertions(+)
create mode 100644 include/linux/vm_sockets_diag.h
diff --git a/include/linux/vm_sockets_diag.h b/include/linux/vm_sockets_diag.h
new file mode 100644
index 00000000..14cd7dc5
--- /dev/null
+++ b/include/linux/vm_sockets_diag.h
@@ -0,0 +1,33 @@
+/* AF_VSOCK sock_diag(7) interface for querying open sockets */
+
+#ifndef _UAPI__VM_SOCKETS_DIAG_H__
+#define _UAPI__VM_SOCKETS_DIAG_H__
+
+#include <linux/types.h>
+
+/* Request */
+struct vsock_diag_req {
+ __u8 sdiag_family; /* must be AF_VSOCK */
+ __u8 sdiag_protocol; /* must be 0 */
+ __u16 pad; /* must be 0 */
+ __u32 vdiag_states; /* query bitmap (e.g. 1 << TCP_LISTEN) */
+ __u32 vdiag_ino; /* must be 0 (reserved) */
+ __u32 vdiag_show; /* must be 0 (reserved) */
+ __u32 vdiag_cookie[2];
+};
+
+/* Response */
+struct vsock_diag_msg {
+ __u8 vdiag_family; /* AF_VSOCK */
+ __u8 vdiag_type; /* SOCK_STREAM or SOCK_DGRAM */
+ __u8 vdiag_state; /* sk_state (e.g. TCP_LISTEN) */
+ __u8 vdiag_shutdown; /* local RCV_SHUTDOWN | SEND_SHUTDOWN */
+ __u32 vdiag_src_cid;
+ __u32 vdiag_src_port;
+ __u32 vdiag_dst_cid;
+ __u32 vdiag_dst_port;
+ __u32 vdiag_ino;
+ __u32 vdiag_cookie[2];
+};
+
+#endif /* _UAPI__VM_SOCKETS_DIAG_H__ */
--
2.13.6
^ permalink raw reply related
* [PATCH iproute2 v2 1/3] ss: allow AF_FAMILY constants >32
From: Stefan Hajnoczi @ 2017-10-06 15:48 UTC (permalink / raw)
To: netdev; +Cc: Stephen Hemminger, Jorgen Hansen, Dexuan Cui, Stefan Hajnoczi
In-Reply-To: <20171006154841.10495-1-stefanha@redhat.com>
Linux has more than 32 address families defined in <bits/socket.h>. Use
a 64-bit type so all of them can be represented in the filter->families
bitmask.
It's easy to introduce bugs when using (1 << AF_FAMILY) because the
value is 32-bit. This can produce incorrect results from bitmask
operations so introduce the FAMILY_MASK() macro to eliminate these bugs.
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
---
misc/ss.c | 54 ++++++++++++++++++++++++++++--------------------------
1 file changed, 28 insertions(+), 26 deletions(-)
diff --git a/misc/ss.c b/misc/ss.c
index dd8dfaa4..005e781d 100644
--- a/misc/ss.c
+++ b/misc/ss.c
@@ -170,55 +170,57 @@ enum {
struct filter {
int dbs;
int states;
- int families;
+ uint64_t families;
struct ssfilter *f;
bool kill;
};
+#define FAMILY_MASK(family) ((uint64_t)1 << (family))
+
static const struct filter default_dbs[MAX_DB] = {
[TCP_DB] = {
.states = SS_CONN,
- .families = (1 << AF_INET) | (1 << AF_INET6),
+ .families = FAMILY_MASK(AF_INET) | FAMILY_MASK(AF_INET6),
},
[DCCP_DB] = {
.states = SS_CONN,
- .families = (1 << AF_INET) | (1 << AF_INET6),
+ .families = FAMILY_MASK(AF_INET) | FAMILY_MASK(AF_INET6),
},
[UDP_DB] = {
.states = (1 << SS_ESTABLISHED),
- .families = (1 << AF_INET) | (1 << AF_INET6),
+ .families = FAMILY_MASK(AF_INET) | FAMILY_MASK(AF_INET6),
},
[RAW_DB] = {
.states = (1 << SS_ESTABLISHED),
- .families = (1 << AF_INET) | (1 << AF_INET6),
+ .families = FAMILY_MASK(AF_INET) | FAMILY_MASK(AF_INET6),
},
[UNIX_DG_DB] = {
.states = (1 << SS_CLOSE),
- .families = (1 << AF_UNIX),
+ .families = FAMILY_MASK(AF_UNIX),
},
[UNIX_ST_DB] = {
.states = SS_CONN,
- .families = (1 << AF_UNIX),
+ .families = FAMILY_MASK(AF_UNIX),
},
[UNIX_SQ_DB] = {
.states = SS_CONN,
- .families = (1 << AF_UNIX),
+ .families = FAMILY_MASK(AF_UNIX),
},
[PACKET_DG_DB] = {
.states = (1 << SS_CLOSE),
- .families = (1 << AF_PACKET),
+ .families = FAMILY_MASK(AF_PACKET),
},
[PACKET_R_DB] = {
.states = (1 << SS_CLOSE),
- .families = (1 << AF_PACKET),
+ .families = FAMILY_MASK(AF_PACKET),
},
[NETLINK_DB] = {
.states = (1 << SS_CLOSE),
- .families = (1 << AF_NETLINK),
+ .families = FAMILY_MASK(AF_NETLINK),
},
[SCTP_DB] = {
.states = SS_CONN,
- .families = (1 << AF_INET) | (1 << AF_INET6),
+ .families = FAMILY_MASK(AF_INET) | FAMILY_MASK(AF_INET6),
},
};
@@ -258,14 +260,14 @@ static void filter_db_set(struct filter *f, int db)
static void filter_af_set(struct filter *f, int af)
{
f->states |= default_afs[af].states;
- f->families |= 1 << af;
+ f->families |= FAMILY_MASK(af);
do_default = 0;
preferred_family = af;
}
static int filter_af_get(struct filter *f, int af)
{
- return f->families & (1 << af);
+ return !!(f->families & FAMILY_MASK(af));
}
static void filter_default_dbs(struct filter *f)
@@ -302,7 +304,7 @@ static void filter_merge_defaults(struct filter *f)
f->families |= default_dbs[db].families;
}
for (af = 0; af < AF_MAX; af++) {
- if (!(f->families & (1 << af)))
+ if (!(f->families & FAMILY_MASK(af)))
continue;
if (!(default_afs[af].dbs & f->dbs))
@@ -2608,7 +2610,7 @@ static int show_one_inet_sock(const struct sockaddr_nl *addr,
struct inet_diag_msg *r = NLMSG_DATA(h);
struct sockstat s = {};
- if (!(diag_arg->f->families & (1 << r->idiag_family)))
+ if (!(diag_arg->f->families & FAMILY_MASK(r->idiag_family)))
return 0;
parse_diag_msg(h, &s);
@@ -2802,7 +2804,7 @@ static int tcp_show(struct filter *f)
return -1;
}
- if (f->families & (1<<AF_INET)) {
+ if (f->families & FAMILY_MASK(AF_INET)) {
if ((fp = net_tcp_open()) == NULL)
goto outerr;
@@ -2812,7 +2814,7 @@ static int tcp_show(struct filter *f)
fclose(fp);
}
- if ((f->families & (1<<AF_INET6)) &&
+ if ((f->families & FAMILY_MASK(AF_INET6)) &&
(fp = net_tcp6_open()) != NULL) {
setbuffer(fp, buf, bufsize);
if (generic_record_read(fp, tcp_show_line, f, AF_INET6))
@@ -2911,7 +2913,7 @@ static int udp_show(struct filter *f)
&& inet_show_netlink(f, NULL, IPPROTO_UDP) == 0)
return 0;
- if (f->families&(1<<AF_INET)) {
+ if (f->families&FAMILY_MASK(AF_INET)) {
if ((fp = net_udp_open()) == NULL)
goto outerr;
if (generic_record_read(fp, dgram_show_line, f, AF_INET))
@@ -2919,7 +2921,7 @@ static int udp_show(struct filter *f)
fclose(fp);
}
- if ((f->families&(1<<AF_INET6)) &&
+ if ((f->families&FAMILY_MASK(AF_INET6)) &&
(fp = net_udp6_open()) != NULL) {
if (generic_record_read(fp, dgram_show_line, f, AF_INET6))
goto outerr;
@@ -2951,7 +2953,7 @@ static int raw_show(struct filter *f)
inet_show_netlink(f, NULL, IPPROTO_RAW) == 0)
return 0;
- if (f->families&(1<<AF_INET)) {
+ if (f->families&FAMILY_MASK(AF_INET)) {
if ((fp = net_raw_open()) == NULL)
goto outerr;
if (generic_record_read(fp, dgram_show_line, f, AF_INET))
@@ -2959,7 +2961,7 @@ static int raw_show(struct filter *f)
fclose(fp);
}
- if ((f->families&(1<<AF_INET6)) &&
+ if ((f->families&FAMILY_MASK(AF_INET6)) &&
(fp = net_raw6_open()) != NULL) {
if (generic_record_read(fp, dgram_show_line, f, AF_INET6))
goto outerr;
@@ -3703,13 +3705,13 @@ static int handle_follow_request(struct filter *f)
int groups = 0;
struct rtnl_handle rth;
- if (f->families & (1 << AF_INET) && f->dbs & (1 << TCP_DB))
+ if (f->families & FAMILY_MASK(AF_INET) && f->dbs & (1 << TCP_DB))
groups |= 1 << (SKNLGRP_INET_TCP_DESTROY - 1);
- if (f->families & (1 << AF_INET) && f->dbs & (1 << UDP_DB))
+ if (f->families & FAMILY_MASK(AF_INET) && f->dbs & (1 << UDP_DB))
groups |= 1 << (SKNLGRP_INET_UDP_DESTROY - 1);
- if (f->families & (1 << AF_INET6) && f->dbs & (1 << TCP_DB))
+ if (f->families & FAMILY_MASK(AF_INET6) && f->dbs & (1 << TCP_DB))
groups |= 1 << (SKNLGRP_INET6_TCP_DESTROY - 1);
- if (f->families & (1 << AF_INET6) && f->dbs & (1 << UDP_DB))
+ if (f->families & FAMILY_MASK(AF_INET6) && f->dbs & (1 << UDP_DB))
groups |= 1 << (SKNLGRP_INET6_UDP_DESTROY - 1);
if (groups == 0)
--
2.13.6
^ permalink raw reply related
* [PATCH iproute2 v2 0/3] ss: add AF_VSOCK support
From: Stefan Hajnoczi @ 2017-10-06 15:48 UTC (permalink / raw)
To: netdev; +Cc: Stephen Hemminger, Jorgen Hansen, Dexuan Cui, Stefan Hajnoczi
v2:
* Use uint64_t instead of __u64 for filter->families
* Added reference to net-next commit that merged vsock_diag.ko
This patch series adds AF_VSOCK support to ss(8). AF_VSOCK is a host<->guest
communications channel supported by VMware, KVM (virtio-vsock), and Hyper-V.
To dump AF_VSOCK sockets:
$ ss --vsock
The vsock_diag.ko module has now been merged in the Linux net-next tree. I
have verified that the <linux/vm_sockets_diag.h> header copy in this patch
series is in sync with Linux net-next. See commit
5820299a271fd3dc9b1733e1e10cd7b983edd028 ("Merge branch 'VSOCK-sock_diag'").
Stefan Hajnoczi (3):
ss: allow AF_FAMILY constants >32
include: add <linux/vm_sockets_diag.h>
ss: add AF_VSOCK support
include/linux/vm_sockets_diag.h | 33 ++++++
misc/ss.c | 238 +++++++++++++++++++++++++++++++++++-----
man/man8/ss.8 | 8 +-
3 files changed, 249 insertions(+), 30 deletions(-)
create mode 100644 include/linux/vm_sockets_diag.h
--
2.13.6
^ permalink raw reply
* Re: [net-next V4 PATCH 2/5] bpf: XDP_REDIRECT enable use of cpumap
From: Jesper Dangaard Brouer @ 2017-10-06 15:45 UTC (permalink / raw)
To: Daniel Borkmann
Cc: netdev, jakub.kicinski, Michael S. Tsirkin, pavel.odintsov,
Jason Wang, mchan, John Fastabend, peter.waskiewicz.jr,
Daniel Borkmann, Alexei Starovoitov, Andy Gospodarek, brouer
In-Reply-To: <20171006131748.75185f65@redhat.com>
On Fri, 6 Oct 2017 13:17:48 +0200
Jesper Dangaard Brouer <brouer@redhat.com> wrote:
> > > +static void *__xdp_map_lookup_elem(struct bpf_map *map, u32 index)
> > > +{
> > > + switch (map->map_type) {
> > > + case BPF_MAP_TYPE_DEVMAP:
> > > + return __dev_map_lookup_elem(map, index);
> > > + case BPF_MAP_TYPE_CPUMAP:
> > > + return __cpu_map_lookup_elem(map, index);
> > > + default:
> > > + return NULL;
> > > + }
> >
> > Should we just have a callback and instead of the above use
> > map->ptr_lookup_elem() (or however we name it) ... lot of it
> > is pretty much the same logic as with devmap.
>
> We could extend struct bpf_map *map with such a callback, I was just
> afraid that this would be too invasive.
>
> Performance wise, I don't thinks will hurt too much.
> http://www.cipht.net/2017/10/03/are-jump-tables-always-fastest.html
Looking at the code, I would like to postpone this callback work until
after this patchset is merged. As this work will also touch devmap +
sockmap.
--
Best regards,
Jesper Dangaard Brouer
MSc.CS, Principal Kernel Engineer at Red Hat
LinkedIn: http://www.linkedin.com/in/brouer
^ permalink raw reply
* Re: [PATCH net-next 0/7] nfp: extend match and action for flower offload
From: Tom Herbert @ 2017-10-06 15:34 UTC (permalink / raw)
To: Simon Horman
Cc: David Miller, Jakub Kicinski, Linux Kernel Network Developers,
oss-drivers
In-Reply-To: <1507278086-3102-1-git-send-email-simon.horman@netronome.com>
Simon,
Maybe a bit off topic, but I had the impression netronome would
support BPF so that filters could be programmed for arbitrary
protocols and fields. Is that true? If so, what is the relationship
between that functionality and these patches?
Thanks,
Tom
On Fri, Oct 6, 2017 at 1:21 AM, Simon Horman <simon.horman@netronome.com> wrote:
> Pieter says:
>
> This series extends flower offload match and action capabilities. It
> specifically adds offload capabilities for matching on MPLS, TTL, TOS
> and flow label. Furthermore offload capabilities for action have been
> expanded to include set ethernet, ipv4, ipv6, tcp and udp headers.
>
> Pieter Jansen van Vuuren (7):
> nfp: add mpls match offloading support
> nfp: add IPv4 ttl and tos match offloading support
> nfp: add IPv6 ttl and tos match offloading support
> nfp: add set ethernet header action flower offload
> nfp: add set ipv4 header action flower offload
> nfp: add set ipv6 source and destination address
> nfp: add set tcp and udp header action flower offload
>
> drivers/net/ethernet/netronome/nfp/flower/action.c | 245 +++++++++++++++++++++
> drivers/net/ethernet/netronome/nfp/flower/cmsg.h | 43 ++++
> drivers/net/ethernet/netronome/nfp/flower/match.c | 39 +++-
> .../net/ethernet/netronome/nfp/flower/offload.c | 20 +-
> 4 files changed, 324 insertions(+), 23 deletions(-)
>
> --
> 2.1.4
>
^ permalink raw reply
* [PATCH net-next,3/3] hv_netvsc: Update netvsc Document for TCP hash level setting
From: Haiyang Zhang @ 2017-10-06 15:33 UTC (permalink / raw)
To: davem, netdev; +Cc: haiyangz, kys, sthemmin, olaf, vkuznets, linux-kernel
In-Reply-To: <20171006153359.8400-1-haiyangz@exchange.microsoft.com>
From: Haiyang Zhang <haiyangz@microsoft.com>
Update Documentation/networking/netvsc.txt for TCP hash level setting
and related info.
Signed-off-by: Haiyang Zhang <haiyangz@microsoft.com>
---
Documentation/networking/netvsc.txt | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/Documentation/networking/netvsc.txt b/Documentation/networking/netvsc.txt
index 93560fb1170a..92f5b31392fa 100644
--- a/Documentation/networking/netvsc.txt
+++ b/Documentation/networking/netvsc.txt
@@ -19,12 +19,12 @@ Features
Receive Side Scaling
--------------------
- Hyper-V supports receive side scaling. For TCP, packets are
- distributed among available queues based on IP address and port
+ Hyper-V supports receive side scaling. For TCP & UDP, packets can
+ be distributed among available queues based on IP address and port
number.
- For UDP, we can switch UDP hash level between L3 and L4 by ethtool
- command. UDP over IPv4 and v6 can be set differently. The default
+ For TCP & UDP, we can switch hash level between L3 and L4 by ethtool
+ command. TCP/UDP over IPv4 and v6 can be set differently. The default
hash level is L4. We currently only allow switching TX hash level
from within the guests.
--
2.14.1
^ permalink raw reply related
* [PATCH net-next,2/3] hv_netvsc: Add ethtool handler to set and get TCP hash levels
From: Haiyang Zhang @ 2017-10-06 15:33 UTC (permalink / raw)
To: davem, netdev; +Cc: haiyangz, kys, sthemmin, olaf, vkuznets, linux-kernel
In-Reply-To: <20171006153359.8400-1-haiyangz@exchange.microsoft.com>
From: Haiyang Zhang <haiyangz@microsoft.com>
The patch supports the options to switch TCP hash level between
L3 and L4 by ethtool command. TCP over IPv4 and v6 can be set
differently. The default hash level is L4. We currently only
allow switching TX hash level from within the guests.
For example, for TCP over IPv4 on eth0:
To include TCP port numbers in hashing:
ethtool -N eth0 rx-flow-hash tcp4 sdfn
To exclude TCP port numbers in hashing:
ethtool -N eth0 rx-flow-hash tcp4 sd
To show TCP hash level:
ethtool -n eth0 rx-flow-hash tcp4
Signed-off-by: Haiyang Zhang <haiyangz@microsoft.com>
---
drivers/net/hyperv/netvsc_drv.c | 25 ++++++++++++++++++++++++-
1 file changed, 24 insertions(+), 1 deletion(-)
diff --git a/drivers/net/hyperv/netvsc_drv.c b/drivers/net/hyperv/netvsc_drv.c
index 9bc7dbab9506..44746de3dd4c 100644
--- a/drivers/net/hyperv/netvsc_drv.c
+++ b/drivers/net/hyperv/netvsc_drv.c
@@ -1264,8 +1264,15 @@ netvsc_get_rss_hash_opts(struct net_device_context *ndc,
switch (info->flow_type) {
case TCP_V4_FLOW:
+ if (ndc->l4_hash & HV_TCP4_L4HASH)
+ info->data |= l4_flag;
+
+ break;
+
case TCP_V6_FLOW:
- info->data |= l4_flag;
+ if (ndc->l4_hash & HV_TCP6_L4HASH)
+ info->data |= l4_flag;
+
break;
case UDP_V4_FLOW:
@@ -1318,6 +1325,14 @@ static int netvsc_set_rss_hash_opts(struct net_device_context *ndc,
if (info->data == (RXH_IP_SRC | RXH_IP_DST |
RXH_L4_B_0_1 | RXH_L4_B_2_3)) {
switch (info->flow_type) {
+ case TCP_V4_FLOW:
+ ndc->l4_hash |= HV_TCP4_L4HASH;
+ break;
+
+ case TCP_V6_FLOW:
+ ndc->l4_hash |= HV_TCP6_L4HASH;
+ break;
+
case UDP_V4_FLOW:
ndc->l4_hash |= HV_UDP4_L4HASH;
break;
@@ -1335,6 +1350,14 @@ static int netvsc_set_rss_hash_opts(struct net_device_context *ndc,
if (info->data == (RXH_IP_SRC | RXH_IP_DST)) {
switch (info->flow_type) {
+ case TCP_V4_FLOW:
+ ndc->l4_hash &= ~HV_TCP4_L4HASH;
+ break;
+
+ case TCP_V6_FLOW:
+ ndc->l4_hash &= ~HV_TCP6_L4HASH;
+ break;
+
case UDP_V4_FLOW:
ndc->l4_hash &= ~HV_UDP4_L4HASH;
break;
--
2.14.1
^ permalink raw reply related
* [PATCH net-next,1/3] hv_netvsc: Change the hash level variable to bit flags
From: Haiyang Zhang @ 2017-10-06 15:33 UTC (permalink / raw)
To: davem, netdev; +Cc: haiyangz, kys, sthemmin, olaf, vkuznets, linux-kernel
In-Reply-To: <20171006153359.8400-1-haiyangz@exchange.microsoft.com>
From: Haiyang Zhang <haiyangz@microsoft.com>
This simplifies the logic and make it easier to add more
options.
Signed-off-by: Haiyang Zhang <haiyangz@microsoft.com>
---
drivers/net/hyperv/hyperv_net.h | 11 +++++--
drivers/net/hyperv/netvsc_drv.c | 73 ++++++++++++++++++++++++++++-------------
2 files changed, 59 insertions(+), 25 deletions(-)
diff --git a/drivers/net/hyperv/hyperv_net.h b/drivers/net/hyperv/hyperv_net.h
index 6f550e15a41c..a81335e8ebe8 100644
--- a/drivers/net/hyperv/hyperv_net.h
+++ b/drivers/net/hyperv/hyperv_net.h
@@ -704,6 +704,14 @@ struct netvsc_reconfig {
u32 event;
};
+/* L4 hash bits for different protocols */
+#define HV_TCP4_L4HASH 1
+#define HV_TCP6_L4HASH 2
+#define HV_UDP4_L4HASH 4
+#define HV_UDP6_L4HASH 8
+#define HV_DEFAULT_L4HASH (HV_TCP4_L4HASH | HV_TCP6_L4HASH | HV_UDP4_L4HASH | \
+ HV_UDP6_L4HASH)
+
/* The context of the netvsc device */
struct net_device_context {
/* point back to our device context */
@@ -726,10 +734,9 @@ struct net_device_context {
u32 tx_send_table[VRSS_SEND_TAB_SIZE];
/* Ethtool settings */
- bool udp4_l4_hash;
- bool udp6_l4_hash;
u8 duplex;
u32 speed;
+ u32 l4_hash; /* L4 hash settings */
struct netvsc_ethtool_stats eth_stats;
/* State to manage the associated VF interface. */
diff --git a/drivers/net/hyperv/netvsc_drv.c b/drivers/net/hyperv/netvsc_drv.c
index dfb986421ec6..9bc7dbab9506 100644
--- a/drivers/net/hyperv/netvsc_drv.c
+++ b/drivers/net/hyperv/netvsc_drv.c
@@ -203,7 +203,7 @@ static inline u32 netvsc_get_hash(
const struct net_device_context *ndc)
{
struct flow_keys flow;
- u32 hash;
+ u32 hash, pkt_proto = 0;
static u32 hashrnd __read_mostly;
net_get_random_once(&hashrnd, sizeof(hashrnd));
@@ -211,11 +211,25 @@ static inline u32 netvsc_get_hash(
if (!skb_flow_dissect_flow_keys(skb, &flow, 0))
return 0;
- if (flow.basic.ip_proto == IPPROTO_TCP ||
- (flow.basic.ip_proto == IPPROTO_UDP &&
- ((flow.basic.n_proto == htons(ETH_P_IP) && ndc->udp4_l4_hash) ||
- (flow.basic.n_proto == htons(ETH_P_IPV6) &&
- ndc->udp6_l4_hash)))) {
+ switch (flow.basic.ip_proto) {
+ case IPPROTO_TCP:
+ if (flow.basic.n_proto == htons(ETH_P_IP))
+ pkt_proto = HV_TCP4_L4HASH;
+ else if (flow.basic.n_proto == htons(ETH_P_IPV6))
+ pkt_proto = HV_TCP6_L4HASH;
+
+ break;
+
+ case IPPROTO_UDP:
+ if (flow.basic.n_proto == htons(ETH_P_IP))
+ pkt_proto = HV_UDP4_L4HASH;
+ else if (flow.basic.n_proto == htons(ETH_P_IPV6))
+ pkt_proto = HV_UDP6_L4HASH;
+
+ break;
+ }
+
+ if (pkt_proto & ndc->l4_hash) {
return skb_get_hash(skb);
} else {
if (flow.basic.n_proto == htons(ETH_P_IP))
@@ -898,8 +912,7 @@ static void netvsc_init_settings(struct net_device *dev)
{
struct net_device_context *ndc = netdev_priv(dev);
- ndc->udp4_l4_hash = true;
- ndc->udp6_l4_hash = true;
+ ndc->l4_hash = HV_DEFAULT_L4HASH;
ndc->speed = SPEED_UNKNOWN;
ndc->duplex = DUPLEX_FULL;
@@ -1245,23 +1258,25 @@ static int
netvsc_get_rss_hash_opts(struct net_device_context *ndc,
struct ethtool_rxnfc *info)
{
+ const u32 l4_flag = RXH_L4_B_0_1 | RXH_L4_B_2_3;
+
info->data = RXH_IP_SRC | RXH_IP_DST;
switch (info->flow_type) {
case TCP_V4_FLOW:
case TCP_V6_FLOW:
- info->data |= RXH_L4_B_0_1 | RXH_L4_B_2_3;
+ info->data |= l4_flag;
break;
case UDP_V4_FLOW:
- if (ndc->udp4_l4_hash)
- info->data |= RXH_L4_B_0_1 | RXH_L4_B_2_3;
+ if (ndc->l4_hash & HV_UDP4_L4HASH)
+ info->data |= l4_flag;
break;
case UDP_V6_FLOW:
- if (ndc->udp6_l4_hash)
- info->data |= RXH_L4_B_0_1 | RXH_L4_B_2_3;
+ if (ndc->l4_hash & HV_UDP6_L4HASH)
+ info->data |= l4_flag;
break;
@@ -1302,23 +1317,35 @@ static int netvsc_set_rss_hash_opts(struct net_device_context *ndc,
{
if (info->data == (RXH_IP_SRC | RXH_IP_DST |
RXH_L4_B_0_1 | RXH_L4_B_2_3)) {
- if (info->flow_type == UDP_V4_FLOW)
- ndc->udp4_l4_hash = true;
- else if (info->flow_type == UDP_V6_FLOW)
- ndc->udp6_l4_hash = true;
- else
+ switch (info->flow_type) {
+ case UDP_V4_FLOW:
+ ndc->l4_hash |= HV_UDP4_L4HASH;
+ break;
+
+ case UDP_V6_FLOW:
+ ndc->l4_hash |= HV_UDP6_L4HASH;
+ break;
+
+ default:
return -EOPNOTSUPP;
+ }
return 0;
}
if (info->data == (RXH_IP_SRC | RXH_IP_DST)) {
- if (info->flow_type == UDP_V4_FLOW)
- ndc->udp4_l4_hash = false;
- else if (info->flow_type == UDP_V6_FLOW)
- ndc->udp6_l4_hash = false;
- else
+ switch (info->flow_type) {
+ case UDP_V4_FLOW:
+ ndc->l4_hash &= ~HV_UDP4_L4HASH;
+ break;
+
+ case UDP_V6_FLOW:
+ ndc->l4_hash &= ~HV_UDP6_L4HASH;
+ break;
+
+ default:
return -EOPNOTSUPP;
+ }
return 0;
}
--
2.14.1
^ permalink raw reply related
* [PATCH net-next,0/3] hv_netvsc: support changing TCP hash level
From: Haiyang Zhang @ 2017-10-06 15:33 UTC (permalink / raw)
To: davem, netdev; +Cc: haiyangz, kys, sthemmin, olaf, vkuznets, linux-kernel
From: Haiyang Zhang <haiyangz@microsoft.com>
The patch set simplifies the existing hash level switching code for
UDP. It also adds the support for changing TCP hash level. So users
can switch between L3 an L4 hash levels for TCP and UDP.
Haiyang Zhang (3):
hv_netvsc: Change the hash level variable to bit flags
hv_netvsc: Add ethtool handler to set and get TCP hash levels
hv_netvsc: Update netvsc Document for TCP hash level setting
Documentation/networking/netvsc.txt | 8 ++--
drivers/net/hyperv/hyperv_net.h | 11 ++++-
drivers/net/hyperv/netvsc_drv.c | 96 ++++++++++++++++++++++++++++---------
3 files changed, 86 insertions(+), 29 deletions(-)
--
2.14.1
^ permalink raw reply
* Re: [PATCH 4/4] tcp: avoid noref dst leak on input path
From: Eric Dumazet @ 2017-10-06 15:32 UTC (permalink / raw)
To: Paolo Abeni
Cc: Eric Dumazet, LKML, Paul E. McKenney, Josh Triplett,
Steven Rostedt, David S. Miller, Hannes Frederic Sowa, netdev
In-Reply-To: <1507303300.2793.25.camel@redhat.com>
On Fri, Oct 6, 2017 at 8:21 AM, Paolo Abeni <pabeni@redhat.com> wrote:
> Hi,
>
> On Fri, 2017-10-06 at 07:37 -0700, Eric Dumazet wrote:
>> On Fri, 2017-10-06 at 14:57 +0200, Paolo Abeni wrote:
>> > Enabling CONFIG_RCU_NOREF_DEBUG gives the following splat when
>> > processing tcp packets:
>> >
>> > to-be-untracked noref entity ffff942cb71ea300 not found in cache
>> > ------------[ cut here ]------------
>> > WARNING: CPU: 24 PID: 178 at kernel/rcu/noref_debug.c:54 rcu_track_noref+0xa4/0xf0
>> > Modules linked in: intel_rapl sb_edac x86_pkg_temp_thermal intel_powerclamp coretemp kvm_intel kvm irqbypass crct10dif_pclmul crc32_pclmul ghash_clmulni_intel pcbc aesni_intel crypto_simd glue_helper cryptd iTCO_wdt ipmi_ssif mei_me iTCO_vendor_support mei dcdbas lpc_ich ipmi_si mxm_wmi sg pcspkr ipmi_devintf ipmi_msghandler acpi_power_meter shpchp wmi nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod mgag200 drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm igb drm ixgbe mdio crc32c_intel ahci ptp i2c_algo_bit libahci pps_core i2c_core libata dca dm_mirror dm_region_hash dm_log dm_mod
>> > CPU: 24 PID: 178 Comm: ksoftirqd/24 Not tainted 4.14.0-rc1.noref_route+ #1610
>> > Hardware name: Dell Inc. PowerEdge R730/072T6D, BIOS 2.4.3 01/17/2017
>> > task: ffff940e48300000 task.stack: ffffaec406a20000
>> > RIP: 0010:rcu_track_noref+0xa4/0xf0
>> > RSP: 0018:ffffaec406a238e0 EFLAGS: 00010246
>> > RAX: 0000000000000040 RBX: 0000000000000000 RCX: 0000000000000002
>> > RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000292
>> > RBP: ffffaec406a238e0 R08: 0000000000000000 R09: 0000000000000000
> Thank you for the feedback.
>
> I most probably messed-up while extracting the info from dmsg, as this
> issue gives a couple of splats almost concurrently. Please let me re-do
> the test and post a more resonable dmsg.
>
> The problem with the current code is that in the tcp_rcv_established()
> -> tcp_queue_rcv() path, the skb_dst() is not cleared.
>
In any case, I would rather put one skb_dst_drop() right after the
last possible use of skb dst in TCP stack,
probably after sk_rx_dst_set() call.
Trying to move it in multiple places has been error prone, even if
current code is not buggy.
^ permalink raw reply
* Re: [PATCH 4/4] tcp: avoid noref dst leak on input path
From: Paolo Abeni @ 2017-10-06 15:21 UTC (permalink / raw)
To: Eric Dumazet
Cc: linux-kernel, Paul E. McKenney, Josh Triplett, Steven Rostedt,
David S. Miller, Eric Dumazet, Hannes Frederic Sowa, netdev
In-Reply-To: <1507300642.14419.17.camel@edumazet-glaptop3.roam.corp.google.com>
Hi,
On Fri, 2017-10-06 at 07:37 -0700, Eric Dumazet wrote:
> On Fri, 2017-10-06 at 14:57 +0200, Paolo Abeni wrote:
> > Enabling CONFIG_RCU_NOREF_DEBUG gives the following splat when
> > processing tcp packets:
> >
> > to-be-untracked noref entity ffff942cb71ea300 not found in cache
> > ------------[ cut here ]------------
> > WARNING: CPU: 24 PID: 178 at kernel/rcu/noref_debug.c:54 rcu_track_noref+0xa4/0xf0
> > Modules linked in: intel_rapl sb_edac x86_pkg_temp_thermal intel_powerclamp coretemp kvm_intel kvm irqbypass crct10dif_pclmul crc32_pclmul ghash_clmulni_intel pcbc aesni_intel crypto_simd glue_helper cryptd iTCO_wdt ipmi_ssif mei_me iTCO_vendor_support mei dcdbas lpc_ich ipmi_si mxm_wmi sg pcspkr ipmi_devintf ipmi_msghandler acpi_power_meter shpchp wmi nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod mgag200 drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm igb drm ixgbe mdio crc32c_intel ahci ptp i2c_algo_bit libahci pps_core i2c_core libata dca dm_mirror dm_region_hash dm_log dm_mod
> > CPU: 24 PID: 178 Comm: ksoftirqd/24 Not tainted 4.14.0-rc1.noref_route+ #1610
> > Hardware name: Dell Inc. PowerEdge R730/072T6D, BIOS 2.4.3 01/17/2017
> > task: ffff940e48300000 task.stack: ffffaec406a20000
> > RIP: 0010:rcu_track_noref+0xa4/0xf0
> > RSP: 0018:ffffaec406a238e0 EFLAGS: 00010246
> > RAX: 0000000000000040 RBX: 0000000000000000 RCX: 0000000000000002
> > RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000292
> > RBP: ffffaec406a238e0 R08: 0000000000000000 R09: 0000000000000000
> > R10: 0000000000000001 R11: 0000000000000003 R12: 0000000000000000
> > R13: ffff942cb5110000 R14: 000000000000fe88 R15: ffff942cb1a20200
> > FS: 0000000000000000(0000) GS:ffff942cbee00000(0000) knlGS:0000000000000000
> > CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
> > CR2: 00007febc072d140 CR3: 0000001feebd6002 CR4: 00000000003606e0
> > DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
> > DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
> > Call Trace:
> > tcp_data_queue+0x82a/0xce0
>
> That is strange, since tcp_data_queue() starts with
>
> skb_dst_drop(skb);
>
> So this stack trace looks suspicious.
Thank you for the feedback.
I most probably messed-up while extracting the info from dmsg, as this
issue gives a couple of splats almost concurrently. Please let me re-do
the test and post a more resonable dmsg.
The problem with the current code is that in the tcp_rcv_established()
-> tcp_queue_rcv() path, the skb_dst() is not cleared.
Thanks,
Paolo
^ permalink raw reply
* Re: [PATCH 0/4] RCU: introduce noref debug
From: Paolo Abeni @ 2017-10-06 15:10 UTC (permalink / raw)
To: paulmck
Cc: linux-kernel, Josh Triplett, Steven Rostedt, David S. Miller,
Eric Dumazet, Hannes Frederic Sowa, netdev
In-Reply-To: <20171006133436.GY3521@linux.vnet.ibm.com>
Hi,
On Fri, 2017-10-06 at 06:34 -0700, Paul E. McKenney wrote:
> On Fri, Oct 06, 2017 at 02:57:45PM +0200, Paolo Abeni wrote:
> > The networking subsystem is currently using some kind of long-lived
> > RCU-protected, references to avoid the overhead of full book-keeping.
> >
> > Such references - skb_dst() noref - are stored inside the skbs and can be
> > moved across relevant slices of the network stack, with the users
> > being in charge of properly clearing the relevant skb - or properly refcount
> > the related dst references - before the skb escapes the RCU section.
> >
> > We currently don't have any deterministic debug infrastructure to check
> > the dst noref usages - and the introduction of others noref artifact is
> > currently under discussion.
> >
> > This series tries to tackle the above introducing an RCU debug infrastructure
> > aimed at spotting incorrect noref pointer usage, in patch one. The
> > infrastructure is small and must be explicitly enabled via a newly introduced
> > build option.
> >
> > Patch two uses such infrastructure to track dst noref usage in the networking
> > stack.
> >
> > Patch 3 and 4 are bugfixes for small buglet found running this infrastructure
> > on basic scenarios.
Thank you for the prompt reply!
>
> This patchset does not look like it handles rcu_read_lock() nesting.
> For example, given code like this:
>
> void foo(void)
> {
> rcu_read_lock();
> rcu_track_noref(&key2, &noref2, true);
> do_something();
> rcu_track_noref(&key2, &noref2, false);
> rcu_read_unlock();
> }
>
> void bar(void)
> {
> rcu_read_lock();
> rcu_track_noref(&key1, &noref1, true);
> do_something_more();
> foo();
> do_something_else();
> rcu_track_noref(&key1, &noref1, false);
> rcu_read_unlock();
> }
>
> void grill(void)
> {
> foo();
> }
>
> It looks like foo()'s rcu_read_unlock() will complain about key1.
> You could remove foo()'s rcu_read_lock() and rcu_read_unlock(), but
> that will break the call from grill().
Actually the code should cope correctly with your example; when foo()'s
rcu_read_unlock() is called, 'cache' contains:
{ { &key1, &noref1, 1}, // ...
and when the related __rcu_check_noref() is invoked preempt_count() is
2 - because the check is called before decreasing the preempt counter.
In the main loop inside __rcu_check_noref() we will hit always the
'continue' statement because 'cache->store[i].nesting != nesting', so
no warn will be triggered.
> Or am I missing something subtle here? Given patch 3/4, I suspect not...
The problem with the code in patch 3/4 is different; currently
ip_route_input_noref() is basically doing:
rcu_read_lock();
rcu_track_noref(&key1, &noref1, true);
rcu_read_unlock();
So the rcu lock there silence any RCU based check inside
ip_route_input_noref() but does not really protect the noref dst.
Please let me know if the above clarify the scenario.
Thanks,
Paolo
^ permalink raw reply
* [PATCH net] ppp: fix race in ppp device destruction
From: Guillaume Nault @ 2017-10-06 15:05 UTC (permalink / raw)
To: netdev; +Cc: Beniamino Galvani, linux-ppp, Paul Mackerras, David Ahern,
Gao Feng
ppp_release() tries to ensure that netdevices are unregistered before
decrementing the unit refcount and running ppp_destroy_interface().
This is all fine as long as the the device is unregistered by
ppp_release(): the unregister_netdevice() call, followed by
rtnl_unlock(), guarantee that the unregistration process completes
before rtnl_unlock() returns.
However, the device may be unregistered by other means (like
ppp_nl_dellink()). If this happens right before ppp_release() calling
rtnl_lock(), then ppp_release() has to wait for the concurrent
unregistration code to release the lock.
But rtnl_unlock() releases the lock before completing the device
unregistration process. This allows ppp_release() to proceed and
eventually call ppp_destroy_interface() before the unregistration
process completes. Calling free_netdev() on this partially unregistered
device will BUG():
------------[ cut here ]------------
kernel BUG at net/core/dev.c:8141!
invalid opcode: 0000 [#1] SMP
CPU: 1 PID: 1557 Comm: pppd Not tainted 4.14.0-rc2+ #4
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.10.2-1.fc26 04/01/2014
Call Trace:
ppp_destroy_interface+0xd8/0xe0 [ppp_generic]
ppp_disconnect_channel+0xda/0x110 [ppp_generic]
ppp_unregister_channel+0x5e/0x110 [ppp_generic]
pppox_unbind_sock+0x23/0x30 [pppox]
pppoe_connect+0x130/0x440 [pppoe]
SYSC_connect+0x98/0x110
? do_fcntl+0x2c0/0x5d0
SyS_connect+0xe/0x10
entry_SYSCALL_64_fastpath+0x1a/0xa5
RIP: free_netdev+0x107/0x110 RSP: ffffc28a40573d88
---[ end trace ed294ff0cc40eeff ]---
We could set the ->needs_free_netdev flag on PPP devices and move the
ppp_destroy_interface() logic in the ->priv_destructor() callback. But
that'd be quite intrusive as we'd first need to unlink from the other
channels and units that depend on the device (the ones that used the
PPPIOCCONNECT and PPPIOCATTACH ioctls).
Instead, we can just let the netdevice hold a reference on its
ppp_file. This reference is dropped in ->priv_destructor(), at the very
end of the unregistration process, so that neither ppp_release() nor
ppp_disconnect_channel() can call ppp_destroy_interface() in the interim.
Reported-by: Beniamino Galvani <bgalvani@redhat.com>
Fixes: 8cb775bc0a34 ("ppp: fix device unregistration upon netns deletion")
Signed-off-by: Guillaume Nault <g.nault@alphalink.fr>
---
drivers/net/ppp/ppp_generic.c | 20 ++++++++++++++++++++
1 file changed, 20 insertions(+)
diff --git a/drivers/net/ppp/ppp_generic.c b/drivers/net/ppp/ppp_generic.c
index c3f77e3b7819..e365866600ba 100644
--- a/drivers/net/ppp/ppp_generic.c
+++ b/drivers/net/ppp/ppp_generic.c
@@ -1339,7 +1339,17 @@ ppp_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats64)
static int ppp_dev_init(struct net_device *dev)
{
+ struct ppp *ppp;
+
netdev_lockdep_set_classes(dev);
+
+ ppp = netdev_priv(dev);
+ /* Let the netdevice take a reference on the ppp file. This ensures
+ * that ppp_destroy_interface() won't run before the device gets
+ * unregistered.
+ */
+ atomic_inc(&ppp->file.refcnt);
+
return 0;
}
@@ -1362,6 +1372,15 @@ static void ppp_dev_uninit(struct net_device *dev)
wake_up_interruptible(&ppp->file.rwait);
}
+static void ppp_dev_priv_destructor(struct net_device *dev)
+{
+ struct ppp *ppp;
+
+ ppp = netdev_priv(dev);
+ if (atomic_dec_and_test(&ppp->file.refcnt))
+ ppp_destroy_interface(ppp);
+}
+
static const struct net_device_ops ppp_netdev_ops = {
.ndo_init = ppp_dev_init,
.ndo_uninit = ppp_dev_uninit,
@@ -1387,6 +1406,7 @@ static void ppp_setup(struct net_device *dev)
dev->tx_queue_len = 3;
dev->type = ARPHRD_PPP;
dev->flags = IFF_POINTOPOINT | IFF_NOARP | IFF_MULTICAST;
+ dev->priv_destructor = ppp_dev_priv_destructor;
netif_keep_dst(dev);
}
--
2.14.2
^ permalink raw reply related
* [PATCH v2] mwifiex: Use put_unaligned_le32
From: Himanshu Jha @ 2017-10-06 15:04 UTC (permalink / raw)
To: amitkarwar
Cc: nishants, gbhat, huxm, kvalo, linux-wireless, netdev,
linux-kernel, Himanshu Jha
Use put_unaligned_le32 rather than using byte ordering function and
memcpy which makes code clear.
Also, add the header file where it is declared.
Done using Coccinelle and semantic patch used is :
@ rule1 @
identifier tmp; expression ptr,x; type T;
@@
- tmp = cpu_to_le32(x);
<+... when != tmp
- memcpy(ptr, (T)&tmp, ...);
+ put_unaligned_le32(x,ptr);
...+>
@ depends on rule1 @
type j; identifier tmp;
@@
- j tmp;
...when != tmp
Signed-off-by: Himanshu Jha <himanshujha199640@gmail.com>
---
v2:
* added correct header file.
drivers/net/wireless/marvell/mwifiex/cmdevt.c | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
diff --git a/drivers/net/wireless/marvell/mwifiex/cmdevt.c b/drivers/net/wireless/marvell/mwifiex/cmdevt.c
index 0edc5d6..e28e119 100644
--- a/drivers/net/wireless/marvell/mwifiex/cmdevt.c
+++ b/drivers/net/wireless/marvell/mwifiex/cmdevt.c
@@ -17,6 +17,7 @@
* this warranty disclaimer.
*/
+#include <asm/unaligned.h>
#include "decl.h"
#include "ioctl.h"
#include "util.h"
@@ -183,7 +184,6 @@ static int mwifiex_dnld_cmd_to_fw(struct mwifiex_private *priv,
uint16_t cmd_code;
uint16_t cmd_size;
unsigned long flags;
- __le32 tmp;
if (!adapter || !cmd_node)
return -1;
@@ -249,9 +249,9 @@ static int mwifiex_dnld_cmd_to_fw(struct mwifiex_private *priv,
mwifiex_dbg_dump(adapter, CMD_D, "cmd buffer:", host_cmd, cmd_size);
if (adapter->iface_type == MWIFIEX_USB) {
- tmp = cpu_to_le32(MWIFIEX_USB_TYPE_CMD);
skb_push(cmd_node->cmd_skb, MWIFIEX_TYPE_LEN);
- memcpy(cmd_node->cmd_skb->data, &tmp, MWIFIEX_TYPE_LEN);
+ put_unaligned_le32(MWIFIEX_USB_TYPE_CMD,
+ cmd_node->cmd_skb->data);
adapter->cmd_sent = true;
ret = adapter->if_ops.host_to_card(adapter,
MWIFIEX_USB_EP_CMD_EVENT,
@@ -317,7 +317,6 @@ static int mwifiex_dnld_sleep_confirm_cmd(struct mwifiex_adapter *adapter)
(struct mwifiex_opt_sleep_confirm *)
adapter->sleep_cfm->data;
struct sk_buff *sleep_cfm_tmp;
- __le32 tmp;
priv = mwifiex_get_priv(adapter, MWIFIEX_BSS_ROLE_ANY);
@@ -342,8 +341,7 @@ static int mwifiex_dnld_sleep_confirm_cmd(struct mwifiex_adapter *adapter)
+ MWIFIEX_TYPE_LEN);
skb_put(sleep_cfm_tmp, sizeof(struct mwifiex_opt_sleep_confirm)
+ MWIFIEX_TYPE_LEN);
- tmp = cpu_to_le32(MWIFIEX_USB_TYPE_CMD);
- memcpy(sleep_cfm_tmp->data, &tmp, MWIFIEX_TYPE_LEN);
+ put_unaligned_le32(MWIFIEX_USB_TYPE_CMD, sleep_cfm_tmp->data);
memcpy(sleep_cfm_tmp->data + MWIFIEX_TYPE_LEN,
adapter->sleep_cfm->data,
sizeof(struct mwifiex_opt_sleep_confirm));
--
2.7.4
^ permalink raw reply related
* Re: [net-next V4 PATCH 1/5] bpf: introduce new bpf cpu map type BPF_MAP_TYPE_CPUMAP
From: Daniel Borkmann @ 2017-10-06 14:52 UTC (permalink / raw)
To: Jesper Dangaard Brouer
Cc: netdev, jakub.kicinski, Michael S. Tsirkin, pavel.odintsov,
Jason Wang, mchan, John Fastabend, peter.waskiewicz.jr,
Daniel Borkmann, Alexei Starovoitov, Andy Gospodarek,
Tobias Klauser
In-Reply-To: <20171006125008.676d5eaf@redhat.com>
On 10/06/2017 12:50 PM, Jesper Dangaard Brouer wrote:
> On Thu, 05 Oct 2017 11:40:15 +0200
> Daniel Borkmann <daniel@iogearbox.net> wrote:
>> On 10/04/2017 02:03 PM, Jesper Dangaard Brouer wrote:
>> [...]
>>> +#define CPU_MAP_BULK_SIZE 8 /* 8 == one cacheline on 64-bit archs */
>>> +struct xdp_bulk_queue {
>>> + void *q[CPU_MAP_BULK_SIZE];
>>> + unsigned int count;
>>> +};
>>> +
>>> +/* Struct for every remote "destination" CPU in map */
>>> +struct bpf_cpu_map_entry {
>>> + u32 cpu; /* kthread CPU and map index */
>>> + int map_id; /* Back reference to map */
>>
>> map_id is not used here if I read it correctly? We should
>> then remove it.
>
> It is actually used in a later patch. Notice, there is no unused
> members in the final patch. I did consider adding back in the later
> patch, but it was annoying to during the devel and split-up patch
> phase, as it creates conflicts when I move between the different
> patches, that need to modify this struct. Thus, I choose to keep the
> end-struct in this cpumap-base-patch. If you insist, I can go though
> the patch-stack and carefully introduce changes to the struct in steps?
It would be great to have every patch as self-contained as possible
since it otherwise makes reviewing much more time consuming to check
through the other patches for possible usage patterns, I noticed you
are using it for the tracepoints in patch 4/5 to dump map_id. Would
definitely be good if you could avoid such split in future sets.
[...]
>>> +void __cpu_map_queue_destructor(void *ptr)
>>> +{
>>> + /* For now, just catch this as an error */
>>> + if (!ptr)
>>> + return;
>>> + pr_err("ERROR: %s() cpu_map queue was not empty\n", __func__);
>>
>> Can you elaborate on this "for now" condition? Is this a race
>> when kthread doesn't consume queue on thread exit, or should it
>> be impossible to trigger (should it then be replaced with a
>> 'if (WARN_ON_ONCE(ptr)) page_frag_free(ptr)' and a more elaborate
>> comment)?
>
> The "for now" is an old comment while developing and testing this.
> In this final state of the patchset it _should_ not be possible to
> trigger this situation. I like your idea of replacing it with a
> WARN_ON_ONCE. (as it might be good to keep in some form, as it would
> catch is someone changing the code which breaks the RCU+WQ+kthread
> tear-down procedure).
Ok.
[...]
>>> + /* Updating qsize cause re-allocation of bpf_cpu_map_entry */
>>> + rcpu = __cpu_map_entry_alloc(qsize, key_cpu, map->id);
>>> + if (!rcpu)
>>> + return -ENOMEM;
>>> + }
>>> + rcu_read_lock();
>>> + __cpu_map_entry_replace(cmap, key_cpu, rcpu);
>>> + rcu_read_unlock();
>>> + return 0;
>>
>> You need to update verifier such that this function cannot be called
>> out of an BPF program,
>
> In the example BPF program, I do a lookup into the map, but only to
> verify that an entry exist (I don't look at the value). I would like
> to support such usage.
Ok, put comment below.
>> otherwise it would be possible under full RCU
>> read context, which is explicitly avoided here and also it would otherwise
>> be allowed for other maps of different type as well, which needs to
>> be avoided.
>
> Sorry, I don't follow this.
What I meant is that check_map_func_compatibility() should check for
BPF_MAP_TYPE_CPUMAP and only allow func_id of BPF_FUNC_redirect_map
and BPF_FUNC_map_lookup_elem to be used, which I haven't seen the set
restricting it to. Some of your later patches do this for the helper
BPF_FUNC_redirect_map but the important point is that map updates
wouldn't be done out of the BPF program itself, but rather from user
space control path given they can't be done under full RCU read lock
context if I read this correctly (which the programs run in though).
[...]
>>> +static void *cpu_map_lookup_elem(struct bpf_map *map, void *key)
>>> +{
>>> + struct bpf_cpu_map_entry *rcpu =
>>> + __cpu_map_lookup_elem(map, *(u32 *)key);
>>> +
>>> + return rcpu ? &rcpu->qsize : NULL;
>>
>> The qsize doesn't seem used anywhere else besides here, but you
>> probably should update verifier such that this cannot be called
>> out of the BPF program, which could then mangle qsize value.
>
> It is true that the BPF prog can modify this qsize value, but it's not
> the authoritative value, so it doesn't really matter.
>
> As I said above, I do want to do a lookup from a BPF program. To allow
> to BPF program to know, if an entry is valid, else it will blindly
> send to a cpu destination. Maybe bpf_prog's just have to use a
> map-on-the-side to coordinate this(?), but then a sysadm modifying the
> real cpumap will be invisible to the program.
>
> Maybe we should just disable BPF-progs from reading this in the first
> iteration? It would allow for more advanced usage schemes later..
Okay, what you could do is the following to prevent unintended value
updates. Just read out the qsize from the value and put that into a
per-cpu scratch buffer that gets returned instead of the map value,
so even if the scratch buffer gets mangled, it's okay because the
actual map value doesn't. Verifier still checks the map value access
bounds for that one.
> One crazy idea is to have the cpu_map_lookup_elem() return if any
> packets are in-flight on this cpu-queue. (Making it easier to avoid OoO
> packets, when switching target CPU, but it can also be implemented by
> the BPF-programmer herself via maps, although via some extra atomic
> cost).
>
>>> +}
>>> +
>>> +static int cpu_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
>>> +{
>>> + struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
>>> + u32 index = key ? *(u32 *)key : U32_MAX;
>>> + u32 *next = next_key;
>>> +
>>> + if (index >= cmap->map.max_entries) {
>>> + *next = 0;
>>> + return 0;
>>> + }
>>> +
>>> + if (index == cmap->map.max_entries - 1)
>>> + return -ENOENT;
>>> + *next = index + 1;
>>> + return 0;
>>> +}
>
> I would have liked to have implemented next_key so it only returned the
> next valid cpu_entry, and used it as a simple round-robin scheduler.
> But AFAIK the next_key function is not allowed from bpf_prog's, right?
Hm, true we don't export this as a helper right now, I currently don't
see a reason why we couldn't. For the array map, the get_next_key is
probably not too useful given we only return key + 1. For user space it
might help for iterating/dumping the map though. Probably one could
enable it and add a flag to search the next valid entry from the map,
though issue could well be that this might need to iterate/test millions
of empty slots until you get to the next valid one which might not be
suitable in a generic case to do out of a BPF prog, perhaps a second
map with keys to round robin over might help.
Cheers,
Daniel
^ permalink raw reply
* Re: [PATCH 4/4] tcp: avoid noref dst leak on input path
From: Eric Dumazet @ 2017-10-06 14:37 UTC (permalink / raw)
To: Paolo Abeni
Cc: linux-kernel, Paul E. McKenney, Josh Triplett, Steven Rostedt,
David S. Miller, Eric Dumazet, Hannes Frederic Sowa, netdev
In-Reply-To: <77fd3b021d5a2f6dd4ad6b08e64f0643e3af95f4.1507294365.git.pabeni@redhat.com>
On Fri, 2017-10-06 at 14:57 +0200, Paolo Abeni wrote:
> Enabling CONFIG_RCU_NOREF_DEBUG gives the following splat when
> processing tcp packets:
>
> to-be-untracked noref entity ffff942cb71ea300 not found in cache
> ------------[ cut here ]------------
> WARNING: CPU: 24 PID: 178 at kernel/rcu/noref_debug.c:54 rcu_track_noref+0xa4/0xf0
> Modules linked in: intel_rapl sb_edac x86_pkg_temp_thermal intel_powerclamp coretemp kvm_intel kvm irqbypass crct10dif_pclmul crc32_pclmul ghash_clmulni_intel pcbc aesni_intel crypto_simd glue_helper cryptd iTCO_wdt ipmi_ssif mei_me iTCO_vendor_support mei dcdbas lpc_ich ipmi_si mxm_wmi sg pcspkr ipmi_devintf ipmi_msghandler acpi_power_meter shpchp wmi nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod mgag200 drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm igb drm ixgbe mdio crc32c_intel ahci ptp i2c_algo_bit libahci pps_core i2c_core libata dca dm_mirror dm_region_hash dm_log dm_mod
> CPU: 24 PID: 178 Comm: ksoftirqd/24 Not tainted 4.14.0-rc1.noref_route+ #1610
> Hardware name: Dell Inc. PowerEdge R730/072T6D, BIOS 2.4.3 01/17/2017
> task: ffff940e48300000 task.stack: ffffaec406a20000
> RIP: 0010:rcu_track_noref+0xa4/0xf0
> RSP: 0018:ffffaec406a238e0 EFLAGS: 00010246
> RAX: 0000000000000040 RBX: 0000000000000000 RCX: 0000000000000002
> RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000292
> RBP: ffffaec406a238e0 R08: 0000000000000000 R09: 0000000000000000
> R10: 0000000000000001 R11: 0000000000000003 R12: 0000000000000000
> R13: ffff942cb5110000 R14: 000000000000fe88 R15: ffff942cb1a20200
> FS: 0000000000000000(0000) GS:ffff942cbee00000(0000) knlGS:0000000000000000
> CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
> CR2: 00007febc072d140 CR3: 0000001feebd6002 CR4: 00000000003606e0
> DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
> DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
> Call Trace:
> tcp_data_queue+0x82a/0xce0
That is strange, since tcp_data_queue() starts with
skb_dst_drop(skb);
So this stack trace looks suspicious.
> tcp_rcv_established+0x283/0x570
> tcp_v4_do_rcv+0x102/0x1e0
> tcp_v4_rcv+0xa9e/0xc10
> ip_local_deliver_finish+0x128/0x380
> ? ip_local_deliver_finish+0x41/0x380
> ip_local_deliver+0x74/0x230
> ip_rcv_finish+0x105/0x5e0
> ip_rcv+0x2a7/0x540
> __netif_receive_skb_core+0x3b9/0xe10
> ? netif_receive_skb_internal+0x40/0x390
> __netif_receive_skb+0x18/0x60
> netif_receive_skb_internal+0x8d/0x390
> ? netif_receive_skb_internal+0x40/0x390
> napi_gro_complete+0x127/0x1d0
> ? napi_gro_complete+0x2a/0x1d0
> napi_gro_flush+0x5f/0x80
> napi_complete_done+0x96/0x100
> ixgbe_poll+0x5f8/0x7c0 [ixgbe]
> net_rx_action+0x27d/0x520
> __do_softirq+0xd1/0x4f5
> run_ksoftirqd+0x25/0x70
> smpboot_thread_fn+0x11a/0x1f0
> kthread+0x155/0x190
> ? sort_range+0x30/0x30
> ? kthread_create_on_node+0x70/0x70
> ret_from_fork+0x2a/0x40
> Code: e9 83 c2 01 48 83 c0 18 83 fa 07 75 ef 80 3d b0 e5 ff 00 00 75 d2 48 c7 c7 50 54 07 9d 31 c0 c6 05 9e e5 ff 00 01 e8 ef af fe ff <0f> ff 5d c3 80 3d 8f e5 ff 00 00 75 b0 48 c7 c7 00 54 07 9d 31
>
> we must clear the skb dst before enqueuing the skb somewhere,
> but currently the dst entry for packets delivered
> via tcp_rcv_established() -> tcp_queue_rcv() is not cleared.
>
> Fix it by adding the explicit drop in tcp_queue_rcv() and moving
> the current skb_dst_drop() just before the other enqueuing
> operation, do avoid unneeded double skb_dst_drop() for some
> path.
>
> The leak itself is not harmful, because the tcp recvmsg() code
> should not access such info.
>
> Signed-off-by: Paolo Abeni <pabeni@redhat.com>
> ---
> net/ipv4/tcp_input.c | 5 ++++-
> 1 file changed, 4 insertions(+), 1 deletion(-)
>
> diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c
> index c5d7656beeee..bf4e17edfe7a 100644
> --- a/net/ipv4/tcp_input.c
> +++ b/net/ipv4/tcp_input.c
> @@ -4422,6 +4422,9 @@ static void tcp_data_queue_ofo(struct sock *sk, struct sk_buff *skb)
> return;
> }
>
> + /* drop the -possibly noref - dst before delivery the skb to ofo tree */
> + skb_dst_drop(skb);
> +
> /* Stash tstamp to avoid being stomped on by rbnode */
> if (TCP_SKB_CB(skb)->has_rxtstamp)
> TCP_SKB_CB(skb)->swtstamp = skb->tstamp;
> @@ -4560,6 +4563,7 @@ static int __must_check tcp_queue_rcv(struct sock *sk, struct sk_buff *skb, int
> skb, fragstolen)) ? 1 : 0;
> tcp_rcv_nxt_update(tcp_sk(sk), TCP_SKB_CB(skb)->end_seq);
> if (!eaten) {
> + skb_dst_drop(skb);
> __skb_queue_tail(&sk->sk_receive_queue, skb);
> skb_set_owner_r(skb, sk);
> }
> @@ -4626,7 +4630,6 @@ static void tcp_data_queue(struct sock *sk, struct sk_buff *skb)
> __kfree_skb(skb);
> return;
> }
> - skb_dst_drop(skb);
> __skb_pull(skb, tcp_hdr(skb)->doff * 4);
>
> tcp_ecn_accept_cwr(tp, skb);
^ permalink raw reply
* Re: [PATCH] net/mlx4_core: Convert timers to use timer_setup()
From: Kees Cook @ 2017-10-06 14:26 UTC (permalink / raw)
To: Leon Romanovsky
Cc: LKML, Tariq Toukan, Network Development, linux-rdma,
Thomas Gleixner
In-Reply-To: <20171005093802.GK25829-U/DQcQFIOTAAJjI8aNfphQ@public.gmane.org>
On Thu, Oct 5, 2017 at 2:38 AM, Leon Romanovsky <leon-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org> wrote:
> On Wed, Oct 04, 2017 at 05:51:54PM -0700, Kees Cook wrote:
>> In preparation for unconditionally passing the struct timer_list pointer to
>> all timer callbacks, switch to using the new timer_setup() and from_timer()
>> to pass the timer pointer explicitly.
>>
>> Cc: Tariq Toukan <tariqt-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
>> Cc: netdev-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
>> Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
>> Cc: Thomas Gleixner <tglx-hfZtesqFncYOwBW4kG4KsQ@public.gmane.org>
>> Signed-off-by: Kees Cook <keescook-F7+t8E8rja9g9hUCZPvPmw@public.gmane.org>
>> ---
>> This requires commit 686fef928bba ("timer: Prepare to change timer
>> callback argument type") in v4.14-rc3, but should be otherwise
>> stand-alone.
>> ---
>> drivers/net/ethernet/mellanox/mlx4/catas.c | 10 ++++------
>> 1 file changed, 4 insertions(+), 6 deletions(-)
>>
>
> Hi Kees,
>
> In RDMA, we had very similar patch [1] to your patch series, but it converts to
> setup_timer, while you are converting to timer_setup.
>
> Which conversion is the right one?
The timer_setup() is the new API designed to eliminate the .data field
and pass in the struct timer_list to callbacks. (Please take the
timer_setup() change instead.)
Thanks!
-Kees
>
> [1] https://patchwork.kernel.org/patch/9980701/
>
> Thanks
>
>> diff --git a/drivers/net/ethernet/mellanox/mlx4/catas.c b/drivers/net/ethernet/mellanox/mlx4/catas.c
>> index 53daa6ca5d83..e2b6b0cac1ac 100644
>> --- a/drivers/net/ethernet/mellanox/mlx4/catas.c
>> +++ b/drivers/net/ethernet/mellanox/mlx4/catas.c
>> @@ -231,10 +231,10 @@ static void dump_err_buf(struct mlx4_dev *dev)
>> i, swab32(readl(priv->catas_err.map + i)));
>> }
>>
>> -static void poll_catas(unsigned long dev_ptr)
>> +static void poll_catas(struct timer_list *t)
>> {
>> - struct mlx4_dev *dev = (struct mlx4_dev *) dev_ptr;
>> - struct mlx4_priv *priv = mlx4_priv(dev);
>> + struct mlx4_priv *priv = from_timer(priv, t, catas_err.timer);
>> + struct mlx4_dev *dev = &priv->dev;
>> u32 slave_read;
>>
>> if (mlx4_is_slave(dev)) {
>> @@ -277,7 +277,7 @@ void mlx4_start_catas_poll(struct mlx4_dev *dev)
>> phys_addr_t addr;
>>
>> INIT_LIST_HEAD(&priv->catas_err.list);
>> - init_timer(&priv->catas_err.timer);
>> + timer_setup(&priv->catas_err.timer, poll_catas, 0);
>> priv->catas_err.map = NULL;
>>
>> if (!mlx4_is_slave(dev)) {
>> @@ -293,8 +293,6 @@ void mlx4_start_catas_poll(struct mlx4_dev *dev)
>> }
>> }
>>
>> - priv->catas_err.timer.data = (unsigned long) dev;
>> - priv->catas_err.timer.function = poll_catas;
>> priv->catas_err.timer.expires =
>> round_jiffies(jiffies + MLX4_CATAS_POLL_INTERVAL);
>> add_timer(&priv->catas_err.timer);
>> --
>> 2.7.4
>>
>>
>> --
>> Kees Cook
>> Pixel Security
>> --
>> To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
>> the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
>> More majordomo info at http://vger.kernel.org/majordomo-info.html
--
Kees Cook
Pixel Security
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* Re: [PATCH 1/4] rcu: introduce noref debug
From: Steven Rostedt @ 2017-10-06 14:13 UTC (permalink / raw)
To: Paolo Abeni
Cc: linux-kernel, Paul E. McKenney, Josh Triplett, David S. Miller,
Eric Dumazet, Hannes Frederic Sowa, netdev
In-Reply-To: <bc39fa36c0765e6d2f7dab82e4c6460989669957.1507294365.git.pabeni@redhat.com>
On Fri, 6 Oct 2017 14:57:46 +0200
Paolo Abeni <pabeni@redhat.com> wrote:
> We currently lack a debugging infrastructure to ensure that
> long-lived noref dst are properly handled - namely dropped
> or converted to a refcounted version before escaping the current
> RCU section.
>
> This changeset implements such infra, tracking the noref pointer
> on a per CPU store and checking that all noref tracked at any
> given RCU nesting level are cleared before leaving such RCU
> section.
>
> Each noref entity is identified using the entity pointer and an
> additional, optional, key/opaque pointer. This is needed to cope
> with usage case scenario where the same noref entity is stored
> in different places (e.g. noref dst on skb_clone()).
>
> To keep the patch/implementation simple RCU_NOREF_DEBUG depends
> on PREEMPT_RCU=n in Kconfig.
>
> Signed-off-by: Paolo Abeni <pabeni@redhat.com>
> ---
> include/linux/rcupdate.h | 11 ++++++
> kernel/rcu/Kconfig.debug | 15 ++++++++
> kernel/rcu/Makefile | 1 +
> kernel/rcu/noref_debug.c | 89 ++++++++++++++++++++++++++++++++++++++++++++++++
> 4 files changed, 116 insertions(+)
> create mode 100644 kernel/rcu/noref_debug.c
>
> diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
> index de50d8a4cf41..20c1ce08e3eb 100644
> --- a/include/linux/rcupdate.h
> +++ b/include/linux/rcupdate.h
> @@ -60,6 +60,15 @@ void call_rcu_sched(struct rcu_head *head, rcu_callback_t func);
> void synchronize_sched(void);
> void rcu_barrier_tasks(void);
>
> +#ifdef CONFIG_RCU_NOREF_DEBUG
> +void rcu_track_noref(void *key, void *noref, bool track);
> +void __rcu_check_noref(void);
> +
> +#else
> +static inline void rcu_track_noref(void *key, void *noref, bool add) { }
> +static inline void __rcu_check_noref(void) { }
> +#endif
> +
> #ifdef CONFIG_PREEMPT_RCU
>
> void __rcu_read_lock(void);
> @@ -85,6 +94,7 @@ static inline void __rcu_read_lock(void)
>
> static inline void __rcu_read_unlock(void)
> {
> + __rcu_check_noref();
> if (IS_ENABLED(CONFIG_PREEMPT_COUNT))
> preempt_enable();
> }
> @@ -723,6 +733,7 @@ static inline void rcu_read_unlock_bh(void)
> "rcu_read_unlock_bh() used illegally while idle");
> rcu_lock_release(&rcu_bh_lock_map);
> __release(RCU_BH);
> + __rcu_check_noref();
> local_bh_enable();
> }
>
> diff --git a/kernel/rcu/Kconfig.debug b/kernel/rcu/Kconfig.debug
> index 0ec7d1d33a14..6c7f52a3e809 100644
> --- a/kernel/rcu/Kconfig.debug
> +++ b/kernel/rcu/Kconfig.debug
> @@ -68,6 +68,21 @@ config RCU_TRACE
> Say Y here if you want to enable RCU tracing
> Say N if you are unsure.
>
> +config RCU_NOREF_DEBUG
> + bool "Debugging for RCU-protected noref entries"
> + depends on PREEMPT_RCU=n
> + select PREEMPT_COUNT
> + default n
> + help
> + In case of long lasting rcu_read_lock sections this debug
> + feature enables one to ensure that no rcu managed dereferenced
> + data leaves the locked section. One use case is the tracking
> + of dst_entries in struct sk_buff ->dst, which is used to pass
> + the dst_entry around in the whole stack while under RCU lock.
> +
> + Say Y here if you want to enable noref RCU debugging
> + Say N if you are unsure.
> +
> config RCU_EQS_DEBUG
> bool "Provide debugging asserts for adding NO_HZ support to an arch"
> depends on DEBUG_KERNEL
> diff --git a/kernel/rcu/Makefile b/kernel/rcu/Makefile
> index 13c0fc852767..c67d7c65c582 100644
> --- a/kernel/rcu/Makefile
> +++ b/kernel/rcu/Makefile
> @@ -11,3 +11,4 @@ obj-$(CONFIG_TREE_RCU) += tree.o
> obj-$(CONFIG_PREEMPT_RCU) += tree.o
> obj-$(CONFIG_TINY_RCU) += tiny.o
> obj-$(CONFIG_RCU_NEED_SEGCBLIST) += rcu_segcblist.o
> +obj-$(CONFIG_RCU_NOREF_DEBUG) += noref_debug.o
> \ No newline at end of file
> diff --git a/kernel/rcu/noref_debug.c b/kernel/rcu/noref_debug.c
> new file mode 100644
> index 000000000000..ae2e104b11d6
> --- /dev/null
> +++ b/kernel/rcu/noref_debug.c
> @@ -0,0 +1,89 @@
> +#include <linux/bug.h>
> +#include <linux/percpu.h>
> +#include <linux/skbuff.h>
> +#include <linux/bitops.h>
> +
> +#define NOREF_MAX 7
> +struct noref_entry {
> + void *noref;
> + void *key;
> + int nesting;
> +};
> +
> +struct noref_cache {
> + struct noref_entry store[NOREF_MAX];
> +};
> +
> +static DEFINE_PER_CPU(struct noref_cache, per_cpu_noref_cache);
> +
> +static int noref_cache_lookup(struct noref_cache *cache, void *noref, void *key)
> +{
> + int i;
> +
> + for (i = 0; i < NOREF_MAX; ++i)
> + if (cache->store[i].noref == noref &&
> + cache->store[i].key == key)
> + return i;
> +
> + return -1;
> +}
> +
Please add a comment above this function on how to use it.
-- Steve
> +void rcu_track_noref(void *key, void *noref, bool track)
> +{
> + struct noref_cache *cache = this_cpu_ptr(&per_cpu_noref_cache);
> + int i;
> +
> + if (track) {
> + /* find the first empty slot */
> + i = noref_cache_lookup(cache, NULL, 0);
> + if (i < 0) {
> + WARN_ONCE(1, "can't find empty an slot to track a noref"
> + " noref tracking will be inaccurate");
> + return;
> + }
> +
> + cache->store[i].noref = noref;
> + cache->store[i].key = key;
> + cache->store[i].nesting = preempt_count();
> + return;
> + }
> +
> + i = noref_cache_lookup(cache, noref, key);
> + if (i == -1) {
> + WARN_ONCE(1, "to-be-untracked noref entity %p not found in "
> + "cache\n", noref);
> + return;
> + }
> +
> + cache->store[i].noref = NULL;
> + cache->store[i].key = NULL;
> + cache->store[i].nesting = 0;
> +}
> +EXPORT_SYMBOL_GPL(rcu_track_noref);
> +
> +void __rcu_check_noref(void)
> +{
> + struct noref_cache *cache = this_cpu_ptr(&per_cpu_noref_cache);
> + char *cur, buf[strlen("0xffffffffffffffff") * NOREF_MAX + 1];
> + int nesting = preempt_count();
> + int i, ret, cnt = 0;
> +
> + cur = buf;
> + for (i = 0; i < NOREF_MAX; ++i) {
> + if (!cache->store[i].noref ||
> + cache->store[i].nesting != nesting)
> + continue;
> +
> + cnt++;
> + ret = sprintf(cur, " %p", cache->store[i].noref);
> + if (ret >= 0)
> + cur += ret;
> + cache->store[i].noref = NULL;
> + cache->store[i].key = NULL;
> + cache->store[i].nesting = 0;
> + }
> +
> + WARN_ONCE(cnt, "%d noref entities escaped an RCU section, "
> + "nesting %d, leaked noref list %s", cnt, nesting, buf);
> +}
> +EXPORT_SYMBOL_GPL(__rcu_check_noref);
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox