* [PATCH mlx5-next v2 02/13] net/mlx5: Use event mask based on device capabilities
From: Leon Romanovsky @ 2019-06-30 16:23 UTC (permalink / raw)
To: Doug Ledford, Jason Gunthorpe
Cc: Leon Romanovsky, RDMA mailing list, Saeed Mahameed, Yishai Hadas,
linux-netdev
In-Reply-To: <20190630162334.22135-1-leon@kernel.org>
From: Yishai Hadas <yishaih@mellanox.com>
Use the reported device capabilities for the supported user events (i.e.
affiliated and un-affiliated) to set the EQ mask.
As the event mask can be up to 256 defined by 4 entries of u64 change
the applicable code to work accordingly.
Signed-off-by: Yishai Hadas <yishaih@mellanox.com>
Acked-by: Saeed Mahameed <saeedm@mellanox.com>
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
---
drivers/infiniband/hw/mlx5/odp.c | 2 +-
drivers/net/ethernet/mellanox/mlx5/core/eq.c | 40 ++++++++++++++++----
drivers/net/ethernet/mellanox/mlx5/core/fw.c | 6 +++
include/linux/mlx5/device.h | 6 ++-
include/linux/mlx5/eq.h | 2 +-
include/linux/mlx5/mlx5_ifc.h | 13 +++++--
6 files changed, 55 insertions(+), 14 deletions(-)
diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c
index 600fe23e2eae..346a7ffd63aa 100644
--- a/drivers/infiniband/hw/mlx5/odp.c
+++ b/drivers/infiniband/hw/mlx5/odp.c
@@ -1559,9 +1559,9 @@ mlx5_ib_create_pf_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq)
eq->irq_nb.notifier_call = mlx5_ib_eq_pf_int;
param = (struct mlx5_eq_param) {
.irq_index = 0,
- .mask = 1 << MLX5_EVENT_TYPE_PAGE_FAULT,
.nent = MLX5_IB_NUM_PF_EQE,
};
+ param.mask[0] = 1ull << MLX5_EVENT_TYPE_PAGE_FAULT;
eq->core = mlx5_eq_create_generic(dev->mdev, ¶m);
if (IS_ERR(eq->core)) {
err = PTR_ERR(eq->core);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c
index 8000d2a4a7e2..33f78d4d3724 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c
@@ -256,6 +256,7 @@ create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq,
int inlen;
u32 *in;
int err;
+ int i;
/* Init CQ table */
memset(cq_table, 0, sizeof(*cq_table));
@@ -283,10 +284,12 @@ create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq,
mlx5_fill_page_array(&eq->buf, pas);
MLX5_SET(create_eq_in, in, opcode, MLX5_CMD_OP_CREATE_EQ);
- if (!param->mask && MLX5_CAP_GEN(dev, log_max_uctx))
+ if (!param->mask[0] && MLX5_CAP_GEN(dev, log_max_uctx))
MLX5_SET(create_eq_in, in, uid, MLX5_SHARED_RESOURCE_UID);
- MLX5_SET64(create_eq_in, in, event_bitmask, param->mask);
+ for (i = 0; i < 4; i++)
+ MLX5_ARRAY_SET64(create_eq_in, in, event_bitmask, i,
+ param->mask[i]);
eqc = MLX5_ADDR_OF(create_eq_in, in, eq_context_entry);
MLX5_SET(eqc, eqc, log_eq_size, ilog2(eq->nent));
@@ -507,7 +510,23 @@ static int cq_err_event_notifier(struct notifier_block *nb,
return NOTIFY_OK;
}
-static u64 gather_async_events_mask(struct mlx5_core_dev *dev)
+static void gather_user_async_events(struct mlx5_core_dev *dev, u64 mask[4])
+{
+ __be64 *user_unaffiliated_events;
+ __be64 *user_affiliated_events;
+ int i;
+
+ user_affiliated_events =
+ MLX5_CAP_DEV_EVENT(dev, user_affiliated_events);
+ user_unaffiliated_events =
+ MLX5_CAP_DEV_EVENT(dev, user_unaffiliated_events);
+
+ for (i = 0; i < 4; i++)
+ mask[i] |= be64_to_cpu(user_affiliated_events[i] |
+ user_unaffiliated_events[i]);
+}
+
+static void gather_async_events_mask(struct mlx5_core_dev *dev, u64 mask[4])
{
u64 async_event_mask = MLX5_ASYNC_EVENT_MASK;
@@ -544,7 +563,10 @@ static u64 gather_async_events_mask(struct mlx5_core_dev *dev)
async_event_mask |=
(1ull << MLX5_EVENT_TYPE_ESW_FUNCTIONS_CHANGED);
- return async_event_mask;
+ mask[0] = async_event_mask;
+
+ if (MLX5_CAP_GEN(dev, event_cap))
+ gather_user_async_events(dev, mask);
}
static int create_async_eqs(struct mlx5_core_dev *dev)
@@ -559,9 +581,10 @@ static int create_async_eqs(struct mlx5_core_dev *dev)
table->cmd_eq.irq_nb.notifier_call = mlx5_eq_async_int;
param = (struct mlx5_eq_param) {
.irq_index = 0,
- .mask = 1ull << MLX5_EVENT_TYPE_CMD,
.nent = MLX5_NUM_CMD_EQE,
};
+
+ param.mask[0] = 1ull << MLX5_EVENT_TYPE_CMD;
err = create_async_eq(dev, &table->cmd_eq.core, ¶m);
if (err) {
mlx5_core_warn(dev, "failed to create cmd EQ %d\n", err);
@@ -577,9 +600,10 @@ static int create_async_eqs(struct mlx5_core_dev *dev)
table->async_eq.irq_nb.notifier_call = mlx5_eq_async_int;
param = (struct mlx5_eq_param) {
.irq_index = 0,
- .mask = gather_async_events_mask(dev),
.nent = MLX5_NUM_ASYNC_EQE,
};
+
+ gather_async_events_mask(dev, param.mask);
err = create_async_eq(dev, &table->async_eq.core, ¶m);
if (err) {
mlx5_core_warn(dev, "failed to create async EQ %d\n", err);
@@ -595,9 +619,10 @@ static int create_async_eqs(struct mlx5_core_dev *dev)
table->pages_eq.irq_nb.notifier_call = mlx5_eq_async_int;
param = (struct mlx5_eq_param) {
.irq_index = 0,
- .mask = 1 << MLX5_EVENT_TYPE_PAGE_REQUEST,
.nent = /* TODO: sriov max_vf + */ 1,
};
+
+ param.mask[0] = 1ull << MLX5_EVENT_TYPE_PAGE_REQUEST;
err = create_async_eq(dev, &table->pages_eq.core, ¶m);
if (err) {
mlx5_core_warn(dev, "failed to create pages EQ %d\n", err);
@@ -789,7 +814,6 @@ static int create_comp_eqs(struct mlx5_core_dev *dev)
eq->irq_nb.notifier_call = mlx5_eq_comp_int;
param = (struct mlx5_eq_param) {
.irq_index = vecidx,
- .mask = 0,
.nent = nent,
};
err = create_map_eq(dev, &eq->core, ¶m);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw.c b/drivers/net/ethernet/mellanox/mlx5/core/fw.c
index 1ab6f7e3bec6..05367f15c3a7 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fw.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fw.c
@@ -202,6 +202,12 @@ int mlx5_query_hca_caps(struct mlx5_core_dev *dev)
return err;
}
+ if (MLX5_CAP_GEN(dev, event_cap)) {
+ err = mlx5_core_get_caps(dev, MLX5_CAP_DEV_EVENT);
+ if (err)
+ return err;
+ }
+
return 0;
}
diff --git a/include/linux/mlx5/device.h b/include/linux/mlx5/device.h
index 5e760067ac41..0d1abe097627 100644
--- a/include/linux/mlx5/device.h
+++ b/include/linux/mlx5/device.h
@@ -351,7 +351,7 @@ enum mlx5_event {
MLX5_EVENT_TYPE_DEVICE_TRACER = 0x26,
- MLX5_EVENT_TYPE_MAX = MLX5_EVENT_TYPE_DEVICE_TRACER + 1,
+ MLX5_EVENT_TYPE_MAX = 0x100,
};
enum {
@@ -1077,6 +1077,7 @@ enum mlx5_cap_type {
MLX5_CAP_DEBUG,
MLX5_CAP_RESERVED_14,
MLX5_CAP_DEV_MEM,
+ MLX5_CAP_DEV_EVENT = 0x14,
/* NUM OF CAP Types */
MLX5_CAP_NUM
};
@@ -1255,6 +1256,9 @@ enum mlx5_qcam_feature_groups {
#define MLX5_CAP64_DEV_MEM(mdev, cap)\
MLX5_GET64(device_mem_cap, mdev->caps.hca_cur[MLX5_CAP_DEV_MEM], cap)
+#define MLX5_CAP_DEV_EVENT(mdev, cap)\
+ MLX5_ADDR_OF(device_event_cap, (mdev)->caps.hca_cur[MLX5_CAP_DEV_EVENT], cap)
+
enum {
MLX5_CMD_STAT_OK = 0x0,
MLX5_CMD_STAT_INT_ERR = 0x1,
diff --git a/include/linux/mlx5/eq.h b/include/linux/mlx5/eq.h
index 70e16dcfb4c4..e49d8c0d4f26 100644
--- a/include/linux/mlx5/eq.h
+++ b/include/linux/mlx5/eq.h
@@ -15,7 +15,7 @@ struct mlx5_core_dev;
struct mlx5_eq_param {
u8 irq_index;
int nent;
- u64 mask;
+ u64 mask[4];
};
struct mlx5_eq *
diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h
index 16348528fef6..3ef716c054c2 100644
--- a/include/linux/mlx5/mlx5_ifc.h
+++ b/include/linux/mlx5/mlx5_ifc.h
@@ -823,6 +823,12 @@ struct mlx5_ifc_device_mem_cap_bits {
u8 reserved_at_180[0x680];
};
+struct mlx5_ifc_device_event_cap_bits {
+ u8 user_affiliated_events[4][0x40];
+
+ u8 user_unaffiliated_events[4][0x40];
+};
+
enum {
MLX5_ATOMIC_CAPS_ATOMIC_SIZE_QP_1_BYTE = 0x0,
MLX5_ATOMIC_CAPS_ATOMIC_SIZE_QP_2_BYTES = 0x2,
@@ -980,7 +986,8 @@ struct mlx5_ifc_cmd_hca_cap_bits {
u8 log_max_srq_sz[0x8];
u8 log_max_qp_sz[0x8];
- u8 reserved_at_90[0x8];
+ u8 event_cap[0x1];
+ u8 reserved_at_91[0x7];
u8 prio_tag_required[0x1];
u8 reserved_at_99[0x2];
u8 log_max_qp[0x5];
@@ -7364,9 +7371,9 @@ struct mlx5_ifc_create_eq_in_bits {
u8 reserved_at_280[0x40];
- u8 event_bitmask[0x40];
+ u8 event_bitmask[4][0x40];
- u8 reserved_at_300[0x580];
+ u8 reserved_at_3c0[0x4c0];
u8 pas[0][0x40];
};
--
2.20.1
^ permalink raw reply related
* [PATCH rdma-next v2 10/13] IB/mlx5: Enable subscription for device events over DEVX
From: Leon Romanovsky @ 2019-06-30 16:23 UTC (permalink / raw)
To: Doug Ledford, Jason Gunthorpe
Cc: Leon Romanovsky, RDMA mailing list, Saeed Mahameed, Yishai Hadas,
linux-netdev
In-Reply-To: <20190630162334.22135-1-leon@kernel.org>
From: Yishai Hadas <yishaih@mellanox.com>
Enable subscription for device events over DEVX.
Each subscription is added to the two level XA data structure according
to its event number and the DEVX object information in case was given
with the given target fd.
Those events will be reported over the given fd once will occur.
Downstream patches will mange the dispatching to any subscription.
Signed-off-by: Yishai Hadas <yishaih@mellanox.com>
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
---
drivers/infiniband/hw/mlx5/devx.c | 560 ++++++++++++++++++++++-
include/uapi/rdma/mlx5_user_ioctl_cmds.h | 9 +
2 files changed, 562 insertions(+), 7 deletions(-)
diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c
index b7b17e882d82..6fb58339b291 100644
--- a/drivers/infiniband/hw/mlx5/devx.c
+++ b/drivers/infiniband/hw/mlx5/devx.c
@@ -14,6 +14,7 @@
#include <linux/mlx5/driver.h>
#include <linux/mlx5/fs.h>
#include "mlx5_ib.h"
+#include <linux/xarray.h>
#define UVERBS_MODULE_NAME mlx5_ib
#include <rdma/uverbs_named_ioctl.h>
@@ -33,6 +34,40 @@ struct devx_async_data {
struct mlx5_ib_uapi_devx_async_cmd_hdr hdr;
};
+/* first level XA value data structure */
+struct devx_event {
+ struct xarray object_ids; /* second XA level, Key = object id */
+ struct list_head unaffiliated_list;
+};
+
+/* second level XA value data structure */
+struct devx_obj_event {
+ struct rcu_head rcu;
+ struct list_head obj_sub_list;
+};
+
+struct devx_event_subscription {
+ struct list_head file_list; /* headed in ev_file->
+ * subscribed_events_list
+ */
+ struct list_head xa_list; /* headed in devx_event->unaffiliated_list or
+ * devx_obj_event->obj_sub_list
+ */
+ struct list_head obj_list; /* headed in devx_object */
+ struct list_head event_list; /* headed in ev_file->event_list or in
+ * temp list via subscription
+ */
+
+ u8 is_cleaned:1;
+ u32 xa_key_level1;
+ u32 xa_key_level2;
+ struct rcu_head rcu;
+ u64 cookie;
+ struct devx_async_event_file *ev_file;
+ struct file *filp; /* Upon hot unplug we need a direct access to */
+ struct eventfd_ctx *eventfd;
+};
+
struct devx_async_event_file {
struct ib_uobject uobj;
/* Head of events that are subscribed to this FD */
@@ -55,6 +90,7 @@ struct devx_obj {
struct mlx5_ib_devx_mr devx_mr;
struct mlx5_core_dct core_dct;
};
+ struct list_head event_sub; /* holds devx_event_subscription entries */
};
struct devx_umem {
@@ -160,6 +196,104 @@ bool mlx5_ib_devx_is_flow_counter(void *obj, u32 *counter_id)
return false;
}
+static bool is_legacy_unaffiliated_event_num(u16 event_num)
+{
+ switch (event_num) {
+ case MLX5_EVENT_TYPE_PORT_CHANGE:
+ return true;
+ default:
+ return false;
+ }
+}
+
+static bool is_legacy_obj_event_num(u16 event_num)
+{
+ switch (event_num) {
+ case MLX5_EVENT_TYPE_PATH_MIG:
+ case MLX5_EVENT_TYPE_COMM_EST:
+ case MLX5_EVENT_TYPE_SQ_DRAINED:
+ case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
+ case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
+ case MLX5_EVENT_TYPE_CQ_ERROR:
+ case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
+ case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
+ case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
+ case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
+ case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
+ case MLX5_EVENT_TYPE_DCT_DRAINED:
+ case MLX5_EVENT_TYPE_COMP:
+ return true;
+ default:
+ return false;
+ }
+}
+
+static u16 get_legacy_obj_type(u16 opcode)
+{
+ switch (opcode) {
+ case MLX5_CMD_OP_CREATE_RQ:
+ return MLX5_EVENT_QUEUE_TYPE_RQ;
+ case MLX5_CMD_OP_CREATE_QP:
+ return MLX5_EVENT_QUEUE_TYPE_QP;
+ case MLX5_CMD_OP_CREATE_SQ:
+ return MLX5_EVENT_QUEUE_TYPE_SQ;
+ case MLX5_CMD_OP_CREATE_DCT:
+ return MLX5_EVENT_QUEUE_TYPE_DCT;
+ default:
+ return 0;
+ }
+}
+
+static u16 get_dec_obj_type(struct devx_obj *obj, u16 event_num)
+{
+ u16 opcode;
+
+ opcode = (obj->obj_id >> 32) & 0xffff;
+
+ if (is_legacy_obj_event_num(event_num))
+ return get_legacy_obj_type(opcode);
+
+ switch (opcode) {
+ case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
+ return (obj->obj_id >> 48);
+ case MLX5_CMD_OP_CREATE_RQ:
+ return MLX5_OBJ_TYPE_RQ;
+ case MLX5_CMD_OP_CREATE_QP:
+ return MLX5_OBJ_TYPE_QP;
+ case MLX5_CMD_OP_CREATE_SQ:
+ return MLX5_OBJ_TYPE_SQ;
+ case MLX5_CMD_OP_CREATE_DCT:
+ return MLX5_OBJ_TYPE_DCT;
+ case MLX5_CMD_OP_CREATE_TIR:
+ return MLX5_OBJ_TYPE_TIR;
+ case MLX5_CMD_OP_CREATE_TIS:
+ return MLX5_OBJ_TYPE_TIS;
+ case MLX5_CMD_OP_CREATE_PSV:
+ return MLX5_OBJ_TYPE_PSV;
+ case MLX5_OBJ_TYPE_MKEY:
+ return MLX5_OBJ_TYPE_MKEY;
+ case MLX5_CMD_OP_CREATE_RMP:
+ return MLX5_OBJ_TYPE_RMP;
+ case MLX5_CMD_OP_CREATE_XRC_SRQ:
+ return MLX5_OBJ_TYPE_XRC_SRQ;
+ case MLX5_CMD_OP_CREATE_XRQ:
+ return MLX5_OBJ_TYPE_XRQ;
+ case MLX5_CMD_OP_CREATE_RQT:
+ return MLX5_OBJ_TYPE_RQT;
+ case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
+ return MLX5_OBJ_TYPE_FLOW_COUNTER;
+ case MLX5_CMD_OP_CREATE_CQ:
+ return MLX5_OBJ_TYPE_CQ;
+ default:
+ return 0;
+ }
+}
+
+static u32 get_dec_obj_id(u64 obj_id)
+{
+ return (obj_id & 0xffffffff);
+}
+
/*
* As the obj_id in the firmware is not globally unique the object type
* must be considered upon checking for a valid object id.
@@ -1136,14 +1270,47 @@ static void devx_cleanup_mkey(struct devx_obj *obj)
write_unlock_irqrestore(&table->lock, flags);
}
+static void devx_cleanup_subscription(struct mlx5_ib_dev *dev,
+ struct devx_event_subscription *sub)
+{
+ struct devx_event *event;
+ struct devx_obj_event *xa_val_level2;
+
+ if (sub->is_cleaned)
+ return;
+
+ sub->is_cleaned = 1;
+ list_del_rcu(&sub->xa_list);
+
+ if (list_empty(&sub->obj_list))
+ return;
+
+ list_del_rcu(&sub->obj_list);
+ /* check whether key level 1 for this obj_sub_list is empty */
+ event = xa_load(&dev->devx_event_table.event_xa,
+ sub->xa_key_level1);
+ WARN_ON(!event);
+
+ xa_val_level2 = xa_load(&event->object_ids, sub->xa_key_level2);
+ if (list_empty(&xa_val_level2->obj_sub_list)) {
+ xa_erase(&event->object_ids,
+ sub->xa_key_level2);
+ kfree_rcu(xa_val_level2, rcu);
+ }
+}
+
static int devx_obj_cleanup(struct ib_uobject *uobject,
enum rdma_remove_reason why,
struct uverbs_attr_bundle *attrs)
{
u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
+ struct mlx5_devx_event_table *devx_event_table;
struct devx_obj *obj = uobject->object;
+ struct devx_event_subscription *sub_entry, *tmp;
+ struct mlx5_ib_dev *dev;
int ret;
+ dev = mlx5_udata_to_mdev(&attrs->driver_udata);
if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY)
devx_cleanup_mkey(obj);
@@ -1155,10 +1322,14 @@ static int devx_obj_cleanup(struct ib_uobject *uobject,
if (ib_is_destroy_retryable(ret, why, uobject))
return ret;
- if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) {
- struct mlx5_ib_dev *dev =
- mlx5_udata_to_mdev(&attrs->driver_udata);
+ devx_event_table = &dev->devx_event_table;
+
+ mutex_lock(&devx_event_table->event_xa_lock);
+ list_for_each_entry_safe(sub_entry, tmp, &obj->event_sub, obj_list)
+ devx_cleanup_subscription(dev, sub_entry);
+ mutex_unlock(&devx_event_table->event_xa_lock);
+ if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) {
call_srcu(&dev->mr_srcu, &obj->devx_mr.rcu,
devx_free_indirect_mkey);
return ret;
@@ -1230,6 +1401,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)(
uobj->object = obj;
obj->mdev = dev->mdev;
+ INIT_LIST_HEAD(&obj->event_sub);
devx_obj_build_destroy_cmd(cmd_in, cmd_out, obj->dinbox, &obj->dinlen,
&obj_id);
WARN_ON(obj->dinlen > MLX5_MAX_DESTROY_INBOX_SIZE_DW * sizeof(u32));
@@ -1414,6 +1586,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC)(
ev_file->omit_data = 1;
INIT_LIST_HEAD(&ev_file->subscribed_events_list);
ev_file->dev = dev;
+ get_device(&dev->ib_dev.dev);
return 0;
}
@@ -1526,6 +1699,331 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY)(
return err;
}
+static void
+subscribe_event_xa_dealloc(struct mlx5_devx_event_table *devx_event_table,
+ u32 key_level1,
+ bool is_level2,
+ u32 key_level2)
+{
+ struct devx_event *event;
+ struct devx_obj_event *xa_val_level2;
+
+ /* Level 1 is valid for future use, no need to free */
+ if (!is_level2)
+ return;
+
+ event = xa_load(&devx_event_table->event_xa, key_level1);
+ WARN_ON(!event);
+
+ xa_val_level2 = xa_load(&event->object_ids,
+ key_level2);
+ if (list_empty(&xa_val_level2->obj_sub_list)) {
+ xa_erase(&event->object_ids,
+ key_level2);
+ kfree_rcu(xa_val_level2, rcu);
+ }
+}
+
+static int
+subscribe_event_xa_alloc(struct mlx5_devx_event_table *devx_event_table,
+ u32 key_level1,
+ bool is_level2,
+ u32 key_level2)
+{
+ struct devx_obj_event *obj_event;
+ struct devx_event *event;
+ int err;
+
+ event = xa_load(&devx_event_table->event_xa, key_level1);
+ if (!event) {
+ event = kzalloc(sizeof(*event), GFP_KERNEL);
+ if (!event)
+ return -ENOMEM;
+
+ INIT_LIST_HEAD(&event->unaffiliated_list);
+ xa_init(&event->object_ids);
+
+ err = xa_insert(&devx_event_table->event_xa,
+ key_level1,
+ event,
+ GFP_KERNEL);
+ if (err) {
+ kfree(event);
+ return err;
+ }
+ }
+
+ if (!is_level2)
+ return 0;
+
+ obj_event = xa_load(&event->object_ids, key_level2);
+ if (!obj_event) {
+ obj_event = kzalloc(sizeof(*obj_event), GFP_KERNEL);
+ if (!obj_event)
+ /* Level1 is valid for future use, no need to free */
+ return -ENOMEM;
+
+ err = xa_insert(&event->object_ids,
+ key_level2,
+ obj_event,
+ GFP_KERNEL);
+ if (err)
+ return err;
+ INIT_LIST_HEAD(&obj_event->obj_sub_list);
+ }
+
+ return 0;
+}
+
+static bool is_valid_events_legacy(int num_events, u16 *event_type_num_list,
+ struct devx_obj *obj)
+{
+ int i;
+
+ for (i = 0; i < num_events; i++) {
+ if (obj) {
+ if (!is_legacy_obj_event_num(event_type_num_list[i]))
+ return false;
+ } else if (!is_legacy_unaffiliated_event_num(
+ event_type_num_list[i])) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+#define MAX_SUPP_EVENT_NUM 255
+static bool is_valid_events(struct mlx5_core_dev *dev,
+ int num_events, u16 *event_type_num_list,
+ struct devx_obj *obj)
+{
+ __be64 *aff_events;
+ __be64 *unaff_events;
+ int mask_entry;
+ int mask_bit;
+ int i;
+
+ if (MLX5_CAP_GEN(dev, event_cap)) {
+ aff_events = MLX5_CAP_DEV_EVENT(dev,
+ user_affiliated_events);
+ unaff_events = MLX5_CAP_DEV_EVENT(dev,
+ user_unaffiliated_events);
+ } else {
+ return is_valid_events_legacy(num_events, event_type_num_list,
+ obj);
+ }
+
+ for (i = 0; i < num_events; i++) {
+ if (event_type_num_list[i] > MAX_SUPP_EVENT_NUM)
+ return false;
+
+ mask_entry = event_type_num_list[i] / 64;
+ mask_bit = event_type_num_list[i] % 64;
+
+ if (obj) {
+ /* CQ completion */
+ if (event_type_num_list[i] == 0)
+ continue;
+
+ if (!(be64_to_cpu(aff_events[mask_entry]) &
+ (1ull << mask_bit)))
+ return false;
+
+ continue;
+ }
+
+ if (!(be64_to_cpu(unaff_events[mask_entry]) &
+ (1ull << mask_bit)))
+ return false;
+ }
+
+ return true;
+}
+
+#define MAX_NUM_EVENTS 16
+static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)(
+ struct uverbs_attr_bundle *attrs)
+{
+ struct ib_uobject *devx_uobj = uverbs_attr_get_uobject(
+ attrs,
+ MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_OBJ_HANDLE);
+ struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
+ &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
+ struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
+ struct ib_uobject *fd_uobj;
+ struct devx_obj *obj = NULL;
+ struct devx_async_event_file *ev_file;
+ struct mlx5_devx_event_table *devx_event_table = &dev->devx_event_table;
+ u16 *event_type_num_list;
+ struct devx_event_subscription *event_sub, *tmp_sub;
+ struct list_head sub_list;
+ int redirect_fd;
+ bool use_eventfd = false;
+ int num_events;
+ int num_alloc_xa_entries = 0;
+ u16 obj_type = 0;
+ u64 cookie = 0;
+ u32 obj_id = 0;
+ int err;
+ int i;
+
+ if (!c->devx_uid)
+ return -EINVAL;
+
+ if (!IS_ERR(devx_uobj)) {
+ obj = (struct devx_obj *)devx_uobj->object;
+ if (obj)
+ obj_id = get_dec_obj_id(obj->obj_id);
+ }
+
+ fd_uobj = uverbs_attr_get_uobject(attrs,
+ MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_HANDLE);
+ if (IS_ERR(fd_uobj))
+ return PTR_ERR(fd_uobj);
+
+ ev_file = container_of(fd_uobj, struct devx_async_event_file,
+ uobj);
+
+ if (uverbs_attr_is_valid(attrs,
+ MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM)) {
+ err = uverbs_copy_from(&redirect_fd, attrs,
+ MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM);
+ if (err)
+ return err;
+
+ use_eventfd = true;
+ }
+
+ if (uverbs_attr_is_valid(attrs,
+ MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE)) {
+ if (use_eventfd)
+ return -EINVAL;
+
+ err = uverbs_copy_from(&cookie, attrs,
+ MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE);
+ if (err)
+ return err;
+ }
+
+ num_events = uverbs_attr_ptr_get_array_size(
+ attrs, MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST,
+ sizeof(u16));
+
+ if (num_events < 0)
+ return num_events;
+
+ if (num_events > MAX_NUM_EVENTS)
+ return -EINVAL;
+
+ event_type_num_list = uverbs_attr_get_alloced_ptr(attrs,
+ MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST);
+
+ if (!is_valid_events(dev->mdev, num_events, event_type_num_list, obj))
+ return -EINVAL;
+
+ INIT_LIST_HEAD(&sub_list);
+
+ /* Protect from concurrent subscriptions to same XA entries to allow
+ * both to succeed
+ */
+ mutex_lock(&devx_event_table->event_xa_lock);
+ for (i = 0; i < num_events; i++) {
+ u32 key_level1;
+
+ if (obj)
+ obj_type = get_dec_obj_type(obj,
+ event_type_num_list[i]);
+ key_level1 = event_type_num_list[i] | obj_type << 16;
+
+ err = subscribe_event_xa_alloc(devx_event_table,
+ key_level1,
+ obj,
+ obj_id);
+ if (err)
+ goto err;
+
+ num_alloc_xa_entries++;
+ event_sub = kzalloc(sizeof(*event_sub), GFP_KERNEL);
+ if (!event_sub)
+ goto err;
+
+ list_add_tail(&event_sub->event_list, &sub_list);
+ if (use_eventfd) {
+ event_sub->eventfd =
+ eventfd_ctx_fdget(redirect_fd);
+
+ if (IS_ERR(event_sub)) {
+ err = PTR_ERR(event_sub->eventfd);
+ event_sub->eventfd = NULL;
+ goto err;
+ }
+ }
+
+ event_sub->cookie = cookie;
+ event_sub->ev_file = ev_file;
+ event_sub->filp = fd_uobj->object;
+ /* May be needed upon cleanup the devx object/subscription */
+ event_sub->xa_key_level1 = key_level1;
+ event_sub->xa_key_level2 = obj_id;
+ INIT_LIST_HEAD(&event_sub->obj_list);
+ }
+
+ /* Once all the allocations and the XA data insertions were done we
+ * can go ahead and add all the subscriptions to the relevant lists
+ * without concern of a failure.
+ */
+ list_for_each_entry_safe(event_sub, tmp_sub, &sub_list, event_list) {
+ struct devx_event *event;
+ struct devx_obj_event *obj_event;
+
+ list_del_init(&event_sub->event_list);
+
+ spin_lock_irq(&ev_file->lock);
+ list_add_tail_rcu(&event_sub->file_list,
+ &ev_file->subscribed_events_list);
+ spin_unlock_irq(&ev_file->lock);
+
+ event = xa_load(&devx_event_table->event_xa,
+ event_sub->xa_key_level1);
+ WARN_ON(!event);
+
+ if (!obj) {
+ list_add_tail_rcu(&event_sub->xa_list,
+ &event->unaffiliated_list);
+ continue;
+ }
+
+ obj_event = xa_load(&event->object_ids, obj_id);
+ WARN_ON(!obj_event);
+ list_add_tail_rcu(&event_sub->xa_list,
+ &obj_event->obj_sub_list);
+ list_add_tail_rcu(&event_sub->obj_list,
+ &obj->event_sub);
+ }
+
+ mutex_unlock(&devx_event_table->event_xa_lock);
+ return 0;
+
+err:
+ list_for_each_entry_safe(event_sub, tmp_sub, &sub_list, event_list) {
+ list_del(&event_sub->event_list);
+
+ subscribe_event_xa_dealloc(devx_event_table,
+ event_sub->xa_key_level1,
+ obj,
+ obj_id);
+
+ if (event_sub->eventfd)
+ eventfd_ctx_put(event_sub->eventfd);
+
+ kfree(event_sub);
+ }
+
+ mutex_unlock(&devx_event_table->event_xa_lock);
+ return err;
+}
+
static int devx_umem_get(struct mlx5_ib_dev *dev, struct ib_ucontext *ucontext,
struct uverbs_attr_bundle *attrs,
struct devx_umem *obj)
@@ -1692,14 +2190,21 @@ void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev)
void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev)
{
struct mlx5_devx_event_table *table = &dev->devx_event_table;
+ struct devx_event_subscription *sub, *tmp;
+ struct devx_event *event;
void *entry;
unsigned long id;
mlx5_eq_notifier_unregister(dev->mdev, &table->devx_nb);
-
- xa_for_each(&table->event_xa, id, entry)
+ mutex_lock(&dev->devx_event_table.event_xa_lock);
+ xa_for_each(&table->event_xa, id, entry) {
+ event = entry;
+ list_for_each_entry_safe(sub, tmp, &event->unaffiliated_list,
+ xa_list)
+ devx_cleanup_subscription(dev, sub);
kfree(entry);
-
+ }
+ mutex_unlock(&dev->devx_event_table.event_xa_lock);
xa_destroy(&table->event_xa);
}
@@ -1815,7 +2320,26 @@ static __poll_t devx_async_event_poll(struct file *filp,
static int devx_async_event_close(struct inode *inode, struct file *filp)
{
+ struct devx_async_event_file *ev_file = filp->private_data;
+ struct devx_event_subscription *event_sub, *event_sub_tmp;
+
+ mutex_lock(&ev_file->dev->devx_event_table.event_xa_lock);
+ /* delete the subscriptions which are related to this FD */
+ list_for_each_entry_safe(event_sub, event_sub_tmp,
+ &ev_file->subscribed_events_list, file_list) {
+ devx_cleanup_subscription(ev_file->dev, event_sub);
+ if (event_sub->eventfd)
+ eventfd_ctx_put(event_sub->eventfd);
+
+ list_del_rcu(&event_sub->file_list);
+ /* subscription may not be used by the read API any more */
+ kfree_rcu(event_sub, rcu);
+ }
+
+ mutex_unlock(&ev_file->dev->devx_event_table.event_xa_lock);
+
uverbs_close_fd(filp);
+ put_device(&ev_file->dev->ib_dev.dev);
return 0;
}
@@ -1983,10 +2507,32 @@ DECLARE_UVERBS_NAMED_METHOD(
UVERBS_ATTR_TYPE(u64),
UA_MANDATORY));
+DECLARE_UVERBS_NAMED_METHOD(
+ MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT,
+ UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_HANDLE,
+ MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
+ UVERBS_ACCESS_READ,
+ UA_MANDATORY),
+ UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_OBJ_HANDLE,
+ MLX5_IB_OBJECT_DEVX_OBJ,
+ UVERBS_ACCESS_READ,
+ UA_OPTIONAL),
+ UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST,
+ UVERBS_ATTR_MIN_SIZE(sizeof(u16)),
+ UA_MANDATORY,
+ UA_ALLOC_AND_COPY),
+ UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE,
+ UVERBS_ATTR_TYPE(u64),
+ UA_OPTIONAL),
+ UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM,
+ UVERBS_ATTR_TYPE(u32),
+ UA_OPTIONAL));
+
DECLARE_UVERBS_GLOBAL_METHODS(MLX5_IB_OBJECT_DEVX,
&UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OTHER),
&UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_UAR),
- &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_EQN));
+ &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_EQN),
+ &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT));
DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_OBJ,
UVERBS_TYPE_ALLOC_IDR(devx_obj_cleanup),
diff --git a/include/uapi/rdma/mlx5_user_ioctl_cmds.h b/include/uapi/rdma/mlx5_user_ioctl_cmds.h
index 6ad8f4f11ddd..d0da070cf0ab 100644
--- a/include/uapi/rdma/mlx5_user_ioctl_cmds.h
+++ b/include/uapi/rdma/mlx5_user_ioctl_cmds.h
@@ -51,6 +51,7 @@ enum mlx5_ib_devx_methods {
MLX5_IB_METHOD_DEVX_OTHER = (1U << UVERBS_ID_NS_SHIFT),
MLX5_IB_METHOD_DEVX_QUERY_UAR,
MLX5_IB_METHOD_DEVX_QUERY_EQN,
+ MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT,
};
enum mlx5_ib_devx_other_attrs {
@@ -93,6 +94,14 @@ enum mlx5_ib_devx_obj_query_async_attrs {
MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_OUT_LEN,
};
+enum mlx5_ib_devx_subscribe_event_attrs {
+ MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
+ MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_OBJ_HANDLE,
+ MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST,
+ MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM,
+ MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE,
+};
+
enum mlx5_ib_devx_query_eqn_attrs {
MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC = (1U << UVERBS_ID_NS_SHIFT),
MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN,
--
2.20.1
^ permalink raw reply related
* [PATCH rdma-next v2 11/13] IB/mlx5: Implement DEVX dispatching event
From: Leon Romanovsky @ 2019-06-30 16:23 UTC (permalink / raw)
To: Doug Ledford, Jason Gunthorpe
Cc: Leon Romanovsky, RDMA mailing list, Saeed Mahameed, Yishai Hadas,
linux-netdev
In-Reply-To: <20190630162334.22135-1-leon@kernel.org>
From: Yishai Hadas <yishaih@mellanox.com>
Implement DEVX dispatching event by looking up for the applicable
subscriptions for the reported event and using their target fd to
signal/set the event.
Signed-off-by: Yishai Hadas <yishaih@mellanox.com>
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
---
drivers/infiniband/hw/mlx5/devx.c | 303 +++++++++++++++++++++-
include/uapi/rdma/mlx5_user_ioctl_verbs.h | 5 +
2 files changed, 305 insertions(+), 3 deletions(-)
diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c
index 6fb58339b291..de068fe94363 100644
--- a/drivers/infiniband/hw/mlx5/devx.c
+++ b/drivers/infiniband/hw/mlx5/devx.c
@@ -34,6 +34,11 @@ struct devx_async_data {
struct mlx5_ib_uapi_devx_async_cmd_hdr hdr;
};
+struct devx_async_event_data {
+ struct list_head list; /* headed in ev_file->event_list */
+ struct mlx5_ib_uapi_devx_async_event_hdr hdr;
+};
+
/* first level XA value data structure */
struct devx_event {
struct xarray object_ids; /* second XA level, Key = object id */
@@ -77,6 +82,8 @@ struct devx_async_event_file {
struct list_head event_list;
struct mlx5_ib_dev *dev;
u8 omit_data:1;
+ u8 is_overflow_err:1;
+ u8 is_destroyed:1;
};
#define MLX5_MAX_DESTROY_INBOX_SIZE_DW MLX5_ST_SZ_DW(delete_fte_in)
@@ -289,6 +296,29 @@ static u16 get_dec_obj_type(struct devx_obj *obj, u16 event_num)
}
}
+static u16 get_event_obj_type(unsigned long event_type, struct mlx5_eqe *eqe)
+{
+ switch (event_type) {
+ case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
+ case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
+ case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
+ case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
+ case MLX5_EVENT_TYPE_PATH_MIG:
+ case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
+ case MLX5_EVENT_TYPE_COMM_EST:
+ case MLX5_EVENT_TYPE_SQ_DRAINED:
+ case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
+ case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
+ return eqe->data.qp_srq.type;
+ case MLX5_EVENT_TYPE_CQ_ERROR:
+ return 0;
+ case MLX5_EVENT_TYPE_DCT_DRAINED:
+ return MLX5_EVENT_QUEUE_TYPE_DCT;
+ default:
+ return MLX5_GET(affiliated_event_header, &eqe->data, obj_type);
+ }
+}
+
static u32 get_dec_obj_id(u64 obj_id)
{
return (obj_id & 0xffffffff);
@@ -2171,10 +2201,170 @@ static int devx_umem_cleanup(struct ib_uobject *uobject,
return 0;
}
+static bool is_unaffiliated_event(struct mlx5_core_dev *dev,
+ unsigned long event_type)
+{
+ __be64 *unaff_events;
+ int mask_entry;
+ int mask_bit;
+
+ if (!MLX5_CAP_GEN(dev, event_cap))
+ return is_legacy_unaffiliated_event_num(event_type);
+
+ unaff_events = MLX5_CAP_DEV_EVENT(dev,
+ user_unaffiliated_events);
+ WARN_ON(event_type > MAX_SUPP_EVENT_NUM);
+
+ mask_entry = event_type / 64;
+ mask_bit = event_type % 64;
+
+ if (!(be64_to_cpu(unaff_events[mask_entry]) & (1ull << mask_bit)))
+ return false;
+
+ return true;
+}
+
+static u32 devx_get_obj_id_from_event(unsigned long event_type, void *data)
+{
+ struct mlx5_eqe *eqe = data;
+ u32 obj_id = 0;
+
+ switch (event_type) {
+ case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
+ case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
+ case MLX5_EVENT_TYPE_PATH_MIG:
+ case MLX5_EVENT_TYPE_COMM_EST:
+ case MLX5_EVENT_TYPE_SQ_DRAINED:
+ case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
+ case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
+ case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
+ case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
+ case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
+ obj_id = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff;
+ break;
+ case MLX5_EVENT_TYPE_DCT_DRAINED:
+ obj_id = be32_to_cpu(eqe->data.dct.dctn) & 0xffffff;
+ break;
+ case MLX5_EVENT_TYPE_CQ_ERROR:
+ obj_id = be32_to_cpu(eqe->data.cq_err.cqn) & 0xffffff;
+ break;
+ default:
+ obj_id = MLX5_GET(affiliated_event_header, &eqe->data, obj_id);
+ break;
+ }
+
+ return obj_id;
+}
+
+static int deliver_event(struct devx_event_subscription *event_sub,
+ const void *data)
+{
+ struct devx_async_event_file *ev_file;
+ struct devx_async_event_data *event_data;
+ unsigned long flags;
+
+ ev_file = event_sub->ev_file;
+
+ if (ev_file->omit_data) {
+ spin_lock_irqsave(&ev_file->lock, flags);
+ if (!list_empty(&event_sub->event_list)) {
+ spin_unlock_irqrestore(&ev_file->lock, flags);
+ return 0;
+ }
+
+ list_add_tail(&event_sub->event_list, &ev_file->event_list);
+ spin_unlock_irqrestore(&ev_file->lock, flags);
+ wake_up_interruptible(&ev_file->poll_wait);
+ return 0;
+ }
+
+ event_data = kzalloc(sizeof(*event_data) + sizeof(struct mlx5_eqe),
+ GFP_ATOMIC);
+ if (!event_data) {
+ spin_lock_irqsave(&ev_file->lock, flags);
+ ev_file->is_overflow_err = 1;
+ spin_unlock_irqrestore(&ev_file->lock, flags);
+ return -ENOMEM;
+ }
+
+ event_data->hdr.cookie = event_sub->cookie;
+ memcpy(event_data->hdr.out_data, data, sizeof(struct mlx5_eqe));
+
+ spin_lock_irqsave(&ev_file->lock, flags);
+ list_add_tail(&event_data->list, &ev_file->event_list);
+ spin_unlock_irqrestore(&ev_file->lock, flags);
+ wake_up_interruptible(&ev_file->poll_wait);
+
+ return 0;
+}
+
+static void dispatch_event_fd(struct list_head *fd_list,
+ const void *data)
+{
+ struct devx_event_subscription *item;
+
+ list_for_each_entry_rcu(item, fd_list, xa_list) {
+ if (!get_file_rcu(item->filp))
+ continue;
+
+ if (item->eventfd) {
+ eventfd_signal(item->eventfd, 1);
+ fput(item->filp);
+ continue;
+ }
+
+ deliver_event(item, data);
+ fput(item->filp);
+ }
+}
+
static int devx_event_notifier(struct notifier_block *nb,
unsigned long event_type, void *data)
{
- return NOTIFY_DONE;
+ struct mlx5_devx_event_table *table;
+ struct mlx5_ib_dev *dev;
+ struct devx_event *event;
+ struct devx_obj_event *obj_event;
+ u16 obj_type = 0;
+ bool is_unaffiliated;
+ u32 obj_id;
+
+ /* Explicit filtering to kernel events which may occur frequently */
+ if (event_type == MLX5_EVENT_TYPE_CMD ||
+ event_type == MLX5_EVENT_TYPE_PAGE_REQUEST)
+ return NOTIFY_OK;
+
+ table = container_of(nb, struct mlx5_devx_event_table, devx_nb.nb);
+ dev = container_of(table, struct mlx5_ib_dev, devx_event_table);
+ is_unaffiliated = is_unaffiliated_event(dev->mdev, event_type);
+
+ if (!is_unaffiliated)
+ obj_type = get_event_obj_type(event_type, data);
+
+ rcu_read_lock();
+ event = xa_load(&table->event_xa, event_type | (obj_type << 16));
+ if (!event) {
+ rcu_read_unlock();
+ return NOTIFY_DONE;
+ }
+
+ if (is_unaffiliated) {
+ dispatch_event_fd(&event->unaffiliated_list, data);
+ rcu_read_unlock();
+ return NOTIFY_OK;
+ }
+
+ obj_id = devx_get_obj_id_from_event(event_type, data);
+ obj_event = xa_load(&event->object_ids, obj_id);
+ if (!obj_event) {
+ rcu_read_unlock();
+ return NOTIFY_DONE;
+ }
+
+ dispatch_event_fd(&obj_event->obj_sub_list, data);
+
+ rcu_read_unlock();
+ return NOTIFY_OK;
}
void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev)
@@ -2309,19 +2499,108 @@ static const struct file_operations devx_async_cmd_event_fops = {
static ssize_t devx_async_event_read(struct file *filp, char __user *buf,
size_t count, loff_t *pos)
{
- return -EINVAL;
+ struct devx_async_event_file *ev_file = filp->private_data;
+ struct devx_event_subscription *event_sub;
+ struct devx_async_event_data *uninitialized_var(event);
+ int ret = 0;
+ size_t eventsz;
+ bool omit_data;
+ void *event_data;
+
+ omit_data = ev_file->omit_data;
+
+ spin_lock_irq(&ev_file->lock);
+
+ if (ev_file->is_overflow_err) {
+ ev_file->is_overflow_err = 0;
+ spin_unlock_irq(&ev_file->lock);
+ return -EOVERFLOW;
+ }
+
+ if (ev_file->is_destroyed) {
+ spin_unlock_irq(&ev_file->lock);
+ return -EIO;
+ }
+
+ while (list_empty(&ev_file->event_list)) {
+ spin_unlock_irq(&ev_file->lock);
+
+ if (filp->f_flags & O_NONBLOCK)
+ return -EAGAIN;
+
+ if (wait_event_interruptible(ev_file->poll_wait,
+ (!list_empty(&ev_file->event_list) ||
+ ev_file->is_destroyed))) {
+ return -ERESTARTSYS;
+ }
+
+ spin_lock_irq(&ev_file->lock);
+ if (ev_file->is_destroyed) {
+ spin_unlock_irq(&ev_file->lock);
+ return -EIO;
+ }
+ }
+
+ if (omit_data) {
+ event_sub = list_first_entry(&ev_file->event_list,
+ struct devx_event_subscription,
+ event_list);
+ eventsz = sizeof(event_sub->cookie);
+ event_data = &event_sub->cookie;
+ } else {
+ event = list_first_entry(&ev_file->event_list,
+ struct devx_async_event_data, list);
+ eventsz = sizeof(struct mlx5_eqe) +
+ sizeof(struct mlx5_ib_uapi_devx_async_event_hdr);
+ event_data = &event->hdr;
+ }
+
+ if (eventsz > count) {
+ spin_unlock_irq(&ev_file->lock);
+ return -EINVAL;
+ }
+
+ if (omit_data)
+ list_del_init(&event_sub->event_list);
+ else
+ list_del(&event->list);
+
+ spin_unlock_irq(&ev_file->lock);
+
+ if (copy_to_user(buf, event_data, eventsz))
+ /* This points to an application issue, not a kernel concern */
+ ret = -EFAULT;
+ else
+ ret = eventsz;
+
+ if (!omit_data)
+ kfree(event);
+ return ret;
}
static __poll_t devx_async_event_poll(struct file *filp,
struct poll_table_struct *wait)
{
- return 0;
+ struct devx_async_event_file *ev_file = filp->private_data;
+ __poll_t pollflags = 0;
+
+ poll_wait(filp, &ev_file->poll_wait, wait);
+
+ spin_lock_irq(&ev_file->lock);
+ if (ev_file->is_destroyed)
+ pollflags = EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
+ else if (!list_empty(&ev_file->event_list))
+ pollflags = EPOLLIN | EPOLLRDNORM;
+ spin_unlock_irq(&ev_file->lock);
+
+ return pollflags;
}
static int devx_async_event_close(struct inode *inode, struct file *filp)
{
struct devx_async_event_file *ev_file = filp->private_data;
struct devx_event_subscription *event_sub, *event_sub_tmp;
+ struct devx_async_event_data *entry, *tmp;
mutex_lock(&ev_file->dev->devx_event_table.event_xa_lock);
/* delete the subscriptions which are related to this FD */
@@ -2338,6 +2617,15 @@ static int devx_async_event_close(struct inode *inode, struct file *filp)
mutex_unlock(&ev_file->dev->devx_event_table.event_xa_lock);
+ /* free the pending events allocation */
+ if (!ev_file->omit_data) {
+ spin_lock_irq(&ev_file->lock);
+ list_for_each_entry_safe(entry, tmp,
+ &ev_file->event_list, list)
+ kfree(entry); /* read can't come any more */
+ spin_unlock_irq(&ev_file->lock);
+ }
+
uverbs_close_fd(filp);
put_device(&ev_file->dev->ib_dev.dev);
return 0;
@@ -2373,6 +2661,15 @@ static int devx_hot_unplug_async_cmd_event_file(struct ib_uobject *uobj,
static int devx_hot_unplug_async_event_file(struct ib_uobject *uobj,
enum rdma_remove_reason why)
{
+ struct devx_async_event_file *ev_file =
+ container_of(uobj, struct devx_async_event_file,
+ uobj);
+
+ spin_lock_irq(&ev_file->lock);
+ ev_file->is_destroyed = 1;
+ spin_unlock_irq(&ev_file->lock);
+
+ wake_up_interruptible(&ev_file->poll_wait);
return 0;
};
diff --git a/include/uapi/rdma/mlx5_user_ioctl_verbs.h b/include/uapi/rdma/mlx5_user_ioctl_verbs.h
index b44691315d39..7e9900b0e746 100644
--- a/include/uapi/rdma/mlx5_user_ioctl_verbs.h
+++ b/include/uapi/rdma/mlx5_user_ioctl_verbs.h
@@ -67,5 +67,10 @@ enum mlx5_ib_uapi_devx_create_event_channel_flags {
MLX5_IB_UAPI_DEVX_CR_EV_CH_FLAGS_OMIT_DATA = 1 << 0,
};
+struct mlx5_ib_uapi_devx_async_event_hdr {
+ __aligned_u64 cookie;
+ __u8 out_data[];
+};
+
#endif
--
2.20.1
^ permalink raw reply related
* [PATCH rdma-next v2 12/13] IB/mlx5: Add DEVX support for CQ events
From: Leon Romanovsky @ 2019-06-30 16:23 UTC (permalink / raw)
To: Doug Ledford, Jason Gunthorpe
Cc: Leon Romanovsky, RDMA mailing list, Saeed Mahameed, Yishai Hadas,
linux-netdev
In-Reply-To: <20190630162334.22135-1-leon@kernel.org>
From: Yishai Hadas <yishaih@mellanox.com>
Add DEVX support for CQ events by creating and destroying the CQ via
mlx5_core and set an handler to manage its completions.
Signed-off-by: Yishai Hadas <yishaih@mellanox.com>
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
---
drivers/infiniband/hw/mlx5/devx.c | 39 +++++++++++++++++++++++++++++++
1 file changed, 39 insertions(+)
diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c
index de068fe94363..38064f4bfdf6 100644
--- a/drivers/infiniband/hw/mlx5/devx.c
+++ b/drivers/infiniband/hw/mlx5/devx.c
@@ -19,9 +19,12 @@
#define UVERBS_MODULE_NAME mlx5_ib
#include <rdma/uverbs_named_ioctl.h>
+static void dispatch_event_fd(struct list_head *fd_list, const void *data);
+
enum devx_obj_flags {
DEVX_OBJ_FLAGS_INDIRECT_MKEY = 1 << 0,
DEVX_OBJ_FLAGS_DCT = 1 << 1,
+ DEVX_OBJ_FLAGS_CQ = 1 << 2,
};
struct devx_async_data {
@@ -89,6 +92,7 @@ struct devx_async_event_file {
#define MLX5_MAX_DESTROY_INBOX_SIZE_DW MLX5_ST_SZ_DW(delete_fte_in)
struct devx_obj {
struct mlx5_core_dev *mdev;
+ struct mlx5_ib_dev *ib_dev;
u64 obj_id;
u32 dinlen; /* destroy inbox length */
u32 dinbox[MLX5_MAX_DESTROY_INBOX_SIZE_DW];
@@ -96,6 +100,7 @@ struct devx_obj {
union {
struct mlx5_ib_devx_mr devx_mr;
struct mlx5_core_dct core_dct;
+ struct mlx5_core_cq core_cq;
};
struct list_head event_sub; /* holds devx_event_subscription entries */
};
@@ -1346,6 +1351,8 @@ static int devx_obj_cleanup(struct ib_uobject *uobject,
if (obj->flags & DEVX_OBJ_FLAGS_DCT)
ret = mlx5_core_destroy_dct(obj->mdev, &obj->core_dct);
+ else if (obj->flags & DEVX_OBJ_FLAGS_CQ)
+ ret = mlx5_core_destroy_cq(obj->mdev, &obj->core_cq);
else
ret = mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out,
sizeof(out));
@@ -1369,6 +1376,29 @@ static int devx_obj_cleanup(struct ib_uobject *uobject,
return ret;
}
+static void devx_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe)
+{
+ struct devx_obj *obj = container_of(mcq, struct devx_obj, core_cq);
+ struct mlx5_devx_event_table *table;
+ struct devx_event *event;
+ struct devx_obj_event *obj_event;
+ u32 obj_id = mcq->cqn;
+
+ table = &obj->ib_dev->devx_event_table;
+ rcu_read_lock();
+ event = xa_load(&table->event_xa, MLX5_EVENT_TYPE_COMP);
+ if (!event)
+ goto out;
+
+ obj_event = xa_load(&event->object_ids, obj_id);
+ if (!obj_event)
+ goto out;
+
+ dispatch_event_fd(&obj_event->obj_sub_list, eqe);
+out:
+ rcu_read_unlock();
+}
+
static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)(
struct uverbs_attr_bundle *attrs)
{
@@ -1420,6 +1450,12 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)(
err = mlx5_core_create_dct(dev->mdev, &obj->core_dct,
cmd_in, cmd_in_len,
cmd_out, cmd_out_len);
+ } else if (opcode == MLX5_CMD_OP_CREATE_CQ) {
+ obj->flags |= DEVX_OBJ_FLAGS_CQ;
+ obj->core_cq.comp = devx_cq_comp;
+ err = mlx5_core_create_cq(dev->mdev, &obj->core_cq,
+ cmd_in, cmd_in_len, cmd_out,
+ cmd_out_len);
} else {
err = mlx5_cmd_exec(dev->mdev, cmd_in,
cmd_in_len,
@@ -1432,6 +1468,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)(
uobj->object = obj;
obj->mdev = dev->mdev;
INIT_LIST_HEAD(&obj->event_sub);
+ obj->ib_dev = dev;
devx_obj_build_destroy_cmd(cmd_in, cmd_out, obj->dinbox, &obj->dinlen,
&obj_id);
WARN_ON(obj->dinlen > MLX5_MAX_DESTROY_INBOX_SIZE_DW * sizeof(u32));
@@ -1459,6 +1496,8 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)(
obj_destroy:
if (obj->flags & DEVX_OBJ_FLAGS_DCT)
mlx5_core_destroy_dct(obj->mdev, &obj->core_dct);
+ else if (obj->flags & DEVX_OBJ_FLAGS_CQ)
+ mlx5_core_destroy_cq(obj->mdev, &obj->core_cq);
else
mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out,
sizeof(out));
--
2.20.1
^ permalink raw reply related
* [PATCH rdma-next v2 13/13] IB/mlx5: DEVX cleanup mdev
From: Leon Romanovsky @ 2019-06-30 16:23 UTC (permalink / raw)
To: Doug Ledford, Jason Gunthorpe
Cc: Leon Romanovsky, RDMA mailing list, Saeed Mahameed, Yishai Hadas,
linux-netdev
In-Reply-To: <20190630162334.22135-1-leon@kernel.org>
From: Yishai Hadas <yishaih@mellanox.com>
No need any more to hold mlx5_core_dev on the devx_object, it can be
accessed from ib_dev.
Signed-off-by: Yishai Hadas <yishaih@mellanox.com>
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
---
drivers/infiniband/hw/mlx5/devx.c | 18 ++++++++----------
1 file changed, 8 insertions(+), 10 deletions(-)
diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c
index 38064f4bfdf6..cb4f0fc79176 100644
--- a/drivers/infiniband/hw/mlx5/devx.c
+++ b/drivers/infiniband/hw/mlx5/devx.c
@@ -91,7 +91,6 @@ struct devx_async_event_file {
#define MLX5_MAX_DESTROY_INBOX_SIZE_DW MLX5_ST_SZ_DW(delete_fte_in)
struct devx_obj {
- struct mlx5_core_dev *mdev;
struct mlx5_ib_dev *ib_dev;
u64 obj_id;
u32 dinlen; /* destroy inbox length */
@@ -1297,7 +1296,7 @@ static void devx_free_indirect_mkey(struct rcu_head *rcu)
*/
static void devx_cleanup_mkey(struct devx_obj *obj)
{
- struct mlx5_mkey_table *table = &obj->mdev->priv.mkey_table;
+ struct mlx5_mkey_table *table = &obj->ib_dev->mdev->priv.mkey_table;
unsigned long flags;
write_lock_irqsave(&table->lock, flags);
@@ -1350,12 +1349,12 @@ static int devx_obj_cleanup(struct ib_uobject *uobject,
devx_cleanup_mkey(obj);
if (obj->flags & DEVX_OBJ_FLAGS_DCT)
- ret = mlx5_core_destroy_dct(obj->mdev, &obj->core_dct);
+ ret = mlx5_core_destroy_dct(obj->ib_dev->mdev, &obj->core_dct);
else if (obj->flags & DEVX_OBJ_FLAGS_CQ)
- ret = mlx5_core_destroy_cq(obj->mdev, &obj->core_cq);
+ ret = mlx5_core_destroy_cq(obj->ib_dev->mdev, &obj->core_cq);
else
- ret = mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out,
- sizeof(out));
+ ret = mlx5_cmd_exec(obj->ib_dev->mdev, obj->dinbox,
+ obj->dinlen, out, sizeof(out));
if (ib_is_destroy_retryable(ret, why, uobject))
return ret;
@@ -1466,7 +1465,6 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)(
goto obj_free;
uobj->object = obj;
- obj->mdev = dev->mdev;
INIT_LIST_HEAD(&obj->event_sub);
obj->ib_dev = dev;
devx_obj_build_destroy_cmd(cmd_in, cmd_out, obj->dinbox, &obj->dinlen,
@@ -1495,11 +1493,11 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)(
devx_cleanup_mkey(obj);
obj_destroy:
if (obj->flags & DEVX_OBJ_FLAGS_DCT)
- mlx5_core_destroy_dct(obj->mdev, &obj->core_dct);
+ mlx5_core_destroy_dct(obj->ib_dev->mdev, &obj->core_dct);
else if (obj->flags & DEVX_OBJ_FLAGS_CQ)
- mlx5_core_destroy_cq(obj->mdev, &obj->core_cq);
+ mlx5_core_destroy_cq(obj->ib_dev->mdev, &obj->core_cq);
else
- mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out,
+ mlx5_cmd_exec(obj->ib_dev->mdev, obj->dinbox, obj->dinlen, out,
sizeof(out));
obj_free:
kfree(obj);
--
2.20.1
^ permalink raw reply related
* [PATCH rdma-next v2 09/13] IB/mlx5: Register DEVX with mlx5_core to get async events
From: Leon Romanovsky @ 2019-06-30 16:23 UTC (permalink / raw)
To: Doug Ledford, Jason Gunthorpe
Cc: Leon Romanovsky, RDMA mailing list, Saeed Mahameed, Yishai Hadas,
linux-netdev
In-Reply-To: <20190630162334.22135-1-leon@kernel.org>
From: Yishai Hadas <yishaih@mellanox.com>
Register DEVX with with mlx5_core to get async events. This will enable
to dispatch the applicable events to its consumers in down stream
patches.
Signed-off-by: Yishai Hadas <yishaih@mellanox.com>
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
---
drivers/infiniband/hw/mlx5/devx.c | 30 ++++++++++++++++++++++++++++
drivers/infiniband/hw/mlx5/main.c | 8 ++++++--
drivers/infiniband/hw/mlx5/mlx5_ib.h | 12 +++++++++++
3 files changed, 48 insertions(+), 2 deletions(-)
diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c
index 89947b4f4f6f..b7b17e882d82 100644
--- a/drivers/infiniband/hw/mlx5/devx.c
+++ b/drivers/infiniband/hw/mlx5/devx.c
@@ -1673,6 +1673,36 @@ static int devx_umem_cleanup(struct ib_uobject *uobject,
return 0;
}
+static int devx_event_notifier(struct notifier_block *nb,
+ unsigned long event_type, void *data)
+{
+ return NOTIFY_DONE;
+}
+
+void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev)
+{
+ struct mlx5_devx_event_table *table = &dev->devx_event_table;
+
+ xa_init(&table->event_xa);
+ mutex_init(&table->event_xa_lock);
+ MLX5_NB_INIT(&table->devx_nb, devx_event_notifier, NOTIFY_ANY);
+ mlx5_eq_notifier_register(dev->mdev, &table->devx_nb);
+}
+
+void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev)
+{
+ struct mlx5_devx_event_table *table = &dev->devx_event_table;
+ void *entry;
+ unsigned long id;
+
+ mlx5_eq_notifier_unregister(dev->mdev, &table->devx_nb);
+
+ xa_for_each(&table->event_xa, id, entry)
+ kfree(entry);
+
+ xa_destroy(&table->event_xa);
+}
+
static ssize_t devx_async_cmd_event_read(struct file *filp, char __user *buf,
size_t count, loff_t *pos)
{
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 23997f744586..be0e04cfed77 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -6652,15 +6652,19 @@ static int mlx5_ib_stage_devx_init(struct mlx5_ib_dev *dev)
int uid;
uid = mlx5_ib_devx_create(dev, false);
- if (uid > 0)
+ if (uid > 0) {
dev->devx_whitelist_uid = uid;
+ mlx5_ib_devx_init_event_table(dev);
+ }
return 0;
}
static void mlx5_ib_stage_devx_cleanup(struct mlx5_ib_dev *dev)
{
- if (dev->devx_whitelist_uid)
+ if (dev->devx_whitelist_uid) {
+ mlx5_ib_devx_cleanup_event_table(dev);
mlx5_ib_devx_destroy(dev, dev->devx_whitelist_uid);
+ }
}
void __mlx5_ib_remove(struct mlx5_ib_dev *dev,
diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index 9cf23ae6324e..556af34b788b 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -944,6 +944,13 @@ struct mlx5_ib_pf_eq {
mempool_t *pool;
};
+struct mlx5_devx_event_table {
+ struct mlx5_nb devx_nb;
+ /* serialize updating the event_xa */
+ struct mutex event_xa_lock;
+ struct xarray event_xa;
+};
+
struct mlx5_ib_dev {
struct ib_device ib_dev;
struct mlx5_core_dev *mdev;
@@ -994,6 +1001,7 @@ struct mlx5_ib_dev {
struct mlx5_srq_table srq_table;
struct mlx5_async_ctx async_ctx;
int free_port;
+ struct mlx5_devx_event_table devx_event_table;
};
static inline struct mlx5_ib_cq *to_mibcq(struct mlx5_core_cq *mcq)
@@ -1333,6 +1341,8 @@ void mlx5_ib_put_native_port_mdev(struct mlx5_ib_dev *dev,
#if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS)
int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user);
void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid);
+void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev);
+void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev);
const struct uverbs_object_tree_def *mlx5_ib_get_devx_tree(void);
extern const struct uapi_definition mlx5_ib_devx_defs[];
extern const struct uapi_definition mlx5_ib_flow_defs[];
@@ -1349,6 +1359,8 @@ static inline int
mlx5_ib_devx_create(struct mlx5_ib_dev *dev,
bool is_user) { return -EOPNOTSUPP; }
static inline void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid) {}
+static inline void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev) {}
+static inline void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev) {}
static inline bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id,
int *dest_type)
{
--
2.20.1
^ permalink raw reply related
* Re: [net-next, PATCH 3/3, v2] net: netsec: add XDP support
From: Ivan Khoronzhuk @ 2019-06-30 16:25 UTC (permalink / raw)
To: Ilias Apalodimas
Cc: netdev, jaswinder.singh, ard.biesheuvel, bjorn.topel,
magnus.karlsson, brouer, daniel, ast, makita.toshiaki,
jakub.kicinski, john.fastabend, davem, maciejromanfijalkowski
In-Reply-To: <1561785805-21647-4-git-send-email-ilias.apalodimas@linaro.org>
On Sat, Jun 29, 2019 at 08:23:25AM +0300, Ilias Apalodimas wrote:
>The interface only supports 1 Tx queue so locking is introduced on
>the Tx queue if XDP is enabled to make sure .ndo_start_xmit and
>.ndo_xdp_xmit won't corrupt Tx ring
>
>- Performance (SMMU off)
>
>Benchmark XDP_SKB XDP_DRV
>xdp1 291kpps 344kpps
>rxdrop 282kpps 342kpps
>
>- Performance (SMMU on)
>Benchmark XDP_SKB XDP_DRV
>xdp1 167kpps 324kpps
>rxdrop 164kpps 323kpps
>
>Signed-off-by: Ilias Apalodimas <ilias.apalodimas@linaro.org>
>---
> drivers/net/ethernet/socionext/netsec.c | 361 ++++++++++++++++++++++--
> 1 file changed, 334 insertions(+), 27 deletions(-)
>
[...]
>+
>+static int netsec_xdp_setup(struct netsec_priv *priv, struct bpf_prog *prog,
>+ struct netlink_ext_ack *extack)
>+{
>+ struct net_device *dev = priv->ndev;
>+ struct bpf_prog *old_prog;
>+
>+ /* For now just support only the usual MTU sized frames */
>+ if (prog && dev->mtu > 1500) {
>+ NL_SET_ERR_MSG_MOD(extack, "Jumbo frames not supported on XDP");
>+ return -EOPNOTSUPP;
>+ }
>+
>+ if (netif_running(dev))
>+ netsec_netdev_stop(dev);
And why to stop the interface. XDP allows to update prog in runtime.
>+
>+ /* Detach old prog, if any */
>+ old_prog = xchg(&priv->xdp_prog, prog);
>+ if (old_prog)
>+ bpf_prog_put(old_prog);
>+
>+ if (netif_running(dev))
>+ netsec_netdev_open(dev);
>+
>+ return 0;
>+}
>+
>+static int netsec_xdp(struct net_device *ndev, struct netdev_bpf *xdp)
>+{
>+ struct netsec_priv *priv = netdev_priv(ndev);
>+
>+ switch (xdp->command) {
>+ case XDP_SETUP_PROG:
>+ return netsec_xdp_setup(priv, xdp->prog, xdp->extack);
>+ case XDP_QUERY_PROG:
>+ xdp->prog_id = priv->xdp_prog ? priv->xdp_prog->aux->id : 0;
>+ return 0;
>+ default:
>+ return -EINVAL;
>+ }
>+}
>+
> static const struct net_device_ops netsec_netdev_ops = {
> .ndo_init = netsec_netdev_init,
> .ndo_uninit = netsec_netdev_uninit,
>@@ -1537,6 +1842,8 @@ static const struct net_device_ops netsec_netdev_ops = {
> .ndo_set_mac_address = eth_mac_addr,
> .ndo_validate_addr = eth_validate_addr,
> .ndo_do_ioctl = netsec_netdev_ioctl,
>+ .ndo_xdp_xmit = netsec_xdp_xmit,
>+ .ndo_bpf = netsec_xdp,
> };
>
> static int netsec_of_probe(struct platform_device *pdev,
>--
>2.20.1
>
--
Regards,
Ivan Khoronzhuk
^ permalink raw reply
* Re: [net-next, PATCH 3/3, v2] net: netsec: add XDP support
From: Ilias Apalodimas @ 2019-06-30 16:32 UTC (permalink / raw)
To: Ivan Khoronzhuk
Cc: netdev, jaswinder.singh, ard.biesheuvel, bjorn.topel,
magnus.karlsson, brouer, daniel, ast, makita.toshiaki,
jakub.kicinski, john.fastabend, davem, maciejromanfijalkowski
In-Reply-To: <20190630162552.GB12704@khorivan>
On Sun, Jun 30, 2019 at 07:25:53PM +0300, Ivan Khoronzhuk wrote:
> On Sat, Jun 29, 2019 at 08:23:25AM +0300, Ilias Apalodimas wrote:
> >The interface only supports 1 Tx queue so locking is introduced on
> >the Tx queue if XDP is enabled to make sure .ndo_start_xmit and
> >.ndo_xdp_xmit won't corrupt Tx ring
> >
> >- Performance (SMMU off)
> >
> >Benchmark XDP_SKB XDP_DRV
> >xdp1 291kpps 344kpps
> >rxdrop 282kpps 342kpps
> >
> >- Performance (SMMU on)
> >Benchmark XDP_SKB XDP_DRV
> >xdp1 167kpps 324kpps
> >rxdrop 164kpps 323kpps
> >
> >Signed-off-by: Ilias Apalodimas <ilias.apalodimas@linaro.org>
> >---
> >drivers/net/ethernet/socionext/netsec.c | 361 ++++++++++++++++++++++--
> >1 file changed, 334 insertions(+), 27 deletions(-)
> >
>
> [...]
>
> >+
> >+static int netsec_xdp_setup(struct netsec_priv *priv, struct bpf_prog *prog,
> >+ struct netlink_ext_ack *extack)
> >+{
> >+ struct net_device *dev = priv->ndev;
> >+ struct bpf_prog *old_prog;
> >+
> >+ /* For now just support only the usual MTU sized frames */
> >+ if (prog && dev->mtu > 1500) {
> >+ NL_SET_ERR_MSG_MOD(extack, "Jumbo frames not supported on XDP");
> >+ return -EOPNOTSUPP;
> >+ }
> >+
> >+ if (netif_running(dev))
> >+ netsec_netdev_stop(dev);
> And why to stop the interface. XDP allows to update prog in runtime.
>
Adding the support is not limited to adding a prog only in this driver.
It also rebuilts the queues which changes the dma mapping of buffers.
Since i don't want to map BIDIRECTIONAL buffers if XDP is not in place,
i am resetting the device and forcing the buffer re-allocation
Thanks
/Ilias
^ permalink raw reply
* Re: [net-next, PATCH 3/3, v2] net: netsec: add XDP support
From: Ilias Apalodimas @ 2019-06-30 16:34 UTC (permalink / raw)
To: Ivan Khoronzhuk
Cc: netdev, jaswinder.singh, ard.biesheuvel, bjorn.topel,
magnus.karlsson, brouer, daniel, ast, makita.toshiaki,
jakub.kicinski, john.fastabend, davem, maciejromanfijalkowski
In-Reply-To: <20190630162042.GA12704@khorivan>
Hi Ivan,
>
> [...]
>
> >+
> >+static int netsec_xdp(struct net_device *ndev, struct netdev_bpf *xdp)
> >+{
> >+ struct netsec_priv *priv = netdev_priv(ndev);
> >+
> >+ switch (xdp->command) {
> >+ case XDP_SETUP_PROG:
> >+ return netsec_xdp_setup(priv, xdp->prog, xdp->extack);
> >+ case XDP_QUERY_PROG:
> >+ xdp->prog_id = priv->xdp_prog ? priv->xdp_prog->aux->id : 0;
> xdp_attachment family to save bpf flags?
Sure why not. This can always be added later though since many drivers are
already doing it similarly no?
>
> >+ return 0;
> >+ default:
> >+ return -EINVAL;
> >+ }
> >+}
> >+
> >static const struct net_device_ops netsec_netdev_ops = {
> > .ndo_init = netsec_netdev_init,
> > .ndo_uninit = netsec_netdev_uninit,
> >@@ -1537,6 +1842,8 @@ static const struct net_device_ops netsec_netdev_ops = {
> > .ndo_set_mac_address = eth_mac_addr,
> > .ndo_validate_addr = eth_validate_addr,
> > .ndo_do_ioctl = netsec_netdev_ioctl,
> >+ .ndo_xdp_xmit = netsec_xdp_xmit,
> >+ .ndo_bpf = netsec_xdp,
> >};
> >
> >static int netsec_of_probe(struct platform_device *pdev,
> >--
> >2.20.1
> >
>
> --
> Regards,
> Ivan Khoronzhuk
Thanks
/Ilias
^ permalink raw reply
* Re: [PATCH net v3] vxlan: do not destroy fdb if register_netdevice() is failed
From: Roopa Prabhu @ 2019-06-30 16:40 UTC (permalink / raw)
To: Taehee Yoo; +Cc: David Miller, Petr Machata, netdev
In-Reply-To: <20190628050725.9445-1-ap420073@gmail.com>
On Thu, Jun 27, 2019 at 10:07 PM Taehee Yoo <ap420073@gmail.com> wrote:
>
> __vxlan_dev_create() destroys FDB using specific pointer which indicates
> a fdb when error occurs.
> But that pointer should not be used when register_netdevice() fails because
> register_netdevice() internally destroys fdb when error occurs.
>
> This patch makes vxlan_fdb_create() to do not link fdb entry to vxlan dev
> internally.
> Instead, a new function vxlan_fdb_insert() is added to link fdb to vxlan
> dev.
>
> vxlan_fdb_insert() is called after calling register_netdevice().
> This routine can avoid situation that ->ndo_uninit() destroys fdb entry
> in error path of register_netdevice().
> Hence, error path of __vxlan_dev_create() routine can have an opportunity
> to destroy default fdb entry by hand.
>
> Test command
> ip link add bonding_masters type vxlan id 0 group 239.1.1.1 \
> dev enp0s9 dstport 4789
>
> Splat looks like:
> [ 213.392816] kasan: GPF could be caused by NULL-ptr deref or user memory access
> [ 213.401257] general protection fault: 0000 [#1] SMP DEBUG_PAGEALLOC KASAN PTI
> [ 213.402178] CPU: 0 PID: 1414 Comm: ip Not tainted 5.2.0-rc5+ #256
> [ 213.402178] RIP: 0010:vxlan_fdb_destroy+0x120/0x220 [vxlan]
> [ 213.402178] Code: df 48 8b 2b 48 89 fa 48 c1 ea 03 80 3c 02 00 0f 85 06 01 00 00 4c 8b 63 08 48 b8 00 00 00 00 00 fc d
> [ 213.402178] RSP: 0018:ffff88810cb9f0a0 EFLAGS: 00010202
> [ 213.402178] RAX: dffffc0000000000 RBX: ffff888101d4a8c8 RCX: 0000000000000000
> [ 213.402178] RDX: 1bd5a00000000040 RSI: ffff888101d4a8c8 RDI: ffff888101d4a8d0
> [ 213.402178] RBP: 0000000000000000 R08: fffffbfff22b72d9 R09: 0000000000000000
> [ 213.402178] R10: 00000000ffffffef R11: 0000000000000000 R12: dead000000000200
> [ 213.402178] R13: ffff88810cb9f1f8 R14: ffff88810efccda0 R15: ffff88810efccda0
> [ 213.402178] FS: 00007f7f6621a0c0(0000) GS:ffff88811b000000(0000) knlGS:0000000000000000
> [ 213.402178] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
> [ 213.402178] CR2: 000055746f0807d0 CR3: 00000001123e0000 CR4: 00000000001006f0
> [ 213.402178] Call Trace:
> [ 213.402178] __vxlan_dev_create+0x3a9/0x7d0 [vxlan]
> [ 213.402178] ? vxlan_changelink+0x740/0x740 [vxlan]
> [ 213.402178] ? rcu_read_unlock+0x60/0x60 [vxlan]
> [ 213.402178] ? __kasan_kmalloc.constprop.3+0xa0/0xd0
> [ 213.402178] vxlan_newlink+0x8d/0xc0 [vxlan]
> [ 213.402178] ? __vxlan_dev_create+0x7d0/0x7d0 [vxlan]
> [ 213.554119] ? __netlink_ns_capable+0xc3/0xf0
> [ 213.554119] __rtnl_newlink+0xb75/0x1180
> [ 213.554119] ? rtnl_link_unregister+0x230/0x230
> [ ... ]
>
> Fixes: 0241b836732f ("vxlan: fix default fdb entry netlink notify ordering during netdev create")
> Suggested-by: Roopa Prabhu <roopa@cumulusnetworks.com>
> Signed-off-by: Taehee Yoo <ap420073@gmail.com>
Acked-by: Roopa Prabhu <roopa@cumulusnetworks.com>
LGTM, thanks!.
> ---
>
> v2 -> v3 :
> - Use function name vxlan_fdb_insert() instead of vxlan_fdb_link()
> - Add and use __vxlan_fdb_free() instead of call_rcu()
>
> v1 -> v2 :
> - Add a new function vxlan_fdb_link().
> - Fix fdb entry leak.
> - Update description.
>
> drivers/net/vxlan.c | 37 +++++++++++++++++++++++++++----------
> 1 file changed, 27 insertions(+), 10 deletions(-)
>
> diff --git a/drivers/net/vxlan.c b/drivers/net/vxlan.c
> index 083f3f0bf37f..b4283f52a09d 100644
> --- a/drivers/net/vxlan.c
> +++ b/drivers/net/vxlan.c
> @@ -804,6 +804,14 @@ static struct vxlan_fdb *vxlan_fdb_alloc(struct vxlan_dev *vxlan,
> return f;
> }
>
> +static void vxlan_fdb_insert(struct vxlan_dev *vxlan, const u8 *mac,
> + __be32 src_vni, struct vxlan_fdb *f)
> +{
> + ++vxlan->addrcnt;
> + hlist_add_head_rcu(&f->hlist,
> + vxlan_fdb_head(vxlan, mac, src_vni));
> +}
> +
> static int vxlan_fdb_create(struct vxlan_dev *vxlan,
> const u8 *mac, union vxlan_addr *ip,
> __u16 state, __be16 port, __be32 src_vni,
> @@ -829,18 +837,13 @@ static int vxlan_fdb_create(struct vxlan_dev *vxlan,
> return rc;
> }
>
> - ++vxlan->addrcnt;
> - hlist_add_head_rcu(&f->hlist,
> - vxlan_fdb_head(vxlan, mac, src_vni));
> -
> *fdb = f;
>
> return 0;
> }
>
> -static void vxlan_fdb_free(struct rcu_head *head)
> +static void __vxlan_fdb_free(struct vxlan_fdb *f)
> {
> - struct vxlan_fdb *f = container_of(head, struct vxlan_fdb, rcu);
> struct vxlan_rdst *rd, *nd;
>
> list_for_each_entry_safe(rd, nd, &f->remotes, list) {
> @@ -850,6 +853,13 @@ static void vxlan_fdb_free(struct rcu_head *head)
> kfree(f);
> }
>
> +static void vxlan_fdb_free(struct rcu_head *head)
> +{
> + struct vxlan_fdb *f = container_of(head, struct vxlan_fdb, rcu);
> +
> + __vxlan_fdb_free(f);
> +}
> +
> static void vxlan_fdb_destroy(struct vxlan_dev *vxlan, struct vxlan_fdb *f,
> bool do_notify, bool swdev_notify)
> {
> @@ -977,6 +987,7 @@ static int vxlan_fdb_update_create(struct vxlan_dev *vxlan,
> if (rc < 0)
> return rc;
>
> + vxlan_fdb_insert(vxlan, mac, src_vni, f);
> rc = vxlan_fdb_notify(vxlan, f, first_remote_rtnl(f), RTM_NEWNEIGH,
> swdev_notify, extack);
> if (rc)
> @@ -3571,12 +3582,17 @@ static int __vxlan_dev_create(struct net *net, struct net_device *dev,
> if (err)
> goto errout;
>
> - /* notify default fdb entry */
> if (f) {
> + vxlan_fdb_insert(vxlan, all_zeros_mac,
> + vxlan->default_dst.remote_vni, f);
> +
> + /* notify default fdb entry */
> err = vxlan_fdb_notify(vxlan, f, first_remote_rtnl(f),
> RTM_NEWNEIGH, true, extack);
> - if (err)
> - goto errout;
> + if (err) {
> + vxlan_fdb_destroy(vxlan, f, false, false);
> + goto unregister;
> + }
> }
>
> list_add(&vxlan->next, &vn->vxlan_list);
> @@ -3588,7 +3604,8 @@ static int __vxlan_dev_create(struct net *net, struct net_device *dev,
> * destroy the entry by hand here.
> */
> if (f)
> - vxlan_fdb_destroy(vxlan, f, false, false);
> + __vxlan_fdb_free(f);
> +unregister:
> if (unregister)
> unregister_netdevice(dev);
> return err;
> --
> 2.17.1
>
^ permalink raw reply
* Re: [net-next, PATCH 3/3, v2] net: netsec: add XDP support
From: Ivan Khoronzhuk @ 2019-06-30 16:41 UTC (permalink / raw)
To: Ilias Apalodimas
Cc: netdev, jaswinder.singh, ard.biesheuvel, bjorn.topel,
magnus.karlsson, brouer, daniel, ast, makita.toshiaki,
jakub.kicinski, john.fastabend, davem, maciejromanfijalkowski
In-Reply-To: <20190630163214.GA10484@apalos>
On Sun, Jun 30, 2019 at 07:32:14PM +0300, Ilias Apalodimas wrote:
>On Sun, Jun 30, 2019 at 07:25:53PM +0300, Ivan Khoronzhuk wrote:
>> On Sat, Jun 29, 2019 at 08:23:25AM +0300, Ilias Apalodimas wrote:
>> >The interface only supports 1 Tx queue so locking is introduced on
>> >the Tx queue if XDP is enabled to make sure .ndo_start_xmit and
>> >.ndo_xdp_xmit won't corrupt Tx ring
>> >
>> >- Performance (SMMU off)
>> >
>> >Benchmark XDP_SKB XDP_DRV
>> >xdp1 291kpps 344kpps
>> >rxdrop 282kpps 342kpps
>> >
>> >- Performance (SMMU on)
>> >Benchmark XDP_SKB XDP_DRV
>> >xdp1 167kpps 324kpps
>> >rxdrop 164kpps 323kpps
>> >
>> >Signed-off-by: Ilias Apalodimas <ilias.apalodimas@linaro.org>
>> >---
>> >drivers/net/ethernet/socionext/netsec.c | 361 ++++++++++++++++++++++--
>> >1 file changed, 334 insertions(+), 27 deletions(-)
>> >
>>
>> [...]
>>
>> >+
>> >+static int netsec_xdp_setup(struct netsec_priv *priv, struct bpf_prog *prog,
>> >+ struct netlink_ext_ack *extack)
>> >+{
>> >+ struct net_device *dev = priv->ndev;
>> >+ struct bpf_prog *old_prog;
>> >+
>> >+ /* For now just support only the usual MTU sized frames */
>> >+ if (prog && dev->mtu > 1500) {
>> >+ NL_SET_ERR_MSG_MOD(extack, "Jumbo frames not supported on XDP");
>> >+ return -EOPNOTSUPP;
>> >+ }
>> >+
>> >+ if (netif_running(dev))
>> >+ netsec_netdev_stop(dev);
>> And why to stop the interface. XDP allows to update prog in runtime.
>>
>Adding the support is not limited to adding a prog only in this driver.
>It also rebuilts the queues which changes the dma mapping of buffers.
>Since i don't want to map BIDIRECTIONAL buffers if XDP is not in place,
>i am resetting the device and forcing the buffer re-allocation
>
>Thanks
>/Ilias
I don't know the internals, probably it has some dependencies, but here you
just update the prog and can at least do it when exchange is happening.
I mean not in case of prog is attached/removed first time.
In case of prog -> prog it seems doable...
It ups to you ofc, but I can run smth like:
ip -force link set dev eth0 xdp obj xdp-example-pass.o sec .text
and expect it's updated w/o interface reset I mean on new prog.
I'm not sure, but maintainers can help, conceptually it's supposed to be in
runtime the prog be update uder rcu as a part of API usage...
--
Regards,
Ivan Khoronzhuk
^ permalink raw reply
* Re: [net-next, PATCH 3/3, v2] net: netsec: add XDP support
From: Ivan Khoronzhuk @ 2019-06-30 16:45 UTC (permalink / raw)
To: Ilias Apalodimas
Cc: netdev, jaswinder.singh, ard.biesheuvel, bjorn.topel,
magnus.karlsson, brouer, daniel, ast, makita.toshiaki,
jakub.kicinski, john.fastabend, davem, maciejromanfijalkowski
In-Reply-To: <20190630163417.GB10484@apalos>
On Sun, Jun 30, 2019 at 07:34:17PM +0300, Ilias Apalodimas wrote:
>Hi Ivan,
>>
>> [...]
>>
>> >+
>> >+static int netsec_xdp(struct net_device *ndev, struct netdev_bpf *xdp)
>> >+{
>> >+ struct netsec_priv *priv = netdev_priv(ndev);
>> >+
>> >+ switch (xdp->command) {
>> >+ case XDP_SETUP_PROG:
>> >+ return netsec_xdp_setup(priv, xdp->prog, xdp->extack);
>> >+ case XDP_QUERY_PROG:
>> >+ xdp->prog_id = priv->xdp_prog ? priv->xdp_prog->aux->id : 0;
>> xdp_attachment family to save bpf flags?
>Sure why not. This can always be added later though since many drivers are
>already doing it similarly no?
yes.
I can work w/o this ofc.
But netronome and cpsw (me) added this.
What I've seen it allows to prevent prog update if flag doesn't allow it.
Usually it doesn't allow, but can be forced with flag. In another case it can
be updated any time w/o reason...and seems like in your case it's sensitive.
>
>>
>> >+ return 0;
>> >+ default:
>> >+ return -EINVAL;
>> >+ }
>> >+}
>> >+
>> >static const struct net_device_ops netsec_netdev_ops = {
>> > .ndo_init = netsec_netdev_init,
>> > .ndo_uninit = netsec_netdev_uninit,
>> >@@ -1537,6 +1842,8 @@ static const struct net_device_ops netsec_netdev_ops = {
>> > .ndo_set_mac_address = eth_mac_addr,
>> > .ndo_validate_addr = eth_validate_addr,
>> > .ndo_do_ioctl = netsec_netdev_ioctl,
>> >+ .ndo_xdp_xmit = netsec_xdp_xmit,
>> >+ .ndo_bpf = netsec_xdp,
>> >};
>> >
>> >static int netsec_of_probe(struct platform_device *pdev,
>> >--
>> >2.20.1
>> >
>>
>> --
>> Regards,
>> Ivan Khoronzhuk
>
>Thanks
>/Ilias
--
Regards,
Ivan Khoronzhuk
^ permalink raw reply
* Re: [net-next, PATCH 3/3, v2] net: netsec: add XDP support
From: Ilias Apalodimas @ 2019-06-30 16:47 UTC (permalink / raw)
To: Ivan Khoronzhuk
Cc: netdev, jaswinder.singh, ard.biesheuvel, bjorn.topel,
magnus.karlsson, brouer, daniel, ast, makita.toshiaki,
jakub.kicinski, john.fastabend, davem, maciejromanfijalkowski
In-Reply-To: <20190630164127.GC12704@khorivan>
On Sun, Jun 30, 2019 at 07:41:28PM +0300, Ivan Khoronzhuk wrote:
> On Sun, Jun 30, 2019 at 07:32:14PM +0300, Ilias Apalodimas wrote:
> >On Sun, Jun 30, 2019 at 07:25:53PM +0300, Ivan Khoronzhuk wrote:
> >>On Sat, Jun 29, 2019 at 08:23:25AM +0300, Ilias Apalodimas wrote:
> >>>The interface only supports 1 Tx queue so locking is introduced on
> >>>the Tx queue if XDP is enabled to make sure .ndo_start_xmit and
> >>>.ndo_xdp_xmit won't corrupt Tx ring
> >>>
> >>>- Performance (SMMU off)
> >>>
> >>>Benchmark XDP_SKB XDP_DRV
> >>>xdp1 291kpps 344kpps
> >>>rxdrop 282kpps 342kpps
> >>>
> >>>- Performance (SMMU on)
> >>>Benchmark XDP_SKB XDP_DRV
> >>>xdp1 167kpps 324kpps
> >>>rxdrop 164kpps 323kpps
> >>>
> >>>Signed-off-by: Ilias Apalodimas <ilias.apalodimas@linaro.org>
> >>>---
> >>>drivers/net/ethernet/socionext/netsec.c | 361 ++++++++++++++++++++++--
> >>>1 file changed, 334 insertions(+), 27 deletions(-)
> >>>
> >>
> >>[...]
> >>
> >>>+
> >>>+static int netsec_xdp_setup(struct netsec_priv *priv, struct bpf_prog *prog,
> >>>+ struct netlink_ext_ack *extack)
> >>>+{
> >>>+ struct net_device *dev = priv->ndev;
> >>>+ struct bpf_prog *old_prog;
> >>>+
> >>>+ /* For now just support only the usual MTU sized frames */
> >>>+ if (prog && dev->mtu > 1500) {
> >>>+ NL_SET_ERR_MSG_MOD(extack, "Jumbo frames not supported on XDP");
> >>>+ return -EOPNOTSUPP;
> >>>+ }
> >>>+
> >>>+ if (netif_running(dev))
> >>>+ netsec_netdev_stop(dev);
> >>And why to stop the interface. XDP allows to update prog in runtime.
> >>
> >Adding the support is not limited to adding a prog only in this driver.
> >It also rebuilts the queues which changes the dma mapping of buffers.
> >Since i don't want to map BIDIRECTIONAL buffers if XDP is not in place,
> >i am resetting the device and forcing the buffer re-allocation
> >
> >Thanks
> >/Ilias
> I don't know the internals, probably it has some dependencies, but here you
> just update the prog and can at least do it when exchange is happening.
> I mean not in case of prog is attached/removed first time.
> In case of prog -> prog it seems doable...
>
> It ups to you ofc, but I can run smth like:
> ip -force link set dev eth0 xdp obj xdp-example-pass.o sec .text
> and expect it's updated w/o interface reset I mean on new prog.
>
> I'm not sure, but maintainers can help, conceptually it's supposed to be in
> runtime the prog be update uder rcu as a part of API usage...
It's doable but it means i'd have to change the buffer allocation again. I'd
also prefer mapping FOR_DEVICE only if XDP is not loaded. Most of the drivers do
restart so i'll stick with this for the current version.
Most of the drivers do restart now so i'll stick to that for now.
Thanks
/Ilias
^ permalink raw reply
* Re: [net-next, PATCH 3/3, v2] net: netsec: add XDP support
From: Ilias Apalodimas @ 2019-06-30 16:50 UTC (permalink / raw)
To: Ivan Khoronzhuk
Cc: netdev, jaswinder.singh, ard.biesheuvel, bjorn.topel,
magnus.karlsson, brouer, daniel, ast, makita.toshiaki,
jakub.kicinski, john.fastabend, davem, maciejromanfijalkowski
In-Reply-To: <20190630164512.GD12704@khorivan>
On Sun, Jun 30, 2019 at 07:45:13PM +0300, Ivan Khoronzhuk wrote:
> On Sun, Jun 30, 2019 at 07:34:17PM +0300, Ilias Apalodimas wrote:
> >Hi Ivan,
> >>
> >>[...]
> >>
> >>>+
> >>>+static int netsec_xdp(struct net_device *ndev, struct netdev_bpf *xdp)
> >>>+{
> >>>+ struct netsec_priv *priv = netdev_priv(ndev);
> >>>+
> >>>+ switch (xdp->command) {
> >>>+ case XDP_SETUP_PROG:
> >>>+ return netsec_xdp_setup(priv, xdp->prog, xdp->extack);
> >>>+ case XDP_QUERY_PROG:
> >>>+ xdp->prog_id = priv->xdp_prog ? priv->xdp_prog->aux->id : 0;
> >>xdp_attachment family to save bpf flags?
> >Sure why not. This can always be added later though since many drivers are
> >already doing it similarly no?
> yes.
> I can work w/o this ofc.
> But netronome and cpsw (me) added this.
Ok let's start using that
> What I've seen it allows to prevent prog update if flag doesn't allow it.
> Usually it doesn't allow, but can be forced with flag. In another case it can
> be updated any time w/o reason...and seems like in your case it's sensitive.
I intend to send a follow up patch anyway to remove the declaration on the top
of the file of netsec_set_tx_de(). I intentionally choose to add that to make
the review easier (since re-arranging would mess that up).
I'll just this optimization as well on the follow up patch since it doesn't
break anything
Thanks
/Ilias
^ permalink raw reply
* Re: [net-next, PATCH 3/3, v2] net: netsec: add XDP support
From: Ivan Khoronzhuk @ 2019-06-30 16:51 UTC (permalink / raw)
To: Ilias Apalodimas
Cc: netdev, jaswinder.singh, ard.biesheuvel, bjorn.topel,
magnus.karlsson, brouer, daniel, ast, makita.toshiaki,
jakub.kicinski, john.fastabend, davem, maciejromanfijalkowski
In-Reply-To: <20190630164745.GA11278@apalos>
On Sun, Jun 30, 2019 at 07:47:45PM +0300, Ilias Apalodimas wrote:
>On Sun, Jun 30, 2019 at 07:41:28PM +0300, Ivan Khoronzhuk wrote:
>> On Sun, Jun 30, 2019 at 07:32:14PM +0300, Ilias Apalodimas wrote:
>> >On Sun, Jun 30, 2019 at 07:25:53PM +0300, Ivan Khoronzhuk wrote:
>> >>On Sat, Jun 29, 2019 at 08:23:25AM +0300, Ilias Apalodimas wrote:
>> >>>The interface only supports 1 Tx queue so locking is introduced on
>> >>>the Tx queue if XDP is enabled to make sure .ndo_start_xmit and
>> >>>.ndo_xdp_xmit won't corrupt Tx ring
>> >>>
>> >>>- Performance (SMMU off)
>> >>>
>> >>>Benchmark XDP_SKB XDP_DRV
>> >>>xdp1 291kpps 344kpps
>> >>>rxdrop 282kpps 342kpps
>> >>>
>> >>>- Performance (SMMU on)
>> >>>Benchmark XDP_SKB XDP_DRV
>> >>>xdp1 167kpps 324kpps
>> >>>rxdrop 164kpps 323kpps
>> >>>
>> >>>Signed-off-by: Ilias Apalodimas <ilias.apalodimas@linaro.org>
>> >>>---
>> >>>drivers/net/ethernet/socionext/netsec.c | 361 ++++++++++++++++++++++--
>> >>>1 file changed, 334 insertions(+), 27 deletions(-)
>> >>>
>> >>
>> >>[...]
>> >>
>> >>>+
>> >>>+static int netsec_xdp_setup(struct netsec_priv *priv, struct bpf_prog *prog,
>> >>>+ struct netlink_ext_ack *extack)
>> >>>+{
>> >>>+ struct net_device *dev = priv->ndev;
>> >>>+ struct bpf_prog *old_prog;
>> >>>+
>> >>>+ /* For now just support only the usual MTU sized frames */
>> >>>+ if (prog && dev->mtu > 1500) {
>> >>>+ NL_SET_ERR_MSG_MOD(extack, "Jumbo frames not supported on XDP");
>> >>>+ return -EOPNOTSUPP;
>> >>>+ }
>> >>>+
>> >>>+ if (netif_running(dev))
>> >>>+ netsec_netdev_stop(dev);
>> >>And why to stop the interface. XDP allows to update prog in runtime.
>> >>
>> >Adding the support is not limited to adding a prog only in this driver.
>> >It also rebuilts the queues which changes the dma mapping of buffers.
>> >Since i don't want to map BIDIRECTIONAL buffers if XDP is not in place,
>> >i am resetting the device and forcing the buffer re-allocation
>> >
>> >Thanks
>> >/Ilias
>> I don't know the internals, probably it has some dependencies, but here you
>> just update the prog and can at least do it when exchange is happening.
>> I mean not in case of prog is attached/removed first time.
>> In case of prog -> prog it seems doable...
>>
>> It ups to you ofc, but I can run smth like:
>> ip -force link set dev eth0 xdp obj xdp-example-pass.o sec .text
>> and expect it's updated w/o interface reset I mean on new prog.
>>
>> I'm not sure, but maintainers can help, conceptually it's supposed to be in
>> runtime the prog be update uder rcu as a part of API usage...
>It's doable but it means i'd have to change the buffer allocation again. I'd
>also prefer mapping FOR_DEVICE only if XDP is not loaded. Most of the drivers do
>restart so i'll stick with this for the current version.
>Most of the drivers do restart now so i'll stick to that for now.
I have nothing against it to be merged. Just pay attention on
XDP_FLAGS_UPDATE_IF_NOEXIST or update it layter or now...ups to your.
>
>Thanks
>/Ilias
--
Regards,
Ivan Khoronzhuk
^ permalink raw reply
* Re: [RFC net-next] net: dsa: add support for MC_DISABLED attribute
From: Linus Lüssing @ 2019-06-30 16:56 UTC (permalink / raw)
To: Ido Schimmel
Cc: Russell King - ARM Linux admin, nikolay, Ido Schimmel,
Vivien Didelot, Florian Fainelli, netdev@vger.kernel.org,
Jiri Pirko, andrew@lunn.ch, davem@davemloft.net, bridge,
b.a.t.m.a.n
In-Reply-To: <20190629162945.GB17143@splinter>
On Sat, Jun 29, 2019 at 07:29:45PM +0300, Ido Schimmel wrote:
> I would like to avoid having drivers take the querier state into account
> as it will only complicate things further.
I absolutely share your pain. Initially in the early prototypes of
multicast awareness in batman-adv we did not consider the querier state.
And doing so later did indeed complicate the code a good bit in batman-adv
(together with the IGMP/MLD suppression issues). I would have loved to
avoid that.
> Is there anything we can do about it? Enable the bridge querier if no
> other querier was detected? Commit c5c23260594c ("bridge: Add
> multicast_querier toggle and disable queries by default") disabled
> queries by default, but I'm only suggesting to turn them on if no other
> querier was detected on the link. Do you think it's still a problem?
As soon as you start becoming the querier, you will not be able to reliably
detect anymore whether you are the only querier candidate.
If any random Linux host using a bridge device were potentially becoming
a querier, that would cause quite some trouble when this host is
behind some bad, bottleneck connection. This host will receive
all multicast traffic, not just IGMP/MLD reports. And with a
congested connection and then unreliable IGMP/MLD, multicast would
become unreliable overall in this domain. So it's important that
your querier is not running in the "dark, remote, dusty closet" of
your network (topologically speaking).
> On Sun, Jun 23, 2019 at 10:44:27AM +0300, Ido Schimmel wrote:
> > See commit b00589af3b04 ("bridge: disable snooping if there is no
> > querier"). I think that's unfortunate behavior that we need because
> > multicast snooping is enabled by default. If it weren't enabled by
> > default, then anyone enabling it would also make sure there's a querier
> > in the network.
I do not quite understand that point. In a way, that's what we
have right now, isn't it? By default it's disabled, because by
default there is no querier on the link. So anyone wanting to use
multicast snooping will need to make sure there's a querier in the
network.
Overall I think the querier (election) mechanism in the standards could
need an update. While the lowest-address first might have
worked well back then, in uniform, fully wired networks where the
position of the querier did not matter, this is not a good
solution anymore in networks involving wireless, dynamic connections.
Especially in wireless mesh networks this is a bit of an issue for
us. Ideally, the querier mechanism were dismissed in favour of simply
unsolicited, periodic IGMP/MLD reports...
But of course, updating IETF standards is no solution for now.
While more complicated, it would not be impossible to consider the
querier state, would it? I mean you probably already need to
consider the case of a user disabling multicast snooping during
runtime, right? So similarly, you could react to appearing or
disappearing queriers?
Cheers, Linus
^ permalink raw reply
* Re: [net-next, PATCH 3/3, v2] net: netsec: add XDP support
From: Ivan Khoronzhuk @ 2019-06-30 17:09 UTC (permalink / raw)
To: Ilias Apalodimas
Cc: netdev, jaswinder.singh, ard.biesheuvel, bjorn.topel,
magnus.karlsson, brouer, daniel, ast, makita.toshiaki,
jakub.kicinski, john.fastabend, davem, maciejromanfijalkowski
In-Reply-To: <20190630164745.GA11278@apalos>
On Sun, Jun 30, 2019 at 07:47:45PM +0300, Ilias Apalodimas wrote:
>On Sun, Jun 30, 2019 at 07:41:28PM +0300, Ivan Khoronzhuk wrote:
>> On Sun, Jun 30, 2019 at 07:32:14PM +0300, Ilias Apalodimas wrote:
>> >On Sun, Jun 30, 2019 at 07:25:53PM +0300, Ivan Khoronzhuk wrote:
>> >>On Sat, Jun 29, 2019 at 08:23:25AM +0300, Ilias Apalodimas wrote:
>> >>>The interface only supports 1 Tx queue so locking is introduced on
>> >>>the Tx queue if XDP is enabled to make sure .ndo_start_xmit and
>> >>>.ndo_xdp_xmit won't corrupt Tx ring
>> >>>
>> >>>- Performance (SMMU off)
>> >>>
>> >>>Benchmark XDP_SKB XDP_DRV
>> >>>xdp1 291kpps 344kpps
>> >>>rxdrop 282kpps 342kpps
>> >>>
>> >>>- Performance (SMMU on)
>> >>>Benchmark XDP_SKB XDP_DRV
>> >>>xdp1 167kpps 324kpps
>> >>>rxdrop 164kpps 323kpps
>> >>>
>> >>>Signed-off-by: Ilias Apalodimas <ilias.apalodimas@linaro.org>
>> >>>---
>> >>>drivers/net/ethernet/socionext/netsec.c | 361 ++++++++++++++++++++++--
>> >>>1 file changed, 334 insertions(+), 27 deletions(-)
>> >>>
>> >>
>> >>[...]
>> >>
>> >>>+
>> >>>+static int netsec_xdp_setup(struct netsec_priv *priv, struct bpf_prog *prog,
>> >>>+ struct netlink_ext_ack *extack)
>> >>>+{
>> >>>+ struct net_device *dev = priv->ndev;
>> >>>+ struct bpf_prog *old_prog;
>> >>>+
>> >>>+ /* For now just support only the usual MTU sized frames */
>> >>>+ if (prog && dev->mtu > 1500) {
>> >>>+ NL_SET_ERR_MSG_MOD(extack, "Jumbo frames not supported on XDP");
>> >>>+ return -EOPNOTSUPP;
>> >>>+ }
>> >>>+
>> >>>+ if (netif_running(dev))
>> >>>+ netsec_netdev_stop(dev);
>> >>And why to stop the interface. XDP allows to update prog in runtime.
>> >>
>> >Adding the support is not limited to adding a prog only in this driver.
>> >It also rebuilts the queues which changes the dma mapping of buffers.
>> >Since i don't want to map BIDIRECTIONAL buffers if XDP is not in place,
>> >i am resetting the device and forcing the buffer re-allocation
>> >
>> >Thanks
>> >/Ilias
>> I don't know the internals, probably it has some dependencies, but here you
>> just update the prog and can at least do it when exchange is happening.
>> I mean not in case of prog is attached/removed first time.
>> In case of prog -> prog it seems doable...
>>
>> It ups to you ofc, but I can run smth like:
>> ip -force link set dev eth0 xdp obj xdp-example-pass.o sec .text
>> and expect it's updated w/o interface reset I mean on new prog.
>>
>> I'm not sure, but maintainers can help, conceptually it's supposed to be in
>> runtime the prog be update uder rcu as a part of API usage...
>It's doable but it means i'd have to change the buffer allocation again. I'd
In case prog -> prog update you don't need to do anyting,
just exchange prog, That's it.
You can add it later, if you want. np.
--
Regards,
Ivan Khoronzhuk
^ permalink raw reply
* [PATCH v5 net-next 0/6] net: ethernet: ti: cpsw: Add XDP support
From: Ivan Khoronzhuk @ 2019-06-30 17:23 UTC (permalink / raw)
To: grygorii.strashko, hawk, davem
Cc: ast, linux-kernel, linux-omap, xdp-newbies, ilias.apalodimas,
netdev, daniel, jakub.kicinski, john.fastabend, Ivan Khoronzhuk
This patchset adds XDP support for TI cpsw driver and base it on
page_pool allocator. It was verified on af_xdp socket drop,
af_xdp l2f, ebpf XDP_DROP, XDP_REDIRECT, XDP_PASS, XDP_TX.
It was verified with following configs enabled:
CONFIG_JIT=y
CONFIG_BPFILTER=y
CONFIG_BPF_SYSCALL=y
CONFIG_XDP_SOCKETS=y
CONFIG_BPF_EVENTS=y
CONFIG_HAVE_EBPF_JIT=y
CONFIG_BPF_JIT=y
CONFIG_CGROUP_BPF=y
Link on previous v4:
https://lkml.org/lkml/2019/6/25/996
Also regular tests with iperf2 were done in order to verify impact on
regular netstack performance, compared with base commit:
https://pastebin.com/JSMT0iZ4
v4..v5:
- added two plreliminary patches:
net: ethernet: ti: davinci_cpdma: allow desc split while down
net: ethernet: ti: cpsw_ethtool: allow res split while down
- added xdp alocator refcnt on xdp level, avoiding page pool refcnt
- moved flush status as separate argument for cpdma_chan_process
- reworked cpsw code according to last changes to allocator
- added missed statistic counter
v3..v4:
- added page pool user counter
- use same pool for ndevs in dual mac
- restructured page pool create/destroy according to the last changes in API
v2..v3:
- each rxq and ndev has its own page pool
v1..v2:
- combined xdp_xmit functions
- used page allocation w/o refcnt juggle
- unmapped page for skb netstack
- moved rxq/page pool allocation to open/close pair
- added several preliminary patches:
net: page_pool: add helper function to retrieve dma addresses
net: page_pool: add helper function to unmap dma addresses
net: ethernet: ti: cpsw: use cpsw as drv data
net: ethernet: ti: cpsw_ethtool: simplify slave loops
Based on net-next/master
Ivan Khoronzhuk (6):
xdp: allow same allocator usage
net: ethernet: ti: davinci_cpdma: add dma mapped submit
net: ethernet: ti: davinci_cpdma: return handler status
net: ethernet: ti: davinci_cpdma: allow desc split while down
net: ethernet: ti: cpsw_ethtool: allow res split while down
net: ethernet: ti: cpsw: add XDP support
drivers/net/ethernet/ti/Kconfig | 1 +
drivers/net/ethernet/ti/cpsw.c | 520 +++++++++++++++++++++---
drivers/net/ethernet/ti/cpsw_ethtool.c | 78 +++-
drivers/net/ethernet/ti/cpsw_priv.h | 9 +-
drivers/net/ethernet/ti/davinci_cpdma.c | 125 +++++-
drivers/net/ethernet/ti/davinci_cpdma.h | 11 +-
drivers/net/ethernet/ti/davinci_emac.c | 17 +-
include/net/xdp_priv.h | 1 +
net/core/xdp.c | 46 +++
9 files changed, 701 insertions(+), 107 deletions(-)
--
2.17.1
^ permalink raw reply
* [PATCH v5 net-next 6/6] net: ethernet: ti: cpsw: add XDP support
From: Ivan Khoronzhuk @ 2019-06-30 17:23 UTC (permalink / raw)
To: grygorii.strashko, hawk, davem
Cc: ast, linux-kernel, linux-omap, xdp-newbies, ilias.apalodimas,
netdev, daniel, jakub.kicinski, john.fastabend, Ivan Khoronzhuk
In-Reply-To: <20190630172348.5692-1-ivan.khoronzhuk@linaro.org>
Add XDP support based on rx page_pool allocator, one frame per page.
Page pool allocator is used with assumption that only one rx_handler
is running simultaneously. DMA map/unmap is reused from page pool
despite there is no need to map whole page.
Due to specific of cpsw, the same TX/RX handler can be used by 2
network devices, so special fields in buffer are added to identify
an interface the frame is destined to. Thus XDP works for both
interfaces, that allows to test xdp redirect between two interfaces
easily. Aslo, each rx queue have own page pools, but common for both
netdevs.
XDP prog is common for all channels till appropriate changes are added
in XDP infrastructure. Also, once page_pool recycling becomes part of
skb netstack some simplifications can be added, like removing
page_pool_release_page() before skb receive.
Signed-off-by: Ivan Khoronzhuk <ivan.khoronzhuk@linaro.org>
---
drivers/net/ethernet/ti/Kconfig | 1 +
drivers/net/ethernet/ti/cpsw.c | 501 ++++++++++++++++++++++---
drivers/net/ethernet/ti/cpsw_ethtool.c | 66 +++-
drivers/net/ethernet/ti/cpsw_priv.h | 7 +
4 files changed, 515 insertions(+), 60 deletions(-)
diff --git a/drivers/net/ethernet/ti/Kconfig b/drivers/net/ethernet/ti/Kconfig
index a800d3417411..834afca3a019 100644
--- a/drivers/net/ethernet/ti/Kconfig
+++ b/drivers/net/ethernet/ti/Kconfig
@@ -50,6 +50,7 @@ config TI_CPSW
depends on ARCH_DAVINCI || ARCH_OMAP2PLUS || COMPILE_TEST
select TI_DAVINCI_MDIO
select MFD_SYSCON
+ select PAGE_POOL
select REGMAP
---help---
This driver supports TI's CPSW Ethernet Switch.
diff --git a/drivers/net/ethernet/ti/cpsw.c b/drivers/net/ethernet/ti/cpsw.c
index 4f72dbb5a428..62ad8d4e231d 100644
--- a/drivers/net/ethernet/ti/cpsw.c
+++ b/drivers/net/ethernet/ti/cpsw.c
@@ -31,6 +31,10 @@
#include <linux/if_vlan.h>
#include <linux/kmemleak.h>
#include <linux/sys_soc.h>
+#include <net/page_pool.h>
+#include <linux/bpf.h>
+#include <linux/bpf_trace.h>
+#include <linux/filter.h>
#include <linux/pinctrl/consumer.h>
#include <net/pkt_cls.h>
@@ -60,6 +64,10 @@ static int descs_pool_size = CPSW_CPDMA_DESCS_POOL_SIZE_DEFAULT;
module_param(descs_pool_size, int, 0444);
MODULE_PARM_DESC(descs_pool_size, "Number of CPDMA CPPI descriptors in pool");
+/* The buf includes headroom compatible with both skb and xdpf */
+#define CPSW_HEADROOM_NA (max(XDP_PACKET_HEADROOM, NET_SKB_PAD) + NET_IP_ALIGN)
+#define CPSW_HEADROOM ALIGN(CPSW_HEADROOM_NA, sizeof(long))
+
#define for_each_slave(priv, func, arg...) \
do { \
struct cpsw_slave *slave; \
@@ -74,6 +82,13 @@ MODULE_PARM_DESC(descs_pool_size, "Number of CPDMA CPPI descriptors in pool");
(func)(slave++, ##arg); \
} while (0)
+#define CPSW_XMETA_OFFSET ALIGN(sizeof(struct xdp_frame), sizeof(long))
+
+#define CPSW_XDP_CONSUMED 1
+#define CPSW_XDP_CONSUMED_FLUSH 2
+#define CPSW_XDP_PASS 0
+#define CPSW_FLUSH_XDP_MAP 1
+
static int cpsw_ndo_vlan_rx_add_vid(struct net_device *ndev,
__be16 proto, u16 vid);
@@ -337,24 +352,58 @@ void cpsw_intr_disable(struct cpsw_common *cpsw)
return;
}
+static int cpsw_is_xdpf_handle(void *handle)
+{
+ return (unsigned long)handle & BIT(0);
+}
+
+static void *cpsw_xdpf_to_handle(struct xdp_frame *xdpf)
+{
+ return (void *)((unsigned long)xdpf | BIT(0));
+}
+
+static struct xdp_frame *cpsw_handle_to_xdpf(void *handle)
+{
+ return (struct xdp_frame *)((unsigned long)handle & ~BIT(0));
+}
+
+struct __aligned(sizeof(long)) cpsw_meta_xdp {
+ struct net_device *ndev;
+ int ch;
+};
+
int cpsw_tx_handler(void *token, int len, int status)
{
+ struct cpsw_meta_xdp *xmeta;
+ struct xdp_frame *xdpf;
+ struct net_device *ndev;
struct netdev_queue *txq;
- struct sk_buff *skb = token;
- struct net_device *ndev = skb->dev;
- struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
+ struct sk_buff *skb;
+ int ch;
+
+ if (cpsw_is_xdpf_handle(token)) {
+ xdpf = cpsw_handle_to_xdpf(token);
+ xmeta = (void *)xdpf + CPSW_XMETA_OFFSET;
+ ndev = xmeta->ndev;
+ ch = xmeta->ch;
+ xdp_return_frame(xdpf);
+ } else {
+ skb = token;
+ ndev = skb->dev;
+ ch = skb_get_queue_mapping(skb);
+ cpts_tx_timestamp(ndev_to_cpsw(ndev)->cpts, skb);
+ dev_kfree_skb_any(skb);
+ }
/* Check whether the queue is stopped due to stalled tx dma, if the
* queue is stopped then start the queue as we have free desc for tx
*/
- txq = netdev_get_tx_queue(ndev, skb_get_queue_mapping(skb));
+ txq = netdev_get_tx_queue(ndev, ch);
if (unlikely(netif_tx_queue_stopped(txq)))
netif_tx_wake_queue(txq);
- cpts_tx_timestamp(cpsw->cpts, skb);
ndev->stats.tx_packets++;
ndev->stats.tx_bytes += len;
- dev_kfree_skb_any(skb);
return 0;
}
@@ -401,24 +450,233 @@ static void cpsw_rx_vlan_encap(struct sk_buff *skb)
}
}
+static int cpsw_xdp_tx_frame(struct cpsw_priv *priv, struct xdp_frame *xdpf,
+ struct page *page)
+{
+ struct cpsw_common *cpsw = priv->cpsw;
+ struct cpsw_meta_xdp *xmeta;
+ struct cpdma_chan *txch;
+ dma_addr_t dma;
+ int ret, port;
+
+ xmeta = (void *)xdpf + CPSW_XMETA_OFFSET;
+ xmeta->ndev = priv->ndev;
+ xmeta->ch = 0;
+ txch = cpsw->txv[0].ch;
+
+ port = priv->emac_port + cpsw->data.dual_emac;
+ if (page) {
+ dma = page_pool_get_dma_addr(page);
+ dma += xdpf->data - (void *)xdpf;
+ ret = cpdma_chan_submit_mapped(txch, cpsw_xdpf_to_handle(xdpf),
+ dma, xdpf->len, port);
+ } else {
+ if (sizeof(*xmeta) > xdpf->headroom) {
+ xdp_return_frame_rx_napi(xdpf);
+ return -EINVAL;
+ }
+
+ ret = cpdma_chan_submit(txch, cpsw_xdpf_to_handle(xdpf),
+ xdpf->data, xdpf->len, port);
+ }
+
+ if (ret) {
+ priv->ndev->stats.tx_dropped++;
+ xdp_return_frame_rx_napi(xdpf);
+ }
+
+ return ret;
+}
+
+static int cpsw_run_xdp(struct cpsw_priv *priv, int ch, struct xdp_buff *xdp,
+ struct page *page)
+{
+ struct cpsw_common *cpsw = priv->cpsw;
+ struct net_device *ndev = priv->ndev;
+ int ret = CPSW_XDP_CONSUMED;
+ struct xdp_frame *xdpf;
+ struct bpf_prog *prog;
+ u32 act;
+
+ rcu_read_lock();
+
+ prog = READ_ONCE(priv->xdp_prog);
+ if (!prog) {
+ ret = CPSW_XDP_PASS;
+ goto out;
+ }
+
+ act = bpf_prog_run_xdp(prog, xdp);
+ switch (act) {
+ case XDP_PASS:
+ ret = CPSW_XDP_PASS;
+ break;
+ case XDP_TX:
+ xdpf = convert_to_xdp_frame(xdp);
+ if (unlikely(!xdpf))
+ goto drop;
+
+ cpsw_xdp_tx_frame(priv, xdpf, page);
+ break;
+ case XDP_REDIRECT:
+ if (xdp_do_redirect(ndev, xdp, prog))
+ goto drop;
+
+ ret = CPSW_XDP_CONSUMED_FLUSH;
+ break;
+ default:
+ bpf_warn_invalid_xdp_action(act);
+ /* fall through */
+ case XDP_ABORTED:
+ trace_xdp_exception(ndev, prog, act);
+ /* fall through -- handle aborts by dropping packet */
+ case XDP_DROP:
+ goto drop;
+ }
+out:
+ rcu_read_unlock();
+ return ret;
+drop:
+ rcu_read_unlock();
+ page_pool_recycle_direct(cpsw->page_pool[ch], page);
+ return ret;
+}
+
+static unsigned int cpsw_rxbuf_total_len(unsigned int len)
+{
+ len += CPSW_HEADROOM;
+ len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
+
+ return SKB_DATA_ALIGN(len);
+}
+
+static struct page_pool *cpsw_create_page_pool(struct cpsw_common *cpsw,
+ int size)
+{
+ struct page_pool_params pp_params;
+ struct page_pool *pool;
+
+ pp_params.order = 0;
+ pp_params.flags = PP_FLAG_DMA_MAP;
+ pp_params.pool_size = size;
+ pp_params.nid = NUMA_NO_NODE;
+ pp_params.dma_dir = DMA_BIDIRECTIONAL;
+ pp_params.dev = cpsw->dev;
+
+ pool = page_pool_create(&pp_params);
+ if (IS_ERR(pool))
+ dev_err(cpsw->dev, "cannot create rx page pool\n");
+
+ return pool;
+}
+
+static int cpsw_create_rx_pool(struct cpsw_common *cpsw, int ch)
+{
+ struct page_pool *pool;
+ int ret = 0, pool_size;
+
+ pool_size = cpdma_chan_get_rx_buf_num(cpsw->rxv[ch].ch);
+ pool = cpsw_create_page_pool(cpsw, pool_size);
+ if (IS_ERR(pool))
+ ret = PTR_ERR(pool);
+ else
+ cpsw->page_pool[ch] = pool;
+
+ return ret;
+}
+
+static int cpsw_ndev_create_xdp_rxq(struct cpsw_priv *priv, int ch)
+{
+ struct cpsw_common *cpsw = priv->cpsw;
+ int ret, new_pool = false;
+ struct xdp_rxq_info *rxq;
+
+ rxq = &priv->xdp_rxq[ch];
+
+ ret = xdp_rxq_info_reg(rxq, priv->ndev, ch);
+ if (ret)
+ return ret;
+
+ if (!cpsw->page_pool[ch]) {
+ ret = cpsw_create_rx_pool(cpsw, ch);
+ if (ret)
+ goto err_rxq;
+
+ new_pool = true;
+ }
+
+ ret = xdp_rxq_info_reg_mem_model(rxq, MEM_TYPE_PAGE_POOL,
+ cpsw->page_pool[ch]);
+ if (!ret)
+ return 0;
+
+ if (new_pool) {
+ page_pool_free(cpsw->page_pool[ch]);
+ cpsw->page_pool[ch] = NULL;
+ }
+
+err_rxq:
+ xdp_rxq_info_unreg(rxq);
+ return ret;
+}
+
+void cpsw_ndev_destroy_xdp_rxqs(struct cpsw_priv *priv)
+{
+ struct cpsw_common *cpsw = priv->cpsw;
+ struct xdp_rxq_info *rxq;
+ int i;
+
+ for (i = 0; i < cpsw->rx_ch_num; i++) {
+ rxq = &priv->xdp_rxq[i];
+ if (xdp_rxq_info_is_reg(rxq))
+ xdp_rxq_info_unreg(rxq);
+ }
+}
+
+int cpsw_ndev_create_xdp_rxqs(struct cpsw_priv *priv)
+{
+ struct cpsw_common *cpsw = priv->cpsw;
+ int i, ret;
+
+ for (i = 0; i < cpsw->rx_ch_num; i++) {
+ ret = cpsw_ndev_create_xdp_rxq(priv, i);
+ if (ret)
+ goto err_cleanup;
+ }
+
+ return 0;
+
+err_cleanup:
+ cpsw_ndev_destroy_xdp_rxqs(priv);
+
+ return ret;
+}
+
static int cpsw_rx_handler(void *token, int len, int status)
{
- struct cpdma_chan *ch;
- struct sk_buff *skb = token;
- struct sk_buff *new_skb;
- struct net_device *ndev = skb->dev;
- int ret = 0, port;
- struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
+ struct page *new_page, *page = token;
+ void *pa = page_address(page);
+ struct cpsw_meta_xdp *xmeta = pa + CPSW_XMETA_OFFSET;
+ struct cpsw_common *cpsw = ndev_to_cpsw(xmeta->ndev);
+ int pkt_size = cpsw->rx_packet_max;
+ int ret = 0, port, ch = xmeta->ch;
+ int headroom = CPSW_HEADROOM;
+ struct net_device *ndev = xmeta->ndev;
+ int res = 0;
struct cpsw_priv *priv;
+ struct page_pool *pool;
+ struct sk_buff *skb;
+ struct xdp_buff xdp;
+ dma_addr_t dma;
- if (cpsw->data.dual_emac) {
+ if (cpsw->data.dual_emac && status >= 0) {
port = CPDMA_RX_SOURCE_PORT(status);
- if (port) {
+ if (port)
ndev = cpsw->slaves[--port].ndev;
- skb->dev = ndev;
- }
}
+ priv = netdev_priv(ndev);
+ pool = cpsw->page_pool[ch];
if (unlikely(status < 0) || unlikely(!netif_running(ndev))) {
/* In dual emac mode check for all interfaces */
if (cpsw->data.dual_emac && cpsw->usage_count &&
@@ -427,46 +685,94 @@ static int cpsw_rx_handler(void *token, int len, int status)
* is already down and the other interface is up
* and running, instead of freeing which results
* in reducing of the number of rx descriptor in
- * DMA engine, requeue skb back to cpdma.
+ * DMA engine, requeue page back to cpdma.
*/
- new_skb = skb;
+ new_page = page;
goto requeue;
}
- /* the interface is going down, skbs are purged */
- dev_kfree_skb_any(skb);
+ /* the interface is going down, pages are purged */
+ page_pool_recycle_direct(pool, page);
return 0;
}
- new_skb = netdev_alloc_skb_ip_align(ndev, cpsw->rx_packet_max);
- if (new_skb) {
- skb_copy_queue_mapping(new_skb, skb);
- skb_put(skb, len);
- if (status & CPDMA_RX_VLAN_ENCAP)
- cpsw_rx_vlan_encap(skb);
- priv = netdev_priv(ndev);
- if (priv->rx_ts_enabled)
- cpts_rx_timestamp(cpsw->cpts, skb);
- skb->protocol = eth_type_trans(skb, ndev);
- netif_receive_skb(skb);
- ndev->stats.rx_bytes += len;
- ndev->stats.rx_packets++;
- kmemleak_not_leak(new_skb);
- } else {
+ new_page = page_pool_dev_alloc_pages(pool);
+ if (unlikely(!new_page)) {
+ new_page = page;
+ ndev->stats.rx_dropped++;
+ goto requeue;
+ }
+
+ if (priv->xdp_prog) {
+ if (status & CPDMA_RX_VLAN_ENCAP) {
+ xdp.data = pa + CPSW_HEADROOM +
+ CPSW_RX_VLAN_ENCAP_HDR_SIZE;
+ xdp.data_end = xdp.data + len -
+ CPSW_RX_VLAN_ENCAP_HDR_SIZE;
+ } else {
+ xdp.data = pa + CPSW_HEADROOM;
+ xdp.data_end = xdp.data + len;
+ }
+
+ xdp_set_data_meta_invalid(&xdp);
+
+ xdp.data_hard_start = pa;
+ xdp.rxq = &priv->xdp_rxq[ch];
+
+ ret = cpsw_run_xdp(priv, ch, &xdp, page);
+ if (ret != CPSW_XDP_PASS) {
+ if (ret == CPSW_XDP_CONSUMED_FLUSH)
+ res = CPSW_FLUSH_XDP_MAP;
+
+ goto requeue;
+ }
+
+ /* XDP prog might have changed packet data and boundaries */
+ len = xdp.data_end - xdp.data;
+ headroom = xdp.data - xdp.data_hard_start;
+
+ /* XDP prog can modify vlan tag, so can't use encap header */
+ status &= ~CPDMA_RX_VLAN_ENCAP;
+ }
+
+ /* pass skb to netstack if no XDP prog or returned XDP_PASS */
+ skb = build_skb(pa, cpsw_rxbuf_total_len(pkt_size));
+ if (!skb) {
ndev->stats.rx_dropped++;
- new_skb = skb;
+ page_pool_recycle_direct(pool, page);
+ goto requeue;
}
+ skb_reserve(skb, headroom);
+ skb_put(skb, len);
+ skb->dev = ndev;
+ if (status & CPDMA_RX_VLAN_ENCAP)
+ cpsw_rx_vlan_encap(skb);
+ if (priv->rx_ts_enabled)
+ cpts_rx_timestamp(cpsw->cpts, skb);
+ skb->protocol = eth_type_trans(skb, ndev);
+
+ /* unmap page as no netstack skb page recycling */
+ page_pool_release_page(pool, page);
+ netif_receive_skb(skb);
+
+ ndev->stats.rx_bytes += len;
+ ndev->stats.rx_packets++;
+
requeue:
- ch = cpsw->rxv[skb_get_queue_mapping(new_skb)].ch;
- ret = cpdma_chan_submit(ch, new_skb, new_skb->data,
- skb_tailroom(new_skb), 0);
+ xmeta = page_address(new_page) + CPSW_XMETA_OFFSET;
+ xmeta->ndev = ndev;
+ xmeta->ch = ch;
+
+ dma = page_pool_get_dma_addr(new_page) + CPSW_HEADROOM;
+ ret = cpdma_chan_submit_mapped(cpsw->rxv[ch].ch, new_page, dma,
+ pkt_size, 0);
if (ret < 0) {
WARN_ON(ret == -ENOMEM);
- dev_kfree_skb_any(new_skb);
+ page_pool_recycle_direct(pool, new_page);
}
- return 0;
+ return res;
}
void cpsw_split_res(struct cpsw_common *cpsw)
@@ -640,8 +946,8 @@ static int cpsw_tx_poll(struct napi_struct *napi_tx, int budget)
static int cpsw_rx_mq_poll(struct napi_struct *napi_rx, int budget)
{
u32 ch_map;
- int num_rx, cur_budget, ch;
struct cpsw_common *cpsw = napi_to_cpsw(napi_rx);
+ int num_rx, cur_budget, ch;
int flags;
struct cpsw_vector *rxv;
@@ -657,7 +963,11 @@ static int cpsw_rx_mq_poll(struct napi_struct *napi_rx, int budget)
else
cur_budget = rxv->budget;
+ flags = 0;
num_rx += cpdma_chan_process(rxv->ch, cur_budget, &flags);
+ if (flags & CPSW_FLUSH_XDP_MAP)
+ xdp_do_flush_map();
+
if (num_rx >= budget)
break;
}
@@ -673,9 +983,12 @@ static int cpsw_rx_mq_poll(struct napi_struct *napi_rx, int budget)
static int cpsw_rx_poll(struct napi_struct *napi_rx, int budget)
{
struct cpsw_common *cpsw = napi_to_cpsw(napi_rx);
- int num_rx, flags;
+ int num_rx, flags = 0;
num_rx = cpdma_chan_process(cpsw->rxv[0].ch, budget, &flags);
+ if (flags & CPSW_FLUSH_XDP_MAP)
+ xdp_do_flush_map();
+
if (num_rx < budget) {
napi_complete_done(napi_rx, num_rx);
writel(0xff, &cpsw->wr_regs->rx_en);
@@ -1037,33 +1350,39 @@ static void cpsw_init_host_port(struct cpsw_priv *priv)
int cpsw_fill_rx_channels(struct cpsw_priv *priv)
{
struct cpsw_common *cpsw = priv->cpsw;
- struct sk_buff *skb;
+ struct cpsw_meta_xdp *xmeta;
+ struct page_pool *pool;
+ struct page *page;
int ch_buf_num;
int ch, i, ret;
+ dma_addr_t dma;
for (ch = 0; ch < cpsw->rx_ch_num; ch++) {
+ pool = cpsw->page_pool[ch];
ch_buf_num = cpdma_chan_get_rx_buf_num(cpsw->rxv[ch].ch);
for (i = 0; i < ch_buf_num; i++) {
- skb = __netdev_alloc_skb_ip_align(priv->ndev,
- cpsw->rx_packet_max,
- GFP_KERNEL);
- if (!skb) {
- cpsw_err(priv, ifup, "cannot allocate skb\n");
+ page = page_pool_dev_alloc_pages(pool);
+ if (!page) {
+ cpsw_err(priv, ifup, "allocate rx page err\n");
return -ENOMEM;
}
- skb_set_queue_mapping(skb, ch);
- ret = cpdma_chan_idle_submit(cpsw->rxv[ch].ch, skb,
- skb->data,
- skb_tailroom(skb), 0);
+ xmeta = page_address(page) + CPSW_XMETA_OFFSET;
+ xmeta->ndev = priv->ndev;
+ xmeta->ch = ch;
+
+ dma = page_pool_get_dma_addr(page) + CPSW_HEADROOM;
+ ret = cpdma_chan_idle_submit_mapped(cpsw->rxv[ch].ch,
+ page, dma,
+ cpsw->rx_packet_max,
+ 0);
if (ret < 0) {
cpsw_err(priv, ifup,
- "cannot submit skb to channel %d rx, error %d\n",
+ "cannot submit page to channel %d rx, error %d\n",
ch, ret);
- kfree_skb(skb);
+ page_pool_recycle_direct(pool, page);
return ret;
}
- kmemleak_not_leak(skb);
}
cpsw_info(priv, ifup, "ch %d rx, submitted %d descriptors\n",
@@ -1375,6 +1694,10 @@ static int cpsw_ndo_open(struct net_device *ndev)
cpsw_ale_add_vlan(cpsw->ale, cpsw->data.default_vlan,
ALE_ALL_PORTS, ALE_ALL_PORTS, 0, 0);
+ ret = cpsw_ndev_create_xdp_rxqs(priv);
+ if (ret)
+ goto err_cleanup;
+
/* initialize shared resources for every ndev */
if (!cpsw->usage_count) {
/* disable priority elevation */
@@ -1427,9 +1750,10 @@ static int cpsw_ndo_open(struct net_device *ndev)
err_cleanup:
if (!cpsw->usage_count) {
cpdma_ctlr_stop(cpsw->dma);
- for_each_slave(priv, cpsw_slave_stop, cpsw);
+ memset(cpsw->page_pool, 0, sizeof(cpsw->page_pool));
}
+ for_each_slave(priv, cpsw_slave_stop, cpsw);
pm_runtime_put_sync(cpsw->dev);
netif_carrier_off(priv->ndev);
return ret;
@@ -1452,9 +1776,12 @@ static int cpsw_ndo_stop(struct net_device *ndev)
cpsw_intr_disable(cpsw);
cpdma_ctlr_stop(cpsw->dma);
cpsw_ale_stop(cpsw->ale);
+ memset(cpsw->page_pool, 0, sizeof(cpsw->page_pool));
}
for_each_slave(priv, cpsw_slave_stop, cpsw);
+ cpsw_ndev_destroy_xdp_rxqs(priv);
+
if (cpsw_need_resplit(cpsw))
cpsw_split_res(cpsw);
@@ -2009,6 +2336,64 @@ static int cpsw_ndo_setup_tc(struct net_device *ndev, enum tc_setup_type type,
}
}
+static int cpsw_xdp_prog_setup(struct cpsw_priv *priv, struct netdev_bpf *bpf)
+{
+ struct bpf_prog *prog = bpf->prog;
+
+ if (!priv->xdpi.prog && !prog)
+ return 0;
+
+ if (!xdp_attachment_flags_ok(&priv->xdpi, bpf))
+ return -EBUSY;
+
+ WRITE_ONCE(priv->xdp_prog, prog);
+
+ xdp_attachment_setup(&priv->xdpi, bpf);
+
+ return 0;
+}
+
+static int cpsw_ndo_bpf(struct net_device *ndev, struct netdev_bpf *bpf)
+{
+ struct cpsw_priv *priv = netdev_priv(ndev);
+
+ switch (bpf->command) {
+ case XDP_SETUP_PROG:
+ return cpsw_xdp_prog_setup(priv, bpf);
+
+ case XDP_QUERY_PROG:
+ return xdp_attachment_query(&priv->xdpi, bpf);
+
+ default:
+ return -EINVAL;
+ }
+}
+
+static int cpsw_ndo_xdp_xmit(struct net_device *ndev, int n,
+ struct xdp_frame **frames, u32 flags)
+{
+ struct cpsw_priv *priv = netdev_priv(ndev);
+ struct xdp_frame *xdpf;
+ int i, drops = 0;
+
+ if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
+ return -EINVAL;
+
+ for (i = 0; i < n; i++) {
+ xdpf = frames[i];
+ if (xdpf->len < CPSW_MIN_PACKET_SIZE) {
+ xdp_return_frame_rx_napi(xdpf);
+ drops++;
+ continue;
+ }
+
+ if (cpsw_xdp_tx_frame(priv, xdpf, NULL))
+ drops++;
+ }
+
+ return n - drops;
+}
+
#ifdef CONFIG_NET_POLL_CONTROLLER
static void cpsw_ndo_poll_controller(struct net_device *ndev)
{
@@ -2037,6 +2422,8 @@ static const struct net_device_ops cpsw_netdev_ops = {
.ndo_vlan_rx_add_vid = cpsw_ndo_vlan_rx_add_vid,
.ndo_vlan_rx_kill_vid = cpsw_ndo_vlan_rx_kill_vid,
.ndo_setup_tc = cpsw_ndo_setup_tc,
+ .ndo_bpf = cpsw_ndo_bpf,
+ .ndo_xdp_xmit = cpsw_ndo_xdp_xmit,
};
static void cpsw_get_drvinfo(struct net_device *ndev,
diff --git a/drivers/net/ethernet/ti/cpsw_ethtool.c b/drivers/net/ethernet/ti/cpsw_ethtool.c
index 99935c1d265d..59f26d1fe09b 100644
--- a/drivers/net/ethernet/ti/cpsw_ethtool.c
+++ b/drivers/net/ethernet/ti/cpsw_ethtool.c
@@ -578,6 +578,48 @@ static int cpsw_update_channels_res(struct cpsw_priv *priv, int ch_num, int rx,
return 0;
}
+static void cpsw_destroy_xdp_rxqs(struct cpsw_common *cpsw)
+{
+ struct net_device *ndev;
+ struct cpsw_priv *priv;
+ int i;
+
+ for (i = 0; i < cpsw->data.slaves; i++) {
+ ndev = cpsw->slaves[i].ndev;
+ if (!ndev || !netif_running(ndev))
+ continue;
+
+ priv = netdev_priv(ndev);
+ cpsw_ndev_destroy_xdp_rxqs(priv);
+ }
+
+ memset(cpsw->page_pool, 0, sizeof(cpsw->page_pool));
+}
+
+static int cpsw_create_xdp_rxqs(struct cpsw_common *cpsw)
+{
+ struct net_device *ndev;
+ struct cpsw_priv *priv;
+ int i, ret;
+
+ for (i = 0; i < cpsw->data.slaves; i++) {
+ ndev = cpsw->slaves[i].ndev;
+ if (!ndev || !netif_running(ndev))
+ continue;
+
+ priv = netdev_priv(ndev);
+ ret = cpsw_ndev_create_xdp_rxqs(priv);
+ if (ret)
+ goto err_cleanup;
+ }
+
+ return 0;
+
+err_cleanup:
+ cpsw_destroy_xdp_rxqs(cpsw);
+ return ret;
+}
+
int cpsw_set_channels_common(struct net_device *ndev,
struct ethtool_channels *chs,
cpdma_handler_fn rx_handler)
@@ -585,7 +627,7 @@ int cpsw_set_channels_common(struct net_device *ndev,
struct cpsw_priv *priv = netdev_priv(ndev);
struct cpsw_common *cpsw = priv->cpsw;
struct net_device *sl_ndev;
- int i, ret;
+ int i, new_pools, ret;
ret = cpsw_check_ch_settings(cpsw, chs);
if (ret < 0)
@@ -593,6 +635,10 @@ int cpsw_set_channels_common(struct net_device *ndev,
cpsw_suspend_data_pass(ndev);
+ new_pools = (chs->rx_count != cpsw->rx_ch_num) && cpsw->usage_count;
+ if (new_pools)
+ cpsw_destroy_xdp_rxqs(cpsw);
+
ret = cpsw_update_channels_res(priv, chs->rx_count, 1, rx_handler);
if (ret)
goto err;
@@ -622,6 +668,12 @@ int cpsw_set_channels_common(struct net_device *ndev,
cpsw_split_res(cpsw);
+ if (new_pools) {
+ ret = cpsw_create_xdp_rxqs(cpsw);
+ if (ret)
+ goto err;
+ }
+
ret = cpsw_resume_data_pass(ndev);
if (!ret)
return 0;
@@ -647,8 +699,7 @@ void cpsw_get_ringparam(struct net_device *ndev,
int cpsw_set_ringparam(struct net_device *ndev,
struct ethtool_ringparam *ering)
{
- struct cpsw_priv *priv = netdev_priv(ndev);
- struct cpsw_common *cpsw = priv->cpsw;
+ struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
int ret;
/* ignore ering->tx_pending - only rx_pending adjustment is supported */
@@ -663,10 +714,19 @@ int cpsw_set_ringparam(struct net_device *ndev,
cpsw_suspend_data_pass(ndev);
+ if (cpsw->usage_count)
+ cpsw_destroy_xdp_rxqs(cpsw);
+
ret = cpdma_set_num_rx_descs(cpsw->dma, ering->rx_pending);
if (ret)
goto err;
+ if (cpsw->usage_count) {
+ ret = cpsw_create_xdp_rxqs(cpsw);
+ if (ret)
+ goto err;
+ }
+
ret = cpsw_resume_data_pass(ndev);
if (!ret)
return 0;
diff --git a/drivers/net/ethernet/ti/cpsw_priv.h b/drivers/net/ethernet/ti/cpsw_priv.h
index 2ecb3af59fe9..b177f94267cd 100644
--- a/drivers/net/ethernet/ti/cpsw_priv.h
+++ b/drivers/net/ethernet/ti/cpsw_priv.h
@@ -346,6 +346,7 @@ struct cpsw_common {
int rx_ch_num, tx_ch_num;
int speed;
int usage_count;
+ struct page_pool *page_pool[CPSW_MAX_QUEUES];
};
struct cpsw_priv {
@@ -360,6 +361,10 @@ struct cpsw_priv {
int shp_cfg_speed;
int tx_ts_enabled;
int rx_ts_enabled;
+ struct bpf_prog *xdp_prog;
+ struct xdp_rxq_info xdp_rxq[CPSW_MAX_QUEUES];
+ struct xdp_attachment_info xdpi;
+
u32 emac_port;
struct cpsw_common *cpsw;
};
@@ -391,6 +396,8 @@ int cpsw_fill_rx_channels(struct cpsw_priv *priv);
void cpsw_intr_enable(struct cpsw_common *cpsw);
void cpsw_intr_disable(struct cpsw_common *cpsw);
int cpsw_tx_handler(void *token, int len, int status);
+int cpsw_ndev_create_xdp_rxqs(struct cpsw_priv *priv);
+void cpsw_ndev_destroy_xdp_rxqs(struct cpsw_priv *priv);
/* ethtool */
u32 cpsw_get_msglevel(struct net_device *ndev);
--
2.17.1
^ permalink raw reply related
* [PATCH v5 net-next 5/6] net: ethernet: ti: cpsw_ethtool: allow res split while down
From: Ivan Khoronzhuk @ 2019-06-30 17:23 UTC (permalink / raw)
To: grygorii.strashko, hawk, davem
Cc: ast, linux-kernel, linux-omap, xdp-newbies, ilias.apalodimas,
netdev, daniel, jakub.kicinski, john.fastabend, Ivan Khoronzhuk
In-Reply-To: <20190630172348.5692-1-ivan.khoronzhuk@linaro.org>
That's possible to set channel num while interfaces are down. When
interface gets up it should resplit budget. This resplit can happen
after phy is up but only if speed is changed, so should be set before
this, for this allow it to happen while changing number of channels,
when interfaces are down.
Signed-off-by: Ivan Khoronzhuk <ivan.khoronzhuk@linaro.org>
---
drivers/net/ethernet/ti/cpsw_ethtool.c | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/drivers/net/ethernet/ti/cpsw_ethtool.c b/drivers/net/ethernet/ti/cpsw_ethtool.c
index 6ab0cec8560a..99935c1d265d 100644
--- a/drivers/net/ethernet/ti/cpsw_ethtool.c
+++ b/drivers/net/ethernet/ti/cpsw_ethtool.c
@@ -620,8 +620,7 @@ int cpsw_set_channels_common(struct net_device *ndev,
}
}
- if (cpsw->usage_count)
- cpsw_split_res(cpsw);
+ cpsw_split_res(cpsw);
ret = cpsw_resume_data_pass(ndev);
if (!ret)
--
2.17.1
^ permalink raw reply related
* [PATCH v5 net-next 3/6] net: ethernet: ti: davinci_cpdma: return handler status
From: Ivan Khoronzhuk @ 2019-06-30 17:23 UTC (permalink / raw)
To: grygorii.strashko, hawk, davem
Cc: ast, linux-kernel, linux-omap, xdp-newbies, ilias.apalodimas,
netdev, daniel, jakub.kicinski, john.fastabend, Ivan Khoronzhuk
In-Reply-To: <20190630172348.5692-1-ivan.khoronzhuk@linaro.org>
This change is needed to return flush status of rx handler for
flushing redirected xdp frames after processing channel packets.
Do it as separate patch for simplicity.
Signed-off-by: Ivan Khoronzhuk <ivan.khoronzhuk@linaro.org>
---
drivers/net/ethernet/ti/cpsw.c | 23 +++++++++++++---------
drivers/net/ethernet/ti/cpsw_ethtool.c | 2 +-
drivers/net/ethernet/ti/cpsw_priv.h | 2 +-
drivers/net/ethernet/ti/davinci_cpdma.c | 26 ++++++++++++++-----------
drivers/net/ethernet/ti/davinci_cpdma.h | 4 ++--
drivers/net/ethernet/ti/davinci_emac.c | 17 ++++++++++------
6 files changed, 44 insertions(+), 30 deletions(-)
diff --git a/drivers/net/ethernet/ti/cpsw.c b/drivers/net/ethernet/ti/cpsw.c
index 32b7b3b74a6b..4f72dbb5a428 100644
--- a/drivers/net/ethernet/ti/cpsw.c
+++ b/drivers/net/ethernet/ti/cpsw.c
@@ -337,7 +337,7 @@ void cpsw_intr_disable(struct cpsw_common *cpsw)
return;
}
-void cpsw_tx_handler(void *token, int len, int status)
+int cpsw_tx_handler(void *token, int len, int status)
{
struct netdev_queue *txq;
struct sk_buff *skb = token;
@@ -355,6 +355,7 @@ void cpsw_tx_handler(void *token, int len, int status)
ndev->stats.tx_packets++;
ndev->stats.tx_bytes += len;
dev_kfree_skb_any(skb);
+ return 0;
}
static void cpsw_rx_vlan_encap(struct sk_buff *skb)
@@ -400,7 +401,7 @@ static void cpsw_rx_vlan_encap(struct sk_buff *skb)
}
}
-static void cpsw_rx_handler(void *token, int len, int status)
+static int cpsw_rx_handler(void *token, int len, int status)
{
struct cpdma_chan *ch;
struct sk_buff *skb = token;
@@ -434,7 +435,7 @@ static void cpsw_rx_handler(void *token, int len, int status)
/* the interface is going down, skbs are purged */
dev_kfree_skb_any(skb);
- return;
+ return 0;
}
new_skb = netdev_alloc_skb_ip_align(ndev, cpsw->rx_packet_max);
@@ -464,6 +465,8 @@ static void cpsw_rx_handler(void *token, int len, int status)
WARN_ON(ret == -ENOMEM);
dev_kfree_skb_any(new_skb);
}
+
+ return 0;
}
void cpsw_split_res(struct cpsw_common *cpsw)
@@ -588,6 +591,7 @@ static int cpsw_tx_mq_poll(struct napi_struct *napi_tx, int budget)
u32 ch_map;
int num_tx, cur_budget, ch;
struct cpsw_common *cpsw = napi_to_cpsw(napi_tx);
+ int flags;
struct cpsw_vector *txv;
/* process every unprocessed channel */
@@ -602,7 +606,7 @@ static int cpsw_tx_mq_poll(struct napi_struct *napi_tx, int budget)
else
cur_budget = txv->budget;
- num_tx += cpdma_chan_process(txv->ch, cur_budget);
+ num_tx += cpdma_chan_process(txv->ch, cur_budget, &flags);
if (num_tx >= budget)
break;
}
@@ -618,9 +622,9 @@ static int cpsw_tx_mq_poll(struct napi_struct *napi_tx, int budget)
static int cpsw_tx_poll(struct napi_struct *napi_tx, int budget)
{
struct cpsw_common *cpsw = napi_to_cpsw(napi_tx);
- int num_tx;
+ int num_tx, flags;
- num_tx = cpdma_chan_process(cpsw->txv[0].ch, budget);
+ num_tx = cpdma_chan_process(cpsw->txv[0].ch, budget, &flags);
if (num_tx < budget) {
napi_complete(napi_tx);
writel(0xff, &cpsw->wr_regs->tx_en);
@@ -638,6 +642,7 @@ static int cpsw_rx_mq_poll(struct napi_struct *napi_rx, int budget)
u32 ch_map;
int num_rx, cur_budget, ch;
struct cpsw_common *cpsw = napi_to_cpsw(napi_rx);
+ int flags;
struct cpsw_vector *rxv;
/* process every unprocessed channel */
@@ -652,7 +657,7 @@ static int cpsw_rx_mq_poll(struct napi_struct *napi_rx, int budget)
else
cur_budget = rxv->budget;
- num_rx += cpdma_chan_process(rxv->ch, cur_budget);
+ num_rx += cpdma_chan_process(rxv->ch, cur_budget, &flags);
if (num_rx >= budget)
break;
}
@@ -668,9 +673,9 @@ static int cpsw_rx_mq_poll(struct napi_struct *napi_rx, int budget)
static int cpsw_rx_poll(struct napi_struct *napi_rx, int budget)
{
struct cpsw_common *cpsw = napi_to_cpsw(napi_rx);
- int num_rx;
+ int num_rx, flags;
- num_rx = cpdma_chan_process(cpsw->rxv[0].ch, budget);
+ num_rx = cpdma_chan_process(cpsw->rxv[0].ch, budget, &flags);
if (num_rx < budget) {
napi_complete_done(napi_rx, num_rx);
writel(0xff, &cpsw->wr_regs->rx_en);
diff --git a/drivers/net/ethernet/ti/cpsw_ethtool.c b/drivers/net/ethernet/ti/cpsw_ethtool.c
index f60dc1dfc443..7c19eebbabcc 100644
--- a/drivers/net/ethernet/ti/cpsw_ethtool.c
+++ b/drivers/net/ethernet/ti/cpsw_ethtool.c
@@ -532,8 +532,8 @@ static int cpsw_update_channels_res(struct cpsw_priv *priv, int ch_num, int rx,
cpdma_handler_fn rx_handler)
{
struct cpsw_common *cpsw = priv->cpsw;
- void (*handler)(void *, int, int);
struct netdev_queue *queue;
+ cpdma_handler_fn handler;
struct cpsw_vector *vec;
int ret, *ch, vch;
diff --git a/drivers/net/ethernet/ti/cpsw_priv.h b/drivers/net/ethernet/ti/cpsw_priv.h
index 04795b97ee71..2ecb3af59fe9 100644
--- a/drivers/net/ethernet/ti/cpsw_priv.h
+++ b/drivers/net/ethernet/ti/cpsw_priv.h
@@ -390,7 +390,7 @@ void cpsw_split_res(struct cpsw_common *cpsw);
int cpsw_fill_rx_channels(struct cpsw_priv *priv);
void cpsw_intr_enable(struct cpsw_common *cpsw);
void cpsw_intr_disable(struct cpsw_common *cpsw);
-void cpsw_tx_handler(void *token, int len, int status);
+int cpsw_tx_handler(void *token, int len, int status);
/* ethtool */
u32 cpsw_get_msglevel(struct net_device *ndev);
diff --git a/drivers/net/ethernet/ti/davinci_cpdma.c b/drivers/net/ethernet/ti/davinci_cpdma.c
index 8da46394c0e7..ea25b23c8058 100644
--- a/drivers/net/ethernet/ti/davinci_cpdma.c
+++ b/drivers/net/ethernet/ti/davinci_cpdma.c
@@ -1191,15 +1191,16 @@ bool cpdma_check_free_tx_desc(struct cpdma_chan *chan)
return free_tx_desc;
}
-static void __cpdma_chan_free(struct cpdma_chan *chan,
- struct cpdma_desc __iomem *desc,
- int outlen, int status)
+static int __cpdma_chan_free(struct cpdma_chan *chan,
+ struct cpdma_desc __iomem *desc, int outlen,
+ int status)
{
struct cpdma_ctlr *ctlr = chan->ctlr;
struct cpdma_desc_pool *pool = ctlr->pool;
dma_addr_t buff_dma;
int origlen;
uintptr_t token;
+ int ret;
token = desc_read(desc, sw_token);
origlen = desc_read(desc, sw_len);
@@ -1214,14 +1215,16 @@ static void __cpdma_chan_free(struct cpdma_chan *chan,
}
cpdma_desc_free(pool, desc, 1);
- (*chan->handler)((void *)token, outlen, status);
+ ret = (*chan->handler)((void *)token, outlen, status);
+
+ return ret;
}
static int __cpdma_chan_process(struct cpdma_chan *chan)
{
+ int status, outlen, ret;
struct cpdma_ctlr *ctlr = chan->ctlr;
struct cpdma_desc __iomem *desc;
- int status, outlen;
int cb_status = 0;
struct cpdma_desc_pool *pool = ctlr->pool;
dma_addr_t desc_dma;
@@ -1232,7 +1235,7 @@ static int __cpdma_chan_process(struct cpdma_chan *chan)
desc = chan->head;
if (!desc) {
chan->stats.empty_dequeue++;
- status = -ENOENT;
+ ret = -ENOENT;
goto unlock_ret;
}
desc_dma = desc_phys(pool, desc);
@@ -1241,7 +1244,7 @@ static int __cpdma_chan_process(struct cpdma_chan *chan)
outlen = status & 0x7ff;
if (status & CPDMA_DESC_OWNER) {
chan->stats.busy_dequeue++;
- status = -EBUSY;
+ ret = -EBUSY;
goto unlock_ret;
}
@@ -1267,15 +1270,15 @@ static int __cpdma_chan_process(struct cpdma_chan *chan)
else
cb_status = status;
- __cpdma_chan_free(chan, desc, outlen, cb_status);
- return status;
+ ret = __cpdma_chan_free(chan, desc, outlen, cb_status);
+ return ret;
unlock_ret:
spin_unlock_irqrestore(&chan->lock, flags);
- return status;
+ return ret;
}
-int cpdma_chan_process(struct cpdma_chan *chan, int quota)
+int cpdma_chan_process(struct cpdma_chan *chan, int quota, int *flags)
{
int used = 0, ret = 0;
@@ -1286,6 +1289,7 @@ int cpdma_chan_process(struct cpdma_chan *chan, int quota)
ret = __cpdma_chan_process(chan);
if (ret < 0)
break;
+ *flags |= ret;
used++;
}
return used;
diff --git a/drivers/net/ethernet/ti/davinci_cpdma.h b/drivers/net/ethernet/ti/davinci_cpdma.h
index 0271a20c2e09..aafa8889c789 100644
--- a/drivers/net/ethernet/ti/davinci_cpdma.h
+++ b/drivers/net/ethernet/ti/davinci_cpdma.h
@@ -61,7 +61,7 @@ struct cpdma_chan_stats {
struct cpdma_ctlr;
struct cpdma_chan;
-typedef void (*cpdma_handler_fn)(void *token, int len, int status);
+typedef int (*cpdma_handler_fn)(void *token, int len, int status);
struct cpdma_ctlr *cpdma_ctlr_create(struct cpdma_params *params);
int cpdma_ctlr_destroy(struct cpdma_ctlr *ctlr);
@@ -85,7 +85,7 @@ int cpdma_chan_idle_submit_mapped(struct cpdma_chan *chan, void *token,
dma_addr_t data, int len, int directed);
int cpdma_chan_idle_submit(struct cpdma_chan *chan, void *token, void *data,
int len, int directed);
-int cpdma_chan_process(struct cpdma_chan *chan, int quota);
+int cpdma_chan_process(struct cpdma_chan *chan, int quota, int *flags);
int cpdma_ctlr_int_ctrl(struct cpdma_ctlr *ctlr, bool enable);
void cpdma_ctlr_eoi(struct cpdma_ctlr *ctlr, u32 value);
diff --git a/drivers/net/ethernet/ti/davinci_emac.c b/drivers/net/ethernet/ti/davinci_emac.c
index 5f4ece0d5a73..06756471d586 100644
--- a/drivers/net/ethernet/ti/davinci_emac.c
+++ b/drivers/net/ethernet/ti/davinci_emac.c
@@ -860,7 +860,7 @@ static struct sk_buff *emac_rx_alloc(struct emac_priv *priv)
return skb;
}
-static void emac_rx_handler(void *token, int len, int status)
+static int emac_rx_handler(void *token, int len, int status)
{
struct sk_buff *skb = token;
struct net_device *ndev = skb->dev;
@@ -871,7 +871,7 @@ static void emac_rx_handler(void *token, int len, int status)
/* free and bail if we are shutting down */
if (unlikely(!netif_running(ndev))) {
dev_kfree_skb_any(skb);
- return;
+ return 0;
}
/* recycle on receive error */
@@ -892,7 +892,7 @@ static void emac_rx_handler(void *token, int len, int status)
if (!skb) {
if (netif_msg_rx_err(priv) && net_ratelimit())
dev_err(emac_dev, "failed rx buffer alloc\n");
- return;
+ return 0;
}
recycle:
@@ -902,9 +902,11 @@ static void emac_rx_handler(void *token, int len, int status)
WARN_ON(ret == -ENOMEM);
if (unlikely(ret < 0))
dev_kfree_skb_any(skb);
+
+ return 0;
}
-static void emac_tx_handler(void *token, int len, int status)
+static int emac_tx_handler(void *token, int len, int status)
{
struct sk_buff *skb = token;
struct net_device *ndev = skb->dev;
@@ -917,6 +919,7 @@ static void emac_tx_handler(void *token, int len, int status)
ndev->stats.tx_packets++;
ndev->stats.tx_bytes += len;
dev_kfree_skb_any(skb);
+ return 0;
}
/**
@@ -1227,6 +1230,7 @@ static int emac_poll(struct napi_struct *napi, int budget)
struct device *emac_dev = &ndev->dev;
u32 status = 0;
u32 num_tx_pkts = 0, num_rx_pkts = 0;
+ int flags;
/* Check interrupt vectors and call packet processing */
status = emac_read(EMAC_MACINVECTOR);
@@ -1238,7 +1242,8 @@ static int emac_poll(struct napi_struct *napi, int budget)
if (status & mask) {
num_tx_pkts = cpdma_chan_process(priv->txchan,
- EMAC_DEF_TX_MAX_SERVICE);
+ EMAC_DEF_TX_MAX_SERVICE,
+ &flags);
} /* TX processing */
mask = EMAC_DM644X_MAC_IN_VECTOR_RX_INT_VEC;
@@ -1247,7 +1252,7 @@ static int emac_poll(struct napi_struct *napi, int budget)
mask = EMAC_DM646X_MAC_IN_VECTOR_RX_INT_VEC;
if (status & mask) {
- num_rx_pkts = cpdma_chan_process(priv->rxchan, budget);
+ num_rx_pkts = cpdma_chan_process(priv->rxchan, budget, &flags);
} /* RX processing */
mask = EMAC_DM644X_MAC_IN_VECTOR_HOST_INT;
--
2.17.1
^ permalink raw reply related
* [PATCH v5 net-next 2/6] net: ethernet: ti: davinci_cpdma: add dma mapped submit
From: Ivan Khoronzhuk @ 2019-06-30 17:23 UTC (permalink / raw)
To: grygorii.strashko, hawk, davem
Cc: ast, linux-kernel, linux-omap, xdp-newbies, ilias.apalodimas,
netdev, daniel, jakub.kicinski, john.fastabend, Ivan Khoronzhuk
In-Reply-To: <20190630172348.5692-1-ivan.khoronzhuk@linaro.org>
In case if dma mapped packet needs to be sent, like with XDP
page pool, the "mapped" submit can be used. This patch adds dma
mapped submit based on regular one.
Signed-off-by: Ivan Khoronzhuk <ivan.khoronzhuk@linaro.org>
---
drivers/net/ethernet/ti/davinci_cpdma.c | 89 ++++++++++++++++++++++---
drivers/net/ethernet/ti/davinci_cpdma.h | 4 ++
2 files changed, 83 insertions(+), 10 deletions(-)
diff --git a/drivers/net/ethernet/ti/davinci_cpdma.c b/drivers/net/ethernet/ti/davinci_cpdma.c
index 5cf1758d425b..8da46394c0e7 100644
--- a/drivers/net/ethernet/ti/davinci_cpdma.c
+++ b/drivers/net/ethernet/ti/davinci_cpdma.c
@@ -139,6 +139,7 @@ struct submit_info {
int directed;
void *token;
void *data;
+ int flags;
int len;
};
@@ -184,6 +185,8 @@ static struct cpdma_control_info controls[] = {
(directed << CPDMA_TO_PORT_SHIFT)); \
} while (0)
+#define CPDMA_DMA_EXT_MAP BIT(16)
+
static void cpdma_desc_pool_destroy(struct cpdma_ctlr *ctlr)
{
struct cpdma_desc_pool *pool = ctlr->pool;
@@ -1015,6 +1018,7 @@ static int cpdma_chan_submit_si(struct submit_info *si)
struct cpdma_chan *chan = si->chan;
struct cpdma_ctlr *ctlr = chan->ctlr;
int len = si->len;
+ int swlen = len;
struct cpdma_desc __iomem *desc;
dma_addr_t buffer;
u32 mode;
@@ -1036,16 +1040,22 @@ static int cpdma_chan_submit_si(struct submit_info *si)
chan->stats.runt_transmit_buff++;
}
- buffer = dma_map_single(ctlr->dev, si->data, len, chan->dir);
- ret = dma_mapping_error(ctlr->dev, buffer);
- if (ret) {
- cpdma_desc_free(ctlr->pool, desc, 1);
- return -EINVAL;
- }
-
mode = CPDMA_DESC_OWNER | CPDMA_DESC_SOP | CPDMA_DESC_EOP;
cpdma_desc_to_port(chan, mode, si->directed);
+ if (si->flags & CPDMA_DMA_EXT_MAP) {
+ buffer = (u32)si->data;
+ dma_sync_single_for_device(ctlr->dev, buffer, len, chan->dir);
+ swlen |= CPDMA_DMA_EXT_MAP;
+ } else {
+ buffer = dma_map_single(ctlr->dev, si->data, len, chan->dir);
+ ret = dma_mapping_error(ctlr->dev, buffer);
+ if (ret) {
+ cpdma_desc_free(ctlr->pool, desc, 1);
+ return -EINVAL;
+ }
+ }
+
/* Relaxed IO accessors can be used here as there is read barrier
* at the end of write sequence.
*/
@@ -1055,7 +1065,7 @@ static int cpdma_chan_submit_si(struct submit_info *si)
writel_relaxed(mode | len, &desc->hw_mode);
writel_relaxed((uintptr_t)si->token, &desc->sw_token);
writel_relaxed(buffer, &desc->sw_buffer);
- writel_relaxed(len, &desc->sw_len);
+ writel_relaxed(swlen, &desc->sw_len);
desc_read(desc, sw_len);
__cpdma_chan_submit(chan, desc);
@@ -1079,6 +1089,32 @@ int cpdma_chan_idle_submit(struct cpdma_chan *chan, void *token, void *data,
si.data = data;
si.len = len;
si.directed = directed;
+ si.flags = 0;
+
+ spin_lock_irqsave(&chan->lock, flags);
+ if (chan->state == CPDMA_STATE_TEARDOWN) {
+ spin_unlock_irqrestore(&chan->lock, flags);
+ return -EINVAL;
+ }
+
+ ret = cpdma_chan_submit_si(&si);
+ spin_unlock_irqrestore(&chan->lock, flags);
+ return ret;
+}
+
+int cpdma_chan_idle_submit_mapped(struct cpdma_chan *chan, void *token,
+ dma_addr_t data, int len, int directed)
+{
+ struct submit_info si;
+ unsigned long flags;
+ int ret;
+
+ si.chan = chan;
+ si.token = token;
+ si.data = (void *)(u32)data;
+ si.len = len;
+ si.directed = directed;
+ si.flags = CPDMA_DMA_EXT_MAP;
spin_lock_irqsave(&chan->lock, flags);
if (chan->state == CPDMA_STATE_TEARDOWN) {
@@ -1103,6 +1139,32 @@ int cpdma_chan_submit(struct cpdma_chan *chan, void *token, void *data,
si.data = data;
si.len = len;
si.directed = directed;
+ si.flags = 0;
+
+ spin_lock_irqsave(&chan->lock, flags);
+ if (chan->state != CPDMA_STATE_ACTIVE) {
+ spin_unlock_irqrestore(&chan->lock, flags);
+ return -EINVAL;
+ }
+
+ ret = cpdma_chan_submit_si(&si);
+ spin_unlock_irqrestore(&chan->lock, flags);
+ return ret;
+}
+
+int cpdma_chan_submit_mapped(struct cpdma_chan *chan, void *token,
+ dma_addr_t data, int len, int directed)
+{
+ struct submit_info si;
+ unsigned long flags;
+ int ret;
+
+ si.chan = chan;
+ si.token = token;
+ si.data = (void *)(u32)data;
+ si.len = len;
+ si.directed = directed;
+ si.flags = CPDMA_DMA_EXT_MAP;
spin_lock_irqsave(&chan->lock, flags);
if (chan->state != CPDMA_STATE_ACTIVE) {
@@ -1140,10 +1202,17 @@ static void __cpdma_chan_free(struct cpdma_chan *chan,
uintptr_t token;
token = desc_read(desc, sw_token);
- buff_dma = desc_read(desc, sw_buffer);
origlen = desc_read(desc, sw_len);
- dma_unmap_single(ctlr->dev, buff_dma, origlen, chan->dir);
+ buff_dma = desc_read(desc, sw_buffer);
+ if (origlen & CPDMA_DMA_EXT_MAP) {
+ origlen &= ~CPDMA_DMA_EXT_MAP;
+ dma_sync_single_for_cpu(ctlr->dev, buff_dma, origlen,
+ chan->dir);
+ } else {
+ dma_unmap_single(ctlr->dev, buff_dma, origlen, chan->dir);
+ }
+
cpdma_desc_free(pool, desc, 1);
(*chan->handler)((void *)token, outlen, status);
}
diff --git a/drivers/net/ethernet/ti/davinci_cpdma.h b/drivers/net/ethernet/ti/davinci_cpdma.h
index 9343c8c73c1b..0271a20c2e09 100644
--- a/drivers/net/ethernet/ti/davinci_cpdma.h
+++ b/drivers/net/ethernet/ti/davinci_cpdma.h
@@ -77,8 +77,12 @@ int cpdma_chan_stop(struct cpdma_chan *chan);
int cpdma_chan_get_stats(struct cpdma_chan *chan,
struct cpdma_chan_stats *stats);
+int cpdma_chan_submit_mapped(struct cpdma_chan *chan, void *token,
+ dma_addr_t data, int len, int directed);
int cpdma_chan_submit(struct cpdma_chan *chan, void *token, void *data,
int len, int directed);
+int cpdma_chan_idle_submit_mapped(struct cpdma_chan *chan, void *token,
+ dma_addr_t data, int len, int directed);
int cpdma_chan_idle_submit(struct cpdma_chan *chan, void *token, void *data,
int len, int directed);
int cpdma_chan_process(struct cpdma_chan *chan, int quota);
--
2.17.1
^ permalink raw reply related
* [PATCH v5 net-next 4/6] net: ethernet: ti: davinci_cpdma: allow desc split while down
From: Ivan Khoronzhuk @ 2019-06-30 17:23 UTC (permalink / raw)
To: grygorii.strashko, hawk, davem
Cc: ast, linux-kernel, linux-omap, xdp-newbies, ilias.apalodimas,
netdev, daniel, jakub.kicinski, john.fastabend, Ivan Khoronzhuk
In-Reply-To: <20190630172348.5692-1-ivan.khoronzhuk@linaro.org>
That's possible to set ring params while interfaces are down. When
interface gets up it uses number of descs to fill rx queue and on
later on changes to create rx pools. Usually, this resplit can happen
after phy is up, but it can be needed before this, so allow it to
happen while setting number of rx descs, when interfaces are down.
Also, if no dependency on intf state, move it to cpdma layer, where
it should be.
Signed-off-by: Ivan Khoronzhuk <ivan.khoronzhuk@linaro.org>
---
drivers/net/ethernet/ti/cpsw_ethtool.c | 9 ++++-----
drivers/net/ethernet/ti/davinci_cpdma.c | 10 +++++++++-
drivers/net/ethernet/ti/davinci_cpdma.h | 3 +--
3 files changed, 14 insertions(+), 8 deletions(-)
diff --git a/drivers/net/ethernet/ti/cpsw_ethtool.c b/drivers/net/ethernet/ti/cpsw_ethtool.c
index 7c19eebbabcc..6ab0cec8560a 100644
--- a/drivers/net/ethernet/ti/cpsw_ethtool.c
+++ b/drivers/net/ethernet/ti/cpsw_ethtool.c
@@ -664,15 +664,14 @@ int cpsw_set_ringparam(struct net_device *ndev,
cpsw_suspend_data_pass(ndev);
- cpdma_set_num_rx_descs(cpsw->dma, ering->rx_pending);
-
- if (cpsw->usage_count)
- cpdma_chan_split_pool(cpsw->dma);
+ ret = cpdma_set_num_rx_descs(cpsw->dma, ering->rx_pending);
+ if (ret)
+ goto err;
ret = cpsw_resume_data_pass(ndev);
if (!ret)
return 0;
-
+err:
dev_err(cpsw->dev, "cannot set ring params, closing device\n");
dev_close(ndev);
return ret;
diff --git a/drivers/net/ethernet/ti/davinci_cpdma.c b/drivers/net/ethernet/ti/davinci_cpdma.c
index ea25b23c8058..7dc2c1ee6238 100644
--- a/drivers/net/ethernet/ti/davinci_cpdma.c
+++ b/drivers/net/ethernet/ti/davinci_cpdma.c
@@ -1427,8 +1427,16 @@ int cpdma_get_num_tx_descs(struct cpdma_ctlr *ctlr)
return ctlr->num_tx_desc;
}
-void cpdma_set_num_rx_descs(struct cpdma_ctlr *ctlr, int num_rx_desc)
+int cpdma_set_num_rx_descs(struct cpdma_ctlr *ctlr, int num_rx_desc)
{
+ unsigned long flags;
+ int ret;
+
+ spin_lock_irqsave(&ctlr->lock, flags);
ctlr->num_rx_desc = num_rx_desc;
ctlr->num_tx_desc = ctlr->pool->num_desc - ctlr->num_rx_desc;
+ ret = cpdma_chan_split_pool(ctlr);
+ spin_unlock_irqrestore(&ctlr->lock, flags);
+
+ return ret;
}
diff --git a/drivers/net/ethernet/ti/davinci_cpdma.h b/drivers/net/ethernet/ti/davinci_cpdma.h
index aafa8889c789..df66b8c797ee 100644
--- a/drivers/net/ethernet/ti/davinci_cpdma.h
+++ b/drivers/net/ethernet/ti/davinci_cpdma.h
@@ -116,8 +116,7 @@ enum cpdma_control {
int cpdma_control_get(struct cpdma_ctlr *ctlr, int control);
int cpdma_control_set(struct cpdma_ctlr *ctlr, int control, int value);
int cpdma_get_num_rx_descs(struct cpdma_ctlr *ctlr);
-void cpdma_set_num_rx_descs(struct cpdma_ctlr *ctlr, int num_rx_desc);
+int cpdma_set_num_rx_descs(struct cpdma_ctlr *ctlr, int num_rx_desc);
int cpdma_get_num_tx_descs(struct cpdma_ctlr *ctlr);
-int cpdma_chan_split_pool(struct cpdma_ctlr *ctlr);
#endif
--
2.17.1
^ permalink raw reply related
* [PATCH v5 net-next 1/6] xdp: allow same allocator usage
From: Ivan Khoronzhuk @ 2019-06-30 17:23 UTC (permalink / raw)
To: grygorii.strashko, hawk, davem
Cc: ast, linux-kernel, linux-omap, xdp-newbies, ilias.apalodimas,
netdev, daniel, jakub.kicinski, john.fastabend, Ivan Khoronzhuk
In-Reply-To: <20190630172348.5692-1-ivan.khoronzhuk@linaro.org>
XDP rxqs can be same for ndevs running under same rx napi softirq.
But there is no ability to register same allocator for both rxqs,
by fact it can same rxq but has different ndev as a reference.
Due to last changes allocator destroy can be defered till the moment
all packets are recycled by destination interface, afterwards it's
freed. In order to schedule allocator destroy only after all users are
unregistered, add refcnt to allocator object and schedule to destroy
only it reaches 0.
Signed-off-by: Ivan Khoronzhuk <ivan.khoronzhuk@linaro.org>
---
include/net/xdp_priv.h | 1 +
net/core/xdp.c | 46 ++++++++++++++++++++++++++++++++++++++++++
2 files changed, 47 insertions(+)
diff --git a/include/net/xdp_priv.h b/include/net/xdp_priv.h
index 6a8cba6ea79a..995b21da2f27 100644
--- a/include/net/xdp_priv.h
+++ b/include/net/xdp_priv.h
@@ -18,6 +18,7 @@ struct xdp_mem_allocator {
struct rcu_head rcu;
struct delayed_work defer_wq;
unsigned long defer_warn;
+ unsigned long refcnt;
};
#endif /* __LINUX_NET_XDP_PRIV_H__ */
diff --git a/net/core/xdp.c b/net/core/xdp.c
index b29d7b513a18..a44621190fdc 100644
--- a/net/core/xdp.c
+++ b/net/core/xdp.c
@@ -98,6 +98,18 @@ bool __mem_id_disconnect(int id, bool force)
WARN(1, "Request remove non-existing id(%d), driver bug?", id);
return true;
}
+
+ /* to avoid calling hash lookup twice, decrement refcnt here till it
+ * reaches zero, then it can be called from workqueue afterwards.
+ */
+ if (xa->refcnt)
+ xa->refcnt--;
+
+ if (xa->refcnt) {
+ mutex_unlock(&mem_id_lock);
+ return true;
+ }
+
xa->disconnect_cnt++;
/* Detects in-flight packet-pages for page_pool */
@@ -312,6 +324,33 @@ static bool __is_supported_mem_type(enum xdp_mem_type type)
return true;
}
+static struct xdp_mem_allocator *xdp_allocator_get(void *allocator)
+{
+ struct xdp_mem_allocator *xae, *xa = NULL;
+ struct rhashtable_iter iter;
+
+ mutex_lock(&mem_id_lock);
+ rhashtable_walk_enter(mem_id_ht, &iter);
+ do {
+ rhashtable_walk_start(&iter);
+
+ while ((xae = rhashtable_walk_next(&iter)) && !IS_ERR(xae)) {
+ if (xae->allocator == allocator) {
+ xae->refcnt++;
+ xa = xae;
+ break;
+ }
+ }
+
+ rhashtable_walk_stop(&iter);
+
+ } while (xae == ERR_PTR(-EAGAIN));
+ rhashtable_walk_exit(&iter);
+ mutex_unlock(&mem_id_lock);
+
+ return xa;
+}
+
int xdp_rxq_info_reg_mem_model(struct xdp_rxq_info *xdp_rxq,
enum xdp_mem_type type, void *allocator)
{
@@ -347,6 +386,12 @@ int xdp_rxq_info_reg_mem_model(struct xdp_rxq_info *xdp_rxq,
}
}
+ xdp_alloc = xdp_allocator_get(allocator);
+ if (xdp_alloc) {
+ xdp_rxq->mem.id = xdp_alloc->mem.id;
+ return 0;
+ }
+
xdp_alloc = kzalloc(sizeof(*xdp_alloc), gfp);
if (!xdp_alloc)
return -ENOMEM;
@@ -360,6 +405,7 @@ int xdp_rxq_info_reg_mem_model(struct xdp_rxq_info *xdp_rxq,
xdp_rxq->mem.id = id;
xdp_alloc->mem = xdp_rxq->mem;
xdp_alloc->allocator = allocator;
+ xdp_alloc->refcnt = 1;
/* Insert allocator into ID lookup table */
ptr = rhashtable_insert_slow(mem_id_ht, &id, &xdp_alloc->node);
--
2.17.1
^ permalink raw reply related
* Re: r8169 not working on 5.2.0rc6 with GPD MicroPC
From: Heiner Kallweit @ 2019-06-30 17:42 UTC (permalink / raw)
To: Karsten Wiborg, Andrew Lunn; +Cc: nic_swsd, romieu, netdev
In-Reply-To: <3825ebc5-15bc-2787-4d73-cccbfe96a0cc@web.de>
On 30.06.2019 18:03, Karsten Wiborg wrote:
> Hi Andrew,
>
> On 30/06/2019 16:55, Andrew Lunn wrote:
>> Hi Karsten
>>
>> What MAC address do you get with the vendor driver? Is it the same MAC
>> address every time you reboot, or does it look random.
>>
>> The BIOS is expected to program the MAC address into the hardware. It
>> could be that the vendor driver is checking if the MAC address is
>> valid, and if not, picking a random MAC address. The mainline driver
>> does not do this.
>
> I programmed a static DHCP-entry on my local DHCP-server so I would
> notice if the MAC address changes. Just turned the computer back on and
> received the intended IP address, so the MAC address seems to stay the
> same with the vendor driver.
>
Vendor driver uses this code, do you see the related messages in syslog?
if (!is_valid_ether_addr(mac_addr)) {
netif_err(tp, probe, dev, "Invalid ether addr %pM\n",
mac_addr);
eth_hw_addr_random(dev);
ether_addr_copy(mac_addr, dev->dev_addr);
netif_info(tp, probe, dev, "Random ether addr %pM\n",
mac_addr);
tp->random_mac = 1;
}
> The vendor part of my MAC is 6e:69:73 which is interesting because
> according to some Vendor-Lookup-pages the vendor is unknown.
>
> Regards,
> Karsten
>
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox