* [PATCH net-next 2/6] sfc: give ef10 its own rwsem in the filter table instead of filter_lock
From: Edward Cree @ 2018-03-27 16:42 UTC (permalink / raw)
To: linux-net-drivers, David Miller; +Cc: netdev
In-Reply-To: <7a932be7-0335-a7f3-4fb4-408ccb191737@solarflare.com>
efx->filter_lock remains in place for use on farch, but EF10 now ignores it.
EFX_EF10_FILTER_FLAG_BUSY is no longer needed, hence it is removed.
Signed-off-by: Edward Cree <ecree@solarflare.com>
---
drivers/net/ethernet/sfc/ef10.c | 361 ++++++++++++++--------------------
drivers/net/ethernet/sfc/efx.c | 1 -
drivers/net/ethernet/sfc/net_driver.h | 2 +-
3 files changed, 151 insertions(+), 213 deletions(-)
diff --git a/drivers/net/ethernet/sfc/ef10.c b/drivers/net/ethernet/sfc/ef10.c
index bcbaba330fb5..9db1b9144e70 100644
--- a/drivers/net/ethernet/sfc/ef10.c
+++ b/drivers/net/ethernet/sfc/ef10.c
@@ -96,17 +96,15 @@ struct efx_ef10_filter_table {
MC_CMD_GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES_MAXNUM * 2];
unsigned int rx_match_count;
+ struct rw_semaphore lock; /* Protects entries */
struct {
unsigned long spec; /* pointer to spec plus flag bits */
-/* BUSY flag indicates that an update is in progress. AUTO_OLD is
- * used to mark and sweep MAC filters for the device address lists.
- */
-#define EFX_EF10_FILTER_FLAG_BUSY 1UL
+/* AUTO_OLD is used to mark and sweep MAC filters for the device address lists. */
+/* unused flag 1UL */
#define EFX_EF10_FILTER_FLAG_AUTO_OLD 2UL
#define EFX_EF10_FILTER_FLAGS 3UL
u64 handle; /* firmware handle */
} *entry;
- wait_queue_head_t waitq;
/* Shadow of net_device address lists, guarded by mac_lock */
struct efx_ef10_dev_addr dev_uc_list[EFX_EF10_FILTER_DEV_UC_MAX];
struct efx_ef10_dev_addr dev_mc_list[EFX_EF10_FILTER_DEV_MC_MAX];
@@ -4302,26 +4300,33 @@ static s32 efx_ef10_filter_insert(struct efx_nic *efx,
struct efx_filter_spec *spec,
bool replace_equal)
{
- struct efx_ef10_filter_table *table = efx->filter_state;
DECLARE_BITMAP(mc_rem_map, EFX_EF10_FILTER_SEARCH_LIMIT);
+ struct efx_ef10_filter_table *table;
struct efx_filter_spec *saved_spec;
struct efx_rss_context *ctx = NULL;
unsigned int match_pri, hash;
unsigned int priv_flags;
bool replacing = false;
+ unsigned int depth, i;
int ins_index = -1;
DEFINE_WAIT(wait);
bool is_mc_recip;
s32 rc;
+ down_read(&efx->filter_sem);
+ table = efx->filter_state;
+ down_write(&table->lock);
+
/* For now, only support RX filters */
if ((spec->flags & (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)) !=
- EFX_FILTER_FLAG_RX)
- return -EINVAL;
+ EFX_FILTER_FLAG_RX) {
+ rc = -EINVAL;
+ goto out_unlock;
+ }
rc = efx_ef10_filter_pri(table, spec);
if (rc < 0)
- return rc;
+ goto out_unlock;
match_pri = rc;
hash = efx_ef10_filter_hash(spec);
@@ -4335,86 +4340,64 @@ static s32 efx_ef10_filter_insert(struct efx_nic *efx,
&efx->rss_context.list);
else
ctx = &efx->rss_context;
- if (!ctx)
- return -ENOENT;
- if (ctx->context_id == EFX_EF10_RSS_CONTEXT_INVALID)
- return -EOPNOTSUPP;
+ if (!ctx) {
+ rc = -ENOENT;
+ goto out_unlock;
+ }
+ if (ctx->context_id == EFX_EF10_RSS_CONTEXT_INVALID) {
+ rc = -EOPNOTSUPP;
+ goto out_unlock;
+ }
}
/* Find any existing filters with the same match tuple or
- * else a free slot to insert at. If any of them are busy,
- * we have to wait and retry.
+ * else a free slot to insert at.
*/
- for (;;) {
- unsigned int depth = 1;
- unsigned int i;
-
- spin_lock_bh(&efx->filter_lock);
+ for (depth = 1; depth < EFX_EF10_FILTER_SEARCH_LIMIT; depth++) {
+ i = (hash + depth) & (HUNT_FILTER_TBL_ROWS - 1);
+ saved_spec = efx_ef10_filter_entry_spec(table, i);
- for (;;) {
- i = (hash + depth) & (HUNT_FILTER_TBL_ROWS - 1);
- saved_spec = efx_ef10_filter_entry_spec(table, i);
-
- if (!saved_spec) {
- if (ins_index < 0)
- ins_index = i;
- } else if (efx_ef10_filter_equal(spec, saved_spec)) {
- if (table->entry[i].spec &
- EFX_EF10_FILTER_FLAG_BUSY)
- break;
- if (spec->priority < saved_spec->priority &&
- spec->priority != EFX_FILTER_PRI_AUTO) {
- rc = -EPERM;
- goto out_unlock;
- }
- if (!is_mc_recip) {
- /* This is the only one */
- if (spec->priority ==
- saved_spec->priority &&
- !replace_equal) {
- rc = -EEXIST;
- goto out_unlock;
- }
- ins_index = i;
- goto found;
- } else if (spec->priority >
- saved_spec->priority ||
- (spec->priority ==
- saved_spec->priority &&
- replace_equal)) {
- if (ins_index < 0)
- ins_index = i;
- else
- __set_bit(depth, mc_rem_map);
- }
+ if (!saved_spec) {
+ if (ins_index < 0)
+ ins_index = i;
+ } else if (efx_ef10_filter_equal(spec, saved_spec)) {
+ if (spec->priority < saved_spec->priority &&
+ spec->priority != EFX_FILTER_PRI_AUTO) {
+ rc = -EPERM;
+ goto out_unlock;
}
-
- /* Once we reach the maximum search depth, use
- * the first suitable slot or return -EBUSY if
- * there was none
- */
- if (depth == EFX_EF10_FILTER_SEARCH_LIMIT) {
- if (ins_index < 0) {
- rc = -EBUSY;
+ if (!is_mc_recip) {
+ /* This is the only one */
+ if (spec->priority ==
+ saved_spec->priority &&
+ !replace_equal) {
+ rc = -EEXIST;
goto out_unlock;
}
- goto found;
+ ins_index = i;
+ break;
+ } else if (spec->priority >
+ saved_spec->priority ||
+ (spec->priority ==
+ saved_spec->priority &&
+ replace_equal)) {
+ if (ins_index < 0)
+ ins_index = i;
+ else
+ __set_bit(depth, mc_rem_map);
}
-
- ++depth;
}
-
- prepare_to_wait(&table->waitq, &wait, TASK_UNINTERRUPTIBLE);
- spin_unlock_bh(&efx->filter_lock);
- schedule();
}
-found:
- /* Create a software table entry if necessary, and mark it
- * busy. We might yet fail to insert, but any attempt to
- * insert a conflicting filter while we're waiting for the
- * firmware must find the busy entry.
+ /* Once we reach the maximum search depth, use the first suitable
+ * slot, or return -EBUSY if there was none
*/
+ if (ins_index < 0) {
+ rc = -EBUSY;
+ goto out_unlock;
+ }
+
+ /* Create a software table entry if necessary. */
saved_spec = efx_ef10_filter_entry_spec(table, ins_index);
if (saved_spec) {
if (spec->priority == EFX_FILTER_PRI_AUTO &&
@@ -4438,28 +4421,13 @@ static s32 efx_ef10_filter_insert(struct efx_nic *efx,
*saved_spec = *spec;
priv_flags = 0;
}
- efx_ef10_filter_set_entry(table, ins_index, saved_spec,
- priv_flags | EFX_EF10_FILTER_FLAG_BUSY);
-
- /* Mark lower-priority multicast recipients busy prior to removal */
- if (is_mc_recip) {
- unsigned int depth, i;
-
- for (depth = 0; depth < EFX_EF10_FILTER_SEARCH_LIMIT; depth++) {
- i = (hash + depth) & (HUNT_FILTER_TBL_ROWS - 1);
- if (test_bit(depth, mc_rem_map))
- table->entry[i].spec |=
- EFX_EF10_FILTER_FLAG_BUSY;
- }
- }
-
- spin_unlock_bh(&efx->filter_lock);
+ efx_ef10_filter_set_entry(table, ins_index, saved_spec, priv_flags);
+ /* Actually insert the filter on the HW */
rc = efx_ef10_filter_push(efx, spec, &table->entry[ins_index].handle,
ctx, replacing);
/* Finalise the software table entry */
- spin_lock_bh(&efx->filter_lock);
if (rc == 0) {
if (replacing) {
/* Update the fields that may differ */
@@ -4475,6 +4443,12 @@ static s32 efx_ef10_filter_insert(struct efx_nic *efx,
} else if (!replacing) {
kfree(saved_spec);
saved_spec = NULL;
+ } else {
+ /* We failed to replace, so the old filter is still present.
+ * Roll back the software table to reflect this. In fact the
+ * efx_ef10_filter_set_entry() call below will do the right
+ * thing, so nothing extra is needed here.
+ */
}
efx_ef10_filter_set_entry(table, ins_index, saved_spec, priv_flags);
@@ -4496,7 +4470,6 @@ static s32 efx_ef10_filter_insert(struct efx_nic *efx,
priv_flags = efx_ef10_filter_entry_flags(table, i);
if (rc == 0) {
- spin_unlock_bh(&efx->filter_lock);
MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP,
MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE);
MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE,
@@ -4504,15 +4477,12 @@ static s32 efx_ef10_filter_insert(struct efx_nic *efx,
rc = efx_mcdi_rpc(efx, MC_CMD_FILTER_OP,
inbuf, sizeof(inbuf),
NULL, 0, NULL);
- spin_lock_bh(&efx->filter_lock);
}
if (rc == 0) {
kfree(saved_spec);
saved_spec = NULL;
priv_flags = 0;
- } else {
- priv_flags &= ~EFX_EF10_FILTER_FLAG_BUSY;
}
efx_ef10_filter_set_entry(table, i, saved_spec,
priv_flags);
@@ -4523,10 +4493,9 @@ static s32 efx_ef10_filter_insert(struct efx_nic *efx,
if (rc == 0)
rc = efx_ef10_make_filter_id(match_pri, ins_index);
- wake_up_all(&table->waitq);
out_unlock:
- spin_unlock_bh(&efx->filter_lock);
- finish_wait(&table->waitq, &wait);
+ up_write(&table->lock);
+ up_read(&efx->filter_sem);
return rc;
}
@@ -4539,6 +4508,8 @@ static void efx_ef10_filter_update_rx_scatter(struct efx_nic *efx)
* If !by_index, remove by ID
* If by_index, remove by index
* Filter ID may come from userland and must be range-checked.
+ * Caller must hold efx->filter_sem for read, and efx->filter_state->lock
+ * for write.
*/
static int efx_ef10_filter_remove_internal(struct efx_nic *efx,
unsigned int priority_mask,
@@ -4553,45 +4524,23 @@ static int efx_ef10_filter_remove_internal(struct efx_nic *efx,
DEFINE_WAIT(wait);
int rc;
- /* Find the software table entry and mark it busy. Don't
- * remove it yet; any attempt to update while we're waiting
- * for the firmware must find the busy entry.
- */
- for (;;) {
- spin_lock_bh(&efx->filter_lock);
- if (!(table->entry[filter_idx].spec &
- EFX_EF10_FILTER_FLAG_BUSY))
- break;
- prepare_to_wait(&table->waitq, &wait, TASK_UNINTERRUPTIBLE);
- spin_unlock_bh(&efx->filter_lock);
- schedule();
- }
-
spec = efx_ef10_filter_entry_spec(table, filter_idx);
if (!spec ||
(!by_index &&
efx_ef10_filter_pri(table, spec) !=
- efx_ef10_filter_get_unsafe_pri(filter_id))) {
- rc = -ENOENT;
- goto out_unlock;
- }
+ efx_ef10_filter_get_unsafe_pri(filter_id)))
+ return -ENOENT;
if (spec->flags & EFX_FILTER_FLAG_RX_OVER_AUTO &&
priority_mask == (1U << EFX_FILTER_PRI_AUTO)) {
/* Just remove flags */
spec->flags &= ~EFX_FILTER_FLAG_RX_OVER_AUTO;
table->entry[filter_idx].spec &= ~EFX_EF10_FILTER_FLAG_AUTO_OLD;
- rc = 0;
- goto out_unlock;
- }
-
- if (!(priority_mask & (1U << spec->priority))) {
- rc = -ENOENT;
- goto out_unlock;
+ return 0;
}
- table->entry[filter_idx].spec |= EFX_EF10_FILTER_FLAG_BUSY;
- spin_unlock_bh(&efx->filter_lock);
+ if (!(priority_mask & (1U << spec->priority)))
+ return -ENOENT;
if (spec->flags & EFX_FILTER_FLAG_RX_OVER_AUTO) {
/* Reset to an automatic filter */
@@ -4609,7 +4558,6 @@ static int efx_ef10_filter_remove_internal(struct efx_nic *efx,
&efx->rss_context,
true);
- spin_lock_bh(&efx->filter_lock);
if (rc == 0)
*spec = new_spec;
} else {
@@ -4624,7 +4572,6 @@ static int efx_ef10_filter_remove_internal(struct efx_nic *efx,
rc = efx_mcdi_rpc_quiet(efx, MC_CMD_FILTER_OP,
inbuf, sizeof(inbuf), NULL, 0, NULL);
- spin_lock_bh(&efx->filter_lock);
if ((rc == 0) || (rc == -ENOENT)) {
/* Filter removed OK or didn't actually exist */
kfree(spec);
@@ -4636,11 +4583,6 @@ static int efx_ef10_filter_remove_internal(struct efx_nic *efx,
}
}
- table->entry[filter_idx].spec &= ~EFX_EF10_FILTER_FLAG_BUSY;
- wake_up_all(&table->waitq);
-out_unlock:
- spin_unlock_bh(&efx->filter_lock);
- finish_wait(&table->waitq, &wait);
return rc;
}
@@ -4648,17 +4590,33 @@ static int efx_ef10_filter_remove_safe(struct efx_nic *efx,
enum efx_filter_priority priority,
u32 filter_id)
{
- return efx_ef10_filter_remove_internal(efx, 1U << priority,
- filter_id, false);
+ struct efx_ef10_filter_table *table;
+ int rc;
+
+ down_read(&efx->filter_sem);
+ table = efx->filter_state;
+ down_write(&table->lock);
+ rc = efx_ef10_filter_remove_internal(efx, 1U << priority, filter_id,
+ false);
+ up_write(&table->lock);
+ up_read(&efx->filter_sem);
+ return rc;
}
+/* Caller must hold efx->filter_sem for read */
static void efx_ef10_filter_remove_unsafe(struct efx_nic *efx,
enum efx_filter_priority priority,
u32 filter_id)
{
+ struct efx_ef10_filter_table *table = efx->filter_state;
+
if (filter_id == EFX_EF10_FILTER_ID_INVALID)
return;
- efx_ef10_filter_remove_internal(efx, 1U << priority, filter_id, true);
+
+ down_write(&table->lock);
+ efx_ef10_filter_remove_internal(efx, 1U << priority, filter_id,
+ true);
+ up_write(&table->lock);
}
static int efx_ef10_filter_get_safe(struct efx_nic *efx,
@@ -4666,11 +4624,13 @@ static int efx_ef10_filter_get_safe(struct efx_nic *efx,
u32 filter_id, struct efx_filter_spec *spec)
{
unsigned int filter_idx = efx_ef10_filter_get_unsafe_id(filter_id);
- struct efx_ef10_filter_table *table = efx->filter_state;
const struct efx_filter_spec *saved_spec;
+ struct efx_ef10_filter_table *table;
int rc;
- spin_lock_bh(&efx->filter_lock);
+ down_read(&efx->filter_sem);
+ table = efx->filter_state;
+ down_read(&table->lock);
saved_spec = efx_ef10_filter_entry_spec(table, filter_idx);
if (saved_spec && saved_spec->priority == priority &&
efx_ef10_filter_pri(table, saved_spec) ==
@@ -4680,13 +4640,15 @@ static int efx_ef10_filter_get_safe(struct efx_nic *efx,
} else {
rc = -ENOENT;
}
- spin_unlock_bh(&efx->filter_lock);
+ up_read(&table->lock);
+ up_read(&efx->filter_sem);
return rc;
}
static int efx_ef10_filter_clear_rx(struct efx_nic *efx,
- enum efx_filter_priority priority)
+ enum efx_filter_priority priority)
{
+ struct efx_ef10_filter_table *table;
unsigned int priority_mask;
unsigned int i;
int rc;
@@ -4694,31 +4656,40 @@ static int efx_ef10_filter_clear_rx(struct efx_nic *efx,
priority_mask = (((1U << (priority + 1)) - 1) &
~(1U << EFX_FILTER_PRI_AUTO));
+ down_read(&efx->filter_sem);
+ table = efx->filter_state;
+ down_write(&table->lock);
for (i = 0; i < HUNT_FILTER_TBL_ROWS; i++) {
rc = efx_ef10_filter_remove_internal(efx, priority_mask,
i, true);
if (rc && rc != -ENOENT)
- return rc;
+ break;
+ rc = 0;
}
- return 0;
+ up_write(&table->lock);
+ up_read(&efx->filter_sem);
+ return rc;
}
static u32 efx_ef10_filter_count_rx_used(struct efx_nic *efx,
enum efx_filter_priority priority)
{
- struct efx_ef10_filter_table *table = efx->filter_state;
+ struct efx_ef10_filter_table *table;
unsigned int filter_idx;
s32 count = 0;
- spin_lock_bh(&efx->filter_lock);
+ down_read(&efx->filter_sem);
+ table = efx->filter_state;
+ down_read(&table->lock);
for (filter_idx = 0; filter_idx < HUNT_FILTER_TBL_ROWS; filter_idx++) {
if (table->entry[filter_idx].spec &&
efx_ef10_filter_entry_spec(table, filter_idx)->priority ==
priority)
++count;
}
- spin_unlock_bh(&efx->filter_lock);
+ up_read(&table->lock);
+ up_read(&efx->filter_sem);
return count;
}
@@ -4733,12 +4704,15 @@ static s32 efx_ef10_filter_get_rx_ids(struct efx_nic *efx,
enum efx_filter_priority priority,
u32 *buf, u32 size)
{
- struct efx_ef10_filter_table *table = efx->filter_state;
+ struct efx_ef10_filter_table *table;
struct efx_filter_spec *spec;
unsigned int filter_idx;
s32 count = 0;
- spin_lock_bh(&efx->filter_lock);
+ down_read(&efx->filter_sem);
+ table = efx->filter_state;
+ down_read(&table->lock);
+
for (filter_idx = 0; filter_idx < HUNT_FILTER_TBL_ROWS; filter_idx++) {
spec = efx_ef10_filter_entry_spec(table, filter_idx);
if (spec && spec->priority == priority) {
@@ -4752,73 +4726,44 @@ static s32 efx_ef10_filter_get_rx_ids(struct efx_nic *efx,
filter_idx);
}
}
- spin_unlock_bh(&efx->filter_lock);
+ up_read(&table->lock);
+ up_read(&efx->filter_sem);
return count;
}
#ifdef CONFIG_RFS_ACCEL
-static void
-efx_ef10_filter_rfs_expire_complete(struct efx_nic *efx,
- unsigned long filter_idx,
- int rc, efx_dword_t *outbuf,
- size_t outlen_actual);
-
static bool efx_ef10_filter_rfs_expire_one(struct efx_nic *efx, u32 flow_id,
unsigned int filter_idx)
{
- struct efx_ef10_filter_table *table = efx->filter_state;
+ struct efx_ef10_filter_table *table;
struct efx_filter_spec *spec;
- MCDI_DECLARE_BUF(inbuf,
- MC_CMD_FILTER_OP_IN_HANDLE_OFST +
- MC_CMD_FILTER_OP_IN_HANDLE_LEN);
- bool ret = true;
+ bool ret;
- spin_lock_bh(&efx->filter_lock);
+ down_read(&efx->filter_sem);
+ table = efx->filter_state;
+ down_write(&table->lock);
spec = efx_ef10_filter_entry_spec(table, filter_idx);
- if (!spec ||
- (table->entry[filter_idx].spec & EFX_EF10_FILTER_FLAG_BUSY) ||
- spec->priority != EFX_FILTER_PRI_HINT ||
- !rps_may_expire_flow(efx->net_dev, spec->dmaq_id,
+
+ if (!spec || spec->priority != EFX_FILTER_PRI_HINT) {
+ ret = true;
+ goto out_unlock;
+ }
+
+ if (!rps_may_expire_flow(efx->net_dev, spec->dmaq_id,
flow_id, filter_idx)) {
ret = false;
goto out_unlock;
}
- MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP,
- MC_CMD_FILTER_OP_IN_OP_REMOVE);
- MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE,
- table->entry[filter_idx].handle);
- if (efx_mcdi_rpc_async(efx, MC_CMD_FILTER_OP, inbuf, sizeof(inbuf), 0,
- efx_ef10_filter_rfs_expire_complete, filter_idx))
- ret = false;
- else
- table->entry[filter_idx].spec |= EFX_EF10_FILTER_FLAG_BUSY;
+ ret = efx_ef10_filter_remove_internal(efx, 1U << spec->priority,
+ filter_idx, true) == 0;
out_unlock:
- spin_unlock_bh(&efx->filter_lock);
+ up_write(&table->lock);
+ up_read(&efx->filter_sem);
return ret;
}
-static void
-efx_ef10_filter_rfs_expire_complete(struct efx_nic *efx,
- unsigned long filter_idx,
- int rc, efx_dword_t *outbuf,
- size_t outlen_actual)
-{
- struct efx_ef10_filter_table *table = efx->filter_state;
- struct efx_filter_spec *spec =
- efx_ef10_filter_entry_spec(table, filter_idx);
-
- spin_lock_bh(&efx->filter_lock);
- if (rc == 0) {
- kfree(spec);
- efx_ef10_filter_set_entry(table, filter_idx, NULL, 0);
- }
- table->entry[filter_idx].spec &= ~EFX_EF10_FILTER_FLAG_BUSY;
- wake_up_all(&table->waitq);
- spin_unlock_bh(&efx->filter_lock);
-}
-
#endif /* CONFIG_RFS_ACCEL */
static int efx_ef10_filter_match_flags_from_mcdi(bool encap, u32 mcdi_flags)
@@ -5011,9 +4956,9 @@ static int efx_ef10_filter_table_probe(struct efx_nic *efx)
table->vlan_filter =
!!(efx->net_dev->features & NETIF_F_HW_VLAN_CTAG_FILTER);
INIT_LIST_HEAD(&table->vlan_list);
+ init_rwsem(&table->lock);
efx->filter_state = table;
- init_waitqueue_head(&table->waitq);
list_for_each_entry(vlan, &nic_data->vlan_list, list) {
rc = efx_ef10_filter_add_vlan(efx, vlan->vid);
@@ -5055,7 +5000,7 @@ static void efx_ef10_filter_table_restore(struct efx_nic *efx)
if (!table)
return;
- spin_lock_bh(&efx->filter_lock);
+ down_write(&table->lock);
for (filter_idx = 0; filter_idx < HUNT_FILTER_TBL_ROWS; filter_idx++) {
spec = efx_ef10_filter_entry_spec(table, filter_idx);
@@ -5093,15 +5038,11 @@ static void efx_ef10_filter_table_restore(struct efx_nic *efx)
}
}
- table->entry[filter_idx].spec |= EFX_EF10_FILTER_FLAG_BUSY;
- spin_unlock_bh(&efx->filter_lock);
-
rc = efx_ef10_filter_push(efx, spec,
&table->entry[filter_idx].handle,
ctx, false);
if (rc)
failed++;
- spin_lock_bh(&efx->filter_lock);
if (rc) {
not_restored:
@@ -5113,13 +5054,10 @@ static void efx_ef10_filter_table_restore(struct efx_nic *efx)
kfree(spec);
efx_ef10_filter_set_entry(table, filter_idx, NULL, 0);
- } else {
- table->entry[filter_idx].spec &=
- ~EFX_EF10_FILTER_FLAG_BUSY;
}
}
- spin_unlock_bh(&efx->filter_lock);
+ up_write(&table->lock);
/* This can happen validly if the MC's capabilities have changed, so
* is not an error.
@@ -5187,6 +5125,8 @@ static void efx_ef10_filter_mark_one_old(struct efx_nic *efx, uint16_t *id)
struct efx_ef10_filter_table *table = efx->filter_state;
unsigned int filter_idx;
+ efx_rwsem_assert_write_locked(&table->lock);
+
if (*id != EFX_EF10_FILTER_ID_INVALID) {
filter_idx = efx_ef10_filter_get_unsafe_id(*id);
if (!table->entry[filter_idx].spec)
@@ -5222,10 +5162,10 @@ static void efx_ef10_filter_mark_old(struct efx_nic *efx)
struct efx_ef10_filter_table *table = efx->filter_state;
struct efx_ef10_filter_vlan *vlan;
- spin_lock_bh(&efx->filter_lock);
+ down_write(&table->lock);
list_for_each_entry(vlan, &table->vlan_list, list)
_efx_ef10_filter_vlan_mark_old(efx, vlan);
- spin_unlock_bh(&efx->filter_lock);
+ up_write(&table->lock);
}
static void efx_ef10_filter_uc_addr_list(struct efx_nic *efx)
@@ -5502,10 +5442,7 @@ static int efx_ef10_filter_insert_def(struct efx_nic *efx,
return rc;
}
-/* Remove filters that weren't renewed. Since nothing else changes the AUTO_OLD
- * flag or removes these filters, we don't need to hold the filter_lock while
- * scanning for these filters.
- */
+/* Remove filters that weren't renewed. */
static void efx_ef10_filter_remove_old(struct efx_nic *efx)
{
struct efx_ef10_filter_table *table = efx->filter_state;
@@ -5514,6 +5451,7 @@ static void efx_ef10_filter_remove_old(struct efx_nic *efx)
int rc;
int i;
+ down_write(&table->lock);
for (i = 0; i < HUNT_FILTER_TBL_ROWS; i++) {
if (READ_ONCE(table->entry[i].spec) &
EFX_EF10_FILTER_FLAG_AUTO_OLD) {
@@ -5525,6 +5463,7 @@ static void efx_ef10_filter_remove_old(struct efx_nic *efx)
remove_failed++;
}
}
+ up_write(&table->lock);
if (remove_failed)
netif_info(efx, drv, efx->net_dev,
diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c
index 0be93abdb2ad..cb9273016916 100644
--- a/drivers/net/ethernet/sfc/efx.c
+++ b/drivers/net/ethernet/sfc/efx.c
@@ -1783,7 +1783,6 @@ static int efx_probe_filters(struct efx_nic *efx)
{
int rc;
- spin_lock_init(&efx->filter_lock);
init_rwsem(&efx->filter_sem);
mutex_lock(&efx->mac_lock);
down_write(&efx->filter_sem);
diff --git a/drivers/net/ethernet/sfc/net_driver.h b/drivers/net/ethernet/sfc/net_driver.h
index fe0c5322454d..6f37a4dbe268 100644
--- a/drivers/net/ethernet/sfc/net_driver.h
+++ b/drivers/net/ethernet/sfc/net_driver.h
@@ -836,7 +836,7 @@ struct efx_rss_context {
* @loopback_mode: Loopback status
* @loopback_modes: Supported loopback mode bitmask
* @loopback_selftest: Offline self-test private state
- * @filter_sem: Filter table rw_semaphore, for freeing the table
+ * @filter_sem: Filter table rw_semaphore, protects existence of @filter_state
* @filter_lock: Filter table lock, for mere content changes
* @filter_state: Architecture-dependent filter table state
* @rps_mutex: Protects RPS state of all channels
^ permalink raw reply related
* [PATCH net-next 3/6] sfc: use a semaphore to lock farch filters too
From: Edward Cree @ 2018-03-27 16:42 UTC (permalink / raw)
To: linux-net-drivers, David Miller; +Cc: netdev
In-Reply-To: <7a932be7-0335-a7f3-4fb4-408ccb191737@solarflare.com>
With this change, the spinlock efx->filter_lock is no longer used and is
thus removed.
Signed-off-by: Edward Cree <ecree@solarflare.com>
---
drivers/net/ethernet/sfc/farch.c | 67 +++++++++++++++++------------------
drivers/net/ethernet/sfc/net_driver.h | 2 --
2 files changed, 33 insertions(+), 36 deletions(-)
diff --git a/drivers/net/ethernet/sfc/farch.c b/drivers/net/ethernet/sfc/farch.c
index 6b5ca569db5e..4a19c7efdf8d 100644
--- a/drivers/net/ethernet/sfc/farch.c
+++ b/drivers/net/ethernet/sfc/farch.c
@@ -1878,6 +1878,7 @@ struct efx_farch_filter_table {
};
struct efx_farch_filter_state {
+ struct rw_semaphore lock; /* Protects table contents */
struct efx_farch_filter_table table[EFX_FARCH_FILTER_TABLE_COUNT];
};
@@ -2397,9 +2398,13 @@ s32 efx_farch_filter_insert(struct efx_nic *efx,
if (rc)
return rc;
+ down_write(&state->lock);
+
table = &state->table[efx_farch_filter_spec_table_id(&spec)];
- if (table->size == 0)
- return -EINVAL;
+ if (table->size == 0) {
+ rc = -EINVAL;
+ goto out_unlock;
+ }
netif_vdbg(efx, hw, efx->net_dev,
"%s: type %d search_limit=%d", __func__, spec.type,
@@ -2412,8 +2417,6 @@ s32 efx_farch_filter_insert(struct efx_nic *efx,
EFX_FARCH_FILTER_MC_DEF - EFX_FARCH_FILTER_UC_DEF);
rep_index = spec.type - EFX_FARCH_FILTER_UC_DEF;
ins_index = rep_index;
-
- spin_lock_bh(&efx->filter_lock);
} else {
/* Search concurrently for
* (1) a filter to be replaced (rep_index): any filter
@@ -2443,8 +2446,6 @@ s32 efx_farch_filter_insert(struct efx_nic *efx,
ins_index = -1;
depth = 1;
- spin_lock_bh(&efx->filter_lock);
-
for (;;) {
if (!test_bit(i, table->used_bitmap)) {
if (ins_index < 0)
@@ -2463,7 +2464,7 @@ s32 efx_farch_filter_insert(struct efx_nic *efx,
/* Case (b) */
if (ins_index < 0) {
rc = -EBUSY;
- goto out;
+ goto out_unlock;
}
rep_index = -1;
break;
@@ -2483,11 +2484,11 @@ s32 efx_farch_filter_insert(struct efx_nic *efx,
if (spec.priority == saved_spec->priority && !replace_equal) {
rc = -EEXIST;
- goto out;
+ goto out_unlock;
}
if (spec.priority < saved_spec->priority) {
rc = -EPERM;
- goto out;
+ goto out_unlock;
}
if (saved_spec->priority == EFX_FILTER_PRI_AUTO ||
saved_spec->flags & EFX_FILTER_FLAG_RX_OVER_AUTO)
@@ -2528,8 +2529,8 @@ s32 efx_farch_filter_insert(struct efx_nic *efx,
__func__, spec.type, ins_index, spec.dmaq_id);
rc = efx_farch_filter_make_id(&spec, ins_index);
-out:
- spin_unlock_bh(&efx->filter_lock);
+out_unlock:
+ up_write(&state->lock);
return rc;
}
@@ -2604,11 +2605,11 @@ int efx_farch_filter_remove_safe(struct efx_nic *efx,
filter_idx = efx_farch_filter_id_index(filter_id);
if (filter_idx >= table->size)
return -ENOENT;
+ down_write(&state->lock);
spec = &table->spec[filter_idx];
- spin_lock_bh(&efx->filter_lock);
rc = efx_farch_filter_remove(efx, table, filter_idx, priority);
- spin_unlock_bh(&efx->filter_lock);
+ up_write(&state->lock);
return rc;
}
@@ -2622,30 +2623,28 @@ int efx_farch_filter_get_safe(struct efx_nic *efx,
struct efx_farch_filter_table *table;
struct efx_farch_filter_spec *spec;
unsigned int filter_idx;
- int rc;
+ int rc = -ENOENT;
+
+ down_read(&state->lock);
table_id = efx_farch_filter_id_table_id(filter_id);
if ((unsigned int)table_id >= EFX_FARCH_FILTER_TABLE_COUNT)
- return -ENOENT;
+ goto out_unlock;
table = &state->table[table_id];
filter_idx = efx_farch_filter_id_index(filter_id);
if (filter_idx >= table->size)
- return -ENOENT;
+ goto out_unlock;
spec = &table->spec[filter_idx];
- spin_lock_bh(&efx->filter_lock);
-
if (test_bit(filter_idx, table->used_bitmap) &&
spec->priority == priority) {
efx_farch_filter_to_gen_spec(spec_buf, spec);
rc = 0;
- } else {
- rc = -ENOENT;
}
- spin_unlock_bh(&efx->filter_lock);
-
+out_unlock:
+ up_read(&state->lock);
return rc;
}
@@ -2658,13 +2657,13 @@ efx_farch_filter_table_clear(struct efx_nic *efx,
struct efx_farch_filter_table *table = &state->table[table_id];
unsigned int filter_idx;
- spin_lock_bh(&efx->filter_lock);
+ down_write(&state->lock);
for (filter_idx = 0; filter_idx < table->size; ++filter_idx) {
if (table->spec[filter_idx].priority != EFX_FILTER_PRI_AUTO)
efx_farch_filter_remove(efx, table,
filter_idx, priority);
}
- spin_unlock_bh(&efx->filter_lock);
+ up_write(&state->lock);
}
int efx_farch_filter_clear_rx(struct efx_nic *efx,
@@ -2688,7 +2687,7 @@ u32 efx_farch_filter_count_rx_used(struct efx_nic *efx,
unsigned int filter_idx;
u32 count = 0;
- spin_lock_bh(&efx->filter_lock);
+ down_read(&state->lock);
for (table_id = EFX_FARCH_FILTER_TABLE_RX_IP;
table_id <= EFX_FARCH_FILTER_TABLE_RX_DEF;
@@ -2701,7 +2700,7 @@ u32 efx_farch_filter_count_rx_used(struct efx_nic *efx,
}
}
- spin_unlock_bh(&efx->filter_lock);
+ up_read(&state->lock);
return count;
}
@@ -2716,7 +2715,7 @@ s32 efx_farch_filter_get_rx_ids(struct efx_nic *efx,
unsigned int filter_idx;
s32 count = 0;
- spin_lock_bh(&efx->filter_lock);
+ down_read(&state->lock);
for (table_id = EFX_FARCH_FILTER_TABLE_RX_IP;
table_id <= EFX_FARCH_FILTER_TABLE_RX_DEF;
@@ -2735,7 +2734,7 @@ s32 efx_farch_filter_get_rx_ids(struct efx_nic *efx,
}
}
out:
- spin_unlock_bh(&efx->filter_lock);
+ up_read(&state->lock);
return count;
}
@@ -2749,7 +2748,7 @@ void efx_farch_filter_table_restore(struct efx_nic *efx)
efx_oword_t filter;
unsigned int filter_idx;
- spin_lock_bh(&efx->filter_lock);
+ down_write(&state->lock);
for (table_id = 0; table_id < EFX_FARCH_FILTER_TABLE_COUNT; table_id++) {
table = &state->table[table_id];
@@ -2770,7 +2769,7 @@ void efx_farch_filter_table_restore(struct efx_nic *efx)
efx_farch_filter_push_rx_config(efx);
efx_farch_filter_push_tx_limits(efx);
- spin_unlock_bh(&efx->filter_lock);
+ up_write(&state->lock);
}
void efx_farch_filter_table_remove(struct efx_nic *efx)
@@ -2864,7 +2863,7 @@ void efx_farch_filter_update_rx_scatter(struct efx_nic *efx)
efx_oword_t filter;
unsigned int filter_idx;
- spin_lock_bh(&efx->filter_lock);
+ down_write(&state->lock);
for (table_id = EFX_FARCH_FILTER_TABLE_RX_IP;
table_id <= EFX_FARCH_FILTER_TABLE_RX_DEF;
@@ -2896,7 +2895,7 @@ void efx_farch_filter_update_rx_scatter(struct efx_nic *efx)
efx_farch_filter_push_rx_config(efx);
- spin_unlock_bh(&efx->filter_lock);
+ up_write(&state->lock);
}
#ifdef CONFIG_RFS_ACCEL
@@ -2908,7 +2907,7 @@ bool efx_farch_filter_rfs_expire_one(struct efx_nic *efx, u32 flow_id,
struct efx_farch_filter_table *table;
bool ret = false;
- spin_lock_bh(&efx->filter_lock);
+ down_write(&state->lock);
table = &state->table[EFX_FARCH_FILTER_TABLE_RX_IP];
if (test_bit(index, table->used_bitmap) &&
table->spec[index].priority == EFX_FILTER_PRI_HINT &&
@@ -2918,7 +2917,7 @@ bool efx_farch_filter_rfs_expire_one(struct efx_nic *efx, u32 flow_id,
ret = true;
}
- spin_unlock_bh(&efx->filter_lock);
+ up_write(&state->lock);
return ret;
}
diff --git a/drivers/net/ethernet/sfc/net_driver.h b/drivers/net/ethernet/sfc/net_driver.h
index 6f37a4dbe268..ef6fc931afb8 100644
--- a/drivers/net/ethernet/sfc/net_driver.h
+++ b/drivers/net/ethernet/sfc/net_driver.h
@@ -837,7 +837,6 @@ struct efx_rss_context {
* @loopback_modes: Supported loopback mode bitmask
* @loopback_selftest: Offline self-test private state
* @filter_sem: Filter table rw_semaphore, protects existence of @filter_state
- * @filter_lock: Filter table lock, for mere content changes
* @filter_state: Architecture-dependent filter table state
* @rps_mutex: Protects RPS state of all channels
* @rps_expire_channel: Next channel to check for expiry
@@ -990,7 +989,6 @@ struct efx_nic {
void *loopback_selftest;
struct rw_semaphore filter_sem;
- spinlock_t filter_lock;
void *filter_state;
#ifdef CONFIG_RFS_ACCEL
struct mutex rps_mutex;
^ permalink raw reply related
* [PATCH net-next 4/6] sfc: return a better error if filter insertion collides with MC reboot
From: Edward Cree @ 2018-03-27 16:44 UTC (permalink / raw)
To: linux-net-drivers, David Miller; +Cc: netdev
In-Reply-To: <7a932be7-0335-a7f3-4fb4-408ccb191737@solarflare.com>
If some other operation gets the MCDI lock ahead of us and performs an MC
reboot, then our attempt to insert the filter will fail with EINVAL,
because the destination VI (spec->dmaq_id, MC_CMD_FILTER_OP_IN_RX_QUEUE) does
not exist. But the caller's request (which might e.g. be an ethtool ntuple
request from userland) isn't invalid, it just got unlucky; so return EAGAIN.
Signed-off-by: Edward Cree <ecree@solarflare.com>
---
drivers/net/ethernet/sfc/ef10.c | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/drivers/net/ethernet/sfc/ef10.c b/drivers/net/ethernet/sfc/ef10.c
index 9db1b9144e70..9a139c390037 100644
--- a/drivers/net/ethernet/sfc/ef10.c
+++ b/drivers/net/ethernet/sfc/ef10.c
@@ -4301,6 +4301,7 @@ static s32 efx_ef10_filter_insert(struct efx_nic *efx,
bool replace_equal)
{
DECLARE_BITMAP(mc_rem_map, EFX_EF10_FILTER_SEARCH_LIMIT);
+ struct efx_ef10_nic_data *nic_data = efx->nic_data;
struct efx_ef10_filter_table *table;
struct efx_filter_spec *saved_spec;
struct efx_rss_context *ctx = NULL;
@@ -4427,6 +4428,12 @@ static s32 efx_ef10_filter_insert(struct efx_nic *efx,
rc = efx_ef10_filter_push(efx, spec, &table->entry[ins_index].handle,
ctx, replacing);
+ if (rc == -EINVAL && nic_data->must_realloc_vis)
+ /* The MC rebooted under us, causing it to reject our filter
+ * insertion as pointing to an invalid VI (spec->dmaq_id).
+ */
+ rc = -EAGAIN;
+
/* Finalise the software table entry */
if (rc == 0) {
if (replacing) {
^ permalink raw reply related
* [PATCH net-next 5/6] sfc: protect list of RSS contexts under a mutex
From: Edward Cree @ 2018-03-27 16:44 UTC (permalink / raw)
To: linux-net-drivers, David Miller; +Cc: netdev
In-Reply-To: <7a932be7-0335-a7f3-4fb4-408ccb191737@solarflare.com>
Otherwise races are possible between ethtool ops and
efx_ef10_rx_restore_rss_contexts().
Also, don't try to perform the restore on every reset, only after an MC
reboot, otherwise we'll leak RSS contexts on the NIC.
Fixes: 42356d9a137b ("sfc: support RSS spreading of ethtool ntuple filters")
Signed-off-by: Edward Cree <ecree@solarflare.com>
---
drivers/net/ethernet/sfc/ef10.c | 32 ++++++++++++++---
drivers/net/ethernet/sfc/efx.c | 14 ++++++--
drivers/net/ethernet/sfc/efx.h | 4 +--
drivers/net/ethernet/sfc/ethtool.c | 66 +++++++++++++++++++++++------------
drivers/net/ethernet/sfc/net_driver.h | 2 ++
drivers/net/ethernet/sfc/nic.h | 3 ++
6 files changed, 89 insertions(+), 32 deletions(-)
diff --git a/drivers/net/ethernet/sfc/ef10.c b/drivers/net/ethernet/sfc/ef10.c
index 9a139c390037..c4c45c94da77 100644
--- a/drivers/net/ethernet/sfc/ef10.c
+++ b/drivers/net/ethernet/sfc/ef10.c
@@ -1499,6 +1499,7 @@ static void efx_ef10_reset_mc_allocations(struct efx_nic *efx)
/* All our allocations have been reset */
nic_data->must_realloc_vis = true;
+ nic_data->must_restore_rss_contexts = true;
nic_data->must_restore_filters = true;
nic_data->must_restore_piobufs = true;
efx_ef10_forget_old_piobufs(efx);
@@ -2899,6 +2900,8 @@ static int efx_ef10_rx_push_rss_context_config(struct efx_nic *efx,
{
int rc;
+ WARN_ON(!mutex_is_locked(&efx->rss_lock));
+
if (ctx->context_id == EFX_EF10_RSS_CONTEXT_INVALID) {
rc = efx_ef10_alloc_rss_context(efx, true, ctx, NULL);
if (rc)
@@ -2929,6 +2932,8 @@ static int efx_ef10_rx_pull_rss_context_config(struct efx_nic *efx,
size_t outlen;
int rc, i;
+ WARN_ON(!mutex_is_locked(&efx->rss_lock));
+
BUILD_BUG_ON(MC_CMD_RSS_CONTEXT_GET_TABLE_IN_LEN !=
MC_CMD_RSS_CONTEXT_GET_KEY_IN_LEN);
@@ -2972,14 +2977,25 @@ static int efx_ef10_rx_pull_rss_context_config(struct efx_nic *efx,
static int efx_ef10_rx_pull_rss_config(struct efx_nic *efx)
{
- return efx_ef10_rx_pull_rss_context_config(efx, &efx->rss_context);
+ int rc;
+
+ mutex_lock(&efx->rss_lock);
+ rc = efx_ef10_rx_pull_rss_context_config(efx, &efx->rss_context);
+ mutex_unlock(&efx->rss_lock);
+ return rc;
}
static void efx_ef10_rx_restore_rss_contexts(struct efx_nic *efx)
{
+ struct efx_ef10_nic_data *nic_data = efx->nic_data;
struct efx_rss_context *ctx;
int rc;
+ WARN_ON(!mutex_is_locked(&efx->rss_lock));
+
+ if (!nic_data->must_restore_rss_contexts)
+ return;
+
list_for_each_entry(ctx, &efx->rss_context.list, list) {
/* previous NIC RSS context is gone */
ctx->context_id = EFX_EF10_RSS_CONTEXT_INVALID;
@@ -2993,6 +3009,7 @@ static void efx_ef10_rx_restore_rss_contexts(struct efx_nic *efx)
"; RSS filters may fail to be applied\n",
ctx->user_id, rc);
}
+ nic_data->must_restore_rss_contexts = false;
}
static int efx_ef10_pf_rx_push_rss_config(struct efx_nic *efx, bool user,
@@ -4307,6 +4324,7 @@ static s32 efx_ef10_filter_insert(struct efx_nic *efx,
struct efx_rss_context *ctx = NULL;
unsigned int match_pri, hash;
unsigned int priv_flags;
+ bool rss_locked = false;
bool replacing = false;
unsigned int depth, i;
int ins_index = -1;
@@ -4336,9 +4354,10 @@ static s32 efx_ef10_filter_insert(struct efx_nic *efx,
bitmap_zero(mc_rem_map, EFX_EF10_FILTER_SEARCH_LIMIT);
if (spec->flags & EFX_FILTER_FLAG_RX_RSS) {
+ mutex_lock(&efx->rss_lock);
+ rss_locked = true;
if (spec->rss_context)
- ctx = efx_find_rss_context_entry(spec->rss_context,
- &efx->rss_context.list);
+ ctx = efx_find_rss_context_entry(efx, spec->rss_context);
else
ctx = &efx->rss_context;
if (!ctx) {
@@ -4501,6 +4520,8 @@ static s32 efx_ef10_filter_insert(struct efx_nic *efx,
rc = efx_ef10_make_filter_id(match_pri, ins_index);
out_unlock:
+ if (rss_locked)
+ mutex_unlock(&efx->rss_lock);
up_write(&table->lock);
up_read(&efx->filter_sem);
return rc;
@@ -5008,6 +5029,7 @@ static void efx_ef10_filter_table_restore(struct efx_nic *efx)
return;
down_write(&table->lock);
+ mutex_lock(&efx->rss_lock);
for (filter_idx = 0; filter_idx < HUNT_FILTER_TBL_ROWS; filter_idx++) {
spec = efx_ef10_filter_entry_spec(table, filter_idx);
@@ -5024,8 +5046,7 @@ static void efx_ef10_filter_table_restore(struct efx_nic *efx)
goto not_restored;
}
if (spec->rss_context)
- ctx = efx_find_rss_context_entry(spec->rss_context,
- &efx->rss_context.list);
+ ctx = efx_find_rss_context_entry(efx, spec->rss_context);
else
ctx = &efx->rss_context;
if (spec->flags & EFX_FILTER_FLAG_RX_RSS) {
@@ -5064,6 +5085,7 @@ static void efx_ef10_filter_table_restore(struct efx_nic *efx)
}
}
+ mutex_unlock(&efx->rss_lock);
up_write(&table->lock);
/* This can happen validly if the MC's capabilities have changed, so
diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c
index cb9273016916..692dd729ee2a 100644
--- a/drivers/net/ethernet/sfc/efx.c
+++ b/drivers/net/ethernet/sfc/efx.c
@@ -2657,6 +2657,7 @@ void efx_reset_down(struct efx_nic *efx, enum reset_type method)
efx_disable_interrupts(efx);
mutex_lock(&efx->mac_lock);
+ mutex_lock(&efx->rss_lock);
if (efx->port_initialized && method != RESET_TYPE_INVISIBLE &&
method != RESET_TYPE_DATAPATH)
efx->phy_op->fini(efx);
@@ -2712,6 +2713,7 @@ int efx_reset_up(struct efx_nic *efx, enum reset_type method, bool ok)
if (efx->type->rx_restore_rss_contexts)
efx->type->rx_restore_rss_contexts(efx);
+ mutex_unlock(&efx->rss_lock);
down_read(&efx->filter_sem);
efx_restore_filters(efx);
up_read(&efx->filter_sem);
@@ -2730,6 +2732,7 @@ int efx_reset_up(struct efx_nic *efx, enum reset_type method, bool ok)
fail:
efx->port_initialized = false;
+ mutex_unlock(&efx->rss_lock);
mutex_unlock(&efx->mac_lock);
return rc;
@@ -3016,6 +3019,7 @@ static int efx_init_struct(struct efx_nic *efx,
efx->rx_packet_ts_offset =
efx->type->rx_ts_offset - efx->type->rx_prefix_size;
INIT_LIST_HEAD(&efx->rss_context.list);
+ mutex_init(&efx->rss_lock);
spin_lock_init(&efx->stats_lock);
efx->vi_stride = EFX_DEFAULT_VI_STRIDE;
efx->num_mac_stats = MC_CMD_MAC_NSTATS;
@@ -3091,11 +3095,14 @@ void efx_update_sw_stats(struct efx_nic *efx, u64 *stats)
/* RSS contexts. We're using linked lists and crappy O(n) algorithms, because
* (a) this is an infrequent control-plane operation and (b) n is small (max 64)
*/
-struct efx_rss_context *efx_alloc_rss_context_entry(struct list_head *head)
+struct efx_rss_context *efx_alloc_rss_context_entry(struct efx_nic *efx)
{
+ struct list_head *head = &efx->rss_context.list;
struct efx_rss_context *ctx, *new;
u32 id = 1; /* Don't use zero, that refers to the master RSS context */
+ WARN_ON(!mutex_is_locked(&efx->rss_lock));
+
/* Search for first gap in the numbering */
list_for_each_entry(ctx, head, list) {
if (ctx->user_id != id)
@@ -3121,10 +3128,13 @@ struct efx_rss_context *efx_alloc_rss_context_entry(struct list_head *head)
return new;
}
-struct efx_rss_context *efx_find_rss_context_entry(u32 id, struct list_head *head)
+struct efx_rss_context *efx_find_rss_context_entry(struct efx_nic *efx, u32 id)
{
+ struct list_head *head = &efx->rss_context.list;
struct efx_rss_context *ctx;
+ WARN_ON(!mutex_is_locked(&efx->rss_lock));
+
list_for_each_entry(ctx, head, list)
if (ctx->user_id == id)
return ctx;
diff --git a/drivers/net/ethernet/sfc/efx.h b/drivers/net/ethernet/sfc/efx.h
index 545c2ea1622e..a3140e16fcef 100644
--- a/drivers/net/ethernet/sfc/efx.h
+++ b/drivers/net/ethernet/sfc/efx.h
@@ -187,8 +187,8 @@ static inline void efx_filter_rfs_expire(struct work_struct *data) {}
bool efx_filter_is_mc_recipient(const struct efx_filter_spec *spec);
/* RSS contexts */
-struct efx_rss_context *efx_alloc_rss_context_entry(struct list_head *list);
-struct efx_rss_context *efx_find_rss_context_entry(u32 id, struct list_head *list);
+struct efx_rss_context *efx_alloc_rss_context_entry(struct efx_nic *efx);
+struct efx_rss_context *efx_find_rss_context_entry(struct efx_nic *efx, u32 id);
void efx_free_rss_context_entry(struct efx_rss_context *ctx);
static inline bool efx_rss_active(struct efx_rss_context *ctx)
{
diff --git a/drivers/net/ethernet/sfc/ethtool.c b/drivers/net/ethernet/sfc/ethtool.c
index 64049e71e6e7..de1fac6cd34f 100644
--- a/drivers/net/ethernet/sfc/ethtool.c
+++ b/drivers/net/ethernet/sfc/ethtool.c
@@ -979,7 +979,7 @@ efx_ethtool_get_rxnfc(struct net_device *net_dev,
{
struct efx_nic *efx = netdev_priv(net_dev);
u32 rss_context = 0;
- s32 rc;
+ s32 rc = 0;
switch (info->cmd) {
case ETHTOOL_GRXRINGS:
@@ -989,15 +989,17 @@ efx_ethtool_get_rxnfc(struct net_device *net_dev,
case ETHTOOL_GRXFH: {
struct efx_rss_context *ctx = &efx->rss_context;
+ mutex_lock(&efx->rss_lock);
if (info->flow_type & FLOW_RSS && info->rss_context) {
- ctx = efx_find_rss_context_entry(info->rss_context,
- &efx->rss_context.list);
- if (!ctx)
- return -ENOENT;
+ ctx = efx_find_rss_context_entry(efx, info->rss_context);
+ if (!ctx) {
+ rc = -ENOENT;
+ goto out_unlock;
+ }
}
info->data = 0;
if (!efx_rss_active(ctx)) /* No RSS */
- return 0;
+ goto out_unlock;
switch (info->flow_type & ~FLOW_RSS) {
case UDP_V4_FLOW:
if (ctx->rx_hash_udp_4tuple)
@@ -1024,7 +1026,9 @@ efx_ethtool_get_rxnfc(struct net_device *net_dev,
default:
break;
}
- return 0;
+out_unlock:
+ mutex_unlock(&efx->rss_lock);
+ return rc;
}
case ETHTOOL_GRXCLSRLCNT:
@@ -1366,16 +1370,20 @@ static int efx_ethtool_get_rxfh_context(struct net_device *net_dev, u32 *indir,
{
struct efx_nic *efx = netdev_priv(net_dev);
struct efx_rss_context *ctx;
- int rc;
+ int rc = 0;
if (!efx->type->rx_pull_rss_context_config)
return -EOPNOTSUPP;
- ctx = efx_find_rss_context_entry(rss_context, &efx->rss_context.list);
- if (!ctx)
- return -ENOENT;
+
+ mutex_lock(&efx->rss_lock);
+ ctx = efx_find_rss_context_entry(efx, rss_context);
+ if (!ctx) {
+ rc = -ENOENT;
+ goto out_unlock;
+ }
rc = efx->type->rx_pull_rss_context_config(efx, ctx);
if (rc)
- return rc;
+ goto out_unlock;
if (hfunc)
*hfunc = ETH_RSS_HASH_TOP;
@@ -1383,7 +1391,9 @@ static int efx_ethtool_get_rxfh_context(struct net_device *net_dev, u32 *indir,
memcpy(indir, ctx->rx_indir_table, sizeof(ctx->rx_indir_table));
if (key)
memcpy(key, ctx->rx_hash_key, efx->type->rx_hash_key_size);
- return 0;
+out_unlock:
+ mutex_unlock(&efx->rss_lock);
+ return rc;
}
static int efx_ethtool_set_rxfh_context(struct net_device *net_dev,
@@ -1401,23 +1411,31 @@ static int efx_ethtool_set_rxfh_context(struct net_device *net_dev,
/* Hash function is Toeplitz, cannot be changed */
if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP)
return -EOPNOTSUPP;
+
+ mutex_lock(&efx->rss_lock);
+
if (*rss_context == ETH_RXFH_CONTEXT_ALLOC) {
- if (delete)
+ if (delete) {
/* alloc + delete == Nothing to do */
- return -EINVAL;
- ctx = efx_alloc_rss_context_entry(&efx->rss_context.list);
- if (!ctx)
- return -ENOMEM;
+ rc = -EINVAL;
+ goto out_unlock;
+ }
+ ctx = efx_alloc_rss_context_entry(efx);
+ if (!ctx) {
+ rc = -ENOMEM;
+ goto out_unlock;
+ }
ctx->context_id = EFX_EF10_RSS_CONTEXT_INVALID;
/* Initialise indir table and key to defaults */
efx_set_default_rx_indir_table(efx, ctx);
netdev_rss_key_fill(ctx->rx_hash_key, sizeof(ctx->rx_hash_key));
allocated = true;
} else {
- ctx = efx_find_rss_context_entry(*rss_context,
- &efx->rss_context.list);
- if (!ctx)
- return -ENOENT;
+ ctx = efx_find_rss_context_entry(efx, *rss_context);
+ if (!ctx) {
+ rc = -ENOENT;
+ goto out_unlock;
+ }
}
if (delete) {
@@ -1425,7 +1443,7 @@ static int efx_ethtool_set_rxfh_context(struct net_device *net_dev,
rc = efx->type->rx_push_rss_context_config(efx, ctx, NULL, NULL);
if (!rc)
efx_free_rss_context_entry(ctx);
- return rc;
+ goto out_unlock;
}
if (!key)
@@ -1438,6 +1456,8 @@ static int efx_ethtool_set_rxfh_context(struct net_device *net_dev,
efx_free_rss_context_entry(ctx);
else
*rss_context = ctx->user_id;
+out_unlock:
+ mutex_unlock(&efx->rss_lock);
return rc;
}
diff --git a/drivers/net/ethernet/sfc/net_driver.h b/drivers/net/ethernet/sfc/net_driver.h
index ef6fc931afb8..15edf98114dd 100644
--- a/drivers/net/ethernet/sfc/net_driver.h
+++ b/drivers/net/ethernet/sfc/net_driver.h
@@ -791,6 +791,7 @@ struct efx_rss_context {
* @rx_scatter: Scatter mode enabled for receives
* @rss_context: Main RSS context. Its @list member is the head of the list of
* RSS contexts created by user requests
+ * @rss_lock: Protects custom RSS context software state in @rss_context.list
* @int_error_count: Number of internal errors seen recently
* @int_error_expire: Time at which error count will be expired
* @irq_soft_enabled: Are IRQs soft-enabled? If not, IRQ handler will
@@ -933,6 +934,7 @@ struct efx_nic {
int rx_packet_ts_offset;
bool rx_scatter;
struct efx_rss_context rss_context;
+ struct mutex rss_lock;
unsigned int_error_count;
unsigned long int_error_expire;
diff --git a/drivers/net/ethernet/sfc/nic.h b/drivers/net/ethernet/sfc/nic.h
index ac59afad6e5c..5640034bda10 100644
--- a/drivers/net/ethernet/sfc/nic.h
+++ b/drivers/net/ethernet/sfc/nic.h
@@ -365,6 +365,8 @@ enum {
* @vi_base: Absolute index of first VI in this function
* @n_allocated_vis: Number of VIs allocated to this function
* @must_realloc_vis: Flag: VIs have yet to be reallocated after MC reboot
+ * @must_restore_rss_contexts: Flag: RSS contexts have yet to be restored after
+ * MC reboot
* @must_restore_filters: Flag: filters have yet to be restored after MC reboot
* @n_piobufs: Number of PIO buffers allocated to this function
* @wc_membase: Base address of write-combining mapping of the memory BAR
@@ -407,6 +409,7 @@ struct efx_ef10_nic_data {
unsigned int vi_base;
unsigned int n_allocated_vis;
bool must_realloc_vis;
+ bool must_restore_rss_contexts;
bool must_restore_filters;
unsigned int n_piobufs;
void __iomem *wc_membase, *pio_write_base;
^ permalink raw reply related
* [PATCH net-next 6/6] sfc: fix flow type handling for RSS filters
From: Edward Cree @ 2018-03-27 16:44 UTC (permalink / raw)
To: linux-net-drivers, David Miller; +Cc: netdev
In-Reply-To: <7a932be7-0335-a7f3-4fb4-408ccb191737@solarflare.com>
The FLOW_RSS flag was causing us to insert UDP filters when TCP was wanted.
Fixes: 42356d9a137b ("sfc: support RSS spreading of ethtool ntuple filters")
Signed-off-by: Edward Cree <ecree@solarflare.com>
---
drivers/net/ethernet/sfc/ethtool.c | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/drivers/net/ethernet/sfc/ethtool.c b/drivers/net/ethernet/sfc/ethtool.c
index de1fac6cd34f..d3ee9df6f4f0 100644
--- a/drivers/net/ethernet/sfc/ethtool.c
+++ b/drivers/net/ethernet/sfc/ethtool.c
@@ -1088,6 +1088,7 @@ static int efx_ethtool_set_class_rule(struct efx_nic *efx,
struct ethtool_tcpip6_spec *ip6_mask = &rule->m_u.tcp_ip6_spec;
struct ethtool_usrip6_spec *uip6_entry = &rule->h_u.usr_ip6_spec;
struct ethtool_usrip6_spec *uip6_mask = &rule->m_u.usr_ip6_spec;
+ u32 flow_type = rule->flow_type & ~(FLOW_EXT | FLOW_RSS);
struct ethhdr *mac_entry = &rule->h_u.ether_spec;
struct ethhdr *mac_mask = &rule->m_u.ether_spec;
enum efx_filter_flags flags = 0;
@@ -1121,14 +1122,14 @@ static int efx_ethtool_set_class_rule(struct efx_nic *efx,
if (rule->flow_type & FLOW_RSS)
spec.rss_context = rss_context;
- switch (rule->flow_type & ~(FLOW_EXT | FLOW_RSS)) {
+ switch (flow_type) {
case TCP_V4_FLOW:
case UDP_V4_FLOW:
spec.match_flags = (EFX_FILTER_MATCH_ETHER_TYPE |
EFX_FILTER_MATCH_IP_PROTO);
spec.ether_type = htons(ETH_P_IP);
- spec.ip_proto = ((rule->flow_type & ~FLOW_EXT) == TCP_V4_FLOW ?
- IPPROTO_TCP : IPPROTO_UDP);
+ spec.ip_proto = flow_type == TCP_V4_FLOW ? IPPROTO_TCP
+ : IPPROTO_UDP;
if (ip_mask->ip4dst) {
if (ip_mask->ip4dst != IP4_ADDR_FULL_MASK)
return -EINVAL;
@@ -1162,8 +1163,8 @@ static int efx_ethtool_set_class_rule(struct efx_nic *efx,
spec.match_flags = (EFX_FILTER_MATCH_ETHER_TYPE |
EFX_FILTER_MATCH_IP_PROTO);
spec.ether_type = htons(ETH_P_IPV6);
- spec.ip_proto = ((rule->flow_type & ~FLOW_EXT) == TCP_V6_FLOW ?
- IPPROTO_TCP : IPPROTO_UDP);
+ spec.ip_proto = flow_type == TCP_V6_FLOW ? IPPROTO_TCP
+ : IPPROTO_UDP;
if (!ip6_mask_is_empty(ip6_mask->ip6dst)) {
if (!ip6_mask_is_full(ip6_mask->ip6dst))
return -EINVAL;
^ permalink raw reply related
* Re: [PATCH v7 0/7] netdev: intel: Eliminate duplicate barriers on weakly-ordered archs
From: Jeff Kirsher @ 2018-03-27 16:54 UTC (permalink / raw)
To: Sinan Kaya
Cc: sulrich, Netdev, Timur Tabi, Alexander Duyck, intel-wired-lan,
linux-arm-msm, linux-arm-kernel
In-Reply-To: <e3ada376-52a5-573b-33f1-9aa84af75f0d@codeaurora.org>
[-- Attachment #1.1: Type: text/plain, Size: 3057 bytes --]
On Tue, 2018-03-27 at 08:42 -0400, Sinan Kaya wrote:
> On 3/23/2018 10:34 PM, okaya@codeaurora.org wrote:
> > On 2018-03-23 19:58, Jeff Kirsher wrote:
> > > On Fri, 2018-03-23 at 14:53 -0700, Alexander Duyck wrote:
> > > > On Fri, Mar 23, 2018 at 11:52 AM, Sinan Kaya <okaya@codeaurora.
> > > > org>
> > > > wrote:
> > > > > Code includes wmb() followed by writel() in multiple places.
> > > > > writel()
> > > > > already has a barrier on some architectures like arm64.
> > > > >
> > > > > This ends up CPU observing two barriers back to back before
> > > > > executing
> > > > > the
> > > > > register write.
> > > > >
> > > > > Since code already has an explicit barrier call, changing
> > > > > writel() to
> > > > > writel_relaxed().
> > > > >
> > > > > I did a regex search for wmb() followed by writel() in each
> > > > > drivers
> > > > > directory.
> > > > > I scrubbed the ones I care about in this series.
> > > > >
> > > > > I considered "ease of change", "popular usage" and
> > > > > "performance
> > > > > critical
> > > > > path" as the determining criteria for my filtering.
> > > > >
> > > > > We used relaxed API heavily on ARM for a long time but
> > > > > it did not exist on other architectures. For this reason,
> > > > > relaxed
> > > > > architectures have been paying double penalty in order to use
> > > > > the
> > > > > common
> > > > > drivers.
> > > > >
> > > > > Now that relaxed API is present on all architectures, we can
> > > > > go and
> > > > > scrub
> > > > > all drivers to see what needs to change and what can remain.
> > > > >
> > > > > We start with mostly used ones and hope to increase the
> > > > > coverage over
> > > > > time.
> > > > > It will take a while to cover all drivers.
> > > > >
> > > > > Feel free to apply patches individually.
> > > >
> > > > I looked over the set and they seem good.
> > > >
> > > > Reviewed-by: Alexander Duyck <alexander.h.duyck@intel.com>
> > >
> > > Grrr, patch 1 does not apply cleanly to my next-queue tree (dev-
> > > queue
> > > branch). I will deal with this series in a day or two, after I
> > > have dealt
> > > with my driver pull requests.
> >
> > Sorry, you will have to replace the ones you took from me.
>
> Double sorry now.
>
> I don't know if you have been following "RFC on writel and
> writel_relaxed" thread
> or not but there are some new developments about wmb() requirement.
>
> Basically, wmb() should never be used before writel() as writel()
> seem to
> provide coherency and observability guarantee.
>
> wmb()+writel_relaxed() is slower on some architectures than plain
> writel()
>
> I'll have to rework these patches to have writel() only.
>
> Are you able to drop the applied ones so that I can post V8 or is it
> too late?
Currently I do not have any of your patches applied to my next-queue
tree (dev-queue branch). So feel free to do any revisions you need to
do and to re-submit to intel-wired-lan@lists.osuosl.org (IWL) mailing
list.
[-- Attachment #1.2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
[-- Attachment #2: Type: text/plain, Size: 176 bytes --]
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply
* [RFC PATCH v2 00/14] Introducing AF_XDP support
From: Björn Töpel @ 2018-03-27 16:59 UTC (permalink / raw)
To: bjorn.topel, magnus.karlsson, alexander.h.duyck, alexander.duyck,
john.fastabend, ast, brouer, willemdebruijn.kernel, daniel,
netdev
Cc: Björn Töpel, michael.lundkvist, jesse.brandeburg,
anjali.singhai, qi.z.zhang, ravineet.singh
From: Björn Töpel <bjorn.topel@intel.com>
This RFC introduces a new address family called AF_XDP that is
optimized for high performance packet processing and, in upcoming
patch sets, zero-copy semantics. In this v2 version, we have removed
all zero-copy related code in order to make it smaller, simpler and
hopefully more review friendly. This RFC only supports copy-mode for
the generic XDP path (XDP_SKB) for both RX and TX and copy-mode for RX
using the XDP_DRV path. Zero-copy support requires XDP and driver
changes that Jesper Dangaard Brouer is working on. Some of his work is
already on the mailing list for review. We will publish our zero-copy
support for RX and TX on top of his patch sets at a later point in
time.
An AF_XDP socket (XSK) is created with the normal socket()
syscall. Associated with each XSK are two queues: the RX queue and the
TX queue. A socket can receive packets on the RX queue and it can send
packets on the TX queue. These queues are registered and sized with
the setsockopts XDP_RX_QUEUE and XDP_TX_QUEUE, respectively. It is
mandatory to have at least one of these queues for each socket. In
contrast to AF_PACKET V2/V3 these descriptor queues are separated from
packet buffers. An RX or TX descriptor points to a data buffer in a
memory area called a UMEM. RX and TX can share the same UMEM so that a
packet does not have to be copied between RX and TX. Moreover, if a
packet needs to be kept for a while due to a possible retransmit, the
descriptor that points to that packet can be changed to point to
another and reused right away. This again avoids copying data.
This new dedicated packet buffer area is called a UMEM. It consists of
a number of equally size frames and each frame has a unique frame
id. A descriptor in one of the queues references a frame by
referencing its frame id. The user space allocates memory for this
UMEM using whatever means it feels is most appropriate (malloc, mmap,
huge pages, etc). This memory area is then registered with the kernel
using the new setsockopt XDP_UMEM_REG. The UMEM also has two queues:
the FILL queue and the COMPLETION queue. The fill queue is used by the
application to send down frame ids for the kernel to fill in with RX
packet data. References to these frames will then appear in the RX
queue of the XSK once they have been received. The completion queue,
on the other hand, contains frame ids that the kernel has transmitted
completely and can now be used again by user space, for either TX or
RX. Thus, the frame ids appearing in the completion queue are ids that
were previously transmitted using the TX queue. In summary, the RX and
FILL queues are used for the RX path and the TX and COMPLETION queues
are used for the TX path.
The socket is then finally bound with a bind() call to a device and a
specific queue id on that device, and it is not until bind is
completed that traffic starts to flow. Note that in this RFC, all
packet data is copied out to user-space.
A new feature in this RFC is that the UMEM can be shared between
processes, if desired. If a process wants to do this, it simply skips
the registration of the UMEM and its corresponding two queues, sets a
flag in the bind call and submits the XSK of the process it would like
to share UMEM with as well as its own newly created XSK socket. The
new process will then receive frame id references in its own RX queue
that point to this shared UMEM. Note that since the queue structures
are single-consumer / single-producer (for performance reasons), the
new process has to create its own socket with associated RX and TX
queues, since it cannot share this with the other process. This is
also the reason that there is only one set of FILL and COMPLETION
queues per UMEM. It is the responsibility of a single process to
handle the UMEM. If multiple-producer / multiple-consumer queues are
implemented in the future, this requirement could be relaxed.
How is then packets distributed between these two XSK? We have
introduced a new BPF map called XSKMAP (or BPF_MAP_TYPE_XSKMAP in
full). The user-space application can place an XSK at an arbitrary
place in this map. The XDP program can then redirect a packet to a
specific index in this map and at this point XDP validates that the
XSK in that map was indeed bound to that device and queue number. If
not, the packet is dropped. If the map is empty at that index, the
packet is also dropped. This also means that it is currently mandatory
to have an XDP program loaded (and one XSK in the XSKMAP) to be able
to get any traffic to user space through the XSK.
AF_XDP can operate in two different modes: XDP_SKB and XDP_DRV. If the
driver does not have support for XDP, or XDP_SKB is explicitly chosen
when loading the XDP program, XDP_SKB mode is employed that uses SKBs
together with the generic XDP support and copies out the data to user
space. A fallback mode that works for any network device. On the other
hand, if the driver has support for XDP, it will be used by the AF_XDP
code to provide better performance, but there is still a copy of the
data into user space.
There is a xdpsock benchmarking/test application included that
demonstrates how to use AF_XDP sockets with both private and shared
UMEMs. Say that you would like your UDP traffic from port 4242 to end
up in queue 16, that we will enable AF_XDP on. Here, we use ethtool
for this:
ethtool -N p3p2 rx-flow-hash udp4 fn
ethtool -N p3p2 flow-type udp4 src-port 4242 dst-port 4242 \
action 16
Running the rxdrop benchmark in XDP_DRV mode can then be done
using:
samples/bpf/xdpsock -i p3p2 -q 16 -r -N
For XDP_SKB mode, use the switch "-S" instead of "-N" and all options
can be displayed with "-h", as usual.
We have run some benchmarks on a dual socket system with two Broadwell
E5 2660 @ 2.0 GHz with hyperthreading turned off. Each socket has 14
cores which gives a total of 28, but only two cores are used in these
experiments. One for TR/RX and one for the user space application. The
memory is DDR4 @ 2133 MT/s (1067 MHz) and the size of each DIMM is
8192MB and with 8 of those DIMMs in the system we have 64 GB of total
memory. The compiler used is gcc version 5.4.0 20160609. The NIC is an
Intel I40E 40Gbit/s using the i40e driver.
Below are the results in Mpps of the I40E NIC benchmark runs for 64
and 1500 byte packets, generated by commercial packet generator HW that is
generating packets at full 40 Gbit/s line rate.
AF_XDP performance 64 byte packets:
Benchmark XDP_SKB XDP_DRV
rxdrop 3.0 9.3
l2fwd 1.7 NA*
AF_XDP performance 1500 byte packets:
Benchmark XDP_SKB XDP_DRV
rxdrop 2.2 3.1
l2fwd 1.1 NA*
* NA since we have no support for TX using the XDP_DRV infrastructure
in this RFC. This is for a future patch set since it involves
changes to the XDP NDOs. Some of this is being worked on by Jesper
Dangaard Brouer in his latest patch set on the netdev mailing list.
XDP performance on our system as a base line:
64 byte packets:
XDP stats CPU pps issue-pps
XDP-RX CPU 16 30,961,380 0
1500 byte packets:
XDP stats CPU pps issue-pps
XDP-RX CPU 16 3,289,493 0
Changes in V2:
* Dropped zero-copy support
* Unidirectional queues for better performance and to allow for
non-symmetric consumer and producer use cases
* Shared packet buffers
* XSKMAP for directing packets with XDP
* Poll() support
* Rebind support
* Changed accounting to be per user as suggested by Willem de Bruijn
* Exterminated packet arrays from the code base
* Some bug fixes and code cleanup
The structure of the patch set is as follows:
Patches 1-2: Basic socket and umem plumbing
Patches 3-9: RX support together with the new XSKMAP
Patches 10-13: TX support
Patch 14: Sample application
We based this patch set on net-next commit: 5306653850b4 ("sctp:
remove unnecessary asoc in sctp_has_association")
For i40e support, this RFC depends on commit: d9314c474d4f ("i40e: add
support for XDP_REDIRECT")
For ixgbe support, this RFC depends on commit: ed93a3987128
("ixgbe: tweak page counting for XDP_REDIRECT")
The two Intel NIC patches above are in net-next as of today.
Questions:
* How to deal with cache alignment for uapi when different
architectures can have different cache line sizes? We have just
aligned it to 64 bytes for now, which works for many popular
architectures, but not all. Please advise.
To do:
* Optimize performance
* Kernel selftest
Post-series plan:
* Kernel load module support of AF_XDP would be nice. Unclear how to
achieve this though since our XDP code depends on net/core.
* Support for AF_XDP sockets without an XPD program loaded. In this
case all the traffic on a queue should go up to the user space
socket.
* Daniel Borkmann's suggestion for a "copy to XDP socket, and return
XDP_PASS" for a tcpdump-like functionality.
* And of course getting to zero-copy support in small increments.
Thanks: Björn and Magnus
Björn Töpel (7):
net: initial AF_XDP skeleton
xsk: add user memory registration support sockopt
xsk: add Rx queue setup and mmap support
xsk: add Rx receive functions and poll support
bpf: introduce new bpf AF_XDP map type BPF_MAP_TYPE_XSKMAP
xsk: wire up XDP_DRV side of AF_XDP
xsk: wire up XDP_SKB side of AF_XDP
Magnus Karlsson (7):
xsk: add umem fill queue support and mmap
xsk: add support for bind for Rx
xsk: add umem completion queue support and mmap
xsk: add Tx queue setup and mmap support
xsk: support for Tx
xsk: statistics support
samples/bpf: sample application for AF_XDP sockets
include/linux/bpf.h | 26 +
include/linux/bpf_types.h | 3 +
include/linux/filter.h | 2 +-
include/linux/socket.h | 5 +-
include/net/xdp_sock.h | 40 ++
include/uapi/linux/bpf.h | 1 +
include/uapi/linux/if_xdp.h | 86 ++++
kernel/bpf/Makefile | 3 +
kernel/bpf/verifier.c | 8 +-
kernel/bpf/xskmap.c | 281 +++++++++++
net/Kconfig | 1 +
net/Makefile | 1 +
net/core/dev.c | 28 +-
net/core/filter.c | 40 +-
net/core/sock.c | 12 +-
net/xdp/Kconfig | 7 +
net/xdp/Makefile | 2 +
net/xdp/xdp_umem.c | 259 ++++++++++
net/xdp/xdp_umem.h | 64 +++
net/xdp/xdp_umem_props.h | 23 +
net/xdp/xsk.c | 710 +++++++++++++++++++++++++++
net/xdp/xsk_queue.c | 59 +++
net/xdp/xsk_queue.h | 297 ++++++++++++
samples/bpf/Makefile | 4 +
samples/bpf/xdpsock.h | 10 +
samples/bpf/xdpsock_kern.c | 55 +++
samples/bpf/xdpsock_user.c | 923 ++++++++++++++++++++++++++++++++++++
security/selinux/hooks.c | 4 +-
security/selinux/include/classmap.h | 4 +-
29 files changed, 2929 insertions(+), 29 deletions(-)
create mode 100644 include/net/xdp_sock.h
create mode 100644 include/uapi/linux/if_xdp.h
create mode 100644 kernel/bpf/xskmap.c
create mode 100644 net/xdp/Kconfig
create mode 100644 net/xdp/Makefile
create mode 100644 net/xdp/xdp_umem.c
create mode 100644 net/xdp/xdp_umem.h
create mode 100644 net/xdp/xdp_umem_props.h
create mode 100644 net/xdp/xsk.c
create mode 100644 net/xdp/xsk_queue.c
create mode 100644 net/xdp/xsk_queue.h
create mode 100644 samples/bpf/xdpsock.h
create mode 100644 samples/bpf/xdpsock_kern.c
create mode 100644 samples/bpf/xdpsock_user.c
--
2.14.1
^ permalink raw reply
* [RFC PATCH v2 01/14] net: initial AF_XDP skeleton
From: Björn Töpel @ 2018-03-27 16:59 UTC (permalink / raw)
To: bjorn.topel, magnus.karlsson, alexander.h.duyck, alexander.duyck,
john.fastabend, ast, brouer, willemdebruijn.kernel, daniel,
netdev
Cc: Björn Töpel, michael.lundkvist, jesse.brandeburg,
anjali.singhai, qi.z.zhang, ravineet.singh
In-Reply-To: <20180327165919.17933-1-bjorn.topel@gmail.com>
From: Björn Töpel <bjorn.topel@intel.com>
Buildable skeleton of AF_XDP without any functionality. Just what it
takes to register a new address family.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
---
include/linux/socket.h | 5 ++++-
net/Kconfig | 1 +
net/core/sock.c | 12 ++++++++----
net/xdp/Kconfig | 7 +++++++
security/selinux/hooks.c | 4 +++-
security/selinux/include/classmap.h | 4 +++-
6 files changed, 26 insertions(+), 7 deletions(-)
create mode 100644 net/xdp/Kconfig
diff --git a/include/linux/socket.h b/include/linux/socket.h
index 60e01482a9c4..141e527b8615 100644
--- a/include/linux/socket.h
+++ b/include/linux/socket.h
@@ -207,8 +207,9 @@ struct ucred {
* PF_SMC protocol family that
* reuses AF_INET address family
*/
+#define AF_XDP 44 /* XDP sockets */
-#define AF_MAX 44 /* For now.. */
+#define AF_MAX 45 /* For now.. */
/* Protocol families, same as address families. */
#define PF_UNSPEC AF_UNSPEC
@@ -257,6 +258,7 @@ struct ucred {
#define PF_KCM AF_KCM
#define PF_QIPCRTR AF_QIPCRTR
#define PF_SMC AF_SMC
+#define PF_XDP AF_XDP
#define PF_MAX AF_MAX
/* Maximum queue length specifiable by listen. */
@@ -338,6 +340,7 @@ struct ucred {
#define SOL_NFC 280
#define SOL_KCM 281
#define SOL_TLS 282
+#define SOL_XDP 283
/* IPX options */
#define IPX_TYPE 1
diff --git a/net/Kconfig b/net/Kconfig
index 0428f12c25c2..308f1a0d265a 100644
--- a/net/Kconfig
+++ b/net/Kconfig
@@ -59,6 +59,7 @@ source "net/tls/Kconfig"
source "net/xfrm/Kconfig"
source "net/iucv/Kconfig"
source "net/smc/Kconfig"
+source "net/xdp/Kconfig"
config INET
bool "TCP/IP networking"
diff --git a/net/core/sock.c b/net/core/sock.c
index 8cee2920a47f..89e692f62c86 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -226,7 +226,8 @@ static struct lock_class_key af_family_kern_slock_keys[AF_MAX];
x "AF_RXRPC" , x "AF_ISDN" , x "AF_PHONET" , \
x "AF_IEEE802154", x "AF_CAIF" , x "AF_ALG" , \
x "AF_NFC" , x "AF_VSOCK" , x "AF_KCM" , \
- x "AF_QIPCRTR", x "AF_SMC" , x "AF_MAX"
+ x "AF_QIPCRTR", x "AF_SMC" , x "AF_XDP" , \
+ x "AF_MAX"
static const char *const af_family_key_strings[AF_MAX+1] = {
_sock_locks("sk_lock-")
@@ -262,7 +263,8 @@ static const char *const af_family_rlock_key_strings[AF_MAX+1] = {
"rlock-AF_RXRPC" , "rlock-AF_ISDN" , "rlock-AF_PHONET" ,
"rlock-AF_IEEE802154", "rlock-AF_CAIF" , "rlock-AF_ALG" ,
"rlock-AF_NFC" , "rlock-AF_VSOCK" , "rlock-AF_KCM" ,
- "rlock-AF_QIPCRTR", "rlock-AF_SMC" , "rlock-AF_MAX"
+ "rlock-AF_QIPCRTR", "rlock-AF_SMC" , "rlock-AF_XDP" ,
+ "rlock-AF_MAX"
};
static const char *const af_family_wlock_key_strings[AF_MAX+1] = {
"wlock-AF_UNSPEC", "wlock-AF_UNIX" , "wlock-AF_INET" ,
@@ -279,7 +281,8 @@ static const char *const af_family_wlock_key_strings[AF_MAX+1] = {
"wlock-AF_RXRPC" , "wlock-AF_ISDN" , "wlock-AF_PHONET" ,
"wlock-AF_IEEE802154", "wlock-AF_CAIF" , "wlock-AF_ALG" ,
"wlock-AF_NFC" , "wlock-AF_VSOCK" , "wlock-AF_KCM" ,
- "wlock-AF_QIPCRTR", "wlock-AF_SMC" , "wlock-AF_MAX"
+ "wlock-AF_QIPCRTR", "wlock-AF_SMC" , "wlock-AF_XDP" ,
+ "wlock-AF_MAX"
};
static const char *const af_family_elock_key_strings[AF_MAX+1] = {
"elock-AF_UNSPEC", "elock-AF_UNIX" , "elock-AF_INET" ,
@@ -296,7 +299,8 @@ static const char *const af_family_elock_key_strings[AF_MAX+1] = {
"elock-AF_RXRPC" , "elock-AF_ISDN" , "elock-AF_PHONET" ,
"elock-AF_IEEE802154", "elock-AF_CAIF" , "elock-AF_ALG" ,
"elock-AF_NFC" , "elock-AF_VSOCK" , "elock-AF_KCM" ,
- "elock-AF_QIPCRTR", "elock-AF_SMC" , "elock-AF_MAX"
+ "elock-AF_QIPCRTR", "elock-AF_SMC" , "elock-AF_XDP" ,
+ "elock-AF_MAX"
};
/*
diff --git a/net/xdp/Kconfig b/net/xdp/Kconfig
new file mode 100644
index 000000000000..90e4a7152854
--- /dev/null
+++ b/net/xdp/Kconfig
@@ -0,0 +1,7 @@
+config XDP_SOCKETS
+ bool "XDP sockets"
+ depends on BPF_SYSCALL
+ default n
+ help
+ XDP sockets allows a channel between XDP programs and
+ userspace applications.
diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c
index b4d7b6242a40..5082dd6e613f 100644
--- a/security/selinux/hooks.c
+++ b/security/selinux/hooks.c
@@ -1436,7 +1436,9 @@ static inline u16 socket_type_to_security_class(int family, int type, int protoc
return SECCLASS_QIPCRTR_SOCKET;
case PF_SMC:
return SECCLASS_SMC_SOCKET;
-#if PF_MAX > 44
+ case PF_XDP:
+ return SECCLASS_XDP_SOCKET;
+#if PF_MAX > 45
#error New address family defined, please update this function.
#endif
}
diff --git a/security/selinux/include/classmap.h b/security/selinux/include/classmap.h
index acdee7795297..e2044cd358bb 100644
--- a/security/selinux/include/classmap.h
+++ b/security/selinux/include/classmap.h
@@ -240,9 +240,11 @@ struct security_class_mapping secclass_map[] = {
{ "manage_subnet", NULL } },
{ "bpf",
{"map_create", "map_read", "map_write", "prog_load", "prog_run"} },
+ { "xdp_socket",
+ { COMMON_SOCK_PERMS, NULL } },
{ NULL }
};
-#if PF_MAX > 44
+#if PF_MAX > 45
#error New address family defined, please update secclass_map.
#endif
--
2.14.1
^ permalink raw reply related
* [RFC PATCH v2 02/14] xsk: add user memory registration support sockopt
From: Björn Töpel @ 2018-03-27 16:59 UTC (permalink / raw)
To: bjorn.topel, magnus.karlsson, alexander.h.duyck, alexander.duyck,
john.fastabend, ast, brouer, willemdebruijn.kernel, daniel,
netdev
Cc: Björn Töpel, michael.lundkvist, jesse.brandeburg,
anjali.singhai, qi.z.zhang, ravineet.singh
In-Reply-To: <20180327165919.17933-1-bjorn.topel@gmail.com>
From: Björn Töpel <bjorn.topel@intel.com>
In this commit the base structure of the AF_XDP address family is set
up. Further, we introduce the abilty register a window of user memory
to the kernel via the XDP_UMEM_REG setsockopt syscall. The memory
window is viewed by an AF_XDP socket as a set of equally large
frames. After a user memory registration all frames are "owned" by the
user application, and not the kernel.
Co-authored-by: Magnus Karlsson <magnus.karlsson@intel.com>
Signed-off-by: Magnus Karlsson <magnus.karlsson@intel.com>
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
---
include/uapi/linux/if_xdp.h | 33 ++++++
net/Makefile | 1 +
net/xdp/Makefile | 2 +
net/xdp/xdp_umem.c | 240 ++++++++++++++++++++++++++++++++++++++++++++
net/xdp/xdp_umem.h | 41 ++++++++
net/xdp/xdp_umem_props.h | 23 +++++
net/xdp/xsk.c | 223 ++++++++++++++++++++++++++++++++++++++++
7 files changed, 563 insertions(+)
create mode 100644 include/uapi/linux/if_xdp.h
create mode 100644 net/xdp/Makefile
create mode 100644 net/xdp/xdp_umem.c
create mode 100644 net/xdp/xdp_umem.h
create mode 100644 net/xdp/xdp_umem_props.h
create mode 100644 net/xdp/xsk.c
diff --git a/include/uapi/linux/if_xdp.h b/include/uapi/linux/if_xdp.h
new file mode 100644
index 000000000000..69ccb3b0a3f2
--- /dev/null
+++ b/include/uapi/linux/if_xdp.h
@@ -0,0 +1,33 @@
+/*
+ * if_xdp: XDP socket user-space interface
+ * Copyright(c) 2018 Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * Author(s): Björn Töpel <bjorn.topel@intel.com>
+ * Magnus Karlsson <magnus.karlsson@intel.com>
+ */
+
+#ifndef _LINUX_IF_XDP_H
+#define _LINUX_IF_XDP_H
+
+#include <linux/types.h>
+
+/* XDP socket options */
+#define XDP_UMEM_REG 3
+
+struct xdp_umem_reg {
+ __u64 addr; /* Start of packet data area */
+ __u64 len; /* Length of packet data area */
+ __u32 frame_size; /* Frame size */
+ __u32 frame_headroom; /* Frame head room */
+};
+
+#endif /* _LINUX_IF_XDP_H */
diff --git a/net/Makefile b/net/Makefile
index a6147c61b174..77aaddedbd29 100644
--- a/net/Makefile
+++ b/net/Makefile
@@ -85,3 +85,4 @@ obj-y += l3mdev/
endif
obj-$(CONFIG_QRTR) += qrtr/
obj-$(CONFIG_NET_NCSI) += ncsi/
+obj-$(CONFIG_XDP_SOCKETS) += xdp/
diff --git a/net/xdp/Makefile b/net/xdp/Makefile
new file mode 100644
index 000000000000..a5d736640a0f
--- /dev/null
+++ b/net/xdp/Makefile
@@ -0,0 +1,2 @@
+obj-$(CONFIG_XDP_SOCKETS) += xsk.o xdp_umem.o
+
diff --git a/net/xdp/xdp_umem.c b/net/xdp/xdp_umem.c
new file mode 100644
index 000000000000..8f768a7887da
--- /dev/null
+++ b/net/xdp/xdp_umem.c
@@ -0,0 +1,240 @@
+/*
+ * XDP user-space packet buffer
+ * Copyright(c) 2018 Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ */
+
+#include <linux/init.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/task.h>
+#include <linux/uaccess.h>
+#include <linux/slab.h>
+#include <linux/bpf.h>
+#include <linux/mm.h>
+
+#include "xdp_umem.h"
+
+#define XDP_UMEM_MIN_FRAME_SIZE 2048
+
+int xdp_umem_create(struct xdp_umem **umem)
+{
+ *umem = kzalloc(sizeof(**umem), GFP_KERNEL);
+
+ if (!(*umem))
+ return -ENOMEM;
+
+ return 0;
+}
+
+static void xdp_umem_unpin_pages(struct xdp_umem *umem)
+{
+ unsigned int i;
+
+ if (umem->pgs) {
+ for (i = 0; i < umem->npgs; i++) {
+ struct page *page = umem->pgs[i];
+
+ set_page_dirty_lock(page);
+ put_page(page);
+ }
+
+ kfree(umem->pgs);
+ umem->pgs = NULL;
+ }
+}
+
+static void xdp_umem_unaccount_pages(struct xdp_umem *umem)
+{
+ if (umem->user) {
+ atomic_long_sub(umem->npgs, &umem->user->locked_vm);
+ free_uid(umem->user);
+ }
+}
+
+static void xdp_umem_release(struct xdp_umem *umem)
+{
+ struct task_struct *task;
+ struct mm_struct *mm;
+ unsigned long diff;
+
+ if (umem->pgs) {
+ xdp_umem_unpin_pages(umem);
+
+ task = get_pid_task(umem->pid, PIDTYPE_PID);
+ put_pid(umem->pid);
+ if (!task)
+ goto out;
+ mm = get_task_mm(task);
+ put_task_struct(task);
+ if (!mm)
+ goto out;
+
+ diff = umem->size >> PAGE_SHIFT;
+
+ down_write(&mm->mmap_sem);
+ mm->pinned_vm -= diff;
+ up_write(&mm->mmap_sem);
+ mmput(mm);
+ umem->pgs = NULL;
+ }
+
+ xdp_umem_unaccount_pages(umem);
+out:
+ kfree(umem);
+}
+
+void xdp_put_umem(struct xdp_umem *umem)
+{
+ if (!umem)
+ return;
+
+ if (atomic_dec_and_test(&umem->users))
+ xdp_umem_release(umem);
+}
+
+static int xdp_umem_pin_pages(struct xdp_umem *umem)
+{
+ unsigned int gup_flags = FOLL_WRITE;
+ long npgs;
+ int err;
+
+ umem->pgs = kcalloc(umem->npgs, sizeof(*umem->pgs), GFP_KERNEL);
+ if (!umem->pgs)
+ return -ENOMEM;
+
+ npgs = get_user_pages(umem->address, umem->npgs,
+ gup_flags, &umem->pgs[0], NULL);
+ if (npgs != umem->npgs) {
+ if (npgs >= 0) {
+ umem->npgs = npgs;
+ err = -ENOMEM;
+ goto out_pin;
+ }
+ err = npgs;
+ goto out_pgs;
+ }
+ return 0;
+
+out_pin:
+ xdp_umem_unpin_pages(umem);
+out_pgs:
+ kfree(umem->pgs);
+ umem->pgs = NULL;
+ return err;
+}
+
+static int xdp_umem_account_pages(struct xdp_umem *umem)
+{
+ unsigned long lock_limit, new_npgs, old_npgs;
+
+ if (capable(CAP_IPC_LOCK))
+ return 0;
+
+ lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
+ umem->user = get_uid(current_user());
+
+ do {
+ old_npgs = atomic_long_read(&umem->user->locked_vm);
+ new_npgs = old_npgs + umem->npgs;
+ if (new_npgs > lock_limit) {
+ free_uid(umem->user);
+ umem->user = NULL;
+ return -ENOBUFS;
+ }
+ } while (atomic_long_cmpxchg(&umem->user->locked_vm, old_npgs,
+ new_npgs) != old_npgs);
+ return 0;
+}
+
+static int __xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr)
+{
+ u32 frame_size = mr->frame_size, frame_headroom = mr->frame_headroom;
+ u64 addr = mr->addr, size = mr->len;
+ unsigned int nframes;
+ int size_chk, err;
+
+ if (frame_size < XDP_UMEM_MIN_FRAME_SIZE || frame_size > PAGE_SIZE) {
+ /* Strictly speaking we could support this, if:
+ * - huge pages, or*
+ * - using an IOMMU, or
+ * - making sure the memory area is consecutive
+ * but for now, we simply say "computer says no".
+ */
+ return -EINVAL;
+ }
+
+ if (!is_power_of_2(frame_size))
+ return -EINVAL;
+
+ if (!PAGE_ALIGNED(addr)) {
+ /* Memory area has to be page size aligned. For
+ * simplicity, this might change.
+ */
+ return -EINVAL;
+ }
+
+ if ((addr + size) < addr)
+ return -EINVAL;
+
+ nframes = size / frame_size;
+ if (nframes == 0 || nframes > UINT_MAX)
+ return -EINVAL;
+
+ frame_headroom = ALIGN(frame_headroom, 64);
+
+ size_chk = frame_size - frame_headroom - XDP_PACKET_HEADROOM;
+ if (size_chk < 0)
+ return -EINVAL;
+
+ umem->pid = get_task_pid(current, PIDTYPE_PID);
+ umem->size = (size_t)size;
+ umem->address = (unsigned long)addr;
+ umem->props.frame_size = frame_size;
+ umem->props.nframes = nframes;
+ umem->frame_headroom = frame_headroom;
+ umem->npgs = size / PAGE_SIZE;
+ umem->pgs = NULL;
+ umem->user = NULL;
+
+ umem->frame_size_log2 = ilog2(frame_size);
+ umem->nfpplog2 = ilog2(PAGE_SIZE / frame_size);
+ atomic_set(&umem->users, 1);
+
+ err = xdp_umem_account_pages(umem);
+ if (err)
+ goto out;
+
+ err = xdp_umem_pin_pages(umem);
+ if (err)
+ goto out;
+ return 0;
+
+out:
+ put_pid(umem->pid);
+ return err;
+}
+
+int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr)
+{
+ int err;
+
+ if (!umem)
+ return -EINVAL;
+
+ down_write(¤t->mm->mmap_sem);
+
+ err = __xdp_umem_reg(umem, mr);
+
+ up_write(¤t->mm->mmap_sem);
+ return err;
+}
+
diff --git a/net/xdp/xdp_umem.h b/net/xdp/xdp_umem.h
new file mode 100644
index 000000000000..d9bbbb880088
--- /dev/null
+++ b/net/xdp/xdp_umem.h
@@ -0,0 +1,41 @@
+/*
+ * XDP user-space packet buffer
+ * Copyright(c) 2018 Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ */
+
+#ifndef XDP_UMEM_H_
+#define XDP_UMEM_H_
+
+#include <linux/mm.h>
+#include <linux/if_xdp.h>
+
+#include "xdp_umem_props.h"
+
+struct xdp_umem {
+ struct pid *pid;
+ struct page **pgs;
+ unsigned long address;
+ size_t size;
+ struct xdp_umem_props props;
+ u32 npgs;
+ u32 frame_headroom;
+ u32 nfpplog2;
+ u32 frame_size_log2;
+ atomic_t users;
+ struct user_struct *user;
+};
+
+int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr);
+void xdp_put_umem(struct xdp_umem *umem);
+int xdp_umem_create(struct xdp_umem **umem);
+
+#endif /* XDP_UMEM_H_ */
diff --git a/net/xdp/xdp_umem_props.h b/net/xdp/xdp_umem_props.h
new file mode 100644
index 000000000000..a2dc91d34262
--- /dev/null
+++ b/net/xdp/xdp_umem_props.h
@@ -0,0 +1,23 @@
+/*
+ * XDP user-space packet buffer
+ * Copyright(c) 2018 Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ */
+
+#ifndef XDP_UMEM_PROPS_H_
+#define XDP_UMEM_PROPS_H_
+
+struct xdp_umem_props {
+ u32 frame_size;
+ u32 nframes;
+};
+
+#endif /* XDP_UMEM_PROPS_H_ */
diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c
new file mode 100644
index 000000000000..cc0f26e17baf
--- /dev/null
+++ b/net/xdp/xsk.c
@@ -0,0 +1,223 @@
+/*
+ * XDP sockets
+ *
+ * AF_XDP sockets allows a channel between XDP programs and userspace
+ * applications.
+ * Copyright(c) 2018 Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * Author(s): Björn Töpel <bjorn.topel@intel.com>
+ * Magnus Karlsson <magnus.karlsson@intel.com>
+ */
+
+#define pr_fmt(fmt) "AF_XDP: %s: " fmt, __func__
+
+#include <linux/if_xdp.h>
+#include <linux/init.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/task.h>
+#include <linux/socket.h>
+#include <linux/file.h>
+#include <linux/uaccess.h>
+#include <linux/net.h>
+#include <linux/netdevice.h>
+#include <net/sock.h>
+
+#include "xdp_umem.h"
+
+struct xdp_sock {
+ /* struct sock must be the first member of struct xdp_sock */
+ struct sock sk;
+ /* Protects multiple processes in the control path */
+ struct mutex mutex;
+ struct xdp_umem *umem;
+};
+
+static struct xdp_sock *xdp_sk(struct sock *sk)
+{
+ return (struct xdp_sock *)sk;
+}
+
+static int xsk_release(struct socket *sock)
+{
+ struct sock *sk = sock->sk;
+ struct net *net;
+
+ if (!sk)
+ return 0;
+
+ net = sock_net(sk);
+
+ local_bh_disable();
+ sock_prot_inuse_add(net, sk->sk_prot, -1);
+ local_bh_enable();
+
+ sock_orphan(sk);
+ sock->sk = NULL;
+
+ sk_refcnt_debug_release(sk);
+ sock_put(sk);
+
+ return 0;
+}
+
+static int xsk_setsockopt(struct socket *sock, int level, int optname,
+ char __user *optval, unsigned int optlen)
+{
+ struct sock *sk = sock->sk;
+ struct xdp_sock *xs = xdp_sk(sk);
+ int err;
+
+ if (level != SOL_XDP)
+ return -ENOPROTOOPT;
+
+ switch (optname) {
+ case XDP_UMEM_REG:
+ {
+ struct xdp_umem_reg mr;
+ struct xdp_umem *umem;
+
+ if (xs->umem)
+ return -EBUSY;
+
+ if (copy_from_user(&mr, optval, sizeof(mr)))
+ return -EFAULT;
+
+ mutex_lock(&xs->mutex);
+ err = xdp_umem_create(&umem);
+
+ err = xdp_umem_reg(umem, &mr);
+ if (err) {
+ kfree(umem);
+ mutex_unlock(&xs->mutex);
+ return err;
+ }
+
+ /* Make sure umem is ready before it can be seen by others */
+ smp_wmb();
+
+ xs->umem = umem;
+ mutex_unlock(&xs->mutex);
+ return 0;
+ }
+ default:
+ break;
+ }
+
+ return -ENOPROTOOPT;
+}
+
+static struct proto xsk_proto = {
+ .name = "XDP",
+ .owner = THIS_MODULE,
+ .obj_size = sizeof(struct xdp_sock),
+};
+
+static const struct proto_ops xsk_proto_ops = {
+ .family = PF_XDP,
+ .owner = THIS_MODULE,
+ .release = xsk_release,
+ .bind = sock_no_bind,
+ .connect = sock_no_connect,
+ .socketpair = sock_no_socketpair,
+ .accept = sock_no_accept,
+ .getname = sock_no_getname,
+ .poll = sock_no_poll,
+ .ioctl = sock_no_ioctl,
+ .listen = sock_no_listen,
+ .shutdown = sock_no_shutdown,
+ .setsockopt = xsk_setsockopt,
+ .getsockopt = sock_no_getsockopt,
+ .sendmsg = sock_no_sendmsg,
+ .recvmsg = sock_no_recvmsg,
+ .mmap = sock_no_mmap,
+ .sendpage = sock_no_sendpage,
+};
+
+static void xsk_destruct(struct sock *sk)
+{
+ struct xdp_sock *xs = xdp_sk(sk);
+
+ if (!sock_flag(sk, SOCK_DEAD))
+ return;
+
+ xdp_put_umem(xs->umem);
+
+ sk_refcnt_debug_dec(sk);
+}
+
+static int xsk_create(struct net *net, struct socket *sock, int protocol,
+ int kern)
+{
+ struct sock *sk;
+ struct xdp_sock *xs;
+
+ if (!ns_capable(net->user_ns, CAP_NET_RAW))
+ return -EPERM;
+ if (sock->type != SOCK_RAW)
+ return -ESOCKTNOSUPPORT;
+
+ if (protocol)
+ return -EPROTONOSUPPORT;
+
+ sock->state = SS_UNCONNECTED;
+
+ sk = sk_alloc(net, PF_XDP, GFP_KERNEL, &xsk_proto, kern);
+ if (!sk)
+ return -ENOBUFS;
+
+ sock->ops = &xsk_proto_ops;
+
+ sock_init_data(sock, sk);
+
+ sk->sk_family = PF_XDP;
+
+ sk->sk_destruct = xsk_destruct;
+ sk_refcnt_debug_inc(sk);
+
+ xs = xdp_sk(sk);
+ mutex_init(&xs->mutex);
+
+ local_bh_disable();
+ sock_prot_inuse_add(net, &xsk_proto, 1);
+ local_bh_enable();
+
+ return 0;
+}
+
+static const struct net_proto_family xsk_family_ops = {
+ .family = PF_XDP,
+ .create = xsk_create,
+ .owner = THIS_MODULE,
+};
+
+static int __init xsk_init(void)
+{
+ int err;
+
+ err = proto_register(&xsk_proto, 0 /* no slab */);
+ if (err)
+ goto out;
+
+ err = sock_register(&xsk_family_ops);
+ if (err)
+ goto out_proto;
+
+ return 0;
+
+out_proto:
+ proto_unregister(&xsk_proto);
+out:
+ return err;
+}
+
+fs_initcall(xsk_init);
--
2.14.1
^ permalink raw reply related
* [RFC PATCH v2 03/14] xsk: add umem fill queue support and mmap
From: Björn Töpel @ 2018-03-27 16:59 UTC (permalink / raw)
To: bjorn.topel, magnus.karlsson, alexander.h.duyck, alexander.duyck,
john.fastabend, ast, brouer, willemdebruijn.kernel, daniel,
netdev
Cc: michael.lundkvist, jesse.brandeburg, anjali.singhai, qi.z.zhang,
ravineet.singh
In-Reply-To: <20180327165919.17933-1-bjorn.topel@gmail.com>
From: Magnus Karlsson <magnus.karlsson@intel.com>
Here, we add another setsockopt for registered user memory (umem)
called XDP_UMEM_FILL_QUEUE. Using this socket option, the process can
ask the kernel to allocate a queue (ring buffer) and also mmap it
(XDP_UMEM_PGOFF_FILL_QUEUE) into the process.
The queue is used to explicitly pass ownership of umem frames from the
user process to the kernel. These frames will in a later patch be
filled in with Rx packet data by the kernel.
Signed-off-by: Magnus Karlsson <magnus.karlsson@intel.com>
---
include/uapi/linux/if_xdp.h | 15 +++++++++++
net/xdp/Makefile | 2 +-
net/xdp/xdp_umem.c | 5 ++++
net/xdp/xdp_umem.h | 2 ++
net/xdp/xsk.c | 65 ++++++++++++++++++++++++++++++++++++++++++++-
net/xdp/xsk_queue.c | 54 +++++++++++++++++++++++++++++++++++++
net/xdp/xsk_queue.h | 49 ++++++++++++++++++++++++++++++++++
7 files changed, 190 insertions(+), 2 deletions(-)
create mode 100644 net/xdp/xsk_queue.c
create mode 100644 net/xdp/xsk_queue.h
diff --git a/include/uapi/linux/if_xdp.h b/include/uapi/linux/if_xdp.h
index 69ccb3b0a3f2..0de1bbf2c5c7 100644
--- a/include/uapi/linux/if_xdp.h
+++ b/include/uapi/linux/if_xdp.h
@@ -22,6 +22,7 @@
/* XDP socket options */
#define XDP_UMEM_REG 3
+#define XDP_UMEM_FILL_QUEUE 4
struct xdp_umem_reg {
__u64 addr; /* Start of packet data area */
@@ -30,4 +31,18 @@ struct xdp_umem_reg {
__u32 frame_headroom; /* Frame head room */
};
+/* Pgoff for mmaping the rings */
+#define XDP_UMEM_PGOFF_FILL_QUEUE 0x100000000
+
+struct xdp_queue {
+ __u32 head_idx __attribute__((aligned(64)));
+ __u32 tail_idx __attribute__((aligned(64)));
+};
+
+/* Used for the fill and completion queues for buffers */
+struct xdp_umem_queue {
+ struct xdp_queue ptrs;
+ __u32 desc[0] __attribute__((aligned(64)));
+};
+
#endif /* _LINUX_IF_XDP_H */
diff --git a/net/xdp/Makefile b/net/xdp/Makefile
index a5d736640a0f..074fb2b2d51c 100644
--- a/net/xdp/Makefile
+++ b/net/xdp/Makefile
@@ -1,2 +1,2 @@
-obj-$(CONFIG_XDP_SOCKETS) += xsk.o xdp_umem.o
+obj-$(CONFIG_XDP_SOCKETS) += xsk.o xdp_umem.o xsk_queue.o
diff --git a/net/xdp/xdp_umem.c b/net/xdp/xdp_umem.c
index 8f768a7887da..f2c0768ca63e 100644
--- a/net/xdp/xdp_umem.c
+++ b/net/xdp/xdp_umem.c
@@ -66,6 +66,11 @@ static void xdp_umem_release(struct xdp_umem *umem)
struct mm_struct *mm;
unsigned long diff;
+ if (umem->fq) {
+ xskq_destroy(umem->fq);
+ umem->fq = NULL;
+ }
+
if (umem->pgs) {
xdp_umem_unpin_pages(umem);
diff --git a/net/xdp/xdp_umem.h b/net/xdp/xdp_umem.h
index d9bbbb880088..b3538dd2118c 100644
--- a/net/xdp/xdp_umem.h
+++ b/net/xdp/xdp_umem.h
@@ -18,9 +18,11 @@
#include <linux/mm.h>
#include <linux/if_xdp.h>
+#include "xsk_queue.h"
#include "xdp_umem_props.h"
struct xdp_umem {
+ struct xsk_queue *fq;
struct pid *pid;
struct page **pgs;
unsigned long address;
diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c
index cc0f26e17baf..6ff1d1f3322f 100644
--- a/net/xdp/xsk.c
+++ b/net/xdp/xsk.c
@@ -32,6 +32,7 @@
#include <linux/netdevice.h>
#include <net/sock.h>
+#include "xsk_queue.h"
#include "xdp_umem.h"
struct xdp_sock {
@@ -47,6 +48,21 @@ static struct xdp_sock *xdp_sk(struct sock *sk)
return (struct xdp_sock *)sk;
}
+static int xsk_init_queue(u32 entries, struct xsk_queue **queue)
+{
+ struct xsk_queue *q;
+
+ if (entries == 0 || *queue || !is_power_of_2(entries))
+ return -EINVAL;
+
+ q = xskq_create(entries);
+ if (!q)
+ return -ENOMEM;
+
+ *queue = q;
+ return 0;
+}
+
static int xsk_release(struct socket *sock)
{
struct sock *sk = sock->sk;
@@ -109,6 +125,23 @@ static int xsk_setsockopt(struct socket *sock, int level, int optname,
mutex_unlock(&xs->mutex);
return 0;
}
+ case XDP_UMEM_FILL_QUEUE:
+ {
+ struct xsk_queue **q;
+ int entries;
+
+ if (!xs->umem)
+ return -EINVAL;
+
+ if (copy_from_user(&entries, optval, sizeof(entries)))
+ return -EFAULT;
+
+ mutex_lock(&xs->mutex);
+ q = &xs->umem->fq;
+ err = xsk_init_queue(entries, q);
+ mutex_unlock(&xs->mutex);
+ return err;
+ }
default:
break;
}
@@ -116,6 +149,36 @@ static int xsk_setsockopt(struct socket *sock, int level, int optname,
return -ENOPROTOOPT;
}
+static int xsk_mmap(struct file *file, struct socket *sock,
+ struct vm_area_struct *vma)
+{
+ unsigned long offset = vma->vm_pgoff << PAGE_SHIFT;
+ unsigned long size = vma->vm_end - vma->vm_start;
+ struct xdp_sock *xs = xdp_sk(sock->sk);
+ struct xsk_queue *q;
+ unsigned long pfn;
+ struct page *qpg;
+ int err;
+
+ if (!xs->umem)
+ return -EINVAL;
+
+ if (offset == XDP_UMEM_PGOFF_FILL_QUEUE)
+ q = xs->umem->fq;
+ else
+ return -EINVAL;
+
+ qpg = virt_to_head_page(q->ring);
+ if (size > (PAGE_SIZE << compound_order(qpg)))
+ return -EINVAL;
+
+ pfn = virt_to_phys(q->ring) >> PAGE_SHIFT;
+ err = remap_pfn_range(vma, vma->vm_start, pfn,
+ size, vma->vm_page_prot);
+
+ return err;
+}
+
static struct proto xsk_proto = {
.name = "XDP",
.owner = THIS_MODULE,
@@ -139,7 +202,7 @@ static const struct proto_ops xsk_proto_ops = {
.getsockopt = sock_no_getsockopt,
.sendmsg = sock_no_sendmsg,
.recvmsg = sock_no_recvmsg,
- .mmap = sock_no_mmap,
+ .mmap = xsk_mmap,
.sendpage = sock_no_sendpage,
};
diff --git a/net/xdp/xsk_queue.c b/net/xdp/xsk_queue.c
new file mode 100644
index 000000000000..fd4bb06aa112
--- /dev/null
+++ b/net/xdp/xsk_queue.c
@@ -0,0 +1,54 @@
+/*
+ * XDP user-space ring structure
+ * Copyright(c) 2018 Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ */
+
+#include <linux/slab.h>
+
+#include "xsk_queue.h"
+
+struct xsk_queue *xskq_create(u32 nentries)
+{
+ struct xsk_queue *q;
+ gfp_t gfp_flags;
+ size_t size;
+
+ q = kzalloc(sizeof(*q), GFP_KERNEL);
+ if (!q)
+ return NULL;
+
+ q->nentries = nentries;
+ q->ring_mask = nentries - 1;
+
+ gfp_flags = GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN |
+ __GFP_COMP | __GFP_NORETRY;
+ size = xskq_umem_get_ring_size(q);
+ q->validation = XSK_VALIDATION_RX;
+
+ q->ring = (struct xdp_queue *)__get_free_pages(gfp_flags,
+ get_order(size));
+ if (!q->ring) {
+ kfree(q);
+ return NULL;
+ }
+
+ return q;
+}
+
+void xskq_destroy(struct xsk_queue *q)
+{
+ if (!q)
+ return;
+
+ page_frag_free(q->ring);
+ kfree(q);
+}
diff --git a/net/xdp/xsk_queue.h b/net/xdp/xsk_queue.h
new file mode 100644
index 000000000000..fe845a20c153
--- /dev/null
+++ b/net/xdp/xsk_queue.h
@@ -0,0 +1,49 @@
+/*
+ * XDP user-space ring structure
+ * Copyright(c) 2018 Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ */
+
+#ifndef _LINUX_XDP_QUEUE_H
+#define _LINUX_XDP_QUEUE_H
+
+#include <linux/types.h>
+#include <linux/if_xdp.h>
+
+#include "xdp_umem_props.h"
+
+enum xsk_validation {
+ XSK_VALIDATION_RX, /* Only address to packet buffer validated */
+};
+
+struct xsk_queue {
+ enum xsk_validation validation;
+ struct xdp_umem_props *umem_props;
+ u32 ring_mask;
+ u32 nentries;
+ u32 iter_head_idx;
+ u32 cached_head;
+ u32 cached_tail;
+ struct xdp_queue *ring;
+ u64 invalid_descs;
+};
+
+/* Functions operating on UMEM queues only */
+
+static inline u32 xskq_umem_get_ring_size(struct xsk_queue *q)
+{
+ return sizeof(struct xdp_umem_queue) + q->nentries * sizeof(u32);
+}
+
+struct xsk_queue *xskq_create(u32 nentries);
+void xskq_destroy(struct xsk_queue *q_ops);
+
+#endif /* _LINUX_XDP_QUEUE_H */
--
2.14.1
^ permalink raw reply related
* [RFC PATCH v2 04/14] xsk: add Rx queue setup and mmap support
From: Björn Töpel @ 2018-03-27 16:59 UTC (permalink / raw)
To: bjorn.topel, magnus.karlsson, alexander.h.duyck, alexander.duyck,
john.fastabend, ast, brouer, willemdebruijn.kernel, daniel,
netdev
Cc: Björn Töpel, michael.lundkvist, jesse.brandeburg,
anjali.singhai, qi.z.zhang, ravineet.singh
In-Reply-To: <20180327165919.17933-1-bjorn.topel@gmail.com>
From: Björn Töpel <bjorn.topel@intel.com>
Another setsockopt (XDP_RX_QUEUE) is added to let the process allocate
a queue, where the kernel can pass completed Rx frames from the kernel
to user process.
The mmapping of the queue is done using the XDP_PGOFF_RX_QUEUE offset.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
---
include/uapi/linux/if_xdp.h | 16 ++++++++++++++++
net/xdp/xsk.c | 42 +++++++++++++++++++++++++++++++++---------
net/xdp/xsk_queue.c | 11 ++++++++---
net/xdp/xsk_queue.h | 11 ++++++++++-
4 files changed, 67 insertions(+), 13 deletions(-)
diff --git a/include/uapi/linux/if_xdp.h b/include/uapi/linux/if_xdp.h
index 0de1bbf2c5c7..118456064e01 100644
--- a/include/uapi/linux/if_xdp.h
+++ b/include/uapi/linux/if_xdp.h
@@ -21,6 +21,7 @@
#include <linux/types.h>
/* XDP socket options */
+#define XDP_RX_QUEUE 1
#define XDP_UMEM_REG 3
#define XDP_UMEM_FILL_QUEUE 4
@@ -32,13 +33,28 @@ struct xdp_umem_reg {
};
/* Pgoff for mmaping the rings */
+#define XDP_PGOFF_RX_QUEUE 0
#define XDP_UMEM_PGOFF_FILL_QUEUE 0x100000000
+struct xdp_desc {
+ __u32 idx;
+ __u32 len;
+ __u16 offset;
+ __u8 flags;
+ __u8 padding[5];
+};
+
struct xdp_queue {
__u32 head_idx __attribute__((aligned(64)));
__u32 tail_idx __attribute__((aligned(64)));
};
+/* Used for the RX and TX queues for packets */
+struct xdp_rxtx_queue {
+ struct xdp_queue ptrs;
+ struct xdp_desc desc[0] __attribute__((aligned(64)));
+};
+
/* Used for the fill and completion queues for buffers */
struct xdp_umem_queue {
struct xdp_queue ptrs;
diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c
index 6ff1d1f3322f..f82f750cadc2 100644
--- a/net/xdp/xsk.c
+++ b/net/xdp/xsk.c
@@ -38,6 +38,8 @@
struct xdp_sock {
/* struct sock must be the first member of struct xdp_sock */
struct sock sk;
+ struct xsk_queue *rx;
+ struct net_device *dev;
/* Protects multiple processes in the control path */
struct mutex mutex;
struct xdp_umem *umem;
@@ -48,14 +50,15 @@ static struct xdp_sock *xdp_sk(struct sock *sk)
return (struct xdp_sock *)sk;
}
-static int xsk_init_queue(u32 entries, struct xsk_queue **queue)
+static int xsk_init_queue(u32 entries, struct xsk_queue **queue,
+ bool umem_queue)
{
struct xsk_queue *q;
if (entries == 0 || *queue || !is_power_of_2(entries))
return -EINVAL;
- q = xskq_create(entries);
+ q = xskq_create(entries, umem_queue);
if (!q)
return -ENOMEM;
@@ -97,6 +100,22 @@ static int xsk_setsockopt(struct socket *sock, int level, int optname,
return -ENOPROTOOPT;
switch (optname) {
+ case XDP_RX_QUEUE:
+ {
+ struct xsk_queue **q;
+ int entries;
+
+ if (optlen < sizeof(entries))
+ return -EINVAL;
+ if (copy_from_user(&entries, optval, sizeof(entries)))
+ return -EFAULT;
+
+ mutex_lock(&xs->mutex);
+ q = &xs->rx;
+ err = xsk_init_queue(entries, q, false);
+ mutex_unlock(&xs->mutex);
+ return err;
+ }
case XDP_UMEM_REG:
{
struct xdp_umem_reg mr;
@@ -138,7 +157,7 @@ static int xsk_setsockopt(struct socket *sock, int level, int optname,
mutex_lock(&xs->mutex);
q = &xs->umem->fq;
- err = xsk_init_queue(entries, q);
+ err = xsk_init_queue(entries, q, true);
mutex_unlock(&xs->mutex);
return err;
}
@@ -160,13 +179,17 @@ static int xsk_mmap(struct file *file, struct socket *sock,
struct page *qpg;
int err;
- if (!xs->umem)
- return -EINVAL;
+ if (offset == XDP_PGOFF_RX_QUEUE) {
+ q = xs->rx;
+ } else {
+ if (!xs->umem)
+ return -EINVAL;
- if (offset == XDP_UMEM_PGOFF_FILL_QUEUE)
- q = xs->umem->fq;
- else
- return -EINVAL;
+ if (offset == XDP_UMEM_PGOFF_FILL_QUEUE)
+ q = xs->umem->fq;
+ else
+ return -EINVAL;
+ }
qpg = virt_to_head_page(q->ring);
if (size > (PAGE_SIZE << compound_order(qpg)))
@@ -213,6 +236,7 @@ static void xsk_destruct(struct sock *sk)
if (!sock_flag(sk, SOCK_DEAD))
return;
+ xskq_destroy(xs->rx);
xdp_put_umem(xs->umem);
sk_refcnt_debug_dec(sk);
diff --git a/net/xdp/xsk_queue.c b/net/xdp/xsk_queue.c
index fd4bb06aa112..3ce6ef350850 100644
--- a/net/xdp/xsk_queue.c
+++ b/net/xdp/xsk_queue.c
@@ -16,7 +16,7 @@
#include "xsk_queue.h"
-struct xsk_queue *xskq_create(u32 nentries)
+struct xsk_queue *xskq_create(u32 nentries, bool umem_queue)
{
struct xsk_queue *q;
gfp_t gfp_flags;
@@ -31,8 +31,13 @@ struct xsk_queue *xskq_create(u32 nentries)
gfp_flags = GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN |
__GFP_COMP | __GFP_NORETRY;
- size = xskq_umem_get_ring_size(q);
- q->validation = XSK_VALIDATION_RX;
+ if (umem_queue) {
+ size = xskq_umem_get_ring_size(q);
+ q->validation = XSK_VALIDATION_RX;
+ } else {
+ size = xskq_rxtx_get_ring_size(q);
+ q->validation = XSK_VALIDATION_TX;
+ }
q->ring = (struct xdp_queue *)__get_free_pages(gfp_flags,
get_order(size));
diff --git a/net/xdp/xsk_queue.h b/net/xdp/xsk_queue.h
index fe845a20c153..d79b613a9e0a 100644
--- a/net/xdp/xsk_queue.h
+++ b/net/xdp/xsk_queue.h
@@ -22,6 +22,7 @@
enum xsk_validation {
XSK_VALIDATION_RX, /* Only address to packet buffer validated */
+ XSK_VALIDATION_TX /* Full descriptor is validated */
};
struct xsk_queue {
@@ -36,6 +37,14 @@ struct xsk_queue {
u64 invalid_descs;
};
+/* Functions operating on RXTX queues only */
+
+static inline u32 xskq_rxtx_get_ring_size(struct xsk_queue *q)
+{
+ return (sizeof(struct xdp_queue) +
+ q->nentries * sizeof(struct xdp_desc));
+}
+
/* Functions operating on UMEM queues only */
static inline u32 xskq_umem_get_ring_size(struct xsk_queue *q)
@@ -43,7 +52,7 @@ static inline u32 xskq_umem_get_ring_size(struct xsk_queue *q)
return sizeof(struct xdp_umem_queue) + q->nentries * sizeof(u32);
}
-struct xsk_queue *xskq_create(u32 nentries);
+struct xsk_queue *xskq_create(u32 nentries, bool umem_queue);
void xskq_destroy(struct xsk_queue *q_ops);
#endif /* _LINUX_XDP_QUEUE_H */
--
2.14.1
^ permalink raw reply related
* [RFC PATCH v2 06/14] xsk: add Rx receive functions and poll support
From: Björn Töpel @ 2018-03-27 16:59 UTC (permalink / raw)
To: bjorn.topel, magnus.karlsson, alexander.h.duyck, alexander.duyck,
john.fastabend, ast, brouer, willemdebruijn.kernel, daniel,
netdev
Cc: Björn Töpel, michael.lundkvist, jesse.brandeburg,
anjali.singhai, qi.z.zhang, ravineet.singh
In-Reply-To: <20180327165919.17933-1-bjorn.topel@gmail.com>
From: Björn Töpel <bjorn.topel@intel.com>
Here the actual receive functions of AF_XDP are implemented, that in a
later commit, will be called from the XDP layers.
There's one set of functions for the XDP_DRV side and another for
XDP_SKB (generic).
Support for the poll syscall is also implemented.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
---
net/xdp/xdp_umem.h | 18 +++++
net/xdp/xsk.c | 81 ++++++++++++++++++++-
net/xdp/xsk_queue.h | 206 ++++++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 304 insertions(+), 1 deletion(-)
diff --git a/net/xdp/xdp_umem.h b/net/xdp/xdp_umem.h
index ad041b911b38..5e7105b7760b 100644
--- a/net/xdp/xdp_umem.h
+++ b/net/xdp/xdp_umem.h
@@ -36,6 +36,24 @@ struct xdp_umem {
struct user_struct *user;
};
+static inline char *xdp_umem_get_data(struct xdp_umem *umem, u32 idx)
+{
+ u64 pg, off;
+ char *data;
+
+ pg = idx >> umem->nfpplog2;
+ off = (idx - (pg << umem->nfpplog2)) << umem->frame_size_log2;
+
+ data = page_address(umem->pgs[pg]);
+ return data + off;
+}
+
+static inline char *xdp_umem_get_data_with_headroom(struct xdp_umem *umem,
+ u32 idx)
+{
+ return xdp_umem_get_data(umem, idx) + umem->frame_headroom;
+}
+
bool xdp_umem_validate_queues(struct xdp_umem *umem);
int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr);
void xdp_get_umem(struct xdp_umem *umem);
diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c
index d99a1b830f94..a60b1fcfb2b3 100644
--- a/net/xdp/xsk.c
+++ b/net/xdp/xsk.c
@@ -35,10 +35,14 @@
#include "xsk_queue.h"
#include "xdp_umem.h"
+#define RX_BATCH_SIZE 16
+
struct xdp_sock {
/* struct sock must be the first member of struct xdp_sock */
struct sock sk;
struct xsk_queue *rx;
+ struct xskq_iter rx_it;
+ u64 rx_dropped;
struct net_device *dev;
/* Protects multiple processes in the control path */
struct mutex mutex;
@@ -52,6 +56,74 @@ static struct xdp_sock *xdp_sk(struct sock *sk)
return (struct xdp_sock *)sk;
}
+static inline int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
+{
+ u32 len = xdp->data_end - xdp->data;
+ void *buffer;
+ int err = 0;
+ u32 id;
+
+ if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index)
+ return -EINVAL;
+
+ if (!xskq_next_frame_deq(xs->umem->fq, &xs->rx_it, RX_BATCH_SIZE))
+ return -ENOSPC;
+
+ id = xdp_umem_get_id(xs->umem->fq, &xs->rx_it);
+ buffer = xdp_umem_get_data_with_headroom(xs->umem, id);
+ memcpy(buffer, xdp->data, len);
+ err = xskq_rxtx_enq_frame(xs->rx, id, len, xs->umem->frame_headroom);
+ if (err)
+ xskq_deq_return_frame(&xs->rx_it);
+
+ return err;
+}
+
+int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
+{
+ int err;
+
+ err = __xsk_rcv(xs, xdp);
+ if (!err)
+ page_frag_free(xdp->data);
+ else
+ xs->rx_dropped++;
+
+ return err;
+}
+
+void xsk_flush(struct xdp_sock *xs)
+{
+ xskq_enq_flush(xs->rx);
+ xs->sk.sk_data_ready(&xs->sk);
+}
+
+int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
+{
+ int err;
+
+ err = __xsk_rcv(xs, xdp);
+ if (!err)
+ xsk_flush(xs);
+ else
+ xs->rx_dropped++;
+
+ return err;
+}
+
+static unsigned int xsk_poll(struct file *file, struct socket *sock,
+ struct poll_table_struct *wait)
+{
+ unsigned int mask = datagram_poll(file, sock, wait);
+ struct sock *sk = sock->sk;
+ struct xdp_sock *xs = xdp_sk(sk);
+
+ if (xs->rx && !xskq_empty(xs->rx))
+ mask |= POLLIN | POLLRDNORM;
+
+ return mask;
+}
+
static int xsk_init_queue(u32 entries, struct xsk_queue **queue,
bool umem_queue)
{
@@ -190,6 +262,9 @@ static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len)
} else if (!xs->umem || !xdp_umem_validate_queues(xs->umem)) {
err = -EINVAL;
goto out_unlock;
+ } else {
+ /* This xsk has its own umem. */
+ xskq_set_umem(xs->umem->fq, &xs->umem->props);
}
/* Rebind? */
@@ -204,6 +279,10 @@ static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len)
xs->ifindex = sxdp->sxdp_ifindex;
xs->queue_id = sxdp->sxdp_queue_id;
+ xskq_init_iter(&xs->rx_it);
+
+ xskq_set_umem(xs->rx, &xs->umem->props);
+
out_unlock:
if (err)
dev_put(dev);
@@ -340,7 +419,7 @@ static const struct proto_ops xsk_proto_ops = {
.socketpair = sock_no_socketpair,
.accept = sock_no_accept,
.getname = sock_no_getname,
- .poll = sock_no_poll,
+ .poll = xsk_poll,
.ioctl = sock_no_ioctl,
.listen = sock_no_listen,
.shutdown = sock_no_shutdown,
diff --git a/net/xdp/xsk_queue.h b/net/xdp/xsk_queue.h
index d79b613a9e0a..af6e651f1207 100644
--- a/net/xdp/xsk_queue.h
+++ b/net/xdp/xsk_queue.h
@@ -37,6 +37,187 @@ struct xsk_queue {
u64 invalid_descs;
};
+struct xskq_iter {
+ u32 head;
+ u32 tail;
+ struct xdp_desc desc_copy;
+};
+
+/* Common functions operating for both RXTX and umem queues */
+
+static inline bool xskq_is_valid_rx_entry(struct xsk_queue *q,
+ u32 idx)
+{
+ if (unlikely(idx >= q->umem_props->nframes)) {
+ q->invalid_descs++;
+ return false;
+ }
+ return true;
+}
+
+static inline bool xskq_is_valid_tx_entry(struct xsk_queue *q,
+ struct xdp_desc *d)
+{
+ u32 buff_len;
+
+ if (unlikely(d->idx >= q->umem_props->nframes)) {
+ q->invalid_descs++;
+ return false;
+ }
+
+ buff_len = q->umem_props->frame_size;
+ if (unlikely(d->len > buff_len || d->len == 0 ||
+ d->offset > buff_len || d->offset + d->len > buff_len)) {
+ q->invalid_descs++;
+ return false;
+ }
+
+ return true;
+}
+
+static inline u32 xskq_nb_free(struct xsk_queue *q, u32 head_idx, u32 dcnt)
+{
+ u32 free_entries = q->nentries - (head_idx - q->cached_tail);
+
+ if (free_entries >= dcnt)
+ return free_entries;
+
+ /* Refresh the local tail pointer */
+ q->cached_tail = READ_ONCE(q->ring->tail_idx);
+ return q->nentries - (head_idx - q->cached_tail);
+}
+
+static inline u32 xskq_nb_avail(struct xsk_queue *q, u32 dcnt)
+{
+ u32 entries = q->cached_head - q->cached_tail;
+
+ if (entries == 0)
+ /* Refresh the local head pointer */
+ q->cached_head = READ_ONCE(q->ring->head_idx);
+
+ entries = q->cached_head - q->cached_tail;
+ return (entries > dcnt) ? dcnt : entries;
+}
+
+static inline bool xskq_empty(struct xsk_queue *q)
+{
+ if (xskq_nb_free(q, q->cached_head, 1) == q->nentries)
+ return true;
+ return false;
+}
+
+static inline bool xskq_full(struct xsk_queue *q)
+{
+ if (xskq_nb_avail(q, q->nentries) == q->nentries)
+ return true;
+ return false;
+}
+
+static inline void xskq_init_iter(struct xskq_iter *it)
+{
+ it->head = 0;
+ it->tail = 0;
+}
+
+static inline void xskq_set_umem(struct xsk_queue *q,
+ struct xdp_umem_props *umem_props)
+{
+ q->umem_props = umem_props;
+}
+
+static inline bool xskq_iter_end(struct xskq_iter *it)
+{
+ return it->tail == it->head;
+}
+
+static inline void xskq_iter_validate(struct xsk_queue *q,
+ struct xskq_iter *it)
+{
+ while (!xskq_iter_end(it)) {
+ unsigned int idx = it->tail & q->ring_mask;
+
+ if (q->validation == XSK_VALIDATION_TX) {
+ struct xdp_rxtx_queue *ring =
+ (struct xdp_rxtx_queue *)q->ring;
+
+ it->desc_copy.idx = ring->desc[idx].idx;
+ it->desc_copy.len = ring->desc[idx].len;
+ it->desc_copy.offset = ring->desc[idx].offset;
+
+ if (xskq_is_valid_tx_entry(q, &it->desc_copy))
+ break;
+ } else {
+ /* XSK_VALIDATION_RX */
+ struct xdp_umem_queue *ring =
+ (struct xdp_umem_queue *)q->ring;
+
+ if (xskq_is_valid_rx_entry(q, ring->desc[idx]))
+ break;
+ }
+
+ it->tail++;
+ }
+}
+
+static inline void xskq_deq_iter(struct xsk_queue *q,
+ struct xskq_iter *it, int cnt)
+{
+ it->tail = q->cached_tail;
+ it->head = q->cached_tail + xskq_nb_avail(q, cnt);
+
+ /* Order tail and data */
+ smp_rmb();
+
+ xskq_iter_validate(q, it);
+}
+
+static inline void xskq_deq_iter_next(struct xsk_queue *q,
+ struct xskq_iter *it)
+{
+ it->tail++;
+ xskq_iter_validate(q, it);
+}
+
+static inline void xskq_deq_iter_done(struct xsk_queue *q,
+ struct xskq_iter *it)
+{
+ q->cached_tail = it->tail;
+ WRITE_ONCE(q->ring->tail_idx, it->tail);
+}
+
+static inline u64 xskq_nb_invalid_descs(struct xsk_queue *q)
+{
+ return q ? q->invalid_descs : 0;
+}
+
+static inline bool xskq_next_frame_deq(struct xsk_queue *q,
+ struct xskq_iter *it,
+ u32 batch_size)
+{
+ if (xskq_iter_end(it)) {
+ xskq_deq_iter_done(q, it);
+ xskq_deq_iter(q, it, batch_size);
+ return !xskq_iter_end(it);
+ }
+
+ xskq_deq_iter_next(q, it);
+ return !xskq_iter_end(it);
+}
+
+static inline void xskq_deq_return_frame(struct xskq_iter *it)
+{
+ it->tail--;
+}
+
+static inline void xskq_enq_flush(struct xsk_queue *q)
+{
+ /* Order flags and data */
+ smp_wmb();
+
+ WRITE_ONCE(q->ring->head_idx, q->iter_head_idx);
+ q->cached_head = q->iter_head_idx;
+}
+
/* Functions operating on RXTX queues only */
static inline u32 xskq_rxtx_get_ring_size(struct xsk_queue *q)
@@ -45,6 +226,23 @@ static inline u32 xskq_rxtx_get_ring_size(struct xsk_queue *q)
q->nentries * sizeof(struct xdp_desc));
}
+static inline int xskq_rxtx_enq_frame(struct xsk_queue *q,
+ u32 id, u32 len, u16 offset)
+{
+ struct xdp_rxtx_queue *ring = (struct xdp_rxtx_queue *)q->ring;
+ unsigned int idx;
+
+ if (xskq_nb_free(q, q->iter_head_idx, 1) == 0)
+ return -ENOSPC;
+
+ idx = (q->iter_head_idx++) & q->ring_mask;
+ ring->desc[idx].idx = id;
+ ring->desc[idx].len = len;
+ ring->desc[idx].offset = offset;
+
+ return 0;
+}
+
/* Functions operating on UMEM queues only */
static inline u32 xskq_umem_get_ring_size(struct xsk_queue *q)
@@ -52,6 +250,14 @@ static inline u32 xskq_umem_get_ring_size(struct xsk_queue *q)
return sizeof(struct xdp_umem_queue) + q->nentries * sizeof(u32);
}
+static inline u32 xdp_umem_get_id(struct xsk_queue *q,
+ struct xskq_iter *it)
+{
+ struct xdp_umem_queue *ring = (struct xdp_umem_queue *)q->ring;
+
+ return ring->desc[it->tail & q->ring_mask];
+}
+
struct xsk_queue *xskq_create(u32 nentries, bool umem_queue);
void xskq_destroy(struct xsk_queue *q_ops);
--
2.14.1
^ permalink raw reply related
* [RFC PATCH v2 05/14] xsk: add support for bind for Rx
From: Björn Töpel @ 2018-03-27 16:59 UTC (permalink / raw)
To: bjorn.topel, magnus.karlsson, alexander.h.duyck, alexander.duyck,
john.fastabend, ast, brouer, willemdebruijn.kernel, daniel,
netdev
Cc: michael.lundkvist, jesse.brandeburg, anjali.singhai, qi.z.zhang,
ravineet.singh
In-Reply-To: <20180327165919.17933-1-bjorn.topel@gmail.com>
From: Magnus Karlsson <magnus.karlsson@intel.com>
Here, the bind syscall is added. Binding an AF_XDP socket, means
associating the socket to an umem, a netdev and a queue index. This
can be done in two ways.
The first way, creating a "socket from scratch". Create the umem using
the XDP_UMEM_REG setsockopt and an associated fill queue with
XDP_UMEM_FILL_QUEUE. Create the Rx queue using the XDP_RX_QUEUE
setsockopt. Call bind passing ifindex and queue index ("channel" in
ethtool speak).
The second way to bind a socket, is simply skipping the
umem/netdev/queue index, and passing another already setup AF_XDP
socket. The new socket will then have the same umem/netdev/queue index
as the parent so it will share the same umem. You must also set the
flags field in the socket address to XDP_SHARED_UMEM.
Signed-off-by: Magnus Karlsson <magnus.karlsson@intel.com>
---
include/uapi/linux/if_xdp.h | 11 ++++
net/xdp/xdp_umem.c | 9 ++++
net/xdp/xdp_umem.h | 2 +
net/xdp/xsk.c | 125 +++++++++++++++++++++++++++++++++++++++++++-
4 files changed, 146 insertions(+), 1 deletion(-)
diff --git a/include/uapi/linux/if_xdp.h b/include/uapi/linux/if_xdp.h
index 118456064e01..e3593df7323c 100644
--- a/include/uapi/linux/if_xdp.h
+++ b/include/uapi/linux/if_xdp.h
@@ -20,6 +20,17 @@
#include <linux/types.h>
+/* Options for the sxdp_flags field */
+#define XDP_SHARED_UMEM 1
+
+struct sockaddr_xdp {
+ __u16 sxdp_family;
+ __u32 sxdp_ifindex;
+ __u32 sxdp_queue_id;
+ __u32 sxdp_shared_umem_fd;
+ __u16 sxdp_flags;
+};
+
/* XDP socket options */
#define XDP_RX_QUEUE 1
#define XDP_UMEM_REG 3
diff --git a/net/xdp/xdp_umem.c b/net/xdp/xdp_umem.c
index f2c0768ca63e..96395beb980e 100644
--- a/net/xdp/xdp_umem.c
+++ b/net/xdp/xdp_umem.c
@@ -97,6 +97,11 @@ static void xdp_umem_release(struct xdp_umem *umem)
kfree(umem);
}
+void xdp_get_umem(struct xdp_umem *umem)
+{
+ atomic_inc(&umem->users);
+}
+
void xdp_put_umem(struct xdp_umem *umem)
{
if (!umem)
@@ -243,3 +248,7 @@ int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr)
return err;
}
+bool xdp_umem_validate_queues(struct xdp_umem *umem)
+{
+ return umem->fq;
+}
diff --git a/net/xdp/xdp_umem.h b/net/xdp/xdp_umem.h
index b3538dd2118c..ad041b911b38 100644
--- a/net/xdp/xdp_umem.h
+++ b/net/xdp/xdp_umem.h
@@ -36,7 +36,9 @@ struct xdp_umem {
struct user_struct *user;
};
+bool xdp_umem_validate_queues(struct xdp_umem *umem);
int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr);
+void xdp_get_umem(struct xdp_umem *umem);
void xdp_put_umem(struct xdp_umem *umem);
int xdp_umem_create(struct xdp_umem **umem);
diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c
index f82f750cadc2..d99a1b830f94 100644
--- a/net/xdp/xsk.c
+++ b/net/xdp/xsk.c
@@ -43,6 +43,8 @@ struct xdp_sock {
/* Protects multiple processes in the control path */
struct mutex mutex;
struct xdp_umem *umem;
+ u32 ifindex;
+ u16 queue_id;
};
static struct xdp_sock *xdp_sk(struct sock *sk)
@@ -66,9 +68,18 @@ static int xsk_init_queue(u32 entries, struct xsk_queue **queue,
return 0;
}
+static void __xsk_release(struct xdp_sock *xs)
+{
+ /* Wait for driver to stop using the xdp socket. */
+ synchronize_net();
+
+ dev_put(xs->dev);
+}
+
static int xsk_release(struct socket *sock)
{
struct sock *sk = sock->sk;
+ struct xdp_sock *xs = xdp_sk(sk);
struct net *net;
if (!sk)
@@ -80,6 +91,11 @@ static int xsk_release(struct socket *sock)
sock_prot_inuse_add(net, sk->sk_prot, -1);
local_bh_enable();
+ if (xs->dev) {
+ __xsk_release(xs);
+ xs->dev = NULL;
+ }
+
sock_orphan(sk);
sock->sk = NULL;
@@ -89,6 +105,113 @@ static int xsk_release(struct socket *sock)
return 0;
}
+static struct socket *xsk_lookup_xsk_from_fd(int fd, int *err)
+{
+ struct socket *sock;
+
+ *err = -ENOTSOCK;
+ sock = sockfd_lookup(fd, err);
+ if (!sock)
+ return NULL;
+
+ if (sock->sk->sk_family != PF_XDP) {
+ *err = -ENOPROTOOPT;
+ sockfd_put(sock);
+ return NULL;
+ }
+
+ *err = 0;
+ return sock;
+}
+
+static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len)
+{
+ struct sockaddr_xdp *sxdp = (struct sockaddr_xdp *)addr;
+ struct sock *sk = sock->sk;
+ struct net_device *dev, *dev_curr;
+ struct xdp_sock *xs = xdp_sk(sk);
+ struct xdp_umem *old_umem = NULL;
+ int err = 0;
+
+ if (addr_len < sizeof(struct sockaddr_xdp))
+ return -EINVAL;
+ if (sxdp->sxdp_family != AF_XDP)
+ return -EINVAL;
+
+ mutex_lock(&xs->mutex);
+ dev_curr = xs->dev;
+ dev = dev_get_by_index(sock_net(sk), sxdp->sxdp_ifindex);
+ if (!dev) {
+ err = -ENODEV;
+ goto out_release;
+ }
+
+ if (!xs->rx) {
+ err = -EINVAL;
+ goto out_unlock;
+ }
+
+ if (sxdp->sxdp_queue_id >= dev->num_rx_queues) {
+ err = -EINVAL;
+ goto out_unlock;
+ }
+
+ if (sxdp->sxdp_flags & XDP_SHARED_UMEM) {
+ struct xdp_sock *umem_xs;
+ struct socket *sock;
+
+ if (xs->umem) {
+ /* We have already our own. */
+ err = -EINVAL;
+ goto out_unlock;
+ }
+
+ sock = xsk_lookup_xsk_from_fd(sxdp->sxdp_shared_umem_fd, &err);
+ if (!sock)
+ goto out_unlock;
+
+ umem_xs = xdp_sk(sock->sk);
+ if (!umem_xs->umem) {
+ /* No umem to inherit. */
+ err = -EBADF;
+ sockfd_put(sock);
+ goto out_unlock;
+ } else if (umem_xs->dev != dev ||
+ umem_xs->queue_id != sxdp->sxdp_queue_id) {
+ err = -EINVAL;
+ sockfd_put(sock);
+ goto out_unlock;
+ }
+
+ xdp_get_umem(umem_xs->umem);
+ old_umem = xs->umem;
+ xs->umem = umem_xs->umem;
+ sockfd_put(sock);
+ } else if (!xs->umem || !xdp_umem_validate_queues(xs->umem)) {
+ err = -EINVAL;
+ goto out_unlock;
+ }
+
+ /* Rebind? */
+ if (dev_curr && (dev_curr != dev ||
+ xs->queue_id != sxdp->sxdp_queue_id)) {
+ __xsk_release(xs);
+ if (old_umem)
+ xdp_put_umem(old_umem);
+ }
+
+ xs->dev = dev;
+ xs->ifindex = sxdp->sxdp_ifindex;
+ xs->queue_id = sxdp->sxdp_queue_id;
+
+out_unlock:
+ if (err)
+ dev_put(dev);
+out_release:
+ mutex_unlock(&xs->mutex);
+ return err;
+}
+
static int xsk_setsockopt(struct socket *sock, int level, int optname,
char __user *optval, unsigned int optlen)
{
@@ -212,7 +335,7 @@ static const struct proto_ops xsk_proto_ops = {
.family = PF_XDP,
.owner = THIS_MODULE,
.release = xsk_release,
- .bind = sock_no_bind,
+ .bind = xsk_bind,
.connect = sock_no_connect,
.socketpair = sock_no_socketpair,
.accept = sock_no_accept,
--
2.14.1
^ permalink raw reply related
* [RFC PATCH v2 07/14] bpf: introduce new bpf AF_XDP map type BPF_MAP_TYPE_XSKMAP
From: Björn Töpel @ 2018-03-27 16:59 UTC (permalink / raw)
To: bjorn.topel, magnus.karlsson, alexander.h.duyck, alexander.duyck,
john.fastabend, ast, brouer, willemdebruijn.kernel, daniel,
netdev
Cc: Björn Töpel, michael.lundkvist, jesse.brandeburg,
anjali.singhai, qi.z.zhang, ravineet.singh
In-Reply-To: <20180327165919.17933-1-bjorn.topel@gmail.com>
From: Björn Töpel <bjorn.topel@intel.com>
The xskmap is yet another BPF map, very much inspired by
dev/cpu/sockmap, and is a holder of AF_XDP sockets. A user application
adds AF_XDP sockets into the map, and by using the bpf_redirect_map
helper, an XDP program can redirect XDP frames to an AF_XDP socket.
Note that a socket that is bound to certain ifindex/queue index will
*only* accept XDP frames from that netdev/queue index. If an XDP
program tries to redirect from a netdev/queue index other than what
the socket is bound to, the frame will not be received on the socket.
A socket can reside in multiple maps.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
---
include/linux/bpf.h | 26 +++++
include/linux/bpf_types.h | 3 +
include/net/xdp_sock.h | 34 ++++++
include/uapi/linux/bpf.h | 1 +
kernel/bpf/Makefile | 3 +
kernel/bpf/verifier.c | 8 +-
kernel/bpf/xskmap.c | 281 ++++++++++++++++++++++++++++++++++++++++++++++
7 files changed, 354 insertions(+), 2 deletions(-)
create mode 100644 include/net/xdp_sock.h
create mode 100644 kernel/bpf/xskmap.c
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 819229c80eca..0fe9d080adc3 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -657,6 +657,32 @@ static inline int sock_map_prog(struct bpf_map *map,
}
#endif
+#if defined(CONFIG_XDP_SOCKETS)
+struct xdp_sock;
+struct xdp_sock *__xsk_map_lookup_elem(struct bpf_map *map, u32 key);
+int __xsk_map_redirect(struct bpf_map *map, u32 index,
+ struct xdp_buff *xdp, struct xdp_sock *xs);
+void __xsk_map_flush(struct bpf_map *map);
+#else
+struct xdp_sock;
+static inline struct xdp_sock *__xsk_map_lookup_elem(struct bpf_map *map,
+ u32 key)
+{
+ return NULL;
+}
+
+static inline int __xsk_map_redirect(struct bpf_map *map, u32 index,
+ struct xdp_buff *xdp,
+ struct xdp_sock *xs)
+{
+ return -EOPNOTSUPP;
+}
+
+static inline void __xsk_map_flush(struct bpf_map *map)
+{
+}
+#endif
+
/* verifier prototypes for helper functions called from eBPF programs */
extern const struct bpf_func_proto bpf_map_lookup_elem_proto;
extern const struct bpf_func_proto bpf_map_update_elem_proto;
diff --git a/include/linux/bpf_types.h b/include/linux/bpf_types.h
index 5e2e8a49fb21..b525862c98ab 100644
--- a/include/linux/bpf_types.h
+++ b/include/linux/bpf_types.h
@@ -47,4 +47,7 @@ BPF_MAP_TYPE(BPF_MAP_TYPE_DEVMAP, dev_map_ops)
BPF_MAP_TYPE(BPF_MAP_TYPE_SOCKMAP, sock_map_ops)
#endif
BPF_MAP_TYPE(BPF_MAP_TYPE_CPUMAP, cpu_map_ops)
+#if defined(CONFIG_XDP_SOCKETS)
+BPF_MAP_TYPE(BPF_MAP_TYPE_XSKMAP, xsk_map_ops)
+#endif
#endif
diff --git a/include/net/xdp_sock.h b/include/net/xdp_sock.h
new file mode 100644
index 000000000000..80d119a685b2
--- /dev/null
+++ b/include/net/xdp_sock.h
@@ -0,0 +1,34 @@
+/*
+ * AF_XDP internal functions
+ * Copyright(c) 2018 Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ */
+
+#ifndef _LINUX_XDP_SOCK_H
+#define _LINUX_XDP_SOCK_H
+
+struct xdp_sock;
+struct xdp_buff;
+#ifdef CONFIG_XDP_SOCKETS
+int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp);
+void xsk_flush(struct xdp_sock *xs);
+#else
+static inline int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
+{
+ return -ENOTSUPP;
+}
+
+static inline void xsk_flush(struct xdp_sock *xs)
+{
+}
+#endif /* CONFIG_XDP_SOCKETS */
+
+#endif /* _LINUX_XDP_SOCK_H */
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 18b7c510c511..c8e1d2977712 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -114,6 +114,7 @@ enum bpf_map_type {
BPF_MAP_TYPE_DEVMAP,
BPF_MAP_TYPE_SOCKMAP,
BPF_MAP_TYPE_CPUMAP,
+ BPF_MAP_TYPE_XSKMAP,
};
enum bpf_prog_type {
diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile
index a713fd23ec88..3c59d9bcae14 100644
--- a/kernel/bpf/Makefile
+++ b/kernel/bpf/Makefile
@@ -7,6 +7,9 @@ obj-$(CONFIG_BPF_SYSCALL) += disasm.o
ifeq ($(CONFIG_NET),y)
obj-$(CONFIG_BPF_SYSCALL) += devmap.o
obj-$(CONFIG_BPF_SYSCALL) += cpumap.o
+ifeq ($(CONFIG_XDP_SOCKETS),y)
+obj-$(CONFIG_BPF_SYSCALL) += xskmap.o
+endif
obj-$(CONFIG_BPF_SYSCALL) += offload.o
ifeq ($(CONFIG_STREAM_PARSER),y)
ifeq ($(CONFIG_INET),y)
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index e9f7c20691c1..46d525539a3b 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -2059,8 +2059,11 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env,
if (func_id != BPF_FUNC_redirect_map)
goto error;
break;
- /* Restrict bpf side of cpumap, open when use-cases appear */
+ /* Restrict bpf side of cpumap and xskmap, open when use-cases
+ * appear.
+ */
case BPF_MAP_TYPE_CPUMAP:
+ case BPF_MAP_TYPE_XSKMAP:
if (func_id != BPF_FUNC_redirect_map)
goto error;
break;
@@ -2107,7 +2110,8 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env,
break;
case BPF_FUNC_redirect_map:
if (map->map_type != BPF_MAP_TYPE_DEVMAP &&
- map->map_type != BPF_MAP_TYPE_CPUMAP)
+ map->map_type != BPF_MAP_TYPE_CPUMAP &&
+ map->map_type != BPF_MAP_TYPE_XSKMAP)
goto error;
break;
case BPF_FUNC_sk_redirect_map:
diff --git a/kernel/bpf/xskmap.c b/kernel/bpf/xskmap.c
new file mode 100644
index 000000000000..77553f24daa2
--- /dev/null
+++ b/kernel/bpf/xskmap.c
@@ -0,0 +1,281 @@
+/*
+ * XSKMAP used for AF_XDP sockets
+ * Copyright(c) 2018 Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ */
+
+#include <linux/bpf.h>
+#include <linux/capability.h>
+#include <net/xdp_sock.h>
+#include <linux/slab.h>
+#include <linux/sched.h>
+#include <net/sock.h>
+
+struct xsk_map_entry {
+ struct xdp_sock *xs;
+ struct rcu_head rcu;
+};
+
+struct xsk_map {
+ struct bpf_map map;
+ struct xsk_map_entry **xsk_map;
+ unsigned long __percpu *flush_needed;
+};
+
+static u64 xsk_map_bitmap_size(const union bpf_attr *attr)
+{
+ return BITS_TO_LONGS((u64) attr->max_entries) * sizeof(unsigned long);
+}
+
+static struct bpf_map *xsk_map_alloc(union bpf_attr *attr)
+{
+ struct xsk_map *m;
+ int err = -EINVAL;
+ u64 cost;
+
+ if (!capable(CAP_NET_ADMIN))
+ return ERR_PTR(-EPERM);
+
+ if (attr->max_entries == 0 || attr->key_size != 4 ||
+ attr->value_size != 4 ||
+ attr->map_flags & ~(BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY))
+ return ERR_PTR(-EINVAL);
+
+ m = kzalloc(sizeof(*m), GFP_USER);
+ if (!m)
+ return ERR_PTR(-ENOMEM);
+
+ bpf_map_init_from_attr(&m->map, attr);
+
+ cost = (u64)m->map.max_entries * sizeof(struct xsk_map_entry *);
+ cost += xsk_map_bitmap_size(attr) * num_possible_cpus();
+ if (cost >= U32_MAX - PAGE_SIZE)
+ goto free_m;
+
+ m->map.pages = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT;
+
+ /* Notice returns -EPERM on if map size is larger than memlock limit */
+ err = bpf_map_precharge_memlock(m->map.pages);
+ if (err)
+ goto free_m;
+
+ m->flush_needed = __alloc_percpu(xsk_map_bitmap_size(attr),
+ __alignof__(unsigned long));
+ if (!m->flush_needed)
+ goto free_m;
+
+ m->xsk_map = bpf_map_area_alloc(m->map.max_entries *
+ sizeof(struct xsk_map_entry *),
+ m->map.numa_node);
+ if (!m->xsk_map)
+ goto free_percpu;
+ return &m->map;
+
+free_percpu:
+ free_percpu(m->flush_needed);
+free_m:
+ kfree(m);
+ return ERR_PTR(err);
+}
+
+static void xsk_map_free(struct bpf_map *map)
+{
+ struct xsk_map *m = container_of(map, struct xsk_map, map);
+ int i, cpu;
+
+ /* At this point bpf_prog->aux->refcnt == 0 and this
+ * map->refcnt == 0, so the programs (can be more than one
+ * that used this map) were disconnected from events. Wait for
+ * outstanding critical sections in these programs to
+ * complete. The rcu critical section only guarantees no
+ * further reads against xsk_map. It does __not__ ensure
+ * pending flush operations (if any) are complete.
+ */
+
+ synchronize_rcu();
+
+ /* To ensure all pending flush operations have completed wait
+ * for flush bitmap to indicate all flush_needed bits to be
+ * zero on _all_ cpus. Because the above synchronize_rcu()
+ * ensures the map is disconnected from the program we can
+ * assume no new bits will be set.
+ */
+ for_each_online_cpu(cpu) {
+ unsigned long *bitmap = per_cpu_ptr(m->flush_needed, cpu);
+
+ while (!bitmap_empty(bitmap, map->max_entries))
+ cond_resched();
+ }
+
+ for (i = 0; i < map->max_entries; i++) {
+ struct xsk_map_entry *entry;
+
+ entry = m->xsk_map[i];
+ if (!entry)
+ continue;
+
+ sock_put((struct sock *)entry->xs);
+ kfree(entry);
+ }
+
+ free_percpu(m->flush_needed);
+ bpf_map_area_free(m->xsk_map);
+ kfree(m);
+}
+
+static int xsk_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
+{
+ struct xsk_map *m = container_of(map, struct xsk_map, map);
+ u32 index = key ? *(u32 *)key : U32_MAX;
+ u32 *next = next_key;
+
+ if (index >= m->map.max_entries) {
+ *next = 0;
+ return 0;
+ }
+
+ if (index == m->map.max_entries - 1)
+ return -ENOENT;
+ *next = index + 1;
+ return 0;
+}
+
+struct xdp_sock *__xsk_map_lookup_elem(struct bpf_map *map, u32 key)
+{
+ struct xsk_map *m = container_of(map, struct xsk_map, map);
+ struct xsk_map_entry *entry;
+
+ if (key >= map->max_entries)
+ return NULL;
+
+ entry = READ_ONCE(m->xsk_map[key]);
+ return entry ? entry->xs : NULL;
+}
+
+int __xsk_map_redirect(struct bpf_map *map, u32 index,
+ struct xdp_buff *xdp, struct xdp_sock *xs)
+{
+ struct xsk_map *m = container_of(map, struct xsk_map, map);
+ unsigned long *bitmap = this_cpu_ptr(m->flush_needed);
+ int err;
+
+ err = xsk_rcv(xs, xdp);
+ if (err)
+ return err;
+
+ __set_bit(index, bitmap);
+ return 0;
+}
+
+void __xsk_map_flush(struct bpf_map *map)
+{
+ struct xsk_map *m = container_of(map, struct xsk_map, map);
+ unsigned long *bitmap = this_cpu_ptr(m->flush_needed);
+ u32 bit;
+
+ for_each_set_bit(bit, bitmap, map->max_entries) {
+ struct xsk_map_entry *entry = READ_ONCE(m->xsk_map[bit]);
+
+ /* This is possible if the entry is removed by user
+ * space between xdp redirect and flush op.
+ */
+ if (unlikely(!entry))
+ continue;
+
+ __clear_bit(bit, bitmap);
+ xsk_flush(entry->xs);
+ }
+}
+
+static void *xsk_map_lookup_elem(struct bpf_map *map, void *key)
+{
+ return NULL;
+}
+
+static void __xsk_map_entry_free(struct rcu_head *rcu)
+{
+ struct xsk_map_entry *entry;
+
+ entry = container_of(rcu, struct xsk_map_entry, rcu);
+ xsk_flush(entry->xs);
+ sock_put((struct sock *)entry->xs);
+ kfree(entry);
+}
+
+static int xsk_map_update_elem(struct bpf_map *map, void *key, void *value,
+ u64 map_flags)
+{
+ struct xsk_map *m = container_of(map, struct xsk_map, map);
+ struct xsk_map_entry *entry, *old_entry;
+ u32 i = *(u32 *)key, fd = *(u32 *)value;
+ struct socket *sock;
+ int err;
+
+ if (unlikely(map_flags > BPF_EXIST))
+ return -EINVAL;
+ if (unlikely(i >= m->map.max_entries))
+ return -E2BIG;
+ if (unlikely(map_flags == BPF_NOEXIST))
+ return -EEXIST;
+
+ sock = sockfd_lookup(fd, &err);
+ if (!sock)
+ return err;
+
+ if (sock->sk->sk_family != PF_XDP) {
+ sockfd_put(sock);
+ return -EOPNOTSUPP;
+ }
+
+ entry = kmalloc_node(sizeof(*entry), GFP_ATOMIC | __GFP_NOWARN,
+ map->numa_node);
+ if (!entry) {
+ sockfd_put(sock);
+ return -ENOMEM;
+ }
+
+ sock_hold(sock->sk);
+ entry->xs = (struct xdp_sock *)sock->sk;
+
+ old_entry = xchg(&m->xsk_map[i], entry);
+ if (old_entry)
+ call_rcu(&old_entry->rcu, __xsk_map_entry_free);
+
+ sockfd_put(sock);
+ return 0;
+}
+
+static int xsk_map_delete_elem(struct bpf_map *map, void *key)
+{
+ struct xsk_map *m = container_of(map, struct xsk_map, map);
+ struct xsk_map_entry *old_entry;
+ int k = *(u32 *)key;
+
+ if (k >= map->max_entries)
+ return -EINVAL;
+
+ old_entry = xchg(&m->xsk_map[k], NULL);
+ if (old_entry)
+ call_rcu(&old_entry->rcu, __xsk_map_entry_free);
+
+ return 0;
+}
+
+const struct bpf_map_ops xsk_map_ops = {
+ .map_alloc = xsk_map_alloc,
+ .map_free = xsk_map_free,
+ .map_get_next_key = xsk_map_get_next_key,
+ .map_lookup_elem = xsk_map_lookup_elem,
+ .map_update_elem = xsk_map_update_elem,
+ .map_delete_elem = xsk_map_delete_elem,
+};
+
+
--
2.14.1
^ permalink raw reply related
* [RFC PATCH v2 08/14] xsk: wire up XDP_DRV side of AF_XDP
From: Björn Töpel @ 2018-03-27 16:59 UTC (permalink / raw)
To: bjorn.topel, magnus.karlsson, alexander.h.duyck, alexander.duyck,
john.fastabend, ast, brouer, willemdebruijn.kernel, daniel,
netdev
Cc: Björn Töpel, michael.lundkvist, jesse.brandeburg,
anjali.singhai, qi.z.zhang, ravineet.singh
In-Reply-To: <20180327165919.17933-1-bjorn.topel@gmail.com>
From: Björn Töpel <bjorn.topel@intel.com>
This commit wires up the xskmap to XDP_DRV layer.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
---
net/core/filter.c | 23 ++++++++++++++++++++---
1 file changed, 20 insertions(+), 3 deletions(-)
diff --git a/net/core/filter.c b/net/core/filter.c
index 00c711c5f1a2..4b09c0a02814 100644
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -2768,7 +2768,8 @@ static int __bpf_tx_xdp_map(struct net_device *dev_rx, void *fwd,
{
int err;
- if (map->map_type == BPF_MAP_TYPE_DEVMAP) {
+ switch (map->map_type) {
+ case BPF_MAP_TYPE_DEVMAP: {
struct net_device *dev = fwd;
if (!dev->netdev_ops->ndo_xdp_xmit)
@@ -2778,14 +2779,25 @@ static int __bpf_tx_xdp_map(struct net_device *dev_rx, void *fwd,
if (err)
return err;
__dev_map_insert_ctx(map, index);
-
- } else if (map->map_type == BPF_MAP_TYPE_CPUMAP) {
+ break;
+ }
+ case BPF_MAP_TYPE_CPUMAP: {
struct bpf_cpu_map_entry *rcpu = fwd;
err = cpu_map_enqueue(rcpu, xdp, dev_rx);
if (err)
return err;
__cpu_map_insert_ctx(map, index);
+ break;
+ }
+ case BPF_MAP_TYPE_XSKMAP: {
+ struct xdp_sock *xs = fwd;
+
+ err = __xsk_map_redirect(map, index, xdp, xs);
+ return err;
+ }
+ default:
+ break;
}
return 0;
}
@@ -2804,6 +2816,9 @@ void xdp_do_flush_map(void)
case BPF_MAP_TYPE_CPUMAP:
__cpu_map_flush(map);
break;
+ case BPF_MAP_TYPE_XSKMAP:
+ __xsk_map_flush(map);
+ break;
default:
break;
}
@@ -2818,6 +2833,8 @@ static void *__xdp_map_lookup_elem(struct bpf_map *map, u32 index)
return __dev_map_lookup_elem(map, index);
case BPF_MAP_TYPE_CPUMAP:
return __cpu_map_lookup_elem(map, index);
+ case BPF_MAP_TYPE_XSKMAP:
+ return __xsk_map_lookup_elem(map, index);
default:
return NULL;
}
--
2.14.1
^ permalink raw reply related
* [RFC PATCH v2 09/14] xsk: wire up XDP_SKB side of AF_XDP
From: Björn Töpel @ 2018-03-27 16:59 UTC (permalink / raw)
To: bjorn.topel, magnus.karlsson, alexander.h.duyck, alexander.duyck,
john.fastabend, ast, brouer, willemdebruijn.kernel, daniel,
netdev
Cc: Björn Töpel, michael.lundkvist, jesse.brandeburg,
anjali.singhai, qi.z.zhang, ravineet.singh
In-Reply-To: <20180327165919.17933-1-bjorn.topel@gmail.com>
From: Björn Töpel <bjorn.topel@intel.com>
This commit wires up the xskmap to XDP_SKB layer.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
---
include/linux/filter.h | 2 +-
include/net/xdp_sock.h | 6 ++++++
net/core/dev.c | 28 +++++++++++++++-------------
net/core/filter.c | 17 ++++++++++++++---
4 files changed, 36 insertions(+), 17 deletions(-)
diff --git a/include/linux/filter.h b/include/linux/filter.h
index 109d05ccea9a..6fa5e53d5fba 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -763,7 +763,7 @@ struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,
* This does not appear to be a real limitation for existing software.
*/
int xdp_do_generic_redirect(struct net_device *dev, struct sk_buff *skb,
- struct bpf_prog *prog);
+ struct xdp_buff *xdp, struct bpf_prog *prog);
int xdp_do_redirect(struct net_device *dev,
struct xdp_buff *xdp,
struct bpf_prog *prog);
diff --git a/include/net/xdp_sock.h b/include/net/xdp_sock.h
index 80d119a685b2..7b3b29ba9ff0 100644
--- a/include/net/xdp_sock.h
+++ b/include/net/xdp_sock.h
@@ -18,9 +18,15 @@
struct xdp_sock;
struct xdp_buff;
#ifdef CONFIG_XDP_SOCKETS
+int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp);
int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp);
void xsk_flush(struct xdp_sock *xs);
#else
+static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
+{
+ return -ENOTSUPP;
+}
+
static inline int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
{
return -ENOTSUPP;
diff --git a/net/core/dev.c b/net/core/dev.c
index 97a96df4b6da..3632d959af1b 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -3985,11 +3985,11 @@ static struct netdev_rx_queue *netif_get_rxqueue(struct sk_buff *skb)
}
static u32 netif_receive_generic_xdp(struct sk_buff *skb,
+ struct xdp_buff *xdp,
struct bpf_prog *xdp_prog)
{
struct netdev_rx_queue *rxqueue;
u32 metalen, act = XDP_DROP;
- struct xdp_buff xdp;
void *orig_data;
int hlen, off;
u32 mac_len;
@@ -4025,18 +4025,18 @@ static u32 netif_receive_generic_xdp(struct sk_buff *skb,
*/
mac_len = skb->data - skb_mac_header(skb);
hlen = skb_headlen(skb) + mac_len;
- xdp.data = skb->data - mac_len;
- xdp.data_meta = xdp.data;
- xdp.data_end = xdp.data + hlen;
- xdp.data_hard_start = skb->data - skb_headroom(skb);
- orig_data = xdp.data;
+ xdp->data = skb->data - mac_len;
+ xdp->data_meta = xdp->data;
+ xdp->data_end = xdp->data + hlen;
+ xdp->data_hard_start = skb->data - skb_headroom(skb);
+ orig_data = xdp->data;
rxqueue = netif_get_rxqueue(skb);
- xdp.rxq = &rxqueue->xdp_rxq;
+ xdp->rxq = &rxqueue->xdp_rxq;
- act = bpf_prog_run_xdp(xdp_prog, &xdp);
+ act = bpf_prog_run_xdp(xdp_prog, xdp);
- off = xdp.data - orig_data;
+ off = xdp->data - orig_data;
if (off > 0)
__skb_pull(skb, off);
else if (off < 0)
@@ -4049,7 +4049,7 @@ static u32 netif_receive_generic_xdp(struct sk_buff *skb,
__skb_push(skb, mac_len);
break;
case XDP_PASS:
- metalen = xdp.data - xdp.data_meta;
+ metalen = xdp->data - xdp->data_meta;
if (metalen)
skb_metadata_set(skb, metalen);
break;
@@ -4099,17 +4099,19 @@ static struct static_key generic_xdp_needed __read_mostly;
int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb)
{
if (xdp_prog) {
- u32 act = netif_receive_generic_xdp(skb, xdp_prog);
+ struct xdp_buff xdp;
+ u32 act;
int err;
+ act = netif_receive_generic_xdp(skb, &xdp, xdp_prog);
if (act != XDP_PASS) {
switch (act) {
case XDP_REDIRECT:
err = xdp_do_generic_redirect(skb->dev, skb,
- xdp_prog);
+ &xdp, xdp_prog);
if (err)
goto out_redir;
- /* fallthru to submit skb */
+ break;
case XDP_TX:
generic_xdp_tx(skb, xdp_prog);
break;
diff --git a/net/core/filter.c b/net/core/filter.c
index 4b09c0a02814..fc71b53542d2 100644
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -57,6 +57,7 @@
#include <net/busy_poll.h>
#include <net/tcp.h>
#include <linux/bpf_trace.h>
+#include <net/xdp_sock.h>
/**
* sk_filter_trim_cap - run a packet through a socket filter
@@ -2932,13 +2933,14 @@ static int __xdp_generic_ok_fwd_dev(struct sk_buff *skb, struct net_device *fwd)
static int xdp_do_generic_redirect_map(struct net_device *dev,
struct sk_buff *skb,
+ struct xdp_buff *xdp,
struct bpf_prog *xdp_prog)
{
struct redirect_info *ri = this_cpu_ptr(&redirect_info);
unsigned long map_owner = ri->map_owner;
struct bpf_map *map = ri->map;
- struct net_device *fwd = NULL;
u32 index = ri->ifindex;
+ void *fwd = NULL;
int err = 0;
ri->ifindex = 0;
@@ -2960,6 +2962,14 @@ static int xdp_do_generic_redirect_map(struct net_device *dev,
if (unlikely((err = __xdp_generic_ok_fwd_dev(skb, fwd))))
goto err;
skb->dev = fwd;
+ generic_xdp_tx(skb, xdp_prog);
+ } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) {
+ struct xdp_sock *xs = fwd;
+
+ err = xsk_generic_rcv(xs, xdp);
+ if (err)
+ goto err;
+ consume_skb(skb);
} else {
/* TODO: Handle BPF_MAP_TYPE_CPUMAP */
err = -EBADRQC;
@@ -2974,7 +2984,7 @@ static int xdp_do_generic_redirect_map(struct net_device *dev,
}
int xdp_do_generic_redirect(struct net_device *dev, struct sk_buff *skb,
- struct bpf_prog *xdp_prog)
+ struct xdp_buff *xdp, struct bpf_prog *xdp_prog)
{
struct redirect_info *ri = this_cpu_ptr(&redirect_info);
u32 index = ri->ifindex;
@@ -2982,7 +2992,7 @@ int xdp_do_generic_redirect(struct net_device *dev, struct sk_buff *skb,
int err = 0;
if (ri->map)
- return xdp_do_generic_redirect_map(dev, skb, xdp_prog);
+ return xdp_do_generic_redirect_map(dev, skb, xdp, xdp_prog);
ri->ifindex = 0;
fwd = dev_get_by_index_rcu(dev_net(dev), index);
@@ -2996,6 +3006,7 @@ int xdp_do_generic_redirect(struct net_device *dev, struct sk_buff *skb,
skb->dev = fwd;
_trace_xdp_redirect(dev, xdp_prog, index);
+ generic_xdp_tx(skb, xdp_prog);
return 0;
err:
_trace_xdp_redirect_err(dev, xdp_prog, index, err);
--
2.14.1
^ permalink raw reply related
* [RFC PATCH v2 10/14] xsk: add umem completion queue support and mmap
From: Björn Töpel @ 2018-03-27 16:59 UTC (permalink / raw)
To: bjorn.topel, magnus.karlsson, alexander.h.duyck, alexander.duyck,
john.fastabend, ast, brouer, willemdebruijn.kernel, daniel,
netdev
Cc: michael.lundkvist, jesse.brandeburg, anjali.singhai, qi.z.zhang,
ravineet.singh
In-Reply-To: <20180327165919.17933-1-bjorn.topel@gmail.com>
From: Magnus Karlsson <magnus.karlsson@intel.com>
Here, we add another setsockopt for registered user memory (umem)
called XDP_UMEM_COMPLETION_QUEUE. Using this socket option, the
process can ask the kernel to allocate a queue (ring buffer) and also
mmap it (XDP_UMEM_PGOFF_COMPLETION_QUEUE) into the process.
The queue is used to explicitly pass ownership of umem frames from the
kernel to user process. This will be used by the TX path to tell user
space that a certain frame has been transmitted and user space can use
it for something else, if it wishes.
Signed-off-by: Magnus Karlsson <magnus.karlsson@intel.com>
---
include/uapi/linux/if_xdp.h | 2 ++
net/xdp/xdp_umem.c | 7 ++++++-
net/xdp/xdp_umem.h | 1 +
net/xdp/xsk.c | 6 +++++-
4 files changed, 14 insertions(+), 2 deletions(-)
diff --git a/include/uapi/linux/if_xdp.h b/include/uapi/linux/if_xdp.h
index e3593df7323c..aec2eee4e751 100644
--- a/include/uapi/linux/if_xdp.h
+++ b/include/uapi/linux/if_xdp.h
@@ -35,6 +35,7 @@ struct sockaddr_xdp {
#define XDP_RX_QUEUE 1
#define XDP_UMEM_REG 3
#define XDP_UMEM_FILL_QUEUE 4
+#define XDP_UMEM_COMPLETION_QUEUE 5
struct xdp_umem_reg {
__u64 addr; /* Start of packet data area */
@@ -46,6 +47,7 @@ struct xdp_umem_reg {
/* Pgoff for mmaping the rings */
#define XDP_PGOFF_RX_QUEUE 0
#define XDP_UMEM_PGOFF_FILL_QUEUE 0x100000000
+#define XDP_UMEM_PGOFF_COMPLETION_QUEUE 0x180000000
struct xdp_desc {
__u32 idx;
diff --git a/net/xdp/xdp_umem.c b/net/xdp/xdp_umem.c
index 96395beb980e..8912022d22ee 100644
--- a/net/xdp/xdp_umem.c
+++ b/net/xdp/xdp_umem.c
@@ -71,6 +71,11 @@ static void xdp_umem_release(struct xdp_umem *umem)
umem->fq = NULL;
}
+ if (umem->cq) {
+ xskq_destroy(umem->cq);
+ umem->cq = NULL;
+ }
+
if (umem->pgs) {
xdp_umem_unpin_pages(umem);
@@ -250,5 +255,5 @@ int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr)
bool xdp_umem_validate_queues(struct xdp_umem *umem)
{
- return umem->fq;
+ return (umem->fq && umem->cq);
}
diff --git a/net/xdp/xdp_umem.h b/net/xdp/xdp_umem.h
index 5e7105b7760b..3eda63797516 100644
--- a/net/xdp/xdp_umem.h
+++ b/net/xdp/xdp_umem.h
@@ -23,6 +23,7 @@
struct xdp_umem {
struct xsk_queue *fq;
+ struct xsk_queue *cq;
struct pid *pid;
struct page **pgs;
unsigned long address;
diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c
index a60b1fcfb2b3..2071365c39b1 100644
--- a/net/xdp/xsk.c
+++ b/net/xdp/xsk.c
@@ -347,6 +347,7 @@ static int xsk_setsockopt(struct socket *sock, int level, int optname,
return 0;
}
case XDP_UMEM_FILL_QUEUE:
+ case XDP_UMEM_COMPLETION_QUEUE:
{
struct xsk_queue **q;
int entries;
@@ -358,7 +359,8 @@ static int xsk_setsockopt(struct socket *sock, int level, int optname,
return -EFAULT;
mutex_lock(&xs->mutex);
- q = &xs->umem->fq;
+ q = (optname == XDP_UMEM_FILL_QUEUE) ? &xs->umem->fq :
+ &xs->umem->cq;
err = xsk_init_queue(entries, q, true);
mutex_unlock(&xs->mutex);
return err;
@@ -389,6 +391,8 @@ static int xsk_mmap(struct file *file, struct socket *sock,
if (offset == XDP_UMEM_PGOFF_FILL_QUEUE)
q = xs->umem->fq;
+ else if (offset == XDP_UMEM_PGOFF_COMPLETION_QUEUE)
+ q = xs->umem->cq;
else
return -EINVAL;
}
--
2.14.1
^ permalink raw reply related
* [RFC PATCH v2 11/14] xsk: add Tx queue setup and mmap support
From: Björn Töpel @ 2018-03-27 16:59 UTC (permalink / raw)
To: bjorn.topel, magnus.karlsson, alexander.h.duyck, alexander.duyck,
john.fastabend, ast, brouer, willemdebruijn.kernel, daniel,
netdev
Cc: michael.lundkvist, jesse.brandeburg, anjali.singhai, qi.z.zhang,
ravineet.singh
In-Reply-To: <20180327165919.17933-1-bjorn.topel@gmail.com>
From: Magnus Karlsson <magnus.karlsson@intel.com>
Another setsockopt (XDP_TX_QUEUE) is added to let the process allocate
a queue, where the user process can pass frames to be transmitted by
the kernel.
The mmapping of the queue is done using the XDP_PGOFF_TX_QUEUE offset.
Signed-off-by: Magnus Karlsson <magnus.karlsson@intel.com>
---
include/uapi/linux/if_xdp.h | 2 ++
net/xdp/xsk.c | 9 +++++++--
2 files changed, 9 insertions(+), 2 deletions(-)
diff --git a/include/uapi/linux/if_xdp.h b/include/uapi/linux/if_xdp.h
index aec2eee4e751..6cab56fb8be6 100644
--- a/include/uapi/linux/if_xdp.h
+++ b/include/uapi/linux/if_xdp.h
@@ -33,6 +33,7 @@ struct sockaddr_xdp {
/* XDP socket options */
#define XDP_RX_QUEUE 1
+#define XDP_TX_QUEUE 2
#define XDP_UMEM_REG 3
#define XDP_UMEM_FILL_QUEUE 4
#define XDP_UMEM_COMPLETION_QUEUE 5
@@ -46,6 +47,7 @@ struct xdp_umem_reg {
/* Pgoff for mmaping the rings */
#define XDP_PGOFF_RX_QUEUE 0
+#define XDP_PGOFF_TX_QUEUE 0x80000000
#define XDP_UMEM_PGOFF_FILL_QUEUE 0x100000000
#define XDP_UMEM_PGOFF_COMPLETION_QUEUE 0x180000000
diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c
index 2071365c39b1..a9bceb0958d8 100644
--- a/net/xdp/xsk.c
+++ b/net/xdp/xsk.c
@@ -43,6 +43,7 @@ struct xdp_sock {
struct xsk_queue *rx;
struct xskq_iter rx_it;
u64 rx_dropped;
+ struct xsk_queue *tx;
struct net_device *dev;
/* Protects multiple processes in the control path */
struct mutex mutex;
@@ -218,7 +219,7 @@ static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len)
goto out_release;
}
- if (!xs->rx) {
+ if (!xs->rx && !xs->tx) {
err = -EINVAL;
goto out_unlock;
}
@@ -303,6 +304,7 @@ static int xsk_setsockopt(struct socket *sock, int level, int optname,
switch (optname) {
case XDP_RX_QUEUE:
+ case XDP_TX_QUEUE:
{
struct xsk_queue **q;
int entries;
@@ -313,7 +315,7 @@ static int xsk_setsockopt(struct socket *sock, int level, int optname,
return -EFAULT;
mutex_lock(&xs->mutex);
- q = &xs->rx;
+ q = (optname == XDP_TX_QUEUE) ? &xs->tx : &xs->rx;
err = xsk_init_queue(entries, q, false);
mutex_unlock(&xs->mutex);
return err;
@@ -385,6 +387,8 @@ static int xsk_mmap(struct file *file, struct socket *sock,
if (offset == XDP_PGOFF_RX_QUEUE) {
q = xs->rx;
+ } else if (offset == XDP_PGOFF_TX_QUEUE) {
+ q = xs->tx;
} else {
if (!xs->umem)
return -EINVAL;
@@ -443,6 +447,7 @@ static void xsk_destruct(struct sock *sk)
return;
xskq_destroy(xs->rx);
+ xskq_destroy(xs->tx);
xdp_put_umem(xs->umem);
sk_refcnt_debug_dec(sk);
--
2.14.1
^ permalink raw reply related
* [RFC PATCH v2 12/14] xsk: support for Tx
From: Björn Töpel @ 2018-03-27 16:59 UTC (permalink / raw)
To: bjorn.topel, magnus.karlsson, alexander.h.duyck, alexander.duyck,
john.fastabend, ast, brouer, willemdebruijn.kernel, daniel,
netdev
Cc: michael.lundkvist, jesse.brandeburg, anjali.singhai, qi.z.zhang,
ravineet.singh
In-Reply-To: <20180327165919.17933-1-bjorn.topel@gmail.com>
From: Magnus Karlsson <magnus.karlsson@intel.com>
Here, Tx support is added. The user fills the Tx queue with frames to
be sent by the kernel, and let's the kernel know using the sendmsg
syscall.
Signed-off-by: Magnus Karlsson <magnus.karlsson@intel.com>
---
net/xdp/xsk.c | 151 +++++++++++++++++++++++++++++++++++++++++++++++++++-
net/xdp/xsk_queue.h | 33 ++++++++++++
2 files changed, 183 insertions(+), 1 deletion(-)
diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c
index a9bceb0958d8..685b6f360628 100644
--- a/net/xdp/xsk.c
+++ b/net/xdp/xsk.c
@@ -35,6 +35,7 @@
#include "xsk_queue.h"
#include "xdp_umem.h"
+#define TX_BATCH_SIZE 16
#define RX_BATCH_SIZE 16
struct xdp_sock {
@@ -44,10 +45,12 @@ struct xdp_sock {
struct xskq_iter rx_it;
u64 rx_dropped;
struct xsk_queue *tx;
+ struct xskq_iter tx_it;
struct net_device *dev;
/* Protects multiple processes in the control path */
struct mutex mutex;
struct xdp_umem *umem;
+ u32 tx_umem_head;
u32 ifindex;
u16 queue_id;
};
@@ -112,6 +115,146 @@ int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
return err;
}
+static void xsk_destruct_skb(struct sk_buff *skb)
+{
+ u32 buff_id = (u32)(long)skb_shinfo(skb)->destructor_arg;
+ struct xdp_sock *xs = xdp_sk(skb->sk);
+
+ WARN_ON_ONCE(xdp_umem_enq_one(xs->umem->cq, buff_id));
+
+ sock_wfree(skb);
+}
+
+static int xsk_xmit_skb(struct sk_buff *skb)
+{
+ struct net_device *dev = skb->dev;
+ struct sk_buff *orig_skb = skb;
+ struct netdev_queue *txq;
+ int ret = NETDEV_TX_BUSY;
+ bool again = false;
+
+ if (unlikely(!netif_running(dev) || !netif_carrier_ok(dev)))
+ goto drop;
+
+ skb = validate_xmit_skb_list(skb, dev, &again);
+ if (skb != orig_skb)
+ return NET_XMIT_DROP;
+
+ txq = skb_get_tx_queue(dev, skb);
+
+ local_bh_disable();
+
+ HARD_TX_LOCK(dev, txq, smp_processor_id());
+ if (!netif_xmit_frozen_or_drv_stopped(txq))
+ ret = netdev_start_xmit(skb, dev, txq, false);
+ HARD_TX_UNLOCK(dev, txq);
+
+ local_bh_enable();
+
+ if (!dev_xmit_complete(ret))
+ goto out_err;
+
+ return ret;
+drop:
+ atomic_long_inc(&dev->tx_dropped);
+out_err:
+ return NET_XMIT_DROP;
+}
+
+static int xsk_generic_xmit(struct sock *sk, struct msghdr *m,
+ size_t total_len)
+{
+ bool need_wait = !(m->msg_flags & MSG_DONTWAIT);
+ struct xdp_sock *xs = xdp_sk(sk);
+ bool sent_frame = false;
+ struct sk_buff *skb;
+ u32 max_outstanding;
+ int err = 0;
+
+ if (unlikely(!xs->tx))
+ return -ENOBUFS;
+ if (need_wait)
+ return -EOPNOTSUPP;
+
+ mutex_lock(&xs->mutex);
+
+ max_outstanding = xskq_nb_free(xs->umem->cq, xs->tx_umem_head,
+ TX_BATCH_SIZE);
+
+ while (xskq_next_frame_deq(xs->tx, &xs->tx_it, TX_BATCH_SIZE)) {
+ char *buffer;
+ u32 id, len;
+
+ if (max_outstanding-- == 0) {
+ err = -EAGAIN;
+ goto out_err;
+ }
+
+ len = xskq_rxtx_get_len(&xs->tx_it);
+ if (unlikely(len > xs->dev->mtu)) {
+ err = -EMSGSIZE;
+ goto out_err;
+ }
+
+ skb = sock_alloc_send_skb(sk, len, !need_wait, &err);
+ if (unlikely(!skb)) {
+ err = -EAGAIN;
+ goto out_err;
+ }
+
+ skb_put(skb, len);
+ id = xskq_rxtx_get_id(&xs->tx_it);
+ buffer = xdp_umem_get_data(xs->umem, id) +
+ xskq_rxtx_get_offset(&xs->tx_it);
+ err = skb_store_bits(skb, 0, buffer, len);
+ if (unlikely(err))
+ goto out_store;
+
+ skb->dev = xs->dev;
+ skb->priority = sk->sk_priority;
+ skb->mark = sk->sk_mark;
+ skb_set_queue_mapping(skb, xs->queue_id);
+ skb_shinfo(skb)->destructor_arg = (void *)(long)id;
+ skb->destructor = xsk_destruct_skb;
+
+ err = xsk_xmit_skb(skb);
+ /* Ignore NET_XMIT_CN as packet might have been sent */
+ if (err == NET_XMIT_DROP || err == NETDEV_TX_BUSY) {
+ err = -EAGAIN;
+ goto out_store;
+ }
+
+ xs->tx_umem_head++;
+ sent_frame = true;
+ }
+
+ goto out;
+
+out_store:
+ kfree_skb(skb);
+out_err:
+ xskq_deq_return_frame(&xs->tx_it);
+out:
+ if (sent_frame)
+ sk->sk_write_space(sk);
+
+ mutex_unlock(&xs->mutex);
+ return err;
+}
+
+static int xsk_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len)
+{
+ struct sock *sk = sock->sk;
+ struct xdp_sock *xs = xdp_sk(sk);
+
+ if (unlikely(!xs->dev))
+ return -ENXIO;
+ if (unlikely(!(xs->dev->flags & IFF_UP)))
+ return -ENETDOWN;
+
+ return xsk_generic_xmit(sk, m, total_len);
+}
+
static unsigned int xsk_poll(struct file *file, struct socket *sock,
struct poll_table_struct *wait)
{
@@ -121,6 +264,8 @@ static unsigned int xsk_poll(struct file *file, struct socket *sock,
if (xs->rx && !xskq_empty(xs->rx))
mask |= POLLIN | POLLRDNORM;
+ if (xs->tx && !xskq_full(xs->tx))
+ mask |= POLLOUT | POLLWRNORM;
return mask;
}
@@ -266,6 +411,7 @@ static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len)
} else {
/* This xsk has its own umem. */
xskq_set_umem(xs->umem->fq, &xs->umem->props);
+ xskq_set_umem(xs->umem->cq, &xs->umem->props);
}
/* Rebind? */
@@ -281,8 +427,11 @@ static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len)
xs->queue_id = sxdp->sxdp_queue_id;
xskq_init_iter(&xs->rx_it);
+ xskq_init_iter(&xs->tx_it);
+ xs->tx_umem_head = 0;
xskq_set_umem(xs->rx, &xs->umem->props);
+ xskq_set_umem(xs->tx, &xs->umem->props);
out_unlock:
if (err)
@@ -433,7 +582,7 @@ static const struct proto_ops xsk_proto_ops = {
.shutdown = sock_no_shutdown,
.setsockopt = xsk_setsockopt,
.getsockopt = sock_no_getsockopt,
- .sendmsg = sock_no_sendmsg,
+ .sendmsg = xsk_sendmsg,
.recvmsg = sock_no_recvmsg,
.mmap = xsk_mmap,
.sendpage = sock_no_sendpage,
diff --git a/net/xdp/xsk_queue.h b/net/xdp/xsk_queue.h
index af6e651f1207..94edc7e7a503 100644
--- a/net/xdp/xsk_queue.h
+++ b/net/xdp/xsk_queue.h
@@ -209,6 +209,11 @@ static inline void xskq_deq_return_frame(struct xskq_iter *it)
it->tail--;
}
+static inline void xskq_enq_return_frame(struct xsk_queue *q)
+{
+ q->iter_head_idx--;
+}
+
static inline void xskq_enq_flush(struct xsk_queue *q)
{
/* Order flags and data */
@@ -226,6 +231,21 @@ static inline u32 xskq_rxtx_get_ring_size(struct xsk_queue *q)
q->nentries * sizeof(struct xdp_desc));
}
+static inline u32 xskq_rxtx_get_id(struct xskq_iter *it)
+{
+ return it->desc_copy.idx;
+}
+
+static inline u32 xskq_rxtx_get_len(struct xskq_iter *it)
+{
+ return it->desc_copy.len;
+}
+
+static inline u32 xskq_rxtx_get_offset(struct xskq_iter *it)
+{
+ return it->desc_copy.offset;
+}
+
static inline int xskq_rxtx_enq_frame(struct xsk_queue *q,
u32 id, u32 len, u16 offset)
{
@@ -258,6 +278,19 @@ static inline u32 xdp_umem_get_id(struct xsk_queue *q,
return ring->desc[it->tail & q->ring_mask];
}
+static inline int xdp_umem_enq_one(struct xsk_queue *q, u32 idx)
+{
+ struct xdp_umem_queue *ring = (struct xdp_umem_queue *)q->ring;
+
+ if (xskq_nb_free(q, q->iter_head_idx, 1) == 0)
+ return -ENOSPC;
+
+ ring->desc[q->iter_head_idx++ & q->ring_mask] = idx;
+
+ xskq_enq_flush(q);
+ return 0;
+}
+
struct xsk_queue *xskq_create(u32 nentries, bool umem_queue);
void xskq_destroy(struct xsk_queue *q_ops);
--
2.14.1
^ permalink raw reply related
* Re: [PATCH iproute2] Revert "iproute: "list/flush/save default" selected all of the routes"
From: Stephen Hemminger @ 2018-03-27 17:00 UTC (permalink / raw)
To: Alexander Zubkov; +Cc: Serhey Popovych, Luca Boccassi, netdev@vger.kernel.org
In-Reply-To: <606551522168171@web35g.yandex.ru>
On Tue, 27 Mar 2018 18:29:31 +0200
Alexander Zubkov <green@msu.ru> wrote:
> Hi Stephen,
>
> Looks like the new patch was applied after the revert of original patch and fix patch for 4.15 branch. Which is not correct and I did not test it. This is how patches were designed:
> 1) your revert patch - rolls back 4.15 branch to old behaviour by reverting the original patch
> 2) my patch for 4.15 - fixes problem is 4.15 branch, it does not require revert patch, it is an alternative solution for the problem, it is designed solely for version 4.15
> 3) my patch for master - fixes problem, it requires neither revert patch nor my patch for 4.15, it is standalone patch designed to do things right in master branch
>
> 27.03.2018, 18:01, "Stephen Hemminger" <stephen@networkplumber.org>:
> > On Wed, 14 Mar 2018 21:26:40 +0100
> > Alexander Zubkov <green@msu.ru> wrote:
> >
> >> Hello,
> >>
> >> For example, it can be fixed in such way (patch is below):
> >> - split handling of default and all/any
> >> - set needed attributes in get_addr: PREFIXLEN_SPECIFIED flag for default
> >> - and AF_UNSPEC for all/any
> >> In this case "ip route show default" shows only default route and "ip
> >> route show all" shows all routes. And both also work when family (-4 or
> >> -6) is specified.
> >> Serhey, does it goes in line with what you wanted to achieve? Because I
> >> do not know - may be there are reasons why all/any should be provided
> >> with specific family. If you think this solution is suitable, I'll do
> >> some additional tests and package the patch in a proper way for this
> >> mailing list.
> >> And I'm unsure if check for AF_DECnet and AF_MPLS should be kept in both
> >> branches. May be someone have some additional thoughts on that?
> >
> > I applied this to master.
> >
> > We can work on the other cases after that.
Please send the update back to what works.
^ permalink raw reply
* [RFC PATCH v2 13/14] xsk: statistics support
From: Björn Töpel @ 2018-03-27 16:59 UTC (permalink / raw)
To: bjorn.topel, magnus.karlsson, alexander.h.duyck, alexander.duyck,
john.fastabend, ast, brouer, willemdebruijn.kernel, daniel,
netdev
Cc: michael.lundkvist, jesse.brandeburg, anjali.singhai, qi.z.zhang,
ravineet.singh
In-Reply-To: <20180327165919.17933-1-bjorn.topel@gmail.com>
From: Magnus Karlsson <magnus.karlsson@intel.com>
In this commit, a new getsockopt is added: XDP_STATISTICS. This is
used to obtain stats from the sockets.
Signed-off-by: Magnus Karlsson <magnus.karlsson@intel.com>
---
include/uapi/linux/if_xdp.h | 7 +++++++
net/xdp/xsk.c | 42 +++++++++++++++++++++++++++++++++++++++++-
2 files changed, 48 insertions(+), 1 deletion(-)
diff --git a/include/uapi/linux/if_xdp.h b/include/uapi/linux/if_xdp.h
index 6cab56fb8be6..4d8142c1596d 100644
--- a/include/uapi/linux/if_xdp.h
+++ b/include/uapi/linux/if_xdp.h
@@ -37,6 +37,7 @@ struct sockaddr_xdp {
#define XDP_UMEM_REG 3
#define XDP_UMEM_FILL_QUEUE 4
#define XDP_UMEM_COMPLETION_QUEUE 5
+#define XDP_STATISTICS 6
struct xdp_umem_reg {
__u64 addr; /* Start of packet data area */
@@ -45,6 +46,12 @@ struct xdp_umem_reg {
__u32 frame_headroom; /* Frame head room */
};
+struct xdp_statistics {
+ __u64 rx_dropped; /* Dropped for reasons other than invalid desc */
+ __u64 rx_invalid_descs; /* Dropped due to invalid descriptor */
+ __u64 tx_invalid_descs; /* Dropped due to invalid descriptor */
+};
+
/* Pgoff for mmaping the rings */
#define XDP_PGOFF_RX_QUEUE 0
#define XDP_PGOFF_TX_QUEUE 0x80000000
diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c
index 685b6f360628..780a5b42d9b2 100644
--- a/net/xdp/xsk.c
+++ b/net/xdp/xsk.c
@@ -523,6 +523,46 @@ static int xsk_setsockopt(struct socket *sock, int level, int optname,
return -ENOPROTOOPT;
}
+static int xsk_getsockopt(struct socket *sock, int level, int optname,
+ char __user *optval, int __user *optlen)
+{
+ struct sock *sk = sock->sk;
+ struct xdp_sock *xs = xdp_sk(sk);
+ int len;
+
+ if (level != SOL_XDP)
+ return -ENOPROTOOPT;
+
+ if (get_user(len, optlen))
+ return -EFAULT;
+ if (len < 0)
+ return -EINVAL;
+
+ switch (optname) {
+ case XDP_STATISTICS:
+ {
+ struct xdp_statistics stats;
+
+ if (len != sizeof(stats))
+ return -EINVAL;
+
+ mutex_lock(&xs->mutex);
+ stats.rx_dropped = xs->rx_dropped;
+ stats.rx_invalid_descs = xskq_nb_invalid_descs(xs->rx);
+ stats.tx_invalid_descs = xskq_nb_invalid_descs(xs->tx);
+ mutex_unlock(&xs->mutex);
+
+ if (copy_to_user(optval, &stats, sizeof(stats)))
+ return -EFAULT;
+ return 0;
+ }
+ default:
+ break;
+ }
+
+ return -EOPNOTSUPP;
+}
+
static int xsk_mmap(struct file *file, struct socket *sock,
struct vm_area_struct *vma)
{
@@ -581,7 +621,7 @@ static const struct proto_ops xsk_proto_ops = {
.listen = sock_no_listen,
.shutdown = sock_no_shutdown,
.setsockopt = xsk_setsockopt,
- .getsockopt = sock_no_getsockopt,
+ .getsockopt = xsk_getsockopt,
.sendmsg = xsk_sendmsg,
.recvmsg = sock_no_recvmsg,
.mmap = xsk_mmap,
--
2.14.1
^ permalink raw reply related
* [RFC PATCH v2 14/14] samples/bpf: sample application for AF_XDP sockets
From: Björn Töpel @ 2018-03-27 16:59 UTC (permalink / raw)
To: bjorn.topel, magnus.karlsson, alexander.h.duyck, alexander.duyck,
john.fastabend, ast, brouer, willemdebruijn.kernel, daniel,
netdev
Cc: michael.lundkvist, jesse.brandeburg, anjali.singhai, qi.z.zhang,
ravineet.singh, Björn Töpel
In-Reply-To: <20180327165919.17933-1-bjorn.topel@gmail.com>
From: Magnus Karlsson <magnus.karlsson@intel.com>
This is a sample application for AF_XDP sockets. The application
supports three different modes of operation: rxdrop, txonly and l2fwd.
To show-case a simple round-robin load-balancing between a set of
sockets in an xskmap, set the RR_LB compile time define option to 1 in
"xdpsock.h".
Co-authored-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Magnus Karlsson <magnus.karlsson@intel.com>
---
samples/bpf/Makefile | 4 +
samples/bpf/xdpsock.h | 10 +
samples/bpf/xdpsock_kern.c | 55 +++
samples/bpf/xdpsock_user.c | 923 +++++++++++++++++++++++++++++++++++++++++++++
4 files changed, 992 insertions(+)
create mode 100644 samples/bpf/xdpsock.h
create mode 100644 samples/bpf/xdpsock_kern.c
create mode 100644 samples/bpf/xdpsock_user.c
diff --git a/samples/bpf/Makefile b/samples/bpf/Makefile
index 2c2a587e0942..11c60c791228 100644
--- a/samples/bpf/Makefile
+++ b/samples/bpf/Makefile
@@ -44,6 +44,7 @@ hostprogs-y += xdp_monitor
hostprogs-y += xdp_rxq_info
hostprogs-y += syscall_tp
hostprogs-y += cpustat
+hostprogs-y += xdpsock
# Libbpf dependencies
LIBBPF := ../../tools/lib/bpf/bpf.o ../../tools/lib/bpf/nlattr.o
@@ -95,6 +96,7 @@ xdp_monitor-objs := bpf_load.o $(LIBBPF) xdp_monitor_user.o
xdp_rxq_info-objs := bpf_load.o $(LIBBPF) xdp_rxq_info_user.o
syscall_tp-objs := bpf_load.o $(LIBBPF) syscall_tp_user.o
cpustat-objs := bpf_load.o $(LIBBPF) cpustat_user.o
+xdpsock-objs := bpf_load.o $(LIBBPF) xdpsock_user.o
# Tell kbuild to always build the programs
always := $(hostprogs-y)
@@ -147,6 +149,7 @@ always += xdp_rxq_info_kern.o
always += xdp2skb_meta_kern.o
always += syscall_tp_kern.o
always += cpustat_kern.o
+always += xdpsock_kern.o
HOSTCFLAGS += -I$(objtree)/usr/include
HOSTCFLAGS += -I$(srctree)/tools/lib/
@@ -192,6 +195,7 @@ HOSTLOADLIBES_xdp_monitor += -lelf
HOSTLOADLIBES_xdp_rxq_info += -lelf
HOSTLOADLIBES_syscall_tp += -lelf
HOSTLOADLIBES_cpustat += -lelf
+HOSTLOADLIBES_xdpsock += -lelf -pthread
# Allows pointing LLC/CLANG to a LLVM backend with bpf support, redefine on cmdline:
# make samples/bpf/ LLC=~/git/llvm/build/bin/llc CLANG=~/git/llvm/build/bin/clang
diff --git a/samples/bpf/xdpsock.h b/samples/bpf/xdpsock.h
new file mode 100644
index 000000000000..4479233dcaae
--- /dev/null
+++ b/samples/bpf/xdpsock.h
@@ -0,0 +1,10 @@
+#ifndef XDPSOCK_H_
+#define XDPSOCK_H_
+
+/* Power-of-2 number of sockets */
+#define MAX_SOCKS 4
+
+/* Round-robin receive */
+#define RR_LB 0
+
+#endif /* XDPSOCK_H_ */
diff --git a/samples/bpf/xdpsock_kern.c b/samples/bpf/xdpsock_kern.c
new file mode 100644
index 000000000000..69af48842182
--- /dev/null
+++ b/samples/bpf/xdpsock_kern.c
@@ -0,0 +1,55 @@
+#define KBUILD_MODNAME "foo"
+#include <uapi/linux/bpf.h>
+#include "bpf_helpers.h"
+
+#include "xdpsock.h"
+
+struct bpf_map_def SEC("maps") qidconf_map = {
+ .type = BPF_MAP_TYPE_ARRAY,
+ .key_size = sizeof(int),
+ .value_size = sizeof(int),
+ .max_entries = 1,
+};
+
+struct bpf_map_def SEC("maps") xsks_map = {
+ .type = BPF_MAP_TYPE_XSKMAP,
+ .key_size = sizeof(int),
+ .value_size = sizeof(int),
+ .max_entries = 4,
+};
+
+struct bpf_map_def SEC("maps") rr_map = {
+ .type = BPF_MAP_TYPE_PERCPU_ARRAY,
+ .key_size = sizeof(int),
+ .value_size = sizeof(unsigned int),
+ .max_entries = 1,
+};
+
+SEC("xdp_sock")
+int xdp_sock_prog(struct xdp_md *ctx)
+{
+ int *qidconf, key = 0, idx;
+ unsigned int *rr;
+
+ qidconf = bpf_map_lookup_elem(&qidconf_map, &key);
+ if (!qidconf)
+ return XDP_ABORTED;
+
+ if (*qidconf != ctx->rx_queue_index)
+ return XDP_PASS;
+
+#if RR_LB /* NB! RR_LB is configured in xdpsock.h */
+ rr = bpf_map_lookup_elem(&rr_map, &key);
+ if (!rr)
+ return XDP_ABORTED;
+
+ *rr = (*rr + 1) & (MAX_SOCKS - 1);
+ idx = *rr;
+#else
+ idx = 0;
+#endif
+
+ return bpf_redirect_map(&xsks_map, idx, 0);
+}
+
+char _license[] SEC("license") = "GPL";
diff --git a/samples/bpf/xdpsock_user.c b/samples/bpf/xdpsock_user.c
new file mode 100644
index 000000000000..e7639a560e85
--- /dev/null
+++ b/samples/bpf/xdpsock_user.c
@@ -0,0 +1,923 @@
+/*
+ * Copyright(c) 2017 - 2018 Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ */
+
+#include <assert.h>
+#include <errno.h>
+#include <getopt.h>
+#include <libgen.h>
+#include <linux/bpf.h>
+#include <linux/if_link.h>
+#include <linux/if_xdp.h>
+#include <linux/if_ether.h>
+#include <net/if.h>
+#include <signal.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <net/ethernet.h>
+#include <sys/resource.h>
+#include <sys/socket.h>
+#include <sys/mman.h>
+#include <time.h>
+#include <unistd.h>
+#include <pthread.h>
+#include <locale.h>
+#include <sys/types.h>
+#include <poll.h>
+
+#include "bpf_load.h"
+#include "bpf_util.h"
+#include "libbpf.h"
+
+#include "xdpsock.h"
+
+#ifndef SOL_XDP
+#define SOL_XDP 283
+#endif
+
+#ifndef AF_XDP
+#define AF_XDP 44
+#endif
+
+#ifndef PF_XDP
+#define PF_XDP AF_XDP
+#endif
+
+#define NUM_FRAMES 131072
+#define FRAME_HEADROOM 0
+#define FRAME_SIZE 2048
+#define NUM_DESCS 1024
+#define BATCH_SIZE 16
+
+#define FQ_NUM_DESCS 1024
+#define CQ_NUM_DESCS 1024
+
+#define DEBUG_HEXDUMP 0
+
+typedef __u32 u32;
+
+static unsigned long start_time;
+
+enum benchmark_type {
+ BENCH_RXDROP = 0,
+ BENCH_TXONLY = 1,
+ BENCH_L2FWD = 2,
+};
+
+static enum benchmark_type opt_bench = BENCH_RXDROP;
+static u32 opt_xdp_flags;
+static const char *opt_if = "";
+static int opt_ifindex;
+static int opt_queue;
+static int opt_poll;
+static int opt_shared_packet_buffer;
+
+struct xdp_umem_uqueue {
+ u32 cached_head;
+ u32 cached_tail;
+ u32 mask;
+ u32 size;
+ struct xdp_umem_queue *q;
+};
+
+struct xdp_umem {
+ char (*frames)[FRAME_SIZE];
+ struct xdp_umem_uqueue fq;
+ struct xdp_umem_uqueue cq;
+ int fd;
+};
+
+struct xdp_uqueue {
+ u32 cached_head;
+ u32 cached_tail;
+ u32 mask;
+ u32 size;
+ struct xdp_rxtx_queue *q;
+};
+
+struct xdpsock {
+ struct xdp_uqueue rx;
+ struct xdp_uqueue tx;
+ int sfd;
+ struct xdp_umem *umem;
+ u32 outstanding_tx;
+ unsigned long rx_npkts;
+ unsigned long tx_npkts;
+};
+
+#define MAX_SOCKS 4
+static int num_socks;
+struct xdpsock *xsks[MAX_SOCKS];
+
+static unsigned long get_nsecs(void)
+{
+ struct timespec ts;
+
+ clock_gettime(CLOCK_MONOTONIC, &ts);
+ return ts.tv_sec * 1000000000UL + ts.tv_nsec;
+}
+
+static void dump_stats(void);
+
+#define lassert(expr) \
+ do { \
+ if (!(expr)) { \
+ fprintf(stderr, "%s:%s:%i: Assertion failed: " \
+ #expr ": errno: %d/\"%s\"\n", \
+ __FILE__, __func__, __LINE__, \
+ errno, strerror(errno)); \
+ dump_stats(); \
+ exit(EXIT_FAILURE); \
+ } \
+ } while (0)
+
+#define barrier() __asm__ __volatile__("": : :"memory")
+#define u_smp_rmb() barrier()
+#define u_smp_wmb() barrier()
+#define likely(x) __builtin_expect(!!(x), 1)
+#define unlikely(x) __builtin_expect(!!(x), 0)
+
+static const char pkt_data[] =
+ "\x3c\xfd\xfe\x9e\x7f\x71\xec\xb1\xd7\x98\x3a\xc0\x08\x00\x45\x00"
+ "\x00\x2e\x00\x00\x00\x00\x40\x11\x88\x97\x05\x08\x07\x08\xc8\x14"
+ "\x1e\x04\x10\x92\x10\x92\x00\x1a\x6d\xa3\x34\x33\x1f\x69\x40\x6b"
+ "\x54\x59\xb6\x14\x2d\x11\x44\xbf\xaf\xd9\xbe\xaa";
+
+static inline u32 umem_nb_free(struct xdp_umem_uqueue *q, u32 nb)
+{
+ u32 free_entries = q->size - (q->cached_head - q->cached_tail);
+
+ if (free_entries >= nb)
+ return free_entries;
+
+ /* Refresh the local tail pointer */
+ q->cached_tail = q->q->ptrs.tail_idx;
+
+ return q->size - (q->cached_head - q->cached_tail);
+}
+
+static inline u32 xq_nb_free(struct xdp_uqueue *q, u32 ndescs)
+{
+ u32 free_entries = q->cached_tail - q->cached_head;
+
+ if (free_entries >= ndescs)
+ return free_entries;
+
+ /* Refreash the local tail pointer */
+ q->cached_tail = q->q->ptrs.tail_idx + q->size;
+ return q->cached_tail - q->cached_head;
+}
+
+static inline u32 umem_nb_avail(struct xdp_umem_uqueue *q, u32 nb)
+{
+ u32 entries = q->cached_head - q->cached_tail;
+
+ if (entries == 0)
+ q->cached_head = q->q->ptrs.head_idx;
+
+ entries = q->cached_head - q->cached_tail;
+
+ return (entries > nb) ? nb : entries;
+}
+
+static inline u32 xq_nb_avail(struct xdp_uqueue *q, u32 ndescs)
+{
+ u32 entries = q->cached_head - q->cached_tail;
+
+ if (entries == 0)
+ q->cached_head = q->q->ptrs.head_idx;
+
+ entries = q->cached_head - q->cached_tail;
+ return (entries > ndescs) ? ndescs : entries;
+}
+
+static inline int umem_fill_to_kernel_ex(struct xdp_umem_uqueue *fq,
+ struct xdp_desc *d,
+ size_t nb)
+{
+ u32 i;
+
+ if (umem_nb_free(fq, nb) < nb)
+ return -ENOSPC;
+
+ for (i = 0; i < nb; i++) {
+ u32 idx = fq->cached_head++ & fq->mask;
+
+ fq->q->desc[idx] = d[i].idx;
+ }
+
+ u_smp_wmb();
+
+ fq->q->ptrs.head_idx = fq->cached_head;
+
+ return 0;
+}
+
+static inline int umem_fill_to_kernel(struct xdp_umem_uqueue *fq, u32 *d,
+ size_t nb)
+{
+ u32 i;
+
+ if (umem_nb_free(fq, nb) < nb)
+ return -ENOSPC;
+
+ for (i = 0; i < nb; i++) {
+ u32 idx = fq->cached_head++ & fq->mask;
+
+ fq->q->desc[idx] = d[i];
+ }
+
+ u_smp_wmb();
+
+ fq->q->ptrs.head_idx = fq->cached_head;
+
+ return 0;
+}
+
+static inline size_t umem_complete_from_kernel(struct xdp_umem_uqueue *cq,
+ u32 *d, size_t nb)
+{
+ u32 idx, i, entries = umem_nb_avail(cq, nb);
+
+ u_smp_rmb();
+
+ for (i = 0; i < entries; i++) {
+ idx = cq->cached_tail++ & cq->mask;
+ d[i] = cq->q->desc[idx];
+ }
+
+ if (entries > 0) {
+ u_smp_wmb();
+
+ cq->q->ptrs.tail_idx = cq->cached_tail;
+ }
+
+ return entries;
+}
+
+static inline void *xq_get_data(struct xdpsock *xsk, __u32 idx, __u32 off)
+{
+ lassert(idx < NUM_FRAMES);
+ return &xsk->umem->frames[idx][off];
+}
+
+static inline int xq_enq(struct xdp_uqueue *uq,
+ const struct xdp_desc *descs,
+ unsigned int ndescs)
+{
+ struct xdp_rxtx_queue *q = uq->q;
+ unsigned int i;
+
+ if (xq_nb_free(uq, ndescs) < ndescs)
+ return -ENOSPC;
+
+ for (i = 0; i < ndescs; i++) {
+ u32 idx = uq->cached_head++ & uq->mask;
+
+ q->desc[idx].idx = descs[i].idx;
+ q->desc[idx].len = descs[i].len;
+ q->desc[idx].offset = descs[i].offset;
+ }
+
+ u_smp_wmb();
+
+ q->ptrs.head_idx = uq->cached_head;
+ return 0;
+}
+
+static inline int xq_deq(struct xdp_uqueue *uq,
+ struct xdp_desc *descs,
+ int ndescs)
+{
+ struct xdp_rxtx_queue *q = uq->q;
+ unsigned int idx;
+ int i, entries;
+
+ entries = xq_nb_avail(uq, ndescs);
+
+ u_smp_rmb();
+
+ for (i = 0; i < entries; i++) {
+ idx = uq->cached_tail++ & uq->mask;
+ descs[i] = q->desc[idx];
+ }
+
+ if (entries > 0) {
+ u_smp_wmb();
+
+ q->ptrs.tail_idx = uq->cached_tail;
+ }
+
+ return entries;
+}
+
+static void swap_mac_addresses(void *data)
+{
+ struct ether_header *eth = (struct ether_header *)data;
+ struct ether_addr *src_addr = (struct ether_addr *)ð->ether_shost;
+ struct ether_addr *dst_addr = (struct ether_addr *)ð->ether_dhost;
+ struct ether_addr tmp;
+
+ tmp = *src_addr;
+ *src_addr = *dst_addr;
+ *dst_addr = tmp;
+}
+
+#if DEBUG_HEXDUMP
+static void hex_dump(void *pkt, size_t length, const char *prefix)
+{
+ int i = 0;
+ const unsigned char *address = (unsigned char *)pkt;
+ const unsigned char *line = address;
+ size_t line_size = 32;
+ unsigned char c;
+
+ printf("length = %zu\n", length);
+ printf("%s | ", prefix);
+ while (length-- > 0) {
+ printf("%02X ", *address++);
+ if (!(++i % line_size) || (length == 0 && i % line_size)) {
+ if (length == 0) {
+ while (i++ % line_size)
+ printf("__ ");
+ }
+ printf(" | "); /* right close */
+ while (line < address) {
+ c = *line++;
+ printf("%c", (c < 33 || c == 255) ? 0x2E : c);
+ }
+ printf("\n");
+ if (length > 0)
+ printf("%s | ", prefix);
+ }
+ }
+ printf("\n");
+}
+#endif
+
+static size_t gen_eth_frame(char *frame)
+{
+ memcpy(frame, pkt_data, sizeof(pkt_data) - 1);
+ return sizeof(pkt_data) - 1;
+}
+
+static struct xdp_umem *xdp_umem_configure(int sfd)
+{
+ int fq_size = FQ_NUM_DESCS, cq_size = CQ_NUM_DESCS;
+ struct xdp_umem_reg mr;
+ struct xdp_umem *umem;
+ void *bufs;
+
+ umem = calloc(1, sizeof(*umem));
+ lassert(umem);
+
+ lassert(posix_memalign(&bufs, getpagesize(), /* PAGE_SIZE aligned */
+ NUM_FRAMES * FRAME_SIZE) == 0);
+
+ mr.addr = (__u64)bufs;
+ mr.len = NUM_FRAMES * FRAME_SIZE;
+ mr.frame_size = FRAME_SIZE;
+ mr.frame_headroom = FRAME_HEADROOM;
+
+ lassert(setsockopt(sfd, SOL_XDP, XDP_UMEM_REG, &mr, sizeof(mr)) == 0);
+ lassert(setsockopt(sfd, SOL_XDP, XDP_UMEM_FILL_QUEUE, &fq_size,
+ sizeof(int)) == 0);
+ lassert(setsockopt(sfd, SOL_XDP, XDP_UMEM_COMPLETION_QUEUE, &cq_size,
+ sizeof(int)) == 0);
+
+ umem->fq.q = mmap(0, sizeof(struct xdp_umem_queue) +
+ FQ_NUM_DESCS * sizeof(u32),
+ PROT_READ | PROT_WRITE,
+ MAP_SHARED | MAP_POPULATE, sfd,
+ XDP_UMEM_PGOFF_FILL_QUEUE);
+ lassert(umem->fq.q != MAP_FAILED);
+
+ umem->fq.mask = FQ_NUM_DESCS - 1;
+ umem->fq.size = FQ_NUM_DESCS;
+
+ umem->cq.q = mmap(0, sizeof(struct xdp_umem_queue) +
+ CQ_NUM_DESCS * sizeof(u32),
+ PROT_READ | PROT_WRITE,
+ MAP_SHARED | MAP_POPULATE, sfd,
+ XDP_UMEM_PGOFF_COMPLETION_QUEUE);
+ lassert(umem->cq.q != MAP_FAILED);
+
+ umem->cq.mask = CQ_NUM_DESCS - 1;
+ umem->cq.size = CQ_NUM_DESCS;
+
+ umem->frames = (char (*)[FRAME_SIZE])bufs;
+ umem->fd = sfd;
+
+ if (opt_bench == BENCH_TXONLY) {
+ int i;
+
+ for (i = 0; i < NUM_FRAMES; i++)
+ (void)gen_eth_frame(&umem->frames[i][0]);
+ }
+
+ return umem;
+}
+
+static struct xdpsock *xsk_configure(struct xdp_umem *umem)
+{
+ struct sockaddr_xdp sxdp = {};
+ int sfd, ndescs = NUM_DESCS;
+ struct xdpsock *xsk;
+ bool shared = true;
+ u32 i;
+
+ sfd = socket(PF_XDP, SOCK_RAW, 0);
+ lassert(sfd >= 0);
+
+ xsk = calloc(1, sizeof(*xsk));
+ lassert(xsk);
+
+ xsk->sfd = sfd;
+ xsk->outstanding_tx = 0;
+
+ if (!umem) {
+ shared = false;
+ xsk->umem = xdp_umem_configure(sfd);
+ } else {
+ xsk->umem = umem;
+ }
+
+ lassert(setsockopt(sfd, SOL_XDP, XDP_RX_QUEUE,
+ &ndescs, sizeof(int)) == 0);
+ lassert(setsockopt(sfd, SOL_XDP, XDP_TX_QUEUE,
+ &ndescs, sizeof(int)) == 0);
+
+ /* Rx */
+ xsk->rx.q = mmap(NULL,
+ sizeof(struct xdp_queue) +
+ NUM_DESCS * sizeof(struct xdp_desc),
+ PROT_READ | PROT_WRITE,
+ MAP_SHARED | MAP_POPULATE, sfd,
+ XDP_PGOFF_RX_QUEUE);
+ lassert(xsk->rx.q != MAP_FAILED);
+
+ if (!shared) {
+ for (i = 0; i < NUM_DESCS / 2; i++)
+ lassert(umem_fill_to_kernel(&xsk->umem->fq, &i, 1)
+ == 0);
+ }
+
+ /* Tx */
+ xsk->tx.q = mmap(NULL,
+ sizeof(struct xdp_queue) +
+ NUM_DESCS * sizeof(struct xdp_desc),
+ PROT_READ | PROT_WRITE,
+ MAP_SHARED | MAP_POPULATE, sfd,
+ XDP_PGOFF_TX_QUEUE);
+ lassert(xsk->tx.q != MAP_FAILED);
+
+ xsk->rx.mask = NUM_DESCS - 1;
+ xsk->rx.size = NUM_DESCS;
+
+ xsk->tx.mask = NUM_DESCS - 1;
+ xsk->tx.size = NUM_DESCS;
+
+ sxdp.sxdp_family = PF_XDP;
+ sxdp.sxdp_ifindex = opt_ifindex;
+ sxdp.sxdp_queue_id = opt_queue;
+ if (shared) {
+ sxdp.sxdp_flags = XDP_SHARED_UMEM;
+ sxdp.sxdp_shared_umem_fd = umem->fd;
+ }
+
+ lassert(bind(sfd, (struct sockaddr *)&sxdp, sizeof(sxdp)) == 0);
+
+ return xsk;
+}
+
+static void print_benchmark(bool running)
+{
+ const char *bench_str = "INVALID";
+
+ if (opt_bench == BENCH_RXDROP)
+ bench_str = "rxdrop";
+ else if (opt_bench == BENCH_TXONLY)
+ bench_str = "txonly";
+ else if (opt_bench == BENCH_L2FWD)
+ bench_str = "l2fwd";
+
+ printf("%s:%d %s ", opt_if, opt_queue, bench_str);
+ if (opt_xdp_flags & XDP_FLAGS_SKB_MODE)
+ printf("xdp-skb ");
+ else if (opt_xdp_flags & XDP_FLAGS_DRV_MODE)
+ printf("xdp-drv ");
+ else
+ printf(" ");
+
+ if (opt_poll)
+ printf("poll() ");
+
+ if (running) {
+ printf("running...");
+ fflush(stdout);
+ }
+}
+
+static void dump_stats(void)
+{
+ unsigned long stop_time = get_nsecs();
+ long dt = stop_time - start_time;
+ int i;
+
+ for (i = 0; i < num_socks; i++) {
+ double rx_pps = xsks[i]->rx_npkts * 1000000000. / dt;
+ double tx_pps = xsks[i]->tx_npkts * 1000000000. / dt;
+ char *fmt = "%-15s %'-11.0f %'-11lu\n";
+
+ printf("\n sock%d@", i);
+ print_benchmark(false);
+ printf("\n");
+
+ printf("%-15s %-11s %-11s %-11.2f\n", "", "pps", "pkts",
+ dt / 1000000000.);
+ printf(fmt, "rx", rx_pps, xsks[i]->rx_npkts);
+ printf(fmt, "tx", tx_pps, xsks[i]->tx_npkts);
+ }
+}
+
+static void *poller(void *arg)
+{
+ (void)arg;
+ for (;;) {
+ sleep(1);
+ dump_stats();
+ }
+
+ return NULL;
+}
+
+static void int_exit(int sig)
+{
+ (void)sig;
+ dump_stats();
+ bpf_set_link_xdp_fd(opt_ifindex, -1, opt_xdp_flags);
+ exit(EXIT_SUCCESS);
+}
+
+static struct option long_options[] = {
+ {"rxdrop", no_argument, 0, 'r'},
+ {"txonly", no_argument, 0, 't'},
+ {"l2fwd", no_argument, 0, 'l'},
+ {"interface", required_argument, 0, 'i'},
+ {"queue", required_argument, 0, 'q'},
+ {"poll", no_argument, 0, 'p'},
+ {"shared-buffer", no_argument, 0, 's'},
+ {"xdp-skb", no_argument, 0, 'S'},
+ {"xdp-native", no_argument, 0, 'N'},
+ {0, 0, 0, 0}
+};
+
+static void usage(const char *prog)
+{
+ const char *str =
+ " Usage: %s [OPTIONS]\n"
+ " Options:\n"
+ " -r, --rxdrop Discard all incoming packets (default)\n"
+ " -t, --txonly Only send packets\n"
+ " -l, --l2fwd MAC swap L2 forwarding\n"
+ " -i, --interface=n Run on interface n\n"
+ " -q, --queue=n Use queue n (default 0)\n"
+ " -p, --poll Use poll syscall\n"
+ " -s, --shared-buffer Use shared packet buffer\n"
+ " -S, --xdp-skb=n Use XDP skb-mod\n"
+ " -N, --xdp-native=n Enfore XDP native mode\n"
+ "\n";
+ fprintf(stderr, str, prog);
+ exit(EXIT_FAILURE);
+}
+
+static void parse_command_line(int argc, char **argv)
+{
+ int option_index, c;
+
+ opterr = 0;
+
+ for (;;) {
+ c = getopt_long(argc, argv, "rtli:q:psSN", long_options,
+ &option_index);
+ if (c == -1)
+ break;
+
+ switch (c) {
+ case 'r':
+ opt_bench = BENCH_RXDROP;
+ break;
+ case 't':
+ opt_bench = BENCH_TXONLY;
+ break;
+ case 'l':
+ opt_bench = BENCH_L2FWD;
+ break;
+ case 'i':
+ opt_if = optarg;
+ break;
+ case 'q':
+ opt_queue = atoi(optarg);
+ break;
+ case 's':
+ opt_shared_packet_buffer = 1;
+ break;
+ case 'p':
+ opt_poll = 1;
+ break;
+ case 'S':
+ opt_xdp_flags |= XDP_FLAGS_SKB_MODE;
+ break;
+ case 'N':
+ opt_xdp_flags |= XDP_FLAGS_DRV_MODE;
+ break;
+ default:
+ usage(basename(argv[0]));
+ }
+ }
+
+ opt_ifindex = if_nametoindex(opt_if);
+ if (!opt_ifindex) {
+ fprintf(stderr, "ERROR: interface \"%s\" does not exist\n",
+ opt_if);
+ usage(basename(argv[0]));
+ }
+}
+
+static void kick_tx(int fd)
+{
+ int ret;
+
+ ret = sendto(fd, NULL, 0, MSG_DONTWAIT, NULL, 0);
+ if (ret >= 0 || errno == ENOBUFS || errno == EAGAIN)
+ return;
+ lassert(0);
+}
+
+static inline void complete_tx_l2fwd(struct xdpsock *xsk)
+{
+ u32 descs[BATCH_SIZE];
+ unsigned int rcvd;
+ size_t ndescs;
+
+ if (!xsk->outstanding_tx)
+ return;
+
+ kick_tx(xsk->sfd);
+ ndescs = (xsk->outstanding_tx > BATCH_SIZE) ? BATCH_SIZE :
+ xsk->outstanding_tx;
+
+ /* re-add completed Tx buffers */
+ rcvd = umem_complete_from_kernel(&xsk->umem->cq, descs, ndescs);
+ if (rcvd > 0) {
+ umem_fill_to_kernel(&xsk->umem->fq, descs, rcvd);
+ xsk->outstanding_tx -= rcvd;
+ xsk->tx_npkts += rcvd;
+ }
+}
+
+static inline void complete_tx_only(struct xdpsock *xsk)
+{
+ u32 descs[BATCH_SIZE];
+ unsigned int rcvd;
+
+ if (!xsk->outstanding_tx)
+ return;
+
+ kick_tx(xsk->sfd);
+
+ rcvd = umem_complete_from_kernel(&xsk->umem->cq, descs, BATCH_SIZE);
+ if (rcvd > 0) {
+ xsk->outstanding_tx -= rcvd;
+ xsk->tx_npkts += rcvd;
+ }
+}
+
+static void rx_drop(struct xdpsock *xsk)
+{
+ struct xdp_desc descs[BATCH_SIZE];
+ unsigned int rcvd, i;
+
+ rcvd = xq_deq(&xsk->rx, descs, BATCH_SIZE);
+ if (!rcvd)
+ return;
+
+ for (i = 0; i < rcvd; i++) {
+ u32 idx = descs[i].idx;
+
+ lassert(idx < NUM_FRAMES);
+#if DEBUG_HEXDUMP
+ char *pkt;
+ char buf[32];
+
+ pkt = xq_get_data(xsk, idx, descs[i].offset);
+ sprintf(buf, "idx=%d", idx);
+ hex_dump(pkt, descs[i].len, buf);
+#endif
+ }
+
+ xsk->rx_npkts += rcvd;
+
+ umem_fill_to_kernel_ex(&xsk->umem->fq, descs, rcvd);
+}
+
+static void rx_drop_all(void)
+{
+ struct pollfd fds[MAX_SOCKS + 1];
+ int i, ret, timeout, nfds = 1;
+
+ memset(fds, 0, sizeof(fds));
+
+ for (i = 0; i < num_socks; i++) {
+ fds[i].fd = xsks[i]->sfd;
+ fds[i].events = POLLIN;
+ timeout = 1000; /* 1sn */
+ }
+
+ for (;;) {
+ if (opt_poll) {
+ ret = poll(fds, nfds, timeout);
+ if (ret <= 0)
+ continue;
+ }
+
+ for (i = 0; i < num_socks; i++)
+ rx_drop(xsks[i]);
+ }
+}
+
+static void gen_tx_descs(struct xdp_desc *descs, unsigned int idx,
+ unsigned int ndescs)
+{
+ int i;
+
+ for (i = 0; i < ndescs; i++) {
+ descs[i].idx = idx + i;
+ descs[i].len = sizeof(pkt_data) - 1;
+ descs[i].offset = 0;
+ descs[i].flags = 0;
+ }
+}
+
+static void tx_only(struct xdpsock *xsk)
+{
+ int timeout, ret, nfds = 1;
+ struct pollfd fds[nfds + 1];
+ unsigned int idx = 0;
+
+ memset(fds, 0, sizeof(fds));
+ fds[0].fd = xsk->sfd;
+ fds[0].events = POLLOUT;
+ timeout = 1000; /* 1sn */
+
+ for (;;) {
+ struct xdp_desc descs[BATCH_SIZE];
+
+ if (opt_poll) {
+ ret = poll(fds, nfds, timeout);
+ if (ret <= 0)
+ continue;
+
+ if (fds[0].fd != xsk->sfd ||
+ !(fds[0].revents & POLLOUT))
+ continue;
+ }
+
+ if (xq_nb_free(&xsk->tx, BATCH_SIZE) >= BATCH_SIZE) {
+ gen_tx_descs(descs, idx, BATCH_SIZE);
+ lassert(xq_enq(&xsk->tx, descs, BATCH_SIZE) == 0);
+
+ xsk->outstanding_tx += BATCH_SIZE;
+ idx += BATCH_SIZE;
+ idx %= NUM_FRAMES;
+ }
+
+ complete_tx_only(xsk);
+ }
+}
+
+static void l2fwd(struct xdpsock *xsk)
+{
+ for (;;) {
+ struct xdp_desc descs[BATCH_SIZE];
+ unsigned int rcvd, i;
+ int ret;
+
+ for (;;) {
+ complete_tx_l2fwd(xsk);
+
+ rcvd = xq_deq(&xsk->rx, descs, BATCH_SIZE);
+ if (rcvd > 0)
+ break;
+ }
+
+ for (i = 0; i < rcvd; i++) {
+ char *pkt = xq_get_data(xsk, descs[i].idx,
+ descs[i].offset);
+
+ swap_mac_addresses(pkt);
+#if DEBUG_HEXDUMP
+ char buf[32];
+ u32 idx = descs[i].idx;
+
+ sprintf(buf, "idx=%d", idx);
+ hex_dump(pkt, descs[i].len, buf);
+#endif
+ }
+
+ xsk->rx_npkts += rcvd;
+
+ ret = xq_enq(&xsk->tx, descs, rcvd);
+ lassert(ret == 0);
+ xsk->outstanding_tx += rcvd;
+ }
+}
+
+int main(int argc, char **argv)
+{
+ struct rlimit r = {RLIM_INFINITY, RLIM_INFINITY};
+ char xdp_filename[256];
+ int i, ret, key = 0;
+ pthread_t pt;
+
+ parse_command_line(argc, argv);
+
+ if (setrlimit(RLIMIT_MEMLOCK, &r)) {
+ fprintf(stderr, "ERROR: setrlimit(RLIMIT_MEMLOCK) \"%s\"\n",
+ strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ snprintf(xdp_filename, sizeof(xdp_filename), "%s_kern.o", argv[0]);
+
+ if (load_bpf_file(xdp_filename)) {
+ fprintf(stderr, "ERROR: load_bpf_file %s\n", bpf_log_buf);
+ exit(EXIT_FAILURE);
+ }
+
+ if (!prog_fd[0]) {
+ fprintf(stderr, "ERROR: load_bpf_file: \"%s\"\n",
+ strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ if (bpf_set_link_xdp_fd(opt_ifindex, prog_fd[0], opt_xdp_flags) < 0) {
+ fprintf(stderr, "ERROR: link set xdp fd failed\n");
+ exit(EXIT_FAILURE);
+ }
+
+ ret = bpf_map_update_elem(map_fd[0], &key, &opt_queue, 0);
+ if (ret) {
+ fprintf(stderr, "ERROR: bpf_map_update_elem qidconf\n");
+ exit(EXIT_FAILURE);
+ }
+
+ /* Create sockets... */
+ xsks[num_socks++] = xsk_configure(NULL);
+
+#if RR_LB
+ for (i = 0; i < MAX_SOCKS - 1; i++)
+ xsks[num_socks++] = xsk_configure(xsks[0]->umem);
+#endif
+
+ /* ...and insert them into the map. */
+ for (i = 0; i < num_socks; i++) {
+ key = i;
+ ret = bpf_map_update_elem(map_fd[1], &key, &xsks[i]->sfd, 0);
+ if (ret) {
+ fprintf(stderr, "ERROR: bpf_map_update_elem %d\n", i);
+ exit(EXIT_FAILURE);
+ }
+ }
+
+ signal(SIGINT, int_exit);
+ signal(SIGTERM, int_exit);
+ signal(SIGABRT, int_exit);
+
+ setlocale(LC_ALL, "");
+
+ ret = pthread_create(&pt, NULL, poller, NULL);
+ lassert(ret == 0);
+
+ start_time = get_nsecs();
+
+ if (opt_bench == BENCH_RXDROP)
+ rx_drop_all();
+ else if (opt_bench == BENCH_TXONLY)
+ tx_only(xsks[0]);
+ else
+ l2fwd(xsks[0]);
+
+ return 0;
+}
--
2.14.1
^ permalink raw reply related
* Re: [PATCH iproute2] ss: Fix rendering of continuous output (-E, --events)
From: Stephen Hemminger @ 2018-03-27 16:11 UTC (permalink / raw)
To: Roman Mashak; +Cc: Stefano Brivio, netdev
In-Reply-To: <85woy2ly1i.fsf@mojatatu.com>
On Fri, 23 Mar 2018 09:30:17 -0400
Roman Mashak <mrv@mojatatu.com> wrote:
> Stefano Brivio <sbrivio@redhat.com> writes:
>
> > Roman Mashak reported that ss currently shows no output when it
> > should continuously report information about terminated sockets
> > (-E, --events switch).
> >
> > This happens because I missed this case in 691bd854bf4a ("ss:
> > Buffer raw fields first, then render them as a table") and the
> > rendering function is simply not called.
> >
> > To fix this, we need to:
> >
> > - call render() every time we need to display new socket events
> > from generic_show_sock(), which is only used to follow events.
> > Always call it even if specific socket display functions
> > return errors to ensure we clean up buffers
> >
> > - get the screen width every time we have new events to display,
> > thus factor out getting the screen width from main() into a
> > function we'll call whenever we calculate columns width
> >
> > - reset the current field pointer after rendering, more output
> > might come after render() is called
> >
> > Reported-by: Roman Mashak <mrv@mojatatu.com>
> > Fixes: 691bd854bf4a ("ss: Buffer raw fields first, then render them as a table")
> > Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
>
> Thanks Stefano.
>
> Tested-by: Roman Mashak <mrv@mojatatu.com>
Applied
^ permalink raw reply
* Re: [patch iproute2] devlink: fix port new monitoring message typo
From: Stephen Hemminger @ 2018-03-27 16:12 UTC (permalink / raw)
To: Jiri Pirko; +Cc: netdev, dsahern
In-Reply-To: <20180323121913.12598-1-jiri@resnulli.us>
On Fri, 23 Mar 2018 13:19:13 +0100
Jiri Pirko <jiri@resnulli.us> wrote:
> From: Jiri Pirko <jiri@mellanox.com>
>
> s/net/new/
>
> Fixes: a3c4b484a1ed ("add devlink tool")
> Signed-off-by: Jiri Pirko <jiri@mellanox.com>
> ---
> devlink/devlink.c | 2 +-
> 1 file changed, 1 insertion(+), 1 deletion(-)
Applied
^ permalink raw reply
* Re: [PATCH iproute2 1/1] tc: print actual action for connmark action
From: Stephen Hemminger @ 2018-03-27 16:03 UTC (permalink / raw)
To: Roman Mashak; +Cc: netdev, jhs, xiyou.wangcong, jiri
In-Reply-To: <1521567938-12408-1-git-send-email-mrv@mojatatu.com>
On Tue, 20 Mar 2018 13:45:38 -0400
Roman Mashak <mrv@mojatatu.com> wrote:
> Signed-off-by: Roman Mashak <mrv@mojatatu.com>
> ---
> tc/m_connmark.c | 3 ++-
> 1 file changed, 2 insertions(+), 1 deletion(-)
Applied
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox