* [PATCH v3 08/27] net/failsafe: convert to stdatomic
From: Stephen Hemminger @ 2026-05-23 19:56 UTC (permalink / raw)
To: dev; +Cc: Stephen Hemminger, Gaetan Rivet
In-Reply-To: <20260523195839.454952-1-stephen@networkplumber.org>
The functions rte_atomic64 are deprecated, convert this
code to use stdatomic for reference count. Use the memory
order implied by naming P/V.
No need for initialization since refcnt is in space
allocated with rte_zmalloc().
Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
drivers/net/failsafe/failsafe_ops.c | 12 +++++-----
drivers/net/failsafe/failsafe_private.h | 29 ++++++++++++++-----------
drivers/net/failsafe/failsafe_rxtx.c | 2 +-
3 files changed, 22 insertions(+), 21 deletions(-)
diff --git a/drivers/net/failsafe/failsafe_ops.c b/drivers/net/failsafe/failsafe_ops.c
index ddc8808ebe..fcb0051777 100644
--- a/drivers/net/failsafe/failsafe_ops.c
+++ b/drivers/net/failsafe/failsafe_ops.c
@@ -11,7 +11,7 @@
#endif
#include <rte_debug.h>
-#include <rte_atomic.h>
+#include <rte_stdatomic.h>
#include <ethdev_driver.h>
#include <rte_malloc.h>
#include <rte_flow.h>
@@ -440,14 +440,13 @@ fs_rx_queue_setup(struct rte_eth_dev *dev,
}
rxq = rte_zmalloc(NULL,
sizeof(*rxq) +
- sizeof(rte_atomic64_t) * PRIV(dev)->subs_tail,
+ sizeof(uint64_t) * PRIV(dev)->subs_tail,
RTE_CACHE_LINE_SIZE);
if (rxq == NULL) {
fs_unlock(dev, 0);
return -ENOMEM;
}
- FOREACH_SUBDEV(sdev, i, dev)
- rte_atomic64_init(&rxq->refcnt[i]);
+
rxq->qid = rx_queue_id;
rxq->socket_id = socket_id;
rxq->info.mp = mb_pool;
@@ -617,14 +616,13 @@ fs_tx_queue_setup(struct rte_eth_dev *dev,
}
txq = rte_zmalloc("ethdev TX queue",
sizeof(*txq) +
- sizeof(rte_atomic64_t) * PRIV(dev)->subs_tail,
+ sizeof(uint64_t) * PRIV(dev)->subs_tail,
RTE_CACHE_LINE_SIZE);
if (txq == NULL) {
fs_unlock(dev, 0);
return -ENOMEM;
}
- FOREACH_SUBDEV(sdev, i, dev)
- rte_atomic64_init(&txq->refcnt[i]);
+
txq->qid = tx_queue_id;
txq->socket_id = socket_id;
txq->info.conf = *tx_conf;
diff --git a/drivers/net/failsafe/failsafe_private.h b/drivers/net/failsafe/failsafe_private.h
index babea6016e..89b06f9756 100644
--- a/drivers/net/failsafe/failsafe_private.h
+++ b/drivers/net/failsafe/failsafe_private.h
@@ -10,7 +10,7 @@
#include <sys/queue.h>
#include <pthread.h>
-#include <rte_atomic.h>
+#include <rte_stdatomic.h>
#include <dev_driver.h>
#include <ethdev_driver.h>
#include <rte_devargs.h>
@@ -75,7 +75,7 @@ struct rxq {
int event_fd;
unsigned int enable_events:1;
struct rte_eth_rxq_info info;
- rte_atomic64_t refcnt[];
+ RTE_ATOMIC(uint64_t) refcnt[];
};
struct txq {
@@ -83,7 +83,7 @@ struct txq {
uint16_t qid;
unsigned int socket_id;
struct rte_eth_txq_info info;
- rte_atomic64_t refcnt[];
+ RTE_ATOMIC(uint64_t) refcnt[];
};
struct rte_flow {
@@ -320,33 +320,36 @@ extern int failsafe_mac_from_arg;
*/
/**
- * a: (rte_atomic64_t)
+ * a: _Atomic uint64_t
*/
#define FS_ATOMIC_P(a) \
- rte_atomic64_set(&(a), 1)
+ rte_atomic_exchange_explicit(&(a), 1, rte_memory_order_acquire)
/**
- * a: (rte_atomic64_t)
+ * a: _Atomic uint64_t
*/
#define FS_ATOMIC_V(a) \
- rte_atomic64_set(&(a), 0)
+ rte_atomic_store_explicit(&(a), 0, rte_memory_order_release)
/**
* s: (struct sub_device *)
* i: uint16_t qid
*/
#define FS_ATOMIC_RX(s, i) \
- rte_atomic64_read( \
- &((struct rxq *) \
- (fs_dev(s)->data->rx_queues[i]))->refcnt[(s)->sid])
+ rte_atomic_load_explicit( \
+ &((struct rxq *) \
+ (fs_dev(s)->data->rx_queues[i]))->refcnt[(s)->sid], \
+ rte_memory_order_seq_cst)
+
/**
* s: (struct sub_device *)
* i: uint16_t qid
*/
#define FS_ATOMIC_TX(s, i) \
- rte_atomic64_read( \
- &((struct txq *) \
- (fs_dev(s)->data->tx_queues[i]))->refcnt[(s)->sid])
+ rte_atomic_load_explicit( \
+ &((struct txq *) \
+ (fs_dev(s)->data->tx_queues[i]))->refcnt[(s)->sid], \
+ rte_memory_order_seq_cst)
#ifdef RTE_EXEC_ENV_FREEBSD
#define FS_THREADID_TYPE void*
diff --git a/drivers/net/failsafe/failsafe_rxtx.c b/drivers/net/failsafe/failsafe_rxtx.c
index fe67293299..500483bda3 100644
--- a/drivers/net/failsafe/failsafe_rxtx.c
+++ b/drivers/net/failsafe/failsafe_rxtx.c
@@ -3,7 +3,7 @@
* Copyright 2017 Mellanox Technologies, Ltd
*/
-#include <rte_atomic.h>
+#include <rte_stdatomic.h>
#include <rte_debug.h>
#include <rte_mbuf.h>
#include <ethdev_driver.h>
--
2.53.0
^ permalink raw reply related
* [PATCH v3 07/27] net/ena: replace use of rte_atomicNN
From: Stephen Hemminger @ 2026-05-23 19:56 UTC (permalink / raw)
To: dev
Cc: Stephen Hemminger, Shai Brandes, Evgeny Schemeilin, Ron Beider,
Amit Bernstein, Wajeeh Atrash
In-Reply-To: <20260523195839.454952-1-stephen@networkplumber.org>
Convert the legacy rte_atomicNN operations to stdatomic.
* Remove variable ena_alloc_cnt is defined by not used.
It is a leftover from previous memzone naming scheme.
* Convert the legacy rte_atomic32_t and rte_atomic32_{inc,dec,set,read}
macros to C11 stdatomic equivalents.
Memory ordering is kept at seq_cst,
matching the implicit ordering of the legacy API.
* Do not use rte_atomic for statistics
The DPDK PMD model is that statistics do not have to be exact
in face of contention.
Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
drivers/net/ena/base/ena_plat_dpdk.h | 14 +++++++++-----
drivers/net/ena/ena_ethdev.c | 21 ++++++---------------
drivers/net/ena/ena_ethdev.h | 7 +++----
3 files changed, 18 insertions(+), 24 deletions(-)
diff --git a/drivers/net/ena/base/ena_plat_dpdk.h b/drivers/net/ena/base/ena_plat_dpdk.h
index c84420de22..83b354d9da 100644
--- a/drivers/net/ena/base/ena_plat_dpdk.h
+++ b/drivers/net/ena/base/ena_plat_dpdk.h
@@ -40,7 +40,7 @@ typedef uint64_t dma_addr_t;
#endif
#define ENA_PRIu64 PRIu64
-#define ena_atomic32_t rte_atomic32_t
+typedef RTE_ATOMIC(int32_t) ena_atomic32_t;
#define ena_mem_handle_t const struct rte_memzone *
#define SZ_256 (256U)
@@ -267,10 +267,14 @@ ena_mem_alloc_coherent(struct rte_eth_dev_data *data, size_t size,
#define ENA_REG_READ32(bus, reg) \
__extension__ ({ (void)(bus); rte_read32_relaxed((reg)); })
-#define ATOMIC32_INC(i32_ptr) rte_atomic32_inc(i32_ptr)
-#define ATOMIC32_DEC(i32_ptr) rte_atomic32_dec(i32_ptr)
-#define ATOMIC32_SET(i32_ptr, val) rte_atomic32_set(i32_ptr, val)
-#define ATOMIC32_READ(i32_ptr) rte_atomic32_read(i32_ptr)
+#define ATOMIC32_INC(i32_ptr) \
+ rte_atomic_fetch_add_explicit((i32_ptr), 1, rte_memory_order_seq_cst)
+#define ATOMIC32_DEC(i32_ptr) \
+ rte_atomic_fetch_sub_explicit((i32_ptr), 1, rte_memory_order_seq_cst)
+#define ATOMIC32_SET(i32_ptr, val) \
+ rte_atomic_store_explicit((i32_ptr), (val), rte_memory_order_seq_cst)
+#define ATOMIC32_READ(i32_ptr) \
+ rte_atomic_load_explicit((i32_ptr), rte_memory_order_seq_cst)
#define msleep(x) rte_delay_us(x * 1000)
#define udelay(x) rte_delay_us(x)
diff --git a/drivers/net/ena/ena_ethdev.c b/drivers/net/ena/ena_ethdev.c
index ea4afbc75d..e9c484456c 100644
--- a/drivers/net/ena/ena_ethdev.c
+++ b/drivers/net/ena/ena_ethdev.c
@@ -121,12 +121,6 @@ struct ena_stats {
*/
#define ENA_DEVARG_ENABLE_FRAG_BYPASS "enable_frag_bypass"
-/*
- * Each rte_memzone should have unique name.
- * To satisfy it, count number of allocation and add it to name.
- */
-rte_atomic64_t ena_alloc_cnt;
-
static const struct ena_stats ena_stats_global_strings[] = {
ENA_STAT_GLOBAL_ENTRY(wd_expired),
ENA_STAT_GLOBAL_ENTRY(dev_start),
@@ -1249,10 +1243,7 @@ static void ena_stats_restart(struct rte_eth_dev *dev)
{
struct ena_adapter *adapter = dev->data->dev_private;
- rte_atomic64_init(&adapter->drv_stats->ierrors);
- rte_atomic64_init(&adapter->drv_stats->oerrors);
- rte_atomic64_init(&adapter->drv_stats->rx_nombuf);
- adapter->drv_stats->rx_drops = 0;
+ memset(adapter->drv_stats, 0, sizeof(struct ena_driver_stats));
}
static int ena_stats_get(struct rte_eth_dev *dev,
@@ -1289,9 +1280,9 @@ static int ena_stats_get(struct rte_eth_dev *dev,
/* Driver related stats */
stats->imissed = adapter->drv_stats->rx_drops;
- stats->ierrors = rte_atomic64_read(&adapter->drv_stats->ierrors);
- stats->oerrors = rte_atomic64_read(&adapter->drv_stats->oerrors);
- stats->rx_nombuf = rte_atomic64_read(&adapter->drv_stats->rx_nombuf);
+ stats->ierrors = adapter->drv_stats->ierrors;
+ stats->oerrors = adapter->drv_stats->oerrors;
+ stats->rx_nombuf = adapter->drv_stats->rx_nombuf;
/* Queue statistics */
if (qstats) {
@@ -1887,7 +1878,7 @@ static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count)
/* get resources for incoming packets */
rc = rte_pktmbuf_alloc_bulk(rxq->mb_pool, mbufs, count);
if (unlikely(rc < 0)) {
- rte_atomic64_inc(&rxq->adapter->drv_stats->rx_nombuf);
+ ++rxq->adapter->drv_stats->rx_nombuf;
++rxq->rx_stats.mbuf_alloc_fail;
PMD_RX_LOG_LINE(DEBUG, "There are not enough free buffers");
return 0;
@@ -3014,7 +3005,7 @@ static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
if (unlikely(mbuf->ol_flags &
(RTE_MBUF_F_RX_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD)))
- rte_atomic64_inc(&rx_ring->adapter->drv_stats->ierrors);
+ ++rx_ring->adapter->drv_stats->ierrors;
rx_pkts[completed] = mbuf;
rx_ring->rx_stats.bytes += mbuf->pkt_len;
diff --git a/drivers/net/ena/ena_ethdev.h b/drivers/net/ena/ena_ethdev.h
index 3a66d79384..b204b07767 100644
--- a/drivers/net/ena/ena_ethdev.h
+++ b/drivers/net/ena/ena_ethdev.h
@@ -6,7 +6,6 @@
#ifndef _ENA_ETHDEV_H_
#define _ENA_ETHDEV_H_
-#include <rte_atomic.h>
#include <rte_ether.h>
#include <ethdev_driver.h>
#include <ethdev_pci.h>
@@ -225,9 +224,9 @@ enum ena_adapter_state {
};
struct ena_driver_stats {
- rte_atomic64_t ierrors;
- rte_atomic64_t oerrors;
- rte_atomic64_t rx_nombuf;
+ u64 ierrors;
+ u64 oerrors;
+ u64 rx_nombuf;
u64 rx_drops;
};
--
2.53.0
^ permalink raw reply related
* [PATCH v3 06/27] net/nbl: remove unused rte_atomic16 field
From: Stephen Hemminger @ 2026-05-23 19:56 UTC (permalink / raw)
To: dev; +Cc: Stephen Hemminger, Dimon Zhao, Leon Yu, Sam Chen
In-Reply-To: <20260523195839.454952-1-stephen@networkplumber.org>
The tx_current_queue was defined as rte_atomic16_t which
is deprecated. Remove it since it was never used.
Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
drivers/net/nbl/nbl_hw/nbl_resource.h | 1 -
1 file changed, 1 deletion(-)
diff --git a/drivers/net/nbl/nbl_hw/nbl_resource.h b/drivers/net/nbl/nbl_hw/nbl_resource.h
index bf5a9461f5..f2182ba6bc 100644
--- a/drivers/net/nbl/nbl_hw/nbl_resource.h
+++ b/drivers/net/nbl/nbl_hw/nbl_resource.h
@@ -225,7 +225,6 @@ struct nbl_res_info {
u16 base_qid;
u16 lcore_max;
u16 *pf_qid_to_lcore_id;
- rte_atomic16_t tx_current_queue;
};
struct nbl_resource_mgt {
--
2.53.0
^ permalink raw reply related
* [PATCH v3 05/27] net/bonding: use stdatomic
From: Stephen Hemminger @ 2026-05-23 19:56 UTC (permalink / raw)
To: dev; +Cc: Stephen Hemminger, Chas Williams, Min Hu (Connor)
In-Reply-To: <20260523195839.454952-1-stephen@networkplumber.org>
The old rte_atomic16 and rte_atomic64 functions are deprecated.
Replace with rte_stdatomic for managing warning and timer flags.
Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
drivers/net/bonding/eth_bond_8023ad_private.h | 6 ++--
drivers/net/bonding/rte_eth_bond_8023ad.c | 35 ++++++++-----------
2 files changed, 17 insertions(+), 24 deletions(-)
diff --git a/drivers/net/bonding/eth_bond_8023ad_private.h b/drivers/net/bonding/eth_bond_8023ad_private.h
index ab7d15f81a..dd3cf3ed26 100644
--- a/drivers/net/bonding/eth_bond_8023ad_private.h
+++ b/drivers/net/bonding/eth_bond_8023ad_private.h
@@ -9,7 +9,7 @@
#include <rte_ether.h>
#include <rte_byteorder.h>
-#include <rte_atomic.h>
+#include <rte_stdatomic.h>
#include <rte_flow.h>
#include "rte_eth_bond_8023ad.h"
@@ -140,10 +140,10 @@ struct port {
/** Timer which is also used as mutex. If is 0 (not running) RX marker
* packet might be responded. Otherwise shall be dropped. It is zeroed in
* mode 4 callback function after expire. */
- volatile uint64_t rx_marker_timer;
+ RTE_ATOMIC(uint64_t) rx_marker_timer;
uint64_t warning_timer;
- volatile uint16_t warnings_to_show;
+ RTE_ATOMIC(uint16_t) warnings_to_show;
/** Memory pool used to allocate slow queues */
struct rte_mempool *slow_pool;
diff --git a/drivers/net/bonding/rte_eth_bond_8023ad.c b/drivers/net/bonding/rte_eth_bond_8023ad.c
index ba88f6d261..cc7e4af2b9 100644
--- a/drivers/net/bonding/rte_eth_bond_8023ad.c
+++ b/drivers/net/bonding/rte_eth_bond_8023ad.c
@@ -171,27 +171,17 @@ timer_is_running(uint64_t *timer)
static void
set_warning_flags(struct port *port, uint16_t flags)
{
- int retval;
- uint16_t old;
- uint16_t new_flag = 0;
-
- do {
- old = port->warnings_to_show;
- new_flag = old | flags;
- retval = rte_atomic16_cmpset(&port->warnings_to_show, old, new_flag);
- } while (unlikely(retval == 0));
+ rte_atomic_fetch_or_explicit(&port->warnings_to_show, flags, rte_memory_order_relaxed);
}
static void
show_warnings(uint16_t member_id)
{
struct port *port = &bond_mode_8023ad_ports[member_id];
- uint8_t warnings;
-
- do {
- warnings = port->warnings_to_show;
- } while (rte_atomic16_cmpset(&port->warnings_to_show, warnings, 0) == 0);
+ uint16_t warnings;
+ warnings = rte_atomic_exchange_explicit(&port->warnings_to_show, 0,
+ rte_memory_order_relaxed);
if (!warnings)
return;
@@ -1337,7 +1327,6 @@ bond_mode_8023ad_handle_slow_pkt(struct bond_dev_private *internals,
struct port *port = &bond_mode_8023ad_ports[member_id];
struct marker_header *m_hdr;
uint64_t marker_timer, old_marker_timer;
- int retval;
uint8_t wrn, subtype;
/* If packet is a marker, we send response now by reusing given packet
* and update only source MAC, destination MAC is multicast so don't
@@ -1354,17 +1343,19 @@ bond_mode_8023ad_handle_slow_pkt(struct bond_dev_private *internals,
}
/* Setup marker timer. Do it in loop in case concurrent access. */
+ old_marker_timer = rte_atomic_load_explicit(&port->rx_marker_timer,
+ rte_memory_order_relaxed);
do {
- old_marker_timer = port->rx_marker_timer;
if (!timer_is_expired(&old_marker_timer)) {
wrn = WRN_RX_MARKER_TO_FAST;
goto free_out;
}
timer_set(&marker_timer, mode4->rx_marker_timeout);
- retval = rte_atomic64_cmpset(&port->rx_marker_timer,
- old_marker_timer, marker_timer);
- } while (unlikely(retval == 0));
+
+ } while (!rte_atomic_compare_exchange_weak_explicit(&port->rx_marker_timer,
+ &old_marker_timer, marker_timer,
+ rte_memory_order_seq_cst, rte_memory_order_relaxed));
m_hdr->marker.tlv_type_marker = MARKER_TLV_TYPE_RESP;
rte_eth_macaddr_get(member_id, &m_hdr->eth_hdr.src_addr);
@@ -1372,7 +1363,8 @@ bond_mode_8023ad_handle_slow_pkt(struct bond_dev_private *internals,
if (internals->mode4.dedicated_queues.enabled == 0) {
if (rte_ring_enqueue(port->tx_ring, pkt) != 0) {
/* reset timer */
- port->rx_marker_timer = 0;
+ rte_atomic_store_explicit(&port->rx_marker_timer, 0,
+ rte_memory_order_release);
wrn = WRN_TX_QUEUE_FULL;
goto free_out;
}
@@ -1386,7 +1378,8 @@ bond_mode_8023ad_handle_slow_pkt(struct bond_dev_private *internals,
&pkt, tx_count);
if (tx_count != 1) {
/* reset timer */
- port->rx_marker_timer = 0;
+ rte_atomic_store_explicit(&port->rx_marker_timer, 0,
+ rte_memory_order_release);
wrn = WRN_TX_QUEUE_FULL;
goto free_out;
}
--
2.53.0
^ permalink raw reply related
* [PATCH v3 04/27] bpf: replace atomic op macro with typed helpers
From: Stephen Hemminger @ 2026-05-23 19:56 UTC (permalink / raw)
To: dev; +Cc: Stephen Hemminger, Konstantin Ananyev, Marat Khalili
In-Reply-To: <20260523195839.454952-1-stephen@networkplumber.org>
The BPF_ST_ATOMIC_REG macro token-pasted the legacy rte_atomicNN_*()
API names. It also stacked three casts on the destination pointer
and reached a 'return 0' out of the macro into the caller's control
flow.
Replace it with two small static-inline helpers, bpf_atomic32() and
bpf_atomic64(), that dispatch on ins->imm internally and use the C11
atomic intrinsics directly. The destination is cast once, to a
properly __rte_atomic-qualified pointer. The helpers return a status
and the dispatch loop owns the early exit.
Use memory order seq_cst to preserve the previous behavior of
rte_atomicNN_add() / rte_atomicNN_exchange() and matches
the Linux kernel BPF interpreter for these opcodes.
Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
lib/bpf/bpf_exec.c | 91 ++++++++++++++++++++++++++++++++++------------
1 file changed, 67 insertions(+), 24 deletions(-)
diff --git a/lib/bpf/bpf_exec.c b/lib/bpf/bpf_exec.c
index 18013753b1..b8116db191 100644
--- a/lib/bpf/bpf_exec.c
+++ b/lib/bpf/bpf_exec.c
@@ -64,28 +64,6 @@
(*(type *)(uintptr_t)((reg)[(ins)->dst_reg] + (ins)->off) = \
(type)(reg)[(ins)->src_reg])
-#define BPF_ST_ATOMIC_REG(reg, ins, tp) do { \
- switch (ins->imm) { \
- case BPF_ATOMIC_ADD: \
- rte_atomic##tp##_add((rte_atomic##tp##_t *) \
- (uintptr_t)((reg)[(ins)->dst_reg] + (ins)->off), \
- (reg)[(ins)->src_reg]); \
- break; \
- case BPF_ATOMIC_XCHG: \
- (reg)[(ins)->src_reg] = rte_atomic##tp##_exchange((uint##tp##_t *) \
- (uintptr_t)((reg)[(ins)->dst_reg] + (ins)->off), \
- (reg)[(ins)->src_reg]); \
- break; \
- default: \
- /* this should be caught by validator and never reach here */ \
- RTE_BPF_LOG_LINE(ERR, \
- "%s(%p): unsupported atomic operation at pc: %#zx;", \
- __func__, bpf, \
- (uintptr_t)(ins) - (uintptr_t)(bpf)->prm.ins); \
- return 0; \
- } \
-} while (0)
-
/* BPF_LD | BPF_ABS/BPF_IND */
#define NOP(x) (x)
@@ -105,6 +83,69 @@
reg[EBPF_REG_0] = op(p[0]); \
} while (0)
+/*
+ * Atomic ops on the BPF target memory.
+ *
+ * BPF atomic instructions encode the destination as base register +
+ * signed offset, with the value to combine taken from src_reg.
+ *
+ * Memory order: seq_cst preserves the previous behavior of
+ * rte_atomicNN_add() / rte_atomicNN_exchange() and matches what the
+ * Linux kernel BPF interpreter does for these opcodes.
+ *
+ * Returns 0 on unsupported sub-op (validator should have rejected it),
+ * 1 otherwise.
+ */
+static inline int
+bpf_atomic32(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM],
+ const struct ebpf_insn *ins)
+{
+ /* need to casts to make bpf memory suitable for C11 atomic */
+ uint32_t __rte_atomic *dst
+ = (uint32_t __rte_atomic *)(uintptr_t)(reg[ins->dst_reg] + ins->off);
+ uint32_t val = (uint32_t)reg[ins->src_reg];
+
+ switch (ins->imm) {
+ case BPF_ATOMIC_ADD:
+ rte_atomic_fetch_add_explicit(dst, val, rte_memory_order_seq_cst);
+ return 1;
+ case BPF_ATOMIC_XCHG:
+ reg[ins->src_reg] = rte_atomic_exchange_explicit(dst, val,
+ rte_memory_order_seq_cst);
+ return 1;
+ default:
+ RTE_BPF_LOG_LINE(ERR,
+ "%s(%p): unsupported atomic operation at pc: %#zx;",
+ __func__, bpf,
+ (uintptr_t)ins - (uintptr_t)bpf->prm.ins);
+ return 0;
+ }
+}
+
+static inline int
+bpf_atomic64(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM],
+ const struct ebpf_insn *ins)
+{
+ uint64_t __rte_atomic *dst
+ = (uint64_t __rte_atomic *)(uintptr_t) (reg[ins->dst_reg] + ins->off);
+ uint64_t val = reg[ins->src_reg];
+
+ switch (ins->imm) {
+ case BPF_ATOMIC_ADD:
+ rte_atomic_fetch_add_explicit(dst, val, rte_memory_order_seq_cst);
+ return 1;
+ case BPF_ATOMIC_XCHG:
+ reg[ins->src_reg] = rte_atomic_exchange_explicit(dst, val,
+ rte_memory_order_seq_cst);
+ return 1;
+ default:
+ RTE_BPF_LOG_LINE(ERR,
+ "%s(%p): unsupported atomic operation at pc: %#zx;",
+ __func__, bpf,
+ (uintptr_t)ins - (uintptr_t)bpf->prm.ins);
+ return 0;
+ }
+}
static inline void
bpf_alu_be(uint64_t reg[EBPF_REG_NUM], const struct ebpf_insn *ins)
@@ -392,10 +433,12 @@ bpf_exec(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM])
break;
/* atomic instructions */
case (BPF_STX | EBPF_ATOMIC | BPF_W):
- BPF_ST_ATOMIC_REG(reg, ins, 32);
+ if (bpf_atomic32(bpf, reg, ins) == 0)
+ return 0;
break;
case (BPF_STX | EBPF_ATOMIC | EBPF_DW):
- BPF_ST_ATOMIC_REG(reg, ins, 64);
+ if (bpf_atomic64(bpf, reg, ins) == 0)
+ return 0;
break;
/* jump instructions */
case (BPF_JMP | BPF_JA):
--
2.53.0
^ permalink raw reply related
* [PATCH v3 03/27] ring: use compare-and-swap wrapper
From: Stephen Hemminger @ 2026-05-23 19:56 UTC (permalink / raw)
To: dev; +Cc: Stephen Hemminger, Konstantin Ananyev, Wathsala Vithanage
In-Reply-To: <20260523195839.454952-1-stephen@networkplumber.org>
The rte_atomic32_cmpset is deprecated. Initial attempts at
changing this with direct conversion to
rte_atomic_compare_exchange_weak_explicit()
regressed MP/MC contended performance on x86 by 10-30%,
because the C11 builtin's failure-writeback semantic forces
GCC to emit extra instructions on the CAS critical path.
Add an internal __rte_ring_compare_and_swap() wrapper that calls
__sync_bool_compare_and_swap() directly, which keeps the original
instruction sequence. Add equivalent function for MSVC.
Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
lib/ring/rte_ring_generic_pvt.h | 32 ++++++++++++++++++++++++++++----
1 file changed, 28 insertions(+), 4 deletions(-)
diff --git a/lib/ring/rte_ring_generic_pvt.h b/lib/ring/rte_ring_generic_pvt.h
index affd2d5ba7..0fb972de9e 100644
--- a/lib/ring/rte_ring_generic_pvt.h
+++ b/lib/ring/rte_ring_generic_pvt.h
@@ -18,6 +18,30 @@
* For more information please refer to <rte_ring.h>.
*/
+/**
+ * @internal optimized version of compare exchange
+ *
+ * The C11 builtin's failure-writeback semantic generates worse code on x86.
+ * Unlike rte_atomic_compare_exchange_*_explicit(), this wrapper does NOT
+ * write the actual value back to a pointer on failure. Callers in a retry
+ * loop must reload the expected value explicitly on the next iteration.
+ *
+ * Full memory barrier, equivalent to rte_memory_order_seq_cst on both
+ * success and failure.
+ */
+static __rte_always_inline bool
+__rte_ring_compare_and_swap(volatile uint32_t *dst,
+ uint32_t expected, uint32_t desired)
+{
+#if defined(RTE_TOOLCHAIN_MSVC)
+ return _InterlockedCompareExchange((volatile long *)dst,
+ (long)desired, (long)expected)
+ == (long)expected;
+#else
+ return __sync_bool_compare_and_swap(dst, expected, desired);
+#endif
+}
+
/**
* @internal This function updates tail values.
*/
@@ -108,10 +132,10 @@ __rte_ring_headtail_move_head(struct rte_ring_headtail *d,
if (is_st) {
d->head = *new_head;
success = 1;
- } else
- success = rte_atomic32_cmpset(
- (uint32_t *)(uintptr_t)&d->head,
- *old_head, *new_head);
+ } else {
+ success = __rte_ring_compare_and_swap(
+ &d->head, *old_head, *new_head);
+ }
} while (unlikely(success == 0));
return n;
}
--
2.53.0
^ permalink raw reply related
* [PATCH v3 02/27] eal: reimplement rte_smp_*mb with rte_atomic_thread_fence
From: Stephen Hemminger @ 2026-05-23 19:56 UTC (permalink / raw)
To: dev
Cc: Stephen Hemminger, Thomas Monjalon, Wathsala Vithanage, Bibo Mao,
David Christensen, Sun Yuechi, Bruce Richardson,
Konstantin Ananyev
In-Reply-To: <20260523195839.454952-1-stephen@networkplumber.org>
The rte_smp_mb(), rte_smp_wmb() and rte_smp_rmb() functions were
flagged as deprecated by commit 3ec965b6de12 ("doc: update atomic
operation deprecation") in 2021 but nothing came of it.
Reimplement them as inline wrappers over rte_atomic_thread_fence()
and drop the deprecation notice.
The API is preserved; only the implementation changes.
Generated code is unchanged on x86 (seq_cst keeps the lock-addl
trick, release/acquire collapse to a compiler barrier under TSO).
On arm64, release/acquire emit dmb ish instead of dmb ishst/ishld;
the difference is below measurement noise.
Drop restriction frm checkpatch since they are no longer
really on deprecation cycle.
Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
devtools/checkpatches.sh | 8 --
doc/guides/rel_notes/deprecation.rst | 8 --
lib/eal/arm/include/rte_atomic_32.h | 6 --
lib/eal/arm/include/rte_atomic_64.h | 6 --
lib/eal/include/generic/rte_atomic.h | 130 +++++--------------------
lib/eal/loongarch/include/rte_atomic.h | 6 --
lib/eal/ppc/include/rte_atomic.h | 6 --
lib/eal/riscv/include/rte_atomic.h | 6 --
lib/eal/x86/include/rte_atomic.h | 33 +++----
9 files changed, 37 insertions(+), 172 deletions(-)
diff --git a/devtools/checkpatches.sh b/devtools/checkpatches.sh
index f5dd77443f..81bb0fe4e8 100755
--- a/devtools/checkpatches.sh
+++ b/devtools/checkpatches.sh
@@ -121,14 +121,6 @@ check_forbidden_additions() { # <patch>
-f $(dirname $(readlink -f $0))/check-forbidden-tokens.awk \
"$1" || res=1
- # refrain from new additions of rte_smp_[r/w]mb()
- awk -v FOLDERS="lib drivers app examples" \
- -v EXPRESSIONS="rte_smp_(r|w)?mb\\\(" \
- -v RET_ON_FAIL=1 \
- -v MESSAGE='Using rte_smp_[r/w]mb' \
- -f $(dirname $(readlink -f $0))/check-forbidden-tokens.awk \
- "$1" || res=1
-
# refrain from using compiler __sync_xxx builtins
awk -v FOLDERS="lib drivers app examples" \
-v EXPRESSIONS="__sync_.*\\\(" \
diff --git a/doc/guides/rel_notes/deprecation.rst b/doc/guides/rel_notes/deprecation.rst
index 35c9b4e06c..2190419f79 100644
--- a/doc/guides/rel_notes/deprecation.rst
+++ b/doc/guides/rel_notes/deprecation.rst
@@ -47,14 +47,6 @@ Deprecation Notices
operations must be used for patches that need to be merged in 20.08 onwards.
This change will not introduce any performance degradation.
-* rte_smp_*mb: These APIs provide full barrier functionality. However, many
- use cases do not require full barriers. To support such use cases, DPDK has
- adopted atomic operations from
- https://gcc.gnu.org/onlinedocs/gcc/_005f_005fatomic-Builtins.html. These
- operations and a new wrapper ``rte_atomic_thread_fence`` instead of
- ``__atomic_thread_fence`` must be used for patches that need to be merged in
- 20.08 onwards. This change will not introduce any performance degradation.
-
* lib: will fix extending some enum/define breaking the ABI. There are multiple
samples in DPDK that enum/define terminated with a ``.*MAX.*`` value which is
used by iterators, and arrays holding these values are sized with this
diff --git a/lib/eal/arm/include/rte_atomic_32.h b/lib/eal/arm/include/rte_atomic_32.h
index 696a539fef..4115271091 100644
--- a/lib/eal/arm/include/rte_atomic_32.h
+++ b/lib/eal/arm/include/rte_atomic_32.h
@@ -17,12 +17,6 @@ extern "C" {
#define rte_rmb() __sync_synchronize()
-#define rte_smp_mb() rte_mb()
-
-#define rte_smp_wmb() rte_wmb()
-
-#define rte_smp_rmb() rte_rmb()
-
#define rte_io_mb() rte_mb()
#define rte_io_wmb() rte_wmb()
diff --git a/lib/eal/arm/include/rte_atomic_64.h b/lib/eal/arm/include/rte_atomic_64.h
index 9f790238df..604e777bcd 100644
--- a/lib/eal/arm/include/rte_atomic_64.h
+++ b/lib/eal/arm/include/rte_atomic_64.h
@@ -20,12 +20,6 @@ extern "C" {
#define rte_rmb() asm volatile("dmb oshld" : : : "memory")
-#define rte_smp_mb() asm volatile("dmb ish" : : : "memory")
-
-#define rte_smp_wmb() asm volatile("dmb ishst" : : : "memory")
-
-#define rte_smp_rmb() asm volatile("dmb ishld" : : : "memory")
-
#define rte_io_mb() rte_mb()
#define rte_io_wmb() rte_wmb()
diff --git a/lib/eal/include/generic/rte_atomic.h b/lib/eal/include/generic/rte_atomic.h
index 292e52fade..1b04b43cbb 100644
--- a/lib/eal/include/generic/rte_atomic.h
+++ b/lib/eal/include/generic/rte_atomic.h
@@ -59,55 +59,25 @@ static inline void rte_rmb(void);
*
* Guarantees that the LOAD and STORE operations that precede the
* rte_smp_mb() call are globally visible across the lcores
- * before the LOAD and STORE operations that follows it.
- *
- * @note
- * This function is deprecated.
- * It provides similar synchronization primitive as atomic fence,
- * but has different syntax and memory ordering semantic. Hence
- * deprecated for the simplicity of memory ordering semantics in use.
- *
- * rte_atomic_thread_fence(rte_memory_order_acq_rel) should be used instead.
+ * before the LOAD and STORE operations that follow it.
*/
static inline void rte_smp_mb(void);
/**
* Write memory barrier between lcores
*
- * Guarantees that the STORE operations that precede the
- * rte_smp_wmb() call are globally visible across the lcores
- * before the STORE operations that follows it.
- *
- * @note
- * This function is deprecated.
- * It provides similar synchronization primitive as atomic fence,
- * but has different syntax and memory ordering semantic. Hence
- * deprecated for the simplicity of memory ordering semantics in use.
- *
- * rte_atomic_thread_fence(rte_memory_order_release) should be used instead.
- * The fence also guarantees LOAD operations that precede the call
- * are globally visible across the lcores before the STORE operations
- * that follows it.
+ * Guarantees that the LOAD and STORE operations that precede the
+ * rte_smp_wmb() call are globally visible across the lcores before
+ * any STORE operations that follow it.
*/
static inline void rte_smp_wmb(void);
/**
* Read memory barrier between lcores
*
- * Guarantees that the LOAD operations that precede the
- * rte_smp_rmb() call are globally visible across the lcores
- * before the LOAD operations that follows it.
- *
- * @note
- * This function is deprecated.
- * It provides similar synchronization primitive as atomic fence,
- * but has different syntax and memory ordering semantic. Hence
- * deprecated for the simplicity of memory ordering semantics in use.
- *
- * rte_atomic_thread_fence(rte_memory_order_acquire) should be used instead.
- * The fence also guarantees LOAD operations that precede the call
- * are globally visible across the lcores before the STORE operations
- * that follows it.
+ * Guarantees that any LOAD operations that precede the rte_smp_rmb()
+ * call complete before LOAD and STORE operations that follow it
+ * become globally visible.
*/
static inline void rte_smp_rmb(void);
///@}
@@ -164,6 +134,24 @@ static inline void rte_io_rmb(void);
*/
static inline void rte_atomic_thread_fence(rte_memory_order memorder);
+static __rte_always_inline void
+rte_smp_mb(void)
+{
+ rte_atomic_thread_fence(rte_memory_order_seq_cst);
+}
+
+static __rte_always_inline void
+rte_smp_wmb(void)
+{
+ rte_atomic_thread_fence(rte_memory_order_release);
+}
+
+static __rte_always_inline void
+rte_smp_rmb(void)
+{
+ rte_atomic_thread_fence(rte_memory_order_acquire);
+}
+
/*------------------------- 16 bit atomic operations -------------------------*/
#ifndef RTE_TOOLCHAIN_MSVC
@@ -184,9 +172,6 @@ static inline void rte_atomic_thread_fence(rte_memory_order memorder);
* @return
* Non-zero on success; 0 on failure.
*/
-static inline int
-rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src);
-
static inline int
rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src)
{
@@ -303,9 +288,6 @@ rte_atomic16_sub(rte_atomic16_t *v, int16_t dec)
* @param v
* A pointer to the atomic counter.
*/
-static inline void
-rte_atomic16_inc(rte_atomic16_t *v);
-
static inline void
rte_atomic16_inc(rte_atomic16_t *v)
{
@@ -318,9 +300,6 @@ rte_atomic16_inc(rte_atomic16_t *v)
* @param v
* A pointer to the atomic counter.
*/
-static inline void
-rte_atomic16_dec(rte_atomic16_t *v);
-
static inline void
rte_atomic16_dec(rte_atomic16_t *v)
{
@@ -379,8 +358,6 @@ rte_atomic16_sub_return(rte_atomic16_t *v, int16_t dec)
* @return
* True if the result after the increment operation is 0; false otherwise.
*/
-static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v);
-
static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v)
{
return rte_atomic_fetch_add_explicit((volatile __rte_atomic int16_t *)&v->cnt, 1,
@@ -398,8 +375,6 @@ static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v)
* @return
* True if the result after the decrement operation is 0; false otherwise.
*/
-static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v);
-
static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v)
{
return rte_atomic_fetch_sub_explicit((volatile __rte_atomic int16_t *)&v->cnt, 1,
@@ -417,8 +392,6 @@ static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v)
* @return
* 0 if failed; else 1, success.
*/
-static inline int rte_atomic16_test_and_set(rte_atomic16_t *v);
-
static inline int rte_atomic16_test_and_set(rte_atomic16_t *v)
{
return rte_atomic16_cmpset((volatile uint16_t *)&v->cnt, 0, 1);
@@ -453,9 +426,6 @@ static inline void rte_atomic16_clear(rte_atomic16_t *v)
* @return
* Non-zero on success; 0 on failure.
*/
-static inline int
-rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src);
-
static inline int
rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src)
{
@@ -572,9 +542,6 @@ rte_atomic32_sub(rte_atomic32_t *v, int32_t dec)
* @param v
* A pointer to the atomic counter.
*/
-static inline void
-rte_atomic32_inc(rte_atomic32_t *v);
-
static inline void
rte_atomic32_inc(rte_atomic32_t *v)
{
@@ -587,9 +554,6 @@ rte_atomic32_inc(rte_atomic32_t *v)
* @param v
* A pointer to the atomic counter.
*/
-static inline void
-rte_atomic32_dec(rte_atomic32_t *v);
-
static inline void
rte_atomic32_dec(rte_atomic32_t *v)
{
@@ -648,8 +612,6 @@ rte_atomic32_sub_return(rte_atomic32_t *v, int32_t dec)
* @return
* True if the result after the increment operation is 0; false otherwise.
*/
-static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v);
-
static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v)
{
return rte_atomic_fetch_add_explicit((volatile __rte_atomic int32_t *)&v->cnt, 1,
@@ -667,8 +629,6 @@ static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v)
* @return
* True if the result after the decrement operation is 0; false otherwise.
*/
-static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v);
-
static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v)
{
return rte_atomic_fetch_sub_explicit((volatile __rte_atomic int32_t *)&v->cnt, 1,
@@ -686,8 +646,6 @@ static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v)
* @return
* 0 if failed; else 1, success.
*/
-static inline int rte_atomic32_test_and_set(rte_atomic32_t *v);
-
static inline int rte_atomic32_test_and_set(rte_atomic32_t *v)
{
return rte_atomic32_cmpset((volatile uint32_t *)&v->cnt, 0, 1);
@@ -721,9 +679,6 @@ static inline void rte_atomic32_clear(rte_atomic32_t *v)
* @return
* Non-zero on success; 0 on failure.
*/
-static inline int
-rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src);
-
static inline int
rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src)
{
@@ -770,9 +725,6 @@ typedef struct {
* @param v
* A pointer to the atomic counter.
*/
-static inline void
-rte_atomic64_init(rte_atomic64_t *v);
-
static inline void
rte_atomic64_init(rte_atomic64_t *v)
{
@@ -798,9 +750,6 @@ rte_atomic64_init(rte_atomic64_t *v)
* @return
* The value of the counter.
*/
-static inline int64_t
-rte_atomic64_read(rte_atomic64_t *v);
-
static inline int64_t
rte_atomic64_read(rte_atomic64_t *v)
{
@@ -828,9 +777,6 @@ rte_atomic64_read(rte_atomic64_t *v)
* @param new_value
* The new value of the counter.
*/
-static inline void
-rte_atomic64_set(rte_atomic64_t *v, int64_t new_value);
-
static inline void
rte_atomic64_set(rte_atomic64_t *v, int64_t new_value)
{
@@ -856,9 +802,6 @@ rte_atomic64_set(rte_atomic64_t *v, int64_t new_value)
* @param inc
* The value to be added to the counter.
*/
-static inline void
-rte_atomic64_add(rte_atomic64_t *v, int64_t inc);
-
static inline void
rte_atomic64_add(rte_atomic64_t *v, int64_t inc)
{
@@ -874,9 +817,6 @@ rte_atomic64_add(rte_atomic64_t *v, int64_t inc)
* @param dec
* The value to be subtracted from the counter.
*/
-static inline void
-rte_atomic64_sub(rte_atomic64_t *v, int64_t dec);
-
static inline void
rte_atomic64_sub(rte_atomic64_t *v, int64_t dec)
{
@@ -890,9 +830,6 @@ rte_atomic64_sub(rte_atomic64_t *v, int64_t dec)
* @param v
* A pointer to the atomic counter.
*/
-static inline void
-rte_atomic64_inc(rte_atomic64_t *v);
-
static inline void
rte_atomic64_inc(rte_atomic64_t *v)
{
@@ -905,9 +842,6 @@ rte_atomic64_inc(rte_atomic64_t *v)
* @param v
* A pointer to the atomic counter.
*/
-static inline void
-rte_atomic64_dec(rte_atomic64_t *v);
-
static inline void
rte_atomic64_dec(rte_atomic64_t *v)
{
@@ -927,9 +861,6 @@ rte_atomic64_dec(rte_atomic64_t *v)
* @return
* The value of v after the addition.
*/
-static inline int64_t
-rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc);
-
static inline int64_t
rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc)
{
@@ -950,9 +881,6 @@ rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc)
* @return
* The value of v after the subtraction.
*/
-static inline int64_t
-rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec);
-
static inline int64_t
rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec)
{
@@ -971,8 +899,6 @@ rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec)
* @return
* True if the result after the addition is 0; false otherwise.
*/
-static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v);
-
static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v)
{
return rte_atomic64_add_return(v, 1) == 0;
@@ -989,8 +915,6 @@ static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v)
* @return
* True if the result after subtraction is 0; false otherwise.
*/
-static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v);
-
static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v)
{
return rte_atomic64_sub_return(v, 1) == 0;
@@ -1007,8 +931,6 @@ static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v)
* @return
* 0 if failed; else 1, success.
*/
-static inline int rte_atomic64_test_and_set(rte_atomic64_t *v);
-
static inline int rte_atomic64_test_and_set(rte_atomic64_t *v)
{
return rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, 0, 1);
@@ -1020,8 +942,6 @@ static inline int rte_atomic64_test_and_set(rte_atomic64_t *v)
* @param v
* A pointer to the atomic counter.
*/
-static inline void rte_atomic64_clear(rte_atomic64_t *v);
-
static inline void rte_atomic64_clear(rte_atomic64_t *v)
{
rte_atomic64_set(v, 0);
diff --git a/lib/eal/loongarch/include/rte_atomic.h b/lib/eal/loongarch/include/rte_atomic.h
index 785a452c9e..a789e3ab4d 100644
--- a/lib/eal/loongarch/include/rte_atomic.h
+++ b/lib/eal/loongarch/include/rte_atomic.h
@@ -18,12 +18,6 @@ extern "C" {
#define rte_rmb() rte_mb()
-#define rte_smp_mb() rte_mb()
-
-#define rte_smp_wmb() rte_mb()
-
-#define rte_smp_rmb() rte_mb()
-
#define rte_io_mb() rte_mb()
#define rte_io_wmb() rte_mb()
diff --git a/lib/eal/ppc/include/rte_atomic.h b/lib/eal/ppc/include/rte_atomic.h
index 64f4c3d670..0e64db2a35 100644
--- a/lib/eal/ppc/include/rte_atomic.h
+++ b/lib/eal/ppc/include/rte_atomic.h
@@ -24,12 +24,6 @@ extern "C" {
#define rte_rmb() asm volatile("sync" : : : "memory")
-#define rte_smp_mb() rte_mb()
-
-#define rte_smp_wmb() rte_wmb()
-
-#define rte_smp_rmb() rte_rmb()
-
#define rte_io_mb() rte_mb()
#define rte_io_wmb() rte_wmb()
diff --git a/lib/eal/riscv/include/rte_atomic.h b/lib/eal/riscv/include/rte_atomic.h
index 061b175f33..04c40e4e9b 100644
--- a/lib/eal/riscv/include/rte_atomic.h
+++ b/lib/eal/riscv/include/rte_atomic.h
@@ -23,12 +23,6 @@ extern "C" {
#define rte_rmb() asm volatile("fence r, r" : : : "memory")
-#define rte_smp_mb() rte_mb()
-
-#define rte_smp_wmb() rte_wmb()
-
-#define rte_smp_rmb() rte_rmb()
-
#define rte_io_mb() asm volatile("fence iorw, iorw" : : : "memory")
#define rte_io_wmb() asm volatile("fence orw, ow" : : : "memory")
diff --git a/lib/eal/x86/include/rte_atomic.h b/lib/eal/x86/include/rte_atomic.h
index 4f05302c9f..f4d39ce4fe 100644
--- a/lib/eal/x86/include/rte_atomic.h
+++ b/lib/eal/x86/include/rte_atomic.h
@@ -23,10 +23,6 @@
#define rte_rmb() _mm_lfence()
-#define rte_smp_wmb() rte_compiler_barrier()
-
-#define rte_smp_rmb() rte_compiler_barrier()
-
#ifdef __cplusplus
extern "C" {
#endif
@@ -63,20 +59,6 @@ extern "C" {
* So below we use that technique for rte_smp_mb() implementation.
*/
-static __rte_always_inline void
-rte_smp_mb(void)
-{
-#ifdef RTE_TOOLCHAIN_MSVC
- _mm_mfence();
-#else
-#ifdef RTE_ARCH_I686
- asm volatile("lock addl $0, -128(%%esp); " ::: "memory");
-#else
- asm volatile("lock addl $0, -128(%%rsp); " ::: "memory");
-#endif
-#endif
-}
-
#define rte_io_mb() rte_mb()
#define rte_io_wmb() rte_compiler_barrier()
@@ -93,10 +75,19 @@ rte_smp_mb(void)
static __rte_always_inline void
rte_atomic_thread_fence(rte_memory_order memorder)
{
- if (memorder == rte_memory_order_seq_cst)
- rte_smp_mb();
- else
+ if (memorder == rte_memory_order_seq_cst) {
+#ifdef RTE_TOOLCHAIN_MSVC
+ _mm_mfence();
+#else
+#ifdef RTE_ARCH_I686
+ asm volatile("lock addl $0, -128(%%esp); " ::: "memory");
+#else
+ asm volatile("lock addl $0, -128(%%rsp); " ::: "memory");
+#endif
+#endif
+ } else {
__rte_atomic_thread_fence(memorder);
+ }
}
#ifdef __cplusplus
--
2.53.0
^ permalink raw reply related
* [PATCH v3 01/27] eal: use intrinsics for rte_atomic on all platforms
From: Stephen Hemminger @ 2026-05-23 19:56 UTC (permalink / raw)
To: dev
Cc: Stephen Hemminger, Wathsala Vithanage, Bibo Mao,
David Christensen, Sun Yuechi, Bruce Richardson,
Konstantin Ananyev
In-Reply-To: <20260523195839.454952-1-stephen@networkplumber.org>
Next step is to deprecate the rte_atomicNN_*() family. Rather than
maintaining both the inline asm and intrinsic fallbacks, drop the
asm paths and use intrinsics everywhere.
Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
lib/eal/arm/include/rte_atomic_32.h | 4 -
lib/eal/arm/include/rte_atomic_64.h | 4 -
lib/eal/include/generic/rte_atomic.h | 76 +---------
lib/eal/loongarch/include/rte_atomic.h | 4 -
lib/eal/ppc/include/rte_atomic.h | 173 -----------------------
lib/eal/riscv/include/rte_atomic.h | 4 -
lib/eal/x86/include/rte_atomic.h | 172 ----------------------
lib/eal/x86/include/rte_atomic_32.h | 188 -------------------------
lib/eal/x86/include/rte_atomic_64.h | 157 ---------------------
9 files changed, 6 insertions(+), 776 deletions(-)
diff --git a/lib/eal/arm/include/rte_atomic_32.h b/lib/eal/arm/include/rte_atomic_32.h
index 0b9a0dfa30..696a539fef 100644
--- a/lib/eal/arm/include/rte_atomic_32.h
+++ b/lib/eal/arm/include/rte_atomic_32.h
@@ -5,10 +5,6 @@
#ifndef _RTE_ATOMIC_ARM32_H_
#define _RTE_ATOMIC_ARM32_H_
-#ifndef RTE_FORCE_INTRINSICS
-# error Platform must be built with RTE_FORCE_INTRINSICS
-#endif
-
#include "generic/rte_atomic.h"
#ifdef __cplusplus
diff --git a/lib/eal/arm/include/rte_atomic_64.h b/lib/eal/arm/include/rte_atomic_64.h
index 181bb60929..9f790238df 100644
--- a/lib/eal/arm/include/rte_atomic_64.h
+++ b/lib/eal/arm/include/rte_atomic_64.h
@@ -6,10 +6,6 @@
#ifndef _RTE_ATOMIC_ARM64_H_
#define _RTE_ATOMIC_ARM64_H_
-#ifndef RTE_FORCE_INTRINSICS
-# error Platform must be built with RTE_FORCE_INTRINSICS
-#endif
-
#include "generic/rte_atomic.h"
#include <rte_branch_prediction.h>
#include <rte_debug.h>
diff --git a/lib/eal/include/generic/rte_atomic.h b/lib/eal/include/generic/rte_atomic.h
index 0a4f3f8528..292e52fade 100644
--- a/lib/eal/include/generic/rte_atomic.h
+++ b/lib/eal/include/generic/rte_atomic.h
@@ -187,13 +187,11 @@ static inline void rte_atomic_thread_fence(rte_memory_order memorder);
static inline int
rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src);
-#ifdef RTE_FORCE_INTRINSICS
static inline int
rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src)
{
return __sync_bool_compare_and_swap(dst, exp, src);
}
-#endif
/**
* Atomic exchange.
@@ -211,15 +209,11 @@ rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src)
* The original value at that location
*/
static inline uint16_t
-rte_atomic16_exchange(volatile uint16_t *dst, uint16_t val);
-
-#ifdef RTE_FORCE_INTRINSICS
-static inline uint16_t
rte_atomic16_exchange(volatile uint16_t *dst, uint16_t val)
{
- return rte_atomic_exchange_explicit(dst, val, rte_memory_order_seq_cst);
+ return rte_atomic_exchange_explicit((volatile __rte_atomic uint16_t *)dst,
+ val, rte_memory_order_seq_cst);
}
-#endif
/**
* The atomic counter structure.
@@ -312,13 +306,11 @@ rte_atomic16_sub(rte_atomic16_t *v, int16_t dec)
static inline void
rte_atomic16_inc(rte_atomic16_t *v);
-#ifdef RTE_FORCE_INTRINSICS
static inline void
rte_atomic16_inc(rte_atomic16_t *v)
{
rte_atomic16_add(v, 1);
}
-#endif
/**
* Atomically decrement a counter by one.
@@ -329,13 +321,11 @@ rte_atomic16_inc(rte_atomic16_t *v)
static inline void
rte_atomic16_dec(rte_atomic16_t *v);
-#ifdef RTE_FORCE_INTRINSICS
static inline void
rte_atomic16_dec(rte_atomic16_t *v)
{
rte_atomic16_sub(v, 1);
}
-#endif
/**
* Atomically add a 16-bit value to a counter and return the result.
@@ -391,13 +381,11 @@ rte_atomic16_sub_return(rte_atomic16_t *v, int16_t dec)
*/
static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v);
-#ifdef RTE_FORCE_INTRINSICS
static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v)
{
return rte_atomic_fetch_add_explicit((volatile __rte_atomic int16_t *)&v->cnt, 1,
rte_memory_order_seq_cst) + 1 == 0;
}
-#endif
/**
* Atomically decrement a 16-bit counter by one and test.
@@ -412,13 +400,11 @@ static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v)
*/
static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v);
-#ifdef RTE_FORCE_INTRINSICS
static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v)
{
return rte_atomic_fetch_sub_explicit((volatile __rte_atomic int16_t *)&v->cnt, 1,
rte_memory_order_seq_cst) - 1 == 0;
}
-#endif
/**
* Atomically test and set a 16-bit atomic counter.
@@ -433,12 +419,10 @@ static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v)
*/
static inline int rte_atomic16_test_and_set(rte_atomic16_t *v);
-#ifdef RTE_FORCE_INTRINSICS
static inline int rte_atomic16_test_and_set(rte_atomic16_t *v)
{
return rte_atomic16_cmpset((volatile uint16_t *)&v->cnt, 0, 1);
}
-#endif
/**
* Atomically set a 16-bit counter to 0.
@@ -472,13 +456,11 @@ static inline void rte_atomic16_clear(rte_atomic16_t *v)
static inline int
rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src);
-#ifdef RTE_FORCE_INTRINSICS
static inline int
rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src)
{
return __sync_bool_compare_and_swap(dst, exp, src);
}
-#endif
/**
* Atomic exchange.
@@ -496,15 +478,11 @@ rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src)
* The original value at that location
*/
static inline uint32_t
-rte_atomic32_exchange(volatile uint32_t *dst, uint32_t val);
-
-#ifdef RTE_FORCE_INTRINSICS
-static inline uint32_t
rte_atomic32_exchange(volatile uint32_t *dst, uint32_t val)
{
- return rte_atomic_exchange_explicit(dst, val, rte_memory_order_seq_cst);
+ return rte_atomic_exchange_explicit((volatile __rte_atomic uint32_t *)dst,
+ val, rte_memory_order_seq_cst);
}
-#endif
/**
* The atomic counter structure.
@@ -597,13 +575,11 @@ rte_atomic32_sub(rte_atomic32_t *v, int32_t dec)
static inline void
rte_atomic32_inc(rte_atomic32_t *v);
-#ifdef RTE_FORCE_INTRINSICS
static inline void
rte_atomic32_inc(rte_atomic32_t *v)
{
rte_atomic32_add(v, 1);
}
-#endif
/**
* Atomically decrement a counter by one.
@@ -614,13 +590,11 @@ rte_atomic32_inc(rte_atomic32_t *v)
static inline void
rte_atomic32_dec(rte_atomic32_t *v);
-#ifdef RTE_FORCE_INTRINSICS
static inline void
rte_atomic32_dec(rte_atomic32_t *v)
{
rte_atomic32_sub(v,1);
}
-#endif
/**
* Atomically add a 32-bit value to a counter and return the result.
@@ -676,13 +650,11 @@ rte_atomic32_sub_return(rte_atomic32_t *v, int32_t dec)
*/
static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v);
-#ifdef RTE_FORCE_INTRINSICS
static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v)
{
return rte_atomic_fetch_add_explicit((volatile __rte_atomic int32_t *)&v->cnt, 1,
rte_memory_order_seq_cst) + 1 == 0;
}
-#endif
/**
* Atomically decrement a 32-bit counter by one and test.
@@ -697,13 +669,11 @@ static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v)
*/
static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v);
-#ifdef RTE_FORCE_INTRINSICS
static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v)
{
return rte_atomic_fetch_sub_explicit((volatile __rte_atomic int32_t *)&v->cnt, 1,
rte_memory_order_seq_cst) - 1 == 0;
}
-#endif
/**
* Atomically test and set a 32-bit atomic counter.
@@ -718,12 +688,10 @@ static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v)
*/
static inline int rte_atomic32_test_and_set(rte_atomic32_t *v);
-#ifdef RTE_FORCE_INTRINSICS
static inline int rte_atomic32_test_and_set(rte_atomic32_t *v)
{
return rte_atomic32_cmpset((volatile uint32_t *)&v->cnt, 0, 1);
}
-#endif
/**
* Atomically set a 32-bit counter to 0.
@@ -756,13 +724,11 @@ static inline void rte_atomic32_clear(rte_atomic32_t *v)
static inline int
rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src);
-#ifdef RTE_FORCE_INTRINSICS
static inline int
rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src)
{
return __sync_bool_compare_and_swap(dst, exp, src);
}
-#endif
/**
* Atomic exchange.
@@ -780,15 +746,11 @@ rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src)
* The original value at that location
*/
static inline uint64_t
-rte_atomic64_exchange(volatile uint64_t *dst, uint64_t val);
-
-#ifdef RTE_FORCE_INTRINSICS
-static inline uint64_t
rte_atomic64_exchange(volatile uint64_t *dst, uint64_t val)
{
- return rte_atomic_exchange_explicit(dst, val, rte_memory_order_seq_cst);
+ return rte_atomic_exchange_explicit((volatile __rte_atomic uint64_t *)dst,
+ val, rte_memory_order_seq_cst);
}
-#endif
/**
* The atomic counter structure.
@@ -811,7 +773,6 @@ typedef struct {
static inline void
rte_atomic64_init(rte_atomic64_t *v);
-#ifdef RTE_FORCE_INTRINSICS
static inline void
rte_atomic64_init(rte_atomic64_t *v)
{
@@ -828,7 +789,6 @@ rte_atomic64_init(rte_atomic64_t *v)
}
#endif
}
-#endif
/**
* Atomically read a 64-bit counter.
@@ -841,7 +801,6 @@ rte_atomic64_init(rte_atomic64_t *v)
static inline int64_t
rte_atomic64_read(rte_atomic64_t *v);
-#ifdef RTE_FORCE_INTRINSICS
static inline int64_t
rte_atomic64_read(rte_atomic64_t *v)
{
@@ -860,7 +819,6 @@ rte_atomic64_read(rte_atomic64_t *v)
return tmp;
#endif
}
-#endif
/**
* Atomically set a 64-bit counter.
@@ -873,7 +831,6 @@ rte_atomic64_read(rte_atomic64_t *v)
static inline void
rte_atomic64_set(rte_atomic64_t *v, int64_t new_value);
-#ifdef RTE_FORCE_INTRINSICS
static inline void
rte_atomic64_set(rte_atomic64_t *v, int64_t new_value)
{
@@ -890,7 +847,6 @@ rte_atomic64_set(rte_atomic64_t *v, int64_t new_value)
}
#endif
}
-#endif
/**
* Atomically add a 64-bit value to a counter.
@@ -903,14 +859,12 @@ rte_atomic64_set(rte_atomic64_t *v, int64_t new_value)
static inline void
rte_atomic64_add(rte_atomic64_t *v, int64_t inc);
-#ifdef RTE_FORCE_INTRINSICS
static inline void
rte_atomic64_add(rte_atomic64_t *v, int64_t inc)
{
rte_atomic_fetch_add_explicit((volatile __rte_atomic int64_t *)&v->cnt, inc,
rte_memory_order_seq_cst);
}
-#endif
/**
* Atomically subtract a 64-bit value from a counter.
@@ -923,14 +877,12 @@ rte_atomic64_add(rte_atomic64_t *v, int64_t inc)
static inline void
rte_atomic64_sub(rte_atomic64_t *v, int64_t dec);
-#ifdef RTE_FORCE_INTRINSICS
static inline void
rte_atomic64_sub(rte_atomic64_t *v, int64_t dec)
{
rte_atomic_fetch_sub_explicit((volatile __rte_atomic int64_t *)&v->cnt, dec,
rte_memory_order_seq_cst);
}
-#endif
/**
* Atomically increment a 64-bit counter by one and test.
@@ -941,13 +893,11 @@ rte_atomic64_sub(rte_atomic64_t *v, int64_t dec)
static inline void
rte_atomic64_inc(rte_atomic64_t *v);
-#ifdef RTE_FORCE_INTRINSICS
static inline void
rte_atomic64_inc(rte_atomic64_t *v)
{
rte_atomic64_add(v, 1);
}
-#endif
/**
* Atomically decrement a 64-bit counter by one and test.
@@ -958,13 +908,11 @@ rte_atomic64_inc(rte_atomic64_t *v)
static inline void
rte_atomic64_dec(rte_atomic64_t *v);
-#ifdef RTE_FORCE_INTRINSICS
static inline void
rte_atomic64_dec(rte_atomic64_t *v)
{
rte_atomic64_sub(v, 1);
}
-#endif
/**
* Add a 64-bit value to an atomic counter and return the result.
@@ -982,14 +930,12 @@ rte_atomic64_dec(rte_atomic64_t *v)
static inline int64_t
rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc);
-#ifdef RTE_FORCE_INTRINSICS
static inline int64_t
rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc)
{
return rte_atomic_fetch_add_explicit((volatile __rte_atomic int64_t *)&v->cnt, inc,
rte_memory_order_seq_cst) + inc;
}
-#endif
/**
* Subtract a 64-bit value from an atomic counter and return the result.
@@ -1007,14 +953,12 @@ rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc)
static inline int64_t
rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec);
-#ifdef RTE_FORCE_INTRINSICS
static inline int64_t
rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec)
{
return rte_atomic_fetch_sub_explicit((volatile __rte_atomic int64_t *)&v->cnt, dec,
rte_memory_order_seq_cst) - dec;
}
-#endif
/**
* Atomically increment a 64-bit counter by one and test.
@@ -1029,12 +973,10 @@ rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec)
*/
static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v);
-#ifdef RTE_FORCE_INTRINSICS
static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v)
{
return rte_atomic64_add_return(v, 1) == 0;
}
-#endif
/**
* Atomically decrement a 64-bit counter by one and test.
@@ -1049,12 +991,10 @@ static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v)
*/
static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v);
-#ifdef RTE_FORCE_INTRINSICS
static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v)
{
return rte_atomic64_sub_return(v, 1) == 0;
}
-#endif
/**
* Atomically test and set a 64-bit atomic counter.
@@ -1069,12 +1009,10 @@ static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v)
*/
static inline int rte_atomic64_test_and_set(rte_atomic64_t *v);
-#ifdef RTE_FORCE_INTRINSICS
static inline int rte_atomic64_test_and_set(rte_atomic64_t *v)
{
return rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, 0, 1);
}
-#endif
/**
* Atomically set a 64-bit counter to 0.
@@ -1084,12 +1022,10 @@ static inline int rte_atomic64_test_and_set(rte_atomic64_t *v)
*/
static inline void rte_atomic64_clear(rte_atomic64_t *v);
-#ifdef RTE_FORCE_INTRINSICS
static inline void rte_atomic64_clear(rte_atomic64_t *v)
{
rte_atomic64_set(v, 0);
}
-#endif
#endif
diff --git a/lib/eal/loongarch/include/rte_atomic.h b/lib/eal/loongarch/include/rte_atomic.h
index c8066a4612..785a452c9e 100644
--- a/lib/eal/loongarch/include/rte_atomic.h
+++ b/lib/eal/loongarch/include/rte_atomic.h
@@ -5,10 +5,6 @@
#ifndef RTE_ATOMIC_LOONGARCH_H
#define RTE_ATOMIC_LOONGARCH_H
-#ifndef RTE_FORCE_INTRINSICS
-# error Platform must be built with RTE_FORCE_INTRINSICS
-#endif
-
#include <rte_common.h>
#include "generic/rte_atomic.h"
diff --git a/lib/eal/ppc/include/rte_atomic.h b/lib/eal/ppc/include/rte_atomic.h
index 10acc238f9..64f4c3d670 100644
--- a/lib/eal/ppc/include/rte_atomic.h
+++ b/lib/eal/ppc/include/rte_atomic.h
@@ -43,179 +43,6 @@ rte_atomic_thread_fence(rte_memory_order memorder)
}
/*------------------------- 16 bit atomic operations -------------------------*/
-#ifndef RTE_FORCE_INTRINSICS
-static inline int
-rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src)
-{
- return rte_atomic_compare_exchange_strong_explicit(dst, &exp, src, rte_memory_order_acquire,
- rte_memory_order_acquire) ? 1 : 0;
-}
-
-static inline int rte_atomic16_test_and_set(rte_atomic16_t *v)
-{
- return rte_atomic16_cmpset((volatile uint16_t *)&v->cnt, 0, 1);
-}
-
-static inline void
-rte_atomic16_inc(rte_atomic16_t *v)
-{
- rte_atomic_fetch_add_explicit(&v->cnt, 1, rte_memory_order_acquire);
-}
-
-static inline void
-rte_atomic16_dec(rte_atomic16_t *v)
-{
- rte_atomic_fetch_sub_explicit(&v->cnt, 1, rte_memory_order_acquire);
-}
-
-static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v)
-{
- return rte_atomic_fetch_add_explicit(&v->cnt, 1, rte_memory_order_acquire) + 1 == 0;
-}
-
-static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v)
-{
- return rte_atomic_fetch_sub_explicit(&v->cnt, 1, rte_memory_order_acquire) - 1 == 0;
-}
-
-static inline uint16_t
-rte_atomic16_exchange(volatile uint16_t *dst, uint16_t val)
-{
- return __atomic_exchange_2(dst, val, rte_memory_order_seq_cst);
-}
-
-/*------------------------- 32 bit atomic operations -------------------------*/
-
-static inline int
-rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src)
-{
- return rte_atomic_compare_exchange_strong_explicit(dst, &exp, src, rte_memory_order_acquire,
- rte_memory_order_acquire) ? 1 : 0;
-}
-
-static inline int rte_atomic32_test_and_set(rte_atomic32_t *v)
-{
- return rte_atomic32_cmpset((volatile uint32_t *)&v->cnt, 0, 1);
-}
-
-static inline void
-rte_atomic32_inc(rte_atomic32_t *v)
-{
- rte_atomic_fetch_add_explicit(&v->cnt, 1, rte_memory_order_acquire);
-}
-
-static inline void
-rte_atomic32_dec(rte_atomic32_t *v)
-{
- rte_atomic_fetch_sub_explicit(&v->cnt, 1, rte_memory_order_acquire);
-}
-
-static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v)
-{
- return rte_atomic_fetch_add_explicit(&v->cnt, 1, rte_memory_order_acquire) + 1 == 0;
-}
-
-static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v)
-{
- return rte_atomic_fetch_sub_explicit(&v->cnt, 1, rte_memory_order_acquire) - 1 == 0;
-}
-
-static inline uint32_t
-rte_atomic32_exchange(volatile uint32_t *dst, uint32_t val)
-{
- return __atomic_exchange_4(dst, val, rte_memory_order_seq_cst);
-}
-
-/*------------------------- 64 bit atomic operations -------------------------*/
-
-static inline int
-rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src)
-{
- return rte_atomic_compare_exchange_strong_explicit(dst, &exp, src, rte_memory_order_acquire,
- rte_memory_order_acquire) ? 1 : 0;
-}
-
-static inline void
-rte_atomic64_init(rte_atomic64_t *v)
-{
- v->cnt = 0;
-}
-
-static inline int64_t
-rte_atomic64_read(rte_atomic64_t *v)
-{
- return v->cnt;
-}
-
-static inline void
-rte_atomic64_set(rte_atomic64_t *v, int64_t new_value)
-{
- v->cnt = new_value;
-}
-
-static inline void
-rte_atomic64_add(rte_atomic64_t *v, int64_t inc)
-{
- rte_atomic_fetch_add_explicit(&v->cnt, inc, rte_memory_order_acquire);
-}
-
-static inline void
-rte_atomic64_sub(rte_atomic64_t *v, int64_t dec)
-{
- rte_atomic_fetch_sub_explicit(&v->cnt, dec, rte_memory_order_acquire);
-}
-
-static inline void
-rte_atomic64_inc(rte_atomic64_t *v)
-{
- rte_atomic_fetch_add_explicit(&v->cnt, 1, rte_memory_order_acquire);
-}
-
-static inline void
-rte_atomic64_dec(rte_atomic64_t *v)
-{
- rte_atomic_fetch_sub_explicit(&v->cnt, 1, rte_memory_order_acquire);
-}
-
-static inline int64_t
-rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc)
-{
- return rte_atomic_fetch_add_explicit(&v->cnt, inc, rte_memory_order_acquire) + inc;
-}
-
-static inline int64_t
-rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec)
-{
- return rte_atomic_fetch_sub_explicit(&v->cnt, dec, rte_memory_order_acquire) - dec;
-}
-
-static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v)
-{
- return rte_atomic_fetch_add_explicit(&v->cnt, 1, rte_memory_order_acquire) + 1 == 0;
-}
-
-static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v)
-{
- return rte_atomic_fetch_sub_explicit(&v->cnt, 1, rte_memory_order_acquire) - 1 == 0;
-}
-
-static inline int rte_atomic64_test_and_set(rte_atomic64_t *v)
-{
- return rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, 0, 1);
-}
-
-static inline void rte_atomic64_clear(rte_atomic64_t *v)
-{
- v->cnt = 0;
-}
-
-static inline uint64_t
-rte_atomic64_exchange(volatile uint64_t *dst, uint64_t val)
-{
- return __atomic_exchange_8(dst, val, rte_memory_order_seq_cst);
-}
-
-#endif
#ifdef __cplusplus
}
diff --git a/lib/eal/riscv/include/rte_atomic.h b/lib/eal/riscv/include/rte_atomic.h
index 66346ad474..061b175f33 100644
--- a/lib/eal/riscv/include/rte_atomic.h
+++ b/lib/eal/riscv/include/rte_atomic.h
@@ -8,10 +8,6 @@
#ifndef RTE_ATOMIC_RISCV_H
#define RTE_ATOMIC_RISCV_H
-#ifndef RTE_FORCE_INTRINSICS
-# error Platform must be built with RTE_FORCE_INTRINSICS
-#endif
-
#include <stdint.h>
#include <rte_common.h>
#include <rte_config.h>
diff --git a/lib/eal/x86/include/rte_atomic.h b/lib/eal/x86/include/rte_atomic.h
index e071e4234e..4f05302c9f 100644
--- a/lib/eal/x86/include/rte_atomic.h
+++ b/lib/eal/x86/include/rte_atomic.h
@@ -111,178 +111,6 @@ rte_atomic_thread_fence(rte_memory_order memorder)
extern "C" {
#endif
-#ifndef RTE_FORCE_INTRINSICS
-static inline int
-rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src)
-{
- uint8_t res;
-
- asm volatile(
- MPLOCKED
- "cmpxchgw %[src], %[dst];"
- "sete %[res];"
- : [res] "=a" (res), /* output */
- [dst] "=m" (*dst)
- : [src] "r" (src), /* input */
- "a" (exp),
- "m" (*dst)
- : "memory"); /* no-clobber list */
- return res;
-}
-
-static inline uint16_t
-rte_atomic16_exchange(volatile uint16_t *dst, uint16_t val)
-{
- asm volatile(
- MPLOCKED
- "xchgw %0, %1;"
- : "=r" (val), "=m" (*dst)
- : "0" (val), "m" (*dst)
- : "memory"); /* no-clobber list */
- return val;
-}
-
-static inline int rte_atomic16_test_and_set(rte_atomic16_t *v)
-{
- return rte_atomic16_cmpset((volatile uint16_t *)&v->cnt, 0, 1);
-}
-
-static inline void
-rte_atomic16_inc(rte_atomic16_t *v)
-{
- asm volatile(
- MPLOCKED
- "incw %[cnt]"
- : [cnt] "=m" (v->cnt) /* output */
- : "m" (v->cnt) /* input */
- );
-}
-
-static inline void
-rte_atomic16_dec(rte_atomic16_t *v)
-{
- asm volatile(
- MPLOCKED
- "decw %[cnt]"
- : [cnt] "=m" (v->cnt) /* output */
- : "m" (v->cnt) /* input */
- );
-}
-
-static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v)
-{
- uint8_t ret;
-
- asm volatile(
- MPLOCKED
- "incw %[cnt] ; "
- "sete %[ret]"
- : [cnt] "+m" (v->cnt), /* output */
- [ret] "=qm" (ret)
- );
- return ret != 0;
-}
-
-static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v)
-{
- uint8_t ret;
-
- asm volatile(MPLOCKED
- "decw %[cnt] ; "
- "sete %[ret]"
- : [cnt] "+m" (v->cnt), /* output */
- [ret] "=qm" (ret)
- );
- return ret != 0;
-}
-
-/*------------------------- 32 bit atomic operations -------------------------*/
-
-static inline int
-rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src)
-{
- uint8_t res;
-
- asm volatile(
- MPLOCKED
- "cmpxchgl %[src], %[dst];"
- "sete %[res];"
- : [res] "=a" (res), /* output */
- [dst] "=m" (*dst)
- : [src] "r" (src), /* input */
- "a" (exp),
- "m" (*dst)
- : "memory"); /* no-clobber list */
- return res;
-}
-
-static inline uint32_t
-rte_atomic32_exchange(volatile uint32_t *dst, uint32_t val)
-{
- asm volatile(
- MPLOCKED
- "xchgl %0, %1;"
- : "=r" (val), "=m" (*dst)
- : "0" (val), "m" (*dst)
- : "memory"); /* no-clobber list */
- return val;
-}
-
-static inline int rte_atomic32_test_and_set(rte_atomic32_t *v)
-{
- return rte_atomic32_cmpset((volatile uint32_t *)&v->cnt, 0, 1);
-}
-
-static inline void
-rte_atomic32_inc(rte_atomic32_t *v)
-{
- asm volatile(
- MPLOCKED
- "incl %[cnt]"
- : [cnt] "=m" (v->cnt) /* output */
- : "m" (v->cnt) /* input */
- );
-}
-
-static inline void
-rte_atomic32_dec(rte_atomic32_t *v)
-{
- asm volatile(
- MPLOCKED
- "decl %[cnt]"
- : [cnt] "=m" (v->cnt) /* output */
- : "m" (v->cnt) /* input */
- );
-}
-
-static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v)
-{
- uint8_t ret;
-
- asm volatile(
- MPLOCKED
- "incl %[cnt] ; "
- "sete %[ret]"
- : [cnt] "+m" (v->cnt), /* output */
- [ret] "=qm" (ret)
- );
- return ret != 0;
-}
-
-static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v)
-{
- uint8_t ret;
-
- asm volatile(MPLOCKED
- "decl %[cnt] ; "
- "sete %[ret]"
- : [cnt] "+m" (v->cnt), /* output */
- [ret] "=qm" (ret)
- );
- return ret != 0;
-}
-
-#endif /* !RTE_FORCE_INTRINSICS */
#ifdef __cplusplus
}
diff --git a/lib/eal/x86/include/rte_atomic_32.h b/lib/eal/x86/include/rte_atomic_32.h
index 0f25863aa5..37d139f30d 100644
--- a/lib/eal/x86/include/rte_atomic_32.h
+++ b/lib/eal/x86/include/rte_atomic_32.h
@@ -20,193 +20,5 @@
/*------------------------- 64 bit atomic operations -------------------------*/
-#ifndef RTE_FORCE_INTRINSICS
-static inline int
-rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src)
-{
- uint8_t res;
- union {
- struct {
- uint32_t l32;
- uint32_t h32;
- };
- uint64_t u64;
- } _exp, _src;
-
- _exp.u64 = exp;
- _src.u64 = src;
-
-#ifndef __PIC__
- asm volatile (
- MPLOCKED
- "cmpxchg8b (%[dst]);"
- "setz %[res];"
- : [res] "=a" (res) /* result in eax */
- : [dst] "S" (dst), /* esi */
- "b" (_src.l32), /* ebx */
- "c" (_src.h32), /* ecx */
- "a" (_exp.l32), /* eax */
- "d" (_exp.h32) /* edx */
- : "memory" ); /* no-clobber list */
-#else
- asm volatile (
- "xchgl %%ebx, %%edi;\n"
- MPLOCKED
- "cmpxchg8b (%[dst]);"
- "setz %[res];"
- "xchgl %%ebx, %%edi;\n"
- : [res] "=a" (res) /* result in eax */
- : [dst] "S" (dst), /* esi */
- "D" (_src.l32), /* ebx */
- "c" (_src.h32), /* ecx */
- "a" (_exp.l32), /* eax */
- "d" (_exp.h32) /* edx */
- : "memory" ); /* no-clobber list */
-#endif
-
- return res;
-}
-
-static inline uint64_t
-rte_atomic64_exchange(volatile uint64_t *dest, uint64_t val)
-{
- uint64_t old;
-
- do {
- old = *dest;
- } while (rte_atomic64_cmpset(dest, old, val) == 0);
-
- return old;
-}
-
-static inline void
-rte_atomic64_init(rte_atomic64_t *v)
-{
- int success = 0;
- uint64_t tmp;
-
- while (success == 0) {
- tmp = v->cnt;
- success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
- tmp, 0);
- }
-}
-
-static inline int64_t
-rte_atomic64_read(rte_atomic64_t *v)
-{
- int success = 0;
- uint64_t tmp;
-
- while (success == 0) {
- tmp = v->cnt;
- /* replace the value by itself */
- success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
- tmp, tmp);
- }
- return tmp;
-}
-
-static inline void
-rte_atomic64_set(rte_atomic64_t *v, int64_t new_value)
-{
- int success = 0;
- uint64_t tmp;
-
- while (success == 0) {
- tmp = v->cnt;
- success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
- tmp, new_value);
- }
-}
-
-static inline void
-rte_atomic64_add(rte_atomic64_t *v, int64_t inc)
-{
- int success = 0;
- uint64_t tmp;
-
- while (success == 0) {
- tmp = v->cnt;
- success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
- tmp, tmp + inc);
- }
-}
-
-static inline void
-rte_atomic64_sub(rte_atomic64_t *v, int64_t dec)
-{
- int success = 0;
- uint64_t tmp;
-
- while (success == 0) {
- tmp = v->cnt;
- success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
- tmp, tmp - dec);
- }
-}
-
-static inline void
-rte_atomic64_inc(rte_atomic64_t *v)
-{
- rte_atomic64_add(v, 1);
-}
-
-static inline void
-rte_atomic64_dec(rte_atomic64_t *v)
-{
- rte_atomic64_sub(v, 1);
-}
-
-static inline int64_t
-rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc)
-{
- int success = 0;
- uint64_t tmp;
-
- while (success == 0) {
- tmp = v->cnt;
- success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
- tmp, tmp + inc);
- }
-
- return tmp + inc;
-}
-
-static inline int64_t
-rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec)
-{
- int success = 0;
- uint64_t tmp;
-
- while (success == 0) {
- tmp = v->cnt;
- success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
- tmp, tmp - dec);
- }
-
- return tmp - dec;
-}
-
-static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v)
-{
- return rte_atomic64_add_return(v, 1) == 0;
-}
-
-static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v)
-{
- return rte_atomic64_sub_return(v, 1) == 0;
-}
-
-static inline int rte_atomic64_test_and_set(rte_atomic64_t *v)
-{
- return rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, 0, 1);
-}
-
-static inline void rte_atomic64_clear(rte_atomic64_t *v)
-{
- rte_atomic64_set(v, 0);
-}
-#endif
#endif /* _RTE_ATOMIC_I686_H_ */
diff --git a/lib/eal/x86/include/rte_atomic_64.h b/lib/eal/x86/include/rte_atomic_64.h
index 0a7a2131e0..1cd12695a2 100644
--- a/lib/eal/x86/include/rte_atomic_64.h
+++ b/lib/eal/x86/include/rte_atomic_64.h
@@ -22,163 +22,6 @@
/*------------------------- 64 bit atomic operations -------------------------*/
-#ifndef RTE_FORCE_INTRINSICS
-static inline int
-rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src)
-{
- uint8_t res;
-
-
- asm volatile(
- MPLOCKED
- "cmpxchgq %[src], %[dst];"
- "sete %[res];"
- : [res] "=a" (res), /* output */
- [dst] "=m" (*dst)
- : [src] "r" (src), /* input */
- "a" (exp),
- "m" (*dst)
- : "memory"); /* no-clobber list */
-
- return res;
-}
-
-static inline uint64_t
-rte_atomic64_exchange(volatile uint64_t *dst, uint64_t val)
-{
- asm volatile(
- MPLOCKED
- "xchgq %0, %1;"
- : "=r" (val), "=m" (*dst)
- : "0" (val), "m" (*dst)
- : "memory"); /* no-clobber list */
- return val;
-}
-
-static inline void
-rte_atomic64_init(rte_atomic64_t *v)
-{
- v->cnt = 0;
-}
-
-static inline int64_t
-rte_atomic64_read(rte_atomic64_t *v)
-{
- return v->cnt;
-}
-
-static inline void
-rte_atomic64_set(rte_atomic64_t *v, int64_t new_value)
-{
- v->cnt = new_value;
-}
-
-static inline void
-rte_atomic64_add(rte_atomic64_t *v, int64_t inc)
-{
- asm volatile(
- MPLOCKED
- "addq %[inc], %[cnt]"
- : [cnt] "=m" (v->cnt) /* output */
- : [inc] "ir" (inc), /* input */
- "m" (v->cnt)
- );
-}
-
-static inline void
-rte_atomic64_sub(rte_atomic64_t *v, int64_t dec)
-{
- asm volatile(
- MPLOCKED
- "subq %[dec], %[cnt]"
- : [cnt] "=m" (v->cnt) /* output */
- : [dec] "ir" (dec), /* input */
- "m" (v->cnt)
- );
-}
-
-static inline void
-rte_atomic64_inc(rte_atomic64_t *v)
-{
- asm volatile(
- MPLOCKED
- "incq %[cnt]"
- : [cnt] "=m" (v->cnt) /* output */
- : "m" (v->cnt) /* input */
- );
-}
-
-static inline void
-rte_atomic64_dec(rte_atomic64_t *v)
-{
- asm volatile(
- MPLOCKED
- "decq %[cnt]"
- : [cnt] "=m" (v->cnt) /* output */
- : "m" (v->cnt) /* input */
- );
-}
-
-static inline int64_t
-rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc)
-{
- int64_t prev = inc;
-
- asm volatile(
- MPLOCKED
- "xaddq %[prev], %[cnt]"
- : [prev] "+r" (prev), /* output */
- [cnt] "=m" (v->cnt)
- : "m" (v->cnt) /* input */
- );
- return prev + inc;
-}
-
-static inline int64_t
-rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec)
-{
- return rte_atomic64_add_return(v, -dec);
-}
-
-static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v)
-{
- uint8_t ret;
-
- asm volatile(
- MPLOCKED
- "incq %[cnt] ; "
- "sete %[ret]"
- : [cnt] "+m" (v->cnt), /* output */
- [ret] "=qm" (ret)
- );
-
- return ret != 0;
-}
-
-static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v)
-{
- uint8_t ret;
-
- asm volatile(
- MPLOCKED
- "decq %[cnt] ; "
- "sete %[ret]"
- : [cnt] "+m" (v->cnt), /* output */
- [ret] "=qm" (ret)
- );
- return ret != 0;
-}
-
-static inline int rte_atomic64_test_and_set(rte_atomic64_t *v)
-{
- return rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, 0, 1);
-}
-
-static inline void rte_atomic64_clear(rte_atomic64_t *v)
-{
- v->cnt = 0;
-}
-#endif
/*------------------------ 128 bit atomic operations -------------------------*/
--
2.53.0
^ permalink raw reply related
* [PATCH v3 00/27] deprecate rte_atomicNN family
From: Stephen Hemminger @ 2026-05-23 19:56 UTC (permalink / raw)
To: dev; +Cc: Stephen Hemminger
In-Reply-To: <20260521042043.1590536-1-stephen@networkplumber.org>
The rte_atomicNN_* family was flagged for deprecation in 2021 by
commit 3ec965b6de12 ("doc: update atomic operation deprecation")
but enforcement never landed and in-tree usage continued to grow.
v2 covered the EAL changes, lib/ring, and a starter set of drivers.
This series finishes the job: convert every remaining in-tree
caller to the C11-style rte_atomic_*_explicit() / RTE_ATOMIC()
API, then mark the legacy functions __rte_deprecated so future
in-tree and out-of-tree uses are caught at compile time.
Performance: ran the DPDK perf-tests suite (mempool, hash, stack,
ring, distributor, rcu_qsbr, etc.) on the full series; only
lib/ring showed a regression, addressed by the wrapper in patch 03.
Patch organisation
==================
01-02 EAL: drop the inline-asm fallback paths now that intrinsics
work on all platforms; reimplement rte_smp_*mb on top of
rte_atomic_thread_fence.
03-04 lib/ring and lib/bpf -- the last legacy callers in lib/.
05-25 Drivers and selftests, one patch per directory.
26 Suppress deprecation warnings in app/test/test_atomic.c,
which exercises the legacy API until it goes away.
27 Mark rte_atomicNN_* with __rte_deprecated and drop the
corresponding checkpatch grep; new uses are now caught
at compile time.
Changes since v2
================
Scope: v2 stopped at crypto/ccp (11 patches). v3 adds:
04 lib/bpf -- the bpf interpreter's atomic op macro
13-25 Remaining driver/bus/event/vdpa conversions
26-27 Test-suite warning suppression and the actual
__rte_deprecated marking
Substantive changes to patches that were in v2:
02 Also drop the rte_smp_*mb forbidden-token check from
devtools/checkpatches.sh, since the API is no longer
on a deprecation cycle.
03 lib/ring -- keep most of the original code, introduce wrapper
for the one performance sensitive CAS. This fixes the
20-30% drop in ring_perf test on x86 which was observed
when using atomic_compare_exchange_weak_explicit() with GCC.
Feedback wanted
===============
Series is targeting 26.11 rather than the next release. The driver
conversions touch many maintainers' code and several are likely to
need cycles of review/respin; a longer review window avoids rushing
contested orderings into an earlier release.
- vmbus producer commit-order pattern (patch 17)
- the ring CAS GCC bug workaround might be needed on other similar
uses of ring buffers in vmbus and netvsc.
- Dekker-style seq_cst handshake in net/vhost (patch 24), which
also closes a pre-existing ordering hole on weakly-ordered ISAs
- netvsc rndis_pending claim/timeout/clear cmpxchg orderings
(patch 15)
Stephen Hemminger (27):
eal: use intrinsics for rte_atomic on all platforms
eal: reimplement rte_smp_*mb with rte_atomic_thread_fence
ring: use compare-and-swap wrapper
bpf: replace atomic op macro with typed helpers
net/bonding: use stdatomic
net/nbl: remove unused rte_atomic16 field
net/ena: replace use of rte_atomicNN
net/failsafe: convert to stdatomic
net/enic: do not use deprecated rte_atomic64
net/pfe: use ethdev linkstatus helpers
net/sfc: replace rte_atomic with stdatomic
crypto/ccp: replace use of rte_atomic64 with stdatomic
bus/dpaa: replace rte_atomic16 with stdatomic
drivers: replace rte_atomic16 with stdatomic
net/netvsc: replace rte_atomic32 with stdatomic
event/sw: convert from rte_atomic32 to stdatomic
bus/vmbus: convert from rte_atomic to stdatomic
common/dpaax: remove unused atomic macros
net/bnx2x: convert from rte_atomic32 to stdatomic
bus/fslmc: replace rte_atomic32 with stdatomic
drivers/event: replace rte_atomic32 in selftests
net/hinic: replace rte_atomic32 with stdatomic
net/txgbe: replace rte_atomic32 with stdatomic
net/vhost: use stdatomic instead of rte_atomic32
vdpa/ifc: replace rte_atomic32 with stdatomic
test/atomic: suppress deprecation warnings for legacy APIs
eal: mark rte_atomicNN as deprecated
app/test/test_atomic.c | 12 +
devtools/checkpatches.sh | 16 -
doc/guides/rel_notes/deprecation.rst | 12 +-
doc/guides/rel_notes/release_26_07.rst | 4 +
drivers/bus/dpaa/base/qbman/qman.c | 9 +-
drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c | 10 +-
drivers/bus/fslmc/portal/dpaa2_hw_dpci.c | 10 +-
drivers/bus/fslmc/portal/dpaa2_hw_dpio.c | 12 +-
drivers/bus/fslmc/portal/dpaa2_hw_pvt.h | 8 +-
drivers/bus/fslmc/qbman/include/compat.h | 21 +-
drivers/bus/vmbus/private.h | 2 +-
drivers/bus/vmbus/vmbus_bufring.c | 39 ++-
drivers/common/dpaax/compat.h | 14 -
drivers/crypto/ccp/ccp_crypto.c | 11 +-
drivers/crypto/ccp/ccp_crypto.h | 2 +-
drivers/crypto/ccp/ccp_dev.c | 10 +-
drivers/crypto/ccp/ccp_dev.h | 4 +-
drivers/event/dpaa2/dpaa2_eventdev_selftest.c | 26 +-
drivers/event/dpaa2/dpaa2_hw_dpcon.c | 11 +-
drivers/event/octeontx/ssovf_evdev_selftest.c | 58 ++--
drivers/event/sw/sw_evdev.c | 8 +-
drivers/event/sw/sw_evdev.h | 4 +-
drivers/event/sw/sw_evdev_worker.c | 16 +-
drivers/net/bnx2x/bnx2x.c | 6 +-
drivers/net/bnx2x/bnx2x.h | 2 +-
drivers/net/bnx2x/ecore_sp.c | 6 +-
drivers/net/bonding/eth_bond_8023ad_private.h | 6 +-
drivers/net/bonding/rte_eth_bond_8023ad.c | 35 +-
drivers/net/ena/base/ena_plat_dpdk.h | 14 +-
drivers/net/ena/ena_ethdev.c | 21 +-
drivers/net/ena/ena_ethdev.h | 7 +-
drivers/net/enic/enic.h | 6 +-
drivers/net/enic/enic_compat.h | 1 -
drivers/net/enic/enic_main.c | 17 +-
drivers/net/enic/enic_rxtx.c | 14 +-
drivers/net/enic/enic_rxtx_vec_avx2.c | 4 +-
drivers/net/failsafe/failsafe_ops.c | 12 +-
drivers/net/failsafe/failsafe_private.h | 29 +-
drivers/net/failsafe/failsafe_rxtx.c | 2 +-
drivers/net/hinic/base/hinic_compat.h | 2 +-
drivers/net/hinic/base/hinic_pmd_hwdev.c | 24 +-
drivers/net/hinic/base/hinic_pmd_hwdev.h | 4 +-
drivers/net/nbl/nbl_hw/nbl_resource.h | 1 -
drivers/net/netvsc/hn_rndis.c | 28 +-
drivers/net/netvsc/hn_rxtx.c | 12 +-
drivers/net/netvsc/hn_var.h | 6 +-
drivers/net/pfe/pfe_ethdev.c | 32 +-
drivers/net/sfc/sfc.c | 9 +-
drivers/net/sfc/sfc.h | 4 +-
drivers/net/sfc/sfc_port.c | 7 +-
drivers/net/sfc/sfc_stats.h | 2 +-
drivers/net/txgbe/base/txgbe_mng.c | 4 +-
drivers/net/txgbe/base/txgbe_type.h | 2 +-
drivers/net/vhost/rte_eth_vhost.c | 103 +++---
drivers/vdpa/ifc/ifcvf_vdpa.c | 37 +--
lib/bpf/bpf_exec.c | 91 ++++--
lib/eal/arm/include/rte_atomic_32.h | 10 -
lib/eal/arm/include/rte_atomic_64.h | 10 -
lib/eal/include/generic/rte_atomic.h | 305 +++++-------------
lib/eal/loongarch/include/rte_atomic.h | 10 -
lib/eal/ppc/include/rte_atomic.h | 179 ----------
lib/eal/riscv/include/rte_atomic.h | 10 -
lib/eal/x86/include/rte_atomic.h | 205 +-----------
lib/eal/x86/include/rte_atomic_32.h | 188 -----------
lib/eal/x86/include/rte_atomic_64.h | 157 ---------
lib/ring/rte_ring_generic_pvt.h | 32 +-
66 files changed, 585 insertions(+), 1390 deletions(-)
--
2.53.0
^ permalink raw reply
* [PATCH v3 09/27] net/enic: do not use deprecated rte_atomic64
From: Stephen Hemminger @ 2026-05-23 19:16 UTC (permalink / raw)
To: dev; +Cc: Stephen Hemminger
In-Reply-To: <20260523195604.441947-1-stephen@networkplumber.org>
The rte_atomic64 datatype and functions are deprecated.
This driver was only using it for error statistics where atomic
is not necessary. The DPDK PMD model is that statistics do
not have to be exact in face of contention.
Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
drivers/net/enic/enic.h | 6 +++---
drivers/net/enic/enic_compat.h | 1 -
drivers/net/enic/enic_main.c | 17 +++++++----------
drivers/net/enic/enic_rxtx.c | 14 ++++++--------
drivers/net/enic/enic_rxtx_vec_avx2.c | 4 ++--
5 files changed, 18 insertions(+), 24 deletions(-)
diff --git a/drivers/net/enic/enic.h b/drivers/net/enic/enic.h
index 87f6b35fcd..0a8d4a29ca 100644
--- a/drivers/net/enic/enic.h
+++ b/drivers/net/enic/enic.h
@@ -59,9 +59,9 @@
#define ENICPMD_RXQ_INTR_OFFSET 1
struct enic_soft_stats {
- rte_atomic64_t rx_nombuf;
- rte_atomic64_t rx_packet_errors;
- rte_atomic64_t tx_oversized;
+ uint64_t rx_nombuf;
+ uint64_t rx_packet_errors;
+ uint64_t tx_oversized;
};
struct enic_memzone_entry {
diff --git a/drivers/net/enic/enic_compat.h b/drivers/net/enic/enic_compat.h
index 7cff6831b9..3ce4299e81 100644
--- a/drivers/net/enic/enic_compat.h
+++ b/drivers/net/enic/enic_compat.h
@@ -9,7 +9,6 @@
#include <stdio.h>
#include <unistd.h>
-#include <rte_atomic.h>
#include <rte_malloc.h>
#include <rte_log.h>
#include <rte_io.h>
diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c
index 2696fa77d4..fb9a5754c9 100644
--- a/drivers/net/enic/enic_main.c
+++ b/drivers/net/enic/enic_main.c
@@ -83,17 +83,15 @@ static void enic_log_q_error(struct enic *enic)
static void enic_clear_soft_stats(struct enic *enic)
{
struct enic_soft_stats *soft_stats = &enic->soft_stats;
- rte_atomic64_clear(&soft_stats->rx_nombuf);
- rte_atomic64_clear(&soft_stats->rx_packet_errors);
- rte_atomic64_clear(&soft_stats->tx_oversized);
+
+ memset(soft_stats, 0, sizeof(*soft_stats));
}
static void enic_init_soft_stats(struct enic *enic)
{
struct enic_soft_stats *soft_stats = &enic->soft_stats;
- rte_atomic64_init(&soft_stats->rx_nombuf);
- rte_atomic64_init(&soft_stats->rx_packet_errors);
- rte_atomic64_init(&soft_stats->tx_oversized);
+
+ memset(soft_stats, 0, sizeof(*soft_stats));
enic_clear_soft_stats(enic);
}
@@ -132,7 +130,7 @@ int enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats,
* counted in ibytes even though truncated packets are dropped
* which can make ibytes be slightly higher than it should be.
*/
- rx_packet_errors = rte_atomic64_read(&soft_stats->rx_packet_errors);
+ rx_packet_errors = soft_stats->rx_packet_errors;
rx_truncated = rx_packet_errors - stats->rx.rx_errors;
r_stats->ipackets = stats->rx.rx_frames_ok - rx_truncated;
@@ -142,12 +140,11 @@ int enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats,
r_stats->obytes = stats->tx.tx_bytes_ok;
r_stats->ierrors = stats->rx.rx_errors + stats->rx.rx_drop;
- r_stats->oerrors = stats->tx.tx_errors
- + rte_atomic64_read(&soft_stats->tx_oversized);
+ r_stats->oerrors = stats->tx.tx_errors + soft_stats->tx_oversized;
r_stats->imissed = stats->rx.rx_no_bufs + rx_truncated;
- r_stats->rx_nombuf = rte_atomic64_read(&soft_stats->rx_nombuf);
+ r_stats->rx_nombuf = soft_stats->rx_nombuf;
return 0;
}
diff --git a/drivers/net/enic/enic_rxtx.c b/drivers/net/enic/enic_rxtx.c
index 549a153332..c87d947b93 100644
--- a/drivers/net/enic/enic_rxtx.c
+++ b/drivers/net/enic/enic_rxtx.c
@@ -112,7 +112,7 @@ enic_recv_pkts_common(void *rx_queue, struct rte_mbuf **rx_pkts,
/* allocate a new mbuf */
nmb = rte_mbuf_raw_alloc(rq->mp);
if (nmb == NULL) {
- rte_atomic64_inc(&enic->soft_stats.rx_nombuf);
+ ++enic->soft_stats.rx_nombuf;
break;
}
@@ -185,7 +185,7 @@ enic_recv_pkts_common(void *rx_queue, struct rte_mbuf **rx_pkts,
}
if (unlikely(packet_error)) {
rte_pktmbuf_free(first_seg);
- rte_atomic64_inc(&enic->soft_stats.rx_packet_errors);
+ ++enic->soft_stats.rx_packet_errors;
continue;
}
@@ -303,7 +303,7 @@ enic_noscatter_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
if (unlikely(cqd->bytes_written_flags &
CQ_ENET_RQ_DESC_FLAGS_TRUNCATED)) {
rte_pktmbuf_free(*rxmb++);
- rte_atomic64_inc(&enic->soft_stats.rx_packet_errors);
+ ++enic->soft_stats.rx_packet_errors;
cqd++;
continue;
}
@@ -505,14 +505,12 @@ uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
uint8_t offload_mode;
uint16_t header_len;
uint64_t tso;
- rte_atomic64_t *tx_oversized;
enic_cleanup_wq(enic, wq);
wq_desc_avail = vnic_wq_desc_avail(wq);
head_idx = wq->head_idx;
desc_count = wq->ring.desc_count;
ol_flags_mask = RTE_MBUF_F_TX_VLAN | RTE_MBUF_F_TX_IP_CKSUM | RTE_MBUF_F_TX_L4_MASK;
- tx_oversized = &enic->soft_stats.tx_oversized;
nb_pkts = RTE_MIN(nb_pkts, ENIC_TX_XMIT_MAX);
@@ -527,7 +525,7 @@ uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
/* drop packet if it's too big to send */
if (unlikely(!tso && pkt_len > ENIC_TX_MAX_PKT_SIZE)) {
rte_pktmbuf_free(tx_pkt);
- rte_atomic64_inc(tx_oversized);
+ ++enic->soft_stats.tx_oversized;
continue;
}
@@ -558,7 +556,7 @@ uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
if (unlikely(header_len == 0 || ((tx_pkt->tso_segsz +
header_len) > ENIC_TX_MAX_PKT_SIZE))) {
rte_pktmbuf_free(tx_pkt);
- rte_atomic64_inc(tx_oversized);
+ ++enic->soft_stats.tx_oversized;
continue;
}
@@ -681,7 +679,7 @@ static void enqueue_simple_pkts(struct rte_mbuf **pkts,
*/
if (unlikely(p->pkt_len > ENIC_TX_MAX_PKT_SIZE)) {
desc->length = ENIC_TX_MAX_PKT_SIZE;
- rte_atomic64_inc(&enic->soft_stats.tx_oversized);
+ ++enic->soft_stats.tx_oversized;
}
desc++;
}
diff --git a/drivers/net/enic/enic_rxtx_vec_avx2.c b/drivers/net/enic/enic_rxtx_vec_avx2.c
index 600efff270..53589ab788 100644
--- a/drivers/net/enic/enic_rxtx_vec_avx2.c
+++ b/drivers/net/enic/enic_rxtx_vec_avx2.c
@@ -81,7 +81,7 @@ enic_noscatter_vec_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
if (unlikely(cqd->bytes_written_flags &
CQ_ENET_RQ_DESC_FLAGS_TRUNCATED)) {
rte_pktmbuf_free(*rxmb++);
- rte_atomic64_inc(&enic->soft_stats.rx_packet_errors);
+ ++enic->soft_stats.rx_packet_errors;
} else {
*rx++ = rx_one(cqd, *rxmb++, enic);
}
@@ -761,7 +761,7 @@ enic_noscatter_vec_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
if (unlikely(cqd->bytes_written_flags &
CQ_ENET_RQ_DESC_FLAGS_TRUNCATED)) {
rte_pktmbuf_free(*rxmb++);
- rte_atomic64_inc(&enic->soft_stats.rx_packet_errors);
+ ++enic->soft_stats.rx_packet_errors;
} else {
*rx++ = rx_one(cqd, *rxmb++, enic);
}
--
2.53.0
^ permalink raw reply related
* [PATCH v3 08/27] net/failsafe: convert to stdatomic
From: Stephen Hemminger @ 2026-05-23 19:16 UTC (permalink / raw)
To: dev; +Cc: Stephen Hemminger
In-Reply-To: <20260523195604.441947-1-stephen@networkplumber.org>
The functions rte_atomic64 are deprecated, convert this
code to use stdatomic for reference count. Use the memory
order implied by naming P/V.
No need for initialization since refcnt is in space
allocated with rte_zmalloc().
Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
drivers/net/failsafe/failsafe_ops.c | 12 +++++-----
drivers/net/failsafe/failsafe_private.h | 29 ++++++++++++++-----------
drivers/net/failsafe/failsafe_rxtx.c | 2 +-
3 files changed, 22 insertions(+), 21 deletions(-)
diff --git a/drivers/net/failsafe/failsafe_ops.c b/drivers/net/failsafe/failsafe_ops.c
index ddc8808ebe..fcb0051777 100644
--- a/drivers/net/failsafe/failsafe_ops.c
+++ b/drivers/net/failsafe/failsafe_ops.c
@@ -11,7 +11,7 @@
#endif
#include <rte_debug.h>
-#include <rte_atomic.h>
+#include <rte_stdatomic.h>
#include <ethdev_driver.h>
#include <rte_malloc.h>
#include <rte_flow.h>
@@ -440,14 +440,13 @@ fs_rx_queue_setup(struct rte_eth_dev *dev,
}
rxq = rte_zmalloc(NULL,
sizeof(*rxq) +
- sizeof(rte_atomic64_t) * PRIV(dev)->subs_tail,
+ sizeof(uint64_t) * PRIV(dev)->subs_tail,
RTE_CACHE_LINE_SIZE);
if (rxq == NULL) {
fs_unlock(dev, 0);
return -ENOMEM;
}
- FOREACH_SUBDEV(sdev, i, dev)
- rte_atomic64_init(&rxq->refcnt[i]);
+
rxq->qid = rx_queue_id;
rxq->socket_id = socket_id;
rxq->info.mp = mb_pool;
@@ -617,14 +616,13 @@ fs_tx_queue_setup(struct rte_eth_dev *dev,
}
txq = rte_zmalloc("ethdev TX queue",
sizeof(*txq) +
- sizeof(rte_atomic64_t) * PRIV(dev)->subs_tail,
+ sizeof(uint64_t) * PRIV(dev)->subs_tail,
RTE_CACHE_LINE_SIZE);
if (txq == NULL) {
fs_unlock(dev, 0);
return -ENOMEM;
}
- FOREACH_SUBDEV(sdev, i, dev)
- rte_atomic64_init(&txq->refcnt[i]);
+
txq->qid = tx_queue_id;
txq->socket_id = socket_id;
txq->info.conf = *tx_conf;
diff --git a/drivers/net/failsafe/failsafe_private.h b/drivers/net/failsafe/failsafe_private.h
index babea6016e..89b06f9756 100644
--- a/drivers/net/failsafe/failsafe_private.h
+++ b/drivers/net/failsafe/failsafe_private.h
@@ -10,7 +10,7 @@
#include <sys/queue.h>
#include <pthread.h>
-#include <rte_atomic.h>
+#include <rte_stdatomic.h>
#include <dev_driver.h>
#include <ethdev_driver.h>
#include <rte_devargs.h>
@@ -75,7 +75,7 @@ struct rxq {
int event_fd;
unsigned int enable_events:1;
struct rte_eth_rxq_info info;
- rte_atomic64_t refcnt[];
+ RTE_ATOMIC(uint64_t) refcnt[];
};
struct txq {
@@ -83,7 +83,7 @@ struct txq {
uint16_t qid;
unsigned int socket_id;
struct rte_eth_txq_info info;
- rte_atomic64_t refcnt[];
+ RTE_ATOMIC(uint64_t) refcnt[];
};
struct rte_flow {
@@ -320,33 +320,36 @@ extern int failsafe_mac_from_arg;
*/
/**
- * a: (rte_atomic64_t)
+ * a: _Atomic uint64_t
*/
#define FS_ATOMIC_P(a) \
- rte_atomic64_set(&(a), 1)
+ rte_atomic_exchange_explicit(&(a), 1, rte_memory_order_acquire)
/**
- * a: (rte_atomic64_t)
+ * a: _Atomic uint64_t
*/
#define FS_ATOMIC_V(a) \
- rte_atomic64_set(&(a), 0)
+ rte_atomic_store_explicit(&(a), 0, rte_memory_order_release)
/**
* s: (struct sub_device *)
* i: uint16_t qid
*/
#define FS_ATOMIC_RX(s, i) \
- rte_atomic64_read( \
- &((struct rxq *) \
- (fs_dev(s)->data->rx_queues[i]))->refcnt[(s)->sid])
+ rte_atomic_load_explicit( \
+ &((struct rxq *) \
+ (fs_dev(s)->data->rx_queues[i]))->refcnt[(s)->sid], \
+ rte_memory_order_seq_cst)
+
/**
* s: (struct sub_device *)
* i: uint16_t qid
*/
#define FS_ATOMIC_TX(s, i) \
- rte_atomic64_read( \
- &((struct txq *) \
- (fs_dev(s)->data->tx_queues[i]))->refcnt[(s)->sid])
+ rte_atomic_load_explicit( \
+ &((struct txq *) \
+ (fs_dev(s)->data->tx_queues[i]))->refcnt[(s)->sid], \
+ rte_memory_order_seq_cst)
#ifdef RTE_EXEC_ENV_FREEBSD
#define FS_THREADID_TYPE void*
diff --git a/drivers/net/failsafe/failsafe_rxtx.c b/drivers/net/failsafe/failsafe_rxtx.c
index fe67293299..500483bda3 100644
--- a/drivers/net/failsafe/failsafe_rxtx.c
+++ b/drivers/net/failsafe/failsafe_rxtx.c
@@ -3,7 +3,7 @@
* Copyright 2017 Mellanox Technologies, Ltd
*/
-#include <rte_atomic.h>
+#include <rte_stdatomic.h>
#include <rte_debug.h>
#include <rte_mbuf.h>
#include <ethdev_driver.h>
--
2.53.0
^ permalink raw reply related
* [PATCH v3 07/27] net/ena: replace use of rte_atomicNN
From: Stephen Hemminger @ 2026-05-23 19:16 UTC (permalink / raw)
To: dev; +Cc: Stephen Hemminger
In-Reply-To: <20260523195604.441947-1-stephen@networkplumber.org>
Convert the legacy rte_atomicNN operations to stdatomic.
* Remove variable ena_alloc_cnt is defined by not used.
It is a leftover from previous memzone naming scheme.
* Convert the legacy rte_atomic32_t and rte_atomic32_{inc,dec,set,read}
macros to C11 stdatomic equivalents.
Memory ordering is kept at seq_cst,
matching the implicit ordering of the legacy API.
* Do not use rte_atomic for statistics
The DPDK PMD model is that statistics do not have to be exact
in face of contention.
Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
drivers/net/ena/base/ena_plat_dpdk.h | 14 +++++++++-----
drivers/net/ena/ena_ethdev.c | 21 ++++++---------------
drivers/net/ena/ena_ethdev.h | 7 +++----
3 files changed, 18 insertions(+), 24 deletions(-)
diff --git a/drivers/net/ena/base/ena_plat_dpdk.h b/drivers/net/ena/base/ena_plat_dpdk.h
index c84420de22..83b354d9da 100644
--- a/drivers/net/ena/base/ena_plat_dpdk.h
+++ b/drivers/net/ena/base/ena_plat_dpdk.h
@@ -40,7 +40,7 @@ typedef uint64_t dma_addr_t;
#endif
#define ENA_PRIu64 PRIu64
-#define ena_atomic32_t rte_atomic32_t
+typedef RTE_ATOMIC(int32_t) ena_atomic32_t;
#define ena_mem_handle_t const struct rte_memzone *
#define SZ_256 (256U)
@@ -267,10 +267,14 @@ ena_mem_alloc_coherent(struct rte_eth_dev_data *data, size_t size,
#define ENA_REG_READ32(bus, reg) \
__extension__ ({ (void)(bus); rte_read32_relaxed((reg)); })
-#define ATOMIC32_INC(i32_ptr) rte_atomic32_inc(i32_ptr)
-#define ATOMIC32_DEC(i32_ptr) rte_atomic32_dec(i32_ptr)
-#define ATOMIC32_SET(i32_ptr, val) rte_atomic32_set(i32_ptr, val)
-#define ATOMIC32_READ(i32_ptr) rte_atomic32_read(i32_ptr)
+#define ATOMIC32_INC(i32_ptr) \
+ rte_atomic_fetch_add_explicit((i32_ptr), 1, rte_memory_order_seq_cst)
+#define ATOMIC32_DEC(i32_ptr) \
+ rte_atomic_fetch_sub_explicit((i32_ptr), 1, rte_memory_order_seq_cst)
+#define ATOMIC32_SET(i32_ptr, val) \
+ rte_atomic_store_explicit((i32_ptr), (val), rte_memory_order_seq_cst)
+#define ATOMIC32_READ(i32_ptr) \
+ rte_atomic_load_explicit((i32_ptr), rte_memory_order_seq_cst)
#define msleep(x) rte_delay_us(x * 1000)
#define udelay(x) rte_delay_us(x)
diff --git a/drivers/net/ena/ena_ethdev.c b/drivers/net/ena/ena_ethdev.c
index ea4afbc75d..e9c484456c 100644
--- a/drivers/net/ena/ena_ethdev.c
+++ b/drivers/net/ena/ena_ethdev.c
@@ -121,12 +121,6 @@ struct ena_stats {
*/
#define ENA_DEVARG_ENABLE_FRAG_BYPASS "enable_frag_bypass"
-/*
- * Each rte_memzone should have unique name.
- * To satisfy it, count number of allocation and add it to name.
- */
-rte_atomic64_t ena_alloc_cnt;
-
static const struct ena_stats ena_stats_global_strings[] = {
ENA_STAT_GLOBAL_ENTRY(wd_expired),
ENA_STAT_GLOBAL_ENTRY(dev_start),
@@ -1249,10 +1243,7 @@ static void ena_stats_restart(struct rte_eth_dev *dev)
{
struct ena_adapter *adapter = dev->data->dev_private;
- rte_atomic64_init(&adapter->drv_stats->ierrors);
- rte_atomic64_init(&adapter->drv_stats->oerrors);
- rte_atomic64_init(&adapter->drv_stats->rx_nombuf);
- adapter->drv_stats->rx_drops = 0;
+ memset(adapter->drv_stats, 0, sizeof(struct ena_driver_stats));
}
static int ena_stats_get(struct rte_eth_dev *dev,
@@ -1289,9 +1280,9 @@ static int ena_stats_get(struct rte_eth_dev *dev,
/* Driver related stats */
stats->imissed = adapter->drv_stats->rx_drops;
- stats->ierrors = rte_atomic64_read(&adapter->drv_stats->ierrors);
- stats->oerrors = rte_atomic64_read(&adapter->drv_stats->oerrors);
- stats->rx_nombuf = rte_atomic64_read(&adapter->drv_stats->rx_nombuf);
+ stats->ierrors = adapter->drv_stats->ierrors;
+ stats->oerrors = adapter->drv_stats->oerrors;
+ stats->rx_nombuf = adapter->drv_stats->rx_nombuf;
/* Queue statistics */
if (qstats) {
@@ -1887,7 +1878,7 @@ static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count)
/* get resources for incoming packets */
rc = rte_pktmbuf_alloc_bulk(rxq->mb_pool, mbufs, count);
if (unlikely(rc < 0)) {
- rte_atomic64_inc(&rxq->adapter->drv_stats->rx_nombuf);
+ ++rxq->adapter->drv_stats->rx_nombuf;
++rxq->rx_stats.mbuf_alloc_fail;
PMD_RX_LOG_LINE(DEBUG, "There are not enough free buffers");
return 0;
@@ -3014,7 +3005,7 @@ static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
if (unlikely(mbuf->ol_flags &
(RTE_MBUF_F_RX_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD)))
- rte_atomic64_inc(&rx_ring->adapter->drv_stats->ierrors);
+ ++rx_ring->adapter->drv_stats->ierrors;
rx_pkts[completed] = mbuf;
rx_ring->rx_stats.bytes += mbuf->pkt_len;
diff --git a/drivers/net/ena/ena_ethdev.h b/drivers/net/ena/ena_ethdev.h
index 3a66d79384..b204b07767 100644
--- a/drivers/net/ena/ena_ethdev.h
+++ b/drivers/net/ena/ena_ethdev.h
@@ -6,7 +6,6 @@
#ifndef _ENA_ETHDEV_H_
#define _ENA_ETHDEV_H_
-#include <rte_atomic.h>
#include <rte_ether.h>
#include <ethdev_driver.h>
#include <ethdev_pci.h>
@@ -225,9 +224,9 @@ enum ena_adapter_state {
};
struct ena_driver_stats {
- rte_atomic64_t ierrors;
- rte_atomic64_t oerrors;
- rte_atomic64_t rx_nombuf;
+ u64 ierrors;
+ u64 oerrors;
+ u64 rx_nombuf;
u64 rx_drops;
};
--
2.53.0
^ permalink raw reply related
* [PATCH v3 06/27] net/nbl: remove unused rte_atomic16 field
From: Stephen Hemminger @ 2026-05-23 19:16 UTC (permalink / raw)
To: dev; +Cc: Stephen Hemminger
In-Reply-To: <20260523195604.441947-1-stephen@networkplumber.org>
The tx_current_queue was defined as rte_atomic16_t which
is deprecated. Remove it since it was never used.
Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
drivers/net/nbl/nbl_hw/nbl_resource.h | 1 -
1 file changed, 1 deletion(-)
diff --git a/drivers/net/nbl/nbl_hw/nbl_resource.h b/drivers/net/nbl/nbl_hw/nbl_resource.h
index bf5a9461f5..f2182ba6bc 100644
--- a/drivers/net/nbl/nbl_hw/nbl_resource.h
+++ b/drivers/net/nbl/nbl_hw/nbl_resource.h
@@ -225,7 +225,6 @@ struct nbl_res_info {
u16 base_qid;
u16 lcore_max;
u16 *pf_qid_to_lcore_id;
- rte_atomic16_t tx_current_queue;
};
struct nbl_resource_mgt {
--
2.53.0
^ permalink raw reply related
* [PATCH v3 05/27] net/bonding: use stdatomic
From: Stephen Hemminger @ 2026-05-23 19:16 UTC (permalink / raw)
To: dev; +Cc: Stephen Hemminger
In-Reply-To: <20260523195604.441947-1-stephen@networkplumber.org>
The old rte_atomic16 and rte_atomic64 functions are deprecated.
Replace with rte_stdatomic for managing warning and timer flags.
Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
drivers/net/bonding/eth_bond_8023ad_private.h | 6 ++--
drivers/net/bonding/rte_eth_bond_8023ad.c | 35 ++++++++-----------
2 files changed, 17 insertions(+), 24 deletions(-)
diff --git a/drivers/net/bonding/eth_bond_8023ad_private.h b/drivers/net/bonding/eth_bond_8023ad_private.h
index ab7d15f81a..dd3cf3ed26 100644
--- a/drivers/net/bonding/eth_bond_8023ad_private.h
+++ b/drivers/net/bonding/eth_bond_8023ad_private.h
@@ -9,7 +9,7 @@
#include <rte_ether.h>
#include <rte_byteorder.h>
-#include <rte_atomic.h>
+#include <rte_stdatomic.h>
#include <rte_flow.h>
#include "rte_eth_bond_8023ad.h"
@@ -140,10 +140,10 @@ struct port {
/** Timer which is also used as mutex. If is 0 (not running) RX marker
* packet might be responded. Otherwise shall be dropped. It is zeroed in
* mode 4 callback function after expire. */
- volatile uint64_t rx_marker_timer;
+ RTE_ATOMIC(uint64_t) rx_marker_timer;
uint64_t warning_timer;
- volatile uint16_t warnings_to_show;
+ RTE_ATOMIC(uint16_t) warnings_to_show;
/** Memory pool used to allocate slow queues */
struct rte_mempool *slow_pool;
diff --git a/drivers/net/bonding/rte_eth_bond_8023ad.c b/drivers/net/bonding/rte_eth_bond_8023ad.c
index ba88f6d261..cc7e4af2b9 100644
--- a/drivers/net/bonding/rte_eth_bond_8023ad.c
+++ b/drivers/net/bonding/rte_eth_bond_8023ad.c
@@ -171,27 +171,17 @@ timer_is_running(uint64_t *timer)
static void
set_warning_flags(struct port *port, uint16_t flags)
{
- int retval;
- uint16_t old;
- uint16_t new_flag = 0;
-
- do {
- old = port->warnings_to_show;
- new_flag = old | flags;
- retval = rte_atomic16_cmpset(&port->warnings_to_show, old, new_flag);
- } while (unlikely(retval == 0));
+ rte_atomic_fetch_or_explicit(&port->warnings_to_show, flags, rte_memory_order_relaxed);
}
static void
show_warnings(uint16_t member_id)
{
struct port *port = &bond_mode_8023ad_ports[member_id];
- uint8_t warnings;
-
- do {
- warnings = port->warnings_to_show;
- } while (rte_atomic16_cmpset(&port->warnings_to_show, warnings, 0) == 0);
+ uint16_t warnings;
+ warnings = rte_atomic_exchange_explicit(&port->warnings_to_show, 0,
+ rte_memory_order_relaxed);
if (!warnings)
return;
@@ -1337,7 +1327,6 @@ bond_mode_8023ad_handle_slow_pkt(struct bond_dev_private *internals,
struct port *port = &bond_mode_8023ad_ports[member_id];
struct marker_header *m_hdr;
uint64_t marker_timer, old_marker_timer;
- int retval;
uint8_t wrn, subtype;
/* If packet is a marker, we send response now by reusing given packet
* and update only source MAC, destination MAC is multicast so don't
@@ -1354,17 +1343,19 @@ bond_mode_8023ad_handle_slow_pkt(struct bond_dev_private *internals,
}
/* Setup marker timer. Do it in loop in case concurrent access. */
+ old_marker_timer = rte_atomic_load_explicit(&port->rx_marker_timer,
+ rte_memory_order_relaxed);
do {
- old_marker_timer = port->rx_marker_timer;
if (!timer_is_expired(&old_marker_timer)) {
wrn = WRN_RX_MARKER_TO_FAST;
goto free_out;
}
timer_set(&marker_timer, mode4->rx_marker_timeout);
- retval = rte_atomic64_cmpset(&port->rx_marker_timer,
- old_marker_timer, marker_timer);
- } while (unlikely(retval == 0));
+
+ } while (!rte_atomic_compare_exchange_weak_explicit(&port->rx_marker_timer,
+ &old_marker_timer, marker_timer,
+ rte_memory_order_seq_cst, rte_memory_order_relaxed));
m_hdr->marker.tlv_type_marker = MARKER_TLV_TYPE_RESP;
rte_eth_macaddr_get(member_id, &m_hdr->eth_hdr.src_addr);
@@ -1372,7 +1363,8 @@ bond_mode_8023ad_handle_slow_pkt(struct bond_dev_private *internals,
if (internals->mode4.dedicated_queues.enabled == 0) {
if (rte_ring_enqueue(port->tx_ring, pkt) != 0) {
/* reset timer */
- port->rx_marker_timer = 0;
+ rte_atomic_store_explicit(&port->rx_marker_timer, 0,
+ rte_memory_order_release);
wrn = WRN_TX_QUEUE_FULL;
goto free_out;
}
@@ -1386,7 +1378,8 @@ bond_mode_8023ad_handle_slow_pkt(struct bond_dev_private *internals,
&pkt, tx_count);
if (tx_count != 1) {
/* reset timer */
- port->rx_marker_timer = 0;
+ rte_atomic_store_explicit(&port->rx_marker_timer, 0,
+ rte_memory_order_release);
wrn = WRN_TX_QUEUE_FULL;
goto free_out;
}
--
2.53.0
^ permalink raw reply related
* [PATCH v3 04/27] bpf: replace atomic op macro with typed helpers
From: Stephen Hemminger @ 2026-05-23 19:16 UTC (permalink / raw)
To: dev; +Cc: Stephen Hemminger
In-Reply-To: <20260523195604.441947-1-stephen@networkplumber.org>
The BPF_ST_ATOMIC_REG macro token-pasted the legacy rte_atomicNN_*()
API names. It also stacked three casts on the destination pointer
and reached a 'return 0' out of the macro into the caller's control
flow.
Replace it with two small static-inline helpers, bpf_atomic32() and
bpf_atomic64(), that dispatch on ins->imm internally and use the C11
atomic intrinsics directly. The destination is cast once, to a
properly __rte_atomic-qualified pointer. The helpers return a status
and the dispatch loop owns the early exit.
Use memory order seq_cst to preserve the previous behavior of
rte_atomicNN_add() / rte_atomicNN_exchange() and matches
the Linux kernel BPF interpreter for these opcodes.
Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
lib/bpf/bpf_exec.c | 91 ++++++++++++++++++++++++++++++++++------------
1 file changed, 67 insertions(+), 24 deletions(-)
diff --git a/lib/bpf/bpf_exec.c b/lib/bpf/bpf_exec.c
index 18013753b1..b8116db191 100644
--- a/lib/bpf/bpf_exec.c
+++ b/lib/bpf/bpf_exec.c
@@ -64,28 +64,6 @@
(*(type *)(uintptr_t)((reg)[(ins)->dst_reg] + (ins)->off) = \
(type)(reg)[(ins)->src_reg])
-#define BPF_ST_ATOMIC_REG(reg, ins, tp) do { \
- switch (ins->imm) { \
- case BPF_ATOMIC_ADD: \
- rte_atomic##tp##_add((rte_atomic##tp##_t *) \
- (uintptr_t)((reg)[(ins)->dst_reg] + (ins)->off), \
- (reg)[(ins)->src_reg]); \
- break; \
- case BPF_ATOMIC_XCHG: \
- (reg)[(ins)->src_reg] = rte_atomic##tp##_exchange((uint##tp##_t *) \
- (uintptr_t)((reg)[(ins)->dst_reg] + (ins)->off), \
- (reg)[(ins)->src_reg]); \
- break; \
- default: \
- /* this should be caught by validator and never reach here */ \
- RTE_BPF_LOG_LINE(ERR, \
- "%s(%p): unsupported atomic operation at pc: %#zx;", \
- __func__, bpf, \
- (uintptr_t)(ins) - (uintptr_t)(bpf)->prm.ins); \
- return 0; \
- } \
-} while (0)
-
/* BPF_LD | BPF_ABS/BPF_IND */
#define NOP(x) (x)
@@ -105,6 +83,69 @@
reg[EBPF_REG_0] = op(p[0]); \
} while (0)
+/*
+ * Atomic ops on the BPF target memory.
+ *
+ * BPF atomic instructions encode the destination as base register +
+ * signed offset, with the value to combine taken from src_reg.
+ *
+ * Memory order: seq_cst preserves the previous behavior of
+ * rte_atomicNN_add() / rte_atomicNN_exchange() and matches what the
+ * Linux kernel BPF interpreter does for these opcodes.
+ *
+ * Returns 0 on unsupported sub-op (validator should have rejected it),
+ * 1 otherwise.
+ */
+static inline int
+bpf_atomic32(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM],
+ const struct ebpf_insn *ins)
+{
+ /* need to casts to make bpf memory suitable for C11 atomic */
+ uint32_t __rte_atomic *dst
+ = (uint32_t __rte_atomic *)(uintptr_t)(reg[ins->dst_reg] + ins->off);
+ uint32_t val = (uint32_t)reg[ins->src_reg];
+
+ switch (ins->imm) {
+ case BPF_ATOMIC_ADD:
+ rte_atomic_fetch_add_explicit(dst, val, rte_memory_order_seq_cst);
+ return 1;
+ case BPF_ATOMIC_XCHG:
+ reg[ins->src_reg] = rte_atomic_exchange_explicit(dst, val,
+ rte_memory_order_seq_cst);
+ return 1;
+ default:
+ RTE_BPF_LOG_LINE(ERR,
+ "%s(%p): unsupported atomic operation at pc: %#zx;",
+ __func__, bpf,
+ (uintptr_t)ins - (uintptr_t)bpf->prm.ins);
+ return 0;
+ }
+}
+
+static inline int
+bpf_atomic64(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM],
+ const struct ebpf_insn *ins)
+{
+ uint64_t __rte_atomic *dst
+ = (uint64_t __rte_atomic *)(uintptr_t) (reg[ins->dst_reg] + ins->off);
+ uint64_t val = reg[ins->src_reg];
+
+ switch (ins->imm) {
+ case BPF_ATOMIC_ADD:
+ rte_atomic_fetch_add_explicit(dst, val, rte_memory_order_seq_cst);
+ return 1;
+ case BPF_ATOMIC_XCHG:
+ reg[ins->src_reg] = rte_atomic_exchange_explicit(dst, val,
+ rte_memory_order_seq_cst);
+ return 1;
+ default:
+ RTE_BPF_LOG_LINE(ERR,
+ "%s(%p): unsupported atomic operation at pc: %#zx;",
+ __func__, bpf,
+ (uintptr_t)ins - (uintptr_t)bpf->prm.ins);
+ return 0;
+ }
+}
static inline void
bpf_alu_be(uint64_t reg[EBPF_REG_NUM], const struct ebpf_insn *ins)
@@ -392,10 +433,12 @@ bpf_exec(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM])
break;
/* atomic instructions */
case (BPF_STX | EBPF_ATOMIC | BPF_W):
- BPF_ST_ATOMIC_REG(reg, ins, 32);
+ if (bpf_atomic32(bpf, reg, ins) == 0)
+ return 0;
break;
case (BPF_STX | EBPF_ATOMIC | EBPF_DW):
- BPF_ST_ATOMIC_REG(reg, ins, 64);
+ if (bpf_atomic64(bpf, reg, ins) == 0)
+ return 0;
break;
/* jump instructions */
case (BPF_JMP | BPF_JA):
--
2.53.0
^ permalink raw reply related
* [PATCH v3 03/27] ring: use compare-and-swap wrapper
From: Stephen Hemminger @ 2026-05-23 19:16 UTC (permalink / raw)
To: dev; +Cc: Stephen Hemminger
In-Reply-To: <20260523195604.441947-1-stephen@networkplumber.org>
The rte_atomic32_cmpset is deprecated. Initial attempts at
changing this with direct conversion to
rte_atomic_compare_exchange_weak_explicit()
regressed MP/MC contended performance on x86 by 10-30%,
because the C11 builtin's failure-writeback semantic forces
GCC to emit extra instructions on the CAS critical path.
Add an internal __rte_ring_compare_and_swap() wrapper that calls
__sync_bool_compare_and_swap() directly, which keeps the original
instruction sequence. Add equivalent function for MSVC.
Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
lib/ring/rte_ring_generic_pvt.h | 32 ++++++++++++++++++++++++++++----
1 file changed, 28 insertions(+), 4 deletions(-)
diff --git a/lib/ring/rte_ring_generic_pvt.h b/lib/ring/rte_ring_generic_pvt.h
index affd2d5ba7..0fb972de9e 100644
--- a/lib/ring/rte_ring_generic_pvt.h
+++ b/lib/ring/rte_ring_generic_pvt.h
@@ -18,6 +18,30 @@
* For more information please refer to <rte_ring.h>.
*/
+/**
+ * @internal optimized version of compare exchange
+ *
+ * The C11 builtin's failure-writeback semantic generates worse code on x86.
+ * Unlike rte_atomic_compare_exchange_*_explicit(), this wrapper does NOT
+ * write the actual value back to a pointer on failure. Callers in a retry
+ * loop must reload the expected value explicitly on the next iteration.
+ *
+ * Full memory barrier, equivalent to rte_memory_order_seq_cst on both
+ * success and failure.
+ */
+static __rte_always_inline bool
+__rte_ring_compare_and_swap(volatile uint32_t *dst,
+ uint32_t expected, uint32_t desired)
+{
+#if defined(RTE_TOOLCHAIN_MSVC)
+ return _InterlockedCompareExchange((volatile long *)dst,
+ (long)desired, (long)expected)
+ == (long)expected;
+#else
+ return __sync_bool_compare_and_swap(dst, expected, desired);
+#endif
+}
+
/**
* @internal This function updates tail values.
*/
@@ -108,10 +132,10 @@ __rte_ring_headtail_move_head(struct rte_ring_headtail *d,
if (is_st) {
d->head = *new_head;
success = 1;
- } else
- success = rte_atomic32_cmpset(
- (uint32_t *)(uintptr_t)&d->head,
- *old_head, *new_head);
+ } else {
+ success = __rte_ring_compare_and_swap(
+ &d->head, *old_head, *new_head);
+ }
} while (unlikely(success == 0));
return n;
}
--
2.53.0
^ permalink raw reply related
* [PATCH v3 02/27] eal: reimplement rte_smp_*mb with rte_atomic_thread_fence
From: Stephen Hemminger @ 2026-05-23 19:16 UTC (permalink / raw)
To: dev; +Cc: Stephen Hemminger
In-Reply-To: <20260523195604.441947-1-stephen@networkplumber.org>
The rte_smp_mb(), rte_smp_wmb() and rte_smp_rmb() functions were
flagged as deprecated by commit 3ec965b6de12 ("doc: update atomic
operation deprecation") in 2021 but nothing came of it.
Reimplement them as inline wrappers over rte_atomic_thread_fence()
and drop the deprecation notice.
The API is preserved; only the implementation changes.
Generated code is unchanged on x86 (seq_cst keeps the lock-addl
trick, release/acquire collapse to a compiler barrier under TSO).
On arm64, release/acquire emit dmb ish instead of dmb ishst/ishld;
the difference is below measurement noise.
Drop restriction frm checkpatch since they are no longer
really on deprecation cycle.
Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
devtools/checkpatches.sh | 8 --
doc/guides/rel_notes/deprecation.rst | 8 --
lib/eal/arm/include/rte_atomic_32.h | 6 --
lib/eal/arm/include/rte_atomic_64.h | 6 --
lib/eal/include/generic/rte_atomic.h | 130 +++++--------------------
lib/eal/loongarch/include/rte_atomic.h | 6 --
lib/eal/ppc/include/rte_atomic.h | 6 --
lib/eal/riscv/include/rte_atomic.h | 6 --
lib/eal/x86/include/rte_atomic.h | 33 +++----
9 files changed, 37 insertions(+), 172 deletions(-)
diff --git a/devtools/checkpatches.sh b/devtools/checkpatches.sh
index f5dd77443f..81bb0fe4e8 100755
--- a/devtools/checkpatches.sh
+++ b/devtools/checkpatches.sh
@@ -121,14 +121,6 @@ check_forbidden_additions() { # <patch>
-f $(dirname $(readlink -f $0))/check-forbidden-tokens.awk \
"$1" || res=1
- # refrain from new additions of rte_smp_[r/w]mb()
- awk -v FOLDERS="lib drivers app examples" \
- -v EXPRESSIONS="rte_smp_(r|w)?mb\\\(" \
- -v RET_ON_FAIL=1 \
- -v MESSAGE='Using rte_smp_[r/w]mb' \
- -f $(dirname $(readlink -f $0))/check-forbidden-tokens.awk \
- "$1" || res=1
-
# refrain from using compiler __sync_xxx builtins
awk -v FOLDERS="lib drivers app examples" \
-v EXPRESSIONS="__sync_.*\\\(" \
diff --git a/doc/guides/rel_notes/deprecation.rst b/doc/guides/rel_notes/deprecation.rst
index 35c9b4e06c..2190419f79 100644
--- a/doc/guides/rel_notes/deprecation.rst
+++ b/doc/guides/rel_notes/deprecation.rst
@@ -47,14 +47,6 @@ Deprecation Notices
operations must be used for patches that need to be merged in 20.08 onwards.
This change will not introduce any performance degradation.
-* rte_smp_*mb: These APIs provide full barrier functionality. However, many
- use cases do not require full barriers. To support such use cases, DPDK has
- adopted atomic operations from
- https://gcc.gnu.org/onlinedocs/gcc/_005f_005fatomic-Builtins.html. These
- operations and a new wrapper ``rte_atomic_thread_fence`` instead of
- ``__atomic_thread_fence`` must be used for patches that need to be merged in
- 20.08 onwards. This change will not introduce any performance degradation.
-
* lib: will fix extending some enum/define breaking the ABI. There are multiple
samples in DPDK that enum/define terminated with a ``.*MAX.*`` value which is
used by iterators, and arrays holding these values are sized with this
diff --git a/lib/eal/arm/include/rte_atomic_32.h b/lib/eal/arm/include/rte_atomic_32.h
index 696a539fef..4115271091 100644
--- a/lib/eal/arm/include/rte_atomic_32.h
+++ b/lib/eal/arm/include/rte_atomic_32.h
@@ -17,12 +17,6 @@ extern "C" {
#define rte_rmb() __sync_synchronize()
-#define rte_smp_mb() rte_mb()
-
-#define rte_smp_wmb() rte_wmb()
-
-#define rte_smp_rmb() rte_rmb()
-
#define rte_io_mb() rte_mb()
#define rte_io_wmb() rte_wmb()
diff --git a/lib/eal/arm/include/rte_atomic_64.h b/lib/eal/arm/include/rte_atomic_64.h
index 9f790238df..604e777bcd 100644
--- a/lib/eal/arm/include/rte_atomic_64.h
+++ b/lib/eal/arm/include/rte_atomic_64.h
@@ -20,12 +20,6 @@ extern "C" {
#define rte_rmb() asm volatile("dmb oshld" : : : "memory")
-#define rte_smp_mb() asm volatile("dmb ish" : : : "memory")
-
-#define rte_smp_wmb() asm volatile("dmb ishst" : : : "memory")
-
-#define rte_smp_rmb() asm volatile("dmb ishld" : : : "memory")
-
#define rte_io_mb() rte_mb()
#define rte_io_wmb() rte_wmb()
diff --git a/lib/eal/include/generic/rte_atomic.h b/lib/eal/include/generic/rte_atomic.h
index 292e52fade..1b04b43cbb 100644
--- a/lib/eal/include/generic/rte_atomic.h
+++ b/lib/eal/include/generic/rte_atomic.h
@@ -59,55 +59,25 @@ static inline void rte_rmb(void);
*
* Guarantees that the LOAD and STORE operations that precede the
* rte_smp_mb() call are globally visible across the lcores
- * before the LOAD and STORE operations that follows it.
- *
- * @note
- * This function is deprecated.
- * It provides similar synchronization primitive as atomic fence,
- * but has different syntax and memory ordering semantic. Hence
- * deprecated for the simplicity of memory ordering semantics in use.
- *
- * rte_atomic_thread_fence(rte_memory_order_acq_rel) should be used instead.
+ * before the LOAD and STORE operations that follow it.
*/
static inline void rte_smp_mb(void);
/**
* Write memory barrier between lcores
*
- * Guarantees that the STORE operations that precede the
- * rte_smp_wmb() call are globally visible across the lcores
- * before the STORE operations that follows it.
- *
- * @note
- * This function is deprecated.
- * It provides similar synchronization primitive as atomic fence,
- * but has different syntax and memory ordering semantic. Hence
- * deprecated for the simplicity of memory ordering semantics in use.
- *
- * rte_atomic_thread_fence(rte_memory_order_release) should be used instead.
- * The fence also guarantees LOAD operations that precede the call
- * are globally visible across the lcores before the STORE operations
- * that follows it.
+ * Guarantees that the LOAD and STORE operations that precede the
+ * rte_smp_wmb() call are globally visible across the lcores before
+ * any STORE operations that follow it.
*/
static inline void rte_smp_wmb(void);
/**
* Read memory barrier between lcores
*
- * Guarantees that the LOAD operations that precede the
- * rte_smp_rmb() call are globally visible across the lcores
- * before the LOAD operations that follows it.
- *
- * @note
- * This function is deprecated.
- * It provides similar synchronization primitive as atomic fence,
- * but has different syntax and memory ordering semantic. Hence
- * deprecated for the simplicity of memory ordering semantics in use.
- *
- * rte_atomic_thread_fence(rte_memory_order_acquire) should be used instead.
- * The fence also guarantees LOAD operations that precede the call
- * are globally visible across the lcores before the STORE operations
- * that follows it.
+ * Guarantees that any LOAD operations that precede the rte_smp_rmb()
+ * call complete before LOAD and STORE operations that follow it
+ * become globally visible.
*/
static inline void rte_smp_rmb(void);
///@}
@@ -164,6 +134,24 @@ static inline void rte_io_rmb(void);
*/
static inline void rte_atomic_thread_fence(rte_memory_order memorder);
+static __rte_always_inline void
+rte_smp_mb(void)
+{
+ rte_atomic_thread_fence(rte_memory_order_seq_cst);
+}
+
+static __rte_always_inline void
+rte_smp_wmb(void)
+{
+ rte_atomic_thread_fence(rte_memory_order_release);
+}
+
+static __rte_always_inline void
+rte_smp_rmb(void)
+{
+ rte_atomic_thread_fence(rte_memory_order_acquire);
+}
+
/*------------------------- 16 bit atomic operations -------------------------*/
#ifndef RTE_TOOLCHAIN_MSVC
@@ -184,9 +172,6 @@ static inline void rte_atomic_thread_fence(rte_memory_order memorder);
* @return
* Non-zero on success; 0 on failure.
*/
-static inline int
-rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src);
-
static inline int
rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src)
{
@@ -303,9 +288,6 @@ rte_atomic16_sub(rte_atomic16_t *v, int16_t dec)
* @param v
* A pointer to the atomic counter.
*/
-static inline void
-rte_atomic16_inc(rte_atomic16_t *v);
-
static inline void
rte_atomic16_inc(rte_atomic16_t *v)
{
@@ -318,9 +300,6 @@ rte_atomic16_inc(rte_atomic16_t *v)
* @param v
* A pointer to the atomic counter.
*/
-static inline void
-rte_atomic16_dec(rte_atomic16_t *v);
-
static inline void
rte_atomic16_dec(rte_atomic16_t *v)
{
@@ -379,8 +358,6 @@ rte_atomic16_sub_return(rte_atomic16_t *v, int16_t dec)
* @return
* True if the result after the increment operation is 0; false otherwise.
*/
-static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v);
-
static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v)
{
return rte_atomic_fetch_add_explicit((volatile __rte_atomic int16_t *)&v->cnt, 1,
@@ -398,8 +375,6 @@ static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v)
* @return
* True if the result after the decrement operation is 0; false otherwise.
*/
-static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v);
-
static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v)
{
return rte_atomic_fetch_sub_explicit((volatile __rte_atomic int16_t *)&v->cnt, 1,
@@ -417,8 +392,6 @@ static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v)
* @return
* 0 if failed; else 1, success.
*/
-static inline int rte_atomic16_test_and_set(rte_atomic16_t *v);
-
static inline int rte_atomic16_test_and_set(rte_atomic16_t *v)
{
return rte_atomic16_cmpset((volatile uint16_t *)&v->cnt, 0, 1);
@@ -453,9 +426,6 @@ static inline void rte_atomic16_clear(rte_atomic16_t *v)
* @return
* Non-zero on success; 0 on failure.
*/
-static inline int
-rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src);
-
static inline int
rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src)
{
@@ -572,9 +542,6 @@ rte_atomic32_sub(rte_atomic32_t *v, int32_t dec)
* @param v
* A pointer to the atomic counter.
*/
-static inline void
-rte_atomic32_inc(rte_atomic32_t *v);
-
static inline void
rte_atomic32_inc(rte_atomic32_t *v)
{
@@ -587,9 +554,6 @@ rte_atomic32_inc(rte_atomic32_t *v)
* @param v
* A pointer to the atomic counter.
*/
-static inline void
-rte_atomic32_dec(rte_atomic32_t *v);
-
static inline void
rte_atomic32_dec(rte_atomic32_t *v)
{
@@ -648,8 +612,6 @@ rte_atomic32_sub_return(rte_atomic32_t *v, int32_t dec)
* @return
* True if the result after the increment operation is 0; false otherwise.
*/
-static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v);
-
static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v)
{
return rte_atomic_fetch_add_explicit((volatile __rte_atomic int32_t *)&v->cnt, 1,
@@ -667,8 +629,6 @@ static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v)
* @return
* True if the result after the decrement operation is 0; false otherwise.
*/
-static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v);
-
static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v)
{
return rte_atomic_fetch_sub_explicit((volatile __rte_atomic int32_t *)&v->cnt, 1,
@@ -686,8 +646,6 @@ static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v)
* @return
* 0 if failed; else 1, success.
*/
-static inline int rte_atomic32_test_and_set(rte_atomic32_t *v);
-
static inline int rte_atomic32_test_and_set(rte_atomic32_t *v)
{
return rte_atomic32_cmpset((volatile uint32_t *)&v->cnt, 0, 1);
@@ -721,9 +679,6 @@ static inline void rte_atomic32_clear(rte_atomic32_t *v)
* @return
* Non-zero on success; 0 on failure.
*/
-static inline int
-rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src);
-
static inline int
rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src)
{
@@ -770,9 +725,6 @@ typedef struct {
* @param v
* A pointer to the atomic counter.
*/
-static inline void
-rte_atomic64_init(rte_atomic64_t *v);
-
static inline void
rte_atomic64_init(rte_atomic64_t *v)
{
@@ -798,9 +750,6 @@ rte_atomic64_init(rte_atomic64_t *v)
* @return
* The value of the counter.
*/
-static inline int64_t
-rte_atomic64_read(rte_atomic64_t *v);
-
static inline int64_t
rte_atomic64_read(rte_atomic64_t *v)
{
@@ -828,9 +777,6 @@ rte_atomic64_read(rte_atomic64_t *v)
* @param new_value
* The new value of the counter.
*/
-static inline void
-rte_atomic64_set(rte_atomic64_t *v, int64_t new_value);
-
static inline void
rte_atomic64_set(rte_atomic64_t *v, int64_t new_value)
{
@@ -856,9 +802,6 @@ rte_atomic64_set(rte_atomic64_t *v, int64_t new_value)
* @param inc
* The value to be added to the counter.
*/
-static inline void
-rte_atomic64_add(rte_atomic64_t *v, int64_t inc);
-
static inline void
rte_atomic64_add(rte_atomic64_t *v, int64_t inc)
{
@@ -874,9 +817,6 @@ rte_atomic64_add(rte_atomic64_t *v, int64_t inc)
* @param dec
* The value to be subtracted from the counter.
*/
-static inline void
-rte_atomic64_sub(rte_atomic64_t *v, int64_t dec);
-
static inline void
rte_atomic64_sub(rte_atomic64_t *v, int64_t dec)
{
@@ -890,9 +830,6 @@ rte_atomic64_sub(rte_atomic64_t *v, int64_t dec)
* @param v
* A pointer to the atomic counter.
*/
-static inline void
-rte_atomic64_inc(rte_atomic64_t *v);
-
static inline void
rte_atomic64_inc(rte_atomic64_t *v)
{
@@ -905,9 +842,6 @@ rte_atomic64_inc(rte_atomic64_t *v)
* @param v
* A pointer to the atomic counter.
*/
-static inline void
-rte_atomic64_dec(rte_atomic64_t *v);
-
static inline void
rte_atomic64_dec(rte_atomic64_t *v)
{
@@ -927,9 +861,6 @@ rte_atomic64_dec(rte_atomic64_t *v)
* @return
* The value of v after the addition.
*/
-static inline int64_t
-rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc);
-
static inline int64_t
rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc)
{
@@ -950,9 +881,6 @@ rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc)
* @return
* The value of v after the subtraction.
*/
-static inline int64_t
-rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec);
-
static inline int64_t
rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec)
{
@@ -971,8 +899,6 @@ rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec)
* @return
* True if the result after the addition is 0; false otherwise.
*/
-static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v);
-
static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v)
{
return rte_atomic64_add_return(v, 1) == 0;
@@ -989,8 +915,6 @@ static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v)
* @return
* True if the result after subtraction is 0; false otherwise.
*/
-static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v);
-
static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v)
{
return rte_atomic64_sub_return(v, 1) == 0;
@@ -1007,8 +931,6 @@ static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v)
* @return
* 0 if failed; else 1, success.
*/
-static inline int rte_atomic64_test_and_set(rte_atomic64_t *v);
-
static inline int rte_atomic64_test_and_set(rte_atomic64_t *v)
{
return rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, 0, 1);
@@ -1020,8 +942,6 @@ static inline int rte_atomic64_test_and_set(rte_atomic64_t *v)
* @param v
* A pointer to the atomic counter.
*/
-static inline void rte_atomic64_clear(rte_atomic64_t *v);
-
static inline void rte_atomic64_clear(rte_atomic64_t *v)
{
rte_atomic64_set(v, 0);
diff --git a/lib/eal/loongarch/include/rte_atomic.h b/lib/eal/loongarch/include/rte_atomic.h
index 785a452c9e..a789e3ab4d 100644
--- a/lib/eal/loongarch/include/rte_atomic.h
+++ b/lib/eal/loongarch/include/rte_atomic.h
@@ -18,12 +18,6 @@ extern "C" {
#define rte_rmb() rte_mb()
-#define rte_smp_mb() rte_mb()
-
-#define rte_smp_wmb() rte_mb()
-
-#define rte_smp_rmb() rte_mb()
-
#define rte_io_mb() rte_mb()
#define rte_io_wmb() rte_mb()
diff --git a/lib/eal/ppc/include/rte_atomic.h b/lib/eal/ppc/include/rte_atomic.h
index 64f4c3d670..0e64db2a35 100644
--- a/lib/eal/ppc/include/rte_atomic.h
+++ b/lib/eal/ppc/include/rte_atomic.h
@@ -24,12 +24,6 @@ extern "C" {
#define rte_rmb() asm volatile("sync" : : : "memory")
-#define rte_smp_mb() rte_mb()
-
-#define rte_smp_wmb() rte_wmb()
-
-#define rte_smp_rmb() rte_rmb()
-
#define rte_io_mb() rte_mb()
#define rte_io_wmb() rte_wmb()
diff --git a/lib/eal/riscv/include/rte_atomic.h b/lib/eal/riscv/include/rte_atomic.h
index 061b175f33..04c40e4e9b 100644
--- a/lib/eal/riscv/include/rte_atomic.h
+++ b/lib/eal/riscv/include/rte_atomic.h
@@ -23,12 +23,6 @@ extern "C" {
#define rte_rmb() asm volatile("fence r, r" : : : "memory")
-#define rte_smp_mb() rte_mb()
-
-#define rte_smp_wmb() rte_wmb()
-
-#define rte_smp_rmb() rte_rmb()
-
#define rte_io_mb() asm volatile("fence iorw, iorw" : : : "memory")
#define rte_io_wmb() asm volatile("fence orw, ow" : : : "memory")
diff --git a/lib/eal/x86/include/rte_atomic.h b/lib/eal/x86/include/rte_atomic.h
index 4f05302c9f..f4d39ce4fe 100644
--- a/lib/eal/x86/include/rte_atomic.h
+++ b/lib/eal/x86/include/rte_atomic.h
@@ -23,10 +23,6 @@
#define rte_rmb() _mm_lfence()
-#define rte_smp_wmb() rte_compiler_barrier()
-
-#define rte_smp_rmb() rte_compiler_barrier()
-
#ifdef __cplusplus
extern "C" {
#endif
@@ -63,20 +59,6 @@ extern "C" {
* So below we use that technique for rte_smp_mb() implementation.
*/
-static __rte_always_inline void
-rte_smp_mb(void)
-{
-#ifdef RTE_TOOLCHAIN_MSVC
- _mm_mfence();
-#else
-#ifdef RTE_ARCH_I686
- asm volatile("lock addl $0, -128(%%esp); " ::: "memory");
-#else
- asm volatile("lock addl $0, -128(%%rsp); " ::: "memory");
-#endif
-#endif
-}
-
#define rte_io_mb() rte_mb()
#define rte_io_wmb() rte_compiler_barrier()
@@ -93,10 +75,19 @@ rte_smp_mb(void)
static __rte_always_inline void
rte_atomic_thread_fence(rte_memory_order memorder)
{
- if (memorder == rte_memory_order_seq_cst)
- rte_smp_mb();
- else
+ if (memorder == rte_memory_order_seq_cst) {
+#ifdef RTE_TOOLCHAIN_MSVC
+ _mm_mfence();
+#else
+#ifdef RTE_ARCH_I686
+ asm volatile("lock addl $0, -128(%%esp); " ::: "memory");
+#else
+ asm volatile("lock addl $0, -128(%%rsp); " ::: "memory");
+#endif
+#endif
+ } else {
__rte_atomic_thread_fence(memorder);
+ }
}
#ifdef __cplusplus
--
2.53.0
^ permalink raw reply related
* [PATCH v3 01/27] eal: use intrinsics for rte_atomic on all platforms
From: Stephen Hemminger @ 2026-05-23 19:16 UTC (permalink / raw)
To: dev; +Cc: Stephen Hemminger
In-Reply-To: <20260523195604.441947-1-stephen@networkplumber.org>
Next step is to deprecate the rte_atomicNN_*() family. Rather than
maintaining both the inline asm and intrinsic fallbacks, drop the
asm paths and use intrinsics everywhere.
Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
lib/eal/arm/include/rte_atomic_32.h | 4 -
lib/eal/arm/include/rte_atomic_64.h | 4 -
lib/eal/include/generic/rte_atomic.h | 76 +---------
lib/eal/loongarch/include/rte_atomic.h | 4 -
lib/eal/ppc/include/rte_atomic.h | 173 -----------------------
lib/eal/riscv/include/rte_atomic.h | 4 -
lib/eal/x86/include/rte_atomic.h | 172 ----------------------
lib/eal/x86/include/rte_atomic_32.h | 188 -------------------------
lib/eal/x86/include/rte_atomic_64.h | 157 ---------------------
9 files changed, 6 insertions(+), 776 deletions(-)
diff --git a/lib/eal/arm/include/rte_atomic_32.h b/lib/eal/arm/include/rte_atomic_32.h
index 0b9a0dfa30..696a539fef 100644
--- a/lib/eal/arm/include/rte_atomic_32.h
+++ b/lib/eal/arm/include/rte_atomic_32.h
@@ -5,10 +5,6 @@
#ifndef _RTE_ATOMIC_ARM32_H_
#define _RTE_ATOMIC_ARM32_H_
-#ifndef RTE_FORCE_INTRINSICS
-# error Platform must be built with RTE_FORCE_INTRINSICS
-#endif
-
#include "generic/rte_atomic.h"
#ifdef __cplusplus
diff --git a/lib/eal/arm/include/rte_atomic_64.h b/lib/eal/arm/include/rte_atomic_64.h
index 181bb60929..9f790238df 100644
--- a/lib/eal/arm/include/rte_atomic_64.h
+++ b/lib/eal/arm/include/rte_atomic_64.h
@@ -6,10 +6,6 @@
#ifndef _RTE_ATOMIC_ARM64_H_
#define _RTE_ATOMIC_ARM64_H_
-#ifndef RTE_FORCE_INTRINSICS
-# error Platform must be built with RTE_FORCE_INTRINSICS
-#endif
-
#include "generic/rte_atomic.h"
#include <rte_branch_prediction.h>
#include <rte_debug.h>
diff --git a/lib/eal/include/generic/rte_atomic.h b/lib/eal/include/generic/rte_atomic.h
index 0a4f3f8528..292e52fade 100644
--- a/lib/eal/include/generic/rte_atomic.h
+++ b/lib/eal/include/generic/rte_atomic.h
@@ -187,13 +187,11 @@ static inline void rte_atomic_thread_fence(rte_memory_order memorder);
static inline int
rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src);
-#ifdef RTE_FORCE_INTRINSICS
static inline int
rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src)
{
return __sync_bool_compare_and_swap(dst, exp, src);
}
-#endif
/**
* Atomic exchange.
@@ -211,15 +209,11 @@ rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src)
* The original value at that location
*/
static inline uint16_t
-rte_atomic16_exchange(volatile uint16_t *dst, uint16_t val);
-
-#ifdef RTE_FORCE_INTRINSICS
-static inline uint16_t
rte_atomic16_exchange(volatile uint16_t *dst, uint16_t val)
{
- return rte_atomic_exchange_explicit(dst, val, rte_memory_order_seq_cst);
+ return rte_atomic_exchange_explicit((volatile __rte_atomic uint16_t *)dst,
+ val, rte_memory_order_seq_cst);
}
-#endif
/**
* The atomic counter structure.
@@ -312,13 +306,11 @@ rte_atomic16_sub(rte_atomic16_t *v, int16_t dec)
static inline void
rte_atomic16_inc(rte_atomic16_t *v);
-#ifdef RTE_FORCE_INTRINSICS
static inline void
rte_atomic16_inc(rte_atomic16_t *v)
{
rte_atomic16_add(v, 1);
}
-#endif
/**
* Atomically decrement a counter by one.
@@ -329,13 +321,11 @@ rte_atomic16_inc(rte_atomic16_t *v)
static inline void
rte_atomic16_dec(rte_atomic16_t *v);
-#ifdef RTE_FORCE_INTRINSICS
static inline void
rte_atomic16_dec(rte_atomic16_t *v)
{
rte_atomic16_sub(v, 1);
}
-#endif
/**
* Atomically add a 16-bit value to a counter and return the result.
@@ -391,13 +381,11 @@ rte_atomic16_sub_return(rte_atomic16_t *v, int16_t dec)
*/
static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v);
-#ifdef RTE_FORCE_INTRINSICS
static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v)
{
return rte_atomic_fetch_add_explicit((volatile __rte_atomic int16_t *)&v->cnt, 1,
rte_memory_order_seq_cst) + 1 == 0;
}
-#endif
/**
* Atomically decrement a 16-bit counter by one and test.
@@ -412,13 +400,11 @@ static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v)
*/
static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v);
-#ifdef RTE_FORCE_INTRINSICS
static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v)
{
return rte_atomic_fetch_sub_explicit((volatile __rte_atomic int16_t *)&v->cnt, 1,
rte_memory_order_seq_cst) - 1 == 0;
}
-#endif
/**
* Atomically test and set a 16-bit atomic counter.
@@ -433,12 +419,10 @@ static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v)
*/
static inline int rte_atomic16_test_and_set(rte_atomic16_t *v);
-#ifdef RTE_FORCE_INTRINSICS
static inline int rte_atomic16_test_and_set(rte_atomic16_t *v)
{
return rte_atomic16_cmpset((volatile uint16_t *)&v->cnt, 0, 1);
}
-#endif
/**
* Atomically set a 16-bit counter to 0.
@@ -472,13 +456,11 @@ static inline void rte_atomic16_clear(rte_atomic16_t *v)
static inline int
rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src);
-#ifdef RTE_FORCE_INTRINSICS
static inline int
rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src)
{
return __sync_bool_compare_and_swap(dst, exp, src);
}
-#endif
/**
* Atomic exchange.
@@ -496,15 +478,11 @@ rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src)
* The original value at that location
*/
static inline uint32_t
-rte_atomic32_exchange(volatile uint32_t *dst, uint32_t val);
-
-#ifdef RTE_FORCE_INTRINSICS
-static inline uint32_t
rte_atomic32_exchange(volatile uint32_t *dst, uint32_t val)
{
- return rte_atomic_exchange_explicit(dst, val, rte_memory_order_seq_cst);
+ return rte_atomic_exchange_explicit((volatile __rte_atomic uint32_t *)dst,
+ val, rte_memory_order_seq_cst);
}
-#endif
/**
* The atomic counter structure.
@@ -597,13 +575,11 @@ rte_atomic32_sub(rte_atomic32_t *v, int32_t dec)
static inline void
rte_atomic32_inc(rte_atomic32_t *v);
-#ifdef RTE_FORCE_INTRINSICS
static inline void
rte_atomic32_inc(rte_atomic32_t *v)
{
rte_atomic32_add(v, 1);
}
-#endif
/**
* Atomically decrement a counter by one.
@@ -614,13 +590,11 @@ rte_atomic32_inc(rte_atomic32_t *v)
static inline void
rte_atomic32_dec(rte_atomic32_t *v);
-#ifdef RTE_FORCE_INTRINSICS
static inline void
rte_atomic32_dec(rte_atomic32_t *v)
{
rte_atomic32_sub(v,1);
}
-#endif
/**
* Atomically add a 32-bit value to a counter and return the result.
@@ -676,13 +650,11 @@ rte_atomic32_sub_return(rte_atomic32_t *v, int32_t dec)
*/
static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v);
-#ifdef RTE_FORCE_INTRINSICS
static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v)
{
return rte_atomic_fetch_add_explicit((volatile __rte_atomic int32_t *)&v->cnt, 1,
rte_memory_order_seq_cst) + 1 == 0;
}
-#endif
/**
* Atomically decrement a 32-bit counter by one and test.
@@ -697,13 +669,11 @@ static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v)
*/
static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v);
-#ifdef RTE_FORCE_INTRINSICS
static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v)
{
return rte_atomic_fetch_sub_explicit((volatile __rte_atomic int32_t *)&v->cnt, 1,
rte_memory_order_seq_cst) - 1 == 0;
}
-#endif
/**
* Atomically test and set a 32-bit atomic counter.
@@ -718,12 +688,10 @@ static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v)
*/
static inline int rte_atomic32_test_and_set(rte_atomic32_t *v);
-#ifdef RTE_FORCE_INTRINSICS
static inline int rte_atomic32_test_and_set(rte_atomic32_t *v)
{
return rte_atomic32_cmpset((volatile uint32_t *)&v->cnt, 0, 1);
}
-#endif
/**
* Atomically set a 32-bit counter to 0.
@@ -756,13 +724,11 @@ static inline void rte_atomic32_clear(rte_atomic32_t *v)
static inline int
rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src);
-#ifdef RTE_FORCE_INTRINSICS
static inline int
rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src)
{
return __sync_bool_compare_and_swap(dst, exp, src);
}
-#endif
/**
* Atomic exchange.
@@ -780,15 +746,11 @@ rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src)
* The original value at that location
*/
static inline uint64_t
-rte_atomic64_exchange(volatile uint64_t *dst, uint64_t val);
-
-#ifdef RTE_FORCE_INTRINSICS
-static inline uint64_t
rte_atomic64_exchange(volatile uint64_t *dst, uint64_t val)
{
- return rte_atomic_exchange_explicit(dst, val, rte_memory_order_seq_cst);
+ return rte_atomic_exchange_explicit((volatile __rte_atomic uint64_t *)dst,
+ val, rte_memory_order_seq_cst);
}
-#endif
/**
* The atomic counter structure.
@@ -811,7 +773,6 @@ typedef struct {
static inline void
rte_atomic64_init(rte_atomic64_t *v);
-#ifdef RTE_FORCE_INTRINSICS
static inline void
rte_atomic64_init(rte_atomic64_t *v)
{
@@ -828,7 +789,6 @@ rte_atomic64_init(rte_atomic64_t *v)
}
#endif
}
-#endif
/**
* Atomically read a 64-bit counter.
@@ -841,7 +801,6 @@ rte_atomic64_init(rte_atomic64_t *v)
static inline int64_t
rte_atomic64_read(rte_atomic64_t *v);
-#ifdef RTE_FORCE_INTRINSICS
static inline int64_t
rte_atomic64_read(rte_atomic64_t *v)
{
@@ -860,7 +819,6 @@ rte_atomic64_read(rte_atomic64_t *v)
return tmp;
#endif
}
-#endif
/**
* Atomically set a 64-bit counter.
@@ -873,7 +831,6 @@ rte_atomic64_read(rte_atomic64_t *v)
static inline void
rte_atomic64_set(rte_atomic64_t *v, int64_t new_value);
-#ifdef RTE_FORCE_INTRINSICS
static inline void
rte_atomic64_set(rte_atomic64_t *v, int64_t new_value)
{
@@ -890,7 +847,6 @@ rte_atomic64_set(rte_atomic64_t *v, int64_t new_value)
}
#endif
}
-#endif
/**
* Atomically add a 64-bit value to a counter.
@@ -903,14 +859,12 @@ rte_atomic64_set(rte_atomic64_t *v, int64_t new_value)
static inline void
rte_atomic64_add(rte_atomic64_t *v, int64_t inc);
-#ifdef RTE_FORCE_INTRINSICS
static inline void
rte_atomic64_add(rte_atomic64_t *v, int64_t inc)
{
rte_atomic_fetch_add_explicit((volatile __rte_atomic int64_t *)&v->cnt, inc,
rte_memory_order_seq_cst);
}
-#endif
/**
* Atomically subtract a 64-bit value from a counter.
@@ -923,14 +877,12 @@ rte_atomic64_add(rte_atomic64_t *v, int64_t inc)
static inline void
rte_atomic64_sub(rte_atomic64_t *v, int64_t dec);
-#ifdef RTE_FORCE_INTRINSICS
static inline void
rte_atomic64_sub(rte_atomic64_t *v, int64_t dec)
{
rte_atomic_fetch_sub_explicit((volatile __rte_atomic int64_t *)&v->cnt, dec,
rte_memory_order_seq_cst);
}
-#endif
/**
* Atomically increment a 64-bit counter by one and test.
@@ -941,13 +893,11 @@ rte_atomic64_sub(rte_atomic64_t *v, int64_t dec)
static inline void
rte_atomic64_inc(rte_atomic64_t *v);
-#ifdef RTE_FORCE_INTRINSICS
static inline void
rte_atomic64_inc(rte_atomic64_t *v)
{
rte_atomic64_add(v, 1);
}
-#endif
/**
* Atomically decrement a 64-bit counter by one and test.
@@ -958,13 +908,11 @@ rte_atomic64_inc(rte_atomic64_t *v)
static inline void
rte_atomic64_dec(rte_atomic64_t *v);
-#ifdef RTE_FORCE_INTRINSICS
static inline void
rte_atomic64_dec(rte_atomic64_t *v)
{
rte_atomic64_sub(v, 1);
}
-#endif
/**
* Add a 64-bit value to an atomic counter and return the result.
@@ -982,14 +930,12 @@ rte_atomic64_dec(rte_atomic64_t *v)
static inline int64_t
rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc);
-#ifdef RTE_FORCE_INTRINSICS
static inline int64_t
rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc)
{
return rte_atomic_fetch_add_explicit((volatile __rte_atomic int64_t *)&v->cnt, inc,
rte_memory_order_seq_cst) + inc;
}
-#endif
/**
* Subtract a 64-bit value from an atomic counter and return the result.
@@ -1007,14 +953,12 @@ rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc)
static inline int64_t
rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec);
-#ifdef RTE_FORCE_INTRINSICS
static inline int64_t
rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec)
{
return rte_atomic_fetch_sub_explicit((volatile __rte_atomic int64_t *)&v->cnt, dec,
rte_memory_order_seq_cst) - dec;
}
-#endif
/**
* Atomically increment a 64-bit counter by one and test.
@@ -1029,12 +973,10 @@ rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec)
*/
static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v);
-#ifdef RTE_FORCE_INTRINSICS
static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v)
{
return rte_atomic64_add_return(v, 1) == 0;
}
-#endif
/**
* Atomically decrement a 64-bit counter by one and test.
@@ -1049,12 +991,10 @@ static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v)
*/
static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v);
-#ifdef RTE_FORCE_INTRINSICS
static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v)
{
return rte_atomic64_sub_return(v, 1) == 0;
}
-#endif
/**
* Atomically test and set a 64-bit atomic counter.
@@ -1069,12 +1009,10 @@ static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v)
*/
static inline int rte_atomic64_test_and_set(rte_atomic64_t *v);
-#ifdef RTE_FORCE_INTRINSICS
static inline int rte_atomic64_test_and_set(rte_atomic64_t *v)
{
return rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, 0, 1);
}
-#endif
/**
* Atomically set a 64-bit counter to 0.
@@ -1084,12 +1022,10 @@ static inline int rte_atomic64_test_and_set(rte_atomic64_t *v)
*/
static inline void rte_atomic64_clear(rte_atomic64_t *v);
-#ifdef RTE_FORCE_INTRINSICS
static inline void rte_atomic64_clear(rte_atomic64_t *v)
{
rte_atomic64_set(v, 0);
}
-#endif
#endif
diff --git a/lib/eal/loongarch/include/rte_atomic.h b/lib/eal/loongarch/include/rte_atomic.h
index c8066a4612..785a452c9e 100644
--- a/lib/eal/loongarch/include/rte_atomic.h
+++ b/lib/eal/loongarch/include/rte_atomic.h
@@ -5,10 +5,6 @@
#ifndef RTE_ATOMIC_LOONGARCH_H
#define RTE_ATOMIC_LOONGARCH_H
-#ifndef RTE_FORCE_INTRINSICS
-# error Platform must be built with RTE_FORCE_INTRINSICS
-#endif
-
#include <rte_common.h>
#include "generic/rte_atomic.h"
diff --git a/lib/eal/ppc/include/rte_atomic.h b/lib/eal/ppc/include/rte_atomic.h
index 10acc238f9..64f4c3d670 100644
--- a/lib/eal/ppc/include/rte_atomic.h
+++ b/lib/eal/ppc/include/rte_atomic.h
@@ -43,179 +43,6 @@ rte_atomic_thread_fence(rte_memory_order memorder)
}
/*------------------------- 16 bit atomic operations -------------------------*/
-#ifndef RTE_FORCE_INTRINSICS
-static inline int
-rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src)
-{
- return rte_atomic_compare_exchange_strong_explicit(dst, &exp, src, rte_memory_order_acquire,
- rte_memory_order_acquire) ? 1 : 0;
-}
-
-static inline int rte_atomic16_test_and_set(rte_atomic16_t *v)
-{
- return rte_atomic16_cmpset((volatile uint16_t *)&v->cnt, 0, 1);
-}
-
-static inline void
-rte_atomic16_inc(rte_atomic16_t *v)
-{
- rte_atomic_fetch_add_explicit(&v->cnt, 1, rte_memory_order_acquire);
-}
-
-static inline void
-rte_atomic16_dec(rte_atomic16_t *v)
-{
- rte_atomic_fetch_sub_explicit(&v->cnt, 1, rte_memory_order_acquire);
-}
-
-static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v)
-{
- return rte_atomic_fetch_add_explicit(&v->cnt, 1, rte_memory_order_acquire) + 1 == 0;
-}
-
-static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v)
-{
- return rte_atomic_fetch_sub_explicit(&v->cnt, 1, rte_memory_order_acquire) - 1 == 0;
-}
-
-static inline uint16_t
-rte_atomic16_exchange(volatile uint16_t *dst, uint16_t val)
-{
- return __atomic_exchange_2(dst, val, rte_memory_order_seq_cst);
-}
-
-/*------------------------- 32 bit atomic operations -------------------------*/
-
-static inline int
-rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src)
-{
- return rte_atomic_compare_exchange_strong_explicit(dst, &exp, src, rte_memory_order_acquire,
- rte_memory_order_acquire) ? 1 : 0;
-}
-
-static inline int rte_atomic32_test_and_set(rte_atomic32_t *v)
-{
- return rte_atomic32_cmpset((volatile uint32_t *)&v->cnt, 0, 1);
-}
-
-static inline void
-rte_atomic32_inc(rte_atomic32_t *v)
-{
- rte_atomic_fetch_add_explicit(&v->cnt, 1, rte_memory_order_acquire);
-}
-
-static inline void
-rte_atomic32_dec(rte_atomic32_t *v)
-{
- rte_atomic_fetch_sub_explicit(&v->cnt, 1, rte_memory_order_acquire);
-}
-
-static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v)
-{
- return rte_atomic_fetch_add_explicit(&v->cnt, 1, rte_memory_order_acquire) + 1 == 0;
-}
-
-static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v)
-{
- return rte_atomic_fetch_sub_explicit(&v->cnt, 1, rte_memory_order_acquire) - 1 == 0;
-}
-
-static inline uint32_t
-rte_atomic32_exchange(volatile uint32_t *dst, uint32_t val)
-{
- return __atomic_exchange_4(dst, val, rte_memory_order_seq_cst);
-}
-
-/*------------------------- 64 bit atomic operations -------------------------*/
-
-static inline int
-rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src)
-{
- return rte_atomic_compare_exchange_strong_explicit(dst, &exp, src, rte_memory_order_acquire,
- rte_memory_order_acquire) ? 1 : 0;
-}
-
-static inline void
-rte_atomic64_init(rte_atomic64_t *v)
-{
- v->cnt = 0;
-}
-
-static inline int64_t
-rte_atomic64_read(rte_atomic64_t *v)
-{
- return v->cnt;
-}
-
-static inline void
-rte_atomic64_set(rte_atomic64_t *v, int64_t new_value)
-{
- v->cnt = new_value;
-}
-
-static inline void
-rte_atomic64_add(rte_atomic64_t *v, int64_t inc)
-{
- rte_atomic_fetch_add_explicit(&v->cnt, inc, rte_memory_order_acquire);
-}
-
-static inline void
-rte_atomic64_sub(rte_atomic64_t *v, int64_t dec)
-{
- rte_atomic_fetch_sub_explicit(&v->cnt, dec, rte_memory_order_acquire);
-}
-
-static inline void
-rte_atomic64_inc(rte_atomic64_t *v)
-{
- rte_atomic_fetch_add_explicit(&v->cnt, 1, rte_memory_order_acquire);
-}
-
-static inline void
-rte_atomic64_dec(rte_atomic64_t *v)
-{
- rte_atomic_fetch_sub_explicit(&v->cnt, 1, rte_memory_order_acquire);
-}
-
-static inline int64_t
-rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc)
-{
- return rte_atomic_fetch_add_explicit(&v->cnt, inc, rte_memory_order_acquire) + inc;
-}
-
-static inline int64_t
-rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec)
-{
- return rte_atomic_fetch_sub_explicit(&v->cnt, dec, rte_memory_order_acquire) - dec;
-}
-
-static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v)
-{
- return rte_atomic_fetch_add_explicit(&v->cnt, 1, rte_memory_order_acquire) + 1 == 0;
-}
-
-static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v)
-{
- return rte_atomic_fetch_sub_explicit(&v->cnt, 1, rte_memory_order_acquire) - 1 == 0;
-}
-
-static inline int rte_atomic64_test_and_set(rte_atomic64_t *v)
-{
- return rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, 0, 1);
-}
-
-static inline void rte_atomic64_clear(rte_atomic64_t *v)
-{
- v->cnt = 0;
-}
-
-static inline uint64_t
-rte_atomic64_exchange(volatile uint64_t *dst, uint64_t val)
-{
- return __atomic_exchange_8(dst, val, rte_memory_order_seq_cst);
-}
-
-#endif
#ifdef __cplusplus
}
diff --git a/lib/eal/riscv/include/rte_atomic.h b/lib/eal/riscv/include/rte_atomic.h
index 66346ad474..061b175f33 100644
--- a/lib/eal/riscv/include/rte_atomic.h
+++ b/lib/eal/riscv/include/rte_atomic.h
@@ -8,10 +8,6 @@
#ifndef RTE_ATOMIC_RISCV_H
#define RTE_ATOMIC_RISCV_H
-#ifndef RTE_FORCE_INTRINSICS
-# error Platform must be built with RTE_FORCE_INTRINSICS
-#endif
-
#include <stdint.h>
#include <rte_common.h>
#include <rte_config.h>
diff --git a/lib/eal/x86/include/rte_atomic.h b/lib/eal/x86/include/rte_atomic.h
index e071e4234e..4f05302c9f 100644
--- a/lib/eal/x86/include/rte_atomic.h
+++ b/lib/eal/x86/include/rte_atomic.h
@@ -111,178 +111,6 @@ rte_atomic_thread_fence(rte_memory_order memorder)
extern "C" {
#endif
-#ifndef RTE_FORCE_INTRINSICS
-static inline int
-rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src)
-{
- uint8_t res;
-
- asm volatile(
- MPLOCKED
- "cmpxchgw %[src], %[dst];"
- "sete %[res];"
- : [res] "=a" (res), /* output */
- [dst] "=m" (*dst)
- : [src] "r" (src), /* input */
- "a" (exp),
- "m" (*dst)
- : "memory"); /* no-clobber list */
- return res;
-}
-
-static inline uint16_t
-rte_atomic16_exchange(volatile uint16_t *dst, uint16_t val)
-{
- asm volatile(
- MPLOCKED
- "xchgw %0, %1;"
- : "=r" (val), "=m" (*dst)
- : "0" (val), "m" (*dst)
- : "memory"); /* no-clobber list */
- return val;
-}
-
-static inline int rte_atomic16_test_and_set(rte_atomic16_t *v)
-{
- return rte_atomic16_cmpset((volatile uint16_t *)&v->cnt, 0, 1);
-}
-
-static inline void
-rte_atomic16_inc(rte_atomic16_t *v)
-{
- asm volatile(
- MPLOCKED
- "incw %[cnt]"
- : [cnt] "=m" (v->cnt) /* output */
- : "m" (v->cnt) /* input */
- );
-}
-
-static inline void
-rte_atomic16_dec(rte_atomic16_t *v)
-{
- asm volatile(
- MPLOCKED
- "decw %[cnt]"
- : [cnt] "=m" (v->cnt) /* output */
- : "m" (v->cnt) /* input */
- );
-}
-
-static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v)
-{
- uint8_t ret;
-
- asm volatile(
- MPLOCKED
- "incw %[cnt] ; "
- "sete %[ret]"
- : [cnt] "+m" (v->cnt), /* output */
- [ret] "=qm" (ret)
- );
- return ret != 0;
-}
-
-static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v)
-{
- uint8_t ret;
-
- asm volatile(MPLOCKED
- "decw %[cnt] ; "
- "sete %[ret]"
- : [cnt] "+m" (v->cnt), /* output */
- [ret] "=qm" (ret)
- );
- return ret != 0;
-}
-
-/*------------------------- 32 bit atomic operations -------------------------*/
-
-static inline int
-rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src)
-{
- uint8_t res;
-
- asm volatile(
- MPLOCKED
- "cmpxchgl %[src], %[dst];"
- "sete %[res];"
- : [res] "=a" (res), /* output */
- [dst] "=m" (*dst)
- : [src] "r" (src), /* input */
- "a" (exp),
- "m" (*dst)
- : "memory"); /* no-clobber list */
- return res;
-}
-
-static inline uint32_t
-rte_atomic32_exchange(volatile uint32_t *dst, uint32_t val)
-{
- asm volatile(
- MPLOCKED
- "xchgl %0, %1;"
- : "=r" (val), "=m" (*dst)
- : "0" (val), "m" (*dst)
- : "memory"); /* no-clobber list */
- return val;
-}
-
-static inline int rte_atomic32_test_and_set(rte_atomic32_t *v)
-{
- return rte_atomic32_cmpset((volatile uint32_t *)&v->cnt, 0, 1);
-}
-
-static inline void
-rte_atomic32_inc(rte_atomic32_t *v)
-{
- asm volatile(
- MPLOCKED
- "incl %[cnt]"
- : [cnt] "=m" (v->cnt) /* output */
- : "m" (v->cnt) /* input */
- );
-}
-
-static inline void
-rte_atomic32_dec(rte_atomic32_t *v)
-{
- asm volatile(
- MPLOCKED
- "decl %[cnt]"
- : [cnt] "=m" (v->cnt) /* output */
- : "m" (v->cnt) /* input */
- );
-}
-
-static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v)
-{
- uint8_t ret;
-
- asm volatile(
- MPLOCKED
- "incl %[cnt] ; "
- "sete %[ret]"
- : [cnt] "+m" (v->cnt), /* output */
- [ret] "=qm" (ret)
- );
- return ret != 0;
-}
-
-static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v)
-{
- uint8_t ret;
-
- asm volatile(MPLOCKED
- "decl %[cnt] ; "
- "sete %[ret]"
- : [cnt] "+m" (v->cnt), /* output */
- [ret] "=qm" (ret)
- );
- return ret != 0;
-}
-
-#endif /* !RTE_FORCE_INTRINSICS */
#ifdef __cplusplus
}
diff --git a/lib/eal/x86/include/rte_atomic_32.h b/lib/eal/x86/include/rte_atomic_32.h
index 0f25863aa5..37d139f30d 100644
--- a/lib/eal/x86/include/rte_atomic_32.h
+++ b/lib/eal/x86/include/rte_atomic_32.h
@@ -20,193 +20,5 @@
/*------------------------- 64 bit atomic operations -------------------------*/
-#ifndef RTE_FORCE_INTRINSICS
-static inline int
-rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src)
-{
- uint8_t res;
- union {
- struct {
- uint32_t l32;
- uint32_t h32;
- };
- uint64_t u64;
- } _exp, _src;
-
- _exp.u64 = exp;
- _src.u64 = src;
-
-#ifndef __PIC__
- asm volatile (
- MPLOCKED
- "cmpxchg8b (%[dst]);"
- "setz %[res];"
- : [res] "=a" (res) /* result in eax */
- : [dst] "S" (dst), /* esi */
- "b" (_src.l32), /* ebx */
- "c" (_src.h32), /* ecx */
- "a" (_exp.l32), /* eax */
- "d" (_exp.h32) /* edx */
- : "memory" ); /* no-clobber list */
-#else
- asm volatile (
- "xchgl %%ebx, %%edi;\n"
- MPLOCKED
- "cmpxchg8b (%[dst]);"
- "setz %[res];"
- "xchgl %%ebx, %%edi;\n"
- : [res] "=a" (res) /* result in eax */
- : [dst] "S" (dst), /* esi */
- "D" (_src.l32), /* ebx */
- "c" (_src.h32), /* ecx */
- "a" (_exp.l32), /* eax */
- "d" (_exp.h32) /* edx */
- : "memory" ); /* no-clobber list */
-#endif
-
- return res;
-}
-
-static inline uint64_t
-rte_atomic64_exchange(volatile uint64_t *dest, uint64_t val)
-{
- uint64_t old;
-
- do {
- old = *dest;
- } while (rte_atomic64_cmpset(dest, old, val) == 0);
-
- return old;
-}
-
-static inline void
-rte_atomic64_init(rte_atomic64_t *v)
-{
- int success = 0;
- uint64_t tmp;
-
- while (success == 0) {
- tmp = v->cnt;
- success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
- tmp, 0);
- }
-}
-
-static inline int64_t
-rte_atomic64_read(rte_atomic64_t *v)
-{
- int success = 0;
- uint64_t tmp;
-
- while (success == 0) {
- tmp = v->cnt;
- /* replace the value by itself */
- success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
- tmp, tmp);
- }
- return tmp;
-}
-
-static inline void
-rte_atomic64_set(rte_atomic64_t *v, int64_t new_value)
-{
- int success = 0;
- uint64_t tmp;
-
- while (success == 0) {
- tmp = v->cnt;
- success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
- tmp, new_value);
- }
-}
-
-static inline void
-rte_atomic64_add(rte_atomic64_t *v, int64_t inc)
-{
- int success = 0;
- uint64_t tmp;
-
- while (success == 0) {
- tmp = v->cnt;
- success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
- tmp, tmp + inc);
- }
-}
-
-static inline void
-rte_atomic64_sub(rte_atomic64_t *v, int64_t dec)
-{
- int success = 0;
- uint64_t tmp;
-
- while (success == 0) {
- tmp = v->cnt;
- success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
- tmp, tmp - dec);
- }
-}
-
-static inline void
-rte_atomic64_inc(rte_atomic64_t *v)
-{
- rte_atomic64_add(v, 1);
-}
-
-static inline void
-rte_atomic64_dec(rte_atomic64_t *v)
-{
- rte_atomic64_sub(v, 1);
-}
-
-static inline int64_t
-rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc)
-{
- int success = 0;
- uint64_t tmp;
-
- while (success == 0) {
- tmp = v->cnt;
- success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
- tmp, tmp + inc);
- }
-
- return tmp + inc;
-}
-
-static inline int64_t
-rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec)
-{
- int success = 0;
- uint64_t tmp;
-
- while (success == 0) {
- tmp = v->cnt;
- success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
- tmp, tmp - dec);
- }
-
- return tmp - dec;
-}
-
-static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v)
-{
- return rte_atomic64_add_return(v, 1) == 0;
-}
-
-static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v)
-{
- return rte_atomic64_sub_return(v, 1) == 0;
-}
-
-static inline int rte_atomic64_test_and_set(rte_atomic64_t *v)
-{
- return rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, 0, 1);
-}
-
-static inline void rte_atomic64_clear(rte_atomic64_t *v)
-{
- rte_atomic64_set(v, 0);
-}
-#endif
#endif /* _RTE_ATOMIC_I686_H_ */
diff --git a/lib/eal/x86/include/rte_atomic_64.h b/lib/eal/x86/include/rte_atomic_64.h
index 0a7a2131e0..1cd12695a2 100644
--- a/lib/eal/x86/include/rte_atomic_64.h
+++ b/lib/eal/x86/include/rte_atomic_64.h
@@ -22,163 +22,6 @@
/*------------------------- 64 bit atomic operations -------------------------*/
-#ifndef RTE_FORCE_INTRINSICS
-static inline int
-rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src)
-{
- uint8_t res;
-
-
- asm volatile(
- MPLOCKED
- "cmpxchgq %[src], %[dst];"
- "sete %[res];"
- : [res] "=a" (res), /* output */
- [dst] "=m" (*dst)
- : [src] "r" (src), /* input */
- "a" (exp),
- "m" (*dst)
- : "memory"); /* no-clobber list */
-
- return res;
-}
-
-static inline uint64_t
-rte_atomic64_exchange(volatile uint64_t *dst, uint64_t val)
-{
- asm volatile(
- MPLOCKED
- "xchgq %0, %1;"
- : "=r" (val), "=m" (*dst)
- : "0" (val), "m" (*dst)
- : "memory"); /* no-clobber list */
- return val;
-}
-
-static inline void
-rte_atomic64_init(rte_atomic64_t *v)
-{
- v->cnt = 0;
-}
-
-static inline int64_t
-rte_atomic64_read(rte_atomic64_t *v)
-{
- return v->cnt;
-}
-
-static inline void
-rte_atomic64_set(rte_atomic64_t *v, int64_t new_value)
-{
- v->cnt = new_value;
-}
-
-static inline void
-rte_atomic64_add(rte_atomic64_t *v, int64_t inc)
-{
- asm volatile(
- MPLOCKED
- "addq %[inc], %[cnt]"
- : [cnt] "=m" (v->cnt) /* output */
- : [inc] "ir" (inc), /* input */
- "m" (v->cnt)
- );
-}
-
-static inline void
-rte_atomic64_sub(rte_atomic64_t *v, int64_t dec)
-{
- asm volatile(
- MPLOCKED
- "subq %[dec], %[cnt]"
- : [cnt] "=m" (v->cnt) /* output */
- : [dec] "ir" (dec), /* input */
- "m" (v->cnt)
- );
-}
-
-static inline void
-rte_atomic64_inc(rte_atomic64_t *v)
-{
- asm volatile(
- MPLOCKED
- "incq %[cnt]"
- : [cnt] "=m" (v->cnt) /* output */
- : "m" (v->cnt) /* input */
- );
-}
-
-static inline void
-rte_atomic64_dec(rte_atomic64_t *v)
-{
- asm volatile(
- MPLOCKED
- "decq %[cnt]"
- : [cnt] "=m" (v->cnt) /* output */
- : "m" (v->cnt) /* input */
- );
-}
-
-static inline int64_t
-rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc)
-{
- int64_t prev = inc;
-
- asm volatile(
- MPLOCKED
- "xaddq %[prev], %[cnt]"
- : [prev] "+r" (prev), /* output */
- [cnt] "=m" (v->cnt)
- : "m" (v->cnt) /* input */
- );
- return prev + inc;
-}
-
-static inline int64_t
-rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec)
-{
- return rte_atomic64_add_return(v, -dec);
-}
-
-static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v)
-{
- uint8_t ret;
-
- asm volatile(
- MPLOCKED
- "incq %[cnt] ; "
- "sete %[ret]"
- : [cnt] "+m" (v->cnt), /* output */
- [ret] "=qm" (ret)
- );
-
- return ret != 0;
-}
-
-static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v)
-{
- uint8_t ret;
-
- asm volatile(
- MPLOCKED
- "decq %[cnt] ; "
- "sete %[ret]"
- : [cnt] "+m" (v->cnt), /* output */
- [ret] "=qm" (ret)
- );
- return ret != 0;
-}
-
-static inline int rte_atomic64_test_and_set(rte_atomic64_t *v)
-{
- return rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, 0, 1);
-}
-
-static inline void rte_atomic64_clear(rte_atomic64_t *v)
-{
- v->cnt = 0;
-}
-#endif
/*------------------------ 128 bit atomic operations -------------------------*/
--
2.53.0
^ permalink raw reply related
* [PATCH v3 00/27] deprecate rte_atomicNN family
From: Stephen Hemminger @ 2026-05-23 19:16 UTC (permalink / raw)
To: dev; +Cc: Stephen Hemminger
In-Reply-To: <20260521042043.1590536-1-stephen@networkplumber.org>
The rte_atomicNN_* family was flagged for deprecation in 2021 by
commit 3ec965b6de12 ("doc: update atomic operation deprecation")
but enforcement never landed and in-tree usage continued to grow.
v2 covered the EAL changes, lib/ring, and a starter set of drivers.
This series finishes the job: convert every remaining in-tree
caller to the C11-style rte_atomic_*_explicit() / RTE_ATOMIC()
API, then mark the legacy functions __rte_deprecated so future
in-tree and out-of-tree uses are caught at compile time.
Performance: ran the DPDK perf-tests suite (mempool, hash, stack,
ring, distributor, rcu_qsbr, etc.) on the full series; only
lib/ring showed a regression, addressed by the wrapper in patch 03.
Patch organisation
==================
01-02 EAL: drop the inline-asm fallback paths now that intrinsics
work on all platforms; reimplement rte_smp_*mb on top of
rte_atomic_thread_fence.
03-04 lib/ring and lib/bpf -- the last legacy callers in lib/.
05-25 Drivers and selftests, one patch per directory.
26 Suppress deprecation warnings in app/test/test_atomic.c,
which exercises the legacy API until it goes away.
27 Mark rte_atomicNN_* with __rte_deprecated and drop the
corresponding checkpatch grep; new uses are now caught
at compile time.
Changes since v2
================
Scope: v2 stopped at crypto/ccp (11 patches). v3 adds:
04 lib/bpf -- the bpf interpreter's atomic op macro
13-25 Remaining driver/bus/event/vdpa conversions
26-27 Test-suite warning suppression and the actual
__rte_deprecated marking
Substantive changes to patches that were in v2:
02 Also drop the rte_smp_*mb forbidden-token check from
devtools/checkpatches.sh, since the API is no longer
on a deprecation cycle.
03 lib/ring -- keep most of the original code, introduce wrapper
for the one performance sensitive CAS. This fixes the
20-30% drop in ring_perf test on x86 which was observed
when using atomic_compare_exchange_weak_explicit() with GCC.
Feedback wanted
===============
Series is targeting 26.11 rather than the next release. The driver
conversions touch many maintainers' code and several are likely to
need cycles of review/respin; a longer review window avoids rushing
contested orderings into an earlier release.
- vmbus producer commit-order pattern (patch 17)
- the ring CAS wrapper might be improve performance on other similar
uses of ring buffers in vmbus and netvsc.
- Dekker-style seq_cst handshake in net/vhost (patch 24), which
also closes a pre-existing ordering hole on weakly-ordered ISAs
- netvsc rndis_pending claim/timeout/clear cmpxchg orderings
(patch 15)
Stephen Hemminger (27):
eal: use intrinsics for rte_atomic on all platforms
eal: reimplement rte_smp_*mb with rte_atomic_thread_fence
ring: use compare-and-swap wrapper
bpf: replace atomic op macro with typed helpers
net/bonding: use stdatomic
net/nbl: remove unused rte_atomic16 field
net/ena: replace use of rte_atomicNN
net/failsafe: convert to stdatomic
net/enic: do not use deprecated rte_atomic64
net/pfe: use ethdev linkstatus helpers
net/sfc: replace rte_atomic with stdatomic
crypto/ccp: replace use of rte_atomic64 with stdatomic
bus/dpaa: replace rte_atomic16 with stdatomic
drivers: replace rte_atomic16 with stdatomic
net/netvsc: replace rte_atomic32 with stdatomic
event/sw: convert from rte_atomic32 to stdatomic
bus/vmbus: convert from rte_atomic to stdatomic
common/dpaax: remove unused atomic macros
net/bnx2x: convert from rte_atomic32 to stdatomic
bus/fslmc: replace rte_atomic32 with stdatomic
drivers/event: replace rte_atomic32 in selftests
net/hinic: replace rte_atomic32 with stdatomic
net/txgbe: replace rte_atomic32 with stdatomic
net/vhost: use stdatomic instead of rte_atomic32
vdpa/ifc: replace rte_atomic32 with stdatomic
test/atomic: suppress deprecation warnings for legacy APIs
eal: mark rte_atomicNN as deprecated
app/test/test_atomic.c | 12 +
devtools/checkpatches.sh | 16 -
doc/guides/rel_notes/deprecation.rst | 12 +-
doc/guides/rel_notes/release_26_07.rst | 4 +
drivers/bus/dpaa/base/qbman/qman.c | 9 +-
drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c | 10 +-
drivers/bus/fslmc/portal/dpaa2_hw_dpci.c | 10 +-
drivers/bus/fslmc/portal/dpaa2_hw_dpio.c | 12 +-
drivers/bus/fslmc/portal/dpaa2_hw_pvt.h | 8 +-
drivers/bus/fslmc/qbman/include/compat.h | 21 +-
drivers/bus/vmbus/private.h | 2 +-
drivers/bus/vmbus/vmbus_bufring.c | 39 ++-
drivers/common/dpaax/compat.h | 14 -
drivers/crypto/ccp/ccp_crypto.c | 11 +-
drivers/crypto/ccp/ccp_crypto.h | 2 +-
drivers/crypto/ccp/ccp_dev.c | 10 +-
drivers/crypto/ccp/ccp_dev.h | 4 +-
drivers/event/dpaa2/dpaa2_eventdev_selftest.c | 26 +-
drivers/event/dpaa2/dpaa2_hw_dpcon.c | 11 +-
drivers/event/octeontx/ssovf_evdev_selftest.c | 58 ++--
drivers/event/sw/sw_evdev.c | 8 +-
drivers/event/sw/sw_evdev.h | 4 +-
drivers/event/sw/sw_evdev_worker.c | 16 +-
drivers/net/bnx2x/bnx2x.c | 6 +-
drivers/net/bnx2x/bnx2x.h | 2 +-
drivers/net/bnx2x/ecore_sp.c | 6 +-
drivers/net/bonding/eth_bond_8023ad_private.h | 6 +-
drivers/net/bonding/rte_eth_bond_8023ad.c | 35 +-
drivers/net/ena/base/ena_plat_dpdk.h | 14 +-
drivers/net/ena/ena_ethdev.c | 21 +-
drivers/net/ena/ena_ethdev.h | 7 +-
drivers/net/enic/enic.h | 6 +-
drivers/net/enic/enic_compat.h | 1 -
drivers/net/enic/enic_main.c | 17 +-
drivers/net/enic/enic_rxtx.c | 14 +-
drivers/net/enic/enic_rxtx_vec_avx2.c | 4 +-
drivers/net/failsafe/failsafe_ops.c | 12 +-
drivers/net/failsafe/failsafe_private.h | 29 +-
drivers/net/failsafe/failsafe_rxtx.c | 2 +-
drivers/net/hinic/base/hinic_compat.h | 2 +-
drivers/net/hinic/base/hinic_pmd_hwdev.c | 24 +-
drivers/net/hinic/base/hinic_pmd_hwdev.h | 4 +-
drivers/net/nbl/nbl_hw/nbl_resource.h | 1 -
drivers/net/netvsc/hn_rndis.c | 28 +-
drivers/net/netvsc/hn_rxtx.c | 12 +-
drivers/net/netvsc/hn_var.h | 6 +-
drivers/net/pfe/pfe_ethdev.c | 32 +-
drivers/net/sfc/sfc.c | 9 +-
drivers/net/sfc/sfc.h | 4 +-
drivers/net/sfc/sfc_port.c | 7 +-
drivers/net/sfc/sfc_stats.h | 2 +-
drivers/net/txgbe/base/txgbe_mng.c | 4 +-
drivers/net/txgbe/base/txgbe_type.h | 2 +-
drivers/net/vhost/rte_eth_vhost.c | 103 +++---
drivers/vdpa/ifc/ifcvf_vdpa.c | 37 +--
lib/bpf/bpf_exec.c | 91 ++++--
lib/eal/arm/include/rte_atomic_32.h | 10 -
lib/eal/arm/include/rte_atomic_64.h | 10 -
lib/eal/include/generic/rte_atomic.h | 305 +++++-------------
lib/eal/loongarch/include/rte_atomic.h | 10 -
lib/eal/ppc/include/rte_atomic.h | 179 ----------
lib/eal/riscv/include/rte_atomic.h | 10 -
lib/eal/x86/include/rte_atomic.h | 205 +-----------
lib/eal/x86/include/rte_atomic_32.h | 188 -----------
lib/eal/x86/include/rte_atomic_64.h | 157 ---------
lib/ring/rte_ring_generic_pvt.h | 32 +-
66 files changed, 585 insertions(+), 1390 deletions(-)
--
2.53.0
^ permalink raw reply
* [V2 1/1] net/hinic3: Fix VXLAN TSO issue
From: Feifei Wang @ 2026-05-23 7:36 UTC (permalink / raw)
To: dev; +Cc: Feifei Wang
In-Reply-To: <20260523073619.996-1-wff_light@vip.163.com>
From: Feifei Wang <wangfeifei40@huawei.com>
VXLAN TSO lacks a flag bit, causing the processing function
to determine that the hardware does not support it, leading
to improper handling.
Signed-off-by: Feifei Wang <wangfeifei40@huawei.com>
---
drivers/net/hinic3/hinic3_ethdev.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/drivers/net/hinic3/hinic3_ethdev.c b/drivers/net/hinic3/hinic3_ethdev.c
index f4eb788..fbadb1e 100644
--- a/drivers/net/hinic3/hinic3_ethdev.c
+++ b/drivers/net/hinic3/hinic3_ethdev.c
@@ -696,7 +696,7 @@ hinic3_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info)
RTE_ETH_TX_OFFLOAD_VLAN_INSERT | RTE_ETH_TX_OFFLOAD_IPV4_CKSUM |
RTE_ETH_TX_OFFLOAD_UDP_CKSUM | RTE_ETH_TX_OFFLOAD_TCP_CKSUM |
RTE_ETH_TX_OFFLOAD_SCTP_CKSUM |
- RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM |
+ RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM | RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO |
RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_MULTI_SEGS;
if (nic_dev->feature_cap & NIC_F_HTN_CMDQ)
hinic3_dev_tnl_tso_support(info, nic_dev);
--
2.47.0.windows.2
^ permalink raw reply related
* [V2 0/1] net/hinic3: Fix VXLAN TSO issue
From: Feifei Wang @ 2026-05-23 7:36 UTC (permalink / raw)
To: dev
In-Reply-To: <20260520065817.931-2-wff_light@vip.163.com>
V1: The RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO flag is added to support the VXLAN TSO function
V2: Modify the commit information issue and supplement the commit information
Feifei Wang (1):
net/hinic3: Fix VXLAN TSO issue
drivers/net/hinic3/hinic3_ethdev.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
2.47.0.windows.2
^ permalink raw reply
* Re: [RFC PATCH 0/3] flow_compile: textual flow rule compiler
From: Stephen Hemminger @ 2026-05-22 18:16 UTC (permalink / raw)
To: Lukáš Šišmiš; +Cc: dev
In-Reply-To: <CAPQRu6GMtF1B24vaOA1Er7j2iB5HpmCP9CAAcbj_T6FmXys6Eg@mail.gmail.com>
On Fri, 22 May 2026 17:27:26 +0200
Lukáš Šišmiš <sismis@dyna-nic.com> wrote:
> (My) original idea was to reuse one parser for both the testpmd and the
> library to avoid multiple implementations of essentially the same thing.
>
> How would "flow compile" be useful if testpmd already has the "flow create"
> command? Perhaps as a library demo only? I would expect testpmd users to
> like the `flow create` endpoint more for its more comprehensive support.
> Similarly, at this moment, since testpmd supports interactivity, I cannot
> see how the `_compile` API would replace functions in testpmd. But perhaps
> it would be part of a greater effort to migrate to flex/bison solution.
> Since this is also a simple, single-rule, stateless parser, it is not fully
> compatible with all of TestPMD's commands (e.g., "set raw_encap"). (Ok,
> noticed, you highlighted this)
> Perhaps it is fine, just wanted to highlight this.
>
> From the user-as-an-engineer's perspective, I would be generally happy for
> this parser to exist. The drop-filter feature in Suricata, in my view,
> requires just a simple network pattern specification so looking at the
> supported patterns already ticks off most of the items I've had in mind,
> except, e.g., MPLS. I can also see this as a viable way for Suricata
> user-specified decap options for better applicability of RSS.
>
I am taking this is in a slightly different direction in next step.
The syntax of testpmd filters is just raw expression of what rte_flow items
are. The compiler should not need all the seperators and end marker.
It should be user friendly and hide the internals not expose them
^ permalink raw reply
* Re: [PATCH v5] mempool: improve cache behaviour and performance
From: Bruce Richardson @ 2026-05-22 16:12 UTC (permalink / raw)
To: Morten Brørup
Cc: dev, Andrew Rybchenko, Jingjing Wu, Praveen Shetty,
Hemant Agrawal, Sachin Saxena
In-Reply-To: <20260419095526.39526-1-mb@smartsharesystems.com>
On Sun, Apr 19, 2026 at 09:55:26AM +0000, Morten Brørup wrote:
> This patch refactors the mempool cache to eliminate some unexpected
> behaviour and reduce the mempool cache miss rate.
>
> 1.
> The actual cache size was 1.5 times the cache size specified at run-time
> mempool creation.
> This was obviously not expected by application developers.
>
> 2.
> In get operations, the check for when to use the cache as bounce buffer
> did not respect the run-time configured cache size,
> but compared to the build time maximum possible cache size
> (RTE_MEMPOOL_CACHE_MAX_SIZE, default 512).
> E.g. with a configured cache size of 32 objects, getting 256 objects
> would first fetch 32 + 256 = 288 objects into the cache,
> and then move the 256 objects from the cache to the destination memory,
> instead of fetching the 256 objects directly to the destination memory.
> This had a performance cost.
> However, this is unlikely to occur in real applications, so it is not
> important in itself.
>
> 3.
> When putting objects into a mempool, and the mempool cache did not have
> free space for so many objects,
> the cache was flushed completely, and the new objects were then put into
> the cache.
> I.e. the cache drain level was zero.
> This (complete cache flush) meant that a subsequent get operation (with
> the same number of objects) completely emptied the cache,
> so another subsequent get operation required replenishing the cache.
>
> Similarly,
> When getting objects from a mempool, and the mempool cache did not hold so
> many objects,
> the cache was replenished to cache->size + remaining objects,
> and then (the remaining part of) the requested objects were fetched via
> the cache,
> which left the cache filled (to cache->size) at completion.
> I.e. the cache refill level was cache->size (plus some, depending on
> request size).
>
> (1) was improved by generally comparing to cache->size instead of
> cache->flushthresh, when considering the capacity of the cache.
> The cache->flushthresh field is kept for API/ABI compatibility purposes,
> and initialized to cache->size instead of cache->size * 1.5.
>
> (2) was improved by generally comparing to cache->size / 2 instead of
> RTE_MEMPOOL_CACHE_MAX_SIZE, when checking the bounce buffer limit.
>
> (3) was improved by flushing and replenishing the cache by half its size,
> so a flush/refill can be followed randomly by get or put requests.
> This also reduced the number of objects in each flush/refill operation.
>
> As a consequence of these changes, the size of the array holding the
> objects in the cache (cache->objs[]) no longer needs to be
> 2 * RTE_MEMPOOL_CACHE_MAX_SIZE, and can be reduced to
> RTE_MEMPOOL_CACHE_MAX_SIZE at an API/ABI breaking release.
>
> Performance data:
> With a real WAN Optimization application, where the number of allocated
> packets varies (as they are held in e.g. shaper queues), the mempool
> cache miss rate dropped from ca. 1/20 objects to ca. 1/48 objects.
> This was deployed in production at an ISP, and using an effective cache
> size of 384 objects.
>
> As a consequence of the improved mempool cache algorithm, some drivers
> were updated accordingly:
> - The Intel idpf PMD was updated regarding how much to backfill the
> mempool cache in the AVX512 code.
> - The NXP dpaa and dpaa2 mempool drivers were updated to not set the
> mempool cache flush threshold; doing this no longer has any effect, and
> thus became superfluous.
>
> Bugzilla ID: 1027
> Fixes: ea5dd2744b90 ("mempool: cache optimisations")
> Signed-off-by: Morten Brørup <mb@smartsharesystems.com>
> ---
> Depends-on: patch-163181 ("net/intel: do not bypass mbuf lib for mbuf fast-free")
> ---
> v5:
> * Flush the cache from the bottom, where objects are colder, and move down
> the remaining objects, which are hotter.
> * In the Intel idpf PMD, move up the hot objects in the cache and refill
> with cold objects at the bottom.
> v4:
> * Added Bugzilla ID.
> * Added Fixes tag. For reference only.
> * Moved fast-free related update of Intel common driver out as a separate
> patch, and depend on that patch.
> * Omitted unrelated changes to the Intel idpf AVX512 driver, specifically
> fixing an indentation and adding mbuf instrumentation.
> * Omitted unrelated changes to the mempool library, specifically adding
> __rte_restrict and changing a couple of comments to proper sentences.
> * Please checkpatches by swapping operators in a couple of comparisons.
> v3:
> * Fixed my copy-paste bug in idpf_splitq_rearm().
> v2:
> * Fixed issue found by abidiff:
> Reverted cache objects array size reduction. Added a note instead.
> * Added missing mbuf instrumentation to the Intel idpf AVX512 driver.
> * Updated idpf_splitq_rearm() like idpf_singleq_rearm().
> * Added a few more __rte_assume(). (Inspired by AI review)
> * Updated NXP dpaa and dpaa2 mempool drivers to not set mempool cache
> flush threshold.
> * Added release notes.
> * Added deprecation notes.
> ---
> doc/guides/rel_notes/deprecation.rst | 7 ++
> doc/guides/rel_notes/release_26_07.rst | 10 +++
> drivers/mempool/dpaa/dpaa_mempool.c | 14 ----
> drivers/mempool/dpaa2/dpaa2_hw_mempool.c | 14 ----
> .../net/intel/idpf/idpf_common_rxtx_avx512.c | 52 +++++++++++---
> lib/mempool/rte_mempool.c | 14 +---
> lib/mempool/rte_mempool.h | 70 ++++++++++++-------
> 7 files changed, 104 insertions(+), 77 deletions(-)
>
Can the idpf and dpaa changes be made in separate patches, so we can review
the mempool changes along in a single patch? Even if the commits can't work
logically together, perhaps they can be separated for review, and then
squashed on apply?
/Bruce
^ permalink raw reply
* Re: [PATCH v5] mempool: improve cache behaviour and performance
From: Bruce Richardson @ 2026-05-22 16:11 UTC (permalink / raw)
To: Morten Brørup
Cc: dev, Andrew Rybchenko, Jingjing Wu, Praveen Shetty,
Hemant Agrawal, Sachin Saxena
In-Reply-To: <20260419095526.39526-1-mb@smartsharesystems.com>
On Sun, Apr 19, 2026 at 09:55:26AM +0000, Morten Brørup wrote:
> This patch refactors the mempool cache to eliminate some unexpected
> behaviour and reduce the mempool cache miss rate.
>
Agree in principle with most of these changes. As we dicussed at the DPDK
summit conference, only issue I really have is with the threshold limits
here - allocating and freeing only half the cache at a time seems overly
conservative. Thinking about use-cases:
1 for apps where alloc + free (generally Rx+Tx) is on the same core(s),
then we should run (almost) entirely out of cache.
2 for apps where we have alloc and free on different cores, then we have
some caches always being filled and others always being emptied
For case #1, we only need worry about the thresholds for the odd case where
we have a burst that causes us to overflow our cache (and we can't increase
our cache size to cope and avoid that). Otherwise the thresholds don't
matter. However, for case #2, the thresholds are constantly involved as we
always are going to backing store. In this case, we really want to have the
allocs *always* fill the cache completely, and the frees completely empty
the cache.
Because of this, while we want to avoid cases where we fill the cache
completely only to have a further free causing it to be flushed, because of
case #2 we cannot be overly conservative in how much we free/empty.
Ideally, we want to fill to full less a single burst, and empty leaving
only a single burst in the cache. Unfortunately, we don't know what those
burst limits are, so we have to try and guess the best behaviour from
everything else.
All that said, commits with specific suggestions inline.
/Bruce
<snip>
> diff --git a/lib/mempool/rte_mempool.h b/lib/mempool/rte_mempool.h
> index 2e54fc4466..432c43ab15 100644
> --- a/lib/mempool/rte_mempool.h
> +++ b/lib/mempool/rte_mempool.h
> @@ -89,7 +89,7 @@ struct __rte_cache_aligned rte_mempool_debug_stats {
> */
> struct __rte_cache_aligned rte_mempool_cache {
> uint32_t size; /**< Size of the cache */
> - uint32_t flushthresh; /**< Threshold before we flush excess elements */
> + uint32_t flushthresh; /**< Obsolete; for API/ABI compatibility purposes only */
> uint32_t len; /**< Current cache count */
> #ifdef RTE_LIBRTE_MEMPOOL_STATS
> uint32_t unused;
> @@ -107,8 +107,10 @@ struct __rte_cache_aligned rte_mempool_cache {
> /**
> * Cache objects
> *
> - * Cache is allocated to this size to allow it to overflow in certain
> - * cases to avoid needless emptying of cache.
> + * Note:
> + * Cache is allocated at double size for API/ABI compatibility purposes only.
> + * When reducing its size at an API/ABI breaking release,
> + * remember to add a cache guard after it.
> */
> alignas(RTE_CACHE_LINE_SIZE) void *objs[RTE_MEMPOOL_CACHE_MAX_SIZE * 2];
> };
> @@ -1046,12 +1048,17 @@ rte_mempool_free(struct rte_mempool *mp);
> * @param cache_size
> * If cache_size is non-zero, the rte_mempool library will try to
> * limit the accesses to the common lockless pool, by maintaining a
> - * per-lcore object cache. This argument must be lower or equal to
> - * RTE_MEMPOOL_CACHE_MAX_SIZE and n / 1.5.
> + * per-lcore object cache. This argument must be an even number,
> + * lower or equal to RTE_MEMPOOL_CACHE_MAX_SIZE and n.
> * The access to the per-lcore table is of course
> * faster than the multi-producer/consumer pool. The cache can be
> * disabled if the cache_size argument is set to 0; it can be useful to
> * avoid losing objects in cache.
> + * Note:
> + * Mempool put/get requests of more than cache_size / 2 objects may be
> + * partially or fully served directly by the multi-producer/consumer
> + * pool, to avoid the overhead of copying the objects twice (instead of
> + * once) when using the cache as a bounce buffer.
> * @param private_data_size
> * The size of the private data appended after the mempool
> * structure. This is useful for storing some private data after the
> @@ -1390,24 +1397,32 @@ rte_mempool_do_generic_put(struct rte_mempool *mp, void * const *obj_table,
> RTE_MEMPOOL_CACHE_STAT_ADD(cache, put_bulk, 1);
> RTE_MEMPOOL_CACHE_STAT_ADD(cache, put_objs, n);
>
> - __rte_assume(cache->flushthresh <= RTE_MEMPOOL_CACHE_MAX_SIZE * 2);
> - __rte_assume(cache->len <= RTE_MEMPOOL_CACHE_MAX_SIZE * 2);
> - __rte_assume(cache->len <= cache->flushthresh);
> - if (likely(cache->len + n <= cache->flushthresh)) {
> + __rte_assume(cache->size <= RTE_MEMPOOL_CACHE_MAX_SIZE);
> + __rte_assume(cache->size / 2 <= RTE_MEMPOOL_CACHE_MAX_SIZE / 2);
> + __rte_assume(cache->len <= RTE_MEMPOOL_CACHE_MAX_SIZE);
> + __rte_assume(cache->len <= cache->size);
> + if (likely(cache->len + n <= cache->size)) {
> /* Sufficient room in the cache for the objects. */
> cache_objs = &cache->objs[cache->len];
> cache->len += n;
> - } else if (n <= cache->flushthresh) {
> + } else if (n <= cache->size / 2) {
> /*
> - * The cache is big enough for the objects, but - as detected by
> - * the comparison above - has insufficient room for them.
> - * Flush the cache to make room for the objects.
> + * The number of objects is within the cache bounce buffer limit,
> + * but - as detected by the comparison above - the cache has
> + * insufficient room for them.
> + * Flush the cache to the backend to make room for the objects;
> + * flush (size / 2) objects from the bottom of the cache, where
> + * objects are less hot, and move down the remaining objects, which
> + * are more hot, from the upper half of the cache.
> */
> - cache_objs = &cache->objs[0];
> - rte_mempool_ops_enqueue_bulk(mp, cache_objs, cache->len);
> - cache->len = n;
> + __rte_assume(cache->len > cache->size / 2);
> + rte_mempool_ops_enqueue_bulk(mp, &cache->objs[0], cache->size / 2);
> + rte_memcpy(&cache->objs[0], &cache->objs[cache->size / 2],
> + sizeof(void *) * (cache->len - cache->size / 2));
> + cache_objs = &cache->objs[cache->len - cache->size / 2];
> + cache->len = cache->len - cache->size / 2 + n;
The flushing of only half the cache I'm not so certain about. I agree that
we want to not flush to empty, but I also think that we want to do more
than a half-flush, especially since we do an enqueue to the cache
immediately afterwards. Consider the case where we have a cache size of
128, and we do an enqueue of 32, with the cache currently full. In that
case we only flush 64, reducing the cache to 64, but then immediately
bringing it back up to 96. For cases where we have pipelines with all alloc
on one core and all free on another, this half-flush would be inefficient.
Instead, I would look to have a lower target threshold post-flush, and I
would suggest 25% - taking into account the newly freed buffers. For
example:
/* if n > our target of 1/4 full, flush everything,
* else flush so that we end up with 1/4 full after n added.
*/
flush_count = n > cache->size/4 ? cache->len :
(cache->len + n) - cache->size/4;
> } else {
> - /* The request itself is too big for the cache. */
> + /* The request itself is too big. */
> goto driver_enqueue_stats_incremented;
I think original comment is better. The request itself is not too big for
the whole mempool, just for the cache.
> }
>
> @@ -1524,7 +1539,7 @@ rte_mempool_do_generic_get(struct rte_mempool *mp, void **obj_table,
> /* The cache is a stack, so copy will be in reverse order. */
> cache_objs = &cache->objs[cache->len];
>
> - __rte_assume(cache->len <= RTE_MEMPOOL_CACHE_MAX_SIZE * 2);
> + __rte_assume(cache->len <= RTE_MEMPOOL_CACHE_MAX_SIZE);
> if (likely(n <= cache->len)) {
> /* The entire request can be satisfied from the cache. */
> RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_bulk, 1);
> @@ -1548,13 +1563,13 @@ rte_mempool_do_generic_get(struct rte_mempool *mp, void **obj_table,
> for (index = 0; index < len; index++)
> *obj_table++ = *--cache_objs;
>
> - /* Dequeue below would overflow mem allocated for cache? */
> - if (unlikely(remaining > RTE_MEMPOOL_CACHE_MAX_SIZE))
> + /* Dequeue below would exceed the cache bounce buffer limit? */
> + __rte_assume(cache->size / 2 <= RTE_MEMPOOL_CACHE_MAX_SIZE / 2);
> + if (unlikely(remaining > cache->size / 2))
> goto driver_dequeue;
>
> - /* Fill the cache from the backend; fetch size + remaining objects. */
> - ret = rte_mempool_ops_dequeue_bulk(mp, cache->objs,
> - cache->size + remaining);
> + /* Fill the cache from the backend; fetch (size / 2) objects. */
> + ret = rte_mempool_ops_dequeue_bulk(mp, cache->objs, cache->size / 2);
Again, the cache->size / 2 doesn't seem right here. We at most half-fill
the cache and then take some objects from that, meaning that have just done
a re-fill of cache but end the function with it less than half full. Since
we take from this value, I'd suggest just filling the cache completely.
> if (unlikely(ret < 0)) {
> /*
> * We are buffer constrained, and not able to fetch all that.
> @@ -1568,10 +1583,11 @@ rte_mempool_do_generic_get(struct rte_mempool *mp, void **obj_table,
> RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_bulk, 1);
> RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_objs, n);
>
> - __rte_assume(cache->size <= RTE_MEMPOOL_CACHE_MAX_SIZE);
> - __rte_assume(remaining <= RTE_MEMPOOL_CACHE_MAX_SIZE);
> - cache_objs = &cache->objs[cache->size + remaining];
> - cache->len = cache->size;
> + __rte_assume(cache->size / 2 <= RTE_MEMPOOL_CACHE_MAX_SIZE / 2);
> + __rte_assume(remaining <= RTE_MEMPOOL_CACHE_MAX_SIZE / 2);
> + __rte_assume(remaining <= cache->size / 2);
> + cache_objs = &cache->objs[cache->size / 2];
> + cache->len = cache->size / 2 - remaining;
> for (index = 0; index < remaining; index++)
> *obj_table++ = *--cache_objs;
>
> --
> 2.43.0
>
^ permalink raw reply
* [PATCH v1 2/2] dts: add build arguments to test run configuration
From: Koushik Bhargav Nimoji @ 2026-05-22 15:46 UTC (permalink / raw)
To: luca.vizzarro, patrickrobb1997
Cc: dev, abailey, ahassick, lylavoie, Koushik Bhargav Nimoji
In-Reply-To: <20260522154637.952588-1-knimoji@iol.unh.edu>
This patch adds the ability to specify build arguments when building DPDK
through DTS. Doing so allows users to build DPDK with the desired build
arguments, which allows for a more configurable DTS run.
Signed-off-by: Koushik Bhargav Nimoji <knimoji@iol.unh.edu>
---
dts/configurations/test_run.example.yaml | 13 +++++++++++++
dts/framework/config/test_run.py | 2 ++
dts/framework/remote_session/dpdk.py | 12 ++++++++----
dts/framework/utils.py | 21 ++++++++++++++++++++-
4 files changed, 43 insertions(+), 5 deletions(-)
diff --git a/dts/configurations/test_run.example.yaml b/dts/configurations/test_run.example.yaml
index ee641f5dce..0bd5151801 100644
--- a/dts/configurations/test_run.example.yaml
+++ b/dts/configurations/test_run.example.yaml
@@ -16,6 +16,8 @@
# `precompiled_build_dir` or `build_options` can be defined, but not both.
# `compiler_wrapper`:
# Optional, adds a compiler wrapper if present.
+# `build_args`:
+# The additional build arguments to be used when building DPDK.
# `func_traffic_generator` & `perf_traffic_generator`:
# Define `func_traffic_generator` when `func` set to true.
# Define `perf_traffic_generator` when `perf` set to true.
@@ -40,6 +42,17 @@ dpdk:
# the combination of the following two makes CC="ccache gcc"
compiler: gcc
compiler_wrapper: ccache # see `Optional Fields`
+ # arguments to be used when building DPDK
+ # build_args:
+ # c_args:
+ # - O3
+ # - g
+ # b_coverage:
+ # - "true"
+ # buildtype:
+ # - release
+ # flags:
+ # - strip
func_traffic_generator:
type: SCAPY
# perf_traffic_generator:
diff --git a/dts/framework/config/test_run.py b/dts/framework/config/test_run.py
index 76e24d1785..eab12041fc 100644
--- a/dts/framework/config/test_run.py
+++ b/dts/framework/config/test_run.py
@@ -191,6 +191,8 @@ class DPDKBuildOptionsConfiguration(FrozenModel):
#: This string will be put in front of the compiler when executing the build. Useful for adding
#: wrapper commands, such as ``ccache``.
compiler_wrapper: str = ""
+ #: The build arguments to build dpdk with
+ build_args: dict[str, list[str]] = {}
class DPDKUncompiledBuildConfiguration(BaseDPDKBuildConfiguration):
diff --git a/dts/framework/remote_session/dpdk.py b/dts/framework/remote_session/dpdk.py
index 865f97f6ca..4dc0ceeaaf 100644
--- a/dts/framework/remote_session/dpdk.py
+++ b/dts/framework/remote_session/dpdk.py
@@ -100,8 +100,8 @@ def setup(self) -> None:
match self.config:
case DPDKPrecompiledBuildConfiguration(precompiled_build_dir=build_dir):
self._set_remote_dpdk_build_dir(build_dir)
- case DPDKUncompiledBuildConfiguration(build_options=build_options):
- self._configure_dpdk_build(build_options)
+ case DPDKUncompiledBuildConfiguration():
+ self._configure_dpdk_build(self.config.build_options)
self._build_dpdk()
def teardown(self) -> None:
@@ -277,16 +277,20 @@ def _build_dpdk(self) -> None:
`remote_dpdk_tree_path` has already been set on the SUT node.
"""
ctx = get_ctx()
+ build_options = getattr(self.config, "build_options")
# If the SUT is an ice driver device, make sure to build with 16B descriptors.
if (
ctx.topology.sut_port_ingress
and ctx.topology.sut_port_ingress.config.os_driver == "ice"
):
meson_args = MesonArgs(
- default_library="static", libdir="lib", c_args="-DRTE_NET_INTEL_USE_16BYTE_DESC"
+ build_options.build_args,
+ default_library="static",
+ libdir="lib",
+ c_args="-DRTE_NET_INTEL_USE_16BYTE_DESC",
)
else:
- meson_args = MesonArgs(default_library="static", libdir="lib")
+ meson_args = MesonArgs(build_options.build_args, default_library="static", libdir="lib")
if SETTINGS.code_coverage:
meson_args._add_arg("-Db_coverage=true")
diff --git a/dts/framework/utils.py b/dts/framework/utils.py
index 38da88cd9c..e0ed35066c 100644
--- a/dts/framework/utils.py
+++ b/dts/framework/utils.py
@@ -99,10 +99,16 @@ class MesonArgs:
_default_library: str
- def __init__(self, default_library: str | None = None, **dpdk_args: str | bool):
+ def __init__(
+ self,
+ dpdk_build_args: dict[str, list[str]],
+ default_library: str | None = None,
+ **dpdk_args: str | bool,
+ ):
"""Initialize the meson arguments.
Args:
+ dpdk_build_args: The DPDK build arguments specified in the test run configuration file.
default_library: The default library type, Meson supports ``shared``, ``static`` and
``both``. Defaults to :data:`None`, in which case the argument won't be used.
dpdk_args: The arguments found in ``meson_options.txt`` in root DPDK directory.
@@ -121,6 +127,19 @@ def __init__(self, default_library: str | None = None, **dpdk_args: str | bool):
)
)
+ arguments = []
+ for option, value in dpdk_build_args.items():
+ if option == "c_args":
+ values = " ".join(f"-{val}" for val in value)
+ arguments.append(f'-D{option}="{values}"')
+ elif option == "flags":
+ values = " ".join(f"--{val}" for val in value)
+ arguments.append(values)
+ else:
+ arguments.append(f" -D{option}={value[0]}")
+
+ self._dpdk_args = " ".join(arguments)
+
def __str__(self) -> str:
"""The actual args."""
return " ".join(f"{self._default_library} {self._dpdk_args}".split())
--
2.54.0
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox