[RFC 00/32] remove rte_smp barrier functions

[RFC 00/32] remove rte_smp barrier functions

[Stephen Hemminger]

This patch series completes the removal of rte_smp_XX barrier
functions that was announced over five years ago (20.05).

Most of this is mechanical with AI assistance, so strongly
encourage maintainers to take a good look.

Stephen Hemminger (32):
  bpf: replace deprecated SMP barriers with C11 fences
  test: remove test for rte_smp_mb
  bus/vmbus: fix ring buffer ordering on weakly ordered CPUs
  bus/vmbus: fix missing acquire on receive ring index
  bus/vmbus: replace SMP barriers with C11 memory fences
  baseband: convert rte_smp_rmb to fence
  net/hinic: replace rte_smp_rmb
  net/intel: replace rte_smp_rmb
  crypto_caam_jr: replace rte_smp_rmb
  net/virtio: replcae rte_smp_rmb
  net/thunderx: replace rte_smp_rmb
  stack: always use C11 memory model implementation
  ring: replace SMP read barrier with C11 acquire fence
  crypto/virtio: update comment reference to rte_smp_rmb
  event/sw: fix unlinks in progress counter races
  event/sw: replace SMP barriers with C11 atomics
  eal/x86: move optimized fence out of SMP barrier
  common/octeontx: remove redundant barrier in mbox
  crypto/caam_jr: use IO barrier before job ring doorbell
  crypto/octeontx: use IO barrier before doorbell
  mempool/octeontx: use IO barrier in pool destroy
  event/octeontx: replace deprecated SMP barriers
  event/dpaa2: replace deprecated barrier in selftest
  event/dsw: replace SMP barriers with release fences
  event/opdl: replace SMP barriers with C11 atomics
  net/netvsc: replace SMP barrier in RNDIS response
  net/thunderx: replace deprecated SMP barriers
  net/virtio: replace deprecated barrier in avail index update
  eal: remove stale SMP barrier in rte_service
  eal: remove rte_smp_XX
  checkpatches: no longer warn about rte_smp_XX
  doc: update release notes about rte_smp_XX removal

 app/test/meson.build                          |   1 -
 app/test/test_barrier.c                       | 288 ------------------
 devtools/checkpatches.sh                      |   8 -
 doc/guides/rel_notes/deprecation.rst          |   7 -
 doc/guides/rel_notes/release_26_11.rst        |   6 +
 .../fpga_5gnr_fec/rte_fpga_5gnr_fec.c         |   8 +-
 drivers/baseband/fpga_lte_fec/fpga_lte_fec.c  |  12 +-
 drivers/bus/vmbus/private.h                   |  12 +-
 drivers/bus/vmbus/vmbus_bufring.c             |  17 +-
 drivers/bus/vmbus/vmbus_channel.c             |  10 +-
 drivers/common/octeontx/octeontx_mbox.c       |   6 +-
 drivers/crypto/caam_jr/caam_jr.c              |   5 +-
 .../crypto/octeontx/otx_cryptodev_hw_access.h |   4 +-
 drivers/crypto/virtio/virtqueue.h             |   2 +-
 drivers/event/dpaa2/dpaa2_eventdev_selftest.c |   4 +-
 drivers/event/dsw/dsw_event.c                 |   8 +-
 drivers/event/octeontx/ssovf_evdev_selftest.c |   6 +-
 drivers/event/octeontx/ssovf_worker.c         |   8 +-
 drivers/event/opdl/opdl_evdev.c               |   7 +-
 drivers/event/sw/sw_evdev.c                   |  37 ++-
 drivers/event/sw/sw_evdev.h                   |   2 +-
 drivers/event/sw/sw_evdev_scheduler.c         |  13 +-
 drivers/mempool/octeontx/octeontx_fpavf.c     |   4 +-
 drivers/net/hinic/base/hinic_pmd_cmdq.c       |   2 +-
 drivers/net/hinic/base/hinic_pmd_mgmt.c       |   2 +-
 drivers/net/intel/ice/ice_rxtx.c              |   2 +-
 drivers/net/intel/ixgbe/ixgbe_rxtx.c          |   2 +-
 drivers/net/netvsc/hn_rndis.c                 |   2 +-
 drivers/net/thunderx/base/nicvf_hw.c          |   3 +-
 drivers/net/thunderx/base/nicvf_plat.h        |   5 +-
 drivers/net/virtio/virtqueue.h                |  18 +-
 lib/bpf/bpf_pkt.c                             |  18 +-
 lib/eal/arm/include/rte_atomic_32.h           |   6 -
 lib/eal/arm/include/rte_atomic_64.h           |   6 -
 lib/eal/common/rte_service.c                  |   1 -
 lib/eal/include/generic/rte_atomic.h          |  61 ----
 lib/eal/loongarch/include/rte_atomic.h        |   6 -
 lib/eal/ppc/include/rte_atomic.h              |   6 -
 lib/eal/riscv/include/rte_atomic.h            |   6 -
 lib/eal/x86/include/rte_atomic.h              |  41 +--
 lib/ring/rte_ring_gcc_pvt.h                   |   6 +-
 lib/stack/meson.build                         |   1 -
 lib/stack/rte_stack_lf.h                      |   4 -
 lib/stack/rte_stack_lf_generic.h              | 153 ----------
 44 files changed, 156 insertions(+), 670 deletions(-)
 delete mode 100644 app/test/test_barrier.c
 delete mode 100644 lib/stack/rte_stack_lf_generic.h

-- 
2.53.0

[RFC 01/32] bpf: replace deprecated SMP barriers with C11 fences

[Stephen Hemminger]

The use counter handshake is a store-load pattern, so rte_smp_mb()
becomes a seq_cst thread fence; the read barrier in
bpf_eth_cbi_unuse() becomes an acquire fence. Same code generated
on x86 and arm64.

Use relaxed loads and stores for the counter itself. With
enable_stdatomic, the plain increment of an RTE_ATOMIC() field
compiled to a seq_cst add, i.e. two locked operations per burst.

Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
 lib/bpf/bpf_pkt.c | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/lib/bpf/bpf_pkt.c b/lib/bpf/bpf_pkt.c
index f072fdaaed..a831b5ad86 100644
--- a/lib/bpf/bpf_pkt.c
+++ b/lib/bpf/bpf_pkt.c
@@ -80,9 +80,11 @@ static struct bpf_eth_cbh tx_cbh = {
 static __rte_always_inline void
 bpf_eth_cbi_inuse(struct bpf_eth_cbi *cbi)
 {
-	cbi->use++;
+	rte_atomic_store_explicit(&cbi->use,
+		rte_atomic_load_explicit(&cbi->use, rte_memory_order_relaxed) + 1,
+		rte_memory_order_relaxed);
 	/* make sure no store/load reordering could happen */
-	rte_smp_mb();
+	rte_atomic_thread_fence(rte_memory_order_seq_cst);
 }
 
 /*
@@ -92,8 +94,10 @@ static __rte_always_inline void
 bpf_eth_cbi_unuse(struct bpf_eth_cbi *cbi)
 {
 	/* make sure all previous loads are completed */
-	rte_smp_rmb();
-	cbi->use++;
+	rte_atomic_thread_fence(rte_memory_order_acquire);
+	rte_atomic_store_explicit(&cbi->use,
+		rte_atomic_load_explicit(&cbi->use, rte_memory_order_relaxed) + 1,
+		rte_memory_order_relaxed);
 }
 
 /*
@@ -105,9 +109,9 @@ bpf_eth_cbi_wait(const struct bpf_eth_cbi *cbi)
 	uint32_t puse;
 
 	/* make sure all previous loads and stores are completed */
-	rte_smp_mb();
+	rte_atomic_thread_fence(rte_memory_order_seq_cst);
 
-	puse = cbi->use;
+	puse = rte_atomic_load_explicit(&cbi->use, rte_memory_order_relaxed);
 
 	/* in use, busy wait till current RX/TX iteration is finished */
 	if ((puse & BPF_ETH_CBI_INUSE) != 0) {
@@ -439,7 +443,7 @@ bpf_eth_cbi_unload(struct bpf_eth_cbi *bc)
 {
 	/* mark this cbi as empty */
 	bc->cb = NULL;
-	rte_smp_mb();
+	rte_atomic_thread_fence(rte_memory_order_seq_cst);
 
 	/* make sure datapath doesn't use bpf anymore, then destroy bpf */
 	bpf_eth_cbi_wait(bc);
-- 
2.53.0

[RFC 02/32] test: remove test for rte_smp_mb

[Stephen Hemminger]

The DPDK version of SMP barriers is deprecated and
being removed. Drop the test as no longer relevant.

Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
 app/test/meson.build    |   1 -
 app/test/test_barrier.c | 288 ----------------------------------------
 2 files changed, 289 deletions(-)
 delete mode 100644 app/test/test_barrier.c

diff --git a/app/test/meson.build b/app/test/meson.build
index 51abeeb732..6df9dfa222 100644
--- a/app/test/meson.build
+++ b/app/test/meson.build
@@ -28,7 +28,6 @@ source_file_deps = {
     'test_alarm.c': [],
     'test_argparse.c': ['argparse'],
     'test_atomic.c': ['hash'],
-    'test_barrier.c': [],
     'test_bitcount.c': [],
     'test_bitmap.c': [],
     'test_bitops.c': [],
diff --git a/app/test/test_barrier.c b/app/test/test_barrier.c
deleted file mode 100644
index 925a88b68a..0000000000
--- a/app/test/test_barrier.c
+++ /dev/null
@@ -1,288 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2010-2018 Intel Corporation
- */
-
- /*
-  * This is a simple functional test for rte_smp_mb() implementation.
-  * I.E. make sure that LOAD and STORE operations that precede the
-  * rte_smp_mb() call are globally visible across the lcores
-  * before the LOAD and STORE operations that follows it.
-  * The test uses simple implementation of Peterson's lock algorithm
-  * (https://en.wikipedia.org/wiki/Peterson%27s_algorithm)
-  * for two execution units to make sure that rte_smp_mb() prevents
-  * store-load reordering to happen.
-  * Also when executed on a single lcore could be used as a approximate
-  * estimation of number of cycles particular implementation of rte_smp_mb()
-  * will take.
-  */
-
-#include <stdio.h>
-#include <string.h>
-#include <stdint.h>
-#include <inttypes.h>
-
-#include <rte_memory.h>
-#include <rte_per_lcore.h>
-#include <rte_launch.h>
-#include <rte_eal.h>
-#include <rte_lcore.h>
-#include <rte_pause.h>
-#include <rte_random.h>
-#include <rte_cycles.h>
-#include <rte_vect.h>
-#include <rte_debug.h>
-
-#include "test.h"
-
-#define ADD_MAX		8
-#define ITER_MAX	0x1000000
-
-enum plock_use_type {
-	USE_MB,
-	USE_SMP_MB,
-	USE_NUM
-};
-
-struct plock {
-	volatile uint32_t flag[2];
-	volatile uint32_t victim;
-	enum plock_use_type utype;
-};
-
-/*
- * Lock plus protected by it two counters.
- */
-struct plock_test {
-	struct plock lock;
-	uint64_t val;
-	uint64_t iter;
-};
-
-/*
- * Each active lcore shares plock_test struct with it's left and right
- * neighbours.
- */
-struct lcore_plock_test {
-	struct plock_test *pt[2]; /* shared, lock-protected data */
-	uint64_t sum[2];          /* local copy of the shared data */
-	uint64_t iter;            /* number of iterations to perform */
-	uint32_t lc;              /* given lcore id */
-};
-
-static inline void
-store_load_barrier(uint32_t utype)
-{
-	if (utype == USE_MB)
-		rte_mb();
-	else if (utype == USE_SMP_MB)
-		rte_smp_mb();
-	else
-		RTE_VERIFY(0);
-}
-
-/*
- * Peterson lock implementation.
- */
-static void
-plock_lock(struct plock *l, uint32_t self)
-{
-	uint32_t other;
-
-	other = self ^ 1;
-
-	l->flag[self] = 1;
-	rte_smp_wmb();
-	l->victim = self;
-
-	store_load_barrier(l->utype);
-
-	while (l->flag[other] == 1 && l->victim == self)
-		rte_pause();
-	rte_smp_rmb();
-}
-
-static void
-plock_unlock(struct plock *l, uint32_t self)
-{
-	rte_smp_wmb();
-	l->flag[self] = 0;
-}
-
-static void
-plock_reset(struct plock *l, enum plock_use_type utype)
-{
-	memset(l, 0, sizeof(*l));
-	l->utype = utype;
-}
-
-/*
- * grab the lock, update both counters, release the lock.
- */
-static void
-plock_add(struct plock_test *pt, uint32_t self, uint32_t n)
-{
-	plock_lock(&pt->lock, self);
-	pt->iter++;
-	pt->val += n;
-	plock_unlock(&pt->lock, self);
-}
-
-static int
-plock_test1_lcore(void *data)
-{
-	uint64_t tm;
-	uint32_t lc, ln;
-	uint64_t i, n;
-	struct lcore_plock_test *lpt;
-
-	lpt = data;
-	lc = rte_lcore_id();
-
-	/* find lcore_plock_test struct for given lcore */
-	for (ln = rte_lcore_count(); ln != 0 && lpt->lc != lc; lpt++, ln--)
-		;
-
-	if (ln == 0) {
-		printf("%s(%u) error at init\n", __func__, lc);
-		return -1;
-	}
-
-	n = rte_rand() % ADD_MAX;
-	tm = rte_get_timer_cycles();
-
-	/*
-	 * for each iteration:
-	 * - update shared, locked protected data in a safe manner
-	 * - update local copy of the shared data
-	 */
-	for (i = 0; i != lpt->iter; i++) {
-
-		plock_add(lpt->pt[0], 0, n);
-		plock_add(lpt->pt[1], 1, n);
-
-		lpt->sum[0] += n;
-		lpt->sum[1] += n;
-
-		n = (n + 1) % ADD_MAX;
-	}
-
-	tm = rte_get_timer_cycles() - tm;
-
-	printf("%s(%u): %" PRIu64 " iterations finished, in %" PRIu64
-		" cycles, %#Lf cycles/iteration, "
-		"local sum={%" PRIu64 ", %" PRIu64 "}\n",
-		__func__, lc, i, tm, (long double)tm / i,
-		lpt->sum[0], lpt->sum[1]);
-	return 0;
-}
-
-/*
- * For N active lcores we allocate N+1 lcore_plock_test structures.
- * Each active lcore shares one lcore_plock_test structure with its
- * left lcore neighbor and one lcore_plock_test structure with its
- * right lcore neighbor.
- * During the test each lcore updates data in both shared structures and
- * its local copies. Then at validation phase we check that our shared
- * and local data are the same.
- */
-static int
-plock_test(uint64_t iter, enum plock_use_type utype)
-{
-	int32_t rc;
-	uint32_t i, lc, n;
-	uint64_t *sum;
-	struct plock_test *pt;
-	struct lcore_plock_test *lpt;
-
-	/* init phase, allocate and initialize shared data */
-
-	n = rte_lcore_count();
-	pt = calloc(n + 1, sizeof(*pt));
-	lpt = calloc(n, sizeof(*lpt));
-	sum = calloc(n + 1, sizeof(*sum));
-
-	printf("%s(iter=%" PRIu64 ", utype=%u) started on %u lcores\n",
-		__func__, iter, utype, n);
-
-	if (pt == NULL || lpt == NULL || sum == NULL) {
-		printf("%s: failed to allocate memory for %u lcores\n",
-			__func__, n);
-		free(pt);
-		free(lpt);
-		free(sum);
-		return -ENOMEM;
-	}
-
-	for (i = 0; i != n + 1; i++)
-		plock_reset(&pt[i].lock, utype);
-
-	i = 0;
-	RTE_LCORE_FOREACH(lc) {
-
-		lpt[i].lc = lc;
-		lpt[i].iter = iter;
-		lpt[i].pt[0] = pt + i;
-		lpt[i].pt[1] = pt + i + 1;
-		i++;
-	}
-
-	lpt[i - 1].pt[1] = pt;
-
-	for (i = 0; i != n; i++)
-		printf("lpt[%u]={lc=%u, pt={%p, %p},};\n",
-			i, lpt[i].lc, lpt[i].pt[0], lpt[i].pt[1]);
-
-
-	/* test phase - start and wait for completion on each active lcore */
-
-	rte_eal_mp_remote_launch(plock_test1_lcore, lpt, CALL_MAIN);
-	rte_eal_mp_wait_lcore();
-
-	/* validation phase - make sure that shared and local data match */
-
-	for (i = 0; i != n; i++) {
-		sum[i] += lpt[i].sum[0];
-		sum[i + 1] += lpt[i].sum[1];
-	}
-
-	sum[0] += sum[i];
-
-	rc = 0;
-	for (i = 0; i != n; i++) {
-		printf("%s: sum[%u]=%" PRIu64 ", pt[%u].val=%" PRIu64 ", pt[%u].iter=%" PRIu64 ";\n",
-			__func__, i, sum[i], i, pt[i].val, i, pt[i].iter);
-
-		/* race condition occurred, lock doesn't work properly */
-		if (sum[i] != pt[i].val || 2 * iter != pt[i].iter) {
-			printf("error: local and shared sums don't match\n");
-			rc = -1;
-		}
-	}
-
-	free(pt);
-	free(lpt);
-	free(sum);
-
-	printf("%s(utype=%u) returns %d\n", __func__, utype, rc);
-	return rc;
-}
-
-static int
-test_barrier(void)
-{
-	int32_t i, ret, rc[USE_NUM];
-
-	for (i = 0; i != RTE_DIM(rc); i++)
-		rc[i] = plock_test(ITER_MAX, i);
-
-	ret = 0;
-	for (i = 0; i != RTE_DIM(rc); i++) {
-		printf("%s for utype=%d %s\n",
-			__func__, i, rc[i] == 0 ? "passed" : "failed");
-		ret |= rc[i];
-	}
-
-	return ret;
-}
-
-REGISTER_PERF_TEST(barrier_autotest, test_barrier);
-- 
2.53.0

[RFC 03/32] bus/vmbus: fix ring buffer ordering on weakly ordered CPUs

[Stephen Hemminger]

The read index update in vmbus_rxbr_read() was only protected by a
compiler barrier. On a weakly ordered architecture the store of rindex
can become visible to the host before the data copy completes, allowing
the host to reuse and overwrite ring data still being read. Use a
release store for the read index.

The barrier in rte_vmbus_chan_signal_tx() must order the earlier ring
index update against the load of the host interrupt mask, and a write
barrier does not order stores against loads. Use a full fence, which
matches virt_mb() in the equivalent Linux and FreeBSD code.

Neither change affects generated code on x86.

Fixes: 831dba47bd36 ("bus/vmbus: add Hyper-V virtual bus support")
Cc: stable@dpdk.org

Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
 drivers/bus/vmbus/vmbus_bufring.c | 11 +++++++----
 drivers/bus/vmbus/vmbus_channel.c |  4 ++--
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/drivers/bus/vmbus/vmbus_bufring.c b/drivers/bus/vmbus/vmbus_bufring.c
index fcb97287dc..f622869974 100644
--- a/drivers/bus/vmbus/vmbus_bufring.c
+++ b/drivers/bus/vmbus/vmbus_bufring.c
@@ -237,10 +237,13 @@ vmbus_rxbr_read(struct vmbus_br *rbr, void *data, size_t dlen, size_t skip)
 	 */
 	rindex = vmbus_br_idxinc(rindex, sizeof(uint64_t), br_dsize);
 
-	/* Update the read index _after_ the channel packet is fetched.	 */
-	rte_compiler_barrier();
-
-	vbr->rindex = rindex;
+	/*
+	 * Update the read index after the channel packet is fetched.
+	 * Release store ensures the host can not observe the new read
+	 * index before the data copy is complete.
+	 */
+	rte_atomic_store_explicit((volatile uint32_t __rte_atomic *)&vbr->rindex,
+				  rindex, rte_memory_order_release);
 
 	return 0;
 }
diff --git a/drivers/bus/vmbus/vmbus_channel.c b/drivers/bus/vmbus/vmbus_channel.c
index 6887fbad46..2648d8e3fd 100644
--- a/drivers/bus/vmbus/vmbus_channel.c
+++ b/drivers/bus/vmbus/vmbus_channel.c
@@ -104,8 +104,8 @@ rte_vmbus_chan_signal_tx(struct rte_vmbus_device *dev, const struct vmbus_channe
 {
 	const struct vmbus_br *tbr = &chan->txbr;
 
-	/* Make sure all updates are done before signaling host */
-	rte_smp_wmb();
+	/* Order ring index update before reading host interrupt mask */
+	rte_atomic_thread_fence(rte_memory_order_seq_cst);
 
 	/* If host is ignoring interrupts? */
 	if (tbr->vbr->imask)
-- 
2.53.0

[RFC 04/32] bus/vmbus: fix missing acquire on receive ring index

[Stephen Hemminger]

The receive path checks available data by reading the write index and
then copies ring data. The link between the two is only the comparison,
a control dependency, which does not order load against load on a
weakly ordered architecture: packet data can be read before the host
has finished writing it. Load the write index with acquire, pairing
with the host's release store. This is the equivalent of virt_rmb()
in the Linux hv_pkt_iter code. No change in generated code on x86.

Fixes: 831dba47bd36 ("bus/vmbus: add Hyper-V virtual bus support")
Cc: stable@dpdk.org

Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
 drivers/bus/vmbus/private.h | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/drivers/bus/vmbus/private.h b/drivers/bus/vmbus/private.h
index 6efac86b77..aca5b534f9 100644
--- a/drivers/bus/vmbus/private.h
+++ b/drivers/bus/vmbus/private.h
@@ -12,6 +12,7 @@
 #include <bus_driver.h>
 #include <bus_vmbus_driver.h>
 #include <rte_log.h>
+#include <rte_stdatomic.h>
 #include <rte_eal_paging.h>
 #include <rte_vmbus_reg.h>
 
@@ -130,7 +131,16 @@ vmbus_br_availwrite(const struct vmbus_br *br, uint32_t windex)
 static inline uint32_t
 vmbus_br_availread(const struct vmbus_br *br)
 {
-	return br->dsize - vmbus_br_availwrite(br, br->vbr->windex);
+	uint32_t windex;
+
+	/* Pairs with the host's release store of the write index;
+	 * ring data must not be read before the index.
+	 */
+	windex = rte_atomic_load_explicit(
+		(volatile uint32_t __rte_atomic *)&br->vbr->windex,
+		rte_memory_order_acquire);
+
+	return br->dsize - vmbus_br_availwrite(br, windex);
 }
 
 int vmbus_txbr_write(struct vmbus_br *tbr, const struct iovec iov[], int iovlen,
-- 
2.53.0

[RFC 05/32] bus/vmbus: replace SMP barriers with C11 memory fences

[Stephen Hemminger]

Full barriers order the preceding index store against the following
load of the interrupt mask or pending size and stay seq_cst; read
barriers become acquire fences. Same code generated on x86 and arm64.

The write barrier before publishing the transmit index becomes a
release fence. Nominally stronger on arm64, but it is immediately
followed by a full barrier cmpset; converting that to a release
compare-exchange is left for the rte_atomic32 removal series.

Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
 drivers/bus/vmbus/vmbus_bufring.c | 6 +++---
 drivers/bus/vmbus/vmbus_channel.c | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/drivers/bus/vmbus/vmbus_bufring.c b/drivers/bus/vmbus/vmbus_bufring.c
index f622869974..31478d3bfa 100644
--- a/drivers/bus/vmbus/vmbus_bufring.c
+++ b/drivers/bus/vmbus/vmbus_bufring.c
@@ -58,11 +58,11 @@ void vmbus_br_setup(struct vmbus_br *br, void *buf, unsigned int blen)
 static inline bool
 vmbus_txbr_need_signal(const struct vmbus_bufring *vbr, uint32_t old_windex)
 {
-	rte_smp_mb();
+	rte_atomic_thread_fence(rte_memory_order_seq_cst);
 	if (vbr->imask)
 		return false;
 
-	rte_smp_rmb();
+	rte_atomic_thread_fence(rte_memory_order_acquire);
 
 	/*
 	 * This is the only case we need to signal when the
@@ -158,7 +158,7 @@ vmbus_txbr_write(struct vmbus_br *tbr, const struct iovec iov[], int iovlen,
 	RTE_ASSERT(windex == next_windex);
 
 	/* Ensure that data is available before updating host index */
-	rte_smp_wmb();
+	rte_atomic_thread_fence(rte_memory_order_release);
 
 	/* Checkin for our reservation. wait for our turn to update host */
 	while (!rte_atomic32_cmpset(&vbr->windex, old_windex, next_windex))
diff --git a/drivers/bus/vmbus/vmbus_channel.c b/drivers/bus/vmbus/vmbus_channel.c
index 2648d8e3fd..10293f59be 100644
--- a/drivers/bus/vmbus/vmbus_channel.c
+++ b/drivers/bus/vmbus/vmbus_channel.c
@@ -211,7 +211,7 @@ bool rte_vmbus_chan_rx_empty(const struct vmbus_channel *channel)
 {
 	const struct vmbus_br *br = &channel->rxbr;
 
-	rte_smp_rmb();
+	rte_atomic_thread_fence(rte_memory_order_acquire);
 	return br->vbr->rindex == br->vbr->windex;
 }
 
@@ -229,13 +229,13 @@ void rte_vmbus_chan_signal_read(struct rte_vmbus_device *dev,
 		return;
 
 	/* Make sure reading of pending happens after new read index */
-	rte_smp_mb();
+	rte_atomic_thread_fence(rte_memory_order_seq_cst);
 
 	pending_sz = rbr->vbr->pending_send;
 	if (!pending_sz)
 		return;
 
-	rte_smp_rmb();
+	rte_atomic_thread_fence(rte_memory_order_acquire);
 	write_sz = vmbus_br_availwrite(rbr, rbr->vbr->windex);
 
 	/* If there was space before then host was not blocked */
-- 
2.53.0

[RFC 06/32] baseband: convert rte_smp_rmb to fence

[Stephen Hemminger]

Replace deprecated rte_smp_rmb with equivalent
te_atomic_thread_fence(rte_memory_order_acquire).

Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
 drivers/baseband/fpga_5gnr_fec/rte_fpga_5gnr_fec.c |  8 ++++----
 drivers/baseband/fpga_lte_fec/fpga_lte_fec.c       | 12 ++++++------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/drivers/baseband/fpga_5gnr_fec/rte_fpga_5gnr_fec.c b/drivers/baseband/fpga_5gnr_fec/rte_fpga_5gnr_fec.c
index 45bd171ca7..e804be4cea 100644
--- a/drivers/baseband/fpga_5gnr_fec/rte_fpga_5gnr_fec.c
+++ b/drivers/baseband/fpga_5gnr_fec/rte_fpga_5gnr_fec.c
@@ -2662,7 +2662,7 @@ vc_5gnr_dequeue_ldpc_enc_one_op_cb(struct fpga_5gnr_queue *q, struct rte_bbdev_e
 		return -1;
 
 	/* make sure the response is read atomically */
-	rte_smp_rmb();
+	rte_atomic_thread_fence(rte_memory_order_acquire);
 
 	rte_bbdev_log_debug("DMA response desc %p", desc);
 
@@ -2691,7 +2691,7 @@ agx100_dequeue_ldpc_enc_one_op_cb(struct fpga_5gnr_queue *q, struct rte_bbdev_en
 		return -1;
 
 	/* make sure the response is read atomically. */
-	rte_smp_rmb();
+	rte_atomic_thread_fence(rte_memory_order_acquire);
 
 	rte_bbdev_log_debug("DMA response desc %p", desc);
 
@@ -2723,7 +2723,7 @@ vc_5gnr_dequeue_ldpc_dec_one_op_cb(struct fpga_5gnr_queue *q, struct rte_bbdev_d
 		return -1;
 
 	/* make sure the response is read atomically */
-	rte_smp_rmb();
+	rte_atomic_thread_fence(rte_memory_order_acquire);
 
 #ifdef RTE_LIBRTE_BBDEV_DEBUG
 	vc_5gnr_print_dma_dec_desc_debug_info(desc);
@@ -2769,7 +2769,7 @@ agx100_dequeue_ldpc_dec_one_op_cb(struct fpga_5gnr_queue *q, struct rte_bbdev_de
 		return -1;
 
 	/* make sure the response is read atomically. */
-	rte_smp_rmb();
+	rte_atomic_thread_fence(rte_memory_order_acquire);
 
 #ifdef RTE_LIBRTE_BBDEV_DEBUG
 	agx100_print_dma_dec_desc_debug_info(desc);
diff --git a/drivers/baseband/fpga_lte_fec/fpga_lte_fec.c b/drivers/baseband/fpga_lte_fec/fpga_lte_fec.c
index 04ac445820..cb3f29c10b 100644
--- a/drivers/baseband/fpga_lte_fec/fpga_lte_fec.c
+++ b/drivers/baseband/fpga_lte_fec/fpga_lte_fec.c
@@ -2093,7 +2093,7 @@ dequeue_enc_one_op_cb(struct fpga_queue *q, struct rte_bbdev_enc_op **op,
 		return -1;
 
 	/* make sure the response is read atomically */
-	rte_smp_rmb();
+	rte_atomic_thread_fence(rte_memory_order_acquire);
 
 	rte_bbdev_log_debug("DMA response desc %p", desc);
 
@@ -2123,7 +2123,7 @@ dequeue_enc_one_op_tb(struct fpga_queue *q, struct rte_bbdev_enc_op **op,
 		return -1;
 
 	/* Make sure the response is read atomically */
-	rte_smp_rmb();
+	rte_atomic_thread_fence(rte_memory_order_acquire);
 
 	/* Verify if done bit in all CBs is set */
 	cbs_in_op = desc->enc_req.cbs_in_op;
@@ -2135,7 +2135,7 @@ dequeue_enc_one_op_tb(struct fpga_queue *q, struct rte_bbdev_enc_op **op,
 	}
 
 	/* Make sure the response is read atomically */
-	rte_smp_rmb();
+	rte_atomic_thread_fence(rte_memory_order_acquire);
 
 	for (cb_idx = 0; cb_idx < cbs_in_op; ++cb_idx) {
 		desc = q->ring_addr + ((q->head_free_desc + desc_offset +
@@ -2166,7 +2166,7 @@ dequeue_dec_one_op_cb(struct fpga_queue *q, struct rte_bbdev_dec_op **op,
 		return -1;
 
 	/* make sure the response is read atomically */
-	rte_smp_rmb();
+	rte_atomic_thread_fence(rte_memory_order_acquire);
 
 #ifdef RTE_LIBRTE_BBDEV_DEBUG
 	print_dma_dec_desc_debug_info(desc);
@@ -2201,7 +2201,7 @@ dequeue_dec_one_op_tb(struct fpga_queue *q, struct rte_bbdev_dec_op **op,
 		return -1;
 
 	/* Make sure the response is read atomically */
-	rte_smp_rmb();
+	rte_atomic_thread_fence(rte_memory_order_acquire);
 
 	/* Verify if done bit in all CBs is set */
 	cbs_in_op = desc->dec_req.cbs_in_op;
@@ -2213,7 +2213,7 @@ dequeue_dec_one_op_tb(struct fpga_queue *q, struct rte_bbdev_dec_op **op,
 	}
 
 	/* Make sure the response is read atomically */
-	rte_smp_rmb();
+	rte_atomic_thread_fence(rte_memory_order_acquire);
 
 	for (cb_idx = 0; cb_idx < cbs_in_op; ++cb_idx) {
 		desc = q->ring_addr + ((q->head_free_desc + desc_offset +
-- 
2.53.0

[RFC 07/32] net/hinic: replace rte_smp_rmb

[Stephen Hemminger]

Replace deprecated rte_smp_rmb with equivalent
rte_atomic_thread_fence(rte_memory_order_acquire).

Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
 drivers/net/hinic/base/hinic_pmd_cmdq.c | 2 +-
 drivers/net/hinic/base/hinic_pmd_mgmt.c | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/drivers/net/hinic/base/hinic_pmd_cmdq.c b/drivers/net/hinic/base/hinic_pmd_cmdq.c
index 1816636c34..0c805e1528 100644
--- a/drivers/net/hinic/base/hinic_pmd_cmdq.c
+++ b/drivers/net/hinic/base/hinic_pmd_cmdq.c
@@ -825,7 +825,7 @@ static int cmdq_sync_cmd_direct_resp(struct hinic_cmdq *cmdq,
 		goto cmdq_unlock;
 	}
 
-	rte_smp_rmb();	/* read error code after completion */
+	rte_atomic_thread_fence(rte_memory_order_acquire);	/* read error code after completion */
 
 	if (out_param) {
 		wqe_lcmd = &curr_wqe->wqe_lcmd;
diff --git a/drivers/net/hinic/base/hinic_pmd_mgmt.c b/drivers/net/hinic/base/hinic_pmd_mgmt.c
index 6ecca407a1..f1a0b15c21 100644
--- a/drivers/net/hinic/base/hinic_pmd_mgmt.c
+++ b/drivers/net/hinic/base/hinic_pmd_mgmt.c
@@ -459,7 +459,7 @@ hinic_pf_to_mgmt_sync(struct hinic_hwdev *hwdev,
 		goto unlock_sync_msg;
 	}
 
-	rte_smp_rmb();
+	rte_atomic_thread_fence(rte_memory_order_acquire);
 	if (recv_msg->msg_len && buf_out && out_size) {
 		if (recv_msg->msg_len <= *out_size) {
 			memcpy(buf_out, recv_msg->msg,
-- 
2.53.0

[RFC 08/32] net/intel: replace rte_smp_rmb

[Stephen Hemminger]

Replace deprecated rte_smp_rmb with equivalent
rte_atomic_thread_fence(rte_memory_order_acquire).

Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
 drivers/net/intel/ice/ice_rxtx.c     | 2 +-
 drivers/net/intel/ixgbe/ixgbe_rxtx.c | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/drivers/net/intel/ice/ice_rxtx.c b/drivers/net/intel/ice/ice_rxtx.c
index c4b5454c53..6f168e358f 100644
--- a/drivers/net/intel/ice/ice_rxtx.c
+++ b/drivers/net/intel/ice/ice_rxtx.c
@@ -1955,7 +1955,7 @@ ice_rx_scan_hw_ring(struct ci_rx_queue *rxq)
 		for (j = ICE_LOOK_AHEAD - 1; j >= 0; j--)
 			s[j] = rte_le_to_cpu_16(rxdp[j].wb.status_error0);
 
-		rte_smp_rmb();
+		rte_atomic_thread_fence(rte_memory_order_acquire);
 
 		/* Compute how many status bits were set */
 		for (j = 0, nb_dd = 0; j < ICE_LOOK_AHEAD; j++)
diff --git a/drivers/net/intel/ixgbe/ixgbe_rxtx.c b/drivers/net/intel/ixgbe/ixgbe_rxtx.c
index 60222693fe..ce0a3dde34 100644
--- a/drivers/net/intel/ixgbe/ixgbe_rxtx.c
+++ b/drivers/net/intel/ixgbe/ixgbe_rxtx.c
@@ -1549,7 +1549,7 @@ ixgbe_rx_scan_hw_ring(struct ci_rx_queue *rxq)
 		for (j = 0; j < LOOK_AHEAD; j++)
 			s[j] = rte_le_to_cpu_32(rxdp[j].wb.upper.status_error);
 
-		rte_smp_rmb();
+		rte_atomic_thread_fence(rte_memory_order_acquire);
 
 		/* Compute how many status bits were set */
 		for (nb_dd = 0; nb_dd < LOOK_AHEAD &&
-- 
2.53.0

[RFC 09/32] crypto_caam_jr: replace rte_smp_rmb

[Stephen Hemminger]

Replace rte_smp_rmb with equivalent
rte_atomic_thread_fence(rte_memory_order_acquire).

Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
 drivers/crypto/caam_jr/caam_jr.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/crypto/caam_jr/caam_jr.c b/drivers/crypto/caam_jr/caam_jr.c
index a57dc56b80..1f97e72337 100644
--- a/drivers/crypto/caam_jr/caam_jr.c
+++ b/drivers/crypto/caam_jr/caam_jr.c
@@ -572,7 +572,7 @@ hw_poll_job_ring(struct sec_job_ring_t *job_ring,
 		job_ring, job_ring->pidx, job_ring->cidx,
 		limit, number_of_jobs_available, jobs_no_to_notify);
 
-	rte_smp_rmb();
+	rte_atomic_thread_fence(rte_memory_order_acquire);
 
 	while (jobs_no_to_notify > notified_descs_no) {
 		static uint64_t false_alarm;
-- 
2.53.0

[RFC 10/32] net/virtio: replcae rte_smp_rmb

[Stephen Hemminger]

Replace rte_smp_rmb with equivalent
rte_atomic_thread_fence(rte_memory_order_acquire).

Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
 drivers/net/virtio/virtqueue.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/drivers/net/virtio/virtqueue.h b/drivers/net/virtio/virtqueue.h
index 1f0e6ae77e..37fbd47d42 100644
--- a/drivers/net/virtio/virtqueue.h
+++ b/drivers/net/virtio/virtqueue.h
@@ -445,15 +445,15 @@ virtqueue_nused(const struct virtqueue *vq)
 
 	if (vq->hw->weak_barriers) {
 	/**
-	 * x86 prefers to using rte_smp_rmb over rte_atomic_load_explicit as it
-	 * reports a slightly better perf, which comes from the saved
+	 * x86 prefers to using rte_atomic_thread_fence over rte_atomic_load_explicit
+	 * as it reports a slightly better perf, which comes from the saved
 	 * branch by the compiler.
 	 * The if and else branches are identical with the smp and io
 	 * barriers both defined as compiler barriers on x86.
 	 */
 #ifdef RTE_ARCH_X86_64
 		idx = vq->vq_split.ring.used->idx;
-		rte_smp_rmb();
+		rte_atomic_thread_fence(rte_memory_order_acquire);
 #else
 		idx = rte_atomic_load_explicit(&(vq)->vq_split.ring.used->idx,
 				rte_memory_order_acquire);
-- 
2.53.0

[RFC 11/32] net/thunderx: replace rte_smp_rmb

[Stephen Hemminger]

Replace rte_smp_rmb with equivalent
rte_atomic_thread_fence(rte_memory_order_acquire).

Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
 drivers/net/thunderx/base/nicvf_plat.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/net/thunderx/base/nicvf_plat.h b/drivers/net/thunderx/base/nicvf_plat.h
index 8421f57040..7d22563035 100644
--- a/drivers/net/thunderx/base/nicvf_plat.h
+++ b/drivers/net/thunderx/base/nicvf_plat.h
@@ -26,7 +26,7 @@
 /* barrier */
 #include <rte_atomic.h>
 #define nicvf_smp_wmb() rte_smp_wmb()
-#define nicvf_smp_rmb() rte_smp_rmb()
+#define nicvf_smp_rmb() rte_atomic_thread_fence(rte_memory_order_acquire)
 
 /* utils */
 #include <rte_common.h>
-- 
2.53.0

[RFC 12/32] stack: always use C11 memory model implementation

[Stephen Hemminger]

The generic and C11 lock-free stack implementations differ only in
memory ordering. The generic version uses a full barrier where its
own comments state an acquire fence is sufficient, and seq_cst for
all length counter operations.

Only x86 and ThunderX still used the generic version. On x86 the
switch removes a locked add per CAS attempt in push and pop; TSO
provides the acquire semantics. On ThunderX the pop fence weakens
from dmb ish to dmb ishld and the push fence goes away. Unlike the
ring, no platform selected the generic stack for measured
performance reasons.

Remove it and use the C11 implementation everywhere.

Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
 lib/stack/meson.build            |   1 -
 lib/stack/rte_stack_lf.h         |   4 -
 lib/stack/rte_stack_lf_generic.h | 153 -------------------------------
 3 files changed, 158 deletions(-)
 delete mode 100644 lib/stack/rte_stack_lf_generic.h

diff --git a/lib/stack/meson.build b/lib/stack/meson.build
index 18177a742f..1fab46208f 100644
--- a/lib/stack/meson.build
+++ b/lib/stack/meson.build
@@ -7,7 +7,6 @@ headers = files('rte_stack.h')
 indirect_headers += files(
         'rte_stack_std.h',
         'rte_stack_lf.h',
-        'rte_stack_lf_generic.h',
         'rte_stack_lf_c11.h',
         'rte_stack_lf_stubs.h',
 )
diff --git a/lib/stack/rte_stack_lf.h b/lib/stack/rte_stack_lf.h
index f2b012cd0e..1bc6ee8f40 100644
--- a/lib/stack/rte_stack_lf.h
+++ b/lib/stack/rte_stack_lf.h
@@ -8,11 +8,7 @@
 #if !(defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64))
 #include "rte_stack_lf_stubs.h"
 #else
-#ifdef RTE_USE_C11_MEM_MODEL
 #include "rte_stack_lf_c11.h"
-#else
-#include "rte_stack_lf_generic.h"
-#endif
 
 /**
  * Indicates that RTE_STACK_F_LF is supported.
diff --git a/lib/stack/rte_stack_lf_generic.h b/lib/stack/rte_stack_lf_generic.h
deleted file mode 100644
index cc69e4d168..0000000000
--- a/lib/stack/rte_stack_lf_generic.h
+++ /dev/null
@@ -1,153 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2019 Intel Corporation
- */
-
-#ifndef _RTE_STACK_LF_GENERIC_H_
-#define _RTE_STACK_LF_GENERIC_H_
-
-#include <rte_branch_prediction.h>
-#include <rte_prefetch.h>
-
-static __rte_always_inline unsigned int
-__rte_stack_lf_count(struct rte_stack *s)
-{
-	/* stack_lf_push() and stack_lf_pop() do not update the list's contents
-	 * and stack_lf->len atomically, which can cause the list to appear
-	 * shorter than it actually is if this function is called while other
-	 * threads are modifying the list.
-	 *
-	 * However, given the inherently approximate nature of the get_count
-	 * callback -- even if the list and its size were updated atomically,
-	 * the size could change between when get_count executes and when the
-	 * value is returned to the caller -- this is acceptable.
-	 *
-	 * The stack_lf->len updates are placed such that the list may appear to
-	 * have fewer elements than it does, but will never appear to have more
-	 * elements. If the mempool is near-empty to the point that this is a
-	 * concern, the user should consider increasing the mempool size.
-	 */
-	/* NOTE: review for potential ordering optimization */
-	return rte_atomic_load_explicit(&s->stack_lf.used.len, rte_memory_order_seq_cst);
-}
-
-static __rte_always_inline void
-__rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
-			  struct rte_stack_lf_elem *first,
-			  struct rte_stack_lf_elem *last,
-			  unsigned int num)
-{
-	struct rte_stack_lf_head old_head;
-	int success;
-
-	old_head = list->head;
-
-	do {
-		struct rte_stack_lf_head new_head;
-
-		/* An acquire fence (or stronger) is needed for weak memory
-		 * models to establish a synchronized-with relationship between
-		 * the list->head load and store-release operations (as part of
-		 * the rte_atomic128_cmp_exchange()).
-		 */
-		rte_smp_mb();
-
-		/* Swing the top pointer to the first element in the list and
-		 * make the last element point to the old top.
-		 */
-		new_head.top = first;
-		new_head.cnt = old_head.cnt + 1;
-
-		last->next = old_head.top;
-
-		/* old_head is updated on failure */
-		success = rte_atomic128_cmp_exchange(
-				(rte_int128_t *)&list->head,
-				(rte_int128_t *)&old_head,
-				(rte_int128_t *)&new_head,
-				1, rte_memory_order_release,
-				rte_memory_order_relaxed);
-	} while (success == 0);
-	/* NOTE: review for potential ordering optimization */
-	rte_atomic_fetch_add_explicit(&list->len, num, rte_memory_order_seq_cst);
-}
-
-static __rte_always_inline struct rte_stack_lf_elem *
-__rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
-			 unsigned int num,
-			 void **obj_table,
-			 struct rte_stack_lf_elem **last)
-{
-	struct rte_stack_lf_head old_head;
-	int success = 0;
-
-	/* Reserve num elements, if available */
-	while (1) {
-		/* NOTE: review for potential ordering optimization */
-		uint64_t len = rte_atomic_load_explicit(&list->len, rte_memory_order_seq_cst);
-
-		/* Does the list contain enough elements? */
-		if (unlikely(len < num))
-			return NULL;
-
-		/* NOTE: review for potential ordering optimization */
-		if (rte_atomic_compare_exchange_strong_explicit(&list->len, &len, len - num,
-				rte_memory_order_seq_cst, rte_memory_order_seq_cst))
-			break;
-	}
-
-	old_head = list->head;
-
-	/* Pop num elements */
-	do {
-		struct rte_stack_lf_head new_head;
-		struct rte_stack_lf_elem *tmp;
-		unsigned int i;
-
-		/* An acquire fence (or stronger) is needed for weak memory
-		 * models to ensure the LF LIFO element reads are properly
-		 * ordered with respect to the head pointer read.
-		 */
-		rte_smp_mb();
-
-		rte_prefetch0(old_head.top);
-
-		tmp = old_head.top;
-
-		/* Traverse the list to find the new head. A next pointer will
-		 * either point to another element or NULL; if a thread
-		 * encounters a pointer that has already been popped, the CAS
-		 * will fail.
-		 */
-		for (i = 0; i < num && tmp != NULL; i++) {
-			rte_prefetch0(tmp->next);
-			if (obj_table)
-				obj_table[i] = tmp->data;
-			if (last)
-				*last = tmp;
-			tmp = tmp->next;
-		}
-
-		/* If NULL was encountered, the list was modified while
-		 * traversing it. Retry.
-		 */
-		if (i != num) {
-			old_head = list->head;
-			continue;
-		}
-
-		new_head.top = tmp;
-		new_head.cnt = old_head.cnt + 1;
-
-		/* old_head is updated on failure */
-		success = rte_atomic128_cmp_exchange(
-				(rte_int128_t *)&list->head,
-				(rte_int128_t *)&old_head,
-				(rte_int128_t *)&new_head,
-				1, rte_memory_order_release,
-				rte_memory_order_relaxed);
-	} while (success == 0);
-
-	return old_head.top;
-}
-
-#endif /* _RTE_STACK_LF_GENERIC_H_ */
-- 
2.53.0

[RFC 13/32] ring: replace SMP read barrier with C11 acquire fence

[Stephen Hemminger]

rte_smp_rmb() is deprecated. The acquire fence generates the same
code everywhere the gcc implementation is used: a compiler barrier
on x86 and dmb ishld on ThunderX.

The gcc implementation itself is kept: unlike the lock-free stack,
the in-tree comment records a 10% x86 performance drop with the
C11 version, which needs re-measuring with current compilers
before RTE_USE_C11_MEM_MODEL can be removed.

Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
 lib/ring/rte_ring_gcc_pvt.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/ring/rte_ring_gcc_pvt.h b/lib/ring/rte_ring_gcc_pvt.h
index 340ece28c7..9c2c58824d 100644
--- a/lib/ring/rte_ring_gcc_pvt.h
+++ b/lib/ring/rte_ring_gcc_pvt.h
@@ -55,10 +55,10 @@ __rte_ring_headtail_move_head_st(struct rte_ring_headtail *d,
 
 	*old_head = d->head;
 
-	/* add rmb barrier to avoid load/load reorder in weak
-	 * memory model. It is noop on x86
+	/* Acquire fence to avoid load/load reorder in weak
+	 * memory model. It is noop on x86.
 	 */
-	rte_smp_rmb();
+	rte_atomic_thread_fence(rte_memory_order_acquire);
 
 	/*
 	 *  The subtraction is done between two unsigned 32bits value
-- 
2.53.0

[RFC 14/32] crypto/virtio: update comment reference to rte_smp_rmb

[Stephen Hemminger]

The rte_smp_rmb helper is no longer used.

Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
 drivers/crypto/virtio/virtqueue.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/crypto/virtio/virtqueue.h b/drivers/crypto/virtio/virtqueue.h
index 97a3ace48c..23c001a8e6 100644
--- a/drivers/crypto/virtio/virtqueue.h
+++ b/drivers/crypto/virtio/virtqueue.h
@@ -302,7 +302,7 @@ virtqueue_nused(const struct virtqueue *vq)
 
 	if (vq->hw->weak_barriers) {
 	/**
-	 * x86 prefers to using rte_smp_rmb over rte_atomic_load_explicit as it
+	 * x86 prefers to using virtio_rmb over rte_atomic_load_explicit as it
 	 * reports a slightly better perf, which comes from the saved
 	 * branch by the compiler.
 	 * The if and else branches are identical with the smp and io
-- 
2.53.0

[RFC 15/32] event/sw: fix unlinks in progress counter races

[Stephen Hemminger]

The counter is written by both the application thread (increment on
unlink) and the scheduler (clear on ack) as a plain uint8_t. An
increment is lost if it lands between the scheduler's test and clear,
making rte_event_port_unlinks_in_progress() report completion before
the scheduler has seen the unlink. Nothing orders the scheduler's
later cq map reads against the counter test on a weakly ordered CPU
either.

Make the counter atomic: release fetch-add on unlink, acquire
exchange to clear. The exchange cannot lose a concurrent increment,
and the acquire guarantees the scheduler only acks unlinks whose cq
map update it can observe, replacing the full barrier in unlink.

Fixes: bd5ac24fea88 ("event/sw: implement unlinks in progress function")
Cc: stable@dpdk.org

Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
 drivers/event/sw/sw_evdev.c           |  8 +++++---
 drivers/event/sw/sw_evdev.h           |  2 +-
 drivers/event/sw/sw_evdev_scheduler.c | 13 ++++++++++---
 3 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/drivers/event/sw/sw_evdev.c b/drivers/event/sw/sw_evdev.c
index 3ad82e94ac..bb6f50e03b 100644
--- a/drivers/event/sw/sw_evdev.c
+++ b/drivers/event/sw/sw_evdev.c
@@ -119,8 +119,9 @@ sw_port_unlink(struct rte_eventdev *dev, void *port, uint8_t queues[],
 		}
 	}
 
-	p->unlinks_in_progress += unlinked;
-	rte_smp_mb();
+	/* Pairs with the acquire exchange in the scheduler */
+	rte_atomic_fetch_add_explicit(&p->unlinks_in_progress, unlinked,
+				      rte_memory_order_release);
 
 	return unlinked;
 }
@@ -130,7 +131,8 @@ sw_port_unlinks_in_progress(struct rte_eventdev *dev, void *port)
 {
 	RTE_SET_USED(dev);
 	struct sw_port *p = port;
-	return p->unlinks_in_progress;
+	return rte_atomic_load_explicit(&p->unlinks_in_progress,
+					rte_memory_order_relaxed);
 }
 
 static int
diff --git a/drivers/event/sw/sw_evdev.h b/drivers/event/sw/sw_evdev.h
index c159be21be..8b9118bf91 100644
--- a/drivers/event/sw/sw_evdev.h
+++ b/drivers/event/sw/sw_evdev.h
@@ -160,7 +160,7 @@ struct sw_port {
 	 * progress is read by the scheduler, no more events will be pushed to
 	 * the port - hence the scheduler core can just assign zero.
 	 */
-	uint8_t unlinks_in_progress;
+	RTE_ATOMIC(uint8_t) unlinks_in_progress;
 
 	int16_t is_directed; /** Takes from a single directed QID */
 	/**
diff --git a/drivers/event/sw/sw_evdev_scheduler.c b/drivers/event/sw/sw_evdev_scheduler.c
index a5fdcf301b..f4bce2cbb8 100644
--- a/drivers/event/sw/sw_evdev_scheduler.c
+++ b/drivers/event/sw/sw_evdev_scheduler.c
@@ -523,9 +523,16 @@ sw_event_schedule(struct rte_eventdev *dev)
 		do {
 			in_pkts = 0;
 			for (i = 0; i < sw->port_count; i++) {
-				/* ack the unlinks in progress as done */
-				if (sw->ports[i].unlinks_in_progress)
-					sw->ports[i].unlinks_in_progress = 0;
+				/* Ack the unlinks in progress as done. The
+				 * acquire exchange orders the cq map reads
+				 * below after the unlinker's map update.
+				 */
+				if (rte_atomic_load_explicit(
+						&sw->ports[i].unlinks_in_progress,
+						rte_memory_order_relaxed))
+					rte_atomic_exchange_explicit(
+						&sw->ports[i].unlinks_in_progress,
+						0, rte_memory_order_acquire);
 
 				if (sw->ports[i].is_directed)
 					in_pkts += sw_schedule_pull_port_dir(sw, i);
-- 
2.53.0

[RFC 16/32] event/sw: replace SMP barriers with C11 atomics

[Stephen Hemminger]

The write barrier plus plain store publish patterns (cq map count,
port initialized, device started) become release stores; the barrier
after clearing started in sw_stop() becomes a release fence. All are
control path.

Scheduler reads of cq_num_mapped_cqs stay plain loads: acquire loads
would land in the inner scheduling loops on weakly ordered CPUs. The
transient window where a new count is visible before the map entry
on such CPUs predates this change and is unchanged by it; unlink is
covered by the counter handshake.

Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
 drivers/event/sw/sw_evdev.c | 29 ++++++++++++++++++++---------
 1 file changed, 20 insertions(+), 9 deletions(-)

diff --git a/drivers/event/sw/sw_evdev.c b/drivers/event/sw/sw_evdev.c
index bb6f50e03b..bdb3dff997 100644
--- a/drivers/event/sw/sw_evdev.c
+++ b/drivers/event/sw/sw_evdev.c
@@ -84,8 +84,10 @@ sw_port_link(struct rte_eventdev *dev, void *port, const uint8_t queues[],
 		}
 
 		q->cq_map[q->cq_num_mapped_cqs] = p->id;
-		rte_smp_wmb();
-		q->cq_num_mapped_cqs++;
+		/* Release publishes the map entry before the new count */
+		rte_atomic_store_explicit(
+			(uint32_t __rte_atomic *)&q->cq_num_mapped_cqs,
+			q->cq_num_mapped_cqs + 1, rte_memory_order_release);
 	}
 	return i;
 }
@@ -105,8 +107,14 @@ sw_port_unlink(struct rte_eventdev *dev, void *port, uint8_t queues[],
 			if (q->cq_map[j] == p->id) {
 				q->cq_map[j] =
 					q->cq_map[q->cq_num_mapped_cqs - 1];
-				rte_smp_wmb();
-				q->cq_num_mapped_cqs--;
+				/* Release publishes the map update
+				 * before the new count
+				 */
+				rte_atomic_store_explicit(
+					(uint32_t __rte_atomic *)
+						&q->cq_num_mapped_cqs,
+					q->cq_num_mapped_cqs - 1,
+					rte_memory_order_release);
 				unlinked++;
 
 				p->num_qids_mapped--;
@@ -208,8 +216,9 @@ sw_port_setup(struct rte_eventdev *dev, uint8_t port_id,
 	}
 	dev->data->ports[port_id] = p;
 
-	rte_smp_wmb();
-	p->initialized = 1;
+	/* Release publishes the port setup before initialized flag */
+	rte_atomic_store_explicit((uint8_t __rte_atomic *)&p->initialized, 1,
+				  rte_memory_order_release);
 	return 0;
 }
 
@@ -815,8 +824,9 @@ sw_start(struct rte_eventdev *dev)
 	if (sw_xstats_init(sw) < 0)
 		return -EINVAL;
 
-	rte_smp_wmb();
-	sw->started = 1;
+	/* Release publishes device state before the started flag */
+	rte_atomic_store_explicit((uint8_t __rte_atomic *)&sw->started, 1,
+				  rte_memory_order_release);
 
 	return 0;
 }
@@ -845,7 +855,8 @@ sw_stop(struct rte_eventdev *dev)
 	sw_clean_qid_iqs(dev);
 	sw_xstats_uninit(sw);
 	sw->started = 0;
-	rte_smp_wmb();
+	/* Order the started store before re-enabling the service */
+	rte_atomic_thread_fence(rte_memory_order_release);
 
 	if (runstate == 1)
 		rte_service_runstate_set(sw->service_id, 1);
-- 
2.53.0

[RFC 17/32] eal/x86: move optimized fence out of SMP barrier

[Stephen Hemminger]

rte_atomic_thread_fence() implemented the seq_cst case by calling the
deprecated rte_smp_mb(). Invert the dependency: the lock add based
fence moves into rte_atomic_thread_fence() and rte_smp_mb() becomes a
wrapper around it, so removing the deprecated barriers later is a pure
deletion. The optimization itself must stay; a plain seq_cst fence is
an mfence, about twice the cost. No change in generated code.

Drop no longer used rte_smp_mb().

Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
 lib/eal/x86/include/rte_atomic.h | 37 ++++++++++++++------------------
 1 file changed, 16 insertions(+), 21 deletions(-)

diff --git a/lib/eal/x86/include/rte_atomic.h b/lib/eal/x86/include/rte_atomic.h
index e071e4234e..780fdce871 100644
--- a/lib/eal/x86/include/rte_atomic.h
+++ b/lib/eal/x86/include/rte_atomic.h
@@ -60,23 +60,8 @@ extern "C" {
  * Basic idea is to use lock prefixed add with some dummy memory location
  * as the destination. From their experiments 128B(2 cache lines) below
  * current stack pointer looks like a good candidate.
- * So below we use that technique for rte_smp_mb() implementation.
  */
 
-static __rte_always_inline void
-rte_smp_mb(void)
-{
-#ifdef RTE_TOOLCHAIN_MSVC
-	_mm_mfence();
-#else
-#ifdef RTE_ARCH_I686
-	asm volatile("lock addl $0, -128(%%esp); " ::: "memory");
-#else
-	asm volatile("lock addl $0, -128(%%rsp); " ::: "memory");
-#endif
-#endif
-}
-
 #define rte_io_mb() rte_mb()
 
 #define rte_io_wmb() rte_compiler_barrier()
@@ -86,17 +71,27 @@ rte_smp_mb(void)
 /**
  * Synchronization fence between threads based on the specified memory order.
  *
- * On x86 the __rte_atomic_thread_fence(rte_memory_order_seq_cst) generates full 'mfence'
- * which is quite expensive. The optimized implementation of rte_smp_mb is
- * used instead.
+ * On x86 the __rte_atomic_thread_fence(rte_memory_order_seq_cst) generates
+ * a full 'mfence' which is quite expensive. The optimized lock add on a
+ * dummy stack location (see above) is used instead.
  */
 static __rte_always_inline void
 rte_atomic_thread_fence(rte_memory_order memorder)
 {
-	if (memorder == rte_memory_order_seq_cst)
-		rte_smp_mb();
-	else
+	if (memorder != rte_memory_order_seq_cst) {
 		__rte_atomic_thread_fence(memorder);
+		return;
+	}
+
+#ifdef RTE_TOOLCHAIN_MSVC
+	_mm_mfence();
+#else
+#ifdef RTE_ARCH_I686
+	asm volatile("lock addl $0, -128(%%esp); " ::: "memory");
+#else
+	asm volatile("lock addl $0, -128(%%rsp); " ::: "memory");
+#endif
+#endif
 }
 
 #ifdef __cplusplus
-- 
2.53.0

[RFC 18/32] common/octeontx: remove redundant barrier in mbox

[Stephen Hemminger]

The SMP write barrier between writing the mailbox header and ringing
the mailbox register is redundant: rte_write64() begins with
rte_io_wmb(), which already orders the header write before the
trigger, and does so against the device rather than only within the
inner shareable domain.

Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
 drivers/common/octeontx/octeontx_mbox.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/drivers/common/octeontx/octeontx_mbox.c b/drivers/common/octeontx/octeontx_mbox.c
index 9e0bbf453f..8b0da3a98e 100644
--- a/drivers/common/octeontx/octeontx_mbox.c
+++ b/drivers/common/octeontx/octeontx_mbox.c
@@ -113,8 +113,10 @@ mbox_send_request(struct mbox *m, struct octeontx_mbox_hdr *hdr,
 
 	/* Write the msg header */
 	rte_write64(new_hdr.u64, ram_mbox_hdr);
-	rte_smp_wmb();
-	/* Notify PF about the new msg - write to MBOX reg generates PF IRQ */
+	/* Notify PF about the new msg - write to MBOX reg generates PF IRQ.
+	 * rte_write64() issues rte_io_wmb() which orders the header write
+	 * before the trigger; no separate barrier is needed.
+	 */
 	rte_write64(0, m->reg);
 }
 
-- 
2.53.0

[RFC 19/32] crypto/caam_jr: use IO barrier before job ring doorbell

[Stephen Hemminger]

The barrier orders the input ring descriptor write in DMA memory
against the MMIO doorbell write. That is device ordering, not SMP
ordering: rte_io_wmb() is the correct barrier, and on arm64 the
inner shareable dmb the SMP barrier generated was not sufficient
to order against the device in the first place.

Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
 drivers/crypto/caam_jr/caam_jr.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/drivers/crypto/caam_jr/caam_jr.c b/drivers/crypto/caam_jr/caam_jr.c
index 1f97e72337..7a345c436e 100644
--- a/drivers/crypto/caam_jr/caam_jr.c
+++ b/drivers/crypto/caam_jr/caam_jr.c
@@ -1451,7 +1451,8 @@ caam_jr_enqueue_op(struct rte_crypto_op *op, struct caam_jr_qp *qp)
 	/* Set ptr in input ring to current descriptor	*/
 	sec_write_addr(&ring->input_ring[ring->pidx],
 			(phys_addr_t)caam_jr_vtop_ctx(ctx, ctx->jobdes.desc));
-	rte_smp_wmb();
+	/* Descriptor must be visible to the device before the doorbell */
+	rte_io_wmb();
 
 	/* Notify HW that a new job is enqueued */
 	hw_enqueue_desc_on_job_ring(ring);
-- 
2.53.0

[RFC 20/32] crypto/octeontx: use IO barrier before doorbell

[Stephen Hemminger]

The doorbell is written with rte_write64_relaxed(), so the barrier
before it is what orders the queued instructions against the device.
That is device ordering, not SMP ordering: use rte_io_wmb().

Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
 drivers/crypto/octeontx/otx_cryptodev_hw_access.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/drivers/crypto/octeontx/otx_cryptodev_hw_access.h b/drivers/crypto/octeontx/otx_cryptodev_hw_access.h
index 3104e6d31e..096dcf8f87 100644
--- a/drivers/crypto/octeontx/otx_cryptodev_hw_access.h
+++ b/drivers/crypto/octeontx/otx_cryptodev_hw_access.h
@@ -203,8 +203,8 @@ static __rte_always_inline void
 otx_cpt_ring_dbell(struct cpt_instance *instance, uint16_t count)
 {
 	struct cpt_vf *cptvf = (struct cpt_vf *)instance;
-	/* Memory barrier to flush pending writes */
-	rte_smp_wmb();
+	/* Instructions must be visible to the device before the doorbell */
+	rte_io_wmb();
 	otx_cpt_write_vq_doorbell(cptvf, count);
 }
 
-- 
2.53.0

[RFC 21/32] mempool/octeontx: use IO barrier in pool destroy

[Stephen Hemminger]

The barrier flushes outstanding writes to the FPA pool before the
teardown mailbox sequence. That is ordering against the device, so
use rte_io_wmb(). Control path.

Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
 drivers/mempool/octeontx/octeontx_fpavf.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/drivers/mempool/octeontx/octeontx_fpavf.c b/drivers/mempool/octeontx/octeontx_fpavf.c
index 6cd0fc057f..7051422cb7 100644
--- a/drivers/mempool/octeontx/octeontx_fpavf.c
+++ b/drivers/mempool/octeontx/octeontx_fpavf.c
@@ -616,8 +616,8 @@ octeontx_fpa_bufpool_destroy(uintptr_t handle, int node_id)
 
 	RTE_SET_USED(node_id);
 
-	/* Wait for all outstanding writes to be committed */
-	rte_smp_wmb();
+	/* Make outstanding pool writes visible to the device */
+	rte_io_wmb();
 
 	if (unlikely(!octeontx_fpa_handle_valid(handle)))
 		return -EINVAL;
-- 
2.53.0

[RFC 22/32] event/octeontx: replace deprecated SMP barriers

[Stephen Hemminger]

The worker barriers order event payload writes in normal memory
before the add work or tag switch store to the SSO device, so they
are IO ordering: use rte_io_wmb(). The selftest barriers publish
worker parameters before launching lcores; convert them to release
fences.

Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
 drivers/event/octeontx/ssovf_evdev_selftest.c | 6 +++---
 drivers/event/octeontx/ssovf_worker.c         | 8 ++++----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/drivers/event/octeontx/ssovf_evdev_selftest.c b/drivers/event/octeontx/ssovf_evdev_selftest.c
index b54ae126d2..614f9bbaef 100644
--- a/drivers/event/octeontx/ssovf_evdev_selftest.c
+++ b/drivers/event/octeontx/ssovf_evdev_selftest.c
@@ -65,7 +65,7 @@ seqn_list_update(int val)
 		return -1;
 
 	seqn_list[seqn_list_index++] = val;
-	rte_smp_wmb();
+	rte_atomic_thread_fence(rte_memory_order_release);
 	return 0;
 }
 
@@ -637,7 +637,7 @@ launch_workers_and_wait(int (*main_worker)(void *),
 	param[0].sched_type = sched_type;
 	param[0].port = 0;
 	param[0].dequeue_tmo_ticks = dequeue_tmo_ticks;
-	rte_smp_wmb();
+	rte_atomic_thread_fence(rte_memory_order_release);
 
 	w_lcore = rte_get_next_lcore(
 			/* start core */ -1,
@@ -650,7 +650,7 @@ launch_workers_and_wait(int (*main_worker)(void *),
 		param[port].sched_type = sched_type;
 		param[port].port = port;
 		param[port].dequeue_tmo_ticks = dequeue_tmo_ticks;
-		rte_smp_wmb();
+		rte_atomic_thread_fence(rte_memory_order_release);
 		w_lcore = rte_get_next_lcore(w_lcore, 1, 0);
 		rte_eal_remote_launch(worker, &param[port], w_lcore);
 	}
diff --git a/drivers/event/octeontx/ssovf_worker.c b/drivers/event/octeontx/ssovf_worker.c
index 89b5dc056c..f3c6a0da5c 100644
--- a/drivers/event/octeontx/ssovf_worker.c
+++ b/drivers/event/octeontx/ssovf_worker.c
@@ -59,10 +59,10 @@ ssows_fwd_group(struct ssows *ws, const struct rte_event *ev, const uint8_t grp)
 		newtag |= grp << 20;
 		newtag |= tag;
 		ssows_swtag_norm(ws, newtag, SSO_SYNC_ATOMIC);
-		rte_smp_wmb();
+		rte_io_wmb();
 		ssows_swtag_wait(ws);
 	} else {
-		rte_smp_wmb();
+		rte_io_wmb();
 	}
 	ssows_add_work(ws, event_ptr, tag, new_tt, grp);
 }
@@ -158,7 +158,7 @@ ssows_enq_burst(void *port, const struct rte_event ev[], uint16_t nb_events)
 
 	switch (ev->op) {
 	case RTE_EVENT_OP_NEW:
-		rte_smp_wmb();
+		rte_io_wmb();
 		ssows_new_event(ws, ev);
 		break;
 	case RTE_EVENT_OP_FORWARD:
@@ -179,7 +179,7 @@ ssows_enq_new_burst(void *port, const struct rte_event ev[], uint16_t nb_events)
 	uint16_t i;
 	struct ssows *ws = port;
 
-	rte_smp_wmb();
+	rte_io_wmb();
 	for (i = 0; i < nb_events; i++)
 		ssows_new_event(ws,  &ev[i]);
 
-- 
2.53.0

[RFC 23/32] event/dpaa2: replace deprecated barrier in selftest

[Stephen Hemminger]

Publish worker parameters with a release fence before launching the
lcores instead of the deprecated SMP write barrier.

Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
 drivers/event/dpaa2/dpaa2_eventdev_selftest.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/drivers/event/dpaa2/dpaa2_eventdev_selftest.c b/drivers/event/dpaa2/dpaa2_eventdev_selftest.c
index 9d4938efe6..12c1bc9878 100644
--- a/drivers/event/dpaa2/dpaa2_eventdev_selftest.c
+++ b/drivers/event/dpaa2/dpaa2_eventdev_selftest.c
@@ -524,7 +524,7 @@ launch_workers_and_wait(int (*main_worker)(void *),
 	param[0].sched_type = sched_type;
 	param[0].port = 0;
 	param[0].dequeue_tmo_ticks = dequeue_tmo_ticks;
-	rte_smp_wmb();
+	rte_atomic_thread_fence(rte_memory_order_release);
 
 	w_lcore = rte_get_next_lcore(
 			/* start core */ -1,
@@ -537,7 +537,7 @@ launch_workers_and_wait(int (*main_worker)(void *),
 		param[port].sched_type = sched_type;
 		param[port].port = port;
 		param[port].dequeue_tmo_ticks = dequeue_tmo_ticks;
-		rte_smp_wmb();
+		rte_atomic_thread_fence(rte_memory_order_release);
 		w_lcore = rte_get_next_lcore(w_lcore, 1, 0);
 		rte_eal_remote_launch(workers, &param[port], w_lcore);
 	}
-- 
2.53.0

[RFC 24/32] event/dsw: replace SMP barriers with release fences

[Stephen Hemminger]

All four barriers publish flow table or ring updates before a
control message enqueue, which is release ordering. All are on the
migration control path, not per event.

The fences may in fact be removable: the control messages travel
through rte_ring, whose producer tail update is itself a release
store covering the preceding writes. Left in place here to keep
this a like for like conversion.

Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
 drivers/event/dsw/dsw_event.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/drivers/event/dsw/dsw_event.c b/drivers/event/dsw/dsw_event.c
index 399d9f050e..82d3636b29 100644
--- a/drivers/event/dsw/dsw_event.c
+++ b/drivers/event/dsw/dsw_event.c
@@ -339,7 +339,7 @@ dsw_port_handle_pause_flows(struct dsw_evdev *dsw, struct dsw_port *port,
 	/* Make sure any stores to the original port's in_ring is seen
 	 * before the ctl message.
 	 */
-	rte_smp_wmb();
+	rte_atomic_thread_fence(rte_memory_order_release);
 
 	dsw_port_ctl_enqueue(&dsw->ports[originating_port_id], &cfm);
 }
@@ -821,7 +821,7 @@ dsw_port_move_parallel_flows(struct dsw_evdev *dsw,
 		}
 	}
 
-	rte_smp_wmb();
+	rte_atomic_thread_fence(rte_memory_order_release);
 
 	dsw_port_end_emigration(dsw, source_port, RTE_SCHED_TYPE_PARALLEL);
 }
@@ -1190,7 +1190,7 @@ dsw_port_move_emigrating_flows(struct dsw_evdev *dsw,
 		    dest_port_id;
 	}
 
-	rte_smp_wmb();
+	rte_atomic_thread_fence(rte_memory_order_release);
 
 	dsw_port_drain_in_ring(source_port);
 	dsw_port_forward_emigrated_flows(dsw, source_port);
@@ -1213,7 +1213,7 @@ dsw_port_move_emigrating_flows(struct dsw_evdev *dsw,
 	/* Flow table update and migration destination port's enqueues
 	 * must be seen before the control message.
 	 */
-	rte_smp_wmb();
+	rte_atomic_thread_fence(rte_memory_order_release);
 
 	dsw_port_ctl_broadcast(dsw, source_port, DSW_CTL_UNPAUSE_REQ,
 			       source_port->emigration_target_qfs,
-- 
2.53.0

[RFC 25/32] event/opdl: replace SMP barriers with C11 atomics

[Stephen Hemminger]

Publishing the port becomes a release store of the configured flag;
the barrier after clearing started becomes a release fence. Both
are control path.

Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
 drivers/event/opdl/opdl_evdev.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/drivers/event/opdl/opdl_evdev.c b/drivers/event/opdl/opdl_evdev.c
index ffa65ef930..1450fa5ae8 100644
--- a/drivers/event/opdl/opdl_evdev.c
+++ b/drivers/event/opdl/opdl_evdev.c
@@ -200,8 +200,9 @@ opdl_port_setup(struct rte_eventdev *dev,
 	p->queue_id = OPDL_INVALID_QID;
 	p->external_qid = OPDL_INVALID_QID;
 	dev->data->ports[port_id] = p;
-	rte_smp_wmb();
-	p->configured = 1;
+	/* Release publishes port setup before the configured flag */
+	rte_atomic_store_explicit((uint8_t __rte_atomic *)&p->configured, 1,
+				  rte_memory_order_release);
 	device->nb_ports++;
 	return 0;
 }
@@ -477,7 +478,7 @@ opdl_stop(struct rte_eventdev *dev)
 
 	device->started = 0;
 
-	rte_smp_wmb();
+	rte_atomic_thread_fence(rte_memory_order_release);
 }
 
 static int
-- 
2.53.0

[RFC 26/32] net/netvsc: replace SMP barrier in RNDIS response

[Stephen Hemminger]

Publishing the copied response before clearing the pending id is
release ordering. The following cmpset is a full barrier today, but
keep the explicit fence so the ordering survives when cmpset is
converted to a release compare exchange.

Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
 drivers/net/netvsc/hn_rndis.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/net/netvsc/hn_rndis.c b/drivers/net/netvsc/hn_rndis.c
index 7c54eebcef..d1142fa1dd 100644
--- a/drivers/net/netvsc/hn_rndis.c
+++ b/drivers/net/netvsc/hn_rndis.c
@@ -357,7 +357,7 @@ void hn_rndis_receive_response(struct hn_data *hv,
 	memcpy(hv->rndis_resp, data, len);
 
 	/* make sure response copied before update */
-	rte_smp_wmb();
+	rte_atomic_thread_fence(rte_memory_order_release);
 
 	if (rte_atomic32_cmpset(&hv->rndis_pending, hdr->rid, 0) == 0) {
 		PMD_DRV_LOG(NOTICE,
-- 
2.53.0

[RFC 27/32] net/thunderx: replace deprecated SMP barriers

[Stephen Hemminger]

Redefine the nicvf barrier wrappers as C11 fences and add an IO
variant. The RBDR refill barrier orders descriptor writes in DMA
memory against the doorbell register write, which is device
ordering: use the IO barrier there. The mailbox flag barriers are
genuine inter thread ordering and become release/acquire fences.

Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
 drivers/net/thunderx/base/nicvf_hw.c   | 3 ++-
 drivers/net/thunderx/base/nicvf_plat.h | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/drivers/net/thunderx/base/nicvf_hw.c b/drivers/net/thunderx/base/nicvf_hw.c
index 5b1abe2015..d1e31ffdaa 100644
--- a/drivers/net/thunderx/base/nicvf_hw.c
+++ b/drivers/net/thunderx/base/nicvf_hw.c
@@ -499,7 +499,8 @@ nicvf_qset_rbdr_precharge(void *dev, struct nicvf *nic,
 			break;
 		}
 	}
-	nicvf_smp_wmb();
+	/* Descriptors must be visible to the device before the doorbell */
+	nicvf_io_wmb();
 	nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_DOOR, ridx, count);
 	rbdr->tail = nicvf_queue_reg_read(nic,
 				NIC_QSET_RBDR_0_1_TAIL, ridx) >> 3;
diff --git a/drivers/net/thunderx/base/nicvf_plat.h b/drivers/net/thunderx/base/nicvf_plat.h
index 7d22563035..08ebb0cf62 100644
--- a/drivers/net/thunderx/base/nicvf_plat.h
+++ b/drivers/net/thunderx/base/nicvf_plat.h
@@ -25,8 +25,9 @@
 
 /* barrier */
 #include <rte_atomic.h>
-#define nicvf_smp_wmb() rte_smp_wmb()
+#define nicvf_smp_wmb() rte_atomic_thread_fence(rte_memory_order_release)
 #define nicvf_smp_rmb() rte_atomic_thread_fence(rte_memory_order_acquire)
+#define nicvf_io_wmb() rte_io_wmb()
 
 /* utils */
 #include <rte_common.h>
-- 
2.53.0

[RFC 28/32] net/virtio: replace deprecated barrier in avail index update

[Stephen Hemminger]

The x86 special case keeps its shape: a release fence is the same
compiler barrier the SMP write barrier was, so the measured perf
reason for the branch is preserved.

Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
 drivers/net/virtio/virtqueue.h | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/drivers/net/virtio/virtqueue.h b/drivers/net/virtio/virtqueue.h
index 37fbd47d42..cd954108a8 100644
--- a/drivers/net/virtio/virtqueue.h
+++ b/drivers/net/virtio/virtqueue.h
@@ -474,14 +474,14 @@ static inline void
 vq_update_avail_idx(struct virtqueue *vq)
 {
 	if (vq->hw->weak_barriers) {
-	/* x86 prefers to using rte_smp_wmb over rte_atomic_store_explicit as
-	 * it reports a slightly better perf, which comes from the
-	 * saved branch by the compiler.
-	 * The if and else branches are identical with the smp and
-	 * io barriers both defined as compiler barriers on x86.
+	/* x86 prefers a fence plus plain store over
+	 * rte_atomic_store_explicit as it reports a slightly better
+	 * perf, which comes from the saved branch by the compiler.
+	 * The if and else branches are identical with the release
+	 * and io barriers both compiler barriers on x86.
 	 */
 #ifdef RTE_ARCH_X86_64
-		rte_smp_wmb();
+		rte_atomic_thread_fence(rte_memory_order_release);
 		vq->vq_split.ring.avail->idx = vq->vq_avail_idx;
 #else
 		rte_atomic_store_explicit(&vq->vq_split.ring.avail->idx,
-- 
2.53.0

[RFC 29/32] eal: remove stale SMP barrier in rte_service

[Stephen Hemminger]

Leftover missed by the C11 atomics conversion of this file. The
barrier pairs with nothing: no store follows it, and the next use
of the lcore is synchronized by remote launch.

Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
 lib/eal/common/rte_service.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/eal/common/rte_service.c b/lib/eal/common/rte_service.c
index d2ac9d3f14..0c6467285b 100644
--- a/lib/eal/common/rte_service.c
+++ b/lib/eal/common/rte_service.c
@@ -795,7 +795,6 @@ rte_service_lcore_del(uint32_t lcore)
 
 	set_lcore_state(lcore, ROLE_RTE);
 
-	rte_smp_wmb();
 	return 0;
 }
 
-- 
2.53.0

[RFC 30/32] eal: remove rte_smp_XX

[Stephen Hemminger]

These functions were marked for deprecation and no longer used.
Drop from EAL.

Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
 lib/eal/arm/include/rte_atomic_32.h    |  6 ---
 lib/eal/arm/include/rte_atomic_64.h    |  6 ---
 lib/eal/include/generic/rte_atomic.h   | 61 --------------------------
 lib/eal/loongarch/include/rte_atomic.h |  6 ---
 lib/eal/ppc/include/rte_atomic.h       |  6 ---
 lib/eal/riscv/include/rte_atomic.h     |  6 ---
 lib/eal/x86/include/rte_atomic.h       |  4 --
 7 files changed, 95 deletions(-)

diff --git a/lib/eal/arm/include/rte_atomic_32.h b/lib/eal/arm/include/rte_atomic_32.h
index 0b9a0dfa30..3809ddefb7 100644
--- a/lib/eal/arm/include/rte_atomic_32.h
+++ b/lib/eal/arm/include/rte_atomic_32.h
@@ -21,12 +21,6 @@ extern "C" {
 
 #define	rte_rmb() __sync_synchronize()
 
-#define rte_smp_mb() rte_mb()
-
-#define rte_smp_wmb() rte_wmb()
-
-#define rte_smp_rmb() rte_rmb()
-
 #define rte_io_mb() rte_mb()
 
 #define rte_io_wmb() rte_wmb()
diff --git a/lib/eal/arm/include/rte_atomic_64.h b/lib/eal/arm/include/rte_atomic_64.h
index 181bb60929..c9b41f6212 100644
--- a/lib/eal/arm/include/rte_atomic_64.h
+++ b/lib/eal/arm/include/rte_atomic_64.h
@@ -24,12 +24,6 @@ extern "C" {
 
 #define rte_rmb() asm volatile("dmb oshld" : : : "memory")
 
-#define rte_smp_mb() asm volatile("dmb ish" : : : "memory")
-
-#define rte_smp_wmb() asm volatile("dmb ishst" : : : "memory")
-
-#define rte_smp_rmb() asm volatile("dmb ishld" : : : "memory")
-
 #define rte_io_mb() rte_mb()
 
 #define rte_io_wmb() rte_wmb()
diff --git a/lib/eal/include/generic/rte_atomic.h b/lib/eal/include/generic/rte_atomic.h
index 0a4f3f8528..937d344c89 100644
--- a/lib/eal/include/generic/rte_atomic.h
+++ b/lib/eal/include/generic/rte_atomic.h
@@ -51,67 +51,6 @@ static inline void rte_wmb(void);
 static inline void rte_rmb(void);
 ///@}
 
-/** @name SMP Memory Barrier
- */
-///@{
-/**
- * General memory barrier between lcores
- *
- * Guarantees that the LOAD and STORE operations that precede the
- * rte_smp_mb() call are globally visible across the lcores
- * before the LOAD and STORE operations that follows it.
- *
- * @note
- *  This function is deprecated.
- *  It provides similar synchronization primitive as atomic fence,
- *  but has different syntax and memory ordering semantic. Hence
- *  deprecated for the simplicity of memory ordering semantics in use.
- *
- *  rte_atomic_thread_fence(rte_memory_order_acq_rel) should be used instead.
- */
-static inline void rte_smp_mb(void);
-
-/**
- * Write memory barrier between lcores
- *
- * Guarantees that the STORE operations that precede the
- * rte_smp_wmb() call are globally visible across the lcores
- * before the STORE operations that follows it.
- *
- * @note
- *  This function is deprecated.
- *  It provides similar synchronization primitive as atomic fence,
- *  but has different syntax and memory ordering semantic. Hence
- *  deprecated for the simplicity of memory ordering semantics in use.
- *
- *  rte_atomic_thread_fence(rte_memory_order_release) should be used instead.
- *  The fence also guarantees LOAD operations that precede the call
- *  are globally visible across the lcores before the STORE operations
- *  that follows it.
- */
-static inline void rte_smp_wmb(void);
-
-/**
- * Read memory barrier between lcores
- *
- * Guarantees that the LOAD operations that precede the
- * rte_smp_rmb() call are globally visible across the lcores
- * before the LOAD operations that follows it.
- *
- * @note
- *  This function is deprecated.
- *  It provides similar synchronization primitive as atomic fence,
- *  but has different syntax and memory ordering semantic. Hence
- *  deprecated for the simplicity of memory ordering semantics in use.
- *
- *  rte_atomic_thread_fence(rte_memory_order_acquire) should be used instead.
- *  The fence also guarantees LOAD operations that precede the call
- *  are globally visible across the lcores before the STORE operations
- *  that follows it.
- */
-static inline void rte_smp_rmb(void);
-///@}
-
 /** @name I/O Memory Barrier
  */
 ///@{
diff --git a/lib/eal/loongarch/include/rte_atomic.h b/lib/eal/loongarch/include/rte_atomic.h
index c8066a4612..49e0c67020 100644
--- a/lib/eal/loongarch/include/rte_atomic.h
+++ b/lib/eal/loongarch/include/rte_atomic.h
@@ -22,12 +22,6 @@ extern "C" {
 
 #define rte_rmb()	rte_mb()
 
-#define rte_smp_mb()	rte_mb()
-
-#define rte_smp_wmb()	rte_mb()
-
-#define rte_smp_rmb()	rte_mb()
-
 #define rte_io_mb()	rte_mb()
 
 #define rte_io_wmb()	rte_mb()
diff --git a/lib/eal/ppc/include/rte_atomic.h b/lib/eal/ppc/include/rte_atomic.h
index 10acc238f9..1da5afccbf 100644
--- a/lib/eal/ppc/include/rte_atomic.h
+++ b/lib/eal/ppc/include/rte_atomic.h
@@ -24,12 +24,6 @@ extern "C" {
 
 #define	rte_rmb() asm volatile("sync" : : : "memory")
 
-#define rte_smp_mb() rte_mb()
-
-#define rte_smp_wmb() rte_wmb()
-
-#define rte_smp_rmb() rte_rmb()
-
 #define rte_io_mb() rte_mb()
 
 #define rte_io_wmb() rte_wmb()
diff --git a/lib/eal/riscv/include/rte_atomic.h b/lib/eal/riscv/include/rte_atomic.h
index 66346ad474..dd10ad5127 100644
--- a/lib/eal/riscv/include/rte_atomic.h
+++ b/lib/eal/riscv/include/rte_atomic.h
@@ -27,12 +27,6 @@ extern "C" {
 
 #define rte_rmb()	asm volatile("fence r, r" : : : "memory")
 
-#define rte_smp_mb()	rte_mb()
-
-#define rte_smp_wmb()	rte_wmb()
-
-#define rte_smp_rmb()	rte_rmb()
-
 #define rte_io_mb()	asm volatile("fence iorw, iorw" : : : "memory")
 
 #define rte_io_wmb()	asm volatile("fence orw, ow" : : : "memory")
diff --git a/lib/eal/x86/include/rte_atomic.h b/lib/eal/x86/include/rte_atomic.h
index 780fdce871..826b5033d1 100644
--- a/lib/eal/x86/include/rte_atomic.h
+++ b/lib/eal/x86/include/rte_atomic.h
@@ -23,10 +23,6 @@
 
 #define	rte_rmb() _mm_lfence()
 
-#define rte_smp_wmb() rte_compiler_barrier()
-
-#define rte_smp_rmb() rte_compiler_barrier()
-
 #ifdef __cplusplus
 extern "C" {
 #endif
-- 
2.53.0

[RFC 31/32] checkpatches: no longer warn about rte_smp_XX

[Stephen Hemminger]

These macros/functions are now removed.
Any new use will be detected as compile errors.

Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
 devtools/checkpatches.sh | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/devtools/checkpatches.sh b/devtools/checkpatches.sh
index 18bd825372..3649645488 100755
--- a/devtools/checkpatches.sh
+++ b/devtools/checkpatches.sh
@@ -129,14 +129,6 @@ check_forbidden_additions() { # <patch>
 		-f $(dirname $(readlink -f $0))/check-forbidden-tokens.awk \
 		"$1" || res=1
 
-	# refrain from new additions of rte_smp_[r/w]mb()
-	awk -v FOLDERS="lib drivers app examples" \
-		-v EXPRESSIONS="rte_smp_(r|w)?mb\\\(" \
-		-v RET_ON_FAIL=1 \
-		-v MESSAGE='Using rte_smp_[r/w]mb' \
-		-f $(dirname $(readlink -f $0))/check-forbidden-tokens.awk \
-		"$1" || res=1
-
 	# refrain from using compiler __sync_xxx builtins
 	awk -v FOLDERS="lib drivers app examples" \
 		-v EXPRESSIONS="__sync_.*\\\(" \
-- 
2.53.0

[RFC 32/32] doc: update release notes about rte_smp_XX removal

[Stephen Hemminger]

The functions rte_smp_XX were previously marked for removal.
The removal is now complete across the tree.

Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
 doc/guides/rel_notes/deprecation.rst   | 7 -------
 doc/guides/rel_notes/release_26_11.rst | 6 ++++++
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/doc/guides/rel_notes/deprecation.rst b/doc/guides/rel_notes/deprecation.rst
index 6ad7698c6b..b310801b84 100644
--- a/doc/guides/rel_notes/deprecation.rst
+++ b/doc/guides/rel_notes/deprecation.rst
@@ -46,13 +46,6 @@ Deprecation Notices
   operations must be used for patches that need to be merged in 20.08 onwards.
   This change will not introduce any performance degradation.
 
-* rte_smp_*mb: These APIs provide full barrier functionality. However, many
-  use cases do not require full barriers. To support such use cases, DPDK has
-  adopted atomic operations from
-  https://gcc.gnu.org/onlinedocs/gcc/_005f_005fatomic-Builtins.html. These
-  operations and a new wrapper ``rte_atomic_thread_fence`` instead of
-  ``__atomic_thread_fence`` must be used for patches that need to be merged in
-  20.08 onwards. This change will not introduce any performance degradation.
 
 * mempool: The ``flushthresh`` field in ``struct rte_mempool_cache`` is obsolete,
   and will be removed in DPDK 26.11.
diff --git a/doc/guides/rel_notes/release_26_11.rst b/doc/guides/rel_notes/release_26_11.rst
index c8cc86295d..d9ebbac77b 100644
--- a/doc/guides/rel_notes/release_26_11.rst
+++ b/doc/guides/rel_notes/release_26_11.rst
@@ -77,6 +77,12 @@ Removed Items
     ``rte_rib6_is_equal``
   * table: ``RTE_LPM_IPV6_ADDR_SIZE``
 
+* Removed SMP memory barriers:
+
+  * ``rte_smp_mb()`` replaced with ``rte_atomic_thread_fence(rte_memory_order_seq_cst)``
+  * ``rte_smp_rmb()`` replaced with ``rte_atomic_thread_fence(rte_memory_order_acquire)``
+  * ``rte_smp_wmb()`` revised code to use ``rte_atomic_load/store``
+
 
 API Changes
 -----------
-- 
2.53.0

RE: [RFC 02/32] test: remove test for rte_smp_mb

[Konstantin Ananyev]

> The DPDK version of SMP barriers is deprecated and
> being removed. Drop the test as no longer relevant.

Why to remove the whole test? 
I believe it is still useful for testing our custom x86 implementation
of rte_atomic_thread_fence(rte_memory_order_seq_cst).
See patch 17 in this series:
[RFC 17/32] eal/x86: move optimized fence out of SMP barrier
Instead of removing it we can replace rte_smp_mb() with
rte_atomic_thread_fence(rte_memory_order_seq_cst)
In one function below, i.e.: 

static inline void
store_load_barrier(uint32_t utype)
{
	if (utype == USE_MB)
		rte_mb();
	else if (utype == USE_SMP_MB)
 -		rte_smp_mb();
+		rte_atomic_thread_fence(rte_memory_order_seq_cst);
	else
		RTE_VERIFY(0);
}

> Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
> ---
>  app/test/meson.build    |   1 -
>  app/test/test_barrier.c | 288 ----------------------------------------
>  2 files changed, 289 deletions(-)
>  delete mode 100644 app/test/test_barrier.c
> 
> diff --git a/app/test/meson.build b/app/test/meson.build
> index 51abeeb732..6df9dfa222 100644
> --- a/app/test/meson.build
> +++ b/app/test/meson.build
> @@ -28,7 +28,6 @@ source_file_deps = {
>      'test_alarm.c': [],
>      'test_argparse.c': ['argparse'],
>      'test_atomic.c': ['hash'],
> -    'test_barrier.c': [],
>      'test_bitcount.c': [],
>      'test_bitmap.c': [],
>      'test_bitops.c': [],
> diff --git a/app/test/test_barrier.c b/app/test/test_barrier.c
> deleted file mode 100644
> index 925a88b68a..0000000000
> --- a/app/test/test_barrier.c
> +++ /dev/null
> @@ -1,288 +0,0 @@
> -/* SPDX-License-Identifier: BSD-3-Clause
> - * Copyright(c) 2010-2018 Intel Corporation
> - */
> -
> - /*
> -  * This is a simple functional test for rte_smp_mb() implementation.
> -  * I.E. make sure that LOAD and STORE operations that precede the
> -  * rte_smp_mb() call are globally visible across the lcores
> -  * before the LOAD and STORE operations that follows it.
> -  * The test uses simple implementation of Peterson's lock algorithm
> -  * (https://en.wikipedia.org/wiki/Peterson%27s_algorithm)
> -  * for two execution units to make sure that rte_smp_mb() prevents
> -  * store-load reordering to happen.
> -  * Also when executed on a single lcore could be used as a approximate
> -  * estimation of number of cycles particular implementation of rte_smp_mb()
> -  * will take.
> -  */
> -
> -#include <stdio.h>
> -#include <string.h>
> -#include <stdint.h>
> -#include <inttypes.h>
> -
> -#include <rte_memory.h>
> -#include <rte_per_lcore.h>
> -#include <rte_launch.h>
> -#include <rte_eal.h>
> -#include <rte_lcore.h>
> -#include <rte_pause.h>
> -#include <rte_random.h>
> -#include <rte_cycles.h>
> -#include <rte_vect.h>
> -#include <rte_debug.h>
> -
> -#include "test.h"
> -
> -#define ADD_MAX		8
> -#define ITER_MAX	0x1000000
> -
> -enum plock_use_type {
> -	USE_MB,
> -	USE_SMP_MB,
> -	USE_NUM
> -};
> -
> -struct plock {
> -	volatile uint32_t flag[2];
> -	volatile uint32_t victim;
> -	enum plock_use_type utype;
> -};
> -
> -/*
> - * Lock plus protected by it two counters.
> - */
> -struct plock_test {
> -	struct plock lock;
> -	uint64_t val;
> -	uint64_t iter;
> -};
> -
> -/*
> - * Each active lcore shares plock_test struct with it's left and right
> - * neighbours.
> - */
> -struct lcore_plock_test {
> -	struct plock_test *pt[2]; /* shared, lock-protected data */
> -	uint64_t sum[2];          /* local copy of the shared data */
> -	uint64_t iter;            /* number of iterations to perform */
> -	uint32_t lc;              /* given lcore id */
> -};
> -
> -static inline void
> -store_load_barrier(uint32_t utype)
> -{
> -	if (utype == USE_MB)
> -		rte_mb();
> -	else if (utype == USE_SMP_MB)
> -		rte_smp_mb();
> -	else
> -		RTE_VERIFY(0);
> -}
> -
> -/*
> - * Peterson lock implementation.
> - */
> -static void
> -plock_lock(struct plock *l, uint32_t self)
> -{
> -	uint32_t other;
> -
> -	other = self ^ 1;
> -
> -	l->flag[self] = 1;
> -	rte_smp_wmb();
> -	l->victim = self;
> -
> -	store_load_barrier(l->utype);
> -
> -	while (l->flag[other] == 1 && l->victim == self)
> -		rte_pause();
> -	rte_smp_rmb();
> -}
> -
> -static void
> -plock_unlock(struct plock *l, uint32_t self)
> -{
> -	rte_smp_wmb();
> -	l->flag[self] = 0;
> -}
> -
> -static void
> -plock_reset(struct plock *l, enum plock_use_type utype)
> -{
> -	memset(l, 0, sizeof(*l));
> -	l->utype = utype;
> -}
> -
> -/*
> - * grab the lock, update both counters, release the lock.
> - */
> -static void
> -plock_add(struct plock_test *pt, uint32_t self, uint32_t n)
> -{
> -	plock_lock(&pt->lock, self);
> -	pt->iter++;
> -	pt->val += n;
> -	plock_unlock(&pt->lock, self);
> -}
> -
> -static int
> -plock_test1_lcore(void *data)
> -{
> -	uint64_t tm;
> -	uint32_t lc, ln;
> -	uint64_t i, n;
> -	struct lcore_plock_test *lpt;
> -
> -	lpt = data;
> -	lc = rte_lcore_id();
> -
> -	/* find lcore_plock_test struct for given lcore */
> -	for (ln = rte_lcore_count(); ln != 0 && lpt->lc != lc; lpt++, ln--)
> -		;
> -
> -	if (ln == 0) {
> -		printf("%s(%u) error at init\n", __func__, lc);
> -		return -1;
> -	}
> -
> -	n = rte_rand() % ADD_MAX;
> -	tm = rte_get_timer_cycles();
> -
> -	/*
> -	 * for each iteration:
> -	 * - update shared, locked protected data in a safe manner
> -	 * - update local copy of the shared data
> -	 */
> -	for (i = 0; i != lpt->iter; i++) {
> -
> -		plock_add(lpt->pt[0], 0, n);
> -		plock_add(lpt->pt[1], 1, n);
> -
> -		lpt->sum[0] += n;
> -		lpt->sum[1] += n;
> -
> -		n = (n + 1) % ADD_MAX;
> -	}
> -
> -	tm = rte_get_timer_cycles() - tm;
> -
> -	printf("%s(%u): %" PRIu64 " iterations finished, in %" PRIu64
> -		" cycles, %#Lf cycles/iteration, "
> -		"local sum={%" PRIu64 ", %" PRIu64 "}\n",
> -		__func__, lc, i, tm, (long double)tm / i,
> -		lpt->sum[0], lpt->sum[1]);
> -	return 0;
> -}
> -
> -/*
> - * For N active lcores we allocate N+1 lcore_plock_test structures.
> - * Each active lcore shares one lcore_plock_test structure with its
> - * left lcore neighbor and one lcore_plock_test structure with its
> - * right lcore neighbor.
> - * During the test each lcore updates data in both shared structures and
> - * its local copies. Then at validation phase we check that our shared
> - * and local data are the same.
> - */
> -static int
> -plock_test(uint64_t iter, enum plock_use_type utype)
> -{
> -	int32_t rc;
> -	uint32_t i, lc, n;
> -	uint64_t *sum;
> -	struct plock_test *pt;
> -	struct lcore_plock_test *lpt;
> -
> -	/* init phase, allocate and initialize shared data */
> -
> -	n = rte_lcore_count();
> -	pt = calloc(n + 1, sizeof(*pt));
> -	lpt = calloc(n, sizeof(*lpt));
> -	sum = calloc(n + 1, sizeof(*sum));
> -
> -	printf("%s(iter=%" PRIu64 ", utype=%u) started on %u lcores\n",
> -		__func__, iter, utype, n);
> -
> -	if (pt == NULL || lpt == NULL || sum == NULL) {
> -		printf("%s: failed to allocate memory for %u lcores\n",
> -			__func__, n);
> -		free(pt);
> -		free(lpt);
> -		free(sum);
> -		return -ENOMEM;
> -	}
> -
> -	for (i = 0; i != n + 1; i++)
> -		plock_reset(&pt[i].lock, utype);
> -
> -	i = 0;
> -	RTE_LCORE_FOREACH(lc) {
> -
> -		lpt[i].lc = lc;
> -		lpt[i].iter = iter;
> -		lpt[i].pt[0] = pt + i;
> -		lpt[i].pt[1] = pt + i + 1;
> -		i++;
> -	}
> -
> -	lpt[i - 1].pt[1] = pt;
> -
> -	for (i = 0; i != n; i++)
> -		printf("lpt[%u]={lc=%u, pt={%p, %p},};\n",
> -			i, lpt[i].lc, lpt[i].pt[0], lpt[i].pt[1]);
> -
> -
> -	/* test phase - start and wait for completion on each active lcore */
> -
> -	rte_eal_mp_remote_launch(plock_test1_lcore, lpt, CALL_MAIN);
> -	rte_eal_mp_wait_lcore();
> -
> -	/* validation phase - make sure that shared and local data match */
> -
> -	for (i = 0; i != n; i++) {
> -		sum[i] += lpt[i].sum[0];
> -		sum[i + 1] += lpt[i].sum[1];
> -	}
> -
> -	sum[0] += sum[i];
> -
> -	rc = 0;
> -	for (i = 0; i != n; i++) {
> -		printf("%s: sum[%u]=%" PRIu64 ", pt[%u].val=%" PRIu64 ",
> pt[%u].iter=%" PRIu64 ";\n",
> -			__func__, i, sum[i], i, pt[i].val, i, pt[i].iter);
> -
> -		/* race condition occurred, lock doesn't work properly */
> -		if (sum[i] != pt[i].val || 2 * iter != pt[i].iter) {
> -			printf("error: local and shared sums don't match\n");
> -			rc = -1;
> -		}
> -	}
> -
> -	free(pt);
> -	free(lpt);
> -	free(sum);
> -
> -	printf("%s(utype=%u) returns %d\n", __func__, utype, rc);
> -	return rc;
> -}
> -
> -static int
> -test_barrier(void)
> -{
> -	int32_t i, ret, rc[USE_NUM];
> -
> -	for (i = 0; i != RTE_DIM(rc); i++)
> -		rc[i] = plock_test(ITER_MAX, i);
> -
> -	ret = 0;
> -	for (i = 0; i != RTE_DIM(rc); i++) {
> -		printf("%s for utype=%d %s\n",
> -			__func__, i, rc[i] == 0 ? "passed" : "failed");
> -		ret |= rc[i];
> -	}
> -
> -	return ret;
> -}
> -
> -REGISTER_PERF_TEST(barrier_autotest, test_barrier);
> --
> 2.53.0

RE: [RFC 17/32] eal/x86: move optimized fence out of SMP barrier

[Konstantin Ananyev]

> rte_atomic_thread_fence() implemented the seq_cst case by calling the
> deprecated rte_smp_mb(). Invert the dependency: the lock add based
> fence moves into rte_atomic_thread_fence() and rte_smp_mb() becomes a
> wrapper around it, so removing the deprecated barriers later is a pure
> deletion. The optimization itself must stay; a plain seq_cst fence is
> an mfence, about twice the cost. No change in generated code.
> 
> Drop no longer used rte_smp_mb().
> 
> Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
> ---
>  lib/eal/x86/include/rte_atomic.h | 37 ++++++++++++++------------------
>  1 file changed, 16 insertions(+), 21 deletions(-)
> 
> diff --git a/lib/eal/x86/include/rte_atomic.h b/lib/eal/x86/include/rte_atomic.h
> index e071e4234e..780fdce871 100644
> --- a/lib/eal/x86/include/rte_atomic.h
> +++ b/lib/eal/x86/include/rte_atomic.h
> @@ -60,23 +60,8 @@ extern "C" {
>   * Basic idea is to use lock prefixed add with some dummy memory location
>   * as the destination. From their experiments 128B(2 cache lines) below
>   * current stack pointer looks like a good candidate.
> - * So below we use that technique for rte_smp_mb() implementation.
>   */
> 
> -static __rte_always_inline void
> -rte_smp_mb(void)
> -{
> -#ifdef RTE_TOOLCHAIN_MSVC
> -	_mm_mfence();
> -#else
> -#ifdef RTE_ARCH_I686
> -	asm volatile("lock addl $0, -128(%%esp); " ::: "memory");
> -#else
> -	asm volatile("lock addl $0, -128(%%rsp); " ::: "memory");
> -#endif
> -#endif
> -}
> -
>  #define rte_io_mb() rte_mb()
> 
>  #define rte_io_wmb() rte_compiler_barrier()
> @@ -86,17 +71,27 @@ rte_smp_mb(void)
>  /**
>   * Synchronization fence between threads based on the specified memory order.
>   *
> - * On x86 the __rte_atomic_thread_fence(rte_memory_order_seq_cst)
> generates full 'mfence'
> - * which is quite expensive. The optimized implementation of rte_smp_mb is
> - * used instead.
> + * On x86 the __rte_atomic_thread_fence(rte_memory_order_seq_cst)
> generates
> + * a full 'mfence' which is quite expensive. The optimized lock add on a
> + * dummy stack location (see above) is used instead.
>   */
>  static __rte_always_inline void
>  rte_atomic_thread_fence(rte_memory_order memorder)
>  {
> -	if (memorder == rte_memory_order_seq_cst)
> -		rte_smp_mb();
> -	else
> +	if (memorder != rte_memory_order_seq_cst) {
>  		__rte_atomic_thread_fence(memorder);
> +		return;
> +	}
> +
> +#ifdef RTE_TOOLCHAIN_MSVC
> +	_mm_mfence();
> +#else
> +#ifdef RTE_ARCH_I686
> +	asm volatile("lock addl $0, -128(%%esp); " ::: "memory");
> +#else
> +	asm volatile("lock addl $0, -128(%%rsp); " ::: "memory");
> +#endif
> +#endif
>  }
> 
>  #ifdef __cplusplus
> --

Acked-by: Konstantin Ananyev <konstantin.ananyev@huawei.com>

> 2.53.0

RE: [RFC 13/32] ring: replace SMP read barrier with C11 acquire fence

[Konstantin Ananyev]

> rte_smp_rmb() is deprecated. The acquire fence generates the same
> code everywhere the gcc implementation is used: a compiler barrier
> on x86 and dmb ishld on ThunderX.
> 
> The gcc implementation itself is kept: unlike the lock-free stack,
> the in-tree comment records a 10% x86 performance drop with the
> C11 version, which needs re-measuring with current compilers
> before RTE_USE_C11_MEM_MODEL can be removed.
> 
> Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
> ---
>  lib/ring/rte_ring_gcc_pvt.h | 6 +++---
>  1 file changed, 3 insertions(+), 3 deletions(-)
> 
> diff --git a/lib/ring/rte_ring_gcc_pvt.h b/lib/ring/rte_ring_gcc_pvt.h
> index 340ece28c7..9c2c58824d 100644
> --- a/lib/ring/rte_ring_gcc_pvt.h
> +++ b/lib/ring/rte_ring_gcc_pvt.h
> @@ -55,10 +55,10 @@ __rte_ring_headtail_move_head_st(struct
> rte_ring_headtail *d,
> 
>  	*old_head = d->head;
> 
> -	/* add rmb barrier to avoid load/load reorder in weak
> -	 * memory model. It is noop on x86
> +	/* Acquire fence to avoid load/load reorder in weak
> +	 * memory model. It is noop on x86.
>  	 */
> -	rte_smp_rmb();
> +	rte_atomic_thread_fence(rte_memory_order_acquire);
> 
>  	/*
>  	 *  The subtraction is done between two unsigned 32bits value
> --

Acked-by: Konstantin Ananyev <konstantin.ananyev@huawei.com>

As a side note - do we really need two versions of __rte_ring_headtail_move_head_st()?
Might be just keep 2 versions of _mt, while _st can be merged into one?

> 2.53.0

RE: [RFC 01/32] bpf: replace deprecated SMP barriers with C11 fences

[Marat Khalili]

> diff --git a/lib/bpf/bpf_pkt.c b/lib/bpf/bpf_pkt.c
> index f072fdaaed..a831b5ad86 100644
> --- a/lib/bpf/bpf_pkt.c
> +++ b/lib/bpf/bpf_pkt.c
> @@ -80,9 +80,11 @@ static struct bpf_eth_cbh tx_cbh = {
>  static __rte_always_inline void
>  bpf_eth_cbi_inuse(struct bpf_eth_cbi *cbi)
>  {
> -	cbi->use++;
> +	rte_atomic_store_explicit(&cbi->use,
> +		rte_atomic_load_explicit(&cbi->use, rte_memory_order_relaxed) + 1,
> +		rte_memory_order_relaxed);
>  	/* make sure no store/load reordering could happen */
> -	rte_smp_mb();
> +	rte_atomic_thread_fence(rte_memory_order_seq_cst);
>  }

I think this needs to be analyzed further by AI to see if a more canonical way
of doing things exists now. To me the whole construct above looks very strange.

> 
>  /*
> @@ -92,8 +94,10 @@ static __rte_always_inline void
>  bpf_eth_cbi_unuse(struct bpf_eth_cbi *cbi)
>  {
>  	/* make sure all previous loads are completed */
> -	rte_smp_rmb();
> -	cbi->use++;
> +	rte_atomic_thread_fence(rte_memory_order_acquire);
> +	rte_atomic_store_explicit(&cbi->use,
> +		rte_atomic_load_explicit(&cbi->use, rte_memory_order_relaxed) + 1,
> +		rte_memory_order_relaxed);
>  }
> 
>  /*
> @@ -105,9 +109,9 @@ bpf_eth_cbi_wait(const struct bpf_eth_cbi *cbi)
>  	uint32_t puse;
> 
>  	/* make sure all previous loads and stores are completed */
> -	rte_smp_mb();
> +	rte_atomic_thread_fence(rte_memory_order_seq_cst);
> 
> -	puse = cbi->use;
> +	puse = rte_atomic_load_explicit(&cbi->use, rte_memory_order_relaxed);
> 
>  	/* in use, busy wait till current RX/TX iteration is finished */
>  	if ((puse & BPF_ETH_CBI_INUSE) != 0) {
> @@ -439,7 +443,7 @@ bpf_eth_cbi_unload(struct bpf_eth_cbi *bc)
>  {
>  	/* mark this cbi as empty */
>  	bc->cb = NULL;
> -	rte_smp_mb();
> +	rte_atomic_thread_fence(rte_memory_order_seq_cst);
> 
>  	/* make sure datapath doesn't use bpf anymore, then destroy bpf */
>  	bpf_eth_cbi_wait(bc);
> --
> 2.53.0