* [PATCH net-next] sctp: replace cb->args[4] with a local variable in sctp_diag_dump()
From: Xin Long @ 2026-07-09 21:28 UTC (permalink / raw)
To: network dev, linux-sctp
Cc: davem, kuba, Eric Dumazet, Paolo Abeni, Simon Horman,
Marcelo Ricardo Leitner
cb->args[4] is currently used as a temporary counter to track the assoc
index while traversing ep->asocs in sctp_sock_dump(). However, this
state is only needed locally within a single dump iteration and does not
need to be preserved across callbacks.
Replace cb->args[4] with a local idx variable in sctp_sock_dump(), and
update cb->args[1] directly when the dump is interrupted due to skb
space exhaustion.
This simplifies the dump state handling and removes the dependency on an
extra cb->args[] slot, making the traversal logic easier to follow.
Signed-off-by: Xin Long <lucien.xin@gmail.com>
---
net/sctp/diag.c | 45 ++++++++++++++++++++++-----------------------
1 file changed, 22 insertions(+), 23 deletions(-)
diff --git a/net/sctp/diag.c b/net/sctp/diag.c
index c2a0de2adf6f..a9bb31303613 100644
--- a/net/sctp/diag.c
+++ b/net/sctp/diag.c
@@ -299,18 +299,21 @@ static int sctp_sock_dump_one(struct sctp_endpoint *ep, struct sctp_transport *t
static int sctp_sock_dump(struct sctp_endpoint *ep, struct sctp_transport *tsp, void *p)
{
struct sctp_comm_param *commp = p;
- struct sock *sk = ep->base.sk;
+ struct sock *sk = ep->base.sk, *nsk;
struct sk_buff *skb = commp->skb;
struct netlink_callback *cb = commp->cb;
const struct inet_diag_req_v2 *r = commp->r;
+ u32 portid = NETLINK_CB(cb->skb).portid;
struct sctp_association *assoc;
- int err = 0;
+ int err = 0, idx = 0;
+
+ nsk = NETLINK_CB(cb->skb).sk;
lock_sock(sk);
if (ep != tsp->asoc->ep)
goto release;
list_for_each_entry(assoc, &ep->asocs, asocs) {
- if (cb->args[4] < cb->args[1])
+ if (idx < cb->args[1])
goto next;
if (r->id.idiag_sport != htons(assoc->base.bind_addr.port) &&
@@ -320,32 +323,31 @@ static int sctp_sock_dump(struct sctp_endpoint *ep, struct sctp_transport *tsp,
r->id.idiag_dport)
goto next;
- if (!cb->args[3] &&
- inet_sctp_diag_fill(sk, NULL, skb, r,
- sk_user_ns(NETLINK_CB(cb->skb).sk),
- NETLINK_CB(cb->skb).portid,
- cb->nlh->nlmsg_seq,
- NLM_F_MULTI, cb->nlh,
- commp->net_admin) < 0) {
- err = 1;
- goto release;
+ if (!cb->args[3]) {
+ err = inet_sctp_diag_fill(sk, NULL, skb, r,
+ sk_user_ns(nsk), portid,
+ cb->nlh->nlmsg_seq,
+ NLM_F_MULTI, cb->nlh,
+ commp->net_admin);
+ if (err < 0) {
+ cb->args[1] = idx;
+ goto release;
+ }
}
cb->args[3] = 1;
- if (inet_sctp_diag_fill(sk, assoc, skb, r,
- sk_user_ns(NETLINK_CB(cb->skb).sk),
- NETLINK_CB(cb->skb).portid,
- cb->nlh->nlmsg_seq, 0, cb->nlh,
- commp->net_admin) < 0) {
- err = 1;
+ err = inet_sctp_diag_fill(sk, assoc, skb, r, sk_user_ns(nsk),
+ portid, cb->nlh->nlmsg_seq, 0,
+ cb->nlh, commp->net_admin);
+ if (err < 0) {
+ cb->args[1] = idx;
goto release;
}
next:
- cb->args[4]++;
+ idx++;
}
cb->args[1] = 0;
cb->args[3] = 0;
- cb->args[4] = 0;
release:
release_sock(sk);
return err;
@@ -505,14 +507,11 @@ static void sctp_diag_dump(struct sk_buff *skb, struct netlink_callback *cb,
* 1 : to record the assoc pos of this time's traversal
* 2 : to record the transport pos of this time's traversal
* 3 : to mark if we have dumped the ep info of the current asoc
- * 4 : to track position within ep->asocs list in sctp_sock_dump()
*/
pos = cb->args[2];
sctp_transport_traverse_process(sctp_sock_filter, sctp_sock_dump,
net, &pos, &commp);
cb->args[2] = pos;
- cb->args[1] = cb->args[4];
- cb->args[4] = 0;
}
static const struct inet_diag_handler sctp_diag_handler = {
--
2.47.1
^ permalink raw reply related
* [PATCH net v2] gve: fix Rx queue stall on alloc failure
From: Harshitha Ramamurthy @ 2026-07-09 21:19 UTC (permalink / raw)
To: netdev
Cc: joshwash, hramamurthy, andrew+netdev, davem, edumazet, kuba,
pabeni, willemb, jordanrhee, nktgrg, maolson, thostet, csully,
bcf, maciej.fijalkowski, linux-kernel, stable, Eddie Phillips
From: Eddie Phillips <eddiephillips@google.com>
When the system is under extreme memory pressure, page allocations can
fail during the Rx buffer refill loop. If the number of buffers posted
to hardware falls below a critical low threshold and the refill loop
exits due to allocation failures, the queue can stall:
1. The device drops incoming packets because there are no descriptors.
2. Since no packets are processed, no Rx completions are generated.
3. Because no completions occur, NAPI is never scheduled, preventing
the refill loop from running again even after memory is freed.
This results in a permanent queue stall.
Resolve this by introducing a starvation recovery timer for each Rx queue.
If the number of buffers posted to hardware falls below a critical low
threshold, start a timer to periodically reschedule NAPI. Once NAPI runs
and successfully refills the queue above the threshold, the timer is
not rescheduled.
The threshold is set to 32 because a single maximum-sized Receive Segment
Coalescing (RSC) packet can consume up to 19 descriptors in the Rx path.
Lower thresholds (such as 8 or 16) would be insufficient to process a
complete maximum-sized RSC packet, risking packet drops or unexpected
hardware behavior under memory pressure. Setting the threshold to 32
guarantees a safe margin to handle at least one full RSC packet.
Cc: stable@vger.kernel.org
Fixes: 9b8dd5e5ea48 ("gve: DQO: Add RX path")
Reviewed-by: Jordan Rhee <jordanrhee@google.com>
Signed-off-by: Eddie Phillips <eddiephillips@google.com>
Signed-off-by: Harshitha Ramamurthy <hramamurthy@google.com>
---
Changes in v2:
- Link to v1: https://lore.kernel.org/netdev/20260701005341.3699161-1-hramamurthy@google.com/
- Relocated the starvation timer to the end of gve_rx_ring to avoid polluting
hotpath cachelines
- Decoupled timer lifecycle from allocation cycles by moving initialization
and shutdown to start/stop pathways instead of setup/remove pathways.
- Added explicit rationale for the 32-descriptor threshold
(GVE_RX_BUF_THRESH_DQO) ensuring it is safe for maximum-sized RSC packets.
- Removed addition of a stat tracking critical low buffer events
drivers/net/ethernet/google/gve/gve.h | 3 +++
drivers/net/ethernet/google/gve/gve_rx_dqo.c | 34 ++++++++++++++++++++++++++++++++++
2 files changed, 37 insertions(+)
diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
index 2f7bd330..bdd53d08 100644
--- a/drivers/net/ethernet/google/gve/gve.h
+++ b/drivers/net/ethernet/google/gve/gve.h
@@ -13,6 +13,7 @@
#include <linux/netdevice.h>
#include <linux/net_tstamp.h>
#include <linux/pci.h>
+#include <linux/timer.h>
#include <linux/ptp_clock_kernel.h>
#include <linux/u64_stats_sync.h>
#include <net/page_pool/helpers.h>
@@ -41,6 +42,7 @@
/* Interval to schedule a stats report update, 20000ms. */
#define GVE_STATS_REPORT_TIMER_PERIOD 20000
+#define GVE_RX_NAPI_RESCHED_MS 20 /* msecs */
/* Numbers of NIC tx/rx stats in stats report. */
#define NIC_TX_STATS_REPORT_NUM 0
@@ -341,6 +343,7 @@ struct gve_rx_ring {
struct xdp_rxq_info xdp_rxq;
struct xsk_buff_pool *xsk_pool;
struct page_frag_cache page_cache; /* Page cache to allocate XDP frames */
+ struct timer_list starvation_timer; /* for queue starvation recovery */
};
/* A TX desc ring entry */
diff --git a/drivers/net/ethernet/google/gve/gve_rx_dqo.c b/drivers/net/ethernet/google/gve/gve_rx_dqo.c
index 02cba280..8271f731 100644
--- a/drivers/net/ethernet/google/gve/gve_rx_dqo.c
+++ b/drivers/net/ethernet/google/gve/gve_rx_dqo.c
@@ -18,6 +18,16 @@
#include <net/tcp.h>
#include <net/xdp_sock_drv.h>
+static void gve_rx_starvation_timer(struct timer_list *t)
+{
+ struct gve_rx_ring *rx = timer_container_of(rx, t, starvation_timer);
+ struct gve_priv *priv = rx->gve;
+ struct gve_notify_block *block;
+
+ block = &priv->ntfy_blocks[rx->ntfy_id];
+ napi_schedule(&block->napi);
+}
+
static void gve_rx_free_hdr_bufs(struct gve_priv *priv, struct gve_rx_ring *rx)
{
struct device *hdev = &priv->pdev->dev;
@@ -120,6 +130,7 @@ void gve_rx_stop_ring_dqo(struct gve_priv *priv, int idx)
if (rx->dqo.page_pool)
page_pool_disable_direct_recycling(rx->dqo.page_pool);
+ timer_shutdown_sync(&rx->starvation_timer);
gve_remove_napi(priv, ntfy_idx);
gve_rx_remove_from_block(priv, idx);
gve_rx_reset_ring_dqo(priv, idx);
@@ -208,8 +219,10 @@ static int gve_rx_alloc_hdr_bufs(struct gve_priv *priv, struct gve_rx_ring *rx,
void gve_rx_start_ring_dqo(struct gve_priv *priv, int idx)
{
int ntfy_idx = gve_rx_idx_to_ntfy(priv, idx);
+ struct gve_rx_ring *rx = &priv->rx[idx];
gve_rx_add_to_block(priv, idx);
+ timer_setup(&rx->starvation_timer, gve_rx_starvation_timer, 0);
gve_add_napi(priv, ntfy_idx, gve_napi_poll_dqo);
}
@@ -365,6 +378,7 @@ void gve_rx_post_buffers_dqo(struct gve_rx_ring *rx)
struct gve_rx_compl_queue_dqo *complq = &rx->dqo.complq;
struct gve_rx_buf_queue_dqo *bufq = &rx->dqo.bufq;
struct gve_priv *priv = rx->gve;
+ u32 num_bufs_avail_to_hw;
u32 num_avail_slots;
u32 num_full_slots;
u32 num_posted = 0;
@@ -400,6 +414,26 @@ void gve_rx_post_buffers_dqo(struct gve_rx_ring *rx)
}
rx->fill_cnt += num_posted;
+
+ /* If the queue has fewer than GVE_RX_BUF_THRESH_DQO descriptors
+ * visible to the hardware, the hardware is in danger of starving
+ * and cannot trigger interrupts.
+ *
+ * We use a threshold of 32 because a single maximum-sized RSC
+ * packet can consume up to 19 descriptors in the Rx path. Lower
+ * thresholds (e.g., 8 or 16) would be unsafe as they could cause
+ * the device to drop/stall on a maximum-sized RSC packet.
+ *
+ * Start the timer to periodically reschedule NAPI and recover.
+ */
+ num_bufs_avail_to_hw =
+ ((bufq->tail & ~(GVE_RX_BUF_THRESH_DQO - 1)) -
+ bufq->head) & bufq->mask;
+
+ if (num_bufs_avail_to_hw < GVE_RX_BUF_THRESH_DQO) {
+ mod_timer(&rx->starvation_timer,
+ jiffies + msecs_to_jiffies(GVE_RX_NAPI_RESCHED_MS));
+ }
}
static void gve_rx_skb_csum(struct sk_buff *skb,
--
2.55.0.795.g602f6c329a-goog
^ permalink raw reply related
* Re: [PATCH net-next v6 1/2] udp: fix encapsulation packet resubmit in multicast deliver
From: Willem de Bruijn @ 2026-07-09 21:18 UTC (permalink / raw)
To: Anton Danilov, Willem de Bruijn
Cc: netdev, Willem de Bruijn, David S . Miller, David Ahern,
Eric Dumazet, Kuniyuki Iwashima, Jakub Kicinski, Paolo Abeni,
Simon Horman, Shuah Khan, linux-kselftest
In-Reply-To: <alAAXv60KUZ9KYx1@dau-home-pc>
Anton Danilov wrote:
> On Wed, Jul 08, 2026 at 09:24:10PM -0400, Willem de Bruijn wrote:
> > This helps the case of one encap_rcv socket in the multicast receiver
> > group, so is a useful fix on its own.
> >
> > But is Sashiko correct that this would still leave the same issue for
> > other sockets in the group? If so, something to address in this series
> > or leave for later?
> >
> > Might be worthwhile to extend the test to capture that case too.
>
> The clone loop is not reachable for encapsulation sockets, so there is
> no remaining gap.
>
> FOU/GUE tunnel sockets are created via udp_sock_create() /
> setup_udp_tunnel_sock() and do not set SO_REUSEADDR or SO_REUSEPORT.
> Without either, a UDP socket cannot share its port, so an encap socket
> is always the only socket bound to its port. In
> __udp[46]_lib_mcast_deliver() it is therefore always delivered as
> 'first', and the clone loop -- which handles the second and subsequent
> sockets in the group -- never runs for it. The positive (resubmit)
> return from udp_queue_rcv_skb() only happens for encap sockets; plain
> UDP sockets return 0 or a negative value there. So the resubmit case
> in the clone loop cannot occur.
>
> For the same reason a selftest cannot exercise it: a second socket
> cannot bind to the FOU port.
>
> If useful, I can add a brief comment above the clone loop documenting
> why no resubmit is needed there.
Thanks for that context. No need to respin for that.
^ permalink raw reply
* [PATCH net v4] tipc: fix u16 MTU truncation in media and bearer MTU validation
From: Cen Zhang (Microsoft) @ 2026-07-09 21:16 UTC (permalink / raw)
To: jmaloy, davem, edumazet, kuba, pabeni, horms
Cc: netdev, tipc-discussion, linux-kernel, vadim.fedorenko,
tung.quang.nguyen, AutonomousCodeSecurity, tgopinath, kys,
blbllhy
Both TIPC_NL_MEDIA_SET and TIPC_NL_BEARER_SET accept user-supplied
MTU values but only enforce a minimum bound, not a maximum. When a user
sets the MTU to a value exceeding U16_MAX (65535), it passes validation
but is silently truncated when assigned to u16 fields l->mtu and
l->advertised_mtu in tipc_link_create(). Values like 65536 (0x10000)
truncate to 0, causing a division by zero in tipc_link_set_queue_limits()
which computes TIPC_MAX_PUBL / (l->mtu / ITEM_SIZE). Other overflowing
values (e.g. 65537-131071) produce small incorrect MTU values, resulting
in link malfunction behaviors.
Crash stack (triggered as unprivileged user via user namespace):
tipc_link_set_queue_limits net/tipc/link.c:2531
tipc_link_create net/tipc/link.c:520
tipc_node_check_dest net/tipc/node.c:1279
tipc_disc_rcv net/tipc/discover.c:252
tipc_rcv net/tipc/node.c:2129
tipc_udp_recv net/tipc/udp_media.c:392
Two independent paths lack the upper bound check:
1. tipc_udp_mtu_bad() -- called from __tipc_nl_media_set() (MEDIA_SET)
2. inline check in __tipc_nl_bearer_set() at bearer.c:1160 (BEARER_SET)
Fix both by rejecting MTU values above U16_MAX.
Fixes: 901271e0403a ("tipc: implement configuration of UDP media MTU")
Reported-by: AutonomousCodeSecurity@microsoft.com
Closes: https://lore.kernel.org/all/CAB8m9WgETt0AjmFwE=F-CKjGXsK6_WDv0=kbYRcC8-noo+amnA@mail.gmail.com
Signed-off-by: Cen Zhang (Microsoft) <blbllhy@gmail.com>
---
v4: Add .min check value
v3: Use nla_policy check to limit MTU max value as suggested by Vadim
v2: Solved format issue
Link: https://lore.kernel.org/all/CAB8m9WgETt0AjmFwE=F-CKjGXsK6_WDv0=kbYRcC8-noo+amnA@mail.gmail.com
net/tipc/netlink.c | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/net/tipc/netlink.c b/net/tipc/netlink.c
index 8336a9664703..811438c44542 100644
--- a/net/tipc/netlink.c
+++ b/net/tipc/netlink.c
@@ -113,12 +113,17 @@ const struct nla_policy tipc_nl_node_policy[TIPC_NLA_NODE_MAX + 1] = {
};
/* Properties valid for media, bearer and link */
+static const struct netlink_range_validation tipc_nl_mtu_range = {
+ .min = TIPC_MIN_BEARER_MTU,
+ .max = U16_MAX,
+};
+
const struct nla_policy tipc_nl_prop_policy[TIPC_NLA_PROP_MAX + 1] = {
[TIPC_NLA_PROP_UNSPEC] = { .type = NLA_UNSPEC },
[TIPC_NLA_PROP_PRIO] = { .type = NLA_U32 },
[TIPC_NLA_PROP_TOL] = { .type = NLA_U32 },
[TIPC_NLA_PROP_WIN] = { .type = NLA_U32 },
- [TIPC_NLA_PROP_MTU] = { .type = NLA_U32 },
+ [TIPC_NLA_PROP_MTU] = NLA_POLICY_FULL_RANGE(NLA_U32, &tipc_nl_mtu_range),
[TIPC_NLA_PROP_BROADCAST] = { .type = NLA_U32 },
[TIPC_NLA_PROP_BROADCAST_RATIO] = { .type = NLA_U32 }
};
--
2.53.0
^ permalink raw reply related
* [PATCH v15 9/9] selftests: net: add TLS hardware offload test
From: Rishikesh Jethwani @ 2026-07-09 20:53 UTC (permalink / raw)
To: netdev
Cc: saeedm, tariqt, mbloch, borisp, john.fastabend, kuba, sd, davem,
pabeni, edumazet, leon, andrew.gospodarek, Rishikesh Jethwani
In-Reply-To: <20260709205325.1591196-1-rjethwani@purestorage.com>
Two-node kTLS HW offload test using NetDrvEpEnv. A C helper binary
acts as TLS client or server; a Python harness drives it and verifies
TLS stat counters (RekeyOk, RekeyReceived, RekeyFallback,
RekeyInProgress, RekeyAborted, RekeyError, DecryptError).
Covers TLS 1.2/1.3 with AES-GCM-128/256, rekey with various buffer
sizes, and burst variants that stress TX rekey (temporary SW phase,
HW reinstall) and RX rekey (boundary tracking, old-key reencryption,
deferred dev_add).
Signed-off-by: Rishikesh Jethwani <rjethwani@purestorage.com>
---
MAINTAINERS | 2 +
.../selftests/drivers/net/hw/.gitignore | 1 +
.../testing/selftests/drivers/net/hw/Makefile | 2 +
.../selftests/drivers/net/hw/tls_hw_offload.c | 975 ++++++++++++++++++
.../drivers/net/hw/tls_hw_offload.py | 295 ++++++
5 files changed, 1275 insertions(+)
create mode 100644 tools/testing/selftests/drivers/net/hw/tls_hw_offload.c
create mode 100755 tools/testing/selftests/drivers/net/hw/tls_hw_offload.py
diff --git a/MAINTAINERS b/MAINTAINERS
index 52f1a55eca99..d132681f07dc 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -19047,6 +19047,8 @@ F: Documentation/networking/tls*
F: include/net/tls.h
F: include/uapi/linux/tls.h
F: net/tls/
+F: tools/testing/selftests/drivers/net/hw/tls_hw_offload.c
+F: tools/testing/selftests/drivers/net/hw/tls_hw_offload.py
F: tools/testing/selftests/net/tls.c
NETWORKING [SOCKETS]
diff --git a/tools/testing/selftests/drivers/net/hw/.gitignore b/tools/testing/selftests/drivers/net/hw/.gitignore
index 46540468a775..f0a5d15b469b 100644
--- a/tools/testing/selftests/drivers/net/hw/.gitignore
+++ b/tools/testing/selftests/drivers/net/hw/.gitignore
@@ -2,3 +2,4 @@
iou-zcrx
ncdevmem
toeplitz
+tls_hw_offload
diff --git a/tools/testing/selftests/drivers/net/hw/Makefile b/tools/testing/selftests/drivers/net/hw/Makefile
index 234db5c2c90c..c274c41f97e8 100644
--- a/tools/testing/selftests/drivers/net/hw/Makefile
+++ b/tools/testing/selftests/drivers/net/hw/Makefile
@@ -15,6 +15,7 @@ endif
TEST_GEN_FILES := \
$(COND_GEN_FILES) \
+ tls_hw_offload \
# end of TEST_GEN_FILES
TEST_PROGS = \
@@ -46,6 +47,7 @@ TEST_PROGS = \
rss_drv.py \
rss_flow_label.py \
rss_input_xfrm.py \
+ tls_hw_offload.py \
toeplitz.py \
tso.py \
userns_devmem.py \
diff --git a/tools/testing/selftests/drivers/net/hw/tls_hw_offload.c b/tools/testing/selftests/drivers/net/hw/tls_hw_offload.c
new file mode 100644
index 000000000000..1882e2f073ad
--- /dev/null
+++ b/tools/testing/selftests/drivers/net/hw/tls_hw_offload.c
@@ -0,0 +1,975 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * TLS Hardware Offload Two-Node Test
+ *
+ * Tests kTLS hardware offload between two physical nodes using
+ * hardcoded keys. Supports TLS 1.2/1.3, AES-GCM-128/256, and rekey.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <limits.h>
+#include <time.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+#include <arpa/inet.h>
+#include <linux/tls.h>
+
+#define TLS_RECORD_TYPE_HANDSHAKE 22
+#define TLS_HANDSHAKE_KEY_UPDATE 0x18
+
+/* Large enough for a TLS 1.3 KeyUpdate handshake record's plaintext. */
+#define MIN_BUF_SIZE 16
+
+/* Initial key material */
+static struct tls12_crypto_info_aes_gcm_128 tls_info_key0_128 = {
+ .info = {
+ .version = TLS_1_3_VERSION,
+ .cipher_type = TLS_CIPHER_AES_GCM_128,
+ },
+ .iv = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08 },
+ .key = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
+ 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10 },
+ .salt = { 0x01, 0x02, 0x03, 0x04 },
+ .rec_seq = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+};
+
+static struct tls12_crypto_info_aes_gcm_256 tls_info_key0_256 = {
+ .info = {
+ .version = TLS_1_3_VERSION,
+ .cipher_type = TLS_CIPHER_AES_GCM_256,
+ },
+ .iv = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08 },
+ .key = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
+ 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10,
+ 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18,
+ 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20 },
+ .salt = { 0x01, 0x02, 0x03, 0x04 },
+ .rec_seq = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+};
+
+static int num_rekeys;
+static int num_iterations = 100;
+static int cipher_type = TLS_CIPHER_AES_GCM_128;
+static int tls_version = TLS_1_3_VERSION;
+static int server_port = 4433;
+static char *server_ip;
+
+static int send_size = 16384;
+static int random_size_max;
+/* Burst mode: sender keeps pushing records without reading from the peer;
+ * receiver drains without echoing back. Only the client initiates rekey.
+ */
+static int burst_mode;
+static int zc_rx;
+
+/* XOR each byte with the generation so both endpoints derive the
+ * same per-generation key without a real KDF. Generation 0 leaves
+ * the base key unchanged.
+ */
+static void derive_key_fields(unsigned char *key, int key_size,
+ unsigned char *iv, int iv_size,
+ unsigned char *salt, int salt_size,
+ unsigned char *rec_seq, int rec_seq_size,
+ int generation)
+{
+ int i;
+
+ for (i = 0; i < key_size; i++)
+ key[i] ^= generation;
+ for (i = 0; i < iv_size; i++)
+ iv[i] ^= generation;
+ for (i = 0; i < salt_size; i++)
+ salt[i] ^= generation;
+ memset(rec_seq, 0, rec_seq_size);
+}
+
+static void derive_key_128(struct tls12_crypto_info_aes_gcm_128 *key,
+ int generation)
+{
+ memcpy(key, &tls_info_key0_128, sizeof(*key));
+ key->info.version = tls_version;
+ derive_key_fields(key->key, TLS_CIPHER_AES_GCM_128_KEY_SIZE,
+ key->iv, TLS_CIPHER_AES_GCM_128_IV_SIZE,
+ key->salt, TLS_CIPHER_AES_GCM_128_SALT_SIZE,
+ key->rec_seq, TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE,
+ generation);
+}
+
+static void derive_key_256(struct tls12_crypto_info_aes_gcm_256 *key,
+ int generation)
+{
+ memcpy(key, &tls_info_key0_256, sizeof(*key));
+ key->info.version = tls_version;
+ derive_key_fields(key->key, TLS_CIPHER_AES_GCM_256_KEY_SIZE,
+ key->iv, TLS_CIPHER_AES_GCM_256_IV_SIZE,
+ key->salt, TLS_CIPHER_AES_GCM_256_SALT_SIZE,
+ key->rec_seq, TLS_CIPHER_AES_GCM_256_REC_SEQ_SIZE,
+ generation);
+}
+
+static const char *cipher_name(int cipher)
+{
+ switch (cipher) {
+ case TLS_CIPHER_AES_GCM_128: return "AES-GCM-128";
+ case TLS_CIPHER_AES_GCM_256: return "AES-GCM-256";
+ default: return "unknown";
+ }
+}
+
+static const char *version_name(int version)
+{
+ switch (version) {
+ case TLS_1_2_VERSION: return "TLS 1.2";
+ case TLS_1_3_VERSION: return "TLS 1.3";
+ default: return "unknown";
+ }
+}
+
+static int setup_tls_ulp(int fd)
+{
+ int ret;
+
+ ret = setsockopt(fd, IPPROTO_TCP, TCP_ULP, "tls", sizeof("tls"));
+ if (ret < 0) {
+ printf("SETUP ERROR: TCP_ULP failed: %s\n", strerror(errno));
+ return -1;
+ }
+ return 0;
+}
+
+static int set_zc_rx(int fd)
+{
+ int val = 1;
+
+ if (setsockopt(fd, SOL_TLS, TLS_RX_EXPECT_NO_PAD, &val,
+ sizeof(val)) < 0) {
+ printf("SETUP ERROR: TLS_RX_EXPECT_NO_PAD failed: %s\n",
+ strerror(errno));
+ return -1;
+ }
+ return 0;
+}
+
+/* Send a TLS 1.3 KeyUpdate handshake record. The kernel only
+ * inspects the HandshakeType byte to detect KeyUpdate, so don't
+ * bother with the 3-byte length or request_update fields.
+ */
+static int send_tls_key_update(int fd)
+{
+ char cmsg_buf[CMSG_SPACE(sizeof(unsigned char))];
+ unsigned char key_update_msg = TLS_HANDSHAKE_KEY_UPDATE;
+ struct msghdr msg = {0};
+ struct cmsghdr *cmsg;
+ struct iovec iov;
+
+ iov.iov_base = &key_update_msg;
+ iov.iov_len = sizeof(key_update_msg);
+
+ msg.msg_iov = &iov;
+ msg.msg_iovlen = 1;
+ msg.msg_control = cmsg_buf;
+ msg.msg_controllen = sizeof(cmsg_buf);
+
+ cmsg = CMSG_FIRSTHDR(&msg);
+ cmsg->cmsg_level = SOL_TLS;
+ cmsg->cmsg_type = TLS_SET_RECORD_TYPE;
+ cmsg->cmsg_len = CMSG_LEN(sizeof(unsigned char));
+ *CMSG_DATA(cmsg) = TLS_RECORD_TYPE_HANDSHAKE;
+ msg.msg_controllen = cmsg->cmsg_len;
+
+ if (sendmsg(fd, &msg, 0) < 0) {
+ printf("sendmsg KeyUpdate failed: %s\n", strerror(errno));
+ return -1;
+ }
+
+ printf("Sent TLS KeyUpdate handshake message\n");
+ return 0;
+}
+
+static int recv_tls_message(int fd, char *buf, size_t buflen, int *record_type,
+ int flags)
+{
+ char cmsg_buf[CMSG_SPACE(sizeof(unsigned char))];
+ struct msghdr msg = {0};
+ struct cmsghdr *cmsg;
+ struct iovec iov;
+ int ret;
+
+ iov.iov_base = buf;
+ iov.iov_len = buflen;
+
+ msg.msg_iov = &iov;
+ msg.msg_iovlen = 1;
+ msg.msg_control = cmsg_buf;
+ msg.msg_controllen = sizeof(cmsg_buf);
+
+ ret = recvmsg(fd, &msg, flags);
+ if (ret <= 0)
+ return ret;
+
+ cmsg = CMSG_FIRSTHDR(&msg);
+ if (cmsg && cmsg->cmsg_level == SOL_TLS &&
+ cmsg->cmsg_type == TLS_GET_RECORD_TYPE)
+ *record_type = *((unsigned char *)CMSG_DATA(cmsg));
+
+ return ret;
+}
+
+/* Confirm a handshake record starting with HandshakeType KeyUpdate. */
+static int check_keyupdate(const char *buf, int len, int record_type)
+{
+ if (record_type != TLS_RECORD_TYPE_HANDSHAKE) {
+ printf("Expected handshake record (0x%02x), got 0x%02x\n",
+ TLS_RECORD_TYPE_HANDSHAKE, record_type);
+ return -1;
+ }
+ if (len < 1 || (unsigned char)buf[0] != TLS_HANDSHAKE_KEY_UPDATE) {
+ printf("Expected KeyUpdate (0x%02x), got 0x%02x\n",
+ TLS_HANDSHAKE_KEY_UPDATE,
+ len ? (unsigned char)buf[0] : 0);
+ return -1;
+ }
+ printf("Received TLS KeyUpdate\n");
+ return 0;
+}
+
+static int recv_tls_keyupdate(int fd)
+{
+ char buf[MIN_BUF_SIZE];
+ int record_type = 0;
+ int ret;
+
+ ret = recv_tls_message(fd, buf, sizeof(buf), &record_type, 0);
+ if (ret < 0) {
+ printf("recv_tls_message failed: %s\n", strerror(errno));
+ return -1;
+ }
+
+ return check_keyupdate(buf, ret, record_type);
+}
+
+static int check_ekeyexpired(int fd)
+{
+ char buf[MIN_BUF_SIZE];
+ int ret;
+
+ ret = recv(fd, buf, sizeof(buf), MSG_DONTWAIT);
+ if (ret == -1 && errno == EKEYEXPIRED) {
+ printf("recv() returned EKEYEXPIRED as expected\n");
+ return 0;
+ } else if (ret == -1 && errno == EAGAIN) {
+ printf("recv() returned EAGAIN (no pending data)\n");
+ return 0;
+ } else if (ret > 0) {
+ printf("FAIL: recv() returned %d bytes, expected EKEYEXPIRED\n",
+ ret);
+ return -1;
+ } else {
+ printf("FAIL: recv() returned unexpected error: %s\n",
+ strerror(errno));
+ return -1;
+ }
+}
+
+static int do_tls_rekey(int fd, int direction, int generation, int cipher)
+{
+ const char *dir = direction == TLS_TX ? "TX" : "RX";
+ int ret;
+
+ printf("%s TLS_%s %s gen %d...\n",
+ generation ? "Rekeying" : "Installing",
+ dir, cipher_name(cipher), generation);
+
+ if (cipher == TLS_CIPHER_AES_GCM_256) {
+ struct tls12_crypto_info_aes_gcm_256 key;
+
+ derive_key_256(&key, generation);
+ ret = setsockopt(fd, SOL_TLS, direction, &key, sizeof(key));
+ } else {
+ struct tls12_crypto_info_aes_gcm_128 key;
+
+ derive_key_128(&key, generation);
+ ret = setsockopt(fd, SOL_TLS, direction, &key, sizeof(key));
+ }
+
+ if (ret < 0) {
+ printf("%sTLS_%s %s gen %d failed: %s\n",
+ generation ? "" : "SETUP ERROR: ", dir,
+ cipher_name(cipher), generation, strerror(errno));
+ return -1;
+ }
+ printf("TLS_%s %s gen %d installed\n",
+ dir, cipher_name(cipher), generation);
+ return 0;
+}
+
+/* Open a TCP connection to server_ip:server_port, switch to the TLS
+ * ULP, and install initial generation-0 TX/RX keys. Returns the fd on
+ * success, -1 on error (with the fd already closed).
+ */
+static int client_connect_tls(void)
+{
+ struct sockaddr_in sa;
+ int csk;
+
+ csk = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
+ if (csk < 0) {
+ printf("SETUP ERROR: failed to create socket: %s\n",
+ strerror(errno));
+ return -1;
+ }
+
+ memset(&sa, 0, sizeof(sa));
+ sa.sin_family = AF_INET;
+ sa.sin_addr.s_addr = inet_addr(server_ip);
+ sa.sin_port = htons(server_port);
+ printf("Connecting to %s:%d...\n", server_ip, server_port);
+
+ if (connect(csk, (struct sockaddr *)&sa, sizeof(sa)) < 0) {
+ printf("SETUP ERROR: connect failed: %s\n", strerror(errno));
+ goto err;
+ }
+ printf("Connected!\n");
+
+ if (setup_tls_ulp(csk) < 0)
+ goto err;
+
+ if (do_tls_rekey(csk, TLS_TX, 0, cipher_type) < 0 ||
+ do_tls_rekey(csk, TLS_RX, 0, cipher_type) < 0)
+ goto err;
+
+ return csk;
+err:
+ close(csk);
+ return -1;
+}
+
+/* Drain `len` echoed bytes from the server and verify they match the
+ * payload we just sent.
+ */
+static int client_recv_echo(int fd, const char *sent, char *echo_buf,
+ ssize_t len)
+{
+ ssize_t total = 0;
+ ssize_t n;
+
+ while (total < len) {
+ n = recv(fd, echo_buf + total, len - total, 0);
+ if (n < 0) {
+ printf("FAIL: Echo recv failed: %s\n", strerror(errno));
+ return -1;
+ }
+ if (n == 0) {
+ printf("FAIL: Connection closed during echo\n");
+ return -1;
+ }
+ total += n;
+ }
+
+ if (memcmp(sent, echo_buf, len) != 0) {
+ printf("FAIL: Echo data mismatch!\n");
+ return -1;
+ }
+ printf("Received echo %zd bytes (ok)\n", total);
+ return 0;
+}
+
+/* Client side of a rekey: send KeyUpdate and rotate TX. In echo mode
+ * also wait for the peer's KeyUpdate and rotate RX.
+ */
+static int client_rekey(int fd, int generation)
+{
+ if (send_tls_key_update(fd) < 0) {
+ printf("FAIL: send KeyUpdate\n");
+ return -1;
+ }
+
+ if (do_tls_rekey(fd, TLS_TX, generation, cipher_type) < 0)
+ return -1;
+
+ if (burst_mode)
+ return 0;
+
+ if (recv_tls_keyupdate(fd) < 0) {
+ printf("FAIL: recv KeyUpdate from server\n");
+ return -1;
+ }
+
+ if (check_ekeyexpired(fd) < 0)
+ return -1;
+
+ return do_tls_rekey(fd, TLS_RX, generation, cipher_type);
+}
+
+static int do_client(void)
+{
+ char *buf = NULL, *echo_buf = NULL;
+ int max_size, rekey_interval;
+ int csk = -1, i;
+ int test_result = -1;
+ int current_gen = 0;
+ int next_rekey_at;
+ ssize_t n;
+
+ max_size = random_size_max > 0 ? random_size_max : send_size;
+ if (max_size < MIN_BUF_SIZE)
+ max_size = MIN_BUF_SIZE;
+ buf = malloc(max_size);
+ if (!burst_mode)
+ echo_buf = malloc(max_size);
+ if (!buf || (!burst_mode && !echo_buf)) {
+ printf("SETUP ERROR: failed to allocate buffers\n");
+ goto out;
+ }
+
+ csk = client_connect_tls();
+ if (csk < 0)
+ goto out;
+
+ if (num_rekeys)
+ printf("TLS %s setup complete. Will perform %d rekey(s).\n",
+ cipher_name(cipher_type), num_rekeys);
+ else
+ printf("TLS setup complete.\n");
+
+ if (random_size_max > 0)
+ printf("Sending %d messages of random size (1..%d bytes)...\n",
+ num_iterations, random_size_max);
+ else
+ printf("Sending %d messages of %d bytes...\n",
+ num_iterations, send_size);
+
+ rekey_interval = num_iterations / (num_rekeys + 1);
+ next_rekey_at = rekey_interval;
+
+ for (i = 1; i <= num_iterations; i++) {
+ int this_size;
+
+ if (random_size_max > 0)
+ this_size = (rand() % random_size_max) + 1;
+ else
+ this_size = send_size;
+
+ /* In burst mode, use a per-iteration fill pattern so the
+ * receiver can detect any plaintext corruption without a
+ * round-trip echo.
+ */
+ if (burst_mode) {
+ memset(buf, i & 0xFF, this_size);
+ } else {
+ int j;
+
+ for (j = 0; j < this_size; j++)
+ buf[j] = rand() & 0xFF;
+ }
+
+ n = send(csk, buf, this_size, 0);
+ if (n != this_size) {
+ printf("FAIL: send failed: %s\n", strerror(errno));
+ goto out;
+ }
+
+ if (!burst_mode) {
+ printf("Sent %zd bytes (iteration %d)\n", n, i);
+ if (client_recv_echo(csk, buf, echo_buf, n) < 0)
+ goto out;
+ }
+
+ /* Rekey at intervals. In echo mode this is a full bidirectional
+ * exchange; in burst mode the client only rotates its TX key
+ * and sends KeyUpdate - the peer is expected to follow.
+ */
+ if (num_rekeys && current_gen < num_rekeys &&
+ i == next_rekey_at) {
+ current_gen++;
+ printf("\n=== Client Rekey gen %d ===\n", current_gen);
+
+ if (client_rekey(csk, current_gen) < 0)
+ goto out;
+
+ next_rekey_at += rekey_interval;
+ printf("=== Client Rekey gen %d Complete ===\n\n",
+ current_gen);
+ }
+ }
+
+ test_result = 0;
+out:
+ if (num_rekeys)
+ printf("Rekeys completed: %d/%d\n", current_gen, num_rekeys);
+ if (csk >= 0)
+ close(csk);
+ free(buf);
+ free(echo_buf);
+ return test_result;
+}
+
+/* Bind/listen on server_port, accept one client, switch to the TLS ULP
+ * and install initial generation-0 keys (plus zc_rx if requested).
+ * Returns the connected fd on success and writes the listener fd to
+ * *lsk_out so the caller can close it. Returns -1 on error, with all
+ * intermediate fds already closed and *lsk_out left at -1.
+ */
+static int server_accept_tls(int *lsk_out)
+{
+ int lsk, csk, one = 1;
+ struct sockaddr_in sa;
+
+ *lsk_out = -1;
+
+ lsk = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
+ if (lsk < 0) {
+ printf("SETUP ERROR: failed to create socket: %s\n",
+ strerror(errno));
+ return -1;
+ }
+
+ setsockopt(lsk, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one));
+
+ memset(&sa, 0, sizeof(sa));
+ sa.sin_family = AF_INET;
+ sa.sin_addr.s_addr = INADDR_ANY;
+ sa.sin_port = htons(server_port);
+
+ if (bind(lsk, (struct sockaddr *)&sa, sizeof(sa)) < 0) {
+ printf("SETUP ERROR: bind failed: %s\n", strerror(errno));
+ close(lsk);
+ return -1;
+ }
+
+ if (listen(lsk, 1) < 0) {
+ printf("SETUP ERROR: listen failed: %s\n", strerror(errno));
+ close(lsk);
+ return -1;
+ }
+
+ printf("Server listening on 0.0.0.0:%d\n", server_port);
+ printf("Waiting for client connection...\n");
+
+ csk = accept(lsk, (struct sockaddr *)NULL, (socklen_t *)NULL);
+ if (csk < 0) {
+ printf("SETUP ERROR: accept failed: %s\n", strerror(errno));
+ close(lsk);
+ return -1;
+ }
+ printf("Client connected!\n");
+
+ if (setup_tls_ulp(csk) < 0)
+ goto err;
+
+ if (do_tls_rekey(csk, TLS_TX, 0, cipher_type) < 0 ||
+ do_tls_rekey(csk, TLS_RX, 0, cipher_type) < 0)
+ goto err;
+
+ if (zc_rx && set_zc_rx(csk) < 0)
+ goto err;
+
+ *lsk_out = lsk;
+ return csk;
+err:
+ close(csk);
+ close(lsk);
+ return -1;
+}
+
+/* Server side of a rekey: drain any in-flight ciphertext that hit
+ * EKEYEXPIRED and rotate RX. In echo mode also send a KeyUpdate back
+ * and rotate TX.
+ */
+static int server_rekey(int fd, int generation)
+{
+ if (check_ekeyexpired(fd) < 0)
+ return -1;
+
+ if (do_tls_rekey(fd, TLS_RX, generation, cipher_type) < 0)
+ return -1;
+
+ if (burst_mode)
+ return 0;
+
+ if (send_tls_key_update(fd) < 0) {
+ printf("FAIL: send KeyUpdate\n");
+ return -1;
+ }
+
+ return do_tls_rekey(fd, TLS_TX, generation, cipher_type);
+}
+
+/* Burst mode: MSG_WAITALL gives us exactly one iteration's payload,
+ * filled with (send_iter & 0xff). Catches decrypt-succeeded-but-
+ * plaintext-corrupt bugs that AEAD counters alone would miss.
+ */
+static int server_verify_burst(int fd, char *buf, int buf_size,
+ ssize_t n, int send_iter)
+{
+ unsigned char expect = send_iter & 0xFF;
+ int j;
+
+ if (n != send_size) {
+ int record_type = 0;
+ ssize_t n2;
+
+ /* MSG_WAITALL returned short; probe one more recv to
+ * surface any pending socket error for the diagnostic.
+ */
+ n2 = recv_tls_message(fd, buf, buf_size, &record_type, 0);
+ printf("FAIL: short recv in burst mode: got %zd, expected %d (iter %d)\n",
+ n, send_size, send_iter);
+ printf(" follow-up recv: %zd errno=%s\n",
+ n2, n2 < 0 ? strerror(errno) : "ok");
+ return -1;
+ }
+
+ for (j = 0; j < n; j++) {
+ if ((unsigned char)buf[j] != expect) {
+ printf("FAIL: data mismatch iter %d offset %d: expected 0x%02x got 0x%02x\n",
+ send_iter, j, expect, (unsigned char)buf[j]);
+ return -1;
+ }
+ }
+ return 0;
+}
+
+static int server_echo_send(int fd, const char *buf, ssize_t n)
+{
+ ssize_t sent;
+ int ret;
+
+ for (sent = 0; sent < n; sent += ret) {
+ ret = send(fd, buf + sent, n - sent, 0);
+ if (ret < 0) {
+ printf("FAIL: Echo send failed: %s\n", strerror(errno));
+ return -1;
+ }
+ }
+ return 0;
+}
+
+static int do_server(void)
+{
+ int lsk = -1, csk = -1;
+ ssize_t n, total = 0;
+ int test_result = -1;
+ int current_gen = 0;
+ int recv_count = 0;
+ int send_iter = 1;
+ char *buf = NULL;
+ int record_type = 0;
+ int recv_flags;
+ int buf_size;
+
+ buf_size = send_size;
+ if (buf_size < MIN_BUF_SIZE)
+ buf_size = MIN_BUF_SIZE;
+ buf = malloc(buf_size);
+ if (!buf) {
+ printf("SETUP ERROR: failed to allocate buffer\n");
+ goto out;
+ }
+
+ csk = server_accept_tls(&lsk);
+ if (csk < 0)
+ goto out;
+
+ printf("TLS %s setup complete. Receiving...\n",
+ cipher_name(cipher_type));
+
+ /* Burst mode: ask for a full iteration's worth of plaintext per
+ * recv. kTLS accumulates across data records when MSG_WAITALL is
+ * set (target == len), and breaks cleanly at control records, so
+ * each recv returns exactly send_size data bytes or a small KU.
+ */
+ recv_flags = burst_mode ? MSG_WAITALL : 0;
+
+ /* Main receive loop */
+ while (1) {
+ n = recv_tls_message(csk, buf, buf_size, &record_type,
+ recv_flags);
+ if (n == 0) {
+ printf("Connection closed by client\n");
+ break;
+ }
+ if (n < 0) {
+ printf("FAIL: recv failed: %s\n", strerror(errno));
+ goto out;
+ }
+
+ /* Handle KeyUpdate. In echo mode the server mirrors the
+ * rekey back to the peer; in burst mode it only rotates
+ * its RX key and keeps draining.
+ */
+ if (record_type == TLS_RECORD_TYPE_HANDSHAKE) {
+ if (check_keyupdate(buf, n, record_type) < 0)
+ goto out;
+ current_gen++;
+ printf("\n=== Server Rekey gen %d ===\n", current_gen);
+
+ if (server_rekey(csk, current_gen) < 0)
+ goto out;
+
+ printf("=== Server Rekey gen %d Complete ===\n\n",
+ current_gen);
+ continue;
+ }
+
+ total += n;
+ recv_count++;
+
+ if (burst_mode) {
+ if (server_verify_burst(csk, buf, buf_size, n,
+ send_iter) < 0)
+ goto out;
+ send_iter++;
+ continue;
+ }
+
+ printf("Received %zd bytes (total: %zd, count: %d)\n",
+ n, total, recv_count);
+
+ if (server_echo_send(csk, buf, n) < 0)
+ goto out;
+ printf("Echoed %zd bytes back to client\n", n);
+ }
+
+ test_result = 0;
+out:
+ printf("Connection closed. Total received: %zd bytes\n", total);
+ if (num_rekeys)
+ printf("Rekeys completed: %d\n", current_gen);
+
+ if (csk >= 0)
+ close(csk);
+ if (lsk >= 0)
+ close(lsk);
+ free(buf);
+ return test_result;
+}
+
+static int parse_int_arg(const char *arg, int min, int max,
+ const char *name, int *out)
+{
+ char *endp;
+ long val;
+
+ errno = 0;
+ val = strtol(arg, &endp, 10);
+ if (errno || endp == arg || *endp != '\0' || val < min || val > max) {
+ if (max == INT_MAX)
+ printf("ERROR: Invalid %s '%s'. Must be >= %d.\n",
+ name, arg, min);
+ else
+ printf("ERROR: Invalid %s '%s'. Must be %d..%d.\n",
+ name, arg, min, max);
+ return -1;
+ }
+ *out = (int)val;
+ return 0;
+}
+
+static int parse_cipher_option(const char *arg)
+{
+ if (strcmp(arg, "128") == 0) {
+ cipher_type = TLS_CIPHER_AES_GCM_128;
+ return 0;
+ } else if (strcmp(arg, "256") == 0) {
+ cipher_type = TLS_CIPHER_AES_GCM_256;
+ return 0;
+ }
+ printf("ERROR: Invalid cipher '%s'. Must be 128 or 256.\n", arg);
+ return -1;
+}
+
+static int parse_version_option(const char *arg)
+{
+ if (strcmp(arg, "1.2") == 0) {
+ tls_version = TLS_1_2_VERSION;
+ return 0;
+ } else if (strcmp(arg, "1.3") == 0) {
+ tls_version = TLS_1_3_VERSION;
+ return 0;
+ }
+ printf("ERROR: Invalid TLS version '%s'. Must be 1.2 or 1.3.\n", arg);
+ return -1;
+}
+
+static void print_usage(const char *prog)
+{
+ printf("TLS Hardware Offload Two-Node Test\n\n");
+ printf("Usage:\n");
+ printf(" %s server [OPTIONS]\n", prog);
+ printf(" %s client -s <ip> [OPTIONS]\n", prog);
+ printf("\nOptions:\n");
+ printf(" -s <ip> Server IPv4 address (client, required)\n");
+ printf(" -p <port> Server port (default: 4433)\n");
+ printf(" -b <size> Send buffer size in bytes (default: 16384)\n");
+ printf(" -r <max> Use random send buffer sizes (1..<max>)\n");
+ printf(" -v <version> TLS version: 1.2 or 1.3 (default: 1.3)\n");
+ printf(" -c <cipher> Cipher: 128 or 256 (default: 128)\n");
+ printf(" -n <N> Number of send/echo iterations (default: 100)\n");
+ printf(" -k <N> Perform N rekeys (client only, TLS 1.3; N < iterations)\n");
+ printf(" -B Burst mode: client sends continuously without echo;\n");
+ printf(" server drains and handles KeyUpdate without responding.\n");
+ printf(" -Z Set TLS_RX_EXPECT_NO_PAD on the server: TLS 1.3\n");
+ printf(" opt-in to the zero-copy RX fast path. Not needed\n");
+ printf(" for TLS 1.2 (always eligible). Server only.\n");
+ printf(" -h Show this help message\n");
+ printf("\nExample:\n");
+ printf(" Node A: %s server\n", prog);
+ printf(" Node B: %s client -s 192.168.20.2\n", prog);
+ printf("\nRekey Example (3 rekeys, TLS 1.3 only):\n");
+ printf(" Node A: %s server\n", prog);
+ printf(" Node B: %s client -s 192.168.20.2 -k 3\n", prog);
+ printf("\nBurst Mode Example (client stresses TX rekey under load):\n");
+ printf(" Node A: %s server -B\n", prog);
+ printf(" Node B: %s client -s 192.168.20.2 -B -k 3\n", prog);
+}
+
+int main(int argc, char *argv[])
+{
+ int send_size_set = 0;
+ int is_server;
+ int opt;
+
+ if (argc < 2 ||
+ (strcmp(argv[1], "server") && strcmp(argv[1], "client"))) {
+ print_usage(argv[0]);
+ return 1;
+ }
+ is_server = !strcmp(argv[1], "server");
+
+ optind = 2; /* skip subcommand */
+ while ((opt = getopt(argc, argv, "s:p:b:r:c:v:k:n:BZh")) != -1) {
+ switch (opt) {
+ case 's':
+ server_ip = optarg;
+ break;
+ case 'B':
+ burst_mode = 1;
+ break;
+ case 'Z':
+ zc_rx = 1;
+ break;
+ case 'p':
+ if (parse_int_arg(optarg, 1, 65535, "port",
+ &server_port) < 0)
+ return 1;
+ break;
+ case 'b':
+ if (parse_int_arg(optarg, 1, INT_MAX, "buffer size",
+ &send_size) < 0)
+ return 1;
+ send_size_set = 1;
+ break;
+ case 'r':
+ if (parse_int_arg(optarg, 1, INT_MAX, "random size",
+ &random_size_max) < 0)
+ return 1;
+ break;
+ case 'c':
+ if (parse_cipher_option(optarg) < 0)
+ return 1;
+ break;
+ case 'v':
+ if (parse_version_option(optarg) < 0)
+ return 1;
+ break;
+ case 'k':
+ if (parse_int_arg(optarg, 1, 255, "rekey count",
+ &num_rekeys) < 0)
+ return 1;
+ break;
+ case 'n':
+ if (parse_int_arg(optarg, 1, INT_MAX, "iteration count",
+ &num_iterations) < 0)
+ return 1;
+ break;
+ case 'h':
+ print_usage(argv[0]);
+ return 0;
+ default:
+ print_usage(argv[0]);
+ return 1;
+ }
+ }
+
+ if (send_size_set && random_size_max > 0) {
+ printf("ERROR: -b and -r are mutually exclusive\n");
+ return 1;
+ }
+
+ if (zc_rx && tls_version != TLS_1_3_VERSION) {
+ printf("ERROR: -Z (TLS_RX_EXPECT_NO_PAD) requires TLS 1.3\n");
+ return 1;
+ }
+
+ if (burst_mode && random_size_max > 0) {
+ printf("ERROR: -B and -r are mutually exclusive\n");
+ return 1;
+ }
+
+ if (burst_mode && send_size < MIN_BUF_SIZE) {
+ printf("ERROR: -b must be >= %d in burst mode (-B)\n",
+ MIN_BUF_SIZE);
+ return 1;
+ }
+
+ if (is_server) {
+ if (server_ip) {
+ printf("warning: -s is ignored in server mode\n");
+ server_ip = NULL;
+ }
+ if (random_size_max > 0) {
+ printf("warning: -r is ignored in server mode\n");
+ random_size_max = 0;
+ }
+ if (num_rekeys) {
+ printf("warning: -k is ignored in server mode\n");
+ num_rekeys = 0;
+ }
+ } else {
+ if (!server_ip) {
+ printf("ERROR: Client requires -s <ip> option\n");
+ return 1;
+ }
+ if (tls_version == TLS_1_2_VERSION && num_rekeys) {
+ printf("ERROR: TLS 1.2 does not support rekey\n");
+ return 1;
+ }
+ if (num_rekeys >= num_iterations) {
+ printf("ERROR: num_rekeys (%d) must be < num_iterations (%d)\n",
+ num_rekeys, num_iterations);
+ return 1;
+ }
+ if (zc_rx) {
+ printf("ERROR: -Z applies to the server (receiver) only\n");
+ return 1;
+ }
+ }
+
+ printf("TLS Version: %s\n", version_name(tls_version));
+ printf("Cipher: %s\n", cipher_name(cipher_type));
+ if (random_size_max > 0)
+ printf("Buffer size: random (1..%d)\n", random_size_max);
+ else
+ printf("Buffer size: %d\n", send_size);
+
+ if (num_rekeys)
+ printf("Rekey testing ENABLED: %d rekey(s)\n", num_rekeys);
+ if (burst_mode)
+ printf("Burst mode ENABLED\n");
+ if (zc_rx)
+ printf("TLS_RX_EXPECT_NO_PAD ENABLED\n");
+
+ srand(time(NULL));
+
+ if (is_server)
+ return do_server() ? 1 : 0;
+
+ return do_client() ? 1 : 0;
+}
diff --git a/tools/testing/selftests/drivers/net/hw/tls_hw_offload.py b/tools/testing/selftests/drivers/net/hw/tls_hw_offload.py
new file mode 100755
index 000000000000..b8f5a3314030
--- /dev/null
+++ b/tools/testing/selftests/drivers/net/hw/tls_hw_offload.py
@@ -0,0 +1,295 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0
+
+"""Test kTLS hardware offload using a C helper binary."""
+
+from collections import defaultdict
+
+from lib.py import ksft_run, ksft_exit, ksft_pr, KsftSkipEx, ksft_true
+from lib.py import ksft_variants, KsftNamedVariant
+from lib.py import NetDrvEpEnv
+from lib.py import cmd, bkg, wait_port_listen, rand_port
+from lib.py import CmdExitFailure
+
+# Burst variants push hundreds of MB and perform many rekeys; the
+# default cmd() timeout (5s) is too short.
+BURST_TIMEOUT_S = 180
+
+
+def check_tls_support(cfg):
+ try:
+ cmd("test -f /proc/net/tls_stat")
+ cmd("test -f /proc/net/tls_stat", host=cfg.remote)
+ except CmdExitFailure as e:
+ raise KsftSkipEx(f"kTLS not supported: {e}")
+
+ try:
+ features = cmd(f"ethtool -k {cfg.ifname}").stdout
+ if 'tls-hw-tx-offload: on' not in features:
+ raise KsftSkipEx("Device does not support TLS HW TX offload")
+ if 'tls-hw-rx-offload: on' not in features:
+ raise KsftSkipEx("Device does not support TLS HW RX offload")
+ except CmdExitFailure as e:
+ raise KsftSkipEx(f"Cannot determine TLS HW offload support: {e}")
+
+
+def read_tls_stats(host=None):
+ stats = defaultdict(int)
+ output = cmd("cat /proc/net/tls_stat", host=host)
+ for line in output.stdout.strip().split('\n'):
+ parts = line.split()
+ if len(parts) == 2:
+ stats[parts[0]] = int(parts[1])
+ return stats
+
+
+def stat_diff(before, after, key):
+ return after[key] - before[key]
+
+
+def check_path(before, after, direction, role, require_hw):
+ """On the DUT, require HW offload; on the remote, HW or SW is fine."""
+ dev = stat_diff(before, after, f'Tls{direction}Device')
+ sw = stat_diff(before, after, f'Tls{direction}Sw')
+ if require_hw:
+ if dev < 1:
+ ksft_pr(f"FAIL: {role} {direction}: HW offload not engaged "
+ f"(Device={dev}, Sw={sw})")
+ return 1
+ elif dev < 1 and sw < 1:
+ ksft_pr(f"FAIL: {role} {direction}: no TLS activity "
+ f"(Device={dev}, Sw={sw})")
+ return 1
+ return 0
+
+
+def check_min(before, after, key, minimum, role):
+ diff = stat_diff(before, after, key)
+ if diff < minimum:
+ ksft_pr(f"FAIL: {role} {key}: expected >= {minimum}, got {diff}")
+ return 1
+ return 0
+
+
+def check_max(before, after, key, maximum, role):
+ diff = stat_diff(before, after, key)
+ if diff > maximum:
+ ksft_pr(f"FAIL: {role} {key}: expected <= {maximum}, got {diff}")
+ return 1
+ return 0
+
+
+def check_eq_sum(before, after, keys, expected, role):
+ """Verify the sum of `keys` increased by exactly `expected`."""
+ diff = sum(stat_diff(before, after, k) for k in keys)
+ if diff != expected:
+ ksft_pr(f"FAIL: {role} sum({'+'.join(keys)}): "
+ f"expected == {expected}, got {diff}")
+ return 1
+ return 0
+
+
+def check_zero(before, after, key, role):
+ diff = stat_diff(before, after, key)
+ if diff != 0:
+ ksft_pr(f"FAIL: {role} {key} changed by {diff}, expected 0")
+ return 1
+ return 0
+
+
+def verify_tls_counters(stats_before, stats_after, expected_rekeys,
+ tls_role, is_dut, burst=False):
+ """Verify TLS counters on one side of the connection.
+
+ tls_role: 'client' or 'server' (TLS role this side played).
+ is_dut: True for the local DUT; requires HW offload counters.
+ burst: burst mode - only the TLS client rotates its TX key; the TLS
+ server only follows with an RX rotation on KeyUpdate receipt.
+ """
+ role = 'DUT' if is_dut else 'Peer'
+
+ # In burst mode the TLS client only TXs and the TLS server only RXs.
+ # In echo mode both sides drive both directions.
+ with_tx = not burst or tls_role == 'client'
+ with_rx = not burst or tls_role != 'client'
+
+ errors = 0
+ if with_tx:
+ errors += check_path(stats_before, stats_after, 'Tx', role,
+ require_hw=is_dut)
+ if with_rx:
+ errors += check_path(stats_before, stats_after, 'Rx', role,
+ require_hw=is_dut)
+
+ if expected_rekeys > 0:
+ if with_tx:
+ # At most one rekey can be PENDING at socket close (single
+ # TLS_TX_REKEY_PENDING bit), so at most one rekey lands in
+ # TlsTxRekeyAborted; the rest must complete as TlsTxRekeyOk.
+ # Together with the sum equality this pins:
+ # Aborted=0 -> Ok=N, Aborted=1 -> Ok=N-1.
+ errors += check_max(stats_before, stats_after,
+ 'TlsTxRekeyAborted', 1, role)
+ errors += check_eq_sum(stats_before, stats_after,
+ ['TlsTxRekeyOk', 'TlsTxRekeyAborted'],
+ expected_rekeys, role)
+ errors += check_zero(stats_before, stats_after,
+ 'TlsTxRekeyError', role)
+ errors += check_zero(stats_before, stats_after,
+ 'TlsTxRekeyFallback', role)
+ errors += check_zero(stats_before, stats_after,
+ 'TlsTxRekeyInProgress', role)
+ if with_rx:
+ # At most one rekey can be deferred (single dev_add_pending)
+ # at socket close, landing in TlsRxRekeyAborted; the rest
+ # complete as TlsRxRekeyOk. Together with the sum equality this
+ # pins: Aborted=0 -> Ok=N, Aborted=1 -> Ok=N-1.
+ errors += check_max(stats_before, stats_after,
+ 'TlsRxRekeyAborted', 1, role)
+ errors += check_eq_sum(stats_before, stats_after,
+ ['TlsRxRekeyOk', 'TlsRxRekeyAborted'],
+ expected_rekeys, role)
+ errors += check_min(stats_before, stats_after,
+ 'TlsRxRekeyReceived', expected_rekeys, role)
+ errors += check_zero(stats_before, stats_after,
+ 'TlsRxRekeyError', role)
+ errors += check_zero(stats_before, stats_after,
+ 'TlsRxRekeyFallback', role)
+ errors += check_zero(stats_before, stats_after,
+ 'TlsRxRekeyInProgress', role)
+
+ errors += check_zero(stats_before, stats_after, 'TlsDecryptError', role)
+
+ return errors
+
+
+def run_tls_test(cfg, cipher="128", tls_version="1.3", rekey=0,
+ buffer_size=None, random_max=None, burst=False, zc=False,
+ dut_role="client", num_iterations=None):
+ """Run the TLS offload test.
+
+ dut_role: 'client' (default) - DUT runs the TLS client, remote the server.
+ 'server' - swap: DUT listens, remote connects. Used for burst_rx
+ so the DUT's RX path is the one under rekey pressure.
+
+ The DUT (local) is the kernel under test; the remote is just a traffic
+ source/sink and may run any kernel without HW offload. Both sides run
+ kTLS because TLS is pairwise, but verify_tls_counters() requires HW
+ offload only on the DUT (is_dut=True); the peer may use SW kTLS.
+ """
+ port = rand_port()
+ send_size = random_max or buffer_size
+
+ if dut_role == "client":
+ server_bin, server_host = cfg.bin_remote, cfg.remote
+ client_bin, client_host = cfg.bin_local, None
+ client_target = cfg.remote_addr_v['4']
+ else:
+ server_bin, server_host = cfg.bin_local, None
+ client_bin, client_host = cfg.bin_remote, cfg.remote
+ client_target = cfg.addr_v['4']
+
+ server_parts = [f"{server_bin} server -p {port} -c {cipher}",
+ f"-v {tls_version}"]
+ if burst:
+ server_parts.append("-B")
+ if zc:
+ server_parts.append("-Z")
+ if send_size:
+ server_parts.append(f"-b {send_size}")
+ server_cmd = " ".join(server_parts)
+
+ client_parts = [f"{client_bin} client -s {client_target}",
+ f"-p {port} -c {cipher} -v {tls_version}"]
+ if rekey:
+ client_parts.append(f"-k {rekey}")
+ if burst:
+ client_parts.append("-B")
+ if num_iterations:
+ client_parts.append(f"-n {num_iterations}")
+ if random_max:
+ client_parts.append(f"-r {random_max}")
+ elif buffer_size:
+ client_parts.append(f"-b {buffer_size}")
+ client_cmd = " ".join(client_parts)
+
+ cmd_timeout = BURST_TIMEOUT_S if burst else 10
+
+ stats_before_local = read_tls_stats()
+ stats_before_remote = read_tls_stats(host=cfg.remote)
+
+ with bkg(server_cmd, host=server_host, exit_wait=True):
+ wait_port_listen(port, host=server_host)
+ cmd(client_cmd, host=client_host, timeout=cmd_timeout)
+
+ stats_after_local = read_tls_stats()
+ stats_after_remote = read_tls_stats(host=cfg.remote)
+
+ peer_tls_role = 'server' if dut_role == 'client' else 'client'
+
+ dut_errors = verify_tls_counters(stats_before_local, stats_after_local,
+ rekey, dut_role, is_dut=True,
+ burst=burst)
+ peer_errors = verify_tls_counters(stats_before_remote, stats_after_remote,
+ rekey, peer_tls_role, is_dut=False,
+ burst=burst)
+
+ ksft_true(dut_errors == 0,
+ f"DUT TLS counters verified ({dut_errors} failures)")
+ ksft_true(peer_errors == 0,
+ f"Peer TLS counters verified ({peer_errors} failures)")
+
+
+@ksft_variants([
+ KsftNamedVariant("tls13_aes128", "128", "1.3"),
+ KsftNamedVariant("tls13_aes256", "256", "1.3"),
+ KsftNamedVariant("tls12_aes128", "128", "1.2"),
+ KsftNamedVariant("tls12_aes256", "256", "1.2"),
+])
+def test_tls_offload(cfg, cipher, tls_version):
+ run_tls_test(cfg, cipher=cipher, tls_version=tls_version)
+
+
+@ksft_variants([
+ KsftNamedVariant("single", 1),
+ KsftNamedVariant("multiple", 99),
+ KsftNamedVariant("small_buf", 30, 512),
+ KsftNamedVariant("large_buf", 10, 2097152),
+ KsftNamedVariant("random_buf", 20, None, 8192),
+])
+def test_tls_offload_rekey(cfg, rekey, buffer_size=None, random_max=None):
+ run_tls_test(cfg, cipher="128", tls_version="1.3", rekey=rekey,
+ buffer_size=buffer_size, random_max=random_max)
+
+
+# Columns: dut_role zc interval rekeys buffer_size
+@ksft_variants([
+ KsftNamedVariant("burst_tx_rekey_every_1", "client", False, 1, 50, 65536),
+ KsftNamedVariant("burst_tx_rekey_every_1000", "client", False, 1000, 3, 65536),
+ KsftNamedVariant("burst_rx_rekey_every_10", "server", False, 10, 20, 65536),
+ KsftNamedVariant("burst_rx_rekey_every_10000", "server", False, 10000, 1, 32768),
+ KsftNamedVariant("burst_rx_zc_rekey_every_100", "server", True, 100, 10, 65536),
+ KsftNamedVariant("burst_rx_zc_rekey_every_20000", "server", True, 20000, 1, 16384),
+])
+def test_tls_offload_burst(cfg, dut_role, zc, interval, rekeys, buffer_size):
+ run_tls_test(cfg, cipher="128", tls_version="1.3", rekey=rekeys,
+ buffer_size=buffer_size, burst=True, zc=zc, dut_role=dut_role,
+ num_iterations=interval * (rekeys + 1))
+
+
+def main() -> None:
+ with NetDrvEpEnv(__file__, nsim_test=False) as cfg:
+ cfg.bin_local = cfg.test_dir / "tls_hw_offload"
+ if not cfg.bin_local.exists():
+ raise KsftSkipEx(f"tls_hw_offload binary not found at {cfg.bin_local}")
+ cfg.bin_remote = cfg.remote.deploy(cfg.bin_local)
+ cfg.require_ipver("4")
+ check_tls_support(cfg)
+
+ ksft_run([test_tls_offload, test_tls_offload_rekey,
+ test_tls_offload_burst], args=(cfg, ))
+ ksft_exit()
+
+
+if __name__ == "__main__":
+ main()
--
2.25.1
^ permalink raw reply related
* [PATCH v15 8/9] tls: device: add tracepoints for the KeyUpdate path
From: Rishikesh Jethwani @ 2026-07-09 20:53 UTC (permalink / raw)
To: netdev
Cc: saeedm, tariqt, mbloch, borisp, john.fastabend, kuba, sd, davem,
pabeni, edumazet, leon, andrew.gospodarek, Rishikesh Jethwani
In-Reply-To: <20260709205325.1591196-1-rjethwani@purestorage.com>
Add four trace events covering the rekey state machine in
tls_device.c:
tls_device_rekey_start - rekey accepted; inflight=1 means old-key
data is still queued, dev_add deferred
tls_device_rekey_reencrypt - old-key undo pass for a boundary record
tls_device_rekey_done - boundary crossed, old_aead_recv freed,
deferred dev_add issued if pending
tls_device_complete_rekey_fail - TX rekey completion failed in sendmsg;
READY is left set and the next sendmsg
retries
Signed-off-by: Rishikesh Jethwani <rjethwani@purestorage.com>
---
net/tls/tls_device.c | 17 +++++++-
net/tls/trace.h | 98 ++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 113 insertions(+), 2 deletions(-)
diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c
index 74583433b593..aa4ef04e9c43 100644
--- a/net/tls/tls_device.c
+++ b/net/tls/tls_device.c
@@ -761,8 +761,14 @@ int tls_device_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
}
/* Old-key records all ACKed; switch back to HW. */
- if (test_bit(TLS_TX_REKEY_READY, &tls_ctx->flags))
- tls_device_complete_rekey(sk, tls_ctx, true);
+ if (test_bit(TLS_TX_REKEY_READY, &tls_ctx->flags)) {
+ rc = tls_device_complete_rekey(sk, tls_ctx, true);
+ /* On failure the READY bit is left set; the next sendmsg
+ * retries.
+ */
+ if (rc)
+ trace_tls_device_complete_rekey_fail(sk, rc);
+ }
/* Use SW path if rekey is in progress (PENDING) or if HW rekey
* failed (FAILED).
@@ -1244,6 +1250,9 @@ int tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx)
return 0;
}
+ trace_tls_device_rekey_reencrypt(sk, rec_start_seq,
+ ctx->rekey.old_nic_boundary);
+
/* rekey_fixup sets decrypted flags in case if NIC clears
* decrypted flags on auth failure
*/
@@ -1254,6 +1263,8 @@ int tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx)
sw_ctx, tls_ctx);
}
+ trace_tls_device_rekey_done(sk, rec_start_seq,
+ ctx->rekey.old_nic_boundary);
crypto_free_aead(ctx->rekey.old_aead_recv);
ctx->rekey.old_aead_recv = NULL;
@@ -1848,6 +1859,8 @@ int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx,
netdev->tlsdev_ops->tls_dev_rx_rekey_fixup;
context->dev_add_pending = 1;
}
+ trace_tls_device_rekey_start(sk, copied_seq, rcv_nxt,
+ before(copied_seq, rcv_nxt));
}
}
diff --git a/net/tls/trace.h b/net/tls/trace.h
index 2d8ce4ff3265..2a90b77d75e8 100644
--- a/net/tls/trace.h
+++ b/net/tls/trace.h
@@ -192,6 +192,104 @@ TRACE_EVENT(tls_device_tx_resync_send,
)
);
+TRACE_EVENT(tls_device_rekey_start,
+
+ TP_PROTO(struct sock *sk, u32 copied_seq, u32 nic_boundary,
+ bool inflight),
+
+ TP_ARGS(sk, copied_seq, nic_boundary, inflight),
+
+ TP_STRUCT__entry(
+ __field( struct sock *, sk )
+ __field( u32, copied_seq )
+ __field( u32, nic_boundary )
+ __field( bool, inflight )
+ ),
+
+ TP_fast_assign(
+ __entry->sk = sk;
+ __entry->copied_seq = copied_seq;
+ __entry->nic_boundary = nic_boundary;
+ __entry->inflight = inflight;
+ ),
+
+ TP_printk(
+ "sk=%p copied_seq=%u nic_boundary=%u inflight=%d",
+ __entry->sk, __entry->copied_seq, __entry->nic_boundary,
+ __entry->inflight
+ )
+);
+
+TRACE_EVENT(tls_device_rekey_reencrypt,
+
+ TP_PROTO(struct sock *sk, u32 tcp_seq, u32 nic_boundary),
+
+ TP_ARGS(sk, tcp_seq, nic_boundary),
+
+ TP_STRUCT__entry(
+ __field( struct sock *, sk )
+ __field( u32, tcp_seq )
+ __field( u32, nic_boundary )
+ ),
+
+ TP_fast_assign(
+ __entry->sk = sk;
+ __entry->tcp_seq = tcp_seq;
+ __entry->nic_boundary = nic_boundary;
+ ),
+
+ TP_printk(
+ "sk=%p tcp_seq=%u nic_boundary=%u",
+ __entry->sk, __entry->tcp_seq, __entry->nic_boundary
+ )
+);
+
+TRACE_EVENT(tls_device_rekey_done,
+
+ TP_PROTO(struct sock *sk, u32 tcp_seq, u32 nic_boundary),
+
+ TP_ARGS(sk, tcp_seq, nic_boundary),
+
+ TP_STRUCT__entry(
+ __field( struct sock *, sk )
+ __field( u32, tcp_seq )
+ __field( u32, nic_boundary )
+ ),
+
+ TP_fast_assign(
+ __entry->sk = sk;
+ __entry->tcp_seq = tcp_seq;
+ __entry->nic_boundary = nic_boundary;
+ ),
+
+ TP_printk(
+ "sk=%p tcp_seq=%u nic_boundary=%u",
+ __entry->sk, __entry->tcp_seq, __entry->nic_boundary
+ )
+);
+
+TRACE_EVENT(tls_device_complete_rekey_fail,
+
+ TP_PROTO(struct sock *sk, int rc),
+
+ TP_ARGS(sk, rc),
+
+ TP_STRUCT__entry(
+ __field( struct sock *, sk )
+ __field( int, rc )
+ ),
+
+ TP_fast_assign(
+ __entry->sk = sk;
+ __entry->rc = rc;
+ ),
+
+ TP_printk(
+ "sk=%p rc=%d",
+ __entry->sk, __entry->rc
+ )
+);
+
#endif /* _TLS_TRACE_H_ */
#undef TRACE_INCLUDE_PATH
--
2.25.1
^ permalink raw reply related
* [PATCH v15 7/9] tls: device: add RX KeyUpdate support
From: Rishikesh Jethwani @ 2026-07-09 20:53 UTC (permalink / raw)
To: netdev
Cc: saeedm, tariqt, mbloch, borisp, john.fastabend, kuba, sd, davem,
pabeni, edumazet, leon, andrew.gospodarek, Rishikesh Jethwani
In-Reply-To: <20260709205325.1591196-1-rjethwani@purestorage.com>
On RX, the NIC may have already decrypted in-flight records with
the old key before the peer's KeyUpdate is parsed, so the old
AEAD, IV and rec_seq are retained on tls_offload_context_rx.
tls_device_rx_del_key() is called from tls_check_pending_rekey()
when a KeyUpdate record is decoded; otherwise post-KeyUpdate records
(carrying new-key wire encryption) would be decrypted with the retired key.
tls_device_decrypted() classifies records by old_nic_boundary:
- after the boundary: new-key record; drop the old key.
- before, fully encrypted: advance old_rec_seq, let SW AEAD decrypt.
- before, (partially) decrypted: reencrypt with the old key so SW
AEAD can decrypt with the new key.
For mixed records the NIC may have cleared skb->decrypted on auth failure;
the optional tls_dev_rx_rekey_fixup callback re-asserts it,
then the record is reencrypted with the old key so the SW AEAD can
decrypt it with the new key.
The new key's tls_dev_add is deferred until the old key is fully
consumed: tls_set_device_offload_rx() sets dev_add_pending while
old_aead_recv is retained, and tls_device_deferred_dev_add_rx()
installs the new key once copied_seq crosses old_nic_boundary.
Tested on Mellanox ConnectX-6 Dx (Crypto Enabled) with multiple
TLS 1.3 RX KeyUpdate cycles.
Signed-off-by: Rishikesh Jethwani <rjethwani@purestorage.com>
---
.../mellanox/mlx5/core/en_accel/ktls.c | 10 +
include/net/tls.h | 10 +
include/uapi/linux/snmp.h | 3 +
net/tls/tls.h | 9 +-
net/tls/tls_device.c | 314 ++++++++++++++++--
net/tls/tls_main.c | 46 +--
net/tls/tls_proc.c | 3 +
net/tls/tls_sw.c | 1 +
8 files changed, 346 insertions(+), 50 deletions(-)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c
index 337201f12895..71c9d3ca947d 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c
@@ -86,10 +86,20 @@ static int mlx5e_ktls_resync(struct net_device *netdev,
return 0;
}
+static void mlx5e_ktls_rx_rekey_fixup(struct sk_buff *skb)
+{
+ struct sk_buff *frag_iter;
+
+ skb->decrypted = 1;
+ skb_walk_frags(skb, frag_iter)
+ frag_iter->decrypted = 1;
+}
+
static const struct tlsdev_ops mlx5e_ktls_ops = {
.tls_dev_add = mlx5e_ktls_add,
.tls_dev_del = mlx5e_ktls_del,
.tls_dev_resync = mlx5e_ktls_resync,
+ .tls_dev_rx_rekey_fixup = mlx5e_ktls_rx_rekey_fixup,
};
bool mlx5e_is_ktls_rx(struct mlx5_core_dev *mdev)
diff --git a/include/net/tls.h b/include/net/tls.h
index 2c6e314355fd..de04dfcf7a8c 100644
--- a/include/net/tls.h
+++ b/include/net/tls.h
@@ -320,6 +320,7 @@ struct tlsdev_ops {
int (*tls_dev_resync)(struct net_device *netdev,
struct sock *sk, u32 seq, u8 *rcd_sn,
enum tls_offload_ctx_dir direction);
+ void (*tls_dev_rx_rekey_fixup)(struct sk_buff *skb);
};
enum tls_offload_sync_type {
@@ -348,6 +349,15 @@ struct tls_offload_context_rx {
u8 resync_nh_reset:1;
/* CORE_NEXT_HINT-only member, but use the hole here */
u8 resync_nh_do_now:1;
+ /* tls_dev_add deferred until old key is freed */
+ u8 dev_add_pending:1;
+ struct {
+ struct crypto_aead *old_aead_recv; /* old key AEAD cipher */
+ char old_iv[TLS_MAX_IV_SIZE + TLS_MAX_SALT_SIZE]; /* old key IV */
+ char old_rec_seq[TLS_MAX_REC_SEQ_SIZE]; /* old key TLS record seq */
+ u32 old_nic_boundary; /* TCP seq: NIC switched to next key */
+ void (*rekey_fixup)(struct sk_buff *skb);
+ } rekey;
union {
/* TLS_OFFLOAD_SYNC_TYPE_DRIVER_REQ */
struct {
diff --git a/include/uapi/linux/snmp.h b/include/uapi/linux/snmp.h
index 119fbd49179c..284f879c92cc 100644
--- a/include/uapi/linux/snmp.h
+++ b/include/uapi/linux/snmp.h
@@ -370,8 +370,11 @@ enum
LINUX_MIB_TLSTXREKEYERROR, /* TlsTxRekeyError */
LINUX_MIB_TLSRXREKEYRECEIVED, /* TlsRxRekeyReceived */
LINUX_MIB_TLSTXREKEYFALLBACK, /* TlsTxRekeyFallback */
+ LINUX_MIB_TLSRXREKEYFALLBACK, /* TlsRxRekeyFallback */
LINUX_MIB_TLSTXREKEYINPROGRESS, /* TlsTxRekeyInProgress */
+ LINUX_MIB_TLSRXREKEYINPROGRESS, /* TlsRxRekeyInProgress */
LINUX_MIB_TLSTXREKEYABORTED, /* TlsTxRekeyAborted */
+ LINUX_MIB_TLSRXREKEYABORTED, /* TlsRxRekeyAborted */
__LINUX_MIB_TLSMAX
};
diff --git a/net/tls/tls.h b/net/tls/tls.h
index 6e8fafca42ca..0983b29dc46d 100644
--- a/net/tls/tls.h
+++ b/net/tls/tls.h
@@ -241,8 +241,10 @@ void tls_device_cleanup(void);
int tls_set_device_offload(struct sock *sk,
struct tls_crypto_info *crypto_info);
void tls_device_free_resources_tx(struct sock *sk);
-int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx);
+int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx,
+ struct tls_crypto_info *crypto_info);
void tls_device_offload_cleanup_rx(struct sock *sk);
+void tls_device_rx_del_key(struct sock *sk, struct tls_context *ctx);
void tls_device_rx_resync_new_rec(struct sock *sk, u32 rcd_len, u32 seq);
int tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx);
#else
@@ -258,13 +260,16 @@ tls_set_device_offload(struct sock *sk, struct tls_crypto_info *crypto_info)
static inline void tls_device_free_resources_tx(struct sock *sk) {}
static inline int
-tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx)
+tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx,
+ struct tls_crypto_info *crypto_info)
{
return -EOPNOTSUPP;
}
static inline void tls_device_offload_cleanup_rx(struct sock *sk) {}
static inline void
+tls_device_rx_del_key(struct sock *sk, struct tls_context *ctx) {}
+static inline void
tls_device_rx_resync_new_rec(struct sock *sk, u32 rcd_len, u32 seq) {}
static inline int
diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c
index ef837bbd9560..74583433b593 100644
--- a/net/tls/tls_device.c
+++ b/net/tls/tls_device.c
@@ -66,8 +66,14 @@ static void tls_device_free_ctx(struct tls_context *ctx)
kfree(offload_ctx);
}
- if (ctx->rx_conf == TLS_HW)
- kfree(tls_offload_ctx_rx(ctx));
+ if (ctx->rx_conf == TLS_HW) {
+ struct tls_offload_context_rx *offload_ctx =
+ tls_offload_ctx_rx(ctx);
+
+ memzero_explicit(&offload_ctx->rekey,
+ sizeof(offload_ctx->rekey));
+ kfree(offload_ctx);
+ }
tls_ctx_free(NULL, ctx);
}
@@ -185,6 +191,82 @@ static void tls_device_commit_start_marker(struct sock *sk,
tcp_write_collapse_fence(sk);
}
+static int tls_device_dev_add_rx(struct sock *sk, struct tls_context *tls_ctx,
+ struct net_device *netdev,
+ struct tls_crypto_info *crypto_info,
+ u32 cur_seq, bool is_rekey)
+{
+ const struct tls_cipher_desc *cipher_desc;
+ char *rec_seq;
+ int rc;
+
+ cipher_desc = get_cipher_desc(crypto_info->cipher_type);
+ DEBUG_NET_WARN_ON_ONCE(!cipher_desc || !cipher_desc->offloadable);
+
+ rc = netdev->tlsdev_ops->tls_dev_add(netdev, sk,
+ TLS_OFFLOAD_CTX_DIR_RX,
+ crypto_info, cur_seq);
+ rec_seq = crypto_info_rec_seq(crypto_info, cipher_desc);
+ trace_tls_device_offload_set(sk, TLS_OFFLOAD_CTX_DIR_RX,
+ cur_seq, rec_seq, rc);
+ if (!rc) {
+ clear_bit(TLS_RX_DEV_DEGRADED, &tls_ctx->flags);
+ clear_bit(TLS_RX_DEV_CLOSED, &tls_ctx->flags);
+ if (is_rekey)
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYOK);
+ } else if (is_rekey) {
+ set_bit(TLS_RX_DEV_DEGRADED, &tls_ctx->flags);
+ set_bit(TLS_RX_DEV_CLOSED, &tls_ctx->flags);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYFALLBACK);
+ }
+ return rc;
+}
+
+static void tls_device_deferred_dev_add_rx(struct sock *sk,
+ struct tls_context *tls_ctx,
+ struct tls_offload_context_rx *ctx)
+{
+ struct net_device *netdev;
+
+ ctx->dev_add_pending = 0;
+
+ down_read(&device_offload_lock);
+ netdev = rcu_dereference_protected(tls_ctx->netdev,
+ lockdep_is_held(&device_offload_lock));
+ if (netdev)
+ tls_device_dev_add_rx(sk, tls_ctx, netdev,
+ &tls_ctx->crypto_recv.info,
+ tcp_sk(sk)->copied_seq, true);
+ else
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYFALLBACK);
+ up_read(&device_offload_lock);
+ TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYINPROGRESS);
+}
+
+void tls_device_rx_del_key(struct sock *sk, struct tls_context *ctx)
+{
+ struct net_device *netdev;
+
+ if (ctx->rx_conf != TLS_HW)
+ return;
+ if (test_bit(TLS_RX_DEV_CLOSED, &ctx->flags))
+ return;
+
+ down_read(&device_offload_lock);
+ netdev = rcu_dereference_protected(ctx->netdev,
+ lockdep_is_held(&device_offload_lock));
+ if (!netdev) {
+ up_read(&device_offload_lock);
+ return;
+ }
+
+ set_bit(TLS_RX_DEV_CLOSED, &ctx->flags);
+ synchronize_net();
+ netdev->tlsdev_ops->tls_dev_del(netdev, ctx,
+ TLS_OFFLOAD_CTX_DIR_RX);
+ up_read(&device_offload_lock);
+}
+
static void destroy_record(struct tls_record_info *record)
{
int i;
@@ -893,6 +975,8 @@ void tls_device_rx_resync_new_rec(struct sock *sk, u32 rcd_len, u32 seq)
return;
if (unlikely(test_bit(TLS_RX_DEV_DEGRADED, &tls_ctx->flags)))
return;
+ if (unlikely(test_bit(TLS_RX_DEV_CLOSED, &tls_ctx->flags)))
+ return;
prot = &tls_ctx->prot_info;
rx_ctx = tls_offload_ctx_rx(tls_ctx);
@@ -1082,13 +1166,56 @@ tls_device_reencrypt(struct sock *sk, struct tls_context *tls_ctx)
return err;
}
+/*
+ * Temporarily swap in the old key, run
+ * tls_device_reencrypt(), then restore the current key.
+ */
+static int tls_device_reencrypt_old_key(struct sock *sk,
+ struct tls_offload_context_rx *ctx,
+ struct tls_sw_context_rx *sw_ctx,
+ struct tls_context *tls_ctx)
+{
+ struct crypto_aead *saved_aead = sw_ctx->aead_recv;
+ char saved_iv[TLS_MAX_IV_SIZE + TLS_MAX_SALT_SIZE];
+ char saved_rec_seq[TLS_MAX_REC_SEQ_SIZE];
+ int ret;
+
+ memcpy(saved_iv, tls_ctx->rx.iv, sizeof(saved_iv));
+ memcpy(saved_rec_seq, tls_ctx->rx.rec_seq, sizeof(saved_rec_seq));
+
+ sw_ctx->aead_recv = ctx->rekey.old_aead_recv;
+ memcpy(tls_ctx->rx.iv, ctx->rekey.old_iv, sizeof(ctx->rekey.old_iv));
+ memcpy(tls_ctx->rx.rec_seq, ctx->rekey.old_rec_seq,
+ sizeof(ctx->rekey.old_rec_seq));
+
+ ret = tls_device_reencrypt(sk, tls_ctx);
+
+ memcpy(ctx->rekey.old_rec_seq, tls_ctx->rx.rec_seq,
+ sizeof(ctx->rekey.old_rec_seq));
+
+ sw_ctx->aead_recv = saved_aead;
+ memcpy(tls_ctx->rx.iv, saved_iv, sizeof(saved_iv));
+ memcpy(tls_ctx->rx.rec_seq, saved_rec_seq, sizeof(saved_rec_seq));
+
+ if (ret)
+ return ret;
+
+ tls_bigint_increment(ctx->rekey.old_rec_seq,
+ tls_ctx->prot_info.rec_seq_size);
+ ctx->resync_nh_reset = 1;
+
+ return 0;
+}
+
int tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx)
{
struct tls_offload_context_rx *ctx = tls_offload_ctx_rx(tls_ctx);
struct tls_sw_context_rx *sw_ctx = tls_sw_ctx_rx(tls_ctx);
struct sk_buff *skb = tls_strp_msg(sw_ctx);
+ u32 copied_seq = tcp_sk(sk)->copied_seq;
struct strp_msg *rxm = strp_msg(skb);
int is_decrypted, is_encrypted;
+ u32 rec_start_seq;
if (!tls_strp_msg_mixed_decrypted(sw_ctx)) {
is_decrypted = skb->decrypted;
@@ -1098,10 +1225,42 @@ int tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx)
is_encrypted = 0;
}
- trace_tls_device_decrypted(sk, tcp_sk(sk)->copied_seq - rxm->full_len,
+ rec_start_seq = sw_ctx->strp.copy_mode
+ ? copied_seq - rxm->full_len
+ : copied_seq;
+
+ trace_tls_device_decrypted(sk, rec_start_seq,
tls_ctx->rx.rec_seq, rxm->full_len,
is_encrypted, is_decrypted);
+ if (unlikely(ctx->rekey.old_aead_recv)) {
+ bool before_nic_boundary =
+ before(rec_start_seq, ctx->rekey.old_nic_boundary);
+
+ if (before_nic_boundary) {
+ if (is_encrypted) {
+ tls_bigint_increment(ctx->rekey.old_rec_seq,
+ tls_ctx->prot_info.rec_seq_size);
+ return 0;
+ }
+
+ /* rekey_fixup sets decrypted flags in case if NIC clears
+ * decrypted flags on auth failure
+ */
+ if (!is_decrypted && ctx->rekey.rekey_fixup)
+ ctx->rekey.rekey_fixup(skb);
+
+ return tls_device_reencrypt_old_key(sk, ctx,
+ sw_ctx, tls_ctx);
+ }
+
+ crypto_free_aead(ctx->rekey.old_aead_recv);
+ ctx->rekey.old_aead_recv = NULL;
+
+ if (ctx->dev_add_pending)
+ tls_device_deferred_dev_add_rx(sk, tls_ctx, ctx);
+ }
+
if (unlikely(test_bit(TLS_RX_DEV_DEGRADED, &tls_ctx->flags))) {
if (likely(is_encrypted || is_decrypted))
return is_decrypted;
@@ -1580,13 +1739,31 @@ int tls_set_device_offload(struct sock *sk,
return rc;
}
-int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx)
+int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx,
+ struct tls_crypto_info *new_crypto_info)
{
- struct tls12_crypto_info_aes_gcm_128 *info;
+ struct tls_crypto_info *crypto_info, *src_crypto_info;
+ const struct tls_cipher_desc *cipher_desc;
+ u32 copied_seq = tcp_sk(sk)->copied_seq;
struct tls_offload_context_rx *context;
struct net_device *netdev;
+ bool was_dev_add_pending;
+ bool moved_aead_recv = false;
int rc = 0;
+ /* Rekey is only supported for connections that are already
+ * using HW offload. For SW offload connections, the caller
+ * should fall back to tls_set_sw_offload() for rekey.
+ */
+ if (new_crypto_info && ctx->rx_conf != TLS_HW)
+ return -EINVAL;
+
+ crypto_info = &ctx->crypto_recv.info;
+ src_crypto_info = new_crypto_info ?: crypto_info;
+ cipher_desc = get_cipher_desc(src_crypto_info->cipher_type);
+ if (!cipher_desc || !cipher_desc->offloadable)
+ return -EINVAL;
+
netdev = get_netdev_for_sock(sk);
if (!netdev) {
pr_err_ratelimited("%s: netdev not found\n", __func__);
@@ -1612,29 +1789,88 @@ int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx)
goto release_lock;
}
- context = kzalloc_obj(*context);
- if (!context) {
- rc = -ENOMEM;
- goto release_lock;
+ if (!new_crypto_info) {
+ context = kzalloc_obj(*context);
+ if (!context) {
+ rc = -ENOMEM;
+ goto release_lock;
+ }
+ ctx->priv_ctx_rx = context;
+ } else {
+ context = tls_offload_ctx_rx(ctx);
}
+ was_dev_add_pending = context->dev_add_pending;
context->resync_nh_reset = 1;
- ctx->priv_ctx_rx = context;
- rc = tls_sw_ctx_init(sk, 0, NULL);
+ if (new_crypto_info) {
+ struct tls_sw_context_rx *sw_ctx = tls_sw_ctx_rx(ctx);
+
+ if (!test_bit(TLS_RX_DEV_CLOSED, &ctx->flags)) {
+ set_bit(TLS_RX_DEV_CLOSED, &ctx->flags);
+ synchronize_net();
+ netdev->tlsdev_ops->tls_dev_del(netdev, ctx,
+ TLS_OFFLOAD_CTX_DIR_RX);
+ }
+
+ if (context->rekey.old_aead_recv &&
+ before(copied_seq, context->rekey.old_nic_boundary)) {
+ /* Previous rekey still draining. Keep rekey.old_aead_recv,
+ * it is the only key that can undo the NIC-XOR on queued
+ * records. sw_ctx->aead_recv may be re-setkey'd by
+ * tls_sw_ctx_init(); that intermediate key was never on
+ * the NIC and its wire era is drained, so it is needed
+ * for neither undo nor AEAD. Defer dev_add; the new key
+ * is installed once copied_seq crosses rekey.old_nic_boundary.
+ */
+ context->dev_add_pending = 1;
+ } else {
+ u32 rcv_nxt;
+
+ if (context->rekey.old_aead_recv) {
+ crypto_free_aead(context->rekey.old_aead_recv);
+ context->rekey.old_aead_recv = NULL;
+ }
+
+ /* flush the backlog so rcv_nxt is accurate */
+ __sk_flush_backlog(sk);
+ rcv_nxt = tcp_sk(sk)->rcv_nxt;
+
+ if (before(copied_seq, rcv_nxt)) {
+ context->rekey.old_aead_recv = sw_ctx->aead_recv;
+ sw_ctx->aead_recv = NULL;
+ moved_aead_recv = true;
+ memcpy(context->rekey.old_iv, ctx->rx.iv,
+ sizeof(context->rekey.old_iv));
+ memcpy(context->rekey.old_rec_seq, ctx->rx.rec_seq,
+ sizeof(context->rekey.old_rec_seq));
+ context->rekey.old_nic_boundary = rcv_nxt;
+ context->rekey.rekey_fixup =
+ netdev->tlsdev_ops->tls_dev_rx_rekey_fixup;
+ context->dev_add_pending = 1;
+ }
+ }
+ }
+
+ rc = tls_sw_ctx_init(sk, 0, new_crypto_info);
if (rc)
goto release_ctx;
- rc = netdev->tlsdev_ops->tls_dev_add(netdev, sk, TLS_OFFLOAD_CTX_DIR_RX,
- &ctx->crypto_recv.info,
- tcp_sk(sk)->copied_seq);
- info = (void *)&ctx->crypto_recv.info;
- trace_tls_device_offload_set(sk, TLS_OFFLOAD_CTX_DIR_RX,
- tcp_sk(sk)->copied_seq, info->rec_seq, rc);
- if (rc)
- goto free_sw_resources;
+ if (!context->dev_add_pending) {
+ rc = tls_device_dev_add_rx(sk, ctx, netdev, src_crypto_info,
+ copied_seq, !!new_crypto_info);
+ if (!new_crypto_info) {
+ if (rc)
+ goto free_sw_resources;
+ tls_device_attach(ctx, sk, netdev);
+ }
+ } else if (!was_dev_add_pending) {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYINPROGRESS);
+ } else {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYOK);
+ }
+
+ tls_sw_ctx_finalize(sk, 0, new_crypto_info);
- tls_device_attach(ctx, sk, netdev);
- tls_sw_ctx_finalize(sk, 0, NULL);
up_read(&device_offload_lock);
dev_put(netdev);
@@ -1646,7 +1882,18 @@ int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx)
tls_sw_free_resources_rx(sk);
down_read(&device_offload_lock);
release_ctx:
- ctx->priv_ctx_rx = NULL;
+ if (!new_crypto_info) {
+ ctx->priv_ctx_rx = NULL;
+ } else {
+ context->dev_add_pending = was_dev_add_pending;
+ if (moved_aead_recv) {
+ struct tls_sw_context_rx *sw_ctx = tls_sw_ctx_rx(ctx);
+
+ crypto_free_aead(sw_ctx->aead_recv);
+ sw_ctx->aead_recv = context->rekey.old_aead_recv;
+ context->rekey.old_aead_recv = NULL;
+ }
+ }
release_lock:
up_read(&device_offload_lock);
release_netdev:
@@ -1657,6 +1904,7 @@ int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx)
void tls_device_offload_cleanup_rx(struct sock *sk)
{
struct tls_context *tls_ctx = tls_get_ctx(sk);
+ struct tls_offload_context_rx *rx_ctx;
struct net_device *netdev;
down_read(&device_offload_lock);
@@ -1665,8 +1913,9 @@ void tls_device_offload_cleanup_rx(struct sock *sk)
if (!netdev)
goto out;
- netdev->tlsdev_ops->tls_dev_del(netdev, tls_ctx,
- TLS_OFFLOAD_CTX_DIR_RX);
+ if (!test_bit(TLS_RX_DEV_CLOSED, &tls_ctx->flags))
+ netdev->tlsdev_ops->tls_dev_del(netdev, tls_ctx,
+ TLS_OFFLOAD_CTX_DIR_RX);
if (tls_ctx->tx_conf != TLS_HW) {
dev_put(netdev);
@@ -1676,6 +1925,19 @@ void tls_device_offload_cleanup_rx(struct sock *sk)
}
out:
up_read(&device_offload_lock);
+
+ rx_ctx = tls_offload_ctx_rx(tls_ctx);
+ if (rx_ctx && rx_ctx->rekey.old_aead_recv) {
+ crypto_free_aead(rx_ctx->rekey.old_aead_recv);
+ rx_ctx->rekey.old_aead_recv = NULL;
+ }
+
+ if (rx_ctx && rx_ctx->dev_add_pending) {
+ rx_ctx->dev_add_pending = 0;
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYABORTED);
+ TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYINPROGRESS);
+ }
+
tls_sw_release_resources_rx(sk);
}
@@ -1733,9 +1995,11 @@ static int tls_device_down(struct net_device *netdev)
set_bit(TLS_TX_DEV_CLOSED, &ctx->flags);
}
if (ctx->rx_conf == TLS_HW &&
- !test_bit(TLS_RX_DEV_CLOSED, &ctx->flags))
+ !test_bit(TLS_RX_DEV_CLOSED, &ctx->flags)) {
netdev->tlsdev_ops->tls_dev_del(netdev, ctx,
TLS_OFFLOAD_CTX_DIR_RX);
+ set_bit(TLS_RX_DEV_CLOSED, &ctx->flags);
+ }
dev_put(netdev);
diff --git a/net/tls/tls_main.c b/net/tls/tls_main.c
index b51f9ea1ed6c..4519c8943ebf 100644
--- a/net/tls/tls_main.c
+++ b/net/tls/tls_main.c
@@ -757,37 +757,37 @@ static int do_tls_setsockopt_conf(struct sock *sk, sockptr_t optval,
conf = TLS_SW;
}
} else {
- if (update && ctx->rx_conf == TLS_HW) {
- rc = -EOPNOTSUPP;
- goto err_crypto_info;
- }
-
- if (!update) {
- rc = tls_set_device_offload_rx(sk, ctx);
- conf = TLS_HW;
- if (!rc) {
+ rc = tls_set_device_offload_rx(sk, ctx,
+ update ? crypto_info : NULL);
+ conf = TLS_HW;
+ if (!rc) {
+ if (!update) {
TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXDEVICE);
TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXDEVICE);
- tls_sw_strparser_arm(sk, ctx);
- goto out;
}
- }
-
- rc = tls_set_sw_offload(sk, 0, update ? crypto_info : NULL);
- if (rc)
+ } else if (update && ctx->rx_conf == TLS_HW) {
+ /* HW rekey failed - return the actual error.
+ * Cannot fall back to SW for an existing HW connection.
+ */
goto err_crypto_info;
-
- if (update) {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYOK);
} else {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXSW);
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXSW);
- tls_sw_strparser_arm(sk, ctx);
+ rc = tls_set_sw_offload(sk, 0,
+ update ? crypto_info : NULL);
+ if (rc)
+ goto err_crypto_info;
+
+ if (update) {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYOK);
+ } else {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXSW);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXSW);
+ }
+ conf = TLS_SW;
}
- conf = TLS_SW;
+ if (!update)
+ tls_sw_strparser_arm(sk, ctx);
}
-out:
if (tx)
ctx->tx_conf = conf;
else
diff --git a/net/tls/tls_proc.c b/net/tls/tls_proc.c
index adca8561c7da..ae2322481c71 100644
--- a/net/tls/tls_proc.c
+++ b/net/tls/tls_proc.c
@@ -28,8 +28,11 @@ static const struct snmp_mib tls_mib_list[] = {
SNMP_MIB_ITEM("TlsTxRekeyError", LINUX_MIB_TLSTXREKEYERROR),
SNMP_MIB_ITEM("TlsRxRekeyReceived", LINUX_MIB_TLSRXREKEYRECEIVED),
SNMP_MIB_ITEM("TlsTxRekeyFallback", LINUX_MIB_TLSTXREKEYFALLBACK),
+ SNMP_MIB_ITEM("TlsRxRekeyFallback", LINUX_MIB_TLSRXREKEYFALLBACK),
SNMP_MIB_ITEM("TlsTxRekeyInProgress", LINUX_MIB_TLSTXREKEYINPROGRESS),
+ SNMP_MIB_ITEM("TlsRxRekeyInProgress", LINUX_MIB_TLSRXREKEYINPROGRESS),
SNMP_MIB_ITEM("TlsTxRekeyAborted", LINUX_MIB_TLSTXREKEYABORTED),
+ SNMP_MIB_ITEM("TlsRxRekeyAborted", LINUX_MIB_TLSRXREKEYABORTED),
};
static int tls_statistics_seq_show(struct seq_file *seq, void *v)
diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c
index 0364fae8a7e0..445517d8bc90 100644
--- a/net/tls/tls_sw.c
+++ b/net/tls/tls_sw.c
@@ -1534,6 +1534,7 @@ static int tls_check_pending_rekey(struct sock *sk, struct tls_context *ctx,
if (hs_type == TLS_HANDSHAKE_KEYUPDATE) {
struct tls_sw_context_rx *rx_ctx = ctx->priv_ctx_rx;
+ tls_device_rx_del_key(sk, ctx);
WRITE_ONCE(rx_ctx->key_update_pending, true);
TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYRECEIVED);
}
--
2.25.1
^ permalink raw reply related
* [PATCH v15 6/9] tls: device: add TX KeyUpdate support
From: Rishikesh Jethwani @ 2026-07-09 20:53 UTC (permalink / raw)
To: netdev
Cc: saeedm, tariqt, mbloch, borisp, john.fastabend, kuba, sd, davem,
pabeni, edumazet, leon, andrew.gospodarek, Rishikesh Jethwani
In-Reply-To: <20260709205325.1591196-1-rjethwani@purestorage.com>
The NIC key cannot be replaced while HW-offloaded records
are still unacked. tls_device_start_rekey() installs a temporary SW
context with the new key and redirects sendmsg through
tls_sw_sendmsg_locked. If no records are pending,
tls_device_complete_rekey() runs inline during setsockopt; otherwise
tls_tcp_clean_acked sets REKEY_READY once all old-key records are ACKed
and the next sendmsg completes the rekey, flushing SW records and
reinstalling HW offload at the current write_seq. A KeyUpdate
arriving while one is pending re-keys the SW AEAD in place; if the
HW reinstall fails the socket stays in SW mode (REKEY_FAILED).
Tested on Mellanox ConnectX-6 Dx (Crypto Enabled) with multiple
TLS 1.3 TX KeyUpdate cycles.
Signed-off-by: Rishikesh Jethwani <rjethwani@purestorage.com>
---
include/net/tls.h | 42 ++++
include/uapi/linux/snmp.h | 3 +
net/tls/tls.h | 8 +-
net/tls/tls_device.c | 384 ++++++++++++++++++++++++++++++++--
net/tls/tls_device_fallback.c | 24 +++
net/tls/tls_main.c | 81 ++++---
net/tls/tls_proc.c | 3 +
net/tls/tls_sw.c | 22 +-
8 files changed, 526 insertions(+), 41 deletions(-)
diff --git a/include/net/tls.h b/include/net/tls.h
index eb258bcd62bc..2c6e314355fd 100644
--- a/include/net/tls.h
+++ b/include/net/tls.h
@@ -185,6 +185,13 @@ struct tls_offload_context_tx {
void (*sk_destruct)(struct sock *sk);
struct work_struct destruct_work;
struct tls_context *ctx;
+
+ struct {
+ struct tls_sw_context_tx sw; /* SW context for new key */
+ struct cipher_context tx; /* IV, rec_seq for new key */
+ union tls_crypto_context crypto_send; /* Crypto for new key */
+ } rekey;
+
/* The TLS layer reserves room for driver specific state
* Currently the belief is that there is not enough
* driver specific state to justify another layer of indirection
@@ -209,6 +216,21 @@ enum tls_context_flags {
* tls_dev_del call in tls_device_down if it happens simultaneously.
*/
TLS_RX_DEV_CLOSED = 2,
+ /* Flag for TX HW context deleted during failed rekey.
+ * Prevents double tls_dev_del in cleanup paths.
+ */
+ TLS_TX_DEV_CLOSED = 3,
+ /* TX rekey is pending, waiting for old-key data to be ACKed.
+ * While set, new data uses SW path with new key, HW keeps old key
+ * for retransmissions.
+ */
+ TLS_TX_REKEY_PENDING = 4,
+ /* All old-key data has been ACKed, ready to install new key in HW. */
+ TLS_TX_REKEY_READY = 5,
+ /* HW rekey failed, permanently stay in SW encrypt mode.
+ * Prevents tls_tcp_clean_acked from re-setting TLS_TX_REKEY_READY.
+ */
+ TLS_TX_REKEY_FAILED = 6,
};
struct tls_prot_info {
@@ -257,6 +279,17 @@ struct tls_context {
*/
unsigned long flags;
+ struct {
+ /* TCP sequence number boundary for pending rekey.
+ * Packets with seq < this use old key, >= use new key.
+ */
+ u32 boundary_seq;
+
+ /* Pointers to rekey contexts for SW encryption with new key */
+ struct tls_sw_context_tx *sw_ctx;
+ struct cipher_context *cipher_ctx;
+ } rekey;
+
/* cache cold stuff */
struct proto *sk_proto;
struct sock *sk;
@@ -389,12 +422,18 @@ static inline struct tls_sw_context_rx *tls_sw_ctx_rx(
static inline struct tls_sw_context_tx *tls_sw_ctx_tx(
const struct tls_context *tls_ctx)
{
+ if (unlikely(tls_ctx->rekey.sw_ctx))
+ return tls_ctx->rekey.sw_ctx;
+
return (struct tls_sw_context_tx *)tls_ctx->priv_ctx_tx;
}
static inline struct cipher_context *tls_tx_cipher_ctx(
const struct tls_context *tls_ctx)
{
+ if (unlikely(tls_ctx->rekey.cipher_ctx))
+ return tls_ctx->rekey.cipher_ctx;
+
return (struct cipher_context *)&tls_ctx->tx;
}
@@ -510,6 +549,9 @@ struct sk_buff *tls_encrypt_skb(struct sk_buff *skb);
#ifdef CONFIG_TLS_DEVICE
void tls_device_sk_destruct(struct sock *sk);
void tls_offload_tx_resync_request(struct sock *sk, u32 got_seq, u32 exp_seq);
+struct sk_buff *
+tls_validate_xmit_skb_rekey(struct sock *sk, struct net_device *dev,
+ struct sk_buff *skb);
static inline bool tls_is_sk_rx_device_offloaded(struct sock *sk)
{
diff --git a/include/uapi/linux/snmp.h b/include/uapi/linux/snmp.h
index 49f5640092a0..119fbd49179c 100644
--- a/include/uapi/linux/snmp.h
+++ b/include/uapi/linux/snmp.h
@@ -369,6 +369,9 @@ enum
LINUX_MIB_TLSTXREKEYOK, /* TlsTxRekeyOk */
LINUX_MIB_TLSTXREKEYERROR, /* TlsTxRekeyError */
LINUX_MIB_TLSRXREKEYRECEIVED, /* TlsRxRekeyReceived */
+ LINUX_MIB_TLSTXREKEYFALLBACK, /* TlsTxRekeyFallback */
+ LINUX_MIB_TLSTXREKEYINPROGRESS, /* TlsTxRekeyInProgress */
+ LINUX_MIB_TLSTXREKEYABORTED, /* TlsTxRekeyAborted */
__LINUX_MIB_TLSMAX
};
diff --git a/net/tls/tls.h b/net/tls/tls.h
index bf46d3471da6..6e8fafca42ca 100644
--- a/net/tls/tls.h
+++ b/net/tls/tls.h
@@ -157,7 +157,10 @@ void tls_update_rx_zc_capable(struct tls_context *tls_ctx);
void tls_sw_strparser_arm(struct sock *sk, struct tls_context *ctx);
void tls_sw_strparser_done(struct tls_context *tls_ctx);
int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
+int tls_sw_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size);
void tls_sw_ctx_tx_init(struct sock *sk, struct tls_sw_context_tx *sw_ctx);
+int tls_sw_drain_tx(struct sock *sk, struct tls_context *ctx);
+int tls_encrypt_async_wait(struct tls_sw_context_tx *ctx);
void tls_sw_splice_eof(struct socket *sock);
void tls_sw_cancel_work_tx(struct tls_context *tls_ctx);
void tls_sw_release_resources_tx(struct sock *sk);
@@ -235,7 +238,8 @@ static inline bool tls_strp_msg_mixed_decrypted(struct tls_sw_context_rx *ctx)
#ifdef CONFIG_TLS_DEVICE
int tls_device_init(void);
void tls_device_cleanup(void);
-int tls_set_device_offload(struct sock *sk);
+int tls_set_device_offload(struct sock *sk,
+ struct tls_crypto_info *crypto_info);
void tls_device_free_resources_tx(struct sock *sk);
int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx);
void tls_device_offload_cleanup_rx(struct sock *sk);
@@ -246,7 +250,7 @@ static inline int tls_device_init(void) { return 0; }
static inline void tls_device_cleanup(void) {}
static inline int
-tls_set_device_offload(struct sock *sk)
+tls_set_device_offload(struct sock *sk, struct tls_crypto_info *crypto_info)
{
return -EOPNOTSUPP;
}
diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c
index 7a98d2f6cbd3..ef837bbd9560 100644
--- a/net/tls/tls_device.c
+++ b/net/tls/tls_device.c
@@ -57,8 +57,14 @@ static struct page *dummy_page;
static void tls_device_free_ctx(struct tls_context *ctx)
{
- if (ctx->tx_conf == TLS_HW)
- kfree(tls_offload_ctx_tx(ctx));
+ if (ctx->tx_conf == TLS_HW) {
+ struct tls_offload_context_tx *offload_ctx =
+ tls_offload_ctx_tx(ctx);
+
+ memzero_explicit(&offload_ctx->rekey,
+ sizeof(offload_ctx->rekey));
+ kfree(offload_ctx);
+ }
if (ctx->rx_conf == TLS_HW)
kfree(tls_offload_ctx_rx(ctx));
@@ -79,7 +85,9 @@ static void tls_device_tx_del_task(struct work_struct *work)
netdev = rcu_dereference_protected(ctx->netdev,
!refcount_read(&ctx->refcount));
- netdev->tlsdev_ops->tls_dev_del(netdev, ctx, TLS_OFFLOAD_CTX_DIR_TX);
+ if (!test_bit(TLS_TX_DEV_CLOSED, &ctx->flags))
+ netdev->tlsdev_ops->tls_dev_del(netdev, ctx,
+ TLS_OFFLOAD_CTX_DIR_TX);
dev_put(netdev);
ctx->netdev = NULL;
tls_device_free_ctx(ctx);
@@ -161,10 +169,14 @@ static void tls_device_commit_start_marker(struct sock *sk,
struct tls_offload_context_tx *offload_ctx,
struct tls_record_info *start_marker_record)
{
+ unsigned long flags;
+
+ spin_lock_irqsave(&offload_ctx->lock, flags);
start_marker_record->end_seq = tcp_sk(sk)->write_seq;
start_marker_record->len = 0;
start_marker_record->num_frags = 0;
list_add_tail_rcu(&start_marker_record->list, &offload_ctx->records_list);
+ spin_unlock_irqrestore(&offload_ctx->lock, flags);
/* TLS offload is greatly simplified if we don't send
* SKBs where only part of the payload needs to be encrypted.
@@ -194,6 +206,24 @@ static void delete_all_records(struct tls_offload_context_tx *offload_ctx)
offload_ctx->retransmit_hint = NULL;
}
+static bool tls_has_unacked_records(struct tls_offload_context_tx *offload_ctx)
+{
+ struct tls_record_info *info;
+ bool has_unacked = false;
+ unsigned long flags;
+
+ spin_lock_irqsave(&offload_ctx->lock, flags);
+ list_for_each_entry(info, &offload_ctx->records_list, list) {
+ if (!tls_record_is_start_marker(info)) {
+ has_unacked = true;
+ break;
+ }
+ }
+ spin_unlock_irqrestore(&offload_ctx->lock, flags);
+
+ return has_unacked;
+}
+
static void tls_tcp_clean_acked(struct sock *sk, u32 acked_seq)
{
struct tls_context *tls_ctx = tls_get_ctx(sk);
@@ -222,6 +252,19 @@ static void tls_tcp_clean_acked(struct sock *sk, u32 acked_seq)
}
ctx->unacked_record_sn += deleted_records;
+
+ /* Once all old-key HW records are ACKed, set REKEY_READY to
+ * let sendmsg know it can finish the rekey and switch back
+ * to HW offload.
+ */
+ if (test_bit(TLS_TX_REKEY_PENDING, &tls_ctx->flags) &&
+ !test_bit(TLS_TX_REKEY_FAILED, &tls_ctx->flags)) {
+ u32 boundary_seq = READ_ONCE(tls_ctx->rekey.boundary_seq);
+
+ if (!before(acked_seq, boundary_seq))
+ set_bit(TLS_TX_REKEY_READY, &tls_ctx->flags);
+ }
+
spin_unlock_irqrestore(&ctx->lock, flags);
}
@@ -253,6 +296,14 @@ void tls_device_free_resources_tx(struct sock *sk)
struct tls_context *tls_ctx = tls_get_ctx(sk);
tls_free_partial_record(sk, tls_ctx);
+
+ if (unlikely(tls_ctx->rekey.sw_ctx))
+ tls_sw_release_resources_tx(sk);
+
+ if (test_bit(TLS_TX_REKEY_PENDING, &tls_ctx->flags)) {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYABORTED);
+ TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYINPROGRESS);
+ }
}
void tls_offload_tx_resync_request(struct sock *sk, u32 got_seq, u32 exp_seq)
@@ -462,6 +513,9 @@ static int tls_device_copy_data(void *addr, size_t bytes, struct iov_iter *i)
return 0;
}
+static int tls_device_complete_rekey(struct sock *sk, struct tls_context *ctx,
+ bool deferred);
+
static int tls_push_data(struct sock *sk,
struct iov_iter *iter,
size_t size, int flags,
@@ -624,6 +678,19 @@ int tls_device_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
goto out;
}
+ /* Old-key records all ACKed; switch back to HW. */
+ if (test_bit(TLS_TX_REKEY_READY, &tls_ctx->flags))
+ tls_device_complete_rekey(sk, tls_ctx, true);
+
+ /* Use SW path if rekey is in progress (PENDING) or if HW rekey
+ * failed (FAILED).
+ */
+ if (test_bit(TLS_TX_REKEY_PENDING, &tls_ctx->flags) ||
+ test_bit(TLS_TX_REKEY_FAILED, &tls_ctx->flags)) {
+ rc = tls_sw_sendmsg_locked(sk, msg, size);
+ goto out;
+ }
+
rc = tls_push_data(sk, &msg->msg_iter, size, msg->msg_flags,
record_type);
@@ -1103,6 +1170,282 @@ static struct tls_offload_context_tx *alloc_offload_ctx_tx(struct tls_context *c
return offload_ctx;
}
+static int tls_device_init_rekey_sw(struct sock *sk,
+ struct tls_context *ctx,
+ struct tls_offload_context_tx *offload_ctx,
+ struct tls_crypto_info *new_crypto_info)
+{
+ struct tls_sw_context_tx *sw_ctx = &offload_ctx->rekey.sw;
+ const struct tls_cipher_desc *cipher_desc;
+ char *key;
+ int rc;
+
+ cipher_desc = get_cipher_desc(new_crypto_info->cipher_type);
+ DEBUG_NET_WARN_ON_ONCE(!cipher_desc || !cipher_desc->offloadable);
+
+ memset(sw_ctx, 0, sizeof(*sw_ctx));
+ tls_sw_ctx_tx_init(sk, sw_ctx);
+
+ sw_ctx->aead_send = crypto_alloc_aead(cipher_desc->cipher_name, 0, 0);
+ if (IS_ERR(sw_ctx->aead_send)) {
+ rc = PTR_ERR(sw_ctx->aead_send);
+ sw_ctx->aead_send = NULL;
+ return rc;
+ }
+
+ key = crypto_info_key(new_crypto_info, cipher_desc);
+ rc = crypto_aead_setkey(sw_ctx->aead_send, key, cipher_desc->key);
+ if (rc)
+ goto free_aead;
+
+ rc = crypto_aead_setauthsize(sw_ctx->aead_send, cipher_desc->tag);
+ if (rc)
+ goto free_aead;
+
+ return 0;
+
+free_aead:
+ crypto_free_aead(sw_ctx->aead_send);
+ sw_ctx->aead_send = NULL;
+ return rc;
+}
+
+static int tls_device_start_rekey(struct sock *sk,
+ struct tls_context *ctx,
+ struct tls_offload_context_tx *offload_ctx,
+ struct tls_crypto_info *new_crypto_info)
+{
+ bool rekey_pending = test_bit(TLS_TX_REKEY_PENDING, &ctx->flags);
+ bool rekey_failed = test_bit(TLS_TX_REKEY_FAILED, &ctx->flags);
+ const struct tls_cipher_desc *cipher_desc;
+ char *key, *iv, *rec_seq, *salt;
+ unsigned long flags;
+ int rc;
+
+ cipher_desc = get_cipher_desc(new_crypto_info->cipher_type);
+ DEBUG_NET_WARN_ON_ONCE(!cipher_desc || !cipher_desc->offloadable);
+
+ key = crypto_info_key(new_crypto_info, cipher_desc);
+ iv = crypto_info_iv(new_crypto_info, cipher_desc);
+ rec_seq = crypto_info_rec_seq(new_crypto_info, cipher_desc);
+ salt = crypto_info_salt(new_crypto_info, cipher_desc);
+
+ if (rekey_pending || rekey_failed) {
+ /* Wait for in-flight async encryptions submitted to this tfm
+ * with the previous key before changing it.
+ */
+ rc = tls_encrypt_async_wait(&offload_ctx->rekey.sw);
+ if (rc)
+ return rc;
+
+ rc = crypto_aead_setkey(offload_ctx->rekey.sw.aead_send,
+ key, cipher_desc->key);
+ if (rc)
+ return rc;
+
+ memcpy(offload_ctx->rekey.tx.iv, salt, cipher_desc->salt);
+ memcpy(offload_ctx->rekey.tx.iv + cipher_desc->salt, iv,
+ cipher_desc->iv);
+ memcpy(offload_ctx->rekey.tx.rec_seq, rec_seq,
+ cipher_desc->rec_seq);
+
+ if (rekey_failed) {
+ set_bit(TLS_TX_REKEY_PENDING, &ctx->flags);
+ clear_bit(TLS_TX_REKEY_FAILED, &ctx->flags);
+ TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXDEVICE);
+ }
+ } else {
+ /* Flush any HW open_record before switching to SW */
+ if (tls_is_pending_open_record(ctx)) {
+ rc = ctx->push_pending_record(sk, 0);
+ if (rc < 0)
+ return rc;
+ }
+
+ rc = tls_device_init_rekey_sw(sk, ctx, offload_ctx,
+ new_crypto_info);
+ if (rc)
+ return rc;
+
+ memcpy(offload_ctx->rekey.tx.iv, salt, cipher_desc->salt);
+ memcpy(offload_ctx->rekey.tx.iv + cipher_desc->salt, iv,
+ cipher_desc->iv);
+ memcpy(offload_ctx->rekey.tx.rec_seq, rec_seq,
+ cipher_desc->rec_seq);
+
+ /* Prevent a partial record straddling the SW/HW boundary. */
+ tcp_write_collapse_fence(sk);
+
+ ctx->rekey.sw_ctx = &offload_ctx->rekey.sw;
+ ctx->rekey.cipher_ctx = &offload_ctx->rekey.tx;
+
+ spin_lock_irqsave(&offload_ctx->lock, flags);
+ WRITE_ONCE(ctx->rekey.boundary_seq, tcp_sk(sk)->write_seq);
+ set_bit(TLS_TX_REKEY_PENDING, &ctx->flags);
+ spin_unlock_irqrestore(&offload_ctx->lock, flags);
+
+ /* Switch to rekey validator; new sends won't use HW offload */
+ smp_store_release(&sk->sk_validate_xmit_skb,
+ tls_validate_xmit_skb_rekey);
+ }
+
+ unsafe_memcpy(&offload_ctx->rekey.crypto_send.info, new_crypto_info,
+ cipher_desc->crypto_info,
+ /* checked in do_tls_setsockopt_conf */);
+ memzero_explicit(new_crypto_info, cipher_desc->crypto_info);
+
+ return 0;
+}
+
+static int tls_device_complete_rekey(struct sock *sk, struct tls_context *ctx,
+ bool deferred)
+{
+ struct tls_offload_context_tx *offload_ctx = tls_offload_ctx_tx(ctx);
+ struct tls_record_info *start_marker_record;
+ const struct tls_cipher_desc *cipher_desc;
+ struct net_device *netdev;
+ unsigned long flags;
+ __be64 rcd_sn;
+ char *key;
+ int rc;
+
+ cipher_desc = get_cipher_desc(offload_ctx->rekey.crypto_send.info.cipher_type);
+ DEBUG_NET_WARN_ON_ONCE(!cipher_desc || !cipher_desc->offloadable);
+
+ rc = tls_sw_drain_tx(sk, ctx);
+ if (rc)
+ return rc;
+
+ start_marker_record = kmalloc_obj(*start_marker_record);
+ if (!start_marker_record)
+ return -ENOMEM;
+
+ down_read(&device_offload_lock);
+
+ netdev = rcu_dereference_protected(ctx->netdev,
+ lockdep_is_held(&device_offload_lock));
+ if (!netdev) {
+ rc = -ENODEV;
+ goto release_lock;
+ }
+
+ if (!test_bit(TLS_TX_DEV_CLOSED, &ctx->flags)) {
+ netdev->tlsdev_ops->tls_dev_del(netdev, ctx,
+ TLS_OFFLOAD_CTX_DIR_TX);
+ set_bit(TLS_TX_DEV_CLOSED, &ctx->flags);
+ }
+
+ memcpy(crypto_info_rec_seq(&offload_ctx->rekey.crypto_send.info, cipher_desc),
+ offload_ctx->rekey.tx.rec_seq, cipher_desc->rec_seq);
+
+ rc = tls_device_dev_add_tx(sk, netdev, &offload_ctx->rekey.crypto_send.info,
+ tcp_sk(sk)->write_seq);
+ if (!rc)
+ clear_bit(TLS_TX_DEV_CLOSED, &ctx->flags);
+
+release_lock:
+ up_read(&device_offload_lock);
+
+ if (rc)
+ goto rekey_fail;
+
+ spin_lock_irqsave(&offload_ctx->lock, flags);
+ memcpy(&rcd_sn, offload_ctx->rekey.tx.rec_seq, sizeof(rcd_sn));
+ offload_ctx->unacked_record_sn = be64_to_cpu(rcd_sn) - 1;
+ spin_unlock_irqrestore(&offload_ctx->lock, flags);
+
+ memcpy(ctx->tx.iv, offload_ctx->rekey.tx.iv,
+ cipher_desc->salt + cipher_desc->iv);
+ memcpy(ctx->tx.rec_seq, offload_ctx->rekey.tx.rec_seq,
+ cipher_desc->rec_seq);
+ unsafe_memcpy(&ctx->crypto_send.info,
+ &offload_ctx->rekey.crypto_send.info,
+ cipher_desc->crypto_info,
+ /* checked during rekey setup */);
+
+ key = crypto_info_key(&offload_ctx->rekey.crypto_send.info, cipher_desc);
+ rc = crypto_aead_setkey(offload_ctx->aead_send, key, cipher_desc->key);
+ if (rc)
+ goto rekey_fail;
+
+ /* Start marker: the NIC passes through everything before
+ * write_seq unencrypted (already SW-encrypted during rekey),
+ * same as during initial offload setup.
+ */
+ tls_device_commit_start_marker(sk, offload_ctx, start_marker_record);
+
+ spin_lock_irqsave(&offload_ctx->lock, flags);
+ clear_bit(TLS_TX_REKEY_PENDING, &ctx->flags);
+ clear_bit(TLS_TX_REKEY_READY, &ctx->flags);
+ clear_bit(TLS_TX_REKEY_FAILED, &ctx->flags);
+ spin_unlock_irqrestore(&offload_ctx->lock, flags);
+
+ /* Switch back to HW offload validator */
+ smp_store_release(&sk->sk_validate_xmit_skb, tls_validate_xmit_skb);
+
+ crypto_free_aead(tls_sw_ctx_tx(ctx)->aead_send);
+ ctx->rekey.sw_ctx = NULL;
+ ctx->rekey.cipher_ctx = NULL;
+ memzero_explicit(&offload_ctx->rekey, sizeof(offload_ctx->rekey));
+
+ if (deferred)
+ TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYINPROGRESS);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYOK);
+ return 0;
+
+rekey_fail:
+ kfree(start_marker_record);
+ spin_lock_irqsave(&offload_ctx->lock, flags);
+ set_bit(TLS_TX_REKEY_FAILED, &ctx->flags);
+ clear_bit(TLS_TX_REKEY_READY, &ctx->flags);
+ clear_bit(TLS_TX_REKEY_PENDING, &ctx->flags);
+ spin_unlock_irqrestore(&offload_ctx->lock, flags);
+ if (deferred)
+ TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYINPROGRESS);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYFALLBACK);
+ TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXDEVICE);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW);
+
+ return 0;
+}
+
+static int tls_set_device_offload_rekey(struct sock *sk,
+ struct tls_context *ctx,
+ struct net_device *netdev,
+ struct tls_crypto_info *new_crypto_info)
+{
+ struct tls_offload_context_tx *offload_ctx = tls_offload_ctx_tx(ctx);
+ bool rekey_pending = test_bit(TLS_TX_REKEY_PENDING, &ctx->flags);
+ bool rekey_failed = test_bit(TLS_TX_REKEY_FAILED, &ctx->flags);
+ bool defer = true;
+ int rc;
+
+ if (!rekey_pending && !rekey_failed)
+ defer = tls_has_unacked_records(offload_ctx) ||
+ tls_is_pending_open_record(ctx);
+
+ down_read(&device_offload_lock);
+
+ rc = tls_device_start_rekey(sk, ctx, offload_ctx, new_crypto_info);
+ if (rc) {
+ up_read(&device_offload_lock);
+ return rc;
+ }
+
+ up_read(&device_offload_lock);
+
+ if (defer) {
+ if (!rekey_pending)
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYINPROGRESS);
+ else
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYOK);
+ return 0;
+ }
+
+ return tls_device_complete_rekey(sk, ctx, false);
+}
+
static int tls_set_device_offload_initial(struct sock *sk,
struct tls_context *ctx,
struct net_device *netdev,
@@ -1187,18 +1530,23 @@ static int tls_set_device_offload_initial(struct sock *sk,
return rc;
}
-int tls_set_device_offload(struct sock *sk)
+int tls_set_device_offload(struct sock *sk,
+ struct tls_crypto_info *new_crypto_info)
{
+ struct tls_crypto_info *crypto_info, *src_crypto_info;
const struct tls_cipher_desc *cipher_desc;
- struct tls_crypto_info *crypto_info;
struct net_device *netdev;
struct tls_context *ctx;
int rc;
ctx = tls_get_ctx(sk);
- if (ctx->priv_ctx_tx)
- return -EEXIST;
+ /* Rekey is only supported for connections that are already
+ * using HW offload. For SW offload connections, the caller
+ * should fall back to tls_set_sw_offload() for rekey.
+ */
+ if (new_crypto_info && ctx->tx_conf != TLS_HW)
+ return -EINVAL;
netdev = get_netdev_for_sock(sk);
if (!netdev) {
@@ -1212,14 +1560,20 @@ int tls_set_device_offload(struct sock *sk)
}
crypto_info = &ctx->crypto_send.info;
- cipher_desc = get_cipher_desc(crypto_info->cipher_type);
+ src_crypto_info = new_crypto_info ?: crypto_info;
+ cipher_desc = get_cipher_desc(src_crypto_info->cipher_type);
if (!cipher_desc || !cipher_desc->offloadable) {
rc = -EINVAL;
goto release_netdev;
}
- rc = tls_set_device_offload_initial(sk, ctx, netdev, crypto_info,
- cipher_desc);
+ if (new_crypto_info)
+ rc = tls_set_device_offload_rekey(sk, ctx, netdev,
+ src_crypto_info);
+ else
+ rc = tls_set_device_offload_initial(sk, ctx, netdev,
+ src_crypto_info,
+ cipher_desc);
release_netdev:
dev_put(netdev);
@@ -1352,7 +1706,10 @@ static int tls_device_down(struct net_device *netdev)
/* Stop offloaded TX and switch to the fallback.
* tls_is_skb_tx_device_offloaded will return false.
*/
- WRITE_ONCE(ctx->sk->sk_validate_xmit_skb, tls_validate_xmit_skb_sw);
+ if (!test_bit(TLS_TX_REKEY_PENDING, &ctx->flags) &&
+ !test_bit(TLS_TX_REKEY_FAILED, &ctx->flags))
+ WRITE_ONCE(ctx->sk->sk_validate_xmit_skb,
+ tls_validate_xmit_skb_sw);
/* Stop the RX and TX resync.
* tls_dev_resync must not be called after tls_dev_del.
@@ -1369,9 +1726,12 @@ static int tls_device_down(struct net_device *netdev)
synchronize_net();
/* Release the offload context on the driver side. */
- if (ctx->tx_conf == TLS_HW)
+ if (ctx->tx_conf == TLS_HW &&
+ !test_bit(TLS_TX_DEV_CLOSED, &ctx->flags)) {
netdev->tlsdev_ops->tls_dev_del(netdev, ctx,
TLS_OFFLOAD_CTX_DIR_TX);
+ set_bit(TLS_TX_DEV_CLOSED, &ctx->flags);
+ }
if (ctx->rx_conf == TLS_HW &&
!test_bit(TLS_RX_DEV_CLOSED, &ctx->flags))
netdev->tlsdev_ops->tls_dev_del(netdev, ctx,
diff --git a/net/tls/tls_device_fallback.c b/net/tls/tls_device_fallback.c
index 1110f7ac6bcb..64ac4ef4012b 100644
--- a/net/tls/tls_device_fallback.c
+++ b/net/tls/tls_device_fallback.c
@@ -435,6 +435,30 @@ struct sk_buff *tls_validate_xmit_skb_sw(struct sock *sk,
return tls_sw_fallback(sk, skb);
}
+struct sk_buff *tls_validate_xmit_skb_rekey(struct sock *sk,
+ struct net_device *dev,
+ struct sk_buff *skb)
+{
+ struct tls_context *tls_ctx = tls_get_ctx(sk);
+ u32 tcp_seq = ntohl(tcp_hdr(skb)->seq);
+ u32 boundary_seq;
+
+ if (test_bit(TLS_TX_REKEY_FAILED, &tls_ctx->flags))
+ return skb;
+
+ /* If this packet is at or after the rekey boundary, it's already
+ * SW-encrypted with the new key, pass through unchanged
+ */
+ boundary_seq = READ_ONCE(tls_ctx->rekey.boundary_seq);
+ if (!before(tcp_seq, boundary_seq))
+ return skb;
+
+ /* Packet before boundary means retransmit of old data,
+ * use SW fallback with the old key
+ */
+ return tls_sw_fallback(sk, skb);
+}
+
struct sk_buff *tls_encrypt_skb(struct sk_buff *skb)
{
return tls_sw_fallback(skb->sk, skb);
diff --git a/net/tls/tls_main.c b/net/tls/tls_main.c
index c4c15c0e731a..b51f9ea1ed6c 100644
--- a/net/tls/tls_main.c
+++ b/net/tls/tls_main.c
@@ -347,8 +347,14 @@ static void tls_sk_proto_cleanup(struct sock *sk,
tls_sw_release_resources_tx(sk);
TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW);
} else if (ctx->tx_conf == TLS_HW) {
+ bool rekey_failed = test_bit(TLS_TX_REKEY_FAILED, &ctx->flags);
+
tls_device_free_resources_tx(sk);
- TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXDEVICE);
+
+ if (rekey_failed)
+ TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW);
+ else
+ TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXDEVICE);
}
if (ctx->rx_conf == TLS_SW) {
@@ -369,6 +375,8 @@ static void tls_sk_proto_close(struct sock *sk, long timeout)
if (ctx->tx_conf == TLS_SW)
tls_sw_cancel_work_tx(ctx);
+ else if (ctx->tx_conf == TLS_HW && ctx->rekey.sw_ctx)
+ tls_sw_cancel_work_tx(ctx);
lock_sock(sk);
free_ctx = ctx->tx_conf != TLS_HW && ctx->rx_conf != TLS_HW;
@@ -449,8 +457,16 @@ static int do_tls_getsockopt_conf(struct sock *sk, char __user *optval,
/* get user crypto info */
if (tx) {
- crypto_info = &ctx->crypto_send.info;
- cctx = &ctx->tx;
+ if (ctx->rekey.cipher_ctx) {
+ struct tls_offload_context_tx *offload_ctx =
+ tls_offload_ctx_tx(ctx);
+
+ crypto_info = &offload_ctx->rekey.crypto_send.info;
+ cctx = &offload_ctx->rekey.tx;
+ } else {
+ crypto_info = &ctx->crypto_send.info;
+ cctx = &ctx->tx;
+ }
} else {
crypto_info = &ctx->crypto_recv.info;
cctx = &ctx->rx;
@@ -714,32 +730,32 @@ static int do_tls_setsockopt_conf(struct sock *sk, sockptr_t optval,
}
if (tx) {
- if (update && ctx->tx_conf == TLS_HW) {
- rc = -EOPNOTSUPP;
- goto err_crypto_info;
- }
-
- if (!update) {
- rc = tls_set_device_offload(sk);
- conf = TLS_HW;
- if (!rc) {
+ rc = tls_set_device_offload(sk, update ? crypto_info : NULL);
+ conf = TLS_HW;
+ if (!rc) {
+ if (!update) {
TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXDEVICE);
TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXDEVICE);
- goto out;
}
- }
-
- rc = tls_set_sw_offload(sk, 1, update ? crypto_info : NULL);
- if (rc)
+ } else if (update && ctx->tx_conf == TLS_HW) {
+ /* HW rekey failed - return the actual error.
+ * Cannot fall back to SW for an existing HW connection.
+ */
goto err_crypto_info;
-
- if (update) {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYOK);
} else {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXSW);
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW);
+ rc = tls_set_sw_offload(sk, 1,
+ update ? crypto_info : NULL);
+ if (rc)
+ goto err_crypto_info;
+
+ if (update) {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYOK);
+ } else {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXSW);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW);
+ }
+ conf = TLS_SW;
}
- conf = TLS_SW;
} else {
if (update && ctx->rx_conf == TLS_HW) {
rc = -EOPNOTSUPP;
@@ -885,12 +901,29 @@ static int do_tls_setsockopt(struct sock *sk, int optname, sockptr_t optval,
switch (optname) {
case TLS_TX:
- case TLS_RX:
+ case TLS_RX: {
+ struct tls_context *ctx = tls_get_ctx(sk);
+ /* A HW-offload TX rekey arrives mid-stream and races with an
+ * in-flight tls_device_sendmsg(), which holds tx_lock across
+ * the lock_sock drop in sk_stream_wait_memory() while an
+ * open_record is half built. start_rekey() would flush that
+ * record and flip the SW/HW boundary under the sender,
+ * corrupting record framing. Serialize against the data path
+ * with tx_lock. Only the HW TX path reaches start_rekey, and
+ * initial setup (tx_conf == BASE) has no concurrent sender.
+ */
+ bool tx_rekey = optname == TLS_TX && ctx->tx_conf == TLS_HW;
+
+ if (tx_rekey)
+ mutex_lock(&ctx->tx_lock);
lock_sock(sk);
rc = do_tls_setsockopt_conf(sk, optval, optlen,
optname == TLS_TX);
release_sock(sk);
+ if (tx_rekey)
+ mutex_unlock(&ctx->tx_lock);
break;
+ }
case TLS_TX_ZEROCOPY_RO:
lock_sock(sk);
rc = do_tls_setsockopt_tx_zc(sk, optval, optlen);
diff --git a/net/tls/tls_proc.c b/net/tls/tls_proc.c
index 4012c4372d4c..adca8561c7da 100644
--- a/net/tls/tls_proc.c
+++ b/net/tls/tls_proc.c
@@ -27,6 +27,9 @@ static const struct snmp_mib tls_mib_list[] = {
SNMP_MIB_ITEM("TlsTxRekeyOk", LINUX_MIB_TLSTXREKEYOK),
SNMP_MIB_ITEM("TlsTxRekeyError", LINUX_MIB_TLSTXREKEYERROR),
SNMP_MIB_ITEM("TlsRxRekeyReceived", LINUX_MIB_TLSRXREKEYRECEIVED),
+ SNMP_MIB_ITEM("TlsTxRekeyFallback", LINUX_MIB_TLSTXREKEYFALLBACK),
+ SNMP_MIB_ITEM("TlsTxRekeyInProgress", LINUX_MIB_TLSTXREKEYINPROGRESS),
+ SNMP_MIB_ITEM("TlsTxRekeyAborted", LINUX_MIB_TLSTXREKEYABORTED),
};
static int tls_statistics_seq_show(struct seq_file *seq, void *v)
diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c
index b8e06b751efe..0364fae8a7e0 100644
--- a/net/tls/tls_sw.c
+++ b/net/tls/tls_sw.c
@@ -522,7 +522,7 @@ static void tls_encrypt_done(void *data, int err)
complete(&ctx->async_wait.completion);
}
-static int tls_encrypt_async_wait(struct tls_sw_context_tx *ctx)
+int tls_encrypt_async_wait(struct tls_sw_context_tx *ctx)
{
if (!atomic_dec_and_test(&ctx->encrypt_pending))
crypto_wait_req(-EINPROGRESS, &ctx->async_wait);
@@ -763,8 +763,7 @@ static int tls_sw_sendmsg_splice(struct sock *sk, struct msghdr *msg,
return 0;
}
-static int tls_sw_sendmsg_locked(struct sock *sk, struct msghdr *msg,
- size_t size)
+int tls_sw_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size)
{
long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
struct tls_context *tls_ctx = tls_get_ctx(sk);
@@ -2378,6 +2377,23 @@ void tls_sw_ctx_tx_init(struct sock *sk, struct tls_sw_context_tx *sw_ctx)
sw_ctx->tx_work.sk = sk;
}
+int tls_sw_drain_tx(struct sock *sk, struct tls_context *ctx)
+{
+ struct tls_sw_context_tx *sw_ctx = tls_sw_ctx_tx(ctx);
+ int rc;
+
+ if (tls_is_pending_open_record(ctx))
+ tls_sw_push_pending_record(sk, 0);
+ tls_encrypt_async_wait(sw_ctx);
+ rc = tls_tx_records(sk, -1);
+ if (rc < 0 || tls_is_partially_sent_record(ctx) ||
+ tls_is_pending_open_record(ctx))
+ return rc < 0 ? rc : -EAGAIN;
+
+ cancel_delayed_work_sync(&sw_ctx->tx_work.work);
+ return 0;
+}
+
static bool tls_is_tx_ready(struct tls_sw_context_tx *ctx)
{
struct tls_rec *rec;
--
2.25.1
^ permalink raw reply related
* [PATCH v15 5/9] tls: prep helpers and refactors for HW offload KeyUpdate
From: Rishikesh Jethwani @ 2026-07-09 20:53 UTC (permalink / raw)
To: netdev
Cc: saeedm, tariqt, mbloch, borisp, john.fastabend, kuba, sd, davem,
pabeni, edumazet, leon, andrew.gospodarek, Rishikesh Jethwani
In-Reply-To: <20260709205325.1591196-1-rjethwani@purestorage.com>
Preparatory refactors for TX and RX HW rekey support; no functional
change.
- Hoist cipher_context / tls_crypto_context above
tls_offload_context_tx so they can be embedded in offload
contexts.
- Add tls_tx_cipher_ctx() accessor and factor tls_sw_ctx_tx_init()
so the TX path can redirect to a temporary SW context during
rekey.
- Split tls_set_device_offload() into a dispatcher and
tls_set_device_offload_initial(); a _rekey() sibling follows.
- Factor tls_device_dev_add_tx() and tls_device_commit_start_marker()
so the rekey completion path can reuse them.
- Move crypto_aead_setauthsize() into the !*aead block so a fresh
AEAD is correctly configured when RX HW rekey allocates one.
Signed-off-by: Rishikesh Jethwani <rjethwani@purestorage.com>
---
include/net/tls.h | 38 +++++++-----
net/tls/tls.h | 1 +
net/tls/tls_device.c | 139 ++++++++++++++++++++++++++-----------------
net/tls/tls_sw.c | 33 +++++-----
4 files changed, 127 insertions(+), 84 deletions(-)
diff --git a/include/net/tls.h b/include/net/tls.h
index e57bef58851e..eb258bcd62bc 100644
--- a/include/net/tls.h
+++ b/include/net/tls.h
@@ -155,6 +155,22 @@ struct tls_record_info {
skb_frag_t frags[MAX_SKB_FRAGS];
};
+struct cipher_context {
+ char iv[TLS_MAX_IV_SIZE + TLS_MAX_SALT_SIZE];
+ char rec_seq[TLS_MAX_REC_SEQ_SIZE];
+};
+
+union tls_crypto_context {
+ struct tls_crypto_info info;
+ union {
+ struct tls12_crypto_info_aes_gcm_128 aes_gcm_128;
+ struct tls12_crypto_info_aes_gcm_256 aes_gcm_256;
+ struct tls12_crypto_info_chacha20_poly1305 chacha20_poly1305;
+ struct tls12_crypto_info_sm4_gcm sm4_gcm;
+ struct tls12_crypto_info_sm4_ccm sm4_ccm;
+ };
+};
+
#define TLS_DRIVER_STATE_SIZE_TX 16
struct tls_offload_context_tx {
struct crypto_aead *aead_send;
@@ -195,22 +211,6 @@ enum tls_context_flags {
TLS_RX_DEV_CLOSED = 2,
};
-struct cipher_context {
- char iv[TLS_MAX_IV_SIZE + TLS_MAX_SALT_SIZE];
- char rec_seq[TLS_MAX_REC_SEQ_SIZE];
-};
-
-union tls_crypto_context {
- struct tls_crypto_info info;
- union {
- struct tls12_crypto_info_aes_gcm_128 aes_gcm_128;
- struct tls12_crypto_info_aes_gcm_256 aes_gcm_256;
- struct tls12_crypto_info_chacha20_poly1305 chacha20_poly1305;
- struct tls12_crypto_info_sm4_gcm sm4_gcm;
- struct tls12_crypto_info_sm4_ccm sm4_ccm;
- };
-};
-
struct tls_prot_info {
u16 version;
u16 cipher_type;
@@ -392,6 +392,12 @@ static inline struct tls_sw_context_tx *tls_sw_ctx_tx(
return (struct tls_sw_context_tx *)tls_ctx->priv_ctx_tx;
}
+static inline struct cipher_context *tls_tx_cipher_ctx(
+ const struct tls_context *tls_ctx)
+{
+ return (struct cipher_context *)&tls_ctx->tx;
+}
+
static inline struct tls_offload_context_tx *
tls_offload_ctx_tx(const struct tls_context *tls_ctx)
{
diff --git a/net/tls/tls.h b/net/tls/tls.h
index 5a6ee1ea00f8..bf46d3471da6 100644
--- a/net/tls/tls.h
+++ b/net/tls/tls.h
@@ -157,6 +157,7 @@ void tls_update_rx_zc_capable(struct tls_context *tls_ctx);
void tls_sw_strparser_arm(struct sock *sk, struct tls_context *ctx);
void tls_sw_strparser_done(struct tls_context *tls_ctx);
int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
+void tls_sw_ctx_tx_init(struct sock *sk, struct tls_sw_context_tx *sw_ctx);
void tls_sw_splice_eof(struct socket *sock);
void tls_sw_cancel_work_tx(struct tls_context *tls_ctx);
void tls_sw_release_resources_tx(struct sock *sk);
diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c
index f22f8a550c82..7a98d2f6cbd3 100644
--- a/net/tls/tls_device.c
+++ b/net/tls/tls_device.c
@@ -138,6 +138,41 @@ static struct net_device *get_netdev_for_sock(struct sock *sk)
return lowest_dev;
}
+static int tls_device_dev_add_tx(struct sock *sk, struct net_device *netdev,
+ struct tls_crypto_info *crypto_info,
+ u32 write_seq)
+{
+ const struct tls_cipher_desc *cipher_desc;
+ char *rec_seq;
+ int rc;
+
+ cipher_desc = get_cipher_desc(crypto_info->cipher_type);
+ DEBUG_NET_WARN_ON_ONCE(!cipher_desc || !cipher_desc->offloadable);
+
+ rc = netdev->tlsdev_ops->tls_dev_add(netdev, sk, TLS_OFFLOAD_CTX_DIR_TX,
+ crypto_info, write_seq);
+ rec_seq = crypto_info_rec_seq(crypto_info, cipher_desc);
+ trace_tls_device_offload_set(sk, TLS_OFFLOAD_CTX_DIR_TX,
+ write_seq, rec_seq, rc);
+ return rc;
+}
+
+static void tls_device_commit_start_marker(struct sock *sk,
+ struct tls_offload_context_tx *offload_ctx,
+ struct tls_record_info *start_marker_record)
+{
+ start_marker_record->end_seq = tcp_sk(sk)->write_seq;
+ start_marker_record->len = 0;
+ start_marker_record->num_frags = 0;
+ list_add_tail_rcu(&start_marker_record->list, &offload_ctx->records_list);
+
+ /* TLS offload is greatly simplified if we don't send
+ * SKBs where only part of the payload needs to be encrypted.
+ * So mark the last skb in the write queue as end of record.
+ */
+ tcp_write_collapse_fence(sk);
+}
+
static void destroy_record(struct tls_record_info *record)
{
int i;
@@ -1068,57 +1103,31 @@ static struct tls_offload_context_tx *alloc_offload_ctx_tx(struct tls_context *c
return offload_ctx;
}
-int tls_set_device_offload(struct sock *sk)
+static int tls_set_device_offload_initial(struct sock *sk,
+ struct tls_context *ctx,
+ struct net_device *netdev,
+ struct tls_crypto_info *crypto_info,
+ const struct tls_cipher_desc *cipher_desc)
{
+ struct tls_prot_info *prot = &ctx->prot_info;
struct tls_record_info *start_marker_record;
struct tls_offload_context_tx *offload_ctx;
- const struct tls_cipher_desc *cipher_desc;
- struct tls_crypto_info *crypto_info;
- struct tls_prot_info *prot;
- struct net_device *netdev;
- struct tls_context *ctx;
char *iv, *rec_seq;
int rc;
- ctx = tls_get_ctx(sk);
- prot = &ctx->prot_info;
-
- if (ctx->priv_ctx_tx)
- return -EEXIST;
-
- netdev = get_netdev_for_sock(sk);
- if (!netdev) {
- pr_err_ratelimited("%s: netdev not found\n", __func__);
- return -EINVAL;
- }
-
- if (!(netdev->features & NETIF_F_HW_TLS_TX)) {
- rc = -EOPNOTSUPP;
- goto release_netdev;
- }
-
- crypto_info = &ctx->crypto_send.info;
- cipher_desc = get_cipher_desc(crypto_info->cipher_type);
- if (!cipher_desc || !cipher_desc->offloadable) {
- rc = -EINVAL;
- goto release_netdev;
- }
+ iv = crypto_info_iv(crypto_info, cipher_desc);
+ rec_seq = crypto_info_rec_seq(crypto_info, cipher_desc);
rc = init_prot_info(prot, crypto_info, cipher_desc);
if (rc)
- goto release_netdev;
-
- iv = crypto_info_iv(crypto_info, cipher_desc);
- rec_seq = crypto_info_rec_seq(crypto_info, cipher_desc);
+ return rc;
memcpy(ctx->tx.iv + cipher_desc->salt, iv, cipher_desc->iv);
memcpy(ctx->tx.rec_seq, rec_seq, cipher_desc->rec_seq);
start_marker_record = kmalloc_obj(*start_marker_record);
- if (!start_marker_record) {
- rc = -ENOMEM;
- goto release_netdev;
- }
+ if (!start_marker_record)
+ return -ENOMEM;
offload_ctx = alloc_offload_ctx_tx(ctx);
if (!offload_ctx) {
@@ -1130,20 +1139,11 @@ int tls_set_device_offload(struct sock *sk)
if (rc)
goto free_offload_ctx;
- start_marker_record->end_seq = tcp_sk(sk)->write_seq;
- start_marker_record->len = 0;
- start_marker_record->num_frags = 0;
- list_add_tail(&start_marker_record->list, &offload_ctx->records_list);
+ tls_device_commit_start_marker(sk, offload_ctx, start_marker_record);
clean_acked_data_enable(tcp_sk(sk), &tls_tcp_clean_acked);
ctx->push_pending_record = tls_device_push_pending_record;
- /* TLS offload is greatly simplified if we don't send
- * SKBs where only part of the payload needs to be encrypted.
- * So mark the last skb in the write queue as end of record.
- */
- tcp_write_collapse_fence(sk);
-
/* Avoid offloading if the device is down
* We don't want to offload new flows after
* the NETDEV_DOWN event
@@ -1159,11 +1159,8 @@ int tls_set_device_offload(struct sock *sk)
}
ctx->priv_ctx_tx = offload_ctx;
- rc = netdev->tlsdev_ops->tls_dev_add(netdev, sk, TLS_OFFLOAD_CTX_DIR_TX,
- &ctx->crypto_send.info,
- tcp_sk(sk)->write_seq);
- trace_tls_device_offload_set(sk, TLS_OFFLOAD_CTX_DIR_TX,
- tcp_sk(sk)->write_seq, rec_seq, rc);
+ rc = tls_device_dev_add_tx(sk, netdev, crypto_info,
+ tcp_sk(sk)->write_seq);
if (rc)
goto release_lock;
@@ -1175,7 +1172,6 @@ int tls_set_device_offload(struct sock *sk)
* by the netdev's xmit function.
*/
smp_store_release(&sk->sk_validate_xmit_skb, tls_validate_xmit_skb);
- dev_put(netdev);
return 0;
@@ -1188,6 +1184,43 @@ int tls_set_device_offload(struct sock *sk)
ctx->priv_ctx_tx = NULL;
free_marker_record:
kfree(start_marker_record);
+ return rc;
+}
+
+int tls_set_device_offload(struct sock *sk)
+{
+ const struct tls_cipher_desc *cipher_desc;
+ struct tls_crypto_info *crypto_info;
+ struct net_device *netdev;
+ struct tls_context *ctx;
+ int rc;
+
+ ctx = tls_get_ctx(sk);
+
+ if (ctx->priv_ctx_tx)
+ return -EEXIST;
+
+ netdev = get_netdev_for_sock(sk);
+ if (!netdev) {
+ pr_err_ratelimited("%s: netdev not found\n", __func__);
+ return -EINVAL;
+ }
+
+ if (!(netdev->features & NETIF_F_HW_TLS_TX)) {
+ rc = -EOPNOTSUPP;
+ goto release_netdev;
+ }
+
+ crypto_info = &ctx->crypto_send.info;
+ cipher_desc = get_cipher_desc(crypto_info->cipher_type);
+ if (!cipher_desc || !cipher_desc->offloadable) {
+ rc = -EINVAL;
+ goto release_netdev;
+ }
+
+ rc = tls_set_device_offload_initial(sk, ctx, netdev, crypto_info,
+ cipher_desc);
+
release_netdev:
dev_put(netdev);
return rc;
diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c
index 383eab5150ce..b8e06b751efe 100644
--- a/net/tls/tls_sw.c
+++ b/net/tls/tls_sw.c
@@ -555,11 +555,11 @@ static int tls_do_encryption(struct sock *sk,
break;
}
- memcpy(&rec->iv_data[iv_offset], tls_ctx->tx.iv,
+ memcpy(&rec->iv_data[iv_offset], tls_tx_cipher_ctx(tls_ctx)->iv,
prot->iv_size + prot->salt_size);
tls_xor_iv_with_seq(prot, rec->iv_data + iv_offset,
- tls_ctx->tx.rec_seq);
+ tls_tx_cipher_ctx(tls_ctx)->rec_seq);
sge->offset += prot->prepend_size;
sge->length -= prot->prepend_size;
@@ -610,7 +610,7 @@ static int tls_do_encryption(struct sock *sk,
/* Unhook the record from context if encryption is not failure */
ctx->open_rec = NULL;
- tls_advance_record_sn(sk, prot, &tls_ctx->tx);
+ tls_advance_record_sn(sk, prot, tls_tx_cipher_ctx(tls_ctx));
return rc;
}
@@ -676,7 +676,7 @@ static int tls_push_record(struct sock *sk, int flags,
sg_chain(rec->sg_aead_out, 2, &msg_en->sg.data[i]);
tls_make_aad(rec->aad_space, msg_pl->sg.size + prot->tail_size,
- tls_ctx->tx.rec_seq, record_type, prot);
+ tls_tx_cipher_ctx(tls_ctx)->rec_seq, record_type, prot);
tls_fill_prepend(tls_ctx,
page_address(sg_page(&msg_en->sg.data[i])) +
@@ -2369,6 +2369,15 @@ static void tx_work_handler(struct work_struct *work)
}
}
+void tls_sw_ctx_tx_init(struct sock *sk, struct tls_sw_context_tx *sw_ctx)
+{
+ crypto_init_wait(&sw_ctx->async_wait);
+ atomic_set(&sw_ctx->encrypt_pending, 1);
+ INIT_LIST_HEAD(&sw_ctx->tx_list);
+ INIT_DELAYED_WORK(&sw_ctx->tx_work.work, tx_work_handler);
+ sw_ctx->tx_work.sk = sk;
+}
+
static bool tls_is_tx_ready(struct tls_sw_context_tx *ctx)
{
struct tls_rec *rec;
@@ -2420,11 +2429,7 @@ static struct tls_sw_context_tx *init_ctx_tx(struct tls_context *ctx, struct soc
sw_ctx_tx = ctx->priv_ctx_tx;
}
- crypto_init_wait(&sw_ctx_tx->async_wait);
- atomic_set(&sw_ctx_tx->encrypt_pending, 1);
- INIT_LIST_HEAD(&sw_ctx_tx->tx_list);
- INIT_DELAYED_WORK(&sw_ctx_tx->tx_work.work, tx_work_handler);
- sw_ctx_tx->tx_work.sk = sk;
+ tls_sw_ctx_tx_init(sk, sw_ctx_tx);
return sw_ctx_tx;
}
@@ -2551,6 +2556,10 @@ int tls_sw_ctx_init(struct sock *sk, int tx,
*aead = NULL;
goto free_priv;
}
+
+ rc = crypto_aead_setauthsize(*aead, prot->tag_size);
+ if (rc)
+ goto free_aead;
}
ctx->push_pending_record = tls_sw_push_pending_record;
@@ -2567,12 +2576,6 @@ int tls_sw_ctx_init(struct sock *sk, int tx,
goto free_aead;
}
- if (!new_crypto_info) {
- rc = crypto_aead_setauthsize(*aead, prot->tag_size);
- if (rc)
- goto free_aead;
- }
-
if (!tx && !new_crypto_info) {
tfm = crypto_aead_tfm(sw_ctx_rx->aead_recv);
--
2.25.1
^ permalink raw reply related
* [PATCH v15 4/9] tls: split tls_set_sw_offload into init and finalize stages
From: Rishikesh Jethwani @ 2026-07-09 20:53 UTC (permalink / raw)
To: netdev
Cc: saeedm, tariqt, mbloch, borisp, john.fastabend, kuba, sd, davem,
pabeni, edumazet, leon, andrew.gospodarek, Rishikesh Jethwani
In-Reply-To: <20260709205325.1591196-1-rjethwani@purestorage.com>
Separate cipher context initialization from key material finalization
to support staged setup for hardware offload fallback paths.
Signed-off-by: Rishikesh Jethwani <rjethwani@purestorage.com>
---
net/tls/tls.h | 4 +++
net/tls/tls_device.c | 3 +-
net/tls/tls_sw.c | 77 +++++++++++++++++++++++++++++++-------------
3 files changed, 61 insertions(+), 23 deletions(-)
diff --git a/net/tls/tls.h b/net/tls/tls.h
index 60a37bdaaa25..5a6ee1ea00f8 100644
--- a/net/tls/tls.h
+++ b/net/tls/tls.h
@@ -147,6 +147,10 @@ void tls_strp_abort_strp(struct tls_strparser *strp, int err);
int init_prot_info(struct tls_prot_info *prot,
const struct tls_crypto_info *crypto_info,
const struct tls_cipher_desc *cipher_desc);
+int tls_sw_ctx_init(struct sock *sk, int tx,
+ struct tls_crypto_info *new_crypto_info);
+void tls_sw_ctx_finalize(struct sock *sk, int tx,
+ struct tls_crypto_info *new_crypto_info);
int tls_set_sw_offload(struct sock *sk, int tx,
struct tls_crypto_info *new_crypto_info);
void tls_update_rx_zc_capable(struct tls_context *tls_ctx);
diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c
index a087cf3f544f..f22f8a550c82 100644
--- a/net/tls/tls_device.c
+++ b/net/tls/tls_device.c
@@ -1233,7 +1233,7 @@ int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx)
context->resync_nh_reset = 1;
ctx->priv_ctx_rx = context;
- rc = tls_set_sw_offload(sk, 0, NULL);
+ rc = tls_sw_ctx_init(sk, 0, NULL);
if (rc)
goto release_ctx;
@@ -1247,6 +1247,7 @@ int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx)
goto free_sw_resources;
tls_device_attach(ctx, sk, netdev);
+ tls_sw_ctx_finalize(sk, 0, NULL);
up_read(&device_offload_lock);
dev_put(netdev);
diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c
index 9324e4ed20a3..383eab5150ce 100644
--- a/net/tls/tls_sw.c
+++ b/net/tls/tls_sw.c
@@ -2490,20 +2490,19 @@ static void tls_finish_key_update(struct sock *sk, struct tls_context *tls_ctx)
ctx->saved_data_ready(sk);
}
-int tls_set_sw_offload(struct sock *sk, int tx,
- struct tls_crypto_info *new_crypto_info)
+int tls_sw_ctx_init(struct sock *sk, int tx,
+ struct tls_crypto_info *new_crypto_info)
{
struct tls_crypto_info *crypto_info, *src_crypto_info;
struct tls_sw_context_tx *sw_ctx_tx = NULL;
struct tls_sw_context_rx *sw_ctx_rx = NULL;
const struct tls_cipher_desc *cipher_desc;
- char *iv, *rec_seq, *key, *salt;
- struct cipher_context *cctx;
struct tls_prot_info *prot;
struct crypto_aead **aead;
struct tls_context *ctx;
struct crypto_tfm *tfm;
int rc = 0;
+ char *key;
ctx = tls_get_ctx(sk);
prot = &ctx->prot_info;
@@ -2524,12 +2523,10 @@ int tls_set_sw_offload(struct sock *sk, int tx,
if (tx) {
sw_ctx_tx = ctx->priv_ctx_tx;
crypto_info = &ctx->crypto_send.info;
- cctx = &ctx->tx;
aead = &sw_ctx_tx->aead_send;
} else {
sw_ctx_rx = ctx->priv_ctx_rx;
crypto_info = &ctx->crypto_recv.info;
- cctx = &ctx->rx;
aead = &sw_ctx_rx->aead_recv;
}
@@ -2545,10 +2542,7 @@ int tls_set_sw_offload(struct sock *sk, int tx,
if (rc)
goto free_priv;
- iv = crypto_info_iv(src_crypto_info, cipher_desc);
key = crypto_info_key(src_crypto_info, cipher_desc);
- salt = crypto_info_salt(src_crypto_info, cipher_desc);
- rec_seq = crypto_info_rec_seq(src_crypto_info, cipher_desc);
if (!*aead) {
*aead = crypto_alloc_aead(cipher_desc->cipher_name, 0, 0);
@@ -2592,19 +2586,6 @@ int tls_set_sw_offload(struct sock *sk, int tx,
goto free_aead;
}
- memcpy(cctx->iv, salt, cipher_desc->salt);
- memcpy(cctx->iv + cipher_desc->salt, iv, cipher_desc->iv);
- memcpy(cctx->rec_seq, rec_seq, cipher_desc->rec_seq);
-
- if (new_crypto_info) {
- unsafe_memcpy(crypto_info, new_crypto_info,
- cipher_desc->crypto_info,
- /* size was checked in do_tls_setsockopt_conf */);
- memzero_explicit(new_crypto_info, cipher_desc->crypto_info);
- if (!tx)
- tls_finish_key_update(sk, ctx);
- }
-
goto out;
free_aead:
@@ -2623,3 +2604,55 @@ int tls_set_sw_offload(struct sock *sk, int tx,
out:
return rc;
}
+
+void tls_sw_ctx_finalize(struct sock *sk, int tx,
+ struct tls_crypto_info *new_crypto_info)
+{
+ struct tls_crypto_info *crypto_info, *src_crypto_info;
+ const struct tls_cipher_desc *cipher_desc;
+ struct tls_context *ctx = tls_get_ctx(sk);
+ struct cipher_context *cctx;
+ char *iv, *salt, *rec_seq;
+
+ if (tx) {
+ crypto_info = &ctx->crypto_send.info;
+ cctx = &ctx->tx;
+ } else {
+ crypto_info = &ctx->crypto_recv.info;
+ cctx = &ctx->rx;
+ }
+
+ src_crypto_info = new_crypto_info ?: crypto_info;
+ cipher_desc = get_cipher_desc(src_crypto_info->cipher_type);
+
+ iv = crypto_info_iv(src_crypto_info, cipher_desc);
+ salt = crypto_info_salt(src_crypto_info, cipher_desc);
+ rec_seq = crypto_info_rec_seq(src_crypto_info, cipher_desc);
+
+ memcpy(cctx->iv, salt, cipher_desc->salt);
+ memcpy(cctx->iv + cipher_desc->salt, iv, cipher_desc->iv);
+ memcpy(cctx->rec_seq, rec_seq, cipher_desc->rec_seq);
+
+ if (new_crypto_info) {
+ unsafe_memcpy(crypto_info, new_crypto_info,
+ cipher_desc->crypto_info,
+ /* size was checked in do_tls_setsockopt_conf */);
+ memzero_explicit(new_crypto_info, cipher_desc->crypto_info);
+
+ if (!tx)
+ tls_finish_key_update(sk, ctx);
+ }
+}
+
+int tls_set_sw_offload(struct sock *sk, int tx,
+ struct tls_crypto_info *new_crypto_info)
+{
+ int rc;
+
+ rc = tls_sw_ctx_init(sk, tx, new_crypto_info);
+ if (rc)
+ return rc;
+
+ tls_sw_ctx_finalize(sk, tx, new_crypto_info);
+ return 0;
+}
--
2.25.1
^ permalink raw reply related
* [PATCH v15 3/9] tls: add TLS 1.3 hardware offload support
From: Rishikesh Jethwani @ 2026-07-09 20:53 UTC (permalink / raw)
To: netdev
Cc: saeedm, tariqt, mbloch, borisp, john.fastabend, kuba, sd, davem,
pabeni, edumazet, leon, andrew.gospodarek, Rishikesh Jethwani
In-Reply-To: <20260709205325.1591196-1-rjethwani@purestorage.com>
Add TLS 1.3 support to the kernel TLS hardware offload infrastructure,
enabling hardware acceleration for TLS 1.3 connections on capable NICs.
Tested on Mellanox ConnectX-6 Dx (Crypto Enabled) with TLS 1.3 AES-GCM-128
and AES-GCM-256 cipher suites.
Signed-off-by: Rishikesh Jethwani <rjethwani@purestorage.com>
---
net/tls/tls_device.c | 65 ++++++++++++++++-----------
net/tls/tls_device_fallback.c | 58 +++++++++++++-----------
net/tls/tls_main.c | 85 ++++++++++++++++++++---------------
3 files changed, 121 insertions(+), 87 deletions(-)
diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c
index 741aef09bfd3..a087cf3f544f 100644
--- a/net/tls/tls_device.c
+++ b/net/tls/tls_device.c
@@ -317,25 +317,34 @@ static void tls_device_record_close(struct sock *sk,
unsigned char record_type)
{
struct tls_prot_info *prot = &ctx->prot_info;
- struct page_frag dummy_tag_frag;
-
- /* append tag
- * device will fill in the tag, we just need to append a placeholder
- * use socket memory to improve coalescing (re-using a single buffer
- * increases frag count)
- * if we can't allocate memory now use the dummy page
+ int tail = prot->tag_size + prot->tail_size;
+
+ /* Append tail: tag for TLS 1.2, content_type + tag for TLS 1.3.
+ * Device fills in the tag, we just need to append a placeholder.
+ * Use socket memory to improve coalescing (re-using a single buffer
+ * increases frag count); if allocation fails use dummy_page
+ * (offset = record_type gives correct content_type byte via
+ * identity mapping)
*/
- if (unlikely(pfrag->size - pfrag->offset < prot->tag_size) &&
- !skb_page_frag_refill(prot->tag_size, pfrag, sk->sk_allocation)) {
- dummy_tag_frag.page = dummy_page;
- dummy_tag_frag.offset = 0;
- pfrag = &dummy_tag_frag;
+ if (unlikely(pfrag->size - pfrag->offset < tail) &&
+ !skb_page_frag_refill(tail, pfrag, sk->sk_allocation)) {
+ struct page_frag dummy_pfrag = {
+ .page = dummy_page,
+ .offset = record_type,
+ };
+ tls_append_frag(record, &dummy_pfrag, tail);
+ } else {
+ if (prot->tail_size) {
+ char *content_type_addr = page_address(pfrag->page) +
+ pfrag->offset;
+ *content_type_addr = record_type;
+ }
+ tls_append_frag(record, pfrag, tail);
}
- tls_append_frag(record, pfrag, prot->tag_size);
/* fill prepend */
tls_fill_prepend(ctx, skb_frag_address(&record->frags[0]),
- record->len - prot->overhead_size,
+ record->len - prot->overhead_size + prot->tail_size,
record_type);
}
@@ -883,6 +892,7 @@ static int
tls_device_reencrypt(struct sock *sk, struct tls_context *tls_ctx)
{
struct tls_sw_context_rx *sw_ctx = tls_sw_ctx_rx(tls_ctx);
+ struct tls_prot_info *prot = &tls_ctx->prot_info;
const struct tls_cipher_desc *cipher_desc;
int err, offset, copy, data_len, pos;
struct sk_buff *skb, *skb_iter;
@@ -894,7 +904,7 @@ tls_device_reencrypt(struct sock *sk, struct tls_context *tls_ctx)
DEBUG_NET_WARN_ON_ONCE(!cipher_desc || !cipher_desc->offloadable);
rxm = strp_msg(tls_strp_msg(sw_ctx));
- orig_buf = kmalloc(rxm->full_len + TLS_HEADER_SIZE + cipher_desc->iv,
+ orig_buf = kmalloc(rxm->full_len + prot->prepend_size,
sk->sk_allocation);
if (!orig_buf)
return -ENOMEM;
@@ -909,9 +919,8 @@ tls_device_reencrypt(struct sock *sk, struct tls_context *tls_ctx)
offset = rxm->offset;
sg_init_table(sg, 1);
- sg_set_buf(&sg[0], buf,
- rxm->full_len + TLS_HEADER_SIZE + cipher_desc->iv);
- err = skb_copy_bits(skb, offset, buf, TLS_HEADER_SIZE + cipher_desc->iv);
+ sg_set_buf(&sg[0], buf, rxm->full_len + prot->prepend_size);
+ err = skb_copy_bits(skb, offset, buf, prot->prepend_size);
if (err)
goto free_buf;
@@ -1089,11 +1098,6 @@ int tls_set_device_offload(struct sock *sk)
}
crypto_info = &ctx->crypto_send.info;
- if (crypto_info->version != TLS_1_2_VERSION) {
- rc = -EOPNOTSUPP;
- goto release_netdev;
- }
-
cipher_desc = get_cipher_desc(crypto_info->cipher_type);
if (!cipher_desc || !cipher_desc->offloadable) {
rc = -EINVAL;
@@ -1196,9 +1200,6 @@ int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx)
struct net_device *netdev;
int rc = 0;
- if (ctx->crypto_recv.info.version != TLS_1_2_VERSION)
- return -EOPNOTSUPP;
-
netdev = get_netdev_for_sock(sk);
if (!netdev) {
pr_err_ratelimited("%s: netdev not found\n", __func__);
@@ -1408,12 +1409,22 @@ static struct notifier_block tls_dev_notifier = {
int __init tls_device_init(void)
{
- int err;
+ unsigned char *page_addr;
+ int err, i;
dummy_page = alloc_page(GFP_KERNEL);
if (!dummy_page)
return -ENOMEM;
+ /* Pre-populate dummy_page with identity mapping for all byte values.
+ * This is used as fallback for TLS 1.3 content type when memory
+ * allocation fails. By populating all 256 values, we avoid needing
+ * to validate record_type at runtime.
+ */
+ page_addr = page_address(dummy_page);
+ for (i = 0; i < 256; i++)
+ page_addr[i] = (unsigned char)i;
+
destruct_wq = alloc_workqueue("ktls_device_destruct", WQ_PERCPU, 0);
if (!destruct_wq) {
err = -ENOMEM;
diff --git a/net/tls/tls_device_fallback.c b/net/tls/tls_device_fallback.c
index 3b7d0ab2bcf1..1110f7ac6bcb 100644
--- a/net/tls/tls_device_fallback.c
+++ b/net/tls/tls_device_fallback.c
@@ -37,14 +37,15 @@
#include "tls.h"
-static int tls_enc_record(struct aead_request *aead_req,
+static int tls_enc_record(struct tls_context *tls_ctx,
+ struct aead_request *aead_req,
struct crypto_aead *aead, char *aad,
char *iv, __be64 rcd_sn,
struct scatter_walk *in,
- struct scatter_walk *out, int *in_len,
- struct tls_prot_info *prot)
+ struct scatter_walk *out, int *in_len)
{
unsigned char buf[TLS_HEADER_SIZE + TLS_MAX_IV_SIZE];
+ struct tls_prot_info *prot = &tls_ctx->prot_info;
const struct tls_cipher_desc *cipher_desc;
struct scatterlist sg_in[3];
struct scatterlist sg_out[3];
@@ -55,7 +56,7 @@ static int tls_enc_record(struct aead_request *aead_req,
cipher_desc = get_cipher_desc(prot->cipher_type);
DEBUG_NET_WARN_ON_ONCE(!cipher_desc || !cipher_desc->offloadable);
- buf_size = TLS_HEADER_SIZE + cipher_desc->iv;
+ buf_size = prot->prepend_size;
len = min_t(int, *in_len, buf_size);
memcpy_from_scatterwalk(buf, in, len);
@@ -66,16 +67,27 @@ static int tls_enc_record(struct aead_request *aead_req,
return 0;
len = buf[4] | (buf[3] << 8);
- len -= cipher_desc->iv;
+ if (prot->version != TLS_1_3_VERSION)
+ len -= cipher_desc->iv;
tls_make_aad(aad, len - cipher_desc->tag, (char *)&rcd_sn, buf[0], prot);
- memcpy(iv + cipher_desc->salt, buf + TLS_HEADER_SIZE, cipher_desc->iv);
+ if (prot->version == TLS_1_3_VERSION) {
+ void *iv_src = crypto_info_iv(&tls_ctx->crypto_send.info,
+ cipher_desc);
+
+ memcpy(iv + cipher_desc->salt, iv_src, cipher_desc->iv);
+ } else {
+ memcpy(iv + cipher_desc->salt, buf + TLS_HEADER_SIZE,
+ cipher_desc->iv);
+ }
+
+ tls_xor_iv_with_seq(prot, iv, (char *)&rcd_sn);
sg_init_table(sg_in, ARRAY_SIZE(sg_in));
sg_init_table(sg_out, ARRAY_SIZE(sg_out));
- sg_set_buf(sg_in, aad, TLS_AAD_SPACE_SIZE);
- sg_set_buf(sg_out, aad, TLS_AAD_SPACE_SIZE);
+ sg_set_buf(sg_in, aad, prot->aad_size);
+ sg_set_buf(sg_out, aad, prot->aad_size);
scatterwalk_get_sglist(in, sg_in + 1);
scatterwalk_get_sglist(out, sg_out + 1);
@@ -108,13 +120,6 @@ static int tls_enc_record(struct aead_request *aead_req,
return rc;
}
-static void tls_init_aead_request(struct aead_request *aead_req,
- struct crypto_aead *aead)
-{
- aead_request_set_tfm(aead_req, aead);
- aead_request_set_ad(aead_req, TLS_AAD_SPACE_SIZE);
-}
-
static struct aead_request *tls_alloc_aead_request(struct crypto_aead *aead,
gfp_t flags)
{
@@ -124,14 +129,15 @@ static struct aead_request *tls_alloc_aead_request(struct crypto_aead *aead,
aead_req = kzalloc(req_size, flags);
if (aead_req)
- tls_init_aead_request(aead_req, aead);
+ aead_request_set_tfm(aead_req, aead);
return aead_req;
}
-static int tls_enc_records(struct aead_request *aead_req,
+static int tls_enc_records(struct tls_context *tls_ctx,
+ struct aead_request *aead_req,
struct crypto_aead *aead, struct scatterlist *sg_in,
struct scatterlist *sg_out, char *aad, char *iv,
- u64 rcd_sn, int len, struct tls_prot_info *prot)
+ u64 rcd_sn, int len)
{
struct scatter_walk out, in;
int rc;
@@ -140,8 +146,8 @@ static int tls_enc_records(struct aead_request *aead_req,
scatterwalk_start(&out, sg_out);
do {
- rc = tls_enc_record(aead_req, aead, aad, iv,
- cpu_to_be64(rcd_sn), &in, &out, &len, prot);
+ rc = tls_enc_record(tls_ctx, aead_req, aead, aad, iv,
+ cpu_to_be64(rcd_sn), &in, &out, &len);
rcd_sn++;
} while (rc == 0 && len);
@@ -314,7 +320,10 @@ static struct sk_buff *tls_enc_skb(struct tls_context *tls_ctx,
cipher_desc = get_cipher_desc(tls_ctx->crypto_send.info.cipher_type);
DEBUG_NET_WARN_ON_ONCE(!cipher_desc || !cipher_desc->offloadable);
- buf_len = cipher_desc->salt + cipher_desc->iv + TLS_AAD_SPACE_SIZE +
+ aead_request_set_ad(aead_req, tls_ctx->prot_info.aad_size);
+
+ buf_len = cipher_desc->salt + cipher_desc->iv +
+ tls_ctx->prot_info.aad_size +
sync_size + cipher_desc->tag;
buf = kmalloc(buf_len, GFP_ATOMIC);
if (!buf)
@@ -324,7 +333,7 @@ static struct sk_buff *tls_enc_skb(struct tls_context *tls_ctx,
salt = crypto_info_salt(&tls_ctx->crypto_send.info, cipher_desc);
memcpy(iv, salt, cipher_desc->salt);
aad = buf + cipher_desc->salt + cipher_desc->iv;
- dummy_buf = aad + TLS_AAD_SPACE_SIZE;
+ dummy_buf = aad + tls_ctx->prot_info.aad_size;
nskb = alloc_skb(skb_headroom(skb) + skb->len, GFP_ATOMIC);
if (!nskb)
@@ -335,9 +344,8 @@ static struct sk_buff *tls_enc_skb(struct tls_context *tls_ctx,
fill_sg_out(sg_out, buf, tls_ctx, nskb, tcp_payload_offset,
payload_len, sync_size, dummy_buf);
- if (tls_enc_records(aead_req, ctx->aead_send, sg_in, sg_out, aad, iv,
- rcd_sn, sync_size + payload_len,
- &tls_ctx->prot_info) < 0)
+ if (tls_enc_records(tls_ctx, aead_req, ctx->aead_send, sg_in, sg_out,
+ aad, iv, rcd_sn, sync_size + payload_len) < 0)
goto free_nskb;
complete_skb(nskb, skb, tcp_payload_offset);
diff --git a/net/tls/tls_main.c b/net/tls/tls_main.c
index 8c588cdab733..c4c15c0e731a 100644
--- a/net/tls/tls_main.c
+++ b/net/tls/tls_main.c
@@ -714,49 +714,64 @@ static int do_tls_setsockopt_conf(struct sock *sk, sockptr_t optval,
}
if (tx) {
- rc = tls_set_device_offload(sk);
- conf = TLS_HW;
- if (!rc) {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXDEVICE);
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXDEVICE);
- } else {
- rc = tls_set_sw_offload(sk, 1,
- update ? crypto_info : NULL);
- if (rc)
- goto err_crypto_info;
-
- if (update) {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYOK);
- } else {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXSW);
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW);
+ if (update && ctx->tx_conf == TLS_HW) {
+ rc = -EOPNOTSUPP;
+ goto err_crypto_info;
+ }
+
+ if (!update) {
+ rc = tls_set_device_offload(sk);
+ conf = TLS_HW;
+ if (!rc) {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXDEVICE);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXDEVICE);
+ goto out;
}
- conf = TLS_SW;
}
- } else {
- rc = tls_set_device_offload_rx(sk, ctx);
- conf = TLS_HW;
- if (!rc) {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXDEVICE);
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXDEVICE);
+
+ rc = tls_set_sw_offload(sk, 1, update ? crypto_info : NULL);
+ if (rc)
+ goto err_crypto_info;
+
+ if (update) {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYOK);
} else {
- rc = tls_set_sw_offload(sk, 0,
- update ? crypto_info : NULL);
- if (rc)
- goto err_crypto_info;
-
- if (update) {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYOK);
- } else {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXSW);
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXSW);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXSW);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW);
+ }
+ conf = TLS_SW;
+ } else {
+ if (update && ctx->rx_conf == TLS_HW) {
+ rc = -EOPNOTSUPP;
+ goto err_crypto_info;
+ }
+
+ if (!update) {
+ rc = tls_set_device_offload_rx(sk, ctx);
+ conf = TLS_HW;
+ if (!rc) {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXDEVICE);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXDEVICE);
+ tls_sw_strparser_arm(sk, ctx);
+ goto out;
}
- conf = TLS_SW;
}
- if (!update)
+
+ rc = tls_set_sw_offload(sk, 0, update ? crypto_info : NULL);
+ if (rc)
+ goto err_crypto_info;
+
+ if (update) {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYOK);
+ } else {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXSW);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXSW);
tls_sw_strparser_arm(sk, ctx);
+ }
+ conf = TLS_SW;
}
+out:
if (tx)
ctx->tx_conf = conf;
else
--
2.25.1
^ permalink raw reply related
* [PATCH v15 2/9] net/mlx5e: add TLS 1.3 hardware offload support
From: Rishikesh Jethwani @ 2026-07-09 20:53 UTC (permalink / raw)
To: netdev
Cc: saeedm, tariqt, mbloch, borisp, john.fastabend, kuba, sd, davem,
pabeni, edumazet, leon, andrew.gospodarek, Rishikesh Jethwani
In-Reply-To: <20260709205325.1591196-1-rjethwani@purestorage.com>
Enable TLS 1.3 TX/RX hardware offload on ConnectX-6 Dx and newer
crypto-enabled adapters.
Key changes:
- Add TLS 1.3 capability checking and version validation
- Use MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_3 (0x3) for crypto context
- Handle TLS 1.3 IV format: full 12-byte IV copied to gcm_iv +
implicit_iv (vs TLS 1.2's 4-byte salt only)
Tested with TLS 1.3 AES-GCM-128 and AES-GCM-256 cipher suites.
Signed-off-by: Rishikesh Jethwani <rjethwani@purestorage.com>
Tested-by: Tariq Toukan <tariqt@nvidia.com>
Reviewed-by: Tariq Toukan <tariqt@nvidia.com>
---
.../ethernet/mellanox/mlx5/core/en_accel/ktls.h | 8 +++++++-
.../mellanox/mlx5/core/en_accel/ktls_txrx.c | 14 +++++++++++---
2 files changed, 18 insertions(+), 4 deletions(-)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h
index 07a04a142a2e..0469ca6a0762 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h
@@ -30,7 +30,9 @@ static inline bool mlx5e_is_ktls_device(struct mlx5_core_dev *mdev)
return false;
return (MLX5_CAP_TLS(mdev, tls_1_2_aes_gcm_128) ||
- MLX5_CAP_TLS(mdev, tls_1_2_aes_gcm_256));
+ MLX5_CAP_TLS(mdev, tls_1_2_aes_gcm_256) ||
+ MLX5_CAP_TLS(mdev, tls_1_3_aes_gcm_128) ||
+ MLX5_CAP_TLS(mdev, tls_1_3_aes_gcm_256));
}
static inline bool mlx5e_ktls_type_check(struct mlx5_core_dev *mdev,
@@ -40,10 +42,14 @@ static inline bool mlx5e_ktls_type_check(struct mlx5_core_dev *mdev,
case TLS_CIPHER_AES_GCM_128:
if (crypto_info->version == TLS_1_2_VERSION)
return MLX5_CAP_TLS(mdev, tls_1_2_aes_gcm_128);
+ else if (crypto_info->version == TLS_1_3_VERSION)
+ return MLX5_CAP_TLS(mdev, tls_1_3_aes_gcm_128);
break;
case TLS_CIPHER_AES_GCM_256:
if (crypto_info->version == TLS_1_2_VERSION)
return MLX5_CAP_TLS(mdev, tls_1_2_aes_gcm_256);
+ else if (crypto_info->version == TLS_1_3_VERSION)
+ return MLX5_CAP_TLS(mdev, tls_1_3_aes_gcm_256);
break;
}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c
index 570a912dd6fa..f3f1be1d4034 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c
@@ -6,6 +6,7 @@
enum {
MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_2 = 0x2,
+ MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_3 = 0x3,
};
enum {
@@ -15,8 +16,10 @@ enum {
#define EXTRACT_INFO_FIELDS do { \
salt = info->salt; \
rec_seq = info->rec_seq; \
+ iv = info->iv; \
salt_sz = sizeof(info->salt); \
rec_seq_sz = sizeof(info->rec_seq); \
+ iv_sz = sizeof(info->iv); \
} while (0)
static void
@@ -24,9 +27,9 @@ fill_static_params(struct mlx5_wqe_tls_static_params_seg *params,
union mlx5e_crypto_info *crypto_info,
u32 key_id, u32 resync_tcp_sn)
{
+ u16 salt_sz, rec_seq_sz, iv_sz;
+ char *salt, *rec_seq, *iv;
char *initial_rn, *gcm_iv;
- u16 salt_sz, rec_seq_sz;
- char *salt, *rec_seq;
u8 tls_version;
u8 *ctx;
@@ -59,7 +62,12 @@ fill_static_params(struct mlx5_wqe_tls_static_params_seg *params,
memcpy(gcm_iv, salt, salt_sz);
memcpy(initial_rn, rec_seq, rec_seq_sz);
- tls_version = MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_2;
+ if (crypto_info->crypto_info.version == TLS_1_3_VERSION) {
+ memcpy(gcm_iv + salt_sz, iv, iv_sz);
+ tls_version = MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_3;
+ } else {
+ tls_version = MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_2;
+ }
MLX5_SET(tls_static_params, ctx, tls_version, tls_version);
MLX5_SET(tls_static_params, ctx, const_1, 1);
--
2.25.1
^ permalink raw reply related
* [PATCH v15 1/9] net: tls: reject TLS 1.3 offload in chcr_ktls and nfp drivers
From: Rishikesh Jethwani @ 2026-07-09 20:53 UTC (permalink / raw)
To: netdev
Cc: saeedm, tariqt, mbloch, borisp, john.fastabend, kuba, sd, davem,
pabeni, edumazet, leon, andrew.gospodarek, Rishikesh Jethwani
In-Reply-To: <20260709205325.1591196-1-rjethwani@purestorage.com>
These drivers only support TLS 1.2. Return early when TLS 1.3
is requested to prevent unsupported hardware offload attempts.
Signed-off-by: Rishikesh Jethwani <rjethwani@purestorage.com>
---
drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c | 3 +++
drivers/net/ethernet/netronome/nfp/crypto/tls.c | 3 +++
2 files changed, 6 insertions(+)
diff --git a/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c
index f5acd4be1e69..29e108ce6764 100644
--- a/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c
+++ b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c
@@ -431,6 +431,9 @@ static int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk,
atomic64_inc(&port_stats->ktls_tx_connection_open);
u_ctx = adap->uld[CXGB4_ULD_KTLS].handle;
+ if (crypto_info->version != TLS_1_2_VERSION)
+ goto out;
+
if (direction == TLS_OFFLOAD_CTX_DIR_RX) {
pr_err("not expecting for RX direction\n");
goto out;
diff --git a/drivers/net/ethernet/netronome/nfp/crypto/tls.c b/drivers/net/ethernet/netronome/nfp/crypto/tls.c
index 9983d7aa2b9c..13864c6a55dc 100644
--- a/drivers/net/ethernet/netronome/nfp/crypto/tls.c
+++ b/drivers/net/ethernet/netronome/nfp/crypto/tls.c
@@ -287,6 +287,9 @@ nfp_net_tls_add(struct net_device *netdev, struct sock *sk,
BUILD_BUG_ON(offsetof(struct nfp_net_tls_offload_ctx, rx_end) >
TLS_DRIVER_STATE_SIZE_RX);
+ if (crypto_info->version != TLS_1_2_VERSION)
+ return -EOPNOTSUPP;
+
if (!nfp_net_cipher_supported(nn, crypto_info->cipher_type, direction))
return -EOPNOTSUPP;
--
2.25.1
^ permalink raw reply related
* [PATCH net-next v15 0/9] tls: Add TLS 1.3 hardware offload support
From: Rishikesh Jethwani @ 2026-07-09 20:53 UTC (permalink / raw)
To: netdev
Cc: saeedm, tariqt, mbloch, borisp, john.fastabend, kuba, sd, davem,
pabeni, edumazet, leon, andrew.gospodarek, Rishikesh Jethwani
Hi all,
This series adds TLS 1.3 hardware offload support including KeyUpdate
(rekey) and a selftest for validation.
Patch 1: Reject TLS 1.3 offload in chcr_ktls and nfp drivers
These drivers only support TLS 1.2; add explicit version check.
Patch 2: mlx5e TLS 1.3 hardware offload
Add TLS 1.3 TX/RX offload on ConnectX-6 Dx and newer.
Handle 12-byte IV format and TLS_1_3 context type.
Patch 3: Core TLS 1.3 hardware offload support
Extend tls_device.c for TLS 1.3 record format (content type
appended before tag). Handle TLS 1.3 IV construction in fallback.
Patch 4: Split tls_set_sw_offload into init/finalize
Patch 5: Prep helpers and refactors for HW offload KeyUpdate
Patch 6: TX KeyUpdate support
Patch 7: RX KeyUpdate support
Patch 8: Tracepoints for RX KeyUpdate path
Patch 9: Selftest for hardware offload
Rishikesh
Changes in v15:
- Added tls_dev_rx_rekey_fixup callback
- Addressed review comments
Rishikesh Jethwani (9):
net: tls: reject TLS 1.3 offload in chcr_ktls and nfp drivers
net/mlx5e: add TLS 1.3 hardware offload support
tls: add TLS 1.3 hardware offload support
tls: split tls_set_sw_offload into init and finalize stages
tls: prep helpers and refactors for HW offload KeyUpdate
tls: device: add TX KeyUpdate support
tls: device: add RX KeyUpdate support
tls: device: add tracepoints for the KeyUpdate path
selftests: net: add TLS hardware offload test
MAINTAINERS | 2 +
.../chelsio/inline_crypto/ch_ktls/chcr_ktls.c | 3 +
.../mellanox/mlx5/core/en_accel/ktls.c | 10 +
.../mellanox/mlx5/core/en_accel/ktls.h | 8 +-
.../mellanox/mlx5/core/en_accel/ktls_txrx.c | 14 +-
.../net/ethernet/netronome/nfp/crypto/tls.c | 3 +
include/net/tls.h | 90 +-
include/uapi/linux/snmp.h | 6 +
net/tls/tls.h | 22 +-
net/tls/tls_device.c | 876 ++++++++++++++--
net/tls/tls_device_fallback.c | 82 +-
net/tls/tls_main.c | 68 +-
net/tls/tls_proc.c | 6 +
net/tls/tls_sw.c | 133 ++-
net/tls/trace.h | 98 ++
.../selftests/drivers/net/hw/.gitignore | 1 +
.../testing/selftests/drivers/net/hw/Makefile | 2 +
.../selftests/drivers/net/hw/tls_hw_offload.c | 975 ++++++++++++++++++
.../drivers/net/hw/tls_hw_offload.py | 295 ++++++
19 files changed, 2498 insertions(+), 196 deletions(-)
create mode 100644 tools/testing/selftests/drivers/net/hw/tls_hw_offload.c
create mode 100755 tools/testing/selftests/drivers/net/hw/tls_hw_offload.py
--
2.25.1
^ permalink raw reply
* Re: [PATCH nf] netfilter: flowtable: tear down HW offloaded flows on FIB route changes
From: Ahmed Zaki @ 2026-07-09 20:33 UTC (permalink / raw)
To: Pablo Neira Ayuso
Cc: netfilter-devel, fw, kuba, edumazet, davem, pabeni, horms, netdev,
Ahmed Zaki
In-Reply-To: <ak_VoSJ7fozDdOzM@chamomile>
On Thu, Jul 9, 2026 at 11:08 AM Pablo Neira Ayuso <pablo@netfilter.org> wrote:
>
> On Wed, Jul 08, 2026 at 02:54:04PM -0600, Ahmed Zaki wrote:
> > Hardware-offloaded flows bypass the CPU and, unlike the software
> > datapath, dst_check() does not invalidate them when a route changes.
> > For ephemeral flows, this is usually not a problem as the flow expire on
> > its own and the driver clears the entry in the HW. However, for persistent
> > flows forwarded through the device, the HW is never informed that the
> > route has expired.
> >
> > For tables marked with NF_FLOWTABLE_HW_OFFLOAD, listen to the per-net FIB
> > notifier chain and tear down the affected flows so they are re-evaluated by
> > the SW forwarding path.
> >
> > A lockless list is used to reduce the work items overhead in case of a
> > route change storm allowing many FIB events to be processed by one work
> > item.
>
> This walks the hashtable anyway in case of fib event, maybe simply
> walk over the hashtable and call dst_check() to check if the cached
> dst is still current.
Good idea. I will replace the tuple matches and the lockless list with a
dst_check in the hashtable iter func: nf_flow_offload_fib_cb().
> > Fixes: c29f74e0df7a ("netfilter: nf_flow_table: hardware offload support")
>
> No, this is an enhancement, not a fix. This must be targeted to nf-next.
>
Will tag v2 to nf-next.
Thanks.
Ahmed
^ permalink raw reply
* [PATCH net-next v5 2/2] net: dsa: realtek: rtl8365mb: add HSGMII support for RTL8367S
From: Johan Alvarado @ 2026-07-09 20:26 UTC (permalink / raw)
To: linusw, alsi, andrew, olteanv, kuba, davem, edumazet, pabeni,
linux
Cc: luizluca, maxime.chevallier, namiltd, netdev, linux-kernel,
contact
In-Reply-To: <20260709202531.17166-1-contact@c127.dev>
In addition to SGMII, the RTL8367S SerDes also supports HSGMII, which
carries 2.5 Gbps with the same signaling as SGMII at 2.5x clock rate.
The chip info table already declares HSGMII as a supported interface
mode for external interface 1.
Extend the SerDes PCS to handle HSGMII, which phylink represents as
2500base-x:
- Select the HSGMII SerDes tuning parameters and external interface
mode, and mux the SerDes to MAC8 in HSGMII mode, from pcs_config()
according to the interface. The parameters are again lifted from the
GPL-licensed Realtek rtl8367c vendor driver, and again only cover
the tuning variant for a non-zero chip option, so the mode is gated
on the option probed at setup.
- Advertise 2500base-x and MAC_2500FD on ports whose external
interface supports HSGMII.
- Accept SPEED_2500 in the forced link configuration. The MAC speed
field has no 2.5 Gbps value: the rate is determined by the HSGMII
SerDes configuration, and the vendor driver programs the 1 Gbps
value here, so do the same.
- Raise the port 6 ingress and egress rate limiters to their maximum
at setup time, as the vendor switch init does unconditionally for
the whole chip family. The chip resets them to 0x1FFFF (~1.048 Gbps
in units of 8 Kbps), which caps the aggregate HSGMII throughput at
roughly 1 Gbps. The vendor documentation describes the reset
default as disabling the limiter, but the cap is real: on an
RTL8367S-based Mercusys MR85X running an OpenWrt backport of this
series, several clients on 1 Gbps user ports were limited to about
1.02 Gbps combined across the HSGMII CPU port until these limiters
were raised, after which throughput reached about 2 Gbps [1]. The
related HSGMII scheduler line rate (LINE_RATE_HSG_H) is already set
to its maximum by the common init jam table.
Tested on a Mercusys MR80X v2.20, where the RTL8367S is connected to
the SoC over HSGMII.
Link: https://github.com/openwrt/openwrt/pull/19445#issuecomment-4505613294 [1]
Suggested-by: Luiz Angelo Daros de Luca <luizluca@gmail.com>
Suggested-by: Mieczyslaw Nalewaj <namiltd@yahoo.com>
Signed-off-by: Johan Alvarado <contact@c127.dev>
---
drivers/net/dsa/realtek/rtl8365mb_main.c | 134 ++++++++++++++++++++---
1 file changed, 118 insertions(+), 16 deletions(-)
diff --git a/drivers/net/dsa/realtek/rtl8365mb_main.c b/drivers/net/dsa/realtek/rtl8365mb_main.c
index ea03c42d0f1a..d1ba0cc9426f 100644
--- a/drivers/net/dsa/realtek/rtl8365mb_main.c
+++ b/drivers/net/dsa/realtek/rtl8365mb_main.c
@@ -40,8 +40,8 @@
* driver has only been tested with a fixed-link, but in principle it should not
* matter.
*
- * NOTE: Currently, only the RGMII and SGMII interfaces are implemented in this
- * driver.
+ * NOTE: Currently, only the RGMII, SGMII and HSGMII interfaces are implemented
+ * in this driver.
*
* The interrupt line is asserted on link UP/DOWN events. The driver creates a
* custom irqchip to handle this interrupt and demultiplex the events by reading
@@ -251,6 +251,18 @@
#define RTL8365MB_BYPASS_LINE_RATE_REG 0x03F7
#define RTL8365MB_BYPASS_LINE_RATE_MASK(_port) BIT((_port) - 5)
+/* Port 6 ingress and egress rate limiter registers. Each limit is a 19-bit
+ * value in units of 8 Kbps, split across a 16-bit LSB register (CTRL0) and a
+ * 3-bit MSB field (CTRL1). The chip resets them to 0x1FFFF; see
+ * rtl8365mb_sds_raise_rate_limits().
+ */
+#define RTL8365MB_INGRESSBW_PORT6_RATE_CTRL0_REG 0x00CF
+#define RTL8365MB_INGRESSBW_PORT6_RATE_CTRL1_REG 0x00D0
+#define RTL8365MB_INGRESSBW_PORT6_RATE_CTRL1_MASK 0x0007
+#define RTL8365MB_PORT6_EGRESSBW_CTRL0_REG 0x0398
+#define RTL8365MB_PORT6_EGRESSBW_CTRL1_REG 0x0399
+#define RTL8365MB_PORT6_EGRESSBW_CTRL1_MASK 0x0007
+
/* SerDes indirect access registers */
#define RTL8365MB_SDS_INDACS_CMD_REG 0x6600
#define RTL8365MB_SDS_INDACS_CMD_BUSY_MASK 0x0100
@@ -637,6 +649,18 @@ static const struct rtl8365mb_jam_tbl_entry rtl8365mb_sds_jam_sgmii[] = {
{ 0x0424, 0xD810 }, { 0x002E, 0x83F2 },
};
+/* HSGMII SerDes tuning parameters, lifted from the vendor driver sources. As
+ * with the SGMII table, the vendor driver keeps several variants and selects
+ * one based on the chip option register; these are the values for a non-zero
+ * option, which is what RTL8367S parts seen so far report. See
+ * rtl8365mb_sds_probe_option().
+ */
+static const struct rtl8365mb_jam_tbl_entry rtl8365mb_sds_jam_hsgmii[] = {
+ { 0x0500, 0x82F0 }, { 0x0501, 0xF195 }, { 0x0502, 0x31A2 },
+ { 0x0503, 0x7960 }, { 0x0504, 0x9728 }, { 0x0423, 0x9D85 },
+ { 0x0424, 0xD810 }, { 0x0001, 0x0F80 }, { 0x002E, 0x83F2 },
+};
+
enum rtl8365mb_phy_interface_mode {
RTL8365MB_PHY_INTERFACE_MODE_INVAL = 0,
RTL8365MB_PHY_INTERFACE_MODE_INTERNAL = BIT(0),
@@ -1242,20 +1266,70 @@ static int rtl8365mb_sds_probe_option(struct realtek_priv *priv)
return 0;
}
+/* The vendor driver raises the port 6 ingress and egress rate limiters to
+ * their maximum in its switch init, unconditionally for the whole chip
+ * family. The chip reset in rtl8365mb_setup() puts them back to their reset
+ * default of 0x1FFFF, a ~1.048 Gbps limit which caps the aggregate
+ * throughput of an HSGMII CPU port at roughly 1 Gbps. The vendor
+ * documentation describes the reset default as disabling the limiter, but
+ * the cap has been observed on hardware. Raise them likewise, to 0x7FFFF
+ * (~4.19 Gbps, above the HSGMII line rate). The related HSGMII scheduler
+ * line rate register (LINE_RATE_HSG_H, 0x03FA) is already set to its
+ * maximum by the common init jam table.
+ */
+static int rtl8365mb_sds_raise_rate_limits(struct realtek_priv *priv)
+{
+ int ret;
+
+ ret = regmap_write(priv->map, RTL8365MB_INGRESSBW_PORT6_RATE_CTRL0_REG,
+ 0xFFFF);
+ if (ret)
+ return ret;
+
+ ret = regmap_update_bits(priv->map,
+ RTL8365MB_INGRESSBW_PORT6_RATE_CTRL1_REG,
+ RTL8365MB_INGRESSBW_PORT6_RATE_CTRL1_MASK,
+ RTL8365MB_INGRESSBW_PORT6_RATE_CTRL1_MASK);
+ if (ret)
+ return ret;
+
+ ret = regmap_write(priv->map, RTL8365MB_PORT6_EGRESSBW_CTRL0_REG,
+ 0xFFFF);
+ if (ret)
+ return ret;
+
+ return regmap_update_bits(priv->map, RTL8365MB_PORT6_EGRESSBW_CTRL1_REG,
+ RTL8365MB_PORT6_EGRESSBW_CTRL1_MASK,
+ RTL8365MB_PORT6_EGRESSBW_CTRL1_MASK);
+}
+
static int rtl8365mb_pcs_config(struct phylink_pcs *pcs, unsigned int neg_mode,
phy_interface_t interface,
const unsigned long *advertising,
bool permit_pause_to_mac)
{
+ const struct rtl8365mb_jam_tbl_entry *sds_jam;
const int id = RTL8365MB_SDS_EXT_INTERFACE_ID;
struct rtl8365mb *mb = pcs_to_rtl8365mb(pcs);
struct realtek_priv *priv;
+ size_t sds_jam_size;
+ u32 mode;
u16 val;
int ret;
int i;
priv = mb->priv;
+ if (interface == PHY_INTERFACE_MODE_2500BASEX) {
+ sds_jam = rtl8365mb_sds_jam_hsgmii;
+ sds_jam_size = ARRAY_SIZE(rtl8365mb_sds_jam_hsgmii);
+ mode = RTL8365MB_EXT_PORT_MODE_HSGMII;
+ } else {
+ sds_jam = rtl8365mb_sds_jam_sgmii;
+ sds_jam_size = ARRAY_SIZE(rtl8365mb_sds_jam_sgmii);
+ mode = RTL8365MB_EXT_PORT_MODE_SGMII;
+ }
+
/* Hold the embedded DW8051 microcontroller in reset and keep it
* disabled. The vendor driver loads firmware into it to manage the
* SerDes link, but the firmware only duplicates work that phylink
@@ -1283,24 +1357,24 @@ static int rtl8365mb_pcs_config(struct phylink_pcs *pcs, unsigned int neg_mode,
return ret;
/* Tune the SerDes with vendor-prescribed parameters */
- for (i = 0; i < ARRAY_SIZE(rtl8365mb_sds_jam_sgmii); i++) {
- ret = rtl8365mb_sds_write(priv,
- rtl8365mb_sds_jam_sgmii[i].reg,
- rtl8365mb_sds_jam_sgmii[i].val);
+ for (i = 0; i < sds_jam_size; i++) {
+ ret = rtl8365mb_sds_write(priv, sds_jam[i].reg,
+ sds_jam[i].val);
if (ret)
return ret;
}
- /* Mux the SerDes to MAC8 in SGMII mode */
+ /* Mux the SerDes to MAC8 in the requested mode */
ret = regmap_update_bits(priv->map, RTL8365MB_SDS_MISC_REG,
RTL8365MB_SDS_MISC_MAC8_SEL_SGMII_MASK |
RTL8365MB_SDS_MISC_MAC8_SEL_HSGMII_MASK,
- RTL8365MB_SDS_MISC_MAC8_SEL_SGMII_MASK);
+ mode == RTL8365MB_EXT_PORT_MODE_SGMII ?
+ RTL8365MB_SDS_MISC_MAC8_SEL_SGMII_MASK :
+ RTL8365MB_SDS_MISC_MAC8_SEL_HSGMII_MASK);
if (ret)
return ret;
- val = RTL8365MB_EXT_PORT_MODE_SGMII
- << RTL8365MB_DIGITAL_INTERFACE_SELECT_MODE_OFFSET(id);
+ val = mode << RTL8365MB_DIGITAL_INTERFACE_SELECT_MODE_OFFSET(id);
ret = regmap_update_bits(priv->map,
RTL8365MB_DIGITAL_INTERFACE_SELECT_REG(id),
RTL8365MB_DIGITAL_INTERFACE_SELECT_MODE_MASK(id),
@@ -1346,7 +1420,8 @@ static int rtl8365mb_pcs_config(struct phylink_pcs *pcs, unsigned int neg_mode,
static bool rtl8365mb_interface_is_serdes(phy_interface_t interface)
{
- return interface == PHY_INTERFACE_MODE_SGMII;
+ return interface == PHY_INTERFACE_MODE_SGMII ||
+ interface == PHY_INTERFACE_MODE_2500BASEX;
}
static unsigned int rtl8365mb_pcs_inband_caps(struct phylink_pcs *pcs,
@@ -1401,7 +1476,9 @@ static void rtl8365mb_pcs_get_state(struct phylink_pcs *pcs,
switch (FIELD_GET(RTL8365MB_SDS_MISC_SGMII_SPD_MASK, val)) {
case RTL8365MB_PORT_SPEED_1000M:
- state->speed = SPEED_1000;
+ state->speed =
+ state->interface == PHY_INTERFACE_MODE_2500BASEX ?
+ SPEED_2500 : SPEED_1000;
break;
case RTL8365MB_PORT_SPEED_100M:
state->speed = SPEED_100;
@@ -1426,7 +1503,11 @@ static void rtl8365mb_pcs_link_up(struct phylink_pcs *pcs,
u32 r_speed;
int ret;
- if (speed == SPEED_1000) {
+ /* The speed field has no value for 2.5 Gbps: the rate is determined by
+ * the HSGMII SerDes configuration, and the vendor driver programs the
+ * 1 Gbps value here.
+ */
+ if (speed == SPEED_2500 || speed == SPEED_1000) {
r_speed = RTL8365MB_PORT_SPEED_1000M;
} else if (speed == SPEED_100) {
r_speed = RTL8365MB_PORT_SPEED_100M;
@@ -1486,7 +1567,11 @@ static int rtl8365mb_ext_config_forcemode(struct realtek_priv *priv, int port,
r_rx_pause = rx_pause ? 1 : 0;
r_tx_pause = tx_pause ? 1 : 0;
- if (speed == SPEED_1000) {
+ /* The speed field has no value for 2.5 Gbps: the rate is
+ * determined by the HSGMII SerDes configuration, and the
+ * vendor driver programs the 1 Gbps value here.
+ */
+ if (speed == SPEED_2500 || speed == SPEED_1000) {
r_speed = RTL8365MB_PORT_SPEED_1000M;
} else if (speed == SPEED_100) {
r_speed = RTL8365MB_PORT_SPEED_100M;
@@ -1569,6 +1654,13 @@ static void rtl8365mb_phylink_get_caps(struct dsa_switch *ds, int port,
mb->sds_supported)
__set_bit(PHY_INTERFACE_MODE_SGMII,
config->supported_interfaces);
+
+ if (extint->supported_interfaces & RTL8365MB_PHY_INTERFACE_MODE_HSGMII &&
+ mb->sds_supported) {
+ __set_bit(PHY_INTERFACE_MODE_2500BASEX,
+ config->supported_interfaces);
+ config->mac_capabilities |= MAC_2500FD;
+ }
}
static struct phylink_pcs *
@@ -1610,8 +1702,8 @@ static void rtl8365mb_phylink_mac_config(struct phylink_config *config,
return;
}
- /* SGMII is handled by the SerDes PCS, configured through the
- * phylink_pcs ops, so there is nothing to do here for it.
+ /* SGMII and 2500base-x are handled by the SerDes PCS, configured
+ * through the phylink_pcs ops, so nothing to do here for them.
*/
if (rtl8365mb_interface_is_serdes(state->interface))
return;
@@ -2940,6 +3032,16 @@ static int rtl8365mb_setup(struct dsa_switch *ds)
goto out_error;
}
+ if (mb->sds_supported) {
+ ret = rtl8365mb_sds_raise_rate_limits(priv);
+ if (ret) {
+ dev_err(priv->dev,
+ "failed to raise port rate limits: %pe\n",
+ ERR_PTR(ret));
+ goto out_error;
+ }
+ }
+
/* Set up cascading IRQs */
ret = rtl8365mb_irq_setup(priv);
if (ret == -EPROBE_DEFER)
--
2.55.0
^ permalink raw reply related
* [PATCH net-next v5 1/2] net: dsa: realtek: rtl8365mb: add SGMII support for RTL8367S
From: Johan Alvarado @ 2026-07-09 20:25 UTC (permalink / raw)
To: linusw, alsi, andrew, olteanv, kuba, davem, edumazet, pabeni,
linux
Cc: luizluca, maxime.chevallier, namiltd, netdev, linux-kernel,
contact
In-Reply-To: <20260709202531.17166-1-contact@c127.dev>
The RTL8367S can mux its embedded SerDes to external interface 1,
which is typically used to connect the switch to a CPU port. The chip
info table already declares SGMII as a supported interface mode for
this chip, but the driver only implements RGMII so far.
Implement SGMII support as a phylink PCS, with the configuration
sequence derived from the GPL-licensed Realtek rtl8367c vendor driver
as distributed in the Mercusys MR80X GPL code drop:
- Add accessors for the SerDes indirect access registers (SDS_INDACS),
through which the SerDes internal registers are reached.
- Register a phylink_pcs for the SerDes, selected from mac_select_pcs
for the SGMII interface, so the SerDes handling lives in the PCS
operations rather than in the MAC operations.
- Probe the SerDes tuning variant from the chip option register once
at setup. The vendor driver keeps two sets of SerDes tuning
parameters and selects between them based on this option; only the
variant for a non-zero option (which all RTL8367S parts seen so far
report) has been validated on hardware, so the SerDes interface
modes are only advertised in that case. An unsupported variant thus
fails at phylink validation time instead of at link configuration
time.
- Keep the embedded DW8051 microcontroller in reset and disabled. The
vendor driver loads firmware into it to manage the SerDes link, but
analysis of that firmware shows it only duplicates the link
management phylink already performs: it polls the port status and
writes the external interface force registers behind the driver's
back.
- Clear the line rate bypass bit for the external interface, tune the
SerDes with the vendor-prescribed parameters, mux the SerDes to MAC8
in SGMII mode and only then take the SerDes out of reset, as the
vendor driver does.
- After deasserting the SerDes reset, reset the SerDes data path via
the SerDes BMCR register to flush the FIFOs and resync the PLL.
This mirrors what the vendor firmware does right after deasserting
the SerDes reset, and ensures a clean link state from cold boot.
- Force the SGMII link parameters (link, speed, duplex) in the SDS_MISC
register from pcs_link_up(). SGMII in-band autonegotiation is not
implemented, so only fixed-link and conventional PHY setups are
supported, just like RGMII. This is reported to phylink through
pcs_inband_caps() returning LINK_INBAND_DISABLE, so phylink never
selects an in-band-enabled negotiation mode for this PCS.
- Program the SerDes pause enables in SDS_MISC from the resolved
pause modes when forcing the MAC external interface in mac_link_up,
as the vendor driver does, rather than leaving whatever state the
boot firmware left there. Flow control testing shows these bits,
not the MAC force pause bits, gate pause on the SerDes external
interface. This is done in the MAC layer because pcs_link_up()
carries no pause information.
- Implement pcs_get_state() by reading the link status from the
SerDes, with the forced speed and duplex read back from SDS_MISC.
Although the supported fixed-link and conventional PHY setups do not
use it, the PCS owns the SerDes link state, and phylink consults
pcs_get_state() to track the physical link when operating in in-band
mode with autonegotiation disabled. The SerDes has no link interrupt
wired up, so the PCS sets its poll flag.
Tested on a Mercusys MR80X v2.20, where the RTL8367S is connected to
the SoC over SGMII.
Suggested-by: Luiz Angelo Daros de Luca <luizluca@gmail.com>
Suggested-by: Maxime Chevallier <maxime.chevallier@bootlin.com>
Suggested-by: Mieczyslaw Nalewaj <namiltd@yahoo.com>
Signed-off-by: Johan Alvarado <contact@c127.dev>
---
drivers/net/dsa/realtek/rtl8365mb_main.c | 515 ++++++++++++++++++++++-
1 file changed, 511 insertions(+), 4 deletions(-)
diff --git a/drivers/net/dsa/realtek/rtl8365mb_main.c b/drivers/net/dsa/realtek/rtl8365mb_main.c
index 5ac091bf93c9..ea03c42d0f1a 100644
--- a/drivers/net/dsa/realtek/rtl8365mb_main.c
+++ b/drivers/net/dsa/realtek/rtl8365mb_main.c
@@ -40,7 +40,8 @@
* driver has only been tested with a fixed-link, but in principle it should not
* matter.
*
- * NOTE: Currently, only the RGMII interface is implemented in this driver.
+ * NOTE: Currently, only the RGMII and SGMII interfaces are implemented in this
+ * driver.
*
* The interrupt line is asserted on link UP/DOWN events. The driver creates a
* custom irqchip to handle this interrupt and demultiplex the events by reading
@@ -94,11 +95,13 @@
#include <linux/bitops.h>
#include <linux/interrupt.h>
#include <linux/irqdomain.h>
+#include <linux/mii.h>
#include <linux/mutex.h>
#include <linux/of_irq.h>
#include <linux/regmap.h>
#include <linux/if_bridge.h>
#include <linux/if_vlan.h>
+#include <linux/phylink.h>
#include "realtek.h"
#include "realtek-smi.h"
@@ -129,6 +132,7 @@
/* Chip reset register */
#define RTL8365MB_CHIP_RESET_REG 0x1322
+#define RTL8365MB_CHIP_RESET_DW8051_MASK 0x0010
#define RTL8365MB_CHIP_RESET_SW_MASK 0x0002
#define RTL8365MB_CHIP_RESET_HW_MASK 0x0001
@@ -238,6 +242,76 @@
#define RTL8365MB_EXT_RGMXF_RXDELAY_MASK 0x0007
#define RTL8365MB_EXT_RGMXF_TXDELAY_MASK 0x0008
+/* External interface line rate bypass register - one bit per external
+ * interface, indexed by the external port number with port 5 (the first
+ * external port) as the base. Other RTL8367 families index this register
+ * differently (e.g. the RTL8367R uses (id + 1) % 2), so this mapping only
+ * holds for the RTL8367C-style parts this driver supports.
+ */
+#define RTL8365MB_BYPASS_LINE_RATE_REG 0x03F7
+#define RTL8365MB_BYPASS_LINE_RATE_MASK(_port) BIT((_port) - 5)
+
+/* SerDes indirect access registers */
+#define RTL8365MB_SDS_INDACS_CMD_REG 0x6600
+#define RTL8365MB_SDS_INDACS_CMD_BUSY_MASK 0x0100
+#define RTL8365MB_SDS_INDACS_CMD_RUN_MASK 0x0080
+#define RTL8365MB_SDS_INDACS_CMD_WR_MASK 0x0040
+#define RTL8365MB_SDS_INDACS_ADR_REG 0x6601
+#define RTL8365MB_SDS_INDACS_DATA_REG 0x6602
+
+/* SerDes miscellaneous configuration register */
+#define RTL8365MB_SDS_MISC_REG 0x1D11
+#define RTL8365MB_SDS_MISC_SGMII_RXFC_MASK 0x4000
+#define RTL8365MB_SDS_MISC_SGMII_TXFC_MASK 0x2000
+#define RTL8365MB_SDS_MISC_MAC8_SEL_HSGMII_MASK 0x0800
+#define RTL8365MB_SDS_MISC_SGMII_FDUP_MASK 0x0400
+#define RTL8365MB_SDS_MISC_SGMII_LINK_MASK 0x0200
+#define RTL8365MB_SDS_MISC_SGMII_SPD_MASK 0x0180
+#define RTL8365MB_SDS_MISC_MAC8_SEL_SGMII_MASK 0x0040
+
+/* SerDes internal registers, accessed via the SDS_INDACS registers. The BMCR
+ * data path reset holds BMCR_ANENABLE | BMCR_ISOLATE while toggling the
+ * vendor-specific low bits from phase 1 to phase 2, which triggers a data path
+ * reset and PLL resync.
+ */
+#define RTL8365MB_SDS_REG_BMCR 0x0000
+#define RTL8365MB_SDS_BMCR_DPRST_PHASE1 (BMCR_ANENABLE | BMCR_ISOLATE | 0x1)
+#define RTL8365MB_SDS_BMCR_DPRST_PHASE2 (BMCR_ANENABLE | BMCR_ISOLATE | 0x3)
+#define RTL8365MB_SDS_REG_NWAY 0x0002
+#define RTL8365MB_SDS_NWAY_EN_MASK 0x0200
+#define RTL8365MB_SDS_NWAY_RESTART_MASK 0x0100
+#define RTL8365MB_SDS_REG_RESET 0x0003
+#define RTL8365MB_SDS_RESET_DEASSERT 0x7106
+#define RTL8365MB_SDS_REG_LINK_STATUS 0x003d
+#define RTL8365MB_SDS_LINK_STATUS_LINK_MASK 0x0010
+
+/* The embedded SerDes can only be muxed to external interface 1 (MAC8),
+ * which is port 6.
+ */
+#define RTL8365MB_SDS_EXT_INTERFACE_ID 1
+#define RTL8365MB_SDS_EXT_INTERFACE_PORT 6
+
+/* Line rate bypass bit for the SerDes external interface */
+#define RTL8365MB_SDS_BYPASS_LINE_RATE_MASK \
+ RTL8365MB_BYPASS_LINE_RATE_MASK(RTL8365MB_SDS_EXT_INTERFACE_PORT)
+
+/* SerDes tuning parameter variant selector. The vendor driver picks between
+ * two sets of SerDes tuning parameters based on this chip option. Reading it
+ * requires first arming the read by writing a magic key to the arm register,
+ * then disarming it afterwards.
+ */
+#define RTL8365MB_SDS_OPTION_ARM_REG 0x13C0
+#define RTL8365MB_SDS_OPTION_ARM_KEY 0x0249
+#define RTL8365MB_SDS_OPTION_REG 0x13C1
+
+/* Embedded DW8051 microcontroller control registers. The microcontroller
+ * can run firmware to manage the SerDes link, but this driver keeps it in
+ * reset and disabled: phylink already performs the link management that
+ * the firmware would otherwise do.
+ */
+#define RTL8365MB_MISC_CFG0_REG 0x130C
+#define RTL8365MB_MISC_CFG0_DW8051_EN_MASK 0x0020
+
/* External interface port speed values - used in DIGITAL_INTERFACE_FORCE */
#define RTL8365MB_PORT_SPEED_10M 0
#define RTL8365MB_PORT_SPEED_100M 1
@@ -551,6 +625,18 @@ static const struct rtl8365mb_jam_tbl_entry rtl8365mb_init_jam_common[] = {
{ 0x1D32, 0x0002 },
};
+/* SGMII SerDes tuning parameters, lifted from the vendor driver sources. The
+ * vendor driver keeps two variants of this table and selects between them
+ * based on the chip option register; these are the values for a non-zero
+ * option, which is what RTL8367S parts seen so far report. See
+ * rtl8365mb_sds_probe_option().
+ */
+static const struct rtl8365mb_jam_tbl_entry rtl8365mb_sds_jam_sgmii[] = {
+ { 0x0480, 0x04D7 }, { 0x0481, 0xF994 }, { 0x0482, 0x2420 },
+ { 0x0483, 0x6960 }, { 0x0484, 0x9728 }, { 0x0423, 0x9D85 },
+ { 0x0424, 0xD810 }, { 0x002E, 0x83F2 },
+};
+
enum rtl8365mb_phy_interface_mode {
RTL8365MB_PHY_INTERFACE_MODE_INVAL = 0,
RTL8365MB_PHY_INTERFACE_MODE_INTERNAL = BIT(0),
@@ -730,6 +816,9 @@ struct rtl8365mb_port {
* @cpu: CPU tagging and CPU port configuration for this chip
* @mib_lock: prevent concurrent reads of MIB counters
* @ports: per-port data
+ * @pcs: PCS for the SerDes external interface
+ * @sds_supported: SerDes tuning parameters match the chip option, so the
+ * SerDes interface modes can be advertised
*
* Private data for this driver.
*/
@@ -740,8 +829,12 @@ struct rtl8365mb {
struct rtl8365mb_cpu cpu;
struct mutex mib_lock;
struct rtl8365mb_port ports[RTL8365MB_MAX_NUM_PORTS];
+ struct phylink_pcs pcs;
+ bool sds_supported;
};
+#define pcs_to_rtl8365mb(_pcs) container_of((_pcs), struct rtl8365mb, pcs)
+
static int rtl8365mb_phy_poll_busy(struct realtek_priv *priv)
{
u32 val;
@@ -1042,6 +1135,334 @@ static int rtl8365mb_ext_config_rgmii(struct realtek_priv *priv, int port,
return 0;
}
+static int rtl8365mb_sds_write(struct realtek_priv *priv, u16 addr, u16 data)
+{
+ int ret;
+
+ ret = regmap_write(priv->map, RTL8365MB_SDS_INDACS_DATA_REG, data);
+ if (ret)
+ return ret;
+
+ ret = regmap_write(priv->map, RTL8365MB_SDS_INDACS_ADR_REG, addr);
+ if (ret)
+ return ret;
+
+ /* The SerDes indirect access engine completes the command within the
+ * register write transaction, so there is no need to wait or poll for
+ * completion before the next access, matching the vendor driver.
+ */
+ return regmap_write(priv->map, RTL8365MB_SDS_INDACS_CMD_REG,
+ RTL8365MB_SDS_INDACS_CMD_RUN_MASK |
+ RTL8365MB_SDS_INDACS_CMD_WR_MASK);
+}
+
+static int rtl8365mb_sds_read(struct realtek_priv *priv, u16 addr, u16 *data)
+{
+ u32 val;
+ int ret;
+
+ ret = regmap_write(priv->map, RTL8365MB_SDS_INDACS_ADR_REG, addr);
+ if (ret)
+ return ret;
+
+ ret = regmap_write(priv->map, RTL8365MB_SDS_INDACS_CMD_REG,
+ RTL8365MB_SDS_INDACS_CMD_RUN_MASK);
+ if (ret)
+ return ret;
+
+ /* Wait for the indirect read to complete: the engine clears the BUSY
+ * bit once the data register holds the result.
+ */
+ ret = regmap_read_poll_timeout(priv->map, RTL8365MB_SDS_INDACS_CMD_REG,
+ val,
+ !(val & RTL8365MB_SDS_INDACS_CMD_BUSY_MASK),
+ 10, 1000);
+ if (ret)
+ return ret;
+
+ ret = regmap_read(priv->map, RTL8365MB_SDS_INDACS_DATA_REG, &val);
+ if (ret)
+ return ret;
+
+ *data = val;
+
+ return 0;
+}
+
+/* The vendor driver selects between two sets of SerDes tuning parameters based
+ * on the chip option register. Only the variant for a non-zero option has been
+ * tested on real hardware - the RTL8367S parts seen so far all report 1. The
+ * variant for option 0 uses different tuning values that cannot be verified,
+ * so probe the option once at setup and only advertise the SerDes interface
+ * modes when the tuning parameters are known to match, so that an unsupported
+ * variant fails at phylink validation time rather than when configuring the
+ * link.
+ */
+static int rtl8365mb_sds_probe_option(struct realtek_priv *priv)
+{
+ struct rtl8365mb *mb = priv->chip_data;
+ const struct rtl8365mb_extint *extint;
+ u32 option;
+ int ret;
+ int i;
+
+ /* Nothing to probe if no external interface is wired to the SerDes */
+ for (i = 0; i < RTL8365MB_MAX_NUM_EXTINTS; i++) {
+ extint = &mb->chip_info->extints[i];
+
+ if (extint->supported_interfaces &
+ (RTL8365MB_PHY_INTERFACE_MODE_SGMII |
+ RTL8365MB_PHY_INTERFACE_MODE_HSGMII))
+ break;
+ }
+ if (i == RTL8365MB_MAX_NUM_EXTINTS)
+ return 0;
+
+ ret = regmap_write(priv->map, RTL8365MB_SDS_OPTION_ARM_REG,
+ RTL8365MB_SDS_OPTION_ARM_KEY);
+ if (ret)
+ return ret;
+
+ ret = regmap_read(priv->map, RTL8365MB_SDS_OPTION_REG, &option);
+ if (ret)
+ return ret;
+
+ ret = regmap_write(priv->map, RTL8365MB_SDS_OPTION_ARM_REG, 0);
+ if (ret)
+ return ret;
+
+ if (option == 0) {
+ dev_warn(priv->dev,
+ "unsupported SerDes tuning variant (chip option 0), disabling SerDes interface modes\n");
+ return 0;
+ }
+
+ mb->sds_supported = true;
+
+ return 0;
+}
+
+static int rtl8365mb_pcs_config(struct phylink_pcs *pcs, unsigned int neg_mode,
+ phy_interface_t interface,
+ const unsigned long *advertising,
+ bool permit_pause_to_mac)
+{
+ const int id = RTL8365MB_SDS_EXT_INTERFACE_ID;
+ struct rtl8365mb *mb = pcs_to_rtl8365mb(pcs);
+ struct realtek_priv *priv;
+ u16 val;
+ int ret;
+ int i;
+
+ priv = mb->priv;
+
+ /* Hold the embedded DW8051 microcontroller in reset and keep it
+ * disabled. The vendor driver loads firmware into it to manage the
+ * SerDes link, but the firmware only duplicates work that phylink
+ * already does: it polls the port status and forces the external
+ * interface configuration in the very registers this driver manages.
+ * Letting it run would race with phylink.
+ */
+ ret = regmap_update_bits(priv->map, RTL8365MB_CHIP_RESET_REG,
+ RTL8365MB_CHIP_RESET_DW8051_MASK,
+ RTL8365MB_CHIP_RESET_DW8051_MASK);
+ if (ret)
+ return ret;
+
+ ret = regmap_update_bits(priv->map, RTL8365MB_MISC_CFG0_REG,
+ RTL8365MB_MISC_CFG0_DW8051_EN_MASK, 0);
+ if (ret)
+ return ret;
+
+ /* The vendor driver clears the line rate bypass for all interface
+ * modes except TMII.
+ */
+ ret = regmap_update_bits(priv->map, RTL8365MB_BYPASS_LINE_RATE_REG,
+ RTL8365MB_SDS_BYPASS_LINE_RATE_MASK, 0);
+ if (ret)
+ return ret;
+
+ /* Tune the SerDes with vendor-prescribed parameters */
+ for (i = 0; i < ARRAY_SIZE(rtl8365mb_sds_jam_sgmii); i++) {
+ ret = rtl8365mb_sds_write(priv,
+ rtl8365mb_sds_jam_sgmii[i].reg,
+ rtl8365mb_sds_jam_sgmii[i].val);
+ if (ret)
+ return ret;
+ }
+
+ /* Mux the SerDes to MAC8 in SGMII mode */
+ ret = regmap_update_bits(priv->map, RTL8365MB_SDS_MISC_REG,
+ RTL8365MB_SDS_MISC_MAC8_SEL_SGMII_MASK |
+ RTL8365MB_SDS_MISC_MAC8_SEL_HSGMII_MASK,
+ RTL8365MB_SDS_MISC_MAC8_SEL_SGMII_MASK);
+ if (ret)
+ return ret;
+
+ val = RTL8365MB_EXT_PORT_MODE_SGMII
+ << RTL8365MB_DIGITAL_INTERFACE_SELECT_MODE_OFFSET(id);
+ ret = regmap_update_bits(priv->map,
+ RTL8365MB_DIGITAL_INTERFACE_SELECT_REG(id),
+ RTL8365MB_DIGITAL_INTERFACE_SELECT_MODE_MASK(id),
+ val);
+ if (ret)
+ return ret;
+
+ /* Take the SerDes out of reset. The vendor driver does this only
+ * after the SerDes mux and the interface mode are configured.
+ */
+ ret = rtl8365mb_sds_write(priv, RTL8365MB_SDS_REG_RESET,
+ RTL8365MB_SDS_RESET_DEASSERT);
+ if (ret)
+ return ret;
+
+ /* Reset the SerDes data path and resync its PLL, mirroring what the
+ * vendor firmware does right after deasserting the SerDes reset.
+ * This flushes the FIFOs and ensures a clean state for the link,
+ * preventing silent drops and CRC errors.
+ */
+ ret = rtl8365mb_sds_write(priv, RTL8365MB_SDS_REG_BMCR,
+ RTL8365MB_SDS_BMCR_DPRST_PHASE1);
+ if (ret)
+ return ret;
+
+ ret = rtl8365mb_sds_write(priv, RTL8365MB_SDS_REG_BMCR,
+ RTL8365MB_SDS_BMCR_DPRST_PHASE2);
+ if (ret)
+ return ret;
+
+ /* Keep SGMII in-band autonegotiation disabled: the link parameters are
+ * forced from rtl8365mb_pcs_link_up() instead.
+ */
+ ret = rtl8365mb_sds_read(priv, RTL8365MB_SDS_REG_NWAY, &val);
+ if (ret)
+ return ret;
+
+ val &= ~RTL8365MB_SDS_NWAY_EN_MASK;
+ val |= RTL8365MB_SDS_NWAY_RESTART_MASK;
+
+ return rtl8365mb_sds_write(priv, RTL8365MB_SDS_REG_NWAY, val);
+}
+
+static bool rtl8365mb_interface_is_serdes(phy_interface_t interface)
+{
+ return interface == PHY_INTERFACE_MODE_SGMII;
+}
+
+static unsigned int rtl8365mb_pcs_inband_caps(struct phylink_pcs *pcs,
+ phy_interface_t interface)
+{
+ /* In-band autonegotiation is not implemented; the link is always
+ * forced. Report that to phylink so that it never selects an
+ * in-band-enabled negotiation mode for this PCS.
+ */
+ return LINK_INBAND_DISABLE;
+}
+
+static void rtl8365mb_pcs_get_state(struct phylink_pcs *pcs,
+ unsigned int neg_mode,
+ struct phylink_link_state *state)
+{
+ struct rtl8365mb *mb = pcs_to_rtl8365mb(pcs);
+ struct realtek_priv *priv = mb->priv;
+ u16 status;
+ u32 val;
+ int ret;
+
+ /* In-band autonegotiation is not implemented, so the link parameters are
+ * forced from rtl8365mb_pcs_link_up(). The real link state must still be
+ * read from the SerDes itself: the embedded DW8051 microcontroller that
+ * the vendor firmware uses to poll the SerDes is kept disabled (see
+ * rtl8365mb_pcs_config()), so the link status register can be read
+ * directly through the SDS_INDACS window without racing the auto-poll.
+ */
+ ret = rtl8365mb_sds_read(priv, RTL8365MB_SDS_REG_LINK_STATUS, &status);
+ if (ret) {
+ state->link = false;
+ return;
+ }
+
+ state->link = !!(status & RTL8365MB_SDS_LINK_STATUS_LINK_MASK);
+ state->an_complete = state->link;
+ if (!state->link)
+ return;
+
+ /* The speed and duplex are forced; read them back from the values
+ * programmed into the SerDes MISC register.
+ */
+ ret = regmap_read(priv->map, RTL8365MB_SDS_MISC_REG, &val);
+ if (ret) {
+ state->link = false;
+ return;
+ }
+
+ state->duplex = (val & RTL8365MB_SDS_MISC_SGMII_FDUP_MASK) ?
+ DUPLEX_FULL : DUPLEX_HALF;
+
+ switch (FIELD_GET(RTL8365MB_SDS_MISC_SGMII_SPD_MASK, val)) {
+ case RTL8365MB_PORT_SPEED_1000M:
+ state->speed = SPEED_1000;
+ break;
+ case RTL8365MB_PORT_SPEED_100M:
+ state->speed = SPEED_100;
+ break;
+ case RTL8365MB_PORT_SPEED_10M:
+ state->speed = SPEED_10;
+ break;
+ }
+}
+
+static void rtl8365mb_pcs_link_up(struct phylink_pcs *pcs,
+ unsigned int neg_mode,
+ phy_interface_t interface, int speed,
+ int duplex)
+{
+ struct rtl8365mb *mb = pcs_to_rtl8365mb(pcs);
+ struct realtek_priv *priv = mb->priv;
+ u32 mask = RTL8365MB_SDS_MISC_SGMII_FDUP_MASK |
+ RTL8365MB_SDS_MISC_SGMII_LINK_MASK |
+ RTL8365MB_SDS_MISC_SGMII_SPD_MASK;
+ u32 val = RTL8365MB_SDS_MISC_SGMII_LINK_MASK;
+ u32 r_speed;
+ int ret;
+
+ if (speed == SPEED_1000) {
+ r_speed = RTL8365MB_PORT_SPEED_1000M;
+ } else if (speed == SPEED_100) {
+ r_speed = RTL8365MB_PORT_SPEED_100M;
+ } else if (speed == SPEED_10) {
+ r_speed = RTL8365MB_PORT_SPEED_10M;
+ } else {
+ dev_err(priv->dev, "unsupported SerDes speed %s\n",
+ phy_speed_to_str(speed));
+ return;
+ }
+
+ val |= FIELD_PREP(RTL8365MB_SDS_MISC_SGMII_SPD_MASK, r_speed);
+
+ if (duplex == DUPLEX_FULL)
+ val |= RTL8365MB_SDS_MISC_SGMII_FDUP_MASK;
+
+ /* pcs_link_up() carries no pause information, so the SerDes flow
+ * control bits are programmed together with the MAC external interface
+ * force from rtl8365mb_phylink_mac_link_up(), where the resolved pause
+ * modes are known.
+ */
+ ret = regmap_update_bits(priv->map, RTL8365MB_SDS_MISC_REG, mask, val);
+ if (ret) {
+ dev_err(priv->dev, "failed to force SerDes link: %pe\n",
+ ERR_PTR(ret));
+ return;
+ }
+}
+
+static const struct phylink_pcs_ops rtl8365mb_pcs_ops = {
+ .pcs_inband_caps = rtl8365mb_pcs_inband_caps,
+ .pcs_config = rtl8365mb_pcs_config,
+ .pcs_get_state = rtl8365mb_pcs_get_state,
+ .pcs_link_up = rtl8365mb_pcs_link_up,
+};
+
static int rtl8365mb_ext_config_forcemode(struct realtek_priv *priv, int port,
bool link, int speed, int duplex,
bool tx_pause, bool rx_pause)
@@ -1118,6 +1539,8 @@ static void rtl8365mb_phylink_get_caps(struct dsa_switch *ds, int port,
{
const struct rtl8365mb_extint *extint =
rtl8365mb_get_port_extint(ds->priv, port);
+ struct realtek_priv *priv = ds->priv;
+ struct rtl8365mb *mb = priv->chip_data;
config->mac_capabilities = MAC_SYM_PAUSE | MAC_ASYM_PAUSE |
MAC_10 | MAC_100 | MAC_1000FD;
@@ -1141,6 +1564,25 @@ static void rtl8365mb_phylink_get_caps(struct dsa_switch *ds, int port,
if (extint->supported_interfaces & RTL8365MB_PHY_INTERFACE_MODE_RGMII)
phy_interface_set_rgmii(config->supported_interfaces);
+
+ if (extint->supported_interfaces & RTL8365MB_PHY_INTERFACE_MODE_SGMII &&
+ mb->sds_supported)
+ __set_bit(PHY_INTERFACE_MODE_SGMII,
+ config->supported_interfaces);
+}
+
+static struct phylink_pcs *
+rtl8365mb_phylink_mac_select_pcs(struct phylink_config *config,
+ phy_interface_t interface)
+{
+ struct dsa_port *dp = dsa_phylink_to_port(config);
+ struct realtek_priv *priv = dp->ds->priv;
+ struct rtl8365mb *mb = priv->chip_data;
+
+ if (rtl8365mb_interface_is_serdes(interface))
+ return &mb->pcs;
+
+ return NULL;
}
static void rtl8365mb_phylink_mac_config(struct phylink_config *config,
@@ -1168,6 +1610,12 @@ static void rtl8365mb_phylink_mac_config(struct phylink_config *config,
return;
}
+ /* SGMII is handled by the SerDes PCS, configured through the
+ * phylink_pcs ops, so there is nothing to do here for it.
+ */
+ if (rtl8365mb_interface_is_serdes(state->interface))
+ return;
+
/* TODO: Implement MII and RMII modes, which the RTL8365MB-VC also
* supports
*/
@@ -1188,7 +1636,13 @@ static void rtl8365mb_phylink_mac_link_down(struct phylink_config *config,
p = &mb->ports[port];
cancel_delayed_work_sync(&p->mib_work);
- if (phy_interface_mode_is_rgmii(interface)) {
+ /* phylink has no pcs_link_down callback, so on the SerDes path only the
+ * MAC external interface force is reset here. Clearing the MAC force is
+ * enough to bring the link down; the SerDes keeps presenting its last
+ * forced state until the next pcs_link_up() reprograms it.
+ */
+ if (phy_interface_mode_is_rgmii(interface) ||
+ rtl8365mb_interface_is_serdes(interface)) {
ret = rtl8365mb_ext_config_forcemode(priv, port, false, 0, 0,
false, false);
if (ret)
@@ -1218,14 +1672,51 @@ static void rtl8365mb_phylink_mac_link_up(struct phylink_config *config,
p = &mb->ports[port];
schedule_delayed_work(&p->mib_work, 0);
- if (phy_interface_mode_is_rgmii(interface)) {
+ /* The SerDes forced link state is programmed by the PCS in
+ * rtl8365mb_pcs_link_up(); here only the MAC external interface force
+ * is configured, for both RGMII and SerDes.
+ */
+ if (phy_interface_mode_is_rgmii(interface) ||
+ rtl8365mb_interface_is_serdes(interface)) {
ret = rtl8365mb_ext_config_forcemode(priv, port, true, speed,
duplex, tx_pause,
rx_pause);
- if (ret)
+ if (ret) {
dev_err(priv->dev,
"failed to force mode on port %d: %pe\n", port,
ERR_PTR(ret));
+ return;
+ }
+
+ /* The SerDes has its own pause enables; program them from
+ * the resolved pause modes, as the vendor driver does when
+ * forcing the link on a SerDes external interface. These
+ * bits, not the MAC force pause bits, gate pause on the
+ * SerDes external interface: flow control testing shows
+ * that pause frames are only emitted with the SerDes TXFC
+ * bit set, while the MAC force pause bits alone have no
+ * effect on this port. This is done here rather than in
+ * rtl8365mb_pcs_link_up() because pcs_link_up() carries no
+ * pause information.
+ */
+ if (rtl8365mb_interface_is_serdes(interface)) {
+ u32 val = 0;
+
+ if (tx_pause)
+ val |= RTL8365MB_SDS_MISC_SGMII_TXFC_MASK;
+ if (rx_pause)
+ val |= RTL8365MB_SDS_MISC_SGMII_RXFC_MASK;
+
+ ret = regmap_update_bits(priv->map,
+ RTL8365MB_SDS_MISC_REG,
+ RTL8365MB_SDS_MISC_SGMII_TXFC_MASK |
+ RTL8365MB_SDS_MISC_SGMII_RXFC_MASK,
+ val);
+ if (ret)
+ dev_err(priv->dev,
+ "failed to force SerDes pause modes on port %d: %pe\n",
+ port, ERR_PTR(ret));
+ }
return;
}
@@ -2419,6 +2910,14 @@ static int rtl8365mb_setup(struct dsa_switch *ds)
mb = priv->chip_data;
cpu = &mb->cpu;
+ mb->pcs.ops = &rtl8365mb_pcs_ops;
+
+ /* The SerDes has no link interrupt wired up, so phylink must poll the
+ * PCS for link changes when it tracks the link through pcs_get_state()
+ * (in-band mode with autonegotiation disabled).
+ */
+ mb->pcs.poll = true;
+
ret = rtl8365mb_reset_chip(priv);
if (ret) {
dev_err(priv->dev, "failed to reset chip: %pe\n",
@@ -2426,6 +2925,13 @@ static int rtl8365mb_setup(struct dsa_switch *ds)
goto out_error;
}
+ ret = rtl8365mb_sds_probe_option(priv);
+ if (ret) {
+ dev_err(priv->dev, "failed to probe SerDes chip option: %pe\n",
+ ERR_PTR(ret));
+ goto out_error;
+ }
+
/* Configure switch to vendor-defined initial state */
ret = rtl8365mb_switch_init(priv);
if (ret) {
@@ -2658,6 +3164,7 @@ static int rtl8365mb_detect(struct realtek_priv *priv)
}
static const struct phylink_mac_ops rtl8365mb_phylink_mac_ops = {
+ .mac_select_pcs = rtl8365mb_phylink_mac_select_pcs,
.mac_config = rtl8365mb_phylink_mac_config,
.mac_link_down = rtl8365mb_phylink_mac_link_down,
.mac_link_up = rtl8365mb_phylink_mac_link_up,
--
2.55.0
^ permalink raw reply related
* [PATCH net-next v5 0/2] net: dsa: realtek: rtl8365mb: add SGMII/HSGMII support for RTL8367S
From: Johan Alvarado @ 2026-07-09 20:25 UTC (permalink / raw)
To: linusw, alsi, andrew, olteanv, kuba, davem, edumazet, pabeni,
linux
Cc: luizluca, maxime.chevallier, namiltd, netdev, linux-kernel,
contact
The RTL8367S is a 5+2 port switch from the same family as the
RTL8365MB-VC already supported by this driver. Its chip info table
entry declares SGMII and HSGMII on external interface 1, but the
driver so far only implements RGMII, leaving boards that wire the
switch to the CPU over the SerDes without a working CPU port.
This series implements both modes. The configuration sequence and the
SerDes tuning parameters are derived from the GPL-licensed Realtek
rtl8367c vendor driver, as distributed in the Mercusys MR80X GPL code
drop, and cross-checked against the real register sequence captured at
runtime by chainloading a custom U-Boot ahead of the stock firmware
and logging the live SerDes accesses on hardware.
The vendor driver brings up the SerDes by loading firmware into the
switch's embedded DW8051 microcontroller. Analysis of that firmware
(by Luiz Angelo Daros de Luca) showed it only performs a SerDes
data-path reset right after the SerDes reset is deasserted, and then
runs a link-polling loop that writes the external interface force
registers -- duplicating, and racing with, the link management phylink
already performs. This series therefore keeps the DW8051 disabled and
performs the one necessary action (the data-path reset via the SerDes
BMCR register) directly in the driver, avoiding both the race and a
dependency on a redistributable firmware blob.
The SerDes is modelled as a phylink PCS: mac_select_pcs() hands the
SerDes interfaces to a phylink_pcs whose pcs_config()/pcs_link_up()
ops own the SerDes register sequence, keeping it out of the MAC
operations. In-band autonegotiation is not implemented; the link is
forced (fixed-link or conventional PHY), as for RGMII, and the PCS
reports this to phylink through pcs_inband_caps().
Patch 1 adds the SerDes indirect access helpers, the PCS and SGMII
(1 Gbps) support. Patch 2 extends the PCS to HSGMII (2.5 Gbps), which
phylink represents as 2500base-x.
Tested on a Mercusys MR80X v2.20 (RTL8367S wired to the SoC over the
SerDes), in both SGMII and HSGMII modes with a fixed-link device tree
description: link bring-up verified across cold boots, warm reboots,
module reloads and link down/up cycles, with sustained traffic and no
CRC/symbol errors. The SerDes pause enables were verified by driving
congestion toward a 100M user port and observing pause frame emission
on the CPU port (dot3OutPauseFrames) toggle with the SDS_MISC
TXFC/RXFC bits. The port 6 rate limiters were verified to be live by
lowering them to 100 Mbps at runtime and observing iperf3 throughput
across the CPU port clamp accordingly in each direction, recovering
once the maximum was restored. The HSGMII link is confirmed running
at 2.5G at the register level (SoC uniphy mode and gmac clocks);
per-direction throughput could not be pushed past ~1 Gbps on this
board because the SoC side is driven by the IPQ5018 SSDK and the
user-facing PHY is 1G, so full 2.5G line-rate throughput remains
unverified on my hardware. Independently, >1 Gbps aggregate HSGMII
throughput (~2 Gbps with multiple clients) has been observed on an
RTL8367S-based Mercusys MR85X running an OpenWrt backport of this
series with the rate limiters raised (see patch 2).
The RTL8367SB also declares SGMII and HSGMII in its chip info entry
and therefore gains both modes as well. The vendor driver drives the
two chips through the same code path, keyed only on the chip option
register (both report chip id 0x6367), so this is expected to work
there too, but I have no RTL8367SB hardware to confirm it.
Signed-off-by: Johan Alvarado <contact@c127.dev>
---
v5:
- Raise the port 6 ingress and egress rate limiters to their maximum
at setup time, as the vendor switch init does. The chip resets them
to 0x1FFFF (~1.048 Gbps in units of 8 Kbps), which caps the
aggregate HSGMII throughput at roughly 1 Gbps; the cap was
confirmed on an RTL8367S-based Mercusys MR85X, where raising the
limiters took multi-client throughput across the HSGMII CPU port
from ~1.02 Gbps combined to ~2 Gbps (link in patch 2). The related
HSGMII scheduler line rate (LINE_RATE_HSG_H) is already set to its
maximum by the common init jam table. Pointed out by Mieczyslaw
Nalewaj.
- Drop the PHYLINK_PCS_NEG_INBAND_ENABLED check from pcs_config().
With pcs_inband_caps() returning LINK_INBAND_DISABLE, phylink can
never select an in-band-enabled negotiation mode for this PCS, so
the check was unreachable. Pointed out by Maxime Chevallier.
- Document the SerDes SDS_MISC pause bits as the functional pause
enables of the SerDes external interface, after testing flow
control on hardware: pause frames are only emitted with the SerDes
TXFC bit set, while the MAC force pause bits alone have no effect
on this port. Comment-only change, prompted by a question from
Maxime Chevallier.
v4: https://lore.kernel.org/netdev/20260702204648.276112-1-contact@c127.dev/
- Drop the chip model name from the driver's NOTE comment; which
interfaces a given chip exposes is described by its chip_info entry,
not the file header. Pointed out by Luiz Angelo Daros de Luca.
- Build the SerDes BMCR data-path-reset values from the standard
BMCR_ANENABLE | BMCR_ISOLATE bits instead of a bare magic number, so
the meaning is in the code rather than only in a comment. Pointed
out by Luiz Angelo Daros de Luca.
- Use a temporary for the DIGITAL_INTERFACE_SELECT value instead of
wrapping the expression across the regmap_update_bits() arguments.
Pointed out by Luiz Angelo Daros de Luca.
- Reject the untested SerDes tuning variant. The vendor driver keeps
two sets of SerDes tuning parameters and selects between them based
on the chip option register (0x13C1); the tables in this series are
the variant for a non-zero option, which is what the RTL8367S parts
seen so far report. The option is probed once at setup and the
SerDes interface modes are only advertised to phylink when the
tuning parameters match, so an unsupported variant fails at phylink
validation time instead of when configuring the link. Thanks to
Luiz Angelo Daros de Luca for pointing out the conditional.
- Express the external interface line rate bypass bit through a
parametric macro keyed on the port number (with port 5 as the base),
instead of an open-coded BIT(interface id) that only matched by
coincidence; other RTL8367 families index this register differently.
Suggested by Luiz Angelo Daros de Luca.
- Drop the arbitrary usleep_range() after each SerDes indirect access.
SerDes writes are now fire-and-forget and reads poll the self-clearing
BUSY bit with regmap_read_poll_timeout(), matching the vendor driver,
which never sleeps. On the MR80X the BUSY bit is never even observed
set: the access completes within the register transaction. Pointed out
by Luiz Angelo Daros de Luca; poll approach suggested by Mieczyslaw
Nalewaj.
- Drop the always-zero SerDes index argument from the SerDes indirect
access helpers, along with the INDACS command index field whose
width was questioned during review; this chip has a single SerDes
block reachable through this window, so the index served no purpose.
Raised by Luiz Angelo Daros de Luca.
- Stop hardcoding external interface 1 with an early -EOPNOTSUPP in
the SerDes configuration path. The SerDes interface modes are now
advertised in phylink_get_caps() from the chip_info
supported_interfaces, and mac_select_pcs() returns the PCS only for
those modes. Pointed out by Luiz Angelo Daros de Luca.
- Keep the new register definitions as raw hex masks, matching the
prevailing style of the file. A file-wide GENMASK/BIT conversion,
raised by Luiz Angelo Daros de Luca during review, is left for a
separate cleanup patch so this series stays focused on the feature.
- Convert the SerDes path to a phylink_pcs, as suggested by Maxime
Chevallier. The SGMII/HSGMII SerDes handling now lives in
pcs_config()/pcs_get_state()/pcs_link_up() selected via
mac_select_pcs(), instead of being driven from the MAC
mac_config()/mac_link_up()/mac_link_down() operations. This
separates the MAC and SerDes layers and makes future in-band
autonegotiation an additive change. No functional change intended
for the forced-link path; retested on the MR80X v2.20. In-band
autonegotiation remains unimplemented and is left for a follow-up,
once hardware is available to validate it.
- Implement pcs_inband_caps(), returning LINK_INBAND_DISABLE so that
phylink knows this PCS cannot do in-band autonegotiation and never
selects an in-band-enabled negotiation mode for it. pcs_config()
rejects PHYLINK_PCS_NEG_INBAND_ENABLED with -EOPNOTSUPP instead of
the previous warn-and-force.
- Program the SerDes pause controls in SDS_MISC from the resolved
pause modes when forcing the MAC external interface, as the vendor
driver does, instead of leaving whatever state the boot firmware
left there. Done in mac_link_up() because pcs_link_up() carries no
pause information.
- Set the PCS poll flag: the SerDes has no link interrupt wired up,
so phylink must poll pcs_get_state() when it tracks the link
through the PCS (in-band mode with autonegotiation disabled).
- Report link down from pcs_get_state() if reading back the forced
speed/duplex fails, rather than reporting link up with a stale
state.
- Reword the misleading "disable in-band aneg" comment.
v3: https://lore.kernel.org/netdev/20260613232136.24246-1-contact@c127.dev/
- Drop the DW8051 firmware loading entirely. Analysis of the vendor
firmware showed it only duplicates the link management phylink
already does; the one needed action (SerDes data-path reset via
the BMCR register) is now performed directly in the driver, with
the DW8051 kept disabled. This removes the dependency on the
rtl8367s-sgmii.bin firmware blob, which could not be redistributed
via linux-firmware (the GPL vendor source ships it as a byte array
without the corresponding microcode source). Thanks to Luiz Angelo
Daros de Luca for the firmware analysis.
v2: https://lore.kernel.org/netdev/0100019eb0b1822e-ffc5626c-1b9f-4c8a-8a1a-759a9e665f4f-000000@email.amazonses.com/
- No code changes; resend because the SMTP provider used for v1
corrupted the mails and patch 1/2 never reached the list.
v1: https://lore.kernel.org/netdev/aebccaad-eca3-4ea4-99dd-ae7edbc8981b@smtp-relay.sendinblue.com/
Johan Alvarado (2):
net: dsa: realtek: rtl8365mb: add SGMII support for RTL8367S
net: dsa: realtek: rtl8365mb: add HSGMII support for RTL8367S
drivers/net/dsa/realtek/rtl8365mb_main.c | 619 ++++++++++++++++++++++++++++++-
1 file changed, 614 insertions(+), 5 deletions(-)
base-commit: fe3e786ef4eb6e47d2901f568a27bd920477bbe9
--
2.55.0
^ permalink raw reply
* [PATCH net v2] vmxnet3: fix BUG_ON in vmxnet3_get_hdr_len() for Geneve packets
From: Harshaka Narayana @ 2026-07-09 20:16 UTC (permalink / raw)
To: harshaka.narayana
Cc: andrew+netdev, bcm-kernel-feedback-list, davem, edumazet,
guolin.yang, kuba, linux-kernel, netdev, pabeni, ronak.doshi,
sankararaman.jayaraman
vmxnet3_get_hdr_len() assumes gdesc->rcd.v4/v6/tcp always describe the
outer header, but for a Geneve-encapsulated packet the device can set
them based on the inner header instead, signalled by the
VMXNET3_RCD_HDR_INNER_SHIFT bit in the completion descriptor. Since the
function never skips the outer encapsulation, this mismatch triggers:
- BUG_ON(hdr.ipv4->protocol != IPPROTO_TCP), because the outer
protocol is UDP (Geneve), not TCP.
- BUG_ON(hdr.eth->h_proto != ...), when the tunnel's outer and inner
IP versions differ (e.g. outer IPv6/inner IPv4 or vice versa).
Check VMXNET3_RCD_HDR_INNER_SHIFT up front and bail out, since the
function cannot locate the inner header it would need to parse. Also
convert the remaining BUG_ON()s in this function to return 0
defensively.
Fixes: 45dac1d6ea04 ("vmxnet3: Changes for vmxnet3 adapter version 2 (fwd)")
Signed-off-by: Harshaka Narayana <harshaka.narayana@broadcom.com>
Signed-off-by: Ronak Doshi <ronak.doshi@broadcom.com>
Signed-off-by: Sankararaman Jayaraman <sankararaman.jayaraman@broadcom.com>
---
v2:
- Check VMXNET3_RCD_HDR_INNER_SHIFT up front to catch the Geneve
inner-header case directly, and convert the remaining
BUG_ON(hdr.eth->h_proto != ...) checks to return 0
- Reworded commit message to describe the root cause via
VMXNET3_RCD_HDR_INNER_SHIFT
v1: https://lore.kernel.org/netdev/20260707165248.1859188-1-harshaka.narayana@broadcom.com/
---
drivers/net/vmxnet3/vmxnet3_drv.c | 24 ++++++++++++++++++------
1 file changed, 18 insertions(+), 6 deletions(-)
diff --git a/drivers/net/vmxnet3/vmxnet3_drv.c b/drivers/net/vmxnet3/vmxnet3_drv.c
index 40522afc0532..dedf4082eb6c 100644
--- a/drivers/net/vmxnet3/vmxnet3_drv.c
+++ b/drivers/net/vmxnet3/vmxnet3_drv.c
@@ -1530,7 +1530,13 @@ vmxnet3_get_hdr_len(struct vmxnet3_adapter *adapter, struct sk_buff *skb,
struct ipv6hdr *ipv6;
struct tcphdr *tcp;
} hdr;
- BUG_ON(gdesc->rcd.tcp == 0);
+
+ /* v4/v6/tcp then describe the inner header, which we can't locate. */
+ if (le32_to_cpu(gdesc->dword[0]) & (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))
+ return 0;
+
+ if (gdesc->rcd.tcp == 0)
+ return 0;
maplen = skb_headlen(skb);
if (unlikely(sizeof(struct iphdr) + sizeof(struct tcphdr) > maplen))
@@ -1544,15 +1550,21 @@ vmxnet3_get_hdr_len(struct vmxnet3_adapter *adapter, struct sk_buff *skb,
hdr.eth = eth_hdr(skb);
if (gdesc->rcd.v4) {
- BUG_ON(hdr.eth->h_proto != htons(ETH_P_IP) &&
- hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IP));
+ if (hdr.eth->h_proto != htons(ETH_P_IP) &&
+ hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IP))
+ return 0;
+
hdr.ptr += hlen;
- BUG_ON(hdr.ipv4->protocol != IPPROTO_TCP);
+ if (hdr.ipv4->protocol != IPPROTO_TCP)
+ return 0;
+
hlen = hdr.ipv4->ihl << 2;
hdr.ptr += hdr.ipv4->ihl << 2;
} else if (gdesc->rcd.v6) {
- BUG_ON(hdr.eth->h_proto != htons(ETH_P_IPV6) &&
- hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IPV6));
+ if (hdr.eth->h_proto != htons(ETH_P_IPV6) &&
+ hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IPV6))
+ return 0;
+
hdr.ptr += hlen;
/* Use an estimated value, since we also need to handle
* TSO case.
--
2.52.0
^ permalink raw reply related
* Re: Ethtool : PRBS feature
From: Alexander Duyck @ 2026-07-09 20:13 UTC (permalink / raw)
To: Andrew Lunn
Cc: Lee Trager, Das, Shubham, Srinivasan, Vijay, Maxime Chevallier,
netdev@vger.kernel.org, mkubecek@suse.cz, D H, Siddaraju,
Chintalapalle, Balaji, Lindberg, Magnus,
niklas.damberg@ericsson.com, Wirandi, Jonas
In-Reply-To: <4ccc869f-a724-41fc-9bfc-da5332f54790@lunn.ch>
On Thu, Jul 9, 2026 at 7:37 AM Andrew Lunn <andrew@lunn.ch> wrote:
>
> On Wed, Jul 08, 2026 at 05:58:52PM -0700, Lee Trager wrote:
> > On 7/7/26 2:06 AM, Das, Shubham wrote:
> >
> > > Thanks Andrew, Lee for the feedback.
> > >
> > > Lee,
> > > I don't see prbs11.0, prbs11.1, prbs11.2, prbs11.3, prbs13.0, prbs13.1, prbs13.2, prbs13.3, prbs16 and prbs32 in IEEE 802.3 2022 standard.
> > > Is this specific to fnic based on base PRBS pattern or it is mentioned in some other standard ?
> > Those were from the fbnic spec, I'm not sure where they originate from. I
> > suppose we can drop those for now but its good to keep in mind more tests
> > may be added in the future.
>
> Can you find out what they actually do. It might be something as silly
> as, the .1 indicate it only runs on lane 1? So these are in fact
> standard prbs11 and being able to specify the lane via another
> attribute is sufficient?
Looks like they are different derivations of the polynomial for the
given pattern. For example PRBS11 is 1 + x^9 + x^11, but PRBS11_0 is 1
+ x^5 + x^6 + X^10 + x^11. That is one thing to think about. When we
say PRBS7 the assumption is we are all talking about 1 + x^6 + x^7 for
the polynomial. We may want to have that clearly recorded somewhere so
there isn't any confusion on which polynomials we are using for this
testing as there is always a risk that somebody is playing with
bleeding edge hardware and ends up defining a PRBS polynomial
differently than what is expected. I almost wonder if we couldn't
define the sequence as a bitmap instead of an enum with each bit
representing which fields are in the sequence.
The 13_X values seem to be used to generate QPRBS13. I would assume
the 11_X values are something similar. Odds are they aren't in the
standard. The comphy we use supports both PCIe and SATA use cases as
well as Ethernet so it has support for additional testing that may not
be referred to in the standard. In the case of QPRBS13 it looks like
it was rejected as a testing sequence for 400G so it may not be
something that we have to concern ourselves with, at least when it
comes to testing Ethernet PHYs.
^ permalink raw reply
* Re: [PATCH net-next v11 7/7] selftests: netconsole: validate target resume
From: Andre Carvalho @ 2026-07-09 20:13 UTC (permalink / raw)
To: Matthieu Baerts
Cc: Breno Leitao, netdev, linux-kernel, linux-kselftest, Andrew Lunn,
David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Shuah Khan, Simon Horman
In-Reply-To: <1fb14039-b56d-4757-add9-22b97d3c9f44@kernel.org>
On Thu, Jul 09, 2026 at 12:21:29PM +0200, Matthieu Baerts wrote:
> Good! I will send this patch.
>
> Just to avoid a deadlock, do you plan to send one to handle the MAC
> address that has not been modified (and eventually one to get more debug
> messages in case of errors)?
Ack! I'll send a patch to making the test handle the MAC address unchanged
scenario (taking your suggestion above) in the next couple of days. I'll follow
up on improving debuggability across this and other netconsole selftests as well
as a lot of them share the same issues.
> Cheers,
> Matt
Sorry for the noise here and thank you for testing the fix!
--
Andre Carvalho
^ permalink raw reply
* Re: [PATCH net-next v6 1/2] udp: fix encapsulation packet resubmit in multicast deliver
From: Anton Danilov @ 2026-07-09 20:11 UTC (permalink / raw)
To: Willem de Bruijn
Cc: netdev, Willem de Bruijn, David S . Miller, David Ahern,
Eric Dumazet, Kuniyuki Iwashima, Jakub Kicinski, Paolo Abeni,
Simon Horman, Shuah Khan, linux-kselftest
In-Reply-To: <willemdebruijn.kernel.2a3e532d3c28f@gmail.com>
On Wed, Jul 08, 2026 at 09:24:10PM -0400, Willem de Bruijn wrote:
> This helps the case of one encap_rcv socket in the multicast receiver
> group, so is a useful fix on its own.
>
> But is Sashiko correct that this would still leave the same issue for
> other sockets in the group? If so, something to address in this series
> or leave for later?
>
> Might be worthwhile to extend the test to capture that case too.
The clone loop is not reachable for encapsulation sockets, so there is
no remaining gap.
FOU/GUE tunnel sockets are created via udp_sock_create() /
setup_udp_tunnel_sock() and do not set SO_REUSEADDR or SO_REUSEPORT.
Without either, a UDP socket cannot share its port, so an encap socket
is always the only socket bound to its port. In
__udp[46]_lib_mcast_deliver() it is therefore always delivered as
'first', and the clone loop -- which handles the second and subsequent
sockets in the group -- never runs for it. The positive (resubmit)
return from udp_queue_rcv_skb() only happens for encap sockets; plain
UDP sockets return 0 or a negative value there. So the resubmit case
in the clone loop cannot occur.
For the same reason a selftest cannot exercise it: a second socket
cannot bind to the FOU port.
If useful, I can add a brief comment above the clone loop documenting
why no resubmit is needed there.
^ permalink raw reply
* [PATCH net v2] vmxnet3: fix BUG_ON in vmxnet3_get_hdr_len() for Geneve packets
From: Harshaka Narayana @ 2026-07-09 19:47 UTC (permalink / raw)
To: harshaka.narayana
Cc: andrew+netdev, bcm-kernel-feedback-list, davem, edumazet,
guolin.yang, kuba, linux-kernel, netdev, pabeni, ronak.doshi,
sankararaman.jayaraman
In-Reply-To: <20260707165248.1859188-1-harshaka.narayana@broadcom.com>
vmxnet3_get_hdr_len() assumes gdesc->rcd.v4/v6/tcp always describe the
outer header, but for a Geneve-encapsulated packet the device can set
them based on the inner header instead, signalled by the
VMXNET3_RCD_HDR_INNER_SHIFT bit in the completion descriptor. Since the
function never skips the outer encapsulation, this mismatch triggers:
- BUG_ON(hdr.ipv4->protocol != IPPROTO_TCP), because the outer
protocol is UDP (Geneve), not TCP.
- BUG_ON(hdr.eth->h_proto != ...), when the tunnel's outer and inner
IP versions differ (e.g. outer IPv6/inner IPv4 or vice versa).
Check VMXNET3_RCD_HDR_INNER_SHIFT up front and bail out, since the
function cannot locate the inner header it would need to parse. Also
convert the remaining BUG_ON()s in this function to return 0
defensively.
Fixes: 45dac1d6ea04 ("vmxnet3: Changes for vmxnet3 adapter version 2 (fwd)")
Signed-off-by: Harshaka Narayana <harshaka.narayana@broadcom.com>
Signed-off-by: Ronak Doshi <ronak.doshi@broadcom.com>
Signed-off-by: Sankararaman Jayaraman <sankararaman.jayaraman@broadcom.com>
---
v2:
- Check VMXNET3_RCD_HDR_INNER_SHIFT up front to catch the Geneve
inner-header case directly, and convert the remaining
BUG_ON(hdr.eth->h_proto != ...) checks to return 0
- Reworded commit message to describe the root cause via
VMXNET3_RCD_HDR_INNER_SHIFT
v1: https://lore.kernel.org/netdev/20260707165248.1859188-1-harshaka.narayana@broadcom.com/
---
drivers/net/vmxnet3/vmxnet3_drv.c | 24 ++++++++++++++++++------
1 file changed, 18 insertions(+), 6 deletions(-)
diff --git a/drivers/net/vmxnet3/vmxnet3_drv.c b/drivers/net/vmxnet3/vmxnet3_drv.c
index 40522afc0532..dedf4082eb6c 100644
--- a/drivers/net/vmxnet3/vmxnet3_drv.c
+++ b/drivers/net/vmxnet3/vmxnet3_drv.c
@@ -1530,7 +1530,13 @@ vmxnet3_get_hdr_len(struct vmxnet3_adapter *adapter, struct sk_buff *skb,
struct ipv6hdr *ipv6;
struct tcphdr *tcp;
} hdr;
- BUG_ON(gdesc->rcd.tcp == 0);
+
+ /* v4/v6/tcp then describe the inner header, which we can't locate. */
+ if (le32_to_cpu(gdesc->dword[0]) & (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))
+ return 0;
+
+ if (gdesc->rcd.tcp == 0)
+ return 0;
maplen = skb_headlen(skb);
if (unlikely(sizeof(struct iphdr) + sizeof(struct tcphdr) > maplen))
@@ -1544,15 +1550,21 @@ vmxnet3_get_hdr_len(struct vmxnet3_adapter *adapter, struct sk_buff *skb,
hdr.eth = eth_hdr(skb);
if (gdesc->rcd.v4) {
- BUG_ON(hdr.eth->h_proto != htons(ETH_P_IP) &&
- hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IP));
+ if (hdr.eth->h_proto != htons(ETH_P_IP) &&
+ hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IP))
+ return 0;
+
hdr.ptr += hlen;
- BUG_ON(hdr.ipv4->protocol != IPPROTO_TCP);
+ if (hdr.ipv4->protocol != IPPROTO_TCP)
+ return 0;
+
hlen = hdr.ipv4->ihl << 2;
hdr.ptr += hdr.ipv4->ihl << 2;
} else if (gdesc->rcd.v6) {
- BUG_ON(hdr.eth->h_proto != htons(ETH_P_IPV6) &&
- hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IPV6));
+ if (hdr.eth->h_proto != htons(ETH_P_IPV6) &&
+ hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IPV6))
+ return 0;
+
hdr.ptr += hlen;
/* Use an estimated value, since we also need to handle
* TSO case.
--
2.52.0
^ permalink raw reply related
* Re: Please backport bridge multicast exponential field encoding fix series to 6.1.y/6.6.y/6.12.y/6.18.y/7.0.y
From: Andrew Lunn @ 2026-07-09 20:01 UTC (permalink / raw)
To: Ujjal Roy
Cc: Greg KH, Linux Stable, David S . Miller, Eric Dumazet,
Jakub Kicinski, Paolo Abeni, Simon Horman, Nikolay Aleksandrov,
Ido Schimmel, David Ahern, Shuah Khan, Andy Roulin, Yong Wang,
Petr Machata, Ujjal Roy, bridge, netdev, linux-kernel,
linux-kselftest
In-Reply-To: <CAE2MWkn3L7V3x8i0F-soGLxsBBo_Umgs1pJ3FwCw1OW7=U55zg@mail.gmail.com>
> > > History: The multicast stack currently supports decoding of IGMPv3 and
> > > MLDv2 exponential timer field encodings, but lacks the corresponding
> > > encoding logic when generating multicast query packets.
RFC 3376 says:
4.1.1. Max Resp Code
The Max Resp Code field specifies the maximum time allowed before
sending a responding report. The actual time allowed, called the Max
Resp Time, is represented in units of 1/10 second and is derived from
the Max Resp Code as follows:
If Max Resp Code < 128, Max Resp Time = Max Resp Code
If Max Resp Code >= 128, Max Resp Code represents a floating-point
value as follows:
0 1 2 3 4 5 6 7
+-+-+-+-+-+-+-+-+
|1| exp | mant |
+-+-+-+-+-+-+-+-+
Max Resp Time = (mant | 0x10) << (exp + 3)
Small values of Max Resp Time allow IGMPv3 routers to tune the "leave
latency" (the time between the moment the last host leaves a group
and the moment the routing protocol is notified that there are no
more members). Larger values, especially in the exponential range,
allow tuning of the burstiness of IGMP traffic on a network.
Let me check i understand the issue. If the user configures a value >
127, linux continues to use the linear encoding, but a peer decodes it
as a floating value.
128 linear is 0 | 0x10) << (0 + 3) = 0x40 = 64. So the peer sends the
reports earlier than required?
255 linear is (0xf | 0x10) << (7 + 3) = 0x1F0000 = 2031616. So the
peer can send the reports much later than the 255 1/10 of a second
than userspace expected.
What is useful here is, 'maximum time allowed'. The RFC does not
appear to say how to pick a value between 0 and the maximum time
allowed. Which gives us some flexibility.
I think a much simpler fix for stable is to clamp the user space
request for setting the max response time to 127. That seems like a
one line patch.
Andrew
^ permalink raw reply
* Re: [PATCH net v2 2/2] vsock/test: add test for small packets under pressure
From: Bobby Eshleman @ 2026-07-09 19:48 UTC (permalink / raw)
To: Stefano Garzarella
Cc: netdev, Jason Wang, Xuan Zhuo, Eric Dumazet, Eugenio Pérez,
Simon Horman, Stefan Hajnoczi, David S. Miller, linux-kernel,
Michael S. Tsirkin, kvm, Paolo Abeni, virtualization,
Jakub Kicinski, Jason Wang
In-Reply-To: <20260708102904.50732-3-sgarzare@redhat.com>
On Wed, Jul 08, 2026 at 12:29:04PM +0200, Stefano Garzarella wrote:
> From: Stefano Garzarella <sgarzare@redhat.com>
>
> Add a test that sends 2 MB of data using randomly sized small packets
> (129-512 bytes) over a SOCK_STREAM connection. Packets above
> GOOD_COPY_LEN (128) bypass the in-place coalescing in recv_enqueue(),
> forcing each one into its own skb.
>
> Without receive queue collapsing, the per-skb overhead eventually
> exceeds buf_alloc and the connection is reset. The test verifies
> that all data arrives and that content integrity is preserved.
>
> Signed-off-by: Stefano Garzarella <sgarzare@redhat.com>
> ---
> tools/testing/vsock/vsock_test.c | 87 ++++++++++++++++++++++++++++++++
> 1 file changed, 87 insertions(+)
>
> diff --git a/tools/testing/vsock/vsock_test.c b/tools/testing/vsock/vsock_test.c
> index 76be0e4a7f0e..b4ff9f946565 100644
> --- a/tools/testing/vsock/vsock_test.c
> +++ b/tools/testing/vsock/vsock_test.c
> @@ -2347,6 +2347,88 @@ static void test_stream_tx_credit_bounds_server(const struct test_opts *opts)
> close(fd);
> }
>
> +/* Test that many small packets don't cause a connection reset under pressure
> + * and that data integrity is preserved. Packet sizes vary randomly between
> + * 129 and 512 bytes, above GOOD_COPY_LEN (128) to bypass in-place coalescing
> + * in recv_enqueue, forcing each one into its own skb. Without receive queue
> + * collapsing, the per-skb overhead eventually exceeds buf_alloc and the
> + * connection is reset.
> + */
> +#define COLLAPSE_PKT_MIN 129
> +#define COLLAPSE_PKT_MAX 512
> +#define COLLAPSE_TOTAL (2 * 1024 * 1024)
> +
> +static void test_stream_collapse_client(const struct test_opts *opts)
> +{
> + unsigned char *data;
> + unsigned long hash;
> + size_t offset = 0;
> + int i, fd;
> +
> + data = malloc(COLLAPSE_TOTAL);
> + if (!data) {
> + perror("malloc");
> + exit(EXIT_FAILURE);
> + }
> +
> + for (i = 0; i < COLLAPSE_TOTAL; i++)
> + data[i] = rand() & 0xff;
> +
> + fd = vsock_stream_connect(opts->peer_cid, opts->peer_port);
> + if (fd < 0) {
> + perror("connect");
> + exit(EXIT_FAILURE);
> + }
> +
> + while (offset < COLLAPSE_TOTAL) {
> + size_t pkt_size = COLLAPSE_PKT_MIN +
> + rand() % (COLLAPSE_PKT_MAX - COLLAPSE_PKT_MIN + 1);
> +
> + pkt_size = min(pkt_size, COLLAPSE_TOTAL - offset);
> +
> + send_buf(fd, data + offset, pkt_size, 0, pkt_size);
> + offset += pkt_size;
> + }
> +
> + hash = hash_djb2(data, COLLAPSE_TOTAL);
> + control_writeulong(hash);
> +
> + free(data);
> + close(fd);
> +}
> +
> +static void test_stream_collapse_server(const struct test_opts *opts)
> +{
> + unsigned long hash, remote_hash;
> + unsigned char *data;
> + int fd;
> +
> + data = malloc(COLLAPSE_TOTAL);
> + if (!data) {
> + perror("malloc");
> + exit(EXIT_FAILURE);
> + }
> +
> + fd = vsock_stream_accept(VMADDR_CID_ANY, opts->peer_port, NULL);
> + if (fd < 0) {
> + perror("accept");
> + exit(EXIT_FAILURE);
> + }
> +
> + recv_buf(fd, data, COLLAPSE_TOTAL, 0, COLLAPSE_TOTAL);
> +
> + hash = hash_djb2(data, COLLAPSE_TOTAL);
> + remote_hash = control_readulong();
> + if (hash != remote_hash) {
> + fprintf(stderr, "hash mismatch: local %lu remote %lu\n",
> + hash, remote_hash);
> + exit(EXIT_FAILURE);
> + }
> +
> + free(data);
> + close(fd);
> +}
> +
> static struct test_case test_cases[] = {
> {
> .name = "SOCK_STREAM connection reset",
> @@ -2546,6 +2628,11 @@ static struct test_case test_cases[] = {
> .run_client = test_stream_msg_peek_client,
> .run_server = test_stream_peek_after_recv_server,
> },
> + {
> + .name = "SOCK_STREAM small packets backpressure",
> + .run_client = test_stream_collapse_client,
> + .run_server = test_stream_collapse_server,
> + },
> {},
> };
>
> --
> 2.55.0
>
Reviewed-by: Bobby Eshleman <bobbyeshleman@meta.com>
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox