* [PATCH v15 1/9] net: tls: reject TLS 1.3 offload in chcr_ktls and nfp drivers
2026-07-09 20:53 [PATCH net-next v15 0/9] tls: Add TLS 1.3 hardware offload support Rishikesh Jethwani
@ 2026-07-09 20:53 ` Rishikesh Jethwani
2026-07-09 20:53 ` [PATCH v15 2/9] net/mlx5e: add TLS 1.3 hardware offload support Rishikesh Jethwani
` (7 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Rishikesh Jethwani @ 2026-07-09 20:53 UTC (permalink / raw)
To: netdev
Cc: saeedm, tariqt, mbloch, borisp, john.fastabend, kuba, sd, davem,
pabeni, edumazet, leon, andrew.gospodarek, Rishikesh Jethwani
These drivers only support TLS 1.2. Return early when TLS 1.3
is requested to prevent unsupported hardware offload attempts.
Signed-off-by: Rishikesh Jethwani <rjethwani@purestorage.com>
---
drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c | 3 +++
drivers/net/ethernet/netronome/nfp/crypto/tls.c | 3 +++
2 files changed, 6 insertions(+)
diff --git a/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c
index f5acd4be1e69..29e108ce6764 100644
--- a/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c
+++ b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c
@@ -431,6 +431,9 @@ static int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk,
atomic64_inc(&port_stats->ktls_tx_connection_open);
u_ctx = adap->uld[CXGB4_ULD_KTLS].handle;
+ if (crypto_info->version != TLS_1_2_VERSION)
+ goto out;
+
if (direction == TLS_OFFLOAD_CTX_DIR_RX) {
pr_err("not expecting for RX direction\n");
goto out;
diff --git a/drivers/net/ethernet/netronome/nfp/crypto/tls.c b/drivers/net/ethernet/netronome/nfp/crypto/tls.c
index 9983d7aa2b9c..13864c6a55dc 100644
--- a/drivers/net/ethernet/netronome/nfp/crypto/tls.c
+++ b/drivers/net/ethernet/netronome/nfp/crypto/tls.c
@@ -287,6 +287,9 @@ nfp_net_tls_add(struct net_device *netdev, struct sock *sk,
BUILD_BUG_ON(offsetof(struct nfp_net_tls_offload_ctx, rx_end) >
TLS_DRIVER_STATE_SIZE_RX);
+ if (crypto_info->version != TLS_1_2_VERSION)
+ return -EOPNOTSUPP;
+
if (!nfp_net_cipher_supported(nn, crypto_info->cipher_type, direction))
return -EOPNOTSUPP;
--
2.25.1
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH v15 2/9] net/mlx5e: add TLS 1.3 hardware offload support
2026-07-09 20:53 [PATCH net-next v15 0/9] tls: Add TLS 1.3 hardware offload support Rishikesh Jethwani
2026-07-09 20:53 ` [PATCH v15 1/9] net: tls: reject TLS 1.3 offload in chcr_ktls and nfp drivers Rishikesh Jethwani
@ 2026-07-09 20:53 ` Rishikesh Jethwani
2026-07-09 20:53 ` [PATCH v15 3/9] tls: " Rishikesh Jethwani
` (6 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Rishikesh Jethwani @ 2026-07-09 20:53 UTC (permalink / raw)
To: netdev
Cc: saeedm, tariqt, mbloch, borisp, john.fastabend, kuba, sd, davem,
pabeni, edumazet, leon, andrew.gospodarek, Rishikesh Jethwani
Enable TLS 1.3 TX/RX hardware offload on ConnectX-6 Dx and newer
crypto-enabled adapters.
Key changes:
- Add TLS 1.3 capability checking and version validation
- Use MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_3 (0x3) for crypto context
- Handle TLS 1.3 IV format: full 12-byte IV copied to gcm_iv +
implicit_iv (vs TLS 1.2's 4-byte salt only)
Tested with TLS 1.3 AES-GCM-128 and AES-GCM-256 cipher suites.
Signed-off-by: Rishikesh Jethwani <rjethwani@purestorage.com>
Tested-by: Tariq Toukan <tariqt@nvidia.com>
Reviewed-by: Tariq Toukan <tariqt@nvidia.com>
---
.../ethernet/mellanox/mlx5/core/en_accel/ktls.h | 8 +++++++-
.../mellanox/mlx5/core/en_accel/ktls_txrx.c | 14 +++++++++++---
2 files changed, 18 insertions(+), 4 deletions(-)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h
index 07a04a142a2e..0469ca6a0762 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h
@@ -30,7 +30,9 @@ static inline bool mlx5e_is_ktls_device(struct mlx5_core_dev *mdev)
return false;
return (MLX5_CAP_TLS(mdev, tls_1_2_aes_gcm_128) ||
- MLX5_CAP_TLS(mdev, tls_1_2_aes_gcm_256));
+ MLX5_CAP_TLS(mdev, tls_1_2_aes_gcm_256) ||
+ MLX5_CAP_TLS(mdev, tls_1_3_aes_gcm_128) ||
+ MLX5_CAP_TLS(mdev, tls_1_3_aes_gcm_256));
}
static inline bool mlx5e_ktls_type_check(struct mlx5_core_dev *mdev,
@@ -40,10 +42,14 @@ static inline bool mlx5e_ktls_type_check(struct mlx5_core_dev *mdev,
case TLS_CIPHER_AES_GCM_128:
if (crypto_info->version == TLS_1_2_VERSION)
return MLX5_CAP_TLS(mdev, tls_1_2_aes_gcm_128);
+ else if (crypto_info->version == TLS_1_3_VERSION)
+ return MLX5_CAP_TLS(mdev, tls_1_3_aes_gcm_128);
break;
case TLS_CIPHER_AES_GCM_256:
if (crypto_info->version == TLS_1_2_VERSION)
return MLX5_CAP_TLS(mdev, tls_1_2_aes_gcm_256);
+ else if (crypto_info->version == TLS_1_3_VERSION)
+ return MLX5_CAP_TLS(mdev, tls_1_3_aes_gcm_256);
break;
}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c
index 570a912dd6fa..f3f1be1d4034 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c
@@ -6,6 +6,7 @@
enum {
MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_2 = 0x2,
+ MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_3 = 0x3,
};
enum {
@@ -15,8 +16,10 @@ enum {
#define EXTRACT_INFO_FIELDS do { \
salt = info->salt; \
rec_seq = info->rec_seq; \
+ iv = info->iv; \
salt_sz = sizeof(info->salt); \
rec_seq_sz = sizeof(info->rec_seq); \
+ iv_sz = sizeof(info->iv); \
} while (0)
static void
@@ -24,9 +27,9 @@ fill_static_params(struct mlx5_wqe_tls_static_params_seg *params,
union mlx5e_crypto_info *crypto_info,
u32 key_id, u32 resync_tcp_sn)
{
+ u16 salt_sz, rec_seq_sz, iv_sz;
+ char *salt, *rec_seq, *iv;
char *initial_rn, *gcm_iv;
- u16 salt_sz, rec_seq_sz;
- char *salt, *rec_seq;
u8 tls_version;
u8 *ctx;
@@ -59,7 +62,12 @@ fill_static_params(struct mlx5_wqe_tls_static_params_seg *params,
memcpy(gcm_iv, salt, salt_sz);
memcpy(initial_rn, rec_seq, rec_seq_sz);
- tls_version = MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_2;
+ if (crypto_info->crypto_info.version == TLS_1_3_VERSION) {
+ memcpy(gcm_iv + salt_sz, iv, iv_sz);
+ tls_version = MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_3;
+ } else {
+ tls_version = MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_2;
+ }
MLX5_SET(tls_static_params, ctx, tls_version, tls_version);
MLX5_SET(tls_static_params, ctx, const_1, 1);
--
2.25.1
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH v15 3/9] tls: add TLS 1.3 hardware offload support
2026-07-09 20:53 [PATCH net-next v15 0/9] tls: Add TLS 1.3 hardware offload support Rishikesh Jethwani
2026-07-09 20:53 ` [PATCH v15 1/9] net: tls: reject TLS 1.3 offload in chcr_ktls and nfp drivers Rishikesh Jethwani
2026-07-09 20:53 ` [PATCH v15 2/9] net/mlx5e: add TLS 1.3 hardware offload support Rishikesh Jethwani
@ 2026-07-09 20:53 ` Rishikesh Jethwani
2026-07-09 20:53 ` [PATCH v15 4/9] tls: split tls_set_sw_offload into init and finalize stages Rishikesh Jethwani
` (5 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Rishikesh Jethwani @ 2026-07-09 20:53 UTC (permalink / raw)
To: netdev
Cc: saeedm, tariqt, mbloch, borisp, john.fastabend, kuba, sd, davem,
pabeni, edumazet, leon, andrew.gospodarek, Rishikesh Jethwani
Add TLS 1.3 support to the kernel TLS hardware offload infrastructure,
enabling hardware acceleration for TLS 1.3 connections on capable NICs.
Tested on Mellanox ConnectX-6 Dx (Crypto Enabled) with TLS 1.3 AES-GCM-128
and AES-GCM-256 cipher suites.
Signed-off-by: Rishikesh Jethwani <rjethwani@purestorage.com>
---
net/tls/tls_device.c | 65 ++++++++++++++++-----------
net/tls/tls_device_fallback.c | 58 +++++++++++++-----------
net/tls/tls_main.c | 85 ++++++++++++++++++++---------------
3 files changed, 121 insertions(+), 87 deletions(-)
diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c
index 741aef09bfd3..a087cf3f544f 100644
--- a/net/tls/tls_device.c
+++ b/net/tls/tls_device.c
@@ -317,25 +317,34 @@ static void tls_device_record_close(struct sock *sk,
unsigned char record_type)
{
struct tls_prot_info *prot = &ctx->prot_info;
- struct page_frag dummy_tag_frag;
-
- /* append tag
- * device will fill in the tag, we just need to append a placeholder
- * use socket memory to improve coalescing (re-using a single buffer
- * increases frag count)
- * if we can't allocate memory now use the dummy page
+ int tail = prot->tag_size + prot->tail_size;
+
+ /* Append tail: tag for TLS 1.2, content_type + tag for TLS 1.3.
+ * Device fills in the tag, we just need to append a placeholder.
+ * Use socket memory to improve coalescing (re-using a single buffer
+ * increases frag count); if allocation fails use dummy_page
+ * (offset = record_type gives correct content_type byte via
+ * identity mapping)
*/
- if (unlikely(pfrag->size - pfrag->offset < prot->tag_size) &&
- !skb_page_frag_refill(prot->tag_size, pfrag, sk->sk_allocation)) {
- dummy_tag_frag.page = dummy_page;
- dummy_tag_frag.offset = 0;
- pfrag = &dummy_tag_frag;
+ if (unlikely(pfrag->size - pfrag->offset < tail) &&
+ !skb_page_frag_refill(tail, pfrag, sk->sk_allocation)) {
+ struct page_frag dummy_pfrag = {
+ .page = dummy_page,
+ .offset = record_type,
+ };
+ tls_append_frag(record, &dummy_pfrag, tail);
+ } else {
+ if (prot->tail_size) {
+ char *content_type_addr = page_address(pfrag->page) +
+ pfrag->offset;
+ *content_type_addr = record_type;
+ }
+ tls_append_frag(record, pfrag, tail);
}
- tls_append_frag(record, pfrag, prot->tag_size);
/* fill prepend */
tls_fill_prepend(ctx, skb_frag_address(&record->frags[0]),
- record->len - prot->overhead_size,
+ record->len - prot->overhead_size + prot->tail_size,
record_type);
}
@@ -883,6 +892,7 @@ static int
tls_device_reencrypt(struct sock *sk, struct tls_context *tls_ctx)
{
struct tls_sw_context_rx *sw_ctx = tls_sw_ctx_rx(tls_ctx);
+ struct tls_prot_info *prot = &tls_ctx->prot_info;
const struct tls_cipher_desc *cipher_desc;
int err, offset, copy, data_len, pos;
struct sk_buff *skb, *skb_iter;
@@ -894,7 +904,7 @@ tls_device_reencrypt(struct sock *sk, struct tls_context *tls_ctx)
DEBUG_NET_WARN_ON_ONCE(!cipher_desc || !cipher_desc->offloadable);
rxm = strp_msg(tls_strp_msg(sw_ctx));
- orig_buf = kmalloc(rxm->full_len + TLS_HEADER_SIZE + cipher_desc->iv,
+ orig_buf = kmalloc(rxm->full_len + prot->prepend_size,
sk->sk_allocation);
if (!orig_buf)
return -ENOMEM;
@@ -909,9 +919,8 @@ tls_device_reencrypt(struct sock *sk, struct tls_context *tls_ctx)
offset = rxm->offset;
sg_init_table(sg, 1);
- sg_set_buf(&sg[0], buf,
- rxm->full_len + TLS_HEADER_SIZE + cipher_desc->iv);
- err = skb_copy_bits(skb, offset, buf, TLS_HEADER_SIZE + cipher_desc->iv);
+ sg_set_buf(&sg[0], buf, rxm->full_len + prot->prepend_size);
+ err = skb_copy_bits(skb, offset, buf, prot->prepend_size);
if (err)
goto free_buf;
@@ -1089,11 +1098,6 @@ int tls_set_device_offload(struct sock *sk)
}
crypto_info = &ctx->crypto_send.info;
- if (crypto_info->version != TLS_1_2_VERSION) {
- rc = -EOPNOTSUPP;
- goto release_netdev;
- }
-
cipher_desc = get_cipher_desc(crypto_info->cipher_type);
if (!cipher_desc || !cipher_desc->offloadable) {
rc = -EINVAL;
@@ -1196,9 +1200,6 @@ int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx)
struct net_device *netdev;
int rc = 0;
- if (ctx->crypto_recv.info.version != TLS_1_2_VERSION)
- return -EOPNOTSUPP;
-
netdev = get_netdev_for_sock(sk);
if (!netdev) {
pr_err_ratelimited("%s: netdev not found\n", __func__);
@@ -1408,12 +1409,22 @@ static struct notifier_block tls_dev_notifier = {
int __init tls_device_init(void)
{
- int err;
+ unsigned char *page_addr;
+ int err, i;
dummy_page = alloc_page(GFP_KERNEL);
if (!dummy_page)
return -ENOMEM;
+ /* Pre-populate dummy_page with identity mapping for all byte values.
+ * This is used as fallback for TLS 1.3 content type when memory
+ * allocation fails. By populating all 256 values, we avoid needing
+ * to validate record_type at runtime.
+ */
+ page_addr = page_address(dummy_page);
+ for (i = 0; i < 256; i++)
+ page_addr[i] = (unsigned char)i;
+
destruct_wq = alloc_workqueue("ktls_device_destruct", WQ_PERCPU, 0);
if (!destruct_wq) {
err = -ENOMEM;
diff --git a/net/tls/tls_device_fallback.c b/net/tls/tls_device_fallback.c
index 3b7d0ab2bcf1..1110f7ac6bcb 100644
--- a/net/tls/tls_device_fallback.c
+++ b/net/tls/tls_device_fallback.c
@@ -37,14 +37,15 @@
#include "tls.h"
-static int tls_enc_record(struct aead_request *aead_req,
+static int tls_enc_record(struct tls_context *tls_ctx,
+ struct aead_request *aead_req,
struct crypto_aead *aead, char *aad,
char *iv, __be64 rcd_sn,
struct scatter_walk *in,
- struct scatter_walk *out, int *in_len,
- struct tls_prot_info *prot)
+ struct scatter_walk *out, int *in_len)
{
unsigned char buf[TLS_HEADER_SIZE + TLS_MAX_IV_SIZE];
+ struct tls_prot_info *prot = &tls_ctx->prot_info;
const struct tls_cipher_desc *cipher_desc;
struct scatterlist sg_in[3];
struct scatterlist sg_out[3];
@@ -55,7 +56,7 @@ static int tls_enc_record(struct aead_request *aead_req,
cipher_desc = get_cipher_desc(prot->cipher_type);
DEBUG_NET_WARN_ON_ONCE(!cipher_desc || !cipher_desc->offloadable);
- buf_size = TLS_HEADER_SIZE + cipher_desc->iv;
+ buf_size = prot->prepend_size;
len = min_t(int, *in_len, buf_size);
memcpy_from_scatterwalk(buf, in, len);
@@ -66,16 +67,27 @@ static int tls_enc_record(struct aead_request *aead_req,
return 0;
len = buf[4] | (buf[3] << 8);
- len -= cipher_desc->iv;
+ if (prot->version != TLS_1_3_VERSION)
+ len -= cipher_desc->iv;
tls_make_aad(aad, len - cipher_desc->tag, (char *)&rcd_sn, buf[0], prot);
- memcpy(iv + cipher_desc->salt, buf + TLS_HEADER_SIZE, cipher_desc->iv);
+ if (prot->version == TLS_1_3_VERSION) {
+ void *iv_src = crypto_info_iv(&tls_ctx->crypto_send.info,
+ cipher_desc);
+
+ memcpy(iv + cipher_desc->salt, iv_src, cipher_desc->iv);
+ } else {
+ memcpy(iv + cipher_desc->salt, buf + TLS_HEADER_SIZE,
+ cipher_desc->iv);
+ }
+
+ tls_xor_iv_with_seq(prot, iv, (char *)&rcd_sn);
sg_init_table(sg_in, ARRAY_SIZE(sg_in));
sg_init_table(sg_out, ARRAY_SIZE(sg_out));
- sg_set_buf(sg_in, aad, TLS_AAD_SPACE_SIZE);
- sg_set_buf(sg_out, aad, TLS_AAD_SPACE_SIZE);
+ sg_set_buf(sg_in, aad, prot->aad_size);
+ sg_set_buf(sg_out, aad, prot->aad_size);
scatterwalk_get_sglist(in, sg_in + 1);
scatterwalk_get_sglist(out, sg_out + 1);
@@ -108,13 +120,6 @@ static int tls_enc_record(struct aead_request *aead_req,
return rc;
}
-static void tls_init_aead_request(struct aead_request *aead_req,
- struct crypto_aead *aead)
-{
- aead_request_set_tfm(aead_req, aead);
- aead_request_set_ad(aead_req, TLS_AAD_SPACE_SIZE);
-}
-
static struct aead_request *tls_alloc_aead_request(struct crypto_aead *aead,
gfp_t flags)
{
@@ -124,14 +129,15 @@ static struct aead_request *tls_alloc_aead_request(struct crypto_aead *aead,
aead_req = kzalloc(req_size, flags);
if (aead_req)
- tls_init_aead_request(aead_req, aead);
+ aead_request_set_tfm(aead_req, aead);
return aead_req;
}
-static int tls_enc_records(struct aead_request *aead_req,
+static int tls_enc_records(struct tls_context *tls_ctx,
+ struct aead_request *aead_req,
struct crypto_aead *aead, struct scatterlist *sg_in,
struct scatterlist *sg_out, char *aad, char *iv,
- u64 rcd_sn, int len, struct tls_prot_info *prot)
+ u64 rcd_sn, int len)
{
struct scatter_walk out, in;
int rc;
@@ -140,8 +146,8 @@ static int tls_enc_records(struct aead_request *aead_req,
scatterwalk_start(&out, sg_out);
do {
- rc = tls_enc_record(aead_req, aead, aad, iv,
- cpu_to_be64(rcd_sn), &in, &out, &len, prot);
+ rc = tls_enc_record(tls_ctx, aead_req, aead, aad, iv,
+ cpu_to_be64(rcd_sn), &in, &out, &len);
rcd_sn++;
} while (rc == 0 && len);
@@ -314,7 +320,10 @@ static struct sk_buff *tls_enc_skb(struct tls_context *tls_ctx,
cipher_desc = get_cipher_desc(tls_ctx->crypto_send.info.cipher_type);
DEBUG_NET_WARN_ON_ONCE(!cipher_desc || !cipher_desc->offloadable);
- buf_len = cipher_desc->salt + cipher_desc->iv + TLS_AAD_SPACE_SIZE +
+ aead_request_set_ad(aead_req, tls_ctx->prot_info.aad_size);
+
+ buf_len = cipher_desc->salt + cipher_desc->iv +
+ tls_ctx->prot_info.aad_size +
sync_size + cipher_desc->tag;
buf = kmalloc(buf_len, GFP_ATOMIC);
if (!buf)
@@ -324,7 +333,7 @@ static struct sk_buff *tls_enc_skb(struct tls_context *tls_ctx,
salt = crypto_info_salt(&tls_ctx->crypto_send.info, cipher_desc);
memcpy(iv, salt, cipher_desc->salt);
aad = buf + cipher_desc->salt + cipher_desc->iv;
- dummy_buf = aad + TLS_AAD_SPACE_SIZE;
+ dummy_buf = aad + tls_ctx->prot_info.aad_size;
nskb = alloc_skb(skb_headroom(skb) + skb->len, GFP_ATOMIC);
if (!nskb)
@@ -335,9 +344,8 @@ static struct sk_buff *tls_enc_skb(struct tls_context *tls_ctx,
fill_sg_out(sg_out, buf, tls_ctx, nskb, tcp_payload_offset,
payload_len, sync_size, dummy_buf);
- if (tls_enc_records(aead_req, ctx->aead_send, sg_in, sg_out, aad, iv,
- rcd_sn, sync_size + payload_len,
- &tls_ctx->prot_info) < 0)
+ if (tls_enc_records(tls_ctx, aead_req, ctx->aead_send, sg_in, sg_out,
+ aad, iv, rcd_sn, sync_size + payload_len) < 0)
goto free_nskb;
complete_skb(nskb, skb, tcp_payload_offset);
diff --git a/net/tls/tls_main.c b/net/tls/tls_main.c
index 8c588cdab733..c4c15c0e731a 100644
--- a/net/tls/tls_main.c
+++ b/net/tls/tls_main.c
@@ -714,49 +714,64 @@ static int do_tls_setsockopt_conf(struct sock *sk, sockptr_t optval,
}
if (tx) {
- rc = tls_set_device_offload(sk);
- conf = TLS_HW;
- if (!rc) {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXDEVICE);
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXDEVICE);
- } else {
- rc = tls_set_sw_offload(sk, 1,
- update ? crypto_info : NULL);
- if (rc)
- goto err_crypto_info;
-
- if (update) {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYOK);
- } else {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXSW);
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW);
+ if (update && ctx->tx_conf == TLS_HW) {
+ rc = -EOPNOTSUPP;
+ goto err_crypto_info;
+ }
+
+ if (!update) {
+ rc = tls_set_device_offload(sk);
+ conf = TLS_HW;
+ if (!rc) {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXDEVICE);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXDEVICE);
+ goto out;
}
- conf = TLS_SW;
}
- } else {
- rc = tls_set_device_offload_rx(sk, ctx);
- conf = TLS_HW;
- if (!rc) {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXDEVICE);
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXDEVICE);
+
+ rc = tls_set_sw_offload(sk, 1, update ? crypto_info : NULL);
+ if (rc)
+ goto err_crypto_info;
+
+ if (update) {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYOK);
} else {
- rc = tls_set_sw_offload(sk, 0,
- update ? crypto_info : NULL);
- if (rc)
- goto err_crypto_info;
-
- if (update) {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYOK);
- } else {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXSW);
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXSW);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXSW);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW);
+ }
+ conf = TLS_SW;
+ } else {
+ if (update && ctx->rx_conf == TLS_HW) {
+ rc = -EOPNOTSUPP;
+ goto err_crypto_info;
+ }
+
+ if (!update) {
+ rc = tls_set_device_offload_rx(sk, ctx);
+ conf = TLS_HW;
+ if (!rc) {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXDEVICE);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXDEVICE);
+ tls_sw_strparser_arm(sk, ctx);
+ goto out;
}
- conf = TLS_SW;
}
- if (!update)
+
+ rc = tls_set_sw_offload(sk, 0, update ? crypto_info : NULL);
+ if (rc)
+ goto err_crypto_info;
+
+ if (update) {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYOK);
+ } else {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXSW);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXSW);
tls_sw_strparser_arm(sk, ctx);
+ }
+ conf = TLS_SW;
}
+out:
if (tx)
ctx->tx_conf = conf;
else
--
2.25.1
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH v15 4/9] tls: split tls_set_sw_offload into init and finalize stages
2026-07-09 20:53 [PATCH net-next v15 0/9] tls: Add TLS 1.3 hardware offload support Rishikesh Jethwani
` (2 preceding siblings ...)
2026-07-09 20:53 ` [PATCH v15 3/9] tls: " Rishikesh Jethwani
@ 2026-07-09 20:53 ` Rishikesh Jethwani
2026-07-09 20:53 ` [PATCH v15 5/9] tls: prep helpers and refactors for HW offload KeyUpdate Rishikesh Jethwani
` (4 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Rishikesh Jethwani @ 2026-07-09 20:53 UTC (permalink / raw)
To: netdev
Cc: saeedm, tariqt, mbloch, borisp, john.fastabend, kuba, sd, davem,
pabeni, edumazet, leon, andrew.gospodarek, Rishikesh Jethwani
Separate cipher context initialization from key material finalization
to support staged setup for hardware offload fallback paths.
Signed-off-by: Rishikesh Jethwani <rjethwani@purestorage.com>
---
net/tls/tls.h | 4 +++
net/tls/tls_device.c | 3 +-
net/tls/tls_sw.c | 77 +++++++++++++++++++++++++++++++-------------
3 files changed, 61 insertions(+), 23 deletions(-)
diff --git a/net/tls/tls.h b/net/tls/tls.h
index 60a37bdaaa25..5a6ee1ea00f8 100644
--- a/net/tls/tls.h
+++ b/net/tls/tls.h
@@ -147,6 +147,10 @@ void tls_strp_abort_strp(struct tls_strparser *strp, int err);
int init_prot_info(struct tls_prot_info *prot,
const struct tls_crypto_info *crypto_info,
const struct tls_cipher_desc *cipher_desc);
+int tls_sw_ctx_init(struct sock *sk, int tx,
+ struct tls_crypto_info *new_crypto_info);
+void tls_sw_ctx_finalize(struct sock *sk, int tx,
+ struct tls_crypto_info *new_crypto_info);
int tls_set_sw_offload(struct sock *sk, int tx,
struct tls_crypto_info *new_crypto_info);
void tls_update_rx_zc_capable(struct tls_context *tls_ctx);
diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c
index a087cf3f544f..f22f8a550c82 100644
--- a/net/tls/tls_device.c
+++ b/net/tls/tls_device.c
@@ -1233,7 +1233,7 @@ int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx)
context->resync_nh_reset = 1;
ctx->priv_ctx_rx = context;
- rc = tls_set_sw_offload(sk, 0, NULL);
+ rc = tls_sw_ctx_init(sk, 0, NULL);
if (rc)
goto release_ctx;
@@ -1247,6 +1247,7 @@ int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx)
goto free_sw_resources;
tls_device_attach(ctx, sk, netdev);
+ tls_sw_ctx_finalize(sk, 0, NULL);
up_read(&device_offload_lock);
dev_put(netdev);
diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c
index 9324e4ed20a3..383eab5150ce 100644
--- a/net/tls/tls_sw.c
+++ b/net/tls/tls_sw.c
@@ -2490,20 +2490,19 @@ static void tls_finish_key_update(struct sock *sk, struct tls_context *tls_ctx)
ctx->saved_data_ready(sk);
}
-int tls_set_sw_offload(struct sock *sk, int tx,
- struct tls_crypto_info *new_crypto_info)
+int tls_sw_ctx_init(struct sock *sk, int tx,
+ struct tls_crypto_info *new_crypto_info)
{
struct tls_crypto_info *crypto_info, *src_crypto_info;
struct tls_sw_context_tx *sw_ctx_tx = NULL;
struct tls_sw_context_rx *sw_ctx_rx = NULL;
const struct tls_cipher_desc *cipher_desc;
- char *iv, *rec_seq, *key, *salt;
- struct cipher_context *cctx;
struct tls_prot_info *prot;
struct crypto_aead **aead;
struct tls_context *ctx;
struct crypto_tfm *tfm;
int rc = 0;
+ char *key;
ctx = tls_get_ctx(sk);
prot = &ctx->prot_info;
@@ -2524,12 +2523,10 @@ int tls_set_sw_offload(struct sock *sk, int tx,
if (tx) {
sw_ctx_tx = ctx->priv_ctx_tx;
crypto_info = &ctx->crypto_send.info;
- cctx = &ctx->tx;
aead = &sw_ctx_tx->aead_send;
} else {
sw_ctx_rx = ctx->priv_ctx_rx;
crypto_info = &ctx->crypto_recv.info;
- cctx = &ctx->rx;
aead = &sw_ctx_rx->aead_recv;
}
@@ -2545,10 +2542,7 @@ int tls_set_sw_offload(struct sock *sk, int tx,
if (rc)
goto free_priv;
- iv = crypto_info_iv(src_crypto_info, cipher_desc);
key = crypto_info_key(src_crypto_info, cipher_desc);
- salt = crypto_info_salt(src_crypto_info, cipher_desc);
- rec_seq = crypto_info_rec_seq(src_crypto_info, cipher_desc);
if (!*aead) {
*aead = crypto_alloc_aead(cipher_desc->cipher_name, 0, 0);
@@ -2592,19 +2586,6 @@ int tls_set_sw_offload(struct sock *sk, int tx,
goto free_aead;
}
- memcpy(cctx->iv, salt, cipher_desc->salt);
- memcpy(cctx->iv + cipher_desc->salt, iv, cipher_desc->iv);
- memcpy(cctx->rec_seq, rec_seq, cipher_desc->rec_seq);
-
- if (new_crypto_info) {
- unsafe_memcpy(crypto_info, new_crypto_info,
- cipher_desc->crypto_info,
- /* size was checked in do_tls_setsockopt_conf */);
- memzero_explicit(new_crypto_info, cipher_desc->crypto_info);
- if (!tx)
- tls_finish_key_update(sk, ctx);
- }
-
goto out;
free_aead:
@@ -2623,3 +2604,55 @@ int tls_set_sw_offload(struct sock *sk, int tx,
out:
return rc;
}
+
+void tls_sw_ctx_finalize(struct sock *sk, int tx,
+ struct tls_crypto_info *new_crypto_info)
+{
+ struct tls_crypto_info *crypto_info, *src_crypto_info;
+ const struct tls_cipher_desc *cipher_desc;
+ struct tls_context *ctx = tls_get_ctx(sk);
+ struct cipher_context *cctx;
+ char *iv, *salt, *rec_seq;
+
+ if (tx) {
+ crypto_info = &ctx->crypto_send.info;
+ cctx = &ctx->tx;
+ } else {
+ crypto_info = &ctx->crypto_recv.info;
+ cctx = &ctx->rx;
+ }
+
+ src_crypto_info = new_crypto_info ?: crypto_info;
+ cipher_desc = get_cipher_desc(src_crypto_info->cipher_type);
+
+ iv = crypto_info_iv(src_crypto_info, cipher_desc);
+ salt = crypto_info_salt(src_crypto_info, cipher_desc);
+ rec_seq = crypto_info_rec_seq(src_crypto_info, cipher_desc);
+
+ memcpy(cctx->iv, salt, cipher_desc->salt);
+ memcpy(cctx->iv + cipher_desc->salt, iv, cipher_desc->iv);
+ memcpy(cctx->rec_seq, rec_seq, cipher_desc->rec_seq);
+
+ if (new_crypto_info) {
+ unsafe_memcpy(crypto_info, new_crypto_info,
+ cipher_desc->crypto_info,
+ /* size was checked in do_tls_setsockopt_conf */);
+ memzero_explicit(new_crypto_info, cipher_desc->crypto_info);
+
+ if (!tx)
+ tls_finish_key_update(sk, ctx);
+ }
+}
+
+int tls_set_sw_offload(struct sock *sk, int tx,
+ struct tls_crypto_info *new_crypto_info)
+{
+ int rc;
+
+ rc = tls_sw_ctx_init(sk, tx, new_crypto_info);
+ if (rc)
+ return rc;
+
+ tls_sw_ctx_finalize(sk, tx, new_crypto_info);
+ return 0;
+}
--
2.25.1
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH v15 5/9] tls: prep helpers and refactors for HW offload KeyUpdate
2026-07-09 20:53 [PATCH net-next v15 0/9] tls: Add TLS 1.3 hardware offload support Rishikesh Jethwani
` (3 preceding siblings ...)
2026-07-09 20:53 ` [PATCH v15 4/9] tls: split tls_set_sw_offload into init and finalize stages Rishikesh Jethwani
@ 2026-07-09 20:53 ` Rishikesh Jethwani
2026-07-09 20:53 ` [PATCH v15 6/9] tls: device: add TX KeyUpdate support Rishikesh Jethwani
` (3 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Rishikesh Jethwani @ 2026-07-09 20:53 UTC (permalink / raw)
To: netdev
Cc: saeedm, tariqt, mbloch, borisp, john.fastabend, kuba, sd, davem,
pabeni, edumazet, leon, andrew.gospodarek, Rishikesh Jethwani
Preparatory refactors for TX and RX HW rekey support; no functional
change.
- Hoist cipher_context / tls_crypto_context above
tls_offload_context_tx so they can be embedded in offload
contexts.
- Add tls_tx_cipher_ctx() accessor and factor tls_sw_ctx_tx_init()
so the TX path can redirect to a temporary SW context during
rekey.
- Split tls_set_device_offload() into a dispatcher and
tls_set_device_offload_initial(); a _rekey() sibling follows.
- Factor tls_device_dev_add_tx() and tls_device_commit_start_marker()
so the rekey completion path can reuse them.
- Move crypto_aead_setauthsize() into the !*aead block so a fresh
AEAD is correctly configured when RX HW rekey allocates one.
Signed-off-by: Rishikesh Jethwani <rjethwani@purestorage.com>
---
include/net/tls.h | 38 +++++++-----
net/tls/tls.h | 1 +
net/tls/tls_device.c | 139 ++++++++++++++++++++++++++-----------------
net/tls/tls_sw.c | 33 +++++-----
4 files changed, 127 insertions(+), 84 deletions(-)
diff --git a/include/net/tls.h b/include/net/tls.h
index e57bef58851e..eb258bcd62bc 100644
--- a/include/net/tls.h
+++ b/include/net/tls.h
@@ -155,6 +155,22 @@ struct tls_record_info {
skb_frag_t frags[MAX_SKB_FRAGS];
};
+struct cipher_context {
+ char iv[TLS_MAX_IV_SIZE + TLS_MAX_SALT_SIZE];
+ char rec_seq[TLS_MAX_REC_SEQ_SIZE];
+};
+
+union tls_crypto_context {
+ struct tls_crypto_info info;
+ union {
+ struct tls12_crypto_info_aes_gcm_128 aes_gcm_128;
+ struct tls12_crypto_info_aes_gcm_256 aes_gcm_256;
+ struct tls12_crypto_info_chacha20_poly1305 chacha20_poly1305;
+ struct tls12_crypto_info_sm4_gcm sm4_gcm;
+ struct tls12_crypto_info_sm4_ccm sm4_ccm;
+ };
+};
+
#define TLS_DRIVER_STATE_SIZE_TX 16
struct tls_offload_context_tx {
struct crypto_aead *aead_send;
@@ -195,22 +211,6 @@ enum tls_context_flags {
TLS_RX_DEV_CLOSED = 2,
};
-struct cipher_context {
- char iv[TLS_MAX_IV_SIZE + TLS_MAX_SALT_SIZE];
- char rec_seq[TLS_MAX_REC_SEQ_SIZE];
-};
-
-union tls_crypto_context {
- struct tls_crypto_info info;
- union {
- struct tls12_crypto_info_aes_gcm_128 aes_gcm_128;
- struct tls12_crypto_info_aes_gcm_256 aes_gcm_256;
- struct tls12_crypto_info_chacha20_poly1305 chacha20_poly1305;
- struct tls12_crypto_info_sm4_gcm sm4_gcm;
- struct tls12_crypto_info_sm4_ccm sm4_ccm;
- };
-};
-
struct tls_prot_info {
u16 version;
u16 cipher_type;
@@ -392,6 +392,12 @@ static inline struct tls_sw_context_tx *tls_sw_ctx_tx(
return (struct tls_sw_context_tx *)tls_ctx->priv_ctx_tx;
}
+static inline struct cipher_context *tls_tx_cipher_ctx(
+ const struct tls_context *tls_ctx)
+{
+ return (struct cipher_context *)&tls_ctx->tx;
+}
+
static inline struct tls_offload_context_tx *
tls_offload_ctx_tx(const struct tls_context *tls_ctx)
{
diff --git a/net/tls/tls.h b/net/tls/tls.h
index 5a6ee1ea00f8..bf46d3471da6 100644
--- a/net/tls/tls.h
+++ b/net/tls/tls.h
@@ -157,6 +157,7 @@ void tls_update_rx_zc_capable(struct tls_context *tls_ctx);
void tls_sw_strparser_arm(struct sock *sk, struct tls_context *ctx);
void tls_sw_strparser_done(struct tls_context *tls_ctx);
int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
+void tls_sw_ctx_tx_init(struct sock *sk, struct tls_sw_context_tx *sw_ctx);
void tls_sw_splice_eof(struct socket *sock);
void tls_sw_cancel_work_tx(struct tls_context *tls_ctx);
void tls_sw_release_resources_tx(struct sock *sk);
diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c
index f22f8a550c82..7a98d2f6cbd3 100644
--- a/net/tls/tls_device.c
+++ b/net/tls/tls_device.c
@@ -138,6 +138,41 @@ static struct net_device *get_netdev_for_sock(struct sock *sk)
return lowest_dev;
}
+static int tls_device_dev_add_tx(struct sock *sk, struct net_device *netdev,
+ struct tls_crypto_info *crypto_info,
+ u32 write_seq)
+{
+ const struct tls_cipher_desc *cipher_desc;
+ char *rec_seq;
+ int rc;
+
+ cipher_desc = get_cipher_desc(crypto_info->cipher_type);
+ DEBUG_NET_WARN_ON_ONCE(!cipher_desc || !cipher_desc->offloadable);
+
+ rc = netdev->tlsdev_ops->tls_dev_add(netdev, sk, TLS_OFFLOAD_CTX_DIR_TX,
+ crypto_info, write_seq);
+ rec_seq = crypto_info_rec_seq(crypto_info, cipher_desc);
+ trace_tls_device_offload_set(sk, TLS_OFFLOAD_CTX_DIR_TX,
+ write_seq, rec_seq, rc);
+ return rc;
+}
+
+static void tls_device_commit_start_marker(struct sock *sk,
+ struct tls_offload_context_tx *offload_ctx,
+ struct tls_record_info *start_marker_record)
+{
+ start_marker_record->end_seq = tcp_sk(sk)->write_seq;
+ start_marker_record->len = 0;
+ start_marker_record->num_frags = 0;
+ list_add_tail_rcu(&start_marker_record->list, &offload_ctx->records_list);
+
+ /* TLS offload is greatly simplified if we don't send
+ * SKBs where only part of the payload needs to be encrypted.
+ * So mark the last skb in the write queue as end of record.
+ */
+ tcp_write_collapse_fence(sk);
+}
+
static void destroy_record(struct tls_record_info *record)
{
int i;
@@ -1068,57 +1103,31 @@ static struct tls_offload_context_tx *alloc_offload_ctx_tx(struct tls_context *c
return offload_ctx;
}
-int tls_set_device_offload(struct sock *sk)
+static int tls_set_device_offload_initial(struct sock *sk,
+ struct tls_context *ctx,
+ struct net_device *netdev,
+ struct tls_crypto_info *crypto_info,
+ const struct tls_cipher_desc *cipher_desc)
{
+ struct tls_prot_info *prot = &ctx->prot_info;
struct tls_record_info *start_marker_record;
struct tls_offload_context_tx *offload_ctx;
- const struct tls_cipher_desc *cipher_desc;
- struct tls_crypto_info *crypto_info;
- struct tls_prot_info *prot;
- struct net_device *netdev;
- struct tls_context *ctx;
char *iv, *rec_seq;
int rc;
- ctx = tls_get_ctx(sk);
- prot = &ctx->prot_info;
-
- if (ctx->priv_ctx_tx)
- return -EEXIST;
-
- netdev = get_netdev_for_sock(sk);
- if (!netdev) {
- pr_err_ratelimited("%s: netdev not found\n", __func__);
- return -EINVAL;
- }
-
- if (!(netdev->features & NETIF_F_HW_TLS_TX)) {
- rc = -EOPNOTSUPP;
- goto release_netdev;
- }
-
- crypto_info = &ctx->crypto_send.info;
- cipher_desc = get_cipher_desc(crypto_info->cipher_type);
- if (!cipher_desc || !cipher_desc->offloadable) {
- rc = -EINVAL;
- goto release_netdev;
- }
+ iv = crypto_info_iv(crypto_info, cipher_desc);
+ rec_seq = crypto_info_rec_seq(crypto_info, cipher_desc);
rc = init_prot_info(prot, crypto_info, cipher_desc);
if (rc)
- goto release_netdev;
-
- iv = crypto_info_iv(crypto_info, cipher_desc);
- rec_seq = crypto_info_rec_seq(crypto_info, cipher_desc);
+ return rc;
memcpy(ctx->tx.iv + cipher_desc->salt, iv, cipher_desc->iv);
memcpy(ctx->tx.rec_seq, rec_seq, cipher_desc->rec_seq);
start_marker_record = kmalloc_obj(*start_marker_record);
- if (!start_marker_record) {
- rc = -ENOMEM;
- goto release_netdev;
- }
+ if (!start_marker_record)
+ return -ENOMEM;
offload_ctx = alloc_offload_ctx_tx(ctx);
if (!offload_ctx) {
@@ -1130,20 +1139,11 @@ int tls_set_device_offload(struct sock *sk)
if (rc)
goto free_offload_ctx;
- start_marker_record->end_seq = tcp_sk(sk)->write_seq;
- start_marker_record->len = 0;
- start_marker_record->num_frags = 0;
- list_add_tail(&start_marker_record->list, &offload_ctx->records_list);
+ tls_device_commit_start_marker(sk, offload_ctx, start_marker_record);
clean_acked_data_enable(tcp_sk(sk), &tls_tcp_clean_acked);
ctx->push_pending_record = tls_device_push_pending_record;
- /* TLS offload is greatly simplified if we don't send
- * SKBs where only part of the payload needs to be encrypted.
- * So mark the last skb in the write queue as end of record.
- */
- tcp_write_collapse_fence(sk);
-
/* Avoid offloading if the device is down
* We don't want to offload new flows after
* the NETDEV_DOWN event
@@ -1159,11 +1159,8 @@ int tls_set_device_offload(struct sock *sk)
}
ctx->priv_ctx_tx = offload_ctx;
- rc = netdev->tlsdev_ops->tls_dev_add(netdev, sk, TLS_OFFLOAD_CTX_DIR_TX,
- &ctx->crypto_send.info,
- tcp_sk(sk)->write_seq);
- trace_tls_device_offload_set(sk, TLS_OFFLOAD_CTX_DIR_TX,
- tcp_sk(sk)->write_seq, rec_seq, rc);
+ rc = tls_device_dev_add_tx(sk, netdev, crypto_info,
+ tcp_sk(sk)->write_seq);
if (rc)
goto release_lock;
@@ -1175,7 +1172,6 @@ int tls_set_device_offload(struct sock *sk)
* by the netdev's xmit function.
*/
smp_store_release(&sk->sk_validate_xmit_skb, tls_validate_xmit_skb);
- dev_put(netdev);
return 0;
@@ -1188,6 +1184,43 @@ int tls_set_device_offload(struct sock *sk)
ctx->priv_ctx_tx = NULL;
free_marker_record:
kfree(start_marker_record);
+ return rc;
+}
+
+int tls_set_device_offload(struct sock *sk)
+{
+ const struct tls_cipher_desc *cipher_desc;
+ struct tls_crypto_info *crypto_info;
+ struct net_device *netdev;
+ struct tls_context *ctx;
+ int rc;
+
+ ctx = tls_get_ctx(sk);
+
+ if (ctx->priv_ctx_tx)
+ return -EEXIST;
+
+ netdev = get_netdev_for_sock(sk);
+ if (!netdev) {
+ pr_err_ratelimited("%s: netdev not found\n", __func__);
+ return -EINVAL;
+ }
+
+ if (!(netdev->features & NETIF_F_HW_TLS_TX)) {
+ rc = -EOPNOTSUPP;
+ goto release_netdev;
+ }
+
+ crypto_info = &ctx->crypto_send.info;
+ cipher_desc = get_cipher_desc(crypto_info->cipher_type);
+ if (!cipher_desc || !cipher_desc->offloadable) {
+ rc = -EINVAL;
+ goto release_netdev;
+ }
+
+ rc = tls_set_device_offload_initial(sk, ctx, netdev, crypto_info,
+ cipher_desc);
+
release_netdev:
dev_put(netdev);
return rc;
diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c
index 383eab5150ce..b8e06b751efe 100644
--- a/net/tls/tls_sw.c
+++ b/net/tls/tls_sw.c
@@ -555,11 +555,11 @@ static int tls_do_encryption(struct sock *sk,
break;
}
- memcpy(&rec->iv_data[iv_offset], tls_ctx->tx.iv,
+ memcpy(&rec->iv_data[iv_offset], tls_tx_cipher_ctx(tls_ctx)->iv,
prot->iv_size + prot->salt_size);
tls_xor_iv_with_seq(prot, rec->iv_data + iv_offset,
- tls_ctx->tx.rec_seq);
+ tls_tx_cipher_ctx(tls_ctx)->rec_seq);
sge->offset += prot->prepend_size;
sge->length -= prot->prepend_size;
@@ -610,7 +610,7 @@ static int tls_do_encryption(struct sock *sk,
/* Unhook the record from context if encryption is not failure */
ctx->open_rec = NULL;
- tls_advance_record_sn(sk, prot, &tls_ctx->tx);
+ tls_advance_record_sn(sk, prot, tls_tx_cipher_ctx(tls_ctx));
return rc;
}
@@ -676,7 +676,7 @@ static int tls_push_record(struct sock *sk, int flags,
sg_chain(rec->sg_aead_out, 2, &msg_en->sg.data[i]);
tls_make_aad(rec->aad_space, msg_pl->sg.size + prot->tail_size,
- tls_ctx->tx.rec_seq, record_type, prot);
+ tls_tx_cipher_ctx(tls_ctx)->rec_seq, record_type, prot);
tls_fill_prepend(tls_ctx,
page_address(sg_page(&msg_en->sg.data[i])) +
@@ -2369,6 +2369,15 @@ static void tx_work_handler(struct work_struct *work)
}
}
+void tls_sw_ctx_tx_init(struct sock *sk, struct tls_sw_context_tx *sw_ctx)
+{
+ crypto_init_wait(&sw_ctx->async_wait);
+ atomic_set(&sw_ctx->encrypt_pending, 1);
+ INIT_LIST_HEAD(&sw_ctx->tx_list);
+ INIT_DELAYED_WORK(&sw_ctx->tx_work.work, tx_work_handler);
+ sw_ctx->tx_work.sk = sk;
+}
+
static bool tls_is_tx_ready(struct tls_sw_context_tx *ctx)
{
struct tls_rec *rec;
@@ -2420,11 +2429,7 @@ static struct tls_sw_context_tx *init_ctx_tx(struct tls_context *ctx, struct soc
sw_ctx_tx = ctx->priv_ctx_tx;
}
- crypto_init_wait(&sw_ctx_tx->async_wait);
- atomic_set(&sw_ctx_tx->encrypt_pending, 1);
- INIT_LIST_HEAD(&sw_ctx_tx->tx_list);
- INIT_DELAYED_WORK(&sw_ctx_tx->tx_work.work, tx_work_handler);
- sw_ctx_tx->tx_work.sk = sk;
+ tls_sw_ctx_tx_init(sk, sw_ctx_tx);
return sw_ctx_tx;
}
@@ -2551,6 +2556,10 @@ int tls_sw_ctx_init(struct sock *sk, int tx,
*aead = NULL;
goto free_priv;
}
+
+ rc = crypto_aead_setauthsize(*aead, prot->tag_size);
+ if (rc)
+ goto free_aead;
}
ctx->push_pending_record = tls_sw_push_pending_record;
@@ -2567,12 +2576,6 @@ int tls_sw_ctx_init(struct sock *sk, int tx,
goto free_aead;
}
- if (!new_crypto_info) {
- rc = crypto_aead_setauthsize(*aead, prot->tag_size);
- if (rc)
- goto free_aead;
- }
-
if (!tx && !new_crypto_info) {
tfm = crypto_aead_tfm(sw_ctx_rx->aead_recv);
--
2.25.1
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH v15 6/9] tls: device: add TX KeyUpdate support
2026-07-09 20:53 [PATCH net-next v15 0/9] tls: Add TLS 1.3 hardware offload support Rishikesh Jethwani
` (4 preceding siblings ...)
2026-07-09 20:53 ` [PATCH v15 5/9] tls: prep helpers and refactors for HW offload KeyUpdate Rishikesh Jethwani
@ 2026-07-09 20:53 ` Rishikesh Jethwani
2026-07-09 20:53 ` [PATCH v15 7/9] tls: device: add RX " Rishikesh Jethwani
` (2 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Rishikesh Jethwani @ 2026-07-09 20:53 UTC (permalink / raw)
To: netdev
Cc: saeedm, tariqt, mbloch, borisp, john.fastabend, kuba, sd, davem,
pabeni, edumazet, leon, andrew.gospodarek, Rishikesh Jethwani
The NIC key cannot be replaced while HW-offloaded records
are still unacked. tls_device_start_rekey() installs a temporary SW
context with the new key and redirects sendmsg through
tls_sw_sendmsg_locked. If no records are pending,
tls_device_complete_rekey() runs inline during setsockopt; otherwise
tls_tcp_clean_acked sets REKEY_READY once all old-key records are ACKed
and the next sendmsg completes the rekey, flushing SW records and
reinstalling HW offload at the current write_seq. A KeyUpdate
arriving while one is pending re-keys the SW AEAD in place; if the
HW reinstall fails the socket stays in SW mode (REKEY_FAILED).
Tested on Mellanox ConnectX-6 Dx (Crypto Enabled) with multiple
TLS 1.3 TX KeyUpdate cycles.
Signed-off-by: Rishikesh Jethwani <rjethwani@purestorage.com>
---
include/net/tls.h | 42 ++++
include/uapi/linux/snmp.h | 3 +
net/tls/tls.h | 8 +-
net/tls/tls_device.c | 384 ++++++++++++++++++++++++++++++++--
net/tls/tls_device_fallback.c | 24 +++
net/tls/tls_main.c | 81 ++++---
net/tls/tls_proc.c | 3 +
net/tls/tls_sw.c | 22 +-
8 files changed, 526 insertions(+), 41 deletions(-)
diff --git a/include/net/tls.h b/include/net/tls.h
index eb258bcd62bc..2c6e314355fd 100644
--- a/include/net/tls.h
+++ b/include/net/tls.h
@@ -185,6 +185,13 @@ struct tls_offload_context_tx {
void (*sk_destruct)(struct sock *sk);
struct work_struct destruct_work;
struct tls_context *ctx;
+
+ struct {
+ struct tls_sw_context_tx sw; /* SW context for new key */
+ struct cipher_context tx; /* IV, rec_seq for new key */
+ union tls_crypto_context crypto_send; /* Crypto for new key */
+ } rekey;
+
/* The TLS layer reserves room for driver specific state
* Currently the belief is that there is not enough
* driver specific state to justify another layer of indirection
@@ -209,6 +216,21 @@ enum tls_context_flags {
* tls_dev_del call in tls_device_down if it happens simultaneously.
*/
TLS_RX_DEV_CLOSED = 2,
+ /* Flag for TX HW context deleted during failed rekey.
+ * Prevents double tls_dev_del in cleanup paths.
+ */
+ TLS_TX_DEV_CLOSED = 3,
+ /* TX rekey is pending, waiting for old-key data to be ACKed.
+ * While set, new data uses SW path with new key, HW keeps old key
+ * for retransmissions.
+ */
+ TLS_TX_REKEY_PENDING = 4,
+ /* All old-key data has been ACKed, ready to install new key in HW. */
+ TLS_TX_REKEY_READY = 5,
+ /* HW rekey failed, permanently stay in SW encrypt mode.
+ * Prevents tls_tcp_clean_acked from re-setting TLS_TX_REKEY_READY.
+ */
+ TLS_TX_REKEY_FAILED = 6,
};
struct tls_prot_info {
@@ -257,6 +279,17 @@ struct tls_context {
*/
unsigned long flags;
+ struct {
+ /* TCP sequence number boundary for pending rekey.
+ * Packets with seq < this use old key, >= use new key.
+ */
+ u32 boundary_seq;
+
+ /* Pointers to rekey contexts for SW encryption with new key */
+ struct tls_sw_context_tx *sw_ctx;
+ struct cipher_context *cipher_ctx;
+ } rekey;
+
/* cache cold stuff */
struct proto *sk_proto;
struct sock *sk;
@@ -389,12 +422,18 @@ static inline struct tls_sw_context_rx *tls_sw_ctx_rx(
static inline struct tls_sw_context_tx *tls_sw_ctx_tx(
const struct tls_context *tls_ctx)
{
+ if (unlikely(tls_ctx->rekey.sw_ctx))
+ return tls_ctx->rekey.sw_ctx;
+
return (struct tls_sw_context_tx *)tls_ctx->priv_ctx_tx;
}
static inline struct cipher_context *tls_tx_cipher_ctx(
const struct tls_context *tls_ctx)
{
+ if (unlikely(tls_ctx->rekey.cipher_ctx))
+ return tls_ctx->rekey.cipher_ctx;
+
return (struct cipher_context *)&tls_ctx->tx;
}
@@ -510,6 +549,9 @@ struct sk_buff *tls_encrypt_skb(struct sk_buff *skb);
#ifdef CONFIG_TLS_DEVICE
void tls_device_sk_destruct(struct sock *sk);
void tls_offload_tx_resync_request(struct sock *sk, u32 got_seq, u32 exp_seq);
+struct sk_buff *
+tls_validate_xmit_skb_rekey(struct sock *sk, struct net_device *dev,
+ struct sk_buff *skb);
static inline bool tls_is_sk_rx_device_offloaded(struct sock *sk)
{
diff --git a/include/uapi/linux/snmp.h b/include/uapi/linux/snmp.h
index 49f5640092a0..119fbd49179c 100644
--- a/include/uapi/linux/snmp.h
+++ b/include/uapi/linux/snmp.h
@@ -369,6 +369,9 @@ enum
LINUX_MIB_TLSTXREKEYOK, /* TlsTxRekeyOk */
LINUX_MIB_TLSTXREKEYERROR, /* TlsTxRekeyError */
LINUX_MIB_TLSRXREKEYRECEIVED, /* TlsRxRekeyReceived */
+ LINUX_MIB_TLSTXREKEYFALLBACK, /* TlsTxRekeyFallback */
+ LINUX_MIB_TLSTXREKEYINPROGRESS, /* TlsTxRekeyInProgress */
+ LINUX_MIB_TLSTXREKEYABORTED, /* TlsTxRekeyAborted */
__LINUX_MIB_TLSMAX
};
diff --git a/net/tls/tls.h b/net/tls/tls.h
index bf46d3471da6..6e8fafca42ca 100644
--- a/net/tls/tls.h
+++ b/net/tls/tls.h
@@ -157,7 +157,10 @@ void tls_update_rx_zc_capable(struct tls_context *tls_ctx);
void tls_sw_strparser_arm(struct sock *sk, struct tls_context *ctx);
void tls_sw_strparser_done(struct tls_context *tls_ctx);
int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
+int tls_sw_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size);
void tls_sw_ctx_tx_init(struct sock *sk, struct tls_sw_context_tx *sw_ctx);
+int tls_sw_drain_tx(struct sock *sk, struct tls_context *ctx);
+int tls_encrypt_async_wait(struct tls_sw_context_tx *ctx);
void tls_sw_splice_eof(struct socket *sock);
void tls_sw_cancel_work_tx(struct tls_context *tls_ctx);
void tls_sw_release_resources_tx(struct sock *sk);
@@ -235,7 +238,8 @@ static inline bool tls_strp_msg_mixed_decrypted(struct tls_sw_context_rx *ctx)
#ifdef CONFIG_TLS_DEVICE
int tls_device_init(void);
void tls_device_cleanup(void);
-int tls_set_device_offload(struct sock *sk);
+int tls_set_device_offload(struct sock *sk,
+ struct tls_crypto_info *crypto_info);
void tls_device_free_resources_tx(struct sock *sk);
int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx);
void tls_device_offload_cleanup_rx(struct sock *sk);
@@ -246,7 +250,7 @@ static inline int tls_device_init(void) { return 0; }
static inline void tls_device_cleanup(void) {}
static inline int
-tls_set_device_offload(struct sock *sk)
+tls_set_device_offload(struct sock *sk, struct tls_crypto_info *crypto_info)
{
return -EOPNOTSUPP;
}
diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c
index 7a98d2f6cbd3..ef837bbd9560 100644
--- a/net/tls/tls_device.c
+++ b/net/tls/tls_device.c
@@ -57,8 +57,14 @@ static struct page *dummy_page;
static void tls_device_free_ctx(struct tls_context *ctx)
{
- if (ctx->tx_conf == TLS_HW)
- kfree(tls_offload_ctx_tx(ctx));
+ if (ctx->tx_conf == TLS_HW) {
+ struct tls_offload_context_tx *offload_ctx =
+ tls_offload_ctx_tx(ctx);
+
+ memzero_explicit(&offload_ctx->rekey,
+ sizeof(offload_ctx->rekey));
+ kfree(offload_ctx);
+ }
if (ctx->rx_conf == TLS_HW)
kfree(tls_offload_ctx_rx(ctx));
@@ -79,7 +85,9 @@ static void tls_device_tx_del_task(struct work_struct *work)
netdev = rcu_dereference_protected(ctx->netdev,
!refcount_read(&ctx->refcount));
- netdev->tlsdev_ops->tls_dev_del(netdev, ctx, TLS_OFFLOAD_CTX_DIR_TX);
+ if (!test_bit(TLS_TX_DEV_CLOSED, &ctx->flags))
+ netdev->tlsdev_ops->tls_dev_del(netdev, ctx,
+ TLS_OFFLOAD_CTX_DIR_TX);
dev_put(netdev);
ctx->netdev = NULL;
tls_device_free_ctx(ctx);
@@ -161,10 +169,14 @@ static void tls_device_commit_start_marker(struct sock *sk,
struct tls_offload_context_tx *offload_ctx,
struct tls_record_info *start_marker_record)
{
+ unsigned long flags;
+
+ spin_lock_irqsave(&offload_ctx->lock, flags);
start_marker_record->end_seq = tcp_sk(sk)->write_seq;
start_marker_record->len = 0;
start_marker_record->num_frags = 0;
list_add_tail_rcu(&start_marker_record->list, &offload_ctx->records_list);
+ spin_unlock_irqrestore(&offload_ctx->lock, flags);
/* TLS offload is greatly simplified if we don't send
* SKBs where only part of the payload needs to be encrypted.
@@ -194,6 +206,24 @@ static void delete_all_records(struct tls_offload_context_tx *offload_ctx)
offload_ctx->retransmit_hint = NULL;
}
+static bool tls_has_unacked_records(struct tls_offload_context_tx *offload_ctx)
+{
+ struct tls_record_info *info;
+ bool has_unacked = false;
+ unsigned long flags;
+
+ spin_lock_irqsave(&offload_ctx->lock, flags);
+ list_for_each_entry(info, &offload_ctx->records_list, list) {
+ if (!tls_record_is_start_marker(info)) {
+ has_unacked = true;
+ break;
+ }
+ }
+ spin_unlock_irqrestore(&offload_ctx->lock, flags);
+
+ return has_unacked;
+}
+
static void tls_tcp_clean_acked(struct sock *sk, u32 acked_seq)
{
struct tls_context *tls_ctx = tls_get_ctx(sk);
@@ -222,6 +252,19 @@ static void tls_tcp_clean_acked(struct sock *sk, u32 acked_seq)
}
ctx->unacked_record_sn += deleted_records;
+
+ /* Once all old-key HW records are ACKed, set REKEY_READY to
+ * let sendmsg know it can finish the rekey and switch back
+ * to HW offload.
+ */
+ if (test_bit(TLS_TX_REKEY_PENDING, &tls_ctx->flags) &&
+ !test_bit(TLS_TX_REKEY_FAILED, &tls_ctx->flags)) {
+ u32 boundary_seq = READ_ONCE(tls_ctx->rekey.boundary_seq);
+
+ if (!before(acked_seq, boundary_seq))
+ set_bit(TLS_TX_REKEY_READY, &tls_ctx->flags);
+ }
+
spin_unlock_irqrestore(&ctx->lock, flags);
}
@@ -253,6 +296,14 @@ void tls_device_free_resources_tx(struct sock *sk)
struct tls_context *tls_ctx = tls_get_ctx(sk);
tls_free_partial_record(sk, tls_ctx);
+
+ if (unlikely(tls_ctx->rekey.sw_ctx))
+ tls_sw_release_resources_tx(sk);
+
+ if (test_bit(TLS_TX_REKEY_PENDING, &tls_ctx->flags)) {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYABORTED);
+ TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYINPROGRESS);
+ }
}
void tls_offload_tx_resync_request(struct sock *sk, u32 got_seq, u32 exp_seq)
@@ -462,6 +513,9 @@ static int tls_device_copy_data(void *addr, size_t bytes, struct iov_iter *i)
return 0;
}
+static int tls_device_complete_rekey(struct sock *sk, struct tls_context *ctx,
+ bool deferred);
+
static int tls_push_data(struct sock *sk,
struct iov_iter *iter,
size_t size, int flags,
@@ -624,6 +678,19 @@ int tls_device_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
goto out;
}
+ /* Old-key records all ACKed; switch back to HW. */
+ if (test_bit(TLS_TX_REKEY_READY, &tls_ctx->flags))
+ tls_device_complete_rekey(sk, tls_ctx, true);
+
+ /* Use SW path if rekey is in progress (PENDING) or if HW rekey
+ * failed (FAILED).
+ */
+ if (test_bit(TLS_TX_REKEY_PENDING, &tls_ctx->flags) ||
+ test_bit(TLS_TX_REKEY_FAILED, &tls_ctx->flags)) {
+ rc = tls_sw_sendmsg_locked(sk, msg, size);
+ goto out;
+ }
+
rc = tls_push_data(sk, &msg->msg_iter, size, msg->msg_flags,
record_type);
@@ -1103,6 +1170,282 @@ static struct tls_offload_context_tx *alloc_offload_ctx_tx(struct tls_context *c
return offload_ctx;
}
+static int tls_device_init_rekey_sw(struct sock *sk,
+ struct tls_context *ctx,
+ struct tls_offload_context_tx *offload_ctx,
+ struct tls_crypto_info *new_crypto_info)
+{
+ struct tls_sw_context_tx *sw_ctx = &offload_ctx->rekey.sw;
+ const struct tls_cipher_desc *cipher_desc;
+ char *key;
+ int rc;
+
+ cipher_desc = get_cipher_desc(new_crypto_info->cipher_type);
+ DEBUG_NET_WARN_ON_ONCE(!cipher_desc || !cipher_desc->offloadable);
+
+ memset(sw_ctx, 0, sizeof(*sw_ctx));
+ tls_sw_ctx_tx_init(sk, sw_ctx);
+
+ sw_ctx->aead_send = crypto_alloc_aead(cipher_desc->cipher_name, 0, 0);
+ if (IS_ERR(sw_ctx->aead_send)) {
+ rc = PTR_ERR(sw_ctx->aead_send);
+ sw_ctx->aead_send = NULL;
+ return rc;
+ }
+
+ key = crypto_info_key(new_crypto_info, cipher_desc);
+ rc = crypto_aead_setkey(sw_ctx->aead_send, key, cipher_desc->key);
+ if (rc)
+ goto free_aead;
+
+ rc = crypto_aead_setauthsize(sw_ctx->aead_send, cipher_desc->tag);
+ if (rc)
+ goto free_aead;
+
+ return 0;
+
+free_aead:
+ crypto_free_aead(sw_ctx->aead_send);
+ sw_ctx->aead_send = NULL;
+ return rc;
+}
+
+static int tls_device_start_rekey(struct sock *sk,
+ struct tls_context *ctx,
+ struct tls_offload_context_tx *offload_ctx,
+ struct tls_crypto_info *new_crypto_info)
+{
+ bool rekey_pending = test_bit(TLS_TX_REKEY_PENDING, &ctx->flags);
+ bool rekey_failed = test_bit(TLS_TX_REKEY_FAILED, &ctx->flags);
+ const struct tls_cipher_desc *cipher_desc;
+ char *key, *iv, *rec_seq, *salt;
+ unsigned long flags;
+ int rc;
+
+ cipher_desc = get_cipher_desc(new_crypto_info->cipher_type);
+ DEBUG_NET_WARN_ON_ONCE(!cipher_desc || !cipher_desc->offloadable);
+
+ key = crypto_info_key(new_crypto_info, cipher_desc);
+ iv = crypto_info_iv(new_crypto_info, cipher_desc);
+ rec_seq = crypto_info_rec_seq(new_crypto_info, cipher_desc);
+ salt = crypto_info_salt(new_crypto_info, cipher_desc);
+
+ if (rekey_pending || rekey_failed) {
+ /* Wait for in-flight async encryptions submitted to this tfm
+ * with the previous key before changing it.
+ */
+ rc = tls_encrypt_async_wait(&offload_ctx->rekey.sw);
+ if (rc)
+ return rc;
+
+ rc = crypto_aead_setkey(offload_ctx->rekey.sw.aead_send,
+ key, cipher_desc->key);
+ if (rc)
+ return rc;
+
+ memcpy(offload_ctx->rekey.tx.iv, salt, cipher_desc->salt);
+ memcpy(offload_ctx->rekey.tx.iv + cipher_desc->salt, iv,
+ cipher_desc->iv);
+ memcpy(offload_ctx->rekey.tx.rec_seq, rec_seq,
+ cipher_desc->rec_seq);
+
+ if (rekey_failed) {
+ set_bit(TLS_TX_REKEY_PENDING, &ctx->flags);
+ clear_bit(TLS_TX_REKEY_FAILED, &ctx->flags);
+ TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXDEVICE);
+ }
+ } else {
+ /* Flush any HW open_record before switching to SW */
+ if (tls_is_pending_open_record(ctx)) {
+ rc = ctx->push_pending_record(sk, 0);
+ if (rc < 0)
+ return rc;
+ }
+
+ rc = tls_device_init_rekey_sw(sk, ctx, offload_ctx,
+ new_crypto_info);
+ if (rc)
+ return rc;
+
+ memcpy(offload_ctx->rekey.tx.iv, salt, cipher_desc->salt);
+ memcpy(offload_ctx->rekey.tx.iv + cipher_desc->salt, iv,
+ cipher_desc->iv);
+ memcpy(offload_ctx->rekey.tx.rec_seq, rec_seq,
+ cipher_desc->rec_seq);
+
+ /* Prevent a partial record straddling the SW/HW boundary. */
+ tcp_write_collapse_fence(sk);
+
+ ctx->rekey.sw_ctx = &offload_ctx->rekey.sw;
+ ctx->rekey.cipher_ctx = &offload_ctx->rekey.tx;
+
+ spin_lock_irqsave(&offload_ctx->lock, flags);
+ WRITE_ONCE(ctx->rekey.boundary_seq, tcp_sk(sk)->write_seq);
+ set_bit(TLS_TX_REKEY_PENDING, &ctx->flags);
+ spin_unlock_irqrestore(&offload_ctx->lock, flags);
+
+ /* Switch to rekey validator; new sends won't use HW offload */
+ smp_store_release(&sk->sk_validate_xmit_skb,
+ tls_validate_xmit_skb_rekey);
+ }
+
+ unsafe_memcpy(&offload_ctx->rekey.crypto_send.info, new_crypto_info,
+ cipher_desc->crypto_info,
+ /* checked in do_tls_setsockopt_conf */);
+ memzero_explicit(new_crypto_info, cipher_desc->crypto_info);
+
+ return 0;
+}
+
+static int tls_device_complete_rekey(struct sock *sk, struct tls_context *ctx,
+ bool deferred)
+{
+ struct tls_offload_context_tx *offload_ctx = tls_offload_ctx_tx(ctx);
+ struct tls_record_info *start_marker_record;
+ const struct tls_cipher_desc *cipher_desc;
+ struct net_device *netdev;
+ unsigned long flags;
+ __be64 rcd_sn;
+ char *key;
+ int rc;
+
+ cipher_desc = get_cipher_desc(offload_ctx->rekey.crypto_send.info.cipher_type);
+ DEBUG_NET_WARN_ON_ONCE(!cipher_desc || !cipher_desc->offloadable);
+
+ rc = tls_sw_drain_tx(sk, ctx);
+ if (rc)
+ return rc;
+
+ start_marker_record = kmalloc_obj(*start_marker_record);
+ if (!start_marker_record)
+ return -ENOMEM;
+
+ down_read(&device_offload_lock);
+
+ netdev = rcu_dereference_protected(ctx->netdev,
+ lockdep_is_held(&device_offload_lock));
+ if (!netdev) {
+ rc = -ENODEV;
+ goto release_lock;
+ }
+
+ if (!test_bit(TLS_TX_DEV_CLOSED, &ctx->flags)) {
+ netdev->tlsdev_ops->tls_dev_del(netdev, ctx,
+ TLS_OFFLOAD_CTX_DIR_TX);
+ set_bit(TLS_TX_DEV_CLOSED, &ctx->flags);
+ }
+
+ memcpy(crypto_info_rec_seq(&offload_ctx->rekey.crypto_send.info, cipher_desc),
+ offload_ctx->rekey.tx.rec_seq, cipher_desc->rec_seq);
+
+ rc = tls_device_dev_add_tx(sk, netdev, &offload_ctx->rekey.crypto_send.info,
+ tcp_sk(sk)->write_seq);
+ if (!rc)
+ clear_bit(TLS_TX_DEV_CLOSED, &ctx->flags);
+
+release_lock:
+ up_read(&device_offload_lock);
+
+ if (rc)
+ goto rekey_fail;
+
+ spin_lock_irqsave(&offload_ctx->lock, flags);
+ memcpy(&rcd_sn, offload_ctx->rekey.tx.rec_seq, sizeof(rcd_sn));
+ offload_ctx->unacked_record_sn = be64_to_cpu(rcd_sn) - 1;
+ spin_unlock_irqrestore(&offload_ctx->lock, flags);
+
+ memcpy(ctx->tx.iv, offload_ctx->rekey.tx.iv,
+ cipher_desc->salt + cipher_desc->iv);
+ memcpy(ctx->tx.rec_seq, offload_ctx->rekey.tx.rec_seq,
+ cipher_desc->rec_seq);
+ unsafe_memcpy(&ctx->crypto_send.info,
+ &offload_ctx->rekey.crypto_send.info,
+ cipher_desc->crypto_info,
+ /* checked during rekey setup */);
+
+ key = crypto_info_key(&offload_ctx->rekey.crypto_send.info, cipher_desc);
+ rc = crypto_aead_setkey(offload_ctx->aead_send, key, cipher_desc->key);
+ if (rc)
+ goto rekey_fail;
+
+ /* Start marker: the NIC passes through everything before
+ * write_seq unencrypted (already SW-encrypted during rekey),
+ * same as during initial offload setup.
+ */
+ tls_device_commit_start_marker(sk, offload_ctx, start_marker_record);
+
+ spin_lock_irqsave(&offload_ctx->lock, flags);
+ clear_bit(TLS_TX_REKEY_PENDING, &ctx->flags);
+ clear_bit(TLS_TX_REKEY_READY, &ctx->flags);
+ clear_bit(TLS_TX_REKEY_FAILED, &ctx->flags);
+ spin_unlock_irqrestore(&offload_ctx->lock, flags);
+
+ /* Switch back to HW offload validator */
+ smp_store_release(&sk->sk_validate_xmit_skb, tls_validate_xmit_skb);
+
+ crypto_free_aead(tls_sw_ctx_tx(ctx)->aead_send);
+ ctx->rekey.sw_ctx = NULL;
+ ctx->rekey.cipher_ctx = NULL;
+ memzero_explicit(&offload_ctx->rekey, sizeof(offload_ctx->rekey));
+
+ if (deferred)
+ TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYINPROGRESS);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYOK);
+ return 0;
+
+rekey_fail:
+ kfree(start_marker_record);
+ spin_lock_irqsave(&offload_ctx->lock, flags);
+ set_bit(TLS_TX_REKEY_FAILED, &ctx->flags);
+ clear_bit(TLS_TX_REKEY_READY, &ctx->flags);
+ clear_bit(TLS_TX_REKEY_PENDING, &ctx->flags);
+ spin_unlock_irqrestore(&offload_ctx->lock, flags);
+ if (deferred)
+ TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYINPROGRESS);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYFALLBACK);
+ TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXDEVICE);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW);
+
+ return 0;
+}
+
+static int tls_set_device_offload_rekey(struct sock *sk,
+ struct tls_context *ctx,
+ struct net_device *netdev,
+ struct tls_crypto_info *new_crypto_info)
+{
+ struct tls_offload_context_tx *offload_ctx = tls_offload_ctx_tx(ctx);
+ bool rekey_pending = test_bit(TLS_TX_REKEY_PENDING, &ctx->flags);
+ bool rekey_failed = test_bit(TLS_TX_REKEY_FAILED, &ctx->flags);
+ bool defer = true;
+ int rc;
+
+ if (!rekey_pending && !rekey_failed)
+ defer = tls_has_unacked_records(offload_ctx) ||
+ tls_is_pending_open_record(ctx);
+
+ down_read(&device_offload_lock);
+
+ rc = tls_device_start_rekey(sk, ctx, offload_ctx, new_crypto_info);
+ if (rc) {
+ up_read(&device_offload_lock);
+ return rc;
+ }
+
+ up_read(&device_offload_lock);
+
+ if (defer) {
+ if (!rekey_pending)
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYINPROGRESS);
+ else
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYOK);
+ return 0;
+ }
+
+ return tls_device_complete_rekey(sk, ctx, false);
+}
+
static int tls_set_device_offload_initial(struct sock *sk,
struct tls_context *ctx,
struct net_device *netdev,
@@ -1187,18 +1530,23 @@ static int tls_set_device_offload_initial(struct sock *sk,
return rc;
}
-int tls_set_device_offload(struct sock *sk)
+int tls_set_device_offload(struct sock *sk,
+ struct tls_crypto_info *new_crypto_info)
{
+ struct tls_crypto_info *crypto_info, *src_crypto_info;
const struct tls_cipher_desc *cipher_desc;
- struct tls_crypto_info *crypto_info;
struct net_device *netdev;
struct tls_context *ctx;
int rc;
ctx = tls_get_ctx(sk);
- if (ctx->priv_ctx_tx)
- return -EEXIST;
+ /* Rekey is only supported for connections that are already
+ * using HW offload. For SW offload connections, the caller
+ * should fall back to tls_set_sw_offload() for rekey.
+ */
+ if (new_crypto_info && ctx->tx_conf != TLS_HW)
+ return -EINVAL;
netdev = get_netdev_for_sock(sk);
if (!netdev) {
@@ -1212,14 +1560,20 @@ int tls_set_device_offload(struct sock *sk)
}
crypto_info = &ctx->crypto_send.info;
- cipher_desc = get_cipher_desc(crypto_info->cipher_type);
+ src_crypto_info = new_crypto_info ?: crypto_info;
+ cipher_desc = get_cipher_desc(src_crypto_info->cipher_type);
if (!cipher_desc || !cipher_desc->offloadable) {
rc = -EINVAL;
goto release_netdev;
}
- rc = tls_set_device_offload_initial(sk, ctx, netdev, crypto_info,
- cipher_desc);
+ if (new_crypto_info)
+ rc = tls_set_device_offload_rekey(sk, ctx, netdev,
+ src_crypto_info);
+ else
+ rc = tls_set_device_offload_initial(sk, ctx, netdev,
+ src_crypto_info,
+ cipher_desc);
release_netdev:
dev_put(netdev);
@@ -1352,7 +1706,10 @@ static int tls_device_down(struct net_device *netdev)
/* Stop offloaded TX and switch to the fallback.
* tls_is_skb_tx_device_offloaded will return false.
*/
- WRITE_ONCE(ctx->sk->sk_validate_xmit_skb, tls_validate_xmit_skb_sw);
+ if (!test_bit(TLS_TX_REKEY_PENDING, &ctx->flags) &&
+ !test_bit(TLS_TX_REKEY_FAILED, &ctx->flags))
+ WRITE_ONCE(ctx->sk->sk_validate_xmit_skb,
+ tls_validate_xmit_skb_sw);
/* Stop the RX and TX resync.
* tls_dev_resync must not be called after tls_dev_del.
@@ -1369,9 +1726,12 @@ static int tls_device_down(struct net_device *netdev)
synchronize_net();
/* Release the offload context on the driver side. */
- if (ctx->tx_conf == TLS_HW)
+ if (ctx->tx_conf == TLS_HW &&
+ !test_bit(TLS_TX_DEV_CLOSED, &ctx->flags)) {
netdev->tlsdev_ops->tls_dev_del(netdev, ctx,
TLS_OFFLOAD_CTX_DIR_TX);
+ set_bit(TLS_TX_DEV_CLOSED, &ctx->flags);
+ }
if (ctx->rx_conf == TLS_HW &&
!test_bit(TLS_RX_DEV_CLOSED, &ctx->flags))
netdev->tlsdev_ops->tls_dev_del(netdev, ctx,
diff --git a/net/tls/tls_device_fallback.c b/net/tls/tls_device_fallback.c
index 1110f7ac6bcb..64ac4ef4012b 100644
--- a/net/tls/tls_device_fallback.c
+++ b/net/tls/tls_device_fallback.c
@@ -435,6 +435,30 @@ struct sk_buff *tls_validate_xmit_skb_sw(struct sock *sk,
return tls_sw_fallback(sk, skb);
}
+struct sk_buff *tls_validate_xmit_skb_rekey(struct sock *sk,
+ struct net_device *dev,
+ struct sk_buff *skb)
+{
+ struct tls_context *tls_ctx = tls_get_ctx(sk);
+ u32 tcp_seq = ntohl(tcp_hdr(skb)->seq);
+ u32 boundary_seq;
+
+ if (test_bit(TLS_TX_REKEY_FAILED, &tls_ctx->flags))
+ return skb;
+
+ /* If this packet is at or after the rekey boundary, it's already
+ * SW-encrypted with the new key, pass through unchanged
+ */
+ boundary_seq = READ_ONCE(tls_ctx->rekey.boundary_seq);
+ if (!before(tcp_seq, boundary_seq))
+ return skb;
+
+ /* Packet before boundary means retransmit of old data,
+ * use SW fallback with the old key
+ */
+ return tls_sw_fallback(sk, skb);
+}
+
struct sk_buff *tls_encrypt_skb(struct sk_buff *skb)
{
return tls_sw_fallback(skb->sk, skb);
diff --git a/net/tls/tls_main.c b/net/tls/tls_main.c
index c4c15c0e731a..b51f9ea1ed6c 100644
--- a/net/tls/tls_main.c
+++ b/net/tls/tls_main.c
@@ -347,8 +347,14 @@ static void tls_sk_proto_cleanup(struct sock *sk,
tls_sw_release_resources_tx(sk);
TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW);
} else if (ctx->tx_conf == TLS_HW) {
+ bool rekey_failed = test_bit(TLS_TX_REKEY_FAILED, &ctx->flags);
+
tls_device_free_resources_tx(sk);
- TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXDEVICE);
+
+ if (rekey_failed)
+ TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW);
+ else
+ TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXDEVICE);
}
if (ctx->rx_conf == TLS_SW) {
@@ -369,6 +375,8 @@ static void tls_sk_proto_close(struct sock *sk, long timeout)
if (ctx->tx_conf == TLS_SW)
tls_sw_cancel_work_tx(ctx);
+ else if (ctx->tx_conf == TLS_HW && ctx->rekey.sw_ctx)
+ tls_sw_cancel_work_tx(ctx);
lock_sock(sk);
free_ctx = ctx->tx_conf != TLS_HW && ctx->rx_conf != TLS_HW;
@@ -449,8 +457,16 @@ static int do_tls_getsockopt_conf(struct sock *sk, char __user *optval,
/* get user crypto info */
if (tx) {
- crypto_info = &ctx->crypto_send.info;
- cctx = &ctx->tx;
+ if (ctx->rekey.cipher_ctx) {
+ struct tls_offload_context_tx *offload_ctx =
+ tls_offload_ctx_tx(ctx);
+
+ crypto_info = &offload_ctx->rekey.crypto_send.info;
+ cctx = &offload_ctx->rekey.tx;
+ } else {
+ crypto_info = &ctx->crypto_send.info;
+ cctx = &ctx->tx;
+ }
} else {
crypto_info = &ctx->crypto_recv.info;
cctx = &ctx->rx;
@@ -714,32 +730,32 @@ static int do_tls_setsockopt_conf(struct sock *sk, sockptr_t optval,
}
if (tx) {
- if (update && ctx->tx_conf == TLS_HW) {
- rc = -EOPNOTSUPP;
- goto err_crypto_info;
- }
-
- if (!update) {
- rc = tls_set_device_offload(sk);
- conf = TLS_HW;
- if (!rc) {
+ rc = tls_set_device_offload(sk, update ? crypto_info : NULL);
+ conf = TLS_HW;
+ if (!rc) {
+ if (!update) {
TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXDEVICE);
TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXDEVICE);
- goto out;
}
- }
-
- rc = tls_set_sw_offload(sk, 1, update ? crypto_info : NULL);
- if (rc)
+ } else if (update && ctx->tx_conf == TLS_HW) {
+ /* HW rekey failed - return the actual error.
+ * Cannot fall back to SW for an existing HW connection.
+ */
goto err_crypto_info;
-
- if (update) {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYOK);
} else {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXSW);
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW);
+ rc = tls_set_sw_offload(sk, 1,
+ update ? crypto_info : NULL);
+ if (rc)
+ goto err_crypto_info;
+
+ if (update) {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYOK);
+ } else {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXSW);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW);
+ }
+ conf = TLS_SW;
}
- conf = TLS_SW;
} else {
if (update && ctx->rx_conf == TLS_HW) {
rc = -EOPNOTSUPP;
@@ -885,12 +901,29 @@ static int do_tls_setsockopt(struct sock *sk, int optname, sockptr_t optval,
switch (optname) {
case TLS_TX:
- case TLS_RX:
+ case TLS_RX: {
+ struct tls_context *ctx = tls_get_ctx(sk);
+ /* A HW-offload TX rekey arrives mid-stream and races with an
+ * in-flight tls_device_sendmsg(), which holds tx_lock across
+ * the lock_sock drop in sk_stream_wait_memory() while an
+ * open_record is half built. start_rekey() would flush that
+ * record and flip the SW/HW boundary under the sender,
+ * corrupting record framing. Serialize against the data path
+ * with tx_lock. Only the HW TX path reaches start_rekey, and
+ * initial setup (tx_conf == BASE) has no concurrent sender.
+ */
+ bool tx_rekey = optname == TLS_TX && ctx->tx_conf == TLS_HW;
+
+ if (tx_rekey)
+ mutex_lock(&ctx->tx_lock);
lock_sock(sk);
rc = do_tls_setsockopt_conf(sk, optval, optlen,
optname == TLS_TX);
release_sock(sk);
+ if (tx_rekey)
+ mutex_unlock(&ctx->tx_lock);
break;
+ }
case TLS_TX_ZEROCOPY_RO:
lock_sock(sk);
rc = do_tls_setsockopt_tx_zc(sk, optval, optlen);
diff --git a/net/tls/tls_proc.c b/net/tls/tls_proc.c
index 4012c4372d4c..adca8561c7da 100644
--- a/net/tls/tls_proc.c
+++ b/net/tls/tls_proc.c
@@ -27,6 +27,9 @@ static const struct snmp_mib tls_mib_list[] = {
SNMP_MIB_ITEM("TlsTxRekeyOk", LINUX_MIB_TLSTXREKEYOK),
SNMP_MIB_ITEM("TlsTxRekeyError", LINUX_MIB_TLSTXREKEYERROR),
SNMP_MIB_ITEM("TlsRxRekeyReceived", LINUX_MIB_TLSRXREKEYRECEIVED),
+ SNMP_MIB_ITEM("TlsTxRekeyFallback", LINUX_MIB_TLSTXREKEYFALLBACK),
+ SNMP_MIB_ITEM("TlsTxRekeyInProgress", LINUX_MIB_TLSTXREKEYINPROGRESS),
+ SNMP_MIB_ITEM("TlsTxRekeyAborted", LINUX_MIB_TLSTXREKEYABORTED),
};
static int tls_statistics_seq_show(struct seq_file *seq, void *v)
diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c
index b8e06b751efe..0364fae8a7e0 100644
--- a/net/tls/tls_sw.c
+++ b/net/tls/tls_sw.c
@@ -522,7 +522,7 @@ static void tls_encrypt_done(void *data, int err)
complete(&ctx->async_wait.completion);
}
-static int tls_encrypt_async_wait(struct tls_sw_context_tx *ctx)
+int tls_encrypt_async_wait(struct tls_sw_context_tx *ctx)
{
if (!atomic_dec_and_test(&ctx->encrypt_pending))
crypto_wait_req(-EINPROGRESS, &ctx->async_wait);
@@ -763,8 +763,7 @@ static int tls_sw_sendmsg_splice(struct sock *sk, struct msghdr *msg,
return 0;
}
-static int tls_sw_sendmsg_locked(struct sock *sk, struct msghdr *msg,
- size_t size)
+int tls_sw_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size)
{
long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
struct tls_context *tls_ctx = tls_get_ctx(sk);
@@ -2378,6 +2377,23 @@ void tls_sw_ctx_tx_init(struct sock *sk, struct tls_sw_context_tx *sw_ctx)
sw_ctx->tx_work.sk = sk;
}
+int tls_sw_drain_tx(struct sock *sk, struct tls_context *ctx)
+{
+ struct tls_sw_context_tx *sw_ctx = tls_sw_ctx_tx(ctx);
+ int rc;
+
+ if (tls_is_pending_open_record(ctx))
+ tls_sw_push_pending_record(sk, 0);
+ tls_encrypt_async_wait(sw_ctx);
+ rc = tls_tx_records(sk, -1);
+ if (rc < 0 || tls_is_partially_sent_record(ctx) ||
+ tls_is_pending_open_record(ctx))
+ return rc < 0 ? rc : -EAGAIN;
+
+ cancel_delayed_work_sync(&sw_ctx->tx_work.work);
+ return 0;
+}
+
static bool tls_is_tx_ready(struct tls_sw_context_tx *ctx)
{
struct tls_rec *rec;
--
2.25.1
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH v15 7/9] tls: device: add RX KeyUpdate support
2026-07-09 20:53 [PATCH net-next v15 0/9] tls: Add TLS 1.3 hardware offload support Rishikesh Jethwani
` (5 preceding siblings ...)
2026-07-09 20:53 ` [PATCH v15 6/9] tls: device: add TX KeyUpdate support Rishikesh Jethwani
@ 2026-07-09 20:53 ` Rishikesh Jethwani
2026-07-09 20:53 ` [PATCH v15 8/9] tls: device: add tracepoints for the KeyUpdate path Rishikesh Jethwani
2026-07-09 20:53 ` [PATCH v15 9/9] selftests: net: add TLS hardware offload test Rishikesh Jethwani
8 siblings, 0 replies; 10+ messages in thread
From: Rishikesh Jethwani @ 2026-07-09 20:53 UTC (permalink / raw)
To: netdev
Cc: saeedm, tariqt, mbloch, borisp, john.fastabend, kuba, sd, davem,
pabeni, edumazet, leon, andrew.gospodarek, Rishikesh Jethwani
On RX, the NIC may have already decrypted in-flight records with
the old key before the peer's KeyUpdate is parsed, so the old
AEAD, IV and rec_seq are retained on tls_offload_context_rx.
tls_device_rx_del_key() is called from tls_check_pending_rekey()
when a KeyUpdate record is decoded; otherwise post-KeyUpdate records
(carrying new-key wire encryption) would be decrypted with the retired key.
tls_device_decrypted() classifies records by old_nic_boundary:
- after the boundary: new-key record; drop the old key.
- before, fully encrypted: advance old_rec_seq, let SW AEAD decrypt.
- before, (partially) decrypted: reencrypt with the old key so SW
AEAD can decrypt with the new key.
For mixed records the NIC may have cleared skb->decrypted on auth failure;
the optional tls_dev_rx_rekey_fixup callback re-asserts it,
then the record is reencrypted with the old key so the SW AEAD can
decrypt it with the new key.
The new key's tls_dev_add is deferred until the old key is fully
consumed: tls_set_device_offload_rx() sets dev_add_pending while
old_aead_recv is retained, and tls_device_deferred_dev_add_rx()
installs the new key once copied_seq crosses old_nic_boundary.
Tested on Mellanox ConnectX-6 Dx (Crypto Enabled) with multiple
TLS 1.3 RX KeyUpdate cycles.
Signed-off-by: Rishikesh Jethwani <rjethwani@purestorage.com>
---
.../mellanox/mlx5/core/en_accel/ktls.c | 10 +
include/net/tls.h | 10 +
include/uapi/linux/snmp.h | 3 +
net/tls/tls.h | 9 +-
net/tls/tls_device.c | 314 ++++++++++++++++--
net/tls/tls_main.c | 46 +--
net/tls/tls_proc.c | 3 +
net/tls/tls_sw.c | 1 +
8 files changed, 346 insertions(+), 50 deletions(-)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c
index 337201f12895..71c9d3ca947d 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c
@@ -86,10 +86,20 @@ static int mlx5e_ktls_resync(struct net_device *netdev,
return 0;
}
+static void mlx5e_ktls_rx_rekey_fixup(struct sk_buff *skb)
+{
+ struct sk_buff *frag_iter;
+
+ skb->decrypted = 1;
+ skb_walk_frags(skb, frag_iter)
+ frag_iter->decrypted = 1;
+}
+
static const struct tlsdev_ops mlx5e_ktls_ops = {
.tls_dev_add = mlx5e_ktls_add,
.tls_dev_del = mlx5e_ktls_del,
.tls_dev_resync = mlx5e_ktls_resync,
+ .tls_dev_rx_rekey_fixup = mlx5e_ktls_rx_rekey_fixup,
};
bool mlx5e_is_ktls_rx(struct mlx5_core_dev *mdev)
diff --git a/include/net/tls.h b/include/net/tls.h
index 2c6e314355fd..de04dfcf7a8c 100644
--- a/include/net/tls.h
+++ b/include/net/tls.h
@@ -320,6 +320,7 @@ struct tlsdev_ops {
int (*tls_dev_resync)(struct net_device *netdev,
struct sock *sk, u32 seq, u8 *rcd_sn,
enum tls_offload_ctx_dir direction);
+ void (*tls_dev_rx_rekey_fixup)(struct sk_buff *skb);
};
enum tls_offload_sync_type {
@@ -348,6 +349,15 @@ struct tls_offload_context_rx {
u8 resync_nh_reset:1;
/* CORE_NEXT_HINT-only member, but use the hole here */
u8 resync_nh_do_now:1;
+ /* tls_dev_add deferred until old key is freed */
+ u8 dev_add_pending:1;
+ struct {
+ struct crypto_aead *old_aead_recv; /* old key AEAD cipher */
+ char old_iv[TLS_MAX_IV_SIZE + TLS_MAX_SALT_SIZE]; /* old key IV */
+ char old_rec_seq[TLS_MAX_REC_SEQ_SIZE]; /* old key TLS record seq */
+ u32 old_nic_boundary; /* TCP seq: NIC switched to next key */
+ void (*rekey_fixup)(struct sk_buff *skb);
+ } rekey;
union {
/* TLS_OFFLOAD_SYNC_TYPE_DRIVER_REQ */
struct {
diff --git a/include/uapi/linux/snmp.h b/include/uapi/linux/snmp.h
index 119fbd49179c..284f879c92cc 100644
--- a/include/uapi/linux/snmp.h
+++ b/include/uapi/linux/snmp.h
@@ -370,8 +370,11 @@ enum
LINUX_MIB_TLSTXREKEYERROR, /* TlsTxRekeyError */
LINUX_MIB_TLSRXREKEYRECEIVED, /* TlsRxRekeyReceived */
LINUX_MIB_TLSTXREKEYFALLBACK, /* TlsTxRekeyFallback */
+ LINUX_MIB_TLSRXREKEYFALLBACK, /* TlsRxRekeyFallback */
LINUX_MIB_TLSTXREKEYINPROGRESS, /* TlsTxRekeyInProgress */
+ LINUX_MIB_TLSRXREKEYINPROGRESS, /* TlsRxRekeyInProgress */
LINUX_MIB_TLSTXREKEYABORTED, /* TlsTxRekeyAborted */
+ LINUX_MIB_TLSRXREKEYABORTED, /* TlsRxRekeyAborted */
__LINUX_MIB_TLSMAX
};
diff --git a/net/tls/tls.h b/net/tls/tls.h
index 6e8fafca42ca..0983b29dc46d 100644
--- a/net/tls/tls.h
+++ b/net/tls/tls.h
@@ -241,8 +241,10 @@ void tls_device_cleanup(void);
int tls_set_device_offload(struct sock *sk,
struct tls_crypto_info *crypto_info);
void tls_device_free_resources_tx(struct sock *sk);
-int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx);
+int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx,
+ struct tls_crypto_info *crypto_info);
void tls_device_offload_cleanup_rx(struct sock *sk);
+void tls_device_rx_del_key(struct sock *sk, struct tls_context *ctx);
void tls_device_rx_resync_new_rec(struct sock *sk, u32 rcd_len, u32 seq);
int tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx);
#else
@@ -258,13 +260,16 @@ tls_set_device_offload(struct sock *sk, struct tls_crypto_info *crypto_info)
static inline void tls_device_free_resources_tx(struct sock *sk) {}
static inline int
-tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx)
+tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx,
+ struct tls_crypto_info *crypto_info)
{
return -EOPNOTSUPP;
}
static inline void tls_device_offload_cleanup_rx(struct sock *sk) {}
static inline void
+tls_device_rx_del_key(struct sock *sk, struct tls_context *ctx) {}
+static inline void
tls_device_rx_resync_new_rec(struct sock *sk, u32 rcd_len, u32 seq) {}
static inline int
diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c
index ef837bbd9560..74583433b593 100644
--- a/net/tls/tls_device.c
+++ b/net/tls/tls_device.c
@@ -66,8 +66,14 @@ static void tls_device_free_ctx(struct tls_context *ctx)
kfree(offload_ctx);
}
- if (ctx->rx_conf == TLS_HW)
- kfree(tls_offload_ctx_rx(ctx));
+ if (ctx->rx_conf == TLS_HW) {
+ struct tls_offload_context_rx *offload_ctx =
+ tls_offload_ctx_rx(ctx);
+
+ memzero_explicit(&offload_ctx->rekey,
+ sizeof(offload_ctx->rekey));
+ kfree(offload_ctx);
+ }
tls_ctx_free(NULL, ctx);
}
@@ -185,6 +191,82 @@ static void tls_device_commit_start_marker(struct sock *sk,
tcp_write_collapse_fence(sk);
}
+static int tls_device_dev_add_rx(struct sock *sk, struct tls_context *tls_ctx,
+ struct net_device *netdev,
+ struct tls_crypto_info *crypto_info,
+ u32 cur_seq, bool is_rekey)
+{
+ const struct tls_cipher_desc *cipher_desc;
+ char *rec_seq;
+ int rc;
+
+ cipher_desc = get_cipher_desc(crypto_info->cipher_type);
+ DEBUG_NET_WARN_ON_ONCE(!cipher_desc || !cipher_desc->offloadable);
+
+ rc = netdev->tlsdev_ops->tls_dev_add(netdev, sk,
+ TLS_OFFLOAD_CTX_DIR_RX,
+ crypto_info, cur_seq);
+ rec_seq = crypto_info_rec_seq(crypto_info, cipher_desc);
+ trace_tls_device_offload_set(sk, TLS_OFFLOAD_CTX_DIR_RX,
+ cur_seq, rec_seq, rc);
+ if (!rc) {
+ clear_bit(TLS_RX_DEV_DEGRADED, &tls_ctx->flags);
+ clear_bit(TLS_RX_DEV_CLOSED, &tls_ctx->flags);
+ if (is_rekey)
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYOK);
+ } else if (is_rekey) {
+ set_bit(TLS_RX_DEV_DEGRADED, &tls_ctx->flags);
+ set_bit(TLS_RX_DEV_CLOSED, &tls_ctx->flags);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYFALLBACK);
+ }
+ return rc;
+}
+
+static void tls_device_deferred_dev_add_rx(struct sock *sk,
+ struct tls_context *tls_ctx,
+ struct tls_offload_context_rx *ctx)
+{
+ struct net_device *netdev;
+
+ ctx->dev_add_pending = 0;
+
+ down_read(&device_offload_lock);
+ netdev = rcu_dereference_protected(tls_ctx->netdev,
+ lockdep_is_held(&device_offload_lock));
+ if (netdev)
+ tls_device_dev_add_rx(sk, tls_ctx, netdev,
+ &tls_ctx->crypto_recv.info,
+ tcp_sk(sk)->copied_seq, true);
+ else
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYFALLBACK);
+ up_read(&device_offload_lock);
+ TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYINPROGRESS);
+}
+
+void tls_device_rx_del_key(struct sock *sk, struct tls_context *ctx)
+{
+ struct net_device *netdev;
+
+ if (ctx->rx_conf != TLS_HW)
+ return;
+ if (test_bit(TLS_RX_DEV_CLOSED, &ctx->flags))
+ return;
+
+ down_read(&device_offload_lock);
+ netdev = rcu_dereference_protected(ctx->netdev,
+ lockdep_is_held(&device_offload_lock));
+ if (!netdev) {
+ up_read(&device_offload_lock);
+ return;
+ }
+
+ set_bit(TLS_RX_DEV_CLOSED, &ctx->flags);
+ synchronize_net();
+ netdev->tlsdev_ops->tls_dev_del(netdev, ctx,
+ TLS_OFFLOAD_CTX_DIR_RX);
+ up_read(&device_offload_lock);
+}
+
static void destroy_record(struct tls_record_info *record)
{
int i;
@@ -893,6 +975,8 @@ void tls_device_rx_resync_new_rec(struct sock *sk, u32 rcd_len, u32 seq)
return;
if (unlikely(test_bit(TLS_RX_DEV_DEGRADED, &tls_ctx->flags)))
return;
+ if (unlikely(test_bit(TLS_RX_DEV_CLOSED, &tls_ctx->flags)))
+ return;
prot = &tls_ctx->prot_info;
rx_ctx = tls_offload_ctx_rx(tls_ctx);
@@ -1082,13 +1166,56 @@ tls_device_reencrypt(struct sock *sk, struct tls_context *tls_ctx)
return err;
}
+/*
+ * Temporarily swap in the old key, run
+ * tls_device_reencrypt(), then restore the current key.
+ */
+static int tls_device_reencrypt_old_key(struct sock *sk,
+ struct tls_offload_context_rx *ctx,
+ struct tls_sw_context_rx *sw_ctx,
+ struct tls_context *tls_ctx)
+{
+ struct crypto_aead *saved_aead = sw_ctx->aead_recv;
+ char saved_iv[TLS_MAX_IV_SIZE + TLS_MAX_SALT_SIZE];
+ char saved_rec_seq[TLS_MAX_REC_SEQ_SIZE];
+ int ret;
+
+ memcpy(saved_iv, tls_ctx->rx.iv, sizeof(saved_iv));
+ memcpy(saved_rec_seq, tls_ctx->rx.rec_seq, sizeof(saved_rec_seq));
+
+ sw_ctx->aead_recv = ctx->rekey.old_aead_recv;
+ memcpy(tls_ctx->rx.iv, ctx->rekey.old_iv, sizeof(ctx->rekey.old_iv));
+ memcpy(tls_ctx->rx.rec_seq, ctx->rekey.old_rec_seq,
+ sizeof(ctx->rekey.old_rec_seq));
+
+ ret = tls_device_reencrypt(sk, tls_ctx);
+
+ memcpy(ctx->rekey.old_rec_seq, tls_ctx->rx.rec_seq,
+ sizeof(ctx->rekey.old_rec_seq));
+
+ sw_ctx->aead_recv = saved_aead;
+ memcpy(tls_ctx->rx.iv, saved_iv, sizeof(saved_iv));
+ memcpy(tls_ctx->rx.rec_seq, saved_rec_seq, sizeof(saved_rec_seq));
+
+ if (ret)
+ return ret;
+
+ tls_bigint_increment(ctx->rekey.old_rec_seq,
+ tls_ctx->prot_info.rec_seq_size);
+ ctx->resync_nh_reset = 1;
+
+ return 0;
+}
+
int tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx)
{
struct tls_offload_context_rx *ctx = tls_offload_ctx_rx(tls_ctx);
struct tls_sw_context_rx *sw_ctx = tls_sw_ctx_rx(tls_ctx);
struct sk_buff *skb = tls_strp_msg(sw_ctx);
+ u32 copied_seq = tcp_sk(sk)->copied_seq;
struct strp_msg *rxm = strp_msg(skb);
int is_decrypted, is_encrypted;
+ u32 rec_start_seq;
if (!tls_strp_msg_mixed_decrypted(sw_ctx)) {
is_decrypted = skb->decrypted;
@@ -1098,10 +1225,42 @@ int tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx)
is_encrypted = 0;
}
- trace_tls_device_decrypted(sk, tcp_sk(sk)->copied_seq - rxm->full_len,
+ rec_start_seq = sw_ctx->strp.copy_mode
+ ? copied_seq - rxm->full_len
+ : copied_seq;
+
+ trace_tls_device_decrypted(sk, rec_start_seq,
tls_ctx->rx.rec_seq, rxm->full_len,
is_encrypted, is_decrypted);
+ if (unlikely(ctx->rekey.old_aead_recv)) {
+ bool before_nic_boundary =
+ before(rec_start_seq, ctx->rekey.old_nic_boundary);
+
+ if (before_nic_boundary) {
+ if (is_encrypted) {
+ tls_bigint_increment(ctx->rekey.old_rec_seq,
+ tls_ctx->prot_info.rec_seq_size);
+ return 0;
+ }
+
+ /* rekey_fixup sets decrypted flags in case if NIC clears
+ * decrypted flags on auth failure
+ */
+ if (!is_decrypted && ctx->rekey.rekey_fixup)
+ ctx->rekey.rekey_fixup(skb);
+
+ return tls_device_reencrypt_old_key(sk, ctx,
+ sw_ctx, tls_ctx);
+ }
+
+ crypto_free_aead(ctx->rekey.old_aead_recv);
+ ctx->rekey.old_aead_recv = NULL;
+
+ if (ctx->dev_add_pending)
+ tls_device_deferred_dev_add_rx(sk, tls_ctx, ctx);
+ }
+
if (unlikely(test_bit(TLS_RX_DEV_DEGRADED, &tls_ctx->flags))) {
if (likely(is_encrypted || is_decrypted))
return is_decrypted;
@@ -1580,13 +1739,31 @@ int tls_set_device_offload(struct sock *sk,
return rc;
}
-int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx)
+int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx,
+ struct tls_crypto_info *new_crypto_info)
{
- struct tls12_crypto_info_aes_gcm_128 *info;
+ struct tls_crypto_info *crypto_info, *src_crypto_info;
+ const struct tls_cipher_desc *cipher_desc;
+ u32 copied_seq = tcp_sk(sk)->copied_seq;
struct tls_offload_context_rx *context;
struct net_device *netdev;
+ bool was_dev_add_pending;
+ bool moved_aead_recv = false;
int rc = 0;
+ /* Rekey is only supported for connections that are already
+ * using HW offload. For SW offload connections, the caller
+ * should fall back to tls_set_sw_offload() for rekey.
+ */
+ if (new_crypto_info && ctx->rx_conf != TLS_HW)
+ return -EINVAL;
+
+ crypto_info = &ctx->crypto_recv.info;
+ src_crypto_info = new_crypto_info ?: crypto_info;
+ cipher_desc = get_cipher_desc(src_crypto_info->cipher_type);
+ if (!cipher_desc || !cipher_desc->offloadable)
+ return -EINVAL;
+
netdev = get_netdev_for_sock(sk);
if (!netdev) {
pr_err_ratelimited("%s: netdev not found\n", __func__);
@@ -1612,29 +1789,88 @@ int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx)
goto release_lock;
}
- context = kzalloc_obj(*context);
- if (!context) {
- rc = -ENOMEM;
- goto release_lock;
+ if (!new_crypto_info) {
+ context = kzalloc_obj(*context);
+ if (!context) {
+ rc = -ENOMEM;
+ goto release_lock;
+ }
+ ctx->priv_ctx_rx = context;
+ } else {
+ context = tls_offload_ctx_rx(ctx);
}
+ was_dev_add_pending = context->dev_add_pending;
context->resync_nh_reset = 1;
- ctx->priv_ctx_rx = context;
- rc = tls_sw_ctx_init(sk, 0, NULL);
+ if (new_crypto_info) {
+ struct tls_sw_context_rx *sw_ctx = tls_sw_ctx_rx(ctx);
+
+ if (!test_bit(TLS_RX_DEV_CLOSED, &ctx->flags)) {
+ set_bit(TLS_RX_DEV_CLOSED, &ctx->flags);
+ synchronize_net();
+ netdev->tlsdev_ops->tls_dev_del(netdev, ctx,
+ TLS_OFFLOAD_CTX_DIR_RX);
+ }
+
+ if (context->rekey.old_aead_recv &&
+ before(copied_seq, context->rekey.old_nic_boundary)) {
+ /* Previous rekey still draining. Keep rekey.old_aead_recv,
+ * it is the only key that can undo the NIC-XOR on queued
+ * records. sw_ctx->aead_recv may be re-setkey'd by
+ * tls_sw_ctx_init(); that intermediate key was never on
+ * the NIC and its wire era is drained, so it is needed
+ * for neither undo nor AEAD. Defer dev_add; the new key
+ * is installed once copied_seq crosses rekey.old_nic_boundary.
+ */
+ context->dev_add_pending = 1;
+ } else {
+ u32 rcv_nxt;
+
+ if (context->rekey.old_aead_recv) {
+ crypto_free_aead(context->rekey.old_aead_recv);
+ context->rekey.old_aead_recv = NULL;
+ }
+
+ /* flush the backlog so rcv_nxt is accurate */
+ __sk_flush_backlog(sk);
+ rcv_nxt = tcp_sk(sk)->rcv_nxt;
+
+ if (before(copied_seq, rcv_nxt)) {
+ context->rekey.old_aead_recv = sw_ctx->aead_recv;
+ sw_ctx->aead_recv = NULL;
+ moved_aead_recv = true;
+ memcpy(context->rekey.old_iv, ctx->rx.iv,
+ sizeof(context->rekey.old_iv));
+ memcpy(context->rekey.old_rec_seq, ctx->rx.rec_seq,
+ sizeof(context->rekey.old_rec_seq));
+ context->rekey.old_nic_boundary = rcv_nxt;
+ context->rekey.rekey_fixup =
+ netdev->tlsdev_ops->tls_dev_rx_rekey_fixup;
+ context->dev_add_pending = 1;
+ }
+ }
+ }
+
+ rc = tls_sw_ctx_init(sk, 0, new_crypto_info);
if (rc)
goto release_ctx;
- rc = netdev->tlsdev_ops->tls_dev_add(netdev, sk, TLS_OFFLOAD_CTX_DIR_RX,
- &ctx->crypto_recv.info,
- tcp_sk(sk)->copied_seq);
- info = (void *)&ctx->crypto_recv.info;
- trace_tls_device_offload_set(sk, TLS_OFFLOAD_CTX_DIR_RX,
- tcp_sk(sk)->copied_seq, info->rec_seq, rc);
- if (rc)
- goto free_sw_resources;
+ if (!context->dev_add_pending) {
+ rc = tls_device_dev_add_rx(sk, ctx, netdev, src_crypto_info,
+ copied_seq, !!new_crypto_info);
+ if (!new_crypto_info) {
+ if (rc)
+ goto free_sw_resources;
+ tls_device_attach(ctx, sk, netdev);
+ }
+ } else if (!was_dev_add_pending) {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYINPROGRESS);
+ } else {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYOK);
+ }
+
+ tls_sw_ctx_finalize(sk, 0, new_crypto_info);
- tls_device_attach(ctx, sk, netdev);
- tls_sw_ctx_finalize(sk, 0, NULL);
up_read(&device_offload_lock);
dev_put(netdev);
@@ -1646,7 +1882,18 @@ int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx)
tls_sw_free_resources_rx(sk);
down_read(&device_offload_lock);
release_ctx:
- ctx->priv_ctx_rx = NULL;
+ if (!new_crypto_info) {
+ ctx->priv_ctx_rx = NULL;
+ } else {
+ context->dev_add_pending = was_dev_add_pending;
+ if (moved_aead_recv) {
+ struct tls_sw_context_rx *sw_ctx = tls_sw_ctx_rx(ctx);
+
+ crypto_free_aead(sw_ctx->aead_recv);
+ sw_ctx->aead_recv = context->rekey.old_aead_recv;
+ context->rekey.old_aead_recv = NULL;
+ }
+ }
release_lock:
up_read(&device_offload_lock);
release_netdev:
@@ -1657,6 +1904,7 @@ int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx)
void tls_device_offload_cleanup_rx(struct sock *sk)
{
struct tls_context *tls_ctx = tls_get_ctx(sk);
+ struct tls_offload_context_rx *rx_ctx;
struct net_device *netdev;
down_read(&device_offload_lock);
@@ -1665,8 +1913,9 @@ void tls_device_offload_cleanup_rx(struct sock *sk)
if (!netdev)
goto out;
- netdev->tlsdev_ops->tls_dev_del(netdev, tls_ctx,
- TLS_OFFLOAD_CTX_DIR_RX);
+ if (!test_bit(TLS_RX_DEV_CLOSED, &tls_ctx->flags))
+ netdev->tlsdev_ops->tls_dev_del(netdev, tls_ctx,
+ TLS_OFFLOAD_CTX_DIR_RX);
if (tls_ctx->tx_conf != TLS_HW) {
dev_put(netdev);
@@ -1676,6 +1925,19 @@ void tls_device_offload_cleanup_rx(struct sock *sk)
}
out:
up_read(&device_offload_lock);
+
+ rx_ctx = tls_offload_ctx_rx(tls_ctx);
+ if (rx_ctx && rx_ctx->rekey.old_aead_recv) {
+ crypto_free_aead(rx_ctx->rekey.old_aead_recv);
+ rx_ctx->rekey.old_aead_recv = NULL;
+ }
+
+ if (rx_ctx && rx_ctx->dev_add_pending) {
+ rx_ctx->dev_add_pending = 0;
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYABORTED);
+ TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYINPROGRESS);
+ }
+
tls_sw_release_resources_rx(sk);
}
@@ -1733,9 +1995,11 @@ static int tls_device_down(struct net_device *netdev)
set_bit(TLS_TX_DEV_CLOSED, &ctx->flags);
}
if (ctx->rx_conf == TLS_HW &&
- !test_bit(TLS_RX_DEV_CLOSED, &ctx->flags))
+ !test_bit(TLS_RX_DEV_CLOSED, &ctx->flags)) {
netdev->tlsdev_ops->tls_dev_del(netdev, ctx,
TLS_OFFLOAD_CTX_DIR_RX);
+ set_bit(TLS_RX_DEV_CLOSED, &ctx->flags);
+ }
dev_put(netdev);
diff --git a/net/tls/tls_main.c b/net/tls/tls_main.c
index b51f9ea1ed6c..4519c8943ebf 100644
--- a/net/tls/tls_main.c
+++ b/net/tls/tls_main.c
@@ -757,37 +757,37 @@ static int do_tls_setsockopt_conf(struct sock *sk, sockptr_t optval,
conf = TLS_SW;
}
} else {
- if (update && ctx->rx_conf == TLS_HW) {
- rc = -EOPNOTSUPP;
- goto err_crypto_info;
- }
-
- if (!update) {
- rc = tls_set_device_offload_rx(sk, ctx);
- conf = TLS_HW;
- if (!rc) {
+ rc = tls_set_device_offload_rx(sk, ctx,
+ update ? crypto_info : NULL);
+ conf = TLS_HW;
+ if (!rc) {
+ if (!update) {
TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXDEVICE);
TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXDEVICE);
- tls_sw_strparser_arm(sk, ctx);
- goto out;
}
- }
-
- rc = tls_set_sw_offload(sk, 0, update ? crypto_info : NULL);
- if (rc)
+ } else if (update && ctx->rx_conf == TLS_HW) {
+ /* HW rekey failed - return the actual error.
+ * Cannot fall back to SW for an existing HW connection.
+ */
goto err_crypto_info;
-
- if (update) {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYOK);
} else {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXSW);
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXSW);
- tls_sw_strparser_arm(sk, ctx);
+ rc = tls_set_sw_offload(sk, 0,
+ update ? crypto_info : NULL);
+ if (rc)
+ goto err_crypto_info;
+
+ if (update) {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYOK);
+ } else {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXSW);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXSW);
+ }
+ conf = TLS_SW;
}
- conf = TLS_SW;
+ if (!update)
+ tls_sw_strparser_arm(sk, ctx);
}
-out:
if (tx)
ctx->tx_conf = conf;
else
diff --git a/net/tls/tls_proc.c b/net/tls/tls_proc.c
index adca8561c7da..ae2322481c71 100644
--- a/net/tls/tls_proc.c
+++ b/net/tls/tls_proc.c
@@ -28,8 +28,11 @@ static const struct snmp_mib tls_mib_list[] = {
SNMP_MIB_ITEM("TlsTxRekeyError", LINUX_MIB_TLSTXREKEYERROR),
SNMP_MIB_ITEM("TlsRxRekeyReceived", LINUX_MIB_TLSRXREKEYRECEIVED),
SNMP_MIB_ITEM("TlsTxRekeyFallback", LINUX_MIB_TLSTXREKEYFALLBACK),
+ SNMP_MIB_ITEM("TlsRxRekeyFallback", LINUX_MIB_TLSRXREKEYFALLBACK),
SNMP_MIB_ITEM("TlsTxRekeyInProgress", LINUX_MIB_TLSTXREKEYINPROGRESS),
+ SNMP_MIB_ITEM("TlsRxRekeyInProgress", LINUX_MIB_TLSRXREKEYINPROGRESS),
SNMP_MIB_ITEM("TlsTxRekeyAborted", LINUX_MIB_TLSTXREKEYABORTED),
+ SNMP_MIB_ITEM("TlsRxRekeyAborted", LINUX_MIB_TLSRXREKEYABORTED),
};
static int tls_statistics_seq_show(struct seq_file *seq, void *v)
diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c
index 0364fae8a7e0..445517d8bc90 100644
--- a/net/tls/tls_sw.c
+++ b/net/tls/tls_sw.c
@@ -1534,6 +1534,7 @@ static int tls_check_pending_rekey(struct sock *sk, struct tls_context *ctx,
if (hs_type == TLS_HANDSHAKE_KEYUPDATE) {
struct tls_sw_context_rx *rx_ctx = ctx->priv_ctx_rx;
+ tls_device_rx_del_key(sk, ctx);
WRITE_ONCE(rx_ctx->key_update_pending, true);
TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYRECEIVED);
}
--
2.25.1
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH v15 8/9] tls: device: add tracepoints for the KeyUpdate path
2026-07-09 20:53 [PATCH net-next v15 0/9] tls: Add TLS 1.3 hardware offload support Rishikesh Jethwani
` (6 preceding siblings ...)
2026-07-09 20:53 ` [PATCH v15 7/9] tls: device: add RX " Rishikesh Jethwani
@ 2026-07-09 20:53 ` Rishikesh Jethwani
2026-07-09 20:53 ` [PATCH v15 9/9] selftests: net: add TLS hardware offload test Rishikesh Jethwani
8 siblings, 0 replies; 10+ messages in thread
From: Rishikesh Jethwani @ 2026-07-09 20:53 UTC (permalink / raw)
To: netdev
Cc: saeedm, tariqt, mbloch, borisp, john.fastabend, kuba, sd, davem,
pabeni, edumazet, leon, andrew.gospodarek, Rishikesh Jethwani
Add four trace events covering the rekey state machine in
tls_device.c:
tls_device_rekey_start - rekey accepted; inflight=1 means old-key
data is still queued, dev_add deferred
tls_device_rekey_reencrypt - old-key undo pass for a boundary record
tls_device_rekey_done - boundary crossed, old_aead_recv freed,
deferred dev_add issued if pending
tls_device_complete_rekey_fail - TX rekey completion failed in sendmsg;
READY is left set and the next sendmsg
retries
Signed-off-by: Rishikesh Jethwani <rjethwani@purestorage.com>
---
net/tls/tls_device.c | 17 +++++++-
net/tls/trace.h | 98 ++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 113 insertions(+), 2 deletions(-)
diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c
index 74583433b593..aa4ef04e9c43 100644
--- a/net/tls/tls_device.c
+++ b/net/tls/tls_device.c
@@ -761,8 +761,14 @@ int tls_device_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
}
/* Old-key records all ACKed; switch back to HW. */
- if (test_bit(TLS_TX_REKEY_READY, &tls_ctx->flags))
- tls_device_complete_rekey(sk, tls_ctx, true);
+ if (test_bit(TLS_TX_REKEY_READY, &tls_ctx->flags)) {
+ rc = tls_device_complete_rekey(sk, tls_ctx, true);
+ /* On failure the READY bit is left set; the next sendmsg
+ * retries.
+ */
+ if (rc)
+ trace_tls_device_complete_rekey_fail(sk, rc);
+ }
/* Use SW path if rekey is in progress (PENDING) or if HW rekey
* failed (FAILED).
@@ -1244,6 +1250,9 @@ int tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx)
return 0;
}
+ trace_tls_device_rekey_reencrypt(sk, rec_start_seq,
+ ctx->rekey.old_nic_boundary);
+
/* rekey_fixup sets decrypted flags in case if NIC clears
* decrypted flags on auth failure
*/
@@ -1254,6 +1263,8 @@ int tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx)
sw_ctx, tls_ctx);
}
+ trace_tls_device_rekey_done(sk, rec_start_seq,
+ ctx->rekey.old_nic_boundary);
crypto_free_aead(ctx->rekey.old_aead_recv);
ctx->rekey.old_aead_recv = NULL;
@@ -1848,6 +1859,8 @@ int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx,
netdev->tlsdev_ops->tls_dev_rx_rekey_fixup;
context->dev_add_pending = 1;
}
+ trace_tls_device_rekey_start(sk, copied_seq, rcv_nxt,
+ before(copied_seq, rcv_nxt));
}
}
diff --git a/net/tls/trace.h b/net/tls/trace.h
index 2d8ce4ff3265..2a90b77d75e8 100644
--- a/net/tls/trace.h
+++ b/net/tls/trace.h
@@ -192,6 +192,104 @@ TRACE_EVENT(tls_device_tx_resync_send,
)
);
+TRACE_EVENT(tls_device_rekey_start,
+
+ TP_PROTO(struct sock *sk, u32 copied_seq, u32 nic_boundary,
+ bool inflight),
+
+ TP_ARGS(sk, copied_seq, nic_boundary, inflight),
+
+ TP_STRUCT__entry(
+ __field( struct sock *, sk )
+ __field( u32, copied_seq )
+ __field( u32, nic_boundary )
+ __field( bool, inflight )
+ ),
+
+ TP_fast_assign(
+ __entry->sk = sk;
+ __entry->copied_seq = copied_seq;
+ __entry->nic_boundary = nic_boundary;
+ __entry->inflight = inflight;
+ ),
+
+ TP_printk(
+ "sk=%p copied_seq=%u nic_boundary=%u inflight=%d",
+ __entry->sk, __entry->copied_seq, __entry->nic_boundary,
+ __entry->inflight
+ )
+);
+
+TRACE_EVENT(tls_device_rekey_reencrypt,
+
+ TP_PROTO(struct sock *sk, u32 tcp_seq, u32 nic_boundary),
+
+ TP_ARGS(sk, tcp_seq, nic_boundary),
+
+ TP_STRUCT__entry(
+ __field( struct sock *, sk )
+ __field( u32, tcp_seq )
+ __field( u32, nic_boundary )
+ ),
+
+ TP_fast_assign(
+ __entry->sk = sk;
+ __entry->tcp_seq = tcp_seq;
+ __entry->nic_boundary = nic_boundary;
+ ),
+
+ TP_printk(
+ "sk=%p tcp_seq=%u nic_boundary=%u",
+ __entry->sk, __entry->tcp_seq, __entry->nic_boundary
+ )
+);
+
+TRACE_EVENT(tls_device_rekey_done,
+
+ TP_PROTO(struct sock *sk, u32 tcp_seq, u32 nic_boundary),
+
+ TP_ARGS(sk, tcp_seq, nic_boundary),
+
+ TP_STRUCT__entry(
+ __field( struct sock *, sk )
+ __field( u32, tcp_seq )
+ __field( u32, nic_boundary )
+ ),
+
+ TP_fast_assign(
+ __entry->sk = sk;
+ __entry->tcp_seq = tcp_seq;
+ __entry->nic_boundary = nic_boundary;
+ ),
+
+ TP_printk(
+ "sk=%p tcp_seq=%u nic_boundary=%u",
+ __entry->sk, __entry->tcp_seq, __entry->nic_boundary
+ )
+);
+
+TRACE_EVENT(tls_device_complete_rekey_fail,
+
+ TP_PROTO(struct sock *sk, int rc),
+
+ TP_ARGS(sk, rc),
+
+ TP_STRUCT__entry(
+ __field( struct sock *, sk )
+ __field( int, rc )
+ ),
+
+ TP_fast_assign(
+ __entry->sk = sk;
+ __entry->rc = rc;
+ ),
+
+ TP_printk(
+ "sk=%p rc=%d",
+ __entry->sk, __entry->rc
+ )
+);
+
#endif /* _TLS_TRACE_H_ */
#undef TRACE_INCLUDE_PATH
--
2.25.1
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH v15 9/9] selftests: net: add TLS hardware offload test
2026-07-09 20:53 [PATCH net-next v15 0/9] tls: Add TLS 1.3 hardware offload support Rishikesh Jethwani
` (7 preceding siblings ...)
2026-07-09 20:53 ` [PATCH v15 8/9] tls: device: add tracepoints for the KeyUpdate path Rishikesh Jethwani
@ 2026-07-09 20:53 ` Rishikesh Jethwani
8 siblings, 0 replies; 10+ messages in thread
From: Rishikesh Jethwani @ 2026-07-09 20:53 UTC (permalink / raw)
To: netdev
Cc: saeedm, tariqt, mbloch, borisp, john.fastabend, kuba, sd, davem,
pabeni, edumazet, leon, andrew.gospodarek, Rishikesh Jethwani
Two-node kTLS HW offload test using NetDrvEpEnv. A C helper binary
acts as TLS client or server; a Python harness drives it and verifies
TLS stat counters (RekeyOk, RekeyReceived, RekeyFallback,
RekeyInProgress, RekeyAborted, RekeyError, DecryptError).
Covers TLS 1.2/1.3 with AES-GCM-128/256, rekey with various buffer
sizes, and burst variants that stress TX rekey (temporary SW phase,
HW reinstall) and RX rekey (boundary tracking, old-key reencryption,
deferred dev_add).
Signed-off-by: Rishikesh Jethwani <rjethwani@purestorage.com>
---
MAINTAINERS | 2 +
.../selftests/drivers/net/hw/.gitignore | 1 +
.../testing/selftests/drivers/net/hw/Makefile | 2 +
.../selftests/drivers/net/hw/tls_hw_offload.c | 975 ++++++++++++++++++
.../drivers/net/hw/tls_hw_offload.py | 295 ++++++
5 files changed, 1275 insertions(+)
create mode 100644 tools/testing/selftests/drivers/net/hw/tls_hw_offload.c
create mode 100755 tools/testing/selftests/drivers/net/hw/tls_hw_offload.py
diff --git a/MAINTAINERS b/MAINTAINERS
index 52f1a55eca99..d132681f07dc 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -19047,6 +19047,8 @@ F: Documentation/networking/tls*
F: include/net/tls.h
F: include/uapi/linux/tls.h
F: net/tls/
+F: tools/testing/selftests/drivers/net/hw/tls_hw_offload.c
+F: tools/testing/selftests/drivers/net/hw/tls_hw_offload.py
F: tools/testing/selftests/net/tls.c
NETWORKING [SOCKETS]
diff --git a/tools/testing/selftests/drivers/net/hw/.gitignore b/tools/testing/selftests/drivers/net/hw/.gitignore
index 46540468a775..f0a5d15b469b 100644
--- a/tools/testing/selftests/drivers/net/hw/.gitignore
+++ b/tools/testing/selftests/drivers/net/hw/.gitignore
@@ -2,3 +2,4 @@
iou-zcrx
ncdevmem
toeplitz
+tls_hw_offload
diff --git a/tools/testing/selftests/drivers/net/hw/Makefile b/tools/testing/selftests/drivers/net/hw/Makefile
index 234db5c2c90c..c274c41f97e8 100644
--- a/tools/testing/selftests/drivers/net/hw/Makefile
+++ b/tools/testing/selftests/drivers/net/hw/Makefile
@@ -15,6 +15,7 @@ endif
TEST_GEN_FILES := \
$(COND_GEN_FILES) \
+ tls_hw_offload \
# end of TEST_GEN_FILES
TEST_PROGS = \
@@ -46,6 +47,7 @@ TEST_PROGS = \
rss_drv.py \
rss_flow_label.py \
rss_input_xfrm.py \
+ tls_hw_offload.py \
toeplitz.py \
tso.py \
userns_devmem.py \
diff --git a/tools/testing/selftests/drivers/net/hw/tls_hw_offload.c b/tools/testing/selftests/drivers/net/hw/tls_hw_offload.c
new file mode 100644
index 000000000000..1882e2f073ad
--- /dev/null
+++ b/tools/testing/selftests/drivers/net/hw/tls_hw_offload.c
@@ -0,0 +1,975 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * TLS Hardware Offload Two-Node Test
+ *
+ * Tests kTLS hardware offload between two physical nodes using
+ * hardcoded keys. Supports TLS 1.2/1.3, AES-GCM-128/256, and rekey.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <limits.h>
+#include <time.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+#include <arpa/inet.h>
+#include <linux/tls.h>
+
+#define TLS_RECORD_TYPE_HANDSHAKE 22
+#define TLS_HANDSHAKE_KEY_UPDATE 0x18
+
+/* Large enough for a TLS 1.3 KeyUpdate handshake record's plaintext. */
+#define MIN_BUF_SIZE 16
+
+/* Initial key material */
+static struct tls12_crypto_info_aes_gcm_128 tls_info_key0_128 = {
+ .info = {
+ .version = TLS_1_3_VERSION,
+ .cipher_type = TLS_CIPHER_AES_GCM_128,
+ },
+ .iv = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08 },
+ .key = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
+ 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10 },
+ .salt = { 0x01, 0x02, 0x03, 0x04 },
+ .rec_seq = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+};
+
+static struct tls12_crypto_info_aes_gcm_256 tls_info_key0_256 = {
+ .info = {
+ .version = TLS_1_3_VERSION,
+ .cipher_type = TLS_CIPHER_AES_GCM_256,
+ },
+ .iv = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08 },
+ .key = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
+ 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10,
+ 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18,
+ 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20 },
+ .salt = { 0x01, 0x02, 0x03, 0x04 },
+ .rec_seq = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+};
+
+static int num_rekeys;
+static int num_iterations = 100;
+static int cipher_type = TLS_CIPHER_AES_GCM_128;
+static int tls_version = TLS_1_3_VERSION;
+static int server_port = 4433;
+static char *server_ip;
+
+static int send_size = 16384;
+static int random_size_max;
+/* Burst mode: sender keeps pushing records without reading from the peer;
+ * receiver drains without echoing back. Only the client initiates rekey.
+ */
+static int burst_mode;
+static int zc_rx;
+
+/* XOR each byte with the generation so both endpoints derive the
+ * same per-generation key without a real KDF. Generation 0 leaves
+ * the base key unchanged.
+ */
+static void derive_key_fields(unsigned char *key, int key_size,
+ unsigned char *iv, int iv_size,
+ unsigned char *salt, int salt_size,
+ unsigned char *rec_seq, int rec_seq_size,
+ int generation)
+{
+ int i;
+
+ for (i = 0; i < key_size; i++)
+ key[i] ^= generation;
+ for (i = 0; i < iv_size; i++)
+ iv[i] ^= generation;
+ for (i = 0; i < salt_size; i++)
+ salt[i] ^= generation;
+ memset(rec_seq, 0, rec_seq_size);
+}
+
+static void derive_key_128(struct tls12_crypto_info_aes_gcm_128 *key,
+ int generation)
+{
+ memcpy(key, &tls_info_key0_128, sizeof(*key));
+ key->info.version = tls_version;
+ derive_key_fields(key->key, TLS_CIPHER_AES_GCM_128_KEY_SIZE,
+ key->iv, TLS_CIPHER_AES_GCM_128_IV_SIZE,
+ key->salt, TLS_CIPHER_AES_GCM_128_SALT_SIZE,
+ key->rec_seq, TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE,
+ generation);
+}
+
+static void derive_key_256(struct tls12_crypto_info_aes_gcm_256 *key,
+ int generation)
+{
+ memcpy(key, &tls_info_key0_256, sizeof(*key));
+ key->info.version = tls_version;
+ derive_key_fields(key->key, TLS_CIPHER_AES_GCM_256_KEY_SIZE,
+ key->iv, TLS_CIPHER_AES_GCM_256_IV_SIZE,
+ key->salt, TLS_CIPHER_AES_GCM_256_SALT_SIZE,
+ key->rec_seq, TLS_CIPHER_AES_GCM_256_REC_SEQ_SIZE,
+ generation);
+}
+
+static const char *cipher_name(int cipher)
+{
+ switch (cipher) {
+ case TLS_CIPHER_AES_GCM_128: return "AES-GCM-128";
+ case TLS_CIPHER_AES_GCM_256: return "AES-GCM-256";
+ default: return "unknown";
+ }
+}
+
+static const char *version_name(int version)
+{
+ switch (version) {
+ case TLS_1_2_VERSION: return "TLS 1.2";
+ case TLS_1_3_VERSION: return "TLS 1.3";
+ default: return "unknown";
+ }
+}
+
+static int setup_tls_ulp(int fd)
+{
+ int ret;
+
+ ret = setsockopt(fd, IPPROTO_TCP, TCP_ULP, "tls", sizeof("tls"));
+ if (ret < 0) {
+ printf("SETUP ERROR: TCP_ULP failed: %s\n", strerror(errno));
+ return -1;
+ }
+ return 0;
+}
+
+static int set_zc_rx(int fd)
+{
+ int val = 1;
+
+ if (setsockopt(fd, SOL_TLS, TLS_RX_EXPECT_NO_PAD, &val,
+ sizeof(val)) < 0) {
+ printf("SETUP ERROR: TLS_RX_EXPECT_NO_PAD failed: %s\n",
+ strerror(errno));
+ return -1;
+ }
+ return 0;
+}
+
+/* Send a TLS 1.3 KeyUpdate handshake record. The kernel only
+ * inspects the HandshakeType byte to detect KeyUpdate, so don't
+ * bother with the 3-byte length or request_update fields.
+ */
+static int send_tls_key_update(int fd)
+{
+ char cmsg_buf[CMSG_SPACE(sizeof(unsigned char))];
+ unsigned char key_update_msg = TLS_HANDSHAKE_KEY_UPDATE;
+ struct msghdr msg = {0};
+ struct cmsghdr *cmsg;
+ struct iovec iov;
+
+ iov.iov_base = &key_update_msg;
+ iov.iov_len = sizeof(key_update_msg);
+
+ msg.msg_iov = &iov;
+ msg.msg_iovlen = 1;
+ msg.msg_control = cmsg_buf;
+ msg.msg_controllen = sizeof(cmsg_buf);
+
+ cmsg = CMSG_FIRSTHDR(&msg);
+ cmsg->cmsg_level = SOL_TLS;
+ cmsg->cmsg_type = TLS_SET_RECORD_TYPE;
+ cmsg->cmsg_len = CMSG_LEN(sizeof(unsigned char));
+ *CMSG_DATA(cmsg) = TLS_RECORD_TYPE_HANDSHAKE;
+ msg.msg_controllen = cmsg->cmsg_len;
+
+ if (sendmsg(fd, &msg, 0) < 0) {
+ printf("sendmsg KeyUpdate failed: %s\n", strerror(errno));
+ return -1;
+ }
+
+ printf("Sent TLS KeyUpdate handshake message\n");
+ return 0;
+}
+
+static int recv_tls_message(int fd, char *buf, size_t buflen, int *record_type,
+ int flags)
+{
+ char cmsg_buf[CMSG_SPACE(sizeof(unsigned char))];
+ struct msghdr msg = {0};
+ struct cmsghdr *cmsg;
+ struct iovec iov;
+ int ret;
+
+ iov.iov_base = buf;
+ iov.iov_len = buflen;
+
+ msg.msg_iov = &iov;
+ msg.msg_iovlen = 1;
+ msg.msg_control = cmsg_buf;
+ msg.msg_controllen = sizeof(cmsg_buf);
+
+ ret = recvmsg(fd, &msg, flags);
+ if (ret <= 0)
+ return ret;
+
+ cmsg = CMSG_FIRSTHDR(&msg);
+ if (cmsg && cmsg->cmsg_level == SOL_TLS &&
+ cmsg->cmsg_type == TLS_GET_RECORD_TYPE)
+ *record_type = *((unsigned char *)CMSG_DATA(cmsg));
+
+ return ret;
+}
+
+/* Confirm a handshake record starting with HandshakeType KeyUpdate. */
+static int check_keyupdate(const char *buf, int len, int record_type)
+{
+ if (record_type != TLS_RECORD_TYPE_HANDSHAKE) {
+ printf("Expected handshake record (0x%02x), got 0x%02x\n",
+ TLS_RECORD_TYPE_HANDSHAKE, record_type);
+ return -1;
+ }
+ if (len < 1 || (unsigned char)buf[0] != TLS_HANDSHAKE_KEY_UPDATE) {
+ printf("Expected KeyUpdate (0x%02x), got 0x%02x\n",
+ TLS_HANDSHAKE_KEY_UPDATE,
+ len ? (unsigned char)buf[0] : 0);
+ return -1;
+ }
+ printf("Received TLS KeyUpdate\n");
+ return 0;
+}
+
+static int recv_tls_keyupdate(int fd)
+{
+ char buf[MIN_BUF_SIZE];
+ int record_type = 0;
+ int ret;
+
+ ret = recv_tls_message(fd, buf, sizeof(buf), &record_type, 0);
+ if (ret < 0) {
+ printf("recv_tls_message failed: %s\n", strerror(errno));
+ return -1;
+ }
+
+ return check_keyupdate(buf, ret, record_type);
+}
+
+static int check_ekeyexpired(int fd)
+{
+ char buf[MIN_BUF_SIZE];
+ int ret;
+
+ ret = recv(fd, buf, sizeof(buf), MSG_DONTWAIT);
+ if (ret == -1 && errno == EKEYEXPIRED) {
+ printf("recv() returned EKEYEXPIRED as expected\n");
+ return 0;
+ } else if (ret == -1 && errno == EAGAIN) {
+ printf("recv() returned EAGAIN (no pending data)\n");
+ return 0;
+ } else if (ret > 0) {
+ printf("FAIL: recv() returned %d bytes, expected EKEYEXPIRED\n",
+ ret);
+ return -1;
+ } else {
+ printf("FAIL: recv() returned unexpected error: %s\n",
+ strerror(errno));
+ return -1;
+ }
+}
+
+static int do_tls_rekey(int fd, int direction, int generation, int cipher)
+{
+ const char *dir = direction == TLS_TX ? "TX" : "RX";
+ int ret;
+
+ printf("%s TLS_%s %s gen %d...\n",
+ generation ? "Rekeying" : "Installing",
+ dir, cipher_name(cipher), generation);
+
+ if (cipher == TLS_CIPHER_AES_GCM_256) {
+ struct tls12_crypto_info_aes_gcm_256 key;
+
+ derive_key_256(&key, generation);
+ ret = setsockopt(fd, SOL_TLS, direction, &key, sizeof(key));
+ } else {
+ struct tls12_crypto_info_aes_gcm_128 key;
+
+ derive_key_128(&key, generation);
+ ret = setsockopt(fd, SOL_TLS, direction, &key, sizeof(key));
+ }
+
+ if (ret < 0) {
+ printf("%sTLS_%s %s gen %d failed: %s\n",
+ generation ? "" : "SETUP ERROR: ", dir,
+ cipher_name(cipher), generation, strerror(errno));
+ return -1;
+ }
+ printf("TLS_%s %s gen %d installed\n",
+ dir, cipher_name(cipher), generation);
+ return 0;
+}
+
+/* Open a TCP connection to server_ip:server_port, switch to the TLS
+ * ULP, and install initial generation-0 TX/RX keys. Returns the fd on
+ * success, -1 on error (with the fd already closed).
+ */
+static int client_connect_tls(void)
+{
+ struct sockaddr_in sa;
+ int csk;
+
+ csk = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
+ if (csk < 0) {
+ printf("SETUP ERROR: failed to create socket: %s\n",
+ strerror(errno));
+ return -1;
+ }
+
+ memset(&sa, 0, sizeof(sa));
+ sa.sin_family = AF_INET;
+ sa.sin_addr.s_addr = inet_addr(server_ip);
+ sa.sin_port = htons(server_port);
+ printf("Connecting to %s:%d...\n", server_ip, server_port);
+
+ if (connect(csk, (struct sockaddr *)&sa, sizeof(sa)) < 0) {
+ printf("SETUP ERROR: connect failed: %s\n", strerror(errno));
+ goto err;
+ }
+ printf("Connected!\n");
+
+ if (setup_tls_ulp(csk) < 0)
+ goto err;
+
+ if (do_tls_rekey(csk, TLS_TX, 0, cipher_type) < 0 ||
+ do_tls_rekey(csk, TLS_RX, 0, cipher_type) < 0)
+ goto err;
+
+ return csk;
+err:
+ close(csk);
+ return -1;
+}
+
+/* Drain `len` echoed bytes from the server and verify they match the
+ * payload we just sent.
+ */
+static int client_recv_echo(int fd, const char *sent, char *echo_buf,
+ ssize_t len)
+{
+ ssize_t total = 0;
+ ssize_t n;
+
+ while (total < len) {
+ n = recv(fd, echo_buf + total, len - total, 0);
+ if (n < 0) {
+ printf("FAIL: Echo recv failed: %s\n", strerror(errno));
+ return -1;
+ }
+ if (n == 0) {
+ printf("FAIL: Connection closed during echo\n");
+ return -1;
+ }
+ total += n;
+ }
+
+ if (memcmp(sent, echo_buf, len) != 0) {
+ printf("FAIL: Echo data mismatch!\n");
+ return -1;
+ }
+ printf("Received echo %zd bytes (ok)\n", total);
+ return 0;
+}
+
+/* Client side of a rekey: send KeyUpdate and rotate TX. In echo mode
+ * also wait for the peer's KeyUpdate and rotate RX.
+ */
+static int client_rekey(int fd, int generation)
+{
+ if (send_tls_key_update(fd) < 0) {
+ printf("FAIL: send KeyUpdate\n");
+ return -1;
+ }
+
+ if (do_tls_rekey(fd, TLS_TX, generation, cipher_type) < 0)
+ return -1;
+
+ if (burst_mode)
+ return 0;
+
+ if (recv_tls_keyupdate(fd) < 0) {
+ printf("FAIL: recv KeyUpdate from server\n");
+ return -1;
+ }
+
+ if (check_ekeyexpired(fd) < 0)
+ return -1;
+
+ return do_tls_rekey(fd, TLS_RX, generation, cipher_type);
+}
+
+static int do_client(void)
+{
+ char *buf = NULL, *echo_buf = NULL;
+ int max_size, rekey_interval;
+ int csk = -1, i;
+ int test_result = -1;
+ int current_gen = 0;
+ int next_rekey_at;
+ ssize_t n;
+
+ max_size = random_size_max > 0 ? random_size_max : send_size;
+ if (max_size < MIN_BUF_SIZE)
+ max_size = MIN_BUF_SIZE;
+ buf = malloc(max_size);
+ if (!burst_mode)
+ echo_buf = malloc(max_size);
+ if (!buf || (!burst_mode && !echo_buf)) {
+ printf("SETUP ERROR: failed to allocate buffers\n");
+ goto out;
+ }
+
+ csk = client_connect_tls();
+ if (csk < 0)
+ goto out;
+
+ if (num_rekeys)
+ printf("TLS %s setup complete. Will perform %d rekey(s).\n",
+ cipher_name(cipher_type), num_rekeys);
+ else
+ printf("TLS setup complete.\n");
+
+ if (random_size_max > 0)
+ printf("Sending %d messages of random size (1..%d bytes)...\n",
+ num_iterations, random_size_max);
+ else
+ printf("Sending %d messages of %d bytes...\n",
+ num_iterations, send_size);
+
+ rekey_interval = num_iterations / (num_rekeys + 1);
+ next_rekey_at = rekey_interval;
+
+ for (i = 1; i <= num_iterations; i++) {
+ int this_size;
+
+ if (random_size_max > 0)
+ this_size = (rand() % random_size_max) + 1;
+ else
+ this_size = send_size;
+
+ /* In burst mode, use a per-iteration fill pattern so the
+ * receiver can detect any plaintext corruption without a
+ * round-trip echo.
+ */
+ if (burst_mode) {
+ memset(buf, i & 0xFF, this_size);
+ } else {
+ int j;
+
+ for (j = 0; j < this_size; j++)
+ buf[j] = rand() & 0xFF;
+ }
+
+ n = send(csk, buf, this_size, 0);
+ if (n != this_size) {
+ printf("FAIL: send failed: %s\n", strerror(errno));
+ goto out;
+ }
+
+ if (!burst_mode) {
+ printf("Sent %zd bytes (iteration %d)\n", n, i);
+ if (client_recv_echo(csk, buf, echo_buf, n) < 0)
+ goto out;
+ }
+
+ /* Rekey at intervals. In echo mode this is a full bidirectional
+ * exchange; in burst mode the client only rotates its TX key
+ * and sends KeyUpdate - the peer is expected to follow.
+ */
+ if (num_rekeys && current_gen < num_rekeys &&
+ i == next_rekey_at) {
+ current_gen++;
+ printf("\n=== Client Rekey gen %d ===\n", current_gen);
+
+ if (client_rekey(csk, current_gen) < 0)
+ goto out;
+
+ next_rekey_at += rekey_interval;
+ printf("=== Client Rekey gen %d Complete ===\n\n",
+ current_gen);
+ }
+ }
+
+ test_result = 0;
+out:
+ if (num_rekeys)
+ printf("Rekeys completed: %d/%d\n", current_gen, num_rekeys);
+ if (csk >= 0)
+ close(csk);
+ free(buf);
+ free(echo_buf);
+ return test_result;
+}
+
+/* Bind/listen on server_port, accept one client, switch to the TLS ULP
+ * and install initial generation-0 keys (plus zc_rx if requested).
+ * Returns the connected fd on success and writes the listener fd to
+ * *lsk_out so the caller can close it. Returns -1 on error, with all
+ * intermediate fds already closed and *lsk_out left at -1.
+ */
+static int server_accept_tls(int *lsk_out)
+{
+ int lsk, csk, one = 1;
+ struct sockaddr_in sa;
+
+ *lsk_out = -1;
+
+ lsk = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
+ if (lsk < 0) {
+ printf("SETUP ERROR: failed to create socket: %s\n",
+ strerror(errno));
+ return -1;
+ }
+
+ setsockopt(lsk, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one));
+
+ memset(&sa, 0, sizeof(sa));
+ sa.sin_family = AF_INET;
+ sa.sin_addr.s_addr = INADDR_ANY;
+ sa.sin_port = htons(server_port);
+
+ if (bind(lsk, (struct sockaddr *)&sa, sizeof(sa)) < 0) {
+ printf("SETUP ERROR: bind failed: %s\n", strerror(errno));
+ close(lsk);
+ return -1;
+ }
+
+ if (listen(lsk, 1) < 0) {
+ printf("SETUP ERROR: listen failed: %s\n", strerror(errno));
+ close(lsk);
+ return -1;
+ }
+
+ printf("Server listening on 0.0.0.0:%d\n", server_port);
+ printf("Waiting for client connection...\n");
+
+ csk = accept(lsk, (struct sockaddr *)NULL, (socklen_t *)NULL);
+ if (csk < 0) {
+ printf("SETUP ERROR: accept failed: %s\n", strerror(errno));
+ close(lsk);
+ return -1;
+ }
+ printf("Client connected!\n");
+
+ if (setup_tls_ulp(csk) < 0)
+ goto err;
+
+ if (do_tls_rekey(csk, TLS_TX, 0, cipher_type) < 0 ||
+ do_tls_rekey(csk, TLS_RX, 0, cipher_type) < 0)
+ goto err;
+
+ if (zc_rx && set_zc_rx(csk) < 0)
+ goto err;
+
+ *lsk_out = lsk;
+ return csk;
+err:
+ close(csk);
+ close(lsk);
+ return -1;
+}
+
+/* Server side of a rekey: drain any in-flight ciphertext that hit
+ * EKEYEXPIRED and rotate RX. In echo mode also send a KeyUpdate back
+ * and rotate TX.
+ */
+static int server_rekey(int fd, int generation)
+{
+ if (check_ekeyexpired(fd) < 0)
+ return -1;
+
+ if (do_tls_rekey(fd, TLS_RX, generation, cipher_type) < 0)
+ return -1;
+
+ if (burst_mode)
+ return 0;
+
+ if (send_tls_key_update(fd) < 0) {
+ printf("FAIL: send KeyUpdate\n");
+ return -1;
+ }
+
+ return do_tls_rekey(fd, TLS_TX, generation, cipher_type);
+}
+
+/* Burst mode: MSG_WAITALL gives us exactly one iteration's payload,
+ * filled with (send_iter & 0xff). Catches decrypt-succeeded-but-
+ * plaintext-corrupt bugs that AEAD counters alone would miss.
+ */
+static int server_verify_burst(int fd, char *buf, int buf_size,
+ ssize_t n, int send_iter)
+{
+ unsigned char expect = send_iter & 0xFF;
+ int j;
+
+ if (n != send_size) {
+ int record_type = 0;
+ ssize_t n2;
+
+ /* MSG_WAITALL returned short; probe one more recv to
+ * surface any pending socket error for the diagnostic.
+ */
+ n2 = recv_tls_message(fd, buf, buf_size, &record_type, 0);
+ printf("FAIL: short recv in burst mode: got %zd, expected %d (iter %d)\n",
+ n, send_size, send_iter);
+ printf(" follow-up recv: %zd errno=%s\n",
+ n2, n2 < 0 ? strerror(errno) : "ok");
+ return -1;
+ }
+
+ for (j = 0; j < n; j++) {
+ if ((unsigned char)buf[j] != expect) {
+ printf("FAIL: data mismatch iter %d offset %d: expected 0x%02x got 0x%02x\n",
+ send_iter, j, expect, (unsigned char)buf[j]);
+ return -1;
+ }
+ }
+ return 0;
+}
+
+static int server_echo_send(int fd, const char *buf, ssize_t n)
+{
+ ssize_t sent;
+ int ret;
+
+ for (sent = 0; sent < n; sent += ret) {
+ ret = send(fd, buf + sent, n - sent, 0);
+ if (ret < 0) {
+ printf("FAIL: Echo send failed: %s\n", strerror(errno));
+ return -1;
+ }
+ }
+ return 0;
+}
+
+static int do_server(void)
+{
+ int lsk = -1, csk = -1;
+ ssize_t n, total = 0;
+ int test_result = -1;
+ int current_gen = 0;
+ int recv_count = 0;
+ int send_iter = 1;
+ char *buf = NULL;
+ int record_type = 0;
+ int recv_flags;
+ int buf_size;
+
+ buf_size = send_size;
+ if (buf_size < MIN_BUF_SIZE)
+ buf_size = MIN_BUF_SIZE;
+ buf = malloc(buf_size);
+ if (!buf) {
+ printf("SETUP ERROR: failed to allocate buffer\n");
+ goto out;
+ }
+
+ csk = server_accept_tls(&lsk);
+ if (csk < 0)
+ goto out;
+
+ printf("TLS %s setup complete. Receiving...\n",
+ cipher_name(cipher_type));
+
+ /* Burst mode: ask for a full iteration's worth of plaintext per
+ * recv. kTLS accumulates across data records when MSG_WAITALL is
+ * set (target == len), and breaks cleanly at control records, so
+ * each recv returns exactly send_size data bytes or a small KU.
+ */
+ recv_flags = burst_mode ? MSG_WAITALL : 0;
+
+ /* Main receive loop */
+ while (1) {
+ n = recv_tls_message(csk, buf, buf_size, &record_type,
+ recv_flags);
+ if (n == 0) {
+ printf("Connection closed by client\n");
+ break;
+ }
+ if (n < 0) {
+ printf("FAIL: recv failed: %s\n", strerror(errno));
+ goto out;
+ }
+
+ /* Handle KeyUpdate. In echo mode the server mirrors the
+ * rekey back to the peer; in burst mode it only rotates
+ * its RX key and keeps draining.
+ */
+ if (record_type == TLS_RECORD_TYPE_HANDSHAKE) {
+ if (check_keyupdate(buf, n, record_type) < 0)
+ goto out;
+ current_gen++;
+ printf("\n=== Server Rekey gen %d ===\n", current_gen);
+
+ if (server_rekey(csk, current_gen) < 0)
+ goto out;
+
+ printf("=== Server Rekey gen %d Complete ===\n\n",
+ current_gen);
+ continue;
+ }
+
+ total += n;
+ recv_count++;
+
+ if (burst_mode) {
+ if (server_verify_burst(csk, buf, buf_size, n,
+ send_iter) < 0)
+ goto out;
+ send_iter++;
+ continue;
+ }
+
+ printf("Received %zd bytes (total: %zd, count: %d)\n",
+ n, total, recv_count);
+
+ if (server_echo_send(csk, buf, n) < 0)
+ goto out;
+ printf("Echoed %zd bytes back to client\n", n);
+ }
+
+ test_result = 0;
+out:
+ printf("Connection closed. Total received: %zd bytes\n", total);
+ if (num_rekeys)
+ printf("Rekeys completed: %d\n", current_gen);
+
+ if (csk >= 0)
+ close(csk);
+ if (lsk >= 0)
+ close(lsk);
+ free(buf);
+ return test_result;
+}
+
+static int parse_int_arg(const char *arg, int min, int max,
+ const char *name, int *out)
+{
+ char *endp;
+ long val;
+
+ errno = 0;
+ val = strtol(arg, &endp, 10);
+ if (errno || endp == arg || *endp != '\0' || val < min || val > max) {
+ if (max == INT_MAX)
+ printf("ERROR: Invalid %s '%s'. Must be >= %d.\n",
+ name, arg, min);
+ else
+ printf("ERROR: Invalid %s '%s'. Must be %d..%d.\n",
+ name, arg, min, max);
+ return -1;
+ }
+ *out = (int)val;
+ return 0;
+}
+
+static int parse_cipher_option(const char *arg)
+{
+ if (strcmp(arg, "128") == 0) {
+ cipher_type = TLS_CIPHER_AES_GCM_128;
+ return 0;
+ } else if (strcmp(arg, "256") == 0) {
+ cipher_type = TLS_CIPHER_AES_GCM_256;
+ return 0;
+ }
+ printf("ERROR: Invalid cipher '%s'. Must be 128 or 256.\n", arg);
+ return -1;
+}
+
+static int parse_version_option(const char *arg)
+{
+ if (strcmp(arg, "1.2") == 0) {
+ tls_version = TLS_1_2_VERSION;
+ return 0;
+ } else if (strcmp(arg, "1.3") == 0) {
+ tls_version = TLS_1_3_VERSION;
+ return 0;
+ }
+ printf("ERROR: Invalid TLS version '%s'. Must be 1.2 or 1.3.\n", arg);
+ return -1;
+}
+
+static void print_usage(const char *prog)
+{
+ printf("TLS Hardware Offload Two-Node Test\n\n");
+ printf("Usage:\n");
+ printf(" %s server [OPTIONS]\n", prog);
+ printf(" %s client -s <ip> [OPTIONS]\n", prog);
+ printf("\nOptions:\n");
+ printf(" -s <ip> Server IPv4 address (client, required)\n");
+ printf(" -p <port> Server port (default: 4433)\n");
+ printf(" -b <size> Send buffer size in bytes (default: 16384)\n");
+ printf(" -r <max> Use random send buffer sizes (1..<max>)\n");
+ printf(" -v <version> TLS version: 1.2 or 1.3 (default: 1.3)\n");
+ printf(" -c <cipher> Cipher: 128 or 256 (default: 128)\n");
+ printf(" -n <N> Number of send/echo iterations (default: 100)\n");
+ printf(" -k <N> Perform N rekeys (client only, TLS 1.3; N < iterations)\n");
+ printf(" -B Burst mode: client sends continuously without echo;\n");
+ printf(" server drains and handles KeyUpdate without responding.\n");
+ printf(" -Z Set TLS_RX_EXPECT_NO_PAD on the server: TLS 1.3\n");
+ printf(" opt-in to the zero-copy RX fast path. Not needed\n");
+ printf(" for TLS 1.2 (always eligible). Server only.\n");
+ printf(" -h Show this help message\n");
+ printf("\nExample:\n");
+ printf(" Node A: %s server\n", prog);
+ printf(" Node B: %s client -s 192.168.20.2\n", prog);
+ printf("\nRekey Example (3 rekeys, TLS 1.3 only):\n");
+ printf(" Node A: %s server\n", prog);
+ printf(" Node B: %s client -s 192.168.20.2 -k 3\n", prog);
+ printf("\nBurst Mode Example (client stresses TX rekey under load):\n");
+ printf(" Node A: %s server -B\n", prog);
+ printf(" Node B: %s client -s 192.168.20.2 -B -k 3\n", prog);
+}
+
+int main(int argc, char *argv[])
+{
+ int send_size_set = 0;
+ int is_server;
+ int opt;
+
+ if (argc < 2 ||
+ (strcmp(argv[1], "server") && strcmp(argv[1], "client"))) {
+ print_usage(argv[0]);
+ return 1;
+ }
+ is_server = !strcmp(argv[1], "server");
+
+ optind = 2; /* skip subcommand */
+ while ((opt = getopt(argc, argv, "s:p:b:r:c:v:k:n:BZh")) != -1) {
+ switch (opt) {
+ case 's':
+ server_ip = optarg;
+ break;
+ case 'B':
+ burst_mode = 1;
+ break;
+ case 'Z':
+ zc_rx = 1;
+ break;
+ case 'p':
+ if (parse_int_arg(optarg, 1, 65535, "port",
+ &server_port) < 0)
+ return 1;
+ break;
+ case 'b':
+ if (parse_int_arg(optarg, 1, INT_MAX, "buffer size",
+ &send_size) < 0)
+ return 1;
+ send_size_set = 1;
+ break;
+ case 'r':
+ if (parse_int_arg(optarg, 1, INT_MAX, "random size",
+ &random_size_max) < 0)
+ return 1;
+ break;
+ case 'c':
+ if (parse_cipher_option(optarg) < 0)
+ return 1;
+ break;
+ case 'v':
+ if (parse_version_option(optarg) < 0)
+ return 1;
+ break;
+ case 'k':
+ if (parse_int_arg(optarg, 1, 255, "rekey count",
+ &num_rekeys) < 0)
+ return 1;
+ break;
+ case 'n':
+ if (parse_int_arg(optarg, 1, INT_MAX, "iteration count",
+ &num_iterations) < 0)
+ return 1;
+ break;
+ case 'h':
+ print_usage(argv[0]);
+ return 0;
+ default:
+ print_usage(argv[0]);
+ return 1;
+ }
+ }
+
+ if (send_size_set && random_size_max > 0) {
+ printf("ERROR: -b and -r are mutually exclusive\n");
+ return 1;
+ }
+
+ if (zc_rx && tls_version != TLS_1_3_VERSION) {
+ printf("ERROR: -Z (TLS_RX_EXPECT_NO_PAD) requires TLS 1.3\n");
+ return 1;
+ }
+
+ if (burst_mode && random_size_max > 0) {
+ printf("ERROR: -B and -r are mutually exclusive\n");
+ return 1;
+ }
+
+ if (burst_mode && send_size < MIN_BUF_SIZE) {
+ printf("ERROR: -b must be >= %d in burst mode (-B)\n",
+ MIN_BUF_SIZE);
+ return 1;
+ }
+
+ if (is_server) {
+ if (server_ip) {
+ printf("warning: -s is ignored in server mode\n");
+ server_ip = NULL;
+ }
+ if (random_size_max > 0) {
+ printf("warning: -r is ignored in server mode\n");
+ random_size_max = 0;
+ }
+ if (num_rekeys) {
+ printf("warning: -k is ignored in server mode\n");
+ num_rekeys = 0;
+ }
+ } else {
+ if (!server_ip) {
+ printf("ERROR: Client requires -s <ip> option\n");
+ return 1;
+ }
+ if (tls_version == TLS_1_2_VERSION && num_rekeys) {
+ printf("ERROR: TLS 1.2 does not support rekey\n");
+ return 1;
+ }
+ if (num_rekeys >= num_iterations) {
+ printf("ERROR: num_rekeys (%d) must be < num_iterations (%d)\n",
+ num_rekeys, num_iterations);
+ return 1;
+ }
+ if (zc_rx) {
+ printf("ERROR: -Z applies to the server (receiver) only\n");
+ return 1;
+ }
+ }
+
+ printf("TLS Version: %s\n", version_name(tls_version));
+ printf("Cipher: %s\n", cipher_name(cipher_type));
+ if (random_size_max > 0)
+ printf("Buffer size: random (1..%d)\n", random_size_max);
+ else
+ printf("Buffer size: %d\n", send_size);
+
+ if (num_rekeys)
+ printf("Rekey testing ENABLED: %d rekey(s)\n", num_rekeys);
+ if (burst_mode)
+ printf("Burst mode ENABLED\n");
+ if (zc_rx)
+ printf("TLS_RX_EXPECT_NO_PAD ENABLED\n");
+
+ srand(time(NULL));
+
+ if (is_server)
+ return do_server() ? 1 : 0;
+
+ return do_client() ? 1 : 0;
+}
diff --git a/tools/testing/selftests/drivers/net/hw/tls_hw_offload.py b/tools/testing/selftests/drivers/net/hw/tls_hw_offload.py
new file mode 100755
index 000000000000..b8f5a3314030
--- /dev/null
+++ b/tools/testing/selftests/drivers/net/hw/tls_hw_offload.py
@@ -0,0 +1,295 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0
+
+"""Test kTLS hardware offload using a C helper binary."""
+
+from collections import defaultdict
+
+from lib.py import ksft_run, ksft_exit, ksft_pr, KsftSkipEx, ksft_true
+from lib.py import ksft_variants, KsftNamedVariant
+from lib.py import NetDrvEpEnv
+from lib.py import cmd, bkg, wait_port_listen, rand_port
+from lib.py import CmdExitFailure
+
+# Burst variants push hundreds of MB and perform many rekeys; the
+# default cmd() timeout (5s) is too short.
+BURST_TIMEOUT_S = 180
+
+
+def check_tls_support(cfg):
+ try:
+ cmd("test -f /proc/net/tls_stat")
+ cmd("test -f /proc/net/tls_stat", host=cfg.remote)
+ except CmdExitFailure as e:
+ raise KsftSkipEx(f"kTLS not supported: {e}")
+
+ try:
+ features = cmd(f"ethtool -k {cfg.ifname}").stdout
+ if 'tls-hw-tx-offload: on' not in features:
+ raise KsftSkipEx("Device does not support TLS HW TX offload")
+ if 'tls-hw-rx-offload: on' not in features:
+ raise KsftSkipEx("Device does not support TLS HW RX offload")
+ except CmdExitFailure as e:
+ raise KsftSkipEx(f"Cannot determine TLS HW offload support: {e}")
+
+
+def read_tls_stats(host=None):
+ stats = defaultdict(int)
+ output = cmd("cat /proc/net/tls_stat", host=host)
+ for line in output.stdout.strip().split('\n'):
+ parts = line.split()
+ if len(parts) == 2:
+ stats[parts[0]] = int(parts[1])
+ return stats
+
+
+def stat_diff(before, after, key):
+ return after[key] - before[key]
+
+
+def check_path(before, after, direction, role, require_hw):
+ """On the DUT, require HW offload; on the remote, HW or SW is fine."""
+ dev = stat_diff(before, after, f'Tls{direction}Device')
+ sw = stat_diff(before, after, f'Tls{direction}Sw')
+ if require_hw:
+ if dev < 1:
+ ksft_pr(f"FAIL: {role} {direction}: HW offload not engaged "
+ f"(Device={dev}, Sw={sw})")
+ return 1
+ elif dev < 1 and sw < 1:
+ ksft_pr(f"FAIL: {role} {direction}: no TLS activity "
+ f"(Device={dev}, Sw={sw})")
+ return 1
+ return 0
+
+
+def check_min(before, after, key, minimum, role):
+ diff = stat_diff(before, after, key)
+ if diff < minimum:
+ ksft_pr(f"FAIL: {role} {key}: expected >= {minimum}, got {diff}")
+ return 1
+ return 0
+
+
+def check_max(before, after, key, maximum, role):
+ diff = stat_diff(before, after, key)
+ if diff > maximum:
+ ksft_pr(f"FAIL: {role} {key}: expected <= {maximum}, got {diff}")
+ return 1
+ return 0
+
+
+def check_eq_sum(before, after, keys, expected, role):
+ """Verify the sum of `keys` increased by exactly `expected`."""
+ diff = sum(stat_diff(before, after, k) for k in keys)
+ if diff != expected:
+ ksft_pr(f"FAIL: {role} sum({'+'.join(keys)}): "
+ f"expected == {expected}, got {diff}")
+ return 1
+ return 0
+
+
+def check_zero(before, after, key, role):
+ diff = stat_diff(before, after, key)
+ if diff != 0:
+ ksft_pr(f"FAIL: {role} {key} changed by {diff}, expected 0")
+ return 1
+ return 0
+
+
+def verify_tls_counters(stats_before, stats_after, expected_rekeys,
+ tls_role, is_dut, burst=False):
+ """Verify TLS counters on one side of the connection.
+
+ tls_role: 'client' or 'server' (TLS role this side played).
+ is_dut: True for the local DUT; requires HW offload counters.
+ burst: burst mode - only the TLS client rotates its TX key; the TLS
+ server only follows with an RX rotation on KeyUpdate receipt.
+ """
+ role = 'DUT' if is_dut else 'Peer'
+
+ # In burst mode the TLS client only TXs and the TLS server only RXs.
+ # In echo mode both sides drive both directions.
+ with_tx = not burst or tls_role == 'client'
+ with_rx = not burst or tls_role != 'client'
+
+ errors = 0
+ if with_tx:
+ errors += check_path(stats_before, stats_after, 'Tx', role,
+ require_hw=is_dut)
+ if with_rx:
+ errors += check_path(stats_before, stats_after, 'Rx', role,
+ require_hw=is_dut)
+
+ if expected_rekeys > 0:
+ if with_tx:
+ # At most one rekey can be PENDING at socket close (single
+ # TLS_TX_REKEY_PENDING bit), so at most one rekey lands in
+ # TlsTxRekeyAborted; the rest must complete as TlsTxRekeyOk.
+ # Together with the sum equality this pins:
+ # Aborted=0 -> Ok=N, Aborted=1 -> Ok=N-1.
+ errors += check_max(stats_before, stats_after,
+ 'TlsTxRekeyAborted', 1, role)
+ errors += check_eq_sum(stats_before, stats_after,
+ ['TlsTxRekeyOk', 'TlsTxRekeyAborted'],
+ expected_rekeys, role)
+ errors += check_zero(stats_before, stats_after,
+ 'TlsTxRekeyError', role)
+ errors += check_zero(stats_before, stats_after,
+ 'TlsTxRekeyFallback', role)
+ errors += check_zero(stats_before, stats_after,
+ 'TlsTxRekeyInProgress', role)
+ if with_rx:
+ # At most one rekey can be deferred (single dev_add_pending)
+ # at socket close, landing in TlsRxRekeyAborted; the rest
+ # complete as TlsRxRekeyOk. Together with the sum equality this
+ # pins: Aborted=0 -> Ok=N, Aborted=1 -> Ok=N-1.
+ errors += check_max(stats_before, stats_after,
+ 'TlsRxRekeyAborted', 1, role)
+ errors += check_eq_sum(stats_before, stats_after,
+ ['TlsRxRekeyOk', 'TlsRxRekeyAborted'],
+ expected_rekeys, role)
+ errors += check_min(stats_before, stats_after,
+ 'TlsRxRekeyReceived', expected_rekeys, role)
+ errors += check_zero(stats_before, stats_after,
+ 'TlsRxRekeyError', role)
+ errors += check_zero(stats_before, stats_after,
+ 'TlsRxRekeyFallback', role)
+ errors += check_zero(stats_before, stats_after,
+ 'TlsRxRekeyInProgress', role)
+
+ errors += check_zero(stats_before, stats_after, 'TlsDecryptError', role)
+
+ return errors
+
+
+def run_tls_test(cfg, cipher="128", tls_version="1.3", rekey=0,
+ buffer_size=None, random_max=None, burst=False, zc=False,
+ dut_role="client", num_iterations=None):
+ """Run the TLS offload test.
+
+ dut_role: 'client' (default) - DUT runs the TLS client, remote the server.
+ 'server' - swap: DUT listens, remote connects. Used for burst_rx
+ so the DUT's RX path is the one under rekey pressure.
+
+ The DUT (local) is the kernel under test; the remote is just a traffic
+ source/sink and may run any kernel without HW offload. Both sides run
+ kTLS because TLS is pairwise, but verify_tls_counters() requires HW
+ offload only on the DUT (is_dut=True); the peer may use SW kTLS.
+ """
+ port = rand_port()
+ send_size = random_max or buffer_size
+
+ if dut_role == "client":
+ server_bin, server_host = cfg.bin_remote, cfg.remote
+ client_bin, client_host = cfg.bin_local, None
+ client_target = cfg.remote_addr_v['4']
+ else:
+ server_bin, server_host = cfg.bin_local, None
+ client_bin, client_host = cfg.bin_remote, cfg.remote
+ client_target = cfg.addr_v['4']
+
+ server_parts = [f"{server_bin} server -p {port} -c {cipher}",
+ f"-v {tls_version}"]
+ if burst:
+ server_parts.append("-B")
+ if zc:
+ server_parts.append("-Z")
+ if send_size:
+ server_parts.append(f"-b {send_size}")
+ server_cmd = " ".join(server_parts)
+
+ client_parts = [f"{client_bin} client -s {client_target}",
+ f"-p {port} -c {cipher} -v {tls_version}"]
+ if rekey:
+ client_parts.append(f"-k {rekey}")
+ if burst:
+ client_parts.append("-B")
+ if num_iterations:
+ client_parts.append(f"-n {num_iterations}")
+ if random_max:
+ client_parts.append(f"-r {random_max}")
+ elif buffer_size:
+ client_parts.append(f"-b {buffer_size}")
+ client_cmd = " ".join(client_parts)
+
+ cmd_timeout = BURST_TIMEOUT_S if burst else 10
+
+ stats_before_local = read_tls_stats()
+ stats_before_remote = read_tls_stats(host=cfg.remote)
+
+ with bkg(server_cmd, host=server_host, exit_wait=True):
+ wait_port_listen(port, host=server_host)
+ cmd(client_cmd, host=client_host, timeout=cmd_timeout)
+
+ stats_after_local = read_tls_stats()
+ stats_after_remote = read_tls_stats(host=cfg.remote)
+
+ peer_tls_role = 'server' if dut_role == 'client' else 'client'
+
+ dut_errors = verify_tls_counters(stats_before_local, stats_after_local,
+ rekey, dut_role, is_dut=True,
+ burst=burst)
+ peer_errors = verify_tls_counters(stats_before_remote, stats_after_remote,
+ rekey, peer_tls_role, is_dut=False,
+ burst=burst)
+
+ ksft_true(dut_errors == 0,
+ f"DUT TLS counters verified ({dut_errors} failures)")
+ ksft_true(peer_errors == 0,
+ f"Peer TLS counters verified ({peer_errors} failures)")
+
+
+@ksft_variants([
+ KsftNamedVariant("tls13_aes128", "128", "1.3"),
+ KsftNamedVariant("tls13_aes256", "256", "1.3"),
+ KsftNamedVariant("tls12_aes128", "128", "1.2"),
+ KsftNamedVariant("tls12_aes256", "256", "1.2"),
+])
+def test_tls_offload(cfg, cipher, tls_version):
+ run_tls_test(cfg, cipher=cipher, tls_version=tls_version)
+
+
+@ksft_variants([
+ KsftNamedVariant("single", 1),
+ KsftNamedVariant("multiple", 99),
+ KsftNamedVariant("small_buf", 30, 512),
+ KsftNamedVariant("large_buf", 10, 2097152),
+ KsftNamedVariant("random_buf", 20, None, 8192),
+])
+def test_tls_offload_rekey(cfg, rekey, buffer_size=None, random_max=None):
+ run_tls_test(cfg, cipher="128", tls_version="1.3", rekey=rekey,
+ buffer_size=buffer_size, random_max=random_max)
+
+
+# Columns: dut_role zc interval rekeys buffer_size
+@ksft_variants([
+ KsftNamedVariant("burst_tx_rekey_every_1", "client", False, 1, 50, 65536),
+ KsftNamedVariant("burst_tx_rekey_every_1000", "client", False, 1000, 3, 65536),
+ KsftNamedVariant("burst_rx_rekey_every_10", "server", False, 10, 20, 65536),
+ KsftNamedVariant("burst_rx_rekey_every_10000", "server", False, 10000, 1, 32768),
+ KsftNamedVariant("burst_rx_zc_rekey_every_100", "server", True, 100, 10, 65536),
+ KsftNamedVariant("burst_rx_zc_rekey_every_20000", "server", True, 20000, 1, 16384),
+])
+def test_tls_offload_burst(cfg, dut_role, zc, interval, rekeys, buffer_size):
+ run_tls_test(cfg, cipher="128", tls_version="1.3", rekey=rekeys,
+ buffer_size=buffer_size, burst=True, zc=zc, dut_role=dut_role,
+ num_iterations=interval * (rekeys + 1))
+
+
+def main() -> None:
+ with NetDrvEpEnv(__file__, nsim_test=False) as cfg:
+ cfg.bin_local = cfg.test_dir / "tls_hw_offload"
+ if not cfg.bin_local.exists():
+ raise KsftSkipEx(f"tls_hw_offload binary not found at {cfg.bin_local}")
+ cfg.bin_remote = cfg.remote.deploy(cfg.bin_local)
+ cfg.require_ipver("4")
+ check_tls_support(cfg)
+
+ ksft_run([test_tls_offload, test_tls_offload_rekey,
+ test_tls_offload_burst], args=(cfg, ))
+ ksft_exit()
+
+
+if __name__ == "__main__":
+ main()
--
2.25.1
^ permalink raw reply related [flat|nested] 10+ messages in thread