* [PATCH v13 1/6] net: tls: reject TLS 1.3 offload in chcr_ktls and nfp drivers
2026-04-29 18:10 [PATCH net-next v13 0/6] tls: Add TLS 1.3 hardware offload support Rishikesh Jethwani
@ 2026-04-29 18:10 ` Rishikesh Jethwani
2026-04-29 18:10 ` [PATCH v13 2/6] net/mlx5e: add TLS 1.3 hardware offload support Rishikesh Jethwani
` (4 subsequent siblings)
5 siblings, 0 replies; 9+ messages in thread
From: Rishikesh Jethwani @ 2026-04-29 18:10 UTC (permalink / raw)
To: netdev
Cc: saeedm, tariqt, mbloch, borisp, john.fastabend, kuba, sd, davem,
pabeni, edumazet, leon, Rishikesh Jethwani
These drivers only support TLS 1.2. Return early when TLS 1.3
is requested to prevent unsupported hardware offload attempts.
Signed-off-by: Rishikesh Jethwani <rjethwani@purestorage.com>
---
drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c | 3 +++
drivers/net/ethernet/netronome/nfp/crypto/tls.c | 3 +++
2 files changed, 6 insertions(+)
diff --git a/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c
index f5acd4be1e69..29e108ce6764 100644
--- a/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c
+++ b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c
@@ -431,6 +431,9 @@ static int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk,
atomic64_inc(&port_stats->ktls_tx_connection_open);
u_ctx = adap->uld[CXGB4_ULD_KTLS].handle;
+ if (crypto_info->version != TLS_1_2_VERSION)
+ goto out;
+
if (direction == TLS_OFFLOAD_CTX_DIR_RX) {
pr_err("not expecting for RX direction\n");
goto out;
diff --git a/drivers/net/ethernet/netronome/nfp/crypto/tls.c b/drivers/net/ethernet/netronome/nfp/crypto/tls.c
index 9983d7aa2b9c..13864c6a55dc 100644
--- a/drivers/net/ethernet/netronome/nfp/crypto/tls.c
+++ b/drivers/net/ethernet/netronome/nfp/crypto/tls.c
@@ -287,6 +287,9 @@ nfp_net_tls_add(struct net_device *netdev, struct sock *sk,
BUILD_BUG_ON(offsetof(struct nfp_net_tls_offload_ctx, rx_end) >
TLS_DRIVER_STATE_SIZE_RX);
+ if (crypto_info->version != TLS_1_2_VERSION)
+ return -EOPNOTSUPP;
+
if (!nfp_net_cipher_supported(nn, crypto_info->cipher_type, direction))
return -EOPNOTSUPP;
--
2.25.1
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH v13 2/6] net/mlx5e: add TLS 1.3 hardware offload support
2026-04-29 18:10 [PATCH net-next v13 0/6] tls: Add TLS 1.3 hardware offload support Rishikesh Jethwani
2026-04-29 18:10 ` [PATCH v13 1/6] net: tls: reject TLS 1.3 offload in chcr_ktls and nfp drivers Rishikesh Jethwani
@ 2026-04-29 18:10 ` Rishikesh Jethwani
2026-04-29 18:10 ` [PATCH v13 3/6] tls: " Rishikesh Jethwani
` (3 subsequent siblings)
5 siblings, 0 replies; 9+ messages in thread
From: Rishikesh Jethwani @ 2026-04-29 18:10 UTC (permalink / raw)
To: netdev
Cc: saeedm, tariqt, mbloch, borisp, john.fastabend, kuba, sd, davem,
pabeni, edumazet, leon, Rishikesh Jethwani
Enable TLS 1.3 TX/RX hardware offload on ConnectX-6 Dx and newer
crypto-enabled adapters.
Key changes:
- Add TLS 1.3 capability checking and version validation
- Use MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_3 (0x3) for crypto context
- Handle TLS 1.3 IV format: full 12-byte IV copied to gcm_iv +
implicit_iv (vs TLS 1.2's 4-byte salt only)
Tested with TLS 1.3 AES-GCM-128 and AES-GCM-256 cipher suites.
Signed-off-by: Rishikesh Jethwani <rjethwani@purestorage.com>
---
.../ethernet/mellanox/mlx5/core/en_accel/ktls.h | 8 +++++++-
.../mellanox/mlx5/core/en_accel/ktls_txrx.c | 14 +++++++++++---
2 files changed, 18 insertions(+), 4 deletions(-)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h
index 07a04a142a2e..0469ca6a0762 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h
@@ -30,7 +30,9 @@ static inline bool mlx5e_is_ktls_device(struct mlx5_core_dev *mdev)
return false;
return (MLX5_CAP_TLS(mdev, tls_1_2_aes_gcm_128) ||
- MLX5_CAP_TLS(mdev, tls_1_2_aes_gcm_256));
+ MLX5_CAP_TLS(mdev, tls_1_2_aes_gcm_256) ||
+ MLX5_CAP_TLS(mdev, tls_1_3_aes_gcm_128) ||
+ MLX5_CAP_TLS(mdev, tls_1_3_aes_gcm_256));
}
static inline bool mlx5e_ktls_type_check(struct mlx5_core_dev *mdev,
@@ -40,10 +42,14 @@ static inline bool mlx5e_ktls_type_check(struct mlx5_core_dev *mdev,
case TLS_CIPHER_AES_GCM_128:
if (crypto_info->version == TLS_1_2_VERSION)
return MLX5_CAP_TLS(mdev, tls_1_2_aes_gcm_128);
+ else if (crypto_info->version == TLS_1_3_VERSION)
+ return MLX5_CAP_TLS(mdev, tls_1_3_aes_gcm_128);
break;
case TLS_CIPHER_AES_GCM_256:
if (crypto_info->version == TLS_1_2_VERSION)
return MLX5_CAP_TLS(mdev, tls_1_2_aes_gcm_256);
+ else if (crypto_info->version == TLS_1_3_VERSION)
+ return MLX5_CAP_TLS(mdev, tls_1_3_aes_gcm_256);
break;
}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c
index 570a912dd6fa..f3f1be1d4034 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c
@@ -6,6 +6,7 @@
enum {
MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_2 = 0x2,
+ MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_3 = 0x3,
};
enum {
@@ -15,8 +16,10 @@ enum {
#define EXTRACT_INFO_FIELDS do { \
salt = info->salt; \
rec_seq = info->rec_seq; \
+ iv = info->iv; \
salt_sz = sizeof(info->salt); \
rec_seq_sz = sizeof(info->rec_seq); \
+ iv_sz = sizeof(info->iv); \
} while (0)
static void
@@ -24,9 +27,9 @@ fill_static_params(struct mlx5_wqe_tls_static_params_seg *params,
union mlx5e_crypto_info *crypto_info,
u32 key_id, u32 resync_tcp_sn)
{
+ u16 salt_sz, rec_seq_sz, iv_sz;
+ char *salt, *rec_seq, *iv;
char *initial_rn, *gcm_iv;
- u16 salt_sz, rec_seq_sz;
- char *salt, *rec_seq;
u8 tls_version;
u8 *ctx;
@@ -59,7 +62,12 @@ fill_static_params(struct mlx5_wqe_tls_static_params_seg *params,
memcpy(gcm_iv, salt, salt_sz);
memcpy(initial_rn, rec_seq, rec_seq_sz);
- tls_version = MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_2;
+ if (crypto_info->crypto_info.version == TLS_1_3_VERSION) {
+ memcpy(gcm_iv + salt_sz, iv, iv_sz);
+ tls_version = MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_3;
+ } else {
+ tls_version = MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_2;
+ }
MLX5_SET(tls_static_params, ctx, tls_version, tls_version);
MLX5_SET(tls_static_params, ctx, const_1, 1);
--
2.25.1
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH v13 3/6] tls: add TLS 1.3 hardware offload support
2026-04-29 18:10 [PATCH net-next v13 0/6] tls: Add TLS 1.3 hardware offload support Rishikesh Jethwani
2026-04-29 18:10 ` [PATCH v13 1/6] net: tls: reject TLS 1.3 offload in chcr_ktls and nfp drivers Rishikesh Jethwani
2026-04-29 18:10 ` [PATCH v13 2/6] net/mlx5e: add TLS 1.3 hardware offload support Rishikesh Jethwani
@ 2026-04-29 18:10 ` Rishikesh Jethwani
2026-04-29 18:10 ` [PATCH v13 4/6] tls: split tls_set_sw_offload into init and finalize stages Rishikesh Jethwani
` (2 subsequent siblings)
5 siblings, 0 replies; 9+ messages in thread
From: Rishikesh Jethwani @ 2026-04-29 18:10 UTC (permalink / raw)
To: netdev
Cc: saeedm, tariqt, mbloch, borisp, john.fastabend, kuba, sd, davem,
pabeni, edumazet, leon, Rishikesh Jethwani
Add TLS 1.3 support to the kernel TLS hardware offload infrastructure,
enabling hardware acceleration for TLS 1.3 connections on capable NICs.
Tested on Mellanox ConnectX-6 Dx (Crypto Enabled) with TLS 1.3 AES-GCM-128
and AES-GCM-256 cipher suites.
Signed-off-by: Rishikesh Jethwani <rjethwani@purestorage.com>
---
net/tls/tls_device.c | 65 ++++++++++++++++-----------
net/tls/tls_device_fallback.c | 58 +++++++++++++-----------
net/tls/tls_main.c | 85 ++++++++++++++++++++---------------
3 files changed, 121 insertions(+), 87 deletions(-)
diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c
index 99c8eff9783e..1321bf9b59b0 100644
--- a/net/tls/tls_device.c
+++ b/net/tls/tls_device.c
@@ -317,25 +317,34 @@ static void tls_device_record_close(struct sock *sk,
unsigned char record_type)
{
struct tls_prot_info *prot = &ctx->prot_info;
- struct page_frag dummy_tag_frag;
-
- /* append tag
- * device will fill in the tag, we just need to append a placeholder
- * use socket memory to improve coalescing (re-using a single buffer
- * increases frag count)
- * if we can't allocate memory now use the dummy page
+ int tail = prot->tag_size + prot->tail_size;
+
+ /* Append tail: tag for TLS 1.2, content_type + tag for TLS 1.3.
+ * Device fills in the tag, we just need to append a placeholder.
+ * Use socket memory to improve coalescing (re-using a single buffer
+ * increases frag count); if allocation fails use dummy_page
+ * (offset = record_type gives correct content_type byte via
+ * identity mapping)
*/
- if (unlikely(pfrag->size - pfrag->offset < prot->tag_size) &&
- !skb_page_frag_refill(prot->tag_size, pfrag, sk->sk_allocation)) {
- dummy_tag_frag.page = dummy_page;
- dummy_tag_frag.offset = 0;
- pfrag = &dummy_tag_frag;
+ if (unlikely(pfrag->size - pfrag->offset < tail) &&
+ !skb_page_frag_refill(tail, pfrag, sk->sk_allocation)) {
+ struct page_frag dummy_pfrag = {
+ .page = dummy_page,
+ .offset = record_type,
+ };
+ tls_append_frag(record, &dummy_pfrag, tail);
+ } else {
+ if (prot->tail_size) {
+ char *content_type_addr = page_address(pfrag->page) +
+ pfrag->offset;
+ *content_type_addr = record_type;
+ }
+ tls_append_frag(record, pfrag, tail);
}
- tls_append_frag(record, pfrag, prot->tag_size);
/* fill prepend */
tls_fill_prepend(ctx, skb_frag_address(&record->frags[0]),
- record->len - prot->overhead_size,
+ record->len - prot->overhead_size + prot->tail_size,
record_type);
}
@@ -883,6 +892,7 @@ static int
tls_device_reencrypt(struct sock *sk, struct tls_context *tls_ctx)
{
struct tls_sw_context_rx *sw_ctx = tls_sw_ctx_rx(tls_ctx);
+ struct tls_prot_info *prot = &tls_ctx->prot_info;
const struct tls_cipher_desc *cipher_desc;
int err, offset, copy, data_len, pos;
struct sk_buff *skb, *skb_iter;
@@ -894,7 +904,7 @@ tls_device_reencrypt(struct sock *sk, struct tls_context *tls_ctx)
DEBUG_NET_WARN_ON_ONCE(!cipher_desc || !cipher_desc->offloadable);
rxm = strp_msg(tls_strp_msg(sw_ctx));
- orig_buf = kmalloc(rxm->full_len + TLS_HEADER_SIZE + cipher_desc->iv,
+ orig_buf = kmalloc(rxm->full_len + prot->prepend_size,
sk->sk_allocation);
if (!orig_buf)
return -ENOMEM;
@@ -909,9 +919,8 @@ tls_device_reencrypt(struct sock *sk, struct tls_context *tls_ctx)
offset = rxm->offset;
sg_init_table(sg, 1);
- sg_set_buf(&sg[0], buf,
- rxm->full_len + TLS_HEADER_SIZE + cipher_desc->iv);
- err = skb_copy_bits(skb, offset, buf, TLS_HEADER_SIZE + cipher_desc->iv);
+ sg_set_buf(&sg[0], buf, rxm->full_len + prot->prepend_size);
+ err = skb_copy_bits(skb, offset, buf, prot->prepend_size);
if (err)
goto free_buf;
@@ -1089,11 +1098,6 @@ int tls_set_device_offload(struct sock *sk)
}
crypto_info = &ctx->crypto_send.info;
- if (crypto_info->version != TLS_1_2_VERSION) {
- rc = -EOPNOTSUPP;
- goto release_netdev;
- }
-
cipher_desc = get_cipher_desc(crypto_info->cipher_type);
if (!cipher_desc || !cipher_desc->offloadable) {
rc = -EINVAL;
@@ -1196,9 +1200,6 @@ int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx)
struct net_device *netdev;
int rc = 0;
- if (ctx->crypto_recv.info.version != TLS_1_2_VERSION)
- return -EOPNOTSUPP;
-
netdev = get_netdev_for_sock(sk);
if (!netdev) {
pr_err_ratelimited("%s: netdev not found\n", __func__);
@@ -1409,12 +1410,22 @@ static struct notifier_block tls_dev_notifier = {
int __init tls_device_init(void)
{
- int err;
+ unsigned char *page_addr;
+ int err, i;
dummy_page = alloc_page(GFP_KERNEL);
if (!dummy_page)
return -ENOMEM;
+ /* Pre-populate dummy_page with identity mapping for all byte values.
+ * This is used as fallback for TLS 1.3 content type when memory
+ * allocation fails. By populating all 256 values, we avoid needing
+ * to validate record_type at runtime.
+ */
+ page_addr = page_address(dummy_page);
+ for (i = 0; i < 256; i++)
+ page_addr[i] = (unsigned char)i;
+
destruct_wq = alloc_workqueue("ktls_device_destruct", WQ_PERCPU, 0);
if (!destruct_wq) {
err = -ENOMEM;
diff --git a/net/tls/tls_device_fallback.c b/net/tls/tls_device_fallback.c
index 3b7d0ab2bcf1..1110f7ac6bcb 100644
--- a/net/tls/tls_device_fallback.c
+++ b/net/tls/tls_device_fallback.c
@@ -37,14 +37,15 @@
#include "tls.h"
-static int tls_enc_record(struct aead_request *aead_req,
+static int tls_enc_record(struct tls_context *tls_ctx,
+ struct aead_request *aead_req,
struct crypto_aead *aead, char *aad,
char *iv, __be64 rcd_sn,
struct scatter_walk *in,
- struct scatter_walk *out, int *in_len,
- struct tls_prot_info *prot)
+ struct scatter_walk *out, int *in_len)
{
unsigned char buf[TLS_HEADER_SIZE + TLS_MAX_IV_SIZE];
+ struct tls_prot_info *prot = &tls_ctx->prot_info;
const struct tls_cipher_desc *cipher_desc;
struct scatterlist sg_in[3];
struct scatterlist sg_out[3];
@@ -55,7 +56,7 @@ static int tls_enc_record(struct aead_request *aead_req,
cipher_desc = get_cipher_desc(prot->cipher_type);
DEBUG_NET_WARN_ON_ONCE(!cipher_desc || !cipher_desc->offloadable);
- buf_size = TLS_HEADER_SIZE + cipher_desc->iv;
+ buf_size = prot->prepend_size;
len = min_t(int, *in_len, buf_size);
memcpy_from_scatterwalk(buf, in, len);
@@ -66,16 +67,27 @@ static int tls_enc_record(struct aead_request *aead_req,
return 0;
len = buf[4] | (buf[3] << 8);
- len -= cipher_desc->iv;
+ if (prot->version != TLS_1_3_VERSION)
+ len -= cipher_desc->iv;
tls_make_aad(aad, len - cipher_desc->tag, (char *)&rcd_sn, buf[0], prot);
- memcpy(iv + cipher_desc->salt, buf + TLS_HEADER_SIZE, cipher_desc->iv);
+ if (prot->version == TLS_1_3_VERSION) {
+ void *iv_src = crypto_info_iv(&tls_ctx->crypto_send.info,
+ cipher_desc);
+
+ memcpy(iv + cipher_desc->salt, iv_src, cipher_desc->iv);
+ } else {
+ memcpy(iv + cipher_desc->salt, buf + TLS_HEADER_SIZE,
+ cipher_desc->iv);
+ }
+
+ tls_xor_iv_with_seq(prot, iv, (char *)&rcd_sn);
sg_init_table(sg_in, ARRAY_SIZE(sg_in));
sg_init_table(sg_out, ARRAY_SIZE(sg_out));
- sg_set_buf(sg_in, aad, TLS_AAD_SPACE_SIZE);
- sg_set_buf(sg_out, aad, TLS_AAD_SPACE_SIZE);
+ sg_set_buf(sg_in, aad, prot->aad_size);
+ sg_set_buf(sg_out, aad, prot->aad_size);
scatterwalk_get_sglist(in, sg_in + 1);
scatterwalk_get_sglist(out, sg_out + 1);
@@ -108,13 +120,6 @@ static int tls_enc_record(struct aead_request *aead_req,
return rc;
}
-static void tls_init_aead_request(struct aead_request *aead_req,
- struct crypto_aead *aead)
-{
- aead_request_set_tfm(aead_req, aead);
- aead_request_set_ad(aead_req, TLS_AAD_SPACE_SIZE);
-}
-
static struct aead_request *tls_alloc_aead_request(struct crypto_aead *aead,
gfp_t flags)
{
@@ -124,14 +129,15 @@ static struct aead_request *tls_alloc_aead_request(struct crypto_aead *aead,
aead_req = kzalloc(req_size, flags);
if (aead_req)
- tls_init_aead_request(aead_req, aead);
+ aead_request_set_tfm(aead_req, aead);
return aead_req;
}
-static int tls_enc_records(struct aead_request *aead_req,
+static int tls_enc_records(struct tls_context *tls_ctx,
+ struct aead_request *aead_req,
struct crypto_aead *aead, struct scatterlist *sg_in,
struct scatterlist *sg_out, char *aad, char *iv,
- u64 rcd_sn, int len, struct tls_prot_info *prot)
+ u64 rcd_sn, int len)
{
struct scatter_walk out, in;
int rc;
@@ -140,8 +146,8 @@ static int tls_enc_records(struct aead_request *aead_req,
scatterwalk_start(&out, sg_out);
do {
- rc = tls_enc_record(aead_req, aead, aad, iv,
- cpu_to_be64(rcd_sn), &in, &out, &len, prot);
+ rc = tls_enc_record(tls_ctx, aead_req, aead, aad, iv,
+ cpu_to_be64(rcd_sn), &in, &out, &len);
rcd_sn++;
} while (rc == 0 && len);
@@ -314,7 +320,10 @@ static struct sk_buff *tls_enc_skb(struct tls_context *tls_ctx,
cipher_desc = get_cipher_desc(tls_ctx->crypto_send.info.cipher_type);
DEBUG_NET_WARN_ON_ONCE(!cipher_desc || !cipher_desc->offloadable);
- buf_len = cipher_desc->salt + cipher_desc->iv + TLS_AAD_SPACE_SIZE +
+ aead_request_set_ad(aead_req, tls_ctx->prot_info.aad_size);
+
+ buf_len = cipher_desc->salt + cipher_desc->iv +
+ tls_ctx->prot_info.aad_size +
sync_size + cipher_desc->tag;
buf = kmalloc(buf_len, GFP_ATOMIC);
if (!buf)
@@ -324,7 +333,7 @@ static struct sk_buff *tls_enc_skb(struct tls_context *tls_ctx,
salt = crypto_info_salt(&tls_ctx->crypto_send.info, cipher_desc);
memcpy(iv, salt, cipher_desc->salt);
aad = buf + cipher_desc->salt + cipher_desc->iv;
- dummy_buf = aad + TLS_AAD_SPACE_SIZE;
+ dummy_buf = aad + tls_ctx->prot_info.aad_size;
nskb = alloc_skb(skb_headroom(skb) + skb->len, GFP_ATOMIC);
if (!nskb)
@@ -335,9 +344,8 @@ static struct sk_buff *tls_enc_skb(struct tls_context *tls_ctx,
fill_sg_out(sg_out, buf, tls_ctx, nskb, tcp_payload_offset,
payload_len, sync_size, dummy_buf);
- if (tls_enc_records(aead_req, ctx->aead_send, sg_in, sg_out, aad, iv,
- rcd_sn, sync_size + payload_len,
- &tls_ctx->prot_info) < 0)
+ if (tls_enc_records(tls_ctx, aead_req, ctx->aead_send, sg_in, sg_out,
+ aad, iv, rcd_sn, sync_size + payload_len) < 0)
goto free_nskb;
complete_skb(nskb, skb, tcp_payload_offset);
diff --git a/net/tls/tls_main.c b/net/tls/tls_main.c
index fd39acf41a61..fd04857fa0ab 100644
--- a/net/tls/tls_main.c
+++ b/net/tls/tls_main.c
@@ -711,49 +711,64 @@ static int do_tls_setsockopt_conf(struct sock *sk, sockptr_t optval,
}
if (tx) {
- rc = tls_set_device_offload(sk);
- conf = TLS_HW;
- if (!rc) {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXDEVICE);
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXDEVICE);
- } else {
- rc = tls_set_sw_offload(sk, 1,
- update ? crypto_info : NULL);
- if (rc)
- goto err_crypto_info;
-
- if (update) {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYOK);
- } else {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXSW);
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW);
+ if (update && ctx->tx_conf == TLS_HW) {
+ rc = -EOPNOTSUPP;
+ goto err_crypto_info;
+ }
+
+ if (!update) {
+ rc = tls_set_device_offload(sk);
+ conf = TLS_HW;
+ if (!rc) {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXDEVICE);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXDEVICE);
+ goto out;
}
- conf = TLS_SW;
}
- } else {
- rc = tls_set_device_offload_rx(sk, ctx);
- conf = TLS_HW;
- if (!rc) {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXDEVICE);
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXDEVICE);
+
+ rc = tls_set_sw_offload(sk, 1, update ? crypto_info : NULL);
+ if (rc)
+ goto err_crypto_info;
+
+ if (update) {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYOK);
} else {
- rc = tls_set_sw_offload(sk, 0,
- update ? crypto_info : NULL);
- if (rc)
- goto err_crypto_info;
-
- if (update) {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYOK);
- } else {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXSW);
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXSW);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXSW);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW);
+ }
+ conf = TLS_SW;
+ } else {
+ if (update && ctx->rx_conf == TLS_HW) {
+ rc = -EOPNOTSUPP;
+ goto err_crypto_info;
+ }
+
+ if (!update) {
+ rc = tls_set_device_offload_rx(sk, ctx);
+ conf = TLS_HW;
+ if (!rc) {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXDEVICE);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXDEVICE);
+ tls_sw_strparser_arm(sk, ctx);
+ goto out;
}
- conf = TLS_SW;
}
- if (!update)
+
+ rc = tls_set_sw_offload(sk, 0, update ? crypto_info : NULL);
+ if (rc)
+ goto err_crypto_info;
+
+ if (update) {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYOK);
+ } else {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXSW);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXSW);
tls_sw_strparser_arm(sk, ctx);
+ }
+ conf = TLS_SW;
}
+out:
if (tx)
ctx->tx_conf = conf;
else
--
2.25.1
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH v13 4/6] tls: split tls_set_sw_offload into init and finalize stages
2026-04-29 18:10 [PATCH net-next v13 0/6] tls: Add TLS 1.3 hardware offload support Rishikesh Jethwani
` (2 preceding siblings ...)
2026-04-29 18:10 ` [PATCH v13 3/6] tls: " Rishikesh Jethwani
@ 2026-04-29 18:10 ` Rishikesh Jethwani
2026-04-29 18:10 ` [PATCH v13 5/6] tls: add hardware offload key update support Rishikesh Jethwani
2026-04-29 18:10 ` [PATCH v13 6/6] selftests: net: add TLS hardware offload test Rishikesh Jethwani
5 siblings, 0 replies; 9+ messages in thread
From: Rishikesh Jethwani @ 2026-04-29 18:10 UTC (permalink / raw)
To: netdev
Cc: saeedm, tariqt, mbloch, borisp, john.fastabend, kuba, sd, davem,
pabeni, edumazet, leon, Rishikesh Jethwani
Separate cipher context initialization from key material finalization
to support staged setup for hardware offload fallback paths.
Signed-off-by: Rishikesh Jethwani <rjethwani@purestorage.com>
---
net/tls/tls.h | 4 +++
net/tls/tls_device.c | 3 +-
net/tls/tls_sw.c | 77 +++++++++++++++++++++++++++++++-------------
3 files changed, 61 insertions(+), 23 deletions(-)
diff --git a/net/tls/tls.h b/net/tls/tls.h
index e8f81a006520..a65cf9bab190 100644
--- a/net/tls/tls.h
+++ b/net/tls/tls.h
@@ -147,6 +147,10 @@ void tls_strp_abort_strp(struct tls_strparser *strp, int err);
int init_prot_info(struct tls_prot_info *prot,
const struct tls_crypto_info *crypto_info,
const struct tls_cipher_desc *cipher_desc);
+int tls_sw_ctx_init(struct sock *sk, int tx,
+ struct tls_crypto_info *new_crypto_info);
+void tls_sw_ctx_finalize(struct sock *sk, int tx,
+ struct tls_crypto_info *new_crypto_info);
int tls_set_sw_offload(struct sock *sk, int tx,
struct tls_crypto_info *new_crypto_info);
void tls_update_rx_zc_capable(struct tls_context *tls_ctx);
diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c
index 1321bf9b59b0..cd26873e9063 100644
--- a/net/tls/tls_device.c
+++ b/net/tls/tls_device.c
@@ -1233,7 +1233,7 @@ int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx)
context->resync_nh_reset = 1;
ctx->priv_ctx_rx = context;
- rc = tls_set_sw_offload(sk, 0, NULL);
+ rc = tls_sw_ctx_init(sk, 0, NULL);
if (rc)
goto release_ctx;
@@ -1247,6 +1247,7 @@ int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx)
goto free_sw_resources;
tls_device_attach(ctx, sk, netdev);
+ tls_sw_ctx_finalize(sk, 0, NULL);
up_read(&device_offload_lock);
dev_put(netdev);
diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c
index 94d2ae0daa8c..1412b3dcce4c 100644
--- a/net/tls/tls_sw.c
+++ b/net/tls/tls_sw.c
@@ -2784,20 +2784,19 @@ static void tls_finish_key_update(struct sock *sk, struct tls_context *tls_ctx)
ctx->saved_data_ready(sk);
}
-int tls_set_sw_offload(struct sock *sk, int tx,
- struct tls_crypto_info *new_crypto_info)
+int tls_sw_ctx_init(struct sock *sk, int tx,
+ struct tls_crypto_info *new_crypto_info)
{
struct tls_crypto_info *crypto_info, *src_crypto_info;
struct tls_sw_context_tx *sw_ctx_tx = NULL;
struct tls_sw_context_rx *sw_ctx_rx = NULL;
const struct tls_cipher_desc *cipher_desc;
- char *iv, *rec_seq, *key, *salt;
- struct cipher_context *cctx;
struct tls_prot_info *prot;
struct crypto_aead **aead;
struct tls_context *ctx;
struct crypto_tfm *tfm;
int rc = 0;
+ char *key;
ctx = tls_get_ctx(sk);
prot = &ctx->prot_info;
@@ -2818,12 +2817,10 @@ int tls_set_sw_offload(struct sock *sk, int tx,
if (tx) {
sw_ctx_tx = ctx->priv_ctx_tx;
crypto_info = &ctx->crypto_send.info;
- cctx = &ctx->tx;
aead = &sw_ctx_tx->aead_send;
} else {
sw_ctx_rx = ctx->priv_ctx_rx;
crypto_info = &ctx->crypto_recv.info;
- cctx = &ctx->rx;
aead = &sw_ctx_rx->aead_recv;
}
@@ -2839,10 +2836,7 @@ int tls_set_sw_offload(struct sock *sk, int tx,
if (rc)
goto free_priv;
- iv = crypto_info_iv(src_crypto_info, cipher_desc);
key = crypto_info_key(src_crypto_info, cipher_desc);
- salt = crypto_info_salt(src_crypto_info, cipher_desc);
- rec_seq = crypto_info_rec_seq(src_crypto_info, cipher_desc);
if (!*aead) {
*aead = crypto_alloc_aead(cipher_desc->cipher_name, 0, 0);
@@ -2886,19 +2880,6 @@ int tls_set_sw_offload(struct sock *sk, int tx,
goto free_aead;
}
- memcpy(cctx->iv, salt, cipher_desc->salt);
- memcpy(cctx->iv + cipher_desc->salt, iv, cipher_desc->iv);
- memcpy(cctx->rec_seq, rec_seq, cipher_desc->rec_seq);
-
- if (new_crypto_info) {
- unsafe_memcpy(crypto_info, new_crypto_info,
- cipher_desc->crypto_info,
- /* size was checked in do_tls_setsockopt_conf */);
- memzero_explicit(new_crypto_info, cipher_desc->crypto_info);
- if (!tx)
- tls_finish_key_update(sk, ctx);
- }
-
goto out;
free_aead:
@@ -2917,3 +2898,55 @@ int tls_set_sw_offload(struct sock *sk, int tx,
out:
return rc;
}
+
+void tls_sw_ctx_finalize(struct sock *sk, int tx,
+ struct tls_crypto_info *new_crypto_info)
+{
+ struct tls_crypto_info *crypto_info, *src_crypto_info;
+ const struct tls_cipher_desc *cipher_desc;
+ struct tls_context *ctx = tls_get_ctx(sk);
+ struct cipher_context *cctx;
+ char *iv, *salt, *rec_seq;
+
+ if (tx) {
+ crypto_info = &ctx->crypto_send.info;
+ cctx = &ctx->tx;
+ } else {
+ crypto_info = &ctx->crypto_recv.info;
+ cctx = &ctx->rx;
+ }
+
+ src_crypto_info = new_crypto_info ?: crypto_info;
+ cipher_desc = get_cipher_desc(src_crypto_info->cipher_type);
+
+ iv = crypto_info_iv(src_crypto_info, cipher_desc);
+ salt = crypto_info_salt(src_crypto_info, cipher_desc);
+ rec_seq = crypto_info_rec_seq(src_crypto_info, cipher_desc);
+
+ memcpy(cctx->iv, salt, cipher_desc->salt);
+ memcpy(cctx->iv + cipher_desc->salt, iv, cipher_desc->iv);
+ memcpy(cctx->rec_seq, rec_seq, cipher_desc->rec_seq);
+
+ if (new_crypto_info) {
+ unsafe_memcpy(crypto_info, new_crypto_info,
+ cipher_desc->crypto_info,
+ /* size was checked in do_tls_setsockopt_conf */);
+ memzero_explicit(new_crypto_info, cipher_desc->crypto_info);
+
+ if (!tx)
+ tls_finish_key_update(sk, ctx);
+ }
+}
+
+int tls_set_sw_offload(struct sock *sk, int tx,
+ struct tls_crypto_info *new_crypto_info)
+{
+ int rc;
+
+ rc = tls_sw_ctx_init(sk, tx, new_crypto_info);
+ if (rc)
+ return rc;
+
+ tls_sw_ctx_finalize(sk, tx, new_crypto_info);
+ return 0;
+}
--
2.25.1
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH v13 5/6] tls: add hardware offload key update support
2026-04-29 18:10 [PATCH net-next v13 0/6] tls: Add TLS 1.3 hardware offload support Rishikesh Jethwani
` (3 preceding siblings ...)
2026-04-29 18:10 ` [PATCH v13 4/6] tls: split tls_set_sw_offload into init and finalize stages Rishikesh Jethwani
@ 2026-04-29 18:10 ` Rishikesh Jethwani
2026-05-05 8:40 ` Paolo Abeni
2026-04-29 18:10 ` [PATCH v13 6/6] selftests: net: add TLS hardware offload test Rishikesh Jethwani
5 siblings, 1 reply; 9+ messages in thread
From: Rishikesh Jethwani @ 2026-04-29 18:10 UTC (permalink / raw)
To: netdev
Cc: saeedm, tariqt, mbloch, borisp, john.fastabend, kuba, sd, davem,
pabeni, edumazet, leon, Rishikesh Jethwani
On TX, the NIC key cannot be replaced while HW-offloaded records
are still unacked. tls_dev_start_rekey() installs a temporary SW
context with the new key and redirects sendmsg through
tls_sw_sendmsg_locked. If no records are pending,
tls_dev_complete_rekey() runs inline during setsockopt; otherwise
clean_acked sets REKEY_READY once all old-key records are ACKed
and the next sendmsg completes the rekey, flushing SW records and
reinstalling HW offload at the current write_seq. A KeyUpdate
arriving while one is pending re-keys the SW AEAD in place; if the
HW reinstall fails the socket stays in SW mode (REKEY_FAILED).
On RX, the NIC may have already decrypted in-flight records with
the old key before the peer's KeyUpdate is parsed, so the old
AEAD, IV and rec_seq are retained on tls_offload_context_rx.
tls_check_pending_rekey() invokes tls_device_rx_del_key() to drop
the NIC key; otherwise post-KeyUpdate records (carrying new-key
wire encryption) would be XOR'd with the retired key.
tls_device_decrypted() classifies records by old_nic_boundary:
- after the boundary: new-key record; drop the old key.
- before, fully encrypted: advance old_rec_seq, let SW AEAD decrypt.
- before, (partially) decrypted: reencrypt with the old key so SW
AEAD can decrypt with the new key.
For mixed records skb->decrypted flags can be wrong (NIC clears
them on auth failure); on -EBADMSG, tls_rx_rekey_retry() toggles
those flags, decrements old_rec_seq to reuse the nonce, and
retries once (gated by old_key_reencrypted).
The new key's tls_dev_add is deferred until the old key is fully
consumed: tls_set_device_offload_rx() sets dev_add_pending while
old_aead_recv is retained, and tls_device_deferred_dev_add()
installs the new key once copied_seq crosses old_nic_boundary.
Tested on Mellanox ConnectX-6 Dx (Crypto Enabled) with multiple
TLS 1.3 TX and RX KeyUpdate cycles.
Signed-off-by: Rishikesh Jethwani <rjethwani@purestorage.com>
---
include/net/tls.h | 84 +++-
include/uapi/linux/snmp.h | 2 +
net/tls/tls.h | 29 +-
net/tls/tls_device.c | 753 +++++++++++++++++++++++++++++++---
net/tls/tls_device_fallback.c | 24 ++
net/tls/tls_main.c | 92 +++--
net/tls/tls_proc.c | 2 +
net/tls/tls_sw.c | 76 +++-
net/tls/trace.h | 79 ++++
9 files changed, 992 insertions(+), 149 deletions(-)
diff --git a/include/net/tls.h b/include/net/tls.h
index ebd2550280ae..6891aa6b484c 100644
--- a/include/net/tls.h
+++ b/include/net/tls.h
@@ -151,6 +151,22 @@ struct tls_record_info {
skb_frag_t frags[MAX_SKB_FRAGS];
};
+struct cipher_context {
+ char iv[TLS_MAX_IV_SIZE + TLS_MAX_SALT_SIZE];
+ char rec_seq[TLS_MAX_REC_SEQ_SIZE];
+};
+
+union tls_crypto_context {
+ struct tls_crypto_info info;
+ union {
+ struct tls12_crypto_info_aes_gcm_128 aes_gcm_128;
+ struct tls12_crypto_info_aes_gcm_256 aes_gcm_256;
+ struct tls12_crypto_info_chacha20_poly1305 chacha20_poly1305;
+ struct tls12_crypto_info_sm4_gcm sm4_gcm;
+ struct tls12_crypto_info_sm4_ccm sm4_ccm;
+ };
+};
+
#define TLS_DRIVER_STATE_SIZE_TX 16
struct tls_offload_context_tx {
struct crypto_aead *aead_send;
@@ -165,6 +181,11 @@ struct tls_offload_context_tx {
void (*sk_destruct)(struct sock *sk);
struct work_struct destruct_work;
struct tls_context *ctx;
+
+ struct tls_sw_context_tx rekey_sw; /* SW context for new key */
+ struct cipher_context rekey_tx; /* IV, rec_seq for new key */
+ union tls_crypto_context rekey_crypto_send; /* Crypto for new key */
+
/* The TLS layer reserves room for driver specific state
* Currently the belief is that there is not enough
* driver specific state to justify another layer of indirection
@@ -189,22 +210,21 @@ enum tls_context_flags {
* tls_dev_del call in tls_device_down if it happens simultaneously.
*/
TLS_RX_DEV_CLOSED = 2,
-};
-
-struct cipher_context {
- char iv[TLS_MAX_IV_SIZE + TLS_MAX_SALT_SIZE];
- char rec_seq[TLS_MAX_REC_SEQ_SIZE];
-};
-
-union tls_crypto_context {
- struct tls_crypto_info info;
- union {
- struct tls12_crypto_info_aes_gcm_128 aes_gcm_128;
- struct tls12_crypto_info_aes_gcm_256 aes_gcm_256;
- struct tls12_crypto_info_chacha20_poly1305 chacha20_poly1305;
- struct tls12_crypto_info_sm4_gcm sm4_gcm;
- struct tls12_crypto_info_sm4_ccm sm4_ccm;
- };
+ /* Flag for TX HW context deleted during failed rekey.
+ * Prevents double tls_dev_del in cleanup paths.
+ */
+ TLS_TX_DEV_CLOSED = 3,
+ /* TX rekey is pending, waiting for old-key data to be ACKed.
+ * While set, new data uses SW path with new key, HW keeps old key
+ * for retransmissions.
+ */
+ TLS_TX_REKEY_PENDING = 4,
+ /* All old-key data has been ACKed, ready to install new key in HW. */
+ TLS_TX_REKEY_READY = 5,
+ /* HW rekey failed, permanently stay in SW encrypt mode.
+ * Prevents tls_tcp_clean_acked from re-setting TLS_TX_REKEY_READY.
+ */
+ TLS_TX_REKEY_FAILED = 6,
};
struct tls_prot_info {
@@ -253,6 +273,15 @@ struct tls_context {
*/
unsigned long flags;
+ /* TCP sequence number boundary for pending rekey.
+ * Packets with seq < this use old key, >= use new key.
+ */
+ u32 rekey_boundary_seq;
+
+ /* Pointers to rekey contexts for SW encryption with new key */
+ struct tls_sw_context_tx *rekey_sw_ctx;
+ struct cipher_context *rekey_cipher_ctx;
+
/* cache cold stuff */
struct proto *sk_proto;
struct sock *sk;
@@ -311,6 +340,14 @@ struct tls_offload_context_rx {
u8 resync_nh_reset:1;
/* CORE_NEXT_HINT-only member, but use the hole here */
u8 resync_nh_do_now:1;
+ /* retry reencrypt of mixed record during rekey */
+ u8 old_key_reencrypted:1;
+ /* tls_dev_add deferred until old key is freed */
+ u8 dev_add_pending:1;
+ struct crypto_aead *old_aead_recv; /* old key AEAD cipher */
+ char old_iv[TLS_MAX_IV_SIZE + TLS_MAX_SALT_SIZE]; /* old key IV */
+ char old_rec_seq[TLS_MAX_REC_SEQ_SIZE]; /* old key TLS record seq */
+ u32 old_nic_boundary; /* TCP seq: NIC switched to next key */
union {
/* TLS_OFFLOAD_SYNC_TYPE_DRIVER_REQ */
struct {
@@ -385,9 +422,21 @@ static inline struct tls_sw_context_rx *tls_sw_ctx_rx(
static inline struct tls_sw_context_tx *tls_sw_ctx_tx(
const struct tls_context *tls_ctx)
{
+ if (unlikely(tls_ctx->rekey_sw_ctx))
+ return tls_ctx->rekey_sw_ctx;
+
return (struct tls_sw_context_tx *)tls_ctx->priv_ctx_tx;
}
+static inline struct cipher_context *tls_tx_cipher_ctx(
+ const struct tls_context *tls_ctx)
+{
+ if (unlikely(tls_ctx->rekey_cipher_ctx))
+ return tls_ctx->rekey_cipher_ctx;
+
+ return (struct cipher_context *)&tls_ctx->tx;
+}
+
static inline struct tls_offload_context_tx *
tls_offload_ctx_tx(const struct tls_context *tls_ctx)
{
@@ -500,6 +549,9 @@ struct sk_buff *tls_encrypt_skb(struct sk_buff *skb);
#ifdef CONFIG_TLS_DEVICE
void tls_device_sk_destruct(struct sock *sk);
void tls_offload_tx_resync_request(struct sock *sk, u32 got_seq, u32 exp_seq);
+struct sk_buff *
+tls_validate_xmit_skb_rekey(struct sock *sk, struct net_device *dev,
+ struct sk_buff *skb);
static inline bool tls_is_sk_rx_device_offloaded(struct sock *sk)
{
diff --git a/include/uapi/linux/snmp.h b/include/uapi/linux/snmp.h
index 49f5640092a0..39fa48821faa 100644
--- a/include/uapi/linux/snmp.h
+++ b/include/uapi/linux/snmp.h
@@ -369,6 +369,8 @@ enum
LINUX_MIB_TLSTXREKEYOK, /* TlsTxRekeyOk */
LINUX_MIB_TLSTXREKEYERROR, /* TlsTxRekeyError */
LINUX_MIB_TLSRXREKEYRECEIVED, /* TlsRxRekeyReceived */
+ LINUX_MIB_TLSTXREKEYHWFAIL, /* TlsTxRekeyHwFail */
+ LINUX_MIB_TLSRXREKEYHWFAIL, /* TlsRxRekeyHwFail */
__LINUX_MIB_TLSMAX
};
diff --git a/net/tls/tls.h b/net/tls/tls.h
index a65cf9bab190..03d558e80f9a 100644
--- a/net/tls/tls.h
+++ b/net/tls/tls.h
@@ -157,6 +157,9 @@ void tls_update_rx_zc_capable(struct tls_context *tls_ctx);
void tls_sw_strparser_arm(struct sock *sk, struct tls_context *ctx);
void tls_sw_strparser_done(struct tls_context *tls_ctx);
int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
+int tls_sw_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size);
+void tls_tx_work_handler(struct work_struct *work);
+void tls_sw_ctx_tx_init(struct sock *sk, struct tls_sw_context_tx *sw_ctx);
void tls_sw_splice_eof(struct socket *sock);
void tls_sw_cancel_work_tx(struct tls_context *tls_ctx);
void tls_sw_release_resources_tx(struct sock *sk);
@@ -176,6 +179,8 @@ int tls_sw_read_sock(struct sock *sk, read_descriptor_t *desc,
int tls_device_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
void tls_device_splice_eof(struct socket *sock);
int tls_tx_records(struct sock *sk, int flags);
+int tls_sw_push_pending_record(struct sock *sk, int flags);
+int tls_encrypt_async_wait(struct tls_sw_context_tx *ctx);
void tls_sw_write_space(struct sock *sk, struct tls_context *ctx);
void tls_device_write_space(struct sock *sk, struct tls_context *ctx);
@@ -233,10 +238,13 @@ static inline bool tls_strp_msg_mixed_decrypted(struct tls_sw_context_rx *ctx)
#ifdef CONFIG_TLS_DEVICE
int tls_device_init(void);
void tls_device_cleanup(void);
-int tls_set_device_offload(struct sock *sk);
+int tls_set_device_offload(struct sock *sk,
+ struct tls_crypto_info *crypto_info);
void tls_device_free_resources_tx(struct sock *sk);
-int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx);
+int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx,
+ struct tls_crypto_info *crypto_info);
void tls_device_offload_cleanup_rx(struct sock *sk);
+void tls_device_rx_del_key(struct sock *sk, struct tls_context *ctx);
void tls_device_rx_resync_new_rec(struct sock *sk, u32 rcd_len, u32 seq);
int tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx);
#else
@@ -244,7 +252,7 @@ static inline int tls_device_init(void) { return 0; }
static inline void tls_device_cleanup(void) {}
static inline int
-tls_set_device_offload(struct sock *sk)
+tls_set_device_offload(struct sock *sk, struct tls_crypto_info *crypto_info)
{
return -EOPNOTSUPP;
}
@@ -252,13 +260,16 @@ tls_set_device_offload(struct sock *sk)
static inline void tls_device_free_resources_tx(struct sock *sk) {}
static inline int
-tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx)
+tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx,
+ struct tls_crypto_info *crypto_info)
{
return -EOPNOTSUPP;
}
static inline void tls_device_offload_cleanup_rx(struct sock *sk) {}
static inline void
+tls_device_rx_del_key(struct sock *sk, struct tls_context *ctx) {}
+static inline void
tls_device_rx_resync_new_rec(struct sock *sk, u32 rcd_len, u32 seq) {}
static inline int
@@ -298,6 +309,16 @@ static inline bool tls_bigint_increment(unsigned char *seq, int len)
return (i == -1);
}
+static inline void tls_bigint_decrement(unsigned char *seq, int len)
+{
+ int i;
+
+ for (i = len - 1; i >= 0; i--) {
+ if (seq[i]-- != 0)
+ break;
+ }
+}
+
static inline void tls_bigint_subtract(unsigned char *seq, int n)
{
u64 rcd_sn;
diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c
index cd26873e9063..51f1cc783336 100644
--- a/net/tls/tls_device.c
+++ b/net/tls/tls_device.c
@@ -79,7 +79,9 @@ static void tls_device_tx_del_task(struct work_struct *work)
netdev = rcu_dereference_protected(ctx->netdev,
!refcount_read(&ctx->refcount));
- netdev->tlsdev_ops->tls_dev_del(netdev, ctx, TLS_OFFLOAD_CTX_DIR_TX);
+ if (!test_bit(TLS_TX_DEV_CLOSED, &ctx->flags))
+ netdev->tlsdev_ops->tls_dev_del(netdev, ctx,
+ TLS_OFFLOAD_CTX_DIR_TX);
dev_put(netdev);
ctx->netdev = NULL;
tls_device_free_ctx(ctx);
@@ -159,6 +161,262 @@ static void delete_all_records(struct tls_offload_context_tx *offload_ctx)
offload_ctx->retransmit_hint = NULL;
}
+static bool tls_has_unacked_records(struct tls_offload_context_tx *offload_ctx)
+{
+ struct tls_record_info *info;
+ bool has_unacked = false;
+ unsigned long flags;
+
+ spin_lock_irqsave(&offload_ctx->lock, flags);
+ list_for_each_entry(info, &offload_ctx->records_list, list) {
+ if (!tls_record_is_start_marker(info)) {
+ has_unacked = true;
+ break;
+ }
+ }
+ spin_unlock_irqrestore(&offload_ctx->lock, flags);
+
+ return has_unacked;
+}
+
+static int tls_device_init_rekey_sw(struct sock *sk,
+ struct tls_context *ctx,
+ struct tls_offload_context_tx *offload_ctx,
+ struct tls_crypto_info *new_crypto_info)
+{
+ struct tls_sw_context_tx *sw_ctx = &offload_ctx->rekey_sw;
+ const struct tls_cipher_desc *cipher_desc;
+ char *key;
+ int rc;
+
+ cipher_desc = get_cipher_desc(new_crypto_info->cipher_type);
+ DEBUG_NET_WARN_ON_ONCE(!cipher_desc || !cipher_desc->offloadable);
+
+ memset(sw_ctx, 0, sizeof(*sw_ctx));
+ tls_sw_ctx_tx_init(sk, sw_ctx);
+
+ sw_ctx->aead_send = crypto_alloc_aead(cipher_desc->cipher_name, 0, 0);
+ if (IS_ERR(sw_ctx->aead_send)) {
+ rc = PTR_ERR(sw_ctx->aead_send);
+ sw_ctx->aead_send = NULL;
+ return rc;
+ }
+
+ key = crypto_info_key(new_crypto_info, cipher_desc);
+ rc = crypto_aead_setkey(sw_ctx->aead_send, key, cipher_desc->key);
+ if (rc)
+ goto free_aead;
+
+ rc = crypto_aead_setauthsize(sw_ctx->aead_send, cipher_desc->tag);
+ if (rc)
+ goto free_aead;
+
+ return 0;
+
+free_aead:
+ crypto_free_aead(sw_ctx->aead_send);
+ sw_ctx->aead_send = NULL;
+ return rc;
+}
+
+static int tls_device_start_rekey(struct sock *sk,
+ struct tls_context *ctx,
+ struct tls_offload_context_tx *offload_ctx,
+ struct tls_crypto_info *new_crypto_info)
+{
+ bool rekey_pending = test_bit(TLS_TX_REKEY_PENDING, &ctx->flags);
+ bool rekey_failed = test_bit(TLS_TX_REKEY_FAILED, &ctx->flags);
+ const struct tls_cipher_desc *cipher_desc;
+ char *key, *iv, *rec_seq, *salt;
+ int rc;
+
+ cipher_desc = get_cipher_desc(new_crypto_info->cipher_type);
+ DEBUG_NET_WARN_ON_ONCE(!cipher_desc || !cipher_desc->offloadable);
+
+ key = crypto_info_key(new_crypto_info, cipher_desc);
+ iv = crypto_info_iv(new_crypto_info, cipher_desc);
+ rec_seq = crypto_info_rec_seq(new_crypto_info, cipher_desc);
+ salt = crypto_info_salt(new_crypto_info, cipher_desc);
+
+ if (rekey_pending || rekey_failed) {
+ rc = crypto_aead_setkey(offload_ctx->rekey_sw.aead_send,
+ key, cipher_desc->key);
+ if (rc)
+ return rc;
+
+ memcpy(offload_ctx->rekey_tx.iv, salt, cipher_desc->salt);
+ memcpy(offload_ctx->rekey_tx.iv + cipher_desc->salt, iv,
+ cipher_desc->iv);
+ memcpy(offload_ctx->rekey_tx.rec_seq, rec_seq,
+ cipher_desc->rec_seq);
+
+ if (rekey_failed) {
+ set_bit(TLS_TX_REKEY_PENDING, &ctx->flags);
+ clear_bit(TLS_TX_REKEY_FAILED, &ctx->flags);
+ }
+ } else {
+ rc = tls_device_init_rekey_sw(sk, ctx, offload_ctx,
+ new_crypto_info);
+ if (rc)
+ return rc;
+
+ memcpy(offload_ctx->rekey_tx.iv, salt, cipher_desc->salt);
+ memcpy(offload_ctx->rekey_tx.iv + cipher_desc->salt, iv,
+ cipher_desc->iv);
+ memcpy(offload_ctx->rekey_tx.rec_seq, rec_seq,
+ cipher_desc->rec_seq);
+
+ WRITE_ONCE(ctx->rekey_boundary_seq, tcp_sk(sk)->write_seq);
+
+ /* Prevent a partial record straddling the SW/HW boundary. */
+ tcp_write_collapse_fence(sk);
+
+ ctx->rekey_sw_ctx = &offload_ctx->rekey_sw;
+ ctx->rekey_cipher_ctx = &offload_ctx->rekey_tx;
+
+ set_bit(TLS_TX_REKEY_PENDING, &ctx->flags);
+
+ /* Switch to rekey validator; new sends won't use HW offload */
+ smp_store_release(&sk->sk_validate_xmit_skb,
+ tls_validate_xmit_skb_rekey);
+ }
+
+ unsafe_memcpy(&offload_ctx->rekey_crypto_send.info, new_crypto_info,
+ cipher_desc->crypto_info,
+ /* checked in do_tls_setsockopt_conf */);
+ memzero_explicit(new_crypto_info, cipher_desc->crypto_info);
+
+ return 0;
+}
+
+static int tls_device_complete_rekey(struct sock *sk, struct tls_context *ctx)
+{
+ struct tls_offload_context_tx *offload_ctx = tls_offload_ctx_tx(ctx);
+ struct tls_record_info *start_marker_record;
+ const struct tls_cipher_desc *cipher_desc;
+ struct net_device *netdev;
+ unsigned long flags;
+ __be64 rcd_sn;
+ char *key;
+ int rc;
+
+ cipher_desc = get_cipher_desc(offload_ctx->rekey_crypto_send.info.cipher_type);
+ DEBUG_NET_WARN_ON_ONCE(!cipher_desc || !cipher_desc->offloadable);
+
+ /* Flush all pending SW data before switching back to HW:
+ * 1. Close any open_rec left by MSG_MORE and encrypt it.
+ * 2. Wait for async crypto completions.
+ * 3. Push all ready records into TCP.
+ * If the send buffer is full, bail out and retry next sendmsg.
+ */
+ if (tls_is_pending_open_record(ctx))
+ tls_sw_push_pending_record(sk, 0);
+ tls_encrypt_async_wait(tls_sw_ctx_tx(ctx));
+ rc = tls_tx_records(sk, -1);
+ if (rc < 0 || tls_is_partially_sent_record(ctx) ||
+ tls_is_pending_open_record(ctx))
+ return rc < 0 ? rc : -EAGAIN;
+
+ cancel_delayed_work_sync(&offload_ctx->rekey_sw.tx_work.work);
+
+ start_marker_record = kmalloc_obj(*start_marker_record);
+ if (!start_marker_record)
+ return -ENOMEM;
+
+ down_read(&device_offload_lock);
+
+ netdev = rcu_dereference_protected(ctx->netdev,
+ lockdep_is_held(&device_offload_lock));
+ if (!netdev) {
+ rc = -ENODEV;
+ goto release_lock;
+ }
+
+ if (!test_bit(TLS_TX_DEV_CLOSED, &ctx->flags)) {
+ netdev->tlsdev_ops->tls_dev_del(netdev, ctx,
+ TLS_OFFLOAD_CTX_DIR_TX);
+ set_bit(TLS_TX_DEV_CLOSED, &ctx->flags);
+ }
+
+ memcpy(crypto_info_rec_seq(&offload_ctx->rekey_crypto_send.info, cipher_desc),
+ offload_ctx->rekey_tx.rec_seq, cipher_desc->rec_seq);
+
+ rc = netdev->tlsdev_ops->tls_dev_add(netdev, sk, TLS_OFFLOAD_CTX_DIR_TX,
+ &offload_ctx->rekey_crypto_send.info,
+ tcp_sk(sk)->write_seq);
+ trace_tls_device_offload_set(sk, TLS_OFFLOAD_CTX_DIR_TX,
+ tcp_sk(sk)->write_seq,
+ offload_ctx->rekey_tx.rec_seq, rc);
+
+release_lock:
+ up_read(&device_offload_lock);
+
+ spin_lock_irqsave(&offload_ctx->lock, flags);
+ memcpy(&rcd_sn, offload_ctx->rekey_tx.rec_seq, sizeof(rcd_sn));
+ offload_ctx->unacked_record_sn = be64_to_cpu(rcd_sn) - 1;
+ spin_unlock_irqrestore(&offload_ctx->lock, flags);
+
+ memcpy(ctx->tx.iv, offload_ctx->rekey_tx.iv,
+ cipher_desc->salt + cipher_desc->iv);
+ memcpy(ctx->tx.rec_seq, offload_ctx->rekey_tx.rec_seq,
+ cipher_desc->rec_seq);
+ unsafe_memcpy(&ctx->crypto_send.info,
+ &offload_ctx->rekey_crypto_send.info,
+ cipher_desc->crypto_info,
+ /* checked during rekey setup */);
+
+ if (rc)
+ goto rekey_fail;
+
+ clear_bit(TLS_TX_DEV_CLOSED, &ctx->flags);
+
+ key = crypto_info_key(&offload_ctx->rekey_crypto_send.info, cipher_desc);
+ rc = crypto_aead_setkey(offload_ctx->aead_send, key, cipher_desc->key);
+ if (rc)
+ goto rekey_fail;
+
+ /* Start marker: the NIC passes through everything before
+ * write_seq unencrypted (already SW-encrypted during rekey),
+ * same as during initial offload setup.
+ */
+ spin_lock_irqsave(&offload_ctx->lock, flags);
+ start_marker_record->end_seq = tcp_sk(sk)->write_seq;
+ start_marker_record->len = 0;
+ start_marker_record->num_frags = 0;
+ list_add_tail_rcu(&start_marker_record->list,
+ &offload_ctx->records_list);
+ spin_unlock_irqrestore(&offload_ctx->lock, flags);
+
+ /* Prevent a partial record straddling the SW/HW boundary. */
+ tcp_write_collapse_fence(sk);
+
+ /* PENDING before READY: prevents clean_acked from
+ * re-setting REKEY_READY after we clear it.
+ */
+ clear_bit(TLS_TX_REKEY_PENDING, &ctx->flags);
+ smp_mb__after_atomic();
+ clear_bit(TLS_TX_REKEY_READY, &ctx->flags);
+ clear_bit(TLS_TX_REKEY_FAILED, &ctx->flags);
+
+ /* Switch back to HW offload validator */
+ smp_store_release(&sk->sk_validate_xmit_skb, tls_validate_xmit_skb);
+
+ crypto_free_aead(tls_sw_ctx_tx(ctx)->aead_send);
+ ctx->rekey_sw_ctx = NULL;
+ ctx->rekey_cipher_ctx = NULL;
+
+ return 0;
+
+rekey_fail:
+ kfree(start_marker_record);
+ set_bit(TLS_TX_REKEY_FAILED, &ctx->flags);
+ clear_bit(TLS_TX_REKEY_READY, &ctx->flags);
+ clear_bit(TLS_TX_REKEY_PENDING, &ctx->flags);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYHWFAIL);
+
+ return 0;
+}
+
static void tls_tcp_clean_acked(struct sock *sk, u32 acked_seq)
{
struct tls_context *tls_ctx = tls_get_ctx(sk);
@@ -187,6 +445,19 @@ static void tls_tcp_clean_acked(struct sock *sk, u32 acked_seq)
}
ctx->unacked_record_sn += deleted_records;
+
+ /* Once all old-key HW records are ACKed, set REKEY_READY to
+ * let sendmsg know it can finish the rekey and switch back
+ * to HW offload.
+ */
+ if (test_bit(TLS_TX_REKEY_PENDING, &tls_ctx->flags) &&
+ !test_bit(TLS_TX_REKEY_FAILED, &tls_ctx->flags)) {
+ u32 boundary_seq = READ_ONCE(tls_ctx->rekey_boundary_seq);
+
+ if (!before(acked_seq, boundary_seq))
+ set_bit(TLS_TX_REKEY_READY, &tls_ctx->flags);
+ }
+
spin_unlock_irqrestore(&ctx->lock, flags);
}
@@ -218,6 +489,9 @@ void tls_device_free_resources_tx(struct sock *sk)
struct tls_context *tls_ctx = tls_get_ctx(sk);
tls_free_partial_record(sk, tls_ctx);
+
+ if (unlikely(tls_ctx->rekey_sw_ctx))
+ tls_sw_release_resources_tx(sk);
}
void tls_offload_tx_resync_request(struct sock *sk, u32 got_seq, u32 exp_seq)
@@ -589,6 +863,19 @@ int tls_device_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
goto out;
}
+ /* Old-key records all ACKed; switch back to HW. */
+ if (test_bit(TLS_TX_REKEY_READY, &tls_ctx->flags))
+ tls_device_complete_rekey(sk, tls_ctx);
+
+ /* Use SW path if rekey is in progress (PENDING) or if HW rekey
+ * failed (FAILED).
+ */
+ if (test_bit(TLS_TX_REKEY_PENDING, &tls_ctx->flags) ||
+ test_bit(TLS_TX_REKEY_FAILED, &tls_ctx->flags)) {
+ rc = tls_sw_sendmsg_locked(sk, msg, size);
+ goto out;
+ }
+
rc = tls_push_data(sk, &msg->msg_iter, size, msg->msg_flags,
record_type);
@@ -791,6 +1078,8 @@ void tls_device_rx_resync_new_rec(struct sock *sk, u32 rcd_len, u32 seq)
return;
if (unlikely(test_bit(TLS_RX_DEV_DEGRADED, &tls_ctx->flags)))
return;
+ if (unlikely(test_bit(TLS_RX_DEV_CLOSED, &tls_ctx->flags)))
+ return;
prot = &tls_ctx->prot_info;
rx_ctx = tls_offload_ctx_rx(tls_ctx);
@@ -980,13 +1269,144 @@ tls_device_reencrypt(struct sock *sk, struct tls_context *tls_ctx)
return err;
}
+/*
+ * temporarily swap in the old key, run
+ * tls_device_reencrypt(), then restore the current key.
+ */
+static int tls_old_key_reencrypt(struct sock *sk,
+ struct tls_offload_context_rx *ctx,
+ struct tls_sw_context_rx *sw_ctx,
+ struct tls_context *tls_ctx)
+{
+ struct crypto_aead *saved_aead = sw_ctx->aead_recv;
+ char saved_iv[TLS_MAX_IV_SIZE + TLS_MAX_SALT_SIZE];
+ char saved_rec_seq[TLS_MAX_REC_SEQ_SIZE];
+ int ret;
+
+ memcpy(saved_iv, tls_ctx->rx.iv, sizeof(saved_iv));
+ memcpy(saved_rec_seq, tls_ctx->rx.rec_seq, sizeof(saved_rec_seq));
+
+ sw_ctx->aead_recv = ctx->old_aead_recv;
+ memcpy(tls_ctx->rx.iv, ctx->old_iv, sizeof(ctx->old_iv));
+ memcpy(tls_ctx->rx.rec_seq, ctx->old_rec_seq,
+ sizeof(ctx->old_rec_seq));
+
+ ret = tls_device_reencrypt(sk, tls_ctx);
+
+ memcpy(ctx->old_rec_seq, tls_ctx->rx.rec_seq,
+ sizeof(ctx->old_rec_seq));
+
+ sw_ctx->aead_recv = saved_aead;
+ memcpy(tls_ctx->rx.iv, saved_iv, sizeof(saved_iv));
+ memcpy(tls_ctx->rx.rec_seq, saved_rec_seq, sizeof(saved_rec_seq));
+
+ return ret;
+}
+
+/* Undo old-key XOR so SW AEAD can decrypt with the new key. */
+static int tls_device_reencrypt_old_key(struct sock *sk,
+ struct tls_offload_context_rx *ctx,
+ struct tls_sw_context_rx *sw_ctx,
+ struct tls_context *tls_ctx)
+{
+ int ret;
+
+ ret = tls_old_key_reencrypt(sk, ctx, sw_ctx, tls_ctx);
+ if (ret)
+ return ret;
+
+ tls_bigint_increment(ctx->old_rec_seq,
+ tls_ctx->prot_info.rec_seq_size);
+ ctx->resync_nh_reset = 1;
+
+ return 0;
+}
+
+/* Tear down NIC offload on peer KeyUpdate so post-KU records
+ * (new-key wire encryption) are not NIC-XOR'd with the retired key.
+ * NIC stays keyless until tls_set_device_offload_rx installs the new key.
+ */
+void tls_device_rx_del_key(struct sock *sk, struct tls_context *ctx)
+{
+ struct net_device *netdev;
+
+ if (ctx->rx_conf != TLS_HW)
+ return;
+ if (test_bit(TLS_RX_DEV_CLOSED, &ctx->flags))
+ return;
+
+ down_read(&device_offload_lock);
+ netdev = rcu_dereference_protected(ctx->netdev,
+ lockdep_is_held(&device_offload_lock));
+ if (!netdev) {
+ up_read(&device_offload_lock);
+ return;
+ }
+
+ set_bit(TLS_RX_DEV_CLOSED, &ctx->flags);
+ synchronize_net();
+ netdev->tlsdev_ops->tls_dev_del(netdev, ctx,
+ TLS_OFFLOAD_CTX_DIR_RX);
+ up_read(&device_offload_lock);
+}
+
+static int tls_device_dev_add(struct sock *sk, struct tls_context *tls_ctx,
+ struct net_device *netdev,
+ struct tls_crypto_info *crypto_info,
+ u32 cur_seq, bool is_rekey)
+{
+ const struct tls_cipher_desc *cipher_desc;
+ char *rec_seq;
+ int rc;
+
+ cipher_desc = get_cipher_desc(crypto_info->cipher_type);
+ DEBUG_NET_WARN_ON_ONCE(!cipher_desc || !cipher_desc->offloadable);
+
+ rc = netdev->tlsdev_ops->tls_dev_add(netdev, sk,
+ TLS_OFFLOAD_CTX_DIR_RX,
+ crypto_info, cur_seq);
+ rec_seq = crypto_info_rec_seq(crypto_info, cipher_desc);
+ trace_tls_device_offload_set(sk, TLS_OFFLOAD_CTX_DIR_RX,
+ cur_seq, rec_seq, rc);
+ if (!rc) {
+ clear_bit(TLS_RX_DEV_DEGRADED, &tls_ctx->flags);
+ clear_bit(TLS_RX_DEV_CLOSED, &tls_ctx->flags);
+ } else if (is_rekey) {
+ set_bit(TLS_RX_DEV_DEGRADED, &tls_ctx->flags);
+ set_bit(TLS_RX_DEV_CLOSED, &tls_ctx->flags);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYHWFAIL);
+ }
+ return rc;
+}
+
+static void tls_device_deferred_dev_add(struct sock *sk,
+ struct tls_context *tls_ctx,
+ struct tls_offload_context_rx *ctx)
+{
+ struct net_device *netdev;
+
+ ctx->dev_add_pending = 0;
+
+ down_read(&device_offload_lock);
+ netdev = rcu_dereference_protected(tls_ctx->netdev,
+ lockdep_is_held(&device_offload_lock));
+ if (netdev)
+ tls_device_dev_add(sk, tls_ctx, netdev,
+ &tls_ctx->crypto_recv.info,
+ tcp_sk(sk)->copied_seq, true);
+ up_read(&device_offload_lock);
+}
+
+
int tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx)
{
struct tls_offload_context_rx *ctx = tls_offload_ctx_rx(tls_ctx);
struct tls_sw_context_rx *sw_ctx = tls_sw_ctx_rx(tls_ctx);
struct sk_buff *skb = tls_strp_msg(sw_ctx);
+ u32 copied_seq = tcp_sk(sk)->copied_seq;
struct strp_msg *rxm = strp_msg(skb);
int is_decrypted, is_encrypted;
+ u32 rec_start_seq;
if (!tls_strp_msg_mixed_decrypted(sw_ctx)) {
is_decrypted = skb->decrypted;
@@ -996,10 +1416,67 @@ int tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx)
is_encrypted = 0;
}
- trace_tls_device_decrypted(sk, tcp_sk(sk)->copied_seq - rxm->full_len,
+ rec_start_seq = sw_ctx->strp.copy_mode
+ ? copied_seq - rxm->full_len
+ : copied_seq;
+
+ trace_tls_device_decrypted(sk, rec_start_seq,
tls_ctx->rx.rec_seq, rxm->full_len,
is_encrypted, is_decrypted);
+ if (unlikely(ctx->old_aead_recv)) {
+ bool before_nic_boundary =
+ before(rec_start_seq, ctx->old_nic_boundary);
+
+ /* Retry path: mixed record first-pass XOR-undo produced
+ * EBADMSG because per-fragment decrypted flags don't
+ * reflect which fragments were actually XOR'd (NIC auth
+ * failure clearing flags). Toggle decrypted flag and re-XOR,
+ * decrement old_rec_seq to reuse the same nonce.
+ */
+ if (ctx->old_key_reencrypted) {
+ struct sk_buff *frag_iter;
+
+ trace_tls_device_rekey_reencrypt(sk, rec_start_seq,
+ ctx->old_nic_boundary,
+ true);
+ skb->decrypted = !skb->decrypted;
+ skb_walk_frags(skb, frag_iter)
+ frag_iter->decrypted = !frag_iter->decrypted;
+
+ tls_bigint_decrement(ctx->old_rec_seq,
+ tls_ctx->prot_info.rec_seq_size);
+ return tls_device_reencrypt_old_key(sk, ctx,
+ sw_ctx, tls_ctx);
+ }
+
+ if (before_nic_boundary) {
+ if (is_encrypted) {
+ tls_bigint_increment(ctx->old_rec_seq,
+ tls_ctx->prot_info.rec_seq_size);
+ return 0;
+ }
+ /* For mixed records, first old key rencrypt and if
+ * SW AEAD fails then retry with decrypted flags toggled
+ */
+ trace_tls_device_rekey_reencrypt(sk, rec_start_seq,
+ ctx->old_nic_boundary,
+ false);
+ if (!is_decrypted)
+ ctx->old_key_reencrypted = 1;
+ return tls_device_reencrypt_old_key(sk, ctx,
+ sw_ctx, tls_ctx);
+ }
+
+ trace_tls_device_rekey_done(sk, rec_start_seq,
+ ctx->old_nic_boundary);
+ crypto_free_aead(ctx->old_aead_recv);
+ ctx->old_aead_recv = NULL;
+
+ if (ctx->dev_add_pending)
+ tls_device_deferred_dev_add(sk, tls_ctx, ctx);
+ }
+
if (unlikely(test_bit(TLS_RX_DEV_DEGRADED, &tls_ctx->flags))) {
if (likely(is_encrypted || is_decrypted))
return is_decrypted;
@@ -1068,57 +1545,31 @@ static struct tls_offload_context_tx *alloc_offload_ctx_tx(struct tls_context *c
return offload_ctx;
}
-int tls_set_device_offload(struct sock *sk)
+static int tls_set_device_offload_initial(struct sock *sk,
+ struct tls_context *ctx,
+ struct net_device *netdev,
+ struct tls_crypto_info *crypto_info,
+ const struct tls_cipher_desc *cipher_desc)
{
+ struct tls_prot_info *prot = &ctx->prot_info;
struct tls_record_info *start_marker_record;
struct tls_offload_context_tx *offload_ctx;
- const struct tls_cipher_desc *cipher_desc;
- struct tls_crypto_info *crypto_info;
- struct tls_prot_info *prot;
- struct net_device *netdev;
- struct tls_context *ctx;
char *iv, *rec_seq;
int rc;
- ctx = tls_get_ctx(sk);
- prot = &ctx->prot_info;
-
- if (ctx->priv_ctx_tx)
- return -EEXIST;
-
- netdev = get_netdev_for_sock(sk);
- if (!netdev) {
- pr_err_ratelimited("%s: netdev not found\n", __func__);
- return -EINVAL;
- }
-
- if (!(netdev->features & NETIF_F_HW_TLS_TX)) {
- rc = -EOPNOTSUPP;
- goto release_netdev;
- }
-
- crypto_info = &ctx->crypto_send.info;
- cipher_desc = get_cipher_desc(crypto_info->cipher_type);
- if (!cipher_desc || !cipher_desc->offloadable) {
- rc = -EINVAL;
- goto release_netdev;
- }
+ iv = crypto_info_iv(crypto_info, cipher_desc);
+ rec_seq = crypto_info_rec_seq(crypto_info, cipher_desc);
rc = init_prot_info(prot, crypto_info, cipher_desc);
if (rc)
- goto release_netdev;
-
- iv = crypto_info_iv(crypto_info, cipher_desc);
- rec_seq = crypto_info_rec_seq(crypto_info, cipher_desc);
+ return rc;
memcpy(ctx->tx.iv + cipher_desc->salt, iv, cipher_desc->iv);
memcpy(ctx->tx.rec_seq, rec_seq, cipher_desc->rec_seq);
start_marker_record = kmalloc_obj(*start_marker_record);
- if (!start_marker_record) {
- rc = -ENOMEM;
- goto release_netdev;
- }
+ if (!start_marker_record)
+ return -ENOMEM;
offload_ctx = alloc_offload_ctx_tx(ctx);
if (!offload_ctx) {
@@ -1159,8 +1610,10 @@ int tls_set_device_offload(struct sock *sk)
}
ctx->priv_ctx_tx = offload_ctx;
- rc = netdev->tlsdev_ops->tls_dev_add(netdev, sk, TLS_OFFLOAD_CTX_DIR_TX,
- &ctx->crypto_send.info,
+
+ rc = netdev->tlsdev_ops->tls_dev_add(netdev, sk,
+ TLS_OFFLOAD_CTX_DIR_TX,
+ crypto_info,
tcp_sk(sk)->write_seq);
trace_tls_device_offload_set(sk, TLS_OFFLOAD_CTX_DIR_TX,
tcp_sk(sk)->write_seq, rec_seq, rc);
@@ -1175,7 +1628,6 @@ int tls_set_device_offload(struct sock *sk)
* by the netdev's xmit function.
*/
smp_store_release(&sk->sk_validate_xmit_skb, tls_validate_xmit_skb);
- dev_put(netdev);
return 0;
@@ -1188,18 +1640,112 @@ int tls_set_device_offload(struct sock *sk)
ctx->priv_ctx_tx = NULL;
free_marker_record:
kfree(start_marker_record);
+ return rc;
+}
+
+static int tls_set_device_offload_rekey(struct sock *sk,
+ struct tls_context *ctx,
+ struct net_device *netdev,
+ struct tls_crypto_info *new_crypto_info)
+{
+ struct tls_offload_context_tx *offload_ctx = tls_offload_ctx_tx(ctx);
+ bool rekey_pending = test_bit(TLS_TX_REKEY_PENDING, &ctx->flags);
+ bool rekey_failed = test_bit(TLS_TX_REKEY_FAILED, &ctx->flags);
+ bool defer = true;
+ int rc;
+
+ if (!rekey_pending && !rekey_failed)
+ defer = tls_has_unacked_records(offload_ctx);
+
+ down_read(&device_offload_lock);
+
+ rc = tls_device_start_rekey(sk, ctx, offload_ctx, new_crypto_info);
+ if (rc) {
+ up_read(&device_offload_lock);
+ return rc;
+ }
+
+ up_read(&device_offload_lock);
+
+ if (!defer)
+ rc = tls_device_complete_rekey(sk, ctx);
+
+ return rc;
+}
+
+int tls_set_device_offload(struct sock *sk,
+ struct tls_crypto_info *new_crypto_info)
+{
+ struct tls_crypto_info *crypto_info, *src_crypto_info;
+ const struct tls_cipher_desc *cipher_desc;
+ struct net_device *netdev;
+ struct tls_context *ctx;
+ int rc;
+
+ ctx = tls_get_ctx(sk);
+
+ /* Rekey is only supported for connections that are already
+ * using HW offload. For SW offload connections, the caller
+ * should fall back to tls_set_sw_offload() for rekey.
+ */
+ if (new_crypto_info && ctx->tx_conf != TLS_HW)
+ return -EINVAL;
+
+ netdev = get_netdev_for_sock(sk);
+ if (!netdev) {
+ pr_err_ratelimited("%s: netdev not found\n", __func__);
+ return -EINVAL;
+ }
+
+ if (!(netdev->features & NETIF_F_HW_TLS_TX)) {
+ rc = -EOPNOTSUPP;
+ goto release_netdev;
+ }
+
+ crypto_info = &ctx->crypto_send.info;
+ src_crypto_info = new_crypto_info ?: crypto_info;
+ cipher_desc = get_cipher_desc(src_crypto_info->cipher_type);
+ if (!cipher_desc || !cipher_desc->offloadable) {
+ rc = -EINVAL;
+ goto release_netdev;
+ }
+
+ if (new_crypto_info)
+ rc = tls_set_device_offload_rekey(sk, ctx, netdev,
+ src_crypto_info);
+ else
+ rc = tls_set_device_offload_initial(sk, ctx, netdev,
+ src_crypto_info,
+ cipher_desc);
+
release_netdev:
dev_put(netdev);
return rc;
}
-int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx)
+int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx,
+ struct tls_crypto_info *new_crypto_info)
{
- struct tls12_crypto_info_aes_gcm_128 *info;
+ struct tls_crypto_info *crypto_info, *src_crypto_info;
+ const struct tls_cipher_desc *cipher_desc;
+ u32 copied_seq = tcp_sk(sk)->copied_seq;
struct tls_offload_context_rx *context;
struct net_device *netdev;
int rc = 0;
+ /* Rekey is only supported for connections that are already
+ * using HW offload. For SW offload connections, the caller
+ * should fall back to tls_set_sw_offload() for rekey.
+ */
+ if (new_crypto_info && ctx->rx_conf != TLS_HW)
+ return -EINVAL;
+
+ crypto_info = &ctx->crypto_recv.info;
+ src_crypto_info = new_crypto_info ?: crypto_info;
+ cipher_desc = get_cipher_desc(src_crypto_info->cipher_type);
+ if (!cipher_desc || !cipher_desc->offloadable)
+ return -EINVAL;
+
netdev = get_netdev_for_sock(sk);
if (!netdev) {
pr_err_ratelimited("%s: netdev not found\n", __func__);
@@ -1225,29 +1771,82 @@ int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx)
goto release_lock;
}
- context = kzalloc_obj(*context);
- if (!context) {
- rc = -ENOMEM;
- goto release_lock;
+ if (!new_crypto_info) {
+ context = kzalloc_obj(*context);
+ if (!context) {
+ rc = -ENOMEM;
+ goto release_lock;
+ }
+ ctx->priv_ctx_rx = context;
+ } else {
+ context = tls_offload_ctx_rx(ctx);
}
context->resync_nh_reset = 1;
- ctx->priv_ctx_rx = context;
- rc = tls_sw_ctx_init(sk, 0, NULL);
+ if (new_crypto_info) {
+ struct tls_sw_context_rx *sw_ctx = tls_sw_ctx_rx(ctx);
+
+ if (!test_bit(TLS_RX_DEV_CLOSED, &ctx->flags)) {
+ set_bit(TLS_RX_DEV_CLOSED, &ctx->flags);
+ synchronize_net();
+ netdev->tlsdev_ops->tls_dev_del(netdev, ctx,
+ TLS_OFFLOAD_CTX_DIR_RX);
+ }
+
+ if (context->old_aead_recv &&
+ before(copied_seq, context->old_nic_boundary)) {
+ /* Previous rekey still draining. Keep old_aead_recv,
+ * it is the only key that can undo the NIC-XOR on queued
+ * records. sw_ctx->aead_recv may be re-setkey'd by
+ * tls_sw_ctx_init(); that intermediate key was never on
+ * the NIC and its wire era is drained, so it is needed
+ * for neither undo nor AEAD. Defer dev_add; the new key
+ * is installed once copied_seq crosses old_nic_boundary.
+ */
+ context->dev_add_pending = 1;
+ } else {
+ u32 rcv_nxt;
+
+ if (context->old_aead_recv) {
+ crypto_free_aead(context->old_aead_recv);
+ context->old_aead_recv = NULL;
+ }
+
+ /* flush the backlog so rcv_nxt is accurate */
+ __sk_flush_backlog(sk);
+ rcv_nxt = tcp_sk(sk)->rcv_nxt;
+
+ if (before(copied_seq, rcv_nxt)) {
+ context->old_aead_recv = sw_ctx->aead_recv;
+ sw_ctx->aead_recv = NULL;
+ memcpy(context->old_iv, ctx->rx.iv,
+ sizeof(context->old_iv));
+ memcpy(context->old_rec_seq, ctx->rx.rec_seq,
+ sizeof(context->old_rec_seq));
+ context->old_nic_boundary = rcv_nxt;
+ context->dev_add_pending = 1;
+ }
+ trace_tls_device_rekey_start(sk, copied_seq, rcv_nxt,
+ before(copied_seq, rcv_nxt));
+ }
+ }
+
+ rc = tls_sw_ctx_init(sk, 0, new_crypto_info);
if (rc)
goto release_ctx;
- rc = netdev->tlsdev_ops->tls_dev_add(netdev, sk, TLS_OFFLOAD_CTX_DIR_RX,
- &ctx->crypto_recv.info,
- tcp_sk(sk)->copied_seq);
- info = (void *)&ctx->crypto_recv.info;
- trace_tls_device_offload_set(sk, TLS_OFFLOAD_CTX_DIR_RX,
- tcp_sk(sk)->copied_seq, info->rec_seq, rc);
- if (rc)
- goto free_sw_resources;
+ if (!context->dev_add_pending) {
+ rc = tls_device_dev_add(sk, ctx, netdev, src_crypto_info,
+ copied_seq, !!new_crypto_info);
+ if (!new_crypto_info) {
+ if (rc)
+ goto free_sw_resources;
+ tls_device_attach(ctx, sk, netdev);
+ }
+ }
+
+ tls_sw_ctx_finalize(sk, 0, new_crypto_info);
- tls_device_attach(ctx, sk, netdev);
- tls_sw_ctx_finalize(sk, 0, NULL);
up_read(&device_offload_lock);
dev_put(netdev);
@@ -1256,10 +1855,13 @@ int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx)
free_sw_resources:
up_read(&device_offload_lock);
- tls_sw_free_resources_rx(sk);
+ tls_sw_release_resources_rx(sk);
down_read(&device_offload_lock);
release_ctx:
- ctx->priv_ctx_rx = NULL;
+ if (!new_crypto_info) {
+ kfree(ctx->priv_ctx_rx);
+ ctx->priv_ctx_rx = NULL;
+ }
release_lock:
up_read(&device_offload_lock);
release_netdev:
@@ -1270,6 +1872,7 @@ int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx)
void tls_device_offload_cleanup_rx(struct sock *sk)
{
struct tls_context *tls_ctx = tls_get_ctx(sk);
+ struct tls_offload_context_rx *rx_ctx;
struct net_device *netdev;
down_read(&device_offload_lock);
@@ -1278,8 +1881,9 @@ void tls_device_offload_cleanup_rx(struct sock *sk)
if (!netdev)
goto out;
- netdev->tlsdev_ops->tls_dev_del(netdev, tls_ctx,
- TLS_OFFLOAD_CTX_DIR_RX);
+ if (!test_bit(TLS_RX_DEV_CLOSED, &tls_ctx->flags))
+ netdev->tlsdev_ops->tls_dev_del(netdev, tls_ctx,
+ TLS_OFFLOAD_CTX_DIR_RX);
if (tls_ctx->tx_conf != TLS_HW) {
dev_put(netdev);
@@ -1289,6 +1893,13 @@ void tls_device_offload_cleanup_rx(struct sock *sk)
}
out:
up_read(&device_offload_lock);
+
+ rx_ctx = tls_offload_ctx_rx(tls_ctx);
+ if (rx_ctx && rx_ctx->old_aead_recv) {
+ crypto_free_aead(rx_ctx->old_aead_recv);
+ rx_ctx->old_aead_recv = NULL;
+ }
+
tls_sw_release_resources_rx(sk);
}
@@ -1319,7 +1930,10 @@ static int tls_device_down(struct net_device *netdev)
/* Stop offloaded TX and switch to the fallback.
* tls_is_skb_tx_device_offloaded will return false.
*/
- WRITE_ONCE(ctx->sk->sk_validate_xmit_skb, tls_validate_xmit_skb_sw);
+ if (!test_bit(TLS_TX_REKEY_PENDING, &ctx->flags) &&
+ !test_bit(TLS_TX_REKEY_FAILED, &ctx->flags))
+ WRITE_ONCE(ctx->sk->sk_validate_xmit_skb,
+ tls_validate_xmit_skb_sw);
/* Stop the RX and TX resync.
* tls_dev_resync must not be called after tls_dev_del.
@@ -1336,13 +1950,18 @@ static int tls_device_down(struct net_device *netdev)
synchronize_net();
/* Release the offload context on the driver side. */
- if (ctx->tx_conf == TLS_HW)
+ if (ctx->tx_conf == TLS_HW &&
+ !test_bit(TLS_TX_DEV_CLOSED, &ctx->flags)) {
netdev->tlsdev_ops->tls_dev_del(netdev, ctx,
TLS_OFFLOAD_CTX_DIR_TX);
+ set_bit(TLS_TX_DEV_CLOSED, &ctx->flags);
+ }
if (ctx->rx_conf == TLS_HW &&
- !test_bit(TLS_RX_DEV_CLOSED, &ctx->flags))
+ !test_bit(TLS_RX_DEV_CLOSED, &ctx->flags)) {
netdev->tlsdev_ops->tls_dev_del(netdev, ctx,
TLS_OFFLOAD_CTX_DIR_RX);
+ set_bit(TLS_RX_DEV_CLOSED, &ctx->flags);
+ }
dev_put(netdev);
diff --git a/net/tls/tls_device_fallback.c b/net/tls/tls_device_fallback.c
index 1110f7ac6bcb..5be425a32c82 100644
--- a/net/tls/tls_device_fallback.c
+++ b/net/tls/tls_device_fallback.c
@@ -435,6 +435,30 @@ struct sk_buff *tls_validate_xmit_skb_sw(struct sock *sk,
return tls_sw_fallback(sk, skb);
}
+struct sk_buff *tls_validate_xmit_skb_rekey(struct sock *sk,
+ struct net_device *dev,
+ struct sk_buff *skb)
+{
+ struct tls_context *tls_ctx = tls_get_ctx(sk);
+ u32 tcp_seq = ntohl(tcp_hdr(skb)->seq);
+ u32 boundary_seq;
+
+ if (test_bit(TLS_TX_REKEY_FAILED, &tls_ctx->flags))
+ return skb;
+
+ /* If this packet is at or after the rekey boundary, it's already
+ * SW-encrypted with the new key, pass through unchanged
+ */
+ boundary_seq = READ_ONCE(tls_ctx->rekey_boundary_seq);
+ if (!before(tcp_seq, boundary_seq))
+ return skb;
+
+ /* Packet before boundary means retransmit of old data,
+ * use SW fallback with the old key
+ */
+ return tls_sw_fallback(sk, skb);
+}
+
struct sk_buff *tls_encrypt_skb(struct sk_buff *skb)
{
return tls_sw_fallback(skb->sk, skb);
diff --git a/net/tls/tls_main.c b/net/tls/tls_main.c
index fd04857fa0ab..ab701f166b57 100644
--- a/net/tls/tls_main.c
+++ b/net/tls/tls_main.c
@@ -371,6 +371,8 @@ static void tls_sk_proto_close(struct sock *sk, long timeout)
if (ctx->tx_conf == TLS_SW)
tls_sw_cancel_work_tx(ctx);
+ else if (ctx->tx_conf == TLS_HW && ctx->rekey_sw_ctx)
+ tls_sw_cancel_work_tx(ctx);
lock_sock(sk);
free_ctx = ctx->tx_conf != TLS_HW && ctx->rx_conf != TLS_HW;
@@ -711,64 +713,68 @@ static int do_tls_setsockopt_conf(struct sock *sk, sockptr_t optval,
}
if (tx) {
- if (update && ctx->tx_conf == TLS_HW) {
- rc = -EOPNOTSUPP;
- goto err_crypto_info;
- }
-
- if (!update) {
- rc = tls_set_device_offload(sk);
- conf = TLS_HW;
- if (!rc) {
+ rc = tls_set_device_offload(sk, update ? crypto_info : NULL);
+ conf = TLS_HW;
+ if (!rc) {
+ if (update) {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYOK);
+ } else {
TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXDEVICE);
TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXDEVICE);
- goto out;
}
- }
-
- rc = tls_set_sw_offload(sk, 1, update ? crypto_info : NULL);
- if (rc)
+ } else if (update && ctx->tx_conf == TLS_HW) {
+ /* HW rekey failed - return the actual error.
+ * Cannot fall back to SW for an existing HW connection.
+ */
goto err_crypto_info;
-
- if (update) {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYOK);
} else {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXSW);
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW);
+ rc = tls_set_sw_offload(sk, 1,
+ update ? crypto_info : NULL);
+ if (rc)
+ goto err_crypto_info;
+
+ if (update) {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYOK);
+ } else {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXSW);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW);
+ }
+ conf = TLS_SW;
}
- conf = TLS_SW;
} else {
- if (update && ctx->rx_conf == TLS_HW) {
- rc = -EOPNOTSUPP;
- goto err_crypto_info;
- }
-
- if (!update) {
- rc = tls_set_device_offload_rx(sk, ctx);
- conf = TLS_HW;
- if (!rc) {
+ rc = tls_set_device_offload_rx(sk, ctx,
+ update ? crypto_info : NULL);
+ conf = TLS_HW;
+ if (!rc) {
+ if (update) {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYOK);
+ } else {
TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXDEVICE);
TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXDEVICE);
- tls_sw_strparser_arm(sk, ctx);
- goto out;
}
- }
-
- rc = tls_set_sw_offload(sk, 0, update ? crypto_info : NULL);
- if (rc)
+ } else if (update && ctx->rx_conf == TLS_HW) {
+ /* HW rekey failed - return the actual error.
+ * Cannot fall back to SW for an existing HW connection.
+ */
goto err_crypto_info;
-
- if (update) {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYOK);
} else {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXSW);
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXSW);
- tls_sw_strparser_arm(sk, ctx);
+ rc = tls_set_sw_offload(sk, 0,
+ update ? crypto_info : NULL);
+ if (rc)
+ goto err_crypto_info;
+
+ if (update) {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYOK);
+ } else {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXSW);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXSW);
+ }
+ conf = TLS_SW;
}
- conf = TLS_SW;
+ if (!update)
+ tls_sw_strparser_arm(sk, ctx);
}
-out:
if (tx)
ctx->tx_conf = conf;
else
diff --git a/net/tls/tls_proc.c b/net/tls/tls_proc.c
index 4012c4372d4c..5599af306aab 100644
--- a/net/tls/tls_proc.c
+++ b/net/tls/tls_proc.c
@@ -27,6 +27,8 @@ static const struct snmp_mib tls_mib_list[] = {
SNMP_MIB_ITEM("TlsTxRekeyOk", LINUX_MIB_TLSTXREKEYOK),
SNMP_MIB_ITEM("TlsTxRekeyError", LINUX_MIB_TLSTXREKEYERROR),
SNMP_MIB_ITEM("TlsRxRekeyReceived", LINUX_MIB_TLSRXREKEYRECEIVED),
+ SNMP_MIB_ITEM("TlsTxRekeyHwFail", LINUX_MIB_TLSTXREKEYHWFAIL),
+ SNMP_MIB_ITEM("TlsRxRekeyHwFail", LINUX_MIB_TLSRXREKEYHWFAIL),
};
static int tls_statistics_seq_show(struct seq_file *seq, void *v)
diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c
index 1412b3dcce4c..fc60e8c0f24c 100644
--- a/net/tls/tls_sw.c
+++ b/net/tls/tls_sw.c
@@ -522,7 +522,7 @@ static void tls_encrypt_done(void *data, int err)
complete(&ctx->async_wait.completion);
}
-static int tls_encrypt_async_wait(struct tls_sw_context_tx *ctx)
+int tls_encrypt_async_wait(struct tls_sw_context_tx *ctx)
{
if (!atomic_dec_and_test(&ctx->encrypt_pending))
crypto_wait_req(-EINPROGRESS, &ctx->async_wait);
@@ -555,11 +555,11 @@ static int tls_do_encryption(struct sock *sk,
break;
}
- memcpy(&rec->iv_data[iv_offset], tls_ctx->tx.iv,
+ memcpy(&rec->iv_data[iv_offset], tls_tx_cipher_ctx(tls_ctx)->iv,
prot->iv_size + prot->salt_size);
tls_xor_iv_with_seq(prot, rec->iv_data + iv_offset,
- tls_ctx->tx.rec_seq);
+ tls_tx_cipher_ctx(tls_ctx)->rec_seq);
sge->offset += prot->prepend_size;
sge->length -= prot->prepend_size;
@@ -610,7 +610,7 @@ static int tls_do_encryption(struct sock *sk,
/* Unhook the record from context if encryption is not failure */
ctx->open_rec = NULL;
- tls_advance_record_sn(sk, prot, &tls_ctx->tx);
+ tls_advance_record_sn(sk, prot, tls_tx_cipher_ctx(tls_ctx));
return rc;
}
@@ -817,7 +817,7 @@ static int tls_push_record(struct sock *sk, int flags,
sg_chain(rec->sg_aead_out, 2, &msg_en->sg.data[i]);
tls_make_aad(rec->aad_space, msg_pl->sg.size + prot->tail_size,
- tls_ctx->tx.rec_seq, record_type, prot);
+ tls_tx_cipher_ctx(tls_ctx)->rec_seq, record_type, prot);
tls_fill_prepend(tls_ctx,
page_address(sg_page(&msg_en->sg.data[i])) +
@@ -982,7 +982,7 @@ static int bpf_exec_tx_verdict(struct sk_msg *msg, struct sock *sk,
return err;
}
-static int tls_sw_push_pending_record(struct sock *sk, int flags)
+int tls_sw_push_pending_record(struct sock *sk, int flags)
{
struct tls_context *tls_ctx = tls_get_ctx(sk);
struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
@@ -1033,8 +1033,7 @@ static int tls_sw_sendmsg_splice(struct sock *sk, struct msghdr *msg,
return 0;
}
-static int tls_sw_sendmsg_locked(struct sock *sk, struct msghdr *msg,
- size_t size)
+int tls_sw_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size)
{
long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
struct tls_context *tls_ctx = tls_get_ctx(sk);
@@ -1802,6 +1801,8 @@ static int tls_check_pending_rekey(struct sock *sk, struct tls_context *ctx,
if (hs_type == TLS_HANDSHAKE_KEYUPDATE) {
struct tls_sw_context_rx *rx_ctx = ctx->priv_ctx_rx;
+ /* Stop NIC from XOR-ing post-KU records with the retired key */
+ tls_device_rx_del_key(sk, ctx);
WRITE_ONCE(rx_ctx->key_update_pending, true);
TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYRECEIVED);
}
@@ -1809,6 +1810,36 @@ static int tls_check_pending_rekey(struct sock *sk, struct tls_context *ctx,
return 0;
}
+static int tls_rx_rekey_retry(struct sock *sk, struct msghdr *msg,
+ struct tls_context *tls_ctx,
+ struct tls_decrypt_arg *darg, int err)
+{
+ struct tls_offload_context_rx *rx_ctx = tls_offload_ctx_rx(tls_ctx);
+ struct tls_prot_info *prot = &tls_ctx->prot_info;
+
+ if (!rx_ctx->old_key_reencrypted)
+ return err;
+
+ if (err == -EBADMSG) {
+ if (darg->zc) {
+ struct tls_sw_context_rx *sw_ctx =
+ tls_sw_ctx_rx(tls_ctx);
+ struct strp_msg *rxm;
+
+ rxm = strp_msg(tls_strp_msg(sw_ctx));
+ iov_iter_revert(&msg->msg_iter,
+ rxm->full_len - prot->overhead_size);
+ }
+
+ err = tls_decrypt_device(sk, msg, tls_ctx, darg);
+ if (!err)
+ err = tls_decrypt_sw(sk, tls_ctx, msg, darg);
+ }
+
+ rx_ctx->old_key_reencrypted = 0;
+ return err;
+}
+
static int tls_rx_one_record(struct sock *sk, struct msghdr *msg,
struct tls_decrypt_arg *darg)
{
@@ -1820,6 +1851,10 @@ static int tls_rx_one_record(struct sock *sk, struct msghdr *msg,
err = tls_decrypt_device(sk, msg, tls_ctx, darg);
if (!err)
err = tls_decrypt_sw(sk, tls_ctx, msg, darg);
+
+ if (tls_ctx->rx_conf == TLS_HW)
+ err = tls_rx_rekey_retry(sk, msg, tls_ctx, darg, err);
+
if (err < 0)
return err;
@@ -2630,7 +2665,7 @@ void tls_sw_free_resources_rx(struct sock *sk)
}
/* The work handler to transmitt the encrypted records in tx_list */
-static void tx_work_handler(struct work_struct *work)
+void tls_tx_work_handler(struct work_struct *work)
{
struct delayed_work *delayed_work = to_delayed_work(work);
struct tx_work *tx_work = container_of(delayed_work,
@@ -2663,6 +2698,15 @@ static void tx_work_handler(struct work_struct *work)
}
}
+void tls_sw_ctx_tx_init(struct sock *sk, struct tls_sw_context_tx *sw_ctx)
+{
+ crypto_init_wait(&sw_ctx->async_wait);
+ atomic_set(&sw_ctx->encrypt_pending, 1);
+ INIT_LIST_HEAD(&sw_ctx->tx_list);
+ INIT_DELAYED_WORK(&sw_ctx->tx_work.work, tls_tx_work_handler);
+ sw_ctx->tx_work.sk = sk;
+}
+
static bool tls_is_tx_ready(struct tls_sw_context_tx *ctx)
{
struct tls_rec *rec;
@@ -2714,11 +2758,7 @@ static struct tls_sw_context_tx *init_ctx_tx(struct tls_context *ctx, struct soc
sw_ctx_tx = ctx->priv_ctx_tx;
}
- crypto_init_wait(&sw_ctx_tx->async_wait);
- atomic_set(&sw_ctx_tx->encrypt_pending, 1);
- INIT_LIST_HEAD(&sw_ctx_tx->tx_list);
- INIT_DELAYED_WORK(&sw_ctx_tx->tx_work.work, tx_work_handler);
- sw_ctx_tx->tx_work.sk = sk;
+ tls_sw_ctx_tx_init(sk, sw_ctx_tx);
return sw_ctx_tx;
}
@@ -2861,11 +2901,9 @@ int tls_sw_ctx_init(struct sock *sk, int tx,
goto free_aead;
}
- if (!new_crypto_info) {
- rc = crypto_aead_setauthsize(*aead, prot->tag_size);
- if (rc)
- goto free_aead;
- }
+ rc = crypto_aead_setauthsize(*aead, prot->tag_size);
+ if (rc)
+ goto free_aead;
if (!tx && !new_crypto_info) {
tfm = crypto_aead_tfm(sw_ctx_rx->aead_recv);
diff --git a/net/tls/trace.h b/net/tls/trace.h
index 2d8ce4ff3265..56fcf95c5aaf 100644
--- a/net/tls/trace.h
+++ b/net/tls/trace.h
@@ -192,6 +192,85 @@ TRACE_EVENT(tls_device_tx_resync_send,
)
);
+TRACE_EVENT(tls_device_rekey_start,
+
+ TP_PROTO(struct sock *sk, u32 copied_seq, u32 nic_boundary,
+ bool inflight),
+
+ TP_ARGS(sk, copied_seq, nic_boundary, inflight),
+
+ TP_STRUCT__entry(
+ __field( struct sock *, sk )
+ __field( u32, copied_seq )
+ __field( u32, nic_boundary )
+ __field( bool, inflight )
+ ),
+
+ TP_fast_assign(
+ __entry->sk = sk;
+ __entry->copied_seq = copied_seq;
+ __entry->nic_boundary = nic_boundary;
+ __entry->inflight = inflight;
+ ),
+
+ TP_printk(
+ "sk=%p copied_seq=%u nic_boundary=%u inflight=%d",
+ __entry->sk, __entry->copied_seq, __entry->nic_boundary,
+ __entry->inflight
+ )
+);
+
+TRACE_EVENT(tls_device_rekey_reencrypt,
+
+ TP_PROTO(struct sock *sk, u32 tcp_seq, u32 nic_boundary, bool retry),
+
+ TP_ARGS(sk, tcp_seq, nic_boundary, retry),
+
+ TP_STRUCT__entry(
+ __field( struct sock *, sk )
+ __field( u32, tcp_seq )
+ __field( u32, nic_boundary )
+ __field( bool, retry )
+ ),
+
+ TP_fast_assign(
+ __entry->sk = sk;
+ __entry->tcp_seq = tcp_seq;
+ __entry->nic_boundary = nic_boundary;
+ __entry->retry = retry;
+ ),
+
+ TP_printk(
+ "sk=%p tcp_seq=%u nic_boundary=%u retry=%d",
+ __entry->sk, __entry->tcp_seq, __entry->nic_boundary,
+ __entry->retry
+ )
+);
+
+TRACE_EVENT(tls_device_rekey_done,
+
+ TP_PROTO(struct sock *sk, u32 tcp_seq, u32 nic_boundary),
+
+ TP_ARGS(sk, tcp_seq, nic_boundary),
+
+ TP_STRUCT__entry(
+ __field( struct sock *, sk )
+ __field( u32, tcp_seq )
+ __field( u32, nic_boundary )
+ ),
+
+ TP_fast_assign(
+ __entry->sk = sk;
+ __entry->tcp_seq = tcp_seq;
+ __entry->nic_boundary = nic_boundary;
+ ),
+
+ TP_printk(
+ "sk=%p tcp_seq=%u nic_boundary=%u",
+ __entry->sk, __entry->tcp_seq, __entry->nic_boundary
+ )
+);
+
#endif /* _TLS_TRACE_H_ */
#undef TRACE_INCLUDE_PATH
--
2.25.1
^ permalink raw reply related [flat|nested] 9+ messages in thread* Re: [PATCH v13 5/6] tls: add hardware offload key update support
2026-04-29 18:10 ` [PATCH v13 5/6] tls: add hardware offload key update support Rishikesh Jethwani
@ 2026-05-05 8:40 ` Paolo Abeni
2026-05-05 8:41 ` Paolo Abeni
0 siblings, 1 reply; 9+ messages in thread
From: Paolo Abeni @ 2026-05-05 8:40 UTC (permalink / raw)
To: Rishikesh Jethwani, netdev
Cc: saeedm, tariqt, mbloch, borisp, john.fastabend, kuba, sd, davem,
edumazet, leon
On 4/29/26 8:10 PM, Rishikesh Jethwani wrote:
> On TX, the NIC key cannot be replaced while HW-offloaded records
> are still unacked. tls_dev_start_rekey() installs a temporary SW
> context with the new key and redirects sendmsg through
> tls_sw_sendmsg_locked. If no records are pending,
> tls_dev_complete_rekey() runs inline during setsockopt; otherwise
> clean_acked sets REKEY_READY once all old-key records are ACKed
> and the next sendmsg completes the rekey, flushing SW records and
> reinstalling HW offload at the current write_seq. A KeyUpdate
> arriving while one is pending re-keys the SW AEAD in place; if the
> HW reinstall fails the socket stays in SW mode (REKEY_FAILED).
>
> On RX, the NIC may have already decrypted in-flight records with
> the old key before the peer's KeyUpdate is parsed, so the old
> AEAD, IV and rec_seq are retained on tls_offload_context_rx.
> tls_check_pending_rekey() invokes tls_device_rx_del_key() to drop
> the NIC key; otherwise post-KeyUpdate records (carrying new-key
> wire encryption) would be XOR'd with the retired key.
> tls_device_decrypted() classifies records by old_nic_boundary:
>
> - after the boundary: new-key record; drop the old key.
> - before, fully encrypted: advance old_rec_seq, let SW AEAD decrypt.
> - before, (partially) decrypted: reencrypt with the old key so SW
> AEAD can decrypt with the new key.
>
> For mixed records skb->decrypted flags can be wrong (NIC clears
> them on auth failure); on -EBADMSG, tls_rx_rekey_retry() toggles
> those flags, decrements old_rec_seq to reuse the nonce, and
> retries once (gated by old_key_reencrypted).
>
> The new key's tls_dev_add is deferred until the old key is fully
> consumed: tls_set_device_offload_rx() sets dev_add_pending while
> old_aead_recv is retained, and tls_device_deferred_dev_add()
> installs the new key once copied_seq crosses old_nic_boundary.
>
> Tested on Mellanox ConnectX-6 Dx (Crypto Enabled) with multiple
> TLS 1.3 TX and RX KeyUpdate cycles.
>
> Signed-off-by: Rishikesh Jethwani <rjethwani@purestorage.com>
> ---
> include/net/tls.h | 84 +++-
> include/uapi/linux/snmp.h | 2 +
> net/tls/tls.h | 29 +-
> net/tls/tls_device.c | 753 +++++++++++++++++++++++++++++++---
> net/tls/tls_device_fallback.c | 24 ++
> net/tls/tls_main.c | 92 +++--
> net/tls/tls_proc.c | 2 +
> net/tls/tls_sw.c | 76 +++-
> net/tls/trace.h | 79 ++++
This patch is really big and complex and you should break it to help
reviewers.
At very least you can split out the tracing bits and the trivial
refactor moving around declaration and definitions to separate patches.
/P
^ permalink raw reply [flat|nested] 9+ messages in thread
* Re: [PATCH v13 5/6] tls: add hardware offload key update support
2026-05-05 8:40 ` Paolo Abeni
@ 2026-05-05 8:41 ` Paolo Abeni
0 siblings, 0 replies; 9+ messages in thread
From: Paolo Abeni @ 2026-05-05 8:41 UTC (permalink / raw)
To: Rishikesh Jethwani, netdev
Cc: saeedm, tariqt, mbloch, borisp, john.fastabend, kuba, sd, davem,
edumazet, leon
On 5/5/26 10:40 AM, Paolo Abeni wrote:
> On 4/29/26 8:10 PM, Rishikesh Jethwani wrote:
>> On TX, the NIC key cannot be replaced while HW-offloaded records
>> are still unacked. tls_dev_start_rekey() installs a temporary SW
>> context with the new key and redirects sendmsg through
>> tls_sw_sendmsg_locked. If no records are pending,
>> tls_dev_complete_rekey() runs inline during setsockopt; otherwise
>> clean_acked sets REKEY_READY once all old-key records are ACKed
>> and the next sendmsg completes the rekey, flushing SW records and
>> reinstalling HW offload at the current write_seq. A KeyUpdate
>> arriving while one is pending re-keys the SW AEAD in place; if the
>> HW reinstall fails the socket stays in SW mode (REKEY_FAILED).
>>
>> On RX, the NIC may have already decrypted in-flight records with
>> the old key before the peer's KeyUpdate is parsed, so the old
>> AEAD, IV and rec_seq are retained on tls_offload_context_rx.
>> tls_check_pending_rekey() invokes tls_device_rx_del_key() to drop
>> the NIC key; otherwise post-KeyUpdate records (carrying new-key
>> wire encryption) would be XOR'd with the retired key.
>> tls_device_decrypted() classifies records by old_nic_boundary:
>>
>> - after the boundary: new-key record; drop the old key.
>> - before, fully encrypted: advance old_rec_seq, let SW AEAD decrypt.
>> - before, (partially) decrypted: reencrypt with the old key so SW
>> AEAD can decrypt with the new key.
>>
>> For mixed records skb->decrypted flags can be wrong (NIC clears
>> them on auth failure); on -EBADMSG, tls_rx_rekey_retry() toggles
>> those flags, decrements old_rec_seq to reuse the nonce, and
>> retries once (gated by old_key_reencrypted).
>>
>> The new key's tls_dev_add is deferred until the old key is fully
>> consumed: tls_set_device_offload_rx() sets dev_add_pending while
>> old_aead_recv is retained, and tls_device_deferred_dev_add()
>> installs the new key once copied_seq crosses old_nic_boundary.
>>
>> Tested on Mellanox ConnectX-6 Dx (Crypto Enabled) with multiple
>> TLS 1.3 TX and RX KeyUpdate cycles.
>>
>> Signed-off-by: Rishikesh Jethwani <rjethwani@purestorage.com>
>> ---
>> include/net/tls.h | 84 +++-
>> include/uapi/linux/snmp.h | 2 +
>> net/tls/tls.h | 29 +-
>> net/tls/tls_device.c | 753 +++++++++++++++++++++++++++++++---
>> net/tls/tls_device_fallback.c | 24 ++
>> net/tls/tls_main.c | 92 +++--
>> net/tls/tls_proc.c | 2 +
>> net/tls/tls_sw.c | 76 +++-
>> net/tls/trace.h | 79 ++++
>
> This patch is really big and complex and you should break it to help
> reviewers.
>
> At very least you can split out the tracing bits and the trivial
> refactor moving around declaration and definitions to separate patches.
I should have mentioned the following in my previous email:
please wait for more feedback from Sabrina and/or Jakub before posting a
new revision.
Thanks,
Paolo
^ permalink raw reply [flat|nested] 9+ messages in thread
* [PATCH v13 6/6] selftests: net: add TLS hardware offload test
2026-04-29 18:10 [PATCH net-next v13 0/6] tls: Add TLS 1.3 hardware offload support Rishikesh Jethwani
` (4 preceding siblings ...)
2026-04-29 18:10 ` [PATCH v13 5/6] tls: add hardware offload key update support Rishikesh Jethwani
@ 2026-04-29 18:10 ` Rishikesh Jethwani
5 siblings, 0 replies; 9+ messages in thread
From: Rishikesh Jethwani @ 2026-04-29 18:10 UTC (permalink / raw)
To: netdev
Cc: saeedm, tariqt, mbloch, borisp, john.fastabend, kuba, sd, davem,
pabeni, edumazet, leon, Rishikesh Jethwani
Two-node kTLS hardware offload test using NetDrvEpEnv. Tests TLS
1.2/1.3 with AES-GCM-128/256, rekey operations, and various buffer
sizes.
Signed-off-by: Rishikesh Jethwani <rjethwani@purestorage.com>
---
MAINTAINERS | 2 +
.../selftests/drivers/net/hw/.gitignore | 1 +
.../testing/selftests/drivers/net/hw/Makefile | 2 +
.../selftests/drivers/net/hw/tls_hw_offload.c | 887 ++++++++++++++++++
.../drivers/net/hw/tls_hw_offload.py | 256 +++++
5 files changed, 1148 insertions(+)
create mode 100644 tools/testing/selftests/drivers/net/hw/tls_hw_offload.c
create mode 100755 tools/testing/selftests/drivers/net/hw/tls_hw_offload.py
diff --git a/MAINTAINERS b/MAINTAINERS
index 2fb1c75afd16..aedf42890094 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -18776,6 +18776,8 @@ F: Documentation/networking/tls*
F: include/net/tls.h
F: include/uapi/linux/tls.h
F: net/tls/
+F: tools/testing/selftests/drivers/net/hw/tls_hw_offload.c
+F: tools/testing/selftests/drivers/net/hw/tls_hw_offload.py
F: tools/testing/selftests/net/tls.c
NETWORKING [SOCKETS]
diff --git a/tools/testing/selftests/drivers/net/hw/.gitignore b/tools/testing/selftests/drivers/net/hw/.gitignore
index 46540468a775..f0a5d15b469b 100644
--- a/tools/testing/selftests/drivers/net/hw/.gitignore
+++ b/tools/testing/selftests/drivers/net/hw/.gitignore
@@ -2,3 +2,4 @@
iou-zcrx
ncdevmem
toeplitz
+tls_hw_offload
diff --git a/tools/testing/selftests/drivers/net/hw/Makefile b/tools/testing/selftests/drivers/net/hw/Makefile
index 85ca4d1ecf9e..2dd633619f40 100644
--- a/tools/testing/selftests/drivers/net/hw/Makefile
+++ b/tools/testing/selftests/drivers/net/hw/Makefile
@@ -15,6 +15,7 @@ endif
TEST_GEN_FILES := \
$(COND_GEN_FILES) \
+ tls_hw_offload \
# end of TEST_GEN_FILES
TEST_PROGS = \
@@ -43,6 +44,7 @@ TEST_PROGS = \
rss_drv.py \
rss_flow_label.py \
rss_input_xfrm.py \
+ tls_hw_offload.py \
toeplitz.py \
tso.py \
uso.py \
diff --git a/tools/testing/selftests/drivers/net/hw/tls_hw_offload.c b/tools/testing/selftests/drivers/net/hw/tls_hw_offload.c
new file mode 100644
index 000000000000..db7c61d8b4e7
--- /dev/null
+++ b/tools/testing/selftests/drivers/net/hw/tls_hw_offload.c
@@ -0,0 +1,887 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * TLS Hardware Offload Two-Node Test
+ *
+ * Tests kTLS hardware offload between two physical nodes using
+ * hardcoded keys. Supports TLS 1.2/1.3, AES-GCM-128/256, and rekey.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <time.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+#include <arpa/inet.h>
+#include <linux/tls.h>
+
+#define TLS_RECORD_TYPE_HANDSHAKE 22
+#define TLS_HANDSHAKE_KEY_UPDATE 0x18
+
+#define MIN_BUF_SIZE 16
+
+/* Initial key material */
+static struct tls12_crypto_info_aes_gcm_128 tls_info_key0_128 = {
+ .info = {
+ .version = TLS_1_3_VERSION,
+ .cipher_type = TLS_CIPHER_AES_GCM_128,
+ },
+ .iv = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08 },
+ .key = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
+ 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10 },
+ .salt = { 0x01, 0x02, 0x03, 0x04 },
+ .rec_seq = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+};
+
+static struct tls12_crypto_info_aes_gcm_256 tls_info_key0_256 = {
+ .info = {
+ .version = TLS_1_3_VERSION,
+ .cipher_type = TLS_CIPHER_AES_GCM_256,
+ },
+ .iv = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08 },
+ .key = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
+ 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10,
+ 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18,
+ 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20 },
+ .salt = { 0x01, 0x02, 0x03, 0x04 },
+ .rec_seq = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+};
+
+static int num_rekeys;
+static int num_iterations = 100;
+static int cipher_type = TLS_CIPHER_AES_GCM_128;
+static int tls_version = TLS_1_3_VERSION;
+static int server_port = 4433;
+static char *server_ip;
+
+static int send_size = 16384;
+static int random_size_max;
+/* Burst mode: sender keeps pushing records without reading from the peer;
+ * receiver drains without echoing back. Only the client initiates rekey.
+ */
+static int burst_mode;
+static int zc_rx;
+
+/* XOR each byte with the generation so both endpoints derive the
+ * same per-generation key without a real KDF. Generation 0 leaves
+ * the base key unchanged.
+ */
+static void derive_key_fields(unsigned char *key, int key_size,
+ unsigned char *iv, int iv_size,
+ unsigned char *salt, int salt_size,
+ unsigned char *rec_seq, int rec_seq_size,
+ int generation)
+{
+ int i;
+
+ for (i = 0; i < key_size; i++)
+ key[i] ^= generation;
+ for (i = 0; i < iv_size; i++)
+ iv[i] ^= generation;
+ for (i = 0; i < salt_size; i++)
+ salt[i] ^= generation;
+ memset(rec_seq, 0, rec_seq_size);
+}
+
+static void derive_key_128(struct tls12_crypto_info_aes_gcm_128 *key,
+ int generation)
+{
+ memcpy(key, &tls_info_key0_128, sizeof(*key));
+ key->info.version = tls_version;
+ derive_key_fields(key->key, TLS_CIPHER_AES_GCM_128_KEY_SIZE,
+ key->iv, TLS_CIPHER_AES_GCM_128_IV_SIZE,
+ key->salt, TLS_CIPHER_AES_GCM_128_SALT_SIZE,
+ key->rec_seq, TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE,
+ generation);
+}
+
+static void derive_key_256(struct tls12_crypto_info_aes_gcm_256 *key,
+ int generation)
+{
+ memcpy(key, &tls_info_key0_256, sizeof(*key));
+ key->info.version = tls_version;
+ derive_key_fields(key->key, TLS_CIPHER_AES_GCM_256_KEY_SIZE,
+ key->iv, TLS_CIPHER_AES_GCM_256_IV_SIZE,
+ key->salt, TLS_CIPHER_AES_GCM_256_SALT_SIZE,
+ key->rec_seq, TLS_CIPHER_AES_GCM_256_REC_SEQ_SIZE,
+ generation);
+}
+
+static const char *cipher_name(int cipher)
+{
+ switch (cipher) {
+ case TLS_CIPHER_AES_GCM_128: return "AES-GCM-128";
+ case TLS_CIPHER_AES_GCM_256: return "AES-GCM-256";
+ default: return "unknown";
+ }
+}
+
+static const char *version_name(int version)
+{
+ switch (version) {
+ case TLS_1_2_VERSION: return "TLS 1.2";
+ case TLS_1_3_VERSION: return "TLS 1.3";
+ default: return "unknown";
+ }
+}
+
+static int setup_tls_ulp(int fd)
+{
+ int ret;
+
+ ret = setsockopt(fd, IPPROTO_TCP, TCP_ULP, "tls", sizeof("tls"));
+ if (ret < 0) {
+ printf("SETUP ERROR: TCP_ULP failed: %s\n", strerror(errno));
+ return -1;
+ }
+ return 0;
+}
+
+static int set_zc_rx(int fd)
+{
+ int val = 1;
+
+ if (setsockopt(fd, SOL_TLS, TLS_RX_EXPECT_NO_PAD, &val,
+ sizeof(val)) < 0) {
+ printf("SETUP ERROR: TLS_RX_EXPECT_NO_PAD failed: %s\n",
+ strerror(errno));
+ return -1;
+ }
+ return 0;
+}
+
+/* Send a TLS 1.3 KeyUpdate handshake record. The kernel only
+ * inspects the HandshakeType byte to detect KeyUpdate, so don't
+ * bother with the 3-byte length or request_update fields.
+ */
+static int send_tls_key_update(int fd)
+{
+ char cmsg_buf[CMSG_SPACE(sizeof(unsigned char))];
+ unsigned char key_update_msg = TLS_HANDSHAKE_KEY_UPDATE;
+ struct msghdr msg = {0};
+ struct cmsghdr *cmsg;
+ struct iovec iov;
+
+ iov.iov_base = &key_update_msg;
+ iov.iov_len = sizeof(key_update_msg);
+
+ msg.msg_iov = &iov;
+ msg.msg_iovlen = 1;
+ msg.msg_control = cmsg_buf;
+ msg.msg_controllen = sizeof(cmsg_buf);
+
+ cmsg = CMSG_FIRSTHDR(&msg);
+ cmsg->cmsg_level = SOL_TLS;
+ cmsg->cmsg_type = TLS_SET_RECORD_TYPE;
+ cmsg->cmsg_len = CMSG_LEN(sizeof(unsigned char));
+ *CMSG_DATA(cmsg) = TLS_RECORD_TYPE_HANDSHAKE;
+ msg.msg_controllen = cmsg->cmsg_len;
+
+ if (sendmsg(fd, &msg, 0) < 0) {
+ printf("sendmsg KeyUpdate failed: %s\n", strerror(errno));
+ return -1;
+ }
+
+ printf("Sent TLS KeyUpdate handshake message\n");
+ return 0;
+}
+
+static int recv_tls_message(int fd, char *buf, size_t buflen, int *record_type)
+{
+ char cmsg_buf[CMSG_SPACE(sizeof(unsigned char))];
+ struct msghdr msg = {0};
+ struct cmsghdr *cmsg;
+ struct iovec iov;
+ int ret;
+
+ iov.iov_base = buf;
+ iov.iov_len = buflen;
+
+ msg.msg_iov = &iov;
+ msg.msg_iovlen = 1;
+ msg.msg_control = cmsg_buf;
+ msg.msg_controllen = sizeof(cmsg_buf);
+
+ ret = recvmsg(fd, &msg, 0);
+ if (ret <= 0)
+ return ret;
+
+ cmsg = CMSG_FIRSTHDR(&msg);
+ if (cmsg && cmsg->cmsg_level == SOL_TLS &&
+ cmsg->cmsg_type == TLS_GET_RECORD_TYPE)
+ *record_type = *((unsigned char *)CMSG_DATA(cmsg));
+
+ return ret;
+}
+
+/* Confirm a handshake record starting with HandshakeType KeyUpdate. */
+static int check_keyupdate(const char *buf, int len, int record_type)
+{
+ if (record_type != TLS_RECORD_TYPE_HANDSHAKE) {
+ printf("Expected handshake record (0x%02x), got 0x%02x\n",
+ TLS_RECORD_TYPE_HANDSHAKE, record_type);
+ return -1;
+ }
+ if (len < 1 || (unsigned char)buf[0] != TLS_HANDSHAKE_KEY_UPDATE) {
+ printf("Expected KeyUpdate (0x%02x), got 0x%02x\n",
+ TLS_HANDSHAKE_KEY_UPDATE,
+ len ? (unsigned char)buf[0] : 0);
+ return -1;
+ }
+ printf("Received TLS KeyUpdate\n");
+ return 0;
+}
+
+static int recv_tls_keyupdate(int fd)
+{
+ char buf[MIN_BUF_SIZE];
+ int record_type = 0;
+ int ret;
+
+ ret = recv_tls_message(fd, buf, sizeof(buf), &record_type);
+ if (ret < 0) {
+ printf("recv_tls_message failed: %s\n", strerror(errno));
+ return -1;
+ }
+
+ return check_keyupdate(buf, ret, record_type);
+}
+
+static int check_ekeyexpired(int fd)
+{
+ char buf[MIN_BUF_SIZE];
+ int ret;
+
+ ret = recv(fd, buf, sizeof(buf), MSG_DONTWAIT);
+ if (ret == -1 && errno == EKEYEXPIRED) {
+ printf("recv() returned EKEYEXPIRED as expected\n");
+ return 0;
+ } else if (ret == -1 && errno == EAGAIN) {
+ printf("recv() returned EAGAIN (no pending data)\n");
+ return 0;
+ } else if (ret > 0) {
+ printf("FAIL: recv() returned %d bytes, expected EKEYEXPIRED\n",
+ ret);
+ return -1;
+ } else {
+ printf("FAIL: recv() returned unexpected error: %s\n",
+ strerror(errno));
+ return -1;
+ }
+}
+
+static int do_tls_rekey(int fd, int direction, int generation, int cipher)
+{
+ const char *dir = direction == TLS_TX ? "TX" : "RX";
+ int ret;
+
+ printf("%s TLS_%s %s gen %d...\n",
+ generation ? "Rekeying" : "Installing",
+ dir, cipher_name(cipher), generation);
+
+ if (cipher == TLS_CIPHER_AES_GCM_256) {
+ struct tls12_crypto_info_aes_gcm_256 key;
+
+ derive_key_256(&key, generation);
+ ret = setsockopt(fd, SOL_TLS, direction, &key, sizeof(key));
+ } else {
+ struct tls12_crypto_info_aes_gcm_128 key;
+
+ derive_key_128(&key, generation);
+ ret = setsockopt(fd, SOL_TLS, direction, &key, sizeof(key));
+ }
+
+ if (ret < 0) {
+ printf("%sTLS_%s %s gen %d failed: %s\n",
+ generation ? "" : "SETUP ERROR: ", dir,
+ cipher_name(cipher), generation, strerror(errno));
+ return -1;
+ }
+ printf("TLS_%s %s gen %d installed\n",
+ dir, cipher_name(cipher), generation);
+ return 0;
+}
+
+static int do_client(void)
+{
+ char *buf = NULL, *echo_buf = NULL;
+ int max_size, rekey_interval;
+ ssize_t echo_total, echo_n;
+ int csk = -1, ret, i, j;
+ struct sockaddr_in sa;
+ int test_result = -1;
+ int current_gen = 0;
+ int next_rekey_at;
+ ssize_t n;
+
+ max_size = random_size_max > 0 ? random_size_max : send_size;
+ if (max_size < MIN_BUF_SIZE)
+ max_size = MIN_BUF_SIZE;
+ buf = malloc(max_size);
+ if (!burst_mode)
+ echo_buf = malloc(max_size);
+ if (!buf || (!burst_mode && !echo_buf)) {
+ printf("SETUP ERROR: failed to allocate buffers\n");
+ goto out;
+ }
+
+ csk = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
+ if (csk < 0) {
+ printf("SETUP ERROR: failed to create socket: %s\n",
+ strerror(errno));
+ goto out;
+ }
+
+ memset(&sa, 0, sizeof(sa));
+ sa.sin_family = AF_INET;
+ sa.sin_addr.s_addr = inet_addr(server_ip);
+ sa.sin_port = htons(server_port);
+ printf("Connecting to %s:%d...\n", server_ip, server_port);
+
+ ret = connect(csk, (struct sockaddr *)&sa, sizeof(sa));
+ if (ret < 0) {
+ printf("SETUP ERROR: connect failed: %s\n", strerror(errno));
+ goto out;
+ }
+ printf("Connected!\n");
+
+ if (setup_tls_ulp(csk) < 0)
+ goto out;
+
+ if (do_tls_rekey(csk, TLS_TX, 0, cipher_type) < 0 ||
+ do_tls_rekey(csk, TLS_RX, 0, cipher_type) < 0)
+ goto out;
+
+ if (num_rekeys)
+ printf("TLS %s setup complete. Will perform %d rekey(s).\n",
+ cipher_name(cipher_type), num_rekeys);
+ else
+ printf("TLS setup complete.\n");
+
+ if (random_size_max > 0)
+ printf("Sending %d messages of random size (1..%d bytes)...\n",
+ num_iterations, random_size_max);
+ else
+ printf("Sending %d messages of %d bytes...\n",
+ num_iterations, send_size);
+
+ rekey_interval = num_iterations / (num_rekeys + 1);
+ next_rekey_at = rekey_interval;
+
+ for (i = 1; i <= num_iterations; i++) {
+ int this_size;
+
+ if (random_size_max > 0)
+ this_size = (rand() % random_size_max) + 1;
+ else
+ this_size = send_size;
+
+ /* In burst mode, use a per-iteration fill pattern so the
+ * receiver can detect any plaintext corruption without a
+ * round-trip echo.
+ */
+ if (burst_mode) {
+ memset(buf, i & 0xFF, this_size);
+ } else {
+ for (j = 0; j < this_size; j++)
+ buf[j] = rand() & 0xFF;
+ }
+
+ n = send(csk, buf, this_size, 0);
+ if (n != this_size) {
+ printf("FAIL: send failed: %s\n", strerror(errno));
+ goto out;
+ }
+ /* Throttle per-iteration progress lines on long burst runs so
+ * stdout over ssh doesn't become the bottleneck.
+ */
+ if (!burst_mode || num_iterations <= 1000 || (i % 1000) == 0 ||
+ i == num_iterations)
+ printf("Sent %zd bytes (iteration %d)\n", n, i);
+
+ if (!burst_mode) {
+ echo_total = 0;
+ while (echo_total < n) {
+ echo_n = recv(csk, echo_buf + echo_total,
+ n - echo_total, 0);
+ if (echo_n < 0) {
+ printf("FAIL: Echo recv failed: %s\n",
+ strerror(errno));
+ goto out;
+ }
+ if (echo_n == 0) {
+ printf("FAIL: Connection closed during echo\n");
+ goto out;
+ }
+ echo_total += echo_n;
+ }
+
+ if (memcmp(buf, echo_buf, n) != 0) {
+ printf("FAIL: Echo data mismatch!\n");
+ goto out;
+ }
+ printf("Received echo %zd bytes (ok)\n", echo_total);
+ }
+
+ /* Rekey at intervals. In echo mode this is a full bidirectional
+ * exchange; in burst mode the client only rotates its TX key
+ * and sends KeyUpdate - the peer is expected to follow.
+ */
+ if (num_rekeys && current_gen < num_rekeys &&
+ i == next_rekey_at) {
+ current_gen++;
+ printf("\n=== Client Rekey gen %d ===\n", current_gen);
+
+ ret = send_tls_key_update(csk);
+ if (ret < 0) {
+ printf("FAIL: send KeyUpdate\n");
+ goto out;
+ }
+
+ ret = do_tls_rekey(csk, TLS_TX, current_gen, cipher_type);
+ if (ret < 0)
+ goto out;
+
+ if (!burst_mode) {
+ if (recv_tls_keyupdate(csk) < 0) {
+ printf("FAIL: recv KeyUpdate from server\n");
+ goto out;
+ }
+
+ if (check_ekeyexpired(csk) < 0)
+ goto out;
+
+ ret = do_tls_rekey(csk, TLS_RX, current_gen,
+ cipher_type);
+ if (ret < 0)
+ goto out;
+ }
+
+ next_rekey_at += rekey_interval;
+ printf("=== Client Rekey gen %d Complete ===\n\n",
+ current_gen);
+ }
+ }
+
+ test_result = 0;
+out:
+ if (num_rekeys)
+ printf("Rekeys completed: %d/%d\n", current_gen, num_rekeys);
+ if (csk >= 0)
+ close(csk);
+ free(buf);
+ free(echo_buf);
+ return test_result;
+}
+
+static int do_server(void)
+{
+ int lsk = -1, csk = -1, ret;
+ ssize_t n, total = 0, sent;
+ /* Burst-mode data verification state: client fills each iteration's
+ * send_size-byte block with (send_iter & 0xff). A single recv may
+ * return part of a block or span iterations, so track position as
+ * (send_iter, remaining) - bytes left in the current block.
+ */
+ int send_iter = 1;
+ int remaining = send_size;
+ struct sockaddr_in sa;
+ int test_result = -1;
+ int current_gen = 0;
+ int recv_count = 0;
+ char *buf = NULL;
+ int record_type;
+ int buf_size;
+ int one = 1;
+
+ buf_size = send_size;
+ if (buf_size < MIN_BUF_SIZE)
+ buf_size = MIN_BUF_SIZE;
+ buf = malloc(buf_size);
+ if (!buf) {
+ printf("SETUP ERROR: failed to allocate buffer\n");
+ goto out;
+ }
+
+ lsk = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
+ if (lsk < 0) {
+ printf("SETUP ERROR: failed to create socket: %s\n",
+ strerror(errno));
+ goto out;
+ }
+
+ setsockopt(lsk, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one));
+
+ memset(&sa, 0, sizeof(sa));
+ sa.sin_family = AF_INET;
+ sa.sin_addr.s_addr = INADDR_ANY;
+ sa.sin_port = htons(server_port);
+
+ ret = bind(lsk, (struct sockaddr *)&sa, sizeof(sa));
+ if (ret < 0) {
+ printf("SETUP ERROR: bind failed: %s\n", strerror(errno));
+ goto out;
+ }
+
+ ret = listen(lsk, 1);
+ if (ret < 0) {
+ printf("SETUP ERROR: listen failed: %s\n", strerror(errno));
+ goto out;
+ }
+
+ printf("Server listening on 0.0.0.0:%d\n", server_port);
+ printf("Waiting for client connection...\n");
+
+ csk = accept(lsk, (struct sockaddr *)NULL, (socklen_t *)NULL);
+ if (csk < 0) {
+ printf("SETUP ERROR: accept failed: %s\n", strerror(errno));
+ goto out;
+ }
+ printf("Client connected!\n");
+
+ if (setup_tls_ulp(csk) < 0)
+ goto out;
+
+ if (do_tls_rekey(csk, TLS_TX, 0, cipher_type) < 0 ||
+ do_tls_rekey(csk, TLS_RX, 0, cipher_type) < 0)
+ goto out;
+
+ if (zc_rx && set_zc_rx(csk) < 0)
+ goto out;
+
+ printf("TLS %s setup complete. Receiving...\n",
+ cipher_name(cipher_type));
+
+ /* Main receive loop */
+ while (1) {
+ n = recv_tls_message(csk, buf, buf_size, &record_type);
+ if (n == 0) {
+ printf("Connection closed by client\n");
+ break;
+ }
+ if (n < 0) {
+ printf("FAIL: recv failed: %s\n", strerror(errno));
+ goto out;
+ }
+
+ /* Handle KeyUpdate. In echo mode the server mirrors the
+ * rekey back to the peer; in burst mode it only rotates
+ * its RX key and keeps draining.
+ */
+ if (record_type == TLS_RECORD_TYPE_HANDSHAKE) {
+ if (check_keyupdate(buf, n, record_type) < 0)
+ goto out;
+ current_gen++;
+ printf("\n=== Server Rekey gen %d ===\n", current_gen);
+
+ if (check_ekeyexpired(csk) < 0)
+ goto out;
+
+ ret = do_tls_rekey(csk, TLS_RX, current_gen, cipher_type);
+ if (ret < 0)
+ goto out;
+
+ if (!burst_mode) {
+ ret = send_tls_key_update(csk);
+ if (ret < 0) {
+ printf("FAIL: send KeyUpdate\n");
+ goto out;
+ }
+
+ ret = do_tls_rekey(csk, TLS_TX, current_gen,
+ cipher_type);
+ if (ret < 0)
+ goto out;
+ }
+
+ printf("=== Server Rekey gen %d Complete ===\n\n",
+ current_gen);
+ continue;
+ }
+
+ total += n;
+ recv_count++;
+ /* Throttle per-record progress lines on long burst runs. */
+ if (!burst_mode || (recv_count % 1000) == 0)
+ printf("Received %zd bytes (total: %zd, count: %d)\n",
+ n, total, recv_count);
+
+ /* Burst mode: verify payload matches the client's fill
+ * pattern. TLS record boundaries may differ from send()
+ * boundaries, so walk the received buffer in chunks that
+ * fit within the current iteration's remaining bytes.
+ * Catches decrypt-succeeded-but-plaintext-corrupt bugs
+ * that AEAD counters alone would miss.
+ */
+ if (burst_mode) {
+ int off = 0;
+
+ while (off < n) {
+ unsigned char expect = send_iter & 0xFF;
+ int chunk = n - off;
+ int j;
+
+ if (chunk > remaining)
+ chunk = remaining;
+
+ for (j = 0; j < chunk; j++) {
+ if ((unsigned char)buf[off + j] != expect) {
+ printf("FAIL: data mismatch recv #%d offset %d:"
+ " expected 0x%02x got 0x%02x"
+ " (iter %d, remaining %d)\n",
+ recv_count, off + j,
+ expect,
+ (unsigned char)buf[off + j],
+ send_iter, remaining);
+ goto out;
+ }
+ }
+
+ off += chunk;
+ remaining -= chunk;
+ if (remaining == 0) {
+ send_iter++;
+ remaining = send_size;
+ }
+ }
+ continue;
+ }
+
+ for (sent = 0; sent < n; sent += ret) {
+ ret = send(csk, buf + sent, n - sent, 0);
+ if (ret < 0) {
+ printf("FAIL: Echo send failed: %s\n",
+ strerror(errno));
+ goto out;
+ }
+ }
+ printf("Echoed %zd bytes back to client\n", n);
+ }
+
+ test_result = 0;
+out:
+ printf("Connection closed. Total received: %zd bytes\n", total);
+ if (num_rekeys)
+ printf("Rekeys completed: %d\n", current_gen);
+
+ if (csk >= 0)
+ close(csk);
+ if (lsk >= 0)
+ close(lsk);
+ free(buf);
+ return test_result;
+}
+
+static int parse_cipher_option(const char *arg)
+{
+ if (strcmp(arg, "128") == 0) {
+ cipher_type = TLS_CIPHER_AES_GCM_128;
+ return 0;
+ } else if (strcmp(arg, "256") == 0) {
+ cipher_type = TLS_CIPHER_AES_GCM_256;
+ return 0;
+ }
+ printf("ERROR: Invalid cipher '%s'. Must be 128 or 256.\n", arg);
+ return -1;
+}
+
+static int parse_version_option(const char *arg)
+{
+ if (strcmp(arg, "1.2") == 0) {
+ tls_version = TLS_1_2_VERSION;
+ return 0;
+ } else if (strcmp(arg, "1.3") == 0) {
+ tls_version = TLS_1_3_VERSION;
+ return 0;
+ }
+ printf("ERROR: Invalid TLS version '%s'. Must be 1.2 or 1.3.\n", arg);
+ return -1;
+}
+
+static void print_usage(const char *prog)
+{
+ printf("TLS Hardware Offload Two-Node Test\n\n");
+ printf("Usage:\n");
+ printf(" %s server [OPTIONS]\n", prog);
+ printf(" %s client -s <ip> [OPTIONS]\n", prog);
+ printf("\nOptions:\n");
+ printf(" -s <ip> Server IPv4 address (client, required)\n");
+ printf(" -p <port> Server port (default: 4433)\n");
+ printf(" -b <size> Send buffer size in bytes (default: 16384)\n");
+ printf(" -r <max> Use random send buffer sizes (1..<max>)\n");
+ printf(" -v <version> TLS version: 1.2 or 1.3 (default: 1.3)\n");
+ printf(" -c <cipher> Cipher: 128 or 256 (default: 128)\n");
+ printf(" -n <N> Number of send/echo iterations (default: 100)\n");
+ printf(" -k <N> Perform N rekeys (client only, TLS 1.3; N < iterations)\n");
+ printf(" -B Burst mode: client sends continuously without echo;\n");
+ printf(" server drains and handles KeyUpdate without responding.\n");
+ printf(" -Z Enable zero-copy RX (TLS_RX_EXPECT_NO_PAD);\n");
+ printf(" server only, TLS 1.3 only.\n");
+ printf(" -h Show this help message\n");
+ printf("\nExample:\n");
+ printf(" Node A: %s server\n", prog);
+ printf(" Node B: %s client -s 192.168.20.2\n", prog);
+ printf("\nRekey Example (3 rekeys, TLS 1.3 only):\n");
+ printf(" Node A: %s server\n", prog);
+ printf(" Node B: %s client -s 192.168.20.2 -k 3\n", prog);
+ printf("\nBurst Mode Example (client stresses TX rekey under load):\n");
+ printf(" Node A: %s server -B\n", prog);
+ printf(" Node B: %s client -s 192.168.20.2 -B -k 3\n", prog);
+}
+
+int main(int argc, char *argv[])
+{
+ int send_size_set = 0;
+ int is_server;
+ int opt;
+
+ if (argc < 2 ||
+ (strcmp(argv[1], "server") && strcmp(argv[1], "client"))) {
+ print_usage(argv[0]);
+ return 1;
+ }
+ is_server = !strcmp(argv[1], "server");
+
+ optind = 2; /* skip subcommand */
+ while ((opt = getopt(argc, argv, "s:p:b:r:c:v:k:n:BZh")) != -1) {
+ switch (opt) {
+ case 's':
+ server_ip = optarg;
+ break;
+ case 'B':
+ burst_mode = 1;
+ break;
+ case 'Z':
+ zc_rx = 1;
+ break;
+ case 'p':
+ server_port = atoi(optarg);
+ if (server_port < 1 || server_port > 65535) {
+ printf("ERROR: Invalid port '%s'. Must be 1..65535.\n",
+ optarg);
+ return 1;
+ }
+ break;
+ case 'b':
+ send_size = atoi(optarg);
+ if (send_size < 1) {
+ printf("ERROR: Invalid buffer size '%s'. Must be >= 1.\n",
+ optarg);
+ return 1;
+ }
+ send_size_set = 1;
+ break;
+ case 'r':
+ random_size_max = atoi(optarg);
+ if (random_size_max < 1) {
+ printf("ERROR: Invalid random size '%s'. Must be >= 1.\n",
+ optarg);
+ return 1;
+ }
+ break;
+ case 'c':
+ if (parse_cipher_option(optarg) < 0)
+ return 1;
+ break;
+ case 'v':
+ if (parse_version_option(optarg) < 0)
+ return 1;
+ break;
+ case 'k':
+ num_rekeys = atoi(optarg);
+ if (num_rekeys < 1) {
+ printf("ERROR: Invalid rekey count '%s'. Must be >= 1.\n",
+ optarg);
+ return 1;
+ }
+ break;
+ case 'n':
+ num_iterations = atoi(optarg);
+ if (num_iterations < 1) {
+ printf("ERROR: Invalid iteration count '%s'. Must be >= 1.\n",
+ optarg);
+ return 1;
+ }
+ break;
+ case 'h':
+ print_usage(argv[0]);
+ return 0;
+ default:
+ print_usage(argv[0]);
+ return 1;
+ }
+ }
+
+ if (send_size_set && random_size_max > 0) {
+ printf("ERROR: -b and -r are mutually exclusive\n");
+ return 1;
+ }
+
+ if (zc_rx && tls_version != TLS_1_3_VERSION) {
+ printf("ERROR: -Z (TLS_RX_EXPECT_NO_PAD) requires TLS 1.3\n");
+ return 1;
+ }
+
+ if (burst_mode && random_size_max > 0) {
+ printf("ERROR: -B and -r are mutually exclusive\n");
+ return 1;
+ }
+
+ if (is_server) {
+ if (server_ip) {
+ printf("warning: -s is ignored in server mode\n");
+ server_ip = NULL;
+ }
+ if (random_size_max > 0) {
+ printf("warning: -r is ignored in server mode\n");
+ random_size_max = 0;
+ }
+ if (num_rekeys) {
+ printf("warning: -k is ignored in server mode\n");
+ num_rekeys = 0;
+ }
+ } else {
+ if (!server_ip) {
+ printf("ERROR: Client requires -s <ip> option\n");
+ return 1;
+ }
+ if (tls_version == TLS_1_2_VERSION && num_rekeys) {
+ printf("ERROR: TLS 1.2 does not support rekey\n");
+ return 1;
+ }
+ if (num_rekeys >= num_iterations) {
+ printf("ERROR: num_rekeys (%d) must be < num_iterations (%d)\n",
+ num_rekeys, num_iterations);
+ return 1;
+ }
+ if (zc_rx) {
+ printf("ERROR: -Z applies to the server (receiver) only\n");
+ return 1;
+ }
+ }
+
+ printf("TLS Version: %s\n", version_name(tls_version));
+ printf("Cipher: %s\n", cipher_name(cipher_type));
+ if (random_size_max > 0)
+ printf("Buffer size: random (1..%d)\n", random_size_max);
+ else
+ printf("Buffer size: %d\n", send_size);
+
+ if (num_rekeys)
+ printf("Rekey testing ENABLED: %d rekey(s)\n", num_rekeys);
+ if (burst_mode)
+ printf("Burst mode ENABLED\n");
+ if (zc_rx)
+ printf("Zero-copy RX ENABLED\n");
+
+ srand(time(NULL));
+
+ if (is_server)
+ return do_server() ? 1 : 0;
+
+ return do_client() ? 1 : 0;
+}
diff --git a/tools/testing/selftests/drivers/net/hw/tls_hw_offload.py b/tools/testing/selftests/drivers/net/hw/tls_hw_offload.py
new file mode 100755
index 000000000000..f12da0e66afd
--- /dev/null
+++ b/tools/testing/selftests/drivers/net/hw/tls_hw_offload.py
@@ -0,0 +1,256 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0
+
+"""Test kTLS hardware offload using a C helper binary."""
+
+from collections import defaultdict
+
+from lib.py import ksft_run, ksft_exit, ksft_pr, KsftSkipEx, ksft_true
+from lib.py import ksft_variants, KsftNamedVariant
+from lib.py import NetDrvEpEnv
+from lib.py import cmd, bkg, wait_port_listen, rand_port
+
+
+def check_tls_support(cfg):
+ try:
+ cmd("test -f /proc/net/tls_stat")
+ cmd("test -f /proc/net/tls_stat", host=cfg.remote)
+ except Exception as e:
+ raise KsftSkipEx(f"kTLS not supported: {e}")
+
+
+def read_tls_stats(host=None):
+ stats = defaultdict(int)
+ output = cmd("cat /proc/net/tls_stat", host=host)
+ for line in output.stdout.strip().split('\n'):
+ parts = line.split()
+ if len(parts) == 2:
+ stats[parts[0]] = int(parts[1])
+ return stats
+
+
+def stat_diff(before, after, key):
+ return after[key] - before[key]
+
+
+def check_path(before, after, direction, role, require_hw):
+ """On the DUT, require HW offload; on the remote, HW or SW is fine."""
+ dev = stat_diff(before, after, f'Tls{direction}Device')
+ sw = stat_diff(before, after, f'Tls{direction}Sw')
+ if require_hw:
+ if dev < 1:
+ ksft_pr(f"FAIL: {role} {direction}: HW offload not engaged "
+ f"(Device={dev}, Sw={sw})")
+ return 1
+ elif dev < 1 and sw < 1:
+ ksft_pr(f"FAIL: {role} {direction}: no TLS activity "
+ f"(Device={dev}, Sw={sw})")
+ return 1
+ return 0
+
+
+def check_min(before, after, key, minimum, role):
+ diff = stat_diff(before, after, key)
+ if diff < minimum:
+ ksft_pr(f"FAIL: {role} {key}: expected >= {minimum}, got {diff}")
+ return 1
+ return 0
+
+
+def check_zero(before, after, key, role):
+ diff = stat_diff(before, after, key)
+ if diff > 0:
+ ksft_pr(f"FAIL: {role} {key} increased by {diff}")
+ return 1
+ return 0
+
+
+def verify_tls_counters(stats_before, stats_after, expected_rekeys,
+ tls_role, is_dut, burst=False):
+ """Verify TLS counters on one side of the connection.
+
+ tls_role: 'client' or 'server' (TLS role this side played).
+ is_dut: True for the local DUT; requires HW offload counters.
+ burst: burst mode - only the TLS client rotates its TX key; the TLS
+ server only follows with an RX rotation on KeyUpdate receipt.
+ """
+ errors = 0
+ role = 'DUT' if is_dut else 'Peer'
+
+ # In burst mode only one direction carries TLS traffic per side
+ # (TLS client sends, TLS server receives). Check HW offload only on
+ # the active direction(s); require HW on the DUT's active direction.
+ if burst:
+ if tls_role == 'client':
+ errors += check_path(stats_before, stats_after, 'Tx', role,
+ require_hw=is_dut)
+ else:
+ errors += check_path(stats_before, stats_after, 'Rx', role,
+ require_hw=is_dut)
+ else:
+ errors += check_path(stats_before, stats_after, 'Tx', role,
+ require_hw=is_dut)
+ errors += check_path(stats_before, stats_after, 'Rx', role,
+ require_hw=is_dut)
+
+ if expected_rekeys > 0:
+ if burst:
+ if tls_role == 'client':
+ errors += check_min(stats_before, stats_after,
+ 'TlsTxRekeyOk', expected_rekeys, role)
+ errors += check_zero(stats_before, stats_after,
+ 'TlsTxRekeyError', role)
+ else:
+ errors += check_min(stats_before, stats_after,
+ 'TlsRxRekeyOk', expected_rekeys, role)
+ errors += check_min(stats_before, stats_after,
+ 'TlsRxRekeyReceived', expected_rekeys,
+ role)
+ errors += check_zero(stats_before, stats_after,
+ 'TlsRxRekeyError', role)
+ else:
+ errors += check_min(stats_before, stats_after,
+ 'TlsTxRekeyOk', expected_rekeys, role)
+ errors += check_min(stats_before, stats_after,
+ 'TlsRxRekeyOk', expected_rekeys, role)
+ if tls_role == 'server':
+ errors += check_min(stats_before, stats_after,
+ 'TlsRxRekeyReceived', expected_rekeys,
+ role)
+ errors += check_zero(stats_before, stats_after,
+ 'TlsTxRekeyError', role)
+ errors += check_zero(stats_before, stats_after,
+ 'TlsRxRekeyError', role)
+
+ errors += check_zero(stats_before, stats_after, 'TlsDecryptError', role)
+
+ return errors == 0
+
+
+def run_tls_test(cfg, cipher="128", tls_version="1.3", rekey=0,
+ buffer_size=None, random_max=None, burst=False, zc=False,
+ dut_role="client", num_iterations=None):
+ """Run the TLS offload test.
+
+ dut_role: 'client' (default) - DUT runs the TLS client, remote the server.
+ 'server' - swap: DUT listens, remote connects. Used for burst_rx
+ so the DUT's RX path is the one under rekey pressure.
+
+ The DUT (local) is the kernel under test; the remote is just a traffic
+ source/sink and may run any kernel without HW offload. Both sides run
+ kTLS because TLS is pairwise, but verify_tls_counters() requires HW
+ offload only on the DUT (is_dut=True); the peer may use SW kTLS.
+ """
+ port = rand_port()
+ send_size = random_max or buffer_size
+
+ if dut_role == "client":
+ server_bin, server_host = cfg.bin_remote, cfg.remote
+ client_bin, client_host = cfg.bin_local, None
+ client_target = cfg.remote_addr_v['4']
+ else:
+ server_bin, server_host = cfg.bin_local, None
+ client_bin, client_host = cfg.bin_remote, cfg.remote
+ client_target = cfg.addr_v['4']
+
+ server_cmd = f"{server_bin} server -p {port} -c {cipher} -v {tls_version}"
+ if burst:
+ server_cmd += " -B"
+ if zc:
+ server_cmd += " -Z"
+ if send_size:
+ server_cmd += f" -b {send_size}"
+
+ client_cmd = (f"{client_bin} client -s {client_target} "
+ f"-p {port} -c {cipher} -v {tls_version}")
+ if rekey:
+ client_cmd += f" -k {rekey}"
+ if burst:
+ client_cmd += " -B"
+ if num_iterations:
+ client_cmd += f" -n {num_iterations}"
+ if random_max:
+ client_cmd += f" -r {random_max}"
+ elif buffer_size:
+ client_cmd += f" -b {buffer_size}"
+
+ # Burst variants push hundreds of MB and perform many rekeys; the
+ # default cmd() timeout (5s) is too short.
+ cmd_timeout = 180 if burst else 5
+
+ stats_before_local = read_tls_stats()
+ stats_before_remote = read_tls_stats(host=cfg.remote)
+
+ with bkg(server_cmd, host=server_host, exit_wait=True):
+ wait_port_listen(port, host=server_host)
+ cmd(client_cmd, host=client_host, timeout=cmd_timeout)
+
+ stats_after_local = read_tls_stats()
+ stats_after_remote = read_tls_stats(host=cfg.remote)
+
+ dut_tls_role = dut_role
+ peer_tls_role = 'server' if dut_role == 'client' else 'client'
+
+ dut_ok = verify_tls_counters(stats_before_local, stats_after_local,
+ rekey, dut_tls_role, is_dut=True,
+ burst=burst)
+ peer_ok = verify_tls_counters(stats_before_remote, stats_after_remote,
+ rekey, peer_tls_role, is_dut=False,
+ burst=burst)
+
+ ksft_true(dut_ok, "DUT TLS counters verified")
+ ksft_true(peer_ok, "Peer TLS counters verified")
+
+
+@ksft_variants([
+ KsftNamedVariant("tls13_aes128", "128", "1.3"),
+ KsftNamedVariant("tls13_aes256", "256", "1.3"),
+ KsftNamedVariant("tls12_aes128", "128", "1.2"),
+ KsftNamedVariant("tls12_aes256", "256", "1.2"),
+])
+def test_tls_offload(cfg, cipher, tls_version):
+ run_tls_test(cfg, cipher=cipher, tls_version=tls_version)
+
+
+@ksft_variants([
+ KsftNamedVariant("single", 1),
+ KsftNamedVariant("multiple", 99),
+ KsftNamedVariant("small_buf", 30, 512),
+ KsftNamedVariant("large_buf", 10, 2097152),
+ KsftNamedVariant("random_buf", 20, None, 8192),
+])
+def test_tls_offload_rekey(cfg, rekey, buffer_size=None, random_max=None):
+ run_tls_test(cfg, cipher="128", tls_version="1.3", rekey=rekey,
+ buffer_size=buffer_size, random_max=random_max)
+
+
+@ksft_variants([
+ KsftNamedVariant("burst_tx_rekey_every_1", "client", False, 1, 50, 65536),
+ KsftNamedVariant("burst_tx_rekey_every_1000", "client", False, 1000, 3, 65536),
+ KsftNamedVariant("burst_rx_rekey_every_10", "server", False, 10, 20, 65536),
+ KsftNamedVariant("burst_rx_rekey_every_10000", "server", False, 10000, 1, 32768),
+ KsftNamedVariant("burst_rx_zc_rekey_every_100", "server", True, 100, 10, 65536),
+ KsftNamedVariant("burst_rx_zc_rekey_every_20000", "server", True, 20000, 1, 16384),
+])
+def test_tls_offload_burst(cfg, dut_role, zc, interval, rekeys, buffer_size):
+ run_tls_test(cfg, cipher="128", tls_version="1.3", rekey=rekeys,
+ buffer_size=buffer_size, burst=True, zc=zc, dut_role=dut_role,
+ num_iterations=interval * (rekeys + 1))
+
+
+def main() -> None:
+ with NetDrvEpEnv(__file__, nsim_test=False) as cfg:
+ cfg.bin_local = cfg.test_dir / "tls_hw_offload"
+ if not cfg.bin_local.exists():
+ raise KsftSkipEx(f"tls_hw_offload binary not found at {cfg.bin_local}")
+ cfg.bin_remote = cfg.remote.deploy(cfg.bin_local)
+ cfg.require_ipver("4")
+ check_tls_support(cfg)
+
+ ksft_run([test_tls_offload, test_tls_offload_rekey,
+ test_tls_offload_burst], args=(cfg, ))
+ ksft_exit()
+
+
+if __name__ == "__main__":
+ main()
--
2.25.1
^ permalink raw reply related [flat|nested] 9+ messages in thread