* [PATCH v9 04/10] x86/vmscape: Rename x86_ibpb_exit_to_user to x86_predictor_flush_exit_to_user
From: Pawan Gupta @ 2026-04-03 0:31 UTC (permalink / raw)
To: x86, Jon Kohler, Nikolay Borisov, H. Peter Anvin, Josh Poimboeuf,
David Kaplan, Sean Christopherson, Borislav Petkov, Dave Hansen,
Peter Zijlstra, Alexei Starovoitov, Daniel Borkmann,
Andrii Nakryiko, KP Singh, Jiri Olsa, David S. Miller,
David Laight, Andy Lutomirski, Thomas Gleixner, Ingo Molnar,
David Ahern, Martin KaFai Lau, Eduard Zingerman, Song Liu,
Yonghong Song, John Fastabend, Stanislav Fomichev, Hao Luo,
Paolo Bonzini, Jonathan Corbet
Cc: linux-kernel, kvm, Asit Mallick, Tao Zhang, bpf, netdev,
linux-doc
In-Reply-To: <20260402-vmscape-bhb-v9-0-94d16bc29774@linux.intel.com>
With the upcoming changes x86_ibpb_exit_to_user will also be used when BHB
clearing sequence is used. Rename it cover both the cases.
No functional change.
Suggested-by: Sean Christopherson <seanjc@google.com>
Tested-by: Jon Kohler <jon@nutanix.com>
Acked-by: Sean Christopherson <seanjc@google.com>
Signed-off-by: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
---
arch/x86/include/asm/entry-common.h | 6 +++---
arch/x86/include/asm/nospec-branch.h | 2 +-
arch/x86/kernel/cpu/bugs.c | 4 ++--
arch/x86/kvm/x86.c | 2 +-
4 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/arch/x86/include/asm/entry-common.h b/arch/x86/include/asm/entry-common.h
index ce3eb6d5fdf9..c45858db16c9 100644
--- a/arch/x86/include/asm/entry-common.h
+++ b/arch/x86/include/asm/entry-common.h
@@ -94,11 +94,11 @@ static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs,
*/
choose_random_kstack_offset(rdtsc());
- /* Avoid unnecessary reads of 'x86_ibpb_exit_to_user' */
+ /* Avoid unnecessary reads of 'x86_predictor_flush_exit_to_user' */
if (cpu_feature_enabled(X86_FEATURE_IBPB_EXIT_TO_USER) &&
- this_cpu_read(x86_ibpb_exit_to_user)) {
+ this_cpu_read(x86_predictor_flush_exit_to_user)) {
indirect_branch_prediction_barrier();
- this_cpu_write(x86_ibpb_exit_to_user, false);
+ this_cpu_write(x86_predictor_flush_exit_to_user, false);
}
}
#define arch_exit_to_user_mode_prepare arch_exit_to_user_mode_prepare
diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index 157eb69c7f0f..0381db59c39d 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -533,7 +533,7 @@ void alternative_msr_write(unsigned int msr, u64 val, unsigned int feature)
: "memory");
}
-DECLARE_PER_CPU(bool, x86_ibpb_exit_to_user);
+DECLARE_PER_CPU(bool, x86_predictor_flush_exit_to_user);
static inline void indirect_branch_prediction_barrier(void)
{
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 2cb4a96247d8..002bf4adccc3 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -65,8 +65,8 @@ EXPORT_PER_CPU_SYMBOL_GPL(x86_spec_ctrl_current);
* be needed to before running userspace. That IBPB will flush the branch
* predictor content.
*/
-DEFINE_PER_CPU(bool, x86_ibpb_exit_to_user);
-EXPORT_PER_CPU_SYMBOL_GPL(x86_ibpb_exit_to_user);
+DEFINE_PER_CPU(bool, x86_predictor_flush_exit_to_user);
+EXPORT_PER_CPU_SYMBOL_GPL(x86_predictor_flush_exit_to_user);
u64 x86_pred_cmd __ro_after_init = PRED_CMD_IBPB;
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index fd1c4a36b593..45d7cfedc507 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -11464,7 +11464,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
* may migrate to.
*/
if (cpu_feature_enabled(X86_FEATURE_IBPB_EXIT_TO_USER))
- this_cpu_write(x86_ibpb_exit_to_user, true);
+ this_cpu_write(x86_predictor_flush_exit_to_user, true);
/*
* Consume any pending interrupts, including the possible source of
--
2.34.1
^ permalink raw reply related
* [PATCH v9 03/10] x86/bhi: Rename clear_bhb_loop() to clear_bhb_loop_nofence()
From: Pawan Gupta @ 2026-04-03 0:31 UTC (permalink / raw)
To: x86, Jon Kohler, Nikolay Borisov, H. Peter Anvin, Josh Poimboeuf,
David Kaplan, Sean Christopherson, Borislav Petkov, Dave Hansen,
Peter Zijlstra, Alexei Starovoitov, Daniel Borkmann,
Andrii Nakryiko, KP Singh, Jiri Olsa, David S. Miller,
David Laight, Andy Lutomirski, Thomas Gleixner, Ingo Molnar,
David Ahern, Martin KaFai Lau, Eduard Zingerman, Song Liu,
Yonghong Song, John Fastabend, Stanislav Fomichev, Hao Luo,
Paolo Bonzini, Jonathan Corbet
Cc: linux-kernel, kvm, Asit Mallick, Tao Zhang, bpf, netdev,
linux-doc
In-Reply-To: <20260402-vmscape-bhb-v9-0-94d16bc29774@linux.intel.com>
To reflect the recent change that moved LFENCE to the caller side.
Suggested-by: Borislav Petkov <bp@alien8.de>
Reviewed-by: Nikolay Borisov <nik.borisov@suse.com>
Tested-by: Jon Kohler <jon@nutanix.com>
Signed-off-by: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
---
arch/x86/entry/entry_64.S | 8 ++++----
arch/x86/include/asm/nospec-branch.h | 6 +++---
arch/x86/net/bpf_jit_comp.c | 2 +-
3 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index bbd4b1c7ec04..1f56d086d312 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -1532,7 +1532,7 @@ SYM_CODE_END(rewind_stack_and_make_dead)
* Note, callers should use a speculation barrier like LFENCE immediately after
* a call to this function to ensure BHB is cleared before indirect branches.
*/
-SYM_FUNC_START(clear_bhb_loop)
+SYM_FUNC_START(clear_bhb_loop_nofence)
ANNOTATE_NOENDBR
push %rbp
mov %rsp, %rbp
@@ -1570,6 +1570,6 @@ SYM_FUNC_START(clear_bhb_loop)
5:
pop %rbp
RET
-SYM_FUNC_END(clear_bhb_loop)
-EXPORT_SYMBOL_FOR_KVM(clear_bhb_loop)
-STACK_FRAME_NON_STANDARD(clear_bhb_loop)
+SYM_FUNC_END(clear_bhb_loop_nofence)
+EXPORT_SYMBOL_FOR_KVM(clear_bhb_loop_nofence)
+STACK_FRAME_NON_STANDARD(clear_bhb_loop_nofence)
diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index 87b83ae7c97f..157eb69c7f0f 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -331,11 +331,11 @@
#ifdef CONFIG_X86_64
.macro CLEAR_BRANCH_HISTORY
- ALTERNATIVE "", "call clear_bhb_loop; lfence", X86_FEATURE_CLEAR_BHB_LOOP
+ ALTERNATIVE "", "call clear_bhb_loop_nofence; lfence", X86_FEATURE_CLEAR_BHB_LOOP
.endm
.macro CLEAR_BRANCH_HISTORY_VMEXIT
- ALTERNATIVE "", "call clear_bhb_loop; lfence", X86_FEATURE_CLEAR_BHB_VMEXIT
+ ALTERNATIVE "", "call clear_bhb_loop_nofence; lfence", X86_FEATURE_CLEAR_BHB_VMEXIT
.endm
#else
#define CLEAR_BRANCH_HISTORY
@@ -389,7 +389,7 @@ extern void entry_untrain_ret(void);
extern void write_ibpb(void);
#ifdef CONFIG_X86_64
-extern void clear_bhb_loop(void);
+extern void clear_bhb_loop_nofence(void);
#endif
extern void (*x86_return_thunk)(void);
diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 63d6c9fa5e80..f40e88f87273 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -1619,7 +1619,7 @@ static int emit_spectre_bhb_barrier(u8 **pprog, u8 *ip,
EMIT1(0x51); /* push rcx */
ip += 2;
- func = (u8 *)clear_bhb_loop;
+ func = (u8 *)clear_bhb_loop_nofence;
ip += x86_call_depth_emit_accounting(&prog, func, ip);
if (emit_call(&prog, func, ip))
--
2.34.1
^ permalink raw reply related
* [PATCH v9 02/10] x86/bhi: Make clear_bhb_loop() effective on newer CPUs
From: Pawan Gupta @ 2026-04-03 0:31 UTC (permalink / raw)
To: x86, Jon Kohler, Nikolay Borisov, H. Peter Anvin, Josh Poimboeuf,
David Kaplan, Sean Christopherson, Borislav Petkov, Dave Hansen,
Peter Zijlstra, Alexei Starovoitov, Daniel Borkmann,
Andrii Nakryiko, KP Singh, Jiri Olsa, David S. Miller,
David Laight, Andy Lutomirski, Thomas Gleixner, Ingo Molnar,
David Ahern, Martin KaFai Lau, Eduard Zingerman, Song Liu,
Yonghong Song, John Fastabend, Stanislav Fomichev, Hao Luo,
Paolo Bonzini, Jonathan Corbet
Cc: linux-kernel, kvm, Asit Mallick, Tao Zhang, bpf, netdev,
linux-doc
In-Reply-To: <20260402-vmscape-bhb-v9-0-94d16bc29774@linux.intel.com>
As a mitigation for BHI, clear_bhb_loop() executes branches that overwrite
the Branch History Buffer (BHB). On Alder Lake and newer parts this
sequence is not sufficient because it doesn't clear enough entries. This
was not an issue because these CPUs use the BHI_DIS_S hardware mitigation
in the kernel.
Now with VMSCAPE (BHI variant) it is also required to isolate branch
history between guests and userspace. Since BHI_DIS_S only protects the
kernel, the newer CPUs also use IBPB.
A cheaper alternative to the current IBPB mitigation is clear_bhb_loop().
But it currently does not clear enough BHB entries to be effective on newer
CPUs with larger BHB. At boot, dynamically set the loop count of
clear_bhb_loop() such that it is effective on newer CPUs too. Use the
X86_FEATURE_BHI_CTRL feature flag to select the appropriate loop count.
Suggested-by: Dave Hansen <dave.hansen@linux.intel.com>
Signed-off-by: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
---
arch/x86/entry/entry_64.S | 8 +++++---
arch/x86/include/asm/nospec-branch.h | 2 ++
arch/x86/kernel/cpu/bugs.c | 13 +++++++++++++
3 files changed, 20 insertions(+), 3 deletions(-)
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index 3a180a36ca0e..bbd4b1c7ec04 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -1536,7 +1536,9 @@ SYM_FUNC_START(clear_bhb_loop)
ANNOTATE_NOENDBR
push %rbp
mov %rsp, %rbp
- movl $5, %ecx
+
+ movzbl bhb_seq_outer_loop(%rip), %ecx
+
ANNOTATE_INTRA_FUNCTION_CALL
call 1f
jmp 5f
@@ -1556,8 +1558,8 @@ SYM_FUNC_START(clear_bhb_loop)
* This should be ideally be: .skip 32 - (.Lret2 - 2f), 0xcc
* but some Clang versions (e.g. 18) don't like this.
*/
- .skip 32 - 18, 0xcc
-2: movl $5, %eax
+ .skip 32 - 20, 0xcc
+2: movzbl bhb_seq_inner_loop(%rip), %eax
3: jmp 4f
nop
4: sub $1, %eax
diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index 70b377fcbc1c..87b83ae7c97f 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -548,6 +548,8 @@ DECLARE_PER_CPU(u64, x86_spec_ctrl_current);
extern void update_spec_ctrl_cond(u64 val);
extern u64 spec_ctrl_current(void);
+extern u8 bhb_seq_inner_loop, bhb_seq_outer_loop;
+
/*
* With retpoline, we must use IBRS to restrict branch prediction
* before calling into firmware.
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 83f51cab0b1e..2cb4a96247d8 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -2047,6 +2047,10 @@ enum bhi_mitigations {
static enum bhi_mitigations bhi_mitigation __ro_after_init =
IS_ENABLED(CONFIG_MITIGATION_SPECTRE_BHI) ? BHI_MITIGATION_AUTO : BHI_MITIGATION_OFF;
+/* Default to short BHB sequence values */
+u8 bhb_seq_outer_loop __ro_after_init = 5;
+u8 bhb_seq_inner_loop __ro_after_init = 5;
+
static int __init spectre_bhi_parse_cmdline(char *str)
{
if (!str)
@@ -3242,6 +3246,15 @@ void __init cpu_select_mitigations(void)
x86_spec_ctrl_base &= ~SPEC_CTRL_MITIGATIONS_MASK;
}
+ /*
+ * Switch to long BHB clear sequence on newer CPUs (with BHI_CTRL
+ * support), see Intel's BHI guidance.
+ */
+ if (cpu_feature_enabled(X86_FEATURE_BHI_CTRL)) {
+ bhb_seq_outer_loop = 12;
+ bhb_seq_inner_loop = 7;
+ }
+
x86_arch_cap_msr = x86_read_arch_cap_msr();
cpu_print_attack_vectors();
--
2.34.1
^ permalink raw reply related
* [PATCH v9 01/10] x86/bhi: x86/vmscape: Move LFENCE out of clear_bhb_loop()
From: Pawan Gupta @ 2026-04-03 0:30 UTC (permalink / raw)
To: x86, Jon Kohler, Nikolay Borisov, H. Peter Anvin, Josh Poimboeuf,
David Kaplan, Sean Christopherson, Borislav Petkov, Dave Hansen,
Peter Zijlstra, Alexei Starovoitov, Daniel Borkmann,
Andrii Nakryiko, KP Singh, Jiri Olsa, David S. Miller,
David Laight, Andy Lutomirski, Thomas Gleixner, Ingo Molnar,
David Ahern, Martin KaFai Lau, Eduard Zingerman, Song Liu,
Yonghong Song, John Fastabend, Stanislav Fomichev, Hao Luo,
Paolo Bonzini, Jonathan Corbet
Cc: linux-kernel, kvm, Asit Mallick, Tao Zhang, bpf, netdev,
linux-doc
In-Reply-To: <20260402-vmscape-bhb-v9-0-94d16bc29774@linux.intel.com>
Currently, the BHB clearing sequence is followed by an LFENCE to prevent
transient execution of subsequent indirect branches prematurely. However,
the LFENCE barrier could be unnecessary in certain cases. For example, when
the kernel is using the BHI_DIS_S mitigation, and BHB clearing is only
needed for userspace. In such cases, the LFENCE is redundant because ring
transitions would provide the necessary serialization.
Below is a quick recap of BHI mitigation options:
On Alder Lake and newer
BHI_DIS_S: Hardware control to mitigate BHI in ring0. This has low
performance overhead.
Long loop: Alternatively, a longer version of the BHB clearing sequence
can be used to mitigate BHI. It can also be used to mitigate the BHI
variant of VMSCAPE. This is not yet implemented in Linux.
On older CPUs
Short loop: Clears BHB at kernel entry and VMexit. The "Long loop" is
effective on older CPUs as well, but should be avoided because of
unnecessary overhead.
On Alder Lake and newer CPUs, eIBRS isolates the indirect targets between
guest and host. But when affected by the BHI variant of VMSCAPE, a guest's
branch history may still influence indirect branches in userspace. This
also means the big hammer IBPB could be replaced with a cheaper option that
clears the BHB at exit-to-userspace after a VMexit.
In preparation for adding the support for the BHB sequence (without LFENCE)
on newer CPUs, move the LFENCE to the caller side after clear_bhb_loop() is
executed. Allow callers to decide whether they need the LFENCE or not. This
adds a few extra bytes to the call sites, but it obviates the need for
multiple variants of clear_bhb_loop().
Suggested-by: Dave Hansen <dave.hansen@linux.intel.com>
Tested-by: Jon Kohler <jon@nutanix.com>
Reviewed-by: Nikolay Borisov <nik.borisov@suse.com>
Signed-off-by: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
---
arch/x86/entry/entry_64.S | 5 ++++-
arch/x86/include/asm/nospec-branch.h | 4 ++--
arch/x86/net/bpf_jit_comp.c | 2 ++
3 files changed, 8 insertions(+), 3 deletions(-)
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index 42447b1e1dff..3a180a36ca0e 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -1528,6 +1528,9 @@ SYM_CODE_END(rewind_stack_and_make_dead)
* refactored in the future if needed. The .skips are for safety, to ensure
* that all RETs are in the second half of a cacheline to mitigate Indirect
* Target Selection, rather than taking the slowpath via its_return_thunk.
+ *
+ * Note, callers should use a speculation barrier like LFENCE immediately after
+ * a call to this function to ensure BHB is cleared before indirect branches.
*/
SYM_FUNC_START(clear_bhb_loop)
ANNOTATE_NOENDBR
@@ -1562,7 +1565,7 @@ SYM_FUNC_START(clear_bhb_loop)
sub $1, %ecx
jnz 1b
.Lret2: RET
-5: lfence
+5:
pop %rbp
RET
SYM_FUNC_END(clear_bhb_loop)
diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index 4f4b5e8a1574..70b377fcbc1c 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -331,11 +331,11 @@
#ifdef CONFIG_X86_64
.macro CLEAR_BRANCH_HISTORY
- ALTERNATIVE "", "call clear_bhb_loop", X86_FEATURE_CLEAR_BHB_LOOP
+ ALTERNATIVE "", "call clear_bhb_loop; lfence", X86_FEATURE_CLEAR_BHB_LOOP
.endm
.macro CLEAR_BRANCH_HISTORY_VMEXIT
- ALTERNATIVE "", "call clear_bhb_loop", X86_FEATURE_CLEAR_BHB_VMEXIT
+ ALTERNATIVE "", "call clear_bhb_loop; lfence", X86_FEATURE_CLEAR_BHB_VMEXIT
.endm
#else
#define CLEAR_BRANCH_HISTORY
diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index e9b78040d703..63d6c9fa5e80 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -1624,6 +1624,8 @@ static int emit_spectre_bhb_barrier(u8 **pprog, u8 *ip,
if (emit_call(&prog, func, ip))
return -EINVAL;
+ /* Don't speculate past this until BHB is cleared */
+ EMIT_LFENCE();
EMIT1(0x59); /* pop rcx */
EMIT1(0x58); /* pop rax */
}
--
2.34.1
^ permalink raw reply related
* [PATCH v9 00/10] VMSCAPE optimization for BHI variant
From: Pawan Gupta @ 2026-04-03 0:30 UTC (permalink / raw)
To: x86, Jon Kohler, Nikolay Borisov, H. Peter Anvin, Josh Poimboeuf,
David Kaplan, Sean Christopherson, Borislav Petkov, Dave Hansen,
Peter Zijlstra, Alexei Starovoitov, Daniel Borkmann,
Andrii Nakryiko, KP Singh, Jiri Olsa, David S. Miller,
David Laight, Andy Lutomirski, Thomas Gleixner, Ingo Molnar,
David Ahern, Martin KaFai Lau, Eduard Zingerman, Song Liu,
Yonghong Song, John Fastabend, Stanislav Fomichev, Hao Luo,
Paolo Bonzini, Jonathan Corbet
Cc: linux-kernel, kvm, Asit Mallick, Tao Zhang, bpf, netdev,
linux-doc
v9:
- Use global variables for BHB loop counters instead of ALTERNATIVE-based
approach. (Dave & others)
- Use 32-bit registers (%eax/%ecx) for loop counters, loaded via movzbl
from 8-bit globals. 8-bit registers (e.g. %ah in the inner loop) caused
performance regression on certain CPUs due to partial-register stalls. (David Laight)
- Let BPF save/restore %rax/%rcx as in the original implementation, since
it is the only caller that needs these registers preserved across the
BHB clearing sequence.
- Drop Reviewed-by from patch 2/10 as the implementation changed significantly.
- Apply Tested-by from Jon Kohler to the series (except patch 2/10).
- Fix commit message grammar. (Borislav)
- Rebased to v7.0-rc6.
v8: https://lore.kernel.org/r/20260324-vmscape-bhb-v8-0-68bb524b3ab9@linux.intel.com
- Use helper in KVM to convey the mitigation status. (PeterZ/Borisov)
- Fix the documentation for default vmscape mitigation. (BPF bot)
- Remove the stray lines in bug.c (BPF bot).
- Updated commit messages and comments.
- Rebased to v7.0-rc5.
v7: https://lore.kernel.org/r/20260319-vmscape-bhb-v7-0-b76a777a98af@linux.intel.com
- s/This allows/Allow/ and s/This does adds/This adds/ in patch 1/10 commit
message (Borislav).
- Minimize register usage in BHB clearing seq. (David Laight)
- Instead of separate ecx/eax counters, use al/ah.
- Adjust the alignment of RET due to register size change.
- save/restore rax in the seq itself.
- Remove the save/restore of rax/rcx for BPF callers.
- Rename clear_bhb_loop() to clear_bhb_loop_nofence() to make it
obvious that the LFENCE is not part of the sequence (Borislav).
- Fix Kconfig: s/select/depends on/ HAVE_STATIC_CALL (PeterZ).
- Rebased to v7.0-rc4.
v6: https://lore.kernel.org/r/20251201-vmscape-bhb-v6-0-d610dd515714@linux.intel.com
- Remove semicolon at the end of asm in ALTERNATIVE (Uros).
- Fix build warning in vmscape_select_mitigation() (LKP).
- Rebased to v6.18.
v5: https://lore.kernel.org/r/20251126-vmscape-bhb-v5-2-02d66e423b00@linux.intel.com
- For BHI seq, limit runtime-patching to loop counts only (Dave).
Dropped 2 patches that moved the BHB seq to a macro.
- Remove redundant switch cases in vmscape_select_mitigation() (Nikolay).
- Improve commit message (Nikolay).
- Collected tags.
v4: https://lore.kernel.org/r/20251119-vmscape-bhb-v4-0-1adad4e69ddc@linux.intel.com
- Move LFENCE to the callsite, out of clear_bhb_loop(). (Dave)
- Make clear_bhb_loop() work for larger BHB. (Dave)
This now uses hardware enumeration to determine the BHB size to clear.
- Use write_ibpb() instead of indirect_branch_prediction_barrier() when
IBPB is known to be available. (Dave)
- Use static_call() to simplify mitigation at exit-to-userspace. (Dave)
- Refactor vmscape_select_mitigation(). (Dave)
- Fix vmscape=on which was wrongly behaving as AUTO. (Dave)
- Split the patches. (Dave)
- Patch 1-4 prepares for making the sequence flexible for VMSCAPE use.
- Patch 5 trivial rename of variable.
- Patch 6-8 prepares for deploying BHB mitigation for VMSCAPE.
- Patch 9 deploys the mitigation.
- Patch 10-11 fixes ON Vs AUTO mode.
v3: https://lore.kernel.org/r/20251027-vmscape-bhb-v3-0-5793c2534e93@linux.intel.com
- s/x86_pred_flush_pending/x86_predictor_flush_exit_to_user/ (Sean).
- Removed IBPB & BHB-clear mutual exclusion at exit-to-userspace.
- Collected tags.
v2: https://lore.kernel.org/r/20251015-vmscape-bhb-v2-0-91cbdd9c3a96@linux.intel.com
- Added check for IBPB feature in vmscape_select_mitigation(). (David)
- s/vmscape=auto/vmscape=on/ (David)
- Added patch to remove LFENCE from VMSCAPE BHB-clear sequence.
- Rebased to v6.18-rc1.
v1: https://lore.kernel.org/r/20250924-vmscape-bhb-v1-0-da51f0e1934d@linux.intel.com
Hi All,
These patches aim to improve the performance of a recent mitigation for
VMSCAPE[1] vulnerability. This improvement is relevant for BHI variant of
VMSCAPE that affect Alder Lake and newer processors.
The current mitigation approach uses IBPB on kvm-exit-to-userspace for all
affected range of CPUs. This is an overkill for CPUs that are only affected
by the BHI variant. On such CPUs clearing the branch history is sufficient
for VMSCAPE, and also more apt as the underlying issue is due to poisoned
branch history.
Below is the iPerf data for transfer between guest and host, comparing IBPB
and BHB-clear mitigation. BHB-clear shows performance improvement over IBPB
in most cases.
Platform: Emerald Rapids
Baseline: vmscape=off
Target: IBPB at VMexit-to-userspace Vs the new BHB-clear at
VMexit-to-userspace mitigation (both compared against baseline).
(pN = N parallel connections)
| iPerf user-net | IBPB | BHB Clear |
|----------------|---------|-----------|
| UDP 1-vCPU_p1 | -12.5% | 1.3% |
| TCP 1-vCPU_p1 | -10.4% | -1.5% |
| TCP 1-vCPU_p1 | -7.5% | -3.0% |
| UDP 4-vCPU_p16 | -3.7% | -3.7% |
| TCP 4-vCPU_p4 | -2.9% | -1.4% |
| UDP 4-vCPU_p4 | -0.6% | 0.0% |
| TCP 4-vCPU_p4 | 3.5% | 0.0% |
| iPerf bridge-net | IBPB | BHB Clear |
|------------------|---------|-----------|
| UDP 1-vCPU_p1 | -9.4% | -0.4% |
| TCP 1-vCPU_p1 | -3.9% | -0.5% |
| UDP 4-vCPU_p16 | -2.2% | -3.8% |
| TCP 4-vCPU_p4 | -1.0% | -1.0% |
| TCP 4-vCPU_p4 | 0.5% | 0.5% |
| UDP 4-vCPU_p4 | 0.0% | 0.9% |
| TCP 1-vCPU_p1 | 0.0% | 0.9% |
| iPerf vhost-net | IBPB | BHB Clear |
|-----------------|---------|-----------|
| UDP 1-vCPU_p1 | -4.3% | 1.0% |
| TCP 1-vCPU_p1 | -3.8% | -0.5% |
| TCP 1-vCPU_p1 | -2.7% | -0.7% |
| UDP 4-vCPU_p16 | -0.7% | -2.2% |
| TCP 4-vCPU_p4 | -0.4% | 0.8% |
| UDP 4-vCPU_p4 | 0.4% | -0.7% |
| TCP 4-vCPU_p4 | 0.0% | 0.6% |
[1] https://comsec.ethz.ch/research/microarch/vmscape-exposing-and-exploiting-incomplete-branch-predictor-isolation-in-cloud-environments/
---
Pawan Gupta (10):
x86/bhi: x86/vmscape: Move LFENCE out of clear_bhb_loop()
x86/bhi: Make clear_bhb_loop() effective on newer CPUs
x86/bhi: Rename clear_bhb_loop() to clear_bhb_loop_nofence()
x86/vmscape: Rename x86_ibpb_exit_to_user to x86_predictor_flush_exit_to_user
x86/vmscape: Move mitigation selection to a switch()
x86/vmscape: Use write_ibpb() instead of indirect_branch_prediction_barrier()
x86/vmscape: Use static_call() for predictor flush
x86/vmscape: Deploy BHB clearing mitigation
x86/vmscape: Resolve conflict between attack-vectors and vmscape=force
x86/vmscape: Add cmdline vmscape=on to override attack vector controls
Documentation/admin-guide/hw-vuln/vmscape.rst | 15 ++++-
Documentation/admin-guide/kernel-parameters.txt | 6 +-
arch/x86/Kconfig | 1 +
arch/x86/entry/entry_64.S | 21 +++---
arch/x86/include/asm/cpufeatures.h | 2 +-
arch/x86/include/asm/entry-common.h | 13 ++--
arch/x86/include/asm/nospec-branch.h | 15 +++--
arch/x86/include/asm/processor.h | 1 +
arch/x86/kernel/cpu/bugs.c | 89 +++++++++++++++++++++----
arch/x86/kvm/x86.c | 4 +-
arch/x86/net/bpf_jit_comp.c | 4 +-
11 files changed, 135 insertions(+), 36 deletions(-)
---
base-commit: 7aaa8047eafd0bd628065b15757d9b48c5f9c07d
change-id: 20250916-vmscape-bhb-d7d469977f2f
Best regards,
--
Thanks,
Pawan
^ permalink raw reply
* Re: [PATCH net-next v2 2/4] net: dsa: yt921x: Refactor long register helpers
From: Andrew Lunn @ 2026-04-03 0:26 UTC (permalink / raw)
To: David Yang
Cc: netdev, Vladimir Oltean, David S. Miller, Eric Dumazet,
Jakub Kicinski, Paolo Abeni, linux-kernel
In-Reply-To: <20260402223437.109097-3-mmyangfl@gmail.com>
> -static int yt921x_reg64_write(struct yt921x_priv *priv, u32 reg, u64 val)
> +yt921x_reg64_write(struct yt921x_priv *priv, u32 reg, const u32 *vals)
> +{
The old reg64_write actually took a u64 value, so the name fit. The
new one takes 2x a u32. So i'm not sure calling is reg64 actually
fits. Maybe yt921x_reg32x2_write()?
> +static int yt921x_vlan_del(struct yt921x_priv *priv, int port, u16 vid)
> {
> - u64 mask64;
> + u32 masks[2];
>
> - mask64 = YT921X_VLAN_CTRL_PORTS(port) |
> - YT921X_VLAN_CTRL_UNTAG_PORTn(port);
> + masks[0] = YT921X_VLAN_CTRLa_PORTn(port);
> + masks[1] = YT921X_VLAN_CTRLb_UNTAG_PORTn(port);
>
> - return yt921x_reg64_clear_bits(priv, YT921X_VLANn_CTRL(vid), mask64);
> + return yt921x_reg64_clear_bits(priv, YT921X_VLANn_CTRL(vid), masks);
Or you could make yt921x_reg64_write() continue to take a u64, and
avoid all these changes?
How does the datasheet describe these registers? Are they 64bit
registers, or 2x 32bit registers?
Andrew
^ permalink raw reply
* Re: [PATCH net-next v2 1/4] net: dsa: pass extack to user tc policers
From: Andrew Lunn @ 2026-04-03 0:12 UTC (permalink / raw)
To: David Yang
Cc: netdev, Vladimir Oltean, UNGLinuxDriver, David S. Miller,
Eric Dumazet, Jakub Kicinski, Paolo Abeni, Simon Horman,
linux-kernel, Vladimir Oltean
In-Reply-To: <20260402223437.109097-2-mmyangfl@gmail.com>
On Fri, Apr 03, 2026 at 06:34:29AM +0800, David Yang wrote:
> Users may use extack for a friendly error message instead of dumping
> everything into dmesg.
>
> Make the according transformations to the two users (sja1105 and
> felix).
>
> Signed-off-by: David Yang <mmyangfl@gmail.com>
Reviewed-by: Andrew Lunn <andrew@lunn.ch>
Andrew
^ permalink raw reply
* [PATCH v12 6/6] selftests: net: add TLS hardware offload test
From: Rishikesh Jethwani @ 2026-04-02 23:55 UTC (permalink / raw)
To: netdev
Cc: saeedm, tariqt, mbloch, borisp, john.fastabend, kuba, sd, davem,
pabeni, edumazet, leon, Rishikesh Jethwani
In-Reply-To: <20260402235511.664801-1-rjethwani@purestorage.com>
Two-node kTLS hardware offload test using NetDrvEpEnv. Tests TLS
1.2/1.3 with AES-GCM-128/256, rekey operations, and various buffer
sizes.
Signed-off-by: Rishikesh Jethwani <rjethwani@purestorage.com>
---
.../selftests/drivers/net/hw/.gitignore | 1 +
.../testing/selftests/drivers/net/hw/Makefile | 2 +
.../selftests/drivers/net/hw/tls_hw_offload.c | 767 ++++++++++++++++++
.../drivers/net/hw/tls_hw_offload.py | 171 ++++
4 files changed, 941 insertions(+)
create mode 100644 tools/testing/selftests/drivers/net/hw/tls_hw_offload.c
create mode 100755 tools/testing/selftests/drivers/net/hw/tls_hw_offload.py
diff --git a/tools/testing/selftests/drivers/net/hw/.gitignore b/tools/testing/selftests/drivers/net/hw/.gitignore
index 46540468a775..f0a5d15b469b 100644
--- a/tools/testing/selftests/drivers/net/hw/.gitignore
+++ b/tools/testing/selftests/drivers/net/hw/.gitignore
@@ -2,3 +2,4 @@
iou-zcrx
ncdevmem
toeplitz
+tls_hw_offload
diff --git a/tools/testing/selftests/drivers/net/hw/Makefile b/tools/testing/selftests/drivers/net/hw/Makefile
index deeca3f8d080..261ee453610f 100644
--- a/tools/testing/selftests/drivers/net/hw/Makefile
+++ b/tools/testing/selftests/drivers/net/hw/Makefile
@@ -15,6 +15,7 @@ endif
TEST_GEN_FILES := \
$(COND_GEN_FILES) \
+ tls_hw_offload \
# end of TEST_GEN_FILES
TEST_PROGS = \
@@ -41,6 +42,7 @@ TEST_PROGS = \
rss_drv.py \
rss_flow_label.py \
rss_input_xfrm.py \
+ tls_hw_offload.py \
toeplitz.py \
tso.py \
xdp_metadata.py \
diff --git a/tools/testing/selftests/drivers/net/hw/tls_hw_offload.c b/tools/testing/selftests/drivers/net/hw/tls_hw_offload.c
new file mode 100644
index 000000000000..788891890ec8
--- /dev/null
+++ b/tools/testing/selftests/drivers/net/hw/tls_hw_offload.c
@@ -0,0 +1,767 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * TLS Hardware Offload Two-Node Test
+ *
+ * Tests kTLS hardware offload between two physical nodes using
+ * hardcoded keys. Supports TLS 1.2/1.3, AES-GCM-128/256, and rekey.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <time.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+#include <arpa/inet.h>
+#include <linux/tls.h>
+
+#define TLS_RECORD_TYPE_HANDSHAKE 22
+#define TLS_HANDSHAKE_KEY_UPDATE 0x18
+#define KEY_UPDATE_NOT_REQUESTED 0
+#define KEY_UPDATE_REQUESTED 1
+
+#define TEST_ITERATIONS 100
+#define MAX_REKEYS 99
+#define MIN_BUF_SIZE 16 /* must fit TLS handshake msg (KeyUpdate = 5 B) */
+
+/* Initial key material */
+static struct tls12_crypto_info_aes_gcm_128 tls_info_key0_128 = {
+ .info = {
+ .version = TLS_1_3_VERSION,
+ .cipher_type = TLS_CIPHER_AES_GCM_128,
+ },
+ .iv = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08 },
+ .key = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
+ 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10 },
+ .salt = { 0x01, 0x02, 0x03, 0x04 },
+ .rec_seq = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+};
+
+static struct tls12_crypto_info_aes_gcm_256 tls_info_key0_256 = {
+ .info = {
+ .version = TLS_1_3_VERSION,
+ .cipher_type = TLS_CIPHER_AES_GCM_256,
+ },
+ .iv = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08 },
+ .key = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
+ 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10,
+ 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18,
+ 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20 },
+ .salt = { 0x01, 0x02, 0x03, 0x04 },
+ .rec_seq = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+};
+
+static int num_rekeys;
+static int cipher_type = TLS_CIPHER_AES_GCM_128;
+static int tls_version = TLS_1_3_VERSION;
+static int server_port = 4433;
+static char *server_ip;
+
+static int send_size = 16384;
+static int random_size_max;
+
+/*
+ * Scramble key material fields for a given rekey generation.
+ * Generation 0 uses the base key unchanged; generation N XORs a
+ * deterministic pattern into each field so both endpoints derive
+ * identical keys without a real KDF.
+ */
+static void derive_key_fields(unsigned char *key, int key_size,
+ unsigned char *iv, int iv_size,
+ unsigned char *salt, int salt_size,
+ unsigned char *rec_seq, int rec_seq_size,
+ int generation)
+{
+ unsigned char pattern;
+ int i;
+
+ if (generation == 0)
+ return;
+
+ pattern = (unsigned char)((generation * 0x1B) ^ 0x63);
+ for (i = 0; i < key_size; i++) {
+ key[i] ^= pattern;
+ pattern = (pattern << 1) | (pattern >> 7);
+ }
+
+ pattern = (unsigned char)((generation * 0x2D) ^ 0x7C);
+ for (i = 0; i < iv_size; i++) {
+ iv[i] ^= pattern;
+ pattern = (pattern << 1) | (pattern >> 7);
+ }
+
+ for (i = 0; i < salt_size; i++)
+ salt[i] ^= (unsigned char)(generation & 0xFF);
+
+ memset(rec_seq, 0, rec_seq_size);
+}
+
+static void derive_key_128(struct tls12_crypto_info_aes_gcm_128 *key,
+ int generation)
+{
+ memcpy(key, &tls_info_key0_128, sizeof(*key));
+ key->info.version = tls_version;
+ derive_key_fields(key->key, TLS_CIPHER_AES_GCM_128_KEY_SIZE,
+ key->iv, TLS_CIPHER_AES_GCM_128_IV_SIZE,
+ key->salt, TLS_CIPHER_AES_GCM_128_SALT_SIZE,
+ key->rec_seq, TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE,
+ generation);
+}
+
+static void derive_key_256(struct tls12_crypto_info_aes_gcm_256 *key,
+ int generation)
+{
+ memcpy(key, &tls_info_key0_256, sizeof(*key));
+ key->info.version = tls_version;
+ derive_key_fields(key->key, TLS_CIPHER_AES_GCM_256_KEY_SIZE,
+ key->iv, TLS_CIPHER_AES_GCM_256_IV_SIZE,
+ key->salt, TLS_CIPHER_AES_GCM_256_SALT_SIZE,
+ key->rec_seq, TLS_CIPHER_AES_GCM_256_REC_SEQ_SIZE,
+ generation);
+}
+
+static const char *cipher_name(int cipher)
+{
+ switch (cipher) {
+ case TLS_CIPHER_AES_GCM_128: return "AES-GCM-128";
+ case TLS_CIPHER_AES_GCM_256: return "AES-GCM-256";
+ default: return "unknown";
+ }
+}
+
+static const char *version_name(int version)
+{
+ switch (version) {
+ case TLS_1_2_VERSION: return "TLS 1.2";
+ case TLS_1_3_VERSION: return "TLS 1.3";
+ default: return "unknown";
+ }
+}
+
+static int setup_tls_ulp(int fd)
+{
+ int ret;
+
+ ret = setsockopt(fd, IPPROTO_TCP, TCP_ULP, "tls", sizeof("tls"));
+ if (ret < 0) {
+ printf("TCP_ULP failed: %s\n", strerror(errno));
+ return -1;
+ }
+ return 0;
+}
+
+/* Send TLS 1.3 KeyUpdate handshake message */
+static int send_tls_key_update(int fd, int request_update)
+{
+ char cmsg_buf[CMSG_SPACE(sizeof(unsigned char))];
+ unsigned char key_update_msg[5];
+ struct msghdr msg = {0};
+ struct cmsghdr *cmsg;
+ struct iovec iov;
+
+ key_update_msg[0] = TLS_HANDSHAKE_KEY_UPDATE;
+ key_update_msg[1] = 0;
+ key_update_msg[2] = 0;
+ key_update_msg[3] = 1;
+ key_update_msg[4] = request_update ? KEY_UPDATE_REQUESTED
+ : KEY_UPDATE_NOT_REQUESTED;
+
+ iov.iov_base = key_update_msg;
+ iov.iov_len = sizeof(key_update_msg);
+
+ msg.msg_iov = &iov;
+ msg.msg_iovlen = 1;
+ msg.msg_control = cmsg_buf;
+ msg.msg_controllen = sizeof(cmsg_buf);
+
+ cmsg = CMSG_FIRSTHDR(&msg);
+ cmsg->cmsg_level = SOL_TLS;
+ cmsg->cmsg_type = TLS_SET_RECORD_TYPE;
+ cmsg->cmsg_len = CMSG_LEN(sizeof(unsigned char));
+ *CMSG_DATA(cmsg) = TLS_RECORD_TYPE_HANDSHAKE;
+ msg.msg_controllen = cmsg->cmsg_len;
+
+ if (sendmsg(fd, &msg, 0) < 0) {
+ printf("sendmsg KeyUpdate failed: %s\n", strerror(errno));
+ return -1;
+ }
+
+ printf("Sent TLS KeyUpdate handshake message\n");
+ return 0;
+}
+
+static int recv_tls_message(int fd, char *buf, size_t buflen, int *record_type)
+{
+ char cmsg_buf[CMSG_SPACE(sizeof(unsigned char))];
+ struct msghdr msg = {0};
+ struct cmsghdr *cmsg;
+ struct iovec iov;
+ int ret;
+
+ iov.iov_base = buf;
+ iov.iov_len = buflen;
+
+ msg.msg_iov = &iov;
+ msg.msg_iovlen = 1;
+ msg.msg_control = cmsg_buf;
+ msg.msg_controllen = sizeof(cmsg_buf);
+
+ ret = recvmsg(fd, &msg, 0);
+ if (ret <= 0)
+ return ret;
+
+ cmsg = CMSG_FIRSTHDR(&msg);
+ if (cmsg && cmsg->cmsg_level == SOL_TLS &&
+ cmsg->cmsg_type == TLS_GET_RECORD_TYPE)
+ *record_type = *((unsigned char *)CMSG_DATA(cmsg));
+
+ return ret;
+}
+
+/*
+ * Validate a KeyUpdate handshake message per RFC 8446:
+ * HandshakeType (1) = 0x18, Length (3) = 0x000001,
+ * KeyUpdateRequest (1) = 0 or 1.
+ */
+static int validate_keyupdate(const char *buf, int len)
+{
+ if (len != 5) {
+ printf("KeyUpdate: expected 5 bytes, got %d\n", len);
+ return -1;
+ }
+
+ if ((unsigned char)buf[0] != TLS_HANDSHAKE_KEY_UPDATE) {
+ printf("Expected KeyUpdate (0x%02x), got 0x%02x\n",
+ TLS_HANDSHAKE_KEY_UPDATE, (unsigned char)buf[0]);
+ return -1;
+ }
+
+ if (buf[1] != 0 || buf[2] != 0 || buf[3] != 1) {
+ printf("KeyUpdate: bad length field %02x%02x%02x\n",
+ (unsigned char)buf[1], (unsigned char)buf[2],
+ (unsigned char)buf[3]);
+ return -1;
+ }
+
+ if ((unsigned char)buf[4] != KEY_UPDATE_NOT_REQUESTED &&
+ (unsigned char)buf[4] != KEY_UPDATE_REQUESTED) {
+ printf("KeyUpdate: invalid request_update value %u\n",
+ (unsigned char)buf[4]);
+ return -1;
+ }
+
+ printf("Received TLS KeyUpdate (request_update=%u)\n",
+ (unsigned char)buf[4]);
+ return 0;
+}
+
+static int recv_tls_keyupdate(int fd)
+{
+ char buf[MIN_BUF_SIZE];
+ int record_type;
+ int ret;
+
+ ret = recv_tls_message(fd, buf, sizeof(buf), &record_type);
+ if (ret < 0) {
+ printf("recv_tls_message failed: %s\n", strerror(errno));
+ return -1;
+ }
+
+ if (record_type != TLS_RECORD_TYPE_HANDSHAKE) {
+ printf("Expected handshake record (0x%02x), got 0x%02x\n",
+ TLS_RECORD_TYPE_HANDSHAKE, record_type);
+ return -1;
+ }
+
+ return validate_keyupdate(buf, ret);
+}
+
+static int check_ekeyexpired(int fd)
+{
+ char buf[MIN_BUF_SIZE];
+ int ret;
+
+ ret = recv(fd, buf, sizeof(buf), MSG_DONTWAIT);
+ if (ret == -1 && errno == EKEYEXPIRED) {
+ printf("recv() returned EKEYEXPIRED as expected\n");
+ return 0;
+ } else if (ret == -1 && errno == EAGAIN) {
+ printf("recv() returned EAGAIN (no pending data)\n");
+ return 0;
+ } else if (ret > 0) {
+ printf("FAIL: recv() returned %d bytes, expected EKEYEXPIRED\n",
+ ret);
+ return -1;
+ } else {
+ printf("FAIL: recv() returned unexpected error: %s\n",
+ strerror(errno));
+ return -1;
+ }
+}
+
+static int do_tls_rekey(int fd, int direction, int generation, int cipher)
+{
+ const char *dir = direction == TLS_TX ? "TX" : "RX";
+ int ret;
+
+ printf("%s TLS_%s %s gen %d...\n",
+ generation ? "Rekeying" : "Installing",
+ dir, cipher_name(cipher), generation);
+
+ if (cipher == TLS_CIPHER_AES_GCM_256) {
+ struct tls12_crypto_info_aes_gcm_256 key;
+
+ derive_key_256(&key, generation);
+ ret = setsockopt(fd, SOL_TLS, direction, &key, sizeof(key));
+ } else {
+ struct tls12_crypto_info_aes_gcm_128 key;
+
+ derive_key_128(&key, generation);
+ ret = setsockopt(fd, SOL_TLS, direction, &key, sizeof(key));
+ }
+
+ if (ret < 0) {
+ printf("TLS_%s %s gen %d failed: %s\n", dir,
+ cipher_name(cipher), generation, strerror(errno));
+ return -1;
+ }
+ printf("TLS_%s %s gen %d installed\n",
+ dir, cipher_name(cipher), generation);
+ return 0;
+}
+
+static int do_client(void)
+{
+ char *buf = NULL, *echo_buf = NULL;
+ int max_size, rekey_interval;
+ ssize_t echo_total, echo_n;
+ int csk = -1, ret, i, j;
+ struct sockaddr_in sa;
+ int test_result = -1;
+ int current_gen = 0;
+ int next_rekey_at;
+ ssize_t n;
+
+ if (!server_ip) {
+ printf("ERROR: Client requires -s <ip> option\n");
+ return -1;
+ }
+
+ max_size = random_size_max > 0 ? random_size_max : send_size;
+ if (max_size < MIN_BUF_SIZE)
+ max_size = MIN_BUF_SIZE;
+ buf = malloc(max_size);
+ echo_buf = malloc(max_size);
+ if (!buf || !echo_buf) {
+ printf("failed to allocate buffers\n");
+ goto out;
+ }
+
+ csk = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
+ if (csk < 0) {
+ printf("failed to create socket: %s\n", strerror(errno));
+ goto out;
+ }
+
+ memset(&sa, 0, sizeof(sa));
+ sa.sin_family = AF_INET;
+ sa.sin_addr.s_addr = inet_addr(server_ip);
+ sa.sin_port = htons(server_port);
+ printf("Connecting to %s:%d...\n", server_ip, server_port);
+
+ ret = connect(csk, (struct sockaddr *)&sa, sizeof(sa));
+ if (ret < 0) {
+ printf("connect failed: %s\n", strerror(errno));
+ goto out;
+ }
+ printf("Connected!\n");
+
+ if (setup_tls_ulp(csk) < 0)
+ goto out;
+
+ if (do_tls_rekey(csk, TLS_TX, 0, cipher_type) < 0 ||
+ do_tls_rekey(csk, TLS_RX, 0, cipher_type) < 0)
+ goto out;
+
+ if (num_rekeys)
+ printf("TLS %s setup complete. Will perform %d rekey(s).\n",
+ cipher_name(cipher_type), num_rekeys);
+ else
+ printf("TLS setup complete.\n");
+
+ if (random_size_max > 0)
+ printf("Sending %d messages of random size (1..%d bytes)...\n",
+ TEST_ITERATIONS, random_size_max);
+ else
+ printf("Sending %d messages of %d bytes...\n",
+ TEST_ITERATIONS, send_size);
+
+ rekey_interval = TEST_ITERATIONS / (num_rekeys + 1);
+ if (rekey_interval < 1)
+ rekey_interval = 1;
+ next_rekey_at = rekey_interval;
+
+ for (i = 0; i < TEST_ITERATIONS; i++) {
+ int this_size;
+
+ if (random_size_max > 0)
+ this_size = (rand() % random_size_max) + 1;
+ else
+ this_size = send_size;
+
+ for (j = 0; j < this_size; j++)
+ buf[j] = rand() & 0xFF;
+
+ n = send(csk, buf, this_size, 0);
+ if (n != this_size) {
+ printf("FAIL: send failed: %s\n", strerror(errno));
+ goto out;
+ }
+ printf("Sent %zd bytes (iteration %d)\n", n, i + 1);
+
+ echo_total = 0;
+ while (echo_total < n) {
+ echo_n = recv(csk, echo_buf + echo_total,
+ n - echo_total, 0);
+ if (echo_n < 0) {
+ printf("FAIL: Echo recv failed: %s\n",
+ strerror(errno));
+ goto out;
+ }
+ if (echo_n == 0) {
+ printf("FAIL: Connection closed during echo\n");
+ goto out;
+ }
+ echo_total += echo_n;
+ }
+
+ if (memcmp(buf, echo_buf, n) != 0) {
+ printf("FAIL: Echo data mismatch!\n");
+ goto out;
+ }
+ printf("Received echo %zd bytes (ok)\n", echo_total);
+
+ /* Rekey at intervals: send KeyUpdate, update TX, recv KeyUpdate, update RX */
+ if (num_rekeys && current_gen < num_rekeys &&
+ (i + 1) == next_rekey_at) {
+ current_gen++;
+ printf("\n=== Client Rekey gen %d ===\n", current_gen);
+
+ ret = send_tls_key_update(csk, KEY_UPDATE_REQUESTED);
+ if (ret < 0) {
+ printf("FAIL: send KeyUpdate\n");
+ goto out;
+ }
+
+ ret = do_tls_rekey(csk, TLS_TX, current_gen, cipher_type);
+ if (ret < 0)
+ goto out;
+
+ if (recv_tls_keyupdate(csk) < 0) {
+ printf("FAIL: recv KeyUpdate from server\n");
+ goto out;
+ }
+
+ if (check_ekeyexpired(csk) < 0)
+ goto out;
+
+ ret = do_tls_rekey(csk, TLS_RX, current_gen, cipher_type);
+ if (ret < 0)
+ goto out;
+
+ next_rekey_at += rekey_interval;
+ printf("=== Client Rekey gen %d Complete ===\n\n",
+ current_gen);
+ }
+ }
+
+ test_result = 0;
+out:
+ if (num_rekeys)
+ printf("Rekeys completed: %d/%d\n", current_gen, num_rekeys);
+ if (csk >= 0)
+ close(csk);
+ free(buf);
+ free(echo_buf);
+ return test_result;
+}
+
+static int do_server(void)
+{
+ int lsk = -1, csk = -1, ret;
+ ssize_t n, total = 0, sent;
+ struct sockaddr_in sa;
+ int test_result = -1;
+ int current_gen = 0;
+ int recv_count = 0;
+ char *buf = NULL;
+ int record_type;
+ int buf_size;
+ int one = 1;
+
+ buf_size = send_size;
+ if (buf_size < MIN_BUF_SIZE)
+ buf_size = MIN_BUF_SIZE;
+ buf = malloc(buf_size);
+ if (!buf) {
+ printf("failed to allocate buffer\n");
+ goto out;
+ }
+
+ lsk = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
+ if (lsk < 0) {
+ printf("failed to create socket: %s\n", strerror(errno));
+ goto out;
+ }
+
+ setsockopt(lsk, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one));
+
+ memset(&sa, 0, sizeof(sa));
+ sa.sin_family = AF_INET;
+ sa.sin_addr.s_addr = INADDR_ANY;
+ sa.sin_port = htons(server_port);
+
+ ret = bind(lsk, (struct sockaddr *)&sa, sizeof(sa));
+ if (ret < 0) {
+ printf("bind failed: %s\n", strerror(errno));
+ goto out;
+ }
+
+ ret = listen(lsk, 1);
+ if (ret < 0) {
+ printf("listen failed: %s\n", strerror(errno));
+ goto out;
+ }
+
+ printf("Server listening on 0.0.0.0:%d\n", server_port);
+ printf("Waiting for client connection...\n");
+
+ csk = accept(lsk, (struct sockaddr *)NULL, (socklen_t *)NULL);
+ if (csk < 0) {
+ printf("accept failed: %s\n", strerror(errno));
+ goto out;
+ }
+ printf("Client connected!\n");
+
+ if (setup_tls_ulp(csk) < 0)
+ goto out;
+
+ if (do_tls_rekey(csk, TLS_TX, 0, cipher_type) < 0 ||
+ do_tls_rekey(csk, TLS_RX, 0, cipher_type) < 0)
+ goto out;
+
+ printf("TLS %s setup complete. Receiving...\n",
+ cipher_name(cipher_type));
+
+ /* Main receive loop */
+ while (1) {
+ n = recv_tls_message(csk, buf, buf_size, &record_type);
+ if (n == 0) {
+ printf("Connection closed by client\n");
+ break;
+ }
+ if (n < 0) {
+ printf("recv failed: %s\n", strerror(errno));
+ break;
+ }
+
+ /* Handle KeyUpdate: validate, update RX, respond, update TX */
+ if (record_type == TLS_RECORD_TYPE_HANDSHAKE) {
+ if (validate_keyupdate(buf, n) < 0)
+ goto out;
+ current_gen++;
+ printf("\n=== Server Rekey gen %d ===\n", current_gen);
+
+ if (check_ekeyexpired(csk) < 0)
+ goto out;
+
+ ret = do_tls_rekey(csk, TLS_RX, current_gen, cipher_type);
+ if (ret < 0)
+ goto out;
+
+ ret = send_tls_key_update(csk,
+ KEY_UPDATE_NOT_REQUESTED);
+ if (ret < 0) {
+ printf("Failed to send KeyUpdate\n");
+ goto out;
+ }
+
+ ret = do_tls_rekey(csk, TLS_TX, current_gen, cipher_type);
+ if (ret < 0)
+ goto out;
+
+ printf("=== Server Rekey gen %d Complete ===\n\n",
+ current_gen);
+ continue;
+ }
+
+ total += n;
+ recv_count++;
+ printf("Received %zd bytes (total: %zd, count: %d)\n",
+ n, total, recv_count);
+
+ for (sent = 0; sent < n; sent += ret) {
+ ret = send(csk, buf + sent, n - sent, 0);
+ if (ret < 0) {
+ printf("Echo send failed: %s\n",
+ strerror(errno));
+ goto out;
+ }
+ }
+ printf("Echoed %zd bytes back to client\n", n);
+ }
+
+ test_result = 0;
+out:
+ printf("Connection closed. Total received: %zd bytes\n", total);
+ if (num_rekeys)
+ printf("Rekeys completed: %d\n", current_gen);
+
+ if (csk >= 0)
+ close(csk);
+ if (lsk >= 0)
+ close(lsk);
+ free(buf);
+ return test_result;
+}
+
+static int parse_cipher_option(const char *arg)
+{
+ if (strcmp(arg, "128") == 0) {
+ cipher_type = TLS_CIPHER_AES_GCM_128;
+ return 0;
+ } else if (strcmp(arg, "256") == 0) {
+ cipher_type = TLS_CIPHER_AES_GCM_256;
+ return 0;
+ }
+ printf("ERROR: Invalid cipher '%s'. Must be 128 or 256.\n", arg);
+ return -1;
+}
+
+static int parse_version_option(const char *arg)
+{
+ if (strcmp(arg, "1.2") == 0) {
+ tls_version = TLS_1_2_VERSION;
+ return 0;
+ } else if (strcmp(arg, "1.3") == 0) {
+ tls_version = TLS_1_3_VERSION;
+ return 0;
+ }
+ printf("ERROR: Invalid TLS version '%s'. Must be 1.2 or 1.3.\n", arg);
+ return -1;
+}
+
+static void print_usage(const char *prog)
+{
+ printf("TLS Hardware Offload Two-Node Test\n\n");
+ printf("Usage:\n");
+ printf(" %s server [OPTIONS]\n", prog);
+ printf(" %s client -s <ip> [OPTIONS]\n", prog);
+ printf("\nOptions:\n");
+ printf(" -s <ip> Server IPv4 address (client, required)\n");
+ printf(" -p <port> Server port (default: 4433)\n");
+ printf(" -b <size> Send buffer size in bytes (default: 16384)\n");
+ printf(" -r <max> Use random send buffer sizes (1..<max>)\n");
+ printf(" -v <version> TLS version: 1.2 or 1.3 (default: 1.3)\n");
+ printf(" -c <cipher> Cipher: 128 or 256 (default: 128)\n");
+ printf(" -k <N> Perform N rekeys (client only, TLS 1.3)\n");
+ printf(" -h Show this help message\n");
+ printf("\nExample:\n");
+ printf(" Node A: %s server\n", prog);
+ printf(" Node B: %s client -s 192.168.20.2\n", prog);
+ printf("\nRekey Example (3 rekeys, TLS 1.3 only):\n");
+ printf(" Node A: %s server\n", prog);
+ printf(" Node B: %s client -s 192.168.20.2 -k 3\n", prog);
+}
+
+int main(int argc, char *argv[])
+{
+ int opt;
+
+ if (argc < 2 ||
+ (strcmp(argv[1], "server") && strcmp(argv[1], "client"))) {
+ print_usage(argv[0]);
+ return -1;
+ }
+
+ optind = 2; /* skip subcommand */
+ while ((opt = getopt(argc, argv, "s:p:b:r:c:v:k:h")) != -1) {
+ switch (opt) {
+ case 's':
+ server_ip = optarg;
+ break;
+ case 'p':
+ server_port = atoi(optarg);
+ if (server_port < 1 || server_port > 65535) {
+ printf("ERROR: Invalid port '%s'. Must be 1..65535.\n",
+ optarg);
+ return -1;
+ }
+ break;
+ case 'b':
+ send_size = atoi(optarg);
+ if (send_size < 1) {
+ printf("ERROR: Invalid buffer size '%s'. Must be >= 1.\n",
+ optarg);
+ return -1;
+ }
+ break;
+ case 'r':
+ random_size_max = atoi(optarg);
+ if (random_size_max < 1) {
+ printf("ERROR: Invalid random size '%s'. Must be >= 1.\n",
+ optarg);
+ return -1;
+ }
+ break;
+ case 'c':
+ if (parse_cipher_option(optarg) < 0)
+ return -1;
+ break;
+ case 'v':
+ if (parse_version_option(optarg) < 0)
+ return -1;
+ break;
+ case 'k':
+ num_rekeys = atoi(optarg);
+ if (num_rekeys < 1 || num_rekeys > MAX_REKEYS) {
+ printf("ERROR: Invalid rekey count '%s'. Must be 1..%d.\n",
+ optarg, MAX_REKEYS);
+ return -1;
+ }
+ break;
+ case 'h':
+ print_usage(argv[0]);
+ return 0;
+ default:
+ print_usage(argv[0]);
+ return -1;
+ }
+ }
+
+ if (tls_version == TLS_1_2_VERSION && num_rekeys) {
+ printf("ERROR: TLS 1.2 does not support rekey\n");
+ return -1;
+ }
+
+ printf("TLS Version: %s\n", version_name(tls_version));
+ printf("Cipher: %s\n", cipher_name(cipher_type));
+ if (random_size_max > 0)
+ printf("Buffer size: random (1..%d)\n", random_size_max);
+ else
+ printf("Buffer size: %d\n", send_size);
+
+ if (num_rekeys)
+ printf("Rekey testing ENABLED: %d rekey(s)\n", num_rekeys);
+
+ srand(time(NULL));
+
+ if (!strcmp(argv[1], "client"))
+ return do_client();
+
+ return do_server();
+}
diff --git a/tools/testing/selftests/drivers/net/hw/tls_hw_offload.py b/tools/testing/selftests/drivers/net/hw/tls_hw_offload.py
new file mode 100755
index 000000000000..66c5ddfd8125
--- /dev/null
+++ b/tools/testing/selftests/drivers/net/hw/tls_hw_offload.py
@@ -0,0 +1,171 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0
+
+"""Test kTLS hardware offload using a C helper binary."""
+
+from collections import defaultdict
+
+from lib.py import ksft_run, ksft_exit, ksft_pr, KsftSkipEx, ksft_true
+from lib.py import ksft_variants, KsftNamedVariant
+from lib.py import NetDrvEpEnv
+from lib.py import cmd, bkg, wait_port_listen, rand_port
+
+
+def check_tls_support(cfg):
+ try:
+ cmd("test -f /proc/net/tls_stat")
+ cmd("test -f /proc/net/tls_stat", host=cfg.remote)
+ except Exception as e:
+ raise KsftSkipEx(f"kTLS not supported: {e}")
+
+
+def read_tls_stats(host=None):
+ stats = defaultdict(int)
+ output = cmd("cat /proc/net/tls_stat", host=host)
+ for line in output.stdout.strip().split('\n'):
+ parts = line.split()
+ if len(parts) == 2:
+ stats[parts[0]] = int(parts[1])
+ return stats
+
+
+def stat_diff(before, after, key):
+ """Print counter before/after and return the diff."""
+ before_val, after_val = before[key], after[key]
+ diff = after_val - before_val
+ ksft_pr(f"{key}: {before_val} -> {after_val} (diff: {diff})")
+ return diff
+
+
+def check_path(before, after, direction):
+ """Check that HW or SW offload was used for a given direction."""
+ dev = stat_diff(before, after, f'Tls{direction}Device')
+ sw = stat_diff(before, after, f'Tls{direction}Sw')
+ if dev >= 1:
+ ksft_pr(f"{direction} Path: HARDWARE OFFLOAD")
+ return 0
+ if sw >= 1:
+ ksft_pr(f"{direction} Path: SOFTWARE")
+ return 0
+ ksft_pr(f"{direction} Path: FAIL (no TLS {direction} activity detected)")
+ return 1
+
+
+def check_min(before, after, key, minimum, label):
+ """Check that a counter increased by at least minimum."""
+ diff = stat_diff(before, after, key)
+ if diff < minimum:
+ ksft_pr(f"FAIL: Expected >= {minimum} {label}")
+ return 1
+ return 0
+
+
+def check_zero(before, after, key):
+ """Check that an error counter did not increase."""
+ diff = stat_diff(before, after, key)
+ if diff > 0:
+ ksft_pr(f"ERROR: {key} increased by {diff}")
+ return 1
+ return 0
+
+
+def verify_tls_counters(stats_before, stats_after, expected_rekeys, is_server):
+ errors = 0
+ role = 'Server' if is_server else 'Client'
+ ksft_pr(f"=== Counter Verification ({role}) ===")
+
+ errors += check_path(stats_before, stats_after, 'Tx')
+ errors += check_path(stats_before, stats_after, 'Rx')
+
+ if expected_rekeys > 0:
+ errors += check_min(stats_before, stats_after,
+ 'TlsTxRekeyOk', expected_rekeys, "TX rekeys")
+ errors += check_min(stats_before, stats_after,
+ 'TlsRxRekeyOk', expected_rekeys, "RX rekeys")
+ if is_server:
+ errors += check_min(stats_before, stats_after,
+ 'TlsRxRekeyReceived', expected_rekeys,
+ "KeyUpdate messages")
+ errors += check_zero(stats_before, stats_after, 'TlsTxRekeyError')
+ errors += check_zero(stats_before, stats_after, 'TlsRxRekeyError')
+
+ errors += check_zero(stats_before, stats_after, 'TlsDecryptError')
+
+ ksft_pr(f"=== Verification {'PASSED' if errors == 0 else 'FAILED'} ===\n")
+ return errors == 0
+
+
+def run_tls_test(cfg, cipher="128", tls_version="1.3", rekey=0, buffer_size=None, random_max=None):
+ port = rand_port()
+ send_size = random_max or buffer_size
+
+ server_args = f"{cfg.bin_remote} server -p {port} -c {cipher} -v {tls_version}"
+ if send_size:
+ server_args += f" -b {send_size}"
+
+ client_args = (f"{cfg.bin_local} client -s {cfg.remote_addr_v['4']} "
+ f"-p {port} -c {cipher} -v {tls_version}")
+ if rekey:
+ client_args += f" -k {rekey}"
+ if random_max:
+ client_args += f" -r {random_max}"
+ elif send_size:
+ client_args += f" -b {send_size}"
+
+ stats_before_local = read_tls_stats()
+ stats_before_remote = read_tls_stats(host=cfg.remote)
+
+ with bkg(server_args, host=cfg.remote, exit_wait=True):
+ wait_port_listen(port, host=cfg.remote)
+ result = cmd(client_args, fail=False)
+
+ stats_after_local = read_tls_stats()
+ stats_after_remote = read_tls_stats(host=cfg.remote)
+
+ client_ok = verify_tls_counters(stats_before_local, stats_after_local,
+ rekey, False)
+ server_ok = verify_tls_counters(stats_before_remote, stats_after_remote,
+ rekey, True)
+
+ ksft_true(result.ret == 0, "Client completed successfully")
+ ksft_true(client_ok, "Client TLS counters verified")
+ ksft_true(server_ok, "Server TLS counters verified")
+
+
+@ksft_variants([
+ KsftNamedVariant("tls13_aes128", "128", "1.3"),
+ KsftNamedVariant("tls13_aes256", "256", "1.3"),
+ KsftNamedVariant("tls12_aes128", "128", "1.2"),
+ KsftNamedVariant("tls12_aes256", "256", "1.2"),
+])
+def test_tls_offload(cfg, cipher, tls_version):
+ run_tls_test(cfg, cipher=cipher, tls_version=tls_version)
+
+
+@ksft_variants([
+ KsftNamedVariant("single", 1),
+ KsftNamedVariant("multiple", 99),
+ KsftNamedVariant("small_buf", 30, 512),
+ KsftNamedVariant("large_buf", 10, 2097152),
+ KsftNamedVariant("random_sizes", 20, None, 8192),
+])
+def test_tls_offload_rekey(cfg, rekey, buffer_size=None, random_max=None):
+ run_tls_test(cfg, cipher="128", tls_version="1.3", rekey=rekey,
+ buffer_size=buffer_size, random_max=random_max)
+
+
+def main() -> None:
+ with NetDrvEpEnv(__file__, nsim_test=False) as cfg:
+ cfg.bin_local = cfg.test_dir / "tls_hw_offload"
+ if not cfg.bin_local.exists():
+ raise KsftSkipEx(f"tls_hw_offload binary not found at {cfg.bin_local}")
+ cfg.bin_remote = cfg.remote.deploy(cfg.bin_local)
+ cfg.require_ipver("4")
+ check_tls_support(cfg)
+
+ ksft_run([test_tls_offload, test_tls_offload_rekey], args=(cfg, ))
+ ksft_exit()
+
+
+if __name__ == "__main__":
+ main()
--
2.25.1
^ permalink raw reply related
* [PATCH v12 5/6] tls: add hardware offload key update support
From: Rishikesh Jethwani @ 2026-04-02 23:55 UTC (permalink / raw)
To: netdev
Cc: saeedm, tariqt, mbloch, borisp, john.fastabend, kuba, sd, davem,
pabeni, edumazet, leon, Rishikesh Jethwani
In-Reply-To: <20260402235511.664801-1-rjethwani@purestorage.com>
During a TLS 1.3 KeyUpdate the NIC key cannot be replaced immediately
if previously encrypted HW records are awaiting ACK. start_rekey sets
up a temporary SW context with the new key and redirects sendmsg through
tls_sw_sendmsg_locked. When no records are pending, complete_rekey runs
inline during setsockopt. Otherwise, clean_acked sets REKEY_READY once
all old-key records are ACKed, and the next sendmsg calls complete_rekey.
complete_rekey flushes remaining SW records, reinstalls HW offload at
the current write_seq, and frees the temporary context.
If another KeyUpdate arrives while a rekey is already pending,
start_rekey just re-keys the existing SW AEAD in place.
If complete_rekey fails (tls_dev_add or crypto_aead_setkey),
we stay in SW mode (REKEY_FAILED) until a subsequent rekey
succeeds, while maintaining TLS_HW configuration.
Tested on Mellanox ConnectX-6 Dx (Crypto Enabled) with multiple
TLS 1.3 key update cycles.
Signed-off-by: Rishikesh Jethwani <rjethwani@purestorage.com>
---
include/net/tls.h | 76 +++--
include/uapi/linux/snmp.h | 2 +
net/tls/tls.h | 16 +-
net/tls/tls_device.c | 508 +++++++++++++++++++++++++++++-----
net/tls/tls_device_fallback.c | 24 ++
net/tls/tls_main.c | 92 +++---
net/tls/tls_proc.c | 2 +
net/tls/tls_sw.c | 32 ++-
8 files changed, 611 insertions(+), 141 deletions(-)
diff --git a/include/net/tls.h b/include/net/tls.h
index ebd2550280ae..f4c5579cd9b5 100644
--- a/include/net/tls.h
+++ b/include/net/tls.h
@@ -151,6 +151,22 @@ struct tls_record_info {
skb_frag_t frags[MAX_SKB_FRAGS];
};
+struct cipher_context {
+ char iv[TLS_MAX_IV_SIZE + TLS_MAX_SALT_SIZE];
+ char rec_seq[TLS_MAX_REC_SEQ_SIZE];
+};
+
+union tls_crypto_context {
+ struct tls_crypto_info info;
+ union {
+ struct tls12_crypto_info_aes_gcm_128 aes_gcm_128;
+ struct tls12_crypto_info_aes_gcm_256 aes_gcm_256;
+ struct tls12_crypto_info_chacha20_poly1305 chacha20_poly1305;
+ struct tls12_crypto_info_sm4_gcm sm4_gcm;
+ struct tls12_crypto_info_sm4_ccm sm4_ccm;
+ };
+};
+
#define TLS_DRIVER_STATE_SIZE_TX 16
struct tls_offload_context_tx {
struct crypto_aead *aead_send;
@@ -165,6 +181,11 @@ struct tls_offload_context_tx {
void (*sk_destruct)(struct sock *sk);
struct work_struct destruct_work;
struct tls_context *ctx;
+
+ struct tls_sw_context_tx rekey_sw; /* SW context for new key */
+ struct cipher_context rekey_tx; /* IV, rec_seq for new key */
+ union tls_crypto_context rekey_crypto_send; /* Crypto for new key */
+
/* The TLS layer reserves room for driver specific state
* Currently the belief is that there is not enough
* driver specific state to justify another layer of indirection
@@ -189,22 +210,21 @@ enum tls_context_flags {
* tls_dev_del call in tls_device_down if it happens simultaneously.
*/
TLS_RX_DEV_CLOSED = 2,
-};
-
-struct cipher_context {
- char iv[TLS_MAX_IV_SIZE + TLS_MAX_SALT_SIZE];
- char rec_seq[TLS_MAX_REC_SEQ_SIZE];
-};
-
-union tls_crypto_context {
- struct tls_crypto_info info;
- union {
- struct tls12_crypto_info_aes_gcm_128 aes_gcm_128;
- struct tls12_crypto_info_aes_gcm_256 aes_gcm_256;
- struct tls12_crypto_info_chacha20_poly1305 chacha20_poly1305;
- struct tls12_crypto_info_sm4_gcm sm4_gcm;
- struct tls12_crypto_info_sm4_ccm sm4_ccm;
- };
+ /* Flag for TX HW context deleted during failed rekey.
+ * Prevents double tls_dev_del in cleanup paths.
+ */
+ TLS_TX_DEV_CLOSED = 3,
+ /* TX rekey is pending, waiting for old-key data to be ACKed.
+ * While set, new data uses SW path with new key, HW keeps old key
+ * for retransmissions.
+ */
+ TLS_TX_REKEY_PENDING = 4,
+ /* All old-key data has been ACKed, ready to install new key in HW. */
+ TLS_TX_REKEY_READY = 5,
+ /* HW rekey failed, permanently stay in SW encrypt mode.
+ * Prevents tls_tcp_clean_acked from re-setting TLS_TX_REKEY_READY.
+ */
+ TLS_TX_REKEY_FAILED = 6,
};
struct tls_prot_info {
@@ -253,6 +273,15 @@ struct tls_context {
*/
unsigned long flags;
+ /* TCP sequence number boundary for pending rekey.
+ * Packets with seq < this use old key, >= use new key.
+ */
+ u32 rekey_boundary_seq;
+
+ /* Pointers to rekey contexts for SW encryption with new key */
+ struct tls_sw_context_tx *rekey_sw_ctx;
+ struct cipher_context *rekey_cipher_ctx;
+
/* cache cold stuff */
struct proto *sk_proto;
struct sock *sk;
@@ -385,9 +414,21 @@ static inline struct tls_sw_context_rx *tls_sw_ctx_rx(
static inline struct tls_sw_context_tx *tls_sw_ctx_tx(
const struct tls_context *tls_ctx)
{
+ if (unlikely(tls_ctx->rekey_sw_ctx))
+ return tls_ctx->rekey_sw_ctx;
+
return (struct tls_sw_context_tx *)tls_ctx->priv_ctx_tx;
}
+static inline struct cipher_context *tls_tx_cipher_ctx(
+ const struct tls_context *tls_ctx)
+{
+ if (unlikely(tls_ctx->rekey_cipher_ctx))
+ return tls_ctx->rekey_cipher_ctx;
+
+ return (struct cipher_context *)&tls_ctx->tx;
+}
+
static inline struct tls_offload_context_tx *
tls_offload_ctx_tx(const struct tls_context *tls_ctx)
{
@@ -500,6 +541,9 @@ struct sk_buff *tls_encrypt_skb(struct sk_buff *skb);
#ifdef CONFIG_TLS_DEVICE
void tls_device_sk_destruct(struct sock *sk);
void tls_offload_tx_resync_request(struct sock *sk, u32 got_seq, u32 exp_seq);
+struct sk_buff *
+tls_validate_xmit_skb_rekey(struct sock *sk, struct net_device *dev,
+ struct sk_buff *skb);
static inline bool tls_is_sk_rx_device_offloaded(struct sock *sk)
{
diff --git a/include/uapi/linux/snmp.h b/include/uapi/linux/snmp.h
index 49f5640092a0..39fa48821faa 100644
--- a/include/uapi/linux/snmp.h
+++ b/include/uapi/linux/snmp.h
@@ -369,6 +369,8 @@ enum
LINUX_MIB_TLSTXREKEYOK, /* TlsTxRekeyOk */
LINUX_MIB_TLSTXREKEYERROR, /* TlsTxRekeyError */
LINUX_MIB_TLSRXREKEYRECEIVED, /* TlsRxRekeyReceived */
+ LINUX_MIB_TLSTXREKEYHWFAIL, /* TlsTxRekeyHwFail */
+ LINUX_MIB_TLSRXREKEYHWFAIL, /* TlsRxRekeyHwFail */
__LINUX_MIB_TLSMAX
};
diff --git a/net/tls/tls.h b/net/tls/tls.h
index a65cf9bab190..df7f892307ff 100644
--- a/net/tls/tls.h
+++ b/net/tls/tls.h
@@ -157,6 +157,9 @@ void tls_update_rx_zc_capable(struct tls_context *tls_ctx);
void tls_sw_strparser_arm(struct sock *sk, struct tls_context *ctx);
void tls_sw_strparser_done(struct tls_context *tls_ctx);
int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
+int tls_sw_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size);
+void tls_tx_work_handler(struct work_struct *work);
+void tls_sw_ctx_tx_init(struct sock *sk, struct tls_sw_context_tx *sw_ctx);
void tls_sw_splice_eof(struct socket *sock);
void tls_sw_cancel_work_tx(struct tls_context *tls_ctx);
void tls_sw_release_resources_tx(struct sock *sk);
@@ -176,6 +179,8 @@ int tls_sw_read_sock(struct sock *sk, read_descriptor_t *desc,
int tls_device_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
void tls_device_splice_eof(struct socket *sock);
int tls_tx_records(struct sock *sk, int flags);
+int tls_sw_push_pending_record(struct sock *sk, int flags);
+int tls_encrypt_async_wait(struct tls_sw_context_tx *ctx);
void tls_sw_write_space(struct sock *sk, struct tls_context *ctx);
void tls_device_write_space(struct sock *sk, struct tls_context *ctx);
@@ -233,9 +238,11 @@ static inline bool tls_strp_msg_mixed_decrypted(struct tls_sw_context_rx *ctx)
#ifdef CONFIG_TLS_DEVICE
int tls_device_init(void);
void tls_device_cleanup(void);
-int tls_set_device_offload(struct sock *sk);
+int tls_set_device_offload(struct sock *sk,
+ struct tls_crypto_info *crypto_info);
void tls_device_free_resources_tx(struct sock *sk);
-int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx);
+int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx,
+ struct tls_crypto_info *crypto_info);
void tls_device_offload_cleanup_rx(struct sock *sk);
void tls_device_rx_resync_new_rec(struct sock *sk, u32 rcd_len, u32 seq);
int tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx);
@@ -244,7 +251,7 @@ static inline int tls_device_init(void) { return 0; }
static inline void tls_device_cleanup(void) {}
static inline int
-tls_set_device_offload(struct sock *sk)
+tls_set_device_offload(struct sock *sk, struct tls_crypto_info *crypto_info)
{
return -EOPNOTSUPP;
}
@@ -252,7 +259,8 @@ tls_set_device_offload(struct sock *sk)
static inline void tls_device_free_resources_tx(struct sock *sk) {}
static inline int
-tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx)
+tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx,
+ struct tls_crypto_info *crypto_info)
{
return -EOPNOTSUPP;
}
diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c
index cd26873e9063..d36159a7302f 100644
--- a/net/tls/tls_device.c
+++ b/net/tls/tls_device.c
@@ -79,7 +79,9 @@ static void tls_device_tx_del_task(struct work_struct *work)
netdev = rcu_dereference_protected(ctx->netdev,
!refcount_read(&ctx->refcount));
- netdev->tlsdev_ops->tls_dev_del(netdev, ctx, TLS_OFFLOAD_CTX_DIR_TX);
+ if (!test_bit(TLS_TX_DEV_CLOSED, &ctx->flags))
+ netdev->tlsdev_ops->tls_dev_del(netdev, ctx,
+ TLS_OFFLOAD_CTX_DIR_TX);
dev_put(netdev);
ctx->netdev = NULL;
tls_device_free_ctx(ctx);
@@ -159,6 +161,254 @@ static void delete_all_records(struct tls_offload_context_tx *offload_ctx)
offload_ctx->retransmit_hint = NULL;
}
+static bool tls_has_unacked_records(struct tls_offload_context_tx *offload_ctx)
+{
+ struct tls_record_info *info;
+ bool has_unacked = false;
+ unsigned long flags;
+
+ spin_lock_irqsave(&offload_ctx->lock, flags);
+ list_for_each_entry(info, &offload_ctx->records_list, list) {
+ if (!tls_record_is_start_marker(info)) {
+ has_unacked = true;
+ break;
+ }
+ }
+ spin_unlock_irqrestore(&offload_ctx->lock, flags);
+
+ return has_unacked;
+}
+
+static int tls_device_init_rekey_sw(struct sock *sk,
+ struct tls_context *ctx,
+ struct tls_offload_context_tx *offload_ctx,
+ struct tls_crypto_info *new_crypto_info)
+{
+ struct tls_sw_context_tx *sw_ctx = &offload_ctx->rekey_sw;
+ const struct tls_cipher_desc *cipher_desc;
+ char *key;
+ int rc;
+
+ cipher_desc = get_cipher_desc(new_crypto_info->cipher_type);
+ DEBUG_NET_WARN_ON_ONCE(!cipher_desc || !cipher_desc->offloadable);
+
+ memset(sw_ctx, 0, sizeof(*sw_ctx));
+ tls_sw_ctx_tx_init(sk, sw_ctx);
+
+ sw_ctx->aead_send = crypto_alloc_aead(cipher_desc->cipher_name, 0, 0);
+ if (IS_ERR(sw_ctx->aead_send)) {
+ rc = PTR_ERR(sw_ctx->aead_send);
+ sw_ctx->aead_send = NULL;
+ return rc;
+ }
+
+ key = crypto_info_key(new_crypto_info, cipher_desc);
+ rc = crypto_aead_setkey(sw_ctx->aead_send, key, cipher_desc->key);
+ if (rc)
+ goto free_aead;
+
+ rc = crypto_aead_setauthsize(sw_ctx->aead_send, cipher_desc->tag);
+ if (rc)
+ goto free_aead;
+
+ return 0;
+
+free_aead:
+ crypto_free_aead(sw_ctx->aead_send);
+ sw_ctx->aead_send = NULL;
+ return rc;
+}
+
+static int tls_device_start_rekey(struct sock *sk,
+ struct tls_context *ctx,
+ struct tls_offload_context_tx *offload_ctx,
+ struct tls_crypto_info *new_crypto_info)
+{
+ bool rekey_pending = test_bit(TLS_TX_REKEY_PENDING, &ctx->flags);
+ bool rekey_failed = test_bit(TLS_TX_REKEY_FAILED, &ctx->flags);
+ const struct tls_cipher_desc *cipher_desc;
+ char *key, *iv, *rec_seq, *salt;
+ int rc;
+
+ cipher_desc = get_cipher_desc(new_crypto_info->cipher_type);
+ DEBUG_NET_WARN_ON_ONCE(!cipher_desc || !cipher_desc->offloadable);
+
+ key = crypto_info_key(new_crypto_info, cipher_desc);
+ iv = crypto_info_iv(new_crypto_info, cipher_desc);
+ rec_seq = crypto_info_rec_seq(new_crypto_info, cipher_desc);
+ salt = crypto_info_salt(new_crypto_info, cipher_desc);
+
+ if (rekey_pending || rekey_failed) {
+ rc = crypto_aead_setkey(offload_ctx->rekey_sw.aead_send,
+ key, cipher_desc->key);
+ if (rc)
+ return rc;
+
+ memcpy(offload_ctx->rekey_tx.iv, salt, cipher_desc->salt);
+ memcpy(offload_ctx->rekey_tx.iv + cipher_desc->salt, iv,
+ cipher_desc->iv);
+ memcpy(offload_ctx->rekey_tx.rec_seq, rec_seq,
+ cipher_desc->rec_seq);
+
+ if (rekey_failed) {
+ set_bit(TLS_TX_REKEY_PENDING, &ctx->flags);
+ clear_bit(TLS_TX_REKEY_FAILED, &ctx->flags);
+ }
+ } else {
+ rc = tls_device_init_rekey_sw(sk, ctx, offload_ctx,
+ new_crypto_info);
+ if (rc)
+ return rc;
+
+ memcpy(offload_ctx->rekey_tx.iv, salt, cipher_desc->salt);
+ memcpy(offload_ctx->rekey_tx.iv + cipher_desc->salt, iv,
+ cipher_desc->iv);
+ memcpy(offload_ctx->rekey_tx.rec_seq, rec_seq,
+ cipher_desc->rec_seq);
+
+ WRITE_ONCE(ctx->rekey_boundary_seq, tcp_sk(sk)->write_seq);
+
+ ctx->rekey_sw_ctx = &offload_ctx->rekey_sw;
+ ctx->rekey_cipher_ctx = &offload_ctx->rekey_tx;
+
+ set_bit(TLS_TX_REKEY_PENDING, &ctx->flags);
+
+ /* Switch to rekey validator; new sends won't use HW offload */
+ smp_store_release(&sk->sk_validate_xmit_skb,
+ tls_validate_xmit_skb_rekey);
+ }
+
+ unsafe_memcpy(&offload_ctx->rekey_crypto_send.info, new_crypto_info,
+ cipher_desc->crypto_info,
+ /* checked in do_tls_setsockopt_conf */);
+ memzero_explicit(new_crypto_info, cipher_desc->crypto_info);
+
+ return 0;
+}
+
+static int tls_device_complete_rekey(struct sock *sk, struct tls_context *ctx)
+{
+ struct tls_offload_context_tx *offload_ctx = tls_offload_ctx_tx(ctx);
+ struct tls_record_info *start_marker_record;
+ const struct tls_cipher_desc *cipher_desc;
+ struct net_device *netdev;
+ unsigned long flags;
+ __be64 rcd_sn;
+ char *key;
+ int rc;
+
+ cipher_desc = get_cipher_desc(offload_ctx->rekey_crypto_send.info.cipher_type);
+ DEBUG_NET_WARN_ON_ONCE(!cipher_desc || !cipher_desc->offloadable);
+
+ /* Flush all pending SW data before switching back to HW:
+ * 1. Close any open_rec left by MSG_MORE and encrypt it.
+ * 2. Wait for async crypto completions.
+ * 3. Push all ready records into TCP.
+ * If the send buffer is full, bail out and retry next sendmsg.
+ */
+ if (tls_is_pending_open_record(ctx))
+ tls_sw_push_pending_record(sk, 0);
+ tls_encrypt_async_wait(tls_sw_ctx_tx(ctx));
+ rc = tls_tx_records(sk, -1);
+ if (rc < 0 || tls_is_partially_sent_record(ctx) ||
+ tls_is_pending_open_record(ctx))
+ return rc < 0 ? rc : -EAGAIN;
+
+ start_marker_record = kmalloc_obj(*start_marker_record);
+ if (!start_marker_record)
+ return -ENOMEM;
+
+ down_read(&device_offload_lock);
+
+ netdev = rcu_dereference_protected(ctx->netdev,
+ lockdep_is_held(&device_offload_lock));
+ if (!netdev) {
+ rc = -ENODEV;
+ goto release_lock;
+ }
+
+ if (!test_bit(TLS_TX_DEV_CLOSED, &ctx->flags)) {
+ netdev->tlsdev_ops->tls_dev_del(netdev, ctx,
+ TLS_OFFLOAD_CTX_DIR_TX);
+ set_bit(TLS_TX_DEV_CLOSED, &ctx->flags);
+ }
+
+ memcpy(crypto_info_rec_seq(&offload_ctx->rekey_crypto_send.info, cipher_desc),
+ offload_ctx->rekey_tx.rec_seq, cipher_desc->rec_seq);
+
+ rc = netdev->tlsdev_ops->tls_dev_add(netdev, sk, TLS_OFFLOAD_CTX_DIR_TX,
+ &offload_ctx->rekey_crypto_send.info,
+ tcp_sk(sk)->write_seq);
+
+release_lock:
+ up_read(&device_offload_lock);
+
+ spin_lock_irqsave(&offload_ctx->lock, flags);
+ memcpy(&rcd_sn, offload_ctx->rekey_tx.rec_seq, sizeof(rcd_sn));
+ offload_ctx->unacked_record_sn = be64_to_cpu(rcd_sn) - 1;
+ spin_unlock_irqrestore(&offload_ctx->lock, flags);
+
+ memcpy(ctx->tx.iv, offload_ctx->rekey_tx.iv,
+ cipher_desc->salt + cipher_desc->iv);
+ memcpy(ctx->tx.rec_seq, offload_ctx->rekey_tx.rec_seq,
+ cipher_desc->rec_seq);
+ unsafe_memcpy(&ctx->crypto_send.info,
+ &offload_ctx->rekey_crypto_send.info,
+ cipher_desc->crypto_info,
+ /* checked during rekey setup */);
+
+ if (rc)
+ goto rekey_fail;
+
+ clear_bit(TLS_TX_DEV_CLOSED, &ctx->flags);
+
+ key = crypto_info_key(&offload_ctx->rekey_crypto_send.info, cipher_desc);
+ rc = crypto_aead_setkey(offload_ctx->aead_send, key, cipher_desc->key);
+ if (rc)
+ goto rekey_fail;
+
+ /* Start marker: the NIC passes through everything before
+ * write_seq unencrypted (already SW-encrypted during rekey),
+ * same as during initial offload setup.
+ */
+ spin_lock_irqsave(&offload_ctx->lock, flags);
+ start_marker_record->end_seq = tcp_sk(sk)->write_seq;
+ start_marker_record->len = 0;
+ start_marker_record->num_frags = 0;
+ list_add_tail_rcu(&start_marker_record->list,
+ &offload_ctx->records_list);
+ spin_unlock_irqrestore(&offload_ctx->lock, flags);
+
+ /* Prevent a partial record straddling the SW/HW boundary. */
+ tcp_write_collapse_fence(sk);
+
+ /* PENDING before READY: prevents clean_acked from
+ * re-setting REKEY_READY after we clear it.
+ */
+ clear_bit(TLS_TX_REKEY_PENDING, &ctx->flags);
+ smp_mb__after_atomic();
+ clear_bit(TLS_TX_REKEY_READY, &ctx->flags);
+ clear_bit(TLS_TX_REKEY_FAILED, &ctx->flags);
+
+ /* Switch back to HW offload validator */
+ smp_store_release(&sk->sk_validate_xmit_skb, tls_validate_xmit_skb);
+
+ crypto_free_aead(tls_sw_ctx_tx(ctx)->aead_send);
+ ctx->rekey_sw_ctx = NULL;
+ ctx->rekey_cipher_ctx = NULL;
+
+ return 0;
+
+rekey_fail:
+ kfree(start_marker_record);
+ set_bit(TLS_TX_REKEY_FAILED, &ctx->flags);
+ clear_bit(TLS_TX_REKEY_READY, &ctx->flags);
+ clear_bit(TLS_TX_REKEY_PENDING, &ctx->flags);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYHWFAIL);
+
+ return 0;
+}
+
static void tls_tcp_clean_acked(struct sock *sk, u32 acked_seq)
{
struct tls_context *tls_ctx = tls_get_ctx(sk);
@@ -187,6 +437,19 @@ static void tls_tcp_clean_acked(struct sock *sk, u32 acked_seq)
}
ctx->unacked_record_sn += deleted_records;
+
+ /* Once all old-key HW records are ACKed, set REKEY_READY to
+ * let sendmsg know it can finish the rekey and switch back
+ * to HW offload.
+ */
+ if (test_bit(TLS_TX_REKEY_PENDING, &tls_ctx->flags) &&
+ !test_bit(TLS_TX_REKEY_FAILED, &tls_ctx->flags)) {
+ u32 boundary_seq = READ_ONCE(tls_ctx->rekey_boundary_seq);
+
+ if (!before(acked_seq, boundary_seq))
+ set_bit(TLS_TX_REKEY_READY, &tls_ctx->flags);
+ }
+
spin_unlock_irqrestore(&ctx->lock, flags);
}
@@ -218,6 +481,9 @@ void tls_device_free_resources_tx(struct sock *sk)
struct tls_context *tls_ctx = tls_get_ctx(sk);
tls_free_partial_record(sk, tls_ctx);
+
+ if (unlikely(tls_ctx->rekey_sw_ctx))
+ tls_sw_release_resources_tx(sk);
}
void tls_offload_tx_resync_request(struct sock *sk, u32 got_seq, u32 exp_seq)
@@ -589,6 +855,19 @@ int tls_device_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
goto out;
}
+ /* Old-key records all ACKed; switch back to HW. */
+ if (test_bit(TLS_TX_REKEY_READY, &tls_ctx->flags))
+ tls_device_complete_rekey(sk, tls_ctx);
+
+ /* Use SW path if rekey is in progress (PENDING) or if HW rekey
+ * failed (FAILED).
+ */
+ if (test_bit(TLS_TX_REKEY_PENDING, &tls_ctx->flags) ||
+ test_bit(TLS_TX_REKEY_FAILED, &tls_ctx->flags)) {
+ rc = tls_sw_sendmsg_locked(sk, msg, size);
+ goto out;
+ }
+
rc = tls_push_data(sk, &msg->msg_iter, size, msg->msg_flags,
record_type);
@@ -1068,57 +1347,31 @@ static struct tls_offload_context_tx *alloc_offload_ctx_tx(struct tls_context *c
return offload_ctx;
}
-int tls_set_device_offload(struct sock *sk)
+static int tls_set_device_offload_initial(struct sock *sk,
+ struct tls_context *ctx,
+ struct net_device *netdev,
+ struct tls_crypto_info *crypto_info,
+ const struct tls_cipher_desc *cipher_desc)
{
+ struct tls_prot_info *prot = &ctx->prot_info;
struct tls_record_info *start_marker_record;
struct tls_offload_context_tx *offload_ctx;
- const struct tls_cipher_desc *cipher_desc;
- struct tls_crypto_info *crypto_info;
- struct tls_prot_info *prot;
- struct net_device *netdev;
- struct tls_context *ctx;
char *iv, *rec_seq;
int rc;
- ctx = tls_get_ctx(sk);
- prot = &ctx->prot_info;
-
- if (ctx->priv_ctx_tx)
- return -EEXIST;
-
- netdev = get_netdev_for_sock(sk);
- if (!netdev) {
- pr_err_ratelimited("%s: netdev not found\n", __func__);
- return -EINVAL;
- }
-
- if (!(netdev->features & NETIF_F_HW_TLS_TX)) {
- rc = -EOPNOTSUPP;
- goto release_netdev;
- }
-
- crypto_info = &ctx->crypto_send.info;
- cipher_desc = get_cipher_desc(crypto_info->cipher_type);
- if (!cipher_desc || !cipher_desc->offloadable) {
- rc = -EINVAL;
- goto release_netdev;
- }
+ iv = crypto_info_iv(crypto_info, cipher_desc);
+ rec_seq = crypto_info_rec_seq(crypto_info, cipher_desc);
rc = init_prot_info(prot, crypto_info, cipher_desc);
if (rc)
- goto release_netdev;
-
- iv = crypto_info_iv(crypto_info, cipher_desc);
- rec_seq = crypto_info_rec_seq(crypto_info, cipher_desc);
+ return rc;
memcpy(ctx->tx.iv + cipher_desc->salt, iv, cipher_desc->iv);
memcpy(ctx->tx.rec_seq, rec_seq, cipher_desc->rec_seq);
start_marker_record = kmalloc_obj(*start_marker_record);
- if (!start_marker_record) {
- rc = -ENOMEM;
- goto release_netdev;
- }
+ if (!start_marker_record)
+ return -ENOMEM;
offload_ctx = alloc_offload_ctx_tx(ctx);
if (!offload_ctx) {
@@ -1159,8 +1412,10 @@ int tls_set_device_offload(struct sock *sk)
}
ctx->priv_ctx_tx = offload_ctx;
- rc = netdev->tlsdev_ops->tls_dev_add(netdev, sk, TLS_OFFLOAD_CTX_DIR_TX,
- &ctx->crypto_send.info,
+
+ rc = netdev->tlsdev_ops->tls_dev_add(netdev, sk,
+ TLS_OFFLOAD_CTX_DIR_TX,
+ crypto_info,
tcp_sk(sk)->write_seq);
trace_tls_device_offload_set(sk, TLS_OFFLOAD_CTX_DIR_TX,
tcp_sk(sk)->write_seq, rec_seq, rc);
@@ -1175,7 +1430,6 @@ int tls_set_device_offload(struct sock *sk)
* by the netdev's xmit function.
*/
smp_store_release(&sk->sk_validate_xmit_skb, tls_validate_xmit_skb);
- dev_put(netdev);
return 0;
@@ -1188,18 +1442,111 @@ int tls_set_device_offload(struct sock *sk)
ctx->priv_ctx_tx = NULL;
free_marker_record:
kfree(start_marker_record);
+ return rc;
+}
+
+static int tls_set_device_offload_rekey(struct sock *sk,
+ struct tls_context *ctx,
+ struct net_device *netdev,
+ struct tls_crypto_info *new_crypto_info)
+{
+ struct tls_offload_context_tx *offload_ctx = tls_offload_ctx_tx(ctx);
+ bool rekey_pending = test_bit(TLS_TX_REKEY_PENDING, &ctx->flags);
+ bool has_unacked = false;
+ int rc;
+
+ if (!rekey_pending)
+ has_unacked = tls_has_unacked_records(offload_ctx);
+
+ down_read(&device_offload_lock);
+
+ rc = tls_device_start_rekey(sk, ctx, offload_ctx, new_crypto_info);
+ if (rc) {
+ up_read(&device_offload_lock);
+ return rc;
+ }
+
+ up_read(&device_offload_lock);
+
+ if (!rekey_pending && !has_unacked)
+ rc = tls_device_complete_rekey(sk, ctx);
+
+ return rc;
+}
+
+int tls_set_device_offload(struct sock *sk,
+ struct tls_crypto_info *new_crypto_info)
+{
+ struct tls_crypto_info *crypto_info, *src_crypto_info;
+ const struct tls_cipher_desc *cipher_desc;
+ struct net_device *netdev;
+ struct tls_context *ctx;
+ int rc;
+
+ ctx = tls_get_ctx(sk);
+
+ /* Rekey is only supported for connections that are already
+ * using HW offload. For SW offload connections, the caller
+ * should fall back to tls_set_sw_offload() for rekey.
+ */
+ if (new_crypto_info && ctx->tx_conf != TLS_HW)
+ return -EINVAL;
+
+ netdev = get_netdev_for_sock(sk);
+ if (!netdev) {
+ pr_err_ratelimited("%s: netdev not found\n", __func__);
+ return -EINVAL;
+ }
+
+ if (!(netdev->features & NETIF_F_HW_TLS_TX)) {
+ rc = -EOPNOTSUPP;
+ goto release_netdev;
+ }
+
+ crypto_info = &ctx->crypto_send.info;
+ src_crypto_info = new_crypto_info ?: crypto_info;
+ cipher_desc = get_cipher_desc(src_crypto_info->cipher_type);
+ if (!cipher_desc || !cipher_desc->offloadable) {
+ rc = -EINVAL;
+ goto release_netdev;
+ }
+
+ if (new_crypto_info)
+ rc = tls_set_device_offload_rekey(sk, ctx, netdev,
+ src_crypto_info);
+ else
+ rc = tls_set_device_offload_initial(sk, ctx, netdev,
+ src_crypto_info,
+ cipher_desc);
+
release_netdev:
dev_put(netdev);
return rc;
}
-int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx)
+int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx,
+ struct tls_crypto_info *new_crypto_info)
{
- struct tls12_crypto_info_aes_gcm_128 *info;
+ struct tls_crypto_info *crypto_info, *src_crypto_info;
+ const struct tls_cipher_desc *cipher_desc;
struct tls_offload_context_rx *context;
struct net_device *netdev;
+ char *rec_seq;
int rc = 0;
+ /* Rekey is only supported for connections that are already
+ * using HW offload. For SW offload connections, the caller
+ * should fall back to tls_set_sw_offload() for rekey.
+ */
+ if (new_crypto_info && ctx->rx_conf != TLS_HW)
+ return -EINVAL;
+
+ crypto_info = &ctx->crypto_recv.info;
+ src_crypto_info = new_crypto_info ?: crypto_info;
+ cipher_desc = get_cipher_desc(src_crypto_info->cipher_type);
+ if (!cipher_desc || !cipher_desc->offloadable)
+ return -EINVAL;
+
netdev = get_netdev_for_sock(sk);
if (!netdev) {
pr_err_ratelimited("%s: netdev not found\n", __func__);
@@ -1225,29 +1572,50 @@ int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx)
goto release_lock;
}
- context = kzalloc_obj(*context);
- if (!context) {
- rc = -ENOMEM;
- goto release_lock;
+ if (!new_crypto_info) {
+ context = kzalloc_obj(*context);
+ if (!context) {
+ rc = -ENOMEM;
+ goto release_lock;
+ }
+ context->resync_nh_reset = 1;
+ ctx->priv_ctx_rx = context;
}
- context->resync_nh_reset = 1;
- ctx->priv_ctx_rx = context;
- rc = tls_sw_ctx_init(sk, 0, NULL);
+ rc = tls_sw_ctx_init(sk, 0, new_crypto_info);
if (rc)
goto release_ctx;
+ if (new_crypto_info && !test_bit(TLS_RX_DEV_CLOSED, &ctx->flags))
+ netdev->tlsdev_ops->tls_dev_del(netdev, ctx,
+ TLS_OFFLOAD_CTX_DIR_RX);
+
rc = netdev->tlsdev_ops->tls_dev_add(netdev, sk, TLS_OFFLOAD_CTX_DIR_RX,
- &ctx->crypto_recv.info,
+ src_crypto_info,
tcp_sk(sk)->copied_seq);
- info = (void *)&ctx->crypto_recv.info;
+
+ rec_seq = crypto_info_rec_seq(src_crypto_info, cipher_desc);
trace_tls_device_offload_set(sk, TLS_OFFLOAD_CTX_DIR_RX,
- tcp_sk(sk)->copied_seq, info->rec_seq, rc);
- if (rc)
- goto free_sw_resources;
+ tcp_sk(sk)->copied_seq, rec_seq, rc);
+ if (rc) {
+ if (new_crypto_info) {
+ set_bit(TLS_RX_DEV_DEGRADED, &ctx->flags);
+ set_bit(TLS_RX_DEV_CLOSED, &ctx->flags);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYHWFAIL);
+ } else {
+ goto free_sw_resources;
+ }
+ } else {
+ if (new_crypto_info) {
+ clear_bit(TLS_RX_DEV_DEGRADED, &ctx->flags);
+ clear_bit(TLS_RX_DEV_CLOSED, &ctx->flags);
+ }
+
+ tls_device_attach(ctx, sk, netdev);
+ }
+
+ tls_sw_ctx_finalize(sk, 0, new_crypto_info);
- tls_device_attach(ctx, sk, netdev);
- tls_sw_ctx_finalize(sk, 0, NULL);
up_read(&device_offload_lock);
dev_put(netdev);
@@ -1256,10 +1624,13 @@ int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx)
free_sw_resources:
up_read(&device_offload_lock);
- tls_sw_free_resources_rx(sk);
+ tls_sw_release_resources_rx(sk);
down_read(&device_offload_lock);
release_ctx:
- ctx->priv_ctx_rx = NULL;
+ if (!new_crypto_info) {
+ kfree(ctx->priv_ctx_rx);
+ ctx->priv_ctx_rx = NULL;
+ }
release_lock:
up_read(&device_offload_lock);
release_netdev:
@@ -1278,8 +1649,9 @@ void tls_device_offload_cleanup_rx(struct sock *sk)
if (!netdev)
goto out;
- netdev->tlsdev_ops->tls_dev_del(netdev, tls_ctx,
- TLS_OFFLOAD_CTX_DIR_RX);
+ if (!test_bit(TLS_RX_DEV_CLOSED, &tls_ctx->flags))
+ netdev->tlsdev_ops->tls_dev_del(netdev, tls_ctx,
+ TLS_OFFLOAD_CTX_DIR_RX);
if (tls_ctx->tx_conf != TLS_HW) {
dev_put(netdev);
@@ -1319,7 +1691,10 @@ static int tls_device_down(struct net_device *netdev)
/* Stop offloaded TX and switch to the fallback.
* tls_is_skb_tx_device_offloaded will return false.
*/
- WRITE_ONCE(ctx->sk->sk_validate_xmit_skb, tls_validate_xmit_skb_sw);
+ if (!test_bit(TLS_TX_REKEY_PENDING, &ctx->flags) &&
+ !test_bit(TLS_TX_REKEY_FAILED, &ctx->flags))
+ WRITE_ONCE(ctx->sk->sk_validate_xmit_skb,
+ tls_validate_xmit_skb_sw);
/* Stop the RX and TX resync.
* tls_dev_resync must not be called after tls_dev_del.
@@ -1336,13 +1711,18 @@ static int tls_device_down(struct net_device *netdev)
synchronize_net();
/* Release the offload context on the driver side. */
- if (ctx->tx_conf == TLS_HW)
+ if (ctx->tx_conf == TLS_HW &&
+ !test_bit(TLS_TX_DEV_CLOSED, &ctx->flags)) {
netdev->tlsdev_ops->tls_dev_del(netdev, ctx,
TLS_OFFLOAD_CTX_DIR_TX);
+ set_bit(TLS_TX_DEV_CLOSED, &ctx->flags);
+ }
if (ctx->rx_conf == TLS_HW &&
- !test_bit(TLS_RX_DEV_CLOSED, &ctx->flags))
+ !test_bit(TLS_RX_DEV_CLOSED, &ctx->flags)) {
netdev->tlsdev_ops->tls_dev_del(netdev, ctx,
TLS_OFFLOAD_CTX_DIR_RX);
+ set_bit(TLS_RX_DEV_CLOSED, &ctx->flags);
+ }
dev_put(netdev);
diff --git a/net/tls/tls_device_fallback.c b/net/tls/tls_device_fallback.c
index 1110f7ac6bcb..5be425a32c82 100644
--- a/net/tls/tls_device_fallback.c
+++ b/net/tls/tls_device_fallback.c
@@ -435,6 +435,30 @@ struct sk_buff *tls_validate_xmit_skb_sw(struct sock *sk,
return tls_sw_fallback(sk, skb);
}
+struct sk_buff *tls_validate_xmit_skb_rekey(struct sock *sk,
+ struct net_device *dev,
+ struct sk_buff *skb)
+{
+ struct tls_context *tls_ctx = tls_get_ctx(sk);
+ u32 tcp_seq = ntohl(tcp_hdr(skb)->seq);
+ u32 boundary_seq;
+
+ if (test_bit(TLS_TX_REKEY_FAILED, &tls_ctx->flags))
+ return skb;
+
+ /* If this packet is at or after the rekey boundary, it's already
+ * SW-encrypted with the new key, pass through unchanged
+ */
+ boundary_seq = READ_ONCE(tls_ctx->rekey_boundary_seq);
+ if (!before(tcp_seq, boundary_seq))
+ return skb;
+
+ /* Packet before boundary means retransmit of old data,
+ * use SW fallback with the old key
+ */
+ return tls_sw_fallback(sk, skb);
+}
+
struct sk_buff *tls_encrypt_skb(struct sk_buff *skb)
{
return tls_sw_fallback(skb->sk, skb);
diff --git a/net/tls/tls_main.c b/net/tls/tls_main.c
index fd04857fa0ab..ab701f166b57 100644
--- a/net/tls/tls_main.c
+++ b/net/tls/tls_main.c
@@ -371,6 +371,8 @@ static void tls_sk_proto_close(struct sock *sk, long timeout)
if (ctx->tx_conf == TLS_SW)
tls_sw_cancel_work_tx(ctx);
+ else if (ctx->tx_conf == TLS_HW && ctx->rekey_sw_ctx)
+ tls_sw_cancel_work_tx(ctx);
lock_sock(sk);
free_ctx = ctx->tx_conf != TLS_HW && ctx->rx_conf != TLS_HW;
@@ -711,64 +713,68 @@ static int do_tls_setsockopt_conf(struct sock *sk, sockptr_t optval,
}
if (tx) {
- if (update && ctx->tx_conf == TLS_HW) {
- rc = -EOPNOTSUPP;
- goto err_crypto_info;
- }
-
- if (!update) {
- rc = tls_set_device_offload(sk);
- conf = TLS_HW;
- if (!rc) {
+ rc = tls_set_device_offload(sk, update ? crypto_info : NULL);
+ conf = TLS_HW;
+ if (!rc) {
+ if (update) {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYOK);
+ } else {
TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXDEVICE);
TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXDEVICE);
- goto out;
}
- }
-
- rc = tls_set_sw_offload(sk, 1, update ? crypto_info : NULL);
- if (rc)
+ } else if (update && ctx->tx_conf == TLS_HW) {
+ /* HW rekey failed - return the actual error.
+ * Cannot fall back to SW for an existing HW connection.
+ */
goto err_crypto_info;
-
- if (update) {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYOK);
} else {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXSW);
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW);
+ rc = tls_set_sw_offload(sk, 1,
+ update ? crypto_info : NULL);
+ if (rc)
+ goto err_crypto_info;
+
+ if (update) {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYOK);
+ } else {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXSW);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW);
+ }
+ conf = TLS_SW;
}
- conf = TLS_SW;
} else {
- if (update && ctx->rx_conf == TLS_HW) {
- rc = -EOPNOTSUPP;
- goto err_crypto_info;
- }
-
- if (!update) {
- rc = tls_set_device_offload_rx(sk, ctx);
- conf = TLS_HW;
- if (!rc) {
+ rc = tls_set_device_offload_rx(sk, ctx,
+ update ? crypto_info : NULL);
+ conf = TLS_HW;
+ if (!rc) {
+ if (update) {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYOK);
+ } else {
TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXDEVICE);
TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXDEVICE);
- tls_sw_strparser_arm(sk, ctx);
- goto out;
}
- }
-
- rc = tls_set_sw_offload(sk, 0, update ? crypto_info : NULL);
- if (rc)
+ } else if (update && ctx->rx_conf == TLS_HW) {
+ /* HW rekey failed - return the actual error.
+ * Cannot fall back to SW for an existing HW connection.
+ */
goto err_crypto_info;
-
- if (update) {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYOK);
} else {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXSW);
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXSW);
- tls_sw_strparser_arm(sk, ctx);
+ rc = tls_set_sw_offload(sk, 0,
+ update ? crypto_info : NULL);
+ if (rc)
+ goto err_crypto_info;
+
+ if (update) {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYOK);
+ } else {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXSW);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXSW);
+ }
+ conf = TLS_SW;
}
- conf = TLS_SW;
+ if (!update)
+ tls_sw_strparser_arm(sk, ctx);
}
-out:
if (tx)
ctx->tx_conf = conf;
else
diff --git a/net/tls/tls_proc.c b/net/tls/tls_proc.c
index 4012c4372d4c..5599af306aab 100644
--- a/net/tls/tls_proc.c
+++ b/net/tls/tls_proc.c
@@ -27,6 +27,8 @@ static const struct snmp_mib tls_mib_list[] = {
SNMP_MIB_ITEM("TlsTxRekeyOk", LINUX_MIB_TLSTXREKEYOK),
SNMP_MIB_ITEM("TlsTxRekeyError", LINUX_MIB_TLSTXREKEYERROR),
SNMP_MIB_ITEM("TlsRxRekeyReceived", LINUX_MIB_TLSRXREKEYRECEIVED),
+ SNMP_MIB_ITEM("TlsTxRekeyHwFail", LINUX_MIB_TLSTXREKEYHWFAIL),
+ SNMP_MIB_ITEM("TlsRxRekeyHwFail", LINUX_MIB_TLSRXREKEYHWFAIL),
};
static int tls_statistics_seq_show(struct seq_file *seq, void *v)
diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c
index 5df27493c2a7..6f46d45f836d 100644
--- a/net/tls/tls_sw.c
+++ b/net/tls/tls_sw.c
@@ -522,7 +522,7 @@ static void tls_encrypt_done(void *data, int err)
complete(&ctx->async_wait.completion);
}
-static int tls_encrypt_async_wait(struct tls_sw_context_tx *ctx)
+int tls_encrypt_async_wait(struct tls_sw_context_tx *ctx)
{
if (!atomic_dec_and_test(&ctx->encrypt_pending))
crypto_wait_req(-EINPROGRESS, &ctx->async_wait);
@@ -555,11 +555,11 @@ static int tls_do_encryption(struct sock *sk,
break;
}
- memcpy(&rec->iv_data[iv_offset], tls_ctx->tx.iv,
+ memcpy(&rec->iv_data[iv_offset], tls_tx_cipher_ctx(tls_ctx)->iv,
prot->iv_size + prot->salt_size);
tls_xor_iv_with_seq(prot, rec->iv_data + iv_offset,
- tls_ctx->tx.rec_seq);
+ tls_tx_cipher_ctx(tls_ctx)->rec_seq);
sge->offset += prot->prepend_size;
sge->length -= prot->prepend_size;
@@ -600,7 +600,7 @@ static int tls_do_encryption(struct sock *sk,
/* Unhook the record from context if encryption is not failure */
ctx->open_rec = NULL;
- tls_advance_record_sn(sk, prot, &tls_ctx->tx);
+ tls_advance_record_sn(sk, prot, tls_tx_cipher_ctx(tls_ctx));
return rc;
}
@@ -807,7 +807,7 @@ static int tls_push_record(struct sock *sk, int flags,
sg_chain(rec->sg_aead_out, 2, &msg_en->sg.data[i]);
tls_make_aad(rec->aad_space, msg_pl->sg.size + prot->tail_size,
- tls_ctx->tx.rec_seq, record_type, prot);
+ tls_tx_cipher_ctx(tls_ctx)->rec_seq, record_type, prot);
tls_fill_prepend(tls_ctx,
page_address(sg_page(&msg_en->sg.data[i])) +
@@ -972,7 +972,7 @@ static int bpf_exec_tx_verdict(struct sk_msg *msg, struct sock *sk,
return err;
}
-static int tls_sw_push_pending_record(struct sock *sk, int flags)
+int tls_sw_push_pending_record(struct sock *sk, int flags)
{
struct tls_context *tls_ctx = tls_get_ctx(sk);
struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
@@ -1023,8 +1023,7 @@ static int tls_sw_sendmsg_splice(struct sock *sk, struct msghdr *msg,
return 0;
}
-static int tls_sw_sendmsg_locked(struct sock *sk, struct msghdr *msg,
- size_t size)
+int tls_sw_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size)
{
long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
struct tls_context *tls_ctx = tls_get_ctx(sk);
@@ -2620,7 +2619,7 @@ void tls_sw_free_resources_rx(struct sock *sk)
}
/* The work handler to transmitt the encrypted records in tx_list */
-static void tx_work_handler(struct work_struct *work)
+void tls_tx_work_handler(struct work_struct *work)
{
struct delayed_work *delayed_work = to_delayed_work(work);
struct tx_work *tx_work = container_of(delayed_work,
@@ -2653,6 +2652,15 @@ static void tx_work_handler(struct work_struct *work)
}
}
+void tls_sw_ctx_tx_init(struct sock *sk, struct tls_sw_context_tx *sw_ctx)
+{
+ crypto_init_wait(&sw_ctx->async_wait);
+ atomic_set(&sw_ctx->encrypt_pending, 1);
+ INIT_LIST_HEAD(&sw_ctx->tx_list);
+ INIT_DELAYED_WORK(&sw_ctx->tx_work.work, tls_tx_work_handler);
+ sw_ctx->tx_work.sk = sk;
+}
+
static bool tls_is_tx_ready(struct tls_sw_context_tx *ctx)
{
struct tls_rec *rec;
@@ -2704,11 +2712,7 @@ static struct tls_sw_context_tx *init_ctx_tx(struct tls_context *ctx, struct soc
sw_ctx_tx = ctx->priv_ctx_tx;
}
- crypto_init_wait(&sw_ctx_tx->async_wait);
- atomic_set(&sw_ctx_tx->encrypt_pending, 1);
- INIT_LIST_HEAD(&sw_ctx_tx->tx_list);
- INIT_DELAYED_WORK(&sw_ctx_tx->tx_work.work, tx_work_handler);
- sw_ctx_tx->tx_work.sk = sk;
+ tls_sw_ctx_tx_init(sk, sw_ctx_tx);
return sw_ctx_tx;
}
--
2.25.1
^ permalink raw reply related
* [PATCH v12 4/6] tls: split tls_set_sw_offload into init and finalize stages
From: Rishikesh Jethwani @ 2026-04-02 23:55 UTC (permalink / raw)
To: netdev
Cc: saeedm, tariqt, mbloch, borisp, john.fastabend, kuba, sd, davem,
pabeni, edumazet, leon, Rishikesh Jethwani
In-Reply-To: <20260402235511.664801-1-rjethwani@purestorage.com>
Separate cipher context initialization from key material finalization
to support staged setup for hardware offload fallback paths.
Signed-off-by: Rishikesh Jethwani <rjethwani@purestorage.com>
---
net/tls/tls.h | 4 +++
net/tls/tls_device.c | 3 +-
net/tls/tls_sw.c | 77 +++++++++++++++++++++++++++++++-------------
3 files changed, 61 insertions(+), 23 deletions(-)
diff --git a/net/tls/tls.h b/net/tls/tls.h
index e8f81a006520..a65cf9bab190 100644
--- a/net/tls/tls.h
+++ b/net/tls/tls.h
@@ -147,6 +147,10 @@ void tls_strp_abort_strp(struct tls_strparser *strp, int err);
int init_prot_info(struct tls_prot_info *prot,
const struct tls_crypto_info *crypto_info,
const struct tls_cipher_desc *cipher_desc);
+int tls_sw_ctx_init(struct sock *sk, int tx,
+ struct tls_crypto_info *new_crypto_info);
+void tls_sw_ctx_finalize(struct sock *sk, int tx,
+ struct tls_crypto_info *new_crypto_info);
int tls_set_sw_offload(struct sock *sk, int tx,
struct tls_crypto_info *new_crypto_info);
void tls_update_rx_zc_capable(struct tls_context *tls_ctx);
diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c
index 1321bf9b59b0..cd26873e9063 100644
--- a/net/tls/tls_device.c
+++ b/net/tls/tls_device.c
@@ -1233,7 +1233,7 @@ int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx)
context->resync_nh_reset = 1;
ctx->priv_ctx_rx = context;
- rc = tls_set_sw_offload(sk, 0, NULL);
+ rc = tls_sw_ctx_init(sk, 0, NULL);
if (rc)
goto release_ctx;
@@ -1247,6 +1247,7 @@ int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx)
goto free_sw_resources;
tls_device_attach(ctx, sk, netdev);
+ tls_sw_ctx_finalize(sk, 0, NULL);
up_read(&device_offload_lock);
dev_put(netdev);
diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c
index 20f8fc84c5f5..5df27493c2a7 100644
--- a/net/tls/tls_sw.c
+++ b/net/tls/tls_sw.c
@@ -2774,20 +2774,19 @@ static void tls_finish_key_update(struct sock *sk, struct tls_context *tls_ctx)
ctx->saved_data_ready(sk);
}
-int tls_set_sw_offload(struct sock *sk, int tx,
- struct tls_crypto_info *new_crypto_info)
+int tls_sw_ctx_init(struct sock *sk, int tx,
+ struct tls_crypto_info *new_crypto_info)
{
struct tls_crypto_info *crypto_info, *src_crypto_info;
struct tls_sw_context_tx *sw_ctx_tx = NULL;
struct tls_sw_context_rx *sw_ctx_rx = NULL;
const struct tls_cipher_desc *cipher_desc;
- char *iv, *rec_seq, *key, *salt;
- struct cipher_context *cctx;
struct tls_prot_info *prot;
struct crypto_aead **aead;
struct tls_context *ctx;
struct crypto_tfm *tfm;
int rc = 0;
+ char *key;
ctx = tls_get_ctx(sk);
prot = &ctx->prot_info;
@@ -2808,12 +2807,10 @@ int tls_set_sw_offload(struct sock *sk, int tx,
if (tx) {
sw_ctx_tx = ctx->priv_ctx_tx;
crypto_info = &ctx->crypto_send.info;
- cctx = &ctx->tx;
aead = &sw_ctx_tx->aead_send;
} else {
sw_ctx_rx = ctx->priv_ctx_rx;
crypto_info = &ctx->crypto_recv.info;
- cctx = &ctx->rx;
aead = &sw_ctx_rx->aead_recv;
}
@@ -2829,10 +2826,7 @@ int tls_set_sw_offload(struct sock *sk, int tx,
if (rc)
goto free_priv;
- iv = crypto_info_iv(src_crypto_info, cipher_desc);
key = crypto_info_key(src_crypto_info, cipher_desc);
- salt = crypto_info_salt(src_crypto_info, cipher_desc);
- rec_seq = crypto_info_rec_seq(src_crypto_info, cipher_desc);
if (!*aead) {
*aead = crypto_alloc_aead(cipher_desc->cipher_name, 0, 0);
@@ -2876,19 +2870,6 @@ int tls_set_sw_offload(struct sock *sk, int tx,
goto free_aead;
}
- memcpy(cctx->iv, salt, cipher_desc->salt);
- memcpy(cctx->iv + cipher_desc->salt, iv, cipher_desc->iv);
- memcpy(cctx->rec_seq, rec_seq, cipher_desc->rec_seq);
-
- if (new_crypto_info) {
- unsafe_memcpy(crypto_info, new_crypto_info,
- cipher_desc->crypto_info,
- /* size was checked in do_tls_setsockopt_conf */);
- memzero_explicit(new_crypto_info, cipher_desc->crypto_info);
- if (!tx)
- tls_finish_key_update(sk, ctx);
- }
-
goto out;
free_aead:
@@ -2907,3 +2888,55 @@ int tls_set_sw_offload(struct sock *sk, int tx,
out:
return rc;
}
+
+void tls_sw_ctx_finalize(struct sock *sk, int tx,
+ struct tls_crypto_info *new_crypto_info)
+{
+ struct tls_crypto_info *crypto_info, *src_crypto_info;
+ const struct tls_cipher_desc *cipher_desc;
+ struct tls_context *ctx = tls_get_ctx(sk);
+ struct cipher_context *cctx;
+ char *iv, *salt, *rec_seq;
+
+ if (tx) {
+ crypto_info = &ctx->crypto_send.info;
+ cctx = &ctx->tx;
+ } else {
+ crypto_info = &ctx->crypto_recv.info;
+ cctx = &ctx->rx;
+ }
+
+ src_crypto_info = new_crypto_info ?: crypto_info;
+ cipher_desc = get_cipher_desc(src_crypto_info->cipher_type);
+
+ iv = crypto_info_iv(src_crypto_info, cipher_desc);
+ salt = crypto_info_salt(src_crypto_info, cipher_desc);
+ rec_seq = crypto_info_rec_seq(src_crypto_info, cipher_desc);
+
+ memcpy(cctx->iv, salt, cipher_desc->salt);
+ memcpy(cctx->iv + cipher_desc->salt, iv, cipher_desc->iv);
+ memcpy(cctx->rec_seq, rec_seq, cipher_desc->rec_seq);
+
+ if (new_crypto_info) {
+ unsafe_memcpy(crypto_info, new_crypto_info,
+ cipher_desc->crypto_info,
+ /* size was checked in do_tls_setsockopt_conf */);
+ memzero_explicit(new_crypto_info, cipher_desc->crypto_info);
+
+ if (!tx)
+ tls_finish_key_update(sk, ctx);
+ }
+}
+
+int tls_set_sw_offload(struct sock *sk, int tx,
+ struct tls_crypto_info *new_crypto_info)
+{
+ int rc;
+
+ rc = tls_sw_ctx_init(sk, tx, new_crypto_info);
+ if (rc)
+ return rc;
+
+ tls_sw_ctx_finalize(sk, tx, new_crypto_info);
+ return 0;
+}
--
2.25.1
^ permalink raw reply related
* [PATCH v12 3/6] tls: add TLS 1.3 hardware offload support
From: Rishikesh Jethwani @ 2026-04-02 23:55 UTC (permalink / raw)
To: netdev
Cc: saeedm, tariqt, mbloch, borisp, john.fastabend, kuba, sd, davem,
pabeni, edumazet, leon, Rishikesh Jethwani
In-Reply-To: <20260402235511.664801-1-rjethwani@purestorage.com>
Add TLS 1.3 support to the kernel TLS hardware offload infrastructure,
enabling hardware acceleration for TLS 1.3 connections on capable NICs.
Tested on Mellanox ConnectX-6 Dx (Crypto Enabled) with TLS 1.3 AES-GCM-128
and AES-GCM-256 cipher suites.
Signed-off-by: Rishikesh Jethwani <rjethwani@purestorage.com>
---
net/tls/tls_device.c | 65 ++++++++++++++++-----------
net/tls/tls_device_fallback.c | 58 +++++++++++++-----------
net/tls/tls_main.c | 85 ++++++++++++++++++++---------------
3 files changed, 121 insertions(+), 87 deletions(-)
diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c
index 99c8eff9783e..1321bf9b59b0 100644
--- a/net/tls/tls_device.c
+++ b/net/tls/tls_device.c
@@ -317,25 +317,34 @@ static void tls_device_record_close(struct sock *sk,
unsigned char record_type)
{
struct tls_prot_info *prot = &ctx->prot_info;
- struct page_frag dummy_tag_frag;
-
- /* append tag
- * device will fill in the tag, we just need to append a placeholder
- * use socket memory to improve coalescing (re-using a single buffer
- * increases frag count)
- * if we can't allocate memory now use the dummy page
+ int tail = prot->tag_size + prot->tail_size;
+
+ /* Append tail: tag for TLS 1.2, content_type + tag for TLS 1.3.
+ * Device fills in the tag, we just need to append a placeholder.
+ * Use socket memory to improve coalescing (re-using a single buffer
+ * increases frag count); if allocation fails use dummy_page
+ * (offset = record_type gives correct content_type byte via
+ * identity mapping)
*/
- if (unlikely(pfrag->size - pfrag->offset < prot->tag_size) &&
- !skb_page_frag_refill(prot->tag_size, pfrag, sk->sk_allocation)) {
- dummy_tag_frag.page = dummy_page;
- dummy_tag_frag.offset = 0;
- pfrag = &dummy_tag_frag;
+ if (unlikely(pfrag->size - pfrag->offset < tail) &&
+ !skb_page_frag_refill(tail, pfrag, sk->sk_allocation)) {
+ struct page_frag dummy_pfrag = {
+ .page = dummy_page,
+ .offset = record_type,
+ };
+ tls_append_frag(record, &dummy_pfrag, tail);
+ } else {
+ if (prot->tail_size) {
+ char *content_type_addr = page_address(pfrag->page) +
+ pfrag->offset;
+ *content_type_addr = record_type;
+ }
+ tls_append_frag(record, pfrag, tail);
}
- tls_append_frag(record, pfrag, prot->tag_size);
/* fill prepend */
tls_fill_prepend(ctx, skb_frag_address(&record->frags[0]),
- record->len - prot->overhead_size,
+ record->len - prot->overhead_size + prot->tail_size,
record_type);
}
@@ -883,6 +892,7 @@ static int
tls_device_reencrypt(struct sock *sk, struct tls_context *tls_ctx)
{
struct tls_sw_context_rx *sw_ctx = tls_sw_ctx_rx(tls_ctx);
+ struct tls_prot_info *prot = &tls_ctx->prot_info;
const struct tls_cipher_desc *cipher_desc;
int err, offset, copy, data_len, pos;
struct sk_buff *skb, *skb_iter;
@@ -894,7 +904,7 @@ tls_device_reencrypt(struct sock *sk, struct tls_context *tls_ctx)
DEBUG_NET_WARN_ON_ONCE(!cipher_desc || !cipher_desc->offloadable);
rxm = strp_msg(tls_strp_msg(sw_ctx));
- orig_buf = kmalloc(rxm->full_len + TLS_HEADER_SIZE + cipher_desc->iv,
+ orig_buf = kmalloc(rxm->full_len + prot->prepend_size,
sk->sk_allocation);
if (!orig_buf)
return -ENOMEM;
@@ -909,9 +919,8 @@ tls_device_reencrypt(struct sock *sk, struct tls_context *tls_ctx)
offset = rxm->offset;
sg_init_table(sg, 1);
- sg_set_buf(&sg[0], buf,
- rxm->full_len + TLS_HEADER_SIZE + cipher_desc->iv);
- err = skb_copy_bits(skb, offset, buf, TLS_HEADER_SIZE + cipher_desc->iv);
+ sg_set_buf(&sg[0], buf, rxm->full_len + prot->prepend_size);
+ err = skb_copy_bits(skb, offset, buf, prot->prepend_size);
if (err)
goto free_buf;
@@ -1089,11 +1098,6 @@ int tls_set_device_offload(struct sock *sk)
}
crypto_info = &ctx->crypto_send.info;
- if (crypto_info->version != TLS_1_2_VERSION) {
- rc = -EOPNOTSUPP;
- goto release_netdev;
- }
-
cipher_desc = get_cipher_desc(crypto_info->cipher_type);
if (!cipher_desc || !cipher_desc->offloadable) {
rc = -EINVAL;
@@ -1196,9 +1200,6 @@ int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx)
struct net_device *netdev;
int rc = 0;
- if (ctx->crypto_recv.info.version != TLS_1_2_VERSION)
- return -EOPNOTSUPP;
-
netdev = get_netdev_for_sock(sk);
if (!netdev) {
pr_err_ratelimited("%s: netdev not found\n", __func__);
@@ -1409,12 +1410,22 @@ static struct notifier_block tls_dev_notifier = {
int __init tls_device_init(void)
{
- int err;
+ unsigned char *page_addr;
+ int err, i;
dummy_page = alloc_page(GFP_KERNEL);
if (!dummy_page)
return -ENOMEM;
+ /* Pre-populate dummy_page with identity mapping for all byte values.
+ * This is used as fallback for TLS 1.3 content type when memory
+ * allocation fails. By populating all 256 values, we avoid needing
+ * to validate record_type at runtime.
+ */
+ page_addr = page_address(dummy_page);
+ for (i = 0; i < 256; i++)
+ page_addr[i] = (unsigned char)i;
+
destruct_wq = alloc_workqueue("ktls_device_destruct", WQ_PERCPU, 0);
if (!destruct_wq) {
err = -ENOMEM;
diff --git a/net/tls/tls_device_fallback.c b/net/tls/tls_device_fallback.c
index 3b7d0ab2bcf1..1110f7ac6bcb 100644
--- a/net/tls/tls_device_fallback.c
+++ b/net/tls/tls_device_fallback.c
@@ -37,14 +37,15 @@
#include "tls.h"
-static int tls_enc_record(struct aead_request *aead_req,
+static int tls_enc_record(struct tls_context *tls_ctx,
+ struct aead_request *aead_req,
struct crypto_aead *aead, char *aad,
char *iv, __be64 rcd_sn,
struct scatter_walk *in,
- struct scatter_walk *out, int *in_len,
- struct tls_prot_info *prot)
+ struct scatter_walk *out, int *in_len)
{
unsigned char buf[TLS_HEADER_SIZE + TLS_MAX_IV_SIZE];
+ struct tls_prot_info *prot = &tls_ctx->prot_info;
const struct tls_cipher_desc *cipher_desc;
struct scatterlist sg_in[3];
struct scatterlist sg_out[3];
@@ -55,7 +56,7 @@ static int tls_enc_record(struct aead_request *aead_req,
cipher_desc = get_cipher_desc(prot->cipher_type);
DEBUG_NET_WARN_ON_ONCE(!cipher_desc || !cipher_desc->offloadable);
- buf_size = TLS_HEADER_SIZE + cipher_desc->iv;
+ buf_size = prot->prepend_size;
len = min_t(int, *in_len, buf_size);
memcpy_from_scatterwalk(buf, in, len);
@@ -66,16 +67,27 @@ static int tls_enc_record(struct aead_request *aead_req,
return 0;
len = buf[4] | (buf[3] << 8);
- len -= cipher_desc->iv;
+ if (prot->version != TLS_1_3_VERSION)
+ len -= cipher_desc->iv;
tls_make_aad(aad, len - cipher_desc->tag, (char *)&rcd_sn, buf[0], prot);
- memcpy(iv + cipher_desc->salt, buf + TLS_HEADER_SIZE, cipher_desc->iv);
+ if (prot->version == TLS_1_3_VERSION) {
+ void *iv_src = crypto_info_iv(&tls_ctx->crypto_send.info,
+ cipher_desc);
+
+ memcpy(iv + cipher_desc->salt, iv_src, cipher_desc->iv);
+ } else {
+ memcpy(iv + cipher_desc->salt, buf + TLS_HEADER_SIZE,
+ cipher_desc->iv);
+ }
+
+ tls_xor_iv_with_seq(prot, iv, (char *)&rcd_sn);
sg_init_table(sg_in, ARRAY_SIZE(sg_in));
sg_init_table(sg_out, ARRAY_SIZE(sg_out));
- sg_set_buf(sg_in, aad, TLS_AAD_SPACE_SIZE);
- sg_set_buf(sg_out, aad, TLS_AAD_SPACE_SIZE);
+ sg_set_buf(sg_in, aad, prot->aad_size);
+ sg_set_buf(sg_out, aad, prot->aad_size);
scatterwalk_get_sglist(in, sg_in + 1);
scatterwalk_get_sglist(out, sg_out + 1);
@@ -108,13 +120,6 @@ static int tls_enc_record(struct aead_request *aead_req,
return rc;
}
-static void tls_init_aead_request(struct aead_request *aead_req,
- struct crypto_aead *aead)
-{
- aead_request_set_tfm(aead_req, aead);
- aead_request_set_ad(aead_req, TLS_AAD_SPACE_SIZE);
-}
-
static struct aead_request *tls_alloc_aead_request(struct crypto_aead *aead,
gfp_t flags)
{
@@ -124,14 +129,15 @@ static struct aead_request *tls_alloc_aead_request(struct crypto_aead *aead,
aead_req = kzalloc(req_size, flags);
if (aead_req)
- tls_init_aead_request(aead_req, aead);
+ aead_request_set_tfm(aead_req, aead);
return aead_req;
}
-static int tls_enc_records(struct aead_request *aead_req,
+static int tls_enc_records(struct tls_context *tls_ctx,
+ struct aead_request *aead_req,
struct crypto_aead *aead, struct scatterlist *sg_in,
struct scatterlist *sg_out, char *aad, char *iv,
- u64 rcd_sn, int len, struct tls_prot_info *prot)
+ u64 rcd_sn, int len)
{
struct scatter_walk out, in;
int rc;
@@ -140,8 +146,8 @@ static int tls_enc_records(struct aead_request *aead_req,
scatterwalk_start(&out, sg_out);
do {
- rc = tls_enc_record(aead_req, aead, aad, iv,
- cpu_to_be64(rcd_sn), &in, &out, &len, prot);
+ rc = tls_enc_record(tls_ctx, aead_req, aead, aad, iv,
+ cpu_to_be64(rcd_sn), &in, &out, &len);
rcd_sn++;
} while (rc == 0 && len);
@@ -314,7 +320,10 @@ static struct sk_buff *tls_enc_skb(struct tls_context *tls_ctx,
cipher_desc = get_cipher_desc(tls_ctx->crypto_send.info.cipher_type);
DEBUG_NET_WARN_ON_ONCE(!cipher_desc || !cipher_desc->offloadable);
- buf_len = cipher_desc->salt + cipher_desc->iv + TLS_AAD_SPACE_SIZE +
+ aead_request_set_ad(aead_req, tls_ctx->prot_info.aad_size);
+
+ buf_len = cipher_desc->salt + cipher_desc->iv +
+ tls_ctx->prot_info.aad_size +
sync_size + cipher_desc->tag;
buf = kmalloc(buf_len, GFP_ATOMIC);
if (!buf)
@@ -324,7 +333,7 @@ static struct sk_buff *tls_enc_skb(struct tls_context *tls_ctx,
salt = crypto_info_salt(&tls_ctx->crypto_send.info, cipher_desc);
memcpy(iv, salt, cipher_desc->salt);
aad = buf + cipher_desc->salt + cipher_desc->iv;
- dummy_buf = aad + TLS_AAD_SPACE_SIZE;
+ dummy_buf = aad + tls_ctx->prot_info.aad_size;
nskb = alloc_skb(skb_headroom(skb) + skb->len, GFP_ATOMIC);
if (!nskb)
@@ -335,9 +344,8 @@ static struct sk_buff *tls_enc_skb(struct tls_context *tls_ctx,
fill_sg_out(sg_out, buf, tls_ctx, nskb, tcp_payload_offset,
payload_len, sync_size, dummy_buf);
- if (tls_enc_records(aead_req, ctx->aead_send, sg_in, sg_out, aad, iv,
- rcd_sn, sync_size + payload_len,
- &tls_ctx->prot_info) < 0)
+ if (tls_enc_records(tls_ctx, aead_req, ctx->aead_send, sg_in, sg_out,
+ aad, iv, rcd_sn, sync_size + payload_len) < 0)
goto free_nskb;
complete_skb(nskb, skb, tcp_payload_offset);
diff --git a/net/tls/tls_main.c b/net/tls/tls_main.c
index fd39acf41a61..fd04857fa0ab 100644
--- a/net/tls/tls_main.c
+++ b/net/tls/tls_main.c
@@ -711,49 +711,64 @@ static int do_tls_setsockopt_conf(struct sock *sk, sockptr_t optval,
}
if (tx) {
- rc = tls_set_device_offload(sk);
- conf = TLS_HW;
- if (!rc) {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXDEVICE);
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXDEVICE);
- } else {
- rc = tls_set_sw_offload(sk, 1,
- update ? crypto_info : NULL);
- if (rc)
- goto err_crypto_info;
-
- if (update) {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYOK);
- } else {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXSW);
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW);
+ if (update && ctx->tx_conf == TLS_HW) {
+ rc = -EOPNOTSUPP;
+ goto err_crypto_info;
+ }
+
+ if (!update) {
+ rc = tls_set_device_offload(sk);
+ conf = TLS_HW;
+ if (!rc) {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXDEVICE);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXDEVICE);
+ goto out;
}
- conf = TLS_SW;
}
- } else {
- rc = tls_set_device_offload_rx(sk, ctx);
- conf = TLS_HW;
- if (!rc) {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXDEVICE);
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXDEVICE);
+
+ rc = tls_set_sw_offload(sk, 1, update ? crypto_info : NULL);
+ if (rc)
+ goto err_crypto_info;
+
+ if (update) {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYOK);
} else {
- rc = tls_set_sw_offload(sk, 0,
- update ? crypto_info : NULL);
- if (rc)
- goto err_crypto_info;
-
- if (update) {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYOK);
- } else {
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXSW);
- TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXSW);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXSW);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW);
+ }
+ conf = TLS_SW;
+ } else {
+ if (update && ctx->rx_conf == TLS_HW) {
+ rc = -EOPNOTSUPP;
+ goto err_crypto_info;
+ }
+
+ if (!update) {
+ rc = tls_set_device_offload_rx(sk, ctx);
+ conf = TLS_HW;
+ if (!rc) {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXDEVICE);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXDEVICE);
+ tls_sw_strparser_arm(sk, ctx);
+ goto out;
}
- conf = TLS_SW;
}
- if (!update)
+
+ rc = tls_set_sw_offload(sk, 0, update ? crypto_info : NULL);
+ if (rc)
+ goto err_crypto_info;
+
+ if (update) {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYOK);
+ } else {
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXSW);
+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXSW);
tls_sw_strparser_arm(sk, ctx);
+ }
+ conf = TLS_SW;
}
+out:
if (tx)
ctx->tx_conf = conf;
else
--
2.25.1
^ permalink raw reply related
* [PATCH v12 2/6] net/mlx5e: add TLS 1.3 hardware offload support
From: Rishikesh Jethwani @ 2026-04-02 23:55 UTC (permalink / raw)
To: netdev
Cc: saeedm, tariqt, mbloch, borisp, john.fastabend, kuba, sd, davem,
pabeni, edumazet, leon, Rishikesh Jethwani
In-Reply-To: <20260402235511.664801-1-rjethwani@purestorage.com>
Enable TLS 1.3 TX/RX hardware offload on ConnectX-6 Dx and newer
crypto-enabled adapters.
Key changes:
- Add TLS 1.3 capability checking and version validation
- Use MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_3 (0x3) for crypto context
- Handle TLS 1.3 IV format: full 12-byte IV copied to gcm_iv +
implicit_iv (vs TLS 1.2's 4-byte salt only)
Tested with TLS 1.3 AES-GCM-128 and AES-GCM-256 cipher suites.
Signed-off-by: Rishikesh Jethwani <rjethwani@purestorage.com>
---
.../ethernet/mellanox/mlx5/core/en_accel/ktls.h | 8 +++++++-
.../mellanox/mlx5/core/en_accel/ktls_txrx.c | 14 +++++++++++---
2 files changed, 18 insertions(+), 4 deletions(-)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h
index 07a04a142a2e..0469ca6a0762 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h
@@ -30,7 +30,9 @@ static inline bool mlx5e_is_ktls_device(struct mlx5_core_dev *mdev)
return false;
return (MLX5_CAP_TLS(mdev, tls_1_2_aes_gcm_128) ||
- MLX5_CAP_TLS(mdev, tls_1_2_aes_gcm_256));
+ MLX5_CAP_TLS(mdev, tls_1_2_aes_gcm_256) ||
+ MLX5_CAP_TLS(mdev, tls_1_3_aes_gcm_128) ||
+ MLX5_CAP_TLS(mdev, tls_1_3_aes_gcm_256));
}
static inline bool mlx5e_ktls_type_check(struct mlx5_core_dev *mdev,
@@ -40,10 +42,14 @@ static inline bool mlx5e_ktls_type_check(struct mlx5_core_dev *mdev,
case TLS_CIPHER_AES_GCM_128:
if (crypto_info->version == TLS_1_2_VERSION)
return MLX5_CAP_TLS(mdev, tls_1_2_aes_gcm_128);
+ else if (crypto_info->version == TLS_1_3_VERSION)
+ return MLX5_CAP_TLS(mdev, tls_1_3_aes_gcm_128);
break;
case TLS_CIPHER_AES_GCM_256:
if (crypto_info->version == TLS_1_2_VERSION)
return MLX5_CAP_TLS(mdev, tls_1_2_aes_gcm_256);
+ else if (crypto_info->version == TLS_1_3_VERSION)
+ return MLX5_CAP_TLS(mdev, tls_1_3_aes_gcm_256);
break;
}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c
index 570a912dd6fa..f3f1be1d4034 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c
@@ -6,6 +6,7 @@
enum {
MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_2 = 0x2,
+ MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_3 = 0x3,
};
enum {
@@ -15,8 +16,10 @@ enum {
#define EXTRACT_INFO_FIELDS do { \
salt = info->salt; \
rec_seq = info->rec_seq; \
+ iv = info->iv; \
salt_sz = sizeof(info->salt); \
rec_seq_sz = sizeof(info->rec_seq); \
+ iv_sz = sizeof(info->iv); \
} while (0)
static void
@@ -24,9 +27,9 @@ fill_static_params(struct mlx5_wqe_tls_static_params_seg *params,
union mlx5e_crypto_info *crypto_info,
u32 key_id, u32 resync_tcp_sn)
{
+ u16 salt_sz, rec_seq_sz, iv_sz;
+ char *salt, *rec_seq, *iv;
char *initial_rn, *gcm_iv;
- u16 salt_sz, rec_seq_sz;
- char *salt, *rec_seq;
u8 tls_version;
u8 *ctx;
@@ -59,7 +62,12 @@ fill_static_params(struct mlx5_wqe_tls_static_params_seg *params,
memcpy(gcm_iv, salt, salt_sz);
memcpy(initial_rn, rec_seq, rec_seq_sz);
- tls_version = MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_2;
+ if (crypto_info->crypto_info.version == TLS_1_3_VERSION) {
+ memcpy(gcm_iv + salt_sz, iv, iv_sz);
+ tls_version = MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_3;
+ } else {
+ tls_version = MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_2;
+ }
MLX5_SET(tls_static_params, ctx, tls_version, tls_version);
MLX5_SET(tls_static_params, ctx, const_1, 1);
--
2.25.1
^ permalink raw reply related
* [PATCH v12 1/6] net: tls: reject TLS 1.3 offload in chcr_ktls and nfp drivers
From: Rishikesh Jethwani @ 2026-04-02 23:55 UTC (permalink / raw)
To: netdev
Cc: saeedm, tariqt, mbloch, borisp, john.fastabend, kuba, sd, davem,
pabeni, edumazet, leon, Rishikesh Jethwani
In-Reply-To: <20260402235511.664801-1-rjethwani@purestorage.com>
These drivers only support TLS 1.2. Return early when TLS 1.3
is requested to prevent unsupported hardware offload attempts.
Signed-off-by: Rishikesh Jethwani <rjethwani@purestorage.com>
---
drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c | 3 +++
drivers/net/ethernet/netronome/nfp/crypto/tls.c | 3 +++
2 files changed, 6 insertions(+)
diff --git a/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c
index f5acd4be1e69..29e108ce6764 100644
--- a/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c
+++ b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c
@@ -431,6 +431,9 @@ static int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk,
atomic64_inc(&port_stats->ktls_tx_connection_open);
u_ctx = adap->uld[CXGB4_ULD_KTLS].handle;
+ if (crypto_info->version != TLS_1_2_VERSION)
+ goto out;
+
if (direction == TLS_OFFLOAD_CTX_DIR_RX) {
pr_err("not expecting for RX direction\n");
goto out;
diff --git a/drivers/net/ethernet/netronome/nfp/crypto/tls.c b/drivers/net/ethernet/netronome/nfp/crypto/tls.c
index 9983d7aa2b9c..13864c6a55dc 100644
--- a/drivers/net/ethernet/netronome/nfp/crypto/tls.c
+++ b/drivers/net/ethernet/netronome/nfp/crypto/tls.c
@@ -287,6 +287,9 @@ nfp_net_tls_add(struct net_device *netdev, struct sock *sk,
BUILD_BUG_ON(offsetof(struct nfp_net_tls_offload_ctx, rx_end) >
TLS_DRIVER_STATE_SIZE_RX);
+ if (crypto_info->version != TLS_1_2_VERSION)
+ return -EOPNOTSUPP;
+
if (!nfp_net_cipher_supported(nn, crypto_info->cipher_type, direction))
return -EOPNOTSUPP;
--
2.25.1
^ permalink raw reply related
* [PATCH net-next v12 0/6] tls: Add TLS 1.3 hardware offload support
From: Rishikesh Jethwani @ 2026-04-02 23:55 UTC (permalink / raw)
To: netdev
Cc: saeedm, tariqt, mbloch, borisp, john.fastabend, kuba, sd, davem,
pabeni, edumazet, leon, Rishikesh Jethwani
Hi all,
This series adds TLS 1.3 hardware offload support including KeyUpdate
(rekey) and a selftest for validation.
Patch 1: Reject TLS 1.3 offload in chcr_ktls and nfp drivers
These drivers only support TLS 1.2; add explicit version check.
Patch 2: mlx5e TLS 1.3 hardware offload
Add TLS 1.3 TX/RX offload on ConnectX-6 Dx and newer.
Handle 12-byte IV format and TLS_1_3 context type.
Patch 3: Core TLS 1.3 hardware offload support
Extend tls_device.c for TLS 1.3 record format (content type
appended before tag). Handle TLS 1.3 IV construction in fallback.
Patch 4: Split tls_set_sw_offload into init/finalize
Allows HW RX path to init SW context, attempt HW setup, then
finalize. Required for proper rekey error handling.
Patch 5: Hardware offload key update (rekey) support
Delete old HW context and add new one with updated key.
Track ACKs to ensure old-key data is flushed before HW switch.
Patch 6: Selftest for hardware offload
Python wrapper + C binary using NetDrvEpEnv framework.
Tests TLS 1.2/1.3, AES-GCM-128/256, rekey, various buffer sizes.
Tested on Mellanox ConnectX-6 Dx (Crypto Enabled) with TLS 1.3 AES-GCM-128/256
and multiple rekey cycles.
Rishikesh
Changes in v12:
- Rebased onto current net-next/main
- Addressed selftest review comments (patch 6/6)
Rishikesh Jethwani (6):
net: tls: reject TLS 1.3 offload in chcr_ktls and nfp drivers
net/mlx5e: add TLS 1.3 hardware offload support
tls: add TLS 1.3 hardware offload support
tls: split tls_set_sw_offload into init and finalize stages
tls: add hardware offload key update support
selftests: net: add TLS hardware offload test
.../chelsio/inline_crypto/ch_ktls/chcr_ktls.c | 3 +
.../mellanox/mlx5/core/en_accel/ktls.h | 8 +-
.../mellanox/mlx5/core/en_accel/ktls_txrx.c | 14 +-
.../net/ethernet/netronome/nfp/crypto/tls.c | 3 +
include/net/tls.h | 76 +-
include/uapi/linux/snmp.h | 2 +
net/tls/tls.h | 20 +-
net/tls/tls_device.c | 570 +++++++++++--
net/tls/tls_device_fallback.c | 82 +-
net/tls/tls_main.c | 33 +-
net/tls/tls_proc.c | 2 +
net/tls/tls_sw.c | 109 ++-
.../selftests/drivers/net/hw/.gitignore | 1 +
.../testing/selftests/drivers/net/hw/Makefile | 2 +
.../selftests/drivers/net/hw/tls_hw_offload.c | 767 ++++++++++++++++++
.../drivers/net/hw/tls_hw_offload.py | 171 ++++
16 files changed, 1683 insertions(+), 180 deletions(-)
create mode 100644 tools/testing/selftests/drivers/net/hw/tls_hw_offload.c
create mode 100755 tools/testing/selftests/drivers/net/hw/tls_hw_offload.py
--
2.25.1
^ permalink raw reply
* Re: [Intel-wired-lan] [PATCH] igc: fix Tx timestamp timeout caused by unlocked TIMINCA write in adj fine]
From: Vinicius Costa Gomes @ 2026-04-02 23:48 UTC (permalink / raw)
To: Bob Van Valzah, Vadim Fedorenko
Cc: intel-wired-lan, anthony.l.nguyen, netdev, julianstj, jeff,
Lasse Johnsen, Ian Gough
In-Reply-To: <CDC20B85-E83B-4AC9-9261-24B3D967DCA6@vanvalzah.com>
Hi,
Bob Van Valzah <bob@vanvalzah.com> writes:
> Vadim,
>
> Thanks for the feedback on our first patch. We've spent more time in
> the lab studying the igc TX timestamp behavior under stress. We
> understand the failure modes much better now.
>
> You were right that ptp_tx_lock was the wrong lock — it guards the TX
> queue, not the timing registers. You suggested tmreg_lock instead. We
> tested tmreg_lock alone (v2) and found it doesn't fix the bug: it appears
> that the race is between the software TIMINCA write and the hardware’s
> asynchronous TX timestamp capture pipeline, not between two software threads.
> tmreg_lock serializes software register accesses but can't prevent the hardware
> from reading TIMINCA at the instant software writes it.
>
> Our v3 patch (attached) takes tmreg_lock as you suggested, and
> additionally disables TX timestamping in hardware via TSYNCTXCTL around
> the TIMINCA write. This prevents the hardware from starting new
> timestamp captures during the rate change:
>
> spin_lock_irqsave(&igc->tmreg_lock, flags);
> txctl = rd32(IGC_TSYNCTXCTL);
> wr32(IGC_TSYNCTXCTL, txctl & ~IGC_TSYNCTXCTL_ENABLED);
> wr32(IGC_TIMINCA, inca);
> wr32(IGC_TSYNCTXCTL, txctl);
> spin_unlock_irqrestore(&igc->tmreg_lock, flags);
>
I sent, a couple of days ago, the link to your report to our hardware
folks, waiting for them to take a look.
I think that this workaround, even if incomplete, will be interesting to
them as well. Again, thanks for the detailed report.
Cheers,
--
Vinicius
^ permalink raw reply
* Re: [PATCH] vsock: avoid timeout for non-blocking accept() with empty backlog
From: Laurence Rowe @ 2026-04-02 23:30 UTC (permalink / raw)
To: Stefano Garzarella; +Cc: virtualization, netdev, Bobby Eshleman
In-Reply-To: <CAOycyLTy=ZT0AXiwPrbEfyxX_ZZgwzaetKOqP9cNywTO1_wNZw@mail.gmail.com>
On Thu, Apr 2, 2026 at 12:22 PM Laurence Rowe <laurencerowe@gmail.com> wrote:
>
> On Thu, Apr 2, 2026 at 5:03 AM Stefano Garzarella <sgarzare@redhat.com> wrote:
>
> > >Scheduling a timeout for a non-blocking accept with an empty backlog
> > >meant AF_VSOCK sockets used by epoll network servers incurred hundreds
> > >of microseconds of additional latency per accept loop compared to
> > >AF_INET or AF_UNIX sockets.
> >
> > Not related to this patch, but should we do something similar (in
> > another patch) also in vsock_connect() or doesn't matter since usually
> > it's always blocking?
>
> Looking at vsock_connect it's not immediately obvious to me whether it
> is affected
> in the same way. I'll capture some ftraces and follow up after
> updating this patch.
This does not seem to be a problem for vsock_connect since it checks
for `if (flags & O_NONBLOCK) {` in the while loop before calling
`schedule_timeout`.
Timings and ftraces:
https://github.com/lrowe/linux-vsock-accept-timeout-investigation?tab=readme-ov-file#a-quick-look-at-connect
Laurence
^ permalink raw reply
* [RFC PATCH 5/6] net: add Rust reference driver for nlmon
From: Wenzhao Liao @ 2026-04-02 16:36 UTC (permalink / raw)
To: rust-for-linux, netdev
Cc: linux-kernel, ojeda, boqun, gary, bjorn3_gh, lossin, a.hindborg,
aliceryhl, tmgross, dakr, andrew+netdev, davem, edumazet, kuba,
pabeni
In-Reply-To: <20260402163640.1079056-1-wenzhaoliao@ruc.edu.cn>
Implement nlmon as a small Rust reference driver on top of the new
networking abstractions.
nlmon is a good validation target because it is netlink-centric and
exercises rtnl registration, netdevice setup, tap lifecycle, and packet
stats without hardware dependencies.
Signed-off-by: Wenzhao Liao <wenzhaoliao@ruc.edu.cn>
---
drivers/net/Kconfig | 9 ++++
drivers/net/Makefile | 6 ++-
drivers/net/nlmon_rust.rs | 93 +++++++++++++++++++++++++++++++++++++++
3 files changed, 107 insertions(+), 1 deletion(-)
create mode 100644 drivers/net/nlmon_rust.rs
diff --git a/drivers/net/Kconfig b/drivers/net/Kconfig
index 17108c359216..0eccd0af8b75 100644
--- a/drivers/net/Kconfig
+++ b/drivers/net/Kconfig
@@ -466,6 +466,15 @@ config NLMON
diagnostics, etc. This is mostly intended for developers or support
to debug netlink issues. If unsure, say N.
+config NLMON_RUST
+ bool "Rust implementation of nlmon"
+ depends on RUST && NLMON
+ help
+ Select this option to build the Rust implementation of
+ the nlmon driver instead of the original C version.
+ This keeps a small reference driver available while the
+ supporting Rust networking abstractions are reviewed.
+
config NETKIT
bool "BPF-programmable network device"
depends on BPF_SYSCALL
diff --git a/drivers/net/Makefile b/drivers/net/Makefile
index 5b01215f6829..5d3357d56cc8 100644
--- a/drivers/net/Makefile
+++ b/drivers/net/Makefile
@@ -38,7 +38,11 @@ obj-$(CONFIG_VXLAN) += vxlan/
obj-$(CONFIG_GENEVE) += geneve.o
obj-$(CONFIG_BAREUDP) += bareudp.o
obj-$(CONFIG_GTP) += gtp.o
-obj-$(CONFIG_NLMON) += nlmon.o
+ifdef CONFIG_NLMON_RUST
+ obj-$(CONFIG_NLMON) += nlmon_rust.o
+else
+ obj-$(CONFIG_NLMON) += nlmon.o
+endif
obj-$(CONFIG_PFCP) += pfcp.o
obj-$(CONFIG_NET_VRF) += vrf.o
obj-$(CONFIG_VSOCKMON) += vsockmon.o
diff --git a/drivers/net/nlmon_rust.rs b/drivers/net/nlmon_rust.rs
new file mode 100644
index 000000000000..d10e320c4635
--- /dev/null
+++ b/drivers/net/nlmon_rust.rs
@@ -0,0 +1,93 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#![forbid(unsafe_code)]
+
+//! Rust netlink monitoring device.
+//!
+//! C version of this driver: [`drivers/net/nlmon.c`](./nlmon.c)
+
+use kernel::{
+ net::{netdevice, netlink_tap, rtnl, skbuff, stats},
+ prelude::*,
+};
+
+module! {
+ type: NlmonModule,
+ name: "nlmon_rust",
+ authors: [
+ "Daniel Borkmann <dborkman@redhat.com>",
+ "Mathieu Geli <geli@enseirb.fr>",
+ ],
+ description: "Rust netlink monitoring device",
+ license: "GPL v2",
+ alias: ["rtnl-link-nlmon"],
+}
+
+#[pin_data]
+struct NlmonModule {
+ #[pin]
+ registration: rtnl::Registration<NlmonDriver>,
+}
+
+impl kernel::InPlaceModule for NlmonModule {
+ fn init(_module: &'static ThisModule) -> impl PinInit<Self, Error> {
+ try_pin_init!(Self {
+ registration <- rtnl::Registration::new(),
+ })
+ }
+}
+
+struct NlmonDriver;
+
+#[pin_data]
+#[derive(Zeroable)]
+#[repr(C)]
+struct NlmonPriv {
+ #[pin]
+ tap: netlink_tap::Tap,
+}
+
+impl netdevice::Operations for NlmonDriver {
+ type Private = NlmonPriv;
+
+ fn open(dev: &mut netdevice::Device, private: Pin<&mut Self::Private>) -> Result {
+ private.project().tap.add(dev, &THIS_MODULE)
+ }
+
+ fn stop(_dev: &mut netdevice::Device, private: Pin<&mut Self::Private>) -> Result {
+ private.project().tap.remove()
+ }
+
+ fn start_xmit(skb: skbuff::SkBuff, dev: &netdevice::Device) -> netdevice::TxOutcome {
+ stats::dev_lstats_add(dev, skb.len());
+ netdevice::TxOutcome::Ok
+ }
+}
+
+impl rtnl::Driver for NlmonDriver {
+ const KIND: &'static CStr = c"nlmon";
+
+ fn setup(dev: &mut netdevice::Device) {
+ let features = netdevice::features::SG
+ | netdevice::features::FRAGLIST
+ | netdevice::features::HIGHDMA;
+
+ dev.set_type(netdevice::device_type::NETLINK);
+ dev.add_priv_flag(netdevice::priv_flags::NO_QUEUE);
+ dev.set_lltx(true);
+ dev.set_needs_free_netdev(true);
+ dev.set_features(features);
+ dev.set_flags(netdevice::flags::NO_ARP);
+ dev.set_pcpu_stat_type(netdevice::pcpu_stat_type::LSTATS);
+ dev.set_mtu(netdevice::mtu::nlmsg_goodsize());
+ dev.set_min_mtu(netdevice::mtu::NLMSGHDR);
+ }
+
+ fn validate(ctx: &mut rtnl::ValidateContext<'_>) -> Result {
+ if ctx.has_link_attr(rtnl::LinkAttr::ADDRESS) {
+ return Err(EINVAL);
+ }
+
+ Ok(())
+ }
+}
--
2.34.1
^ permalink raw reply related
* [RFC PATCH 3/6] rust: net: add minimal skbuff, netdevice, and stats abstractions
From: Wenzhao Liao @ 2026-04-02 16:36 UTC (permalink / raw)
To: rust-for-linux, netdev
Cc: linux-kernel, ojeda, boqun, gary, bjorn3_gh, lossin, a.hindborg,
aliceryhl, tmgross, dakr, andrew+netdev, davem, edumazet, kuba,
pabeni
In-Reply-To: <20260402163640.1079056-1-wenzhaoliao@ruc.edu.cn>
Add narrow Rust wrappers for sk_buff access, net_device private state,
and lightweight stats updates.
These APIs keep ownership with kernel-managed networking objects while
confining unsafe operations to the abstraction boundary.
Signed-off-by: Wenzhao Liao <wenzhaoliao@ruc.edu.cn>
---
rust/kernel/net.rs | 3 +
rust/kernel/net/netdevice.rs | 319 +++++++++++++++++++++++++++++++++++
rust/kernel/net/skbuff.rs | 67 ++++++++
rust/kernel/net/stats.rs | 17 ++
4 files changed, 406 insertions(+)
create mode 100644 rust/kernel/net/netdevice.rs
create mode 100644 rust/kernel/net/skbuff.rs
create mode 100644 rust/kernel/net/stats.rs
diff --git a/rust/kernel/net.rs b/rust/kernel/net.rs
index fe415cb369d3..a61bc76f4499 100644
--- a/rust/kernel/net.rs
+++ b/rust/kernel/net.rs
@@ -4,3 +4,6 @@
#[cfg(CONFIG_RUST_PHYLIB_ABSTRACTIONS)]
pub mod phy;
+pub mod netdevice;
+pub mod skbuff;
+pub mod stats;
diff --git a/rust/kernel/net/netdevice.rs b/rust/kernel/net/netdevice.rs
new file mode 100644
index 000000000000..1e6a63741422
--- /dev/null
+++ b/rust/kernel/net/netdevice.rs
@@ -0,0 +1,319 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Network device support.
+//!
+//! C headers: [`include/linux/netdevice.h`](srctree/include/linux/netdevice.h)
+
+use crate::{
+ bindings,
+ error::from_result,
+ net::skbuff,
+ prelude::*,
+ types::Opaque,
+};
+
+/// Network device feature flags.
+pub type Features = bindings::netdev_features_t;
+
+/// `features` bits used by the MVP link-type setup path.
+pub mod features {
+ use super::{bindings, Features};
+
+ const fn bit(index: u32) -> Features {
+ 1u64 << index
+ }
+
+ /// Equivalent to `NETIF_F_SG`.
+ pub const SG: Features = bit(bindings::NETIF_F_SG_BIT as u32);
+
+ /// Equivalent to `NETIF_F_FRAGLIST`.
+ pub const FRAGLIST: Features = bit(bindings::NETIF_F_FRAGLIST_BIT as u32);
+
+ /// Equivalent to `NETIF_F_HIGHDMA`.
+ pub const HIGHDMA: Features = bit(bindings::NETIF_F_HIGHDMA_BIT as u32);
+}
+
+/// MTU-related constants and helpers used by the MVP link-type setup path.
+pub mod mtu {
+ use super::bindings;
+
+ const fn align(value: usize, alignment: usize) -> usize {
+ (value + alignment - 1) & !(alignment - 1)
+ }
+
+ /// Equivalent to `NLMSG_GOODSIZE`.
+ pub const fn nlmsg_goodsize() -> u32 {
+ let page = if bindings::PAGE_SIZE < 8192 {
+ bindings::PAGE_SIZE
+ } else {
+ 8192
+ };
+ let overhead = align(
+ core::mem::size_of::<bindings::skb_shared_info>(),
+ bindings::SMP_CACHE_BYTES as usize,
+ );
+
+ (page - overhead) as u32
+ }
+
+ /// Equivalent to `sizeof(struct nlmsghdr)`.
+ pub const NLMSGHDR: u32 = core::mem::size_of::<bindings::nlmsghdr>() as u32;
+}
+
+/// `priv_flags` bits used by the MVP link-type setup path.
+pub mod priv_flags {
+ use super::bindings;
+
+ /// Equivalent to `IFF_NO_QUEUE`.
+ pub const NO_QUEUE: u32 = bindings::netdev_priv_flags_IFF_NO_QUEUE;
+}
+
+/// `flags` bits used by the MVP link-type setup path.
+pub mod flags {
+ use super::bindings;
+
+ /// Equivalent to `IFF_NOARP`.
+ pub const NO_ARP: u32 = bindings::net_device_flags_IFF_NOARP;
+}
+
+/// Device type constants used by the MVP link-type setup path.
+pub mod device_type {
+ use super::bindings;
+
+ /// Equivalent to `ARPHRD_NETLINK`.
+ pub const NETLINK: u16 = bindings::ARPHRD_NETLINK as u16;
+}
+
+/// Per-cpu stat type constants used by the MVP link-type setup path.
+pub mod pcpu_stat_type {
+ use super::bindings;
+
+ /// Equivalent to `NETDEV_PCPU_STAT_LSTATS`.
+ pub const LSTATS: bindings::netdev_stat_type =
+ bindings::netdev_stat_type_NETDEV_PCPU_STAT_LSTATS;
+}
+
+/// Result of an `ndo_start_xmit` callback.
+pub enum TxOutcome {
+ /// The skb has been consumed and may be dropped by the abstraction.
+ Ok,
+
+ /// The device is temporarily busy; ownership of the skb goes back to the networking core.
+ Busy(skbuff::SkBuff),
+}
+
+/// Operations exposed by a network device for the MVP `nlmon` loop.
+///
+/// The callback trampolines live in this abstraction layer so that driver code does not have to
+/// touch raw kernel pointers or private storage casts directly.
+pub trait Operations: Send + Sync + 'static {
+ /// The type stored in `net_device` private storage.
+ ///
+ /// The RTNL/netdevice allocation path zero-initializes private storage before any driver
+ /// callback observes it, so implementations must accept the all-zero bit pattern.
+ type Private: Zeroable;
+
+ /// Device open callback.
+ fn open(dev: &mut Device, private: Pin<&mut Self::Private>) -> Result;
+
+ /// Device stop callback.
+ fn stop(dev: &mut Device, private: Pin<&mut Self::Private>) -> Result;
+
+ /// Packet transmit callback.
+ ///
+ /// This only exposes a shared device view: TX callbacks may run concurrently, so the
+ /// abstraction must not synthesize an exclusive borrow of either `net_device` or private
+ /// driver state here.
+ fn start_xmit(skb: skbuff::SkBuff, dev: &Device) -> TxOutcome;
+}
+
+/// A Rust wrapper over `struct net_device`.
+///
+/// # Invariants
+///
+/// - The wrapped pointer always points to a valid `struct net_device`.
+/// - The caller is responsible for only creating references in contexts where the kernel allows
+/// the accessed fields to be mutated.
+#[repr(transparent)]
+pub struct Device(Opaque<bindings::net_device>);
+
+impl Device {
+ /// Creates a mutable wrapper from a raw `net_device` pointer.
+ ///
+ /// # Safety
+ ///
+ /// The pointer must point to a valid `struct net_device` for the lifetime of the returned
+ /// reference, and the caller must ensure that it is safe to mutate through it.
+ pub(crate) unsafe fn from_raw<'a>(ptr: *mut bindings::net_device) -> &'a mut Self {
+ let ptr = ptr.cast::<Self>();
+ // SAFETY: The caller guarantees the pointer is valid for the returned lifetime.
+ unsafe { &mut *ptr }
+ }
+
+ /// Creates a shared wrapper from a raw `net_device` pointer.
+ ///
+ /// # Safety
+ ///
+ /// The pointer must point to a valid `struct net_device` for the lifetime of the returned
+ /// reference.
+ pub(crate) unsafe fn from_raw_ref<'a>(ptr: *mut bindings::net_device) -> &'a Self {
+ let ptr = ptr.cast::<Self>();
+ // SAFETY: The caller guarantees the pointer is valid for the returned lifetime.
+ unsafe { &*ptr }
+ }
+
+ /// Returns the wrapped raw `net_device` pointer.
+ pub(crate) fn as_ptr(&self) -> *mut bindings::net_device {
+ self.0.get()
+ }
+
+ /// Sets `dev->type`.
+ pub fn set_type(&mut self, device_type: u16) {
+ let dev = self.as_ptr();
+ // SAFETY: The type invariant guarantees that `dev` points to a valid `net_device`.
+ unsafe { (*dev).type_ = device_type };
+ }
+
+ /// ORs a bit into `dev->priv_flags`.
+ pub fn add_priv_flag(&mut self, flag: u32) {
+ let dev = self.as_ptr();
+ // SAFETY: The type invariant guarantees that `dev` points to a valid `net_device`, and the
+ // bindgen-generated accessors preserve the layout of the `priv_flags/lltx` union.
+ unsafe {
+ let flags = &mut (*dev).__bindgen_anon_1.__bindgen_anon_1;
+ let current = flags.priv_flags();
+ let flag = flag as usize;
+ flags.set_priv_flags(current | flag);
+ }
+ }
+
+ /// Sets `dev->lltx`.
+ pub fn set_lltx(&mut self, enabled: bool) {
+ let dev = self.as_ptr();
+ // SAFETY: The type invariant guarantees that `dev` points to a valid `net_device`, and the
+ // bindgen-generated accessors preserve the layout of the `priv_flags/lltx` union.
+ unsafe {
+ let flags = &mut (*dev).__bindgen_anon_1.__bindgen_anon_1;
+ flags.set_lltx(if enabled { 1 } else { 0 });
+ }
+ }
+
+ /// Stores a typed `net_device_ops` vtable pointer into `dev->netdev_ops`.
+ pub fn set_netdevice_ops<T: Operations>(&mut self) {
+ let dev = self.as_ptr();
+ // SAFETY: The type invariant guarantees that `dev` points to a valid `net_device`.
+ unsafe { (*dev).netdev_ops = OperationsVTable::<T>::build() };
+ }
+
+ /// Sets `dev->needs_free_netdev`.
+ pub fn set_needs_free_netdev(&mut self, enabled: bool) {
+ let dev = self.as_ptr();
+ // SAFETY: The type invariant guarantees that `dev` points to a valid `net_device`.
+ unsafe { (*dev).needs_free_netdev = enabled };
+ }
+
+ /// Sets `dev->features`.
+ pub fn set_features(&mut self, features: Features) {
+ let dev = self.as_ptr();
+ // SAFETY: The type invariant guarantees that `dev` points to a valid `net_device`.
+ unsafe { (*dev).features = features };
+ }
+
+ /// Sets `dev->flags`.
+ pub fn set_flags(&mut self, flags: u32) {
+ let dev = self.as_ptr();
+ // SAFETY: The type invariant guarantees that `dev` points to a valid `net_device`.
+ unsafe { (*dev).flags = flags };
+ }
+
+ /// Sets `dev->mtu`.
+ pub fn set_mtu(&mut self, mtu: u32) {
+ let dev = self.as_ptr();
+ // SAFETY: The type invariant guarantees that `dev` points to a valid `net_device`.
+ unsafe { (*dev).mtu = mtu };
+ }
+
+ /// Sets `dev->min_mtu`.
+ pub fn set_min_mtu(&mut self, mtu: u32) {
+ let dev = self.as_ptr();
+ // SAFETY: The type invariant guarantees that `dev` points to a valid `net_device`.
+ unsafe { (*dev).min_mtu = mtu };
+ }
+
+ /// Sets `dev->pcpu_stat_type`.
+ pub fn set_pcpu_stat_type(&mut self, stat_type: bindings::netdev_stat_type) {
+ let dev = self.as_ptr();
+ // SAFETY: The type invariant guarantees that `dev` points to a valid `net_device`.
+ unsafe { (*dev).set_pcpu_stat_type(stat_type) };
+ }
+
+ /// Returns a typed raw pointer to `net_device` private storage.
+ ///
+ /// # Safety
+ ///
+ /// The caller must ensure that the current `net_device` was allocated with private storage for
+ /// `T`. Dereferencing the returned pointer additionally requires exclusive access to that
+ /// storage.
+ pub(crate) unsafe fn private_ptr<T>(&self) -> *mut T {
+ // SAFETY: The caller guarantees that the private storage really contains a `T`, and the
+ // helper/exported wrapper preserves the C `netdev_priv` semantics.
+ unsafe { bindings::netdev_priv(self.as_ptr() as *const bindings::net_device).cast::<T>() }
+ }
+}
+
+struct OperationsVTable<T: Operations>(core::marker::PhantomData<T>);
+
+impl<T: Operations> OperationsVTable<T> {
+ extern "C" fn open_callback(dev: *mut bindings::net_device) -> c_int {
+ from_result(|| {
+ // SAFETY: The networking core only calls this callback with a valid `net_device`.
+ let dev = unsafe { Device::from_raw(dev) };
+ // SAFETY: The `rtnl::Registration<T>` abstraction ties `priv_size` to `T::Private`.
+ let private = unsafe { Pin::new_unchecked(&mut *dev.private_ptr::<T::Private>()) };
+ T::open(dev, private)?;
+ Ok(0)
+ })
+ }
+
+ extern "C" fn stop_callback(dev: *mut bindings::net_device) -> c_int {
+ from_result(|| {
+ // SAFETY: The networking core only calls this callback with a valid `net_device`.
+ let dev = unsafe { Device::from_raw(dev) };
+ // SAFETY: The `rtnl::Registration<T>` abstraction ties `priv_size` to `T::Private`.
+ let private = unsafe { Pin::new_unchecked(&mut *dev.private_ptr::<T::Private>()) };
+ T::stop(dev, private)?;
+ Ok(0)
+ })
+ }
+
+ extern "C" fn start_xmit_callback(
+ skb: *mut bindings::sk_buff,
+ dev: *mut bindings::net_device,
+ ) -> bindings::netdev_tx {
+ // SAFETY: The networking core only calls this callback with a valid `net_device`.
+ let dev = unsafe { Device::from_raw_ref(dev) };
+ // SAFETY: The networking core transfers ownership of the callback skb to the driver.
+ let skb = unsafe { skbuff::SkBuff::from_raw_owned(skb) };
+
+ match T::start_xmit(skb, dev) {
+ TxOutcome::Ok => bindings::netdev_tx_NETDEV_TX_OK,
+ TxOutcome::Busy(skb) => {
+ let _ = skb.into_raw();
+ bindings::netdev_tx_NETDEV_TX_BUSY
+ }
+ }
+ }
+
+ const VTABLE: bindings::net_device_ops = bindings::net_device_ops {
+ ndo_open: Some(Self::open_callback),
+ ndo_stop: Some(Self::stop_callback),
+ ndo_start_xmit: Some(Self::start_xmit_callback),
+ // SAFETY: A zeroed `net_device_ops` is valid because omitted callbacks are represented by
+ // null pointers and all remaining fields are plain data/pointers.
+ ..unsafe { core::mem::zeroed() }
+ };
+
+ pub(crate) const fn build() -> &'static bindings::net_device_ops {
+ &Self::VTABLE
+ }
+}
diff --git a/rust/kernel/net/skbuff.rs b/rust/kernel/net/skbuff.rs
new file mode 100644
index 000000000000..32ca2bb24a75
--- /dev/null
+++ b/rust/kernel/net/skbuff.rs
@@ -0,0 +1,67 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Socket-buffer ownership wrappers.
+//!
+//! C header: [`include/linux/skbuff.h`](srctree/include/linux/skbuff.h)
+
+use crate::bindings;
+
+use core::ptr::NonNull;
+
+/// Owns a single `struct sk_buff` passed into Rust from `ndo_start_xmit`.
+///
+/// # Invariants
+///
+/// - `ptr` is either `None` after ownership was handed back to the networking core, or it points
+/// to a valid `struct sk_buff` exclusively owned by this value.
+/// - Dropping this wrapper releases the skb exactly once via `dev_kfree_skb`.
+pub struct SkBuff {
+ ptr: Option<NonNull<bindings::sk_buff>>,
+}
+
+impl SkBuff {
+ /// Creates an owned skb wrapper from the callback argument.
+ ///
+ /// # Safety
+ ///
+ /// `ptr` must be a valid non-null skb pointer whose ownership is transferred to Rust for the
+ /// duration of the callback.
+ pub(crate) unsafe fn from_raw_owned(ptr: *mut bindings::sk_buff) -> Self {
+ Self {
+ // SAFETY: The caller guarantees ownership of a valid non-null skb pointer.
+ ptr: Some(unsafe { NonNull::new_unchecked(ptr) }),
+ }
+ }
+
+ fn ptr(&self) -> *mut bindings::sk_buff {
+ self.ptr
+ .expect("SkBuff ownership already transferred back to the core")
+ .as_ptr()
+ }
+
+ /// Returns `skb->len`.
+ pub fn len(&self) -> u32 {
+ // SAFETY: `Self` owns a valid skb until ownership is explicitly returned with `into_raw`.
+ unsafe { (*self.ptr()).len }
+ }
+
+ /// Hands ownership back to the networking core without freeing the skb.
+ pub(crate) fn into_raw(mut self) -> *mut bindings::sk_buff {
+ self.ptr
+ .take()
+ .expect("SkBuff ownership already transferred back to the core")
+ .as_ptr()
+ }
+}
+
+impl Drop for SkBuff {
+ fn drop(&mut self) {
+ let Some(ptr) = self.ptr.take() else {
+ return;
+ };
+
+ // SAFETY: `Self` owns the skb exactly once unless ownership has been transferred back to
+ // the networking core via `into_raw`.
+ unsafe { bindings::dev_kfree_skb(ptr.as_ptr()) };
+ }
+}
diff --git a/rust/kernel/net/stats.rs b/rust/kernel/net/stats.rs
new file mode 100644
index 000000000000..3e17f3b19c05
--- /dev/null
+++ b/rust/kernel/net/stats.rs
@@ -0,0 +1,17 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Network-device statistics helpers.
+//!
+//! C header: [`include/linux/netdevice.h`](srctree/include/linux/netdevice.h)
+
+use crate::{
+ bindings,
+ net::netdevice,
+};
+
+/// Equivalent to `dev_lstats_add(dev, len)`.
+pub fn dev_lstats_add(dev: &netdevice::Device, len: u32) {
+ // SAFETY: The helper expects a valid `net_device *`; `netdevice::Device` maintains that
+ // invariant and the helper handles the required per-cpu synchronization internally.
+ unsafe { bindings::dev_lstats_add(dev.as_ptr(), len) };
+}
--
2.34.1
^ permalink raw reply related
* [RFC PATCH 2/6] rust: helpers: add net_device and sk_buff helper wrappers
From: Wenzhao Liao @ 2026-04-02 16:36 UTC (permalink / raw)
To: rust-for-linux, netdev
Cc: linux-kernel, ojeda, boqun, gary, bjorn3_gh, lossin, a.hindborg,
aliceryhl, tmgross, dakr, andrew+netdev, davem, edumazet, kuba,
pabeni
In-Reply-To: <20260402163640.1079056-1-wenzhaoliao@ruc.edu.cn>
Add C helper entry points for the small networking helpers that are not
exposed through bindgen-friendly interfaces.
This covers netdev_priv() and dev_lstats_add(), which are used by the
reference driver and remain hidden behind Rust-side safe wrappers.
Signed-off-by: Wenzhao Liao <wenzhaoliao@ruc.edu.cn>
---
rust/helpers/helpers.c | 1 +
rust/helpers/net.c | 19 +++++++++++++++++++
2 files changed, 20 insertions(+)
create mode 100644 rust/helpers/net.c
diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c
index a3c42e51f00a..90fe1c7cbadc 100644
--- a/rust/helpers/helpers.c
+++ b/rust/helpers/helpers.c
@@ -38,6 +38,7 @@
#include "maple_tree.c"
#include "mm.c"
#include "mutex.c"
+#include "net.c"
#include "of.c"
#include "page.c"
#include "pci.c"
diff --git a/rust/helpers/net.c b/rust/helpers/net.c
new file mode 100644
index 000000000000..da04c718acd4
--- /dev/null
+++ b/rust/helpers/net.c
@@ -0,0 +1,19 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/netdevice.h>
+#include <linux/skbuff.h>
+
+__rust_helper void rust_helper_dev_kfree_skb(struct sk_buff *skb)
+{
+ dev_kfree_skb(skb);
+}
+
+__rust_helper void rust_helper_dev_lstats_add(struct net_device *dev, unsigned int len)
+{
+ dev_lstats_add(dev, len);
+}
+
+__rust_helper void *rust_helper_netdev_priv(const struct net_device *dev)
+{
+ return netdev_priv(dev);
+}
--
2.34.1
^ permalink raw reply related
* [RFC PATCH 0/6] rust: net: introduce minimal rtnl/netdevice abstractions and nlmon reference driver
From: Wenzhao Liao @ 2026-04-02 16:36 UTC (permalink / raw)
To: rust-for-linux, netdev
Cc: linux-kernel, ojeda, boqun, gary, bjorn3_gh, lossin, a.hindborg,
aliceryhl, tmgross, dakr, andrew+netdev, davem, edumazet, kuba,
pabeni
Hi,
This RFC proposes a minimal set of Rust networking abstractions for
link-type drivers, together with a Rust implementation of nlmon as a
reference driver.
The scope is intentionally narrow. The series only tries to cover the
pieces needed to model:
- rtnl link registration and validation
- net_device private state allocation
- sk_buff transmit ownership
- netlink tap lifetime management
- per-device stats updates
nlmon is a useful target for this because it is small, self-contained,
and exercises rtnl registration, netdevice setup, tap lifecycle, and
stats handling without introducing hardware-specific probe or teardown
paths.
The series is structured as follows:
- patches 1-2 expose the required bindings and helper glue
- patches 3-4 add the minimal Rust net abstractions
- patch 5 adds the Rust nlmon reference driver
- patch 6 adds scoped MAINTAINERS entries
The driver itself is built with #![forbid(unsafe_code)] and does not
call raw bindings directly. Unsafe is confined to small abstraction
implementations under rust/kernel/net/.
Validation so far:
- drivers/net/nlmon_rust.o builds successfully
- x86_64 bzImage builds successfully
- QEMU smoke passes:
ip link add nlmon0 type nlmon
ip link set nlmon0 up
ip link show nlmon0
ip link set nlmon0 down
ip link del nlmon0
The Rust implementation is kept behind CONFIG_NLMON_RUST so the new
abstractions and the reference driver can be reviewed without displacing
the existing C implementation.
Feedback on the abstraction shape, the lifetime model, and whether
nlmon is a suitable reference driver would be especially appreciated.
I would also be grateful for guidance on any aspect of this series that
does not match Rust-for-Linux or netdev expectations. If the direction
is useful but the submission, factoring, or abstraction boundary needs
to change, I am happy to revise the series accordingly and resubmit.
Thanks,
Wenzhao Liao
Wenzhao Liao (6):
rust: bindings: expose networking headers needed by nlmon
rust: helpers: add net_device and sk_buff helper wrappers
rust: net: add minimal skbuff, netdevice, and stats abstractions
rust: net: add minimal rtnl registration and netlink tap support
net: add Rust reference driver for nlmon
MAINTAINERS: add Rust net and nlmon entries
MAINTAINERS | 19 ++
drivers/net/Kconfig | 9 +
drivers/net/Makefile | 6 +-
drivers/net/nlmon_rust.rs | 93 ++++++++++
rust/bindings/bindings_helper.h | 4 +
rust/helpers/helpers.c | 1 +
rust/helpers/net.c | 19 ++
rust/kernel/net.rs | 5 +
rust/kernel/net/netdevice.rs | 319 ++++++++++++++++++++++++++++++++
rust/kernel/net/netlink_tap.rs | 89 +++++++++
rust/kernel/net/rtnl.rs | 221 ++++++++++++++++++++++
rust/kernel/net/skbuff.rs | 67 +++++++
rust/kernel/net/stats.rs | 17 ++
13 files changed, 868 insertions(+), 1 deletion(-)
create mode 100644 drivers/net/nlmon_rust.rs
create mode 100644 rust/helpers/net.c
create mode 100644 rust/kernel/net/netdevice.rs
create mode 100644 rust/kernel/net/netlink_tap.rs
create mode 100644 rust/kernel/net/rtnl.rs
create mode 100644 rust/kernel/net/skbuff.rs
create mode 100644 rust/kernel/net/stats.rs
--
2.34.1
^ permalink raw reply
* [PATCH v2] mm/vmpressure: skip socket pressure for costly order reclaim
From: JP Kobryn (Meta) @ 2026-04-02 23:25 UTC (permalink / raw)
To: linux-mm, willy, hannes, akpm, david, ljs, Liam.Howlett, vbabka,
rppt, surenb, mhocko, kasong, qi.zheng, shakeel.butt, baohua,
axelrasmussen, yuanchu, weixugc, riel, kuba, edumazet
Cc: netdev, linux-kernel, kernel-team
When kswapd reclaims at high order due to fragmentation, vmpressure() can
report poor reclaim efficiency even though the system has plenty of free
memory. This is because kswapd scans many pages but finds little to reclaim
- the pages are actively in use and don't need to be freed. The resulting
scan:reclaim ratio triggers socket pressure, throttling TCP throughput
unnecessarily.
Net allocations do not exceed order 3 (PAGE_ALLOC_COSTLY_ORDER), so high
order reclaim difficulty should not trigger socket pressure. The kernel
already treats this order as the boundary where reclaim is no longer
expected to succeed and compaction may take over.
Make vmpressure() order-aware through an additional parameter sourced from
scan_control at existing call sites. Socket pressure is now only asserted
when order <= PAGE_ALLOC_COSTLY_ORDER.
Memcg reclaim is unaffected since try_to_free_mem_cgroup_pages() always
uses order 0, which passes the filter unconditionally. Similarly,
vmpressure_prio() now passes order 0 internally when calling vmpressure(),
ensuring critical pressure from low reclaim priority is not suppressed by
the order filter.
Signed-off-by: JP Kobryn (Meta) <jp.kobryn@linux.dev>
---
v2
- dropped extern specifier from vmpressure decl
- added comment to explain rationale of adjusted conditional
v1: https://lore.kernel.org/all/20260401203752.643259-1-jp.kobryn@linux.dev/
include/linux/vmpressure.h | 9 +++++----
mm/vmpressure.c | 15 ++++++++++++---
mm/vmscan.c | 8 ++++----
3 files changed, 21 insertions(+), 11 deletions(-)
diff --git a/include/linux/vmpressure.h b/include/linux/vmpressure.h
index 6a2f51ebbfd35..faecd55224017 100644
--- a/include/linux/vmpressure.h
+++ b/include/linux/vmpressure.h
@@ -30,8 +30,8 @@ struct vmpressure {
struct mem_cgroup;
#ifdef CONFIG_MEMCG
-extern void vmpressure(gfp_t gfp, struct mem_cgroup *memcg, bool tree,
- unsigned long scanned, unsigned long reclaimed);
+void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg, bool tree,
+ unsigned long scanned, unsigned long reclaimed);
extern void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, int prio);
extern void vmpressure_init(struct vmpressure *vmpr);
@@ -44,8 +44,9 @@ extern int vmpressure_register_event(struct mem_cgroup *memcg,
extern void vmpressure_unregister_event(struct mem_cgroup *memcg,
struct eventfd_ctx *eventfd);
#else
-static inline void vmpressure(gfp_t gfp, struct mem_cgroup *memcg, bool tree,
- unsigned long scanned, unsigned long reclaimed) {}
+static inline void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg,
+ bool tree, unsigned long scanned,
+ unsigned long reclaimed) {}
static inline void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg,
int prio) {}
#endif /* CONFIG_MEMCG */
diff --git a/mm/vmpressure.c b/mm/vmpressure.c
index 3fbb86996c4d2..f053554e58264 100644
--- a/mm/vmpressure.c
+++ b/mm/vmpressure.c
@@ -218,6 +218,7 @@ static void vmpressure_work_fn(struct work_struct *work)
/**
* vmpressure() - Account memory pressure through scanned/reclaimed ratio
* @gfp: reclaimer's gfp mask
+ * @order: allocation order being reclaimed for
* @memcg: cgroup memory controller handle
* @tree: legacy subtree mode
* @scanned: number of pages scanned
@@ -236,7 +237,7 @@ static void vmpressure_work_fn(struct work_struct *work)
*
* This function does not return any value.
*/
-void vmpressure(gfp_t gfp, struct mem_cgroup *memcg, bool tree,
+void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg, bool tree,
unsigned long scanned, unsigned long reclaimed)
{
struct vmpressure *vmpr;
@@ -307,7 +308,15 @@ void vmpressure(gfp_t gfp, struct mem_cgroup *memcg, bool tree,
level = vmpressure_calc_level(scanned, reclaimed);
- if (level > VMPRESSURE_LOW) {
+ /*
+ * Once we go above COSTLY_ORDER, reclaim relies heavily on
+ * compaction to make progress. Reclaim efficiency was never a
+ * great proxy for pressure to begin with, but it's outright
+ * misleading with these high orders. Don't throttle sockets
+ * because somebody is attempting something crazy like an order-7
+ * and predictably struggling.
+ */
+ if (level > VMPRESSURE_LOW && order <= PAGE_ALLOC_COSTLY_ORDER) {
/*
* Let the socket buffer allocator know that
* we are having trouble reclaiming LRU pages.
@@ -348,7 +357,7 @@ void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, int prio)
* to the vmpressure() basically means that we signal 'critical'
* level.
*/
- vmpressure(gfp, memcg, true, vmpressure_win, 0);
+ vmpressure(gfp, 0, memcg, true, vmpressure_win, 0);
}
#define MAX_VMPRESSURE_ARGS_LEN (strlen("critical") + strlen("hierarchy") + 2)
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 5a8c8fcccbfc9..1342323a0b41f 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -5071,8 +5071,8 @@ static int shrink_one(struct lruvec *lruvec, struct scan_control *sc)
shrink_slab(sc->gfp_mask, pgdat->node_id, memcg, sc->priority);
if (!sc->proactive)
- vmpressure(sc->gfp_mask, memcg, false, sc->nr_scanned - scanned,
- sc->nr_reclaimed - reclaimed);
+ vmpressure(sc->gfp_mask, sc->order, memcg, false,
+ sc->nr_scanned - scanned, sc->nr_reclaimed - reclaimed);
flush_reclaim_state(sc);
@@ -6175,7 +6175,7 @@ static void shrink_node_memcgs(pg_data_t *pgdat, struct scan_control *sc)
/* Record the group's reclaim efficiency */
if (!sc->proactive)
- vmpressure(sc->gfp_mask, memcg, false,
+ vmpressure(sc->gfp_mask, sc->order, memcg, false,
sc->nr_scanned - scanned,
sc->nr_reclaimed - reclaimed);
@@ -6220,7 +6220,7 @@ static void shrink_node(pg_data_t *pgdat, struct scan_control *sc)
/* Record the subtree's reclaim efficiency */
if (!sc->proactive)
- vmpressure(sc->gfp_mask, sc->target_mem_cgroup, true,
+ vmpressure(sc->gfp_mask, sc->order, sc->target_mem_cgroup, true,
sc->nr_scanned - nr_scanned, nr_node_reclaimed);
if (nr_node_reclaimed)
--
2.52.0
^ permalink raw reply related
* [net-next 3/3] selftests/net: Test PACKET_AUXDATA
From: Joe Damato @ 2026-04-02 23:21 UTC (permalink / raw)
To: netdev, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Simon Horman, Shuah Khan
Cc: andrew+netdev, linux-kernel, willemb, Joe Damato, linux-kselftest
In-Reply-To: <20260402232203.2004496-1-joe@dama.to>
Extend the packet socket selftest, adding a recvmsg path, to test
PACKET_AUXDATA. Check basic attributes of tpacket_auxdata.
Signed-off-by: Joe Damato <joe@dama.to>
---
tools/testing/selftests/net/psock_snd.c | 57 ++++++++++++++++++++++--
tools/testing/selftests/net/psock_snd.sh | 5 +++
2 files changed, 59 insertions(+), 3 deletions(-)
diff --git a/tools/testing/selftests/net/psock_snd.c b/tools/testing/selftests/net/psock_snd.c
index 7bbce274b9f0..682e0afe364d 100644
--- a/tools/testing/selftests/net/psock_snd.c
+++ b/tools/testing/selftests/net/psock_snd.c
@@ -40,6 +40,7 @@ static bool cfg_use_qdisc_bypass;
static bool cfg_use_vlan;
static bool cfg_use_vnet;
static bool cfg_drop;
+static bool cfg_aux_data;
static char *cfg_ifname = "lo";
static int cfg_mtu = 1500;
@@ -281,9 +282,26 @@ static int setup_rx(void)
static void do_rx(int fd, int expected_len, char *expected)
{
+ char cmsg_buf[1024] = {};
+ struct msghdr msg = {};
+ struct iovec iov[1];
int ret;
- ret = recv(fd, rbuf, sizeof(rbuf), 0);
+ if (cfg_aux_data) {
+ iov[0].iov_base = rbuf;
+ iov[0].iov_len = sizeof(rbuf);
+
+ msg.msg_iov = iov;
+ msg.msg_iovlen = 1;
+
+ msg.msg_control = cmsg_buf;
+ msg.msg_controllen = sizeof(cmsg_buf);
+
+ ret = recvmsg(fd, &msg, 0);
+ } else {
+ ret = recv(fd, rbuf, sizeof(rbuf), 0);
+ }
+
if (ret == -1)
error(1, errno, "recv");
if (ret != expected_len)
@@ -292,6 +310,31 @@ static void do_rx(int fd, int expected_len, char *expected)
if (memcmp(rbuf, expected, ret))
error(1, 0, "recv: data mismatch");
+ if (cfg_aux_data) {
+ struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg);
+ struct tpacket_auxdata *adata;
+
+ if (!cmsg)
+ error(1, 0, "auxdata null");
+
+ if (cmsg->cmsg_level != SOL_PACKET)
+ error(1, 0, "cmsg_level != SOL_PACKET");
+
+ if (cmsg->cmsg_type != PACKET_AUXDATA)
+ error(1, 0, "cmsg_type != PACKET_AUXDATA");
+
+ adata = (struct tpacket_auxdata *)CMSG_DATA(cmsg);
+
+ if (adata->tp_net != ETH_HLEN)
+ error(1, 0, "cmsg tp_net != ETH_HLEN");
+
+ if (adata->tp_len != expected_len)
+ error(1, 0, "cmsg tp_len != %u", expected_len);
+
+ if (adata->tp_snaplen != expected_len)
+ error(1, 0, "cmsg tp_snaplen != %u", expected_len);
+ }
+
fprintf(stderr, "rx: %u\n", ret);
}
@@ -312,6 +355,10 @@ static int setup_sniffer(void)
if (setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &one, sizeof(one)))
error(1, errno, "setsockopt SO_RCVBUF");
+ if (cfg_aux_data)
+ if (setsockopt(fd, SOL_PACKET, PACKET_AUXDATA, &one, sizeof(one)))
+ error(1, errno, "setsockopt PACKET_AUXDATA");
+
pair_udp_setfilter(fd);
do_bind(fd);
@@ -322,8 +369,11 @@ static void parse_opts(int argc, char **argv)
{
int c;
- while ((c = getopt(argc, argv, "bcCdDgl:qt:vV")) != -1) {
+ while ((c = getopt(argc, argv, "abcCdDgl:qt:vV")) != -1) {
switch (c) {
+ case 'a':
+ cfg_aux_data = true;
+ break;
case 'b':
cfg_use_bind = true;
break;
@@ -436,7 +486,8 @@ static void run_test(void)
check_packet_stats(fds);
}
- do_rx(fdr, cfg_payload_len, tbuf + total_len - cfg_payload_len);
+ if (!cfg_aux_data)
+ do_rx(fdr, cfg_payload_len, tbuf + total_len - cfg_payload_len);
out:
if (close(fds))
diff --git a/tools/testing/selftests/net/psock_snd.sh b/tools/testing/selftests/net/psock_snd.sh
index b6ef12fad5d5..111c9e2f0d21 100755
--- a/tools/testing/selftests/net/psock_snd.sh
+++ b/tools/testing/selftests/net/psock_snd.sh
@@ -97,4 +97,9 @@ echo "raw gso max size + 1 (expected to fail)"
echo "test drops statistics"
./in_netns.sh ./psock_snd -D
+# test aux data
+
+echo "test aux data"
+./in_netns.sh ./psock_snd -a
+
echo "OK. All tests passed"
--
2.52.0
^ permalink raw reply related
* [net-next 2/3] selftests/net: Test PACKET_STATISTICS drops
From: Joe Damato @ 2026-04-02 23:21 UTC (permalink / raw)
To: netdev, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Simon Horman, Shuah Khan
Cc: andrew+netdev, linux-kernel, willemb, Joe Damato, linux-kselftest
In-Reply-To: <20260402232203.2004496-1-joe@dama.to>
Extend psock_snd to test drops by setting a tiny receive buffer and
sending a large burst of packets.
Signed-off-by: Joe Damato <joe@dama.to>
---
tools/testing/selftests/net/psock_snd.c | 52 ++++++++++++++++++++----
tools/testing/selftests/net/psock_snd.sh | 5 +++
2 files changed, 48 insertions(+), 9 deletions(-)
diff --git a/tools/testing/selftests/net/psock_snd.c b/tools/testing/selftests/net/psock_snd.c
index 5be481a3d2bd..7bbce274b9f0 100644
--- a/tools/testing/selftests/net/psock_snd.c
+++ b/tools/testing/selftests/net/psock_snd.c
@@ -39,6 +39,7 @@ static bool cfg_use_gso;
static bool cfg_use_qdisc_bypass;
static bool cfg_use_vlan;
static bool cfg_use_vnet;
+static bool cfg_drop;
static char *cfg_ifname = "lo";
static int cfg_mtu = 1500;
@@ -49,6 +50,8 @@ static uint16_t cfg_port = 8000;
/* test sending up to max mtu + 1 */
#define TEST_SZ (sizeof(struct virtio_net_hdr) + ETH_HLEN + ETH_MAX_MTU + 1)
+#define BURST_CNT (1000)
+
static char tbuf[TEST_SZ], rbuf[TEST_SZ];
static unsigned long add_csum_hword(const uint16_t *start, int num_u16)
@@ -212,13 +215,14 @@ static void do_send(int fd, char *buf, int len)
if (ret != len)
error(1, 0, "write: %u %u", ret, len);
- fprintf(stderr, "tx: %u\n", ret);
+ if (!cfg_drop)
+ fprintf(stderr, "tx: %u\n", ret);
}
-static int do_tx(void)
+static int do_tx(bool burst)
{
const int one = 1;
- int fd, len;
+ int i, fd, len;
fd = socket(PF_PACKET, cfg_use_dgram ? SOCK_DGRAM : SOCK_RAW, 0);
if (fd == -1)
@@ -242,6 +246,10 @@ static int do_tx(void)
do_send(fd, tbuf, len);
+ if (burst)
+ for (i = 0; i < BURST_CNT; i++)
+ do_send(fd, tbuf, len);
+
if (close(fd))
error(1, errno, "close t");
@@ -290,6 +298,7 @@ static void do_rx(int fd, int expected_len, char *expected)
static int setup_sniffer(void)
{
struct timeval tv = { .tv_usec = 100 * 1000 };
+ const int one = 1;
int fd;
fd = socket(PF_PACKET, SOCK_RAW, 0);
@@ -299,6 +308,10 @@ static int setup_sniffer(void)
if (setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)))
error(1, errno, "setsockopt rcv timeout");
+ if (cfg_drop)
+ if (setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &one, sizeof(one)))
+ error(1, errno, "setsockopt SO_RCVBUF");
+
pair_udp_setfilter(fd);
do_bind(fd);
@@ -309,7 +322,7 @@ static void parse_opts(int argc, char **argv)
{
int c;
- while ((c = getopt(argc, argv, "bcCdgl:qt:vV")) != -1) {
+ while ((c = getopt(argc, argv, "bcCdDgl:qt:vV")) != -1) {
switch (c) {
case 'b':
cfg_use_bind = true;
@@ -323,6 +336,9 @@ static void parse_opts(int argc, char **argv)
case 'd':
cfg_use_dgram = true;
break;
+ case 'D':
+ cfg_drop = true;
+ break;
case 'g':
cfg_use_gso = true;
break;
@@ -367,11 +383,23 @@ static void check_packet_stats(int fd)
if (getsockopt(fd, SOL_PACKET, PACKET_STATISTICS, &st, &len))
error(1, errno, "getsockopt packet statistics");
- if (st.tp_packets != 1)
- error(1, 0, "stats: tp_packets %u != 1", st.tp_packets);
+ if (cfg_drop) {
+ /* PACKET_STATISTICS reports all packets seen (including
+ * drops) in tp_packets
+ */
+ if (st.tp_packets < st.tp_drops)
+ error(1, 0, "stats: tp_packets %u < tp_drops %u",
+ st.tp_packets, st.tp_drops);
- if (st.tp_drops != 0)
- error(1, 0, "stats: tp_drops %u != 0", st.tp_drops);
+ if (st.tp_drops == 0)
+ error(1, 0, "stats: expected drops but tp_drops == 0");
+ } else {
+ if (st.tp_packets != 1)
+ error(1, 0, "stats: tp_packets %u != 1", st.tp_packets);
+
+ if (st.tp_drops != 0)
+ error(1, 0, "stats: tp_drops %u != 0", st.tp_drops);
+ }
/* verify clear on read */
memset(&st, 0xff, sizeof(st));
@@ -394,7 +422,12 @@ static void run_test(void)
fdr = setup_rx();
fds = setup_sniffer();
- total_len = do_tx();
+ total_len = do_tx(cfg_drop);
+
+ if (cfg_drop) {
+ check_packet_stats(fds);
+ goto out;
+ }
/* BPF filter accepts only this length, vlan changes MAC */
if (cfg_payload_len == DATA_LEN && !cfg_use_vlan) {
@@ -405,6 +438,7 @@ static void run_test(void)
do_rx(fdr, cfg_payload_len, tbuf + total_len - cfg_payload_len);
+out:
if (close(fds))
error(1, errno, "close s");
if (close(fdr))
diff --git a/tools/testing/selftests/net/psock_snd.sh b/tools/testing/selftests/net/psock_snd.sh
index 1cbfeb5052ec..b6ef12fad5d5 100755
--- a/tools/testing/selftests/net/psock_snd.sh
+++ b/tools/testing/selftests/net/psock_snd.sh
@@ -92,4 +92,9 @@ echo "raw gso max size"
echo "raw gso max size + 1 (expected to fail)"
(! ./in_netns.sh ./psock_snd -v -c -g -l "${max_mss_exceeds}")
+# test drops statistics
+
+echo "test drops statistics"
+./in_netns.sh ./psock_snd -D
+
echo "OK. All tests passed"
--
2.52.0
^ permalink raw reply related
* [net-next 1/3] selftests/net: Test PACKET_STATISTICS
From: Joe Damato @ 2026-04-02 23:21 UTC (permalink / raw)
To: netdev, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Simon Horman, Shuah Khan
Cc: andrew+netdev, linux-kernel, willemb, Joe Damato, linux-kselftest
In-Reply-To: <20260402232203.2004496-1-joe@dama.to>
Update the existing packet socket test to include a test for the sockopt
PACKET_STATISTICS.
Signed-off-by: Joe Damato <joe@dama.to>
---
tools/testing/selftests/net/psock_snd.c | 32 ++++++++++++++++++++++++-
1 file changed, 31 insertions(+), 1 deletion(-)
diff --git a/tools/testing/selftests/net/psock_snd.c b/tools/testing/selftests/net/psock_snd.c
index edf1e6f80d41..5be481a3d2bd 100644
--- a/tools/testing/selftests/net/psock_snd.c
+++ b/tools/testing/selftests/net/psock_snd.c
@@ -359,6 +359,34 @@ static void parse_opts(int argc, char **argv)
error(1, 0, "option gso (-g) requires csum offload (-c)");
}
+static void check_packet_stats(int fd)
+{
+ struct tpacket_stats st = {};
+ socklen_t len = sizeof(st);
+
+ if (getsockopt(fd, SOL_PACKET, PACKET_STATISTICS, &st, &len))
+ error(1, errno, "getsockopt packet statistics");
+
+ if (st.tp_packets != 1)
+ error(1, 0, "stats: tp_packets %u != 1", st.tp_packets);
+
+ if (st.tp_drops != 0)
+ error(1, 0, "stats: tp_drops %u != 0", st.tp_drops);
+
+ /* verify clear on read */
+ memset(&st, 0xff, sizeof(st));
+ len = sizeof(st);
+
+ if (getsockopt(fd, SOL_PACKET, PACKET_STATISTICS, &st, &len))
+ error(1, errno, "getsockopt packet statistics");
+
+ if (st.tp_packets != 0)
+ error(1, 0, "stats: tp_packets %u != 0 after clear", st.tp_packets);
+
+ if (st.tp_drops != 0)
+ error(1, 0, "stats: tp_drops %u != 0 after clear", st.tp_drops);
+}
+
static void run_test(void)
{
int fdr, fds, total_len;
@@ -369,9 +397,11 @@ static void run_test(void)
total_len = do_tx();
/* BPF filter accepts only this length, vlan changes MAC */
- if (cfg_payload_len == DATA_LEN && !cfg_use_vlan)
+ if (cfg_payload_len == DATA_LEN && !cfg_use_vlan) {
do_rx(fds, total_len - sizeof(struct virtio_net_hdr),
tbuf + sizeof(struct virtio_net_hdr));
+ check_packet_stats(fds);
+ }
do_rx(fdr, cfg_payload_len, tbuf + total_len - cfg_payload_len);
--
2.52.0
^ permalink raw reply related
* [net-next 0/3] Extend packet socket selftests
From: Joe Damato @ 2026-04-02 23:21 UTC (permalink / raw)
To: netdev
Cc: andrew+netdev, davem, edumazet, kuba, pabeni, horms, linux-kernel,
willemb, Joe Damato, linux-kselftest, Shuah Khan
Greetings:
I was looking around the packet socket code and noticed there were a few
features that could be tested by extending the existing packet socket
tests.
I extended the test to test stats, drops, and auxdata and re-ran the tests.
The existing and new tests passed.
Thanks,
Joe
Joe Damato (3):
selftests/net: Test PACKET_STATISTICS
selftests/net: Test PACKET_STATISTICS drops
selftests/net: Test PACKET_AUXDATA
tools/testing/selftests/net/psock_snd.c | 131 +++++++++++++++++++++--
tools/testing/selftests/net/psock_snd.sh | 10 ++
2 files changed, 133 insertions(+), 8 deletions(-)
base-commit: f35340f2d653f1003602878403c901396ab03c17
--
2.52.0
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox