[PATCH v4 0/8] s390: Reintroduce support for DCACHE_WORD_ACCESS

[PATCH v4 0/8] s390: Reintroduce support for DCACHE_WORD_ACCESS

[Heiko Carstens]

v4:
- Add two more cleanup patches based on Sashiko feedback, however I
  don't consider them as bug fixes. This is more cosmetics:
  - Use handle_fault_error() in do_secure_storage_access()
  - Use goto statement in do_secure_storage_access()

v3:
- Fix two more pre-existing bugs reported by Sashiko
- Add Christian's Tested-by tag to four of the six patches

v2:
- Explicitely PAGE_ALIGN size in uv_alloc_stor_var() [Sashiko]
- Document that lazy PTE update / TLB flushing is intended [Sashiko]

v1:
Support for DCACHE_WORD_ACCESS was recently removed [1] since it caused
problems with the incomplete handling of secure storage access
exceptions. It looked like fixing the exception handler would be a larger
effort; therefore support for DCACHE_WORD_ACCESS was removed as a work
around.

Address the potential problems that exist with secure storage access
exceptions and add support for DCACHE_WORD_ACCESS again.

In particular address the following problems:

- Reading the guest variable storage area via the /proc/kcore interface
  results in short reads. Address this by using a VM_SPARSE area for the
  guest variable storage area. VM_SPARSE areas will be handled like
  memory holes (zeros will be read).

- Fix handling of secure storage access exceptions in vmalloc area.

- Remove folio handling for secure storage access exceptions to avoid
  potential data corruption.

[1] 37540b8c287f ("s390: Revert support for DCACHE_WORD_ACCESS")

Heiko Carstens (8):
  KVM: s390: pv: Use VM_SPARSE area for guest variable storage area
  s390/mm: Add missing mm check to do_secure_storage_access()
  s390/mm: Use lock_mm_and_find_vma() in do_secure_storage_access()
  s390/mm: Fix handling of vmalloc area in do_secure_storage_access()
  s390/mm: Remove folio handling for kernel faults in do_secure_storage_access()
  s390/mm: Use handle_fault_error() in do_secure_storage_access()
  s390/mm: Use goto statement in do_secure_storage_access()
  s390: Add support for DCACHE_WORD_ACCESS (again)

 arch/s390/Kconfig                      |  1 +
 arch/s390/include/asm/asm-extable.h    |  4 ++
 arch/s390/include/asm/uv.h             |  2 +
 arch/s390/include/asm/word-at-a-time.h | 22 +++++++++
 arch/s390/kernel/uv.c                  | 65 ++++++++++++++++++++++++++
 arch/s390/kvm/pv.c                     |  6 +--
 arch/s390/mm/extable.c                 | 18 +++++++
 arch/s390/mm/fault.c                   | 35 ++++++--------
 8 files changed, 128 insertions(+), 25 deletions(-)

-- 
2.53.0

[PATCH v4 1/8] KVM: s390: pv: Use VM_SPARSE area for guest variable storage area

[Heiko Carstens]

The guest variable storage area is allocated with vmalloc and then
donated to the ultravisor. Any kernel access to that area will result
in a secure storage access exception (aka fault).

This is a problem if such a memory area is read via /proc/kcore. This
causes an exception via vread_iter() and results in an unexpected short
read. Avoid this by allocating a custom VM_SPARSE area. If such an area
is read, vread_iter() returns zeroes for the entire area.

Note that the function which frees the area does not update ptes. This
is intentional to allow for deferred / lazy pte updates and TLB flushing
like the generic vfree() code is doing that. See vunmap_pte_range().

This assumes that s390 will gain full support for lazy_mmu_mode_enable()
and lazy_mmu_mode_disable() in the future, since as of now the used
ptep_get_and_clear() in vunmap_pte_range() does indeed invalidate and
flush every single pte entry, but only for s390.

Tested-by: Christian Borntraeger <borntraeger@linux.ibm.com>
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
---
 arch/s390/include/asm/uv.h |  2 ++
 arch/s390/kernel/uv.c      | 65 ++++++++++++++++++++++++++++++++++++++
 arch/s390/kvm/pv.c         |  6 ++--
 3 files changed, 70 insertions(+), 3 deletions(-)

diff --git a/arch/s390/include/asm/uv.h b/arch/s390/include/asm/uv.h
index d919e69662f5..153fed63adda 100644
--- a/arch/s390/include/asm/uv.h
+++ b/arch/s390/include/asm/uv.h
@@ -635,6 +635,8 @@ int s390_wiggle_split_folio(struct mm_struct *mm, struct folio *folio);
 int __make_folio_secure(struct folio *folio, struct uv_cb_header *uvcb);
 int uv_convert_from_secure(unsigned long paddr);
 int uv_convert_from_secure_folio(struct folio *folio);
+void *uv_alloc_stor_var(unsigned long size);
+void uv_free_stor_var(void *stor_var);
 
 void setup_uv(void);
 
diff --git a/arch/s390/kernel/uv.c b/arch/s390/kernel/uv.c
index a284f98d9716..d1cdc5ae5e27 100644
--- a/arch/s390/kernel/uv.c
+++ b/arch/s390/kernel/uv.c
@@ -16,6 +16,7 @@
 #include <linux/swap.h>
 #include <linux/pagewalk.h>
 #include <linux/backing-dev.h>
+#include <linux/vmalloc.h>
 #include <asm/facility.h>
 #include <asm/sections.h>
 #include <asm/uv.h>
@@ -209,6 +210,70 @@ int uv_convert_from_secure_pte(pte_t pte)
 	return uv_convert_from_secure_folio(pfn_folio(pte_pfn(pte)));
 }
 
+static int uv_free_range_cb(pte_t *ptep, unsigned long addr, void *data)
+{
+	pte_t pte = ptep_get(ptep);
+
+	if (!pte_present(pte))
+		return 0;
+	/*
+	 * Note: do not update the pte here, since there is no code which
+	 * accesses the memory range, besides bugs. The invalidation of ptes
+	 * and TLB flushing is deferred like for regular vfree() calls.
+	 */
+	__free_page(pte_page(pte));
+	return 0;
+}
+
+void uv_free_stor_var(void *stor_var)
+{
+	unsigned long addr, size;
+	struct vm_struct *area;
+
+	if (!stor_var)
+		return;
+	area = find_vm_area(stor_var);
+	if (WARN_ON_ONCE(!area || !(area->flags & VM_SPARSE)))
+		return;
+	size = get_vm_area_size(area);
+	addr = (unsigned long)area->addr;
+	apply_to_existing_page_range(&init_mm, addr, size, uv_free_range_cb, NULL);
+	free_vm_area(area);
+}
+EXPORT_SYMBOL_FOR_MODULES(uv_free_stor_var, "kvm");
+
+static int uv_alloc_range_cb(pte_t *ptep, unsigned long addr, void *data)
+{
+	struct page *page;
+	pte_t pte;
+
+	page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
+	if (!page)
+		return -ENOMEM;
+	pte = __pte(page_to_phys(page) | pgprot_val(PAGE_KERNEL));
+	set_pte(ptep, pte);
+	return 0;
+}
+
+void *uv_alloc_stor_var(unsigned long size)
+{
+	struct vm_struct *area;
+	unsigned long addr;
+
+	size = PAGE_ALIGN(size);
+	area = get_vm_area(size, VM_SPARSE);
+	if (!area)
+		return NULL;
+	addr = (unsigned long)area->addr;
+	if (apply_to_page_range(&init_mm, addr, size, uv_alloc_range_cb, NULL))
+		goto out;
+	return area->addr;
+out:
+	uv_free_stor_var(area->addr);
+	return NULL;
+}
+EXPORT_SYMBOL_FOR_MODULES(uv_alloc_stor_var, "kvm");
+
 /*
  * Calculate the expected ref_count for a folio that would otherwise have no
  * further pins. This was cribbed from similar functions in other places in
diff --git a/arch/s390/kvm/pv.c b/arch/s390/kvm/pv.c
index 1beacc841ca8..dc5ac29b4c31 100644
--- a/arch/s390/kvm/pv.c
+++ b/arch/s390/kvm/pv.c
@@ -337,7 +337,7 @@ int kvm_s390_pv_create_cpu(struct kvm_vcpu *vcpu, u16 *rc, u16 *rrc)
 /* only free resources when the destroy was successful */
 static void kvm_s390_pv_dealloc_vm(struct kvm *kvm)
 {
-	vfree(kvm->arch.pv.stor_var);
+	uv_free_stor_var(kvm->arch.pv.stor_var);
 	free_pages(kvm->arch.pv.stor_base,
 		   get_order(uv_info.guest_base_stor_len));
 	kvm_s390_clear_pv_state(kvm);
@@ -369,7 +369,7 @@ static int kvm_s390_pv_alloc_vm(struct kvm *kvm)
 	/* Allocate variable storage */
 	vlen = ALIGN(virt * ((npages * PAGE_SIZE) / HPAGE_SIZE), PAGE_SIZE);
 	vlen += uv_info.guest_virt_base_stor_len;
-	kvm->arch.pv.stor_var = vzalloc(vlen);
+	kvm->arch.pv.stor_var = uv_alloc_stor_var(vlen);
 	if (!kvm->arch.pv.stor_var)
 		goto out_err;
 	return 0;
@@ -414,7 +414,7 @@ static int kvm_s390_pv_dispose_one_leftover(struct kvm *kvm,
 	 */
 	free_pages(leftover->stor_base, get_order(uv_info.guest_base_stor_len));
 	free_pages(leftover->old_gmap_table, CRST_ALLOC_ORDER);
-	vfree(leftover->stor_var);
+	uv_free_stor_var(leftover->stor_var);
 done_fast:
 	atomic_dec(&kvm->mm->context.protected_count);
 	return 0;
-- 
2.53.0

[PATCH v4 2/8] s390/mm: Add missing mm check to do_secure_storage_access()

[Heiko Carstens]

Similar like the normal exception handler the secure storage access
exception handler needs to verify if mm is not NULL, which may happen
with kernel threads.

Reported-by: sashiko-bot <sashiko-bot@kernel.org>
Closes: https://lore.kernel.org/all/20260717132345.539A11F00A3A@smtp.kernel.org/
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
---
 arch/s390/mm/fault.c | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c
index 028aeb9c48d6..2839d7a64401 100644
--- a/arch/s390/mm/fault.c
+++ b/arch/s390/mm/fault.c
@@ -409,9 +409,9 @@ void do_secure_storage_access(struct pt_regs *regs)
 {
 	union teid teid = { .val = regs->int_parm_long };
 	unsigned long addr = get_fault_address(regs);
+	struct mm_struct *mm = current->mm;
 	struct vm_area_struct *vma;
 	struct folio_walk fw;
-	struct mm_struct *mm;
 	struct folio *folio;
 	int rc;
 
@@ -453,9 +453,8 @@ void do_secure_storage_access(struct pt_regs *regs)
 		if (rc)
 			return handle_fault_error_nolock(regs, 0);
 	} else {
-		if (faulthandler_disabled())
+		if (faulthandler_disabled() || !mm)
 			return handle_fault_error_nolock(regs, 0);
-		mm = current->mm;
 		mmap_read_lock(mm);
 		vma = find_vma(mm, addr);
 		if (!vma)
-- 
2.53.0

[PATCH v4 3/8] s390/mm: Use lock_mm_and_find_vma() in do_secure_storage_access()

[Heiko Carstens]

do_secure_storage_access() uses find_vma() without verifying that the
faulting address is within the returned vma. Add this missing check by
converting to lock_mm_and_find_vma().

This is not a critical fix, since the worst that could happen is a
WARN_ON_ONCE() in folio_walk_start().

Reported-by: sashiko-bot <sashiko-bot@kernel.org>
Closes: https://lore.kernel.org/all/20260717093904.E4A421F00A3E@smtp.kernel.org/
Link: https://lore.kernel.org/all/20260717093904.E4A421F00A3E@smtp.kernel.org/
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
---
 arch/s390/mm/fault.c | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c
index 2839d7a64401..f150d97cac2e 100644
--- a/arch/s390/mm/fault.c
+++ b/arch/s390/mm/fault.c
@@ -455,10 +455,9 @@ void do_secure_storage_access(struct pt_regs *regs)
 	} else {
 		if (faulthandler_disabled() || !mm)
 			return handle_fault_error_nolock(regs, 0);
-		mmap_read_lock(mm);
-		vma = find_vma(mm, addr);
+		vma = lock_mm_and_find_vma(mm, addr, regs);
 		if (!vma)
-			return handle_fault_error(regs, SEGV_MAPERR);
+			return handle_fault_error_nolock(regs, SEGV_MAPERR);
 		folio = folio_walk_start(&fw, vma, addr, 0);
 		if (!folio) {
 			mmap_read_unlock(mm);
-- 
2.53.0

[PATCH v4 4/8] s390/mm: Fix handling of vmalloc area in do_secure_storage_access()

[Heiko Carstens]

Secure storage access exceptions in vmalloc area need to be handled
separately since folio_try_get() or virt_to_phys() do not work on such
addresses. Given that there cannot be a valid access let such exceptions
fail immediately. The kernel will either fixup or crash.

Tested-by: Christian Borntraeger <borntraeger@linux.ibm.com>
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
---
 arch/s390/mm/fault.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c
index f150d97cac2e..e650cdfa2523 100644
--- a/arch/s390/mm/fault.c
+++ b/arch/s390/mm/fault.c
@@ -438,6 +438,8 @@ void do_secure_storage_access(struct pt_regs *regs)
 		panic("Unexpected PGM 0x3d with TEID bit 61=0");
 	}
 	if (is_kernel_fault(regs)) {
+		if (is_vmalloc_addr((void *)addr))
+			return handle_fault_error_nolock(regs, 0);
 		folio = virt_to_folio((void *)addr);
 		if (unlikely(!folio_try_get(folio)))
 			return;
-- 
2.53.0

[PATCH v4 5/8] s390/mm: Remove folio handling for kernel faults in do_secure_storage_access()

[Heiko Carstens]

Remove folio handling for secure storage access exceptions that happen
via the kernel mapping. Using folio_try_get() for exceptions via the
kernel mapping is racy: for example such an exception may happen on one
CPU, while a different CPU frees the corresponding page. Subsequently the
page can be part of a folio of different size, before the faulting CPU
executes folio_try_get(). This may lead to data corruption.

As of now this scenario does not seem to be possible, however with the
upcoming load_unaligned_zeropad() this is possible.

As a side effect PG_arch_1 is not cleared anymore for such faults via the
kernel mapping. However given that this bit is over-indicating it
shouldn't matter.

Tested-by: Christian Borntraeger <borntraeger@linux.ibm.com>
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
---
 arch/s390/mm/fault.c | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c
index e650cdfa2523..776549d55c94 100644
--- a/arch/s390/mm/fault.c
+++ b/arch/s390/mm/fault.c
@@ -440,13 +440,7 @@ void do_secure_storage_access(struct pt_regs *regs)
 	if (is_kernel_fault(regs)) {
 		if (is_vmalloc_addr((void *)addr))
 			return handle_fault_error_nolock(regs, 0);
-		folio = virt_to_folio((void *)addr);
-		if (unlikely(!folio_try_get(folio)))
-			return;
-		rc = uv_convert_from_secure(folio_to_phys(folio));
-		if (!rc)
-			clear_bit(PG_arch_1, &folio->flags.f);
-		folio_put(folio);
+		rc = uv_convert_from_secure(__pa(addr));
 		/*
 		 * There are some valid fixup types for kernel
 		 * accesses to donated secure memory. zeropad is one
-- 
2.53.0

[PATCH v4 6/8] s390/mm: Use handle_fault_error() in do_secure_storage_access()

[Heiko Carstens]

Similar to the normal exception handler use handle_fault_error() which
handles accesses from user space and kernel space differently.

This also addresses failing kernel accesses via e.g. copy_from_user()to
user space: instead of an endless loop, the fixup handling is performed.
Even though this could only happen if arch_make_folio_accessible() would
permanently fail, which is not realistic.

Therefore this is rather a cleanup patch.

Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
---
 arch/s390/mm/fault.c | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c
index 776549d55c94..2a9b9fb30d09 100644
--- a/arch/s390/mm/fault.c
+++ b/arch/s390/mm/fault.c
@@ -427,10 +427,8 @@ void do_secure_storage_access(struct pt_regs *regs)
 		 * was not supposed to do, e.g. branching into secure
 		 * memory. Trigger a segmentation fault.
 		 */
-		if (user_mode(regs)) {
-			send_sig(SIGSEGV, current, 0);
-			return;
-		}
+		if (user_mode(regs))
+			return handle_fault_error_nolock(regs, SEGV_ACCERR);
 		/*
 		 * The kernel should never run into this case and
 		 * there is no way out of this situation.
@@ -465,7 +463,7 @@ void do_secure_storage_access(struct pt_regs *regs)
 		folio_put(folio);
 		folio_walk_end(&fw, vma);
 		if (rc)
-			send_sig(SIGSEGV, current, 0);
+			return handle_fault_error(regs, SEGV_ACCERR);
 		mmap_read_unlock(mm);
 	}
 }
-- 
2.53.0

[PATCH v4 7/8] s390/mm: Use goto statement in do_secure_storage_access()

[Heiko Carstens]

Make do_secure_storage_access() look more like the do_exception()
by using a goto statement to get rid of an extra open coded
mmap_read_unlock() call.

Just cosmetic, no functional change.

Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
---
 arch/s390/mm/fault.c | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c
index 2a9b9fb30d09..666fe8e99480 100644
--- a/arch/s390/mm/fault.c
+++ b/arch/s390/mm/fault.c
@@ -453,10 +453,8 @@ void do_secure_storage_access(struct pt_regs *regs)
 		if (!vma)
 			return handle_fault_error_nolock(regs, SEGV_MAPERR);
 		folio = folio_walk_start(&fw, vma, addr, 0);
-		if (!folio) {
-			mmap_read_unlock(mm);
-			return;
-		}
+		if (!folio)
+			goto out;
 		/* arch_make_folio_accessible() needs a raised refcount. */
 		folio_get(folio);
 		rc = arch_make_folio_accessible(folio);
@@ -464,6 +462,7 @@ void do_secure_storage_access(struct pt_regs *regs)
 		folio_walk_end(&fw, vma);
 		if (rc)
 			return handle_fault_error(regs, SEGV_ACCERR);
+out:
 		mmap_read_unlock(mm);
 	}
 }
-- 
2.53.0

[PATCH v4 8/8] s390: Add support for DCACHE_WORD_ACCESS (again)

[Heiko Carstens]

Implement load_unaligned_zeropad() and enable DCACHE_WORD_ACCESS to
speed up string operations in fs/dcache.c and fs/namei.c.

With the secure storage access exception cases addressed, add support
for DCACHE_WORD_ACCESS again.

Tested-by: Christian Borntraeger <borntraeger@linux.ibm.com>
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
---
 arch/s390/Kconfig                      |  1 +
 arch/s390/include/asm/asm-extable.h    |  4 ++++
 arch/s390/include/asm/word-at-a-time.h | 22 ++++++++++++++++++++++
 arch/s390/mm/extable.c                 | 18 ++++++++++++++++++
 4 files changed, 45 insertions(+)

diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index 84404e6778d5..fe81c539ef2f 100644
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -163,6 +163,7 @@ config S390
 	select ARCH_WANTS_THP_SWAP
 	select BUILDTIME_TABLE_SORT
 	select CLONE_BACKWARDS2
+	select DCACHE_WORD_ACCESS if !KMSAN
 	select DYNAMIC_FTRACE if FUNCTION_TRACER
 	select FUNCTION_ALIGNMENT_8B if CC_IS_GCC
 	select FUNCTION_ALIGNMENT_16B if !CC_IS_GCC
diff --git a/arch/s390/include/asm/asm-extable.h b/arch/s390/include/asm/asm-extable.h
index 99748c20e767..d23ea0c94e4e 100644
--- a/arch/s390/include/asm/asm-extable.h
+++ b/arch/s390/include/asm/asm-extable.h
@@ -12,6 +12,7 @@
 #define EX_TYPE_UA_FAULT	3
 #define EX_TYPE_UA_LOAD_REG	5
 #define EX_TYPE_UA_LOAD_REGPAIR	6
+#define EX_TYPE_ZEROPAD		7
 #define EX_TYPE_FPC		8
 #define EX_TYPE_UA_MVCOS_TO	9
 #define EX_TYPE_UA_MVCOS_FROM	10
@@ -79,6 +80,9 @@
 #define EX_TABLE_UA_LOAD_REGPAIR(_fault, _target, _regerr, _regzero)	\
 	__EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_REGPAIR, _regerr, _regzero, 0)
 
+#define EX_TABLE_ZEROPAD(_fault, _target, _regdata, _regaddr)		\
+	__EX_TABLE(__ex_table, _fault, _target, EX_TYPE_ZEROPAD, _regdata, _regaddr, 0)
+
 #define EX_TABLE_FPC(_fault, _target)					\
 	__EX_TABLE(__ex_table, _fault, _target, EX_TYPE_FPC, __stringify(%%r0), __stringify(%%r0), 0)
 
diff --git a/arch/s390/include/asm/word-at-a-time.h b/arch/s390/include/asm/word-at-a-time.h
index e9287036392d..eaa19dee7699 100644
--- a/arch/s390/include/asm/word-at-a-time.h
+++ b/arch/s390/include/asm/word-at-a-time.h
@@ -4,6 +4,7 @@
 
 #include <linux/bitops.h>
 #include <linux/wordpart.h>
+#include <asm/asm-extable.h>
 #include <asm/bitsperlong.h>
 
 struct word_at_a_time {
@@ -40,4 +41,25 @@ static inline unsigned long zero_bytemask(unsigned long data)
 	return ~1UL << data;
 }
 
+/*
+ * Load an unaligned word from kernel space.
+ *
+ * In the (very unlikely) case of the word being a page-crosser
+ * and the next page not being mapped, take the exception and
+ * return zeroes in the non-existing part.
+ */
+static inline unsigned long load_unaligned_zeropad(const void *addr)
+{
+	unsigned long data;
+
+	asm_inline volatile(
+		"0:	lg	%[data],0(%[addr])\n"
+		"1:	nopr	%%r7\n"
+		EX_TABLE_ZEROPAD(0b, 1b, %[data], %[addr])
+		EX_TABLE_ZEROPAD(1b, 1b, %[data], %[addr])
+		: [data] "=d" (data)
+		: [addr] "a" (addr), "m" (*(unsigned long *)addr));
+	return data;
+}
+
 #endif /* _ASM_WORD_AT_A_TIME_H */
diff --git a/arch/s390/mm/extable.c b/arch/s390/mm/extable.c
index 063b4346742d..7498e858c401 100644
--- a/arch/s390/mm/extable.c
+++ b/arch/s390/mm/extable.c
@@ -50,6 +50,22 @@ static bool ex_handler_ua_load_reg(const struct exception_table_entry *ex,
 	return true;
 }
 
+static bool ex_handler_zeropad(const struct exception_table_entry *ex, struct pt_regs *regs)
+{
+	unsigned int reg_addr = FIELD_GET(EX_DATA_REG_ADDR, ex->data);
+	unsigned int reg_data = FIELD_GET(EX_DATA_REG_ERR, ex->data);
+	unsigned long data, addr, offset;
+
+	addr = regs->gprs[reg_addr];
+	offset = addr & (sizeof(unsigned long) - 1);
+	addr &= ~(sizeof(unsigned long) - 1);
+	data = *(unsigned long *)addr;
+	data <<= BITS_PER_BYTE * offset;
+	regs->gprs[reg_data] = data;
+	regs->psw.addr = extable_fixup(ex);
+	return true;
+}
+
 static bool ex_handler_fpc(const struct exception_table_entry *ex, struct pt_regs *regs)
 {
 	fpu_sfpc(0);
@@ -118,6 +134,8 @@ bool fixup_exception(struct pt_regs *regs)
 		return ex_handler_ua_load_reg(ex, false, regs);
 	case EX_TYPE_UA_LOAD_REGPAIR:
 		return ex_handler_ua_load_reg(ex, true, regs);
+	case EX_TYPE_ZEROPAD:
+		return ex_handler_zeropad(ex, regs);
 	case EX_TYPE_FPC:
 		return ex_handler_fpc(ex, regs);
 	case EX_TYPE_UA_MVCOS_TO:
-- 
2.53.0

Re: [PATCH v4 0/8] s390: Reintroduce support for DCACHE_WORD_ACCESS

[Christian Borntraeger]

Am 20.07.26 um 10:58 schrieb Heiko Carstens:
> v4:
> - Add two more cleanup patches based on Sashiko feedback, however I
>    don't consider them as bug fixes. This is more cosmetics:
>    - Use handle_fault_error() in do_secure_storage_access()
>    - Use goto statement in do_secure_storage_access()
Shall we run the kmemleak fix (for the initial memblock) and maybe additional
kmemleak changes for the other secure memory arady  on top of your patch set
or shall I run them independently?

Re: [PATCH v4 0/8] s390: Reintroduce support for DCACHE_WORD_ACCESS

[Heiko Carstens]

On Mon, Jul 20, 2026 at 11:03:01AM +0200, Christian Borntraeger wrote:
> Am 20.07.26 um 10:58 schrieb Heiko Carstens:
> > v4:
> > - Add two more cleanup patches based on Sashiko feedback, however I
> >    don't consider them as bug fixes. This is more cosmetics:
> >    - Use handle_fault_error() in do_secure_storage_access()
> >    - Use goto statement in do_secure_storage_access()
> Shall we run the kmemleak fix (for the initial memblock) and maybe additional
> kmemleak changes for the other secure memory arady  on top of your patch set
> or shall I run them independently?

Please handle that independently of this series.

After reading through all the Sashiko reports, at least (Sashiko) AI review is
now complete. All current findings are either false positives, describe
intended behavior, or are addressed with later patches in this series.

Now waiting for human review :)

Re: [PATCH v4 1/8] KVM: s390: pv: Use VM_SPARSE area for guest variable storage area

[Christian Borntraeger]

Am 20.07.26 um 10:58 schrieb Heiko Carstens:
> The guest variable storage area is allocated with vmalloc and then
> donated to the ultravisor. Any kernel access to that area will result
> in a secure storage access exception (aka fault).
> 
> This is a problem if such a memory area is read via /proc/kcore. This
> causes an exception via vread_iter() and results in an unexpected short
> read. Avoid this by allocating a custom VM_SPARSE area. If such an area
> is read, vread_iter() returns zeroes for the entire area.
> 
> Note that the function which frees the area does not update ptes. This
> is intentional to allow for deferred / lazy pte updates and TLB flushing
> like the generic vfree() code is doing that. See vunmap_pte_range().
> 
> This assumes that s390 will gain full support for lazy_mmu_mode_enable()
> and lazy_mmu_mode_disable() in the future, since as of now the used
> ptep_get_and_clear() in vunmap_pte_range() does indeed invalidate and
> flush every single pte entry, but only for s390.
> 
> Tested-by: Christian Borntraeger <borntraeger@linux.ibm.com>
> Signed-off-by: Heiko Carstens <hca@linux.ibm.com>

Reviewed-by: Christian Borntraeger <borntraeger@linux.ibm.com>


[..]> +static int uv_free_range_cb(pte_t *ptep, unsigned long addr, void *data)
> +{
> +	pte_t pte = ptep_get(ptep);
> +
> +	if (!pte_present(pte))
> +		return 0;
I guess we do not need it today, but it certainly does not hurt?

Re: [PATCH v4 1/8] KVM: s390: pv: Use VM_SPARSE area for guest variable storage area

[Heiko Carstens]

On Mon, Jul 20, 2026 at 11:56:36AM +0200, Christian Borntraeger wrote:
> [..]> +static int uv_free_range_cb(pte_t *ptep, unsigned long addr, void *data)
> > +{
> > +	pte_t pte = ptep_get(ptep);
> > +
> > +	if (!pte_present(pte))
> > +		return 0;
> I guess we do not need it today, but it certainly does not hurt?

It is not needed, just a sanity check.

Re: [PATCH v4 4/8] s390/mm: Fix handling of vmalloc area in do_secure_storage_access()

[Christian Borntraeger]

Am 20.07.26 um 10:58 schrieb Heiko Carstens:
> Secure storage access exceptions in vmalloc area need to be handled
> separately since folio_try_get() or virt_to_phys() do not work on such
> addresses. Given that there cannot be a valid access let such exceptions
> fail immediately. The kernel will either fixup or crash.
> 
> Tested-by: Christian Borntraeger <borntraeger@linux.ibm.com>
> Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
Reviewed-by: Christian Borntraeger <borntraeger@linux.ibm.com>> ---
>   arch/s390/mm/fault.c | 2 ++
>   1 file changed, 2 insertions(+)
> 
> diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c
> index f150d97cac2e..e650cdfa2523 100644
> --- a/arch/s390/mm/fault.c
> +++ b/arch/s390/mm/fault.c
> @@ -438,6 +438,8 @@ void do_secure_storage_access(struct pt_regs *regs)
>   		panic("Unexpected PGM 0x3d with TEID bit 61=0");
>   	}
>   	if (is_kernel_fault(regs)) {
> +		if (is_vmalloc_addr((void *)addr))
> +			return handle_fault_error_nolock(regs, 0);
>   		folio = virt_to_folio((void *)addr);
>   		if (unlikely(!folio_try_get(folio)))
>   			return;

Re: [PATCH v4 7/8] s390/mm: Use goto statement in do_secure_storage_access()

[Christian Borntraeger]

Am 20.07.26 um 10:58 schrieb Heiko Carstens:
> Make do_secure_storage_access() look more like the do_exception()
> by using a goto statement to get rid of an extra open coded
> mmap_read_unlock() call.
> 
> Just cosmetic, no functional change.
> 
> Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
Reviewed-by: Christian Borntraeger <borntraeger@linux.ibm.com>

> ---
>   arch/s390/mm/fault.c | 7 +++----
>   1 file changed, 3 insertions(+), 4 deletions(-)
> 
> diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c
> index 2a9b9fb30d09..666fe8e99480 100644
> --- a/arch/s390/mm/fault.c
> +++ b/arch/s390/mm/fault.c
> @@ -453,10 +453,8 @@ void do_secure_storage_access(struct pt_regs *regs)
>   		if (!vma)
>   			return handle_fault_error_nolock(regs, SEGV_MAPERR);
>   		folio = folio_walk_start(&fw, vma, addr, 0);
> -		if (!folio) {
> -			mmap_read_unlock(mm);
> -			return;
> -		}
> +		if (!folio)
> +			goto out;
>   		/* arch_make_folio_accessible() needs a raised refcount. */
>   		folio_get(folio);
>   		rc = arch_make_folio_accessible(folio);
> @@ -464,6 +462,7 @@ void do_secure_storage_access(struct pt_regs *regs)
>   		folio_walk_end(&fw, vma);
>   		if (rc)
>   			return handle_fault_error(regs, SEGV_ACCERR);
> +out:
>   		mmap_read_unlock(mm);
>   	}
>   }

Re: [PATCH v4 2/8] s390/mm: Add missing mm check to do_secure_storage_access()

[Christian Borntraeger]

Am 20.07.26 um 10:58 schrieb Heiko Carstens:
> Similar like the normal exception handler the secure storage access
> exception handler needs to verify if mm is not NULL, which may happen
> with kernel threads.
> 
> Reported-by: sashiko-bot <sashiko-bot@kernel.org>
> Closes: https://lore.kernel.org/all/20260717132345.539A11F00A3A@smtp.kernel.org/
> Signed-off-by: Heiko Carstens <hca@linux.ibm.com>

Reviewed-by: Christian Borntraeger <borntraeger@linux.ibm.com>> ---
>   arch/s390/mm/fault.c | 5 ++---
>   1 file changed, 2 insertions(+), 3 deletions(-)
> 
> diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c
> index 028aeb9c48d6..2839d7a64401 100644
> --- a/arch/s390/mm/fault.c
> +++ b/arch/s390/mm/fault.c
> @@ -409,9 +409,9 @@ void do_secure_storage_access(struct pt_regs *regs)
>   {
>   	union teid teid = { .val = regs->int_parm_long };
>   	unsigned long addr = get_fault_address(regs);
> +	struct mm_struct *mm = current->mm;
>   	struct vm_area_struct *vma;
>   	struct folio_walk fw;
> -	struct mm_struct *mm;
>   	struct folio *folio;
>   	int rc;
>   
> @@ -453,9 +453,8 @@ void do_secure_storage_access(struct pt_regs *regs)
>   		if (rc)
>   			return handle_fault_error_nolock(regs, 0);
>   	} else {
> -		if (faulthandler_disabled())
> +		if (faulthandler_disabled() || !mm)
>   			return handle_fault_error_nolock(regs, 0);
> -		mm = current->mm;
>   		mmap_read_lock(mm);
>   		vma = find_vma(mm, addr);
>   		if (!vma)

Re: [PATCH v4 3/8] s390/mm: Use lock_mm_and_find_vma() in do_secure_storage_access()

[Christian Borntraeger]

Am 20.07.26 um 10:58 schrieb Heiko Carstens:
> do_secure_storage_access() uses find_vma() without verifying that the
> faulting address is within the returned vma. Add this missing check by
> converting to lock_mm_and_find_vma().
> 
> This is not a critical fix, since the worst that could happen is a
> WARN_ON_ONCE() in folio_walk_start().
> 
> Reported-by: sashiko-bot <sashiko-bot@kernel.org>
> Closes: https://lore.kernel.org/all/20260717093904.E4A421F00A3E@smtp.kernel.org/
> Link: https://lore.kernel.org/all/20260717093904.E4A421F00A3E@smtp.kernel.org/
> Signed-off-by: Heiko Carstens <hca@linux.ibm.com>

Reviewed-by: Christian Borntraeger <borntraeger@linux.ibm.com>

> ---
>   arch/s390/mm/fault.c | 5 ++---
>   1 file changed, 2 insertions(+), 3 deletions(-)
> 
> diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c
> index 2839d7a64401..f150d97cac2e 100644
> --- a/arch/s390/mm/fault.c
> +++ b/arch/s390/mm/fault.c
> @@ -455,10 +455,9 @@ void do_secure_storage_access(struct pt_regs *regs)
>   	} else {
>   		if (faulthandler_disabled() || !mm)
>   			return handle_fault_error_nolock(regs, 0);
> -		mmap_read_lock(mm);
> -		vma = find_vma(mm, addr);
> +		vma = lock_mm_and_find_vma(mm, addr, regs);
>   		if (!vma)
> -			return handle_fault_error(regs, SEGV_MAPERR);
> +			return handle_fault_error_nolock(regs, SEGV_MAPERR);
>   		folio = folio_walk_start(&fw, vma, addr, 0);
>   		if (!folio) {
>   			mmap_read_unlock(mm);

Re: [PATCH v4 5/8] s390/mm: Remove folio handling for kernel faults in do_secure_storage_access()

[Christian Borntraeger]

Am 20.07.26 um 10:58 schrieb Heiko Carstens:
> Remove folio handling for secure storage access exceptions that happen
> via the kernel mapping. Using folio_try_get() for exceptions via the
> kernel mapping is racy: for example such an exception may happen on one
> CPU, while a different CPU frees the corresponding page. Subsequently the
> page can be part of a folio of different size, before the faulting CPU
> executes folio_try_get(). This may lead to data corruption.
> 
> As of now this scenario does not seem to be possible, however with the
> upcoming load_unaligned_zeropad() this is possible.
> 
> As a side effect PG_arch_1 is not cleared anymore for such faults via the
> kernel mapping. However given that this bit is over-indicating it
> shouldn't matter.
> 
> Tested-by: Christian Borntraeger <borntraeger@linux.ibm.com>
> Signed-off-by: Heiko Carstens <hca@linux.ibm.com>

Ideally Claudio should confirm that PG_arch_1 is over-indicating.

Reviewed-by: Christian Borntraeger <borntraeger@linux.ibm.com>

> ---
>   arch/s390/mm/fault.c | 8 +-------
>   1 file changed, 1 insertion(+), 7 deletions(-)
> 
> diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c
> index e650cdfa2523..776549d55c94 100644
> --- a/arch/s390/mm/fault.c
> +++ b/arch/s390/mm/fault.c
> @@ -440,13 +440,7 @@ void do_secure_storage_access(struct pt_regs *regs)
>   	if (is_kernel_fault(regs)) {
>   		if (is_vmalloc_addr((void *)addr))
>   			return handle_fault_error_nolock(regs, 0);
> -		folio = virt_to_folio((void *)addr);
> -		if (unlikely(!folio_try_get(folio)))
> -			return;
> -		rc = uv_convert_from_secure(folio_to_phys(folio));
> -		if (!rc)
> -			clear_bit(PG_arch_1, &folio->flags.f);
> -		folio_put(folio);
> +		rc = uv_convert_from_secure(__pa(addr));
>   		/*
>   		 * There are some valid fixup types for kernel
>   		 * accesses to donated secure memory. zeropad is one

Re: [PATCH v4 8/8] s390: Add support for DCACHE_WORD_ACCESS (again)

[Sven Schnelle]

Heiko Carstens <hca@linux.ibm.com> writes:

> Implement load_unaligned_zeropad() and enable DCACHE_WORD_ACCESS to
> speed up string operations in fs/dcache.c and fs/namei.c.
>
> With the secure storage access exception cases addressed, add support
> for DCACHE_WORD_ACCESS again.
>
> Tested-by: Christian Borntraeger <borntraeger@linux.ibm.com>
> Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
> ---
>  arch/s390/Kconfig                      |  1 +
>  arch/s390/include/asm/asm-extable.h    |  4 ++++
>  arch/s390/include/asm/word-at-a-time.h | 22 ++++++++++++++++++++++
>  arch/s390/mm/extable.c                 | 18 ++++++++++++++++++
>  4 files changed, 45 insertions(+)
>
> diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
> index 84404e6778d5..fe81c539ef2f 100644
> --- a/arch/s390/Kconfig
> +++ b/arch/s390/Kconfig
> @@ -163,6 +163,7 @@ config S390
>  	select ARCH_WANTS_THP_SWAP
>  	select BUILDTIME_TABLE_SORT
>  	select CLONE_BACKWARDS2
> +	select DCACHE_WORD_ACCESS if !KMSAN
>  	select DYNAMIC_FTRACE if FUNCTION_TRACER
>  	select FUNCTION_ALIGNMENT_8B if CC_IS_GCC
>  	select FUNCTION_ALIGNMENT_16B if !CC_IS_GCC
> diff --git a/arch/s390/include/asm/asm-extable.h b/arch/s390/include/asm/asm-extable.h
> index 99748c20e767..d23ea0c94e4e 100644
> --- a/arch/s390/include/asm/asm-extable.h
> +++ b/arch/s390/include/asm/asm-extable.h
> @@ -12,6 +12,7 @@
>  #define EX_TYPE_UA_FAULT	3
>  #define EX_TYPE_UA_LOAD_REG	5
>  #define EX_TYPE_UA_LOAD_REGPAIR	6
> +#define EX_TYPE_ZEROPAD		7
>  #define EX_TYPE_FPC		8
>  #define EX_TYPE_UA_MVCOS_TO	9
>  #define EX_TYPE_UA_MVCOS_FROM	10
> @@ -79,6 +80,9 @@
>  #define EX_TABLE_UA_LOAD_REGPAIR(_fault, _target, _regerr, _regzero)	\
>  	__EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_REGPAIR, _regerr, _regzero, 0)
>  
> +#define EX_TABLE_ZEROPAD(_fault, _target, _regdata, _regaddr)		\
> +	__EX_TABLE(__ex_table, _fault, _target, EX_TYPE_ZEROPAD, _regdata, _regaddr, 0)
> +
>  #define EX_TABLE_FPC(_fault, _target)					\
>  	__EX_TABLE(__ex_table, _fault, _target, EX_TYPE_FPC, __stringify(%%r0), __stringify(%%r0), 0)
>  
> diff --git a/arch/s390/include/asm/word-at-a-time.h b/arch/s390/include/asm/word-at-a-time.h
> index e9287036392d..eaa19dee7699 100644
> --- a/arch/s390/include/asm/word-at-a-time.h
> +++ b/arch/s390/include/asm/word-at-a-time.h
> @@ -4,6 +4,7 @@
>  
>  #include <linux/bitops.h>
>  #include <linux/wordpart.h>
> +#include <asm/asm-extable.h>
>  #include <asm/bitsperlong.h>
>  
>  struct word_at_a_time {
> @@ -40,4 +41,25 @@ static inline unsigned long zero_bytemask(unsigned long data)
>  	return ~1UL << data;
>  }
>  
> +/*
> + * Load an unaligned word from kernel space.
> + *
> + * In the (very unlikely) case of the word being a page-crosser
> + * and the next page not being mapped, take the exception and
> + * return zeroes in the non-existing part.
> + */
> +static inline unsigned long load_unaligned_zeropad(const void *addr)
> +{
> +	unsigned long data;
> +
> +	asm_inline volatile(
> +		"0:	lg	%[data],0(%[addr])\n"
> +		"1:	nopr	%%r7\n"
> +		EX_TABLE_ZEROPAD(0b, 1b, %[data], %[addr])
> +		EX_TABLE_ZEROPAD(1b, 1b, %[data], %[addr])
> +		: [data] "=d" (data)
> +		: [addr] "a" (addr), "m" (*(unsigned long *)addr));
> +	return data;
> +}
> +
>  #endif /* _ASM_WORD_AT_A_TIME_H */
> diff --git a/arch/s390/mm/extable.c b/arch/s390/mm/extable.c
> index 063b4346742d..7498e858c401 100644
> --- a/arch/s390/mm/extable.c
> +++ b/arch/s390/mm/extable.c
> @@ -50,6 +50,22 @@ static bool ex_handler_ua_load_reg(const struct exception_table_entry *ex,
>  	return true;
>  }
>  
> +static bool ex_handler_zeropad(const struct exception_table_entry *ex, struct pt_regs *regs)
> +{
> +	unsigned int reg_addr = FIELD_GET(EX_DATA_REG_ADDR, ex->data);
> +	unsigned int reg_data = FIELD_GET(EX_DATA_REG_ERR, ex->data);
> +	unsigned long data, addr, offset;
> +
> +	addr = regs->gprs[reg_addr];
> +	offset = addr & (sizeof(unsigned long) - 1);
> +	addr &= ~(sizeof(unsigned long) - 1);
> +	data = *(unsigned long *)addr;
> +	data <<= BITS_PER_BYTE * offset;
> +	regs->gprs[reg_data] = data;
> +	regs->psw.addr = extable_fixup(ex);
> +	return true;
> +}
> +
>  static bool ex_handler_fpc(const struct exception_table_entry *ex, struct pt_regs *regs)
>  {
>  	fpu_sfpc(0);
> @@ -118,6 +134,8 @@ bool fixup_exception(struct pt_regs *regs)
>  		return ex_handler_ua_load_reg(ex, false, regs);
>  	case EX_TYPE_UA_LOAD_REGPAIR:
>  		return ex_handler_ua_load_reg(ex, true, regs);
> +	case EX_TYPE_ZEROPAD:
> +		return ex_handler_zeropad(ex, regs);
>  	case EX_TYPE_FPC:
>  		return ex_handler_fpc(ex, regs);
>  	case EX_TYPE_UA_MVCOS_TO:

Reviewed-by: Sven Schnelle <svens@linux.ibm.com>