Attempting to unify some user space copying APIs

Attempting to unify some user space copying APIs

[Linus Torvalds]

[-- Attachment #1: Type: text/plain, Size: 2822 bytes --]

I'm sending this out to the architecture maintainer list (and a random
collection of arch maintainers that I just took from "recent pull
requests") in the hope that people can test this on platforms that I'm
too incompetent to check myself (read: I haven't installed all the
cross-compiler environments).

The impetus for this was trying to have some shared code in
lib/iov_iter.c by having a common wrapper function for a couple of
user copy functions that most architectures don't even have. But when
doing so I noticed that even the common function that we all *do* have
doesn't have a consistent calling convention: raw_copy_from_user().

So the compiler ended up being unhappy just because I want to pass a
function pointer around, and different architectures had different
types for the functions (and in the case of arm64, used a macro to
implement it, which really didn't work for this).

So I fixed it all up, and I think the end result looks better, but I'd
like people to at least build-test this on the platforms that I didn't
have immediate access to. I've tested this on arm64, x86-64 and i386
(and then I made changes, and didn't necessarily test all the changes
on all plaforms, but it LooksGoodToMe(tm), and some earlier version of
this patch did work on those architectures, so it must be perfect).

The patch was generated on current tip-of-the-day, but I don't think
any of this has changed in a while, so it should apply and be testable
pretty much on any half-way modern code base.

I fixed up powerpc blindly (we really had very odd types for
__copy_from_user_flushcache), and as mentioned the arm64 and x86 side
should be tested. Everything else lacks the flushcache and nocache
versions of the user copy functions, and it *looks* like only arm64
had that odd pattern for raw_copy_from_user().

But maybe my grep missed some case... Considering that three
architectures needed some header file fixups, let's make sure it's
_only_ those three, ok?

My initial version tried to use 'size_t', but the type of that is
typically - but not always - 'unsigned int' on 32-bit architectures.
And since I didn't want to mess with this low-level architecture
function more than necessary, the traditional 'unsigned long' it is
and most architectures should JustWork(tm).

[ Traditional, that is, except for those cache control versions, which
 just used completely random types with wrong sizes and signs and
everything.

  Not that we even allow IO that big in the first place, but it was
still a 'that shouldn't even work' kind of reaction from me ]

Anyway, could I please ask architecture maintainers to check that
there isn't some odd corner case that I missed? I could put this in
linux-next, but even before I do that, I'd like to do the basic "it
won't break anything" smell test.

             Linus

[-- Attachment #2: patch.diff --]
[-- Type: text/x-patch, Size: 7879 bytes --]

 arch/arm64/include/asm/uaccess.h   | 36 ++++++++++++++----------------
 arch/powerpc/include/asm/uaccess.h |  4 ++--
 arch/powerpc/lib/pmem.c            |  4 ++--
 arch/x86/include/asm/uaccess_64.h  | 13 +++++------
 arch/x86/lib/usercopy_64.c         |  2 +-
 lib/iov_iter.c                     | 19 +++++++++++-----
 6 files changed, 42 insertions(+), 36 deletions(-)

diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h
index 9810106a3f66..2d36a0c4ec6a 100644
--- a/arch/arm64/include/asm/uaccess.h
+++ b/arch/arm64/include/asm/uaccess.h
@@ -390,26 +390,24 @@ do {									\
 } while(0)
 
 extern unsigned long __must_check __arch_copy_from_user(void *to, const void __user *from, unsigned long n);
-#define raw_copy_from_user(to, from, n)					\
-({									\
-	unsigned long __acfu_ret;					\
-	uaccess_ttbr0_enable();						\
-	__acfu_ret = __arch_copy_from_user((to),			\
-				      __uaccess_mask_ptr(from), (n));	\
-	uaccess_ttbr0_disable();					\
-	__acfu_ret;							\
-})
+static __must_check __always_inline unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n)
+{
+	unsigned long ret;
+	uaccess_ttbr0_enable();
+	ret = __arch_copy_from_user(to, __uaccess_mask_ptr(from), n);
+	uaccess_ttbr0_disable();
+	return ret;
+}
 
 extern unsigned long __must_check __arch_copy_to_user(void __user *to, const void *from, unsigned long n);
-#define raw_copy_to_user(to, from, n)					\
-({									\
-	unsigned long __actu_ret;					\
-	uaccess_ttbr0_enable();						\
-	__actu_ret = __arch_copy_to_user(__uaccess_mask_ptr(to),	\
-				    (from), (n));			\
-	uaccess_ttbr0_disable();					\
-	__actu_ret;							\
-})
+static __must_check __always_inline unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n)
+{
+	unsigned long ret;
+	uaccess_ttbr0_enable();
+	ret = __arch_copy_to_user(__uaccess_mask_ptr(to), from, n);
+	uaccess_ttbr0_disable();
+	return ret;
+}
 
 static __must_check __always_inline bool user_access_begin(const void __user *ptr, size_t len)
 {
@@ -478,7 +476,7 @@ extern __must_check long strnlen_user(const char __user *str, long n);
 #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
 extern unsigned long __must_check __copy_user_flushcache(void *to, const void __user *from, unsigned long n);
 
-static inline int __copy_from_user_flushcache(void *dst, const void __user *src, unsigned size)
+static inline unsigned long __copy_from_user_flushcache(void *dst, const void __user *src, unsigned long size)
 {
 	kasan_check_write(dst, size);
 	return __copy_user_flushcache(dst, __uaccess_mask_ptr(src), size);
diff --git a/arch/powerpc/include/asm/uaccess.h b/arch/powerpc/include/asm/uaccess.h
index 17e63244e885..8ae10aa5de63 100644
--- a/arch/powerpc/include/asm/uaccess.h
+++ b/arch/powerpc/include/asm/uaccess.h
@@ -434,8 +434,8 @@ copy_mc_to_user(void __user *to, const void *from, unsigned long n)
 }
 #endif
 
-extern long __copy_from_user_flushcache(void *dst, const void __user *src,
-		unsigned size);
+extern unsigned long __copy_from_user_flushcache(void *dst, const void __user *src,
+		unsigned long size);
 
 static __must_check __always_inline bool __user_access_begin(const void __user *ptr, size_t len,
 							     unsigned long dir)
diff --git a/arch/powerpc/lib/pmem.c b/arch/powerpc/lib/pmem.c
index 4e724c4c01ad..2e733ea9de3a 100644
--- a/arch/powerpc/lib/pmem.c
+++ b/arch/powerpc/lib/pmem.c
@@ -66,8 +66,8 @@ EXPORT_SYMBOL_GPL(arch_invalidate_pmem);
 /*
  * CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE symbols
  */
-long __copy_from_user_flushcache(void *dest, const void __user *src,
-		unsigned size)
+unsigned long __copy_from_user_flushcache(void *dest, const void __user *src,
+		unsigned long size)
 {
 	unsigned long copied, start = (unsigned long) dest;
 
diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h
index 915124011c27..6c369481174a 100644
--- a/arch/x86/include/asm/uaccess_64.h
+++ b/arch/x86/include/asm/uaccess_64.h
@@ -147,12 +147,11 @@ raw_copy_to_user(void __user *dst, const void *src, unsigned long size)
 	return copy_user_generic((__force void *)dst, src, size);
 }
 
-extern long __copy_user_nocache(void *dst, const void __user *src, unsigned size);
-extern long __copy_user_flushcache(void *dst, const void __user *src, unsigned size);
+extern unsigned long __copy_user_nocache(void *dst, const void __user *src, unsigned long size);
+extern unsigned long __copy_user_flushcache(void *dst, const void __user *src, unsigned long size);
 
-static inline int
-__copy_from_user_inatomic_nocache(void *dst, const void __user *src,
-				  unsigned size)
+static inline unsigned long
+__copy_from_user_inatomic_nocache(void *dst, const void __user *src, unsigned long size)
 {
 	long ret;
 	kasan_check_write(dst, size);
@@ -162,8 +161,8 @@ __copy_from_user_inatomic_nocache(void *dst, const void __user *src,
 	return ret;
 }
 
-static inline int
-__copy_from_user_flushcache(void *dst, const void __user *src, unsigned size)
+static inline unsigned long
+__copy_from_user_flushcache(void *dst, const void __user *src, unsigned long size)
 {
 	kasan_check_write(dst, size);
 	return __copy_user_flushcache(dst, src, size);
diff --git a/arch/x86/lib/usercopy_64.c b/arch/x86/lib/usercopy_64.c
index 654280aaa3e9..10b0026e85b7 100644
--- a/arch/x86/lib/usercopy_64.c
+++ b/arch/x86/lib/usercopy_64.c
@@ -43,7 +43,7 @@ void arch_wb_cache_pmem(void *addr, size_t size)
 }
 EXPORT_SYMBOL_GPL(arch_wb_cache_pmem);
 
-long __copy_user_flushcache(void *dst, const void __user *src, unsigned size)
+unsigned long __copy_user_flushcache(void *dst, const void __user *src, unsigned long size)
 {
 	unsigned long flushed, dest = (unsigned long) dst;
 	long rc;
diff --git a/lib/iov_iter.c b/lib/iov_iter.c
index 0a63c7fba313..9c8fe94c72b4 100644
--- a/lib/iov_iter.c
+++ b/lib/iov_iter.c
@@ -41,9 +41,11 @@ size_t copy_to_user_iter_nofault(void __user *iter_to, size_t progress,
 	return res < 0 ? len : res;
 }
 
+typedef unsigned long (*from_user_copy_t)(void *, const void __user *, unsigned long);
+
 static __always_inline
-size_t copy_from_user_iter(void __user *iter_from, size_t progress,
-			   size_t len, void *to, void *priv2)
+size_t copy_from_user_iter_boilerplate(void __user *iter_from, size_t progress,
+			   size_t len, void *to, from_user_copy_t actor)
 {
 	size_t res = len;
 
@@ -64,12 +66,19 @@ size_t copy_from_user_iter(void __user *iter_from, size_t progress,
 	}
 	to += progress;
 	instrument_copy_from_user_before(to, iter_from, len);
-	res = raw_copy_from_user(to, iter_from, len);
+	res = actor(to, iter_from, len);
 	instrument_copy_from_user_after(to, iter_from, len, res);
 
 	return res;
 }
 
+static __always_inline
+size_t copy_from_user_iter(void __user *iter_from, size_t progress,
+			   size_t len, void *to, void *priv2)
+{
+	return copy_from_user_iter_boilerplate(iter_from, progress, len, to, raw_copy_from_user);
+}
+
 static __always_inline
 size_t memcpy_to_iter(void *iter_to, size_t progress,
 		      size_t len, void *from, void *priv2)
@@ -277,7 +286,7 @@ static __always_inline
 size_t copy_from_user_iter_nocache(void __user *iter_from, size_t progress,
 				   size_t len, void *to, void *priv2)
 {
-	return __copy_from_user_inatomic_nocache(to + progress, iter_from, len);
+	return copy_from_user_iter_boilerplate(iter_from, progress, len, to, __copy_from_user_inatomic_nocache);
 }
 
 size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
@@ -296,7 +305,7 @@ static __always_inline
 size_t copy_from_user_iter_flushcache(void __user *iter_from, size_t progress,
 				      size_t len, void *to, void *priv2)
 {
-	return __copy_from_user_flushcache(to + progress, iter_from, len);
+	return copy_from_user_iter_boilerplate(iter_from, progress, len, to, __copy_from_user_flushcache);
 }
 
 static __always_inline
-- 
2.53.0.479.g83c58c3406

Re: Attempting to unify some user space copying APIs

[Helge Deller]

On 3/28/26 01:20, Linus Torvalds wrote:
> I'm sending this out to the architecture maintainer list (and a random
> collection of arch maintainers that I just took from "recent pull
> requests") in the hope that people can test this on platforms that I'm
> too incompetent to check myself (read: I haven't installed all the
> cross-compiler environments).

Successfully compile- and run-tested on 32- & 64bit parisc.
Patch works as-is.

Helge

Re: Attempting to unify some user space copying APIs

[Vasily Gorbik]

On Fri, Mar 27, 2026 at 05:20:57PM -0700, Linus Torvalds wrote:
> I'm sending this out to the architecture maintainer list (and a random
> collection of arch maintainers that I just took from "recent pull
> requests") in the hope that people can test this on platforms that I'm
> too incompetent to check myself (read: I haven't installed all the
> cross-compiler environments).
...
>  arch/arm64/include/asm/uaccess.h   | 36 ++++++++++++++----------------
>  arch/powerpc/include/asm/uaccess.h |  4 ++--
>  arch/powerpc/lib/pmem.c            |  4 ++--
>  arch/x86/include/asm/uaccess_64.h  | 13 +++++------
>  arch/x86/lib/usercopy_64.c         |  2 +-
>  lib/iov_iter.c                     | 19 +++++++++++-----
>  6 files changed, 42 insertions(+), 36 deletions(-)

On s390 this builds and runs fine.

One thing I noticed: copy_from_user_iter is clean - the boilerplate just
wraps raw_copy_from_user which has no instrumentation of its own.

copy_from_user_iter_nocache is a bit different: the boilerplate wraps
the actor with instrument_copy_from_user_before/after, but the nocache
actors already instrument internally.

On s390 (and all arches without ARCH_HAS_NOCACHE_UACCESS) the generic
__copy_from_user_inatomic_nocache calls __copy_from_user_inatomic
which does its own instrument_copy_from_user_before/after around
raw_copy_from_user, so we end up with double kasan/kcsan/kmsan calls.

On x86 __copy_from_user_inatomic_nocache only does a bare
kasan_check_write(), so that one gets doubled, while kcsan and kmsan are
newly added through the boilerplate.

I only looked at s390 and compared to x86.

Re: Attempting to unify some user space copying APIs

[Huacai Chen]

On Sat, Mar 28, 2026 at 8:21 AM Linus Torvalds
<torvalds@linux-foundation.org> wrote:
>
> I'm sending this out to the architecture maintainer list (and a random
> collection of arch maintainers that I just took from "recent pull
> requests") in the hope that people can test this on platforms that I'm
> too incompetent to check myself (read: I haven't installed all the
> cross-compiler environments).
>
> The impetus for this was trying to have some shared code in
> lib/iov_iter.c by having a common wrapper function for a couple of
> user copy functions that most architectures don't even have. But when
> doing so I noticed that even the common function that we all *do* have
> doesn't have a consistent calling convention: raw_copy_from_user().
>
> So the compiler ended up being unhappy just because I want to pass a
> function pointer around, and different architectures had different
> types for the functions (and in the case of arm64, used a macro to
> implement it, which really didn't work for this).
>
> So I fixed it all up, and I think the end result looks better, but I'd
> like people to at least build-test this on the platforms that I didn't
> have immediate access to. I've tested this on arm64, x86-64 and i386
> (and then I made changes, and didn't necessarily test all the changes
> on all plaforms, but it LooksGoodToMe(tm), and some earlier version of
> this patch did work on those architectures, so it must be perfect).
>
> The patch was generated on current tip-of-the-day, but I don't think
> any of this has changed in a while, so it should apply and be testable
> pretty much on any half-way modern code base.
>
> I fixed up powerpc blindly (we really had very odd types for
> __copy_from_user_flushcache), and as mentioned the arm64 and x86 side
> should be tested. Everything else lacks the flushcache and nocache
> versions of the user copy functions, and it *looks* like only arm64
> had that odd pattern for raw_copy_from_user().
>
> But maybe my grep missed some case... Considering that three
> architectures needed some header file fixups, let's make sure it's
> _only_ those three, ok?
>
> My initial version tried to use 'size_t', but the type of that is
> typically - but not always - 'unsigned int' on 32-bit architectures.
> And since I didn't want to mess with this low-level architecture
> function more than necessary, the traditional 'unsigned long' it is
> and most architectures should JustWork(tm).
>
> [ Traditional, that is, except for those cache control versions, which
>  just used completely random types with wrong sizes and signs and
> everything.
>
>   Not that we even allow IO that big in the first place, but it was
> still a 'that shouldn't even work' kind of reaction from me ]
>
> Anyway, could I please ask architecture maintainers to check that
> there isn't some odd corner case that I missed? I could put this in
> linux-next, but even before I do that, I'd like to do the basic "it
> won't break anything" smell test.
Successfully compile- and run-tested on LoongArch.

Huacai

>
>              Linus

Re: Attempting to unify some user space copying APIs

[Madhavan Srinivasan]

On 3/28/26 5:50 AM, Linus Torvalds wrote:
> I'm sending this out to the architecture maintainer list (and a random
> collection of arch maintainers that I just took from "recent pull
> requests") in the hope that people can test this on platforms that I'm
> too incompetent to check myself (read: I haven't installed all the
> cross-compiler environments).
>
> The impetus for this was trying to have some shared code in
> lib/iov_iter.c by having a common wrapper function for a couple of
> user copy functions that most architectures don't even have. But when
> doing so I noticed that even the common function that we all *do* have
> doesn't have a consistent calling convention: raw_copy_from_user().
>
> So the compiler ended up being unhappy just because I want to pass a
> function pointer around, and different architectures had different
> types for the functions (and in the case of arm64, used a macro to
> implement it, which really didn't work for this).
>
> So I fixed it all up, and I think the end result looks better, but I'd
> like people to at least build-test this on the platforms that I didn't
> have immediate access to. I've tested this on arm64, x86-64 and i386
> (and then I made changes, and didn't necessarily test all the changes
> on all plaforms, but it LooksGoodToMe(tm), and some earlier version of
> this patch did work on those architectures, so it must be perfect).
>
> The patch was generated on current tip-of-the-day, but I don't think
> any of this has changed in a while, so it should apply and be testable
> pretty much on any half-way modern code base.
>
> I fixed up powerpc blindly (we really had very odd types for
> __copy_from_user_flushcache), and as mentioned the arm64 and x86 side
> should be tested. Everything else lacks the flushcache and nocache
> versions of the user copy functions, and it *looks* like only arm64
> had that odd pattern for raw_copy_from_user().

Compile- and boot tested for powerpc.

We also did a quick basic test via fsdax pmem device (qemu+TCG)

             bash-1759    [002] .....   532.873958: fc: (__copy_from_user_flushcache+0xc/0x3a4) dest=0xc0000005027a0000 src=0x132c01400 size=0xd
             bash-1759    [002] .....   532.874021: <stack trace>
  => __copy_from_user_flushcache
  => _copy_from_iter_flushcache
  => dax_copy_from_iter
  => dax_iomap_rw
  => ext4_file_write_iter
  => vfs_write
  => ksys_write
  => system_call_exception
  => system_call_vectored_common
root@buildroot:/#

Patch works as-is.

Maddy
>
> But maybe my grep missed some case... Considering that three
> architectures needed some header file fixups, let's make sure it's
> _only_ those three, ok?
>
> My initial version tried to use 'size_t', but the type of that is
> typically - but not always - 'unsigned int' on 32-bit architectures.
> And since I didn't want to mess with this low-level architecture
> function more than necessary, the traditional 'unsigned long' it is
> and most architectures should JustWork(tm).
>
> [ Traditional, that is, except for those cache control versions, which
>   just used completely random types with wrong sizes and signs and
> everything.
>
>    Not that we even allow IO that big in the first place, but it was
> still a 'that shouldn't even work' kind of reaction from me ]
>
> Anyway, could I please ask architecture maintainers to check that
> there isn't some odd corner case that I missed? I could put this in
> linux-next, but even before I do that, I'd like to do the basic "it
> won't break anything" smell test.
>
>               Linus

Re: Attempting to unify some user space copying APIs

[Linus Torvalds]

On Fri, 27 Mar 2026 at 21:14, Vasily Gorbik <gor@linux.ibm.com> wrote:
>
> One thing I noticed: copy_from_user_iter is clean - the boilerplate just
> wraps raw_copy_from_user which has no instrumentation of its own.
>
> copy_from_user_iter_nocache is a bit different: the boilerplate wraps
> the actor with instrument_copy_from_user_before/after, but the nocache
> actors already instrument internally.

Good catch. I only looked at the iov_iter side and reacted to the
"that's not doing all the required steps", without noticing that we
had added that instrumentation separately to the nocache actor (well,
the __copy_from_user_inatomic() wrapper), but not to the
__copy_from_user_flushcache() one.

And as you say, it's all also very inconsistent wrt kasan/kmsan/kcsan.

So the whole inconsistency goes even deeper.

Gaah. What a pain. Following all the randomly named wrapper functions
that do slightly different things on different architectures is a
nightmare. Thanks for noticing.

This only makes me more convinced we need to make this all more
consistent, but clearly I need to think about this some more.

I suspect I should make the consistency be about naming too, and
introduce "raw_copy_from_user_inatomic()" and
"raw_copy_from_user_flushcache()", which do just that part - matching
our raw_copy_from_user() model (which in turn evolved from many years
of crazy inconsistency in the *basic* user access code, with varying
amounts of underscores etc).

The good news being that the inatomic and flushcache cases aren't
actually implemented by most architectures, so that cleanup should be
minor.

Again, thanks for noticing, and I have to go back and improve on that
patch. But it looks like it's _almost_ good.

                 Linus

Re: Attempting to unify some user space copying APIs

[Linus Torvalds]

On Sat, 28 Mar 2026 at 09:46, Linus Torvalds
<torvalds@linux-foundation.org> wrote:
>
> I suspect I should make the consistency be about naming too, and
> introduce "raw_copy_from_user_inatomic()" and
> "raw_copy_from_user_flushcache()"

There's another strange inconsistency here: the "nocache" iov_iter
case actually uses the name "inatomic_nocache", but the "inatomic"
part of it seems to be purely a historical implementation accident (ie
it started out as a helper function that could also be used in
inatomic context, but here the "inatomic" part seems not actually
relevant).

The only user of copy_from_user_iter_nocache() is
_copy_from_iter_nocache(), which has that underscore to avoid the
hardened-usercopy check_copy_size() overhead on user copy.

And nobody actually appears to use that "avoid hardned copy"
_copy_from_iter_nocache() function directly, it's only used as the
fallback for _copy_from_iter_flushcache() for all architectures that
don't implement it natively:

  #define _copy_from_iter_flushcache _copy_from_iter_nocache

with only x86 having an actual distinction between "flushcache" and "nocache".

So in reality we essentially have just two cases:
copy_from_iter_nocache() that *does* do the copy source hardening
check, and then we have _copy_from_iter_flushcache() that avoids it,
and on x86 does something subtly different than on all other
architectures.

And all of this completely crazy ad-hoc stuff for iovec appear to have
all of *seven* users in the whole kernel:

 - networking has two users of copy_from_iter_nocache(): skmsg and tls_device

 - networking also has two users of "copy_from_iter_full_nocache()"
which is another wrapper around copy_from_iter_nocache (which
basically walks back the source if it couldn't copy everything)

 - tracing has one copy_from_iter_nocache

 - dax_copy_from_iter() uses _copy_from_iter_flushcache()

 - dm-pcache also uses _copy_from_iter_flushcache(), but it doesn't
actually care about the user space case at all, having created the
iter from a bvec with iov_iter_bvec()

 - nvdimm uses _copy_from_iter_flushcache

but it's possible that I missed some other case that uses some other
indirection wrapper that I didn't notice.

I'm starting to seriously regret ever looking at this.

               Linus

Re: Attempting to unify some user space copying APIs

[Linus Torvalds]

[-- Attachment #1: Type: text/plain, Size: 934 bytes --]

On Fri, 27 Mar 2026 at 21:14, Vasily Gorbik <gor@linux.ibm.com> wrote:
>
> On s390 (and all arches without ARCH_HAS_NOCACHE_UACCESS) the generic
> __copy_from_user_inatomic_nocache calls __copy_from_user_inatomic
> which does its own instrument_copy_from_user_before/after around
> raw_copy_from_user, so we end up with double kasan/kcsan/kmsan calls.

You also end up with a completely pointless duplication of
_copy_from_iter_nocache, which really is just the regular
_copy_from_iter, since there's no "nocache".

And the "inatomic" part looks like an accidental implementation issue,
not actually meaningful semantically or something anybody cares about.

And the "flushcache" version is even more pointless.

Oh well. I walked away from this all in disgust, and then came back
and tried to make sense of it all to myself. I'm not sure I succeeded,
but this is what I have in my private test-tree right now...

              Linus

[-- Attachment #2: 0001-iov_iter-make-the-specialty-user-copies-match-the-st.patch --]
[-- Type: text/x-patch, Size: 9867 bytes --]

From 184dceace996af5ed1c9c818050eb21a897c24ba Mon Sep 17 00:00:00 2001
From: Linus Torvalds <torvalds@linux-foundation.org>
Date: Fri, 27 Mar 2026 12:12:16 -0700
Subject: [PATCH 1/3] iov_iter: make the specialty user copies match the
 standard one

Jann Horn pointed out that the 'nocache' and 'flushcache' iterators
never got modernized.  Al did the basic checks on the normal iterators
almost a decade ago, and they have been updated since to do various
modern user copy things like instrumenting user copies better, and using
masked user accesses.

But the 'nocache' and 'flushcache' cases have been left to rot, because
they aren't actually used by any normal code paths.

Out of sight, out of mind.

Let's just use a generic wrapper to do all this on the 'copy from user
space' side, to avoid missing the unusual cases or duplicating the
logic.

Using that wrapper with a "user copy actor" function pointer made it
clear that we've also never unified the exact function prototypes for
these specialty user-copy functions, so then then required some cleanup
in the architecture uaccess.h files to unify it all.

Note that this only does the "copy from user" cases.  Copying the other
way has some similar issues, but in different - and less messy - ways.
Instead of "nocache" and "cacheflush", that side has "nofault" and "mc"
cases, but they both have proper iterator functions that have been
mostly been kept up-to-date over the years.

So unlike the from-user case, the only thing missing on the to-user side
seems to be an instrument_copy_to_user() call for the nofault case.  And
that might even be intentional - the nofault case only exists for
"copy_page_to_iter_nofault()", so it's fundamentally different in other
cases too.

So I'm leaving that alone for now, but it might be worth looking at.

Reported-by: Jann Horn <jannh@google.com>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: Al Viro <viro@zeniv.linux.org.uk>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
---
 arch/arm64/include/asm/uaccess.h   | 36 ++++++++++++++----------------
 arch/powerpc/include/asm/uaccess.h |  4 ++--
 arch/powerpc/lib/pmem.c            |  4 ++--
 arch/x86/include/asm/uaccess_64.h  | 13 +++++------
 arch/x86/lib/usercopy_64.c         |  2 +-
 lib/iov_iter.c                     | 19 +++++++++++-----
 6 files changed, 42 insertions(+), 36 deletions(-)

diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h
index 9810106a3f66..2d36a0c4ec6a 100644
--- a/arch/arm64/include/asm/uaccess.h
+++ b/arch/arm64/include/asm/uaccess.h
@@ -390,26 +390,24 @@ do {									\
 } while(0)
 
 extern unsigned long __must_check __arch_copy_from_user(void *to, const void __user *from, unsigned long n);
-#define raw_copy_from_user(to, from, n)					\
-({									\
-	unsigned long __acfu_ret;					\
-	uaccess_ttbr0_enable();						\
-	__acfu_ret = __arch_copy_from_user((to),			\
-				      __uaccess_mask_ptr(from), (n));	\
-	uaccess_ttbr0_disable();					\
-	__acfu_ret;							\
-})
+static __must_check __always_inline unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n)
+{
+	unsigned long ret;
+	uaccess_ttbr0_enable();
+	ret = __arch_copy_from_user(to, __uaccess_mask_ptr(from), n);
+	uaccess_ttbr0_disable();
+	return ret;
+}
 
 extern unsigned long __must_check __arch_copy_to_user(void __user *to, const void *from, unsigned long n);
-#define raw_copy_to_user(to, from, n)					\
-({									\
-	unsigned long __actu_ret;					\
-	uaccess_ttbr0_enable();						\
-	__actu_ret = __arch_copy_to_user(__uaccess_mask_ptr(to),	\
-				    (from), (n));			\
-	uaccess_ttbr0_disable();					\
-	__actu_ret;							\
-})
+static __must_check __always_inline unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n)
+{
+	unsigned long ret;
+	uaccess_ttbr0_enable();
+	ret = __arch_copy_to_user(__uaccess_mask_ptr(to), from, n);
+	uaccess_ttbr0_disable();
+	return ret;
+}
 
 static __must_check __always_inline bool user_access_begin(const void __user *ptr, size_t len)
 {
@@ -478,7 +476,7 @@ extern __must_check long strnlen_user(const char __user *str, long n);
 #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
 extern unsigned long __must_check __copy_user_flushcache(void *to, const void __user *from, unsigned long n);
 
-static inline int __copy_from_user_flushcache(void *dst, const void __user *src, unsigned size)
+static inline unsigned long __copy_from_user_flushcache(void *dst, const void __user *src, unsigned long size)
 {
 	kasan_check_write(dst, size);
 	return __copy_user_flushcache(dst, __uaccess_mask_ptr(src), size);
diff --git a/arch/powerpc/include/asm/uaccess.h b/arch/powerpc/include/asm/uaccess.h
index 17e63244e885..8ae10aa5de63 100644
--- a/arch/powerpc/include/asm/uaccess.h
+++ b/arch/powerpc/include/asm/uaccess.h
@@ -434,8 +434,8 @@ copy_mc_to_user(void __user *to, const void *from, unsigned long n)
 }
 #endif
 
-extern long __copy_from_user_flushcache(void *dst, const void __user *src,
-		unsigned size);
+extern unsigned long __copy_from_user_flushcache(void *dst, const void __user *src,
+		unsigned long size);
 
 static __must_check __always_inline bool __user_access_begin(const void __user *ptr, size_t len,
 							     unsigned long dir)
diff --git a/arch/powerpc/lib/pmem.c b/arch/powerpc/lib/pmem.c
index 4e724c4c01ad..2e733ea9de3a 100644
--- a/arch/powerpc/lib/pmem.c
+++ b/arch/powerpc/lib/pmem.c
@@ -66,8 +66,8 @@ EXPORT_SYMBOL_GPL(arch_invalidate_pmem);
 /*
  * CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE symbols
  */
-long __copy_from_user_flushcache(void *dest, const void __user *src,
-		unsigned size)
+unsigned long __copy_from_user_flushcache(void *dest, const void __user *src,
+		unsigned long size)
 {
 	unsigned long copied, start = (unsigned long) dest;
 
diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h
index 915124011c27..6c369481174a 100644
--- a/arch/x86/include/asm/uaccess_64.h
+++ b/arch/x86/include/asm/uaccess_64.h
@@ -147,12 +147,11 @@ raw_copy_to_user(void __user *dst, const void *src, unsigned long size)
 	return copy_user_generic((__force void *)dst, src, size);
 }
 
-extern long __copy_user_nocache(void *dst, const void __user *src, unsigned size);
-extern long __copy_user_flushcache(void *dst, const void __user *src, unsigned size);
+extern unsigned long __copy_user_nocache(void *dst, const void __user *src, unsigned long size);
+extern unsigned long __copy_user_flushcache(void *dst, const void __user *src, unsigned long size);
 
-static inline int
-__copy_from_user_inatomic_nocache(void *dst, const void __user *src,
-				  unsigned size)
+static inline unsigned long
+__copy_from_user_inatomic_nocache(void *dst, const void __user *src, unsigned long size)
 {
 	long ret;
 	kasan_check_write(dst, size);
@@ -162,8 +161,8 @@ __copy_from_user_inatomic_nocache(void *dst, const void __user *src,
 	return ret;
 }
 
-static inline int
-__copy_from_user_flushcache(void *dst, const void __user *src, unsigned size)
+static inline unsigned long
+__copy_from_user_flushcache(void *dst, const void __user *src, unsigned long size)
 {
 	kasan_check_write(dst, size);
 	return __copy_user_flushcache(dst, src, size);
diff --git a/arch/x86/lib/usercopy_64.c b/arch/x86/lib/usercopy_64.c
index 654280aaa3e9..10b0026e85b7 100644
--- a/arch/x86/lib/usercopy_64.c
+++ b/arch/x86/lib/usercopy_64.c
@@ -43,7 +43,7 @@ void arch_wb_cache_pmem(void *addr, size_t size)
 }
 EXPORT_SYMBOL_GPL(arch_wb_cache_pmem);
 
-long __copy_user_flushcache(void *dst, const void __user *src, unsigned size)
+unsigned long __copy_user_flushcache(void *dst, const void __user *src, unsigned long size)
 {
 	unsigned long flushed, dest = (unsigned long) dst;
 	long rc;
diff --git a/lib/iov_iter.c b/lib/iov_iter.c
index 0a63c7fba313..9c8fe94c72b4 100644
--- a/lib/iov_iter.c
+++ b/lib/iov_iter.c
@@ -41,9 +41,11 @@ size_t copy_to_user_iter_nofault(void __user *iter_to, size_t progress,
 	return res < 0 ? len : res;
 }
 
+typedef unsigned long (*from_user_copy_t)(void *, const void __user *, unsigned long);
+
 static __always_inline
-size_t copy_from_user_iter(void __user *iter_from, size_t progress,
-			   size_t len, void *to, void *priv2)
+size_t copy_from_user_iter_boilerplate(void __user *iter_from, size_t progress,
+			   size_t len, void *to, from_user_copy_t actor)
 {
 	size_t res = len;
 
@@ -64,12 +66,19 @@ size_t copy_from_user_iter(void __user *iter_from, size_t progress,
 	}
 	to += progress;
 	instrument_copy_from_user_before(to, iter_from, len);
-	res = raw_copy_from_user(to, iter_from, len);
+	res = actor(to, iter_from, len);
 	instrument_copy_from_user_after(to, iter_from, len, res);
 
 	return res;
 }
 
+static __always_inline
+size_t copy_from_user_iter(void __user *iter_from, size_t progress,
+			   size_t len, void *to, void *priv2)
+{
+	return copy_from_user_iter_boilerplate(iter_from, progress, len, to, raw_copy_from_user);
+}
+
 static __always_inline
 size_t memcpy_to_iter(void *iter_to, size_t progress,
 		      size_t len, void *from, void *priv2)
@@ -277,7 +286,7 @@ static __always_inline
 size_t copy_from_user_iter_nocache(void __user *iter_from, size_t progress,
 				   size_t len, void *to, void *priv2)
 {
-	return __copy_from_user_inatomic_nocache(to + progress, iter_from, len);
+	return copy_from_user_iter_boilerplate(iter_from, progress, len, to, __copy_from_user_inatomic_nocache);
 }
 
 size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
@@ -296,7 +305,7 @@ static __always_inline
 size_t copy_from_user_iter_flushcache(void __user *iter_from, size_t progress,
 				      size_t len, void *to, void *priv2)
 {
-	return __copy_from_user_flushcache(to + progress, iter_from, len);
+	return copy_from_user_iter_boilerplate(iter_from, progress, len, to, __copy_from_user_flushcache);
 }
 
 static __always_inline
-- 
2.53.0.479.g83c58c3406


[-- Attachment #3: 0002-Rename-and-clean-up-__copy_from_user_flushcache-func.patch --]
[-- Type: text/x-patch, Size: 6889 bytes --]

From c9e08013715c49b9ed67e9b9f9bf507bbcbedaa7 Mon Sep 17 00:00:00 2001
From: Linus Torvalds <torvalds@linux-foundation.org>
Date: Sat, 28 Mar 2026 17:00:44 -0700
Subject: [PATCH 2/3] Rename and clean up __copy_from_user_flushcache function

This function is conditional on CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE, and
had exactly one call-site: the _copy_from_iter_flushcache() iov_iter
walking code.

Three architectures enabled this feature: arm64 does it if you enable
ARCH64_PMEM, and powerpc and x86-64 enable it unconditionally.

Despite having only three implementations and one single call-site, that
function was impressively confused.

For example, two of the three implementations (x86-64 and arm64) had a
kasan_check_write() for it, but powerpc did not.  And arm64 called its
actual internally wrapped function '__copy_user_flushcache()', which was
the original x86-64 internal implementation name too: but that was
because on x86-64, the implementation was agnostic about which way to
copy.  Not so on arm64.

And now, with the unification of the iov_iter code, the single call site
doesn't even want the kasan_check_write() any more, because we have a
unified wrapper that just does it for us.

So get rid of all this confusion, the inconsistent kasan call, and the
unnecessary wrappers.  Just call it 'raw_copy_from_user_flushcache()' to
match the regular 'raw_copy_from_user()', which makes the single caller
look more logical since it now matches the normal case.

So this avoids some confusing wrappers, and avoids the doubled-up KASAN
overhead I unwittingly introduced in the iov_iter cleanup.

Reported-by: Vasily Gorbik <gor@linux.ibm.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
---
 arch/arm64/include/asm/uaccess.h    | 8 +-------
 arch/arm64/lib/uaccess_flushcache.c | 4 ++--
 arch/powerpc/include/asm/uaccess.h  | 2 +-
 arch/powerpc/lib/pmem.c             | 2 +-
 arch/x86/include/asm/uaccess_64.h   | 9 +--------
 arch/x86/lib/usercopy_64.c          | 2 +-
 lib/iov_iter.c                      | 2 +-
 7 files changed, 8 insertions(+), 21 deletions(-)

diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h
index 2d36a0c4ec6a..21c85fe4b993 100644
--- a/arch/arm64/include/asm/uaccess.h
+++ b/arch/arm64/include/asm/uaccess.h
@@ -474,13 +474,7 @@ extern long strncpy_from_user(char *dest, const char __user *src, long count);
 extern __must_check long strnlen_user(const char __user *str, long n);
 
 #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
-extern unsigned long __must_check __copy_user_flushcache(void *to, const void __user *from, unsigned long n);
-
-static inline unsigned long __copy_from_user_flushcache(void *dst, const void __user *src, unsigned long size)
-{
-	kasan_check_write(dst, size);
-	return __copy_user_flushcache(dst, __uaccess_mask_ptr(src), size);
-}
+extern unsigned long __must_check raw_copy_from_user_flushcache(void *to, const void __user *from, unsigned long n);
 #endif
 
 #ifdef CONFIG_ARCH_HAS_SUBPAGE_FAULTS
diff --git a/arch/arm64/lib/uaccess_flushcache.c b/arch/arm64/lib/uaccess_flushcache.c
index 7510d1a23124..73517f3a7811 100644
--- a/arch/arm64/lib/uaccess_flushcache.c
+++ b/arch/arm64/lib/uaccess_flushcache.c
@@ -19,8 +19,8 @@ void memcpy_flushcache(void *dst, const void *src, size_t cnt)
 }
 EXPORT_SYMBOL_GPL(memcpy_flushcache);
 
-unsigned long __copy_user_flushcache(void *to, const void __user *from,
-				     unsigned long n)
+unsigned long raw_copy_from_user_flushcache(void *to, const void __user *from,
+					    unsigned long n)
 {
 	unsigned long rc;
 
diff --git a/arch/powerpc/include/asm/uaccess.h b/arch/powerpc/include/asm/uaccess.h
index 8ae10aa5de63..f9aaaef84528 100644
--- a/arch/powerpc/include/asm/uaccess.h
+++ b/arch/powerpc/include/asm/uaccess.h
@@ -434,7 +434,7 @@ copy_mc_to_user(void __user *to, const void *from, unsigned long n)
 }
 #endif
 
-extern unsigned long __copy_from_user_flushcache(void *dst, const void __user *src,
+extern unsigned long raw_copy_from_user_flushcache(void *dst, const void __user *src,
 		unsigned long size);
 
 static __must_check __always_inline bool __user_access_begin(const void __user *ptr, size_t len,
diff --git a/arch/powerpc/lib/pmem.c b/arch/powerpc/lib/pmem.c
index 2e733ea9de3a..d31655e7ff0e 100644
--- a/arch/powerpc/lib/pmem.c
+++ b/arch/powerpc/lib/pmem.c
@@ -66,7 +66,7 @@ EXPORT_SYMBOL_GPL(arch_invalidate_pmem);
 /*
  * CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE symbols
  */
-unsigned long __copy_from_user_flushcache(void *dest, const void __user *src,
+unsigned long raw_copy_from_user_flushcache(void *dest, const void __user *src,
 		unsigned long size)
 {
 	unsigned long copied, start = (unsigned long) dest;
diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h
index 6c369481174a..dfe5fff8fbc1 100644
--- a/arch/x86/include/asm/uaccess_64.h
+++ b/arch/x86/include/asm/uaccess_64.h
@@ -148,7 +148,7 @@ raw_copy_to_user(void __user *dst, const void *src, unsigned long size)
 }
 
 extern unsigned long __copy_user_nocache(void *dst, const void __user *src, unsigned long size);
-extern unsigned long __copy_user_flushcache(void *dst, const void __user *src, unsigned long size);
+extern unsigned long raw_copy_from_user_flushcache(void *dst, const void __user *src, unsigned long size);
 
 static inline unsigned long
 __copy_from_user_inatomic_nocache(void *dst, const void __user *src, unsigned long size)
@@ -161,13 +161,6 @@ __copy_from_user_inatomic_nocache(void *dst, const void __user *src, unsigned lo
 	return ret;
 }
 
-static inline unsigned long
-__copy_from_user_flushcache(void *dst, const void __user *src, unsigned long size)
-{
-	kasan_check_write(dst, size);
-	return __copy_user_flushcache(dst, src, size);
-}
-
 /*
  * Zero Userspace.
  */
diff --git a/arch/x86/lib/usercopy_64.c b/arch/x86/lib/usercopy_64.c
index 10b0026e85b7..a4240dc0a787 100644
--- a/arch/x86/lib/usercopy_64.c
+++ b/arch/x86/lib/usercopy_64.c
@@ -43,7 +43,7 @@ void arch_wb_cache_pmem(void *addr, size_t size)
 }
 EXPORT_SYMBOL_GPL(arch_wb_cache_pmem);
 
-unsigned long __copy_user_flushcache(void *dst, const void __user *src, unsigned long size)
+unsigned long raw_copy_from_user_flushcache(void *dst, const void __user *src, unsigned long size)
 {
 	unsigned long flushed, dest = (unsigned long) dst;
 	long rc;
diff --git a/lib/iov_iter.c b/lib/iov_iter.c
index 9c8fe94c72b4..c7524fa43b8e 100644
--- a/lib/iov_iter.c
+++ b/lib/iov_iter.c
@@ -305,7 +305,7 @@ static __always_inline
 size_t copy_from_user_iter_flushcache(void __user *iter_from, size_t progress,
 				      size_t len, void *to, void *priv2)
 {
-	return copy_from_user_iter_boilerplate(iter_from, progress, len, to, __copy_from_user_flushcache);
+	return copy_from_user_iter_boilerplate(iter_from, progress, len, to, raw_copy_from_user_flushcache);
 }
 
 static __always_inline
-- 
2.53.0.479.g83c58c3406


[-- Attachment #4: 0003-x86-Introduce-raw_copy_from_user_nocache-function.patch --]
[-- Type: text/x-patch, Size: 6257 bytes --]

From 1aca05b885c9ba8f1a86983f522e17911347318d Mon Sep 17 00:00:00 2001
From: Linus Torvalds <torvalds@linux-foundation.org>
Date: Sat, 28 Mar 2026 17:22:46 -0700
Subject: [PATCH 3/3] x86: Introduce raw_copy_from_user_nocache() function

This isn't a new function per se, but I'm trying to clean up the
existing case of __copy_from_user_inatomic_nocache().  That function is
oddly named, oddly used, and now the iov_iter code wanted saner sematics
to match the other user access functions.

It's oddly named: the "inatomic" part of the name seems to be purely a
historical artifact of the implementation, with none of the users
actually caring about it.  And the double underscore is because of the
traditional "use access_ok() separately" that we used to do, but is
frowned upon now.

It's also very oddly used: there aren't many users, but one of them
doesn't want the "user" part either, and actively just mis-uses this for
a kernel-to-kernel copy because that user - ntb, the non-transparent PCI
bridge driver - wants the "nocache" semantics for what is really a
"memcpy_toio()".

That's particularly disgusting, because that only works on x86 and is
mostly an implementation accident even there (and it obviously also
means that it requires that "double underscore semantics" thing).

This is all conditional on ARCH_HAS_NOCACHE_UACCESS, and x86 is the only
platform that enables it, so this all happens to work, but that doesn't
make it right.

Even on x86 it's not just ugly, it also means that the horrible NTB copy
then entirely unnecessarily does a STAC/CLAC pair around it because it's
written as if it was a user space copy.

Other x86-only users include two GPU drivers.  At least in those two
cases the source is actually user space.

Anyway, since a couple of other users do exist, this doesn't just rename
the function and remove the KASAN check it does.  The old crufty name is
kept for those users (as a warning to others), but this introduces a
plain "raw_copy_from_user_nocache()" for the iov_iter case, and now
explains what that case actually really wants and matches modern user
access semantics and naming.

And yes, we should clean up the legacy cases too at some point.  Other
architectures can do the non-temporal copies too, and the NTB case is
just particularly bad form in general.  But this change only tries to
isolate the generic code from the mess.

Reported-by: Vasily Gorbik <gor@linux.ibm.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
---
 arch/x86/include/asm/uaccess_32.h |  4 +++-
 arch/x86/include/asm/uaccess_64.h | 10 ++++++++--
 include/linux/uio.h               |  4 ++++
 lib/iov_iter.c                    |  6 +++++-
 4 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/arch/x86/include/asm/uaccess_32.h b/arch/x86/include/asm/uaccess_32.h
index 40379a1adbb8..7c9b602af261 100644
--- a/arch/x86/include/asm/uaccess_32.h
+++ b/arch/x86/include/asm/uaccess_32.h
@@ -27,12 +27,14 @@ raw_copy_from_user(void *to, const void __user *from, unsigned long n)
 }
 
 static __always_inline unsigned long
-__copy_from_user_inatomic_nocache(void *to, const void __user *from,
+raw_copy_from_user_nocache(void *to, const void __user *from,
 				  unsigned long n)
 {
        return __copy_from_user_ll_nocache_nozero(to, from, n);
 }
 
+#define __copy_from_user_inatomic_nocache raw_copy_from_user_nocache
+
 unsigned long __must_check clear_user(void __user *mem, unsigned long len);
 unsigned long __must_check __clear_user(void __user *mem, unsigned long len);
 
diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h
index dfe5fff8fbc1..d3af6afbf765 100644
--- a/arch/x86/include/asm/uaccess_64.h
+++ b/arch/x86/include/asm/uaccess_64.h
@@ -151,16 +151,22 @@ extern unsigned long __copy_user_nocache(void *dst, const void __user *src, unsi
 extern unsigned long raw_copy_from_user_flushcache(void *dst, const void __user *src, unsigned long size);
 
 static inline unsigned long
-__copy_from_user_inatomic_nocache(void *dst, const void __user *src, unsigned long size)
+raw_copy_from_user_nocache(void *dst, const void __user *src, unsigned long size)
 {
 	long ret;
-	kasan_check_write(dst, size);
 	stac();
 	ret = __copy_user_nocache(dst, src, size);
 	clac();
 	return ret;
 }
 
+static inline unsigned long
+__copy_from_user_inatomic_nocache(void *dst, const void __user *src, unsigned long size)
+{
+	kasan_check_write(dst, size);
+	return raw_copy_from_user_nocache(dst, src, size);
+}
+
 /*
  * Zero Userspace.
  */
diff --git a/include/linux/uio.h b/include/linux/uio.h
index a9bc5b3067e3..473cfa5ade9a 100644
--- a/include/linux/uio.h
+++ b/include/linux/uio.h
@@ -196,7 +196,11 @@ size_t copy_folio_from_iter_atomic(struct folio *folio, size_t offset,
 
 size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i);
 size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i);
+#ifdef ARCH_HAS_NOCACHE_UACCESS
 size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i);
+#else
+#define _copy_from_iter_nocache _copy_from_iter
+#endif
 
 static inline size_t copy_folio_to_iter(struct folio *folio, size_t offset,
 		size_t bytes, struct iov_iter *i)
diff --git a/lib/iov_iter.c b/lib/iov_iter.c
index c7524fa43b8e..cce3d86a3cb9 100644
--- a/lib/iov_iter.c
+++ b/lib/iov_iter.c
@@ -282,11 +282,13 @@ size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
 }
 EXPORT_SYMBOL(_copy_from_iter);
 
+#ifdef ARCH_HAS_NOCACHE_UACCESS
+
 static __always_inline
 size_t copy_from_user_iter_nocache(void __user *iter_from, size_t progress,
 				   size_t len, void *to, void *priv2)
 {
-	return copy_from_user_iter_boilerplate(iter_from, progress, len, to, __copy_from_user_inatomic_nocache);
+	return copy_from_user_iter_boilerplate(iter_from, progress, len, to, raw_copy_from_user_nocache);
 }
 
 size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
@@ -300,6 +302,8 @@ size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
 }
 EXPORT_SYMBOL(_copy_from_iter_nocache);
 
+#endif
+
 #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
 static __always_inline
 size_t copy_from_user_iter_flushcache(void __user *iter_from, size_t progress,
-- 
2.53.0.479.g83c58c3406