From: Paul Mackerras <paulus@ozlabs.org>
To: linuxppc-dev@ozlabs.org
Subject: [PATCH v3 09/17] powerpc: Make load/store emulation use larger memory accesses
Date: Wed, 30 Aug 2017 14:12:32 +1000 [thread overview]
Message-ID: <1504066360-30128-10-git-send-email-paulus@ozlabs.org> (raw)
In-Reply-To: <1504066360-30128-1-git-send-email-paulus@ozlabs.org>
At the moment, emulation of loads and stores of up to 8 bytes to
unaligned addresses on a little-endian system uses a sequence of
single-byte loads or stores to memory. This is rather inefficient,
and the code is hard to follow because it has many ifdefs.
In addition, the Power ISA has requirements on how unaligned accesses
are performed, which are not met by doing all accesses as
sequences of single-byte accesses.
Emulation of VSX loads and stores uses __copy_{to,from}_user,
which means the emulation code has no control on the size of
accesses.
To simplify this, we add new copy_mem_in() and copy_mem_out()
functions for accessing memory. These use a sequence of the largest
possible aligned accesses, up to 8 bytes (or 4 on 32-bit systems),
to copy memory between a local buffer and user memory. We then
rewrite {read,write}_mem_unaligned and the VSX load/store
emulation using these new functions.
These new functions also simplify the code in do_fp_load() and
do_fp_store() for the unaligned cases.
Signed-off-by: Paul Mackerras <paulus@ozlabs.org>
---
arch/powerpc/lib/sstep.c | 235 +++++++++++++++++++++--------------------------
1 file changed, 106 insertions(+), 129 deletions(-)
diff --git a/arch/powerpc/lib/sstep.c b/arch/powerpc/lib/sstep.c
index ed2bc4c..6cc2911 100644
--- a/arch/powerpc/lib/sstep.c
+++ b/arch/powerpc/lib/sstep.c
@@ -193,7 +193,6 @@ static nokprobe_inline unsigned long max_align(unsigned long x)
return x & -x; /* isolates rightmost bit */
}
-
static nokprobe_inline unsigned long byterev_2(unsigned long x)
{
return ((x >> 8) & 0xff) | ((x & 0xff) << 8);
@@ -239,56 +238,69 @@ static nokprobe_inline int read_mem_aligned(unsigned long *dest,
return err;
}
-static nokprobe_inline int read_mem_unaligned(unsigned long *dest,
- unsigned long ea, int nb, struct pt_regs *regs)
+/*
+ * Copy from userspace to a buffer, using the largest possible
+ * aligned accesses, up to sizeof(long).
+ */
+static int nokprobe_inline copy_mem_in(u8 *dest, unsigned long ea, int nb)
{
- int err;
- unsigned long x, b, c;
-#ifdef __LITTLE_ENDIAN__
- int len = nb; /* save a copy of the length for byte reversal */
-#endif
+ int err = 0;
+ int c;
- /* unaligned, do this in pieces */
- x = 0;
for (; nb > 0; nb -= c) {
-#ifdef __LITTLE_ENDIAN__
- c = 1;
-#endif
-#ifdef __BIG_ENDIAN__
c = max_align(ea);
-#endif
if (c > nb)
c = max_align(nb);
- err = read_mem_aligned(&b, ea, c);
+ switch (c) {
+ case 1:
+ err = __get_user(*dest, (unsigned char __user *) ea);
+ break;
+ case 2:
+ err = __get_user(*(u16 *)dest,
+ (unsigned short __user *) ea);
+ break;
+ case 4:
+ err = __get_user(*(u32 *)dest,
+ (unsigned int __user *) ea);
+ break;
+#ifdef __powerpc64__
+ case 8:
+ err = __get_user(*(unsigned long *)dest,
+ (unsigned long __user *) ea);
+ break;
+#endif
+ }
if (err)
return err;
- x = (x << (8 * c)) + b;
+ dest += c;
ea += c;
}
-#ifdef __LITTLE_ENDIAN__
- switch (len) {
- case 2:
- *dest = byterev_2(x);
- break;
- case 4:
- *dest = byterev_4(x);
- break;
-#ifdef __powerpc64__
- case 8:
- *dest = byterev_8(x);
- break;
-#endif
- }
-#endif
-#ifdef __BIG_ENDIAN__
- *dest = x;
-#endif
return 0;
}
+static nokprobe_inline int read_mem_unaligned(unsigned long *dest,
+ unsigned long ea, int nb,
+ struct pt_regs *regs)
+{
+ union {
+ unsigned long ul;
+ u8 b[sizeof(unsigned long)];
+ } u;
+ int i;
+ int err;
+
+ u.ul = 0;
+ i = IS_BE ? sizeof(unsigned long) - nb : 0;
+ err = copy_mem_in(&u.b[i], ea, nb);
+ if (!err)
+ *dest = u.ul;
+ return err;
+}
+
/*
* Read memory at address ea for nb bytes, return 0 for success
- * or -EFAULT if an error occurred.
+ * or -EFAULT if an error occurred. N.B. nb must be 1, 2, 4 or 8.
+ * If nb < sizeof(long), the result is right-justified on BE systems.
*/
static int read_mem(unsigned long *dest, unsigned long ea, int nb,
struct pt_regs *regs)
@@ -325,48 +337,64 @@ static nokprobe_inline int write_mem_aligned(unsigned long val,
return err;
}
-static nokprobe_inline int write_mem_unaligned(unsigned long val,
- unsigned long ea, int nb, struct pt_regs *regs)
+/*
+ * Copy from a buffer to userspace, using the largest possible
+ * aligned accesses, up to sizeof(long).
+ */
+static int nokprobe_inline copy_mem_out(u8 *dest, unsigned long ea, int nb)
{
- int err;
- unsigned long c;
+ int err = 0;
+ int c;
-#ifdef __LITTLE_ENDIAN__
- switch (nb) {
- case 2:
- val = byterev_2(val);
- break;
- case 4:
- val = byterev_4(val);
- break;
-#ifdef __powerpc64__
- case 8:
- val = byterev_8(val);
- break;
-#endif
- }
-#endif
- /* unaligned or little-endian, do this in pieces */
for (; nb > 0; nb -= c) {
-#ifdef __LITTLE_ENDIAN__
- c = 1;
-#endif
-#ifdef __BIG_ENDIAN__
c = max_align(ea);
-#endif
if (c > nb)
c = max_align(nb);
- err = write_mem_aligned(val >> (nb - c) * 8, ea, c);
+ switch (c) {
+ case 1:
+ err = __put_user(*dest, (unsigned char __user *) ea);
+ break;
+ case 2:
+ err = __put_user(*(u16 *)dest,
+ (unsigned short __user *) ea);
+ break;
+ case 4:
+ err = __put_user(*(u32 *)dest,
+ (unsigned int __user *) ea);
+ break;
+#ifdef __powerpc64__
+ case 8:
+ err = __put_user(*(unsigned long *)dest,
+ (unsigned long __user *) ea);
+ break;
+#endif
+ }
if (err)
return err;
+ dest += c;
ea += c;
}
return 0;
}
+static nokprobe_inline int write_mem_unaligned(unsigned long val,
+ unsigned long ea, int nb,
+ struct pt_regs *regs)
+{
+ union {
+ unsigned long ul;
+ u8 b[sizeof(unsigned long)];
+ } u;
+ int i;
+
+ u.ul = val;
+ i = IS_BE ? sizeof(unsigned long) - nb : 0;
+ return copy_mem_out(&u.b[i], ea, nb);
+}
+
/*
* Write memory at address ea for nb bytes, return 0 for success
- * or -EFAULT if an error occurred.
+ * or -EFAULT if an error occurred. N.B. nb must be 1, 2, 4 or 8.
*/
static int write_mem(unsigned long val, unsigned long ea, int nb,
struct pt_regs *regs)
@@ -389,40 +417,17 @@ static int do_fp_load(int rn, int (*func)(int, unsigned long),
struct pt_regs *regs)
{
int err;
- union {
- double dbl;
- unsigned long ul[2];
- struct {
-#ifdef __BIG_ENDIAN__
- unsigned _pad_;
- unsigned word;
-#endif
-#ifdef __LITTLE_ENDIAN__
- unsigned word;
- unsigned _pad_;
-#endif
- } single;
- } data;
- unsigned long ptr;
+ u8 buf[sizeof(double)] __attribute__((aligned(sizeof(double))));
if (!address_ok(regs, ea, nb))
return -EFAULT;
- if ((ea & 3) == 0)
- return (*func)(rn, ea);
- ptr = (unsigned long) &data.ul;
- if (sizeof(unsigned long) == 8 || nb == 4) {
- err = read_mem_unaligned(&data.ul[0], ea, nb, regs);
- if (nb == 4)
- ptr = (unsigned long)&(data.single.word);
- } else {
- /* reading a double on 32-bit */
- err = read_mem_unaligned(&data.ul[0], ea, 4, regs);
- if (!err)
- err = read_mem_unaligned(&data.ul[1], ea + 4, 4, regs);
+ if (ea & 3) {
+ err = copy_mem_in(buf, ea, nb);
+ if (err)
+ return err;
+ ea = (unsigned long) buf;
}
- if (err)
- return err;
- return (*func)(rn, ptr);
+ return (*func)(rn, ea);
}
NOKPROBE_SYMBOL(do_fp_load);
@@ -431,43 +436,15 @@ static int do_fp_store(int rn, int (*func)(int, unsigned long),
struct pt_regs *regs)
{
int err;
- union {
- double dbl;
- unsigned long ul[2];
- struct {
-#ifdef __BIG_ENDIAN__
- unsigned _pad_;
- unsigned word;
-#endif
-#ifdef __LITTLE_ENDIAN__
- unsigned word;
- unsigned _pad_;
-#endif
- } single;
- } data;
- unsigned long ptr;
+ u8 buf[sizeof(double)] __attribute__((aligned(sizeof(double))));
if (!address_ok(regs, ea, nb))
return -EFAULT;
if ((ea & 3) == 0)
return (*func)(rn, ea);
- ptr = (unsigned long) &data.ul[0];
- if (sizeof(unsigned long) == 8 || nb == 4) {
- if (nb == 4)
- ptr = (unsigned long)&(data.single.word);
- err = (*func)(rn, ptr);
- if (err)
- return err;
- err = write_mem_unaligned(data.ul[0], ea, nb, regs);
- } else {
- /* writing a double on 32-bit */
- err = (*func)(rn, ptr);
- if (err)
- return err;
- err = write_mem_unaligned(data.ul[0], ea, 4, regs);
- if (!err)
- err = write_mem_unaligned(data.ul[1], ea + 4, 4, regs);
- }
+ err = (*func)(rn, (unsigned long) buf);
+ if (!err)
+ err = copy_mem_out(buf, ea, nb);
return err;
}
NOKPROBE_SYMBOL(do_fp_store);
@@ -2564,7 +2541,7 @@ int emulate_step(struct pt_regs *regs, unsigned int instr)
#endif
#ifdef CONFIG_VSX
case LOAD_VSX: {
- char mem[16];
+ u8 mem[16];
union vsx_reg buf;
unsigned long msrbit = MSR_VSX;
@@ -2577,7 +2554,7 @@ int emulate_step(struct pt_regs *regs, unsigned int instr)
if (!(regs->msr & msrbit))
return 0;
if (!address_ok(regs, ea, size) ||
- __copy_from_user(mem, (void __user *)ea, size))
+ copy_mem_in(mem, ea, size))
return 0;
emulate_vsx_load(&op, &buf, mem);
@@ -2639,7 +2616,7 @@ int emulate_step(struct pt_regs *regs, unsigned int instr)
#endif
#ifdef CONFIG_VSX
case STORE_VSX: {
- char mem[16];
+ u8 mem[16];
union vsx_reg buf;
unsigned long msrbit = MSR_VSX;
@@ -2656,7 +2633,7 @@ int emulate_step(struct pt_regs *regs, unsigned int instr)
store_vsrn(op.reg, &buf);
emulate_vsx_store(&op, &buf, mem);
- if (__copy_to_user((void __user *)ea, mem, size))
+ if (copy_mem_out(mem, ea, size))
return 0;
goto ldst_done;
}
--
2.7.4
next prev parent reply other threads:[~2017-08-30 4:12 UTC|newest]
Thread overview: 23+ messages / expand[flat|nested] mbox.gz Atom feed top
2017-08-30 4:12 [PATCH v3 00/17] powerpc: Do alignment fixups using analyse_instr etc Paul Mackerras
2017-08-30 4:12 ` [PATCH v3 01/17] powerpc: Correct instruction code for xxlor instruction Paul Mackerras
2017-09-01 13:29 ` [v3, " Michael Ellerman
2017-08-30 4:12 ` [PATCH v3 02/17] powerpc: Change analyse_instr so it doesn't modify *regs Paul Mackerras
2017-08-30 4:12 ` [PATCH v3 03/17] powerpc: Don't check MSR FP/VMX/VSX enable bits in analyse_instr() Paul Mackerras
2017-08-30 4:12 ` [PATCH v3 04/17] powerpc: Handle most loads and stores in instruction emulation code Paul Mackerras
2017-08-30 4:12 ` [PATCH v3 05/17] powerpc/64: Fix update forms of loads and stores to write 64-bit EA Paul Mackerras
2017-08-30 4:12 ` [PATCH v3 06/17] powerpc: Fix emulation of the isel instruction Paul Mackerras
2017-08-30 4:12 ` [PATCH v3 07/17] powerpc: Don't update CR0 in emulation of popcnt, prty, bpermd instructions Paul Mackerras
2017-08-30 4:12 ` [PATCH v3 08/17] powerpc: Add emulation for the addpcis instruction Paul Mackerras
2017-08-30 4:12 ` Paul Mackerras [this message]
2017-08-30 4:12 ` [PATCH v3 10/17] powerpc: Emulate FP/vector/VSX loads/stores correctly when regs not live Paul Mackerras
2017-08-30 4:12 ` [PATCH v3 11/17] powerpc: Emulate vector element load/store instructions Paul Mackerras
2017-08-30 4:12 ` [PATCH v3 12/17] powerpc: Emulate load/store floating double pair instructions Paul Mackerras
2017-08-30 4:12 ` [PATCH v3 13/17] powerpc: Emulate the dcbz instruction Paul Mackerras
2017-08-30 4:12 ` [PATCH v3 14/17] powerpc: Set regs->dar if memory access fails in emulate_step() Paul Mackerras
2017-08-30 4:12 ` [PATCH v3 15/17] powerpc: Handle opposite-endian processes in emulation code Paul Mackerras
2017-08-30 4:12 ` [PATCH v3 16/17] powerpc: Separate out load/store emulation into its own function Paul Mackerras
2017-08-30 4:12 ` [PATCH v3 17/17] powerpc: Use instruction emulation infrastructure to handle alignment faults Paul Mackerras
2017-08-30 6:34 ` [PATCH v3 18/17] powerpc: Emulate load/store floating point as integer word instructions Paul Mackerras
2017-08-31 0:49 ` [PATCH v3 00/17] powerpc: Do alignment fixups using analyse_instr etc Michael Neuling
2017-08-31 0:54 ` Michael Neuling
2017-08-31 23:51 ` [PATCH 19/17] powerpc: Wrap register number correctly for string load/store instructions Paul Mackerras
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1504066360-30128-10-git-send-email-paulus@ozlabs.org \
--to=paulus@ozlabs.org \
--cc=linuxppc-dev@ozlabs.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).