All of lore.kernel.org
 help / color / mirror / Atom feed
From: Andrew Holden <andy0x48@linux.ibm.com>
To: qemu-devel@nongnu.org, iii@linux.ibm.com, freude@linux.ibm.com
Cc: qemu-s390x@nongnu.org, richard.henderson@linaro.org,
	david@kernel.org, cohuck@redhat.com, farman@linux.ibm.com,
	mjrosato@linux.ibm.com, Andrew Holden <andy0x48@linux.ibm.com>
Subject: [PATCH v2 2/3] target/s390x/tcg: update regs and IV/tweak per block in CPACF AES/PAES
Date: Fri, 20 Nov 2026 13:14:42 +0000	[thread overview]
Message-ID: <20261120131443.61606-3-andy0x48@linux.ibm.com> (raw)
In-Reply-To: <20261120131443.61606-1-andy0x48@linux.ibm.com>

The CPACF AES and PAES helpers (ECB, CBC, CTR, and XTS) updated the guest
registers, and the CBC IV and XTS tweak, only after the whole block
loop. A guest fault unwinds out of the helper, so the registers kept
their start-of-instruction values, with the instruction having
restarted at the first block instead of resuming after the last
completed block, as expected regarding PoP. A block crossing a page
boundary could also be partly written before the fault.

Probe the source, destination, and IV/tweak of each block with
access_prepare() before any data is written, then update the registers
last, a fault leaves the block untouched and a re-invocation would
redo this. aes_read_block() and aes_write_block() do all the copying
for each mode. Addresses are wrapped per block.

The IV/tweak write is probed first but it is not atomic with the data
write. It's unlikely a fault happens between the data store and
IV/tweak, but it could happen due to a specific thread unmapping, page
change, etc., and only matters when source and destination overlap.

Signed-off-by: Andrew Holden <andy0x48@linux.ibm.com>
---
 target/s390x/tcg/cpacf_aes.c | 380 ++++++++++++++++++++++++-----------
 1 file changed, 259 insertions(+), 121 deletions(-)

diff --git a/target/s390x/tcg/cpacf_aes.c b/target/s390x/tcg/cpacf_aes.c
index 92884c4120..117d43040b 100644
--- a/target/s390x/tcg/cpacf_aes.c
+++ b/target/s390x/tcg/cpacf_aes.c
@@ -12,6 +12,7 @@
 #include "tcg_s390x.h"
 #include "accel/tcg/cpu-ldst-common.h"
 #include "accel/tcg/cpu-mmu-index.h"
+#include "accel/tcg/helper-retaddr.h"
 #include "crypto/aes.h"
 #include "crypto/aes-helpers.h"
 #include "target/s390x/tcg/cpacf-arch.h"
@@ -19,23 +20,40 @@
 #include "target/s390x/tcg/crypto_helper.h"
 
 /*
- * read exactly one AES block from guest memory into a local buffer
+ * probe one AES block at guest address, which is wrapped according to current
+ * addressing mode, since access_prepare() only wraps second page.
  */
-static inline void aes_read_block(CPUS390XState *env, const int mmu_idx,
-                                  const uintptr_t ra, uint64_t guest_addr,
-                                  uint8_t *buf)
+static inline void aes_prepare_block(CPUS390XState *env, const int mmu_idx,
+                                     const uintptr_t ra, S390Access *acc,
+                                     uint64_t addr, MMUAccessType type)
 {
-    read_guest_wrap_u8(env, mmu_idx, ra, guest_addr, buf, AES_BLOCK_SIZE);
+    uint64_t waddr;
+    waddr = wrap_address(env, addr);
+    access_prepare(acc, env, waddr, AES_BLOCK_SIZE, type, mmu_idx, ra);
 }
 
 /*
- * write exactly one AES block from local buffer to guest memory
+ * read exactly one AES block from a prepared access into a local buffer.
  */
-static void aes_write_block(CPUS390XState *env, const int mmu_idx,
-                            const uintptr_t ra, uint64_t guest_addr,
-                            uint8_t *buf)
+static inline void aes_read_block(CPUS390XState *env, const uintptr_t ra,
+                                  S390Access *acc, uint8_t *buf)
 {
-    write_guest_wrap_u8(env, mmu_idx, ra, guest_addr, buf, AES_BLOCK_SIZE);
+    /* gather block */
+    for (int i = 0; i < AES_BLOCK_SIZE; i++) {
+        buf[i] = access_get_byte(env, acc, i, ra);
+    }
+}
+
+/*
+ * write exactly one AES block from a local buffer to a prepared access.
+ */
+static inline void aes_write_block(CPUS390XState *env, const uintptr_t ra,
+                                   S390Access *acc, uint8_t *buf)
+{
+    /* scatter block */
+    for (int i = 0; i < AES_BLOCK_SIZE; i++) {
+        access_set_byte(env, acc, i, buf[i], ra);
+    }
 }
 
 /*
@@ -62,8 +80,9 @@ int cpacf_aes_ecb(CPUS390XState *env, const int mmu_idx, uintptr_t ra,
 {
     enum { MAX_BLOCKS_PER_RUN = 8192 / AES_BLOCK_SIZE };
     uint8_t in[AES_BLOCK_SIZE], out[AES_BLOCK_SIZE];
-    uint64_t len = *src_len_reg, done = 0;
     int i, keysize, addr_reg_size;
+    uint64_t len = *src_len_reg;
+    S390Access srca, dsta;
     uint8_t key[32];
     AES_KEY exkey;
 
@@ -116,23 +135,37 @@ int cpacf_aes_ecb(CPUS390XState *env, const int mmu_idx, uintptr_t ra,
 
     /* process up to MAX_BLOCKS_PER_RUN aes blocks */
     for (i = 0; i < MAX_BLOCKS_PER_RUN && len >= AES_BLOCK_SIZE; i++) {
-        aes_read_block(env, mmu_idx, ra, *src_ptr_reg + done, in);
+        /* probe source and destination access */
+        aes_prepare_block(env, mmu_idx, ra, &srca, *src_ptr_reg,
+                          MMU_DATA_LOAD);
+        aes_prepare_block(env, mmu_idx, ra, &dsta, *dst_ptr_reg,
+                          MMU_DATA_STORE);
+
+        /*
+         * linux-user: the accessors below use host pointers. Record ra so a
+         * fault from a racing unmap of the probed page is unwound correctly.
+         */
+        set_helper_retaddr(ra);
+
+        aes_read_block(env, ra, &srca, in);
         if (mod) {
             AES_decrypt(in, out, &exkey);
         } else {
             AES_encrypt(in, out, &exkey);
         }
-        aes_write_block(env, mmu_idx, ra, *dst_ptr_reg + done, out);
+        aes_write_block(env, ra, &dsta, out);
+
+        clear_helper_retaddr();
+
+        /* update registers at end of each iteration */
+        *src_ptr_reg = deposit64(*src_ptr_reg, 0, addr_reg_size,
+                                 *src_ptr_reg + AES_BLOCK_SIZE);
+        *dst_ptr_reg = deposit64(*dst_ptr_reg, 0, addr_reg_size,
+                                 *dst_ptr_reg + AES_BLOCK_SIZE);
+        *src_len_reg -= AES_BLOCK_SIZE;
         len -= AES_BLOCK_SIZE;
-        done += AES_BLOCK_SIZE;
     }
 
-    *src_ptr_reg = deposit64(*src_ptr_reg, 0, addr_reg_size,
-                             *src_ptr_reg + done);
-    *dst_ptr_reg = deposit64(*dst_ptr_reg, 0, addr_reg_size,
-                             *dst_ptr_reg + done);
-    *src_len_reg -= done;
-
     return !len ? 0 : 3;
 }
 
@@ -143,9 +176,10 @@ int cpacf_aes_cbc(CPUS390XState *env, const int mmu_idx, uintptr_t ra,
 {
     enum { MAX_BLOCKS_PER_RUN = 8192 / AES_BLOCK_SIZE };
     uint8_t in[AES_BLOCK_SIZE], out[AES_BLOCK_SIZE];
-    uint64_t len = *src_len_reg, done = 0;
     uint8_t key[32], iv[AES_BLOCK_SIZE];
     int i, keysize, addr_reg_size;
+    uint64_t len = *src_len_reg;
+    S390Access srca, dsta, iva;
     AES_KEY exkey;
 
     g_assert(type == S390_FEAT_TYPE_KMC);
@@ -202,7 +236,21 @@ int cpacf_aes_cbc(CPUS390XState *env, const int mmu_idx, uintptr_t ra,
 
     /* process up to MAX_BLOCKS_PER_RUN aes blocks */
     for (i = 0; i < MAX_BLOCKS_PER_RUN && len >= AES_BLOCK_SIZE; i++) {
-        aes_read_block(env, mmu_idx, ra, *src_ptr_reg + done, in);
+        /* probe source, destination, and iv access */
+        aes_prepare_block(env, mmu_idx, ra, &srca, *src_ptr_reg,
+                          MMU_DATA_LOAD);
+        aes_prepare_block(env, mmu_idx, ra, &dsta, *dst_ptr_reg,
+                          MMU_DATA_STORE);
+        aes_prepare_block(env, mmu_idx, ra, &iva, param_addr,
+                          MMU_DATA_STORE);
+
+        /*
+         * linux-user: the accessors below use host pointers. Record ra so a
+         * fault from a racing unmap of the probed page is unwound correctly.
+         */
+        set_helper_retaddr(ra);
+
+        aes_read_block(env, ra, &srca, in);
         if (mod) {
             /* decrypt in => out */
             AES_cbc_decrypt(in, out, iv, &exkey);
@@ -210,20 +258,22 @@ int cpacf_aes_cbc(CPUS390XState *env, const int mmu_idx, uintptr_t ra,
             /* encrypt in => out */
             AES_cbc_encrypt(in, out, iv, &exkey);
         }
-        aes_write_block(env, mmu_idx, ra, *dst_ptr_reg + done, out);
+        /* scatter block to destination */
+        aes_write_block(env, ra, &dsta, out);
+        /* write back running iv */
+        aes_write_block(env, ra, &iva, iv);
+
+        clear_helper_retaddr();
+
+        /* update registers at end of each iteration */
+        *src_ptr_reg = deposit64(*src_ptr_reg, 0, addr_reg_size,
+                                 *src_ptr_reg + AES_BLOCK_SIZE);
+        *dst_ptr_reg = deposit64(*dst_ptr_reg, 0, addr_reg_size,
+                                 *dst_ptr_reg + AES_BLOCK_SIZE);
+        *src_len_reg -= AES_BLOCK_SIZE;
         len -= AES_BLOCK_SIZE;
-        done += AES_BLOCK_SIZE;
     }
 
-    /* update iv in param block */
-    write_guest_wrap_u8(env, mmu_idx, ra, param_addr, iv, AES_BLOCK_SIZE);
-
-    *src_ptr_reg = deposit64(*src_ptr_reg, 0, addr_reg_size,
-                             *src_ptr_reg + done);
-    *dst_ptr_reg = deposit64(*dst_ptr_reg, 0, addr_reg_size,
-                             *dst_ptr_reg + done);
-    *src_len_reg -= done;
-
     return !len ? 0 : 3;
 }
 
@@ -235,9 +285,10 @@ int cpacf_aes_ctr(CPUS390XState *env, const int mmu_idx, uintptr_t ra,
 {
     enum { MAX_BLOCKS_PER_RUN = 8192 / AES_BLOCK_SIZE };
     uint8_t in[AES_BLOCK_SIZE], out[AES_BLOCK_SIZE];
-    uint64_t len = *src_len_reg, done = 0;
     uint8_t ctr[AES_BLOCK_SIZE], key[32];
     int i, keysize, addr_reg_size;
+    uint64_t len = *src_len_reg;
+    S390Access srca, dsta, ctra;
     AES_KEY exkey;
 
     g_assert(type == S390_FEAT_TYPE_KMCTR);
@@ -286,26 +337,39 @@ int cpacf_aes_ctr(CPUS390XState *env, const int mmu_idx, uintptr_t ra,
 
     /* process up to MAX_BLOCKS_PER_RUN aes blocks */
     for (i = 0; i < MAX_BLOCKS_PER_RUN && len >= AES_BLOCK_SIZE; i++) {
-        /* read in nonce/ctr => ctr */
-        aes_read_block(env, mmu_idx, ra, *ctr_ptr_reg + done, ctr);
-        /* read in one block of input data => in */
-        aes_read_block(env, mmu_idx, ra, *src_ptr_reg + done, in);
+        /* probe source, counter, and destination access */
+        aes_prepare_block(env, mmu_idx, ra, &srca, *src_ptr_reg,
+                          MMU_DATA_LOAD);
+        aes_prepare_block(env, mmu_idx, ra, &ctra, *ctr_ptr_reg,
+                          MMU_DATA_LOAD);
+        aes_prepare_block(env, mmu_idx, ra, &dsta, *dst_ptr_reg,
+                          MMU_DATA_STORE);
+
+        /*
+         * linux-user: the accessors below use host pointers. Record ra so a
+         * fault from a racing unmap of the probed page is unwound correctly.
+         */
+        set_helper_retaddr(ra);
+
+        aes_read_block(env, ra, &srca, in);
+        aes_read_block(env, ra, &ctra, ctr);
         /* encrypt ctr and xor with in => out */
         AES_ctr_encrypt(in, out, ctr, &exkey);
-        /* write out the processed block */
-        aes_write_block(env, mmu_idx, ra, *dst_ptr_reg + done, out);
+        aes_write_block(env, ra, &dsta, out);
+
+        clear_helper_retaddr();
+
+        /* update registers at end of each iteration */
+        *src_ptr_reg = deposit64(*src_ptr_reg, 0, addr_reg_size,
+                                 *src_ptr_reg + AES_BLOCK_SIZE);
+        *dst_ptr_reg = deposit64(*dst_ptr_reg, 0, addr_reg_size,
+                                 *dst_ptr_reg + AES_BLOCK_SIZE);
+        *ctr_ptr_reg = deposit64(*ctr_ptr_reg, 0, addr_reg_size,
+                                 *ctr_ptr_reg + AES_BLOCK_SIZE);
+        *src_len_reg -= AES_BLOCK_SIZE;
         len -= AES_BLOCK_SIZE;
-        done += AES_BLOCK_SIZE;
     }
 
-    *src_ptr_reg = deposit64(*src_ptr_reg, 0, addr_reg_size,
-                             *src_ptr_reg + done);
-    *dst_ptr_reg = deposit64(*dst_ptr_reg, 0, addr_reg_size,
-                             *dst_ptr_reg + done);
-    *ctr_ptr_reg = deposit64(*ctr_ptr_reg, 0, addr_reg_size,
-                             *ctr_ptr_reg + done);
-    *src_len_reg -= done;
-
     return !len ? 0 : 3;
 }
 
@@ -369,9 +433,10 @@ int cpacf_aes_xts(CPUS390XState *env, const int mmu_idx, uintptr_t ra,
 {
     enum { MAX_BLOCKS_PER_RUN = 8192 / AES_BLOCK_SIZE };
     uint8_t in[AES_BLOCK_SIZE], out[AES_BLOCK_SIZE];
-    uint64_t len = *src_len_reg, done = 0;
     uint8_t key[32], tweak[AES_BLOCK_SIZE];
+    S390Access srca, dsta, tweaka;
     int i, keysize, addr_reg_size;
+    uint64_t len = *src_len_reg;
     AES_KEY exkey;
 
     g_assert(type == S390_FEAT_TYPE_KM);
@@ -425,8 +490,21 @@ int cpacf_aes_xts(CPUS390XState *env, const int mmu_idx, uintptr_t ra,
 
     /* process up to MAX_BLOCKS_PER_RUN aes blocks */
     for (i = 0; i < MAX_BLOCKS_PER_RUN && len >= AES_BLOCK_SIZE; i++) {
-        /* fetch one AES block into in  */
-        aes_read_block(env, mmu_idx, ra, *src_ptr_reg + done, in);
+        /* probe source, destination, and tweak access */
+        aes_prepare_block(env, mmu_idx, ra, &srca, *src_ptr_reg,
+                          MMU_DATA_LOAD);
+        aes_prepare_block(env, mmu_idx, ra, &dsta, *dst_ptr_reg,
+                          MMU_DATA_STORE);
+        aes_prepare_block(env, mmu_idx, ra, &tweaka, param_addr + keysize,
+                          MMU_DATA_STORE);
+
+        /*
+         * linux-user: the accessors below use host pointers. Record ra so a
+         * fault from a racing unmap of the probed page is unwound correctly.
+         */
+        set_helper_retaddr(ra);
+
+        aes_read_block(env, ra, &srca, in);
         if (mod) {
             /* decrypt in => out */
             AES_xts_decrypt(in, out, tweak, &exkey);
@@ -436,22 +514,22 @@ int cpacf_aes_xts(CPUS390XState *env, const int mmu_idx, uintptr_t ra,
         }
         /* prep tweak for next round */
         AES_xts_prep_next_tweak(tweak);
-        /* write out this processed block from out */
-        aes_write_block(env, mmu_idx, ra, *dst_ptr_reg + done, out);
+        /* scatter block to destination */
+        aes_write_block(env, ra, &dsta, out);
+        /* write back running tweak */
+        aes_write_block(env, ra, &tweaka, tweak);
+
+        clear_helper_retaddr();
+
+        /* update registers at end of each iteration */
+        *src_ptr_reg = deposit64(*src_ptr_reg, 0, addr_reg_size,
+                                 *src_ptr_reg + AES_BLOCK_SIZE);
+        *dst_ptr_reg = deposit64(*dst_ptr_reg, 0, addr_reg_size,
+                                 *dst_ptr_reg + AES_BLOCK_SIZE);
+        *src_len_reg -= AES_BLOCK_SIZE;
         len -= AES_BLOCK_SIZE;
-        done += AES_BLOCK_SIZE;
     }
 
-    /* update tweak in param block */
-    write_guest_wrap_u8(env, mmu_idx, ra,
-                        param_addr + keysize, tweak, AES_BLOCK_SIZE);
-
-    *src_ptr_reg = deposit64(*src_ptr_reg, 0, addr_reg_size,
-                             *src_ptr_reg + done);
-    *dst_ptr_reg = deposit64(*dst_ptr_reg, 0, addr_reg_size,
-                             *dst_ptr_reg + done);
-    *src_len_reg -= done;
-
     return !len ? 0 : 3;
 }
 
@@ -537,9 +615,10 @@ int cpacf_paes_ecb(CPUS390XState *env, const int mmu_idx, uintptr_t ra,
 {
     enum { MAX_BLOCKS_PER_RUN = 8192 / AES_BLOCK_SIZE };
     uint8_t in[AES_BLOCK_SIZE], out[AES_BLOCK_SIZE];
-    uint64_t len = *src_len_reg, done = 0;
     int i, keysize, addr_reg_size;
+    uint64_t len = *src_len_reg;
     uint8_t key[32], wkvp[32];
+    S390Access srca, dsta;
     AES_KEY exkey;
 
     g_assert(type == S390_FEAT_TYPE_KM);
@@ -602,23 +681,37 @@ int cpacf_paes_ecb(CPUS390XState *env, const int mmu_idx, uintptr_t ra,
 
     /* process up to MAX_BLOCKS_PER_RUN aes blocks */
     for (i = 0; i < MAX_BLOCKS_PER_RUN && len >= AES_BLOCK_SIZE; i++) {
-        aes_read_block(env, mmu_idx, ra, *src_ptr_reg + done, in);
+        /* probe source and destination access */
+        aes_prepare_block(env, mmu_idx, ra, &srca, *src_ptr_reg,
+                          MMU_DATA_LOAD);
+        aes_prepare_block(env, mmu_idx, ra, &dsta, *dst_ptr_reg,
+                          MMU_DATA_STORE);
+
+        /*
+         * linux-user: the accessors below use host pointers. Record ra so a
+         * fault from a racing unmap of the probed page is unwound correctly.
+         */
+        set_helper_retaddr(ra);
+
+        aes_read_block(env, ra, &srca, in);
         if (mod) {
             AES_decrypt(in, out, &exkey);
         } else {
             AES_encrypt(in, out, &exkey);
         }
-        aes_write_block(env, mmu_idx, ra, *dst_ptr_reg + done, out);
+        aes_write_block(env, ra, &dsta, out);
+
+        clear_helper_retaddr();
+
+        /* update registers at end of each iteration */
+        *src_ptr_reg = deposit64(*src_ptr_reg, 0, addr_reg_size,
+                                 *src_ptr_reg + AES_BLOCK_SIZE);
+        *dst_ptr_reg = deposit64(*dst_ptr_reg, 0, addr_reg_size,
+                                 *dst_ptr_reg + AES_BLOCK_SIZE);
+        *src_len_reg -= AES_BLOCK_SIZE;
         len -= AES_BLOCK_SIZE;
-        done += AES_BLOCK_SIZE;
     }
 
-    *src_ptr_reg = deposit64(*src_ptr_reg, 0, addr_reg_size,
-                             *src_ptr_reg + done);
-    *dst_ptr_reg = deposit64(*dst_ptr_reg, 0, addr_reg_size,
-                             *dst_ptr_reg + done);
-    *src_len_reg -= done;
-
     return !len ? 0 : 3;
 }
 
@@ -630,8 +723,9 @@ int cpacf_paes_cbc(CPUS390XState *env, const int mmu_idx, uintptr_t ra,
     enum { MAX_BLOCKS_PER_RUN = 8192 / AES_BLOCK_SIZE };
     uint8_t in[AES_BLOCK_SIZE], out[AES_BLOCK_SIZE];
     uint8_t key[32], wkvp[32], iv[AES_BLOCK_SIZE];
-    uint64_t len = *src_len_reg, done = 0;
     int i, keysize, addr_reg_size;
+    uint64_t len = *src_len_reg;
+    S390Access srca, dsta, iva;
     AES_KEY exkey;
 
     g_assert(type == S390_FEAT_TYPE_KMC);
@@ -699,7 +793,21 @@ int cpacf_paes_cbc(CPUS390XState *env, const int mmu_idx, uintptr_t ra,
 
     /* process up to MAX_BLOCKS_PER_RUN aes blocks */
     for (i = 0; i < MAX_BLOCKS_PER_RUN && len >= AES_BLOCK_SIZE; i++) {
-        aes_read_block(env, mmu_idx, ra, *src_ptr_reg + done, in);
+        /* probe source, destination, and iv access */
+        aes_prepare_block(env, mmu_idx, ra, &srca, *src_ptr_reg,
+                          MMU_DATA_LOAD);
+        aes_prepare_block(env, mmu_idx, ra, &dsta, *dst_ptr_reg,
+                          MMU_DATA_STORE);
+        aes_prepare_block(env, mmu_idx, ra, &iva, param_addr,
+                          MMU_DATA_STORE);
+
+        /*
+         * linux-user: the accessors below use host pointers. Record ra so a
+         * fault from a racing unmap of the probed page is unwound correctly.
+         */
+        set_helper_retaddr(ra);
+
+        aes_read_block(env, ra, &srca, in);
         if (mod) {
             /* decrypt in => out */
             AES_cbc_decrypt(in, out, iv, &exkey);
@@ -707,20 +815,22 @@ int cpacf_paes_cbc(CPUS390XState *env, const int mmu_idx, uintptr_t ra,
             /* encrypt in => out */
             AES_cbc_encrypt(in, out, iv, &exkey);
         }
-        aes_write_block(env, mmu_idx, ra, *dst_ptr_reg + done, out);
+        /* scatter block to destination */
+        aes_write_block(env, ra, &dsta, out);
+        /* write back running iv */
+        aes_write_block(env, ra, &iva, iv);
+
+        clear_helper_retaddr();
+
+        /* update registers at end of each iteration */
+        *src_ptr_reg = deposit64(*src_ptr_reg, 0, addr_reg_size,
+                                 *src_ptr_reg + AES_BLOCK_SIZE);
+        *dst_ptr_reg = deposit64(*dst_ptr_reg, 0, addr_reg_size,
+                                 *dst_ptr_reg + AES_BLOCK_SIZE);
+        *src_len_reg -= AES_BLOCK_SIZE;
         len -= AES_BLOCK_SIZE;
-        done += AES_BLOCK_SIZE;
     }
 
-    /* update iv in param block */
-    write_guest_wrap_u8(env, mmu_idx, ra, param_addr, iv, AES_BLOCK_SIZE);
-
-    *src_ptr_reg = deposit64(*src_ptr_reg, 0, addr_reg_size,
-                             *src_ptr_reg + done);
-    *dst_ptr_reg = deposit64(*dst_ptr_reg, 0, addr_reg_size,
-                             *dst_ptr_reg + done);
-    *src_len_reg -= done;
-
     return !len ? 0 : 3;
 }
 
@@ -733,8 +843,9 @@ int cpacf_paes_ctr(CPUS390XState *env, const int mmu_idx, uintptr_t ra,
     enum { MAX_BLOCKS_PER_RUN = 8192 / AES_BLOCK_SIZE };
     uint8_t in[AES_BLOCK_SIZE], out[AES_BLOCK_SIZE];
     uint8_t ctr[AES_BLOCK_SIZE], key[32], wkvp[32];
-    uint64_t len = *src_len_reg, done = 0;
     int i, keysize, addr_reg_size;
+    uint64_t len = *src_len_reg;
+    S390Access srca, dsta, ctra;
     AES_KEY exkey;
 
     g_assert(type == S390_FEAT_TYPE_KMCTR);
@@ -793,26 +904,39 @@ int cpacf_paes_ctr(CPUS390XState *env, const int mmu_idx, uintptr_t ra,
 
     /* process up to MAX_BLOCKS_PER_RUN aes blocks */
     for (i = 0; i < MAX_BLOCKS_PER_RUN && len >= AES_BLOCK_SIZE; i++) {
-        /* read in nonce/ctr => ctr */
-        aes_read_block(env, mmu_idx, ra, *ctr_ptr_reg + done, ctr);
-        /* read in one block of input data => in */
-        aes_read_block(env, mmu_idx, ra, *src_ptr_reg + done, in);
+        /* probe source, counter, and destination access */
+        aes_prepare_block(env, mmu_idx, ra, &srca, *src_ptr_reg,
+                          MMU_DATA_LOAD);
+        aes_prepare_block(env, mmu_idx, ra, &ctra, *ctr_ptr_reg,
+                          MMU_DATA_LOAD);
+        aes_prepare_block(env, mmu_idx, ra, &dsta, *dst_ptr_reg,
+                          MMU_DATA_STORE);
+
+        /*
+         * linux-user: the accessors below use host pointers. Record ra so a
+         * fault from a racing unmap of the probed page is unwound correctly.
+         */
+        set_helper_retaddr(ra);
+
+        aes_read_block(env, ra, &srca, in);
+        aes_read_block(env, ra, &ctra, ctr);
         /* encrypt ctr and xor with in => out */
         AES_ctr_encrypt(in, out, ctr, &exkey);
-        /* write out the processed block */
-        aes_write_block(env, mmu_idx, ra, *dst_ptr_reg + done, out);
+        aes_write_block(env, ra, &dsta, out);
+
+        clear_helper_retaddr();
+
+        /* update registers at end of each iteration */
+        *src_ptr_reg = deposit64(*src_ptr_reg, 0, addr_reg_size,
+                                 *src_ptr_reg + AES_BLOCK_SIZE);
+        *dst_ptr_reg = deposit64(*dst_ptr_reg, 0, addr_reg_size,
+                                 *dst_ptr_reg + AES_BLOCK_SIZE);
+        *ctr_ptr_reg = deposit64(*ctr_ptr_reg, 0, addr_reg_size,
+                                 *ctr_ptr_reg + AES_BLOCK_SIZE);
+        *src_len_reg -= AES_BLOCK_SIZE;
         len -= AES_BLOCK_SIZE;
-        done += AES_BLOCK_SIZE;
     }
 
-    *src_ptr_reg = deposit64(*src_ptr_reg, 0, addr_reg_size,
-                             *src_ptr_reg + done);
-    *dst_ptr_reg = deposit64(*dst_ptr_reg, 0, addr_reg_size,
-                             *dst_ptr_reg + done);
-    *ctr_ptr_reg = deposit64(*ctr_ptr_reg, 0, addr_reg_size,
-                             *ctr_ptr_reg + done);
-    *src_len_reg -= done;
-
     return !len ? 0 : 3;
 }
 
@@ -888,8 +1012,9 @@ int cpacf_paes_xts(CPUS390XState *env, const int mmu_idx, uintptr_t ra,
     enum { MAX_BLOCKS_PER_RUN = 8192 / AES_BLOCK_SIZE };
     uint8_t in[AES_BLOCK_SIZE], out[AES_BLOCK_SIZE];
     uint8_t key[32], wkvp[32], tweak[AES_BLOCK_SIZE];
-    uint64_t len = *src_len_reg, done = 0;
+    S390Access srca, dsta, tweaka;
     int i, keysize, addr_reg_size;
+    uint64_t len = *src_len_reg;
     AES_KEY exkey;
 
     g_assert(type == S390_FEAT_TYPE_KM);
@@ -954,8 +1079,22 @@ int cpacf_paes_xts(CPUS390XState *env, const int mmu_idx, uintptr_t ra,
 
     /* process up to MAX_BLOCKS_PER_RUN aes blocks */
     for (i = 0; i < MAX_BLOCKS_PER_RUN && len >= AES_BLOCK_SIZE; i++) {
-        /* fetch one AES block into in */
-        aes_read_block(env, mmu_idx, ra, *src_ptr_reg + done, in);
+        /* probe source, destination, and tweak access */
+        aes_prepare_block(env, mmu_idx, ra, &srca, *src_ptr_reg,
+                          MMU_DATA_LOAD);
+        aes_prepare_block(env, mmu_idx, ra, &dsta, *dst_ptr_reg,
+                          MMU_DATA_STORE);
+        aes_prepare_block(env, mmu_idx, ra, &tweaka,
+                          param_addr + keysize + sizeof(wkvp),
+                          MMU_DATA_STORE);
+
+        /*
+         * linux-user: the accessors below use host pointers. Record ra so a
+         * fault from a racing unmap of the probed page is unwound correctly.
+         */
+        set_helper_retaddr(ra);
+
+        aes_read_block(env, ra, &srca, in);
         if (mod) {
             /* decrypt in => out */
             AES_xts_decrypt(in, out, tweak, &exkey);
@@ -965,22 +1104,21 @@ int cpacf_paes_xts(CPUS390XState *env, const int mmu_idx, uintptr_t ra,
         }
         /* prep tweak for next round */
         AES_xts_prep_next_tweak(tweak);
-        /* write out this processed block from out */
-        aes_write_block(env, mmu_idx, ra, *dst_ptr_reg + done, out);
+        /* scatter block to destination */
+        aes_write_block(env, ra, &dsta, out);
+        /* write back running tweak */
+        aes_write_block(env, ra, &tweaka, tweak);
+
+        clear_helper_retaddr();
+
+        /* update registers at end of each iteration */
+        *src_ptr_reg = deposit64(*src_ptr_reg, 0, addr_reg_size,
+                                 *src_ptr_reg + AES_BLOCK_SIZE);
+        *dst_ptr_reg = deposit64(*dst_ptr_reg, 0, addr_reg_size,
+                                 *dst_ptr_reg + AES_BLOCK_SIZE);
+        *src_len_reg -= AES_BLOCK_SIZE;
         len -= AES_BLOCK_SIZE;
-        done += AES_BLOCK_SIZE;
     }
 
-    /* update tweak in param block */
-    write_guest_wrap_u8(env, mmu_idx, ra,
-                        param_addr + keysize + sizeof(wkvp),
-                        tweak, AES_BLOCK_SIZE);
-
-    *src_ptr_reg = deposit64(*src_ptr_reg, 0, addr_reg_size,
-                             *src_ptr_reg + done);
-    *dst_ptr_reg = deposit64(*dst_ptr_reg, 0, addr_reg_size,
-                             *dst_ptr_reg + done);
-    *src_len_reg -= done;
-
     return !len ? 0 : 3;
 }
-- 
2.56.0



  parent reply	other threads:[~2026-11-20 13:16 UTC|newest]

Thread overview: 4+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-11-20 13:14 [PATCH v2 0/3] Fix AES/PAES CPACF register update on faults Andrew Holden
2026-11-20 13:14 ` [PATCH v2 1/3] target/s390x: export access_prepare() and byte accessors Andrew Holden
2026-11-20 13:14 ` Andrew Holden [this message]
2026-11-20 13:14 ` [PATCH v2 3/3] tests/s390x/tcg: Add cpacf clear key register update tests Andrew Holden

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20261120131443.61606-3-andy0x48@linux.ibm.com \
    --to=andy0x48@linux.ibm.com \
    --cc=cohuck@redhat.com \
    --cc=david@kernel.org \
    --cc=farman@linux.ibm.com \
    --cc=freude@linux.ibm.com \
    --cc=iii@linux.ibm.com \
    --cc=mjrosato@linux.ibm.com \
    --cc=qemu-devel@nongnu.org \
    --cc=qemu-s390x@nongnu.org \
    --cc=richard.henderson@linaro.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.