[PATCH v2 00/37] target/arm: Implement FEAT_SYSREG128

[PATCH v2 00/37] target/arm: Implement FEAT_SYSREG128

[Richard Henderson]

Based-on: 20251014195017.421681-1-richard.henderson@linaro.org
("[PATCH v2 0/7] target/arm: Implement FEAT_AIE")

Changes for v2:
  - The VHE rewrite has been merged.
  - The ARMCPU data are two separate uint64_t not Int128.
  - The write128fn callback uses two separate uint64_t arguments.
  - The read128fn callback continues to return Int128.
  - Updated arm_gdb_get_sysreg for 128-bit registers.

The final patch may be used to test all of this, but is not intended
for merge.  All of this should enabled if and only if FEAT_D128.


r~


Richard Henderson (37):
  target/arm: Implement isar tests for FEAT_SYSREG128, FEAT_SYSINSTR128
  target/arm: Define CP_REG_SIZE_U128
  target/arm: Update ARMCPRegInfo for 128-bit sysregs
  target/arm: Asserts for ARM_CP_128BIT in define_one_arm_cp_reg
  target/arm: Split add_cpreg_to_hashtable_aa64
  target/arm: Add raw_read128, raw_write128
  target/arm: Add read_raw_cp_reg128, write_raw_cp_reg128
  target/arm: Use cpreg_field_type in arm_gen_one_feature_sysreg
  target/arm: Merge arm_gen_one_feature_sysreg into the single caller
  target/arm: Handle MO_128 in arm_gdb_get_sysreg
  target/arm: Handle ARM_CP_128BIT in cpu reset
  target/arm: Put 128-bit sysregs into a separate list
  target/arm/kvm: Assert no 128-bit sysregs in kvm_arm_init_cpreg_list
  target/arm/hvf: Assert no 128-bit sysregs in hvf_arch_init_vcpu
  migration: Add vmstate_info_int128
  target/arm: Migrate cpreg128 registers
  target/arm: Add syn_aa64_sysreg128trap
  target/arm: Introduce helper_{get,set}_cp_reg128
  target/arm: Implement MRRS, MSRR, SYSP
  target/arm: Consolidate definitions of PAR
  target/arm: Extend PAR_EL1 to 128-bit
  target/arm: Consolidate definitions of TTBR[01]
  target/arm: Split out flush_if_asid_change
  target/arm: Use flush_if_asid_change in vmsa_ttbr_write
  target/arm: Extend TTBR system registers to 128-bit
  target/arm: Implement TLBIP IPAS2E1, IPAS2LE1
  target/arm: Implement TLBIP IPAS2E1IS, IPAS2LE1IS
  target/arm: Implement TLBIP RVAE1, RVAAE1, RVALE1, RVAALE1
  target/arm: Implement TLBIP RIPAS1E1, RIPAS1LE1, RIPAS2E1IS,
    RIPAS2LE1IS
  target/arm: Implement TLBIP RVA{L}E2{IS,OS}
  target/arm: Implement TLBIP RVA{L}E3{IS,OS}
  target/arm: Implement TLBIP VA{L}E1{IS,OS}
  target/arm: Implement TLBIP VAE2, VALE2
  target/arm: Implement TLBIP VAE3, VALE3
  target/arm: Implement TLBIP VA{L}E2{IS,OS}
  target/arm: Implement TLBIP VA{L}E3{IS,OS}
  NOTFORMERGE: Enable FEAT_SYSREG128, FEAT_SYSINSTR128 for cpu max

 include/migration/vmstate.h    |   1 +
 target/arm/cpregs.h            |  31 ++
 target/arm/cpu-features.h      |  10 +
 target/arm/cpu.h               |  33 +-
 target/arm/internals.h         |   1 +
 target/arm/kvm-consts.h        |   2 +
 target/arm/syndrome.h          |  10 +
 target/arm/tcg/helper.h        |   2 +
 migration/vmstate-types.c      |  30 ++
 target/arm/cpu.c               |  20 +-
 target/arm/gdbstub.c           |  87 +++--
 target/arm/helper.c            | 663 ++++++++++++++++++++++++++-------
 target/arm/hvf/hvf.c           |   4 +
 target/arm/kvm.c               |   2 +
 target/arm/machine.c           |  50 +++
 target/arm/tcg/cpregs-at.c     |   4 +
 target/arm/tcg/cpu64.c         |   2 +
 target/arm/tcg/op_helper.c     |  29 ++
 target/arm/tcg/tlb-insns.c     | 494 ++++++++++++++++++------
 target/arm/tcg/translate-a64.c | 169 +++++++--
 target/arm/tcg/a64.decode      |  12 +-
 21 files changed, 1312 insertions(+), 344 deletions(-)

-- 
2.43.0

[PATCH v2 01/37] target/arm: Implement isar tests for FEAT_SYSREG128, FEAT_SYSINSTR128

[Richard Henderson]

Reviewed-by: Manos Pitsidianakis <manos.pitsidianakis@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/cpu-features.h | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/target/arm/cpu-features.h b/target/arm/cpu-features.h
index 579fa8f8f4..9b7fdde0d7 100644
--- a/target/arm/cpu-features.h
+++ b/target/arm/cpu-features.h
@@ -1019,6 +1019,16 @@ static inline bool isar_feature_aa64_mops(const ARMISARegisters *id)
     return FIELD_EX64_IDREG(id, ID_AA64ISAR2, MOPS);
 }
 
+static inline bool isar_feature_aa64_sysreg128(const ARMISARegisters *id)
+{
+    return FIELD_EX64_IDREG(id, ID_AA64ISAR2, SYSREG_128) != 0;
+}
+
+static inline bool isar_feature_aa64_sysinstr128(const ARMISARegisters *id)
+{
+    return FIELD_EX64_IDREG(id, ID_AA64ISAR2, SYSINSTR_128) != 0;
+}
+
 static inline bool isar_feature_aa64_rpres(const ARMISARegisters *id)
 {
     return FIELD_EX64_IDREG(id, ID_AA64ISAR2, RPRES);
-- 
2.43.0

[PATCH v2 02/37] target/arm: Define CP_REG_SIZE_U128

[Richard Henderson]

Reviewed-by: Manos Pitsidianakis <manos.pitsidianakis@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/kvm-consts.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/target/arm/kvm-consts.h b/target/arm/kvm-consts.h
index 54ae5da7ce..1b3c10c0d3 100644
--- a/target/arm/kvm-consts.h
+++ b/target/arm/kvm-consts.h
@@ -30,6 +30,7 @@
 #define CP_REG_SIZE_MASK       0x00f0000000000000ULL
 #define CP_REG_SIZE_U32        0x0020000000000000ULL
 #define CP_REG_SIZE_U64        0x0030000000000000ULL
+#define CP_REG_SIZE_U128       0x0040000000000000ULL
 #define CP_REG_ARM             0x4000000000000000ULL
 #define CP_REG_ARCH_MASK       0xff00000000000000ULL
 
@@ -37,6 +38,7 @@ MISMATCH_CHECK(CP_REG_SIZE_SHIFT, KVM_REG_SIZE_SHIFT);
 MISMATCH_CHECK(CP_REG_SIZE_MASK, KVM_REG_SIZE_MASK);
 MISMATCH_CHECK(CP_REG_SIZE_U32, KVM_REG_SIZE_U32);
 MISMATCH_CHECK(CP_REG_SIZE_U64, KVM_REG_SIZE_U64);
+MISMATCH_CHECK(CP_REG_SIZE_U128, KVM_REG_SIZE_U128);
 MISMATCH_CHECK(CP_REG_ARM, KVM_REG_ARM);
 MISMATCH_CHECK(CP_REG_ARCH_MASK, KVM_REG_ARCH_MASK);
 
-- 
2.43.0

[PATCH v2 03/37] target/arm: Update ARMCPRegInfo for 128-bit sysregs

[Richard Henderson]

Add ARM_CP_128BIT for type and functions describing 128-bit access.

Reviewed-by: Manos Pitsidianakis <manos.pitsidianakis@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/cpregs.h | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/target/arm/cpregs.h b/target/arm/cpregs.h
index 48a406a5fb..e1d54baead 100644
--- a/target/arm/cpregs.h
+++ b/target/arm/cpregs.h
@@ -149,6 +149,8 @@ enum {
      * should not trap to EL2 when HCR_EL2.NV is set.
      */
     ARM_CP_NV_NO_TRAP            = 1 << 22,
+    /* Flag: For ARM_CP_STATE_AA64, sysreg is 128-bit. */
+    ARM_CP_128BIT                = 1 << 23,
 };
 
 /*
@@ -191,6 +193,10 @@ enum {
 #define CP_REG_AA32_NS_SHIFT     29
 #define CP_REG_AA32_NS_MASK      (1 << CP_REG_AA32_NS_SHIFT)
 
+/* Distinguish 64-bit and 128-bit views of AArch64 system registers. */
+#define CP_REG_AA64_128BIT_SHIFT 30
+#define CP_REG_AA64_128BIT_MASK  (1 << CP_REG_AA64_128BIT_SHIFT)
+
 /* Distinguish 32-bit and 64-bit views of AArch32 system registers. */
 #define CP_REG_AA32_64BIT_SHIFT  15
 #define CP_REG_AA32_64BIT_MASK   (1 << CP_REG_AA32_64BIT_SHIFT)
@@ -903,6 +909,9 @@ typedef struct ARMCPRegInfo ARMCPRegInfo;
 typedef uint64_t CPReadFn(CPUARMState *env, const ARMCPRegInfo *ri);
 typedef void CPWriteFn(CPUARMState *env, const ARMCPRegInfo *ri,
                        uint64_t value);
+typedef Int128 CPRead128Fn(CPUARMState *env, const ARMCPRegInfo *opaque);
+typedef void CPWrite128Fn(CPUARMState *env, const ARMCPRegInfo *opaque,
+                          uint64_t valuelo, uint64_t valuehi);
 /* Access permission check functions for coprocessor registers. */
 typedef CPAccessResult CPAccessFn(CPUARMState *env,
                                   const ARMCPRegInfo *ri,
@@ -991,6 +1000,11 @@ struct ARMCPRegInfo {
      *  2. both readfn and writefn are specified
      */
     ptrdiff_t fieldoffset; /* offsetof(CPUARMState, field) */
+    /*
+     * Offset of the high 64-bits of the field in CPUARMState.
+     * Similarly, may be omitted if read128fn and write128fn are set.
+     */
+    ptrdiff_t fieldoffsethi;
 
     /*
      * Offsets of the secure and non-secure fields in CPUARMState for the
@@ -1046,6 +1060,13 @@ struct ARMCPRegInfo {
      * fieldoffset is 0 then no reset will be done.
      */
     CPResetFn *resetfn;
+
+    /* For ARM_CP_128BIT, when accessed via MRRS/MSRR. */
+    CPAccessFn *access128fn;
+    CPRead128Fn *read128fn;
+    CPWrite128Fn *write128fn;
+    CPRead128Fn *raw_read128fn;
+    CPWrite128Fn *raw_write128fn;
 };
 
 void define_one_arm_cp_reg(ARMCPU *cpu, const ARMCPRegInfo *regs);
@@ -1115,6 +1136,9 @@ void arm_cp_reset_ignore(CPUARMState *env, const ARMCPRegInfo *ri);
  */
 static inline MemOp cpreg_field_type(const ARMCPRegInfo *ri)
 {
+    if (ri->type & ARM_CP_128BIT) {
+        return MO_128;
+    }
     return (ri->state == ARM_CP_STATE_AA64 || (ri->type & ARM_CP_64BIT)
             ? MO_64 : MO_32);
 }
-- 
2.43.0

[PATCH v2 04/37] target/arm: Asserts for ARM_CP_128BIT in define_one_arm_cp_reg

[Richard Henderson]

128-bit registers are aarch64 only; disallow non-zero reset values.

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/helper.c | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/target/arm/helper.c b/target/arm/helper.c
index 8c0b8889db..3b06704963 100644
--- a/target/arm/helper.c
+++ b/target/arm/helper.c
@@ -7805,6 +7805,22 @@ void define_one_arm_cp_reg(ARMCPU *cpu, const ARMCPRegInfo *r)
     assert(r->state == ARM_CP_STATE_AA32 || !(r->type & ARM_CP_64BIT));
     /* AArch32 64-bit registers have only CRm and Opc1 fields. */
     assert(!(r->type & ARM_CP_64BIT) || !(r->opc2 || r->crn));
+    if (r->type & ARM_CP_128BIT) {
+        /*
+         * Only AArch64 regs are 128-bit.  There is usually an AArch32 64-bit
+         * register aliasing the low half, which must be defined separately due
+         * to encoding conflicts above.
+         */
+        assert(r->state == ARM_CP_STATE_AA64);
+        /*
+         * All 128-bit regs are UNKNOWN at reset, so there's no need
+         * for either resetvalue or resetfn.  For those EL2 registers
+         * that become CONST RES0 for EL3 with EL2 disabled, we allow
+         * ARM_CP_CONST, but only with value 0.
+         */
+        assert(r->resetvalue == 0);
+        assert(r->resetfn == NULL);
+    }
     /* op0 only exists in the AArch64 encodings */
     assert(r->state != ARM_CP_STATE_AA32 || r->opc0 == 0);
 
-- 
2.43.0

[PATCH v2 05/37] target/arm: Split add_cpreg_to_hashtable_aa64

[Richard Henderson]

Rename the existing add_cpreg_to_hashtable_aa64 as *_1.
Introduce a new add_cpreg_to_hashtable_aa64 that handles
128-bit and 64-bit views of an AArch64 system register.

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/helper.c | 61 ++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 57 insertions(+), 4 deletions(-)

diff --git a/target/arm/helper.c b/target/arm/helper.c
index 3b06704963..c240edf182 100644
--- a/target/arm/helper.c
+++ b/target/arm/helper.c
@@ -7668,11 +7668,9 @@ static void add_cpreg_to_hashtable_aa32(ARMCPU *cpu, ARMCPRegInfo *r)
     }
 }
 
-static void add_cpreg_to_hashtable_aa64(ARMCPU *cpu, ARMCPRegInfo *r)
+static void add_cpreg_to_hashtable_aa64_1(ARMCPU *cpu, ARMCPRegInfo *r,
+                                          uint32_t key)
 {
-    uint32_t key = ENCODE_AA64_CP_REG(r->opc0, r->opc1,
-                                      r->crn, r->crm, r->opc2);
-
     if ((r->type & ARM_CP_ADD_TLBI_NXS) &&
         cpu_isar_feature(aa64_xs, cpu)) {
         /*
@@ -7740,6 +7738,10 @@ static void add_cpreg_to_hashtable_aa64(ARMCPU *cpu, ARMCPRegInfo *r)
         r2->writefn = NULL;
         r2->raw_readfn = NULL;
         r2->raw_writefn = NULL;
+        r2->read128fn = NULL;
+        r2->write128fn = NULL;
+        r2->raw_read128fn = NULL;
+        r2->raw_write128fn = NULL;
         r2->accessfn = NULL;
         r2->fieldoffset = 0;
 
@@ -7761,6 +7763,57 @@ static void add_cpreg_to_hashtable_aa64(ARMCPU *cpu, ARMCPRegInfo *r)
                            ARM_CP_SECSTATE_NS, key);
 }
 
+static void add_cpreg_to_hashtable_aa64(ARMCPU *cpu, ARMCPRegInfo *r)
+{
+    uint32_t key64 = ENCODE_AA64_CP_REG(r->opc0, r->opc1,
+                                        r->crn, r->crm, r->opc2);
+
+    /*
+     * All 128-bit system registers and instructions have 64-bit aliases.
+     * If the 128-bit feature is enabled, create a duplicate.
+     */
+    if (r->type & ARM_CP_128BIT) {
+        if (cpu_isar_feature(aa64_sysreg128, cpu) ||
+            cpu_isar_feature(aa64_sysinstr128, cpu)) {
+            ARMCPRegInfo *r128 = alloc_cpreg(r, NULL);
+            uint32_t key128 = key64 | CP_REG_AA64_128BIT_MASK;
+
+            r128->accessfn = r128->access128fn;
+            r128->access128fn = NULL;
+            r128->readfn = NULL;
+            r128->writefn = NULL;
+            r128->raw_readfn = NULL;
+            r128->raw_writefn = NULL;
+
+            if (r128->vhe_redir_to_el2) {
+                r128->vhe_redir_to_el2 |= CP_REG_AA64_128BIT_MASK;
+            }
+            if (r128->vhe_redir_to_el01) {
+                r128->vhe_redir_to_el01 |= CP_REG_AA64_128BIT_MASK;
+            }
+
+            add_cpreg_to_hashtable_aa64_1(cpu, r128, key128);
+
+            /*
+             * The 128-bit definition is the canonical view.
+             * The 64-bit definition is an alias, hidden from gdb.
+             */
+            r->type |= ARM_CP_ALIAS | ARM_CP_NO_GDB;
+        }
+
+        /* Squash the original to create the 64-bit view. */
+        r->type &= ~ARM_CP_128BIT;
+        r->access128fn = NULL;
+        r->read128fn = NULL;
+        r->write128fn = NULL;
+        r->raw_read128fn = NULL;
+        r->raw_write128fn = NULL;
+        r->fieldoffsethi = 0;
+    }
+
+    add_cpreg_to_hashtable_aa64_1(cpu, r, key64);
+}
+
 void define_one_arm_cp_reg(ARMCPU *cpu, const ARMCPRegInfo *r)
 {
     /*
-- 
2.43.0

[PATCH v2 06/37] target/arm: Add raw_read128, raw_write128

[Richard Henderson]

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/cpregs.h |  6 ++++++
 target/arm/helper.c | 21 +++++++++++++++++++++
 2 files changed, 27 insertions(+)

diff --git a/target/arm/cpregs.h b/target/arm/cpregs.h
index e1d54baead..0b0004eff9 100644
--- a/target/arm/cpregs.h
+++ b/target/arm/cpregs.h
@@ -21,6 +21,7 @@
 #ifndef TARGET_ARM_CPREGS_H
 #define TARGET_ARM_CPREGS_H
 
+#include "qemu/int128.h"
 #include "hw/registerfields.h"
 #include "exec/memop.h"
 #include "target/arm/kvm-consts.h"
@@ -1124,6 +1125,11 @@ uint64_t raw_read(CPUARMState *env, const ARMCPRegInfo *ri);
 /* CPWriteFn that just writes the value to ri->fieldoffset */
 void raw_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value);
 
+/* Likewise for 128-bit fields. */
+Int128 raw_read128(CPUARMState *env, const ARMCPRegInfo *opaque);
+void raw_write128(CPUARMState *env, const ARMCPRegInfo *opaque,
+                  uint64_t lo, uint64_t hi);
+
 /*
  * CPResetFn that does nothing, for use if no reset is required even
  * if fieldoffset is non zero.
diff --git a/target/arm/helper.c b/target/arm/helper.c
index c240edf182..e321f404e6 100644
--- a/target/arm/helper.c
+++ b/target/arm/helper.c
@@ -59,6 +59,8 @@ int compare_u64(const void *a, const void *b)
     (*(uint32_t *)((char *)(env) + (ri)->fieldoffset))
 #define CPREG_FIELD64(env, ri) \
     (*(uint64_t *)((char *)(env) + (ri)->fieldoffset))
+#define CPREG_FIELD128_H64(env, ri) \
+    (*(uint64_t *)((char *)(env) + (ri)->fieldoffsethi))
 
 uint64_t raw_read(CPUARMState *env, const ARMCPRegInfo *ri)
 {
@@ -88,8 +90,27 @@ void raw_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
     }
 }
 
+Int128 raw_read128(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    assert(ri->type & ARM_CP_128BIT);
+    assert(ri->fieldoffset);
+    assert(ri->fieldoffsethi);
+    return int128_make128(CPREG_FIELD64(env, ri), CPREG_FIELD128_H64(env, ri));
+}
+
+void raw_write128(CPUARMState *env, const ARMCPRegInfo *ri,
+                  uint64_t valuelo, uint64_t valuehi)
+{
+    assert(ri->type & ARM_CP_128BIT);
+    assert(ri->fieldoffset);
+    assert(ri->fieldoffsethi);
+    CPREG_FIELD64(env, ri) = valuelo;
+    CPREG_FIELD128_H64(env, ri) = valuehi;
+}
+
 #undef CPREG_FIELD32
 #undef CPREG_FIELD64
+#undef CPREG_FIELD128_H64
 
 static void *raw_ptr(CPUARMState *env, const ARMCPRegInfo *ri)
 {
-- 
2.43.0

[PATCH v2 07/37] target/arm: Add read_raw_cp_reg128, write_raw_cp_reg128

[Richard Henderson]

Add the functions and update raw_accessors_invalid to match.
Add assertions for !ARM_CP_128BIT in read_raw_cp_reg and
write_raw_cp_reg.

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/cpregs.h |  1 +
 target/arm/helper.c | 49 +++++++++++++++++++++++++++++++++++++++++----
 2 files changed, 46 insertions(+), 4 deletions(-)

diff --git a/target/arm/cpregs.h b/target/arm/cpregs.h
index 0b0004eff9..f6658abc57 100644
--- a/target/arm/cpregs.h
+++ b/target/arm/cpregs.h
@@ -1157,6 +1157,7 @@ static inline bool cp_access_ok(int current_el,
 
 /* Raw read of a coprocessor register (as needed for migration, etc) */
 uint64_t read_raw_cp_reg(CPUARMState *env, const ARMCPRegInfo *ri);
+Int128 read_raw_cp_reg128(CPUARMState *env, const ARMCPRegInfo *ri);
 
 /*
  * Return true if the cp register encoding is in the "feature ID space" as
diff --git a/target/arm/helper.c b/target/arm/helper.c
index e321f404e6..d9d8ae56e8 100644
--- a/target/arm/helper.c
+++ b/target/arm/helper.c
@@ -119,6 +119,7 @@ static void *raw_ptr(CPUARMState *env, const ARMCPRegInfo *ri)
 
 uint64_t read_raw_cp_reg(CPUARMState *env, const ARMCPRegInfo *ri)
 {
+    assert(!(ri->type & ARM_CP_128BIT));
     /* Raw read of a coprocessor register (as needed for migration, etc). */
     if (ri->type & ARM_CP_CONST) {
         return ri->resetvalue;
@@ -134,6 +135,7 @@ uint64_t read_raw_cp_reg(CPUARMState *env, const ARMCPRegInfo *ri)
 static void write_raw_cp_reg(CPUARMState *env, const ARMCPRegInfo *ri,
                              uint64_t v)
 {
+    assert(!(ri->type & ARM_CP_128BIT));
     /*
      * Raw write of a coprocessor register (as needed for migration, etc).
      * Note that constant registers are treated as write-ignored; the
@@ -151,6 +153,35 @@ static void write_raw_cp_reg(CPUARMState *env, const ARMCPRegInfo *ri,
     }
 }
 
+Int128 read_raw_cp_reg128(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    assert(ri->type & ARM_CP_128BIT);
+    if (ri->raw_read128fn) {
+        return ri->raw_read128fn(env, ri);
+    } else if (ri->read128fn) {
+        return ri->read128fn(env, ri);
+    } else {
+        return raw_read128(env, ri);
+    }
+}
+
+__attribute__((unused))
+static void write_raw_cp_reg128(CPUARMState *env, const ARMCPRegInfo *ri,
+                                Int128 v)
+{
+    uint64_t lo = int128_getlo(v);
+    uint64_t hi = int128_gethi(v);
+
+    assert(ri->type & ARM_CP_128BIT);
+    if (ri->raw_write128fn) {
+        ri->raw_write128fn(env, ri, lo, hi);
+    } else if (ri->write128fn) {
+        ri->write128fn(env, ri, lo, hi);
+    } else {
+        raw_write128(env, ri, lo, hi);
+    }
+}
+
 static bool raw_accessors_invalid(const ARMCPRegInfo *ri)
 {
    /*
@@ -165,12 +196,22 @@ static bool raw_accessors_invalid(const ARMCPRegInfo *ri)
     * The tests here line up with the conditions in read/write_raw_cp_reg()
     * and assertions in raw_read()/raw_write().
     */
-    if ((ri->type & ARM_CP_CONST) ||
-        ri->fieldoffset ||
-        ((ri->raw_writefn || ri->writefn) && (ri->raw_readfn || ri->readfn))) {
+    if (ri->type & ARM_CP_CONST) {
         return false;
     }
-    return true;
+    if (ri->fieldoffset) {
+        return false;
+    }
+    if (ri->type & ARM_CP_128BIT) {
+        if (ri->fieldoffsethi) {
+            return false;
+        }
+        return !((ri->raw_write128fn || ri->write128fn) &&
+                 (ri->raw_read128fn || ri->read128fn));
+    } else {
+        return !((ri->raw_writefn || ri->writefn) &&
+                 (ri->raw_readfn || ri->readfn));
+    }
 }
 
 bool write_cpustate_to_list(ARMCPU *cpu, bool kvm_sync)
-- 
2.43.0

[PATCH v2 08/37] target/arm: Use cpreg_field_type in arm_gen_one_feature_sysreg

[Richard Henderson]

Deduce bitsize from cpreg_field_type.  With this, we can
simplify arm_register_sysreg_for_feature.

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/gdbstub.c | 40 +++++++++++++++++++---------------------
 1 file changed, 19 insertions(+), 21 deletions(-)

diff --git a/target/arm/gdbstub.c b/target/arm/gdbstub.c
index 8d2229f519..89324e4ae9 100644
--- a/target/arm/gdbstub.c
+++ b/target/arm/gdbstub.c
@@ -274,8 +274,9 @@ static int arm_gdb_set_sysreg(CPUState *cs, uint8_t *buf, int reg)
 static void arm_gen_one_feature_sysreg(GDBFeatureBuilder *builder,
                                        DynamicGDBFeatureInfo *dyn_feature,
                                        ARMCPRegInfo *ri, uint32_t ri_key,
-                                       int bitsize, int n)
+                                       int n)
 {
+    int bitsize = 8 << cpreg_field_type(ri);
     gdb_feature_builder_append_reg(builder, ri->name, bitsize, n,
                                    "int", "cp_regs");
 
@@ -292,28 +293,25 @@ static void arm_register_sysreg_for_feature(gpointer key, gpointer value,
     CPUARMState *env = &cpu->env;
     DynamicGDBFeatureInfo *dyn_feature = &cpu->dyn_sysreg_feature;
 
-    if (!(ri->type & (ARM_CP_NO_RAW | ARM_CP_NO_GDB))) {
-        if (arm_feature(env, ARM_FEATURE_AARCH64)) {
-            if (ri->state == ARM_CP_STATE_AA64) {
-                arm_gen_one_feature_sysreg(&param->builder, dyn_feature,
-                                           ri, ri_key, 64, param->n++);
-            }
-        } else {
-            if (ri->state == ARM_CP_STATE_AA32) {
-                if (!arm_feature(env, ARM_FEATURE_EL3) &&
-                    (ri->secure & ARM_CP_SECSTATE_S)) {
-                    return;
-                }
-                if (ri->type & ARM_CP_64BIT) {
-                    arm_gen_one_feature_sysreg(&param->builder, dyn_feature,
-                                               ri, ri_key, 64, param->n++);
-                } else {
-                    arm_gen_one_feature_sysreg(&param->builder, dyn_feature,
-                                               ri, ri_key, 32, param->n++);
-                }
-            }
+    if (ri->type & (ARM_CP_NO_RAW | ARM_CP_NO_GDB)) {
+        return;
+    }
+    if (arm_feature(env, ARM_FEATURE_AARCH64)) {
+        if (ri->state != ARM_CP_STATE_AA64) {
+            return;
+        }
+    } else {
+        if (ri->state != ARM_CP_STATE_AA32) {
+            return;
+        }
+        if (!arm_feature(env, ARM_FEATURE_EL3)
+            && (ri->secure & ARM_CP_SECSTATE_S)) {
+            return;
         }
     }
+
+    arm_gen_one_feature_sysreg(&param->builder, dyn_feature,
+                               ri, ri_key, param->n++);
 }
 
 static GDBFeature *arm_gen_dynamic_sysreg_feature(CPUState *cs, int base_reg)
-- 
2.43.0

[PATCH v2 09/37] target/arm: Merge arm_gen_one_feature_sysreg into the single caller

[Richard Henderson]

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/gdbstub.c | 19 ++++---------------
 1 file changed, 4 insertions(+), 15 deletions(-)

diff --git a/target/arm/gdbstub.c b/target/arm/gdbstub.c
index 89324e4ae9..8583057b58 100644
--- a/target/arm/gdbstub.c
+++ b/target/arm/gdbstub.c
@@ -271,18 +271,6 @@ static int arm_gdb_set_sysreg(CPUState *cs, uint8_t *buf, int reg)
     return 0;
 }
 
-static void arm_gen_one_feature_sysreg(GDBFeatureBuilder *builder,
-                                       DynamicGDBFeatureInfo *dyn_feature,
-                                       ARMCPRegInfo *ri, uint32_t ri_key,
-                                       int n)
-{
-    int bitsize = 8 << cpreg_field_type(ri);
-    gdb_feature_builder_append_reg(builder, ri->name, bitsize, n,
-                                   "int", "cp_regs");
-
-    dyn_feature->data.cpregs.keys[n] = ri_key;
-}
-
 static void arm_register_sysreg_for_feature(gpointer key, gpointer value,
                                             gpointer p)
 {
@@ -291,7 +279,6 @@ static void arm_register_sysreg_for_feature(gpointer key, gpointer value,
     RegisterSysregFeatureParam *param = p;
     ARMCPU *cpu = ARM_CPU(param->cs);
     CPUARMState *env = &cpu->env;
-    DynamicGDBFeatureInfo *dyn_feature = &cpu->dyn_sysreg_feature;
 
     if (ri->type & (ARM_CP_NO_RAW | ARM_CP_NO_GDB)) {
         return;
@@ -310,8 +297,10 @@ static void arm_register_sysreg_for_feature(gpointer key, gpointer value,
         }
     }
 
-    arm_gen_one_feature_sysreg(&param->builder, dyn_feature,
-                               ri, ri_key, param->n++);
+    gdb_feature_builder_append_reg(&param->builder, ri->name,
+                                   8 << cpreg_field_type(ri),
+                                   param->n, "int", "cp_regs");
+    cpu->dyn_sysreg_feature.data.cpregs.keys[param->n++] = ri_key;
 }
 
 static GDBFeature *arm_gen_dynamic_sysreg_feature(CPUState *cs, int base_reg)
-- 
2.43.0

[PATCH v2 10/37] target/arm: Handle MO_128 in arm_gdb_get_sysreg

[Richard Henderson]

Handle gdb reads of 128-bit system registers.

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/gdbstub.c | 44 +++++++++++++++++++++++++++++++++-----------
 1 file changed, 33 insertions(+), 11 deletions(-)

diff --git a/target/arm/gdbstub.c b/target/arm/gdbstub.c
index 8583057b58..f25bd50b71 100644
--- a/target/arm/gdbstub.c
+++ b/target/arm/gdbstub.c
@@ -247,15 +247,20 @@ static int arm_gdb_get_sysreg(CPUState *cs, GByteArray *buf, int reg)
     key = cpu->dyn_sysreg_feature.data.cpregs.keys[reg];
     ri = get_arm_cp_reginfo(cpu->cp_regs, key);
     if (ri) {
+        if (ri->vhe_redir_to_el2 &&
+            (arm_hcr_el2_eff(env) & HCR_E2H) &&
+            arm_current_el(env) == 2) {
+            ri = get_arm_cp_reginfo(cpu->cp_regs, ri->vhe_redir_to_el2);
+        } else if (ri->vhe_redir_to_el01) {
+            ri = get_arm_cp_reginfo(cpu->cp_regs, ri->vhe_redir_to_el01);
+        }
         switch (cpreg_field_type(ri)) {
-        case MO_64:
-            if (ri->vhe_redir_to_el2 &&
-                (arm_hcr_el2_eff(env) & HCR_E2H) &&
-                arm_current_el(env) == 2) {
-                ri = get_arm_cp_reginfo(cpu->cp_regs, ri->vhe_redir_to_el2);
-            } else if (ri->vhe_redir_to_el01) {
-                ri = get_arm_cp_reginfo(cpu->cp_regs, ri->vhe_redir_to_el01);
+        case MO_128:
+            {
+                Int128 v = read_raw_cp_reg128(env, ri);
+                return gdb_get_reg128(buf, int128_gethi(v), int128_getlo(v));
             }
+        case MO_64:
             return gdb_get_reg64(buf, (uint64_t)read_raw_cp_reg(env, ri));
         case MO_32:
             return gdb_get_reg32(buf, (uint32_t)read_raw_cp_reg(env, ri));
@@ -279,6 +284,7 @@ static void arm_register_sysreg_for_feature(gpointer key, gpointer value,
     RegisterSysregFeatureParam *param = p;
     ARMCPU *cpu = ARM_CPU(param->cs);
     CPUARMState *env = &cpu->env;
+    int bitsize, n;
 
     if (ri->type & (ARM_CP_NO_RAW | ARM_CP_NO_GDB)) {
         return;
@@ -297,10 +303,26 @@ static void arm_register_sysreg_for_feature(gpointer key, gpointer value,
         }
     }
 
-    gdb_feature_builder_append_reg(&param->builder, ri->name,
-                                   8 << cpreg_field_type(ri),
-                                   param->n, "int", "cp_regs");
-    cpu->dyn_sysreg_feature.data.cpregs.keys[param->n++] = ri_key;
+    n = param->n++;
+    bitsize = 8 << cpreg_field_type(ri);
+    /*
+     * GDB generates an error for type="int" and bitsize=128.
+     * We need to use type="uint128" instead.
+     */
+    switch (bitsize) {
+    case 128:
+        gdb_feature_builder_append_reg(&param->builder, ri->name,
+                                       bitsize, n, "uint128", "cp_regs");
+        break;
+    case 64:
+    case 32:
+        gdb_feature_builder_append_reg(&param->builder, ri->name,
+                                       bitsize, n, "int", "cp_regs");
+        break;
+    default:
+        g_assert_not_reached();
+    }
+    cpu->dyn_sysreg_feature.data.cpregs.keys[n] = ri_key;
 }
 
 static GDBFeature *arm_gen_dynamic_sysreg_feature(CPUState *cs, int base_reg)
-- 
2.43.0

[PATCH v2 11/37] target/arm: Handle ARM_CP_128BIT in cpu reset

[Richard Henderson]

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/cpu.c | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/target/arm/cpu.c b/target/arm/cpu.c
index d2fc17eab6..000fa20bc6 100644
--- a/target/arm/cpu.c
+++ b/target/arm/cpu.c
@@ -180,7 +180,7 @@ static void cp_reg_reset(gpointer key, gpointer value, gpointer opaque)
     ARMCPRegInfo *ri = value;
     ARMCPU *cpu = opaque;
 
-    if (ri->type & (ARM_CP_SPECIAL_MASK | ARM_CP_ALIAS)) {
+    if (ri->type & (ARM_CP_SPECIAL_MASK | ARM_CP_ALIAS | ARM_CP_128BIT)) {
         return;
     }
 
@@ -208,16 +208,24 @@ static void cp_reg_check_reset(gpointer key, gpointer value,  gpointer opaque)
      */
     ARMCPRegInfo *ri = value;
     ARMCPU *cpu = opaque;
-    uint64_t oldvalue, newvalue;
 
     if (ri->type & (ARM_CP_SPECIAL_MASK | ARM_CP_ALIAS | ARM_CP_NO_RAW)) {
         return;
     }
+    if (ri->type & ARM_CP_128BIT) {
+        /*
+         * All 128-bit registers are UNKNOWN at reset.
+         * For qemu, they are all cleared by memset to end_reset_fields.
+         */
+        assert(!int128_nz(read_raw_cp_reg128(&cpu->env, ri)));
+    } else {
+        uint64_t oldvalue, newvalue;
 
-    oldvalue = read_raw_cp_reg(&cpu->env, ri);
-    cp_reg_reset(key, value, opaque);
-    newvalue = read_raw_cp_reg(&cpu->env, ri);
-    assert(oldvalue == newvalue);
+        oldvalue = read_raw_cp_reg(&cpu->env, ri);
+        cp_reg_reset(key, value, opaque);
+        newvalue = read_raw_cp_reg(&cpu->env, ri);
+        assert(oldvalue == newvalue);
+    }
 }
 
 static void arm_cpu_reset_hold(Object *obj, ResetType type)
-- 
2.43.0

[PATCH v2 12/37] target/arm: Put 128-bit sysregs into a separate list

[Richard Henderson]

Duplicate the cpreg_{indexes,values,array_len} array as
cpreg128_{indexes,values,array_len}.  Similarly for
cpreg_vmstate_{indexes,values,array_len}.

Split the values between the two arrays during
write_cpustate_to_list, write_list_to_cpustate,
count_cpreg, and add_cpreg_to_list.

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/cpu.h    |  19 +++++--
 target/arm/helper.c | 125 +++++++++++++++++++++++++++++++++++---------
 2 files changed, 116 insertions(+), 28 deletions(-)

diff --git a/target/arm/cpu.h b/target/arm/cpu.h
index e21612b898..a030539488 100644
--- a/target/arm/cpu.h
+++ b/target/arm/cpu.h
@@ -918,25 +918,36 @@ struct ArchCPU {
 
     /* Coprocessor information */
     GHashTable *cp_regs;
-    /* For marshalling (mostly coprocessor) register state between the
+    /*
+     * For marshalling (mostly coprocessor) register state between the
      * kernel and QEMU (for KVM) and between two QEMUs (for migration),
      * we use these arrays.
      */
-    /* List of register indexes managed via these arrays; (full KVM style
-     * 64 bit indexes, not CPRegInfo 32 bit indexes)
+    /*
+     * List of register indexes managed via these arrays (full KVM style
+     * 64 bit indexes, not CPRegInfo 32 bit indexes).  The registers are
+     * segregated by size, with 64-bit registers in cpreg_indexes and
+     * 128-bit registers in cpreg128_indexes.
      */
     uint64_t *cpreg_indexes;
+    uint64_t *cpreg128_indexes;
     /* Values of the registers (cpreg_indexes[i]'s value is cpreg_values[i]) */
     uint64_t *cpreg_values;
+    Int128 *cpreg128_values;
     /* Length of the indexes, values, reset_values arrays */
     int32_t cpreg_array_len;
-    /* These are used only for migration: incoming data arrives in
+    int32_t cpreg128_array_len;
+    /*
+     * These are used only for migration: incoming data arrives in
      * these fields and is sanity checked in post_load before copying
      * to the working data structures above.
      */
     uint64_t *cpreg_vmstate_indexes;
+    uint64_t *cpreg128_vmstate_indexes;
     uint64_t *cpreg_vmstate_values;
+    Int128 *cpreg128_vmstate_values;
     int32_t cpreg_vmstate_array_len;
+    int32_t cpreg128_vmstate_array_len;
 
     DynamicGDBFeatureInfo dyn_sysreg_feature;
     DynamicGDBFeatureInfo dyn_svereg_feature;
diff --git a/target/arm/helper.c b/target/arm/helper.c
index d9d8ae56e8..d3a425e259 100644
--- a/target/arm/helper.c
+++ b/target/arm/helper.c
@@ -217,10 +217,11 @@ static bool raw_accessors_invalid(const ARMCPRegInfo *ri)
 bool write_cpustate_to_list(ARMCPU *cpu, bool kvm_sync)
 {
     /* Write the coprocessor state from cpu->env to the (index,value) list. */
-    int i;
+    int i, n;
     bool ok = true;
 
-    for (i = 0; i < cpu->cpreg_array_len; i++) {
+    n = cpu->cpreg_array_len;
+    for (i = 0; i < n; i++) {
         uint32_t regidx = kvm_to_cpreg_id(cpu->cpreg_indexes[i]);
         const ARMCPRegInfo *ri;
         uint64_t newval;
@@ -230,6 +231,10 @@ bool write_cpustate_to_list(ARMCPU *cpu, bool kvm_sync)
             ok = false;
             continue;
         }
+        if (ri->type & ARM_CP_128BIT) {
+            ok = false;
+            continue;
+        }
         if (ri->type & ARM_CP_NO_RAW) {
             continue;
         }
@@ -257,35 +262,77 @@ bool write_cpustate_to_list(ARMCPU *cpu, bool kvm_sync)
         }
         cpu->cpreg_values[i] = newval;
     }
-    return ok;
-}
 
-bool write_list_to_cpustate(ARMCPU *cpu)
-{
-    int i;
-    bool ok = true;
+    n = cpu->cpreg128_array_len;
+    if (n == 0) {
+        return ok;
+    }
+    assert(!kvm_sync);
 
-    for (i = 0; i < cpu->cpreg_array_len; i++) {
-        uint32_t regidx = kvm_to_cpreg_id(cpu->cpreg_indexes[i]);
-        uint64_t v = cpu->cpreg_values[i];
-        const ARMCPRegInfo *ri;
+    for (i = 0; i < n; i++) {
+        uint32_t regidx = kvm_to_cpreg_id(cpu->cpreg128_indexes[i]);
+        const ARMCPRegInfo *ri = get_arm_cp_reginfo(cpu->cp_regs, regidx);
 
-        ri = get_arm_cp_reginfo(cpu->cp_regs, regidx);
         if (!ri) {
             ok = false;
             continue;
         }
+        if (!(ri->type & ARM_CP_128BIT)) {
+            ok = false;
+            continue;
+        }
         if (ri->type & ARM_CP_NO_RAW) {
             continue;
         }
-        /*
-         * Write value and confirm it reads back as written
-         * (to catch read-only registers and partially read-only
-         * registers where the incoming migration value doesn't match)
-         */
-        write_raw_cp_reg(&cpu->env, ri, v);
-        if (read_raw_cp_reg(&cpu->env, ri) != v) {
+
+        cpu->cpreg128_values[i] = read_raw_cp_reg128(&cpu->env, ri);
+    }
+    return ok;
+}
+
+bool write_list_to_cpustate(ARMCPU *cpu)
+{
+    int i, n;
+    bool ok = true;
+
+    n = cpu->cpreg_array_len;
+    for (i = 0; i < n; i++) {
+        uint32_t regidx = kvm_to_cpreg_id(cpu->cpreg_indexes[i]);
+        const ARMCPRegInfo *ri = get_arm_cp_reginfo(cpu->cp_regs, regidx);
+
+        if (!ri) {
             ok = false;
+        } else if (ri->type & ARM_CP_128BIT) {
+            ok = false;
+        } else if (!(ri->type & ARM_CP_NO_RAW)) {
+            /*
+             * Write value and confirm it reads back as written
+             * (to catch read-only registers and partially read-only
+             * registers where the incoming migration value doesn't match)
+             */
+            uint64_t v = cpu->cpreg_values[i];
+            write_raw_cp_reg(&cpu->env, ri, v);
+            if (read_raw_cp_reg(&cpu->env, ri) != v) {
+                ok = false;
+            }
+        }
+    }
+
+    n = cpu->cpreg128_array_len;
+    for (i = 0; i < n; i++) {
+        uint32_t regidx = kvm_to_cpreg_id(cpu->cpreg128_indexes[i]);
+        const ARMCPRegInfo *ri = get_arm_cp_reginfo(cpu->cp_regs, regidx);
+
+        if (!ri) {
+            ok = false;
+        } else if (!(ri->type & ARM_CP_128BIT)) {
+            ok = false;
+        } else if (!(ri->type & ARM_CP_NO_RAW)) {
+            Int128 v = cpu->cpreg128_values[i];
+            write_raw_cp_reg128(&cpu->env, ri, v);
+            if (int128_ne(read_raw_cp_reg128(&cpu->env, ri), v)) {
+                ok = false;
+            }
         }
     }
     return ok;
@@ -298,9 +345,14 @@ static void add_cpreg_to_list(gpointer key, gpointer value, gpointer opaque)
     const ARMCPRegInfo *ri = value;
 
     if (!(ri->type & (ARM_CP_NO_RAW | ARM_CP_ALIAS))) {
-        cpu->cpreg_indexes[cpu->cpreg_array_len] = cpreg_to_kvm_id(regidx);
+        uint64_t idx = cpreg_to_kvm_id(regidx);
+
         /* The value array need not be initialized at this point */
-        cpu->cpreg_array_len++;
+        if (ri->type & ARM_CP_128BIT) {
+            cpu->cpreg128_indexes[cpu->cpreg128_array_len++] = idx;
+        } else {
+            cpu->cpreg_indexes[cpu->cpreg_array_len++] = idx;
+        }
     }
 }
 
@@ -310,7 +362,11 @@ static void count_cpreg(gpointer key, gpointer value, gpointer opaque)
     const ARMCPRegInfo *ri = value;
 
     if (!(ri->type & (ARM_CP_NO_RAW | ARM_CP_ALIAS))) {
-        cpu->cpreg_array_len++;
+        if (ri->type & ARM_CP_128BIT) {
+            cpu->cpreg128_array_len++;
+        } else {
+            cpu->cpreg_array_len++;
+        }
     }
 }
 
@@ -320,9 +376,10 @@ void init_cpreg_list(ARMCPU *cpu)
      * Initialise the cpreg_tuples[] array based on the cp_regs hash.
      * Note that we require cpreg_tuples[] to be sorted by key ID.
      */
-    int arraylen;
+    int arraylen, array128len;
 
     cpu->cpreg_array_len = 0;
+    cpu->cpreg128_array_len = 0;
     g_hash_table_foreach(cpu->cp_regs, count_cpreg, cpu);
 
     arraylen = cpu->cpreg_array_len;
@@ -340,13 +397,33 @@ void init_cpreg_list(ARMCPU *cpu)
     cpu->cpreg_vmstate_array_len = arraylen;
     cpu->cpreg_array_len = 0;
 
+    array128len = cpu->cpreg128_array_len;
+    if (array128len) {
+        cpu->cpreg128_indexes = g_new(uint64_t, array128len);
+        cpu->cpreg128_values = g_new(Int128, array128len);
+        cpu->cpreg128_vmstate_indexes = g_new(uint64_t, array128len);
+        cpu->cpreg128_vmstate_values = g_new(Int128, array128len);
+    } else {
+        cpu->cpreg128_indexes = NULL;
+        cpu->cpreg128_values = NULL;
+        cpu->cpreg128_vmstate_indexes = NULL;
+        cpu->cpreg128_vmstate_values = NULL;
+    }
+    cpu->cpreg128_vmstate_array_len = array128len;
+    cpu->cpreg128_array_len = 0;
+
     g_hash_table_foreach(cpu->cp_regs, add_cpreg_to_list, cpu);
 
     assert(cpu->cpreg_array_len == arraylen);
+    assert(cpu->cpreg128_array_len == array128len);
 
     if (arraylen) {
         qsort(cpu->cpreg_indexes, arraylen, sizeof(uint64_t), compare_u64);
     }
+    if (array128len) {
+        qsort(cpu->cpreg128_indexes, array128len,
+              sizeof(uint64_t), compare_u64);
+    }
 }
 
 bool arm_pan_enabled(CPUARMState *env)
-- 
2.43.0

[PATCH v2 13/37] target/arm/kvm: Assert no 128-bit sysregs in kvm_arm_init_cpreg_list

[Richard Henderson]

KVM has not yet enabled 128-bit system registers.

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/kvm.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/target/arm/kvm.c b/target/arm/kvm.c
index 4f769d69b3..3e7738592f 100644
--- a/target/arm/kvm.c
+++ b/target/arm/kvm.c
@@ -804,6 +804,8 @@ static int kvm_arm_init_cpreg_list(ARMCPU *cpu)
         arraylen++;
     }
 
+    assert(cpu->cpreg128_array_len == 0);
+
     cpu->cpreg_indexes = g_renew(uint64_t, cpu->cpreg_indexes, arraylen);
     cpu->cpreg_values = g_renew(uint64_t, cpu->cpreg_values, arraylen);
     cpu->cpreg_vmstate_indexes = g_renew(uint64_t, cpu->cpreg_vmstate_indexes,
-- 
2.43.0

[PATCH v2 14/37] target/arm/hvf: Assert no 128-bit sysregs in hvf_arch_init_vcpu

[Richard Henderson]

HVF has not yet enabled 128-bit system registers.

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/hvf/hvf.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/target/arm/hvf/hvf.c b/target/arm/hvf/hvf.c
index 0658a99a2d..aa42fa09c3 100644
--- a/target/arm/hvf/hvf.c
+++ b/target/arm/hvf/hvf.c
@@ -897,6 +897,9 @@ int hvf_arch_init_vcpu(CPUState *cpu)
     env->aarch64 = true;
     asm volatile("mrs %0, cntfrq_el0" : "=r"(arm_cpu->gt_cntfrq_hz));
 
+    /* No support yet for FEAT_D128 */
+    assert(arm_cpu->cpreg128_array_len == 0);
+
     /* Allocate enough space for our sysreg sync */
     arm_cpu->cpreg_indexes = g_renew(uint64_t, arm_cpu->cpreg_indexes,
                                      sregs_match_len);
@@ -920,6 +923,7 @@ int hvf_arch_init_vcpu(CPUState *cpu)
 
         if (ri) {
             assert(!(ri->type & ARM_CP_NO_RAW));
+            assert(!(ri->type & ARM_CP_128BIT));
             arm_cpu->cpreg_indexes[sregs_cnt++] = kvm_id;
         }
     }
-- 
2.43.0

[PATCH v2 15/37] migration: Add vmstate_info_int128

[Richard Henderson]

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 include/migration/vmstate.h |  1 +
 migration/vmstate-types.c   | 30 ++++++++++++++++++++++++++++++
 2 files changed, 31 insertions(+)

diff --git a/include/migration/vmstate.h b/include/migration/vmstate.h
index 63ccaee07a..d8b9e157da 100644
--- a/include/migration/vmstate.h
+++ b/include/migration/vmstate.h
@@ -237,6 +237,7 @@ extern const VMStateInfo vmstate_info_int8;
 extern const VMStateInfo vmstate_info_int16;
 extern const VMStateInfo vmstate_info_int32;
 extern const VMStateInfo vmstate_info_int64;
+extern const VMStateInfo vmstate_info_int128;
 
 extern const VMStateInfo vmstate_info_uint8_equal;
 extern const VMStateInfo vmstate_info_uint16_equal;
diff --git a/migration/vmstate-types.c b/migration/vmstate-types.c
index 4b01dc19c2..d335340e34 100644
--- a/migration/vmstate-types.c
+++ b/migration/vmstate-types.c
@@ -18,6 +18,7 @@
 #include "migration/client-options.h"
 #include "qemu/error-report.h"
 #include "qemu/queue.h"
+#include "qemu/int128.h"
 #include "trace.h"
 #include "qapi/error.h"
 
@@ -194,6 +195,35 @@ const VMStateInfo vmstate_info_int64 = {
     .put  = put_int64,
 };
 
+/* 128 bit int */
+
+static int get_int128(QEMUFile *f, void *pv, size_t size,
+                      const VMStateField *field)
+{
+    Int128 *v = pv;
+    uint64_t hi, lo;
+
+    qemu_get_be64s(f, &hi);
+    qemu_get_be64s(f, &lo);
+    *v = int128_make128(lo, hi);
+    return 0;
+}
+
+static int put_int128(QEMUFile *f, void *pv, size_t size,
+                      const VMStateField *field, JSONWriter *vmdesc)
+{
+    Int128 *v = pv;
+    qemu_put_be64(f, int128_gethi(*v));
+    qemu_put_be64(f, int128_getlo(*v));
+    return 0;
+}
+
+const VMStateInfo vmstate_info_int128 = {
+    .name = "int128",
+    .get  = get_int128,
+    .put  = put_int128,
+};
+
 /* 8 bit unsigned int */
 
 static int get_uint8(QEMUFile *f, void *pv, size_t size,
-- 
2.43.0

[PATCH v2 16/37] target/arm: Migrate cpreg128 registers

[Richard Henderson]

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/machine.c | 50 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 50 insertions(+)

diff --git a/target/arm/machine.c b/target/arm/machine.c
index 44a0cf844b..2cc6234832 100644
--- a/target/arm/machine.c
+++ b/target/arm/machine.c
@@ -939,6 +939,34 @@ static const VMStateDescription vmstate_syndrome64 = {
     },
 };
 
+static bool sysreg128_needed(void *opaque)
+{
+    ARMCPU *cpu = opaque;
+    return cpu->cpreg128_array_len != 0;
+}
+
+static const VMStateDescription vmstate_sysreg128 = {
+    .name = "cpu/sysreg128",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = sysreg128_needed,
+    .fields = (const VMStateField[]) {
+        /*
+         * The length-check must come before the arrays to avoid
+         * incoming data possibly overflowing the array.
+         */
+        VMSTATE_INT32_POSITIVE_LE(cpreg128_vmstate_array_len, ARMCPU),
+        VMSTATE_VARRAY_INT32(cpreg128_vmstate_indexes, ARMCPU,
+                             cpreg128_vmstate_array_len,
+                             0, vmstate_info_uint64, uint64_t),
+        VMSTATE_VARRAY_INT32(cpreg128_vmstate_values, ARMCPU,
+                             cpreg128_vmstate_array_len,
+                             0, vmstate_info_int128, Int128),
+
+        VMSTATE_END_OF_LIST()
+    },
+};
+
 static int cpu_pre_save(void *opaque)
 {
     ARMCPU *cpu = opaque;
@@ -971,6 +999,12 @@ static int cpu_pre_save(void *opaque)
     memcpy(cpu->cpreg_vmstate_values, cpu->cpreg_values,
            cpu->cpreg_array_len * sizeof(uint64_t));
 
+    cpu->cpreg128_vmstate_array_len = cpu->cpreg128_array_len;
+    memcpy(cpu->cpreg128_vmstate_indexes, cpu->cpreg128_indexes,
+           cpu->cpreg128_array_len * sizeof(uint64_t));
+    memcpy(cpu->cpreg128_vmstate_values, cpu->cpreg128_values,
+           cpu->cpreg128_array_len * sizeof(Int128));
+
     return 0;
 }
 
@@ -1065,6 +1099,21 @@ static int cpu_post_load(void *opaque, int version_id)
         v++;
     }
 
+    for (i = 0, v = 0; i < cpu->cpreg128_array_len
+             && v < cpu->cpreg128_vmstate_array_len; i++) {
+        if (cpu->cpreg128_vmstate_indexes[v] > cpu->cpreg128_indexes[i]) {
+            /* register in our list but not incoming : skip it */
+            continue;
+        }
+        if (cpu->cpreg128_vmstate_indexes[v] < cpu->cpreg128_indexes[i]) {
+            /* register in their list but not ours: fail migration */
+            return -1;
+        }
+        /* matching register, copy the value over */
+        cpu->cpreg128_values[i] = cpu->cpreg128_vmstate_values[v];
+        v++;
+    }
+
     if (kvm_enabled()) {
         if (!kvm_arm_cpu_post_load(cpu)) {
             return -1;
@@ -1209,6 +1258,7 @@ const VMStateDescription vmstate_arm_cpu = {
         &vmstate_wfxt_timer,
         &vmstate_syndrome64,
         &vmstate_pstate64,
+        &vmstate_sysreg128,
         NULL
     }
 };
-- 
2.43.0

[PATCH v2 17/37] target/arm: Add syn_aa64_sysreg128trap

[Richard Henderson]

System register access with MRRS/MSRR use a different
exception class.

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/syndrome.h | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/target/arm/syndrome.h b/target/arm/syndrome.h
index bff61f052c..2c7dc53507 100644
--- a/target/arm/syndrome.h
+++ b/target/arm/syndrome.h
@@ -45,6 +45,7 @@ enum arm_exception_class {
     EC_AA32_SVC               = 0x11,
     EC_AA32_HVC               = 0x12,
     EC_AA32_SMC               = 0x13,
+    EC_SYSTEMREGISTER128TRAP  = 0x14,
     EC_AA64_SVC               = 0x15,
     EC_AA64_HVC               = 0x16,
     EC_AA64_SMC               = 0x17,
@@ -187,6 +188,15 @@ static inline uint32_t syn_aa64_sysregtrap(int op0, int op1, int op2,
         | (crm << 1) | isread;
 }
 
+static inline uint32_t syn_aa64_sysreg128trap(int op0, int op1, int op2,
+                                              int crn, int crm, int rt,
+                                              int isread)
+{
+    return (EC_SYSTEMREGISTER128TRAP << ARM_EL_EC_SHIFT) | ARM_EL_IL
+        | (op0 << 20) | (op2 << 17) | (op1 << 14) | (crn << 10) | (rt << 5)
+        | (crm << 1) | isread;
+}
+
 static inline uint32_t syn_cp14_rt_trap(int cv, int cond, int opc1, int opc2,
                                         int crn, int crm, int rt, int isread,
                                         bool is_16bit)
-- 
2.43.0

[PATCH v2 18/37] target/arm: Introduce helper_{get,set}_cp_reg128

[Richard Henderson]

Helper functions to invoke the write128fn and read128fn
methods of ARMCPRegInfo.

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/tcg/helper.h    |  2 ++
 target/arm/tcg/op_helper.c | 29 +++++++++++++++++++++++++++++
 2 files changed, 31 insertions(+)

diff --git a/target/arm/tcg/helper.h b/target/arm/tcg/helper.h
index 4636d1bc03..45bb87fbaf 100644
--- a/target/arm/tcg/helper.h
+++ b/target/arm/tcg/helper.h
@@ -90,6 +90,8 @@ DEF_HELPER_3(set_cp_reg, void, env, cptr, i32)
 DEF_HELPER_2(get_cp_reg, i32, env, cptr)
 DEF_HELPER_3(set_cp_reg64, void, env, cptr, i64)
 DEF_HELPER_2(get_cp_reg64, i64, env, cptr)
+DEF_HELPER_4(set_cp_reg128, void, env, cptr, i64, i64)
+DEF_HELPER_2(get_cp_reg128, i128, env, cptr)
 
 DEF_HELPER_2(get_r13_banked, i32, env, i32)
 DEF_HELPER_3(set_r13_banked, void, env, i32, i32)
diff --git a/target/arm/tcg/op_helper.c b/target/arm/tcg/op_helper.c
index 4fbd219555..9c7b18fe65 100644
--- a/target/arm/tcg/op_helper.c
+++ b/target/arm/tcg/op_helper.c
@@ -1026,6 +1026,35 @@ uint64_t HELPER(get_cp_reg64)(CPUARMState *env, const void *rip)
     return res;
 }
 
+void HELPER(set_cp_reg128)(CPUARMState *env, const void *rip,
+                           uint64_t valuelo, uint64_t valuehi)
+{
+    const ARMCPRegInfo *ri = rip;
+
+    if (ri->type & ARM_CP_IO) {
+        bql_lock();
+        ri->write128fn(env, ri, valuelo, valuehi);
+        bql_unlock();
+    } else {
+        ri->write128fn(env, ri, valuelo, valuehi);
+    }
+}
+
+Int128 HELPER(get_cp_reg128)(CPUARMState *env, const void *rip)
+{
+    const ARMCPRegInfo *ri = rip;
+    Int128 res;
+
+    if (ri->type & ARM_CP_IO) {
+        bql_lock();
+        res = ri->read128fn(env, ri);
+        bql_unlock();
+    } else {
+        res = ri->read128fn(env, ri);
+    }
+    return res;
+}
+
 void HELPER(pre_hvc)(CPUARMState *env)
 {
     ARMCPU *cpu = env_archcpu(env);
-- 
2.43.0

[PATCH v2 19/37] target/arm: Implement MRRS, MSRR, SYSP

[Richard Henderson]

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/tcg/translate-a64.c | 169 +++++++++++++++++++++++++--------
 target/arm/tcg/a64.decode      |  12 ++-
 2 files changed, 139 insertions(+), 42 deletions(-)

diff --git a/target/arm/tcg/translate-a64.c b/target/arm/tcg/translate-a64.c
index 918d5ed112..c3b717b408 100644
--- a/target/arm/tcg/translate-a64.c
+++ b/target/arm/tcg/translate-a64.c
@@ -2752,17 +2752,23 @@ redirect_cpreg(DisasContext *s, uint32_t key, bool isread)
 
 /* MRS - move from system register
  * MSR (register) - move to system register
+ * MRRS
+ * MSRR
  * SYS
  * SYSL
+ * SYSP
  * These are all essentially the same insn in 'read' and 'write'
  * versions, with varying op0 fields.
+ *
+ * RT2 is non-zero if and only if this is a 128-bit access.
  */
 static void handle_sys(DisasContext *s, bool isread,
                        unsigned int op0, unsigned int op1, unsigned int op2,
-                       unsigned int crn, unsigned int crm, unsigned int rt)
+                       unsigned int crn, unsigned int crm, unsigned int rt,
+                       unsigned int rt2)
 {
     uint32_t key = ENCODE_AA64_CP_REG(op0, op1, crn, crm, op2);
-    const ARMCPRegInfo *ri = get_arm_cp_reginfo(s->cp_regs, key);
+    const ARMCPRegInfo *ri;
     bool need_exit_tb = false;
     bool nv_trap_to_el2 = false;
     bool nv_redirect_reg = false;
@@ -2770,7 +2776,16 @@ static void handle_sys(DisasContext *s, bool isread,
     bool nv2_mem_redirect = false;
     TCGv_ptr tcg_ri = NULL;
     TCGv_i64 tcg_rt;
-    uint32_t syndrome = syn_aa64_sysregtrap(op0, op1, op2, crn, crm, rt, isread);
+    uint32_t syndrome;
+    bool is128 = rt2 != 0;
+
+    if (is128) {
+        key |= CP_REG_AA64_128BIT_MASK;
+        syndrome = syn_aa64_sysreg128trap(op0, op1, op2, crn, crm, rt, isread);
+    } else {
+        syndrome = syn_aa64_sysregtrap(op0, op1, op2, crn, crm, rt, isread);
+    }
+    ri = get_arm_cp_reginfo(s->cp_regs, key);
 
     if (crn == 11 || crn == 15) {
         /*
@@ -2790,12 +2805,14 @@ static void handle_sys(DisasContext *s, bool isread,
     }
 
     if (!ri) {
-        /* Unknown register; this might be a guest error or a QEMU
-         * unimplemented feature.
+        /*
+         * Unknown register, or 128-bit access to a 64-bit register.
+         * This might be a guest error or a QEMU unimplemented feature.
          */
-        qemu_log_mask(LOG_UNIMP, "%s access to unsupported AArch64 "
+        qemu_log_mask(LOG_UNIMP, "%s%s access to unsupported AArch64 "
                       "system register op0:%d op1:%d crn:%d crm:%d op2:%d\n",
-                      isread ? "read" : "write", op0, op1, crn, crm, op2);
+                      isread ? "read" : "write", is128 ? "128" : "",
+                      op0, op1, crn, crm, op2);
         gen_sysreg_undef(s, isread, op0, op1, op2, crn, crm, rt);
         return;
     }
@@ -2918,7 +2935,7 @@ static void handle_sys(DisasContext *s, bool isread,
          * We don't use the EL1 register's access function, and
          * fine-grained-traps on EL1 also do not apply here.
          */
-        key = ENCODE_AA64_CP_REG(op0, 0, crn, crm, op2);
+        key &= ~CP_REG_ARM64_SYSREG_OP1_MASK;
         ri = redirect_cpreg(s, key, isread);
         /*
          * We might not have done an update_pc earlier, so check we don't
@@ -2933,32 +2950,44 @@ static void handle_sys(DisasContext *s, bool isread,
          * This means it is not an IO operation, doesn't change hflags,
          * and need not end the TB, because it has no side effects.
          *
-         * The access is 64-bit single copy atomic, guaranteed aligned because
-         * of the definition of VCNR_EL2. Its endianness depends on
-         * SCTLR_EL2.EE, not on the data endianness of EL1.
-         * It is done under either the EL2 translation regime or the EL2&0
-         * translation regime, depending on HCR_EL2.E2H. It behaves as if
-         * PSTATE.PAN is 0.
+         * The access is 64-bit (R_VFMQB) or 128-bit (R_BSBZP) single copy
+         * atomic, guaranteed aligned because of the definition of VCNR_EL2.
+         * Its endianness depends on SCTLR_EL2.EE, not on the data endianness
+         * of EL1. It is done under either the EL2 translation regime or
+         * the EL2&0 translation regime, depending on HCR_EL2.E2H.
+         * It behaves as if PSTATE.PAN is 0.
          */
         TCGv_i64 ptr = tcg_temp_new_i64();
-        MemOp mop = MO_64 | MO_ALIGN | MO_ATOM_IFALIGN;
+        MemOp mop = MO_ALIGN | MO_ATOM_IFALIGN;
         ARMMMUIdx armmemidx = s->nv2_mem_e20 ? ARMMMUIdx_E20_2 : ARMMMUIdx_E2;
         int memidx = arm_to_core_mmu_idx(armmemidx);
-        uint32_t syn;
 
         mop |= (s->nv2_mem_be ? MO_BE : MO_LE);
+        disas_set_insn_syndrome(s, syn_data_abort_vncr(0, !isread, 0));
 
         tcg_gen_ld_i64(ptr, tcg_env, offsetof(CPUARMState, cp15.vncr_el2));
         tcg_gen_addi_i64(ptr, ptr,
                          (ri->nv2_redirect_offset & ~NV2_REDIR_FLAG_MASK));
-        tcg_rt = cpu_reg(s, rt);
 
-        syn = syn_data_abort_vncr(0, !isread, 0);
-        disas_set_insn_syndrome(s, syn);
-        if (isread) {
-            tcg_gen_qemu_ld_i64(tcg_rt, ptr, memidx, mop);
+        if (is128) {
+            TCGv_i128 t128 = tcg_temp_new_i128();
+
+            mop |= MO_128;
+            if (isread) {
+                tcg_gen_qemu_ld_i128(t128, ptr, memidx, mop);
+                tcg_gen_extr_i128_i64(cpu_reg(s, rt), cpu_reg(s, rt2), t128);
+            } else {
+                tcg_gen_concat_i64_i128(t128, cpu_reg(s, rt), cpu_reg(s, rt2));
+                tcg_gen_qemu_st_i128(t128, ptr, memidx, mop);
+            }
         } else {
-            tcg_gen_qemu_st_i64(tcg_rt, ptr, memidx, mop);
+            mop |= MO_64;
+            tcg_rt = cpu_reg(s, rt);
+            if (isread) {
+                tcg_gen_qemu_ld_i64(tcg_rt, ptr, memidx, mop);
+            } else {
+                tcg_gen_qemu_st_i64(tcg_rt, ptr, memidx, mop);
+            }
         }
         return;
     }
@@ -3099,28 +3128,63 @@ static void handle_sys(DisasContext *s, bool isread,
 
     tcg_rt = cpu_reg(s, rt);
 
-    if (isread) {
-        if (ri->type & ARM_CP_CONST) {
-            tcg_gen_movi_i64(tcg_rt, ri->resetvalue);
-        } else if (ri->readfn) {
-            if (!tcg_ri) {
-                tcg_ri = gen_lookup_cp_reg(key);
+    if (is128) {
+        TCGv_i64 tcg_rt2 = cpu_reg(s, rt2);
+
+        if (isread) {
+            if (ri->type & ARM_CP_CONST) {
+                /* E.g. TTBR0_EL2 is RES0 from EL3 when !FEAT_EL2. */
+                tcg_gen_movi_i64(tcg_rt, ri->resetvalue);
+                tcg_gen_movi_i64(tcg_rt2, 0);
+            } else if (ri->read128fn) {
+                TCGv_i128 t128 = tcg_temp_new_i128();
+                if (!tcg_ri) {
+                    tcg_ri = gen_lookup_cp_reg(key);
+                }
+                gen_helper_get_cp_reg128(t128, tcg_env, tcg_ri);
+                tcg_gen_extr_i128_i64(tcg_rt, tcg_rt2, t128);
+            } else {
+                tcg_gen_ld_i64(tcg_rt, tcg_env, ri->fieldoffset);
+                tcg_gen_ld_i64(tcg_rt2, tcg_env, ri->fieldoffsethi);
             }
-            gen_helper_get_cp_reg64(tcg_rt, tcg_env, tcg_ri);
         } else {
-            tcg_gen_ld_i64(tcg_rt, tcg_env, ri->fieldoffset);
+            if (ri->type & ARM_CP_CONST) {
+                /* If not forbidden by access permissions, treat as WI */
+                return;
+            } else if (ri->write128fn) {
+                if (!tcg_ri) {
+                    tcg_ri = gen_lookup_cp_reg(key);
+                }
+                gen_helper_set_cp_reg128(tcg_env, tcg_ri, tcg_rt, tcg_rt2);
+            } else {
+                tcg_gen_st_i64(tcg_rt, tcg_env, ri->fieldoffset);
+                tcg_gen_st_i64(tcg_rt2, tcg_env, ri->fieldoffsethi);
+            }
         }
     } else {
-        if (ri->type & ARM_CP_CONST) {
-            /* If not forbidden by access permissions, treat as WI */
-            return;
-        } else if (ri->writefn) {
-            if (!tcg_ri) {
-                tcg_ri = gen_lookup_cp_reg(key);
+        if (isread) {
+            if (ri->type & ARM_CP_CONST) {
+                tcg_gen_movi_i64(tcg_rt, ri->resetvalue);
+            } else if (ri->readfn) {
+                if (!tcg_ri) {
+                    tcg_ri = gen_lookup_cp_reg(key);
+                }
+                gen_helper_get_cp_reg64(tcg_rt, tcg_env, tcg_ri);
+            } else {
+                tcg_gen_ld_i64(tcg_rt, tcg_env, ri->fieldoffset);
             }
-            gen_helper_set_cp_reg64(tcg_env, tcg_ri, tcg_rt);
         } else {
-            tcg_gen_st_i64(tcg_rt, tcg_env, ri->fieldoffset);
+            if (ri->type & ARM_CP_CONST) {
+                /* If not forbidden by access permissions, treat as WI */
+                return;
+            } else if (ri->writefn) {
+                if (!tcg_ri) {
+                    tcg_ri = gen_lookup_cp_reg(key);
+                }
+                gen_helper_set_cp_reg64(tcg_env, tcg_ri, tcg_rt);
+            } else {
+                tcg_gen_st_i64(tcg_rt, tcg_env, ri->fieldoffset);
+            }
         }
     }
 
@@ -3144,7 +3208,34 @@ static void handle_sys(DisasContext *s, bool isread,
 
 static bool trans_SYS(DisasContext *s, arg_SYS *a)
 {
-    handle_sys(s, a->l, a->op0, a->op1, a->op2, a->crn, a->crm, a->rt);
+    handle_sys(s, a->l, a->op0, a->op1, a->op2, a->crn, a->crm, a->rt, 0);
+    return true;
+}
+
+static bool trans_SYS128(DisasContext *s, arg_sys *a)
+{
+    if (!dc_isar_feature(aa64_sysreg128, s) || (a->rt & 1)) {
+        return false;
+    }
+    handle_sys(s, a->l, a->op0, a->op1, a->op2, a->crn, a->crm,
+               a->rt, a->rt + 1);
+    return true;
+}
+
+static bool trans_SYSP(DisasContext *s, arg_sys *a)
+{
+    int rt2;
+    if (!dc_isar_feature(aa64_sysinstr128, s)) {
+        return false;
+    }
+    if (a->rt == 31) {
+        rt2 = 31;
+    } else if (a->rt & 1) {
+        return false;
+    } else {
+        rt2 = a->rt + 1;
+    }
+    handle_sys(s, a->l, a->op0, a->op1, a->op2, a->crn, a->crm, a->rt, rt2);
     return true;
 }
 
diff --git a/target/arm/tcg/a64.decode b/target/arm/tcg/a64.decode
index 01b1b3e38b..c76757ed01 100644
--- a/target/arm/tcg/a64.decode
+++ b/target/arm/tcg/a64.decode
@@ -302,9 +302,15 @@ MSR_i_SVCR      1101 0101 0000 0 011 0100 0 mask:2 imm:1 011 11111
 # same instruction as far as QEMU is concerned.
 # NB: op0 is bits [20:19], but op0=0b00 is other insns, so we have
 # to hand-decode it.
-SYS             1101 0101 00 l:1 01 op1:3 crn:4 crm:4 op2:3 rt:5 op0=1
-SYS             1101 0101 00 l:1 10 op1:3 crn:4 crm:4 op2:3 rt:5 op0=2
-SYS             1101 0101 00 l:1 11 op1:3 crn:4 crm:4 op2:3 rt:5 op0=3
+&sys            l op0 op1 op2 crn crm rt
+SYS             1101 0101 00 l:1 01 op1:3 crn:4 crm:4 op2:3 rt:5 &sys op0=1
+SYS             1101 0101 00 l:1 10 op1:3 crn:4 crm:4 op2:3 rt:5 &sys op0=2
+SYS             1101 0101 00 l:1 11 op1:3 crn:4 crm:4 op2:3 rt:5 &sys op0=3
+
+# MRRS, MSRR
+SYS128          1101 0101 01 l:1 10 op1:3 crn:4 crm:4 op2:3 rt:5 &sys op0=2
+SYS128          1101 0101 01 l:1 11 op1:3 crn:4 crm:4 op2:3 rt:5 &sys op0=3
+SYSP            1101 0101 01 0   01 op1:3 crn:4 crm:4 op2:3 rt:5 &sys op0=1 l=0
 
 # Exception generation
 
-- 
2.43.0

[PATCH v2 20/37] target/arm: Consolidate definitions of PAR

[Richard Henderson]

Create a function define_par_register which handles the 3
distinct cases for PAR.  It is easier to understand with
the definitions all in one place.

Make the aarch64 to be the primary definition, when present,
rather than being an alias of the 64-bit non-secure aa32 reg.
Remove the unnecessary .writefn from the aarch64 defintion,
and drop it from the 32-bit definition with LPAE.

Remove the LPAE test from par_write, since it will no longer
be used in that situation.

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/helper.c | 103 +++++++++++++++++++++++++++++++-------------
 1 file changed, 73 insertions(+), 30 deletions(-)

diff --git a/target/arm/helper.c b/target/arm/helper.c
index d3a425e259..7800d83f48 100644
--- a/target/arm/helper.c
+++ b/target/arm/helper.c
@@ -2475,9 +2475,7 @@ static const ARMCPRegInfo gen_timer_ecv_cp_reginfo[] = {
 
 static void par_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
 {
-    if (arm_feature(env, ARM_FEATURE_LPAE)) {
-        raw_write(env, ri, value);
-    } else if (arm_feature(env, ARM_FEATURE_V7)) {
+    if (arm_feature(env, ARM_FEATURE_V7)) {
         raw_write(env, ri, value & 0xfffff6ff);
     } else {
         raw_write(env, ri, value & 0xfffff1ff);
@@ -3244,10 +3242,11 @@ static const ARMCPRegInfo lpae_cp_reginfo[] = {
     { .name = "AMAIR1", .cp = 15, .crn = 10, .crm = 3, .opc1 = 0, .opc2 = 1,
       .access = PL1_RW, .accessfn = access_tvm_trvm,
       .type = ARM_CP_CONST, .resetvalue = 0 },
-    { .name = "PAR", .cp = 15, .crm = 7, .opc1 = 0,
-      .access = PL1_RW, .type = ARM_CP_64BIT, .resetvalue = 0,
-      .bank_fieldoffsets = { offsetof(CPUARMState, cp15.par_s),
-                             offsetof(CPUARMState, cp15.par_ns)} },
+
+    /*
+     * The primary definitions of TTBR[01]_EL1 are in vmsa_cp_reginfo[].
+     * Here we need only provide the 64-bit views for AArch32.
+     */
     { .name = "TTBR0", .cp = 15, .crm = 2, .opc1 = 0,
       .access = PL1_RW, .accessfn = access_tvm_trvm,
       .type = ARM_CP_64BIT | ARM_CP_ALIAS,
@@ -3262,6 +3261,71 @@ static const ARMCPRegInfo lpae_cp_reginfo[] = {
       .writefn = vmsa_ttbr_write, .raw_writefn = raw_write },
 };
 
+static void define_par_register(ARMCPU *cpu)
+{
+    /*
+     * For v8:
+     * The aarch64 reg is primary, since it might be 128-bit.
+     * The aarch32 64-bit non-secure reg is secondary to aa64.
+     * The aarch32 64-bit secure reg is primary.
+     *
+     * For v7:
+     * The aarch32 64-bit s+ns regs are primary.
+     *
+     * The aarch32 32-bit regs are secondary to one of the above,
+     * and we also don't expose them to gdb.
+     */
+    static const ARMCPRegInfo parv8_reginfo = {
+        .name = "PAR_EL1", .state = ARM_CP_STATE_AA64,
+        .opc0 = 3, .opc1 = 0, .crn = 7, .crm = 4, .opc2 = 0,
+        .access = PL1_RW, .fgt = FGT_PAR_EL1,
+        .fieldoffset = offsetof(CPUARMState, cp15.par_el[1])
+    };
+
+    static ARMCPRegInfo par64_reginfo[2] = {
+        [0 ... 1] = {
+            .state = ARM_CP_STATE_AA32,
+            .cp = 15, .crm = 7, .opc1 = 0,
+            .type = ARM_CP_64BIT, .access = PL1_RW,
+        },
+        [0].name = "PAR",
+        [0].secure = ARM_CP_SECSTATE_NS,
+        [0].fieldoffset = offsetof(CPUARMState, cp15.par_ns),
+        [1].name = "PAR_S",
+        [1].secure = ARM_CP_SECSTATE_S,
+        [1].fieldoffset = offsetof(CPUARMState, cp15.par_s),
+    };
+
+    static ARMCPRegInfo par32_reginfo = {
+        .name = "PAR", .state = ARM_CP_STATE_AA32,
+        .cp = 15, .crn = 7, .crm = 4, .opc1 = 0, .opc2 = 0,
+        .access = PL1_RW, .resetvalue = 0,
+        .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.par_s),
+                               offsetoflow32(CPUARMState, cp15.par_ns) },
+        .writefn = par_write,
+    };
+
+    CPUARMState *env = &cpu->env;
+
+    /* With only VAPA, define a 32-bit reg that filters bits from write. */
+    if (!arm_feature(env, ARM_FEATURE_LPAE)) {
+        define_one_arm_cp_reg(cpu, &par32_reginfo);
+        return;
+    }
+
+    /* With LPAE, the 32-bit regs are aliases of 64-bit regs. */
+    par32_reginfo.type = ARM_CP_ALIAS | ARM_CP_NO_GDB;
+    par32_reginfo.writefn = NULL;
+    define_one_arm_cp_reg(cpu, &par32_reginfo);
+
+    if (arm_feature(env, ARM_FEATURE_V8)) {
+        define_one_arm_cp_reg(cpu, &parv8_reginfo);
+        par64_reginfo[0].type |= ARM_CP_ALIAS;
+    }
+
+    define_arm_cp_regs(cpu, par64_reginfo);
+}
+
 static uint64_t aa64_fpcr_read(CPUARMState *env, const ARMCPRegInfo *ri)
 {
     return vfp_get_fpcr(env);
@@ -3765,13 +3829,6 @@ static const ARMCPRegInfo v8_cp_reginfo[] = {
       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 14, .opc2 = 2,
       .fgt = FGT_DCCISW,
       .access = PL1_W, .accessfn = access_tsw, .type = ARM_CP_NOP },
-    { .name = "PAR_EL1", .state = ARM_CP_STATE_AA64,
-      .type = ARM_CP_ALIAS,
-      .opc0 = 3, .opc1 = 0, .crn = 7, .crm = 4, .opc2 = 0,
-      .access = PL1_RW, .resetvalue = 0,
-      .fgt = FGT_PAR_EL1,
-      .fieldoffset = offsetof(CPUARMState, cp15.par_el[1]),
-      .writefn = par_write },
     /* 32 bit cache operations */
     { .name = "ICIALLUIS", .cp = 15, .opc1 = 0, .crn = 7, .crm = 1, .opc2 = 0,
       .type = ARM_CP_NOP, .access = PL1_W, .accessfn = access_ticab },
@@ -7120,23 +7177,9 @@ void register_cp_regs_for_features(ARMCPU *cpu)
         define_one_arm_cp_reg(cpu, &gen_timer_cntpoff_reginfo);
     }
 #endif
-    if (arm_feature(env, ARM_FEATURE_VAPA)) {
-        ARMCPRegInfo vapa_cp_reginfo[] = {
-            { .name = "PAR", .cp = 15, .crn = 7, .crm = 4, .opc1 = 0, .opc2 = 0,
-              .access = PL1_RW, .resetvalue = 0,
-              .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.par_s),
-                                     offsetoflow32(CPUARMState, cp15.par_ns) },
-              .writefn = par_write},
-        };
 
-        /*
-         * When LPAE exists this 32-bit PAR register is an alias of the
-         * 64-bit AArch32 PAR register defined in lpae_cp_reginfo[]
-         */
-        if (arm_feature(env, ARM_FEATURE_LPAE)) {
-            vapa_cp_reginfo[0].type = ARM_CP_ALIAS | ARM_CP_NO_GDB;
-        }
-        define_arm_cp_regs(cpu, vapa_cp_reginfo);
+    if (arm_feature(env, ARM_FEATURE_VAPA)) {
+        define_par_register(cpu);
     }
     if (arm_feature(env, ARM_FEATURE_CACHE_TEST_CLEAN)) {
         define_arm_cp_regs(cpu, cache_test_clean_cp_reginfo);
-- 
2.43.0

[PATCH v2 21/37] target/arm: Extend PAR_EL1 to 128-bit

[Richard Henderson]

Do not yet produce the 128-bit AT format result, but zero the
high bits whenever the low bits are written.  This corresponds
to PAR_EL1.D128 = 0, and bits [127:65] as RES0.

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/cpu.h           |  5 ++++-
 target/arm/internals.h     |  1 +
 target/arm/helper.c        | 18 +++++++++++++++++-
 target/arm/tcg/cpregs-at.c |  4 ++++
 4 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/target/arm/cpu.h b/target/arm/cpu.h
index a030539488..e03d832717 100644
--- a/target/arm/cpu.h
+++ b/target/arm/cpu.h
@@ -429,7 +429,8 @@ typedef struct CPUArchState {
         };
         uint64_t hpfar_el2;
         uint64_t hstr_el2;
-        union { /* Translation result. */
+        /* Translation result. */
+        union {
             struct {
                 uint64_t _unused_par_0;
                 uint64_t par_ns;
@@ -438,6 +439,7 @@ typedef struct CPUArchState {
             };
             uint64_t par_el[4];
         };
+        uint64_t par_el1_hi;  /* high 64 bits of 128-bit PAR_EL1 */
 
         uint32_t c9_insn; /* Cache lockdown registers.  */
         uint32_t c9_data;
@@ -1750,6 +1752,7 @@ static inline void xpsr_write(CPUARMState *env, uint32_t val, uint32_t mask)
 #define SCR_SCTLR2EN          (1ULL << 44)
 #define SCR_PIEN              (1ULL << 45)
 #define SCR_AIEN              (1ULL << 46)
+#define SCR_D128EN            (1ULL << 47)
 #define SCR_GPF               (1ULL << 48)
 #define SCR_MECEN             (1ULL << 49)
 #define SCR_NSE               (1ULL << 62)
diff --git a/target/arm/internals.h b/target/arm/internals.h
index a65386aaed..bda2400b65 100644
--- a/target/arm/internals.h
+++ b/target/arm/internals.h
@@ -251,6 +251,7 @@ FIELD(VSTCR, SA, 30, 1)
 #define HCRX_MSCEN    (1ULL << 11)
 #define HCRX_TCR2EN   (1ULL << 14)
 #define HCRX_SCTLR2EN (1ULL << 15)
+#define HCRX_D128EN   (1ULL << 17)
 #define HCRX_GCSEN    (1ULL << 22)
 
 #define HPFAR_NS      (1ULL << 63)
diff --git a/target/arm/helper.c b/target/arm/helper.c
index 7800d83f48..f9a2a7d62d 100644
--- a/target/arm/helper.c
+++ b/target/arm/helper.c
@@ -508,6 +508,20 @@ static CPAccessResult access_tacr(CPUARMState *env, const ARMCPRegInfo *ri,
     return CP_ACCESS_OK;
 }
 
+static CPAccessResult access_d128(CPUARMState *env, const ARMCPRegInfo *ri,
+                                  bool isread)
+{
+    int el = arm_current_el(env);
+
+    if (el <= 1 && !(arm_hcrx_el2_eff(env) & HCRX_D128EN)) {
+        return CP_ACCESS_TRAP_EL2;
+    }
+    if (el <= 2 && !(env->cp15.scr_el3 & SCR_D128EN)) {
+        return CP_ACCESS_TRAP_EL3;
+    }
+    return CP_ACCESS_OK;
+}
+
 static void dacr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
 {
     ARMCPU *cpu = env_archcpu(env);
@@ -3279,7 +3293,9 @@ static void define_par_register(ARMCPU *cpu)
         .name = "PAR_EL1", .state = ARM_CP_STATE_AA64,
         .opc0 = 3, .opc1 = 0, .crn = 7, .crm = 4, .opc2 = 0,
         .access = PL1_RW, .fgt = FGT_PAR_EL1,
-        .fieldoffset = offsetof(CPUARMState, cp15.par_el[1])
+        .type = ARM_CP_128BIT, .access128fn = access_d128,
+        .fieldoffset = offsetof(CPUARMState, cp15.par_el[1]),
+        .fieldoffsethi = offsetof(CPUARMState, cp15.par_el1_hi),
     };
 
     static ARMCPRegInfo par64_reginfo[2] = {
diff --git a/target/arm/tcg/cpregs-at.c b/target/arm/tcg/cpregs-at.c
index 0e8f229aa7..9e6af3d974 100644
--- a/target/arm/tcg/cpregs-at.c
+++ b/target/arm/tcg/cpregs-at.c
@@ -353,6 +353,7 @@ static void ats_write64(CPUARMState *env, const ARMCPRegInfo *ri,
 
     ss = for_el3 ? arm_security_space(env) : arm_security_space_below_el3(env);
     env->cp15.par_el[1] = do_ats_write(env, value, access_perm, mmu_idx, ss);
+    env->cp15.par_el1_hi = 0;
 }
 
 static CPAccessResult ats_access(CPUARMState *env, const ARMCPRegInfo *ri,
@@ -496,6 +497,7 @@ static void ats_s1e1a(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
     ARMSecuritySpace ss = arm_security_space_below_el3(env);
 
     env->cp15.par_el[1] = do_ats_write(env, value, 0, mmu_idx, ss);
+    env->cp15.par_el1_hi = 0;
 }
 
 static void ats_s1e2a(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
@@ -505,12 +507,14 @@ static void ats_s1e2a(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
     ARMSecuritySpace ss = arm_security_space_below_el3(env);
 
     env->cp15.par_el[1] = do_ats_write(env, value, 0, mmu_idx, ss);
+    env->cp15.par_el1_hi = 0;
 }
 
 static void ats_s1e3a(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
 {
     env->cp15.par_el[1] = do_ats_write(env, value, 0, ARMMMUIdx_E3,
                                        arm_security_space(env));
+    env->cp15.par_el1_hi = 0;
 }
 
 static const ARMCPRegInfo ats1a_reginfo[] = {
-- 
2.43.0

[PATCH v2 22/37] target/arm: Consolidate definitions of TTBR[01]

[Richard Henderson]

Create a function define_ttbr_register which handles the 3
distinct cases for TTBR[01] registers.

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/helper.c | 165 +++++++++++++++++++++++++++++++-------------
 1 file changed, 116 insertions(+), 49 deletions(-)

diff --git a/target/arm/helper.c b/target/arm/helper.c
index f9a2a7d62d..550ba77c5c 100644
--- a/target/arm/helper.c
+++ b/target/arm/helper.c
@@ -3011,26 +3011,6 @@ static const ARMCPRegInfo vmsa_cp_reginfo[] = {
       .vhe_redir_to_el2 = ENCODE_AA64_CP_REG(3, 4, 5, 2, 0),
       .vhe_redir_to_el01 = ENCODE_AA64_CP_REG(3, 5, 5, 2, 0),
       .fieldoffset = offsetof(CPUARMState, cp15.esr_el[1]), .resetvalue = 0, },
-    { .name = "TTBR0_EL1", .state = ARM_CP_STATE_BOTH,
-      .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 0, .opc2 = 0,
-      .access = PL1_RW, .accessfn = access_tvm_trvm,
-      .fgt = FGT_TTBR0_EL1,
-      .nv2_redirect_offset = 0x200 | NV2_REDIR_NV1,
-      .vhe_redir_to_el2 = ENCODE_AA64_CP_REG(3, 4, 2, 0, 0),
-      .vhe_redir_to_el01 = ENCODE_AA64_CP_REG(3, 5, 2, 0, 0),
-      .writefn = vmsa_ttbr_write, .resetvalue = 0, .raw_writefn = raw_write,
-      .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr0_s),
-                             offsetof(CPUARMState, cp15.ttbr0_ns) } },
-    { .name = "TTBR1_EL1", .state = ARM_CP_STATE_BOTH,
-      .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 0, .opc2 = 1,
-      .access = PL1_RW, .accessfn = access_tvm_trvm,
-      .fgt = FGT_TTBR1_EL1,
-      .nv2_redirect_offset = 0x210 | NV2_REDIR_NV1,
-      .vhe_redir_to_el2 = ENCODE_AA64_CP_REG(3, 4, 2, 0, 1),
-      .vhe_redir_to_el01 = ENCODE_AA64_CP_REG(3, 5, 2, 0, 1),
-      .writefn = vmsa_ttbr_write, .resetvalue = 0, .raw_writefn = raw_write,
-      .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr1_s),
-                             offsetof(CPUARMState, cp15.ttbr1_ns) } },
     { .name = "TCR_EL1", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 2,
       .access = PL1_RW, .accessfn = access_tvm_trvm,
@@ -3256,23 +3236,6 @@ static const ARMCPRegInfo lpae_cp_reginfo[] = {
     { .name = "AMAIR1", .cp = 15, .crn = 10, .crm = 3, .opc1 = 0, .opc2 = 1,
       .access = PL1_RW, .accessfn = access_tvm_trvm,
       .type = ARM_CP_CONST, .resetvalue = 0 },
-
-    /*
-     * The primary definitions of TTBR[01]_EL1 are in vmsa_cp_reginfo[].
-     * Here we need only provide the 64-bit views for AArch32.
-     */
-    { .name = "TTBR0", .cp = 15, .crm = 2, .opc1 = 0,
-      .access = PL1_RW, .accessfn = access_tvm_trvm,
-      .type = ARM_CP_64BIT | ARM_CP_ALIAS,
-      .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr0_s),
-                             offsetof(CPUARMState, cp15.ttbr0_ns) },
-      .writefn = vmsa_ttbr_write, .raw_writefn = raw_write },
-    { .name = "TTBR1", .cp = 15, .crm = 2, .opc1 = 1,
-      .access = PL1_RW, .accessfn = access_tvm_trvm,
-      .type = ARM_CP_64BIT | ARM_CP_ALIAS,
-      .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr1_s),
-                             offsetof(CPUARMState, cp15.ttbr1_ns) },
-      .writefn = vmsa_ttbr_write, .raw_writefn = raw_write },
 };
 
 static void define_par_register(ARMCPU *cpu)
@@ -3342,6 +3305,121 @@ static void define_par_register(ARMCPU *cpu)
     define_arm_cp_regs(cpu, par64_reginfo);
 }
 
+static void define_ttbr_registers(ARMCPU *cpu)
+{
+    /*
+     * For v8:
+     * The aarch64 regs are primary, since they might be 128-bit.
+     * The aarch32 64-bit non-secure regs are secondary to aa64 el1.
+     * The aarch32 64-bit httbr is secondary to aa64 el2.
+     * The aarch32 64-bit secure ttbr0 is secondary to aa64 el3.
+     * The aarch32 64-bit secure ttbr1 is primary.
+     *
+     * For v7:
+     * The aarch32 64-bit s+ns regs are primary.
+     *
+     * The aarch32 32-bit regs are secondary to one of the above,
+     * and we also don't expose them to gdb.
+     */
+    static const ARMCPRegInfo ttbrv8_reginfo[] = {
+        { .name = "TTBR0_EL1", .state = ARM_CP_STATE_AA64,
+          .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 0, .opc2 = 0,
+          .access = PL1_RW, .accessfn = access_tvm_trvm,
+          .fgt = FGT_TTBR0_EL1,
+          .nv2_redirect_offset = 0x200 | NV2_REDIR_NV1,
+          .vhe_redir_to_el2 = ENCODE_AA64_CP_REG(3, 4, 2, 0, 0),
+          .vhe_redir_to_el01 = ENCODE_AA64_CP_REG(3, 5, 2, 0, 0),
+          .writefn = vmsa_ttbr_write, .raw_writefn = raw_write,
+          .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el[1]) },
+        { .name = "TTBR1_EL1", .state = ARM_CP_STATE_AA64,
+          .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 0, .opc2 = 1,
+          .access = PL1_RW, .accessfn = access_tvm_trvm,
+          .fgt = FGT_TTBR1_EL1,
+          .nv2_redirect_offset = 0x210 | NV2_REDIR_NV1,
+          .vhe_redir_to_el2 = ENCODE_AA64_CP_REG(3, 4, 2, 0, 1),
+          .vhe_redir_to_el01 = ENCODE_AA64_CP_REG(3, 5, 2, 0, 1),
+          .writefn = vmsa_ttbr_write, .raw_writefn = raw_write,
+          .fieldoffset = offsetof(CPUARMState, cp15.ttbr1_el[1]) },
+        { .name = "TTBR0_EL2", .state = ARM_CP_STATE_AA64,
+          .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 0,
+          .access = PL2_RW, .resetvalue = 0,
+          .writefn = vmsa_tcr_ttbr_el2_write, .raw_writefn = raw_write,
+          .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el[2]) },
+        { .name = "TTBR0_EL3", .state = ARM_CP_STATE_AA64,
+          .opc0 = 3, .opc1 = 6, .crn = 2, .crm = 0, .opc2 = 0,
+          .access = PL3_RW, .resetvalue = 0,
+          .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el[3]) },
+    };
+
+    static ARMCPRegInfo ttbr64_reginfo[] = {
+        [0 ... 3] = {
+            .cp = 15, .crm = 2, .type = ARM_CP_64BIT,
+            .access = PL1_RW, .accessfn = access_tvm_trvm,
+            .writefn = vmsa_ttbr_write, .raw_writefn = raw_write
+        },
+        [0 ... 1].opc1 = 0,
+        [0].name = "TTBR0",
+        [0].secure = ARM_CP_SECSTATE_NS,
+        [0].fieldoffset = offsetof(CPUARMState, cp15.ttbr0_ns),
+        [1].name = "TTBR0_S",
+        [1].secure = ARM_CP_SECSTATE_S,
+        [1].fieldoffset = offsetof(CPUARMState, cp15.ttbr0_s),
+
+        [2 ... 3].opc1 = 1,
+        [2].name = "TTBR1",
+        [2].secure = ARM_CP_SECSTATE_NS,
+        [2].fieldoffset = offsetof(CPUARMState, cp15.ttbr1_ns),
+        [3].name = "TTBR1_S",
+        [3].secure = ARM_CP_SECSTATE_S,
+        [3].fieldoffset = offsetof(CPUARMState, cp15.ttbr1_s),
+
+        [4] = {
+            .name = "HTTBR", .cp = 15, .crm = 2, .opc1 = 4,
+            .access = PL2_RW, .type = ARM_CP_64BIT,
+            .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el[2])
+        },
+    };
+
+    static ARMCPRegInfo ttbr32_reginfo[] = {
+        { .name = "TTBR0", .cp = 15, .opc1 = 0, .crn = 2, .crm = 0, .opc2 = 0,
+          .access = PL1_RW, .accessfn = access_tvm_trvm,
+          .writefn = vmsa_ttbr_write, .raw_writefn = raw_write,
+          .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr0_s),
+                                 offsetof(CPUARMState, cp15.ttbr0_ns) } },
+        { .name = "TTBR1", .cp = 15, .opc1 = 0, .crn = 2, .crm = 0, .opc2 = 1,
+          .access = PL1_RW, .accessfn = access_tvm_trvm,
+          .writefn = vmsa_ttbr_write, .raw_writefn = raw_write,
+          .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr1_s),
+                                 offsetof(CPUARMState, cp15.ttbr1_ns) } },
+    };
+
+    CPUARMState *env = &cpu->env;
+
+    /* With only VMSA, define a 32-bit reg that filters bits from write. */
+    if (!arm_feature(env, ARM_FEATURE_LPAE)) {
+        define_arm_cp_regs(cpu, ttbr32_reginfo);
+        return;
+    }
+
+    /* With LPAE, the 32-bit regs are aliases of 64-bit regs. */
+    for (int i = 0; i < ARRAY_SIZE(ttbr32_reginfo); ++i) {
+        ttbr32_reginfo[i].type = ARM_CP_ALIAS | ARM_CP_NO_GDB;
+    }
+    define_arm_cp_regs(cpu, ttbr32_reginfo);
+
+    if (arm_feature(env, ARM_FEATURE_V8)) {
+        define_arm_cp_regs(cpu, ttbrv8_reginfo);
+
+        ttbr64_reginfo[0].type |= ARM_CP_ALIAS;
+        ttbr64_reginfo[1].type |= ARM_CP_ALIAS;
+        ttbr64_reginfo[2].type |= ARM_CP_ALIAS;
+        /* Index 3, TTBR1_S, is not an alias. */
+        ttbr64_reginfo[4].type |= ARM_CP_ALIAS;
+    }
+
+    define_arm_cp_regs(cpu, ttbr64_reginfo);
+}
+
 static uint64_t aa64_fpcr_read(CPUARMState *env, const ARMCPRegInfo *ri)
 {
     return vfp_get_fpcr(env);
@@ -4468,14 +4546,6 @@ static const ARMCPRegInfo el2_cp_reginfo[] = {
       .access = PL2_RW, .resetvalue = 0,
       .nv2_redirect_offset = 0x90,
       .fieldoffset = offsetof(CPUARMState, cp15.tpidr_el[2]) },
-    { .name = "TTBR0_EL2", .state = ARM_CP_STATE_AA64,
-      .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 0,
-      .access = PL2_RW, .resetvalue = 0,
-      .writefn = vmsa_tcr_ttbr_el2_write, .raw_writefn = raw_write,
-      .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el[2]) },
-    { .name = "HTTBR", .cp = 15, .opc1 = 4, .crm = 2,
-      .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_ALIAS,
-      .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el[2]) },
 #ifndef CONFIG_USER_ONLY
     { .name = "CNTHCTL_EL2", .state = ARM_CP_STATE_BOTH,
       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 1, .opc2 = 0,
@@ -4660,10 +4730,6 @@ static const ARMCPRegInfo el3_cp_reginfo[] = {
       .access = PL1_RW, .accessfn = access_trap_aa32s_el1,
       .writefn = vbar_write, .resetvalue = 0,
       .fieldoffset = offsetof(CPUARMState, cp15.mvbar) },
-    { .name = "TTBR0_EL3", .state = ARM_CP_STATE_AA64,
-      .opc0 = 3, .opc1 = 6, .crn = 2, .crm = 0, .opc2 = 0,
-      .access = PL3_RW, .resetvalue = 0,
-      .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el[3]) },
     { .name = "TCR_EL3", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 6, .crn = 2, .crm = 0, .opc2 = 2,
       .access = PL3_RW,
@@ -7174,6 +7240,7 @@ void register_cp_regs_for_features(ARMCPU *cpu)
     } else {
         define_arm_cp_regs(cpu, vmsa_pmsa_cp_reginfo);
         define_arm_cp_regs(cpu, vmsa_cp_reginfo);
+        define_ttbr_registers(cpu);
         /* TTCBR2 is introduced with ARMv8.2-AA32HPD.  */
         if (cpu_isar_feature(aa32_hpd, cpu)) {
             define_one_arm_cp_reg(cpu, &ttbcr2_reginfo);
-- 
2.43.0

[PATCH v2 23/37] target/arm: Split out flush_if_asid_change

[Richard Henderson]

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/helper.c | 33 +++++++++++++++++++--------------
 1 file changed, 19 insertions(+), 14 deletions(-)

diff --git a/target/arm/helper.c b/target/arm/helper.c
index 550ba77c5c..c6d290ce7c 100644
--- a/target/arm/helper.c
+++ b/target/arm/helper.c
@@ -2929,6 +2929,17 @@ static void vmsa_tcr_el12_write(CPUARMState *env, const ARMCPRegInfo *ri,
     raw_write(env, ri, value);
 }
 
+static void flush_if_asid_change(CPUARMState *env, const ARMCPRegInfo *ri,
+                                 uint64_t new, unsigned mask)
+{
+    uint64_t old = raw_read(env, ri);
+
+    /* The ASID or VMID is in bits [63:48]. */
+    if ((old ^ new) >> 48) {
+        tlb_flush_by_mmuidx(env_cpu(env), mask);
+    }
+}
+
 static void vmsa_ttbr_write(CPUARMState *env, const ARMCPRegInfo *ri,
                             uint64_t value)
 {
@@ -2950,14 +2961,13 @@ static void vmsa_tcr_ttbr_el2_write(CPUARMState *env, const ARMCPRegInfo *ri,
      * TCR_EL2.A1 to know if this is really the TTBRx_EL2 that
      * holds the active ASID, only checking the field that might.
      */
-    if (extract64(raw_read(env, ri) ^ value, 48, 16) &&
-        (arm_hcr_el2_eff(env) & HCR_E2H)) {
-        uint16_t mask = ARMMMUIdxBit_E20_2 |
-                        ARMMMUIdxBit_E20_2_PAN |
-                        ARMMMUIdxBit_E20_2_GCS |
-                        ARMMMUIdxBit_E20_0 |
-                        ARMMMUIdxBit_E20_0_GCS;
-        tlb_flush_by_mmuidx(env_cpu(env), mask);
+    if (arm_hcr_el2_eff(env) & HCR_E2H) {
+        flush_if_asid_change(env, ri, value,
+                             ARMMMUIdxBit_E20_2 |
+                             ARMMMUIdxBit_E20_2_PAN |
+                             ARMMMUIdxBit_E20_2_GCS |
+                             ARMMMUIdxBit_E20_0 |
+                             ARMMMUIdxBit_E20_0_GCS);
     }
     raw_write(env, ri, value);
 }
@@ -2965,16 +2975,11 @@ static void vmsa_tcr_ttbr_el2_write(CPUARMState *env, const ARMCPRegInfo *ri,
 static void vttbr_write(CPUARMState *env, const ARMCPRegInfo *ri,
                         uint64_t value)
 {
-    ARMCPU *cpu = env_archcpu(env);
-    CPUState *cs = CPU(cpu);
-
     /*
      * A change in VMID to the stage2 page table (Stage2) invalidates
      * the stage2 and combined stage 1&2 tlbs (EL10_1 and EL10_0).
      */
-    if (extract64(raw_read(env, ri) ^ value, 48, 16) != 0) {
-        tlb_flush_by_mmuidx(cs, alle1_tlbmask(env));
-    }
+    flush_if_asid_change(env, ri, value, alle1_tlbmask(env));
     raw_write(env, ri, value);
 }
 
-- 
2.43.0

[PATCH v2 24/37] target/arm: Use flush_if_asid_change in vmsa_ttbr_write

[Richard Henderson]

Only flush the subset of tlbs that are affected by the ttbr
register to which we are writing.

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/helper.c | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/target/arm/helper.c b/target/arm/helper.c
index c6d290ce7c..2b55e219c2 100644
--- a/target/arm/helper.c
+++ b/target/arm/helper.c
@@ -2943,11 +2943,20 @@ static void flush_if_asid_change(CPUARMState *env, const ARMCPRegInfo *ri,
 static void vmsa_ttbr_write(CPUARMState *env, const ARMCPRegInfo *ri,
                             uint64_t value)
 {
-    /* If the ASID changes (with a 64-bit write), we must flush the TLB.  */
-    if (cpreg_field_type(ri) == MO_64 &&
-        extract64(raw_read(env, ri) ^ value, 48, 16) != 0) {
-        ARMCPU *cpu = env_archcpu(env);
-        tlb_flush(CPU(cpu));
+    /*
+     * If the ASID changes (with a 64-bit write), we must flush the TLB.
+     * The non-secure ttbr registers affect the EL1 regime;
+     * the secure ttbr registers affect the AA32 EL3 regime.
+     */
+    if (cpreg_field_type(ri) == MO_64) {
+        flush_if_asid_change(env, ri, value,
+                             ri->secure & ARM_CP_SECSTATE_S
+                             ? (ARMMMUIdxBit_E30_0 |
+                                ARMMMUIdxBit_E30_3_PAN |
+                                ARMMMUIdxBit_E3)
+                             : (ARMMMUIdxBit_E10_1 |
+                                ARMMMUIdxBit_E10_1_PAN |
+                                ARMMMUIdxBit_E10_0));
     }
     raw_write(env, ri, value);
 }
-- 
2.43.0

[PATCH v2 25/37] target/arm: Extend TTBR system registers to 128-bit

[Richard Henderson]

So far, just extend the data type and check access; do not yet
consume the 128-bit table format.

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/cpu.h    |  9 +++--
 target/arm/helper.c | 83 ++++++++++++++++++++++++++++++++++++---------
 2 files changed, 74 insertions(+), 18 deletions(-)

diff --git a/target/arm/cpu.h b/target/arm/cpu.h
index e03d832717..65037f1dd3 100644
--- a/target/arm/cpu.h
+++ b/target/arm/cpu.h
@@ -344,7 +344,8 @@ typedef struct CPUArchState {
         uint64_t cptr_el[4];  /* ARMv8 feature trap registers */
         uint64_t sder; /* Secure debug enable register. */
         uint32_t nsacr; /* Non-secure access control register. */
-        union { /* MMU translation table base 0. */
+        /* MMU translation table bases. */
+        union {
             struct {
                 uint64_t _unused_ttbr0_0;
                 uint64_t ttbr0_ns;
@@ -353,7 +354,7 @@ typedef struct CPUArchState {
             };
             uint64_t ttbr0_el[4];
         };
-        union { /* MMU translation table base 1. */
+        union {
             struct {
                 uint64_t _unused_ttbr1_0;
                 uint64_t ttbr1_ns;
@@ -364,6 +365,10 @@ typedef struct CPUArchState {
         };
         uint64_t vttbr_el2; /* Virtualization Translation Table Base.  */
         uint64_t vsttbr_el2; /* Secure Virtualization Translation Table. */
+        /* High 64 bits of 128-bit Translation Table Bases. */
+        uint64_t ttbr0_el_hi[4];
+        uint64_t ttbr1_el_hi[4];
+        uint64_t vttbr_el2_hi;
         /* MMU translation table base control. */
         uint64_t tcr_el[4];
         uint64_t tcr2_el[3];
diff --git a/target/arm/helper.c b/target/arm/helper.c
index 2b55e219c2..90a940aeb9 100644
--- a/target/arm/helper.c
+++ b/target/arm/helper.c
@@ -108,10 +108,6 @@ void raw_write128(CPUARMState *env, const ARMCPRegInfo *ri,
     CPREG_FIELD128_H64(env, ri) = valuehi;
 }
 
-#undef CPREG_FIELD32
-#undef CPREG_FIELD64
-#undef CPREG_FIELD128_H64
-
 static void *raw_ptr(CPUARMState *env, const ARMCPRegInfo *ri)
 {
     return (char *)env + ri->fieldoffset;
@@ -522,6 +518,16 @@ static CPAccessResult access_d128(CPUARMState *env, const ARMCPRegInfo *ri,
     return CP_ACCESS_OK;
 }
 
+static CPAccessResult access_tvm_trvm_d128(CPUARMState *env,
+                                           const ARMCPRegInfo *ri, bool isread)
+{
+    CPAccessResult ret = access_tvm_trvm(env, ri, isread);
+    if (ret == CP_ACCESS_OK) {
+        ret = access_d128(env, ri, isread);
+    }
+    return ret;
+}
+
 static void dacr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
 {
     ARMCPU *cpu = env_archcpu(env);
@@ -2932,7 +2938,7 @@ static void vmsa_tcr_el12_write(CPUARMState *env, const ARMCPRegInfo *ri,
 static void flush_if_asid_change(CPUARMState *env, const ARMCPRegInfo *ri,
                                  uint64_t new, unsigned mask)
 {
-    uint64_t old = raw_read(env, ri);
+    uint64_t old = CPREG_FIELD64(env, ri);
 
     /* The ASID or VMID is in bits [63:48]. */
     if ((old ^ new) >> 48) {
@@ -2961,6 +2967,16 @@ static void vmsa_ttbr_write(CPUARMState *env, const ARMCPRegInfo *ri,
     raw_write(env, ri, value);
 }
 
+static void vmsa_ttbr_write128(CPUARMState *env, const ARMCPRegInfo *ri,
+                               uint64_t vallo, uint64_t valhi)
+{
+    flush_if_asid_change(env, ri, vallo,
+                         ARMMMUIdxBit_E10_1 |
+                         ARMMMUIdxBit_E10_1_PAN |
+                         ARMMMUIdxBit_E10_0);
+    raw_write128(env, ri, vallo, valhi);
+}
+
 static void vmsa_tcr_ttbr_el2_write(CPUARMState *env, const ARMCPRegInfo *ri,
                                     uint64_t value)
 {
@@ -2981,6 +2997,18 @@ static void vmsa_tcr_ttbr_el2_write(CPUARMState *env, const ARMCPRegInfo *ri,
     raw_write(env, ri, value);
 }
 
+static void vmsa_tcr_ttbr_el2_write128(CPUARMState *env, const ARMCPRegInfo *ri,
+                                       uint64_t vallo, uint64_t valhi)
+{
+    if (arm_hcr_el2_eff(env) & HCR_E2H) {
+        flush_if_asid_change(env, ri, vallo,
+                             ARMMMUIdxBit_E20_2 |
+                             ARMMMUIdxBit_E20_2_PAN |
+                             ARMMMUIdxBit_E20_0);
+    }
+    raw_write128(env, ri, vallo, valhi);
+}
+
 static void vttbr_write(CPUARMState *env, const ARMCPRegInfo *ri,
                         uint64_t value)
 {
@@ -2992,6 +3020,13 @@ static void vttbr_write(CPUARMState *env, const ARMCPRegInfo *ri,
     raw_write(env, ri, value);
 }
 
+static void vttbr_write128(CPUARMState *env, const ARMCPRegInfo *ri,
+                           uint64_t vallo, uint64_t valhi)
+{
+    flush_if_asid_change(env, ri, vallo, alle1_tlbmask(env));
+    raw_write128(env, ri, vallo, valhi);
+}
+
 static const ARMCPRegInfo vmsa_pmsa_cp_reginfo[] = {
     { .name = "DFSR", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 0,
       .access = PL1_RW, .accessfn = access_tvm_trvm, .type = ARM_CP_ALIAS,
@@ -3339,26 +3374,35 @@ static void define_ttbr_registers(ARMCPU *cpu)
         { .name = "TTBR0_EL1", .state = ARM_CP_STATE_AA64,
           .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 0, .opc2 = 0,
           .access = PL1_RW, .accessfn = access_tvm_trvm,
-          .fgt = FGT_TTBR0_EL1,
+          .access128fn = access_tvm_trvm_d128,
+          .fgt = FGT_TTBR0_EL1, .type = ARM_CP_128BIT,
           .nv2_redirect_offset = 0x200 | NV2_REDIR_NV1,
           .vhe_redir_to_el2 = ENCODE_AA64_CP_REG(3, 4, 2, 0, 0),
           .vhe_redir_to_el01 = ENCODE_AA64_CP_REG(3, 5, 2, 0, 0),
           .writefn = vmsa_ttbr_write, .raw_writefn = raw_write,
-          .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el[1]) },
+          .write128fn = vmsa_ttbr_write128, .raw_write128fn = raw_write128,
+          .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el[1]),
+          .fieldoffsethi = offsetof(CPUARMState, cp15.ttbr0_el_hi[1]) },
         { .name = "TTBR1_EL1", .state = ARM_CP_STATE_AA64,
           .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 0, .opc2 = 1,
           .access = PL1_RW, .accessfn = access_tvm_trvm,
-          .fgt = FGT_TTBR1_EL1,
+          .access128fn = access_tvm_trvm_d128,
+          .fgt = FGT_TTBR1_EL1, .type = ARM_CP_128BIT,
           .nv2_redirect_offset = 0x210 | NV2_REDIR_NV1,
           .vhe_redir_to_el2 = ENCODE_AA64_CP_REG(3, 4, 2, 0, 1),
           .vhe_redir_to_el01 = ENCODE_AA64_CP_REG(3, 5, 2, 0, 1),
           .writefn = vmsa_ttbr_write, .raw_writefn = raw_write,
-          .fieldoffset = offsetof(CPUARMState, cp15.ttbr1_el[1]) },
+          .write128fn = vmsa_ttbr_write128, .raw_write128fn = raw_write128,
+          .fieldoffset = offsetof(CPUARMState, cp15.ttbr1_el[1]),
+          .fieldoffsethi = offsetof(CPUARMState, cp15.ttbr1_el_hi[1]) },
         { .name = "TTBR0_EL2", .state = ARM_CP_STATE_AA64,
           .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 0,
-          .access = PL2_RW, .resetvalue = 0,
+          .access = PL2_RW, .access128fn = access_d128, .type = ARM_CP_128BIT,
           .writefn = vmsa_tcr_ttbr_el2_write, .raw_writefn = raw_write,
-          .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el[2]) },
+          .write128fn = vmsa_tcr_ttbr_el2_write128,
+          .raw_write128fn = raw_write128,
+          .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el[2]),
+          .fieldoffsethi = offsetof(CPUARMState, cp15.ttbr0_el_hi[2]) },
         { .name = "TTBR0_EL3", .state = ARM_CP_STATE_AA64,
           .opc0 = 3, .opc1 = 6, .crn = 2, .crm = 0, .opc2 = 0,
           .access = PL3_RW, .resetvalue = 0,
@@ -4548,9 +4592,13 @@ static const ARMCPRegInfo el2_cp_reginfo[] = {
       .writefn = vttbr_write, .raw_writefn = raw_write },
     { .name = "VTTBR_EL2", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 0,
-      .access = PL2_RW, .writefn = vttbr_write, .raw_writefn = raw_write,
+      .type = ARM_CP_128BIT,
+      .access = PL2_RW, .access128fn = access_d128,
+      .writefn = vttbr_write, .raw_writefn = raw_write,
+      .write128fn = vttbr_write128, .raw_write128fn = raw_write128,
       .nv2_redirect_offset = 0x20,
-      .fieldoffset = offsetof(CPUARMState, cp15.vttbr_el2) },
+      .fieldoffset = offsetof(CPUARMState, cp15.vttbr_el2),
+      .fieldoffsethi = offsetof(CPUARMState, cp15.vttbr_el2_hi) },
     { .name = "SCTLR_EL2", .state = ARM_CP_STATE_BOTH,
       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 0, .opc2 = 0,
       .access = PL2_RW, .raw_writefn = raw_write, .writefn = sctlr_write,
@@ -6196,9 +6244,12 @@ static const ARMCPRegInfo contextidr_el2 = {
 static const ARMCPRegInfo vhe_reginfo[] = {
     { .name = "TTBR1_EL2", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 1,
-      .access = PL2_RW, .writefn = vmsa_tcr_ttbr_el2_write,
-      .raw_writefn = raw_write,
-      .fieldoffset = offsetof(CPUARMState, cp15.ttbr1_el[2]) },
+      .type = ARM_CP_128BIT,
+      .access = PL2_RW, .access128fn = access_d128,
+      .writefn = vmsa_tcr_ttbr_el2_write, .raw_writefn = raw_write,
+      .write128fn = vmsa_tcr_ttbr_el2_write128, .raw_write128fn = raw_write128,
+      .fieldoffset = offsetof(CPUARMState, cp15.ttbr1_el[2]),
+      .fieldoffset = offsetof(CPUARMState, cp15.ttbr1_el_hi[2]) },
 #ifndef CONFIG_USER_ONLY
     { .name = "CNTHV_CVAL_EL2", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 3, .opc2 = 2,
-- 
2.43.0

[PATCH v2 26/37] target/arm: Implement TLBIP IPAS2E1, IPAS2LE1

[Richard Henderson]

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/tcg/tlb-insns.c | 26 ++++++++++++++++++++++----
 1 file changed, 22 insertions(+), 4 deletions(-)

diff --git a/target/arm/tcg/tlb-insns.c b/target/arm/tcg/tlb-insns.c
index 1a0a332583..bf8f007869 100644
--- a/target/arm/tcg/tlb-insns.c
+++ b/target/arm/tcg/tlb-insns.c
@@ -512,6 +512,20 @@ static void tlbi_aa64_ipas2e1_write(CPUARMState *env, const ARMCPRegInfo *ri,
     }
 }
 
+static void tlbi_aa64_ipas2e1_write128(CPUARMState *env, const ARMCPRegInfo *ri,
+                                       uint64_t vallo, uint64_t valhi)
+{
+    CPUState *cs = env_cpu(env);
+    int mask = ipas2e1_tlbmask(env, vallo);
+    uint64_t pageaddr = sextract64(valhi << 12, 0, 56);
+
+    if (tlb_force_broadcast(env)) {
+        tlb_flush_page_by_mmuidx_all_cpus_synced(cs, pageaddr, mask);
+    } else {
+        tlb_flush_page_by_mmuidx(cs, pageaddr, mask);
+    }
+}
+
 static void tlbi_aa64_ipas2e1is_write(CPUARMState *env, const ARMCPRegInfo *ri,
                                       uint64_t value)
 {
@@ -721,12 +735,16 @@ static const ARMCPRegInfo tlbi_v8_cp_reginfo[] = {
       .writefn = tlbi_aa64_alle1is_write },
     { .name = "TLBI_IPAS2E1", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 1,
-      .access = PL2_W, .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
-      .writefn = tlbi_aa64_ipas2e1_write },
+      .access = PL2_W,
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_128BIT,
+      .writefn = tlbi_aa64_ipas2e1_write,
+      .write128fn = tlbi_aa64_ipas2e1_write128 },
     { .name = "TLBI_IPAS2LE1", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 5,
-      .access = PL2_W, .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
-      .writefn = tlbi_aa64_ipas2e1_write },
+      .access = PL2_W,
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_128BIT,
+      .writefn = tlbi_aa64_ipas2e1_write,
+      .write128fn = tlbi_aa64_ipas2e1_write128 },
     { .name = "TLBI_ALLE1", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 4,
       .access = PL2_W, .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
-- 
2.43.0

[PATCH v2 27/37] target/arm: Implement TLBIP IPAS2E1IS, IPAS2LE1IS

[Richard Henderson]

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/tcg/tlb-insns.c | 23 +++++++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

diff --git a/target/arm/tcg/tlb-insns.c b/target/arm/tcg/tlb-insns.c
index bf8f007869..f7510a1208 100644
--- a/target/arm/tcg/tlb-insns.c
+++ b/target/arm/tcg/tlb-insns.c
@@ -536,6 +536,17 @@ static void tlbi_aa64_ipas2e1is_write(CPUARMState *env, const ARMCPRegInfo *ri,
     tlb_flush_page_by_mmuidx_all_cpus_synced(cs, pageaddr, mask);
 }
 
+static void tlbi_aa64_ipas2e1is_write128(CPUARMState *env,
+                                         const ARMCPRegInfo *ri,
+                                         uint64_t vallo, uint64_t valhi)
+{
+    CPUState *cs = env_cpu(env);
+    int mask = ipas2e1_tlbmask(env, vallo);
+    uint64_t pageaddr = sextract64(valhi << 12, 0, 56);
+
+    tlb_flush_page_by_mmuidx_all_cpus_synced(cs, pageaddr, mask);
+}
+
 static const ARMCPRegInfo tlbi_not_v7_cp_reginfo[] = {
     /*
      * MMU TLB control. Note that the wildcarding means we cover not just
@@ -719,12 +730,16 @@ static const ARMCPRegInfo tlbi_v8_cp_reginfo[] = {
       .writefn = tlbi_aa64_vae1_write },
     { .name = "TLBI_IPAS2E1IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 1,
-      .access = PL2_W, .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
-      .writefn = tlbi_aa64_ipas2e1is_write },
+      .access = PL2_W,
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_128BIT,
+      .writefn = tlbi_aa64_ipas2e1is_write,
+      .write128fn = tlbi_aa64_ipas2e1is_write128 },
     { .name = "TLBI_IPAS2LE1IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 5,
-      .access = PL2_W, .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
-      .writefn = tlbi_aa64_ipas2e1is_write },
+      .access = PL2_W,
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_128BIT,
+      .writefn = tlbi_aa64_ipas2e1is_write,
+      .write128fn = tlbi_aa64_ipas2e1is_write128 },
     { .name = "TLBI_ALLE1IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 4,
       .access = PL2_W, .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
-- 
2.43.0

[PATCH v2 28/37] target/arm: Implement TLBIP RVAE1, RVAAE1, RVALE1, RVAALE1

[Richard Henderson]

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/tcg/tlb-insns.c | 93 ++++++++++++++++++++++++++++++--------
 1 file changed, 74 insertions(+), 19 deletions(-)

diff --git a/target/arm/tcg/tlb-insns.c b/target/arm/tcg/tlb-insns.c
index f7510a1208..daadba7bfc 100644
--- a/target/arm/tcg/tlb-insns.c
+++ b/target/arm/tcg/tlb-insns.c
@@ -918,16 +918,43 @@ static TLBIRange tlbi_aa64_get_range(CPUARMState *env, ARMMMUIdx mmuidx,
     return ret;
 }
 
-static void do_rvae_write(CPUARMState *env, uint64_t value,
-                          int idxmap, bool synced)
+static TLBIRange tlbi_aa64_get_range128(CPUARMState *env, ARMMMUIdx mmuidx,
+                                        uint64_t vallo, uint64_t valhi)
 {
-    ARMMMUIdx one_idx = ARM_MMU_IDX_A | ctz32(idxmap);
-    TLBIRange range;
-    int bits;
+    uint64_t uaddr = extract64(valhi << 12, 0, 56);
+    ARMVAParameters param = aa64_va_parameters(env, uaddr, mmuidx, true, false);
+    TLBIRange ret = { };
+    unsigned page_size_granule = extract64(vallo, 46, 2);
+    ARMGranuleSize gran = tlbi_range_tg_to_gran_size(page_size_granule);
 
-    range = tlbi_aa64_get_range(env, one_idx, value);
-    bits = tlbbits_for_regime(env, one_idx, range.base);
+    /* The granule encoded in value must match the granule in use. */
+    if (gran != param.gran) {
+        qemu_log_mask(LOG_GUEST_ERROR, "Invalid tlbi page size granule %d\n",
+                      page_size_granule);
+    } else {
+        unsigned page_shift = arm_granule_bits(gran);
+        unsigned num = extract64(vallo, 39, 5);
+        unsigned scale = extract64(vallo, 44, 2);
+        unsigned exponent = (5 * scale) + 1;
+        uint64_t max = 1ull << 56;
 
+        ret.length = (num + 1) << (exponent + page_shift);
+        ret.length = MIN(ret.length, max - uaddr);
+        /*
+         * Note that TLBIPRange ignores the high bits, because the HW TLB
+         * does not use it.  But the qemu softmmu tlb does, so sign-extend
+         * if and only if the regime has two ranges.
+         */
+        ret.base = uaddr | (-(uint64_t)param.select << 56);
+    }
+
+    return ret;
+}
+
+static void do_flush_range(CPUARMState *env, ARMMMUIdx one_idx, int idxmap,
+                           bool synced, TLBIRange range)
+{
+    int bits = tlbbits_for_regime(env, one_idx, range.base);
     if (synced) {
         tlb_flush_range_by_mmuidx_all_cpus_synced(env_cpu(env),
                                                   range.base,
@@ -940,6 +967,22 @@ static void do_rvae_write(CPUARMState *env, uint64_t value,
     }
 }
 
+static void do_rvae_write(CPUARMState *env, uint64_t value,
+                          int idxmap, bool synced)
+{
+    ARMMMUIdx one_idx = ARM_MMU_IDX_A | ctz32(idxmap);
+    do_flush_range(env, one_idx, idxmap, synced,
+                   tlbi_aa64_get_range(env, one_idx, value));
+}
+
+static void do_rvae_write128(CPUARMState *env, uint64_t vallo, uint64_t valhi,
+                             int idxmap, bool synced)
+{
+    ARMMMUIdx one_idx = ARM_MMU_IDX_A | ctz32(idxmap);
+    do_flush_range(env, one_idx, idxmap, synced,
+                   tlbi_aa64_get_range128(env, one_idx, vallo, valhi));
+}
+
 static void tlbi_aa64_rvae1_write(CPUARMState *env,
                                   const ARMCPRegInfo *ri,
                                   uint64_t value)
@@ -955,6 +998,14 @@ static void tlbi_aa64_rvae1_write(CPUARMState *env,
                   tlb_force_broadcast(env));
 }
 
+static void tlbi_aa64_rvae1_write128(CPUARMState *env,
+                                     const ARMCPRegInfo *ri,
+                                     uint64_t vallo, uint64_t valhi)
+{
+    do_rvae_write128(env, vallo, valhi, vae1_tlbmask(env),
+                     tlb_force_broadcast(env));
+}
+
 static void tlbi_aa64_rvae1is_write(CPUARMState *env,
                                     const ARMCPRegInfo *ri,
                                     uint64_t value)
@@ -1095,28 +1146,32 @@ static const ARMCPRegInfo tlbirange_reginfo[] = {
       .writefn = tlbi_aa64_rvae1is_write },
     { .name = "TLBI_RVAE1", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 1,
-      .access = PL1_W, .accessfn = access_ttlb,
-      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
+      .access = PL1_W, .accessfn = access_ttlb, .access128fn = access_ttlb,
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_128BIT,
       .fgt = FGT_TLBIRVAE1,
-      .writefn = tlbi_aa64_rvae1_write },
+      .writefn = tlbi_aa64_rvae1_write,
+      .write128fn = tlbi_aa64_rvae1_write128 },
     { .name = "TLBI_RVAAE1", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 3,
-      .access = PL1_W, .accessfn = access_ttlb,
-      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
+      .access = PL1_W, .accessfn = access_ttlb, .access128fn = access_ttlb,
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_128BIT,
       .fgt = FGT_TLBIRVAAE1,
-      .writefn = tlbi_aa64_rvae1_write },
+      .writefn = tlbi_aa64_rvae1_write,
+      .write128fn = tlbi_aa64_rvae1_write128 },
    { .name = "TLBI_RVALE1", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 5,
-      .access = PL1_W, .accessfn = access_ttlb,
-      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
+      .access = PL1_W, .accessfn = access_ttlb, .access128fn = access_ttlb,
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_128BIT,
       .fgt = FGT_TLBIRVALE1,
-      .writefn = tlbi_aa64_rvae1_write },
+      .writefn = tlbi_aa64_rvae1_write,
+      .write128fn = tlbi_aa64_rvae1_write128 },
     { .name = "TLBI_RVAALE1", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 7,
-      .access = PL1_W, .accessfn = access_ttlb,
-      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
+      .access = PL1_W, .accessfn = access_ttlb, .access128fn = access_ttlb,
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_128BIT,
       .fgt = FGT_TLBIRVAALE1,
-      .writefn = tlbi_aa64_rvae1_write },
+      .writefn = tlbi_aa64_rvae1_write,
+      .write128fn = tlbi_aa64_rvae1_write128 },
     { .name = "TLBI_RIPAS2E1IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 2,
       .access = PL2_W, .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
-- 
2.43.0

[PATCH v2 29/37] target/arm: Implement TLBIP RIPAS1E1, RIPAS1LE1, RIPAS2E1IS, RIPAS2LE1IS

[Richard Henderson]

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/tcg/tlb-insns.c | 39 ++++++++++++++++++++++++++++++--------
 1 file changed, 31 insertions(+), 8 deletions(-)

diff --git a/target/arm/tcg/tlb-insns.c b/target/arm/tcg/tlb-insns.c
index daadba7bfc..6c07839e3f 100644
--- a/target/arm/tcg/tlb-insns.c
+++ b/target/arm/tcg/tlb-insns.c
@@ -1088,6 +1088,14 @@ static void tlbi_aa64_ripas2e1_write(CPUARMState *env, const ARMCPRegInfo *ri,
                   tlb_force_broadcast(env));
 }
 
+static void tlbi_aa64_ripas2e1_write128(CPUARMState *env,
+                                        const ARMCPRegInfo *ri,
+                                        uint64_t vallo, uint64_t valhi)
+{
+    do_rvae_write128(env, vallo, valhi, ipas2e1_tlbmask(env, vallo),
+                     tlb_force_broadcast(env));
+}
+
 static void tlbi_aa64_ripas2e1is_write(CPUARMState *env,
                                        const ARMCPRegInfo *ri,
                                        uint64_t value)
@@ -1095,6 +1103,13 @@ static void tlbi_aa64_ripas2e1is_write(CPUARMState *env,
     do_rvae_write(env, value, ipas2e1_tlbmask(env, value), true);
 }
 
+static void tlbi_aa64_ripas2e1is_write128(CPUARMState *env,
+                                          const ARMCPRegInfo *ri,
+                                          uint64_t vallo, uint64_t valhi)
+{
+    do_rvae_write128(env, vallo, valhi, ipas2e1_tlbmask(env, vallo), true);
+}
+
 static const ARMCPRegInfo tlbirange_reginfo[] = {
     { .name = "TLBI_RVAE1IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 2, .opc2 = 1,
@@ -1174,12 +1189,16 @@ static const ARMCPRegInfo tlbirange_reginfo[] = {
       .write128fn = tlbi_aa64_rvae1_write128 },
     { .name = "TLBI_RIPAS2E1IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 2,
-      .access = PL2_W, .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
-      .writefn = tlbi_aa64_ripas2e1is_write },
+      .access = PL2_W,
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_128BIT,
+      .writefn = tlbi_aa64_ripas2e1is_write,
+      .write128fn = tlbi_aa64_ripas2e1is_write128 },
     { .name = "TLBI_RIPAS2LE1IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 6,
-      .access = PL2_W, .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
-      .writefn = tlbi_aa64_ripas2e1is_write },
+      .access = PL2_W,
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_128BIT,
+      .writefn = tlbi_aa64_ripas2e1is_write,
+      .write128fn = tlbi_aa64_ripas2e1is_write128 },
     { .name = "TLBI_RVAE2IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 2, .opc2 = 1,
       .access = PL2_W,
@@ -1192,12 +1211,16 @@ static const ARMCPRegInfo tlbirange_reginfo[] = {
       .writefn = tlbi_aa64_rvae2is_write },
     { .name = "TLBI_RIPAS2E1", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 2,
-      .access = PL2_W, .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
-      .writefn = tlbi_aa64_ripas2e1_write },
+      .access = PL2_W,
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_128BIT,
+      .writefn = tlbi_aa64_ripas2e1_write,
+      .write128fn = tlbi_aa64_ripas2e1_write128 },
     { .name = "TLBI_RIPAS2LE1", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 6,
-      .access = PL2_W, .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
-      .writefn = tlbi_aa64_ripas2e1_write },
+      .access = PL2_W,
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_128BIT,
+      .writefn = tlbi_aa64_ripas2e1_write,
+      .write128fn = tlbi_aa64_ripas2e1_write128 },
    { .name = "TLBI_RVAE2OS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 5, .opc2 = 1,
       .access = PL2_W,
-- 
2.43.0

[PATCH v2 30/37] target/arm: Implement TLBIP RVA{L}E2{IS,OS}

[Richard Henderson]

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/tcg/tlb-insns.c | 52 +++++++++++++++++++++++++++-----------
 1 file changed, 37 insertions(+), 15 deletions(-)

diff --git a/target/arm/tcg/tlb-insns.c b/target/arm/tcg/tlb-insns.c
index 6c07839e3f..cbab7f81f9 100644
--- a/target/arm/tcg/tlb-insns.c
+++ b/target/arm/tcg/tlb-insns.c
@@ -1031,11 +1031,16 @@ static void tlbi_aa64_rvae2_write(CPUARMState *env,
      * since we don't support flush-for-specific-ASID-only or
      * flush-last-level-only.
      */
-
     do_rvae_write(env, value, vae2_tlbmask(env),
                   tlb_force_broadcast(env));
+}
 
-
+static void tlbi_aa64_rvae2_write128(CPUARMState *env,
+                                     const ARMCPRegInfo *ri,
+                                     uint64_t vallo, uint64_t valhi)
+{
+    do_rvae_write128(env, vallo, valhi, vae2_tlbmask(env),
+                     tlb_force_broadcast(env));
 }
 
 static void tlbi_aa64_rvae2is_write(CPUARMState *env,
@@ -1048,9 +1053,14 @@ static void tlbi_aa64_rvae2is_write(CPUARMState *env,
      * since we don't support flush-for-specific-ASID-only,
      * flush-last-level-only or inner/outer shareable specific flushes.
      */
-
     do_rvae_write(env, value, vae2_tlbmask(env), true);
+}
 
+static void tlbi_aa64_rvae2is_write128(CPUARMState *env,
+                                       const ARMCPRegInfo *ri,
+                                       uint64_t vallo, uint64_t valhi)
+{
+    do_rvae_write128(env, vallo, valhi, vae2_tlbmask(env), true);
 }
 
 static void tlbi_aa64_rvae3_write(CPUARMState *env,
@@ -1202,13 +1212,17 @@ static const ARMCPRegInfo tlbirange_reginfo[] = {
     { .name = "TLBI_RVAE2IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 2, .opc2 = 1,
       .access = PL2_W,
-      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_EL3_NO_EL2_UNDEF,
-      .writefn = tlbi_aa64_rvae2is_write },
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS |
+              ARM_CP_EL3_NO_EL2_UNDEF | ARM_CP_128BIT,
+      .writefn = tlbi_aa64_rvae2is_write,
+      .write128fn = tlbi_aa64_rvae2is_write128 },
    { .name = "TLBI_RVALE2IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 2, .opc2 = 5,
       .access = PL2_W,
-      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_EL3_NO_EL2_UNDEF,
-      .writefn = tlbi_aa64_rvae2is_write },
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS |
+              ARM_CP_EL3_NO_EL2_UNDEF | ARM_CP_128BIT,
+      .writefn = tlbi_aa64_rvae2is_write,
+      .write128fn = tlbi_aa64_rvae2is_write128 },
     { .name = "TLBI_RIPAS2E1", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 2,
       .access = PL2_W,
@@ -1224,23 +1238,31 @@ static const ARMCPRegInfo tlbirange_reginfo[] = {
    { .name = "TLBI_RVAE2OS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 5, .opc2 = 1,
       .access = PL2_W,
-      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_EL3_NO_EL2_UNDEF,
-      .writefn = tlbi_aa64_rvae2is_write },
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS |
+              ARM_CP_EL3_NO_EL2_UNDEF | ARM_CP_128BIT,
+      .writefn = tlbi_aa64_rvae2is_write,
+      .write128fn = tlbi_aa64_rvae2is_write128 },
    { .name = "TLBI_RVALE2OS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 5, .opc2 = 5,
       .access = PL2_W,
-      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_EL3_NO_EL2_UNDEF,
-      .writefn = tlbi_aa64_rvae2is_write },
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS |
+              ARM_CP_EL3_NO_EL2_UNDEF | ARM_CP_128BIT,
+      .writefn = tlbi_aa64_rvae2is_write,
+      .write128fn = tlbi_aa64_rvae2is_write128 },
     { .name = "TLBI_RVAE2", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 6, .opc2 = 1,
       .access = PL2_W,
-      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_EL3_NO_EL2_UNDEF,
-      .writefn = tlbi_aa64_rvae2_write },
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS |
+              ARM_CP_EL3_NO_EL2_UNDEF | ARM_CP_128BIT,
+      .writefn = tlbi_aa64_rvae2_write,
+      .write128fn = tlbi_aa64_rvae2_write128 },
    { .name = "TLBI_RVALE2", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 6, .opc2 = 5,
       .access = PL2_W,
-      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_EL3_NO_EL2_UNDEF,
-      .writefn = tlbi_aa64_rvae2_write },
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS |
+              ARM_CP_EL3_NO_EL2_UNDEF | ARM_CP_128BIT,
+      .writefn = tlbi_aa64_rvae2_write,
+      .write128fn = tlbi_aa64_rvae2_write128 },
    { .name = "TLBI_RVAE3IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 2, .opc2 = 1,
       .access = PL3_W, .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
-- 
2.43.0

[PATCH v2 31/37] target/arm: Implement TLBIP RVA{L}E3{IS,OS}

[Richard Henderson]

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/tcg/tlb-insns.c | 53 ++++++++++++++++++++++++++++----------
 1 file changed, 39 insertions(+), 14 deletions(-)

diff --git a/target/arm/tcg/tlb-insns.c b/target/arm/tcg/tlb-insns.c
index cbab7f81f9..c7583957b0 100644
--- a/target/arm/tcg/tlb-insns.c
+++ b/target/arm/tcg/tlb-insns.c
@@ -1073,10 +1073,17 @@ static void tlbi_aa64_rvae3_write(CPUARMState *env,
      * since we don't support flush-for-specific-ASID-only or
      * flush-last-level-only.
      */
-
     do_rvae_write(env, value, vae3_tlbmask(), tlb_force_broadcast(env));
 }
 
+static void tlbi_aa64_rvae3_write128(CPUARMState *env,
+                                     const ARMCPRegInfo *ri,
+                                     uint64_t vallo, uint64_t valhi)
+{
+    do_rvae_write128(env, vallo, valhi, vae3_tlbmask(),
+                     tlb_force_broadcast(env));
+}
+
 static void tlbi_aa64_rvae3is_write(CPUARMState *env,
                                     const ARMCPRegInfo *ri,
                                     uint64_t value)
@@ -1087,10 +1094,16 @@ static void tlbi_aa64_rvae3is_write(CPUARMState *env,
      * since we don't support flush-for-specific-ASID-only,
      * flush-last-level-only or inner/outer specific flushes.
      */
-
     do_rvae_write(env, value, vae3_tlbmask(), true);
 }
 
+static void tlbi_aa64_rvae3is_write128(CPUARMState *env,
+                                       const ARMCPRegInfo *ri,
+                                       uint64_t vallo, uint64_t valhi)
+{
+    do_rvae_write128(env, vallo, valhi, vae3_tlbmask(), true);
+}
+
 static void tlbi_aa64_ripas2e1_write(CPUARMState *env, const ARMCPRegInfo *ri,
                                      uint64_t value)
 {
@@ -1265,28 +1278,40 @@ static const ARMCPRegInfo tlbirange_reginfo[] = {
       .write128fn = tlbi_aa64_rvae2_write128 },
    { .name = "TLBI_RVAE3IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 2, .opc2 = 1,
-      .access = PL3_W, .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
-      .writefn = tlbi_aa64_rvae3is_write },
+      .access = PL3_W,
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_128BIT,
+      .writefn = tlbi_aa64_rvae3is_write,
+      .write128fn = tlbi_aa64_rvae3is_write128 },
    { .name = "TLBI_RVALE3IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 2, .opc2 = 5,
-      .access = PL3_W, .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
-      .writefn = tlbi_aa64_rvae3is_write },
+      .access = PL3_W,
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_128BIT,
+      .writefn = tlbi_aa64_rvae3is_write,
+      .write128fn = tlbi_aa64_rvae3is_write128 },
    { .name = "TLBI_RVAE3OS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 5, .opc2 = 1,
-      .access = PL3_W, .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
-      .writefn = tlbi_aa64_rvae3is_write },
+      .access = PL3_W,
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_128BIT,
+      .writefn = tlbi_aa64_rvae3is_write,
+      .write128fn = tlbi_aa64_rvae3is_write128 },
    { .name = "TLBI_RVALE3OS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 5, .opc2 = 5,
-      .access = PL3_W, .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
-      .writefn = tlbi_aa64_rvae3is_write },
+      .access = PL3_W,
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_128BIT,
+      .writefn = tlbi_aa64_rvae3is_write,
+      .write128fn = tlbi_aa64_rvae3is_write128 },
    { .name = "TLBI_RVAE3", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 6, .opc2 = 1,
-      .access = PL3_W, .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
-      .writefn = tlbi_aa64_rvae3_write },
+      .access = PL3_W,
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_128BIT,
+      .writefn = tlbi_aa64_rvae3_write,
+      .write128fn = tlbi_aa64_rvae3_write128 },
    { .name = "TLBI_RVALE3", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 6, .opc2 = 5,
-      .access = PL3_W, .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
-      .writefn = tlbi_aa64_rvae3_write },
+      .access = PL3_W,
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_128BIT,
+      .writefn = tlbi_aa64_rvae3_write,
+      .write128fn = tlbi_aa64_rvae3_write128 },
 };
 
 static const ARMCPRegInfo tlbios_reginfo[] = {
-- 
2.43.0

[PATCH v2 32/37] target/arm: Implement TLBIP VA{L}E1{IS,OS}

[Richard Henderson]

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/tcg/tlb-insns.c | 96 ++++++++++++++++++++++++++------------
 1 file changed, 66 insertions(+), 30 deletions(-)

diff --git a/target/arm/tcg/tlb-insns.c b/target/arm/tcg/tlb-insns.c
index c7583957b0..beacb92f32 100644
--- a/target/arm/tcg/tlb-insns.c
+++ b/target/arm/tcg/tlb-insns.c
@@ -442,6 +442,17 @@ static void tlbi_aa64_vae1is_write(CPUARMState *env, const ARMCPRegInfo *ri,
     tlb_flush_page_bits_by_mmuidx_all_cpus_synced(cs, pageaddr, mask, bits);
 }
 
+static void tlbi_aa64_vae1is_write128(CPUARMState *env, const ARMCPRegInfo *ri,
+                                      uint64_t vallo, uint64_t valhi)
+{
+    CPUState *cs = env_cpu(env);
+    int mask = vae1_tlbmask(env);
+    uint64_t pageaddr = sextract64(valhi << 12, 0, 56);
+    int bits = vae1_tlbbits(env, pageaddr);
+
+    tlb_flush_page_bits_by_mmuidx_all_cpus_synced(cs, pageaddr, mask, bits);
+}
+
 static void tlbi_aa64_vae1_write(CPUARMState *env, const ARMCPRegInfo *ri,
                                  uint64_t value)
 {
@@ -463,6 +474,21 @@ static void tlbi_aa64_vae1_write(CPUARMState *env, const ARMCPRegInfo *ri,
     }
 }
 
+static void tlbi_aa64_vae1_write128(CPUARMState *env, const ARMCPRegInfo *ri,
+                                    uint64_t vallo, uint64_t valhi)
+{
+    CPUState *cs = env_cpu(env);
+    int mask = vae1_tlbmask(env);
+    uint64_t pageaddr = sextract64(valhi << 12, 0, 56);
+    int bits = vae1_tlbbits(env, pageaddr);
+
+    if (tlb_force_broadcast(env)) {
+        tlb_flush_page_bits_by_mmuidx_all_cpus_synced(cs, pageaddr, mask, bits);
+    } else {
+        tlb_flush_page_bits_by_mmuidx(cs, pageaddr, mask, bits);
+    }
+}
+
 static void tlbi_aa64_vae2is_write(CPUARMState *env, const ARMCPRegInfo *ri,
                                    uint64_t value)
 {
@@ -664,10 +690,11 @@ static const ARMCPRegInfo tlbi_v8_cp_reginfo[] = {
       .writefn = tlbi_aa64_vmalle1is_write },
     { .name = "TLBI_VAE1IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 1,
-      .access = PL1_W, .accessfn = access_ttlbis,
-      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
+      .access = PL1_W, .accessfn = access_ttlbis, .access128fn = access_ttlbis,
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_128BIT,
       .fgt = FGT_TLBIVAE1IS,
-      .writefn = tlbi_aa64_vae1is_write },
+      .writefn = tlbi_aa64_vae1is_write,
+      .write128fn = tlbi_aa64_vae1is_write128 },
     { .name = "TLBI_ASIDE1IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 2,
       .access = PL1_W, .accessfn = access_ttlbis,
@@ -676,10 +703,11 @@ static const ARMCPRegInfo tlbi_v8_cp_reginfo[] = {
       .writefn = tlbi_aa64_vmalle1is_write },
     { .name = "TLBI_VAAE1IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 3,
-      .access = PL1_W, .accessfn = access_ttlbis,
-      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
+      .access = PL1_W, .accessfn = access_ttlbis, .access128fn = access_ttlbis,
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_128BIT,
       .fgt = FGT_TLBIVAAE1IS,
-      .writefn = tlbi_aa64_vae1is_write },
+      .writefn = tlbi_aa64_vae1is_write,
+      .write128fn = tlbi_aa64_vae1is_write128 },
     { .name = "TLBI_VALE1IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 5,
       .access = PL1_W, .accessfn = access_ttlbis,
@@ -700,10 +728,11 @@ static const ARMCPRegInfo tlbi_v8_cp_reginfo[] = {
       .writefn = tlbi_aa64_vmalle1_write },
     { .name = "TLBI_VAE1", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 1,
-      .access = PL1_W, .accessfn = access_ttlb,
-      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
+      .access = PL1_W, .accessfn = access_ttlb, .access128fn = access_ttlb,
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_128BIT,
       .fgt = FGT_TLBIVAE1,
-      .writefn = tlbi_aa64_vae1_write },
+      .writefn = tlbi_aa64_vae1_write,
+      .write128fn = tlbi_aa64_vae1_write128 },
     { .name = "TLBI_ASIDE1", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 2,
       .access = PL1_W, .accessfn = access_ttlb,
@@ -712,22 +741,25 @@ static const ARMCPRegInfo tlbi_v8_cp_reginfo[] = {
       .writefn = tlbi_aa64_vmalle1_write },
     { .name = "TLBI_VAAE1", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 3,
-      .access = PL1_W, .accessfn = access_ttlb,
-      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
+      .access = PL1_W, .accessfn = access_ttlb, .access128fn = access_ttlb,
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_128BIT,
       .fgt = FGT_TLBIVAAE1,
-      .writefn = tlbi_aa64_vae1_write },
+      .writefn = tlbi_aa64_vae1_write,
+      .write128fn = tlbi_aa64_vae1_write128 },
     { .name = "TLBI_VALE1", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 5,
-      .access = PL1_W, .accessfn = access_ttlb,
-      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
+      .access = PL1_W, .accessfn = access_ttlb, .access128fn = access_ttlb,
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_128BIT,
       .fgt = FGT_TLBIVALE1,
-      .writefn = tlbi_aa64_vae1_write },
+      .writefn = tlbi_aa64_vae1_write,
+      .write128fn = tlbi_aa64_vae1_write128 },
     { .name = "TLBI_VAALE1", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 7,
-      .access = PL1_W, .accessfn = access_ttlb,
-      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
+      .access = PL1_W, .accessfn = access_ttlb, .access128fn = access_ttlb,
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_128BIT,
       .fgt = FGT_TLBIVAALE1,
-      .writefn = tlbi_aa64_vae1_write },
+      .writefn = tlbi_aa64_vae1_write,
+      .write128fn = tlbi_aa64_vae1_write128 },
     { .name = "TLBI_IPAS2E1IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 1,
       .access = PL2_W,
@@ -1324,9 +1356,10 @@ static const ARMCPRegInfo tlbios_reginfo[] = {
     { .name = "TLBI_VAE1OS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 1, .opc2 = 1,
       .fgt = FGT_TLBIVAE1OS,
-      .access = PL1_W, .accessfn = access_ttlbos,
-      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
-      .writefn = tlbi_aa64_vae1is_write },
+      .access = PL1_W, .accessfn = access_ttlbos, .access128fn = access_ttlbos,
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_128BIT,
+      .writefn = tlbi_aa64_vae1is_write,
+      .write128fn = tlbi_aa64_vae1is_write128 },
     { .name = "TLBI_ASIDE1OS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 1, .opc2 = 2,
       .access = PL1_W, .accessfn = access_ttlbos,
@@ -1335,22 +1368,25 @@ static const ARMCPRegInfo tlbios_reginfo[] = {
       .writefn = tlbi_aa64_vmalle1is_write },
     { .name = "TLBI_VAAE1OS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 1, .opc2 = 3,
-      .access = PL1_W, .accessfn = access_ttlbos,
-      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
+      .access = PL1_W, .accessfn = access_ttlbos, .access128fn = access_ttlbos,
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_128BIT,
       .fgt = FGT_TLBIVAAE1OS,
-      .writefn = tlbi_aa64_vae1is_write },
+      .writefn = tlbi_aa64_vae1is_write,
+      .write128fn = tlbi_aa64_vae1is_write128 },
     { .name = "TLBI_VALE1OS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 1, .opc2 = 5,
-      .access = PL1_W, .accessfn = access_ttlbos,
-      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
+      .access = PL1_W, .accessfn = access_ttlbos, .access128fn = access_ttlbos,
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_128BIT,
       .fgt = FGT_TLBIVALE1OS,
-      .writefn = tlbi_aa64_vae1is_write },
+      .writefn = tlbi_aa64_vae1is_write,
+      .write128fn = tlbi_aa64_vae1is_write128 },
     { .name = "TLBI_VAALE1OS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 1, .opc2 = 7,
-      .access = PL1_W, .accessfn = access_ttlbos,
-      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
+      .access = PL1_W, .accessfn = access_ttlbos, .access128fn = access_ttlbos,
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_128BIT,
       .fgt = FGT_TLBIVAALE1OS,
-      .writefn = tlbi_aa64_vae1is_write },
+      .writefn = tlbi_aa64_vae1is_write,
+      .write128fn = tlbi_aa64_vae1is_write128 },
     { .name = "TLBI_ALLE2OS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 1, .opc2 = 0,
       .access = PL2_W,
-- 
2.43.0

[PATCH v2 33/37] target/arm: Implement TLBIP VAE2, VALE2

[Richard Henderson]

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/tcg/tlb-insns.c | 23 +++++++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

diff --git a/target/arm/tcg/tlb-insns.c b/target/arm/tcg/tlb-insns.c
index beacb92f32..f99559e4a7 100644
--- a/target/arm/tcg/tlb-insns.c
+++ b/target/arm/tcg/tlb-insns.c
@@ -416,6 +416,17 @@ static void tlbi_aa64_vae2_write(CPUARMState *env, const ARMCPRegInfo *ri,
     tlb_flush_page_bits_by_mmuidx(cs, pageaddr, mask, bits);
 }
 
+static void tlbi_aa64_vae2_write128(CPUARMState *env, const ARMCPRegInfo *ri,
+                                    uint64_t vallo, uint64_t valhi)
+{
+    CPUState *cs = env_cpu(env);
+    int mask = vae2_tlbmask(env);
+    uint64_t pageaddr = sextract64(valhi << 12, 0, 56);
+    int bits = vae2_tlbbits(env, pageaddr);
+
+    tlb_flush_page_bits_by_mmuidx(cs, pageaddr, mask, bits);
+}
+
 static void tlbi_aa64_vae3_write(CPUARMState *env, const ARMCPRegInfo *ri,
                                  uint64_t value)
 {
@@ -831,13 +842,17 @@ static const ARMCPRegInfo tlbi_el2_cp_reginfo[] = {
     { .name = "TLBI_VAE2", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 1,
       .access = PL2_W,
-      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_EL3_NO_EL2_UNDEF,
-      .writefn = tlbi_aa64_vae2_write },
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS |
+              ARM_CP_EL3_NO_EL2_UNDEF | ARM_CP_128BIT,
+      .writefn = tlbi_aa64_vae2_write,
+      .write128fn = tlbi_aa64_vae2_write128 },
     { .name = "TLBI_VALE2", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 5,
       .access = PL2_W,
-      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_EL3_NO_EL2_UNDEF,
-      .writefn = tlbi_aa64_vae2_write },
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS |
+              ARM_CP_EL3_NO_EL2_UNDEF | ARM_CP_128BIT,
+      .writefn = tlbi_aa64_vae2_write,
+      .write128fn = tlbi_aa64_vae2_write128 },
     { .name = "TLBI_ALLE2IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 0,
       .access = PL2_W,
-- 
2.43.0

[PATCH v2 34/37] target/arm: Implement TLBIP VAE3, VALE3

[Richard Henderson]

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/tcg/tlb-insns.c | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/target/arm/tcg/tlb-insns.c b/target/arm/tcg/tlb-insns.c
index f99559e4a7..10ee973068 100644
--- a/target/arm/tcg/tlb-insns.c
+++ b/target/arm/tcg/tlb-insns.c
@@ -442,6 +442,13 @@ static void tlbi_aa64_vae3_write(CPUARMState *env, const ARMCPRegInfo *ri,
     tlb_flush_page_by_mmuidx(cs, pageaddr, vae3_tlbmask());
 }
 
+static void tlbi_aa64_vae3_write128(CPUARMState *env, const ARMCPRegInfo *ri,
+                                    uint64_t vallo, uint64_t valhi)
+{
+    uint64_t pageaddr = extract64(valhi << 12, 0, 56);
+    tlb_flush_page_by_mmuidx(env_cpu(env), pageaddr, ARMMMUIdxBit_E3);
+}
+
 static void tlbi_aa64_vae1is_write(CPUARMState *env, const ARMCPRegInfo *ri,
                                    uint64_t value)
 {
@@ -889,12 +896,16 @@ static const ARMCPRegInfo tlbi_el3_cp_reginfo[] = {
       .writefn = tlbi_aa64_alle3_write },
     { .name = "TLBI_VAE3", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 7, .opc2 = 1,
-      .access = PL3_W, .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
-      .writefn = tlbi_aa64_vae3_write },
+      .access = PL3_W,
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_128BIT,
+      .writefn = tlbi_aa64_vae3_write,
+      .write128fn = tlbi_aa64_vae3_write128 },
     { .name = "TLBI_VALE3", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 7, .opc2 = 5,
-      .access = PL3_W, .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
-      .writefn = tlbi_aa64_vae3_write },
+      .access = PL3_W,
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_128BIT,
+      .writefn = tlbi_aa64_vae3_write,
+      .write128fn = tlbi_aa64_vae3_write128 },
 };
 
 typedef struct {
-- 
2.43.0

[PATCH v2 35/37] target/arm: Implement TLBIP VA{L}E2{IS,OS}

[Richard Henderson]

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/tcg/tlb-insns.c | 35 +++++++++++++++++++++++++++--------
 1 file changed, 27 insertions(+), 8 deletions(-)

diff --git a/target/arm/tcg/tlb-insns.c b/target/arm/tcg/tlb-insns.c
index 10ee973068..ea2b56e245 100644
--- a/target/arm/tcg/tlb-insns.c
+++ b/target/arm/tcg/tlb-insns.c
@@ -518,6 +518,17 @@ static void tlbi_aa64_vae2is_write(CPUARMState *env, const ARMCPRegInfo *ri,
     tlb_flush_page_bits_by_mmuidx_all_cpus_synced(cs, pageaddr, mask, bits);
 }
 
+static void tlbi_aa64_vae2is_write128(CPUARMState *env, const ARMCPRegInfo *ri,
+                                      uint64_t vallo, uint64_t valhi)
+{
+    CPUState *cs = env_cpu(env);
+    int mask = vae2_tlbmask(env);
+    uint64_t pageaddr = sextract64(valhi << 12, 0, 56);
+    int bits = vae2_tlbbits(env, pageaddr);
+
+    tlb_flush_page_bits_by_mmuidx_all_cpus_synced(cs, pageaddr, mask, bits);
+}
+
 static void tlbi_aa64_vae3is_write(CPUARMState *env, const ARMCPRegInfo *ri,
                                    uint64_t value)
 {
@@ -868,13 +879,17 @@ static const ARMCPRegInfo tlbi_el2_cp_reginfo[] = {
     { .name = "TLBI_VAE2IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 1,
       .access = PL2_W,
-      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_EL3_NO_EL2_UNDEF,
-      .writefn = tlbi_aa64_vae2is_write },
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS |
+              ARM_CP_EL3_NO_EL2_UNDEF | ARM_CP_128BIT,
+      .writefn = tlbi_aa64_vae2is_write,
+      .write128fn = tlbi_aa64_vae2is_write128 },
     { .name = "TLBI_VALE2IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 5,
       .access = PL2_W,
-      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_EL3_NO_EL2_UNDEF,
-      .writefn = tlbi_aa64_vae2is_write },
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS |
+              ARM_CP_EL3_NO_EL2_UNDEF | ARM_CP_128BIT,
+      .writefn = tlbi_aa64_vae2is_write,
+      .write128fn = tlbi_aa64_vae2is_write128 },
 };
 
 static const ARMCPRegInfo tlbi_el3_cp_reginfo[] = {
@@ -1421,8 +1436,10 @@ static const ARMCPRegInfo tlbios_reginfo[] = {
     { .name = "TLBI_VAE2OS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 1, .opc2 = 1,
       .access = PL2_W,
-      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_EL3_NO_EL2_UNDEF,
-      .writefn = tlbi_aa64_vae2is_write },
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS |
+              ARM_CP_EL3_NO_EL2_UNDEF | ARM_CP_128BIT,
+      .writefn = tlbi_aa64_vae2is_write,
+      .write128fn = tlbi_aa64_vae2is_write128 },
    { .name = "TLBI_ALLE1OS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 1, .opc2 = 4,
       .access = PL2_W,
@@ -1431,8 +1448,10 @@ static const ARMCPRegInfo tlbios_reginfo[] = {
     { .name = "TLBI_VALE2OS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 1, .opc2 = 5,
       .access = PL2_W,
-      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_EL3_NO_EL2_UNDEF,
-      .writefn = tlbi_aa64_vae2is_write },
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS |
+              ARM_CP_EL3_NO_EL2_UNDEF | ARM_CP_128BIT,
+      .writefn = tlbi_aa64_vae2is_write,
+      .write128fn = tlbi_aa64_vae2is_write128 },
     { .name = "TLBI_VMALLS12E1OS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 1, .opc2 = 6,
       .access = PL2_W, .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
-- 
2.43.0

[PATCH v2 36/37] target/arm: Implement TLBIP VA{L}E3{IS,OS}

[Richard Henderson]

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/tcg/tlb-insns.c | 35 +++++++++++++++++++++++++++--------
 1 file changed, 27 insertions(+), 8 deletions(-)

diff --git a/target/arm/tcg/tlb-insns.c b/target/arm/tcg/tlb-insns.c
index ea2b56e245..2b76ea077f 100644
--- a/target/arm/tcg/tlb-insns.c
+++ b/target/arm/tcg/tlb-insns.c
@@ -540,6 +540,17 @@ static void tlbi_aa64_vae3is_write(CPUARMState *env, const ARMCPRegInfo *ri,
                                                   vae3_tlbmask(), bits);
 }
 
+static void tlbi_aa64_vae3is_write128(CPUARMState *env, const ARMCPRegInfo *ri,
+                                      uint64_t vallo, uint64_t valhi)
+{
+    CPUState *cs = env_cpu(env);
+    uint64_t pageaddr = extract64(valhi << 12, 0, 56);
+    int bits = tlbbits_for_regime(env, ARMMMUIdx_E3, pageaddr);
+
+    tlb_flush_page_bits_by_mmuidx_all_cpus_synced(cs, pageaddr,
+                                                  ARMMMUIdxBit_E3, bits);
+}
+
 static int ipas2e1_tlbmask(CPUARMState *env, int64_t value)
 {
     /*
@@ -899,12 +910,16 @@ static const ARMCPRegInfo tlbi_el3_cp_reginfo[] = {
       .writefn = tlbi_aa64_alle3is_write },
     { .name = "TLBI_VAE3IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 3, .opc2 = 1,
-      .access = PL3_W, .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
-      .writefn = tlbi_aa64_vae3is_write },
+      .access = PL3_W,
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_128BIT,
+      .writefn = tlbi_aa64_vae3is_write,
+      .write128fn = tlbi_aa64_vae3is_write128 },
     { .name = "TLBI_VALE3IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 3, .opc2 = 5,
-      .access = PL3_W, .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
-      .writefn = tlbi_aa64_vae3is_write },
+      .access = PL3_W,
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_128BIT,
+      .writefn = tlbi_aa64_vae3is_write,
+      .write128fn = tlbi_aa64_vae3is_write128 },
     { .name = "TLBI_ALLE3", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 7, .opc2 = 0,
       .access = PL3_W, .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
@@ -1474,12 +1489,16 @@ static const ARMCPRegInfo tlbios_reginfo[] = {
       .writefn = tlbi_aa64_alle3is_write },
     { .name = "TLBI_VAE3OS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 1, .opc2 = 1,
-      .access = PL3_W, .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
-      .writefn = tlbi_aa64_vae3is_write },
+      .access = PL3_W,
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_128BIT,
+      .writefn = tlbi_aa64_vae3is_write,
+      .write128fn = tlbi_aa64_vae3is_write128 },
     { .name = "TLBI_VALE3OS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 1, .opc2 = 5,
-      .access = PL3_W, .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS,
-      .writefn = tlbi_aa64_vae3is_write },
+      .access = PL3_W,
+      .type = ARM_CP_NO_RAW | ARM_CP_ADD_TLBI_NXS | ARM_CP_128BIT,
+      .writefn = tlbi_aa64_vae3is_write,
+      .write128fn = tlbi_aa64_vae3is_write128 },
 };
 
 static void tlbi_aa64_paall_write(CPUARMState *env, const ARMCPRegInfo *ri,
-- 
2.43.0

[PATCH v2 37/37] NOTFORMERGE: Enable FEAT_SYSREG128, FEAT_SYSINSTR128 for cpu max

[Richard Henderson]

Do not merge yet, because both of these are enabled
if and only if FEAT_D128 is also enabled.

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/arm/tcg/cpu64.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/target/arm/tcg/cpu64.c b/target/arm/tcg/cpu64.c
index 6871956382..058d5c0ec1 100644
--- a/target/arm/tcg/cpu64.c
+++ b/target/arm/tcg/cpu64.c
@@ -1254,6 +1254,8 @@ void aarch64_max_tcg_initfn(Object *obj)
     t = FIELD_DP64(t, ID_AA64ISAR2, WFXT, 2);     /* FEAT_WFxT */
     t = FIELD_DP64(t, ID_AA64ISAR2, CSSC, 1);     /* FEAT_CSSC */
     t = FIELD_DP64(t, ID_AA64ISAR2, ATS1A, 1);    /* FEAT_ATS1A */
+    t = FIELD_DP64(t, ID_AA64ISAR2, SYSREG_128, 1);   /* FEAT_SYSREG128 */
+    t = FIELD_DP64(t, ID_AA64ISAR2, SYSINSTR_128, 1); /* FEAT_SYSINSTR128 */
     SET_IDREG(isar, ID_AA64ISAR2, t);
 
     t = GET_IDREG(isar, ID_AA64PFR0);
-- 
2.43.0