[PATCH v3] hw/arm/smmuv3: Add GBPA register

[PATCH v3] hw/arm/smmuv3: Add GBPA register

[Mostafa Saleh]

GBPA register can be used to globally abort all
transactions.

It is described in the SMMU manual in "6.3.14 SMMU_GBPA".
ABORT reset value is IMPLEMENTATION DEFINED, it is chosen to
be zero(Do not abort incoming transactions).

Other fields have default values of Use Incoming.

If UPDATE is not set, the write is ignored. This is the only permitted
behavior in SMMUv3.2 and later.(6.3.14.1 Update procedure)

As this patch adds a new state to the SMMU (GBPA), it is added
in a new subsection for forward migration compatibility.
GBPA is only migrated when GBPA.ABORT is set from SW at the time of
migration, to avoid inconsistencies where a qemu instance is aborting
transactions and it is migrated to another older instance bypassing
them.

Signed-off-by: Mostafa Saleh <smostafa@google.com>
---

Changes in v3:
- Remove migrate_gbpa property as it was unnecessary.

Changes in v2:
- GBPA is effective only when SMMU is not enabled.
- Ignore GBPA write when UPDATE is not set.
- Default value for SHCFG is "Use Incoming".
- Support migration.

 hw/arm/smmuv3-internal.h |  5 ++++
 hw/arm/smmuv3.c          | 52 +++++++++++++++++++++++++++++++++++++++-
 include/hw/arm/smmuv3.h  |  1 +
 3 files changed, 57 insertions(+), 1 deletion(-)

diff --git a/hw/arm/smmuv3-internal.h b/hw/arm/smmuv3-internal.h
index bce161870f..3ff704248d 100644
--- a/hw/arm/smmuv3-internal.h
+++ b/hw/arm/smmuv3-internal.h
@@ -79,6 +79,11 @@ REG32(CR0ACK,              0x24)
 REG32(CR1,                 0x28)
 REG32(CR2,                 0x2c)
 REG32(STATUSR,             0x40)
+REG32(GBPA,                0x44)
+    FIELD(GBPA, SHCFG,        12, 2)
+    FIELD(GBPA, ABORT,        20, 1)
+    FIELD(GBPA, UPDATE,       31, 1)
+
 REG32(IRQ_CTRL,            0x50)
     FIELD(IRQ_CTRL, GERROR_IRQEN,        0, 1)
     FIELD(IRQ_CTRL, PRI_IRQEN,           1, 1)
diff --git a/hw/arm/smmuv3.c b/hw/arm/smmuv3.c
index 955b89c8d5..a184665181 100644
--- a/hw/arm/smmuv3.c
+++ b/hw/arm/smmuv3.c
@@ -285,6 +285,8 @@ static void smmuv3_init_regs(SMMUv3State *s)
     s->gerror = 0;
     s->gerrorn = 0;
     s->statusr = 0;
+    /* Use incoming as other fields. */
+    s->gbpa = FIELD_DP32(s->gbpa, GBPA, SHCFG, 1);
 }
 
 static int smmu_get_ste(SMMUv3State *s, dma_addr_t addr, STE *buf,
@@ -659,7 +661,11 @@ static IOMMUTLBEntry smmuv3_translate(IOMMUMemoryRegion *mr, hwaddr addr,
     qemu_mutex_lock(&s->mutex);
 
     if (!smmu_enabled(s)) {
-        status = SMMU_TRANS_DISABLE;
+        if (FIELD_EX32(s->gbpa, GBPA, ABORT)) {
+            status = SMMU_TRANS_ABORT;
+        } else {
+            status = SMMU_TRANS_DISABLE;
+        }
         goto epilogue;
     }
 
@@ -1170,6 +1176,16 @@ static MemTxResult smmu_writel(SMMUv3State *s, hwaddr offset,
     case A_GERROR_IRQ_CFG2:
         s->gerror_irq_cfg2 = data;
         return MEMTX_OK;
+    case A_GBPA:
+        /*
+         * If UPDATE is not set, the write is ignored. This is the only
+         * permitted behavior in SMMUv3.2 and later.
+         */
+        if (data & R_GBPA_UPDATE_MASK) {
+            /* Ignore update bit as write is synchronous. */
+            s->gbpa = data & ~R_GBPA_UPDATE_MASK;
+        }
+        return MEMTX_OK;
     case A_STRTAB_BASE: /* 64b */
         s->strtab_base = deposit64(s->strtab_base, 0, 32, data);
         return MEMTX_OK;
@@ -1318,6 +1334,9 @@ static MemTxResult smmu_readl(SMMUv3State *s, hwaddr offset,
     case A_STATUSR:
         *data = s->statusr;
         return MEMTX_OK;
+    case A_GBPA:
+        *data = s->gbpa;
+        return MEMTX_OK;
     case A_IRQ_CTRL:
     case A_IRQ_CTRL_ACK:
         *data = s->irq_ctrl;
@@ -1482,6 +1501,33 @@ static const VMStateDescription vmstate_smmuv3_queue = {
     },
 };
 
+static bool smmuv3_gbpa_needed(void *opaque)
+{
+    SMMUv3State *s = opaque;
+
+    /*
+     * Only migrate GBPA if ABORT set from SW to 1, which is different from
+     * default behaviour. This allows maximum compatibility with old qemu
+     * instances while being forward compatible.
+     */
+    if (FIELD_EX32(s->gbpa, GBPA, ABORT)) {
+        return true;
+    }
+
+    return false;
+}
+
+static const VMStateDescription vmstate_gbpa = {
+    .name = "smmuv3/gbpa",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = smmuv3_gbpa_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(gbpa, SMMUv3State),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static const VMStateDescription vmstate_smmuv3 = {
     .name = "smmuv3",
     .version_id = 1,
@@ -1512,6 +1558,10 @@ static const VMStateDescription vmstate_smmuv3 = {
 
         VMSTATE_END_OF_LIST(),
     },
+    .subsections = (const VMStateDescription * []) {
+        &vmstate_gbpa,
+        NULL
+    }
 };
 
 static void smmuv3_instance_init(Object *obj)
diff --git a/include/hw/arm/smmuv3.h b/include/hw/arm/smmuv3.h
index f1921fdf9e..9899fa1860 100644
--- a/include/hw/arm/smmuv3.h
+++ b/include/hw/arm/smmuv3.h
@@ -46,6 +46,7 @@ struct SMMUv3State {
     uint32_t cr[3];
     uint32_t cr0ack;
     uint32_t statusr;
+    uint32_t gbpa;
     uint32_t irq_ctrl;
     uint32_t gerror;
     uint32_t gerrorn;
-- 
2.39.1.519.gcb327c4b5f-goog

Re: [PATCH v3] hw/arm/smmuv3: Add GBPA register

[Peter Maydell]

On Tue, 7 Feb 2023 at 18:31, Mostafa Saleh <smostafa@google.com> wrote:
>
> GBPA register can be used to globally abort all
> transactions.
>
> It is described in the SMMU manual in "6.3.14 SMMU_GBPA".
> ABORT reset value is IMPLEMENTATION DEFINED, it is chosen to
> be zero(Do not abort incoming transactions).
>
> Other fields have default values of Use Incoming.
>
> If UPDATE is not set, the write is ignored. This is the only permitted
> behavior in SMMUv3.2 and later.(6.3.14.1 Update procedure)
>
> As this patch adds a new state to the SMMU (GBPA), it is added
> in a new subsection for forward migration compatibility.
> GBPA is only migrated when GBPA.ABORT is set from SW at the time of
> migration, to avoid inconsistencies where a qemu instance is aborting
> transactions and it is migrated to another older instance bypassing
> them.
>
> Signed-off-by: Mostafa Saleh <smostafa@google.com>
> ---
>
> Changes in v3:
> - Remove migrate_gbpa property as it was unnecessary.
>
> Changes in v2:
> - GBPA is effective only when SMMU is not enabled.
> - Ignore GBPA write when UPDATE is not set.
> - Default value for SHCFG is "Use Incoming".
> - Support migration.
>
>  hw/arm/smmuv3-internal.h |  5 ++++
>  hw/arm/smmuv3.c          | 52 +++++++++++++++++++++++++++++++++++++++-
>  include/hw/arm/smmuv3.h  |  1 +
>  3 files changed, 57 insertions(+), 1 deletion(-)
>
> diff --git a/hw/arm/smmuv3-internal.h b/hw/arm/smmuv3-internal.h
> index bce161870f..3ff704248d 100644
> --- a/hw/arm/smmuv3-internal.h
> +++ b/hw/arm/smmuv3-internal.h
> @@ -79,6 +79,11 @@ REG32(CR0ACK,              0x24)
>  REG32(CR1,                 0x28)
>  REG32(CR2,                 0x2c)
>  REG32(STATUSR,             0x40)
> +REG32(GBPA,                0x44)
> +    FIELD(GBPA, SHCFG,        12, 2)
> +    FIELD(GBPA, ABORT,        20, 1)
> +    FIELD(GBPA, UPDATE,       31, 1)
> +
>  REG32(IRQ_CTRL,            0x50)
>      FIELD(IRQ_CTRL, GERROR_IRQEN,        0, 1)
>      FIELD(IRQ_CTRL, PRI_IRQEN,           1, 1)
> diff --git a/hw/arm/smmuv3.c b/hw/arm/smmuv3.c
> index 955b89c8d5..a184665181 100644
> --- a/hw/arm/smmuv3.c
> +++ b/hw/arm/smmuv3.c
> @@ -285,6 +285,8 @@ static void smmuv3_init_regs(SMMUv3State *s)
>      s->gerror = 0;
>      s->gerrorn = 0;
>      s->statusr = 0;
> +    /* Use incoming as other fields. */
> +    s->gbpa = FIELD_DP32(s->gbpa, GBPA, SHCFG, 1);
>  }
>
>  static int smmu_get_ste(SMMUv3State *s, dma_addr_t addr, STE *buf,
> @@ -659,7 +661,11 @@ static IOMMUTLBEntry smmuv3_translate(IOMMUMemoryRegion *mr, hwaddr addr,
>      qemu_mutex_lock(&s->mutex);
>
>      if (!smmu_enabled(s)) {
> -        status = SMMU_TRANS_DISABLE;
> +        if (FIELD_EX32(s->gbpa, GBPA, ABORT)) {
> +            status = SMMU_TRANS_ABORT;
> +        } else {
> +            status = SMMU_TRANS_DISABLE;
> +        }
>          goto epilogue;
>      }
>
> @@ -1170,6 +1176,16 @@ static MemTxResult smmu_writel(SMMUv3State *s, hwaddr offset,
>      case A_GERROR_IRQ_CFG2:
>          s->gerror_irq_cfg2 = data;
>          return MEMTX_OK;
> +    case A_GBPA:
> +        /*
> +         * If UPDATE is not set, the write is ignored. This is the only
> +         * permitted behavior in SMMUv3.2 and later.
> +         */
> +        if (data & R_GBPA_UPDATE_MASK) {
> +            /* Ignore update bit as write is synchronous. */
> +            s->gbpa = data & ~R_GBPA_UPDATE_MASK;
> +        }
> +        return MEMTX_OK;
>      case A_STRTAB_BASE: /* 64b */
>          s->strtab_base = deposit64(s->strtab_base, 0, 32, data);
>          return MEMTX_OK;
> @@ -1318,6 +1334,9 @@ static MemTxResult smmu_readl(SMMUv3State *s, hwaddr offset,
>      case A_STATUSR:
>          *data = s->statusr;
>          return MEMTX_OK;
> +    case A_GBPA:
> +        *data = s->gbpa;
> +        return MEMTX_OK;
>      case A_IRQ_CTRL:
>      case A_IRQ_CTRL_ACK:
>          *data = s->irq_ctrl;
> @@ -1482,6 +1501,33 @@ static const VMStateDescription vmstate_smmuv3_queue = {
>      },
>  };
>
> +static bool smmuv3_gbpa_needed(void *opaque)
> +{
> +    SMMUv3State *s = opaque;
> +
> +    /*
> +     * Only migrate GBPA if ABORT set from SW to 1, which is different from
> +     * default behaviour. This allows maximum compatibility with old qemu
> +     * instances while being forward compatible.
> +     */
> +    if (FIELD_EX32(s->gbpa, GBPA, ABORT)) {
> +        return true;
> +    }

I think we should check the whole register against its reset value,
not just the ABORT bit. Otherwise if the guest writes the other fields
to non default values we won't migrate them. That doesn't change the
device behaviour now but it will have the weird effect that the guest
could write the register and then after a migration find it reading
back as a different value. Plus if in future we implement actual
functionality for any of the other fields then we'll want to know
what their true values written by the guest are.

Linux never changes any fields except ABORT so for the most interesting
guest it won't make a difference right now.

thanks
-- PMM

Re: [PATCH v3] hw/arm/smmuv3: Add GBPA register

[Mostafa Saleh]

On Fri, Feb 10, 2023 at 04:11:37PM +0000, Peter Maydell wrote:
> 
> I think we should check the whole register against its reset value,
> not just the ABORT bit. Otherwise if the guest writes the other fields
> to non default values we won't migrate them. That doesn't change the
> device behaviour now but it will have the weird effect that the guest
> could write the register and then after a migration find it reading
> back as a different value. Plus if in future we implement actual
> functionality for any of the other fields then we'll want to know
> what their true values written by the guest are.
> 
> Linux never changes any fields except ABORT so for the most interesting
> guest it won't make a difference right now.

I agree, I was trying to achieve maximum backward compatibility as we
don’t care about other fields, but this can be weird for forward
migration, if the register is not fully migrated.
And if SW touches this register it would probably want to configure the
ABORT at some point, so it won’t help much for backward migration.

I will update this in v4.

Thanks,
Mostafa