[Qemu-devel] [PATCH 0/3] ARM: fix secondary boot GIC init, GIC bugs

[Qemu-devel] [PATCH 0/3] ARM: fix secondary boot GIC init, GIC bugs

[Peter Maydell]

The secondary CPU boot code we use on ARM had a couple of
places where it was accidentally relying on bugs or implementation
dependent behaviour of QEMU's on GIC implementation:
 * we weren't initialising the GICC_PMR priority mask, which
   in a correct v1 or v2 GIC is set to mask out all interrupts
   from reset. This worked on the QEMU GIC because our GIC (a)
   gets the reset value of PMR wrong on non-11MPCore and (b) is
   doing an incorrect comparison against the PMR value when
   delivering interrupts anyway.
 * no barrier between initialising the GIC and doing a WFI; this
   is fine for TCG QEMU but could potentially result in the GIC
   config not being guaranteed to have happened before we hit the
   WFI when running on real CPU hardware under ARM KVM.

This patch series first fixes the secondary CPU boot code bugs,
and then corrects our GIC model to match the specs.

NB: I don't have a working test setup/images for highbank or
exynos4 so those changes are only compile tested, but they are
basically the same as the generic boot code changes.

Peter Maydell (3):
  hw/arm_boot, exynos4210, highbank: Fix secondary boot GIC init
  hw/arm_gic: Fix comparison with priority mask register
  hw/arm_gic_common: Correct GICC_PMR reset value for newer GICs

 hw/arm_boot.c       |   17 ++++++++++++++---
 hw/arm_gic.c        |    2 +-
 hw/arm_gic_common.c |    6 +++++-
 hw/armv7m_nvic.c    |    4 +++-
 hw/exynos4210.c     |   10 +++++++---
 hw/highbank.c       |    7 +++++--
 6 files changed, 35 insertions(+), 11 deletions(-)

-- 
1.7.9.5

[Qemu-devel] [PATCH 1/3] hw/arm_boot, exynos4210, highbank: Fix secondary boot GIC init

[Peter Maydell]

Fix the code in the secondary CPU boot stubs so that it correctly
initialises the GIC rather than relying on bugs or implementation
dependent aspects of the QEMU GIC implementation:
 * set the GIC_PMR.Priority field to all-ones, so that all
   interrupts are passed through. The default of all-zeroes
   means all interrupts are masked, and QEMU only booted because
   of a bug in the priority masking in our GIC implementation.
 * add a barrier after GIC setup and before WFI to ensure that
   GIC config is complete before we go into a possible low power
   state. This isn't needed with the software GIC model but could
   be required when using KVM and executing this code on the
   real hardware CPU.

Note that of the three secondary stub implementations, only
the common generic one needs to support both v6 and v7 DSB
encodings; highbank and exynos4210 will always be v7 CPUs.

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
---
 hw/arm_boot.c   |   17 ++++++++++++++---
 hw/exynos4210.c |   10 +++++++---
 hw/highbank.c   |    7 +++++--
 3 files changed, 26 insertions(+), 8 deletions(-)

diff --git a/hw/arm_boot.c b/hw/arm_boot.c
index 92e2cab..ec3b8d5 100644
--- a/hw/arm_boot.c
+++ b/hw/arm_boot.c
@@ -44,11 +44,17 @@ static uint32_t bootloader[] = {
  * for an interprocessor interrupt and polling a configurable
  * location for the kernel secondary CPU entry point.
  */
+#define DSB_INSN 0xf57ff04f
+#define CP15_DSB_INSN 0xee070f9a /* mcr cp15, 0, r0, c7, c10, 4 */
+
 static uint32_t smpboot[] = {
-  0xe59f201c, /* ldr r2, gic_cpu_if */
-  0xe59f001c, /* ldr r0, startaddr */
+  0xe59f2028, /* ldr r2, gic_cpu_if */
+  0xe59f0028, /* ldr r0, startaddr */
   0xe3a01001, /* mov r1, #1 */
-  0xe5821000, /* str r1, [r2] */
+  0xe5821000, /* str r1, [r2] - set GICC_CTLR.Enable */
+  0xe3a010ff, /* mov r1, #0xff */
+  0xe5821004, /* str r1, [r2, 4] - set GIC_PMR.Priority to 0xff */
+  DSB_INSN,   /* dsb */
   0xe320f003, /* wfi */
   0xe5901000, /* ldr     r1, [r0] */
   0xe1110001, /* tst     r1, r1 */
@@ -65,6 +71,11 @@ static void default_write_secondary(ARMCPU *cpu,
     smpboot[ARRAY_SIZE(smpboot) - 1] = info->smp_bootreg_addr;
     smpboot[ARRAY_SIZE(smpboot) - 2] = info->gic_cpu_if_addr;
     for (n = 0; n < ARRAY_SIZE(smpboot); n++) {
+        /* Replace DSB with the pre-v7 DSB if necessary. */
+        if (!arm_feature(&cpu->env, ARM_FEATURE_V7) &&
+            smpboot[n] == DSB_INSN) {
+            smpboot[n] = CP15_DSB_INSN;
+        }
         smpboot[n] = tswap32(smpboot[n]);
     }
     rom_add_blob_fixed("smpboot", smpboot, sizeof(smpboot),
diff --git a/hw/exynos4210.c b/hw/exynos4210.c
index 00d4db8..22148cd 100644
--- a/hw/exynos4210.c
+++ b/hw/exynos4210.c
@@ -80,12 +80,16 @@ void exynos4210_write_secondary(ARMCPU *cpu,
 {
     int n;
     uint32_t smpboot[] = {
-        0xe59f3024, /* ldr r3, External gic_cpu_if */
-        0xe59f2024, /* ldr r2, Internal gic_cpu_if */
-        0xe59f0024, /* ldr r0, startaddr */
+        0xe59f3034, /* ldr r3, External gic_cpu_if */
+        0xe59f2034, /* ldr r2, Internal gic_cpu_if */
+        0xe59f0034, /* ldr r0, startaddr */
         0xe3a01001, /* mov r1, #1 */
         0xe5821000, /* str r1, [r2] */
         0xe5831000, /* str r1, [r3] */
+        0xe3a010ff, /* mov r1, #0xff */
+        0xe5821004, /* str r1, [r2, #4] */
+        0xe5831004, /* str r1, [r3, #4] */
+        0xf57ff04f, /* dsb */
         0xe320f003, /* wfi */
         0xe5901000, /* ldr     r1, [r0] */
         0xe1110001, /* tst     r1, r1 */
diff --git a/hw/highbank.c b/hw/highbank.c
index afbb005..447e57d 100644
--- a/hw/highbank.c
+++ b/hw/highbank.c
@@ -44,9 +44,12 @@ static void hb_write_secondary(ARMCPU *cpu, const struct arm_boot_info *info)
         0xe210000f, /* ands r0, r0, #0x0f */
         0xe3a03040, /* mov r3, #0x40 - jump address is 0x40 + 0x10 * core id */
         0xe0830200, /* add r0, r3, r0, lsl #4 */
-        0xe59f2018, /* ldr r2, privbase */
+        0xe59f2024, /* ldr r2, privbase */
         0xe3a01001, /* mov r1, #1 */
-        0xe5821100, /* str r1, [r2, #256] */
+        0xe5821100, /* str r1, [r2, #256] - set GICC_CTLR.Enable */
+        0xe3a010ff, /* mov r1, #0xff */
+        0xe5821104, /* str r1, [r2, #260] - set GICC_PMR.Priority to 0xff */
+        0xf57ff04f, /* dsb */
         0xe320f003, /* wfi */
         0xe5901000, /* ldr     r1, [r0] */
         0xe1110001, /* tst     r1, r1 */
-- 
1.7.9.5

[Qemu-devel] [PATCH 2/3] hw/arm_gic: Fix comparison with priority mask register

[Peter Maydell]

The GIC spec states that only interrupts with higher priority
than the value in the GICC_PMR priority mask register are
passed through to the processor. We were incorrectly allowing
through interrupts with a priority equal to the specified
value: correct the comparison operation to match the spec.

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
---
 hw/arm_gic.c |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hw/arm_gic.c b/hw/arm_gic.c
index f9e423f..672d539 100644
--- a/hw/arm_gic.c
+++ b/hw/arm_gic.c
@@ -73,7 +73,7 @@ void gic_update(GICState *s)
             }
         }
         level = 0;
-        if (best_prio <= s->priority_mask[cpu]) {
+        if (best_prio < s->priority_mask[cpu]) {
             s->current_pending[cpu] = best_irq;
             if (best_prio < s->running_priority[cpu]) {
                 DPRINTF("Raised pending IRQ %d\n", best_irq);
-- 
1.7.9.5

[Qemu-devel] [PATCH 3/3] hw/arm_gic_common: Correct GICC_PMR reset value for newer GICs

[Peter Maydell]

The GIC architecture specification for v1 and v2 GICs (as found
on the Cortex-A9 and newer) states that the GICC_PMR reset value
is zero; this differs from the 0xf0 reset value used on 11MPCore.
The NVIC is different again in not having a CPU interface; since
we share the GIC code we must force the priority mask field to
allow through all interrupts.

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
---
 hw/arm_gic_common.c |    6 +++++-
 hw/armv7m_nvic.c    |    4 +++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/hw/arm_gic_common.c b/hw/arm_gic_common.c
index 8369309..73ae331 100644
--- a/hw/arm_gic_common.c
+++ b/hw/arm_gic_common.c
@@ -127,7 +127,11 @@ static void arm_gic_common_reset(DeviceState *dev)
     int i;
     memset(s->irq_state, 0, GIC_MAXIRQ * sizeof(gic_irq_state));
     for (i = 0 ; i < s->num_cpu; i++) {
-        s->priority_mask[i] = 0xf0;
+        if (s->revision == REV_11MPCORE) {
+            s->priority_mask[i] = 0xf0;
+        } else {
+            s->priority_mask[i] = 0;
+        }
         s->current_pending[i] = 1023;
         s->running_irq[i] = 1023;
         s->running_priority[i] = 0x100;
diff --git a/hw/armv7m_nvic.c b/hw/armv7m_nvic.c
index f0a2e7b..4963678 100644
--- a/hw/armv7m_nvic.c
+++ b/hw/armv7m_nvic.c
@@ -455,9 +455,11 @@ static void armv7m_nvic_reset(DeviceState *dev)
     nc->parent_reset(dev);
     /* Common GIC reset resets to disabled; the NVIC doesn't have
      * per-CPU interfaces so mark our non-existent CPU interface
-     * as enabled by default.
+     * as enabled by default, and with a priority mask which allows
+     * all interrupts through.
      */
     s->gic.cpu_enabled[0] = 1;
+    s->gic.priority_mask[0] = 0x100;
     /* The NVIC as a whole is always enabled. */
     s->gic.enabled = 1;
     systick_reset(s);
-- 
1.7.9.5

Re: [Qemu-devel] [kvmarm] [PATCH 2/3] hw/arm_gic: Fix comparison with priority mask register

[Christoffer Dall]

On Thu, Nov 29, 2012 at 12:02 PM, Peter Maydell
<peter.maydell@linaro.org> wrote:
> The GIC spec states that only interrupts with higher priority
> than the value in the GICC_PMR priority mask register are
> passed through to the processor. We were incorrectly allowing
> through interrupts with a priority equal to the specified
> value: correct the comparison operation to match the spec.
>
> Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
> ---
>  hw/arm_gic.c |    2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
>
> diff --git a/hw/arm_gic.c b/hw/arm_gic.c
> index f9e423f..672d539 100644
> --- a/hw/arm_gic.c
> +++ b/hw/arm_gic.c
> @@ -73,7 +73,7 @@ void gic_update(GICState *s)
>              }
>          }
>          level = 0;
> -        if (best_prio <= s->priority_mask[cpu]) {
> +        if (best_prio < s->priority_mask[cpu]) {
>              s->current_pending[cpu] = best_irq;
>              if (best_prio < s->running_priority[cpu]) {
>                  DPRINTF("Raised pending IRQ %d\n", best_irq);
> --
> 1.7.9.5
>
looks good to me

Re: [Qemu-devel] [PATCH 1/3] hw/arm_boot, exynos4210, highbank: Fix secondary boot GIC init

[Igor Mitsyanko]

On 11/29/2012 9:02 PM, Peter Maydell wrote:
> Fix the code in the secondary CPU boot stubs so that it correctly
> initialises the GIC rather than relying on bugs or implementation
> dependent aspects of the QEMU GIC implementation:
>   * set the GIC_PMR.Priority field to all-ones, so that all
>     interrupts are passed through. The default of all-zeroes
>     means all interrupts are masked, and QEMU only booted because
>     of a bug in the priority masking in our GIC implementation.
>   * add a barrier after GIC setup and before WFI to ensure that
>     GIC config is complete before we go into a possible low power
>     state. This isn't needed with the software GIC model but could
>     be required when using KVM and executing this code on the
>     real hardware CPU.
>
> Note that of the three secondary stub implementations, only
> the common generic one needs to support both v6 and v7 DSB
> encodings; highbank and exynos4210 will always be v7 CPUs.
>
> Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
> ---
>   hw/arm_boot.c   |   17 ++++++++++++++---
>   hw/exynos4210.c |   10 +++++++---
>   hw/highbank.c   |    7 +++++--
>   3 files changed, 26 insertions(+), 8 deletions(-)
>


Reviewed-by: Igor Mitsyanko <i.mitsyanko@samsung.com>


> diff --git a/hw/arm_boot.c b/hw/arm_boot.c
> index 92e2cab..ec3b8d5 100644
> --- a/hw/arm_boot.c
> +++ b/hw/arm_boot.c
> @@ -44,11 +44,17 @@ static uint32_t bootloader[] = {
>    * for an interprocessor interrupt and polling a configurable
>    * location for the kernel secondary CPU entry point.
>    */
> +#define DSB_INSN 0xf57ff04f
> +#define CP15_DSB_INSN 0xee070f9a /* mcr cp15, 0, r0, c7, c10, 4 */
> +
>   static uint32_t smpboot[] = {
> -  0xe59f201c, /* ldr r2, gic_cpu_if */
> -  0xe59f001c, /* ldr r0, startaddr */
> +  0xe59f2028, /* ldr r2, gic_cpu_if */
> +  0xe59f0028, /* ldr r0, startaddr */
>     0xe3a01001, /* mov r1, #1 */
> -  0xe5821000, /* str r1, [r2] */
> +  0xe5821000, /* str r1, [r2] - set GICC_CTLR.Enable */
> +  0xe3a010ff, /* mov r1, #0xff */
> +  0xe5821004, /* str r1, [r2, 4] - set GIC_PMR.Priority to 0xff */
> +  DSB_INSN,   /* dsb */
>     0xe320f003, /* wfi */
>     0xe5901000, /* ldr     r1, [r0] */
>     0xe1110001, /* tst     r1, r1 */
> @@ -65,6 +71,11 @@ static void default_write_secondary(ARMCPU *cpu,
>       smpboot[ARRAY_SIZE(smpboot) - 1] = info->smp_bootreg_addr;
>       smpboot[ARRAY_SIZE(smpboot) - 2] = info->gic_cpu_if_addr;
>       for (n = 0; n < ARRAY_SIZE(smpboot); n++) {
> +        /* Replace DSB with the pre-v7 DSB if necessary. */
> +        if (!arm_feature(&cpu->env, ARM_FEATURE_V7) &&
> +            smpboot[n] == DSB_INSN) {
> +            smpboot[n] = CP15_DSB_INSN;
> +        }
>           smpboot[n] = tswap32(smpboot[n]);
>       }
>       rom_add_blob_fixed("smpboot", smpboot, sizeof(smpboot),
> diff --git a/hw/exynos4210.c b/hw/exynos4210.c
> index 00d4db8..22148cd 100644
> --- a/hw/exynos4210.c
> +++ b/hw/exynos4210.c
> @@ -80,12 +80,16 @@ void exynos4210_write_secondary(ARMCPU *cpu,
>   {
>       int n;
>       uint32_t smpboot[] = {
> -        0xe59f3024, /* ldr r3, External gic_cpu_if */
> -        0xe59f2024, /* ldr r2, Internal gic_cpu_if */
> -        0xe59f0024, /* ldr r0, startaddr */
> +        0xe59f3034, /* ldr r3, External gic_cpu_if */
> +        0xe59f2034, /* ldr r2, Internal gic_cpu_if */
> +        0xe59f0034, /* ldr r0, startaddr */
>           0xe3a01001, /* mov r1, #1 */
>           0xe5821000, /* str r1, [r2] */
>           0xe5831000, /* str r1, [r3] */
> +        0xe3a010ff, /* mov r1, #0xff */
> +        0xe5821004, /* str r1, [r2, #4] */
> +        0xe5831004, /* str r1, [r3, #4] */
> +        0xf57ff04f, /* dsb */
>           0xe320f003, /* wfi */
>           0xe5901000, /* ldr     r1, [r0] */
>           0xe1110001, /* tst     r1, r1 */
> diff --git a/hw/highbank.c b/hw/highbank.c
> index afbb005..447e57d 100644
> --- a/hw/highbank.c
> +++ b/hw/highbank.c
> @@ -44,9 +44,12 @@ static void hb_write_secondary(ARMCPU *cpu, const struct arm_boot_info *info)
>           0xe210000f, /* ands r0, r0, #0x0f */
>           0xe3a03040, /* mov r3, #0x40 - jump address is 0x40 + 0x10 * core id */
>           0xe0830200, /* add r0, r3, r0, lsl #4 */
> -        0xe59f2018, /* ldr r2, privbase */
> +        0xe59f2024, /* ldr r2, privbase */
>           0xe3a01001, /* mov r1, #1 */
> -        0xe5821100, /* str r1, [r2, #256] */
> +        0xe5821100, /* str r1, [r2, #256] - set GICC_CTLR.Enable */
> +        0xe3a010ff, /* mov r1, #0xff */
> +        0xe5821104, /* str r1, [r2, #260] - set GICC_PMR.Priority to 0xff */
> +        0xf57ff04f, /* dsb */
>           0xe320f003, /* wfi */
>           0xe5901000, /* ldr     r1, [r0] */
>           0xe1110001, /* tst     r1, r1 */
>

Re: [Qemu-devel] [PATCH 2/3] hw/arm_gic: Fix comparison with priority mask register

[Igor Mitsyanko]

On 11/29/2012 9:02 PM, Peter Maydell wrote:
> The GIC spec states that only interrupts with higher priority
> than the value in the GICC_PMR priority mask register are
> passed through to the processor. We were incorrectly allowing
> through interrupts with a priority equal to the specified
> value: correct the comparison operation to match the spec.
>
> Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
> ---
>   hw/arm_gic.c |    2 +-
>   1 file changed, 1 insertion(+), 1 deletion(-)
>
> diff --git a/hw/arm_gic.c b/hw/arm_gic.c
> index f9e423f..672d539 100644
> --- a/hw/arm_gic.c
> +++ b/hw/arm_gic.c
> @@ -73,7 +73,7 @@ void gic_update(GICState *s)
>               }
>           }
>           level = 0;
> -        if (best_prio <= s->priority_mask[cpu]) {
> +        if (best_prio < s->priority_mask[cpu]) {
>               s->current_pending[cpu] = best_irq;
>               if (best_prio < s->running_priority[cpu]) {
>                   DPRINTF("Raised pending IRQ %d\n", best_irq);
>


Reviewed-by: Igor Mitsyanko <i.mitsyanko@samsung.com>

Re: [Qemu-devel] [PATCH 3/3] hw/arm_gic_common: Correct GICC_PMR reset value for newer GICs

[Igor Mitsyanko]

On 11/29/2012 9:02 PM, Peter Maydell wrote:
> The GIC architecture specification for v1 and v2 GICs (as found
> on the Cortex-A9 and newer) states that the GICC_PMR reset value
> is zero; this differs from the 0xf0 reset value used on 11MPCore.
> The NVIC is different again in not having a CPU interface; since
> we share the GIC code we must force the priority mask field to
> allow through all interrupts.
>
> Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
> ---
>   hw/arm_gic_common.c |    6 +++++-
>   hw/armv7m_nvic.c    |    4 +++-
>   2 files changed, 8 insertions(+), 2 deletions(-)
>
> diff --git a/hw/arm_gic_common.c b/hw/arm_gic_common.c
> index 8369309..73ae331 100644
> --- a/hw/arm_gic_common.c
> +++ b/hw/arm_gic_common.c
> @@ -127,7 +127,11 @@ static void arm_gic_common_reset(DeviceState *dev)
>       int i;
>       memset(s->irq_state, 0, GIC_MAXIRQ * sizeof(gic_irq_state));
>       for (i = 0 ; i < s->num_cpu; i++) {
> -        s->priority_mask[i] = 0xf0;
> +        if (s->revision == REV_11MPCORE) {
> +            s->priority_mask[i] = 0xf0;
> +        } else {
> +            s->priority_mask[i] = 0;
> +        }
>           s->current_pending[i] = 1023;
>           s->running_irq[i] = 1023;
>           s->running_priority[i] = 0x100;
> diff --git a/hw/armv7m_nvic.c b/hw/armv7m_nvic.c
> index f0a2e7b..4963678 100644
> --- a/hw/armv7m_nvic.c
> +++ b/hw/armv7m_nvic.c
> @@ -455,9 +455,11 @@ static void armv7m_nvic_reset(DeviceState *dev)
>       nc->parent_reset(dev);
>       /* Common GIC reset resets to disabled; the NVIC doesn't have
>        * per-CPU interfaces so mark our non-existent CPU interface
> -     * as enabled by default.
> +     * as enabled by default, and with a priority mask which allows
> +     * all interrupts through.
>        */
>       s->gic.cpu_enabled[0] = 1;
> +    s->gic.priority_mask[0] = 0x100;
>       /* The NVIC as a whole is always enabled. */
>       s->gic.enabled = 1;
>       systick_reset(s);
>

Reviewed-by: Igor Mitsyanko <i.mitsyanko@samsung.com>