[RFC PATCH v1 00/10] xen/arm: Add GICv3 support

[RFC PATCH v1 00/10] xen/arm: Add GICv3 support

[vijay.kilari]

From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>

made changes to existing gic and vgic drivers
to make it generic and added support for GIC v3
hw version

Tested with ARM64 simulator with single core

Vijaya Kumar K (10):
  xen/arm: make secondary gic init as notifier call
  xen/arm: register mmio handler at runtime
  xen/arm: move vgic data to vgic driver
  arm/xen: move gic save and restore registers to gic driver
  xen/arm: move gic definitions to seperate file
  xen/arm: split gic driver into generic and gicv2 driver
  xen/arm: split vgic into generic and GIC v2 specific drivers
  xen/arm: Add support for GIC v3
  xen/arm: Add vgic support for GIC v3
  xen/arm: GICv3 device tree parsing

 xen/arch/arm/Makefile             |    4 +-
 xen/arch/arm/domain.c             |    3 +
 xen/arch/arm/domain_build.c       |   41 +-
 xen/arch/arm/gic-v2.c             |  562 ++++++++++++++++++++++
 xen/arch/arm/gic-v3.c             |  944 +++++++++++++++++++++++++++++++++++++
 xen/arch/arm/gic.c                |  512 +++++---------------
 xen/arch/arm/io.c                 |   32 +-
 xen/arch/arm/io.h                 |   16 +-
 xen/arch/arm/smpboot.c            |    3 +-
 xen/arch/arm/vgic-v2.c            |  636 +++++++++++++++++++++++++
 xen/arch/arm/vgic-v3.c            |  927 ++++++++++++++++++++++++++++++++++++
 xen/arch/arm/vgic.c               |  591 ++---------------------
 xen/arch/arm/vuart.c              |   39 +-
 xen/include/asm-arm/domain.h      |   20 +-
 xen/include/asm-arm/gic.h         |  171 +++----
 xen/include/asm-arm/gic_v2_defs.h |  134 ++++++
 xen/include/asm-arm/gic_v3_defs.h |  211 +++++++++
 17 files changed, 3733 insertions(+), 1113 deletions(-)
 create mode 100644 xen/arch/arm/gic-v2.c
 create mode 100644 xen/arch/arm/gic-v3.c
 create mode 100644 xen/arch/arm/vgic-v2.c
 create mode 100644 xen/arch/arm/vgic-v3.c
 create mode 100644 xen/include/asm-arm/gic_v2_defs.h
 create mode 100644 xen/include/asm-arm/gic_v3_defs.h

-- 
1.7.9.5

[RFC PATCH v1 01/10] xen/arm: make secondary gic init as notifier call

[vijay.kilari]

From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>

make gic init for secondary cpus as notifier call
instead calling directly from secondary boot for
each cpu. This makes secondary gic init generic and runtime.

Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
---
 xen/arch/arm/gic.c        |   35 ++++++++++++++++++++++++++---------
 xen/arch/arm/smpboot.c    |    3 +--
 xen/include/asm-arm/gic.h |    2 --
 3 files changed, 27 insertions(+), 13 deletions(-)

diff --git a/xen/arch/arm/gic.c b/xen/arch/arm/gic.c
index 91a2982..4be0897 100644
--- a/xen/arch/arm/gic.c
+++ b/xen/arch/arm/gic.c
@@ -20,6 +20,7 @@
 #include <xen/config.h>
 #include <xen/lib.h>
 #include <xen/init.h>
+#include <xen/cpu.h>
 #include <xen/mm.h>
 #include <xen/irq.h>
 #include <xen/sched.h>
@@ -380,6 +381,30 @@ static void __cpuinit gic_hyp_disable(void)
     GICH[GICH_HCR] = 0;
 }
 
+/* Set up the per-CPU parts of the GIC for a secondary CPU */
+static int  __cpuinit gic_init_secondary_cpu(struct notifier_block *nfb,
+                                        unsigned long action, void *hcpu)
+{
+    if (action == CPU_STARTING)
+    {
+        spin_lock(&gic.lock);
+        gic_cpu_init();
+        gic_hyp_init();
+        spin_unlock(&gic.lock);
+    }
+    return NOTIFY_DONE;
+}
+
+static struct notifier_block gic_cpu_nb = { 
+    .notifier_call = gic_init_secondary_cpu,
+    .priority = 100 
+};
+
+static void gic_smp_init(void)
+{
+   register_cpu_notifier(&gic_cpu_nb);
+}
+
 int gic_irq_xlate(const u32 *intspec, unsigned int intsize,
                   unsigned int *out_hwirq,
                   unsigned int *out_type)
@@ -469,6 +494,7 @@ void __init gic_init(void)
     spin_lock_init(&gic.lock);
     spin_lock(&gic.lock);
 
+    gic_smp_init();
     gic_dist_init();
     gic_cpu_init();
     gic_hyp_init();
@@ -524,15 +550,6 @@ void smp_send_state_dump(unsigned int cpu)
     send_SGI_one(cpu, GIC_SGI_DUMP_STATE);
 }
 
-/* Set up the per-CPU parts of the GIC for a secondary CPU */
-void __cpuinit gic_init_secondary_cpu(void)
-{
-    spin_lock(&gic.lock);
-    gic_cpu_init();
-    gic_hyp_init();
-    spin_unlock(&gic.lock);
-}
-
 /* Shut down the per-CPU GIC interface */
 void gic_disable_cpu(void)
 {
diff --git a/xen/arch/arm/smpboot.c b/xen/arch/arm/smpboot.c
index a829957..765efcf 100644
--- a/xen/arch/arm/smpboot.c
+++ b/xen/arch/arm/smpboot.c
@@ -283,7 +283,7 @@ void __cpuinit start_secondary(unsigned long boot_phys_offset,
 
     mmu_init_secondary_cpu();
 
-    gic_init_secondary_cpu();
+    notify_cpu_starting(cpuid);
 
     init_secondary_IRQ();
 
@@ -297,7 +297,6 @@ void __cpuinit start_secondary(unsigned long boot_phys_offset,
     setup_cpu_sibling_map(cpuid);
 
     /* Run local notifiers */
-    notify_cpu_starting(cpuid);
     wmb();
 
     /* Now report this CPU is up */
diff --git a/xen/include/asm-arm/gic.h b/xen/include/asm-arm/gic.h
index 071280b..340ef73 100644
--- a/xen/include/asm-arm/gic.h
+++ b/xen/include/asm-arm/gic.h
@@ -189,8 +189,6 @@ extern int gic_route_irq_to_guest(struct domain *d,
 extern void gic_interrupt(struct cpu_user_regs *regs, int is_fiq);
 /* Bring up the interrupt controller, and report # cpus attached */
 extern void gic_init(void);
-/* Bring up a secondary CPU's per-CPU GIC interface */
-extern void gic_init_secondary_cpu(void);
 /* Take down a CPU's per-CPU GIC interface */
 extern void gic_disable_cpu(void);
 /* setup the gic virtual interface for a guest */
-- 
1.7.9.5

Re: [RFC PATCH v1 01/10] xen/arm: make secondary gic init as notifier call

[Julien Grall]

Hello Vijay,

Thanks for your patch.

On 03/19/2014 02:17 PM, vijay.kilari@gmail.com wrote:
> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> 
> make gic init for secondary cpus as notifier call
> instead calling directly from secondary boot for
> each cpu. This makes secondary gic init generic and runtime.
> 
> Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> ---
>  xen/arch/arm/gic.c        |   35 ++++++++++++++++++++++++++---------
>  xen/arch/arm/smpboot.c    |    3 +--
>  xen/include/asm-arm/gic.h |    2 --
>  3 files changed, 27 insertions(+), 13 deletions(-)
> 
> diff --git a/xen/arch/arm/gic.c b/xen/arch/arm/gic.c
> index 91a2982..4be0897 100644
> --- a/xen/arch/arm/gic.c
> +++ b/xen/arch/arm/gic.c
> @@ -20,6 +20,7 @@
>  #include <xen/config.h>
>  #include <xen/lib.h>
>  #include <xen/init.h>
> +#include <xen/cpu.h>
>  #include <xen/mm.h>
>  #include <xen/irq.h>
>  #include <xen/sched.h>
> @@ -380,6 +381,30 @@ static void __cpuinit gic_hyp_disable(void)
>      GICH[GICH_HCR] = 0;
>  }
>  
> +/* Set up the per-CPU parts of the GIC for a secondary CPU */
> +static int  __cpuinit gic_init_secondary_cpu(struct notifier_block *nfb,
> +                                        unsigned long action, void *hcpu)
> +{
> +    if (action == CPU_STARTING)
> +    {
> +        spin_lock(&gic.lock);
> +        gic_cpu_init();
> +        gic_hyp_init();
> +        spin_unlock(&gic.lock);
> +    }
> +    return NOTIFY_DONE;
> +}
> +

This is not the correct way to create a notifier block.

You should have a good similar to (see an example in common/timer.c):

static cpu_callback(struct notifier_block* nfb,
                    unsigned long action, void *hcpu)
{
     unsigned int cpu = (unsigned long)hcpu;

     switch ( action )
     case CPU_STARTING:
        gic_init_secondary_cpu();
        break;
     default:
        break;
     return NOTIFY_DONE;
}

> +static struct notifier_block gic_cpu_nb = { 
> +    .notifier_call = gic_init_secondary_cpu,
> +    .priority = 100 
> +};

> +static void gic_smp_init(void)
> +{
> +   register_cpu_notifier(&gic_cpu_nb);
> +}
> +

You don't need to create a separate function to register the notifier.
You can directly call it in gic_init.

>  int gic_irq_xlate(const u32 *intspec, unsigned int intsize,
>                    unsigned int *out_hwirq,
>                    unsigned int *out_type)
> @@ -469,6 +494,7 @@ void __init gic_init(void)
>      spin_lock_init(&gic.lock);
>      spin_lock(&gic.lock);
>  
> +    gic_smp_init();
>      gic_dist_init();
>      gic_cpu_init();
>      gic_hyp_init();
> @@ -524,15 +550,6 @@ void smp_send_state_dump(unsigned int cpu)
>      send_SGI_one(cpu, GIC_SGI_DUMP_STATE);
>  }
>  
> -/* Set up the per-CPU parts of the GIC for a secondary CPU */
> -void __cpuinit gic_init_secondary_cpu(void)
> -{
> -    spin_lock(&gic.lock);
> -    gic_cpu_init();
> -    gic_hyp_init();
> -    spin_unlock(&gic.lock);
> -}
> -
>  /* Shut down the per-CPU GIC interface */
>  void gic_disable_cpu(void)
>  {
> diff --git a/xen/arch/arm/smpboot.c b/xen/arch/arm/smpboot.c
> index a829957..765efcf 100644
> --- a/xen/arch/arm/smpboot.c
> +++ b/xen/arch/arm/smpboot.c
> @@ -283,7 +283,7 @@ void __cpuinit start_secondary(unsigned long boot_phys_offset,
>  
>      mmu_init_secondary_cpu();
>  
> -    gic_init_secondary_cpu();
> +    notify_cpu_starting(cpuid);

Can you explain why it's safe to move notify_cpu_starting earlier?

>  
>      init_secondary_IRQ();
>  
> @@ -297,7 +297,6 @@ void __cpuinit start_secondary(unsigned long boot_phys_offset,
>      setup_cpu_sibling_map(cpuid);
>  
>      /* Run local notifiers */

Please move also the comment, it's part of notify_cpu_starting. It
doesn't make any sense alone here.

> -    notify_cpu_starting(cpuid);
>      wmb();

Regards,

-- 
Julien Grall

Re: [RFC PATCH v1 01/10] xen/arm: make secondary gic init as notifier call

[Vijay Kilari]

On Thu, Mar 20, 2014 at 6:18 PM, Julien Grall <julien.grall@linaro.org> wrote:
> Hello Vijay,
>
> Thanks for your patch.
>
> On 03/19/2014 02:17 PM, vijay.kilari@gmail.com wrote:
>> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>>
>> make gic init for secondary cpus as notifier call
>> instead calling directly from secondary boot for
>> each cpu. This makes secondary gic init generic and runtime.
>>
>> Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>> ---
>>  xen/arch/arm/gic.c        |   35 ++++++++++++++++++++++++++---------
>>  xen/arch/arm/smpboot.c    |    3 +--
>>  xen/include/asm-arm/gic.h |    2 --
>>  3 files changed, 27 insertions(+), 13 deletions(-)
>>
>> diff --git a/xen/arch/arm/gic.c b/xen/arch/arm/gic.c
>> index 91a2982..4be0897 100644
>> --- a/xen/arch/arm/gic.c
>> +++ b/xen/arch/arm/gic.c
>> @@ -20,6 +20,7 @@
>>  #include <xen/config.h>
>>  #include <xen/lib.h>
>>  #include <xen/init.h>
>> +#include <xen/cpu.h>
>>  #include <xen/mm.h>
>>  #include <xen/irq.h>
>>  #include <xen/sched.h>
>> @@ -380,6 +381,30 @@ static void __cpuinit gic_hyp_disable(void)
>>      GICH[GICH_HCR] = 0;
>>  }
>>
>> +/* Set up the per-CPU parts of the GIC for a secondary CPU */
>> +static int  __cpuinit gic_init_secondary_cpu(struct notifier_block *nfb,
>> +                                        unsigned long action, void *hcpu)
>> +{
>> +    if (action == CPU_STARTING)
>> +    {
>> +        spin_lock(&gic.lock);
>> +        gic_cpu_init();
>> +        gic_hyp_init();
>> +        spin_unlock(&gic.lock);
>> +    }
>> +    return NOTIFY_DONE;
>> +}
>> +
>
> This is not the correct way to create a notifier block.
>
> You should have a good similar to (see an example in common/timer.c):
>
> static cpu_callback(struct notifier_block* nfb,
>                     unsigned long action, void *hcpu)
> {
>      unsigned int cpu = (unsigned long)hcpu;
>
>      switch ( action )
>      case CPU_STARTING:
>         gic_init_secondary_cpu();
>         break;
>      default:
>         break;
>      return NOTIFY_DONE;
> }

Apart from __cpuinit, I could not see any difference with this implementation.
am I missing anything specific?

>
>> +static struct notifier_block gic_cpu_nb = {
>> +    .notifier_call = gic_init_secondary_cpu,
>> +    .priority = 100
>> +};
>
>> +static void gic_smp_init(void)
>> +{
>> +   register_cpu_notifier(&gic_cpu_nb);
>> +}
>> +
>
> You don't need to create a separate function to register the notifier.
> You can directly call it in gic_init.
>
OK
>>  int gic_irq_xlate(const u32 *intspec, unsigned int intsize,
>>                    unsigned int *out_hwirq,
>>                    unsigned int *out_type)
>> @@ -469,6 +494,7 @@ void __init gic_init(void)
>>      spin_lock_init(&gic.lock);
>>      spin_lock(&gic.lock);
>>
>> +    gic_smp_init();
>>      gic_dist_init();
>>      gic_cpu_init();
>>      gic_hyp_init();
>> @@ -524,15 +550,6 @@ void smp_send_state_dump(unsigned int cpu)
>>      send_SGI_one(cpu, GIC_SGI_DUMP_STATE);
>>  }
>>
>> -/* Set up the per-CPU parts of the GIC for a secondary CPU */
>> -void __cpuinit gic_init_secondary_cpu(void)
>> -{
>> -    spin_lock(&gic.lock);
>> -    gic_cpu_init();
>> -    gic_hyp_init();
>> -    spin_unlock(&gic.lock);
>> -}
>> -
>>  /* Shut down the per-CPU GIC interface */
>>  void gic_disable_cpu(void)
>>  {
>> diff --git a/xen/arch/arm/smpboot.c b/xen/arch/arm/smpboot.c
>> index a829957..765efcf 100644
>> --- a/xen/arch/arm/smpboot.c
>> +++ b/xen/arch/arm/smpboot.c
>> @@ -283,7 +283,7 @@ void __cpuinit start_secondary(unsigned long boot_phys_offset,
>>
>>      mmu_init_secondary_cpu();
>>
>> -    gic_init_secondary_cpu();
>> +    notify_cpu_starting(cpuid);
>
> Can you explain why it's safe to move notify_cpu_starting earlier?
>
   When gic registers a notifier with action as CPU_STARTING, I am
getting a panic
   from gic driver because notify_cpu_starting() is called after most of the GIC
   initialization calls as below from start_secondary() are called.
Especially the issue is coming
   with init_maintenanc_interrupt(). So I have moved this notifier
before other GIC initialization
   calls and since I move notifier only before GIC initialization
calls it not be a problem.

    init_secondary_IRQ();

    gic_route_ppis();

    init_maintenance_interrupt();
    init_timer_interrupt();

    In linux also similar problem (not same) with GICv3 is observered
    http://marc.info/?l=git-commits-head&m=138418939813393&w=2

>>
>>      init_secondary_IRQ();
>>
>> @@ -297,7 +297,6 @@ void __cpuinit start_secondary(unsigned long boot_phys_offset,
>>      setup_cpu_sibling_map(cpuid);
>>
>>      /* Run local notifiers */
>
> Please move also the comment, it's part of notify_cpu_starting. It
> doesn't make any sense alone here.
>
OK
>> -    notify_cpu_starting(cpuid);
>>      wmb();
>
> Regards,
>
> --
> Julien Grall

Re: [RFC PATCH v1 01/10] xen/arm: make secondary gic init as notifier call

[Julien Grall]

Hello Vijay,

On 22/03/14 08:16, Vijay Kilari wrote:
> On Thu, Mar 20, 2014 at 6:18 PM, Julien Grall <julien.grall@linaro.org> wrote:
>>> +/* Set up the per-CPU parts of the GIC for a secondary CPU */
>>> +static int  __cpuinit gic_init_secondary_cpu(struct notifier_block *nfb,
>>> +                                        unsigned long action, void *hcpu)
>>> +{
>>> +    if (action == CPU_STARTING)
>>> +    {
>>> +        spin_lock(&gic.lock);
>>> +        gic_cpu_init();
>>> +        gic_hyp_init();
>>> +        spin_unlock(&gic.lock);
>>> +    }
>>> +    return NOTIFY_DONE;
>>> +}
>>> +
>>
>> This is not the correct way to create a notifier block.
>>
>> You should have a good similar to (see an example in common/timer.c):
>>
>> static cpu_callback(struct notifier_block* nfb,
>>                      unsigned long action, void *hcpu)
>> {
>>       unsigned int cpu = (unsigned long)hcpu;
>>
>>       switch ( action )
>>       case CPU_STARTING:
>>          gic_init_secondary_cpu();
>>          break;
>>       default:
>>          break;
>>       return NOTIFY_DONE;
>> }
>
> Apart from __cpuinit, I could not see any difference with this implementation.
> am I missing anything specific?


I'd prefer to use the same pattern function for CPU notifier (see 
common/timer.c) rather than creating a new one.

The main difference are:
	- the function name: the callback will be called on different CPU 
state. The current name "gic_init_secondary_cpu" is too specific. We 
might want to extend this callback later.
         - switch case: It's easier to extend with a switch case compare 
to if (foo)

>>> @@ -283,7 +283,7 @@ void __cpuinit start_secondary(unsigned long boot_phys_offset,
>>>
>>>       mmu_init_secondary_cpu();
>>>
>>> -    gic_init_secondary_cpu();
>>> +    notify_cpu_starting(cpuid);
>>
>> Can you explain why it's safe to move notify_cpu_starting earlier?
>>
>     When gic registers a notifier with action as CPU_STARTING, I am
> getting a panic
>     from gic driver because notify_cpu_starting() is called after most of the GIC
>     initialization calls as below from start_secondary() are called.
> Especially the issue is coming
>     with init_maintenanc_interrupt(). So I have moved this notifier
> before other GIC initialization
>     calls and since I move notifier only before GIC initialization
> calls it not be a problem.

It doesn't explain why it's safe... CPU_STARTING is also used in some 
place to initialize internal data structure. Are you sure that theses 
callbacks can be called earlier?

Regards,

-- 
Julien Grall

Re: [RFC PATCH v1 01/10] xen/arm: make secondary gic init as notifier call

[Vijay Kilari]

Hi Julien,

>>>> @@ -283,7 +283,7 @@ void __cpuinit start_secondary(unsigned long
>>>> boot_phys_offset,
>>>>
>>>>       mmu_init_secondary_cpu();
>>>>
>>>> -    gic_init_secondary_cpu();
>>>> +    notify_cpu_starting(cpuid);
>>>
>>>
>>> Can you explain why it's safe to move notify_cpu_starting earlier?
>>>
>>     When gic registers a notifier with action as CPU_STARTING, I am
>> getting a panic
>>     from gic driver because notify_cpu_starting() is called after most of
>> the GIC
>>     initialization calls as below from start_secondary() are called.
>> Especially the issue is coming
>>     with init_maintenanc_interrupt(). So I have moved this notifier
>> before other GIC initialization
>>     calls and since I move notifier only before GIC initialization
>> calls it not be a problem.
>
>
> It doesn't explain why it's safe... CPU_STARTING is also used in some place
> to initialize internal data structure. Are you sure that theses callbacks
> can be called earlier?
>

The below callback is the only callback that is using CPU_STARTING,
IMO it is only initializing pcpu data.

static int cpu_credit2_callback(
    struct notifier_block *nfb, unsigned long action, void *hcpu)
{
    unsigned int cpu = (unsigned long)hcpu;
    int rc = 0;

    switch ( action )
    {
    case CPU_STARTING:
        csched_cpu_starting(cpu);
        break;
    default:
        break;
    }

    return !rc ? NOTIFY_DONE : notifier_from_errno(rc);
}

With this patch, notifier is only called just before GIC initialization.
So should not be a problem. Let me know your opinion? I am
not very familiar with this piece of code.

> Regards,
>
> --
> Julien Grall

Re: [RFC PATCH v1 01/10] xen/arm: make secondary gic init as notifier call

[Julien Grall]

Hello Vijay,

(Adding George for the scheduler part)

On 03/26/2014 11:27 AM, Vijay Kilari wrote:
> Hi Julien,
> 
>>>>> @@ -283,7 +283,7 @@ void __cpuinit start_secondary(unsigned long
>>>>> boot_phys_offset,
>>>>>
>>>>>       mmu_init_secondary_cpu();
>>>>>
>>>>> -    gic_init_secondary_cpu();
>>>>> +    notify_cpu_starting(cpuid);
>>>>
>>>>
>>>> Can you explain why it's safe to move notify_cpu_starting earlier?
>>>>
>>>     When gic registers a notifier with action as CPU_STARTING, I am
>>> getting a panic
>>>     from gic driver because notify_cpu_starting() is called after most of
>>> the GIC
>>>     initialization calls as below from start_secondary() are called.
>>> Especially the issue is coming
>>>     with init_maintenanc_interrupt(). So I have moved this notifier
>>> before other GIC initialization
>>>     calls and since I move notifier only before GIC initialization
>>> calls it not be a problem.
>>
>>
>> It doesn't explain why it's safe... CPU_STARTING is also used in some place
>> to initialize internal data structure. Are you sure that theses callbacks
>> can be called earlier?
>>
> 
> The below callback is the only callback that is using CPU_STARTING,
> IMO it is only initializing pcpu data.
> 
> static int cpu_credit2_callback(
>     struct notifier_block *nfb, unsigned long action, void *hcpu)
> {
>     unsigned int cpu = (unsigned long)hcpu;
>     int rc = 0;
> 
>     switch ( action )
>     {
>     case CPU_STARTING:
>         csched_cpu_starting(cpu);
>         break;
>     default:
>         break;
>     }
> 
>     return !rc ? NOTIFY_DONE : notifier_from_errno(rc);
> }
> 
> With this patch, notifier is only called just before GIC initialization.
> So should not be a problem. Let me know your opinion? I am
> not very familiar with this piece of code.

I think, the sched credit2 initialization code is relying on the sibling
map (see cpu_to_socket), which is basically a no-op for now on ARM.

You may have to also move setup_cpu_sibling_map(cpuid) earlier.

I don't know enough the scheduler to say it's safe to move earlier.
George any opinion?

The second issue I can see is, a developer wants to add a CPU_STARTING
callback for his shiny driver which will request a PPI. In this case we
will fail because the IRQ desc is not correctly initialized
(init_secondary_IRQ is called after notify_cpu_starting).

Regards,

-- 
Julien Grall

Re: [RFC PATCH v1 01/10] xen/arm: make secondary gic init as notifier call

[George Dunlap]

On Wed, Mar 26, 2014 at 2:41 PM, Julien Grall <julien.grall@linaro.org> wrote:
> Hello Vijay,
>
> (Adding George for the scheduler part)
>
> On 03/26/2014 11:27 AM, Vijay Kilari wrote:
>> Hi Julien,
>>
>>>>>> @@ -283,7 +283,7 @@ void __cpuinit start_secondary(unsigned long
>>>>>> boot_phys_offset,
>>>>>>
>>>>>>       mmu_init_secondary_cpu();
>>>>>>
>>>>>> -    gic_init_secondary_cpu();
>>>>>> +    notify_cpu_starting(cpuid);
>>>>>
>>>>>
>>>>> Can you explain why it's safe to move notify_cpu_starting earlier?
>>>>>
>>>>     When gic registers a notifier with action as CPU_STARTING, I am
>>>> getting a panic
>>>>     from gic driver because notify_cpu_starting() is called after most of
>>>> the GIC
>>>>     initialization calls as below from start_secondary() are called.
>>>> Especially the issue is coming
>>>>     with init_maintenanc_interrupt(). So I have moved this notifier
>>>> before other GIC initialization
>>>>     calls and since I move notifier only before GIC initialization
>>>> calls it not be a problem.
>>>
>>>
>>> It doesn't explain why it's safe... CPU_STARTING is also used in some place
>>> to initialize internal data structure. Are you sure that theses callbacks
>>> can be called earlier?
>>>
>>
>> The below callback is the only callback that is using CPU_STARTING,
>> IMO it is only initializing pcpu data.
>>
>> static int cpu_credit2_callback(
>>     struct notifier_block *nfb, unsigned long action, void *hcpu)
>> {
>>     unsigned int cpu = (unsigned long)hcpu;
>>     int rc = 0;
>>
>>     switch ( action )
>>     {
>>     case CPU_STARTING:
>>         csched_cpu_starting(cpu);
>>         break;
>>     default:
>>         break;
>>     }
>>
>>     return !rc ? NOTIFY_DONE : notifier_from_errno(rc);
>> }
>>
>> With this patch, notifier is only called just before GIC initialization.
>> So should not be a problem. Let me know your opinion? I am
>> not very familiar with this piece of code.
>
> I think, the sched credit2 initialization code is relying on the sibling
> map (see cpu_to_socket), which is basically a no-op for now on ARM.
>
> You may have to also move setup_cpu_sibling_map(cpuid) earlier.
>
> I don't know enough the scheduler to say it's safe to move earlier.
> George any opinion?

Well it definitely breaks the interface of what CPU_STARTING means
defined in cpu.h:

/* CPU_STARTING: CPU nearly online. Runs on new CPU, irqs still disabled. */
#define CPU_STARTING     (0x0003 | NOTIFY_FORWARD)

I'm not sure exactly what you're trying to accomplish by running this
as a notifier instead of just calling it directly, but I think using
CPU_STARTING it not what you want.  In any case, you'd need to change
the definition in generic code, and in the x86 code if that's what you
wanted to do.

 -George

Re: [RFC PATCH v1 01/10] xen/arm: make secondary gic init as notifier call

[Ian Campbell]

On Wed, 2014-03-19 at 19:47 +0530, vijay.kilari@gmail.com wrote:
> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> 
> make gic init for secondary cpus as notifier call
> instead calling directly from secondary boot for
> each cpu.

> This makes secondary gic init generic and runtime.

s/and/at/ perhaps? Otherwise I can't make sense of what you are trying
to say.


> +static struct notifier_block gic_cpu_nb = { 
> +    .notifier_call = gic_init_secondary_cpu,
> +    .priority = 100 
> 
In wondering what 100 was, I notice that the other similar uses have a
"/* Highest priority */" comment, please add one here too.

I notice that the percpu stuff (cpu_percpu_nfb) is also priority == 100,
which makes me suspect that they will run in arbitrary order, which
makes me uncomfortable, not least because at least gic_{cpu,hyp}_init
both touch per-cpu data structures.

Ah, I see the percpu stuff is a CPU_UP_PREPARE notifier, so has already
happened. Good. It does make me wonder if it is wise to do something as
critical as interrupt controller setup in a notifier. Do you think this
is necessary for some reason?

> +};

> +static void gic_smp_init(void)

Like Julien says there isn't really a need for this, but if it were it
should be __init I think.


Ian.

Re: [RFC PATCH v1 01/10] xen/arm: make secondary gic init as notifier call

[Vijay Kilari]

On Fri, Mar 21, 2014 at 10:45 PM, Ian Campbell <Ian.Campbell@citrix.com> wrote:
> On Wed, 2014-03-19 at 19:47 +0530, vijay.kilari@gmail.com wrote:
>> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>>
>> make gic init for secondary cpus as notifier call
>> instead calling directly from secondary boot for
>> each cpu.
>
>> This makes secondary gic init generic and runtime.
>
> s/and/at/ perhaps? Otherwise I can't make sense of what you are trying
> to say.
>
 OK, I will rephrase it.
>
>> +static struct notifier_block gic_cpu_nb = {
>> +    .notifier_call = gic_init_secondary_cpu,
>> +    .priority = 100
>>
> In wondering what 100 was, I notice that the other similar uses have a
> "/* Highest priority */" comment, please add one here too.
>
> I notice that the percpu stuff (cpu_percpu_nfb) is also priority == 100,
> which makes me suspect that they will run in arbitrary order, which
> makes me uncomfortable, not least because at least gic_{cpu,hyp}_init
> both touch per-cpu data structures.
>
> Ah, I see the percpu stuff is a CPU_UP_PREPARE notifier, so has already
> happened. Good. It does make me wonder if it is wise to do something as
> critical as interrupt controller setup in a notifier. Do you think this
> is necessary for some reason?
>
   Yes, I kept it at highest priority thinking that it is critical.
    from the comment in include/xen/cpu.h, CPU_STARTING suits because
    gic_{cpu,hyp}_init should run on new cpu.

      /* CPU_STARTING: CPU nearly online. Runs on new CPU, irqs still
disabled. */

    Is there any difference with Xen compared to kernel? that suggest me to use
    CPU_UP_PREPARE?

    If we don't implement as notifier, then it has to be called from
generic gic code
    through call back as we did for other call ,which is not required
as it is mostly one time called

>> +};
>
>> +static void gic_smp_init(void)
>
> Like Julien says there isn't really a need for this, but if it were it
> should be __init I think.
>
OK
>
> Ian.
>

Re: [RFC PATCH v1 01/10] xen/arm: make secondary gic init as notifier call

[Julien Grall]

Hello Vijay,

On 22/03/14 08:32, Vijay Kilari wrote:
> On Fri, Mar 21, 2014 at 10:45 PM, Ian Campbell <Ian.Campbell@citrix.com> wrote:
>> On Wed, 2014-03-19 at 19:47 +0530, vijay.kilari@gmail.com wrote:
>>> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>>>
>>> make gic init for secondary cpus as notifier call
>>> instead calling directly from secondary boot for
>>> each cpu.
>>
>>> This makes secondary gic init generic and runtime.
>>
>> s/and/at/ perhaps? Otherwise I can't make sense of what you are trying
>> to say.
>>
>   OK, I will rephrase it.
>>
>>> +static struct notifier_block gic_cpu_nb = {
>>> +    .notifier_call = gic_init_secondary_cpu,
>>> +    .priority = 100
>>>
>> In wondering what 100 was, I notice that the other similar uses have a
>> "/* Highest priority */" comment, please add one here too.
>>
>> I notice that the percpu stuff (cpu_percpu_nfb) is also priority == 100,
>> which makes me suspect that they will run in arbitrary order, which
>> makes me uncomfortable, not least because at least gic_{cpu,hyp}_init
>> both touch per-cpu data structures.
>>
>> Ah, I see the percpu stuff is a CPU_UP_PREPARE notifier, so has already
>> happened. Good. It does make me wonder if it is wise to do something as
>> critical as interrupt controller setup in a notifier. Do you think this
>> is necessary for some reason?
>>
>     Yes, I kept it at highest priority thinking that it is critical.
>      from the comment in include/xen/cpu.h, CPU_STARTING suits because
>      gic_{cpu,hyp}_init should run on new cpu.
>
>        /* CPU_STARTING: CPU nearly online. Runs on new CPU, irqs still
> disabled. */
>
>      Is there any difference with Xen compared to kernel? that suggest me to use
>      CPU_UP_PREPARE?

CPU_UP_PREPARE notifiers are called from the boot CPU. Here we want to 
call GIC init on the newly booted CPU.

So CPU_STARTING is fine.

Regards,

-- 
Julien Grall

Re: [RFC PATCH v1 01/10] xen/arm: make secondary gic init as notifier call

[Ian Campbell]

On Sat, 2014-03-22 at 13:54 +0000, Julien Grall wrote:
> Hello Vijay,
> 
> On 22/03/14 08:32, Vijay Kilari wrote:
> > On Fri, Mar 21, 2014 at 10:45 PM, Ian Campbell <Ian.Campbell@citrix.com> wrote:
> >> On Wed, 2014-03-19 at 19:47 +0530, vijay.kilari@gmail.com wrote:
> >>> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> >>>
> >>> make gic init for secondary cpus as notifier call
> >>> instead calling directly from secondary boot for
> >>> each cpu.
> >>
> >>> This makes secondary gic init generic and runtime.
> >>
> >> s/and/at/ perhaps? Otherwise I can't make sense of what you are trying
> >> to say.
> >>
> >   OK, I will rephrase it.
> >>
> >>> +static struct notifier_block gic_cpu_nb = {
> >>> +    .notifier_call = gic_init_secondary_cpu,
> >>> +    .priority = 100
> >>>
> >> In wondering what 100 was, I notice that the other similar uses have a
> >> "/* Highest priority */" comment, please add one here too.
> >>
> >> I notice that the percpu stuff (cpu_percpu_nfb) is also priority == 100,
> >> which makes me suspect that they will run in arbitrary order, which
> >> makes me uncomfortable, not least because at least gic_{cpu,hyp}_init
> >> both touch per-cpu data structures.
> >>
> >> Ah, I see the percpu stuff is a CPU_UP_PREPARE notifier, so has already
> >> happened. Good. It does make me wonder if it is wise to do something as
> >> critical as interrupt controller setup in a notifier. Do you think this
> >> is necessary for some reason?
> >>
> >     Yes, I kept it at highest priority thinking that it is critical.
> >      from the comment in include/xen/cpu.h, CPU_STARTING suits because
> >      gic_{cpu,hyp}_init should run on new cpu.
> >
> >        /* CPU_STARTING: CPU nearly online. Runs on new CPU, irqs still
> > disabled. */
> >
> >      Is there any difference with Xen compared to kernel? that suggest me to use
> >      CPU_UP_PREPARE?
> 
> CPU_UP_PREPARE notifiers are called from the boot CPU. Here we want to 
> call GIC init on the newly booted CPU.
> 
> So CPU_STARTING is fine.

Yeah, thinking about it some more I think this is ok.

Ian.

[RFC PATCH v1 02/10] xen/arm: register mmio handler at runtime

[vijay.kilari]

From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>

mmio handlers are registers at compile time
for drivers like vuart and vgic.
Make mmio handler registered at runtime by
creating linked list of mmio handlers

Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
---
 xen/arch/arm/io.c    |   32 +++++++++++++++++---------
 xen/arch/arm/io.h    |   16 +++++--------
 xen/arch/arm/vgic.c  |   61 ++++++++++++++++++++++++++------------------------
 xen/arch/arm/vuart.c |   39 ++++++++++++++++----------------
 4 files changed, 79 insertions(+), 69 deletions(-)

diff --git a/xen/arch/arm/io.c b/xen/arch/arm/io.c
index a6db00b..d140b29 100644
--- a/xen/arch/arm/io.c
+++ b/xen/arch/arm/io.c
@@ -17,31 +17,41 @@
  */
 
 #include <xen/config.h>
+#include <xen/init.h>
+#include <xen/kernel.h>
 #include <xen/lib.h>
+#include <xen/spinlock.h>
 #include <asm/current.h>
 
 #include "io.h"
 
-static const struct mmio_handler *const mmio_handlers[] =
-{
-    &vgic_distr_mmio_handler,
-    &vuart_mmio_handler,
-};
-#define MMIO_HANDLER_NR ARRAY_SIZE(mmio_handlers)
+LIST_HEAD(handle_head);
+static DEFINE_SPINLOCK(handler_lock);
 
 int handle_mmio(mmio_info_t *info)
 {
     struct vcpu *v = current;
-    int i;
+    struct list_head *pos;
+    struct mmio_handler *mmio_handle;
 
-    for ( i = 0; i < MMIO_HANDLER_NR; i++ )
-        if ( mmio_handlers[i]->check_handler(v, info->gpa) )
+    list_for_each(pos, &handle_head) {
+        mmio_handle = list_entry(pos, struct mmio_handler, handle_list);
+        if ( mmio_handle->check_handler(v, info->gpa) )
             return info->dabt.write ?
-                mmio_handlers[i]->write_handler(v, info) :
-                mmio_handlers[i]->read_handler(v, info);
+                mmio_handle->write_handler(v, info) :
+                mmio_handle->read_handler(v, info);
+    }
 
     return 0;
 }
+
+void register_mmio_handler(struct mmio_handler * handle)
+{
+    spin_lock(&handler_lock);
+    list_add(&handle->handle_list, &handle_head);
+    spin_unlock(&handler_lock);
+}
+
 /*
  * Local variables:
  * mode: C
diff --git a/xen/arch/arm/io.h b/xen/arch/arm/io.h
index 8d252c0..99cd7c3 100644
--- a/xen/arch/arm/io.h
+++ b/xen/arch/arm/io.h
@@ -22,6 +22,7 @@
 #include <xen/lib.h>
 #include <asm/processor.h>
 #include <asm/regs.h>
+#include <xen/list.h>
 
 typedef struct
 {
@@ -30,20 +31,15 @@ typedef struct
     paddr_t gpa;
 } mmio_info_t;
 
-typedef int (*mmio_read_t)(struct vcpu *v, mmio_info_t *info);
-typedef int (*mmio_write_t)(struct vcpu *v, mmio_info_t *info);
-typedef int (*mmio_check_t)(struct vcpu *v, paddr_t addr);
-
 struct mmio_handler {
-    mmio_check_t check_handler;
-    mmio_read_t read_handler;
-    mmio_write_t write_handler;
+    int (*read_handler)(struct vcpu *v, mmio_info_t *info);
+    int (*write_handler)(struct vcpu *v, mmio_info_t *info);
+    int (*check_handler)(struct vcpu *v, paddr_t addr);
+    struct list_head handle_list;
 };
 
-extern const struct mmio_handler vgic_distr_mmio_handler;
-extern const struct mmio_handler vuart_mmio_handler;
-
 extern int handle_mmio(mmio_info_t *info);
+void register_mmio_handler(struct mmio_handler * handle);
 
 #endif
 
diff --git a/xen/arch/arm/vgic.c b/xen/arch/arm/vgic.c
index 553411d..d2a13fb 100644
--- a/xen/arch/arm/vgic.c
+++ b/xen/arch/arm/vgic.c
@@ -73,34 +73,6 @@ static struct vgic_irq_rank *vgic_irq_rank(struct vcpu *v, int b, int n)
         return NULL;
 }
 
-int domain_vgic_init(struct domain *d)
-{
-    int i;
-
-    d->arch.vgic.ctlr = 0;
-
-    /* Currently nr_lines in vgic and gic doesn't have the same meanings
-     * Here nr_lines = number of SPIs
-     */
-    if ( d->domain_id == 0 )
-        d->arch.vgic.nr_lines = gic_number_lines() - 32;
-    else
-        d->arch.vgic.nr_lines = 0; /* We don't need SPIs for the guest */
-
-    d->arch.vgic.shared_irqs =
-        xzalloc_array(struct vgic_irq_rank, DOMAIN_NR_RANKS(d));
-    d->arch.vgic.pending_irqs =
-        xzalloc_array(struct pending_irq, d->arch.vgic.nr_lines);
-    for (i=0; i<d->arch.vgic.nr_lines; i++)
-    {
-        INIT_LIST_HEAD(&d->arch.vgic.pending_irqs[i].inflight);
-        INIT_LIST_HEAD(&d->arch.vgic.pending_irqs[i].lr_queue);
-    }
-    for (i=0; i<DOMAIN_NR_RANKS(d); i++)
-        spin_lock_init(&d->arch.vgic.shared_irqs[i].lock);
-    return 0;
-}
-
 void domain_vgic_free(struct domain *d)
 {
     xfree(d->arch.vgic.shared_irqs);
@@ -655,12 +627,43 @@ static int vgic_distr_mmio_check(struct vcpu *v, paddr_t addr)
     return (addr >= (d->arch.vgic.dbase)) && (addr < (d->arch.vgic.dbase + PAGE_SIZE));
 }
 
-const struct mmio_handler vgic_distr_mmio_handler = {
+static struct mmio_handler vgic_distr_mmio_handler = {
     .check_handler = vgic_distr_mmio_check,
     .read_handler  = vgic_distr_mmio_read,
     .write_handler = vgic_distr_mmio_write,
 };
 
+int domain_vgic_init(struct domain *d)
+{
+    int i;
+
+    d->arch.vgic.ctlr = 0;
+
+    /* Currently nr_lines in vgic and gic doesn't have the same meanings
+     * Here nr_lines = number of SPIs
+     */
+    if ( d->domain_id == 0 )
+        d->arch.vgic.nr_lines = gic_number_lines() - 32;
+    else
+        d->arch.vgic.nr_lines = 0; /* We don't need SPIs for the guest */
+
+    d->arch.vgic.shared_irqs =
+        xzalloc_array(struct vgic_irq_rank, DOMAIN_NR_RANKS(d));
+    d->arch.vgic.pending_irqs =
+        xzalloc_array(struct pending_irq, d->arch.vgic.nr_lines);
+    for (i=0; i<d->arch.vgic.nr_lines; i++)
+    {
+        INIT_LIST_HEAD(&d->arch.vgic.pending_irqs[i].inflight);
+        INIT_LIST_HEAD(&d->arch.vgic.pending_irqs[i].lr_queue);
+    }
+    for (i=0; i<DOMAIN_NR_RANKS(d); i++)
+        spin_lock_init(&d->arch.vgic.shared_irqs[i].lock);
+
+    register_mmio_handler(&vgic_distr_mmio_handler);
+    return 0;
+}
+
+
 struct pending_irq *irq_to_pending(struct vcpu *v, unsigned int irq)
 {
     struct pending_irq *n;
diff --git a/xen/arch/arm/vuart.c b/xen/arch/arm/vuart.c
index b9d3ced..c237d71 100644
--- a/xen/arch/arm/vuart.c
+++ b/xen/arch/arm/vuart.c
@@ -44,24 +44,6 @@
 
 #define domain_has_vuart(d) ((d)->arch.vuart.info != NULL)
 
-int domain_vuart_init(struct domain *d)
-{
-    ASSERT( !d->domain_id );
-
-    d->arch.vuart.info = serial_vuart_info(SERHND_DTUART);
-    if ( !d->arch.vuart.info )
-        return 0;
-
-    spin_lock_init(&d->arch.vuart.lock);
-    d->arch.vuart.idx = 0;
-
-    d->arch.vuart.buf = xzalloc_array(char, VUART_BUF_SIZE);
-    if ( !d->arch.vuart.buf )
-        return -ENOMEM;
-
-    return 0;
-}
-
 void domain_vuart_free(struct domain *d)
 {
     if ( !domain_has_vuart(d) )
@@ -133,12 +115,31 @@ static int vuart_mmio_write(struct vcpu *v, mmio_info_t *info)
     return 1;
 }
 
-const struct mmio_handler vuart_mmio_handler = {
+static struct mmio_handler vuart_mmio_handler = {
     .check_handler = vuart_mmio_check,
     .read_handler  = vuart_mmio_read,
     .write_handler = vuart_mmio_write,
 };
 
+int domain_vuart_init(struct domain *d)
+{
+    ASSERT( !d->domain_id );
+
+    d->arch.vuart.info = serial_vuart_info(SERHND_DTUART);
+    if ( !d->arch.vuart.info )
+        return 0;
+
+    spin_lock_init(&d->arch.vuart.lock);
+    d->arch.vuart.idx = 0;
+
+    d->arch.vuart.buf = xzalloc_array(char, VUART_BUF_SIZE);
+    if ( !d->arch.vuart.buf )
+        return -ENOMEM;
+
+    register_mmio_handler(&vuart_mmio_handler);
+    return 0;
+}
+
 /*
  * Local variables:
  * mode: C
-- 
1.7.9.5

Re: [RFC PATCH v1 02/10] xen/arm: register mmio handler at runtime

[Julien Grall]

Hello Vijay,

(Adding Andrii who is working on a similar patch).

Thanks you for the patch.

On 03/19/2014 02:17 PM, vijay.kilari@gmail.com wrote:
> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> 
> mmio handlers are registers at compile time
> for drivers like vuart and vgic.
> Make mmio handler registered at runtime by
> creating linked list of mmio handlers
> 
> Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> ---
>  xen/arch/arm/io.c    |   32 +++++++++++++++++---------
>  xen/arch/arm/io.h    |   16 +++++--------
>  xen/arch/arm/vgic.c  |   61 ++++++++++++++++++++++++++------------------------
>  xen/arch/arm/vuart.c |   39 ++++++++++++++++----------------
>  4 files changed, 79 insertions(+), 69 deletions(-)
> 
> diff --git a/xen/arch/arm/io.c b/xen/arch/arm/io.c
> index a6db00b..d140b29 100644
> --- a/xen/arch/arm/io.c
> +++ b/xen/arch/arm/io.c
> @@ -17,31 +17,41 @@
>   */
>  
>  #include <xen/config.h>
> +#include <xen/init.h>
> +#include <xen/kernel.h>
>  #include <xen/lib.h>
> +#include <xen/spinlock.h>
>  #include <asm/current.h>
>  
>  #include "io.h"
>  
> -static const struct mmio_handler *const mmio_handlers[] =
> -{
> -    &vgic_distr_mmio_handler,
> -    &vuart_mmio_handler,
> -};
> -#define MMIO_HANDLER_NR ARRAY_SIZE(mmio_handlers)
> +LIST_HEAD(handle_head);
> +static DEFINE_SPINLOCK(handler_lock);

As you change the code, I would prefer a per domain list IO handler. So
we can easily handle GICv2 guest on GICv3 host.

This list would only contains handlers that will be effectively used for
the domain.

>  int handle_mmio(mmio_info_t *info)
>  {
>      struct vcpu *v = current;
> -    int i;
> +    struct list_head *pos;
> +    struct mmio_handler *mmio_handle;
>  
> -    for ( i = 0; i < MMIO_HANDLER_NR; i++ )
> -        if ( mmio_handlers[i]->check_handler(v, info->gpa) )
> +    list_for_each(pos, &handle_head) {
> +        mmio_handle = list_entry(pos, struct mmio_handler, handle_list);

You can use list_for_each_entry here.

> +        if ( mmio_handle->check_handler(v, info->gpa) )
>              return info->dabt.write ?
> -                mmio_handlers[i]->write_handler(v, info) :
> -                mmio_handlers[i]->read_handler(v, info);
> +                mmio_handle->write_handler(v, info) :
> +                mmio_handle->read_handler(v, info);
> +    }
>  
>      return 0;
>  }
> +
> +void register_mmio_handler(struct mmio_handler * handle)
> +{
> +    spin_lock(&handler_lock);

Why do you take the lock here and not in handle_mmio?

> +    list_add(&handle->handle_list, &handle_head);
> +    spin_unlock(&handler_lock);
> +}
> +
>  /*
>   * Local variables:
>   * mode: C
> diff --git a/xen/arch/arm/io.h b/xen/arch/arm/io.h
> index 8d252c0..99cd7c3 100644
> --- a/xen/arch/arm/io.h
> +++ b/xen/arch/arm/io.h
> @@ -22,6 +22,7 @@
>  #include <xen/lib.h>
>  #include <asm/processor.h>
>  #include <asm/regs.h>
> +#include <xen/list.h>
>  
>  typedef struct
>  {
> @@ -30,20 +31,15 @@ typedef struct
>      paddr_t gpa;
>  } mmio_info_t;
>  
> -typedef int (*mmio_read_t)(struct vcpu *v, mmio_info_t *info);
> -typedef int (*mmio_write_t)(struct vcpu *v, mmio_info_t *info);
> -typedef int (*mmio_check_t)(struct vcpu *v, paddr_t addr);
> -

Why did you remove the typedef? It was useful for the code comprehension.

>  struct mmio_handler {
> -    mmio_check_t check_handler;
> -    mmio_read_t read_handler;
> -    mmio_write_t write_handler;
> +    int (*read_handler)(struct vcpu *v, mmio_info_t *info);
> +    int (*write_handler)(struct vcpu *v, mmio_info_t *info);
> +    int (*check_handler)(struct vcpu *v, paddr_t addr);

If we are going to a per domain list IO, I would remove check_handler
and replacing by:

paddr_t addr;
paddr_t size;

> +    struct list_head handle_list;

On a previous mail (see
http://www.gossamer-threads.com/lists/xen/devel/317457#317457) I said
that a list would be better ... but after thinking we can define a fixed
array of 16 cells. It would be enough for now.

You can see an example in arch/x86/hvm/intercept.c

>  };
>  
> -extern const struct mmio_handler vgic_distr_mmio_handler;
> -extern const struct mmio_handler vuart_mmio_handler;
> -
>  extern int handle_mmio(mmio_info_t *info);
> +void register_mmio_handler(struct mmio_handler * handle);
>  
>  #endif
>  
> diff --git a/xen/arch/arm/vgic.c b/xen/arch/arm/vgic.c
> index 553411d..d2a13fb 100644
> --- a/xen/arch/arm/vgic.c
> +++ b/xen/arch/arm/vgic.c
> @@ -73,34 +73,6 @@ static struct vgic_irq_rank *vgic_irq_rank(struct vcpu *v, int b, int n)
>          return NULL;
>  }
>  
> -int domain_vgic_init(struct domain *d)
> -{
> -    int i;
> -
> -    d->arch.vgic.ctlr = 0;
> -
> -    /* Currently nr_lines in vgic and gic doesn't have the same meanings
> -     * Here nr_lines = number of SPIs
> -     */
> -    if ( d->domain_id == 0 )
> -        d->arch.vgic.nr_lines = gic_number_lines() - 32;
> -    else
> -        d->arch.vgic.nr_lines = 0; /* We don't need SPIs for the guest */
> -
> -    d->arch.vgic.shared_irqs =
> -        xzalloc_array(struct vgic_irq_rank, DOMAIN_NR_RANKS(d));
> -    d->arch.vgic.pending_irqs =
> -        xzalloc_array(struct pending_irq, d->arch.vgic.nr_lines);
> -    for (i=0; i<d->arch.vgic.nr_lines; i++)
> -    {
> -        INIT_LIST_HEAD(&d->arch.vgic.pending_irqs[i].inflight);
> -        INIT_LIST_HEAD(&d->arch.vgic.pending_irqs[i].lr_queue);
> -    }
> -    for (i=0; i<DOMAIN_NR_RANKS(d); i++)
> -        spin_lock_init(&d->arch.vgic.shared_irqs[i].lock);
> -    return 0;
> -}
> -

I would predefine vgic_distr_mmio_handler early rather moving the whole
function. It's easier to understand the modification in this patch.

[..]

>  struct pending_irq *irq_to_pending(struct vcpu *v, unsigned int irq)
>  {
>      struct pending_irq *n;
> diff --git a/xen/arch/arm/vuart.c b/xen/arch/arm/vuart.c
> index b9d3ced..c237d71 100644
> --- a/xen/arch/arm/vuart.c
> +++ b/xen/arch/arm/vuart.c
> @@ -44,24 +44,6 @@
>  
>  #define domain_has_vuart(d) ((d)->arch.vuart.info != NULL)
>  
> -int domain_vuart_init(struct domain *d)
> -{
> -    ASSERT( !d->domain_id );
> -
> -    d->arch.vuart.info = serial_vuart_info(SERHND_DTUART);
> -    if ( !d->arch.vuart.info )
> -        return 0;
> -
> -    spin_lock_init(&d->arch.vuart.lock);
> -    d->arch.vuart.idx = 0;
> -
> -    d->arch.vuart.buf = xzalloc_array(char, VUART_BUF_SIZE);
> -    if ( !d->arch.vuart.buf )
> -        return -ENOMEM;
> -
> -    return 0;
> -}
> -

Same remark as domain_vgic_init.

Regards,

-- 
Julien Grall

Re: [RFC PATCH v1 02/10] xen/arm: register mmio handler at runtime

[Andrii Tseglytskyi]

Hi,

On Thu, Mar 20, 2014 at 3:18 PM, Julien Grall <julien.grall@linaro.org> wrote:
> Hello Vijay,
>
> (Adding Andrii who is working on a similar patch).
>

Thank you for pointing me to this thread. Some time ago I posted the
idea of this solution, but my work with it is a bit postponed now.
Hope to resume it asap.
The reason why I started working with this - is a proper OMAP IOMMU
handling, which needs runtime IO traps for sure.
vgic and vuart as standalone modules do not need this - they need boot
time IO mem configuration only. But if you add OMAP IOMMU (hope I'll
post it asap), you'll need to update vgic and vuart too.

> Thanks you for the patch.
>
> On 03/19/2014 02:17 PM, vijay.kilari@gmail.com wrote:
>> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>>
>> mmio handlers are registers at compile time
>> for drivers like vuart and vgic.
>> Make mmio handler registered at runtime by
>> creating linked list of mmio handlers
>>
>> Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>> ---
>>  xen/arch/arm/io.c    |   32 +++++++++++++++++---------
>>  xen/arch/arm/io.h    |   16 +++++--------
>>  xen/arch/arm/vgic.c  |   61 ++++++++++++++++++++++++++------------------------
>>  xen/arch/arm/vuart.c |   39 ++++++++++++++++----------------
>>  4 files changed, 79 insertions(+), 69 deletions(-)
>>
>> diff --git a/xen/arch/arm/io.c b/xen/arch/arm/io.c
>> index a6db00b..d140b29 100644
>> --- a/xen/arch/arm/io.c
>> +++ b/xen/arch/arm/io.c
>> @@ -17,31 +17,41 @@
>>   */
>>
>>  #include <xen/config.h>
>> +#include <xen/init.h>
>> +#include <xen/kernel.h>
>>  #include <xen/lib.h>
>> +#include <xen/spinlock.h>
>>  #include <asm/current.h>
>>
>>  #include "io.h"
>>
>> -static const struct mmio_handler *const mmio_handlers[] =
>> -{
>> -    &vgic_distr_mmio_handler,
>> -    &vuart_mmio_handler,
>> -};
>> -#define MMIO_HANDLER_NR ARRAY_SIZE(mmio_handlers)
>> +LIST_HEAD(handle_head);

static LIST_HEAD

>> +static DEFINE_SPINLOCK(handler_lock);
>
> As you change the code, I would prefer a per domain list IO handler. So
> we can easily handle GICv2 guest on GICv3 host.
>
> This list would only contains handlers that will be effectively used for
> the domain.
>
>>  int handle_mmio(mmio_info_t *info)
>>  {
>>      struct vcpu *v = current;
>> -    int i;
>> +    struct list_head *pos;
>> +    struct mmio_handler *mmio_handle;
>>
>> -    for ( i = 0; i < MMIO_HANDLER_NR; i++ )
>> -        if ( mmio_handlers[i]->check_handler(v, info->gpa) )

Must be locked with irqsave lock.

>> +    list_for_each(pos, &handle_head) {
>> +        mmio_handle = list_entry(pos, struct mmio_handler, handle_list);
>
> You can use list_for_each_entry here.
>

Right. Should be list_for_each_entry

>> +        if ( mmio_handle->check_handler(v, info->gpa) )
>>              return info->dabt.write ?
>> -                mmio_handlers[i]->write_handler(v, info) :
>> -                mmio_handlers[i]->read_handler(v, info);
>> +                mmio_handle->write_handler(v, info) :
>> +                mmio_handle->read_handler(v, info);
>> +    }
>>
>>      return 0;
>>  }
>> +
>> +void register_mmio_handler(struct mmio_handler * handle)
>> +{
>> +    spin_lock(&handler_lock);
>
> Why do you take the lock here and not in handle_mmio?
>

I planned that these functions will be used at runtime and not only
boot time. So spinlocking here is needed for sure. From IOMMU point of
view (which I working with) - this code may be called at any time at
any CPU simultaneously. The only issue here - lock should be irqsave.

>> +    list_add(&handle->handle_list, &handle_head);
>> +    spin_unlock(&handler_lock);
>> +}
>> +
>>  /*
>>   * Local variables:
>>   * mode: C
>> diff --git a/xen/arch/arm/io.h b/xen/arch/arm/io.h
>> index 8d252c0..99cd7c3 100644
>> --- a/xen/arch/arm/io.h
>> +++ b/xen/arch/arm/io.h
>> @@ -22,6 +22,7 @@
>>  #include <xen/lib.h>
>>  #include <asm/processor.h>
>>  #include <asm/regs.h>
>> +#include <xen/list.h>
>>
>>  typedef struct
>>  {
>> @@ -30,20 +31,15 @@ typedef struct
>>      paddr_t gpa;
>>  } mmio_info_t;
>>
>> -typedef int (*mmio_read_t)(struct vcpu *v, mmio_info_t *info);
>> -typedef int (*mmio_write_t)(struct vcpu *v, mmio_info_t *info);
>> -typedef int (*mmio_check_t)(struct vcpu *v, paddr_t addr);
>> -
>
> Why did you remove the typedef? It was useful for the code comprehension.
>
>>  struct mmio_handler {
>> -    mmio_check_t check_handler;
>> -    mmio_read_t read_handler;
>> -    mmio_write_t write_handler;
>> +    int (*read_handler)(struct vcpu *v, mmio_info_t *info);
>> +    int (*write_handler)(struct vcpu *v, mmio_info_t *info);
>> +    int (*check_handler)(struct vcpu *v, paddr_t addr);
>
> If we are going to a per domain list IO, I would remove check_handler
> and replacing by:
>
> paddr_t addr;
> paddr_t size;
>
>> +    struct list_head handle_list;
>
> On a previous mail (see
> http://www.gossamer-threads.com/lists/xen/devel/317457#317457) I said
> that a list would be better ... but after thinking we can define a fixed
> array of 16 cells. It would be enough for now.
>
> You can see an example in arch/x86/hvm/intercept.c
>
>>  };
>>
>> -extern const struct mmio_handler vgic_distr_mmio_handler;
>> -extern const struct mmio_handler vuart_mmio_handler;
>> -
>>  extern int handle_mmio(mmio_info_t *info);
>> +void register_mmio_handler(struct mmio_handler * handle);
>>
>>  #endif
>>
>> diff --git a/xen/arch/arm/vgic.c b/xen/arch/arm/vgic.c
>> index 553411d..d2a13fb 100644
>> --- a/xen/arch/arm/vgic.c
>> +++ b/xen/arch/arm/vgic.c
>> @@ -73,34 +73,6 @@ static struct vgic_irq_rank *vgic_irq_rank(struct vcpu *v, int b, int n)
>>          return NULL;
>>  }
>>
>> -int domain_vgic_init(struct domain *d)
>> -{
>> -    int i;
>> -
>> -    d->arch.vgic.ctlr = 0;
>> -
>> -    /* Currently nr_lines in vgic and gic doesn't have the same meanings
>> -     * Here nr_lines = number of SPIs
>> -     */
>> -    if ( d->domain_id == 0 )
>> -        d->arch.vgic.nr_lines = gic_number_lines() - 32;
>> -    else
>> -        d->arch.vgic.nr_lines = 0; /* We don't need SPIs for the guest */
>> -
>> -    d->arch.vgic.shared_irqs =
>> -        xzalloc_array(struct vgic_irq_rank, DOMAIN_NR_RANKS(d));
>> -    d->arch.vgic.pending_irqs =
>> -        xzalloc_array(struct pending_irq, d->arch.vgic.nr_lines);
>> -    for (i=0; i<d->arch.vgic.nr_lines; i++)
>> -    {
>> -        INIT_LIST_HEAD(&d->arch.vgic.pending_irqs[i].inflight);
>> -        INIT_LIST_HEAD(&d->arch.vgic.pending_irqs[i].lr_queue);
>> -    }
>> -    for (i=0; i<DOMAIN_NR_RANKS(d); i++)
>> -        spin_lock_init(&d->arch.vgic.shared_irqs[i].lock);
>> -    return 0;
>> -}
>> -
>
> I would predefine vgic_distr_mmio_handler early rather moving the whole
> function. It's easier to understand the modification in this patch.
>
> [..]
>
>>  struct pending_irq *irq_to_pending(struct vcpu *v, unsigned int irq)
>>  {
>>      struct pending_irq *n;
>> diff --git a/xen/arch/arm/vuart.c b/xen/arch/arm/vuart.c
>> index b9d3ced..c237d71 100644
>> --- a/xen/arch/arm/vuart.c
>> +++ b/xen/arch/arm/vuart.c
>> @@ -44,24 +44,6 @@
>>
>>  #define domain_has_vuart(d) ((d)->arch.vuart.info != NULL)
>>
>> -int domain_vuart_init(struct domain *d)
>> -{
>> -    ASSERT( !d->domain_id );
>> -
>> -    d->arch.vuart.info = serial_vuart_info(SERHND_DTUART);
>> -    if ( !d->arch.vuart.info )
>> -        return 0;
>> -
>> -    spin_lock_init(&d->arch.vuart.lock);
>> -    d->arch.vuart.idx = 0;
>> -
>> -    d->arch.vuart.buf = xzalloc_array(char, VUART_BUF_SIZE);
>> -    if ( !d->arch.vuart.buf )
>> -        return -ENOMEM;
>> -
>> -    return 0;
>> -}
>> -
>
> Same remark as domain_vgic_init.
>
> Regards,
>
> --
> Julien Grall


Regards,
Andrii


-- 

Andrii Tseglytskyi | Embedded Dev
GlobalLogic
www.globallogic.com

Re: [RFC PATCH v1 02/10] xen/arm: register mmio handler at runtime

[Ian Campbell]

On Wed, 2014-03-19 at 19:47 +0530, vijay.kilari@gmail.com wrote:
> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> 
> mmio handlers are registers at compile time
> for drivers like vuart and vgic.
> Make mmio handler registered at runtime by
> creating linked list of mmio handlers

I'm not convinced of the need for this, certainly the vgic side can just
demux into v2 or v3 as necessary.

Ian.

Re: [RFC PATCH v1 02/10] xen/arm: register mmio handler at runtime

[Julien Grall]

On 03/21/2014 05:17 PM, Ian Campbell wrote:
> On Wed, 2014-03-19 at 19:47 +0530, vijay.kilari@gmail.com wrote:
>> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>>
>> mmio handlers are registers at compile time
>> for drivers like vuart and vgic.
>> Make mmio handler registered at runtime by
>> creating linked list of mmio handlers
> 
> I'm not convinced of the need for this, certainly the vgic side can just
> demux into v2 or v3 as necessary.

Demux the code just add an indirection. We could have a list of mmio
handler per domain and rely on it to call the right handler. A bit like x86.

Regards,

-- 
Julien Grall

Re: [RFC PATCH v1 02/10] xen/arm: register mmio handler at runtime

[Vijay Kilari]

Hi Julien,


On Fri, Mar 21, 2014 at 10:53 PM, Julien Grall <julien.grall@linaro.org> wrote:
> On 03/21/2014 05:17 PM, Ian Campbell wrote:
>> On Wed, 2014-03-19 at 19:47 +0530, vijay.kilari@gmail.com wrote:
>>> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>>>
>>> mmio handlers are registers at compile time
>>> for drivers like vuart and vgic.
>>> Make mmio handler registered at runtime by
>>> creating linked list of mmio handlers
>>
>> I'm not convinced of the need for this, certainly the vgic side can just
>> demux into v2 or v3 as necessary.
>
> Demux the code just add an indirection. We could have a list of mmio
> handler per domain and rely on it to call the right handler. A bit like x86.
>
 Until Andrii adds IOMMU  handling should keep this patch? and adopt
to it later?
  Also vgic & vuart are not per domain specific

> Regards,
>
> --
> Julien Grall

Re: [RFC PATCH v1 02/10] xen/arm: register mmio handler at runtime

[Julien Grall]

On 03/26/2014 12:29 PM, Vijay Kilari wrote:
> Hi Julien,
> 
> 
> On Fri, Mar 21, 2014 at 10:53 PM, Julien Grall <julien.grall@linaro.org> wrote:
>> On 03/21/2014 05:17 PM, Ian Campbell wrote:
>>> On Wed, 2014-03-19 at 19:47 +0530, vijay.kilari@gmail.com wrote:
>>>> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>>>>
>>>> mmio handlers are registers at compile time
>>>> for drivers like vuart and vgic.
>>>> Make mmio handler registered at runtime by
>>>> creating linked list of mmio handlers
>>>
>>> I'm not convinced of the need for this, certainly the vgic side can just
>>> demux into v2 or v3 as necessary.
>>
>> Demux the code just add an indirection. We could have a list of mmio
>> handler per domain and rely on it to call the right handler. A bit like x86.
>>
>  Until Andrii adds IOMMU  handling should keep this patch? and adopt
> to it later?

I'm not sure to understand. IHMO, it doesn't sound right to upstream a
patch that we know it will be superseded in a couple of months. Why
can't you rework your patch to have it in good shape now?

>   Also vgic & vuart are not per domain specific

The vuart is per domain specific. Only dom0 is using the callback.

Regards,

-- 
Julien Grall

Re: [RFC PATCH v1 02/10] xen/arm: register mmio handler at runtime

[Vijay Kilari]

On Wed, Mar 26, 2014 at 8:17 PM, Julien Grall <julien.grall@linaro.org> wrote:
> On 03/26/2014 12:29 PM, Vijay Kilari wrote:
>> Hi Julien,
>>
>>
>> On Fri, Mar 21, 2014 at 10:53 PM, Julien Grall <julien.grall@linaro.org> wrote:
>>> On 03/21/2014 05:17 PM, Ian Campbell wrote:
>>>> On Wed, 2014-03-19 at 19:47 +0530, vijay.kilari@gmail.com wrote:
>>>>> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>>>>>
>>>>> mmio handlers are registers at compile time
>>>>> for drivers like vuart and vgic.
>>>>> Make mmio handler registered at runtime by
>>>>> creating linked list of mmio handlers
>>>>
>>>> I'm not convinced of the need for this, certainly the vgic side can just
>>>> demux into v2 or v3 as necessary.
>>>
>>> Demux the code just add an indirection. We could have a list of mmio
>>> handler per domain and rely on it to call the right handler. A bit like x86.
>>>
>>  Until Andrii adds IOMMU  handling should keep this patch? and adopt
>> to it later?
>
> I'm not sure to understand. IHMO, it doesn't sound right to upstream a
> patch that we know it will be superseded in a couple of months. Why
> can't you rework your patch to have it in good shape now?
>
 I assume that Andii's patch will be required to make mmio handlers
per domain. If not let me know references in x86 that I can make it
per domain specific mmio handlers.

I was more inclined to go with this patch for now because it helps to
go ahead with GICv3 and anything beyond this can be managed separately.
Please let me know you opinion.

>>   Also vgic & vuart are not per domain specific
>
> The vuart is per domain specific. Only dom0 is using the callback.
>
> Regards,
>
> --
> Julien Grall

Re: [RFC PATCH v1 02/10] xen/arm: register mmio handler at runtime

[Julien Grall]

On 03/27/2014 05:40 AM, Vijay Kilari wrote:
> On Wed, Mar 26, 2014 at 8:17 PM, Julien Grall <julien.grall@linaro.org> wrote:
>> On 03/26/2014 12:29 PM, Vijay Kilari wrote:
>>> Hi Julien,
>>>
>>>
>>> On Fri, Mar 21, 2014 at 10:53 PM, Julien Grall <julien.grall@linaro.org> wrote:
>>>> On 03/21/2014 05:17 PM, Ian Campbell wrote:
>>>>> On Wed, 2014-03-19 at 19:47 +0530, vijay.kilari@gmail.com wrote:
>>>>>> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>>>>>>
>>>>>> mmio handlers are registers at compile time
>>>>>> for drivers like vuart and vgic.
>>>>>> Make mmio handler registered at runtime by
>>>>>> creating linked list of mmio handlers
>>>>>
>>>>> I'm not convinced of the need for this, certainly the vgic side can just
>>>>> demux into v2 or v3 as necessary.
>>>>
>>>> Demux the code just add an indirection. We could have a list of mmio
>>>> handler per domain and rely on it to call the right handler. A bit like x86.
>>>>
>>>  Until Andrii adds IOMMU  handling should keep this patch? and adopt
>>> to it later?
>>
>> I'm not sure to understand. IHMO, it doesn't sound right to upstream a
>> patch that we know it will be superseded in a couple of months. Why
>> can't you rework your patch to have it in good shape now?
>>
>  I assume that Andii's patch will be required to make mmio handlers
> per domain. If not let me know references in x86 that I can make it
> per domain specific mmio handlers.

I have already pointed out to the x86 code few mails before.

You can look at xen/arch/x86/intercept.c.

> 
> I was more inclined to go with this patch for now because it helps to
> go ahead with GICv3 and anything beyond this can be managed separately.
> Please let me know you opinion.

As I said on the email you answered, I'm not in favor to allow a this
patch that we know it will superseded in less than a couple of months.

It won't take you longer time than addressing the current requests on
this patch.

In last resort, you can use the demux solution from Ian. I don't agree
with it but it's better than upstreaming an half-feature for no valid
reason.

Regards,

-- 
Julien Grall

Re: [RFC PATCH v1 02/10] xen/arm: register mmio handler at runtime

[Vijay Kilari]

Hi Julien,

On Thu, Mar 27, 2014 at 8:32 PM, Julien Grall <julien.grall@linaro.org> wrote:
> On 03/27/2014 05:40 AM, Vijay Kilari wrote:
>> On Wed, Mar 26, 2014 at 8:17 PM, Julien Grall <julien.grall@linaro.org> wrote:
>>> On 03/26/2014 12:29 PM, Vijay Kilari wrote:
>>>> Hi Julien,
>>>>
>>>>
>>>> On Fri, Mar 21, 2014 at 10:53 PM, Julien Grall <julien.grall@linaro.org> wrote:
>>>>> On 03/21/2014 05:17 PM, Ian Campbell wrote:
>>>>>> On Wed, 2014-03-19 at 19:47 +0530, vijay.kilari@gmail.com wrote:
>>>>>>> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>>>>>>>
>>>>>>> mmio handlers are registers at compile time
>>>>>>> for drivers like vuart and vgic.
>>>>>>> Make mmio handler registered at runtime by
>>>>>>> creating linked list of mmio handlers
>>>>>>
>>>>>> I'm not convinced of the need for this, certainly the vgic side can just
>>>>>> demux into v2 or v3 as necessary.
>>>>>
>>>>> Demux the code just add an indirection. We could have a list of mmio
>>>>> handler per domain and rely on it to call the right handler. A bit like x86.
>>>>>
>>>>  Until Andrii adds IOMMU  handling should keep this patch? and adopt
>>>> to it later?
>>>
>>> I'm not sure to understand. IHMO, it doesn't sound right to upstream a
>>> patch that we know it will be superseded in a couple of months. Why
>>> can't you rework your patch to have it in good shape now?
>>>
>>  I assume that Andii's patch will be required to make mmio handlers
>> per domain. If not let me know references in x86 that I can make it
>> per domain specific mmio handlers.
>
> I have already pointed out to the x86 code few mails before.
>
> You can look at xen/arch/x86/intercept.c.

   From the x86/hvm/intercept.c file, the mmio handling of x86 is
similar to existing arm mmio handling in arm/io.c file, where the
handlers are registered statically.

  I understand that only requirement is to make vgic as domain specific
similar to vuart. Is that what you expect. Can you please make it clear?

 IMO, the io handling in x86/hvm/intercept.c is not required for arm.

>
>>
>> I was more inclined to go with this patch for now because it helps to
>> go ahead with GICv3 and anything beyond this can be managed separately.
>> Please let me know you opinion.
>
> As I said on the email you answered, I'm not in favor to allow a this
> patch that we know it will superseded in less than a couple of months.
>
> It won't take you longer time than addressing the current requests on
> this patch.
>
> In last resort, you can use the demux solution from Ian. I don't agree
> with it but it's better than upstreaming an half-feature for no valid
> reason.
>
> Regards,
>
> --
> Julien Grall

Re: [RFC PATCH v1 02/10] xen/arm: register mmio handler at runtime

[Julien Grall]

On 04/01/2014 10:34 AM, Vijay Kilari wrote:
> Hi Julien,

Hello Vijay,

> On Thu, Mar 27, 2014 at 8:32 PM, Julien Grall <julien.grall@linaro.org> wrote:
>> On 03/27/2014 05:40 AM, Vijay Kilari wrote:
>>> On Wed, Mar 26, 2014 at 8:17 PM, Julien Grall <julien.grall@linaro.org> wrote:
>>>> On 03/26/2014 12:29 PM, Vijay Kilari wrote:
>>>>> Hi Julien,
>>>>>
>>>>>
>>>>> On Fri, Mar 21, 2014 at 10:53 PM, Julien Grall <julien.grall@linaro.org> wrote:
>>>>>> On 03/21/2014 05:17 PM, Ian Campbell wrote:
>>>>>>> On Wed, 2014-03-19 at 19:47 +0530, vijay.kilari@gmail.com wrote:
>>>>>>>> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>>>>>>>>
>>>>>>>> mmio handlers are registers at compile time
>>>>>>>> for drivers like vuart and vgic.
>>>>>>>> Make mmio handler registered at runtime by
>>>>>>>> creating linked list of mmio handlers
>>>>>>>
>>>>>>> I'm not convinced of the need for this, certainly the vgic side can just
>>>>>>> demux into v2 or v3 as necessary.
>>>>>>
>>>>>> Demux the code just add an indirection. We could have a list of mmio
>>>>>> handler per domain and rely on it to call the right handler. A bit like x86.
>>>>>>
>>>>>  Until Andrii adds IOMMU  handling should keep this patch? and adopt
>>>>> to it later?
>>>>
>>>> I'm not sure to understand. IHMO, it doesn't sound right to upstream a
>>>> patch that we know it will be superseded in a couple of months. Why
>>>> can't you rework your patch to have it in good shape now?
>>>>
>>>  I assume that Andii's patch will be required to make mmio handlers
>>> per domain. If not let me know references in x86 that I can make it
>>> per domain specific mmio handlers.
>>
>> I have already pointed out to the x86 code few mails before.
>>
>> You can look at xen/arch/x86/intercept.c.
> 
>    From the x86/hvm/intercept.c file, the mmio handling of x86 is
> similar to existing arm mmio handling in arm/io.c file, where the
> handlers are registered statically.

Not really, x86 use an hybrid approach:
    - static handler for IO region that are handled by every guest (hpet
...)
     - dynamic handler registered by register_io_handler

>   I understand that only requirement is to make vgic as domain specific
> similar to vuart. Is that what you expect. Can you please make it clear?

vuart and vgic handler won't be the only dynamic handlers. On some
platform (e.g the cubieboard) we will have to use a similar solution to
be able to passthrough UARTs to dom0. Currently they are blacklisted
because the UART used by Xen shares the page with the other UARTs.

Using the x86 approach will allow us to register the right vGIC
emulation for every guest. We might want to start a shiny FreeBSD guest
with GICv2 emulation and a Linux guest with a GICv3.

> 
>  IMO, the io handling in x86/hvm/intercept.c is not required for arm.

IHMO, as MMIO is often trapped, the performance is critical. Your
solution won't improve the current implementation because you still have
to browse unused MMIO handler each time the guest will trap into Xen.

Regards,

-- 
Julien Grall

Re: [RFC PATCH v1 02/10] xen/arm: register mmio handler at runtime

[Vijay Kilari]

On Tue, Apr 1, 2014 at 4:30 PM, Julien Grall <julien.grall@linaro.org> wrote:
> On 04/01/2014 10:34 AM, Vijay Kilari wrote:
>> Hi Julien,
>
> Hello Vijay,
>
>> On Thu, Mar 27, 2014 at 8:32 PM, Julien Grall <julien.grall@linaro.org> wrote:
>>> On 03/27/2014 05:40 AM, Vijay Kilari wrote:
>>>> On Wed, Mar 26, 2014 at 8:17 PM, Julien Grall <julien.grall@linaro.org> wrote:
>>>>> On 03/26/2014 12:29 PM, Vijay Kilari wrote:
>>>>>> Hi Julien,
>>>>>>
>>>>>>
>>>>>> On Fri, Mar 21, 2014 at 10:53 PM, Julien Grall <julien.grall@linaro.org> wrote:
>>>>>>> On 03/21/2014 05:17 PM, Ian Campbell wrote:
>>>>>>>> On Wed, 2014-03-19 at 19:47 +0530, vijay.kilari@gmail.com wrote:
>>>>>>>>> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>>>>>>>>>
>>>>>>>>> mmio handlers are registers at compile time
>>>>>>>>> for drivers like vuart and vgic.
>>>>>>>>> Make mmio handler registered at runtime by
>>>>>>>>> creating linked list of mmio handlers
>>>>>>>>
>>>>>>>> I'm not convinced of the need for this, certainly the vgic side can just
>>>>>>>> demux into v2 or v3 as necessary.
>>>>>>>
>>>>>>> Demux the code just add an indirection. We could have a list of mmio
>>>>>>> handler per domain and rely on it to call the right handler. A bit like x86.
>>>>>>>
>>>>>>  Until Andrii adds IOMMU  handling should keep this patch? and adopt
>>>>>> to it later?
>>>>>
>>>>> I'm not sure to understand. IHMO, it doesn't sound right to upstream a
>>>>> patch that we know it will be superseded in a couple of months. Why
>>>>> can't you rework your patch to have it in good shape now?
>>>>>
>>>>  I assume that Andii's patch will be required to make mmio handlers
>>>> per domain. If not let me know references in x86 that I can make it
>>>> per domain specific mmio handlers.
>>>
>>> I have already pointed out to the x86 code few mails before.
>>>
>>> You can look at xen/arch/x86/intercept.c.
>>
>>    From the x86/hvm/intercept.c file, the mmio handling of x86 is
>> similar to existing arm mmio handling in arm/io.c file, where the
>> handlers are registered statically.
>
> Not really, x86 use an hybrid approach:
>     - static handler for IO region that are handled by every guest (hpet
> ...)
>      - dynamic handler registered by register_io_handler
>
I think, I can still keep dynamic handler registration.

>>   I understand that only requirement is to make vgic as domain specific
>> similar to vuart. Is that what you expect. Can you please make it clear?
>
> vuart and vgic handler won't be the only dynamic handlers. On some
> platform (e.g the cubieboard) we will have to use a similar solution to
> be able to passthrough UARTs to dom0. Currently they are blacklisted
> because the UART used by Xen shares the page with the other UARTs.
>
> Using the x86 approach will allow us to register the right vGIC
> emulation for every guest. We might want to start a shiny FreeBSD guest
> with GICv2 emulation and a Linux guest with a GICv3.
>
   To some extent vgic is domain specific because vgic structure defined
within arch_domain holds address of vgic with which check_handler is
comparing against trapped address (gpa)

   However when domain is created it does not check for Dom's GIC
supported version.
>From arch_domain_create() gicv_setup() & domain_vgic_init() is
initializing GICv2 version
always.

With introduction of GICv3, now gicv_setup() & domain_vgic_init()
should able to
identify dom's gic version and initialize accordingly.
Is there any way to know what is Dom's/guest support GIC version?.

Or will below ensure picking right version of vgic?. I just had a doubt that
dt_host always picks dom0's/Xen's device tree.

dt_for_each_device_node(dt_host, node)
{
       rc = device_init(node, DEVICE_VGIC, NULL);
        if ( !rc )
            num_vgics++;
}

>>
>>  IMO, the io handling in x86/hvm/intercept.c is not required for arm.
>
> IHMO, as MMIO is often trapped, the performance is critical. Your
> solution won't improve the current implementation because you still have
> to browse unused MMIO handler each time the guest will trap into Xen.
>
> Regards,
>
> --
> Julien Grall

Re: [RFC PATCH v1 02/10] xen/arm: register mmio handler at runtime

[Ian Campbell]

On Tue, 2014-04-01 at 18:02 +0530, Vijay Kilari wrote:
> On Tue, Apr 1, 2014 at 4:30 PM, Julien Grall <julien.grall@linaro.org> wrote:
> > On 04/01/2014 10:34 AM, Vijay Kilari wrote:
> >> Hi Julien,
> >
> > Hello Vijay,
> >
> >> On Thu, Mar 27, 2014 at 8:32 PM, Julien Grall <julien.grall@linaro.org> wrote:
> >>> On 03/27/2014 05:40 AM, Vijay Kilari wrote:
> >>>> On Wed, Mar 26, 2014 at 8:17 PM, Julien Grall <julien.grall@linaro.org> wrote:
> >>>>> On 03/26/2014 12:29 PM, Vijay Kilari wrote:
> >>>>>> Hi Julien,
> >>>>>>
> >>>>>>
> >>>>>> On Fri, Mar 21, 2014 at 10:53 PM, Julien Grall <julien.grall@linaro.org> wrote:
> >>>>>>> On 03/21/2014 05:17 PM, Ian Campbell wrote:
> >>>>>>>> On Wed, 2014-03-19 at 19:47 +0530, vijay.kilari@gmail.com wrote:
> >>>>>>>>> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> >>>>>>>>>
> >>>>>>>>> mmio handlers are registers at compile time
> >>>>>>>>> for drivers like vuart and vgic.
> >>>>>>>>> Make mmio handler registered at runtime by
> >>>>>>>>> creating linked list of mmio handlers
> >>>>>>>>
> >>>>>>>> I'm not convinced of the need for this, certainly the vgic side can just
> >>>>>>>> demux into v2 or v3 as necessary.
> >>>>>>>
> >>>>>>> Demux the code just add an indirection. We could have a list of mmio
> >>>>>>> handler per domain and rely on it to call the right handler. A bit like x86.
> >>>>>>>
> >>>>>>  Until Andrii adds IOMMU  handling should keep this patch? and adopt
> >>>>>> to it later?
> >>>>>
> >>>>> I'm not sure to understand. IHMO, it doesn't sound right to upstream a
> >>>>> patch that we know it will be superseded in a couple of months. Why
> >>>>> can't you rework your patch to have it in good shape now?
> >>>>>
> >>>>  I assume that Andii's patch will be required to make mmio handlers
> >>>> per domain. If not let me know references in x86 that I can make it
> >>>> per domain specific mmio handlers.
> >>>
> >>> I have already pointed out to the x86 code few mails before.
> >>>
> >>> You can look at xen/arch/x86/intercept.c.
> >>
> >>    From the x86/hvm/intercept.c file, the mmio handling of x86 is
> >> similar to existing arm mmio handling in arm/io.c file, where the
> >> handlers are registered statically.
> >
> > Not really, x86 use an hybrid approach:
> >     - static handler for IO region that are handled by every guest (hpet
> > ...)
> >      - dynamic handler registered by register_io_handler
> >
> I think, I can still keep dynamic handler registration.
> 
> >>   I understand that only requirement is to make vgic as domain specific
> >> similar to vuart. Is that what you expect. Can you please make it clear?
> >
> > vuart and vgic handler won't be the only dynamic handlers. On some
> > platform (e.g the cubieboard) we will have to use a similar solution to
> > be able to passthrough UARTs to dom0. Currently they are blacklisted
> > because the UART used by Xen shares the page with the other UARTs.
> >
> > Using the x86 approach will allow us to register the right vGIC
> > emulation for every guest. We might want to start a shiny FreeBSD guest
> > with GICv2 emulation and a Linux guest with a GICv3.
> >
>    To some extent vgic is domain specific because vgic structure defined
> within arch_domain holds address of vgic with which check_handler is
> comparing against trapped address (gpa)
> 
>    However when domain is created it does not check for Dom's GIC
> supported version.
> From arch_domain_create() gicv_setup() & domain_vgic_init() is
> initializing GICv2 version
> always.
> 
> With introduction of GICv3, now gicv_setup() & domain_vgic_init()
> should able to
> identify dom's gic version and initialize accordingly.
> Is there any way to know what is Dom's/guest support GIC version?.
> 
> Or will below ensure picking right version of vgic?. I just had a doubt that
> dt_host always picks dom0's/Xen's device tree.
> 
> dt_for_each_device_node(dt_host, node)
> {
>        rc = device_init(node, DEVICE_VGIC, NULL);
>         if ( !rc )
>             num_vgics++;
> }

Xen does not currently support any platforms which have multiple GICv2s
so we don't have code to support that case. If such a platform turns up
we will fix that of course.

Are we expecting that there will be platforms with multiple GICv3s in
the short term? I thought most of the reasons why you would need
multiple GICv2s were addressed by v3.

I'd suggest to keep things simple for now and just expose a single GIC
of the same version as the hardware platform. We can cover more complex
configurations as and when they arise, likewise lets leave handling of
guest compatibility with v2 vs v3 until we've got the basics in place.

Ian.

Re: [RFC PATCH v1 02/10] xen/arm: register mmio handler at runtime

[Julien Grall]

On 04/01/2014 01:32 PM, Vijay Kilari wrote:
> On Tue, Apr 1, 2014 at 4:30 PM, Julien Grall <julien.grall@linaro.org> wrote:
>> On 04/01/2014 10:34 AM, Vijay Kilari wrote:
>>> Hi Julien,
>>
>> Hello Vijay,
>>
>>> On Thu, Mar 27, 2014 at 8:32 PM, Julien Grall <julien.grall@linaro.org> wrote:
>>>> On 03/27/2014 05:40 AM, Vijay Kilari wrote:
>>>>> On Wed, Mar 26, 2014 at 8:17 PM, Julien Grall <julien.grall@linaro.org> wrote:
>>>>>> On 03/26/2014 12:29 PM, Vijay Kilari wrote:
>>>>>>> Hi Julien,
>>>>>>>
>>>>>>>
>>>>>>> On Fri, Mar 21, 2014 at 10:53 PM, Julien Grall <julien.grall@linaro.org> wrote:
>>>>>>>> On 03/21/2014 05:17 PM, Ian Campbell wrote:
>>>>>>>>> On Wed, 2014-03-19 at 19:47 +0530, vijay.kilari@gmail.com wrote:
>>>>>>>>>> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>>>>>>>>>>
>>>>>>>>>> mmio handlers are registers at compile time
>>>>>>>>>> for drivers like vuart and vgic.
>>>>>>>>>> Make mmio handler registered at runtime by
>>>>>>>>>> creating linked list of mmio handlers
>>>>>>>>>
>>>>>>>>> I'm not convinced of the need for this, certainly the vgic side can just
>>>>>>>>> demux into v2 or v3 as necessary.
>>>>>>>>
>>>>>>>> Demux the code just add an indirection. We could have a list of mmio
>>>>>>>> handler per domain and rely on it to call the right handler. A bit like x86.
>>>>>>>>
>>>>>>>  Until Andrii adds IOMMU  handling should keep this patch? and adopt
>>>>>>> to it later?
>>>>>>
>>>>>> I'm not sure to understand. IHMO, it doesn't sound right to upstream a
>>>>>> patch that we know it will be superseded in a couple of months. Why
>>>>>> can't you rework your patch to have it in good shape now?
>>>>>>
>>>>>  I assume that Andii's patch will be required to make mmio handlers
>>>>> per domain. If not let me know references in x86 that I can make it
>>>>> per domain specific mmio handlers.
>>>>
>>>> I have already pointed out to the x86 code few mails before.
>>>>
>>>> You can look at xen/arch/x86/intercept.c.
>>>
>>>    From the x86/hvm/intercept.c file, the mmio handling of x86 is
>>> similar to existing arm mmio handling in arm/io.c file, where the
>>> handlers are registered statically.
>>
>> Not really, x86 use an hybrid approach:
>>     - static handler for IO region that are handled by every guest (hpet
>> ...)
>>      - dynamic handler registered by register_io_handler
>>
> I think, I can still keep dynamic handler registration.

Per domain?

-- 
Julien Grall

Re: [RFC PATCH v1 02/10] xen/arm: register mmio handler at runtime

[Vijay Kilari]

On Tue, Apr 1, 2014 at 6:21 PM, Julien Grall <julien.grall@linaro.org> wrote:
> On 04/01/2014 01:32 PM, Vijay Kilari wrote:
>> On Tue, Apr 1, 2014 at 4:30 PM, Julien Grall <julien.grall@linaro.org> wrote:
>>> On 04/01/2014 10:34 AM, Vijay Kilari wrote:
>>>> Hi Julien,
>>>
>>> Hello Vijay,
>>>
>>>> On Thu, Mar 27, 2014 at 8:32 PM, Julien Grall <julien.grall@linaro.org> wrote:
>>>>> On 03/27/2014 05:40 AM, Vijay Kilari wrote:
>>>>>> On Wed, Mar 26, 2014 at 8:17 PM, Julien Grall <julien.grall@linaro.org> wrote:
>>>>>>> On 03/26/2014 12:29 PM, Vijay Kilari wrote:
>>>>>>>> Hi Julien,
>>>>>>>>
>>>>>>>>
>>>>>>>> On Fri, Mar 21, 2014 at 10:53 PM, Julien Grall <julien.grall@linaro.org> wrote:
>>>>>>>>> On 03/21/2014 05:17 PM, Ian Campbell wrote:
>>>>>>>>>> On Wed, 2014-03-19 at 19:47 +0530, vijay.kilari@gmail.com wrote:
>>>>>>>>>>> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>>>>>>>>>>>
>>>>>>>>>>> mmio handlers are registers at compile time
>>>>>>>>>>> for drivers like vuart and vgic.
>>>>>>>>>>> Make mmio handler registered at runtime by
>>>>>>>>>>> creating linked list of mmio handlers
>>>>>>>>>>
>>>>>>>>>> I'm not convinced of the need for this, certainly the vgic side can just
>>>>>>>>>> demux into v2 or v3 as necessary.
>>>>>>>>>
>>>>>>>>> Demux the code just add an indirection. We could have a list of mmio
>>>>>>>>> handler per domain and rely on it to call the right handler. A bit like x86.
>>>>>>>>>
>>>>>>>>  Until Andrii adds IOMMU  handling should keep this patch? and adopt
>>>>>>>> to it later?
>>>>>>>
>>>>>>> I'm not sure to understand. IHMO, it doesn't sound right to upstream a
>>>>>>> patch that we know it will be superseded in a couple of months. Why
>>>>>>> can't you rework your patch to have it in good shape now?
>>>>>>>
>>>>>>  I assume that Andii's patch will be required to make mmio handlers
>>>>>> per domain. If not let me know references in x86 that I can make it
>>>>>> per domain specific mmio handlers.
>>>>>
>>>>> I have already pointed out to the x86 code few mails before.
>>>>>
>>>>> You can look at xen/arch/x86/intercept.c.
>>>>
>>>>    From the x86/hvm/intercept.c file, the mmio handling of x86 is
>>>> similar to existing arm mmio handling in arm/io.c file, where the
>>>> handlers are registered statically.
>>>
>>> Not really, x86 use an hybrid approach:
>>>     - static handler for IO region that are handled by every guest (hpet
>>> ...)
>>>      - dynamic handler registered by register_io_handler
>>>
>> I think, I can still keep dynamic handler registration.
>
> Per domain?

Yes, I plan to make this list fixed with 16 entries and managing
similar to x86/hvm/intercept.c "hvm_domain.io_handler".
>
> --
> Julien Grall

Re: [RFC PATCH v1 02/10] xen/arm: register mmio handler at runtime

[Julien Grall]

On 04/01/2014 02:05 PM, Vijay Kilari wrote:

>> Per domain?
> 
> Yes, I plan to make this list fixed with 16 entries and managing
> similar to x86/hvm/intercept.c "hvm_domain.io_handler".

Great, thanks!

-- 
Julien Grall

[RFC PATCH v1 03/10] xen/arm: move vgic data to vgic driver

[vijay.kilari]

From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>

vgic_irq_rank structure contains gic specific
data elements. Move this out of domain.h and
allocate memory dynamically in vgic driver

Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
---
 xen/arch/arm/vgic.c          |   35 ++++++++++++++++++++++++++++-------
 xen/include/asm-arm/domain.h |   13 ++-----------
 2 files changed, 30 insertions(+), 18 deletions(-)

diff --git a/xen/arch/arm/vgic.c b/xen/arch/arm/vgic.c
index d2a13fb..694a15c 100644
--- a/xen/arch/arm/vgic.c
+++ b/xen/arch/arm/vgic.c
@@ -35,6 +35,15 @@
 /* Number of ranks of interrupt registers for a domain */
 #define DOMAIN_NR_RANKS(d) (((d)->arch.vgic.nr_lines+31)/32)
 
+/* Represents state corresponding to a block of 32 interrupts */
+struct vgic_irq_rank {
+    spinlock_t lock; /* Covers access to all other members of this struct */
+    uint32_t ienable, iactive, ipend, pendsgi;
+    uint32_t icfg[2];
+    uint32_t ipriority[8];
+    uint32_t itargets[8];
+};
+
 /*
  * Rank containing GICD_<FOO><n> for GICD_<FOO> with
  * <b>-bits-per-interrupt
@@ -66,9 +75,10 @@ static struct vgic_irq_rank *vgic_irq_rank(struct vcpu *v, int b, int n)
     int rank = REG_RANK_NR(b, n);
 
     if ( rank == 0 )
-        return &v->arch.vgic.private_irqs;
+        return (struct vgic_irq_rank *)v->arch.vgic.private_irqs;
     else if ( rank <= DOMAIN_NR_RANKS(v->domain) )
-        return &v->domain->arch.vgic.shared_irqs[rank - 1];
+       return (struct vgic_irq_rank *)((unsigned char *)(v->domain->arch.vgic.shared_irqs)
+                           + (sizeof(struct vgic_irq_rank) *(rank - 1)));
     else
         return NULL;
 }
@@ -82,9 +92,11 @@ void domain_vgic_free(struct domain *d)
 int vcpu_vgic_init(struct vcpu *v)
 {
     int i;
-    memset(&v->arch.vgic.private_irqs, 0, sizeof(v->arch.vgic.private_irqs));
+    struct vgic_irq_rank *vir;
 
-    spin_lock_init(&v->arch.vgic.private_irqs.lock);
+    vir =  xzalloc(struct vgic_irq_rank);
+    memset(vir, 0, sizeof(struct vgic_irq_rank));
+    spin_lock_init(&vir->lock);
 
     memset(&v->arch.vgic.pending_irqs, 0, sizeof(v->arch.vgic.pending_irqs));
     for (i = 0; i < 32; i++)
@@ -95,11 +107,14 @@ int vcpu_vgic_init(struct vcpu *v)
 
     /* For SGI and PPI the target is always this CPU */
     for ( i = 0 ; i < 8 ; i++ )
-        v->arch.vgic.private_irqs.itargets[i] =
+        vir->itargets[i] =
               (1<<(v->vcpu_id+0))
             | (1<<(v->vcpu_id+8))
             | (1<<(v->vcpu_id+16))
             | (1<<(v->vcpu_id+24));
+
+    v->arch.vgic.private_irqs = (struct vgic_irq_irank *)vir;
+
     INIT_LIST_HEAD(&v->arch.vgic.inflight_irqs);
     INIT_LIST_HEAD(&v->arch.vgic.lr_pending);
     spin_lock_init(&v->arch.vgic.lock);
@@ -636,6 +651,7 @@ static struct mmio_handler vgic_distr_mmio_handler = {
 int domain_vgic_init(struct domain *d)
 {
     int i;
+    struct vgic_irq_rank *r;
 
     d->arch.vgic.ctlr = 0;
 
@@ -648,7 +664,8 @@ int domain_vgic_init(struct domain *d)
         d->arch.vgic.nr_lines = 0; /* We don't need SPIs for the guest */
 
     d->arch.vgic.shared_irqs =
-        xzalloc_array(struct vgic_irq_rank, DOMAIN_NR_RANKS(d));
+        (struct vgic_irq_rank *)xzalloc_array(struct vgic_irq_rank, DOMAIN_NR_RANKS(d));
+
     d->arch.vgic.pending_irqs =
         xzalloc_array(struct pending_irq, d->arch.vgic.nr_lines);
     for (i=0; i<d->arch.vgic.nr_lines; i++)
@@ -657,7 +674,11 @@ int domain_vgic_init(struct domain *d)
         INIT_LIST_HEAD(&d->arch.vgic.pending_irqs[i].lr_queue);
     }
     for (i=0; i<DOMAIN_NR_RANKS(d); i++)
-        spin_lock_init(&d->arch.vgic.shared_irqs[i].lock);
+    {
+        r = (struct vgic_irq_rank *)((unsigned char *)(d->arch.vgic.shared_irqs)
+                                      + sizeof(struct vgic_irq_rank) * i);
+        spin_lock_init(&r->lock);
+    }
 
     register_mmio_handler(&vgic_distr_mmio_handler);
     return 0;
diff --git a/xen/include/asm-arm/domain.h b/xen/include/asm-arm/domain.h
index bc20a15..d49ab68 100644
--- a/xen/include/asm-arm/domain.h
+++ b/xen/include/asm-arm/domain.h
@@ -10,15 +10,6 @@
 #include <public/hvm/params.h>
 #include <xen/serial.h>
 
-/* Represents state corresponding to a block of 32 interrupts */
-struct vgic_irq_rank {
-    spinlock_t lock; /* Covers access to all other members of this struct */
-    uint32_t ienable, iactive, ipend, pendsgi;
-    uint32_t icfg[2];
-    uint32_t ipriority[8];
-    uint32_t itargets[8];
-};
-
 struct pending_irq
 {
     int irq;
@@ -142,7 +133,7 @@ struct arch_domain
         spinlock_t lock;
         int ctlr;
         int nr_lines; /* Number of SPIs */
-        struct vgic_irq_rank *shared_irqs;
+        void *shared_irqs;
         /*
          * SPIs are domain global, SGIs and PPIs are per-VCPU and stored in
          * struct arch_vcpu.
@@ -263,7 +254,7 @@ struct arch_vcpu
          * struct arch_domain.
          */
         struct pending_irq pending_irqs[32];
-        struct vgic_irq_rank private_irqs;
+        void *private_irqs;
 
         /* This list is ordered by IRQ priority and it is used to keep
          * track of the IRQs that the VGIC injected into the guest.
-- 
1.7.9.5

Re: [RFC PATCH v1 03/10] xen/arm: move vgic data to vgic driver

[Julien Grall]

Hello Vijay,

Thank you for your patch.

On 03/19/2014 02:17 PM, vijay.kilari@gmail.com wrote:
> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> 
> vgic_irq_rank structure contains gic specific
> data elements. Move this out of domain.h and
> allocate memory dynamically in vgic driver

I think we should have a vcpu_vgic_v2 structure which contains
everything related to GICv2 in it, e.g:
	- VGIC rank
        - CTLR
        - cbase, dbase,...

And a domain_vgic_v2 for everything common with all VCPUs.

> 
> Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> ---
>  xen/arch/arm/vgic.c          |   35 ++++++++++++++++++++++++++++-------
>  xen/include/asm-arm/domain.h |   13 ++-----------
>  2 files changed, 30 insertions(+), 18 deletions(-)
> 
> diff --git a/xen/arch/arm/vgic.c b/xen/arch/arm/vgic.c
> index d2a13fb..694a15c 100644
> --- a/xen/arch/arm/vgic.c
> +++ b/xen/arch/arm/vgic.c
> @@ -35,6 +35,15 @@
>  /* Number of ranks of interrupt registers for a domain */
>  #define DOMAIN_NR_RANKS(d) (((d)->arch.vgic.nr_lines+31)/32)
>  
> +/* Represents state corresponding to a block of 32 interrupts */
> +struct vgic_irq_rank {
> +    spinlock_t lock; /* Covers access to all other members of this struct */
> +    uint32_t ienable, iactive, ipend, pendsgi;
> +    uint32_t icfg[2];
> +    uint32_t ipriority[8];
> +    uint32_t itargets[8];
> +};
> +

I would move the definition in a vgic_v2.h

>  /*
>   * Rank containing GICD_<FOO><n> for GICD_<FOO> with
>   * <b>-bits-per-interrupt
> @@ -66,9 +75,10 @@ static struct vgic_irq_rank *vgic_irq_rank(struct vcpu *v, int b, int n)
>      int rank = REG_RANK_NR(b, n);
>  
>      if ( rank == 0 )
> -        return &v->arch.vgic.private_irqs;
> +        return (struct vgic_irq_rank *)v->arch.vgic.private_irqs;
>      else if ( rank <= DOMAIN_NR_RANKS(v->domain) )
> -        return &v->domain->arch.vgic.shared_irqs[rank - 1];
> +       return (struct vgic_irq_rank *)((unsigned char *)(v->domain->arch.vgic.shared_irqs)
> +                           + (sizeof(struct vgic_irq_rank) *(rank - 1)));

It's so ugly, can you use temporary variable?

>      else
>          return NULL;
>  }
> @@ -82,9 +92,11 @@ void domain_vgic_free(struct domain *d)
>  int vcpu_vgic_init(struct vcpu *v)
>  {
>      int i;
> -    memset(&v->arch.vgic.private_irqs, 0, sizeof(v->arch.vgic.private_irqs));
> +    struct vgic_irq_rank *vir;
>  
> -    spin_lock_init(&v->arch.vgic.private_irqs.lock);
> +    vir =  xzalloc(struct vgic_irq_rank);
> +    memset(vir, 0, sizeof(struct vgic_irq_rank));

sizeof(*vir)

> +    spin_lock_init(&vir->lock);
>  
>      memset(&v->arch.vgic.pending_irqs, 0, sizeof(v->arch.vgic.pending_irqs));
>      for (i = 0; i < 32; i++)
> @@ -95,11 +107,14 @@ int vcpu_vgic_init(struct vcpu *v)
>  
>      /* For SGI and PPI the target is always this CPU */
>      for ( i = 0 ; i < 8 ; i++ )
> -        v->arch.vgic.private_irqs.itargets[i] =
> +        vir->itargets[i] =
>                (1<<(v->vcpu_id+0))
>              | (1<<(v->vcpu_id+8))
>              | (1<<(v->vcpu_id+16))
>              | (1<<(v->vcpu_id+24));
> +
> +    v->arch.vgic.private_irqs = (struct vgic_irq_irank *)vir;
> +

The cast is not useful here...

>      INIT_LIST_HEAD(&v->arch.vgic.inflight_irqs);
>      INIT_LIST_HEAD(&v->arch.vgic.lr_pending);
>      spin_lock_init(&v->arch.vgic.lock);
> @@ -636,6 +651,7 @@ static struct mmio_handler vgic_distr_mmio_handler = {
>  int domain_vgic_init(struct domain *d)
>  {
>      int i;
> +    struct vgic_irq_rank *r;
>  
>      d->arch.vgic.ctlr = 0;
>  
> @@ -648,7 +664,8 @@ int domain_vgic_init(struct domain *d)
>          d->arch.vgic.nr_lines = 0; /* We don't need SPIs for the guest */
>  
>      d->arch.vgic.shared_irqs =
> -        xzalloc_array(struct vgic_irq_rank, DOMAIN_NR_RANKS(d));
> +        (struct vgic_irq_rank *)xzalloc_array(struct vgic_irq_rank, DOMAIN_NR_RANKS(d));
> +

Same here.

>      d->arch.vgic.pending_irqs =
>          xzalloc_array(struct pending_irq, d->arch.vgic.nr_lines);
>      for (i=0; i<d->arch.vgic.nr_lines; i++)
> @@ -657,7 +674,11 @@ int domain_vgic_init(struct domain *d)
>          INIT_LIST_HEAD(&d->arch.vgic.pending_irqs[i].lr_queue);
>      }
>      for (i=0; i<DOMAIN_NR_RANKS(d); i++)
> -        spin_lock_init(&d->arch.vgic.shared_irqs[i].lock);
> +    {
> +        r = (struct vgic_irq_rank *)((unsigned char *)(d->arch.vgic.shared_irqs)
> +                                      + sizeof(struct vgic_irq_rank) * i);

Please use temporary variable instead of ugly cast to unsigned char.

Regards,

-- 
Julien Grall

Re: [RFC PATCH v1 03/10] xen/arm: move vgic data to vgic driver

[Ian Campbell]

On Thu, 2014-03-20 at 13:51 +0000, Julien Grall wrote:

> > 
> > Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> > ---
> >  xen/arch/arm/vgic.c          |   35 ++++++++++++++++++++++++++++-------
> >  xen/include/asm-arm/domain.h |   13 ++-----------
> >  2 files changed, 30 insertions(+), 18 deletions(-)
> > 
> > diff --git a/xen/arch/arm/vgic.c b/xen/arch/arm/vgic.c
> > index d2a13fb..694a15c 100644
> > --- a/xen/arch/arm/vgic.c
> > +++ b/xen/arch/arm/vgic.c
> > @@ -35,6 +35,15 @@
> >  /* Number of ranks of interrupt registers for a domain */
> >  #define DOMAIN_NR_RANKS(d) (((d)->arch.vgic.nr_lines+31)/32)
> >  
> > +/* Represents state corresponding to a block of 32 interrupts */
> > +struct vgic_irq_rank {
> > +    spinlock_t lock; /* Covers access to all other members of this struct */
> > +    uint32_t ienable, iactive, ipend, pendsgi;
> > +    uint32_t icfg[2];
> > +    uint32_t ipriority[8];
> > +    uint32_t itargets[8];
> > +};
> > +
> 
> I would move the definition in a vgic_v2.h

Is there going to be lots of this stuff which is different for the two?
I'd rather wait and see how unwieldy gic.h gets first.
> 
> >  /*
> >   * Rank containing GICD_<FOO><n> for GICD_<FOO> with
> >   * <b>-bits-per-interrupt
> > @@ -66,9 +75,10 @@ static struct vgic_irq_rank *vgic_irq_rank(struct vcpu *v, int b, int n)
> >      int rank = REG_RANK_NR(b, n);
> >  
> >      if ( rank == 0 )
> > -        return &v->arch.vgic.private_irqs;
> > +        return (struct vgic_irq_rank *)v->arch.vgic.private_irqs;
> >      else if ( rank <= DOMAIN_NR_RANKS(v->domain) )
> > -        return &v->domain->arch.vgic.shared_irqs[rank - 1];
> > +       return (struct vgic_irq_rank *)((unsigned char *)(v->domain->arch.vgic.shared_irqs)
> > +                           + (sizeof(struct vgic_irq_rank) *(rank - 1)));
> 
> It's so ugly, can you use temporary variable?

The exact same thing was present later on, which is crying out for a
helper function.

But actually if v->domain->arch.vgic.shared_irqs were correctly typed
this would be &v->domain->arch.vgic.shared_irqs[rank -i ], which is
clearly the right way to go about this, casting to char * is too ugly to
live.

> 
> >      else
> >          return NULL;
> >  }
> > @@ -82,9 +92,11 @@ void domain_vgic_free(struct domain *d)
> >  int vcpu_vgic_init(struct vcpu *v)
> >  {
> >      int i;
> > -    memset(&v->arch.vgic.private_irqs, 0, sizeof(v->arch.vgic.private_irqs));
> > +    struct vgic_irq_rank *vir;
> >  
> > -    spin_lock_init(&v->arch.vgic.private_irqs.lock);
> > +    vir =  xzalloc(struct vgic_irq_rank);
> > +    memset(vir, 0, sizeof(struct vgic_irq_rank));
> 
> sizeof(*vir)
> 
> > +    spin_lock_init(&vir->lock);
> >  
> >      memset(&v->arch.vgic.pending_irqs, 0, sizeof(v->arch.vgic.pending_irqs));
> >      for (i = 0; i < 32; i++)
> > @@ -95,11 +107,14 @@ int vcpu_vgic_init(struct vcpu *v)
> >  
> >      /* For SGI and PPI the target is always this CPU */
> >      for ( i = 0 ; i < 8 ; i++ )
> > -        v->arch.vgic.private_irqs.itargets[i] =
> > +        vir->itargets[i] =
> >                (1<<(v->vcpu_id+0))
> >              | (1<<(v->vcpu_id+8))
> >              | (1<<(v->vcpu_id+16))
> >              | (1<<(v->vcpu_id+24));
> > +
> > +    v->arch.vgic.private_irqs = (struct vgic_irq_irank *)vir;
> > +
> 
> The cast is not useful here...

And if this line were done up near the xzalloc the rest of the changes
would fall away too, the vir variable isn't all that helpful.

Ian.

Re: [RFC PATCH v1 03/10] xen/arm: move vgic data to vgic driver

[Vijay Kilari]

On Fri, Mar 21, 2014 at 10:53 PM, Ian Campbell <Ian.Campbell@citrix.com> wrote:
> On Thu, 2014-03-20 at 13:51 +0000, Julien Grall wrote:
>
>> >
>> > Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>> > ---
>> >  xen/arch/arm/vgic.c          |   35 ++++++++++++++++++++++++++++-------
>> >  xen/include/asm-arm/domain.h |   13 ++-----------
>> >  2 files changed, 30 insertions(+), 18 deletions(-)
>> >
>> > diff --git a/xen/arch/arm/vgic.c b/xen/arch/arm/vgic.c
>> > index d2a13fb..694a15c 100644
>> > --- a/xen/arch/arm/vgic.c
>> > +++ b/xen/arch/arm/vgic.c
>> > @@ -35,6 +35,15 @@
>> >  /* Number of ranks of interrupt registers for a domain */
>> >  #define DOMAIN_NR_RANKS(d) (((d)->arch.vgic.nr_lines+31)/32)
>> >
>> > +/* Represents state corresponding to a block of 32 interrupts */
>> > +struct vgic_irq_rank {
>> > +    spinlock_t lock; /* Covers access to all other members of this struct */
>> > +    uint32_t ienable, iactive, ipend, pendsgi;
>> > +    uint32_t icfg[2];
>> > +    uint32_t ipriority[8];
>> > +    uint32_t itargets[8];
>> > +};
>> > +
>>
>> I would move the definition in a vgic_v2.h
>
OK

> Is there going to be lots of this stuff which is different for the two?
> I'd rather wait and see how unwieldy gic.h gets first.

For now, the difference is in v3 uint32_t  itargets[8] is replaced
with u64 itargets[32]
may be for v4 it other registers might differ

>>
>> >  /*
>> >   * Rank containing GICD_<FOO><n> for GICD_<FOO> with
>> >   * <b>-bits-per-interrupt
>> > @@ -66,9 +75,10 @@ static struct vgic_irq_rank *vgic_irq_rank(struct vcpu *v, int b, int n)
>> >      int rank = REG_RANK_NR(b, n);
>> >
>> >      if ( rank == 0 )
>> > -        return &v->arch.vgic.private_irqs;
>> > +        return (struct vgic_irq_rank *)v->arch.vgic.private_irqs;
>> >      else if ( rank <= DOMAIN_NR_RANKS(v->domain) )
>> > -        return &v->domain->arch.vgic.shared_irqs[rank - 1];
>> > +       return (struct vgic_irq_rank *)((unsigned char *)(v->domain->arch.vgic.shared_irqs)
>> > +                           + (sizeof(struct vgic_irq_rank) *(rank - 1)));
>>
>> It's so ugly, can you use temporary variable?
>
> The exact same thing was present later on, which is crying out for a
> helper function.
>
> But actually if v->domain->arch.vgic.shared_irqs were correctly typed
> this would be &v->domain->arch.vgic.shared_irqs[rank -i ], which is
> clearly the right way to go about this, casting to char * is too ugly to
> live.
>
OK

>>
>> >      else
>> >          return NULL;
>> >  }
>> > @@ -82,9 +92,11 @@ void domain_vgic_free(struct domain *d)
>> >  int vcpu_vgic_init(struct vcpu *v)
>> >  {
>> >      int i;
>> > -    memset(&v->arch.vgic.private_irqs, 0, sizeof(v->arch.vgic.private_irqs));
>> > +    struct vgic_irq_rank *vir;
>> >
>> > -    spin_lock_init(&v->arch.vgic.private_irqs.lock);
>> > +    vir =  xzalloc(struct vgic_irq_rank);
>> > +    memset(vir, 0, sizeof(struct vgic_irq_rank));
>>
>> sizeof(*vir)
>>
>> > +    spin_lock_init(&vir->lock);
>> >
>> >      memset(&v->arch.vgic.pending_irqs, 0, sizeof(v->arch.vgic.pending_irqs));
>> >      for (i = 0; i < 32; i++)
>> > @@ -95,11 +107,14 @@ int vcpu_vgic_init(struct vcpu *v)
>> >
>> >      /* For SGI and PPI the target is always this CPU */
>> >      for ( i = 0 ; i < 8 ; i++ )
>> > -        v->arch.vgic.private_irqs.itargets[i] =
>> > +        vir->itargets[i] =
>> >                (1<<(v->vcpu_id+0))
>> >              | (1<<(v->vcpu_id+8))
>> >              | (1<<(v->vcpu_id+16))
>> >              | (1<<(v->vcpu_id+24));
>> > +
>> > +    v->arch.vgic.private_irqs = (struct vgic_irq_irank *)vir;
>> > +
>>
>> The cast is not useful here...
>
> And if this line were done up near the xzalloc the rest of the changes
> would fall away too, the vir variable isn't all that helpful.
>
OK
> Ian.
>

Re: [RFC PATCH v1 03/10] xen/arm: move vgic data to vgic driver

[Ian Campbell]

On Sat, 2014-03-22 at 14:50 +0530, Vijay Kilari wrote:
> On Fri, Mar 21, 2014 at 10:53 PM, Ian Campbell <Ian.Campbell@citrix.com> wrote:
> > On Thu, 2014-03-20 at 13:51 +0000, Julien Grall wrote:
> >
> >> >
> >> > Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> >> > ---
> >> >  xen/arch/arm/vgic.c          |   35 ++++++++++++++++++++++++++++-------
> >> >  xen/include/asm-arm/domain.h |   13 ++-----------
> >> >  2 files changed, 30 insertions(+), 18 deletions(-)
> >> >
> >> > diff --git a/xen/arch/arm/vgic.c b/xen/arch/arm/vgic.c
> >> > index d2a13fb..694a15c 100644
> >> > --- a/xen/arch/arm/vgic.c
> >> > +++ b/xen/arch/arm/vgic.c
> >> > @@ -35,6 +35,15 @@
> >> >  /* Number of ranks of interrupt registers for a domain */
> >> >  #define DOMAIN_NR_RANKS(d) (((d)->arch.vgic.nr_lines+31)/32)
> >> >
> >> > +/* Represents state corresponding to a block of 32 interrupts */
> >> > +struct vgic_irq_rank {
> >> > +    spinlock_t lock; /* Covers access to all other members of this struct */
> >> > +    uint32_t ienable, iactive, ipend, pendsgi;
> >> > +    uint32_t icfg[2];
> >> > +    uint32_t ipriority[8];
> >> > +    uint32_t itargets[8];
> >> > +};
> >> > +
> >>
> >> I would move the definition in a vgic_v2.h
> >
> OK
> 
> > Is there going to be lots of this stuff which is different for the two?
> > I'd rather wait and see how unwieldy gic.h gets first.
> 
> For now, the difference is in v3 uint32_t  itargets[8] is replaced
> with u64 itargets[32]

But the rest of the state remains the same?

If that is the case then I don't think there is any reason to split this
struct into v2 and v3 versions. Just keep a single struct with the
larger itargets using v3 semantics and have the v2 driver do the obvious
packing and unpacking.

> may be for v4 it other registers might differ

Lets cross that bridge when we come to it.

Ian.

Re: [RFC PATCH v1 03/10] xen/arm: move vgic data to vgic driver

[Vijay Kilari]

On Mon, Mar 24, 2014 at 4:27 PM, Ian Campbell <Ian.Campbell@citrix.com> wrote:
> On Sat, 2014-03-22 at 14:50 +0530, Vijay Kilari wrote:
>> On Fri, Mar 21, 2014 at 10:53 PM, Ian Campbell <Ian.Campbell@citrix.com> wrote:
>> > On Thu, 2014-03-20 at 13:51 +0000, Julien Grall wrote:
>> >
>> >> >
>> >> > Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>> >> > ---
>> >> >  xen/arch/arm/vgic.c          |   35 ++++++++++++++++++++++++++++-------
>> >> >  xen/include/asm-arm/domain.h |   13 ++-----------
>> >> >  2 files changed, 30 insertions(+), 18 deletions(-)
>> >> >
>> >> > diff --git a/xen/arch/arm/vgic.c b/xen/arch/arm/vgic.c
>> >> > index d2a13fb..694a15c 100644
>> >> > --- a/xen/arch/arm/vgic.c
>> >> > +++ b/xen/arch/arm/vgic.c
>> >> > @@ -35,6 +35,15 @@
>> >> >  /* Number of ranks of interrupt registers for a domain */
>> >> >  #define DOMAIN_NR_RANKS(d) (((d)->arch.vgic.nr_lines+31)/32)
>> >> >
>> >> > +/* Represents state corresponding to a block of 32 interrupts */
>> >> > +struct vgic_irq_rank {
>> >> > +    spinlock_t lock; /* Covers access to all other members of this struct */
>> >> > +    uint32_t ienable, iactive, ipend, pendsgi;
>> >> > +    uint32_t icfg[2];
>> >> > +    uint32_t ipriority[8];
>> >> > +    uint32_t itargets[8];
>> >> > +};
>> >> > +
>> >>
>> >> I would move the definition in a vgic_v2.h
>> >
>> OK
>>
>> > Is there going to be lots of this stuff which is different for the two?
>> > I'd rather wait and see how unwieldy gic.h gets first.
>>
>> For now, the difference is in v3 uint32_t  itargets[8] is replaced
>> with u64 itargets[32]
>
> But the rest of the state remains the same?
>
> If that is the case then I don't think there is any reason to split this
> struct into v2 and v3 versions. Just keep a single struct with the
> larger itargets using v3 semantics and have the v2 driver do the obvious
> packing and unpacking.
>
  The complete vgic driver is based on this structure and with change in size
and number of registers for itargets register is not clean enough to manage
single vgic driver for both v2 & v3.

Anyway, we have separate vgic v2 & v3 driver and having two different
structures and it helps to manage it easily.

>> may be for v4 it other registers might differ
>
> Lets cross that bridge when we come to it.
>
> Ian.
>

Re: [RFC PATCH v1 03/10] xen/arm: move vgic data to vgic driver

[Ian Campbell]

On Wed, 2014-03-26 at 17:14 +0530, Vijay Kilari wrote:
> > If that is the case then I don't think there is any reason to split this
> > struct into v2 and v3 versions. Just keep a single struct with the
> > larger itargets using v3 semantics and have the v2 driver do the obvious
> > packing and unpacking.
> >
>   The complete vgic driver is based on this structure and with change in size
> and number of registers for itargets register is not clean enough to manage
> single vgic driver for both v2 & v3.

What does "not clean enough" mean?

BTW I was suggesting a common datastructure, not a single driver.

Ian.

Re: [RFC PATCH v1 03/10] xen/arm: move vgic data to vgic driver

[Vijay Kilari]

On Wed, Mar 26, 2014 at 5:30 PM, Ian Campbell <Ian.Campbell@citrix.com> wrote:
> On Wed, 2014-03-26 at 17:14 +0530, Vijay Kilari wrote:
>> > If that is the case then I don't think there is any reason to split this
>> > struct into v2 and v3 versions. Just keep a single struct with the
>> > larger itargets using v3 semantics and have the v2 driver do the obvious
>> > packing and unpacking.
>> >
>>   The complete vgic driver is based on this structure and with change in size
>> and number of registers for itargets register is not clean enough to manage
>> single vgic driver for both v2 & v3.
>
> What does "not clean enough" mean?
>

   I mean for v2 the itargets is uint32_t itargets[8] where each
interrupt routing
information is specified with 8 bits and hence 8 32-bit registers can
handle 32 IRQs.

   Where as in v3, itargets is uint64_t itargets[32] where each
interrupt routing
information is specified with 64 bits and hence 32 64-bit registers
are required to handle
32 IRQ's.

  I will make it common structure by putting this itargets under union.

> BTW I was suggesting a common datastructure, not a single driver.
>
> Ian.
>
>

Re: [RFC PATCH v1 03/10] xen/arm: move vgic data to vgic driver

[Vijay Kilari]

On Thu, Mar 20, 2014 at 7:21 PM, Julien Grall <julien.grall@linaro.org> wrote:
> Hello Vijay,
>
> Thank you for your patch.
>
> On 03/19/2014 02:17 PM, vijay.kilari@gmail.com wrote:
>> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>>
>> vgic_irq_rank structure contains gic specific
>> data elements. Move this out of domain.h and
>> allocate memory dynamically in vgic driver
>
> I think we should have a vcpu_vgic_v2 structure which contains
> everything related to GICv2 in it, e.g:
>         - VGIC rank
>         - CTLR
>         - cbase, dbase,...
>
> And a domain_vgic_v2 for everything common with all VCPUs.
>
OK. I will check the feasibility and rework accordingly

>>
>> Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>> ---
>>  xen/arch/arm/vgic.c          |   35 ++++++++++++++++++++++++++++-------
>>  xen/include/asm-arm/domain.h |   13 ++-----------
>>  2 files changed, 30 insertions(+), 18 deletions(-)
>>
>> diff --git a/xen/arch/arm/vgic.c b/xen/arch/arm/vgic.c
>> index d2a13fb..694a15c 100644
>> --- a/xen/arch/arm/vgic.c
>> +++ b/xen/arch/arm/vgic.c
>> @@ -35,6 +35,15 @@
>>  /* Number of ranks of interrupt registers for a domain */
>>  #define DOMAIN_NR_RANKS(d) (((d)->arch.vgic.nr_lines+31)/32)
>>
>> +/* Represents state corresponding to a block of 32 interrupts */
>> +struct vgic_irq_rank {
>> +    spinlock_t lock; /* Covers access to all other members of this struct */
>> +    uint32_t ienable, iactive, ipend, pendsgi;
>> +    uint32_t icfg[2];
>> +    uint32_t ipriority[8];
>> +    uint32_t itargets[8];
>> +};
>> +
>
> I would move the definition in a vgic_v2.h
>
OK

>>  /*
>>   * Rank containing GICD_<FOO><n> for GICD_<FOO> with
>>   * <b>-bits-per-interrupt
>> @@ -66,9 +75,10 @@ static struct vgic_irq_rank *vgic_irq_rank(struct vcpu *v, int b, int n)
>>      int rank = REG_RANK_NR(b, n);
>>
>>      if ( rank == 0 )
>> -        return &v->arch.vgic.private_irqs;
>> +        return (struct vgic_irq_rank *)v->arch.vgic.private_irqs;
>>      else if ( rank <= DOMAIN_NR_RANKS(v->domain) )
>> -        return &v->domain->arch.vgic.shared_irqs[rank - 1];
>> +       return (struct vgic_irq_rank *)((unsigned char *)(v->domain->arch.vgic.shared_irqs)
>> +                           + (sizeof(struct vgic_irq_rank) *(rank - 1)));
>
> It's so ugly, can you use temporary variable?
>
OK

>>      else
>>          return NULL;
>>  }
>> @@ -82,9 +92,11 @@ void domain_vgic_free(struct domain *d)
>>  int vcpu_vgic_init(struct vcpu *v)
>>  {
>>      int i;
>> -    memset(&v->arch.vgic.private_irqs, 0, sizeof(v->arch.vgic.private_irqs));
>> +    struct vgic_irq_rank *vir;
>>
>> -    spin_lock_init(&v->arch.vgic.private_irqs.lock);
>> +    vir =  xzalloc(struct vgic_irq_rank);
>> +    memset(vir, 0, sizeof(struct vgic_irq_rank));
>
> sizeof(*vir)
>
OK

>> +    spin_lock_init(&vir->lock);
>>
>>      memset(&v->arch.vgic.pending_irqs, 0, sizeof(v->arch.vgic.pending_irqs));
>>      for (i = 0; i < 32; i++)
>> @@ -95,11 +107,14 @@ int vcpu_vgic_init(struct vcpu *v)
>>
>>      /* For SGI and PPI the target is always this CPU */
>>      for ( i = 0 ; i < 8 ; i++ )
>> -        v->arch.vgic.private_irqs.itargets[i] =
>> +        vir->itargets[i] =
>>                (1<<(v->vcpu_id+0))
>>              | (1<<(v->vcpu_id+8))
>>              | (1<<(v->vcpu_id+16))
>>              | (1<<(v->vcpu_id+24));
>> +
>> +    v->arch.vgic.private_irqs = (struct vgic_irq_irank *)vir;
>> +
>
> The cast is not useful here...
OK

>
>>      INIT_LIST_HEAD(&v->arch.vgic.inflight_irqs);
>>      INIT_LIST_HEAD(&v->arch.vgic.lr_pending);
>>      spin_lock_init(&v->arch.vgic.lock);
>> @@ -636,6 +651,7 @@ static struct mmio_handler vgic_distr_mmio_handler = {
>>  int domain_vgic_init(struct domain *d)
>>  {
>>      int i;
>> +    struct vgic_irq_rank *r;
>>
>>      d->arch.vgic.ctlr = 0;
>>
>> @@ -648,7 +664,8 @@ int domain_vgic_init(struct domain *d)
>>          d->arch.vgic.nr_lines = 0; /* We don't need SPIs for the guest */
>>
>>      d->arch.vgic.shared_irqs =
>> -        xzalloc_array(struct vgic_irq_rank, DOMAIN_NR_RANKS(d));
>> +        (struct vgic_irq_rank *)xzalloc_array(struct vgic_irq_rank, DOMAIN_NR_RANKS(d));
>> +
>
> Same here.
OK

>
>>      d->arch.vgic.pending_irqs =
>>          xzalloc_array(struct pending_irq, d->arch.vgic.nr_lines);
>>      for (i=0; i<d->arch.vgic.nr_lines; i++)
>> @@ -657,7 +674,11 @@ int domain_vgic_init(struct domain *d)
>>          INIT_LIST_HEAD(&d->arch.vgic.pending_irqs[i].lr_queue);
>>      }
>>      for (i=0; i<DOMAIN_NR_RANKS(d); i++)
>> -        spin_lock_init(&d->arch.vgic.shared_irqs[i].lock);
>> +    {
>> +        r = (struct vgic_irq_rank *)((unsigned char *)(d->arch.vgic.shared_irqs)
>> +                                      + sizeof(struct vgic_irq_rank) * i);
>
> Please use temporary variable instead of ugly cast to unsigned char.
>
OK

> Regards,
>
> --
> Julien Grall

Re: [RFC PATCH v1 03/10] xen/arm: move vgic data to vgic driver

[Stefano Stabellini]

On Wed, 19 Mar 2014, vijay.kilari@gmail.com wrote:
> @@ -142,7 +133,7 @@ struct arch_domain
>          spinlock_t lock;
>          int ctlr;
>          int nr_lines; /* Number of SPIs */
> -        struct vgic_irq_rank *shared_irqs;
> +        void *shared_irqs;

I would prefer to avoid void* fields. Either we introduce a
vgic_irq_rank union that can be v2 or v3, or we could have a single
opaque void* pointer that each vgic implementation can use for whatever
they like. I think that the union is probably better because it would
let us avoid quite a few casts in the code.

Re: [RFC PATCH v1 03/10] xen/arm: move vgic data to vgic driver

[Julien Grall]

On 03/20/2014 05:14 PM, Stefano Stabellini wrote:
> On Wed, 19 Mar 2014, vijay.kilari@gmail.com wrote:
>> @@ -142,7 +133,7 @@ struct arch_domain
>>          spinlock_t lock;
>>          int ctlr;
>>          int nr_lines; /* Number of SPIs */
>> -        struct vgic_irq_rank *shared_irqs;
>> +        void *shared_irqs;
> 
> I would prefer to avoid void* fields. Either we introduce a
> vgic_irq_rank union that can be v2 or v3, or we could have a single
> opaque void* pointer that each vgic implementation can use for whatever
> they like. I think that the union is probably better because it would
> let us avoid quite a few casts in the code.
> 

Most of the cast can be removed without this trick :).

IHMO, void * might better to avoid expanding the domain/VCPU structure.
We are already close to the 1 page size.

Regards,

-- 
Julien Grall

Re: [RFC PATCH v1 03/10] xen/arm: move vgic data to vgic driver

[Stefano Stabellini]

On Thu, 20 Mar 2014, Julien Grall wrote:
> On 03/20/2014 05:14 PM, Stefano Stabellini wrote:
> > On Wed, 19 Mar 2014, vijay.kilari@gmail.com wrote:
> >> @@ -142,7 +133,7 @@ struct arch_domain
> >>          spinlock_t lock;
> >>          int ctlr;
> >>          int nr_lines; /* Number of SPIs */
> >> -        struct vgic_irq_rank *shared_irqs;
> >> +        void *shared_irqs;
> > 
> > I would prefer to avoid void* fields. Either we introduce a
> > vgic_irq_rank union that can be v2 or v3, or we could have a single
> > opaque void* pointer that each vgic implementation can use for whatever
> > they like. I think that the union is probably better because it would
> > let us avoid quite a few casts in the code.
> > 
> 
> Most of the cast can be removed without this trick :).
> 
> IHMO, void * might better to avoid expanding the domain/VCPU structure.
> We are already close to the 1 page size.

A pointer is fine, that is not an issue.
However we should try to avoid the usage of void* as a way to avoid type
checking at compile time.

Re: [RFC PATCH v1 03/10] xen/arm: move vgic data to vgic driver

[Ian Campbell]

On Thu, 2014-03-20 at 18:11 +0000, Stefano Stabellini wrote:
> On Thu, 20 Mar 2014, Julien Grall wrote:
> > On 03/20/2014 05:14 PM, Stefano Stabellini wrote:
> > > On Wed, 19 Mar 2014, vijay.kilari@gmail.com wrote:
> > >> @@ -142,7 +133,7 @@ struct arch_domain
> > >>          spinlock_t lock;
> > >>          int ctlr;
> > >>          int nr_lines; /* Number of SPIs */
> > >> -        struct vgic_irq_rank *shared_irqs;
> > >> +        void *shared_irqs;
> > > 
> > > I would prefer to avoid void* fields. Either we introduce a
> > > vgic_irq_rank union that can be v2 or v3, or we could have a single
> > > opaque void* pointer that each vgic implementation can use for whatever
> > > they like. I think that the union is probably better because it would
> > > let us avoid quite a few casts in the code.
> > > 
> > 
> > Most of the cast can be removed without this trick :).
> > 
> > IHMO, void * might better to avoid expanding the domain/VCPU structure.
> > We are already close to the 1 page size.
> 
> A pointer is fine, that is not an issue.
> However we should try to avoid the usage of void* as a way to avoid type
> checking at compile time.

Yes. A (pointer to a) union is probably the way to go (not having
reviewed the series myself yet mind).

Ian.

[RFC PATCH v1 04/10] arm/xen: move gic save and restore registers to gic driver

[vijay.kilari]

From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>

gic saved registers are moved to gic driver.
This required structure is allocated at runtime
and is saved & restored.

Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
---
 xen/arch/arm/domain.c        |    3 +++
 xen/arch/arm/gic.c           |   36 +++++++++++++++++++++++++++++-------
 xen/include/asm-arm/domain.h |    3 +--
 xen/include/asm-arm/gic.h    |    1 +
 4 files changed, 34 insertions(+), 9 deletions(-)

diff --git a/xen/arch/arm/domain.c b/xen/arch/arm/domain.c
index 82a1e79..292716a 100644
--- a/xen/arch/arm/domain.c
+++ b/xen/arch/arm/domain.c
@@ -466,6 +466,9 @@ int vcpu_initialise(struct vcpu *v)
     v->arch.saved_context.sp = (register_t)v->arch.cpu_info;
     v->arch.saved_context.pc = (register_t)continue_new_vcpu;
 
+    if ( (rc = vcpu_gic_init(v)) != 0 )
+        return rc;
+
     /* Idle VCPUs don't need the rest of this setup */
     if ( is_idle_vcpu(v) )
         return rc;
diff --git a/xen/arch/arm/gic.c b/xen/arch/arm/gic.c
index 4be0897..973fcf9 100644
--- a/xen/arch/arm/gic.c
+++ b/xen/arch/arm/gic.c
@@ -41,6 +41,13 @@
 #define GICH ((volatile uint32_t *) FIXMAP_ADDR(FIXMAP_GICH))
 static void gic_restore_pending_irqs(struct vcpu *v);
 
+struct gic_state_data {
+    uint32_t gic_hcr;
+    uint32_t gic_vmcr;
+    uint32_t gic_apr;
+    uint32_t gic_lr[64];
+};
+
 /* Global state */
 static struct {
     paddr_t dbase;       /* Address of distributor registers */
@@ -98,6 +105,9 @@ irq_desc_t *__irq_to_desc(int irq)
 void gic_save_state(struct vcpu *v)
 {
     int i;
+    struct gic_state_data *d;
+    d = (struct gic_state_data *)v->arch.gic_state;
+
     ASSERT(!local_irq_is_enabled());
 
     /* No need for spinlocks here because interrupts are disabled around
@@ -105,10 +115,10 @@ void gic_save_state(struct vcpu *v)
      * accessed simultaneously by another pCPU.
      */
     for ( i=0; i<nr_lrs; i++)
-        v->arch.gic_lr[i] = GICH[GICH_LR + i];
+        d->gic_lr[i] = GICH[GICH_LR + i];
     v->arch.lr_mask = this_cpu(lr_mask);
-    v->arch.gic_apr = GICH[GICH_APR];
-    v->arch.gic_vmcr = GICH[GICH_VMCR];
+    d->gic_apr = GICH[GICH_APR];
+    d->gic_vmcr = GICH[GICH_VMCR];
     /* Disable until next VCPU scheduled */
     GICH[GICH_HCR] = 0;
     isb();
@@ -117,15 +127,17 @@ void gic_save_state(struct vcpu *v)
 void gic_restore_state(struct vcpu *v)
 {
     int i;
+    struct gic_state_data *d;
+    d = (struct gic_state_data *)v->arch.gic_state;
 
     if ( is_idle_vcpu(v) )
         return;
 
     this_cpu(lr_mask) = v->arch.lr_mask;
     for ( i=0; i<nr_lrs; i++)
-        GICH[GICH_LR + i] = v->arch.gic_lr[i];
-    GICH[GICH_APR] = v->arch.gic_apr;
-    GICH[GICH_VMCR] = v->arch.gic_vmcr;
+        GICH[GICH_LR + i] = d->gic_lr[i];
+    GICH[GICH_APR] = d->gic_apr;
+    GICH[GICH_VMCR] = d->gic_vmcr;
     GICH[GICH_HCR] = GICH_HCR_EN;
     isb();
 
@@ -877,6 +889,14 @@ void gic_interrupt(struct cpu_user_regs *regs, int is_fiq)
     } while (1);
 }
 
+int vcpu_gic_init(struct vcpu *v)
+{
+     v->arch.gic_state = xzalloc(struct gic_state_data);
+     if(!v->arch.gic_state)
+        return -ENOMEM;
+     return 0;
+}
+
 int gicv_setup(struct domain *d)
 {
     int ret;
@@ -1001,6 +1021,8 @@ void gic_dump_info(struct vcpu *v)
 {
     int i;
     struct pending_irq *p;
+    struct gic_state_data *d;
+    d = (struct gic_state_data *)v->arch.gic_state;
 
     printk("GICH_LRs (vcpu %d) mask=%"PRIx64"\n", v->vcpu_id, v->arch.lr_mask);
     if ( v == current )
@@ -1009,7 +1031,7 @@ void gic_dump_info(struct vcpu *v)
             printk("   HW_LR[%d]=%x\n", i, GICH[GICH_LR + i]);
     } else {
         for ( i = 0; i < nr_lrs; i++ )
-            printk("   VCPU_LR[%d]=%x\n", i, v->arch.gic_lr[i]);
+            printk("   VCPU_LR[%d]=%x\n", i, d->gic_lr[i]);
     }
 
     list_for_each_entry ( p, &v->arch.vgic.inflight_irqs, inflight )
diff --git a/xen/include/asm-arm/domain.h b/xen/include/asm-arm/domain.h
index d49ab68..38df789 100644
--- a/xen/include/asm-arm/domain.h
+++ b/xen/include/asm-arm/domain.h
@@ -243,8 +243,7 @@ struct arch_vcpu
     uint32_t csselr;
     register_t vmpidr;
 
-    uint32_t gic_hcr, gic_vmcr, gic_apr;
-    uint32_t gic_lr[64];
+    void *gic_state;
     uint64_t event_mask;
     uint64_t lr_mask;
 
diff --git a/xen/include/asm-arm/gic.h b/xen/include/asm-arm/gic.h
index 340ef73..debfab8 100644
--- a/xen/include/asm-arm/gic.h
+++ b/xen/include/asm-arm/gic.h
@@ -161,6 +161,7 @@ extern int domain_vgic_init(struct domain *d);
 extern void domain_vgic_free(struct domain *d);
 
 extern int vcpu_vgic_init(struct vcpu *v);
+extern int vcpu_gic_init(struct vcpu *v);
 
 extern void vgic_vcpu_inject_irq(struct vcpu *v, unsigned int irq,int virtual);
 extern void vgic_clear_pending_irqs(struct vcpu *v);
-- 
1.7.9.5

Re: [RFC PATCH v1 04/10] arm/xen: move gic save and restore registers to gic driver

[Julien Grall]

Hello Vijay,

Thanks for the patch.

On 03/19/2014 02:17 PM, vijay.kilari@gmail.com wrote:
> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> 
> gic saved registers are moved to gic driver.
> This required structure is allocated at runtime
> and is saved & restored.
> 
> Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> ---
>  xen/arch/arm/domain.c        |    3 +++
>  xen/arch/arm/gic.c           |   36 +++++++++++++++++++++++++++++-------
>  xen/include/asm-arm/domain.h |    3 +--
>  xen/include/asm-arm/gic.h    |    1 +
>  4 files changed, 34 insertions(+), 9 deletions(-)
> 
> diff --git a/xen/arch/arm/domain.c b/xen/arch/arm/domain.c
> index 82a1e79..292716a 100644
> --- a/xen/arch/arm/domain.c
> +++ b/xen/arch/arm/domain.c
> @@ -466,6 +466,9 @@ int vcpu_initialise(struct vcpu *v)
>      v->arch.saved_context.sp = (register_t)v->arch.cpu_info;
>      v->arch.saved_context.pc = (register_t)continue_new_vcpu;
>  
> +    if ( (rc = vcpu_gic_init(v)) != 0 )
> +        return rc;
> +
>      /* Idle VCPUs don't need the rest of this setup */
>      if ( is_idle_vcpu(v) )
>          return rc;
> diff --git a/xen/arch/arm/gic.c b/xen/arch/arm/gic.c
> index 4be0897..973fcf9 100644
> --- a/xen/arch/arm/gic.c
> +++ b/xen/arch/arm/gic.c
> @@ -41,6 +41,13 @@
>  #define GICH ((volatile uint32_t *) FIXMAP_ADDR(FIXMAP_GICH))
>  static void gic_restore_pending_irqs(struct vcpu *v);
>  
> +struct gic_state_data {
> +    uint32_t gic_hcr;
> +    uint32_t gic_vmcr;
> +    uint32_t gic_apr;
> +    uint32_t gic_lr[64];
> +};
> +

Can you move this structure in a gic_v2 header?

>  /* Global state */
>  static struct {
>      paddr_t dbase;       /* Address of distributor registers */
> @@ -98,6 +105,9 @@ irq_desc_t *__irq_to_desc(int irq)
>  void gic_save_state(struct vcpu *v)
>  {
>      int i;
> +    struct gic_state_data *d;
> +    d = (struct gic_state_data *)v->arch.gic_state;
> +

You don't need the cast. Can you also rename d in state? Using d is very
confusing as it's a common alias used for domain.

struct gic_state_data *d = v->arch.gic_state;

>      ASSERT(!local_irq_is_enabled());
>  
>      /* No need for spinlocks here because interrupts are disabled around
> @@ -105,10 +115,10 @@ void gic_save_state(struct vcpu *v)
>       * accessed simultaneously by another pCPU.
>       */
>      for ( i=0; i<nr_lrs; i++)
> -        v->arch.gic_lr[i] = GICH[GICH_LR + i];
> +        d->gic_lr[i] = GICH[GICH_LR + i];
>      v->arch.lr_mask = this_cpu(lr_mask);
> -    v->arch.gic_apr = GICH[GICH_APR];
> -    v->arch.gic_vmcr = GICH[GICH_VMCR];
> +    d->gic_apr = GICH[GICH_APR];
> +    d->gic_vmcr = GICH[GICH_VMCR];
>      /* Disable until next VCPU scheduled */
>      GICH[GICH_HCR] = 0;
>      isb();
> @@ -117,15 +127,17 @@ void gic_save_state(struct vcpu *v)
>  void gic_restore_state(struct vcpu *v)
>  {
>      int i;
> +    struct gic_state_data *d;
> +    d = (struct gic_state_data *)v->arch.gic_state;

Same comments here.

>  
>      if ( is_idle_vcpu(v) )
>          return;
>  
>      this_cpu(lr_mask) = v->arch.lr_mask;
>      for ( i=0; i<nr_lrs; i++)
> -        GICH[GICH_LR + i] = v->arch.gic_lr[i];
> -    GICH[GICH_APR] = v->arch.gic_apr;
> -    GICH[GICH_VMCR] = v->arch.gic_vmcr;
> +        GICH[GICH_LR + i] = d->gic_lr[i];
> +    GICH[GICH_APR] = d->gic_apr;
> +    GICH[GICH_VMCR] = d->gic_vmcr;
>      GICH[GICH_HCR] = GICH_HCR_EN;
>      isb();
>  
> @@ -877,6 +889,14 @@ void gic_interrupt(struct cpu_user_regs *regs, int is_fiq)
>      } while (1);
>  }
>  
> +int vcpu_gic_init(struct vcpu *v)
> +{
> +     v->arch.gic_state = xzalloc(struct gic_state_data);
> +     if(!v->arch.gic_state)

Coding style
if ( .. )

> +        return -ENOMEM;
> +     return 0;
> +}
> +

Where is the function to free v->arch.gic_state when the domain is
destroyed?

>  int gicv_setup(struct domain *d)
>  {
>      int ret;
> @@ -1001,6 +1021,8 @@ void gic_dump_info(struct vcpu *v)
>  {
>      int i;
>      struct pending_irq *p;
> +    struct gic_state_data *d;
> +    d = (struct gic_state_data *)v->arch.gic_state;

Don't need the cast and can you rename the d variable?

Regards,

-- 
Julien Grall

Re: [RFC PATCH v1 04/10] arm/xen: move gic save and restore registers to gic driver

[Ian Campbell]

On Thu, 2014-03-20 at 15:22 +0000, Julien Grall wrote:
> > @@ -98,6 +105,9 @@ irq_desc_t *__irq_to_desc(int irq)
> >  void gic_save_state(struct vcpu *v)
> >  {
> >      int i;
> > +    struct gic_state_data *d;
> > +    d = (struct gic_state_data *)v->arch.gic_state;
> > +
> 
> You don't need the cast. Can you also rename d in state? Using d is very
> confusing as it's a common alias used for domain.

Yes, I was about to query how it was correct to move all this per-vcpu
state into a per-domain structure.

I don't think accessing this as v->arch.gic_state->gic_apr etc would be
the end of the world, although in context it seems like the field could
become just apr and even s/gic_state/gic/ I think. Everything in struct
vcpu_info is state.

Ian.

Re: [RFC PATCH v1 04/10] arm/xen: move gic save and restore registers to gic driver

[Vijay Kilari]

On Thu, Mar 20, 2014 at 8:52 PM, Julien Grall <julien.grall@linaro.org> wrote:
> Hello Vijay,
>
> Thanks for the patch.
>
> On 03/19/2014 02:17 PM, vijay.kilari@gmail.com wrote:
>> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>>
>> gic saved registers are moved to gic driver.
>> This required structure is allocated at runtime
>> and is saved & restored.
>>
>> Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>> ---
>>  xen/arch/arm/domain.c        |    3 +++
>>  xen/arch/arm/gic.c           |   36 +++++++++++++++++++++++++++++-------
>>  xen/include/asm-arm/domain.h |    3 +--
>>  xen/include/asm-arm/gic.h    |    1 +
>>  4 files changed, 34 insertions(+), 9 deletions(-)
>>
>> diff --git a/xen/arch/arm/domain.c b/xen/arch/arm/domain.c
>> index 82a1e79..292716a 100644
>> --- a/xen/arch/arm/domain.c
>> +++ b/xen/arch/arm/domain.c
>> @@ -466,6 +466,9 @@ int vcpu_initialise(struct vcpu *v)
>>      v->arch.saved_context.sp = (register_t)v->arch.cpu_info;
>>      v->arch.saved_context.pc = (register_t)continue_new_vcpu;
>>
>> +    if ( (rc = vcpu_gic_init(v)) != 0 )
>> +        return rc;
>> +
>>      /* Idle VCPUs don't need the rest of this setup */
>>      if ( is_idle_vcpu(v) )
>>          return rc;
>> diff --git a/xen/arch/arm/gic.c b/xen/arch/arm/gic.c
>> index 4be0897..973fcf9 100644
>> --- a/xen/arch/arm/gic.c
>> +++ b/xen/arch/arm/gic.c
>> @@ -41,6 +41,13 @@
>>  #define GICH ((volatile uint32_t *) FIXMAP_ADDR(FIXMAP_GICH))
>>  static void gic_restore_pending_irqs(struct vcpu *v);
>>
>> +struct gic_state_data {
>> +    uint32_t gic_hcr;
>> +    uint32_t gic_vmcr;
>> +    uint32_t gic_apr;
>> +    uint32_t gic_lr[64];
>> +};
>> +
>
> Can you move this structure in a gic_v2 header?
>
OK

>>  /* Global state */
>>  static struct {
>>      paddr_t dbase;       /* Address of distributor registers */
>> @@ -98,6 +105,9 @@ irq_desc_t *__irq_to_desc(int irq)
>>  void gic_save_state(struct vcpu *v)
>>  {
>>      int i;
>> +    struct gic_state_data *d;
>> +    d = (struct gic_state_data *)v->arch.gic_state;
>> +
>
> You don't need the cast. Can you also rename d in state? Using d is very
> confusing as it's a common alias used for domain.
>
OK
> struct gic_state_data *d = v->arch.gic_state;
>
>>      ASSERT(!local_irq_is_enabled());
>>
>>      /* No need for spinlocks here because interrupts are disabled around
>> @@ -105,10 +115,10 @@ void gic_save_state(struct vcpu *v)
>>       * accessed simultaneously by another pCPU.
>>       */
>>      for ( i=0; i<nr_lrs; i++)
>> -        v->arch.gic_lr[i] = GICH[GICH_LR + i];
>> +        d->gic_lr[i] = GICH[GICH_LR + i];
>>      v->arch.lr_mask = this_cpu(lr_mask);
>> -    v->arch.gic_apr = GICH[GICH_APR];
>> -    v->arch.gic_vmcr = GICH[GICH_VMCR];
>> +    d->gic_apr = GICH[GICH_APR];
>> +    d->gic_vmcr = GICH[GICH_VMCR];
>>      /* Disable until next VCPU scheduled */
>>      GICH[GICH_HCR] = 0;
>>      isb();
>> @@ -117,15 +127,17 @@ void gic_save_state(struct vcpu *v)
>>  void gic_restore_state(struct vcpu *v)
>>  {
>>      int i;
>> +    struct gic_state_data *d;
>> +    d = (struct gic_state_data *)v->arch.gic_state;
>
> Same comments here.
>
>>
>>      if ( is_idle_vcpu(v) )
>>          return;
>>
>>      this_cpu(lr_mask) = v->arch.lr_mask;
>>      for ( i=0; i<nr_lrs; i++)
>> -        GICH[GICH_LR + i] = v->arch.gic_lr[i];
>> -    GICH[GICH_APR] = v->arch.gic_apr;
>> -    GICH[GICH_VMCR] = v->arch.gic_vmcr;
>> +        GICH[GICH_LR + i] = d->gic_lr[i];
>> +    GICH[GICH_APR] = d->gic_apr;
>> +    GICH[GICH_VMCR] = d->gic_vmcr;
>>      GICH[GICH_HCR] = GICH_HCR_EN;
>>      isb();
>>
>> @@ -877,6 +889,14 @@ void gic_interrupt(struct cpu_user_regs *regs, int is_fiq)
>>      } while (1);
>>  }
>>
>> +int vcpu_gic_init(struct vcpu *v)
>> +{
>> +     v->arch.gic_state = xzalloc(struct gic_state_data);
>> +     if(!v->arch.gic_state)
>
> Coding style
> if ( .. )
>
OK
>> +        return -ENOMEM;
>> +     return 0;
>> +}
>> +
>
> Where is the function to free v->arch.gic_state when the domain is
> destroyed?
>
>>  int gicv_setup(struct domain *d)
>>  {
>>      int ret;
>> @@ -1001,6 +1021,8 @@ void gic_dump_info(struct vcpu *v)
>>  {
>>      int i;
>>      struct pending_irq *p;
>> +    struct gic_state_data *d;
>> +    d = (struct gic_state_data *)v->arch.gic_state;
>
> Don't need the cast and can you rename the d variable?
>
OK
> Regards,
>
> --
> Julien Grall

Re: [RFC PATCH v1 04/10] arm/xen: move gic save and restore registers to gic driver

[Stefano Stabellini]

On Wed, 19 Mar 2014, vijay.kilari@gmail.com wrote:
> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> 
> gic saved registers are moved to gic driver.
> This required structure is allocated at runtime
> and is saved & restored.
> 
> Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> ---
>  xen/arch/arm/domain.c        |    3 +++
>  xen/arch/arm/gic.c           |   36 +++++++++++++++++++++++++++++-------
>  xen/include/asm-arm/domain.h |    3 +--
>  xen/include/asm-arm/gic.h    |    1 +
>  4 files changed, 34 insertions(+), 9 deletions(-)
> 
> diff --git a/xen/arch/arm/domain.c b/xen/arch/arm/domain.c
> index 82a1e79..292716a 100644
> --- a/xen/arch/arm/domain.c
> +++ b/xen/arch/arm/domain.c
> @@ -466,6 +466,9 @@ int vcpu_initialise(struct vcpu *v)
>      v->arch.saved_context.sp = (register_t)v->arch.cpu_info;
>      v->arch.saved_context.pc = (register_t)continue_new_vcpu;
>  
> +    if ( (rc = vcpu_gic_init(v)) != 0 )
> +        return rc;
> +
>      /* Idle VCPUs don't need the rest of this setup */
>      if ( is_idle_vcpu(v) )
>          return rc;
> diff --git a/xen/arch/arm/gic.c b/xen/arch/arm/gic.c
> index 4be0897..973fcf9 100644
> --- a/xen/arch/arm/gic.c
> +++ b/xen/arch/arm/gic.c
> @@ -41,6 +41,13 @@
>  #define GICH ((volatile uint32_t *) FIXMAP_ADDR(FIXMAP_GICH))
>  static void gic_restore_pending_irqs(struct vcpu *v);
>  
> +struct gic_state_data {
> +    uint32_t gic_hcr;
> +    uint32_t gic_vmcr;
> +    uint32_t gic_apr;
> +    uint32_t gic_lr[64];
> +};
> +
>  /* Global state */
>  static struct {
>      paddr_t dbase;       /* Address of distributor registers */
> @@ -98,6 +105,9 @@ irq_desc_t *__irq_to_desc(int irq)
>  void gic_save_state(struct vcpu *v)
>  {
>      int i;
> +    struct gic_state_data *d;
> +    d = (struct gic_state_data *)v->arch.gic_state;
> +
>      ASSERT(!local_irq_is_enabled());
>  
>      /* No need for spinlocks here because interrupts are disabled around
> @@ -105,10 +115,10 @@ void gic_save_state(struct vcpu *v)
>       * accessed simultaneously by another pCPU.
>       */
>      for ( i=0; i<nr_lrs; i++)
> -        v->arch.gic_lr[i] = GICH[GICH_LR + i];
> +        d->gic_lr[i] = GICH[GICH_LR + i];
>      v->arch.lr_mask = this_cpu(lr_mask);
> -    v->arch.gic_apr = GICH[GICH_APR];
> -    v->arch.gic_vmcr = GICH[GICH_VMCR];
> +    d->gic_apr = GICH[GICH_APR];
> +    d->gic_vmcr = GICH[GICH_VMCR];
>      /* Disable until next VCPU scheduled */
>      GICH[GICH_HCR] = 0;
>      isb();
> @@ -117,15 +127,17 @@ void gic_save_state(struct vcpu *v)
>  void gic_restore_state(struct vcpu *v)
>  {
>      int i;
> +    struct gic_state_data *d;
> +    d = (struct gic_state_data *)v->arch.gic_state;
>  
>      if ( is_idle_vcpu(v) )
>          return;
>  
>      this_cpu(lr_mask) = v->arch.lr_mask;
>      for ( i=0; i<nr_lrs; i++)
> -        GICH[GICH_LR + i] = v->arch.gic_lr[i];
> -    GICH[GICH_APR] = v->arch.gic_apr;
> -    GICH[GICH_VMCR] = v->arch.gic_vmcr;
> +        GICH[GICH_LR + i] = d->gic_lr[i];
> +    GICH[GICH_APR] = d->gic_apr;
> +    GICH[GICH_VMCR] = d->gic_vmcr;
>      GICH[GICH_HCR] = GICH_HCR_EN;
>      isb();
>  
> @@ -877,6 +889,14 @@ void gic_interrupt(struct cpu_user_regs *regs, int is_fiq)
>      } while (1);
>  }
>  
> +int vcpu_gic_init(struct vcpu *v)
> +{
> +     v->arch.gic_state = xzalloc(struct gic_state_data);
> +     if(!v->arch.gic_state)
> +        return -ENOMEM;
> +     return 0;
> +}
> +
>  int gicv_setup(struct domain *d)
>  {
>      int ret;
> @@ -1001,6 +1021,8 @@ void gic_dump_info(struct vcpu *v)
>  {
>      int i;
>      struct pending_irq *p;
> +    struct gic_state_data *d;
> +    d = (struct gic_state_data *)v->arch.gic_state;
>  
>      printk("GICH_LRs (vcpu %d) mask=%"PRIx64"\n", v->vcpu_id, v->arch.lr_mask);
>      if ( v == current )
> @@ -1009,7 +1031,7 @@ void gic_dump_info(struct vcpu *v)
>              printk("   HW_LR[%d]=%x\n", i, GICH[GICH_LR + i]);
>      } else {
>          for ( i = 0; i < nr_lrs; i++ )
> -            printk("   VCPU_LR[%d]=%x\n", i, v->arch.gic_lr[i]);
> +            printk("   VCPU_LR[%d]=%x\n", i, d->gic_lr[i]);
>      }
>  
>      list_for_each_entry ( p, &v->arch.vgic.inflight_irqs, inflight )
> diff --git a/xen/include/asm-arm/domain.h b/xen/include/asm-arm/domain.h
> index d49ab68..38df789 100644
> --- a/xen/include/asm-arm/domain.h
> +++ b/xen/include/asm-arm/domain.h
> @@ -243,8 +243,7 @@ struct arch_vcpu
>      uint32_t csselr;
>      register_t vmpidr;
>  
> -    uint32_t gic_hcr, gic_vmcr, gic_apr;
> -    uint32_t gic_lr[64];
> +    void *gic_state;
>      uint64_t event_mask;
>      uint64_t lr_mask;
>  
> diff --git a/xen/include/asm-arm/gic.h b/xen/include/asm-arm/gic.h
> index 340ef73..debfab8 100644
> --- a/xen/include/asm-arm/gic.h
> +++ b/xen/include/asm-arm/gic.h
> @@ -161,6 +161,7 @@ extern int domain_vgic_init(struct domain *d);
>  extern void domain_vgic_free(struct domain *d);
>  
>  extern int vcpu_vgic_init(struct vcpu *v);
> +extern int vcpu_gic_init(struct vcpu *v);
>  
>  extern void vgic_vcpu_inject_irq(struct vcpu *v, unsigned int irq,int virtual);
>  extern void vgic_clear_pending_irqs(struct vcpu *v);

Same here about avoiding void*.
We could move the definition of struct gic_state_data to gic.h and use
it in domain.h.  We can turn it into a union in the later patches.

Re: [RFC PATCH v1 04/10] arm/xen: move gic save and restore registers to gic driver

[Ian Campbell]

On Thu, 2014-03-20 at 17:23 +0000, Stefano Stabellini wrote:

> Same here about avoiding void*.

Yes, please remove all uses of void * from this series, and consider
each cast you are thinking about adding very carefully, there should be
almost no need to be casting anything in any of this code.

Ian.

Re: [RFC PATCH v1 04/10] arm/xen: move gic save and restore registers to gic driver

[Vijay Kilari]

On Thu, Mar 20, 2014 at 10:53 PM, Stefano Stabellini
<stefano.stabellini@eu.citrix.com> wrote:
> On Wed, 19 Mar 2014, vijay.kilari@gmail.com wrote:
>> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>>
>> gic saved registers are moved to gic driver.
>> This required structure is allocated at runtime
>> and is saved & restored.
>>
>> Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>> ---
>>  xen/arch/arm/domain.c        |    3 +++
>>  xen/arch/arm/gic.c           |   36 +++++++++++++++++++++++++++++-------
>>  xen/include/asm-arm/domain.h |    3 +--
>>  xen/include/asm-arm/gic.h    |    1 +
>>  4 files changed, 34 insertions(+), 9 deletions(-)
>>
>> diff --git a/xen/arch/arm/domain.c b/xen/arch/arm/domain.c
>> index 82a1e79..292716a 100644
>> --- a/xen/arch/arm/domain.c
>> +++ b/xen/arch/arm/domain.c
>> @@ -466,6 +466,9 @@ int vcpu_initialise(struct vcpu *v)
>>      v->arch.saved_context.sp = (register_t)v->arch.cpu_info;
>>      v->arch.saved_context.pc = (register_t)continue_new_vcpu;
>>
>> +    if ( (rc = vcpu_gic_init(v)) != 0 )
>> +        return rc;
>> +
>>      /* Idle VCPUs don't need the rest of this setup */
>>      if ( is_idle_vcpu(v) )
>>          return rc;
>> diff --git a/xen/arch/arm/gic.c b/xen/arch/arm/gic.c
>> index 4be0897..973fcf9 100644
>> --- a/xen/arch/arm/gic.c
>> +++ b/xen/arch/arm/gic.c
>> @@ -41,6 +41,13 @@
>>  #define GICH ((volatile uint32_t *) FIXMAP_ADDR(FIXMAP_GICH))
>>  static void gic_restore_pending_irqs(struct vcpu *v);
>>
>> +struct gic_state_data {
>> +    uint32_t gic_hcr;
>> +    uint32_t gic_vmcr;
>> +    uint32_t gic_apr;
>> +    uint32_t gic_lr[64];
>> +};
>> +
>>  /* Global state */
>>  static struct {
>>      paddr_t dbase;       /* Address of distributor registers */
>> @@ -98,6 +105,9 @@ irq_desc_t *__irq_to_desc(int irq)
>>  void gic_save_state(struct vcpu *v)
>>  {
>>      int i;
>> +    struct gic_state_data *d;
>> +    d = (struct gic_state_data *)v->arch.gic_state;
>> +
>>      ASSERT(!local_irq_is_enabled());
>>
>>      /* No need for spinlocks here because interrupts are disabled around
>> @@ -105,10 +115,10 @@ void gic_save_state(struct vcpu *v)
>>       * accessed simultaneously by another pCPU.
>>       */
>>      for ( i=0; i<nr_lrs; i++)
>> -        v->arch.gic_lr[i] = GICH[GICH_LR + i];
>> +        d->gic_lr[i] = GICH[GICH_LR + i];
>>      v->arch.lr_mask = this_cpu(lr_mask);
>> -    v->arch.gic_apr = GICH[GICH_APR];
>> -    v->arch.gic_vmcr = GICH[GICH_VMCR];
>> +    d->gic_apr = GICH[GICH_APR];
>> +    d->gic_vmcr = GICH[GICH_VMCR];
>>      /* Disable until next VCPU scheduled */
>>      GICH[GICH_HCR] = 0;
>>      isb();
>> @@ -117,15 +127,17 @@ void gic_save_state(struct vcpu *v)
>>  void gic_restore_state(struct vcpu *v)
>>  {
>>      int i;
>> +    struct gic_state_data *d;
>> +    d = (struct gic_state_data *)v->arch.gic_state;
>>
>>      if ( is_idle_vcpu(v) )
>>          return;
>>
>>      this_cpu(lr_mask) = v->arch.lr_mask;
>>      for ( i=0; i<nr_lrs; i++)
>> -        GICH[GICH_LR + i] = v->arch.gic_lr[i];
>> -    GICH[GICH_APR] = v->arch.gic_apr;
>> -    GICH[GICH_VMCR] = v->arch.gic_vmcr;
>> +        GICH[GICH_LR + i] = d->gic_lr[i];
>> +    GICH[GICH_APR] = d->gic_apr;
>> +    GICH[GICH_VMCR] = d->gic_vmcr;
>>      GICH[GICH_HCR] = GICH_HCR_EN;
>>      isb();
>>
>> @@ -877,6 +889,14 @@ void gic_interrupt(struct cpu_user_regs *regs, int is_fiq)
>>      } while (1);
>>  }
>>
>> +int vcpu_gic_init(struct vcpu *v)
>> +{
>> +     v->arch.gic_state = xzalloc(struct gic_state_data);
>> +     if(!v->arch.gic_state)
>> +        return -ENOMEM;
>> +     return 0;
>> +}
>> +
>>  int gicv_setup(struct domain *d)
>>  {
>>      int ret;
>> @@ -1001,6 +1021,8 @@ void gic_dump_info(struct vcpu *v)
>>  {
>>      int i;
>>      struct pending_irq *p;
>> +    struct gic_state_data *d;
>> +    d = (struct gic_state_data *)v->arch.gic_state;
>>
>>      printk("GICH_LRs (vcpu %d) mask=%"PRIx64"\n", v->vcpu_id, v->arch.lr_mask);
>>      if ( v == current )
>> @@ -1009,7 +1031,7 @@ void gic_dump_info(struct vcpu *v)
>>              printk("   HW_LR[%d]=%x\n", i, GICH[GICH_LR + i]);
>>      } else {
>>          for ( i = 0; i < nr_lrs; i++ )
>> -            printk("   VCPU_LR[%d]=%x\n", i, v->arch.gic_lr[i]);
>> +            printk("   VCPU_LR[%d]=%x\n", i, d->gic_lr[i]);
>>      }
>>
>>      list_for_each_entry ( p, &v->arch.vgic.inflight_irqs, inflight )
>> diff --git a/xen/include/asm-arm/domain.h b/xen/include/asm-arm/domain.h
>> index d49ab68..38df789 100644
>> --- a/xen/include/asm-arm/domain.h
>> +++ b/xen/include/asm-arm/domain.h
>> @@ -243,8 +243,7 @@ struct arch_vcpu
>>      uint32_t csselr;
>>      register_t vmpidr;
>>
>> -    uint32_t gic_hcr, gic_vmcr, gic_apr;
>> -    uint32_t gic_lr[64];
>> +    void *gic_state;
>>      uint64_t event_mask;
>>      uint64_t lr_mask;
>>
>> diff --git a/xen/include/asm-arm/gic.h b/xen/include/asm-arm/gic.h
>> index 340ef73..debfab8 100644
>> --- a/xen/include/asm-arm/gic.h
>> +++ b/xen/include/asm-arm/gic.h
>> @@ -161,6 +161,7 @@ extern int domain_vgic_init(struct domain *d);
>>  extern void domain_vgic_free(struct domain *d);
>>
>>  extern int vcpu_vgic_init(struct vcpu *v);
>> +extern int vcpu_gic_init(struct vcpu *v);
>>
>>  extern void vgic_vcpu_inject_irq(struct vcpu *v, unsigned int irq,int virtual);
>>  extern void vgic_clear_pending_irqs(struct vcpu *v);
>
> Same here about avoiding void*.
> We could move the definition of struct gic_state_data to gic.h and use
> it in domain.h.  We can turn it into a union in the later patches.
Since I made all the changes together, I have directly moved this
to v2 file. Yes, I will try to make it union

[RFC PATCH v1 05/10] xen/arm: move gic definitions to seperate file

[vijay.kilari]

From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>

Move gic v2 register definitions to seperate file
so that gic.h will hold only common definitions
and helps to define gic v3 definitions.

Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
---
 xen/arch/arm/gic.c                |    1 +
 xen/arch/arm/vgic.c               |    1 +
 xen/include/asm-arm/gic.h         |  111 ------------------------------
 xen/include/asm-arm/gic_v2_defs.h |  136 +++++++++++++++++++++++++++++++++++++
 4 files changed, 138 insertions(+), 111 deletions(-)

diff --git a/xen/arch/arm/gic.c b/xen/arch/arm/gic.c
index 973fcf9..bb718f6 100644
--- a/xen/arch/arm/gic.c
+++ b/xen/arch/arm/gic.c
@@ -33,6 +33,7 @@
 #include <asm/domain.h>
 #include <asm/platform.h>
 
+#include <asm/gic_v2_defs.h>
 #include <asm/gic.h>
 
 /* Access to the GIC Distributor registers through the fixmap */
diff --git a/xen/arch/arm/vgic.c b/xen/arch/arm/vgic.c
index 694a15c..9c907b5 100644
--- a/xen/arch/arm/vgic.c
+++ b/xen/arch/arm/vgic.c
@@ -28,6 +28,7 @@
 #include <asm/current.h>
 
 #include "io.h"
+#include <asm/gic_v2_defs.h>
 #include <asm/gic.h>
 
 #define REG(n) (n/4)
diff --git a/xen/include/asm-arm/gic.h b/xen/include/asm-arm/gic.h
index debfab8..18656fd 100644
--- a/xen/include/asm-arm/gic.h
+++ b/xen/include/asm-arm/gic.h
@@ -18,117 +18,6 @@
 #ifndef __ASM_ARM_GIC_H__
 #define __ASM_ARM_GIC_H__
 
-#define GICD_CTLR       (0x000/4)
-#define GICD_TYPER      (0x004/4)
-#define GICD_IIDR       (0x008/4)
-#define GICD_IGROUPR    (0x080/4)
-#define GICD_IGROUPRN   (0x0FC/4)
-#define GICD_ISENABLER  (0x100/4)
-#define GICD_ISENABLERN (0x17C/4)
-#define GICD_ICENABLER  (0x180/4)
-#define GICD_ICENABLERN (0x1fC/4)
-#define GICD_ISPENDR    (0x200/4)
-#define GICD_ISPENDRN   (0x27C/4)
-#define GICD_ICPENDR    (0x280/4)
-#define GICD_ICPENDRN   (0x2FC/4)
-#define GICD_ISACTIVER  (0x300/4)
-#define GICD_ISACTIVERN (0x37C/4)
-#define GICD_ICACTIVER  (0x380/4)
-#define GICD_ICACTIVERN (0x3FC/4)
-#define GICD_IPRIORITYR (0x400/4)
-#define GICD_IPRIORITYRN (0x7F8/4)
-#define GICD_ITARGETSR  (0x800/4)
-#define GICD_ITARGETSRN (0xBF8/4)
-#define GICD_ICFGR      (0xC00/4)
-#define GICD_ICFGRN     (0xCFC/4)
-#define GICD_NSACR      (0xE00/4)
-#define GICD_NSACRN     (0xEFC/4)
-#define GICD_SGIR       (0xF00/4)
-#define GICD_CPENDSGIR  (0xF10/4)
-#define GICD_CPENDSGIRN (0xF1C/4)
-#define GICD_SPENDSGIR  (0xF20/4)
-#define GICD_SPENDSGIRN (0xF2C/4)
-#define GICD_ICPIDR2    (0xFE8/4)
-
-#define GICD_SGI_TARGET_LIST_SHIFT   (24)
-#define GICD_SGI_TARGET_LIST_MASK    (0x3UL << GICD_SGI_TARGET_LIST_SHIFT)
-#define GICD_SGI_TARGET_LIST         (0UL<<GICD_SGI_TARGET_LIST_SHIFT)
-#define GICD_SGI_TARGET_OTHERS       (1UL<<GICD_SGI_TARGET_LIST_SHIFT)
-#define GICD_SGI_TARGET_SELF         (2UL<<GICD_SGI_TARGET_LIST_SHIFT)
-#define GICD_SGI_TARGET_SHIFT        (16)
-#define GICD_SGI_TARGET_MASK         (0xFFUL<<GICD_SGI_TARGET_SHIFT)
-#define GICD_SGI_GROUP1              (1UL<<15)
-#define GICD_SGI_INTID_MASK          (0xFUL)
-
-#define GICC_CTLR       (0x0000/4)
-#define GICC_PMR        (0x0004/4)
-#define GICC_BPR        (0x0008/4)
-#define GICC_IAR        (0x000C/4)
-#define GICC_EOIR       (0x0010/4)
-#define GICC_RPR        (0x0014/4)
-#define GICC_HPPIR      (0x0018/4)
-#define GICC_APR        (0x00D0/4)
-#define GICC_NSAPR      (0x00E0/4)
-#define GICC_DIR        (0x1000/4)
-
-#define GICH_HCR        (0x00/4)
-#define GICH_VTR        (0x04/4)
-#define GICH_VMCR       (0x08/4)
-#define GICH_MISR       (0x10/4)
-#define GICH_EISR0      (0x20/4)
-#define GICH_EISR1      (0x24/4)
-#define GICH_ELSR0      (0x30/4)
-#define GICH_ELSR1      (0x34/4)
-#define GICH_APR        (0xF0/4)
-#define GICH_LR         (0x100/4)
-
-/* Register bits */
-#define GICD_CTL_ENABLE 0x1
-
-#define GICD_TYPE_LINES 0x01f
-#define GICD_TYPE_CPUS  0x0e0
-#define GICD_TYPE_SEC   0x400
-
-#define GICC_CTL_ENABLE 0x1
-#define GICC_CTL_EOI    (0x1 << 9)
-
-#define GICC_IA_IRQ       0x03ff
-#define GICC_IA_CPU_MASK  0x1c00
-#define GICC_IA_CPU_SHIFT 10
-
-#define GICH_HCR_EN       (1 << 0)
-#define GICH_HCR_UIE      (1 << 1)
-#define GICH_HCR_LRENPIE  (1 << 2)
-#define GICH_HCR_NPIE     (1 << 3)
-#define GICH_HCR_VGRP0EIE (1 << 4)
-#define GICH_HCR_VGRP0DIE (1 << 5)
-#define GICH_HCR_VGRP1EIE (1 << 6)
-#define GICH_HCR_VGRP1DIE (1 << 7)
-
-#define GICH_MISR_EOI     (1 << 0)
-#define GICH_MISR_U       (1 << 1)
-#define GICH_MISR_LRENP   (1 << 2)
-#define GICH_MISR_NP      (1 << 3)
-#define GICH_MISR_VGRP0E  (1 << 4)
-#define GICH_MISR_VGRP0D  (1 << 5)
-#define GICH_MISR_VGRP1E  (1 << 6)
-#define GICH_MISR_VGRP1D  (1 << 7)
-
-#define GICH_LR_VIRTUAL_MASK    0x3ff
-#define GICH_LR_VIRTUAL_SHIFT   0
-#define GICH_LR_PHYSICAL_MASK   0x3ff
-#define GICH_LR_PHYSICAL_SHIFT  10
-#define GICH_LR_STATE_MASK      0x3
-#define GICH_LR_STATE_SHIFT     28
-#define GICH_LR_PRIORITY_SHIFT  23
-#define GICH_LR_MAINTENANCE_IRQ (1<<19)
-#define GICH_LR_PENDING         (1<<28)
-#define GICH_LR_ACTIVE          (1<<29)
-#define GICH_LR_GRP1            (1<<30)
-#define GICH_LR_HW              (1<<31)
-#define GICH_LR_CPUID_SHIFT     9
-#define GICH_VTR_NRLRGS         0x3f
-
 /*
  * The minimum GICC_BPR is required to be in the range 0-3. We set
  * GICC_BPR to 0 but we must expect that it might be 3. This means we
diff --git a/xen/include/asm-arm/gic_v2_defs.h b/xen/include/asm-arm/gic_v2_defs.h
new file mode 100644
index 0000000..2366685
--- /dev/null
+++ b/xen/include/asm-arm/gic_v2_defs.h
@@ -0,0 +1,136 @@
+/*
+ * ARM Generic Interrupt Controller support
+ *
+ * Tim Deegan <tim@xen.org>
+ * Copyright (c) 2011 Citrix Systems.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#define GICD_CTLR       (0x000/4)
+#define GICD_TYPER      (0x004/4)
+#define GICD_IIDR       (0x008/4)
+#define GICD_IGROUPR    (0x080/4)
+#define GICD_IGROUPRN   (0x0FC/4)
+#define GICD_ISENABLER  (0x100/4)
+#define GICD_ISENABLERN (0x17C/4)
+#define GICD_ICENABLER  (0x180/4)
+#define GICD_ICENABLERN (0x1fC/4)
+#define GICD_ISPENDR    (0x200/4)
+#define GICD_ISPENDRN   (0x27C/4)
+#define GICD_ICPENDR    (0x280/4)
+#define GICD_ICPENDRN   (0x2FC/4)
+#define GICD_ISACTIVER  (0x300/4)
+#define GICD_ISACTIVERN (0x37C/4)
+#define GICD_ICACTIVER  (0x380/4)
+#define GICD_ICACTIVERN (0x3FC/4)
+#define GICD_IPRIORITYR (0x400/4)
+#define GICD_IPRIORITYRN (0x7F8/4)
+#define GICD_ITARGETSR  (0x800/4)
+#define GICD_ITARGETSRN (0xBF8/4)
+#define GICD_ICFGR      (0xC00/4)
+#define GICD_ICFGRN     (0xCFC/4)
+#define GICD_NSACR      (0xE00/4)
+#define GICD_NSACRN     (0xEFC/4)
+#define GICD_SGIR       (0xF00/4)
+#define GICD_CPENDSGIR  (0xF10/4)
+#define GICD_CPENDSGIRN (0xF1C/4)
+#define GICD_SPENDSGIR  (0xF20/4)
+#define GICD_SPENDSGIRN (0xF2C/4)
+#define GICD_ICPIDR2    (0xFE8/4)
+
+#define GICD_SGI_TARGET_LIST_SHIFT   (24)
+#define GICD_SGI_TARGET_LIST_MASK    (0x3UL << GICD_SGI_TARGET_LIST_SHIFT)
+#define GICD_SGI_TARGET_LIST         (0UL<<GICD_SGI_TARGET_LIST_SHIFT)
+#define GICD_SGI_TARGET_OTHERS       (1UL<<GICD_SGI_TARGET_LIST_SHIFT)
+#define GICD_SGI_TARGET_SELF         (2UL<<GICD_SGI_TARGET_LIST_SHIFT)
+#define GICD_SGI_TARGET_SHIFT        (16)
+#define GICD_SGI_TARGET_MASK         (0xFFUL<<GICD_SGI_TARGET_SHIFT)
+#define GICD_SGI_GROUP1              (1UL<<15)
+#define GICD_SGI_INTID_MASK          (0xFUL)
+
+#define GICC_CTLR       (0x0000/4)
+#define GICC_PMR        (0x0004/4)
+#define GICC_BPR        (0x0008/4)
+#define GICC_IAR        (0x000C/4)
+#define GICC_EOIR       (0x0010/4)
+#define GICC_RPR        (0x0014/4)
+#define GICC_HPPIR      (0x0018/4)
+#define GICC_APR        (0x00D0/4)
+#define GICC_NSAPR      (0x00E0/4)
+#define GICC_DIR        (0x1000/4)
+
+#define GICH_HCR        (0x00/4)
+#define GICH_VTR        (0x04/4)
+#define GICH_VMCR       (0x08/4)
+#define GICH_MISR       (0x10/4)
+#define GICH_EISR0      (0x20/4)
+#define GICH_EISR1      (0x24/4)
+#define GICH_ELSR0      (0x30/4)
+#define GICH_ELSR1      (0x34/4)
+#define GICH_APR        (0xF0/4)
+#define GICH_LR         (0x100/4)
+
+/* Register bits */
+#define GICD_CTL_ENABLE 0x1
+
+#define GICD_TYPE_LINES 0x01f
+#define GICD_TYPE_CPUS  0x0e0
+#define GICD_TYPE_SEC   0x400
+
+#define GICC_CTL_ENABLE 0x1
+#define GICC_CTL_EOI    (0x1 << 9)
+
+#define GICC_IA_IRQ       0x03ff
+#define GICC_IA_CPU_MASK  0x1c00
+#define GICC_IA_CPU_SHIFT 10
+
+#define GICH_HCR_EN       (1 << 0)
+#define GICH_HCR_UIE      (1 << 1)
+#define GICH_HCR_LRENPIE  (1 << 2)
+#define GICH_HCR_NPIE     (1 << 3)
+#define GICH_HCR_VGRP0EIE (1 << 4)
+#define GICH_HCR_VGRP0DIE (1 << 5)
+#define GICH_HCR_VGRP1EIE (1 << 6)
+#define GICH_HCR_VGRP1DIE (1 << 7)
+
+#define GICH_MISR_EOI     (1 << 0)
+#define GICH_MISR_U       (1 << 1)
+#define GICH_MISR_LRENP   (1 << 2)
+#define GICH_MISR_NP      (1 << 3)
+#define GICH_MISR_VGRP0E  (1 << 4)
+#define GICH_MISR_VGRP0D  (1 << 5)
+#define GICH_MISR_VGRP1E  (1 << 6)
+#define GICH_MISR_VGRP1D  (1 << 7)
+
+#define GICH_LR_VIRTUAL_MASK    0x3ff
+#define GICH_LR_VIRTUAL_SHIFT   0
+#define GICH_LR_PHYSICAL_MASK   0x3ff
+#define GICH_LR_PHYSICAL_SHIFT  10
+#define GICH_LR_STATE_MASK      0x3
+#define GICH_LR_STATE_SHIFT     28
+#define GICH_LR_PRIORITY_SHIFT  23
+#define GICH_LR_MAINTENANCE_IRQ (1<<19)
+#define GICH_LR_PENDING         (1<<28)
+#define GICH_LR_ACTIVE          (1<<29)
+#define GICH_LR_GRP1            (1<<30)
+#define GICH_LR_HW              (1<<31)
+#define GICH_LR_CPUID_SHIFT     9
+#define GICH_VTR_NRLRGS         0x3f
+
+/*
+ * Local variables:
+ * mode: C
+ * c-file-style: "BSD"
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
-- 
1.7.9.5

Re: [RFC PATCH v1 05/10] xen/arm: move gic definitions to seperate file

[Julien Grall]

Hello Vijay,

Thanks for the patch.

On 03/19/2014 02:17 PM, vijay.kilari@gmail.com wrote:
> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> 
> Move gic v2 register definitions to seperate file

separate

> so that gic.h will hold only common definitions
> and helps to define gic v3 definitions.
> 
> Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> ---
> diff --git a/xen/include/asm-arm/gic_v2_defs.h b/xen/include/asm-arm/gic_v2_defs.h
> new file mode 100644
> index 0000000..2366685
> --- /dev/null
> +++ b/xen/include/asm-arm/gic_v2_defs.h
> @@ -0,0 +1,136 @@
> +/*
> + * ARM Generic Interrupt Controller support
> + *
> + * Tim Deegan <tim@xen.org>
> + * Copyright (c) 2011 Citrix Systems.
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License as published by
> + * the Free Software Foundation; either version 2 of the License, or
> + * (at your option) any later version.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> + * GNU General Public License for more details.
> + */
> +

Please add guard for this header.

Regards,

-- 
Julien Grall

[RFC PATCH v1 06/10] xen/arm: split gic driver into generic and gicv2 driver

[vijay.kilari]

From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>

Existing GIC driver has both generic code and hw specific
code. Segregate GIC low level driver into gic-v2.c and
keep generic code in existing gic.c file

GIC v2 driver registers required functions
to generic GIC driver. This helps to plug in next version
of GIC drivers like GIC v3.

Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
---
 xen/arch/arm/Makefile             |    2 +-
 xen/arch/arm/gic-v2.c             |  562 +++++++++++++++++++++++++++++++++++++
 xen/arch/arm/gic.c                |  530 +++++++---------------------------
 xen/include/asm-arm/gic.h         |   40 ++-
 xen/include/asm-arm/gic_v2_defs.h |    2 -
 5 files changed, 708 insertions(+), 428 deletions(-)

diff --git a/xen/arch/arm/Makefile b/xen/arch/arm/Makefile
index 63e0460..969ee52 100644
--- a/xen/arch/arm/Makefile
+++ b/xen/arch/arm/Makefile
@@ -10,7 +10,7 @@ obj-y += vpsci.o
 obj-y += domctl.o
 obj-y += sysctl.o
 obj-y += domain_build.o
-obj-y += gic.o
+obj-y += gic.o gic-v2.o
 obj-y += io.o
 obj-y += irq.o
 obj-y += kernel.o
diff --git a/xen/arch/arm/gic-v2.c b/xen/arch/arm/gic-v2.c
new file mode 100644
index 0000000..9378427
--- /dev/null
+++ b/xen/arch/arm/gic-v2.c
@@ -0,0 +1,562 @@
+/*
+ * xen/arch/arm/gic-v2.c
+ *
+ * ARM Generic Interrupt Controller support v2
+ *
+ * Tim Deegan <tim@xen.org>
+ * Copyright (c) 2011 Citrix Systems.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <xen/config.h>
+#include <xen/lib.h>
+#include <xen/init.h>
+#include <xen/cpu.h>
+#include <xen/mm.h>
+#include <xen/irq.h>
+#include <xen/sched.h>
+#include <xen/errno.h>
+#include <xen/serial.h>
+#include <xen/softirq.h>
+#include <xen/list.h>
+#include <xen/device_tree.h>
+#include <asm/p2m.h>
+#include <asm/domain.h>
+#include <asm/platform.h>
+
+#include <asm/gic_v2_defs.h>
+#include <asm/gic.h>
+
+/* Access to the GIC Distributor registers through the fixmap */
+#define GICD ((volatile uint32_t *) FIXMAP_ADDR(FIXMAP_GICD))
+#define GICC ((volatile uint32_t *) FIXMAP_ADDR(FIXMAP_GICC1))
+#define GICH ((volatile uint32_t *) FIXMAP_ADDR(FIXMAP_GICH))
+
+struct gic_state_data {
+    uint32_t gic_hcr;
+    uint32_t gic_vmcr;
+    uint32_t gic_apr;
+    uint32_t gic_lr[64];
+};
+
+/* Global state */
+static struct {
+    int hw_version;
+    paddr_t dbase;       /* Address of distributor registers */
+    paddr_t cbase;       /* Address of CPU interface registers */
+    paddr_t hbase;       /* Address of virtual interface registers */
+    paddr_t vbase;       /* Address of virtual cpu interface registers */
+    unsigned int lines;  /* Number of interrupts (SPIs + PPIs + SGIs) */
+    struct dt_irq maintenance; /* IRQ maintenance */
+    unsigned int cpus;
+    spinlock_t lock;
+} gic;
+
+static unsigned nr_lrs;
+
+/* The GIC mapping of CPU interfaces does not necessarily match the
+ * logical CPU numbering. Let's use mapping as returned by the GIC
+ * itself
+ */
+static DEFINE_PER_CPU(u8, gic_cpu_id);
+
+/* Maximum cpu interface per GIC */
+#define NR_GIC_CPU_IF 8
+
+static unsigned int gic_cpu_mask(const cpumask_t *cpumask)
+{
+    unsigned int cpu;
+    unsigned int mask = 0;
+    cpumask_t possible_mask;
+
+    cpumask_and(&possible_mask, cpumask, &cpu_possible_map);
+    for_each_cpu(cpu, &possible_mask)
+    {
+        ASSERT(cpu < NR_GIC_CPU_IF);
+        mask |= per_cpu(gic_cpu_id, cpu);
+    }
+
+    return mask;
+}
+
+static unsigned int gic_nr_lines(void)
+{
+    return gic.lines;
+}
+
+static unsigned int gic_nr_lrs(void)
+{
+    return nr_lrs;
+}
+
+static int gic_state_init(struct vcpu *v)
+{
+     v->arch.gic_state = (struct gic_state_data *)xzalloc(struct gic_state_data);
+     if(!v->arch.gic_state)
+        return -ENOMEM;
+     return 0;
+}
+
+static void save_state(struct vcpu *v)
+{
+    int i;
+    struct gic_state_data *d;
+    d = (struct gic_state_data *)v->arch.gic_state;
+
+    ASSERT(!local_irq_is_enabled());
+
+    /* No need for spinlocks here because interrupts are disabled around
+     * this call and it only accesses struct vcpu fields that cannot be
+     * accessed simultaneously by another pCPU.
+     */
+    for ( i = 0; i < nr_lrs; i++)
+        d->gic_lr[i] = GICH[GICH_LR + i];
+    d->gic_apr = GICH[GICH_APR];
+    d->gic_vmcr = GICH[GICH_VMCR];
+    /* Disable until next VCPU scheduled */
+    GICH[GICH_HCR] = 0;
+}
+
+static void restore_state(struct vcpu *v)
+{
+    int i;
+    struct gic_state_data *d;
+    d = (struct gic_state_data *)v->arch.gic_state;
+
+    for ( i = 0; i < nr_lrs; i++)
+        GICH[GICH_LR + i] = d->gic_lr[i];
+    GICH[GICH_APR] = d->gic_apr;
+    GICH[GICH_VMCR] = d->gic_vmcr;
+    GICH[GICH_HCR] = GICH_HCR_EN;
+}
+
+static void gic_dump_state(struct vcpu *v)
+{
+    int i;
+    struct gic_state_data *d;
+    d = (struct gic_state_data *)v->arch.gic_state;
+
+    if ( v == current )
+    {
+        for ( i = 0; i < nr_lrs; i++ )
+            printk("   HW_LR[%d]=%x\n", i, GICH[GICH_LR + i]);
+    } else {
+        for ( i = 0; i < nr_lrs; i++ )
+            printk("   VCPU_LR[%d]=%x\n", i, d->gic_lr[i]);
+    }
+}
+
+static void gic_enable_irq(int irq)
+{
+    /* Enable routing */
+    GICD[GICD_ISENABLER + irq / 32] = (1u << (irq % 32));
+}
+
+static void gic_disable_irq(int irq)
+{
+    /* Disable routing */
+    GICD[GICD_ICENABLER + irq / 32] = (1u << (irq % 32));
+}
+
+static void gic_eoi_irq(int irq)
+{
+    /* Lower the priority */
+    GICC[GICC_EOIR] = irq;
+}
+
+static void gic_dir_irq(int irq)
+{
+    /* Deactivate */
+    GICC[GICC_DIR] = irq;
+}
+
+static unsigned int gic_ack_irq(void)
+{
+    return (GICC[GICC_IAR] & GICC_IA_IRQ);
+}
+
+/*
+ * - needs to be called with gic.lock held
+ * - needs to be called with a valid cpu_mask, ie each cpu in the mask has
+ * already called gic_cpu_init
+ */
+static void gic_set_irq_properties(unsigned int irq, bool_t level,
+                                   const cpumask_t *cpu_mask,
+                                   unsigned int priority)
+{
+    volatile unsigned char *bytereg;
+    uint32_t cfg, edgebit;
+    unsigned int mask = gic_cpu_mask(cpu_mask);
+
+    /* Set edge / level */
+    cfg = GICD[GICD_ICFGR + irq / 16];
+    edgebit = 2u << (2 * (irq % 16));
+    if ( level )
+        cfg &= ~edgebit;
+    else
+        cfg |= edgebit;
+    GICD[GICD_ICFGR + irq / 16] = cfg;
+
+    /* Set target CPU mask (RAZ/WI on uniprocessor) */
+    bytereg = (unsigned char *) (GICD + GICD_ITARGETSR);
+    bytereg[irq] = mask;
+
+    /* Set priority */
+    bytereg = (unsigned char *) (GICD + GICD_IPRIORITYR);
+    bytereg[irq] = priority;
+
+}
+
+static void __init gic_dist_init(void)
+{
+    uint32_t type;
+    uint32_t cpumask;
+    int i;
+
+    cpumask = GICD[GICD_ITARGETSR] & 0xff;
+    cpumask |= cpumask << 8;
+    cpumask |= cpumask << 16;
+
+    /* Disable the distributor */
+    GICD[GICD_CTLR] = 0;
+
+    type = GICD[GICD_TYPER];
+    gic.lines = 32 * ((type & GICD_TYPE_LINES) + 1);
+    gic.cpus = 1 + ((type & GICD_TYPE_CPUS) >> 5);
+    printk("GIC: %d lines, %d cpu%s%s (IID %8.8x).\n",
+           gic.lines, gic.cpus, (gic.cpus == 1) ? "" : "s",
+           (type & GICD_TYPE_SEC) ? ", secure" : "",
+           GICD[GICD_IIDR]);
+
+    /* Default all global IRQs to level, active low */
+    for ( i = 32; i < gic.lines; i += 16 )
+        GICD[GICD_ICFGR + i / 16] = 0x0;
+
+    /* Route all global IRQs to this CPU */
+    for ( i = 32; i < gic.lines; i += 4 )
+        GICD[GICD_ITARGETSR + i / 4] = cpumask;
+
+    /* Default priority for global interrupts */
+    for ( i = 32; i < gic.lines; i += 4 )
+        GICD[GICD_IPRIORITYR + i / 4] =
+            GIC_PRI_IRQ<<24 | GIC_PRI_IRQ<<16 | GIC_PRI_IRQ<<8 | GIC_PRI_IRQ;
+
+    /* Disable all global interrupts */
+    for ( i = 32; i < gic.lines; i += 32 )
+        GICD[GICD_ICENABLER + i / 32] = (uint32_t)~0ul;
+
+    /* Turn on the distributor */
+    GICD[GICD_CTLR] = GICD_CTL_ENABLE;
+}
+
+static void __cpuinit gic_cpu_init(void)
+{
+    int i;
+
+    this_cpu(gic_cpu_id) = GICD[GICD_ITARGETSR] & 0xff;
+
+    /* The first 32 interrupts (PPI and SGI) are banked per-cpu, so
+     * even though they are controlled with GICD registers, they must
+     * be set up here with the other per-cpu state. */
+    GICD[GICD_ICENABLER] = 0xffff0000; /* Disable all PPI */
+    GICD[GICD_ISENABLER] = 0x0000ffff; /* Enable all SGI */
+    /* Set SGI priorities */
+    for (i = 0; i < 16; i += 4)
+        GICD[GICD_IPRIORITYR + i / 4] =
+            GIC_PRI_IPI<<24 | GIC_PRI_IPI<<16 | GIC_PRI_IPI<<8 | GIC_PRI_IPI;
+    /* Set PPI priorities */
+    for (i = 16; i < 32; i += 4)
+        GICD[GICD_IPRIORITYR + i / 4] =
+            GIC_PRI_IRQ<<24 | GIC_PRI_IRQ<<16 | GIC_PRI_IRQ<<8 | GIC_PRI_IRQ;
+
+    /* Local settings: interface controller */
+    GICC[GICC_PMR] = 0xff;                /* Don't mask by priority */
+    GICC[GICC_BPR] = 0;                   /* Finest granularity of priority */
+    GICC[GICC_CTLR] = GICC_CTL_ENABLE|GICC_CTL_EOI;    /* Turn on delivery */
+}
+
+static void gic_cpu_disable(void)
+{
+    GICC[GICC_CTLR] = 0;
+}
+
+static void __cpuinit gic_hyp_init(void)
+{
+    uint32_t vtr;
+
+    vtr = GICH[GICH_VTR];
+    nr_lrs  = (vtr & GICH_VTR_NRLRGS) + 1;
+
+    GICH[GICH_MISR] = GICH_MISR_EOI;
+    update_cpu_lr_mask();
+}
+
+static void __cpuinit gic_hyp_disable(void)
+{
+    GICH[GICH_HCR] = 0;
+}
+
+/* Set up the per-CPU parts of the GIC for a secondary CPU */
+static int __cpuinit gic_init_secondary_cpu(struct notifier_block *nfb,
+                                       unsigned long action, void *hcpu)
+{
+    if (action == CPU_STARTING)
+    {
+        spin_lock(&gic.lock);
+        gic_cpu_init();
+        gic_hyp_init();
+        spin_unlock(&gic.lock);
+    }
+    return NOTIFY_DONE;
+}
+
+static struct notifier_block gic_cpu_nb = {
+    .notifier_call = gic_init_secondary_cpu,
+    .priority = 100
+};
+
+static void gic_smp_init(void)
+{
+   register_cpu_notifier(&gic_cpu_nb);
+}
+
+static int gic_hw_type(void)
+{
+    return gic.hw_version;
+}
+
+static struct dt_irq * gic_maintenance_irq(void)
+{
+    return &gic.maintenance;
+}
+
+/* Set up the GIC */
+
+static void gic_send_sgi(const cpumask_t *cpumask, enum gic_sgi sgi)
+{
+    unsigned int mask = 0;
+    cpumask_t online_mask;
+
+    ASSERT(sgi < 16); /* There are only 16 SGIs */
+
+    cpumask_and(&online_mask, cpumask, &cpu_online_map);
+    mask = gic_cpu_mask(&online_mask);
+
+    dsb();
+
+    GICD[GICD_SGIR] = GICD_SGI_TARGET_LIST
+        | (mask<<GICD_SGI_TARGET_SHIFT)
+        | sgi;
+}
+
+/* Shut down the per-CPU GIC interface */
+static void gic_disable_interface(void)
+{
+    ASSERT(!local_irq_is_enabled());
+
+    spin_lock(&gic.lock);
+    gic_cpu_disable();
+    gic_hyp_disable();
+    spin_unlock(&gic.lock);
+}
+
+static void gic_update_lr(int lr, unsigned int virtual_irq,
+        unsigned int state, unsigned int priority)
+{
+    int maintenance_int = GICH_LR_MAINTENANCE_IRQ;
+
+    BUG_ON(lr >= nr_lrs);
+    BUG_ON(lr < 0);
+    BUG_ON(state & ~(GICH_LR_STATE_MASK<<GICH_LR_STATE_SHIFT));
+
+    GICH[GICH_LR + lr] = ((state & 0x3) << GICH_LR_STATE_SHIFT) |
+        maintenance_int |
+        ((priority >> 3) << GICH_LR_PRIORITY_SHIFT) |
+        ((virtual_irq & GICH_LR_VIRTUAL_MASK) << GICH_LR_VIRTUAL_SHIFT);
+
+}
+
+static int gicv_init(struct domain *d)
+{
+    int ret;
+
+    /*
+     * Domain 0 gets the hardware address.
+     * Guests get the virtual platform layout.
+     */
+    if ( d->domain_id == 0 )
+    {
+        d->arch.vgic.dbase = gic.dbase;
+        d->arch.vgic.cbase = gic.cbase;
+    }
+    else
+    {
+        d->arch.vgic.dbase = GUEST_GICD_BASE;
+        d->arch.vgic.cbase = GUEST_GICC_BASE;
+    }
+
+    d->arch.vgic.nr_lines = 0;
+
+    /*
+     * Map the gic virtual cpu interface in the gic cpu interface
+     * region of the guest.
+     *
+     * The second page is always mapped at +4K irrespective of the
+     * GIC_64K_STRIDE quirk. The DTB passed to the guest reflects this.
+     */
+    ret = map_mmio_regions(d, d->arch.vgic.cbase,
+                           d->arch.vgic.cbase + PAGE_SIZE - 1,
+                           gic.vbase);
+    if (ret)
+        return ret;
+
+    if ( !platform_has_quirk(PLATFORM_QUIRK_GIC_64K_STRIDE) )
+        ret = map_mmio_regions(d, d->arch.vgic.cbase + PAGE_SIZE,
+                               d->arch.vgic.cbase + (2 * PAGE_SIZE) - 1,
+                               gic.vbase + PAGE_SIZE);
+    else
+        ret = map_mmio_regions(d, d->arch.vgic.cbase + PAGE_SIZE,
+                               d->arch.vgic.cbase + (2 * PAGE_SIZE) - 1,
+                               gic.vbase + 16*PAGE_SIZE);
+
+    return ret;
+
+}
+
+static unsigned long gic_read_eisr(void)
+{
+    return ((unsigned long)(GICH[GICH_EISR0] | (((uint64_t) GICH[GICH_EISR1]) << 32)));
+}
+
+static unsigned int gic_update_lr_for_mi(int lr)
+{
+    u64 val;
+    uint32_t virq;
+
+    spin_lock_irq(&gic.lock);
+    val = GICH[GICH_LR + lr];
+    virq = val & GICH_LR_VIRTUAL_MASK;
+    GICH[GICH_LR + lr] = 0;
+    spin_unlock_irq(&gic.lock);
+    return virq;
+}
+
+static struct gic_hw_operations gic_ops = {
+    .gic_type            = gic_hw_type,
+    .nr_lines            = gic_nr_lines,
+    .nr_lrs              = gic_nr_lrs,
+    .get_maintenance_irq = gic_maintenance_irq,
+    .state_init        = gic_state_init,
+    .save_state          = save_state,
+    .restore_state       = restore_state,
+    .dump_state          = gic_dump_state,
+    .gicv_setup          = gicv_init,
+    .enable_irq          = gic_enable_irq,
+    .disable_irq         = gic_disable_irq,
+    .eoi_irq             = gic_eoi_irq,
+    .deactivate_irq      = gic_dir_irq,
+    .ack_irq             = gic_ack_irq,
+    .set_irq_property    = gic_set_irq_properties,
+    .send_sgi            = gic_send_sgi,
+    .disable_interface   = gic_disable_interface,
+    .update_lr           = gic_update_lr,
+    .update_lr_for_mi    = gic_update_lr_for_mi,
+    .read_eisr           = gic_read_eisr,
+};
+
+void __init gicv2_init(void)
+{
+    static const struct dt_device_match gic_ids[] __initconst =
+    {
+        DT_MATCH_GIC,
+        { /* sentinel */ },
+    };
+    struct dt_device_node *node;
+    int res;
+
+    node = dt_find_interrupt_controller(gic_ids);
+    if ( !node )
+        panic("Unable to find compatible GIC in the device tree");
+
+    dt_device_set_used_by(node, DOMID_XEN);
+
+    gic.hw_version = GIC_VERSION_V2;
+
+    res = dt_device_get_address(node, 0, &gic.dbase, NULL);
+    if ( res || !gic.dbase || (gic.dbase & ~PAGE_MASK) )
+        panic("GIC: Cannot find a valid address for the distributor");
+
+    res = dt_device_get_address(node, 1, &gic.cbase, NULL);
+    if ( res || !gic.cbase || (gic.cbase & ~PAGE_MASK) )
+        panic("GIC: Cannot find a valid address for the CPU");
+
+    res = dt_device_get_address(node, 2, &gic.hbase, NULL);
+    if ( res || !gic.hbase || (gic.hbase & ~PAGE_MASK) )
+        panic("GIC: Cannot find a valid address for the hypervisor");
+
+    res = dt_device_get_address(node, 3, &gic.vbase, NULL);
+    if ( res || !gic.vbase || (gic.vbase & ~PAGE_MASK) )
+        panic("GIC: Cannot find a valid address for the virtual CPU");
+
+    res = dt_device_get_irq(node, 0, &gic.maintenance);
+    if ( res )
+        panic("GIC: Cannot find the maintenance IRQ");
+
+    /* Set the GIC as the primary interrupt controller */
+    dt_interrupt_controller = node;
+
+    /* TODO: Add check on distributor, cpu size */
+
+    printk("GIC initialization:\n"
+              "        gic_dist_addr=%"PRIpaddr"\n"
+              "        gic_cpu_addr=%"PRIpaddr"\n"
+              "        gic_hyp_addr=%"PRIpaddr"\n"
+              "        gic_vcpu_addr=%"PRIpaddr"\n"
+              "        gic_maintenance_irq=%u\n",
+              gic.dbase, gic.cbase, gic.hbase, gic.vbase,
+              gic.maintenance.irq);
+
+    if ( (gic.dbase & ~PAGE_MASK) || (gic.cbase & ~PAGE_MASK) ||
+         (gic.hbase & ~PAGE_MASK) || (gic.vbase & ~PAGE_MASK) )
+        panic("GIC interfaces not page aligned");
+
+    set_fixmap(FIXMAP_GICD, gic.dbase >> PAGE_SHIFT, DEV_SHARED);
+    BUILD_BUG_ON(FIXMAP_ADDR(FIXMAP_GICC1) !=
+                 FIXMAP_ADDR(FIXMAP_GICC2)-PAGE_SIZE);
+    set_fixmap(FIXMAP_GICC1, gic.cbase >> PAGE_SHIFT, DEV_SHARED);
+    if ( platform_has_quirk(PLATFORM_QUIRK_GIC_64K_STRIDE) )
+        set_fixmap(FIXMAP_GICC2, (gic.cbase >> PAGE_SHIFT) + 0x10, DEV_SHARED);
+    else
+        set_fixmap(FIXMAP_GICC2, (gic.cbase >> PAGE_SHIFT) + 0x1, DEV_SHARED);
+    set_fixmap(FIXMAP_GICH, gic.hbase >> PAGE_SHIFT, DEV_SHARED);
+
+    /* Global settings: interrupt distributor */
+    spin_lock_init(&gic.lock);
+    spin_lock(&gic.lock);
+
+    gic_smp_init();
+    gic_dist_init();
+    gic_cpu_init();
+    gic_hyp_init();
+
+    register_gic_ops(&gic_ops);
+    spin_unlock(&gic.lock);
+}
+
+/*
+ * Local variables:
+ * mode: C
+ * c-file-style: "BSD"
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff --git a/xen/arch/arm/gic.c b/xen/arch/arm/gic.c
index bb718f6..e0859ae 100644
--- a/xen/arch/arm/gic.c
+++ b/xen/arch/arm/gic.c
@@ -33,68 +33,46 @@
 #include <asm/domain.h>
 #include <asm/platform.h>
 
-#include <asm/gic_v2_defs.h>
 #include <asm/gic.h>
 
-/* Access to the GIC Distributor registers through the fixmap */
-#define GICD ((volatile uint32_t *) FIXMAP_ADDR(FIXMAP_GICD))
-#define GICC ((volatile uint32_t *) FIXMAP_ADDR(FIXMAP_GICC1))
-#define GICH ((volatile uint32_t *) FIXMAP_ADDR(FIXMAP_GICH))
 static void gic_restore_pending_irqs(struct vcpu *v);
 
-struct gic_state_data {
-    uint32_t gic_hcr;
-    uint32_t gic_vmcr;
-    uint32_t gic_apr;
-    uint32_t gic_lr[64];
-};
-
-/* Global state */
-static struct {
-    paddr_t dbase;       /* Address of distributor registers */
-    paddr_t cbase;       /* Address of CPU interface registers */
-    paddr_t hbase;       /* Address of virtual interface registers */
-    paddr_t vbase;       /* Address of virtual cpu interface registers */
-    unsigned int lines;  /* Number of interrupts (SPIs + PPIs + SGIs) */
-    struct dt_irq maintenance; /* IRQ maintenance */
-    unsigned int cpus;
-    spinlock_t lock;
-} gic;
-
 static irq_desc_t irq_desc[NR_IRQS];
 static DEFINE_PER_CPU(irq_desc_t[NR_LOCAL_IRQS], local_irq_desc);
 static DEFINE_PER_CPU(uint64_t, lr_mask);
 
-static unsigned nr_lrs;
-
-/* The GIC mapping of CPU interfaces does not necessarily match the
- * logical CPU numbering. Let's use mapping as returned by the GIC
- * itself
- */
-static DEFINE_PER_CPU(u8, gic_cpu_id);
+static struct gic_hw_operations *gic_hw_ops;
 
-/* Maximum cpu interface per GIC */
-#define NR_GIC_CPU_IF 8
+spinlock_t gic_lock;
 
-static unsigned int gic_cpu_mask(const cpumask_t *cpumask)
+void register_gic_ops(struct gic_hw_operations *ops)
 {
-    unsigned int cpu;
-    unsigned int mask = 0;
-    cpumask_t possible_mask;
+    gic_hw_ops = ops;
+}
 
-    cpumask_and(&possible_mask, cpumask, &cpu_possible_map);
-    for_each_cpu(cpu, &possible_mask)
-    {
-        ASSERT(cpu < NR_GIC_CPU_IF);
-        mask |= per_cpu(gic_cpu_id, cpu);
-    }
+void update_cpu_lr_mask(void)
+{    
+    this_cpu(lr_mask) = 0ULL;
+}
 
-    return mask;
+int gic_hw_version(void)
+{
+   return gic_hw_ops->gic_type();
 }
 
 unsigned int gic_number_lines(void)
 {
-    return gic.lines;
+    return gic_hw_ops->nr_lines();
+}
+
+unsigned long gic_data_rdist_rd_base(void)
+{
+   return gic_hw_ops->read_cpu_rbase();
+}
+
+unsigned long gic_data_rdist_sgi_base(void)
+{
+   return gic_hw_ops->read_cpu_sgi_rbase();
 }
 
 irq_desc_t *__irq_to_desc(int irq)
@@ -105,43 +83,21 @@ irq_desc_t *__irq_to_desc(int irq)
 
 void gic_save_state(struct vcpu *v)
 {
-    int i;
-    struct gic_state_data *d;
-    d = (struct gic_state_data *)v->arch.gic_state;
-
     ASSERT(!local_irq_is_enabled());
 
-    /* No need for spinlocks here because interrupts are disabled around
-     * this call and it only accesses struct vcpu fields that cannot be
-     * accessed simultaneously by another pCPU.
-     */
-    for ( i=0; i<nr_lrs; i++)
-        d->gic_lr[i] = GICH[GICH_LR + i];
     v->arch.lr_mask = this_cpu(lr_mask);
-    d->gic_apr = GICH[GICH_APR];
-    d->gic_vmcr = GICH[GICH_VMCR];
-    /* Disable until next VCPU scheduled */
-    GICH[GICH_HCR] = 0;
+    gic_hw_ops->save_state(v);
     isb();
 }
 
 void gic_restore_state(struct vcpu *v)
 {
-    int i;
-    struct gic_state_data *d;
-    d = (struct gic_state_data *)v->arch.gic_state;
-
     if ( is_idle_vcpu(v) )
         return;
 
     this_cpu(lr_mask) = v->arch.lr_mask;
-    for ( i=0; i<nr_lrs; i++)
-        GICH[GICH_LR + i] = d->gic_lr[i];
-    GICH[GICH_APR] = d->gic_apr;
-    GICH[GICH_VMCR] = d->gic_vmcr;
-    GICH[GICH_HCR] = GICH_HCR_EN;
+    gic_hw_ops->restore_state(v);
     isb();
-
     gic_restore_pending_irqs(v);
 }
 
@@ -151,12 +107,12 @@ static void gic_irq_enable(struct irq_desc *desc)
     unsigned long flags;
 
     spin_lock_irqsave(&desc->lock, flags);
-    spin_lock(&gic.lock);
+    spin_lock(&gic_lock);
     desc->status &= ~IRQ_DISABLED;
     dsb();
     /* Enable routing */
-    GICD[GICD_ISENABLER + irq / 32] = (1u << (irq % 32));
-    spin_unlock(&gic.lock);
+    gic_hw_ops->enable_irq(irq);
+    spin_unlock(&gic_lock);
     spin_unlock_irqrestore(&desc->lock, flags);
 }
 
@@ -166,11 +122,11 @@ static void gic_irq_disable(struct irq_desc *desc)
     unsigned long flags;
 
     spin_lock_irqsave(&desc->lock, flags);
-    spin_lock(&gic.lock);
+    spin_lock(&gic_lock);
     /* Disable routing */
-    GICD[GICD_ICENABLER + irq / 32] = (1u << (irq % 32));
+    gic_hw_ops->disable_irq(irq);
     desc->status |= IRQ_DISABLED;
-    spin_unlock(&gic.lock);
+    spin_unlock(&gic_lock);
     spin_unlock_irqrestore(&desc->lock, flags);
 }
 
@@ -194,17 +150,16 @@ static void gic_host_irq_end(struct irq_desc *desc)
 {
     int irq = desc->irq;
     /* Lower the priority */
-    GICC[GICC_EOIR] = irq;
-    /* Deactivate */
-    GICC[GICC_DIR] = irq;
+    gic_hw_ops->eoi_irq(irq);
+    gic_hw_ops->deactivate_irq(irq);
 }
 
 static void gic_guest_irq_end(struct irq_desc *desc)
 {
     int irq = desc->irq;
     /* Lower the priority of the IRQ */
-    GICC[GICC_EOIR] = irq;
     /* Deactivation happens in maintenance interrupt / via GICV */
+    gic_hw_ops->eoi_irq(irq);
 }
 
 static void gic_irq_set_affinity(struct irq_desc *desc, const cpumask_t *mask)
@@ -243,27 +198,7 @@ static void gic_set_irq_properties(unsigned int irq, bool_t level,
                                    const cpumask_t *cpu_mask,
                                    unsigned int priority)
 {
-    volatile unsigned char *bytereg;
-    uint32_t cfg, edgebit;
-    unsigned int mask = gic_cpu_mask(cpu_mask);
-
-    /* Set edge / level */
-    cfg = GICD[GICD_ICFGR + irq / 16];
-    edgebit = 2u << (2 * (irq % 16));
-    if ( level )
-        cfg &= ~edgebit;
-    else
-        cfg |= edgebit;
-    GICD[GICD_ICFGR + irq / 16] = cfg;
-
-    /* Set target CPU mask (RAZ/WI on uniprocessor) */
-    bytereg = (unsigned char *) (GICD + GICD_ITARGETSR);
-    bytereg[irq] = mask;
-
-    /* Set priority */
-    bytereg = (unsigned char *) (GICD + GICD_IPRIORITYR);
-    bytereg[irq] = priority;
-
+    return gic_hw_ops->set_irq_property(irq, level, cpu_mask, priority);
 }
 
 /* Program the GIC to route an interrupt */
@@ -273,8 +208,8 @@ static int gic_route_irq(unsigned int irq, bool_t level,
     struct irq_desc *desc = irq_to_desc(irq);
     unsigned long flags;
 
-    ASSERT(priority <= 0xff);     /* Only 8 bits of priority */
-    ASSERT(irq < gic.lines);      /* Can't route interrupts that don't exist */
+    ASSERT(priority <= 0xff);  /* Only 8 bits of priority */
+    ASSERT(irq < gic_number_lines());
 
     if ( desc->action != NULL )
         return -EBUSY;
@@ -286,9 +221,9 @@ static int gic_route_irq(unsigned int irq, bool_t level,
 
     desc->handler = &gic_host_irq_type;
 
-    spin_lock(&gic.lock);
+    spin_lock(&gic_lock);
     gic_set_irq_properties(irq, level, cpu_mask, priority);
-    spin_unlock(&gic.lock);
+    spin_unlock(&gic_lock);
 
     spin_unlock_irqrestore(&desc->lock, flags);
     return 0;
@@ -305,119 +240,6 @@ void gic_route_dt_irq(const struct dt_irq *irq, const cpumask_t *cpu_mask,
     gic_route_irq(irq->irq, level, cpu_mask, priority);
 }
 
-static void __init gic_dist_init(void)
-{
-    uint32_t type;
-    uint32_t cpumask;
-    int i;
-
-    cpumask = GICD[GICD_ITARGETSR] & 0xff;
-    cpumask |= cpumask << 8;
-    cpumask |= cpumask << 16;
-
-    /* Disable the distributor */
-    GICD[GICD_CTLR] = 0;
-
-    type = GICD[GICD_TYPER];
-    gic.lines = 32 * ((type & GICD_TYPE_LINES) + 1);
-    gic.cpus = 1 + ((type & GICD_TYPE_CPUS) >> 5);
-    printk("GIC: %d lines, %d cpu%s%s (IID %8.8x).\n",
-           gic.lines, gic.cpus, (gic.cpus == 1) ? "" : "s",
-           (type & GICD_TYPE_SEC) ? ", secure" : "",
-           GICD[GICD_IIDR]);
-
-    /* Default all global IRQs to level, active low */
-    for ( i = 32; i < gic.lines; i += 16 )
-        GICD[GICD_ICFGR + i / 16] = 0x0;
-
-    /* Route all global IRQs to this CPU */
-    for ( i = 32; i < gic.lines; i += 4 )
-        GICD[GICD_ITARGETSR + i / 4] = cpumask;
-
-    /* Default priority for global interrupts */
-    for ( i = 32; i < gic.lines; i += 4 )
-        GICD[GICD_IPRIORITYR + i / 4] =
-            GIC_PRI_IRQ<<24 | GIC_PRI_IRQ<<16 | GIC_PRI_IRQ<<8 | GIC_PRI_IRQ;
-
-    /* Disable all global interrupts */
-    for ( i = 32; i < gic.lines; i += 32 )
-        GICD[GICD_ICENABLER + i / 32] = (uint32_t)~0ul;
-
-    /* Turn on the distributor */
-    GICD[GICD_CTLR] = GICD_CTL_ENABLE;
-}
-
-static void __cpuinit gic_cpu_init(void)
-{
-    int i;
-
-    this_cpu(gic_cpu_id) = GICD[GICD_ITARGETSR] & 0xff;
-
-    /* The first 32 interrupts (PPI and SGI) are banked per-cpu, so
-     * even though they are controlled with GICD registers, they must
-     * be set up here with the other per-cpu state. */
-    GICD[GICD_ICENABLER] = 0xffff0000; /* Disable all PPI */
-    GICD[GICD_ISENABLER] = 0x0000ffff; /* Enable all SGI */
-    /* Set SGI priorities */
-    for (i = 0; i < 16; i += 4)
-        GICD[GICD_IPRIORITYR + i / 4] =
-            GIC_PRI_IPI<<24 | GIC_PRI_IPI<<16 | GIC_PRI_IPI<<8 | GIC_PRI_IPI;
-    /* Set PPI priorities */
-    for (i = 16; i < 32; i += 4)
-        GICD[GICD_IPRIORITYR + i / 4] =
-            GIC_PRI_IRQ<<24 | GIC_PRI_IRQ<<16 | GIC_PRI_IRQ<<8 | GIC_PRI_IRQ;
-
-    /* Local settings: interface controller */
-    GICC[GICC_PMR] = 0xff;                /* Don't mask by priority */
-    GICC[GICC_BPR] = 0;                   /* Finest granularity of priority */
-    GICC[GICC_CTLR] = GICC_CTL_ENABLE|GICC_CTL_EOI;    /* Turn on delivery */
-}
-
-static void gic_cpu_disable(void)
-{
-    GICC[GICC_CTLR] = 0;
-}
-
-static void __cpuinit gic_hyp_init(void)
-{
-    uint32_t vtr;
-
-    vtr = GICH[GICH_VTR];
-    nr_lrs  = (vtr & GICH_VTR_NRLRGS) + 1;
-
-    GICH[GICH_MISR] = GICH_MISR_EOI;
-    this_cpu(lr_mask) = 0ULL;
-}
-
-static void __cpuinit gic_hyp_disable(void)
-{
-    GICH[GICH_HCR] = 0;
-}
-
-/* Set up the per-CPU parts of the GIC for a secondary CPU */
-static int  __cpuinit gic_init_secondary_cpu(struct notifier_block *nfb,
-                                        unsigned long action, void *hcpu)
-{
-    if (action == CPU_STARTING)
-    {
-        spin_lock(&gic.lock);
-        gic_cpu_init();
-        gic_hyp_init();
-        spin_unlock(&gic.lock);
-    }
-    return NOTIFY_DONE;
-}
-
-static struct notifier_block gic_cpu_nb = { 
-    .notifier_call = gic_init_secondary_cpu,
-    .priority = 100 
-};
-
-static void gic_smp_init(void)
-{
-   register_cpu_notifier(&gic_cpu_nb);
-}
-
 int gic_irq_xlate(const u32 *intspec, unsigned int intsize,
                   unsigned int *out_hwirq,
                   unsigned int *out_type)
@@ -438,124 +260,25 @@ int gic_irq_xlate(const u32 *intspec, unsigned int intsize,
     return 0;
 }
 
-/* Set up the GIC */
-void __init gic_init(void)
-{
-    static const struct dt_device_match gic_ids[] __initconst =
-    {
-        DT_MATCH_GIC,
-        { /* sentinel */ },
-    };
-    struct dt_device_node *node;
-    int res;
-
-    node = dt_find_interrupt_controller(gic_ids);
-    if ( !node )
-        panic("Unable to find compatible GIC in the device tree");
-
-    dt_device_set_used_by(node, DOMID_XEN);
-
-    res = dt_device_get_address(node, 0, &gic.dbase, NULL);
-    if ( res || !gic.dbase || (gic.dbase & ~PAGE_MASK) )
-        panic("GIC: Cannot find a valid address for the distributor");
-
-    res = dt_device_get_address(node, 1, &gic.cbase, NULL);
-    if ( res || !gic.cbase || (gic.cbase & ~PAGE_MASK) )
-        panic("GIC: Cannot find a valid address for the CPU");
-
-    res = dt_device_get_address(node, 2, &gic.hbase, NULL);
-    if ( res || !gic.hbase || (gic.hbase & ~PAGE_MASK) )
-        panic("GIC: Cannot find a valid address for the hypervisor");
-
-    res = dt_device_get_address(node, 3, &gic.vbase, NULL);
-    if ( res || !gic.vbase || (gic.vbase & ~PAGE_MASK) )
-        panic("GIC: Cannot find a valid address for the virtual CPU");
-
-    res = dt_device_get_irq(node, 0, &gic.maintenance);
-    if ( res )
-        panic("GIC: Cannot find the maintenance IRQ");
-
-    /* Set the GIC as the primary interrupt controller */
-    dt_interrupt_controller = node;
-
-    /* TODO: Add check on distributor, cpu size */
-
-    printk("GIC initialization:\n"
-              "        gic_dist_addr=%"PRIpaddr"\n"
-              "        gic_cpu_addr=%"PRIpaddr"\n"
-              "        gic_hyp_addr=%"PRIpaddr"\n"
-              "        gic_vcpu_addr=%"PRIpaddr"\n"
-              "        gic_maintenance_irq=%u\n",
-              gic.dbase, gic.cbase, gic.hbase, gic.vbase,
-              gic.maintenance.irq);
-
-    if ( (gic.dbase & ~PAGE_MASK) || (gic.cbase & ~PAGE_MASK) ||
-         (gic.hbase & ~PAGE_MASK) || (gic.vbase & ~PAGE_MASK) )
-        panic("GIC interfaces not page aligned");
-
-    set_fixmap(FIXMAP_GICD, gic.dbase >> PAGE_SHIFT, DEV_SHARED);
-    BUILD_BUG_ON(FIXMAP_ADDR(FIXMAP_GICC1) !=
-                 FIXMAP_ADDR(FIXMAP_GICC2)-PAGE_SIZE);
-    set_fixmap(FIXMAP_GICC1, gic.cbase >> PAGE_SHIFT, DEV_SHARED);
-    if ( platform_has_quirk(PLATFORM_QUIRK_GIC_64K_STRIDE) )
-        set_fixmap(FIXMAP_GICC2, (gic.cbase >> PAGE_SHIFT) + 0x10, DEV_SHARED);
-    else
-        set_fixmap(FIXMAP_GICC2, (gic.cbase >> PAGE_SHIFT) + 0x1, DEV_SHARED);
-    set_fixmap(FIXMAP_GICH, gic.hbase >> PAGE_SHIFT, DEV_SHARED);
-
-    /* Global settings: interrupt distributor */
-    spin_lock_init(&gic.lock);
-    spin_lock(&gic.lock);
-
-    gic_smp_init();
-    gic_dist_init();
-    gic_cpu_init();
-    gic_hyp_init();
-
-    spin_unlock(&gic.lock);
-}
-
 void send_SGI_mask(const cpumask_t *cpumask, enum gic_sgi sgi)
 {
-    unsigned int mask = 0;
-    cpumask_t online_mask;
-
-    ASSERT(sgi < 16); /* There are only 16 SGIs */
-
-    cpumask_and(&online_mask, cpumask, &cpu_online_map);
-    mask = gic_cpu_mask(&online_mask);
-
-    dsb();
-
-    GICD[GICD_SGIR] = GICD_SGI_TARGET_LIST
-        | (mask<<GICD_SGI_TARGET_SHIFT)
-        | sgi;
+    gic_hw_ops->send_sgi(cpumask, sgi);
 }
 
-void send_SGI_one(unsigned int cpu, enum gic_sgi sgi)
+static void send_SGI_one(unsigned int cpu, enum gic_sgi sgi)
 {
-    ASSERT(cpu < NR_GIC_CPU_IF);  /* Targets bitmap only supports 8 CPUs */
     send_SGI_mask(cpumask_of(cpu), sgi);
 }
 
-void send_SGI_self(enum gic_sgi sgi)
-{
-    ASSERT(sgi < 16); /* There are only 16 SGIs */
-
-    dsb();
-
-    GICD[GICD_SGIR] = GICD_SGI_TARGET_SELF
-        | sgi;
-}
-
 void send_SGI_allbutself(enum gic_sgi sgi)
 {
-   ASSERT(sgi < 16); /* There are only 16 SGIs */
+    cpumask_t all_others_mask;
+    ASSERT(sgi < 16); /* There are only 16 SGIs */
 
-   dsb();
+    cpumask_andnot(&all_others_mask, &cpu_possible_map, cpumask_of(smp_processor_id()));  
 
-   GICD[GICD_SGIR] = GICD_SGI_TARGET_OTHERS
-       | sgi;
+    dsb();
+    send_SGI_mask(&all_others_mask, sgi);
 }
 
 void smp_send_state_dump(unsigned int cpu)
@@ -568,16 +291,15 @@ void gic_disable_cpu(void)
 {
     ASSERT(!local_irq_is_enabled());
 
-    spin_lock(&gic.lock);
-    gic_cpu_disable();
-    gic_hyp_disable();
-    spin_unlock(&gic.lock);
+    spin_lock(&gic_lock);
+    gic_hw_ops->disable_interface();
+    spin_unlock(&gic_lock);
 }
 
 void gic_route_ppis(void)
 {
     /* GIC maintenance */
-    gic_route_dt_irq(&gic.maintenance, cpumask_of(smp_processor_id()),
+    gic_route_dt_irq(gic_hw_ops->get_maintenance_irq(), cpumask_of(smp_processor_id()),
                      GIC_PRI_IRQ);
     /* Route timer interrupt */
     route_timer_interrupt();
@@ -652,20 +374,10 @@ int __init setup_dt_irq(const struct dt_irq *irq, struct irqaction *new)
     return rc;
 }
 
-static inline void gic_set_lr(int lr, struct pending_irq *p,
+static void gic_set_lr(int lr, struct pending_irq *p,
         unsigned int state)
 {
-    int maintenance_int = GICH_LR_MAINTENANCE_IRQ;
-
-    BUG_ON(lr >= nr_lrs);
-    BUG_ON(lr < 0);
-    BUG_ON(state & ~(GICH_LR_STATE_MASK<<GICH_LR_STATE_SHIFT));
-
-    GICH[GICH_LR + lr] = state |
-        maintenance_int |
-        ((p->priority >> 3) << GICH_LR_PRIORITY_SHIFT) |
-        ((p->irq & GICH_LR_VIRTUAL_MASK) << GICH_LR_VIRTUAL_SHIFT);
-
+    gic_hw_ops->update_lr(lr, p->irq, state, p->priority);
     set_bit(GIC_IRQ_GUEST_VISIBLE, &p->status);
     clear_bit(GIC_IRQ_GUEST_PENDING, &p->status);
 }
@@ -693,10 +405,10 @@ void gic_remove_from_queues(struct vcpu *v, unsigned int virtual_irq)
     struct pending_irq *p = irq_to_pending(v, virtual_irq);
     unsigned long flags;
 
-    spin_lock_irqsave(&gic.lock, flags);
+    spin_lock_irqsave(&gic_lock, flags);
     if ( !list_empty(&p->lr_queue) )
         list_del_init(&p->lr_queue);
-    spin_unlock_irqrestore(&gic.lock, flags);
+    spin_unlock_irqrestore(&gic_lock, flags);
 }
 
 void gic_set_guest_irq(struct vcpu *v, unsigned int virtual_irq,
@@ -704,9 +416,9 @@ void gic_set_guest_irq(struct vcpu *v, unsigned int virtual_irq,
 {
     int i;
     unsigned long flags;
+    unsigned int nr_lrs = gic_hw_ops->nr_lrs();
 
-    spin_lock_irqsave(&gic.lock, flags);
-
+    spin_lock_irqsave(&gic_lock, flags);
     if ( v == current && list_empty(&v->arch.vgic.lr_pending) )
     {
         i = find_first_zero_bit(&this_cpu(lr_mask), nr_lrs);
@@ -720,7 +432,7 @@ void gic_set_guest_irq(struct vcpu *v, unsigned int virtual_irq,
     gic_add_to_lr_pending(v, irq_to_pending(v, virtual_irq));
 
 out:
-    spin_unlock_irqrestore(&gic.lock, flags);
+    spin_unlock_irqrestore(&gic_lock, flags);
     return;
 }
 
@@ -729,17 +441,18 @@ static void gic_restore_pending_irqs(struct vcpu *v)
     int i;
     struct pending_irq *p, *t;
     unsigned long flags;
+    unsigned int nr_lrs = gic_hw_ops->nr_lrs();
 
     list_for_each_entry_safe ( p, t, &v->arch.vgic.lr_pending, lr_queue )
     {
         i = find_first_zero_bit(&this_cpu(lr_mask), nr_lrs);
         if ( i >= nr_lrs ) return;
 
-        spin_lock_irqsave(&gic.lock, flags);
+        spin_lock_irqsave(&gic_lock, flags);
         gic_set_lr(i, p, GICH_LR_PENDING);
         list_del_init(&p->lr_queue);
         set_bit(i, &this_cpu(lr_mask));
-        spin_unlock_irqrestore(&gic.lock, flags);
+        spin_unlock_irqrestore(&gic_lock, flags);
     }
 
 }
@@ -749,11 +462,11 @@ void gic_clear_pending_irqs(struct vcpu *v)
     struct pending_irq *p, *t;
     unsigned long flags;
 
-    spin_lock_irqsave(&gic.lock, flags);
+    spin_lock_irqsave(&gic_lock, flags);
     v->arch.lr_mask = 0;
     list_for_each_entry_safe ( p, t, &v->arch.vgic.lr_pending, lr_queue )
         list_del_init(&p->lr_queue);
-    spin_unlock_irqrestore(&gic.lock, flags);
+    spin_unlock_irqrestore(&gic_lock, flags);
 }
 
 static void gic_inject_irq_start(void)
@@ -809,7 +522,7 @@ int gic_route_irq_to_guest(struct domain *d, const struct dt_irq *irq,
     action->free_on_release = 1;
 
     spin_lock_irqsave(&desc->lock, flags);
-    spin_lock(&gic.lock);
+    spin_lock(&gic_lock);
 
     desc->handler = &gic_guest_irq_type;
     desc->status |= IRQ_GUEST;
@@ -830,15 +543,15 @@ int gic_route_irq_to_guest(struct domain *d, const struct dt_irq *irq,
     p->desc = desc;
 
 out:
-    spin_unlock(&gic.lock);
+    spin_unlock(&gic_lock);
     spin_unlock_irqrestore(&desc->lock, flags);
     return retval;
 }
 
-static void do_sgi(struct cpu_user_regs *regs, int othercpu, enum gic_sgi sgi)
+static void do_sgi(struct cpu_user_regs *regs, enum gic_sgi sgi)
 {
     /* Lower the priority */
-    GICC[GICC_EOIR] = sgi;
+    gic_hw_ops->eoi_irq(sgi);
 
     switch (sgi)
     {
@@ -857,20 +570,17 @@ static void do_sgi(struct cpu_user_regs *regs, int othercpu, enum gic_sgi sgi)
     }
 
     /* Deactivate */
-    GICC[GICC_DIR] = sgi;
+    gic_hw_ops->deactivate_irq(sgi);
 }
 
 /* Accept an interrupt from the GIC and dispatch its handler */
 void gic_interrupt(struct cpu_user_regs *regs, int is_fiq)
 {
-    uint32_t intack;
     unsigned int irq;
 
 
     do  {
-        intack = GICC[GICC_IAR];
-        irq = intack & GICC_IA_IRQ;
-
+        irq = gic_hw_ops->ack_irq();
         if ( likely(irq >= 16 && irq < 1021) )
         {
             local_irq_enable();
@@ -879,8 +589,7 @@ void gic_interrupt(struct cpu_user_regs *regs, int is_fiq)
         }
         else if (unlikely(irq < 16))
         {
-            unsigned int cpu = (intack & GICC_IA_CPU_MASK) >> GICC_IA_CPU_SHIFT;
-            do_sgi(regs, cpu, irq);
+            do_sgi(regs, irq);
         }
         else
         {
@@ -892,72 +601,31 @@ void gic_interrupt(struct cpu_user_regs *regs, int is_fiq)
 
 int vcpu_gic_init(struct vcpu *v)
 {
-     v->arch.gic_state = xzalloc(struct gic_state_data);
-     if(!v->arch.gic_state)
-        return -ENOMEM;
-     return 0;
+     return gic_hw_ops->state_init(v);
 }
 
 int gicv_setup(struct domain *d)
 {
     int ret;
 
-    /*
-     * Domain 0 gets the hardware address.
-     * Guests get the virtual platform layout.
-     */
-    if ( d->domain_id == 0 )
-    {
-        d->arch.vgic.dbase = gic.dbase;
-        d->arch.vgic.cbase = gic.cbase;
-    }
-    else
-    {
-        d->arch.vgic.dbase = GUEST_GICD_BASE;
-        d->arch.vgic.cbase = GUEST_GICC_BASE;
-    }
-
-    d->arch.vgic.nr_lines = 0;
-
-    /*
-     * Map the gic virtual cpu interface in the gic cpu interface
-     * region of the guest.
-     *
-     * The second page is always mapped at +4K irrespective of the
-     * GIC_64K_STRIDE quirk. The DTB passed to the guest reflects this.
-     */
-    ret = map_mmio_regions(d, d->arch.vgic.cbase,
-                           d->arch.vgic.cbase + PAGE_SIZE - 1,
-                           gic.vbase);
-    if (ret)
-        return ret;
-
-    if ( !platform_has_quirk(PLATFORM_QUIRK_GIC_64K_STRIDE) )
-        ret = map_mmio_regions(d, d->arch.vgic.cbase + PAGE_SIZE,
-                               d->arch.vgic.cbase + (2 * PAGE_SIZE) - 1,
-                               gic.vbase + PAGE_SIZE);
-    else
-        ret = map_mmio_regions(d, d->arch.vgic.cbase + PAGE_SIZE,
-                               d->arch.vgic.cbase + (2 * PAGE_SIZE) - 1,
-                               gic.vbase + 16*PAGE_SIZE);
-
+    ret = gic_hw_ops->gicv_setup(d);
     return ret;
 
 }
 
 static void gic_irq_eoi(void *info)
-{
+{ 
     int virq = (uintptr_t) info;
-    GICC[GICC_DIR] = virq;
+    gic_hw_ops->deactivate_irq(virq);
 }
 
 static void maintenance_interrupt(int irq, void *dev_id, struct cpu_user_regs *regs)
 {
     int i = 0, virq, pirq = -1;
-    uint32_t lr;
     struct vcpu *v = current;
-    uint64_t eisr = GICH[GICH_EISR0] | (((uint64_t) GICH[GICH_EISR1]) << 32);
+    unsigned long eisr = 0;
 
+    eisr = gic_hw_ops->read_eisr();
     while ((i = find_next_bit((const long unsigned int *) &eisr,
                               64, i)) < 64) {
         struct pending_irq *p, *p2;
@@ -967,10 +635,9 @@ static void maintenance_interrupt(int irq, void *dev_id, struct cpu_user_regs *r
         cpu = -1;
         inflight = 0;
 
-        spin_lock_irq(&gic.lock);
-        lr = GICH[GICH_LR + i];
-        virq = lr & GICH_LR_VIRTUAL_MASK;
-        GICH[GICH_LR + i] = 0;
+        spin_lock_irq(&gic_lock);
+        virq = gic_hw_ops->update_lr_for_mi(i);
+
         clear_bit(i, &this_cpu(lr_mask));
 
         p = irq_to_pending(v, virq);
@@ -995,7 +662,7 @@ static void maintenance_interrupt(int irq, void *dev_id, struct cpu_user_regs *r
             list_del_init(&p2->lr_queue);
             set_bit(i, &this_cpu(lr_mask));
         }
-        spin_unlock_irq(&gic.lock);
+        spin_unlock_irq(&gic_lock);
 
         if ( !inflight )
         {
@@ -1018,22 +685,37 @@ static void maintenance_interrupt(int irq, void *dev_id, struct cpu_user_regs *r
     }
 }
 
+/* Set up the GIC */
+void __init gic_init(void)
+{
+    static const struct dt_device_match gicv2_ids[] __initconst =
+    {
+        DT_MATCH_GIC,
+        { /* sentinel */ },
+    };
+    struct dt_device_node *node;
+
+    spin_lock_init(&gic_lock);
+
+    spin_lock(&gic_lock);
+    node = dt_find_interrupt_controller(gicv2_ids);
+    if ( node )
+    {
+        gicv2_init();
+        spin_unlock(&gic_lock);
+        return;
+    }
+    spin_unlock(&gic_lock);
+    if ( !node )
+        panic("Unable to find compatible GIC in the device tree");
+}
+
 void gic_dump_info(struct vcpu *v)
 {
-    int i;
     struct pending_irq *p;
-    struct gic_state_data *d;
-    d = (struct gic_state_data *)v->arch.gic_state;
 
     printk("GICH_LRs (vcpu %d) mask=%"PRIx64"\n", v->vcpu_id, v->arch.lr_mask);
-    if ( v == current )
-    {
-        for ( i = 0; i < nr_lrs; i++ )
-            printk("   HW_LR[%d]=%x\n", i, GICH[GICH_LR + i]);
-    } else {
-        for ( i = 0; i < nr_lrs; i++ )
-            printk("   VCPU_LR[%d]=%x\n", i, d->gic_lr[i]);
-    }
+    gic_hw_ops->dump_state(v);
 
     list_for_each_entry ( p, &v->arch.vgic.inflight_irqs, inflight )
     {
@@ -1049,7 +731,7 @@ void gic_dump_info(struct vcpu *v)
 
 void __cpuinit init_maintenance_interrupt(void)
 {
-    request_dt_irq(&gic.maintenance, maintenance_interrupt,
+    request_dt_irq(gic_hw_ops->get_maintenance_irq(), maintenance_interrupt,
                    "irq-maintenance", NULL);
 }
 
diff --git a/xen/include/asm-arm/gic.h b/xen/include/asm-arm/gic.h
index 18656fd..4244491 100644
--- a/xen/include/asm-arm/gic.h
+++ b/xen/include/asm-arm/gic.h
@@ -39,6 +39,8 @@
 #define GIC_PRI_IPI        0x90 /* IPIs must preempt normal interrupts */
 #define GIC_PRI_HIGHEST    0x80 /* Higher priorities belong to Secure-World */
 
+#define GICH_LR_PENDING    1
+#define GICH_LR_ACTIVE     2
 
 #ifndef __ASSEMBLY__
 #include <xen/device_tree.h>
@@ -46,12 +48,15 @@
 #define DT_MATCH_GIC    DT_MATCH_COMPATIBLE("arm,cortex-a15-gic"), \
                         DT_MATCH_COMPATIBLE("arm,cortex-a7-gic")
 
+extern int gic_hw_version(void);
 extern int domain_vgic_init(struct domain *d);
 extern void domain_vgic_free(struct domain *d);
 
 extern int vcpu_vgic_init(struct vcpu *v);
 extern int vcpu_gic_init(struct vcpu *v);
 
+extern void gicv2_init(void);
+
 extern void vgic_vcpu_inject_irq(struct vcpu *v, unsigned int irq,int virtual);
 extern void vgic_clear_pending_irqs(struct vcpu *v);
 extern struct pending_irq *irq_to_pending(struct vcpu *v, unsigned int irq);
@@ -88,14 +93,47 @@ extern int gicv_setup(struct domain *d);
 extern void gic_save_state(struct vcpu *v);
 extern void gic_restore_state(struct vcpu *v);
 
+#define GIC_VERSION_V2 0x2
+
 /* SGI (AKA IPIs) */
 enum gic_sgi {
     GIC_SGI_EVENT_CHECK = 0,
     GIC_SGI_DUMP_STATE  = 1,
     GIC_SGI_CALL_FUNCTION = 2,
 };
+
+struct gic_hw_operations {
+    int (*gic_type)(void);
+    struct dt_irq * (*get_maintenance_irq)(void);
+    unsigned int (*nr_lines)(void);
+    unsigned int (*nr_lrs)(void);
+    int (*state_init)(struct vcpu *);
+    void (*save_state)(struct vcpu *);
+    void (*restore_state)(struct vcpu *);
+    void (*dump_state)(struct vcpu *);
+    int (*gicv_setup)(struct domain *);
+    void (*enable_irq)(int);
+    void (*disable_irq)(int);
+    void (*eoi_irq)(int);
+    void (*deactivate_irq)(int);
+    unsigned int (*ack_irq)(void);
+    void (*set_irq_property)(unsigned int irq, bool_t level,
+                            const cpumask_t *cpu_mask,
+                            unsigned int priority);
+    void (*send_sgi)(const cpumask_t *, enum gic_sgi);
+    void (*disable_interface)(void);
+    void (*update_lr)(int lr, unsigned int virtual_irq,
+                          unsigned int state, unsigned int priority);
+    unsigned int (*update_lr_for_mi)(int lr);
+    unsigned long (*read_eisr)(void);
+    unsigned long (*read_cpu_rbase)(void);
+    unsigned long (*read_cpu_sgi_rbase)(void);
+};
+
+extern void register_gic_ops(struct gic_hw_operations *);
+
+extern void update_cpu_lr_mask(void);
 extern void send_SGI_mask(const cpumask_t *cpumask, enum gic_sgi sgi);
-extern void send_SGI_one(unsigned int cpu, enum gic_sgi sgi);
 extern void send_SGI_self(enum gic_sgi sgi);
 extern void send_SGI_allbutself(enum gic_sgi sgi);
 
diff --git a/xen/include/asm-arm/gic_v2_defs.h b/xen/include/asm-arm/gic_v2_defs.h
index 2366685..5ddd48a 100644
--- a/xen/include/asm-arm/gic_v2_defs.h
+++ b/xen/include/asm-arm/gic_v2_defs.h
@@ -119,8 +119,6 @@
 #define GICH_LR_STATE_SHIFT     28
 #define GICH_LR_PRIORITY_SHIFT  23
 #define GICH_LR_MAINTENANCE_IRQ (1<<19)
-#define GICH_LR_PENDING         (1<<28)
-#define GICH_LR_ACTIVE          (1<<29)
 #define GICH_LR_GRP1            (1<<30)
 #define GICH_LR_HW              (1<<31)
 #define GICH_LR_CPUID_SHIFT     9
-- 
1.7.9.5

Re: [RFC PATCH v1 06/10] xen/arm: split gic driver into generic and gicv2 driver

[Stefano Stabellini]

On Wed, 19 Mar 2014, vijay.kilari@gmail.com wrote:
> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> 
> Existing GIC driver has both generic code and hw specific
> code. Segregate GIC low level driver into gic-v2.c and
> keep generic code in existing gic.c file
> 
> GIC v2 driver registers required functions
> to generic GIC driver. This helps to plug in next version
> of GIC drivers like GIC v3.
> 
> Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>

Am I correct in assuming that this patch is just code movement plus the
introduction of struct gic_hw_operations?


>  xen/arch/arm/Makefile             |    2 +-
>  xen/arch/arm/gic-v2.c             |  562 +++++++++++++++++++++++++++++++++++++
>  xen/arch/arm/gic.c                |  530 +++++++---------------------------
>  xen/include/asm-arm/gic.h         |   40 ++-
>  xen/include/asm-arm/gic_v2_defs.h |    2 -
>  5 files changed, 708 insertions(+), 428 deletions(-)
> 
> diff --git a/xen/arch/arm/Makefile b/xen/arch/arm/Makefile
> index 63e0460..969ee52 100644
> --- a/xen/arch/arm/Makefile
> +++ b/xen/arch/arm/Makefile
> @@ -10,7 +10,7 @@ obj-y += vpsci.o
>  obj-y += domctl.o
>  obj-y += sysctl.o
>  obj-y += domain_build.o
> -obj-y += gic.o
> +obj-y += gic.o gic-v2.o
>  obj-y += io.o
>  obj-y += irq.o
>  obj-y += kernel.o
> diff --git a/xen/arch/arm/gic-v2.c b/xen/arch/arm/gic-v2.c
> new file mode 100644
> index 0000000..9378427
> --- /dev/null
> +++ b/xen/arch/arm/gic-v2.c
> @@ -0,0 +1,562 @@
> +/*
> + * xen/arch/arm/gic-v2.c
> + *
> + * ARM Generic Interrupt Controller support v2
> + *
> + * Tim Deegan <tim@xen.org>
> + * Copyright (c) 2011 Citrix Systems.
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License as published by
> + * the Free Software Foundation; either version 2 of the License, or
> + * (at your option) any later version.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> + * GNU General Public License for more details.
> + */
> +
> +#include <xen/config.h>
> +#include <xen/lib.h>
> +#include <xen/init.h>
> +#include <xen/cpu.h>
> +#include <xen/mm.h>
> +#include <xen/irq.h>
> +#include <xen/sched.h>
> +#include <xen/errno.h>
> +#include <xen/serial.h>
> +#include <xen/softirq.h>
> +#include <xen/list.h>
> +#include <xen/device_tree.h>
> +#include <asm/p2m.h>
> +#include <asm/domain.h>
> +#include <asm/platform.h>
> +
> +#include <asm/gic_v2_defs.h>
> +#include <asm/gic.h>
> +
> +/* Access to the GIC Distributor registers through the fixmap */
> +#define GICD ((volatile uint32_t *) FIXMAP_ADDR(FIXMAP_GICD))
> +#define GICC ((volatile uint32_t *) FIXMAP_ADDR(FIXMAP_GICC1))
> +#define GICH ((volatile uint32_t *) FIXMAP_ADDR(FIXMAP_GICH))
> +
> +struct gic_state_data {
> +    uint32_t gic_hcr;
> +    uint32_t gic_vmcr;
> +    uint32_t gic_apr;
> +    uint32_t gic_lr[64];
> +};
> +
> +/* Global state */
> +static struct {
> +    int hw_version;
> +    paddr_t dbase;       /* Address of distributor registers */
> +    paddr_t cbase;       /* Address of CPU interface registers */
> +    paddr_t hbase;       /* Address of virtual interface registers */
> +    paddr_t vbase;       /* Address of virtual cpu interface registers */
> +    unsigned int lines;  /* Number of interrupts (SPIs + PPIs + SGIs) */
> +    struct dt_irq maintenance; /* IRQ maintenance */
> +    unsigned int cpus;
> +    spinlock_t lock;
> +} gic;
> +
> +static unsigned nr_lrs;
> +
> +/* The GIC mapping of CPU interfaces does not necessarily match the
> + * logical CPU numbering. Let's use mapping as returned by the GIC
> + * itself
> + */
> +static DEFINE_PER_CPU(u8, gic_cpu_id);
> +
> +/* Maximum cpu interface per GIC */
> +#define NR_GIC_CPU_IF 8
> +
> +static unsigned int gic_cpu_mask(const cpumask_t *cpumask)
> +{
> +    unsigned int cpu;
> +    unsigned int mask = 0;
> +    cpumask_t possible_mask;
> +
> +    cpumask_and(&possible_mask, cpumask, &cpu_possible_map);
> +    for_each_cpu(cpu, &possible_mask)
> +    {
> +        ASSERT(cpu < NR_GIC_CPU_IF);
> +        mask |= per_cpu(gic_cpu_id, cpu);
> +    }
> +
> +    return mask;
> +}
> +
> +static unsigned int gic_nr_lines(void)
> +{
> +    return gic.lines;
> +}
> +
> +static unsigned int gic_nr_lrs(void)
> +{
> +    return nr_lrs;
> +}
> +
> +static int gic_state_init(struct vcpu *v)
> +{
> +     v->arch.gic_state = (struct gic_state_data *)xzalloc(struct gic_state_data);
> +     if(!v->arch.gic_state)
> +        return -ENOMEM;
> +     return 0;
> +}
> +
> +static void save_state(struct vcpu *v)
> +{
> +    int i;
> +    struct gic_state_data *d;
> +    d = (struct gic_state_data *)v->arch.gic_state;
> +
> +    ASSERT(!local_irq_is_enabled());
> +
> +    /* No need for spinlocks here because interrupts are disabled around
> +     * this call and it only accesses struct vcpu fields that cannot be
> +     * accessed simultaneously by another pCPU.
> +     */
> +    for ( i = 0; i < nr_lrs; i++)
> +        d->gic_lr[i] = GICH[GICH_LR + i];
> +    d->gic_apr = GICH[GICH_APR];
> +    d->gic_vmcr = GICH[GICH_VMCR];
> +    /* Disable until next VCPU scheduled */
> +    GICH[GICH_HCR] = 0;
> +}
> +
> +static void restore_state(struct vcpu *v)
> +{
> +    int i;
> +    struct gic_state_data *d;
> +    d = (struct gic_state_data *)v->arch.gic_state;
> +
> +    for ( i = 0; i < nr_lrs; i++)
> +        GICH[GICH_LR + i] = d->gic_lr[i];
> +    GICH[GICH_APR] = d->gic_apr;
> +    GICH[GICH_VMCR] = d->gic_vmcr;
> +    GICH[GICH_HCR] = GICH_HCR_EN;
> +}
> +
> +static void gic_dump_state(struct vcpu *v)
> +{
> +    int i;
> +    struct gic_state_data *d;
> +    d = (struct gic_state_data *)v->arch.gic_state;
> +
> +    if ( v == current )
> +    {
> +        for ( i = 0; i < nr_lrs; i++ )
> +            printk("   HW_LR[%d]=%x\n", i, GICH[GICH_LR + i]);
> +    } else {
> +        for ( i = 0; i < nr_lrs; i++ )
> +            printk("   VCPU_LR[%d]=%x\n", i, d->gic_lr[i]);
> +    }
> +}
> +
> +static void gic_enable_irq(int irq)
> +{
> +    /* Enable routing */
> +    GICD[GICD_ISENABLER + irq / 32] = (1u << (irq % 32));
> +}
> +
> +static void gic_disable_irq(int irq)
> +{
> +    /* Disable routing */
> +    GICD[GICD_ICENABLER + irq / 32] = (1u << (irq % 32));
> +}
> +
> +static void gic_eoi_irq(int irq)
> +{
> +    /* Lower the priority */
> +    GICC[GICC_EOIR] = irq;
> +}
> +
> +static void gic_dir_irq(int irq)
> +{
> +    /* Deactivate */
> +    GICC[GICC_DIR] = irq;
> +}
> +
> +static unsigned int gic_ack_irq(void)
> +{
> +    return (GICC[GICC_IAR] & GICC_IA_IRQ);
> +}
> +
> +/*
> + * - needs to be called with gic.lock held
> + * - needs to be called with a valid cpu_mask, ie each cpu in the mask has
> + * already called gic_cpu_init
> + */
> +static void gic_set_irq_properties(unsigned int irq, bool_t level,
> +                                   const cpumask_t *cpu_mask,
> +                                   unsigned int priority)
> +{
> +    volatile unsigned char *bytereg;
> +    uint32_t cfg, edgebit;
> +    unsigned int mask = gic_cpu_mask(cpu_mask);
> +
> +    /* Set edge / level */
> +    cfg = GICD[GICD_ICFGR + irq / 16];
> +    edgebit = 2u << (2 * (irq % 16));
> +    if ( level )
> +        cfg &= ~edgebit;
> +    else
> +        cfg |= edgebit;
> +    GICD[GICD_ICFGR + irq / 16] = cfg;
> +
> +    /* Set target CPU mask (RAZ/WI on uniprocessor) */
> +    bytereg = (unsigned char *) (GICD + GICD_ITARGETSR);
> +    bytereg[irq] = mask;
> +
> +    /* Set priority */
> +    bytereg = (unsigned char *) (GICD + GICD_IPRIORITYR);
> +    bytereg[irq] = priority;
> +
> +}
> +
> +static void __init gic_dist_init(void)
> +{
> +    uint32_t type;
> +    uint32_t cpumask;
> +    int i;
> +
> +    cpumask = GICD[GICD_ITARGETSR] & 0xff;
> +    cpumask |= cpumask << 8;
> +    cpumask |= cpumask << 16;
> +
> +    /* Disable the distributor */
> +    GICD[GICD_CTLR] = 0;
> +
> +    type = GICD[GICD_TYPER];
> +    gic.lines = 32 * ((type & GICD_TYPE_LINES) + 1);
> +    gic.cpus = 1 + ((type & GICD_TYPE_CPUS) >> 5);
> +    printk("GIC: %d lines, %d cpu%s%s (IID %8.8x).\n",
> +           gic.lines, gic.cpus, (gic.cpus == 1) ? "" : "s",
> +           (type & GICD_TYPE_SEC) ? ", secure" : "",
> +           GICD[GICD_IIDR]);
> +
> +    /* Default all global IRQs to level, active low */
> +    for ( i = 32; i < gic.lines; i += 16 )
> +        GICD[GICD_ICFGR + i / 16] = 0x0;
> +
> +    /* Route all global IRQs to this CPU */
> +    for ( i = 32; i < gic.lines; i += 4 )
> +        GICD[GICD_ITARGETSR + i / 4] = cpumask;
> +
> +    /* Default priority for global interrupts */
> +    for ( i = 32; i < gic.lines; i += 4 )
> +        GICD[GICD_IPRIORITYR + i / 4] =
> +            GIC_PRI_IRQ<<24 | GIC_PRI_IRQ<<16 | GIC_PRI_IRQ<<8 | GIC_PRI_IRQ;
> +
> +    /* Disable all global interrupts */
> +    for ( i = 32; i < gic.lines; i += 32 )
> +        GICD[GICD_ICENABLER + i / 32] = (uint32_t)~0ul;
> +
> +    /* Turn on the distributor */
> +    GICD[GICD_CTLR] = GICD_CTL_ENABLE;
> +}
> +
> +static void __cpuinit gic_cpu_init(void)
> +{
> +    int i;
> +
> +    this_cpu(gic_cpu_id) = GICD[GICD_ITARGETSR] & 0xff;
> +
> +    /* The first 32 interrupts (PPI and SGI) are banked per-cpu, so
> +     * even though they are controlled with GICD registers, they must
> +     * be set up here with the other per-cpu state. */
> +    GICD[GICD_ICENABLER] = 0xffff0000; /* Disable all PPI */
> +    GICD[GICD_ISENABLER] = 0x0000ffff; /* Enable all SGI */
> +    /* Set SGI priorities */
> +    for (i = 0; i < 16; i += 4)
> +        GICD[GICD_IPRIORITYR + i / 4] =
> +            GIC_PRI_IPI<<24 | GIC_PRI_IPI<<16 | GIC_PRI_IPI<<8 | GIC_PRI_IPI;
> +    /* Set PPI priorities */
> +    for (i = 16; i < 32; i += 4)
> +        GICD[GICD_IPRIORITYR + i / 4] =
> +            GIC_PRI_IRQ<<24 | GIC_PRI_IRQ<<16 | GIC_PRI_IRQ<<8 | GIC_PRI_IRQ;
> +
> +    /* Local settings: interface controller */
> +    GICC[GICC_PMR] = 0xff;                /* Don't mask by priority */
> +    GICC[GICC_BPR] = 0;                   /* Finest granularity of priority */
> +    GICC[GICC_CTLR] = GICC_CTL_ENABLE|GICC_CTL_EOI;    /* Turn on delivery */
> +}
> +
> +static void gic_cpu_disable(void)
> +{
> +    GICC[GICC_CTLR] = 0;
> +}
> +
> +static void __cpuinit gic_hyp_init(void)
> +{
> +    uint32_t vtr;
> +
> +    vtr = GICH[GICH_VTR];
> +    nr_lrs  = (vtr & GICH_VTR_NRLRGS) + 1;
> +
> +    GICH[GICH_MISR] = GICH_MISR_EOI;
> +    update_cpu_lr_mask();
> +}
> +
> +static void __cpuinit gic_hyp_disable(void)
> +{
> +    GICH[GICH_HCR] = 0;
> +}
> +
> +/* Set up the per-CPU parts of the GIC for a secondary CPU */
> +static int __cpuinit gic_init_secondary_cpu(struct notifier_block *nfb,
> +                                       unsigned long action, void *hcpu)
> +{
> +    if (action == CPU_STARTING)
> +    {
> +        spin_lock(&gic.lock);
> +        gic_cpu_init();
> +        gic_hyp_init();
> +        spin_unlock(&gic.lock);
> +    }
> +    return NOTIFY_DONE;
> +}
> +
> +static struct notifier_block gic_cpu_nb = {
> +    .notifier_call = gic_init_secondary_cpu,
> +    .priority = 100
> +};
> +
> +static void gic_smp_init(void)
> +{
> +   register_cpu_notifier(&gic_cpu_nb);
> +}
> +
> +static int gic_hw_type(void)
> +{
> +    return gic.hw_version;
> +}
> +
> +static struct dt_irq * gic_maintenance_irq(void)
> +{
> +    return &gic.maintenance;
> +}
> +
> +/* Set up the GIC */
> +
> +static void gic_send_sgi(const cpumask_t *cpumask, enum gic_sgi sgi)
> +{
> +    unsigned int mask = 0;
> +    cpumask_t online_mask;
> +
> +    ASSERT(sgi < 16); /* There are only 16 SGIs */
> +
> +    cpumask_and(&online_mask, cpumask, &cpu_online_map);
> +    mask = gic_cpu_mask(&online_mask);
> +
> +    dsb();
> +
> +    GICD[GICD_SGIR] = GICD_SGI_TARGET_LIST
> +        | (mask<<GICD_SGI_TARGET_SHIFT)
> +        | sgi;
> +}
> +
> +/* Shut down the per-CPU GIC interface */
> +static void gic_disable_interface(void)
> +{
> +    ASSERT(!local_irq_is_enabled());
> +
> +    spin_lock(&gic.lock);
> +    gic_cpu_disable();
> +    gic_hyp_disable();
> +    spin_unlock(&gic.lock);
> +}
> +
> +static void gic_update_lr(int lr, unsigned int virtual_irq,
> +        unsigned int state, unsigned int priority)
> +{
> +    int maintenance_int = GICH_LR_MAINTENANCE_IRQ;
> +
> +    BUG_ON(lr >= nr_lrs);
> +    BUG_ON(lr < 0);
> +    BUG_ON(state & ~(GICH_LR_STATE_MASK<<GICH_LR_STATE_SHIFT));
> +
> +    GICH[GICH_LR + lr] = ((state & 0x3) << GICH_LR_STATE_SHIFT) |
> +        maintenance_int |
> +        ((priority >> 3) << GICH_LR_PRIORITY_SHIFT) |
> +        ((virtual_irq & GICH_LR_VIRTUAL_MASK) << GICH_LR_VIRTUAL_SHIFT);
> +
> +}
> +
> +static int gicv_init(struct domain *d)
> +{
> +    int ret;
> +
> +    /*
> +     * Domain 0 gets the hardware address.
> +     * Guests get the virtual platform layout.
> +     */
> +    if ( d->domain_id == 0 )
> +    {
> +        d->arch.vgic.dbase = gic.dbase;
> +        d->arch.vgic.cbase = gic.cbase;
> +    }
> +    else
> +    {
> +        d->arch.vgic.dbase = GUEST_GICD_BASE;
> +        d->arch.vgic.cbase = GUEST_GICC_BASE;
> +    }
> +
> +    d->arch.vgic.nr_lines = 0;
> +
> +    /*
> +     * Map the gic virtual cpu interface in the gic cpu interface
> +     * region of the guest.
> +     *
> +     * The second page is always mapped at +4K irrespective of the
> +     * GIC_64K_STRIDE quirk. The DTB passed to the guest reflects this.
> +     */
> +    ret = map_mmio_regions(d, d->arch.vgic.cbase,
> +                           d->arch.vgic.cbase + PAGE_SIZE - 1,
> +                           gic.vbase);
> +    if (ret)
> +        return ret;
> +
> +    if ( !platform_has_quirk(PLATFORM_QUIRK_GIC_64K_STRIDE) )
> +        ret = map_mmio_regions(d, d->arch.vgic.cbase + PAGE_SIZE,
> +                               d->arch.vgic.cbase + (2 * PAGE_SIZE) - 1,
> +                               gic.vbase + PAGE_SIZE);
> +    else
> +        ret = map_mmio_regions(d, d->arch.vgic.cbase + PAGE_SIZE,
> +                               d->arch.vgic.cbase + (2 * PAGE_SIZE) - 1,
> +                               gic.vbase + 16*PAGE_SIZE);
> +
> +    return ret;
> +
> +}
> +
> +static unsigned long gic_read_eisr(void)
> +{
> +    return ((unsigned long)(GICH[GICH_EISR0] | (((uint64_t) GICH[GICH_EISR1]) << 32)));
> +}
> +
> +static unsigned int gic_update_lr_for_mi(int lr)
> +{
> +    u64 val;
> +    uint32_t virq;
> +
> +    spin_lock_irq(&gic.lock);
> +    val = GICH[GICH_LR + lr];
> +    virq = val & GICH_LR_VIRTUAL_MASK;
> +    GICH[GICH_LR + lr] = 0;
> +    spin_unlock_irq(&gic.lock);
> +    return virq;
> +}
> +
> +static struct gic_hw_operations gic_ops = {
> +    .gic_type            = gic_hw_type,
> +    .nr_lines            = gic_nr_lines,
> +    .nr_lrs              = gic_nr_lrs,
> +    .get_maintenance_irq = gic_maintenance_irq,
> +    .state_init        = gic_state_init,
> +    .save_state          = save_state,
> +    .restore_state       = restore_state,
> +    .dump_state          = gic_dump_state,
> +    .gicv_setup          = gicv_init,
> +    .enable_irq          = gic_enable_irq,
> +    .disable_irq         = gic_disable_irq,
> +    .eoi_irq             = gic_eoi_irq,
> +    .deactivate_irq      = gic_dir_irq,
> +    .ack_irq             = gic_ack_irq,
> +    .set_irq_property    = gic_set_irq_properties,
> +    .send_sgi            = gic_send_sgi,
> +    .disable_interface   = gic_disable_interface,
> +    .update_lr           = gic_update_lr,
> +    .update_lr_for_mi    = gic_update_lr_for_mi,
> +    .read_eisr           = gic_read_eisr,
> +};
> +
> +void __init gicv2_init(void)
> +{
> +    static const struct dt_device_match gic_ids[] __initconst =
> +    {
> +        DT_MATCH_GIC,
> +        { /* sentinel */ },
> +    };
> +    struct dt_device_node *node;
> +    int res;
> +
> +    node = dt_find_interrupt_controller(gic_ids);
> +    if ( !node )
> +        panic("Unable to find compatible GIC in the device tree");
> +
> +    dt_device_set_used_by(node, DOMID_XEN);
> +
> +    gic.hw_version = GIC_VERSION_V2;
> +
> +    res = dt_device_get_address(node, 0, &gic.dbase, NULL);
> +    if ( res || !gic.dbase || (gic.dbase & ~PAGE_MASK) )
> +        panic("GIC: Cannot find a valid address for the distributor");
> +
> +    res = dt_device_get_address(node, 1, &gic.cbase, NULL);
> +    if ( res || !gic.cbase || (gic.cbase & ~PAGE_MASK) )
> +        panic("GIC: Cannot find a valid address for the CPU");
> +
> +    res = dt_device_get_address(node, 2, &gic.hbase, NULL);
> +    if ( res || !gic.hbase || (gic.hbase & ~PAGE_MASK) )
> +        panic("GIC: Cannot find a valid address for the hypervisor");
> +
> +    res = dt_device_get_address(node, 3, &gic.vbase, NULL);
> +    if ( res || !gic.vbase || (gic.vbase & ~PAGE_MASK) )
> +        panic("GIC: Cannot find a valid address for the virtual CPU");
> +
> +    res = dt_device_get_irq(node, 0, &gic.maintenance);
> +    if ( res )
> +        panic("GIC: Cannot find the maintenance IRQ");
> +
> +    /* Set the GIC as the primary interrupt controller */
> +    dt_interrupt_controller = node;
> +
> +    /* TODO: Add check on distributor, cpu size */
> +
> +    printk("GIC initialization:\n"
> +              "        gic_dist_addr=%"PRIpaddr"\n"
> +              "        gic_cpu_addr=%"PRIpaddr"\n"
> +              "        gic_hyp_addr=%"PRIpaddr"\n"
> +              "        gic_vcpu_addr=%"PRIpaddr"\n"
> +              "        gic_maintenance_irq=%u\n",
> +              gic.dbase, gic.cbase, gic.hbase, gic.vbase,
> +              gic.maintenance.irq);
> +
> +    if ( (gic.dbase & ~PAGE_MASK) || (gic.cbase & ~PAGE_MASK) ||
> +         (gic.hbase & ~PAGE_MASK) || (gic.vbase & ~PAGE_MASK) )
> +        panic("GIC interfaces not page aligned");
> +
> +    set_fixmap(FIXMAP_GICD, gic.dbase >> PAGE_SHIFT, DEV_SHARED);
> +    BUILD_BUG_ON(FIXMAP_ADDR(FIXMAP_GICC1) !=
> +                 FIXMAP_ADDR(FIXMAP_GICC2)-PAGE_SIZE);
> +    set_fixmap(FIXMAP_GICC1, gic.cbase >> PAGE_SHIFT, DEV_SHARED);
> +    if ( platform_has_quirk(PLATFORM_QUIRK_GIC_64K_STRIDE) )
> +        set_fixmap(FIXMAP_GICC2, (gic.cbase >> PAGE_SHIFT) + 0x10, DEV_SHARED);
> +    else
> +        set_fixmap(FIXMAP_GICC2, (gic.cbase >> PAGE_SHIFT) + 0x1, DEV_SHARED);
> +    set_fixmap(FIXMAP_GICH, gic.hbase >> PAGE_SHIFT, DEV_SHARED);
> +
> +    /* Global settings: interrupt distributor */
> +    spin_lock_init(&gic.lock);
> +    spin_lock(&gic.lock);
> +
> +    gic_smp_init();
> +    gic_dist_init();
> +    gic_cpu_init();
> +    gic_hyp_init();
> +
> +    register_gic_ops(&gic_ops);
> +    spin_unlock(&gic.lock);
> +}
> +
> +/*
> + * Local variables:
> + * mode: C
> + * c-file-style: "BSD"
> + * c-basic-offset: 4
> + * indent-tabs-mode: nil
> + * End:
> + */
> diff --git a/xen/arch/arm/gic.c b/xen/arch/arm/gic.c
> index bb718f6..e0859ae 100644
> --- a/xen/arch/arm/gic.c
> +++ b/xen/arch/arm/gic.c
> @@ -33,68 +33,46 @@
>  #include <asm/domain.h>
>  #include <asm/platform.h>
>  
> -#include <asm/gic_v2_defs.h>
>  #include <asm/gic.h>
>  
> -/* Access to the GIC Distributor registers through the fixmap */
> -#define GICD ((volatile uint32_t *) FIXMAP_ADDR(FIXMAP_GICD))
> -#define GICC ((volatile uint32_t *) FIXMAP_ADDR(FIXMAP_GICC1))
> -#define GICH ((volatile uint32_t *) FIXMAP_ADDR(FIXMAP_GICH))
>  static void gic_restore_pending_irqs(struct vcpu *v);
>  
> -struct gic_state_data {
> -    uint32_t gic_hcr;
> -    uint32_t gic_vmcr;
> -    uint32_t gic_apr;
> -    uint32_t gic_lr[64];
> -};
> -
> -/* Global state */
> -static struct {
> -    paddr_t dbase;       /* Address of distributor registers */
> -    paddr_t cbase;       /* Address of CPU interface registers */
> -    paddr_t hbase;       /* Address of virtual interface registers */
> -    paddr_t vbase;       /* Address of virtual cpu interface registers */
> -    unsigned int lines;  /* Number of interrupts (SPIs + PPIs + SGIs) */
> -    struct dt_irq maintenance; /* IRQ maintenance */
> -    unsigned int cpus;
> -    spinlock_t lock;
> -} gic;
> -
>  static irq_desc_t irq_desc[NR_IRQS];
>  static DEFINE_PER_CPU(irq_desc_t[NR_LOCAL_IRQS], local_irq_desc);
>  static DEFINE_PER_CPU(uint64_t, lr_mask);
>  
> -static unsigned nr_lrs;
> -
> -/* The GIC mapping of CPU interfaces does not necessarily match the
> - * logical CPU numbering. Let's use mapping as returned by the GIC
> - * itself
> - */
> -static DEFINE_PER_CPU(u8, gic_cpu_id);
> +static struct gic_hw_operations *gic_hw_ops;
>  
> -/* Maximum cpu interface per GIC */
> -#define NR_GIC_CPU_IF 8
> +spinlock_t gic_lock;
>  
> -static unsigned int gic_cpu_mask(const cpumask_t *cpumask)
> +void register_gic_ops(struct gic_hw_operations *ops)
>  {
> -    unsigned int cpu;
> -    unsigned int mask = 0;
> -    cpumask_t possible_mask;
> +    gic_hw_ops = ops;
> +}
>  
> -    cpumask_and(&possible_mask, cpumask, &cpu_possible_map);
> -    for_each_cpu(cpu, &possible_mask)
> -    {
> -        ASSERT(cpu < NR_GIC_CPU_IF);
> -        mask |= per_cpu(gic_cpu_id, cpu);
> -    }
> +void update_cpu_lr_mask(void)
> +{    
> +    this_cpu(lr_mask) = 0ULL;
> +}
>  
> -    return mask;
> +int gic_hw_version(void)
> +{
> +   return gic_hw_ops->gic_type();
>  }
>  
>  unsigned int gic_number_lines(void)
>  {
> -    return gic.lines;
> +    return gic_hw_ops->nr_lines();
> +}
> +
> +unsigned long gic_data_rdist_rd_base(void)
> +{
> +   return gic_hw_ops->read_cpu_rbase();
> +}
> +
> +unsigned long gic_data_rdist_sgi_base(void)
> +{
> +   return gic_hw_ops->read_cpu_sgi_rbase();
>  }
>  
>  irq_desc_t *__irq_to_desc(int irq)
> @@ -105,43 +83,21 @@ irq_desc_t *__irq_to_desc(int irq)
>  
>  void gic_save_state(struct vcpu *v)
>  {
> -    int i;
> -    struct gic_state_data *d;
> -    d = (struct gic_state_data *)v->arch.gic_state;
> -
>      ASSERT(!local_irq_is_enabled());
>  
> -    /* No need for spinlocks here because interrupts are disabled around
> -     * this call and it only accesses struct vcpu fields that cannot be
> -     * accessed simultaneously by another pCPU.
> -     */
> -    for ( i=0; i<nr_lrs; i++)
> -        d->gic_lr[i] = GICH[GICH_LR + i];
>      v->arch.lr_mask = this_cpu(lr_mask);
> -    d->gic_apr = GICH[GICH_APR];
> -    d->gic_vmcr = GICH[GICH_VMCR];
> -    /* Disable until next VCPU scheduled */
> -    GICH[GICH_HCR] = 0;
> +    gic_hw_ops->save_state(v);
>      isb();
>  }
>  
>  void gic_restore_state(struct vcpu *v)
>  {
> -    int i;
> -    struct gic_state_data *d;
> -    d = (struct gic_state_data *)v->arch.gic_state;
> -
>      if ( is_idle_vcpu(v) )
>          return;
>  
>      this_cpu(lr_mask) = v->arch.lr_mask;
> -    for ( i=0; i<nr_lrs; i++)
> -        GICH[GICH_LR + i] = d->gic_lr[i];
> -    GICH[GICH_APR] = d->gic_apr;
> -    GICH[GICH_VMCR] = d->gic_vmcr;
> -    GICH[GICH_HCR] = GICH_HCR_EN;
> +    gic_hw_ops->restore_state(v);
>      isb();
> -
>      gic_restore_pending_irqs(v);
>  }
>  
> @@ -151,12 +107,12 @@ static void gic_irq_enable(struct irq_desc *desc)
>      unsigned long flags;
>  
>      spin_lock_irqsave(&desc->lock, flags);
> -    spin_lock(&gic.lock);
> +    spin_lock(&gic_lock);
>      desc->status &= ~IRQ_DISABLED;
>      dsb();
>      /* Enable routing */
> -    GICD[GICD_ISENABLER + irq / 32] = (1u << (irq % 32));
> -    spin_unlock(&gic.lock);
> +    gic_hw_ops->enable_irq(irq);
> +    spin_unlock(&gic_lock);
>      spin_unlock_irqrestore(&desc->lock, flags);
>  }
>  
> @@ -166,11 +122,11 @@ static void gic_irq_disable(struct irq_desc *desc)
>      unsigned long flags;
>  
>      spin_lock_irqsave(&desc->lock, flags);
> -    spin_lock(&gic.lock);
> +    spin_lock(&gic_lock);
>      /* Disable routing */
> -    GICD[GICD_ICENABLER + irq / 32] = (1u << (irq % 32));
> +    gic_hw_ops->disable_irq(irq);
>      desc->status |= IRQ_DISABLED;
> -    spin_unlock(&gic.lock);
> +    spin_unlock(&gic_lock);
>      spin_unlock_irqrestore(&desc->lock, flags);
>  }
>  
> @@ -194,17 +150,16 @@ static void gic_host_irq_end(struct irq_desc *desc)
>  {
>      int irq = desc->irq;
>      /* Lower the priority */
> -    GICC[GICC_EOIR] = irq;
> -    /* Deactivate */
> -    GICC[GICC_DIR] = irq;
> +    gic_hw_ops->eoi_irq(irq);
> +    gic_hw_ops->deactivate_irq(irq);
>  }
>  
>  static void gic_guest_irq_end(struct irq_desc *desc)
>  {
>      int irq = desc->irq;
>      /* Lower the priority of the IRQ */
> -    GICC[GICC_EOIR] = irq;
>      /* Deactivation happens in maintenance interrupt / via GICV */
> +    gic_hw_ops->eoi_irq(irq);
>  }
>  
>  static void gic_irq_set_affinity(struct irq_desc *desc, const cpumask_t *mask)
> @@ -243,27 +198,7 @@ static void gic_set_irq_properties(unsigned int irq, bool_t level,
>                                     const cpumask_t *cpu_mask,
>                                     unsigned int priority)
>  {
> -    volatile unsigned char *bytereg;
> -    uint32_t cfg, edgebit;
> -    unsigned int mask = gic_cpu_mask(cpu_mask);
> -
> -    /* Set edge / level */
> -    cfg = GICD[GICD_ICFGR + irq / 16];
> -    edgebit = 2u << (2 * (irq % 16));
> -    if ( level )
> -        cfg &= ~edgebit;
> -    else
> -        cfg |= edgebit;
> -    GICD[GICD_ICFGR + irq / 16] = cfg;
> -
> -    /* Set target CPU mask (RAZ/WI on uniprocessor) */
> -    bytereg = (unsigned char *) (GICD + GICD_ITARGETSR);
> -    bytereg[irq] = mask;
> -
> -    /* Set priority */
> -    bytereg = (unsigned char *) (GICD + GICD_IPRIORITYR);
> -    bytereg[irq] = priority;
> -
> +    return gic_hw_ops->set_irq_property(irq, level, cpu_mask, priority);
>  }
>  
>  /* Program the GIC to route an interrupt */
> @@ -273,8 +208,8 @@ static int gic_route_irq(unsigned int irq, bool_t level,
>      struct irq_desc *desc = irq_to_desc(irq);
>      unsigned long flags;
>  
> -    ASSERT(priority <= 0xff);     /* Only 8 bits of priority */
> -    ASSERT(irq < gic.lines);      /* Can't route interrupts that don't exist */
> +    ASSERT(priority <= 0xff);  /* Only 8 bits of priority */
> +    ASSERT(irq < gic_number_lines());
>  
>      if ( desc->action != NULL )
>          return -EBUSY;
> @@ -286,9 +221,9 @@ static int gic_route_irq(unsigned int irq, bool_t level,
>  
>      desc->handler = &gic_host_irq_type;
>  
> -    spin_lock(&gic.lock);
> +    spin_lock(&gic_lock);
>      gic_set_irq_properties(irq, level, cpu_mask, priority);
> -    spin_unlock(&gic.lock);
> +    spin_unlock(&gic_lock);
>  
>      spin_unlock_irqrestore(&desc->lock, flags);
>      return 0;
> @@ -305,119 +240,6 @@ void gic_route_dt_irq(const struct dt_irq *irq, const cpumask_t *cpu_mask,
>      gic_route_irq(irq->irq, level, cpu_mask, priority);
>  }
>  
> -static void __init gic_dist_init(void)
> -{
> -    uint32_t type;
> -    uint32_t cpumask;
> -    int i;
> -
> -    cpumask = GICD[GICD_ITARGETSR] & 0xff;
> -    cpumask |= cpumask << 8;
> -    cpumask |= cpumask << 16;
> -
> -    /* Disable the distributor */
> -    GICD[GICD_CTLR] = 0;
> -
> -    type = GICD[GICD_TYPER];
> -    gic.lines = 32 * ((type & GICD_TYPE_LINES) + 1);
> -    gic.cpus = 1 + ((type & GICD_TYPE_CPUS) >> 5);
> -    printk("GIC: %d lines, %d cpu%s%s (IID %8.8x).\n",
> -           gic.lines, gic.cpus, (gic.cpus == 1) ? "" : "s",
> -           (type & GICD_TYPE_SEC) ? ", secure" : "",
> -           GICD[GICD_IIDR]);
> -
> -    /* Default all global IRQs to level, active low */
> -    for ( i = 32; i < gic.lines; i += 16 )
> -        GICD[GICD_ICFGR + i / 16] = 0x0;
> -
> -    /* Route all global IRQs to this CPU */
> -    for ( i = 32; i < gic.lines; i += 4 )
> -        GICD[GICD_ITARGETSR + i / 4] = cpumask;
> -
> -    /* Default priority for global interrupts */
> -    for ( i = 32; i < gic.lines; i += 4 )
> -        GICD[GICD_IPRIORITYR + i / 4] =
> -            GIC_PRI_IRQ<<24 | GIC_PRI_IRQ<<16 | GIC_PRI_IRQ<<8 | GIC_PRI_IRQ;
> -
> -    /* Disable all global interrupts */
> -    for ( i = 32; i < gic.lines; i += 32 )
> -        GICD[GICD_ICENABLER + i / 32] = (uint32_t)~0ul;
> -
> -    /* Turn on the distributor */
> -    GICD[GICD_CTLR] = GICD_CTL_ENABLE;
> -}
> -
> -static void __cpuinit gic_cpu_init(void)
> -{
> -    int i;
> -
> -    this_cpu(gic_cpu_id) = GICD[GICD_ITARGETSR] & 0xff;
> -
> -    /* The first 32 interrupts (PPI and SGI) are banked per-cpu, so
> -     * even though they are controlled with GICD registers, they must
> -     * be set up here with the other per-cpu state. */
> -    GICD[GICD_ICENABLER] = 0xffff0000; /* Disable all PPI */
> -    GICD[GICD_ISENABLER] = 0x0000ffff; /* Enable all SGI */
> -    /* Set SGI priorities */
> -    for (i = 0; i < 16; i += 4)
> -        GICD[GICD_IPRIORITYR + i / 4] =
> -            GIC_PRI_IPI<<24 | GIC_PRI_IPI<<16 | GIC_PRI_IPI<<8 | GIC_PRI_IPI;
> -    /* Set PPI priorities */
> -    for (i = 16; i < 32; i += 4)
> -        GICD[GICD_IPRIORITYR + i / 4] =
> -            GIC_PRI_IRQ<<24 | GIC_PRI_IRQ<<16 | GIC_PRI_IRQ<<8 | GIC_PRI_IRQ;
> -
> -    /* Local settings: interface controller */
> -    GICC[GICC_PMR] = 0xff;                /* Don't mask by priority */
> -    GICC[GICC_BPR] = 0;                   /* Finest granularity of priority */
> -    GICC[GICC_CTLR] = GICC_CTL_ENABLE|GICC_CTL_EOI;    /* Turn on delivery */
> -}
> -
> -static void gic_cpu_disable(void)
> -{
> -    GICC[GICC_CTLR] = 0;
> -}
> -
> -static void __cpuinit gic_hyp_init(void)
> -{
> -    uint32_t vtr;
> -
> -    vtr = GICH[GICH_VTR];
> -    nr_lrs  = (vtr & GICH_VTR_NRLRGS) + 1;
> -
> -    GICH[GICH_MISR] = GICH_MISR_EOI;
> -    this_cpu(lr_mask) = 0ULL;
> -}
> -
> -static void __cpuinit gic_hyp_disable(void)
> -{
> -    GICH[GICH_HCR] = 0;
> -}
> -
> -/* Set up the per-CPU parts of the GIC for a secondary CPU */
> -static int  __cpuinit gic_init_secondary_cpu(struct notifier_block *nfb,
> -                                        unsigned long action, void *hcpu)
> -{
> -    if (action == CPU_STARTING)
> -    {
> -        spin_lock(&gic.lock);
> -        gic_cpu_init();
> -        gic_hyp_init();
> -        spin_unlock(&gic.lock);
> -    }
> -    return NOTIFY_DONE;
> -}
> -
> -static struct notifier_block gic_cpu_nb = { 
> -    .notifier_call = gic_init_secondary_cpu,
> -    .priority = 100 
> -};
> -
> -static void gic_smp_init(void)
> -{
> -   register_cpu_notifier(&gic_cpu_nb);
> -}
> -
>  int gic_irq_xlate(const u32 *intspec, unsigned int intsize,
>                    unsigned int *out_hwirq,
>                    unsigned int *out_type)
> @@ -438,124 +260,25 @@ int gic_irq_xlate(const u32 *intspec, unsigned int intsize,
>      return 0;
>  }
>  
> -/* Set up the GIC */
> -void __init gic_init(void)
> -{
> -    static const struct dt_device_match gic_ids[] __initconst =
> -    {
> -        DT_MATCH_GIC,
> -        { /* sentinel */ },
> -    };
> -    struct dt_device_node *node;
> -    int res;
> -
> -    node = dt_find_interrupt_controller(gic_ids);
> -    if ( !node )
> -        panic("Unable to find compatible GIC in the device tree");
> -
> -    dt_device_set_used_by(node, DOMID_XEN);
> -
> -    res = dt_device_get_address(node, 0, &gic.dbase, NULL);
> -    if ( res || !gic.dbase || (gic.dbase & ~PAGE_MASK) )
> -        panic("GIC: Cannot find a valid address for the distributor");
> -
> -    res = dt_device_get_address(node, 1, &gic.cbase, NULL);
> -    if ( res || !gic.cbase || (gic.cbase & ~PAGE_MASK) )
> -        panic("GIC: Cannot find a valid address for the CPU");
> -
> -    res = dt_device_get_address(node, 2, &gic.hbase, NULL);
> -    if ( res || !gic.hbase || (gic.hbase & ~PAGE_MASK) )
> -        panic("GIC: Cannot find a valid address for the hypervisor");
> -
> -    res = dt_device_get_address(node, 3, &gic.vbase, NULL);
> -    if ( res || !gic.vbase || (gic.vbase & ~PAGE_MASK) )
> -        panic("GIC: Cannot find a valid address for the virtual CPU");
> -
> -    res = dt_device_get_irq(node, 0, &gic.maintenance);
> -    if ( res )
> -        panic("GIC: Cannot find the maintenance IRQ");
> -
> -    /* Set the GIC as the primary interrupt controller */
> -    dt_interrupt_controller = node;
> -
> -    /* TODO: Add check on distributor, cpu size */
> -
> -    printk("GIC initialization:\n"
> -              "        gic_dist_addr=%"PRIpaddr"\n"
> -              "        gic_cpu_addr=%"PRIpaddr"\n"
> -              "        gic_hyp_addr=%"PRIpaddr"\n"
> -              "        gic_vcpu_addr=%"PRIpaddr"\n"
> -              "        gic_maintenance_irq=%u\n",
> -              gic.dbase, gic.cbase, gic.hbase, gic.vbase,
> -              gic.maintenance.irq);
> -
> -    if ( (gic.dbase & ~PAGE_MASK) || (gic.cbase & ~PAGE_MASK) ||
> -         (gic.hbase & ~PAGE_MASK) || (gic.vbase & ~PAGE_MASK) )
> -        panic("GIC interfaces not page aligned");
> -
> -    set_fixmap(FIXMAP_GICD, gic.dbase >> PAGE_SHIFT, DEV_SHARED);
> -    BUILD_BUG_ON(FIXMAP_ADDR(FIXMAP_GICC1) !=
> -                 FIXMAP_ADDR(FIXMAP_GICC2)-PAGE_SIZE);
> -    set_fixmap(FIXMAP_GICC1, gic.cbase >> PAGE_SHIFT, DEV_SHARED);
> -    if ( platform_has_quirk(PLATFORM_QUIRK_GIC_64K_STRIDE) )
> -        set_fixmap(FIXMAP_GICC2, (gic.cbase >> PAGE_SHIFT) + 0x10, DEV_SHARED);
> -    else
> -        set_fixmap(FIXMAP_GICC2, (gic.cbase >> PAGE_SHIFT) + 0x1, DEV_SHARED);
> -    set_fixmap(FIXMAP_GICH, gic.hbase >> PAGE_SHIFT, DEV_SHARED);
> -
> -    /* Global settings: interrupt distributor */
> -    spin_lock_init(&gic.lock);
> -    spin_lock(&gic.lock);
> -
> -    gic_smp_init();
> -    gic_dist_init();
> -    gic_cpu_init();
> -    gic_hyp_init();
> -
> -    spin_unlock(&gic.lock);
> -}
> -
>  void send_SGI_mask(const cpumask_t *cpumask, enum gic_sgi sgi)
>  {
> -    unsigned int mask = 0;
> -    cpumask_t online_mask;
> -
> -    ASSERT(sgi < 16); /* There are only 16 SGIs */
> -
> -    cpumask_and(&online_mask, cpumask, &cpu_online_map);
> -    mask = gic_cpu_mask(&online_mask);
> -
> -    dsb();
> -
> -    GICD[GICD_SGIR] = GICD_SGI_TARGET_LIST
> -        | (mask<<GICD_SGI_TARGET_SHIFT)
> -        | sgi;
> +    gic_hw_ops->send_sgi(cpumask, sgi);
>  }
>  
> -void send_SGI_one(unsigned int cpu, enum gic_sgi sgi)
> +static void send_SGI_one(unsigned int cpu, enum gic_sgi sgi)
>  {
> -    ASSERT(cpu < NR_GIC_CPU_IF);  /* Targets bitmap only supports 8 CPUs */
>      send_SGI_mask(cpumask_of(cpu), sgi);
>  }
>  
> -void send_SGI_self(enum gic_sgi sgi)
> -{
> -    ASSERT(sgi < 16); /* There are only 16 SGIs */
> -
> -    dsb();
> -
> -    GICD[GICD_SGIR] = GICD_SGI_TARGET_SELF
> -        | sgi;
> -}
> -
>  void send_SGI_allbutself(enum gic_sgi sgi)
>  {
> -   ASSERT(sgi < 16); /* There are only 16 SGIs */
> +    cpumask_t all_others_mask;
> +    ASSERT(sgi < 16); /* There are only 16 SGIs */
>  
> -   dsb();
> +    cpumask_andnot(&all_others_mask, &cpu_possible_map, cpumask_of(smp_processor_id()));  
>  
> -   GICD[GICD_SGIR] = GICD_SGI_TARGET_OTHERS
> -       | sgi;
> +    dsb();
> +    send_SGI_mask(&all_others_mask, sgi);
>  }
>  
>  void smp_send_state_dump(unsigned int cpu)
> @@ -568,16 +291,15 @@ void gic_disable_cpu(void)
>  {
>      ASSERT(!local_irq_is_enabled());
>  
> -    spin_lock(&gic.lock);
> -    gic_cpu_disable();
> -    gic_hyp_disable();
> -    spin_unlock(&gic.lock);
> +    spin_lock(&gic_lock);
> +    gic_hw_ops->disable_interface();
> +    spin_unlock(&gic_lock);
>  }
>  
>  void gic_route_ppis(void)
>  {
>      /* GIC maintenance */
> -    gic_route_dt_irq(&gic.maintenance, cpumask_of(smp_processor_id()),
> +    gic_route_dt_irq(gic_hw_ops->get_maintenance_irq(), cpumask_of(smp_processor_id()),
>                       GIC_PRI_IRQ);
>      /* Route timer interrupt */
>      route_timer_interrupt();
> @@ -652,20 +374,10 @@ int __init setup_dt_irq(const struct dt_irq *irq, struct irqaction *new)
>      return rc;
>  }
>  
> -static inline void gic_set_lr(int lr, struct pending_irq *p,
> +static void gic_set_lr(int lr, struct pending_irq *p,
>          unsigned int state)
>  {
> -    int maintenance_int = GICH_LR_MAINTENANCE_IRQ;
> -
> -    BUG_ON(lr >= nr_lrs);
> -    BUG_ON(lr < 0);
> -    BUG_ON(state & ~(GICH_LR_STATE_MASK<<GICH_LR_STATE_SHIFT));
> -
> -    GICH[GICH_LR + lr] = state |
> -        maintenance_int |
> -        ((p->priority >> 3) << GICH_LR_PRIORITY_SHIFT) |
> -        ((p->irq & GICH_LR_VIRTUAL_MASK) << GICH_LR_VIRTUAL_SHIFT);
> -
> +    gic_hw_ops->update_lr(lr, p->irq, state, p->priority);
>      set_bit(GIC_IRQ_GUEST_VISIBLE, &p->status);
>      clear_bit(GIC_IRQ_GUEST_PENDING, &p->status);
>  }
> @@ -693,10 +405,10 @@ void gic_remove_from_queues(struct vcpu *v, unsigned int virtual_irq)
>      struct pending_irq *p = irq_to_pending(v, virtual_irq);
>      unsigned long flags;
>  
> -    spin_lock_irqsave(&gic.lock, flags);
> +    spin_lock_irqsave(&gic_lock, flags);
>      if ( !list_empty(&p->lr_queue) )
>          list_del_init(&p->lr_queue);
> -    spin_unlock_irqrestore(&gic.lock, flags);
> +    spin_unlock_irqrestore(&gic_lock, flags);
>  }
>  
>  void gic_set_guest_irq(struct vcpu *v, unsigned int virtual_irq,
> @@ -704,9 +416,9 @@ void gic_set_guest_irq(struct vcpu *v, unsigned int virtual_irq,
>  {
>      int i;
>      unsigned long flags;
> +    unsigned int nr_lrs = gic_hw_ops->nr_lrs();
>  
> -    spin_lock_irqsave(&gic.lock, flags);
> -
> +    spin_lock_irqsave(&gic_lock, flags);
>      if ( v == current && list_empty(&v->arch.vgic.lr_pending) )
>      {
>          i = find_first_zero_bit(&this_cpu(lr_mask), nr_lrs);
> @@ -720,7 +432,7 @@ void gic_set_guest_irq(struct vcpu *v, unsigned int virtual_irq,
>      gic_add_to_lr_pending(v, irq_to_pending(v, virtual_irq));
>  
>  out:
> -    spin_unlock_irqrestore(&gic.lock, flags);
> +    spin_unlock_irqrestore(&gic_lock, flags);
>      return;
>  }
>  
> @@ -729,17 +441,18 @@ static void gic_restore_pending_irqs(struct vcpu *v)
>      int i;
>      struct pending_irq *p, *t;
>      unsigned long flags;
> +    unsigned int nr_lrs = gic_hw_ops->nr_lrs();
>  
>      list_for_each_entry_safe ( p, t, &v->arch.vgic.lr_pending, lr_queue )
>      {
>          i = find_first_zero_bit(&this_cpu(lr_mask), nr_lrs);
>          if ( i >= nr_lrs ) return;
>  
> -        spin_lock_irqsave(&gic.lock, flags);
> +        spin_lock_irqsave(&gic_lock, flags);
>          gic_set_lr(i, p, GICH_LR_PENDING);
>          list_del_init(&p->lr_queue);
>          set_bit(i, &this_cpu(lr_mask));
> -        spin_unlock_irqrestore(&gic.lock, flags);
> +        spin_unlock_irqrestore(&gic_lock, flags);
>      }
>  
>  }
> @@ -749,11 +462,11 @@ void gic_clear_pending_irqs(struct vcpu *v)
>      struct pending_irq *p, *t;
>      unsigned long flags;
>  
> -    spin_lock_irqsave(&gic.lock, flags);
> +    spin_lock_irqsave(&gic_lock, flags);
>      v->arch.lr_mask = 0;
>      list_for_each_entry_safe ( p, t, &v->arch.vgic.lr_pending, lr_queue )
>          list_del_init(&p->lr_queue);
> -    spin_unlock_irqrestore(&gic.lock, flags);
> +    spin_unlock_irqrestore(&gic_lock, flags);
>  }
>  
>  static void gic_inject_irq_start(void)
> @@ -809,7 +522,7 @@ int gic_route_irq_to_guest(struct domain *d, const struct dt_irq *irq,
>      action->free_on_release = 1;
>  
>      spin_lock_irqsave(&desc->lock, flags);
> -    spin_lock(&gic.lock);
> +    spin_lock(&gic_lock);
>  
>      desc->handler = &gic_guest_irq_type;
>      desc->status |= IRQ_GUEST;
> @@ -830,15 +543,15 @@ int gic_route_irq_to_guest(struct domain *d, const struct dt_irq *irq,
>      p->desc = desc;
>  
>  out:
> -    spin_unlock(&gic.lock);
> +    spin_unlock(&gic_lock);
>      spin_unlock_irqrestore(&desc->lock, flags);
>      return retval;
>  }
>  
> -static void do_sgi(struct cpu_user_regs *regs, int othercpu, enum gic_sgi sgi)
> +static void do_sgi(struct cpu_user_regs *regs, enum gic_sgi sgi)
>  {
>      /* Lower the priority */
> -    GICC[GICC_EOIR] = sgi;
> +    gic_hw_ops->eoi_irq(sgi);
>  
>      switch (sgi)
>      {
> @@ -857,20 +570,17 @@ static void do_sgi(struct cpu_user_regs *regs, int othercpu, enum gic_sgi sgi)
>      }
>  
>      /* Deactivate */
> -    GICC[GICC_DIR] = sgi;
> +    gic_hw_ops->deactivate_irq(sgi);
>  }
>  
>  /* Accept an interrupt from the GIC and dispatch its handler */
>  void gic_interrupt(struct cpu_user_regs *regs, int is_fiq)
>  {
> -    uint32_t intack;
>      unsigned int irq;
>  
>  
>      do  {
> -        intack = GICC[GICC_IAR];
> -        irq = intack & GICC_IA_IRQ;
> -
> +        irq = gic_hw_ops->ack_irq();
>          if ( likely(irq >= 16 && irq < 1021) )
>          {
>              local_irq_enable();
> @@ -879,8 +589,7 @@ void gic_interrupt(struct cpu_user_regs *regs, int is_fiq)
>          }
>          else if (unlikely(irq < 16))
>          {
> -            unsigned int cpu = (intack & GICC_IA_CPU_MASK) >> GICC_IA_CPU_SHIFT;
> -            do_sgi(regs, cpu, irq);
> +            do_sgi(regs, irq);
>          }
>          else
>          {
> @@ -892,72 +601,31 @@ void gic_interrupt(struct cpu_user_regs *regs, int is_fiq)
>  
>  int vcpu_gic_init(struct vcpu *v)
>  {
> -     v->arch.gic_state = xzalloc(struct gic_state_data);
> -     if(!v->arch.gic_state)
> -        return -ENOMEM;
> -     return 0;
> +     return gic_hw_ops->state_init(v);
>  }
>  
>  int gicv_setup(struct domain *d)
>  {
>      int ret;
>  
> -    /*
> -     * Domain 0 gets the hardware address.
> -     * Guests get the virtual platform layout.
> -     */
> -    if ( d->domain_id == 0 )
> -    {
> -        d->arch.vgic.dbase = gic.dbase;
> -        d->arch.vgic.cbase = gic.cbase;
> -    }
> -    else
> -    {
> -        d->arch.vgic.dbase = GUEST_GICD_BASE;
> -        d->arch.vgic.cbase = GUEST_GICC_BASE;
> -    }
> -
> -    d->arch.vgic.nr_lines = 0;
> -
> -    /*
> -     * Map the gic virtual cpu interface in the gic cpu interface
> -     * region of the guest.
> -     *
> -     * The second page is always mapped at +4K irrespective of the
> -     * GIC_64K_STRIDE quirk. The DTB passed to the guest reflects this.
> -     */
> -    ret = map_mmio_regions(d, d->arch.vgic.cbase,
> -                           d->arch.vgic.cbase + PAGE_SIZE - 1,
> -                           gic.vbase);
> -    if (ret)
> -        return ret;
> -
> -    if ( !platform_has_quirk(PLATFORM_QUIRK_GIC_64K_STRIDE) )
> -        ret = map_mmio_regions(d, d->arch.vgic.cbase + PAGE_SIZE,
> -                               d->arch.vgic.cbase + (2 * PAGE_SIZE) - 1,
> -                               gic.vbase + PAGE_SIZE);
> -    else
> -        ret = map_mmio_regions(d, d->arch.vgic.cbase + PAGE_SIZE,
> -                               d->arch.vgic.cbase + (2 * PAGE_SIZE) - 1,
> -                               gic.vbase + 16*PAGE_SIZE);
> -
> +    ret = gic_hw_ops->gicv_setup(d);
>      return ret;
>  
>  }
>  
>  static void gic_irq_eoi(void *info)
> -{
> +{ 
>      int virq = (uintptr_t) info;
> -    GICC[GICC_DIR] = virq;
> +    gic_hw_ops->deactivate_irq(virq);
>  }
>  
>  static void maintenance_interrupt(int irq, void *dev_id, struct cpu_user_regs *regs)
>  {
>      int i = 0, virq, pirq = -1;
> -    uint32_t lr;
>      struct vcpu *v = current;
> -    uint64_t eisr = GICH[GICH_EISR0] | (((uint64_t) GICH[GICH_EISR1]) << 32);
> +    unsigned long eisr = 0;
>  
> +    eisr = gic_hw_ops->read_eisr();
>      while ((i = find_next_bit((const long unsigned int *) &eisr,
>                                64, i)) < 64) {
>          struct pending_irq *p, *p2;
> @@ -967,10 +635,9 @@ static void maintenance_interrupt(int irq, void *dev_id, struct cpu_user_regs *r
>          cpu = -1;
>          inflight = 0;
>  
> -        spin_lock_irq(&gic.lock);
> -        lr = GICH[GICH_LR + i];
> -        virq = lr & GICH_LR_VIRTUAL_MASK;
> -        GICH[GICH_LR + i] = 0;
> +        spin_lock_irq(&gic_lock);
> +        virq = gic_hw_ops->update_lr_for_mi(i);
> +
>          clear_bit(i, &this_cpu(lr_mask));
>  
>          p = irq_to_pending(v, virq);
> @@ -995,7 +662,7 @@ static void maintenance_interrupt(int irq, void *dev_id, struct cpu_user_regs *r
>              list_del_init(&p2->lr_queue);
>              set_bit(i, &this_cpu(lr_mask));
>          }
> -        spin_unlock_irq(&gic.lock);
> +        spin_unlock_irq(&gic_lock);
>  
>          if ( !inflight )
>          {
> @@ -1018,22 +685,37 @@ static void maintenance_interrupt(int irq, void *dev_id, struct cpu_user_regs *r
>      }
>  }
>  
> +/* Set up the GIC */
> +void __init gic_init(void)
> +{
> +    static const struct dt_device_match gicv2_ids[] __initconst =
> +    {
> +        DT_MATCH_GIC,
> +        { /* sentinel */ },
> +    };
> +    struct dt_device_node *node;
> +
> +    spin_lock_init(&gic_lock);
> +
> +    spin_lock(&gic_lock);
> +    node = dt_find_interrupt_controller(gicv2_ids);
> +    if ( node )
> +    {
> +        gicv2_init();
> +        spin_unlock(&gic_lock);
> +        return;
> +    }
> +    spin_unlock(&gic_lock);
> +    if ( !node )
> +        panic("Unable to find compatible GIC in the device tree");
> +}
> +
>  void gic_dump_info(struct vcpu *v)
>  {
> -    int i;
>      struct pending_irq *p;
> -    struct gic_state_data *d;
> -    d = (struct gic_state_data *)v->arch.gic_state;
>  
>      printk("GICH_LRs (vcpu %d) mask=%"PRIx64"\n", v->vcpu_id, v->arch.lr_mask);
> -    if ( v == current )
> -    {
> -        for ( i = 0; i < nr_lrs; i++ )
> -            printk("   HW_LR[%d]=%x\n", i, GICH[GICH_LR + i]);
> -    } else {
> -        for ( i = 0; i < nr_lrs; i++ )
> -            printk("   VCPU_LR[%d]=%x\n", i, d->gic_lr[i]);
> -    }
> +    gic_hw_ops->dump_state(v);
>  
>      list_for_each_entry ( p, &v->arch.vgic.inflight_irqs, inflight )
>      {
> @@ -1049,7 +731,7 @@ void gic_dump_info(struct vcpu *v)
>  
>  void __cpuinit init_maintenance_interrupt(void)
>  {
> -    request_dt_irq(&gic.maintenance, maintenance_interrupt,
> +    request_dt_irq(gic_hw_ops->get_maintenance_irq(), maintenance_interrupt,
>                     "irq-maintenance", NULL);
>  }
>  
> diff --git a/xen/include/asm-arm/gic.h b/xen/include/asm-arm/gic.h
> index 18656fd..4244491 100644
> --- a/xen/include/asm-arm/gic.h
> +++ b/xen/include/asm-arm/gic.h
> @@ -39,6 +39,8 @@
>  #define GIC_PRI_IPI        0x90 /* IPIs must preempt normal interrupts */
>  #define GIC_PRI_HIGHEST    0x80 /* Higher priorities belong to Secure-World */
>  
> +#define GICH_LR_PENDING    1
> +#define GICH_LR_ACTIVE     2
>  
>  #ifndef __ASSEMBLY__
>  #include <xen/device_tree.h>
> @@ -46,12 +48,15 @@
>  #define DT_MATCH_GIC    DT_MATCH_COMPATIBLE("arm,cortex-a15-gic"), \
>                          DT_MATCH_COMPATIBLE("arm,cortex-a7-gic")
>  
> +extern int gic_hw_version(void);
>  extern int domain_vgic_init(struct domain *d);
>  extern void domain_vgic_free(struct domain *d);
>  
>  extern int vcpu_vgic_init(struct vcpu *v);
>  extern int vcpu_gic_init(struct vcpu *v);
>  
> +extern void gicv2_init(void);
> +
>  extern void vgic_vcpu_inject_irq(struct vcpu *v, unsigned int irq,int virtual);
>  extern void vgic_clear_pending_irqs(struct vcpu *v);
>  extern struct pending_irq *irq_to_pending(struct vcpu *v, unsigned int irq);
> @@ -88,14 +93,47 @@ extern int gicv_setup(struct domain *d);
>  extern void gic_save_state(struct vcpu *v);
>  extern void gic_restore_state(struct vcpu *v);
>  
> +#define GIC_VERSION_V2 0x2
> +
>  /* SGI (AKA IPIs) */
>  enum gic_sgi {
>      GIC_SGI_EVENT_CHECK = 0,
>      GIC_SGI_DUMP_STATE  = 1,
>      GIC_SGI_CALL_FUNCTION = 2,
>  };
> +
> +struct gic_hw_operations {
> +    int (*gic_type)(void);
> +    struct dt_irq * (*get_maintenance_irq)(void);
> +    unsigned int (*nr_lines)(void);
> +    unsigned int (*nr_lrs)(void);
> +    int (*state_init)(struct vcpu *);
> +    void (*save_state)(struct vcpu *);
> +    void (*restore_state)(struct vcpu *);
> +    void (*dump_state)(struct vcpu *);
> +    int (*gicv_setup)(struct domain *);
> +    void (*enable_irq)(int);
> +    void (*disable_irq)(int);
> +    void (*eoi_irq)(int);
> +    void (*deactivate_irq)(int);
> +    unsigned int (*ack_irq)(void);
> +    void (*set_irq_property)(unsigned int irq, bool_t level,
> +                            const cpumask_t *cpu_mask,
> +                            unsigned int priority);
> +    void (*send_sgi)(const cpumask_t *, enum gic_sgi);
> +    void (*disable_interface)(void);
> +    void (*update_lr)(int lr, unsigned int virtual_irq,
> +                          unsigned int state, unsigned int priority);
> +    unsigned int (*update_lr_for_mi)(int lr);
> +    unsigned long (*read_eisr)(void);
> +    unsigned long (*read_cpu_rbase)(void);
> +    unsigned long (*read_cpu_sgi_rbase)(void);
> +};
> +
> +extern void register_gic_ops(struct gic_hw_operations *);
> +
> +extern void update_cpu_lr_mask(void);
>  extern void send_SGI_mask(const cpumask_t *cpumask, enum gic_sgi sgi);
> -extern void send_SGI_one(unsigned int cpu, enum gic_sgi sgi);
>  extern void send_SGI_self(enum gic_sgi sgi);
>  extern void send_SGI_allbutself(enum gic_sgi sgi);
>  
> diff --git a/xen/include/asm-arm/gic_v2_defs.h b/xen/include/asm-arm/gic_v2_defs.h
> index 2366685..5ddd48a 100644
> --- a/xen/include/asm-arm/gic_v2_defs.h
> +++ b/xen/include/asm-arm/gic_v2_defs.h
> @@ -119,8 +119,6 @@
>  #define GICH_LR_STATE_SHIFT     28
>  #define GICH_LR_PRIORITY_SHIFT  23
>  #define GICH_LR_MAINTENANCE_IRQ (1<<19)
> -#define GICH_LR_PENDING         (1<<28)
> -#define GICH_LR_ACTIVE          (1<<29)
>  #define GICH_LR_GRP1            (1<<30)
>  #define GICH_LR_HW              (1<<31)
>  #define GICH_LR_CPUID_SHIFT     9
> -- 
> 1.7.9.5
> 

Re: [RFC PATCH v1 06/10] xen/arm: split gic driver into generic and gicv2 driver

[Vijay Kilari]

On Thu, Mar 20, 2014 at 5:25 PM, Stefano Stabellini
<stefano.stabellini@eu.citrix.com> wrote:
> On Wed, 19 Mar 2014, vijay.kilari@gmail.com wrote:
>> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>>
>> Existing GIC driver has both generic code and hw specific
>> code. Segregate GIC low level driver into gic-v2.c and
>> keep generic code in existing gic.c file
>>
>> GIC v2 driver registers required functions
>> to generic GIC driver. This helps to plug in next version
>> of GIC drivers like GIC v3.
>>
>> Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>
> Am I correct in assuming that this patch is just code movement plus the
> introduction of struct gic_hw_operations?
>
Yes, as a part of code movement, I introduced this structure and
used this callbacks in generic code

>
>>  xen/arch/arm/Makefile             |    2 +-
>>  xen/arch/arm/gic-v2.c             |  562 +++++++++++++++++++++++++++++++++++++
>>  xen/arch/arm/gic.c                |  530 +++++++---------------------------
>>  xen/include/asm-arm/gic.h         |   40 ++-
>>  xen/include/asm-arm/gic_v2_defs.h |    2 -
>>  5 files changed, 708 insertions(+), 428 deletions(-)
>>
>> diff --git a/xen/arch/arm/Makefile b/xen/arch/arm/Makefile
>> index 63e0460..969ee52 100644
>> --- a/xen/arch/arm/Makefile
>> +++ b/xen/arch/arm/Makefile
>> @@ -10,7 +10,7 @@ obj-y += vpsci.o
>>  obj-y += domctl.o
>>  obj-y += sysctl.o
>>  obj-y += domain_build.o
>> -obj-y += gic.o
>> +obj-y += gic.o gic-v2.o
>>  obj-y += io.o
>>  obj-y += irq.o
>>  obj-y += kernel.o
>> diff --git a/xen/arch/arm/gic-v2.c b/xen/arch/arm/gic-v2.c
>> new file mode 100644
>> index 0000000..9378427
>> --- /dev/null
>> +++ b/xen/arch/arm/gic-v2.c
>> @@ -0,0 +1,562 @@
>> +/*
>> + * xen/arch/arm/gic-v2.c
>> + *
>> + * ARM Generic Interrupt Controller support v2
>> + *
>> + * Tim Deegan <tim@xen.org>
>> + * Copyright (c) 2011 Citrix Systems.
>> + *
>> + * This program is free software; you can redistribute it and/or modify
>> + * it under the terms of the GNU General Public License as published by
>> + * the Free Software Foundation; either version 2 of the License, or
>> + * (at your option) any later version.
>> + *
>> + * This program is distributed in the hope that it will be useful,
>> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
>> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
>> + * GNU General Public License for more details.
>> + */
>> +
>> +#include <xen/config.h>
>> +#include <xen/lib.h>
>> +#include <xen/init.h>
>> +#include <xen/cpu.h>
>> +#include <xen/mm.h>
>> +#include <xen/irq.h>
>> +#include <xen/sched.h>
>> +#include <xen/errno.h>
>> +#include <xen/serial.h>
>> +#include <xen/softirq.h>
>> +#include <xen/list.h>
>> +#include <xen/device_tree.h>
>> +#include <asm/p2m.h>
>> +#include <asm/domain.h>
>> +#include <asm/platform.h>
>> +
>> +#include <asm/gic_v2_defs.h>
>> +#include <asm/gic.h>
>> +
>> +/* Access to the GIC Distributor registers through the fixmap */
>> +#define GICD ((volatile uint32_t *) FIXMAP_ADDR(FIXMAP_GICD))
>> +#define GICC ((volatile uint32_t *) FIXMAP_ADDR(FIXMAP_GICC1))
>> +#define GICH ((volatile uint32_t *) FIXMAP_ADDR(FIXMAP_GICH))
>> +
>> +struct gic_state_data {
>> +    uint32_t gic_hcr;
>> +    uint32_t gic_vmcr;
>> +    uint32_t gic_apr;
>> +    uint32_t gic_lr[64];
>> +};
>> +
>> +/* Global state */
>> +static struct {
>> +    int hw_version;
>> +    paddr_t dbase;       /* Address of distributor registers */
>> +    paddr_t cbase;       /* Address of CPU interface registers */
>> +    paddr_t hbase;       /* Address of virtual interface registers */
>> +    paddr_t vbase;       /* Address of virtual cpu interface registers */
>> +    unsigned int lines;  /* Number of interrupts (SPIs + PPIs + SGIs) */
>> +    struct dt_irq maintenance; /* IRQ maintenance */
>> +    unsigned int cpus;
>> +    spinlock_t lock;
>> +} gic;
>> +
>> +static unsigned nr_lrs;
>> +
>> +/* The GIC mapping of CPU interfaces does not necessarily match the
>> + * logical CPU numbering. Let's use mapping as returned by the GIC
>> + * itself
>> + */
>> +static DEFINE_PER_CPU(u8, gic_cpu_id);
>> +
>> +/* Maximum cpu interface per GIC */
>> +#define NR_GIC_CPU_IF 8
>> +
>> +static unsigned int gic_cpu_mask(const cpumask_t *cpumask)
>> +{
>> +    unsigned int cpu;
>> +    unsigned int mask = 0;
>> +    cpumask_t possible_mask;
>> +
>> +    cpumask_and(&possible_mask, cpumask, &cpu_possible_map);
>> +    for_each_cpu(cpu, &possible_mask)
>> +    {
>> +        ASSERT(cpu < NR_GIC_CPU_IF);
>> +        mask |= per_cpu(gic_cpu_id, cpu);
>> +    }
>> +
>> +    return mask;
>> +}
>> +
>> +static unsigned int gic_nr_lines(void)
>> +{
>> +    return gic.lines;
>> +}
>> +
>> +static unsigned int gic_nr_lrs(void)
>> +{
>> +    return nr_lrs;
>> +}
>> +
>> +static int gic_state_init(struct vcpu *v)
>> +{
>> +     v->arch.gic_state = (struct gic_state_data *)xzalloc(struct gic_state_data);
>> +     if(!v->arch.gic_state)
>> +        return -ENOMEM;
>> +     return 0;
>> +}
>> +
>> +static void save_state(struct vcpu *v)
>> +{
>> +    int i;
>> +    struct gic_state_data *d;
>> +    d = (struct gic_state_data *)v->arch.gic_state;
>> +
>> +    ASSERT(!local_irq_is_enabled());
>> +
>> +    /* No need for spinlocks here because interrupts are disabled around
>> +     * this call and it only accesses struct vcpu fields that cannot be
>> +     * accessed simultaneously by another pCPU.
>> +     */
>> +    for ( i = 0; i < nr_lrs; i++)
>> +        d->gic_lr[i] = GICH[GICH_LR + i];
>> +    d->gic_apr = GICH[GICH_APR];
>> +    d->gic_vmcr = GICH[GICH_VMCR];
>> +    /* Disable until next VCPU scheduled */
>> +    GICH[GICH_HCR] = 0;
>> +}
>> +
>> +static void restore_state(struct vcpu *v)
>> +{
>> +    int i;
>> +    struct gic_state_data *d;
>> +    d = (struct gic_state_data *)v->arch.gic_state;
>> +
>> +    for ( i = 0; i < nr_lrs; i++)
>> +        GICH[GICH_LR + i] = d->gic_lr[i];
>> +    GICH[GICH_APR] = d->gic_apr;
>> +    GICH[GICH_VMCR] = d->gic_vmcr;
>> +    GICH[GICH_HCR] = GICH_HCR_EN;
>> +}
>> +
>> +static void gic_dump_state(struct vcpu *v)
>> +{
>> +    int i;
>> +    struct gic_state_data *d;
>> +    d = (struct gic_state_data *)v->arch.gic_state;
>> +
>> +    if ( v == current )
>> +    {
>> +        for ( i = 0; i < nr_lrs; i++ )
>> +            printk("   HW_LR[%d]=%x\n", i, GICH[GICH_LR + i]);
>> +    } else {
>> +        for ( i = 0; i < nr_lrs; i++ )
>> +            printk("   VCPU_LR[%d]=%x\n", i, d->gic_lr[i]);
>> +    }
>> +}
>> +
>> +static void gic_enable_irq(int irq)
>> +{
>> +    /* Enable routing */
>> +    GICD[GICD_ISENABLER + irq / 32] = (1u << (irq % 32));
>> +}
>> +
>> +static void gic_disable_irq(int irq)
>> +{
>> +    /* Disable routing */
>> +    GICD[GICD_ICENABLER + irq / 32] = (1u << (irq % 32));
>> +}
>> +
>> +static void gic_eoi_irq(int irq)
>> +{
>> +    /* Lower the priority */
>> +    GICC[GICC_EOIR] = irq;
>> +}
>> +
>> +static void gic_dir_irq(int irq)
>> +{
>> +    /* Deactivate */
>> +    GICC[GICC_DIR] = irq;
>> +}
>> +
>> +static unsigned int gic_ack_irq(void)
>> +{
>> +    return (GICC[GICC_IAR] & GICC_IA_IRQ);
>> +}
>> +
>> +/*
>> + * - needs to be called with gic.lock held
>> + * - needs to be called with a valid cpu_mask, ie each cpu in the mask has
>> + * already called gic_cpu_init
>> + */
>> +static void gic_set_irq_properties(unsigned int irq, bool_t level,
>> +                                   const cpumask_t *cpu_mask,
>> +                                   unsigned int priority)
>> +{
>> +    volatile unsigned char *bytereg;
>> +    uint32_t cfg, edgebit;
>> +    unsigned int mask = gic_cpu_mask(cpu_mask);
>> +
>> +    /* Set edge / level */
>> +    cfg = GICD[GICD_ICFGR + irq / 16];
>> +    edgebit = 2u << (2 * (irq % 16));
>> +    if ( level )
>> +        cfg &= ~edgebit;
>> +    else
>> +        cfg |= edgebit;
>> +    GICD[GICD_ICFGR + irq / 16] = cfg;
>> +
>> +    /* Set target CPU mask (RAZ/WI on uniprocessor) */
>> +    bytereg = (unsigned char *) (GICD + GICD_ITARGETSR);
>> +    bytereg[irq] = mask;
>> +
>> +    /* Set priority */
>> +    bytereg = (unsigned char *) (GICD + GICD_IPRIORITYR);
>> +    bytereg[irq] = priority;
>> +
>> +}
>> +
>> +static void __init gic_dist_init(void)
>> +{
>> +    uint32_t type;
>> +    uint32_t cpumask;
>> +    int i;
>> +
>> +    cpumask = GICD[GICD_ITARGETSR] & 0xff;
>> +    cpumask |= cpumask << 8;
>> +    cpumask |= cpumask << 16;
>> +
>> +    /* Disable the distributor */
>> +    GICD[GICD_CTLR] = 0;
>> +
>> +    type = GICD[GICD_TYPER];
>> +    gic.lines = 32 * ((type & GICD_TYPE_LINES) + 1);
>> +    gic.cpus = 1 + ((type & GICD_TYPE_CPUS) >> 5);
>> +    printk("GIC: %d lines, %d cpu%s%s (IID %8.8x).\n",
>> +           gic.lines, gic.cpus, (gic.cpus == 1) ? "" : "s",
>> +           (type & GICD_TYPE_SEC) ? ", secure" : "",
>> +           GICD[GICD_IIDR]);
>> +
>> +    /* Default all global IRQs to level, active low */
>> +    for ( i = 32; i < gic.lines; i += 16 )
>> +        GICD[GICD_ICFGR + i / 16] = 0x0;
>> +
>> +    /* Route all global IRQs to this CPU */
>> +    for ( i = 32; i < gic.lines; i += 4 )
>> +        GICD[GICD_ITARGETSR + i / 4] = cpumask;
>> +
>> +    /* Default priority for global interrupts */
>> +    for ( i = 32; i < gic.lines; i += 4 )
>> +        GICD[GICD_IPRIORITYR + i / 4] =
>> +            GIC_PRI_IRQ<<24 | GIC_PRI_IRQ<<16 | GIC_PRI_IRQ<<8 | GIC_PRI_IRQ;
>> +
>> +    /* Disable all global interrupts */
>> +    for ( i = 32; i < gic.lines; i += 32 )
>> +        GICD[GICD_ICENABLER + i / 32] = (uint32_t)~0ul;
>> +
>> +    /* Turn on the distributor */
>> +    GICD[GICD_CTLR] = GICD_CTL_ENABLE;
>> +}
>> +
>> +static void __cpuinit gic_cpu_init(void)
>> +{
>> +    int i;
>> +
>> +    this_cpu(gic_cpu_id) = GICD[GICD_ITARGETSR] & 0xff;
>> +
>> +    /* The first 32 interrupts (PPI and SGI) are banked per-cpu, so
>> +     * even though they are controlled with GICD registers, they must
>> +     * be set up here with the other per-cpu state. */
>> +    GICD[GICD_ICENABLER] = 0xffff0000; /* Disable all PPI */
>> +    GICD[GICD_ISENABLER] = 0x0000ffff; /* Enable all SGI */
>> +    /* Set SGI priorities */
>> +    for (i = 0; i < 16; i += 4)
>> +        GICD[GICD_IPRIORITYR + i / 4] =
>> +            GIC_PRI_IPI<<24 | GIC_PRI_IPI<<16 | GIC_PRI_IPI<<8 | GIC_PRI_IPI;
>> +    /* Set PPI priorities */
>> +    for (i = 16; i < 32; i += 4)
>> +        GICD[GICD_IPRIORITYR + i / 4] =
>> +            GIC_PRI_IRQ<<24 | GIC_PRI_IRQ<<16 | GIC_PRI_IRQ<<8 | GIC_PRI_IRQ;
>> +
>> +    /* Local settings: interface controller */
>> +    GICC[GICC_PMR] = 0xff;                /* Don't mask by priority */
>> +    GICC[GICC_BPR] = 0;                   /* Finest granularity of priority */
>> +    GICC[GICC_CTLR] = GICC_CTL_ENABLE|GICC_CTL_EOI;    /* Turn on delivery */
>> +}
>> +
>> +static void gic_cpu_disable(void)
>> +{
>> +    GICC[GICC_CTLR] = 0;
>> +}
>> +
>> +static void __cpuinit gic_hyp_init(void)
>> +{
>> +    uint32_t vtr;
>> +
>> +    vtr = GICH[GICH_VTR];
>> +    nr_lrs  = (vtr & GICH_VTR_NRLRGS) + 1;
>> +
>> +    GICH[GICH_MISR] = GICH_MISR_EOI;
>> +    update_cpu_lr_mask();
>> +}
>> +
>> +static void __cpuinit gic_hyp_disable(void)
>> +{
>> +    GICH[GICH_HCR] = 0;
>> +}
>> +
>> +/* Set up the per-CPU parts of the GIC for a secondary CPU */
>> +static int __cpuinit gic_init_secondary_cpu(struct notifier_block *nfb,
>> +                                       unsigned long action, void *hcpu)
>> +{
>> +    if (action == CPU_STARTING)
>> +    {
>> +        spin_lock(&gic.lock);
>> +        gic_cpu_init();
>> +        gic_hyp_init();
>> +        spin_unlock(&gic.lock);
>> +    }
>> +    return NOTIFY_DONE;
>> +}
>> +
>> +static struct notifier_block gic_cpu_nb = {
>> +    .notifier_call = gic_init_secondary_cpu,
>> +    .priority = 100
>> +};
>> +
>> +static void gic_smp_init(void)
>> +{
>> +   register_cpu_notifier(&gic_cpu_nb);
>> +}
>> +
>> +static int gic_hw_type(void)
>> +{
>> +    return gic.hw_version;
>> +}
>> +
>> +static struct dt_irq * gic_maintenance_irq(void)
>> +{
>> +    return &gic.maintenance;
>> +}
>> +
>> +/* Set up the GIC */
>> +
>> +static void gic_send_sgi(const cpumask_t *cpumask, enum gic_sgi sgi)
>> +{
>> +    unsigned int mask = 0;
>> +    cpumask_t online_mask;
>> +
>> +    ASSERT(sgi < 16); /* There are only 16 SGIs */
>> +
>> +    cpumask_and(&online_mask, cpumask, &cpu_online_map);
>> +    mask = gic_cpu_mask(&online_mask);
>> +
>> +    dsb();
>> +
>> +    GICD[GICD_SGIR] = GICD_SGI_TARGET_LIST
>> +        | (mask<<GICD_SGI_TARGET_SHIFT)
>> +        | sgi;
>> +}
>> +
>> +/* Shut down the per-CPU GIC interface */
>> +static void gic_disable_interface(void)
>> +{
>> +    ASSERT(!local_irq_is_enabled());
>> +
>> +    spin_lock(&gic.lock);
>> +    gic_cpu_disable();
>> +    gic_hyp_disable();
>> +    spin_unlock(&gic.lock);
>> +}
>> +
>> +static void gic_update_lr(int lr, unsigned int virtual_irq,
>> +        unsigned int state, unsigned int priority)
>> +{
>> +    int maintenance_int = GICH_LR_MAINTENANCE_IRQ;
>> +
>> +    BUG_ON(lr >= nr_lrs);
>> +    BUG_ON(lr < 0);
>> +    BUG_ON(state & ~(GICH_LR_STATE_MASK<<GICH_LR_STATE_SHIFT));
>> +
>> +    GICH[GICH_LR + lr] = ((state & 0x3) << GICH_LR_STATE_SHIFT) |
>> +        maintenance_int |
>> +        ((priority >> 3) << GICH_LR_PRIORITY_SHIFT) |
>> +        ((virtual_irq & GICH_LR_VIRTUAL_MASK) << GICH_LR_VIRTUAL_SHIFT);
>> +
>> +}
>> +
>> +static int gicv_init(struct domain *d)
>> +{
>> +    int ret;
>> +
>> +    /*
>> +     * Domain 0 gets the hardware address.
>> +     * Guests get the virtual platform layout.
>> +     */
>> +    if ( d->domain_id == 0 )
>> +    {
>> +        d->arch.vgic.dbase = gic.dbase;
>> +        d->arch.vgic.cbase = gic.cbase;
>> +    }
>> +    else
>> +    {
>> +        d->arch.vgic.dbase = GUEST_GICD_BASE;
>> +        d->arch.vgic.cbase = GUEST_GICC_BASE;
>> +    }
>> +
>> +    d->arch.vgic.nr_lines = 0;
>> +
>> +    /*
>> +     * Map the gic virtual cpu interface in the gic cpu interface
>> +     * region of the guest.
>> +     *
>> +     * The second page is always mapped at +4K irrespective of the
>> +     * GIC_64K_STRIDE quirk. The DTB passed to the guest reflects this.
>> +     */
>> +    ret = map_mmio_regions(d, d->arch.vgic.cbase,
>> +                           d->arch.vgic.cbase + PAGE_SIZE - 1,
>> +                           gic.vbase);
>> +    if (ret)
>> +        return ret;
>> +
>> +    if ( !platform_has_quirk(PLATFORM_QUIRK_GIC_64K_STRIDE) )
>> +        ret = map_mmio_regions(d, d->arch.vgic.cbase + PAGE_SIZE,
>> +                               d->arch.vgic.cbase + (2 * PAGE_SIZE) - 1,
>> +                               gic.vbase + PAGE_SIZE);
>> +    else
>> +        ret = map_mmio_regions(d, d->arch.vgic.cbase + PAGE_SIZE,
>> +                               d->arch.vgic.cbase + (2 * PAGE_SIZE) - 1,
>> +                               gic.vbase + 16*PAGE_SIZE);
>> +
>> +    return ret;
>> +
>> +}
>> +
>> +static unsigned long gic_read_eisr(void)
>> +{
>> +    return ((unsigned long)(GICH[GICH_EISR0] | (((uint64_t) GICH[GICH_EISR1]) << 32)));
>> +}
>> +
>> +static unsigned int gic_update_lr_for_mi(int lr)
>> +{
>> +    u64 val;
>> +    uint32_t virq;
>> +
>> +    spin_lock_irq(&gic.lock);
>> +    val = GICH[GICH_LR + lr];
>> +    virq = val & GICH_LR_VIRTUAL_MASK;
>> +    GICH[GICH_LR + lr] = 0;
>> +    spin_unlock_irq(&gic.lock);
>> +    return virq;
>> +}
>> +
>> +static struct gic_hw_operations gic_ops = {
>> +    .gic_type            = gic_hw_type,
>> +    .nr_lines            = gic_nr_lines,
>> +    .nr_lrs              = gic_nr_lrs,
>> +    .get_maintenance_irq = gic_maintenance_irq,
>> +    .state_init        = gic_state_init,
>> +    .save_state          = save_state,
>> +    .restore_state       = restore_state,
>> +    .dump_state          = gic_dump_state,
>> +    .gicv_setup          = gicv_init,
>> +    .enable_irq          = gic_enable_irq,
>> +    .disable_irq         = gic_disable_irq,
>> +    .eoi_irq             = gic_eoi_irq,
>> +    .deactivate_irq      = gic_dir_irq,
>> +    .ack_irq             = gic_ack_irq,
>> +    .set_irq_property    = gic_set_irq_properties,
>> +    .send_sgi            = gic_send_sgi,
>> +    .disable_interface   = gic_disable_interface,
>> +    .update_lr           = gic_update_lr,
>> +    .update_lr_for_mi    = gic_update_lr_for_mi,
>> +    .read_eisr           = gic_read_eisr,
>> +};
>> +
>> +void __init gicv2_init(void)
>> +{
>> +    static const struct dt_device_match gic_ids[] __initconst =
>> +    {
>> +        DT_MATCH_GIC,
>> +        { /* sentinel */ },
>> +    };
>> +    struct dt_device_node *node;
>> +    int res;
>> +
>> +    node = dt_find_interrupt_controller(gic_ids);
>> +    if ( !node )
>> +        panic("Unable to find compatible GIC in the device tree");
>> +
>> +    dt_device_set_used_by(node, DOMID_XEN);
>> +
>> +    gic.hw_version = GIC_VERSION_V2;
>> +
>> +    res = dt_device_get_address(node, 0, &gic.dbase, NULL);
>> +    if ( res || !gic.dbase || (gic.dbase & ~PAGE_MASK) )
>> +        panic("GIC: Cannot find a valid address for the distributor");
>> +
>> +    res = dt_device_get_address(node, 1, &gic.cbase, NULL);
>> +    if ( res || !gic.cbase || (gic.cbase & ~PAGE_MASK) )
>> +        panic("GIC: Cannot find a valid address for the CPU");
>> +
>> +    res = dt_device_get_address(node, 2, &gic.hbase, NULL);
>> +    if ( res || !gic.hbase || (gic.hbase & ~PAGE_MASK) )
>> +        panic("GIC: Cannot find a valid address for the hypervisor");
>> +
>> +    res = dt_device_get_address(node, 3, &gic.vbase, NULL);
>> +    if ( res || !gic.vbase || (gic.vbase & ~PAGE_MASK) )
>> +        panic("GIC: Cannot find a valid address for the virtual CPU");
>> +
>> +    res = dt_device_get_irq(node, 0, &gic.maintenance);
>> +    if ( res )
>> +        panic("GIC: Cannot find the maintenance IRQ");
>> +
>> +    /* Set the GIC as the primary interrupt controller */
>> +    dt_interrupt_controller = node;
>> +
>> +    /* TODO: Add check on distributor, cpu size */
>> +
>> +    printk("GIC initialization:\n"
>> +              "        gic_dist_addr=%"PRIpaddr"\n"
>> +              "        gic_cpu_addr=%"PRIpaddr"\n"
>> +              "        gic_hyp_addr=%"PRIpaddr"\n"
>> +              "        gic_vcpu_addr=%"PRIpaddr"\n"
>> +              "        gic_maintenance_irq=%u\n",
>> +              gic.dbase, gic.cbase, gic.hbase, gic.vbase,
>> +              gic.maintenance.irq);
>> +
>> +    if ( (gic.dbase & ~PAGE_MASK) || (gic.cbase & ~PAGE_MASK) ||
>> +         (gic.hbase & ~PAGE_MASK) || (gic.vbase & ~PAGE_MASK) )
>> +        panic("GIC interfaces not page aligned");
>> +
>> +    set_fixmap(FIXMAP_GICD, gic.dbase >> PAGE_SHIFT, DEV_SHARED);
>> +    BUILD_BUG_ON(FIXMAP_ADDR(FIXMAP_GICC1) !=
>> +                 FIXMAP_ADDR(FIXMAP_GICC2)-PAGE_SIZE);
>> +    set_fixmap(FIXMAP_GICC1, gic.cbase >> PAGE_SHIFT, DEV_SHARED);
>> +    if ( platform_has_quirk(PLATFORM_QUIRK_GIC_64K_STRIDE) )
>> +        set_fixmap(FIXMAP_GICC2, (gic.cbase >> PAGE_SHIFT) + 0x10, DEV_SHARED);
>> +    else
>> +        set_fixmap(FIXMAP_GICC2, (gic.cbase >> PAGE_SHIFT) + 0x1, DEV_SHARED);
>> +    set_fixmap(FIXMAP_GICH, gic.hbase >> PAGE_SHIFT, DEV_SHARED);
>> +
>> +    /* Global settings: interrupt distributor */
>> +    spin_lock_init(&gic.lock);
>> +    spin_lock(&gic.lock);
>> +
>> +    gic_smp_init();
>> +    gic_dist_init();
>> +    gic_cpu_init();
>> +    gic_hyp_init();
>> +
>> +    register_gic_ops(&gic_ops);
>> +    spin_unlock(&gic.lock);
>> +}
>> +
>> +/*
>> + * Local variables:
>> + * mode: C
>> + * c-file-style: "BSD"
>> + * c-basic-offset: 4
>> + * indent-tabs-mode: nil
>> + * End:
>> + */
>> diff --git a/xen/arch/arm/gic.c b/xen/arch/arm/gic.c
>> index bb718f6..e0859ae 100644
>> --- a/xen/arch/arm/gic.c
>> +++ b/xen/arch/arm/gic.c
>> @@ -33,68 +33,46 @@
>>  #include <asm/domain.h>
>>  #include <asm/platform.h>
>>
>> -#include <asm/gic_v2_defs.h>
>>  #include <asm/gic.h>
>>
>> -/* Access to the GIC Distributor registers through the fixmap */
>> -#define GICD ((volatile uint32_t *) FIXMAP_ADDR(FIXMAP_GICD))
>> -#define GICC ((volatile uint32_t *) FIXMAP_ADDR(FIXMAP_GICC1))
>> -#define GICH ((volatile uint32_t *) FIXMAP_ADDR(FIXMAP_GICH))
>>  static void gic_restore_pending_irqs(struct vcpu *v);
>>
>> -struct gic_state_data {
>> -    uint32_t gic_hcr;
>> -    uint32_t gic_vmcr;
>> -    uint32_t gic_apr;
>> -    uint32_t gic_lr[64];
>> -};
>> -
>> -/* Global state */
>> -static struct {
>> -    paddr_t dbase;       /* Address of distributor registers */
>> -    paddr_t cbase;       /* Address of CPU interface registers */
>> -    paddr_t hbase;       /* Address of virtual interface registers */
>> -    paddr_t vbase;       /* Address of virtual cpu interface registers */
>> -    unsigned int lines;  /* Number of interrupts (SPIs + PPIs + SGIs) */
>> -    struct dt_irq maintenance; /* IRQ maintenance */
>> -    unsigned int cpus;
>> -    spinlock_t lock;
>> -} gic;
>> -
>>  static irq_desc_t irq_desc[NR_IRQS];
>>  static DEFINE_PER_CPU(irq_desc_t[NR_LOCAL_IRQS], local_irq_desc);
>>  static DEFINE_PER_CPU(uint64_t, lr_mask);
>>
>> -static unsigned nr_lrs;
>> -
>> -/* The GIC mapping of CPU interfaces does not necessarily match the
>> - * logical CPU numbering. Let's use mapping as returned by the GIC
>> - * itself
>> - */
>> -static DEFINE_PER_CPU(u8, gic_cpu_id);
>> +static struct gic_hw_operations *gic_hw_ops;
>>
>> -/* Maximum cpu interface per GIC */
>> -#define NR_GIC_CPU_IF 8
>> +spinlock_t gic_lock;
>>
>> -static unsigned int gic_cpu_mask(const cpumask_t *cpumask)
>> +void register_gic_ops(struct gic_hw_operations *ops)
>>  {
>> -    unsigned int cpu;
>> -    unsigned int mask = 0;
>> -    cpumask_t possible_mask;
>> +    gic_hw_ops = ops;
>> +}
>>
>> -    cpumask_and(&possible_mask, cpumask, &cpu_possible_map);
>> -    for_each_cpu(cpu, &possible_mask)
>> -    {
>> -        ASSERT(cpu < NR_GIC_CPU_IF);
>> -        mask |= per_cpu(gic_cpu_id, cpu);
>> -    }
>> +void update_cpu_lr_mask(void)
>> +{
>> +    this_cpu(lr_mask) = 0ULL;
>> +}
>>
>> -    return mask;
>> +int gic_hw_version(void)
>> +{
>> +   return gic_hw_ops->gic_type();
>>  }
>>
>>  unsigned int gic_number_lines(void)
>>  {
>> -    return gic.lines;
>> +    return gic_hw_ops->nr_lines();
>> +}
>> +
>> +unsigned long gic_data_rdist_rd_base(void)
>> +{
>> +   return gic_hw_ops->read_cpu_rbase();
>> +}
>> +
>> +unsigned long gic_data_rdist_sgi_base(void)
>> +{
>> +   return gic_hw_ops->read_cpu_sgi_rbase();
>>  }
>>
>>  irq_desc_t *__irq_to_desc(int irq)
>> @@ -105,43 +83,21 @@ irq_desc_t *__irq_to_desc(int irq)
>>
>>  void gic_save_state(struct vcpu *v)
>>  {
>> -    int i;
>> -    struct gic_state_data *d;
>> -    d = (struct gic_state_data *)v->arch.gic_state;
>> -
>>      ASSERT(!local_irq_is_enabled());
>>
>> -    /* No need for spinlocks here because interrupts are disabled around
>> -     * this call and it only accesses struct vcpu fields that cannot be
>> -     * accessed simultaneously by another pCPU.
>> -     */
>> -    for ( i=0; i<nr_lrs; i++)
>> -        d->gic_lr[i] = GICH[GICH_LR + i];
>>      v->arch.lr_mask = this_cpu(lr_mask);
>> -    d->gic_apr = GICH[GICH_APR];
>> -    d->gic_vmcr = GICH[GICH_VMCR];
>> -    /* Disable until next VCPU scheduled */
>> -    GICH[GICH_HCR] = 0;
>> +    gic_hw_ops->save_state(v);
>>      isb();
>>  }
>>
>>  void gic_restore_state(struct vcpu *v)
>>  {
>> -    int i;
>> -    struct gic_state_data *d;
>> -    d = (struct gic_state_data *)v->arch.gic_state;
>> -
>>      if ( is_idle_vcpu(v) )
>>          return;
>>
>>      this_cpu(lr_mask) = v->arch.lr_mask;
>> -    for ( i=0; i<nr_lrs; i++)
>> -        GICH[GICH_LR + i] = d->gic_lr[i];
>> -    GICH[GICH_APR] = d->gic_apr;
>> -    GICH[GICH_VMCR] = d->gic_vmcr;
>> -    GICH[GICH_HCR] = GICH_HCR_EN;
>> +    gic_hw_ops->restore_state(v);
>>      isb();
>> -
>>      gic_restore_pending_irqs(v);
>>  }
>>
>> @@ -151,12 +107,12 @@ static void gic_irq_enable(struct irq_desc *desc)
>>      unsigned long flags;
>>
>>      spin_lock_irqsave(&desc->lock, flags);
>> -    spin_lock(&gic.lock);
>> +    spin_lock(&gic_lock);
>>      desc->status &= ~IRQ_DISABLED;
>>      dsb();
>>      /* Enable routing */
>> -    GICD[GICD_ISENABLER + irq / 32] = (1u << (irq % 32));
>> -    spin_unlock(&gic.lock);
>> +    gic_hw_ops->enable_irq(irq);
>> +    spin_unlock(&gic_lock);
>>      spin_unlock_irqrestore(&desc->lock, flags);
>>  }
>>
>> @@ -166,11 +122,11 @@ static void gic_irq_disable(struct irq_desc *desc)
>>      unsigned long flags;
>>
>>      spin_lock_irqsave(&desc->lock, flags);
>> -    spin_lock(&gic.lock);
>> +    spin_lock(&gic_lock);
>>      /* Disable routing */
>> -    GICD[GICD_ICENABLER + irq / 32] = (1u << (irq % 32));
>> +    gic_hw_ops->disable_irq(irq);
>>      desc->status |= IRQ_DISABLED;
>> -    spin_unlock(&gic.lock);
>> +    spin_unlock(&gic_lock);
>>      spin_unlock_irqrestore(&desc->lock, flags);
>>  }
>>
>> @@ -194,17 +150,16 @@ static void gic_host_irq_end(struct irq_desc *desc)
>>  {
>>      int irq = desc->irq;
>>      /* Lower the priority */
>> -    GICC[GICC_EOIR] = irq;
>> -    /* Deactivate */
>> -    GICC[GICC_DIR] = irq;
>> +    gic_hw_ops->eoi_irq(irq);
>> +    gic_hw_ops->deactivate_irq(irq);
>>  }
>>
>>  static void gic_guest_irq_end(struct irq_desc *desc)
>>  {
>>      int irq = desc->irq;
>>      /* Lower the priority of the IRQ */
>> -    GICC[GICC_EOIR] = irq;
>>      /* Deactivation happens in maintenance interrupt / via GICV */
>> +    gic_hw_ops->eoi_irq(irq);
>>  }
>>
>>  static void gic_irq_set_affinity(struct irq_desc *desc, const cpumask_t *mask)
>> @@ -243,27 +198,7 @@ static void gic_set_irq_properties(unsigned int irq, bool_t level,
>>                                     const cpumask_t *cpu_mask,
>>                                     unsigned int priority)
>>  {
>> -    volatile unsigned char *bytereg;
>> -    uint32_t cfg, edgebit;
>> -    unsigned int mask = gic_cpu_mask(cpu_mask);
>> -
>> -    /* Set edge / level */
>> -    cfg = GICD[GICD_ICFGR + irq / 16];
>> -    edgebit = 2u << (2 * (irq % 16));
>> -    if ( level )
>> -        cfg &= ~edgebit;
>> -    else
>> -        cfg |= edgebit;
>> -    GICD[GICD_ICFGR + irq / 16] = cfg;
>> -
>> -    /* Set target CPU mask (RAZ/WI on uniprocessor) */
>> -    bytereg = (unsigned char *) (GICD + GICD_ITARGETSR);
>> -    bytereg[irq] = mask;
>> -
>> -    /* Set priority */
>> -    bytereg = (unsigned char *) (GICD + GICD_IPRIORITYR);
>> -    bytereg[irq] = priority;
>> -
>> +    return gic_hw_ops->set_irq_property(irq, level, cpu_mask, priority);
>>  }
>>
>>  /* Program the GIC to route an interrupt */
>> @@ -273,8 +208,8 @@ static int gic_route_irq(unsigned int irq, bool_t level,
>>      struct irq_desc *desc = irq_to_desc(irq);
>>      unsigned long flags;
>>
>> -    ASSERT(priority <= 0xff);     /* Only 8 bits of priority */
>> -    ASSERT(irq < gic.lines);      /* Can't route interrupts that don't exist */
>> +    ASSERT(priority <= 0xff);  /* Only 8 bits of priority */
>> +    ASSERT(irq < gic_number_lines());
>>
>>      if ( desc->action != NULL )
>>          return -EBUSY;
>> @@ -286,9 +221,9 @@ static int gic_route_irq(unsigned int irq, bool_t level,
>>
>>      desc->handler = &gic_host_irq_type;
>>
>> -    spin_lock(&gic.lock);
>> +    spin_lock(&gic_lock);
>>      gic_set_irq_properties(irq, level, cpu_mask, priority);
>> -    spin_unlock(&gic.lock);
>> +    spin_unlock(&gic_lock);
>>
>>      spin_unlock_irqrestore(&desc->lock, flags);
>>      return 0;
>> @@ -305,119 +240,6 @@ void gic_route_dt_irq(const struct dt_irq *irq, const cpumask_t *cpu_mask,
>>      gic_route_irq(irq->irq, level, cpu_mask, priority);
>>  }
>>
>> -static void __init gic_dist_init(void)
>> -{
>> -    uint32_t type;
>> -    uint32_t cpumask;
>> -    int i;
>> -
>> -    cpumask = GICD[GICD_ITARGETSR] & 0xff;
>> -    cpumask |= cpumask << 8;
>> -    cpumask |= cpumask << 16;
>> -
>> -    /* Disable the distributor */
>> -    GICD[GICD_CTLR] = 0;
>> -
>> -    type = GICD[GICD_TYPER];
>> -    gic.lines = 32 * ((type & GICD_TYPE_LINES) + 1);
>> -    gic.cpus = 1 + ((type & GICD_TYPE_CPUS) >> 5);
>> -    printk("GIC: %d lines, %d cpu%s%s (IID %8.8x).\n",
>> -           gic.lines, gic.cpus, (gic.cpus == 1) ? "" : "s",
>> -           (type & GICD_TYPE_SEC) ? ", secure" : "",
>> -           GICD[GICD_IIDR]);
>> -
>> -    /* Default all global IRQs to level, active low */
>> -    for ( i = 32; i < gic.lines; i += 16 )
>> -        GICD[GICD_ICFGR + i / 16] = 0x0;
>> -
>> -    /* Route all global IRQs to this CPU */
>> -    for ( i = 32; i < gic.lines; i += 4 )
>> -        GICD[GICD_ITARGETSR + i / 4] = cpumask;
>> -
>> -    /* Default priority for global interrupts */
>> -    for ( i = 32; i < gic.lines; i += 4 )
>> -        GICD[GICD_IPRIORITYR + i / 4] =
>> -            GIC_PRI_IRQ<<24 | GIC_PRI_IRQ<<16 | GIC_PRI_IRQ<<8 | GIC_PRI_IRQ;
>> -
>> -    /* Disable all global interrupts */
>> -    for ( i = 32; i < gic.lines; i += 32 )
>> -        GICD[GICD_ICENABLER + i / 32] = (uint32_t)~0ul;
>> -
>> -    /* Turn on the distributor */
>> -    GICD[GICD_CTLR] = GICD_CTL_ENABLE;
>> -}
>> -
>> -static void __cpuinit gic_cpu_init(void)
>> -{
>> -    int i;
>> -
>> -    this_cpu(gic_cpu_id) = GICD[GICD_ITARGETSR] & 0xff;
>> -
>> -    /* The first 32 interrupts (PPI and SGI) are banked per-cpu, so
>> -     * even though they are controlled with GICD registers, they must
>> -     * be set up here with the other per-cpu state. */
>> -    GICD[GICD_ICENABLER] = 0xffff0000; /* Disable all PPI */
>> -    GICD[GICD_ISENABLER] = 0x0000ffff; /* Enable all SGI */
>> -    /* Set SGI priorities */
>> -    for (i = 0; i < 16; i += 4)
>> -        GICD[GICD_IPRIORITYR + i / 4] =
>> -            GIC_PRI_IPI<<24 | GIC_PRI_IPI<<16 | GIC_PRI_IPI<<8 | GIC_PRI_IPI;
>> -    /* Set PPI priorities */
>> -    for (i = 16; i < 32; i += 4)
>> -        GICD[GICD_IPRIORITYR + i / 4] =
>> -            GIC_PRI_IRQ<<24 | GIC_PRI_IRQ<<16 | GIC_PRI_IRQ<<8 | GIC_PRI_IRQ;
>> -
>> -    /* Local settings: interface controller */
>> -    GICC[GICC_PMR] = 0xff;                /* Don't mask by priority */
>> -    GICC[GICC_BPR] = 0;                   /* Finest granularity of priority */
>> -    GICC[GICC_CTLR] = GICC_CTL_ENABLE|GICC_CTL_EOI;    /* Turn on delivery */
>> -}
>> -
>> -static void gic_cpu_disable(void)
>> -{
>> -    GICC[GICC_CTLR] = 0;
>> -}
>> -
>> -static void __cpuinit gic_hyp_init(void)
>> -{
>> -    uint32_t vtr;
>> -
>> -    vtr = GICH[GICH_VTR];
>> -    nr_lrs  = (vtr & GICH_VTR_NRLRGS) + 1;
>> -
>> -    GICH[GICH_MISR] = GICH_MISR_EOI;
>> -    this_cpu(lr_mask) = 0ULL;
>> -}
>> -
>> -static void __cpuinit gic_hyp_disable(void)
>> -{
>> -    GICH[GICH_HCR] = 0;
>> -}
>> -
>> -/* Set up the per-CPU parts of the GIC for a secondary CPU */
>> -static int  __cpuinit gic_init_secondary_cpu(struct notifier_block *nfb,
>> -                                        unsigned long action, void *hcpu)
>> -{
>> -    if (action == CPU_STARTING)
>> -    {
>> -        spin_lock(&gic.lock);
>> -        gic_cpu_init();
>> -        gic_hyp_init();
>> -        spin_unlock(&gic.lock);
>> -    }
>> -    return NOTIFY_DONE;
>> -}
>> -
>> -static struct notifier_block gic_cpu_nb = {
>> -    .notifier_call = gic_init_secondary_cpu,
>> -    .priority = 100
>> -};
>> -
>> -static void gic_smp_init(void)
>> -{
>> -   register_cpu_notifier(&gic_cpu_nb);
>> -}
>> -
>>  int gic_irq_xlate(const u32 *intspec, unsigned int intsize,
>>                    unsigned int *out_hwirq,
>>                    unsigned int *out_type)
>> @@ -438,124 +260,25 @@ int gic_irq_xlate(const u32 *intspec, unsigned int intsize,
>>      return 0;
>>  }
>>
>> -/* Set up the GIC */
>> -void __init gic_init(void)
>> -{
>> -    static const struct dt_device_match gic_ids[] __initconst =
>> -    {
>> -        DT_MATCH_GIC,
>> -        { /* sentinel */ },
>> -    };
>> -    struct dt_device_node *node;
>> -    int res;
>> -
>> -    node = dt_find_interrupt_controller(gic_ids);
>> -    if ( !node )
>> -        panic("Unable to find compatible GIC in the device tree");
>> -
>> -    dt_device_set_used_by(node, DOMID_XEN);
>> -
>> -    res = dt_device_get_address(node, 0, &gic.dbase, NULL);
>> -    if ( res || !gic.dbase || (gic.dbase & ~PAGE_MASK) )
>> -        panic("GIC: Cannot find a valid address for the distributor");
>> -
>> -    res = dt_device_get_address(node, 1, &gic.cbase, NULL);
>> -    if ( res || !gic.cbase || (gic.cbase & ~PAGE_MASK) )
>> -        panic("GIC: Cannot find a valid address for the CPU");
>> -
>> -    res = dt_device_get_address(node, 2, &gic.hbase, NULL);
>> -    if ( res || !gic.hbase || (gic.hbase & ~PAGE_MASK) )
>> -        panic("GIC: Cannot find a valid address for the hypervisor");
>> -
>> -    res = dt_device_get_address(node, 3, &gic.vbase, NULL);
>> -    if ( res || !gic.vbase || (gic.vbase & ~PAGE_MASK) )
>> -        panic("GIC: Cannot find a valid address for the virtual CPU");
>> -
>> -    res = dt_device_get_irq(node, 0, &gic.maintenance);
>> -    if ( res )
>> -        panic("GIC: Cannot find the maintenance IRQ");
>> -
>> -    /* Set the GIC as the primary interrupt controller */
>> -    dt_interrupt_controller = node;
>> -
>> -    /* TODO: Add check on distributor, cpu size */
>> -
>> -    printk("GIC initialization:\n"
>> -              "        gic_dist_addr=%"PRIpaddr"\n"
>> -              "        gic_cpu_addr=%"PRIpaddr"\n"
>> -              "        gic_hyp_addr=%"PRIpaddr"\n"
>> -              "        gic_vcpu_addr=%"PRIpaddr"\n"
>> -              "        gic_maintenance_irq=%u\n",
>> -              gic.dbase, gic.cbase, gic.hbase, gic.vbase,
>> -              gic.maintenance.irq);
>> -
>> -    if ( (gic.dbase & ~PAGE_MASK) || (gic.cbase & ~PAGE_MASK) ||
>> -         (gic.hbase & ~PAGE_MASK) || (gic.vbase & ~PAGE_MASK) )
>> -        panic("GIC interfaces not page aligned");
>> -
>> -    set_fixmap(FIXMAP_GICD, gic.dbase >> PAGE_SHIFT, DEV_SHARED);
>> -    BUILD_BUG_ON(FIXMAP_ADDR(FIXMAP_GICC1) !=
>> -                 FIXMAP_ADDR(FIXMAP_GICC2)-PAGE_SIZE);
>> -    set_fixmap(FIXMAP_GICC1, gic.cbase >> PAGE_SHIFT, DEV_SHARED);
>> -    if ( platform_has_quirk(PLATFORM_QUIRK_GIC_64K_STRIDE) )
>> -        set_fixmap(FIXMAP_GICC2, (gic.cbase >> PAGE_SHIFT) + 0x10, DEV_SHARED);
>> -    else
>> -        set_fixmap(FIXMAP_GICC2, (gic.cbase >> PAGE_SHIFT) + 0x1, DEV_SHARED);
>> -    set_fixmap(FIXMAP_GICH, gic.hbase >> PAGE_SHIFT, DEV_SHARED);
>> -
>> -    /* Global settings: interrupt distributor */
>> -    spin_lock_init(&gic.lock);
>> -    spin_lock(&gic.lock);
>> -
>> -    gic_smp_init();
>> -    gic_dist_init();
>> -    gic_cpu_init();
>> -    gic_hyp_init();
>> -
>> -    spin_unlock(&gic.lock);
>> -}
>> -
>>  void send_SGI_mask(const cpumask_t *cpumask, enum gic_sgi sgi)
>>  {
>> -    unsigned int mask = 0;
>> -    cpumask_t online_mask;
>> -
>> -    ASSERT(sgi < 16); /* There are only 16 SGIs */
>> -
>> -    cpumask_and(&online_mask, cpumask, &cpu_online_map);
>> -    mask = gic_cpu_mask(&online_mask);
>> -
>> -    dsb();
>> -
>> -    GICD[GICD_SGIR] = GICD_SGI_TARGET_LIST
>> -        | (mask<<GICD_SGI_TARGET_SHIFT)
>> -        | sgi;
>> +    gic_hw_ops->send_sgi(cpumask, sgi);
>>  }
>>
>> -void send_SGI_one(unsigned int cpu, enum gic_sgi sgi)
>> +static void send_SGI_one(unsigned int cpu, enum gic_sgi sgi)
>>  {
>> -    ASSERT(cpu < NR_GIC_CPU_IF);  /* Targets bitmap only supports 8 CPUs */
>>      send_SGI_mask(cpumask_of(cpu), sgi);
>>  }
>>
>> -void send_SGI_self(enum gic_sgi sgi)
>> -{
>> -    ASSERT(sgi < 16); /* There are only 16 SGIs */
>> -
>> -    dsb();
>> -
>> -    GICD[GICD_SGIR] = GICD_SGI_TARGET_SELF
>> -        | sgi;
>> -}
>> -
>>  void send_SGI_allbutself(enum gic_sgi sgi)
>>  {
>> -   ASSERT(sgi < 16); /* There are only 16 SGIs */
>> +    cpumask_t all_others_mask;
>> +    ASSERT(sgi < 16); /* There are only 16 SGIs */
>>
>> -   dsb();
>> +    cpumask_andnot(&all_others_mask, &cpu_possible_map, cpumask_of(smp_processor_id()));
>>
>> -   GICD[GICD_SGIR] = GICD_SGI_TARGET_OTHERS
>> -       | sgi;
>> +    dsb();
>> +    send_SGI_mask(&all_others_mask, sgi);
>>  }
>>
>>  void smp_send_state_dump(unsigned int cpu)
>> @@ -568,16 +291,15 @@ void gic_disable_cpu(void)
>>  {
>>      ASSERT(!local_irq_is_enabled());
>>
>> -    spin_lock(&gic.lock);
>> -    gic_cpu_disable();
>> -    gic_hyp_disable();
>> -    spin_unlock(&gic.lock);
>> +    spin_lock(&gic_lock);
>> +    gic_hw_ops->disable_interface();
>> +    spin_unlock(&gic_lock);
>>  }
>>
>>  void gic_route_ppis(void)
>>  {
>>      /* GIC maintenance */
>> -    gic_route_dt_irq(&gic.maintenance, cpumask_of(smp_processor_id()),
>> +    gic_route_dt_irq(gic_hw_ops->get_maintenance_irq(), cpumask_of(smp_processor_id()),
>>                       GIC_PRI_IRQ);
>>      /* Route timer interrupt */
>>      route_timer_interrupt();
>> @@ -652,20 +374,10 @@ int __init setup_dt_irq(const struct dt_irq *irq, struct irqaction *new)
>>      return rc;
>>  }
>>
>> -static inline void gic_set_lr(int lr, struct pending_irq *p,
>> +static void gic_set_lr(int lr, struct pending_irq *p,
>>          unsigned int state)
>>  {
>> -    int maintenance_int = GICH_LR_MAINTENANCE_IRQ;
>> -
>> -    BUG_ON(lr >= nr_lrs);
>> -    BUG_ON(lr < 0);
>> -    BUG_ON(state & ~(GICH_LR_STATE_MASK<<GICH_LR_STATE_SHIFT));
>> -
>> -    GICH[GICH_LR + lr] = state |
>> -        maintenance_int |
>> -        ((p->priority >> 3) << GICH_LR_PRIORITY_SHIFT) |
>> -        ((p->irq & GICH_LR_VIRTUAL_MASK) << GICH_LR_VIRTUAL_SHIFT);
>> -
>> +    gic_hw_ops->update_lr(lr, p->irq, state, p->priority);
>>      set_bit(GIC_IRQ_GUEST_VISIBLE, &p->status);
>>      clear_bit(GIC_IRQ_GUEST_PENDING, &p->status);
>>  }
>> @@ -693,10 +405,10 @@ void gic_remove_from_queues(struct vcpu *v, unsigned int virtual_irq)
>>      struct pending_irq *p = irq_to_pending(v, virtual_irq);
>>      unsigned long flags;
>>
>> -    spin_lock_irqsave(&gic.lock, flags);
>> +    spin_lock_irqsave(&gic_lock, flags);
>>      if ( !list_empty(&p->lr_queue) )
>>          list_del_init(&p->lr_queue);
>> -    spin_unlock_irqrestore(&gic.lock, flags);
>> +    spin_unlock_irqrestore(&gic_lock, flags);
>>  }
>>
>>  void gic_set_guest_irq(struct vcpu *v, unsigned int virtual_irq,
>> @@ -704,9 +416,9 @@ void gic_set_guest_irq(struct vcpu *v, unsigned int virtual_irq,
>>  {
>>      int i;
>>      unsigned long flags;
>> +    unsigned int nr_lrs = gic_hw_ops->nr_lrs();
>>
>> -    spin_lock_irqsave(&gic.lock, flags);
>> -
>> +    spin_lock_irqsave(&gic_lock, flags);
>>      if ( v == current && list_empty(&v->arch.vgic.lr_pending) )
>>      {
>>          i = find_first_zero_bit(&this_cpu(lr_mask), nr_lrs);
>> @@ -720,7 +432,7 @@ void gic_set_guest_irq(struct vcpu *v, unsigned int virtual_irq,
>>      gic_add_to_lr_pending(v, irq_to_pending(v, virtual_irq));
>>
>>  out:
>> -    spin_unlock_irqrestore(&gic.lock, flags);
>> +    spin_unlock_irqrestore(&gic_lock, flags);
>>      return;
>>  }
>>
>> @@ -729,17 +441,18 @@ static void gic_restore_pending_irqs(struct vcpu *v)
>>      int i;
>>      struct pending_irq *p, *t;
>>      unsigned long flags;
>> +    unsigned int nr_lrs = gic_hw_ops->nr_lrs();
>>
>>      list_for_each_entry_safe ( p, t, &v->arch.vgic.lr_pending, lr_queue )
>>      {
>>          i = find_first_zero_bit(&this_cpu(lr_mask), nr_lrs);
>>          if ( i >= nr_lrs ) return;
>>
>> -        spin_lock_irqsave(&gic.lock, flags);
>> +        spin_lock_irqsave(&gic_lock, flags);
>>          gic_set_lr(i, p, GICH_LR_PENDING);
>>          list_del_init(&p->lr_queue);
>>          set_bit(i, &this_cpu(lr_mask));
>> -        spin_unlock_irqrestore(&gic.lock, flags);
>> +        spin_unlock_irqrestore(&gic_lock, flags);
>>      }
>>
>>  }
>> @@ -749,11 +462,11 @@ void gic_clear_pending_irqs(struct vcpu *v)
>>      struct pending_irq *p, *t;
>>      unsigned long flags;
>>
>> -    spin_lock_irqsave(&gic.lock, flags);
>> +    spin_lock_irqsave(&gic_lock, flags);
>>      v->arch.lr_mask = 0;
>>      list_for_each_entry_safe ( p, t, &v->arch.vgic.lr_pending, lr_queue )
>>          list_del_init(&p->lr_queue);
>> -    spin_unlock_irqrestore(&gic.lock, flags);
>> +    spin_unlock_irqrestore(&gic_lock, flags);
>>  }
>>
>>  static void gic_inject_irq_start(void)
>> @@ -809,7 +522,7 @@ int gic_route_irq_to_guest(struct domain *d, const struct dt_irq *irq,
>>      action->free_on_release = 1;
>>
>>      spin_lock_irqsave(&desc->lock, flags);
>> -    spin_lock(&gic.lock);
>> +    spin_lock(&gic_lock);
>>
>>      desc->handler = &gic_guest_irq_type;
>>      desc->status |= IRQ_GUEST;
>> @@ -830,15 +543,15 @@ int gic_route_irq_to_guest(struct domain *d, const struct dt_irq *irq,
>>      p->desc = desc;
>>
>>  out:
>> -    spin_unlock(&gic.lock);
>> +    spin_unlock(&gic_lock);
>>      spin_unlock_irqrestore(&desc->lock, flags);
>>      return retval;
>>  }
>>
>> -static void do_sgi(struct cpu_user_regs *regs, int othercpu, enum gic_sgi sgi)
>> +static void do_sgi(struct cpu_user_regs *regs, enum gic_sgi sgi)
>>  {
>>      /* Lower the priority */
>> -    GICC[GICC_EOIR] = sgi;
>> +    gic_hw_ops->eoi_irq(sgi);
>>
>>      switch (sgi)
>>      {
>> @@ -857,20 +570,17 @@ static void do_sgi(struct cpu_user_regs *regs, int othercpu, enum gic_sgi sgi)
>>      }
>>
>>      /* Deactivate */
>> -    GICC[GICC_DIR] = sgi;
>> +    gic_hw_ops->deactivate_irq(sgi);
>>  }
>>
>>  /* Accept an interrupt from the GIC and dispatch its handler */
>>  void gic_interrupt(struct cpu_user_regs *regs, int is_fiq)
>>  {
>> -    uint32_t intack;
>>      unsigned int irq;
>>
>>
>>      do  {
>> -        intack = GICC[GICC_IAR];
>> -        irq = intack & GICC_IA_IRQ;
>> -
>> +        irq = gic_hw_ops->ack_irq();
>>          if ( likely(irq >= 16 && irq < 1021) )
>>          {
>>              local_irq_enable();
>> @@ -879,8 +589,7 @@ void gic_interrupt(struct cpu_user_regs *regs, int is_fiq)
>>          }
>>          else if (unlikely(irq < 16))
>>          {
>> -            unsigned int cpu = (intack & GICC_IA_CPU_MASK) >> GICC_IA_CPU_SHIFT;
>> -            do_sgi(regs, cpu, irq);
>> +            do_sgi(regs, irq);
>>          }
>>          else
>>          {
>> @@ -892,72 +601,31 @@ void gic_interrupt(struct cpu_user_regs *regs, int is_fiq)
>>
>>  int vcpu_gic_init(struct vcpu *v)
>>  {
>> -     v->arch.gic_state = xzalloc(struct gic_state_data);
>> -     if(!v->arch.gic_state)
>> -        return -ENOMEM;
>> -     return 0;
>> +     return gic_hw_ops->state_init(v);
>>  }
>>
>>  int gicv_setup(struct domain *d)
>>  {
>>      int ret;
>>
>> -    /*
>> -     * Domain 0 gets the hardware address.
>> -     * Guests get the virtual platform layout.
>> -     */
>> -    if ( d->domain_id == 0 )
>> -    {
>> -        d->arch.vgic.dbase = gic.dbase;
>> -        d->arch.vgic.cbase = gic.cbase;
>> -    }
>> -    else
>> -    {
>> -        d->arch.vgic.dbase = GUEST_GICD_BASE;
>> -        d->arch.vgic.cbase = GUEST_GICC_BASE;
>> -    }
>> -
>> -    d->arch.vgic.nr_lines = 0;
>> -
>> -    /*
>> -     * Map the gic virtual cpu interface in the gic cpu interface
>> -     * region of the guest.
>> -     *
>> -     * The second page is always mapped at +4K irrespective of the
>> -     * GIC_64K_STRIDE quirk. The DTB passed to the guest reflects this.
>> -     */
>> -    ret = map_mmio_regions(d, d->arch.vgic.cbase,
>> -                           d->arch.vgic.cbase + PAGE_SIZE - 1,
>> -                           gic.vbase);
>> -    if (ret)
>> -        return ret;
>> -
>> -    if ( !platform_has_quirk(PLATFORM_QUIRK_GIC_64K_STRIDE) )
>> -        ret = map_mmio_regions(d, d->arch.vgic.cbase + PAGE_SIZE,
>> -                               d->arch.vgic.cbase + (2 * PAGE_SIZE) - 1,
>> -                               gic.vbase + PAGE_SIZE);
>> -    else
>> -        ret = map_mmio_regions(d, d->arch.vgic.cbase + PAGE_SIZE,
>> -                               d->arch.vgic.cbase + (2 * PAGE_SIZE) - 1,
>> -                               gic.vbase + 16*PAGE_SIZE);
>> -
>> +    ret = gic_hw_ops->gicv_setup(d);
>>      return ret;
>>
>>  }
>>
>>  static void gic_irq_eoi(void *info)
>> -{
>> +{
>>      int virq = (uintptr_t) info;
>> -    GICC[GICC_DIR] = virq;
>> +    gic_hw_ops->deactivate_irq(virq);
>>  }
>>
>>  static void maintenance_interrupt(int irq, void *dev_id, struct cpu_user_regs *regs)
>>  {
>>      int i = 0, virq, pirq = -1;
>> -    uint32_t lr;
>>      struct vcpu *v = current;
>> -    uint64_t eisr = GICH[GICH_EISR0] | (((uint64_t) GICH[GICH_EISR1]) << 32);
>> +    unsigned long eisr = 0;
>>
>> +    eisr = gic_hw_ops->read_eisr();
>>      while ((i = find_next_bit((const long unsigned int *) &eisr,
>>                                64, i)) < 64) {
>>          struct pending_irq *p, *p2;
>> @@ -967,10 +635,9 @@ static void maintenance_interrupt(int irq, void *dev_id, struct cpu_user_regs *r
>>          cpu = -1;
>>          inflight = 0;
>>
>> -        spin_lock_irq(&gic.lock);
>> -        lr = GICH[GICH_LR + i];
>> -        virq = lr & GICH_LR_VIRTUAL_MASK;
>> -        GICH[GICH_LR + i] = 0;
>> +        spin_lock_irq(&gic_lock);
>> +        virq = gic_hw_ops->update_lr_for_mi(i);
>> +
>>          clear_bit(i, &this_cpu(lr_mask));
>>
>>          p = irq_to_pending(v, virq);
>> @@ -995,7 +662,7 @@ static void maintenance_interrupt(int irq, void *dev_id, struct cpu_user_regs *r
>>              list_del_init(&p2->lr_queue);
>>              set_bit(i, &this_cpu(lr_mask));
>>          }
>> -        spin_unlock_irq(&gic.lock);
>> +        spin_unlock_irq(&gic_lock);
>>
>>          if ( !inflight )
>>          {
>> @@ -1018,22 +685,37 @@ static void maintenance_interrupt(int irq, void *dev_id, struct cpu_user_regs *r
>>      }
>>  }
>>
>> +/* Set up the GIC */
>> +void __init gic_init(void)
>> +{
>> +    static const struct dt_device_match gicv2_ids[] __initconst =
>> +    {
>> +        DT_MATCH_GIC,
>> +        { /* sentinel */ },
>> +    };
>> +    struct dt_device_node *node;
>> +
>> +    spin_lock_init(&gic_lock);
>> +
>> +    spin_lock(&gic_lock);
>> +    node = dt_find_interrupt_controller(gicv2_ids);
>> +    if ( node )
>> +    {
>> +        gicv2_init();
>> +        spin_unlock(&gic_lock);
>> +        return;
>> +    }
>> +    spin_unlock(&gic_lock);
>> +    if ( !node )
>> +        panic("Unable to find compatible GIC in the device tree");
>> +}
>> +
>>  void gic_dump_info(struct vcpu *v)
>>  {
>> -    int i;
>>      struct pending_irq *p;
>> -    struct gic_state_data *d;
>> -    d = (struct gic_state_data *)v->arch.gic_state;
>>
>>      printk("GICH_LRs (vcpu %d) mask=%"PRIx64"\n", v->vcpu_id, v->arch.lr_mask);
>> -    if ( v == current )
>> -    {
>> -        for ( i = 0; i < nr_lrs; i++ )
>> -            printk("   HW_LR[%d]=%x\n", i, GICH[GICH_LR + i]);
>> -    } else {
>> -        for ( i = 0; i < nr_lrs; i++ )
>> -            printk("   VCPU_LR[%d]=%x\n", i, d->gic_lr[i]);
>> -    }
>> +    gic_hw_ops->dump_state(v);
>>
>>      list_for_each_entry ( p, &v->arch.vgic.inflight_irqs, inflight )
>>      {
>> @@ -1049,7 +731,7 @@ void gic_dump_info(struct vcpu *v)
>>
>>  void __cpuinit init_maintenance_interrupt(void)
>>  {
>> -    request_dt_irq(&gic.maintenance, maintenance_interrupt,
>> +    request_dt_irq(gic_hw_ops->get_maintenance_irq(), maintenance_interrupt,
>>                     "irq-maintenance", NULL);
>>  }
>>
>> diff --git a/xen/include/asm-arm/gic.h b/xen/include/asm-arm/gic.h
>> index 18656fd..4244491 100644
>> --- a/xen/include/asm-arm/gic.h
>> +++ b/xen/include/asm-arm/gic.h
>> @@ -39,6 +39,8 @@
>>  #define GIC_PRI_IPI        0x90 /* IPIs must preempt normal interrupts */
>>  #define GIC_PRI_HIGHEST    0x80 /* Higher priorities belong to Secure-World */
>>
>> +#define GICH_LR_PENDING    1
>> +#define GICH_LR_ACTIVE     2
>>
>>  #ifndef __ASSEMBLY__
>>  #include <xen/device_tree.h>
>> @@ -46,12 +48,15 @@
>>  #define DT_MATCH_GIC    DT_MATCH_COMPATIBLE("arm,cortex-a15-gic"), \
>>                          DT_MATCH_COMPATIBLE("arm,cortex-a7-gic")
>>
>> +extern int gic_hw_version(void);
>>  extern int domain_vgic_init(struct domain *d);
>>  extern void domain_vgic_free(struct domain *d);
>>
>>  extern int vcpu_vgic_init(struct vcpu *v);
>>  extern int vcpu_gic_init(struct vcpu *v);
>>
>> +extern void gicv2_init(void);
>> +
>>  extern void vgic_vcpu_inject_irq(struct vcpu *v, unsigned int irq,int virtual);
>>  extern void vgic_clear_pending_irqs(struct vcpu *v);
>>  extern struct pending_irq *irq_to_pending(struct vcpu *v, unsigned int irq);
>> @@ -88,14 +93,47 @@ extern int gicv_setup(struct domain *d);
>>  extern void gic_save_state(struct vcpu *v);
>>  extern void gic_restore_state(struct vcpu *v);
>>
>> +#define GIC_VERSION_V2 0x2
>> +
>>  /* SGI (AKA IPIs) */
>>  enum gic_sgi {
>>      GIC_SGI_EVENT_CHECK = 0,
>>      GIC_SGI_DUMP_STATE  = 1,
>>      GIC_SGI_CALL_FUNCTION = 2,
>>  };
>> +
>> +struct gic_hw_operations {
>> +    int (*gic_type)(void);
>> +    struct dt_irq * (*get_maintenance_irq)(void);
>> +    unsigned int (*nr_lines)(void);
>> +    unsigned int (*nr_lrs)(void);
>> +    int (*state_init)(struct vcpu *);
>> +    void (*save_state)(struct vcpu *);
>> +    void (*restore_state)(struct vcpu *);
>> +    void (*dump_state)(struct vcpu *);
>> +    int (*gicv_setup)(struct domain *);
>> +    void (*enable_irq)(int);
>> +    void (*disable_irq)(int);
>> +    void (*eoi_irq)(int);
>> +    void (*deactivate_irq)(int);
>> +    unsigned int (*ack_irq)(void);
>> +    void (*set_irq_property)(unsigned int irq, bool_t level,
>> +                            const cpumask_t *cpu_mask,
>> +                            unsigned int priority);
>> +    void (*send_sgi)(const cpumask_t *, enum gic_sgi);
>> +    void (*disable_interface)(void);
>> +    void (*update_lr)(int lr, unsigned int virtual_irq,
>> +                          unsigned int state, unsigned int priority);
>> +    unsigned int (*update_lr_for_mi)(int lr);
>> +    unsigned long (*read_eisr)(void);
>> +    unsigned long (*read_cpu_rbase)(void);
>> +    unsigned long (*read_cpu_sgi_rbase)(void);
>> +};
>> +
>> +extern void register_gic_ops(struct gic_hw_operations *);
>> +
>> +extern void update_cpu_lr_mask(void);
>>  extern void send_SGI_mask(const cpumask_t *cpumask, enum gic_sgi sgi);
>> -extern void send_SGI_one(unsigned int cpu, enum gic_sgi sgi);
>>  extern void send_SGI_self(enum gic_sgi sgi);
>>  extern void send_SGI_allbutself(enum gic_sgi sgi);
>>
>> diff --git a/xen/include/asm-arm/gic_v2_defs.h b/xen/include/asm-arm/gic_v2_defs.h
>> index 2366685..5ddd48a 100644
>> --- a/xen/include/asm-arm/gic_v2_defs.h
>> +++ b/xen/include/asm-arm/gic_v2_defs.h
>> @@ -119,8 +119,6 @@
>>  #define GICH_LR_STATE_SHIFT     28
>>  #define GICH_LR_PRIORITY_SHIFT  23
>>  #define GICH_LR_MAINTENANCE_IRQ (1<<19)
>> -#define GICH_LR_PENDING         (1<<28)
>> -#define GICH_LR_ACTIVE          (1<<29)
>>  #define GICH_LR_GRP1            (1<<30)
>>  #define GICH_LR_HW              (1<<31)
>>  #define GICH_LR_CPUID_SHIFT     9
>> --
>> 1.7.9.5
>>

Re: [RFC PATCH v1 06/10] xen/arm: split gic driver into generic and gicv2 driver

[Julien Grall]

Hello Vijay,

On 22/03/14 09:32, Vijay Kilari wrote:
> On Thu, Mar 20, 2014 at 5:25 PM, Stefano Stabellini
> <stefano.stabellini@eu.citrix.com> wrote:
>> On Wed, 19 Mar 2014, vijay.kilari@gmail.com wrote:
>>> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>>>
>>> Existing GIC driver has both generic code and hw specific
>>> code. Segregate GIC low level driver into gic-v2.c and
>>> keep generic code in existing gic.c file
>>>
>>> GIC v2 driver registers required functions
>>> to generic GIC driver. This helps to plug in next version
>>> of GIC drivers like GIC v3.
>>>
>>> Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>>
>> Am I correct in assuming that this patch is just code movement plus the
>> introduction of struct gic_hw_operations?
>>
> Yes, as a part of code movement, I introduced this structure and
> used this callbacks in generic code

 From what I saw, this patch is not only code movement. You are 
modifying/removing some functions without any explanation (see SGI stuffs).

I would prefer if you modify theses on another patch. So this patch will 
only be code movement.

Regards

-- 
Julien Grall

Re: [RFC PATCH v1 06/10] xen/arm: split gic driver into generic and gicv2 driver

[Ian Campbell]

On Sun, 2014-03-23 at 14:43 +0000, Julien Grall wrote:
> Hello Vijay,
> 
> On 22/03/14 09:32, Vijay Kilari wrote:
> > On Thu, Mar 20, 2014 at 5:25 PM, Stefano Stabellini
> > <stefano.stabellini@eu.citrix.com> wrote:
> >> On Wed, 19 Mar 2014, vijay.kilari@gmail.com wrote:
> >>> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> >>>
> >>> Existing GIC driver has both generic code and hw specific
> >>> code. Segregate GIC low level driver into gic-v2.c and
> >>> keep generic code in existing gic.c file
> >>>
> >>> GIC v2 driver registers required functions
> >>> to generic GIC driver. This helps to plug in next version
> >>> of GIC drivers like GIC v3.
> >>>
> >>> Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> >>
> >> Am I correct in assuming that this patch is just code movement plus the
> >> introduction of struct gic_hw_operations?
> >>
> > Yes, as a part of code movement, I introduced this structure and
> > used this callbacks in generic code
> 
>  From what I saw, this patch is not only code movement. You are 
> modifying/removing some functions without any explanation (see SGI stuffs).
> 
> I would prefer if you modify theses on another patch. So this patch will 
> only be code movement.

Yes, please.

One plausible sequence of patches might be:
- Add struct gic_hw_operations for v2 stuff all in the same file
- Move the v2 stuff out into the new file (pure motion)
- Other changes/cleanups
- Add the v3 stuff

Ian.

Re: [RFC PATCH v1 06/10] xen/arm: split gic driver into generic and gicv2 driver

[Julien Grall]

Hello Vijay,

Thanks for the patch.

On 03/19/2014 02:17 PM, vijay.kilari@gmail.com wrote:
> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> 
> Existing GIC driver has both generic code and hw specific
> code. Segregate GIC low level driver into gic-v2.c and
> keep generic code in existing gic.c file
> 
> GIC v2 driver registers required functions
> to generic GIC driver. This helps to plug in next version
> of GIC drivers like GIC v3.
> 
> Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> ---

[..]

> +
> +void __init gicv2_init(void)
> +{

Instead of calling gicv2_init and gicv3_init from generic, it would be
better to the device API (see xen/include/asm-arm/device.h). An example
of use is my iommu series (see https://patches.linaro.org/26032/
iommu_hardware_setup).

[..]

> -void send_SGI_one(unsigned int cpu, enum gic_sgi sgi)
> +static void send_SGI_one(unsigned int cpu, enum gic_sgi sgi)

Why did you put static here?

>  {
> -    ASSERT(cpu < NR_GIC_CPU_IF);  /* Targets bitmap only supports 8
CPUs */
>      send_SGI_mask(cpumask_of(cpu), sgi);
>  }
>
> -void send_SGI_self(enum gic_sgi sgi)
> -{
> -    ASSERT(sgi < 16); /* There are only 16 SGIs */
> -
> -    dsb();
> -
> -    GICD[GICD_SGIR] = GICD_SGI_TARGET_SELF
> -        | sgi;
> -}
> -

Why did you remove send_SGI_self?

>  void send_SGI_allbutself(enum gic_sgi sgi)
>  {
> -   ASSERT(sgi < 16); /* There are only 16 SGIs */
> +    cpumask_t all_others_mask;
> +    ASSERT(sgi < 16); /* There are only 16 SGIs */
>
> -   dsb();
> +    cpumask_andnot(&all_others_mask, &cpu_possible_map,
cpumask_of(smp_processor_id()));
>
> -   GICD[GICD_SGIR] = GICD_SGI_TARGET_OTHERS
> -       | sgi;
> +    dsb();
> +    send_SGI_mask(&all_others_mask, sgi);

Why did you remove the optmization for GICv2?

[..]

> +#define GIC_VERSION_V2 0x2
> +

I would prefer if you define an enum here with for now only one value
GIC_VERSION_V2.

>  /* SGI (AKA IPIs) */
>  enum gic_sgi {
>      GIC_SGI_EVENT_CHECK = 0,
>      GIC_SGI_DUMP_STATE  = 1,
>      GIC_SGI_CALL_FUNCTION = 2,
>  };
> +
> +struct gic_hw_operations {

Can you explain a bit more your structure? Why do you need so all theses
callbacks...

> +    int (*gic_type)(void);

This functions always return the same value on GICv2 (e.g
GIC_VERSION_V2) and GICv3 (e.g GIC_VERSION_V3), right?

If so, using int type directly is better.

[..]
> +    void (*enable_irq)(int);
> +    void (*disable_irq)(int);
> +    void (*eoi_irq)(int);
> +    void (*deactivate_irq)(int);
> +    unsigned int (*ack_irq)(void);

I would prefer to introduce a new hw_controller here rather than adding
another abstraction.

[..]

> +    unsigned long (*read_cpu_rbase)(void);
> +    unsigned long (*read_cpu_sgi_rbase)(void);

The both function above are GICv3 specific and not defined in GICv2 ...
why do you need it? If you plan to implement later, please add them in
the corresponding patch.

Regards,

-- 
Julien Grall

Re: [RFC PATCH v1 06/10] xen/arm: split gic driver into generic and gicv2 driver

[Ian Campbell]

On Thu, 2014-03-20 at 16:02 +0000, Julien Grall wrote:
> Hello Vijay,
> 
> Thanks for the patch.
> 
> On 03/19/2014 02:17 PM, vijay.kilari@gmail.com wrote:
> > From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> > 
> > Existing GIC driver has both generic code and hw specific
> > code. Segregate GIC low level driver into gic-v2.c and
> > keep generic code in existing gic.c file
> > 
> > GIC v2 driver registers required functions
> > to generic GIC driver. This helps to plug in next version
> > of GIC drivers like GIC v3.
> > 
> > Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> > ---
> 
> [..]
> 
> > +
> > +void __init gicv2_init(void)
> > +{
> 
> Instead of calling gicv2_init and gicv3_init from generic, it would be
> better to the device API (see xen/include/asm-arm/device.h). An example
> of use is my iommu series (see https://patches.linaro.org/26032/
> iommu_hardware_setup).

Is that usable for something needed as soon as interrupt controller
setup?

> > +struct gic_hw_operations {
> 
> Can you explain a bit more your structure? Why do you need so all theses
> callbacks...

In fact each should come a little doc comment about their usage, e.g.
like platform.h.

> [..]
> > +    void (*enable_irq)(int);
> > +    void (*disable_irq)(int);
> > +    void (*eoi_irq)(int);
> > +    void (*deactivate_irq)(int);
> > +    unsigned int (*ack_irq)(void);
> 
> I would prefer to introduce a new hw_controller here rather than adding
> another abstraction.

Agreed.

Ian.

Re: [RFC PATCH v1 06/10] xen/arm: split gic driver into generic and gicv2 driver

[Julien Grall]

On 03/21/2014 05:32 PM, Ian Campbell wrote:
> On Thu, 2014-03-20 at 16:02 +0000, Julien Grall wrote:
>> Hello Vijay,
>>
>> Thanks for the patch.
>>
>> On 03/19/2014 02:17 PM, vijay.kilari@gmail.com wrote:
>>> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>>>
>>> Existing GIC driver has both generic code and hw specific
>>> code. Segregate GIC low level driver into gic-v2.c and
>>> keep generic code in existing gic.c file
>>>
>>> GIC v2 driver registers required functions
>>> to generic GIC driver. This helps to plug in next version
>>> of GIC drivers like GIC v3.
>>>
>>> Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>>> ---
>>
>> [..]
>>
>>> +
>>> +void __init gicv2_init(void)
>>> +{
>>
>> Instead of calling gicv2_init and gicv3_init from generic, it would be
>> better to the device API (see xen/include/asm-arm/device.h). An example
>> of use is my iommu series (see https://patches.linaro.org/26032/
>> iommu_hardware_setup).
> 
> Is that usable for something needed as soon as interrupt controller
> setup?

As soon as the device tree is parsed, the "device API" can be used :).

Regards,

-- 
Julien Grall

Re: [RFC PATCH v1 06/10] xen/arm: split gic driver into generic and gicv2 driver

[Vijay Kilari]

On Thu, Mar 20, 2014 at 9:32 PM, Julien Grall <julien.grall@linaro.org> wrote:
> Hello Vijay,
>
> Thanks for the patch.
>
> On 03/19/2014 02:17 PM, vijay.kilari@gmail.com wrote:
>> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>>
>> Existing GIC driver has both generic code and hw specific
>> code. Segregate GIC low level driver into gic-v2.c and
>> keep generic code in existing gic.c file
>>
>> GIC v2 driver registers required functions
>> to generic GIC driver. This helps to plug in next version
>> of GIC drivers like GIC v3.
>>
>> Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>> ---
>
> [..]
>
>> +
>> +void __init gicv2_init(void)
>> +{
>
> Instead of calling gicv2_init and gicv3_init from generic, it would be
> better to the device API (see xen/include/asm-arm/device.h). An example
> of use is my iommu series (see https://patches.linaro.org/26032/
> iommu_hardware_setup).
>
OK. I will check

> [..]
>
>> -void send_SGI_one(unsigned int cpu, enum gic_sgi sgi)
>> +static void send_SGI_one(unsigned int cpu, enum gic_sgi sgi)
>
> Why did you put static here?
>
I think it is not being called from outside of this file.
Should I keep it non static for future use?

>>  {
>> -    ASSERT(cpu < NR_GIC_CPU_IF);  /* Targets bitmap only supports 8
> CPUs */
>>      send_SGI_mask(cpumask_of(cpu), sgi);
>>  }
>>
>> -void send_SGI_self(enum gic_sgi sgi)
>> -{
>> -    ASSERT(sgi < 16); /* There are only 16 SGIs */
>> -
>> -    dsb();
>> -
>> -    GICD[GICD_SGIR] = GICD_SGI_TARGET_SELF
>> -        | sgi;
>> -}
>> -
>
> Why did you remove send_SGI_self?
>
It is not used at all.

>>  void send_SGI_allbutself(enum gic_sgi sgi)
>>  {
>> -   ASSERT(sgi < 16); /* There are only 16 SGIs */
>> +    cpumask_t all_others_mask;
>> +    ASSERT(sgi < 16); /* There are only 16 SGIs */
>>
>> -   dsb();
>> +    cpumask_andnot(&all_others_mask, &cpu_possible_map,
> cpumask_of(smp_processor_id()));
>>
>> -   GICD[GICD_SGIR] = GICD_SGI_TARGET_OTHERS
>> -       | sgi;
>> +    dsb();
>> +    send_SGI_mask(&all_others_mask, sgi);
>
> Why did you remove the optmization for GICv2?
What was the optimization that was there earlier?
The gic-v2.c is handling the hw specific code

>
> [..]
>
>> +#define GIC_VERSION_V2 0x2
>> +
>
> I would prefer if you define an enum here with for now only one value
> GIC_VERSION_V2.
>
>>  /* SGI (AKA IPIs) */
>>  enum gic_sgi {
>>      GIC_SGI_EVENT_CHECK = 0,
>>      GIC_SGI_DUMP_STATE  = 1,
>>      GIC_SGI_CALL_FUNCTION = 2,
>>  };
>> +
>> +struct gic_hw_operations {
>
> Can you explain a bit more your structure? Why do you need so all theses
> callbacks...
>
>> +    int (*gic_type)(void);
>
> This functions always return the same value on GICv2 (e.g
> GIC_VERSION_V2) and GICv3 (e.g GIC_VERSION_V3), right?
>
> If so, using int type directly is better.
>
> [..]
>> +    void (*enable_irq)(int);
>> +    void (*disable_irq)(int);
>> +    void (*eoi_irq)(int);
>> +    void (*deactivate_irq)(int);
>> +    unsigned int (*ack_irq)(void);
>
> I would prefer to introduce a new hw_controller here rather than adding
> another abstraction.
>
Can you please explain more on what is meant by new hw_controller?

> [..]
>
>> +    unsigned long (*read_cpu_rbase)(void);
>> +    unsigned long (*read_cpu_sgi_rbase)(void);
>
> The both function above are GICv3 specific and not defined in GICv2 ...
> why do you need it? If you plan to implement later, please add them in
> the corresponding patch.
>
OK. I will introduce with GICv3 patch
> Regards,
>
> --
> Julien Grall

Re: [RFC PATCH v1 06/10] xen/arm: split gic driver into generic and gicv2 driver

[Julien Grall]

Hello Vijay,

On 22/03/14 09:40, Vijay Kilari wrote:
> On Thu, Mar 20, 2014 at 9:32 PM, Julien Grall <julien.grall@linaro.org> wrote:
>> Hello Vijay,
>>
>> Thanks for the patch.
>>
>> On 03/19/2014 02:17 PM, vijay.kilari@gmail.com wrote:
>>> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>>>
>>> Existing GIC driver has both generic code and hw specific
>>> code. Segregate GIC low level driver into gic-v2.c and
>>> keep generic code in existing gic.c file
>>>
>>> GIC v2 driver registers required functions
>>> to generic GIC driver. This helps to plug in next version
>>> of GIC drivers like GIC v3.
>>>
>>> Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>>> ---
>>
>> [..]
>>
>>> +
>>> +void __init gicv2_init(void)
>>> +{
>>
>> Instead of calling gicv2_init and gicv3_init from generic, it would be
>> better to the device API (see xen/include/asm-arm/device.h). An example
>> of use is my iommu series (see https://patches.linaro.org/26032/
>> iommu_hardware_setup).
>>
> OK. I will check
>
>> [..]
>>
>>> -void send_SGI_one(unsigned int cpu, enum gic_sgi sgi)
>>> +static void send_SGI_one(unsigned int cpu, enum gic_sgi sgi)
>>
>> Why did you put static here?
>>
> I think it is not being called from outside of this file.
> Should I keep it non static for future use?

I think this change should not be part of this patch. As you said this 
patch should be only code movement, and adding prototype is definitely 
not code movement.

If you want to keep it, I would prefer a separate patch.

>>>   {
>>> -    ASSERT(cpu < NR_GIC_CPU_IF);  /* Targets bitmap only supports 8
>> CPUs */
>>>       send_SGI_mask(cpumask_of(cpu), sgi);
>>>   }
>>>
It might interesting to keep it just in case.

If you plan to remove it from
>>> -void send_SGI_self(enum gic_sgi sgi)
>>> -{
>>> -    ASSERT(sgi < 16); /* There are only 16 SGIs */
>>> -
>>> -    dsb();
>>> -
>>> -    GICD[GICD_SGIR] = GICD_SGI_TARGET_SELF
>>> -        | sgi;
>>> -}
>>> -
>>
>> Why did you remove send_SGI_self?
>>
> It is not used at all.

Same as send_SGI_one. It's not part of this patch.

>>>   void send_SGI_allbutself(enum gic_sgi sgi)
>>>   {
>>> -   ASSERT(sgi < 16); /* There are only 16 SGIs */
>>> +    cpumask_t all_others_mask;
>>> +    ASSERT(sgi < 16); /* There are only 16 SGIs */
>>>
>>> -   dsb();
>>> +    cpumask_andnot(&all_others_mask, &cpu_possible_map,
>> cpumask_of(smp_processor_id()));
>>>
>>> -   GICD[GICD_SGIR] = GICD_SGI_TARGET_OTHERS
>>> -       | sgi;
>>> +    dsb();
>>> +    send_SGI_mask(&all_others_mask, sgi);
>>
>> Why did you remove the optmization for GICv2?
> What was the optimization that was there earlier?

GICD_SGI_TARGET_OTHERS, will send an SGI to every CPU even if the CPU is 
not yet online (e.g not registered by Xen). It's used during secondary 
boot (see cpu_up_send_sgi).

Here you are breaking SMP boot on GICv2.

>> [..]
>>> +    void (*enable_irq)(int);
>>> +    void (*disable_irq)(int);
>>> +    void (*eoi_irq)(int);
>>> +    void (*deactivate_irq)(int);
>>> +    unsigned int (*ack_irq)(void);
>>
>> I would prefer to introduce a new hw_controller here rather than adding
>> another abstraction.
>>
> Can you please explain more on what is meant by new hw_controller?

oops, I meant hw_irq_controller sorry. (see xen/include/xen/irq.h).

Regards,

-- 
Julien Grall

Re: [RFC PATCH v1 06/10] xen/arm: split gic driver into generic and gicv2 driver

[Stefano Stabellini]

On Wed, 19 Mar 2014, vijay.kilari@gmail.com wrote:
> +static struct gic_hw_operations gic_ops = {
> +    .gic_type            = gic_hw_type,
> +    .nr_lines            = gic_nr_lines,
> +    .nr_lrs              = gic_nr_lrs,
> +    .get_maintenance_irq = gic_maintenance_irq,
> +    .state_init        = gic_state_init,
> +    .save_state          = save_state,
> +    .restore_state       = restore_state,
> +    .dump_state          = gic_dump_state,
> +    .gicv_setup          = gicv_init,
> +    .enable_irq          = gic_enable_irq,
> +    .disable_irq         = gic_disable_irq,
> +    .eoi_irq             = gic_eoi_irq,
> +    .deactivate_irq      = gic_dir_irq,
> +    .ack_irq             = gic_ack_irq,
> +    .set_irq_property    = gic_set_irq_properties,
> +    .send_sgi            = gic_send_sgi,
> +    .disable_interface   = gic_disable_interface,
> +    .update_lr           = gic_update_lr,
> +    .update_lr_for_mi    = gic_update_lr_for_mi,

I would prefer to have this function called clear_lr


> +    .read_eisr           = gic_read_eisr,
> +};

Re: [RFC PATCH v1 06/10] xen/arm: split gic driver into generic and gicv2 driver

[Ian Campbell]

On Wed, 2014-03-19 at 19:47 +0530, vijay.kilari@gmail.com wrote:

> +/* Global state */
> +static struct {
> +    int hw_version;
> +    paddr_t dbase;       /* Address of distributor registers */
> +    paddr_t cbase;       /* Address of CPU interface registers */
> +    paddr_t hbase;       /* Address of virtual interface registers */
> +    paddr_t vbase;       /* Address of virtual cpu interface registers */
> +    unsigned int lines;  /* Number of interrupts (SPIs + PPIs + SGIs) */
> +    struct dt_irq maintenance; /* IRQ maintenance */
> +    unsigned int cpus;
> +    spinlock_t lock;
> +} gic;
> +
[...]
> +        spin_lock(&gic.lock);
> [...]
> +    spin_lock(&gic.lock);
> diff --git a/xen/arch/arm/gic.c b/xen/arch/arm/gic.c
> index bb718f6..e0859ae 100644
> --- a/xen/arch/arm/gic.c
> +++ b/xen/arch/arm/gic.c
[...]
> +spinlock_t gic_lock;
> [...]
> @@ -151,12 +107,12 @@ static void gic_irq_enable(struct irq_desc *desc)
>      unsigned long flags;
>  
>      spin_lock_irqsave(&desc->lock, flags);
> -    spin_lock(&gic.lock);
> +    spin_lock(&gic_lock);

You seem to have divided the gic lock into two, is that deliberate?

What is the locking hierarchy between them?

> @@ -286,9 +221,9 @@ static int gic_route_irq(unsigned int irq, bool_t level,
>  
>      desc->handler = &gic_host_irq_type;
>  
> -    spin_lock(&gic.lock);
> +    spin_lock(&gic_lock);

e.g. this function will call through to hook in the gic-v2 code which
used to require gic.lock to be taken, and accordig to the comment it
still does after this refactoring. Yet that lock isn't held here any
more (nor is it accessible to this code).

>      gic_set_irq_properties(irq, level, cpu_mask, priority);
> -    spin_unlock(&gic.lock);
> +    spin_unlock(&gic_lock);
>  
>      spin_unlock_irqrestore(&desc->lock, flags);
>      return 0;

Ian.

Re: [RFC PATCH v1 06/10] xen/arm: split gic driver into generic and gicv2 driver

[Vijay Kilari]

On Fri, Mar 21, 2014 at 11:08 PM, Ian Campbell <Ian.Campbell@citrix.com> wrote:
> On Wed, 2014-03-19 at 19:47 +0530, vijay.kilari@gmail.com wrote:
>
>> +/* Global state */
>> +static struct {
>> +    int hw_version;
>> +    paddr_t dbase;       /* Address of distributor registers */
>> +    paddr_t cbase;       /* Address of CPU interface registers */
>> +    paddr_t hbase;       /* Address of virtual interface registers */
>> +    paddr_t vbase;       /* Address of virtual cpu interface registers */
>> +    unsigned int lines;  /* Number of interrupts (SPIs + PPIs + SGIs) */
>> +    struct dt_irq maintenance; /* IRQ maintenance */
>> +    unsigned int cpus;
>> +    spinlock_t lock;
>> +} gic;
>> +
> [...]
>> +        spin_lock(&gic.lock);
>> [...]
>> +    spin_lock(&gic.lock);
>> diff --git a/xen/arch/arm/gic.c b/xen/arch/arm/gic.c
>> index bb718f6..e0859ae 100644
>> --- a/xen/arch/arm/gic.c
>> +++ b/xen/arch/arm/gic.c
> [...]
>> +spinlock_t gic_lock;
>> [...]
>> @@ -151,12 +107,12 @@ static void gic_irq_enable(struct irq_desc *desc)
>>      unsigned long flags;
>>
>>      spin_lock_irqsave(&desc->lock, flags);
>> -    spin_lock(&gic.lock);
>> +    spin_lock(&gic_lock);
>
> You seem to have divided the gic lock into two, is that deliberate?
>
> What is the locking hierarchy between them?
>

>> @@ -286,9 +221,9 @@ static int gic_route_irq(unsigned int irq, bool_t level,
>>
>>      desc->handler = &gic_host_irq_type;
>>
>> -    spin_lock(&gic.lock);
>> +    spin_lock(&gic_lock);
>
> e.g. this function will call through to hook in the gic-v2 code which
> used to require gic.lock to be taken, and accordig to the comment it
> still does after this refactoring. Yet that lock isn't held here any
> more (nor is it accessible to this code).
>

  Yes, the lock in generic code gic.c file is kept as it is.
and gic-v2.c which contains the callbacks will not hold any lock
and assumes that caller is taking care of it.
I made this approach because there is some functions in generic
code in gic. c which require lock even after calling gic-v2 callbacks
So I kept lock with generic code in gic.c

I have add new lock in gic-v2.c file to only syn for the calls that
are not callback functions which are called from gic.c.
Ex: the gic_(hyp/cpu}_init are called directly from notifier.
Only these functions are locked.

>>      gic_set_irq_properties(irq, level, cpu_mask, priority);
>> -    spin_unlock(&gic.lock);
>> +    spin_unlock(&gic_lock);
>>
>>      spin_unlock_irqrestore(&desc->lock, flags);
>>      return 0;
>
> Ian.
>

Re: [RFC PATCH v1 06/10] xen/arm: split gic driver into generic and gicv2 driver

[Ian Campbell]

On Sat, 2014-03-22 at 15:29 +0530, Vijay Kilari wrote:
> On Fri, Mar 21, 2014 at 11:08 PM, Ian Campbell <Ian.Campbell@citrix.com> wrote:
> > On Wed, 2014-03-19 at 19:47 +0530, vijay.kilari@gmail.com wrote:
> >
> >> +/* Global state */
> >> +static struct {
> >> +    int hw_version;
> >> +    paddr_t dbase;       /* Address of distributor registers */
> >> +    paddr_t cbase;       /* Address of CPU interface registers */
> >> +    paddr_t hbase;       /* Address of virtual interface registers */
> >> +    paddr_t vbase;       /* Address of virtual cpu interface registers */
> >> +    unsigned int lines;  /* Number of interrupts (SPIs + PPIs + SGIs) */
> >> +    struct dt_irq maintenance; /* IRQ maintenance */
> >> +    unsigned int cpus;
> >> +    spinlock_t lock;
> >> +} gic;
> >> +
> > [...]
> >> +        spin_lock(&gic.lock);
> >> [...]
> >> +    spin_lock(&gic.lock);
> >> diff --git a/xen/arch/arm/gic.c b/xen/arch/arm/gic.c
> >> index bb718f6..e0859ae 100644
> >> --- a/xen/arch/arm/gic.c
> >> +++ b/xen/arch/arm/gic.c
> > [...]
> >> +spinlock_t gic_lock;
> >> [...]
> >> @@ -151,12 +107,12 @@ static void gic_irq_enable(struct irq_desc *desc)
> >>      unsigned long flags;
> >>
> >>      spin_lock_irqsave(&desc->lock, flags);
> >> -    spin_lock(&gic.lock);
> >> +    spin_lock(&gic_lock);
> >
> > You seem to have divided the gic lock into two, is that deliberate?
> >
> > What is the locking hierarchy between them?
> >
> 
> >> @@ -286,9 +221,9 @@ static int gic_route_irq(unsigned int irq, bool_t level,
> >>
> >>      desc->handler = &gic_host_irq_type;
> >>
> >> -    spin_lock(&gic.lock);
> >> +    spin_lock(&gic_lock);
> >
> > e.g. this function will call through to hook in the gic-v2 code which
> > used to require gic.lock to be taken, and accordig to the comment it
> > still does after this refactoring. Yet that lock isn't held here any
> > more (nor is it accessible to this code).
> >
> 
>   Yes, the lock in generic code gic.c file is kept as it is.
> and gic-v2.c which contains the callbacks will not hold any lock
> and assumes that caller is taking care of it.
> I made this approach because there is some functions in generic
> code in gic. c which require lock even after calling gic-v2 callbacks
> So I kept lock with generic code in gic.c
> 
> I have add new lock in gic-v2.c file to only syn for the calls that
> are not callback functions which are called from gic.c.
> Ex: the gic_(hyp/cpu}_init are called directly from notifier.
> Only these functions are locked.

But against what? What is the purpose of this lock if not to enforce a
critical section against the callbacks?

Ian.

[RFC PATCH v1 07/10] xen/arm: split vgic into generic and GIC v2 specific drivers

[vijay.kilari]

From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>

vgic driver contains both generic and GIC v2 specific
funtionality together. Segregate vgic driver into generic
code into vgic.c and GIC v2 specific functionality into
vgic-v2.c files. This helps to and GIC v3 specific funtionality.

Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
---
 xen/arch/arm/Makefile     |    2 +-
 xen/arch/arm/vgic-v2.c    |  636 ++++++++++++++++++++++++++++++++++++++++++++
 xen/arch/arm/vgic.c       |  640 ++++-----------------------------------------
 xen/include/asm-arm/gic.h |    9 +
 4 files changed, 695 insertions(+), 592 deletions(-)

diff --git a/xen/arch/arm/Makefile b/xen/arch/arm/Makefile
index 969ee52..20f59f4 100644
--- a/xen/arch/arm/Makefile
+++ b/xen/arch/arm/Makefile
@@ -26,7 +26,7 @@ obj-y += smpboot.o
 obj-y += smp.o
 obj-y += shutdown.o
 obj-y += traps.o
-obj-y += vgic.o
+obj-y += vgic.o vgic-v2.o
 obj-y += vtimer.o
 obj-y += vuart.o
 obj-y += hvm.o
diff --git a/xen/arch/arm/vgic-v2.c b/xen/arch/arm/vgic-v2.c
new file mode 100644
index 0000000..1e69763
--- /dev/null
+++ b/xen/arch/arm/vgic-v2.c
@@ -0,0 +1,636 @@
+/*
+ * xen/arch/arm/vgic-v2.c
+ *
+ * ARM Virtual Generic Interrupt Controller support v2
+ *
+ * Ian Campbell <ian.campbell@citrix.com>
+ * Copyright (c) 2011 Citrix Systems.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <xen/bitops.h>
+#include <xen/config.h>
+#include <xen/lib.h>
+#include <xen/init.h>
+#include <xen/softirq.h>
+#include <xen/irq.h>
+#include <xen/sched.h>
+
+#include <asm/current.h>
+
+#include "io.h"
+#include <asm/gic_v2_defs.h>
+#include <asm/gic.h>
+
+struct vgic_irq_rank {
+    spinlock_t lock; /* Covers access to all other members of this struct */
+    uint32_t ienable, iactive, ipend, pendsgi;
+    uint32_t icfg[2];
+    uint32_t ipriority[8];
+    uint32_t itargets[8];
+};
+
+#define REG(n) (n/4)
+
+/* Number of ranks of interrupt registers for a domain */
+#define DOMAIN_NR_RANKS(d) (((d)->arch.vgic.nr_lines+31)/32)
+
+/*
+ * Rank containing GICD_<FOO><n> for GICD_<FOO> with
+ * <b>-bits-per-interrupt
+ */
+static inline int REG_RANK_NR(int b, uint32_t n)
+{
+    switch ( b )
+    {
+    case 8: return n >> 3;
+    case 4: return n >> 2;
+    case 2: return n >> 1;
+    case 1: return n;
+    default: BUG();
+    }
+}
+
+/*
+ * Offset of GICD_<FOO><n> with its rank, for GICD_<FOO> with
+ * <b>-bits-per-interrupt.
+ */
+#define REG_RANK_INDEX(b, n) ((n) & ((b)-1))
+
+/*
+ * Returns rank corresponding to a GICD_<FOO><n> register for
+ * GICD_<FOO> with <b>-bits-per-interrupt.
+ */
+static struct vgic_irq_rank *vgic_irq_rank(struct vcpu *v, int b, int n)
+{
+    int rank = REG_RANK_NR(b, n);
+
+    if ( rank == 0 )
+       return (struct vgic_irq_rank *)v->arch.vgic.private_irqs;
+    else if ( rank <= DOMAIN_NR_RANKS(v->domain) )
+       return (struct vgic_irq_rank *)((unsigned char *)(v->domain->arch.vgic.shared_irqs)
+                           + (sizeof(struct vgic_irq_rank) *(rank - 1)));
+    else
+        return NULL;
+}
+
+static uint32_t byte_read(uint32_t val, int sign, int offset)
+{
+    int byte = offset & 0x3;
+
+    val = val >> (8*byte);
+    if ( sign && (val & 0x80) )
+        val |= 0xffffff00;
+    else
+        val &= 0x000000ff;
+    return val;
+}
+
+static void byte_write(uint32_t *reg, uint32_t var, int offset)
+{
+    int byte = offset & 0x3;
+
+    var &= (0xff << (8*byte));
+
+    *reg &= ~(0xff << (8*byte));
+    *reg |= var;
+}
+
+static int vgic_read_priority(struct vcpu *v, int irq) 
+{
+   int idx = irq >> 2;
+   struct vgic_irq_rank *rank = vgic_irq_rank(v, 8, idx);
+   return byte_read(rank->ipriority[REG_RANK_INDEX(8, idx)], 0, irq & 0x3);
+}
+
+static int vgic_vcpu_init(struct vcpu *v)
+{
+    int i;
+    struct vgic_irq_rank *vir;
+
+    vir =  xzalloc(struct vgic_irq_rank);
+    memset(vir, 0, sizeof(struct vgic_irq_rank));
+
+    spin_lock_init(&vir->lock);
+
+    /* For SGI and PPI the target is always this CPU */
+    for ( i = 0 ; i < 8 ; i++ )
+        vir->itargets[i] =
+              (1<<(v->vcpu_id+0))
+            | (1<<(v->vcpu_id+8))
+            | (1<<(v->vcpu_id+16))
+            | (1<<(v->vcpu_id+24));
+
+    v->arch.vgic.private_irqs = (struct vgic_irq_irank *)vir;
+    return 0;
+}
+
+static struct vgic_ops ops = {
+    .vgic_vcpu_init   = vgic_vcpu_init,
+    .read_priority    = vgic_read_priority,
+};
+
+#define vgic_lock(v)   spin_lock_irq(&(v)->domain->arch.vgic.lock)
+#define vgic_unlock(v) spin_unlock_irq(&(v)->domain->arch.vgic.lock)
+
+#define vgic_lock_rank(v, r) spin_lock(&(r)->lock)
+#define vgic_unlock_rank(v, r) spin_unlock(&(r)->lock)
+
+static int vgic_distr_mmio_read(struct vcpu *v, mmio_info_t *info)
+{
+    struct hsr_dabt dabt = info->dabt;
+    struct cpu_user_regs *regs = guest_cpu_user_regs();
+    register_t *r = select_user_reg(regs, dabt.reg);
+    struct vgic_irq_rank *rank;
+    int offset = (int)(info->gpa - v->domain->arch.vgic.dbase);
+    int gicd_reg = REG(offset);
+
+    switch ( gicd_reg )
+    {
+    case GICD_CTLR:
+        if ( dabt.size != 2 ) goto bad_width;
+        vgic_lock(v);
+        *r = v->domain->arch.vgic.ctlr;
+        vgic_unlock(v);
+        return 1;
+    case GICD_TYPER:
+        if ( dabt.size != 2 ) goto bad_width;
+        /* No secure world support for guests. */
+        vgic_lock(v);
+        *r = ( (v->domain->max_vcpus<<5) & GICD_TYPE_CPUS )
+            |( ((v->domain->arch.vgic.nr_lines/32)) & GICD_TYPE_LINES );
+        vgic_unlock(v);
+        return 1;
+    case GICD_IIDR:
+        if ( dabt.size != 2 ) goto bad_width;
+        /*
+         * XXX Do we need a JEP106 manufacturer ID?
+         * Just use the physical h/w value for now
+         */
+        *r = 0x0000043b;
+        return 1;
+
+    /* Implementation defined -- read as zero */
+    case REG(0x020) ... REG(0x03c):
+        goto read_as_zero;
+
+    case GICD_IGROUPR ... GICD_IGROUPRN:
+        /* We do not implement security extensions for guests, read zero */
+        goto read_as_zero;
+
+    case GICD_ISENABLER ... GICD_ISENABLERN:
+        if ( dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 1, gicd_reg - GICD_ISENABLER);
+        if ( rank == NULL) goto read_as_zero;
+        vgic_lock_rank(v, rank);
+        *r = rank->ienable;
+        vgic_unlock_rank(v, rank);
+        return 1;
+
+    case GICD_ICENABLER ... GICD_ICENABLERN:
+        if ( dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 1, gicd_reg - GICD_ICENABLER);
+        if ( rank == NULL) goto read_as_zero;
+        vgic_lock_rank(v, rank);
+        *r = rank->ienable;
+        vgic_unlock_rank(v, rank);
+        return 1;
+
+    case GICD_ISPENDR ... GICD_ISPENDRN:
+        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 1, gicd_reg - GICD_ISPENDR);
+        if ( rank == NULL) goto read_as_zero;
+        vgic_lock_rank(v, rank);
+        *r = byte_read(rank->ipend, dabt.sign, offset);
+        vgic_unlock_rank(v, rank);
+        return 1;
+
+    case GICD_ICPENDR ... GICD_ICPENDRN:
+        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 1, gicd_reg - GICD_ICPENDR);
+        if ( rank == NULL) goto read_as_zero;
+        vgic_lock_rank(v, rank);
+        *r = byte_read(rank->ipend, dabt.sign, offset);
+        vgic_unlock_rank(v, rank);
+        return 1;
+
+    case GICD_ISACTIVER ... GICD_ISACTIVERN:
+        if ( dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 1, gicd_reg - GICD_ISACTIVER);
+        if ( rank == NULL) goto read_as_zero;
+        vgic_lock_rank(v, rank);
+        *r = rank->iactive;
+        vgic_unlock_rank(v, rank);
+        return 1;
+
+    case GICD_ICACTIVER ... GICD_ICACTIVERN:
+        if ( dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 1, gicd_reg - GICD_ICACTIVER);
+        if ( rank == NULL) goto read_as_zero;
+        vgic_lock_rank(v, rank);
+        *r = rank->iactive;
+        vgic_unlock_rank(v, rank);
+        return 1;
+
+    case GICD_ITARGETSR ... GICD_ITARGETSRN:
+        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 8, gicd_reg - GICD_ITARGETSR);
+        if ( rank == NULL) goto read_as_zero;
+
+        vgic_lock_rank(v, rank);
+        *r = rank->itargets[REG_RANK_INDEX(8, gicd_reg - GICD_ITARGETSR)];
+        if ( dabt.size == 0 )
+            *r = byte_read(*r, dabt.sign, offset);
+        vgic_unlock_rank(v, rank);
+        return 1;
+
+    case GICD_IPRIORITYR ... GICD_IPRIORITYRN:
+        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 8, gicd_reg - GICD_IPRIORITYR);
+        if ( rank == NULL) goto read_as_zero;
+
+        vgic_lock_rank(v, rank);
+        *r = rank->ipriority[REG_RANK_INDEX(8, gicd_reg - GICD_IPRIORITYR)];
+        if ( dabt.size == 0 )
+            *r = byte_read(*r, dabt.sign, offset);
+        vgic_unlock_rank(v, rank);
+        return 1;
+
+    case GICD_ICFGR ... GICD_ICFGRN:
+        if ( dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 2, gicd_reg - GICD_ICFGR);
+        if ( rank == NULL) goto read_as_zero;
+        vgic_lock_rank(v, rank);
+        *r = rank->icfg[REG_RANK_INDEX(2, gicd_reg - GICD_ICFGR)];
+        vgic_unlock_rank(v, rank);
+        return 1;
+
+    case GICD_NSACR ... GICD_NSACRN:
+        /* We do not implement security extensions for guests, read zero */
+        goto read_as_zero;
+
+    case GICD_SGIR:
+        if ( dabt.size != 2 ) goto bad_width;
+        /* Write only -- read unknown */
+        *r = 0xdeadbeef;
+        return 1;
+
+    case GICD_CPENDSGIR ... GICD_CPENDSGIRN:
+        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 1, gicd_reg - GICD_CPENDSGIR);
+        if ( rank == NULL) goto read_as_zero;
+        vgic_lock_rank(v, rank);
+        *r = byte_read(rank->pendsgi, dabt.sign, offset);
+        vgic_unlock_rank(v, rank);
+        return 1;
+
+    case GICD_SPENDSGIR ... GICD_SPENDSGIRN:
+        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 1, gicd_reg - GICD_SPENDSGIR);
+        if ( rank == NULL) goto read_as_zero;
+        vgic_lock_rank(v, rank);
+        *r = byte_read(rank->pendsgi, dabt.sign, offset);
+        vgic_unlock_rank(v, rank);
+        return 1;
+
+    /* Implementation defined -- read as zero */
+    case REG(0xfd0) ... REG(0xfe4):
+        goto read_as_zero;
+
+    case GICD_ICPIDR2:
+        if ( dabt.size != 2 ) goto bad_width;
+        printk("vGICD: unhandled read from ICPIDR2\n");
+        return 0;
+
+    /* Implementation defined -- read as zero */
+    case REG(0xfec) ... REG(0xffc):
+        goto read_as_zero;
+
+    /* Reserved -- read as zero */
+    case REG(0x00c) ... REG(0x01c):
+    case REG(0x040) ... REG(0x07c):
+    case REG(0x7fc):
+    case REG(0xbfc):
+    case REG(0xf04) ... REG(0xf0c):
+    case REG(0xf30) ... REG(0xfcc):
+        goto read_as_zero;
+
+    default:
+        printk("vGICD: unhandled read r%d offset %#08x\n",
+               dabt.reg, offset);
+        return 0;
+    }
+
+bad_width:
+    printk("vGICD: bad read width %d r%d offset %#08x\n",
+           dabt.size, dabt.reg, offset);
+    domain_crash_synchronous();
+    return 0;
+
+read_as_zero:
+    if ( dabt.size != 2 ) goto bad_width;
+    *r = 0;
+    return 1;
+}
+
+static inline int is_vcpu_running(struct domain *d, int vcpuid)
+{
+    struct vcpu *v;
+
+    if ( vcpuid >= d->max_vcpus )
+        return 0;
+
+    v = d->vcpu[vcpuid];
+    if ( v == NULL )
+        return 0;
+    if (test_bit(_VPF_down, &v->pause_flags) )
+        return 0;
+
+    return 1;
+}
+
+static int vgic_to_sgi(struct vcpu *v, register_t sgir)
+{
+    struct domain *d = v->domain;
+    int virtual_irq;
+    int filter;
+    int vcpuid;
+    int i;
+    unsigned long vcpu_mask = 0;
+
+    ASSERT(d->max_vcpus < 8*sizeof(vcpu_mask));
+
+    filter = (sgir & GICD_SGI_TARGET_LIST_MASK);
+    virtual_irq = (sgir & GICD_SGI_INTID_MASK);
+    ASSERT( virtual_irq < 16 );
+
+    switch ( filter )
+    {
+        case GICD_SGI_TARGET_LIST:
+            vcpu_mask = (sgir & GICD_SGI_TARGET_MASK) >> GICD_SGI_TARGET_SHIFT;
+            break;
+        case GICD_SGI_TARGET_OTHERS:
+            for ( i = 0; i < d->max_vcpus; i++ )
+            {
+                if ( i != current->vcpu_id && is_vcpu_running(d, i) )
+                    set_bit(i, &vcpu_mask);
+            }
+            break;
+        case GICD_SGI_TARGET_SELF:
+            set_bit(current->vcpu_id, &vcpu_mask);
+            break;
+        default:
+            gdprintk(XENLOG_WARNING, "vGICD: unhandled GICD_SGIR write %"PRIregister" with wrong TargetListFilter field\n",
+                     sgir);
+            return 0;
+    }
+
+    for_each_set_bit( vcpuid, &vcpu_mask, d->max_vcpus )
+    {
+        if ( !is_vcpu_running(d, vcpuid) )
+        {
+            gdprintk(XENLOG_WARNING, "vGICD: GICD_SGIR write r=%"PRIregister" vcpu_mask=%lx, wrong CPUTargetList\n",
+                     sgir, vcpu_mask);
+            continue;
+        }
+        vgic_vcpu_inject_irq(d->vcpu[vcpuid], virtual_irq, 1);
+    }
+    return 1;
+}
+
+static int vgic_distr_mmio_write(struct vcpu *v, mmio_info_t *info)
+{
+    struct hsr_dabt dabt = info->dabt;
+    struct cpu_user_regs *regs = guest_cpu_user_regs();
+    register_t *r = select_user_reg(regs, dabt.reg);
+    struct vgic_irq_rank *rank;
+    int offset = (int)(info->gpa - v->domain->arch.vgic.dbase);
+    int gicd_reg = REG(offset);
+    uint32_t tr;
+
+    switch ( gicd_reg )
+    {
+    case GICD_CTLR:
+        if ( dabt.size != 2 ) goto bad_width;
+        /* Ignore all but the enable bit */
+        v->domain->arch.vgic.ctlr = (*r) & GICD_CTL_ENABLE;
+        return 1;
+
+    /* R/O -- write ignored */
+    case GICD_TYPER:
+    case GICD_IIDR:
+        goto write_ignore;
+
+    /* Implementation defined -- write ignored */
+    case REG(0x020) ... REG(0x03c):
+        goto write_ignore;
+
+    case GICD_IGROUPR ... GICD_IGROUPRN:
+        /* We do not implement security extensions for guests, write ignore */
+        goto write_ignore;
+
+    case GICD_ISENABLER ... GICD_ISENABLERN:
+        if ( dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 1, gicd_reg - GICD_ISENABLER);
+        if ( rank == NULL) goto write_ignore;
+        vgic_lock_rank(v, rank);
+        tr = rank->ienable;
+        rank->ienable |= *r;
+        vgic_unlock_rank(v, rank);
+        vgic_enable_irqs(v, (*r) & (~tr), gicd_reg - GICD_ISENABLER);
+        return 1;
+
+    case GICD_ICENABLER ... GICD_ICENABLERN:
+        if ( dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 1, gicd_reg - GICD_ICENABLER);
+        if ( rank == NULL) goto write_ignore;
+        vgic_lock_rank(v, rank);
+        tr = rank->ienable;
+        rank->ienable &= ~*r;
+        vgic_unlock_rank(v, rank);
+        vgic_disable_irqs(v, (*r) & tr, gicd_reg - GICD_ICENABLER);
+        return 1;
+
+    case GICD_ISPENDR ... GICD_ISPENDRN:
+        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
+        printk("vGICD: unhandled %s write %#"PRIregister" to ISPENDR%d\n",
+               dabt.size ? "word" : "byte", *r, gicd_reg - GICD_ISPENDR);
+        return 0;
+
+    case GICD_ICPENDR ... GICD_ICPENDRN:
+        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
+        printk("vGICD: unhandled %s write %#"PRIregister" to ICPENDR%d\n",
+               dabt.size ? "word" : "byte", *r, gicd_reg - GICD_ICPENDR);
+        return 0;
+
+    case GICD_ISACTIVER ... GICD_ISACTIVERN:
+        if ( dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 1, gicd_reg - GICD_ISACTIVER);
+        if ( rank == NULL) goto write_ignore;
+        vgic_lock_rank(v, rank);
+        rank->iactive &= ~*r;
+        vgic_unlock_rank(v, rank);
+        return 1;
+
+    case GICD_ICACTIVER ... GICD_ICACTIVERN:
+        if ( dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 1, gicd_reg - GICD_ICACTIVER);
+        if ( rank == NULL) goto write_ignore;
+        vgic_lock_rank(v, rank);
+        rank->iactive &= ~*r;
+        vgic_unlock_rank(v, rank);
+        return 1;
+
+    case GICD_ITARGETSR ... GICD_ITARGETSR + 7:
+        /* SGI/PPI target is read only */
+        goto write_ignore;
+
+    case GICD_ITARGETSR + 8 ... GICD_ITARGETSRN:
+        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 8, gicd_reg - GICD_ITARGETSR);
+        if ( rank == NULL) goto write_ignore;
+        vgic_lock_rank(v, rank);
+        if ( dabt.size == 2 )
+            rank->itargets[REG_RANK_INDEX(8, gicd_reg - GICD_ITARGETSR)] = *r;
+        else
+            byte_write(&rank->itargets[REG_RANK_INDEX(8, gicd_reg - GICD_ITARGETSR)],
+                       *r, offset);
+        vgic_unlock_rank(v, rank);
+        return 1;
+
+    case GICD_IPRIORITYR ... GICD_IPRIORITYRN:
+        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 8, gicd_reg - GICD_IPRIORITYR);
+        if ( rank == NULL) goto write_ignore;
+        vgic_lock_rank(v, rank);
+        if ( dabt.size == 2 )
+            rank->ipriority[REG_RANK_INDEX(8, gicd_reg - GICD_IPRIORITYR)] = *r;
+        else
+            byte_write(&rank->ipriority[REG_RANK_INDEX(8, gicd_reg - GICD_IPRIORITYR)],
+                       *r, offset);
+        vgic_unlock_rank(v, rank);
+        return 1;
+
+    case GICD_ICFGR: /* SGIs */
+        goto write_ignore;
+    case GICD_ICFGR + 1: /* PPIs */
+        /* It is implementation defined if these are writeable. We chose not */
+        goto write_ignore;
+    case GICD_ICFGR + 2 ... GICD_ICFGRN: /* SPIs */
+        if ( dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 2, gicd_reg - GICD_ICFGR);
+        vgic_lock_rank(v, rank);
+        if ( rank == NULL) goto write_ignore;
+        rank->icfg[REG_RANK_INDEX(2, gicd_reg - GICD_ICFGR)] = *r;
+        vgic_unlock_rank(v, rank);
+        return 1;
+
+    case GICD_NSACR ... GICD_NSACRN:
+        /* We do not implement security extensions for guests, write ignore */
+        goto write_ignore;
+
+    case GICD_SGIR:
+        if ( dabt.size != 2 )
+            goto bad_width;
+        return vgic_to_sgi(v, *r);
+
+    case GICD_CPENDSGIR ... GICD_CPENDSGIRN:
+        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
+        printk("vGICD: unhandled %s write %#"PRIregister" to ICPENDSGIR%d\n",
+               dabt.size ? "word" : "byte", *r, gicd_reg - GICD_CPENDSGIR);
+        return 0;
+
+    case GICD_SPENDSGIR ... GICD_SPENDSGIRN:
+        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
+        printk("vGICD: unhandled %s write %#"PRIregister" to ISPENDSGIR%d\n",
+               dabt.size ? "word" : "byte", *r, gicd_reg - GICD_SPENDSGIR);
+        return 0;
+
+    /* Implementation defined -- write ignored */
+    case REG(0xfd0) ... REG(0xfe4):
+        goto write_ignore;
+
+    /* R/O -- write ignore */
+    case GICD_ICPIDR2:
+        goto write_ignore;
+
+    /* Implementation defined -- write ignored */
+    case REG(0xfec) ... REG(0xffc):
+        goto write_ignore;
+
+    /* Reserved -- write ignored */
+    case REG(0x00c) ... REG(0x01c):
+    case REG(0x040) ... REG(0x07c):
+    case REG(0x7fc):
+    case REG(0xbfc):
+    case REG(0xf04) ... REG(0xf0c):
+    case REG(0xf30) ... REG(0xfcc):
+        goto write_ignore;
+
+    default:
+        printk("vGICD: unhandled write r%d=%"PRIregister" offset %#08x\n",
+               dabt.reg, *r, offset);
+        return 0;
+    }
+
+bad_width:
+    printk("vGICD: bad write width %d r%d=%"PRIregister" offset %#08x\n",
+           dabt.size, dabt.reg, *r, offset);
+    domain_crash_synchronous();
+    return 0;
+
+write_ignore:
+    if ( dabt.size != 2 ) goto bad_width;
+    return 1;
+}
+
+static int vgic_distr_mmio_check(struct vcpu *v, paddr_t addr)
+{
+    struct domain *d = v->domain;
+
+    return (addr >= (d->arch.vgic.dbase)) && (addr < (d->arch.vgic.dbase + PAGE_SIZE));
+}
+
+static struct mmio_handler vgic_distr_mmio_handler = {
+    .check_handler = vgic_distr_mmio_check,
+    .read_handler  = vgic_distr_mmio_read,
+    .write_handler = vgic_distr_mmio_write,
+};
+
+int vgic_v2_init(struct domain *d)
+{
+    int i;
+    struct vgic_irq_rank *r;
+
+    d->arch.vgic.shared_irqs =
+        (struct vgic_irq_rank *)xzalloc_array(struct vgic_irq_rank, DOMAIN_NR_RANKS(d));
+
+    for (i=0; i<DOMAIN_NR_RANKS(d); i++) {
+        r = (struct vgic_irq_rank *)((unsigned char *)(d->arch.vgic.shared_irqs)
+                                      + sizeof(struct vgic_irq_rank) * i);
+        spin_lock_init(&r->lock);
+    }
+    register_vgic_ops(&ops);
+
+    register_mmio_handler(&vgic_distr_mmio_handler);
+    return 0;
+}
+
+/*
+ * Local variables:
+ * mode: C
+ * c-file-style: "BSD"
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
+
diff --git a/xen/arch/arm/vgic.c b/xen/arch/arm/vgic.c
index 9c907b5..b60f012 100644
--- a/xen/arch/arm/vgic.c
+++ b/xen/arch/arm/vgic.c
@@ -31,57 +31,53 @@
 #include <asm/gic_v2_defs.h>
 #include <asm/gic.h>
 
-#define REG(n) (n/4)
 
-/* Number of ranks of interrupt registers for a domain */
-#define DOMAIN_NR_RANKS(d) (((d)->arch.vgic.nr_lines+31)/32)
+static struct vgic_ops *vgic_ops;
 
-/* Represents state corresponding to a block of 32 interrupts */
-struct vgic_irq_rank {
-    spinlock_t lock; /* Covers access to all other members of this struct */
-    uint32_t ienable, iactive, ipend, pendsgi;
-    uint32_t icfg[2];
-    uint32_t ipriority[8];
-    uint32_t itargets[8];
-};
-
-/*
- * Rank containing GICD_<FOO><n> for GICD_<FOO> with
- * <b>-bits-per-interrupt
- */
-static inline int REG_RANK_NR(int b, uint32_t n)
+void register_vgic_ops(struct vgic_ops *ops)
 {
-    switch ( b )
-    {
-    case 8: return n >> 3;
-    case 4: return n >> 2;
-    case 2: return n >> 1;
-    case 1: return n;
-    default: BUG();
-    }
+   vgic_ops = ops;
 }
 
-/*
- * Offset of GICD_<FOO><n> with its rank, for GICD_<FOO> with
- * <b>-bits-per-interrupt.
- */
-#define REG_RANK_INDEX(b, n) ((n) & ((b)-1))
-
-/*
- * Returns rank corresponding to a GICD_<FOO><n> register for
- * GICD_<FOO> with <b>-bits-per-interrupt.
- */
-static struct vgic_irq_rank *vgic_irq_rank(struct vcpu *v, int b, int n)
+int domain_vgic_init(struct domain *d)
 {
-    int rank = REG_RANK_NR(b, n);
+    int i;
+
+    static const struct dt_device_match gicv2_ids[] __initconst =
+    {
+        DT_MATCH_GIC,
+        { /* sentinel */ },
+    };
+    struct dt_device_node *node;
+
+    d->arch.vgic.ctlr = 0;
 
-    if ( rank == 0 )
-        return (struct vgic_irq_rank *)v->arch.vgic.private_irqs;
-    else if ( rank <= DOMAIN_NR_RANKS(v->domain) )
-       return (struct vgic_irq_rank *)((unsigned char *)(v->domain->arch.vgic.shared_irqs)
-                           + (sizeof(struct vgic_irq_rank) *(rank - 1)));
+    /* Currently nr_lines in vgic and gic doesn't have the same meanings
+     * Here nr_lines = number of SPIs
+     */
+    if ( d->domain_id == 0 )
+        d->arch.vgic.nr_lines = gic_number_lines() - 32;
     else
-        return NULL;
+        d->arch.vgic.nr_lines = 0; /* We don't need SPIs for the guest */
+
+    node = dt_find_interrupt_controller(gicv2_ids);
+    if ( node )
+    {
+        if (vgic_v2_init(d))
+            panic("vgic v2 initialization failed");
+        goto out;
+    }
+    if ( !node )
+        panic("Unable to find compatible vGIC");
+out:
+    d->arch.vgic.pending_irqs =
+        xzalloc_array(struct pending_irq, d->arch.vgic.nr_lines);
+    for (i=0; i<d->arch.vgic.nr_lines; i++)
+    {
+        INIT_LIST_HEAD(&d->arch.vgic.pending_irqs[i].inflight);
+        INIT_LIST_HEAD(&d->arch.vgic.pending_irqs[i].lr_queue);
+    }
+    return 0;
 }
 
 void domain_vgic_free(struct domain *d)
@@ -93,11 +89,11 @@ void domain_vgic_free(struct domain *d)
 int vcpu_vgic_init(struct vcpu *v)
 {
     int i;
-    struct vgic_irq_rank *vir;
 
-    vir =  xzalloc(struct vgic_irq_rank);
-    memset(vir, 0, sizeof(struct vgic_irq_rank));
-    spin_lock_init(&vir->lock);
+    if (vgic_ops)
+       vgic_ops->vgic_vcpu_init(v);
+    else
+       panic("No VGIC ops found\n");
 
     memset(&v->arch.vgic.pending_irqs, 0, sizeof(v->arch.vgic.pending_irqs));
     for (i = 0; i < 32; i++)
@@ -106,16 +102,6 @@ int vcpu_vgic_init(struct vcpu *v)
         INIT_LIST_HEAD(&v->arch.vgic.pending_irqs[i].lr_queue);
     }
 
-    /* For SGI and PPI the target is always this CPU */
-    for ( i = 0 ; i < 8 ; i++ )
-        vir->itargets[i] =
-              (1<<(v->vcpu_id+0))
-            | (1<<(v->vcpu_id+8))
-            | (1<<(v->vcpu_id+16))
-            | (1<<(v->vcpu_id+24));
-
-    v->arch.vgic.private_irqs = (struct vgic_irq_irank *)vir;
-
     INIT_LIST_HEAD(&v->arch.vgic.inflight_irqs);
     INIT_LIST_HEAD(&v->arch.vgic.lr_pending);
     spin_lock_init(&v->arch.vgic.lock);
@@ -123,232 +109,7 @@ int vcpu_vgic_init(struct vcpu *v)
     return 0;
 }
 
-#define vgic_lock(v)   spin_lock_irq(&(v)->domain->arch.vgic.lock)
-#define vgic_unlock(v) spin_unlock_irq(&(v)->domain->arch.vgic.lock)
-
-#define vgic_lock_rank(v, r) spin_lock(&(r)->lock)
-#define vgic_unlock_rank(v, r) spin_unlock(&(r)->lock)
-
-static uint32_t byte_read(uint32_t val, int sign, int offset)
-{
-    int byte = offset & 0x3;
-
-    val = val >> (8*byte);
-    if ( sign && (val & 0x80) )
-        val |= 0xffffff00;
-    else
-        val &= 0x000000ff;
-    return val;
-}
-
-static void byte_write(uint32_t *reg, uint32_t var, int offset)
-{
-    int byte = offset & 0x3;
-
-    var &= (0xff << (8*byte));
-
-    *reg &= ~(0xff << (8*byte));
-    *reg |= var;
-}
-
-static int vgic_distr_mmio_read(struct vcpu *v, mmio_info_t *info)
-{
-    struct hsr_dabt dabt = info->dabt;
-    struct cpu_user_regs *regs = guest_cpu_user_regs();
-    register_t *r = select_user_reg(regs, dabt.reg);
-    struct vgic_irq_rank *rank;
-    int offset = (int)(info->gpa - v->domain->arch.vgic.dbase);
-    int gicd_reg = REG(offset);
-
-    switch ( gicd_reg )
-    {
-    case GICD_CTLR:
-        if ( dabt.size != 2 ) goto bad_width;
-        vgic_lock(v);
-        *r = v->domain->arch.vgic.ctlr;
-        vgic_unlock(v);
-        return 1;
-    case GICD_TYPER:
-        if ( dabt.size != 2 ) goto bad_width;
-        /* No secure world support for guests. */
-        vgic_lock(v);
-        *r = ( (v->domain->max_vcpus<<5) & GICD_TYPE_CPUS )
-            |( ((v->domain->arch.vgic.nr_lines/32)) & GICD_TYPE_LINES );
-        vgic_unlock(v);
-        return 1;
-    case GICD_IIDR:
-        if ( dabt.size != 2 ) goto bad_width;
-        /*
-         * XXX Do we need a JEP106 manufacturer ID?
-         * Just use the physical h/w value for now
-         */
-        *r = 0x0000043b;
-        return 1;
-
-    /* Implementation defined -- read as zero */
-    case REG(0x020) ... REG(0x03c):
-        goto read_as_zero;
-
-    case GICD_IGROUPR ... GICD_IGROUPRN:
-        /* We do not implement security extensions for guests, read zero */
-        goto read_as_zero;
-
-    case GICD_ISENABLER ... GICD_ISENABLERN:
-        if ( dabt.size != 2 ) goto bad_width;
-        rank = vgic_irq_rank(v, 1, gicd_reg - GICD_ISENABLER);
-        if ( rank == NULL) goto read_as_zero;
-        vgic_lock_rank(v, rank);
-        *r = rank->ienable;
-        vgic_unlock_rank(v, rank);
-        return 1;
-
-    case GICD_ICENABLER ... GICD_ICENABLERN:
-        if ( dabt.size != 2 ) goto bad_width;
-        rank = vgic_irq_rank(v, 1, gicd_reg - GICD_ICENABLER);
-        if ( rank == NULL) goto read_as_zero;
-        vgic_lock_rank(v, rank);
-        *r = rank->ienable;
-        vgic_unlock_rank(v, rank);
-        return 1;
-
-    case GICD_ISPENDR ... GICD_ISPENDRN:
-        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
-        rank = vgic_irq_rank(v, 1, gicd_reg - GICD_ISPENDR);
-        if ( rank == NULL) goto read_as_zero;
-        vgic_lock_rank(v, rank);
-        *r = byte_read(rank->ipend, dabt.sign, offset);
-        vgic_unlock_rank(v, rank);
-        return 1;
-
-    case GICD_ICPENDR ... GICD_ICPENDRN:
-        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
-        rank = vgic_irq_rank(v, 1, gicd_reg - GICD_ICPENDR);
-        if ( rank == NULL) goto read_as_zero;
-        vgic_lock_rank(v, rank);
-        *r = byte_read(rank->ipend, dabt.sign, offset);
-        vgic_unlock_rank(v, rank);
-        return 1;
-
-    case GICD_ISACTIVER ... GICD_ISACTIVERN:
-        if ( dabt.size != 2 ) goto bad_width;
-        rank = vgic_irq_rank(v, 1, gicd_reg - GICD_ISACTIVER);
-        if ( rank == NULL) goto read_as_zero;
-        vgic_lock_rank(v, rank);
-        *r = rank->iactive;
-        vgic_unlock_rank(v, rank);
-        return 1;
-
-    case GICD_ICACTIVER ... GICD_ICACTIVERN:
-        if ( dabt.size != 2 ) goto bad_width;
-        rank = vgic_irq_rank(v, 1, gicd_reg - GICD_ICACTIVER);
-        if ( rank == NULL) goto read_as_zero;
-        vgic_lock_rank(v, rank);
-        *r = rank->iactive;
-        vgic_unlock_rank(v, rank);
-        return 1;
-
-    case GICD_ITARGETSR ... GICD_ITARGETSRN:
-        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
-        rank = vgic_irq_rank(v, 8, gicd_reg - GICD_ITARGETSR);
-        if ( rank == NULL) goto read_as_zero;
-
-        vgic_lock_rank(v, rank);
-        *r = rank->itargets[REG_RANK_INDEX(8, gicd_reg - GICD_ITARGETSR)];
-        if ( dabt.size == 0 )
-            *r = byte_read(*r, dabt.sign, offset);
-        vgic_unlock_rank(v, rank);
-        return 1;
-
-    case GICD_IPRIORITYR ... GICD_IPRIORITYRN:
-        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
-        rank = vgic_irq_rank(v, 8, gicd_reg - GICD_IPRIORITYR);
-        if ( rank == NULL) goto read_as_zero;
-
-        vgic_lock_rank(v, rank);
-        *r = rank->ipriority[REG_RANK_INDEX(8, gicd_reg - GICD_IPRIORITYR)];
-        if ( dabt.size == 0 )
-            *r = byte_read(*r, dabt.sign, offset);
-        vgic_unlock_rank(v, rank);
-        return 1;
-
-    case GICD_ICFGR ... GICD_ICFGRN:
-        if ( dabt.size != 2 ) goto bad_width;
-        rank = vgic_irq_rank(v, 2, gicd_reg - GICD_ICFGR);
-        if ( rank == NULL) goto read_as_zero;
-        vgic_lock_rank(v, rank);
-        *r = rank->icfg[REG_RANK_INDEX(2, gicd_reg - GICD_ICFGR)];
-        vgic_unlock_rank(v, rank);
-        return 1;
-
-    case GICD_NSACR ... GICD_NSACRN:
-        /* We do not implement security extensions for guests, read zero */
-        goto read_as_zero;
-
-    case GICD_SGIR:
-        if ( dabt.size != 2 ) goto bad_width;
-        /* Write only -- read unknown */
-        *r = 0xdeadbeef;
-        return 1;
-
-    case GICD_CPENDSGIR ... GICD_CPENDSGIRN:
-        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
-        rank = vgic_irq_rank(v, 1, gicd_reg - GICD_CPENDSGIR);
-        if ( rank == NULL) goto read_as_zero;
-        vgic_lock_rank(v, rank);
-        *r = byte_read(rank->pendsgi, dabt.sign, offset);
-        vgic_unlock_rank(v, rank);
-        return 1;
-
-    case GICD_SPENDSGIR ... GICD_SPENDSGIRN:
-        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
-        rank = vgic_irq_rank(v, 1, gicd_reg - GICD_SPENDSGIR);
-        if ( rank == NULL) goto read_as_zero;
-        vgic_lock_rank(v, rank);
-        *r = byte_read(rank->pendsgi, dabt.sign, offset);
-        vgic_unlock_rank(v, rank);
-        return 1;
-
-    /* Implementation defined -- read as zero */
-    case REG(0xfd0) ... REG(0xfe4):
-        goto read_as_zero;
-
-    case GICD_ICPIDR2:
-        if ( dabt.size != 2 ) goto bad_width;
-        printk("vGICD: unhandled read from ICPIDR2\n");
-        return 0;
-
-    /* Implementation defined -- read as zero */
-    case REG(0xfec) ... REG(0xffc):
-        goto read_as_zero;
-
-    /* Reserved -- read as zero */
-    case REG(0x00c) ... REG(0x01c):
-    case REG(0x040) ... REG(0x07c):
-    case REG(0x7fc):
-    case REG(0xbfc):
-    case REG(0xf04) ... REG(0xf0c):
-    case REG(0xf30) ... REG(0xfcc):
-        goto read_as_zero;
-
-    default:
-        printk("vGICD: unhandled read r%d offset %#08x\n",
-               dabt.reg, offset);
-        return 0;
-    }
-
-bad_width:
-    printk("vGICD: bad read width %d r%d offset %#08x\n",
-           dabt.size, dabt.reg, offset);
-    domain_crash_synchronous();
-    return 0;
-
-read_as_zero:
-    if ( dabt.size != 2 ) goto bad_width;
-    *r = 0;
-    return 1;
-}
-
-static void vgic_disable_irqs(struct vcpu *v, uint32_t r, int n)
+void vgic_disable_irqs(struct vcpu *v, uint32_t r, int n)
 {
     const unsigned long mask = r;
     struct pending_irq *p;
@@ -366,7 +127,7 @@ static void vgic_disable_irqs(struct vcpu *v, uint32_t r, int n)
     }
 }
 
-static void vgic_enable_irqs(struct vcpu *v, uint32_t r, int n)
+void vgic_enable_irqs(struct vcpu *v, uint32_t r, int n)
 {
     const unsigned long mask = r;
     struct pending_irq *p;
@@ -382,310 +143,9 @@ static void vgic_enable_irqs(struct vcpu *v, uint32_t r, int n)
         if ( p->desc != NULL )
             p->desc->handler->enable(p->desc);
         i++;
-    }
+   }
 }
 
-static inline int is_vcpu_running(struct domain *d, int vcpuid)
-{
-    struct vcpu *v;
-
-    if ( vcpuid >= d->max_vcpus )
-        return 0;
-
-    v = d->vcpu[vcpuid];
-    if ( v == NULL )
-        return 0;
-    if (test_bit(_VPF_down, &v->pause_flags) )
-        return 0;
-
-    return 1;
-}
-
-static int vgic_to_sgi(struct vcpu *v, register_t sgir)
-{
-    struct domain *d = v->domain;
-    int virtual_irq;
-    int filter;
-    int vcpuid;
-    int i;
-    unsigned long vcpu_mask = 0;
-
-    ASSERT(d->max_vcpus < 8*sizeof(vcpu_mask));
-
-    filter = (sgir & GICD_SGI_TARGET_LIST_MASK);
-    virtual_irq = (sgir & GICD_SGI_INTID_MASK);
-    ASSERT( virtual_irq < 16 );
-
-    switch ( filter )
-    {
-        case GICD_SGI_TARGET_LIST:
-            vcpu_mask = (sgir & GICD_SGI_TARGET_MASK) >> GICD_SGI_TARGET_SHIFT;
-            break;
-        case GICD_SGI_TARGET_OTHERS:
-            for ( i = 0; i < d->max_vcpus; i++ )
-            {
-                if ( i != current->vcpu_id && is_vcpu_running(d, i) )
-                    set_bit(i, &vcpu_mask);
-            }
-            break;
-        case GICD_SGI_TARGET_SELF:
-            set_bit(current->vcpu_id, &vcpu_mask);
-            break;
-        default:
-            gdprintk(XENLOG_WARNING, "vGICD: unhandled GICD_SGIR write %"PRIregister" with wrong TargetListFilter field\n",
-                     sgir);
-            return 0;
-    }
-
-    for_each_set_bit( vcpuid, &vcpu_mask, d->max_vcpus )
-    {
-        if ( !is_vcpu_running(d, vcpuid) )
-        {
-            gdprintk(XENLOG_WARNING, "vGICD: GICD_SGIR write r=%"PRIregister" vcpu_mask=%lx, wrong CPUTargetList\n",
-                     sgir, vcpu_mask);
-            continue;
-        }
-        vgic_vcpu_inject_irq(d->vcpu[vcpuid], virtual_irq, 1);
-    }
-    return 1;
-}
-
-static int vgic_distr_mmio_write(struct vcpu *v, mmio_info_t *info)
-{
-    struct hsr_dabt dabt = info->dabt;
-    struct cpu_user_regs *regs = guest_cpu_user_regs();
-    register_t *r = select_user_reg(regs, dabt.reg);
-    struct vgic_irq_rank *rank;
-    int offset = (int)(info->gpa - v->domain->arch.vgic.dbase);
-    int gicd_reg = REG(offset);
-    uint32_t tr;
-
-    switch ( gicd_reg )
-    {
-    case GICD_CTLR:
-        if ( dabt.size != 2 ) goto bad_width;
-        /* Ignore all but the enable bit */
-        v->domain->arch.vgic.ctlr = (*r) & GICD_CTL_ENABLE;
-        return 1;
-
-    /* R/O -- write ignored */
-    case GICD_TYPER:
-    case GICD_IIDR:
-        goto write_ignore;
-
-    /* Implementation defined -- write ignored */
-    case REG(0x020) ... REG(0x03c):
-        goto write_ignore;
-
-    case GICD_IGROUPR ... GICD_IGROUPRN:
-        /* We do not implement security extensions for guests, write ignore */
-        goto write_ignore;
-
-    case GICD_ISENABLER ... GICD_ISENABLERN:
-        if ( dabt.size != 2 ) goto bad_width;
-        rank = vgic_irq_rank(v, 1, gicd_reg - GICD_ISENABLER);
-        if ( rank == NULL) goto write_ignore;
-        vgic_lock_rank(v, rank);
-        tr = rank->ienable;
-        rank->ienable |= *r;
-        vgic_unlock_rank(v, rank);
-        vgic_enable_irqs(v, (*r) & (~tr), gicd_reg - GICD_ISENABLER);
-        return 1;
-
-    case GICD_ICENABLER ... GICD_ICENABLERN:
-        if ( dabt.size != 2 ) goto bad_width;
-        rank = vgic_irq_rank(v, 1, gicd_reg - GICD_ICENABLER);
-        if ( rank == NULL) goto write_ignore;
-        vgic_lock_rank(v, rank);
-        tr = rank->ienable;
-        rank->ienable &= ~*r;
-        vgic_unlock_rank(v, rank);
-        vgic_disable_irqs(v, (*r) & tr, gicd_reg - GICD_ICENABLER);
-        return 1;
-
-    case GICD_ISPENDR ... GICD_ISPENDRN:
-        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
-        printk("vGICD: unhandled %s write %#"PRIregister" to ISPENDR%d\n",
-               dabt.size ? "word" : "byte", *r, gicd_reg - GICD_ISPENDR);
-        return 0;
-
-    case GICD_ICPENDR ... GICD_ICPENDRN:
-        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
-        printk("vGICD: unhandled %s write %#"PRIregister" to ICPENDR%d\n",
-               dabt.size ? "word" : "byte", *r, gicd_reg - GICD_ICPENDR);
-        return 0;
-
-    case GICD_ISACTIVER ... GICD_ISACTIVERN:
-        if ( dabt.size != 2 ) goto bad_width;
-        rank = vgic_irq_rank(v, 1, gicd_reg - GICD_ISACTIVER);
-        if ( rank == NULL) goto write_ignore;
-        vgic_lock_rank(v, rank);
-        rank->iactive &= ~*r;
-        vgic_unlock_rank(v, rank);
-        return 1;
-
-    case GICD_ICACTIVER ... GICD_ICACTIVERN:
-        if ( dabt.size != 2 ) goto bad_width;
-        rank = vgic_irq_rank(v, 1, gicd_reg - GICD_ICACTIVER);
-        if ( rank == NULL) goto write_ignore;
-        vgic_lock_rank(v, rank);
-        rank->iactive &= ~*r;
-        vgic_unlock_rank(v, rank);
-        return 1;
-
-    case GICD_ITARGETSR ... GICD_ITARGETSR + 7:
-        /* SGI/PPI target is read only */
-        goto write_ignore;
-
-    case GICD_ITARGETSR + 8 ... GICD_ITARGETSRN:
-        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
-        rank = vgic_irq_rank(v, 8, gicd_reg - GICD_ITARGETSR);
-        if ( rank == NULL) goto write_ignore;
-        vgic_lock_rank(v, rank);
-        if ( dabt.size == 2 )
-            rank->itargets[REG_RANK_INDEX(8, gicd_reg - GICD_ITARGETSR)] = *r;
-        else
-            byte_write(&rank->itargets[REG_RANK_INDEX(8, gicd_reg - GICD_ITARGETSR)],
-                       *r, offset);
-        vgic_unlock_rank(v, rank);
-        return 1;
-
-    case GICD_IPRIORITYR ... GICD_IPRIORITYRN:
-        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
-        rank = vgic_irq_rank(v, 8, gicd_reg - GICD_IPRIORITYR);
-        if ( rank == NULL) goto write_ignore;
-        vgic_lock_rank(v, rank);
-        if ( dabt.size == 2 )
-            rank->ipriority[REG_RANK_INDEX(8, gicd_reg - GICD_IPRIORITYR)] = *r;
-        else
-            byte_write(&rank->ipriority[REG_RANK_INDEX(8, gicd_reg - GICD_IPRIORITYR)],
-                       *r, offset);
-        vgic_unlock_rank(v, rank);
-        return 1;
-
-    case GICD_ICFGR: /* SGIs */
-        goto write_ignore;
-    case GICD_ICFGR + 1: /* PPIs */
-        /* It is implementation defined if these are writeable. We chose not */
-        goto write_ignore;
-    case GICD_ICFGR + 2 ... GICD_ICFGRN: /* SPIs */
-        if ( dabt.size != 2 ) goto bad_width;
-        rank = vgic_irq_rank(v, 2, gicd_reg - GICD_ICFGR);
-        vgic_lock_rank(v, rank);
-        if ( rank == NULL) goto write_ignore;
-        rank->icfg[REG_RANK_INDEX(2, gicd_reg - GICD_ICFGR)] = *r;
-        vgic_unlock_rank(v, rank);
-        return 1;
-
-    case GICD_NSACR ... GICD_NSACRN:
-        /* We do not implement security extensions for guests, write ignore */
-        goto write_ignore;
-
-    case GICD_SGIR:
-        if ( dabt.size != 2 )
-            goto bad_width;
-        return vgic_to_sgi(v, *r);
-
-    case GICD_CPENDSGIR ... GICD_CPENDSGIRN:
-        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
-        printk("vGICD: unhandled %s write %#"PRIregister" to ICPENDSGIR%d\n",
-               dabt.size ? "word" : "byte", *r, gicd_reg - GICD_CPENDSGIR);
-        return 0;
-
-    case GICD_SPENDSGIR ... GICD_SPENDSGIRN:
-        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
-        printk("vGICD: unhandled %s write %#"PRIregister" to ISPENDSGIR%d\n",
-               dabt.size ? "word" : "byte", *r, gicd_reg - GICD_SPENDSGIR);
-        return 0;
-
-    /* Implementation defined -- write ignored */
-    case REG(0xfd0) ... REG(0xfe4):
-        goto write_ignore;
-
-    /* R/O -- write ignore */
-    case GICD_ICPIDR2:
-        goto write_ignore;
-
-    /* Implementation defined -- write ignored */
-    case REG(0xfec) ... REG(0xffc):
-        goto write_ignore;
-
-    /* Reserved -- write ignored */
-    case REG(0x00c) ... REG(0x01c):
-    case REG(0x040) ... REG(0x07c):
-    case REG(0x7fc):
-    case REG(0xbfc):
-    case REG(0xf04) ... REG(0xf0c):
-    case REG(0xf30) ... REG(0xfcc):
-        goto write_ignore;
-
-    default:
-        printk("vGICD: unhandled write r%d=%"PRIregister" offset %#08x\n",
-               dabt.reg, *r, offset);
-        return 0;
-    }
-
-bad_width:
-    printk("vGICD: bad write width %d r%d=%"PRIregister" offset %#08x\n",
-           dabt.size, dabt.reg, *r, offset);
-    domain_crash_synchronous();
-    return 0;
-
-write_ignore:
-    if ( dabt.size != 2 ) goto bad_width;
-    return 1;
-}
-
-static int vgic_distr_mmio_check(struct vcpu *v, paddr_t addr)
-{
-    struct domain *d = v->domain;
-
-    return (addr >= (d->arch.vgic.dbase)) && (addr < (d->arch.vgic.dbase + PAGE_SIZE));
-}
-
-static struct mmio_handler vgic_distr_mmio_handler = {
-    .check_handler = vgic_distr_mmio_check,
-    .read_handler  = vgic_distr_mmio_read,
-    .write_handler = vgic_distr_mmio_write,
-};
-
-int domain_vgic_init(struct domain *d)
-{
-    int i;
-    struct vgic_irq_rank *r;
-
-    d->arch.vgic.ctlr = 0;
-
-    /* Currently nr_lines in vgic and gic doesn't have the same meanings
-     * Here nr_lines = number of SPIs
-     */
-    if ( d->domain_id == 0 )
-        d->arch.vgic.nr_lines = gic_number_lines() - 32;
-    else
-        d->arch.vgic.nr_lines = 0; /* We don't need SPIs for the guest */
-
-    d->arch.vgic.shared_irqs =
-        (struct vgic_irq_rank *)xzalloc_array(struct vgic_irq_rank, DOMAIN_NR_RANKS(d));
-
-    d->arch.vgic.pending_irqs =
-        xzalloc_array(struct pending_irq, d->arch.vgic.nr_lines);
-    for (i=0; i<d->arch.vgic.nr_lines; i++)
-    {
-        INIT_LIST_HEAD(&d->arch.vgic.pending_irqs[i].inflight);
-        INIT_LIST_HEAD(&d->arch.vgic.pending_irqs[i].lr_queue);
-    }
-    for (i=0; i<DOMAIN_NR_RANKS(d); i++)
-    {
-        r = (struct vgic_irq_rank *)((unsigned char *)(d->arch.vgic.shared_irqs)
-                                      + sizeof(struct vgic_irq_rank) * i);
-        spin_lock_init(&r->lock);
-    }
-
-    register_mmio_handler(&vgic_distr_mmio_handler);
-    return 0;
-}
-
-
 struct pending_irq *irq_to_pending(struct vcpu *v, unsigned int irq)
 {
     struct pending_irq *n;
@@ -712,15 +172,13 @@ void vgic_clear_pending_irqs(struct vcpu *v)
 
 void vgic_vcpu_inject_irq(struct vcpu *v, unsigned int irq, int virtual)
 {
-    int idx = irq >> 2, byte = irq & 0x3;
     uint8_t priority;
-    struct vgic_irq_rank *rank = vgic_irq_rank(v, 8, idx);
-    struct pending_irq *iter, *n = irq_to_pending(v, irq);
+    struct pending_irq *iter;
+    struct pending_irq  *n = irq_to_pending(v, irq);
     unsigned long flags;
     bool_t running;
 
     spin_lock_irqsave(&v->arch.vgic.lock, flags);
-
     if ( !list_empty(&n->inflight) )
     {
         if ( (irq != current->domain->arch.evtchn_irq) ||
@@ -737,7 +195,7 @@ void vgic_vcpu_inject_irq(struct vcpu *v, unsigned int irq, int virtual)
         return;
     }
 
-    priority = byte_read(rank->ipriority[REG_RANK_INDEX(8, idx)], 0, byte);
+    priority = vgic_ops->read_priority(v, irq);
 
     n->irq = irq;
     set_bit(GIC_IRQ_GUEST_PENDING, &n->status);
diff --git a/xen/include/asm-arm/gic.h b/xen/include/asm-arm/gic.h
index 4244491..2de6c6a 100644
--- a/xen/include/asm-arm/gic.h
+++ b/xen/include/asm-arm/gic.h
@@ -54,6 +54,9 @@ extern void domain_vgic_free(struct domain *d);
 
 extern int vcpu_vgic_init(struct vcpu *v);
 extern int vcpu_gic_init(struct vcpu *v);
+extern void vgic_disable_irqs(struct vcpu *v, uint32_t r, int n);
+extern void vgic_enable_irqs(struct vcpu *v, uint32_t r, int n);
+extern int vgic_v2_init(struct domain *d);
 
 extern void gicv2_init(void);
 
@@ -130,7 +133,13 @@ struct gic_hw_operations {
     unsigned long (*read_cpu_sgi_rbase)(void);
 };
 
+struct vgic_ops {
+    int (*vgic_vcpu_init)(struct vcpu *v);
+    int (*read_priority)(struct vcpu *v, int irq);
+};
+
 extern void register_gic_ops(struct gic_hw_operations *);
+extern void register_vgic_ops(struct vgic_ops *ops);
 
 extern void update_cpu_lr_mask(void);
 extern void send_SGI_mask(const cpumask_t *cpumask, enum gic_sgi sgi);
-- 
1.7.9.5

[RFC PATCH v1 08/10] xen/arm: Add support for GIC v3

[vijay.kilari]

From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>

Add support for GIC v3 specification.
This driver assumes that ARE and SRE
is enable by default.

Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
---
 xen/arch/arm/Makefile             |    2 +-
 xen/arch/arm/gic-v3.c             |  944 +++++++++++++++++++++++++++++++++++++
 xen/arch/arm/gic.c                |   12 +
 xen/include/asm-arm/domain.h      |    4 +
 xen/include/asm-arm/gic.h         |    9 +
 xen/include/asm-arm/gic_v3_defs.h |  211 +++++++++
 6 files changed, 1181 insertions(+), 1 deletion(-)

diff --git a/xen/arch/arm/Makefile b/xen/arch/arm/Makefile
index 20f59f4..a11c699 100644
--- a/xen/arch/arm/Makefile
+++ b/xen/arch/arm/Makefile
@@ -10,7 +10,7 @@ obj-y += vpsci.o
 obj-y += domctl.o
 obj-y += sysctl.o
 obj-y += domain_build.o
-obj-y += gic.o gic-v2.o
+obj-y += gic.o gic-v2.o gic-v3.o
 obj-y += io.o
 obj-y += irq.o
 obj-y += kernel.o
diff --git a/xen/arch/arm/gic-v3.c b/xen/arch/arm/gic-v3.c
new file mode 100644
index 0000000..81a36ba
--- /dev/null
+++ b/xen/arch/arm/gic-v3.c
@@ -0,0 +1,944 @@
+/*
+ * xen/arch/arm/gic-v3.c
+ *
+ * ARM Generic Interrupt Controller support v3 version
+ * based on xen/arch/arm/gic-v2.c
+ * 
+ * Vijaya Kumar K <vijaya.kumar@caviumnetworks.com>
+ * Copyright (c) 2014 Cavium Inc.
+ * 
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <xen/config.h>
+#include <xen/lib.h>
+#include <xen/init.h>
+#include <xen/cpu.h>
+#include <xen/mm.h>
+#include <xen/irq.h>
+#include <xen/sched.h>
+#include <xen/errno.h>
+#include <xen/serial.h>
+#include <xen/softirq.h>
+#include <xen/list.h>
+#include <xen/device_tree.h>
+#include <asm/p2m.h>
+#include <asm/domain.h>
+#include <asm/platform.h>
+
+#include <asm/gic_v3_defs.h>
+#include <asm/gic.h>
+#include <asm/io.h>
+
+struct rdist_region {
+    paddr_t rdist_base;
+    paddr_t rdist_base_size;
+    void __iomem *map_rdist_base;
+};
+	
+/* Global state */
+static struct {
+    paddr_t dbase;            /* Address of distributor registers */
+    paddr_t dbase_size;
+    void __iomem *map_dbase;  /* Mapped address of distributor registers */
+    struct rdist_region *rdist_regions;
+    u32  rdist_stride;
+    unsigned int rdist_count; /* Number of rdist regions count */
+    unsigned int lines;       /* Number of interrupts (SPIs + PPIs + SGIs) */
+    struct dt_irq maintenance;
+    unsigned int cpus;
+    int hw_version;
+    spinlock_t lock;
+} gic;
+
+struct gic_state_data {
+    uint32_t gic_hcr, gic_vmcr;
+    uint32_t gic_apr0[4];
+    uint32_t gic_apr1[4];
+    uint64_t gic_lr[16];
+};
+
+#define GICD ((volatile unsigned char *) gic.map_dbase)
+/* Only one region is implemented which is enough for 0-31 cpus */
+#define GICR ((volatile unsigned char *) gic.rdist_regions[0].map_rdist_base)
+
+/* per-cpu re-distributor base */
+static DEFINE_PER_CPU(paddr_t, rbase);
+static DEFINE_PER_CPU(paddr_t, phy_rbase);
+
+static unsigned nr_lrs;
+static uint32_t nr_priorities;
+
+/* The GIC mapping of CPU interfaces does not necessarily match the
+ * logical CPU numbering. Let's use mapping as returned by the GIC
+ * itself
+ */
+
+#define gic_data_rdist_rd_base()        (this_cpu(rbase))
+#define gic_data_rdist_sgi_base()       (gic_data_rdist_rd_base() + SZ_64K)
+
+static inline u64 read_cpuid_mpidr(void)
+{
+   return READ_SYSREG(MPIDR_EL1);
+}
+
+static u64 gich_read_lr(int lr)
+{
+    switch (lr) 
+    {
+        case 0: /* ICH_LRn is 64 bit */
+            return READ_SYSREG(ICH_LR0_EL2);
+            break;
+        case 1:
+            return READ_SYSREG(ICH_LR1_EL2);
+            break;
+        case 2:
+            return READ_SYSREG(ICH_LR2_EL2);
+            break;
+        case 3:
+            return READ_SYSREG(ICH_LR3_EL2);
+            break;
+        case 4:
+            return READ_SYSREG(ICH_LR4_EL2);
+            break;
+        case 5:
+            return READ_SYSREG(ICH_LR5_EL2);
+            break;
+        case 6:
+            return READ_SYSREG(ICH_LR6_EL2);
+            break;
+        case 7:
+            return READ_SYSREG(ICH_LR7_EL2);
+            break;
+        case 8:
+            return READ_SYSREG(ICH_LR8_EL2);
+            break;
+        case 9:
+            return READ_SYSREG(ICH_LR9_EL2);
+            break;
+        case 10:
+            return READ_SYSREG(ICH_LR10_EL2);
+            break;
+        case 11:
+            return READ_SYSREG(ICH_LR11_EL2);
+            break;
+        case 12:
+            return READ_SYSREG(ICH_LR12_EL2);
+            break;
+        case 13:
+            return READ_SYSREG(ICH_LR13_EL2);
+            break;
+        case 14:
+            return READ_SYSREG(ICH_LR14_EL2);
+            break;
+        case 15:
+            return READ_SYSREG(ICH_LR15_EL2);
+            break;
+        default:
+            return 0;
+    }
+}
+
+static void gich_write_lr(int lr, u64 val)
+{
+    switch (lr) 
+    {
+        case 0:
+           WRITE_SYSREG(val, ICH_LR0_EL2);
+           break;
+        case 1:
+           WRITE_SYSREG(val, ICH_LR1_EL2);
+           break;
+        case 2:
+           WRITE_SYSREG(val, ICH_LR2_EL2);
+           break;
+        case 3:
+           WRITE_SYSREG(val, ICH_LR3_EL2);
+           break;
+        case 4:
+           WRITE_SYSREG(val, ICH_LR4_EL2);
+           break;
+        case 5:
+           WRITE_SYSREG(val, ICH_LR5_EL2);
+           break;
+        case 6:
+           WRITE_SYSREG(val, ICH_LR6_EL2);
+           break;
+        case 7:
+           WRITE_SYSREG(val, ICH_LR7_EL2);
+           break;
+        case 8:
+           WRITE_SYSREG(val, ICH_LR8_EL2);
+           break;
+        case 9:
+           WRITE_SYSREG(val, ICH_LR9_EL2);
+           break;
+        case 10:
+           WRITE_SYSREG(val, ICH_LR10_EL2);
+           break;
+        case 11:
+           WRITE_SYSREG(val, ICH_LR11_EL2);
+           break;
+        case 12:
+           WRITE_SYSREG(val, ICH_LR12_EL2);
+           break;
+        case 13:
+           WRITE_SYSREG(val, ICH_LR13_EL2);
+           break;
+        case 14:
+           WRITE_SYSREG(val, ICH_LR14_EL2);
+           break;
+        case 15:
+           WRITE_SYSREG(val, ICH_LR15_EL2);
+           break;
+        default:
+           return;
+    }
+}
+
+static void gic_enable_sre(void)
+{
+    uint32_t val;
+
+    val = READ_SYSREG32(ICC_SRE_EL2);
+    val |= GICC_SRE_EL2_SRE | GICC_SRE_EL2_ENEL1 | GICC_SRE_EL2_DFB | GICC_SRE_EL2_DIB;
+    WRITE_SYSREG32(val, ICC_SRE_EL2);
+    isb();
+}
+
+/* Wait for completion of a distributor change */
+static void gic_do_wait_for_rwp(paddr_t base)
+{
+    u32 val;
+    do {
+        val = readl_relaxed((void *)base + GICD_CTLR);
+        val = readl_relaxed(GICD + GICD_CTLR);
+        val = GICD[GICD_CTLR];
+        cpu_relax();
+    } while (val & GICD_CTLR_RWP);
+}
+
+static void gic_dist_wait_for_rwp(void)
+{
+    gic_do_wait_for_rwp((paddr_t)GICD);
+}
+
+static void gic_redist_wait_for_rwp(void)
+{
+    gic_do_wait_for_rwp(gic_data_rdist_rd_base());
+}
+
+static void gic_wait_for_rwp(int irq)
+{
+    if (irq < 32)
+         gic_redist_wait_for_rwp();
+    else
+         gic_dist_wait_for_rwp();
+}
+
+static unsigned int gic_mask_cpu(const cpumask_t *cpumask)
+{
+    unsigned int cpu;
+    cpumask_t possible_mask;
+
+    cpumask_and(&possible_mask, cpumask, &cpu_possible_map);
+    cpu = cpumask_any(&possible_mask);
+    return cpu;
+}
+
+static unsigned int gic_nr_lines(void)
+{
+    return gic.lines;
+}
+
+static unsigned int gic_nr_lrs(void)
+{
+    return nr_lrs;
+}
+
+static void write_aprn_regs(struct gic_state_data *d)
+{
+    switch(nr_priorities)
+    {
+        case 7:
+            WRITE_SYSREG32(d->gic_apr0[2], ICH_AP0R2_EL2);
+            WRITE_SYSREG32(d->gic_apr1[2], ICH_AP1R2_EL2);
+        case 6:
+            WRITE_SYSREG32(d->gic_apr0[1], ICH_AP0R1_EL2);
+            WRITE_SYSREG32(d->gic_apr1[1], ICH_AP1R1_EL2);
+        case 5:
+            WRITE_SYSREG32(d->gic_apr0[0], ICH_AP0R0_EL2);
+            WRITE_SYSREG32(d->gic_apr1[0], ICH_AP1R0_EL2);
+            break;
+        default:
+          panic("Write Undefined active priorities \n");
+    }
+}
+
+static void read_aprn_regs(struct gic_state_data *d)
+{
+    switch(nr_priorities)
+    {
+        case 7:
+            d->gic_apr0[2] = READ_SYSREG32(ICH_AP0R2_EL2);
+            d->gic_apr1[2] = READ_SYSREG32(ICH_AP1R2_EL2);
+        case 6:
+            d->gic_apr0[1] = READ_SYSREG32(ICH_AP0R1_EL2);
+            d->gic_apr1[1] = READ_SYSREG32(ICH_AP1R1_EL2);
+        case 5:
+            d->gic_apr0[0] = READ_SYSREG32(ICH_AP0R0_EL2);
+            d->gic_apr1[0] = READ_SYSREG32(ICH_AP1R0_EL2);
+            break;
+        default:
+          panic("Read Undefined active priorities \n");
+    }
+}
+
+static int gic_state_init(struct vcpu *v)
+{
+     v->arch.gic_state = (struct gic_state_data *)xzalloc(struct gic_state_data);
+     if(!v->arch.gic_state)
+        return -ENOMEM;
+     return 0; 
+}
+
+static void save_state(struct vcpu *v)
+{
+    int i;
+    struct gic_state_data *d;
+    d = (struct gic_state_data *)v->arch.gic_state;
+
+    /* No need for spinlocks here because interrupts are disabled around
+     * this call and it only accesses struct vcpu fields that cannot be
+     * accessed simultaneously by another pCPU.
+     */
+    for ( i=0; i<nr_lrs; i++)
+        d->gic_lr[i] = gich_read_lr(i);
+
+    read_aprn_regs(d); 
+
+    d->gic_vmcr = READ_SYSREG32(ICH_VMCR_EL2);
+}
+
+static void restore_state(struct vcpu *v)
+{
+    int i;
+    struct gic_state_data *d;
+    d = (struct gic_state_data *)v->arch.gic_state;
+
+    for ( i=0; i<nr_lrs; i++)
+        gich_write_lr(i, d->gic_lr[i]);
+
+    write_aprn_regs(d);
+
+    WRITE_SYSREG32(d->gic_vmcr, ICH_VMCR_EL2);
+}
+
+static void gic_dump_state(struct vcpu *v)
+{
+    int i;
+    struct gic_state_data *d;
+    d = (struct gic_state_data *)v->arch.gic_state;
+    if ( v == current )
+    {
+        for ( i = 0; i < nr_lrs; i++ )
+            printk("   HW_LR[%d]=%lx\n", i, gich_read_lr(i));
+    }
+    else
+    {
+        for ( i = 0; i < nr_lrs; i++ )
+            printk("   VCPU_LR[%d]=%lx\n", i, d->gic_lr[i]);
+    }
+}
+ 
+static void gic_enable_irq(int irq)
+{
+    uint32_t enabler;
+
+    /* Enable routing */
+    if(irq > 31)
+    {
+        enabler = readl_relaxed(GICD + GICD_ISENABLER + (irq / 32) * 4);
+        writel_relaxed(enabler | (1u << (irq % 32)), GICD + GICD_ISENABLER + (irq / 32) * 4);
+    }
+    else
+    {
+        enabler = readl_relaxed((void *)gic_data_rdist_sgi_base() + GICR_ISENABLER0);
+        writel_relaxed(enabler | (1u << irq), (void *)gic_data_rdist_sgi_base() + GICR_ISENABLER0);
+    }
+    gic_wait_for_rwp(irq);
+}
+
+static void gic_disable_irq(int irq)
+{
+    /* Disable routing */
+    if(irq > 31)
+        writel_relaxed(1u << (irq % 32), GICD + GICD_ICENABLER + ((irq / 32) * 4));
+    else
+        writel_relaxed(1u << irq, (void *)gic_data_rdist_sgi_base() + GICR_ICENABLER0);
+}
+
+static void gic_eoi_irq(int irq)
+{
+    /* Lower the priority */
+    WRITE_SYSREG32(irq, ICC_EOIR1_EL1);
+}
+
+static void gic_dir_irq(int irq)
+{
+    /* Deactivate */
+    WRITE_SYSREG32(irq, ICC_DIR_EL1);
+}
+
+static unsigned int gic_ack_irq(void)
+{
+    return (READ_SYSREG32(ICC_IAR1_EL1) & GICC_IA_IRQ);
+}
+
+static u64 gic_mpidr_to_affinity(u64 mpidr)
+{
+    /* Make sure we don't broadcast the interrupt */
+    return mpidr & ~GICD_IROUTER_SPI_MODE_ANY;
+}
+
+/*
+ * - needs to be called with gic.lock held
+ * - needs to be called with a valid cpu_mask, ie each cpu in the mask has
+ * already called gic_cpu_init
+ */
+static void gic_set_irq_property(unsigned int irq, bool_t level,
+                                   const cpumask_t *cpu_mask,
+                                   unsigned int priority)
+{
+    uint32_t cfg, edgebit;
+    u64 affinity;
+    unsigned int cpu = gic_mask_cpu(cpu_mask);
+    paddr_t rebase;
+
+
+    /* Set edge / level */
+    if (irq < 16)
+	/* SGI's are always edge-triggered not need to call GICD_ICFGR0 */
+       cfg = readl_relaxed((void *)gic_data_rdist_sgi_base() + GICR_ICFGR0);
+    else if (irq < 32)
+       cfg = readl_relaxed((void *)gic_data_rdist_sgi_base() + GICR_ICFGR1);
+    else
+       cfg = readl_relaxed(GICD + GICD_ICFGR + (irq / 16) * 4);
+
+    edgebit = 2u << (2 * (irq % 16));
+    if ( level )
+        cfg &= ~edgebit;
+    else
+        cfg |= edgebit;
+
+    if (irq < 16)
+       writel_relaxed(cfg, (void *)gic_data_rdist_sgi_base() + GICR_ICFGR0);
+    else if (irq < 32)
+       writel_relaxed(cfg, (void *)gic_data_rdist_sgi_base() + GICR_ICFGR1);
+    else
+       writel_relaxed(cfg, GICD + GICD_ICFGR + (irq / 16) * 4);
+
+
+    /* need to check if ARE is set to access IROUTER */
+    affinity = gic_mpidr_to_affinity(cpu_logical_map(cpu));
+    if (irq > 31)
+        writeq_relaxed(affinity, (GICD + GICD_IROUTER + irq * 8));
+
+    /* Set priority */
+    if (irq < 32)
+    {
+	rebase = gic_data_rdist_sgi_base();
+        writeb_relaxed(priority, (void *)rebase + GICR_IPRIORITYR0 + irq);
+    }
+    else 
+    {
+        writeb_relaxed(priority, GICD + GICD_IPRIORITYR + irq);
+
+    }
+}
+
+static void __init gic_dist_init(void)
+{
+    uint32_t type;
+    u64 affinity;
+    int i;
+
+    /* Disable the distributor */
+    writel_relaxed(0, GICD + GICD_CTLR);
+
+    type = readl_relaxed(GICD + GICD_TYPER);
+    gic.lines = 32 * ((type & GICD_TYPE_LINES) + 1);
+
+    printk("GIC: %d lines, (IID %8.8x).\n",
+           gic.lines, readl_relaxed(GICD + GICD_IIDR));
+
+    /* Default all global IRQs to level, active low */
+    for ( i = 32; i < gic.lines; i += 16 )
+        writel_relaxed(0, GICD + GICD_ICFGR + (i / 16) * 4);
+
+    /* Default priority for global interrupts */
+    for ( i = 32; i < gic.lines; i += 4 )
+        writel_relaxed((GIC_PRI_IRQ<<24 | GIC_PRI_IRQ<<16 | GIC_PRI_IRQ<<8 | GIC_PRI_IRQ), GICD + GICD_IPRIORITYR + (i / 4) * 4);
+
+    /* Disable all global interrupts */
+    for ( i = 32; i < gic.lines; i += 32 )
+        writel_relaxed(0xffffffff, GICD + GICD_ICENABLER + (i / 32) * 4);
+
+    gic_dist_wait_for_rwp();
+
+    /* Turn on the distributor */
+    writel_relaxed(GICD_CTL_ENABLE | GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1, GICD + GICD_CTLR);
+ 
+    /* Route all global IRQs to this CPU */
+    affinity = gic_mpidr_to_affinity(read_cpuid_mpidr());
+    for ( i = 31; i < gic.lines; i++ )
+        writeq_relaxed(affinity, GICD + GICD_IROUTER + i * 8);
+}
+
+static void gic_enable_redist(void)
+{
+    paddr_t rbase;
+    u32 val;
+
+    rbase = this_cpu(rbase);
+
+    /* Wake up this CPU redistributor */
+    val = readl_relaxed((void *)rbase + GICR_WAKER);
+    val &= ~GICR_WAKER_ProcessorSleep;
+    writel_relaxed(val, (void *)rbase + GICR_WAKER);
+
+    do {
+         val = readl_relaxed((void *)rbase + GICR_WAKER);
+         cpu_relax();
+    } while (val & GICR_WAKER_ChildrenAsleep);
+}
+
+static int __init gic_populate_rdist(void)
+{
+    u64 mpidr = cpu_logical_map(smp_processor_id());
+    u64 typer;
+    u64 aff;
+    int i;
+    uint32_t reg;
+
+    aff  = mpidr & ((1 << 24) - 1);
+    aff |= (mpidr >> 8) & (0xffUL << 24);
+
+    for (i = 0; i < gic.rdist_count; i++) {
+        uint32_t ptr = 0;
+
+        reg = readl_relaxed(GICR + ptr + GICR_PIDR0);
+        if ((reg & 0xff) != GICR_PIDR0_GICv3) { /* We're in trouble... */
+            printk("No redistributor present @%x\n", ptr);
+            break;
+        }
+
+        do {
+            typer = readq_relaxed(GICR + ptr + GICR_TYPER);
+            if ((typer >> 32) == aff) {
+                              
+                this_cpu(rbase) = (u64)GICR + ptr;
+                this_cpu(phy_rbase) = gic.rdist_regions[i].rdist_base + ptr;
+
+                printk("CPU%d: found redistributor %llx region %d\n",
+                            smp_processor_id(), (unsigned long long) mpidr, i);
+                return 0;
+            }
+
+            if (gic.rdist_stride) {
+                ptr += gic.rdist_stride;
+            } else {
+                ptr += SZ_64K * 2; /* Skip RD_base + SGI_base */
+                if (typer & GICR_TYPER_VLPIS)
+                    ptr += SZ_64K * 2; /* Skip VLPI_base + reserved page */
+                }
+            } while (!(typer & GICR_TYPER_LAST));
+        }
+
+        /* We couldn't even deal with ourselves... */
+        printk("CPU%d: mpidr %lx has no re-distributor!\n",
+                  smp_processor_id(), (unsigned long)mpidr);
+        return -ENODEV;
+}
+
+static void __cpuinit gic_cpu_init(void)
+{
+    int i;
+    paddr_t rbase_sgi;
+
+    /* Register ourselves with the rest of the world */
+    if (gic_populate_rdist())
+        return;
+
+    gic_enable_redist();
+
+    rbase_sgi = gic_data_rdist_sgi_base();
+
+    /*
+     * Set priority on PPI and SGI interrupts
+     */
+    for (i = 0; i < 16; i += 4)
+        writel_relaxed((GIC_PRI_IPI<<24 | GIC_PRI_IPI<<16 | GIC_PRI_IPI<<8 | GIC_PRI_IPI), (void *)rbase_sgi + GICR_IPRIORITYR0 + (i / 4) * 4);
+        //writel_relaxed(0x0, (void *)rbase + GICR_IPRIORITYR0 + (i / 4) * 4);
+        //writel_relaxed(0xa0a0a0a0, (void *)rbase + GICR_IPRIORITYR0 + (i / 4) * 4);
+
+    for (i = 16; i < 32; i += 4)
+        writel_relaxed((GIC_PRI_IRQ<<24 | GIC_PRI_IRQ<<16 | GIC_PRI_IRQ<<8 | GIC_PRI_IRQ), (void *)rbase_sgi + GICR_IPRIORITYR0 + (i / 4) * 4);
+
+    /*
+     * Disable all PPI interrupts, ensure all SGI interrupts are
+     * enabled.
+     */
+    writel_relaxed(0xffff0000, (void *)rbase_sgi + GICR_ICENABLER0);
+    writel_relaxed(0x0000ffff, (void *)rbase_sgi + GICR_ISENABLER0);
+
+    gic_redist_wait_for_rwp();
+
+    /* Enable system registers */
+    gic_enable_sre();
+
+    WRITE_SYSREG32(0, ICC_BPR1_EL1);
+    /* Set priority mask register */
+    WRITE_SYSREG32(DEFAULT_PMR_VALUE, ICC_PMR_EL1);
+
+    /* EOI drops priority too (mode 0) */
+    WRITE_SYSREG32(GICC_CTLR_EL1_EOImode_drop, ICC_CTLR_EL1);
+
+    WRITE_SYSREG32(1, ICC_IGRPEN1_EL1);
+}
+
+static void gic_cpu_disable(void)
+{
+    WRITE_SYSREG32(0, ICC_CTLR_EL1);
+}
+
+static void __cpuinit gic_hyp_init(void)
+{
+    uint32_t vtr;
+
+    vtr = READ_SYSREG32(ICH_VTR_EL2);
+    nr_lrs  = (vtr & GICH_VTR_NRLRGS) + 1;
+    nr_priorities = ((vtr >> GICH_VTR_PRIBITS_SHIFT) & GICH_VTR_PRIBITS_MASK) + 1;
+
+    WRITE_SYSREG32(GICH_VMCR_EOI | GICH_VMCR_VENG1, ICH_VMCR_EL2);
+    WRITE_SYSREG32(GICH_HCR_VGRP1EIE | GICH_HCR_EN, ICH_HCR_EL2);
+
+    update_cpu_lr_mask();
+    vtr = READ_SYSREG32(ICH_HCR_EL2);
+}
+
+/* Set up the per-CPU parts of the GIC for a secondary CPU */
+static int __cpuinit gic_init_secondary_cpu(struct notifier_block *nfb,
+                                       unsigned long action, void *hcpu)
+{
+    if (action == CPU_STARTING)
+    {
+        spin_lock(&gic.lock);
+        gic_cpu_init();
+        gic_hyp_init();
+        spin_unlock(&gic.lock);
+    }
+    return NOTIFY_DONE;
+}
+
+static struct notifier_block gic_cpu_nb = {
+    .notifier_call = gic_init_secondary_cpu,
+    .priority = 100
+};
+
+static void gic_smp_init(void)
+{
+   register_cpu_notifier(&gic_cpu_nb);
+}
+
+static void __cpuinit gic_hyp_disable(void)
+{
+    uint32_t vtr;
+    vtr = READ_SYSREG32(ICH_HCR_EL2);
+    vtr &= ~0x1;
+    WRITE_SYSREG32( vtr, ICH_HCR_EL2);
+}
+
+static u16 gic_compute_target_list(int *base_cpu, const struct cpumask *mask,
+                                   u64 cluster_id)
+{
+    int cpu = *base_cpu;
+    u64 mpidr = cpu_logical_map(cpu);
+    u16 tlist = 0;
+
+    while (cpu < nr_cpu_ids) {
+        /*
+         * If we ever get a cluster of more than 16 CPUs, just
+         * scream and skip that CPU.
+         */
+        tlist |= 1 << (mpidr & 0xf);
+
+        cpu = cpumask_next(cpu, mask);
+        mpidr = cpu_logical_map(cpu);
+
+        if (cluster_id != (mpidr & ~0xffUL)) {
+            cpu--;
+            goto out;
+        }
+    }
+out:
+    *base_cpu = cpu;
+    return tlist;
+}
+
+static void send_sgi(u64 cluster_id, u16 tlist, unsigned int irq)
+{
+    u64 val;
+
+    val  = (cluster_id & 0xff00ff0000UL) << 16; /* Aff3 + Aff2 */
+    val |= (cluster_id & 0xff00) << 8;          /* Aff1 */
+    val |= irq << 24;
+    val |= tlist;
+
+    WRITE_SYSREG(val, ICC_SGI1R_EL1);   
+}
+
+static void gic_send_sgi(const cpumask_t *cpumask, enum gic_sgi sgi)
+{
+    int cpu = 0;
+
+    dsb();
+
+    for_each_cpu(cpu, cpumask) {
+        u64 cluster_id = cpu_logical_map(cpu) & ~0xffUL;
+        u16 tlist;
+
+        tlist = gic_compute_target_list(&cpu, cpumask, cluster_id);
+        send_sgi(cluster_id, tlist, sgi);
+    }
+}
+
+/* Shut down the per-CPU GIC interface */
+static void gic_disable_interface(void)
+{
+    ASSERT(!local_irq_is_enabled());
+
+    spin_lock(&gic.lock);
+    gic_cpu_disable();
+    gic_hyp_disable();
+    spin_unlock(&gic.lock);
+}
+
+static void gic_update_lr(int lr, unsigned int virtual_irq,
+        unsigned int state, unsigned int priority)
+{
+    u64 maintenance_int = GICH_LR_MAINTENANCE_IRQ;
+    u64 grp = GICH_LR_GRP1;
+    u64 val = 0;
+
+    BUG_ON(lr >= nr_lrs);
+    BUG_ON(lr < 0);
+
+    val =  ((((u64)state) & 0x3) << GICH_LR_STATE_SHIFT) | grp | maintenance_int |
+        ((((u64)priority) & 0xff) << GICH_LR_PRIORITY_SHIFT) |
+        (((u64)virtual_irq & GICH_LR_VIRTUAL_MASK) << GICH_LR_VIRTUAL_SHIFT) |
+        (((u64)virtual_irq & GICH_LR_PHYSICAL_MASK) << GICH_LR_PHYSICAL_SHIFT);
+
+    gich_write_lr(lr, val);
+}
+
+int gicv_init(struct domain *d)
+{
+    /*
+     * Domain 0 gets the hardware address.
+     * Guests get the virtual platform layout.
+     */
+    if ( d->domain_id == 0 )
+    {
+        d->arch.vgic.dbase = gic.dbase;
+        d->arch.vgic.dbase_size = gic.dbase_size;
+        d->arch.vgic.rbase = gic.rdist_regions[0].rdist_base;
+        d->arch.vgic.rbase_size = gic.rdist_regions[0].rdist_base_size;
+        d->arch.vgic.rdist_stride = gic.rdist_stride;
+    }
+    else
+    {
+        d->arch.vgic.dbase = GUEST_GICD_BASE;
+    }
+
+    d->arch.vgic.nr_lines = 0;
+
+    return 0;
+
+}
+
+static unsigned long gic_read_eisr(void)
+{
+    return (unsigned long)READ_SYSREG32(ICH_EISR_EL2);
+}
+
+static unsigned int gic_update_lr_for_mi(int lr)
+{
+    u64 val;
+    uint32_t virq;
+
+    spin_lock_irq(&gic.lock);
+    val = gich_read_lr(lr);
+    virq = val & GICH_LR_VIRTUAL_MASK;
+
+    gich_write_lr(lr, 0);
+
+    spin_unlock_irq(&gic.lock);
+    return virq;
+}
+
+static unsigned long gic_rdist_rd_base(void)
+{
+   return this_cpu(phy_rbase);
+}
+
+static unsigned long gic_rdist_sgi_base(void)
+{
+   return this_cpu(phy_rbase) + SZ_64K;
+}
+
+static int gic_hw_type(void)
+{
+    return gic.hw_version;
+}
+
+static struct dt_irq * gic_maintenance_irq(void)
+{
+    return &gic.maintenance;
+}
+
+static struct gic_hw_operations gic_ops = {
+    .gic_type            = gic_hw_type,
+    .nr_lines            = gic_nr_lines,
+    .nr_lrs              = gic_nr_lrs,
+    .get_maintenance_irq = gic_maintenance_irq,
+    .state_init          = gic_state_init,
+    .save_state          = save_state,
+    .restore_state       = restore_state,
+    .dump_state          = gic_dump_state,
+    .gicv_setup          = gicv_init,
+    .enable_irq          = gic_enable_irq,
+    .disable_irq         = gic_disable_irq,
+    .eoi_irq             = gic_eoi_irq,
+    .deactivate_irq      = gic_dir_irq,
+    .ack_irq             = gic_ack_irq,
+    .set_irq_property    = gic_set_irq_property,
+    .send_sgi            = gic_send_sgi,
+    .disable_interface   = gic_disable_interface,
+    .update_lr           = gic_update_lr,
+    .update_lr_for_mi    = gic_update_lr_for_mi,
+    .read_eisr           = gic_read_eisr,
+    .read_cpu_rbase      = gic_rdist_rd_base,
+    .read_cpu_sgi_rbase  = gic_rdist_sgi_base,
+};
+
+/* Set up the GIC */
+void __init gicv3_init(void)
+{
+    static const struct dt_device_match gic_ids[] __initconst =
+    {
+        DT_MATCH_GIC_V3,
+        { /* sentinel */ },
+    };
+    struct dt_device_node *node;
+    struct rdist_region *rdist_regs;
+    int res, i;
+    uint32_t reg;
+
+    node = dt_find_interrupt_controller(gic_ids);
+    if ( !node )
+        panic("Unable to find compatible GIC in the device tree");
+
+    dt_device_set_used_by(node, DOMID_XEN);
+
+    res = dt_device_get_address(node, 0, &gic.dbase, &gic.dbase_size);
+    if ( res || !gic.dbase  || (gic.dbase & ~PAGE_MASK) || (gic.dbase_size & ~PAGE_MASK) )
+        panic("GIC: Cannot find a valid address for the distributor");
+
+    gic.map_dbase = ioremap_nocache(gic.dbase, gic.dbase_size);
+
+    reg = readl_relaxed(GICD + GICD_PIDR0);
+    if ((reg & 0xff) != GICD_PIDR0_GICv3)
+        panic("GIC: no distributor detected, giving up\n"); 
+
+    gic.hw_version = GIC_VERSION_V3;
+ 
+    if (!dt_property_read_u32(node, "#redistributor-regions", &gic.rdist_count))
+        gic.rdist_count = 1;
+
+    rdist_regs = xzalloc_array(struct rdist_region, gic.rdist_count);
+    if (!rdist_regs)
+        panic("GIC: no distributor detected, giving up\n");
+
+    for (i = 0; i < gic.rdist_count; i++) {
+	u64 rdist_base, rdist_size;
+
+        res = dt_device_get_address(node, 1 + i, &rdist_base, &rdist_size);
+        if ( res || !rdist_base)
+		printk("No rdist base found\n");
+		
+        rdist_regs[i].rdist_base = rdist_base;
+        rdist_regs[i].rdist_base_size = rdist_size;
+    }
+
+    if(!dt_property_read_u32(node, "redistributor-stride", &gic.rdist_stride))
+        gic.rdist_stride = 0x0;
+
+    gic.rdist_regions= rdist_regs;
+ 
+    res = dt_device_get_irq(node, 0, &gic.maintenance);
+    if ( res )
+        panic("GIC: Cannot find the maintenance IRQ");
+
+    /* Set the GIC as the primary interrupt controller */
+    dt_interrupt_controller = node;
+
+    /* map dbase & rdist regions */
+    gic.rdist_regions[0].map_rdist_base = ioremap_nocache(gic.rdist_regions[0].
+                             rdist_base, gic.rdist_regions[0].rdist_base_size);
+
+    printk("GIC initialization:\n"
+              "        gic_dist_addr=%"PRIpaddr"\n"
+              "        gic_dist_size=%"PRIpaddr"\n"
+              "        gic_dist_mapaddr=%"PRIpaddr"\n"
+              "        gic_rdist_regions=%d\n"
+              "        gic_rdist_stride=%x\n"
+              "        gic_rdist_base=%"PRIpaddr"\n"
+              "        gic_rdist_base_size=%"PRIpaddr"\n"
+              "        gic_rdist_base_mapaddr=%"PRIpaddr"\n"
+              "        gic_maintenance_irq=%u\n",
+              gic.dbase, gic.dbase_size, (u64)gic.map_dbase, gic.rdist_count,
+              gic.rdist_stride, gic.rdist_regions[0].rdist_base,
+              gic.rdist_regions[0].rdist_base_size,
+              (u64)gic.rdist_regions[0].map_rdist_base, gic.maintenance.irq);
+
+    /* Global settings: interrupt distributor */
+    spin_lock_init(&gic.lock);
+    spin_lock(&gic.lock);
+
+    gic_smp_init();
+    gic_dist_init();
+    gic_cpu_init();
+    gic_hyp_init();
+
+    /* Register hw ops*/
+    register_gic_ops(&gic_ops);
+    spin_unlock(&gic.lock);
+}
+
+/*
+ * Local variables:
+ * mode: C
+ * c-file-style: "BSD"
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff --git a/xen/arch/arm/gic.c b/xen/arch/arm/gic.c
index e0859ae..291e34c 100644
--- a/xen/arch/arm/gic.c
+++ b/xen/arch/arm/gic.c
@@ -688,6 +688,11 @@ static void maintenance_interrupt(int irq, void *dev_id, struct cpu_user_regs *r
 /* Set up the GIC */
 void __init gic_init(void)
 {
+    static const struct dt_device_match gicv3_ids[] __initconst =
+    {
+        DT_MATCH_GIC_V3,
+        { /* sentinel */ },
+    };
     static const struct dt_device_match gicv2_ids[] __initconst =
     {
         DT_MATCH_GIC,
@@ -698,6 +703,13 @@ void __init gic_init(void)
     spin_lock_init(&gic_lock);
 
     spin_lock(&gic_lock);
+    node = dt_find_interrupt_controller(gicv3_ids);
+    if ( node )
+    {
+        gicv3_init();
+        spin_unlock(&gic_lock);
+        return;
+    }
     node = dt_find_interrupt_controller(gicv2_ids);
     if ( node )
     {
diff --git a/xen/include/asm-arm/domain.h b/xen/include/asm-arm/domain.h
index 38df789..15e83e8 100644
--- a/xen/include/asm-arm/domain.h
+++ b/xen/include/asm-arm/domain.h
@@ -142,6 +142,10 @@ struct arch_domain
         /* Base address for guest GIC */
         paddr_t dbase; /* Distributor base address */
         paddr_t cbase; /* CPU base address */
+        paddr_t dbase_size; /* Distributor base size */
+        paddr_t rbase;      /* Re-Distributor base address */
+        paddr_t rbase_size; /* Re-Distributor size */
+        uint32_t rdist_stride;
     } vgic;
 
     struct vuart {
diff --git a/xen/include/asm-arm/gic.h b/xen/include/asm-arm/gic.h
index 2de6c6a..b6fbd5e 100644
--- a/xen/include/asm-arm/gic.h
+++ b/xen/include/asm-arm/gic.h
@@ -18,6 +18,8 @@
 #ifndef __ASM_ARM_GIC_H__
 #define __ASM_ARM_GIC_H__
 
+#define SZ_64K  0x00010000
+
 /*
  * The minimum GICC_BPR is required to be in the range 0-3. We set
  * GICC_BPR to 0 but we must expect that it might be 3. This means we
@@ -48,6 +50,8 @@
 #define DT_MATCH_GIC    DT_MATCH_COMPATIBLE("arm,cortex-a15-gic"), \
                         DT_MATCH_COMPATIBLE("arm,cortex-a7-gic")
 
+#define DT_MATCH_GIC_V3 DT_MATCH_COMPATIBLE("arm,gic-v3")
+
 extern int gic_hw_version(void);
 extern int domain_vgic_init(struct domain *d);
 extern void domain_vgic_free(struct domain *d);
@@ -58,6 +62,7 @@ extern void vgic_disable_irqs(struct vcpu *v, uint32_t r, int n);
 extern void vgic_enable_irqs(struct vcpu *v, uint32_t r, int n);
 extern int vgic_v2_init(struct domain *d);
 
+extern void gicv3_init(void);
 extern void gicv2_init(void);
 
 extern void vgic_vcpu_inject_irq(struct vcpu *v, unsigned int irq,int virtual);
@@ -96,6 +101,7 @@ extern int gicv_setup(struct domain *d);
 extern void gic_save_state(struct vcpu *v);
 extern void gic_restore_state(struct vcpu *v);
 
+#define GIC_VERSION_V3 0x3
 #define GIC_VERSION_V2 0x2
 
 /* SGI (AKA IPIs) */
@@ -156,6 +162,9 @@ extern unsigned int gic_number_lines(void);
 int gic_irq_xlate(const u32 *intspec, unsigned int intsize,
                   unsigned int *out_hwirq, unsigned int *out_type);
 
+extern unsigned long gic_data_rdist_rd_base(void);
+extern unsigned long gic_data_rdist_sgi_base(void);
+
 #endif /* __ASSEMBLY__ */
 #endif
 
diff --git a/xen/include/asm-arm/gic_v3_defs.h b/xen/include/asm-arm/gic_v3_defs.h
new file mode 100644
index 0000000..24242ea
--- /dev/null
+++ b/xen/include/asm-arm/gic_v3_defs.h
@@ -0,0 +1,211 @@
+/*
+ * ARM Generic Interrupt Controller support
+ *
+ * Vijaya Kumar K <vijaya.kumar@caviumnetworks.com>
+ * Copyright (c) 2014 Cavium Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#define SZ_64K  0x00010000
+
+#define GICD_CTLR       (0x000)
+#define GICD_TYPER      (0x004)
+#define GICD_IIDR       (0x008)
+#define GICD_STATUSR    (0x010)
+#define GICD_SETSPI_NSR (0x040)
+#define GICD_CLRSPI_NSR (0x048)
+#define GICD_SETSPI_SR  (0x050)
+#define GICD_CLRSPI_SR  (0x058)
+#define GICD_IGROUPR    (0x080)
+#define GICD_IGROUPRN   (0x0FC)
+#define GICD_ISENABLER  (0x100)
+#define GICD_ISENABLERN (0x17C)
+#define GICD_ICENABLER  (0x180)
+#define GICD_ICENABLERN (0x1fC)
+#define GICD_ISPENDR    (0x200)
+#define GICD_ISPENDRN   (0x27C)
+#define GICD_ICPENDR    (0x280)
+#define GICD_ICPENDRN   (0x2FC)
+#define GICD_ISACTIVER  (0x300)
+#define GICD_ISACTIVERN (0x37C)
+#define GICD_ICACTIVER  (0x380)
+#define GICD_ICACTIVERN (0x3FC)
+#define GICD_IPRIORITYR (0x400)
+#define GICD_IPRIORITYRN (0x7F8)
+#define GICD_ICFGR      (0xC00)
+#define GICD_ICFGRN     (0xCFC)
+#define GICD_NSACR      (0xE00)
+#define GICD_NSACRN     (0xEFC)
+#define GICD_SGIR       (0xF00)
+#define GICD_CPENDSGIR  (0xF10)
+#define GICD_CPENDSGIRN (0xF1C)
+#define GICD_SPENDSGIR  (0xF20)
+#define GICD_SPENDSGIRN (0xF2C)
+#define GICD_IROUTER    (0x6000)
+#define GICD_IROUTERN   (0x7FF8)
+#define GICD_PIDR0      (0xFFE0)
+#define GICD_PIDR7      (0xFFDC)
+
+#define GICD_SGI_TARGET_LIST_SHIFT   (24)
+#define GICD_SGI_TARGET_LIST_MASK    (0x3UL << GICD_SGI_TARGET_LIST_SHIFT)
+#define GICD_SGI_TARGET_LIST         (0UL<<GICD_SGI_TARGET_LIST_SHIFT)
+#define GICD_SGI_TARGET_OTHERS       (1UL<<GICD_SGI_TARGET_LIST_SHIFT)
+#define GICD_SGI_TARGET_SELF         (2UL<<GICD_SGI_TARGET_LIST_SHIFT)
+#define GICD_SGI_TARGET_SHIFT        (16)
+#define GICD_SGI_TARGET_MASK         (0xFFUL<<GICD_SGI_TARGET_SHIFT)
+#define GICD_SGI_GROUP1              (1UL<<15)
+#define GICD_SGI_INTID_MASK          (0xFUL)
+
+#define GICC_SRE_EL2_SRE             (1UL << 0)
+#define GICC_SRE_EL2_DFB             (1UL << 1)
+#define GICC_SRE_EL2_DIB             (1UL << 2)
+#define GICC_SRE_EL2_ENEL1           (1UL << 3)
+
+#define GICD_PIDR0_GICv3                0x92
+
+#define GICD_CTLR_RWP                (1UL << 31)
+#define GICD_CTLR_ARE_NS                (1U << 4)
+#define GICD_CTLR_ENABLE_G1A            (1U << 1)
+#define GICD_CTLR_ENABLE_G1             (1U << 0)
+#define GICD_IROUTER_SPI_MODE_ONE    (0UL << 31)
+#define GICD_IROUTER_SPI_MODE_ANY    (1UL << 31)
+
+#define GICH_HCR_EN                  (1 << 0)
+
+#define GICC_CTLR_EL1_EOImode_drop_dir  (0U << 1)
+#define GICC_CTLR_EL1_EOImode_drop      (1U << 1)
+
+#define GICR_WAKER_ProcessorSleep       (1U << 1)
+#define GICR_WAKER_ChildrenAsleep       (1U << 2)
+#define GICR_PIDR0_GICv3                0x93
+
+#define GICR_CTLR       (0x0000)
+#define GICR_IIDR       (0x0004)
+#define GICR_TYPER      (0x0008)
+#define GICR_STATUSR    (0x0010)
+#define GICR_WAKER      (0x0014)
+#define GICR_SETLPIR    (0x0040)
+#define GICR_CLRLPIR    (0x0048)
+#define GICR_PROPBASER  (0x0070)
+#define GICR_PENDBASER  (0x0078)
+#define GICR_INVLPIR    (0x00A0)
+#define GICR_INVALLR    (0x00B0)
+#define GICR_SYNCR      (0x00C0)
+#define GICR_MOVLPIR    (0x100)
+#define GICR_MOVALLR    (0x0110)
+#define GICR_PIDR0      GICD_PIDR0
+#define GICR_PIDR7      GICD_PIDR7
+
+/* GICR for SGI's & PPI's */
+
+#define GICR_IGROUPR0    (0x0080)
+#define GICR_IGRPMODR0   (0x0F80)
+#define GICR_ISENABLER0  (0x0100)
+#define GICR_ICENABLER0  (0x0180)
+#define GICR_ISPENDR0    (0x0200)
+#define GICR_ICPENDR0    (0x0280)
+#define GICR_ISACTIVER0  (0x0300)
+#define GICR_ICACTIVER0  (0x0380)
+#define GICR_IPRIORITYR0 (0x0400)
+#define GICR_IPRIORITYR7 (0x041C)
+#define GICR_ICFGR0      (0x0C00)
+#define GICR_ICFGR1      (0x0C04)
+#define GICR_NSACR       (0x0E00)
+
+#define GICR_TYPER_PLPIS                (1U << 0)
+#define GICR_TYPER_VLPIS                (1U << 1)
+#define GICR_TYPER_LAST                 (1U << 4)
+
+/* Register bits */
+#define GICD_CTL_ENABLE 0x1
+
+#define GICD_TYPE_LINES 0x01f
+#define GICD_TYPE_CPUS  0x0e0
+#define GICD_TYPE_SEC   0x400
+
+#define GICC_CTL_ENABLE 0x1
+#define GICC_CTL_EOI    (0x1 << 9)
+
+#define GICC_IA_IRQ       0x03ff
+#define GICC_IA_CPU_MASK  0x1c00
+#define GICC_IA_CPU_SHIFT 10
+
+#define DEFAULT_PMR_VALUE       0xff
+
+#define GICH_HCR_EN       (1 << 0)
+#define GICH_HCR_UIE      (1 << 1)
+#define GICH_HCR_LRENPIE  (1 << 2)
+#define GICH_HCR_NPIE     (1 << 3)
+#define GICH_HCR_VGRP0EIE (1 << 4)
+#define GICH_HCR_VGRP0DIE (1 << 5)
+#define GICH_HCR_VGRP1EIE (1 << 6)
+#define GICH_HCR_VGRP1DIE (1 << 7)
+#define GICH_HCR_TC       (1 << 10)
+
+#define GICH_MISR_EOI     (1 << 0)
+#define GICH_MISR_U       (1 << 1)
+#define GICH_MISR_LRENP   (1 << 2)
+#define GICH_MISR_NP      (1 << 3)
+#define GICH_MISR_VGRP0E  (1 << 4)
+#define GICH_MISR_VGRP0D  (1 << 5)
+#define GICH_MISR_VGRP1E  (1 << 6)
+#define GICH_MISR_VGRP1D  (1 << 7)
+
+#define GICH_VMCR_EOI     (1 << 9)
+#define GICH_VMCR_VENG1    (1 << 1)
+
+#define GICH_LR_VIRTUAL_MASK    0xffff
+#define GICH_LR_VIRTUAL_SHIFT   0
+#define GICH_LR_PHYSICAL_MASK   0x3ff
+#define GICH_LR_PHYSICAL_SHIFT  32
+#define GICH_LR_STATE_MASK      0x3
+#define GICH_LR_STATE_SHIFT     62
+#define GICH_LR_PRIORITY_SHIFT  48
+#define GICH_LR_MAINTENANCE_IRQ (1UL<<41)
+#define GICH_LR_PENDING         1
+#define GICH_LR_ACTIVE          2
+#define GICH_LR_GRP1            (1UL<<60)
+#define GICH_LR_HW              (1UL<<61)
+
+#define GICH_VTR_NRLRGS         0x3f
+#define GICH_VTR_PRIBITS_MASK   0x7
+#define GICH_VTR_PRIBITS_SHIFT  29
+
+/*
+ * The minimum GICC_BPR is required to be in the range 0-3. We set
+ * GICC_BPR to 0 but we must expect that it might be 3. This means we
+ * can rely on premption between the following ranges:
+ * 0xf0..0xff
+ * 0xe0..0xdf
+ * 0xc0..0xcf
+ * 0xb0..0xbf
+ * 0xa0..0xaf
+ * 0x90..0x9f
+ * 0x80..0x8f
+ *
+ * Priorities within a range will not preempt each other.
+ *
+ * A GIC must support a mimimum of 16 priority levels.
+ */
+#define GIC_PRI_LOWEST 0xf0
+#define GIC_PRI_IRQ 0xa0
+#define GIC_PRI_IPI 0x90 /* IPIs must preempt normal interrupts */
+#define GIC_PRI_HIGHEST 0x80 /* Higher priorities belong to Secure-World */
+
+/*
+ * Local variables:
+ * mode: C
+ * c-file-style: "BSD"
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
-- 
1.7.9.5

Re: [RFC PATCH v1 08/10] xen/arm: Add support for GIC v3

[Stefano Stabellini]

On Wed, 19 Mar 2014, vijay.kilari@gmail.com wrote:
> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> 
> Add support for GIC v3 specification.
> This driver assumes that ARE and SRE
> is enable by default.
> 
> Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> ---
>  xen/arch/arm/Makefile             |    2 +-
>  xen/arch/arm/gic-v3.c             |  944 +++++++++++++++++++++++++++++++++++++
>  xen/arch/arm/gic.c                |   12 +
>  xen/include/asm-arm/domain.h      |    4 +
>  xen/include/asm-arm/gic.h         |    9 +
>  xen/include/asm-arm/gic_v3_defs.h |  211 +++++++++
>  6 files changed, 1181 insertions(+), 1 deletion(-)
> 
> diff --git a/xen/arch/arm/Makefile b/xen/arch/arm/Makefile
> index 20f59f4..a11c699 100644
> --- a/xen/arch/arm/Makefile
> +++ b/xen/arch/arm/Makefile
> @@ -10,7 +10,7 @@ obj-y += vpsci.o
>  obj-y += domctl.o
>  obj-y += sysctl.o
>  obj-y += domain_build.o
> -obj-y += gic.o gic-v2.o
> +obj-y += gic.o gic-v2.o gic-v3.o
>  obj-y += io.o
>  obj-y += irq.o
>  obj-y += kernel.o
> diff --git a/xen/arch/arm/gic-v3.c b/xen/arch/arm/gic-v3.c
> new file mode 100644
> index 0000000..81a36ba
> --- /dev/null
> +++ b/xen/arch/arm/gic-v3.c
> @@ -0,0 +1,944 @@
> +/*
> + * xen/arch/arm/gic-v3.c
> + *
> + * ARM Generic Interrupt Controller support v3 version
> + * based on xen/arch/arm/gic-v2.c
> + * 
> + * Vijaya Kumar K <vijaya.kumar@caviumnetworks.com>
> + * Copyright (c) 2014 Cavium Inc.
> + * 
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License as published by
> + * the Free Software Foundation; either version 2 of the License, or
> + * (at your option) any later version.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> + * GNU General Public License for more details.
> + */
> +
> +#include <xen/config.h>
> +#include <xen/lib.h>
> +#include <xen/init.h>
> +#include <xen/cpu.h>
> +#include <xen/mm.h>
> +#include <xen/irq.h>
> +#include <xen/sched.h>
> +#include <xen/errno.h>
> +#include <xen/serial.h>
> +#include <xen/softirq.h>
> +#include <xen/list.h>
> +#include <xen/device_tree.h>
> +#include <asm/p2m.h>
> +#include <asm/domain.h>
> +#include <asm/platform.h>
> +
> +#include <asm/gic_v3_defs.h>
> +#include <asm/gic.h>
> +#include <asm/io.h>
> +
> +struct rdist_region {
> +    paddr_t rdist_base;
> +    paddr_t rdist_base_size;
> +    void __iomem *map_rdist_base;
> +};
> +	
> +/* Global state */
> +static struct {
> +    paddr_t dbase;            /* Address of distributor registers */
> +    paddr_t dbase_size;
> +    void __iomem *map_dbase;  /* Mapped address of distributor registers */
> +    struct rdist_region *rdist_regions;
> +    u32  rdist_stride;
> +    unsigned int rdist_count; /* Number of rdist regions count */
> +    unsigned int lines;       /* Number of interrupts (SPIs + PPIs + SGIs) */
> +    struct dt_irq maintenance;
> +    unsigned int cpus;
> +    int hw_version;
> +    spinlock_t lock;
> +} gic;
> +
> +struct gic_state_data {
> +    uint32_t gic_hcr, gic_vmcr;
> +    uint32_t gic_apr0[4];
> +    uint32_t gic_apr1[4];
> +    uint64_t gic_lr[16];
> +};
> +
> +#define GICD ((volatile unsigned char *) gic.map_dbase)
> +/* Only one region is implemented which is enough for 0-31 cpus */
> +#define GICR ((volatile unsigned char *) gic.rdist_regions[0].map_rdist_base)
> +
> +/* per-cpu re-distributor base */
> +static DEFINE_PER_CPU(paddr_t, rbase);
> +static DEFINE_PER_CPU(paddr_t, phy_rbase);
> +
> +static unsigned nr_lrs;
> +static uint32_t nr_priorities;
> +
> +/* The GIC mapping of CPU interfaces does not necessarily match the
> + * logical CPU numbering. Let's use mapping as returned by the GIC
> + * itself
> + */
> +
> +#define gic_data_rdist_rd_base()        (this_cpu(rbase))
> +#define gic_data_rdist_sgi_base()       (gic_data_rdist_rd_base() + SZ_64K)
> +
> +static inline u64 read_cpuid_mpidr(void)
> +{
> +   return READ_SYSREG(MPIDR_EL1);
> +}
> +
> +static u64 gich_read_lr(int lr)
> +{
> +    switch (lr) 
> +    {
> +        case 0: /* ICH_LRn is 64 bit */
> +            return READ_SYSREG(ICH_LR0_EL2);
> +            break;
> +        case 1:
> +            return READ_SYSREG(ICH_LR1_EL2);
> +            break;
> +        case 2:
> +            return READ_SYSREG(ICH_LR2_EL2);
> +            break;
> +        case 3:
> +            return READ_SYSREG(ICH_LR3_EL2);
> +            break;
> +        case 4:
> +            return READ_SYSREG(ICH_LR4_EL2);
> +            break;
> +        case 5:
> +            return READ_SYSREG(ICH_LR5_EL2);
> +            break;
> +        case 6:
> +            return READ_SYSREG(ICH_LR6_EL2);
> +            break;
> +        case 7:
> +            return READ_SYSREG(ICH_LR7_EL2);
> +            break;
> +        case 8:
> +            return READ_SYSREG(ICH_LR8_EL2);
> +            break;
> +        case 9:
> +            return READ_SYSREG(ICH_LR9_EL2);
> +            break;
> +        case 10:
> +            return READ_SYSREG(ICH_LR10_EL2);
> +            break;
> +        case 11:
> +            return READ_SYSREG(ICH_LR11_EL2);
> +            break;
> +        case 12:
> +            return READ_SYSREG(ICH_LR12_EL2);
> +            break;
> +        case 13:
> +            return READ_SYSREG(ICH_LR13_EL2);
> +            break;
> +        case 14:
> +            return READ_SYSREG(ICH_LR14_EL2);
> +            break;
> +        case 15:
> +            return READ_SYSREG(ICH_LR15_EL2);
> +            break;
> +        default:
> +            return 0;
> +    }
> +}
> +
> +static void gich_write_lr(int lr, u64 val)
> +{
> +    switch (lr) 
> +    {
> +        case 0:
> +           WRITE_SYSREG(val, ICH_LR0_EL2);
> +           break;
> +        case 1:
> +           WRITE_SYSREG(val, ICH_LR1_EL2);
> +           break;
> +        case 2:
> +           WRITE_SYSREG(val, ICH_LR2_EL2);
> +           break;
> +        case 3:
> +           WRITE_SYSREG(val, ICH_LR3_EL2);
> +           break;
> +        case 4:
> +           WRITE_SYSREG(val, ICH_LR4_EL2);
> +           break;
> +        case 5:
> +           WRITE_SYSREG(val, ICH_LR5_EL2);
> +           break;
> +        case 6:
> +           WRITE_SYSREG(val, ICH_LR6_EL2);
> +           break;
> +        case 7:
> +           WRITE_SYSREG(val, ICH_LR7_EL2);
> +           break;
> +        case 8:
> +           WRITE_SYSREG(val, ICH_LR8_EL2);
> +           break;
> +        case 9:
> +           WRITE_SYSREG(val, ICH_LR9_EL2);
> +           break;
> +        case 10:
> +           WRITE_SYSREG(val, ICH_LR10_EL2);
> +           break;
> +        case 11:
> +           WRITE_SYSREG(val, ICH_LR11_EL2);
> +           break;
> +        case 12:
> +           WRITE_SYSREG(val, ICH_LR12_EL2);
> +           break;
> +        case 13:
> +           WRITE_SYSREG(val, ICH_LR13_EL2);
> +           break;
> +        case 14:
> +           WRITE_SYSREG(val, ICH_LR14_EL2);
> +           break;
> +        case 15:
> +           WRITE_SYSREG(val, ICH_LR15_EL2);
> +           break;
> +        default:
> +           return;
> +    }
> +}
> +
> +static void gic_enable_sre(void)
> +{
> +    uint32_t val;
> +
> +    val = READ_SYSREG32(ICC_SRE_EL2);
> +    val |= GICC_SRE_EL2_SRE | GICC_SRE_EL2_ENEL1 | GICC_SRE_EL2_DFB | GICC_SRE_EL2_DIB;
> +    WRITE_SYSREG32(val, ICC_SRE_EL2);
> +    isb();
> +}
> +
> +/* Wait for completion of a distributor change */
> +static void gic_do_wait_for_rwp(paddr_t base)
> +{
> +    u32 val;
> +    do {
> +        val = readl_relaxed((void *)base + GICD_CTLR);
> +        val = readl_relaxed(GICD + GICD_CTLR);
> +        val = GICD[GICD_CTLR];
> +        cpu_relax();
> +    } while (val & GICD_CTLR_RWP);
> +}

As much I think that this busy loop is terrible unfortunately it is
part of the spec :-(
Might be worth adding a comment on the function to explain why and when
it is required.
It is also worth considering whether it makes sense to use the
notification of command completion by interrupt instead.



> +static void gic_dist_wait_for_rwp(void)
> +{
> +    gic_do_wait_for_rwp((paddr_t)GICD);
> +}
> +
> +static void gic_redist_wait_for_rwp(void)
> +{
> +    gic_do_wait_for_rwp(gic_data_rdist_rd_base());
> +}
> +
> +static void gic_wait_for_rwp(int irq)
> +{
> +    if (irq < 32)
> +         gic_redist_wait_for_rwp();
> +    else
> +         gic_dist_wait_for_rwp();
> +}
> +
> +static unsigned int gic_mask_cpu(const cpumask_t *cpumask)
> +{
> +    unsigned int cpu;
> +    cpumask_t possible_mask;
> +
> +    cpumask_and(&possible_mask, cpumask, &cpu_possible_map);
> +    cpu = cpumask_any(&possible_mask);
> +    return cpu;
> +}
> +
> +static unsigned int gic_nr_lines(void)
> +{
> +    return gic.lines;
> +}
> +
> +static unsigned int gic_nr_lrs(void)
> +{
> +    return nr_lrs;
> +}
> +
> +static void write_aprn_regs(struct gic_state_data *d)
> +{
> +    switch(nr_priorities)
> +    {
> +        case 7:
> +            WRITE_SYSREG32(d->gic_apr0[2], ICH_AP0R2_EL2);
> +            WRITE_SYSREG32(d->gic_apr1[2], ICH_AP1R2_EL2);
> +        case 6:
> +            WRITE_SYSREG32(d->gic_apr0[1], ICH_AP0R1_EL2);
> +            WRITE_SYSREG32(d->gic_apr1[1], ICH_AP1R1_EL2);
> +        case 5:
> +            WRITE_SYSREG32(d->gic_apr0[0], ICH_AP0R0_EL2);
> +            WRITE_SYSREG32(d->gic_apr1[0], ICH_AP1R0_EL2);
> +            break;
> +        default:
> +          panic("Write Undefined active priorities \n");
> +    }
> +}
> +
> +static void read_aprn_regs(struct gic_state_data *d)
> +{
> +    switch(nr_priorities)
> +    {
> +        case 7:
> +            d->gic_apr0[2] = READ_SYSREG32(ICH_AP0R2_EL2);
> +            d->gic_apr1[2] = READ_SYSREG32(ICH_AP1R2_EL2);
> +        case 6:
> +            d->gic_apr0[1] = READ_SYSREG32(ICH_AP0R1_EL2);
> +            d->gic_apr1[1] = READ_SYSREG32(ICH_AP1R1_EL2);
> +        case 5:
> +            d->gic_apr0[0] = READ_SYSREG32(ICH_AP0R0_EL2);
> +            d->gic_apr1[0] = READ_SYSREG32(ICH_AP1R0_EL2);
> +            break;
> +        default:
> +          panic("Read Undefined active priorities \n");
> +    }
> +}
> +
> +static int gic_state_init(struct vcpu *v)
> +{
> +     v->arch.gic_state = (struct gic_state_data *)xzalloc(struct gic_state_data);
> +     if(!v->arch.gic_state)
> +        return -ENOMEM;
> +     return 0; 
> +}
> +
> +static void save_state(struct vcpu *v)
> +{
> +    int i;
> +    struct gic_state_data *d;
> +    d = (struct gic_state_data *)v->arch.gic_state;
> +
> +    /* No need for spinlocks here because interrupts are disabled around
> +     * this call and it only accesses struct vcpu fields that cannot be
> +     * accessed simultaneously by another pCPU.
> +     */
> +    for ( i=0; i<nr_lrs; i++)
> +        d->gic_lr[i] = gich_read_lr(i);
> +
> +    read_aprn_regs(d); 
> +
> +    d->gic_vmcr = READ_SYSREG32(ICH_VMCR_EL2);
> +}
> +
> +static void restore_state(struct vcpu *v)
> +{
> +    int i;
> +    struct gic_state_data *d;
> +    d = (struct gic_state_data *)v->arch.gic_state;
> +
> +    for ( i=0; i<nr_lrs; i++)
> +        gich_write_lr(i, d->gic_lr[i]);
> +
> +    write_aprn_regs(d);
> +
> +    WRITE_SYSREG32(d->gic_vmcr, ICH_VMCR_EL2);
> +}
> +
> +static void gic_dump_state(struct vcpu *v)
> +{
> +    int i;
> +    struct gic_state_data *d;
> +    d = (struct gic_state_data *)v->arch.gic_state;
> +    if ( v == current )
> +    {
> +        for ( i = 0; i < nr_lrs; i++ )
> +            printk("   HW_LR[%d]=%lx\n", i, gich_read_lr(i));
> +    }
> +    else
> +    {
> +        for ( i = 0; i < nr_lrs; i++ )
> +            printk("   VCPU_LR[%d]=%lx\n", i, d->gic_lr[i]);
> +    }
> +}
> + 
> +static void gic_enable_irq(int irq)
> +{
> +    uint32_t enabler;
> +
> +    /* Enable routing */
> +    if(irq > 31)
> +    {
> +        enabler = readl_relaxed(GICD + GICD_ISENABLER + (irq / 32) * 4);
> +        writel_relaxed(enabler | (1u << (irq % 32)), GICD + GICD_ISENABLER + (irq / 32) * 4);
> +    }
> +    else
> +    {
> +        enabler = readl_relaxed((void *)gic_data_rdist_sgi_base() + GICR_ISENABLER0);
> +        writel_relaxed(enabler | (1u << irq), (void *)gic_data_rdist_sgi_base() + GICR_ISENABLER0);
> +    }
> +    gic_wait_for_rwp(irq);
> +}
> +
> +static void gic_disable_irq(int irq)
> +{
> +    /* Disable routing */
> +    if(irq > 31)
> +        writel_relaxed(1u << (irq % 32), GICD + GICD_ICENABLER + ((irq / 32) * 4));
> +    else
> +        writel_relaxed(1u << irq, (void *)gic_data_rdist_sgi_base() + GICR_ICENABLER0);
> +}
> +
> +static void gic_eoi_irq(int irq)
> +{
> +    /* Lower the priority */
> +    WRITE_SYSREG32(irq, ICC_EOIR1_EL1);
> +}
> +
> +static void gic_dir_irq(int irq)
> +{
> +    /* Deactivate */
> +    WRITE_SYSREG32(irq, ICC_DIR_EL1);
> +}
> +
> +static unsigned int gic_ack_irq(void)
> +{
> +    return (READ_SYSREG32(ICC_IAR1_EL1) & GICC_IA_IRQ);
> +}
> +
> +static u64 gic_mpidr_to_affinity(u64 mpidr)
> +{
> +    /* Make sure we don't broadcast the interrupt */
> +    return mpidr & ~GICD_IROUTER_SPI_MODE_ANY;
> +}
> +
> +/*
> + * - needs to be called with gic.lock held
> + * - needs to be called with a valid cpu_mask, ie each cpu in the mask has
> + * already called gic_cpu_init
> + */
> +static void gic_set_irq_property(unsigned int irq, bool_t level,
> +                                   const cpumask_t *cpu_mask,
> +                                   unsigned int priority)
> +{
> +    uint32_t cfg, edgebit;
> +    u64 affinity;
> +    unsigned int cpu = gic_mask_cpu(cpu_mask);
> +    paddr_t rebase;
> +
> +
> +    /* Set edge / level */
> +    if (irq < 16)
> +	/* SGI's are always edge-triggered not need to call GICD_ICFGR0 */
> +       cfg = readl_relaxed((void *)gic_data_rdist_sgi_base() + GICR_ICFGR0);
> +    else if (irq < 32)
> +       cfg = readl_relaxed((void *)gic_data_rdist_sgi_base() + GICR_ICFGR1);
> +    else
> +       cfg = readl_relaxed(GICD + GICD_ICFGR + (irq / 16) * 4);
> +
> +    edgebit = 2u << (2 * (irq % 16));
> +    if ( level )
> +        cfg &= ~edgebit;
> +    else
> +        cfg |= edgebit;
> +
> +    if (irq < 16)
> +       writel_relaxed(cfg, (void *)gic_data_rdist_sgi_base() + GICR_ICFGR0);
> +    else if (irq < 32)
> +       writel_relaxed(cfg, (void *)gic_data_rdist_sgi_base() + GICR_ICFGR1);
> +    else
> +       writel_relaxed(cfg, GICD + GICD_ICFGR + (irq / 16) * 4);
> +
> +
> +    /* need to check if ARE is set to access IROUTER */
> +    affinity = gic_mpidr_to_affinity(cpu_logical_map(cpu));
> +    if (irq > 31)
> +        writeq_relaxed(affinity, (GICD + GICD_IROUTER + irq * 8));
> +
> +    /* Set priority */
> +    if (irq < 32)
> +    {
> +	rebase = gic_data_rdist_sgi_base();

code style

> +        writeb_relaxed(priority, (void *)rebase + GICR_IPRIORITYR0 + irq);
> +    }
> +    else 
> +    {
> +        writeb_relaxed(priority, GICD + GICD_IPRIORITYR + irq);
> +
> +    }
> +}
> +
> +static void __init gic_dist_init(void)
> +{
> +    uint32_t type;
> +    u64 affinity;
> +    int i;
> +
> +    /* Disable the distributor */
> +    writel_relaxed(0, GICD + GICD_CTLR);
> +
> +    type = readl_relaxed(GICD + GICD_TYPER);
> +    gic.lines = 32 * ((type & GICD_TYPE_LINES) + 1);
> +
> +    printk("GIC: %d lines, (IID %8.8x).\n",
> +           gic.lines, readl_relaxed(GICD + GICD_IIDR));
> +
> +    /* Default all global IRQs to level, active low */
> +    for ( i = 32; i < gic.lines; i += 16 )
> +        writel_relaxed(0, GICD + GICD_ICFGR + (i / 16) * 4);
> +
> +    /* Default priority for global interrupts */
> +    for ( i = 32; i < gic.lines; i += 4 )
> +        writel_relaxed((GIC_PRI_IRQ<<24 | GIC_PRI_IRQ<<16 | GIC_PRI_IRQ<<8 | GIC_PRI_IRQ), GICD + GICD_IPRIORITYR + (i / 4) * 4);
> +
> +    /* Disable all global interrupts */
> +    for ( i = 32; i < gic.lines; i += 32 )
> +        writel_relaxed(0xffffffff, GICD + GICD_ICENABLER + (i / 32) * 4);
> +
> +    gic_dist_wait_for_rwp();
> +
> +    /* Turn on the distributor */
> +    writel_relaxed(GICD_CTL_ENABLE | GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1, GICD + GICD_CTLR);
> + 
> +    /* Route all global IRQs to this CPU */
> +    affinity = gic_mpidr_to_affinity(read_cpuid_mpidr());
> +    for ( i = 31; i < gic.lines; i++ )
> +        writeq_relaxed(affinity, GICD + GICD_IROUTER + i * 8);
> +}
> +
> +static void gic_enable_redist(void)
> +{
> +    paddr_t rbase;
> +    u32 val;
> +
> +    rbase = this_cpu(rbase);
> +
> +    /* Wake up this CPU redistributor */
> +    val = readl_relaxed((void *)rbase + GICR_WAKER);
> +    val &= ~GICR_WAKER_ProcessorSleep;
> +    writel_relaxed(val, (void *)rbase + GICR_WAKER);
> +
> +    do {
> +         val = readl_relaxed((void *)rbase + GICR_WAKER);
> +         cpu_relax();
> +    } while (val & GICR_WAKER_ChildrenAsleep);
> +}
> +
> +static int __init gic_populate_rdist(void)
> +{
> +    u64 mpidr = cpu_logical_map(smp_processor_id());
> +    u64 typer;
> +    u64 aff;
> +    int i;
> +    uint32_t reg;
> +
> +    aff  = mpidr & ((1 << 24) - 1);
> +    aff |= (mpidr >> 8) & (0xffUL << 24);
> +
> +    for (i = 0; i < gic.rdist_count; i++) {
> +        uint32_t ptr = 0;
> +
> +        reg = readl_relaxed(GICR + ptr + GICR_PIDR0);
> +        if ((reg & 0xff) != GICR_PIDR0_GICv3) { /* We're in trouble... */
> +            printk("No redistributor present @%x\n", ptr);
> +            break;
> +        }
> +
> +        do {
> +            typer = readq_relaxed(GICR + ptr + GICR_TYPER);
> +            if ((typer >> 32) == aff) {
> +                              
> +                this_cpu(rbase) = (u64)GICR + ptr;
> +                this_cpu(phy_rbase) = gic.rdist_regions[i].rdist_base + ptr;
> +
> +                printk("CPU%d: found redistributor %llx region %d\n",
> +                            smp_processor_id(), (unsigned long long) mpidr, i);
> +                return 0;
> +            }
> +
> +            if (gic.rdist_stride) {
> +                ptr += gic.rdist_stride;
> +            } else {
> +                ptr += SZ_64K * 2; /* Skip RD_base + SGI_base */
> +                if (typer & GICR_TYPER_VLPIS)
> +                    ptr += SZ_64K * 2; /* Skip VLPI_base + reserved page */
> +                }
> +            } while (!(typer & GICR_TYPER_LAST));

code style: the indentation is wrong


> +        }
> +
> +        /* We couldn't even deal with ourselves... */
> +        printk("CPU%d: mpidr %lx has no re-distributor!\n",
> +                  smp_processor_id(), (unsigned long)mpidr);
> +        return -ENODEV;
> +}
> +
> +static void __cpuinit gic_cpu_init(void)
> +{
> +    int i;
> +    paddr_t rbase_sgi;
> +
> +    /* Register ourselves with the rest of the world */
> +    if (gic_populate_rdist())
> +        return;
> +
> +    gic_enable_redist();
> +
> +    rbase_sgi = gic_data_rdist_sgi_base();
> +
> +    /*
> +     * Set priority on PPI and SGI interrupts
> +     */
> +    for (i = 0; i < 16; i += 4)
> +        writel_relaxed((GIC_PRI_IPI<<24 | GIC_PRI_IPI<<16 | GIC_PRI_IPI<<8 | GIC_PRI_IPI), (void *)rbase_sgi + GICR_IPRIORITYR0 + (i / 4) * 4);
> +        //writel_relaxed(0x0, (void *)rbase + GICR_IPRIORITYR0 + (i / 4) * 4);
> +        //writel_relaxed(0xa0a0a0a0, (void *)rbase + GICR_IPRIORITYR0 + (i / 4) * 4);

What?


> +
> +    for (i = 16; i < 32; i += 4)
> +        writel_relaxed((GIC_PRI_IRQ<<24 | GIC_PRI_IRQ<<16 | GIC_PRI_IRQ<<8 | GIC_PRI_IRQ), (void *)rbase_sgi + GICR_IPRIORITYR0 + (i / 4) * 4);
> +
> +    /*
> +     * Disable all PPI interrupts, ensure all SGI interrupts are
> +     * enabled.
> +     */
> +    writel_relaxed(0xffff0000, (void *)rbase_sgi + GICR_ICENABLER0);
> +    writel_relaxed(0x0000ffff, (void *)rbase_sgi + GICR_ISENABLER0);
> +
> +    gic_redist_wait_for_rwp();
> +
> +    /* Enable system registers */
> +    gic_enable_sre();
> +
> +    WRITE_SYSREG32(0, ICC_BPR1_EL1);
> +    /* Set priority mask register */
> +    WRITE_SYSREG32(DEFAULT_PMR_VALUE, ICC_PMR_EL1);
> +
> +    /* EOI drops priority too (mode 0) */
> +    WRITE_SYSREG32(GICC_CTLR_EL1_EOImode_drop, ICC_CTLR_EL1);
> +
> +    WRITE_SYSREG32(1, ICC_IGRPEN1_EL1);
> +}
> +
> +static void gic_cpu_disable(void)
> +{
> +    WRITE_SYSREG32(0, ICC_CTLR_EL1);
> +}
> +
> +static void __cpuinit gic_hyp_init(void)
> +{
> +    uint32_t vtr;
> +
> +    vtr = READ_SYSREG32(ICH_VTR_EL2);
> +    nr_lrs  = (vtr & GICH_VTR_NRLRGS) + 1;
> +    nr_priorities = ((vtr >> GICH_VTR_PRIBITS_SHIFT) & GICH_VTR_PRIBITS_MASK) + 1;
> +
> +    WRITE_SYSREG32(GICH_VMCR_EOI | GICH_VMCR_VENG1, ICH_VMCR_EL2);
> +    WRITE_SYSREG32(GICH_HCR_VGRP1EIE | GICH_HCR_EN, ICH_HCR_EL2);
> +
> +    update_cpu_lr_mask();
> +    vtr = READ_SYSREG32(ICH_HCR_EL2);
> +}
> +
> +/* Set up the per-CPU parts of the GIC for a secondary CPU */
> +static int __cpuinit gic_init_secondary_cpu(struct notifier_block *nfb,
> +                                       unsigned long action, void *hcpu)
> +{
> +    if (action == CPU_STARTING)
> +    {
> +        spin_lock(&gic.lock);
> +        gic_cpu_init();
> +        gic_hyp_init();
> +        spin_unlock(&gic.lock);
> +    }
> +    return NOTIFY_DONE;
> +}
> +
> +static struct notifier_block gic_cpu_nb = {
> +    .notifier_call = gic_init_secondary_cpu,
> +    .priority = 100
> +};
> +
> +static void gic_smp_init(void)
> +{
> +   register_cpu_notifier(&gic_cpu_nb);
> +}
> +
> +static void __cpuinit gic_hyp_disable(void)
> +{
> +    uint32_t vtr;
> +    vtr = READ_SYSREG32(ICH_HCR_EL2);
> +    vtr &= ~0x1;
> +    WRITE_SYSREG32( vtr, ICH_HCR_EL2);
> +}
> +
> +static u16 gic_compute_target_list(int *base_cpu, const struct cpumask *mask,
> +                                   u64 cluster_id)
> +{
> +    int cpu = *base_cpu;
> +    u64 mpidr = cpu_logical_map(cpu);
> +    u16 tlist = 0;
> +
> +    while (cpu < nr_cpu_ids) {
> +        /*
> +         * If we ever get a cluster of more than 16 CPUs, just
> +         * scream and skip that CPU.
> +         */
> +        tlist |= 1 << (mpidr & 0xf);
> +
> +        cpu = cpumask_next(cpu, mask);
> +        mpidr = cpu_logical_map(cpu);
> +
> +        if (cluster_id != (mpidr & ~0xffUL)) {
> +            cpu--;
> +            goto out;
> +        }
> +    }
> +out:
> +    *base_cpu = cpu;
> +    return tlist;
> +}
> +
> +static void send_sgi(u64 cluster_id, u16 tlist, unsigned int irq)
> +{
> +    u64 val;
> +
> +    val  = (cluster_id & 0xff00ff0000UL) << 16; /* Aff3 + Aff2 */
> +    val |= (cluster_id & 0xff00) << 8;          /* Aff1 */
> +    val |= irq << 24;
> +    val |= tlist;
> +
> +    WRITE_SYSREG(val, ICC_SGI1R_EL1);   
> +}
> +
> +static void gic_send_sgi(const cpumask_t *cpumask, enum gic_sgi sgi)
> +{
> +    int cpu = 0;
> +
> +    dsb();
> +
> +    for_each_cpu(cpu, cpumask) {
> +        u64 cluster_id = cpu_logical_map(cpu) & ~0xffUL;
> +        u16 tlist;
> +
> +        tlist = gic_compute_target_list(&cpu, cpumask, cluster_id);
> +        send_sgi(cluster_id, tlist, sgi);
> +    }
> +}
> +
> +/* Shut down the per-CPU GIC interface */
> +static void gic_disable_interface(void)
> +{
> +    ASSERT(!local_irq_is_enabled());
> +
> +    spin_lock(&gic.lock);
> +    gic_cpu_disable();
> +    gic_hyp_disable();
> +    spin_unlock(&gic.lock);
> +}
> +
> +static void gic_update_lr(int lr, unsigned int virtual_irq,
> +        unsigned int state, unsigned int priority)
> +{
> +    u64 maintenance_int = GICH_LR_MAINTENANCE_IRQ;
> +    u64 grp = GICH_LR_GRP1;
> +    u64 val = 0;
> +
> +    BUG_ON(lr >= nr_lrs);
> +    BUG_ON(lr < 0);
> +
> +    val =  ((((u64)state) & 0x3) << GICH_LR_STATE_SHIFT) | grp | maintenance_int |
> +        ((((u64)priority) & 0xff) << GICH_LR_PRIORITY_SHIFT) |
> +        (((u64)virtual_irq & GICH_LR_VIRTUAL_MASK) << GICH_LR_VIRTUAL_SHIFT) |
> +        (((u64)virtual_irq & GICH_LR_PHYSICAL_MASK) << GICH_LR_PHYSICAL_SHIFT);
> +
> +    gich_write_lr(lr, val);
> +}

This function has to change after 

http://marc.info/?l=xen-devel&m=139523241201086

Re: [RFC PATCH v1 08/10] xen/arm: Add support for GIC v3

[Vijay Kilari]

On Thu, Mar 20, 2014 at 6:07 PM, Stefano Stabellini
<stefano.stabellini@eu.citrix.com> wrote:
> On Wed, 19 Mar 2014, vijay.kilari@gmail.com wrote:
>> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>>
>> Add support for GIC v3 specification.
>> This driver assumes that ARE and SRE
>> is enable by default.
>>
>> Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>> ---
>>  xen/arch/arm/Makefile             |    2 +-
>>  xen/arch/arm/gic-v3.c             |  944 +++++++++++++++++++++++++++++++++++++
>>  xen/arch/arm/gic.c                |   12 +
>>  xen/include/asm-arm/domain.h      |    4 +
>>  xen/include/asm-arm/gic.h         |    9 +
>>  xen/include/asm-arm/gic_v3_defs.h |  211 +++++++++
>>  6 files changed, 1181 insertions(+), 1 deletion(-)
>>
>> diff --git a/xen/arch/arm/Makefile b/xen/arch/arm/Makefile
>> index 20f59f4..a11c699 100644
>> --- a/xen/arch/arm/Makefile
>> +++ b/xen/arch/arm/Makefile
>> @@ -10,7 +10,7 @@ obj-y += vpsci.o
>>  obj-y += domctl.o
>>  obj-y += sysctl.o
>>  obj-y += domain_build.o
>> -obj-y += gic.o gic-v2.o
>> +obj-y += gic.o gic-v2.o gic-v3.o
>>  obj-y += io.o
>>  obj-y += irq.o
>>  obj-y += kernel.o
>> diff --git a/xen/arch/arm/gic-v3.c b/xen/arch/arm/gic-v3.c
>> new file mode 100644
>> index 0000000..81a36ba
>> --- /dev/null
>> +++ b/xen/arch/arm/gic-v3.c
>> @@ -0,0 +1,944 @@
>> +/*
>> + * xen/arch/arm/gic-v3.c
>> + *
>> + * ARM Generic Interrupt Controller support v3 version
>> + * based on xen/arch/arm/gic-v2.c
>> + *
>> + * Vijaya Kumar K <vijaya.kumar@caviumnetworks.com>
>> + * Copyright (c) 2014 Cavium Inc.
>> + *
>> + * This program is free software; you can redistribute it and/or modify
>> + * it under the terms of the GNU General Public License as published by
>> + * the Free Software Foundation; either version 2 of the License, or
>> + * (at your option) any later version.
>> + *
>> + * This program is distributed in the hope that it will be useful,
>> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
>> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
>> + * GNU General Public License for more details.
>> + */
>> +
>> +#include <xen/config.h>
>> +#include <xen/lib.h>
>> +#include <xen/init.h>
>> +#include <xen/cpu.h>
>> +#include <xen/mm.h>
>> +#include <xen/irq.h>
>> +#include <xen/sched.h>
>> +#include <xen/errno.h>
>> +#include <xen/serial.h>
>> +#include <xen/softirq.h>
>> +#include <xen/list.h>
>> +#include <xen/device_tree.h>
>> +#include <asm/p2m.h>
>> +#include <asm/domain.h>
>> +#include <asm/platform.h>
>> +
>> +#include <asm/gic_v3_defs.h>
>> +#include <asm/gic.h>
>> +#include <asm/io.h>
>> +
>> +struct rdist_region {
>> +    paddr_t rdist_base;
>> +    paddr_t rdist_base_size;
>> +    void __iomem *map_rdist_base;
>> +};
>> +
>> +/* Global state */
>> +static struct {
>> +    paddr_t dbase;            /* Address of distributor registers */
>> +    paddr_t dbase_size;
>> +    void __iomem *map_dbase;  /* Mapped address of distributor registers */
>> +    struct rdist_region *rdist_regions;
>> +    u32  rdist_stride;
>> +    unsigned int rdist_count; /* Number of rdist regions count */
>> +    unsigned int lines;       /* Number of interrupts (SPIs + PPIs + SGIs) */
>> +    struct dt_irq maintenance;
>> +    unsigned int cpus;
>> +    int hw_version;
>> +    spinlock_t lock;
>> +} gic;
>> +
>> +struct gic_state_data {
>> +    uint32_t gic_hcr, gic_vmcr;
>> +    uint32_t gic_apr0[4];
>> +    uint32_t gic_apr1[4];
>> +    uint64_t gic_lr[16];
>> +};
>> +
>> +#define GICD ((volatile unsigned char *) gic.map_dbase)
>> +/* Only one region is implemented which is enough for 0-31 cpus */
>> +#define GICR ((volatile unsigned char *) gic.rdist_regions[0].map_rdist_base)
>> +
>> +/* per-cpu re-distributor base */
>> +static DEFINE_PER_CPU(paddr_t, rbase);
>> +static DEFINE_PER_CPU(paddr_t, phy_rbase);
>> +
>> +static unsigned nr_lrs;
>> +static uint32_t nr_priorities;
>> +
>> +/* The GIC mapping of CPU interfaces does not necessarily match the
>> + * logical CPU numbering. Let's use mapping as returned by the GIC
>> + * itself
>> + */
>> +
>> +#define gic_data_rdist_rd_base()        (this_cpu(rbase))
>> +#define gic_data_rdist_sgi_base()       (gic_data_rdist_rd_base() + SZ_64K)
>> +
>> +static inline u64 read_cpuid_mpidr(void)
>> +{
>> +   return READ_SYSREG(MPIDR_EL1);
>> +}
>> +
>> +static u64 gich_read_lr(int lr)
>> +{
>> +    switch (lr)
>> +    {
>> +        case 0: /* ICH_LRn is 64 bit */
>> +            return READ_SYSREG(ICH_LR0_EL2);
>> +            break;
>> +        case 1:
>> +            return READ_SYSREG(ICH_LR1_EL2);
>> +            break;
>> +        case 2:
>> +            return READ_SYSREG(ICH_LR2_EL2);
>> +            break;
>> +        case 3:
>> +            return READ_SYSREG(ICH_LR3_EL2);
>> +            break;
>> +        case 4:
>> +            return READ_SYSREG(ICH_LR4_EL2);
>> +            break;
>> +        case 5:
>> +            return READ_SYSREG(ICH_LR5_EL2);
>> +            break;
>> +        case 6:
>> +            return READ_SYSREG(ICH_LR6_EL2);
>> +            break;
>> +        case 7:
>> +            return READ_SYSREG(ICH_LR7_EL2);
>> +            break;
>> +        case 8:
>> +            return READ_SYSREG(ICH_LR8_EL2);
>> +            break;
>> +        case 9:
>> +            return READ_SYSREG(ICH_LR9_EL2);
>> +            break;
>> +        case 10:
>> +            return READ_SYSREG(ICH_LR10_EL2);
>> +            break;
>> +        case 11:
>> +            return READ_SYSREG(ICH_LR11_EL2);
>> +            break;
>> +        case 12:
>> +            return READ_SYSREG(ICH_LR12_EL2);
>> +            break;
>> +        case 13:
>> +            return READ_SYSREG(ICH_LR13_EL2);
>> +            break;
>> +        case 14:
>> +            return READ_SYSREG(ICH_LR14_EL2);
>> +            break;
>> +        case 15:
>> +            return READ_SYSREG(ICH_LR15_EL2);
>> +            break;
>> +        default:
>> +            return 0;
>> +    }
>> +}
>> +
>> +static void gich_write_lr(int lr, u64 val)
>> +{
>> +    switch (lr)
>> +    {
>> +        case 0:
>> +           WRITE_SYSREG(val, ICH_LR0_EL2);
>> +           break;
>> +        case 1:
>> +           WRITE_SYSREG(val, ICH_LR1_EL2);
>> +           break;
>> +        case 2:
>> +           WRITE_SYSREG(val, ICH_LR2_EL2);
>> +           break;
>> +        case 3:
>> +           WRITE_SYSREG(val, ICH_LR3_EL2);
>> +           break;
>> +        case 4:
>> +           WRITE_SYSREG(val, ICH_LR4_EL2);
>> +           break;
>> +        case 5:
>> +           WRITE_SYSREG(val, ICH_LR5_EL2);
>> +           break;
>> +        case 6:
>> +           WRITE_SYSREG(val, ICH_LR6_EL2);
>> +           break;
>> +        case 7:
>> +           WRITE_SYSREG(val, ICH_LR7_EL2);
>> +           break;
>> +        case 8:
>> +           WRITE_SYSREG(val, ICH_LR8_EL2);
>> +           break;
>> +        case 9:
>> +           WRITE_SYSREG(val, ICH_LR9_EL2);
>> +           break;
>> +        case 10:
>> +           WRITE_SYSREG(val, ICH_LR10_EL2);
>> +           break;
>> +        case 11:
>> +           WRITE_SYSREG(val, ICH_LR11_EL2);
>> +           break;
>> +        case 12:
>> +           WRITE_SYSREG(val, ICH_LR12_EL2);
>> +           break;
>> +        case 13:
>> +           WRITE_SYSREG(val, ICH_LR13_EL2);
>> +           break;
>> +        case 14:
>> +           WRITE_SYSREG(val, ICH_LR14_EL2);
>> +           break;
>> +        case 15:
>> +           WRITE_SYSREG(val, ICH_LR15_EL2);
>> +           break;
>> +        default:
>> +           return;
>> +    }
>> +}
>> +
>> +static void gic_enable_sre(void)
>> +{
>> +    uint32_t val;
>> +
>> +    val = READ_SYSREG32(ICC_SRE_EL2);
>> +    val |= GICC_SRE_EL2_SRE | GICC_SRE_EL2_ENEL1 | GICC_SRE_EL2_DFB | GICC_SRE_EL2_DIB;
>> +    WRITE_SYSREG32(val, ICC_SRE_EL2);
>> +    isb();
>> +}
>> +
>> +/* Wait for completion of a distributor change */
>> +static void gic_do_wait_for_rwp(paddr_t base)
>> +{
>> +    u32 val;
>> +    do {
>> +        val = readl_relaxed((void *)base + GICD_CTLR);
>> +        val = readl_relaxed(GICD + GICD_CTLR);
>> +        val = GICD[GICD_CTLR];
>> +        cpu_relax();
>> +    } while (val & GICD_CTLR_RWP);
>> +}
>
> As much I think that this busy loop is terrible unfortunately it is
> part of the spec :-(
> Might be worth adding a comment on the function to explain why and when
> it is required.
> It is also worth considering whether it makes sense to use the
> notification of command completion by interrupt instead.
>
As per spec, we have to wait for this bit to be set to be
sure that register programming is done. IMO, there is no
interrupt to notify it and also we have to be sure
that programming these registers are complete before we proceed.

I will add a comment

>
>
>> +static void gic_dist_wait_for_rwp(void)
>> +{
>> +    gic_do_wait_for_rwp((paddr_t)GICD);
>> +}
>> +
>> +static void gic_redist_wait_for_rwp(void)
>> +{
>> +    gic_do_wait_for_rwp(gic_data_rdist_rd_base());
>> +}
>> +
>> +static void gic_wait_for_rwp(int irq)
>> +{
>> +    if (irq < 32)
>> +         gic_redist_wait_for_rwp();
>> +    else
>> +         gic_dist_wait_for_rwp();
>> +}
>> +
>> +static unsigned int gic_mask_cpu(const cpumask_t *cpumask)
>> +{
>> +    unsigned int cpu;
>> +    cpumask_t possible_mask;
>> +
>> +    cpumask_and(&possible_mask, cpumask, &cpu_possible_map);
>> +    cpu = cpumask_any(&possible_mask);
>> +    return cpu;
>> +}
>> +
>> +static unsigned int gic_nr_lines(void)
>> +{
>> +    return gic.lines;
>> +}
>> +
>> +static unsigned int gic_nr_lrs(void)
>> +{
>> +    return nr_lrs;
>> +}
>> +
>> +static void write_aprn_regs(struct gic_state_data *d)
>> +{
>> +    switch(nr_priorities)
>> +    {
>> +        case 7:
>> +            WRITE_SYSREG32(d->gic_apr0[2], ICH_AP0R2_EL2);
>> +            WRITE_SYSREG32(d->gic_apr1[2], ICH_AP1R2_EL2);
>> +        case 6:
>> +            WRITE_SYSREG32(d->gic_apr0[1], ICH_AP0R1_EL2);
>> +            WRITE_SYSREG32(d->gic_apr1[1], ICH_AP1R1_EL2);
>> +        case 5:
>> +            WRITE_SYSREG32(d->gic_apr0[0], ICH_AP0R0_EL2);
>> +            WRITE_SYSREG32(d->gic_apr1[0], ICH_AP1R0_EL2);
>> +            break;
>> +        default:
>> +          panic("Write Undefined active priorities \n");
>> +    }
>> +}
>> +
>> +static void read_aprn_regs(struct gic_state_data *d)
>> +{
>> +    switch(nr_priorities)
>> +    {
>> +        case 7:
>> +            d->gic_apr0[2] = READ_SYSREG32(ICH_AP0R2_EL2);
>> +            d->gic_apr1[2] = READ_SYSREG32(ICH_AP1R2_EL2);
>> +        case 6:
>> +            d->gic_apr0[1] = READ_SYSREG32(ICH_AP0R1_EL2);
>> +            d->gic_apr1[1] = READ_SYSREG32(ICH_AP1R1_EL2);
>> +        case 5:
>> +            d->gic_apr0[0] = READ_SYSREG32(ICH_AP0R0_EL2);
>> +            d->gic_apr1[0] = READ_SYSREG32(ICH_AP1R0_EL2);
>> +            break;
>> +        default:
>> +          panic("Read Undefined active priorities \n");
>> +    }
>> +}
>> +
>> +static int gic_state_init(struct vcpu *v)
>> +{
>> +     v->arch.gic_state = (struct gic_state_data *)xzalloc(struct gic_state_data);
>> +     if(!v->arch.gic_state)
>> +        return -ENOMEM;
>> +     return 0;
>> +}
>> +
>> +static void save_state(struct vcpu *v)
>> +{
>> +    int i;
>> +    struct gic_state_data *d;
>> +    d = (struct gic_state_data *)v->arch.gic_state;
>> +
>> +    /* No need for spinlocks here because interrupts are disabled around
>> +     * this call and it only accesses struct vcpu fields that cannot be
>> +     * accessed simultaneously by another pCPU.
>> +     */
>> +    for ( i=0; i<nr_lrs; i++)
>> +        d->gic_lr[i] = gich_read_lr(i);
>> +
>> +    read_aprn_regs(d);
>> +
>> +    d->gic_vmcr = READ_SYSREG32(ICH_VMCR_EL2);
>> +}
>> +
>> +static void restore_state(struct vcpu *v)
>> +{
>> +    int i;
>> +    struct gic_state_data *d;
>> +    d = (struct gic_state_data *)v->arch.gic_state;
>> +
>> +    for ( i=0; i<nr_lrs; i++)
>> +        gich_write_lr(i, d->gic_lr[i]);
>> +
>> +    write_aprn_regs(d);
>> +
>> +    WRITE_SYSREG32(d->gic_vmcr, ICH_VMCR_EL2);
>> +}
>> +
>> +static void gic_dump_state(struct vcpu *v)
>> +{
>> +    int i;
>> +    struct gic_state_data *d;
>> +    d = (struct gic_state_data *)v->arch.gic_state;
>> +    if ( v == current )
>> +    {
>> +        for ( i = 0; i < nr_lrs; i++ )
>> +            printk("   HW_LR[%d]=%lx\n", i, gich_read_lr(i));
>> +    }
>> +    else
>> +    {
>> +        for ( i = 0; i < nr_lrs; i++ )
>> +            printk("   VCPU_LR[%d]=%lx\n", i, d->gic_lr[i]);
>> +    }
>> +}
>> +
>> +static void gic_enable_irq(int irq)
>> +{
>> +    uint32_t enabler;
>> +
>> +    /* Enable routing */
>> +    if(irq > 31)
>> +    {
>> +        enabler = readl_relaxed(GICD + GICD_ISENABLER + (irq / 32) * 4);
>> +        writel_relaxed(enabler | (1u << (irq % 32)), GICD + GICD_ISENABLER + (irq / 32) * 4);
>> +    }
>> +    else
>> +    {
>> +        enabler = readl_relaxed((void *)gic_data_rdist_sgi_base() + GICR_ISENABLER0);
>> +        writel_relaxed(enabler | (1u << irq), (void *)gic_data_rdist_sgi_base() + GICR_ISENABLER0);
>> +    }
>> +    gic_wait_for_rwp(irq);
>> +}
>> +
>> +static void gic_disable_irq(int irq)
>> +{
>> +    /* Disable routing */
>> +    if(irq > 31)
>> +        writel_relaxed(1u << (irq % 32), GICD + GICD_ICENABLER + ((irq / 32) * 4));
>> +    else
>> +        writel_relaxed(1u << irq, (void *)gic_data_rdist_sgi_base() + GICR_ICENABLER0);
>> +}
>> +
>> +static void gic_eoi_irq(int irq)
>> +{
>> +    /* Lower the priority */
>> +    WRITE_SYSREG32(irq, ICC_EOIR1_EL1);
>> +}
>> +
>> +static void gic_dir_irq(int irq)
>> +{
>> +    /* Deactivate */
>> +    WRITE_SYSREG32(irq, ICC_DIR_EL1);
>> +}
>> +
>> +static unsigned int gic_ack_irq(void)
>> +{
>> +    return (READ_SYSREG32(ICC_IAR1_EL1) & GICC_IA_IRQ);
>> +}
>> +
>> +static u64 gic_mpidr_to_affinity(u64 mpidr)
>> +{
>> +    /* Make sure we don't broadcast the interrupt */
>> +    return mpidr & ~GICD_IROUTER_SPI_MODE_ANY;
>> +}
>> +
>> +/*
>> + * - needs to be called with gic.lock held
>> + * - needs to be called with a valid cpu_mask, ie each cpu in the mask has
>> + * already called gic_cpu_init
>> + */
>> +static void gic_set_irq_property(unsigned int irq, bool_t level,
>> +                                   const cpumask_t *cpu_mask,
>> +                                   unsigned int priority)
>> +{
>> +    uint32_t cfg, edgebit;
>> +    u64 affinity;
>> +    unsigned int cpu = gic_mask_cpu(cpu_mask);
>> +    paddr_t rebase;
>> +
>> +
>> +    /* Set edge / level */
>> +    if (irq < 16)
>> +     /* SGI's are always edge-triggered not need to call GICD_ICFGR0 */
>> +       cfg = readl_relaxed((void *)gic_data_rdist_sgi_base() + GICR_ICFGR0);
>> +    else if (irq < 32)
>> +       cfg = readl_relaxed((void *)gic_data_rdist_sgi_base() + GICR_ICFGR1);
>> +    else
>> +       cfg = readl_relaxed(GICD + GICD_ICFGR + (irq / 16) * 4);
>> +
>> +    edgebit = 2u << (2 * (irq % 16));
>> +    if ( level )
>> +        cfg &= ~edgebit;
>> +    else
>> +        cfg |= edgebit;
>> +
>> +    if (irq < 16)
>> +       writel_relaxed(cfg, (void *)gic_data_rdist_sgi_base() + GICR_ICFGR0);
>> +    else if (irq < 32)
>> +       writel_relaxed(cfg, (void *)gic_data_rdist_sgi_base() + GICR_ICFGR1);
>> +    else
>> +       writel_relaxed(cfg, GICD + GICD_ICFGR + (irq / 16) * 4);
>> +
>> +
>> +    /* need to check if ARE is set to access IROUTER */
>> +    affinity = gic_mpidr_to_affinity(cpu_logical_map(cpu));
>> +    if (irq > 31)
>> +        writeq_relaxed(affinity, (GICD + GICD_IROUTER + irq * 8));
>> +
>> +    /* Set priority */
>> +    if (irq < 32)
>> +    {
>> +     rebase = gic_data_rdist_sgi_base();
>
> code style
>
>> +        writeb_relaxed(priority, (void *)rebase + GICR_IPRIORITYR0 + irq);
>> +    }
>> +    else
>> +    {
>> +        writeb_relaxed(priority, GICD + GICD_IPRIORITYR + irq);
>> +
>> +    }
>> +}
>> +
>> +static void __init gic_dist_init(void)
>> +{
>> +    uint32_t type;
>> +    u64 affinity;
>> +    int i;
>> +
>> +    /* Disable the distributor */
>> +    writel_relaxed(0, GICD + GICD_CTLR);
>> +
>> +    type = readl_relaxed(GICD + GICD_TYPER);
>> +    gic.lines = 32 * ((type & GICD_TYPE_LINES) + 1);
>> +
>> +    printk("GIC: %d lines, (IID %8.8x).\n",
>> +           gic.lines, readl_relaxed(GICD + GICD_IIDR));
>> +
>> +    /* Default all global IRQs to level, active low */
>> +    for ( i = 32; i < gic.lines; i += 16 )
>> +        writel_relaxed(0, GICD + GICD_ICFGR + (i / 16) * 4);
>> +
>> +    /* Default priority for global interrupts */
>> +    for ( i = 32; i < gic.lines; i += 4 )
>> +        writel_relaxed((GIC_PRI_IRQ<<24 | GIC_PRI_IRQ<<16 | GIC_PRI_IRQ<<8 | GIC_PRI_IRQ), GICD + GICD_IPRIORITYR + (i / 4) * 4);
>> +
>> +    /* Disable all global interrupts */
>> +    for ( i = 32; i < gic.lines; i += 32 )
>> +        writel_relaxed(0xffffffff, GICD + GICD_ICENABLER + (i / 32) * 4);
>> +
>> +    gic_dist_wait_for_rwp();
>> +
>> +    /* Turn on the distributor */
>> +    writel_relaxed(GICD_CTL_ENABLE | GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1, GICD + GICD_CTLR);
>> +
>> +    /* Route all global IRQs to this CPU */
>> +    affinity = gic_mpidr_to_affinity(read_cpuid_mpidr());
>> +    for ( i = 31; i < gic.lines; i++ )
>> +        writeq_relaxed(affinity, GICD + GICD_IROUTER + i * 8);
>> +}
>> +
>> +static void gic_enable_redist(void)
>> +{
>> +    paddr_t rbase;
>> +    u32 val;
>> +
>> +    rbase = this_cpu(rbase);
>> +
>> +    /* Wake up this CPU redistributor */
>> +    val = readl_relaxed((void *)rbase + GICR_WAKER);
>> +    val &= ~GICR_WAKER_ProcessorSleep;
>> +    writel_relaxed(val, (void *)rbase + GICR_WAKER);
>> +
>> +    do {
>> +         val = readl_relaxed((void *)rbase + GICR_WAKER);
>> +         cpu_relax();
>> +    } while (val & GICR_WAKER_ChildrenAsleep);
>> +}
>> +
>> +static int __init gic_populate_rdist(void)
>> +{
>> +    u64 mpidr = cpu_logical_map(smp_processor_id());
>> +    u64 typer;
>> +    u64 aff;
>> +    int i;
>> +    uint32_t reg;
>> +
>> +    aff  = mpidr & ((1 << 24) - 1);
>> +    aff |= (mpidr >> 8) & (0xffUL << 24);
>> +
>> +    for (i = 0; i < gic.rdist_count; i++) {
>> +        uint32_t ptr = 0;
>> +
>> +        reg = readl_relaxed(GICR + ptr + GICR_PIDR0);
>> +        if ((reg & 0xff) != GICR_PIDR0_GICv3) { /* We're in trouble... */
>> +            printk("No redistributor present @%x\n", ptr);
>> +            break;
>> +        }
>> +
>> +        do {
>> +            typer = readq_relaxed(GICR + ptr + GICR_TYPER);
>> +            if ((typer >> 32) == aff) {
>> +
>> +                this_cpu(rbase) = (u64)GICR + ptr;
>> +                this_cpu(phy_rbase) = gic.rdist_regions[i].rdist_base + ptr;
>> +
>> +                printk("CPU%d: found redistributor %llx region %d\n",
>> +                            smp_processor_id(), (unsigned long long) mpidr, i);
>> +                return 0;
>> +            }
>> +
>> +            if (gic.rdist_stride) {
>> +                ptr += gic.rdist_stride;
>> +            } else {
>> +                ptr += SZ_64K * 2; /* Skip RD_base + SGI_base */
>> +                if (typer & GICR_TYPER_VLPIS)
>> +                    ptr += SZ_64K * 2; /* Skip VLPI_base + reserved page */
>> +                }
>> +            } while (!(typer & GICR_TYPER_LAST));
>
> code style: the indentation is wrong
>
OK.
>
>> +        }
>> +
>> +        /* We couldn't even deal with ourselves... */
>> +        printk("CPU%d: mpidr %lx has no re-distributor!\n",
>> +                  smp_processor_id(), (unsigned long)mpidr);
>> +        return -ENODEV;
>> +}
>> +
>> +static void __cpuinit gic_cpu_init(void)
>> +{
>> +    int i;
>> +    paddr_t rbase_sgi;
>> +
>> +    /* Register ourselves with the rest of the world */
>> +    if (gic_populate_rdist())
>> +        return;
>> +
>> +    gic_enable_redist();
>> +
>> +    rbase_sgi = gic_data_rdist_sgi_base();
>> +
>> +    /*
>> +     * Set priority on PPI and SGI interrupts
>> +     */
>> +    for (i = 0; i < 16; i += 4)
>> +        writel_relaxed((GIC_PRI_IPI<<24 | GIC_PRI_IPI<<16 | GIC_PRI_IPI<<8 | GIC_PRI_IPI), (void *)rbase_sgi + GICR_IPRIORITYR0 + (i / 4) * 4);
>> +        //writel_relaxed(0x0, (void *)rbase + GICR_IPRIORITYR0 + (i / 4) * 4);
>> +        //writel_relaxed(0xa0a0a0a0, (void *)rbase + GICR_IPRIORITYR0 + (i / 4) * 4);
>
> What?
>
>
>> +
>> +    for (i = 16; i < 32; i += 4)
>> +        writel_relaxed((GIC_PRI_IRQ<<24 | GIC_PRI_IRQ<<16 | GIC_PRI_IRQ<<8 | GIC_PRI_IRQ), (void *)rbase_sgi + GICR_IPRIORITYR0 + (i / 4) * 4);
>> +
>> +    /*
>> +     * Disable all PPI interrupts, ensure all SGI interrupts are
>> +     * enabled.
>> +     */
>> +    writel_relaxed(0xffff0000, (void *)rbase_sgi + GICR_ICENABLER0);
>> +    writel_relaxed(0x0000ffff, (void *)rbase_sgi + GICR_ISENABLER0);
>> +
>> +    gic_redist_wait_for_rwp();
>> +
>> +    /* Enable system registers */
>> +    gic_enable_sre();
>> +
>> +    WRITE_SYSREG32(0, ICC_BPR1_EL1);
>> +    /* Set priority mask register */
>> +    WRITE_SYSREG32(DEFAULT_PMR_VALUE, ICC_PMR_EL1);
>> +
>> +    /* EOI drops priority too (mode 0) */
>> +    WRITE_SYSREG32(GICC_CTLR_EL1_EOImode_drop, ICC_CTLR_EL1);
>> +
>> +    WRITE_SYSREG32(1, ICC_IGRPEN1_EL1);
>> +}
>> +
>> +static void gic_cpu_disable(void)
>> +{
>> +    WRITE_SYSREG32(0, ICC_CTLR_EL1);
>> +}
>> +
>> +static void __cpuinit gic_hyp_init(void)
>> +{
>> +    uint32_t vtr;
>> +
>> +    vtr = READ_SYSREG32(ICH_VTR_EL2);
>> +    nr_lrs  = (vtr & GICH_VTR_NRLRGS) + 1;
>> +    nr_priorities = ((vtr >> GICH_VTR_PRIBITS_SHIFT) & GICH_VTR_PRIBITS_MASK) + 1;
>> +
>> +    WRITE_SYSREG32(GICH_VMCR_EOI | GICH_VMCR_VENG1, ICH_VMCR_EL2);
>> +    WRITE_SYSREG32(GICH_HCR_VGRP1EIE | GICH_HCR_EN, ICH_HCR_EL2);
>> +
>> +    update_cpu_lr_mask();
>> +    vtr = READ_SYSREG32(ICH_HCR_EL2);
>> +}
>> +
>> +/* Set up the per-CPU parts of the GIC for a secondary CPU */
>> +static int __cpuinit gic_init_secondary_cpu(struct notifier_block *nfb,
>> +                                       unsigned long action, void *hcpu)
>> +{
>> +    if (action == CPU_STARTING)
>> +    {
>> +        spin_lock(&gic.lock);
>> +        gic_cpu_init();
>> +        gic_hyp_init();
>> +        spin_unlock(&gic.lock);
>> +    }
>> +    return NOTIFY_DONE;
>> +}
>> +
>> +static struct notifier_block gic_cpu_nb = {
>> +    .notifier_call = gic_init_secondary_cpu,
>> +    .priority = 100
>> +};
>> +
>> +static void gic_smp_init(void)
>> +{
>> +   register_cpu_notifier(&gic_cpu_nb);
>> +}
>> +
>> +static void __cpuinit gic_hyp_disable(void)
>> +{
>> +    uint32_t vtr;
>> +    vtr = READ_SYSREG32(ICH_HCR_EL2);
>> +    vtr &= ~0x1;
>> +    WRITE_SYSREG32( vtr, ICH_HCR_EL2);
>> +}
>> +
>> +static u16 gic_compute_target_list(int *base_cpu, const struct cpumask *mask,
>> +                                   u64 cluster_id)
>> +{
>> +    int cpu = *base_cpu;
>> +    u64 mpidr = cpu_logical_map(cpu);
>> +    u16 tlist = 0;
>> +
>> +    while (cpu < nr_cpu_ids) {
>> +        /*
>> +         * If we ever get a cluster of more than 16 CPUs, just
>> +         * scream and skip that CPU.
>> +         */
>> +        tlist |= 1 << (mpidr & 0xf);
>> +
>> +        cpu = cpumask_next(cpu, mask);
>> +        mpidr = cpu_logical_map(cpu);
>> +
>> +        if (cluster_id != (mpidr & ~0xffUL)) {
>> +            cpu--;
>> +            goto out;
>> +        }
>> +    }
>> +out:
>> +    *base_cpu = cpu;
>> +    return tlist;
>> +}
>> +
>> +static void send_sgi(u64 cluster_id, u16 tlist, unsigned int irq)
>> +{
>> +    u64 val;
>> +
>> +    val  = (cluster_id & 0xff00ff0000UL) << 16; /* Aff3 + Aff2 */
>> +    val |= (cluster_id & 0xff00) << 8;          /* Aff1 */
>> +    val |= irq << 24;
>> +    val |= tlist;
>> +
>> +    WRITE_SYSREG(val, ICC_SGI1R_EL1);
>> +}
>> +
>> +static void gic_send_sgi(const cpumask_t *cpumask, enum gic_sgi sgi)
>> +{
>> +    int cpu = 0;
>> +
>> +    dsb();
>> +
>> +    for_each_cpu(cpu, cpumask) {
>> +        u64 cluster_id = cpu_logical_map(cpu) & ~0xffUL;
>> +        u16 tlist;
>> +
>> +        tlist = gic_compute_target_list(&cpu, cpumask, cluster_id);
>> +        send_sgi(cluster_id, tlist, sgi);
>> +    }
>> +}
>> +
>> +/* Shut down the per-CPU GIC interface */
>> +static void gic_disable_interface(void)
>> +{
>> +    ASSERT(!local_irq_is_enabled());
>> +
>> +    spin_lock(&gic.lock);
>> +    gic_cpu_disable();
>> +    gic_hyp_disable();
>> +    spin_unlock(&gic.lock);
>> +}
>> +
>> +static void gic_update_lr(int lr, unsigned int virtual_irq,
>> +        unsigned int state, unsigned int priority)
>> +{
>> +    u64 maintenance_int = GICH_LR_MAINTENANCE_IRQ;
>> +    u64 grp = GICH_LR_GRP1;
>> +    u64 val = 0;
>> +
>> +    BUG_ON(lr >= nr_lrs);
>> +    BUG_ON(lr < 0);
>> +
>> +    val =  ((((u64)state) & 0x3) << GICH_LR_STATE_SHIFT) | grp | maintenance_int |
>> +        ((((u64)priority) & 0xff) << GICH_LR_PRIORITY_SHIFT) |
>> +        (((u64)virtual_irq & GICH_LR_VIRTUAL_MASK) << GICH_LR_VIRTUAL_SHIFT) |
>> +        (((u64)virtual_irq & GICH_LR_PHYSICAL_MASK) << GICH_LR_PHYSICAL_SHIFT);
>> +
>> +    gich_write_lr(lr, val);
>> +}
>
> This function has to change after
>
> http://marc.info/?l=xen-devel&m=139523241201086
>
OK. I will try to rebase on top of your patches if you consider that your
patches are complete

Re: [RFC PATCH v1 08/10] xen/arm: Add support for GIC v3

[Ian Campbell]

On Sat, 2014-03-22 at 15:37 +0530, Vijay Kilari wrote:

> >> +/* Wait for completion of a distributor change */
> >> +static void gic_do_wait_for_rwp(paddr_t base)
> >> +{
> >> +    u32 val;
> >> +    do {
> >> +        val = readl_relaxed((void *)base + GICD_CTLR);
> >> +        val = readl_relaxed(GICD + GICD_CTLR);

Are these both required for some reason? Looks like the second might be
a left over from a previous iteration?

> >> +        val = GICD[GICD_CTLR];
> >> +        cpu_relax();
> >> +    } while (val & GICD_CTLR_RWP);
> >> +}
> >

Ian.

Re: [RFC PATCH v1 08/10] xen/arm: Add support for GIC v3

[Stefano Stabellini]

On Sat, 22 Mar 2014, Vijay Kilari wrote:
> On Thu, Mar 20, 2014 at 6:07 PM, Stefano Stabellini
> <stefano.stabellini@eu.citrix.com> wrote:
> > On Wed, 19 Mar 2014, vijay.kilari@gmail.com wrote:
> >> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> >>
> >> Add support for GIC v3 specification.
> >> This driver assumes that ARE and SRE
> >> is enable by default.
> >>
> >> Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> >> ---
> >>  xen/arch/arm/Makefile             |    2 +-
> >>  xen/arch/arm/gic-v3.c             |  944 +++++++++++++++++++++++++++++++++++++
> >>  xen/arch/arm/gic.c                |   12 +
> >>  xen/include/asm-arm/domain.h      |    4 +
> >>  xen/include/asm-arm/gic.h         |    9 +
> >>  xen/include/asm-arm/gic_v3_defs.h |  211 +++++++++
> >>  6 files changed, 1181 insertions(+), 1 deletion(-)
> >>
> >> diff --git a/xen/arch/arm/Makefile b/xen/arch/arm/Makefile
> >> index 20f59f4..a11c699 100644
> >> --- a/xen/arch/arm/Makefile
> >> +++ b/xen/arch/arm/Makefile
> >> @@ -10,7 +10,7 @@ obj-y += vpsci.o
> >>  obj-y += domctl.o
> >>  obj-y += sysctl.o
> >>  obj-y += domain_build.o
> >> -obj-y += gic.o gic-v2.o
> >> +obj-y += gic.o gic-v2.o gic-v3.o
> >>  obj-y += io.o
> >>  obj-y += irq.o
> >>  obj-y += kernel.o
> >> diff --git a/xen/arch/arm/gic-v3.c b/xen/arch/arm/gic-v3.c
> >> new file mode 100644
> >> index 0000000..81a36ba
> >> --- /dev/null
> >> +++ b/xen/arch/arm/gic-v3.c
> >> @@ -0,0 +1,944 @@
> >> +/*
> >> + * xen/arch/arm/gic-v3.c
> >> + *
> >> + * ARM Generic Interrupt Controller support v3 version
> >> + * based on xen/arch/arm/gic-v2.c
> >> + *
> >> + * Vijaya Kumar K <vijaya.kumar@caviumnetworks.com>
> >> + * Copyright (c) 2014 Cavium Inc.
> >> + *
> >> + * This program is free software; you can redistribute it and/or modify
> >> + * it under the terms of the GNU General Public License as published by
> >> + * the Free Software Foundation; either version 2 of the License, or
> >> + * (at your option) any later version.
> >> + *
> >> + * This program is distributed in the hope that it will be useful,
> >> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> >> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> >> + * GNU General Public License for more details.
> >> + */
> >> +
> >> +#include <xen/config.h>
> >> +#include <xen/lib.h>
> >> +#include <xen/init.h>
> >> +#include <xen/cpu.h>
> >> +#include <xen/mm.h>
> >> +#include <xen/irq.h>
> >> +#include <xen/sched.h>
> >> +#include <xen/errno.h>
> >> +#include <xen/serial.h>
> >> +#include <xen/softirq.h>
> >> +#include <xen/list.h>
> >> +#include <xen/device_tree.h>
> >> +#include <asm/p2m.h>
> >> +#include <asm/domain.h>
> >> +#include <asm/platform.h>
> >> +
> >> +#include <asm/gic_v3_defs.h>
> >> +#include <asm/gic.h>
> >> +#include <asm/io.h>
> >> +
> >> +struct rdist_region {
> >> +    paddr_t rdist_base;
> >> +    paddr_t rdist_base_size;
> >> +    void __iomem *map_rdist_base;
> >> +};
> >> +
> >> +/* Global state */
> >> +static struct {
> >> +    paddr_t dbase;            /* Address of distributor registers */
> >> +    paddr_t dbase_size;
> >> +    void __iomem *map_dbase;  /* Mapped address of distributor registers */
> >> +    struct rdist_region *rdist_regions;
> >> +    u32  rdist_stride;
> >> +    unsigned int rdist_count; /* Number of rdist regions count */
> >> +    unsigned int lines;       /* Number of interrupts (SPIs + PPIs + SGIs) */
> >> +    struct dt_irq maintenance;
> >> +    unsigned int cpus;
> >> +    int hw_version;
> >> +    spinlock_t lock;
> >> +} gic;
> >> +
> >> +struct gic_state_data {
> >> +    uint32_t gic_hcr, gic_vmcr;
> >> +    uint32_t gic_apr0[4];
> >> +    uint32_t gic_apr1[4];
> >> +    uint64_t gic_lr[16];
> >> +};
> >> +
> >> +#define GICD ((volatile unsigned char *) gic.map_dbase)
> >> +/* Only one region is implemented which is enough for 0-31 cpus */
> >> +#define GICR ((volatile unsigned char *) gic.rdist_regions[0].map_rdist_base)
> >> +
> >> +/* per-cpu re-distributor base */
> >> +static DEFINE_PER_CPU(paddr_t, rbase);
> >> +static DEFINE_PER_CPU(paddr_t, phy_rbase);
> >> +
> >> +static unsigned nr_lrs;
> >> +static uint32_t nr_priorities;
> >> +
> >> +/* The GIC mapping of CPU interfaces does not necessarily match the
> >> + * logical CPU numbering. Let's use mapping as returned by the GIC
> >> + * itself
> >> + */
> >> +
> >> +#define gic_data_rdist_rd_base()        (this_cpu(rbase))
> >> +#define gic_data_rdist_sgi_base()       (gic_data_rdist_rd_base() + SZ_64K)
> >> +
> >> +static inline u64 read_cpuid_mpidr(void)
> >> +{
> >> +   return READ_SYSREG(MPIDR_EL1);
> >> +}
> >> +
> >> +static u64 gich_read_lr(int lr)
> >> +{
> >> +    switch (lr)
> >> +    {
> >> +        case 0: /* ICH_LRn is 64 bit */
> >> +            return READ_SYSREG(ICH_LR0_EL2);
> >> +            break;
> >> +        case 1:
> >> +            return READ_SYSREG(ICH_LR1_EL2);
> >> +            break;
> >> +        case 2:
> >> +            return READ_SYSREG(ICH_LR2_EL2);
> >> +            break;
> >> +        case 3:
> >> +            return READ_SYSREG(ICH_LR3_EL2);
> >> +            break;
> >> +        case 4:
> >> +            return READ_SYSREG(ICH_LR4_EL2);
> >> +            break;
> >> +        case 5:
> >> +            return READ_SYSREG(ICH_LR5_EL2);
> >> +            break;
> >> +        case 6:
> >> +            return READ_SYSREG(ICH_LR6_EL2);
> >> +            break;
> >> +        case 7:
> >> +            return READ_SYSREG(ICH_LR7_EL2);
> >> +            break;
> >> +        case 8:
> >> +            return READ_SYSREG(ICH_LR8_EL2);
> >> +            break;
> >> +        case 9:
> >> +            return READ_SYSREG(ICH_LR9_EL2);
> >> +            break;
> >> +        case 10:
> >> +            return READ_SYSREG(ICH_LR10_EL2);
> >> +            break;
> >> +        case 11:
> >> +            return READ_SYSREG(ICH_LR11_EL2);
> >> +            break;
> >> +        case 12:
> >> +            return READ_SYSREG(ICH_LR12_EL2);
> >> +            break;
> >> +        case 13:
> >> +            return READ_SYSREG(ICH_LR13_EL2);
> >> +            break;
> >> +        case 14:
> >> +            return READ_SYSREG(ICH_LR14_EL2);
> >> +            break;
> >> +        case 15:
> >> +            return READ_SYSREG(ICH_LR15_EL2);
> >> +            break;
> >> +        default:
> >> +            return 0;
> >> +    }
> >> +}
> >> +
> >> +static void gich_write_lr(int lr, u64 val)
> >> +{
> >> +    switch (lr)
> >> +    {
> >> +        case 0:
> >> +           WRITE_SYSREG(val, ICH_LR0_EL2);
> >> +           break;
> >> +        case 1:
> >> +           WRITE_SYSREG(val, ICH_LR1_EL2);
> >> +           break;
> >> +        case 2:
> >> +           WRITE_SYSREG(val, ICH_LR2_EL2);
> >> +           break;
> >> +        case 3:
> >> +           WRITE_SYSREG(val, ICH_LR3_EL2);
> >> +           break;
> >> +        case 4:
> >> +           WRITE_SYSREG(val, ICH_LR4_EL2);
> >> +           break;
> >> +        case 5:
> >> +           WRITE_SYSREG(val, ICH_LR5_EL2);
> >> +           break;
> >> +        case 6:
> >> +           WRITE_SYSREG(val, ICH_LR6_EL2);
> >> +           break;
> >> +        case 7:
> >> +           WRITE_SYSREG(val, ICH_LR7_EL2);
> >> +           break;
> >> +        case 8:
> >> +           WRITE_SYSREG(val, ICH_LR8_EL2);
> >> +           break;
> >> +        case 9:
> >> +           WRITE_SYSREG(val, ICH_LR9_EL2);
> >> +           break;
> >> +        case 10:
> >> +           WRITE_SYSREG(val, ICH_LR10_EL2);
> >> +           break;
> >> +        case 11:
> >> +           WRITE_SYSREG(val, ICH_LR11_EL2);
> >> +           break;
> >> +        case 12:
> >> +           WRITE_SYSREG(val, ICH_LR12_EL2);
> >> +           break;
> >> +        case 13:
> >> +           WRITE_SYSREG(val, ICH_LR13_EL2);
> >> +           break;
> >> +        case 14:
> >> +           WRITE_SYSREG(val, ICH_LR14_EL2);
> >> +           break;
> >> +        case 15:
> >> +           WRITE_SYSREG(val, ICH_LR15_EL2);
> >> +           break;
> >> +        default:
> >> +           return;
> >> +    }
> >> +}
> >> +
> >> +static void gic_enable_sre(void)
> >> +{
> >> +    uint32_t val;
> >> +
> >> +    val = READ_SYSREG32(ICC_SRE_EL2);
> >> +    val |= GICC_SRE_EL2_SRE | GICC_SRE_EL2_ENEL1 | GICC_SRE_EL2_DFB | GICC_SRE_EL2_DIB;
> >> +    WRITE_SYSREG32(val, ICC_SRE_EL2);
> >> +    isb();
> >> +}
> >> +
> >> +/* Wait for completion of a distributor change */
> >> +static void gic_do_wait_for_rwp(paddr_t base)
> >> +{
> >> +    u32 val;
> >> +    do {
> >> +        val = readl_relaxed((void *)base + GICD_CTLR);
> >> +        val = readl_relaxed(GICD + GICD_CTLR);
> >> +        val = GICD[GICD_CTLR];
> >> +        cpu_relax();
> >> +    } while (val & GICD_CTLR_RWP);
> >> +}
> >
> > As much I think that this busy loop is terrible unfortunately it is
> > part of the spec :-(
> > Might be worth adding a comment on the function to explain why and when
> > it is required.
> > It is also worth considering whether it makes sense to use the
> > notification of command completion by interrupt instead.
> >
> As per spec, we have to wait for this bit to be set to be
> sure that register programming is done. IMO, there is no
> interrupt to notify it and also we have to be sure
> that programming these registers are complete before we proceed.
> 
> I will add a comment
> 
> >
> >
> >> +static void gic_dist_wait_for_rwp(void)
> >> +{
> >> +    gic_do_wait_for_rwp((paddr_t)GICD);
> >> +}
> >> +
> >> +static void gic_redist_wait_for_rwp(void)
> >> +{
> >> +    gic_do_wait_for_rwp(gic_data_rdist_rd_base());
> >> +}
> >> +
> >> +static void gic_wait_for_rwp(int irq)
> >> +{
> >> +    if (irq < 32)
> >> +         gic_redist_wait_for_rwp();
> >> +    else
> >> +         gic_dist_wait_for_rwp();
> >> +}
> >> +
> >> +static unsigned int gic_mask_cpu(const cpumask_t *cpumask)
> >> +{
> >> +    unsigned int cpu;
> >> +    cpumask_t possible_mask;
> >> +
> >> +    cpumask_and(&possible_mask, cpumask, &cpu_possible_map);
> >> +    cpu = cpumask_any(&possible_mask);
> >> +    return cpu;
> >> +}
> >> +
> >> +static unsigned int gic_nr_lines(void)
> >> +{
> >> +    return gic.lines;
> >> +}
> >> +
> >> +static unsigned int gic_nr_lrs(void)
> >> +{
> >> +    return nr_lrs;
> >> +}
> >> +
> >> +static void write_aprn_regs(struct gic_state_data *d)
> >> +{
> >> +    switch(nr_priorities)
> >> +    {
> >> +        case 7:
> >> +            WRITE_SYSREG32(d->gic_apr0[2], ICH_AP0R2_EL2);
> >> +            WRITE_SYSREG32(d->gic_apr1[2], ICH_AP1R2_EL2);
> >> +        case 6:
> >> +            WRITE_SYSREG32(d->gic_apr0[1], ICH_AP0R1_EL2);
> >> +            WRITE_SYSREG32(d->gic_apr1[1], ICH_AP1R1_EL2);
> >> +        case 5:
> >> +            WRITE_SYSREG32(d->gic_apr0[0], ICH_AP0R0_EL2);
> >> +            WRITE_SYSREG32(d->gic_apr1[0], ICH_AP1R0_EL2);
> >> +            break;
> >> +        default:
> >> +          panic("Write Undefined active priorities \n");
> >> +    }
> >> +}
> >> +
> >> +static void read_aprn_regs(struct gic_state_data *d)
> >> +{
> >> +    switch(nr_priorities)
> >> +    {
> >> +        case 7:
> >> +            d->gic_apr0[2] = READ_SYSREG32(ICH_AP0R2_EL2);
> >> +            d->gic_apr1[2] = READ_SYSREG32(ICH_AP1R2_EL2);
> >> +        case 6:
> >> +            d->gic_apr0[1] = READ_SYSREG32(ICH_AP0R1_EL2);
> >> +            d->gic_apr1[1] = READ_SYSREG32(ICH_AP1R1_EL2);
> >> +        case 5:
> >> +            d->gic_apr0[0] = READ_SYSREG32(ICH_AP0R0_EL2);
> >> +            d->gic_apr1[0] = READ_SYSREG32(ICH_AP1R0_EL2);
> >> +            break;
> >> +        default:
> >> +          panic("Read Undefined active priorities \n");
> >> +    }
> >> +}
> >> +
> >> +static int gic_state_init(struct vcpu *v)
> >> +{
> >> +     v->arch.gic_state = (struct gic_state_data *)xzalloc(struct gic_state_data);
> >> +     if(!v->arch.gic_state)
> >> +        return -ENOMEM;
> >> +     return 0;
> >> +}
> >> +
> >> +static void save_state(struct vcpu *v)
> >> +{
> >> +    int i;
> >> +    struct gic_state_data *d;
> >> +    d = (struct gic_state_data *)v->arch.gic_state;
> >> +
> >> +    /* No need for spinlocks here because interrupts are disabled around
> >> +     * this call and it only accesses struct vcpu fields that cannot be
> >> +     * accessed simultaneously by another pCPU.
> >> +     */
> >> +    for ( i=0; i<nr_lrs; i++)
> >> +        d->gic_lr[i] = gich_read_lr(i);
> >> +
> >> +    read_aprn_regs(d);
> >> +
> >> +    d->gic_vmcr = READ_SYSREG32(ICH_VMCR_EL2);
> >> +}
> >> +
> >> +static void restore_state(struct vcpu *v)
> >> +{
> >> +    int i;
> >> +    struct gic_state_data *d;
> >> +    d = (struct gic_state_data *)v->arch.gic_state;
> >> +
> >> +    for ( i=0; i<nr_lrs; i++)
> >> +        gich_write_lr(i, d->gic_lr[i]);
> >> +
> >> +    write_aprn_regs(d);
> >> +
> >> +    WRITE_SYSREG32(d->gic_vmcr, ICH_VMCR_EL2);
> >> +}
> >> +
> >> +static void gic_dump_state(struct vcpu *v)
> >> +{
> >> +    int i;
> >> +    struct gic_state_data *d;
> >> +    d = (struct gic_state_data *)v->arch.gic_state;
> >> +    if ( v == current )
> >> +    {
> >> +        for ( i = 0; i < nr_lrs; i++ )
> >> +            printk("   HW_LR[%d]=%lx\n", i, gich_read_lr(i));
> >> +    }
> >> +    else
> >> +    {
> >> +        for ( i = 0; i < nr_lrs; i++ )
> >> +            printk("   VCPU_LR[%d]=%lx\n", i, d->gic_lr[i]);
> >> +    }
> >> +}
> >> +
> >> +static void gic_enable_irq(int irq)
> >> +{
> >> +    uint32_t enabler;
> >> +
> >> +    /* Enable routing */
> >> +    if(irq > 31)
> >> +    {
> >> +        enabler = readl_relaxed(GICD + GICD_ISENABLER + (irq / 32) * 4);
> >> +        writel_relaxed(enabler | (1u << (irq % 32)), GICD + GICD_ISENABLER + (irq / 32) * 4);
> >> +    }
> >> +    else
> >> +    {
> >> +        enabler = readl_relaxed((void *)gic_data_rdist_sgi_base() + GICR_ISENABLER0);
> >> +        writel_relaxed(enabler | (1u << irq), (void *)gic_data_rdist_sgi_base() + GICR_ISENABLER0);
> >> +    }
> >> +    gic_wait_for_rwp(irq);
> >> +}
> >> +
> >> +static void gic_disable_irq(int irq)
> >> +{
> >> +    /* Disable routing */
> >> +    if(irq > 31)
> >> +        writel_relaxed(1u << (irq % 32), GICD + GICD_ICENABLER + ((irq / 32) * 4));
> >> +    else
> >> +        writel_relaxed(1u << irq, (void *)gic_data_rdist_sgi_base() + GICR_ICENABLER0);
> >> +}
> >> +
> >> +static void gic_eoi_irq(int irq)
> >> +{
> >> +    /* Lower the priority */
> >> +    WRITE_SYSREG32(irq, ICC_EOIR1_EL1);
> >> +}
> >> +
> >> +static void gic_dir_irq(int irq)
> >> +{
> >> +    /* Deactivate */
> >> +    WRITE_SYSREG32(irq, ICC_DIR_EL1);
> >> +}
> >> +
> >> +static unsigned int gic_ack_irq(void)
> >> +{
> >> +    return (READ_SYSREG32(ICC_IAR1_EL1) & GICC_IA_IRQ);
> >> +}
> >> +
> >> +static u64 gic_mpidr_to_affinity(u64 mpidr)
> >> +{
> >> +    /* Make sure we don't broadcast the interrupt */
> >> +    return mpidr & ~GICD_IROUTER_SPI_MODE_ANY;
> >> +}
> >> +
> >> +/*
> >> + * - needs to be called with gic.lock held
> >> + * - needs to be called with a valid cpu_mask, ie each cpu in the mask has
> >> + * already called gic_cpu_init
> >> + */
> >> +static void gic_set_irq_property(unsigned int irq, bool_t level,
> >> +                                   const cpumask_t *cpu_mask,
> >> +                                   unsigned int priority)
> >> +{
> >> +    uint32_t cfg, edgebit;
> >> +    u64 affinity;
> >> +    unsigned int cpu = gic_mask_cpu(cpu_mask);
> >> +    paddr_t rebase;
> >> +
> >> +
> >> +    /* Set edge / level */
> >> +    if (irq < 16)
> >> +     /* SGI's are always edge-triggered not need to call GICD_ICFGR0 */
> >> +       cfg = readl_relaxed((void *)gic_data_rdist_sgi_base() + GICR_ICFGR0);
> >> +    else if (irq < 32)
> >> +       cfg = readl_relaxed((void *)gic_data_rdist_sgi_base() + GICR_ICFGR1);
> >> +    else
> >> +       cfg = readl_relaxed(GICD + GICD_ICFGR + (irq / 16) * 4);
> >> +
> >> +    edgebit = 2u << (2 * (irq % 16));
> >> +    if ( level )
> >> +        cfg &= ~edgebit;
> >> +    else
> >> +        cfg |= edgebit;
> >> +
> >> +    if (irq < 16)
> >> +       writel_relaxed(cfg, (void *)gic_data_rdist_sgi_base() + GICR_ICFGR0);
> >> +    else if (irq < 32)
> >> +       writel_relaxed(cfg, (void *)gic_data_rdist_sgi_base() + GICR_ICFGR1);
> >> +    else
> >> +       writel_relaxed(cfg, GICD + GICD_ICFGR + (irq / 16) * 4);
> >> +
> >> +
> >> +    /* need to check if ARE is set to access IROUTER */
> >> +    affinity = gic_mpidr_to_affinity(cpu_logical_map(cpu));
> >> +    if (irq > 31)
> >> +        writeq_relaxed(affinity, (GICD + GICD_IROUTER + irq * 8));
> >> +
> >> +    /* Set priority */
> >> +    if (irq < 32)
> >> +    {
> >> +     rebase = gic_data_rdist_sgi_base();
> >
> > code style
> >
> >> +        writeb_relaxed(priority, (void *)rebase + GICR_IPRIORITYR0 + irq);
> >> +    }
> >> +    else
> >> +    {
> >> +        writeb_relaxed(priority, GICD + GICD_IPRIORITYR + irq);
> >> +
> >> +    }
> >> +}
> >> +
> >> +static void __init gic_dist_init(void)
> >> +{
> >> +    uint32_t type;
> >> +    u64 affinity;
> >> +    int i;
> >> +
> >> +    /* Disable the distributor */
> >> +    writel_relaxed(0, GICD + GICD_CTLR);
> >> +
> >> +    type = readl_relaxed(GICD + GICD_TYPER);
> >> +    gic.lines = 32 * ((type & GICD_TYPE_LINES) + 1);
> >> +
> >> +    printk("GIC: %d lines, (IID %8.8x).\n",
> >> +           gic.lines, readl_relaxed(GICD + GICD_IIDR));
> >> +
> >> +    /* Default all global IRQs to level, active low */
> >> +    for ( i = 32; i < gic.lines; i += 16 )
> >> +        writel_relaxed(0, GICD + GICD_ICFGR + (i / 16) * 4);
> >> +
> >> +    /* Default priority for global interrupts */
> >> +    for ( i = 32; i < gic.lines; i += 4 )
> >> +        writel_relaxed((GIC_PRI_IRQ<<24 | GIC_PRI_IRQ<<16 | GIC_PRI_IRQ<<8 | GIC_PRI_IRQ), GICD + GICD_IPRIORITYR + (i / 4) * 4);
> >> +
> >> +    /* Disable all global interrupts */
> >> +    for ( i = 32; i < gic.lines; i += 32 )
> >> +        writel_relaxed(0xffffffff, GICD + GICD_ICENABLER + (i / 32) * 4);
> >> +
> >> +    gic_dist_wait_for_rwp();
> >> +
> >> +    /* Turn on the distributor */
> >> +    writel_relaxed(GICD_CTL_ENABLE | GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1, GICD + GICD_CTLR);
> >> +
> >> +    /* Route all global IRQs to this CPU */
> >> +    affinity = gic_mpidr_to_affinity(read_cpuid_mpidr());
> >> +    for ( i = 31; i < gic.lines; i++ )
> >> +        writeq_relaxed(affinity, GICD + GICD_IROUTER + i * 8);
> >> +}
> >> +
> >> +static void gic_enable_redist(void)
> >> +{
> >> +    paddr_t rbase;
> >> +    u32 val;
> >> +
> >> +    rbase = this_cpu(rbase);
> >> +
> >> +    /* Wake up this CPU redistributor */
> >> +    val = readl_relaxed((void *)rbase + GICR_WAKER);
> >> +    val &= ~GICR_WAKER_ProcessorSleep;
> >> +    writel_relaxed(val, (void *)rbase + GICR_WAKER);
> >> +
> >> +    do {
> >> +         val = readl_relaxed((void *)rbase + GICR_WAKER);
> >> +         cpu_relax();
> >> +    } while (val & GICR_WAKER_ChildrenAsleep);
> >> +}
> >> +
> >> +static int __init gic_populate_rdist(void)
> >> +{
> >> +    u64 mpidr = cpu_logical_map(smp_processor_id());
> >> +    u64 typer;
> >> +    u64 aff;
> >> +    int i;
> >> +    uint32_t reg;
> >> +
> >> +    aff  = mpidr & ((1 << 24) - 1);
> >> +    aff |= (mpidr >> 8) & (0xffUL << 24);
> >> +
> >> +    for (i = 0; i < gic.rdist_count; i++) {
> >> +        uint32_t ptr = 0;
> >> +
> >> +        reg = readl_relaxed(GICR + ptr + GICR_PIDR0);
> >> +        if ((reg & 0xff) != GICR_PIDR0_GICv3) { /* We're in trouble... */
> >> +            printk("No redistributor present @%x\n", ptr);
> >> +            break;
> >> +        }
> >> +
> >> +        do {
> >> +            typer = readq_relaxed(GICR + ptr + GICR_TYPER);
> >> +            if ((typer >> 32) == aff) {
> >> +
> >> +                this_cpu(rbase) = (u64)GICR + ptr;
> >> +                this_cpu(phy_rbase) = gic.rdist_regions[i].rdist_base + ptr;
> >> +
> >> +                printk("CPU%d: found redistributor %llx region %d\n",
> >> +                            smp_processor_id(), (unsigned long long) mpidr, i);
> >> +                return 0;
> >> +            }
> >> +
> >> +            if (gic.rdist_stride) {
> >> +                ptr += gic.rdist_stride;
> >> +            } else {
> >> +                ptr += SZ_64K * 2; /* Skip RD_base + SGI_base */
> >> +                if (typer & GICR_TYPER_VLPIS)
> >> +                    ptr += SZ_64K * 2; /* Skip VLPI_base + reserved page */
> >> +                }
> >> +            } while (!(typer & GICR_TYPER_LAST));
> >
> > code style: the indentation is wrong
> >
> OK.
> >
> >> +        }
> >> +
> >> +        /* We couldn't even deal with ourselves... */
> >> +        printk("CPU%d: mpidr %lx has no re-distributor!\n",
> >> +                  smp_processor_id(), (unsigned long)mpidr);
> >> +        return -ENODEV;
> >> +}
> >> +
> >> +static void __cpuinit gic_cpu_init(void)
> >> +{
> >> +    int i;
> >> +    paddr_t rbase_sgi;
> >> +
> >> +    /* Register ourselves with the rest of the world */
> >> +    if (gic_populate_rdist())
> >> +        return;
> >> +
> >> +    gic_enable_redist();
> >> +
> >> +    rbase_sgi = gic_data_rdist_sgi_base();
> >> +
> >> +    /*
> >> +     * Set priority on PPI and SGI interrupts
> >> +     */
> >> +    for (i = 0; i < 16; i += 4)
> >> +        writel_relaxed((GIC_PRI_IPI<<24 | GIC_PRI_IPI<<16 | GIC_PRI_IPI<<8 | GIC_PRI_IPI), (void *)rbase_sgi + GICR_IPRIORITYR0 + (i / 4) * 4);
> >> +        //writel_relaxed(0x0, (void *)rbase + GICR_IPRIORITYR0 + (i / 4) * 4);
> >> +        //writel_relaxed(0xa0a0a0a0, (void *)rbase + GICR_IPRIORITYR0 + (i / 4) * 4);
> >
> > What?
> >
> >
> >> +
> >> +    for (i = 16; i < 32; i += 4)
> >> +        writel_relaxed((GIC_PRI_IRQ<<24 | GIC_PRI_IRQ<<16 | GIC_PRI_IRQ<<8 | GIC_PRI_IRQ), (void *)rbase_sgi + GICR_IPRIORITYR0 + (i / 4) * 4);
> >> +
> >> +    /*
> >> +     * Disable all PPI interrupts, ensure all SGI interrupts are
> >> +     * enabled.
> >> +     */
> >> +    writel_relaxed(0xffff0000, (void *)rbase_sgi + GICR_ICENABLER0);
> >> +    writel_relaxed(0x0000ffff, (void *)rbase_sgi + GICR_ISENABLER0);
> >> +
> >> +    gic_redist_wait_for_rwp();
> >> +
> >> +    /* Enable system registers */
> >> +    gic_enable_sre();
> >> +
> >> +    WRITE_SYSREG32(0, ICC_BPR1_EL1);
> >> +    /* Set priority mask register */
> >> +    WRITE_SYSREG32(DEFAULT_PMR_VALUE, ICC_PMR_EL1);
> >> +
> >> +    /* EOI drops priority too (mode 0) */
> >> +    WRITE_SYSREG32(GICC_CTLR_EL1_EOImode_drop, ICC_CTLR_EL1);
> >> +
> >> +    WRITE_SYSREG32(1, ICC_IGRPEN1_EL1);
> >> +}
> >> +
> >> +static void gic_cpu_disable(void)
> >> +{
> >> +    WRITE_SYSREG32(0, ICC_CTLR_EL1);
> >> +}
> >> +
> >> +static void __cpuinit gic_hyp_init(void)
> >> +{
> >> +    uint32_t vtr;
> >> +
> >> +    vtr = READ_SYSREG32(ICH_VTR_EL2);
> >> +    nr_lrs  = (vtr & GICH_VTR_NRLRGS) + 1;
> >> +    nr_priorities = ((vtr >> GICH_VTR_PRIBITS_SHIFT) & GICH_VTR_PRIBITS_MASK) + 1;
> >> +
> >> +    WRITE_SYSREG32(GICH_VMCR_EOI | GICH_VMCR_VENG1, ICH_VMCR_EL2);
> >> +    WRITE_SYSREG32(GICH_HCR_VGRP1EIE | GICH_HCR_EN, ICH_HCR_EL2);
> >> +
> >> +    update_cpu_lr_mask();
> >> +    vtr = READ_SYSREG32(ICH_HCR_EL2);
> >> +}
> >> +
> >> +/* Set up the per-CPU parts of the GIC for a secondary CPU */
> >> +static int __cpuinit gic_init_secondary_cpu(struct notifier_block *nfb,
> >> +                                       unsigned long action, void *hcpu)
> >> +{
> >> +    if (action == CPU_STARTING)
> >> +    {
> >> +        spin_lock(&gic.lock);
> >> +        gic_cpu_init();
> >> +        gic_hyp_init();
> >> +        spin_unlock(&gic.lock);
> >> +    }
> >> +    return NOTIFY_DONE;
> >> +}
> >> +
> >> +static struct notifier_block gic_cpu_nb = {
> >> +    .notifier_call = gic_init_secondary_cpu,
> >> +    .priority = 100
> >> +};
> >> +
> >> +static void gic_smp_init(void)
> >> +{
> >> +   register_cpu_notifier(&gic_cpu_nb);
> >> +}
> >> +
> >> +static void __cpuinit gic_hyp_disable(void)
> >> +{
> >> +    uint32_t vtr;
> >> +    vtr = READ_SYSREG32(ICH_HCR_EL2);
> >> +    vtr &= ~0x1;
> >> +    WRITE_SYSREG32( vtr, ICH_HCR_EL2);
> >> +}
> >> +
> >> +static u16 gic_compute_target_list(int *base_cpu, const struct cpumask *mask,
> >> +                                   u64 cluster_id)
> >> +{
> >> +    int cpu = *base_cpu;
> >> +    u64 mpidr = cpu_logical_map(cpu);
> >> +    u16 tlist = 0;
> >> +
> >> +    while (cpu < nr_cpu_ids) {
> >> +        /*
> >> +         * If we ever get a cluster of more than 16 CPUs, just
> >> +         * scream and skip that CPU.
> >> +         */
> >> +        tlist |= 1 << (mpidr & 0xf);
> >> +
> >> +        cpu = cpumask_next(cpu, mask);
> >> +        mpidr = cpu_logical_map(cpu);
> >> +
> >> +        if (cluster_id != (mpidr & ~0xffUL)) {
> >> +            cpu--;
> >> +            goto out;
> >> +        }
> >> +    }
> >> +out:
> >> +    *base_cpu = cpu;
> >> +    return tlist;
> >> +}
> >> +
> >> +static void send_sgi(u64 cluster_id, u16 tlist, unsigned int irq)
> >> +{
> >> +    u64 val;
> >> +
> >> +    val  = (cluster_id & 0xff00ff0000UL) << 16; /* Aff3 + Aff2 */
> >> +    val |= (cluster_id & 0xff00) << 8;          /* Aff1 */
> >> +    val |= irq << 24;
> >> +    val |= tlist;
> >> +
> >> +    WRITE_SYSREG(val, ICC_SGI1R_EL1);
> >> +}
> >> +
> >> +static void gic_send_sgi(const cpumask_t *cpumask, enum gic_sgi sgi)
> >> +{
> >> +    int cpu = 0;
> >> +
> >> +    dsb();
> >> +
> >> +    for_each_cpu(cpu, cpumask) {
> >> +        u64 cluster_id = cpu_logical_map(cpu) & ~0xffUL;
> >> +        u16 tlist;
> >> +
> >> +        tlist = gic_compute_target_list(&cpu, cpumask, cluster_id);
> >> +        send_sgi(cluster_id, tlist, sgi);
> >> +    }
> >> +}
> >> +
> >> +/* Shut down the per-CPU GIC interface */
> >> +static void gic_disable_interface(void)
> >> +{
> >> +    ASSERT(!local_irq_is_enabled());
> >> +
> >> +    spin_lock(&gic.lock);
> >> +    gic_cpu_disable();
> >> +    gic_hyp_disable();
> >> +    spin_unlock(&gic.lock);
> >> +}
> >> +
> >> +static void gic_update_lr(int lr, unsigned int virtual_irq,
> >> +        unsigned int state, unsigned int priority)
> >> +{
> >> +    u64 maintenance_int = GICH_LR_MAINTENANCE_IRQ;
> >> +    u64 grp = GICH_LR_GRP1;
> >> +    u64 val = 0;
> >> +
> >> +    BUG_ON(lr >= nr_lrs);
> >> +    BUG_ON(lr < 0);
> >> +
> >> +    val =  ((((u64)state) & 0x3) << GICH_LR_STATE_SHIFT) | grp | maintenance_int |
> >> +        ((((u64)priority) & 0xff) << GICH_LR_PRIORITY_SHIFT) |
> >> +        (((u64)virtual_irq & GICH_LR_VIRTUAL_MASK) << GICH_LR_VIRTUAL_SHIFT) |
> >> +        (((u64)virtual_irq & GICH_LR_PHYSICAL_MASK) << GICH_LR_PHYSICAL_SHIFT);
> >> +
> >> +    gich_write_lr(lr, val);
> >> +}
> >
> > This function has to change after
> >
> > http://marc.info/?l=xen-devel&m=139523241201086
> >
> OK. I will try to rebase on top of your patches if you consider that your
> patches are complete

They are functionlly complete but they might still need few cosmetic
changes.  I'll resend another version of my patch series later today
that addresses all the comments so far.

Re: [RFC PATCH v1 08/10] xen/arm: Add support for GIC v3

[Vijay Kilari]

Hi Stefano,

On Mon, Mar 24, 2014 at 10:31 PM, Stefano Stabellini
<stefano.stabellini@eu.citrix.com> wrote:
> On Sat, 22 Mar 2014, Vijay Kilari wrote:
>> On Thu, Mar 20, 2014 at 6:07 PM, Stefano Stabellini
>> <stefano.stabellini@eu.citrix.com> wrote:
>> > On Wed, 19 Mar 2014, vijay.kilari@gmail.com wrote:
>> >> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>> >>
>> >> Add support for GIC v3 specification.
>> >> This driver assumes that ARE and SRE
>> >> is enable by default.
>> >>
>> >> Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>

>> > This function has to change after
>> >
>> > http://marc.info/?l=xen-devel&m=139523241201086
>> >
>> OK. I will try to rebase on top of your patches if you consider that your
>> patches are complete
>
> They are functionlly complete but they might still need few cosmetic
> changes.  I'll resend another version of my patch series later today
> that addresses all the comments so far.

I tried your V5 patch set. It works with my changes.
I rebase on top of your patch set and will resend my next version.

Re: [RFC PATCH v1 08/10] xen/arm: Add support for GIC v3

[Stefano Stabellini]

On Wed, 26 Mar 2014, Vijay Kilari wrote:
> Hi Stefano,
> 
> On Mon, Mar 24, 2014 at 10:31 PM, Stefano Stabellini
> <stefano.stabellini@eu.citrix.com> wrote:
> > On Sat, 22 Mar 2014, Vijay Kilari wrote:
> >> On Thu, Mar 20, 2014 at 6:07 PM, Stefano Stabellini
> >> <stefano.stabellini@eu.citrix.com> wrote:
> >> > On Wed, 19 Mar 2014, vijay.kilari@gmail.com wrote:
> >> >> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> >> >>
> >> >> Add support for GIC v3 specification.
> >> >> This driver assumes that ARE and SRE
> >> >> is enable by default.
> >> >>
> >> >> Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> 
> >> > This function has to change after
> >> >
> >> > http://marc.info/?l=xen-devel&m=139523241201086
> >> >
> >> OK. I will try to rebase on top of your patches if you consider that your
> >> patches are complete
> >
> > They are functionlly complete but they might still need few cosmetic
> > changes.  I'll resend another version of my patch series later today
> > that addresses all the comments so far.
> 
> I tried your V5 patch set. It works with my changes.
> I rebase on top of your patch set and will resend my next version.

Great!

Re: [RFC PATCH v1 08/10] xen/arm: Add support for GIC v3

[Julien Grall]

Hello Vijay,

Thank you for the patch.

On 03/19/2014 02:17 PM, vijay.kilari@gmail.com wrote:
> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> 
> Add support for GIC v3 specification.
> This driver assumes that ARE and SRE
> is enable by default.

Can you expand what mean ARE and SRE here?
Can you explain how do you configure the GICv3 ...

[..]

> +struct gic_state_data {
> +    uint32_t gic_hcr, gic_vmcr;
> +    uint32_t gic_apr0[4];
> +    uint32_t gic_apr1[4];
> +    uint64_t gic_lr[16];
> +};
> +
> +#define GICD ((volatile unsigned char *) gic.map_dbase)

You only need to cast into unsigned char *. IO read/write will take care
of the volatile.

> +/* Only one region is implemented which is enough for 0-31 cpus */
> +#define GICR ((volatile unsigned char *) gic.rdist_regions[0].map_rdist_base)
> +
> +/* per-cpu re-distributor base */
> +static DEFINE_PER_CPU(paddr_t, rbase);
> +static DEFINE_PER_CPU(paddr_t, phy_rbase);
> +
> +static unsigned nr_lrs;
> +static uint32_t nr_priorities;
> +
> +/* The GIC mapping of CPU interfaces does not necessarily match the
> + * logical CPU numbering. Let's use mapping as returned by the GIC
> + * itself
> + */
> +
> +#define gic_data_rdist_rd_base()        (this_cpu(rbase))
> +#define gic_data_rdist_sgi_base()       (gic_data_rdist_rd_base() + SZ_64K)
> +
> +static inline u64 read_cpuid_mpidr(void)
> +{
> +   return READ_SYSREG(MPIDR_EL1);

MPDIR_EL1 is already replicated in current_cpu_data.mpidr.bits

[..]

> +static void gic_enable_sre(void)
> +{
> +    uint32_t val;
> +

Can you explain in a comment what you are enabling here ....

> +    val = READ_SYSREG32(ICC_SRE_EL2);
> +    val |= GICC_SRE_EL2_SRE | GICC_SRE_EL2_ENEL1 | GICC_SRE_EL2_DFB | GICC_SRE_EL2_DIB;
> +    WRITE_SYSREG32(val, ICC_SRE_EL2);
> +    isb();
> +}
> +
> +/* Wait for completion of a distributor change */
> +static void gic_do_wait_for_rwp(paddr_t base)
> +{
> +    u32 val;
> +    do {
> +        val = readl_relaxed((void *)base + GICD_CTLR);
> +        val = readl_relaxed(GICD + GICD_CTLR);
> +        val = GICD[GICD_CTLR];
> +        cpu_relax();
> +    } while (val & GICD_CTLR_RWP);
> +}
> +
> +static void gic_dist_wait_for_rwp(void)
> +{
> +    gic_do_wait_for_rwp((paddr_t)GICD);
> +}
> +
> +static void gic_redist_wait_for_rwp(void)
> +{
> +    gic_do_wait_for_rwp(gic_data_rdist_rd_base());
> +}
> +
> +static void gic_wait_for_rwp(int irq)
> +{
> +    if (irq < 32)
> +         gic_redist_wait_for_rwp();
> +    else
> +         gic_dist_wait_for_rwp();
> +}
> +
> +static unsigned int gic_mask_cpu(const cpumask_t *cpumask)
> +{
> +    unsigned int cpu;
> +    cpumask_t possible_mask;
> +
> +    cpumask_and(&possible_mask, cpumask, &cpu_possible_map);
> +    cpu = cpumask_any(&possible_mask);
> +    return cpu;
> +}
> +
> +static unsigned int gic_nr_lines(void)
> +{
> +    return gic.lines;
> +}
> +
> +static unsigned int gic_nr_lrs(void)
> +{
> +    return nr_lrs;
> +}
> +
> +static void write_aprn_regs(struct gic_state_data *d)
> +{
> +    switch(nr_priorities)

Coding style

switch ( .. )

> +    {
> +        case 7:
       case align to the {

Please check all the file against CODING_STYLE. I won't shout anymore on
every coding style error in this mail.

[..]

> +static int gic_state_init(struct vcpu *v)
> +{
> +     v->arch.gic_state = (struct gic_state_data *)xzalloc(struct gic_state_data);

Don't need to cast.

> +     if(!v->arch.gic_state)
> +        return -ENOMEM;
> +     return 0; 
> +}
> +
> +static void save_state(struct vcpu *v)
> +{
> +    int i;
> +    struct gic_state_data *d;
> +    d = (struct gic_state_data *)v->arch.gic_state;
> +
> +    /* No need for spinlocks here because interrupts are disabled around
> +     * this call and it only accesses struct vcpu fields that cannot be
> +     * accessed simultaneously by another pCPU.
> +     */
> +    for ( i=0; i<nr_lrs; i++)
> +        d->gic_lr[i] = gich_read_lr(i);

You are introducing a helper to read/write lr. How the compiler handle
it? Will it optimize?

For me it seems very slow...

> +static u64 gic_mpidr_to_affinity(u64 mpidr)
> +{
> +    /* Make sure we don't broadcast the interrupt */
> +    return mpidr & ~GICD_IROUTER_SPI_MODE_ANY;
> +}
> +
> +/*
> + * - needs to be called with gic.lock held
> + * - needs to be called with a valid cpu_mask, ie each cpu in the mask has

Please add assert in the function to check that the function is
effectively called with this requirements.

[..]

> +
> +    edgebit = 2u << (2 * (irq % 16));
> +    if ( level )
> +        cfg &= ~edgebit;
> +    else
> +        cfg |= edgebit;
> +
> +    if (irq < 16)
> +       writel_relaxed(cfg, (void *)gic_data_rdist_sgi_base() + GICR_ICFGR0);
> +    else if (irq < 32)
> +       writel_relaxed(cfg, (void *)gic_data_rdist_sgi_base() + GICR_ICFGR1);
> +    else
> +       writel_relaxed(cfg, GICD + GICD_ICFGR + (irq / 16) * 4);
> +
> +

Spurious line

> +    /* need to check if ARE is set to access IROUTER */
> +    affinity = gic_mpidr_to_affinity(cpu_logical_map(cpu));
> +    if (irq > 31)
> +        writeq_relaxed(affinity, (GICD + GICD_IROUTER + irq * 8));
> +
> +    /* Set priority */
> +    if (irq < 32)
> +    {
> +	rebase = gic_data_rdist_sgi_base();

HARD tab.

> +        writeb_relaxed(priority, (void *)rebase + GICR_IPRIORITYR0 + irq);
> +    }
> +    else 
> +    {
> +        writeb_relaxed(priority, GICD + GICD_IPRIORITYR + irq);
> +

Spurious line

[..]

> +static void __cpuinit gic_cpu_init(void)
> +{
> +    int i;
> +    paddr_t rbase_sgi;
> +
> +    /* Register ourselves with the rest of the world */
> +    if (gic_populate_rdist())
> +        return;
> +
> +    gic_enable_redist();
> +
> +    rbase_sgi = gic_data_rdist_sgi_base();
> +
> +    /*
> +     * Set priority on PPI and SGI interrupts
> +     */
> +    for (i = 0; i < 16; i += 4)
> +        writel_relaxed((GIC_PRI_IPI<<24 | GIC_PRI_IPI<<16 | GIC_PRI_IPI<<8 | GIC_PRI_IPI), (void *)rbase_sgi + GICR_IPRIORITYR0 + (i / 4) * 4);
> +        //writel_relaxed(0x0, (void *)rbase + GICR_IPRIORITYR0 + (i / 4) * 4);
> +        //writel_relaxed(0xa0a0a0a0, (void *)rbase + GICR_IPRIORITYR0 + (i / 4) * 4);

Please remove this comment.

[..]

> +/* Set up the GIC */
> +void __init gicv3_init(void)
> +{
> +    static const struct dt_device_match gic_ids[] __initconst =
> +    {
> +        DT_MATCH_GIC_V3,
> +        { /* sentinel */ },
> +    };
> +    struct dt_device_node *node;
> +    struct rdist_region *rdist_regs;
> +    int res, i;
> +    uint32_t reg;
> +
> +    node = dt_find_interrupt_controller(gic_ids);
> +    if ( !node )
> +        panic("Unable to find compatible GIC in the device tree");
> +
> +    dt_device_set_used_by(node, DOMID_XEN);
> +
> +    res = dt_device_get_address(node, 0, &gic.dbase, &gic.dbase_size);
> +    if ( res || !gic.dbase  || (gic.dbase & ~PAGE_MASK) || (gic.dbase_size & ~PAGE_MASK) )
> +        panic("GIC: Cannot find a valid address for the distributor");
> +
> +    gic.map_dbase = ioremap_nocache(gic.dbase, gic.dbase_size);

ioremap_nocache can fail. Please check the return.

> +
> +    reg = readl_relaxed(GICD + GICD_PIDR0);
> +    if ((reg & 0xff) != GICD_PIDR0_GICv3)
> +        panic("GIC: no distributor detected, giving up\n"); 
> +
> +    gic.hw_version = GIC_VERSION_V3;
> + 
> +    if (!dt_property_read_u32(node, "#redistributor-regions", &gic.rdist_count))
> +        gic.rdist_count = 1;
> +
> +    rdist_regs = xzalloc_array(struct rdist_region, gic.rdist_count);
> +    if (!rdist_regs)
> +        panic("GIC: no distributor detected, giving up\n");
> +
> +    for (i = 0; i < gic.rdist_count; i++) {
> +	u64 rdist_base, rdist_size;

HARD tab.

> +
> +        res = dt_device_get_address(node, 1 + i, &rdist_base, &rdist_size);
> +        if ( res || !rdist_base)
> +		printk("No rdist base found\n");


You are setup rdist_base and rdist_base_size to and unknown value if an
error occurred. Should not Xen panic?

> diff --git a/xen/arch/arm/gic.c b/xen/arch/arm/gic.c
> index e0859ae..291e34c 100644
> --- a/xen/arch/arm/gic.c
> +++ b/xen/arch/arm/gic.c
> @@ -688,6 +688,11 @@ static void maintenance_interrupt(int irq, void *dev_id, struct cpu_user_regs *r
>  /* Set up the GIC */
>  void __init gic_init(void)
>  {
> +    static const struct dt_device_match gicv3_ids[] __initconst =
> +    {
> +        DT_MATCH_GIC_V3,
> +        { /* sentinel */ },
> +    };

As I said on patch #6, you should use the device API. It will help use
when a new GIC driver will be introduced.

[..]

> diff --git a/xen/include/asm-arm/domain.h b/xen/include/asm-arm/domain.h
> index 38df789..15e83e8 100644
> --- a/xen/include/asm-arm/domain.h
> +++ b/xen/include/asm-arm/domain.h
> @@ -142,6 +142,10 @@ struct arch_domain
>          /* Base address for guest GIC */
>          paddr_t dbase; /* Distributor base address */
>          paddr_t cbase; /* CPU base address */
> +        paddr_t dbase_size; /* Distributor base size */
> +        paddr_t rbase;      /* Re-Distributor base address */
> +        paddr_t rbase_size; /* Re-Distributor size */
> +        uint32_t rdist_stride;

Theses values should go in a GICv3 specific structure. It's *NOT* common
code.

>      } vgic;
>  
>      struct vuart {
> diff --git a/xen/include/asm-arm/gic.h b/xen/include/asm-arm/gic.h
> index 2de6c6a..b6fbd5e 100644
> --- a/xen/include/asm-arm/gic.h
> +++ b/xen/include/asm-arm/gic.h
> @@ -18,6 +18,8 @@
>  #ifndef __ASM_ARM_GIC_H__
>  #define __ASM_ARM_GIC_H__
>  
> +#define SZ_64K  0x00010000

Please don't hardcode SZ_64K like that. It should also go in common code.

[..]

> +/*
> + * The minimum GICC_BPR is required to be in the range 0-3. We set
> + * GICC_BPR to 0 but we must expect that it might be 3. This means we
> + * can rely on premption between the following ranges:
> + * 0xf0..0xff
> + * 0xe0..0xdf
> + * 0xc0..0xcf
> + * 0xb0..0xbf
> + * 0xa0..0xaf
> + * 0x90..0x9f
> + * 0x80..0x8f
> + *
> + * Priorities within a range will not preempt each other.
> + *
> + * A GIC must support a mimimum of 16 priority levels.
> + */
> +#define GIC_PRI_LOWEST 0xf0
> +#define GIC_PRI_IRQ 0xa0
> +#define GIC_PRI_IPI 0x90 /* IPIs must preempt normal interrupts */
> +#define GIC_PRI_HIGHEST 0x80 /* Higher priorities belong to Secure-World */
> +

Hmmm you let this part on common header patch #6. Why do you move know?

Regards,

-- 
Julien Grall

Re: [RFC PATCH v1 08/10] xen/arm: Add support for GIC v3

[Vijay Kilari]

On Thu, Mar 20, 2014 at 10:10 PM, Julien Grall <julien.grall@linaro.org> wrote:
> Hello Vijay,
>
> Thank you for the patch.
>
> On 03/19/2014 02:17 PM, vijay.kilari@gmail.com wrote:
>> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>>
>> Add support for GIC v3 specification.
>> This driver assumes that ARE and SRE
>> is enable by default.
>
> Can you expand what mean ARE and SRE here?
> Can you explain how do you configure the GICv3 ...
>
> [..]
>
>> +struct gic_state_data {
>> +    uint32_t gic_hcr, gic_vmcr;
>> +    uint32_t gic_apr0[4];
>> +    uint32_t gic_apr1[4];
>> +    uint64_t gic_lr[16];
>> +};
>> +
>> +#define GICD ((volatile unsigned char *) gic.map_dbase)
>
> You only need to cast into unsigned char *. IO read/write will take care
> of the volatile.
>
>> +/* Only one region is implemented which is enough for 0-31 cpus */
>> +#define GICR ((volatile unsigned char *) gic.rdist_regions[0].map_rdist_base)
>> +
>> +/* per-cpu re-distributor base */
>> +static DEFINE_PER_CPU(paddr_t, rbase);
>> +static DEFINE_PER_CPU(paddr_t, phy_rbase);
>> +
>> +static unsigned nr_lrs;
>> +static uint32_t nr_priorities;
>> +
>> +/* The GIC mapping of CPU interfaces does not necessarily match the
>> + * logical CPU numbering. Let's use mapping as returned by the GIC
>> + * itself
>> + */
>> +
>> +#define gic_data_rdist_rd_base()        (this_cpu(rbase))
>> +#define gic_data_rdist_sgi_base()       (gic_data_rdist_rd_base() + SZ_64K)
>> +
>> +static inline u64 read_cpuid_mpidr(void)
>> +{
>> +   return READ_SYSREG(MPIDR_EL1);
>
> MPDIR_EL1 is already replicated in current_cpu_data.mpidr.bits
>
ok

> [..]
>
>> +static void gic_enable_sre(void)
>> +{
>> +    uint32_t val;
>> +
>
> Can you explain in a comment what you are enabling here ....
>
ok

>> +    val = READ_SYSREG32(ICC_SRE_EL2);
>> +    val |= GICC_SRE_EL2_SRE | GICC_SRE_EL2_ENEL1 | GICC_SRE_EL2_DFB | GICC_SRE_EL2_DIB;
>> +    WRITE_SYSREG32(val, ICC_SRE_EL2);
>> +    isb();
>> +}
>> +
>> +/* Wait for completion of a distributor change */
>> +static void gic_do_wait_for_rwp(paddr_t base)
>> +{
>> +    u32 val;
>> +    do {
>> +        val = readl_relaxed((void *)base + GICD_CTLR);
>> +        val = readl_relaxed(GICD + GICD_CTLR);
>> +        val = GICD[GICD_CTLR];
>> +        cpu_relax();
>> +    } while (val & GICD_CTLR_RWP);
>> +}
>> +
>> +static void gic_dist_wait_for_rwp(void)
>> +{
>> +    gic_do_wait_for_rwp((paddr_t)GICD);
>> +}
>> +
>> +static void gic_redist_wait_for_rwp(void)
>> +{
>> +    gic_do_wait_for_rwp(gic_data_rdist_rd_base());
>> +}
>> +
>> +static void gic_wait_for_rwp(int irq)
>> +{
>> +    if (irq < 32)
>> +         gic_redist_wait_for_rwp();
>> +    else
>> +         gic_dist_wait_for_rwp();
>> +}
>> +
>> +static unsigned int gic_mask_cpu(const cpumask_t *cpumask)
>> +{
>> +    unsigned int cpu;
>> +    cpumask_t possible_mask;
>> +
>> +    cpumask_and(&possible_mask, cpumask, &cpu_possible_map);
>> +    cpu = cpumask_any(&possible_mask);
>> +    return cpu;
>> +}
>> +
>> +static unsigned int gic_nr_lines(void)
>> +{
>> +    return gic.lines;
>> +}
>> +
>> +static unsigned int gic_nr_lrs(void)
>> +{
>> +    return nr_lrs;
>> +}
>> +
>> +static void write_aprn_regs(struct gic_state_data *d)
>> +{
>> +    switch(nr_priorities)
>
> Coding style
>
> switch ( .. )
>
>> +    {
>> +        case 7:
>        case align to the {
>
> Please check all the file against CODING_STYLE. I won't shout anymore on
> every coding style error in this mail.
OK. Is there any script to check coding style of Xen?

>
> [..]
>
>> +static int gic_state_init(struct vcpu *v)
>> +{
>> +     v->arch.gic_state = (struct gic_state_data *)xzalloc(struct gic_state_data);
>
> Don't need to cast.
>
>> +     if(!v->arch.gic_state)
>> +        return -ENOMEM;
>> +     return 0;
>> +}
>> +
>> +static void save_state(struct vcpu *v)
>> +{
>> +    int i;
>> +    struct gic_state_data *d;
>> +    d = (struct gic_state_data *)v->arch.gic_state;
>> +
>> +    /* No need for spinlocks here because interrupts are disabled around
>> +     * this call and it only accesses struct vcpu fields that cannot be
>> +     * accessed simultaneously by another pCPU.
>> +     */
>> +    for ( i=0; i<nr_lrs; i++)
>> +        d->gic_lr[i] = gich_read_lr(i);
>
> You are introducing a helper to read/write lr. How the compiler handle
> it? Will it optimize?
>
> For me it seems very slow...
>
  because LR registers are system registers, we have to
use READ/WRITE_SYSREG

>> +static u64 gic_mpidr_to_affinity(u64 mpidr)
>> +{
>> +    /* Make sure we don't broadcast the interrupt */
>> +    return mpidr & ~GICD_IROUTER_SPI_MODE_ANY;
>> +}
>> +
>> +/*
>> + * - needs to be called with gic.lock held
>> + * - needs to be called with a valid cpu_mask, ie each cpu in the mask has
>
> Please add assert in the function to check that the function is
> effectively called with this requirements.
>
ok

> [..]
>
>> +
>> +    edgebit = 2u << (2 * (irq % 16));
>> +    if ( level )
>> +        cfg &= ~edgebit;
>> +    else
>> +        cfg |= edgebit;
>> +
>> +    if (irq < 16)
>> +       writel_relaxed(cfg, (void *)gic_data_rdist_sgi_base() + GICR_ICFGR0);
>> +    else if (irq < 32)
>> +       writel_relaxed(cfg, (void *)gic_data_rdist_sgi_base() + GICR_ICFGR1);
>> +    else
>> +       writel_relaxed(cfg, GICD + GICD_ICFGR + (irq / 16) * 4);
>> +
>> +
>
> Spurious line
>
>> +    /* need to check if ARE is set to access IROUTER */
>> +    affinity = gic_mpidr_to_affinity(cpu_logical_map(cpu));
>> +    if (irq > 31)
>> +        writeq_relaxed(affinity, (GICD + GICD_IROUTER + irq * 8));
>> +
>> +    /* Set priority */
>> +    if (irq < 32)
>> +    {
>> +     rebase = gic_data_rdist_sgi_base();
>
> HARD tab.
>
>> +        writeb_relaxed(priority, (void *)rebase + GICR_IPRIORITYR0 + irq);
>> +    }
>> +    else
>> +    {
>> +        writeb_relaxed(priority, GICD + GICD_IPRIORITYR + irq);
>> +
>
> Spurious line
>
> [..]
>
>> +static void __cpuinit gic_cpu_init(void)
>> +{
>> +    int i;
>> +    paddr_t rbase_sgi;
>> +
>> +    /* Register ourselves with the rest of the world */
>> +    if (gic_populate_rdist())
>> +        return;
>> +
>> +    gic_enable_redist();
>> +
>> +    rbase_sgi = gic_data_rdist_sgi_base();
>> +
>> +    /*
>> +     * Set priority on PPI and SGI interrupts
>> +     */
>> +    for (i = 0; i < 16; i += 4)
>> +        writel_relaxed((GIC_PRI_IPI<<24 | GIC_PRI_IPI<<16 | GIC_PRI_IPI<<8 | GIC_PRI_IPI), (void *)rbase_sgi + GICR_IPRIORITYR0 + (i / 4) * 4);
>> +        //writel_relaxed(0x0, (void *)rbase + GICR_IPRIORITYR0 + (i / 4) * 4);
>> +        //writel_relaxed(0xa0a0a0a0, (void *)rbase + GICR_IPRIORITYR0 + (i / 4) * 4);
>
> Please remove this comment.
>
> [..]
>
>> +/* Set up the GIC */
>> +void __init gicv3_init(void)
>> +{
>> +    static const struct dt_device_match gic_ids[] __initconst =
>> +    {
>> +        DT_MATCH_GIC_V3,
>> +        { /* sentinel */ },
>> +    };
>> +    struct dt_device_node *node;
>> +    struct rdist_region *rdist_regs;
>> +    int res, i;
>> +    uint32_t reg;
>> +
>> +    node = dt_find_interrupt_controller(gic_ids);
>> +    if ( !node )
>> +        panic("Unable to find compatible GIC in the device tree");
>> +
>> +    dt_device_set_used_by(node, DOMID_XEN);
>> +
>> +    res = dt_device_get_address(node, 0, &gic.dbase, &gic.dbase_size);
>> +    if ( res || !gic.dbase  || (gic.dbase & ~PAGE_MASK) || (gic.dbase_size & ~PAGE_MASK) )
>> +        panic("GIC: Cannot find a valid address for the distributor");
>> +
>> +    gic.map_dbase = ioremap_nocache(gic.dbase, gic.dbase_size);
>
> ioremap_nocache can fail. Please check the return.
>
ok

>> +
>> +    reg = readl_relaxed(GICD + GICD_PIDR0);
>> +    if ((reg & 0xff) != GICD_PIDR0_GICv3)
>> +        panic("GIC: no distributor detected, giving up\n");
>> +
>> +    gic.hw_version = GIC_VERSION_V3;
>> +
>> +    if (!dt_property_read_u32(node, "#redistributor-regions", &gic.rdist_count))
>> +        gic.rdist_count = 1;
>> +
>> +    rdist_regs = xzalloc_array(struct rdist_region, gic.rdist_count);
>> +    if (!rdist_regs)
>> +        panic("GIC: no distributor detected, giving up\n");
>> +
>> +    for (i = 0; i < gic.rdist_count; i++) {
>> +     u64 rdist_base, rdist_size;
>
> HARD tab.
>
>> +
>> +        res = dt_device_get_address(node, 1 + i, &rdist_base, &rdist_size);
>> +        if ( res || !rdist_base)
>> +             printk("No rdist base found\n");
>
>
> You are setup rdist_base and rdist_base_size to and unknown value if an
> error occurred. Should not Xen panic?
>
>> diff --git a/xen/arch/arm/gic.c b/xen/arch/arm/gic.c
>> index e0859ae..291e34c 100644
>> --- a/xen/arch/arm/gic.c
>> +++ b/xen/arch/arm/gic.c
>> @@ -688,6 +688,11 @@ static void maintenance_interrupt(int irq, void *dev_id, struct cpu_user_regs *r
>>  /* Set up the GIC */
>>  void __init gic_init(void)
>>  {
>> +    static const struct dt_device_match gicv3_ids[] __initconst =
>> +    {
>> +        DT_MATCH_GIC_V3,
>> +        { /* sentinel */ },
>> +    };
>
> As I said on patch #6, you should use the device API. It will help use
> when a new GIC driver will be introduced.
>
ok

> [..]
>
>> diff --git a/xen/include/asm-arm/domain.h b/xen/include/asm-arm/domain.h
>> index 38df789..15e83e8 100644
>> --- a/xen/include/asm-arm/domain.h
>> +++ b/xen/include/asm-arm/domain.h
>> @@ -142,6 +142,10 @@ struct arch_domain
>>          /* Base address for guest GIC */
>>          paddr_t dbase; /* Distributor base address */
>>          paddr_t cbase; /* CPU base address */
>> +        paddr_t dbase_size; /* Distributor base size */
>> +        paddr_t rbase;      /* Re-Distributor base address */
>> +        paddr_t rbase_size; /* Re-Distributor size */
>> +        uint32_t rdist_stride;
>
> Theses values should go in a GICv3 specific structure. It's *NOT* common
> code.
>
Yes, all the variables should be moved out. similar to other gic stuff

>>      } vgic;
>>
>>      struct vuart {
>> diff --git a/xen/include/asm-arm/gic.h b/xen/include/asm-arm/gic.h
>> index 2de6c6a..b6fbd5e 100644
>> --- a/xen/include/asm-arm/gic.h
>> +++ b/xen/include/asm-arm/gic.h
>> @@ -18,6 +18,8 @@
>>  #ifndef __ASM_ARM_GIC_H__
>>  #define __ASM_ARM_GIC_H__
>>
>> +#define SZ_64K  0x00010000
>
> Please don't hardcode SZ_64K like that. It should also go in common code.
>
> [..]
>
>> +/*
>> + * The minimum GICC_BPR is required to be in the range 0-3. We set
>> + * GICC_BPR to 0 but we must expect that it might be 3. This means we
>> + * can rely on premption between the following ranges:
>> + * 0xf0..0xff
>> + * 0xe0..0xdf
>> + * 0xc0..0xcf
>> + * 0xb0..0xbf
>> + * 0xa0..0xaf
>> + * 0x90..0x9f
>> + * 0x80..0x8f
>> + *
>> + * Priorities within a range will not preempt each other.
>> + *
>> + * A GIC must support a mimimum of 16 priority levels.
>> + */
>> +#define GIC_PRI_LOWEST 0xf0
>> +#define GIC_PRI_IRQ 0xa0
>> +#define GIC_PRI_IPI 0x90 /* IPIs must preempt normal interrupts */
>> +#define GIC_PRI_HIGHEST 0x80 /* Higher priorities belong to Secure-World */
>> +
>
> Hmmm you let this part on common header patch #6. Why do you move know?
>

 ok

> Regards,
>
> --
> Julien Grall

Re: [RFC PATCH v1 08/10] xen/arm: Add support for GIC v3

[Julien Grall]

Hello Vijay,

For next time, can you try to quote only what you need? It's hard to 
find your answer in a long mail.

On 22/03/14 10:21, Vijay Kilari wrote:
> On Thu, Mar 20, 2014 at 10:10 PM, Julien Grall <julien.grall@linaro.org> wrote:
>> Please check all the file against CODING_STYLE. I won't shout anymore on
>> every coding style error in this mail.
> OK. Is there any script to check coding style of Xen?

Unfortunately no. It might be interesting to have one for Xen as the 
coding style differs from Linux. People are often confusing between them :)

>>> +static void save_state(struct vcpu *v)
>>> +{
>>> +    int i;
>>> +    struct gic_state_data *d;
>>> +    d = (struct gic_state_data *)v->arch.gic_state;
>>> +
>>> +    /* No need for spinlocks here because interrupts are disabled around
>>> +     * this call and it only accesses struct vcpu fields that cannot be
>>> +     * accessed simultaneously by another pCPU.
>>> +     */
>>> +    for ( i=0; i<nr_lrs; i++)
>>> +        d->gic_lr[i] = gich_read_lr(i);
>>
>> You are introducing a helper to read/write lr. How the compiler handle
>> it? Will it optimize?
>>
>> For me it seems very slow...
>>
>    because LR registers are system registers, we have to
> use READ/WRITE_SYSREG

Is it possible to read all of them (without looking at nr_lrs)?

I see that KVM is using this solution, e.g not looping.

Regards,

-- 
Julien Grall

Re: [RFC PATCH v1 08/10] xen/arm: Add support for GIC v3

[Ian Campbell]

On Sun, 2014-03-23 at 14:49 +0000, Julien Grall wrote:
> Hello Vijay,
> 
> For next time, can you try to quote only what you need? It's hard to 
> find your answer in a long mail.
> 
> On 22/03/14 10:21, Vijay Kilari wrote:
> > On Thu, Mar 20, 2014 at 10:10 PM, Julien Grall <julien.grall@linaro.org> wrote:
> >> Please check all the file against CODING_STYLE. I won't shout anymore on
> >> every coding style error in this mail.
> > OK. Is there any script to check coding style of Xen?
> 
> Unfortunately no. It might be interesting to have one for Xen as the 
> coding style differs from Linux. People are often confusing between them :)
> 
> >>> +static void save_state(struct vcpu *v)
> >>> +{
> >>> +    int i;
> >>> +    struct gic_state_data *d;
> >>> +    d = (struct gic_state_data *)v->arch.gic_state;
> >>> +
> >>> +    /* No need for spinlocks here because interrupts are disabled around
> >>> +     * this call and it only accesses struct vcpu fields that cannot be
> >>> +     * accessed simultaneously by another pCPU.
> >>> +     */
> >>> +    for ( i=0; i<nr_lrs; i++)
> >>> +        d->gic_lr[i] = gich_read_lr(i);
> >>
> >> You are introducing a helper to read/write lr. How the compiler handle
> >> it? Will it optimize?
> >>
> >> For me it seems very slow...
> >    because LR registers are system registers, we have to
> > use READ/WRITE_SYSREG
> 
> Is it possible to read all of them (without looking at nr_lrs)?

What's the concern here? That the compiler doesn't inline and then do
dead code elimination of gich_read_lr to effectively unroll the loop
into a series of sysreg reads+memory stores?

> I see that KVM is using this solution, e.g not looping.

You mean they've just manually unrolled the loop? That's likely to be as
much to do with having to write it in asm any way as anything else.

I'm not saying it isn't worth measuring+optimising this code, but the
measuring bit should happen first not just guessing what might optimise
badly (a simple form of "measure" might be "look at the disassembly").

Ian.

Re: [RFC PATCH v1 08/10] xen/arm: Add support for GIC v3

[Julien Grall]

On 03/24/2014 11:26 AM, Ian Campbell wrote:
> On Sun, 2014-03-23 at 14:49 +0000, Julien Grall wrote:
>> Hello Vijay,
>>
>> For next time, can you try to quote only what you need? It's hard to 
>> find your answer in a long mail.
>>
>> On 22/03/14 10:21, Vijay Kilari wrote:
>>> On Thu, Mar 20, 2014 at 10:10 PM, Julien Grall <julien.grall@linaro.org> wrote:
>>>> Please check all the file against CODING_STYLE. I won't shout anymore on
>>>> every coding style error in this mail.
>>> OK. Is there any script to check coding style of Xen?
>>
>> Unfortunately no. It might be interesting to have one for Xen as the 
>> coding style differs from Linux. People are often confusing between them :)
>>
>>>>> +static void save_state(struct vcpu *v)
>>>>> +{
>>>>> +    int i;
>>>>> +    struct gic_state_data *d;
>>>>> +    d = (struct gic_state_data *)v->arch.gic_state;
>>>>> +
>>>>> +    /* No need for spinlocks here because interrupts are disabled around
>>>>> +     * this call and it only accesses struct vcpu fields that cannot be
>>>>> +     * accessed simultaneously by another pCPU.
>>>>> +     */
>>>>> +    for ( i=0; i<nr_lrs; i++)
>>>>> +        d->gic_lr[i] = gich_read_lr(i);
>>>>
>>>> You are introducing a helper to read/write lr. How the compiler handle
>>>> it? Will it optimize?
>>>>
>>>> For me it seems very slow...
>>>    because LR registers are system registers, we have to
>>> use READ/WRITE_SYSREG
>>
>> Is it possible to read all of them (without looking at nr_lrs)?
> 
> What's the concern here? That the compiler doesn't inline and then do
> dead code elimination of gich_read_lr to effectively unroll the loop
> into a series of sysreg reads+memory stores?
> 
>> I see that KVM is using this solution, e.g not looping.
> 
> You mean they've just manually unrolled the loop? That's likely to be as
> much to do with having to write it in asm any way as anything else.
> 
> I'm not saying it isn't worth measuring+optimising this code, but the
> measuring bit should happen first not just guessing what might optimise
> badly (a simple form of "measure" might be "look at the disassembly").

The compiler will likely not optimize this code because nr_lrs is set at
boot time.

-- 
Julien Grall

Re: [RFC PATCH v1 08/10] xen/arm: Add support for GIC v3

[Stefano Stabellini]

On Sat, 22 Mar 2014, Vijay Kilari wrote:
> > Please check all the file against CODING_STYLE. I won't shout anymore on
> > every coding style error in this mail.
> OK. Is there any script to check coding style of Xen?

Unfortunately we don't have one.

[RFC PATCH v1 09/10] xen/arm: Add vgic support for GIC v3

[vijay.kilari]

From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>

Add vgic driver support for GIC v3 version.

Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
---
 xen/arch/arm/Makefile     |    2 +-
 xen/arch/arm/vgic-v3.c    |  927 +++++++++++++++++++++++++++++++++++++++++++++
 xen/arch/arm/vgic.c       |   12 +
 xen/include/asm-arm/gic.h |    1 +
 4 files changed, 941 insertions(+), 1 deletion(-)

diff --git a/xen/arch/arm/Makefile b/xen/arch/arm/Makefile
index a11c699..5a6cb07 100644
--- a/xen/arch/arm/Makefile
+++ b/xen/arch/arm/Makefile
@@ -26,7 +26,7 @@ obj-y += smpboot.o
 obj-y += smp.o
 obj-y += shutdown.o
 obj-y += traps.o
-obj-y += vgic.o vgic-v2.o
+obj-y += vgic.o vgic-v2.o vgic-v3.o
 obj-y += vtimer.o
 obj-y += vuart.o
 obj-y += hvm.o
diff --git a/xen/arch/arm/vgic-v3.c b/xen/arch/arm/vgic-v3.c
new file mode 100644
index 0000000..b5c1d1e
--- /dev/null
+++ b/xen/arch/arm/vgic-v3.c
@@ -0,0 +1,927 @@
+/*
+ * xen/arch/arm/vgic-v3.c
+ *
+ * ARM Virtual Generic Interrupt Controller v3 support
+ * based on xen/arch/arm/vgic.c
+ *
+ * Vijaya Kumar K <vijaya.kumar@caviumnetworks.com>
+ * Copyright (c) 2014 Cavium Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <xen/bitops.h>
+#include <xen/config.h>
+#include <xen/lib.h>
+#include <xen/init.h>
+#include <xen/softirq.h>
+#include <xen/irq.h>
+#include <xen/sched.h>
+
+#include <asm/current.h>
+
+#include "io.h"
+#include <asm/gic-v3.h>
+#include <asm/gic.h>
+
+struct vgic_irq_rank {
+    spinlock_t lock; /* Covers access to all other members of this struct */
+    uint32_t ienable, iactive, ipend, pendsgi;
+    uint32_t icfg[2];
+    uint32_t ipriority[8];
+    uint64_t irouter[32];
+};
+
+#define REG(n) (n)
+
+/* Number of ranks of interrupt registers for a domain */
+#define DOMAIN_NR_RANKS(d) (((d)->arch.vgic.nr_lines+31)/32)
+
+/*
+ * Rank containing GICD_<FOO><n> for GICD_<FOO> with
+ * <b>-bits-per-interrupt
+ */
+static inline int REG_RANK_NR(int b, uint32_t n)
+{
+    switch ( b )
+    {
+        case 64: return n >> 6;
+        case 32: return n >> 5;
+        case 16: return n >> 4;
+        case 8: return n >> 3;
+        case 4: return n >> 2;
+        case 2: return n >> 1;
+        case 1: return n;
+        default: BUG();
+    }
+}
+
+/*
+ * Offset of GICD_<FOO><n> with its rank, for GICD_<FOO> with
+ * <b>-bits-per-interrupt.
+ */
+/* Shift n >> 2 to make it byte register diff */
+#define REG_RANK_INDEX(b, n) (((n) >> 2) & ((b)-1))
+
+/*
+ * Returns rank corresponding to a GICD_<FOO><n> register for
+ * GICD_<FOO> with <b>-bits-per-interrupt.
+ */
+static struct vgic_irq_rank *vgic_irq_rank(struct vcpu *v, int b, int n)
+{
+    int rank;
+
+    n = n >> 2;
+    rank  = REG_RANK_NR(b, n);
+
+    if ( rank == 0 ) /* Rank 0 is nothing but GICR registers in GICv3 */
+        return (struct vgic_irq_rank *)v->arch.vgic.private_irqs;
+    else if ( rank <= DOMAIN_NR_RANKS(v->domain) )
+        return (struct vgic_irq_rank *)((unsigned char *)(v->domain->arch.vgic.shared_irqs)
+                + (sizeof(struct vgic_irq_rank) *(rank - 1)));
+    else
+        return NULL;
+}
+
+#define vgic_lock(v)   spin_lock_irq(&(v)->domain->arch.vgic.lock)
+#define vgic_unlock(v) spin_unlock_irq(&(v)->domain->arch.vgic.lock)
+
+#define vgic_lock_rank(v, r) spin_lock(&(r)->lock)
+#define vgic_unlock_rank(v, r) spin_unlock(&(r)->lock)
+
+static uint32_t byte_read(uint32_t val, int sign, int offset)
+{
+    int byte = offset & 0x3;
+
+    val = val >> (8*byte);
+    if ( sign && (val & 0x80) )
+        val |= 0xffffff00;
+    else
+        val &= 0x000000ff;
+    return val;
+}
+
+static void byte_write(uint32_t *reg, uint32_t var, int offset)
+{
+    int byte = offset & 0x3;
+
+    var &= (0xff << (8*byte));
+
+    *reg &= ~(0xff << (8*byte));
+    *reg |= var;
+}
+
+static int vgic_read_priority(struct vcpu *v, int irq)
+{
+   struct vgic_irq_rank *rank = vgic_irq_rank(v, 8, irq);
+   return byte_read(rank->ipriority[REG_RANK_INDEX(8, irq)], 0, irq & 0x3);
+}
+ 
+static int __vgic_rdistr_mmio_read(struct vcpu *v, mmio_info_t *info)
+{
+    struct hsr_dabt dabt = info->dabt;
+    struct cpu_user_regs *regs = guest_cpu_user_regs();
+    register_t *r = select_user_reg(regs, dabt.reg);
+    int offset = (int)(info->gpa - gic_data_rdist_rd_base());
+    int gicr_reg = REG(offset);
+    u64 mpidr;
+    u64 aff;
+
+    switch ( gicr_reg )
+    {
+    case GICR_CTLR:
+        /* We have implemented LPI's, read zero */
+        goto read_as_zero;
+    case GICR_IIDR:
+        *r = 0x34;
+        return 1;
+    case GICR_TYPER:
+        if(((info->gpa) & (~((SZ_64K * 2 * smp_processor_id()) - 1))) ==
+            (gic_data_rdist_rd_base() & (~((SZ_64K * 2 * smp_processor_id()) - 1))) )
+        {
+            mpidr = cpu_logical_map(smp_processor_id());
+            aff  = mpidr & ((1 << 24) - 1);
+            aff |= (mpidr >> 8) & (0xffUL << 24);
+            aff = aff << 32;
+            aff = (aff | ((u64)smp_processor_id() << 8));
+            aff = (aff | (0x9UL << 0));
+            *r = aff;
+            return 1;
+        }
+        *r = 0;
+        return 0;
+    case GICR_STATUSR:
+        /* Not implemented */
+        goto read_as_zero;
+    case GICR_WAKER:
+        /* Not implemented */
+        goto read_as_zero;
+    case GICR_SETLPIR:
+    case GICR_CLRLPIR:
+        /* WO */
+        return 0;
+    case GICR_PROPBASER:
+        /* LPI's not implemented */
+        goto read_as_zero;
+    case GICR_PENDBASER:
+        /* LPI's not implemented */
+        goto read_as_zero;
+    case GICR_INVLPIR:
+    case GICR_INVALLR:
+        /* WO */
+        return 0;
+    case GICR_SYNCR:
+        if ( dabt.size != 2 ) goto bad_width;
+        /* RO */
+        /* Return as not busy */
+        *r = 1;
+        return 1;
+    case GICR_MOVLPIR:
+    case GICR_MOVALLR:
+        /* WO */
+        return 0;
+    case GICR_PIDR7... GICR_PIDR0:
+        *r = 0x93;
+         return 1;
+    default:
+        printk("vGICR: read r%d offset %#08x\n not found", dabt.reg, offset);
+       
+    }
+bad_width:
+    printk("vGICD: bad read width %d r%d offset %#08x\n",
+           dabt.size, dabt.reg, offset);
+    domain_crash_synchronous();
+    return 0;
+
+read_as_zero:
+    if ( dabt.size != 2 ) goto bad_width;
+    *r = 0;
+    return 1;
+}
+
+static int __vgic_rdistr_mmio_write(struct vcpu *v, mmio_info_t *info)
+{
+    struct hsr_dabt dabt = info->dabt;
+    struct cpu_user_regs *regs = guest_cpu_user_regs();
+    register_t *r = select_user_reg(regs, dabt.reg);
+    int offset = (int)(info->gpa - gic_data_rdist_rd_base());
+    int gicr_reg = REG(offset);
+
+    switch ( gicr_reg )
+    {
+    case GICR_CTLR:
+        /* We do not implement LPI's, read zero */
+        goto write_ignore;
+    case GICR_IIDR:
+    case GICR_TYPER:
+        /* RO */
+        goto write_ignore;
+    case GICR_STATUSR:
+        /* Not implemented */
+        goto write_ignore;
+    case GICR_WAKER:
+        /* Not implemented */
+        goto write_ignore;
+    case GICR_SETLPIR:
+        return 1;
+    case GICR_CLRLPIR:
+        return 1;
+    case GICR_PROPBASER:
+        /* LPI is not implemented */
+        goto write_ignore;
+    case GICR_PENDBASER:
+        /* LPI is not implemented */
+        goto write_ignore;
+    case GICR_INVLPIR:
+        return 1;
+    case GICR_INVALLR:
+        return 1;
+    case GICR_SYNCR:
+        /* RO */
+        goto write_ignore;
+    case GICR_MOVLPIR:
+        return 1;
+    case GICR_MOVALLR:
+        return 1;
+    case GICR_PIDR7... GICR_PIDR0:
+        /* RO */
+        goto write_ignore;
+    default:
+        printk("vGICR: write r%d offset %#08x\n not found", dabt.reg, offset);
+    }
+bad_width:
+    printk("vGICD: bad write width %d r%d=%"PRIregister" offset %#08x\n",
+           dabt.size, dabt.reg, *r, offset);
+    domain_crash_synchronous();
+    return 0;
+
+write_ignore:
+    if ( dabt.size != 2 ) goto bad_width;
+    return 1;
+}
+
+static int vgic_rdistr_sgi_mmio_read(struct vcpu *v, mmio_info_t *info)
+{
+    struct hsr_dabt dabt = info->dabt;
+    struct cpu_user_regs *regs = guest_cpu_user_regs();
+    register_t *r = select_user_reg(regs, dabt.reg);
+    struct vgic_irq_rank *rank;
+    int offset = (int)(info->gpa - gic_data_rdist_sgi_base());
+    int gicr_reg = REG(offset);
+
+    switch ( gicr_reg )
+    {
+    case GICR_IGROUPR0:
+        /* We do not implement security extensions for guests, read zero */
+        goto read_as_zero;
+    case GICR_IGRPMODR0:
+        /* We do not implement security extensions for guests, read zero */
+        goto read_as_zero;
+    case GICR_ISENABLER0:
+        if ( dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 1, gicr_reg - GICR_ISENABLER0);
+        if ( rank == NULL ) goto read_as_zero;
+        vgic_lock_rank(v, rank);
+        *r = rank->ienable;
+        vgic_unlock_rank(v, rank);
+        return 1;
+    case GICR_ICENABLER0:
+        if ( dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 1, gicr_reg - GICR_ICENABLER0);
+        if ( rank == NULL ) goto read_as_zero;
+        vgic_lock_rank(v, rank);
+        *r = rank->ienable;
+        vgic_unlock_rank(v, rank);
+        return 1;
+    case GICR_ISPENDR0:
+        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 1, gicr_reg - GICR_ISPENDR0);
+        if ( rank == NULL ) goto read_as_zero;
+        vgic_lock_rank(v, rank);
+        *r = byte_read(rank->ipend, dabt.sign, offset);
+        vgic_unlock_rank(v, rank);
+        return 1;
+    case GICR_ICPENDR0:
+        if ( dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 1, gicr_reg - GICR_ICPENDR0);
+        if ( rank == NULL ) goto read_as_zero;
+        vgic_lock_rank(v, rank);
+        *r = byte_read(rank->ipend, dabt.sign, offset);
+        vgic_unlock_rank(v, rank);
+        return 1;
+    case GICR_ISACTIVER0:
+        if ( dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 1, gicr_reg - GICR_ISACTIVER0);
+        if ( rank == NULL ) goto read_as_zero;
+        vgic_lock_rank(v, rank);
+        *r = rank->iactive;
+        vgic_unlock_rank(v, rank);
+        return 1;
+    case GICR_ICACTIVER0:
+        if ( dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 1, gicr_reg - GICR_ICACTIVER0);
+        if ( rank == NULL ) goto read_as_zero;
+        vgic_lock_rank(v, rank);
+        *r = rank->iactive;
+        vgic_unlock_rank(v, rank);
+        return 1;
+    case GICR_IPRIORITYR0...GICR_IPRIORITYR7:
+        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 8, gicr_reg - GICR_IPRIORITYR0);
+        if ( rank == NULL ) goto read_as_zero;
+
+        vgic_lock_rank(v, rank);
+        *r = rank->ipriority[REG_RANK_INDEX(8, gicr_reg - GICR_IPRIORITYR0)];
+        if ( dabt.size == 0 )
+            *r = byte_read(*r, dabt.sign, offset);
+        vgic_unlock_rank(v, rank);
+        return 1;
+    case GICR_ICFGR0... GICR_ICFGR1:
+        if ( dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 2, gicr_reg - GICR_ICFGR0);
+        if ( rank == NULL) goto read_as_zero;
+        vgic_lock_rank(v, rank);
+        *r = rank->icfg[REG_RANK_INDEX(2, gicr_reg - GICR_ICFGR0)];
+        vgic_unlock_rank(v, rank);
+        return 1;
+    case GICR_NSACR:
+        if ( dabt.size != 2 ) goto bad_width;
+        return 1;
+    default:
+        printk("vGICR: read r%d offset %#08x\n not found", dabt.reg, offset);
+       
+    }
+bad_width:
+    printk("vGICD: bad read width %d r%d offset %#08x\n",
+           dabt.size, dabt.reg, offset);
+    domain_crash_synchronous();
+    return 0;
+
+read_as_zero:
+    if ( dabt.size != 2 ) goto bad_width;
+    *r = 0;
+    return 1;
+}
+
+static int vgic_rdistr_sgi_mmio_write(struct vcpu *v, mmio_info_t *info)
+{
+    struct hsr_dabt dabt = info->dabt;
+    struct cpu_user_regs *regs = guest_cpu_user_regs();
+    register_t *r = select_user_reg(regs, dabt.reg);
+    struct vgic_irq_rank *rank;
+    int offset = (int)(info->gpa - gic_data_rdist_sgi_base());
+    int gicr_reg = REG(offset);
+    uint32_t tr;
+
+    switch ( gicr_reg )
+    {
+    case GICR_IGROUPR0:
+        /* We do not implement security extensions for guests, write ignore */
+        goto write_ignore;
+    case GICR_IGRPMODR0:
+        /* We do not implement security extensions for guests, write ignore */
+        goto write_ignore;
+    case GICR_ISENABLER0:
+        if ( dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 1, gicr_reg - GICR_ISENABLER0);
+        if ( rank == NULL ) goto write_ignore;
+        vgic_lock_rank(v, rank);
+        tr = rank->ienable;
+        rank->ienable |= *r;
+        vgic_unlock_rank(v, rank);
+        vgic_enable_irqs(v, (*r) & (~tr), (gicr_reg - GICR_ISENABLER0) >> 2);
+        return 1;
+    case GICR_ICENABLER0:
+        if ( dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 1, gicr_reg - GICR_ICENABLER0);
+        if ( rank == NULL ) goto write_ignore;
+        vgic_lock_rank(v, rank);
+        rank->ienable &= ~*r;
+        vgic_unlock_rank(v, rank);
+        return 1;
+    case GICR_ISPENDR0:
+        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
+        return 0;
+    case GICR_ICPENDR0:
+        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
+        return 0;
+    case GICR_ISACTIVER0:
+        if ( dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 1, gicr_reg - GICR_ISACTIVER0);
+        if ( rank == NULL ) goto write_ignore;
+        vgic_lock_rank(v, rank);
+        rank->iactive &= ~*r;
+        vgic_unlock_rank(v, rank);
+        return 1;
+    case GICR_ICACTIVER0:
+        if ( dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 1, gicr_reg - GICR_ICACTIVER0);
+        if ( rank == NULL) goto write_ignore;
+        vgic_lock_rank(v, rank);
+        rank->iactive &= ~*r;
+        vgic_unlock_rank(v, rank);
+        return 1;
+    case GICR_IPRIORITYR0...GICR_IPRIORITYR7:
+        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 8, gicr_reg - GICR_IPRIORITYR0);
+        if ( rank == NULL) goto write_ignore;
+        vgic_lock_rank(v, rank);
+        if ( dabt.size == 2 )
+            rank->ipriority[REG_RANK_INDEX(8, gicr_reg - GICR_IPRIORITYR0)] = *r;
+        else
+            byte_write(&rank->ipriority[REG_RANK_INDEX(8, gicr_reg - GICR_IPRIORITYR0)],
+                       *r, offset);
+        vgic_unlock_rank(v, rank);
+        return 1;
+    case GICR_ICFGR0:
+        /* RO */
+    case GICR_ICFGR1:
+        goto write_ignore;
+    case GICR_NSACR:
+        /* We do not implement security extensions for guests, write ignore */
+        goto write_ignore;
+    default:
+        printk("vGICR: write r%d offset %#08x\n not found", dabt.reg, offset);
+    }
+bad_width:
+    printk("vGICD: bad write width %d r%d=%"PRIregister" offset %#08x\n",
+           dabt.size, dabt.reg, *r, offset);
+    domain_crash_synchronous();
+    return 0;
+
+write_ignore:
+    if ( dabt.size != 2 ) goto bad_width;
+    return 1;
+}
+
+static int vgic_rdistr_mmio_read(struct vcpu *v, mmio_info_t *info)
+{
+    uint32_t offset = info->gpa - gic_data_rdist_rd_base();
+    uint32_t sgi_offset = info->gpa - gic_data_rdist_sgi_base();
+
+    if ( offset < SZ_64K)
+       return __vgic_rdistr_mmio_read(v, info);
+    else if (sgi_offset < SZ_64K)
+       return vgic_rdistr_sgi_mmio_read(v, info);
+    else 
+       printk("vGICR: wrong gpa read %"PRIpaddr"\n",info->gpa);
+    return 0;
+}
+      
+static int vgic_rdistr_mmio_write(struct vcpu *v, mmio_info_t *info)
+{
+    uint32_t offset = info->gpa - gic_data_rdist_rd_base();
+    uint32_t sgi_offset = info->gpa - gic_data_rdist_sgi_base();
+
+    if (offset < SZ_64K)
+       return __vgic_rdistr_mmio_write(v, info);
+    else if (sgi_offset < SZ_64K)
+       return vgic_rdistr_sgi_mmio_write(v, info);
+    else
+       printk("vGICR: wrong gpa write %"PRIpaddr"\n",info->gpa);
+    return 0;
+}
+
+static int vgic_distr_mmio_read(struct vcpu *v, mmio_info_t *info)
+{
+    struct hsr_dabt dabt = info->dabt;
+    struct cpu_user_regs *regs = guest_cpu_user_regs();
+    register_t *r = select_user_reg(regs, dabt.reg);
+    struct vgic_irq_rank *rank;
+    int offset = (int)(info->gpa - v->domain->arch.vgic.dbase);
+    int gicd_reg = REG(offset);
+
+    switch ( gicd_reg )
+    {
+    case GICD_CTLR:
+        if ( dabt.size != 2 ) goto bad_width;
+        vgic_lock(v);
+        *r = v->domain->arch.vgic.ctlr;
+        vgic_unlock(v);
+        return 1;
+    case GICD_TYPER:
+        if ( dabt.size != 2 ) goto bad_width;
+        /* No secure world support for guests. */
+        vgic_lock(v);
+        *r = ( (v->domain->max_vcpus<<5) & GICD_TYPE_CPUS )
+            |( ((v->domain->arch.vgic.nr_lines/32)) & GICD_TYPE_LINES );
+        vgic_unlock(v);
+        return 1;
+    case GICD_STATUSR:
+        /* Not implemented */
+        goto read_as_zero;
+    case GICD_IIDR:
+        if ( dabt.size != 2 ) goto bad_width;
+        /*
+         * XXX Do we need a JEP106 manufacturer ID?
+         * Just use the physical h/w value for now
+         */
+        *r = 0x0000034c;
+        return 1;
+    /* Implementation defined -- read as zero */
+    case REG(0x020) ... REG(0x03c):
+        goto read_as_zero;
+    case GICD_IGROUPR ... GICD_IGROUPRN:
+        /* We do not implement security extensions for guests, read zero */
+        goto read_as_zero;
+    case GICD_ISENABLER ... GICD_ISENABLERN:
+        if ( dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 1, gicd_reg - GICD_ISENABLER);
+        if ( rank == NULL ) goto read_as_zero;
+        vgic_lock_rank(v, rank);
+        *r = rank->ienable;
+        vgic_unlock_rank(v, rank);
+        return 1;
+    case GICD_ICENABLER ... GICD_ICENABLERN:
+        if ( dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 1, gicd_reg - GICD_ICENABLER);
+        if ( rank == NULL) goto read_as_zero;
+        vgic_lock_rank(v, rank);
+        *r = rank->ienable;
+        vgic_unlock_rank(v, rank);
+        return 1;
+    case GICD_ISPENDR ... GICD_ISPENDRN:
+        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 1, gicd_reg - GICD_ISPENDR);
+        if ( rank == NULL ) goto read_as_zero;
+        vgic_lock_rank(v, rank);
+        *r = byte_read(rank->ipend, dabt.sign, offset);
+        vgic_unlock_rank(v, rank);
+        return 1;
+    case GICD_ICPENDR ... GICD_ICPENDRN:
+        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 1, gicd_reg - GICD_ICPENDR);
+        if ( rank == NULL) goto read_as_zero;
+        vgic_lock_rank(v, rank);
+        *r = byte_read(rank->ipend, dabt.sign, offset);
+        vgic_unlock_rank(v, rank);
+        return 1;
+    case GICD_ISACTIVER ... GICD_ISACTIVERN:
+        if ( dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 1, gicd_reg - GICD_ISACTIVER);
+        if ( rank == NULL) goto read_as_zero;
+        vgic_lock_rank(v, rank);
+        *r = rank->iactive;
+        vgic_unlock_rank(v, rank);
+        return 1;
+    case GICD_ICACTIVER ... GICD_ICACTIVERN:
+        if ( dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 1, gicd_reg - GICD_ICACTIVER);
+        if ( rank == NULL) goto read_as_zero;
+        vgic_lock_rank(v, rank);
+        *r = rank->iactive;
+        vgic_unlock_rank(v, rank);
+        return 1;
+    case GICD_IROUTER ... GICD_IROUTERN:
+        rank = vgic_irq_rank(v, 64, gicd_reg - GICD_IROUTERN);
+        if ( rank == NULL) goto read_as_zero;
+
+        vgic_lock_rank(v, rank);
+        /* IROUTER is 64 bit so, to make it byte size right shift by 3. 
+           Here once. macro REG_RANK_INDEX will do it twice */
+        *r = rank->irouter[REG_RANK_INDEX(64, (gicd_reg - GICD_IROUTER)>>1)];
+        if ( dabt.size == 0 )
+            *r = byte_read(*r, dabt.sign, offset);
+        vgic_unlock_rank(v, rank);
+        return 1;
+    case GICD_IPRIORITYR ... GICD_IPRIORITYRN:
+        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 8, gicd_reg - GICD_IPRIORITYR);
+        if ( rank == NULL) goto read_as_zero;
+
+        vgic_lock_rank(v, rank);
+        *r = rank->ipriority[REG_RANK_INDEX(8, gicd_reg - GICD_IPRIORITYR)];
+        if ( dabt.size == 0 )
+            *r = byte_read(*r, dabt.sign, offset);
+        vgic_unlock_rank(v, rank);
+        return 1;
+    case GICD_ICFGR ... GICD_ICFGRN:
+        if ( dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 2, gicd_reg - GICD_ICFGR);
+        if ( rank == NULL) goto read_as_zero;
+        vgic_lock_rank(v, rank);
+        *r = rank->icfg[REG_RANK_INDEX(2, gicd_reg - GICD_ICFGR)];
+        vgic_unlock_rank(v, rank);
+        return 1;
+    case GICD_NSACR ... GICD_NSACRN:
+        /* We do not implement security extensions for guests, read zero */
+        goto read_as_zero;
+    case GICD_SGIR:
+        if ( dabt.size != 2 ) goto bad_width;
+        /* Write only -- read unknown */
+        *r = 0xdeadbeef;
+        return 1;
+    case GICD_CPENDSGIR ... GICD_CPENDSGIRN:
+        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 1, gicd_reg - GICD_CPENDSGIR);
+        if ( rank == NULL) goto read_as_zero;
+        vgic_lock_rank(v, rank);
+        *r = byte_read(rank->pendsgi, dabt.sign, offset);
+        vgic_unlock_rank(v, rank);
+        return 1;
+    case GICD_SPENDSGIR ... GICD_SPENDSGIRN:
+        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 1, gicd_reg - GICD_SPENDSGIR);
+        if ( rank == NULL) goto read_as_zero;
+        vgic_lock_rank(v, rank);
+        *r = byte_read(rank->pendsgi, dabt.sign, offset);
+        vgic_unlock_rank(v, rank);
+        return 1;
+    case GICD_PIDR7... GICD_PIDR0:
+        /* GICv3 identification value */
+        *r = 0x92;
+        return 1;
+    case REG(0x00c):
+    case REG(0x044):
+    case REG(0x04c):
+    case REG(0x05c) ... REG(0x07c):
+    case REG(0xf30) ... REG(0x5fcc):
+    case REG(0x8000) ... REG(0xbfcc):
+    case REG(0xc000) ... REG(0xffcc):
+    	printk("vGICD: read unknown 0x00c .. 0xfcc r%d offset %#08x\n", dabt.reg, offset);
+        goto read_as_zero;
+
+    default:
+        printk("vGICD: unhandled read r%d offset %#08x\n",
+               dabt.reg, offset);
+        return 0;
+    }
+
+bad_width:
+    printk("vGICD: bad read width %d r%d offset %#08x\n",
+           dabt.size, dabt.reg, offset);
+    domain_crash_synchronous();
+    return 0;
+
+read_as_zero:
+    printk("vGICD: read as zero %d r%d offset %#08x\n",
+           dabt.size, dabt.reg, offset);
+    if ( dabt.size != 2 ) goto bad_width;
+    *r = 0;
+    return 1;
+}
+
+static int vgic_distr_mmio_write(struct vcpu *v, mmio_info_t *info)
+{
+    struct hsr_dabt dabt = info->dabt;
+    struct cpu_user_regs *regs = guest_cpu_user_regs();
+    register_t *r = select_user_reg(regs, dabt.reg);
+    struct vgic_irq_rank *rank;
+    int offset = (int)(info->gpa - v->domain->arch.vgic.dbase);
+    int gicd_reg = REG(offset);
+    uint32_t tr;
+
+    switch ( gicd_reg )
+    {
+    case GICD_CTLR:
+        if ( dabt.size != 2 ) goto bad_width;
+        /* Ignore all but the enable bit */
+        v->domain->arch.vgic.ctlr = (*r) & GICD_CTL_ENABLE;
+        return 1;
+
+    /* R/O -- write ignored */
+    case GICD_TYPER:
+    case GICD_IIDR:
+        goto write_ignore;
+
+    case GICD_STATUSR:
+        goto write_ignore;
+    case GICD_SETSPI_NSR:
+        goto write_ignore;
+    case GICD_CLRSPI_NSR:
+        goto write_ignore;
+    case GICD_SETSPI_SR:
+        goto write_ignore;
+    case GICD_CLRSPI_SR:
+        goto write_ignore;
+    /* Implementation defined -- write ignored */
+    case REG(0x020) ... REG(0x03c):
+    	printk("vGICD: write unknown 0x020 - 0x03c r%d offset %#08x\n", dabt.reg, offset);
+        goto write_ignore;
+
+    case GICD_IGROUPR ... GICD_IGROUPRN:
+        /* We do not implement security extensions for guests, write ignore */
+        goto write_ignore;
+
+    case GICD_ISENABLER ... GICD_ISENABLERN:
+        if ( dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 1, gicd_reg - GICD_ISENABLER);
+        if ( rank == NULL) goto write_ignore;
+        vgic_lock_rank(v, rank);
+        tr = rank->ienable;
+        rank->ienable |= *r;
+        vgic_unlock_rank(v, rank);
+        vgic_enable_irqs(v, (*r) & (~tr), (gicd_reg - GICD_ISENABLER) >> 2);
+        return 1;
+
+    case GICD_ICENABLER ... GICD_ICENABLERN:
+        if ( dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 1, gicd_reg - GICD_ICENABLER);
+        if ( rank == NULL) goto write_ignore;
+        vgic_lock_rank(v, rank);
+        tr = rank->ienable;
+        rank->ienable &= ~*r;
+        vgic_unlock_rank(v, rank);
+        vgic_disable_irqs(v, (*r) & tr, gicd_reg - GICD_ICENABLER);
+        return 1;
+
+    case GICD_ISPENDR ... GICD_ISPENDRN:
+        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
+        return 0;
+
+    case GICD_ICPENDR ... GICD_ICPENDRN:
+        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
+        return 0;
+
+    case GICD_ISACTIVER ... GICD_ISACTIVERN:
+        if ( dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 1, gicd_reg - GICD_ISACTIVER);
+        if ( rank == NULL) goto write_ignore;
+        vgic_lock_rank(v, rank);
+        rank->iactive &= ~*r;
+        vgic_unlock_rank(v, rank);
+        return 1;
+
+    case GICD_ICACTIVER ... GICD_ICACTIVERN:
+        if ( dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 1, gicd_reg - GICD_ICACTIVER);
+        if ( rank == NULL) goto write_ignore;
+        vgic_lock_rank(v, rank);
+        rank->iactive &= ~*r;
+        vgic_unlock_rank(v, rank);
+        return 1;
+
+    case GICD_IROUTER ... GICD_IROUTER + 8*7:
+        /* SGI/PPI target is read only */
+        goto write_ignore;
+
+    case GICD_IROUTER + 8*8 ... GICD_IROUTERN:
+        rank = vgic_irq_rank(v, 64, gicd_reg - GICD_IROUTER);
+        if ( rank == NULL) goto write_ignore;
+        vgic_lock_rank(v, rank);
+        rank->irouter[REG_RANK_INDEX(64, (gicd_reg - GICD_IROUTER)>>1)] = *r;
+        vgic_unlock_rank(v, rank);
+        return 1;
+
+    case GICD_IPRIORITYR ... GICD_IPRIORITYRN:
+        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 8, gicd_reg - GICD_IPRIORITYR);
+        if ( rank == NULL) goto write_ignore;
+        vgic_lock_rank(v, rank);
+        if ( dabt.size == 2 )
+            rank->ipriority[REG_RANK_INDEX(8, gicd_reg - GICD_IPRIORITYR)] = *r;
+        else
+            byte_write(&rank->ipriority[REG_RANK_INDEX(8, gicd_reg - GICD_IPRIORITYR)],
+                       *r, offset);
+        vgic_unlock_rank(v, rank);
+        return 1;
+
+    case GICD_ICFGR: /* SGIs */
+        goto write_ignore;
+    case GICD_ICFGR + 4: /* PPIs */
+        /* It is implementation defined if these are writeable. We chose not */
+        goto write_ignore;
+    case GICD_ICFGR + 8 ... GICD_ICFGRN: /* SPIs */
+        if ( dabt.size != 2 ) goto bad_width;
+        rank = vgic_irq_rank(v, 2, gicd_reg - GICD_ICFGR);
+        vgic_lock_rank(v, rank);
+        if ( rank == NULL) goto write_ignore;
+        rank->icfg[REG_RANK_INDEX(2, gicd_reg - GICD_ICFGR)] = *r;
+        vgic_unlock_rank(v, rank);
+        return 1;
+
+    case GICD_NSACR ... GICD_NSACRN:
+        /* We do not implement security extensions for guests, write ignore */
+        goto write_ignore;
+
+    case GICD_SGIR:
+        /* it is accessed as system register */
+        goto write_ignore;
+
+    case GICD_CPENDSGIR ... GICD_CPENDSGIRN:
+        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
+        return 0;
+
+    case GICD_SPENDSGIR ... GICD_SPENDSGIRN:
+        if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
+        return 0;
+
+    case REG(0x00c):
+    case REG(0x044):
+    case REG(0x04c):
+    case REG(0x05c) ... REG(0x07c):
+    case REG(0xf30) ... REG(0x5fcc):
+    case REG(0x8000) ... REG(0xbfcc):
+    case REG(0xc000) ... REG(0xffcc):
+    	printk("vGICD: write unknown 0x00c 0xfcc  r%d offset %#08x\n", dabt.reg, offset);
+        goto write_ignore;
+
+    default:
+        printk("vGICD: unhandled write r%d=%"PRIregister" offset %#08x\n",
+               dabt.reg, *r, offset);
+        return 0;
+    }
+
+bad_width:
+    printk("vGICD: bad write width %d r%d=%"PRIregister" offset %#08x\n",
+           dabt.size, dabt.reg, *r, offset);
+    domain_crash_synchronous();
+    return 0;
+
+write_ignore:
+    if ( dabt.size != 2 ) goto bad_width;
+    return 1;
+}
+
+
+static int vgic_rdistr_mmio_check(struct vcpu *v, paddr_t addr)
+{
+    struct domain *d = v->domain;
+
+    /* Compare with global rdist base. the addr should fall in this region */
+    return (addr >= (d->arch.vgic.rbase)) && (addr < (d->arch.vgic.rbase + d->arch.vgic.rbase_size));
+}
+
+static struct mmio_handler vgic_rdistr_mmio_handler = {
+    .check_handler = vgic_rdistr_mmio_check,
+    .read_handler  = vgic_rdistr_mmio_read,
+    .write_handler = vgic_rdistr_mmio_write,
+};
+
+static int vgic_distr_mmio_check(struct vcpu *v, paddr_t addr)
+{
+    struct domain *d = v->domain;
+
+    return (addr >= (d->arch.vgic.dbase)) && (addr < (d->arch.vgic.dbase + d->arch.vgic.dbase_size));
+}
+
+static struct mmio_handler vgic_distr_mmio_handler = {
+    .check_handler = vgic_distr_mmio_check,
+    .read_handler  = vgic_distr_mmio_read,
+    .write_handler = vgic_distr_mmio_write,
+};
+
+
+static int vgic_vcpu_init(struct vcpu *v)
+{
+    int i;
+    u64 affinity;
+    struct vgic_irq_rank *vir;
+
+    vir =  xzalloc(struct vgic_irq_rank);
+
+    memset(vir, 0, sizeof(struct vgic_irq_rank));
+
+    spin_lock_init(&vir->lock);
+
+    v->arch.vgic.private_irqs = (struct vgic_irq_irank *)vir;
+
+    /* For SGI and PPI the target is always this CPU */
+    affinity = cpu_logical_map(v->vcpu_id);
+    for ( i = 0 ; i < 32 ; i++ )
+        ((struct vgic_irq_rank *)(v->arch.vgic.private_irqs))->irouter[i] = affinity;
+    return 0;
+}
+
+static struct vgic_ops ops = {
+    .vgic_vcpu_init   = vgic_vcpu_init,
+    .read_priority = vgic_read_priority,
+};
+
+int vgic_v3_init(struct domain *d)
+{
+    int i;
+    struct vgic_irq_rank *r;
+
+    d->arch.vgic.shared_irqs =
+        (struct vgic_irq_rank *)xzalloc_array(struct vgic_irq_rank, DOMAIN_NR_RANKS(d));
+
+    for (i=0; i < DOMAIN_NR_RANKS(d); i++) {
+        r = (struct vgic_irq_rank *)((unsigned char *)(d->arch.vgic.shared_irqs)
+                       + sizeof(struct vgic_irq_rank) * i);
+        spin_lock_init(&r->lock);
+    }
+    register_vgic_ops(&ops);
+
+    register_mmio_handler(&vgic_distr_mmio_handler);
+    register_mmio_handler(&vgic_rdistr_mmio_handler);
+    return 0;
+}
+
+/*
+ * Local variables:
+ * mode: C
+ * c-file-style: "BSD"
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
+
diff --git a/xen/arch/arm/vgic.c b/xen/arch/arm/vgic.c
index b60f012..79b9587 100644
--- a/xen/arch/arm/vgic.c
+++ b/xen/arch/arm/vgic.c
@@ -43,6 +43,11 @@ int domain_vgic_init(struct domain *d)
 {
     int i;
 
+    static const struct dt_device_match gicv3_ids[] __initconst =
+    {
+        DT_MATCH_GIC_V3,
+        { /* sentinel */ },
+    };
     static const struct dt_device_match gicv2_ids[] __initconst =
     {
         DT_MATCH_GIC,
@@ -60,6 +65,13 @@ int domain_vgic_init(struct domain *d)
     else
         d->arch.vgic.nr_lines = 0; /* We don't need SPIs for the guest */
 
+    node = dt_find_interrupt_controller(gicv3_ids);
+    if ( node )
+    {
+        if (vgic_v3_init(d))
+            panic("vgic v3 initialization failed");
+        goto out;
+    }
     node = dt_find_interrupt_controller(gicv2_ids);
     if ( node )
     {
diff --git a/xen/include/asm-arm/gic.h b/xen/include/asm-arm/gic.h
index b6fbd5e..0bce7f5 100644
--- a/xen/include/asm-arm/gic.h
+++ b/xen/include/asm-arm/gic.h
@@ -60,6 +60,7 @@ extern int vcpu_vgic_init(struct vcpu *v);
 extern int vcpu_gic_init(struct vcpu *v);
 extern void vgic_disable_irqs(struct vcpu *v, uint32_t r, int n);
 extern void vgic_enable_irqs(struct vcpu *v, uint32_t r, int n);
+extern int vgic_v3_init(struct domain *d);
 extern int vgic_v2_init(struct domain *d);
 
 extern void gicv3_init(void);
-- 
1.7.9.5

Re: [RFC PATCH v1 09/10] xen/arm: Add vgic support for GIC v3

[Stefano Stabellini]

On Wed, 19 Mar 2014, vijay.kilari@gmail.com wrote:
> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> 
> Add vgic driver support for GIC v3 version.
> 
> Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> ---
>  xen/arch/arm/Makefile     |    2 +-
>  xen/arch/arm/vgic-v3.c    |  927 +++++++++++++++++++++++++++++++++++++++++++++
>  xen/arch/arm/vgic.c       |   12 +
>  xen/include/asm-arm/gic.h |    1 +
>  4 files changed, 941 insertions(+), 1 deletion(-)
> 
> diff --git a/xen/arch/arm/Makefile b/xen/arch/arm/Makefile
> index a11c699..5a6cb07 100644
> --- a/xen/arch/arm/Makefile
> +++ b/xen/arch/arm/Makefile
> @@ -26,7 +26,7 @@ obj-y += smpboot.o
>  obj-y += smp.o
>  obj-y += shutdown.o
>  obj-y += traps.o
> -obj-y += vgic.o vgic-v2.o
> +obj-y += vgic.o vgic-v2.o vgic-v3.o
>  obj-y += vtimer.o
>  obj-y += vuart.o
>  obj-y += hvm.o
> diff --git a/xen/arch/arm/vgic-v3.c b/xen/arch/arm/vgic-v3.c
> new file mode 100644
> index 0000000..b5c1d1e
> --- /dev/null
> +++ b/xen/arch/arm/vgic-v3.c
> @@ -0,0 +1,927 @@
> +/*
> + * xen/arch/arm/vgic-v3.c
> + *
> + * ARM Virtual Generic Interrupt Controller v3 support
> + * based on xen/arch/arm/vgic.c
> + *
> + * Vijaya Kumar K <vijaya.kumar@caviumnetworks.com>
> + * Copyright (c) 2014 Cavium Inc.
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License as published by
> + * the Free Software Foundation; either version 2 of the License, or
> + * (at your option) any later version.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> + * GNU General Public License for more details.
> + */
> +
> +#include <xen/bitops.h>
> +#include <xen/config.h>
> +#include <xen/lib.h>
> +#include <xen/init.h>
> +#include <xen/softirq.h>
> +#include <xen/irq.h>
> +#include <xen/sched.h>
> +
> +#include <asm/current.h>
> +
> +#include "io.h"
> +#include <asm/gic-v3.h>
> +#include <asm/gic.h>
> +
> +struct vgic_irq_rank {
> +    spinlock_t lock; /* Covers access to all other members of this struct */
> +    uint32_t ienable, iactive, ipend, pendsgi;
> +    uint32_t icfg[2];
> +    uint32_t ipriority[8];
> +    uint64_t irouter[32];
> +};
> +
> +#define REG(n) (n)
> +
> +/* Number of ranks of interrupt registers for a domain */
> +#define DOMAIN_NR_RANKS(d) (((d)->arch.vgic.nr_lines+31)/32)
> +
> +/*
> + * Rank containing GICD_<FOO><n> for GICD_<FOO> with
> + * <b>-bits-per-interrupt
> + */
> +static inline int REG_RANK_NR(int b, uint32_t n)
> +{
> +    switch ( b )
> +    {
> +        case 64: return n >> 6;
> +        case 32: return n >> 5;
> +        case 16: return n >> 4;
> +        case 8: return n >> 3;
> +        case 4: return n >> 2;
> +        case 2: return n >> 1;
> +        case 1: return n;
> +        default: BUG();
> +    }
> +}
> +
> +/*
> + * Offset of GICD_<FOO><n> with its rank, for GICD_<FOO> with
> + * <b>-bits-per-interrupt.
> + */
> +/* Shift n >> 2 to make it byte register diff */
> +#define REG_RANK_INDEX(b, n) (((n) >> 2) & ((b)-1))
> +
> +/*
> + * Returns rank corresponding to a GICD_<FOO><n> register for
> + * GICD_<FOO> with <b>-bits-per-interrupt.
> + */
> +static struct vgic_irq_rank *vgic_irq_rank(struct vcpu *v, int b, int n)
> +{
> +    int rank;
> +
> +    n = n >> 2;
> +    rank  = REG_RANK_NR(b, n);
> +
> +    if ( rank == 0 ) /* Rank 0 is nothing but GICR registers in GICv3 */
> +        return (struct vgic_irq_rank *)v->arch.vgic.private_irqs;
> +    else if ( rank <= DOMAIN_NR_RANKS(v->domain) )
> +        return (struct vgic_irq_rank *)((unsigned char *)(v->domain->arch.vgic.shared_irqs)
> +                + (sizeof(struct vgic_irq_rank) *(rank - 1)));
> +    else
> +        return NULL;

code style


> +}
> +
> +#define vgic_lock(v)   spin_lock_irq(&(v)->domain->arch.vgic.lock)
> +#define vgic_unlock(v) spin_unlock_irq(&(v)->domain->arch.vgic.lock)
> +
> +#define vgic_lock_rank(v, r) spin_lock(&(r)->lock)
> +#define vgic_unlock_rank(v, r) spin_unlock(&(r)->lock)
> +
> +static uint32_t byte_read(uint32_t val, int sign, int offset)
> +{
> +    int byte = offset & 0x3;
> +
> +    val = val >> (8*byte);
> +    if ( sign && (val & 0x80) )
> +        val |= 0xffffff00;
> +    else
> +        val &= 0x000000ff;
> +    return val;
> +}
> +
> +static void byte_write(uint32_t *reg, uint32_t var, int offset)
> +{
> +    int byte = offset & 0x3;
> +
> +    var &= (0xff << (8*byte));
> +
> +    *reg &= ~(0xff << (8*byte));
> +    *reg |= var;
> +}
> +
> +static int vgic_read_priority(struct vcpu *v, int irq)
> +{
> +   struct vgic_irq_rank *rank = vgic_irq_rank(v, 8, irq);
> +   return byte_read(rank->ipriority[REG_RANK_INDEX(8, irq)], 0, irq & 0x3);
> +}
> + 
> +static int __vgic_rdistr_mmio_read(struct vcpu *v, mmio_info_t *info)
> +{
> +    struct hsr_dabt dabt = info->dabt;
> +    struct cpu_user_regs *regs = guest_cpu_user_regs();
> +    register_t *r = select_user_reg(regs, dabt.reg);
> +    int offset = (int)(info->gpa - gic_data_rdist_rd_base());
> +    int gicr_reg = REG(offset);
> +    u64 mpidr;
> +    u64 aff;
> +
> +    switch ( gicr_reg )
> +    {
> +    case GICR_CTLR:
> +        /* We have implemented LPI's, read zero */
> +        goto read_as_zero;
> +    case GICR_IIDR:
> +        *r = 0x34;
> +        return 1;
> +    case GICR_TYPER:
> +        if(((info->gpa) & (~((SZ_64K * 2 * smp_processor_id()) - 1))) ==
> +            (gic_data_rdist_rd_base() & (~((SZ_64K * 2 * smp_processor_id()) - 1))) )

Please add a comment to explain what you are doing here

[RFC PATCH v1 10/10] xen/arm: GICv3 device tree parsing

[vijay.kilari]

From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>

GICv3 supports system register access to GIC cpu interface.
So no need to read device tree for cpu interface.
GICv3 adds Re-distributor region and redistributor stride
which are parsed.

Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
---
 xen/arch/arm/domain_build.c |   41 +++++++++++++++++++++++++++++++++++------
 1 file changed, 35 insertions(+), 6 deletions(-)

diff --git a/xen/arch/arm/domain_build.c b/xen/arch/arm/domain_build.c
index 5ca2f15..c88c5ae 100644
--- a/xen/arch/arm/domain_build.c
+++ b/xen/arch/arm/domain_build.c
@@ -532,6 +532,8 @@ static int make_gic_node(const struct domain *d, void *fdt,
     u32 len;
     __be32 *new_cells, *tmp;
     int res = 0;
+    int hw_type = GIC_VERSION_V2;
+    u32 rd_stride = 0;
 
     /*
      * Xen currently supports only a single GIC. Discard any secondary
@@ -545,6 +547,8 @@ static int make_gic_node(const struct domain *d, void *fdt,
 
     DPRINT("Create gic node\n");
 
+    hw_type = gic_hw_version();
+
     compatible = dt_get_property(gic, "compatible", &len);
     if ( !compatible )
     {
@@ -552,6 +556,12 @@ static int make_gic_node(const struct domain *d, void *fdt,
         return -FDT_ERR_XEN(ENOENT);
     }
 
+    if (hw_type == GIC_VERSION_V3)
+    {
+        res = dt_property_read_u32(gic, "redistributor-stride", &rd_stride);
+        if ( !res )
+            rd_stride = 0;
+    }
     res = fdt_begin_node(fdt, "interrupt-controller");
     if ( res )
         return res;
@@ -569,6 +579,13 @@ static int make_gic_node(const struct domain *d, void *fdt,
     if ( res )
         return res;
 
+    if (hw_type == GIC_VERSION_V3)
+    {
+        res = fdt_property_cell(fdt, "redistributor-stride", rd_stride);
+        if ( res )
+            return res;
+        DPRINT("  gicv3 rd stride 0x%x\n", rd_stride);
+    }
     len = dt_cells_to_size(dt_n_addr_cells(node) + dt_n_size_cells(node));
     len *= 2; /* GIC has two memory regions: Distributor + CPU interface */
     new_cells = xzalloc_bytes(len);
@@ -576,14 +593,26 @@ static int make_gic_node(const struct domain *d, void *fdt,
         return -FDT_ERR_XEN(ENOMEM);
 
     tmp = new_cells;
-    DPRINT("  Set Distributor Base 0x%"PRIpaddr"-0x%"PRIpaddr"\n",
-           d->arch.vgic.dbase, d->arch.vgic.dbase + PAGE_SIZE - 1);
-    dt_set_range(&tmp, node, d->arch.vgic.dbase, PAGE_SIZE);
+    if (hw_type == GIC_VERSION_V3)
+    {
+        DPRINT("  Set Distributor Base 0x%"PRIpaddr"-0x%"PRIpaddr"\n",
+        d->arch.vgic.dbase, d->arch.vgic.dbase + d->arch.vgic.dbase_size - 1);
+        dt_set_range(&tmp, node, d->arch.vgic.dbase, d->arch.vgic.dbase_size);
 
-    DPRINT("  Set Cpu Base 0x%"PRIpaddr"-0x%"PRIpaddr"\n",
-           d->arch.vgic.cbase, d->arch.vgic.cbase + (PAGE_SIZE * 2) - 1);
-    dt_set_range(&tmp, node, d->arch.vgic.cbase, PAGE_SIZE * 2);
+        DPRINT("  Set  Re-distributor Base 0x%"PRIpaddr"-0x%"PRIpaddr"\n",
+        d->arch.vgic.cbase, d->arch.vgic.rbase + d->arch.vgic.rbase_size - 1);
+        dt_set_range(&tmp, node, d->arch.vgic.rbase, d->arch.vgic.rbase_size);
+    }
+    else
+    {
+        DPRINT("  Set Distributor Base 0x%"PRIpaddr"-0x%"PRIpaddr"\n",
+        d->arch.vgic.dbase, d->arch.vgic.dbase + PAGE_SIZE - 1);
+        dt_set_range(&tmp, node, d->arch.vgic.dbase, PAGE_SIZE);
 
+        DPRINT("  Set Cpu Base 0x%"PRIpaddr"-0x%"PRIpaddr"\n",
+        d->arch.vgic.cbase, d->arch.vgic.cbase + (PAGE_SIZE * 2) - 1);
+        dt_set_range(&tmp, node, d->arch.vgic.cbase, PAGE_SIZE * 2);
+    }
     res = fdt_property(fdt, "reg", new_cells, len);
     xfree(new_cells);
 
-- 
1.7.9.5

Re: [RFC PATCH v1 10/10] xen/arm: GICv3 device tree parsing

[Julien Grall]

Hello Vijay,

Thanks for the patch. I think the name of this patch is wrong. You are
not parsing the device tree but creating the node for DOM0.

On 03/19/2014 02:17 PM, vijay.kilari@gmail.com wrote:
> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> 
> GICv3 supports system register access to GIC cpu interface.
> So no need to read device tree for cpu interface.
> GICv3 adds Re-distributor region and redistributor stride
> which are parsed.

If I understand correctly, DOM0 kernel must support GICv3, right? What
about booting DOM0 with only GICv2 support, e.g. because the kernel is
only using less than 8 CPUs?

[..]

> +    if (hw_type == GIC_VERSION_V3)
> +    {
> +        res = dt_property_read_u32(gic, "redistributor-stride", &rd_stride);
> +        if ( !res )
> +            rd_stride = 0;
> +    }

Actually I would prefer if you introduce an callback to create these new
property.

Regards,

-- 
Julien Grall

Re: [RFC PATCH v1 10/10] xen/arm: GICv3 device tree parsing

[Vijay Kilari]

Hi,

 Julien,

On Thu, Mar 20, 2014 at 9:38 PM, Julien Grall <julien.grall@linaro.org> wrote:
> Hello Vijay,
>
> Thanks for the patch. I think the name of this patch is wrong. You are
> not parsing the device tree but creating the node for DOM0.
>
OK

> On 03/19/2014 02:17 PM, vijay.kilari@gmail.com wrote:
>> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>>
>> GICv3 supports system register access to GIC cpu interface.
>> So no need to read device tree for cpu interface.
>> GICv3 adds Re-distributor region and redistributor stride
>> which are parsed.
>
> If I understand correctly, DOM0 kernel must support GICv3, right? What
> about booting DOM0 with only GICv2 support, e.g. because the kernel is
> only using less than 8 CPUs?
>
Yes, correct.
If DOM0 supports only V2, then Xen should load vgic v2 driver?
if so, how to detect it?

> [..]
>
>> +    if (hw_type == GIC_VERSION_V3)
>> +    {
>> +        res = dt_property_read_u32(gic, "redistributor-stride", &rd_stride);
>> +        if ( !res )
>> +            rd_stride = 0;
>> +    }
>
> Actually I would prefer if you introduce an callback to create these new
> property.
>
OK. you mean callback of driver should be called from here

> Regards,
>
> --
> Julien Grall

Re: [RFC PATCH v1 10/10] xen/arm: GICv3 device tree parsing

[Ian Campbell]

On Sat, 2014-03-22 at 16:00 +0530, Vijay Kilari wrote:
> Hi,
> 
>  Julien,
> 
> On Thu, Mar 20, 2014 at 9:38 PM, Julien Grall <julien.grall@linaro.org> wrote:
> > Hello Vijay,
> >
> > Thanks for the patch. I think the name of this patch is wrong. You are
> > not parsing the device tree but creating the node for DOM0.
> >
> OK
> 
> > On 03/19/2014 02:17 PM, vijay.kilari@gmail.com wrote:
> >> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> >>
> >> GICv3 supports system register access to GIC cpu interface.
> >> So no need to read device tree for cpu interface.
> >> GICv3 adds Re-distributor region and redistributor stride
> >> which are parsed.
> >
> > If I understand correctly, DOM0 kernel must support GICv3, right? What
> > about booting DOM0 with only GICv2 support, e.g. because the kernel is
> > only using less than 8 CPUs?
> >
> Yes, correct.
> If DOM0 supports only V2, then Xen should load vgic v2 driver?
> if so, how to detect it?

There are four cases I suppose:

Two obvious ones:
HW: GICv2 + Xen: GICv2 + Dom0: GICv2
HW: GICv3 + Xen: GICv3 + Dom0: GICv3

Then:
HW: GICv3 + Xen: GICv2 (h/w compat mode) + Dom0: GICv2

Which I think we effectively aren't implementing, since we will always
use the GICv3 driver on GICv3 hardware. Or maybe this is just a case of
the first one above when the DT describes a GICv2 even though the h/w is
actually v3. I think we can ignore this case really.

Which leaves:

HW: GICv3 + Xen: GICv3 + Dom0: GICv2

Which would require extra support in Xen I suppose, to provide the v2
emulation in terms of v3?

There is also the question of what guests use, which I suppose it mostly
the same as dom0, but without the expectation that the guest kernel has
to boot natively on a specific platform.

I'm not sure what to suggest here, but I think for dom0 we can rely on
"dom0 kernel should be usable as a native kernel on this platform" and
we can punt for now on what to do about guests.

Ian.

Re: [RFC PATCH v1 10/10] xen/arm: GICv3 device tree parsing

[Julien Grall]

On 03/24/2014 11:43 AM, Ian Campbell wrote:
> There is also the question of what guests use, which I suppose it mostly
> the same as dom0, but without the expectation that the guest kernel has
> to boot natively on a specific platform.
> 
> I'm not sure what to suggest here, but I think for dom0 we can rely on
> "dom0 kernel should be usable as a native kernel on this platform" and
> we can punt for now on what to do about guests.

As GICv3 is 64-bits only, that would mean you can't boot 32bit dom0 on
this platform, right?

-- 
Julien Grall

Re: [RFC PATCH v1 10/10] xen/arm: GICv3 device tree parsing

[Ian Campbell]

On Mon, 2014-03-24 at 12:03 +0000, Julien Grall wrote:
> On 03/24/2014 11:43 AM, Ian Campbell wrote:
> > There is also the question of what guests use, which I suppose it mostly
> > the same as dom0, but without the expectation that the guest kernel has
> > to boot natively on a specific platform.
> > 
> > I'm not sure what to suggest here, but I think for dom0 we can rely on
> > "dom0 kernel should be usable as a native kernel on this platform" and
> > we can punt for now on what to do about guests.
> 
> As GICv3 is 64-bits only, that would mean you can't boot 32bit dom0 on
> this platform, right?

I guess not. Is that a huge deal? At the very least I reckon we can punt
on it for now.

Ian.

Re: [RFC PATCH v1 10/10] xen/arm: GICv3 device tree parsing

[Julien Grall]

On 03/24/2014 12:03 PM, Julien Grall wrote:
> On 03/24/2014 11:43 AM, Ian Campbell wrote:
>> There is also the question of what guests use, which I suppose it mostly
>> the same as dom0, but without the expectation that the guest kernel has
>> to boot natively on a specific platform.
>>
>> I'm not sure what to suggest here, but I think for dom0 we can rely on
>> "dom0 kernel should be usable as a native kernel on this platform" and
>> we can punt for now on what to do about guests.
> 
> As GICv3 is 64-bits only, that would mean you can't boot 32bit dom0 on
> this platform, right?

Hmmm ... this question was stupid. GICv8 is ARMv8 specific not 64bits.
Sorry for the noise.

I think for the guest we have to support VGICv2 on GICv3 (if the
hardware support it). The user might want to try his shiny OS which only
support GICv2 on any hardware. It's easier than writing a support for GICv3.

-- 
Julien Grall

Re: [RFC PATCH v1 10/10] xen/arm: GICv3 device tree parsing

[Stefano Stabellini]

On Mon, 24 Mar 2014, Julien Grall wrote:
> On 03/24/2014 12:03 PM, Julien Grall wrote:
> > On 03/24/2014 11:43 AM, Ian Campbell wrote:
> >> There is also the question of what guests use, which I suppose it mostly
> >> the same as dom0, but without the expectation that the guest kernel has
> >> to boot natively on a specific platform.
> >>
> >> I'm not sure what to suggest here, but I think for dom0 we can rely on
> >> "dom0 kernel should be usable as a native kernel on this platform" and
> >> we can punt for now on what to do about guests.
> > 
> > As GICv3 is 64-bits only, that would mean you can't boot 32bit dom0 on
> > this platform, right?
> 
> Hmmm ... this question was stupid. GICv8 is ARMv8 specific not 64bits.
> Sorry for the noise.
> 
> I think for the guest we have to support VGICv2 on GICv3 (if the
> hardware support it). The user might want to try his shiny OS which only
> support GICv2 on any hardware. It's easier than writing a support for GICv3.

I think that for Dom0 we have to use vgic_v3, because it doesn't only
give you support for more vcpus but also MSI and MSI-X delivery.


For DomUs it might be important to support vgic_v2 on gicv3 hardware,
however I wouldn't want to defer the decision to the user (i.e.
introduce yet another VM config option), if not for debugging.

Would it be possible to advirtise both gicv2 and gicv3 on device tree?
What would the guest kernel do in that case?

Otherwise we could default to vgic_v2 if vcpus <= 8 and vgic_v3 if
vcpus > 8.

Re: [RFC PATCH v1 10/10] xen/arm: GICv3 device tree parsing

[Julien Grall]

Hi Stefano,

On 24/03/14 17:34, Stefano Stabellini wrote:
> I think that for Dom0 we have to use vgic_v3, because it doesn't only
> give you support for more vcpus but also MSI and MSI-X delivery.

Right.

>
> For DomUs it might be important to support vgic_v2 on gicv3 hardware,
> however I wouldn't want to defer the decision to the user (i.e.
> introduce yet another VM config option), if not for debugging.
>
> Would it be possible to advirtise both gicv2 and gicv3 on device tree?
> What would the guest kernel do in that case?

Linux will try to load both GICv2 and GICv3 drivers. In any case I don't 
think it's a solution because:
	- you don't know how the kernel will react
         - how will you choose which backend to use?

> Otherwise we could default to vgic_v2 if vcpus <= 8 and vgic_v3 if
> vcpus > 8.

What about kernel which only support GICv3 and have less than 8 VCPUS?
What about of device (MSI, ...) passthrough with this kind of kernel?

I think we need to have a VM config option in this specific case.

Regards,

-- 
Julien Grall

Re: [RFC PATCH v1 10/10] xen/arm: GICv3 device tree parsing

[Stefano Stabellini]

On Mon, 24 Mar 2014, Julien Grall wrote:
> Hi Stefano,
> 
> On 24/03/14 17:34, Stefano Stabellini wrote:
> > I think that for Dom0 we have to use vgic_v3, because it doesn't only
> > give you support for more vcpus but also MSI and MSI-X delivery.
> 
> Right.
> 
> > 
> > For DomUs it might be important to support vgic_v2 on gicv3 hardware,
> > however I wouldn't want to defer the decision to the user (i.e.
> > introduce yet another VM config option), if not for debugging.
> > 
> > Would it be possible to advirtise both gicv2 and gicv3 on device tree?
> > What would the guest kernel do in that case?
> 
> Linux will try to load both GICv2 and GICv3 drivers. In any case I don't think
> it's a solution because:
> 	- you don't know how the kernel will react
>         - how will you choose which backend to use?
> 
> > Otherwise we could default to vgic_v2 if vcpus <= 8 and vgic_v3 if
> > vcpus > 8.
> 
> What about kernel which only support GICv3 and have less than 8 VCPUS?
> What about of device (MSI, ...) passthrough with this kind of kernel?
> 
> I think we need to have a VM config option in this specific case.

Having a VM config option is OK, but it should be the last resort for
people doing something very uncommon.

GICv3 support in Linux and distros is probably going to be widespread
soon, so we might not have to worry about it though.

Re: [RFC PATCH v1 10/10] xen/arm: GICv3 device tree parsing

[Julien Grall]

On 03/25/2014 11:04 AM, Stefano Stabellini wrote:
> On Mon, 24 Mar 2014, Julien Grall wrote:
>> Hi Stefano,
>>
>> On 24/03/14 17:34, Stefano Stabellini wrote:
>>> I think that for Dom0 we have to use vgic_v3, because it doesn't only
>>> give you support for more vcpus but also MSI and MSI-X delivery.
>>
>> Right.
>>
>>>
>>> For DomUs it might be important to support vgic_v2 on gicv3 hardware,
>>> however I wouldn't want to defer the decision to the user (i.e.
>>> introduce yet another VM config option), if not for debugging.
>>>
>>> Would it be possible to advirtise both gicv2 and gicv3 on device tree?
>>> What would the guest kernel do in that case?
>>
>> Linux will try to load both GICv2 and GICv3 drivers. In any case I don't think
>> it's a solution because:
>> 	- you don't know how the kernel will react
>>         - how will you choose which backend to use?
>>
>>> Otherwise we could default to vgic_v2 if vcpus <= 8 and vgic_v3 if
>>> vcpus > 8.
>>
>> What about kernel which only support GICv3 and have less than 8 VCPUS?
>> What about of device (MSI, ...) passthrough with this kind of kernel?
>>
>> I think we need to have a VM config option in this specific case.
> 
> Having a VM config option is OK, but it should be the last resort for
> people doing something very uncommon.
> 
> GICv3 support in Linux and distros is probably going to be widespread
> soon, so we might not have to worry about it though.

I don't think GICv3 will supported in 3.15. The current series doesn't
support GICv3 on 32 bits
(http://www.spinics.net/lists/arm-kernel/msg317028.html).

But, I'm more worry about other OS than Linux (e.g FreeRTOS, *BSD,...)
which may not support GICv2 before a while.

Regards,

-- 
Julien Grall

Re: [RFC PATCH v1 10/10] xen/arm: GICv3 device tree parsing

[Julien Grall]

On 03/25/2014 12:33 PM, Julien Grall wrote:
> On 03/25/2014 11:04 AM, Stefano Stabellini wrote:
>> On Mon, 24 Mar 2014, Julien Grall wrote:
>>> Hi Stefano,
>>>
>>> On 24/03/14 17:34, Stefano Stabellini wrote:
>>>> I think that for Dom0 we have to use vgic_v3, because it doesn't only
>>>> give you support for more vcpus but also MSI and MSI-X delivery.
>>>
>>> Right.
>>>
>>>>
>>>> For DomUs it might be important to support vgic_v2 on gicv3 hardware,
>>>> however I wouldn't want to defer the decision to the user (i.e.
>>>> introduce yet another VM config option), if not for debugging.
>>>>
>>>> Would it be possible to advirtise both gicv2 and gicv3 on device tree?
>>>> What would the guest kernel do in that case?
>>>
>>> Linux will try to load both GICv2 and GICv3 drivers. In any case I don't think
>>> it's a solution because:
>>> 	- you don't know how the kernel will react
>>>         - how will you choose which backend to use?
>>>
>>>> Otherwise we could default to vgic_v2 if vcpus <= 8 and vgic_v3 if
>>>> vcpus > 8.
>>>
>>> What about kernel which only support GICv3 and have less than 8 VCPUS?
>>> What about of device (MSI, ...) passthrough with this kind of kernel?
>>>
>>> I think we need to have a VM config option in this specific case.
>>
>> Having a VM config option is OK, but it should be the last resort for
>> people doing something very uncommon.
>>
>> GICv3 support in Linux and distros is probably going to be widespread
>> soon, so we might not have to worry about it though.
> 
> I don't think GICv3 will supported in 3.15. The current series doesn't
> support GICv3 on 32 bits
> (http://www.spinics.net/lists/arm-kernel/msg317028.html).
> 
> But, I'm more worry about other OS than Linux (e.g FreeRTOS, *BSD,...)
> which may not support GICv2 before a while.

I should have write GICv3.

-- 
Julien Grall

Re: [RFC PATCH v1 10/10] xen/arm: GICv3 device tree parsing

[Ian Campbell]

On Tue, 2014-03-25 at 12:33 +0000, Julien Grall wrote:
> But, I'm more worry about other OS than Linux (e.g FreeRTOS, *BSD,...)
> which may not support GICv2 before a while.

Note that this is dependent on the hardware, v2 compat mode is optional
in v3 and emulating a v2 gic on v3 without this functionality is going
to be hard and complicated enough that I think we should rule that out
right now, at least for the medium term.

Even if v2 compat is available in the h/w I'd much rather we got basic
support for GICv3 in as a first step and then start worrying about
guests which need v2 compat mode to be present.

Ian.

Re: [RFC PATCH v1 10/10] xen/arm: GICv3 device tree parsing

[Julien Grall]

On 04/01/2014 01:59 PM, Ian Campbell wrote:
> On Tue, 2014-03-25 at 12:33 +0000, Julien Grall wrote:
>> But, I'm more worry about other OS than Linux (e.g FreeRTOS, *BSD,...)
>> which may not support GICv2 before a while.
> 
> Note that this is dependent on the hardware, v2 compat mode is optional
> in v3 and emulating a v2 gic on v3 without this functionality is going
> to be hard and complicated enough that I think we should rule that out
> right now, at least for the medium term.

> Even if v2 compat is available in the h/w I'd much rather we got basic
> support for GICv3 in as a first step and then start worrying about
> guests which need v2 compat mode to be present.

Right. I'd like to make sure that even if we don't support vGICv2 on
GICv3 right now, we are going to the right direction to support it. So
the interface is clearly defined between GIC and VGIC emulation and we
don't have lots of "if gicv2_host then foo" in guest code.

-- 
Julien Grall

Re: [RFC PATCH v1 00/10] xen/arm: Add GICv3 support

[Stefano Stabellini]

On Wed, 19 Mar 2014, vijay.kilari@gmail.com wrote:
> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
> 
> made changes to existing gic and vgic drivers
> to make it generic and added support for GIC v3
> hw version
> 
> Tested with ARM64 simulator with single core
> 
> Vijaya Kumar K (10):
>   xen/arm: make secondary gic init as notifier call
>   xen/arm: register mmio handler at runtime
>   xen/arm: move vgic data to vgic driver
>   arm/xen: move gic save and restore registers to gic driver
>   xen/arm: move gic definitions to seperate file
>   xen/arm: split gic driver into generic and gicv2 driver
>   xen/arm: split vgic into generic and GIC v2 specific drivers
>   xen/arm: Add support for GIC v3
>   xen/arm: Add vgic support for GIC v3
>   xen/arm: GICv3 device tree parsing
> 
>  xen/arch/arm/Makefile             |    4 +-
>  xen/arch/arm/domain.c             |    3 +
>  xen/arch/arm/domain_build.c       |   41 +-
>  xen/arch/arm/gic-v2.c             |  562 ++++++++++++++++++++++
>  xen/arch/arm/gic-v3.c             |  944 +++++++++++++++++++++++++++++++++++++
>  xen/arch/arm/gic.c                |  512 +++++---------------
>  xen/arch/arm/io.c                 |   32 +-
>  xen/arch/arm/io.h                 |   16 +-
>  xen/arch/arm/smpboot.c            |    3 +-
>  xen/arch/arm/vgic-v2.c            |  636 +++++++++++++++++++++++++
>  xen/arch/arm/vgic-v3.c            |  927 ++++++++++++++++++++++++++++++++++++
>  xen/arch/arm/vgic.c               |  591 ++---------------------
>  xen/arch/arm/vuart.c              |   39 +-
>  xen/include/asm-arm/domain.h      |   20 +-
>  xen/include/asm-arm/gic.h         |  171 +++----
>  xen/include/asm-arm/gic_v2_defs.h |  134 ++++++
>  xen/include/asm-arm/gic_v3_defs.h |  211 +++++++++
>  17 files changed, 3733 insertions(+), 1113 deletions(-)
>  create mode 100644 xen/arch/arm/gic-v2.c
>  create mode 100644 xen/arch/arm/gic-v3.c
>  create mode 100644 xen/arch/arm/vgic-v2.c
>  create mode 100644 xen/arch/arm/vgic-v3.c
>  create mode 100644 xen/include/asm-arm/gic_v2_defs.h
>  create mode 100644 xen/include/asm-arm/gic_v3_defs.h

Do you have a git tree somewhere with the patches?