Re: [Qemu-devel] [RFC v2 2/3] intc/arm_gic: Support IRQ injection for more than 256 vpus

[Auger Eric <eric.auger@redhat.com>]

Hi Drew,

On 9/12/19 9:36 AM, Andrew Jones wrote:
> On Wed, Sep 11, 2019 at 05:51:24PM +0200, Eric Auger wrote:
>> Host kernels that expose the KVM_CAP_ARM_IRQ_LINE_LAYOUT_2 capability
>> allow injection of interrupts along with vcpu ids larger than 255.
>> Let's encode the vpcu id on 12 bits according to the upgraded KVM_IRQ_LINE
>> ABI when needed.
>>
>> Given that we have two callsites that need to assemble
>> the value for kvm_set_irq(), a new helper routine, kvm_arm_set_irq
>> is introduced.
>>
>> Without that patch qemu exits with "kvm_set_irq: Invalid argument"
>> message.
>>
>> Signed-off-by: Eric Auger <eric.auger@redhat.com>
>> Reported-by: Zenghui Yu <yuzenghui@huawei.com>
>> ---
>>  hw/intc/arm_gic_kvm.c |  7 ++-----
>>  target/arm/cpu.c      | 10 ++++------
>>  target/arm/kvm.c      | 16 ++++++++++++++++
>>  target/arm/kvm_arm.h  |  1 +
>>  4 files changed, 23 insertions(+), 11 deletions(-)
>>
>> diff --git a/hw/intc/arm_gic_kvm.c b/hw/intc/arm_gic_kvm.c
>> index b56fda144f..9deb15e7e6 100644
>> --- a/hw/intc/arm_gic_kvm.c
>> +++ b/hw/intc/arm_gic_kvm.c
>> @@ -55,7 +55,7 @@ void kvm_arm_gic_set_irq(uint32_t num_irq, int irq, int level)
>>       * has separate fields in the irq number for type,
>>       * CPU number and interrupt number.
>>       */
>> -    int kvm_irq, irqtype, cpu;
>> +    int irqtype, cpu;
>>  
>>      if (irq < (num_irq - GIC_INTERNAL)) {
>>          /* External interrupt. The kernel numbers these like the GIC
>> @@ -72,10 +72,7 @@ void kvm_arm_gic_set_irq(uint32_t num_irq, int irq, int level)
>>          cpu = irq / GIC_INTERNAL;
>>          irq %= GIC_INTERNAL;
>>      }
>> -    kvm_irq = (irqtype << KVM_ARM_IRQ_TYPE_SHIFT)
>> -        | (cpu << KVM_ARM_IRQ_VCPU_SHIFT) | irq;
>> -
>> -    kvm_set_irq(kvm_state, kvm_irq, !!level);
>> +    kvm_arm_set_irq(cpu, irqtype, irq, !!level);
>>  }
>>  
>>  static void kvm_arm_gicv2_set_irq(void *opaque, int irq, int level)
>> diff --git a/target/arm/cpu.c b/target/arm/cpu.c
>> index 2399c14471..13813fb213 100644
>> --- a/target/arm/cpu.c
>> +++ b/target/arm/cpu.c
>> @@ -576,16 +576,16 @@ static void arm_cpu_kvm_set_irq(void *opaque, int irq, int level)
>>      ARMCPU *cpu = opaque;
>>      CPUARMState *env = &cpu->env;
>>      CPUState *cs = CPU(cpu);
>> -    int kvm_irq = KVM_ARM_IRQ_TYPE_CPU << KVM_ARM_IRQ_TYPE_SHIFT;
>>      uint32_t linestate_bit;
>> +    int irq_id;
>>  
>>      switch (irq) {
>>      case ARM_CPU_IRQ:
>> -        kvm_irq |= KVM_ARM_IRQ_CPU_IRQ;
>> +        irq_id = KVM_ARM_IRQ_CPU_IRQ;
>>          linestate_bit = CPU_INTERRUPT_HARD;
>>          break;
>>      case ARM_CPU_FIQ:
>> -        kvm_irq |= KVM_ARM_IRQ_CPU_FIQ;
>> +        irq_id = KVM_ARM_IRQ_CPU_FIQ;
>>          linestate_bit = CPU_INTERRUPT_FIQ;
>>          break;
>>      default:
>> @@ -597,9 +597,7 @@ static void arm_cpu_kvm_set_irq(void *opaque, int irq, int level)
>>      } else {
>>          env->irq_line_state &= ~linestate_bit;
>>      }
>> -
>> -    kvm_irq |= cs->cpu_index << KVM_ARM_IRQ_VCPU_SHIFT;
>> -    kvm_set_irq(kvm_state, kvm_irq, level ? 1 : 0);
>> +    kvm_arm_set_irq(cs->cpu_index, KVM_ARM_IRQ_TYPE_CPU, irq_id, !!level);
>>  #endif
>>  }
>>  
>> diff --git a/target/arm/kvm.c b/target/arm/kvm.c
>> index b2eaa50b8d..6cdfa2204f 100644
>> --- a/target/arm/kvm.c
>> +++ b/target/arm/kvm.c
>> @@ -744,6 +744,22 @@ int kvm_arm_vgic_probe(void)
>>      }
>>  }
>>  
>> +int kvm_arm_set_irq(int cpu, int irqtype, int irq, int level)
>> +{
>> +    int kvm_irq = 0;
> 
> No need to init to zero, and could just immediately init with the
> line below instead.
> 
>> +
>> +    kvm_irq = (irqtype << KVM_ARM_IRQ_TYPE_SHIFT) | irq;
>> +
>> +    if (cpu != 0) {
> 
> No need for this 'if'
> 
>> +        int cpu_idx2 = cpu / 256;
>> +        int cpu_idx1 = cpu % 256;
>> +
>> +        kvm_irq |= (cpu_idx1 << KVM_ARM_IRQ_VCPU_SHIFT) |
>> +           ((cpu_idx2 & KVM_ARM_IRQ_VCPU2_MASK) << KVM_ARM_IRQ_VCPU2_SHIFT);
> 
> Masking should be unnecessary as the only way it'll do anything is if we
> have vcpus >= 4096, which I imagine will never happen or will be guarded
> against happening elsewhere. Silently masking doesn't look right anyway,
> so I'd either add an assert(cpu_idx2 < 16) and drop the masking or just
> drop the masking.

All that makes sense.

Thank you for the review.

Eric
> 
>> +    }
>> +    return kvm_set_irq(kvm_state, kvm_irq, !!level);
>> +}
>> +
>>  int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route,
>>                               uint64_t address, uint32_t data, PCIDevice *dev)
>>  {
>> diff --git a/target/arm/kvm_arm.h b/target/arm/kvm_arm.h
>> index b3106c8600..b4e19457a0 100644
>> --- a/target/arm/kvm_arm.h
>> +++ b/target/arm/kvm_arm.h
>> @@ -253,6 +253,7 @@ int kvm_arm_vgic_probe(void);
>>  
>>  void kvm_arm_pmu_set_irq(CPUState *cs, int irq);
>>  void kvm_arm_pmu_init(CPUState *cs);
>> +int kvm_arm_set_irq(int cpu, int irqtype, int irq, int level);
>>  
>>  #else
>>  
>> -- 
>> 2.20.1
>>
>>
> 
> Thanks,
> drew 
>