[PATCH 0/3] qemu-kvm: pci-assign: Host IRQ sharing suppport

[PATCH 0/3] qemu-kvm: pci-assign: Host IRQ sharing suppport

[Jan Kiszka]

This enables support for the host IRQ sharing support via PCI-2.3
interrupt masking. See the patches for details.

Patch 1 could be considered for uq/master although we have no use case
for the changes in upstream yet.

Jan Kiszka (3):
  kvm: Update kernel headers against kvm.git
  pci-assign: Use PCI-2.3-based shared legacy interrupts
  pci_assign: Flip defaults of prefer_msi and share_intx

 hw/device-assignment.c       |   23 ++++++++++++++++++++++-
 hw/device-assignment.h       |   10 ++++++----
 linux-headers/asm-s390/kvm.h |    2 ++
 linux-headers/linux/kvm.h    |    6 ++++++
 qemu-kvm.c                   |    9 +++++++++
 qemu-kvm.h                   |    2 ++
 6 files changed, 47 insertions(+), 5 deletions(-)

-- 
1.7.3.4

[PATCH 1/3] kvm: Update kernel headers against kvm.git

[Jan Kiszka]

Required for INTx masking / host IRQ sharing support of assigned
devices.

Corresponding kvm.git hash: 186195928e

Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
---
 linux-headers/asm-s390/kvm.h |    2 ++
 linux-headers/linux/kvm.h    |    6 ++++++
 2 files changed, 8 insertions(+), 0 deletions(-)

diff --git a/linux-headers/asm-s390/kvm.h b/linux-headers/asm-s390/kvm.h
index 9acbde4..9607667 100644
--- a/linux-headers/asm-s390/kvm.h
+++ b/linux-headers/asm-s390/kvm.h
@@ -44,10 +44,12 @@ struct kvm_guest_debug_arch {
 #define KVM_SYNC_PREFIX (1UL << 0)
 #define KVM_SYNC_GPRS   (1UL << 1)
 #define KVM_SYNC_ACRS   (1UL << 2)
+#define KVM_SYNC_CRS    (1UL << 3)
 /* definition of registers in kvm_run */
 struct kvm_sync_regs {
 	__u64 prefix;	/* prefix register */
 	__u64 gprs[16];	/* general purpose registers */
 	__u32 acrs[16];	/* access registers */
+	__u64 crs[16];	/* control registers */
 };
 #endif
diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
index f6b5343..cf7b098 100644
--- a/linux-headers/linux/kvm.h
+++ b/linux-headers/linux/kvm.h
@@ -588,6 +588,7 @@ struct kvm_ppc_pvinfo {
 #define KVM_CAP_TSC_DEADLINE_TIMER 72
 #define KVM_CAP_S390_UCONTROL 73
 #define KVM_CAP_SYNC_REGS 74
+#define KVM_CAP_PCI_2_3 75
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
@@ -784,6 +785,9 @@ struct kvm_s390_ucas_mapping {
 /* Available with KVM_CAP_TSC_CONTROL */
 #define KVM_SET_TSC_KHZ           _IO(KVMIO,  0xa2)
 #define KVM_GET_TSC_KHZ           _IO(KVMIO,  0xa3)
+/* Available with KVM_CAP_PCI_2_3 */
+#define KVM_ASSIGN_SET_INTX_MASK  _IOW(KVMIO,  0xa4, \
+				       struct kvm_assigned_pci_dev)
 
 /*
  * ioctls for vcpu fds
@@ -857,6 +861,8 @@ struct kvm_s390_ucas_mapping {
 #define KVM_SET_ONE_REG		  _IOW(KVMIO,  0xac, struct kvm_one_reg)
 
 #define KVM_DEV_ASSIGN_ENABLE_IOMMU	(1 << 0)
+#define KVM_DEV_ASSIGN_PCI_2_3		(1 << 1)
+#define KVM_DEV_ASSIGN_MASK_INTX	(1 << 2)
 
 struct kvm_assigned_pci_dev {
 	__u32 assigned_dev_id;
-- 
1.7.3.4

[PATCH 2/3] pci-assign: Use PCI-2.3-based shared legacy interrupts

[Jan Kiszka]

Enable the new KVM feature that allows legacy interrupt sharing for
PCI-2.3-compliant devices. This requires to synchronize any guest
change of the INTx mask bit to the kernel.

The feature is controlled by the property 'share_intx' and is off by
default for now.

Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
---
 hw/device-assignment.c |   21 +++++++++++++++++++++
 hw/device-assignment.h |   10 ++++++----
 qemu-kvm.c             |    9 +++++++++
 qemu-kvm.h             |    2 ++
 4 files changed, 38 insertions(+), 4 deletions(-)

diff --git a/hw/device-assignment.c b/hw/device-assignment.c
index 0e2c7b5..45e7913 100644
--- a/hw/device-assignment.c
+++ b/hw/device-assignment.c
@@ -782,6 +782,13 @@ static int assign_device(AssignedDevice *dev)
                 "cause host memory corruption if the device issues DMA write "
                 "requests!\n");
     }
+    if (dev->features & ASSIGNED_DEVICE_SHARE_INTX_MASK) {
+        assigned_dev_data.flags |= KVM_DEV_ASSIGN_PCI_2_3;
+
+        /* hide host-side INTx masking from the guest */
+        dev->emulate_config_read[PCI_COMMAND + 1] |=
+            PCI_COMMAND_INTX_DISABLE >> 8;
+    }
 
     r = kvm_assign_pci_device(kvm_state, &assigned_dev_data);
     if (r < 0) {
@@ -1122,9 +1129,21 @@ static void assigned_dev_pci_write_config(PCIDevice *pci_dev, uint32_t address,
 {
     AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
     uint32_t emulate_mask, full_emulation_mask;
+    int ret;
 
     pci_default_write_config(pci_dev, address, val, len);
 
+    if (range_covers_byte(address, len, PCI_COMMAND + 1)) {
+        bool intx_masked = (pci_get_word(pci_dev->config + PCI_COMMAND) &
+                            PCI_COMMAND_INTX_DISABLE);
+
+        ret = kvm_device_intx_set_mask(kvm_state,
+                                       calc_assigned_dev_id(assigned_dev),
+                                       intx_masked);
+        if (ret) {
+            perror("assigned_dev_pci_write_config: set intx mask");
+        }
+    }
     if (assigned_dev->cap.available & ASSIGNED_DEVICE_CAP_MSI) {
         if (range_covers_byte(address, len,
                               pci_dev->msi_cap + PCI_MSI_FLAGS)) {
@@ -1748,6 +1767,8 @@ static Property da_properties[] =
                    ASSIGNED_DEVICE_USE_IOMMU_BIT, true),
     DEFINE_PROP_BIT("prefer_msi", AssignedDevice, features,
                    ASSIGNED_DEVICE_PREFER_MSI_BIT, true),
+    DEFINE_PROP_BIT("share_intx", AssignedDevice, features,
+                    ASSIGNED_DEVICE_SHARE_INTX_BIT, false),
     DEFINE_PROP_INT32("bootindex", AssignedDevice, bootindex, -1),
     DEFINE_PROP_STRING("configfd", AssignedDevice, configfd_name),
     DEFINE_PROP_END_OF_LIST(),
diff --git a/hw/device-assignment.h b/hw/device-assignment.h
index b4bcfa6..5d271d5 100644
--- a/hw/device-assignment.h
+++ b/hw/device-assignment.h
@@ -74,11 +74,13 @@ typedef struct {
     PCIRegion *region;
 } AssignedDevRegion;
 
-#define ASSIGNED_DEVICE_USE_IOMMU_BIT	0
-#define ASSIGNED_DEVICE_PREFER_MSI_BIT	1
+#define ASSIGNED_DEVICE_USE_IOMMU_BIT   0
+#define ASSIGNED_DEVICE_PREFER_MSI_BIT  1
+#define ASSIGNED_DEVICE_SHARE_INTX_BIT  2
 
-#define ASSIGNED_DEVICE_USE_IOMMU_MASK	(1 << ASSIGNED_DEVICE_USE_IOMMU_BIT)
-#define ASSIGNED_DEVICE_PREFER_MSI_MASK	(1 << ASSIGNED_DEVICE_PREFER_MSI_BIT)
+#define ASSIGNED_DEVICE_USE_IOMMU_MASK  (1 << ASSIGNED_DEVICE_USE_IOMMU_BIT)
+#define ASSIGNED_DEVICE_PREFER_MSI_MASK (1 << ASSIGNED_DEVICE_PREFER_MSI_BIT)
+#define ASSIGNED_DEVICE_SHARE_INTX_MASK (1 << ASSIGNED_DEVICE_SHARE_INTX_BIT)
 
 typedef struct {
     uint32_t addr_lo;
diff --git a/qemu-kvm.c b/qemu-kvm.c
index b1eee80..520d737 100644
--- a/qemu-kvm.c
+++ b/qemu-kvm.c
@@ -189,6 +189,15 @@ static int kvm_old_assign_irq(KVMState *s,
     return kvm_vm_ioctl(s, KVM_ASSIGN_IRQ, assigned_irq);
 }
 
+int kvm_device_intx_set_mask(KVMState *s, uint32_t dev_id, bool masked)
+{
+    struct kvm_assigned_pci_dev assigned_dev;
+
+    assigned_dev.assigned_dev_id = dev_id;
+    assigned_dev.flags = masked ? KVM_DEV_ASSIGN_MASK_INTX : 0;
+    return kvm_vm_ioctl(s, KVM_ASSIGN_SET_INTX_MASK, &assigned_dev);
+}
+
 #ifdef KVM_CAP_ASSIGN_DEV_IRQ
 int kvm_assign_irq(KVMState *s, struct kvm_assigned_irq *assigned_irq)
 {
diff --git a/qemu-kvm.h b/qemu-kvm.h
index 975b6fa..63f46a5 100644
--- a/qemu-kvm.h
+++ b/qemu-kvm.h
@@ -161,6 +161,8 @@ int kvm_assign_irq(KVMState *s, struct kvm_assigned_irq *assigned_irq);
  */
 int kvm_deassign_irq(KVMState *s, struct kvm_assigned_irq *assigned_irq);
 
+int kvm_device_intx_set_mask(KVMState *s, uint32_t dev_id, bool masked);
+
 /*!
  * \brief Notifies host kernel about a PCI device to be deassigned from a guest
  *
-- 
1.7.3.4

[PATCH 3/3] pci_assign: Flip defaults of prefer_msi and share_intx

[Jan Kiszka]

INTx sharing is a bit more expensive than exclusive host interrupts, but
this channel is not supposed to be used for high-performance scenarios
anyway. Modern devices support MSI/MSI-X and do not depend on using INTx
under critical workload, real old devices do not support INTx sharing
anyway.

For those in the middle, the user experience is much better if they just
work even when IRQ sharing is required. If there is nothing to share,
share_intx=off can still be applied as tuning parameter.

With INTx sharing as default, the primary reason for prefer_msi=on is
gone. Make it default off, specifically as it is known to cause troubles
with devices that have incomplete/broken MSI support or otherwise
stumble if host IRQ configuration does not match guest driver
expectation.

Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
---
 hw/device-assignment.c |    4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/hw/device-assignment.c b/hw/device-assignment.c
index 45e7913..0b2dfba 100644
--- a/hw/device-assignment.c
+++ b/hw/device-assignment.c
@@ -1766,9 +1766,9 @@ static Property da_properties[] =
     DEFINE_PROP_BIT("iommu", AssignedDevice, features,
                    ASSIGNED_DEVICE_USE_IOMMU_BIT, true),
     DEFINE_PROP_BIT("prefer_msi", AssignedDevice, features,
-                   ASSIGNED_DEVICE_PREFER_MSI_BIT, true),
+                    ASSIGNED_DEVICE_PREFER_MSI_BIT, false),
     DEFINE_PROP_BIT("share_intx", AssignedDevice, features,
-                    ASSIGNED_DEVICE_SHARE_INTX_BIT, false),
+                    ASSIGNED_DEVICE_SHARE_INTX_BIT, true),
     DEFINE_PROP_INT32("bootindex", AssignedDevice, bootindex, -1),
     DEFINE_PROP_STRING("configfd", AssignedDevice, configfd_name),
     DEFINE_PROP_END_OF_LIST(),
-- 
1.7.3.4

Re: [PATCH 2/3] pci-assign: Use PCI-2.3-based shared legacy interrupts

[Alex Williamson]

On Wed, 2012-03-07 at 18:19 +0100, Jan Kiszka wrote:
> Enable the new KVM feature that allows legacy interrupt sharing for
> PCI-2.3-compliant devices. This requires to synchronize any guest
> change of the INTx mask bit to the kernel.
> 
> The feature is controlled by the property 'share_intx' and is off by
> default for now.
> 
> Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
> ---
>  hw/device-assignment.c |   21 +++++++++++++++++++++
>  hw/device-assignment.h |   10 ++++++----
>  qemu-kvm.c             |    9 +++++++++
>  qemu-kvm.h             |    2 ++
>  4 files changed, 38 insertions(+), 4 deletions(-)
> 
> diff --git a/hw/device-assignment.c b/hw/device-assignment.c
> index 0e2c7b5..45e7913 100644
> --- a/hw/device-assignment.c
> +++ b/hw/device-assignment.c
> @@ -782,6 +782,13 @@ static int assign_device(AssignedDevice *dev)
>                  "cause host memory corruption if the device issues DMA write "
>                  "requests!\n");
>      }
> +    if (dev->features & ASSIGNED_DEVICE_SHARE_INTX_MASK) {
> +        assigned_dev_data.flags |= KVM_DEV_ASSIGN_PCI_2_3;
> +
> +        /* hide host-side INTx masking from the guest */
> +        dev->emulate_config_read[PCI_COMMAND + 1] |=
> +            PCI_COMMAND_INTX_DISABLE >> 8;
> +    }
>  
>      r = kvm_assign_pci_device(kvm_state, &assigned_dev_data);
>      if (r < 0) {
> @@ -1122,9 +1129,21 @@ static void assigned_dev_pci_write_config(PCIDevice *pci_dev, uint32_t address,
>  {
>      AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
>      uint32_t emulate_mask, full_emulation_mask;
> +    int ret;
>  
>      pci_default_write_config(pci_dev, address, val, len);
>  
> +    if (range_covers_byte(address, len, PCI_COMMAND + 1)) {
> +        bool intx_masked = (pci_get_word(pci_dev->config + PCI_COMMAND) &
> +                            PCI_COMMAND_INTX_DISABLE);
> +
> +        ret = kvm_device_intx_set_mask(kvm_state,
> +                                       calc_assigned_dev_id(assigned_dev),
> +                                       intx_masked);
> +        if (ret) {
> +            perror("assigned_dev_pci_write_config: set intx mask");
> +        }
> +    }

Can we be a little more strategic and only call intx_set_mask when the
bit changes?  Looks ok otherwise.  Thanks,

Alex

[PATCH v2 2/3] pci-assign: Use PCI-2.3-based shared legacy interrupts

[Jan Kiszka]

Enable the new KVM feature that allows legacy interrupt sharing for
PCI-2.3-compliant devices. This requires to synchronize any guest
change of the INTx mask bit to the kernel.

The feature is controlled by the property 'share_intx' and is off by
default for now.

Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
---

Changes in v2:
 - only call into kvm_device_intx_set_mask if the mask actually changed

 hw/device-assignment.c |   24 ++++++++++++++++++++++++
 hw/device-assignment.h |   10 ++++++----
 qemu-kvm.c             |    9 +++++++++
 qemu-kvm.h             |    2 ++
 4 files changed, 41 insertions(+), 4 deletions(-)

diff --git a/hw/device-assignment.c b/hw/device-assignment.c
index a5f1abb..12e2195 100644
--- a/hw/device-assignment.c
+++ b/hw/device-assignment.c
@@ -782,6 +782,13 @@ static int assign_device(AssignedDevice *dev)
                 "cause host memory corruption if the device issues DMA write "
                 "requests!\n");
     }
+    if (dev->features & ASSIGNED_DEVICE_SHARE_INTX_MASK) {
+        assigned_dev_data.flags |= KVM_DEV_ASSIGN_PCI_2_3;
+
+        /* hide host-side INTx masking from the guest */
+        dev->emulate_config_read[PCI_COMMAND + 1] |=
+            PCI_COMMAND_INTX_DISABLE >> 8;
+    }
 
     r = kvm_assign_pci_device(kvm_state, &assigned_dev_data);
     if (r < 0) {
@@ -1121,10 +1128,25 @@ static void assigned_dev_pci_write_config(PCIDevice *pci_dev, uint32_t address,
                                           uint32_t val, int len)
 {
     AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    uint32_t old_cmd = pci_get_word(pci_dev->config + PCI_COMMAND);
     uint32_t emulate_mask, full_emulation_mask;
+    int ret;
 
     pci_default_write_config(pci_dev, address, val, len);
 
+    if (range_covers_byte(address, len, PCI_COMMAND + 1)) {
+        bool intx_masked = (pci_get_word(pci_dev->config + PCI_COMMAND) &
+                            PCI_COMMAND_INTX_DISABLE);
+
+        if (intx_masked != !!(old_cmd & PCI_COMMAND_INTX_DISABLE)) {
+            ret = kvm_device_intx_set_mask(kvm_state,
+                                           calc_assigned_dev_id(assigned_dev),
+                                           intx_masked);
+            if (ret) {
+                perror("assigned_dev_pci_write_config: set intx mask");
+            }
+        }
+    }
     if (assigned_dev->cap.available & ASSIGNED_DEVICE_CAP_MSI) {
         if (range_covers_byte(address, len,
                               pci_dev->msi_cap + PCI_MSI_FLAGS)) {
@@ -1748,6 +1770,8 @@ static Property da_properties[] =
                    ASSIGNED_DEVICE_USE_IOMMU_BIT, true),
     DEFINE_PROP_BIT("prefer_msi", AssignedDevice, features,
                    ASSIGNED_DEVICE_PREFER_MSI_BIT, true),
+    DEFINE_PROP_BIT("share_intx", AssignedDevice, features,
+                    ASSIGNED_DEVICE_SHARE_INTX_BIT, false),
     DEFINE_PROP_INT32("bootindex", AssignedDevice, bootindex, -1),
     DEFINE_PROP_STRING("configfd", AssignedDevice, configfd_name),
     DEFINE_PROP_END_OF_LIST(),
diff --git a/hw/device-assignment.h b/hw/device-assignment.h
index b4bcfa6..5d271d5 100644
--- a/hw/device-assignment.h
+++ b/hw/device-assignment.h
@@ -74,11 +74,13 @@ typedef struct {
     PCIRegion *region;
 } AssignedDevRegion;
 
-#define ASSIGNED_DEVICE_USE_IOMMU_BIT	0
-#define ASSIGNED_DEVICE_PREFER_MSI_BIT	1
+#define ASSIGNED_DEVICE_USE_IOMMU_BIT   0
+#define ASSIGNED_DEVICE_PREFER_MSI_BIT  1
+#define ASSIGNED_DEVICE_SHARE_INTX_BIT  2
 
-#define ASSIGNED_DEVICE_USE_IOMMU_MASK	(1 << ASSIGNED_DEVICE_USE_IOMMU_BIT)
-#define ASSIGNED_DEVICE_PREFER_MSI_MASK	(1 << ASSIGNED_DEVICE_PREFER_MSI_BIT)
+#define ASSIGNED_DEVICE_USE_IOMMU_MASK  (1 << ASSIGNED_DEVICE_USE_IOMMU_BIT)
+#define ASSIGNED_DEVICE_PREFER_MSI_MASK (1 << ASSIGNED_DEVICE_PREFER_MSI_BIT)
+#define ASSIGNED_DEVICE_SHARE_INTX_MASK (1 << ASSIGNED_DEVICE_SHARE_INTX_BIT)
 
 typedef struct {
     uint32_t addr_lo;
diff --git a/qemu-kvm.c b/qemu-kvm.c
index 09a35f0..2047ebb 100644
--- a/qemu-kvm.c
+++ b/qemu-kvm.c
@@ -54,6 +54,15 @@ static int kvm_old_assign_irq(KVMState *s,
     return kvm_vm_ioctl(s, KVM_ASSIGN_IRQ, assigned_irq);
 }
 
+int kvm_device_intx_set_mask(KVMState *s, uint32_t dev_id, bool masked)
+{
+    struct kvm_assigned_pci_dev assigned_dev;
+
+    assigned_dev.assigned_dev_id = dev_id;
+    assigned_dev.flags = masked ? KVM_DEV_ASSIGN_MASK_INTX : 0;
+    return kvm_vm_ioctl(s, KVM_ASSIGN_SET_INTX_MASK, &assigned_dev);
+}
+
 #ifdef KVM_CAP_ASSIGN_DEV_IRQ
 int kvm_assign_irq(KVMState *s, struct kvm_assigned_irq *assigned_irq)
 {
diff --git a/qemu-kvm.h b/qemu-kvm.h
index 3c4f023..2b23daf 100644
--- a/qemu-kvm.h
+++ b/qemu-kvm.h
@@ -114,6 +114,8 @@ int kvm_assign_irq(KVMState *s, struct kvm_assigned_irq *assigned_irq);
  */
 int kvm_deassign_irq(KVMState *s, struct kvm_assigned_irq *assigned_irq);
 
+int kvm_device_intx_set_mask(KVMState *s, uint32_t dev_id, bool masked);
+
 /*!
  * \brief Notifies host kernel about a PCI device to be deassigned from a guest
  *
-- 
1.7.3.4

Re: [PATCH v2 2/3] pci-assign: Use PCI-2.3-based shared legacy interrupts

[Alex Williamson]

On Wed, 2012-03-07 at 19:05 +0100, Jan Kiszka wrote:
> Enable the new KVM feature that allows legacy interrupt sharing for
> PCI-2.3-compliant devices. This requires to synchronize any guest
> change of the INTx mask bit to the kernel.
> 
> The feature is controlled by the property 'share_intx' and is off by
> default for now.
> 
> Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
> ---
> 
> Changes in v2:
>  - only call into kvm_device_intx_set_mask if the mask actually changed
> 
>  hw/device-assignment.c |   24 ++++++++++++++++++++++++
>  hw/device-assignment.h |   10 ++++++----
>  qemu-kvm.c             |    9 +++++++++
>  qemu-kvm.h             |    2 ++
>  4 files changed, 41 insertions(+), 4 deletions(-)
> 
> diff --git a/hw/device-assignment.c b/hw/device-assignment.c
> index a5f1abb..12e2195 100644
> --- a/hw/device-assignment.c
> +++ b/hw/device-assignment.c
> @@ -782,6 +782,13 @@ static int assign_device(AssignedDevice *dev)
>                  "cause host memory corruption if the device issues DMA write "
>                  "requests!\n");
>      }
> +    if (dev->features & ASSIGNED_DEVICE_SHARE_INTX_MASK) {
> +        assigned_dev_data.flags |= KVM_DEV_ASSIGN_PCI_2_3;
> +
> +        /* hide host-side INTx masking from the guest */
> +        dev->emulate_config_read[PCI_COMMAND + 1] |=
> +            PCI_COMMAND_INTX_DISABLE >> 8;
> +    }
>  
>      r = kvm_assign_pci_device(kvm_state, &assigned_dev_data);
>      if (r < 0) {
> @@ -1121,10 +1128,25 @@ static void assigned_dev_pci_write_config(PCIDevice *pci_dev, uint32_t address,
>                                            uint32_t val, int len)
>  {
>      AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
> +    uint32_t old_cmd = pci_get_word(pci_dev->config + PCI_COMMAND);

nit, uint16_t

[PATCH v3 2/3] pci-assign: Use PCI-2.3-based shared legacy interrupts

[Jan Kiszka]

Enable the new KVM feature that allows legacy interrupt sharing for
PCI-2.3-compliant devices. This requires to synchronize any guest
change of the INTx mask bit to the kernel.

The feature is controlled by the property 'share_intx' and is off by
default for now.

Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
---

Changes in v3:
 - fixed nit :)

 hw/device-assignment.c |   24 ++++++++++++++++++++++++
 hw/device-assignment.h |   10 ++++++----
 qemu-kvm.c             |    9 +++++++++
 qemu-kvm.h             |    2 ++
 4 files changed, 41 insertions(+), 4 deletions(-)

diff --git a/hw/device-assignment.c b/hw/device-assignment.c
index a5f1abb..e2a8479 100644
--- a/hw/device-assignment.c
+++ b/hw/device-assignment.c
@@ -782,6 +782,13 @@ static int assign_device(AssignedDevice *dev)
                 "cause host memory corruption if the device issues DMA write "
                 "requests!\n");
     }
+    if (dev->features & ASSIGNED_DEVICE_SHARE_INTX_MASK) {
+        assigned_dev_data.flags |= KVM_DEV_ASSIGN_PCI_2_3;
+
+        /* hide host-side INTx masking from the guest */
+        dev->emulate_config_read[PCI_COMMAND + 1] |=
+            PCI_COMMAND_INTX_DISABLE >> 8;
+    }
 
     r = kvm_assign_pci_device(kvm_state, &assigned_dev_data);
     if (r < 0) {
@@ -1121,10 +1128,25 @@ static void assigned_dev_pci_write_config(PCIDevice *pci_dev, uint32_t address,
                                           uint32_t val, int len)
 {
     AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    uint16_t old_cmd = pci_get_word(pci_dev->config + PCI_COMMAND);
     uint32_t emulate_mask, full_emulation_mask;
+    int ret;
 
     pci_default_write_config(pci_dev, address, val, len);
 
+    if (range_covers_byte(address, len, PCI_COMMAND + 1)) {
+        bool intx_masked = (pci_get_word(pci_dev->config + PCI_COMMAND) &
+                            PCI_COMMAND_INTX_DISABLE);
+
+        if (intx_masked != !!(old_cmd & PCI_COMMAND_INTX_DISABLE)) {
+            ret = kvm_device_intx_set_mask(kvm_state,
+                                           calc_assigned_dev_id(assigned_dev),
+                                           intx_masked);
+            if (ret) {
+                perror("assigned_dev_pci_write_config: set intx mask");
+            }
+        }
+    }
     if (assigned_dev->cap.available & ASSIGNED_DEVICE_CAP_MSI) {
         if (range_covers_byte(address, len,
                               pci_dev->msi_cap + PCI_MSI_FLAGS)) {
@@ -1748,6 +1770,8 @@ static Property da_properties[] =
                    ASSIGNED_DEVICE_USE_IOMMU_BIT, true),
     DEFINE_PROP_BIT("prefer_msi", AssignedDevice, features,
                    ASSIGNED_DEVICE_PREFER_MSI_BIT, true),
+    DEFINE_PROP_BIT("share_intx", AssignedDevice, features,
+                    ASSIGNED_DEVICE_SHARE_INTX_BIT, false),
     DEFINE_PROP_INT32("bootindex", AssignedDevice, bootindex, -1),
     DEFINE_PROP_STRING("configfd", AssignedDevice, configfd_name),
     DEFINE_PROP_END_OF_LIST(),
diff --git a/hw/device-assignment.h b/hw/device-assignment.h
index b4bcfa6..5d271d5 100644
--- a/hw/device-assignment.h
+++ b/hw/device-assignment.h
@@ -74,11 +74,13 @@ typedef struct {
     PCIRegion *region;
 } AssignedDevRegion;
 
-#define ASSIGNED_DEVICE_USE_IOMMU_BIT	0
-#define ASSIGNED_DEVICE_PREFER_MSI_BIT	1
+#define ASSIGNED_DEVICE_USE_IOMMU_BIT   0
+#define ASSIGNED_DEVICE_PREFER_MSI_BIT  1
+#define ASSIGNED_DEVICE_SHARE_INTX_BIT  2
 
-#define ASSIGNED_DEVICE_USE_IOMMU_MASK	(1 << ASSIGNED_DEVICE_USE_IOMMU_BIT)
-#define ASSIGNED_DEVICE_PREFER_MSI_MASK	(1 << ASSIGNED_DEVICE_PREFER_MSI_BIT)
+#define ASSIGNED_DEVICE_USE_IOMMU_MASK  (1 << ASSIGNED_DEVICE_USE_IOMMU_BIT)
+#define ASSIGNED_DEVICE_PREFER_MSI_MASK (1 << ASSIGNED_DEVICE_PREFER_MSI_BIT)
+#define ASSIGNED_DEVICE_SHARE_INTX_MASK (1 << ASSIGNED_DEVICE_SHARE_INTX_BIT)
 
 typedef struct {
     uint32_t addr_lo;
diff --git a/qemu-kvm.c b/qemu-kvm.c
index 09a35f0..2047ebb 100644
--- a/qemu-kvm.c
+++ b/qemu-kvm.c
@@ -54,6 +54,15 @@ static int kvm_old_assign_irq(KVMState *s,
     return kvm_vm_ioctl(s, KVM_ASSIGN_IRQ, assigned_irq);
 }
 
+int kvm_device_intx_set_mask(KVMState *s, uint32_t dev_id, bool masked)
+{
+    struct kvm_assigned_pci_dev assigned_dev;
+
+    assigned_dev.assigned_dev_id = dev_id;
+    assigned_dev.flags = masked ? KVM_DEV_ASSIGN_MASK_INTX : 0;
+    return kvm_vm_ioctl(s, KVM_ASSIGN_SET_INTX_MASK, &assigned_dev);
+}
+
 #ifdef KVM_CAP_ASSIGN_DEV_IRQ
 int kvm_assign_irq(KVMState *s, struct kvm_assigned_irq *assigned_irq)
 {
diff --git a/qemu-kvm.h b/qemu-kvm.h
index 3c4f023..2b23daf 100644
--- a/qemu-kvm.h
+++ b/qemu-kvm.h
@@ -114,6 +114,8 @@ int kvm_assign_irq(KVMState *s, struct kvm_assigned_irq *assigned_irq);
  */
 int kvm_deassign_irq(KVMState *s, struct kvm_assigned_irq *assigned_irq);
 
+int kvm_device_intx_set_mask(KVMState *s, uint32_t dev_id, bool masked);
+
 /*!
  * \brief Notifies host kernel about a PCI device to be deassigned from a guest
  *
-- 
1.7.3.4

Re: [PATCH 0/3] qemu-kvm: pci-assign: Host IRQ sharing suppport

[Alex Williamson]

On Wed, 2012-03-07 at 18:19 +0100, Jan Kiszka wrote:
> This enables support for the host IRQ sharing support via PCI-2.3
> interrupt masking. See the patches for details.
> 
> Patch 1 could be considered for uq/master although we have no use case
> for the changes in upstream yet.
> 
> Jan Kiszka (3):
>   kvm: Update kernel headers against kvm.git
>   pci-assign: Use PCI-2.3-based shared legacy interrupts
>   pci_assign: Flip defaults of prefer_msi and share_intx
> 
>  hw/device-assignment.c       |   23 ++++++++++++++++++++++-
>  hw/device-assignment.h       |   10 ++++++----
>  linux-headers/asm-s390/kvm.h |    2 ++
>  linux-headers/linux/kvm.h    |    6 ++++++
>  qemu-kvm.c                   |    9 +++++++++
>  qemu-kvm.h                   |    2 ++
>  6 files changed, 47 insertions(+), 5 deletions(-)
> 

For series w/ v3 2/3:

Acked-by: Alex Williamson <alex.williamson@redhat.com>

Re: [PATCH v3 2/3] pci-assign: Use PCI-2.3-based shared legacy interrupts

[Alex Williamson]

On Wed, 2012-03-07 at 19:18 +0100, Jan Kiszka wrote:
> Enable the new KVM feature that allows legacy interrupt sharing for
> PCI-2.3-compliant devices. This requires to synchronize any guest
> change of the INTx mask bit to the kernel.
> 
> The feature is controlled by the property 'share_intx' and is off by
> default for now.
> 
> Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
> ---
> 
> Changes in v3:
>  - fixed nit :)
> 
>  hw/device-assignment.c |   24 ++++++++++++++++++++++++
>  hw/device-assignment.h |   10 ++++++----
>  qemu-kvm.c             |    9 +++++++++
>  qemu-kvm.h             |    2 ++
>  4 files changed, 41 insertions(+), 4 deletions(-)
> 
> diff --git a/hw/device-assignment.c b/hw/device-assignment.c
> index a5f1abb..e2a8479 100644
> --- a/hw/device-assignment.c
> +++ b/hw/device-assignment.c
> @@ -782,6 +782,13 @@ static int assign_device(AssignedDevice *dev)
>                  "cause host memory corruption if the device issues DMA write "
>                  "requests!\n");
>      }
> +    if (dev->features & ASSIGNED_DEVICE_SHARE_INTX_MASK) {
> +        assigned_dev_data.flags |= KVM_DEV_ASSIGN_PCI_2_3;
> +
> +        /* hide host-side INTx masking from the guest */
> +        dev->emulate_config_read[PCI_COMMAND + 1] |=
> +            PCI_COMMAND_INTX_DISABLE >> 8;
> +    }
>  
>      r = kvm_assign_pci_device(kvm_state, &assigned_dev_data);
>      if (r < 0) {
> @@ -1121,10 +1128,25 @@ static void assigned_dev_pci_write_config(PCIDevice *pci_dev, uint32_t address,
>                                            uint32_t val, int len)
>  {
>      AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
> +    uint16_t old_cmd = pci_get_word(pci_dev->config + PCI_COMMAND);
>      uint32_t emulate_mask, full_emulation_mask;
> +    int ret;
>  
>      pci_default_write_config(pci_dev, address, val, len);
>  
> +    if (range_covers_byte(address, len, PCI_COMMAND + 1)) {

Hmm, now that I've acked this... shouldn't we have a feature check here
to avoid the ioctl when share_intx=false or when the host kernel doesn't
support this?  Thanks,

Alex

> +        bool intx_masked = (pci_get_word(pci_dev->config + PCI_COMMAND) &
> +                            PCI_COMMAND_INTX_DISABLE);
> +
> +        if (intx_masked != !!(old_cmd & PCI_COMMAND_INTX_DISABLE)) {
> +            ret = kvm_device_intx_set_mask(kvm_state,
> +                                           calc_assigned_dev_id(assigned_dev),
> +                                           intx_masked);
> +            if (ret) {
> +                perror("assigned_dev_pci_write_config: set intx mask");
> +            }
> +        }
> +    }
>      if (assigned_dev->cap.available & ASSIGNED_DEVICE_CAP_MSI) {
>          if (range_covers_byte(address, len,
>                                pci_dev->msi_cap + PCI_MSI_FLAGS)) {
> @@ -1748,6 +1770,8 @@ static Property da_properties[] =
>                     ASSIGNED_DEVICE_USE_IOMMU_BIT, true),
>      DEFINE_PROP_BIT("prefer_msi", AssignedDevice, features,
>                     ASSIGNED_DEVICE_PREFER_MSI_BIT, true),
> +    DEFINE_PROP_BIT("share_intx", AssignedDevice, features,
> +                    ASSIGNED_DEVICE_SHARE_INTX_BIT, false),
>      DEFINE_PROP_INT32("bootindex", AssignedDevice, bootindex, -1),
>      DEFINE_PROP_STRING("configfd", AssignedDevice, configfd_name),
>      DEFINE_PROP_END_OF_LIST(),
> diff --git a/hw/device-assignment.h b/hw/device-assignment.h
> index b4bcfa6..5d271d5 100644
> --- a/hw/device-assignment.h
> +++ b/hw/device-assignment.h
> @@ -74,11 +74,13 @@ typedef struct {
>      PCIRegion *region;
>  } AssignedDevRegion;
>  
> -#define ASSIGNED_DEVICE_USE_IOMMU_BIT	0
> -#define ASSIGNED_DEVICE_PREFER_MSI_BIT	1
> +#define ASSIGNED_DEVICE_USE_IOMMU_BIT   0
> +#define ASSIGNED_DEVICE_PREFER_MSI_BIT  1
> +#define ASSIGNED_DEVICE_SHARE_INTX_BIT  2
>  
> -#define ASSIGNED_DEVICE_USE_IOMMU_MASK	(1 << ASSIGNED_DEVICE_USE_IOMMU_BIT)
> -#define ASSIGNED_DEVICE_PREFER_MSI_MASK	(1 << ASSIGNED_DEVICE_PREFER_MSI_BIT)
> +#define ASSIGNED_DEVICE_USE_IOMMU_MASK  (1 << ASSIGNED_DEVICE_USE_IOMMU_BIT)
> +#define ASSIGNED_DEVICE_PREFER_MSI_MASK (1 << ASSIGNED_DEVICE_PREFER_MSI_BIT)
> +#define ASSIGNED_DEVICE_SHARE_INTX_MASK (1 << ASSIGNED_DEVICE_SHARE_INTX_BIT)
>  
>  typedef struct {
>      uint32_t addr_lo;
> diff --git a/qemu-kvm.c b/qemu-kvm.c
> index 09a35f0..2047ebb 100644
> --- a/qemu-kvm.c
> +++ b/qemu-kvm.c
> @@ -54,6 +54,15 @@ static int kvm_old_assign_irq(KVMState *s,
>      return kvm_vm_ioctl(s, KVM_ASSIGN_IRQ, assigned_irq);
>  }
>  
> +int kvm_device_intx_set_mask(KVMState *s, uint32_t dev_id, bool masked)
> +{
> +    struct kvm_assigned_pci_dev assigned_dev;
> +
> +    assigned_dev.assigned_dev_id = dev_id;
> +    assigned_dev.flags = masked ? KVM_DEV_ASSIGN_MASK_INTX : 0;
> +    return kvm_vm_ioctl(s, KVM_ASSIGN_SET_INTX_MASK, &assigned_dev);
> +}
> +
>  #ifdef KVM_CAP_ASSIGN_DEV_IRQ
>  int kvm_assign_irq(KVMState *s, struct kvm_assigned_irq *assigned_irq)
>  {
> diff --git a/qemu-kvm.h b/qemu-kvm.h
> index 3c4f023..2b23daf 100644
> --- a/qemu-kvm.h
> +++ b/qemu-kvm.h
> @@ -114,6 +114,8 @@ int kvm_assign_irq(KVMState *s, struct kvm_assigned_irq *assigned_irq);
>   */
>  int kvm_deassign_irq(KVMState *s, struct kvm_assigned_irq *assigned_irq);
>  
> +int kvm_device_intx_set_mask(KVMState *s, uint32_t dev_id, bool masked);
> +
>  /*!
>   * \brief Notifies host kernel about a PCI device to be deassigned from a guest
>   *

Re: [PATCH v3 2/3] pci-assign: Use PCI-2.3-based shared legacy interrupts

[Jan Kiszka]

On 2012-03-07 19:32, Alex Williamson wrote:
> On Wed, 2012-03-07 at 19:18 +0100, Jan Kiszka wrote:
>> Enable the new KVM feature that allows legacy interrupt sharing for
>> PCI-2.3-compliant devices. This requires to synchronize any guest
>> change of the INTx mask bit to the kernel.
>>
>> The feature is controlled by the property 'share_intx' and is off by
>> default for now.
>>
>> Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
>> ---
>>
>> Changes in v3:
>>  - fixed nit :)
>>
>>  hw/device-assignment.c |   24 ++++++++++++++++++++++++
>>  hw/device-assignment.h |   10 ++++++----
>>  qemu-kvm.c             |    9 +++++++++
>>  qemu-kvm.h             |    2 ++
>>  4 files changed, 41 insertions(+), 4 deletions(-)
>>
>> diff --git a/hw/device-assignment.c b/hw/device-assignment.c
>> index a5f1abb..e2a8479 100644
>> --- a/hw/device-assignment.c
>> +++ b/hw/device-assignment.c
>> @@ -782,6 +782,13 @@ static int assign_device(AssignedDevice *dev)
>>                  "cause host memory corruption if the device issues DMA write "
>>                  "requests!\n");
>>      }
>> +    if (dev->features & ASSIGNED_DEVICE_SHARE_INTX_MASK) {
>> +        assigned_dev_data.flags |= KVM_DEV_ASSIGN_PCI_2_3;
>> +
>> +        /* hide host-side INTx masking from the guest */
>> +        dev->emulate_config_read[PCI_COMMAND + 1] |=
>> +            PCI_COMMAND_INTX_DISABLE >> 8;
>> +    }
>>  
>>      r = kvm_assign_pci_device(kvm_state, &assigned_dev_data);
>>      if (r < 0) {
>> @@ -1121,10 +1128,25 @@ static void assigned_dev_pci_write_config(PCIDevice *pci_dev, uint32_t address,
>>                                            uint32_t val, int len)
>>  {
>>      AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
>> +    uint16_t old_cmd = pci_get_word(pci_dev->config + PCI_COMMAND);
>>      uint32_t emulate_mask, full_emulation_mask;
>> +    int ret;
>>  
>>      pci_default_write_config(pci_dev, address, val, len);
>>  
>> +    if (range_covers_byte(address, len, PCI_COMMAND + 1)) {
> 
> Hmm, now that I've acked this... shouldn't we have a feature check here
> to avoid the ioctl when share_intx=false or when the host kernel doesn't
> support this?  Thanks,

Hmm, I thought I handled this, somehow. But it's not there. v4 follows...

Thanks,
Jan

-- 
Siemens AG, Corporate Technology, CT T DE IT 1
Corporate Competence Center Embedded Linux

Re: [PATCH v3 2/3] pci-assign: Use PCI-2.3-based shared legacy interrupts

[Jan Kiszka]

On 2012-03-07 19:42, Jan Kiszka wrote:
> On 2012-03-07 19:32, Alex Williamson wrote:
>> On Wed, 2012-03-07 at 19:18 +0100, Jan Kiszka wrote:
>>> Enable the new KVM feature that allows legacy interrupt sharing for
>>> PCI-2.3-compliant devices. This requires to synchronize any guest
>>> change of the INTx mask bit to the kernel.
>>>
>>> The feature is controlled by the property 'share_intx' and is off by
>>> default for now.
>>>
>>> Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
>>> ---
>>>
>>> Changes in v3:
>>>  - fixed nit :)
>>>
>>>  hw/device-assignment.c |   24 ++++++++++++++++++++++++
>>>  hw/device-assignment.h |   10 ++++++----
>>>  qemu-kvm.c             |    9 +++++++++
>>>  qemu-kvm.h             |    2 ++
>>>  4 files changed, 41 insertions(+), 4 deletions(-)
>>>
>>> diff --git a/hw/device-assignment.c b/hw/device-assignment.c
>>> index a5f1abb..e2a8479 100644
>>> --- a/hw/device-assignment.c
>>> +++ b/hw/device-assignment.c
>>> @@ -782,6 +782,13 @@ static int assign_device(AssignedDevice *dev)
>>>                  "cause host memory corruption if the device issues DMA write "
>>>                  "requests!\n");
>>>      }
>>> +    if (dev->features & ASSIGNED_DEVICE_SHARE_INTX_MASK) {
>>> +        assigned_dev_data.flags |= KVM_DEV_ASSIGN_PCI_2_3;
>>> +
>>> +        /* hide host-side INTx masking from the guest */
>>> +        dev->emulate_config_read[PCI_COMMAND + 1] |=
>>> +            PCI_COMMAND_INTX_DISABLE >> 8;
>>> +    }
>>>  
>>>      r = kvm_assign_pci_device(kvm_state, &assigned_dev_data);
>>>      if (r < 0) {
>>> @@ -1121,10 +1128,25 @@ static void assigned_dev_pci_write_config(PCIDevice *pci_dev, uint32_t address,
>>>                                            uint32_t val, int len)
>>>  {
>>>      AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
>>> +    uint16_t old_cmd = pci_get_word(pci_dev->config + PCI_COMMAND);
>>>      uint32_t emulate_mask, full_emulation_mask;
>>> +    int ret;
>>>  
>>>      pci_default_write_config(pci_dev, address, val, len);
>>>  
>>> +    if (range_covers_byte(address, len, PCI_COMMAND + 1)) {
>>
>> Hmm, now that I've acked this... shouldn't we have a feature check here
>> to avoid the ioctl when share_intx=false or when the host kernel doesn't
>> support this?  Thanks,
> 
> Hmm, I thought I handled this, somehow. But it's not there. v4 follows...

To be precise: We only need to worry about false positive error messages
in the absence of KVM_CAP_PCI_2_3. Devices not supporting INTx sharing
will either refuse to work due to IRQ conflicts, or we will map INTx
masking on IRQ line masking.

Jan

-- 
Siemens AG, Corporate Technology, CT T DE IT 1
Corporate Competence Center Embedded Linux