From: Sheng Yang <sheng@linux.intel.com>
To: Avi Kivity <avi@redhat.com>, Marcelo Tosatti <mtosatti@redhat.com>
Cc: "Michael S. Tsirkin" <mst@redhat.com>,
kvm@vger.kernel.org, Sheng Yang <sheng@linux.intel.com>
Subject: [PATCH 4/5] qemu-kvm: device assignment: emulate MSI-X mask bits
Date: Wed, 20 Oct 2010 16:29:54 +0800 [thread overview]
Message-ID: <1287563395-30039-5-git-send-email-sheng@linux.intel.com> (raw)
In-Reply-To: <1287563395-30039-1-git-send-email-sheng@linux.intel.com>
This patch emulated MSI-X per vector mask bit on assigned device.
Signed-off-by: Sheng Yang <sheng@linux.intel.com>
---
hw/device-assignment.c | 151 ++++++++++++++++++++++++++++++++++++++++++++++--
1 files changed, 145 insertions(+), 6 deletions(-)
diff --git a/hw/device-assignment.c b/hw/device-assignment.c
index 8a98876..d1a6282 100644
--- a/hw/device-assignment.c
+++ b/hw/device-assignment.c
@@ -62,6 +62,11 @@ static void assigned_dev_load_option_rom(AssignedDevice *dev);
static void assigned_dev_unregister_msix_mmio(AssignedDevice *dev);
+static uint32_t calc_assigned_dev_id(uint16_t seg, uint8_t bus, uint8_t devfn)
+{
+ return (uint32_t)seg << 16 | (uint32_t)bus << 8 | (uint32_t)devfn;
+}
+
static uint32_t assigned_dev_ioport_rw(AssignedDevRegion *dev_region,
uint32_t addr, int len, uint32_t *val)
{
@@ -824,11 +829,6 @@ static void free_assigned_device(AssignedDevice *dev)
}
}
-static uint32_t calc_assigned_dev_id(uint16_t seg, uint8_t bus, uint8_t devfn)
-{
- return (uint32_t)seg << 16 | (uint32_t)bus << 8 | (uint32_t)devfn;
-}
-
static void assign_failed_examine(AssignedDevice *dev)
{
char name[PATH_MAX], dir[PATH_MAX], driver[PATH_MAX] = {}, *ns;
@@ -1123,6 +1123,7 @@ static int get_msix_entries_max_nr(AssignedDevice *adev)
return entries_max_nr;
}
+#define MSIX_VECTOR_MASK 0x1
static int get_msix_valid_entries_nr(AssignedDevice *adev,
uint16_t entries_max_nr)
{
@@ -1136,7 +1137,11 @@ static int get_msix_valid_entries_nr(AssignedDevice *adev,
memcpy(&msg_ctrl, va + i * 16 + 12, 4);
memcpy(&msg_data, va + i * 16 + 8, 4);
/* Ignore unused entry even it's unmasked */
+#ifdef KVM_CAP_DEVICE_MSIX_MASK
+ if (msg_data == 0 || (msg_ctrl & MSIX_VECTOR_MASK))
+#else
if (msg_data == 0)
+#endif
continue;
entries_nr ++;
}
@@ -1165,6 +1170,8 @@ static int assigned_dev_update_msix_mmio(PCIDevice *pci_dev,
}
free_dev_irq_entries(adev);
+ memset(pci_dev->msix_entry_used, 0, KVM_MAX_MSIX_PER_DEV *
+ sizeof(*pci_dev->msix_entry_used));
adev->irq_entries_nr = entries_nr;
adev->entry = calloc(entries_nr, sizeof(struct kvm_irq_routing_entry));
if (!adev->entry) {
@@ -1179,7 +1186,11 @@ static int assigned_dev_update_msix_mmio(PCIDevice *pci_dev,
break;
memcpy(&msg_ctrl, va + i * 16 + 12, 4);
memcpy(&msg_data, va + i * 16 + 8, 4);
+#ifdef KVM_CAP_DEVICE_MSIX_MASK
+ if (msg_data == 0 || (msg_ctrl & MSIX_VECTOR_MASK))
+#else
if (msg_data == 0)
+#endif
continue;
memcpy(&msg_addr, va + i * 16, 4);
@@ -1200,6 +1211,7 @@ static int assigned_dev_update_msix_mmio(PCIDevice *pci_dev,
msix_entry.gsi = adev->entry[entries_nr].gsi;
msix_entry.entry = i;
+ pci_dev->msix_entry_used[i] = 1;
r = kvm_assign_set_msix_entry(kvm_context, &msix_entry);
if (r) {
fprintf(stderr, "fail to set MSI-X entry! %s\n", strerror(-r));
@@ -1243,6 +1255,8 @@ static void assigned_dev_update_msix(PCIDevice *pci_dev, int enable_msix)
perror("assigned_dev_update_msix: deassign irq");
assigned_dev->irq_requested_type = 0;
+ memset(pci_dev->msix_entry_used, 0, KVM_MAX_MSIX_PER_DEV *
+ sizeof(*pci_dev->msix_entry_used));
}
entries_max_nr = get_msix_entries_max_nr(assigned_dev);
@@ -1250,10 +1264,16 @@ static void assigned_dev_update_msix(PCIDevice *pci_dev, int enable_msix)
fprintf(stderr, "assigned_dev_update_msix: MSI-X entries_max_nr == 0");
return;
}
+ /*
+ * Guest may try to enable MSI-X before setting MSI-X entry done, so
+ * let's wait until guest unmask the entries.
+ */
entries_nr = get_msix_valid_entries_nr(assigned_dev, entries_max_nr);
if (entries_nr == 0) {
+#ifndef KVM_CAP_DEVICE_MSIX_MASK
if (enable_msix)
fprintf(stderr, "MSI-X entry number is zero!\n");
+#endif
return;
}
if (enable_msix) {
@@ -1297,7 +1317,8 @@ static void assigned_device_pci_cap_write_config(PCIDevice *pci_dev, uint32_t ad
if (address <= ctrl_pos && address + len > ctrl_pos) {
ctrl_pos--; /* control is word long */
ctrl_word = (uint16_t *)(pci_dev->config + ctrl_pos);
- assigned_dev_update_msix(pci_dev, (*ctrl_word & PCI_MSIX_ENABLE));
+ assigned_dev_update_msix(pci_dev,
+ (*ctrl_word & PCI_MSIX_ENABLE) && !(*ctrl_word & PCI_MSIX_MASK));
}
pos += PCI_CAPABILITY_CONFIG_MSIX_LENGTH;
}
@@ -1389,6 +1410,104 @@ static uint32_t msix_mmio_readw(void *opaque, target_phys_addr_t addr)
(8 * (addr & 3))) & 0xffff;
}
+#ifdef KVM_CAP_DEVICE_MSIX_MASK
+static void msix_mmio_access_mask_bit(AssignedDevice *adev, int index)
+{
+ void *page = adev->msix_table_page;
+ uint32_t msg_ctrl, msg_data, msg_upper_addr, msg_addr;
+ struct kvm_assigned_msix_entry msix_entry;
+ int r = 0, pos, ctrl_word, entry_idx, entries_max_nr;
+ struct kvm_irq_routing_entry new_entry = {};
+
+ memcpy(&msg_addr, (char *)page + index * 16, 4);
+ memcpy(&msg_upper_addr, (char *)page + index * 16 + 4, 4);
+ memcpy(&msg_data, (char *)page + index * 16 + 8, 4);
+ memcpy(&msg_ctrl, (char *)page + index * 16 + 12, 4);
+ DEBUG("Access mask bit: MSI-X entries index %d: "
+ "msg_addr 0x%x, msg_upper_addr 0x%x, msg_data 0x%x, vec_ctl 0x%x\n",
+ index, msg_addr, msg_upper_addr, msg_data, msg_ctrl);
+
+ if (adev->cap.available & ASSIGNED_DEVICE_CAP_MSI)
+ pos = adev->dev.cap.start + PCI_CAPABILITY_CONFIG_MSI_LENGTH;
+ else
+ pos = adev->dev.cap.start;
+
+ ctrl_word = *(uint16_t *)(adev->dev.config + pos + 2);
+
+ if (!((ctrl_word & PCI_MSIX_ENABLE) && !(ctrl_word & PCI_MSIX_MASK)))
+ return;
+
+ if (!adev->dev.msix_entry_used[index]) {
+ if ((msg_ctrl & MSIX_VECTOR_MASK) == 0) {
+ DEBUG("Try to modify unenabled MSI-X entry %d's mask. "
+ "Reenable MSI-X.\n",
+ index);
+ assigned_dev_update_msix(&adev->dev, 1);
+ }
+ return;
+ }
+
+ /* find the correlated index of adev->entry */
+ entries_max_nr = get_msix_entries_max_nr(adev);
+ entry_idx = 0;
+ while (entry_idx < entries_max_nr) {
+ if (adev->dev.msix_entry_used[entry_idx] && entry_idx == index)
+ break;
+ entry_idx ++;
+ }
+ if (entry_idx >= entries_max_nr) {
+ fprintf(stderr, "msix_mmio_access_mask_bit: Entry idx exceed limit!\n");
+ return;
+ }
+
+ msix_entry.assigned_dev_id = calc_assigned_dev_id(adev->h_segnr,
+ adev->h_busnr,
+ (uint8_t)adev->h_devfn);
+ msix_entry.gsi = adev->entry[entry_idx].gsi;
+ msix_entry.entry = index;
+ if (msg_ctrl & MSIX_VECTOR_MASK)
+ msix_entry.flags = KVM_MSIX_FLAG_MASK;
+ else
+ msix_entry.flags = 0;
+ DEBUG("set MSI-X index %d, esi 0x%x, mask %d\n",
+ index, msix_entry.gsi, msix_entry.flags);
+ r = kvm_assign_set_msix_entry(kvm_context, &msix_entry);
+ if (r) {
+ perror("msix_mmio_access_mask_bit: "
+ "fail to set MSI-X entry!");
+ return;
+ }
+
+ if (msix_entry.flags == 0) {
+ new_entry.gsi = adev->entry[entry_idx].gsi;
+ new_entry.type = KVM_IRQ_ROUTING_MSI;
+ new_entry.flags = 0;
+ new_entry.u.msi.address_lo = msg_addr;
+ new_entry.u.msi.address_hi = msg_upper_addr;
+ new_entry.u.msi.data = msg_data;
+ if (memcmp(&adev->entry[entry_idx].u.msi, &new_entry.u.msi,
+ sizeof new_entry.u.msi)) {
+ int r;
+ r = kvm_update_routing_entry(&adev->entry[entry_idx], &new_entry);
+ if (r) {
+ perror("msix_mmio_access_mask_bit: "
+ "kvm_update_routing_entry failed\n");
+ return;
+ }
+ r = kvm_commit_irq_routes();
+ if (r) {
+ perror("msix_mmio_access_mask_bit: "
+ "kvm_commit_irq_routes failed\n");
+ return;
+ }
+ }
+ adev->entry[entry_idx].u.msi.address_lo = msg_addr;
+ adev->entry[entry_idx].u.msi.address_hi = msg_upper_addr;
+ adev->entry[entry_idx].u.msi.data = msg_data;
+ }
+}
+#endif
+
static void msix_mmio_writel(void *opaque,
target_phys_addr_t addr, uint32_t val)
{
@@ -1399,6 +1518,12 @@ static void msix_mmio_writel(void *opaque,
DEBUG("write to MSI-X entry table mmio offset 0x%lx, val 0x%x\n",
addr, val);
memcpy((void *)((char *)page + offset), &val, 4);
+
+#ifdef KVM_CAP_DEVICE_MSIX_MASK
+ /* Check if mask bit is being accessed */
+ if (offset % 16 == 12)
+ msix_mmio_access_mask_bit(adev, offset / 16);
+#endif
}
static void msix_mmio_writew(void *opaque,
@@ -1425,6 +1550,16 @@ static CPUReadMemoryFunc *msix_mmio_read[] = {
static int assigned_dev_register_msix_mmio(AssignedDevice *dev)
{
+#ifdef KVM_CAP_DEVICE_MSIX_MASK
+#ifndef KVM_CAP_DEVICE_MSIX_EXT
+#error
+#endif
+ /* Request for MSI-X masking support */
+ if (kvm_enable_cap(first_cpu, KVM_CAP_DEVICE_MSIX_EXT) != 0) {
+ fprintf(stderr, "fail to enable MSI-X extending support!\n");
+ return -EFAULT;
+ }
+#endif
dev->msix_table_page = mmap(NULL, 0x1000,
PROT_READ|PROT_WRITE,
MAP_ANONYMOUS|MAP_PRIVATE, 0, 0);
@@ -1436,6 +1571,8 @@ static int assigned_dev_register_msix_mmio(AssignedDevice *dev)
memset(dev->msix_table_page, 0, 0x1000);
dev->mmio_index = cpu_register_io_memory(
msix_mmio_read, msix_mmio_write, dev);
+ dev->dev.msix_entry_used = qemu_mallocz(KVM_MAX_MSIX_PER_DEV *
+ sizeof *dev->dev.msix_entry_used);
return 0;
}
@@ -1452,6 +1589,8 @@ static void assigned_dev_unregister_msix_mmio(AssignedDevice *dev)
strerror(errno));
}
dev->msix_table_page = NULL;
+ free(dev->dev.msix_entry_used);
+ dev->dev.msix_entry_used = NULL;
}
static int assigned_initfn(struct PCIDevice *pci_dev)
--
1.7.0.1
next prev parent reply other threads:[~2010-10-20 8:28 UTC|newest]
Thread overview: 9+ messages / expand[flat|nested] mbox.gz Atom feed top
2010-10-20 8:29 [PATCH 0/5] MSI-X mask support for assigned device on userspace Sheng Yang
2010-10-20 8:29 ` [PATCH 1/5] qemu-kvm: Add kvm_enable_cap() interface for x86 Sheng Yang
2010-10-20 8:29 ` [PATCH 2/5] qemu-kvm: Ioctl for in-kernel mask support Sheng Yang
2010-10-20 8:29 ` [PATCH 3/5] qemu-kvm: device assignment: Some clean up about MSI-X code Sheng Yang
2010-10-20 8:29 ` Sheng Yang [this message]
2010-10-20 8:29 ` [PATCH 5/5] qemu-kvm: device assignment: Enable in-kernel MSI-X mask support Sheng Yang
2010-10-20 22:36 ` Michael S. Tsirkin
2010-10-21 7:12 ` Sheng Yang
-- strict thread matches above, loose matches on Subject: below --
2010-11-11 7:51 [PATCH 0/5 v4] MSI-X mask support for userspace Sheng Yang
2010-11-11 7:51 ` [PATCH 4/5] qemu-kvm: device assignment: emulate MSI-X mask bits Sheng Yang
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1287563395-30039-5-git-send-email-sheng@linux.intel.com \
--to=sheng@linux.intel.com \
--cc=avi@redhat.com \
--cc=kvm@vger.kernel.org \
--cc=mst@redhat.com \
--cc=mtosatti@redhat.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.