[PATCH 0/2] Bug fix for VFIO EEH

[PATCH 0/2] Bug fix for VFIO EEH

[Gavin Shan]

Those 2 patches are bug fix for VFIO EEH support, which isn't merged yet though
all reviewers gave their ack. So I'm sending this to avoid revert or something
like that.

The problem is that dma_offset/iommu_table_base
are sharing same memory location. When disabling bypass mode, we missed to restore
iommu_table_base. EEH is utilizing that to translate IOMMU group ID to PE. The
patches fix the issue.

Another issue is that we're searching all online PCI devices for translating
IOMMU group ID to PE. That's incorrect since we're uncertain that one speicific
device (except those in current IOMMU group) is running in bypassed mode or
not. So we should have search current IOMMU group.

It should be applied on top of unmerged VFIO EEH support patchset:

http://patchwork.ozlabs.org/patch/357665/

Gavin Shan (2):
  powerpc/powernv: Fix IOMMU table for VFIO dev
  powerpc/eeh: Fetch IOMMU table in reliable way

 arch/powerpc/kernel/eeh.c                 | 33 ++++++++++++++++++++-----------
 arch/powerpc/platforms/powernv/pci-ioda.c | 30 +++++++++++++++++++---------
 2 files changed, 43 insertions(+), 20 deletions(-)

-- 
1.8.3.2

[PATCH 1/2] powerpc/powernv: Fix IOMMU table for VFIO dev

[Gavin Shan]

On PHB3, PCI devices can bypass IOMMU for DMA access. If we pass
through one PCI device, whose hose driver ever enable the bypass
mode, pdev->dev.archdata.dma_data.iommu_table_base isn't IOMMU
table. However, EEH needs access the IOMMU table when the device
is owned by guest.

The patch fixes pdev->dev.archdata.dma_data.iommu_table when
passing through the device to guest in pnv_pci_ioda2_set_bypass().

Signed-off-by: Gavin Shan <gwshan@linux.vnet.ibm.com>
---
 arch/powerpc/platforms/powernv/pci-ioda.c | 30 +++++++++++++++++++++---------
 1 file changed, 21 insertions(+), 9 deletions(-)

diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
index de19ede..93fd815 100644
--- a/arch/powerpc/platforms/powernv/pci-ioda.c
+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
@@ -494,14 +494,22 @@ static int pnv_pci_ioda_dma_set_mask(struct pnv_phb *phb,
 	return 0;
 }
 
-static void pnv_ioda_setup_bus_dma(struct pnv_ioda_pe *pe, struct pci_bus *bus)
+static void pnv_ioda_setup_bus_dma(struct pnv_ioda_pe *pe,
+				   struct pci_bus *bus,
+				   bool add_to_iommu_group)
 {
 	struct pci_dev *dev;
 
 	list_for_each_entry(dev, &bus->devices, bus_list) {
-		set_iommu_table_base_and_group(&dev->dev, &pe->tce32_table);
+		if (add_to_iommu_group)
+			set_iommu_table_base_and_group(&dev->dev,
+						       &pe->tce32_table);
+		else
+			set_iommu_table_base(&dev->dev, &pe->tce32_table);
+
 		if (dev->subordinate)
-			pnv_ioda_setup_bus_dma(pe, dev->subordinate);
+			pnv_ioda_setup_bus_dma(pe, dev->subordinate,
+					       add_to_iommu_group);
 	}
 }
 
@@ -677,7 +685,7 @@ static void pnv_pci_ioda_setup_dma_pe(struct pnv_phb *phb,
 	if (pe->pdev)
 		set_iommu_table_base_and_group(&pe->pdev->dev, tbl);
 	else
-		pnv_ioda_setup_bus_dma(pe, pe->pbus);
+		pnv_ioda_setup_bus_dma(pe, pe->pbus, true);
 
 	return;
  fail:
@@ -713,11 +721,15 @@ static void pnv_pci_ioda2_set_bypass(struct iommu_table *tbl, bool enable)
 						     0);
 
 		/*
-		 * We might want to reset the DMA ops of all devices on
-		 * this PE. However in theory, that shouldn't be necessary
-		 * as this is used for VFIO/KVM pass-through and the device
-		 * hasn't yet been returned to its kernel driver
+		 * EEH needs the mapping between IOMMU table and group
+		 * of those VFIO/KVM pass-through devices. We can postpone
+		 * resetting DMA ops until the DMA mask is configured in
+		 * host side.
 		 */
+		if (pe->pdev)
+			set_iommu_table_base(&pe->pdev->dev, tbl);
+		else
+			pnv_ioda_setup_bus_dma(pe, pe->pbus, false);
 	}
 	if (rc)
 		pe_err(pe, "OPAL error %lld configuring bypass window\n", rc);
@@ -805,7 +817,7 @@ static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
 	if (pe->pdev)
 		set_iommu_table_base_and_group(&pe->pdev->dev, tbl);
 	else
-		pnv_ioda_setup_bus_dma(pe, pe->pbus);
+		pnv_ioda_setup_bus_dma(pe, pe->pbus, true);
 
 	/* Also create a bypass window */
 	pnv_pci_ioda2_setup_bypass_pe(phb, pe);
-- 
1.8.3.2

[PATCH 2/2] powerpc/eeh: Fetch IOMMU table in reliable way

[Gavin Shan]

Function eeh_iommu_group_to_pe() iterates each PCI device to check
the binding IOMMU group with get_iommu_table_base(), which possibly
fetches pdev->dev.archdata.dma_data.dma_offset. It's (0x1 << 59)
for "bypass" cases.

The patch fixes the issue by iterating devices hooked to the IOMMU
group and fetch IOMMU table there.

Signed-off-by: Gavin Shan <gwshan@linux.vnet.ibm.com>
---
 arch/powerpc/kernel/eeh.c | 33 ++++++++++++++++++++++-----------
 1 file changed, 22 insertions(+), 11 deletions(-)

diff --git a/arch/powerpc/kernel/eeh.c b/arch/powerpc/kernel/eeh.c
index 18c40fd..4de2103 100644
--- a/arch/powerpc/kernel/eeh.c
+++ b/arch/powerpc/kernel/eeh.c
@@ -27,6 +27,7 @@
 #include <linux/init.h>
 #include <linux/list.h>
 #include <linux/pci.h>
+#include <linux/iommu.h>
 #include <linux/proc_fs.h>
 #include <linux/rbtree.h>
 #include <linux/reboot.h>
@@ -1178,6 +1179,24 @@ out:
 }
 EXPORT_SYMBOL(eeh_dev_release);
 
+static int dev_has_iommu_table(struct device *dev, void *data)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct pci_dev **ppdev = data;
+	struct iommu_table *tbl;
+
+	if (!dev)
+		return 0;
+
+	tbl = get_iommu_table_base(dev);
+	if (tbl && tbl->it_group) {
+		*ppdev = pdev;
+		return 1;
+	}
+
+	return 0;
+}
+
 /**
  * eeh_iommu_group_to_pe - Convert IOMMU group to EEH PE
  * @group: IOMMU group
@@ -1186,24 +1205,16 @@ EXPORT_SYMBOL(eeh_dev_release);
  */
 struct eeh_pe *eeh_iommu_group_to_pe(struct iommu_group *group)
 {
-	struct iommu_table *tbl;
 	struct pci_dev *pdev = NULL;
 	struct eeh_dev *edev;
-	bool found = false;
+	int ret;
 
 	/* No IOMMU group ? */
 	if (!group)
 		return NULL;
 
-	/* No PCI device ? */
-	for_each_pci_dev(pdev) {
-		tbl = get_iommu_table_base(&pdev->dev);
-		if (tbl && tbl->it_group == group) {
-			found = true;
-			break;
-		}
-	}
-	if (!found)
+	ret = iommu_group_for_each_dev(group, &pdev, dev_has_iommu_table);
+	if (!ret || !pdev)
 		return NULL;
 
 	/* No EEH device or PE ? */
-- 
1.8.3.2