[PATCH 1/2] powerpc/eeh: More reliability of PCI dev reset

[PATCH 1/2] powerpc/eeh: More reliability of PCI dev reset

[Gavin Shan]

The PCI core has function pci_reset_function() to do reset on the
specified PCI device. Before the reset starts, the sate of the PCI
device is saved and it is restored after reset. The real reset work
could be routed to pcibios_set_pcie_reset_state() by quirks. However,
the PCI bus or PCI device isn't settled down fully for restore (PCI
config and MMIO for MSIx table) after reset and it would introduce
unnecessary frozen PE. One of the observed cases is failure of passing
IPR adapter from host to KVM-based guest because of this.

The patch adds delay in pcibios_set_pcie_reset_state() so that the
PCI bus/device can settle down fully before restoring PCI device
states. The patch also does cleanup on the names of those macros for
PE reset hold and settle time.

CC: <stable@vger.kernel.org>
Signed-off-by: Gavin Shan <shangw@linux.vnet.ibm.com>
---
 arch/powerpc/kernel/eeh.c |   19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/arch/powerpc/kernel/eeh.c b/arch/powerpc/kernel/eeh.c
index e7b76a6..251e370 100644
--- a/arch/powerpc/kernel/eeh.c
+++ b/arch/powerpc/kernel/eeh.c
@@ -84,8 +84,10 @@
  */
 #define EEH_MAX_FAILS	2100000
 
-/* Time to wait for a PCI slot to report status, in milliseconds */
-#define PCI_BUS_RESET_WAIT_MSEC (5*60*1000)
+/* All in milliseconds */
+#define EEH_PE_STATUS_WAIT_TIME		(5 * 60 * 1000)
+#define EEH_PE_RESET_HOLD_TIME		250
+#define EEH_PE_RESET_SETTLE_TIME	1800
 
 /* Platform dependent EEH operations */
 struct eeh_ops *eeh_ops = NULL;
@@ -522,7 +524,7 @@ int eeh_pci_enable(struct eeh_pe *pe, int function)
 		pr_warning("%s: Unexpected state change %d on PHB#%d-PE#%x, err=%d\n",
 			__func__, function, pe->phb->global_number, pe->addr, rc);
 
-	rc = eeh_ops->wait_state(pe, PCI_BUS_RESET_WAIT_MSEC);
+	rc = eeh_ops->wait_state(pe, EEH_PE_STATUS_WAIT_TIME);
 	if (rc > 0 && (rc & EEH_STATE_MMIO_ENABLED) &&
 	   (function == EEH_OPT_THAW_MMIO))
 		return 0;
@@ -552,12 +554,15 @@ int pcibios_set_pcie_reset_state(struct pci_dev *dev, enum pcie_reset_state stat
 	switch (state) {
 	case pcie_deassert_reset:
 		eeh_ops->reset(pe, EEH_RESET_DEACTIVATE);
+		msleep(EEH_PE_RESET_HOLD_TIME);
 		break;
 	case pcie_hot_reset:
 		eeh_ops->reset(pe, EEH_RESET_HOT);
+		msleep(EEH_PE_RESET_HOLD_TIME);
 		break;
 	case pcie_warm_reset:
 		eeh_ops->reset(pe, EEH_RESET_FUNDAMENTAL);
+		msleep(EEH_PE_RESET_SETTLE_TIME);
 		break;
 	default:
 		return -EINVAL;
@@ -615,8 +620,7 @@ static void eeh_reset_pe_once(struct eeh_pe *pe)
 	/* The PCI bus requires that the reset be held high for at least
 	 * a 100 milliseconds. We wait a bit longer 'just in case'.
 	 */
-#define PCI_BUS_RST_HOLD_TIME_MSEC 250
-	msleep(PCI_BUS_RST_HOLD_TIME_MSEC);
+	msleep(EEH_PE_RESET_HOLD_TIME);
 
 	/* We might get hit with another EEH freeze as soon as the
 	 * pci slot reset line is dropped. Make sure we don't miss
@@ -630,8 +634,7 @@ static void eeh_reset_pe_once(struct eeh_pe *pe)
 	 * a 1.5 second idle time for the bus to stabilize, before starting
 	 * up traffic.
 	 */
-#define PCI_BUS_SETTLE_TIME_MSEC 1800
-	msleep(PCI_BUS_SETTLE_TIME_MSEC);
+	msleep(EEH_PE_RESET_SETTLE_TIME);
 }
 
 /**
@@ -651,7 +654,7 @@ int eeh_reset_pe(struct eeh_pe *pe)
 	for (i=0; i<3; i++) {
 		eeh_reset_pe_once(pe);
 
-		rc = eeh_ops->wait_state(pe, PCI_BUS_RESET_WAIT_MSEC);
+		rc = eeh_ops->wait_state(pe, EEH_PE_STATUS_WAIT_TIME);
 		if ((rc & flags) == flags)
 			return 0;
 
-- 
1.7.10.4

[PATCH 2/2] powerpc/eeh: Check PCIe link in pcibios_set_pcie_reset_state()

[Gavin Shan]

After PE reset in pcibios_set_pcie_reset_state(), the PCIe link
might be not ready after settle time of PE primary bus. The
subsequent access to PCI config and MMIO of the affected domain
would cause more problems (e.g. unexpected frozen PE).

The patch checks the PCIe link in pcibios_set_pcie_reset_state()
to make sure all PCIe links are up after PE reset so that to
avoid unexpected problems.

Signed-off-by: Gavin Shan <shangw@linux.vnet.ibm.com>
---
 arch/powerpc/include/asm/eeh.h |    1 +
 arch/powerpc/kernel/eeh.c      |   29 +++++++++++++++++++++++++++++
 arch/powerpc/kernel/eeh_pe.c   |   21 +++++++++++++--------
 3 files changed, 43 insertions(+), 8 deletions(-)

diff --git a/arch/powerpc/include/asm/eeh.h b/arch/powerpc/include/asm/eeh.h
index d4dd41f..e96ed32 100644
--- a/arch/powerpc/include/asm/eeh.h
+++ b/arch/powerpc/include/asm/eeh.h
@@ -231,6 +231,7 @@ void *eeh_pe_traverse(struct eeh_pe *root,
 		eeh_traverse_func fn, void *flag);
 void *eeh_pe_dev_traverse(struct eeh_pe *root,
 		eeh_traverse_func fn, void *flag);
+void eeh_bridge_check_link(struct eeh_dev *edev);
 void eeh_pe_restore_bars(struct eeh_pe *pe);
 struct pci_bus *eeh_pe_bus_get(struct eeh_pe *pe);
 
diff --git a/arch/powerpc/kernel/eeh.c b/arch/powerpc/kernel/eeh.c
index 251e370..ba2dd2d 100644
--- a/arch/powerpc/kernel/eeh.c
+++ b/arch/powerpc/kernel/eeh.c
@@ -532,6 +532,14 @@ int eeh_pci_enable(struct eeh_pe *pe, int function)
 	return rc;
 }
 
+static void *eeh_dev_check_link(void *data, void *flag)
+{
+	struct eeh_dev *edev = data;
+
+	eeh_bridge_check_link(edev);
+	return NULL;
+}
+
 /**
  * pcibios_set_pcie_slot_reset - Set PCI-E reset state
  * @dev: pci device struct
@@ -544,6 +552,7 @@ int pcibios_set_pcie_reset_state(struct pci_dev *dev, enum pcie_reset_state stat
 {
 	struct eeh_dev *edev = pci_dev_to_eeh_dev(dev);
 	struct eeh_pe *pe = edev->pe;
+	struct pci_bus *bus;
 
 	if (!pe) {
 		pr_err("%s: No PE found on PCI device %s\n",
@@ -551,10 +560,30 @@ int pcibios_set_pcie_reset_state(struct pci_dev *dev, enum pcie_reset_state stat
 		return -EINVAL;
 	}
 
+	bus = eeh_pe_bus_get(pe);
+	if (!bus) {
+		pr_err("%s: No PE primary bus found for PCI dev %s\n",
+			__func__, pci_name(dev));
+		return -EINVAL;
+	}
+
 	switch (state) {
 	case pcie_deassert_reset:
 		eeh_ops->reset(pe, EEH_RESET_DEACTIVATE);
 		msleep(EEH_PE_RESET_HOLD_TIME);
+
+		/*
+		 * After PE reset, the PCIe link is probably
+		 * not ready after settle period. We're checking
+		 * all PCIe downstream port of the affected PE
+		 * ensure that.
+		 */
+		if (bus->self) {
+			edev = pci_dev_to_eeh_dev(bus->self);
+			eeh_bridge_check_link(edev);
+		}
+		eeh_pe_dev_traverse(pe, eeh_dev_check_link, NULL);
+
 		break;
 	case pcie_hot_reset:
 		eeh_ops->reset(pe, EEH_RESET_HOT);
diff --git a/arch/powerpc/kernel/eeh_pe.c b/arch/powerpc/kernel/eeh_pe.c
index f0c353f..a49f9dc 100644
--- a/arch/powerpc/kernel/eeh_pe.c
+++ b/arch/powerpc/kernel/eeh_pe.c
@@ -567,6 +567,9 @@ void eeh_pe_state_clear(struct eeh_pe *pe, int state)
 }
 
 /*
+ * eeh_bridge_check_link - Check PCI link is up or down
+ * @edev: EEH device
+ *
  * Some PCI bridges (e.g. PLX bridges) have primary/secondary
  * buses assigned explicitly by firmware, and we probably have
  * lost that after reset. So we have to delay the check until
@@ -577,18 +580,20 @@ void eeh_pe_state_clear(struct eeh_pe *pe, int state)
  * blocked on normal path during the stage. So we need utilize
  * eeh operations, which is always permitted.
  */
-static void eeh_bridge_check_link(struct eeh_dev *edev,
-				  struct device_node *dn)
+void eeh_bridge_check_link(struct eeh_dev *edev)
 {
+	struct device_node *dn;
 	int cap;
 	uint32_t val;
 	int timeout = 0;
 
-	/*
-	 * We only check root port and downstream ports of
-	 * PCIe switches
-	 */
-	if (!(edev->mode & (EEH_DEV_ROOT_PORT | EEH_DEV_DS_PORT)))
+	/* Only for root port and downstream ports */
+	if (!edev || !(edev->mode & (EEH_DEV_ROOT_PORT | EEH_DEV_DS_PORT)))
+		return;
+
+	/* Device node */
+	dn = eeh_dev_to_of_node(edev);
+	if (!dn)
 		return;
 
 	pr_debug("%s: Check PCIe link for %04x:%02x:%02x.%01x ...\n",
@@ -678,7 +683,7 @@ static void eeh_restore_bridge_bars(struct eeh_dev *edev,
 	eeh_ops->write_config(dn, PCI_COMMAND, 4, edev->config_space[1]);
 
 	/* Check the PCIe link is ready */
-	eeh_bridge_check_link(edev, dn);
+	eeh_bridge_check_link(edev);
 }
 
 static void eeh_restore_device_bars(struct eeh_dev *edev,
-- 
1.7.10.4