[PATCH 0/2] PCI: mvebu: Add support for unbinding and unloading driver

[PATCH 0/2] PCI: mvebu: Add support for unbinding and unloading driver

[Pali Rohár]

This patch series add support for unbinding device from pci-mvebu driver,
allow to compile pci-mvebu.c as kernel module pci-mvebu.ko and allow to
rmmod this module.

This patch series depends on another patch series which converts
pci-mvebu.c driver to use devm_pci_remap_iospace() function:

"arm: ioremap: Remove pci_ioremap_io() and mvebu_pci_host_probe()":
https://lore.kernel.org/linux-pci/20211124154116.916-1-pali@kernel.org/

Without usage of devm_pci_remap_iospace() it is not possible to bind
device to pci-mvebu driver again after unbinding it.

Tested on Turris Omnia - Armada 385 board.

Pali Rohár (2):
  bus: mvebu-mbus: Export symbols for public API window functions
  PCI: mvebu: Add support for compiling driver as module

 drivers/bus/mvebu-mbus.c           |  5 ++
 drivers/pci/controller/Kconfig     |  2 +-
 drivers/pci/controller/pci-mvebu.c | 91 +++++++++++++++++++++++++-----
 3 files changed, 82 insertions(+), 16 deletions(-)

-- 
2.20.1

[PATCH 1/2] bus: mvebu-mbus: Export symbols for public API window functions

[Pali Rohár]

This would allow to compile pci-mvebu.c driver as module.

Signed-off-by: Pali Rohár <pali@kernel.org>
---
 drivers/bus/mvebu-mbus.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/drivers/bus/mvebu-mbus.c b/drivers/bus/mvebu-mbus.c
index ea0424922de7..db612045616f 100644
--- a/drivers/bus/mvebu-mbus.c
+++ b/drivers/bus/mvebu-mbus.c
@@ -914,6 +914,7 @@ int mvebu_mbus_add_window_remap_by_id(unsigned int target,
 
 	return mvebu_mbus_alloc_window(s, base, size, remap, target, attribute);
 }
+EXPORT_SYMBOL_GPL(mvebu_mbus_add_window_remap_by_id);
 
 int mvebu_mbus_add_window_by_id(unsigned int target, unsigned int attribute,
 				phys_addr_t base, size_t size)
@@ -921,6 +922,7 @@ int mvebu_mbus_add_window_by_id(unsigned int target, unsigned int attribute,
 	return mvebu_mbus_add_window_remap_by_id(target, attribute, base,
 						 size, MVEBU_MBUS_NO_REMAP);
 }
+EXPORT_SYMBOL_GPL(mvebu_mbus_add_window_by_id);
 
 int mvebu_mbus_del_window(phys_addr_t base, size_t size)
 {
@@ -933,6 +935,7 @@ int mvebu_mbus_del_window(phys_addr_t base, size_t size)
 	mvebu_mbus_disable_window(&mbus_state, win);
 	return 0;
 }
+EXPORT_SYMBOL_GPL(mvebu_mbus_del_window);
 
 void mvebu_mbus_get_pcie_mem_aperture(struct resource *res)
 {
@@ -940,6 +943,7 @@ void mvebu_mbus_get_pcie_mem_aperture(struct resource *res)
 		return;
 	*res = mbus_state.pcie_mem_aperture;
 }
+EXPORT_SYMBOL_GPL(mvebu_mbus_get_pcie_mem_aperture);
 
 void mvebu_mbus_get_pcie_io_aperture(struct resource *res)
 {
@@ -947,6 +951,7 @@ void mvebu_mbus_get_pcie_io_aperture(struct resource *res)
 		return;
 	*res = mbus_state.pcie_io_aperture;
 }
+EXPORT_SYMBOL_GPL(mvebu_mbus_get_pcie_io_aperture);
 
 int mvebu_mbus_get_dram_win_info(phys_addr_t phyaddr, u8 *target, u8 *attr)
 {
-- 
2.20.1

[PATCH 2/2] PCI: mvebu: Add support for compiling driver as module

[Pali Rohár]

Now when driver uses devm_pci_remap_iospace() function, it is possible
implement ->remove() callback for unbinding device from driver.

Implement mvebu_pcie_remove() callback with proper cleanup phase, drop
driver's suppress_bind_attrs flag and switch type of CONFIG_PCI_MVEBU
option from bool to tristate.

This allows to compile pci-mvebu.c driver as loadable module pci-mvebu.ko
with ability to unload it.

Signed-off-by: Pali Rohár <pali@kernel.org>
---
 drivers/pci/controller/Kconfig     |  2 +-
 drivers/pci/controller/pci-mvebu.c | 91 +++++++++++++++++++++++++-----
 2 files changed, 77 insertions(+), 16 deletions(-)

diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig
index 93b141110537..67189bcd5d89 100644
--- a/drivers/pci/controller/Kconfig
+++ b/drivers/pci/controller/Kconfig
@@ -4,7 +4,7 @@ menu "PCI controller drivers"
 	depends on PCI
 
 config PCI_MVEBU
-	bool "Marvell EBU PCIe controller"
+	tristate "Marvell EBU PCIe controller"
 	depends on ARCH_MVEBU || ARCH_DOVE || COMPILE_TEST
 	depends on MVEBU_MBUS
 	depends on ARM
diff --git a/drivers/pci/controller/pci-mvebu.c b/drivers/pci/controller/pci-mvebu.c
index f2180e4630a1..f43492695f94 100644
--- a/drivers/pci/controller/pci-mvebu.c
+++ b/drivers/pci/controller/pci-mvebu.c
@@ -6,6 +6,7 @@
  */
 
 #include <linux/kernel.h>
+#include <linux/module.h>
 #include <linux/pci.h>
 #include <linux/clk.h>
 #include <linux/delay.h>
@@ -145,22 +146,13 @@ static void mvebu_pcie_set_local_dev_nr(struct mvebu_pcie_port *port, int nr)
 	mvebu_writel(port, stat, PCIE_STAT_OFF);
 }
 
-/*
- * Setup PCIE BARs and Address Decode Wins:
- * BAR[0] -> internal registers (needed for MSI)
- * BAR[1] -> covers all DRAM banks
- * BAR[2] -> Disabled
- * WIN[0-3] -> DRAM bank[0-3]
- */
-static void mvebu_pcie_setup_wins(struct mvebu_pcie_port *port)
+static void mvebu_pcie_disable_wins(struct mvebu_pcie_port *port)
 {
-	const struct mbus_dram_target_info *dram;
-	u32 size;
 	int i;
 
-	dram = mv_mbus_dram_info();
+	mvebu_writel(port, 0, PCIE_BAR_LO_OFF(0));
+	mvebu_writel(port, 0, PCIE_BAR_HI_OFF(0));
 
-	/* First, disable and clear BARs and windows. */
 	for (i = 1; i < 3; i++) {
 		mvebu_writel(port, 0, PCIE_BAR_CTRL_OFF(i));
 		mvebu_writel(port, 0, PCIE_BAR_LO_OFF(i));
@@ -176,6 +168,25 @@ static void mvebu_pcie_setup_wins(struct mvebu_pcie_port *port)
 	mvebu_writel(port, 0, PCIE_WIN5_CTRL_OFF);
 	mvebu_writel(port, 0, PCIE_WIN5_BASE_OFF);
 	mvebu_writel(port, 0, PCIE_WIN5_REMAP_OFF);
+}
+
+/*
+ * Setup PCIE BARs and Address Decode Wins:
+ * BAR[0] -> internal registers (needed for MSI)
+ * BAR[1] -> covers all DRAM banks
+ * BAR[2] -> Disabled
+ * WIN[0-3] -> DRAM bank[0-3]
+ */
+static void mvebu_pcie_setup_wins(struct mvebu_pcie_port *port)
+{
+	const struct mbus_dram_target_info *dram;
+	u32 size;
+	int i;
+
+	dram = mv_mbus_dram_info();
+
+	/* First, disable and clear BARs and windows. */
+	mvebu_pcie_disable_wins(port);
 
 	/* Setup windows for DDR banks.  Count total DDR size on the fly. */
 	size = 0;
@@ -1082,6 +1093,52 @@ static int mvebu_pcie_probe(struct platform_device *pdev)
 	return pci_host_probe(bridge);
 }
 
+static int mvebu_pcie_remove(struct platform_device *pdev)
+{
+	struct mvebu_pcie *pcie = platform_get_drvdata(pdev);
+	struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie);
+	u32 cmd;
+	int i;
+
+	/* Remove PCI bus with all devices. */
+	pci_lock_rescan_remove();
+	pci_stop_root_bus(bridge->bus);
+	pci_remove_root_bus(bridge->bus);
+	pci_unlock_rescan_remove();
+
+	for (i = 0; i < pcie->nports; i++) {
+		struct mvebu_pcie_port *port = &pcie->ports[i];
+
+		if (!port->base)
+			continue;
+
+		/* Disable Root Bridge I/O space, memory space and bus mastering. */
+		cmd = mvebu_readl(port, PCIE_CMD_OFF);
+		cmd &= ~(PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER);
+		mvebu_writel(port, cmd, PCIE_CMD_OFF);
+
+		/* Mask all interrupt sources. */
+		mvebu_writel(port, 0, PCIE_MASK_OFF);
+
+		/* Free config space for emulated root bridge. */
+		pci_bridge_emul_cleanup(&port->bridge);
+
+		/* Disable and clear BARs and windows. */
+		mvebu_pcie_disable_wins(port);
+
+		/* Delete PCIe IO and MEM windows. */
+		if (port->iowin.size)
+			mvebu_pcie_del_windows(port, port->iowin.base, port->iowin.size);
+		if (port->memwin.size)
+			mvebu_pcie_del_windows(port, port->memwin.base, port->memwin.size);
+
+		/* Power down card and disable clocks. Must be the last step. */
+		mvebu_pcie_powerdown(port);
+	}
+
+	return 0;
+}
+
 static const struct of_device_id mvebu_pcie_of_match_table[] = {
 	{ .compatible = "marvell,armada-xp-pcie", },
 	{ .compatible = "marvell,armada-370-pcie", },
@@ -1098,10 +1155,14 @@ static struct platform_driver mvebu_pcie_driver = {
 	.driver = {
 		.name = "mvebu-pcie",
 		.of_match_table = mvebu_pcie_of_match_table,
-		/* driver unloading/unbinding currently not supported */
-		.suppress_bind_attrs = true,
 		.pm = &mvebu_pcie_pm_ops,
 	},
 	.probe = mvebu_pcie_probe,
+	.remove = mvebu_pcie_remove,
 };
-builtin_platform_driver(mvebu_pcie_driver);
+module_platform_driver(mvebu_pcie_driver);
+
+MODULE_AUTHOR("Thomas Petazzoni <thomas.petazzoni@bootlin.com>");
+MODULE_AUTHOR("Pali Rohár <pali@kernel.org>");
+MODULE_DESCRIPTION("Marvell EBU PCIe controller");
+MODULE_LICENSE("GPL v2");
-- 
2.20.1

Re: [PATCH 0/2] PCI: mvebu: Add support for unbinding and unloading driver

[Lorenzo Pieralisi]

On Fri, 26 Nov 2021 15:43:05 +0100, Pali Rohár wrote:
> This patch series add support for unbinding device from pci-mvebu driver,
> allow to compile pci-mvebu.c as kernel module pci-mvebu.ko and allow to
> rmmod this module.
> 
> This patch series depends on another patch series which converts
> pci-mvebu.c driver to use devm_pci_remap_iospace() function:
> 
> [...]

Applied to pci/mvebu, thanks!

[1/2] bus: mvebu-mbus: Export symbols for public API window functions
      https://git.kernel.org/lpieralisi/pci/c/859186e238
[2/2] PCI: mvebu: Add support for compiling driver as module
      https://git.kernel.org/lpieralisi/pci/c/0746ae1be1

Thanks,
Lorenzo