[PATCH v2 00/10] cxl: Remove abandonned capi support for the Mellanox CX4

[PATCH v2 00/10] cxl: Remove abandonned capi support for the Mellanox CX4

[Frederic Barrat]

An attempt was made to add capi support for the Mellanox CX4 card, so
that it could operate in "traditional" PCI mode or capi mode. The
project ended up being canceled and a different approach was taken for
CX5. Mellanox never upstreamed any capi support in their CX4 driver.

CX4 was not following exactly the CAIA 1.0 model, so some CX4-specific
code was added. That code is now dead and hasn't been tested for a
while, so it's probably broken anyway. Let's remove it. It has been
agreed with engineers at Mellanox, of course.

No user API is affected. Some (unused) symbols exported by cxl are removed.

Most of the patch set consists of reverts of older patches, with
sometimes very minor tweaking. The last patch aggregates a few changes
where reverting was not possible easily.

Tests run to prevent regressions:
-  memcpy tests on POWER8 and POWER9
-  Mellanox CX5, which uses a different code path, based on cxllib
-  cxlflash tests on POWER8


Changelog:
v2: add missing signed-offs for the revert patches


Alastair D'Silva (7):
  Revert "cxl: Add kernel API to allow a context to operate with
    relocate disabled"
  Revert "cxl: Add support for interrupts on the Mellanox CX4"
  Revert "cxl: Add preliminary workaround for CX4 interrupt limitation"
  Revert "cxl: Add kernel APIs to get & set the max irqs per context"
  Revert "cxl: Add cxl_check_and_switch_mode() API to switch bi-modal
    cards"
  Revert "cxl: Add support for using the kernel API with a real PHB"
  Revert "powerpc/powernv: Add support for the cxl kernel api on the
    real phb"

Frederic Barrat (3):
  Revert "cxl: Add cxl_slot_is_supported API"
  Revert "cxl: Allow a default context to be associated with an external
    pci_dev"
  cxl: Remove abandonned capi support for the Mellanox CX4, final
    cleanup

 arch/powerpc/include/asm/pnv-pci.h        |   7 -
 arch/powerpc/platforms/powernv/pci-cxl.c  | 199 ------------
 arch/powerpc/platforms/powernv/pci-ioda.c |  22 +-
 arch/powerpc/platforms/powernv/pci.h      |  15 -
 drivers/misc/cxl/Kconfig                  |   8 -
 drivers/misc/cxl/Makefile                 |   2 +-
 drivers/misc/cxl/api.c                    | 132 --------
 drivers/misc/cxl/base.c                   |  83 -----
 drivers/misc/cxl/context.c                |   3 +-
 drivers/misc/cxl/cxl.h                    |  33 --
 drivers/misc/cxl/debugfs.c                |   5 -
 drivers/misc/cxl/guest.c                  |   3 -
 drivers/misc/cxl/main.c                   |   5 -
 drivers/misc/cxl/native.c                 |   3 +-
 drivers/misc/cxl/pci.c                    | 351 ++--------------------
 drivers/misc/cxl/phb.c                    |  44 ---
 drivers/misc/cxl/vphb.c                   |  46 +--
 include/misc/cxl-base.h                   |  10 -
 include/misc/cxl.h                        |  68 -----
 19 files changed, 58 insertions(+), 981 deletions(-)
 delete mode 100644 drivers/misc/cxl/phb.c

-- 
2.17.1

[PATCH v2 01/10] Revert "cxl: Add kernel API to allow a context to operate with relocate disabled"

[Frederic Barrat]

From: Alastair D'Silva <alastair@d-silva.org>

Remove abandonned capi support for the Mellanox CX4.
The symbol 'cxl_set_translation_mode' is never called, so
ctx->real_mode is always false.

This reverts commit 7a0d85d313c2066712e530e668bc02bb741a685c.

Signed-off-by: Alastair D'Silva <alastair@d-silva.org>
---
 drivers/misc/cxl/api.c    | 19 -------------------
 drivers/misc/cxl/cxl.h    |  1 -
 drivers/misc/cxl/guest.c  |  3 ---
 drivers/misc/cxl/native.c |  3 ++-
 include/misc/cxl.h        |  8 --------
 5 files changed, 2 insertions(+), 32 deletions(-)

diff --git a/drivers/misc/cxl/api.c b/drivers/misc/cxl/api.c
index 753b1a698fc4..21d620e29fea 100644
--- a/drivers/misc/cxl/api.c
+++ b/drivers/misc/cxl/api.c
@@ -324,7 +324,6 @@ int cxl_start_context(struct cxl_context *ctx, u64 wed,
 	if (task) {
 		ctx->pid = get_task_pid(task, PIDTYPE_PID);
 		kernel = false;
-		ctx->real_mode = false;
 
 		/* acquire a reference to the task's mm */
 		ctx->mm = get_task_mm(current);
@@ -388,24 +387,6 @@ void cxl_set_master(struct cxl_context *ctx)
 }
 EXPORT_SYMBOL_GPL(cxl_set_master);
 
-int cxl_set_translation_mode(struct cxl_context *ctx, bool real_mode)
-{
-	if (ctx->status == STARTED) {
-		/*
-		 * We could potentially update the PE and issue an update LLCMD
-		 * to support this, but it doesn't seem to have a good use case
-		 * since it's trivial to just create a second kernel context
-		 * with different translation modes, so until someone convinces
-		 * me otherwise:
-		 */
-		return -EBUSY;
-	}
-
-	ctx->real_mode = real_mode;
-	return 0;
-}
-EXPORT_SYMBOL_GPL(cxl_set_translation_mode);
-
 /* wrappers around afu_* file ops which are EXPORTED */
 int cxl_fd_open(struct inode *inode, struct file *file)
 {
diff --git a/drivers/misc/cxl/cxl.h b/drivers/misc/cxl/cxl.h
index 918d4fb742d1..af8794719956 100644
--- a/drivers/misc/cxl/cxl.h
+++ b/drivers/misc/cxl/cxl.h
@@ -613,7 +613,6 @@ struct cxl_context {
 	bool pe_inserted;
 	bool master;
 	bool kernel;
-	bool real_mode;
 	bool pending_irq;
 	bool pending_fault;
 	bool pending_afu_err;
diff --git a/drivers/misc/cxl/guest.c b/drivers/misc/cxl/guest.c
index 4644f16606a3..f5dc740fcd13 100644
--- a/drivers/misc/cxl/guest.c
+++ b/drivers/misc/cxl/guest.c
@@ -623,9 +623,6 @@ static int guest_attach_process(struct cxl_context *ctx, bool kernel, u64 wed, u
 {
 	pr_devel("in %s\n", __func__);
 
-	if (ctx->real_mode)
-		return -EPERM;
-
 	ctx->kernel = kernel;
 	if (ctx->afu->current_mode == CXL_MODE_DIRECTED)
 		return attach_afu_directed(ctx, wed, amr);
diff --git a/drivers/misc/cxl/native.c b/drivers/misc/cxl/native.c
index 98f867fcef24..c9d5d82dce8e 100644
--- a/drivers/misc/cxl/native.c
+++ b/drivers/misc/cxl/native.c
@@ -605,6 +605,7 @@ u64 cxl_calculate_sr(bool master, bool kernel, bool real_mode, bool p9)
 		sr |= CXL_PSL_SR_An_MP;
 	if (mfspr(SPRN_LPCR) & LPCR_TC)
 		sr |= CXL_PSL_SR_An_TC;
+
 	if (kernel) {
 		if (!real_mode)
 			sr |= CXL_PSL_SR_An_R;
@@ -629,7 +630,7 @@ u64 cxl_calculate_sr(bool master, bool kernel, bool real_mode, bool p9)
 
 static u64 calculate_sr(struct cxl_context *ctx)
 {
-	return cxl_calculate_sr(ctx->master, ctx->kernel, ctx->real_mode,
+	return cxl_calculate_sr(ctx->master, ctx->kernel, false,
 				cxl_is_power9());
 }
 
diff --git a/include/misc/cxl.h b/include/misc/cxl.h
index b712be544f8c..82cc6ffafe2d 100644
--- a/include/misc/cxl.h
+++ b/include/misc/cxl.h
@@ -173,14 +173,6 @@ int cxl_afu_reset(struct cxl_context *ctx);
  */
 void cxl_set_master(struct cxl_context *ctx);
 
-/*
- * Sets the context to use real mode memory accesses to operate with
- * translation disabled. Note that this only makes sense for kernel contexts
- * under bare metal, and will not work with virtualisation. May only be
- * performed on stopped contexts.
- */
-int cxl_set_translation_mode(struct cxl_context *ctx, bool real_mode);
-
 /*
  * Map and unmap the AFU Problem Space area. The amount and location mapped
  * depends on if this context is a master or slave.
-- 
2.17.1

Re: [PATCH v2 01/10] Revert "cxl: Add kernel API to allow a context to operate with relocate disabled"

[Andrew Donnellan]

On 28/06/18 20:05, Frederic Barrat wrote:
> From: Alastair D'Silva <alastair@d-silva.org>
> 
> Remove abandonned capi support for the Mellanox CX4.
> The symbol 'cxl_set_translation_mode' is never called, so
> ctx->real_mode is always false.
> 
> This reverts commit 7a0d85d313c2066712e530e668bc02bb741a685c.
> 
> Signed-off-by: Alastair D'Silva <alastair@d-silva.org>

Acked-by: Andrew Donnellan <andrew.donnellan@au1.ibm.com>

> ---
>   drivers/misc/cxl/api.c    | 19 -------------------
>   drivers/misc/cxl/cxl.h    |  1 -
>   drivers/misc/cxl/guest.c  |  3 ---
>   drivers/misc/cxl/native.c |  3 ++-
>   include/misc/cxl.h        |  8 --------
>   5 files changed, 2 insertions(+), 32 deletions(-)
> 
> diff --git a/drivers/misc/cxl/api.c b/drivers/misc/cxl/api.c
> index 753b1a698fc4..21d620e29fea 100644
> --- a/drivers/misc/cxl/api.c
> +++ b/drivers/misc/cxl/api.c
> @@ -324,7 +324,6 @@ int cxl_start_context(struct cxl_context *ctx, u64 wed,
>   	if (task) {
>   		ctx->pid = get_task_pid(task, PIDTYPE_PID);
>   		kernel = false;
> -		ctx->real_mode = false;
>   
>   		/* acquire a reference to the task's mm */
>   		ctx->mm = get_task_mm(current);
> @@ -388,24 +387,6 @@ void cxl_set_master(struct cxl_context *ctx)
>   }
>   EXPORT_SYMBOL_GPL(cxl_set_master);
>   
> -int cxl_set_translation_mode(struct cxl_context *ctx, bool real_mode)
> -{
> -	if (ctx->status == STARTED) {
> -		/*
> -		 * We could potentially update the PE and issue an update LLCMD
> -		 * to support this, but it doesn't seem to have a good use case
> -		 * since it's trivial to just create a second kernel context
> -		 * with different translation modes, so until someone convinces
> -		 * me otherwise:
> -		 */
> -		return -EBUSY;
> -	}
> -
> -	ctx->real_mode = real_mode;
> -	return 0;
> -}
> -EXPORT_SYMBOL_GPL(cxl_set_translation_mode);
> -
>   /* wrappers around afu_* file ops which are EXPORTED */
>   int cxl_fd_open(struct inode *inode, struct file *file)
>   {
> diff --git a/drivers/misc/cxl/cxl.h b/drivers/misc/cxl/cxl.h
> index 918d4fb742d1..af8794719956 100644
> --- a/drivers/misc/cxl/cxl.h
> +++ b/drivers/misc/cxl/cxl.h
> @@ -613,7 +613,6 @@ struct cxl_context {
>   	bool pe_inserted;
>   	bool master;
>   	bool kernel;
> -	bool real_mode;
>   	bool pending_irq;
>   	bool pending_fault;
>   	bool pending_afu_err;
> diff --git a/drivers/misc/cxl/guest.c b/drivers/misc/cxl/guest.c
> index 4644f16606a3..f5dc740fcd13 100644
> --- a/drivers/misc/cxl/guest.c
> +++ b/drivers/misc/cxl/guest.c
> @@ -623,9 +623,6 @@ static int guest_attach_process(struct cxl_context *ctx, bool kernel, u64 wed, u
>   {
>   	pr_devel("in %s\n", __func__);
>   
> -	if (ctx->real_mode)
> -		return -EPERM;
> -
>   	ctx->kernel = kernel;
>   	if (ctx->afu->current_mode == CXL_MODE_DIRECTED)
>   		return attach_afu_directed(ctx, wed, amr);
> diff --git a/drivers/misc/cxl/native.c b/drivers/misc/cxl/native.c
> index 98f867fcef24..c9d5d82dce8e 100644
> --- a/drivers/misc/cxl/native.c
> +++ b/drivers/misc/cxl/native.c
> @@ -605,6 +605,7 @@ u64 cxl_calculate_sr(bool master, bool kernel, bool real_mode, bool p9)
>   		sr |= CXL_PSL_SR_An_MP;
>   	if (mfspr(SPRN_LPCR) & LPCR_TC)
>   		sr |= CXL_PSL_SR_An_TC;
> +
>   	if (kernel) {
>   		if (!real_mode)
>   			sr |= CXL_PSL_SR_An_R;
> @@ -629,7 +630,7 @@ u64 cxl_calculate_sr(bool master, bool kernel, bool real_mode, bool p9)
>   
>   static u64 calculate_sr(struct cxl_context *ctx)
>   {
> -	return cxl_calculate_sr(ctx->master, ctx->kernel, ctx->real_mode,
> +	return cxl_calculate_sr(ctx->master, ctx->kernel, false,
>   				cxl_is_power9());
>   }
>   
> diff --git a/include/misc/cxl.h b/include/misc/cxl.h
> index b712be544f8c..82cc6ffafe2d 100644
> --- a/include/misc/cxl.h
> +++ b/include/misc/cxl.h
> @@ -173,14 +173,6 @@ int cxl_afu_reset(struct cxl_context *ctx);
>    */
>   void cxl_set_master(struct cxl_context *ctx);
>   
> -/*
> - * Sets the context to use real mode memory accesses to operate with
> - * translation disabled. Note that this only makes sense for kernel contexts
> - * under bare metal, and will not work with virtualisation. May only be
> - * performed on stopped contexts.
> - */
> -int cxl_set_translation_mode(struct cxl_context *ctx, bool real_mode);
> -
>   /*
>    * Map and unmap the AFU Problem Space area. The amount and location mapped
>    * depends on if this context is a master or slave.
> 

-- 
Andrew Donnellan              OzLabs, ADL Canberra
andrew.donnellan@au1.ibm.com  IBM Australia Limited

Re: [v2, 01/10] Revert "cxl: Add kernel API to allow a context to operate with relocate disabled"

[Michael Ellerman]

On Thu, 2018-06-28 at 10:05:00 UTC, Frederic Barrat wrote:
> From: Alastair D'Silva <alastair@d-silva.org>
> 
> Remove abandonned capi support for the Mellanox CX4.
> The symbol 'cxl_set_translation_mode' is never called, so
> ctx->real_mode is always false.
> 
> This reverts commit 7a0d85d313c2066712e530e668bc02bb741a685c.
> 
> Signed-off-by: Alastair D'Silva <alastair@d-silva.org>
> Acked-by: Andrew Donnellan <andrew.donnellan@au1.ibm.com>

Series applied to powerpc next, thanks.

https://git.kernel.org/powerpc/c/c5828150067c47a97f30e690a472e0

cheers

[PATCH v2 02/10] Revert "cxl: Add support for interrupts on the Mellanox CX4"

[Frederic Barrat]

From: Alastair D'Silva <alastair@d-silva.org>

Remove abandonned capi support for the Mellanox CX4.

This reverts commit a2f67d5ee8d950caaa7a6144cf0bfb256500b73e.

Signed-off-by: Alastair D'Silva <alastair@d-silva.org>
---
 arch/powerpc/platforms/powernv/pci-cxl.c  | 84 -----------------------
 arch/powerpc/platforms/powernv/pci-ioda.c |  4 --
 arch/powerpc/platforms/powernv/pci.h      |  2 -
 drivers/misc/cxl/api.c                    | 71 -------------------
 drivers/misc/cxl/base.c                   | 31 ---------
 drivers/misc/cxl/cxl.h                    |  4 --
 drivers/misc/cxl/main.c                   |  2 -
 include/misc/cxl-base.h                   |  4 --
 8 files changed, 202 deletions(-)

diff --git a/arch/powerpc/platforms/powernv/pci-cxl.c b/arch/powerpc/platforms/powernv/pci-cxl.c
index cee003de63af..c447b7f03c09 100644
--- a/arch/powerpc/platforms/powernv/pci-cxl.c
+++ b/arch/powerpc/platforms/powernv/pci-cxl.c
@@ -8,7 +8,6 @@
  */
 
 #include <linux/module.h>
-#include <linux/msi.h>
 #include <asm/pci-bridge.h>
 #include <asm/pnv-pci.h>
 #include <asm/opal.h>
@@ -292,86 +291,3 @@ void pnv_cxl_disable_device(struct pci_dev *dev)
 	cxl_pci_disable_device(dev);
 	cxl_afu_put(afu);
 }
-
-/*
- * This is a special version of pnv_setup_msi_irqs for cards in cxl mode. This
- * function handles setting up the IVTE entries for the XSL to use.
- *
- * We are currently not filling out the MSIX table, since the only currently
- * supported adapter (CX4) uses a custom MSIX table format in cxl mode and it
- * is up to their driver to fill that out. In the future we may fill out the
- * MSIX table (and change the IVTE entries to be an index to the MSIX table)
- * for adapters implementing the Full MSI-X mode described in the CAIA.
- */
-int pnv_cxl_cx4_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
-{
-	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
-	struct pnv_phb *phb = hose->private_data;
-	struct msi_desc *entry;
-	struct cxl_context *ctx = NULL;
-	unsigned int virq;
-	int hwirq;
-	int afu_irq = 0;
-	int rc;
-
-	if (WARN_ON(!phb) || !phb->msi_bmp.bitmap)
-		return -ENODEV;
-
-	if (pdev->no_64bit_msi && !phb->msi32_support)
-		return -ENODEV;
-
-	rc = cxl_cx4_setup_msi_irqs(pdev, nvec, type);
-	if (rc)
-		return rc;
-
-	for_each_pci_msi_entry(entry, pdev) {
-		if (!entry->msi_attrib.is_64 && !phb->msi32_support) {
-			pr_warn("%s: Supports only 64-bit MSIs\n",
-				pci_name(pdev));
-			return -ENXIO;
-		}
-
-		hwirq = cxl_next_msi_hwirq(pdev, &ctx, &afu_irq);
-		if (WARN_ON(hwirq <= 0))
-			return (hwirq ? hwirq : -ENOMEM);
-
-		virq = irq_create_mapping(NULL, hwirq);
-		if (!virq) {
-			pr_warn("%s: Failed to map cxl mode MSI to linux irq\n",
-				pci_name(pdev));
-			return -ENOMEM;
-		}
-
-		rc = pnv_cxl_ioda_msi_setup(pdev, hwirq, virq);
-		if (rc) {
-			pr_warn("%s: Failed to setup cxl mode MSI\n", pci_name(pdev));
-			irq_dispose_mapping(virq);
-			return rc;
-		}
-
-		irq_set_msi_desc(virq, entry);
-	}
-
-	return 0;
-}
-
-void pnv_cxl_cx4_teardown_msi_irqs(struct pci_dev *pdev)
-{
-	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
-	struct pnv_phb *phb = hose->private_data;
-	struct msi_desc *entry;
-	irq_hw_number_t hwirq;
-
-	if (WARN_ON(!phb))
-		return;
-
-	for_each_pci_msi_entry(entry, pdev) {
-		if (!entry->irq)
-			continue;
-		hwirq = virq_to_hw(entry->irq);
-		irq_set_msi_desc(entry->irq, NULL);
-		irq_dispose_mapping(entry->irq);
-	}
-
-	cxl_cx4_teardown_msi_irqs(pdev);
-}
diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
index 5bd0eb6681bc..41f8f0ff4a55 100644
--- a/arch/powerpc/platforms/powernv/pci-ioda.c
+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
@@ -3847,10 +3847,6 @@ static const struct pci_controller_ops pnv_npu_ocapi_ioda_controller_ops = {
 const struct pci_controller_ops pnv_cxl_cx4_ioda_controller_ops = {
 	.dma_dev_setup		= pnv_pci_dma_dev_setup,
 	.dma_bus_setup		= pnv_pci_dma_bus_setup,
-#ifdef CONFIG_PCI_MSI
-	.setup_msi_irqs		= pnv_cxl_cx4_setup_msi_irqs,
-	.teardown_msi_irqs	= pnv_cxl_cx4_teardown_msi_irqs,
-#endif
 	.enable_device_hook	= pnv_cxl_enable_device_hook,
 	.disable_device		= pnv_cxl_disable_device,
 	.release_device		= pnv_pci_release_device,
diff --git a/arch/powerpc/platforms/powernv/pci.h b/arch/powerpc/platforms/powernv/pci.h
index eada4b6068cb..ba41913c7e21 100644
--- a/arch/powerpc/platforms/powernv/pci.h
+++ b/arch/powerpc/platforms/powernv/pci.h
@@ -265,8 +265,6 @@ extern int pnv_npu2_init(struct pnv_phb *phb);
 /* cxl functions */
 extern bool pnv_cxl_enable_device_hook(struct pci_dev *dev);
 extern void pnv_cxl_disable_device(struct pci_dev *dev);
-extern int pnv_cxl_cx4_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type);
-extern void pnv_cxl_cx4_teardown_msi_irqs(struct pci_dev *pdev);
 
 
 /* phb ops (cxl switches these when enabling the kernel api on the phb) */
diff --git a/drivers/misc/cxl/api.c b/drivers/misc/cxl/api.c
index 21d620e29fea..2e5862b7a074 100644
--- a/drivers/misc/cxl/api.c
+++ b/drivers/misc/cxl/api.c
@@ -11,7 +11,6 @@
 #include <linux/slab.h>
 #include <linux/file.h>
 #include <misc/cxl.h>
-#include <linux/msi.h>
 #include <linux/module.h>
 #include <linux/mount.h>
 #include <linux/sched/mm.h>
@@ -595,73 +594,3 @@ int cxl_get_max_irqs_per_process(struct pci_dev *dev)
 	return afu->irqs_max;
 }
 EXPORT_SYMBOL_GPL(cxl_get_max_irqs_per_process);
-
-/*
- * This is a special interrupt allocation routine called from the PHB's MSI
- * setup function. When capi interrupts are allocated in this manner they must
- * still be associated with a running context, but since the MSI APIs have no
- * way to specify this we use the default context associated with the device.
- *
- * The Mellanox CX4 has a hardware limitation that restricts the maximum AFU
- * interrupt number, so in order to overcome this their driver informs us of
- * the restriction by setting the maximum interrupts per context, and we
- * allocate additional contexts as necessary so that we can keep the AFU
- * interrupt number within the supported range.
- */
-int _cxl_cx4_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
-{
-	struct cxl_context *ctx, *new_ctx, *default_ctx;
-	int remaining;
-	int rc;
-
-	ctx = default_ctx = cxl_get_context(pdev);
-	if (WARN_ON(!default_ctx))
-		return -ENODEV;
-
-	remaining = nvec;
-	while (remaining > 0) {
-		rc = cxl_allocate_afu_irqs(ctx, min(remaining, ctx->afu->irqs_max));
-		if (rc) {
-			pr_warn("%s: Failed to find enough free MSIs\n", pci_name(pdev));
-			return rc;
-		}
-		remaining -= ctx->afu->irqs_max;
-
-		if (ctx != default_ctx && default_ctx->status == STARTED) {
-			WARN_ON(cxl_start_context(ctx,
-				be64_to_cpu(default_ctx->elem->common.wed),
-				NULL));
-		}
-
-		if (remaining > 0) {
-			new_ctx = cxl_dev_context_init(pdev);
-			if (IS_ERR(new_ctx)) {
-				pr_warn("%s: Failed to allocate enough contexts for MSIs\n", pci_name(pdev));
-				return -ENOSPC;
-			}
-			list_add(&new_ctx->extra_irq_contexts, &ctx->extra_irq_contexts);
-			ctx = new_ctx;
-		}
-	}
-
-	return 0;
-}
-/* Exported via cxl_base */
-
-void _cxl_cx4_teardown_msi_irqs(struct pci_dev *pdev)
-{
-	struct cxl_context *ctx, *pos, *tmp;
-
-	ctx = cxl_get_context(pdev);
-	if (WARN_ON(!ctx))
-		return;
-
-	cxl_free_afu_irqs(ctx);
-	list_for_each_entry_safe(pos, tmp, &ctx->extra_irq_contexts, extra_irq_contexts) {
-		cxl_stop_context(pos);
-		cxl_free_afu_irqs(pos);
-		list_del(&pos->extra_irq_contexts);
-		cxl_release_context(pos);
-	}
-}
-/* Exported via cxl_base */
diff --git a/drivers/misc/cxl/base.c b/drivers/misc/cxl/base.c
index cd54ce6f6230..fe90f895bb10 100644
--- a/drivers/misc/cxl/base.c
+++ b/drivers/misc/cxl/base.c
@@ -158,37 +158,6 @@ int cxl_next_msi_hwirq(struct pci_dev *pdev, struct cxl_context **ctx, int *afu_
 }
 EXPORT_SYMBOL_GPL(cxl_next_msi_hwirq);
 
-int cxl_cx4_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
-{
-	int ret;
-	struct cxl_calls *calls;
-
-	calls = cxl_calls_get();
-	if (!calls)
-		return false;
-
-	ret = calls->cxl_cx4_setup_msi_irqs(pdev, nvec, type);
-
-	cxl_calls_put(calls);
-
-	return ret;
-}
-EXPORT_SYMBOL_GPL(cxl_cx4_setup_msi_irqs);
-
-void cxl_cx4_teardown_msi_irqs(struct pci_dev *pdev)
-{
-	struct cxl_calls *calls;
-
-	calls = cxl_calls_get();
-	if (!calls)
-		return;
-
-	calls->cxl_cx4_teardown_msi_irqs(pdev);
-
-	cxl_calls_put(calls);
-}
-EXPORT_SYMBOL_GPL(cxl_cx4_teardown_msi_irqs);
-
 static int __init cxl_base_init(void)
 {
 	struct device_node *np;
diff --git a/drivers/misc/cxl/cxl.h b/drivers/misc/cxl/cxl.h
index af8794719956..9688fe8b4d80 100644
--- a/drivers/misc/cxl/cxl.h
+++ b/drivers/misc/cxl/cxl.h
@@ -879,16 +879,12 @@ ssize_t cxl_pci_afu_read_err_buffer(struct cxl_afu *afu, char *buf,
 bool _cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu);
 void _cxl_pci_disable_device(struct pci_dev *dev);
 int _cxl_next_msi_hwirq(struct pci_dev *pdev, struct cxl_context **ctx, int *afu_irq);
-int _cxl_cx4_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type);
-void _cxl_cx4_teardown_msi_irqs(struct pci_dev *pdev);
 
 struct cxl_calls {
 	void (*cxl_slbia)(struct mm_struct *mm);
 	bool (*cxl_pci_associate_default_context)(struct pci_dev *dev, struct cxl_afu *afu);
 	void (*cxl_pci_disable_device)(struct pci_dev *dev);
 	int (*cxl_next_msi_hwirq)(struct pci_dev *pdev, struct cxl_context **ctx, int *afu_irq);
-	int (*cxl_cx4_setup_msi_irqs)(struct pci_dev *pdev, int nvec, int type);
-	void (*cxl_cx4_teardown_msi_irqs)(struct pci_dev *pdev);
 
 	struct module *owner;
 };
diff --git a/drivers/misc/cxl/main.c b/drivers/misc/cxl/main.c
index c1ba0d42cbc8..59a904efd104 100644
--- a/drivers/misc/cxl/main.c
+++ b/drivers/misc/cxl/main.c
@@ -107,8 +107,6 @@ static struct cxl_calls cxl_calls = {
 	.cxl_pci_associate_default_context = _cxl_pci_associate_default_context,
 	.cxl_pci_disable_device = _cxl_pci_disable_device,
 	.cxl_next_msi_hwirq = _cxl_next_msi_hwirq,
-	.cxl_cx4_setup_msi_irqs = _cxl_cx4_setup_msi_irqs,
-	.cxl_cx4_teardown_msi_irqs = _cxl_cx4_teardown_msi_irqs,
 	.owner = THIS_MODULE,
 };
 
diff --git a/include/misc/cxl-base.h b/include/misc/cxl-base.h
index b2ebc91fe09a..bb7e629ae492 100644
--- a/include/misc/cxl-base.h
+++ b/include/misc/cxl-base.h
@@ -43,8 +43,6 @@ void cxl_afu_put(struct cxl_afu *afu);
 void cxl_slbia(struct mm_struct *mm);
 bool cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu);
 void cxl_pci_disable_device(struct pci_dev *dev);
-int cxl_cx4_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type);
-void cxl_cx4_teardown_msi_irqs(struct pci_dev *pdev);
 
 #else /* CONFIG_CXL_BASE */
 
@@ -54,8 +52,6 @@ static inline void cxl_afu_put(struct cxl_afu *afu) {}
 static inline void cxl_slbia(struct mm_struct *mm) {}
 static inline bool cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu) { return false; }
 static inline void cxl_pci_disable_device(struct pci_dev *dev) {}
-static inline int cxl_cx4_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) { return -ENODEV; }
-static inline void cxl_cx4_teardown_msi_irqs(struct pci_dev *pdev) {}
 
 #endif /* CONFIG_CXL_BASE */
 
-- 
2.17.1

Re: [PATCH v2 02/10] Revert "cxl: Add support for interrupts on the Mellanox CX4"

[Andrew Donnellan]

On 28/06/18 20:05, Frederic Barrat wrote:
> From: Alastair D'Silva <alastair@d-silva.org>
> 
> Remove abandonned capi support for the Mellanox CX4.
> 
> This reverts commit a2f67d5ee8d950caaa7a6144cf0bfb256500b73e.
> 
> Signed-off-by: Alastair D'Silva <alastair@d-silva.org>

Acked-by: Andrew Donnellan <andrew.donnellan@au1.ibm.com>

> ---
>   arch/powerpc/platforms/powernv/pci-cxl.c  | 84 -----------------------
>   arch/powerpc/platforms/powernv/pci-ioda.c |  4 --
>   arch/powerpc/platforms/powernv/pci.h      |  2 -
>   drivers/misc/cxl/api.c                    | 71 -------------------
>   drivers/misc/cxl/base.c                   | 31 ---------
>   drivers/misc/cxl/cxl.h                    |  4 --
>   drivers/misc/cxl/main.c                   |  2 -
>   include/misc/cxl-base.h                   |  4 --
>   8 files changed, 202 deletions(-)
> 
> diff --git a/arch/powerpc/platforms/powernv/pci-cxl.c b/arch/powerpc/platforms/powernv/pci-cxl.c
> index cee003de63af..c447b7f03c09 100644
> --- a/arch/powerpc/platforms/powernv/pci-cxl.c
> +++ b/arch/powerpc/platforms/powernv/pci-cxl.c
> @@ -8,7 +8,6 @@
>    */
>   
>   #include <linux/module.h>
> -#include <linux/msi.h>
>   #include <asm/pci-bridge.h>
>   #include <asm/pnv-pci.h>
>   #include <asm/opal.h>
> @@ -292,86 +291,3 @@ void pnv_cxl_disable_device(struct pci_dev *dev)
>   	cxl_pci_disable_device(dev);
>   	cxl_afu_put(afu);
>   }
> -
> -/*
> - * This is a special version of pnv_setup_msi_irqs for cards in cxl mode. This
> - * function handles setting up the IVTE entries for the XSL to use.
> - *
> - * We are currently not filling out the MSIX table, since the only currently
> - * supported adapter (CX4) uses a custom MSIX table format in cxl mode and it
> - * is up to their driver to fill that out. In the future we may fill out the
> - * MSIX table (and change the IVTE entries to be an index to the MSIX table)
> - * for adapters implementing the Full MSI-X mode described in the CAIA.
> - */
> -int pnv_cxl_cx4_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
> -{
> -	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
> -	struct pnv_phb *phb = hose->private_data;
> -	struct msi_desc *entry;
> -	struct cxl_context *ctx = NULL;
> -	unsigned int virq;
> -	int hwirq;
> -	int afu_irq = 0;
> -	int rc;
> -
> -	if (WARN_ON(!phb) || !phb->msi_bmp.bitmap)
> -		return -ENODEV;
> -
> -	if (pdev->no_64bit_msi && !phb->msi32_support)
> -		return -ENODEV;
> -
> -	rc = cxl_cx4_setup_msi_irqs(pdev, nvec, type);
> -	if (rc)
> -		return rc;
> -
> -	for_each_pci_msi_entry(entry, pdev) {
> -		if (!entry->msi_attrib.is_64 && !phb->msi32_support) {
> -			pr_warn("%s: Supports only 64-bit MSIs\n",
> -				pci_name(pdev));
> -			return -ENXIO;
> -		}
> -
> -		hwirq = cxl_next_msi_hwirq(pdev, &ctx, &afu_irq);
> -		if (WARN_ON(hwirq <= 0))
> -			return (hwirq ? hwirq : -ENOMEM);
> -
> -		virq = irq_create_mapping(NULL, hwirq);
> -		if (!virq) {
> -			pr_warn("%s: Failed to map cxl mode MSI to linux irq\n",
> -				pci_name(pdev));
> -			return -ENOMEM;
> -		}
> -
> -		rc = pnv_cxl_ioda_msi_setup(pdev, hwirq, virq);
> -		if (rc) {
> -			pr_warn("%s: Failed to setup cxl mode MSI\n", pci_name(pdev));
> -			irq_dispose_mapping(virq);
> -			return rc;
> -		}
> -
> -		irq_set_msi_desc(virq, entry);
> -	}
> -
> -	return 0;
> -}
> -
> -void pnv_cxl_cx4_teardown_msi_irqs(struct pci_dev *pdev)
> -{
> -	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
> -	struct pnv_phb *phb = hose->private_data;
> -	struct msi_desc *entry;
> -	irq_hw_number_t hwirq;
> -
> -	if (WARN_ON(!phb))
> -		return;
> -
> -	for_each_pci_msi_entry(entry, pdev) {
> -		if (!entry->irq)
> -			continue;
> -		hwirq = virq_to_hw(entry->irq);
> -		irq_set_msi_desc(entry->irq, NULL);
> -		irq_dispose_mapping(entry->irq);
> -	}
> -
> -	cxl_cx4_teardown_msi_irqs(pdev);
> -}
> diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
> index 5bd0eb6681bc..41f8f0ff4a55 100644
> --- a/arch/powerpc/platforms/powernv/pci-ioda.c
> +++ b/arch/powerpc/platforms/powernv/pci-ioda.c
> @@ -3847,10 +3847,6 @@ static const struct pci_controller_ops pnv_npu_ocapi_ioda_controller_ops = {
>   const struct pci_controller_ops pnv_cxl_cx4_ioda_controller_ops = {
>   	.dma_dev_setup		= pnv_pci_dma_dev_setup,
>   	.dma_bus_setup		= pnv_pci_dma_bus_setup,
> -#ifdef CONFIG_PCI_MSI
> -	.setup_msi_irqs		= pnv_cxl_cx4_setup_msi_irqs,
> -	.teardown_msi_irqs	= pnv_cxl_cx4_teardown_msi_irqs,
> -#endif
>   	.enable_device_hook	= pnv_cxl_enable_device_hook,
>   	.disable_device		= pnv_cxl_disable_device,
>   	.release_device		= pnv_pci_release_device,
> diff --git a/arch/powerpc/platforms/powernv/pci.h b/arch/powerpc/platforms/powernv/pci.h
> index eada4b6068cb..ba41913c7e21 100644
> --- a/arch/powerpc/platforms/powernv/pci.h
> +++ b/arch/powerpc/platforms/powernv/pci.h
> @@ -265,8 +265,6 @@ extern int pnv_npu2_init(struct pnv_phb *phb);
>   /* cxl functions */
>   extern bool pnv_cxl_enable_device_hook(struct pci_dev *dev);
>   extern void pnv_cxl_disable_device(struct pci_dev *dev);
> -extern int pnv_cxl_cx4_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type);
> -extern void pnv_cxl_cx4_teardown_msi_irqs(struct pci_dev *pdev);
>   
>   
>   /* phb ops (cxl switches these when enabling the kernel api on the phb) */
> diff --git a/drivers/misc/cxl/api.c b/drivers/misc/cxl/api.c
> index 21d620e29fea..2e5862b7a074 100644
> --- a/drivers/misc/cxl/api.c
> +++ b/drivers/misc/cxl/api.c
> @@ -11,7 +11,6 @@
>   #include <linux/slab.h>
>   #include <linux/file.h>
>   #include <misc/cxl.h>
> -#include <linux/msi.h>
>   #include <linux/module.h>
>   #include <linux/mount.h>
>   #include <linux/sched/mm.h>
> @@ -595,73 +594,3 @@ int cxl_get_max_irqs_per_process(struct pci_dev *dev)
>   	return afu->irqs_max;
>   }
>   EXPORT_SYMBOL_GPL(cxl_get_max_irqs_per_process);
> -
> -/*
> - * This is a special interrupt allocation routine called from the PHB's MSI
> - * setup function. When capi interrupts are allocated in this manner they must
> - * still be associated with a running context, but since the MSI APIs have no
> - * way to specify this we use the default context associated with the device.
> - *
> - * The Mellanox CX4 has a hardware limitation that restricts the maximum AFU
> - * interrupt number, so in order to overcome this their driver informs us of
> - * the restriction by setting the maximum interrupts per context, and we
> - * allocate additional contexts as necessary so that we can keep the AFU
> - * interrupt number within the supported range.
> - */
> -int _cxl_cx4_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
> -{
> -	struct cxl_context *ctx, *new_ctx, *default_ctx;
> -	int remaining;
> -	int rc;
> -
> -	ctx = default_ctx = cxl_get_context(pdev);
> -	if (WARN_ON(!default_ctx))
> -		return -ENODEV;
> -
> -	remaining = nvec;
> -	while (remaining > 0) {
> -		rc = cxl_allocate_afu_irqs(ctx, min(remaining, ctx->afu->irqs_max));
> -		if (rc) {
> -			pr_warn("%s: Failed to find enough free MSIs\n", pci_name(pdev));
> -			return rc;
> -		}
> -		remaining -= ctx->afu->irqs_max;
> -
> -		if (ctx != default_ctx && default_ctx->status == STARTED) {
> -			WARN_ON(cxl_start_context(ctx,
> -				be64_to_cpu(default_ctx->elem->common.wed),
> -				NULL));
> -		}
> -
> -		if (remaining > 0) {
> -			new_ctx = cxl_dev_context_init(pdev);
> -			if (IS_ERR(new_ctx)) {
> -				pr_warn("%s: Failed to allocate enough contexts for MSIs\n", pci_name(pdev));
> -				return -ENOSPC;
> -			}
> -			list_add(&new_ctx->extra_irq_contexts, &ctx->extra_irq_contexts);
> -			ctx = new_ctx;
> -		}
> -	}
> -
> -	return 0;
> -}
> -/* Exported via cxl_base */
> -
> -void _cxl_cx4_teardown_msi_irqs(struct pci_dev *pdev)
> -{
> -	struct cxl_context *ctx, *pos, *tmp;
> -
> -	ctx = cxl_get_context(pdev);
> -	if (WARN_ON(!ctx))
> -		return;
> -
> -	cxl_free_afu_irqs(ctx);
> -	list_for_each_entry_safe(pos, tmp, &ctx->extra_irq_contexts, extra_irq_contexts) {
> -		cxl_stop_context(pos);
> -		cxl_free_afu_irqs(pos);
> -		list_del(&pos->extra_irq_contexts);
> -		cxl_release_context(pos);
> -	}
> -}
> -/* Exported via cxl_base */
> diff --git a/drivers/misc/cxl/base.c b/drivers/misc/cxl/base.c
> index cd54ce6f6230..fe90f895bb10 100644
> --- a/drivers/misc/cxl/base.c
> +++ b/drivers/misc/cxl/base.c
> @@ -158,37 +158,6 @@ int cxl_next_msi_hwirq(struct pci_dev *pdev, struct cxl_context **ctx, int *afu_
>   }
>   EXPORT_SYMBOL_GPL(cxl_next_msi_hwirq);
>   
> -int cxl_cx4_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
> -{
> -	int ret;
> -	struct cxl_calls *calls;
> -
> -	calls = cxl_calls_get();
> -	if (!calls)
> -		return false;
> -
> -	ret = calls->cxl_cx4_setup_msi_irqs(pdev, nvec, type);
> -
> -	cxl_calls_put(calls);
> -
> -	return ret;
> -}
> -EXPORT_SYMBOL_GPL(cxl_cx4_setup_msi_irqs);
> -
> -void cxl_cx4_teardown_msi_irqs(struct pci_dev *pdev)
> -{
> -	struct cxl_calls *calls;
> -
> -	calls = cxl_calls_get();
> -	if (!calls)
> -		return;
> -
> -	calls->cxl_cx4_teardown_msi_irqs(pdev);
> -
> -	cxl_calls_put(calls);
> -}
> -EXPORT_SYMBOL_GPL(cxl_cx4_teardown_msi_irqs);
> -
>   static int __init cxl_base_init(void)
>   {
>   	struct device_node *np;
> diff --git a/drivers/misc/cxl/cxl.h b/drivers/misc/cxl/cxl.h
> index af8794719956..9688fe8b4d80 100644
> --- a/drivers/misc/cxl/cxl.h
> +++ b/drivers/misc/cxl/cxl.h
> @@ -879,16 +879,12 @@ ssize_t cxl_pci_afu_read_err_buffer(struct cxl_afu *afu, char *buf,
>   bool _cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu);
>   void _cxl_pci_disable_device(struct pci_dev *dev);
>   int _cxl_next_msi_hwirq(struct pci_dev *pdev, struct cxl_context **ctx, int *afu_irq);
> -int _cxl_cx4_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type);
> -void _cxl_cx4_teardown_msi_irqs(struct pci_dev *pdev);
>   
>   struct cxl_calls {
>   	void (*cxl_slbia)(struct mm_struct *mm);
>   	bool (*cxl_pci_associate_default_context)(struct pci_dev *dev, struct cxl_afu *afu);
>   	void (*cxl_pci_disable_device)(struct pci_dev *dev);
>   	int (*cxl_next_msi_hwirq)(struct pci_dev *pdev, struct cxl_context **ctx, int *afu_irq);
> -	int (*cxl_cx4_setup_msi_irqs)(struct pci_dev *pdev, int nvec, int type);
> -	void (*cxl_cx4_teardown_msi_irqs)(struct pci_dev *pdev);
>   
>   	struct module *owner;
>   };
> diff --git a/drivers/misc/cxl/main.c b/drivers/misc/cxl/main.c
> index c1ba0d42cbc8..59a904efd104 100644
> --- a/drivers/misc/cxl/main.c
> +++ b/drivers/misc/cxl/main.c
> @@ -107,8 +107,6 @@ static struct cxl_calls cxl_calls = {
>   	.cxl_pci_associate_default_context = _cxl_pci_associate_default_context,
>   	.cxl_pci_disable_device = _cxl_pci_disable_device,
>   	.cxl_next_msi_hwirq = _cxl_next_msi_hwirq,
> -	.cxl_cx4_setup_msi_irqs = _cxl_cx4_setup_msi_irqs,
> -	.cxl_cx4_teardown_msi_irqs = _cxl_cx4_teardown_msi_irqs,
>   	.owner = THIS_MODULE,
>   };
>   
> diff --git a/include/misc/cxl-base.h b/include/misc/cxl-base.h
> index b2ebc91fe09a..bb7e629ae492 100644
> --- a/include/misc/cxl-base.h
> +++ b/include/misc/cxl-base.h
> @@ -43,8 +43,6 @@ void cxl_afu_put(struct cxl_afu *afu);
>   void cxl_slbia(struct mm_struct *mm);
>   bool cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu);
>   void cxl_pci_disable_device(struct pci_dev *dev);
> -int cxl_cx4_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type);
> -void cxl_cx4_teardown_msi_irqs(struct pci_dev *pdev);
>   
>   #else /* CONFIG_CXL_BASE */
>   
> @@ -54,8 +52,6 @@ static inline void cxl_afu_put(struct cxl_afu *afu) {}
>   static inline void cxl_slbia(struct mm_struct *mm) {}
>   static inline bool cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu) { return false; }
>   static inline void cxl_pci_disable_device(struct pci_dev *dev) {}
> -static inline int cxl_cx4_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) { return -ENODEV; }
> -static inline void cxl_cx4_teardown_msi_irqs(struct pci_dev *pdev) {}
>   
>   #endif /* CONFIG_CXL_BASE */
>   
> 

-- 
Andrew Donnellan              OzLabs, ADL Canberra
andrew.donnellan@au1.ibm.com  IBM Australia Limited

[PATCH v2 03/10] Revert "cxl: Add preliminary workaround for CX4 interrupt limitation"

[Frederic Barrat]

From: Alastair D'Silva <alastair@d-silva.org>

Remove abandonned capi support for the Mellanox CX4.

This reverts commit cbce0917e2e47d4bf5aa3b5fd6b1247f33e1a126.

Signed-off-by: Alastair D'Silva <alastair@d-silva.org>
---
 drivers/misc/cxl/api.c     | 15 ---------------
 drivers/misc/cxl/base.c    | 17 -----------------
 drivers/misc/cxl/context.c |  1 -
 drivers/misc/cxl/cxl.h     | 10 ----------
 drivers/misc/cxl/main.c    |  1 -
 include/misc/cxl.h         | 20 --------------------
 6 files changed, 64 deletions(-)

diff --git a/drivers/misc/cxl/api.c b/drivers/misc/cxl/api.c
index 2e5862b7a074..34ba67bc41bd 100644
--- a/drivers/misc/cxl/api.c
+++ b/drivers/misc/cxl/api.c
@@ -181,21 +181,6 @@ static irq_hw_number_t cxl_find_afu_irq(struct cxl_context *ctx, int num)
 	return 0;
 }
 
-int _cxl_next_msi_hwirq(struct pci_dev *pdev, struct cxl_context **ctx, int *afu_irq)
-{
-	if (*ctx == NULL || *afu_irq == 0) {
-		*afu_irq = 1;
-		*ctx = cxl_get_context(pdev);
-	} else {
-		(*afu_irq)++;
-		if (*afu_irq > cxl_get_max_irqs_per_process(pdev)) {
-			*ctx = list_next_entry(*ctx, extra_irq_contexts);
-			*afu_irq = 1;
-		}
-	}
-	return cxl_find_afu_irq(*ctx, *afu_irq);
-}
-/* Exported via cxl_base */
 
 int cxl_set_priv(struct cxl_context *ctx, void *priv)
 {
diff --git a/drivers/misc/cxl/base.c b/drivers/misc/cxl/base.c
index fe90f895bb10..e1e80cb99ad9 100644
--- a/drivers/misc/cxl/base.c
+++ b/drivers/misc/cxl/base.c
@@ -141,23 +141,6 @@ void cxl_pci_disable_device(struct pci_dev *dev)
 }
 EXPORT_SYMBOL_GPL(cxl_pci_disable_device);
 
-int cxl_next_msi_hwirq(struct pci_dev *pdev, struct cxl_context **ctx, int *afu_irq)
-{
-	int ret;
-	struct cxl_calls *calls;
-
-	calls = cxl_calls_get();
-	if (!calls)
-		return -EBUSY;
-
-	ret = calls->cxl_next_msi_hwirq(pdev, ctx, afu_irq);
-
-	cxl_calls_put(calls);
-
-	return ret;
-}
-EXPORT_SYMBOL_GPL(cxl_next_msi_hwirq);
-
 static int __init cxl_base_init(void)
 {
 	struct device_node *np;
diff --git a/drivers/misc/cxl/context.c b/drivers/misc/cxl/context.c
index c6ec872800a2..0355d42d367f 100644
--- a/drivers/misc/cxl/context.c
+++ b/drivers/misc/cxl/context.c
@@ -74,7 +74,6 @@ int cxl_context_init(struct cxl_context *ctx, struct cxl_afu *afu, bool master)
 	ctx->pending_afu_err = false;
 
 	INIT_LIST_HEAD(&ctx->irq_names);
-	INIT_LIST_HEAD(&ctx->extra_irq_contexts);
 
 	/*
 	 * When we have to destroy all contexts in cxl_context_detach_all() we
diff --git a/drivers/misc/cxl/cxl.h b/drivers/misc/cxl/cxl.h
index 9688fe8b4d80..d95c2c98f2ab 100644
--- a/drivers/misc/cxl/cxl.h
+++ b/drivers/misc/cxl/cxl.h
@@ -623,14 +623,6 @@ struct cxl_context {
 
 	struct rcu_head rcu;
 
-	/*
-	 * Only used when more interrupts are allocated via
-	 * pci_enable_msix_range than are supported in the default context, to
-	 * use additional contexts to overcome the limitation. i.e. Mellanox
-	 * CX4 only:
-	 */
-	struct list_head extra_irq_contexts;
-
 	struct mm_struct *mm;
 
 	u16 tidr;
@@ -878,13 +870,11 @@ ssize_t cxl_pci_afu_read_err_buffer(struct cxl_afu *afu, char *buf,
 /* Internal functions wrapped in cxl_base to allow PHB to call them */
 bool _cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu);
 void _cxl_pci_disable_device(struct pci_dev *dev);
-int _cxl_next_msi_hwirq(struct pci_dev *pdev, struct cxl_context **ctx, int *afu_irq);
 
 struct cxl_calls {
 	void (*cxl_slbia)(struct mm_struct *mm);
 	bool (*cxl_pci_associate_default_context)(struct pci_dev *dev, struct cxl_afu *afu);
 	void (*cxl_pci_disable_device)(struct pci_dev *dev);
-	int (*cxl_next_msi_hwirq)(struct pci_dev *pdev, struct cxl_context **ctx, int *afu_irq);
 
 	struct module *owner;
 };
diff --git a/drivers/misc/cxl/main.c b/drivers/misc/cxl/main.c
index 59a904efd104..a7e83624034b 100644
--- a/drivers/misc/cxl/main.c
+++ b/drivers/misc/cxl/main.c
@@ -106,7 +106,6 @@ static struct cxl_calls cxl_calls = {
 	.cxl_slbia = cxl_slbia_core,
 	.cxl_pci_associate_default_context = _cxl_pci_associate_default_context,
 	.cxl_pci_disable_device = _cxl_pci_disable_device,
-	.cxl_next_msi_hwirq = _cxl_next_msi_hwirq,
 	.owner = THIS_MODULE,
 };
 
diff --git a/include/misc/cxl.h b/include/misc/cxl.h
index 82cc6ffafe2d..6a3711a2e217 100644
--- a/include/misc/cxl.h
+++ b/include/misc/cxl.h
@@ -183,26 +183,6 @@ void cxl_psa_unmap(void __iomem *addr);
 /*  Get the process element for this context */
 int cxl_process_element(struct cxl_context *ctx);
 
-/*
- * Limit the number of interrupts that a single context can allocate via
- * cxl_start_work. If using the api with a real phb, this may be used to
- * request that additional default contexts be created when allocating
- * interrupts via pci_enable_msix_range. These will be set to the same running
- * state as the default context, and if that is running it will reuse the
- * parameters previously passed to cxl_start_context for the default context.
- */
-int cxl_set_max_irqs_per_process(struct pci_dev *dev, int irqs);
-int cxl_get_max_irqs_per_process(struct pci_dev *dev);
-
-/*
- * Use to simultaneously iterate over hardware interrupt numbers, contexts and
- * afu interrupt numbers allocated for the device via pci_enable_msix_range and
- * is a useful convenience function when working with hardware that has
- * limitations on the number of interrupts per process. *ctx and *afu_irq
- * should be NULL and 0 to start the iteration.
- */
-int cxl_next_msi_hwirq(struct pci_dev *pdev, struct cxl_context **ctx, int *afu_irq);
-
 /*
  * These calls allow drivers to create their own file descriptors and make them
  * identical to the cxl file descriptor user API. An example use case:
-- 
2.17.1

Re: [PATCH v2 03/10] Revert "cxl: Add preliminary workaround for CX4 interrupt limitation"

[Andrew Donnellan]

On 28/06/18 20:05, Frederic Barrat wrote:
> From: Alastair D'Silva <alastair@d-silva.org>
> 
> Remove abandonned capi support for the Mellanox CX4.
> 
> This reverts commit cbce0917e2e47d4bf5aa3b5fd6b1247f33e1a126.
> 
> Signed-off-by: Alastair D'Silva <alastair@d-silva.org>

Acked-by: Andrew Donnellan <andrew.donnellan@au1.ibm.com>

> ---
>   drivers/misc/cxl/api.c     | 15 ---------------
>   drivers/misc/cxl/base.c    | 17 -----------------
>   drivers/misc/cxl/context.c |  1 -
>   drivers/misc/cxl/cxl.h     | 10 ----------
>   drivers/misc/cxl/main.c    |  1 -
>   include/misc/cxl.h         | 20 --------------------
>   6 files changed, 64 deletions(-)
> 
> diff --git a/drivers/misc/cxl/api.c b/drivers/misc/cxl/api.c
> index 2e5862b7a074..34ba67bc41bd 100644
> --- a/drivers/misc/cxl/api.c
> +++ b/drivers/misc/cxl/api.c
> @@ -181,21 +181,6 @@ static irq_hw_number_t cxl_find_afu_irq(struct cxl_context *ctx, int num)
>   	return 0;
>   }
>   
> -int _cxl_next_msi_hwirq(struct pci_dev *pdev, struct cxl_context **ctx, int *afu_irq)
> -{
> -	if (*ctx == NULL || *afu_irq == 0) {
> -		*afu_irq = 1;
> -		*ctx = cxl_get_context(pdev);
> -	} else {
> -		(*afu_irq)++;
> -		if (*afu_irq > cxl_get_max_irqs_per_process(pdev)) {
> -			*ctx = list_next_entry(*ctx, extra_irq_contexts);
> -			*afu_irq = 1;
> -		}
> -	}
> -	return cxl_find_afu_irq(*ctx, *afu_irq);
> -}
> -/* Exported via cxl_base */
>   
>   int cxl_set_priv(struct cxl_context *ctx, void *priv)
>   {
> diff --git a/drivers/misc/cxl/base.c b/drivers/misc/cxl/base.c
> index fe90f895bb10..e1e80cb99ad9 100644
> --- a/drivers/misc/cxl/base.c
> +++ b/drivers/misc/cxl/base.c
> @@ -141,23 +141,6 @@ void cxl_pci_disable_device(struct pci_dev *dev)
>   }
>   EXPORT_SYMBOL_GPL(cxl_pci_disable_device);
>   
> -int cxl_next_msi_hwirq(struct pci_dev *pdev, struct cxl_context **ctx, int *afu_irq)
> -{
> -	int ret;
> -	struct cxl_calls *calls;
> -
> -	calls = cxl_calls_get();
> -	if (!calls)
> -		return -EBUSY;
> -
> -	ret = calls->cxl_next_msi_hwirq(pdev, ctx, afu_irq);
> -
> -	cxl_calls_put(calls);
> -
> -	return ret;
> -}
> -EXPORT_SYMBOL_GPL(cxl_next_msi_hwirq);
> -
>   static int __init cxl_base_init(void)
>   {
>   	struct device_node *np;
> diff --git a/drivers/misc/cxl/context.c b/drivers/misc/cxl/context.c
> index c6ec872800a2..0355d42d367f 100644
> --- a/drivers/misc/cxl/context.c
> +++ b/drivers/misc/cxl/context.c
> @@ -74,7 +74,6 @@ int cxl_context_init(struct cxl_context *ctx, struct cxl_afu *afu, bool master)
>   	ctx->pending_afu_err = false;
>   
>   	INIT_LIST_HEAD(&ctx->irq_names);
> -	INIT_LIST_HEAD(&ctx->extra_irq_contexts);
>   
>   	/*
>   	 * When we have to destroy all contexts in cxl_context_detach_all() we
> diff --git a/drivers/misc/cxl/cxl.h b/drivers/misc/cxl/cxl.h
> index 9688fe8b4d80..d95c2c98f2ab 100644
> --- a/drivers/misc/cxl/cxl.h
> +++ b/drivers/misc/cxl/cxl.h
> @@ -623,14 +623,6 @@ struct cxl_context {
>   
>   	struct rcu_head rcu;
>   
> -	/*
> -	 * Only used when more interrupts are allocated via
> -	 * pci_enable_msix_range than are supported in the default context, to
> -	 * use additional contexts to overcome the limitation. i.e. Mellanox
> -	 * CX4 only:
> -	 */
> -	struct list_head extra_irq_contexts;
> -
>   	struct mm_struct *mm;
>   
>   	u16 tidr;
> @@ -878,13 +870,11 @@ ssize_t cxl_pci_afu_read_err_buffer(struct cxl_afu *afu, char *buf,
>   /* Internal functions wrapped in cxl_base to allow PHB to call them */
>   bool _cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu);
>   void _cxl_pci_disable_device(struct pci_dev *dev);
> -int _cxl_next_msi_hwirq(struct pci_dev *pdev, struct cxl_context **ctx, int *afu_irq);
>   
>   struct cxl_calls {
>   	void (*cxl_slbia)(struct mm_struct *mm);
>   	bool (*cxl_pci_associate_default_context)(struct pci_dev *dev, struct cxl_afu *afu);
>   	void (*cxl_pci_disable_device)(struct pci_dev *dev);
> -	int (*cxl_next_msi_hwirq)(struct pci_dev *pdev, struct cxl_context **ctx, int *afu_irq);
>   
>   	struct module *owner;
>   };
> diff --git a/drivers/misc/cxl/main.c b/drivers/misc/cxl/main.c
> index 59a904efd104..a7e83624034b 100644
> --- a/drivers/misc/cxl/main.c
> +++ b/drivers/misc/cxl/main.c
> @@ -106,7 +106,6 @@ static struct cxl_calls cxl_calls = {
>   	.cxl_slbia = cxl_slbia_core,
>   	.cxl_pci_associate_default_context = _cxl_pci_associate_default_context,
>   	.cxl_pci_disable_device = _cxl_pci_disable_device,
> -	.cxl_next_msi_hwirq = _cxl_next_msi_hwirq,
>   	.owner = THIS_MODULE,
>   };
>   
> diff --git a/include/misc/cxl.h b/include/misc/cxl.h
> index 82cc6ffafe2d..6a3711a2e217 100644
> --- a/include/misc/cxl.h
> +++ b/include/misc/cxl.h
> @@ -183,26 +183,6 @@ void cxl_psa_unmap(void __iomem *addr);
>   /*  Get the process element for this context */
>   int cxl_process_element(struct cxl_context *ctx);
>   
> -/*
> - * Limit the number of interrupts that a single context can allocate via
> - * cxl_start_work. If using the api with a real phb, this may be used to
> - * request that additional default contexts be created when allocating
> - * interrupts via pci_enable_msix_range. These will be set to the same running
> - * state as the default context, and if that is running it will reuse the
> - * parameters previously passed to cxl_start_context for the default context.
> - */
> -int cxl_set_max_irqs_per_process(struct pci_dev *dev, int irqs);
> -int cxl_get_max_irqs_per_process(struct pci_dev *dev);
> -
> -/*
> - * Use to simultaneously iterate over hardware interrupt numbers, contexts and
> - * afu interrupt numbers allocated for the device via pci_enable_msix_range and
> - * is a useful convenience function when working with hardware that has
> - * limitations on the number of interrupts per process. *ctx and *afu_irq
> - * should be NULL and 0 to start the iteration.
> - */
> -int cxl_next_msi_hwirq(struct pci_dev *pdev, struct cxl_context **ctx, int *afu_irq);
> -
>   /*
>    * These calls allow drivers to create their own file descriptors and make them
>    * identical to the cxl file descriptor user API. An example use case:
> 

-- 
Andrew Donnellan              OzLabs, ADL Canberra
andrew.donnellan@au1.ibm.com  IBM Australia Limited

[PATCH v2 04/10] Revert "cxl: Add kernel APIs to get & set the max irqs per context"

[Frederic Barrat]

From: Alastair D'Silva <alastair@d-silva.org>

Remove abandonned capi support for the Mellanox CX4.

This reverts commit 79384e4b71240abf50c375eea56060b0d79c242a.

Signed-off-by: Alastair D'Silva <alastair@d-silva.org>
---
 drivers/misc/cxl/api.c | 27 ---------------------------
 1 file changed, 27 deletions(-)

diff --git a/drivers/misc/cxl/api.c b/drivers/misc/cxl/api.c
index 34ba67bc41bd..a535c1e6aa92 100644
--- a/drivers/misc/cxl/api.c
+++ b/drivers/misc/cxl/api.c
@@ -552,30 +552,3 @@ ssize_t cxl_read_adapter_vpd(struct pci_dev *dev, void *buf, size_t count)
 	return cxl_ops->read_adapter_vpd(afu->adapter, buf, count);
 }
 EXPORT_SYMBOL_GPL(cxl_read_adapter_vpd);
-
-int cxl_set_max_irqs_per_process(struct pci_dev *dev, int irqs)
-{
-	struct cxl_afu *afu = cxl_pci_to_afu(dev);
-	if (IS_ERR(afu))
-		return -ENODEV;
-
-	if (irqs > afu->adapter->user_irqs)
-		return -EINVAL;
-
-	/* Limit user_irqs to prevent the user increasing this via sysfs */
-	afu->adapter->user_irqs = irqs;
-	afu->irqs_max = irqs;
-
-	return 0;
-}
-EXPORT_SYMBOL_GPL(cxl_set_max_irqs_per_process);
-
-int cxl_get_max_irqs_per_process(struct pci_dev *dev)
-{
-	struct cxl_afu *afu = cxl_pci_to_afu(dev);
-	if (IS_ERR(afu))
-		return -ENODEV;
-
-	return afu->irqs_max;
-}
-EXPORT_SYMBOL_GPL(cxl_get_max_irqs_per_process);
-- 
2.17.1

Re: [PATCH v2 04/10] Revert "cxl: Add kernel APIs to get & set the max irqs per context"

[Andrew Donnellan]

On 28/06/18 20:05, Frederic Barrat wrote:
> From: Alastair D'Silva <alastair@d-silva.org>
> 
> Remove abandonned capi support for the Mellanox CX4.
> 
> This reverts commit 79384e4b71240abf50c375eea56060b0d79c242a.
> 
> Signed-off-by: Alastair D'Silva <alastair@d-silva.org>

Acked-by: Andrew Donnellan <andrew.donnellan@au1.ibm.com>

> ---
>   drivers/misc/cxl/api.c | 27 ---------------------------
>   1 file changed, 27 deletions(-)
> 
> diff --git a/drivers/misc/cxl/api.c b/drivers/misc/cxl/api.c
> index 34ba67bc41bd..a535c1e6aa92 100644
> --- a/drivers/misc/cxl/api.c
> +++ b/drivers/misc/cxl/api.c
> @@ -552,30 +552,3 @@ ssize_t cxl_read_adapter_vpd(struct pci_dev *dev, void *buf, size_t count)
>   	return cxl_ops->read_adapter_vpd(afu->adapter, buf, count);
>   }
>   EXPORT_SYMBOL_GPL(cxl_read_adapter_vpd);
> -
> -int cxl_set_max_irqs_per_process(struct pci_dev *dev, int irqs)
> -{
> -	struct cxl_afu *afu = cxl_pci_to_afu(dev);
> -	if (IS_ERR(afu))
> -		return -ENODEV;
> -
> -	if (irqs > afu->adapter->user_irqs)
> -		return -EINVAL;
> -
> -	/* Limit user_irqs to prevent the user increasing this via sysfs */
> -	afu->adapter->user_irqs = irqs;
> -	afu->irqs_max = irqs;
> -
> -	return 0;
> -}
> -EXPORT_SYMBOL_GPL(cxl_set_max_irqs_per_process);
> -
> -int cxl_get_max_irqs_per_process(struct pci_dev *dev)
> -{
> -	struct cxl_afu *afu = cxl_pci_to_afu(dev);
> -	if (IS_ERR(afu))
> -		return -ENODEV;
> -
> -	return afu->irqs_max;
> -}
> -EXPORT_SYMBOL_GPL(cxl_get_max_irqs_per_process);
> 

-- 
Andrew Donnellan              OzLabs, ADL Canberra
andrew.donnellan@au1.ibm.com  IBM Australia Limited

[PATCH v2 05/10] Revert "cxl: Add cxl_check_and_switch_mode() API to switch bi-modal cards"

[Frederic Barrat]

From: Alastair D'Silva <alastair@d-silva.org>

Remove abandonned capi support for the Mellanox CX4.

This reverts commit b0b5e5918ad1babfd1d43d98c7281926a7b57b9f.

Signed-off-by: Alastair D'Silva <alastair@d-silva.org>
---
 drivers/misc/cxl/Kconfig |   8 --
 drivers/misc/cxl/pci.c   | 236 +++------------------------------------
 include/misc/cxl.h       |  25 -----
 3 files changed, 18 insertions(+), 251 deletions(-)

diff --git a/drivers/misc/cxl/Kconfig b/drivers/misc/cxl/Kconfig
index 93397cb05b15..3ce933707828 100644
--- a/drivers/misc/cxl/Kconfig
+++ b/drivers/misc/cxl/Kconfig
@@ -33,11 +33,3 @@ config CXL
 	  CAPI adapters are found in POWER8 based systems.
 
 	  If unsure, say N.
-
-config CXL_BIMODAL
-	bool "Support for bi-modal CAPI cards"
-	depends on HOTPLUG_PCI_POWERNV = y && CXL || HOTPLUG_PCI_POWERNV = m && CXL = m
-	default y
-	help
-	  Select this option to enable support for bi-modal CAPI cards, such as
-	  the Mellanox CX-4.
diff --git a/drivers/misc/cxl/pci.c b/drivers/misc/cxl/pci.c
index 429d6de1dde7..9c5a21fee835 100644
--- a/drivers/misc/cxl/pci.c
+++ b/drivers/misc/cxl/pci.c
@@ -55,8 +55,6 @@
 	pci_read_config_byte(dev, vsec + 0xa, dest)
 #define CXL_WRITE_VSEC_MODE_CONTROL(dev, vsec, val) \
 	pci_write_config_byte(dev, vsec + 0xa, val)
-#define CXL_WRITE_VSEC_MODE_CONTROL_BUS(bus, devfn, vsec, val) \
-	pci_bus_write_config_byte(bus, devfn, vsec + 0xa, val)
 #define CXL_VSEC_PROTOCOL_MASK   0xe0
 #define CXL_VSEC_PROTOCOL_1024TB 0x80
 #define CXL_VSEC_PROTOCOL_512TB  0x40
@@ -800,234 +798,36 @@ static int setup_cxl_bars(struct pci_dev *dev)
 	return 0;
 }
 
-#ifdef CONFIG_CXL_BIMODAL
-
-struct cxl_switch_work {
-	struct pci_dev *dev;
-	struct work_struct work;
-	int vsec;
-	int mode;
-};
-
-static void switch_card_to_cxl(struct work_struct *work)
+/* pciex node: ibm,opal-m64-window = <0x3d058 0x0 0x3d058 0x0 0x8 0x0>; */
+static int switch_card_to_cxl(struct pci_dev *dev)
 {
-	struct cxl_switch_work *switch_work =
-		container_of(work, struct cxl_switch_work, work);
-	struct pci_dev *dev = switch_work->dev;
-	struct pci_bus *bus = dev->bus;
-	struct pci_controller *hose = pci_bus_to_host(bus);
-	struct pci_dev *bridge;
-	struct pnv_php_slot *php_slot;
-	unsigned int devfn;
+	int vsec;
 	u8 val;
 	int rc;
 
-	dev_info(&bus->dev, "cxl: Preparing for mode switch...\n");
-	bridge = list_first_entry_or_null(&hose->bus->devices, struct pci_dev,
-					  bus_list);
-	if (!bridge) {
-		dev_WARN(&bus->dev, "cxl: Couldn't find root port!\n");
-		goto err_dev_put;
-	}
+	dev_info(&dev->dev, "switch card to CXL\n");
 
-	php_slot = pnv_php_find_slot(pci_device_to_OF_node(bridge));
-	if (!php_slot) {
-		dev_err(&bus->dev, "cxl: Failed to find slot hotplug "
-			           "information. You may need to upgrade "
-			           "skiboot. Aborting.\n");
-		goto err_dev_put;
-	}
-
-	rc = CXL_READ_VSEC_MODE_CONTROL(dev, switch_work->vsec, &val);
-	if (rc) {
-		dev_err(&bus->dev, "cxl: Failed to read CAPI mode control: %i\n", rc);
-		goto err_dev_put;
-	}
-	devfn = dev->devfn;
-
-	/* Release the reference obtained in cxl_check_and_switch_mode() */
-	pci_dev_put(dev);
-
-	dev_dbg(&bus->dev, "cxl: Removing PCI devices from kernel\n");
-	pci_lock_rescan_remove();
-	pci_hp_remove_devices(bridge->subordinate);
-	pci_unlock_rescan_remove();
-
-	/* Switch the CXL protocol on the card */
-	if (switch_work->mode == CXL_BIMODE_CXL) {
-		dev_info(&bus->dev, "cxl: Switching card to CXL mode\n");
-		val &= ~CXL_VSEC_PROTOCOL_MASK;
-		val |= CXL_VSEC_PROTOCOL_256TB | CXL_VSEC_PROTOCOL_ENABLE;
-		rc = pnv_cxl_enable_phb_kernel_api(hose, true);
-		if (rc) {
-			dev_err(&bus->dev, "cxl: Failed to enable kernel API"
-				           " on real PHB, aborting\n");
-			goto err_free_work;
-		}
-	} else {
-		dev_WARN(&bus->dev, "cxl: Switching card to PCI mode not supported!\n");
-		goto err_free_work;
-	}
-
-	rc = CXL_WRITE_VSEC_MODE_CONTROL_BUS(bus, devfn, switch_work->vsec, val);
-	if (rc) {
-		dev_err(&bus->dev, "cxl: Failed to configure CXL protocol: %i\n", rc);
-		goto err_free_work;
-	}
-
-	/*
-	 * The CAIA spec (v1.1, Section 10.6 Bi-modal Device Support) states
-	 * we must wait 100ms after this mode switch before touching PCIe config
-	 * space.
-	 */
-	msleep(100);
-
-	/*
-	 * Hot reset to cause the card to come back in cxl mode. A
-	 * OPAL_RESET_PCI_LINK would be sufficient, but currently lacks support
-	 * in skiboot, so we use a hot reset instead.
-	 *
-	 * We call pci_set_pcie_reset_state() on the bridge, as a CAPI card is
-	 * guaranteed to sit directly under the root port, and setting the reset
-	 * state on a device directly under the root port is equivalent to doing
-	 * it on the root port iself.
-	 */
-	dev_info(&bus->dev, "cxl: Configuration write complete, resetting card\n");
-	pci_set_pcie_reset_state(bridge, pcie_hot_reset);
-	pci_set_pcie_reset_state(bridge, pcie_deassert_reset);
-
-	dev_dbg(&bus->dev, "cxl: Offlining slot\n");
-	rc = pnv_php_set_slot_power_state(&php_slot->slot, OPAL_PCI_SLOT_OFFLINE);
-	if (rc) {
-		dev_err(&bus->dev, "cxl: OPAL offlining call failed: %i\n", rc);
-		goto err_free_work;
-	}
-
-	dev_dbg(&bus->dev, "cxl: Onlining and probing slot\n");
-	rc = pnv_php_set_slot_power_state(&php_slot->slot, OPAL_PCI_SLOT_ONLINE);
-	if (rc) {
-		dev_err(&bus->dev, "cxl: OPAL onlining call failed: %i\n", rc);
-		goto err_free_work;
-	}
-
-	pci_lock_rescan_remove();
-	pci_hp_add_devices(bridge->subordinate);
-	pci_unlock_rescan_remove();
-
-	dev_info(&bus->dev, "cxl: CAPI mode switch completed\n");
-	kfree(switch_work);
-	return;
-
-err_dev_put:
-	/* Release the reference obtained in cxl_check_and_switch_mode() */
-	pci_dev_put(dev);
-err_free_work:
-	kfree(switch_work);
-}
-
-int cxl_check_and_switch_mode(struct pci_dev *dev, int mode, int vsec)
-{
-	struct cxl_switch_work *work;
-	u8 val;
-	int rc;
-
-	if (!cpu_has_feature(CPU_FTR_HVMODE))
+	if (!(vsec = find_cxl_vsec(dev))) {
+		dev_err(&dev->dev, "ABORTING: CXL VSEC not found!\n");
 		return -ENODEV;
-
-	if (!vsec) {
-		vsec = find_cxl_vsec(dev);
-		if (!vsec) {
-			dev_info(&dev->dev, "CXL VSEC not found\n");
-			return -ENODEV;
-		}
 	}
 
-	rc = CXL_READ_VSEC_MODE_CONTROL(dev, vsec, &val);
-	if (rc) {
-		dev_err(&dev->dev, "Failed to read current mode control: %i", rc);
+	if ((rc = CXL_READ_VSEC_MODE_CONTROL(dev, vsec, &val))) {
+		dev_err(&dev->dev, "failed to read current mode control: %i", rc);
 		return rc;
 	}
-
-	if (mode == CXL_BIMODE_PCI) {
-		if (!(val & CXL_VSEC_PROTOCOL_ENABLE)) {
-			dev_info(&dev->dev, "Card is already in PCI mode\n");
-			return 0;
-		}
-		/*
-		 * TODO: Before it's safe to switch the card back to PCI mode
-		 * we need to disable the CAPP and make sure any cachelines the
-		 * card holds have been flushed out. Needs skiboot support.
-		 */
-		dev_WARN(&dev->dev, "CXL mode switch to PCI unsupported!\n");
-		return -EIO;
-	}
-
-	if (val & CXL_VSEC_PROTOCOL_ENABLE) {
-		dev_info(&dev->dev, "Card is already in CXL mode\n");
-		return 0;
+	val &= ~CXL_VSEC_PROTOCOL_MASK;
+	val |= CXL_VSEC_PROTOCOL_256TB | CXL_VSEC_PROTOCOL_ENABLE;
+	if ((rc = CXL_WRITE_VSEC_MODE_CONTROL(dev, vsec, val))) {
+		dev_err(&dev->dev, "failed to enable CXL protocol: %i", rc);
+		return rc;
 	}
-
-	dev_info(&dev->dev, "Card is in PCI mode, scheduling kernel thread "
-			    "to switch to CXL mode\n");
-
-	work = kmalloc(sizeof(struct cxl_switch_work), GFP_KERNEL);
-	if (!work)
-		return -ENOMEM;
-
-	pci_dev_get(dev);
-	work->dev = dev;
-	work->vsec = vsec;
-	work->mode = mode;
-	INIT_WORK(&work->work, switch_card_to_cxl);
-
-	schedule_work(&work->work);
-
 	/*
-	 * We return a failure now to abort the driver init. Once the
-	 * link has been cycled and the card is in cxl mode we will
-	 * come back (possibly using the generic cxl driver), but
-	 * return success as the card should then be in cxl mode.
-	 *
-	 * TODO: What if the card comes back in PCI mode even after
-	 *       the switch?  Don't want to spin endlessly.
+	 * The CAIA spec (v0.12 11.6 Bi-modal Device Support) states
+	 * we must wait 100ms after this mode switch before touching
+	 * PCIe config space.
 	 */
-	return -EBUSY;
-}
-EXPORT_SYMBOL_GPL(cxl_check_and_switch_mode);
-
-#endif /* CONFIG_CXL_BIMODAL */
-
-static int setup_cxl_protocol_area(struct pci_dev *dev)
-{
-	u8 val;
-	int rc;
-	int vsec = find_cxl_vsec(dev);
-
-	if (!vsec) {
-		dev_info(&dev->dev, "CXL VSEC not found\n");
-		return -ENODEV;
-	}
-
-	rc = CXL_READ_VSEC_MODE_CONTROL(dev, vsec, &val);
-	if (rc) {
-		dev_err(&dev->dev, "Failed to read current mode control: %i\n", rc);
-		return rc;
-	}
-
-	if (!(val & CXL_VSEC_PROTOCOL_ENABLE)) {
-		dev_err(&dev->dev, "Card not in CAPI mode!\n");
-		return -EIO;
-	}
-
-	if ((val & CXL_VSEC_PROTOCOL_MASK) != CXL_VSEC_PROTOCOL_256TB) {
-		val &= ~CXL_VSEC_PROTOCOL_MASK;
-		val |= CXL_VSEC_PROTOCOL_256TB;
-		rc = CXL_WRITE_VSEC_MODE_CONTROL(dev, vsec, val);
-		if (rc) {
-			dev_err(&dev->dev, "Failed to set CXL protocol area: %i\n", rc);
-			return rc;
-		}
-	}
+	msleep(100);
 
 	return 0;
 }
@@ -1724,7 +1524,7 @@ static int cxl_configure_adapter(struct cxl *adapter, struct pci_dev *dev)
 	if ((rc = setup_cxl_bars(dev)))
 		return rc;
 
-	if ((rc = setup_cxl_protocol_area(dev)))
+	if ((rc = switch_card_to_cxl(dev)))
 		return rc;
 
 	if ((rc = cxl_update_image_control(adapter)))
diff --git a/include/misc/cxl.h b/include/misc/cxl.h
index 6a3711a2e217..74da2e440763 100644
--- a/include/misc/cxl.h
+++ b/include/misc/cxl.h
@@ -39,31 +39,6 @@
 bool cxl_slot_is_supported(struct pci_dev *dev, int flags);
 
 
-#define CXL_BIMODE_CXL 1
-#define CXL_BIMODE_PCI 2
-
-/*
- * Check the mode that the given bi-modal CXL adapter is currently in and
- * change it if necessary. This does not apply to AFU drivers.
- *
- * If the mode matches the requested mode this function will return 0 - if the
- * driver was expecting the generic CXL driver to have bound to the adapter and
- * it gets this return value it should fail the probe function to give the CXL
- * driver a chance to probe it.
- *
- * If the mode does not match it will start a background task to unplug the
- * device from Linux and switch its mode, and will return -EBUSY. At this
- * point the calling driver should make sure it has released the device and
- * fail its probe function.
- *
- * The offset of the CXL VSEC can be provided to this function. If 0 is passed,
- * this function will search for a CXL VSEC with ID 0x1280 and return -ENODEV
- * if it is not found.
- */
-#ifdef CONFIG_CXL_BIMODAL
-int cxl_check_and_switch_mode(struct pci_dev *dev, int mode, int vsec);
-#endif
-
 /* Get the AFU associated with a pci_dev */
 struct cxl_afu *cxl_pci_to_afu(struct pci_dev *dev);
 
-- 
2.17.1

Re: [PATCH v2 05/10] Revert "cxl: Add cxl_check_and_switch_mode() API to switch bi-modal cards"

[Andrew Donnellan]

On 28/06/18 20:05, Frederic Barrat wrote:
> From: Alastair D'Silva <alastair@d-silva.org>
> 
> Remove abandonned capi support for the Mellanox CX4.
> 
> This reverts commit b0b5e5918ad1babfd1d43d98c7281926a7b57b9f.
> 
> Signed-off-by: Alastair D'Silva <alastair@d-silva.org>

I was kinda proud at how dodgy this was and yet how it actually worked...

(Hmm, I should go back and see if there's anything we can rip out of 
pnv_php now...)

Acked-by: Andrew Donnellan <andrew.donnellan@au1.ibm.com>

> ---
>   drivers/misc/cxl/Kconfig |   8 --
>   drivers/misc/cxl/pci.c   | 236 +++------------------------------------
>   include/misc/cxl.h       |  25 -----
>   3 files changed, 18 insertions(+), 251 deletions(-)
> 
> diff --git a/drivers/misc/cxl/Kconfig b/drivers/misc/cxl/Kconfig
> index 93397cb05b15..3ce933707828 100644
> --- a/drivers/misc/cxl/Kconfig
> +++ b/drivers/misc/cxl/Kconfig
> @@ -33,11 +33,3 @@ config CXL
>   	  CAPI adapters are found in POWER8 based systems.
>   
>   	  If unsure, say N.
> -
> -config CXL_BIMODAL
> -	bool "Support for bi-modal CAPI cards"
> -	depends on HOTPLUG_PCI_POWERNV = y && CXL || HOTPLUG_PCI_POWERNV = m && CXL = m
> -	default y
> -	help
> -	  Select this option to enable support for bi-modal CAPI cards, such as
> -	  the Mellanox CX-4.
> diff --git a/drivers/misc/cxl/pci.c b/drivers/misc/cxl/pci.c
> index 429d6de1dde7..9c5a21fee835 100644
> --- a/drivers/misc/cxl/pci.c
> +++ b/drivers/misc/cxl/pci.c
> @@ -55,8 +55,6 @@
>   	pci_read_config_byte(dev, vsec + 0xa, dest)
>   #define CXL_WRITE_VSEC_MODE_CONTROL(dev, vsec, val) \
>   	pci_write_config_byte(dev, vsec + 0xa, val)
> -#define CXL_WRITE_VSEC_MODE_CONTROL_BUS(bus, devfn, vsec, val) \
> -	pci_bus_write_config_byte(bus, devfn, vsec + 0xa, val)
>   #define CXL_VSEC_PROTOCOL_MASK   0xe0
>   #define CXL_VSEC_PROTOCOL_1024TB 0x80
>   #define CXL_VSEC_PROTOCOL_512TB  0x40
> @@ -800,234 +798,36 @@ static int setup_cxl_bars(struct pci_dev *dev)
>   	return 0;
>   }
>   
> -#ifdef CONFIG_CXL_BIMODAL
> -
> -struct cxl_switch_work {
> -	struct pci_dev *dev;
> -	struct work_struct work;
> -	int vsec;
> -	int mode;
> -};
> -
> -static void switch_card_to_cxl(struct work_struct *work)
> +/* pciex node: ibm,opal-m64-window = <0x3d058 0x0 0x3d058 0x0 0x8 0x0>; */
> +static int switch_card_to_cxl(struct pci_dev *dev)
>   {
> -	struct cxl_switch_work *switch_work =
> -		container_of(work, struct cxl_switch_work, work);
> -	struct pci_dev *dev = switch_work->dev;
> -	struct pci_bus *bus = dev->bus;
> -	struct pci_controller *hose = pci_bus_to_host(bus);
> -	struct pci_dev *bridge;
> -	struct pnv_php_slot *php_slot;
> -	unsigned int devfn;
> +	int vsec;
>   	u8 val;
>   	int rc;
>   
> -	dev_info(&bus->dev, "cxl: Preparing for mode switch...\n");
> -	bridge = list_first_entry_or_null(&hose->bus->devices, struct pci_dev,
> -					  bus_list);
> -	if (!bridge) {
> -		dev_WARN(&bus->dev, "cxl: Couldn't find root port!\n");
> -		goto err_dev_put;
> -	}
> +	dev_info(&dev->dev, "switch card to CXL\n");
>   
> -	php_slot = pnv_php_find_slot(pci_device_to_OF_node(bridge));
> -	if (!php_slot) {
> -		dev_err(&bus->dev, "cxl: Failed to find slot hotplug "
> -			           "information. You may need to upgrade "
> -			           "skiboot. Aborting.\n");
> -		goto err_dev_put;
> -	}
> -
> -	rc = CXL_READ_VSEC_MODE_CONTROL(dev, switch_work->vsec, &val);
> -	if (rc) {
> -		dev_err(&bus->dev, "cxl: Failed to read CAPI mode control: %i\n", rc);
> -		goto err_dev_put;
> -	}
> -	devfn = dev->devfn;
> -
> -	/* Release the reference obtained in cxl_check_and_switch_mode() */
> -	pci_dev_put(dev);
> -
> -	dev_dbg(&bus->dev, "cxl: Removing PCI devices from kernel\n");
> -	pci_lock_rescan_remove();
> -	pci_hp_remove_devices(bridge->subordinate);
> -	pci_unlock_rescan_remove();
> -
> -	/* Switch the CXL protocol on the card */
> -	if (switch_work->mode == CXL_BIMODE_CXL) {
> -		dev_info(&bus->dev, "cxl: Switching card to CXL mode\n");
> -		val &= ~CXL_VSEC_PROTOCOL_MASK;
> -		val |= CXL_VSEC_PROTOCOL_256TB | CXL_VSEC_PROTOCOL_ENABLE;
> -		rc = pnv_cxl_enable_phb_kernel_api(hose, true);
> -		if (rc) {
> -			dev_err(&bus->dev, "cxl: Failed to enable kernel API"
> -				           " on real PHB, aborting\n");
> -			goto err_free_work;
> -		}
> -	} else {
> -		dev_WARN(&bus->dev, "cxl: Switching card to PCI mode not supported!\n");
> -		goto err_free_work;
> -	}
> -
> -	rc = CXL_WRITE_VSEC_MODE_CONTROL_BUS(bus, devfn, switch_work->vsec, val);
> -	if (rc) {
> -		dev_err(&bus->dev, "cxl: Failed to configure CXL protocol: %i\n", rc);
> -		goto err_free_work;
> -	}
> -
> -	/*
> -	 * The CAIA spec (v1.1, Section 10.6 Bi-modal Device Support) states
> -	 * we must wait 100ms after this mode switch before touching PCIe config
> -	 * space.
> -	 */
> -	msleep(100);
> -
> -	/*
> -	 * Hot reset to cause the card to come back in cxl mode. A
> -	 * OPAL_RESET_PCI_LINK would be sufficient, but currently lacks support
> -	 * in skiboot, so we use a hot reset instead.
> -	 *
> -	 * We call pci_set_pcie_reset_state() on the bridge, as a CAPI card is
> -	 * guaranteed to sit directly under the root port, and setting the reset
> -	 * state on a device directly under the root port is equivalent to doing
> -	 * it on the root port iself.
> -	 */
> -	dev_info(&bus->dev, "cxl: Configuration write complete, resetting card\n");
> -	pci_set_pcie_reset_state(bridge, pcie_hot_reset);
> -	pci_set_pcie_reset_state(bridge, pcie_deassert_reset);
> -
> -	dev_dbg(&bus->dev, "cxl: Offlining slot\n");
> -	rc = pnv_php_set_slot_power_state(&php_slot->slot, OPAL_PCI_SLOT_OFFLINE);
> -	if (rc) {
> -		dev_err(&bus->dev, "cxl: OPAL offlining call failed: %i\n", rc);
> -		goto err_free_work;
> -	}
> -
> -	dev_dbg(&bus->dev, "cxl: Onlining and probing slot\n");
> -	rc = pnv_php_set_slot_power_state(&php_slot->slot, OPAL_PCI_SLOT_ONLINE);
> -	if (rc) {
> -		dev_err(&bus->dev, "cxl: OPAL onlining call failed: %i\n", rc);
> -		goto err_free_work;
> -	}
> -
> -	pci_lock_rescan_remove();
> -	pci_hp_add_devices(bridge->subordinate);
> -	pci_unlock_rescan_remove();
> -
> -	dev_info(&bus->dev, "cxl: CAPI mode switch completed\n");
> -	kfree(switch_work);
> -	return;
> -
> -err_dev_put:
> -	/* Release the reference obtained in cxl_check_and_switch_mode() */
> -	pci_dev_put(dev);
> -err_free_work:
> -	kfree(switch_work);
> -}
> -
> -int cxl_check_and_switch_mode(struct pci_dev *dev, int mode, int vsec)
> -{
> -	struct cxl_switch_work *work;
> -	u8 val;
> -	int rc;
> -
> -	if (!cpu_has_feature(CPU_FTR_HVMODE))
> +	if (!(vsec = find_cxl_vsec(dev))) {
> +		dev_err(&dev->dev, "ABORTING: CXL VSEC not found!\n");
>   		return -ENODEV;
> -
> -	if (!vsec) {
> -		vsec = find_cxl_vsec(dev);
> -		if (!vsec) {
> -			dev_info(&dev->dev, "CXL VSEC not found\n");
> -			return -ENODEV;
> -		}
>   	}
>   
> -	rc = CXL_READ_VSEC_MODE_CONTROL(dev, vsec, &val);
> -	if (rc) {
> -		dev_err(&dev->dev, "Failed to read current mode control: %i", rc);
> +	if ((rc = CXL_READ_VSEC_MODE_CONTROL(dev, vsec, &val))) {
> +		dev_err(&dev->dev, "failed to read current mode control: %i", rc);
>   		return rc;
>   	}
> -
> -	if (mode == CXL_BIMODE_PCI) {
> -		if (!(val & CXL_VSEC_PROTOCOL_ENABLE)) {
> -			dev_info(&dev->dev, "Card is already in PCI mode\n");
> -			return 0;
> -		}
> -		/*
> -		 * TODO: Before it's safe to switch the card back to PCI mode
> -		 * we need to disable the CAPP and make sure any cachelines the
> -		 * card holds have been flushed out. Needs skiboot support.
> -		 */
> -		dev_WARN(&dev->dev, "CXL mode switch to PCI unsupported!\n");
> -		return -EIO;
> -	}
> -
> -	if (val & CXL_VSEC_PROTOCOL_ENABLE) {
> -		dev_info(&dev->dev, "Card is already in CXL mode\n");
> -		return 0;
> +	val &= ~CXL_VSEC_PROTOCOL_MASK;
> +	val |= CXL_VSEC_PROTOCOL_256TB | CXL_VSEC_PROTOCOL_ENABLE;
> +	if ((rc = CXL_WRITE_VSEC_MODE_CONTROL(dev, vsec, val))) {
> +		dev_err(&dev->dev, "failed to enable CXL protocol: %i", rc);
> +		return rc;
>   	}
> -
> -	dev_info(&dev->dev, "Card is in PCI mode, scheduling kernel thread "
> -			    "to switch to CXL mode\n");
> -
> -	work = kmalloc(sizeof(struct cxl_switch_work), GFP_KERNEL);
> -	if (!work)
> -		return -ENOMEM;
> -
> -	pci_dev_get(dev);
> -	work->dev = dev;
> -	work->vsec = vsec;
> -	work->mode = mode;
> -	INIT_WORK(&work->work, switch_card_to_cxl);
> -
> -	schedule_work(&work->work);
> -
>   	/*
> -	 * We return a failure now to abort the driver init. Once the
> -	 * link has been cycled and the card is in cxl mode we will
> -	 * come back (possibly using the generic cxl driver), but
> -	 * return success as the card should then be in cxl mode.
> -	 *
> -	 * TODO: What if the card comes back in PCI mode even after
> -	 *       the switch?  Don't want to spin endlessly.
> +	 * The CAIA spec (v0.12 11.6 Bi-modal Device Support) states
> +	 * we must wait 100ms after this mode switch before touching
> +	 * PCIe config space.
>   	 */
> -	return -EBUSY;
> -}
> -EXPORT_SYMBOL_GPL(cxl_check_and_switch_mode);
> -
> -#endif /* CONFIG_CXL_BIMODAL */
> -
> -static int setup_cxl_protocol_area(struct pci_dev *dev)
> -{
> -	u8 val;
> -	int rc;
> -	int vsec = find_cxl_vsec(dev);
> -
> -	if (!vsec) {
> -		dev_info(&dev->dev, "CXL VSEC not found\n");
> -		return -ENODEV;
> -	}
> -
> -	rc = CXL_READ_VSEC_MODE_CONTROL(dev, vsec, &val);
> -	if (rc) {
> -		dev_err(&dev->dev, "Failed to read current mode control: %i\n", rc);
> -		return rc;
> -	}
> -
> -	if (!(val & CXL_VSEC_PROTOCOL_ENABLE)) {
> -		dev_err(&dev->dev, "Card not in CAPI mode!\n");
> -		return -EIO;
> -	}
> -
> -	if ((val & CXL_VSEC_PROTOCOL_MASK) != CXL_VSEC_PROTOCOL_256TB) {
> -		val &= ~CXL_VSEC_PROTOCOL_MASK;
> -		val |= CXL_VSEC_PROTOCOL_256TB;
> -		rc = CXL_WRITE_VSEC_MODE_CONTROL(dev, vsec, val);
> -		if (rc) {
> -			dev_err(&dev->dev, "Failed to set CXL protocol area: %i\n", rc);
> -			return rc;
> -		}
> -	}
> +	msleep(100);
>   
>   	return 0;
>   }
> @@ -1724,7 +1524,7 @@ static int cxl_configure_adapter(struct cxl *adapter, struct pci_dev *dev)
>   	if ((rc = setup_cxl_bars(dev)))
>   		return rc;
>   
> -	if ((rc = setup_cxl_protocol_area(dev)))
> +	if ((rc = switch_card_to_cxl(dev)))
>   		return rc;
>   
>   	if ((rc = cxl_update_image_control(adapter)))
> diff --git a/include/misc/cxl.h b/include/misc/cxl.h
> index 6a3711a2e217..74da2e440763 100644
> --- a/include/misc/cxl.h
> +++ b/include/misc/cxl.h
> @@ -39,31 +39,6 @@
>   bool cxl_slot_is_supported(struct pci_dev *dev, int flags);
>   
>   
> -#define CXL_BIMODE_CXL 1
> -#define CXL_BIMODE_PCI 2
> -
> -/*
> - * Check the mode that the given bi-modal CXL adapter is currently in and
> - * change it if necessary. This does not apply to AFU drivers.
> - *
> - * If the mode matches the requested mode this function will return 0 - if the
> - * driver was expecting the generic CXL driver to have bound to the adapter and
> - * it gets this return value it should fail the probe function to give the CXL
> - * driver a chance to probe it.
> - *
> - * If the mode does not match it will start a background task to unplug the
> - * device from Linux and switch its mode, and will return -EBUSY. At this
> - * point the calling driver should make sure it has released the device and
> - * fail its probe function.
> - *
> - * The offset of the CXL VSEC can be provided to this function. If 0 is passed,
> - * this function will search for a CXL VSEC with ID 0x1280 and return -ENODEV
> - * if it is not found.
> - */
> -#ifdef CONFIG_CXL_BIMODAL
> -int cxl_check_and_switch_mode(struct pci_dev *dev, int mode, int vsec);
> -#endif
> -
>   /* Get the AFU associated with a pci_dev */
>   struct cxl_afu *cxl_pci_to_afu(struct pci_dev *dev);
>   
> 

-- 
Andrew Donnellan              OzLabs, ADL Canberra
andrew.donnellan@au1.ibm.com  IBM Australia Limited

[PATCH v2 06/10] Revert "cxl: Add support for using the kernel API with a real PHB"

[Frederic Barrat]

From: Alastair D'Silva <alastair@d-silva.org>

Remove abandonned capi support for the Mellanox CX4.

This reverts commit 317f5ef1b363417b6f1e93b90dfd2ffd6be6e867.

Signed-off-by: Alastair D'Silva <alastair@d-silva.org>
---
 drivers/misc/cxl/pci.c  |  3 ---
 drivers/misc/cxl/vphb.c | 16 ++--------------
 2 files changed, 2 insertions(+), 17 deletions(-)

diff --git a/drivers/misc/cxl/pci.c b/drivers/misc/cxl/pci.c
index 9c5a21fee835..193ff22f610b 100644
--- a/drivers/misc/cxl/pci.c
+++ b/drivers/misc/cxl/pci.c
@@ -1886,9 +1886,6 @@ static int cxl_probe(struct pci_dev *dev, const struct pci_device_id *id)
 			dev_err(&dev->dev, "AFU %i failed to start: %i\n", slice, rc);
 	}
 
-	if (pnv_pci_on_cxl_phb(dev) && adapter->slices >= 1)
-		pnv_cxl_phb_set_peer_afu(dev, adapter->afu[0]);
-
 	return 0;
 }
 
diff --git a/drivers/misc/cxl/vphb.c b/drivers/misc/cxl/vphb.c
index 7fd0bdc1436a..1a99c9c7a6fb 100644
--- a/drivers/misc/cxl/vphb.c
+++ b/drivers/misc/cxl/vphb.c
@@ -9,7 +9,6 @@
 
 #include <linux/pci.h>
 #include <misc/cxl.h>
-#include <asm/pnv-pci.h>
 #include "cxl.h"
 
 static int cxl_dma_set_mask(struct pci_dev *pdev, u64 dma_mask)
@@ -284,18 +283,13 @@ void cxl_pci_vphb_remove(struct cxl_afu *afu)
 	 */
 }
 
-static bool _cxl_pci_is_vphb_device(struct pci_controller *phb)
-{
-	return (phb->ops == &cxl_pcie_pci_ops);
-}
-
 bool cxl_pci_is_vphb_device(struct pci_dev *dev)
 {
 	struct pci_controller *phb;
 
 	phb = pci_bus_to_host(dev->bus);
 
-	return _cxl_pci_is_vphb_device(phb);
+	return (phb->ops == &cxl_pcie_pci_ops);
 }
 
 struct cxl_afu *cxl_pci_to_afu(struct pci_dev *dev)
@@ -304,13 +298,7 @@ struct cxl_afu *cxl_pci_to_afu(struct pci_dev *dev)
 
 	phb = pci_bus_to_host(dev->bus);
 
-	if (_cxl_pci_is_vphb_device(phb))
-		return (struct cxl_afu *)phb->private_data;
-
-	if (pnv_pci_on_cxl_phb(dev))
-		return pnv_cxl_phb_to_afu(phb);
-
-	return ERR_PTR(-ENODEV);
+	return (struct cxl_afu *)phb->private_data;
 }
 EXPORT_SYMBOL_GPL(cxl_pci_to_afu);
 
-- 
2.17.1

Re: [PATCH v2 06/10] Revert "cxl: Add support for using the kernel API with a real PHB"

[Andrew Donnellan]

On 28/06/18 20:05, Frederic Barrat wrote:
> From: Alastair D'Silva <alastair@d-silva.org>
> 
> Remove abandonned capi support for the Mellanox CX4.
> 
> This reverts commit 317f5ef1b363417b6f1e93b90dfd2ffd6be6e867.
> 
> Signed-off-by: Alastair D'Silva <alastair@d-silva.org>

Acked-by: Andrew Donnellan <andrew.donnellan@au1.ibm.com>

> ---
>   drivers/misc/cxl/pci.c  |  3 ---
>   drivers/misc/cxl/vphb.c | 16 ++--------------
>   2 files changed, 2 insertions(+), 17 deletions(-)
> 
> diff --git a/drivers/misc/cxl/pci.c b/drivers/misc/cxl/pci.c
> index 9c5a21fee835..193ff22f610b 100644
> --- a/drivers/misc/cxl/pci.c
> +++ b/drivers/misc/cxl/pci.c
> @@ -1886,9 +1886,6 @@ static int cxl_probe(struct pci_dev *dev, const struct pci_device_id *id)
>   			dev_err(&dev->dev, "AFU %i failed to start: %i\n", slice, rc);
>   	}
>   
> -	if (pnv_pci_on_cxl_phb(dev) && adapter->slices >= 1)
> -		pnv_cxl_phb_set_peer_afu(dev, adapter->afu[0]);
> -
>   	return 0;
>   }
>   
> diff --git a/drivers/misc/cxl/vphb.c b/drivers/misc/cxl/vphb.c
> index 7fd0bdc1436a..1a99c9c7a6fb 100644
> --- a/drivers/misc/cxl/vphb.c
> +++ b/drivers/misc/cxl/vphb.c
> @@ -9,7 +9,6 @@
>   
>   #include <linux/pci.h>
>   #include <misc/cxl.h>
> -#include <asm/pnv-pci.h>
>   #include "cxl.h"
>   
>   static int cxl_dma_set_mask(struct pci_dev *pdev, u64 dma_mask)
> @@ -284,18 +283,13 @@ void cxl_pci_vphb_remove(struct cxl_afu *afu)
>   	 */
>   }
>   
> -static bool _cxl_pci_is_vphb_device(struct pci_controller *phb)
> -{
> -	return (phb->ops == &cxl_pcie_pci_ops);
> -}
> -
>   bool cxl_pci_is_vphb_device(struct pci_dev *dev)
>   {
>   	struct pci_controller *phb;
>   
>   	phb = pci_bus_to_host(dev->bus);
>   
> -	return _cxl_pci_is_vphb_device(phb);
> +	return (phb->ops == &cxl_pcie_pci_ops);
>   }
>   
>   struct cxl_afu *cxl_pci_to_afu(struct pci_dev *dev)
> @@ -304,13 +298,7 @@ struct cxl_afu *cxl_pci_to_afu(struct pci_dev *dev)
>   
>   	phb = pci_bus_to_host(dev->bus);
>   
> -	if (_cxl_pci_is_vphb_device(phb))
> -		return (struct cxl_afu *)phb->private_data;
> -
> -	if (pnv_pci_on_cxl_phb(dev))
> -		return pnv_cxl_phb_to_afu(phb);
> -
> -	return ERR_PTR(-ENODEV);
> +	return (struct cxl_afu *)phb->private_data;
>   }
>   EXPORT_SYMBOL_GPL(cxl_pci_to_afu);
>   
> 

-- 
Andrew Donnellan              OzLabs, ADL Canberra
andrew.donnellan@au1.ibm.com  IBM Australia Limited

[PATCH v2 07/10] Revert "powerpc/powernv: Add support for the cxl kernel api on the real phb"

[Frederic Barrat]

From: Alastair D'Silva <alastair@d-silva.org>

Remove abandonned capi support for the Mellanox CX4.

This reverts commit 4361b03430d685610e5feea3ec7846e8b9ae795f.

Signed-off-by: Alastair D'Silva <alastair@d-silva.org>
---
 arch/powerpc/include/asm/pnv-pci.h        |   7 --
 arch/powerpc/platforms/powernv/pci-cxl.c  | 115 ----------------------
 arch/powerpc/platforms/powernv/pci-ioda.c |  18 +---
 arch/powerpc/platforms/powernv/pci.h      |  13 ---
 4 files changed, 1 insertion(+), 152 deletions(-)

diff --git a/arch/powerpc/include/asm/pnv-pci.h b/arch/powerpc/include/asm/pnv-pci.h
index d2d8c28db336..7f627e3f4da4 100644
--- a/arch/powerpc/include/asm/pnv-pci.h
+++ b/arch/powerpc/include/asm/pnv-pci.h
@@ -50,13 +50,6 @@ int pnv_cxl_alloc_hwirq_ranges(struct cxl_irq_ranges *irqs,
 			       struct pci_dev *dev, int num);
 void pnv_cxl_release_hwirq_ranges(struct cxl_irq_ranges *irqs,
 				  struct pci_dev *dev);
-
-/* Support for the cxl kernel api on the real PHB (instead of vPHB) */
-int pnv_cxl_enable_phb_kernel_api(struct pci_controller *hose, bool enable);
-bool pnv_pci_on_cxl_phb(struct pci_dev *dev);
-struct cxl_afu *pnv_cxl_phb_to_afu(struct pci_controller *hose);
-void pnv_cxl_phb_set_peer_afu(struct pci_dev *dev, struct cxl_afu *afu);
-
 #endif
 
 struct pnv_php_slot {
diff --git a/arch/powerpc/platforms/powernv/pci-cxl.c b/arch/powerpc/platforms/powernv/pci-cxl.c
index c447b7f03c09..1b18111453d7 100644
--- a/arch/powerpc/platforms/powernv/pci-cxl.c
+++ b/arch/powerpc/platforms/powernv/pci-cxl.c
@@ -8,10 +8,8 @@
  */
 
 #include <linux/module.h>
-#include <asm/pci-bridge.h>
 #include <asm/pnv-pci.h>
 #include <asm/opal.h>
-#include <misc/cxl.h>
 
 #include "pci.h"
 
@@ -178,116 +176,3 @@ static inline int get_cxl_module(void)
 #else
 static inline int get_cxl_module(void) { return 0; }
 #endif
-
-/*
- * Sets flags and switches the controller ops to enable the cxl kernel api.
- * Originally the cxl kernel API operated on a virtual PHB, but certain cards
- * such as the Mellanox CX4 use a peer model instead and for these cards the
- * cxl kernel api will operate on the real PHB.
- */
-int pnv_cxl_enable_phb_kernel_api(struct pci_controller *hose, bool enable)
-{
-	struct pnv_phb *phb = hose->private_data;
-	int rc;
-
-	if (!enable) {
-		/*
-		 * Once cxl mode is enabled on the PHB, there is currently no
-		 * known safe method to disable it again, and trying risks a
-		 * checkstop. If we can find a way to safely disable cxl mode
-		 * in the future we can revisit this, but for now the only sane
-		 * thing to do is to refuse to disable cxl mode:
-		 */
-		return -EPERM;
-	}
-
-	/*
-	 * Hold a reference to the cxl module since several PHB operations now
-	 * depend on it, and it would be insane to allow it to be removed so
-	 * long as we are in this mode (and since we can't safely disable this
-	 * mode once enabled...).
-	 */
-	rc = get_cxl_module();
-	if (rc)
-		return rc;
-
-	phb->flags |= PNV_PHB_FLAG_CXL;
-	hose->controller_ops = pnv_cxl_cx4_ioda_controller_ops;
-
-	return 0;
-}
-EXPORT_SYMBOL_GPL(pnv_cxl_enable_phb_kernel_api);
-
-bool pnv_pci_on_cxl_phb(struct pci_dev *dev)
-{
-	struct pci_controller *hose = pci_bus_to_host(dev->bus);
-	struct pnv_phb *phb = hose->private_data;
-
-	return !!(phb->flags & PNV_PHB_FLAG_CXL);
-}
-EXPORT_SYMBOL_GPL(pnv_pci_on_cxl_phb);
-
-struct cxl_afu *pnv_cxl_phb_to_afu(struct pci_controller *hose)
-{
-	struct pnv_phb *phb = hose->private_data;
-
-	return (struct cxl_afu *)phb->cxl_afu;
-}
-EXPORT_SYMBOL_GPL(pnv_cxl_phb_to_afu);
-
-void pnv_cxl_phb_set_peer_afu(struct pci_dev *dev, struct cxl_afu *afu)
-{
-	struct pci_controller *hose = pci_bus_to_host(dev->bus);
-	struct pnv_phb *phb = hose->private_data;
-
-	phb->cxl_afu = afu;
-}
-EXPORT_SYMBOL_GPL(pnv_cxl_phb_set_peer_afu);
-
-/*
- * In the peer cxl model, the XSL/PSL is physical function 0, and will be used
- * by other functions on the device for memory access and interrupts. When the
- * other functions are enabled we explicitly take a reference on the cxl
- * function since they will use it, and allocate a default context associated
- * with that function just like the vPHB model of the cxl kernel API.
- */
-bool pnv_cxl_enable_device_hook(struct pci_dev *dev)
-{
-	struct pci_controller *hose = pci_bus_to_host(dev->bus);
-	struct pnv_phb *phb = hose->private_data;
-	struct cxl_afu *afu = phb->cxl_afu;
-
-	if (!pnv_pci_enable_device_hook(dev))
-		return false;
-
-
-	/* No special handling for the cxl function, which is always PF 0 */
-	if (PCI_FUNC(dev->devfn) == 0)
-		return true;
-
-	if (!afu) {
-		dev_WARN(&dev->dev, "Attempted to enable function > 0 on CXL PHB without a peer AFU\n");
-		return false;
-	}
-
-	dev_info(&dev->dev, "Enabling function on CXL enabled PHB with peer AFU\n");
-
-	/* Make sure the peer AFU can't go away while this device is active */
-	cxl_afu_get(afu);
-
-	return cxl_pci_associate_default_context(dev, afu);
-}
-
-void pnv_cxl_disable_device(struct pci_dev *dev)
-{
-	struct pci_controller *hose = pci_bus_to_host(dev->bus);
-	struct pnv_phb *phb = hose->private_data;
-	struct cxl_afu *afu = phb->cxl_afu;
-
-	/* No special handling for cxl function: */
-	if (PCI_FUNC(dev->devfn) == 0)
-		return;
-
-	cxl_pci_disable_device(dev);
-	cxl_afu_put(afu);
-}
diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
index 41f8f0ff4a55..770c67c4e8f7 100644
--- a/arch/powerpc/platforms/powernv/pci-ioda.c
+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
@@ -3575,7 +3575,7 @@ static resource_size_t pnv_pci_iov_resource_alignment(struct pci_dev *pdev,
 /* Prevent enabling devices for which we couldn't properly
  * assign a PE
  */
-bool pnv_pci_enable_device_hook(struct pci_dev *dev)
+static bool pnv_pci_enable_device_hook(struct pci_dev *dev)
 {
 	struct pci_controller *hose = pci_bus_to_host(dev->bus);
 	struct pnv_phb *phb = hose->private_data;
@@ -3843,22 +3843,6 @@ static const struct pci_controller_ops pnv_npu_ocapi_ioda_controller_ops = {
 	.shutdown		= pnv_pci_ioda_shutdown,
 };
 
-#ifdef CONFIG_CXL_BASE
-const struct pci_controller_ops pnv_cxl_cx4_ioda_controller_ops = {
-	.dma_dev_setup		= pnv_pci_dma_dev_setup,
-	.dma_bus_setup		= pnv_pci_dma_bus_setup,
-	.enable_device_hook	= pnv_cxl_enable_device_hook,
-	.disable_device		= pnv_cxl_disable_device,
-	.release_device		= pnv_pci_release_device,
-	.window_alignment	= pnv_pci_window_alignment,
-	.setup_bridge		= pnv_pci_setup_bridge,
-	.reset_secondary_bus	= pnv_pci_reset_secondary_bus,
-	.dma_set_mask		= pnv_pci_ioda_dma_set_mask,
-	.dma_get_required_mask	= pnv_pci_ioda_dma_get_required_mask,
-	.shutdown		= pnv_pci_ioda_shutdown,
-};
-#endif
-
 static void __init pnv_pci_init_ioda_phb(struct device_node *np,
 					 u64 hub_id, int ioda_type)
 {
diff --git a/arch/powerpc/platforms/powernv/pci.h b/arch/powerpc/platforms/powernv/pci.h
index ba41913c7e21..44dfbc37f547 100644
--- a/arch/powerpc/platforms/powernv/pci.h
+++ b/arch/powerpc/platforms/powernv/pci.h
@@ -88,7 +88,6 @@ struct pnv_ioda_pe {
 };
 
 #define PNV_PHB_FLAG_EEH	(1 << 0)
-#define PNV_PHB_FLAG_CXL	(1 << 1) /* Real PHB supporting the cxl kernel API */
 
 struct pnv_phb {
 	struct pci_controller	*hose;
@@ -194,9 +193,6 @@ struct pnv_phb {
 		bool nmmu_flush;
 	} npu;
 
-#ifdef CONFIG_CXL_BASE
-	struct cxl_afu *cxl_afu;
-#endif
 	int p2p_target_count;
 };
 
@@ -238,7 +234,6 @@ extern int pnv_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type);
 extern void pnv_teardown_msi_irqs(struct pci_dev *pdev);
 extern struct pnv_ioda_pe *pnv_ioda_get_pe(struct pci_dev *dev);
 extern void pnv_set_msi_irq_chip(struct pnv_phb *phb, unsigned int virq);
-extern bool pnv_pci_enable_device_hook(struct pci_dev *dev);
 extern void pnv_pci_ioda2_set_bypass(struct pnv_ioda_pe *pe, bool enable);
 extern int pnv_eeh_post_init(void);
 
@@ -262,12 +257,4 @@ extern void pnv_npu_take_ownership(struct pnv_ioda_pe *npe);
 extern void pnv_npu_release_ownership(struct pnv_ioda_pe *npe);
 extern int pnv_npu2_init(struct pnv_phb *phb);
 
-/* cxl functions */
-extern bool pnv_cxl_enable_device_hook(struct pci_dev *dev);
-extern void pnv_cxl_disable_device(struct pci_dev *dev);
-
-
-/* phb ops (cxl switches these when enabling the kernel api on the phb) */
-extern const struct pci_controller_ops pnv_cxl_cx4_ioda_controller_ops;
-
 #endif /* __POWERNV_PCI_H */
-- 
2.17.1

Re: [PATCH v2 07/10] Revert "powerpc/powernv: Add support for the cxl kernel api on the real phb"

[Andrew Donnellan]

On 28/06/18 20:05, Frederic Barrat wrote:
> From: Alastair D'Silva <alastair@d-silva.org>
> 
> Remove abandonned capi support for the Mellanox CX4.
> 
> This reverts commit 4361b03430d685610e5feea3ec7846e8b9ae795f.
> 
> Signed-off-by: Alastair D'Silva <alastair@d-silva.org>

Acked-by: Andrew Donnellan <andrew.donnellan@au1.ibm.com>

> ---
>   arch/powerpc/include/asm/pnv-pci.h        |   7 --
>   arch/powerpc/platforms/powernv/pci-cxl.c  | 115 ----------------------
>   arch/powerpc/platforms/powernv/pci-ioda.c |  18 +---
>   arch/powerpc/platforms/powernv/pci.h      |  13 ---
>   4 files changed, 1 insertion(+), 152 deletions(-)
> 
> diff --git a/arch/powerpc/include/asm/pnv-pci.h b/arch/powerpc/include/asm/pnv-pci.h
> index d2d8c28db336..7f627e3f4da4 100644
> --- a/arch/powerpc/include/asm/pnv-pci.h
> +++ b/arch/powerpc/include/asm/pnv-pci.h
> @@ -50,13 +50,6 @@ int pnv_cxl_alloc_hwirq_ranges(struct cxl_irq_ranges *irqs,
>   			       struct pci_dev *dev, int num);
>   void pnv_cxl_release_hwirq_ranges(struct cxl_irq_ranges *irqs,
>   				  struct pci_dev *dev);
> -
> -/* Support for the cxl kernel api on the real PHB (instead of vPHB) */
> -int pnv_cxl_enable_phb_kernel_api(struct pci_controller *hose, bool enable);
> -bool pnv_pci_on_cxl_phb(struct pci_dev *dev);
> -struct cxl_afu *pnv_cxl_phb_to_afu(struct pci_controller *hose);
> -void pnv_cxl_phb_set_peer_afu(struct pci_dev *dev, struct cxl_afu *afu);
> -
>   #endif
>   
>   struct pnv_php_slot {
> diff --git a/arch/powerpc/platforms/powernv/pci-cxl.c b/arch/powerpc/platforms/powernv/pci-cxl.c
> index c447b7f03c09..1b18111453d7 100644
> --- a/arch/powerpc/platforms/powernv/pci-cxl.c
> +++ b/arch/powerpc/platforms/powernv/pci-cxl.c
> @@ -8,10 +8,8 @@
>    */
>   
>   #include <linux/module.h>
> -#include <asm/pci-bridge.h>
>   #include <asm/pnv-pci.h>
>   #include <asm/opal.h>
> -#include <misc/cxl.h>
>   
>   #include "pci.h"
>   
> @@ -178,116 +176,3 @@ static inline int get_cxl_module(void)
>   #else
>   static inline int get_cxl_module(void) { return 0; }
>   #endif
> -
> -/*
> - * Sets flags and switches the controller ops to enable the cxl kernel api.
> - * Originally the cxl kernel API operated on a virtual PHB, but certain cards
> - * such as the Mellanox CX4 use a peer model instead and for these cards the
> - * cxl kernel api will operate on the real PHB.
> - */
> -int pnv_cxl_enable_phb_kernel_api(struct pci_controller *hose, bool enable)
> -{
> -	struct pnv_phb *phb = hose->private_data;
> -	int rc;
> -
> -	if (!enable) {
> -		/*
> -		 * Once cxl mode is enabled on the PHB, there is currently no
> -		 * known safe method to disable it again, and trying risks a
> -		 * checkstop. If we can find a way to safely disable cxl mode
> -		 * in the future we can revisit this, but for now the only sane
> -		 * thing to do is to refuse to disable cxl mode:
> -		 */
> -		return -EPERM;
> -	}
> -
> -	/*
> -	 * Hold a reference to the cxl module since several PHB operations now
> -	 * depend on it, and it would be insane to allow it to be removed so
> -	 * long as we are in this mode (and since we can't safely disable this
> -	 * mode once enabled...).
> -	 */
> -	rc = get_cxl_module();
> -	if (rc)
> -		return rc;
> -
> -	phb->flags |= PNV_PHB_FLAG_CXL;
> -	hose->controller_ops = pnv_cxl_cx4_ioda_controller_ops;
> -
> -	return 0;
> -}
> -EXPORT_SYMBOL_GPL(pnv_cxl_enable_phb_kernel_api);
> -
> -bool pnv_pci_on_cxl_phb(struct pci_dev *dev)
> -{
> -	struct pci_controller *hose = pci_bus_to_host(dev->bus);
> -	struct pnv_phb *phb = hose->private_data;
> -
> -	return !!(phb->flags & PNV_PHB_FLAG_CXL);
> -}
> -EXPORT_SYMBOL_GPL(pnv_pci_on_cxl_phb);
> -
> -struct cxl_afu *pnv_cxl_phb_to_afu(struct pci_controller *hose)
> -{
> -	struct pnv_phb *phb = hose->private_data;
> -
> -	return (struct cxl_afu *)phb->cxl_afu;
> -}
> -EXPORT_SYMBOL_GPL(pnv_cxl_phb_to_afu);
> -
> -void pnv_cxl_phb_set_peer_afu(struct pci_dev *dev, struct cxl_afu *afu)
> -{
> -	struct pci_controller *hose = pci_bus_to_host(dev->bus);
> -	struct pnv_phb *phb = hose->private_data;
> -
> -	phb->cxl_afu = afu;
> -}
> -EXPORT_SYMBOL_GPL(pnv_cxl_phb_set_peer_afu);
> -
> -/*
> - * In the peer cxl model, the XSL/PSL is physical function 0, and will be used
> - * by other functions on the device for memory access and interrupts. When the
> - * other functions are enabled we explicitly take a reference on the cxl
> - * function since they will use it, and allocate a default context associated
> - * with that function just like the vPHB model of the cxl kernel API.
> - */
> -bool pnv_cxl_enable_device_hook(struct pci_dev *dev)
> -{
> -	struct pci_controller *hose = pci_bus_to_host(dev->bus);
> -	struct pnv_phb *phb = hose->private_data;
> -	struct cxl_afu *afu = phb->cxl_afu;
> -
> -	if (!pnv_pci_enable_device_hook(dev))
> -		return false;
> -
> -
> -	/* No special handling for the cxl function, which is always PF 0 */
> -	if (PCI_FUNC(dev->devfn) == 0)
> -		return true;
> -
> -	if (!afu) {
> -		dev_WARN(&dev->dev, "Attempted to enable function > 0 on CXL PHB without a peer AFU\n");
> -		return false;
> -	}
> -
> -	dev_info(&dev->dev, "Enabling function on CXL enabled PHB with peer AFU\n");
> -
> -	/* Make sure the peer AFU can't go away while this device is active */
> -	cxl_afu_get(afu);
> -
> -	return cxl_pci_associate_default_context(dev, afu);
> -}
> -
> -void pnv_cxl_disable_device(struct pci_dev *dev)
> -{
> -	struct pci_controller *hose = pci_bus_to_host(dev->bus);
> -	struct pnv_phb *phb = hose->private_data;
> -	struct cxl_afu *afu = phb->cxl_afu;
> -
> -	/* No special handling for cxl function: */
> -	if (PCI_FUNC(dev->devfn) == 0)
> -		return;
> -
> -	cxl_pci_disable_device(dev);
> -	cxl_afu_put(afu);
> -}
> diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
> index 41f8f0ff4a55..770c67c4e8f7 100644
> --- a/arch/powerpc/platforms/powernv/pci-ioda.c
> +++ b/arch/powerpc/platforms/powernv/pci-ioda.c
> @@ -3575,7 +3575,7 @@ static resource_size_t pnv_pci_iov_resource_alignment(struct pci_dev *pdev,
>   /* Prevent enabling devices for which we couldn't properly
>    * assign a PE
>    */
> -bool pnv_pci_enable_device_hook(struct pci_dev *dev)
> +static bool pnv_pci_enable_device_hook(struct pci_dev *dev)
>   {
>   	struct pci_controller *hose = pci_bus_to_host(dev->bus);
>   	struct pnv_phb *phb = hose->private_data;
> @@ -3843,22 +3843,6 @@ static const struct pci_controller_ops pnv_npu_ocapi_ioda_controller_ops = {
>   	.shutdown		= pnv_pci_ioda_shutdown,
>   };
>   
> -#ifdef CONFIG_CXL_BASE
> -const struct pci_controller_ops pnv_cxl_cx4_ioda_controller_ops = {
> -	.dma_dev_setup		= pnv_pci_dma_dev_setup,
> -	.dma_bus_setup		= pnv_pci_dma_bus_setup,
> -	.enable_device_hook	= pnv_cxl_enable_device_hook,
> -	.disable_device		= pnv_cxl_disable_device,
> -	.release_device		= pnv_pci_release_device,
> -	.window_alignment	= pnv_pci_window_alignment,
> -	.setup_bridge		= pnv_pci_setup_bridge,
> -	.reset_secondary_bus	= pnv_pci_reset_secondary_bus,
> -	.dma_set_mask		= pnv_pci_ioda_dma_set_mask,
> -	.dma_get_required_mask	= pnv_pci_ioda_dma_get_required_mask,
> -	.shutdown		= pnv_pci_ioda_shutdown,
> -};
> -#endif
> -
>   static void __init pnv_pci_init_ioda_phb(struct device_node *np,
>   					 u64 hub_id, int ioda_type)
>   {
> diff --git a/arch/powerpc/platforms/powernv/pci.h b/arch/powerpc/platforms/powernv/pci.h
> index ba41913c7e21..44dfbc37f547 100644
> --- a/arch/powerpc/platforms/powernv/pci.h
> +++ b/arch/powerpc/platforms/powernv/pci.h
> @@ -88,7 +88,6 @@ struct pnv_ioda_pe {
>   };
>   
>   #define PNV_PHB_FLAG_EEH	(1 << 0)
> -#define PNV_PHB_FLAG_CXL	(1 << 1) /* Real PHB supporting the cxl kernel API */
>   
>   struct pnv_phb {
>   	struct pci_controller	*hose;
> @@ -194,9 +193,6 @@ struct pnv_phb {
>   		bool nmmu_flush;
>   	} npu;
>   
> -#ifdef CONFIG_CXL_BASE
> -	struct cxl_afu *cxl_afu;
> -#endif
>   	int p2p_target_count;
>   };
>   
> @@ -238,7 +234,6 @@ extern int pnv_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type);
>   extern void pnv_teardown_msi_irqs(struct pci_dev *pdev);
>   extern struct pnv_ioda_pe *pnv_ioda_get_pe(struct pci_dev *dev);
>   extern void pnv_set_msi_irq_chip(struct pnv_phb *phb, unsigned int virq);
> -extern bool pnv_pci_enable_device_hook(struct pci_dev *dev);
>   extern void pnv_pci_ioda2_set_bypass(struct pnv_ioda_pe *pe, bool enable);
>   extern int pnv_eeh_post_init(void);
>   
> @@ -262,12 +257,4 @@ extern void pnv_npu_take_ownership(struct pnv_ioda_pe *npe);
>   extern void pnv_npu_release_ownership(struct pnv_ioda_pe *npe);
>   extern int pnv_npu2_init(struct pnv_phb *phb);
>   
> -/* cxl functions */
> -extern bool pnv_cxl_enable_device_hook(struct pci_dev *dev);
> -extern void pnv_cxl_disable_device(struct pci_dev *dev);
> -
> -
> -/* phb ops (cxl switches these when enabling the kernel api on the phb) */
> -extern const struct pci_controller_ops pnv_cxl_cx4_ioda_controller_ops;
> -
>   #endif /* __POWERNV_PCI_H */
> 

-- 
Andrew Donnellan              OzLabs, ADL Canberra
andrew.donnellan@au1.ibm.com  IBM Australia Limited

[PATCH v2 08/10] Revert "cxl: Add cxl_slot_is_supported API"

[Frederic Barrat]

Remove abandonned capi support for the Mellanox CX4.

This reverts commit 4e56f858bdde5cbfb70f61baddfaa56a8ed851bf.

Signed-off-by: Frederic Barrat <fbarrat@linux.ibm.com>
---
 drivers/misc/cxl/pci.c | 37 -------------------------------------
 include/misc/cxl.h     | 15 ---------------
 2 files changed, 52 deletions(-)

diff --git a/drivers/misc/cxl/pci.c b/drivers/misc/cxl/pci.c
index 193ff22f610b..0ca818396524 100644
--- a/drivers/misc/cxl/pci.c
+++ b/drivers/misc/cxl/pci.c
@@ -1808,43 +1808,6 @@ int cxl_slot_is_switched(struct pci_dev *dev)
 	return (depth > CXL_MAX_PCIEX_PARENT);
 }
 
-bool cxl_slot_is_supported(struct pci_dev *dev, int flags)
-{
-	if (!cpu_has_feature(CPU_FTR_HVMODE))
-		return false;
-
-	if ((flags & CXL_SLOT_FLAG_DMA) && (!pvr_version_is(PVR_POWER8NVL))) {
-		/*
-		 * CAPP DMA mode is technically supported on regular P8, but
-		 * will EEH if the card attempts to access memory < 4GB, which
-		 * we cannot realistically avoid. We might be able to work
-		 * around the issue, but until then return unsupported:
-		 */
-		return false;
-	}
-
-	if (cxl_slot_is_switched(dev))
-		return false;
-
-	/*
-	 * XXX: This gets a little tricky on regular P8 (not POWER8NVL) since
-	 * the CAPP can be connected to PHB 0, 1 or 2 on a first come first
-	 * served basis, which is racy to check from here. If we need to
-	 * support this in future we might need to consider having this
-	 * function effectively reserve it ahead of time.
-	 *
-	 * Currently, the only user of this API is the Mellanox CX4, which is
-	 * only supported on P8NVL due to the above mentioned limitation of
-	 * CAPP DMA mode and therefore does not need to worry about this. If the
-	 * issue with CAPP DMA mode is later worked around on P8 we might need
-	 * to revisit this.
-	 */
-
-	return true;
-}
-EXPORT_SYMBOL_GPL(cxl_slot_is_supported);
-
-
 static int cxl_probe(struct pci_dev *dev, const struct pci_device_id *id)
 {
 	struct cxl *adapter;
diff --git a/include/misc/cxl.h b/include/misc/cxl.h
index 74da2e440763..ea9ff4a1a9ca 100644
--- a/include/misc/cxl.h
+++ b/include/misc/cxl.h
@@ -24,21 +24,6 @@
  * generic PCI API. This API is agnostic to the actual AFU.
  */
 
-#define CXL_SLOT_FLAG_DMA 0x1
-
-/*
- * Checks if the given card is in a cxl capable slot. Pass CXL_SLOT_FLAG_DMA if
- * the card requires CAPP DMA mode to also check if the system supports it.
- * This is intended to be used by bi-modal devices to determine if they can use
- * cxl mode or if they should continue running in PCI mode.
- *
- * Note that this only checks if the slot is cxl capable - it does not
- * currently check if the CAPP is currently available for chips where it can be
- * assigned to different PHBs on a first come first serve basis (i.e. P8)
- */
-bool cxl_slot_is_supported(struct pci_dev *dev, int flags);
-
-
 /* Get the AFU associated with a pci_dev */
 struct cxl_afu *cxl_pci_to_afu(struct pci_dev *dev);
 
-- 
2.17.1

Re: [PATCH v2 08/10] Revert "cxl: Add cxl_slot_is_supported API"

[Andrew Donnellan]

On 28/06/18 20:05, Frederic Barrat wrote:
> Remove abandonned capi support for the Mellanox CX4.
> 
> This reverts commit 4e56f858bdde5cbfb70f61baddfaa56a8ed851bf.
> 
> Signed-off-by: Frederic Barrat <fbarrat@linux.ibm.com>

Acked-by: Andrew Donnellan <andrew.donnellan@au1.ibm.com>

> ---
>   drivers/misc/cxl/pci.c | 37 -------------------------------------
>   include/misc/cxl.h     | 15 ---------------
>   2 files changed, 52 deletions(-)
> 
> diff --git a/drivers/misc/cxl/pci.c b/drivers/misc/cxl/pci.c
> index 193ff22f610b..0ca818396524 100644
> --- a/drivers/misc/cxl/pci.c
> +++ b/drivers/misc/cxl/pci.c
> @@ -1808,43 +1808,6 @@ int cxl_slot_is_switched(struct pci_dev *dev)
>   	return (depth > CXL_MAX_PCIEX_PARENT);
>   }
>   
> -bool cxl_slot_is_supported(struct pci_dev *dev, int flags)
> -{
> -	if (!cpu_has_feature(CPU_FTR_HVMODE))
> -		return false;
> -
> -	if ((flags & CXL_SLOT_FLAG_DMA) && (!pvr_version_is(PVR_POWER8NVL))) {
> -		/*
> -		 * CAPP DMA mode is technically supported on regular P8, but
> -		 * will EEH if the card attempts to access memory < 4GB, which
> -		 * we cannot realistically avoid. We might be able to work
> -		 * around the issue, but until then return unsupported:
> -		 */
> -		return false;
> -	}
> -
> -	if (cxl_slot_is_switched(dev))
> -		return false;
> -
> -	/*
> -	 * XXX: This gets a little tricky on regular P8 (not POWER8NVL) since
> -	 * the CAPP can be connected to PHB 0, 1 or 2 on a first come first
> -	 * served basis, which is racy to check from here. If we need to
> -	 * support this in future we might need to consider having this
> -	 * function effectively reserve it ahead of time.
> -	 *
> -	 * Currently, the only user of this API is the Mellanox CX4, which is
> -	 * only supported on P8NVL due to the above mentioned limitation of
> -	 * CAPP DMA mode and therefore does not need to worry about this. If the
> -	 * issue with CAPP DMA mode is later worked around on P8 we might need
> -	 * to revisit this.
> -	 */
> -
> -	return true;
> -}
> -EXPORT_SYMBOL_GPL(cxl_slot_is_supported);
> -
> -
>   static int cxl_probe(struct pci_dev *dev, const struct pci_device_id *id)
>   {
>   	struct cxl *adapter;
> diff --git a/include/misc/cxl.h b/include/misc/cxl.h
> index 74da2e440763..ea9ff4a1a9ca 100644
> --- a/include/misc/cxl.h
> +++ b/include/misc/cxl.h
> @@ -24,21 +24,6 @@
>    * generic PCI API. This API is agnostic to the actual AFU.
>    */
>   
> -#define CXL_SLOT_FLAG_DMA 0x1
> -
> -/*
> - * Checks if the given card is in a cxl capable slot. Pass CXL_SLOT_FLAG_DMA if
> - * the card requires CAPP DMA mode to also check if the system supports it.
> - * This is intended to be used by bi-modal devices to determine if they can use
> - * cxl mode or if they should continue running in PCI mode.
> - *
> - * Note that this only checks if the slot is cxl capable - it does not
> - * currently check if the CAPP is currently available for chips where it can be
> - * assigned to different PHBs on a first come first serve basis (i.e. P8)
> - */
> -bool cxl_slot_is_supported(struct pci_dev *dev, int flags);
> -
> -
>   /* Get the AFU associated with a pci_dev */
>   struct cxl_afu *cxl_pci_to_afu(struct pci_dev *dev);
>   
> 

-- 
Andrew Donnellan              OzLabs, ADL Canberra
andrew.donnellan@au1.ibm.com  IBM Australia Limited

[PATCH v2 09/10] Revert "cxl: Allow a default context to be associated with an external pci_dev"

[Frederic Barrat]

Remove abandonned capi support for the Mellanox CX4.

This reverts commit a19bd79e31769626d288cc016e21a31b6f47bf6f.

Signed-off-by: Frederic Barrat <fbarrat@linux.ibm.com>
---
 drivers/misc/cxl/Makefile |  2 +-
 drivers/misc/cxl/base.c   | 35 -------------------------------
 drivers/misc/cxl/cxl.h    |  6 ------
 drivers/misc/cxl/main.c   |  2 --
 drivers/misc/cxl/phb.c    | 44 ---------------------------------------
 drivers/misc/cxl/vphb.c   | 30 +++++++++++++++++++++++---
 include/misc/cxl-base.h   |  6 ------
 7 files changed, 28 insertions(+), 97 deletions(-)
 delete mode 100644 drivers/misc/cxl/phb.c

diff --git a/drivers/misc/cxl/Makefile b/drivers/misc/cxl/Makefile
index 502d41fc9ea5..5eea61b9584f 100644
--- a/drivers/misc/cxl/Makefile
+++ b/drivers/misc/cxl/Makefile
@@ -4,7 +4,7 @@ ccflags-$(CONFIG_PPC_WERROR)	+= -Werror
 
 cxl-y				+= main.o file.o irq.o fault.o native.o
 cxl-y				+= context.o sysfs.o pci.o trace.o
-cxl-y				+= vphb.o phb.o api.o cxllib.o
+cxl-y				+= vphb.o api.o cxllib.o
 cxl-$(CONFIG_PPC_PSERIES)	+= flash.o guest.o of.o hcalls.o
 cxl-$(CONFIG_DEBUG_FS)		+= debugfs.o
 obj-$(CONFIG_CXL)		+= cxl.o
diff --git a/drivers/misc/cxl/base.c b/drivers/misc/cxl/base.c
index e1e80cb99ad9..7557835cdfcd 100644
--- a/drivers/misc/cxl/base.c
+++ b/drivers/misc/cxl/base.c
@@ -106,41 +106,6 @@ int cxl_update_properties(struct device_node *dn,
 }
 EXPORT_SYMBOL_GPL(cxl_update_properties);
 
-/*
- * API calls into the driver that may be called from the PHB code and must be
- * built in.
- */
-bool cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu)
-{
-	bool ret;
-	struct cxl_calls *calls;
-
-	calls = cxl_calls_get();
-	if (!calls)
-		return false;
-
-	ret = calls->cxl_pci_associate_default_context(dev, afu);
-
-	cxl_calls_put(calls);
-
-	return ret;
-}
-EXPORT_SYMBOL_GPL(cxl_pci_associate_default_context);
-
-void cxl_pci_disable_device(struct pci_dev *dev)
-{
-	struct cxl_calls *calls;
-
-	calls = cxl_calls_get();
-	if (!calls)
-		return;
-
-	calls->cxl_pci_disable_device(dev);
-
-	cxl_calls_put(calls);
-}
-EXPORT_SYMBOL_GPL(cxl_pci_disable_device);
-
 static int __init cxl_base_init(void)
 {
 	struct device_node *np;
diff --git a/drivers/misc/cxl/cxl.h b/drivers/misc/cxl/cxl.h
index d95c2c98f2ab..aa453448201d 100644
--- a/drivers/misc/cxl/cxl.h
+++ b/drivers/misc/cxl/cxl.h
@@ -867,15 +867,9 @@ static inline bool cxl_is_power9_dd1(void)
 ssize_t cxl_pci_afu_read_err_buffer(struct cxl_afu *afu, char *buf,
 				loff_t off, size_t count);
 
-/* Internal functions wrapped in cxl_base to allow PHB to call them */
-bool _cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu);
-void _cxl_pci_disable_device(struct pci_dev *dev);
 
 struct cxl_calls {
 	void (*cxl_slbia)(struct mm_struct *mm);
-	bool (*cxl_pci_associate_default_context)(struct pci_dev *dev, struct cxl_afu *afu);
-	void (*cxl_pci_disable_device)(struct pci_dev *dev);
-
 	struct module *owner;
 };
 int register_cxl_calls(struct cxl_calls *calls);
diff --git a/drivers/misc/cxl/main.c b/drivers/misc/cxl/main.c
index a7e83624034b..334223b802ee 100644
--- a/drivers/misc/cxl/main.c
+++ b/drivers/misc/cxl/main.c
@@ -104,8 +104,6 @@ static inline void cxl_slbia_core(struct mm_struct *mm)
 
 static struct cxl_calls cxl_calls = {
 	.cxl_slbia = cxl_slbia_core,
-	.cxl_pci_associate_default_context = _cxl_pci_associate_default_context,
-	.cxl_pci_disable_device = _cxl_pci_disable_device,
 	.owner = THIS_MODULE,
 };
 
diff --git a/drivers/misc/cxl/phb.c b/drivers/misc/cxl/phb.c
deleted file mode 100644
index 6ec69ada19f4..000000000000
--- a/drivers/misc/cxl/phb.c
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Copyright 2014-2016 IBM Corp.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version
- * 2 of the License, or (at your option) any later version.
- */
-
-#include <linux/pci.h>
-#include "cxl.h"
-
-bool _cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu)
-{
-	struct cxl_context *ctx;
-
-	/*
-	 * Allocate a context to do cxl things to. This is used for interrupts
-	 * in the peer model using a real phb, and if we eventually do DMA ops
-	 * in the virtual phb, we'll need a default context to attach them to.
-	 */
-	ctx = cxl_dev_context_init(dev);
-	if (IS_ERR(ctx))
-		return false;
-	dev->dev.archdata.cxl_ctx = ctx;
-
-	return (cxl_ops->afu_check_and_enable(afu) == 0);
-}
-/* exported via cxl_base */
-
-void _cxl_pci_disable_device(struct pci_dev *dev)
-{
-	struct cxl_context *ctx = cxl_get_context(dev);
-
-	if (ctx) {
-		if (ctx->status == STARTED) {
-			dev_err(&dev->dev, "Default context started\n");
-			return;
-		}
-		dev->dev.archdata.cxl_ctx = NULL;
-		cxl_release_context(ctx);
-	}
-}
-/* exported via cxl_base */
diff --git a/drivers/misc/cxl/vphb.c b/drivers/misc/cxl/vphb.c
index 1a99c9c7a6fb..7908633d9204 100644
--- a/drivers/misc/cxl/vphb.c
+++ b/drivers/misc/cxl/vphb.c
@@ -44,6 +44,7 @@ static bool cxl_pci_enable_device_hook(struct pci_dev *dev)
 {
 	struct pci_controller *phb;
 	struct cxl_afu *afu;
+	struct cxl_context *ctx;
 
 	phb = pci_bus_to_host(dev->bus);
 	afu = (struct cxl_afu *)phb->private_data;
@@ -56,7 +57,30 @@ static bool cxl_pci_enable_device_hook(struct pci_dev *dev)
 	set_dma_ops(&dev->dev, &dma_nommu_ops);
 	set_dma_offset(&dev->dev, PAGE_OFFSET);
 
-	return _cxl_pci_associate_default_context(dev, afu);
+	/*
+	 * Allocate a context to do cxl things too.  If we eventually do real
+	 * DMA ops, we'll need a default context to attach them to
+	 */
+	ctx = cxl_dev_context_init(dev);
+	if (IS_ERR(ctx))
+		return false;
+	dev->dev.archdata.cxl_ctx = ctx;
+
+	return (cxl_ops->afu_check_and_enable(afu) == 0);
+}
+
+static void cxl_pci_disable_device(struct pci_dev *dev)
+{
+	struct cxl_context *ctx = cxl_get_context(dev);
+
+	if (ctx) {
+		if (ctx->status == STARTED) {
+			dev_err(&dev->dev, "Default context started\n");
+			return;
+		}
+		dev->dev.archdata.cxl_ctx = NULL;
+		cxl_release_context(ctx);
+	}
 }
 
 static resource_size_t cxl_pci_window_alignment(struct pci_bus *bus,
@@ -190,8 +214,8 @@ static struct pci_controller_ops cxl_pci_controller_ops =
 {
 	.probe_mode = cxl_pci_probe_mode,
 	.enable_device_hook = cxl_pci_enable_device_hook,
-	.disable_device = _cxl_pci_disable_device,
-	.release_device = _cxl_pci_disable_device,
+	.disable_device = cxl_pci_disable_device,
+	.release_device = cxl_pci_disable_device,
 	.window_alignment = cxl_pci_window_alignment,
 	.reset_secondary_bus = cxl_pci_reset_secondary_bus,
 	.setup_msi_irqs = cxl_setup_msi_irqs,
diff --git a/include/misc/cxl-base.h b/include/misc/cxl-base.h
index bb7e629ae492..f53808fa638a 100644
--- a/include/misc/cxl-base.h
+++ b/include/misc/cxl-base.h
@@ -10,8 +10,6 @@
 #ifndef _MISC_CXL_BASE_H
 #define _MISC_CXL_BASE_H
 
-#include <misc/cxl.h>
-
 #ifdef CONFIG_CXL_BASE
 
 #define CXL_IRQ_RANGES 4
@@ -41,8 +39,6 @@ static inline void cxl_ctx_put(void)
 struct cxl_afu *cxl_afu_get(struct cxl_afu *afu);
 void cxl_afu_put(struct cxl_afu *afu);
 void cxl_slbia(struct mm_struct *mm);
-bool cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu);
-void cxl_pci_disable_device(struct pci_dev *dev);
 
 #else /* CONFIG_CXL_BASE */
 
@@ -50,8 +46,6 @@ static inline bool cxl_ctx_in_use(void) { return false; }
 static inline struct cxl_afu *cxl_afu_get(struct cxl_afu *afu) { return NULL; }
 static inline void cxl_afu_put(struct cxl_afu *afu) {}
 static inline void cxl_slbia(struct mm_struct *mm) {}
-static inline bool cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu) { return false; }
-static inline void cxl_pci_disable_device(struct pci_dev *dev) {}
 
 #endif /* CONFIG_CXL_BASE */
 
-- 
2.17.1

Re: [PATCH v2 09/10] Revert "cxl: Allow a default context to be associated with an external pci_dev"

[Andrew Donnellan]

On 28/06/18 20:05, Frederic Barrat wrote:
> Remove abandonned capi support for the Mellanox CX4.
> 
> This reverts commit a19bd79e31769626d288cc016e21a31b6f47bf6f.
> 
> Signed-off-by: Frederic Barrat <fbarrat@linux.ibm.com>

Acked-by: Andrew Donnellan <andrew.donnellan@au1.ibm.com>

> ---
>   drivers/misc/cxl/Makefile |  2 +-
>   drivers/misc/cxl/base.c   | 35 -------------------------------
>   drivers/misc/cxl/cxl.h    |  6 ------
>   drivers/misc/cxl/main.c   |  2 --
>   drivers/misc/cxl/phb.c    | 44 ---------------------------------------
>   drivers/misc/cxl/vphb.c   | 30 +++++++++++++++++++++++---
>   include/misc/cxl-base.h   |  6 ------
>   7 files changed, 28 insertions(+), 97 deletions(-)
>   delete mode 100644 drivers/misc/cxl/phb.c
> 
> diff --git a/drivers/misc/cxl/Makefile b/drivers/misc/cxl/Makefile
> index 502d41fc9ea5..5eea61b9584f 100644
> --- a/drivers/misc/cxl/Makefile
> +++ b/drivers/misc/cxl/Makefile
> @@ -4,7 +4,7 @@ ccflags-$(CONFIG_PPC_WERROR)	+= -Werror
>   
>   cxl-y				+= main.o file.o irq.o fault.o native.o
>   cxl-y				+= context.o sysfs.o pci.o trace.o
> -cxl-y				+= vphb.o phb.o api.o cxllib.o
> +cxl-y				+= vphb.o api.o cxllib.o
>   cxl-$(CONFIG_PPC_PSERIES)	+= flash.o guest.o of.o hcalls.o
>   cxl-$(CONFIG_DEBUG_FS)		+= debugfs.o
>   obj-$(CONFIG_CXL)		+= cxl.o
> diff --git a/drivers/misc/cxl/base.c b/drivers/misc/cxl/base.c
> index e1e80cb99ad9..7557835cdfcd 100644
> --- a/drivers/misc/cxl/base.c
> +++ b/drivers/misc/cxl/base.c
> @@ -106,41 +106,6 @@ int cxl_update_properties(struct device_node *dn,
>   }
>   EXPORT_SYMBOL_GPL(cxl_update_properties);
>   
> -/*
> - * API calls into the driver that may be called from the PHB code and must be
> - * built in.
> - */
> -bool cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu)
> -{
> -	bool ret;
> -	struct cxl_calls *calls;
> -
> -	calls = cxl_calls_get();
> -	if (!calls)
> -		return false;
> -
> -	ret = calls->cxl_pci_associate_default_context(dev, afu);
> -
> -	cxl_calls_put(calls);
> -
> -	return ret;
> -}
> -EXPORT_SYMBOL_GPL(cxl_pci_associate_default_context);
> -
> -void cxl_pci_disable_device(struct pci_dev *dev)
> -{
> -	struct cxl_calls *calls;
> -
> -	calls = cxl_calls_get();
> -	if (!calls)
> -		return;
> -
> -	calls->cxl_pci_disable_device(dev);
> -
> -	cxl_calls_put(calls);
> -}
> -EXPORT_SYMBOL_GPL(cxl_pci_disable_device);
> -
>   static int __init cxl_base_init(void)
>   {
>   	struct device_node *np;
> diff --git a/drivers/misc/cxl/cxl.h b/drivers/misc/cxl/cxl.h
> index d95c2c98f2ab..aa453448201d 100644
> --- a/drivers/misc/cxl/cxl.h
> +++ b/drivers/misc/cxl/cxl.h
> @@ -867,15 +867,9 @@ static inline bool cxl_is_power9_dd1(void)
>   ssize_t cxl_pci_afu_read_err_buffer(struct cxl_afu *afu, char *buf,
>   				loff_t off, size_t count);
>   
> -/* Internal functions wrapped in cxl_base to allow PHB to call them */
> -bool _cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu);
> -void _cxl_pci_disable_device(struct pci_dev *dev);
>   
>   struct cxl_calls {
>   	void (*cxl_slbia)(struct mm_struct *mm);
> -	bool (*cxl_pci_associate_default_context)(struct pci_dev *dev, struct cxl_afu *afu);
> -	void (*cxl_pci_disable_device)(struct pci_dev *dev);
> -
>   	struct module *owner;
>   };
>   int register_cxl_calls(struct cxl_calls *calls);
> diff --git a/drivers/misc/cxl/main.c b/drivers/misc/cxl/main.c
> index a7e83624034b..334223b802ee 100644
> --- a/drivers/misc/cxl/main.c
> +++ b/drivers/misc/cxl/main.c
> @@ -104,8 +104,6 @@ static inline void cxl_slbia_core(struct mm_struct *mm)
>   
>   static struct cxl_calls cxl_calls = {
>   	.cxl_slbia = cxl_slbia_core,
> -	.cxl_pci_associate_default_context = _cxl_pci_associate_default_context,
> -	.cxl_pci_disable_device = _cxl_pci_disable_device,
>   	.owner = THIS_MODULE,
>   };
>   
> diff --git a/drivers/misc/cxl/phb.c b/drivers/misc/cxl/phb.c
> deleted file mode 100644
> index 6ec69ada19f4..000000000000
> --- a/drivers/misc/cxl/phb.c
> +++ /dev/null
> @@ -1,44 +0,0 @@
> -/*
> - * Copyright 2014-2016 IBM Corp.
> - *
> - * This program is free software; you can redistribute it and/or
> - * modify it under the terms of the GNU General Public License
> - * as published by the Free Software Foundation; either version
> - * 2 of the License, or (at your option) any later version.
> - */
> -
> -#include <linux/pci.h>
> -#include "cxl.h"
> -
> -bool _cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu)
> -{
> -	struct cxl_context *ctx;
> -
> -	/*
> -	 * Allocate a context to do cxl things to. This is used for interrupts
> -	 * in the peer model using a real phb, and if we eventually do DMA ops
> -	 * in the virtual phb, we'll need a default context to attach them to.
> -	 */
> -	ctx = cxl_dev_context_init(dev);
> -	if (IS_ERR(ctx))
> -		return false;
> -	dev->dev.archdata.cxl_ctx = ctx;
> -
> -	return (cxl_ops->afu_check_and_enable(afu) == 0);
> -}
> -/* exported via cxl_base */
> -
> -void _cxl_pci_disable_device(struct pci_dev *dev)
> -{
> -	struct cxl_context *ctx = cxl_get_context(dev);
> -
> -	if (ctx) {
> -		if (ctx->status == STARTED) {
> -			dev_err(&dev->dev, "Default context started\n");
> -			return;
> -		}
> -		dev->dev.archdata.cxl_ctx = NULL;
> -		cxl_release_context(ctx);
> -	}
> -}
> -/* exported via cxl_base */
> diff --git a/drivers/misc/cxl/vphb.c b/drivers/misc/cxl/vphb.c
> index 1a99c9c7a6fb..7908633d9204 100644
> --- a/drivers/misc/cxl/vphb.c
> +++ b/drivers/misc/cxl/vphb.c
> @@ -44,6 +44,7 @@ static bool cxl_pci_enable_device_hook(struct pci_dev *dev)
>   {
>   	struct pci_controller *phb;
>   	struct cxl_afu *afu;
> +	struct cxl_context *ctx;
>   
>   	phb = pci_bus_to_host(dev->bus);
>   	afu = (struct cxl_afu *)phb->private_data;
> @@ -56,7 +57,30 @@ static bool cxl_pci_enable_device_hook(struct pci_dev *dev)
>   	set_dma_ops(&dev->dev, &dma_nommu_ops);
>   	set_dma_offset(&dev->dev, PAGE_OFFSET);
>   
> -	return _cxl_pci_associate_default_context(dev, afu);
> +	/*
> +	 * Allocate a context to do cxl things too.  If we eventually do real
> +	 * DMA ops, we'll need a default context to attach them to
> +	 */
> +	ctx = cxl_dev_context_init(dev);
> +	if (IS_ERR(ctx))
> +		return false;
> +	dev->dev.archdata.cxl_ctx = ctx;
> +
> +	return (cxl_ops->afu_check_and_enable(afu) == 0);
> +}
> +
> +static void cxl_pci_disable_device(struct pci_dev *dev)
> +{
> +	struct cxl_context *ctx = cxl_get_context(dev);
> +
> +	if (ctx) {
> +		if (ctx->status == STARTED) {
> +			dev_err(&dev->dev, "Default context started\n");
> +			return;
> +		}
> +		dev->dev.archdata.cxl_ctx = NULL;
> +		cxl_release_context(ctx);
> +	}
>   }
>   
>   static resource_size_t cxl_pci_window_alignment(struct pci_bus *bus,
> @@ -190,8 +214,8 @@ static struct pci_controller_ops cxl_pci_controller_ops =
>   {
>   	.probe_mode = cxl_pci_probe_mode,
>   	.enable_device_hook = cxl_pci_enable_device_hook,
> -	.disable_device = _cxl_pci_disable_device,
> -	.release_device = _cxl_pci_disable_device,
> +	.disable_device = cxl_pci_disable_device,
> +	.release_device = cxl_pci_disable_device,
>   	.window_alignment = cxl_pci_window_alignment,
>   	.reset_secondary_bus = cxl_pci_reset_secondary_bus,
>   	.setup_msi_irqs = cxl_setup_msi_irqs,
> diff --git a/include/misc/cxl-base.h b/include/misc/cxl-base.h
> index bb7e629ae492..f53808fa638a 100644
> --- a/include/misc/cxl-base.h
> +++ b/include/misc/cxl-base.h
> @@ -10,8 +10,6 @@
>   #ifndef _MISC_CXL_BASE_H
>   #define _MISC_CXL_BASE_H
>   
> -#include <misc/cxl.h>
> -
>   #ifdef CONFIG_CXL_BASE
>   
>   #define CXL_IRQ_RANGES 4
> @@ -41,8 +39,6 @@ static inline void cxl_ctx_put(void)
>   struct cxl_afu *cxl_afu_get(struct cxl_afu *afu);
>   void cxl_afu_put(struct cxl_afu *afu);
>   void cxl_slbia(struct mm_struct *mm);
> -bool cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu);
> -void cxl_pci_disable_device(struct pci_dev *dev);
>   
>   #else /* CONFIG_CXL_BASE */
>   
> @@ -50,8 +46,6 @@ static inline bool cxl_ctx_in_use(void) { return false; }
>   static inline struct cxl_afu *cxl_afu_get(struct cxl_afu *afu) { return NULL; }
>   static inline void cxl_afu_put(struct cxl_afu *afu) {}
>   static inline void cxl_slbia(struct mm_struct *mm) {}
> -static inline bool cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu) { return false; }
> -static inline void cxl_pci_disable_device(struct pci_dev *dev) {}
>   
>   #endif /* CONFIG_CXL_BASE */
>   
> 

-- 
Andrew Donnellan              OzLabs, ADL Canberra
andrew.donnellan@au1.ibm.com  IBM Australia Limited

[PATCH v2 10/10] cxl: Remove abandonned capi support for the Mellanox CX4, final cleanup

[Frederic Barrat]

Remove a few XSL/CX4 oddities which are no longer needed. A simple
revert of the initial commits was not possible (or not worth it) due
to the history of the code.

Signed-off-by: Frederic Barrat <fbarrat@linux.ibm.com>
---
 drivers/misc/cxl/context.c |  2 +-
 drivers/misc/cxl/cxl.h     | 12 ------
 drivers/misc/cxl/debugfs.c |  5 ---
 drivers/misc/cxl/pci.c     | 75 +++-----------------------------------
 4 files changed, 7 insertions(+), 87 deletions(-)

diff --git a/drivers/misc/cxl/context.c b/drivers/misc/cxl/context.c
index 0355d42d367f..5fe529b43ebe 100644
--- a/drivers/misc/cxl/context.c
+++ b/drivers/misc/cxl/context.c
@@ -95,7 +95,7 @@ int cxl_context_init(struct cxl_context *ctx, struct cxl_afu *afu, bool master)
 	 */
 	mutex_lock(&afu->contexts_lock);
 	idr_preload(GFP_KERNEL);
-	i = idr_alloc(&ctx->afu->contexts_idr, ctx, ctx->afu->adapter->min_pe,
+	i = idr_alloc(&ctx->afu->contexts_idr, ctx, 0,
 		      ctx->afu->num_procs, GFP_NOWAIT);
 	idr_preload_end();
 	mutex_unlock(&afu->contexts_lock);
diff --git a/drivers/misc/cxl/cxl.h b/drivers/misc/cxl/cxl.h
index aa453448201d..44bcfafbb579 100644
--- a/drivers/misc/cxl/cxl.h
+++ b/drivers/misc/cxl/cxl.h
@@ -93,11 +93,6 @@ static const cxl_p1_reg_t CXL_PSL_FIR_CNTL  = {0x0148};
 static const cxl_p1_reg_t CXL_PSL_DSNDCTL   = {0x0150};
 static const cxl_p1_reg_t CXL_PSL_SNWRALLOC = {0x0158};
 static const cxl_p1_reg_t CXL_PSL_TRACE     = {0x0170};
-/* XSL registers (Mellanox CX4) */
-static const cxl_p1_reg_t CXL_XSL_Timebase  = {0x0100};
-static const cxl_p1_reg_t CXL_XSL_TB_CTLSTAT = {0x0108};
-static const cxl_p1_reg_t CXL_XSL_FEC       = {0x0158};
-static const cxl_p1_reg_t CXL_XSL_DSNCTL    = {0x0168};
 /* PSL registers - CAIA 2 */
 static const cxl_p1_reg_t CXL_PSL9_CONTROL  = {0x0020};
 static const cxl_p1_reg_t CXL_XSL9_INV      = {0x0110};
@@ -695,7 +690,6 @@ struct cxl {
 	struct bin_attribute cxl_attr;
 	int adapter_num;
 	int user_irqs;
-	int min_pe;
 	u64 ps_size;
 	u16 psl_rev;
 	u16 base_image;
@@ -934,7 +928,6 @@ int cxl_debugfs_afu_add(struct cxl_afu *afu);
 void cxl_debugfs_afu_remove(struct cxl_afu *afu);
 void cxl_debugfs_add_adapter_regs_psl9(struct cxl *adapter, struct dentry *dir);
 void cxl_debugfs_add_adapter_regs_psl8(struct cxl *adapter, struct dentry *dir);
-void cxl_debugfs_add_adapter_regs_xsl(struct cxl *adapter, struct dentry *dir);
 void cxl_debugfs_add_afu_regs_psl9(struct cxl_afu *afu, struct dentry *dir);
 void cxl_debugfs_add_afu_regs_psl8(struct cxl_afu *afu, struct dentry *dir);
 
@@ -977,11 +970,6 @@ static inline void cxl_debugfs_add_adapter_regs_psl8(struct cxl *adapter,
 {
 }
 
-static inline void cxl_debugfs_add_adapter_regs_xsl(struct cxl *adapter,
-						    struct dentry *dir)
-{
-}
-
 static inline void cxl_debugfs_add_afu_regs_psl9(struct cxl_afu *afu, struct dentry *dir)
 {
 }
diff --git a/drivers/misc/cxl/debugfs.c b/drivers/misc/cxl/debugfs.c
index 1643850d2302..a1921d81593a 100644
--- a/drivers/misc/cxl/debugfs.c
+++ b/drivers/misc/cxl/debugfs.c
@@ -58,11 +58,6 @@ void cxl_debugfs_add_adapter_regs_psl8(struct cxl *adapter, struct dentry *dir)
 	debugfs_create_io_x64("trace", S_IRUSR | S_IWUSR, dir, _cxl_p1_addr(adapter, CXL_PSL_TRACE));
 }
 
-void cxl_debugfs_add_adapter_regs_xsl(struct cxl *adapter, struct dentry *dir)
-{
-	debugfs_create_io_x64("fec", S_IRUSR, dir, _cxl_p1_addr(adapter, CXL_XSL_FEC));
-}
-
 int cxl_debugfs_adapter_add(struct cxl *adapter)
 {
 	struct dentry *dir;
diff --git a/drivers/misc/cxl/pci.c b/drivers/misc/cxl/pci.c
index 0ca818396524..6dfb4ed345d3 100644
--- a/drivers/misc/cxl/pci.c
+++ b/drivers/misc/cxl/pci.c
@@ -593,27 +593,7 @@ static int init_implementation_adapter_regs_psl8(struct cxl *adapter, struct pci
 	return 0;
 }
 
-static int init_implementation_adapter_regs_xsl(struct cxl *adapter, struct pci_dev *dev)
-{
-	u64 xsl_dsnctl;
-	u64 chipid;
-	u32 phb_index;
-	u64 capp_unit_id;
-	int rc;
-
-	rc = cxl_calc_capp_routing(dev, &chipid, &phb_index, &capp_unit_id);
-	if (rc)
-		return rc;
-
-	/* Tell XSL where to route data to */
-	xsl_dsnctl = 0x0000600000000000ULL | (chipid << (63-5));
-	xsl_dsnctl |= (capp_unit_id << (63-13));
-	cxl_p1_write(adapter, CXL_XSL_DSNCTL, xsl_dsnctl);
-
-	return 0;
-}
-
-/* PSL & XSL */
+/* PSL */
 #define TBSYNC_CAL(n) (((u64)n & 0x7) << (63-3))
 #define TBSYNC_CNT(n) (((u64)n & 0x7) << (63-6))
 /* For the PSL this is a multiple for 0 < n <= 7: */
@@ -625,21 +605,6 @@ static void write_timebase_ctrl_psl8(struct cxl *adapter)
 		     TBSYNC_CNT(2 * PSL_2048_250MHZ_CYCLES));
 }
 
-/* XSL */
-#define TBSYNC_ENA (1ULL << 63)
-/* For the XSL this is 2**n * 2000 clocks for 0 < n <= 6: */
-#define XSL_2000_CLOCKS 1
-#define XSL_4000_CLOCKS 2
-#define XSL_8000_CLOCKS 3
-
-static void write_timebase_ctrl_xsl(struct cxl *adapter)
-{
-	cxl_p1_write(adapter, CXL_XSL_TB_CTLSTAT,
-		     TBSYNC_ENA |
-		     TBSYNC_CAL(3) |
-		     TBSYNC_CNT(XSL_4000_CLOCKS));
-}
-
 static u64 timebase_read_psl9(struct cxl *adapter)
 {
 	return cxl_p1_read(adapter, CXL_PSL9_Timebase);
@@ -650,11 +615,6 @@ static u64 timebase_read_psl8(struct cxl *adapter)
 	return cxl_p1_read(adapter, CXL_PSL_Timebase);
 }
 
-static u64 timebase_read_xsl(struct cxl *adapter)
-{
-	return cxl_p1_read(adapter, CXL_XSL_Timebase);
-}
-
 static void cxl_setup_psl_timebase(struct cxl *adapter, struct pci_dev *dev)
 {
 	struct device_node *np;
@@ -1671,37 +1631,14 @@ static const struct cxl_service_layer_ops psl8_ops = {
 	.needs_reset_before_disable = true,
 };
 
-static const struct cxl_service_layer_ops xsl_ops = {
-	.adapter_regs_init = init_implementation_adapter_regs_xsl,
-	.invalidate_all = cxl_invalidate_all_psl8,
-	.sanitise_afu_regs = sanitise_afu_regs_psl8,
-	.handle_interrupt = cxl_irq_psl8,
-	.fail_irq = cxl_fail_irq_psl,
-	.activate_dedicated_process = cxl_activate_dedicated_process_psl8,
-	.attach_afu_directed = cxl_attach_afu_directed_psl8,
-	.attach_dedicated_process = cxl_attach_dedicated_process_psl8,
-	.update_dedicated_ivtes = cxl_update_dedicated_ivtes_psl8,
-	.debugfs_add_adapter_regs = cxl_debugfs_add_adapter_regs_xsl,
-	.write_timebase_ctrl = write_timebase_ctrl_xsl,
-	.timebase_read = timebase_read_xsl,
-	.capi_mode = OPAL_PHB_CAPI_MODE_DMA,
-};
-
 static void set_sl_ops(struct cxl *adapter, struct pci_dev *dev)
 {
-	if (dev->vendor == PCI_VENDOR_ID_MELLANOX && dev->device == 0x1013) {
-		/* Mellanox CX-4 */
-		dev_info(&dev->dev, "Device uses an XSL\n");
-		adapter->native->sl_ops = &xsl_ops;
-		adapter->min_pe = 1; /* Workaround for CX-4 hardware bug */
+	if (cxl_is_power8()) {
+		dev_info(&dev->dev, "Device uses a PSL8\n");
+		adapter->native->sl_ops = &psl8_ops;
 	} else {
-		if (cxl_is_power8()) {
-			dev_info(&dev->dev, "Device uses a PSL8\n");
-			adapter->native->sl_ops = &psl8_ops;
-		} else {
-			dev_info(&dev->dev, "Device uses a PSL9\n");
-			adapter->native->sl_ops = &psl9_ops;
-		}
+		dev_info(&dev->dev, "Device uses a PSL9\n");
+		adapter->native->sl_ops = &psl9_ops;
 	}
 }
 
-- 
2.17.1

Re: [PATCH v2 10/10] cxl: Remove abandonned capi support for the Mellanox CX4, final cleanup

[Andrew Donnellan]

On 28/06/18 20:05, Frederic Barrat wrote:
> Remove a few XSL/CX4 oddities which are no longer needed. A simple
> revert of the initial commits was not possible (or not worth it) due
> to the history of the code.
> 
> Signed-off-by: Frederic Barrat <fbarrat@linux.ibm.com>

Acked-by: Andrew Donnellan <andrew.donnellan@au1.ibm.com>

> ---
>   drivers/misc/cxl/context.c |  2 +-
>   drivers/misc/cxl/cxl.h     | 12 ------
>   drivers/misc/cxl/debugfs.c |  5 ---
>   drivers/misc/cxl/pci.c     | 75 +++-----------------------------------
>   4 files changed, 7 insertions(+), 87 deletions(-)
> 
> diff --git a/drivers/misc/cxl/context.c b/drivers/misc/cxl/context.c
> index 0355d42d367f..5fe529b43ebe 100644
> --- a/drivers/misc/cxl/context.c
> +++ b/drivers/misc/cxl/context.c
> @@ -95,7 +95,7 @@ int cxl_context_init(struct cxl_context *ctx, struct cxl_afu *afu, bool master)
>   	 */
>   	mutex_lock(&afu->contexts_lock);
>   	idr_preload(GFP_KERNEL);
> -	i = idr_alloc(&ctx->afu->contexts_idr, ctx, ctx->afu->adapter->min_pe,
> +	i = idr_alloc(&ctx->afu->contexts_idr, ctx, 0,
>   		      ctx->afu->num_procs, GFP_NOWAIT);
>   	idr_preload_end();
>   	mutex_unlock(&afu->contexts_lock);
> diff --git a/drivers/misc/cxl/cxl.h b/drivers/misc/cxl/cxl.h
> index aa453448201d..44bcfafbb579 100644
> --- a/drivers/misc/cxl/cxl.h
> +++ b/drivers/misc/cxl/cxl.h
> @@ -93,11 +93,6 @@ static const cxl_p1_reg_t CXL_PSL_FIR_CNTL  = {0x0148};
>   static const cxl_p1_reg_t CXL_PSL_DSNDCTL   = {0x0150};
>   static const cxl_p1_reg_t CXL_PSL_SNWRALLOC = {0x0158};
>   static const cxl_p1_reg_t CXL_PSL_TRACE     = {0x0170};
> -/* XSL registers (Mellanox CX4) */
> -static const cxl_p1_reg_t CXL_XSL_Timebase  = {0x0100};
> -static const cxl_p1_reg_t CXL_XSL_TB_CTLSTAT = {0x0108};
> -static const cxl_p1_reg_t CXL_XSL_FEC       = {0x0158};
> -static const cxl_p1_reg_t CXL_XSL_DSNCTL    = {0x0168};
>   /* PSL registers - CAIA 2 */
>   static const cxl_p1_reg_t CXL_PSL9_CONTROL  = {0x0020};
>   static const cxl_p1_reg_t CXL_XSL9_INV      = {0x0110};
> @@ -695,7 +690,6 @@ struct cxl {
>   	struct bin_attribute cxl_attr;
>   	int adapter_num;
>   	int user_irqs;
> -	int min_pe;
>   	u64 ps_size;
>   	u16 psl_rev;
>   	u16 base_image;
> @@ -934,7 +928,6 @@ int cxl_debugfs_afu_add(struct cxl_afu *afu);
>   void cxl_debugfs_afu_remove(struct cxl_afu *afu);
>   void cxl_debugfs_add_adapter_regs_psl9(struct cxl *adapter, struct dentry *dir);
>   void cxl_debugfs_add_adapter_regs_psl8(struct cxl *adapter, struct dentry *dir);
> -void cxl_debugfs_add_adapter_regs_xsl(struct cxl *adapter, struct dentry *dir);
>   void cxl_debugfs_add_afu_regs_psl9(struct cxl_afu *afu, struct dentry *dir);
>   void cxl_debugfs_add_afu_regs_psl8(struct cxl_afu *afu, struct dentry *dir);
>   
> @@ -977,11 +970,6 @@ static inline void cxl_debugfs_add_adapter_regs_psl8(struct cxl *adapter,
>   {
>   }
>   
> -static inline void cxl_debugfs_add_adapter_regs_xsl(struct cxl *adapter,
> -						    struct dentry *dir)
> -{
> -}
> -
>   static inline void cxl_debugfs_add_afu_regs_psl9(struct cxl_afu *afu, struct dentry *dir)
>   {
>   }
> diff --git a/drivers/misc/cxl/debugfs.c b/drivers/misc/cxl/debugfs.c
> index 1643850d2302..a1921d81593a 100644
> --- a/drivers/misc/cxl/debugfs.c
> +++ b/drivers/misc/cxl/debugfs.c
> @@ -58,11 +58,6 @@ void cxl_debugfs_add_adapter_regs_psl8(struct cxl *adapter, struct dentry *dir)
>   	debugfs_create_io_x64("trace", S_IRUSR | S_IWUSR, dir, _cxl_p1_addr(adapter, CXL_PSL_TRACE));
>   }
>   
> -void cxl_debugfs_add_adapter_regs_xsl(struct cxl *adapter, struct dentry *dir)
> -{
> -	debugfs_create_io_x64("fec", S_IRUSR, dir, _cxl_p1_addr(adapter, CXL_XSL_FEC));
> -}
> -
>   int cxl_debugfs_adapter_add(struct cxl *adapter)
>   {
>   	struct dentry *dir;
> diff --git a/drivers/misc/cxl/pci.c b/drivers/misc/cxl/pci.c
> index 0ca818396524..6dfb4ed345d3 100644
> --- a/drivers/misc/cxl/pci.c
> +++ b/drivers/misc/cxl/pci.c
> @@ -593,27 +593,7 @@ static int init_implementation_adapter_regs_psl8(struct cxl *adapter, struct pci
>   	return 0;
>   }
>   
> -static int init_implementation_adapter_regs_xsl(struct cxl *adapter, struct pci_dev *dev)
> -{
> -	u64 xsl_dsnctl;
> -	u64 chipid;
> -	u32 phb_index;
> -	u64 capp_unit_id;
> -	int rc;
> -
> -	rc = cxl_calc_capp_routing(dev, &chipid, &phb_index, &capp_unit_id);
> -	if (rc)
> -		return rc;
> -
> -	/* Tell XSL where to route data to */
> -	xsl_dsnctl = 0x0000600000000000ULL | (chipid << (63-5));
> -	xsl_dsnctl |= (capp_unit_id << (63-13));
> -	cxl_p1_write(adapter, CXL_XSL_DSNCTL, xsl_dsnctl);
> -
> -	return 0;
> -}
> -
> -/* PSL & XSL */
> +/* PSL */
>   #define TBSYNC_CAL(n) (((u64)n & 0x7) << (63-3))
>   #define TBSYNC_CNT(n) (((u64)n & 0x7) << (63-6))
>   /* For the PSL this is a multiple for 0 < n <= 7: */
> @@ -625,21 +605,6 @@ static void write_timebase_ctrl_psl8(struct cxl *adapter)
>   		     TBSYNC_CNT(2 * PSL_2048_250MHZ_CYCLES));
>   }
>   
> -/* XSL */
> -#define TBSYNC_ENA (1ULL << 63)
> -/* For the XSL this is 2**n * 2000 clocks for 0 < n <= 6: */
> -#define XSL_2000_CLOCKS 1
> -#define XSL_4000_CLOCKS 2
> -#define XSL_8000_CLOCKS 3
> -
> -static void write_timebase_ctrl_xsl(struct cxl *adapter)
> -{
> -	cxl_p1_write(adapter, CXL_XSL_TB_CTLSTAT,
> -		     TBSYNC_ENA |
> -		     TBSYNC_CAL(3) |
> -		     TBSYNC_CNT(XSL_4000_CLOCKS));
> -}
> -
>   static u64 timebase_read_psl9(struct cxl *adapter)
>   {
>   	return cxl_p1_read(adapter, CXL_PSL9_Timebase);
> @@ -650,11 +615,6 @@ static u64 timebase_read_psl8(struct cxl *adapter)
>   	return cxl_p1_read(adapter, CXL_PSL_Timebase);
>   }
>   
> -static u64 timebase_read_xsl(struct cxl *adapter)
> -{
> -	return cxl_p1_read(adapter, CXL_XSL_Timebase);
> -}
> -
>   static void cxl_setup_psl_timebase(struct cxl *adapter, struct pci_dev *dev)
>   {
>   	struct device_node *np;
> @@ -1671,37 +1631,14 @@ static const struct cxl_service_layer_ops psl8_ops = {
>   	.needs_reset_before_disable = true,
>   };
>   
> -static const struct cxl_service_layer_ops xsl_ops = {
> -	.adapter_regs_init = init_implementation_adapter_regs_xsl,
> -	.invalidate_all = cxl_invalidate_all_psl8,
> -	.sanitise_afu_regs = sanitise_afu_regs_psl8,
> -	.handle_interrupt = cxl_irq_psl8,
> -	.fail_irq = cxl_fail_irq_psl,
> -	.activate_dedicated_process = cxl_activate_dedicated_process_psl8,
> -	.attach_afu_directed = cxl_attach_afu_directed_psl8,
> -	.attach_dedicated_process = cxl_attach_dedicated_process_psl8,
> -	.update_dedicated_ivtes = cxl_update_dedicated_ivtes_psl8,
> -	.debugfs_add_adapter_regs = cxl_debugfs_add_adapter_regs_xsl,
> -	.write_timebase_ctrl = write_timebase_ctrl_xsl,
> -	.timebase_read = timebase_read_xsl,
> -	.capi_mode = OPAL_PHB_CAPI_MODE_DMA,
> -};
> -
>   static void set_sl_ops(struct cxl *adapter, struct pci_dev *dev)
>   {
> -	if (dev->vendor == PCI_VENDOR_ID_MELLANOX && dev->device == 0x1013) {
> -		/* Mellanox CX-4 */
> -		dev_info(&dev->dev, "Device uses an XSL\n");
> -		adapter->native->sl_ops = &xsl_ops;
> -		adapter->min_pe = 1; /* Workaround for CX-4 hardware bug */
> +	if (cxl_is_power8()) {
> +		dev_info(&dev->dev, "Device uses a PSL8\n");
> +		adapter->native->sl_ops = &psl8_ops;
>   	} else {
> -		if (cxl_is_power8()) {
> -			dev_info(&dev->dev, "Device uses a PSL8\n");
> -			adapter->native->sl_ops = &psl8_ops;
> -		} else {
> -			dev_info(&dev->dev, "Device uses a PSL9\n");
> -			adapter->native->sl_ops = &psl9_ops;
> -		}
> +		dev_info(&dev->dev, "Device uses a PSL9\n");
> +		adapter->native->sl_ops = &psl9_ops;
>   	}
>   }
>   
> 

-- 
Andrew Donnellan              OzLabs, ADL Canberra
andrew.donnellan@au1.ibm.com  IBM Australia Limited