[PATCH 1/5] arm64: dts: imx8-ss-hsio: fix indentation in pcie node

[PATCH 1/5] arm64: dts: imx8-ss-hsio: fix indentation in pcie node

[Frank Li]

Fix indentation in pcie node.

Signed-off-by: Frank Li <Frank.Li@nxp.com>
---
 arch/arm64/boot/dts/freescale/imx8-ss-hsio.dtsi | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/arch/arm64/boot/dts/freescale/imx8-ss-hsio.dtsi b/arch/arm64/boot/dts/freescale/imx8-ss-hsio.dtsi
index 70a8aa1a67911..635b1c801cec7 100644
--- a/arch/arm64/boot/dts/freescale/imx8-ss-hsio.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8-ss-hsio.dtsi
@@ -68,9 +68,9 @@ pcieb: pcie@5f010000 {
 		bus-range = <0x00 0xff>;
 		device_type = "pci";
 		interrupt-map = <0 0 0 1 &gic 0 105 4>,
-				 <0 0 0 2 &gic 0 106 4>,
-				 <0 0 0 3 &gic 0 107 4>,
-				 <0 0 0 4 &gic 0 108 4>;
+				<0 0 0 2 &gic 0 106 4>,
+				<0 0 0 3 &gic 0 107 4>,
+				<0 0 0 4 &gic 0 108 4>;
 		interrupt-map-mask = <0 0 0 0x7>;
 		num-lanes = <1>;
 		num-viewport = <4>;
-- 
2.34.1

[PATCH 2/5] arm64: dts: imx8q: add PCIe EP for i.MX8QM and i.MX8QXP

[Frank Li]

Add PCIe EP support for i.MX8QM and i.MX8QXP.

Signed-off-by: Frank Li <Frank.Li@nxp.com>
---
 .../boot/dts/freescale/imx8-ss-hsio.dtsi      | 19 +++++++++++++++++++
 .../boot/dts/freescale/imx8qm-ss-hsio.dtsi    | 19 +++++++++++++++++++
 2 files changed, 38 insertions(+)

diff --git a/arch/arm64/boot/dts/freescale/imx8-ss-hsio.dtsi b/arch/arm64/boot/dts/freescale/imx8-ss-hsio.dtsi
index 635b1c801cec7..8ec6df02e6381 100644
--- a/arch/arm64/boot/dts/freescale/imx8-ss-hsio.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8-ss-hsio.dtsi
@@ -79,6 +79,25 @@ pcieb: pcie@5f010000 {
 		status = "disabled";
 	};
 
+	pcieb_ep: pcie-ep@5f010000 {
+		compatible = "fsl,imx8q-pcie-ep";
+		reg = <0x5f010000 0x00010000>,
+		      <0x80000000 0x10000000>;
+		reg-names = "dbi", "addr_space";
+		num-lanes = <1>;
+		interrupts = <GIC_SPI 104 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "dma";
+		clocks = <&pcieb_lpcg IMX_LPCG_CLK_6>,
+			 <&pcieb_lpcg IMX_LPCG_CLK_4>,
+			 <&pcieb_lpcg IMX_LPCG_CLK_5>;
+		clock-names = "dbi", "mstr", "slv";
+		power-domains = <&pd IMX_SC_R_PCIE_B>;
+		fsl,max-link-speed = <3>;
+		num-ib-windows = <6>;
+		num-ob-windows = <6>;
+		status = "disabled";
+	};
+
 	pcieb_lpcg: clock-controller@5f060000 {
 		compatible = "fsl,imx8qxp-lpcg";
 		reg = <0x5f060000 0x10000>;
diff --git a/arch/arm64/boot/dts/freescale/imx8qm-ss-hsio.dtsi b/arch/arm64/boot/dts/freescale/imx8qm-ss-hsio.dtsi
index b1d0189a17258..d52609e4fc455 100644
--- a/arch/arm64/boot/dts/freescale/imx8qm-ss-hsio.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8qm-ss-hsio.dtsi
@@ -42,6 +42,25 @@ pciea: pcie@5f000000 {
 		status = "disabled";
 	};
 
+	pciea_ep: pcie-ep@5f000000 {
+		compatible = "fsl,imx8q-pcie-ep";
+		reg = <0x5f000000 0x00010000>,
+		      <0x40000000 0x10000000>;
+		reg-names = "dbi", "addr_space";
+		num-lanes = <1>;
+		interrupts = <GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "dma";
+		clocks = <&pciea_lpcg IMX_LPCG_CLK_6>,
+			 <&pciea_lpcg IMX_LPCG_CLK_4>,
+			 <&pciea_lpcg IMX_LPCG_CLK_5>;
+		clock-names = "dbi", "mstr", "slv";
+		power-domains = <&pd IMX_SC_R_PCIE_A>;
+		fsl,max-link-speed = <3>;
+		num-ib-windows = <6>;
+		num-ob-windows = <6>;
+		status = "disabled";
+	};
+
 	pcieb: pcie@5f010000 {
 		compatible = "fsl,imx8q-pcie";
 		reg = <0x5f010000 0x10000>,
-- 
2.34.1

[PATCH 3/5] arm64: dts: imx8q: add PCIe EP overlay file for i.MX8QXP mek board

[Frank Li]

Add PCIe EP overlay file for i.MX8QXP mek board to let PCI work as endpoint
mode.

Signed-off-by: Frank Li <Frank.Li@nxp.com>
---
 arch/arm64/boot/dts/freescale/Makefile        |  4 ++++
 .../dts/freescale/imx8qxp-mek-pcie-ep.dtso    | 22 +++++++++++++++++++
 2 files changed, 26 insertions(+)
 create mode 100644 arch/arm64/boot/dts/freescale/imx8qxp-mek-pcie-ep.dtso

diff --git a/arch/arm64/boot/dts/freescale/Makefile b/arch/arm64/boot/dts/freescale/Makefile
index 839432153cc7a..9bd32ec898c25 100644
--- a/arch/arm64/boot/dts/freescale/Makefile
+++ b/arch/arm64/boot/dts/freescale/Makefile
@@ -267,6 +267,10 @@ dtb-$(CONFIG_ARCH_MXC) += imx8qxp-colibri-eval-v3.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8qxp-colibri-iris.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8qxp-colibri-iris-v2.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8qxp-mek.dtb
+
+imx8qxp-mek-pcie-ep-dtbs += imx8qxp-mek.dtb imx8qxp-mek-pcie-ep.dtbo
+dtb-$(CONFIG_ARCH_MXC) += imx8qxp-mek-pcie-ep.dtb
+
 dtb-$(CONFIG_ARCH_MXC) += imx8qxp-tqma8xqp-mba8xx.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8ulp-evk.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx93-9x9-qsb.dtb
diff --git a/arch/arm64/boot/dts/freescale/imx8qxp-mek-pcie-ep.dtso b/arch/arm64/boot/dts/freescale/imx8qxp-mek-pcie-ep.dtso
new file mode 100644
index 0000000000000..4f562eb5c5b1d
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8qxp-mek-pcie-ep.dtso
@@ -0,0 +1,22 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2025 NXP
+ */
+
+#include <dt-bindings/phy/phy.h>
+
+/dts-v1/;
+/plugin/;
+
+&pcieb {
+	status = "disabled";
+};
+
+&pcieb_ep {
+	phys = <&hsio_phy 0 PHY_TYPE_PCIE 0>;
+	phy-names = "pcie-phy";
+	pinctrl-0 = <&pinctrl_pcieb>;
+	pinctrl-names = "default";
+	vpcie-supply = <&reg_pcieb>;
+	status = "okay";
+};
-- 
2.34.1

[PATCH 4/5] arm64: dts: imx95: add PCIe's msi-map and iommu-map property

[Frank Li]

Add PCIe's msi-map and iommu-map property because i.MX95 support smmu and
its.

Signed-off-by: Frank Li <Frank.Li@nxp.com>
---
 arch/arm64/boot/dts/freescale/imx95.dtsi | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/arch/arm64/boot/dts/freescale/imx95.dtsi b/arch/arm64/boot/dts/freescale/imx95.dtsi
index 6b8470cb3461a..2cebeda43a52d 100644
--- a/arch/arm64/boot/dts/freescale/imx95.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx95.dtsi
@@ -1573,6 +1573,12 @@ pcie0: pcie@4c300000 {
 			assigned-clock-parents = <0>, <0>,
 						 <&scmi_clk IMX95_CLK_SYSPLL1_PFD1_DIV2>;
 			power-domains = <&scmi_devpd IMX95_PD_HSIO_TOP>;
+			/* pcie0's Devid(BIT[7:6]) is 0x00, stream id(BIT[5:0]) is 0x10~0x17 */
+			msi-map = <0x0 &its 0x10 0x1>,
+				  <0x100 &its 0x11 0x7>;
+			iommu-map = <0x000 &smmu 0x10 0x1>,
+				    <0x100 &smmu 0x11 0x7>;
+			iommu-map-mask = <0x1ff>;
 			fsl,max-link-speed = <3>;
 			status = "disabled";
 		};
@@ -1640,6 +1646,14 @@ pcie1: pcie@4c380000 {
 			assigned-clock-parents = <0>, <0>,
 						 <&scmi_clk IMX95_CLK_SYSPLL1_PFD1_DIV2>;
 			power-domains = <&scmi_devpd IMX95_PD_HSIO_TOP>;
+			/* pcie1's Devid(BIT[7:6]) is 0x10, stream id(BIT[5:0]) is 0x18~0x1f */
+			msi-map = <0x0 &its 0x98 0x1>,
+				  <0x100 &its 0x99 0x7>;
+			msi-map-mask = <0x1ff>;
+			/* smmu have not Devid(BIT[7:6]) */
+			iommu-map = <0x000 &smmu 0x18 0x1>,
+				    <0x100 &smmu 0x19 0x7>;
+			iommu-map-mask = <0x1ff>;
 			fsl,max-link-speed = <3>;
 			status = "disabled";
 		};
-- 
2.34.1

[PATCH 5/5] arm64: dts: imx95: add ref clock for pcie nodes

[Frank Li]

Add "ref" clock for i.MX95's pcie and fix below CHECK_DTBS warnings:
arch/arm64/boot/dts/freescale/imx95-19x19-evk.dtb: pcie@4c300000: clock-names: ['pcie', 'pcie_bus', 'pcie_phy', 'pcie_aux'] is too short
	from schema $id: http://devicetree.org/schemas/pci/fsl,imx6q-pcie.yaml

Signed-off-by: Frank Li <Frank.Li@nxp.com>
---
 arch/arm64/boot/dts/freescale/imx95.dtsi | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/arch/arm64/boot/dts/freescale/imx95.dtsi b/arch/arm64/boot/dts/freescale/imx95.dtsi
index 2cebeda43a52d..f0dc8be2abffa 100644
--- a/arch/arm64/boot/dts/freescale/imx95.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx95.dtsi
@@ -1537,6 +1537,14 @@ smmu: iommu@490d0000 {
 			};
 		};
 
+		hsio_blk_ctl: syscon@4c0100c0 {
+			compatible = "nxp,imx95-hsio-blk-ctl", "syscon";
+			reg = <0x0 0x4c0100c0 0x0 0x1>;
+			#clock-cells = <1>;
+			power-domains = <&scmi_devpd IMX95_PD_HSIO_TOP>;
+			clocks = <&dummy>;
+		};
+
 		pcie0: pcie@4c300000 {
 			compatible = "fsl,imx95-pcie";
 			reg = <0 0x4c300000 0 0x10000>,
@@ -1564,8 +1572,9 @@ pcie0: pcie@4c300000 {
 			clocks = <&scmi_clk IMX95_CLK_HSIO>,
 				 <&scmi_clk IMX95_CLK_HSIOPLL>,
 				 <&scmi_clk IMX95_CLK_HSIOPLL_VCO>,
-				 <&scmi_clk IMX95_CLK_HSIOPCIEAUX>;
-			clock-names = "pcie", "pcie_bus", "pcie_phy", "pcie_aux";
+				 <&scmi_clk IMX95_CLK_HSIOPCIEAUX>,
+				 <&hsio_blk_ctl 0>;
+			clock-names = "pcie", "pcie_bus", "pcie_phy", "pcie_aux", "ref";
 			assigned-clocks =<&scmi_clk IMX95_CLK_HSIOPLL_VCO>,
 					 <&scmi_clk IMX95_CLK_HSIOPLL>,
 					 <&scmi_clk IMX95_CLK_HSIOPCIEAUX>;
@@ -1637,8 +1646,9 @@ pcie1: pcie@4c380000 {
 			clocks = <&scmi_clk IMX95_CLK_HSIO>,
 				 <&scmi_clk IMX95_CLK_HSIOPLL>,
 				 <&scmi_clk IMX95_CLK_HSIOPLL_VCO>,
-				 <&scmi_clk IMX95_CLK_HSIOPCIEAUX>;
-			clock-names = "pcie", "pcie_bus", "pcie_phy", "pcie_aux";
+				 <&scmi_clk IMX95_CLK_HSIOPCIEAUX>,
+				 <&hsio_blk_ctl 0>;
+			clock-names = "pcie", "pcie_bus", "pcie_phy", "pcie_aux", "ref";
 			assigned-clocks =<&scmi_clk IMX95_CLK_HSIOPLL_VCO>,
 					 <&scmi_clk IMX95_CLK_HSIOPLL>,
 					 <&scmi_clk IMX95_CLK_HSIOPCIEAUX>;
-- 
2.34.1

Re: [PATCH 1/5] arm64: dts: imx8-ss-hsio: fix indentation in pcie node

[Rob Herring (Arm)]

On Tue, 28 Jan 2025 16:15:55 -0500, Frank Li wrote:
> Fix indentation in pcie node.
> 
> Signed-off-by: Frank Li <Frank.Li@nxp.com>
> ---
>  arch/arm64/boot/dts/freescale/imx8-ss-hsio.dtsi | 6 +++---
>  1 file changed, 3 insertions(+), 3 deletions(-)
> 


My bot found new DTB warnings on the .dts files added or changed in this
series.

Some warnings may be from an existing SoC .dtsi. Or perhaps the warnings
are fixed by another series. Ultimately, it is up to the platform
maintainer whether these warnings are acceptable or not. No need to reply
unless the platform maintainer has comments.

If you already ran DT checks and didn't see these error(s), then
make sure dt-schema is up to date:

  pip3 install dtschema --upgrade


New warnings running 'make CHECK_DTBS=y for arch/arm64/boot/dts/freescale/' for 20250128211559.1582598-1-Frank.Li@nxp.com:

arch/arm64/boot/dts/freescale/imx8qxp-ai_ml.dtb: /bus@5f000000/pcie-ep@5f010000: failed to match any schema with compatible: ['fsl,imx8q-pcie-ep']
arch/arm64/boot/dts/freescale/imx8dx-colibri-iris-v2.dtb: /bus@5f000000/pcie-ep@5f010000: failed to match any schema with compatible: ['fsl,imx8q-pcie-ep']
arch/arm64/boot/dts/freescale/imx8qxp-tqma8xqp-mba8xx.dtb: /bus@5f000000/pcie-ep@5f010000: failed to match any schema with compatible: ['fsl,imx8q-pcie-ep']
arch/arm64/boot/dts/freescale/imx8dxl-evk.dtb: /bus@5f000000/pcie-ep@5f010000: failed to match any schema with compatible: ['fsl,imx8q-pcie-ep']
arch/arm64/boot/dts/freescale/imx8dx-colibri-aster.dtb: /bus@5f000000/pcie-ep@5f010000: failed to match any schema with compatible: ['fsl,imx8q-pcie-ep']
arch/arm64/boot/dts/freescale/imx8qxp-colibri-iris.dtb: /bus@5f000000/pcie-ep@5f010000: failed to match any schema with compatible: ['fsl,imx8q-pcie-ep']
arch/arm64/boot/dts/freescale/imx8dx-colibri-eval-v3.dtb: /bus@5f000000/pcie-ep@5f010000: failed to match any schema with compatible: ['fsl,imx8q-pcie-ep']
arch/arm64/boot/dts/freescale/imx8qm-apalis-v1.1-eval.dtb: /bus@5f000000/pcie-ep@5f010000: failed to match any schema with compatible: ['fsl,imx8q-pcie-ep']
arch/arm64/boot/dts/freescale/imx8qm-apalis-v1.1-eval.dtb: /bus@5f000000/pcie-ep@5f000000: failed to match any schema with compatible: ['fsl,imx8q-pcie-ep']
arch/arm64/boot/dts/freescale/imx8dx-colibri-iris.dtb: /bus@5f000000/pcie-ep@5f010000: failed to match any schema with compatible: ['fsl,imx8q-pcie-ep']
arch/arm64/boot/dts/freescale/imx8qm-apalis-v1.1-ixora-v1.2.dtb: /bus@5f000000/pcie-ep@5f010000: failed to match any schema with compatible: ['fsl,imx8q-pcie-ep']
arch/arm64/boot/dts/freescale/imx8qm-apalis-v1.1-ixora-v1.2.dtb: /bus@5f000000/pcie-ep@5f000000: failed to match any schema with compatible: ['fsl,imx8q-pcie-ep']
arch/arm64/boot/dts/freescale/imx95-19x19-evk.dtb: pcie@4c300000: clocks: [[27, 87], [27, 36], [27, 35], [27, 88], [61, 0]] is too long
	from schema $id: http://devicetree.org/schemas/pci/fsl,imx6q-pcie.yaml#
arch/arm64/boot/dts/freescale/imx95-19x19-evk.dtb: pcie@4c300000: clock-names: ['pcie', 'pcie_bus', 'pcie_phy', 'pcie_aux', 'ref'] is too long
	from schema $id: http://devicetree.org/schemas/pci/fsl,imx6q-pcie.yaml#
arch/arm64/boot/dts/freescale/imx95-19x19-evk.dtb: pcie@4c300000: clocks: [[27, 87], [27, 36], [27, 35], [27, 88], [61, 0]] is too long
	from schema $id: http://devicetree.org/schemas/pci/fsl,imx6q-pcie.yaml#
arch/arm64/boot/dts/freescale/imx95-19x19-evk.dtb: pcie@4c300000: clock-names: ['pcie', 'pcie_bus', 'pcie_phy', 'pcie_aux', 'ref'] is too long
	from schema $id: http://devicetree.org/schemas/pci/fsl,imx6q-pcie.yaml#
arch/arm64/boot/dts/freescale/imx95-19x19-evk.dtb: pcie@4c300000: clock-names: ['pcie', 'pcie_bus', 'pcie_phy', 'pcie_aux', 'ref'] is too long
	from schema $id: http://devicetree.org/schemas/pci/fsl,imx6q-pcie.yaml#
arch/arm64/boot/dts/freescale/imx95-19x19-evk.dtb: pcie@4c300000: Unevaluated properties are not allowed ('fsl,max-link-speed', 'power-domains', 'vpcie-supply' were unexpected)
	from schema $id: http://devicetree.org/schemas/pci/fsl,imx6q-pcie.yaml#
arch/arm64/boot/dts/freescale/imx95-19x19-evk.dtb: pcie@4c380000: clocks: [[27, 87], [27, 36], [27, 35], [27, 88], [61, 0]] is too long
	from schema $id: http://devicetree.org/schemas/pci/fsl,imx6q-pcie.yaml#
arch/arm64/boot/dts/freescale/imx95-19x19-evk.dtb: pcie@4c380000: clock-names: ['pcie', 'pcie_bus', 'pcie_phy', 'pcie_aux', 'ref'] is too long
	from schema $id: http://devicetree.org/schemas/pci/fsl,imx6q-pcie.yaml#
arch/arm64/boot/dts/freescale/imx95-19x19-evk.dtb: pcie@4c380000: clocks: [[27, 87], [27, 36], [27, 35], [27, 88], [61, 0]] is too long
	from schema $id: http://devicetree.org/schemas/pci/fsl,imx6q-pcie.yaml#
arch/arm64/boot/dts/freescale/imx95-19x19-evk.dtb: pcie@4c380000: clock-names: ['pcie', 'pcie_bus', 'pcie_phy', 'pcie_aux', 'ref'] is too long
	from schema $id: http://devicetree.org/schemas/pci/fsl,imx6q-pcie.yaml#
arch/arm64/boot/dts/freescale/imx95-19x19-evk.dtb: pcie@4c380000: clock-names: ['pcie', 'pcie_bus', 'pcie_phy', 'pcie_aux', 'ref'] is too long
	from schema $id: http://devicetree.org/schemas/pci/fsl,imx6q-pcie.yaml#
arch/arm64/boot/dts/freescale/imx95-19x19-evk.dtb: pcie@4c380000: Unevaluated properties are not allowed ('fsl,max-link-speed', 'power-domains', 'vpcie-supply' were unexpected)
	from schema $id: http://devicetree.org/schemas/pci/fsl,imx6q-pcie.yaml#
arch/arm64/boot/dts/freescale/imx8qm-mek.dtb: /bus@5f000000/pcie-ep@5f010000: failed to match any schema with compatible: ['fsl,imx8q-pcie-ep']
arch/arm64/boot/dts/freescale/imx8qm-apalis-v1.1-eval-v1.2.dtb: /bus@5f000000/pcie-ep@5f010000: failed to match any schema with compatible: ['fsl,imx8q-pcie-ep']
arch/arm64/boot/dts/freescale/imx8qm-apalis-v1.1-eval-v1.2.dtb: /bus@5f000000/pcie-ep@5f000000: failed to match any schema with compatible: ['fsl,imx8q-pcie-ep']
arch/arm64/boot/dts/freescale/imx8qm-mek.dtb: /bus@5f000000/pcie-ep@5f000000: failed to match any schema with compatible: ['fsl,imx8q-pcie-ep']
arch/arm64/boot/dts/freescale/imx8qm-apalis-eval.dtb: /bus@5f000000/pcie-ep@5f010000: failed to match any schema with compatible: ['fsl,imx8q-pcie-ep']
arch/arm64/boot/dts/freescale/imx8qm-apalis-eval.dtb: /bus@5f000000/pcie-ep@5f000000: failed to match any schema with compatible: ['fsl,imx8q-pcie-ep']
arch/arm64/boot/dts/freescale/imx8qm-apalis-v1.1-ixora-v1.1.dtb: /bus@5f000000/pcie-ep@5f010000: failed to match any schema with compatible: ['fsl,imx8q-pcie-ep']
arch/arm64/boot/dts/freescale/imx8qm-apalis-v1.1-ixora-v1.1.dtb: /bus@5f000000/pcie-ep@5f000000: failed to match any schema with compatible: ['fsl,imx8q-pcie-ep']
arch/arm64/boot/dts/freescale/imx8qm-apalis-ixora-v1.1.dtb: /bus@5f000000/pcie-ep@5f010000: failed to match any schema with compatible: ['fsl,imx8q-pcie-ep']
arch/arm64/boot/dts/freescale/imx8qm-apalis-ixora-v1.1.dtb: /bus@5f000000/pcie-ep@5f000000: failed to match any schema with compatible: ['fsl,imx8q-pcie-ep']
arch/arm64/boot/dts/freescale/imx8qxp-colibri-iris-v2.dtb: /bus@5f000000/pcie-ep@5f010000: failed to match any schema with compatible: ['fsl,imx8q-pcie-ep']
arch/arm64/boot/dts/freescale/imx8dxp-tqma8xdp-mba8xx.dtb: /bus@5f000000/pcie-ep@5f010000: failed to match any schema with compatible: ['fsl,imx8q-pcie-ep']
arch/arm64/boot/dts/freescale/imx8qxp-colibri-eval-v3.dtb: /bus@5f000000/pcie-ep@5f010000: failed to match any schema with compatible: ['fsl,imx8q-pcie-ep']
arch/arm64/boot/dts/freescale/imx8qm-apalis-eval-v1.2.dtb: /bus@5f000000/pcie-ep@5f010000: failed to match any schema with compatible: ['fsl,imx8q-pcie-ep']
arch/arm64/boot/dts/freescale/imx8qm-apalis-eval-v1.2.dtb: /bus@5f000000/pcie-ep@5f000000: failed to match any schema with compatible: ['fsl,imx8q-pcie-ep']
arch/arm64/boot/dts/freescale/imx8qxp-colibri-aster.dtb: /bus@5f000000/pcie-ep@5f010000: failed to match any schema with compatible: ['fsl,imx8q-pcie-ep']
arch/arm64/boot/dts/freescale/imx8qxp-mek.dtb: /bus@5f000000/pcie-ep@5f010000: failed to match any schema with compatible: ['fsl,imx8q-pcie-ep']

Re: [PATCH 5/5] arm64: dts: imx95: add ref clock for pcie nodes

[Shawn Guo]

On Tue, Jan 28, 2025 at 04:15:59PM -0500, Frank Li wrote:
> Add "ref" clock for i.MX95's pcie and fix below CHECK_DTBS warnings:
> arch/arm64/boot/dts/freescale/imx95-19x19-evk.dtb: pcie@4c300000: clock-names: ['pcie', 'pcie_bus', 'pcie_phy', 'pcie_aux'] is too short
> 	from schema $id: http://devicetree.org/schemas/pci/fsl,imx6q-pcie.yaml
> 
> Signed-off-by: Frank Li <Frank.Li@nxp.com>

#1 ~ #4 are applied and #5 doesn't apply.

Shawn

RE: [PATCH 5/5] arm64: dts: imx95: add ref clock for pcie nodes

[Hongxing Zhu]

> -----Original Message-----
> From: Shawn Guo <shawnguo2@yeah.net>
> Sent: 2025年2月22日 23:00
> To: Frank Li <frank.li@nxp.com>
> Cc: Rob Herring <robh@kernel.org>; Krzysztof Kozlowski <krzk+dt@kernel.org>;
> Conor Dooley <conor+dt@kernel.org>; Shawn Guo <shawnguo@kernel.org>;
> Sascha Hauer <s.hauer@pengutronix.de>; Pengutronix Kernel Team
> <kernel@pengutronix.de>; Fabio Estevam <festevam@gmail.com>; open
> list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS
> <devicetree@vger.kernel.org>; open list:ARM/FREESCALE IMX / MXC ARM
> ARCHITECTURE <imx@lists.linux.dev>; moderated list:ARM/FREESCALE IMX /
> MXC ARM ARCHITECTURE <linux-arm-kernel@lists.infradead.org>; open list
> <linux-kernel@vger.kernel.org>; Hongxing Zhu <hongxing.zhu@nxp.com>
> Subject: Re: [PATCH 5/5] arm64: dts: imx95: add ref clock for pcie nodes
> 
> On Tue, Jan 28, 2025 at 04:15:59PM -0500, Frank Li wrote:
> > Add "ref" clock for i.MX95's pcie and fix below CHECK_DTBS warnings:
> > arch/arm64/boot/dts/freescale/imx95-19x19-evk.dtb: pcie@4c300000:
> clock-names: ['pcie', 'pcie_bus', 'pcie_phy', 'pcie_aux'] is too short
> > 	from schema $id:
> https://eur01.safelinks.protection.outlook.com/?url=http%3A%2F%2Fdevicetree
> .org%2Fschemas%2Fpci%2Ffsl%2Cimx6q-pcie.yaml&data=05%7C02%7Chongxin
> g.zhu%40nxp.com%7Cf373e5ed1a6b4c7aefc908dd5351a620%7C686ea1d3bc2
> b4c6fa92cd99c5c301635%7C0%7C0%7C638758332322731937%7CUnknown
> %7CTWFpbGZsb3d8eyJFbXB0eU1hcGkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIlAiOi
> JXaW4zMiIsIkFOIjoiTWFpbCIsIldUIjoyfQ%3D%3D%7C0%7C%7C%7C&sdata=glq
> vwWeavp1SMo6%2F8rZ%2FbGMXgJHCeYPYIZVW3vkTFHs%3D&reserved=0
> >
> > Signed-off-by: Frank Li <Frank.Li@nxp.com>
> 
> #1 ~ #4 are applied and #5 doesn't apply.
Hi Shawn:
Can you help to take the last one dts patch in one patch-set below instead?
https://patchwork.kernel.org/project/linux-pci/patch/20241126075702.4099164-11-hongxing.zhu@nxp.com/
Thanks in advanced.
BTW, the others had been merged in PCIe git tree.

Best Regards
Richard Zhu
> 
> Shawn

Re: [PATCH 5/5] arm64: dts: imx95: add ref clock for pcie nodes

[Frank Li]

On Mon, Feb 24, 2025 at 02:10:40AM +0000, Hongxing Zhu wrote:
> > -----Original Message-----
> > From: Shawn Guo <shawnguo2@yeah.net>
> > Sent: 2025年2月22日 23:00
> > To: Frank Li <frank.li@nxp.com>
> > Cc: Rob Herring <robh@kernel.org>; Krzysztof Kozlowski <krzk+dt@kernel.org>;
> > Conor Dooley <conor+dt@kernel.org>; Shawn Guo <shawnguo@kernel.org>;
> > Sascha Hauer <s.hauer@pengutronix.de>; Pengutronix Kernel Team
> > <kernel@pengutronix.de>; Fabio Estevam <festevam@gmail.com>; open
> > list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS
> > <devicetree@vger.kernel.org>; open list:ARM/FREESCALE IMX / MXC ARM
> > ARCHITECTURE <imx@lists.linux.dev>; moderated list:ARM/FREESCALE IMX /
> > MXC ARM ARCHITECTURE <linux-arm-kernel@lists.infradead.org>; open list
> > <linux-kernel@vger.kernel.org>; Hongxing Zhu <hongxing.zhu@nxp.com>
> > Subject: Re: [PATCH 5/5] arm64: dts: imx95: add ref clock for pcie nodes
> >
> > On Tue, Jan 28, 2025 at 04:15:59PM -0500, Frank Li wrote:
> > > Add "ref" clock for i.MX95's pcie and fix below CHECK_DTBS warnings:
> > > arch/arm64/boot/dts/freescale/imx95-19x19-evk.dtb: pcie@4c300000:
> > clock-names: ['pcie', 'pcie_bus', 'pcie_phy', 'pcie_aux'] is too short
> > > 	from schema $id:
> > https://eur01.safelinks.protection.outlook.com/?url=http%3A%2F%2Fdevicetree
> > .org%2Fschemas%2Fpci%2Ffsl%2Cimx6q-pcie.yaml&data=05%7C02%7Chongxin
> > g.zhu%40nxp.com%7Cf373e5ed1a6b4c7aefc908dd5351a620%7C686ea1d3bc2
> > b4c6fa92cd99c5c301635%7C0%7C0%7C638758332322731937%7CUnknown
> > %7CTWFpbGZsb3d8eyJFbXB0eU1hcGkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIlAiOi
> > JXaW4zMiIsIkFOIjoiTWFpbCIsIldUIjoyfQ%3D%3D%7C0%7C%7C%7C&sdata=glq
> > vwWeavp1SMo6%2F8rZ%2FbGMXgJHCeYPYIZVW3vkTFHs%3D&reserved=0
> > >
> > > Signed-off-by: Frank Li <Frank.Li@nxp.com>
> >
> > #1 ~ #4 are applied and #5 doesn't apply.
> Hi Shawn:
> Can you help to take the last one dts patch in one patch-set below instead?
> https://patchwork.kernel.org/project/linux-pci/patch/20241126075702.4099164-11-hongxing.zhu@nxp.com/
> Thanks in advanced.
> BTW, the others had been merged in PCIe git tree.

Richard:

	Still can't apply with your patch because usb3.0 nodes impact this.
	I resend my patch because it is easy to fix conflict and apply your
125mHz input part.

	https://lore.kernel.org/imx/20250224170751.146840-1-Frank.Li@nxp.com/T/#u

Frank

>
> Best Regards
> Richard Zhu
> >
> > Shawn
>

RE: [PATCH 5/5] arm64: dts: imx95: add ref clock for pcie nodes

[Hongxing Zhu]

> -----Original Message-----
> From: Frank Li <frank.li@nxp.com>
> Sent: 2025年2月25日 1:12
> To: Hongxing Zhu <hongxing.zhu@nxp.com>
> Cc: Shawn Guo <shawnguo2@yeah.net>; Rob Herring <robh@kernel.org>;
> Krzysztof Kozlowski <krzk+dt@kernel.org>; Conor Dooley
> <conor+dt@kernel.org>; Shawn Guo <shawnguo@kernel.org>; Sascha Hauer
> <s.hauer@pengutronix.de>; Pengutronix Kernel Team <kernel@pengutronix.de>;
> Fabio Estevam <festevam@gmail.com>; open list:OPEN FIRMWARE AND
> FLATTENED DEVICE TREE BINDINGS <devicetree@vger.kernel.org>; open
> list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE <imx@lists.linux.dev>;
> moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE
> <linux-arm-kernel@lists.infradead.org>; open list <linux-kernel@vger.kernel.org>
> Subject: Re: [PATCH 5/5] arm64: dts: imx95: add ref clock for pcie nodes
> 
> On Mon, Feb 24, 2025 at 02:10:40AM +0000, Hongxing Zhu wrote:
> > > -----Original Message-----
> > > From: Shawn Guo <shawnguo2@yeah.net>
> > > Sent: 2025年2月22日 23:00
> > > To: Frank Li <frank.li@nxp.com>
> > > Cc: Rob Herring <robh@kernel.org>; Krzysztof Kozlowski
> > > <krzk+dt@kernel.org>; Conor Dooley <conor+dt@kernel.org>; Shawn Guo
> > > <shawnguo@kernel.org>; Sascha Hauer <s.hauer@pengutronix.de>;
> > > Pengutronix Kernel Team <kernel@pengutronix.de>; Fabio Estevam
> > > <festevam@gmail.com>; open list:OPEN FIRMWARE AND FLATTENED DEVICE
> > > TREE BINDINGS <devicetree@vger.kernel.org>; open list:ARM/FREESCALE
> > > IMX / MXC ARM ARCHITECTURE <imx@lists.linux.dev>; moderated
> > > list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE
> > > <linux-arm-kernel@lists.infradead.org>; open list
> > > <linux-kernel@vger.kernel.org>; Hongxing Zhu <hongxing.zhu@nxp.com>
> > > Subject: Re: [PATCH 5/5] arm64: dts: imx95: add ref clock for pcie
> > > nodes
> > >
> > > On Tue, Jan 28, 2025 at 04:15:59PM -0500, Frank Li wrote:
> > > > Add "ref" clock for i.MX95's pcie and fix below CHECK_DTBS warnings:
> > > > arch/arm64/boot/dts/freescale/imx95-19x19-evk.dtb: pcie@4c300000:
> > > clock-names: ['pcie', 'pcie_bus', 'pcie_phy', 'pcie_aux'] is too
> > > short
> > > > 	from schema $id:
> > > https://eur01.safelinks.protection.outlook.com/?url=http%3A%2F%2Fdev
> > > icetree
> > > .org%2Fschemas%2Fpci%2Ffsl%2Cimx6q-pcie.yaml&data=05%7C02%7Chon
> gxin
> > >
> g.zhu%40nxp.com%7Cf373e5ed1a6b4c7aefc908dd5351a620%7C686ea1d3bc2
> > >
> b4c6fa92cd99c5c301635%7C0%7C0%7C638758332322731937%7CUnknown
> > > %7CTWFpbGZsb3d8eyJFbXB0eU1hcGkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIl
> AiOi
> > >
> JXaW4zMiIsIkFOIjoiTWFpbCIsIldUIjoyfQ%3D%3D%7C0%7C%7C%7C&sdata=glq
> > > vwWeavp1SMo6%2F8rZ%2FbGMXgJHCeYPYIZVW3vkTFHs%3D&reserved=0
> > > >
> > > > Signed-off-by: Frank Li <Frank.Li@nxp.com>
> > >
> > > #1 ~ #4 are applied and #5 doesn't apply.
> > Hi Shawn:
> > Can you help to take the last one dts patch in one patch-set below instead?
> > https://patchwork.kernel.org/project/linux-pci/patch/20241126075702.40
> > 99164-11-hongxing.zhu@nxp.com/
> > Thanks in advanced.
> > BTW, the others had been merged in PCIe git tree.
> 
> Richard:
> 
> 	Still can't apply with your patch because usb3.0 nodes impact this.
> 	I resend my patch because it is easy to fix conflict and apply your 125mHz
> input part.
> 
> 	https://lore.kernel.org/imx/20250224170751.146840-1-Frank.Li@nxp.co
> m/T/#u
Okay.


Best Regards
Richard Zhu
> 
> Frank
> 
> >
> > Best Regards
> > Richard Zhu
> > >
> > > Shawn
> >

Re: [PATCH 4/5] arm64: dts: imx95: add PCIe's msi-map and iommu-map property

[Alexander Stein]

Hi Frank,

Am Dienstag, 28. Januar 2025, 22:15:58 CET schrieb Frank Li:
> Add PCIe's msi-map and iommu-map property because i.MX95 support smmu and
> its.
> 
> Signed-off-by: Frank Li <Frank.Li@nxp.com>
> ---
>  arch/arm64/boot/dts/freescale/imx95.dtsi | 14 ++++++++++++++
>  1 file changed, 14 insertions(+)
> 
> diff --git a/arch/arm64/boot/dts/freescale/imx95.dtsi b/arch/arm64/boot/dts/freescale/imx95.dtsi
> index 6b8470cb3461a..2cebeda43a52d 100644
> --- a/arch/arm64/boot/dts/freescale/imx95.dtsi
> +++ b/arch/arm64/boot/dts/freescale/imx95.dtsi
> @@ -1573,6 +1573,12 @@ pcie0: pcie@4c300000 {
>  			assigned-clock-parents = <0>, <0>,
>  						 <&scmi_clk IMX95_CLK_SYSPLL1_PFD1_DIV2>;
>  			power-domains = <&scmi_devpd IMX95_PD_HSIO_TOP>;
> +			/* pcie0's Devid(BIT[7:6]) is 0x00, stream id(BIT[5:0]) is 0x10~0x17 */
> +			msi-map = <0x0 &its 0x10 0x1>,
> +				  <0x100 &its 0x11 0x7>;

Aren't you missing msi-map-mask = <0x1ff>; here? Similar to pcie1.
Either way, with this change PCIe on pcie0 is not working anymore,
regardless of msi-map-mask.

Without msi-map-mask:
> OF: /soc/pcie@4c300000: iommu-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> OF: /soc/pcie@4c300000: no msi-map translation for id 0x300 on (null)
> r8169 0000:03:00.0: error -EINVAL: enable failure
> r8169 0000:03:00.0: probe with driver r8169 failed with error -22

With msi-map-mask:
> OF: /soc/pcie@4c300000: iommu-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> OF: /soc/pcie@4c300000: msi-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> r8169 0000:03:00.0: enabling device (0000 -> 0003)
> r8169 0000:03:00.0: enabling Mem-Wr-Inval
> r8169 0000:03:00.0: error -EIO: PCI read failed
> r8169 0000:03:00.0: probe with driver r8169 failed with error -5

Without msi-map/iommu-map:
> r8169 0000:03:00.0: enabling device (0000 -> 0003)
> r8169 0000:03:00.0: enabling Mem-Wr-Inval
> r8169 0000:03:00.0 eth0: RTL8168g/8111g, d8:9d:b9:00:16:10, XID 4c0, IRQ 166
> r8169 0000:03:00.0 eth0: jumbo features [frames: 9194 bytes, tx checksumming: ko]
> r8169 0000:03:00.0 enp3s0: renamed from eth0
> r8169 0000:03:00.0: enabling bus mastering
> r8169 0000:03:00.0 enp3s0: Link is Down

pcie1 works as expected. But this is only a single PCIe device, rather than
having a PCIe bridge.
Any idea what's wrong here?

Best regards,
Alexander

> +			iommu-map = <0x000 &smmu 0x10 0x1>,
> +				    <0x100 &smmu 0x11 0x7>;
> +			iommu-map-mask = <0x1ff>;
>  			fsl,max-link-speed = <3>;
>  			status = "disabled";
>  		};
> @@ -1640,6 +1646,14 @@ pcie1: pcie@4c380000 {
>  			assigned-clock-parents = <0>, <0>,
>  						 <&scmi_clk IMX95_CLK_SYSPLL1_PFD1_DIV2>;
>  			power-domains = <&scmi_devpd IMX95_PD_HSIO_TOP>;
> +			/* pcie1's Devid(BIT[7:6]) is 0x10, stream id(BIT[5:0]) is 0x18~0x1f */
> +			msi-map = <0x0 &its 0x98 0x1>,
> +				  <0x100 &its 0x99 0x7>;
> +			msi-map-mask = <0x1ff>;
> +			/* smmu have not Devid(BIT[7:6]) */
> +			iommu-map = <0x000 &smmu 0x18 0x1>,
> +				    <0x100 &smmu 0x19 0x7>;
> +			iommu-map-mask = <0x1ff>;
>  			fsl,max-link-speed = <3>;
>  			status = "disabled";
>  		};
> 


-- 
TQ-Systems GmbH | Mühlstraße 2, Gut Delling | 82229 Seefeld, Germany
Amtsgericht München, HRB 105018
Geschäftsführer: Detlef Schneider, Rüdiger Stahl, Stefan Schneider
http://www.tq-group.com/

Re: [PATCH 4/5] arm64: dts: imx95: add PCIe's msi-map and iommu-map property

[Frank Li]

On Wed, Feb 26, 2025 at 01:11:37PM +0100, Alexander Stein wrote:
> Hi Frank,
>
> Am Dienstag, 28. Januar 2025, 22:15:58 CET schrieb Frank Li:
> > Add PCIe's msi-map and iommu-map property because i.MX95 support smmu and
> > its.
> >
> > Signed-off-by: Frank Li <Frank.Li@nxp.com>
> > ---
> >  arch/arm64/boot/dts/freescale/imx95.dtsi | 14 ++++++++++++++
> >  1 file changed, 14 insertions(+)
> >
> > diff --git a/arch/arm64/boot/dts/freescale/imx95.dtsi b/arch/arm64/boot/dts/freescale/imx95.dtsi
> > index 6b8470cb3461a..2cebeda43a52d 100644
> > --- a/arch/arm64/boot/dts/freescale/imx95.dtsi
> > +++ b/arch/arm64/boot/dts/freescale/imx95.dtsi
> > @@ -1573,6 +1573,12 @@ pcie0: pcie@4c300000 {
> >  			assigned-clock-parents = <0>, <0>,
> >  						 <&scmi_clk IMX95_CLK_SYSPLL1_PFD1_DIV2>;
> >  			power-domains = <&scmi_devpd IMX95_PD_HSIO_TOP>;
> > +			/* pcie0's Devid(BIT[7:6]) is 0x00, stream id(BIT[5:0]) is 0x10~0x17 */
> > +			msi-map = <0x0 &its 0x10 0x1>,
> > +				  <0x100 &its 0x11 0x7>;
>
> Aren't you missing msi-map-mask = <0x1ff>; here? Similar to pcie1.
> Either way, with this change PCIe on pcie0 is not working anymore,
> regardless of msi-map-mask.

Yes, it should have msi-map-mask. During my test, I have not enable enetc
so I have not found this problem.

>
> Without msi-map-mask:
> > OF: /soc/pcie@4c300000: iommu-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> > OF: /soc/pcie@4c300000: no msi-map translation for id 0x300 on (null)
> > r8169 0000:03:00.0: error -EINVAL: enable failure
> > r8169 0000:03:00.0: probe with driver r8169 failed with error -22
>
> With msi-map-mask:
> > OF: /soc/pcie@4c300000: iommu-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> > OF: /soc/pcie@4c300000: msi-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> > r8169 0000:03:00.0: enabling device (0000 -> 0003)
> > r8169 0000:03:00.0: enabling Mem-Wr-Inval
> > r8169 0000:03:00.0: error -EIO: PCI read failed
> > r8169 0000:03:00.0: probe with driver r8169 failed with error -5

Can you try remove iommu-map and keep msi-map? then remove msi-map and
keep iommu-map to check which one cause this problem.

>
> Without msi-map/iommu-map:
> > r8169 0000:03:00.0: enabling device (0000 -> 0003)
> > r8169 0000:03:00.0: enabling Mem-Wr-Inval
> > r8169 0000:03:00.0 eth0: RTL8168g/8111g, d8:9d:b9:00:16:10, XID 4c0, IRQ 166
> > r8169 0000:03:00.0 eth0: jumbo features [frames: 9194 bytes, tx checksumming: ko]
> > r8169 0000:03:00.0 enp3s0: renamed from eth0
> > r8169 0000:03:00.0: enabling bus mastering
> > r8169 0000:03:00.0 enp3s0: Link is Down
>
> pcie1 works as expected. But this is only a single PCIe device, rather than
> having a PCIe bridge.
> Any idea what's wrong here?

Can you help dump more information at for PCIe bridge case:

imx_pcie_add_lut(), need rid and sid information.
drivers/pci/controller/dwc/pci-imx6.c

>
> Best regards,
> Alexander
>
> > +			iommu-map = <0x000 &smmu 0x10 0x1>,
> > +				    <0x100 &smmu 0x11 0x7>;
> > +			iommu-map-mask = <0x1ff>;
> >  			fsl,max-link-speed = <3>;
> >  			status = "disabled";
> >  		};
> > @@ -1640,6 +1646,14 @@ pcie1: pcie@4c380000 {
> >  			assigned-clock-parents = <0>, <0>,
> >  						 <&scmi_clk IMX95_CLK_SYSPLL1_PFD1_DIV2>;
> >  			power-domains = <&scmi_devpd IMX95_PD_HSIO_TOP>;
> > +			/* pcie1's Devid(BIT[7:6]) is 0x10, stream id(BIT[5:0]) is 0x18~0x1f */
> > +			msi-map = <0x0 &its 0x98 0x1>,
> > +				  <0x100 &its 0x99 0x7>;
> > +			msi-map-mask = <0x1ff>;
> > +			/* smmu have not Devid(BIT[7:6]) */
> > +			iommu-map = <0x000 &smmu 0x18 0x1>,
> > +				    <0x100 &smmu 0x19 0x7>;
> > +			iommu-map-mask = <0x1ff>;
> >  			fsl,max-link-speed = <3>;
> >  			status = "disabled";
> >  		};
> >
>
>
> --
> TQ-Systems GmbH | Mühlstraße 2, Gut Delling | 82229 Seefeld, Germany
> Amtsgericht München, HRB 105018
> Geschäftsführer: Detlef Schneider, Rüdiger Stahl, Stefan Schneider
> http://www.tq-group.com/
>
>

Re: [PATCH 4/5] arm64: dts: imx95: add PCIe's msi-map and iommu-map property

[Frank Li]

On Wed, Feb 26, 2025 at 11:31:26AM -0500, Frank Li wrote:
> On Wed, Feb 26, 2025 at 01:11:37PM +0100, Alexander Stein wrote:
> > Hi Frank,
> >
> > Am Dienstag, 28. Januar 2025, 22:15:58 CET schrieb Frank Li:
> > > Add PCIe's msi-map and iommu-map property because i.MX95 support smmu and
> > > its.
> > >
> > > Signed-off-by: Frank Li <Frank.Li@nxp.com>
> > > ---
> > >  arch/arm64/boot/dts/freescale/imx95.dtsi | 14 ++++++++++++++
> > >  1 file changed, 14 insertions(+)
> > >
> > > diff --git a/arch/arm64/boot/dts/freescale/imx95.dtsi b/arch/arm64/boot/dts/freescale/imx95.dtsi
> > > index 6b8470cb3461a..2cebeda43a52d 100644
> > > --- a/arch/arm64/boot/dts/freescale/imx95.dtsi
> > > +++ b/arch/arm64/boot/dts/freescale/imx95.dtsi
> > > @@ -1573,6 +1573,12 @@ pcie0: pcie@4c300000 {
> > >  			assigned-clock-parents = <0>, <0>,
> > >  						 <&scmi_clk IMX95_CLK_SYSPLL1_PFD1_DIV2>;
> > >  			power-domains = <&scmi_devpd IMX95_PD_HSIO_TOP>;
> > > +			/* pcie0's Devid(BIT[7:6]) is 0x00, stream id(BIT[5:0]) is 0x10~0x17 */
> > > +			msi-map = <0x0 &its 0x10 0x1>,
> > > +				  <0x100 &its 0x11 0x7>;
> >
> > Aren't you missing msi-map-mask = <0x1ff>; here? Similar to pcie1.
> > Either way, with this change PCIe on pcie0 is not working anymore,
> > regardless of msi-map-mask.
>
> Yes, it should have msi-map-mask. During my test, I have not enable enetc
> so I have not found this problem.

I check dts again. There are linux,pci-domain = <0>; It should work without
msi-map-mask.

I am not sure why 0000:03:00.0
                       ^^ bus number is 03 at your system.


>
> >
> > Without msi-map-mask:
> > > OF: /soc/pcie@4c300000: iommu-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> > > OF: /soc/pcie@4c300000: no msi-map translation for id 0x300 on (null)
> > > r8169 0000:03:00.0: error -EINVAL: enable failure
> > > r8169 0000:03:00.0: probe with driver r8169 failed with error -22
> >
> > With msi-map-mask:
> > > OF: /soc/pcie@4c300000: iommu-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> > > OF: /soc/pcie@4c300000: msi-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> > > r8169 0000:03:00.0: enabling device (0000 -> 0003)
> > > r8169 0000:03:00.0: enabling Mem-Wr-Inval
> > > r8169 0000:03:00.0: error -EIO: PCI read failed
> > > r8169 0000:03:00.0: probe with driver r8169 failed with error -5
>
> Can you try remove iommu-map and keep msi-map? then remove msi-map and
> keep iommu-map to check which one cause this problem.
>
> >
> > Without msi-map/iommu-map:
> > > r8169 0000:03:00.0: enabling device (0000 -> 0003)
> > > r8169 0000:03:00.0: enabling Mem-Wr-Inval
> > > r8169 0000:03:00.0 eth0: RTL8168g/8111g, d8:9d:b9:00:16:10, XID 4c0, IRQ 166
> > > r8169 0000:03:00.0 eth0: jumbo features [frames: 9194 bytes, tx checksumming: ko]
> > > r8169 0000:03:00.0 enp3s0: renamed from eth0
> > > r8169 0000:03:00.0: enabling bus mastering
> > > r8169 0000:03:00.0 enp3s0: Link is Down
> >
> > pcie1 works as expected. But this is only a single PCIe device, rather than
> > having a PCIe bridge.
> > Any idea what's wrong here?
>
> Can you help dump more information at for PCIe bridge case:
>
> imx_pcie_add_lut(), need rid and sid information.
> drivers/pci/controller/dwc/pci-imx6.c
>
> >
> > Best regards,
> > Alexander
> >
> > > +			iommu-map = <0x000 &smmu 0x10 0x1>,
> > > +				    <0x100 &smmu 0x11 0x7>;
> > > +			iommu-map-mask = <0x1ff>;
> > >  			fsl,max-link-speed = <3>;
> > >  			status = "disabled";
> > >  		};
> > > @@ -1640,6 +1646,14 @@ pcie1: pcie@4c380000 {
> > >  			assigned-clock-parents = <0>, <0>,
> > >  						 <&scmi_clk IMX95_CLK_SYSPLL1_PFD1_DIV2>;
> > >  			power-domains = <&scmi_devpd IMX95_PD_HSIO_TOP>;
> > > +			/* pcie1's Devid(BIT[7:6]) is 0x10, stream id(BIT[5:0]) is 0x18~0x1f */
> > > +			msi-map = <0x0 &its 0x98 0x1>,
> > > +				  <0x100 &its 0x99 0x7>;
> > > +			msi-map-mask = <0x1ff>;
> > > +			/* smmu have not Devid(BIT[7:6]) */
> > > +			iommu-map = <0x000 &smmu 0x18 0x1>,
> > > +				    <0x100 &smmu 0x19 0x7>;
> > > +			iommu-map-mask = <0x1ff>;
> > >  			fsl,max-link-speed = <3>;
> > >  			status = "disabled";
> > >  		};
> > >
> >
> >
> > --
> > TQ-Systems GmbH | Mühlstraße 2, Gut Delling | 82229 Seefeld, Germany
> > Amtsgericht München, HRB 105018
> > Geschäftsführer: Detlef Schneider, Rüdiger Stahl, Stefan Schneider
> > http://www.tq-group.com/
> >
> >

Re: [PATCH 4/5] arm64: dts: imx95: add PCIe's msi-map and iommu-map property

[Alexander Stein]

Hi Frank,

Am Mittwoch, 26. Februar 2025, 17:31:26 CET schrieb Frank Li:
> On Wed, Feb 26, 2025 at 01:11:37PM +0100, Alexander Stein wrote:
> > Hi Frank,
> >
> > Am Dienstag, 28. Januar 2025, 22:15:58 CET schrieb Frank Li:
> > > Add PCIe's msi-map and iommu-map property because i.MX95 support smmu and
> > > its.
> > >
> > > Signed-off-by: Frank Li <Frank.Li@nxp.com>
> > > ---
> > >  arch/arm64/boot/dts/freescale/imx95.dtsi | 14 ++++++++++++++
> > >  1 file changed, 14 insertions(+)
> > >
> > > diff --git a/arch/arm64/boot/dts/freescale/imx95.dtsi b/arch/arm64/boot/dts/freescale/imx95.dtsi
> > > index 6b8470cb3461a..2cebeda43a52d 100644
> > > --- a/arch/arm64/boot/dts/freescale/imx95.dtsi
> > > +++ b/arch/arm64/boot/dts/freescale/imx95.dtsi
> > > @@ -1573,6 +1573,12 @@ pcie0: pcie@4c300000 {
> > >  			assigned-clock-parents = <0>, <0>,
> > >  						 <&scmi_clk IMX95_CLK_SYSPLL1_PFD1_DIV2>;
> > >  			power-domains = <&scmi_devpd IMX95_PD_HSIO_TOP>;
> > > +			/* pcie0's Devid(BIT[7:6]) is 0x00, stream id(BIT[5:0]) is 0x10~0x17 */
> > > +			msi-map = <0x0 &its 0x10 0x1>,
> > > +				  <0x100 &its 0x11 0x7>;
> >
> > Aren't you missing msi-map-mask = <0x1ff>; here? Similar to pcie1.
> > Either way, with this change PCIe on pcie0 is not working anymore,
> > regardless of msi-map-mask.
> 
> Yes, it should have msi-map-mask. During my test, I have not enable enetc
> so I have not found this problem.

Just to be clear: This is not about enetc. This works fine here.

> > Without msi-map-mask:
> > > OF: /soc/pcie@4c300000: iommu-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> > > OF: /soc/pcie@4c300000: no msi-map translation for id 0x300 on (null)
> > > r8169 0000:03:00.0: error -EINVAL: enable failure
> > > r8169 0000:03:00.0: probe with driver r8169 failed with error -22
> >
> > With msi-map-mask:
> > > OF: /soc/pcie@4c300000: iommu-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> > > OF: /soc/pcie@4c300000: msi-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> > > r8169 0000:03:00.0: enabling device (0000 -> 0003)
> > > r8169 0000:03:00.0: enabling Mem-Wr-Inval
> > > r8169 0000:03:00.0: error -EIO: PCI read failed
> > > r8169 0000:03:00.0: probe with driver r8169 failed with error -5
> 
> Can you try remove iommu-map and keep msi-map? then remove msi-map and
> keep iommu-map to check which one cause this problem.

With only msi-map removed, but smmu enabled:
> arm-smmu-v3 490d0000.iommu: event 0x10 received:
> arm-smmu-v3 490d0000.iommu:      0x0000001100000010
> arm-smmu-v3 490d0000.iommu:      0x0000020a00000000
> arm-smmu-v3 490d0000.iommu:      0x000000009b0cc000
> arm-smmu-v3 490d0000.iommu:      0x0000000000000000
> arm-smmu-v3 490d0000.iommu: event: F_TRANSLATION client: 0000:01:00.0 sid: 0x11 ssid: 0x0 iova: 0x9b0cc000 ipa: 0x0
> arm-smmu-v3 490d0000.iommu: priv data read s1 "Input address caused fault" stag: 0x0 r8169 0000:03:00.0
> enp3s0: Link is Down

With only iommu-map removed, both smmu enabled or disabled:
> OF: /soc/pcie@4c300000: msi-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> r8169 0000:03:00.0: enabling device (0000 -> 0003)
> r8169 0000:03:00.0: enabling Mem-Wr-Inval
> r8169 0000:03:00.0: error -EIO: PCI read failed
> r8169 0000:03:00.0: probe with driver r8169 failed with error -5

Only if smmu is disabled and msi-map is removed the driver probes
successfully:
> r8169 0000:03:00.0: enabling device (0000 -> 0003)
> r8169 0000:03:00.0: enabling Mem-Wr-Inval
> r8169 0000:03:00.0 eth0: RTL8168g/8111g, d8:9d:b9:00:16:10, XID 4c0, IRQ 160
> r8169 0000:03:00.0 eth0: jumbo features [frames: 9194 bytes, tx checksumming: ko]
> r8169 0000:03:00.0 enp3s0: renamed from eth0
> r8169 0000:03:00.0: enabling bus mastering
> r8169 0000:03:00.0 enp3s0: Link is Down

> >
> > Without msi-map/iommu-map:
> > > r8169 0000:03:00.0: enabling device (0000 -> 0003)
> > > r8169 0000:03:00.0: enabling Mem-Wr-Inval
> > > r8169 0000:03:00.0 eth0: RTL8168g/8111g, d8:9d:b9:00:16:10, XID 4c0, IRQ 166
> > > r8169 0000:03:00.0 eth0: jumbo features [frames: 9194 bytes, tx checksumming: ko]
> > > r8169 0000:03:00.0 enp3s0: renamed from eth0
> > > r8169 0000:03:00.0: enabling bus mastering
> > > r8169 0000:03:00.0 enp3s0: Link is Down
> >
> > pcie1 works as expected. But this is only a single PCIe device, rather than
> > having a PCIe bridge.
> > Any idea what's wrong here?
> 
> Can you help dump more information at for PCIe bridge case:
> 
> imx_pcie_add_lut(), need rid and sid information.
> drivers/pci/controller/dwc/pci-imx6.c

Just to be clear, without msi-map and iommu-map I get:
> imx6q-pcie 4c380000.pcie: rid: 0x0, sid: 0x18
> imx6q-pcie 4c380000.pcie: rid: 0x100, sid: 0x19

This function get called once for each device.
Maybe the whole PCIe bus might help here, so I've put lspci output here as well.

$ lspci
0000:00:00.0 PCI bridge: Philips Semiconductors Device 0000
0000:01:00.0 PCI bridge: Pericom Semiconductor Device a303 (rev 03)
0000:02:01.0 PCI bridge: Pericom Semiconductor Device a303 (rev 03)
0000:02:02.0 PCI bridge: Pericom Semiconductor Device a303 (rev 03)
0000:03:00.0 Ethernet controller: Realtek Semiconductor Co., Ltd. RTL8111/8168/8211/8411 PCI Express Gigabit Ethernet Controller (rev 0c)
0000:04:00.0 Ethernet controller: Realtek Semiconductor Co., Ltd. RTL8111/8168/8211/8411 PCI Express Gigabit Ethernet Controller (rev 0c)
0001:00:00.0 PCI bridge: Philips Semiconductors Device 0000
0001:01:00.0 Ethernet controller: Intel Corporation 82574L Gigabit Network Connection
0002:00:00.0 Ethernet controller: Philips Semiconductors Device e101 (rev 04)
0002:00:01.0 Generic system peripheral [0807]: Philips Semiconductors Device e001 (rev 03)
0002:00:08.0 Ethernet controller: Philips Semiconductors Device e101 (rev 04)
0002:00:10.0 Ethernet controller: Philips Semiconductors Device e101 (rev 04)
0002:00:18.0 System peripheral: Philips Semiconductors Device ee02 (rev 04)
0003:01:00.0 System peripheral: Philips Semiconductors Device ee00 (rev 04)
0003:01:01.0 Generic system peripheral [0807]: Philips Semiconductors Device e001 (rev 03)

$ lspci -t
-[0000:00]---00.0-[01-ff]----00.0-[02-04]--+-01.0-[03]----00.0
                                           \-02.0-[04]----00.0
-[0001:00]---00.0-[01-ff]----00.0
-[0002:00]-+-00.0
           +-01.0
           +-08.0
           +-10.0
           \-18.0
-[0003:01]-+-00.0
           \-01.0

Thanks and best regards,
Alexander
-- 
TQ-Systems GmbH | Mühlstraße 2, Gut Delling | 82229 Seefeld, Germany
Amtsgericht München, HRB 105018
Geschäftsführer: Detlef Schneider, Rüdiger Stahl, Stefan Schneider
http://www.tq-group.com/

Re: [PATCH 4/5] arm64: dts: imx95: add PCIe's msi-map and iommu-map property

[Frank Li]

On Thu, Feb 27, 2025 at 08:54:13AM +0100, Alexander Stein wrote:
> Hi Frank,
>
> Am Mittwoch, 26. Februar 2025, 17:31:26 CET schrieb Frank Li:
> > On Wed, Feb 26, 2025 at 01:11:37PM +0100, Alexander Stein wrote:
> > > Hi Frank,
> > >
> > > Am Dienstag, 28. Januar 2025, 22:15:58 CET schrieb Frank Li:
> > > > Add PCIe's msi-map and iommu-map property because i.MX95 support smmu and
> > > > its.
> > > >
> > > > Signed-off-by: Frank Li <Frank.Li@nxp.com>
> > > > ---
> > > >  arch/arm64/boot/dts/freescale/imx95.dtsi | 14 ++++++++++++++
> > > >  1 file changed, 14 insertions(+)
> > > >
> > > > diff --git a/arch/arm64/boot/dts/freescale/imx95.dtsi b/arch/arm64/boot/dts/freescale/imx95.dtsi
> > > > index 6b8470cb3461a..2cebeda43a52d 100644
> > > > --- a/arch/arm64/boot/dts/freescale/imx95.dtsi
> > > > +++ b/arch/arm64/boot/dts/freescale/imx95.dtsi
> > > > @@ -1573,6 +1573,12 @@ pcie0: pcie@4c300000 {
> > > >  			assigned-clock-parents = <0>, <0>,
> > > >  						 <&scmi_clk IMX95_CLK_SYSPLL1_PFD1_DIV2>;
> > > >  			power-domains = <&scmi_devpd IMX95_PD_HSIO_TOP>;
> > > > +			/* pcie0's Devid(BIT[7:6]) is 0x00, stream id(BIT[5:0]) is 0x10~0x17 */
> > > > +			msi-map = <0x0 &its 0x10 0x1>,
> > > > +				  <0x100 &its 0x11 0x7>;
> > >
> > > Aren't you missing msi-map-mask = <0x1ff>; here? Similar to pcie1.
> > > Either way, with this change PCIe on pcie0 is not working anymore,
> > > regardless of msi-map-mask.
> >
> > Yes, it should have msi-map-mask. During my test, I have not enable enetc
> > so I have not found this problem.
>
> Just to be clear: This is not about enetc. This works fine here.
>
> > > Without msi-map-mask:
> > > > OF: /soc/pcie@4c300000: iommu-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> > > > OF: /soc/pcie@4c300000: no msi-map translation for id 0x300 on (null)
> > > > r8169 0000:03:00.0: error -EINVAL: enable failure
> > > > r8169 0000:03:00.0: probe with driver r8169 failed with error -22
> > >
> > > With msi-map-mask:
> > > > OF: /soc/pcie@4c300000: iommu-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> > > > OF: /soc/pcie@4c300000: msi-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> > > > r8169 0000:03:00.0: enabling device (0000 -> 0003)
> > > > r8169 0000:03:00.0: enabling Mem-Wr-Inval
> > > > r8169 0000:03:00.0: error -EIO: PCI read failed
> > > > r8169 0000:03:00.0: probe with driver r8169 failed with error -5
> >
> > Can you try remove iommu-map and keep msi-map? then remove msi-map and
> > keep iommu-map to check which one cause this problem.
>
> With only msi-map removed, but smmu enabled:
> > arm-smmu-v3 490d0000.iommu: event 0x10 received:
> > arm-smmu-v3 490d0000.iommu:      0x0000001100000010
> > arm-smmu-v3 490d0000.iommu:      0x0000020a00000000
> > arm-smmu-v3 490d0000.iommu:      0x000000009b0cc000
> > arm-smmu-v3 490d0000.iommu:      0x0000000000000000
> > arm-smmu-v3 490d0000.iommu: event: F_TRANSLATION client: 0000:01:00.0 sid: 0x11 ssid: 0x0 iova: 0x9b0cc000 ipa: 0x0
> > arm-smmu-v3 490d0000.iommu: priv data read s1 "Input address caused fault" stag: 0x0 r8169 0000:03:00.0
> > enp3s0: Link is Down
>
> With only iommu-map removed, both smmu enabled or disabled:
> > OF: /soc/pcie@4c300000: msi-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> > r8169 0000:03:00.0: enabling device (0000 -> 0003)
> > r8169 0000:03:00.0: enabling Mem-Wr-Inval
> > r8169 0000:03:00.0: error -EIO: PCI read failed
> > r8169 0000:03:00.0: probe with driver r8169 failed with error -5
>
> Only if smmu is disabled and msi-map is removed the driver probes
> successfully:
> > r8169 0000:03:00.0: enabling device (0000 -> 0003)
> > r8169 0000:03:00.0: enabling Mem-Wr-Inval
> > r8169 0000:03:00.0 eth0: RTL8168g/8111g, d8:9d:b9:00:16:10, XID 4c0, IRQ 160
> > r8169 0000:03:00.0 eth0: jumbo features [frames: 9194 bytes, tx checksumming: ko]
> > r8169 0000:03:00.0 enp3s0: renamed from eth0
> > r8169 0000:03:00.0: enabling bus mastering
> > r8169 0000:03:00.0 enp3s0: Link is Down
>
> > >
> > > Without msi-map/iommu-map:
> > > > r8169 0000:03:00.0: enabling device (0000 -> 0003)
> > > > r8169 0000:03:00.0: enabling Mem-Wr-Inval
> > > > r8169 0000:03:00.0 eth0: RTL8168g/8111g, d8:9d:b9:00:16:10, XID 4c0, IRQ 166
> > > > r8169 0000:03:00.0 eth0: jumbo features [frames: 9194 bytes, tx checksumming: ko]
> > > > r8169 0000:03:00.0 enp3s0: renamed from eth0
> > > > r8169 0000:03:00.0: enabling bus mastering
> > > > r8169 0000:03:00.0 enp3s0: Link is Down
> > >
> > > pcie1 works as expected. But this is only a single PCIe device, rather than
> > > having a PCIe bridge.
> > > Any idea what's wrong here?
> >
> > Can you help dump more information at for PCIe bridge case:
> >
> > imx_pcie_add_lut(), need rid and sid information.
> > drivers/pci/controller/dwc/pci-imx6.c
>
> Just to be clear, without msi-map and iommu-map I get:
> > imx6q-pcie 4c380000.pcie: rid: 0x0, sid: 0x18
> > imx6q-pcie 4c380000.pcie: rid: 0x100, sid: 0x19

Can you help dump register value PE0_LUT_CREQID offset 0x101 for your
smmu-map or msi-map enable case

2nd test.
change IMX95_PE0_LUT_MASK to 0x1ff

Frank

>
> This function get called once for each device.
> Maybe the whole PCIe bus might help here, so I've put lspci output here as well.
>
> $ lspci
> 0000:00:00.0 PCI bridge: Philips Semiconductors Device 0000
> 0000:01:00.0 PCI bridge: Pericom Semiconductor Device a303 (rev 03)
> 0000:02:01.0 PCI bridge: Pericom Semiconductor Device a303 (rev 03)
> 0000:02:02.0 PCI bridge: Pericom Semiconductor Device a303 (rev 03)
> 0000:03:00.0 Ethernet controller: Realtek Semiconductor Co., Ltd. RTL8111/8168/8211/8411 PCI Express Gigabit Ethernet Controller (rev 0c)
> 0000:04:00.0 Ethernet controller: Realtek Semiconductor Co., Ltd. RTL8111/8168/8211/8411 PCI Express Gigabit Ethernet Controller (rev 0c)
> 0001:00:00.0 PCI bridge: Philips Semiconductors Device 0000
> 0001:01:00.0 Ethernet controller: Intel Corporation 82574L Gigabit Network Connection
> 0002:00:00.0 Ethernet controller: Philips Semiconductors Device e101 (rev 04)
> 0002:00:01.0 Generic system peripheral [0807]: Philips Semiconductors Device e001 (rev 03)
> 0002:00:08.0 Ethernet controller: Philips Semiconductors Device e101 (rev 04)
> 0002:00:10.0 Ethernet controller: Philips Semiconductors Device e101 (rev 04)
> 0002:00:18.0 System peripheral: Philips Semiconductors Device ee02 (rev 04)
> 0003:01:00.0 System peripheral: Philips Semiconductors Device ee00 (rev 04)
> 0003:01:01.0 Generic system peripheral [0807]: Philips Semiconductors Device e001 (rev 03)
>
> $ lspci -t
> -[0000:00]---00.0-[01-ff]----00.0-[02-04]--+-01.0-[03]----00.0
>                                            \-02.0-[04]----00.0
> -[0001:00]---00.0-[01-ff]----00.0
> -[0002:00]-+-00.0
>            +-01.0
>            +-08.0
>            +-10.0
>            \-18.0
> -[0003:01]-+-00.0
>            \-01.0
>
> Thanks and best regards,
> Alexander
> --
> TQ-Systems GmbH | Mühlstraße 2, Gut Delling | 82229 Seefeld, Germany
> Amtsgericht München, HRB 105018
> Geschäftsführer: Detlef Schneider, Rüdiger Stahl, Stefan Schneider
> http://www.tq-group.com/
>
>

Re: [PATCH 4/5] arm64: dts: imx95: add PCIe's msi-map and iommu-map property

[Alexander Stein]

Hi Frank,

Am Donnerstag, 27. Februar 2025, 17:39:47 CET schrieb Frank Li:
> On Thu, Feb 27, 2025 at 08:54:13AM +0100, Alexander Stein wrote:
> > Hi Frank,
> >
> > Am Mittwoch, 26. Februar 2025, 17:31:26 CET schrieb Frank Li:
> > > On Wed, Feb 26, 2025 at 01:11:37PM +0100, Alexander Stein wrote:
> > > > Hi Frank,
> > > >
> > > > Am Dienstag, 28. Januar 2025, 22:15:58 CET schrieb Frank Li:
> > > > > Add PCIe's msi-map and iommu-map property because i.MX95 support smmu and
> > > > > its.
> > > > >
> > > > > Signed-off-by: Frank Li <Frank.Li@nxp.com>
> > > > > ---
> > > > >  arch/arm64/boot/dts/freescale/imx95.dtsi | 14 ++++++++++++++
> > > > >  1 file changed, 14 insertions(+)
> > > > >
> > > > > diff --git a/arch/arm64/boot/dts/freescale/imx95.dtsi b/arch/arm64/boot/dts/freescale/imx95.dtsi
> > > > > index 6b8470cb3461a..2cebeda43a52d 100644
> > > > > --- a/arch/arm64/boot/dts/freescale/imx95.dtsi
> > > > > +++ b/arch/arm64/boot/dts/freescale/imx95.dtsi
> > > > > @@ -1573,6 +1573,12 @@ pcie0: pcie@4c300000 {
> > > > >  			assigned-clock-parents = <0>, <0>,
> > > > >  						 <&scmi_clk IMX95_CLK_SYSPLL1_PFD1_DIV2>;
> > > > >  			power-domains = <&scmi_devpd IMX95_PD_HSIO_TOP>;
> > > > > +			/* pcie0's Devid(BIT[7:6]) is 0x00, stream id(BIT[5:0]) is 0x10~0x17 */
> > > > > +			msi-map = <0x0 &its 0x10 0x1>,
> > > > > +				  <0x100 &its 0x11 0x7>;
> > > >
> > > > Aren't you missing msi-map-mask = <0x1ff>; here? Similar to pcie1.
> > > > Either way, with this change PCIe on pcie0 is not working anymore,
> > > > regardless of msi-map-mask.
> > >
> > > Yes, it should have msi-map-mask. During my test, I have not enable enetc
> > > so I have not found this problem.
> >
> > Just to be clear: This is not about enetc. This works fine here.
> >
> > > > Without msi-map-mask:
> > > > > OF: /soc/pcie@4c300000: iommu-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> > > > > OF: /soc/pcie@4c300000: no msi-map translation for id 0x300 on (null)
> > > > > r8169 0000:03:00.0: error -EINVAL: enable failure
> > > > > r8169 0000:03:00.0: probe with driver r8169 failed with error -22
> > > >
> > > > With msi-map-mask:
> > > > > OF: /soc/pcie@4c300000: iommu-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> > > > > OF: /soc/pcie@4c300000: msi-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> > > > > r8169 0000:03:00.0: enabling device (0000 -> 0003)
> > > > > r8169 0000:03:00.0: enabling Mem-Wr-Inval
> > > > > r8169 0000:03:00.0: error -EIO: PCI read failed
> > > > > r8169 0000:03:00.0: probe with driver r8169 failed with error -5
> > >
> > > Can you try remove iommu-map and keep msi-map? then remove msi-map and
> > > keep iommu-map to check which one cause this problem.
> >
> > With only msi-map removed, but smmu enabled:
> > > arm-smmu-v3 490d0000.iommu: event 0x10 received:
> > > arm-smmu-v3 490d0000.iommu:      0x0000001100000010
> > > arm-smmu-v3 490d0000.iommu:      0x0000020a00000000
> > > arm-smmu-v3 490d0000.iommu:      0x000000009b0cc000
> > > arm-smmu-v3 490d0000.iommu:      0x0000000000000000
> > > arm-smmu-v3 490d0000.iommu: event: F_TRANSLATION client: 0000:01:00.0 sid: 0x11 ssid: 0x0 iova: 0x9b0cc000 ipa: 0x0
> > > arm-smmu-v3 490d0000.iommu: priv data read s1 "Input address caused fault" stag: 0x0 r8169 0000:03:00.0
> > > enp3s0: Link is Down
> >
> > With only iommu-map removed, both smmu enabled or disabled:
> > > OF: /soc/pcie@4c300000: msi-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> > > r8169 0000:03:00.0: enabling device (0000 -> 0003)
> > > r8169 0000:03:00.0: enabling Mem-Wr-Inval
> > > r8169 0000:03:00.0: error -EIO: PCI read failed
> > > r8169 0000:03:00.0: probe with driver r8169 failed with error -5
> >
> > Only if smmu is disabled and msi-map is removed the driver probes
> > successfully:
> > > r8169 0000:03:00.0: enabling device (0000 -> 0003)
> > > r8169 0000:03:00.0: enabling Mem-Wr-Inval
> > > r8169 0000:03:00.0 eth0: RTL8168g/8111g, d8:9d:b9:00:16:10, XID 4c0, IRQ 160
> > > r8169 0000:03:00.0 eth0: jumbo features [frames: 9194 bytes, tx checksumming: ko]
> > > r8169 0000:03:00.0 enp3s0: renamed from eth0
> > > r8169 0000:03:00.0: enabling bus mastering
> > > r8169 0000:03:00.0 enp3s0: Link is Down
> >
> > > >
> > > > Without msi-map/iommu-map:
> > > > > r8169 0000:03:00.0: enabling device (0000 -> 0003)
> > > > > r8169 0000:03:00.0: enabling Mem-Wr-Inval
> > > > > r8169 0000:03:00.0 eth0: RTL8168g/8111g, d8:9d:b9:00:16:10, XID 4c0, IRQ 166
> > > > > r8169 0000:03:00.0 eth0: jumbo features [frames: 9194 bytes, tx checksumming: ko]
> > > > > r8169 0000:03:00.0 enp3s0: renamed from eth0
> > > > > r8169 0000:03:00.0: enabling bus mastering
> > > > > r8169 0000:03:00.0 enp3s0: Link is Down
> > > >
> > > > pcie1 works as expected. But this is only a single PCIe device, rather than
> > > > having a PCIe bridge.
> > > > Any idea what's wrong here?
> > >
> > > Can you help dump more information at for PCIe bridge case:
> > >
> > > imx_pcie_add_lut(), need rid and sid information.
> > > drivers/pci/controller/dwc/pci-imx6.c
> >
> > Just to be clear, without msi-map and iommu-map I get:
> > > imx6q-pcie 4c380000.pcie: rid: 0x0, sid: 0x18
> > > imx6q-pcie 4c380000.pcie: rid: 0x100, sid: 0x19
> 
> Can you help dump register value PE0_LUT_CREQID offset 0x101 for your
> smmu-map or msi-map enable case

I am assuming you meant offset 0x101c, as stated in the RM.
I added a dump directly before printing "PCI read failed" in r8169_main.c.
Unfortunately this only returns 0 for both PCIe devices, so I'm wondering
if this is correct.

> 2nd test.
> change IMX95_PE0_LUT_MASK to 0x1ff

Unfortunately I do not notice any effect/difference.

Best regards
Alexander
-- 
TQ-Systems GmbH | Mühlstraße 2, Gut Delling | 82229 Seefeld, Germany
Amtsgericht München, HRB 105018
Geschäftsführer: Detlef Schneider, Rüdiger Stahl, Stefan Schneider
http://www.tq-group.com/

Re: [PATCH 4/5] arm64: dts: imx95: add PCIe's msi-map and iommu-map property

[Frank Li]

On Fri, Feb 28, 2025 at 10:08:58AM +0100, Alexander Stein wrote:
> Hi Frank,
>
> Am Donnerstag, 27. Februar 2025, 17:39:47 CET schrieb Frank Li:
> > On Thu, Feb 27, 2025 at 08:54:13AM +0100, Alexander Stein wrote:
> > > Hi Frank,
> > >
> > > Am Mittwoch, 26. Februar 2025, 17:31:26 CET schrieb Frank Li:
> > > > On Wed, Feb 26, 2025 at 01:11:37PM +0100, Alexander Stein wrote:
> > > > > Hi Frank,
> > > > >
> > > > > Am Dienstag, 28. Januar 2025, 22:15:58 CET schrieb Frank Li:
> > > > > > Add PCIe's msi-map and iommu-map property because i.MX95 support smmu and
> > > > > > its.
> > > > > >
> > > > > > Signed-off-by: Frank Li <Frank.Li@nxp.com>
> > > > > > ---
> > > > > >  arch/arm64/boot/dts/freescale/imx95.dtsi | 14 ++++++++++++++
> > > > > >  1 file changed, 14 insertions(+)
> > > > > >
> > > > > > diff --git a/arch/arm64/boot/dts/freescale/imx95.dtsi b/arch/arm64/boot/dts/freescale/imx95.dtsi
> > > > > > index 6b8470cb3461a..2cebeda43a52d 100644
> > > > > > --- a/arch/arm64/boot/dts/freescale/imx95.dtsi
> > > > > > +++ b/arch/arm64/boot/dts/freescale/imx95.dtsi
> > > > > > @@ -1573,6 +1573,12 @@ pcie0: pcie@4c300000 {
> > > > > >  			assigned-clock-parents = <0>, <0>,
> > > > > >  						 <&scmi_clk IMX95_CLK_SYSPLL1_PFD1_DIV2>;
> > > > > >  			power-domains = <&scmi_devpd IMX95_PD_HSIO_TOP>;
> > > > > > +			/* pcie0's Devid(BIT[7:6]) is 0x00, stream id(BIT[5:0]) is 0x10~0x17 */
> > > > > > +			msi-map = <0x0 &its 0x10 0x1>,
> > > > > > +				  <0x100 &its 0x11 0x7>;
> > > > >
> > > > > Aren't you missing msi-map-mask = <0x1ff>; here? Similar to pcie1.
> > > > > Either way, with this change PCIe on pcie0 is not working anymore,
> > > > > regardless of msi-map-mask.
> > > >
> > > > Yes, it should have msi-map-mask. During my test, I have not enable enetc
> > > > so I have not found this problem.
> > >
> > > Just to be clear: This is not about enetc. This works fine here.
> > >
> > > > > Without msi-map-mask:
> > > > > > OF: /soc/pcie@4c300000: iommu-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> > > > > > OF: /soc/pcie@4c300000: no msi-map translation for id 0x300 on (null)
> > > > > > r8169 0000:03:00.0: error -EINVAL: enable failure
> > > > > > r8169 0000:03:00.0: probe with driver r8169 failed with error -22
> > > > >
> > > > > With msi-map-mask:
> > > > > > OF: /soc/pcie@4c300000: iommu-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> > > > > > OF: /soc/pcie@4c300000: msi-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> > > > > > r8169 0000:03:00.0: enabling device (0000 -> 0003)
> > > > > > r8169 0000:03:00.0: enabling Mem-Wr-Inval
> > > > > > r8169 0000:03:00.0: error -EIO: PCI read failed
> > > > > > r8169 0000:03:00.0: probe with driver r8169 failed with error -5
> > > >
> > > > Can you try remove iommu-map and keep msi-map? then remove msi-map and
> > > > keep iommu-map to check which one cause this problem.
> > >
> > > With only msi-map removed, but smmu enabled:
> > > > arm-smmu-v3 490d0000.iommu: event 0x10 received:
> > > > arm-smmu-v3 490d0000.iommu:      0x0000001100000010
> > > > arm-smmu-v3 490d0000.iommu:      0x0000020a00000000
> > > > arm-smmu-v3 490d0000.iommu:      0x000000009b0cc000
> > > > arm-smmu-v3 490d0000.iommu:      0x0000000000000000
> > > > arm-smmu-v3 490d0000.iommu: event: F_TRANSLATION client: 0000:01:00.0 sid: 0x11 ssid: 0x0 iova: 0x9b0cc000 ipa: 0x0
> > > > arm-smmu-v3 490d0000.iommu: priv data read s1 "Input address caused fault" stag: 0x0 r8169 0000:03:00.0
> > > > enp3s0: Link is Down
> > >
> > > With only iommu-map removed, both smmu enabled or disabled:
> > > > OF: /soc/pcie@4c300000: msi-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> > > > r8169 0000:03:00.0: enabling device (0000 -> 0003)
> > > > r8169 0000:03:00.0: enabling Mem-Wr-Inval
> > > > r8169 0000:03:00.0: error -EIO: PCI read failed
> > > > r8169 0000:03:00.0: probe with driver r8169 failed with error -5
> > >
> > > Only if smmu is disabled and msi-map is removed the driver probes
> > > successfully:
> > > > r8169 0000:03:00.0: enabling device (0000 -> 0003)
> > > > r8169 0000:03:00.0: enabling Mem-Wr-Inval
> > > > r8169 0000:03:00.0 eth0: RTL8168g/8111g, d8:9d:b9:00:16:10, XID 4c0, IRQ 160
> > > > r8169 0000:03:00.0 eth0: jumbo features [frames: 9194 bytes, tx checksumming: ko]
> > > > r8169 0000:03:00.0 enp3s0: renamed from eth0
> > > > r8169 0000:03:00.0: enabling bus mastering
> > > > r8169 0000:03:00.0 enp3s0: Link is Down
> > >
> > > > >
> > > > > Without msi-map/iommu-map:
> > > > > > r8169 0000:03:00.0: enabling device (0000 -> 0003)
> > > > > > r8169 0000:03:00.0: enabling Mem-Wr-Inval
> > > > > > r8169 0000:03:00.0 eth0: RTL8168g/8111g, d8:9d:b9:00:16:10, XID 4c0, IRQ 166
> > > > > > r8169 0000:03:00.0 eth0: jumbo features [frames: 9194 bytes, tx checksumming: ko]
> > > > > > r8169 0000:03:00.0 enp3s0: renamed from eth0
> > > > > > r8169 0000:03:00.0: enabling bus mastering
> > > > > > r8169 0000:03:00.0 enp3s0: Link is Down
> > > > >
> > > > > pcie1 works as expected. But this is only a single PCIe device, rather than
> > > > > having a PCIe bridge.
> > > > > Any idea what's wrong here?
> > > >
> > > > Can you help dump more information at for PCIe bridge case:
> > > >
> > > > imx_pcie_add_lut(), need rid and sid information.
> > > > drivers/pci/controller/dwc/pci-imx6.c
> > >
> > > Just to be clear, without msi-map and iommu-map I get:
> > > > imx6q-pcie 4c380000.pcie: rid: 0x0, sid: 0x18
> > > > imx6q-pcie 4c380000.pcie: rid: 0x100, sid: 0x19
> >
> > Can you help dump register value PE0_LUT_CREQID offset 0x101 for your
> > smmu-map or msi-map enable case
>
> I am assuming you meant offset 0x101c, as stated in the RM.
> I added a dump directly before printing "PCI read failed" in r8169_main.c.

Can you point me the code about where "error -EIO: PCI read failed"?

I tested nvme devices worked at both PCIE0 and PCIE1.

Which PCI switch do you use?

Frank

> Unfortunately this only returns 0 for both PCIe devices, so I'm wondering
> if this is correct.
>
> > 2nd test.
> > change IMX95_PE0_LUT_MASK to 0x1ff
>
> Unfortunately I do not notice any effect/difference.
>
> Best regards
> Alexander
> --
> TQ-Systems GmbH | Mühlstraße 2, Gut Delling | 82229 Seefeld, Germany
> Amtsgericht München, HRB 105018
> Geschäftsführer: Detlef Schneider, Rüdiger Stahl, Stefan Schneider
> http://www.tq-group.com/
>
>

Re: [PATCH 4/5] arm64: dts: imx95: add PCIe's msi-map and iommu-map property

[Alexander Stein]

Hi Frank,

Am Freitag, 28. Februar 2025, 16:32:50 CET schrieb Frank Li:
> On Fri, Feb 28, 2025 at 10:08:58AM +0100, Alexander Stein wrote:
> > Hi Frank,
> >
> > Am Donnerstag, 27. Februar 2025, 17:39:47 CET schrieb Frank Li:
> > > On Thu, Feb 27, 2025 at 08:54:13AM +0100, Alexander Stein wrote:
> > > > Hi Frank,
> > > >
> > > > Am Mittwoch, 26. Februar 2025, 17:31:26 CET schrieb Frank Li:
> > > > > On Wed, Feb 26, 2025 at 01:11:37PM +0100, Alexander Stein wrote:
> > > > > > Hi Frank,
> > > > > >
> > > > > > Am Dienstag, 28. Januar 2025, 22:15:58 CET schrieb Frank Li:
> > > > > > > Add PCIe's msi-map and iommu-map property because i.MX95 support smmu and
> > > > > > > its.
> > > > > > >
> > > > > > > Signed-off-by: Frank Li <Frank.Li@nxp.com>
> > > > > > > ---
> > > > > > >  arch/arm64/boot/dts/freescale/imx95.dtsi | 14 ++++++++++++++
> > > > > > >  1 file changed, 14 insertions(+)
> > > > > > >
> > > > > > > diff --git a/arch/arm64/boot/dts/freescale/imx95.dtsi b/arch/arm64/boot/dts/freescale/imx95.dtsi
> > > > > > > index 6b8470cb3461a..2cebeda43a52d 100644
> > > > > > > --- a/arch/arm64/boot/dts/freescale/imx95.dtsi
> > > > > > > +++ b/arch/arm64/boot/dts/freescale/imx95.dtsi
> > > > > > > @@ -1573,6 +1573,12 @@ pcie0: pcie@4c300000 {
> > > > > > >  			assigned-clock-parents = <0>, <0>,
> > > > > > >  						 <&scmi_clk IMX95_CLK_SYSPLL1_PFD1_DIV2>;
> > > > > > >  			power-domains = <&scmi_devpd IMX95_PD_HSIO_TOP>;
> > > > > > > +			/* pcie0's Devid(BIT[7:6]) is 0x00, stream id(BIT[5:0]) is 0x10~0x17 */
> > > > > > > +			msi-map = <0x0 &its 0x10 0x1>,
> > > > > > > +				  <0x100 &its 0x11 0x7>;
> > > > > >
> > > > > > Aren't you missing msi-map-mask = <0x1ff>; here? Similar to pcie1.
> > > > > > Either way, with this change PCIe on pcie0 is not working anymore,
> > > > > > regardless of msi-map-mask.
> > > > >
> > > > > Yes, it should have msi-map-mask. During my test, I have not enable enetc
> > > > > so I have not found this problem.
> > > >
> > > > Just to be clear: This is not about enetc. This works fine here.
> > > >
> > > > > > Without msi-map-mask:
> > > > > > > OF: /soc/pcie@4c300000: iommu-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> > > > > > > OF: /soc/pcie@4c300000: no msi-map translation for id 0x300 on (null)
> > > > > > > r8169 0000:03:00.0: error -EINVAL: enable failure
> > > > > > > r8169 0000:03:00.0: probe with driver r8169 failed with error -22
> > > > > >
> > > > > > With msi-map-mask:
> > > > > > > OF: /soc/pcie@4c300000: iommu-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> > > > > > > OF: /soc/pcie@4c300000: msi-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> > > > > > > r8169 0000:03:00.0: enabling device (0000 -> 0003)
> > > > > > > r8169 0000:03:00.0: enabling Mem-Wr-Inval
> > > > > > > r8169 0000:03:00.0: error -EIO: PCI read failed
> > > > > > > r8169 0000:03:00.0: probe with driver r8169 failed with error -5
> > > > >
> > > > > Can you try remove iommu-map and keep msi-map? then remove msi-map and
> > > > > keep iommu-map to check which one cause this problem.
> > > >
> > > > With only msi-map removed, but smmu enabled:
> > > > > arm-smmu-v3 490d0000.iommu: event 0x10 received:
> > > > > arm-smmu-v3 490d0000.iommu:      0x0000001100000010
> > > > > arm-smmu-v3 490d0000.iommu:      0x0000020a00000000
> > > > > arm-smmu-v3 490d0000.iommu:      0x000000009b0cc000
> > > > > arm-smmu-v3 490d0000.iommu:      0x0000000000000000
> > > > > arm-smmu-v3 490d0000.iommu: event: F_TRANSLATION client: 0000:01:00.0 sid: 0x11 ssid: 0x0 iova: 0x9b0cc000 ipa: 0x0
> > > > > arm-smmu-v3 490d0000.iommu: priv data read s1 "Input address caused fault" stag: 0x0 r8169 0000:03:00.0
> > > > > enp3s0: Link is Down
> > > >
> > > > With only iommu-map removed, both smmu enabled or disabled:
> > > > > OF: /soc/pcie@4c300000: msi-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> > > > > r8169 0000:03:00.0: enabling device (0000 -> 0003)
> > > > > r8169 0000:03:00.0: enabling Mem-Wr-Inval
> > > > > r8169 0000:03:00.0: error -EIO: PCI read failed
> > > > > r8169 0000:03:00.0: probe with driver r8169 failed with error -5
> > > >
> > > > Only if smmu is disabled and msi-map is removed the driver probes
> > > > successfully:
> > > > > r8169 0000:03:00.0: enabling device (0000 -> 0003)
> > > > > r8169 0000:03:00.0: enabling Mem-Wr-Inval
> > > > > r8169 0000:03:00.0 eth0: RTL8168g/8111g, d8:9d:b9:00:16:10, XID 4c0, IRQ 160
> > > > > r8169 0000:03:00.0 eth0: jumbo features [frames: 9194 bytes, tx checksumming: ko]
> > > > > r8169 0000:03:00.0 enp3s0: renamed from eth0
> > > > > r8169 0000:03:00.0: enabling bus mastering
> > > > > r8169 0000:03:00.0 enp3s0: Link is Down
> > > >
> > > > > >
> > > > > > Without msi-map/iommu-map:
> > > > > > > r8169 0000:03:00.0: enabling device (0000 -> 0003)
> > > > > > > r8169 0000:03:00.0: enabling Mem-Wr-Inval
> > > > > > > r8169 0000:03:00.0 eth0: RTL8168g/8111g, d8:9d:b9:00:16:10, XID 4c0, IRQ 166
> > > > > > > r8169 0000:03:00.0 eth0: jumbo features [frames: 9194 bytes, tx checksumming: ko]
> > > > > > > r8169 0000:03:00.0 enp3s0: renamed from eth0
> > > > > > > r8169 0000:03:00.0: enabling bus mastering
> > > > > > > r8169 0000:03:00.0 enp3s0: Link is Down
> > > > > >
> > > > > > pcie1 works as expected. But this is only a single PCIe device, rather than
> > > > > > having a PCIe bridge.
> > > > > > Any idea what's wrong here?
> > > > >
> > > > > Can you help dump more information at for PCIe bridge case:
> > > > >
> > > > > imx_pcie_add_lut(), need rid and sid information.
> > > > > drivers/pci/controller/dwc/pci-imx6.c
> > > >
> > > > Just to be clear, without msi-map and iommu-map I get:
> > > > > imx6q-pcie 4c380000.pcie: rid: 0x0, sid: 0x18
> > > > > imx6q-pcie 4c380000.pcie: rid: 0x100, sid: 0x19
> > >
> > > Can you help dump register value PE0_LUT_CREQID offset 0x101 for your
> > > smmu-map or msi-map enable case
> >
> > I am assuming you meant offset 0x101c, as stated in the RM.
> > I added a dump directly before printing "PCI read failed" in r8169_main.c.
> 
> Can you point me the code about where "error -EIO: PCI read failed"?

The error messages comes from [1]. As far as I can tell, this is the first
read after pcim_iomap_table().

> I tested nvme devices worked at both PCIE0 and PCIE1.
> 
> Which PCI switch do you use?

This is a COTS MiniPCIe ethernet card [2]. If I'm reading correctly
the bridge seems to be a P17C9X20

Thanks and best regards
Alexander

[1] https://web.git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git/tree/drivers/net/ethernet/realtek/r8169_main.c?h=next-20250228#n5454
[2] https://www.delock.com/produkt/95237/merkmale.html?f=s
-- 
TQ-Systems GmbH | Mühlstraße 2, Gut Delling | 82229 Seefeld, Germany
Amtsgericht München, HRB 105018
Geschäftsführer: Detlef Schneider, Rüdiger Stahl, Stefan Schneider
http://www.tq-group.com/

Re: [PATCH 4/5] arm64: dts: imx95: add PCIe's msi-map and iommu-map property

[Frank Li]

On Fri, Feb 28, 2025 at 05:01:15PM +0100, Alexander Stein wrote:
> Hi Frank,
>
> Am Freitag, 28. Februar 2025, 16:32:50 CET schrieb Frank Li:
> > On Fri, Feb 28, 2025 at 10:08:58AM +0100, Alexander Stein wrote:
> > > Hi Frank,
> > >
> > > Am Donnerstag, 27. Februar 2025, 17:39:47 CET schrieb Frank Li:
> > > > On Thu, Feb 27, 2025 at 08:54:13AM +0100, Alexander Stein wrote:
> > > > > Hi Frank,
> > > > >
> > > > > Am Mittwoch, 26. Februar 2025, 17:31:26 CET schrieb Frank Li:
> > > > > > On Wed, Feb 26, 2025 at 01:11:37PM +0100, Alexander Stein wrote:
> > > > > > > Hi Frank,
> > > > > > >
> > > > > > > Am Dienstag, 28. Januar 2025, 22:15:58 CET schrieb Frank Li:
> > > > > > > > Add PCIe's msi-map and iommu-map property because i.MX95 support smmu and
> > > > > > > > its.
> > > > > > > >
> > > > > > > > Signed-off-by: Frank Li <Frank.Li@nxp.com>
> > > > > > > > ---
> > > > > > > >  arch/arm64/boot/dts/freescale/imx95.dtsi | 14 ++++++++++++++
> > > > > > > >  1 file changed, 14 insertions(+)
> > > > > > > >
> > > > > > > > diff --git a/arch/arm64/boot/dts/freescale/imx95.dtsi b/arch/arm64/boot/dts/freescale/imx95.dtsi
> > > > > > > > index 6b8470cb3461a..2cebeda43a52d 100644
> > > > > > > > --- a/arch/arm64/boot/dts/freescale/imx95.dtsi
> > > > > > > > +++ b/arch/arm64/boot/dts/freescale/imx95.dtsi
> > > > > > > > @@ -1573,6 +1573,12 @@ pcie0: pcie@4c300000 {
> > > > > > > >  			assigned-clock-parents = <0>, <0>,
> > > > > > > >  						 <&scmi_clk IMX95_CLK_SYSPLL1_PFD1_DIV2>;
> > > > > > > >  			power-domains = <&scmi_devpd IMX95_PD_HSIO_TOP>;
> > > > > > > > +			/* pcie0's Devid(BIT[7:6]) is 0x00, stream id(BIT[5:0]) is 0x10~0x17 */
> > > > > > > > +			msi-map = <0x0 &its 0x10 0x1>,
> > > > > > > > +				  <0x100 &its 0x11 0x7>;
> > > > > > >
> > > > > > > Aren't you missing msi-map-mask = <0x1ff>; here? Similar to pcie1.
> > > > > > > Either way, with this change PCIe on pcie0 is not working anymore,
> > > > > > > regardless of msi-map-mask.
> > > > > >
> > > > > > Yes, it should have msi-map-mask. During my test, I have not enable enetc
> > > > > > so I have not found this problem.
> > > > >
> > > > > Just to be clear: This is not about enetc. This works fine here.
> > > > >
> > > > > > > Without msi-map-mask:
> > > > > > > > OF: /soc/pcie@4c300000: iommu-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> > > > > > > > OF: /soc/pcie@4c300000: no msi-map translation for id 0x300 on (null)
> > > > > > > > r8169 0000:03:00.0: error -EINVAL: enable failure
> > > > > > > > r8169 0000:03:00.0: probe with driver r8169 failed with error -22
> > > > > > >
> > > > > > > With msi-map-mask:
> > > > > > > > OF: /soc/pcie@4c300000: iommu-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> > > > > > > > OF: /soc/pcie@4c300000: msi-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> > > > > > > > r8169 0000:03:00.0: enabling device (0000 -> 0003)
> > > > > > > > r8169 0000:03:00.0: enabling Mem-Wr-Inval
> > > > > > > > r8169 0000:03:00.0: error -EIO: PCI read failed
> > > > > > > > r8169 0000:03:00.0: probe with driver r8169 failed with error -5
> > > > > >
> > > > > > Can you try remove iommu-map and keep msi-map? then remove msi-map and
> > > > > > keep iommu-map to check which one cause this problem.
> > > > >
> > > > > With only msi-map removed, but smmu enabled:
> > > > > > arm-smmu-v3 490d0000.iommu: event 0x10 received:
> > > > > > arm-smmu-v3 490d0000.iommu:      0x0000001100000010
> > > > > > arm-smmu-v3 490d0000.iommu:      0x0000020a00000000
> > > > > > arm-smmu-v3 490d0000.iommu:      0x000000009b0cc000
> > > > > > arm-smmu-v3 490d0000.iommu:      0x0000000000000000
> > > > > > arm-smmu-v3 490d0000.iommu: event: F_TRANSLATION client: 0000:01:00.0 sid: 0x11 ssid: 0x0 iova: 0x9b0cc000 ipa: 0x0
> > > > > > arm-smmu-v3 490d0000.iommu: priv data read s1 "Input address caused fault" stag: 0x0 r8169 0000:03:00.0
> > > > > > enp3s0: Link is Down
> > > > >
> > > > > With only iommu-map removed, both smmu enabled or disabled:
> > > > > > OF: /soc/pcie@4c300000: msi-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> > > > > > r8169 0000:03:00.0: enabling device (0000 -> 0003)
> > > > > > r8169 0000:03:00.0: enabling Mem-Wr-Inval
> > > > > > r8169 0000:03:00.0: error -EIO: PCI read failed
> > > > > > r8169 0000:03:00.0: probe with driver r8169 failed with error -5
> > > > >
> > > > > Only if smmu is disabled and msi-map is removed the driver probes
> > > > > successfully:
> > > > > > r8169 0000:03:00.0: enabling device (0000 -> 0003)
> > > > > > r8169 0000:03:00.0: enabling Mem-Wr-Inval
> > > > > > r8169 0000:03:00.0 eth0: RTL8168g/8111g, d8:9d:b9:00:16:10, XID 4c0, IRQ 160
> > > > > > r8169 0000:03:00.0 eth0: jumbo features [frames: 9194 bytes, tx checksumming: ko]
> > > > > > r8169 0000:03:00.0 enp3s0: renamed from eth0
> > > > > > r8169 0000:03:00.0: enabling bus mastering
> > > > > > r8169 0000:03:00.0 enp3s0: Link is Down
> > > > >
> > > > > > >
> > > > > > > Without msi-map/iommu-map:
> > > > > > > > r8169 0000:03:00.0: enabling device (0000 -> 0003)
> > > > > > > > r8169 0000:03:00.0: enabling Mem-Wr-Inval
> > > > > > > > r8169 0000:03:00.0 eth0: RTL8168g/8111g, d8:9d:b9:00:16:10, XID 4c0, IRQ 166
> > > > > > > > r8169 0000:03:00.0 eth0: jumbo features [frames: 9194 bytes, tx checksumming: ko]
> > > > > > > > r8169 0000:03:00.0 enp3s0: renamed from eth0
> > > > > > > > r8169 0000:03:00.0: enabling bus mastering
> > > > > > > > r8169 0000:03:00.0 enp3s0: Link is Down
> > > > > > >
> > > > > > > pcie1 works as expected. But this is only a single PCIe device, rather than
> > > > > > > having a PCIe bridge.
> > > > > > > Any idea what's wrong here?
> > > > > >
> > > > > > Can you help dump more information at for PCIe bridge case:
> > > > > >
> > > > > > imx_pcie_add_lut(), need rid and sid information.
> > > > > > drivers/pci/controller/dwc/pci-imx6.c
> > > > >
> > > > > Just to be clear, without msi-map and iommu-map I get:
> > > > > > imx6q-pcie 4c380000.pcie: rid: 0x0, sid: 0x18
> > > > > > imx6q-pcie 4c380000.pcie: rid: 0x100, sid: 0x19
> > > >
> > > > Can you help dump register value PE0_LUT_CREQID offset 0x101 for your
> > > > smmu-map or msi-map enable case
> > >
> > > I am assuming you meant offset 0x101c, as stated in the RM.
> > > I added a dump directly before printing "PCI read failed" in r8169_main.c.
> >
> > Can you point me the code about where "error -EIO: PCI read failed"?
>
> The error messages comes from [1]. As far as I can tell, this is the first
> read after pcim_iomap_table().

Strange, it is CPU read MMIO, should not go through iommu at all.

>
> > I tested nvme devices worked at both PCIE0 and PCIE1.
> >
> > Which PCI switch do you use?
>
> This is a COTS MiniPCIe ethernet card [2]. If I'm reading correctly
> the bridge seems to be a P17C9X20

I order the same chipset one from amazon. Let me debug it after get it.

Frank

>
> Thanks and best regards
> Alexander
>
> [1] https://web.git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git/tree/drivers/net/ethernet/realtek/r8169_main.c?h=next-20250228#n5454
> [2] https://www.delock.com/produkt/95237/merkmale.html?f=s
> --
> TQ-Systems GmbH | Mühlstraße 2, Gut Delling | 82229 Seefeld, Germany
> Amtsgericht München, HRB 105018
> Geschäftsführer: Detlef Schneider, Rüdiger Stahl, Stefan Schneider
> http://www.tq-group.com/
>
>

Re: [PATCH 4/5] arm64: dts: imx95: add PCIe's msi-map and iommu-map property

[Frank Li]

On Fri, Feb 28, 2025 at 12:11:47PM -0500, Frank Li wrote:
> On Fri, Feb 28, 2025 at 05:01:15PM +0100, Alexander Stein wrote:
> > Hi Frank,
> >
> > Am Freitag, 28. Februar 2025, 16:32:50 CET schrieb Frank Li:
> > > On Fri, Feb 28, 2025 at 10:08:58AM +0100, Alexander Stein wrote:
> > > > Hi Frank,
> > > >
> > > > Am Donnerstag, 27. Februar 2025, 17:39:47 CET schrieb Frank Li:
> > > > > On Thu, Feb 27, 2025 at 08:54:13AM +0100, Alexander Stein wrote:
> > > > > > Hi Frank,
> > > > > >
> > > > > > Am Mittwoch, 26. Februar 2025, 17:31:26 CET schrieb Frank Li:
> > > > > > > On Wed, Feb 26, 2025 at 01:11:37PM +0100, Alexander Stein wrote:
> > > > > > > > Hi Frank,
> > > > > > > >
> > > > > > > > Am Dienstag, 28. Januar 2025, 22:15:58 CET schrieb Frank Li:
> > > > > > > > > Add PCIe's msi-map and iommu-map property because i.MX95 support smmu and
> > > > > > > > > its.
> > > > > > > > >
> > > > > > > > > Signed-off-by: Frank Li <Frank.Li@nxp.com>
> > > > > > > > > ---
> > > > > > > > >  arch/arm64/boot/dts/freescale/imx95.dtsi | 14 ++++++++++++++
> > > > > > > > >  1 file changed, 14 insertions(+)
> > > > > > > > >
> > > > > > > > > diff --git a/arch/arm64/boot/dts/freescale/imx95.dtsi b/arch/arm64/boot/dts/freescale/imx95.dtsi
> > > > > > > > > index 6b8470cb3461a..2cebeda43a52d 100644
> > > > > > > > > --- a/arch/arm64/boot/dts/freescale/imx95.dtsi
> > > > > > > > > +++ b/arch/arm64/boot/dts/freescale/imx95.dtsi
> > > > > > > > > @@ -1573,6 +1573,12 @@ pcie0: pcie@4c300000 {
> > > > > > > > >  			assigned-clock-parents = <0>, <0>,
> > > > > > > > >  						 <&scmi_clk IMX95_CLK_SYSPLL1_PFD1_DIV2>;
> > > > > > > > >  			power-domains = <&scmi_devpd IMX95_PD_HSIO_TOP>;
> > > > > > > > > +			/* pcie0's Devid(BIT[7:6]) is 0x00, stream id(BIT[5:0]) is 0x10~0x17 */
> > > > > > > > > +			msi-map = <0x0 &its 0x10 0x1>,
> > > > > > > > > +				  <0x100 &its 0x11 0x7>;
> > > > > > > >
> > > > > > > > Aren't you missing msi-map-mask = <0x1ff>; here? Similar to pcie1.
> > > > > > > > Either way, with this change PCIe on pcie0 is not working anymore,
> > > > > > > > regardless of msi-map-mask.
> > > > > > >
> > > > > > > Yes, it should have msi-map-mask. During my test, I have not enable enetc
> > > > > > > so I have not found this problem.
> > > > > >
> > > > > > Just to be clear: This is not about enetc. This works fine here.
> > > > > >
> > > > > > > > Without msi-map-mask:
> > > > > > > > > OF: /soc/pcie@4c300000: iommu-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> > > > > > > > > OF: /soc/pcie@4c300000: no msi-map translation for id 0x300 on (null)
> > > > > > > > > r8169 0000:03:00.0: error -EINVAL: enable failure
> > > > > > > > > r8169 0000:03:00.0: probe with driver r8169 failed with error -22
> > > > > > > >
> > > > > > > > With msi-map-mask:
> > > > > > > > > OF: /soc/pcie@4c300000: iommu-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> > > > > > > > > OF: /soc/pcie@4c300000: msi-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> > > > > > > > > r8169 0000:03:00.0: enabling device (0000 -> 0003)
> > > > > > > > > r8169 0000:03:00.0: enabling Mem-Wr-Inval
> > > > > > > > > r8169 0000:03:00.0: error -EIO: PCI read failed
> > > > > > > > > r8169 0000:03:00.0: probe with driver r8169 failed with error -5
> > > > > > >
> > > > > > > Can you try remove iommu-map and keep msi-map? then remove msi-map and
> > > > > > > keep iommu-map to check which one cause this problem.
> > > > > >
> > > > > > With only msi-map removed, but smmu enabled:
> > > > > > > arm-smmu-v3 490d0000.iommu: event 0x10 received:
> > > > > > > arm-smmu-v3 490d0000.iommu:      0x0000001100000010
> > > > > > > arm-smmu-v3 490d0000.iommu:      0x0000020a00000000
> > > > > > > arm-smmu-v3 490d0000.iommu:      0x000000009b0cc000
> > > > > > > arm-smmu-v3 490d0000.iommu:      0x0000000000000000
> > > > > > > arm-smmu-v3 490d0000.iommu: event: F_TRANSLATION client: 0000:01:00.0 sid: 0x11 ssid: 0x0 iova: 0x9b0cc000 ipa: 0x0
> > > > > > > arm-smmu-v3 490d0000.iommu: priv data read s1 "Input address caused fault" stag: 0x0 r8169 0000:03:00.0
> > > > > > > enp3s0: Link is Down
> > > > > >
> > > > > > With only iommu-map removed, both smmu enabled or disabled:
> > > > > > > OF: /soc/pcie@4c300000: msi-map, using mask 000001ff, id-base: 00000100, out-base: 00000011, length: 00000007, id: 00000300 -> 00000011
> > > > > > > r8169 0000:03:00.0: enabling device (0000 -> 0003)
> > > > > > > r8169 0000:03:00.0: enabling Mem-Wr-Inval
> > > > > > > r8169 0000:03:00.0: error -EIO: PCI read failed
> > > > > > > r8169 0000:03:00.0: probe with driver r8169 failed with error -5
> > > > > >
> > > > > > Only if smmu is disabled and msi-map is removed the driver probes
> > > > > > successfully:
> > > > > > > r8169 0000:03:00.0: enabling device (0000 -> 0003)
> > > > > > > r8169 0000:03:00.0: enabling Mem-Wr-Inval
> > > > > > > r8169 0000:03:00.0 eth0: RTL8168g/8111g, d8:9d:b9:00:16:10, XID 4c0, IRQ 160
> > > > > > > r8169 0000:03:00.0 eth0: jumbo features [frames: 9194 bytes, tx checksumming: ko]
> > > > > > > r8169 0000:03:00.0 enp3s0: renamed from eth0
> > > > > > > r8169 0000:03:00.0: enabling bus mastering
> > > > > > > r8169 0000:03:00.0 enp3s0: Link is Down
> > > > > >
> > > > > > > >
> > > > > > > > Without msi-map/iommu-map:
> > > > > > > > > r8169 0000:03:00.0: enabling device (0000 -> 0003)
> > > > > > > > > r8169 0000:03:00.0: enabling Mem-Wr-Inval
> > > > > > > > > r8169 0000:03:00.0 eth0: RTL8168g/8111g, d8:9d:b9:00:16:10, XID 4c0, IRQ 166
> > > > > > > > > r8169 0000:03:00.0 eth0: jumbo features [frames: 9194 bytes, tx checksumming: ko]
> > > > > > > > > r8169 0000:03:00.0 enp3s0: renamed from eth0
> > > > > > > > > r8169 0000:03:00.0: enabling bus mastering
> > > > > > > > > r8169 0000:03:00.0 enp3s0: Link is Down
> > > > > > > >
> > > > > > > > pcie1 works as expected. But this is only a single PCIe device, rather than
> > > > > > > > having a PCIe bridge.
> > > > > > > > Any idea what's wrong here?
> > > > > > >
> > > > > > > Can you help dump more information at for PCIe bridge case:
> > > > > > >
> > > > > > > imx_pcie_add_lut(), need rid and sid information.
> > > > > > > drivers/pci/controller/dwc/pci-imx6.c
> > > > > >
> > > > > > Just to be clear, without msi-map and iommu-map I get:
> > > > > > > imx6q-pcie 4c380000.pcie: rid: 0x0, sid: 0x18
> > > > > > > imx6q-pcie 4c380000.pcie: rid: 0x100, sid: 0x19
> > > > >
> > > > > Can you help dump register value PE0_LUT_CREQID offset 0x101 for your
> > > > > smmu-map or msi-map enable case
> > > >
> > > > I am assuming you meant offset 0x101c, as stated in the RM.
> > > > I added a dump directly before printing "PCI read failed" in r8169_main.c.
> > >
> > > Can you point me the code about where "error -EIO: PCI read failed"?
> >
> > The error messages comes from [1]. As far as I can tell, this is the first
> > read after pcim_iomap_table().
>
> Strange, it is CPU read MMIO, should not go through iommu at all.
>
> >
> > > I tested nvme devices worked at both PCIE0 and PCIE1.
> > >
> > > Which PCI switch do you use?
> >
> > This is a COTS MiniPCIe ethernet card [2]. If I'm reading correctly
> > the bridge seems to be a P17C9X20
>
> I order the same chipset one from amazon. Let me debug it after get it.
>

Finially we get realtek PCI card

it quite complex, there are one PCIe switch to split it to two pci bus.

 lspci -t
-[0000:00]---00.0-[01-ff]----00.0-[02-04]--+-03.0-[03]----00.0
                                           \-07.0-[04]----00.0


0000:00:00.0 PCI bridge: Philips Semiconductors Device 0000
0000:01:00.0 PCI bridge: ASMedia Technology Inc. ASM1182e 2-Port PCIe x1 Gen2 Packet Switch
0000:02:03.0 PCI bridge: ASMedia Technology Inc. ASM1182e 2-Port PCIe x1 Gen2 Packet Switch
0000:02:07.0 PCI bridge: ASMedia Technology Inc. ASM1182e 2-Port PCIe x1 Gen2 Packet Switch
0000:03:00.0 Ethernet controller: Realtek Semiconductor Co., Ltd. RTL8111/8168/8211/8411 PCI Express Gigabit Ethernet Controller (rev 09)
0000:04:00.0 Ethernet controller: Realtek Semiconductor Co., Ltd. RTL8111/8168/8211/8411 PCI Express Gigabit Ethernet Controller (rev 09)

It need below change

diff --git a/arch/arm64/boot/dts/freescale/imx95.dtsi b/arch/arm64/boot/dts/freescale/imx95.dtsi
index 9bb26b466a061..9dbf395b9a67b 100644
--- a/arch/arm64/boot/dts/freescale/imx95.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx95.dtsi
@@ -1660,10 +1660,18 @@ pcie0: pcie@4c300000 {
                        power-domains = <&scmi_devpd IMX95_PD_HSIO_TOP>;
                        /* pcie0's Devid(BIT[7:6]) is 0x00, stream id(BIT[5:0]) is 0x10~0x17 */
                        msi-map = <0x0 &its 0x10 0x1>,
-                                 <0x100 &its 0x11 0x7>;
+                                 <0x100 &its 0x11 0x1>,
+                                 <0x218 &its 0x12 0x1>,
+                                 <0x238 &its 0x13 0x1>,
+                                 <0x300 &its 0x14 0x1>,
+                                 <0x400 &its 0x15 0x1>;
                        iommu-map = <0x000 &smmu 0x10 0x1>,
-                                   <0x100 &smmu 0x11 0x7>;
-                       iommu-map-mask = <0x1ff>;
+                                   <0x100 &smmu 0x11 0x1>,
+                                   <0x218 &smmu 0x12 0x1>,
+                                   <0x238 &smmu 0x13 0x1>,
+                                   <0x300 &smmu 0x14 0x1>,
+                                   <0x400 &smmu 0x15 0x1>;
+                       //iommu-map-mask = <0x1ff>;
                        fsl,max-link-speed = <3>;
                        status = "disabled";


Only 8 stream id assign to PCIe0 device, it is hard to dynamaic alloce one,
or need extra works

Frank

> Frank
>
> >
> > Thanks and best regards
> > Alexander
> >
> > [1] https://web.git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git/tree/drivers/net/ethernet/realtek/r8169_main.c?h=next-20250228#n5454
> > [2] https://www.delock.com/produkt/95237/merkmale.html?f=s
> > --
> > TQ-Systems GmbH | Mühlstraße 2, Gut Delling | 82229 Seefeld, Germany
> > Amtsgericht München, HRB 105018
> > Geschäftsführer: Detlef Schneider, Rüdiger Stahl, Stefan Schneider
> > http://www.tq-group.com/
> >
> >

Re: [PATCH 4/5] arm64: dts: imx95: add PCIe's msi-map and iommu-map property

[Alexander Stein]

Hi Frank,

Am Donnerstag, 27. März 2025, 19:48:33 CEST schrieb Frank Li:
> [snip]
> Finially we get realtek PCI card
> 
> it quite complex, there are one PCIe switch to split it to two pci bus.
> 
>  lspci -t
> -[0000:00]---00.0-[01-ff]----00.0-[02-04]--+-03.0-[03]----00.0
>                                            \-07.0-[04]----00.0

Interesting. Mine looks slightly different:

$ lspci -t
-[0000:00]---00.0-[01-ff]----00.0-[02-04]--+-01.0-[03]----00.0
                                           \-02.0-[04]----00.0

> 
> 
> 0000:00:00.0 PCI bridge: Philips Semiconductors Device 0000
> 0000:01:00.0 PCI bridge: ASMedia Technology Inc. ASM1182e 2-Port PCIe x1 Gen2 Packet Switch
> 0000:02:03.0 PCI bridge: ASMedia Technology Inc. ASM1182e 2-Port PCIe x1 Gen2 Packet Switch
> 0000:02:07.0 PCI bridge: ASMedia Technology Inc. ASM1182e 2-Port PCIe x1 Gen2 Packet Switch

It seems you have a newer hardware revision. I have
0000:01:00.0 PCI bridge: Pericom Semiconductor Device a303 (rev 03)
0000:02:01.0 PCI bridge: Pericom Semiconductor Device a303 (rev 03)
0000:02:02.0 PCI bridge: Pericom Semiconductor Device a303 (rev 03)

PCIe bridges.

> 0000:03:00.0 Ethernet controller: Realtek Semiconductor Co., Ltd. RTL8111/8168/8211/8411 PCI Express Gigabit Ethernet Controller (rev 09)
> 0000:04:00.0 Ethernet controller: Realtek Semiconductor Co., Ltd. RTL8111/8168/8211/8411 PCI Express Gigabit Ethernet Controller (rev 09)
> 
> It need below change
> 
> diff --git a/arch/arm64/boot/dts/freescale/imx95.dtsi b/arch/arm64/boot/dts/freescale/imx95.dtsi
> index 9bb26b466a061..9dbf395b9a67b 100644
> --- a/arch/arm64/boot/dts/freescale/imx95.dtsi
> +++ b/arch/arm64/boot/dts/freescale/imx95.dtsi
> @@ -1660,10 +1660,18 @@ pcie0: pcie@4c300000 {
>                         power-domains = <&scmi_devpd IMX95_PD_HSIO_TOP>;
>                         /* pcie0's Devid(BIT[7:6]) is 0x00, stream id(BIT[5:0]) is 0x10~0x17 */
>                         msi-map = <0x0 &its 0x10 0x1>,
> -                                 <0x100 &its 0x11 0x7>;
> +                                 <0x100 &its 0x11 0x1>,
> +                                 <0x218 &its 0x12 0x1>,
> +                                 <0x238 &its 0x13 0x1>,
> +                                 <0x300 &its 0x14 0x1>,
> +                                 <0x400 &its 0x15 0x1>;
>                         iommu-map = <0x000 &smmu 0x10 0x1>,
> -                                   <0x100 &smmu 0x11 0x7>;
> -                       iommu-map-mask = <0x1ff>;
> +                                   <0x100 &smmu 0x11 0x1>,
> +                                   <0x218 &smmu 0x12 0x1>,
> +                                   <0x238 &smmu 0x13 0x1>,
> +                                   <0x300 &smmu 0x14 0x1>,
> +                                   <0x400 &smmu 0x15 0x1>;
> +                       //iommu-map-mask = <0x1ff>;
>                         fsl,max-link-speed = <3>;
>                         status = "disabled";
> 
> 
> Only 8 stream id assign to PCIe0 device, it is hard to dynamaic alloce one,
> or need extra works

Uh, this looks awefully complicated. Even worse this doesn't work on
my hardware. I need mappings for IDs 0x208 and 0x210, so I replaced 0x218
and 0x238 from your diff into my numbers.

So I take that PCIe bridges are not supported properly. What would be
necessary to support this?

Best regards,
Alexander
-- 
TQ-Systems GmbH | Mühlstraße 2, Gut Delling | 82229 Seefeld, Germany
Amtsgericht München, HRB 105018
Geschäftsführer: Detlef Schneider, Rüdiger Stahl, Stefan Schneider
http://www.tq-group.com/

Re: [PATCH 4/5] arm64: dts: imx95: add PCIe's msi-map and iommu-map property

[Frank Li]

On Wed, Apr 09, 2025 at 12:14:48PM +0200, Alexander Stein wrote:
> Hi Frank,
>
> Am Donnerstag, 27. März 2025, 19:48:33 CEST schrieb Frank Li:
> > [snip]
> > Finially we get realtek PCI card
> >
> > it quite complex, there are one PCIe switch to split it to two pci bus.
> >
> >  lspci -t
> > -[0000:00]---00.0-[01-ff]----00.0-[02-04]--+-03.0-[03]----00.0
> >                                            \-07.0-[04]----00.0
>
> Interesting. Mine looks slightly different:
>
> $ lspci -t
> -[0000:00]---00.0-[01-ff]----00.0-[02-04]--+-01.0-[03]----00.0
>                                            \-02.0-[04]----00.0
>
> >
> >
> > 0000:00:00.0 PCI bridge: Philips Semiconductors Device 0000
> > 0000:01:00.0 PCI bridge: ASMedia Technology Inc. ASM1182e 2-Port PCIe x1 Gen2 Packet Switch
> > 0000:02:03.0 PCI bridge: ASMedia Technology Inc. ASM1182e 2-Port PCIe x1 Gen2 Packet Switch
> > 0000:02:07.0 PCI bridge: ASMedia Technology Inc. ASM1182e 2-Port PCIe x1 Gen2 Packet Switch
>
> It seems you have a newer hardware revision. I have
> 0000:01:00.0 PCI bridge: Pericom Semiconductor Device a303 (rev 03)
> 0000:02:01.0 PCI bridge: Pericom Semiconductor Device a303 (rev 03)
> 0000:02:02.0 PCI bridge: Pericom Semiconductor Device a303 (rev 03)
>
> PCIe bridges.
>
> > 0000:03:00.0 Ethernet controller: Realtek Semiconductor Co., Ltd. RTL8111/8168/8211/8411 PCI Express Gigabit Ethernet Controller (rev 09)
> > 0000:04:00.0 Ethernet controller: Realtek Semiconductor Co., Ltd. RTL8111/8168/8211/8411 PCI Express Gigabit Ethernet Controller (rev 09)
> >
> > It need below change
> >
> > diff --git a/arch/arm64/boot/dts/freescale/imx95.dtsi b/arch/arm64/boot/dts/freescale/imx95.dtsi
> > index 9bb26b466a061..9dbf395b9a67b 100644
> > --- a/arch/arm64/boot/dts/freescale/imx95.dtsi
> > +++ b/arch/arm64/boot/dts/freescale/imx95.dtsi
> > @@ -1660,10 +1660,18 @@ pcie0: pcie@4c300000 {
> >                         power-domains = <&scmi_devpd IMX95_PD_HSIO_TOP>;
> >                         /* pcie0's Devid(BIT[7:6]) is 0x00, stream id(BIT[5:0]) is 0x10~0x17 */
> >                         msi-map = <0x0 &its 0x10 0x1>,
> > -                                 <0x100 &its 0x11 0x7>;
> > +                                 <0x100 &its 0x11 0x1>,
> > +                                 <0x218 &its 0x12 0x1>,
> > +                                 <0x238 &its 0x13 0x1>,
> > +                                 <0x300 &its 0x14 0x1>,
> > +                                 <0x400 &its 0x15 0x1>;
> >                         iommu-map = <0x000 &smmu 0x10 0x1>,
> > -                                   <0x100 &smmu 0x11 0x7>;
> > -                       iommu-map-mask = <0x1ff>;
> > +                                   <0x100 &smmu 0x11 0x1>,
> > +                                   <0x218 &smmu 0x12 0x1>,
> > +                                   <0x238 &smmu 0x13 0x1>,
> > +                                   <0x300 &smmu 0x14 0x1>,
> > +                                   <0x400 &smmu 0x15 0x1>;
> > +                       //iommu-map-mask = <0x1ff>;
> >                         fsl,max-link-speed = <3>;
> >                         status = "disabled";
> >
> >
> > Only 8 stream id assign to PCIe0 device, it is hard to dynamaic alloce one,
> > or need extra works
>
> Uh, this looks awefully complicated. Even worse this doesn't work on
> my hardware. I need mappings for IDs 0x208 and 0x210, so I replaced 0x218
> and 0x238 from your diff into my numbers.
>
> So I take that PCIe bridges are not supported properly. What would be
> necessary to support this?

I remember bridge use msi to do port power managements.

ITS msi-map can distribute difference irq to difference cores beside iommu
address protection. It is quite userful for nvme or network devices, which
have multi queues. Of course, we need more elegant solution.

My card use difference pcie switch chip. But suppose it should work after
you update RID information.

which kernel version do you base on?

Frank
>
> Best regards,
> Alexander
> --
> TQ-Systems GmbH | Mühlstraße 2, Gut Delling | 82229 Seefeld, Germany
> Amtsgericht München, HRB 105018
> Geschäftsführer: Detlef Schneider, Rüdiger Stahl, Stefan Schneider
> http://www.tq-group.com/
>
>

Re: [PATCH 4/5] arm64: dts: imx95: add PCIe's msi-map and iommu-map property

[Alexander Stein]

Hi,

Am Mittwoch, 9. April 2025, 16:59:21 CEST schrieb Frank Li:
> On Wed, Apr 09, 2025 at 12:14:48PM +0200, Alexander Stein wrote:
> > Hi Frank,
> >
> > Am Donnerstag, 27. März 2025, 19:48:33 CEST schrieb Frank Li:
> > > [snip]
> > > Finially we get realtek PCI card
> > >
> > > it quite complex, there are one PCIe switch to split it to two pci bus.
> > >
> > >  lspci -t
> > > -[0000:00]---00.0-[01-ff]----00.0-[02-04]--+-03.0-[03]----00.0
> > >                                            \-07.0-[04]----00.0
> >
> > Interesting. Mine looks slightly different:
> >
> > $ lspci -t
> > -[0000:00]---00.0-[01-ff]----00.0-[02-04]--+-01.0-[03]----00.0
> >                                            \-02.0-[04]----00.0
> >
> > >
> > >
> > > 0000:00:00.0 PCI bridge: Philips Semiconductors Device 0000
> > > 0000:01:00.0 PCI bridge: ASMedia Technology Inc. ASM1182e 2-Port PCIe x1 Gen2 Packet Switch
> > > 0000:02:03.0 PCI bridge: ASMedia Technology Inc. ASM1182e 2-Port PCIe x1 Gen2 Packet Switch
> > > 0000:02:07.0 PCI bridge: ASMedia Technology Inc. ASM1182e 2-Port PCIe x1 Gen2 Packet Switch
> >
> > It seems you have a newer hardware revision. I have
> > 0000:01:00.0 PCI bridge: Pericom Semiconductor Device a303 (rev 03)
> > 0000:02:01.0 PCI bridge: Pericom Semiconductor Device a303 (rev 03)
> > 0000:02:02.0 PCI bridge: Pericom Semiconductor Device a303 (rev 03)
> >
> > PCIe bridges.
> >
> > > 0000:03:00.0 Ethernet controller: Realtek Semiconductor Co., Ltd. RTL8111/8168/8211/8411 PCI Express Gigabit Ethernet Controller (rev 09)
> > > 0000:04:00.0 Ethernet controller: Realtek Semiconductor Co., Ltd. RTL8111/8168/8211/8411 PCI Express Gigabit Ethernet Controller (rev 09)
> > >
> > > It need below change
> > >
> > > diff --git a/arch/arm64/boot/dts/freescale/imx95.dtsi b/arch/arm64/boot/dts/freescale/imx95.dtsi
> > > index 9bb26b466a061..9dbf395b9a67b 100644
> > > --- a/arch/arm64/boot/dts/freescale/imx95.dtsi
> > > +++ b/arch/arm64/boot/dts/freescale/imx95.dtsi
> > > @@ -1660,10 +1660,18 @@ pcie0: pcie@4c300000 {
> > >                         power-domains = <&scmi_devpd IMX95_PD_HSIO_TOP>;
> > >                         /* pcie0's Devid(BIT[7:6]) is 0x00, stream id(BIT[5:0]) is 0x10~0x17 */
> > >                         msi-map = <0x0 &its 0x10 0x1>,
> > > -                                 <0x100 &its 0x11 0x7>;
> > > +                                 <0x100 &its 0x11 0x1>,
> > > +                                 <0x218 &its 0x12 0x1>,
> > > +                                 <0x238 &its 0x13 0x1>,
> > > +                                 <0x300 &its 0x14 0x1>,
> > > +                                 <0x400 &its 0x15 0x1>;
> > >                         iommu-map = <0x000 &smmu 0x10 0x1>,
> > > -                                   <0x100 &smmu 0x11 0x7>;
> > > -                       iommu-map-mask = <0x1ff>;
> > > +                                   <0x100 &smmu 0x11 0x1>,
> > > +                                   <0x218 &smmu 0x12 0x1>,
> > > +                                   <0x238 &smmu 0x13 0x1>,
> > > +                                   <0x300 &smmu 0x14 0x1>,
> > > +                                   <0x400 &smmu 0x15 0x1>;
> > > +                       //iommu-map-mask = <0x1ff>;
> > >                         fsl,max-link-speed = <3>;
> > >                         status = "disabled";
> > >
> > >
> > > Only 8 stream id assign to PCIe0 device, it is hard to dynamaic alloce one,
> > > or need extra works
> >
> > Uh, this looks awefully complicated. Even worse this doesn't work on
> > my hardware. I need mappings for IDs 0x208 and 0x210, so I replaced 0x218
> > and 0x238 from your diff into my numbers.
> >
> > So I take that PCIe bridges are not supported properly. What would be
> > necessary to support this?
> 
> I remember bridge use msi to do port power managements.
> 
> ITS msi-map can distribute difference irq to difference cores beside iommu
> address protection. It is quite userful for nvme or network devices, which
> have multi queues. Of course, we need more elegant solution.
> 
> My card use difference pcie switch chip. But suppose it should work after
> you update RID information.

Yep, after adjusting RID mapping, it works here.

> which kernel version do you base on?

My development is usually based on current linux-next versions, so it's
based on v6.15-rc1.

Best regards,
Alexander
-- 
TQ-Systems GmbH | Mühlstraße 2, Gut Delling | 82229 Seefeld, Germany
Amtsgericht München, HRB 105018
Geschäftsführer: Detlef Schneider, Rüdiger Stahl, Stefan Schneider
http://www.tq-group.com/

Re: [PATCH 4/5] arm64: dts: imx95: add PCIe's msi-map and iommu-map property

[Frank Li]

On Fri, Apr 11, 2025 at 08:53:02AM +0200, Alexander Stein wrote:
> Hi,
>
> Am Mittwoch, 9. April 2025, 16:59:21 CEST schrieb Frank Li:
> > On Wed, Apr 09, 2025 at 12:14:48PM +0200, Alexander Stein wrote:
> > > Hi Frank,
> > >
> > > Am Donnerstag, 27. März 2025, 19:48:33 CEST schrieb Frank Li:
> > > > [snip]
> > > > Finially we get realtek PCI card
> > > >
> > > > it quite complex, there are one PCIe switch to split it to two pci bus.
> > > >
> > > >  lspci -t
> > > > -[0000:00]---00.0-[01-ff]----00.0-[02-04]--+-03.0-[03]----00.0
> > > >                                            \-07.0-[04]----00.0
> > >
> > > Interesting. Mine looks slightly different:
> > >
> > > $ lspci -t
> > > -[0000:00]---00.0-[01-ff]----00.0-[02-04]--+-01.0-[03]----00.0
> > >                                            \-02.0-[04]----00.0
> > >
> > > >
> > > >
> > > > 0000:00:00.0 PCI bridge: Philips Semiconductors Device 0000
> > > > 0000:01:00.0 PCI bridge: ASMedia Technology Inc. ASM1182e 2-Port PCIe x1 Gen2 Packet Switch
> > > > 0000:02:03.0 PCI bridge: ASMedia Technology Inc. ASM1182e 2-Port PCIe x1 Gen2 Packet Switch
> > > > 0000:02:07.0 PCI bridge: ASMedia Technology Inc. ASM1182e 2-Port PCIe x1 Gen2 Packet Switch
> > >
> > > It seems you have a newer hardware revision. I have
> > > 0000:01:00.0 PCI bridge: Pericom Semiconductor Device a303 (rev 03)
> > > 0000:02:01.0 PCI bridge: Pericom Semiconductor Device a303 (rev 03)
> > > 0000:02:02.0 PCI bridge: Pericom Semiconductor Device a303 (rev 03)
> > >
> > > PCIe bridges.
> > >
> > > > 0000:03:00.0 Ethernet controller: Realtek Semiconductor Co., Ltd. RTL8111/8168/8211/8411 PCI Express Gigabit Ethernet Controller (rev 09)
> > > > 0000:04:00.0 Ethernet controller: Realtek Semiconductor Co., Ltd. RTL8111/8168/8211/8411 PCI Express Gigabit Ethernet Controller (rev 09)
> > > >
> > > > It need below change
> > > >
> > > > diff --git a/arch/arm64/boot/dts/freescale/imx95.dtsi b/arch/arm64/boot/dts/freescale/imx95.dtsi
> > > > index 9bb26b466a061..9dbf395b9a67b 100644
> > > > --- a/arch/arm64/boot/dts/freescale/imx95.dtsi
> > > > +++ b/arch/arm64/boot/dts/freescale/imx95.dtsi
> > > > @@ -1660,10 +1660,18 @@ pcie0: pcie@4c300000 {
> > > >                         power-domains = <&scmi_devpd IMX95_PD_HSIO_TOP>;
> > > >                         /* pcie0's Devid(BIT[7:6]) is 0x00, stream id(BIT[5:0]) is 0x10~0x17 */
> > > >                         msi-map = <0x0 &its 0x10 0x1>,
> > > > -                                 <0x100 &its 0x11 0x7>;
> > > > +                                 <0x100 &its 0x11 0x1>,
> > > > +                                 <0x218 &its 0x12 0x1>,
> > > > +                                 <0x238 &its 0x13 0x1>,
> > > > +                                 <0x300 &its 0x14 0x1>,
> > > > +                                 <0x400 &its 0x15 0x1>;
> > > >                         iommu-map = <0x000 &smmu 0x10 0x1>,
> > > > -                                   <0x100 &smmu 0x11 0x7>;
> > > > -                       iommu-map-mask = <0x1ff>;
> > > > +                                   <0x100 &smmu 0x11 0x1>,
> > > > +                                   <0x218 &smmu 0x12 0x1>,
> > > > +                                   <0x238 &smmu 0x13 0x1>,
> > > > +                                   <0x300 &smmu 0x14 0x1>,
> > > > +                                   <0x400 &smmu 0x15 0x1>;
> > > > +                       //iommu-map-mask = <0x1ff>;
> > > >                         fsl,max-link-speed = <3>;
> > > >                         status = "disabled";
> > > >
> > > >
> > > > Only 8 stream id assign to PCIe0 device, it is hard to dynamaic alloce one,
> > > > or need extra works
> > >
> > > Uh, this looks awefully complicated. Even worse this doesn't work on
> > > my hardware. I need mappings for IDs 0x208 and 0x210, so I replaced 0x218
> > > and 0x238 from your diff into my numbers.
> > >
> > > So I take that PCIe bridges are not supported properly. What would be
> > > necessary to support this?
> >
> > I remember bridge use msi to do port power managements.
> >
> > ITS msi-map can distribute difference irq to difference cores beside iommu
> > address protection. It is quite userful for nvme or network devices, which
> > have multi queues. Of course, we need more elegant solution.
> >
> > My card use difference pcie switch chip. But suppose it should work after
> > you update RID information.
>
> Yep, after adjusting RID mapping, it works here.

Are you sure it work after adjusting RID mapping? you said
"I take that PCIe bridges are not supported properly"

So I am confused. If it works, I can think how to allocate a stream elegant.

Frank

>
> > which kernel version do you base on?
>
> My development is usually based on current linux-next versions, so it's
> based on v6.15-rc1.
>
> Best regards,
> Alexander
> --
> TQ-Systems GmbH | Mühlstraße 2, Gut Delling | 82229 Seefeld, Germany
> Amtsgericht München, HRB 105018
> Geschäftsführer: Detlef Schneider, Rüdiger Stahl, Stefan Schneider
> http://www.tq-group.com/
>
>

Re: [PATCH 4/5] arm64: dts: imx95: add PCIe's msi-map and iommu-map property

[Alexander Stein]

Hi,

Am Freitag, 11. April 2025, 16:42:29 CEST schrieb Frank Li:
> On Fri, Apr 11, 2025 at 08:53:02AM +0200, Alexander Stein wrote:
> > Hi,
> >
> > Am Mittwoch, 9. April 2025, 16:59:21 CEST schrieb Frank Li:
> > > On Wed, Apr 09, 2025 at 12:14:48PM +0200, Alexander Stein wrote:
> > > > Hi Frank,
> > > >
> > > > Am Donnerstag, 27. März 2025, 19:48:33 CEST schrieb Frank Li:
> > > > > [snip]
> > > > > Finially we get realtek PCI card
> > > > >
> > > > > it quite complex, there are one PCIe switch to split it to two pci bus.
> > > > >
> > > > >  lspci -t
> > > > > -[0000:00]---00.0-[01-ff]----00.0-[02-04]--+-03.0-[03]----00.0
> > > > >                                            \-07.0-[04]----00.0
> > > >
> > > > Interesting. Mine looks slightly different:
> > > >
> > > > $ lspci -t
> > > > -[0000:00]---00.0-[01-ff]----00.0-[02-04]--+-01.0-[03]----00.0
> > > >                                            \-02.0-[04]----00.0
> > > >
> > > > >
> > > > >
> > > > > 0000:00:00.0 PCI bridge: Philips Semiconductors Device 0000
> > > > > 0000:01:00.0 PCI bridge: ASMedia Technology Inc. ASM1182e 2-Port PCIe x1 Gen2 Packet Switch
> > > > > 0000:02:03.0 PCI bridge: ASMedia Technology Inc. ASM1182e 2-Port PCIe x1 Gen2 Packet Switch
> > > > > 0000:02:07.0 PCI bridge: ASMedia Technology Inc. ASM1182e 2-Port PCIe x1 Gen2 Packet Switch
> > > >
> > > > It seems you have a newer hardware revision. I have
> > > > 0000:01:00.0 PCI bridge: Pericom Semiconductor Device a303 (rev 03)
> > > > 0000:02:01.0 PCI bridge: Pericom Semiconductor Device a303 (rev 03)
> > > > 0000:02:02.0 PCI bridge: Pericom Semiconductor Device a303 (rev 03)
> > > >
> > > > PCIe bridges.
> > > >
> > > > > 0000:03:00.0 Ethernet controller: Realtek Semiconductor Co., Ltd. RTL8111/8168/8211/8411 PCI Express Gigabit Ethernet Controller (rev 09)
> > > > > 0000:04:00.0 Ethernet controller: Realtek Semiconductor Co., Ltd. RTL8111/8168/8211/8411 PCI Express Gigabit Ethernet Controller (rev 09)
> > > > >
> > > > > It need below change
> > > > >
> > > > > diff --git a/arch/arm64/boot/dts/freescale/imx95.dtsi b/arch/arm64/boot/dts/freescale/imx95.dtsi
> > > > > index 9bb26b466a061..9dbf395b9a67b 100644
> > > > > --- a/arch/arm64/boot/dts/freescale/imx95.dtsi
> > > > > +++ b/arch/arm64/boot/dts/freescale/imx95.dtsi
> > > > > @@ -1660,10 +1660,18 @@ pcie0: pcie@4c300000 {
> > > > >                         power-domains = <&scmi_devpd IMX95_PD_HSIO_TOP>;
> > > > >                         /* pcie0's Devid(BIT[7:6]) is 0x00, stream id(BIT[5:0]) is 0x10~0x17 */
> > > > >                         msi-map = <0x0 &its 0x10 0x1>,
> > > > > -                                 <0x100 &its 0x11 0x7>;
> > > > > +                                 <0x100 &its 0x11 0x1>,
> > > > > +                                 <0x218 &its 0x12 0x1>,
> > > > > +                                 <0x238 &its 0x13 0x1>,
> > > > > +                                 <0x300 &its 0x14 0x1>,
> > > > > +                                 <0x400 &its 0x15 0x1>;
> > > > >                         iommu-map = <0x000 &smmu 0x10 0x1>,
> > > > > -                                   <0x100 &smmu 0x11 0x7>;
> > > > > -                       iommu-map-mask = <0x1ff>;
> > > > > +                                   <0x100 &smmu 0x11 0x1>,
> > > > > +                                   <0x218 &smmu 0x12 0x1>,
> > > > > +                                   <0x238 &smmu 0x13 0x1>,
> > > > > +                                   <0x300 &smmu 0x14 0x1>,
> > > > > +                                   <0x400 &smmu 0x15 0x1>;
> > > > > +                       //iommu-map-mask = <0x1ff>;
> > > > >                         fsl,max-link-speed = <3>;
> > > > >                         status = "disabled";
> > > > >
> > > > >
> > > > > Only 8 stream id assign to PCIe0 device, it is hard to dynamaic alloce one,
> > > > > or need extra works
> > > >
> > > > Uh, this looks awefully complicated. Even worse this doesn't work on
> > > > my hardware. I need mappings for IDs 0x208 and 0x210, so I replaced 0x218
> > > > and 0x238 from your diff into my numbers.
> > > >
> > > > So I take that PCIe bridges are not supported properly. What would be
> > > > necessary to support this?
> > >
> > > I remember bridge use msi to do port power managements.
> > >
> > > ITS msi-map can distribute difference irq to difference cores beside iommu
> > > address protection. It is quite userful for nvme or network devices, which
> > > have multi queues. Of course, we need more elegant solution.
> > >
> > > My card use difference pcie switch chip. But suppose it should work after
> > > you update RID information.
> >
> > Yep, after adjusting RID mapping, it works here.
> 
> Are you sure it work after adjusting RID mapping? you said
> "I take that PCIe bridges are not supported properly"
> 
> So I am confused. If it works, I can think how to allocate a stream elegant.

Sorry, this sounded misleading. I was referring to the current state.
Without additional adjustments PCIe bridges might not work, AFAIU.
If I adjust the RID mapping, the Ethernet hardware having 2 bridges works on
my platforms.

Best regards
Alexander
-- 
TQ-Systems GmbH | Mühlstraße 2, Gut Delling | 82229 Seefeld, Germany
Amtsgericht München, HRB 105018
Geschäftsführer: Detlef Schneider, Rüdiger Stahl, Stefan Schneider
http://www.tq-group.com/