From: Dave Jiang <dave.jiang@intel.com>
To: Dan Williams <dan.j.williams@intel.com>, <linux-cxl@vger.kernel.org>
Subject: Re: [PATCH v2 11/12] cxl/port: Enumerate cxl link capabilities
Date: Thu, 15 Jun 2023 14:37:07 -0700 [thread overview]
Message-ID: <cc9a65cf-7766-91fc-201d-35dccb64c957@intel.com> (raw)
In-Reply-To: <168679263707.3436160.10946564604121831708.stgit@dwillia2-xfh.jf.intel.com>
On 6/14/23 18:30, Dan Williams wrote:
> Per CXL 3.0 9.14 "Back-Invalidation Configuration", in order to enable
> an HDM-DB range (CXL.mem region with device initiated back-invalidation
> support), all ports in the path between the endpoint and the host bridge
> must be in 256-bit flit-mode.
>
> Even for typical Type-3 class devices it is useful to enumerate link
> capabilities through the topology for debug purposes.
>
> See CXL 3.0 Table-64 "256B Flit Mode vs. 68B Flit Mode Operation", for
> how to determine 64B vs 256B Flit mode operation.
>
> Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Reviewed-by: Dave Jiang <dave.jiang@intel.com>
> ---
> drivers/cxl/core/pci.c | 113 +++++++++++++++++++++++++++++++++++++++++++++++
> drivers/cxl/core/port.c | 6 ++
> drivers/cxl/cxl.h | 7 +++
> drivers/cxl/cxlpci.h | 24 +++++++++-
> drivers/cxl/port.c | 5 ++
> 5 files changed, 153 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
> index 67f4ab6daa34..7440f84be6c8 100644
> --- a/drivers/cxl/core/pci.c
> +++ b/drivers/cxl/core/pci.c
> @@ -519,6 +519,119 @@ int cxl_hdm_decode_init(struct cxl_dev_state *cxlds, struct cxl_hdm *cxlhdm,
> }
> EXPORT_SYMBOL_NS_GPL(cxl_hdm_decode_init, CXL);
>
> +static struct pci_dev *cxl_port_to_pci(struct cxl_port *port)
> +{
> + struct device *dev;
> +
> + if (is_cxl_endpoint(port))
> + dev = port->uport->parent;
> + else
> + dev = port->uport;
> +
> + if (!dev_is_pci(dev))
> + return NULL;
> +
> + return to_pci_dev(dev);
> +}
> +
> +int cxl_probe_link(struct cxl_port *port)
> +{
> + u16 cap, en, lnksta, lnksta2, parent_features;
> + struct pci_dev *pdev = cxl_port_to_pci(port);
> + struct cxl_port *parent_port;
> + unsigned long features;
> + struct device *dev;
> + int rc, dvsec;
> + u32 hdr;
> +
> + if (!pdev) {
> + /*
> + * Assume host bridges support all features, the root
> + * port will dictate the actual enabled set to endpoints.
> + */
> + return 0;
> + }
> +
> + dev = &pdev->dev;
> + dvsec = pci_find_dvsec_capability(pdev, PCI_DVSEC_VENDOR_ID_CXL,
> + CXL_DVSEC_FLEXBUS_PORT);
> + if (!dvsec) {
> + dev_err(dev, "Failed to enumerate port capabilities\n");
> + return -ENXIO;
> + }
> +
> + /*
> + * Cache the link features for future determination of 256B Flit
> + * mode dependent operation support e.g. HDM-DB.
> + */
> + rc = pci_read_config_dword(pdev, dvsec + PCI_DVSEC_HEADER1, &hdr);
> + if (rc)
> + return rc;
> +
> + rc = pci_read_config_word(pdev, dvsec + CXL_DVSEC_FLEXBUS_CAP_OFFSET,
> + &cap);
> + if (rc)
> + return rc;
> +
> + rc = pci_read_config_word(pdev, dvsec + CXL_DVSEC_FLEXBUS_STATUS_OFFSET,
> + &en);
> + if (rc)
> + return rc;
> +
> + features = en & cap & CXL_DVSEC_FLEXBUS_ENABLE_MASK;
> + if ((features & CXL_DVSEC_FLEXBUS_CXL_MASK) == 0)
> + goto no_cxl;
> +
> + /* Determine flit mode from link speed and CXL active */
> + rc = pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnksta);
> + if (rc)
> + return rc;
> +
> + rc = pcie_capability_read_word(pdev, PCI_EXP_LNKSTA2, &lnksta2);
> + if (rc)
> + return rc;
> +
> + /* CXL 3.0 Table-64 256B Flit Mode vs. 68B Flit Mode Operation */
> + switch (FIELD_GET(PCI_EXP_LNKSTA_CLS, lnksta)) {
> + case PCI_EXP_LNKSTA_CLS_2_5GB:
> + case PCI_EXP_LNKSTA_CLS_5_0GB:
> + break;
> + case PCI_EXP_LNKSTA_CLS_8_0GB:
> + case PCI_EXP_LNKSTA_CLS_16_0GB:
> + case PCI_EXP_LNKSTA_CLS_32_0GB:
> + if ((lnksta2 & PCI_EXP_LNKSTA2_FLIT) == 0) {
> + features |= CXL_PORT_FEATURE_FLIT68;
> + break;
> + }
> + fallthrough;
> + case PCI_EXP_LNKSTA_CLS_64_0GB:
> + default:
> + features |= CXL_PORT_FEATURE_FLIT256;
> + break;
> + }
> +
> +no_cxl:
> + parent_port = to_cxl_port(port->dev.parent);
> + parent_features = parent_port->features;
> +
> + /* Enforce port features are plumbed through to the host bridge */
> + features &= parent_features;
> +
> + dev_dbg(dev, "features:%s%s%s%s%s%s%s%s%s\n",
> + features & CXL_DVSEC_FLEXBUS_CACHE_ENABLED ? " cache" : "",
> + features & CXL_DVSEC_FLEXBUS_IO_ENABLED ? " io" : "",
> + features & CXL_DVSEC_FLEXBUS_MEM_ENABLED ? " mem" : "",
> + features & CXL_DVSEC_FLEXBUS_VH_ENABLED ? " vh" : "",
> + features & CXL_DVSEC_FLEXBUS_MLD_ENABLED ? " mld" : "",
> + features & CXL_DVSEC_FLEXBUS_LATOPT_ENABLED ? " latopt" : "",
> + features & CXL_DVSEC_FLEXBUS_PBR_ENABLED ? " pbr" : "",
> + features & CXL_PORT_FEATURE_FLIT68 ? " flit68" : "",
> + features & CXL_PORT_FEATURE_FLIT256 ? " flit256" : "");
> +
> + return 0;
> +}
> +EXPORT_SYMBOL_NS_GPL(cxl_probe_link, CXL);
> +
> #define CXL_DOE_TABLE_ACCESS_REQ_CODE 0x000000ff
> #define CXL_DOE_TABLE_ACCESS_REQ_CODE_READ 0
> #define CXL_DOE_TABLE_ACCESS_TABLE_TYPE 0x0000ff00
> diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c
> index 6d7811b26b5a..bf8f25063914 100644
> --- a/drivers/cxl/core/port.c
> +++ b/drivers/cxl/core/port.c
> @@ -665,6 +665,12 @@ static struct cxl_port *cxl_port_alloc(struct device *uport,
> } else
> dev->parent = uport;
>
> + /*
> + * Assume all CXL link capabilities for root-device-to-host-bridge link,
> + * cxl_probe_link() will fix this up later in cxl_probe_link() for all
> + * other ports.
> + */
> + port->features = CXL_DVSEC_FLEXBUS_ENABLE_MASK;
> port->component_reg_phys = component_reg_phys;
> ida_init(&port->decoder_ida);
> port->hdm_end = -1;
> diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
> index f309b1387858..74548f8f5f4c 100644
> --- a/drivers/cxl/cxl.h
> +++ b/drivers/cxl/cxl.h
> @@ -536,6 +536,10 @@ struct cxl_dax_region {
> struct range hpa_range;
> };
>
> +/* These start after CXL_DVSEC_FLEXBUS_*_ENABLED bits in port->features */
> +#define CXL_PORT_FEATURE_FLIT68 BIT(16)
> +#define CXL_PORT_FEATURE_FLIT256 BIT(17)
> +
> /**
> * struct cxl_port - logical collection of upstream port devices and
> * downstream port devices to construct a CXL memory
> @@ -557,6 +561,8 @@ struct cxl_dax_region {
> * @depth: How deep this port is relative to the root. depth 0 is the root.
> * @cdat: Cached CDAT data
> * @cdat_available: Should a CDAT attribute be available in sysfs
> + * @features: active link features (see CXL_DVSEC_FLEXBUS_*_ENABLED +
> + * CXL_PORT_FEATURE_*)
> */
> struct cxl_port {
> struct device dev;
> @@ -579,6 +585,7 @@ struct cxl_port {
> size_t length;
> } cdat;
> bool cdat_available;
> + unsigned long features;
> };
>
> static inline struct cxl_dport *
> diff --git a/drivers/cxl/cxlpci.h b/drivers/cxl/cxlpci.h
> index 7c02e55b8042..0da6618e0df7 100644
> --- a/drivers/cxl/cxlpci.h
> +++ b/drivers/cxl/cxlpci.h
> @@ -45,8 +45,27 @@
> /* CXL 2.0 8.1.7: GPF DVSEC for CXL Device */
> #define CXL_DVSEC_DEVICE_GPF 5
>
> -/* CXL 2.0 8.1.8: PCIe DVSEC for Flex Bus Port */
> -#define CXL_DVSEC_PCIE_FLEXBUS_PORT 7
> +/* CXL 3.0 8.2.1.3: PCIe DVSEC for Flex Bus Port */
> +#define CXL_DVSEC_FLEXBUS_PORT 7
> +#define CXL_DVSEC_FLEXBUS_CAP_OFFSET 0xA
> +#define CXL_DVSEC_FLEXBUS_CACHE_CAPABLE BIT(0)
> +#define CXL_DVSEC_FLEXBUS_IO_CAPABLE BIT(1)
> +#define CXL_DVSEC_FLEXBUS_MEM_CAPABLE BIT(2)
> +#define CXL_DVSEC_FLEXBUS_VH_CAPABLE BIT(5)
> +#define CXL_DVSEC_FLEXBUS_MLD_CAPABLE BIT(6)
> +#define CXL_DVSEC_FLEXBUS_LATOPT_CAPABLE BIT(13)
> +#define CXL_DVSEC_FLEXBUS_PBR_CAPABLE BIT(14)
> +#define CXL_DVSEC_FLEXBUS_STATUS_OFFSET 0xE
> +#define CXL_DVSEC_FLEXBUS_CACHE_ENABLED BIT(0)
> +#define CXL_DVSEC_FLEXBUS_IO_ENABLED BIT(1)
> +#define CXL_DVSEC_FLEXBUS_MEM_ENABLED BIT(2)
> +#define CXL_DVSEC_FLEXBUS_VH_ENABLED BIT(5)
> +#define CXL_DVSEC_FLEXBUS_MLD_ENABLED BIT(6)
> +#define CXL_DVSEC_FLEXBUS_LATOPT_ENABLED BIT(13)
> +#define CXL_DVSEC_FLEXBUS_PBR_ENABLED BIT(14)
> +#define CXL_DVSEC_FLEXBUS_ENABLE_MASK \
> + (GENMASK(2, 0) | GENMASK(6, 5) | GENMASK(14, 13))
> +#define CXL_DVSEC_FLEXBUS_CXL_MASK GENMASK(2, 0)
>
> /* CXL 2.0 8.1.9: Register Locator DVSEC */
> #define CXL_DVSEC_REG_LOCATOR 8
> @@ -88,6 +107,7 @@ int devm_cxl_port_enumerate_dports(struct cxl_port *port);
> struct cxl_dev_state;
> int cxl_hdm_decode_init(struct cxl_dev_state *cxlds, struct cxl_hdm *cxlhdm,
> struct cxl_endpoint_dvsec_info *info);
> +int cxl_probe_link(struct cxl_port *port);
> void read_cdat_data(struct cxl_port *port);
> void cxl_cor_error_detected(struct pci_dev *pdev);
> pci_ers_result_t cxl_error_detected(struct pci_dev *pdev,
> diff --git a/drivers/cxl/port.c b/drivers/cxl/port.c
> index c23b6164e1c0..5ffe3c7d2f5e 100644
> --- a/drivers/cxl/port.c
> +++ b/drivers/cxl/port.c
> @@ -140,6 +140,11 @@ static int cxl_endpoint_port_probe(struct cxl_port *port)
> static int cxl_port_probe(struct device *dev)
> {
> struct cxl_port *port = to_cxl_port(dev);
> + int rc;
> +
> + rc = cxl_probe_link(port);
> + if (rc)
> + return rc;
>
> if (is_cxl_endpoint(port))
> return cxl_endpoint_port_probe(port);
>
next prev parent reply other threads:[~2023-06-15 21:37 UTC|newest]
Thread overview: 27+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-06-15 1:29 [PATCH v2 00/12] Device memory prep Dan Williams
2023-06-15 1:29 ` [PATCH v2 01/12] cxl/regs: Clarify when a 'struct cxl_register_map' is input vs output Dan Williams
2023-06-15 1:29 ` [PATCH v2 02/12] tools/testing/cxl: Remove unused @cxlds argument Dan Williams
2023-06-15 1:29 ` [PATCH v2 03/12] cxl: Fix kernel-doc warnings Dan Williams
2023-06-15 21:28 ` Dave Jiang
2023-06-22 13:50 ` Jonathan Cameron
2023-06-15 1:29 ` [PATCH v2 04/12] cxl: Remove leftover attribute documentation in 'struct cxl_dev_state' Dan Williams
2023-06-15 21:29 ` Dave Jiang
2023-06-22 13:50 ` Jonathan Cameron
2023-06-15 1:30 ` [PATCH v2 05/12] cxl/mbox: Move mailbox related driver state to its own data structure Dan Williams
2023-06-15 1:30 ` [PATCH v2 06/12] cxl/memdev: Make mailbox functionality optional Dan Williams
2023-06-15 21:30 ` Dave Jiang
2023-06-22 13:52 ` Jonathan Cameron
2023-06-15 1:30 ` [PATCH v2 07/12] cxl/port: Rename CXL_DECODER_{EXPANDER, ACCELERATOR} => {HOSTONLYMEM, DEVMEM} Dan Williams
2023-06-15 21:31 ` Dave Jiang
2023-06-15 1:30 ` [PATCH v2 08/12] cxl/hdm: Default CXL_DEVTYPE_DEVMEM decoders to CXL_DECODER_DEVMEM Dan Williams
2023-06-15 21:32 ` Dave Jiang
2023-06-15 1:30 ` [PATCH v2 09/12] cxl/region: Manage decoder target_type at decoder-attach time Dan Williams
2023-06-15 1:30 ` [PATCH v2 10/12] cxl/pci: Unconditionally unmask 256B Flit errors Dan Williams
2023-06-15 21:34 ` Dave Jiang
2023-06-22 13:55 ` Jonathan Cameron
2023-06-15 1:30 ` [PATCH v2 11/12] cxl/port: Enumerate cxl link capabilities Dan Williams
2023-06-15 21:37 ` Dave Jiang [this message]
2023-06-16 0:07 ` Dan Williams
2023-06-16 19:16 ` Dan Williams
2023-06-22 14:04 ` Jonathan Cameron
2023-06-15 1:30 ` [PATCH v2 12/12] cxl/memdev: Formalize endpoint port linkage Dan Williams
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=cc9a65cf-7766-91fc-201d-35dccb64c957@intel.com \
--to=dave.jiang@intel.com \
--cc=dan.j.williams@intel.com \
--cc=linux-cxl@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox