* [PATCH 0/1][v2] PCI: VF resizable BAR @ 2024-05-23 7:11 Lianjie Shi 2024-05-23 7:11 ` [PATCH 1/1][v2] PCI: Support " Lianjie Shi 0 siblings, 1 reply; 7+ messages in thread From: Lianjie Shi @ 2024-05-23 7:11 UTC (permalink / raw) To: linux-pci, linux-kernel, Bjorn Helgaas Cc: HaiJun Chang, Jerry Jiang, Andy Zhang, Lianjie Shi Similar to regular BAR, drivers can use pci_resize_resource() to resize an IOV BAR to the desired size provided that is supported by the hardware, which can be queried using pci_rebar_get_possible_sizes(). This feature is based on the fact that (default VF BAR size) * (supported VF number) covers all possible resource/address ranges (rounded up to the power of 2 size). For example, the total size of the resource behind the BAR is 256GB, the supported maximum VF number is 4, the default VF BAR size should then be set to 64GB. When the enabled vf_num changes, the VF BAR size will adjust accordingly as - For 1 VF, VF BAR size is 256GB - For 2 VFs, VF BAR size is 128GB - For 4 VFs, VF BAR size is 64GB This feature is necessary to accommodate the limited address per PCI port. Lianjie Shi (1): PCI: Support VF resizable BAR drivers/pci/pci.c | 47 ++++++++++++++++++++++++++++++++++- drivers/pci/setup-res.c | 45 +++++++++++++++++++++++++++------ include/uapi/linux/pci_regs.h | 1 + 3 files changed, 85 insertions(+), 8 deletions(-) base-commit: cf87f46fd34d6c19283d9625a7822f20d90b64a4 -- 2.34.1 ^ permalink raw reply [flat|nested] 7+ messages in thread
* [PATCH 1/1][v2] PCI: Support VF resizable BAR 2024-05-23 7:11 [PATCH 0/1][v2] PCI: VF resizable BAR Lianjie Shi @ 2024-05-23 7:11 ` Lianjie Shi 2024-05-31 4:26 ` Chang, HaiJun 0 siblings, 1 reply; 7+ messages in thread From: Lianjie Shi @ 2024-05-23 7:11 UTC (permalink / raw) To: linux-pci, linux-kernel, Bjorn Helgaas Cc: HaiJun Chang, Jerry Jiang, Andy Zhang, Lianjie Shi Add support for VF resizable BAR PCI extended cap. Similar to regular BAR, drivers can use pci_resize_resource() to resize an IOV BAR. For each VF, dev->sriov->barsz of the IOV BAR is resized, but the total resource size of the IOV resource should not exceed its original size upon init. Based on following patch series: Link: https://lore.kernel.org/lkml/YbqGplTKl5i%2F1%2FkY@rocinante/T/ Signed-off-by: Lianjie Shi <Lianjie.Shi@amd.com> --- drivers/pci/pci.c | 47 ++++++++++++++++++++++++++++++++++- drivers/pci/setup-res.c | 45 +++++++++++++++++++++++++++------ include/uapi/linux/pci_regs.h | 1 + 3 files changed, 85 insertions(+), 8 deletions(-) diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index e5f243dd4..12f86e00a 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -1867,6 +1867,42 @@ static void pci_restore_rebar_state(struct pci_dev *pdev) } } +static void pci_restore_vf_rebar_state(struct pci_dev *pdev) +{ +#ifdef CONFIG_PCI_IOV + unsigned int pos, nbars, i; + u32 ctrl; + u16 total; + + pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_VF_REBAR); + if (!pos) + return; + + pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); + nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, ctrl); + + for (i = 0; i < nbars; i++, pos += 8) { + struct resource *res; + int bar_idx, size; + u64 tmp; + + pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); + bar_idx = ctrl & PCI_REBAR_CTRL_BAR_IDX; + total = pdev->sriov->total_VFs; + if (!total) + return; + + res = pdev->resource + bar_idx + PCI_IOV_RESOURCES; + tmp = resource_size(res); + do_div(tmp, total); + size = pci_rebar_bytes_to_size(tmp); + ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE; + ctrl |= FIELD_PREP(PCI_REBAR_CTRL_BAR_SIZE, size); + pci_write_config_dword(pdev, pos + PCI_REBAR_CTRL, ctrl); + } +#endif +} + /** * pci_restore_state - Restore the saved state of a PCI device * @dev: PCI device that we're dealing with @@ -1882,6 +1918,7 @@ void pci_restore_state(struct pci_dev *dev) pci_restore_ats_state(dev); pci_restore_vc_state(dev); pci_restore_rebar_state(dev); + pci_restore_vf_rebar_state(dev); pci_restore_dpc_state(dev); pci_restore_ptm_state(dev); @@ -3677,10 +3714,18 @@ void pci_acs_init(struct pci_dev *dev) */ static int pci_rebar_find_pos(struct pci_dev *pdev, int bar) { + int cap = PCI_EXT_CAP_ID_REBAR; unsigned int pos, nbars, i; u32 ctrl; - pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_REBAR); +#ifdef CONFIG_PCI_IOV + if (bar >= PCI_IOV_RESOURCES) { + cap = PCI_EXT_CAP_ID_VF_REBAR; + bar -= PCI_IOV_RESOURCES; + } +#endif + + pos = pci_find_ext_capability(pdev, cap); if (!pos) return -ENOTSUPP; diff --git a/drivers/pci/setup-res.c b/drivers/pci/setup-res.c index c6d933ddf..d978a2ccf 100644 --- a/drivers/pci/setup-res.c +++ b/drivers/pci/setup-res.c @@ -427,13 +427,32 @@ void pci_release_resource(struct pci_dev *dev, int resno) } EXPORT_SYMBOL(pci_release_resource); +static int pci_memory_decoding(struct pci_dev *dev, int resno) +{ + u16 cmd; + +#ifdef CONFIG_PCI_IOV + if (resno >= PCI_IOV_RESOURCES) { + pci_read_config_word(dev, dev->sriov->pos + PCI_SRIOV_CTRL, &cmd); + if (cmd & PCI_SRIOV_CTRL_MSE) + return -EBUSY; + else + return 0; + } +#endif + pci_read_config_word(dev, PCI_COMMAND, &cmd); + if (cmd & PCI_COMMAND_MEMORY) + return -EBUSY; + + return 0; +} + int pci_resize_resource(struct pci_dev *dev, int resno, int size) { struct resource *res = dev->resource + resno; struct pci_host_bridge *host; int old, ret; u32 sizes; - u16 cmd; /* Check if we must preserve the firmware's resource assignment */ host = pci_find_host_bridge(dev->bus); @@ -444,9 +463,9 @@ int pci_resize_resource(struct pci_dev *dev, int resno, int size) if (!(res->flags & IORESOURCE_UNSET)) return -EBUSY; - pci_read_config_word(dev, PCI_COMMAND, &cmd); - if (cmd & PCI_COMMAND_MEMORY) - return -EBUSY; + ret = pci_memory_decoding(dev, resno); + if (ret) + return ret; sizes = pci_rebar_get_possible_sizes(dev, resno); if (!sizes) @@ -463,19 +482,31 @@ int pci_resize_resource(struct pci_dev *dev, int resno, int size) if (ret) return ret; - res->end = res->start + pci_rebar_size_to_bytes(size) - 1; +#ifdef CONFIG_PCI_IOV + if (resno >= PCI_IOV_RESOURCES) + dev->sriov->barsz[resno - PCI_IOV_RESOURCES] = pci_rebar_size_to_bytes(size); + else +#endif + res->end = res->start + pci_rebar_size_to_bytes(size) - 1; /* Check if the new config works by trying to assign everything. */ if (dev->bus->self) { ret = pci_reassign_bridge_resources(dev->bus->self, res->flags); - if (ret) + if (ret && ret != -ENOENT) goto error_resize; } return 0; error_resize: pci_rebar_set_size(dev, resno, old); - res->end = res->start + pci_rebar_size_to_bytes(old) - 1; + +#ifdef CONFIG_PCI_IOV + if (resno >= PCI_IOV_RESOURCES) + dev->sriov->barsz[resno - PCI_IOV_RESOURCES] = pci_rebar_size_to_bytes(old); + else +#endif + res->end = res->start + pci_rebar_size_to_bytes(old) - 1; + return ret; } EXPORT_SYMBOL(pci_resize_resource); diff --git a/include/uapi/linux/pci_regs.h b/include/uapi/linux/pci_regs.h index a39193213..a66b90982 100644 --- a/include/uapi/linux/pci_regs.h +++ b/include/uapi/linux/pci_regs.h @@ -738,6 +738,7 @@ #define PCI_EXT_CAP_ID_L1SS 0x1E /* L1 PM Substates */ #define PCI_EXT_CAP_ID_PTM 0x1F /* Precision Time Measurement */ #define PCI_EXT_CAP_ID_DVSEC 0x23 /* Designated Vendor-Specific */ +#define PCI_EXT_CAP_ID_VF_REBAR 0x24 /* VF Resizable BAR */ #define PCI_EXT_CAP_ID_DLF 0x25 /* Data Link Feature */ #define PCI_EXT_CAP_ID_PL_16GT 0x26 /* Physical Layer 16.0 GT/s */ #define PCI_EXT_CAP_ID_PL_32GT 0x2A /* Physical Layer 32.0 GT/s */ -- 2.34.1 ^ permalink raw reply related [flat|nested] 7+ messages in thread
* RE: [PATCH 1/1][v2] PCI: Support VF resizable BAR 2024-05-23 7:11 ` [PATCH 1/1][v2] PCI: Support " Lianjie Shi @ 2024-05-31 4:26 ` Chang, HaiJun 2024-06-07 8:42 ` Chang, HaiJun 0 siblings, 1 reply; 7+ messages in thread From: Chang, HaiJun @ 2024-05-31 4:26 UTC (permalink / raw) To: Shi, Lianjie, linux-pci@vger.kernel.org, linux-kernel@vger.kernel.org, Bjorn Helgaas Cc: Jiang, Jerry (SW), Zhang, Andy, Shi, Lianjie [AMD Official Use Only - AMD Internal Distribution Only] Hi, Bjorn and pci driver maintainers Could you review the VF resizable BAR patch? Thanks, HaiJun -----Original Message----- From: Lianjie Shi <Lianjie.Shi@amd.com> Sent: Thursday, May 23, 2024 3:11 PM To: linux-pci@vger.kernel.org; linux-kernel@vger.kernel.org; Bjorn Helgaas <bhelgaas@google.com> Cc: Chang, HaiJun <HaiJun.Chang@amd.com>; Jiang, Jerry (SW) <Jerry.Jiang@amd.com>; Zhang, Andy <Andy.Zhang@amd.com>; Shi, Lianjie <Lianjie.Shi@amd.com> Subject: [PATCH 1/1][v2] PCI: Support VF resizable BAR Add support for VF resizable BAR PCI extended cap. Similar to regular BAR, drivers can use pci_resize_resource() to resize an IOV BAR. For each VF, dev->sriov->barsz of the IOV BAR is resized, but the total resource size of the IOV resource should not exceed its original size upon init. Based on following patch series: Link: https://lore.kernel.org/lkml/YbqGplTKl5i%2F1%2FkY@rocinante/T/ Signed-off-by: Lianjie Shi <Lianjie.Shi@amd.com> --- drivers/pci/pci.c | 47 ++++++++++++++++++++++++++++++++++- drivers/pci/setup-res.c | 45 +++++++++++++++++++++++++++------ include/uapi/linux/pci_regs.h | 1 + 3 files changed, 85 insertions(+), 8 deletions(-) diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index e5f243dd4..12f86e00a 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -1867,6 +1867,42 @@ static void pci_restore_rebar_state(struct pci_dev *pdev) } } +static void pci_restore_vf_rebar_state(struct pci_dev *pdev) { #ifdef +CONFIG_PCI_IOV + unsigned int pos, nbars, i; + u32 ctrl; + u16 total; + + pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_VF_REBAR); + if (!pos) + return; + + pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); + nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, ctrl); + + for (i = 0; i < nbars; i++, pos += 8) { + struct resource *res; + int bar_idx, size; + u64 tmp; + + pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); + bar_idx = ctrl & PCI_REBAR_CTRL_BAR_IDX; + total = pdev->sriov->total_VFs; + if (!total) + return; + + res = pdev->resource + bar_idx + PCI_IOV_RESOURCES; + tmp = resource_size(res); + do_div(tmp, total); + size = pci_rebar_bytes_to_size(tmp); + ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE; + ctrl |= FIELD_PREP(PCI_REBAR_CTRL_BAR_SIZE, size); + pci_write_config_dword(pdev, pos + PCI_REBAR_CTRL, ctrl); + } +#endif +} + /** * pci_restore_state - Restore the saved state of a PCI device * @dev: PCI device that we're dealing with @@ -1882,6 +1918,7 @@ void pci_restore_state(struct pci_dev *dev) pci_restore_ats_state(dev); pci_restore_vc_state(dev); pci_restore_rebar_state(dev); + pci_restore_vf_rebar_state(dev); pci_restore_dpc_state(dev); pci_restore_ptm_state(dev); @@ -3677,10 +3714,18 @@ void pci_acs_init(struct pci_dev *dev) */ static int pci_rebar_find_pos(struct pci_dev *pdev, int bar) { + int cap = PCI_EXT_CAP_ID_REBAR; unsigned int pos, nbars, i; u32 ctrl; - pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_REBAR); +#ifdef CONFIG_PCI_IOV + if (bar >= PCI_IOV_RESOURCES) { + cap = PCI_EXT_CAP_ID_VF_REBAR; + bar -= PCI_IOV_RESOURCES; + } +#endif + + pos = pci_find_ext_capability(pdev, cap); if (!pos) return -ENOTSUPP; diff --git a/drivers/pci/setup-res.c b/drivers/pci/setup-res.c index c6d933ddf..d978a2ccf 100644 --- a/drivers/pci/setup-res.c +++ b/drivers/pci/setup-res.c @@ -427,13 +427,32 @@ void pci_release_resource(struct pci_dev *dev, int resno) } EXPORT_SYMBOL(pci_release_resource); +static int pci_memory_decoding(struct pci_dev *dev, int resno) { + u16 cmd; + +#ifdef CONFIG_PCI_IOV + if (resno >= PCI_IOV_RESOURCES) { + pci_read_config_word(dev, dev->sriov->pos + PCI_SRIOV_CTRL, &cmd); + if (cmd & PCI_SRIOV_CTRL_MSE) + return -EBUSY; + else + return 0; + } +#endif + pci_read_config_word(dev, PCI_COMMAND, &cmd); + if (cmd & PCI_COMMAND_MEMORY) + return -EBUSY; + + return 0; +} + int pci_resize_resource(struct pci_dev *dev, int resno, int size) { struct resource *res = dev->resource + resno; struct pci_host_bridge *host; int old, ret; u32 sizes; - u16 cmd; /* Check if we must preserve the firmware's resource assignment */ host = pci_find_host_bridge(dev->bus); @@ -444,9 +463,9 @@ int pci_resize_resource(struct pci_dev *dev, int resno, int size) if (!(res->flags & IORESOURCE_UNSET)) return -EBUSY; - pci_read_config_word(dev, PCI_COMMAND, &cmd); - if (cmd & PCI_COMMAND_MEMORY) - return -EBUSY; + ret = pci_memory_decoding(dev, resno); + if (ret) + return ret; sizes = pci_rebar_get_possible_sizes(dev, resno); if (!sizes) @@ -463,19 +482,31 @@ int pci_resize_resource(struct pci_dev *dev, int resno, int size) if (ret) return ret; - res->end = res->start + pci_rebar_size_to_bytes(size) - 1; +#ifdef CONFIG_PCI_IOV + if (resno >= PCI_IOV_RESOURCES) + dev->sriov->barsz[resno - PCI_IOV_RESOURCES] = pci_rebar_size_to_bytes(size); + else +#endif + res->end = res->start + pci_rebar_size_to_bytes(size) - 1; /* Check if the new config works by trying to assign everything. */ if (dev->bus->self) { ret = pci_reassign_bridge_resources(dev->bus->self, res->flags); - if (ret) + if (ret && ret != -ENOENT) goto error_resize; } return 0; error_resize: pci_rebar_set_size(dev, resno, old); - res->end = res->start + pci_rebar_size_to_bytes(old) - 1; + +#ifdef CONFIG_PCI_IOV + if (resno >= PCI_IOV_RESOURCES) + dev->sriov->barsz[resno - PCI_IOV_RESOURCES] = pci_rebar_size_to_bytes(old); + else +#endif + res->end = res->start + pci_rebar_size_to_bytes(old) - 1; + return ret; } EXPORT_SYMBOL(pci_resize_resource); diff --git a/include/uapi/linux/pci_regs.h b/include/uapi/linux/pci_regs.h index a39193213..a66b90982 100644 --- a/include/uapi/linux/pci_regs.h +++ b/include/uapi/linux/pci_regs.h @@ -738,6 +738,7 @@ #define PCI_EXT_CAP_ID_L1SS 0x1E /* L1 PM Substates */ #define PCI_EXT_CAP_ID_PTM 0x1F /* Precision Time Measurement */ #define PCI_EXT_CAP_ID_DVSEC 0x23 /* Designated Vendor-Specific */ +#define PCI_EXT_CAP_ID_VF_REBAR 0x24 /* VF Resizable BAR */ #define PCI_EXT_CAP_ID_DLF 0x25 /* Data Link Feature */ #define PCI_EXT_CAP_ID_PL_16GT 0x26 /* Physical Layer 16.0 GT/s */ #define PCI_EXT_CAP_ID_PL_32GT 0x2A /* Physical Layer 32.0 GT/s */ -- 2.34.1 ^ permalink raw reply [flat|nested] 7+ messages in thread
* RE: [PATCH 1/1][v2] PCI: Support VF resizable BAR 2024-05-31 4:26 ` Chang, HaiJun @ 2024-06-07 8:42 ` Chang, HaiJun 2024-06-07 11:37 ` Christian König 0 siblings, 1 reply; 7+ messages in thread From: Chang, HaiJun @ 2024-06-07 8:42 UTC (permalink / raw) To: Shi, Lianjie, linux-pci@vger.kernel.org, linux-kernel@vger.kernel.org, Bjorn Helgaas, Koenig, Christian Cc: Jiang, Jerry (SW), Zhang, Andy, Shi, Lianjie [AMD Official Use Only - AMD Internal Distribution Only] + Christian to review the VF resizable BAR patch. -----Original Message----- From: Chang, HaiJun Sent: Friday, May 31, 2024 12:26 PM To: Lianjie Shi <Lianjie.Shi@amd.com>; linux-pci@vger.kernel.org; linux-kernel@vger.kernel.org; Bjorn Helgaas <bhelgaas@google.com> Cc: Jiang, Jerry (SW) <Jerry.Jiang@amd.com>; Zhang, Andy <Andy.Zhang@amd.com>; Shi, Lianjie <Lianjie.Shi@amd.com> Subject: RE: [PATCH 1/1][v2] PCI: Support VF resizable BAR Hi, Bjorn and pci driver maintainers Could you review the VF resizable BAR patch? Thanks, HaiJun -----Original Message----- From: Lianjie Shi <Lianjie.Shi@amd.com> Sent: Thursday, May 23, 2024 3:11 PM To: linux-pci@vger.kernel.org; linux-kernel@vger.kernel.org; Bjorn Helgaas <bhelgaas@google.com> Cc: Chang, HaiJun <HaiJun.Chang@amd.com>; Jiang, Jerry (SW) <Jerry.Jiang@amd.com>; Zhang, Andy <Andy.Zhang@amd.com>; Shi, Lianjie <Lianjie.Shi@amd.com> Subject: [PATCH 1/1][v2] PCI: Support VF resizable BAR Add support for VF resizable BAR PCI extended cap. Similar to regular BAR, drivers can use pci_resize_resource() to resize an IOV BAR. For each VF, dev->sriov->barsz of the IOV BAR is resized, but the total resource size of the IOV resource should not exceed its original size upon init. Based on following patch series: Link: https://lore.kernel.org/lkml/YbqGplTKl5i%2F1%2FkY@rocinante/T/ Signed-off-by: Lianjie Shi <Lianjie.Shi@amd.com> --- drivers/pci/pci.c | 47 ++++++++++++++++++++++++++++++++++- drivers/pci/setup-res.c | 45 +++++++++++++++++++++++++++------ include/uapi/linux/pci_regs.h | 1 + 3 files changed, 85 insertions(+), 8 deletions(-) diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index e5f243dd4..12f86e00a 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -1867,6 +1867,42 @@ static void pci_restore_rebar_state(struct pci_dev *pdev) } } +static void pci_restore_vf_rebar_state(struct pci_dev *pdev) { #ifdef +CONFIG_PCI_IOV + unsigned int pos, nbars, i; + u32 ctrl; + u16 total; + + pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_VF_REBAR); + if (!pos) + return; + + pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); + nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, ctrl); + + for (i = 0; i < nbars; i++, pos += 8) { + struct resource *res; + int bar_idx, size; + u64 tmp; + + pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); + bar_idx = ctrl & PCI_REBAR_CTRL_BAR_IDX; + total = pdev->sriov->total_VFs; + if (!total) + return; + + res = pdev->resource + bar_idx + PCI_IOV_RESOURCES; + tmp = resource_size(res); + do_div(tmp, total); + size = pci_rebar_bytes_to_size(tmp); + ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE; + ctrl |= FIELD_PREP(PCI_REBAR_CTRL_BAR_SIZE, size); + pci_write_config_dword(pdev, pos + PCI_REBAR_CTRL, ctrl); + } +#endif +} + /** * pci_restore_state - Restore the saved state of a PCI device * @dev: PCI device that we're dealing with @@ -1882,6 +1918,7 @@ void pci_restore_state(struct pci_dev *dev) pci_restore_ats_state(dev); pci_restore_vc_state(dev); pci_restore_rebar_state(dev); + pci_restore_vf_rebar_state(dev); pci_restore_dpc_state(dev); pci_restore_ptm_state(dev); @@ -3677,10 +3714,18 @@ void pci_acs_init(struct pci_dev *dev) */ static int pci_rebar_find_pos(struct pci_dev *pdev, int bar) { + int cap = PCI_EXT_CAP_ID_REBAR; unsigned int pos, nbars, i; u32 ctrl; - pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_REBAR); +#ifdef CONFIG_PCI_IOV + if (bar >= PCI_IOV_RESOURCES) { + cap = PCI_EXT_CAP_ID_VF_REBAR; + bar -= PCI_IOV_RESOURCES; + } +#endif + + pos = pci_find_ext_capability(pdev, cap); if (!pos) return -ENOTSUPP; diff --git a/drivers/pci/setup-res.c b/drivers/pci/setup-res.c index c6d933ddf..d978a2ccf 100644 --- a/drivers/pci/setup-res.c +++ b/drivers/pci/setup-res.c @@ -427,13 +427,32 @@ void pci_release_resource(struct pci_dev *dev, int resno) } EXPORT_SYMBOL(pci_release_resource); +static int pci_memory_decoding(struct pci_dev *dev, int resno) { + u16 cmd; + +#ifdef CONFIG_PCI_IOV + if (resno >= PCI_IOV_RESOURCES) { + pci_read_config_word(dev, dev->sriov->pos + PCI_SRIOV_CTRL, &cmd); + if (cmd & PCI_SRIOV_CTRL_MSE) + return -EBUSY; + else + return 0; + } +#endif + pci_read_config_word(dev, PCI_COMMAND, &cmd); + if (cmd & PCI_COMMAND_MEMORY) + return -EBUSY; + + return 0; +} + int pci_resize_resource(struct pci_dev *dev, int resno, int size) { struct resource *res = dev->resource + resno; struct pci_host_bridge *host; int old, ret; u32 sizes; - u16 cmd; /* Check if we must preserve the firmware's resource assignment */ host = pci_find_host_bridge(dev->bus); @@ -444,9 +463,9 @@ int pci_resize_resource(struct pci_dev *dev, int resno, int size) if (!(res->flags & IORESOURCE_UNSET)) return -EBUSY; - pci_read_config_word(dev, PCI_COMMAND, &cmd); - if (cmd & PCI_COMMAND_MEMORY) - return -EBUSY; + ret = pci_memory_decoding(dev, resno); + if (ret) + return ret; sizes = pci_rebar_get_possible_sizes(dev, resno); if (!sizes) @@ -463,19 +482,31 @@ int pci_resize_resource(struct pci_dev *dev, int resno, int size) if (ret) return ret; - res->end = res->start + pci_rebar_size_to_bytes(size) - 1; +#ifdef CONFIG_PCI_IOV + if (resno >= PCI_IOV_RESOURCES) + dev->sriov->barsz[resno - PCI_IOV_RESOURCES] = pci_rebar_size_to_bytes(size); + else +#endif + res->end = res->start + pci_rebar_size_to_bytes(size) - 1; /* Check if the new config works by trying to assign everything. */ if (dev->bus->self) { ret = pci_reassign_bridge_resources(dev->bus->self, res->flags); - if (ret) + if (ret && ret != -ENOENT) goto error_resize; } return 0; error_resize: pci_rebar_set_size(dev, resno, old); - res->end = res->start + pci_rebar_size_to_bytes(old) - 1; + +#ifdef CONFIG_PCI_IOV + if (resno >= PCI_IOV_RESOURCES) + dev->sriov->barsz[resno - PCI_IOV_RESOURCES] = pci_rebar_size_to_bytes(old); + else +#endif + res->end = res->start + pci_rebar_size_to_bytes(old) - 1; + return ret; } EXPORT_SYMBOL(pci_resize_resource); diff --git a/include/uapi/linux/pci_regs.h b/include/uapi/linux/pci_regs.h index a39193213..a66b90982 100644 --- a/include/uapi/linux/pci_regs.h +++ b/include/uapi/linux/pci_regs.h @@ -738,6 +738,7 @@ #define PCI_EXT_CAP_ID_L1SS 0x1E /* L1 PM Substates */ #define PCI_EXT_CAP_ID_PTM 0x1F /* Precision Time Measurement */ #define PCI_EXT_CAP_ID_DVSEC 0x23 /* Designated Vendor-Specific */ +#define PCI_EXT_CAP_ID_VF_REBAR 0x24 /* VF Resizable BAR */ #define PCI_EXT_CAP_ID_DLF 0x25 /* Data Link Feature */ #define PCI_EXT_CAP_ID_PL_16GT 0x26 /* Physical Layer 16.0 GT/s */ #define PCI_EXT_CAP_ID_PL_32GT 0x2A /* Physical Layer 32.0 GT/s */ -- 2.34.1 ^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [PATCH 1/1][v2] PCI: Support VF resizable BAR 2024-06-07 8:42 ` Chang, HaiJun @ 2024-06-07 11:37 ` Christian König 2024-06-07 20:40 ` Bjorn Helgaas 0 siblings, 1 reply; 7+ messages in thread From: Christian König @ 2024-06-07 11:37 UTC (permalink / raw) To: Chang, HaiJun, Shi, Lianjie, linux-pci@vger.kernel.org, linux-kernel@vger.kernel.org, Bjorn Helgaas Cc: Jiang, Jerry (SW), Zhang, Andy I need the original set of patches, e.g. directly use git send-email with my mail address. Forwarded and mangled multiple times the patches are not applicable to any tree any more. Regards, Christian. Am 07.06.24 um 10:42 schrieb Chang, HaiJun: > [AMD Official Use Only - AMD Internal Distribution Only] > > + Christian to review the VF resizable BAR patch. > > -----Original Message----- > From: Chang, HaiJun > Sent: Friday, May 31, 2024 12:26 PM > To: Lianjie Shi <Lianjie.Shi@amd.com>; linux-pci@vger.kernel.org; linux-kernel@vger.kernel.org; Bjorn Helgaas <bhelgaas@google.com> > Cc: Jiang, Jerry (SW) <Jerry.Jiang@amd.com>; Zhang, Andy <Andy.Zhang@amd.com>; Shi, Lianjie <Lianjie.Shi@amd.com> > Subject: RE: [PATCH 1/1][v2] PCI: Support VF resizable BAR > > Hi, Bjorn and pci driver maintainers > > Could you review the VF resizable BAR patch? > > Thanks, > HaiJun > > -----Original Message----- > From: Lianjie Shi <Lianjie.Shi@amd.com> > Sent: Thursday, May 23, 2024 3:11 PM > To: linux-pci@vger.kernel.org; linux-kernel@vger.kernel.org; Bjorn Helgaas <bhelgaas@google.com> > Cc: Chang, HaiJun <HaiJun.Chang@amd.com>; Jiang, Jerry (SW) <Jerry.Jiang@amd.com>; Zhang, Andy <Andy.Zhang@amd.com>; Shi, Lianjie <Lianjie.Shi@amd.com> > Subject: [PATCH 1/1][v2] PCI: Support VF resizable BAR > > Add support for VF resizable BAR PCI extended cap. > Similar to regular BAR, drivers can use pci_resize_resource() to resize an IOV BAR. For each VF, dev->sriov->barsz of the IOV BAR is resized, but the total resource size of the IOV resource should not exceed its original size upon init. > > Based on following patch series: > Link: https://lore.kernel.org/lkml/YbqGplTKl5i%2F1%2FkY@rocinante/T/ > > Signed-off-by: Lianjie Shi <Lianjie.Shi@amd.com> > --- > drivers/pci/pci.c | 47 ++++++++++++++++++++++++++++++++++- > drivers/pci/setup-res.c | 45 +++++++++++++++++++++++++++------ > include/uapi/linux/pci_regs.h | 1 + > 3 files changed, 85 insertions(+), 8 deletions(-) > > diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index e5f243dd4..12f86e00a 100644 > --- a/drivers/pci/pci.c > +++ b/drivers/pci/pci.c > @@ -1867,6 +1867,42 @@ static void pci_restore_rebar_state(struct pci_dev *pdev) > } > } > > +static void pci_restore_vf_rebar_state(struct pci_dev *pdev) { #ifdef > +CONFIG_PCI_IOV > + unsigned int pos, nbars, i; > + u32 ctrl; > + u16 total; > + > + pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_VF_REBAR); > + if (!pos) > + return; > + > + pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); > + nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, ctrl); > + > + for (i = 0; i < nbars; i++, pos += 8) { > + struct resource *res; > + int bar_idx, size; > + u64 tmp; > + > + pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); > + bar_idx = ctrl & PCI_REBAR_CTRL_BAR_IDX; > + total = pdev->sriov->total_VFs; > + if (!total) > + return; > + > + res = pdev->resource + bar_idx + PCI_IOV_RESOURCES; > + tmp = resource_size(res); > + do_div(tmp, total); > + size = pci_rebar_bytes_to_size(tmp); > + ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE; > + ctrl |= FIELD_PREP(PCI_REBAR_CTRL_BAR_SIZE, size); > + pci_write_config_dword(pdev, pos + PCI_REBAR_CTRL, ctrl); > + } > +#endif > +} > + > /** > * pci_restore_state - Restore the saved state of a PCI device > * @dev: PCI device that we're dealing with @@ -1882,6 +1918,7 @@ void pci_restore_state(struct pci_dev *dev) > pci_restore_ats_state(dev); > pci_restore_vc_state(dev); > pci_restore_rebar_state(dev); > + pci_restore_vf_rebar_state(dev); > pci_restore_dpc_state(dev); > pci_restore_ptm_state(dev); > > @@ -3677,10 +3714,18 @@ void pci_acs_init(struct pci_dev *dev) > */ > static int pci_rebar_find_pos(struct pci_dev *pdev, int bar) { > + int cap = PCI_EXT_CAP_ID_REBAR; > unsigned int pos, nbars, i; > u32 ctrl; > > - pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_REBAR); > +#ifdef CONFIG_PCI_IOV > + if (bar >= PCI_IOV_RESOURCES) { > + cap = PCI_EXT_CAP_ID_VF_REBAR; > + bar -= PCI_IOV_RESOURCES; > + } > +#endif > + > + pos = pci_find_ext_capability(pdev, cap); > if (!pos) > return -ENOTSUPP; > > diff --git a/drivers/pci/setup-res.c b/drivers/pci/setup-res.c index c6d933ddf..d978a2ccf 100644 > --- a/drivers/pci/setup-res.c > +++ b/drivers/pci/setup-res.c > @@ -427,13 +427,32 @@ void pci_release_resource(struct pci_dev *dev, int resno) } EXPORT_SYMBOL(pci_release_resource); > > +static int pci_memory_decoding(struct pci_dev *dev, int resno) { > + u16 cmd; > + > +#ifdef CONFIG_PCI_IOV > + if (resno >= PCI_IOV_RESOURCES) { > + pci_read_config_word(dev, dev->sriov->pos + PCI_SRIOV_CTRL, &cmd); > + if (cmd & PCI_SRIOV_CTRL_MSE) > + return -EBUSY; > + else > + return 0; > + } > +#endif > + pci_read_config_word(dev, PCI_COMMAND, &cmd); > + if (cmd & PCI_COMMAND_MEMORY) > + return -EBUSY; > + > + return 0; > +} > + > int pci_resize_resource(struct pci_dev *dev, int resno, int size) { > struct resource *res = dev->resource + resno; > struct pci_host_bridge *host; > int old, ret; > u32 sizes; > - u16 cmd; > > /* Check if we must preserve the firmware's resource assignment */ > host = pci_find_host_bridge(dev->bus); @@ -444,9 +463,9 @@ int pci_resize_resource(struct pci_dev *dev, int resno, int size) > if (!(res->flags & IORESOURCE_UNSET)) > return -EBUSY; > > - pci_read_config_word(dev, PCI_COMMAND, &cmd); > - if (cmd & PCI_COMMAND_MEMORY) > - return -EBUSY; > + ret = pci_memory_decoding(dev, resno); > + if (ret) > + return ret; > > sizes = pci_rebar_get_possible_sizes(dev, resno); > if (!sizes) > @@ -463,19 +482,31 @@ int pci_resize_resource(struct pci_dev *dev, int resno, int size) > if (ret) > return ret; > > - res->end = res->start + pci_rebar_size_to_bytes(size) - 1; > +#ifdef CONFIG_PCI_IOV > + if (resno >= PCI_IOV_RESOURCES) > + dev->sriov->barsz[resno - PCI_IOV_RESOURCES] = pci_rebar_size_to_bytes(size); > + else > +#endif > + res->end = res->start + pci_rebar_size_to_bytes(size) - 1; > > /* Check if the new config works by trying to assign everything. */ > if (dev->bus->self) { > ret = pci_reassign_bridge_resources(dev->bus->self, res->flags); > - if (ret) > + if (ret && ret != -ENOENT) > goto error_resize; > } > return 0; > > error_resize: > pci_rebar_set_size(dev, resno, old); > - res->end = res->start + pci_rebar_size_to_bytes(old) - 1; > + > +#ifdef CONFIG_PCI_IOV > + if (resno >= PCI_IOV_RESOURCES) > + dev->sriov->barsz[resno - PCI_IOV_RESOURCES] = pci_rebar_size_to_bytes(old); > + else > +#endif > + res->end = res->start + pci_rebar_size_to_bytes(old) - 1; > + > return ret; > } > EXPORT_SYMBOL(pci_resize_resource); > diff --git a/include/uapi/linux/pci_regs.h b/include/uapi/linux/pci_regs.h index a39193213..a66b90982 100644 > --- a/include/uapi/linux/pci_regs.h > +++ b/include/uapi/linux/pci_regs.h > @@ -738,6 +738,7 @@ > #define PCI_EXT_CAP_ID_L1SS 0x1E /* L1 PM Substates */ > #define PCI_EXT_CAP_ID_PTM 0x1F /* Precision Time Measurement */ > #define PCI_EXT_CAP_ID_DVSEC 0x23 /* Designated Vendor-Specific */ > +#define PCI_EXT_CAP_ID_VF_REBAR 0x24 /* VF Resizable BAR */ > #define PCI_EXT_CAP_ID_DLF 0x25 /* Data Link Feature */ > #define PCI_EXT_CAP_ID_PL_16GT 0x26 /* Physical Layer 16.0 GT/s */ > #define PCI_EXT_CAP_ID_PL_32GT 0x2A /* Physical Layer 32.0 GT/s */ > -- > 2.34.1 > ^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [PATCH 1/1][v2] PCI: Support VF resizable BAR 2024-06-07 11:37 ` Christian König @ 2024-06-07 20:40 ` Bjorn Helgaas 0 siblings, 0 replies; 7+ messages in thread From: Bjorn Helgaas @ 2024-06-07 20:40 UTC (permalink / raw) To: Christian König Cc: Chang, HaiJun, Shi, Lianjie, linux-pci@vger.kernel.org, linux-kernel@vger.kernel.org, Bjorn Helgaas, Jiang, Jerry (SW), Zhang, Andy On Fri, Jun 07, 2024 at 01:37:21PM +0200, Christian König wrote: > I need the original set of patches, e.g. directly use git send-email with my > mail address. This worked for me: $ git checkout -b wip/2406-lianjie-vf-rebar-v2 v6.10-rc1 $ b4 am https://lore.kernel.org/r/20240523071114.2930804-2-Lianjie.Shi@amd.com $ git am ./v2_20240523_lianjie_shi_pci_vf_resizable_bar.mbx > Am 07.06.24 um 10:42 schrieb Chang, HaiJun: > > [AMD Official Use Only - AMD Internal Distribution Only] > > > > + Christian to review the VF resizable BAR patch. > > > > -----Original Message----- > > From: Chang, HaiJun > > Sent: Friday, May 31, 2024 12:26 PM > > To: Lianjie Shi <Lianjie.Shi@amd.com>; linux-pci@vger.kernel.org; linux-kernel@vger.kernel.org; Bjorn Helgaas <bhelgaas@google.com> > > Cc: Jiang, Jerry (SW) <Jerry.Jiang@amd.com>; Zhang, Andy <Andy.Zhang@amd.com>; Shi, Lianjie <Lianjie.Shi@amd.com> > > Subject: RE: [PATCH 1/1][v2] PCI: Support VF resizable BAR > > > > Hi, Bjorn and pci driver maintainers > > > > Could you review the VF resizable BAR patch? > > > > Thanks, > > HaiJun > > > > -----Original Message----- > > From: Lianjie Shi <Lianjie.Shi@amd.com> > > Sent: Thursday, May 23, 2024 3:11 PM > > To: linux-pci@vger.kernel.org; linux-kernel@vger.kernel.org; Bjorn Helgaas <bhelgaas@google.com> > > Cc: Chang, HaiJun <HaiJun.Chang@amd.com>; Jiang, Jerry (SW) <Jerry.Jiang@amd.com>; Zhang, Andy <Andy.Zhang@amd.com>; Shi, Lianjie <Lianjie.Shi@amd.com> > > Subject: [PATCH 1/1][v2] PCI: Support VF resizable BAR > > > > Add support for VF resizable BAR PCI extended cap. > > Similar to regular BAR, drivers can use pci_resize_resource() to resize an IOV BAR. For each VF, dev->sriov->barsz of the IOV BAR is resized, but the total resource size of the IOV resource should not exceed its original size upon init. > > > > Based on following patch series: > > Link: https://lore.kernel.org/lkml/YbqGplTKl5i%2F1%2FkY@rocinante/T/ > > > > Signed-off-by: Lianjie Shi <Lianjie.Shi@amd.com> > > --- > > drivers/pci/pci.c | 47 ++++++++++++++++++++++++++++++++++- > > drivers/pci/setup-res.c | 45 +++++++++++++++++++++++++++------ > > include/uapi/linux/pci_regs.h | 1 + > > 3 files changed, 85 insertions(+), 8 deletions(-) > > > > diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index e5f243dd4..12f86e00a 100644 > > --- a/drivers/pci/pci.c > > +++ b/drivers/pci/pci.c > > @@ -1867,6 +1867,42 @@ static void pci_restore_rebar_state(struct pci_dev *pdev) > > } > > } > > > > +static void pci_restore_vf_rebar_state(struct pci_dev *pdev) { #ifdef > > +CONFIG_PCI_IOV > > + unsigned int pos, nbars, i; > > + u32 ctrl; > > + u16 total; > > + > > + pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_VF_REBAR); > > + if (!pos) > > + return; > > + > > + pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); > > + nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, ctrl); > > + > > + for (i = 0; i < nbars; i++, pos += 8) { > > + struct resource *res; > > + int bar_idx, size; > > + u64 tmp; > > + > > + pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); > > + bar_idx = ctrl & PCI_REBAR_CTRL_BAR_IDX; > > + total = pdev->sriov->total_VFs; > > + if (!total) > > + return; > > + > > + res = pdev->resource + bar_idx + PCI_IOV_RESOURCES; > > + tmp = resource_size(res); > > + do_div(tmp, total); > > + size = pci_rebar_bytes_to_size(tmp); > > + ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE; > > + ctrl |= FIELD_PREP(PCI_REBAR_CTRL_BAR_SIZE, size); > > + pci_write_config_dword(pdev, pos + PCI_REBAR_CTRL, ctrl); > > + } > > +#endif > > +} > > + > > /** > > * pci_restore_state - Restore the saved state of a PCI device > > * @dev: PCI device that we're dealing with @@ -1882,6 +1918,7 @@ void pci_restore_state(struct pci_dev *dev) > > pci_restore_ats_state(dev); > > pci_restore_vc_state(dev); > > pci_restore_rebar_state(dev); > > + pci_restore_vf_rebar_state(dev); > > pci_restore_dpc_state(dev); > > pci_restore_ptm_state(dev); > > > > @@ -3677,10 +3714,18 @@ void pci_acs_init(struct pci_dev *dev) > > */ > > static int pci_rebar_find_pos(struct pci_dev *pdev, int bar) { > > + int cap = PCI_EXT_CAP_ID_REBAR; > > unsigned int pos, nbars, i; > > u32 ctrl; > > > > - pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_REBAR); > > +#ifdef CONFIG_PCI_IOV > > + if (bar >= PCI_IOV_RESOURCES) { > > + cap = PCI_EXT_CAP_ID_VF_REBAR; > > + bar -= PCI_IOV_RESOURCES; > > + } > > +#endif > > + > > + pos = pci_find_ext_capability(pdev, cap); > > if (!pos) > > return -ENOTSUPP; > > > > diff --git a/drivers/pci/setup-res.c b/drivers/pci/setup-res.c index c6d933ddf..d978a2ccf 100644 > > --- a/drivers/pci/setup-res.c > > +++ b/drivers/pci/setup-res.c > > @@ -427,13 +427,32 @@ void pci_release_resource(struct pci_dev *dev, int resno) } EXPORT_SYMBOL(pci_release_resource); > > > > +static int pci_memory_decoding(struct pci_dev *dev, int resno) { > > + u16 cmd; > > + > > +#ifdef CONFIG_PCI_IOV > > + if (resno >= PCI_IOV_RESOURCES) { > > + pci_read_config_word(dev, dev->sriov->pos + PCI_SRIOV_CTRL, &cmd); > > + if (cmd & PCI_SRIOV_CTRL_MSE) > > + return -EBUSY; > > + else > > + return 0; > > + } > > +#endif > > + pci_read_config_word(dev, PCI_COMMAND, &cmd); > > + if (cmd & PCI_COMMAND_MEMORY) > > + return -EBUSY; > > + > > + return 0; > > +} > > + > > int pci_resize_resource(struct pci_dev *dev, int resno, int size) { > > struct resource *res = dev->resource + resno; > > struct pci_host_bridge *host; > > int old, ret; > > u32 sizes; > > - u16 cmd; > > > > /* Check if we must preserve the firmware's resource assignment */ > > host = pci_find_host_bridge(dev->bus); @@ -444,9 +463,9 @@ int pci_resize_resource(struct pci_dev *dev, int resno, int size) > > if (!(res->flags & IORESOURCE_UNSET)) > > return -EBUSY; > > > > - pci_read_config_word(dev, PCI_COMMAND, &cmd); > > - if (cmd & PCI_COMMAND_MEMORY) > > - return -EBUSY; > > + ret = pci_memory_decoding(dev, resno); > > + if (ret) > > + return ret; > > > > sizes = pci_rebar_get_possible_sizes(dev, resno); > > if (!sizes) > > @@ -463,19 +482,31 @@ int pci_resize_resource(struct pci_dev *dev, int resno, int size) > > if (ret) > > return ret; > > > > - res->end = res->start + pci_rebar_size_to_bytes(size) - 1; > > +#ifdef CONFIG_PCI_IOV > > + if (resno >= PCI_IOV_RESOURCES) > > + dev->sriov->barsz[resno - PCI_IOV_RESOURCES] = pci_rebar_size_to_bytes(size); > > + else > > +#endif > > + res->end = res->start + pci_rebar_size_to_bytes(size) - 1; > > > > /* Check if the new config works by trying to assign everything. */ > > if (dev->bus->self) { > > ret = pci_reassign_bridge_resources(dev->bus->self, res->flags); > > - if (ret) > > + if (ret && ret != -ENOENT) > > goto error_resize; > > } > > return 0; > > > > error_resize: > > pci_rebar_set_size(dev, resno, old); > > - res->end = res->start + pci_rebar_size_to_bytes(old) - 1; > > + > > +#ifdef CONFIG_PCI_IOV > > + if (resno >= PCI_IOV_RESOURCES) > > + dev->sriov->barsz[resno - PCI_IOV_RESOURCES] = pci_rebar_size_to_bytes(old); > > + else > > +#endif > > + res->end = res->start + pci_rebar_size_to_bytes(old) - 1; > > + > > return ret; > > } > > EXPORT_SYMBOL(pci_resize_resource); > > diff --git a/include/uapi/linux/pci_regs.h b/include/uapi/linux/pci_regs.h index a39193213..a66b90982 100644 > > --- a/include/uapi/linux/pci_regs.h > > +++ b/include/uapi/linux/pci_regs.h > > @@ -738,6 +738,7 @@ > > #define PCI_EXT_CAP_ID_L1SS 0x1E /* L1 PM Substates */ > > #define PCI_EXT_CAP_ID_PTM 0x1F /* Precision Time Measurement */ > > #define PCI_EXT_CAP_ID_DVSEC 0x23 /* Designated Vendor-Specific */ > > +#define PCI_EXT_CAP_ID_VF_REBAR 0x24 /* VF Resizable BAR */ > > #define PCI_EXT_CAP_ID_DLF 0x25 /* Data Link Feature */ > > #define PCI_EXT_CAP_ID_PL_16GT 0x26 /* Physical Layer 16.0 GT/s */ > > #define PCI_EXT_CAP_ID_PL_32GT 0x2A /* Physical Layer 32.0 GT/s */ > > -- > > 2.34.1 > > > ^ permalink raw reply [flat|nested] 7+ messages in thread
* [PATCH 0/1][v2] PCI: VF resizable BAR @ 2024-06-12 8:42 Lianjie Shi 2024-06-12 8:42 ` [PATCH 1/1][v2] PCI: Support " Lianjie Shi 0 siblings, 1 reply; 7+ messages in thread From: Lianjie Shi @ 2024-06-12 8:42 UTC (permalink / raw) To: christian.koenig Cc: linux-pci, linux-kernel, bhelgaas, HaiJun.Chang, Jerry.Jiang, Andy.Zhang, emily.deng, Lianjie Shi Similar to regular BAR, drivers can use pci_resize_resource() to resize an IOV BAR to the desired size provided that is supported by the hardware, which can be queried using pci_rebar_get_possible_sizes(). This feature is based on the fact that (default VF BAR size) * (supported VF number) covers all possible resource/address ranges (rounded up to the power of 2 size). For example, the total size of the resource behind the BAR is 256GB, the supported maximum VF number is 4, the default VF BAR size should then be set to 64GB. When the enabled vf_num changes, the VF BAR size will adjust accordingly as - For 1 VF, VF BAR size is 256GB - For 2 VFs, VF BAR size is 128GB - For 4 VFs, VF BAR size is 64GB This feature is necessary to accommodate the limited address per PCI port. Lianjie Shi (1): PCI: Support VF resizable BAR drivers/pci/pci.c | 47 ++++++++++++++++++++++++++++++++++- drivers/pci/setup-res.c | 45 +++++++++++++++++++++++++++------ include/uapi/linux/pci_regs.h | 1 + 3 files changed, 85 insertions(+), 8 deletions(-) base-commit: cf87f46fd34d6c19283d9625a7822f20d90b64a4 -- 2.34.1 ^ permalink raw reply [flat|nested] 7+ messages in thread
* [PATCH 1/1][v2] PCI: Support VF resizable BAR 2024-06-12 8:42 [PATCH 0/1][v2] PCI: " Lianjie Shi @ 2024-06-12 8:42 ` Lianjie Shi 0 siblings, 0 replies; 7+ messages in thread From: Lianjie Shi @ 2024-06-12 8:42 UTC (permalink / raw) To: christian.koenig Cc: linux-pci, linux-kernel, bhelgaas, HaiJun.Chang, Jerry.Jiang, Andy.Zhang, emily.deng, Lianjie Shi Add support for VF resizable BAR PCI extended cap. Similar to regular BAR, drivers can use pci_resize_resource() to resize an IOV BAR. For each VF, dev->sriov->barsz of the IOV BAR is resized, but the total resource size of the IOV resource should not exceed its original size upon init. Based on following patch series: Link: https://lore.kernel.org/lkml/YbqGplTKl5i%2F1%2FkY@rocinante/T/ Signed-off-by: Lianjie Shi <Lianjie.Shi@amd.com> --- drivers/pci/pci.c | 47 ++++++++++++++++++++++++++++++++++- drivers/pci/setup-res.c | 45 +++++++++++++++++++++++++++------ include/uapi/linux/pci_regs.h | 1 + 3 files changed, 85 insertions(+), 8 deletions(-) diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index e5f243dd4..12f86e00a 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -1867,6 +1867,42 @@ static void pci_restore_rebar_state(struct pci_dev *pdev) } } +static void pci_restore_vf_rebar_state(struct pci_dev *pdev) +{ +#ifdef CONFIG_PCI_IOV + unsigned int pos, nbars, i; + u32 ctrl; + u16 total; + + pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_VF_REBAR); + if (!pos) + return; + + pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); + nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, ctrl); + + for (i = 0; i < nbars; i++, pos += 8) { + struct resource *res; + int bar_idx, size; + u64 tmp; + + pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); + bar_idx = ctrl & PCI_REBAR_CTRL_BAR_IDX; + total = pdev->sriov->total_VFs; + if (!total) + return; + + res = pdev->resource + bar_idx + PCI_IOV_RESOURCES; + tmp = resource_size(res); + do_div(tmp, total); + size = pci_rebar_bytes_to_size(tmp); + ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE; + ctrl |= FIELD_PREP(PCI_REBAR_CTRL_BAR_SIZE, size); + pci_write_config_dword(pdev, pos + PCI_REBAR_CTRL, ctrl); + } +#endif +} + /** * pci_restore_state - Restore the saved state of a PCI device * @dev: PCI device that we're dealing with @@ -1882,6 +1918,7 @@ void pci_restore_state(struct pci_dev *dev) pci_restore_ats_state(dev); pci_restore_vc_state(dev); pci_restore_rebar_state(dev); + pci_restore_vf_rebar_state(dev); pci_restore_dpc_state(dev); pci_restore_ptm_state(dev); @@ -3677,10 +3714,18 @@ void pci_acs_init(struct pci_dev *dev) */ static int pci_rebar_find_pos(struct pci_dev *pdev, int bar) { + int cap = PCI_EXT_CAP_ID_REBAR; unsigned int pos, nbars, i; u32 ctrl; - pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_REBAR); +#ifdef CONFIG_PCI_IOV + if (bar >= PCI_IOV_RESOURCES) { + cap = PCI_EXT_CAP_ID_VF_REBAR; + bar -= PCI_IOV_RESOURCES; + } +#endif + + pos = pci_find_ext_capability(pdev, cap); if (!pos) return -ENOTSUPP; diff --git a/drivers/pci/setup-res.c b/drivers/pci/setup-res.c index c6d933ddf..d978a2ccf 100644 --- a/drivers/pci/setup-res.c +++ b/drivers/pci/setup-res.c @@ -427,13 +427,32 @@ void pci_release_resource(struct pci_dev *dev, int resno) } EXPORT_SYMBOL(pci_release_resource); +static int pci_memory_decoding(struct pci_dev *dev, int resno) +{ + u16 cmd; + +#ifdef CONFIG_PCI_IOV + if (resno >= PCI_IOV_RESOURCES) { + pci_read_config_word(dev, dev->sriov->pos + PCI_SRIOV_CTRL, &cmd); + if (cmd & PCI_SRIOV_CTRL_MSE) + return -EBUSY; + else + return 0; + } +#endif + pci_read_config_word(dev, PCI_COMMAND, &cmd); + if (cmd & PCI_COMMAND_MEMORY) + return -EBUSY; + + return 0; +} + int pci_resize_resource(struct pci_dev *dev, int resno, int size) { struct resource *res = dev->resource + resno; struct pci_host_bridge *host; int old, ret; u32 sizes; - u16 cmd; /* Check if we must preserve the firmware's resource assignment */ host = pci_find_host_bridge(dev->bus); @@ -444,9 +463,9 @@ int pci_resize_resource(struct pci_dev *dev, int resno, int size) if (!(res->flags & IORESOURCE_UNSET)) return -EBUSY; - pci_read_config_word(dev, PCI_COMMAND, &cmd); - if (cmd & PCI_COMMAND_MEMORY) - return -EBUSY; + ret = pci_memory_decoding(dev, resno); + if (ret) + return ret; sizes = pci_rebar_get_possible_sizes(dev, resno); if (!sizes) @@ -463,19 +482,31 @@ int pci_resize_resource(struct pci_dev *dev, int resno, int size) if (ret) return ret; - res->end = res->start + pci_rebar_size_to_bytes(size) - 1; +#ifdef CONFIG_PCI_IOV + if (resno >= PCI_IOV_RESOURCES) + dev->sriov->barsz[resno - PCI_IOV_RESOURCES] = pci_rebar_size_to_bytes(size); + else +#endif + res->end = res->start + pci_rebar_size_to_bytes(size) - 1; /* Check if the new config works by trying to assign everything. */ if (dev->bus->self) { ret = pci_reassign_bridge_resources(dev->bus->self, res->flags); - if (ret) + if (ret && ret != -ENOENT) goto error_resize; } return 0; error_resize: pci_rebar_set_size(dev, resno, old); - res->end = res->start + pci_rebar_size_to_bytes(old) - 1; + +#ifdef CONFIG_PCI_IOV + if (resno >= PCI_IOV_RESOURCES) + dev->sriov->barsz[resno - PCI_IOV_RESOURCES] = pci_rebar_size_to_bytes(old); + else +#endif + res->end = res->start + pci_rebar_size_to_bytes(old) - 1; + return ret; } EXPORT_SYMBOL(pci_resize_resource); diff --git a/include/uapi/linux/pci_regs.h b/include/uapi/linux/pci_regs.h index a39193213..a66b90982 100644 --- a/include/uapi/linux/pci_regs.h +++ b/include/uapi/linux/pci_regs.h @@ -738,6 +738,7 @@ #define PCI_EXT_CAP_ID_L1SS 0x1E /* L1 PM Substates */ #define PCI_EXT_CAP_ID_PTM 0x1F /* Precision Time Measurement */ #define PCI_EXT_CAP_ID_DVSEC 0x23 /* Designated Vendor-Specific */ +#define PCI_EXT_CAP_ID_VF_REBAR 0x24 /* VF Resizable BAR */ #define PCI_EXT_CAP_ID_DLF 0x25 /* Data Link Feature */ #define PCI_EXT_CAP_ID_PL_16GT 0x26 /* Physical Layer 16.0 GT/s */ #define PCI_EXT_CAP_ID_PL_32GT 0x2A /* Physical Layer 32.0 GT/s */ -- 2.34.1 ^ permalink raw reply related [flat|nested] 7+ messages in thread
end of thread, other threads:[~2024-06-12 8:45 UTC | newest] Thread overview: 7+ messages (download: mbox.gz follow: Atom feed -- links below jump to the message on this page -- 2024-05-23 7:11 [PATCH 0/1][v2] PCI: VF resizable BAR Lianjie Shi 2024-05-23 7:11 ` [PATCH 1/1][v2] PCI: Support " Lianjie Shi 2024-05-31 4:26 ` Chang, HaiJun 2024-06-07 8:42 ` Chang, HaiJun 2024-06-07 11:37 ` Christian König 2024-06-07 20:40 ` Bjorn Helgaas -- strict thread matches above, loose matches on Subject: below -- 2024-06-12 8:42 [PATCH 0/1][v2] PCI: " Lianjie Shi 2024-06-12 8:42 ` [PATCH 1/1][v2] PCI: Support " Lianjie Shi
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox