Re: [PATCH v3 11/20] cxl: define a driver interface for HPA free space enumaration

Jonathan Cameron
2024-09-13T17:52:36Z

On Sat, 7 Sep 2024 09:18:27 +0100
alejandro.lucero-palau@amd.com wrote:

> From: Alejandro Lucero <alucerop@amd.com>
>
> CXL region creation involves allocating capacity from device DPA
> (device-physical-address space) and assigning it to decode a given HPA
> (host-physical-address space). Before determining how much DPA to
> allocate the amount of available HPA must be determined. Also, not all
> HPA is create equal, some specifically targets RAM, some target PMEM,
> some is prepared for device-memory flows like HDM-D and HDM-DB, and some
> is host-only (HDM-H).
>
> Wrap all of those concerns into an API that retrieves a root decoder
> (platform CXL window) that fits the specified constraints and the
> capacity available for a new region.
>
> Based on https://lore.kernel.org/linux-cxl/168592159290.1948938.13522227102445462976.stgit@dwillia2-xfh.jf.intel.com/
>
> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> Co-developed-by: Dan Williams <dan.j.williams@intel.com>
Trivial comment inline.

J
> ---
> drivers/cxl/core/region.c | 141 ++++++++++++++++++++++++++++++++++++++
> drivers/cxl/cxl.h | 3 +
> drivers/cxl/cxlmem.h | 3 +
> include/linux/cxl/cxl.h | 8 +++
> 4 files changed, 155 insertions(+)
>
> diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
> index 21ad5f242875..bb227bf894c4 100644
> --- a/drivers/cxl/core/region.c
> +++ b/drivers/cxl/core/region.c
> @@ -703,6 +703,147 @@ static int free_hpa(struct cxl_region *cxlr)
> return 0;
> }
>
> +struct cxlrd_max_context {
> + struct device *host_bridge;
> + unsigned long flags;
> + resource_size_t max_hpa;
> + struct cxl_root_decoder *cxlrd;
> +};
> +
> +static int find_max_hpa(struct device *dev, void *data)
> +{
> + struct cxlrd_max_context *ctx = data;
> + struct cxl_switch_decoder *cxlsd;
> + struct cxl_root_decoder *cxlrd;
> + struct resource *res, *prev;
> + struct cxl_decoder *cxld;
> + resource_size_t max;
> +
> + if (!is_root_decoder(dev))
> + return 0;
> +
> + cxlrd = to_cxl_root_decoder(dev);
> + cxld = &cxlrd->cxlsd.cxld;
> + if ((cxld->flags & ctx->flags) != ctx->flags) {
> + dev_dbg(dev, "%s, flags not matching: %08lx vs %08lx\n",
> + __func__, cxld->flags, ctx->flags);
> + return 0;
> + }
> +
> + /* An accelerator can not be part of an interleaved HPA range. */
> + if (cxld->interleave_ways != 1) {
> + dev_dbg(dev, "%s, interleave_ways not matching\n", __func__);
> + return 0;
> + }
> +
> + cxlsd = &cxlrd->cxlsd;

Perhaps move this before the
cxld = and use it there as well?

> +
> + guard(rwsem_read)(&cxl_region_rwsem);
> + if (ctx->host_bridge != cxlsd->target[0]->dport_dev) {
> + dev_dbg(dev, "%s, HOST BRIDGE DOES NOT MATCH\n", __func__);
Capitals seem a bit ott.

> + return 0;
> + }
> +
> + /*
> + * Walk the root decoder resource range relying on cxl_region_rwsem to
> + * preclude sibling arrival/departure and find the largest free space
> + * gap.
> + */
> + lockdep_assert_held_read(&cxl_region_rwsem);
> + max = 0;
> + res = cxlrd->res->child;
> + if (!res)
> + max = resource_size(cxlrd->res);
> + else
> + max = 0;
> +
> + for (prev = NULL; res; prev = res, res = res->sibling) {
> + struct resource *next = res->sibling;
> + resource_size_t free = 0;
> +
> + if (!prev && res->start > cxlrd->res->start) {
> + free = res->start - cxlrd->res->start;
> + max = max(free, max);
> + }
> + if (prev && res->start > prev->end + 1) {
> + free = res->start - prev->end + 1;
> + max = max(free, max);
> + }
> + if (next && res->end + 1 < next->start) {
> + free = next->start - res->end + 1;
> + max = max(free, max);
> + }
> + if (!next && res->end + 1 < cxlrd->res->end + 1) {
> + free = cxlrd->res->end + 1 - res->end + 1;
> + max = max(free, max);
> + }
> + }
> +
> + dev_dbg(CXLRD_DEV(cxlrd), "%s, found %pa bytes of free space\n",
> + __func__, &max);
> + if (max > ctx->max_hpa) {
> + if (ctx->cxlrd)
> + put_device(CXLRD_DEV(ctx->cxlrd));
> + get_device(CXLRD_DEV(cxlrd));
> + ctx->cxlrd = cxlrd;
> + ctx->max_hpa = max;
> + dev_dbg(CXLRD_DEV(cxlrd), "%s, found %pa bytes of free space\n",
> + __func__, &max);
> + }
> + return 0;
> +}