Re: [PATCH for-next v2 1/2] RDMA/efa: Add initialization of AH cache rhashtable

[Zhu Yanjun <yanjun.zhu@linux.dev>]

在 2026/5/11 23:11, Yonatan Nachum 写道:
> New EFA devices don't support the creation of multiple address handles
> to the same remote on the same PD.
> To overcome this limitation, introduce an AH cache rhashtable which will
> store the refcounts of the same AH creation on the same PD and will
> allow the driver to manage AH reuse. The hashtable key is the
> combination of PD and GID. Add initialization and teardown logic for the
> rhashtable.
> 
> Reviewed-by: Firas Jahjah <firasj@amazon.com>
> Reviewed-by: Michael Margolin <mrgolin@amazon.com>
> Signed-off-by: Yonatan Nachum <ynachum@amazon.com>
> ---
>   drivers/infiniband/hw/efa/Makefile       |  4 +--
>   drivers/infiniband/hw/efa/efa_ah_cache.c | 30 ++++++++++++++++++++
>   drivers/infiniband/hw/efa/efa_ah_cache.h | 36 ++++++++++++++++++++++++
>   drivers/infiniband/hw/efa/efa_com.c      | 12 +++++++-
>   drivers/infiniband/hw/efa/efa_com.h      |  5 +++-
>   5 files changed, 83 insertions(+), 4 deletions(-)
>   create mode 100644 drivers/infiniband/hw/efa/efa_ah_cache.c
>   create mode 100644 drivers/infiniband/hw/efa/efa_ah_cache.h
> 
> diff --git a/drivers/infiniband/hw/efa/Makefile b/drivers/infiniband/hw/efa/Makefile
> index 6e83083af0bc..a6a433b0ba2f 100644
> --- a/drivers/infiniband/hw/efa/Makefile
> +++ b/drivers/infiniband/hw/efa/Makefile
> @@ -1,9 +1,9 @@
>   # SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
> -# Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
> +# Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved.
>   #
>   # Makefile for Amazon Elastic Fabric Adapter (EFA) device driver.
>   #
>   
>   obj-$(CONFIG_INFINIBAND_EFA) += efa.o
>   
> -efa-y := efa_com_cmd.o efa_com.o efa_main.o efa_verbs.o
> +efa-y := efa_com_cmd.o efa_ah_cache.o efa_com.o efa_main.o efa_verbs.o
> diff --git a/drivers/infiniband/hw/efa/efa_ah_cache.c b/drivers/infiniband/hw/efa/efa_ah_cache.c
> new file mode 100644
> index 000000000000..d31871eb9748
> --- /dev/null
> +++ b/drivers/infiniband/hw/efa/efa_ah_cache.c
> @@ -0,0 +1,30 @@
> +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
> +/*
> + * Copyright 2026 Amazon.com, Inc. or its affiliates. All rights reserved.
> + */
> +
> +#include "efa_ah_cache.h"
> +
> +static const struct rhashtable_params ah_cache_params = {
> +	.key_len = sizeof(struct efa_ah_cache_key),
> +	.key_offset = offsetof(struct efa_ah_cache_entry, key),
> +	.head_offset = offsetof(struct efa_ah_cache_entry, linkage),
> +};
> +
> +int efa_ah_cache_init(struct efa_ah_cache *ah_cache)
> +{
> +	int err;
> +
> +	mutex_init(&ah_cache->lock);
> +	err = rhashtable_init(&ah_cache->hashtable, &ah_cache_params);
> +	if (err)
> +		mutex_destroy(&ah_cache->lock);
> +
> +	return err;
> +}
> +
> +void efa_ah_cache_destroy(struct efa_ah_cache *ah_cache)
> +{
> +	rhashtable_destroy(&ah_cache->hashtable);
> +	mutex_destroy(&ah_cache->lock);
> +}
> diff --git a/drivers/infiniband/hw/efa/efa_ah_cache.h b/drivers/infiniband/hw/efa/efa_ah_cache.h
> new file mode 100644
> index 000000000000..25288fdf778a
> --- /dev/null
> +++ b/drivers/infiniband/hw/efa/efa_ah_cache.h
> @@ -0,0 +1,36 @@
> +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
> +/*
> + * Copyright 2026 Amazon.com, Inc. or its affiliates. All rights reserved.
> + */
> +
> +#ifndef _EFA_AH_CACHE_H_
> +#define _EFA_AH_CACHE_H_
> +
> +#include <linux/refcount.h>
> +#include <linux/rhashtable.h>
> +
> +#define EFA_AH_GID_SIZE 16
> +
> +struct efa_ah_cache_key {
> +	u8 gid[EFA_AH_GID_SIZE];
> +	u16 pd;
> +};

I am not sure if we add __packed to avoid memory hole.

Zhu Yanjun

> +
> +struct efa_ah_cache_entry {
> +	struct efa_ah_cache_key key;
> +	u16 ah;
> +	bool initialized;
> +	refcount_t refcount;
> +	struct rhash_head linkage;
> +	struct mutex lock; /* Serializes device commands per cache entry */
> +};
> +
> +struct efa_ah_cache {
> +	struct rhashtable hashtable;
> +	struct mutex lock; /* Protects AH cache hashtable */
> +};
> +
> +int efa_ah_cache_init(struct efa_ah_cache *ah_cache);
> +void efa_ah_cache_destroy(struct efa_ah_cache *ah_cache);
> +
> +#endif /* _EFA_AH_CACHE_H_ */
> diff --git a/drivers/infiniband/hw/efa/efa_com.c b/drivers/infiniband/hw/efa/efa_com.c
> index e97b5f0d7003..b9b922583709 100644
> --- a/drivers/infiniband/hw/efa/efa_com.c
> +++ b/drivers/infiniband/hw/efa/efa_com.c
> @@ -688,6 +688,8 @@ void efa_com_admin_destroy(struct efa_com_dev *edev)
>   
>   	size = aenq->depth * sizeof(*aenq->entries);
>   	dma_free_coherent(edev->dmadev, size, aenq->entries, aenq->dma_addr);
> +
> +	efa_ah_cache_destroy(&edev->ah_cache);
>   }
>   
>   /**
> @@ -746,6 +748,12 @@ int efa_com_admin_init(struct efa_com_dev *edev,
>   		return -ENODEV;
>   	}
>   
> +	err = efa_ah_cache_init(&edev->ah_cache);
> +	if (err) {
> +		ibdev_err(edev->efa_dev, "Failed to init AH cache\n");
> +		return err;
> +	}
> +
>   	aq->depth = EFA_ADMIN_QUEUE_DEPTH;
>   
>   	aq->dmadev = edev->dmadev;
> @@ -758,7 +766,7 @@ int efa_com_admin_init(struct efa_com_dev *edev,
>   
>   	err = efa_com_init_comp_ctxt(aq);
>   	if (err)
> -		return err;
> +		goto err_destroy_ah_cache;
>   
>   	err = efa_com_admin_init_sq(edev);
>   	if (err)
> @@ -796,6 +804,8 @@ int efa_com_admin_init(struct efa_com_dev *edev,
>   			  aq->sq.entries, aq->sq.dma_addr);
>   err_destroy_comp_ctxt:
>   	devm_kfree(edev->dmadev, aq->comp_ctx);
> +err_destroy_ah_cache:
> +	efa_ah_cache_destroy(&edev->ah_cache);
>   
>   	return err;
>   }
> diff --git a/drivers/infiniband/hw/efa/efa_com.h b/drivers/infiniband/hw/efa/efa_com.h
> index 4d9ca97e4296..dc365df7f3dd 100644
> --- a/drivers/infiniband/hw/efa/efa_com.h
> +++ b/drivers/infiniband/hw/efa/efa_com.h
> @@ -1,6 +1,6 @@
>   /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
>   /*
> - * Copyright 2018-2025 Amazon.com, Inc. or its affiliates. All rights reserved.
> + * Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved.
>    */
>   
>   #ifndef _EFA_COM_H_
> @@ -14,6 +14,7 @@
>   
>   #include <rdma/ib_verbs.h>
>   
> +#include "efa_ah_cache.h"
>   #include "efa_common_defs.h"
>   #include "efa_admin_defs.h"
>   #include "efa_admin_cmds_defs.h"
> @@ -112,6 +113,8 @@ struct efa_com_dev {
>   	u32 supported_features;
>   	u32 dma_addr_bits;
>   
> +	struct efa_ah_cache ah_cache;
> +
>   	struct efa_com_mmio_read mmio_read;
>   };
>