Re: [PATCH rdma-next v5 06/17] RDMA/counter: Add "auto" configuration mode support

[Jason Gunthorpe <jgg@ziepe.ca>]

On Tue, Jul 02, 2019 at 01:02:35PM +0300, Leon Romanovsky wrote:
> From: Mark Zhang <markz@mellanox.com>
> 
> In auto mode all QPs belong to one category are bind automatically to
> a single counter set. Currently only "qp type" is supported.
> 
> In this mode the qp counter is set in RST2INIT modification, and when
> a qp is destroyed the counter is unbound.
> 
> Signed-off-by: Mark Zhang <markz@mellanox.com>
> Reviewed-by: Majd Dibbiny <majd@mellanox.com>
> Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
>  drivers/infiniband/core/counters.c | 221 +++++++++++++++++++++++++++++
>  drivers/infiniband/core/device.c   |   3 +
>  drivers/infiniband/core/verbs.c    |   9 ++
>  include/rdma/ib_verbs.h            |  18 +++
>  include/rdma/rdma_counter.h        |   8 ++
>  5 files changed, 259 insertions(+)
> 
> diff --git a/drivers/infiniband/core/counters.c b/drivers/infiniband/core/counters.c
> index 6167914fba06..60639452669c 100644
> +++ b/drivers/infiniband/core/counters.c
> @@ -54,6 +54,227 @@ int rdma_counter_set_auto_mode(struct ib_device *dev, u8 port,
>  	return ret;
>  }
>  
> +static struct rdma_counter *rdma_counter_alloc(struct ib_device *dev, u8 port,
> +					       enum rdma_nl_counter_mode mode)
> +{
> +	struct rdma_counter *counter;
> +
> +	if (!dev->ops.counter_dealloc)
> +		return NULL;
> +
> +	counter = kzalloc(sizeof(*counter), GFP_KERNEL);
> +	if (!counter)
> +		return NULL;
> +
> +	counter->device    = dev;
> +	counter->port      = port;
> +	counter->res.type  = RDMA_RESTRACK_COUNTER;
> +	counter->mode.mode = mode;
> +	kref_init(&counter->kref);
> +	mutex_init(&counter->lock);
> +
> +	return counter;
> +}
> +
> +static void rdma_counter_free(struct rdma_counter *counter)
> +{
> +	rdma_restrack_del(&counter->res);
> +	kfree(counter);
> +}
> +
> +static void auto_mode_init_counter(struct rdma_counter *counter,
> +				   const struct ib_qp *qp,
> +				   enum rdma_nl_counter_mask new_mask)
> +{
> +	struct auto_mode_param *param = &counter->mode.param;
> +
> +	counter->mode.mode = RDMA_COUNTER_MODE_AUTO;
> +	counter->mode.mask = new_mask;
> +
> +	if (new_mask & RDMA_COUNTER_MASK_QP_TYPE)
> +		param->qp_type = qp->qp_type;
> +}
> +
> +static bool auto_mode_match(struct ib_qp *qp, struct rdma_counter *counter,
> +			    enum rdma_nl_counter_mask auto_mask)
> +{
> +	struct auto_mode_param *param = &counter->mode.param;
> +	bool match = true;
> +
> +	if (rdma_is_kernel_res(&counter->res) != rdma_is_kernel_res(&qp->res))
> +		return false;
> +
> +	/* Ensure that counter belong to right PID */
> +	if (!rdma_is_kernel_res(&counter->res) &&
> +	    !rdma_is_kernel_res(&qp->res) &&
> +	    (task_pid_vnr(counter->res.task) != current->pid))
> +		return false;
> +
> +	if (auto_mask & RDMA_COUNTER_MASK_QP_TYPE)
> +		match &= (param->qp_type == qp->qp_type);
> +
> +	return match;
> +}
> +
> +static int __rdma_counter_bind_qp(struct rdma_counter *counter,
> +				  struct ib_qp *qp)
> +{
> +	int ret;
> +
> +	if (qp->counter)
> +		return -EINVAL;
> +
> +	if (!qp->device->ops.counter_bind_qp)
> +		return -EOPNOTSUPP;
> +
> +	mutex_lock(&counter->lock);
> +	ret = qp->device->ops.counter_bind_qp(counter, qp);
> +	mutex_unlock(&counter->lock);
> +
> +	return ret;
> +}
> +
> +static int __rdma_counter_unbind_qp(struct ib_qp *qp)
> +{
> +	struct rdma_counter *counter = qp->counter;
> +	int ret;
> +
> +	if (!qp->device->ops.counter_unbind_qp)
> +		return -EOPNOTSUPP;
> +
> +	mutex_lock(&counter->lock);
> +	ret = qp->device->ops.counter_unbind_qp(qp);
> +	mutex_unlock(&counter->lock);
> +
> +	return ret;
> +}
> +
> +/**
> + * rdma_get_counter_auto_mode - Find the counter that @qp should be bound
> + *     with in auto mode
> + *
> + * Return: The counter (with ref-count increased) if found
> + */
> +static struct rdma_counter *rdma_get_counter_auto_mode(struct ib_qp *qp,
> +						       u8 port)
> +{
> +	struct rdma_port_counter *port_counter;
> +	struct rdma_counter *counter = NULL;
> +	struct ib_device *dev = qp->device;
> +	struct rdma_restrack_entry *res;
> +	struct rdma_restrack_root *rt;
> +	unsigned long id = 0;
> +
> +	port_counter = &dev->port_data[port].port_counter;
> +	rt = &dev->res[RDMA_RESTRACK_COUNTER];
> +	xa_lock(&rt->xa);
> +	xa_for_each(&rt->xa, id, res) {
> +		if (!rdma_is_visible_in_pid_ns(res))
> +			continue;
> +
> +		counter = container_of(res, struct rdma_counter, res);
> +		if ((counter->device != qp->device) || (counter->port != port))
> +			goto next;
> +
> +		if (auto_mode_match(qp, counter, port_counter->mode.mask))
> +			break;
> +next:
> +		counter = NULL;
> +	}
> +
> +	if (counter)
> +		kref_get(&counter->kref);

This still needs to be kref_get_unless_zero:

	if (counter && !kref_get_unless_zero(&counter->kref))
		counter = NULL;

Jason