Re: [PATCH 14/17] nvmet-tcp: reference counting for queues

[Simon Horman <horms@kernel.org>]

On Thu, Aug 10, 2023 at 05:06:27PM +0200, Hannes Reinecke wrote:
> The 'queue' structure is referenced from various places and
> used as an argument of asynchronous functions, so it's really
> hard to figure out if the queue is still valid when the
> asynchronous function triggers.
> So add reference counting to validate the queue structure.
> 
> Signed-off-by: Hannes Reinecke <hare@suse.de>
> ---
>  drivers/nvme/target/tcp.c | 73 ++++++++++++++++++++++++++++++---------
>  1 file changed, 56 insertions(+), 17 deletions(-)
> 
> diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c
> index ce1d1c5f4e90..a79ede885865 100644
> --- a/drivers/nvme/target/tcp.c
> +++ b/drivers/nvme/target/tcp.c
> @@ -127,6 +127,7 @@ enum nvmet_tcp_queue_state {
>  };
>  
>  struct nvmet_tcp_queue {
> +	struct kref		kref;
>  	struct socket		*sock;
>  	struct nvmet_tcp_port	*port;
>  	struct work_struct	io_work;
> @@ -192,6 +193,8 @@ static struct workqueue_struct *nvmet_tcp_wq;
>  static const struct nvmet_fabrics_ops nvmet_tcp_ops;
>  static void nvmet_tcp_free_cmd(struct nvmet_tcp_cmd *c);
>  static void nvmet_tcp_free_cmd_buffers(struct nvmet_tcp_cmd *cmd);
> +static int nvmet_tcp_get_queue(struct nvmet_tcp_queue *queue);
> +static void nvmet_tcp_put_queue(struct nvmet_tcp_queue *queue);
>  
>  static inline u16 nvmet_tcp_cmd_tag(struct nvmet_tcp_queue *queue,
>  		struct nvmet_tcp_cmd *cmd)
> @@ -1437,11 +1440,21 @@ static void nvmet_tcp_restore_socket_callbacks(struct nvmet_tcp_queue *queue)
>  	struct socket *sock = queue->sock;
>  
>  	write_lock_bh(&sock->sk->sk_callback_lock);
> +	/*
> +	 * Check if nvmet_tcp_set_queue_sock() has been called;
> +	 * if not the queue reference has not been increased
> +	 * and we're getting an refcount error on exit.
> +	 */
> +	if (sock->sk->sk_data_ready != nvmet_tcp_data_ready) {

Hi Hannes,

it seems that nvmet_tcp_data_ready is used here,
but doesn't exist until patch 16/17.

> +		write_unlock_bh(&sock->sk->sk_callback_lock);
> +		return;
> +	}
>  	sock->sk->sk_data_ready =  queue->data_ready;
>  	sock->sk->sk_state_change = queue->state_change;
>  	sock->sk->sk_write_space = queue->write_space;
>  	sock->sk->sk_user_data = NULL;
>  	write_unlock_bh(&sock->sk->sk_callback_lock);
> +	nvmet_tcp_put_queue(queue);
>  }

...