Re: [PATCH 14/17] nvmet-tcp: reference counting for queues

Simon Horman
2023-08-11T10:30:16Z

On Thu, Aug 10, 2023 at 05:06:27PM +0200, Hannes Reinecke wrote:
> The 'queue' structure is referenced from various places and
> used as an argument of asynchronous functions, so it's really
> hard to figure out if the queue is still valid when the
> asynchronous function triggers.
> So add reference counting to validate the queue structure.
>
> Signed-off-by: Hannes Reinecke <hare@suse.de>
> ---
> drivers/nvme/target/tcp.c | 73 ++++++++++++++++++++++++++++++---------
> 1 file changed, 56 insertions(+), 17 deletions(-)
>
> diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c
> index ce1d1c5f4e90..a79ede885865 100644
> --- a/drivers/nvme/target/tcp.c
> +++ b/drivers/nvme/target/tcp.c
> @@ -127,6 +127,7 @@ enum nvmet_tcp_queue_state {
> };
>
> struct nvmet_tcp_queue {
> + struct kref kref;
> struct socket *sock;
> struct nvmet_tcp_port *port;
> struct work_struct io_work;
> @@ -192,6 +193,8 @@ static struct workqueue_struct *nvmet_tcp_wq;
> static const struct nvmet_fabrics_ops nvmet_tcp_ops;
> static void nvmet_tcp_free_cmd(struct nvmet_tcp_cmd *c);
> static void nvmet_tcp_free_cmd_buffers(struct nvmet_tcp_cmd *cmd);
> +static int nvmet_tcp_get_queue(struct nvmet_tcp_queue *queue);
> +static void nvmet_tcp_put_queue(struct nvmet_tcp_queue *queue);
>
> static inline u16 nvmet_tcp_cmd_tag(struct nvmet_tcp_queue *queue,
> struct nvmet_tcp_cmd *cmd)
> @@ -1437,11 +1440,21 @@ static void nvmet_tcp_restore_socket_callbacks(struct nvmet_tcp_queue *queue)
> struct socket *sock = queue->sock;
>
> write_lock_bh(&sock->sk->sk_callback_lock);
> + /*
> + * Check if nvmet_tcp_set_queue_sock() has been called;
> + * if not the queue reference has not been increased
> + * and we're getting an refcount error on exit.
> + */
> + if (sock->sk->sk_data_ready != nvmet_tcp_data_ready) {

Hi Hannes,

it seems that nvmet_tcp_data_ready is used here,
but doesn't exist until patch 16/17.

> + write_unlock_bh(&sock->sk->sk_callback_lock);
> + return;
> + }
> sock->sk->sk_data_ready = queue->data_ready;
> sock->sk->sk_state_change = queue->state_change;
> sock->sk->sk_write_space = queue->write_space;
> sock->sk->sk_user_data = NULL;
> write_unlock_bh(&sock->sk->sk_callback_lock);
> + nvmet_tcp_put_queue(queue);
> }

...