Re: [RFC PATCH v2 13/22] devlink: track snapshot ids using an IDR and refcounts

Jacob Keller
2020-02-18T21:44:37Z

On 2/14/2020 3:22 PM, Jacob Keller wrote:
> New snapshot ids are generated by calling a getter function. The same id
> may be used by multiple snapshots created at the same trigger event.
>
> Currently no effort is made to release any previously used snapshot ids.
>
> Replace the basic logic of using a single devlink integer for tracking
> ids with the IDR interface.
>
> snapshot IDs will be reference counted using a refcount stored in the
> IDR. First, ids are allocated using idr_alloc without a refcount (using
> the NULL pointer).
>
> Once the devlink_region_snapshot_create function is called, it will call
> the new __devlink_region_snapshot_id_ref(). This function will insert
> a new refcount or increment the pre-existing refcount.
>
> devlink_region_snapshot_destroy will call the new
> __devlink_region_snapshot_id_deref(), decrementing the reference count.
> Once there are no other references, the refcount will be removed from
> IDR.
>
> Signed-off-by: Jacob Keller <jacob.e.keller@intel.com>

Based on recent comments on the list it looks like the preference would
be to use xarray directly instead of IDR for this?

(See https://lore.kernel.org/netdev/20200218135359.GA9608@ziepe.ca/ )

I'd be happy to rework this to use xarray instead. I don't believe we
can use an ida directly because of the implementation that enables
re-using the same snapshot id for multiple snapshots. (hence why I went
with a reference count that can be NULL initially to "pre-allocate" the id)

I'm open to also considering whether we should simplify how ids are
managed in some way so that an ida on its own can be used.

Thanks,
Jake

> ---
> include/net/devlink.h | 3 +-
> net/core/devlink.c | 141 ++++++++++++++++++++++++++++++++++++++++--
> 2 files changed, 138 insertions(+), 6 deletions(-)
>
> diff --git a/include/net/devlink.h b/include/net/devlink.h
> index 3a7759355434..3a5ff6bea143 100644
> --- a/include/net/devlink.h
> +++ b/include/net/devlink.h
> @@ -17,6 +17,7 @@
> #include <linux/refcount.h>
> #include <net/net_namespace.h>
> #include <uapi/linux/devlink.h>
> +#include <linux/idr.h>
>
> struct devlink_ops;
>
> @@ -28,13 +29,13 @@ struct devlink {
> struct list_head resource_list;
> struct list_head param_list;
> struct list_head region_list;
> - u32 snapshot_id;
> struct list_head reporter_list;
> struct mutex reporters_lock; /* protects reporter_list */
> struct devlink_dpipe_headers *dpipe_headers;
> struct list_head trap_list;
> struct list_head trap_group_list;
> const struct devlink_ops *ops;
> + struct idr snapshot_idr;
> struct device *dev;
> possible_net_t _net;
> struct mutex lock;
> diff --git a/net/core/devlink.c b/net/core/devlink.c
> index da4e669f425b..9571063846cc 100644
> --- a/net/core/devlink.c
> +++ b/net/core/devlink.c
> @@ -3760,21 +3760,118 @@ static void devlink_nl_region_notify(struct devlink_region *region,
> nlmsg_free(msg);
> }
>
> +/**
> + * __devlink_region_snapshot_id_ref - Increment reference for a snapshot ID
> + * @devlink: devlink instance
> + * @id: the snapshot id being referenced
> + *
> + * Increments the reference count for the given snapshot id. If the id
> + * was not yet allocated, it is allocated immediately. If the id was
> + * allocated but no references exist, a new refcount is created and
> + * inserted.
> + */
> +static int __devlink_region_snapshot_id_ref(struct devlink *devlink, u32 id)
> +{
> + struct idr *idr = &devlink->snapshot_idr;
> + refcount_t *ref;
> + void *old_ptr;
> + int err;
> +
> + lockdep_assert_held(&devlink->lock);
> +
> + if (id < 1 || id >= INT_MAX)
> + return -EINVAL;
> +
> + /* Check if a refcount already exists. If so, increment it and exit */
> + ref = idr_find(idr, id);
> + if (ref) {
> + refcount_inc(ref);
> + return 0;
> + }
> +
> + /* Allocate a new reference count */
> + ref = kzalloc(sizeof(*ref), GFP_KERNEL);
> + refcount_set(ref, 1);
> +
> + /* The id was likely allocated ahead of time using
> + * devlink_region_snapshot_id_get, so attempt to replace the NULL
> + * pointer with the refcount. Since idr_find returned NULL,
> + * idr_replace should either return ERR_PTR(-ENOENT) or NULL.
> + */
> + old_ptr = idr_replace(idr, ref, id);
> + /* if old_ptr is NULL, we've inserted the reference */
> + if (old_ptr == NULL)
> + return 0;
> + if (PTR_ERR(old_ptr) != -ENOENT) {
> + kfree(ref);
> + return PTR_ERR(old_ptr);
> + }
> +
> + /* the snapshot id was not reserved, so reserve it now. */
> + err = idr_alloc(idr, ref, id, id+1, GFP_KERNEL);
> + if (err < 0)
> + return err;
> + WARN_ON(err != id);
> +
> + return 0;
> +}
> +
> +/**
> + * __devlink_region_snapshot_id_deref - Decrement reference for a snapshot ID
> + * @devlink: devlink instance
> + * @id: the snapshot id being referenced
> + *
> + * Decrements the reference count for a given snapshot id. If the
> + * refcount has reached zero then remove the reference from the IDR.
> + */
> +static void __devlink_region_snapshot_id_deref(struct devlink *devlink, u32 id)
> +{
> + struct idr *idr = &devlink->snapshot_idr;
> + refcount_t *ref;
> +
> + lockdep_assert_held(&devlink->lock);
> +
> + if (WARN_ON(id < 1 || id >= INT_MAX))
> + return;
> +
> + /* Find the reference pointer */
> + ref = idr_find(idr, id);
> + if (!ref) {
> + WARN(true, "no previous reference was inserted");
> + /* this shouldn't happen, but at least attempt to cleanup if
> + * something went wrong.
> + */
> + idr_remove(idr, id);
> + return;
> + }
> +
> + if (refcount_dec_and_test(ref)) {
> + /* There are no more references, so remove it from the IDR and
> + * free the reference count.
> + */
> + idr_remove(idr, id);
> + kfree(ref);
> + }
> +}
> +
> /**
> * __devlink_region_snapshot_id_get - get snapshot ID
> * @devlink: devlink instance
> *
> * Returns a new snapshot id or a negative error code on failure. Must be
> * called while holding the devlink instance lock.
> + *
> + * Snapshot ids are stored in an IDR and reference counted by the number
> + * of snapshots currently using that id. This function pre-allocates
> + * a snapshot id but does not fill in a reference count. A later call to
> + * devlink_region_snapshot_create will update the IDR pointer to
> + * a reference count. On devlink_region_snapshot_destory, if there are no
> + * further references, the id will be removed from the IDR.
> */
> static int __devlink_region_snapshot_id_get(struct devlink *devlink)
> {
> lockdep_assert_held(&devlink->lock);
> -
> - if (devlink->snapshot_id >= INT_MAX)
> - return -ENOSPC;
> -
> - return ++devlink->snapshot_id;
> + return idr_alloc(&devlink->snapshot_idr, NULL, 1, INT_MAX, GFP_KERNEL);
> }
>
> /**
> @@ -3797,6 +3894,7 @@ __devlink_region_snapshot_create(struct devlink_region *region,
> {
> struct devlink *devlink = region->devlink;
> struct devlink_snapshot *snapshot;
> + int err;
>
> lockdep_assert_held(&devlink->lock);
>
> @@ -3811,6 +3909,11 @@ __devlink_region_snapshot_create(struct devlink_region *region,
> if (!snapshot)
> return -ENOMEM;
>
> + /* Increment snapshot id reference */
> + err = __devlink_region_snapshot_id_ref(devlink, snapshot_id);
> + if (err)
> + goto err_free_snapshot;
> +
> snapshot->id = snapshot_id;
> snapshot->region = region;
> snapshot->data = data;
> @@ -3821,15 +3924,25 @@ __devlink_region_snapshot_create(struct devlink_region *region,
>
> devlink_nl_region_notify(region, snapshot, DEVLINK_CMD_REGION_NEW);
> return 0;
> +
> +err_free_snapshot:
> + kfree(snapshot);
> + return err;
> }
>
> static void devlink_region_snapshot_del(struct devlink_region *region,
> struct devlink_snapshot *snapshot)
> {
> + struct devlink *devlink = region->devlink;
> +
> + lockdep_assert_held(&devlink_mutex);
> +
> devlink_nl_region_notify(region, snapshot, DEVLINK_CMD_REGION_DEL);
> region->cur_snapshots--;
> list_del(&snapshot->list);
> region->ops->destructor(snapshot->data);
> + __devlink_region_snapshot_id_deref(devlink, snapshot->id);
> +
> kfree(snapshot);
> }
>
> @@ -6388,6 +6501,7 @@ struct devlink *devlink_alloc(const struct devlink_ops *ops, size_t priv_size)
> if (!devlink)
> return NULL;
> devlink->ops = ops;
> + idr_init(&devlink->snapshot_idr);
> __devlink_net_set(devlink, &init_net);
> INIT_LIST_HEAD(&devlink->port_list);
> INIT_LIST_HEAD(&devlink->sb_list);
> @@ -6480,6 +6594,23 @@ EXPORT_SYMBOL_GPL(devlink_reload_disable);
> */
> void devlink_free(struct devlink *devlink)
> {
> + struct idr *idr = &devlink->snapshot_idr;
> +
> + mutex_lock(&devlink->lock);
> + if (!idr_is_empty(idr)) {
> + refcount_t *ref;
> + int id;
> +
> + WARN(true, "snapshot IDR is not empty");
> +
> + idr_for_each_entry(idr, ref, id) {
> + if (ref)
> + kfree(ref);
> + }
> + }
> + idr_destroy(&devlink->snapshot_idr);
> + mutex_unlock(&devlink->lock);
> +
> mutex_destroy(&devlink->reporters_lock);
> mutex_destroy(&devlink->lock);
> WARN_ON(!list_empty(&devlink->trap_group_list));
>