Re: [PATCH v6.11-rc1 06/10] dlm: async freeing of lockspace resources

Alexander Aring
2024-07-30T15:37:56Z

Hi,

On Mon, Jul 29, 2024 at 3:36 PM Alexander Aring <aahringo@redhat.com> wrote:
>
> This patch handles freeing of lockspace resources asynchronously besides
> the release_lockspace() context. The release_lockspace() context is
> sometimes called in a time critical context, e.g. umount syscall. Most
> every user space init system will timeout if it takes too long. To
> reduce the potential waiting time we deregister in release_lockspace()
> the lockspace from the DLM subsystem and do the actual releasing of
> lockspace resource in a worker of a workqueue following recommendation
> of:
>
> https://lore.kernel.org/all/49925af7-78a8-a3dd-bce6-cfc02e1a9236@I-love.SAKURA.ne.jp/T/#u
>
> as flushing of system workqueues are not allowed. The most time to
> release the DLM resources are spent to release the data structures
> "ls->ls_lkbxa" and "ls->ls_rsbtbl" as they iterate over each entries and
> those data structures can contain millions of entries.
>
> Signed-off-by: Alexander Aring <aahringo@redhat.com>
> ---
> fs/dlm/dlm_internal.h | 4 ++
> fs/dlm/lockspace.c | 100 +++++++++++++++++++++++-------------------
> fs/dlm/main.c | 10 +++++
> 3 files changed, 69 insertions(+), 45 deletions(-)
>
> diff --git a/fs/dlm/dlm_internal.h b/fs/dlm/dlm_internal.h
> index 32d98e63d25e..0562099e60eb 100644
> --- a/fs/dlm/dlm_internal.h
> +++ b/fs/dlm/dlm_internal.h
> @@ -660,6 +660,8 @@ struct dlm_ls {
> const struct dlm_lockspace_ops *ls_ops;
> void *ls_ops_arg;
>
> + struct work_struct ls_free_work;
> +
> int ls_namelen;
> char ls_name[DLM_LOCKSPACE_LEN + 1];
> };
> @@ -803,6 +805,8 @@ static inline void dlm_set_sbflags_val(struct dlm_lkb *lkb, uint32_t val)
> __DLM_SBF_MAX_BIT);
> }
>
> +extern struct workqueue_struct *dlm_wq;
> +
> int dlm_plock_init(void);
> void dlm_plock_exit(void);
>
> diff --git a/fs/dlm/lockspace.c b/fs/dlm/lockspace.c
> index bf14016d53e1..e1df92853a53 100644
> --- a/fs/dlm/lockspace.c
> +++ b/fs/dlm/lockspace.c
> @@ -315,6 +315,56 @@ static int threads_start(void)
> return error;
> }
>
> +static int lkb_idr_free(struct dlm_lkb *lkb)
> +{
> + if (lkb->lkb_lvbptr && test_bit(DLM_IFL_MSTCPY_BIT, &lkb->lkb_iflags))
> + dlm_free_lvb(lkb->lkb_lvbptr);
> +
> + dlm_free_lkb(lkb);
> + return 0;
> +}
> +
> +static void rhash_free_rsb(void *ptr, void *arg)
> +{
> + struct dlm_rsb *rsb = ptr;
> +
> + dlm_free_rsb(rsb);
> +}
> +
> +static void free_lockspace(struct work_struct *work)
> +{
> + struct dlm_ls *ls = container_of(work, struct dlm_ls, ls_free_work);
> + struct dlm_lkb *lkb;
> + unsigned long id;
> +
> + xa_destroy(&ls->ls_recover_xa);
> + kfree(ls->ls_recover_buf);
> +
> + /*
> + * Free all lkb's in xa
> + */
> + xa_for_each(&ls->ls_lkbxa, id, lkb) {
> + lkb_idr_free(lkb);
> + }
> + xa_destroy(&ls->ls_lkbxa);
> +
> + /*
> + * Free all rsb's on rsbtbl
> + */
> + rhashtable_free_and_destroy(&ls->ls_rsbtbl, rhash_free_rsb, NULL);
> +
> + /*
> + * Free structures on any other lists
> + */
> +
> + dlm_purge_requestqueue(ls);
> + kfree(ls->ls_recover_args);
> + dlm_clear_members(ls);
> + dlm_clear_members_gone(ls);
> + kfree(ls->ls_node_array);
> + kfree(ls);
> +}
> +
> static int new_lockspace(const char *name, const char *cluster,
> uint32_t flags, int lvblen,
> const struct dlm_lockspace_ops *ops, void *ops_arg,
> @@ -445,6 +495,8 @@ static int new_lockspace(const char *name, const char *cluster,
> spin_lock_init(&ls->ls_cb_lock);
> INIT_LIST_HEAD(&ls->ls_cb_delay);
>
> + INIT_WORK(&ls->ls_free_work, free_lockspace);
> +
> ls->ls_recoverd_task = NULL;
> mutex_init(&ls->ls_recoverd_active);
> spin_lock_init(&ls->ls_recover_lock);
> @@ -627,15 +679,6 @@ int dlm_new_user_lockspace(const char *name, const char *cluster,
> ops_arg, ops_result, lockspace);
> }
>
> -static int lkb_idr_free(struct dlm_lkb *lkb)
> -{
> - if (lkb->lkb_lvbptr && test_bit(DLM_IFL_MSTCPY_BIT, &lkb->lkb_iflags))
> - dlm_free_lvb(lkb->lkb_lvbptr);
> -
> - dlm_free_lkb(lkb);
> - return 0;
> -}
> -
> /* NOTE: We check the lkbxa here rather than the resource table.
> This is because there may be LKBs queued as ASTs that have been unlinked
> from their RSBs and are pending deletion once the AST has been delivered */
> @@ -667,17 +710,8 @@ static int lockspace_busy(struct dlm_ls *ls, int force)
> return rv;
> }
>
> -static void rhash_free_rsb(void *ptr, void *arg)
> -{
> - struct dlm_rsb *rsb = ptr;
> -
> - dlm_free_rsb(rsb);
> -}
> -
> static int release_lockspace(struct dlm_ls *ls, int force)
> {
> - struct dlm_lkb *lkb;
> - unsigned long id;
> int busy, rv;
>
> busy = lockspace_busy(ls, force);
> @@ -732,34 +766,10 @@ static int release_lockspace(struct dlm_ls *ls, int force)
>
> kobject_put(&ls->ls_kobj);
>
> - xa_destroy(&ls->ls_recover_xa);
> - kfree(ls->ls_recover_buf);
> -
> - /*
> - * Free all lkb's in xa
> - */
> - xa_for_each(&ls->ls_lkbxa, id, lkb) {
> - lkb_idr_free(lkb);
> - }
> - xa_destroy(&ls->ls_lkbxa);
> -
> - /*
> - * Free all rsb's on rsbtbl
> - */
> - rhashtable_free_and_destroy(&ls->ls_rsbtbl, rhash_free_rsb, NULL);
> -
> - /*
> - * Free structures on any other lists
> - */
> -
> - dlm_purge_requestqueue(ls);

this.

> - kfree(ls->ls_recover_args);
> - dlm_clear_members(ls);

and this.

Should stay here, it does more than just releasing memory. But the
rest should be okay to handle it in the background in a different
context in some kind of slow path. As mentioned in the commit msg it
is mostly about the data structures here that may contain millions of
entries (we don't know what the DLM user is doing) and we are doing
"our best" to return faster from the release_lockspace() function.

- Alex