Re: [PATCH v18 25/26] drm/virtio: Support shmem shrinking

Re: [PATCH v18 25/26] drm/virtio: Support shmem shrinking

[Gurchetan Singh]

[-- Attachment #1.1: Type: text/plain, Size: 17067 bytes --]

On Sun, Oct 29, 2023 at 4:03 PM Dmitry Osipenko <
dmitry.osipenko@collabora.com> wrote:

> Support generic drm-shmem memory shrinker and add new madvise IOCTL to
> the VirtIO-GPU driver. BO cache manager of Mesa driver will mark BOs as
> "don't need" using the new IOCTL to let shrinker purge the marked BOs on
> OOM, the shrinker will also evict unpurgeable shmem BOs from memory if
> guest supports SWAP file or partition.
>
> Acked-by: Gerd Hoffmann <kraxel@redhat.com>
> Signed-off-by: Daniel Almeida <daniel.almeida@collabora.com>
> Signed-off-by: Dmitry Osipenko <dmitry.osipenko@collabora.com>
> ---
>  drivers/gpu/drm/virtio/virtgpu_drv.h    | 13 +++++-
>  drivers/gpu/drm/virtio/virtgpu_gem.c    | 35 ++++++++++++++
>  drivers/gpu/drm/virtio/virtgpu_ioctl.c  | 25 ++++++++++
>  drivers/gpu/drm/virtio/virtgpu_kms.c    |  8 ++++
>  drivers/gpu/drm/virtio/virtgpu_object.c | 61 +++++++++++++++++++++++++
>  drivers/gpu/drm/virtio/virtgpu_vq.c     | 40 ++++++++++++++++
>  include/uapi/drm/virtgpu_drm.h          | 14 ++++++
>  7 files changed, 195 insertions(+), 1 deletion(-)
>
> diff --git a/drivers/gpu/drm/virtio/virtgpu_drv.h
> b/drivers/gpu/drm/virtio/virtgpu_drv.h
> index 421f524ae1de..33a78b24c272 100644
> --- a/drivers/gpu/drm/virtio/virtgpu_drv.h
> +++ b/drivers/gpu/drm/virtio/virtgpu_drv.h
> @@ -278,7 +278,7 @@ struct virtio_gpu_fpriv {
>  };
>
>  /* virtgpu_ioctl.c */
> -#define DRM_VIRTIO_NUM_IOCTLS 12
> +#define DRM_VIRTIO_NUM_IOCTLS 13
>  extern struct drm_ioctl_desc virtio_gpu_ioctls[DRM_VIRTIO_NUM_IOCTLS];
>  void virtio_gpu_create_context(struct drm_device *dev, struct drm_file
> *file);
>
> @@ -316,6 +316,8 @@ void virtio_gpu_array_put_free_delayed(struct
> virtio_gpu_device *vgdev,
>  void virtio_gpu_array_put_free_work(struct work_struct *work);
>  int virtio_gpu_array_prepare(struct virtio_gpu_device *vgdev,
>                              struct virtio_gpu_object_array *objs);
> +int virtio_gpu_gem_host_mem_release(struct virtio_gpu_object *bo);
> +int virtio_gpu_gem_madvise(struct virtio_gpu_object *obj, int madv);
>  int virtio_gpu_gem_pin(struct virtio_gpu_object *bo);
>  void virtio_gpu_gem_unpin(struct virtio_gpu_object *bo);
>
> @@ -329,6 +331,8 @@ void virtio_gpu_cmd_create_resource(struct
> virtio_gpu_device *vgdev,
>                                     struct virtio_gpu_fence *fence);
>  void virtio_gpu_cmd_unref_resource(struct virtio_gpu_device *vgdev,
>                                    struct virtio_gpu_object *bo);
> +int virtio_gpu_cmd_release_resource(struct virtio_gpu_device *vgdev,
> +                                   struct virtio_gpu_object *bo);
>  void virtio_gpu_cmd_transfer_to_host_2d(struct virtio_gpu_device *vgdev,
>                                         uint64_t offset,
>                                         uint32_t width, uint32_t height,
> @@ -349,6 +353,9 @@ void virtio_gpu_object_attach(struct virtio_gpu_device
> *vgdev,
>                               struct virtio_gpu_object *obj,
>                               struct virtio_gpu_mem_entry *ents,
>                               unsigned int nents);
> +void virtio_gpu_object_detach(struct virtio_gpu_device *vgdev,
> +                             struct virtio_gpu_object *obj,
> +                             struct virtio_gpu_fence *fence);
>  void virtio_gpu_cursor_ping(struct virtio_gpu_device *vgdev,
>                             struct virtio_gpu_output *output);
>  int virtio_gpu_cmd_get_display_info(struct virtio_gpu_device *vgdev);
> @@ -492,4 +499,8 @@ void virtio_gpu_vram_unmap_dma_buf(struct device *dev,
>  int virtio_gpu_execbuffer_ioctl(struct drm_device *dev, void *data,
>                                 struct drm_file *file);
>
> +/* virtgpu_gem_shrinker.c */
> +int virtio_gpu_gem_shrinker_init(struct virtio_gpu_device *vgdev);
> +void virtio_gpu_gem_shrinker_fini(struct virtio_gpu_device *vgdev);
> +
>  #endif
> diff --git a/drivers/gpu/drm/virtio/virtgpu_gem.c
> b/drivers/gpu/drm/virtio/virtgpu_gem.c
> index 97e67064c97e..748f7bbb0e6d 100644
> --- a/drivers/gpu/drm/virtio/virtgpu_gem.c
> +++ b/drivers/gpu/drm/virtio/virtgpu_gem.c
> @@ -147,10 +147,20 @@ void virtio_gpu_gem_object_close(struct
> drm_gem_object *obj,
>         struct virtio_gpu_device *vgdev = obj->dev->dev_private;
>         struct virtio_gpu_fpriv *vfpriv = file->driver_priv;
>         struct virtio_gpu_object_array *objs;
> +       struct virtio_gpu_object *bo;
>
>         if (!vgdev->has_virgl_3d)
>                 return;
>
> +       bo = gem_to_virtio_gpu_obj(obj);
> +
> +       /*
> +        * Purged BO was already detached and released, the resource ID
> +        * is invalid by now.
> +        */
> +       if (!virtio_gpu_gem_madvise(bo, VIRTGPU_MADV_WILLNEED))
> +               return;
> +
>         objs = virtio_gpu_array_alloc(1);
>         if (!objs)
>                 return;
> @@ -315,6 +325,31 @@ int virtio_gpu_array_prepare(struct virtio_gpu_device
> *vgdev,
>         return ret;
>  }
>
> +int virtio_gpu_gem_madvise(struct virtio_gpu_object *bo, int madv)
> +{
> +       if (virtio_gpu_is_shmem(bo))
> +               return drm_gem_shmem_object_madvise(&bo->base.base, madv);
> +
> +       return 1;
> +}
> +
> +int virtio_gpu_gem_host_mem_release(struct virtio_gpu_object *bo)
> +{
> +       struct virtio_gpu_device *vgdev = bo->base.base.dev->dev_private;
> +       int err;
> +
> +       if (bo->created) {
> +               err = virtio_gpu_cmd_release_resource(vgdev, bo);
> +               if (err)
> +                       return err;
> +
> +               virtio_gpu_notify(vgdev);
> +               bo->created = false;
> +       }
> +
> +       return 0;
> +}
> +
>  int virtio_gpu_gem_pin(struct virtio_gpu_object *bo)
>  {
>         int err;
> diff --git a/drivers/gpu/drm/virtio/virtgpu_ioctl.c
> b/drivers/gpu/drm/virtio/virtgpu_ioctl.c
> index 070c29cea26a..44a99166efdc 100644
> --- a/drivers/gpu/drm/virtio/virtgpu_ioctl.c
> +++ b/drivers/gpu/drm/virtio/virtgpu_ioctl.c
> @@ -676,6 +676,28 @@ static int virtio_gpu_context_init_ioctl(struct
> drm_device *dev,
>         return ret;
>  }
>
> +static int virtio_gpu_madvise_ioctl(struct drm_device *dev,
> +                                   void *data,
> +                                   struct drm_file *file)
> +{
> +       struct drm_virtgpu_madvise *args = data;
> +       struct virtio_gpu_object *bo;
> +       struct drm_gem_object *obj;
> +
> +       if (args->madv > VIRTGPU_MADV_DONTNEED)
> +               return -EOPNOTSUPP;
> +
> +       obj = drm_gem_object_lookup(file, args->bo_handle);
> +       if (!obj)
> +               return -ENOENT;
> +
> +       bo = gem_to_virtio_gpu_obj(obj);
> +       args->retained = virtio_gpu_gem_madvise(bo, args->madv);
> +       drm_gem_object_put(obj);
> +
> +       return 0;
> +}
> +
>  struct drm_ioctl_desc virtio_gpu_ioctls[DRM_VIRTIO_NUM_IOCTLS] = {
>         DRM_IOCTL_DEF_DRV(VIRTGPU_MAP, virtio_gpu_map_ioctl,
>                           DRM_RENDER_ALLOW),
> @@ -715,4 +737,7 @@ struct drm_ioctl_desc
> virtio_gpu_ioctls[DRM_VIRTIO_NUM_IOCTLS] = {
>
>         DRM_IOCTL_DEF_DRV(VIRTGPU_CONTEXT_INIT,
> virtio_gpu_context_init_ioctl,
>                           DRM_RENDER_ALLOW),
> +
> +       DRM_IOCTL_DEF_DRV(VIRTGPU_MADVISE, virtio_gpu_madvise_ioctl,
> +                         DRM_RENDER_ALLOW),
>  };
> diff --git a/drivers/gpu/drm/virtio/virtgpu_kms.c
> b/drivers/gpu/drm/virtio/virtgpu_kms.c
> index 5a3b5aaed1f3..43e237082cec 100644
> --- a/drivers/gpu/drm/virtio/virtgpu_kms.c
> +++ b/drivers/gpu/drm/virtio/virtgpu_kms.c
> @@ -245,6 +245,12 @@ int virtio_gpu_init(struct virtio_device *vdev,
> struct drm_device *dev)
>                 goto err_scanouts;
>         }
>
> +       ret = drmm_gem_shmem_init(dev);
> +       if (ret) {
> +               DRM_ERROR("shmem init failed\n");
> +               goto err_modeset;
> +       }
> +
>         virtio_device_ready(vgdev->vdev);
>
>         if (num_capsets)
> @@ -259,6 +265,8 @@ int virtio_gpu_init(struct virtio_device *vdev, struct
> drm_device *dev)
>         }
>         return 0;
>
> +err_modeset:
> +       virtio_gpu_modeset_fini(vgdev);
>  err_scanouts:
>         virtio_gpu_free_vbufs(vgdev);
>  err_vbufs:
> diff --git a/drivers/gpu/drm/virtio/virtgpu_object.c
> b/drivers/gpu/drm/virtio/virtgpu_object.c
> index 000bb7955a57..8fa5f912ae51 100644
> --- a/drivers/gpu/drm/virtio/virtgpu_object.c
> +++ b/drivers/gpu/drm/virtio/virtgpu_object.c
> @@ -98,6 +98,60 @@ static void virtio_gpu_free_object(struct
> drm_gem_object *obj)
>         virtio_gpu_cleanup_object(bo);
>  }
>
> +static int virtio_gpu_detach_object_fenced(struct virtio_gpu_object *bo)
> +{
> +       struct virtio_gpu_device *vgdev = bo->base.base.dev->dev_private;
> +       struct virtio_gpu_fence *fence;
> +
> +       if (bo->detached)
> +               return 0;
> +
> +       fence = virtio_gpu_fence_alloc(vgdev, vgdev->fence_drv.context, 0);
> +       if (!fence)
> +               return -ENOMEM;
> +
> +       virtio_gpu_object_detach(vgdev, bo, fence);
> +       virtio_gpu_notify(vgdev);
> +
> +       dma_fence_wait(&fence->f, false);
> +       dma_fence_put(&fence->f);
> +
> +       bo->detached = true;
> +
> +       return 0;
> +}
> +
> +static int virtio_gpu_shmem_evict(struct drm_gem_object *obj)
> +{
> +       struct virtio_gpu_object *bo = gem_to_virtio_gpu_obj(obj);
> +       int err;
> +
> +       /* blob is not movable, it's impossible to detach it from host */
> +       if (bo->blob_mem)
> +               return -EBUSY;
> +
> +       /*
> +        * At first tell host to stop using guest's memory to ensure that
> +        * host won't touch the released guest's memory once it's gone.
> +        */
> +       err = virtio_gpu_detach_object_fenced(bo);
> +       if (err)
> +               return err;
> +
> +       if (drm_gem_shmem_is_purgeable(&bo->base)) {
> +               err = virtio_gpu_gem_host_mem_release(bo);
> +               if (err)
> +                       return err;
> +
> +               drm_gem_shmem_purge_locked(&bo->base);
> +       } else {
> +               bo->base.pages_mark_dirty_on_put = 1;
> +               drm_gem_shmem_evict_locked(&bo->base);
> +       }
> +
> +       return 0;
> +}
> +
>  static const struct drm_gem_object_funcs virtio_gpu_shmem_funcs = {
>         .free = virtio_gpu_free_object,
>         .open = virtio_gpu_gem_object_open,
> @@ -111,6 +165,7 @@ static const struct drm_gem_object_funcs
> virtio_gpu_shmem_funcs = {
>         .vunmap = drm_gem_shmem_object_vunmap_locked,
>         .mmap = drm_gem_shmem_object_mmap,
>         .vm_ops = &drm_gem_shmem_vm_ops,
> +       .evict = virtio_gpu_shmem_evict,
>  };
>
>  bool virtio_gpu_is_shmem(struct virtio_gpu_object *bo)
> @@ -187,6 +242,10 @@ int virtio_gpu_reattach_shmem_object_locked(struct
> virtio_gpu_object *bo)
>         if (!bo->detached)
>                 return 0;
>
> +       err = drm_gem_shmem_swapin_locked(&bo->base);
> +       if (err)
> +               return err;
> +
>         err = virtio_gpu_object_shmem_init(vgdev, bo, &ents, &nents);
>         if (err)
>                 return err;
> @@ -240,6 +299,8 @@ int virtio_gpu_object_create(struct virtio_gpu_device
> *vgdev,
>                 goto err_put_pages;
>
>         bo->dumb = params->dumb;
> +       bo->blob_mem = params->blob_mem;
> +       bo->blob_flags = params->blob_flags;
>
>         if (bo->blob_mem == VIRTGPU_BLOB_MEM_GUEST)
>                 bo->guest_blob = true;
> diff --git a/drivers/gpu/drm/virtio/virtgpu_vq.c
> b/drivers/gpu/drm/virtio/virtgpu_vq.c
> index b1a00c0c25a7..14ab470f413a 100644
> --- a/drivers/gpu/drm/virtio/virtgpu_vq.c
> +++ b/drivers/gpu/drm/virtio/virtgpu_vq.c
> @@ -545,6 +545,21 @@ void virtio_gpu_cmd_unref_resource(struct
> virtio_gpu_device *vgdev,
>                 virtio_gpu_cleanup_object(bo);
>  }
>
> +int virtio_gpu_cmd_release_resource(struct virtio_gpu_device *vgdev,
> +                                   struct virtio_gpu_object *bo)
> +{
> +       struct virtio_gpu_resource_unref *cmd_p;
> +       struct virtio_gpu_vbuffer *vbuf;
> +
> +       cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));
> +       memset(cmd_p, 0, sizeof(*cmd_p));
> +
> +       cmd_p->hdr.type = cpu_to_le32(VIRTIO_GPU_CMD_RESOURCE_UNREF);
> +       cmd_p->resource_id = cpu_to_le32(bo->hw_res_handle);
> +
> +       return virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);
> +}
> +
>  void virtio_gpu_cmd_set_scanout(struct virtio_gpu_device *vgdev,
>                                 uint32_t scanout_id, uint32_t resource_id,
>                                 uint32_t width, uint32_t height,
> @@ -645,6 +660,23 @@ virtio_gpu_cmd_resource_attach_backing(struct
> virtio_gpu_device *vgdev,
>         virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, fence);
>  }
>
> +static void
> +virtio_gpu_cmd_resource_detach_backing(struct virtio_gpu_device *vgdev,
> +                                      u32 resource_id,
> +                                      struct virtio_gpu_fence *fence)
> +{
> +       struct virtio_gpu_resource_attach_backing *cmd_p;
> +       struct virtio_gpu_vbuffer *vbuf;
> +
> +       cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));
> +       memset(cmd_p, 0, sizeof(*cmd_p));
> +
> +       cmd_p->hdr.type =
> cpu_to_le32(VIRTIO_GPU_CMD_RESOURCE_DETACH_BACKING);
> +       cmd_p->resource_id = cpu_to_le32(resource_id);
> +
> +       virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, fence);
> +}
> +
>  static void virtio_gpu_cmd_get_display_info_cb(struct virtio_gpu_device
> *vgdev,
>                                                struct virtio_gpu_vbuffer
> *vbuf)
>  {
> @@ -1107,6 +1139,14 @@ void virtio_gpu_object_attach(struct
> virtio_gpu_device *vgdev,
>                                                ents, nents, NULL);
>  }
>
> +void virtio_gpu_object_detach(struct virtio_gpu_device *vgdev,
> +                             struct virtio_gpu_object *obj,
> +                             struct virtio_gpu_fence *fence)
> +{
> +       virtio_gpu_cmd_resource_detach_backing(vgdev, obj->hw_res_handle,
> +                                              fence);
> +}
> +
>  void virtio_gpu_cursor_ping(struct virtio_gpu_device *vgdev,
>                             struct virtio_gpu_output *output)
>  {
> diff --git a/include/uapi/drm/virtgpu_drm.h
> b/include/uapi/drm/virtgpu_drm.h
> index b1d0e56565bc..4caba71b2740 100644
> --- a/include/uapi/drm/virtgpu_drm.h
> +++ b/include/uapi/drm/virtgpu_drm.h
> @@ -48,6 +48,7 @@ extern "C" {
>  #define DRM_VIRTGPU_GET_CAPS  0x09
>  #define DRM_VIRTGPU_RESOURCE_CREATE_BLOB 0x0a
>  #define DRM_VIRTGPU_CONTEXT_INIT 0x0b
> +#define DRM_VIRTGPU_MADVISE 0x0c
>
>  #define VIRTGPU_EXECBUF_FENCE_FD_IN    0x01
>  #define VIRTGPU_EXECBUF_FENCE_FD_OUT   0x02
> @@ -211,6 +212,15 @@ struct drm_virtgpu_context_init {
>         __u64 ctx_set_params;
>  };
>
> +#define VIRTGPU_MADV_WILLNEED 0
> +#define VIRTGPU_MADV_DONTNEED 1
> +struct drm_virtgpu_madvise {
> +       __u32 bo_handle;
> +       __u32 retained; /* out, non-zero if BO can be used */
> +       __u32 madv;
> +       __u32 pad;
> +};
> +
>

Link to open-source userspace?

Also, don't you need a VIRTGPU_PARAM_MADVISE_SUPPORTED or is the plan to
use a DRM version?

Overall, the series for a generic shrinker seems useful for a wide variety
of DRM drivers, such as Panfrost.

For virtio-gpu, it could be a tad VIRGIL specific: since other context
types (gfxstream gles, DRM, vk contexts) decrease memory consumption by
either not allocating shadow buffers for textures/buffers, or using blob
memory.

Maybe we need to design with blob in mind, since we expect virgl to be
deprecated.  On Android, it basically is with ANGLE and native contexts.
On Linux, Zink looks good too.  Even with memory issues fixed, virgl is
unattractive compared to those solutions.

Android specific idea: I wonder if we could tie GEM blob buffers usage to
the lmkd and kill based on that ... ?

https://source.android.com/docs/core/perf/lmkd

Is there GEM buffer accounting infrastructure already?

 /*
>   * Event code that's given when VIRTGPU_CONTEXT_PARAM_POLL_RINGS_MASK is
> in
>   * effect.  The event size is sizeof(drm_event), since there is no
> additional
> @@ -261,6 +271,10 @@ struct drm_virtgpu_context_init {
>         DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_CONTEXT_INIT,           \
>                 struct drm_virtgpu_context_init)
>
> +#define DRM_IOCTL_VIRTGPU_MADVISE \
> +       DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_MADVISE, \
> +                struct drm_virtgpu_madvise)
> +
>  #if defined(__cplusplus)
>  }
>  #endif
> --
> 2.41.0
>
>

[-- Attachment #1.2: Type: text/html, Size: 20779 bytes --]

[-- Attachment #2: Type: text/plain, Size: 183 bytes --]

_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

Re: [PATCH v18 05/26] drm/shmem-helper: Remove obsoleted is_iomem test

[Boris Brezillon]

On Mon, 30 Oct 2023 02:01:44 +0300
Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:

> Everything that uses the mapped buffer should be agnostic to is_iomem.
> The only reason for the is_iomem test is that we're setting shmem->vaddr
> to the returned map->vaddr. Now that the shmem->vaddr code is gone, remove
> the obsoleted is_iomem test to clean up the code.
> 
> Suggested-by: Thomas Zimmermann <tzimmermann@suse.de>
> Signed-off-by: Dmitry Osipenko <dmitry.osipenko@collabora.com>

Reviewed-by: Boris Brezillon <boris.brezillon@collabora.com>

> ---
>  drivers/gpu/drm/drm_gem_shmem_helper.c | 6 ------
>  1 file changed, 6 deletions(-)
> 
> diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c
> index 154585ddae08..2cc0601865f6 100644
> --- a/drivers/gpu/drm/drm_gem_shmem_helper.c
> +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c
> @@ -315,12 +315,6 @@ int drm_gem_shmem_vmap_locked(struct drm_gem_shmem_object *shmem,
>  
>  	if (obj->import_attach) {
>  		ret = dma_buf_vmap(obj->import_attach->dmabuf, map);
> -		if (!ret) {
> -			if (drm_WARN_ON(obj->dev, map->is_iomem)) {
> -				dma_buf_vunmap(obj->import_attach->dmabuf, map);
> -				return -EIO;
> -			}
> -		}
>  	} else {
>  		pgprot_t prot = PAGE_KERNEL;
>  

Re: [PATCH v18 05/26] drm/shmem-helper: Remove obsoleted is_iomem test

[Maxime Ripard]

On Mon, 30 Oct 2023 02:01:44 +0300, Dmitry Osipenko wrote:
> Everything that uses the mapped buffer should be agnostic to is_iomem.
> The only reason for the is_iomem test is that we're setting shmem->vaddr
> to the returned map->vaddr. Now that the shmem->vaddr code is gone, remove
> the obsoleted is_iomem test to clean up the code.
> 
> 
> [ ... ]

Acked-by: Maxime Ripard <mripard@kernel.org>

Thanks!
Maxime

Re: [PATCH v18 11/26] drm/shmem-helper: Prepare drm_gem_shmem_free() to shrinker addition

[Boris Brezillon]

On Mon, 30 Oct 2023 02:01:50 +0300
Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:

> Prepare drm_gem_shmem_free() to addition of memory shrinker support
> to drm-shmem by adding and using variant of put_pages() that doesn't
> touch reservation lock. Reservation shouldn't be touched because lockdep
> will trigger a bogus warning about locking contention with fs_reclaim
> code paths that can't happen during the time when GEM is freed and
> lockdep doesn't know about that.
> 
> Signed-off-by: Dmitry Osipenko <dmitry.osipenko@collabora.com>
> ---
>  drivers/gpu/drm/drm_gem_shmem_helper.c | 35 +++++++++++++-------------
>  1 file changed, 18 insertions(+), 17 deletions(-)
> 
> diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c
> index 08b5a57c59d8..24ff2b99e75b 100644
> --- a/drivers/gpu/drm/drm_gem_shmem_helper.c
> +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c
> @@ -128,6 +128,22 @@ struct drm_gem_shmem_object *drm_gem_shmem_create(struct drm_device *dev, size_t
>  }
>  EXPORT_SYMBOL_GPL(drm_gem_shmem_create);
>  
> +static void
> +drm_gem_shmem_free_pages(struct drm_gem_shmem_object *shmem)
> +{
> +	struct drm_gem_object *obj = &shmem->base;
> +
> +#ifdef CONFIG_X86
> +	if (shmem->map_wc)
> +		set_pages_array_wb(shmem->pages, obj->size >> PAGE_SHIFT);
> +#endif
> +
> +	drm_gem_put_pages(obj, shmem->pages,
> +			  shmem->pages_mark_dirty_on_put,
> +			  shmem->pages_mark_accessed_on_put);
> +	shmem->pages = NULL;
> +}
> +
>  /**
>   * drm_gem_shmem_free - Free resources associated with a shmem GEM object
>   * @shmem: shmem GEM object to free
> @@ -142,8 +158,6 @@ void drm_gem_shmem_free(struct drm_gem_shmem_object *shmem)
>  	if (obj->import_attach) {
>  		drm_prime_gem_destroy(obj, shmem->sgt);
>  	} else {
> -		dma_resv_lock(shmem->base.resv, NULL);
> -
>  		drm_WARN_ON(obj->dev, refcount_read(&shmem->vmap_use_count));
>  
>  		if (shmem->sgt) {
> @@ -157,8 +171,6 @@ void drm_gem_shmem_free(struct drm_gem_shmem_object *shmem)
>  

If you drop the dma_resv_lock/unlock(), you should also replace the
drm_gem_shmem_put_pages_locked() by a drm_gem_shmem_free_pages() in this
commit.

>  		drm_WARN_ON(obj->dev, refcount_read(&shmem->pages_use_count));
>  		drm_WARN_ON(obj->dev, refcount_read(&shmem->pages_pin_count));
> -
> -		dma_resv_unlock(shmem->base.resv);
>  	}
>  
>  	drm_gem_object_release(obj);
> @@ -208,21 +220,10 @@ static int drm_gem_shmem_get_pages_locked(struct drm_gem_shmem_object *shmem)
>   */
>  void drm_gem_shmem_put_pages_locked(struct drm_gem_shmem_object *shmem)
>  {
> -	struct drm_gem_object *obj = &shmem->base;
> -
>  	dma_resv_assert_held(shmem->base.resv);
>  
> -	if (refcount_dec_and_test(&shmem->pages_use_count)) {
> -#ifdef CONFIG_X86
> -		if (shmem->map_wc)
> -			set_pages_array_wb(shmem->pages, obj->size >> PAGE_SHIFT);
> -#endif
> -
> -		drm_gem_put_pages(obj, shmem->pages,
> -				  shmem->pages_mark_dirty_on_put,
> -				  shmem->pages_mark_accessed_on_put);
> -		shmem->pages = NULL;
> -	}
> +	if (refcount_dec_and_test(&shmem->pages_use_count))
> +		drm_gem_shmem_free_pages(shmem);
>  }
>  EXPORT_SYMBOL_GPL(drm_gem_shmem_put_pages_locked);
>  

Re: [PATCH v18 11/26] drm/shmem-helper: Prepare drm_gem_shmem_free() to shrinker addition

[Dmitry Osipenko]

On 11/10/23 13:16, Boris Brezillon wrote:
> On Mon, 30 Oct 2023 02:01:50 +0300
> Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:
> 
>> Prepare drm_gem_shmem_free() to addition of memory shrinker support
>> to drm-shmem by adding and using variant of put_pages() that doesn't
>> touch reservation lock. Reservation shouldn't be touched because lockdep
>> will trigger a bogus warning about locking contention with fs_reclaim
>> code paths that can't happen during the time when GEM is freed and
>> lockdep doesn't know about that.
>>
>> Signed-off-by: Dmitry Osipenko <dmitry.osipenko@collabora.com>
>> ---
>>  drivers/gpu/drm/drm_gem_shmem_helper.c | 35 +++++++++++++-------------
>>  1 file changed, 18 insertions(+), 17 deletions(-)
>>
>> diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c
>> index 08b5a57c59d8..24ff2b99e75b 100644
>> --- a/drivers/gpu/drm/drm_gem_shmem_helper.c
>> +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c
>> @@ -128,6 +128,22 @@ struct drm_gem_shmem_object *drm_gem_shmem_create(struct drm_device *dev, size_t
>>  }
>>  EXPORT_SYMBOL_GPL(drm_gem_shmem_create);
>>  
>> +static void
>> +drm_gem_shmem_free_pages(struct drm_gem_shmem_object *shmem)
>> +{
>> +	struct drm_gem_object *obj = &shmem->base;
>> +
>> +#ifdef CONFIG_X86
>> +	if (shmem->map_wc)
>> +		set_pages_array_wb(shmem->pages, obj->size >> PAGE_SHIFT);
>> +#endif
>> +
>> +	drm_gem_put_pages(obj, shmem->pages,
>> +			  shmem->pages_mark_dirty_on_put,
>> +			  shmem->pages_mark_accessed_on_put);
>> +	shmem->pages = NULL;
>> +}
>> +
>>  /**
>>   * drm_gem_shmem_free - Free resources associated with a shmem GEM object
>>   * @shmem: shmem GEM object to free
>> @@ -142,8 +158,6 @@ void drm_gem_shmem_free(struct drm_gem_shmem_object *shmem)
>>  	if (obj->import_attach) {
>>  		drm_prime_gem_destroy(obj, shmem->sgt);
>>  	} else {
>> -		dma_resv_lock(shmem->base.resv, NULL);
>> -
>>  		drm_WARN_ON(obj->dev, refcount_read(&shmem->vmap_use_count));
>>  
>>  		if (shmem->sgt) {
>> @@ -157,8 +171,6 @@ void drm_gem_shmem_free(struct drm_gem_shmem_object *shmem)
>>  
> If you drop the dma_resv_lock/unlock(), you should also replace the
> drm_gem_shmem_put_pages_locked() by a drm_gem_shmem_free_pages() in this
> commit.

drm_gem_shmem_put_pages_locked() is exported by a later patch of this
series, it's not worthwhile to remove this function

-- 
Best regards,
Dmitry

Re: [PATCH v18 11/26] drm/shmem-helper: Prepare drm_gem_shmem_free() to shrinker addition

[Boris Brezillon]

On Mon, 20 Nov 2023 14:02:29 +0300
Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:

> On 11/10/23 13:16, Boris Brezillon wrote:
> > On Mon, 30 Oct 2023 02:01:50 +0300
> > Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:
> >   
> >> Prepare drm_gem_shmem_free() to addition of memory shrinker support
> >> to drm-shmem by adding and using variant of put_pages() that doesn't
> >> touch reservation lock. Reservation shouldn't be touched because lockdep
> >> will trigger a bogus warning about locking contention with fs_reclaim
> >> code paths that can't happen during the time when GEM is freed and
> >> lockdep doesn't know about that.
> >>
> >> Signed-off-by: Dmitry Osipenko <dmitry.osipenko@collabora.com>
> >> ---
> >>  drivers/gpu/drm/drm_gem_shmem_helper.c | 35 +++++++++++++-------------
> >>  1 file changed, 18 insertions(+), 17 deletions(-)
> >>
> >> diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c
> >> index 08b5a57c59d8..24ff2b99e75b 100644
> >> --- a/drivers/gpu/drm/drm_gem_shmem_helper.c
> >> +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c
> >> @@ -128,6 +128,22 @@ struct drm_gem_shmem_object *drm_gem_shmem_create(struct drm_device *dev, size_t
> >>  }
> >>  EXPORT_SYMBOL_GPL(drm_gem_shmem_create);
> >>  
> >> +static void
> >> +drm_gem_shmem_free_pages(struct drm_gem_shmem_object *shmem)
> >> +{
> >> +	struct drm_gem_object *obj = &shmem->base;
> >> +
> >> +#ifdef CONFIG_X86
> >> +	if (shmem->map_wc)
> >> +		set_pages_array_wb(shmem->pages, obj->size >> PAGE_SHIFT);
> >> +#endif
> >> +
> >> +	drm_gem_put_pages(obj, shmem->pages,
> >> +			  shmem->pages_mark_dirty_on_put,
> >> +			  shmem->pages_mark_accessed_on_put);
> >> +	shmem->pages = NULL;
> >> +}
> >> +
> >>  /**
> >>   * drm_gem_shmem_free - Free resources associated with a shmem GEM object
> >>   * @shmem: shmem GEM object to free
> >> @@ -142,8 +158,6 @@ void drm_gem_shmem_free(struct drm_gem_shmem_object *shmem)
> >>  	if (obj->import_attach) {
> >>  		drm_prime_gem_destroy(obj, shmem->sgt);
> >>  	} else {
> >> -		dma_resv_lock(shmem->base.resv, NULL);
> >> -
> >>  		drm_WARN_ON(obj->dev, refcount_read(&shmem->vmap_use_count));
> >>  
> >>  		if (shmem->sgt) {
> >> @@ -157,8 +171,6 @@ void drm_gem_shmem_free(struct drm_gem_shmem_object *shmem)
> >>    
> > If you drop the dma_resv_lock/unlock(), you should also replace the
> > drm_gem_shmem_put_pages_locked() by a drm_gem_shmem_free_pages() in this
> > commit.  
> 
> drm_gem_shmem_put_pages_locked() is exported by a later patch of this
> series, it's not worthwhile to remove this function

I'm not talking about removing drm_gem_shmem_put_pages_locked(), but
replacing the drm_gem_shmem_put_pages_locked() call you have in
drm_gem_shmem_free() by a drm_gem_shmem_free_pages(), so you don't end
up with a lockdep warning when you stop exactly here in the patch
series, which is important if we want to keep things bisectable.

Re: [PATCH v18 11/26] drm/shmem-helper: Prepare drm_gem_shmem_free() to shrinker addition

[Dmitry Osipenko]

On 11/20/23 14:19, Boris Brezillon wrote:
...
>>>> -		dma_resv_lock(shmem->base.resv, NULL);
>>>> -
>>>>  		drm_WARN_ON(obj->dev, refcount_read(&shmem->vmap_use_count));
>>>>  
>>>>  		if (shmem->sgt) {
>>>> @@ -157,8 +171,6 @@ void drm_gem_shmem_free(struct drm_gem_shmem_object *shmem)
>>>>    
>>> If you drop the dma_resv_lock/unlock(), you should also replace the
>>> drm_gem_shmem_put_pages_locked() by a drm_gem_shmem_free_pages() in this
>>> commit.  
>>
>> drm_gem_shmem_put_pages_locked() is exported by a later patch of this
>> series, it's not worthwhile to remove this function
> 
> I'm not talking about removing drm_gem_shmem_put_pages_locked(), but
> replacing the drm_gem_shmem_put_pages_locked() call you have in
> drm_gem_shmem_free() by a drm_gem_shmem_free_pages(), so you don't end
> up with a lockdep warning when you stop exactly here in the patch
> series, which is important if we want to keep things bisectable.

Indeed, there is assert_locked() there. Thanks for the clarification :)

-- 
Best regards,
Dmitry

Re: [PATCH v18 12/26] drm/shmem-helper: Make drm_gem_shmem_get_pages() public

[Boris Brezillon]

On Mon, 30 Oct 2023 02:01:51 +0300
Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:

> We're going to move away from having implicit get_pages() done by
> get_pages_sgt() to ease simplify refcnt handling. Drivers will manage

Nit: ease or simplify, not both.

> get/put_pages() by themselves. Expose the drm_gem_shmem_get_pages()
> in a public drm-shmem API.
> 
> Signed-off-by: Dmitry Osipenko <dmitry.osipenko@collabora.com>
> ---
>  drivers/gpu/drm/drm_gem_shmem_helper.c | 10 +++++++++-
>  include/drm/drm_gem_shmem_helper.h     |  1 +
>  2 files changed, 10 insertions(+), 1 deletion(-)
> 
> diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c
> index 24ff2b99e75b..ca6f422c0dfc 100644
> --- a/drivers/gpu/drm/drm_gem_shmem_helper.c
> +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c
> @@ -227,7 +227,14 @@ void drm_gem_shmem_put_pages_locked(struct drm_gem_shmem_object *shmem)
>  }
>  EXPORT_SYMBOL_GPL(drm_gem_shmem_put_pages_locked);
>  
> -static int drm_gem_shmem_get_pages(struct drm_gem_shmem_object *shmem)
> +/*
> + * drm_gem_shmem_get_pages - Increase use count on the backing pages for a shmem GEM object
> + * @shmem: shmem GEM object
> + *
> + * This function Increases the use count and allocates the backing pages if
> + * use-count equals to zero.
> + */
> +int drm_gem_shmem_get_pages(struct drm_gem_shmem_object *shmem)
>  {
>  	int ret;
>  
> @@ -240,6 +247,7 @@ static int drm_gem_shmem_get_pages(struct drm_gem_shmem_object *shmem)
>  
>  	return ret;
>  }
> +EXPORT_SYMBOL_GPL(drm_gem_shmem_get_pages);
>  
>  static int drm_gem_shmem_pin_locked(struct drm_gem_shmem_object *shmem)
>  {
> diff --git a/include/drm/drm_gem_shmem_helper.h b/include/drm/drm_gem_shmem_helper.h
> index e7b3f4c02bf5..45cd293e10a4 100644
> --- a/include/drm/drm_gem_shmem_helper.h
> +++ b/include/drm/drm_gem_shmem_helper.h
> @@ -110,6 +110,7 @@ struct drm_gem_shmem_object {
>  struct drm_gem_shmem_object *drm_gem_shmem_create(struct drm_device *dev, size_t size);
>  void drm_gem_shmem_free(struct drm_gem_shmem_object *shmem);
>  
> +int drm_gem_shmem_get_pages(struct drm_gem_shmem_object *shmem);
>  void drm_gem_shmem_put_pages_locked(struct drm_gem_shmem_object *shmem);
>  int drm_gem_shmem_pin(struct drm_gem_shmem_object *shmem);
>  void drm_gem_shmem_unpin(struct drm_gem_shmem_object *shmem);

Re: [PATCH v18 14/26] drm/lima: Explicitly get and put drm-shmem pages

[Boris Brezillon]

On Mon, 30 Oct 2023 02:01:53 +0300
Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:

> To simplify the drm-shmem refcnt handling, we're moving away from
> the implicit get_pages() that is used by get_pages_sgt(). From now on
> drivers will have to pin pages while they use sgt. Lima driver doesn't
> have shrinker, hence pages are pinned and sgt is valid as long as pages'
> use-count > 0.
> 
> Signed-off-by: Dmitry Osipenko <dmitry.osipenko@collabora.com>
> ---
>  drivers/gpu/drm/lima/lima_gem.c | 18 ++++++++++++++++--
>  drivers/gpu/drm/lima/lima_gem.h |  1 +
>  2 files changed, 17 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/gpu/drm/lima/lima_gem.c b/drivers/gpu/drm/lima/lima_gem.c
> index 988e74f67465..d255f5775dac 100644
> --- a/drivers/gpu/drm/lima/lima_gem.c
> +++ b/drivers/gpu/drm/lima/lima_gem.c
> @@ -46,6 +46,7 @@ int lima_heap_alloc(struct lima_bo *bo, struct lima_vm *vm)
>  			return -ENOMEM;
>  		}
>  
> +		bo->put_pages = true;
>  		bo->base.pages = pages;
>  		refcount_set(&bo->base.pages_use_count, 1);
>  
> @@ -115,6 +116,7 @@ int lima_gem_create_handle(struct drm_device *dev, struct drm_file *file,
>  		return PTR_ERR(shmem);
>  
>  	obj = &shmem->base;
> +	bo = to_lima_bo(obj);
>  
>  	/* Mali Utgard GPU can only support 32bit address space */
>  	mask = mapping_gfp_mask(obj->filp->f_mapping);
> @@ -123,13 +125,19 @@ int lima_gem_create_handle(struct drm_device *dev, struct drm_file *file,
>  	mapping_set_gfp_mask(obj->filp->f_mapping, mask);
>  
>  	if (is_heap) {
> -		bo = to_lima_bo(obj);
>  		err = lima_heap_alloc(bo, NULL);
>  		if (err)
>  			goto out;
>  	} else {
> -		struct sg_table *sgt = drm_gem_shmem_get_pages_sgt(shmem);
> +		struct sg_table *sgt;
> +
> +		err = drm_gem_shmem_get_pages(shmem);
> +		if (err)
> +			goto out;
> +
> +		bo->put_pages = true;
>  
> +		sgt = drm_gem_shmem_get_pages_sgt(shmem);
>  		if (IS_ERR(sgt)) {
>  			err = PTR_ERR(sgt);
>  			goto out;

Pretty sure we don't need this put_pages flag. We can either check
ba->base.base.pages or refcount_read(&bo->base.pages_use_count). Or,
even better, if it's just used in the error path of the same function,
simply have a dedicated error path for that case:

	drm_gem_object_put(obj);
	return 0;

err_put_pages:
	if (!is_heap)
		drm_gem_shmem_put_pages(shmem);

err_put_bo:
	drm_gem_object_put(obj);
	return err;
}

> @@ -139,6 +147,9 @@ int lima_gem_create_handle(struct drm_device *dev, struct drm_file *file,
>  	err = drm_gem_handle_create(file, obj, handle);
>  
>  out:
> +	if (err && bo->put_pages)
> +		drm_gem_shmem_put_pages(shmem);
> +
>  	/* drop reference from allocate - handle holds it now */
>  	drm_gem_object_put(obj);
>  
> @@ -152,6 +163,9 @@ static void lima_gem_free_object(struct drm_gem_object *obj)
>  	if (!list_empty(&bo->va))
>  		dev_err(obj->dev->dev, "lima gem free bo still has va\n");
>  
> +	if (bo->put_pages)
> +		drm_gem_shmem_put_pages(&bo->base);

This one can be replaced by

	if (!is_heap || bo->base.base.pages)
		drm_gem_shmem_put_pages(&bo->base);

> +
>  	drm_gem_shmem_free(&bo->base);
>  }
>  
> diff --git a/drivers/gpu/drm/lima/lima_gem.h b/drivers/gpu/drm/lima/lima_gem.h
> index ccea06142f4b..dc5a6d465c80 100644
> --- a/drivers/gpu/drm/lima/lima_gem.h
> +++ b/drivers/gpu/drm/lima/lima_gem.h
> @@ -16,6 +16,7 @@ struct lima_bo {
>  	struct list_head va;
>  
>  	size_t heap_size;
> +	bool put_pages;
>  };
>  
>  static inline struct lima_bo *

Re: [PATCH v18 16/26] drm/virtio: Explicitly get and put drm-shmem pages

[Boris Brezillon]

On Mon, 30 Oct 2023 02:01:55 +0300
Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:

> We're moving away from implicit get_pages() that is done by
> get_pages_sgt() to simplify the refcnt handling. Drivers will have
> to pin pages while they use sgt. VirtIO-GPU doesn't support shrinker,
> hence pages are pinned and sgt is valid as long as pages' use-count > 0.
> 
> Signed-off-by: Dmitry Osipenko <dmitry.osipenko@collabora.com>

Reviewed-by: Boris Brezillon <boris.brezillon@collabora.com>

> ---
>  drivers/gpu/drm/virtio/virtgpu_object.c | 9 ++++++++-
>  1 file changed, 8 insertions(+), 1 deletion(-)
> 
> diff --git a/drivers/gpu/drm/virtio/virtgpu_object.c b/drivers/gpu/drm/virtio/virtgpu_object.c
> index ee5d2a70656b..998f8b05ceb1 100644
> --- a/drivers/gpu/drm/virtio/virtgpu_object.c
> +++ b/drivers/gpu/drm/virtio/virtgpu_object.c
> @@ -67,6 +67,7 @@ void virtio_gpu_cleanup_object(struct virtio_gpu_object *bo)
>  
>  	virtio_gpu_resource_id_put(vgdev, bo->hw_res_handle);
>  	if (virtio_gpu_is_shmem(bo)) {
> +		drm_gem_shmem_put_pages(&bo->base);
>  		drm_gem_shmem_free(&bo->base);
>  	} else if (virtio_gpu_is_vram(bo)) {
>  		struct virtio_gpu_object_vram *vram = to_virtio_gpu_vram(bo);
> @@ -196,9 +197,13 @@ int virtio_gpu_object_create(struct virtio_gpu_device *vgdev,
>  		return PTR_ERR(shmem_obj);
>  	bo = gem_to_virtio_gpu_obj(&shmem_obj->base);
>  
> +	ret = drm_gem_shmem_get_pages(shmem_obj);
> +	if (ret)
> +		goto err_free_gem;
> +
>  	ret = virtio_gpu_resource_id_get(vgdev, &bo->hw_res_handle);
>  	if (ret < 0)
> -		goto err_free_gem;
> +		goto err_put_pages;
>  
>  	bo->dumb = params->dumb;
>  
> @@ -243,6 +248,8 @@ int virtio_gpu_object_create(struct virtio_gpu_device *vgdev,
>  	kvfree(ents);
>  err_put_id:
>  	virtio_gpu_resource_id_put(vgdev, bo->hw_res_handle);
> +err_put_pages:
> +	drm_gem_shmem_put_pages(shmem_obj);
>  err_free_gem:
>  	drm_gem_shmem_free(shmem_obj);
>  	return ret;

Re: [PATCH v18 17/26] drm/v3d: Explicitly get and put drm-shmem pages

[Boris Brezillon]

On Mon, 30 Oct 2023 02:01:56 +0300
Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:

> To simplify the drm-shmem refcnt handling, we're moving away from
> the implicit get_pages() that is used by get_pages_sgt(). From now on
> drivers will have to pin pages while they use sgt. V3D driver doesn't
> support shrinker, hence pages are pinned and sgt is valid as long as
> pages' use-count > 0.
> 
> Signed-off-by: Dmitry Osipenko <dmitry.osipenko@collabora.com>

Reviewed-by: Boris Brezillon <boris.brezillon@collabora.com>

> ---
>  drivers/gpu/drm/v3d/v3d_bo.c | 11 ++++++++++-
>  1 file changed, 10 insertions(+), 1 deletion(-)
> 
> diff --git a/drivers/gpu/drm/v3d/v3d_bo.c b/drivers/gpu/drm/v3d/v3d_bo.c
> index 42cd874f6810..0597c6b01b6c 100644
> --- a/drivers/gpu/drm/v3d/v3d_bo.c
> +++ b/drivers/gpu/drm/v3d/v3d_bo.c
> @@ -47,6 +47,9 @@ void v3d_free_object(struct drm_gem_object *obj)
>  	/* GPU execution may have dirtied any pages in the BO. */
>  	bo->base.pages_mark_dirty_on_put = true;
>  
> +	if (!obj->import_attach)
> +		drm_gem_shmem_put_pages(&bo->base);
> +
>  	drm_gem_shmem_free(&bo->base);
>  }
>  
> @@ -135,12 +138,18 @@ struct v3d_bo *v3d_bo_create(struct drm_device *dev, struct drm_file *file_priv,
>  		return ERR_CAST(shmem_obj);
>  	bo = to_v3d_bo(&shmem_obj->base);
>  
> -	ret = v3d_bo_create_finish(&shmem_obj->base);
> +	ret = drm_gem_shmem_get_pages(shmem_obj);
>  	if (ret)
>  		goto free_obj;
>  
> +	ret = v3d_bo_create_finish(&shmem_obj->base);
> +	if (ret)
> +		goto put_pages;
> +
>  	return bo;
>  
> +put_pages:
> +	drm_gem_shmem_put_pages(shmem_obj);
>  free_obj:
>  	drm_gem_shmem_free(shmem_obj);
>  	return ERR_PTR(ret);

Re: [PATCH v18 18/26] drm/shmem-helper: Change sgt allocation policy

[Boris Brezillon]

On Mon, 30 Oct 2023 02:01:57 +0300
Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:

> In a preparation to addition of drm-shmem memory shrinker support, change
> the SGT allocation policy in this way:
> 
> 1. SGT can be allocated only if shmem pages are pinned at the
> time of allocation, otherwise allocation fails.
> 
> 2. Drivers must ensure that pages are pinned during the time of SGT usage
> and should get new SGT if pages were unpinned.

In general, I would discourage drivers from caching the sgt returned by
drm_gem_shmem_get_pages_sgt[_locked](), since the GEM SHMEM layer does
the caching already, so calling drm_gem_shmem_get_pages_sgt_locked()
should be pretty cheap. What this implies is that any portion of the
code using an sgt returned by drm_gem_shmem_get_pages_sgt_locked() must
be surrounded by get/pin_pages()/put/unpin_pages() calls unless the
pages are known to be pinned for the whole BO lifetime. And of course,
as soon as an MMU mapping is created, and even if the sgt is no longer
accessed, the pages must remain pinned until the MMU mapping is torn
down.

> 
> This new policy is required by the shrinker because it will move pages
> to/from SWAP unless pages are pinned, invalidating SGT pointer once pages
> are relocated.
> 
> Previous patches prepared drivers to the new policy.
> 
> Signed-off-by: Dmitry Osipenko <dmitry.osipenko@collabora.com>

Reviewed-by: Boris Brezillon <boris.brezillon@collabora.com>

> ---
>  drivers/gpu/drm/drm_gem_shmem_helper.c | 51 +++++++++++++-------------
>  1 file changed, 26 insertions(+), 25 deletions(-)
> 
> diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c
> index f371ebc6f85c..1420d2166b76 100644
> --- a/drivers/gpu/drm/drm_gem_shmem_helper.c
> +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c
> @@ -133,6 +133,14 @@ drm_gem_shmem_free_pages(struct drm_gem_shmem_object *shmem)
>  {
>  	struct drm_gem_object *obj = &shmem->base;
>  
> +	if (shmem->sgt) {
> +		dma_unmap_sgtable(obj->dev->dev, shmem->sgt,
> +				  DMA_BIDIRECTIONAL, 0);
> +		sg_free_table(shmem->sgt);
> +		kfree(shmem->sgt);
> +		shmem->sgt = NULL;
> +	}
> +
>  #ifdef CONFIG_X86
>  	if (shmem->map_wc)
>  		set_pages_array_wb(shmem->pages, obj->size >> PAGE_SHIFT);
> @@ -155,23 +163,12 @@ void drm_gem_shmem_free(struct drm_gem_shmem_object *shmem)
>  {
>  	struct drm_gem_object *obj = &shmem->base;
>  
> -	if (obj->import_attach) {
> +	if (obj->import_attach)
>  		drm_prime_gem_destroy(obj, shmem->sgt);
> -	} else {
> -		drm_WARN_ON(obj->dev, refcount_read(&shmem->vmap_use_count));
> -
> -		if (shmem->sgt) {
> -			dma_unmap_sgtable(obj->dev->dev, shmem->sgt,
> -					  DMA_BIDIRECTIONAL, 0);
> -			sg_free_table(shmem->sgt);
> -			kfree(shmem->sgt);
> -		}
> -		if (shmem->pages)
> -			drm_gem_shmem_put_pages_locked(shmem);
>  
> -		drm_WARN_ON(obj->dev, refcount_read(&shmem->pages_use_count));
> -		drm_WARN_ON(obj->dev, refcount_read(&shmem->pages_pin_count));
> -	}
> +	drm_WARN_ON(obj->dev, refcount_read(&shmem->vmap_use_count));
> +	drm_WARN_ON(obj->dev, refcount_read(&shmem->pages_use_count));
> +	drm_WARN_ON(obj->dev, refcount_read(&shmem->pages_pin_count));
>  
>  	drm_gem_object_release(obj);
>  	kfree(shmem);
> @@ -705,6 +702,9 @@ struct sg_table *drm_gem_shmem_get_sg_table(struct drm_gem_shmem_object *shmem)
>  
>  	drm_WARN_ON(obj->dev, obj->import_attach);
>  
> +	if (drm_WARN_ON(obj->dev, !shmem->pages))
> +		return ERR_PTR(-ENOMEM);
> +
>  	return drm_prime_pages_to_sg(obj->dev, shmem->pages, obj->size >> PAGE_SHIFT);
>  }
>  EXPORT_SYMBOL_GPL(drm_gem_shmem_get_sg_table);
> @@ -720,15 +720,10 @@ static struct sg_table *drm_gem_shmem_get_pages_sgt_locked(struct drm_gem_shmem_
>  
>  	drm_WARN_ON(obj->dev, obj->import_attach);
>  
> -	ret = drm_gem_shmem_get_pages_locked(shmem);
> -	if (ret)
> -		return ERR_PTR(ret);
> -
>  	sgt = drm_gem_shmem_get_sg_table(shmem);
> -	if (IS_ERR(sgt)) {
> -		ret = PTR_ERR(sgt);
> -		goto err_put_pages;
> -	}
> +	if (IS_ERR(sgt))
> +		return sgt;
> +
>  	/* Map the pages for use by the h/w. */
>  	ret = dma_map_sgtable(obj->dev->dev, sgt, DMA_BIDIRECTIONAL, 0);
>  	if (ret)
> @@ -741,8 +736,6 @@ static struct sg_table *drm_gem_shmem_get_pages_sgt_locked(struct drm_gem_shmem_
>  err_free_sgt:
>  	sg_free_table(sgt);
>  	kfree(sgt);
> -err_put_pages:
> -	drm_gem_shmem_put_pages_locked(shmem);
>  	return ERR_PTR(ret);
>  }
>  
> @@ -759,6 +752,14 @@ static struct sg_table *drm_gem_shmem_get_pages_sgt_locked(struct drm_gem_shmem_
>   * and difference between dma-buf imported and natively allocated objects.
>   * drm_gem_shmem_get_sg_table() should not be directly called by drivers.
>   *
> + * Drivers should adhere to these SGT usage rules:
> + *
> + * 1. SGT should be allocated only if shmem pages are pinned at the
> + *    time of allocation, otherwise allocation will fail.
> + *
> + * 2. Drivers should ensure that pages are pinned during the time of
> + *    SGT usage and should get new SGT if pages were unpinned.
> + *
>   * Returns:
>   * A pointer to the scatter/gather table of pinned pages or errno on failure.
>   */

Re: [PATCH v18 19/26] drm/shmem-helper: Add common memory shrinker

[Boris Brezillon]

On Mon, 30 Oct 2023 02:01:58 +0300
Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:

> @@ -238,6 +308,20 @@ void drm_gem_shmem_put_pages(struct drm_gem_shmem_object *shmem)
>  	if (refcount_dec_not_one(&shmem->pages_use_count))
>  		return;
>  
> +	/*
> +	 * Destroying the object is a special case because acquiring
> +	 * the obj lock can cause a locking order inversion between
> +	 * reservation_ww_class_mutex and fs_reclaim.
> +	 *
> +	 * This deadlock is not actually possible, because no one should
> +	 * be already holding the lock when GEM is released.  Unfortunately
> +	 * lockdep is not aware of this detail.  So when the refcount drops
> +	 * to zero, we pretend it is already locked.
> +	 */
> +	if (!kref_read(&shmem->base.refcount) &&
> +	    refcount_dec_and_test(&shmem->pages_use_count))
> +		return drm_gem_shmem_free_pages(shmem);

Uh, with get/put_pages() being moved to the create/free_gem()
hooks, we're back to a situation where pages_use_count > 0 when we
reach gem->refcount == 0, which is not nice. We really need to patch
drivers so they dissociate GEM creation from the backing storage
allocation/reservation + mapping of the BO in GPU VM space.

Re: [PATCH v18 19/26] drm/shmem-helper: Add common memory shrinker

[Boris Brezillon]

On Fri, 10 Nov 2023 15:58:58 +0100
Boris Brezillon <boris.brezillon@collabora.com> wrote:

> On Mon, 30 Oct 2023 02:01:58 +0300
> Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:
> 
> > @@ -238,6 +308,20 @@ void drm_gem_shmem_put_pages(struct drm_gem_shmem_object *shmem)
> >  	if (refcount_dec_not_one(&shmem->pages_use_count))
> >  		return;
> >  
> > +	/*
> > +	 * Destroying the object is a special case because acquiring
> > +	 * the obj lock can cause a locking order inversion between
> > +	 * reservation_ww_class_mutex and fs_reclaim.
> > +	 *
> > +	 * This deadlock is not actually possible, because no one should
> > +	 * be already holding the lock when GEM is released.  Unfortunately
> > +	 * lockdep is not aware of this detail.  So when the refcount drops
> > +	 * to zero, we pretend it is already locked.
> > +	 */
> > +	if (!kref_read(&shmem->base.refcount) &&
> > +	    refcount_dec_and_test(&shmem->pages_use_count))
> > +		return drm_gem_shmem_free_pages(shmem);  
> 
> Uh, with get/put_pages() being moved to the create/free_gem()
> hooks, we're back to a situation where pages_use_count > 0 when we
> reach gem->refcount == 0, which is not nice. We really need to patch
> drivers so they dissociate GEM creation from the backing storage
> allocation/reservation + mapping of the BO in GPU VM space.

I gave this a try, and I think it'd work fine for lima (I have patches
if you want) and panfrost (I have patches for panfrost too, basically
addressing my comments on patch 15). It's a bit trickier for v3d, but
still possible (we'd have to add a gpu_map_count to v3d_bo, get it
incremented for each BO referenced by a job when the job is created,
decremented for each BO referenced by a job when the job is destroyed,
and have a _put_pages() call when gpu_map_count reached zero). The main
problem is virtio, where BO backing storage lifetime is really tied to
the BO object lifetime, and my virtio knowledge is too limited to come
up with an solution to dissociate those 2 things.

TLDR; let's just keep this hack for now. This being said, there are 2
things that I think should be addressed:

1/ we want this change in patch 18 ("drm/shmem-helper: Change sgt
allocation policy") to keep this series bisectable, otherwise you'll
hit the WARN_ON(shmem->pages_use_count != 0) in drm_gem_shmem_free()

2/ we should do:

	if (!kref_read(&shmem->base.refcount) {
		if (refcount_dec_and_test(&shmem->pages_use_count))
			drm_gem_shmem_free_pages(shmem);

		return;
	}

  if we don't want to try to acquire the resv lock when the user leaked
  a pages ref (the core will complain about this leak in
  drm_gem_shmem_free() anyway)

Regards,

Boris

Re: [PATCH v18 21/26] drm/shmem-helper: Optimize unlocked get_pages_sgt()

[Boris Brezillon]

On Mon, 30 Oct 2023 02:02:00 +0300
Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:

> SGT isn't refcounted. Once SGT pointer has been obtained, it remains the
> same for both locked and unlocked get_pages_sgt(). Return cached SGT
> directly without taking a potentially expensive lock.
> 
> Signed-off-by: Dmitry Osipenko <dmitry.osipenko@collabora.com>
> ---
>  drivers/gpu/drm/drm_gem_shmem_helper.c | 3 +++
>  1 file changed, 3 insertions(+)
> 
> diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c
> index 560ce565f376..6dd087f19ea3 100644
> --- a/drivers/gpu/drm/drm_gem_shmem_helper.c
> +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c
> @@ -955,6 +955,9 @@ struct sg_table *drm_gem_shmem_get_pages_sgt(struct drm_gem_shmem_object *shmem)
>  	drm_WARN_ON(obj->dev, drm_gem_shmem_is_evictable(shmem));
>  	drm_WARN_ON(obj->dev, drm_gem_shmem_is_purgeable(shmem));
>  
> +	if (shmem->sgt)
> +		return shmem->sgt;
> +

By doing that, you introduce a race between drm_gem_shmem_put_pages()
and drm_gem_shmem_get_pages_sgt(). I know it's only happening if the
user doesn't follow the rule listed in drm_gem_shmem_get_pages_sgt() doc
(caller must owns a pages ref before calling
drm_gem_shmem_get_pages_sgt()), but now you return a possibly invalid
pointer instead of returning an error when this happens.
If that's what we want, we should at least update the documentation to
make it clear.

>  	ret = dma_resv_lock_interruptible(shmem->base.resv, NULL);
>  	if (ret)
>  		return ERR_PTR(ret);

Re: [PATCH v18 22/26] drm/shmem-helper: Don't free refcounted GEM

[Boris Brezillon]

On Mon, 30 Oct 2023 02:02:01 +0300
Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:

> Don't free refcounted shmem object to prevent use-after-free bug that
> is worse than a memory leak.
> 
> Signed-off-by: Dmitry Osipenko <dmitry.osipenko@collabora.com>
> ---
>  drivers/gpu/drm/drm_gem_shmem_helper.c | 7 ++++---
>  1 file changed, 4 insertions(+), 3 deletions(-)
> 
> diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c
> index 6dd087f19ea3..4253c367dc07 100644
> --- a/drivers/gpu/drm/drm_gem_shmem_helper.c
> +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c
> @@ -203,9 +203,10 @@ void drm_gem_shmem_free(struct drm_gem_shmem_object *shmem)
>  	if (obj->import_attach)
>  		drm_prime_gem_destroy(obj, shmem->sgt);
>  
> -	drm_WARN_ON(obj->dev, refcount_read(&shmem->vmap_use_count));
> -	drm_WARN_ON(obj->dev, refcount_read(&shmem->pages_use_count));
> -	drm_WARN_ON(obj->dev, refcount_read(&shmem->pages_pin_count));
> +	if (drm_WARN_ON(obj->dev, refcount_read(&shmem->vmap_use_count)) ||
> +	    drm_WARN_ON(obj->dev, refcount_read(&shmem->pages_use_count)) ||
> +	    drm_WARN_ON(obj->dev, refcount_read(&shmem->pages_pin_count)))
> +		return;

I guess you're worried about ->sgt being referenced by the driver after
the GEM is destroyed. If we assume drivers don't cache the sgt and
always call get_pages_sgt() when they need it that shouldn't be an
issue. What we really don't want to release is the pages themselves,
but the GPU MMU might still have active mappings pointing to these
pages.

In any case, I'm not against leaking the GEM object when any of these
counters are not zero, but can we at least have a comment in the
code explaining why we're doing that, so people don't have to go look
at the git history to figure it out.

>  
>  	drm_gem_object_release(obj);
>  	kfree(shmem);

Re: [PATCH v18 22/26] drm/shmem-helper: Don't free refcounted GEM

[Dmitry Osipenko]

On 11/13/23 12:54, Boris Brezillon wrote:
> On Mon, 30 Oct 2023 02:02:01 +0300
> Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:
> 
>> Don't free refcounted shmem object to prevent use-after-free bug that
>> is worse than a memory leak.
>>
>> Signed-off-by: Dmitry Osipenko <dmitry.osipenko@collabora.com>
>> ---
>>  drivers/gpu/drm/drm_gem_shmem_helper.c | 7 ++++---
>>  1 file changed, 4 insertions(+), 3 deletions(-)
>>
>> diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c
>> index 6dd087f19ea3..4253c367dc07 100644
>> --- a/drivers/gpu/drm/drm_gem_shmem_helper.c
>> +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c
>> @@ -203,9 +203,10 @@ void drm_gem_shmem_free(struct drm_gem_shmem_object *shmem)
>>  	if (obj->import_attach)
>>  		drm_prime_gem_destroy(obj, shmem->sgt);
>>  
>> -	drm_WARN_ON(obj->dev, refcount_read(&shmem->vmap_use_count));
>> -	drm_WARN_ON(obj->dev, refcount_read(&shmem->pages_use_count));
>> -	drm_WARN_ON(obj->dev, refcount_read(&shmem->pages_pin_count));
>> +	if (drm_WARN_ON(obj->dev, refcount_read(&shmem->vmap_use_count)) ||
>> +	    drm_WARN_ON(obj->dev, refcount_read(&shmem->pages_use_count)) ||
>> +	    drm_WARN_ON(obj->dev, refcount_read(&shmem->pages_pin_count)))
>> +		return;
> 
> I guess you're worried about ->sgt being referenced by the driver after
> the GEM is destroyed. If we assume drivers don't cache the sgt and
> always call get_pages_sgt() when they need it that shouldn't be an
> issue. What we really don't want to release is the pages themselves,
> but the GPU MMU might still have active mappings pointing to these
> pages.
> 
> In any case, I'm not against leaking the GEM object when any of these
> counters are not zero, but can we at least have a comment in the
> code explaining why we're doing that, so people don't have to go look
> at the git history to figure it out.

This patch is a minor improvement, it doesn't address any specific
issue. This should be a common pattern in kernel. If you're giving a
warning and know about the inevitable catastrophe, then avoid it if you can.

Actually, there are other similar cases in drm-shmem that can be improved.

-- 
Best regards,
Dmitry

Re: [PATCH v18 22/26] drm/shmem-helper: Don't free refcounted GEM

[Boris Brezillon]

On Thu, 23 Nov 2023 01:30:24 +0300
Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:

> On 11/13/23 12:54, Boris Brezillon wrote:
> > On Mon, 30 Oct 2023 02:02:01 +0300
> > Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:
> >   
> >> Don't free refcounted shmem object to prevent use-after-free bug that
> >> is worse than a memory leak.
> >>
> >> Signed-off-by: Dmitry Osipenko <dmitry.osipenko@collabora.com>
> >> ---
> >>  drivers/gpu/drm/drm_gem_shmem_helper.c | 7 ++++---
> >>  1 file changed, 4 insertions(+), 3 deletions(-)
> >>
> >> diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c
> >> index 6dd087f19ea3..4253c367dc07 100644
> >> --- a/drivers/gpu/drm/drm_gem_shmem_helper.c
> >> +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c
> >> @@ -203,9 +203,10 @@ void drm_gem_shmem_free(struct drm_gem_shmem_object *shmem)
> >>  	if (obj->import_attach)
> >>  		drm_prime_gem_destroy(obj, shmem->sgt);
> >>  
> >> -	drm_WARN_ON(obj->dev, refcount_read(&shmem->vmap_use_count));
> >> -	drm_WARN_ON(obj->dev, refcount_read(&shmem->pages_use_count));
> >> -	drm_WARN_ON(obj->dev, refcount_read(&shmem->pages_pin_count));
> >> +	if (drm_WARN_ON(obj->dev, refcount_read(&shmem->vmap_use_count)) ||
> >> +	    drm_WARN_ON(obj->dev, refcount_read(&shmem->pages_use_count)) ||
> >> +	    drm_WARN_ON(obj->dev, refcount_read(&shmem->pages_pin_count)))
> >> +		return;  
> > 
> > I guess you're worried about ->sgt being referenced by the driver after
> > the GEM is destroyed. If we assume drivers don't cache the sgt and
> > always call get_pages_sgt() when they need it that shouldn't be an
> > issue. What we really don't want to release is the pages themselves,
> > but the GPU MMU might still have active mappings pointing to these
> > pages.
> > 
> > In any case, I'm not against leaking the GEM object when any of these
> > counters are not zero, but can we at least have a comment in the
> > code explaining why we're doing that, so people don't have to go look
> > at the git history to figure it out.  
> 
> This patch is a minor improvement, it doesn't address any specific
> issue. This should be a common pattern in kernel. If you're giving a
> warning and know about the inevitable catastrophe, then avoid it if you can.

Sure, I'm just asking that we add a comment to explain why we leak
memory here. Is that too much to ask?

Re: [PATCH v18 22/26] drm/shmem-helper: Don't free refcounted GEM

[Dmitry Osipenko]

On 11/23/23 12:08, Boris Brezillon wrote:
> On Thu, 23 Nov 2023 01:30:24 +0300
> Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:
> 
>> On 11/13/23 12:54, Boris Brezillon wrote:
>>> On Mon, 30 Oct 2023 02:02:01 +0300
>>> Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:
>>>   
>>>> Don't free refcounted shmem object to prevent use-after-free bug that
>>>> is worse than a memory leak.
>>>>
>>>> Signed-off-by: Dmitry Osipenko <dmitry.osipenko@collabora.com>
>>>> ---
>>>>  drivers/gpu/drm/drm_gem_shmem_helper.c | 7 ++++---
>>>>  1 file changed, 4 insertions(+), 3 deletions(-)
>>>>
>>>> diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c
>>>> index 6dd087f19ea3..4253c367dc07 100644
>>>> --- a/drivers/gpu/drm/drm_gem_shmem_helper.c
>>>> +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c
>>>> @@ -203,9 +203,10 @@ void drm_gem_shmem_free(struct drm_gem_shmem_object *shmem)
>>>>  	if (obj->import_attach)
>>>>  		drm_prime_gem_destroy(obj, shmem->sgt);
>>>>  
>>>> -	drm_WARN_ON(obj->dev, refcount_read(&shmem->vmap_use_count));
>>>> -	drm_WARN_ON(obj->dev, refcount_read(&shmem->pages_use_count));
>>>> -	drm_WARN_ON(obj->dev, refcount_read(&shmem->pages_pin_count));
>>>> +	if (drm_WARN_ON(obj->dev, refcount_read(&shmem->vmap_use_count)) ||
>>>> +	    drm_WARN_ON(obj->dev, refcount_read(&shmem->pages_use_count)) ||
>>>> +	    drm_WARN_ON(obj->dev, refcount_read(&shmem->pages_pin_count)))
>>>> +		return;  
>>>
>>> I guess you're worried about ->sgt being referenced by the driver after
>>> the GEM is destroyed. If we assume drivers don't cache the sgt and
>>> always call get_pages_sgt() when they need it that shouldn't be an
>>> issue. What we really don't want to release is the pages themselves,
>>> but the GPU MMU might still have active mappings pointing to these
>>> pages.
>>>
>>> In any case, I'm not against leaking the GEM object when any of these
>>> counters are not zero, but can we at least have a comment in the
>>> code explaining why we're doing that, so people don't have to go look
>>> at the git history to figure it out.  
>>
>> This patch is a minor improvement, it doesn't address any specific
>> issue. This should be a common pattern in kernel. If you're giving a
>> warning and know about the inevitable catastrophe, then avoid it if you can.
> 
> Sure, I'm just asking that we add a comment to explain why we leak
> memory here. Is that too much to ask?

Will add the comment. The reason why I added this patch was unrelated to
the sgt, that's what I'm talking about.

-- 
Best regards,
Dmitry

Re: [PATCH v18 24/26] drm/virtio: Attach shmem BOs dynamically

[Boris Brezillon]

On Mon, 30 Oct 2023 02:02:03 +0300
Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:

> Prepare for addition of memory shrinker support by attaching shmem pages
> to host dynamically on first use. Previously the attachment vq command
> wasn't fenced and there was no vq kick made in the BO creation code path,
> hence the attachment already was happening dynamically, but implicitly.
> Making attachment explicitly dynamic will allow to simplify and reuse more
> code when shrinker will be added. The virtio_gpu_object_shmem_init() now
> works under the held reservation lock, which will be important to have for
> shrinker to avoid moving pages while they are in active use by the driver.

Ah, this commit might actually help getting rid of the workaround
introduced in "drm/shmem-helper: Add common memory shrinker".

> 
> Acked-by: Gerd Hoffmann <kraxel@redhat.com>
> Signed-off-by: Dmitry Osipenko <dmitry.osipenko@collabora.com>
> ---
>  drivers/gpu/drm/virtio/virtgpu_drv.h    |  7 +++
>  drivers/gpu/drm/virtio/virtgpu_gem.c    | 26 +++++++++
>  drivers/gpu/drm/virtio/virtgpu_ioctl.c  | 32 +++++++----
>  drivers/gpu/drm/virtio/virtgpu_object.c | 73 ++++++++++++++++++++-----
>  drivers/gpu/drm/virtio/virtgpu_submit.c | 15 ++++-
>  5 files changed, 125 insertions(+), 28 deletions(-)
> 
> diff --git a/drivers/gpu/drm/virtio/virtgpu_drv.h b/drivers/gpu/drm/virtio/virtgpu_drv.h
> index 56269814fb6d..421f524ae1de 100644
> --- a/drivers/gpu/drm/virtio/virtgpu_drv.h
> +++ b/drivers/gpu/drm/virtio/virtgpu_drv.h
> @@ -89,6 +89,7 @@ struct virtio_gpu_object {
>  	uint32_t hw_res_handle;
>  	bool dumb;
>  	bool created;
> +	bool detached;
>  	bool host3d_blob, guest_blob;
>  	uint32_t blob_mem, blob_flags;
>  
> @@ -313,6 +314,8 @@ void virtio_gpu_array_put_free(struct virtio_gpu_object_array *objs);
>  void virtio_gpu_array_put_free_delayed(struct virtio_gpu_device *vgdev,
>  				       struct virtio_gpu_object_array *objs);
>  void virtio_gpu_array_put_free_work(struct work_struct *work);
> +int virtio_gpu_array_prepare(struct virtio_gpu_device *vgdev,
> +			     struct virtio_gpu_object_array *objs);
>  int virtio_gpu_gem_pin(struct virtio_gpu_object *bo);
>  void virtio_gpu_gem_unpin(struct virtio_gpu_object *bo);
>  
> @@ -453,6 +456,10 @@ int virtio_gpu_object_create(struct virtio_gpu_device *vgdev,
>  
>  bool virtio_gpu_is_shmem(struct virtio_gpu_object *bo);
>  
> +int virtio_gpu_reattach_shmem_object_locked(struct virtio_gpu_object *bo);
> +
> +int virtio_gpu_reattach_shmem_object(struct virtio_gpu_object *bo);
> +
>  int virtio_gpu_resource_id_get(struct virtio_gpu_device *vgdev,
>  			       uint32_t *resid);
>  /* virtgpu_prime.c */
> diff --git a/drivers/gpu/drm/virtio/virtgpu_gem.c b/drivers/gpu/drm/virtio/virtgpu_gem.c
> index 625c05d625bf..97e67064c97e 100644
> --- a/drivers/gpu/drm/virtio/virtgpu_gem.c
> +++ b/drivers/gpu/drm/virtio/virtgpu_gem.c
> @@ -295,6 +295,26 @@ void virtio_gpu_array_put_free_work(struct work_struct *work)
>  	spin_unlock(&vgdev->obj_free_lock);
>  }
>  
> +int virtio_gpu_array_prepare(struct virtio_gpu_device *vgdev,
> +			     struct virtio_gpu_object_array *objs)
> +{
> +	struct virtio_gpu_object *bo;
> +	int ret = 0;
> +	u32 i;
> +
> +	for (i = 0; i < objs->nents; i++) {
> +		bo = gem_to_virtio_gpu_obj(objs->objs[i]);
> +
> +		if (virtio_gpu_is_shmem(bo) && bo->detached) {
> +			ret = virtio_gpu_reattach_shmem_object_locked(bo);
> +			if (ret)
> +				break;
> +		}
> +	}
> +
> +	return ret;
> +}
> +
>  int virtio_gpu_gem_pin(struct virtio_gpu_object *bo)
>  {
>  	int err;
> @@ -303,6 +323,12 @@ int virtio_gpu_gem_pin(struct virtio_gpu_object *bo)
>  		err = drm_gem_shmem_pin(&bo->base);
>  		if (err)
>  			return err;
> +
> +		err = virtio_gpu_reattach_shmem_object(bo);
> +		if (err) {
> +			drm_gem_shmem_unpin(&bo->base);
> +			return err;
> +		}
>  	}
>  
>  	return 0;
> diff --git a/drivers/gpu/drm/virtio/virtgpu_ioctl.c b/drivers/gpu/drm/virtio/virtgpu_ioctl.c
> index b24b11f25197..070c29cea26a 100644
> --- a/drivers/gpu/drm/virtio/virtgpu_ioctl.c
> +++ b/drivers/gpu/drm/virtio/virtgpu_ioctl.c
> @@ -246,6 +246,10 @@ static int virtio_gpu_transfer_from_host_ioctl(struct drm_device *dev,
>  	if (ret != 0)
>  		goto err_put_free;
>  
> +	ret = virtio_gpu_array_prepare(vgdev, objs);
> +	if (ret)
> +		goto err_unlock;
> +
>  	fence = virtio_gpu_fence_alloc(vgdev, vgdev->fence_drv.context, 0);
>  	if (!fence) {
>  		ret = -ENOMEM;
> @@ -288,11 +292,25 @@ static int virtio_gpu_transfer_to_host_ioctl(struct drm_device *dev, void *data,
>  		goto err_put_free;
>  	}
>  
> +	ret = virtio_gpu_array_lock_resv(objs);
> +	if (ret != 0)
> +		goto err_put_free;
> +
> +	ret = virtio_gpu_array_prepare(vgdev, objs);
> +	if (ret)
> +		goto err_unlock;
> +
> +	fence = virtio_gpu_fence_alloc(vgdev, vgdev->fence_drv.context, 0);
> +	if (!fence) {
> +		ret = -ENOMEM;
> +		goto err_unlock;
> +	}
> +
>  	if (!vgdev->has_virgl_3d) {
>  		virtio_gpu_cmd_transfer_to_host_2d
>  			(vgdev, offset,
>  			 args->box.w, args->box.h, args->box.x, args->box.y,
> -			 objs, NULL);
> +			 objs, fence);
>  	} else {
>  		virtio_gpu_create_context(dev, file);
>  
> @@ -301,23 +319,13 @@ static int virtio_gpu_transfer_to_host_ioctl(struct drm_device *dev, void *data,
>  			goto err_put_free;
>  		}
>  
> -		ret = virtio_gpu_array_lock_resv(objs);
> -		if (ret != 0)
> -			goto err_put_free;
> -
> -		ret = -ENOMEM;
> -		fence = virtio_gpu_fence_alloc(vgdev, vgdev->fence_drv.context,
> -					       0);
> -		if (!fence)
> -			goto err_unlock;
> -
>  		virtio_gpu_cmd_transfer_to_host_3d
>  			(vgdev,
>  			 vfpriv ? vfpriv->ctx_id : 0, offset, args->level,
>  			 args->stride, args->layer_stride, &args->box, objs,
>  			 fence);
> -		dma_fence_put(&fence->f);
>  	}
> +	dma_fence_put(&fence->f);
>  	virtio_gpu_notify(vgdev);
>  	return 0;
>  
> diff --git a/drivers/gpu/drm/virtio/virtgpu_object.c b/drivers/gpu/drm/virtio/virtgpu_object.c
> index 998f8b05ceb1..000bb7955a57 100644
> --- a/drivers/gpu/drm/virtio/virtgpu_object.c
> +++ b/drivers/gpu/drm/virtio/virtgpu_object.c
> @@ -143,7 +143,7 @@ static int virtio_gpu_object_shmem_init(struct virtio_gpu_device *vgdev,
>  	struct sg_table *pages;
>  	int si;
>  
> -	pages = drm_gem_shmem_get_pages_sgt(&bo->base);
> +	pages = drm_gem_shmem_get_pages_sgt_locked(&bo->base);
>  	if (IS_ERR(pages))
>  		return PTR_ERR(pages);
>  
> @@ -177,6 +177,40 @@ static int virtio_gpu_object_shmem_init(struct virtio_gpu_device *vgdev,
>  	return 0;
>  }
>  
> +int virtio_gpu_reattach_shmem_object_locked(struct virtio_gpu_object *bo)
> +{
> +	struct virtio_gpu_device *vgdev = bo->base.base.dev->dev_private;
> +	struct virtio_gpu_mem_entry *ents;
> +	unsigned int nents;
> +	int err;
> +
> +	if (!bo->detached)
> +		return 0;
> +
> +	err = virtio_gpu_object_shmem_init(vgdev, bo, &ents, &nents);
> +	if (err)
> +		return err;
> +
> +	virtio_gpu_object_attach(vgdev, bo, ents, nents);
> +
> +	bo->detached = false;
> +
> +	return 0;
> +}
> +
> +int virtio_gpu_reattach_shmem_object(struct virtio_gpu_object *bo)
> +{
> +	int ret;
> +
> +	ret = dma_resv_lock_interruptible(bo->base.base.resv, NULL);
> +	if (ret)
> +		return ret;
> +	ret = virtio_gpu_reattach_shmem_object_locked(bo);
> +	dma_resv_unlock(bo->base.base.resv);
> +
> +	return ret;
> +}
> +
>  int virtio_gpu_object_create(struct virtio_gpu_device *vgdev,
>  			     struct virtio_gpu_object_params *params,
>  			     struct virtio_gpu_object **bo_ptr,
> @@ -207,45 +241,56 @@ int virtio_gpu_object_create(struct virtio_gpu_device *vgdev,
>  
>  	bo->dumb = params->dumb;
>  
> -	ret = virtio_gpu_object_shmem_init(vgdev, bo, &ents, &nents);
> -	if (ret != 0)
> -		goto err_put_id;
> +	if (bo->blob_mem == VIRTGPU_BLOB_MEM_GUEST)
> +		bo->guest_blob = true;
>  
>  	if (fence) {
>  		ret = -ENOMEM;
>  		objs = virtio_gpu_array_alloc(1);
>  		if (!objs)
> -			goto err_free_entry;
> +			goto err_put_id;
>  		virtio_gpu_array_add_obj(objs, &bo->base.base);
>  
>  		ret = virtio_gpu_array_lock_resv(objs);
>  		if (ret != 0)
>  			goto err_put_objs;
> +	} else {
> +		ret = dma_resv_lock(bo->base.base.resv, NULL);
> +		if (ret)
> +			goto err_put_id;
>  	}
>  
>  	if (params->blob) {
> -		if (params->blob_mem == VIRTGPU_BLOB_MEM_GUEST)
> -			bo->guest_blob = true;
> +		ret = virtio_gpu_object_shmem_init(vgdev, bo, &ents, &nents);
> +		if (ret)
> +			goto err_unlock_objs;
> +	} else {
> +		bo->detached = true;
> +	}
>  
> +	if (params->blob)
>  		virtio_gpu_cmd_resource_create_blob(vgdev, bo, params,
>  						    ents, nents);
> -	} else if (params->virgl) {
> +	else if (params->virgl)
>  		virtio_gpu_cmd_resource_create_3d(vgdev, bo, params,
>  						  objs, fence);
> -		virtio_gpu_object_attach(vgdev, bo, ents, nents);
> -	} else {
> +	else
>  		virtio_gpu_cmd_create_resource(vgdev, bo, params,
>  					       objs, fence);
> -		virtio_gpu_object_attach(vgdev, bo, ents, nents);
> -	}
> +
> +	if (!fence)
> +		dma_resv_unlock(bo->base.base.resv);
>  
>  	*bo_ptr = bo;
>  	return 0;
>  
> +err_unlock_objs:
> +	if (fence)
> +		virtio_gpu_array_unlock_resv(objs);
> +	else
> +		dma_resv_unlock(bo->base.base.resv);
>  err_put_objs:
>  	virtio_gpu_array_put_free(objs);
> -err_free_entry:
> -	kvfree(ents);
>  err_put_id:
>  	virtio_gpu_resource_id_put(vgdev, bo->hw_res_handle);
>  err_put_pages:
> diff --git a/drivers/gpu/drm/virtio/virtgpu_submit.c b/drivers/gpu/drm/virtio/virtgpu_submit.c
> index 5c514946bbad..6e4ef2593e8f 100644
> --- a/drivers/gpu/drm/virtio/virtgpu_submit.c
> +++ b/drivers/gpu/drm/virtio/virtgpu_submit.c
> @@ -464,8 +464,19 @@ static void virtio_gpu_install_out_fence_fd(struct virtio_gpu_submit *submit)
>  
>  static int virtio_gpu_lock_buflist(struct virtio_gpu_submit *submit)
>  {
> -	if (submit->buflist)
> -		return virtio_gpu_array_lock_resv(submit->buflist);
> +	int err;
> +
> +	if (submit->buflist) {
> +		err = virtio_gpu_array_lock_resv(submit->buflist);
> +		if (err)
> +			return err;
> +
> +		err = virtio_gpu_array_prepare(submit->vgdev, submit->buflist);
> +		if (err) {
> +			virtio_gpu_array_unlock_resv(submit->buflist);
> +			return err;
> +		}
> +	}
>  
>  	return 0;
>  }

Re: [PATCH v18 24/26] drm/virtio: Attach shmem BOs dynamically

[Dmitry Osipenko]

On 11/13/23 12:57, Boris Brezillon wrote:
> On Mon, 30 Oct 2023 02:02:03 +0300
> Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:
> 
>> Prepare for addition of memory shrinker support by attaching shmem pages
>> to host dynamically on first use. Previously the attachment vq command
>> wasn't fenced and there was no vq kick made in the BO creation code path,
>> hence the attachment already was happening dynamically, but implicitly.
>> Making attachment explicitly dynamic will allow to simplify and reuse more
>> code when shrinker will be added. The virtio_gpu_object_shmem_init() now
>> works under the held reservation lock, which will be important to have for
>> shrinker to avoid moving pages while they are in active use by the driver.
> Ah, this commit might actually help getting rid of the workaround
> introduced in "drm/shmem-helper: Add common memory shrinker".

Not really. The virtio_gpu_free_object() is unchanged, it's only
allocation that is being deferred and it's only done for a one BO type
(virtio-gpu has multiple BO types).

-- 
Best regards,
Dmitry

Re: [PATCH v18 24/26] drm/virtio: Attach shmem BOs dynamically

[Dmitry Osipenko]

On 11/23/23 01:37, Dmitry Osipenko wrote:
> On 11/13/23 12:57, Boris Brezillon wrote:
>> On Mon, 30 Oct 2023 02:02:03 +0300
>> Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:
>>
>>> Prepare for addition of memory shrinker support by attaching shmem pages
>>> to host dynamically on first use. Previously the attachment vq command
>>> wasn't fenced and there was no vq kick made in the BO creation code path,
>>> hence the attachment already was happening dynamically, but implicitly.
>>> Making attachment explicitly dynamic will allow to simplify and reuse more
>>> code when shrinker will be added. The virtio_gpu_object_shmem_init() now
>>> works under the held reservation lock, which will be important to have for
>>> shrinker to avoid moving pages while they are in active use by the driver.
>> Ah, this commit might actually help getting rid of the workaround
>> introduced in "drm/shmem-helper: Add common memory shrinker".
> 
> Not really. The virtio_gpu_free_object() is unchanged, it's only
> allocation that is being deferred and it's only done for a one BO type
> (virtio-gpu has multiple BO types).

s/allocation/attachment/

Pages are still allocated by virtio_gpu_object_create().

-- 
Best regards,
Dmitry

Re: [PATCH v18 15/26] drm/panfrost: Explicitly get and put drm-shmem pages

[Boris Brezillon]

On Mon, 30 Oct 2023 02:01:54 +0300
Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:

> To simplify the drm-shmem refcnt handling, we're moving away from
> the implicit get_pages() that is used by get_pages_sgt(). From now on
> drivers will have to pin pages while they use sgt. Panfrost's shrinker
> doesn't support swapping out BOs, hence pages are pinned and sgt is valid
> as long as pages' use-count > 0.
> 
> Signed-off-by: Dmitry Osipenko <dmitry.osipenko@collabora.com>
> ---
>  drivers/gpu/drm/panfrost/panfrost_gem.c | 17 +++++++++++++++++
>  drivers/gpu/drm/panfrost/panfrost_mmu.c |  6 ++----
>  2 files changed, 19 insertions(+), 4 deletions(-)
> 
> diff --git a/drivers/gpu/drm/panfrost/panfrost_gem.c b/drivers/gpu/drm/panfrost/panfrost_gem.c
> index 6b77d8cebcb2..bb9d43cf7c3c 100644
> --- a/drivers/gpu/drm/panfrost/panfrost_gem.c
> +++ b/drivers/gpu/drm/panfrost/panfrost_gem.c
> @@ -47,8 +47,13 @@ static void panfrost_gem_free_object(struct drm_gem_object *obj)
>  			}
>  		}
>  		kvfree(bo->sgts);
> +
> +		drm_gem_shmem_put_pages(&bo->base);
>  	}
>  
> +	if (!bo->is_heap && !obj->import_attach)
> +		drm_gem_shmem_put_pages(&bo->base);
> +
>  	drm_gem_shmem_free(&bo->base);
>  }
>  
> @@ -269,6 +274,7 @@ panfrost_gem_create(struct drm_device *dev, size_t size, u32 flags)
>  {
>  	struct drm_gem_shmem_object *shmem;
>  	struct panfrost_gem_object *bo;
> +	int err;
>  
>  	/* Round up heap allocations to 2MB to keep fault handling simple */
>  	if (flags & PANFROST_BO_HEAP)
> @@ -282,7 +288,18 @@ panfrost_gem_create(struct drm_device *dev, size_t size, u32 flags)
>  	bo->noexec = !!(flags & PANFROST_BO_NOEXEC);
>  	bo->is_heap = !!(flags & PANFROST_BO_HEAP);
>  
> +	if (!bo->is_heap) {
> +		err = drm_gem_shmem_get_pages(shmem);

Hm, there was no drm_gem_shmem_get_pages_sgt() call here, why should we
add a drm_gem_shmem_get_pages()? What we should do instead is add a
drm_gem_shmem_get_pages() for each drm_gem_shmem_get_pages_sgt() we
have in the driver (in panfrost_mmu_map()), and add
drm_gem_shmem_put_pages() calls where they are missing
(panfrost_mmu_unmap()).

> +		if (err)
> +			goto err_free;
> +	}
> +
>  	return bo;
> +
> +err_free:
> +	drm_gem_shmem_free(&bo->base);
> +
> +	return ERR_PTR(err);
>  }
>  
>  struct drm_gem_object *
> diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.c b/drivers/gpu/drm/panfrost/panfrost_mmu.c
> index 770dab1942c2..ac145a98377b 100644
> --- a/drivers/gpu/drm/panfrost/panfrost_mmu.c
> +++ b/drivers/gpu/drm/panfrost/panfrost_mmu.c
> @@ -504,7 +504,7 @@ static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as,
>  		if (IS_ERR(pages[i])) {
>  			ret = PTR_ERR(pages[i]);
>  			pages[i] = NULL;
> -			goto err_pages;
> +			goto err_unlock;
>  		}
>  	}
>  
> @@ -512,7 +512,7 @@ static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as,
>  	ret = sg_alloc_table_from_pages(sgt, pages + page_offset,
>  					NUM_FAULT_PAGES, 0, SZ_2M, GFP_KERNEL);
>  	if (ret)
> -		goto err_pages;
> +		goto err_unlock;

Feels like the panfrost_gem_mapping object should hold a ref on the BO
pages, not the BO itself, because, ultimately, the user of the BO is
the GPU. This matches what I was saying about moving get/put_pages() to
panfrost_mmu_map/unmap(): everytime a panfrost_gem_mapping becomes
active, to want to take a pages ref, every time it becomes inactive,
you should release the pages ref.

>  
>  	ret = dma_map_sgtable(pfdev->dev, sgt, DMA_BIDIRECTIONAL, 0);
>  	if (ret)
> @@ -535,8 +535,6 @@ static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as,
>  
>  err_map:
>  	sg_free_table(sgt);
> -err_pages:
> -	drm_gem_shmem_put_pages_locked(&bo->base);
>  err_unlock:
>  	dma_resv_unlock(obj->resv);
>  err_bo:

Re: [PATCH v18 15/26] drm/panfrost: Explicitly get and put drm-shmem pages

[Dmitry Osipenko]

On 11/10/23 13:53, Boris Brezillon wrote:
> Hm, there was no drm_gem_shmem_get_pages_sgt() call here, why should we
> add a drm_gem_shmem_get_pages()? What we should do instead is add a
> drm_gem_shmem_get_pages() for each drm_gem_shmem_get_pages_sgt() we
> have in the driver (in panfrost_mmu_map()), and add
> drm_gem_shmem_put_pages() calls where they are missing
> (panfrost_mmu_unmap()).
> 
>> +		if (err)
>> +			goto err_free;
>> +	}
>> +
>>  	return bo;
>> +
>> +err_free:
>> +	drm_gem_shmem_free(&bo->base);
>> +
>> +	return ERR_PTR(err);
>>  }
>>  
>>  struct drm_gem_object *
>> diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.c b/drivers/gpu/drm/panfrost/panfrost_mmu.c
>> index 770dab1942c2..ac145a98377b 100644
>> --- a/drivers/gpu/drm/panfrost/panfrost_mmu.c
>> +++ b/drivers/gpu/drm/panfrost/panfrost_mmu.c
>> @@ -504,7 +504,7 @@ static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as,
>>  		if (IS_ERR(pages[i])) {
>>  			ret = PTR_ERR(pages[i]);
>>  			pages[i] = NULL;
>> -			goto err_pages;
>> +			goto err_unlock;
>>  		}
>>  	}
>>  
>> @@ -512,7 +512,7 @@ static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as,
>>  	ret = sg_alloc_table_from_pages(sgt, pages + page_offset,
>>  					NUM_FAULT_PAGES, 0, SZ_2M, GFP_KERNEL);
>>  	if (ret)
>> -		goto err_pages;
>> +		goto err_unlock;
> Feels like the panfrost_gem_mapping object should hold a ref on the BO
> pages, not the BO itself, because, ultimately, the user of the BO is
> the GPU. This matches what I was saying about moving get/put_pages() to
> panfrost_mmu_map/unmap(): everytime a panfrost_gem_mapping becomes
> active, to want to take a pages ref, every time it becomes inactive,
> you should release the pages ref.

The panfrost_mmu_unmap() is also used by shrinker when BO is purged. I'm
unhappy with how icky it all becomes if unmap is made to put pages.

Previously map() was implicitly allocating pages with get_sgt() and then
pages were implicitly released by drm_gem_shmem_free(). A non-heap BO is
mapped when it's created by Panfrost, hence the actual lifetime of pages
is kept unchanged by this patch. The implicit allocation is turned into
explicit one, i.e. pages are explicitly allocated before BO is mapped.

-- 
Best regards,
Dmitry

Re: [PATCH v18 15/26] drm/panfrost: Explicitly get and put drm-shmem pages

[Boris Brezillon]

On Thu, 23 Nov 2023 01:04:56 +0300
Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:

> On 11/10/23 13:53, Boris Brezillon wrote:
> > Hm, there was no drm_gem_shmem_get_pages_sgt() call here, why should we
> > add a drm_gem_shmem_get_pages()? What we should do instead is add a
> > drm_gem_shmem_get_pages() for each drm_gem_shmem_get_pages_sgt() we
> > have in the driver (in panfrost_mmu_map()), and add
> > drm_gem_shmem_put_pages() calls where they are missing
> > (panfrost_mmu_unmap()).
> >   
> >> +		if (err)
> >> +			goto err_free;
> >> +	}
> >> +
> >>  	return bo;
> >> +
> >> +err_free:
> >> +	drm_gem_shmem_free(&bo->base);
> >> +
> >> +	return ERR_PTR(err);
> >>  }
> >>  
> >>  struct drm_gem_object *
> >> diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.c b/drivers/gpu/drm/panfrost/panfrost_mmu.c
> >> index 770dab1942c2..ac145a98377b 100644
> >> --- a/drivers/gpu/drm/panfrost/panfrost_mmu.c
> >> +++ b/drivers/gpu/drm/panfrost/panfrost_mmu.c
> >> @@ -504,7 +504,7 @@ static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as,
> >>  		if (IS_ERR(pages[i])) {
> >>  			ret = PTR_ERR(pages[i]);
> >>  			pages[i] = NULL;
> >> -			goto err_pages;
> >> +			goto err_unlock;
> >>  		}
> >>  	}
> >>  
> >> @@ -512,7 +512,7 @@ static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as,
> >>  	ret = sg_alloc_table_from_pages(sgt, pages + page_offset,
> >>  					NUM_FAULT_PAGES, 0, SZ_2M, GFP_KERNEL);
> >>  	if (ret)
> >> -		goto err_pages;
> >> +		goto err_unlock;  
> > Feels like the panfrost_gem_mapping object should hold a ref on the BO
> > pages, not the BO itself, because, ultimately, the user of the BO is
> > the GPU. This matches what I was saying about moving get/put_pages() to
> > panfrost_mmu_map/unmap(): everytime a panfrost_gem_mapping becomes
> > active, to want to take a pages ref, every time it becomes inactive,
> > you should release the pages ref.  
> 
> The panfrost_mmu_unmap() is also used by shrinker when BO is purged. I'm
> unhappy with how icky it all becomes if unmap is made to put pages.

Why, that's exactly what's supposed to happen. If you mmu_unmap(), that
means you no longer need the pages ref you got.

> 
> Previously map() was implicitly allocating pages with get_sgt() and then
> pages were implicitly released by drm_gem_shmem_free(). A non-heap BO is
> mapped when it's created by Panfrost, hence the actual lifetime of pages
> is kept unchanged by this patch.

But the whole point of making it explicit is to control when pages are
needed or not, isn't it. The fact we mmu_map() the BO at open time, and
keep it mapped until it's not longer referenced is an implementation
choice, and I don't think having pages_put() in mmu_unmap() changes
that.

> The implicit allocation is turned into
> explicit one, i.e. pages are explicitly allocated before BO is mapped.
> 

Re: [PATCH v18 15/26] drm/panfrost: Explicitly get and put drm-shmem pages

[Dmitry Osipenko]

On 11/23/23 12:05, Boris Brezillon wrote:
> On Thu, 23 Nov 2023 01:04:56 +0300
> Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:
> 
>> On 11/10/23 13:53, Boris Brezillon wrote:
>>> Hm, there was no drm_gem_shmem_get_pages_sgt() call here, why should we
>>> add a drm_gem_shmem_get_pages()? What we should do instead is add a
>>> drm_gem_shmem_get_pages() for each drm_gem_shmem_get_pages_sgt() we
>>> have in the driver (in panfrost_mmu_map()), and add
>>> drm_gem_shmem_put_pages() calls where they are missing
>>> (panfrost_mmu_unmap()).
>>>   
>>>> +		if (err)
>>>> +			goto err_free;
>>>> +	}
>>>> +
>>>>  	return bo;
>>>> +
>>>> +err_free:
>>>> +	drm_gem_shmem_free(&bo->base);
>>>> +
>>>> +	return ERR_PTR(err);
>>>>  }
>>>>  
>>>>  struct drm_gem_object *
>>>> diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.c b/drivers/gpu/drm/panfrost/panfrost_mmu.c
>>>> index 770dab1942c2..ac145a98377b 100644
>>>> --- a/drivers/gpu/drm/panfrost/panfrost_mmu.c
>>>> +++ b/drivers/gpu/drm/panfrost/panfrost_mmu.c
>>>> @@ -504,7 +504,7 @@ static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as,
>>>>  		if (IS_ERR(pages[i])) {
>>>>  			ret = PTR_ERR(pages[i]);
>>>>  			pages[i] = NULL;
>>>> -			goto err_pages;
>>>> +			goto err_unlock;
>>>>  		}
>>>>  	}
>>>>  
>>>> @@ -512,7 +512,7 @@ static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as,
>>>>  	ret = sg_alloc_table_from_pages(sgt, pages + page_offset,
>>>>  					NUM_FAULT_PAGES, 0, SZ_2M, GFP_KERNEL);
>>>>  	if (ret)
>>>> -		goto err_pages;
>>>> +		goto err_unlock;  
>>> Feels like the panfrost_gem_mapping object should hold a ref on the BO
>>> pages, not the BO itself, because, ultimately, the user of the BO is
>>> the GPU. This matches what I was saying about moving get/put_pages() to
>>> panfrost_mmu_map/unmap(): everytime a panfrost_gem_mapping becomes
>>> active, to want to take a pages ref, every time it becomes inactive,
>>> you should release the pages ref.  
>>
>> The panfrost_mmu_unmap() is also used by shrinker when BO is purged. I'm
>> unhappy with how icky it all becomes if unmap is made to put pages.
> 
> Why, that's exactly what's supposed to happen. If you mmu_unmap(), that
> means you no longer need the pages ref you got.

The drm_gem_shmem_purge() frees the pages. If mmu_unmap() frees pages too, then it becomes odd for drm_gem_shmem_purge() that it needs to free pages that were already freed.

>> Previously map() was implicitly allocating pages with get_sgt() and then
>> pages were implicitly released by drm_gem_shmem_free(). A non-heap BO is
>> mapped when it's created by Panfrost, hence the actual lifetime of pages
>> is kept unchanged by this patch.
> 
> But the whole point of making it explicit is to control when pages are
> needed or not, isn't it. The fact we mmu_map() the BO at open time, and
> keep it mapped until it's not longer referenced is an implementation
> choice, and I don't think having pages_put() in mmu_unmap() changes
> that.

Previously, when the last mmu_unmap() was done, the pages were not released.

If you'll make unmap to put pages, then you can't map BO again because pages are released by the last put() of unmap. In order to keep the old pages allocation logic unchanged, the pages must be referenced while BO is alive, not while mapping is alive.

Technically, the code can be changed to put pages on unmap. This requires adding special quirk to drm_gem_shmem_purge() and then for Panfrost pages should have the same lifetime as BO, hence why bother?


diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c
index 5ee98b6f0c94..5492610802a1 100644
--- a/drivers/gpu/drm/drm_gem_shmem_helper.c
+++ b/drivers/gpu/drm/drm_gem_shmem_helper.c
@@ -600,7 +600,9 @@ drm_gem_shmem_shrinker_put_pages_locked(struct drm_gem_shmem_object *shmem)
 	if (shmem->evicted)
 		return;
 
-	drm_gem_shmem_free_pages(shmem);
+	if (refcount_read(&shmem->pages_use_count))
+		drm_gem_shmem_free_pages(shmem);
+
 	drm_vma_node_unmap(&obj->vma_node, dev->anon_inode->i_mapping);
 }
 
@@ -608,7 +610,8 @@ void drm_gem_shmem_purge_locked(struct drm_gem_shmem_object *shmem)
 {
 	struct drm_gem_object *obj = &shmem->base;
 
-	drm_WARN_ON(obj->dev, !drm_gem_shmem_is_purgeable(shmem));
+	if (refcount_read(&shmem->pages_use_count))
+		drm_WARN_ON(obj->dev, !drm_gem_shmem_is_purgeable(shmem));
 
 	drm_gem_shmem_shrinker_put_pages_locked(shmem);
 	drm_gem_free_mmap_offset(obj);
diff --git a/drivers/gpu/drm/panfrost/panfrost_gem.c b/drivers/gpu/drm/panfrost/panfrost_gem.c
index a6128e32f303..499964c43a7b 100644
--- a/drivers/gpu/drm/panfrost/panfrost_gem.c
+++ b/drivers/gpu/drm/panfrost/panfrost_gem.c
@@ -41,9 +41,6 @@ static void panfrost_gem_free_object(struct drm_gem_object *obj)
 		drm_gem_shmem_put_pages(&bo->base);
 	}
 
-	if (!bo->is_heap && !obj->import_attach)
-		drm_gem_shmem_put_pages(&bo->base);
-
 	drm_gem_shmem_free(&bo->base);
 }
 
@@ -302,12 +299,6 @@ panfrost_gem_create(struct drm_device *dev, size_t size, u32 flags)
 	bo->noexec = !!(flags & PANFROST_BO_NOEXEC);
 	bo->is_heap = !!(flags & PANFROST_BO_HEAP);
 
-	if (!bo->is_heap) {
-		err = drm_gem_shmem_get_pages(shmem);
-		if (err)
-			goto err_free;
-	}
-
 	return bo;
 
 err_free:
diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.c b/drivers/gpu/drm/panfrost/panfrost_mmu.c
index 01cd97011ea5..4ed4ba5df420 100644
--- a/drivers/gpu/drm/panfrost/panfrost_mmu.c
+++ b/drivers/gpu/drm/panfrost/panfrost_mmu.c
@@ -335,9 +335,13 @@ int panfrost_mmu_map(struct panfrost_gem_mapping *mapping)
 		prot |= IOMMU_NOEXEC;
 
 	if (!obj->import_attach) {
-		ret = drm_gem_shmem_pin(shmem);
+		ret = drm_gem_shmem_get_pages(shmem);
 		if (ret)
 			return ret;
+
+		ret = drm_gem_shmem_pin(shmem);
+		if (ret)
+			goto put_pages;
 	}
 
 	sgt = drm_gem_shmem_get_pages_sgt(shmem);
@@ -349,9 +353,18 @@ int panfrost_mmu_map(struct panfrost_gem_mapping *mapping)
 	mmu_map_sg(pfdev, mapping->mmu, mapping->mmnode.start << PAGE_SHIFT,
 		   prot, sgt);
 	mapping->active = true;
+
+	if (!obj->import_attach)
+		drm_gem_shmem_unpin(shmem);
+
+	return 0;
+
 unpin:
 	if (!obj->import_attach)
 		drm_gem_shmem_unpin(shmem);
+put_pages:
+	if (!obj->import_attach)
+		drm_gem_shmem_put_pages(shmem);
 
 	return ret;
 }
@@ -360,6 +373,7 @@ void panfrost_mmu_unmap(struct panfrost_gem_mapping *mapping)
 {
 	struct panfrost_gem_object *bo = mapping->obj;
 	struct drm_gem_object *obj = &bo->base.base;
+	struct drm_gem_shmem_object *shmem = &bo->base;
 	struct panfrost_device *pfdev = to_panfrost_device(obj->dev);
 	struct io_pgtable_ops *ops = mapping->mmu->pgtbl_ops;
 	u64 iova = mapping->mmnode.start << PAGE_SHIFT;
@@ -389,6 +403,9 @@ void panfrost_mmu_unmap(struct panfrost_gem_mapping *mapping)
 	panfrost_mmu_flush_range(pfdev, mapping->mmu,
 				 mapping->mmnode.start << PAGE_SHIFT, len);
 	mapping->active = false;
+
+	if (!bo->is_heap && !obj->import_attach)
+		drm_gem_shmem_put_pages_locked(shmem);
 }
 
 static void mmu_tlb_inv_context_s1(void *cookie)


-- 
Best regards,
Dmitry

Re: [PATCH v18 15/26] drm/panfrost: Explicitly get and put drm-shmem pages

[Boris Brezillon]

On Thu, 23 Nov 2023 15:24:32 +0300
Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:

> On 11/23/23 12:05, Boris Brezillon wrote:
> > On Thu, 23 Nov 2023 01:04:56 +0300
> > Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:
> >   
> >> On 11/10/23 13:53, Boris Brezillon wrote:  
> >>> Hm, there was no drm_gem_shmem_get_pages_sgt() call here, why
> >>> should we add a drm_gem_shmem_get_pages()? What we should do
> >>> instead is add a drm_gem_shmem_get_pages() for each
> >>> drm_gem_shmem_get_pages_sgt() we have in the driver (in
> >>> panfrost_mmu_map()), and add drm_gem_shmem_put_pages() calls
> >>> where they are missing (panfrost_mmu_unmap()).
> >>>     
> >>>> +		if (err)
> >>>> +			goto err_free;
> >>>> +	}
> >>>> +
> >>>>  	return bo;
> >>>> +
> >>>> +err_free:
> >>>> +	drm_gem_shmem_free(&bo->base);
> >>>> +
> >>>> +	return ERR_PTR(err);
> >>>>  }
> >>>>  
> >>>>  struct drm_gem_object *
> >>>> diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.c
> >>>> b/drivers/gpu/drm/panfrost/panfrost_mmu.c index
> >>>> 770dab1942c2..ac145a98377b 100644 ---
> >>>> a/drivers/gpu/drm/panfrost/panfrost_mmu.c +++
> >>>> b/drivers/gpu/drm/panfrost/panfrost_mmu.c @@ -504,7 +504,7 @@
> >>>> static int panfrost_mmu_map_fault_addr(struct panfrost_device
> >>>> *pfdev, int as, if (IS_ERR(pages[i])) { ret = PTR_ERR(pages[i]);
> >>>>  			pages[i] = NULL;
> >>>> -			goto err_pages;
> >>>> +			goto err_unlock;
> >>>>  		}
> >>>>  	}
> >>>>  
> >>>> @@ -512,7 +512,7 @@ static int
> >>>> panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int
> >>>> as, ret = sg_alloc_table_from_pages(sgt, pages + page_offset,
> >>>> NUM_FAULT_PAGES, 0, SZ_2M, GFP_KERNEL); if (ret)
> >>>> -		goto err_pages;
> >>>> +		goto err_unlock;    
> >>> Feels like the panfrost_gem_mapping object should hold a ref on
> >>> the BO pages, not the BO itself, because, ultimately, the user of
> >>> the BO is the GPU. This matches what I was saying about moving
> >>> get/put_pages() to panfrost_mmu_map/unmap(): everytime a
> >>> panfrost_gem_mapping becomes active, to want to take a pages ref,
> >>> every time it becomes inactive, you should release the pages ref.
> >>>    
> >>
> >> The panfrost_mmu_unmap() is also used by shrinker when BO is
> >> purged. I'm unhappy with how icky it all becomes if unmap is made
> >> to put pages.  
> > 
> > Why, that's exactly what's supposed to happen. If you mmu_unmap(),
> > that means you no longer need the pages ref you got.  
> 
> The drm_gem_shmem_purge() frees the pages. If mmu_unmap() frees pages
> too, then it becomes odd for drm_gem_shmem_purge() that it needs to
> free pages that were already freed.

Hm, I didn't consider the mmu_unmap() call in the eviction path.

> 
> >> Previously map() was implicitly allocating pages with get_sgt()
> >> and then pages were implicitly released by drm_gem_shmem_free(). A
> >> non-heap BO is mapped when it's created by Panfrost, hence the
> >> actual lifetime of pages is kept unchanged by this patch.  
> > 
> > But the whole point of making it explicit is to control when pages
> > are needed or not, isn't it. The fact we mmu_map() the BO at open
> > time, and keep it mapped until it's not longer referenced is an
> > implementation choice, and I don't think having pages_put() in
> > mmu_unmap() changes that.  
> 
> Previously, when the last mmu_unmap() was done, the pages were not
> released.
> 
> If you'll make unmap to put pages, then you can't map BO again
> because pages are released by the last put() of unmap.

Well, you could, if panfrost_gem_mapping_get() was not only
returning an existing mapping, but was also creating one when none
exist. But you're right, it messes up with the shmem shrinker and also
changes the way we are doing things now.

> In order to
> keep the old pages allocation logic unchanged, the pages must be
> referenced while BO is alive, not while mapping is alive.

Correct.

> 
> Technically, the code can be changed to put pages on unmap. This
> requires adding special quirk to drm_gem_shmem_purge() and then for
> Panfrost pages should have the same lifetime as BO, hence why bother?

No, we certainly don't want to do that.

Re: [PATCH v18 15/26] drm/panfrost: Explicitly get and put drm-shmem pages

[Boris Brezillon]

On Mon, 30 Oct 2023 02:01:54 +0300
Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:

> To simplify the drm-shmem refcnt handling, we're moving away from
> the implicit get_pages() that is used by get_pages_sgt(). From now on
> drivers will have to pin pages while they use sgt. Panfrost's shrinker
> doesn't support swapping out BOs, hence pages are pinned and sgt is valid
> as long as pages' use-count > 0.
> 
> Signed-off-by: Dmitry Osipenko <dmitry.osipenko@collabora.com>
> ---
>  drivers/gpu/drm/panfrost/panfrost_gem.c | 17 +++++++++++++++++
>  drivers/gpu/drm/panfrost/panfrost_mmu.c |  6 ++----
>  2 files changed, 19 insertions(+), 4 deletions(-)
> 
> diff --git a/drivers/gpu/drm/panfrost/panfrost_gem.c b/drivers/gpu/drm/panfrost/panfrost_gem.c
> index 6b77d8cebcb2..bb9d43cf7c3c 100644
> --- a/drivers/gpu/drm/panfrost/panfrost_gem.c
> +++ b/drivers/gpu/drm/panfrost/panfrost_gem.c
> @@ -47,8 +47,13 @@ static void panfrost_gem_free_object(struct drm_gem_object *obj)
>  			}
>  		}
>  		kvfree(bo->sgts);
> +
> +		drm_gem_shmem_put_pages(&bo->base);
>  	}
>  
> +	if (!bo->is_heap && !obj->import_attach)
> +		drm_gem_shmem_put_pages(&bo->base);
> +
>  	drm_gem_shmem_free(&bo->base);
>  }
>  
> @@ -269,6 +274,7 @@ panfrost_gem_create(struct drm_device *dev, size_t size, u32 flags)
>  {
>  	struct drm_gem_shmem_object *shmem;
>  	struct panfrost_gem_object *bo;
> +	int err;
>  
>  	/* Round up heap allocations to 2MB to keep fault handling simple */
>  	if (flags & PANFROST_BO_HEAP)
> @@ -282,7 +288,18 @@ panfrost_gem_create(struct drm_device *dev, size_t size, u32 flags)
>  	bo->noexec = !!(flags & PANFROST_BO_NOEXEC);
>  	bo->is_heap = !!(flags & PANFROST_BO_HEAP);
>  
> +	if (!bo->is_heap) {
> +		err = drm_gem_shmem_get_pages(shmem);

I really hate the fact we request pages here while we call
panfrost_mmu_map() in panfrost_gem_open(), because ultimately, pages
are requested for the MMU mapping. Also hate the quirk we have in shmem
to call free_pages() instead of put_pages_locked() when the BO refcount
dropped to zero, and I was hoping we could get rid of it at some point
by teaching drivers to request pages when they actually need it instead
of tying pages lifetime to the GEM object lifetime.

Maybe what we should do instead is move the get/put_pages() in
panfrost_mmu_map/unmap() (as I suggested), but have a special mapping
panfrost_mmu_evict/restore() helpers that kill/restore the MMU mappings
without releasing/acquiring the pages ref.

Re: [PATCH v18 15/26] drm/panfrost: Explicitly get and put drm-shmem pages

[Boris Brezillon]

On Thu, 23 Nov 2023 15:48:48 +0100
Boris Brezillon <boris.brezillon@collabora.com> wrote:

> On Mon, 30 Oct 2023 02:01:54 +0300
> Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:
> 
> > To simplify the drm-shmem refcnt handling, we're moving away from
> > the implicit get_pages() that is used by get_pages_sgt(). From now on
> > drivers will have to pin pages while they use sgt. Panfrost's shrinker
> > doesn't support swapping out BOs, hence pages are pinned and sgt is valid
> > as long as pages' use-count > 0.
> > 
> > Signed-off-by: Dmitry Osipenko <dmitry.osipenko@collabora.com>
> > ---
> >  drivers/gpu/drm/panfrost/panfrost_gem.c | 17 +++++++++++++++++
> >  drivers/gpu/drm/panfrost/panfrost_mmu.c |  6 ++----
> >  2 files changed, 19 insertions(+), 4 deletions(-)
> > 
> > diff --git a/drivers/gpu/drm/panfrost/panfrost_gem.c b/drivers/gpu/drm/panfrost/panfrost_gem.c
> > index 6b77d8cebcb2..bb9d43cf7c3c 100644
> > --- a/drivers/gpu/drm/panfrost/panfrost_gem.c
> > +++ b/drivers/gpu/drm/panfrost/panfrost_gem.c
> > @@ -47,8 +47,13 @@ static void panfrost_gem_free_object(struct drm_gem_object *obj)
> >  			}
> >  		}
> >  		kvfree(bo->sgts);
> > +
> > +		drm_gem_shmem_put_pages(&bo->base);
> >  	}
> >  
> > +	if (!bo->is_heap && !obj->import_attach)
> > +		drm_gem_shmem_put_pages(&bo->base);
> > +
> >  	drm_gem_shmem_free(&bo->base);
> >  }
> >  
> > @@ -269,6 +274,7 @@ panfrost_gem_create(struct drm_device *dev, size_t size, u32 flags)
> >  {
> >  	struct drm_gem_shmem_object *shmem;
> >  	struct panfrost_gem_object *bo;
> > +	int err;
> >  
> >  	/* Round up heap allocations to 2MB to keep fault handling simple */
> >  	if (flags & PANFROST_BO_HEAP)
> > @@ -282,7 +288,18 @@ panfrost_gem_create(struct drm_device *dev, size_t size, u32 flags)
> >  	bo->noexec = !!(flags & PANFROST_BO_NOEXEC);
> >  	bo->is_heap = !!(flags & PANFROST_BO_HEAP);
> >  
> > +	if (!bo->is_heap) {
> > +		err = drm_gem_shmem_get_pages(shmem);  
> 
> I really hate the fact we request pages here while we call
> panfrost_mmu_map() in panfrost_gem_open(), because ultimately, pages
> are requested for the MMU mapping. Also hate the quirk we have in shmem
> to call free_pages() instead of put_pages_locked() when the BO refcount
> dropped to zero, and I was hoping we could get rid of it at some point
> by teaching drivers to request pages when they actually need it instead
> of tying pages lifetime to the GEM object lifetime.
> 
> Maybe what we should do instead is move the get/put_pages() in
> panfrost_mmu_map/unmap() (as I suggested), but have a special mapping
> panfrost_mmu_evict/restore() helpers that kill/restore the MMU mappings
> without releasing/acquiring the pages ref.

Okay, so I played with your branch and did what I suggested here ^. The
end result is available here [1]. I also split this patch in two:

- A fix for the error path in panfrost_mmu_map_fault_addr() [2]
- The explicit get/put_pages() stuff with pages ref owned by the
  panfrost_gem_mapping object [3]

[1]https://gitlab.freedesktop.org/bbrezillon/linux/-/commits/virtio-gpu-shrinker-v18
[2]https://gitlab.freedesktop.org/bbrezillon/linux/-/commit/9d499e971fdae4d6e52f8871ca27c24b2a2c43d6
[3]https://gitlab.freedesktop.org/bbrezillon/linux/-/commit/ba3de65bf1cf0ca95710e743ec85ca67ff1aa223

Re: [PATCH v18 26/26] drm/panfrost: Switch to generic memory shrinker

[Boris Brezillon]

On Mon, 30 Oct 2023 02:02:05 +0300
Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:

> Replace Panfrost's custom memory shrinker with a common drm-shmem
> memory shrinker.
> 
> Signed-off-by: Dmitry Osipenko <dmitry.osipenko@collabora.com>
> ---
>  drivers/gpu/drm/panfrost/Makefile             |   1 -
>  drivers/gpu/drm/panfrost/panfrost_device.h    |   4 -
>  drivers/gpu/drm/panfrost/panfrost_drv.c       |  27 ++--
>  drivers/gpu/drm/panfrost/panfrost_gem.c       |  34 +++--
>  drivers/gpu/drm/panfrost/panfrost_gem.h       |   9 --
>  .../gpu/drm/panfrost/panfrost_gem_shrinker.c  | 129 ------------------
>  drivers/gpu/drm/panfrost/panfrost_job.c       |  18 ++-
>  drivers/gpu/drm/panfrost/panfrost_mmu.c       |  18 ++-
>  include/drm/drm_gem_shmem_helper.h            |   7 -

Nice diffstat.

>  9 files changed, 66 insertions(+), 181 deletions(-)
>  delete mode 100644 drivers/gpu/drm/panfrost/panfrost_gem_shrinker.c
> 
> diff --git a/drivers/gpu/drm/panfrost/Makefile b/drivers/gpu/drm/panfrost/Makefile
> index 2c01c1e7523e..f2cb1ab0a32d 100644
> --- a/drivers/gpu/drm/panfrost/Makefile
> +++ b/drivers/gpu/drm/panfrost/Makefile
> @@ -5,7 +5,6 @@ panfrost-y := \
>  	panfrost_device.o \
>  	panfrost_devfreq.o \
>  	panfrost_gem.o \
> -	panfrost_gem_shrinker.o \
>  	panfrost_gpu.o \
>  	panfrost_job.o \
>  	panfrost_mmu.o \
> diff --git a/drivers/gpu/drm/panfrost/panfrost_device.h b/drivers/gpu/drm/panfrost/panfrost_device.h
> index 1e85656dc2f7..2b24a0d4f85e 100644
> --- a/drivers/gpu/drm/panfrost/panfrost_device.h
> +++ b/drivers/gpu/drm/panfrost/panfrost_device.h
> @@ -117,10 +117,6 @@ struct panfrost_device {
>  		atomic_t pending;
>  	} reset;
>  
> -	struct mutex shrinker_lock;
> -	struct list_head shrinker_list;
> -	struct shrinker shrinker;
> -
>  	struct panfrost_devfreq pfdevfreq;
>  
>  	struct {
> diff --git a/drivers/gpu/drm/panfrost/panfrost_drv.c b/drivers/gpu/drm/panfrost/panfrost_drv.c
> index 7f2aba96d5b9..ef520d2cc1d2 100644
> --- a/drivers/gpu/drm/panfrost/panfrost_drv.c
> +++ b/drivers/gpu/drm/panfrost/panfrost_drv.c
> @@ -171,7 +171,6 @@ panfrost_lookup_bos(struct drm_device *dev,
>  			break;
>  		}
>  
> -		atomic_inc(&bo->gpu_usecount);
>  		job->mappings[i] = mapping;
>  	}
>  
> @@ -397,7 +396,6 @@ static int panfrost_ioctl_madvise(struct drm_device *dev, void *data,
>  {
>  	struct panfrost_file_priv *priv = file_priv->driver_priv;
>  	struct drm_panfrost_madvise *args = data;
> -	struct panfrost_device *pfdev = dev->dev_private;
>  	struct drm_gem_object *gem_obj;
>  	struct panfrost_gem_object *bo;
>  	int ret = 0;
> @@ -410,11 +408,15 @@ static int panfrost_ioctl_madvise(struct drm_device *dev, void *data,
>  
>  	bo = to_panfrost_bo(gem_obj);
>  
> +	if (bo->is_heap) {
> +		args->retained = 1;
> +		goto out_put_object;
> +	}

After digging a bit, I finally got why you do that:
panfrost_gem_shmem_is_purgeable() had a shmem->sgt != NULL test, and
shmem->sgt is never populated for heap BOs (those have a separate sgts
table, each entry containing an sgt covering a 2MB portion of the
buffer). Looking at the code, I don't think making heap BO
non-reclaimable was intentional, otherwise we'd have filtered it out in
panfrost_ioctl_madvise() to avoid inserting an element that will never
be reclaimable. TLDR; I understand why you do that, and I agree this is
the right thing to do (even if I doubt we have any userspace using the
MADV ioctl on heap BOs), but it definitely deserves a comment
explaining that this is here to keep a pre-existing behavior, so people
are not tempted to remove it, and if they do, they must have a good
explanation.

> +
>  	ret = dma_resv_lock_interruptible(bo->base.base.resv, NULL);
>  	if (ret)
>  		goto out_put_object;
>  
> -	mutex_lock(&pfdev->shrinker_lock);
>  	mutex_lock(&bo->mappings.lock);
>  	if (args->madv == PANFROST_MADV_DONTNEED) {
>  		struct panfrost_gem_mapping *first;
> @@ -440,17 +442,8 @@ static int panfrost_ioctl_madvise(struct drm_device *dev, void *data,
>  
>  	args->retained = drm_gem_shmem_madvise_locked(&bo->base, args->madv);
>  
> -	if (args->retained) {
> -		if (args->madv == PANFROST_MADV_DONTNEED)
> -			list_move_tail(&bo->base.madv_list,
> -				       &pfdev->shrinker_list);
> -		else if (args->madv == PANFROST_MADV_WILLNEED)
> -			list_del_init(&bo->base.madv_list);
> -	}
> -
>  out_unlock_mappings:
>  	mutex_unlock(&bo->mappings.lock);
> -	mutex_unlock(&pfdev->shrinker_lock);
>  	dma_resv_unlock(bo->base.base.resv);
>  out_put_object:
>  	drm_gem_object_put(gem_obj);
> @@ -635,9 +628,6 @@ static int panfrost_probe(struct platform_device *pdev)
>  	ddev->dev_private = pfdev;
>  	pfdev->ddev = ddev;
>  
> -	mutex_init(&pfdev->shrinker_lock);
> -	INIT_LIST_HEAD(&pfdev->shrinker_list);
> -
>  	err = panfrost_device_init(pfdev);
>  	if (err) {
>  		if (err != -EPROBE_DEFER)
> @@ -659,10 +649,14 @@ static int panfrost_probe(struct platform_device *pdev)
>  	if (err < 0)
>  		goto err_out1;
>  
> -	panfrost_gem_shrinker_init(ddev);
> +	err = drmm_gem_shmem_init(ddev);
> +	if (err < 0)
> +		goto err_out2;
>  
>  	return 0;
>  
> +err_out2:

Let's stop this nonsense, and give err labels meaningful names
(err_unregister_dev for that one).

> +	drm_dev_unregister(ddev);
>  err_out1:
>  	pm_runtime_disable(pfdev->dev);
>  	panfrost_device_fini(pfdev);
> @@ -678,7 +672,6 @@ static void panfrost_remove(struct platform_device *pdev)
>  	struct drm_device *ddev = pfdev->ddev;
>  
>  	drm_dev_unregister(ddev);
> -	panfrost_gem_shrinker_cleanup(ddev);
>  
>  	pm_runtime_get_sync(pfdev->dev);
>  	pm_runtime_disable(pfdev->dev);
> diff --git a/drivers/gpu/drm/panfrost/panfrost_gem.c b/drivers/gpu/drm/panfrost/panfrost_gem.c
> index bb9d43cf7c3c..a6128e32f303 100644
> --- a/drivers/gpu/drm/panfrost/panfrost_gem.c
> +++ b/drivers/gpu/drm/panfrost/panfrost_gem.c
> @@ -19,16 +19,6 @@ static void panfrost_gem_free_object(struct drm_gem_object *obj)
>  	struct panfrost_gem_object *bo = to_panfrost_bo(obj);
>  	struct panfrost_device *pfdev = obj->dev->dev_private;
>  
> -	/*
> -	 * Make sure the BO is no longer inserted in the shrinker list before
> -	 * taking care of the destruction itself. If we don't do that we have a
> -	 * race condition between this function and what's done in
> -	 * panfrost_gem_shrinker_scan().
> -	 */
> -	mutex_lock(&pfdev->shrinker_lock);
> -	list_del_init(&bo->base.madv_list);
> -	mutex_unlock(&pfdev->shrinker_lock);
> -
>  	/*
>  	 * If we still have mappings attached to the BO, there's a problem in
>  	 * our refcounting.
> @@ -94,7 +84,11 @@ static void panfrost_gem_mapping_release(struct kref *kref)
>  
>  	mapping = container_of(kref, struct panfrost_gem_mapping, refcount);
>  
> +	/* shrinker that may purge mapping at the same time */

Let's give a detailed explanation here:

	/* We need to do the unmapping with the resv lock held, because
	 * the shrinker might be purging mappings concurrently.
	 */

> +	dma_resv_lock(mapping->obj->base.base.resv, NULL);

This makes me think that maybe we could drop gem->mappings.lock and
protect the mappings list with the GEM resv. Not asking you to do that
in this patch though, it can be addressed in a follow-up.

>  	panfrost_gem_teardown_mapping(mapping);
> +	dma_resv_unlock(mapping->obj->base.base.resv);
> +
>  	drm_gem_object_put(&mapping->obj->base.base);
>  	panfrost_mmu_ctx_put(mapping->mmu);
>  	kfree(mapping);
> @@ -228,6 +222,25 @@ static size_t panfrost_gem_rss(struct drm_gem_object *obj)
>  	return 0;
>  }
>  
> +static int panfrost_shmem_evict(struct drm_gem_object *obj)
> +{
> +	struct panfrost_gem_object *bo = to_panfrost_bo(obj);
> +
> +	if (!drm_gem_shmem_is_purgeable(&bo->base))
> +		return -EBUSY;
> +
> +	if (!mutex_trylock(&bo->mappings.lock))
> +		return -EBUSY;
> +
> +	panfrost_gem_teardown_mappings_locked(bo);

Okay, so I've modified and renamed
panfrost_gem_teardown_mappings_locked() here [1]. The new version is
called panfrost_gem_evict_mappings_locked(), and it just kills the MMU
mapping (with a panfrost_mmu_unmap() call). The mm_node (VA range) is
kept allocated until the panfrost_gem_mapping object is destroyed, which
better matches the eviction semantics IMHO.

> +
> +	drm_gem_shmem_purge_locked(&bo->base);
> +
> +	mutex_unlock(&bo->mappings.lock);
> +
> +	return 0;
> +}
> +
>  static const struct drm_gem_object_funcs panfrost_gem_funcs = {
>  	.free = panfrost_gem_free_object,
>  	.open = panfrost_gem_open,
> @@ -242,6 +255,7 @@ static const struct drm_gem_object_funcs panfrost_gem_funcs = {
>  	.status = panfrost_gem_status,
>  	.rss = panfrost_gem_rss,
>  	.vm_ops = &drm_gem_shmem_vm_ops,
> +	.evict = panfrost_shmem_evict,
>  };
>  
>  /**
> diff --git a/drivers/gpu/drm/panfrost/panfrost_gem.h b/drivers/gpu/drm/panfrost/panfrost_gem.h
> index 13c0a8149c3a..8ddc2d310d29 100644
> --- a/drivers/gpu/drm/panfrost/panfrost_gem.h
> +++ b/drivers/gpu/drm/panfrost/panfrost_gem.h
> @@ -30,12 +30,6 @@ struct panfrost_gem_object {
>  		struct mutex lock;
>  	} mappings;
>  
> -	/*
> -	 * Count the number of jobs referencing this BO so we don't let the
> -	 * shrinker reclaim this object prematurely.
> -	 */
> -	atomic_t gpu_usecount;
> -
>  	/*
>  	 * Object chunk size currently mapped onto physical memory
>  	 */
> @@ -86,7 +80,4 @@ panfrost_gem_mapping_get(struct panfrost_gem_object *bo,
>  void panfrost_gem_mapping_put(struct panfrost_gem_mapping *mapping);
>  void panfrost_gem_teardown_mappings_locked(struct panfrost_gem_object *bo);
>  
> -void panfrost_gem_shrinker_init(struct drm_device *dev);
> -void panfrost_gem_shrinker_cleanup(struct drm_device *dev);
> -
>  #endif /* __PANFROST_GEM_H__ */
> diff --git a/drivers/gpu/drm/panfrost/panfrost_gem_shrinker.c b/drivers/gpu/drm/panfrost/panfrost_gem_shrinker.c
> deleted file mode 100644
> index 1aa94fff7072..000000000000
> --- a/drivers/gpu/drm/panfrost/panfrost_gem_shrinker.c
> +++ /dev/null
> @@ -1,129 +0,0 @@
> -// SPDX-License-Identifier: GPL-2.0
> -/* Copyright (C) 2019 Arm Ltd.
> - *
> - * Based on msm_gem_freedreno.c:
> - * Copyright (C) 2016 Red Hat
> - * Author: Rob Clark <robdclark@gmail.com>
> - */
> -
> -#include <linux/list.h>
> -
> -#include <drm/drm_device.h>
> -#include <drm/drm_gem_shmem_helper.h>
> -
> -#include "panfrost_device.h"
> -#include "panfrost_gem.h"
> -#include "panfrost_mmu.h"
> -
> -static bool panfrost_gem_shmem_is_purgeable(struct drm_gem_shmem_object *shmem)
> -{
> -	return (shmem->madv > 0) &&
> -		!refcount_read(&shmem->pages_pin_count) && shmem->sgt &&
> -		!shmem->base.dma_buf && !shmem->base.import_attach;
> -}
> -
> -static unsigned long
> -panfrost_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
> -{
> -	struct panfrost_device *pfdev =
> -		container_of(shrinker, struct panfrost_device, shrinker);
> -	struct drm_gem_shmem_object *shmem;
> -	unsigned long count = 0;
> -
> -	if (!mutex_trylock(&pfdev->shrinker_lock))
> -		return 0;
> -
> -	list_for_each_entry(shmem, &pfdev->shrinker_list, madv_list) {
> -		if (panfrost_gem_shmem_is_purgeable(shmem))
> -			count += shmem->base.size >> PAGE_SHIFT;
> -	}
> -
> -	mutex_unlock(&pfdev->shrinker_lock);
> -
> -	return count;
> -}
> -
> -static bool panfrost_gem_purge(struct drm_gem_object *obj)
> -{
> -	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
> -	struct panfrost_gem_object *bo = to_panfrost_bo(obj);
> -	bool ret = false;
> -
> -	if (atomic_read(&bo->gpu_usecount))
> -		return false;
> -
> -	if (!mutex_trylock(&bo->mappings.lock))
> -		return false;
> -
> -	if (!dma_resv_trylock(shmem->base.resv))
> -		goto unlock_mappings;
> -
> -	panfrost_gem_teardown_mappings_locked(bo);
> -	drm_gem_shmem_purge_locked(&bo->base);
> -	ret = true;
> -
> -	dma_resv_unlock(shmem->base.resv);
> -
> -unlock_mappings:
> -	mutex_unlock(&bo->mappings.lock);
> -	return ret;
> -}
> -
> -static unsigned long
> -panfrost_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
> -{
> -	struct panfrost_device *pfdev =
> -		container_of(shrinker, struct panfrost_device, shrinker);
> -	struct drm_gem_shmem_object *shmem, *tmp;
> -	unsigned long freed = 0;
> -
> -	if (!mutex_trylock(&pfdev->shrinker_lock))
> -		return SHRINK_STOP;
> -
> -	list_for_each_entry_safe(shmem, tmp, &pfdev->shrinker_list, madv_list) {
> -		if (freed >= sc->nr_to_scan)
> -			break;
> -		if (drm_gem_shmem_is_purgeable(shmem) &&
> -		    panfrost_gem_purge(&shmem->base)) {
> -			freed += shmem->base.size >> PAGE_SHIFT;
> -			list_del_init(&shmem->madv_list);
> -		}
> -	}
> -
> -	mutex_unlock(&pfdev->shrinker_lock);
> -
> -	if (freed > 0)
> -		pr_info_ratelimited("Purging %lu bytes\n", freed << PAGE_SHIFT);
> -
> -	return freed;
> -}
> -
> -/**
> - * panfrost_gem_shrinker_init - Initialize panfrost shrinker
> - * @dev: DRM device
> - *
> - * This function registers and sets up the panfrost shrinker.
> - */
> -void panfrost_gem_shrinker_init(struct drm_device *dev)
> -{
> -	struct panfrost_device *pfdev = dev->dev_private;
> -	pfdev->shrinker.count_objects = panfrost_gem_shrinker_count;
> -	pfdev->shrinker.scan_objects = panfrost_gem_shrinker_scan;
> -	pfdev->shrinker.seeks = DEFAULT_SEEKS;
> -	WARN_ON(register_shrinker(&pfdev->shrinker, "drm-panfrost"));
> -}
> -
> -/**
> - * panfrost_gem_shrinker_cleanup - Clean up panfrost shrinker
> - * @dev: DRM device
> - *
> - * This function unregisters the panfrost shrinker.
> - */
> -void panfrost_gem_shrinker_cleanup(struct drm_device *dev)
> -{
> -	struct panfrost_device *pfdev = dev->dev_private;
> -
> -	if (pfdev->shrinker.nr_deferred) {
> -		unregister_shrinker(&pfdev->shrinker);
> -	}
> -}
> diff --git a/drivers/gpu/drm/panfrost/panfrost_job.c b/drivers/gpu/drm/panfrost/panfrost_job.c
> index fb16de2d0420..da6be590557f 100644
> --- a/drivers/gpu/drm/panfrost/panfrost_job.c
> +++ b/drivers/gpu/drm/panfrost/panfrost_job.c
> @@ -289,6 +289,19 @@ static void panfrost_attach_object_fences(struct drm_gem_object **bos,
>  		dma_resv_add_fence(bos[i]->resv, fence, DMA_RESV_USAGE_WRITE);
>  }
>  
> +static int panfrost_objects_prepare(struct drm_gem_object **bos, int bo_count)
> +{
> +	struct panfrost_gem_object *bo;
> +	int ret = 0;
> +
> +	while (!ret && bo_count--) {
> +		bo = to_panfrost_bo(bos[bo_count]);
> +		ret = bo->base.madv ? -ENOMEM : 0;

That's more an -EINVAL: users are not supposed to pass BOs that have an
madv != WILL_NEED, if they do, that's their mistake, not a memory
allocation failure.

> +	}
> +
> +	return ret;
> +}
> +
>  int panfrost_job_push(struct panfrost_job *job)
>  {
>  	struct panfrost_device *pfdev = job->pfdev;
> @@ -300,6 +313,10 @@ int panfrost_job_push(struct panfrost_job *job)
>  	if (ret)
>  		return ret;
>  
> +	ret = panfrost_objects_prepare(job->bos, job->bo_count);
> +	if (ret)
> +		goto unlock;
> +
>  	mutex_lock(&pfdev->sched_lock);
>  	drm_sched_job_arm(&job->base);
>  
> @@ -341,7 +358,6 @@ static void panfrost_job_cleanup(struct kref *ref)
>  			if (!job->mappings[i])
>  				break;
>  
> -			atomic_dec(&job->mappings[i]->obj->gpu_usecount);
>  			panfrost_gem_mapping_put(job->mappings[i]);
>  		}
>  		kvfree(job->mappings);
> diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.c b/drivers/gpu/drm/panfrost/panfrost_mmu.c
> index ac145a98377b..01cd97011ea5 100644
> --- a/drivers/gpu/drm/panfrost/panfrost_mmu.c
> +++ b/drivers/gpu/drm/panfrost/panfrost_mmu.c
> @@ -326,6 +326,7 @@ int panfrost_mmu_map(struct panfrost_gem_mapping *mapping)
>  	struct panfrost_device *pfdev = to_panfrost_device(obj->dev);
>  	struct sg_table *sgt;
>  	int prot = IOMMU_READ | IOMMU_WRITE;
> +	int ret = 0;
>  
>  	if (WARN_ON(mapping->active))
>  		return 0;
> @@ -333,15 +334,26 @@ int panfrost_mmu_map(struct panfrost_gem_mapping *mapping)
>  	if (bo->noexec)
>  		prot |= IOMMU_NOEXEC;
>  
> +	if (!obj->import_attach) {
> +		ret = drm_gem_shmem_pin(shmem);

I see why you do that. In practice panfrost_mmu_map() is called
before the object has been exposed, so I don't think it can be
purged/evicted at that point (and _get_pages() should have been called
before panfrost_mmu_map()), but I don't mind the extra safety net. Can
we just add a comment explaining the pin/get_pages_sgt/unpin dance here?

> +		if (ret)
> +			return ret;
> +	}
> +
>  	sgt = drm_gem_shmem_get_pages_sgt(shmem);
> -	if (WARN_ON(IS_ERR(sgt)))
> -		return PTR_ERR(sgt);
> +	if (WARN_ON(IS_ERR(sgt))) {
> +		ret = PTR_ERR(sgt);
> +		goto unpin;
> +	}
>  
>  	mmu_map_sg(pfdev, mapping->mmu, mapping->mmnode.start << PAGE_SHIFT,
>  		   prot, sgt);
>  	mapping->active = true;

nit: add a blank line here, please.

> +unpin:
> +	if (!obj->import_attach)
> +		drm_gem_shmem_unpin(shmem);
>  
> -	return 0;
> +	return ret;
>  }
>  
>  void panfrost_mmu_unmap(struct panfrost_gem_mapping *mapping)
> diff --git a/include/drm/drm_gem_shmem_helper.h b/include/drm/drm_gem_shmem_helper.h
> index 6ac77c2082ed..2a506074da46 100644
> --- a/include/drm/drm_gem_shmem_helper.h
> +++ b/include/drm/drm_gem_shmem_helper.h
> @@ -61,13 +61,6 @@ struct drm_gem_shmem_object {
>  	 */
>  	int madv;
>  
> -	/**
> -	 * @madv_list: List entry for madvise tracking
> -	 *
> -	 * Typically used by drivers to track purgeable objects
> -	 */
> -	struct list_head madv_list;
> -
>  	/**
>  	 * @sgt: Scatter/gather table for imported PRIME buffers
>  	 */

[1]https://gitlab.freedesktop.org/bbrezillon/linux/-/commit/8401ba620de63073089517805f9413fac88653cd

Re: [PATCH v18 04/26] drm/shmem-helper: Refactor locked/unlocked functions

[Maxime Ripard]

[-- Attachment #1: Type: text/plain, Size: 2114 bytes --]

On Mon, Oct 30, 2023 at 02:01:43AM +0300, Dmitry Osipenko wrote:
> Add locked and remove unlocked postfixes from drm-shmem function names,
> making names consistent with the drm/gem core code.
> 
> Reviewed-by: Boris Brezillon <boris.brezillon@collabora.com>
> Suggested-by: Boris Brezillon <boris.brezillon@collabora.com>
> Signed-off-by: Dmitry Osipenko <dmitry.osipenko@collabora.com>

This contradicts my earlier ack on a patch but...

> ---
>  drivers/gpu/drm/drm_gem_shmem_helper.c        | 64 +++++++++----------
>  drivers/gpu/drm/lima/lima_gem.c               |  8 +--
>  drivers/gpu/drm/panfrost/panfrost_drv.c       |  2 +-
>  drivers/gpu/drm/panfrost/panfrost_gem.c       |  6 +-
>  .../gpu/drm/panfrost/panfrost_gem_shrinker.c  |  2 +-
>  drivers/gpu/drm/panfrost/panfrost_mmu.c       |  2 +-
>  drivers/gpu/drm/v3d/v3d_bo.c                  |  4 +-
>  drivers/gpu/drm/virtio/virtgpu_object.c       |  4 +-
>  include/drm/drm_gem_shmem_helper.h            | 36 +++++------
>  9 files changed, 64 insertions(+), 64 deletions(-)
> 
> diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c
> index 0d61f2b3e213..154585ddae08 100644
> --- a/drivers/gpu/drm/drm_gem_shmem_helper.c
> +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c
> @@ -43,8 +43,8 @@ static const struct drm_gem_object_funcs drm_gem_shmem_funcs = {
>  	.pin = drm_gem_shmem_object_pin,
>  	.unpin = drm_gem_shmem_object_unpin,
>  	.get_sg_table = drm_gem_shmem_object_get_sg_table,
> -	.vmap = drm_gem_shmem_object_vmap,
> -	.vunmap = drm_gem_shmem_object_vunmap,
> +	.vmap = drm_gem_shmem_object_vmap_locked,
> +	.vunmap = drm_gem_shmem_object_vunmap_locked,

While I think we should indeed be consistent with the names, I would
also expect helpers to get the locking right by default.

I'm not sure how reasonable it is, but I think I'd prefer to turn this
around and keep the drm_gem_shmem_object_vmap/unmap helpers name, and
convert whatever function needs to be converted to the unlock suffix so
we get a consistent naming.

Does that make sense?
Maxime

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 228 bytes --]

Re: [PATCH v18 04/26] drm/shmem-helper: Refactor locked/unlocked functions

[Boris Brezillon]

On Fri, 24 Nov 2023 11:40:06 +0100
Maxime Ripard <mripard@kernel.org> wrote:

> On Mon, Oct 30, 2023 at 02:01:43AM +0300, Dmitry Osipenko wrote:
> > Add locked and remove unlocked postfixes from drm-shmem function names,
> > making names consistent with the drm/gem core code.
> > 
> > Reviewed-by: Boris Brezillon <boris.brezillon@collabora.com>
> > Suggested-by: Boris Brezillon <boris.brezillon@collabora.com>
> > Signed-off-by: Dmitry Osipenko <dmitry.osipenko@collabora.com>  
> 
> This contradicts my earlier ack on a patch but...
> 
> > ---
> >  drivers/gpu/drm/drm_gem_shmem_helper.c        | 64 +++++++++----------
> >  drivers/gpu/drm/lima/lima_gem.c               |  8 +--
> >  drivers/gpu/drm/panfrost/panfrost_drv.c       |  2 +-
> >  drivers/gpu/drm/panfrost/panfrost_gem.c       |  6 +-
> >  .../gpu/drm/panfrost/panfrost_gem_shrinker.c  |  2 +-
> >  drivers/gpu/drm/panfrost/panfrost_mmu.c       |  2 +-
> >  drivers/gpu/drm/v3d/v3d_bo.c                  |  4 +-
> >  drivers/gpu/drm/virtio/virtgpu_object.c       |  4 +-
> >  include/drm/drm_gem_shmem_helper.h            | 36 +++++------
> >  9 files changed, 64 insertions(+), 64 deletions(-)
> > 
> > diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c
> > index 0d61f2b3e213..154585ddae08 100644
> > --- a/drivers/gpu/drm/drm_gem_shmem_helper.c
> > +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c
> > @@ -43,8 +43,8 @@ static const struct drm_gem_object_funcs drm_gem_shmem_funcs = {
> >  	.pin = drm_gem_shmem_object_pin,
> >  	.unpin = drm_gem_shmem_object_unpin,
> >  	.get_sg_table = drm_gem_shmem_object_get_sg_table,
> > -	.vmap = drm_gem_shmem_object_vmap,
> > -	.vunmap = drm_gem_shmem_object_vunmap,
> > +	.vmap = drm_gem_shmem_object_vmap_locked,
> > +	.vunmap = drm_gem_shmem_object_vunmap_locked,  
> 
> While I think we should indeed be consistent with the names, I would
> also expect helpers to get the locking right by default.
> 
> I'm not sure how reasonable it is, but I think I'd prefer to turn this
> around and keep the drm_gem_shmem_object_vmap/unmap helpers name, and
> convert whatever function needs to be converted to the unlock suffix so
> we get a consistent naming.
> 
> Does that make sense?

I don't mind, as long as it's consistent, it's just that that there's
probably more to patch if we do it the other way around.

Re: [PATCH v18 04/26] drm/shmem-helper: Refactor locked/unlocked functions

[Boris Brezillon]

On Fri, 24 Nov 2023 11:40:06 +0100
Maxime Ripard <mripard@kernel.org> wrote:

> On Mon, Oct 30, 2023 at 02:01:43AM +0300, Dmitry Osipenko wrote:
> > Add locked and remove unlocked postfixes from drm-shmem function names,
> > making names consistent with the drm/gem core code.
> > 
> > Reviewed-by: Boris Brezillon <boris.brezillon@collabora.com>
> > Suggested-by: Boris Brezillon <boris.brezillon@collabora.com>
> > Signed-off-by: Dmitry Osipenko <dmitry.osipenko@collabora.com>  
> 
> This contradicts my earlier ack on a patch but...
> 
> > ---
> >  drivers/gpu/drm/drm_gem_shmem_helper.c        | 64 +++++++++----------
> >  drivers/gpu/drm/lima/lima_gem.c               |  8 +--
> >  drivers/gpu/drm/panfrost/panfrost_drv.c       |  2 +-
> >  drivers/gpu/drm/panfrost/panfrost_gem.c       |  6 +-
> >  .../gpu/drm/panfrost/panfrost_gem_shrinker.c  |  2 +-
> >  drivers/gpu/drm/panfrost/panfrost_mmu.c       |  2 +-
> >  drivers/gpu/drm/v3d/v3d_bo.c                  |  4 +-
> >  drivers/gpu/drm/virtio/virtgpu_object.c       |  4 +-
> >  include/drm/drm_gem_shmem_helper.h            | 36 +++++------
> >  9 files changed, 64 insertions(+), 64 deletions(-)
> > 
> > diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c
> > index 0d61f2b3e213..154585ddae08 100644
> > --- a/drivers/gpu/drm/drm_gem_shmem_helper.c
> > +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c
> > @@ -43,8 +43,8 @@ static const struct drm_gem_object_funcs drm_gem_shmem_funcs = {
> >  	.pin = drm_gem_shmem_object_pin,
> >  	.unpin = drm_gem_shmem_object_unpin,
> >  	.get_sg_table = drm_gem_shmem_object_get_sg_table,
> > -	.vmap = drm_gem_shmem_object_vmap,
> > -	.vunmap = drm_gem_shmem_object_vunmap,
> > +	.vmap = drm_gem_shmem_object_vmap_locked,
> > +	.vunmap = drm_gem_shmem_object_vunmap_locked,  
> 
> While I think we should indeed be consistent with the names, I would
> also expect helpers to get the locking right by default.

Wait, actually I think this patch does what you suggest already. The
_locked() prefix tells the caller: "you should take care of the locking,
I expect the lock to be held when this is hook/function is called". So
helpers without the _locked() prefix take care of the locking (which I
guess matches your 'helpers get the locking right' expectation), and
those with the _locked() prefix don't.

> 
> I'm not sure how reasonable it is, but I think I'd prefer to turn this
> around and keep the drm_gem_shmem_object_vmap/unmap helpers name, and
> convert whatever function needs to be converted to the unlock suffix so
> we get a consistent naming.

That would be an _unlocked() prefix if we do it the other way around. I
think the main confusion comes from the names of the hooks in
drm_gem_shmem_funcs. Some of them, like drm_gem_shmem_funcs::v[un]map()
are called with the GEM resv lock held, and locking is handled by the
core, others, like drm_gem_shmem_funcs::[un]pin() are called
without the GEM resv lock held, and locking is deferred to the
implementation. As I said, I don't mind prefixing hooks/helpers with
_unlocked() for those that take care of the locking, and no prefix for
those that expects locks to be held, as long as it's consistent, but I
just wanted to make sure we're on the same page :-).

Re: [PATCH v18 04/26] drm/shmem-helper: Refactor locked/unlocked functions

[Maxime Ripard]

[-- Attachment #1: Type: text/plain, Size: 4522 bytes --]

Hi,

On Fri, Nov 24, 2023 at 11:59:11AM +0100, Boris Brezillon wrote:
> On Fri, 24 Nov 2023 11:40:06 +0100
> Maxime Ripard <mripard@kernel.org> wrote:
> 
> > On Mon, Oct 30, 2023 at 02:01:43AM +0300, Dmitry Osipenko wrote:
> > > Add locked and remove unlocked postfixes from drm-shmem function names,
> > > making names consistent with the drm/gem core code.
> > > 
> > > Reviewed-by: Boris Brezillon <boris.brezillon@collabora.com>
> > > Suggested-by: Boris Brezillon <boris.brezillon@collabora.com>
> > > Signed-off-by: Dmitry Osipenko <dmitry.osipenko@collabora.com>  
> > 
> > This contradicts my earlier ack on a patch but...
> > 
> > > ---
> > >  drivers/gpu/drm/drm_gem_shmem_helper.c        | 64 +++++++++----------
> > >  drivers/gpu/drm/lima/lima_gem.c               |  8 +--
> > >  drivers/gpu/drm/panfrost/panfrost_drv.c       |  2 +-
> > >  drivers/gpu/drm/panfrost/panfrost_gem.c       |  6 +-
> > >  .../gpu/drm/panfrost/panfrost_gem_shrinker.c  |  2 +-
> > >  drivers/gpu/drm/panfrost/panfrost_mmu.c       |  2 +-
> > >  drivers/gpu/drm/v3d/v3d_bo.c                  |  4 +-
> > >  drivers/gpu/drm/virtio/virtgpu_object.c       |  4 +-
> > >  include/drm/drm_gem_shmem_helper.h            | 36 +++++------
> > >  9 files changed, 64 insertions(+), 64 deletions(-)
> > > 
> > > diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c
> > > index 0d61f2b3e213..154585ddae08 100644
> > > --- a/drivers/gpu/drm/drm_gem_shmem_helper.c
> > > +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c
> > > @@ -43,8 +43,8 @@ static const struct drm_gem_object_funcs drm_gem_shmem_funcs = {
> > >  	.pin = drm_gem_shmem_object_pin,
> > >  	.unpin = drm_gem_shmem_object_unpin,
> > >  	.get_sg_table = drm_gem_shmem_object_get_sg_table,
> > > -	.vmap = drm_gem_shmem_object_vmap,
> > > -	.vunmap = drm_gem_shmem_object_vunmap,
> > > +	.vmap = drm_gem_shmem_object_vmap_locked,
> > > +	.vunmap = drm_gem_shmem_object_vunmap_locked,  
> > 
> > While I think we should indeed be consistent with the names, I would
> > also expect helpers to get the locking right by default.
> 
> Wait, actually I think this patch does what you suggest already. The
> _locked() prefix tells the caller: "you should take care of the locking,
> I expect the lock to be held when this is hook/function is called". So
> helpers without the _locked() prefix take care of the locking (which I
> guess matches your 'helpers get the locking right' expectation), and
> those with the _locked() prefix don't.

What I meant by "getting the locking right" is indeed a bit ambiguous,
sorry. What I'm trying to say I guess is that, in this particular case,
I don't think you can expect the vmap implementation to be called with
or without the locks held. The doc for that function will say that it's
either one or the other, but not both.

So helpers should follow what is needed to provide a default vmap/vunmap
implementation, including what locking is expected from a vmap/vunmap
implementation.

If that means that vmap is always called with the locks taken, then
drm_gem_shmem_object_vmap can just assume that it will be called with
the locks taken and there's no need to mention it in the name (and you
can probably sprinkle a couple of lockdep assertion to make sure the
locking is indeed consistent).

> > I'm not sure how reasonable it is, but I think I'd prefer to turn this
> > around and keep the drm_gem_shmem_object_vmap/unmap helpers name, and
> > convert whatever function needs to be converted to the unlock suffix so
> > we get a consistent naming.
> 
> That would be an _unlocked() prefix if we do it the other way around. I
> think the main confusion comes from the names of the hooks in
> drm_gem_shmem_funcs. Some of them, like drm_gem_shmem_funcs::v[un]map()
> are called with the GEM resv lock held, and locking is handled by the
> core, others, like drm_gem_shmem_funcs::[un]pin() are called
> without the GEM resv lock held, and locking is deferred to the
> implementation. As I said, I don't mind prefixing hooks/helpers with
> _unlocked() for those that take care of the locking, and no prefix for
> those that expects locks to be held, as long as it's consistent, but I
> just wanted to make sure we're on the same page :-).

What about _nolock then? It's the same number of characters than
_locked, plus it expresses what the function is (not) doing, not what
context it's supposed to be called in?

Maxime

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 228 bytes --]

Re: [PATCH v18 04/26] drm/shmem-helper: Refactor locked/unlocked functions

[Boris Brezillon]

On Tue, 28 Nov 2023 12:14:42 +0100
Maxime Ripard <mripard@kernel.org> wrote:

> Hi,
> 
> On Fri, Nov 24, 2023 at 11:59:11AM +0100, Boris Brezillon wrote:
> > On Fri, 24 Nov 2023 11:40:06 +0100
> > Maxime Ripard <mripard@kernel.org> wrote:
> >   
> > > On Mon, Oct 30, 2023 at 02:01:43AM +0300, Dmitry Osipenko wrote:  
> > > > Add locked and remove unlocked postfixes from drm-shmem function names,
> > > > making names consistent with the drm/gem core code.
> > > > 
> > > > Reviewed-by: Boris Brezillon <boris.brezillon@collabora.com>
> > > > Suggested-by: Boris Brezillon <boris.brezillon@collabora.com>
> > > > Signed-off-by: Dmitry Osipenko <dmitry.osipenko@collabora.com>    
> > > 
> > > This contradicts my earlier ack on a patch but...
> > >   
> > > > ---
> > > >  drivers/gpu/drm/drm_gem_shmem_helper.c        | 64 +++++++++----------
> > > >  drivers/gpu/drm/lima/lima_gem.c               |  8 +--
> > > >  drivers/gpu/drm/panfrost/panfrost_drv.c       |  2 +-
> > > >  drivers/gpu/drm/panfrost/panfrost_gem.c       |  6 +-
> > > >  .../gpu/drm/panfrost/panfrost_gem_shrinker.c  |  2 +-
> > > >  drivers/gpu/drm/panfrost/panfrost_mmu.c       |  2 +-
> > > >  drivers/gpu/drm/v3d/v3d_bo.c                  |  4 +-
> > > >  drivers/gpu/drm/virtio/virtgpu_object.c       |  4 +-
> > > >  include/drm/drm_gem_shmem_helper.h            | 36 +++++------
> > > >  9 files changed, 64 insertions(+), 64 deletions(-)
> > > > 
> > > > diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c
> > > > index 0d61f2b3e213..154585ddae08 100644
> > > > --- a/drivers/gpu/drm/drm_gem_shmem_helper.c
> > > > +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c
> > > > @@ -43,8 +43,8 @@ static const struct drm_gem_object_funcs drm_gem_shmem_funcs = {
> > > >  	.pin = drm_gem_shmem_object_pin,
> > > >  	.unpin = drm_gem_shmem_object_unpin,
> > > >  	.get_sg_table = drm_gem_shmem_object_get_sg_table,
> > > > -	.vmap = drm_gem_shmem_object_vmap,
> > > > -	.vunmap = drm_gem_shmem_object_vunmap,
> > > > +	.vmap = drm_gem_shmem_object_vmap_locked,
> > > > +	.vunmap = drm_gem_shmem_object_vunmap_locked,    
> > > 
> > > While I think we should indeed be consistent with the names, I would
> > > also expect helpers to get the locking right by default.  
> > 
> > Wait, actually I think this patch does what you suggest already. The
> > _locked() prefix tells the caller: "you should take care of the locking,
> > I expect the lock to be held when this is hook/function is called". So
> > helpers without the _locked() prefix take care of the locking (which I
> > guess matches your 'helpers get the locking right' expectation), and
> > those with the _locked() prefix don't.  
> 
> What I meant by "getting the locking right" is indeed a bit ambiguous,
> sorry. What I'm trying to say I guess is that, in this particular case,
> I don't think you can expect the vmap implementation to be called with
> or without the locks held. The doc for that function will say that it's
> either one or the other, but not both.
> 
> So helpers should follow what is needed to provide a default vmap/vunmap
> implementation, including what locking is expected from a vmap/vunmap
> implementation.

Hm, yeah, I think that's a matter of taste. When locking is often
deferrable, like it is in DRM, I find it beneficial for funcions and
function pointers to reflect the locking scheme, rather than relying on
people properly reading the doc, especially when this is the only
outlier in the group of drm_gem_object_funcs we already have, and it's
not event documented at the drm_gem_object_funcs level [1] :P.

> 
> If that means that vmap is always called with the locks taken, then
> drm_gem_shmem_object_vmap can just assume that it will be called with
> the locks taken and there's no need to mention it in the name (and you
> can probably sprinkle a couple of lockdep assertion to make sure the
> locking is indeed consistent).

Things get very confusing when you end up having drm_gem_shmem helpers
that are suffixed with _locked() to encode the fact locking is the
caller's responsibility and no suffix for the
callee-takes-care-of-the-locking semantics, while other helpers that are
not suffixed at all actually implement the
caller-should-take-care-of-the-locking semantics.

> 
> > > I'm not sure how reasonable it is, but I think I'd prefer to turn this
> > > around and keep the drm_gem_shmem_object_vmap/unmap helpers name, and
> > > convert whatever function needs to be converted to the unlock suffix so
> > > we get a consistent naming.  
> > 
> > That would be an _unlocked() prefix if we do it the other way around. I
> > think the main confusion comes from the names of the hooks in
> > drm_gem_shmem_funcs. Some of them, like drm_gem_shmem_funcs::v[un]map()
> > are called with the GEM resv lock held, and locking is handled by the
> > core, others, like drm_gem_shmem_funcs::[un]pin() are called
> > without the GEM resv lock held, and locking is deferred to the
> > implementation. As I said, I don't mind prefixing hooks/helpers with
> > _unlocked() for those that take care of the locking, and no prefix for
> > those that expects locks to be held, as long as it's consistent, but I
> > just wanted to make sure we're on the same page :-).  
> 
> What about _nolock then? It's the same number of characters than
> _locked, plus it expresses what the function is (not) doing, not what
> context it's supposed to be called in?

Just did a quick

  git grep _nolock drivers/gpu/drm

and it returns zero result, where the _locked/_unlocked pattern seems
to already be widely used. Not saying we shouldn't change that, but it
doesn't feel like a change we should do as part of this series.

Regards,

Boris

[1]https://elixir.bootlin.com/linux/v6.7-rc3/source/include/drm/drm_gem.h#L155

Re: [PATCH v18 04/26] drm/shmem-helper: Refactor locked/unlocked functions

[Dmitry Osipenko]

On 11/28/23 15:37, Boris Brezillon wrote:
> On Tue, 28 Nov 2023 12:14:42 +0100
> Maxime Ripard <mripard@kernel.org> wrote:
> 
>> Hi,
>>
>> On Fri, Nov 24, 2023 at 11:59:11AM +0100, Boris Brezillon wrote:
>>> On Fri, 24 Nov 2023 11:40:06 +0100
>>> Maxime Ripard <mripard@kernel.org> wrote:
>>>   
>>>> On Mon, Oct 30, 2023 at 02:01:43AM +0300, Dmitry Osipenko wrote:  
>>>>> Add locked and remove unlocked postfixes from drm-shmem function names,
>>>>> making names consistent with the drm/gem core code.
>>>>>
>>>>> Reviewed-by: Boris Brezillon <boris.brezillon@collabora.com>
>>>>> Suggested-by: Boris Brezillon <boris.brezillon@collabora.com>
>>>>> Signed-off-by: Dmitry Osipenko <dmitry.osipenko@collabora.com>    
>>>>
>>>> This contradicts my earlier ack on a patch but...
>>>>   
>>>>> ---
>>>>>  drivers/gpu/drm/drm_gem_shmem_helper.c        | 64 +++++++++----------
>>>>>  drivers/gpu/drm/lima/lima_gem.c               |  8 +--
>>>>>  drivers/gpu/drm/panfrost/panfrost_drv.c       |  2 +-
>>>>>  drivers/gpu/drm/panfrost/panfrost_gem.c       |  6 +-
>>>>>  .../gpu/drm/panfrost/panfrost_gem_shrinker.c  |  2 +-
>>>>>  drivers/gpu/drm/panfrost/panfrost_mmu.c       |  2 +-
>>>>>  drivers/gpu/drm/v3d/v3d_bo.c                  |  4 +-
>>>>>  drivers/gpu/drm/virtio/virtgpu_object.c       |  4 +-
>>>>>  include/drm/drm_gem_shmem_helper.h            | 36 +++++------
>>>>>  9 files changed, 64 insertions(+), 64 deletions(-)
>>>>>
>>>>> diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c
>>>>> index 0d61f2b3e213..154585ddae08 100644
>>>>> --- a/drivers/gpu/drm/drm_gem_shmem_helper.c
>>>>> +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c
>>>>> @@ -43,8 +43,8 @@ static const struct drm_gem_object_funcs drm_gem_shmem_funcs = {
>>>>>  	.pin = drm_gem_shmem_object_pin,
>>>>>  	.unpin = drm_gem_shmem_object_unpin,
>>>>>  	.get_sg_table = drm_gem_shmem_object_get_sg_table,
>>>>> -	.vmap = drm_gem_shmem_object_vmap,
>>>>> -	.vunmap = drm_gem_shmem_object_vunmap,
>>>>> +	.vmap = drm_gem_shmem_object_vmap_locked,
>>>>> +	.vunmap = drm_gem_shmem_object_vunmap_locked,    
>>>>
>>>> While I think we should indeed be consistent with the names, I would
>>>> also expect helpers to get the locking right by default.  
>>>
>>> Wait, actually I think this patch does what you suggest already. The
>>> _locked() prefix tells the caller: "you should take care of the locking,
>>> I expect the lock to be held when this is hook/function is called". So
>>> helpers without the _locked() prefix take care of the locking (which I
>>> guess matches your 'helpers get the locking right' expectation), and
>>> those with the _locked() prefix don't.  
>>
>> What I meant by "getting the locking right" is indeed a bit ambiguous,
>> sorry. What I'm trying to say I guess is that, in this particular case,
>> I don't think you can expect the vmap implementation to be called with
>> or without the locks held. The doc for that function will say that it's
>> either one or the other, but not both.
>>
>> So helpers should follow what is needed to provide a default vmap/vunmap
>> implementation, including what locking is expected from a vmap/vunmap
>> implementation.
> 
> Hm, yeah, I think that's a matter of taste. When locking is often
> deferrable, like it is in DRM, I find it beneficial for funcions and
> function pointers to reflect the locking scheme, rather than relying on
> people properly reading the doc, especially when this is the only
> outlier in the group of drm_gem_object_funcs we already have, and it's
> not event documented at the drm_gem_object_funcs level [1] :P.
> 
>>
>> If that means that vmap is always called with the locks taken, then
>> drm_gem_shmem_object_vmap can just assume that it will be called with
>> the locks taken and there's no need to mention it in the name (and you
>> can probably sprinkle a couple of lockdep assertion to make sure the
>> locking is indeed consistent).
> 
> Things get very confusing when you end up having drm_gem_shmem helpers
> that are suffixed with _locked() to encode the fact locking is the
> caller's responsibility and no suffix for the
> callee-takes-care-of-the-locking semantics, while other helpers that are
> not suffixed at all actually implement the
> caller-should-take-care-of-the-locking semantics.
> 
>>
>>>> I'm not sure how reasonable it is, but I think I'd prefer to turn this
>>>> around and keep the drm_gem_shmem_object_vmap/unmap helpers name, and
>>>> convert whatever function needs to be converted to the unlock suffix so
>>>> we get a consistent naming.  
>>>
>>> That would be an _unlocked() prefix if we do it the other way around. I
>>> think the main confusion comes from the names of the hooks in
>>> drm_gem_shmem_funcs. Some of them, like drm_gem_shmem_funcs::v[un]map()
>>> are called with the GEM resv lock held, and locking is handled by the
>>> core, others, like drm_gem_shmem_funcs::[un]pin() are called
>>> without the GEM resv lock held, and locking is deferred to the
>>> implementation. As I said, I don't mind prefixing hooks/helpers with
>>> _unlocked() for those that take care of the locking, and no prefix for
>>> those that expects locks to be held, as long as it's consistent, but I
>>> just wanted to make sure we're on the same page :-).  
>>
>> What about _nolock then? It's the same number of characters than
>> _locked, plus it expresses what the function is (not) doing, not what
>> context it's supposed to be called in?
> 
> Just did a quick
> 
>   git grep _nolock drivers/gpu/drm
> 
> and it returns zero result, where the _locked/_unlocked pattern seems
> to already be widely used. Not saying we shouldn't change that, but it
> doesn't feel like a change we should do as part of this series.
> 
> Regards,
> 
> Boris
> 
> [1]https://elixir.bootlin.com/linux/v6.7-rc3/source/include/drm/drm_gem.h#L155

I'm fine with dropping the _locked() postfix from the common GEM helpers
and documenting the locking rule in drm_gem. Thank you all for the
suggestions :)

-- 
Best regards,
Dmitry

Re: [PATCH v18 04/26] drm/shmem-helper: Refactor locked/unlocked functions

[Boris Brezillon]

On Wed, 29 Nov 2023 01:05:14 +0300
Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:

> On 11/28/23 15:37, Boris Brezillon wrote:
> > On Tue, 28 Nov 2023 12:14:42 +0100
> > Maxime Ripard <mripard@kernel.org> wrote:
> >   
> >> Hi,
> >>
> >> On Fri, Nov 24, 2023 at 11:59:11AM +0100, Boris Brezillon wrote:  
> >>> On Fri, 24 Nov 2023 11:40:06 +0100
> >>> Maxime Ripard <mripard@kernel.org> wrote:
> >>>     
> >>>> On Mon, Oct 30, 2023 at 02:01:43AM +0300, Dmitry Osipenko wrote:    
> >>>>> Add locked and remove unlocked postfixes from drm-shmem function names,
> >>>>> making names consistent with the drm/gem core code.
> >>>>>
> >>>>> Reviewed-by: Boris Brezillon <boris.brezillon@collabora.com>
> >>>>> Suggested-by: Boris Brezillon <boris.brezillon@collabora.com>
> >>>>> Signed-off-by: Dmitry Osipenko <dmitry.osipenko@collabora.com>      
> >>>>
> >>>> This contradicts my earlier ack on a patch but...
> >>>>     
> >>>>> ---
> >>>>>  drivers/gpu/drm/drm_gem_shmem_helper.c        | 64 +++++++++----------
> >>>>>  drivers/gpu/drm/lima/lima_gem.c               |  8 +--
> >>>>>  drivers/gpu/drm/panfrost/panfrost_drv.c       |  2 +-
> >>>>>  drivers/gpu/drm/panfrost/panfrost_gem.c       |  6 +-
> >>>>>  .../gpu/drm/panfrost/panfrost_gem_shrinker.c  |  2 +-
> >>>>>  drivers/gpu/drm/panfrost/panfrost_mmu.c       |  2 +-
> >>>>>  drivers/gpu/drm/v3d/v3d_bo.c                  |  4 +-
> >>>>>  drivers/gpu/drm/virtio/virtgpu_object.c       |  4 +-
> >>>>>  include/drm/drm_gem_shmem_helper.h            | 36 +++++------
> >>>>>  9 files changed, 64 insertions(+), 64 deletions(-)
> >>>>>
> >>>>> diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c
> >>>>> index 0d61f2b3e213..154585ddae08 100644
> >>>>> --- a/drivers/gpu/drm/drm_gem_shmem_helper.c
> >>>>> +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c
> >>>>> @@ -43,8 +43,8 @@ static const struct drm_gem_object_funcs drm_gem_shmem_funcs = {
> >>>>>  	.pin = drm_gem_shmem_object_pin,
> >>>>>  	.unpin = drm_gem_shmem_object_unpin,
> >>>>>  	.get_sg_table = drm_gem_shmem_object_get_sg_table,
> >>>>> -	.vmap = drm_gem_shmem_object_vmap,
> >>>>> -	.vunmap = drm_gem_shmem_object_vunmap,
> >>>>> +	.vmap = drm_gem_shmem_object_vmap_locked,
> >>>>> +	.vunmap = drm_gem_shmem_object_vunmap_locked,      
> >>>>
> >>>> While I think we should indeed be consistent with the names, I would
> >>>> also expect helpers to get the locking right by default.    
> >>>
> >>> Wait, actually I think this patch does what you suggest already. The
> >>> _locked() prefix tells the caller: "you should take care of the locking,
> >>> I expect the lock to be held when this is hook/function is called". So
> >>> helpers without the _locked() prefix take care of the locking (which I
> >>> guess matches your 'helpers get the locking right' expectation), and
> >>> those with the _locked() prefix don't.    
> >>
> >> What I meant by "getting the locking right" is indeed a bit ambiguous,
> >> sorry. What I'm trying to say I guess is that, in this particular case,
> >> I don't think you can expect the vmap implementation to be called with
> >> or without the locks held. The doc for that function will say that it's
> >> either one or the other, but not both.
> >>
> >> So helpers should follow what is needed to provide a default vmap/vunmap
> >> implementation, including what locking is expected from a vmap/vunmap
> >> implementation.  
> > 
> > Hm, yeah, I think that's a matter of taste. When locking is often
> > deferrable, like it is in DRM, I find it beneficial for funcions and
> > function pointers to reflect the locking scheme, rather than relying on
> > people properly reading the doc, especially when this is the only
> > outlier in the group of drm_gem_object_funcs we already have, and it's
> > not event documented at the drm_gem_object_funcs level [1] :P.
> >   
> >>
> >> If that means that vmap is always called with the locks taken, then
> >> drm_gem_shmem_object_vmap can just assume that it will be called with
> >> the locks taken and there's no need to mention it in the name (and you
> >> can probably sprinkle a couple of lockdep assertion to make sure the
> >> locking is indeed consistent).  
> > 
> > Things get very confusing when you end up having drm_gem_shmem helpers
> > that are suffixed with _locked() to encode the fact locking is the
> > caller's responsibility and no suffix for the
> > callee-takes-care-of-the-locking semantics, while other helpers that are
> > not suffixed at all actually implement the
> > caller-should-take-care-of-the-locking semantics.
> >   
> >>  
> >>>> I'm not sure how reasonable it is, but I think I'd prefer to turn this
> >>>> around and keep the drm_gem_shmem_object_vmap/unmap helpers name, and
> >>>> convert whatever function needs to be converted to the unlock suffix so
> >>>> we get a consistent naming.    
> >>>
> >>> That would be an _unlocked() prefix if we do it the other way around. I
> >>> think the main confusion comes from the names of the hooks in
> >>> drm_gem_shmem_funcs. Some of them, like drm_gem_shmem_funcs::v[un]map()
> >>> are called with the GEM resv lock held, and locking is handled by the
> >>> core, others, like drm_gem_shmem_funcs::[un]pin() are called
> >>> without the GEM resv lock held, and locking is deferred to the
> >>> implementation. As I said, I don't mind prefixing hooks/helpers with
> >>> _unlocked() for those that take care of the locking, and no prefix for
> >>> those that expects locks to be held, as long as it's consistent, but I
> >>> just wanted to make sure we're on the same page :-).    
> >>
> >> What about _nolock then? It's the same number of characters than
> >> _locked, plus it expresses what the function is (not) doing, not what
> >> context it's supposed to be called in?  
> > 
> > Just did a quick
> > 
> >   git grep _nolock drivers/gpu/drm
> > 
> > and it returns zero result, where the _locked/_unlocked pattern seems
> > to already be widely used. Not saying we shouldn't change that, but it
> > doesn't feel like a change we should do as part of this series.
> > 
> > Regards,
> > 
> > Boris
> > 
> > [1]https://elixir.bootlin.com/linux/v6.7-rc3/source/include/drm/drm_gem.h#L155  
> 
> I'm fine with dropping the _locked() postfix from the common GEM helpers
> and documenting the locking rule in drm_gem. Thank you all for the
> suggestions :)

Sorry to disagree, but I think a proper function name/suffix is
sometimes worth a few lines of doc. Not saying we should do one or the
other, I think we should do both. But when I see a function suffixed
_locked, _unlocked or _nolock, I can immediately tell if this function
defers the locking to the caller or not, and then go check which lock
in the function doc.

And the second thing I'm not happy with, is the fact we go back to an
inconsistent naming in drm_gem_shmem_helper.c, where some functions
deferring the locking to the caller are suffixed _locked and others are
not, because ultimately, you need a different name when you expose the
two variants...

Re: [PATCH v18 04/26] drm/shmem-helper: Refactor locked/unlocked functions

[Dmitry Osipenko]

On 11/29/23 10:53, Boris Brezillon wrote:
> On Wed, 29 Nov 2023 01:05:14 +0300
> Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:
> 
>> On 11/28/23 15:37, Boris Brezillon wrote:
>>> On Tue, 28 Nov 2023 12:14:42 +0100
>>> Maxime Ripard <mripard@kernel.org> wrote:
>>>   
>>>> Hi,
>>>>
>>>> On Fri, Nov 24, 2023 at 11:59:11AM +0100, Boris Brezillon wrote:  
>>>>> On Fri, 24 Nov 2023 11:40:06 +0100
>>>>> Maxime Ripard <mripard@kernel.org> wrote:
>>>>>     
>>>>>> On Mon, Oct 30, 2023 at 02:01:43AM +0300, Dmitry Osipenko wrote:    
>>>>>>> Add locked and remove unlocked postfixes from drm-shmem function names,
>>>>>>> making names consistent with the drm/gem core code.
>>>>>>>
>>>>>>> Reviewed-by: Boris Brezillon <boris.brezillon@collabora.com>
>>>>>>> Suggested-by: Boris Brezillon <boris.brezillon@collabora.com>
>>>>>>> Signed-off-by: Dmitry Osipenko <dmitry.osipenko@collabora.com>      
>>>>>>
>>>>>> This contradicts my earlier ack on a patch but...
>>>>>>     
>>>>>>> ---
>>>>>>>  drivers/gpu/drm/drm_gem_shmem_helper.c        | 64 +++++++++----------
>>>>>>>  drivers/gpu/drm/lima/lima_gem.c               |  8 +--
>>>>>>>  drivers/gpu/drm/panfrost/panfrost_drv.c       |  2 +-
>>>>>>>  drivers/gpu/drm/panfrost/panfrost_gem.c       |  6 +-
>>>>>>>  .../gpu/drm/panfrost/panfrost_gem_shrinker.c  |  2 +-
>>>>>>>  drivers/gpu/drm/panfrost/panfrost_mmu.c       |  2 +-
>>>>>>>  drivers/gpu/drm/v3d/v3d_bo.c                  |  4 +-
>>>>>>>  drivers/gpu/drm/virtio/virtgpu_object.c       |  4 +-
>>>>>>>  include/drm/drm_gem_shmem_helper.h            | 36 +++++------
>>>>>>>  9 files changed, 64 insertions(+), 64 deletions(-)
>>>>>>>
>>>>>>> diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c
>>>>>>> index 0d61f2b3e213..154585ddae08 100644
>>>>>>> --- a/drivers/gpu/drm/drm_gem_shmem_helper.c
>>>>>>> +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c
>>>>>>> @@ -43,8 +43,8 @@ static const struct drm_gem_object_funcs drm_gem_shmem_funcs = {
>>>>>>>  	.pin = drm_gem_shmem_object_pin,
>>>>>>>  	.unpin = drm_gem_shmem_object_unpin,
>>>>>>>  	.get_sg_table = drm_gem_shmem_object_get_sg_table,
>>>>>>> -	.vmap = drm_gem_shmem_object_vmap,
>>>>>>> -	.vunmap = drm_gem_shmem_object_vunmap,
>>>>>>> +	.vmap = drm_gem_shmem_object_vmap_locked,
>>>>>>> +	.vunmap = drm_gem_shmem_object_vunmap_locked,      
>>>>>>
>>>>>> While I think we should indeed be consistent with the names, I would
>>>>>> also expect helpers to get the locking right by default.    
>>>>>
>>>>> Wait, actually I think this patch does what you suggest already. The
>>>>> _locked() prefix tells the caller: "you should take care of the locking,
>>>>> I expect the lock to be held when this is hook/function is called". So
>>>>> helpers without the _locked() prefix take care of the locking (which I
>>>>> guess matches your 'helpers get the locking right' expectation), and
>>>>> those with the _locked() prefix don't.    
>>>>
>>>> What I meant by "getting the locking right" is indeed a bit ambiguous,
>>>> sorry. What I'm trying to say I guess is that, in this particular case,
>>>> I don't think you can expect the vmap implementation to be called with
>>>> or without the locks held. The doc for that function will say that it's
>>>> either one or the other, but not both.
>>>>
>>>> So helpers should follow what is needed to provide a default vmap/vunmap
>>>> implementation, including what locking is expected from a vmap/vunmap
>>>> implementation.  
>>>
>>> Hm, yeah, I think that's a matter of taste. When locking is often
>>> deferrable, like it is in DRM, I find it beneficial for funcions and
>>> function pointers to reflect the locking scheme, rather than relying on
>>> people properly reading the doc, especially when this is the only
>>> outlier in the group of drm_gem_object_funcs we already have, and it's
>>> not event documented at the drm_gem_object_funcs level [1] :P.
>>>   
>>>>
>>>> If that means that vmap is always called with the locks taken, then
>>>> drm_gem_shmem_object_vmap can just assume that it will be called with
>>>> the locks taken and there's no need to mention it in the name (and you
>>>> can probably sprinkle a couple of lockdep assertion to make sure the
>>>> locking is indeed consistent).  
>>>
>>> Things get very confusing when you end up having drm_gem_shmem helpers
>>> that are suffixed with _locked() to encode the fact locking is the
>>> caller's responsibility and no suffix for the
>>> callee-takes-care-of-the-locking semantics, while other helpers that are
>>> not suffixed at all actually implement the
>>> caller-should-take-care-of-the-locking semantics.
>>>   
>>>>  
>>>>>> I'm not sure how reasonable it is, but I think I'd prefer to turn this
>>>>>> around and keep the drm_gem_shmem_object_vmap/unmap helpers name, and
>>>>>> convert whatever function needs to be converted to the unlock suffix so
>>>>>> we get a consistent naming.    
>>>>>
>>>>> That would be an _unlocked() prefix if we do it the other way around. I
>>>>> think the main confusion comes from the names of the hooks in
>>>>> drm_gem_shmem_funcs. Some of them, like drm_gem_shmem_funcs::v[un]map()
>>>>> are called with the GEM resv lock held, and locking is handled by the
>>>>> core, others, like drm_gem_shmem_funcs::[un]pin() are called
>>>>> without the GEM resv lock held, and locking is deferred to the
>>>>> implementation. As I said, I don't mind prefixing hooks/helpers with
>>>>> _unlocked() for those that take care of the locking, and no prefix for
>>>>> those that expects locks to be held, as long as it's consistent, but I
>>>>> just wanted to make sure we're on the same page :-).    
>>>>
>>>> What about _nolock then? It's the same number of characters than
>>>> _locked, plus it expresses what the function is (not) doing, not what
>>>> context it's supposed to be called in?  
>>>
>>> Just did a quick
>>>
>>>   git grep _nolock drivers/gpu/drm
>>>
>>> and it returns zero result, where the _locked/_unlocked pattern seems
>>> to already be widely used. Not saying we shouldn't change that, but it
>>> doesn't feel like a change we should do as part of this series.
>>>
>>> Regards,
>>>
>>> Boris
>>>
>>> [1]https://elixir.bootlin.com/linux/v6.7-rc3/source/include/drm/drm_gem.h#L155  
>>
>> I'm fine with dropping the _locked() postfix from the common GEM helpers
>> and documenting the locking rule in drm_gem. Thank you all for the
>> suggestions :)
> 
> Sorry to disagree, but I think a proper function name/suffix is
> sometimes worth a few lines of doc. Not saying we should do one or the
> other, I think we should do both. But when I see a function suffixed
> _locked, _unlocked or _nolock, I can immediately tell if this function
> defers the locking to the caller or not, and then go check which lock
> in the function doc.
> 
> And the second thing I'm not happy with, is the fact we go back to an
> inconsistent naming in drm_gem_shmem_helper.c, where some functions
> deferring the locking to the caller are suffixed _locked and others are
> not, because ultimately, you need a different name when you expose the
> two variants...

By the `common GEM helpers` I meant the .vmap drm-shmem common helpers
used for drm_gem_object_funcs, like was suggested by Maxime. The rest of
functions will retain the _locked part. Sorry for the confusion :)

-- 
Best regards,
Dmitry

Re: [PATCH v18 04/26] drm/shmem-helper: Refactor locked/unlocked functions

[Boris Brezillon]

On Wed, 29 Nov 2023 13:47:21 +0300
Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:

> On 11/29/23 10:53, Boris Brezillon wrote:
> > On Wed, 29 Nov 2023 01:05:14 +0300
> > Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:
> >   
> >> On 11/28/23 15:37, Boris Brezillon wrote:  
> >>> On Tue, 28 Nov 2023 12:14:42 +0100
> >>> Maxime Ripard <mripard@kernel.org> wrote:
> >>>     
> >>>> Hi,
> >>>>
> >>>> On Fri, Nov 24, 2023 at 11:59:11AM +0100, Boris Brezillon wrote:    
> >>>>> On Fri, 24 Nov 2023 11:40:06 +0100
> >>>>> Maxime Ripard <mripard@kernel.org> wrote:
> >>>>>       
> >>>>>> On Mon, Oct 30, 2023 at 02:01:43AM +0300, Dmitry Osipenko wrote:      
> >>>>>>> Add locked and remove unlocked postfixes from drm-shmem function names,
> >>>>>>> making names consistent with the drm/gem core code.
> >>>>>>>
> >>>>>>> Reviewed-by: Boris Brezillon <boris.brezillon@collabora.com>
> >>>>>>> Suggested-by: Boris Brezillon <boris.brezillon@collabora.com>
> >>>>>>> Signed-off-by: Dmitry Osipenko <dmitry.osipenko@collabora.com>        
> >>>>>>
> >>>>>> This contradicts my earlier ack on a patch but...
> >>>>>>       
> >>>>>>> ---
> >>>>>>>  drivers/gpu/drm/drm_gem_shmem_helper.c        | 64 +++++++++----------
> >>>>>>>  drivers/gpu/drm/lima/lima_gem.c               |  8 +--
> >>>>>>>  drivers/gpu/drm/panfrost/panfrost_drv.c       |  2 +-
> >>>>>>>  drivers/gpu/drm/panfrost/panfrost_gem.c       |  6 +-
> >>>>>>>  .../gpu/drm/panfrost/panfrost_gem_shrinker.c  |  2 +-
> >>>>>>>  drivers/gpu/drm/panfrost/panfrost_mmu.c       |  2 +-
> >>>>>>>  drivers/gpu/drm/v3d/v3d_bo.c                  |  4 +-
> >>>>>>>  drivers/gpu/drm/virtio/virtgpu_object.c       |  4 +-
> >>>>>>>  include/drm/drm_gem_shmem_helper.h            | 36 +++++------
> >>>>>>>  9 files changed, 64 insertions(+), 64 deletions(-)
> >>>>>>>
> >>>>>>> diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c
> >>>>>>> index 0d61f2b3e213..154585ddae08 100644
> >>>>>>> --- a/drivers/gpu/drm/drm_gem_shmem_helper.c
> >>>>>>> +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c
> >>>>>>> @@ -43,8 +43,8 @@ static const struct drm_gem_object_funcs drm_gem_shmem_funcs = {
> >>>>>>>  	.pin = drm_gem_shmem_object_pin,
> >>>>>>>  	.unpin = drm_gem_shmem_object_unpin,
> >>>>>>>  	.get_sg_table = drm_gem_shmem_object_get_sg_table,
> >>>>>>> -	.vmap = drm_gem_shmem_object_vmap,
> >>>>>>> -	.vunmap = drm_gem_shmem_object_vunmap,
> >>>>>>> +	.vmap = drm_gem_shmem_object_vmap_locked,
> >>>>>>> +	.vunmap = drm_gem_shmem_object_vunmap_locked,        
> >>>>>>
> >>>>>> While I think we should indeed be consistent with the names, I would
> >>>>>> also expect helpers to get the locking right by default.      
> >>>>>
> >>>>> Wait, actually I think this patch does what you suggest already. The
> >>>>> _locked() prefix tells the caller: "you should take care of the locking,
> >>>>> I expect the lock to be held when this is hook/function is called". So
> >>>>> helpers without the _locked() prefix take care of the locking (which I
> >>>>> guess matches your 'helpers get the locking right' expectation), and
> >>>>> those with the _locked() prefix don't.      
> >>>>
> >>>> What I meant by "getting the locking right" is indeed a bit ambiguous,
> >>>> sorry. What I'm trying to say I guess is that, in this particular case,
> >>>> I don't think you can expect the vmap implementation to be called with
> >>>> or without the locks held. The doc for that function will say that it's
> >>>> either one or the other, but not both.
> >>>>
> >>>> So helpers should follow what is needed to provide a default vmap/vunmap
> >>>> implementation, including what locking is expected from a vmap/vunmap
> >>>> implementation.    
> >>>
> >>> Hm, yeah, I think that's a matter of taste. When locking is often
> >>> deferrable, like it is in DRM, I find it beneficial for funcions and
> >>> function pointers to reflect the locking scheme, rather than relying on
> >>> people properly reading the doc, especially when this is the only
> >>> outlier in the group of drm_gem_object_funcs we already have, and it's
> >>> not event documented at the drm_gem_object_funcs level [1] :P.
> >>>     
> >>>>
> >>>> If that means that vmap is always called with the locks taken, then
> >>>> drm_gem_shmem_object_vmap can just assume that it will be called with
> >>>> the locks taken and there's no need to mention it in the name (and you
> >>>> can probably sprinkle a couple of lockdep assertion to make sure the
> >>>> locking is indeed consistent).    
> >>>
> >>> Things get very confusing when you end up having drm_gem_shmem helpers
> >>> that are suffixed with _locked() to encode the fact locking is the
> >>> caller's responsibility and no suffix for the
> >>> callee-takes-care-of-the-locking semantics, while other helpers that are
> >>> not suffixed at all actually implement the
> >>> caller-should-take-care-of-the-locking semantics.
> >>>     
> >>>>    
> >>>>>> I'm not sure how reasonable it is, but I think I'd prefer to turn this
> >>>>>> around and keep the drm_gem_shmem_object_vmap/unmap helpers name, and
> >>>>>> convert whatever function needs to be converted to the unlock suffix so
> >>>>>> we get a consistent naming.      
> >>>>>
> >>>>> That would be an _unlocked() prefix if we do it the other way around. I
> >>>>> think the main confusion comes from the names of the hooks in
> >>>>> drm_gem_shmem_funcs. Some of them, like drm_gem_shmem_funcs::v[un]map()
> >>>>> are called with the GEM resv lock held, and locking is handled by the
> >>>>> core, others, like drm_gem_shmem_funcs::[un]pin() are called
> >>>>> without the GEM resv lock held, and locking is deferred to the
> >>>>> implementation. As I said, I don't mind prefixing hooks/helpers with
> >>>>> _unlocked() for those that take care of the locking, and no prefix for
> >>>>> those that expects locks to be held, as long as it's consistent, but I
> >>>>> just wanted to make sure we're on the same page :-).      
> >>>>
> >>>> What about _nolock then? It's the same number of characters than
> >>>> _locked, plus it expresses what the function is (not) doing, not what
> >>>> context it's supposed to be called in?    
> >>>
> >>> Just did a quick
> >>>
> >>>   git grep _nolock drivers/gpu/drm
> >>>
> >>> and it returns zero result, where the _locked/_unlocked pattern seems
> >>> to already be widely used. Not saying we shouldn't change that, but it
> >>> doesn't feel like a change we should do as part of this series.
> >>>
> >>> Regards,
> >>>
> >>> Boris
> >>>
> >>> [1]https://elixir.bootlin.com/linux/v6.7-rc3/source/include/drm/drm_gem.h#L155    
> >>
> >> I'm fine with dropping the _locked() postfix from the common GEM helpers
> >> and documenting the locking rule in drm_gem. Thank you all for the
> >> suggestions :)  
> > 
> > Sorry to disagree, but I think a proper function name/suffix is
> > sometimes worth a few lines of doc. Not saying we should do one or the
> > other, I think we should do both. But when I see a function suffixed
> > _locked, _unlocked or _nolock, I can immediately tell if this function
> > defers the locking to the caller or not, and then go check which lock
> > in the function doc.
> > 
> > And the second thing I'm not happy with, is the fact we go back to an
> > inconsistent naming in drm_gem_shmem_helper.c, where some functions
> > deferring the locking to the caller are suffixed _locked and others are
> > not, because ultimately, you need a different name when you expose the
> > two variants...  
> 
> By the `common GEM helpers` I meant the .vmap drm-shmem common helpers
> used for drm_gem_object_funcs, like was suggested by Maxime. The rest of
> functions will retain the _locked part. Sorry for the confusion :)

Well, even if it's just
s/drm_gem_shmem_v[un]map_locked/drm_gem_shmem_v[un]map/, it's still
inconsistent with the rest of the helpers we have there (_locked suffix
for those deferring the locking to the caller, and no suffix when the
lock is taken by the helper). To be clear, I won't block the patch
because of that, but I still think this is the wrong move...

Re: [PATCH v18 04/26] drm/shmem-helper: Refactor locked/unlocked functions

[Maxime Ripard]

[-- Attachment #1: Type: text/plain, Size: 8162 bytes --]

On Wed, Nov 29, 2023 at 08:53:30AM +0100, Boris Brezillon wrote:
> On Wed, 29 Nov 2023 01:05:14 +0300
> Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:
> 
> > On 11/28/23 15:37, Boris Brezillon wrote:
> > > On Tue, 28 Nov 2023 12:14:42 +0100
> > > Maxime Ripard <mripard@kernel.org> wrote:
> > >   
> > >> Hi,
> > >>
> > >> On Fri, Nov 24, 2023 at 11:59:11AM +0100, Boris Brezillon wrote:  
> > >>> On Fri, 24 Nov 2023 11:40:06 +0100
> > >>> Maxime Ripard <mripard@kernel.org> wrote:
> > >>>     
> > >>>> On Mon, Oct 30, 2023 at 02:01:43AM +0300, Dmitry Osipenko wrote:    
> > >>>>> Add locked and remove unlocked postfixes from drm-shmem function names,
> > >>>>> making names consistent with the drm/gem core code.
> > >>>>>
> > >>>>> Reviewed-by: Boris Brezillon <boris.brezillon@collabora.com>
> > >>>>> Suggested-by: Boris Brezillon <boris.brezillon@collabora.com>
> > >>>>> Signed-off-by: Dmitry Osipenko <dmitry.osipenko@collabora.com>      
> > >>>>
> > >>>> This contradicts my earlier ack on a patch but...
> > >>>>     
> > >>>>> ---
> > >>>>>  drivers/gpu/drm/drm_gem_shmem_helper.c        | 64 +++++++++----------
> > >>>>>  drivers/gpu/drm/lima/lima_gem.c               |  8 +--
> > >>>>>  drivers/gpu/drm/panfrost/panfrost_drv.c       |  2 +-
> > >>>>>  drivers/gpu/drm/panfrost/panfrost_gem.c       |  6 +-
> > >>>>>  .../gpu/drm/panfrost/panfrost_gem_shrinker.c  |  2 +-
> > >>>>>  drivers/gpu/drm/panfrost/panfrost_mmu.c       |  2 +-
> > >>>>>  drivers/gpu/drm/v3d/v3d_bo.c                  |  4 +-
> > >>>>>  drivers/gpu/drm/virtio/virtgpu_object.c       |  4 +-
> > >>>>>  include/drm/drm_gem_shmem_helper.h            | 36 +++++------
> > >>>>>  9 files changed, 64 insertions(+), 64 deletions(-)
> > >>>>>
> > >>>>> diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c
> > >>>>> index 0d61f2b3e213..154585ddae08 100644
> > >>>>> --- a/drivers/gpu/drm/drm_gem_shmem_helper.c
> > >>>>> +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c
> > >>>>> @@ -43,8 +43,8 @@ static const struct drm_gem_object_funcs drm_gem_shmem_funcs = {
> > >>>>>  	.pin = drm_gem_shmem_object_pin,
> > >>>>>  	.unpin = drm_gem_shmem_object_unpin,
> > >>>>>  	.get_sg_table = drm_gem_shmem_object_get_sg_table,
> > >>>>> -	.vmap = drm_gem_shmem_object_vmap,
> > >>>>> -	.vunmap = drm_gem_shmem_object_vunmap,
> > >>>>> +	.vmap = drm_gem_shmem_object_vmap_locked,
> > >>>>> +	.vunmap = drm_gem_shmem_object_vunmap_locked,      
> > >>>>
> > >>>> While I think we should indeed be consistent with the names, I would
> > >>>> also expect helpers to get the locking right by default.    
> > >>>
> > >>> Wait, actually I think this patch does what you suggest already. The
> > >>> _locked() prefix tells the caller: "you should take care of the locking,
> > >>> I expect the lock to be held when this is hook/function is called". So
> > >>> helpers without the _locked() prefix take care of the locking (which I
> > >>> guess matches your 'helpers get the locking right' expectation), and
> > >>> those with the _locked() prefix don't.    
> > >>
> > >> What I meant by "getting the locking right" is indeed a bit ambiguous,
> > >> sorry. What I'm trying to say I guess is that, in this particular case,
> > >> I don't think you can expect the vmap implementation to be called with
> > >> or without the locks held. The doc for that function will say that it's
> > >> either one or the other, but not both.
> > >>
> > >> So helpers should follow what is needed to provide a default vmap/vunmap
> > >> implementation, including what locking is expected from a vmap/vunmap
> > >> implementation.  
> > > 
> > > Hm, yeah, I think that's a matter of taste. When locking is often
> > > deferrable, like it is in DRM, I find it beneficial for funcions and
> > > function pointers to reflect the locking scheme, rather than relying on
> > > people properly reading the doc, especially when this is the only
> > > outlier in the group of drm_gem_object_funcs we already have, and it's
> > > not event documented at the drm_gem_object_funcs level [1] :P.
> > >   
> > >>
> > >> If that means that vmap is always called with the locks taken, then
> > >> drm_gem_shmem_object_vmap can just assume that it will be called with
> > >> the locks taken and there's no need to mention it in the name (and you
> > >> can probably sprinkle a couple of lockdep assertion to make sure the
> > >> locking is indeed consistent).  
> > > 
> > > Things get very confusing when you end up having drm_gem_shmem helpers
> > > that are suffixed with _locked() to encode the fact locking is the
> > > caller's responsibility and no suffix for the
> > > callee-takes-care-of-the-locking semantics, while other helpers that are
> > > not suffixed at all actually implement the
> > > caller-should-take-care-of-the-locking semantics.
> > >   
> > >>  
> > >>>> I'm not sure how reasonable it is, but I think I'd prefer to turn this
> > >>>> around and keep the drm_gem_shmem_object_vmap/unmap helpers name, and
> > >>>> convert whatever function needs to be converted to the unlock suffix so
> > >>>> we get a consistent naming.    
> > >>>
> > >>> That would be an _unlocked() prefix if we do it the other way around. I
> > >>> think the main confusion comes from the names of the hooks in
> > >>> drm_gem_shmem_funcs. Some of them, like drm_gem_shmem_funcs::v[un]map()
> > >>> are called with the GEM resv lock held, and locking is handled by the
> > >>> core, others, like drm_gem_shmem_funcs::[un]pin() are called
> > >>> without the GEM resv lock held, and locking is deferred to the
> > >>> implementation. As I said, I don't mind prefixing hooks/helpers with
> > >>> _unlocked() for those that take care of the locking, and no prefix for
> > >>> those that expects locks to be held, as long as it's consistent, but I
> > >>> just wanted to make sure we're on the same page :-).    
> > >>
> > >> What about _nolock then? It's the same number of characters than
> > >> _locked, plus it expresses what the function is (not) doing, not what
> > >> context it's supposed to be called in?  
> > > 
> > > Just did a quick
> > > 
> > >   git grep _nolock drivers/gpu/drm
> > > 
> > > and it returns zero result, where the _locked/_unlocked pattern seems
> > > to already be widely used. Not saying we shouldn't change that, but it
> > > doesn't feel like a change we should do as part of this series.
> > > 
> > > Regards,
> > > 
> > > Boris
> > > 
> > > [1]https://elixir.bootlin.com/linux/v6.7-rc3/source/include/drm/drm_gem.h#L155  
> > 
> > I'm fine with dropping the _locked() postfix from the common GEM helpers
> > and documenting the locking rule in drm_gem. Thank you all for the
> > suggestions :)
> 
> Sorry to disagree, but I think a proper function name/suffix is
> sometimes worth a few lines of doc. Not saying we should do one or the
> other, I think we should do both. But when I see a function suffixed
> _locked, _unlocked or _nolock, I can immediately tell if this function
> defers the locking to the caller or not, and then go check which lock
> in the function doc.
> 
> And the second thing I'm not happy with, is the fact we go back to an
> inconsistent naming in drm_gem_shmem_helper.c, where some functions
> deferring the locking to the caller are suffixed _locked and others are
> not, because ultimately, you need a different name when you expose the
> two variants...

I guess one of the point I was trying to make was also: why do you need
both?

If one is better than the other (whatever better means here), then all
drivers should use it.

The counterpart being that if provided a choice, you can be sure that a
lot of people will get it wrong. The one example I have in mind for
example was the drm_atomic_helper_commit_tail vs
drm_atomic_helper_commit_tail_rpm. The latter is now widely used, and
most of it is cargo-cult.

I think you were referring to the locks being deferred vs taken right
now before, why do we need to have the choice between the two?

Maxime

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 228 bytes --]

Re: [PATCH v18 04/26] drm/shmem-helper: Refactor locked/unlocked functions

[Boris Brezillon]

On Wed, 29 Nov 2023 14:09:47 +0100
Maxime Ripard <mripard@kernel.org> wrote:

> On Wed, Nov 29, 2023 at 08:53:30AM +0100, Boris Brezillon wrote:
> > On Wed, 29 Nov 2023 01:05:14 +0300
> > Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:
> >   
> > > On 11/28/23 15:37, Boris Brezillon wrote:  
> > > > On Tue, 28 Nov 2023 12:14:42 +0100
> > > > Maxime Ripard <mripard@kernel.org> wrote:
> > > >     
> > > >> Hi,
> > > >>
> > > >> On Fri, Nov 24, 2023 at 11:59:11AM +0100, Boris Brezillon wrote:    
> > > >>> On Fri, 24 Nov 2023 11:40:06 +0100
> > > >>> Maxime Ripard <mripard@kernel.org> wrote:
> > > >>>       
> > > >>>> On Mon, Oct 30, 2023 at 02:01:43AM +0300, Dmitry Osipenko wrote:      
> > > >>>>> Add locked and remove unlocked postfixes from drm-shmem function names,
> > > >>>>> making names consistent with the drm/gem core code.
> > > >>>>>
> > > >>>>> Reviewed-by: Boris Brezillon <boris.brezillon@collabora.com>
> > > >>>>> Suggested-by: Boris Brezillon <boris.brezillon@collabora.com>
> > > >>>>> Signed-off-by: Dmitry Osipenko <dmitry.osipenko@collabora.com>        
> > > >>>>
> > > >>>> This contradicts my earlier ack on a patch but...
> > > >>>>       
> > > >>>>> ---
> > > >>>>>  drivers/gpu/drm/drm_gem_shmem_helper.c        | 64 +++++++++----------
> > > >>>>>  drivers/gpu/drm/lima/lima_gem.c               |  8 +--
> > > >>>>>  drivers/gpu/drm/panfrost/panfrost_drv.c       |  2 +-
> > > >>>>>  drivers/gpu/drm/panfrost/panfrost_gem.c       |  6 +-
> > > >>>>>  .../gpu/drm/panfrost/panfrost_gem_shrinker.c  |  2 +-
> > > >>>>>  drivers/gpu/drm/panfrost/panfrost_mmu.c       |  2 +-
> > > >>>>>  drivers/gpu/drm/v3d/v3d_bo.c                  |  4 +-
> > > >>>>>  drivers/gpu/drm/virtio/virtgpu_object.c       |  4 +-
> > > >>>>>  include/drm/drm_gem_shmem_helper.h            | 36 +++++------
> > > >>>>>  9 files changed, 64 insertions(+), 64 deletions(-)
> > > >>>>>
> > > >>>>> diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c
> > > >>>>> index 0d61f2b3e213..154585ddae08 100644
> > > >>>>> --- a/drivers/gpu/drm/drm_gem_shmem_helper.c
> > > >>>>> +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c
> > > >>>>> @@ -43,8 +43,8 @@ static const struct drm_gem_object_funcs drm_gem_shmem_funcs = {
> > > >>>>>  	.pin = drm_gem_shmem_object_pin,
> > > >>>>>  	.unpin = drm_gem_shmem_object_unpin,
> > > >>>>>  	.get_sg_table = drm_gem_shmem_object_get_sg_table,
> > > >>>>> -	.vmap = drm_gem_shmem_object_vmap,
> > > >>>>> -	.vunmap = drm_gem_shmem_object_vunmap,
> > > >>>>> +	.vmap = drm_gem_shmem_object_vmap_locked,
> > > >>>>> +	.vunmap = drm_gem_shmem_object_vunmap_locked,        
> > > >>>>
> > > >>>> While I think we should indeed be consistent with the names, I would
> > > >>>> also expect helpers to get the locking right by default.      
> > > >>>
> > > >>> Wait, actually I think this patch does what you suggest already. The
> > > >>> _locked() prefix tells the caller: "you should take care of the locking,
> > > >>> I expect the lock to be held when this is hook/function is called". So
> > > >>> helpers without the _locked() prefix take care of the locking (which I
> > > >>> guess matches your 'helpers get the locking right' expectation), and
> > > >>> those with the _locked() prefix don't.      
> > > >>
> > > >> What I meant by "getting the locking right" is indeed a bit ambiguous,
> > > >> sorry. What I'm trying to say I guess is that, in this particular case,
> > > >> I don't think you can expect the vmap implementation to be called with
> > > >> or without the locks held. The doc for that function will say that it's
> > > >> either one or the other, but not both.
> > > >>
> > > >> So helpers should follow what is needed to provide a default vmap/vunmap
> > > >> implementation, including what locking is expected from a vmap/vunmap
> > > >> implementation.    
> > > > 
> > > > Hm, yeah, I think that's a matter of taste. When locking is often
> > > > deferrable, like it is in DRM, I find it beneficial for funcions and
> > > > function pointers to reflect the locking scheme, rather than relying on
> > > > people properly reading the doc, especially when this is the only
> > > > outlier in the group of drm_gem_object_funcs we already have, and it's
> > > > not event documented at the drm_gem_object_funcs level [1] :P.
> > > >     
> > > >>
> > > >> If that means that vmap is always called with the locks taken, then
> > > >> drm_gem_shmem_object_vmap can just assume that it will be called with
> > > >> the locks taken and there's no need to mention it in the name (and you
> > > >> can probably sprinkle a couple of lockdep assertion to make sure the
> > > >> locking is indeed consistent).    
> > > > 
> > > > Things get very confusing when you end up having drm_gem_shmem helpers
> > > > that are suffixed with _locked() to encode the fact locking is the
> > > > caller's responsibility and no suffix for the
> > > > callee-takes-care-of-the-locking semantics, while other helpers that are
> > > > not suffixed at all actually implement the
> > > > caller-should-take-care-of-the-locking semantics.
> > > >     
> > > >>    
> > > >>>> I'm not sure how reasonable it is, but I think I'd prefer to turn this
> > > >>>> around and keep the drm_gem_shmem_object_vmap/unmap helpers name, and
> > > >>>> convert whatever function needs to be converted to the unlock suffix so
> > > >>>> we get a consistent naming.      
> > > >>>
> > > >>> That would be an _unlocked() prefix if we do it the other way around. I
> > > >>> think the main confusion comes from the names of the hooks in
> > > >>> drm_gem_shmem_funcs. Some of them, like drm_gem_shmem_funcs::v[un]map()
> > > >>> are called with the GEM resv lock held, and locking is handled by the
> > > >>> core, others, like drm_gem_shmem_funcs::[un]pin() are called
> > > >>> without the GEM resv lock held, and locking is deferred to the
> > > >>> implementation. As I said, I don't mind prefixing hooks/helpers with
> > > >>> _unlocked() for those that take care of the locking, and no prefix for
> > > >>> those that expects locks to be held, as long as it's consistent, but I
> > > >>> just wanted to make sure we're on the same page :-).      
> > > >>
> > > >> What about _nolock then? It's the same number of characters than
> > > >> _locked, plus it expresses what the function is (not) doing, not what
> > > >> context it's supposed to be called in?    
> > > > 
> > > > Just did a quick
> > > > 
> > > >   git grep _nolock drivers/gpu/drm
> > > > 
> > > > and it returns zero result, where the _locked/_unlocked pattern seems
> > > > to already be widely used. Not saying we shouldn't change that, but it
> > > > doesn't feel like a change we should do as part of this series.
> > > > 
> > > > Regards,
> > > > 
> > > > Boris
> > > > 
> > > > [1]https://elixir.bootlin.com/linux/v6.7-rc3/source/include/drm/drm_gem.h#L155    
> > > 
> > > I'm fine with dropping the _locked() postfix from the common GEM helpers
> > > and documenting the locking rule in drm_gem. Thank you all for the
> > > suggestions :)  
> > 
> > Sorry to disagree, but I think a proper function name/suffix is
> > sometimes worth a few lines of doc. Not saying we should do one or the
> > other, I think we should do both. But when I see a function suffixed
> > _locked, _unlocked or _nolock, I can immediately tell if this function
> > defers the locking to the caller or not, and then go check which lock
> > in the function doc.
> > 
> > And the second thing I'm not happy with, is the fact we go back to an
> > inconsistent naming in drm_gem_shmem_helper.c, where some functions
> > deferring the locking to the caller are suffixed _locked and others are
> > not, because ultimately, you need a different name when you expose the
> > two variants...  
> 
> I guess one of the point I was trying to make was also: why do you need
> both?
> 
> If one is better than the other (whatever better means here), then all
> drivers should use it.
> 
> The counterpart being that if provided a choice, you can be sure that a
> lot of people will get it wrong. The one example I have in mind for
> example was the drm_atomic_helper_commit_tail vs
> drm_atomic_helper_commit_tail_rpm. The latter is now widely used, and
> most of it is cargo-cult.
> 
> I think you were referring to the locks being deferred vs taken right
> now before, why do we need to have the choice between the two?

Because DRM locking is complex, and you sometimes have to call some
helpers in a context where you already hold the GEM dma_resv lock.
That's not the case for _v[un]map(), because the core always takes the
lock for us if we call drm_gem_vmap_unlocked(). Now, let's assume we
drop the _locked() suffix on drm_gem_shmem_v[un]map(), but keep it on
other helpers that need both variants. This results in an inconsistent
naming scheme inside the same source file, which I find utterly
confusing.

Note that the initial reason I asked Dmitry if he could add the
_locked suffix to drm_gem_shmem_vmap() is because I started using
drm_gem_shmem_vmap() in powervr, before realizing this version wasn't
taking the lock, and I should have used drm_gem_vmap_unlocked()
instead, so this is not something I'm making up. Not saying the
confusion only comes from the naming, because the various layers of
indirection we have clearly don't help, but having a name reflecting
the fact the locking is deferred to the caller would have helped, I
think.

Re: [PATCH v18 04/26] drm/shmem-helper: Refactor locked/unlocked functions

[Maxime Ripard]

[-- Attachment #1: Type: text/plain, Size: 11283 bytes --]

On Wed, Nov 29, 2023 at 02:46:09PM +0100, Boris Brezillon wrote:
> On Wed, 29 Nov 2023 14:09:47 +0100
> Maxime Ripard <mripard@kernel.org> wrote:
> 
> > On Wed, Nov 29, 2023 at 08:53:30AM +0100, Boris Brezillon wrote:
> > > On Wed, 29 Nov 2023 01:05:14 +0300
> > > Dmitry Osipenko <dmitry.osipenko@collabora.com> wrote:
> > >   
> > > > On 11/28/23 15:37, Boris Brezillon wrote:  
> > > > > On Tue, 28 Nov 2023 12:14:42 +0100
> > > > > Maxime Ripard <mripard@kernel.org> wrote:
> > > > >     
> > > > >> Hi,
> > > > >>
> > > > >> On Fri, Nov 24, 2023 at 11:59:11AM +0100, Boris Brezillon wrote:    
> > > > >>> On Fri, 24 Nov 2023 11:40:06 +0100
> > > > >>> Maxime Ripard <mripard@kernel.org> wrote:
> > > > >>>       
> > > > >>>> On Mon, Oct 30, 2023 at 02:01:43AM +0300, Dmitry Osipenko wrote:      
> > > > >>>>> Add locked and remove unlocked postfixes from drm-shmem function names,
> > > > >>>>> making names consistent with the drm/gem core code.
> > > > >>>>>
> > > > >>>>> Reviewed-by: Boris Brezillon <boris.brezillon@collabora.com>
> > > > >>>>> Suggested-by: Boris Brezillon <boris.brezillon@collabora.com>
> > > > >>>>> Signed-off-by: Dmitry Osipenko <dmitry.osipenko@collabora.com>        
> > > > >>>>
> > > > >>>> This contradicts my earlier ack on a patch but...
> > > > >>>>       
> > > > >>>>> ---
> > > > >>>>>  drivers/gpu/drm/drm_gem_shmem_helper.c        | 64 +++++++++----------
> > > > >>>>>  drivers/gpu/drm/lima/lima_gem.c               |  8 +--
> > > > >>>>>  drivers/gpu/drm/panfrost/panfrost_drv.c       |  2 +-
> > > > >>>>>  drivers/gpu/drm/panfrost/panfrost_gem.c       |  6 +-
> > > > >>>>>  .../gpu/drm/panfrost/panfrost_gem_shrinker.c  |  2 +-
> > > > >>>>>  drivers/gpu/drm/panfrost/panfrost_mmu.c       |  2 +-
> > > > >>>>>  drivers/gpu/drm/v3d/v3d_bo.c                  |  4 +-
> > > > >>>>>  drivers/gpu/drm/virtio/virtgpu_object.c       |  4 +-
> > > > >>>>>  include/drm/drm_gem_shmem_helper.h            | 36 +++++------
> > > > >>>>>  9 files changed, 64 insertions(+), 64 deletions(-)
> > > > >>>>>
> > > > >>>>> diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c
> > > > >>>>> index 0d61f2b3e213..154585ddae08 100644
> > > > >>>>> --- a/drivers/gpu/drm/drm_gem_shmem_helper.c
> > > > >>>>> +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c
> > > > >>>>> @@ -43,8 +43,8 @@ static const struct drm_gem_object_funcs drm_gem_shmem_funcs = {
> > > > >>>>>  	.pin = drm_gem_shmem_object_pin,
> > > > >>>>>  	.unpin = drm_gem_shmem_object_unpin,
> > > > >>>>>  	.get_sg_table = drm_gem_shmem_object_get_sg_table,
> > > > >>>>> -	.vmap = drm_gem_shmem_object_vmap,
> > > > >>>>> -	.vunmap = drm_gem_shmem_object_vunmap,
> > > > >>>>> +	.vmap = drm_gem_shmem_object_vmap_locked,
> > > > >>>>> +	.vunmap = drm_gem_shmem_object_vunmap_locked,        
> > > > >>>>
> > > > >>>> While I think we should indeed be consistent with the names, I would
> > > > >>>> also expect helpers to get the locking right by default.      
> > > > >>>
> > > > >>> Wait, actually I think this patch does what you suggest already. The
> > > > >>> _locked() prefix tells the caller: "you should take care of the locking,
> > > > >>> I expect the lock to be held when this is hook/function is called". So
> > > > >>> helpers without the _locked() prefix take care of the locking (which I
> > > > >>> guess matches your 'helpers get the locking right' expectation), and
> > > > >>> those with the _locked() prefix don't.      
> > > > >>
> > > > >> What I meant by "getting the locking right" is indeed a bit ambiguous,
> > > > >> sorry. What I'm trying to say I guess is that, in this particular case,
> > > > >> I don't think you can expect the vmap implementation to be called with
> > > > >> or without the locks held. The doc for that function will say that it's
> > > > >> either one or the other, but not both.
> > > > >>
> > > > >> So helpers should follow what is needed to provide a default vmap/vunmap
> > > > >> implementation, including what locking is expected from a vmap/vunmap
> > > > >> implementation.    
> > > > > 
> > > > > Hm, yeah, I think that's a matter of taste. When locking is often
> > > > > deferrable, like it is in DRM, I find it beneficial for funcions and
> > > > > function pointers to reflect the locking scheme, rather than relying on
> > > > > people properly reading the doc, especially when this is the only
> > > > > outlier in the group of drm_gem_object_funcs we already have, and it's
> > > > > not event documented at the drm_gem_object_funcs level [1] :P.
> > > > >     
> > > > >>
> > > > >> If that means that vmap is always called with the locks taken, then
> > > > >> drm_gem_shmem_object_vmap can just assume that it will be called with
> > > > >> the locks taken and there's no need to mention it in the name (and you
> > > > >> can probably sprinkle a couple of lockdep assertion to make sure the
> > > > >> locking is indeed consistent).    
> > > > > 
> > > > > Things get very confusing when you end up having drm_gem_shmem helpers
> > > > > that are suffixed with _locked() to encode the fact locking is the
> > > > > caller's responsibility and no suffix for the
> > > > > callee-takes-care-of-the-locking semantics, while other helpers that are
> > > > > not suffixed at all actually implement the
> > > > > caller-should-take-care-of-the-locking semantics.
> > > > >     
> > > > >>    
> > > > >>>> I'm not sure how reasonable it is, but I think I'd prefer to turn this
> > > > >>>> around and keep the drm_gem_shmem_object_vmap/unmap helpers name, and
> > > > >>>> convert whatever function needs to be converted to the unlock suffix so
> > > > >>>> we get a consistent naming.      
> > > > >>>
> > > > >>> That would be an _unlocked() prefix if we do it the other way around. I
> > > > >>> think the main confusion comes from the names of the hooks in
> > > > >>> drm_gem_shmem_funcs. Some of them, like drm_gem_shmem_funcs::v[un]map()
> > > > >>> are called with the GEM resv lock held, and locking is handled by the
> > > > >>> core, others, like drm_gem_shmem_funcs::[un]pin() are called
> > > > >>> without the GEM resv lock held, and locking is deferred to the
> > > > >>> implementation. As I said, I don't mind prefixing hooks/helpers with
> > > > >>> _unlocked() for those that take care of the locking, and no prefix for
> > > > >>> those that expects locks to be held, as long as it's consistent, but I
> > > > >>> just wanted to make sure we're on the same page :-).      
> > > > >>
> > > > >> What about _nolock then? It's the same number of characters than
> > > > >> _locked, plus it expresses what the function is (not) doing, not what
> > > > >> context it's supposed to be called in?    
> > > > > 
> > > > > Just did a quick
> > > > > 
> > > > >   git grep _nolock drivers/gpu/drm
> > > > > 
> > > > > and it returns zero result, where the _locked/_unlocked pattern seems
> > > > > to already be widely used. Not saying we shouldn't change that, but it
> > > > > doesn't feel like a change we should do as part of this series.
> > > > > 
> > > > > Regards,
> > > > > 
> > > > > Boris
> > > > > 
> > > > > [1]https://elixir.bootlin.com/linux/v6.7-rc3/source/include/drm/drm_gem.h#L155    
> > > > 
> > > > I'm fine with dropping the _locked() postfix from the common GEM helpers
> > > > and documenting the locking rule in drm_gem. Thank you all for the
> > > > suggestions :)  
> > > 
> > > Sorry to disagree, but I think a proper function name/suffix is
> > > sometimes worth a few lines of doc. Not saying we should do one or the
> > > other, I think we should do both. But when I see a function suffixed
> > > _locked, _unlocked or _nolock, I can immediately tell if this function
> > > defers the locking to the caller or not, and then go check which lock
> > > in the function doc.
> > > 
> > > And the second thing I'm not happy with, is the fact we go back to an
> > > inconsistent naming in drm_gem_shmem_helper.c, where some functions
> > > deferring the locking to the caller are suffixed _locked and others are
> > > not, because ultimately, you need a different name when you expose the
> > > two variants...  
> > 
> > I guess one of the point I was trying to make was also: why do you need
> > both?
> > 
> > If one is better than the other (whatever better means here), then all
> > drivers should use it.
> > 
> > The counterpart being that if provided a choice, you can be sure that a
> > lot of people will get it wrong. The one example I have in mind for
> > example was the drm_atomic_helper_commit_tail vs
> > drm_atomic_helper_commit_tail_rpm. The latter is now widely used, and
> > most of it is cargo-cult.
> > 
> > I think you were referring to the locks being deferred vs taken right
> > now before, why do we need to have the choice between the two?
>
> Because DRM locking is complex, and you sometimes have to call some
> helpers in a context where you already hold the GEM dma_resv lock.
> That's not the case for _v[un]map(), because the core always takes the
> lock for us if we call drm_gem_vmap_unlocked().

Ok

> Now, let's assume we drop the _locked() suffix on
> drm_gem_shmem_v[un]map(), but keep it on other helpers that need both
> variants. This results in an inconsistent naming scheme inside the
> same source file, which I find utterly confusing.
>
> Note that the initial reason I asked Dmitry if he could add the
> _locked suffix to drm_gem_shmem_vmap() is because I started using
> drm_gem_shmem_vmap() in powervr, before realizing this version wasn't
> taking the lock, and I should have used drm_gem_vmap_unlocked()
> instead, so this is not something I'm making up.

Sorry if I gave you the impression I thought that you're making that up,
I'm not.

Thanks for the explanation btw, I think I get what you're saying now:

 - drm_gem_shmem_vmap() is never taking the locks because the core
   expects to take them before calling them.

 - drm_gem_shmem_vunmap() is never taking the locks because the core
   expects to take them before calling them.

 - Some other code path can still call those helpers in drivers, and the
   locking isn't handled by the core anymore.

 - We now have _vmap/vunmap_unlocked functions to take those locks for
   those code paths

 - And the variant names are now confusing, making people use the
   lockless version in situations where they should have use the locked
   one.

Is that a correct summary?

If so, then I agree that we need to change the name.

We discussed it some more on IRC, and we agree that the "default"
function should handle the locking properly and that's what the most
common case should use.

So that means than drm_gem_shmem_vmap/vunmap() should take the lock
itself, and drm_gem_shmem_vmap/vunmap_nolock/unlocked never does.

I think I'd prefer the nolock variant over unlocked still.

And I also think we can improve the documentation and add lockdep calls
to make sure that the difference between variants is clear in the doc,
and if someone still get confused we can catch it.

Does that sound like a plan?

Maxime

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 228 bytes --]

Re: [PATCH v18 04/26] drm/shmem-helper: Refactor locked/unlocked functions

[Boris Brezillon]

On Wed, 29 Nov 2023 16:15:27 +0100
Maxime Ripard <mripard@kernel.org> wrote:

> > Now, let's assume we drop the _locked() suffix on
> > drm_gem_shmem_v[un]map(), but keep it on other helpers that need both
> > variants. This results in an inconsistent naming scheme inside the
> > same source file, which I find utterly confusing.
> >
> > Note that the initial reason I asked Dmitry if he could add the
> > _locked suffix to drm_gem_shmem_vmap() is because I started using
> > drm_gem_shmem_vmap() in powervr, before realizing this version wasn't
> > taking the lock, and I should have used drm_gem_vmap_unlocked()
> > instead, so this is not something I'm making up.  
> 
> Sorry if I gave you the impression I thought that you're making that up,
> I'm not.
> 
> Thanks for the explanation btw, I think I get what you're saying now:
> 
>  - drm_gem_shmem_vmap() is never taking the locks because the core
>    expects to take them before calling them.
> 
>  - drm_gem_shmem_vunmap() is never taking the locks because the core
>    expects to take them before calling them.

Correct.

> 
>  - Some other code path can still call those helpers in drivers, and the
>    locking isn't handled by the core anymore.

They can, if they want to v[un]map a BO and they already acquired the
GEM resv lock. But I'm not sure anyone needs to do that yet. The main
reason for exposing these helpers is if one driver needs to overload the
default gem_shmem_funcs.

> 
>  - We now have _vmap/vunmap_unlocked functions to take those locks for
>    those code paths

We don't have drm_gem_shmem_vmap/vunmap_unlocked(), we have
drm_gem_shmem_vmap/vunmap_locked(), which can be called directly, but
are mainly used to populate the drm_gem_object_funcs vtable. If drivers
want to v[un]map in a path where the resv lock is not held, they should
call drm_gem_vmap/vunmap_unlocked() (which are renamed
drm_gem_vmap/vunmap() in patch 1 of this series). Mind the **drm_gem_**
vs **drm_gem_shmem_** difference in the helper names. drm_gem_ helpers
are provided by drm_gem.c and call drm_gem_object_funcs callback, which
are supposed to be populated with drm_gem_shmem helpers.

> 
>  - And the variant names are now confusing, making people use the
>    lockless version in situations where they should have use the locked
>    one.

That's what happened to me, at least.

> 
> Is that a correct summary?

Almost ;-).

> 
> If so, then I agree that we need to change the name.

Cool.

> 
> We discussed it some more on IRC, and we agree that the "default"
> function should handle the locking properly and that's what the most
> common case should use.

Agree if by 'default' you mean the lock is always acquired by the
helper, not 'let's decide based on what users do most of the time with
this specific helper', because otherwise we'd be back to a situation
where the name doesn't clearly encode the function behavior.

> 
> So that means than drm_gem_shmem_vmap/vunmap() should take the lock
> itself, and drm_gem_shmem_vmap/vunmap_nolock/unlocked never does.

Not sure we have a need for drm_gem_shmem_vmap/vunmap(), but if we ever
add such helpers, they would acquire the resv lock, indeed.

Just to be clear, _nolock == _locked in the current semantics :-).
_nolock means 'don't take the lock', and _locked means 'lock is already
held'.

> 
> I think I'd prefer the nolock variant over unlocked still.

Okay, that means s/_locked/_nolock/ in drm_gem_shmem_helpers.{c,h}, I
guess.

> 
> And I also think we can improve the documentation and add lockdep calls

Lockdep asserts are already there, I think.

> to make sure that the difference between variants is clear in the doc,
> and if someone still get confused we can catch it.
> 
> Does that sound like a plan?

Assuming I understood it correctly, yes. Can you just confirm my
understanding is correct though?

Regards,

Boris

Re: [PATCH v18 04/26] drm/shmem-helper: Refactor locked/unlocked functions

[Maxime Ripard]

[-- Attachment #1: Type: text/plain, Size: 4275 bytes --]

On Wed, Nov 29, 2023 at 04:47:05PM +0100, Boris Brezillon wrote:
> On Wed, 29 Nov 2023 16:15:27 +0100
> Maxime Ripard <mripard@kernel.org> wrote:
> 
> > > Now, let's assume we drop the _locked() suffix on
> > > drm_gem_shmem_v[un]map(), but keep it on other helpers that need both
> > > variants. This results in an inconsistent naming scheme inside the
> > > same source file, which I find utterly confusing.
> > >
> > > Note that the initial reason I asked Dmitry if he could add the
> > > _locked suffix to drm_gem_shmem_vmap() is because I started using
> > > drm_gem_shmem_vmap() in powervr, before realizing this version wasn't
> > > taking the lock, and I should have used drm_gem_vmap_unlocked()
> > > instead, so this is not something I'm making up.  
> > 
> > Sorry if I gave you the impression I thought that you're making that up,
> > I'm not.
> > 
> > Thanks for the explanation btw, I think I get what you're saying now:
> > 
> >  - drm_gem_shmem_vmap() is never taking the locks because the core
> >    expects to take them before calling them.
> > 
> >  - drm_gem_shmem_vunmap() is never taking the locks because the core
> >    expects to take them before calling them.
> 
> Correct.
> 
> > 
> >  - Some other code path can still call those helpers in drivers, and the
> >    locking isn't handled by the core anymore.
> 
> They can, if they want to v[un]map a BO and they already acquired the
> GEM resv lock. But I'm not sure anyone needs to do that yet. The main
> reason for exposing these helpers is if one driver needs to overload the
> default gem_shmem_funcs.
> 
> > 
> >  - We now have _vmap/vunmap_unlocked functions to take those locks for
> >    those code paths
> 
> We don't have drm_gem_shmem_vmap/vunmap_unlocked(), we have
> drm_gem_shmem_vmap/vunmap_locked(), which can be called directly, but
> are mainly used to populate the drm_gem_object_funcs vtable. If drivers
> want to v[un]map in a path where the resv lock is not held, they should
> call drm_gem_vmap/vunmap_unlocked() (which are renamed
> drm_gem_vmap/vunmap() in patch 1 of this series). Mind the **drm_gem_**
> vs **drm_gem_shmem_** difference in the helper names. drm_gem_ helpers
> are provided by drm_gem.c and call drm_gem_object_funcs callback, which
> are supposed to be populated with drm_gem_shmem helpers.
> 
> > 
> >  - And the variant names are now confusing, making people use the
> >    lockless version in situations where they should have use the locked
> >    one.
> 
> That's what happened to me, at least.
> 
> > 
> > Is that a correct summary?
> 
> Almost ;-).
> 
> > 
> > If so, then I agree that we need to change the name.
> 
> Cool.
> 
> > 
> > We discussed it some more on IRC, and we agree that the "default"
> > function should handle the locking properly and that's what the most
> > common case should use.
> 
> Agree if by 'default' you mean the lock is always acquired by the
> helper, not 'let's decide based on what users do most of the time with
> this specific helper', because otherwise we'd be back to a situation
> where the name doesn't clearly encode the function behavior.
> 
> > 
> > So that means than drm_gem_shmem_vmap/vunmap() should take the lock
> > itself, and drm_gem_shmem_vmap/vunmap_nolock/unlocked never does.
> 
> Not sure we have a need for drm_gem_shmem_vmap/vunmap(), but if we ever
> add such helpers, they would acquire the resv lock, indeed.
> 
> Just to be clear, _nolock == _locked in the current semantics :-).
> _nolock means 'don't take the lock', and _locked means 'lock is already
> held'.
> 
> > 
> > I think I'd prefer the nolock variant over unlocked still.
> 
> Okay, that means s/_locked/_nolock/ in drm_gem_shmem_helpers.{c,h}, I
> guess.
> 
> > 
> > And I also think we can improve the documentation and add lockdep calls
> 
> Lockdep asserts are already there, I think.
> 
> > to make sure that the difference between variants is clear in the doc,
> > and if someone still get confused we can catch it.
> > 
> > Does that sound like a plan?
> 
> Assuming I understood it correctly, yes. Can you just confirm my
> understanding is correct though?

We are. Sorry for delaying this :)

Maxime

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 228 bytes --]

Re: [PATCH v18 04/26] drm/shmem-helper: Refactor locked/unlocked functions

[Dmitry Osipenko]

On 12/4/23 15:55, Maxime Ripard wrote:
>> Okay, that means s/_locked/_nolock/ in drm_gem_shmem_helpers.{c,h}, I
>> guess.

DRM subsys and majority of kernel uses common _locked postfix. We should
retain the old naming scheme by using _locked() in DRM. It's not
worthwhile changing the name to a much less popular variant for a no
good reason.

Maxime, are you okay with keeping the _locked name?

-- 
Best regards,
Dmitry

Re: [PATCH v18 04/26] drm/shmem-helper: Refactor locked/unlocked functions

[Maxime Ripard]

[-- Attachment #1: Type: text/plain, Size: 616 bytes --]

On Tue, Dec 05, 2023 at 02:43:16PM +0300, Dmitry Osipenko wrote:
> On 12/4/23 15:55, Maxime Ripard wrote:
> >> Okay, that means s/_locked/_nolock/ in drm_gem_shmem_helpers.{c,h}, I
> >> guess.
> 
> DRM subsys and majority of kernel uses common _locked postfix. We should
> retain the old naming scheme by using _locked() in DRM. It's not
> worthwhile changing the name to a much less popular variant for a no
> good reason.
> 
> Maxime, are you okay with keeping the _locked name?

Yeah... I still don't really like it, but you're right that it's best to
remain consistent over my opinion :)

Maxime

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 228 bytes --]

Re: [PATCH v18 04/26] drm/shmem-helper: Refactor locked/unlocked functions

[Dmitry Osipenko]

On 12/14/23 21:16, Maxime Ripard wrote:
> On Tue, Dec 05, 2023 at 02:43:16PM +0300, Dmitry Osipenko wrote:
>> On 12/4/23 15:55, Maxime Ripard wrote:
>>>> Okay, that means s/_locked/_nolock/ in drm_gem_shmem_helpers.{c,h}, I
>>>> guess.
>>
>> DRM subsys and majority of kernel uses common _locked postfix. We should
>> retain the old naming scheme by using _locked() in DRM. It's not
>> worthwhile changing the name to a much less popular variant for a no
>> good reason.
>>
>> Maxime, are you okay with keeping the _locked name?
> 
> Yeah... I still don't really like it, but you're right that it's best to
> remain consistent over my opinion :)
Thanks for the review!

Best regards,
Dmitry

Re: [PATCH v18 08/26] drm/shmem-helper: Add and use lockless drm_gem_shmem_get_pages()

[Maxime Ripard]

[-- Attachment #1: Type: text/plain, Size: 1769 bytes --]

On Mon, Oct 30, 2023 at 02:01:47AM +0300, Dmitry Osipenko wrote:
> Add lockless drm_gem_shmem_get_pages() helper that skips taking reservation
> lock if pages_use_count is non-zero, leveraging from atomicity of the
> refcount_t. Make drm_gem_shmem_mmap() to utilize the new helper.
> 
> Reviewed-by: Boris Brezillon <boris.brezillon@collabora.com>
> Suggested-by: Boris Brezillon <boris.brezillon@collabora.com>
> Signed-off-by: Dmitry Osipenko <dmitry.osipenko@collabora.com>
> ---
>  drivers/gpu/drm/drm_gem_shmem_helper.c | 19 +++++++++++++++----
>  1 file changed, 15 insertions(+), 4 deletions(-)
> 
> diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c
> index 6e02643ed87e..41b749bedb11 100644
> --- a/drivers/gpu/drm/drm_gem_shmem_helper.c
> +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c
> @@ -226,6 +226,20 @@ void drm_gem_shmem_put_pages_locked(struct drm_gem_shmem_object *shmem)
>  }
>  EXPORT_SYMBOL_GPL(drm_gem_shmem_put_pages_locked);
>  
> +static int drm_gem_shmem_get_pages(struct drm_gem_shmem_object *shmem)
> +{
> +	int ret;
> +
> +	if (refcount_inc_not_zero(&shmem->pages_use_count))
> +		return 0;
> +
> +	dma_resv_lock(shmem->base.resv, NULL);
> +	ret = drm_gem_shmem_get_pages_locked(shmem);
> +	dma_resv_unlock(shmem->base.resv);
> +
> +	return ret;
> +}
> +

Wait, so the locked suffix is to indicate that we need to take the lock
before we call it? I think that's the opposite to all(?) the naming
convention we have

Especially since the function name doesn't describe what the function
does anymore, but the context in which to call it. I'm sure if I was to
use it, I would have gotten it wrong, or at the very least been very
confused about it.

Maxime

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 228 bytes --]

Re: [PATCH v18 08/26] drm/shmem-helper: Add and use lockless drm_gem_shmem_get_pages()

[Boris Brezillon]

On Fri, 24 Nov 2023 11:47:57 +0100
Maxime Ripard <mripard@kernel.org> wrote:

> On Mon, Oct 30, 2023 at 02:01:47AM +0300, Dmitry Osipenko wrote:
> > Add lockless drm_gem_shmem_get_pages() helper that skips taking reservation
> > lock if pages_use_count is non-zero, leveraging from atomicity of the
> > refcount_t. Make drm_gem_shmem_mmap() to utilize the new helper.
> > 
> > Reviewed-by: Boris Brezillon <boris.brezillon@collabora.com>
> > Suggested-by: Boris Brezillon <boris.brezillon@collabora.com>
> > Signed-off-by: Dmitry Osipenko <dmitry.osipenko@collabora.com>
> > ---
> >  drivers/gpu/drm/drm_gem_shmem_helper.c | 19 +++++++++++++++----
> >  1 file changed, 15 insertions(+), 4 deletions(-)
> > 
> > diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c
> > index 6e02643ed87e..41b749bedb11 100644
> > --- a/drivers/gpu/drm/drm_gem_shmem_helper.c
> > +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c
> > @@ -226,6 +226,20 @@ void drm_gem_shmem_put_pages_locked(struct drm_gem_shmem_object *shmem)
> >  }
> >  EXPORT_SYMBOL_GPL(drm_gem_shmem_put_pages_locked);
> >  
> > +static int drm_gem_shmem_get_pages(struct drm_gem_shmem_object *shmem)
> > +{
> > +	int ret;
> > +
> > +	if (refcount_inc_not_zero(&shmem->pages_use_count))
> > +		return 0;
> > +
> > +	dma_resv_lock(shmem->base.resv, NULL);
> > +	ret = drm_gem_shmem_get_pages_locked(shmem);
> > +	dma_resv_unlock(shmem->base.resv);
> > +
> > +	return ret;
> > +}
> > +  
> 
> Wait, so the locked suffix is to indicate that we need to take the lock
> before we call it? I think that's the opposite to all(?) the naming
> convention we have

If you grep for "_locked(" and "_unlocked(" in the DRM sub-tree, you'll
see it's actually mixed, with maybe a few more helpers suffixed
_locked() than we have suffixed with _unlocked().

> 
> Especially since the function name doesn't describe what the function
> does anymore, but the context in which to call it.

Well, that's the same for "_unlocked", and we do have to pick one of
the _locked/_unlocked pattern if we want to expose both flavors.

> I'm sure if I was to
> use it, I would have gotten it wrong, or at the very least been very
> confused about it.

I personally find both equally confusing tbh, but we do have cases
where we need to expose the exact same functionality without the extra
locking. I do have a slight preference for _locked though, because it's
two characters shorter ;-).

Re: [PATCH v18 09/26] drm/shmem-helper: Switch drm_gem_shmem_vmap/vunmap to use pin/unpin

[Maxime Ripard]

[-- Attachment #1: Type: text/plain, Size: 837 bytes --]

On Mon, Oct 30, 2023 at 02:01:48AM +0300, Dmitry Osipenko wrote:
> The vmapped pages shall be pinned in memory and previously get/put_pages()
> were implicitly hard-pinning/unpinning the pages. This will no longer be
> the case with addition of memory shrinker because pages_use_count > 0 won't
> determine anymore whether pages are hard-pinned (they will be soft-pinned),
> while the new pages_pin_count will do the hard-pinning. Switch the
> vmap/vunmap() to use pin/unpin() functions in a preparation of addition
> of the memory shrinker support to drm-shmem.
> 
> Reviewed-by: Boris Brezillon <boris.brezillon@collabora.com>
> Signed-off-by: Dmitry Osipenko <dmitry.osipenko@collabora.com>

The naming convention discussion aside, and once it's settled and fixed:
Acked-by: Maxime Ripard <mripard@kernel.org>

Maxime

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 228 bytes --]