[PATCH v6 0/4] drm: Use full allocated minor range for DRM

[PATCH v6 0/4] drm: Use full allocated minor range for DRM

[Michał Winiarski]

64 DRM device nodes is not enough for everyone.
Upgrade it to ~512K (which definitely is more than enough).

To allow testing userspace support for >64 devices, add additional DRM
modparam (force_extended_minors) which causes DRM to skip allocating minors
in 0-192 range.
Additionally - convert minors to use XArray instead of IDR to simplify the
locking.

v1 -> v2:
Don't touch DRM_MINOR_CONTROL and its range (Simon Ser)

v2 -> v3:
Don't use legacy scheme for >=192 minor range (Dave Airlie)
Add modparam for testing (Dave Airlie)
Add lockdep annotation for IDR (Daniel Vetter)

v3 -> v4:
Convert from IDR to XArray (Matthew Wilcox)

v4 -> v5:
Fixup IDR to XArray conversion (Matthew Wilcox)

v5 -> v6:
Also convert Accel to XArray
Rename skip_legacy_minors to force_extended_minors

Michał Winiarski (4):
  drm: Use XArray instead of IDR for minors
  accel: Use XArray instead of IDR for minors
  drm: Expand max DRM device number to full MINORBITS
  drm: Introduce force_extended_minors modparam

 drivers/accel/drm_accel.c      | 110 +++------------------------------
 drivers/gpu/drm/drm_drv.c      | 105 ++++++++++++++++---------------
 drivers/gpu/drm/drm_file.c     |   2 +-
 drivers/gpu/drm/drm_internal.h |   4 --
 include/drm/drm_accel.h        |  18 +-----
 include/drm/drm_file.h         |   5 ++
 6 files changed, 69 insertions(+), 175 deletions(-)

-- 
2.41.0

[PATCH v6 1/4] drm: Use XArray instead of IDR for minors

[Michał Winiarski]

IDR is deprecated, and since XArray manages its own state with internal
locking, it simplifies the locking on DRM side.
Additionally, don't use the IRQ-safe variant, since operating on drm
minor is not done in IRQ context.

Signed-off-by: Michał Winiarski <michal.winiarski@intel.com>
Suggested-by: Matthew Wilcox <willy@infradead.org>
---
 drivers/gpu/drm/drm_drv.c | 63 ++++++++++++++++-----------------------
 1 file changed, 25 insertions(+), 38 deletions(-)

diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c
index 3eda026ffac6..3faecb01186f 100644
--- a/drivers/gpu/drm/drm_drv.c
+++ b/drivers/gpu/drm/drm_drv.c
@@ -34,6 +34,7 @@
 #include <linux/pseudo_fs.h>
 #include <linux/slab.h>
 #include <linux/srcu.h>
+#include <linux/xarray.h>
 
 #include <drm/drm_accel.h>
 #include <drm/drm_cache.h>
@@ -54,8 +55,7 @@ MODULE_AUTHOR("Gareth Hughes, Leif Delgass, José Fonseca, Jon Smirl");
 MODULE_DESCRIPTION("DRM shared core routines");
 MODULE_LICENSE("GPL and additional rights");
 
-static DEFINE_SPINLOCK(drm_minor_lock);
-static struct idr drm_minors_idr;
+static DEFINE_XARRAY_ALLOC(drm_minors_xa);
 
 /*
  * If the drm core fails to init for whatever reason,
@@ -101,26 +101,23 @@ static struct drm_minor **drm_minor_get_slot(struct drm_device *dev,
 static void drm_minor_alloc_release(struct drm_device *dev, void *data)
 {
 	struct drm_minor *minor = data;
-	unsigned long flags;
 
 	WARN_ON(dev != minor->dev);
 
 	put_device(minor->kdev);
 
-	if (minor->type == DRM_MINOR_ACCEL) {
+	if (minor->type == DRM_MINOR_ACCEL)
 		accel_minor_remove(minor->index);
-	} else {
-		spin_lock_irqsave(&drm_minor_lock, flags);
-		idr_remove(&drm_minors_idr, minor->index);
-		spin_unlock_irqrestore(&drm_minor_lock, flags);
-	}
+	else
+		xa_erase(&drm_minors_xa, minor->index);
 }
 
+#define DRM_MINOR_LIMIT(t) ({ typeof(t) _t = (t); XA_LIMIT(64 * _t, 64 * _t + 63); })
+
 static int drm_minor_alloc(struct drm_device *dev, enum drm_minor_type type)
 {
 	struct drm_minor *minor;
-	unsigned long flags;
-	int r;
+	int index, r;
 
 	minor = drmm_kzalloc(dev, sizeof(*minor), GFP_KERNEL);
 	if (!minor)
@@ -129,24 +126,17 @@ static int drm_minor_alloc(struct drm_device *dev, enum drm_minor_type type)
 	minor->type = type;
 	minor->dev = dev;
 
-	idr_preload(GFP_KERNEL);
 	if (type == DRM_MINOR_ACCEL) {
 		r = accel_minor_alloc();
+		index = r;
 	} else {
-		spin_lock_irqsave(&drm_minor_lock, flags);
-		r = idr_alloc(&drm_minors_idr,
-			NULL,
-			64 * type,
-			64 * (type + 1),
-			GFP_NOWAIT);
-		spin_unlock_irqrestore(&drm_minor_lock, flags);
+		r = xa_alloc(&drm_minors_xa, &index, NULL, DRM_MINOR_LIMIT(type), GFP_KERNEL);
 	}
-	idr_preload_end();
 
 	if (r < 0)
 		return r;
 
-	minor->index = r;
+	minor->index = index;
 
 	r = drmm_add_action_or_reset(dev, drm_minor_alloc_release, minor);
 	if (r)
@@ -163,7 +153,7 @@ static int drm_minor_alloc(struct drm_device *dev, enum drm_minor_type type)
 static int drm_minor_register(struct drm_device *dev, enum drm_minor_type type)
 {
 	struct drm_minor *minor;
-	unsigned long flags;
+	void *entry;
 	int ret;
 
 	DRM_DEBUG("\n");
@@ -190,9 +180,12 @@ static int drm_minor_register(struct drm_device *dev, enum drm_minor_type type)
 	if (minor->type == DRM_MINOR_ACCEL) {
 		accel_minor_replace(minor, minor->index);
 	} else {
-		spin_lock_irqsave(&drm_minor_lock, flags);
-		idr_replace(&drm_minors_idr, minor, minor->index);
-		spin_unlock_irqrestore(&drm_minor_lock, flags);
+		entry = xa_store(&drm_minors_xa, minor->index, minor, GFP_KERNEL);
+		if (xa_is_err(entry)) {
+			ret = xa_err(entry);
+			goto err_debugfs;
+		}
+		WARN_ON(entry);
 	}
 
 	DRM_DEBUG("new minor registered %d\n", minor->index);
@@ -206,20 +199,16 @@ static int drm_minor_register(struct drm_device *dev, enum drm_minor_type type)
 static void drm_minor_unregister(struct drm_device *dev, enum drm_minor_type type)
 {
 	struct drm_minor *minor;
-	unsigned long flags;
 
 	minor = *drm_minor_get_slot(dev, type);
 	if (!minor || !device_is_registered(minor->kdev))
 		return;
 
 	/* replace @minor with NULL so lookups will fail from now on */
-	if (minor->type == DRM_MINOR_ACCEL) {
+	if (minor->type == DRM_MINOR_ACCEL)
 		accel_minor_replace(NULL, minor->index);
-	} else {
-		spin_lock_irqsave(&drm_minor_lock, flags);
-		idr_replace(&drm_minors_idr, NULL, minor->index);
-		spin_unlock_irqrestore(&drm_minor_lock, flags);
-	}
+	else
+		xa_store(&drm_minors_xa, minor->index, NULL, GFP_KERNEL);
 
 	device_del(minor->kdev);
 	dev_set_drvdata(minor->kdev, NULL); /* safety belt */
@@ -238,13 +227,12 @@ static void drm_minor_unregister(struct drm_device *dev, enum drm_minor_type typ
 struct drm_minor *drm_minor_acquire(unsigned int minor_id)
 {
 	struct drm_minor *minor;
-	unsigned long flags;
 
-	spin_lock_irqsave(&drm_minor_lock, flags);
-	minor = idr_find(&drm_minors_idr, minor_id);
+	xa_lock(&drm_minors_xa);
+	minor = xa_load(&drm_minors_xa, minor_id);
 	if (minor)
 		drm_dev_get(minor->dev);
-	spin_unlock_irqrestore(&drm_minor_lock, flags);
+	xa_unlock(&drm_minors_xa);
 
 	if (!minor) {
 		return ERR_PTR(-ENODEV);
@@ -1067,7 +1055,7 @@ static void drm_core_exit(void)
 	unregister_chrdev(DRM_MAJOR, "drm");
 	debugfs_remove(drm_debugfs_root);
 	drm_sysfs_destroy();
-	idr_destroy(&drm_minors_idr);
+	WARN_ON(!xa_empty(&drm_minors_xa));
 	drm_connector_ida_destroy();
 }
 
@@ -1076,7 +1064,6 @@ static int __init drm_core_init(void)
 	int ret;
 
 	drm_connector_ida_init();
-	idr_init(&drm_minors_idr);
 	drm_memcpy_init_early();
 
 	ret = drm_sysfs_init();
-- 
2.41.0

Re: [PATCH v6 1/4] drm: Use XArray instead of IDR for minors

[Michał Winiarski]

On Fri, Aug 25, 2023 at 12:59:26PM -0400, James Zhu wrote:
> 
> On 2023-07-24 17:14, Michał Winiarski wrote:
> > IDR is deprecated, and since XArray manages its own state with internal
> > locking, it simplifies the locking on DRM side.
> > Additionally, don't use the IRQ-safe variant, since operating on drm
> > minor is not done in IRQ context.
> > 
> > Signed-off-by: Michał Winiarski<michal.winiarski@intel.com>
> > Suggested-by: Matthew Wilcox<willy@infradead.org>
> > ---
> >   drivers/gpu/drm/drm_drv.c | 63 ++++++++++++++++-----------------------
> >   1 file changed, 25 insertions(+), 38 deletions(-)
> > 
> > diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c
> > index 3eda026ffac6..3faecb01186f 100644
> > --- a/drivers/gpu/drm/drm_drv.c
> > +++ b/drivers/gpu/drm/drm_drv.c
> > @@ -34,6 +34,7 @@
> >   #include <linux/pseudo_fs.h>
> >   #include <linux/slab.h>
> >   #include <linux/srcu.h>
> > +#include <linux/xarray.h>
> >   #include <drm/drm_accel.h>
> >   #include <drm/drm_cache.h>
> > @@ -54,8 +55,7 @@ MODULE_AUTHOR("Gareth Hughes, Leif Delgass, José Fonseca, Jon Smirl");
> >   MODULE_DESCRIPTION("DRM shared core routines");
> >   MODULE_LICENSE("GPL and additional rights");
> > -static DEFINE_SPINLOCK(drm_minor_lock);
> > -static struct idr drm_minors_idr;
> > +static DEFINE_XARRAY_ALLOC(drm_minors_xa);
> >   /*
> >    * If the drm core fails to init for whatever reason,
> > @@ -101,26 +101,23 @@ static struct drm_minor **drm_minor_get_slot(struct drm_device *dev,
> >   static void drm_minor_alloc_release(struct drm_device *dev, void *data)
> >   {
> >   	struct drm_minor *minor = data;
> > -	unsigned long flags;
> >   	WARN_ON(dev != minor->dev);
> >   	put_device(minor->kdev);
> > -	if (minor->type == DRM_MINOR_ACCEL) {
> > +	if (minor->type == DRM_MINOR_ACCEL)
> >   		accel_minor_remove(minor->index);
> > -	} else {
> > -		spin_lock_irqsave(&drm_minor_lock, flags);
> > -		idr_remove(&drm_minors_idr, minor->index);
> > -		spin_unlock_irqrestore(&drm_minor_lock, flags);
> > -	}
> > +	else
> > +		xa_erase(&drm_minors_xa, minor->index);
> >   }
> > +#define DRM_MINOR_LIMIT(t) ({ typeof(t) _t = (t); XA_LIMIT(64 * _t, 64 * _t + 63); })
> > +
> >   static int drm_minor_alloc(struct drm_device *dev, enum drm_minor_type type)
> >   {
> >   	struct drm_minor *minor;
> > -	unsigned long flags;
> > -	int r;
> > +	int index, r;
> >   	minor = drmm_kzalloc(dev, sizeof(*minor), GFP_KERNEL);
> >   	if (!minor)
> > @@ -129,24 +126,17 @@ static int drm_minor_alloc(struct drm_device *dev, enum drm_minor_type type)
> >   	minor->type = type;
> >   	minor->dev = dev;
> > -	idr_preload(GFP_KERNEL);
> >   	if (type == DRM_MINOR_ACCEL) {
> >   		r = accel_minor_alloc();
> > +		index = r;
> >   	} else {
> > -		spin_lock_irqsave(&drm_minor_lock, flags);
> > -		r = idr_alloc(&drm_minors_idr,
> > -			NULL,
> > -			64 * type,
> > -			64 * (type + 1),
> > -			GFP_NOWAIT);
> > -		spin_unlock_irqrestore(&drm_minor_lock, flags);
> > +		r = xa_alloc(&drm_minors_xa, &index, NULL, DRM_MINOR_LIMIT(type), GFP_KERNEL);
> >   	}
> > -	idr_preload_end();
> >   	if (r < 0)
> >   		return r;
> > -	minor->index = r;
> > +	minor->index = index;
> >   	r = drmm_add_action_or_reset(dev, drm_minor_alloc_release, minor);
> >   	if (r)
> > @@ -163,7 +153,7 @@ static int drm_minor_alloc(struct drm_device *dev, enum drm_minor_type type)
> >   static int drm_minor_register(struct drm_device *dev, enum drm_minor_type type)
> >   {
> >   	struct drm_minor *minor;
> > -	unsigned long flags;
> > +	void *entry;
> >   	int ret;
> >   	DRM_DEBUG("\n");
> > @@ -190,9 +180,12 @@ static int drm_minor_register(struct drm_device *dev, enum drm_minor_type type)
> >   	if (minor->type == DRM_MINOR_ACCEL) {
> >   		accel_minor_replace(minor, minor->index);
> >   	} else {
> > -		spin_lock_irqsave(&drm_minor_lock, flags);
> > -		idr_replace(&drm_minors_idr, minor, minor->index);
> > -		spin_unlock_irqrestore(&drm_minor_lock, flags);
> > +		entry = xa_store(&drm_minors_xa, minor->index, minor, GFP_KERNEL);
> > +		if (xa_is_err(entry)) {
> > +			ret = xa_err(entry);
> > +			goto err_debugfs;
> > +		}
> > +		WARN_ON(entry);
> [JZ] would WARN_ON(entry != minor)be better?

We expect NULL here.
The same one that was previously stored here ⌄⌄⌄
r = xa_alloc(&drm_minors_xa, &minor->index, NULL, DRM_EXTENDED_MINOR_LIMIT, GFP_KERNEL);
in drm_minor_alloc.

> >   	}
> >   	DRM_DEBUG("new minor registered %d\n", minor->index);
> > @@ -206,20 +199,16 @@ static int drm_minor_register(struct drm_device *dev, enum drm_minor_type type)
> >   static void drm_minor_unregister(struct drm_device *dev, enum drm_minor_type type)
> >   {
> >   	struct drm_minor *minor;
> > -	unsigned long flags;
> >   	minor = *drm_minor_get_slot(dev, type);
> >   	if (!minor || !device_is_registered(minor->kdev))
> >   		return;
> >   	/* replace @minor with NULL so lookups will fail from now on */
> > -	if (minor->type == DRM_MINOR_ACCEL) {
> > +	if (minor->type == DRM_MINOR_ACCEL)
> >   		accel_minor_replace(NULL, minor->index);
> > -	} else {
> > -		spin_lock_irqsave(&drm_minor_lock, flags);
> > -		idr_replace(&drm_minors_idr, NULL, minor->index);
> > -		spin_unlock_irqrestore(&drm_minor_lock, flags);
> > -	}
> > +	else
> > +		xa_store(&drm_minors_xa, minor->index, NULL, GFP_KERNEL);
> >   	device_del(minor->kdev);
> >   	dev_set_drvdata(minor->kdev, NULL); /* safety belt */
> > @@ -238,13 +227,12 @@ static void drm_minor_unregister(struct drm_device *dev, enum drm_minor_type typ
> >   struct drm_minor *drm_minor_acquire(unsigned int minor_id)
> >   {
> >   	struct drm_minor *minor;
> > -	unsigned long flags;
> > -	spin_lock_irqsave(&drm_minor_lock, flags);
> > -	minor = idr_find(&drm_minors_idr, minor_id);
> > +	xa_lock(&drm_minors_xa);
> > +	minor = xa_load(&drm_minors_xa, minor_id);
> >   	if (minor)
> >   		drm_dev_get(minor->dev);
> [JZ] why minor->dev need ca_lock here?

We're relying on the ordering for acquire/release (previously
guaranteed by the drm_minor_lock, now by internal XArray locking).
xa_load doesn't take xa_lock internally:
https://docs.kernel.org/core-api/xarray.html#locking

> > -	spin_unlock_irqrestore(&drm_minor_lock, flags);
> > +	xa_unlock(&drm_minors_xa);
> >   	if (!minor) {
> >   		return ERR_PTR(-ENODEV);
> > @@ -1067,7 +1055,7 @@ static void drm_core_exit(void)
> >   	unregister_chrdev(DRM_MAJOR, "drm");
> >   	debugfs_remove(drm_debugfs_root);
> >   	drm_sysfs_destroy();
> > -	idr_destroy(&drm_minors_idr);
> [JZ] Should we call xa_destroy instead here?

We could, if we expect the xarray to potentially not be empty.
From what I understand - all minors should be released at this point.

Thanks,
-Michał

> > +	WARN_ON(!xa_empty(&drm_minors_xa));
> >   	drm_connector_ida_destroy();
> >   }
> > @@ -1076,7 +1064,6 @@ static int __init drm_core_init(void)
> >   	int ret;
> >   	drm_connector_ida_init();
> > -	idr_init(&drm_minors_idr);
> >   	drm_memcpy_init_early();
> >   	ret = drm_sysfs_init();

Re: [PATCH v6 1/4] drm: Use XArray instead of IDR for minors

[James Zhu]

On 2023-08-28 17:08, Michał Winiarski wrote:
> On Fri, Aug 25, 2023 at 12:59:26PM -0400, James Zhu wrote:
>> On 2023-07-24 17:14, Michał Winiarski wrote:
>>> IDR is deprecated, and since XArray manages its own state with internal
>>> locking, it simplifies the locking on DRM side.
>>> Additionally, don't use the IRQ-safe variant, since operating on drm
>>> minor is not done in IRQ context.
>>>
>>> Signed-off-by: Michał Winiarski<michal.winiarski@intel.com>
>>> Suggested-by: Matthew Wilcox<willy@infradead.org>
>>> ---
>>>    drivers/gpu/drm/drm_drv.c | 63 ++++++++++++++++-----------------------
>>>    1 file changed, 25 insertions(+), 38 deletions(-)
>>>
>>> diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c
>>> index 3eda026ffac6..3faecb01186f 100644
>>> --- a/drivers/gpu/drm/drm_drv.c
>>> +++ b/drivers/gpu/drm/drm_drv.c
>>> @@ -34,6 +34,7 @@
>>>    #include <linux/pseudo_fs.h>
>>>    #include <linux/slab.h>
>>>    #include <linux/srcu.h>
>>> +#include <linux/xarray.h>
>>>    #include <drm/drm_accel.h>
>>>    #include <drm/drm_cache.h>
>>> @@ -54,8 +55,7 @@ MODULE_AUTHOR("Gareth Hughes, Leif Delgass, José Fonseca, Jon Smirl");
>>>    MODULE_DESCRIPTION("DRM shared core routines");
>>>    MODULE_LICENSE("GPL and additional rights");
>>> -static DEFINE_SPINLOCK(drm_minor_lock);
>>> -static struct idr drm_minors_idr;
>>> +static DEFINE_XARRAY_ALLOC(drm_minors_xa);
>>>    /*
>>>     * If the drm core fails to init for whatever reason,
>>> @@ -101,26 +101,23 @@ static struct drm_minor **drm_minor_get_slot(struct drm_device *dev,
>>>    static void drm_minor_alloc_release(struct drm_device *dev, void *data)
>>>    {
>>>    	struct drm_minor *minor = data;
>>> -	unsigned long flags;
>>>    	WARN_ON(dev != minor->dev);
>>>    	put_device(minor->kdev);
>>> -	if (minor->type == DRM_MINOR_ACCEL) {
>>> +	if (minor->type == DRM_MINOR_ACCEL)
>>>    		accel_minor_remove(minor->index);
>>> -	} else {
>>> -		spin_lock_irqsave(&drm_minor_lock, flags);
>>> -		idr_remove(&drm_minors_idr, minor->index);
>>> -		spin_unlock_irqrestore(&drm_minor_lock, flags);
>>> -	}
>>> +	else
>>> +		xa_erase(&drm_minors_xa, minor->index);
>>>    }
>>> +#define DRM_MINOR_LIMIT(t) ({ typeof(t) _t = (t); XA_LIMIT(64 * _t, 64 * _t + 63); })
>>> +
>>>    static int drm_minor_alloc(struct drm_device *dev, enum drm_minor_type type)
>>>    {
>>>    	struct drm_minor *minor;
>>> -	unsigned long flags;
>>> -	int r;
>>> +	int index, r;
>>>    	minor = drmm_kzalloc(dev, sizeof(*minor), GFP_KERNEL);
>>>    	if (!minor)
>>> @@ -129,24 +126,17 @@ static int drm_minor_alloc(struct drm_device *dev, enum drm_minor_type type)
>>>    	minor->type = type;
>>>    	minor->dev = dev;
>>> -	idr_preload(GFP_KERNEL);
>>>    	if (type == DRM_MINOR_ACCEL) {
>>>    		r = accel_minor_alloc();
>>> +		index = r;
>>>    	} else {
>>> -		spin_lock_irqsave(&drm_minor_lock, flags);
>>> -		r = idr_alloc(&drm_minors_idr,
>>> -			NULL,
>>> -			64 * type,
>>> -			64 * (type + 1),
>>> -			GFP_NOWAIT);
>>> -		spin_unlock_irqrestore(&drm_minor_lock, flags);
>>> +		r = xa_alloc(&drm_minors_xa, &index, NULL, DRM_MINOR_LIMIT(type), GFP_KERNEL);
>>>    	}
>>> -	idr_preload_end();
>>>    	if (r < 0)
>>>    		return r;
>>> -	minor->index = r;
>>> +	minor->index = index;
>>>    	r = drmm_add_action_or_reset(dev, drm_minor_alloc_release, minor);
>>>    	if (r)
>>> @@ -163,7 +153,7 @@ static int drm_minor_alloc(struct drm_device *dev, enum drm_minor_type type)
>>>    static int drm_minor_register(struct drm_device *dev, enum drm_minor_type type)
>>>    {
>>>    	struct drm_minor *minor;
>>> -	unsigned long flags;
>>> +	void *entry;
>>>    	int ret;
>>>    	DRM_DEBUG("\n");
>>> @@ -190,9 +180,12 @@ static int drm_minor_register(struct drm_device *dev, enum drm_minor_type type)
>>>    	if (minor->type == DRM_MINOR_ACCEL) {
>>>    		accel_minor_replace(minor, minor->index);
>>>    	} else {
>>> -		spin_lock_irqsave(&drm_minor_lock, flags);
>>> -		idr_replace(&drm_minors_idr, minor, minor->index);
>>> -		spin_unlock_irqrestore(&drm_minor_lock, flags);
>>> +		entry = xa_store(&drm_minors_xa, minor->index, minor, GFP_KERNEL);
>>> +		if (xa_is_err(entry)) {
>>> +			ret = xa_err(entry);
>>> +			goto err_debugfs;
>>> +		}
>>> +		WARN_ON(entry);
>> [JZ] would WARN_ON(entry != minor)be better?
> We expect NULL here.
> The same one that was previously stored here ⌄⌄⌄
> r = xa_alloc(&drm_minors_xa, &minor->index, NULL, DRM_EXTENDED_MINOR_LIMIT, GFP_KERNEL);
> in drm_minor_alloc.
[JZ] My mistake.
>
>>>    	}
>>>    	DRM_DEBUG("new minor registered %d\n", minor->index);
>>> @@ -206,20 +199,16 @@ static int drm_minor_register(struct drm_device *dev, enum drm_minor_type type)
>>>    static void drm_minor_unregister(struct drm_device *dev, enum drm_minor_type type)
>>>    {
>>>    	struct drm_minor *minor;
>>> -	unsigned long flags;
>>>    	minor = *drm_minor_get_slot(dev, type);
>>>    	if (!minor || !device_is_registered(minor->kdev))
>>>    		return;
>>>    	/* replace @minor with NULL so lookups will fail from now on */
>>> -	if (minor->type == DRM_MINOR_ACCEL) {
>>> +	if (minor->type == DRM_MINOR_ACCEL)
>>>    		accel_minor_replace(NULL, minor->index);
>>> -	} else {
>>> -		spin_lock_irqsave(&drm_minor_lock, flags);
>>> -		idr_replace(&drm_minors_idr, NULL, minor->index);
>>> -		spin_unlock_irqrestore(&drm_minor_lock, flags);
>>> -	}
>>> +	else
>>> +		xa_store(&drm_minors_xa, minor->index, NULL, GFP_KERNEL);
>>>    	device_del(minor->kdev);
>>>    	dev_set_drvdata(minor->kdev, NULL); /* safety belt */
>>> @@ -238,13 +227,12 @@ static void drm_minor_unregister(struct drm_device *dev, enum drm_minor_type typ
>>>    struct drm_minor *drm_minor_acquire(unsigned int minor_id)
>>>    {
>>>    	struct drm_minor *minor;
>>> -	unsigned long flags;
>>> -	spin_lock_irqsave(&drm_minor_lock, flags);
>>> -	minor = idr_find(&drm_minors_idr, minor_id);
>>> +	xa_lock(&drm_minors_xa);
>>> +	minor = xa_load(&drm_minors_xa, minor_id);
>>>    	if (minor)
>>>    		drm_dev_get(minor->dev);
>> [JZ] why minor->dev need ca_lock here?
> We're relying on the ordering for acquire/release (previously
> guaranteed by the drm_minor_lock, now by internal XArray locking).
> xa_load doesn't take xa_lock internally:
> https://docs.kernel.org/core-api/xarray.html#locking
[JZ] I means that drm_dev_get(minor->dev); can move out of xa_lock.
>>> -	spin_unlock_irqrestore(&drm_minor_lock, flags);
>>> +	xa_unlock(&drm_minors_xa);
>>>    	if (!minor) {
>>>    		return ERR_PTR(-ENODEV);
>>> @@ -1067,7 +1055,7 @@ static void drm_core_exit(void)
>>>    	unregister_chrdev(DRM_MAJOR, "drm");
>>>    	debugfs_remove(drm_debugfs_root);
>>>    	drm_sysfs_destroy();
>>> -	idr_destroy(&drm_minors_idr);
>> [JZ] Should we call xa_destroy instead here?
> We could, if we expect the xarray to potentially not be empty.
>  From what I understand - all minors should be released at this point.
[JZ] In practice,  adding destroy here will be better.
>
> Thanks,
> -Michał
>
>>> +	WARN_ON(!xa_empty(&drm_minors_xa));
>>>    	drm_connector_ida_destroy();
>>>    }
>>> @@ -1076,7 +1064,6 @@ static int __init drm_core_init(void)
>>>    	int ret;
>>>    	drm_connector_ida_init();
>>> -	idr_init(&drm_minors_idr);
>>>    	drm_memcpy_init_early();
>>>    	ret = drm_sysfs_init();

Re: [PATCH v6 1/4] drm: Use XArray instead of IDR for minors

[Matthew Wilcox]

On Tue, Aug 29, 2023 at 01:34:22PM -0400, James Zhu wrote:
> > > > @@ -1067,7 +1055,7 @@ static void drm_core_exit(void)
> > > >    	unregister_chrdev(DRM_MAJOR, "drm");
> > > >    	debugfs_remove(drm_debugfs_root);
> > > >    	drm_sysfs_destroy();
> > > > -	idr_destroy(&drm_minors_idr);
> > > [JZ] Should we call xa_destroy instead here?
> > We could, if we expect the xarray to potentially not be empty.
> >  From what I understand - all minors should be released at this point.
> [JZ] In practice,  adding destroy here will be better.

Why do you say that?

Re: [PATCH v6 1/4] drm: Use XArray instead of IDR for minors

[James Zhu]

On 2023-08-29 14:33, Matthew Wilcox wrote:
> On Tue, Aug 29, 2023 at 01:34:22PM -0400, James Zhu wrote:
>>>>> @@ -1067,7 +1055,7 @@ static void drm_core_exit(void)
>>>>>     	unregister_chrdev(DRM_MAJOR, "drm");
>>>>>     	debugfs_remove(drm_debugfs_root);
>>>>>     	drm_sysfs_destroy();
>>>>> -	idr_destroy(&drm_minors_idr);
>>>> [JZ] Should we call xa_destroy instead here?
>>> We could, if we expect the xarray to potentially not be empty.
>>>   From what I understand - all minors should be released at this point.
>> [JZ] In practice,  adding destroy here will be better.
> Why do you say that?
[JZ] In case, the future, INIT adds something, need DESTROY to free 
completely.

Re: [PATCH v6 1/4] drm: Use XArray instead of IDR for minors

[Matthew Wilcox]

On Tue, Aug 29, 2023 at 02:35:34PM -0400, James Zhu wrote:
> 
> On 2023-08-29 14:33, Matthew Wilcox wrote:
> > On Tue, Aug 29, 2023 at 01:34:22PM -0400, James Zhu wrote:
> > > > > > @@ -1067,7 +1055,7 @@ static void drm_core_exit(void)
> > > > > >     	unregister_chrdev(DRM_MAJOR, "drm");
> > > > > >     	debugfs_remove(drm_debugfs_root);
> > > > > >     	drm_sysfs_destroy();
> > > > > > -	idr_destroy(&drm_minors_idr);
> > > > > [JZ] Should we call xa_destroy instead here?
> > > > We could, if we expect the xarray to potentially not be empty.
> > > >   From what I understand - all minors should be released at this point.
> > > [JZ] In practice,  adding destroy here will be better.
> > Why do you say that?
> [JZ] In case, the future, INIT adds something, need DESTROY to free
> completely.

That isn't going to happen.

[PATCH v6 2/4] accel: Use XArray instead of IDR for minors

[Michał Winiarski]

Accel minor management is based on DRM (and is also using struct
drm_minor internally), since DRM is using XArray for minors, it makes
sense to also convert accel.
As the two implementations are identical (only difference being the
underlying xarray), move the accel_minor_* functionality to DRM.

Signed-off-by: Michał Winiarski <michal.winiarski@intel.com>
---
 drivers/accel/drm_accel.c      | 110 +++------------------------------
 drivers/gpu/drm/drm_drv.c      |  66 ++++++++++----------
 drivers/gpu/drm/drm_file.c     |   2 +-
 drivers/gpu/drm/drm_internal.h |   4 --
 include/drm/drm_accel.h        |  18 +-----
 include/drm/drm_file.h         |   5 ++
 6 files changed, 47 insertions(+), 158 deletions(-)

diff --git a/drivers/accel/drm_accel.c b/drivers/accel/drm_accel.c
index 4a9baf02439e..8827cb78ca9d 100644
--- a/drivers/accel/drm_accel.c
+++ b/drivers/accel/drm_accel.c
@@ -8,7 +8,7 @@
 
 #include <linux/debugfs.h>
 #include <linux/device.h>
-#include <linux/idr.h>
+#include <linux/xarray.h>
 
 #include <drm/drm_accel.h>
 #include <drm/drm_debugfs.h>
@@ -17,8 +17,7 @@
 #include <drm/drm_ioctl.h>
 #include <drm/drm_print.h>
 
-static DEFINE_SPINLOCK(accel_minor_lock);
-static struct idr accel_minors_idr;
+DEFINE_XARRAY_ALLOC(accel_minors_xa);
 
 static struct dentry *accel_debugfs_root;
 static struct class *accel_class;
@@ -120,99 +119,6 @@ void accel_set_device_instance_params(struct device *kdev, int index)
 	kdev->type = &accel_sysfs_device_minor;
 }
 
-/**
- * accel_minor_alloc() - Allocates a new accel minor
- *
- * This function access the accel minors idr and allocates from it
- * a new id to represent a new accel minor
- *
- * Return: A new id on success or error code in case idr_alloc failed
- */
-int accel_minor_alloc(void)
-{
-	unsigned long flags;
-	int r;
-
-	spin_lock_irqsave(&accel_minor_lock, flags);
-	r = idr_alloc(&accel_minors_idr, NULL, 0, ACCEL_MAX_MINORS, GFP_NOWAIT);
-	spin_unlock_irqrestore(&accel_minor_lock, flags);
-
-	return r;
-}
-
-/**
- * accel_minor_remove() - Remove an accel minor
- * @index: The minor id to remove.
- *
- * This function access the accel minors idr and removes from
- * it the member with the id that is passed to this function.
- */
-void accel_minor_remove(int index)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&accel_minor_lock, flags);
-	idr_remove(&accel_minors_idr, index);
-	spin_unlock_irqrestore(&accel_minor_lock, flags);
-}
-
-/**
- * accel_minor_replace() - Replace minor pointer in accel minors idr.
- * @minor: Pointer to the new minor.
- * @index: The minor id to replace.
- *
- * This function access the accel minors idr structure and replaces the pointer
- * that is associated with an existing id. Because the minor pointer can be
- * NULL, we need to explicitly pass the index.
- *
- * Return: 0 for success, negative value for error
- */
-void accel_minor_replace(struct drm_minor *minor, int index)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&accel_minor_lock, flags);
-	idr_replace(&accel_minors_idr, minor, index);
-	spin_unlock_irqrestore(&accel_minor_lock, flags);
-}
-
-/*
- * Looks up the given minor-ID and returns the respective DRM-minor object. The
- * refence-count of the underlying device is increased so you must release this
- * object with accel_minor_release().
- *
- * The object can be only a drm_minor that represents an accel device.
- *
- * As long as you hold this minor, it is guaranteed that the object and the
- * minor->dev pointer will stay valid! However, the device may get unplugged and
- * unregistered while you hold the minor.
- */
-static struct drm_minor *accel_minor_acquire(unsigned int minor_id)
-{
-	struct drm_minor *minor;
-	unsigned long flags;
-
-	spin_lock_irqsave(&accel_minor_lock, flags);
-	minor = idr_find(&accel_minors_idr, minor_id);
-	if (minor)
-		drm_dev_get(minor->dev);
-	spin_unlock_irqrestore(&accel_minor_lock, flags);
-
-	if (!minor) {
-		return ERR_PTR(-ENODEV);
-	} else if (drm_dev_is_unplugged(minor->dev)) {
-		drm_dev_put(minor->dev);
-		return ERR_PTR(-ENODEV);
-	}
-
-	return minor;
-}
-
-static void accel_minor_release(struct drm_minor *minor)
-{
-	drm_dev_put(minor->dev);
-}
-
 /**
  * accel_open - open method for ACCEL file
  * @inode: device inode
@@ -230,7 +136,7 @@ int accel_open(struct inode *inode, struct file *filp)
 	struct drm_minor *minor;
 	int retcode;
 
-	minor = accel_minor_acquire(iminor(inode));
+	minor = drm_minor_acquire(&accel_minors_xa, iminor(inode));
 	if (IS_ERR(minor))
 		return PTR_ERR(minor);
 
@@ -249,7 +155,7 @@ int accel_open(struct inode *inode, struct file *filp)
 
 err_undo:
 	atomic_dec(&dev->open_count);
-	accel_minor_release(minor);
+	drm_minor_release(minor);
 	return retcode;
 }
 EXPORT_SYMBOL_GPL(accel_open);
@@ -260,7 +166,7 @@ static int accel_stub_open(struct inode *inode, struct file *filp)
 	struct drm_minor *minor;
 	int err;
 
-	minor = accel_minor_acquire(iminor(inode));
+	minor = drm_minor_acquire(&accel_minors_xa, iminor(inode));
 	if (IS_ERR(minor))
 		return PTR_ERR(minor);
 
@@ -277,7 +183,7 @@ static int accel_stub_open(struct inode *inode, struct file *filp)
 		err = 0;
 
 out:
-	accel_minor_release(minor);
+	drm_minor_release(minor);
 
 	return err;
 }
@@ -293,15 +199,13 @@ void accel_core_exit(void)
 	unregister_chrdev(ACCEL_MAJOR, "accel");
 	debugfs_remove(accel_debugfs_root);
 	accel_sysfs_destroy();
-	idr_destroy(&accel_minors_idr);
+	WARN_ON(!xa_empty(&accel_minors_xa));
 }
 
 int __init accel_core_init(void)
 {
 	int ret;
 
-	idr_init(&accel_minors_idr);
-
 	ret = accel_sysfs_init();
 	if (ret < 0) {
 		DRM_ERROR("Cannot create ACCEL class: %d\n", ret);
diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c
index 3faecb01186f..34b60196c443 100644
--- a/drivers/gpu/drm/drm_drv.c
+++ b/drivers/gpu/drm/drm_drv.c
@@ -55,7 +55,7 @@ MODULE_AUTHOR("Gareth Hughes, Leif Delgass, José Fonseca, Jon Smirl");
 MODULE_DESCRIPTION("DRM shared core routines");
 MODULE_LICENSE("GPL and additional rights");
 
-static DEFINE_XARRAY_ALLOC(drm_minors_xa);
+DEFINE_XARRAY_ALLOC(drm_minors_xa);
 
 /*
  * If the drm core fails to init for whatever reason,
@@ -83,6 +83,18 @@ DEFINE_STATIC_SRCU(drm_unplug_srcu);
  * registered and unregistered dynamically according to device-state.
  */
 
+static struct xarray *drm_minor_get_xa(enum drm_minor_type type)
+{
+	if (type == DRM_MINOR_PRIMARY || type == DRM_MINOR_RENDER)
+		return &drm_minors_xa;
+#if IS_ENABLED(CONFIG_DRM_ACCEL)
+	else if (type == DRM_MINOR_ACCEL)
+		return &accel_minors_xa;
+#endif
+	else
+		return ERR_PTR(-EOPNOTSUPP);
+}
+
 static struct drm_minor **drm_minor_get_slot(struct drm_device *dev,
 					     enum drm_minor_type type)
 {
@@ -106,18 +118,18 @@ static void drm_minor_alloc_release(struct drm_device *dev, void *data)
 
 	put_device(minor->kdev);
 
-	if (minor->type == DRM_MINOR_ACCEL)
-		accel_minor_remove(minor->index);
-	else
-		xa_erase(&drm_minors_xa, minor->index);
+	xa_erase(drm_minor_get_xa(minor->type), minor->index);
 }
 
-#define DRM_MINOR_LIMIT(t) ({ typeof(t) _t = (t); XA_LIMIT(64 * _t, 64 * _t + 63); })
+#define DRM_MINOR_LIMIT(t) ({ \
+	typeof(t) _t = (t); \
+	_t == DRM_MINOR_ACCEL ? XA_LIMIT(0, ACCEL_MAX_MINORS) : XA_LIMIT(64 * _t, 64 * _t + 63); \
+})
 
 static int drm_minor_alloc(struct drm_device *dev, enum drm_minor_type type)
 {
 	struct drm_minor *minor;
-	int index, r;
+	int r;
 
 	minor = drmm_kzalloc(dev, sizeof(*minor), GFP_KERNEL);
 	if (!minor)
@@ -126,18 +138,11 @@ static int drm_minor_alloc(struct drm_device *dev, enum drm_minor_type type)
 	minor->type = type;
 	minor->dev = dev;
 
-	if (type == DRM_MINOR_ACCEL) {
-		r = accel_minor_alloc();
-		index = r;
-	} else {
-		r = xa_alloc(&drm_minors_xa, &index, NULL, DRM_MINOR_LIMIT(type), GFP_KERNEL);
-	}
-
+	r = xa_alloc(drm_minor_get_xa(type), &minor->index,
+		     NULL, DRM_MINOR_LIMIT(type), GFP_KERNEL);
 	if (r < 0)
 		return r;
 
-	minor->index = index;
-
 	r = drmm_add_action_or_reset(dev, drm_minor_alloc_release, minor);
 	if (r)
 		return r;
@@ -177,16 +182,12 @@ static int drm_minor_register(struct drm_device *dev, enum drm_minor_type type)
 		goto err_debugfs;
 
 	/* replace NULL with @minor so lookups will succeed from now on */
-	if (minor->type == DRM_MINOR_ACCEL) {
-		accel_minor_replace(minor, minor->index);
-	} else {
-		entry = xa_store(&drm_minors_xa, minor->index, minor, GFP_KERNEL);
-		if (xa_is_err(entry)) {
-			ret = xa_err(entry);
-			goto err_debugfs;
-		}
-		WARN_ON(entry);
+	entry = xa_store(drm_minor_get_xa(type), minor->index, minor, GFP_KERNEL);
+	if (xa_is_err(entry)) {
+		ret = xa_err(entry);
+		goto err_debugfs;
 	}
+	WARN_ON(entry);
 
 	DRM_DEBUG("new minor registered %d\n", minor->index);
 	return 0;
@@ -205,10 +206,7 @@ static void drm_minor_unregister(struct drm_device *dev, enum drm_minor_type typ
 		return;
 
 	/* replace @minor with NULL so lookups will fail from now on */
-	if (minor->type == DRM_MINOR_ACCEL)
-		accel_minor_replace(NULL, minor->index);
-	else
-		xa_store(&drm_minors_xa, minor->index, NULL, GFP_KERNEL);
+	xa_store(drm_minor_get_xa(type), minor->index, NULL, GFP_KERNEL);
 
 	device_del(minor->kdev);
 	dev_set_drvdata(minor->kdev, NULL); /* safety belt */
@@ -224,15 +222,15 @@ static void drm_minor_unregister(struct drm_device *dev, enum drm_minor_type typ
  * minor->dev pointer will stay valid! However, the device may get unplugged and
  * unregistered while you hold the minor.
  */
-struct drm_minor *drm_minor_acquire(unsigned int minor_id)
+struct drm_minor *drm_minor_acquire(struct xarray *minor_xa, unsigned int minor_id)
 {
 	struct drm_minor *minor;
 
-	xa_lock(&drm_minors_xa);
-	minor = xa_load(&drm_minors_xa, minor_id);
+	xa_lock(minor_xa);
+	minor = xa_load(minor_xa, minor_id);
 	if (minor)
 		drm_dev_get(minor->dev);
-	xa_unlock(&drm_minors_xa);
+	xa_unlock(minor_xa);
 
 	if (!minor) {
 		return ERR_PTR(-ENODEV);
@@ -1020,7 +1018,7 @@ static int drm_stub_open(struct inode *inode, struct file *filp)
 
 	DRM_DEBUG("\n");
 
-	minor = drm_minor_acquire(iminor(inode));
+	minor = drm_minor_acquire(&drm_minors_xa, iminor(inode));
 	if (IS_ERR(minor))
 		return PTR_ERR(minor);
 
diff --git a/drivers/gpu/drm/drm_file.c b/drivers/gpu/drm/drm_file.c
index 883d83bc0e3d..20c9afe0f5a5 100644
--- a/drivers/gpu/drm/drm_file.c
+++ b/drivers/gpu/drm/drm_file.c
@@ -413,7 +413,7 @@ int drm_open(struct inode *inode, struct file *filp)
 	int retcode;
 	int need_setup = 0;
 
-	minor = drm_minor_acquire(iminor(inode));
+	minor = drm_minor_acquire(&drm_minors_xa, iminor(inode));
 	if (IS_ERR(minor))
 		return PTR_ERR(minor);
 
diff --git a/drivers/gpu/drm/drm_internal.h b/drivers/gpu/drm/drm_internal.h
index d7e023bbb0d5..cb46f4c61991 100644
--- a/drivers/gpu/drm/drm_internal.h
+++ b/drivers/gpu/drm/drm_internal.h
@@ -77,10 +77,6 @@ void drm_prime_destroy_file_private(struct drm_prime_file_private *prime_fpriv);
 void drm_prime_remove_buf_handle(struct drm_prime_file_private *prime_fpriv,
 				 uint32_t handle);
 
-/* drm_drv.c */
-struct drm_minor *drm_minor_acquire(unsigned int minor_id);
-void drm_minor_release(struct drm_minor *minor);
-
 /* drm_managed.c */
 void drm_managed_release(struct drm_device *dev);
 void drmm_add_final_kfree(struct drm_device *dev, void *container);
diff --git a/include/drm/drm_accel.h b/include/drm/drm_accel.h
index d4955062c77e..f93e23985f4e 100644
--- a/include/drm/drm_accel.h
+++ b/include/drm/drm_accel.h
@@ -51,11 +51,10 @@
 
 #if IS_ENABLED(CONFIG_DRM_ACCEL)
 
+extern struct xarray accel_minors_xa;
+
 void accel_core_exit(void);
 int accel_core_init(void);
-void accel_minor_remove(int index);
-int accel_minor_alloc(void);
-void accel_minor_replace(struct drm_minor *minor, int index);
 void accel_set_device_instance_params(struct device *kdev, int index);
 int accel_open(struct inode *inode, struct file *filp);
 void accel_debugfs_init(struct drm_minor *minor, int minor_id);
@@ -72,19 +71,6 @@ static inline int __init accel_core_init(void)
 	return 0;
 }
 
-static inline void accel_minor_remove(int index)
-{
-}
-
-static inline int accel_minor_alloc(void)
-{
-	return -EOPNOTSUPP;
-}
-
-static inline void accel_minor_replace(struct drm_minor *minor, int index)
-{
-}
-
 static inline void accel_set_device_instance_params(struct device *kdev, int index)
 {
 }
diff --git a/include/drm/drm_file.h b/include/drm/drm_file.h
index 010239392adf..7fc09460051f 100644
--- a/include/drm/drm_file.h
+++ b/include/drm/drm_file.h
@@ -45,6 +45,8 @@ struct drm_printer;
 struct device;
 struct file;
 
+extern struct xarray drm_minors_xa;
+
 /*
  * FIXME: Not sure we want to have drm_minor here in the end, but to avoid
  * header include loops we need it here for now.
@@ -420,6 +422,9 @@ static inline bool drm_is_accel_client(const struct drm_file *file_priv)
 	return file_priv->minor->type == DRM_MINOR_ACCEL;
 }
 
+struct drm_minor *drm_minor_acquire(struct xarray *minors_xa, unsigned int minor_id);
+void drm_minor_release(struct drm_minor *minor);
+
 int drm_open(struct inode *inode, struct file *filp);
 int drm_open_helper(struct file *filp, struct drm_minor *minor);
 ssize_t drm_read(struct file *filp, char __user *buffer,
-- 
2.41.0

[PATCH v6 3/4] drm: Expand max DRM device number to full MINORBITS

[Michał Winiarski]

Having a limit of 64 DRM devices is not good enough for modern world
where we have multi-GPU servers, SR-IOV virtual functions and virtual
devices used for testing.
Let's utilize full minor range for DRM devices.
To avoid regressing the existing userspace, we're still maintaining the
numbering scheme where 0-63 is used for primary, 64-127 is reserved
(formerly for control) and 128-191 is used for render.
For minors >= 192, we're allocating minors dynamically on a first-come,
first-served basis.

Signed-off-by: Michał Winiarski <michal.winiarski@intel.com>
---
 drivers/gpu/drm/drm_drv.c | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c
index 34b60196c443..c2c6e80e6b31 100644
--- a/drivers/gpu/drm/drm_drv.c
+++ b/drivers/gpu/drm/drm_drv.c
@@ -121,10 +121,19 @@ static void drm_minor_alloc_release(struct drm_device *dev, void *data)
 	xa_erase(drm_minor_get_xa(minor->type), minor->index);
 }
 
+/*
+ * DRM used to support 64 devices, for backwards compatibility we need to maintain the
+ * minor allocation scheme where minors 0-63 are primary nodes, 64-127 are control nodes,
+ * and 128-191 are render nodes.
+ * After reaching the limit, we're allocating minors dynamically - first-come, first-serve.
+ * Accel nodes are using a distinct major, so the minors are allocated in continuous 0-MAX
+ * range.
+ */
 #define DRM_MINOR_LIMIT(t) ({ \
 	typeof(t) _t = (t); \
 	_t == DRM_MINOR_ACCEL ? XA_LIMIT(0, ACCEL_MAX_MINORS) : XA_LIMIT(64 * _t, 64 * _t + 63); \
 })
+#define DRM_EXTENDED_MINOR_LIMIT XA_LIMIT(192, (1 << MINORBITS) - 1)
 
 static int drm_minor_alloc(struct drm_device *dev, enum drm_minor_type type)
 {
@@ -140,6 +149,9 @@ static int drm_minor_alloc(struct drm_device *dev, enum drm_minor_type type)
 
 	r = xa_alloc(drm_minor_get_xa(type), &minor->index,
 		     NULL, DRM_MINOR_LIMIT(type), GFP_KERNEL);
+	if (r == -EBUSY && (type == DRM_MINOR_PRIMARY || type == DRM_MINOR_RENDER))
+		r = xa_alloc(&drm_minors_xa, &minor->index,
+			     NULL, DRM_EXTENDED_MINOR_LIMIT, GFP_KERNEL);
 	if (r < 0)
 		return r;
 
-- 
2.41.0

Re: [PATCH v6 3/4] drm: Expand max DRM device number to full MINORBITS

[Simon Ser]

On Monday, July 24th, 2023 at 23:14, Michał Winiarski <michal.winiarski@intel.com> wrote:

> Having a limit of 64 DRM devices is not good enough for modern world
> where we have multi-GPU servers, SR-IOV virtual functions and virtual
> devices used for testing.
> Let's utilize full minor range for DRM devices.
> To avoid regressing the existing userspace, we're still maintaining the
> numbering scheme where 0-63 is used for primary, 64-127 is reserved
> (formerly for control) and 128-191 is used for render.
> For minors >= 192, we're allocating minors dynamically on a first-come,
> first-served basis.

In general the approach looks good to me. Old libdrm will see the new
nodes as nodes with an unknown type when it tries to infer the nod type
from the minor, which is as good as it gets.

We do need patches to stop trying to infer the node type from the minor
in libdrm, though. Emil has suggested using sysfs, which we already do
in a few places in libdrm.

Re: [PATCH v6 3/4] drm: Expand max DRM device number to full MINORBITS

[Christian König]

Am 26.07.23 um 20:15 schrieb Simon Ser:
> On Monday, July 24th, 2023 at 23:14, Michał Winiarski <michal.winiarski@intel.com> wrote:
>
>> Having a limit of 64 DRM devices is not good enough for modern world
>> where we have multi-GPU servers, SR-IOV virtual functions and virtual
>> devices used for testing.
>> Let's utilize full minor range for DRM devices.
>> To avoid regressing the existing userspace, we're still maintaining the
>> numbering scheme where 0-63 is used for primary, 64-127 is reserved
>> (formerly for control) and 128-191 is used for render.
>> For minors >= 192, we're allocating minors dynamically on a first-come,
>> first-served basis.
> In general the approach looks good to me. Old libdrm will see the new
> nodes as nodes with an unknown type when it tries to infer the nod type
> from the minor, which is as good as it gets.

Yeah, agree. I wouldn't upstream patch #4, but apart from that it looks 
like it shouldn't break anything which wasn't broken before.

> We do need patches to stop trying to infer the node type from the minor
> in libdrm, though. Emil has suggested using sysfs, which we already do
> in a few places in libdrm.

That sounds like a really good idea to me as well.

But what do we do with DRM_MAX_MINOR? Change it or keep it and say apps 
should use drmGetDevices2() like Emil suggested?

Regards,
Christian.

Re: [PATCH v6 3/4] drm: Expand max DRM device number to full MINORBITS

[Simon Ser]

On Thursday, July 27th, 2023 at 14:01, Christian König <christian.koenig@amd.com> wrote:

> > We do need patches to stop trying to infer the node type from the minor
> > in libdrm, though. Emil has suggested using sysfs, which we already do
> > in a few places in libdrm.
> 
> That sounds like a really good idea to me as well.
> 
> But what do we do with DRM_MAX_MINOR? Change it or keep it and say apps
> should use drmGetDevices2() like Emil suggested?

DRM_MAX_MINOR has been bumped to 64 now.

With the new minor allocation scheme, DRM_MAX_MINOR is meaningless
because there is no "max minor per type" concept anymore: the minor no
longer contains the type.

So I'd suggest leaving it as-is (so that old apps still continue to
work on systems with < 64 devices like they do today) and mark it as
deprecated.

Re: [PATCH v6 3/4] drm: Expand max DRM device number to full MINORBITS

[Christian König]

Am 28.07.23 um 16:22 schrieb Simon Ser:
> On Thursday, July 27th, 2023 at 14:01, Christian König <christian.koenig@amd.com> wrote:
>
>>> We do need patches to stop trying to infer the node type from the minor
>>> in libdrm, though. Emil has suggested using sysfs, which we already do
>>> in a few places in libdrm.
>> That sounds like a really good idea to me as well.
>>
>> But what do we do with DRM_MAX_MINOR? Change it or keep it and say apps
>> should use drmGetDevices2() like Emil suggested?
> DRM_MAX_MINOR has been bumped to 64 now.
>
> With the new minor allocation scheme, DRM_MAX_MINOR is meaningless
> because there is no "max minor per type" concept anymore: the minor no
> longer contains the type.
>
> So I'd suggest leaving it as-is (so that old apps still continue to
> work on systems with < 64 devices like they do today) and mark it as
> deprecated.

Sounds like a plan to me.

Regards,
Christian.

Re: [PATCH v6 3/4] drm: Expand max DRM device number to full MINORBITS

[James Zhu]

I would like if these kernel patches are accepted by everyone, If yes, 
when they can be upstream.

I have a MR for libdrm to support drm nodes type up to 2^MINORBITS  
nodes which can work with these patches,

https://gitlab.freedesktop.org/mesa/drm/-/merge_requests/305

Thanks!

James

On 2023-08-08 09:55, Christian König wrote:
> Am 28.07.23 um 16:22 schrieb Simon Ser:
>> On Thursday, July 27th, 2023 at 14:01, Christian König 
>> <christian.koenig@amd.com> wrote:
>>
>>>> We do need patches to stop trying to infer the node type from the 
>>>> minor
>>>> in libdrm, though. Emil has suggested using sysfs, which we already do
>>>> in a few places in libdrm.
>>> That sounds like a really good idea to me as well.
>>>
>>> But what do we do with DRM_MAX_MINOR? Change it or keep it and say apps
>>> should use drmGetDevices2() like Emil suggested?
>> DRM_MAX_MINOR has been bumped to 64 now.
>>
>> With the new minor allocation scheme, DRM_MAX_MINOR is meaningless
>> because there is no "max minor per type" concept anymore: the minor no
>> longer contains the type.
>>
>> So I'd suggest leaving it as-is (so that old apps still continue to
>> work on systems with < 64 devices like they do today) and mark it as
>> deprecated.
>
> Sounds like a plan to me.
>
> Regards,
> Christian.

Re: [PATCH v6 3/4] drm: Expand max DRM device number to full MINORBITS

[Simon Ser]

On Tuesday, August 8th, 2023 at 17:04, James Zhu <jamesz@amd.com> wrote:

> I have a MR for libdrm to support drm nodes type up to 2^MINORBITS
> nodes which can work with these patches,
> 
> https://gitlab.freedesktop.org/mesa/drm/-/merge_requests/305

FWIW, this MR has been merged, so in theory this kernel patch should
work fine with latest libdrm.

Re: [PATCH v6 3/4] drm: Expand max DRM device number to full MINORBITS

[Simon Ser]

On Wednesday, August 23rd, 2023 at 12:53, Simon Ser <contact@emersion.fr> wrote:

> On Tuesday, August 8th, 2023 at 17:04, James Zhu jamesz@amd.com wrote:
> 
> > I have a MR for libdrm to support drm nodes type up to 2^MINORBITS
> > nodes which can work with these patches,
> > 
> > https://gitlab.freedesktop.org/mesa/drm/-/merge_requests/305
> 
> FWIW, this MR has been merged, so in theory this kernel patch should
> work fine with latest libdrm.

Hm, we might want to adjust MAX_DRM_NODES still. It's set to 256
currently, which should be enough for 128 DRM devices, but not more.
Not bumping this value will make drmGetDevices2() print a warning and
not return all devices on systems with more than 128 DRM devices.

Re: [PATCH v6 3/4] drm: Expand max DRM device number to full MINORBITS

[James Zhu]

Hi Simon,

Thanks! Yes, this kernel patch should work with latest libdrm.

Best regards!

James Zhu

On 2023-08-23 06:53, Simon Ser wrote:
> On Tuesday, August 8th, 2023 at 17:04, James Zhu <jamesz@amd.com> wrote:
>
>> I have a MR for libdrm to support drm nodes type up to 2^MINORBITS
>> nodes which can work with these patches,
>>
>> https://gitlab.freedesktop.org/mesa/drm/-/merge_requests/305
> FWIW, this MR has been merged, so in theory this kernel patch should
> work fine with latest libdrm.

[PATCH v6 4/4] drm: Introduce force_extended_minors modparam

[Michał Winiarski]

While there is support for >64 DRM devices on kernel side, existing
userspace may still have some hardcoded assumptions and it's possible
that it will require changes to be able to use more than 64 devices.
Add a modparam to simplify testing and development of >64 devices
support on userspace side by allocating minors from the >=192 range
(without the need of having >64 physical devices connected).

Signed-off-by: Michał Winiarski <michal.winiarski@intel.com>
---
 drivers/gpu/drm/drm_drv.c | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c
index c2c6e80e6b31..ef6d7b498784 100644
--- a/drivers/gpu/drm/drm_drv.c
+++ b/drivers/gpu/drm/drm_drv.c
@@ -57,6 +57,11 @@ MODULE_LICENSE("GPL and additional rights");
 
 DEFINE_XARRAY_ALLOC(drm_minors_xa);
 
+static bool force_extended_minors;
+module_param_unsafe(force_extended_minors, bool, 0400);
+MODULE_PARM_DESC(force_extended_minors,
+		 "Don't allocate minors in 0-192 range. This can be used for testing userspace support for >64 drm devices (default: false)");
+
 /*
  * If the drm core fails to init for whatever reason,
  * we should prevent any drivers from registering with it.
@@ -138,7 +143,7 @@ static void drm_minor_alloc_release(struct drm_device *dev, void *data)
 static int drm_minor_alloc(struct drm_device *dev, enum drm_minor_type type)
 {
 	struct drm_minor *minor;
-	int r;
+	int r = -EBUSY;
 
 	minor = drmm_kzalloc(dev, sizeof(*minor), GFP_KERNEL);
 	if (!minor)
@@ -147,8 +152,9 @@ static int drm_minor_alloc(struct drm_device *dev, enum drm_minor_type type)
 	minor->type = type;
 	minor->dev = dev;
 
-	r = xa_alloc(drm_minor_get_xa(type), &minor->index,
-		     NULL, DRM_MINOR_LIMIT(type), GFP_KERNEL);
+	if (type == DRM_MINOR_ACCEL || !force_extended_minors)
+		r = xa_alloc(drm_minor_get_xa(type), &minor->index,
+			     NULL, DRM_MINOR_LIMIT(type), GFP_KERNEL);
 	if (r == -EBUSY && (type == DRM_MINOR_PRIMARY || type == DRM_MINOR_RENDER))
 		r = xa_alloc(&drm_minors_xa, &minor->index,
 			     NULL, DRM_EXTENDED_MINOR_LIMIT, GFP_KERNEL);
-- 
2.41.0

Re: [PATCH v6 0/4] drm: Use full allocated minor range for DRM

[Eric Pilmore]

> On Jul 24, 2023, at 2:14 PM, Michał Winiarski <michal.winiarski@intel.com> wrote:
> 
> 64 DRM device nodes is not enough for everyone.
> Upgrade it to ~512K (which definitely is more than enough).
> 
> To allow testing userspace support for >64 devices, add additional DRM
> modparam (force_extended_minors) which causes DRM to skip allocating minors
> in 0-192 range.
> Additionally - convert minors to use XArray instead of IDR to simplify the
> locking.
> 
> v1 -> v2:
> Don't touch DRM_MINOR_CONTROL and its range (Simon Ser)
> 
> v2 -> v3:
> Don't use legacy scheme for >=192 minor range (Dave Airlie)
> Add modparam for testing (Dave Airlie)
> Add lockdep annotation for IDR (Daniel Vetter)
> 
> v3 -> v4:
> Convert from IDR to XArray (Matthew Wilcox)
> 
> v4 -> v5:
> Fixup IDR to XArray conversion (Matthew Wilcox)
> 
> v5 -> v6:
> Also convert Accel to XArray
> Rename skip_legacy_minors to force_extended_minors
> 
> Michał Winiarski (4):
>  drm: Use XArray instead of IDR for minors
>  accel: Use XArray instead of IDR for minors
>  drm: Expand max DRM device number to full MINORBITS
>  drm: Introduce force_extended_minors modparam
> 
> drivers/accel/drm_accel.c      | 110 +++------------------------------
> drivers/gpu/drm/drm_drv.c      | 105 ++++++++++++++++---------------
> drivers/gpu/drm/drm_file.c     |   2 +-
> drivers/gpu/drm/drm_internal.h |   4 --
> include/drm/drm_accel.h        |  18 +-----
> include/drm/drm_file.h         |   5 ++
> 6 files changed, 69 insertions(+), 175 deletions(-)
> 
> -- 
> 2.41.0
> 


Hi Michal,

What is the status on this patch? Did it ever get accepted upstream?
If so, what release? I don’t see the changes in the latest Linux kernel.
I am working on a system that involves a large number of GPUs, where
each GPU consumes a number of DRM devices. As such, I’m easily
exceeding the current limit of 64 in the (6.6) kernel. To workaround this issue,
I have temporarily picked up this patch which is doing the trick, but now
I’m wondering if this patch has seen the light of day in the Linux kernel.

Thanks for any info!

Regards,
Eric

Re: [PATCH v6 0/4] drm: Use full allocated minor range for DRM

[Alex Deucher]

Are there any objections to this series?  We have been running into
this limit as a problem for a while now on big servers.

Alex

On Mon, Jul 24, 2023 at 5:15 PM Michał Winiarski
<michal.winiarski@intel.com> wrote:
>
> 64 DRM device nodes is not enough for everyone.
> Upgrade it to ~512K (which definitely is more than enough).
>
> To allow testing userspace support for >64 devices, add additional DRM
> modparam (force_extended_minors) which causes DRM to skip allocating minors
> in 0-192 range.
> Additionally - convert minors to use XArray instead of IDR to simplify the
> locking.
>
> v1 -> v2:
> Don't touch DRM_MINOR_CONTROL and its range (Simon Ser)
>
> v2 -> v3:
> Don't use legacy scheme for >=192 minor range (Dave Airlie)
> Add modparam for testing (Dave Airlie)
> Add lockdep annotation for IDR (Daniel Vetter)
>
> v3 -> v4:
> Convert from IDR to XArray (Matthew Wilcox)
>
> v4 -> v5:
> Fixup IDR to XArray conversion (Matthew Wilcox)
>
> v5 -> v6:
> Also convert Accel to XArray
> Rename skip_legacy_minors to force_extended_minors
>
> Michał Winiarski (4):
>   drm: Use XArray instead of IDR for minors
>   accel: Use XArray instead of IDR for minors
>   drm: Expand max DRM device number to full MINORBITS
>   drm: Introduce force_extended_minors modparam
>
>  drivers/accel/drm_accel.c      | 110 +++------------------------------
>  drivers/gpu/drm/drm_drv.c      | 105 ++++++++++++++++---------------
>  drivers/gpu/drm/drm_file.c     |   2 +-
>  drivers/gpu/drm/drm_internal.h |   4 --
>  include/drm/drm_accel.h        |  18 +-----
>  include/drm/drm_file.h         |   5 ++
>  6 files changed, 69 insertions(+), 175 deletions(-)
>
> --
> 2.41.0
>

Re: [PATCH v6 0/4] drm: Use full allocated minor range for DRM

[Michał Winiarski]

On Mon, Aug 12, 2024 at 01:38:38PM GMT, Alex Deucher wrote:
> Are there any objections to this series?  We have been running into
> this limit as a problem for a while now on big servers.

I don't think there were any objections, just a general lack of
interest - so there are no R-b / Acks.
If you're interested to have a go at it - I can resend it.
It should still apply on latest drm-tip.

-Michał

> 
> Alex
> 
> On Mon, Jul 24, 2023 at 5:15 PM Michał Winiarski
> <michal.winiarski@intel.com> wrote:
> >
> > 64 DRM device nodes is not enough for everyone.
> > Upgrade it to ~512K (which definitely is more than enough).
> >
> > To allow testing userspace support for >64 devices, add additional DRM
> > modparam (force_extended_minors) which causes DRM to skip allocating minors
> > in 0-192 range.
> > Additionally - convert minors to use XArray instead of IDR to simplify the
> > locking.
> >
> > v1 -> v2:
> > Don't touch DRM_MINOR_CONTROL and its range (Simon Ser)
> >
> > v2 -> v3:
> > Don't use legacy scheme for >=192 minor range (Dave Airlie)
> > Add modparam for testing (Dave Airlie)
> > Add lockdep annotation for IDR (Daniel Vetter)
> >
> > v3 -> v4:
> > Convert from IDR to XArray (Matthew Wilcox)
> >
> > v4 -> v5:
> > Fixup IDR to XArray conversion (Matthew Wilcox)
> >
> > v5 -> v6:
> > Also convert Accel to XArray
> > Rename skip_legacy_minors to force_extended_minors
> >
> > Michał Winiarski (4):
> >   drm: Use XArray instead of IDR for minors
> >   accel: Use XArray instead of IDR for minors
> >   drm: Expand max DRM device number to full MINORBITS
> >   drm: Introduce force_extended_minors modparam
> >
> >  drivers/accel/drm_accel.c      | 110 +++------------------------------
> >  drivers/gpu/drm/drm_drv.c      | 105 ++++++++++++++++---------------
> >  drivers/gpu/drm/drm_file.c     |   2 +-
> >  drivers/gpu/drm/drm_internal.h |   4 --
> >  include/drm/drm_accel.h        |  18 +-----
> >  include/drm/drm_file.h         |   5 ++
> >  6 files changed, 69 insertions(+), 175 deletions(-)
> >
> > --
> > 2.41.0
> >

Re: [PATCH v6 0/4] drm: Use full allocated minor range for DRM

[James Zhu]

Hi Michal

I did give Tested-by before. If you need Acks, Here  I can give
Acked-by: James Zhu <James.Zhu@amd.com> for the series

Best Regards!

James

On 2024-08-13 20:18, Michał Winiarski wrote:
> On Mon, Aug 12, 2024 at 01:38:38PM GMT, Alex Deucher wrote:
>> Are there any objections to this series?  We have been running into
>> this limit as a problem for a while now on big servers.
> I don't think there were any objections, just a general lack of
> interest - so there are no R-b / Acks.
> If you're interested to have a go at it - I can resend it.
> It should still apply on latest drm-tip.
>
> -Michał
>
>> Alex
>>
>> On Mon, Jul 24, 2023 at 5:15 PM Michał Winiarski
>> <michal.winiarski@intel.com> wrote:
>>> 64 DRM device nodes is not enough for everyone.
>>> Upgrade it to ~512K (which definitely is more than enough).
>>>
>>> To allow testing userspace support for >64 devices, add additional DRM
>>> modparam (force_extended_minors) which causes DRM to skip allocating minors
>>> in 0-192 range.
>>> Additionally - convert minors to use XArray instead of IDR to simplify the
>>> locking.
>>>
>>> v1 -> v2:
>>> Don't touch DRM_MINOR_CONTROL and its range (Simon Ser)
>>>
>>> v2 -> v3:
>>> Don't use legacy scheme for >=192 minor range (Dave Airlie)
>>> Add modparam for testing (Dave Airlie)
>>> Add lockdep annotation for IDR (Daniel Vetter)
>>>
>>> v3 -> v4:
>>> Convert from IDR to XArray (Matthew Wilcox)
>>>
>>> v4 -> v5:
>>> Fixup IDR to XArray conversion (Matthew Wilcox)
>>>
>>> v5 -> v6:
>>> Also convert Accel to XArray
>>> Rename skip_legacy_minors to force_extended_minors
>>>
>>> Michał Winiarski (4):
>>>    drm: Use XArray instead of IDR for minors
>>>    accel: Use XArray instead of IDR for minors
>>>    drm: Expand max DRM device number to full MINORBITS
>>>    drm: Introduce force_extended_minors modparam
>>>
>>>   drivers/accel/drm_accel.c      | 110 +++------------------------------
>>>   drivers/gpu/drm/drm_drv.c      | 105 ++++++++++++++++---------------
>>>   drivers/gpu/drm/drm_file.c     |   2 +-
>>>   drivers/gpu/drm/drm_internal.h |   4 --
>>>   include/drm/drm_accel.h        |  18 +-----
>>>   include/drm/drm_file.h         |   5 ++
>>>   6 files changed, 69 insertions(+), 175 deletions(-)
>>>
>>> --
>>> 2.41.0
>>>

Re: [PATCH v6 0/4] drm: Use full allocated minor range for DRM

[Alex Deucher]

On Tue, Aug 13, 2024 at 8:19 PM Michał Winiarski
<michal.winiarski@intel.com> wrote:
>
> On Mon, Aug 12, 2024 at 01:38:38PM GMT, Alex Deucher wrote:
> > Are there any objections to this series?  We have been running into
> > this limit as a problem for a while now on big servers.
>
> I don't think there were any objections, just a general lack of
> interest - so there are no R-b / Acks.
> If you're interested to have a go at it - I can resend it.
> It should still apply on latest drm-tip.

Yeah that would be great!

Thanks,

Alex

>
> -Michał
>
> >
> > Alex
> >
> > On Mon, Jul 24, 2023 at 5:15 PM Michał Winiarski
> > <michal.winiarski@intel.com> wrote:
> > >
> > > 64 DRM device nodes is not enough for everyone.
> > > Upgrade it to ~512K (which definitely is more than enough).
> > >
> > > To allow testing userspace support for >64 devices, add additional DRM
> > > modparam (force_extended_minors) which causes DRM to skip allocating minors
> > > in 0-192 range.
> > > Additionally - convert minors to use XArray instead of IDR to simplify the
> > > locking.
> > >
> > > v1 -> v2:
> > > Don't touch DRM_MINOR_CONTROL and its range (Simon Ser)
> > >
> > > v2 -> v3:
> > > Don't use legacy scheme for >=192 minor range (Dave Airlie)
> > > Add modparam for testing (Dave Airlie)
> > > Add lockdep annotation for IDR (Daniel Vetter)
> > >
> > > v3 -> v4:
> > > Convert from IDR to XArray (Matthew Wilcox)
> > >
> > > v4 -> v5:
> > > Fixup IDR to XArray conversion (Matthew Wilcox)
> > >
> > > v5 -> v6:
> > > Also convert Accel to XArray
> > > Rename skip_legacy_minors to force_extended_minors
> > >
> > > Michał Winiarski (4):
> > >   drm: Use XArray instead of IDR for minors
> > >   accel: Use XArray instead of IDR for minors
> > >   drm: Expand max DRM device number to full MINORBITS
> > >   drm: Introduce force_extended_minors modparam
> > >
> > >  drivers/accel/drm_accel.c      | 110 +++------------------------------
> > >  drivers/gpu/drm/drm_drv.c      | 105 ++++++++++++++++---------------
> > >  drivers/gpu/drm/drm_file.c     |   2 +-
> > >  drivers/gpu/drm/drm_internal.h |   4 --
> > >  include/drm/drm_accel.h        |  18 +-----
> > >  include/drm/drm_file.h         |   5 ++
> > >  6 files changed, 69 insertions(+), 175 deletions(-)
> > >
> > > --
> > > 2.41.0
> > >