[RFC] Some fixes to allow for more than 128 md devices.

[RFC] Some fixes to allow for more than 128 md devices.

[Robert LeBlanc]

I'm stuck and need some help getting this across the finish line. This
is in no way complete, but to help show what I'm working on.

When we added more than 128 md devices, we started getting failures.
Looking through the code it seems that the minor dev number was being
stored in an int and causing overflow and wrecking havoc on everything.
I finally got the mknod in mdadm to correctly make the dev node with
minors up to 1048574 as expected in the mdadm code. However, I can
only create md devices up to 511. Trying to create an md higher than
that has an error where the device can't be read/opened strace reports:
open("/dev/.tmp.md.15341:9:1048574", O_RDWR|O_EXCL|O_DIRECT) = -1 ENXIO
(No such device or address)
while Python reports:
IOError: [Errno 6] No such device or address: '/dev/.tmp.md.3279:9:512'

A corresponding node is not created in /sys/block/md* for mds over 511.

I believe that there may be a bug in the kernel code that is now being
hit. After looking through the kernel code, I can't seem to find where
this might be. Please help me by either pointing me to the source
location that this might be a problem or fixing it based on these
patches I've worked on so far. I'm using 4.7.0 currently.

I'm using this for testing:
./mdadm --create /dev/md1048574 --assume-clean --verbose --level=1 \
--raid-devices=2 /dev/loop0 missing

Yes, we have a real need for more than 128 and 512 md devices.

Please include me in any replies as I'm not on the ML.

Thank you.

Robert LeBlanc (1):
  Some fixes to allow for more than 128 md devices.

 Manage.c |  5 +++--
 lib.c    |  2 +-
 mdadm.h  |  6 +++---
 util.c   | 25 +++++++++++++------------
 4 files changed, 20 insertions(+), 18 deletions(-)

--
2.8.1

[RFC] Some fixes to allow for more than 128 md devices.

[Robert LeBlanc]

---
 Manage.c |  5 +++--
 lib.c    |  2 +-
 mdadm.h  |  6 +++---
 util.c   | 25 +++++++++++++------------
 4 files changed, 20 insertions(+), 18 deletions(-)

diff --git a/Manage.c b/Manage.c
index 30adc99..51d0eae 100644
--- a/Manage.c
+++ b/Manage.c
@@ -1380,7 +1380,8 @@ int Manage_subdevs(char *devname, int fd,
 	for (dv = devlist; dv; dv = dv->next) {
 		unsigned long rdev = 0; /* device to add/remove etc */
 		int rv;
-		int mj,mn;
+		int mj;
+		unsigned long mn;
 
 		raid_slot = -1;
 		if (dv->disposition == 'c') {
@@ -1486,7 +1487,7 @@ int Manage_subdevs(char *devname, int fd,
 			if (sysfd >= 0) {
 				char dn[20];
 				if (sysfs_fd_get_str(sysfd, dn, 20) > 0 &&
-				    sscanf(dn, "%d:%d", &mj,&mn) == 2) {
+				    sscanf(dn, "%d:%lu", &mj,&mn) == 2) {
 					rdev = makedev(mj,mn);
 					found = 1;
 				}
diff --git a/lib.c b/lib.c
index 621edf3..e0df6fd 100644
--- a/lib.c
+++ b/lib.c
@@ -99,7 +99,7 @@ char *fd2kname(int fd)
 	return NULL;
 }
 
-char *devid2devnm(int devid)
+char *devid2devnm(unsigned long devid)
 {
 	char path[30];
 	char link[200];
diff --git a/mdadm.h b/mdadm.h
index d209488..7de6589 100755
--- a/mdadm.h
+++ b/mdadm.h
@@ -1348,7 +1348,7 @@ extern int check_partitions(int fd, char *dname,
 			    unsigned long long size);
 
 extern int get_mdp_major(void);
-extern int get_maj_min(char *dev, int *major, int *minor);
+extern int get_maj_min(char *dev, int *major, unsigned long *minor);
 extern int dev_open(char *dev, int flags);
 extern int open_dev(char *devnm);
 extern void reopen_mddev(int mdfd);
@@ -1438,8 +1438,8 @@ extern char *find_free_devnm(int use_partitions);
 
 extern void put_md_name(char *name);
 extern char *devid2kname(int devid);
-extern char *devid2devnm(int devid);
-extern int devnm2devid(char *devnm);
+extern char *devid2devnm(unsigned long devid);
+extern unsigned long devnm2devid(char *devnm);
 extern char *get_md_name(char *devnm);
 
 extern char DefaultConfFile[];
diff --git a/util.c b/util.c
index 2bcb81f..939cf52 100644
--- a/util.c
+++ b/util.c
@@ -928,7 +928,7 @@ int get_data_disks(int level, int layout, int raid_disks)
 	return data_disks;
 }
 
-int devnm2devid(char *devnm)
+unsigned long devnm2devid(char *devnm)
 {
 	/* First look in /sys/block/$DEVNM/dev for %d:%d
 	 * If that fails, try parsing out a number
@@ -936,7 +936,8 @@ int devnm2devid(char *devnm)
 	char path[100];
 	char *ep;
 	int fd;
-	int mjr,mnr;
+	int mjr;
+	unsigned long mnr;
 
 	sprintf(path, "/sys/block/%s/dev", devnm);
 	fd = open(path, O_RDONLY);
@@ -946,18 +947,18 @@ int devnm2devid(char *devnm)
 		close(fd);
 		if (n > 0)
 			buf[n] = 0;
-		if (n > 0 && sscanf(buf, "%d:%d\n", &mjr, &mnr) == 2)
+		if (n > 0 && sscanf(buf, "%d:%lu\n", &mjr, &mnr) == 2)
 			return makedev(mjr, mnr);
 	}
 	if (strncmp(devnm, "md_d", 4) == 0 &&
 	    isdigit(devnm[4]) &&
-	    (mnr = strtoul(devnm+4, &ep, 10)) >= 0 &&
+	    (mnr = strtoul(devnm+4, &ep, 10)) &&
 	    ep > devnm && *ep == 0)
 		return makedev(get_mdp_major(), mnr << MdpMinorShift);
 
 	if (strncmp(devnm, "md", 2) == 0 &&
 	    isdigit(devnm[2]) &&
-	    (mnr = strtoul(devnm+2, &ep, 10)) >= 0 &&
+	    (mnr = strtoul(devnm+2, &ep, 10)) &&
 	    ep > devnm && *ep == 0)
 		return makedev(MD_MAJOR, mnr);
 
@@ -1020,12 +1021,12 @@ void put_md_name(char *name)
 }
 #endif /* !defined(MDASSEMBLE) || defined(MDASSEMBLE) && defined(MDASSEMBLE_AUTO) */
 
-int get_maj_min(char *dev, int *major, int *minor)
+int get_maj_min(char *dev, int *major, unsigned long *minor)
 {
 	char *e;
 	*major = strtoul(dev, &e, 0);
 	return (e > dev && *e == ':' && e[1] &&
-		(*minor = strtoul(e+1, &e, 0)) >= 0 &&
+		(*minor = strtoul(e+1, &e, 0)) &&
 		*e == 0);
 }
 
@@ -1037,13 +1038,13 @@ int dev_open(char *dev, int flags)
 	int fd = -1;
 	char devname[32];
 	int major;
-	int minor;
+	unsigned long minor;
 
 	if (!dev) return -1;
 	flags |= O_DIRECT;
 
 	if (get_maj_min(dev, &major, &minor)) {
-		snprintf(devname, sizeof(devname), "/dev/.tmp.md.%d:%d:%d",
+		snprintf(devname, sizeof(devname), "/dev/.tmp.md.%d:%d:%lu",
 			 (int)getpid(), major, minor);
 		if (mknod(devname, S_IFBLK|0600, makedev(major, minor)) == 0) {
 			fd = open(devname, flags);
@@ -1051,7 +1052,7 @@ int dev_open(char *dev, int flags)
 		}
 		if (fd < 0) {
 			/* Try /tmp as /dev appear to be read-only */
-			snprintf(devname, sizeof(devname), "/tmp/.tmp.md.%d:%d:%d",
+			snprintf(devname, sizeof(devname), "/tmp/.tmp.md.%d:%d:%lu",
 				 (int)getpid(), major, minor);
 			if (mknod(devname, S_IFBLK|0600, makedev(major, minor)) == 0) {
 				fd = open(devname, flags);
@@ -1069,7 +1070,7 @@ int open_dev_flags(char *devnm, int flags)
 	char buf[20];
 
 	devid = devnm2devid(devnm);
-	sprintf(buf, "%d:%d", major(devid), minor(devid));
+	sprintf(buf, "%d:%lu", major(devid), (unsigned long)minor(devid));
 	return dev_open(buf, flags);
 }
 
@@ -1083,7 +1084,7 @@ int open_dev_excl(char *devnm)
 	char buf[20];
 	int i;
 	int flags = O_RDWR;
-	int devid = devnm2devid(devnm);
+	unsigned long devid = devnm2devid(devnm);
 	long delay = 1000;
 
 	sprintf(buf, "%d:%d", major(devid), minor(devid));
-- 
2.8.1

Re: [RFC] Some fixes to allow for more than 128 md devices.

[Robert LeBlanc]

Apparently, the mdadm source on git-kernel.org (commit 13db17bd)
already has the fixes to properly create the device nodes, but I still
have the unexpected failure opening /dev/md1048574.
----------------
Robert LeBlanc
PGP Fingerprint 79A2 9CA4 6CC4 45DD A904  C70E E654 3BB2 FA62 B9F1


On Fri, Aug 19, 2016 at 8:10 PM, Robert LeBlanc <robert@leblancnet.us> wrote:
> I'm stuck and need some help getting this across the finish line. This
> is in no way complete, but to help show what I'm working on.
>
> When we added more than 128 md devices, we started getting failures.
> Looking through the code it seems that the minor dev number was being
> stored in an int and causing overflow and wrecking havoc on everything.
> I finally got the mknod in mdadm to correctly make the dev node with
> minors up to 1048574 as expected in the mdadm code. However, I can
> only create md devices up to 511. Trying to create an md higher than
> that has an error where the device can't be read/opened strace reports:
> open("/dev/.tmp.md.15341:9:1048574", O_RDWR|O_EXCL|O_DIRECT) = -1 ENXIO
> (No such device or address)
> while Python reports:
> IOError: [Errno 6] No such device or address: '/dev/.tmp.md.3279:9:512'
>
> A corresponding node is not created in /sys/block/md* for mds over 511.
>
> I believe that there may be a bug in the kernel code that is now being
> hit. After looking through the kernel code, I can't seem to find where
> this might be. Please help me by either pointing me to the source
> location that this might be a problem or fixing it based on these
> patches I've worked on so far. I'm using 4.7.0 currently.
>
> I'm using this for testing:
> ./mdadm --create /dev/md1048574 --assume-clean --verbose --level=1 \
> --raid-devices=2 /dev/loop0 missing
>
> Yes, we have a real need for more than 128 and 512 md devices.
>
> Please include me in any replies as I'm not on the ML.
>
> Thank you.
>
> Robert LeBlanc (1):
>   Some fixes to allow for more than 128 md devices.
>
>  Manage.c |  5 +++--
>  lib.c    |  2 +-
>  mdadm.h  |  6 +++---
>  util.c   | 25 +++++++++++++------------
>  4 files changed, 20 insertions(+), 18 deletions(-)
>
> --
> 2.8.1
>

Re: [RFC] Some fixes to allow for more than 128 md devices.

[Robert LeBlanc]

I found an email thread [0] talking about the new way to do this. I
did find a buffer overrun and will submit a patch for it.

Robert LeBlanc

[0] http://www.spinics.net/lists/raid/msg52300.html
----------------
Robert LeBlanc
PGP Fingerprint 79A2 9CA4 6CC4 45DD A904  C70E E654 3BB2 FA62 B9F1


On Mon, Aug 22, 2016 at 10:03 AM, Robert LeBlanc <robert@leblancnet.us> wrote:
> Apparently, the mdadm source on git-kernel.org (commit 13db17bd)
> already has the fixes to properly create the device nodes, but I still
> have the unexpected failure opening /dev/md1048574.
> ----------------
> Robert LeBlanc
> PGP Fingerprint 79A2 9CA4 6CC4 45DD A904  C70E E654 3BB2 FA62 B9F1
>
>
> On Fri, Aug 19, 2016 at 8:10 PM, Robert LeBlanc <robert@leblancnet.us> wrote:
>> I'm stuck and need some help getting this across the finish line. This
>> is in no way complete, but to help show what I'm working on.
>>
>> When we added more than 128 md devices, we started getting failures.
>> Looking through the code it seems that the minor dev number was being
>> stored in an int and causing overflow and wrecking havoc on everything.
>> I finally got the mknod in mdadm to correctly make the dev node with
>> minors up to 1048574 as expected in the mdadm code. However, I can
>> only create md devices up to 511. Trying to create an md higher than
>> that has an error where the device can't be read/opened strace reports:
>> open("/dev/.tmp.md.15341:9:1048574", O_RDWR|O_EXCL|O_DIRECT) = -1 ENXIO
>> (No such device or address)
>> while Python reports:
>> IOError: [Errno 6] No such device or address: '/dev/.tmp.md.3279:9:512'
>>
>> A corresponding node is not created in /sys/block/md* for mds over 511.
>>
>> I believe that there may be a bug in the kernel code that is now being
>> hit. After looking through the kernel code, I can't seem to find where
>> this might be. Please help me by either pointing me to the source
>> location that this might be a problem or fixing it based on these
>> patches I've worked on so far. I'm using 4.7.0 currently.
>>
>> I'm using this for testing:
>> ./mdadm --create /dev/md1048574 --assume-clean --verbose --level=1 \
>> --raid-devices=2 /dev/loop0 missing
>>
>> Yes, we have a real need for more than 128 and 512 md devices.
>>
>> Please include me in any replies as I'm not on the ML.
>>
>> Thank you.
>>
>> Robert LeBlanc (1):
>>   Some fixes to allow for more than 128 md devices.
>>
>>  Manage.c |  5 +++--
>>  lib.c    |  2 +-
>>  mdadm.h  |  6 +++---
>>  util.c   | 25 +++++++++++++------------
>>  4 files changed, 20 insertions(+), 18 deletions(-)
>>
>> --
>> 2.8.1
>>