mdadm-2.3.1 fails to hotadd device

mdadm-2.3.1 fails to hotadd device

[Andre Noll]

# uname -r; mdadm -V; cat /proc/mdstat 
2.6.15.4-gbbde1285
mdadm - v2.3.1 - 6 February 2006
Personalities : [raid1] [raid5] [raid6] 
md1 : active raid1 hdd1[12] sdi1[8] sda1[0] hda1[13] sdl1[11] sdk1[10]
sdj1[9] sdh1[7] sdg1[6] sdf1[5] sde1[4] sdd1[3] sdc1[2] sdb1[1]
251776 blocks [14/14] [UUUUUUUUUUUUUU]
      
unused devices: <none>
# 
# mdadm /dev/md1 -f /dev/hdd1 
mdadm: set /dev/hdd1 faulty in /dev/md1
# 
# mdadm /dev/md1 -r /dev/hdd1
mdadm: hot removed /dev/hdd1
# 
# mdadm /dev/md1 -a /dev/hdd1
mdadm: cannot find valid superblock in this array - HELP
# 
# mdadm-1.12.0 /dev/md1 -a /dev/hdd1
mdadm: hot added /dev/hdd1

Any hints?
Andre
-- 
The only person who always got his work done by Friday was Robinson Crusoe

Re: mdadm-2.3.1 fails to hotadd device

[Neil Brown]

On Sunday February 26, maan@systemlinux.org wrote:
> # uname -r; mdadm -V; cat /proc/mdstat 
> 2.6.15.4-gbbde1285
> mdadm - v2.3.1 - 6 February 2006
> Personalities : [raid1] [raid5] [raid6] 
> md1 : active raid1 hdd1[12] sdi1[8] sda1[0] hda1[13] sdl1[11] sdk1[10]
> sdj1[9] sdh1[7] sdg1[6] sdf1[5] sde1[4] sdd1[3] sdc1[2] sdb1[1]
> 251776 blocks [14/14] [UUUUUUUUUUUUUU]

You have a 14 drive raid1... cool....

> # 
> # mdadm /dev/md1 -a /dev/hdd1
> mdadm: cannot find valid superblock in this array - HELP

Can you
  strace -o /tmp/trace mdadm /dev/md1 -a /dev/hdd1

and send me /tmp/trace?

Thanks.

NeilBrown

> # 
> # mdadm-1.12.0 /dev/md1 -a /dev/hdd1
> mdadm: hot added /dev/hdd1
> 
> Any hints?
> Andre
> -- 
> The only person who always got his work done by Friday was Robinson Crusoe

Re: mdadm-2.3.1 fails to hotadd device

[Andre Noll]

On 22:41, Neil Brown wrote:
> > # mdadm /dev/md1 -a /dev/hdd1
> > mdadm: cannot find valid superblock in this array - HELP
> 
> Can you
>   strace -o /tmp/trace mdadm /dev/md1 -a /dev/hdd1
> 
> and send me /tmp/trace?

Here it comes.
Andre
---
execve("/sbin/mdadm", ["mdadm", "/dev/md1", "-a", "/dev/hdd1"], [/* 99 vars */]) = 0
uname({sys="Linux", node="raspe", ...}) = 0
brk(0)                                  = 0x806a000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
open("/usr/lib/glibc-2.3.6/etc/ld.so.cache", O_RDONLY) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=44713, ...}) = 0
mmap2(NULL, 44713, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb7f4d000
close(3)                                = 0
open("/lib/libc.so.6", O_RDONLY)        = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\0O\1\000"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=1175704, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7f4c000
mmap2(NULL, 1162428, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7e30000
mmap2(0xb7f46000, 16384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x116) = 0xb7f46000
mmap2(0xb7f4a000, 7356, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb7f4a000
close(3)                                = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7e2f000
mprotect(0xb7f46000, 4096, PROT_READ)   = 0
mprotect(0xb7f6d000, 4096, PROT_READ)   = 0
set_thread_area({entry_number:-1 -> 6, base_addr:0xb7e2f6b0, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}) = 0
munmap(0xb7f4d000, 44713)               = 0
time(NULL)                              = 1140955660
getpid()                                = 1392
brk(0)                                  = 0x806a000
brk(0x808b000)                          = 0x808b000
open("/dev/md1", O_RDWR)                = 3
fstat64(3, {st_mode=S_IFBLK|0660, st_rdev=makedev(9, 1), ...}) = 0
ioctl(3, 0x800c0910, 0xbfd671d0)        = 0
ioctl(3, 0x80480911, 0xbfd67310)        = 0
stat64("/dev/hdd1", {st_mode=S_IFBLK|0660, st_rdev=makedev(22, 65), ...}) = 0
open("/dev/hdd1", O_RDONLY|O_EXCL)      = 4
ioctl(4, BLKGETSIZE64, 0xbfd670f0)      = 0
ioctl(4, BLKFLSBUF, 0)                  = 0
_llseek(4, 263061504, [263061504], SEEK_SET) = 0
read(4, "\374N+\251\0\0\0\0Z\0\0\0\0\0\0\0\0\0\0\0a7h\220hTiC\1"..., 4096) = 4096
close(4)                                = 0
fstat64(3, {st_mode=S_IFBLK|0660, st_rdev=makedev(9, 1), ...}) = 0
ioctl(3, 0x800c0910, 0xbfd67090)        = 0
ioctl(3, 0x80140912, 0xbfd672f0)        = 0
lstat64("/dev", {st_mode=S_IFLNK|0777, st_size=5, ...}) = 0
ioctl(3, 0x80140912, 0xbfd672f0)        = 0
ioctl(3, 0x80140912, 0xbfd672f0)        = 0
ioctl(3, 0x80140912, 0xbfd672f0)        = 0
ioctl(3, 0x80140912, 0xbfd672f0)        = 0
ioctl(3, 0x80140912, 0xbfd672f0)        = 0
ioctl(3, 0x80140912, 0xbfd672f0)        = 0
ioctl(3, 0x80140912, 0xbfd672f0)        = 0
ioctl(3, 0x80140912, 0xbfd672f0)        = 0
ioctl(3, 0x80140912, 0xbfd672f0)        = 0
ioctl(3, 0x80140912, 0xbfd672f0)        = 0
ioctl(3, 0x80140912, 0xbfd672f0)        = 0
ioctl(3, 0x80140912, 0xbfd672f0)        = 0
ioctl(3, 0x80140912, 0xbfd672f0)        = 0
ioctl(3, 0x80140912, 0xbfd672f0)        = 0
ioctl(3, 0x80140912, 0xbfd672f0)        = 0
ioctl(3, 0x80140912, 0xbfd672f0)        = 0
ioctl(3, 0x80140912, 0xbfd672f0)        = 0
ioctl(3, 0x80140912, 0xbfd672f0)        = 0
ioctl(3, 0x80140912, 0xbfd672f0)        = 0
ioctl(3, 0x80140912, 0xbfd672f0)        = 0
ioctl(3, 0x80140912, 0xbfd672f0)        = 0
ioctl(3, 0x80140912, 0xbfd672f0)        = 0
ioctl(3, 0x80140912, 0xbfd672f0)        = 0
ioctl(3, 0x80140912, 0xbfd672f0)        = 0
ioctl(3, 0x80140912, 0xbfd672f0)        = 0
ioctl(3, 0x80140912, 0xbfd672f0)        = 0
write(2, "mdadm: cannot find valid superbl"..., 57) = 57
exit_group(1)                           = ?
-- 
The only person who always got his work done by Friday was Robinson Crusoe

Re: mdadm-2.3.1 fails to hotadd device

[Andre Noll]

On 22:41, Neil Brown wrote:
> On Sunday February 26, maan@systemlinux.org wrote:
> > # uname -r; mdadm -V; cat /proc/mdstat 
> > 2.6.15.4-gbbde1285
> > mdadm - v2.3.1 - 6 February 2006
> > Personalities : [raid1] [raid5] [raid6] 
> > md1 : active raid1 hdd1[12] sdi1[8] sda1[0] hda1[13] sdl1[11] sdk1[10]
> > sdj1[9] sdh1[7] sdg1[6] sdf1[5] sde1[4] sdd1[3] sdc1[2] sdb1[1]
> > 251776 blocks [14/14] [UUUUUUUUUUUUUU]
> 
> You have a 14 drive raid1... cool....
> 
> > # 
> > # mdadm /dev/md1 -a /dev/hdd1
> > mdadm: cannot find valid superblock in this array - HELP

I took another look and added some fprintf's to Manage.c and util.c,
see the patch below. The problem appears to be map_dev() always
returning NULL because devlist is NULL.

With the patch applied, "mdadm /dev/md1 -a /dev/hdd1" gives me
the following output:

mdadm: st->maxdevs = 27
mdadm: major/minor/state: 8/1/6
devlist: (nil)
mdadm: major/minor/state: 8/17/6
devlist: (nil)
mdadm: major/minor/state: 8/33/6
devlist: (nil)
mdadm: major/minor/state: 8/49/6
devlist: (nil)
mdadm: major/minor/state: 8/65/6
devlist: (nil)
mdadm: major/minor/state: 8/81/6
devlist: (nil)
mdadm: major/minor/state: 8/97/6
devlist: (nil)
mdadm: major/minor/state: 8/113/6
devlist: (nil)
mdadm: major/minor/state: 8/129/6
devlist: (nil)
mdadm: major/minor/state: 8/145/6
devlist: (nil)
mdadm: major/minor/state: 8/161/6
devlist: (nil)
mdadm: major/minor/state: 8/177/6
devlist: (nil)
mdadm: major/minor/state: 0/0/8
mdadm: major/minor/state: 3/1/6
devlist: (nil)
mdadm: major/minor/state: 0/0/8
mdadm: major/minor/state: 0/0/8
mdadm: major/minor/state: 0/0/8
mdadm: major/minor/state: 0/0/8
mdadm: major/minor/state: 0/0/8
mdadm: major/minor/state: 0/0/8
mdadm: major/minor/state: 0/0/8
mdadm: major/minor/state: 0/0/8
mdadm: major/minor/state: 0/0/8
mdadm: major/minor/state: 0/0/8
mdadm: major/minor/state: 0/0/8
mdadm: major/minor/state: 0/0/8
mdadm: major/minor/state: 0/0/8
mdadm: cannot find valid superblock in this array - HELP

---

diff -urpN mdadm-2.3.1/Manage.c mdadm-2.3.1-hacked/Manage.c
--- mdadm-2.3.1/Manage.c	Mon Dec  5 05:52:22 2005
+++ mdadm-2.3.1-hacked/Manage.c	Sun Feb 26 14:13:07 2006
@@ -236,6 +236,7 @@ int Manage_subdevs(char *devname, int fd
 
 			if (array.not_persistent == 0) {
 
+				fprintf(stderr, Name ": st->maxdevs = %d\n", st->max_devs);
 				/* need to find a sample superblock to copy, and
 				 * a spare slot to use 
 				 */
@@ -245,6 +246,7 @@ int Manage_subdevs(char *devname, int fd
 					disc.number = j;
 					if (ioctl(fd, GET_DISK_INFO, &disc))
 						continue;
+					fprintf(stderr, Name ": major/minor/state: %d/%d/%d\n", disc.major, disc.minor, disc.state);
 					if (disc.major==0 && disc.minor==0)
 						continue;
 					if ((disc.state & 4)==0) continue; /* sync */
diff -urpN mdadm-2.3.1/util.c mdadm-2.3.1-hacked/util.c
--- mdadm-2.3.1/util.c	Tue Jan 31 00:43:03 2006
+++ mdadm-2.3.1-hacked/util.c	Sun Feb 26 14:06:51 2006
@@ -412,6 +412,7 @@ char *map_dev(int major, int minor)
 		devlist_ready=1;
 	}
 
+	fprintf(stderr, "devlist: %p\n", devlist);
 	for (p=devlist; p; p=p->next)
 		if (p->major == major &&
 		    p->minor == minor) {
-- 
The only person who always got his work done by Friday was Robinson Crusoe

Re: mdadm-2.3.1 fails to hotadd device

[Neil Brown]

On Sunday February 26, maan@systemlinux.org wrote:
> On 22:41, Neil Brown wrote:
> > > # mdadm /dev/md1 -a /dev/hdd1
> > > mdadm: cannot find valid superblock in this array - HELP
> > 
> > Can you
> >   strace -o /tmp/trace mdadm /dev/md1 -a /dev/hdd1
> > 
> > and send me /tmp/trace?
> 
> Here it comes.

Thanks.

> lstat64("/dev", {st_mode=S_IFLNK|0777, st_size=5, ...}) = 0

Your '/dev' is a symlink, and mdadm doesn't seem to like that.

Can you change the 'nftw' call in 'map_dev' in 'util.c' to

		nftw("/dev/.", add_dev, 10, FTW_PHYS);

and see if that helps?

It isn't really the right fix, but it is a quick-and-dirty that might
work.

NeilBrown

Re: mdadm-2.3.1 fails to hotadd device

[Andre Noll]

On 08:37, Neil Brown wrote:
> Your '/dev' is a symlink, and mdadm doesn't seem to like that.
> 
> Can you change the 'nftw' call in 'map_dev' in 'util.c' to
> 
> 		nftw("/dev/.", add_dev, 10, FTW_PHYS);
> 
> and see if that helps?

Jup, that does the trick:

	# ./mdadm /dev/md1 -a /dev/hdd1
	mdadm: re-added /dev/hdd1

> It isn't really the right fix, but it is a quick-and-dirty that might
> work.

Is it considered an invalid configuration to have /dev being a symlink
(to /udev as in my case)?

Thanks
Andre

-- 
The only person who always got his work done by Friday was Robinson Crusoe

Re: mdadm-2.3.1 fails to hotadd device

[Neil Brown]

On Sunday February 26, maan@systemlinux.org wrote:
> On 08:37, Neil Brown wrote:
> > Your '/dev' is a symlink, and mdadm doesn't seem to like that.
> > 
> > Can you change the 'nftw' call in 'map_dev' in 'util.c' to
> > 
> > 		nftw("/dev/.", add_dev, 10, FTW_PHYS);
> > 
> > and see if that helps?
> 
> Jup, that does the trick:
> 
> 	# ./mdadm /dev/md1 -a /dev/hdd1
> 	mdadm: re-added /dev/hdd1

Great.

> 
> > It isn't really the right fix, but it is a quick-and-dirty that might
> > work.
> 
> Is it considered an invalid configuration to have /dev being a symlink
> (to /udev as in my case)?

Not invalid.  Just untested.... until now - thanks.

I'll make sure 2.4 handles this a lot more gracefully.

NeilBrown