md regression caused by commit 9e59d609763f70a992a8f3808dabcce60f14eb5c

md regression caused by commit 9e59d609763f70a992a8f3808dabcce60f14eb5c

[Mikulas Patocka]

[-- Attachment #1: Type: text/plain, Size: 3689 bytes --]

Hi

I report that the commit 9e59d609763f70a992a8f3808dabcce60f14eb5c causes 
problem with this mdadm script:

modprobe brd rd_size=1048576
mdadm --create /dev/md/mdmirror --name mdmirror --uuid aaaaaaaa:bbbbbbbb:cccccccc:00000001 /dev/ram0 /dev/ram1 -v -f --level=1 --raid-devices=2
mdadm -v --stop /dev/md/mdmirror
mdadm --assemble /dev/md/mdmirror --name mdmirror -v

Prior to this commit, the last command successfully assembles the array. 
After this commit, it reports an error "mdadm: Unable to initialize 
sysfs".

See https://bugzilla.redhat.com/show_bug.cgi?id=2385871

This is the strace of the failed mdadm --assemble command:

mknodat(AT_FDCWD, "/dev/.tmp.md.2512:9:127", S_IFBLK|0600, makedev(0x9, 0x7f)) = 0
openat(AT_FDCWD, "/dev/.tmp.md.2512:9:127", O_RDWR|O_EXCL|O_DIRECT) = 4
unlink("/dev/.tmp.md.2512:9:127")       = 0
fstat(4, {st_mode=S_IFBLK|0600, st_rdev=makedev(0x9, 0x7f), ...}) = 0
readlink("/sys/dev/block/9:127", "../../devices/virtual/block/md12"..., 199) = 33
openat(AT_FDCWD, "/proc/mdstat", O_RDONLY) = 5
fcntl(5, F_SETFD, FD_CLOEXEC)           = 0
fstat(5, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
read(5, "Personalities : [raid1] \nunused "..., 1024) = 48
read(5, "", 1024)                       = 0
close(5)                                = 0
ioctl(4, STOP_ARRAY, 0)                 = 0
openat(AT_FDCWD, "/dev/ram1", O_RDWR|O_EXCL|O_DIRECT) = 5
ioctl(5, BLKSSZGET, [512])              = 0
fstat(5, {st_mode=S_IFBLK|0660, st_rdev=makedev(0x1, 0x1), ...}) = 0
ioctl(5, BLKGETSIZE64, [1073741824])    = 0
lseek(5, 4096, SEEK_SET)                = 4096
read(5, "\374N+\251\1\0\0\0\0\0\0\0\0\0\0\0\252\252\252\252\273\273\273\273\314\314\314\314\0\0\0\1"..., 4096) = 4096
lseek(5, 0, SEEK_CUR)                   = 8192
fstat(5, {st_mode=S_IFBLK|0660, st_rdev=makedev(0x1, 0x1), ...}) = 0
close(5)                                = 0
write(2, "mdadm: /dev/ram1 is identified a"..., 72) = 72
openat(AT_FDCWD, "/dev/ram0", O_RDWR|O_EXCL|O_DIRECT) = 5
ioctl(5, BLKSSZGET, [512])              = 0
fstat(5, {st_mode=S_IFBLK|0660, st_rdev=makedev(0x1, 0), ...}) = 0
ioctl(5, BLKGETSIZE64, [1073741824])    = 0
lseek(5, 4096, SEEK_SET)                = 4096
read(5, "\374N+\251\1\0\0\0\0\0\0\0\0\0\0\0\252\252\252\252\273\273\273\273\314\314\314\314\0\0\0\1"..., 4096) = 4096
lseek(5, 0, SEEK_CUR)                   = 8192
fstat(5, {st_mode=S_IFBLK|0660, st_rdev=makedev(0x1, 0), ...}) = 0
close(5)                                = 0
write(2, "mdadm: /dev/ram0 is identified a"..., 72) = 72
openat(AT_FDCWD, "/dev/ram0", O_RDONLY|O_EXCL|O_DIRECT) = 5
ioctl(5, BLKSSZGET, [512])              = 0
fstat(5, {st_mode=S_IFBLK|0660, st_rdev=makedev(0x1, 0), ...}) = 0
ioctl(5, BLKGETSIZE64, [1073741824])    = 0
lseek(5, 4096, SEEK_SET)                = 4096
read(5, "\374N+\251\1\0\0\0\0\0\0\0\0\0\0\0\252\252\252\252\273\273\273\273\314\314\314\314\0\0\0\1"..., 4096) = 4096
lseek(5, 0, SEEK_CUR)                   = 8192
close(5)                                = 0
fstat(4, {st_mode=S_IFBLK|0600, st_rdev=makedev(0x9, 0x7f), ...}) = 0
readlink("/sys/dev/block/9:127", 0x7ffcd3ed18b0, 199) = -1 ENOENT (Adresář nebo soubor neexistuje) !!! FAILURE !!!
newfstatat(AT_FDCWD, "/sys/block/md127/md", 0x7ffcd3ed1a30, 0) = -1 ENOENT (Adresář nebo soubor neexistuje)
write(2, "mdadm: Unable to initialize sysf"..., 34) = 34
unlink("/run/mdadm/map.lock")           = 0
close(3)                                = 0
close(4)                                = 0
exit_group(1)                           = ?

See the line that is marked "!!! FAILURE !!!". Prior to the commit 
9e59d609763f70a992a8f3808dabcce60f14eb5c, mdadm is able to read the 
/sys/dev/block/9:127 symlink.

Mikulas

Re: md regression caused by commit 9e59d609763f70a992a8f3808dabcce60f14eb5c

[Xiao Ni]

Hi all

It needs to use the latest upstream mdadm
https://github.com/md-raid-utilities/mdadm/ which has fixed this
problem. And for fedora, it hasn't updated to the latest upstream. So
it has this problem. I'll update fedora mdadm to latest upstream.

Best Regards
Xiao

On Wed, Aug 6, 2025 at 11:28 PM Mikulas Patocka <mpatocka@redhat.com> wrote:
>
> Hi
>
> I report that the commit 9e59d609763f70a992a8f3808dabcce60f14eb5c causes
> problem with this mdadm script:
>
> modprobe brd rd_size=1048576
> mdadm --create /dev/md/mdmirror --name mdmirror --uuid aaaaaaaa:bbbbbbbb:cccccccc:00000001 /dev/ram0 /dev/ram1 -v -f --level=1 --raid-devices=2
> mdadm -v --stop /dev/md/mdmirror
> mdadm --assemble /dev/md/mdmirror --name mdmirror -v
>
> Prior to this commit, the last command successfully assembles the array.
> After this commit, it reports an error "mdadm: Unable to initialize
> sysfs".
>
> See https://bugzilla.redhat.com/show_bug.cgi?id=2385871
>
> This is the strace of the failed mdadm --assemble command:
>
> mknodat(AT_FDCWD, "/dev/.tmp.md.2512:9:127", S_IFBLK|0600, makedev(0x9, 0x7f)) = 0
> openat(AT_FDCWD, "/dev/.tmp.md.2512:9:127", O_RDWR|O_EXCL|O_DIRECT) = 4
> unlink("/dev/.tmp.md.2512:9:127")       = 0
> fstat(4, {st_mode=S_IFBLK|0600, st_rdev=makedev(0x9, 0x7f), ...}) = 0
> readlink("/sys/dev/block/9:127", "../../devices/virtual/block/md12"..., 199) = 33
> openat(AT_FDCWD, "/proc/mdstat", O_RDONLY) = 5
> fcntl(5, F_SETFD, FD_CLOEXEC)           = 0
> fstat(5, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
> read(5, "Personalities : [raid1] \nunused "..., 1024) = 48
> read(5, "", 1024)                       = 0
> close(5)                                = 0
> ioctl(4, STOP_ARRAY, 0)                 = 0
> openat(AT_FDCWD, "/dev/ram1", O_RDWR|O_EXCL|O_DIRECT) = 5
> ioctl(5, BLKSSZGET, [512])              = 0
> fstat(5, {st_mode=S_IFBLK|0660, st_rdev=makedev(0x1, 0x1), ...}) = 0
> ioctl(5, BLKGETSIZE64, [1073741824])    = 0
> lseek(5, 4096, SEEK_SET)                = 4096
> read(5, "\374N+\251\1\0\0\0\0\0\0\0\0\0\0\0\252\252\252\252\273\273\273\273\314\314\314\314\0\0\0\1"..., 4096) = 4096
> lseek(5, 0, SEEK_CUR)                   = 8192
> fstat(5, {st_mode=S_IFBLK|0660, st_rdev=makedev(0x1, 0x1), ...}) = 0
> close(5)                                = 0
> write(2, "mdadm: /dev/ram1 is identified a"..., 72) = 72
> openat(AT_FDCWD, "/dev/ram0", O_RDWR|O_EXCL|O_DIRECT) = 5
> ioctl(5, BLKSSZGET, [512])              = 0
> fstat(5, {st_mode=S_IFBLK|0660, st_rdev=makedev(0x1, 0), ...}) = 0
> ioctl(5, BLKGETSIZE64, [1073741824])    = 0
> lseek(5, 4096, SEEK_SET)                = 4096
> read(5, "\374N+\251\1\0\0\0\0\0\0\0\0\0\0\0\252\252\252\252\273\273\273\273\314\314\314\314\0\0\0\1"..., 4096) = 4096
> lseek(5, 0, SEEK_CUR)                   = 8192
> fstat(5, {st_mode=S_IFBLK|0660, st_rdev=makedev(0x1, 0), ...}) = 0
> close(5)                                = 0
> write(2, "mdadm: /dev/ram0 is identified a"..., 72) = 72
> openat(AT_FDCWD, "/dev/ram0", O_RDONLY|O_EXCL|O_DIRECT) = 5
> ioctl(5, BLKSSZGET, [512])              = 0
> fstat(5, {st_mode=S_IFBLK|0660, st_rdev=makedev(0x1, 0), ...}) = 0
> ioctl(5, BLKGETSIZE64, [1073741824])    = 0
> lseek(5, 4096, SEEK_SET)                = 4096
> read(5, "\374N+\251\1\0\0\0\0\0\0\0\0\0\0\0\252\252\252\252\273\273\273\273\314\314\314\314\0\0\0\1"..., 4096) = 4096
> lseek(5, 0, SEEK_CUR)                   = 8192
> close(5)                                = 0
> fstat(4, {st_mode=S_IFBLK|0600, st_rdev=makedev(0x9, 0x7f), ...}) = 0
> readlink("/sys/dev/block/9:127", 0x7ffcd3ed18b0, 199) = -1 ENOENT (Adresář nebo soubor neexistuje) !!! FAILURE !!!
> newfstatat(AT_FDCWD, "/sys/block/md127/md", 0x7ffcd3ed1a30, 0) = -1 ENOENT (Adresář nebo soubor neexistuje)
> write(2, "mdadm: Unable to initialize sysf"..., 34) = 34
> unlink("/run/mdadm/map.lock")           = 0
> close(3)                                = 0
> close(4)                                = 0
> exit_group(1)                           = ?
>
> See the line that is marked "!!! FAILURE !!!". Prior to the commit
> 9e59d609763f70a992a8f3808dabcce60f14eb5c, mdadm is able to read the
> /sys/dev/block/9:127 symlink.
>
> Mikulas

Re: md regression caused by commit 9e59d609763f70a992a8f3808dabcce60f14eb5c

[Luca Boccassi]

On Thu, 7 Aug 2025 at 01:04, Xiao Ni <xni@redhat.com> wrote:
>
> Hi all
>
> It needs to use the latest upstream mdadm
> https://github.com/md-raid-utilities/mdadm/ which has fixed this
> problem. And for fedora, it hasn't updated to the latest upstream. So
> it has this problem. I'll update fedora mdadm to latest upstream.
>
> Best Regards
> Xiao

Thank you for looking into it and providing a solution - however,
isn't it against the rules to break existing released userspace
components and requiring new versions to be released in order to use a
new kernel version? Is there any way this kernel patch could be
amended to avoid breaking the existing userspace as it is?

Thanks

> On Wed, Aug 6, 2025 at 11:28 PM Mikulas Patocka <mpatocka@redhat.com> wrote:
> >
> > Hi
> >
> > I report that the commit 9e59d609763f70a992a8f3808dabcce60f14eb5c causes
> > problem with this mdadm script:
> >
> > modprobe brd rd_size=1048576
> > mdadm --create /dev/md/mdmirror --name mdmirror --uuid aaaaaaaa:bbbbbbbb:cccccccc:00000001 /dev/ram0 /dev/ram1 -v -f --level=1 --raid-devices=2
> > mdadm -v --stop /dev/md/mdmirror
> > mdadm --assemble /dev/md/mdmirror --name mdmirror -v
> >
> > Prior to this commit, the last command successfully assembles the array.
> > After this commit, it reports an error "mdadm: Unable to initialize
> > sysfs".
> >
> > See https://bugzilla.redhat.com/show_bug.cgi?id=2385871
> >
> > This is the strace of the failed mdadm --assemble command:
> >
> > mknodat(AT_FDCWD, "/dev/.tmp.md.2512:9:127", S_IFBLK|0600, makedev(0x9, 0x7f)) = 0
> > openat(AT_FDCWD, "/dev/.tmp.md.2512:9:127", O_RDWR|O_EXCL|O_DIRECT) = 4
> > unlink("/dev/.tmp.md.2512:9:127")       = 0
> > fstat(4, {st_mode=S_IFBLK|0600, st_rdev=makedev(0x9, 0x7f), ...}) = 0
> > readlink("/sys/dev/block/9:127", "../../devices/virtual/block/md12"..., 199) = 33
> > openat(AT_FDCWD, "/proc/mdstat", O_RDONLY) = 5
> > fcntl(5, F_SETFD, FD_CLOEXEC)           = 0
> > fstat(5, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
> > read(5, "Personalities : [raid1] \nunused "..., 1024) = 48
> > read(5, "", 1024)                       = 0
> > close(5)                                = 0
> > ioctl(4, STOP_ARRAY, 0)                 = 0
> > openat(AT_FDCWD, "/dev/ram1", O_RDWR|O_EXCL|O_DIRECT) = 5
> > ioctl(5, BLKSSZGET, [512])              = 0
> > fstat(5, {st_mode=S_IFBLK|0660, st_rdev=makedev(0x1, 0x1), ...}) = 0
> > ioctl(5, BLKGETSIZE64, [1073741824])    = 0
> > lseek(5, 4096, SEEK_SET)                = 4096
> > read(5, "\374N+\251\1\0\0\0\0\0\0\0\0\0\0\0\252\252\252\252\273\273\273\273\314\314\314\314\0\0\0\1"..., 4096) = 4096
> > lseek(5, 0, SEEK_CUR)                   = 8192
> > fstat(5, {st_mode=S_IFBLK|0660, st_rdev=makedev(0x1, 0x1), ...}) = 0
> > close(5)                                = 0
> > write(2, "mdadm: /dev/ram1 is identified a"..., 72) = 72
> > openat(AT_FDCWD, "/dev/ram0", O_RDWR|O_EXCL|O_DIRECT) = 5
> > ioctl(5, BLKSSZGET, [512])              = 0
> > fstat(5, {st_mode=S_IFBLK|0660, st_rdev=makedev(0x1, 0), ...}) = 0
> > ioctl(5, BLKGETSIZE64, [1073741824])    = 0
> > lseek(5, 4096, SEEK_SET)                = 4096
> > read(5, "\374N+\251\1\0\0\0\0\0\0\0\0\0\0\0\252\252\252\252\273\273\273\273\314\314\314\314\0\0\0\1"..., 4096) = 4096
> > lseek(5, 0, SEEK_CUR)                   = 8192
> > fstat(5, {st_mode=S_IFBLK|0660, st_rdev=makedev(0x1, 0), ...}) = 0
> > close(5)                                = 0
> > write(2, "mdadm: /dev/ram0 is identified a"..., 72) = 72
> > openat(AT_FDCWD, "/dev/ram0", O_RDONLY|O_EXCL|O_DIRECT) = 5
> > ioctl(5, BLKSSZGET, [512])              = 0
> > fstat(5, {st_mode=S_IFBLK|0660, st_rdev=makedev(0x1, 0), ...}) = 0
> > ioctl(5, BLKGETSIZE64, [1073741824])    = 0
> > lseek(5, 4096, SEEK_SET)                = 4096
> > read(5, "\374N+\251\1\0\0\0\0\0\0\0\0\0\0\0\252\252\252\252\273\273\273\273\314\314\314\314\0\0\0\1"..., 4096) = 4096
> > lseek(5, 0, SEEK_CUR)                   = 8192
> > close(5)                                = 0
> > fstat(4, {st_mode=S_IFBLK|0600, st_rdev=makedev(0x9, 0x7f), ...}) = 0
> > readlink("/sys/dev/block/9:127", 0x7ffcd3ed18b0, 199) = -1 ENOENT (Adresář nebo soubor neexistuje) !!! FAILURE !!!
> > newfstatat(AT_FDCWD, "/sys/block/md127/md", 0x7ffcd3ed1a30, 0) = -1 ENOENT (Adresář nebo soubor neexistuje)
> > write(2, "mdadm: Unable to initialize sysf"..., 34) = 34
> > unlink("/run/mdadm/map.lock")           = 0
> > close(3)                                = 0
> > close(4)                                = 0
> > exit_group(1)                           = ?
> >
> > See the line that is marked "!!! FAILURE !!!". Prior to the commit
> > 9e59d609763f70a992a8f3808dabcce60f14eb5c, mdadm is able to read the
> > /sys/dev/block/9:127 symlink.
> >
> > Mikulas
>

Re: md regression caused by commit 9e59d609763f70a992a8f3808dabcce60f14eb5c

[Mikulas Patocka]

On Thu, 7 Aug 2025, Luca Boccassi wrote:

> On Thu, 7 Aug 2025 at 01:04, Xiao Ni <xni@redhat.com> wrote:
> >
> > Hi all
> >
> > It needs to use the latest upstream mdadm
> > https://github.com/md-raid-utilities/mdadm/ which has fixed this
> > problem. And for fedora, it hasn't updated to the latest upstream. So
> > it has this problem. I'll update fedora mdadm to latest upstream.
> >
> > Best Regards
> > Xiao
> 
> Thank you for looking into it and providing a solution - however,
> isn't it against the rules to break existing released userspace
> components and requiring new versions to be released in order to use a
> new kernel version? Is there any way this kernel patch could be
> amended to avoid breaking the existing userspace as it is?
> 
> Thanks

I also think that the misbehavior should be fixed in the kernel.

We shouldn't use arbitrary timeouts to clean up the sysfs entries, because 
it would introduce race conditions.

What about destroying the sysfs entries when the file descriptor is 
closed? (instead of on the STOP_ARRAY ioctl) That wouldn't interfere with 
other code trying to stop the array and it would make it work with the 
buggy mdadm that calls STOP_ARRAY and then tries to find the sysfs entries 
and then calls SET_ARRAY_INFO.

Mikulas

Re: md regression caused by commit 9e59d609763f70a992a8f3808dabcce60f14eb5c

[Xiao Ni]

On Thu, Aug 7, 2025 at 10:18 PM Mikulas Patocka <mpatocka@redhat.com> wrote:
>
>
>
> On Thu, 7 Aug 2025, Luca Boccassi wrote:
>
> > On Thu, 7 Aug 2025 at 01:04, Xiao Ni <xni@redhat.com> wrote:
> > >
> > > Hi all
> > >
> > > It needs to use the latest upstream mdadm
> > > https://github.com/md-raid-utilities/mdadm/ which has fixed this
> > > problem. And for fedora, it hasn't updated to the latest upstream. So
> > > it has this problem. I'll update fedora mdadm to latest upstream.
> > >
> > > Best Regards
> > > Xiao
> >
> > Thank you for looking into it and providing a solution - however,
> > isn't it against the rules to break existing released userspace
> > components and requiring new versions to be released in order to use a
> > new kernel version? Is there any way this kernel patch could be
> > amended to avoid breaking the existing userspace as it is?
> >
> > Thanks
>
> I also think that the misbehavior should be fixed in the kernel.
>
> We shouldn't use arbitrary timeouts to clean up the sysfs entries, because
> it would introduce race conditions.
>
> What about destroying the sysfs entries when the file descriptor is
> closed? (instead of on the STOP_ARRAY ioctl) That wouldn't interfere with
> other code trying to stop the array and it would make it work with the
> buggy mdadm that calls STOP_ARRAY and then tries to find the sysfs entries
> and then calls SET_ARRAY_INFO.
>
> Mikulas
>

Hi all

The assemble process is:
1. create array
2. stop it (STOP_ARRAY). Before the kernel change, del_gendisk is
called at the last release of mddev rather than in STOP_ARRAY ioctl
3. access /sys/block/md0/md

The kernel change tries to call del_gendisk in STOP_ARRAY. So /dev/md0
can be removed and no one can access it. If not, the array can be
created again because md supports create on open.

After the kernel change, the assemble process is:
1. create array
2. stop it (del_gendisk runs and /sys/block/md0 is removed)
3. acces /sys/block/md0/xx (it fails)

So del_gendisk destroys sysfs entries. If we destroy sysfs entries at
the last release of mddev, it will return to the old state that
/dev/md0 can be opened after stop. I don't want to return back.
Because some customers encounter bugs that shutdown is stuck because
/dev/md0 can't be stopped and the regression test usually fails
because of this too.

I know it's not good to break mdadm by a kernel change. But sometimes
it needs userspace tool and kernel work together to fix a problem,
right?
Sorry for bringing the problem, and thanks for the suggestions. Any
more good suggestions?

Best Regards
Xiao

Re: md regression caused by commit 9e59d609763f70a992a8f3808dabcce60f14eb5c

[Yu Kuai]

在 2025/08/08 13:28, Xiao Ni 写道:
> On Thu, Aug 7, 2025 at 10:18 PM Mikulas Patocka <mpatocka@redhat.com> wrote:
>>
>>
>>
>> On Thu, 7 Aug 2025, Luca Boccassi wrote:
>>
>>> On Thu, 7 Aug 2025 at 01:04, Xiao Ni <xni@redhat.com> wrote:
>>>>
>>>> Hi all
>>>>
>>>> It needs to use the latest upstream mdadm
>>>> https://github.com/md-raid-utilities/mdadm/ which has fixed this
>>>> problem. And for fedora, it hasn't updated to the latest upstream. So
>>>> it has this problem. I'll update fedora mdadm to latest upstream.
>>>>
>>>> Best Regards
>>>> Xiao
>>>
>>> Thank you for looking into it and providing a solution - however,
>>> isn't it against the rules to break existing released userspace
>>> components and requiring new versions to be released in order to use a
>>> new kernel version? Is there any way this kernel patch could be
>>> amended to avoid breaking the existing userspace as it is?
>>>
>>> Thanks
>>
>> I also think that the misbehavior should be fixed in the kernel.
>>
>> We shouldn't use arbitrary timeouts to clean up the sysfs entries, because
>> it would introduce race conditions.
>>
>> What about destroying the sysfs entries when the file descriptor is
>> closed? (instead of on the STOP_ARRAY ioctl) That wouldn't interfere with
>> other code trying to stop the array and it would make it work with the
>> buggy mdadm that calls STOP_ARRAY and then tries to find the sysfs entries
>> and then calls SET_ARRAY_INFO.
>>
>> Mikulas
>>
> 
> Hi all
> 
> The assemble process is:
> 1. create array
> 2. stop it (STOP_ARRAY). Before the kernel change, del_gendisk is
> called at the last release of mddev rather than in STOP_ARRAY ioctl
> 3. access /sys/block/md0/md
> 
> The kernel change tries to call del_gendisk in STOP_ARRAY. So /dev/md0
> can be removed and no one can access it. If not, the array can be
> created again because md supports create on open.
> 
> After the kernel change, the assemble process is:
> 1. create array
> 2. stop it (del_gendisk runs and /sys/block/md0 is removed)
> 3. acces /sys/block/md0/xx (it fails)
> 
> So del_gendisk destroys sysfs entries. If we destroy sysfs entries at
> the last release of mddev, it will return to the old state that
> /dev/md0 can be opened after stop. I don't want to return back.
> Because some customers encounter bugs that shutdown is stuck because
> /dev/md0 can't be stopped and the regression test usually fails
> because of this too.

Yes, from kernel side, we think after succeed stop_array ioct, the
kernel disk should be removed in the end. We used to call del_gendisk
asynchronously, leaves a race window that sysfs entries still visible
to user.

We decide to fix this in the last merge window, however, it's true mdadm
has to be fixed together.

> 
> I know it's not good to break mdadm by a kernel change. But sometimes
> it needs userspace tool and kernel work together to fix a problem,
> right?
> Sorry for bringing the problem, and thanks for the suggestions. Any
> more good suggestions?
> 

Idealy, we should fix mdadm first, then after a release, fix kernel.
Sadly the transition stage is missing now. :(

If we want to just avoid this problem in kernel, what I can think of is
adding a switch and mark it deprecated for now. And in new mdadm
releases enable that switch, and after sometime, remove mdadm legacy
code to stop array, and finally remove the deprecated switch in kernel
then everyone will be happy :)

Thanks,
Kuai

> Best Regards
> Xiao
> 
> 
> .
> 

Re: md regression caused by commit 9e59d609763f70a992a8f3808dabcce60f14eb5c

[Xiao Ni]

On Fri, Aug 8, 2025 at 2:41 PM Yu Kuai <yukuai1@huaweicloud.com> wrote:
>
>
>
> 在 2025/08/08 13:28, Xiao Ni 写道:
> > On Thu, Aug 7, 2025 at 10:18 PM Mikulas Patocka <mpatocka@redhat.com> wrote:
> >>
> >>
> >>
> >> On Thu, 7 Aug 2025, Luca Boccassi wrote:
> >>
> >>> On Thu, 7 Aug 2025 at 01:04, Xiao Ni <xni@redhat.com> wrote:
> >>>>
> >>>> Hi all
> >>>>
> >>>> It needs to use the latest upstream mdadm
> >>>> https://github.com/md-raid-utilities/mdadm/ which has fixed this
> >>>> problem. And for fedora, it hasn't updated to the latest upstream. So
> >>>> it has this problem. I'll update fedora mdadm to latest upstream.
> >>>>
> >>>> Best Regards
> >>>> Xiao
> >>>
> >>> Thank you for looking into it and providing a solution - however,
> >>> isn't it against the rules to break existing released userspace
> >>> components and requiring new versions to be released in order to use a
> >>> new kernel version? Is there any way this kernel patch could be
> >>> amended to avoid breaking the existing userspace as it is?
> >>>
> >>> Thanks
> >>
> >> I also think that the misbehavior should be fixed in the kernel.
> >>
> >> We shouldn't use arbitrary timeouts to clean up the sysfs entries, because
> >> it would introduce race conditions.
> >>
> >> What about destroying the sysfs entries when the file descriptor is
> >> closed? (instead of on the STOP_ARRAY ioctl) That wouldn't interfere with
> >> other code trying to stop the array and it would make it work with the
> >> buggy mdadm that calls STOP_ARRAY and then tries to find the sysfs entries
> >> and then calls SET_ARRAY_INFO.
> >>
> >> Mikulas
> >>
> >
> > Hi all
> >
> > The assemble process is:
> > 1. create array
> > 2. stop it (STOP_ARRAY). Before the kernel change, del_gendisk is
> > called at the last release of mddev rather than in STOP_ARRAY ioctl
> > 3. access /sys/block/md0/md
> >
> > The kernel change tries to call del_gendisk in STOP_ARRAY. So /dev/md0
> > can be removed and no one can access it. If not, the array can be
> > created again because md supports create on open.
> >
> > After the kernel change, the assemble process is:
> > 1. create array
> > 2. stop it (del_gendisk runs and /sys/block/md0 is removed)
> > 3. acces /sys/block/md0/xx (it fails)
> >
> > So del_gendisk destroys sysfs entries. If we destroy sysfs entries at
> > the last release of mddev, it will return to the old state that
> > /dev/md0 can be opened after stop. I don't want to return back.
> > Because some customers encounter bugs that shutdown is stuck because
> > /dev/md0 can't be stopped and the regression test usually fails
> > because of this too.
>
> Yes, from kernel side, we think after succeed stop_array ioct, the
> kernel disk should be removed in the end. We used to call del_gendisk
> asynchronously, leaves a race window that sysfs entries still visible
> to user.
>
> We decide to fix this in the last merge window, however, it's true mdadm
> has to be fixed together.
>
> >
> > I know it's not good to break mdadm by a kernel change. But sometimes
> > it needs userspace tool and kernel work together to fix a problem,
> > right?
> > Sorry for bringing the problem, and thanks for the suggestions. Any
> > more good suggestions?
> >
>
> Idealy, we should fix mdadm first, then after a release, fix kernel.
> Sadly the transition stage is missing now. :(
>
> If we want to just avoid this problem in kernel, what I can think of is
> adding a switch and mark it deprecated for now. And in new mdadm
> releases enable that switch, and after sometime, remove mdadm legacy
> code to stop array, and finally remove the deprecated switch in kernel
> then everyone will be happy :)
>
> Thanks,
> Kuai

Hi Kuai

Thanks for the suggestion. I'll use this way.

Regards
Xiao
>
> > Best Regards
> > Xiao
> >
> >
> > .
> >
>

Re: md regression caused by commit 9e59d609763f70a992a8f3808dabcce60f14eb5c

[Luca Boccassi]

On Fri, 8 Aug 2025 at 07:40, Yu Kuai <yukuai1@huaweicloud.com> wrote:
> 在 2025/08/08 13:28, Xiao Ni 写道:
> > I know it's not good to break mdadm by a kernel change. But sometimes
> > it needs userspace tool and kernel work together to fix a problem,
> > right?
> > Sorry for bringing the problem, and thanks for the suggestions. Any
> > more good suggestions?
> >
>
> Idealy, we should fix mdadm first, then after a release, fix kernel.
> Sadly the transition stage is missing now. :(
>
> If we want to just avoid this problem in kernel, what I can think of is
> adding a switch and mark it deprecated for now. And in new mdadm
> releases enable that switch, and after sometime, remove mdadm legacy
> code to stop array, and finally remove the deprecated switch in kernel
> then everyone will be happy :)

Hi,

As long as the change makes the current default behaviour backward
compatible, and the switch is used by mdadm to opt-in the new,
incompatible behaviour, then yes that sounds like a good solution,
thank you.

Re: md regression caused by commit 9e59d609763f70a992a8f3808dabcce60f14eb5c

[Luca Boccassi]

On Fri, 8 Aug 2025 at 09:07, Luca Boccassi <luca.boccassi@gmail.com> wrote:
>
> On Fri, 8 Aug 2025 at 07:40, Yu Kuai <yukuai1@huaweicloud.com> wrote:
> > 在 2025/08/08 13:28, Xiao Ni 写道:
> > > I know it's not good to break mdadm by a kernel change. But sometimes
> > > it needs userspace tool and kernel work together to fix a problem,
> > > right?
> > > Sorry for bringing the problem, and thanks for the suggestions. Any
> > > more good suggestions?
> > >
> >
> > Idealy, we should fix mdadm first, then after a release, fix kernel.
> > Sadly the transition stage is missing now. :(
> >
> > If we want to just avoid this problem in kernel, what I can think of is
> > adding a switch and mark it deprecated for now. And in new mdadm
> > releases enable that switch, and after sometime, remove mdadm legacy
> > code to stop array, and finally remove the deprecated switch in kernel
> > then everyone will be happy :)
>
> Hi,
>
> As long as the change makes the current default behaviour backward
> compatible, and the switch is used by mdadm to opt-in the new,
> incompatible behaviour, then yes that sounds like a good solution,
> thank you.

Hi,

Any update? RC1 was released with this regression. Any ETA on the fix?
If it won't be ready soon, would it be possible to revert the change
for now, until the fix is ready? Thanks!

Re: md regression caused by commit 9e59d609763f70a992a8f3808dabcce60f14eb5c

[Xiao Ni]

On Tue, Aug 12, 2025 at 2:32 AM Luca Boccassi <luca.boccassi@gmail.com> wrote:
>
> On Fri, 8 Aug 2025 at 09:07, Luca Boccassi <luca.boccassi@gmail.com> wrote:
> >
> > On Fri, 8 Aug 2025 at 07:40, Yu Kuai <yukuai1@huaweicloud.com> wrote:
> > > 在 2025/08/08 13:28, Xiao Ni 写道:
> > > > I know it's not good to break mdadm by a kernel change. But sometimes
> > > > it needs userspace tool and kernel work together to fix a problem,
> > > > right?
> > > > Sorry for bringing the problem, and thanks for the suggestions. Any
> > > > more good suggestions?
> > > >
> > >
> > > Idealy, we should fix mdadm first, then after a release, fix kernel.
> > > Sadly the transition stage is missing now. :(
> > >
> > > If we want to just avoid this problem in kernel, what I can think of is
> > > adding a switch and mark it deprecated for now. And in new mdadm
> > > releases enable that switch, and after sometime, remove mdadm legacy
> > > code to stop array, and finally remove the deprecated switch in kernel
> > > then everyone will be happy :)
> >
> > Hi,
> >
> > As long as the change makes the current default behaviour backward
> > compatible, and the switch is used by mdadm to opt-in the new,
> > incompatible behaviour, then yes that sounds like a good solution,
> > thank you.
>
> Hi,
>
> Any update? RC1 was released with this regression. Any ETA on the fix?
> If it won't be ready soon, would it be possible to revert the change
> for now, until the fix is ready? Thanks!
>

Hi

I took two days for a regression test and sent the patch to the mail
list just now.

Regards
Xiao

Re: md regression caused by commit 9e59d609763f70a992a8f3808dabcce60f14eb5c

[Luca Boccassi]

On Tue, 12 Aug 2025 at 08:40, Xiao Ni <xni@redhat.com> wrote:
>
> On Tue, Aug 12, 2025 at 2:32 AM Luca Boccassi <luca.boccassi@gmail.com> wrote:
> >
> > On Fri, 8 Aug 2025 at 09:07, Luca Boccassi <luca.boccassi@gmail.com> wrote:
> > >
> > > On Fri, 8 Aug 2025 at 07:40, Yu Kuai <yukuai1@huaweicloud.com> wrote:
> > > > 在 2025/08/08 13:28, Xiao Ni 写道:
> > > > > I know it's not good to break mdadm by a kernel change. But sometimes
> > > > > it needs userspace tool and kernel work together to fix a problem,
> > > > > right?
> > > > > Sorry for bringing the problem, and thanks for the suggestions. Any
> > > > > more good suggestions?
> > > > >
> > > >
> > > > Idealy, we should fix mdadm first, then after a release, fix kernel.
> > > > Sadly the transition stage is missing now. :(
> > > >
> > > > If we want to just avoid this problem in kernel, what I can think of is
> > > > adding a switch and mark it deprecated for now. And in new mdadm
> > > > releases enable that switch, and after sometime, remove mdadm legacy
> > > > code to stop array, and finally remove the deprecated switch in kernel
> > > > then everyone will be happy :)
> > >
> > > Hi,
> > >
> > > As long as the change makes the current default behaviour backward
> > > compatible, and the switch is used by mdadm to opt-in the new,
> > > incompatible behaviour, then yes that sounds like a good solution,
> > > thank you.
> >
> > Hi,
> >
> > Any update? RC1 was released with this regression. Any ETA on the fix?
> > If it won't be ready soon, would it be possible to revert the change
> > for now, until the fix is ready? Thanks!
> >
>
> Hi
>
> I took two days for a regression test and sent the patch to the mail
> list just now.

That's great news, thank you!