* Re: [PATCH 2/4] md-cluster: remove capabilities
From: Guoqing Jiang @ 2015-04-13 2:27 UTC (permalink / raw)
To: Goldwyn Rodrigues; +Cc: neilb, linux-raid
In-Reply-To: <20150408192247.GA9682@shrek.lan>
Goldwyn Rodrigues wrote:
> Signed-off-by: Goldwyn Rodrigues <rgoldwyn@suse.com>
> ---
> drivers/md/md-cluster.c | 47 +++++++++++++++++++++++++++++++++++++++++++++++
> drivers/md/md-cluster.h | 1 +
> drivers/md/md.c | 24 +++++++++++++++---------
> drivers/md/md.h | 1 +
> 4 files changed, 64 insertions(+), 9 deletions(-)
>
> diff --git a/drivers/md/md-cluster.c b/drivers/md/md-cluster.c
> index 96679b2..d036c83 100644
> --- a/drivers/md/md-cluster.c
> +++ b/drivers/md/md-cluster.c
> @@ -72,6 +72,7 @@ enum msg_type {
> METADATA_UPDATED = 0,
> RESYNCING,
> NEWDISK,
> + REMOVE,
> };
>
> struct cluster_msg {
> @@ -186,6 +187,20 @@ static char *pretty_uuid(char *dest, char *src)
> return dest;
> }
>
> +static struct md_rdev *find_rdev_uuid(struct mddev *mddev, char *uuid)
> +{
> + struct md_rdev *rdev;
> + struct mdp_superblock_1 *sb;
> +
> + rdev_for_each_rcu(rdev, mddev) {
> + sb = page_address(rdev->sb_page);
> + if (!strncmp(uuid, sb->device_uuid, 16)) {
> + return rdev;
> + }
> + }
> + return NULL;
> +}
> +
> static void add_resync_info(struct mddev *mddev, struct dlm_lock_resource *lockres,
> sector_t lo, sector_t hi)
> {
> @@ -401,6 +416,17 @@ static void process_metadata_update(struct mddev *mddev, struct cluster_msg *msg
> dlm_lock_sync(cinfo->no_new_dev_lockres, DLM_LOCK_CR);
> }
>
> +static void process_remove_disk(struct mddev *mddev, struct cluster_msg *msg)
> +{
> + struct md_rdev *rdev = find_rdev_uuid(mddev, msg->uuid);
> + char uuid[32];
> +
> + if (rdev)
> + md_kick_rdev_from_array(rdev);
> + else
> + pr_warn("%s: %d Could not find disk with uuid: %s", __func__, __LINE__, pretty_uuid(uuid, msg->uuid));
> +}
> +
> static void process_recvd_msg(struct mddev *mddev, struct cluster_msg *msg)
> {
> switch (msg->type) {
> @@ -419,6 +445,15 @@ static void process_recvd_msg(struct mddev *mddev, struct cluster_msg *msg)
> pr_info("%s: %d Received message: NEWDISK from %d\n",
> __func__, __LINE__, msg->slot);
> process_add_new_disk(mddev, msg);
> + break;
> + case REMOVE:
> + pr_info("%s: %d Received REMOVE from %d\n",
> + __func__, __LINE__, msg->slot);
> + process_remove_disk(mddev, msg);
> + break;
> + default:
> + pr_warn("%s:%d Received unknown message from %d\n",
> + __func__, __LINE__, msg->slot);
> };
> }
>
> @@ -854,6 +889,17 @@ static int new_disk_ack(struct mddev *mddev, bool ack)
> return 0;
> }
>
> +static int remove_disk(struct mddev *mddev, struct md_rdev *rdev)
> +{
> + struct cluster_msg cmsg;
> + struct md_cluster_info *cinfo = mddev->cluster_info;
> + struct mdp_superblock_1 *sb = page_address(rdev->sb_page);
> + char *uuid = sb->device_uuid;
> + cmsg.type = REMOVE;
> + memcpy(cmsg.uuid, uuid, 16);
> + return __sendmsg(cinfo, &cmsg);
> +}
> +
> static struct md_cluster_operations cluster_ops = {
> .join = join,
> .leave = leave,
> @@ -868,6 +914,7 @@ static struct md_cluster_operations cluster_ops = {
> .add_new_disk_start = add_new_disk_start,
> .add_new_disk_finish = add_new_disk_finish,
> .new_disk_ack = new_disk_ack,
> + .remove_disk = remove_disk,
> };
>
> static int __init cluster_init(void)
> diff --git a/drivers/md/md-cluster.h b/drivers/md/md-cluster.h
> index 7417133..71e5143 100644
> --- a/drivers/md/md-cluster.h
> +++ b/drivers/md/md-cluster.h
> @@ -22,6 +22,7 @@ struct md_cluster_operations {
> int (*add_new_disk_start)(struct mddev *mddev, struct md_rdev *rdev);
> int (*add_new_disk_finish)(struct mddev *mddev);
> int (*new_disk_ack)(struct mddev *mddev, bool ack);
> + int (*remove_disk)(struct mddev *mddev, struct md_rdev *rdev);
> };
>
> #endif /* _MD_CLUSTER_H */
> diff --git a/drivers/md/md.c b/drivers/md/md.c
> index bc11551..0c65e51 100644
> --- a/drivers/md/md.c
> +++ b/drivers/md/md.c
> @@ -2291,11 +2291,12 @@ static void export_rdev(struct md_rdev * rdev)
> kobject_put(&rdev->kobj);
> }
>
> -static void kick_rdev_from_array(struct md_rdev * rdev)
> +void md_kick_rdev_from_array(struct md_rdev * rdev)
> {
> unbind_rdev_from_array(rdev);
> export_rdev(rdev);
> }
> +EXPORT_SYMBOL_GPL(md_kick_rdev_from_array);
>
> static void export_array(struct mddev *mddev)
> {
> @@ -2306,7 +2307,7 @@ static void export_array(struct mddev *mddev)
> MD_BUG();
> continue;
> }
> - kick_rdev_from_array(rdev);
> + md_kick_rdev_from_array(rdev);
> }
> if (!list_empty(&mddev->disks))
> MD_BUG();
> @@ -2750,9 +2751,11 @@ state_store(struct md_rdev *rdev, const char *buf, size_t len)
> err = -EBUSY;
> else {
> struct mddev *mddev = rdev->mddev;
> - if (mddev_is_clustered(mddev))
> + if (mddev_is_clustered(mddev)) {
> md_cluster_ops->metadata_update_start(mddev);
> - kick_rdev_from_array(rdev);
> + md_cluster_ops->remove_disk(mddev, rdev);
> + }
> + md_kick_rdev_from_array(rdev);
>
For md-cluster, seems it is possible that md_kick_rdev_from_array could
be called twice,
is this what you want? Thanks.
> if (mddev->pers) {
> set_bit(MD_CHANGE_DEVS, &mddev->flags);
> md_wakeup_thread(mddev->thread);
> @@ -3424,7 +3427,7 @@ static void analyze_sbs(struct mddev * mddev)
> "md: fatal superblock inconsistency in %s"
> " -- removing from array\n",
> bdevname(rdev->bdev,b));
> - kick_rdev_from_array(rdev);
> + md_kick_rdev_from_array(rdev);
> }
>
>
> @@ -3440,7 +3443,7 @@ static void analyze_sbs(struct mddev * mddev)
> "md: %s: %s: only %d devices permitted\n",
> mdname(mddev), bdevname(rdev->bdev, b),
> mddev->max_disks);
> - kick_rdev_from_array(rdev);
> + md_kick_rdev_from_array(rdev);
> continue;
> }
> if (rdev != freshest) {
> @@ -3449,7 +3452,7 @@ static void analyze_sbs(struct mddev * mddev)
> printk(KERN_WARNING "md: kicking non-fresh %s"
> " from array!\n",
> bdevname(rdev->bdev,b));
> - kick_rdev_from_array(rdev);
> + md_kick_rdev_from_array(rdev);
> continue;
> }
> /* No device should have a Candidate flag
> @@ -3458,7 +3461,7 @@ static void analyze_sbs(struct mddev * mddev)
> if (test_bit(Candidate, &rdev->flags)) {
> pr_info("md: kicking Cluster Candidate %s from array!\n",
> bdevname(rdev->bdev, b));
> - kick_rdev_from_array(rdev);
> + md_kick_rdev_from_array(rdev);
> }
> }
> if (mddev->level == LEVEL_MULTIPATH) {
> @@ -6083,7 +6086,10 @@ static int hot_remove_disk(struct mddev * mddev, dev_t dev)
> if (rdev->raid_disk >= 0)
> goto busy;
>
> - kick_rdev_from_array(rdev);
> + if (mddev_is_clustered(mddev))
> + md_cluster_ops->remove_disk(mddev, rdev);
> +
> + md_kick_rdev_from_array(rdev);
>
Ditto.
Thanks,
Guoqing
^ permalink raw reply
* Re: RAID 5 "magicaly" become a RAID0
From: Phil Turmel @ 2015-04-13 0:18 UTC (permalink / raw)
To: Thomas MARCHESSEAU, linux-raid
In-Reply-To: <D150B36B.8EE37%marchesseau@gmail.com>
Hi Thomas,
On 04/12/2015 05:42 PM, Thomas MARCHESSEAU wrote:
> Hi team ,
>
> Like probably lot of new subscriber , i mail you, guys, for help .
>
> I¹m running a raid5 on 7 HDD for several month now ( and years on other
> system) without problem .
> last week i had a crash disk (sdg) , i¹ve add a new drive (sdi) and
> rebuild .. Works fine , and i dont think this is the cause of my today
> problem.
Agreed. Probably not related.
> Yesterday , i¹v upgraded my ubuntu 14.10 , and the system warm me with a
> message that i can¹t recall and rewrite exactly , but something like :
> md127 doesn¹t not match with /etc/mdadm/mdadm.conf , blah blah , run
> /usr/share/mdadm/mkconf , and fix /etc/mdadm/mdadm.conf
>
> i¹ve done it , and reboot , all looks good .
> All the drive have been rename after reboot ( orginal sdg was extract form
> the bay )
Yes, you cannot trust drives to keep their names through upgrades. The
names are pseudo-random during boot.
You should know that md127 is the default name chosen by mdadm when
assembling an array for which it doesn't know any other name. Followed
by md126, then md125 and so on. You really should give your arrays
other names. Most commonly starting with md0 or md1.
> I¹ve setup a rsync of my most important data on a external drive this
> night, who partially failed (only 25% ha been backuped , bad luck ) ,
> (probably) because this morning i have re-inserted by mistake the faulty
> drive ( for information , i think the drive was in fact ok , the sata
> connector was a bit disconnect )
>
> I did not pay attention of the situation at the moment , but few hour
> later , i ssh my filer and my « home » (on the raid partition) was not
> available anymore .
You should collect your 'dmesg' and post it here. Or cut and paste from
it anything related to your drives or array.
> I didn¹t try to fschk or any thing else than :
>
> Mdadm ‹stop /dev/md127
> mdadm --assemble /dev/md127 /dev/sdb /dev/sdc /dev/sdd /dev/sde /dev/sdf
> /dev/sdg /dev/sdh
> mdadm: /dev/md127 assembled from 5 drives - not enough to start the array.
>
>
> So i¹ve read a bunch of usefull link , one of them :) ,
> https://raid.wiki.kernel.org/index.php/RAID_Recovery , says , don¹t do
> stupid thing until drop a mail on linux-raid mailling Š so i¹m here .
>
> i¹ve collected this usefull info :
> mdadm --examine /dev/sd[a-z] | egrep 'Event|/dev/sd'
> /dev/sda: (system HDD )
> /dev/sdb:
> Events : 21958
> /dev/sdc:
> Events : 21958
> /dev/sdd:
> Events : 21958
> /dev/sde:
> Events : 21958
> /dev/sdf:
> Events : 21958
> /dev/sdg:
> Events : 21954 <‹ here
> /dev/sdh:
> Events : 21954 <‹ and here
In general, people on this list want to see the full --examine reports.
As do I.
Also, you need to record which drive serial numbers correspond to which
device roles, just in case. You can show the smartctl data along with
the examines like so:
for x in /dev/sd[b-h] ; do mdadm -E $x ; smartctl -iA -l scterc $x ;
done > report.txt
Then paste report.txt into your next mail.
> The strange thing is that my raid array is now seen as a RAID0 in
> mdadm --detail /dev/md127
> /dev/md127:
> Version :
> Raid Level : raid0
> Total Devices : 0
>
> State : inactive
It didn't start, so that info isn't meaningful.
> But individually all drive in mdadm ‹examine , are RAID 5 member .
>
> Anyone for help ?
Your array should be fixable. The use of "mdadm --assemble --force" as
recommended by Roger is likely to work. But it won't be enough if you
don't also figure out why the array stopped after a few hours. That
sounds like a common problem with raid5 rebuilds.
> i¹ was on the way to perform a
> mdadm --create --assume-clean ‹level=5 --raid-devices=7 --size=11720300544
> /dev/md127 /dev/sdb /dev/sdc /dev/sdd /dev/sde /dev/sdf /dev/sdg /dev/sdh
>
> Which looks a bit stupid before ask for help
Yes, this is what the wiki means when it refers to doing something
stupid. Any form of --create is destructive and should only be
attempted when all other attempts have failed.
Phil
--
To unsubscribe from this list: send the line "unsubscribe linux-raid" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* [GIT PULL REQUEST] md fixes for late 4.0-rc
From: NeilBrown @ 2015-04-12 23:20 UTC (permalink / raw)
To: Linus Torvalds
Cc: lkml, linux RAID, Joe Landman, Dave Chinner, Gu Zheng,
Simon Kirby, Dave Jones
[-- Attachment #1: Type: text/plain, Size: 1168 bytes --]
Hi Linus,
two regression fixes for md, one fairly recent and minor (diskstats has
confusing data) and one older and more serious - RAID0 with non-power-of-2
chunksize corrupts data. I guess (almost) no-one uses non-power-of-2 chunks.
Thanks,
NeilBrown
The following changes since commit f22e6e847115abc3a0e2ad7bb18d243d42275af1:
Linux 4.0-rc7 (2015-04-06 15:39:45 -0700)
are available in the git repository at:
git://neil.brown.name/md/ tags/md/4.0-rc7-fix
for you to fetch changes up to 47d68979cc968535cb87f3e5f2e6a3533ea48fbd:
md/raid0: fix bug with chunksize not a power of 2. (2015-04-10 15:36:31 +1000)
----------------------------------------------------------------
md fixes for 4.0-rc7
Revert recent change which broke IO accounting.
Fix bug with RAID0 arrays with non-power-of-2 chunk size.
----------------------------------------------------------------
Gu Zheng (1):
md: fix md io stats accounting broken
NeilBrown (1):
md/raid0: fix bug with chunksize not a power of 2.
drivers/md/md.c | 6 +++++-
drivers/md/raid0.c | 3 ++-
2 files changed, 7 insertions(+), 2 deletions(-)
[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 811 bytes --]
^ permalink raw reply
* Re: RAID 5 "magicaly" become a RAID0
From: Roger Heflin @ 2015-04-12 22:28 UTC (permalink / raw)
To: Thomas MARCHESSEAU; +Cc: Linux RAID
In-Reply-To: <D150B36B.8EE37%marchesseau@gmail.com>
Add a "--force" to the --assemble command and it should force the
other 2 online even though they are a bit off on events.
From my understanding that may been some of the last data written
could be lost/corrupted (those last 4 events on those 2 disks).
I would probably also suggest getting another disk and going to raid6,
raid5 is pretty scary if disks start going bad.
On Sun, Apr 12, 2015 at 4:42 PM, Thomas MARCHESSEAU
<marchesseau@gmail.com> wrote:
> Hi team ,
>
> Like probably lot of new subscriber , i mail you, guys, for help .
>
> Iąm running a raid5 on 7 HDD for several month now ( and years on other
> system) without problem .
> last week i had a crash disk (sdg) , iąve add a new drive (sdi) and
> rebuild .. Works fine , and i dont think this is the cause of my today
> problem.
>
> Yesterday , iąv upgraded my ubuntu 14.10 , and the system warm me with a
> message that i canąt recall and rewrite exactly , but something like :
> md127 doesnąt not match with /etc/mdadm/mdadm.conf , blah blah , run
> /usr/share/mdadm/mkconf , and fix /etc/mdadm/mdadm.conf
>
> iąve done it , and reboot , all looks good .
> All the drive have been rename after reboot ( orginal sdg was extract form
> the bay )
>
> Iąve setup a rsync of my most important data on a external drive this
> night, who partially failed (only 25% ha been backuped , bad luck ) ,
> (probably) because this morning i have re-inserted by mistake the faulty
> drive ( for information , i think the drive was in fact ok , the sata
> connector was a bit disconnect )
>
> I did not pay attention of the situation at the moment , but few hour
> later , i ssh my filer and my « home » (on the raid partition) was not
> available anymore .
> I didnąt try to fschk or any thing else than :
>
> Mdadm ‹stop /dev/md127
> mdadm --assemble /dev/md127 /dev/sdb /dev/sdc /dev/sdd /dev/sde /dev/sdf
> /dev/sdg /dev/sdh
> mdadm: /dev/md127 assembled from 5 drives - not enough to start the array.
>
>
> So iąve read a bunch of usefull link , one of them :) ,
> https://raid.wiki.kernel.org/index.php/RAID_Recovery , says , donąt do
> stupid thing until drop a mail on linux-raid mailling Š so iąm here .
>
> iąve collected this usefull info :
> mdadm --examine /dev/sd[a-z] | egrep 'Event|/dev/sd'
> /dev/sda: (system HDD )
> /dev/sdb:
> Events : 21958
> /dev/sdc:
> Events : 21958
> /dev/sdd:
> Events : 21958
> /dev/sde:
> Events : 21958
> /dev/sdf:
> Events : 21958
> /dev/sdg:
> Events : 21954 <‹ here
> /dev/sdh:
> Events : 21954 <‹ and here
>
>
>
> iąve also a full copy of mdadm ‹examine
>
> The strange thing is that my raid array is now seen as a RAID0 in
> mdadm --detail /dev/md127
> /dev/md127:
> Version :
> Raid Level : raid0
> Total Devices : 0
>
> State : inactive
>
>
> But individually all drive in mdadm ‹examine , are RAID 5 member .
>
> Anyone for help ?
>
> ią was on the way to perform a
> mdadm --create --assume-clean ‹level=5 --raid-devices=7 --size=11720300544
> /dev/md127 /dev/sdb /dev/sdc /dev/sdd /dev/sde /dev/sdf /dev/sdg /dev/sdh
>
> Which looks a bit stupid before ask for help
>
> Regards thomas
>
>
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-raid" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at http://vger.kernel.org/majordomo-info.html
--
To unsubscribe from this list: send the line "unsubscribe linux-raid" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* RAID 5 "magicaly" become a RAID0
From: Thomas MARCHESSEAU @ 2015-04-12 21:42 UTC (permalink / raw)
To: linux-raid
Hi team ,
Like probably lot of new subscriber , i mail you, guys, for help .
I¹m running a raid5 on 7 HDD for several month now ( and years on other
system) without problem .
last week i had a crash disk (sdg) , i¹ve add a new drive (sdi) and
rebuild .. Works fine , and i dont think this is the cause of my today
problem.
Yesterday , i¹v upgraded my ubuntu 14.10 , and the system warm me with a
message that i can¹t recall and rewrite exactly , but something like :
md127 doesn¹t not match with /etc/mdadm/mdadm.conf , blah blah , run
/usr/share/mdadm/mkconf , and fix /etc/mdadm/mdadm.conf
i¹ve done it , and reboot , all looks good .
All the drive have been rename after reboot ( orginal sdg was extract form
the bay )
I¹ve setup a rsync of my most important data on a external drive this
night, who partially failed (only 25% ha been backuped , bad luck ) ,
(probably) because this morning i have re-inserted by mistake the faulty
drive ( for information , i think the drive was in fact ok , the sata
connector was a bit disconnect )
I did not pay attention of the situation at the moment , but few hour
later , i ssh my filer and my « home » (on the raid partition) was not
available anymore .
I didn¹t try to fschk or any thing else than :
Mdadm ‹stop /dev/md127
mdadm --assemble /dev/md127 /dev/sdb /dev/sdc /dev/sdd /dev/sde /dev/sdf
/dev/sdg /dev/sdh
mdadm: /dev/md127 assembled from 5 drives - not enough to start the array.
So i¹ve read a bunch of usefull link , one of them :) ,
https://raid.wiki.kernel.org/index.php/RAID_Recovery , says , don¹t do
stupid thing until drop a mail on linux-raid mailling Š so i¹m here .
i¹ve collected this usefull info :
mdadm --examine /dev/sd[a-z] | egrep 'Event|/dev/sd'
/dev/sda: (system HDD )
/dev/sdb:
Events : 21958
/dev/sdc:
Events : 21958
/dev/sdd:
Events : 21958
/dev/sde:
Events : 21958
/dev/sdf:
Events : 21958
/dev/sdg:
Events : 21954 <‹ here
/dev/sdh:
Events : 21954 <‹ and here
i¹ve also a full copy of mdadm ‹examine
The strange thing is that my raid array is now seen as a RAID0 in
mdadm --detail /dev/md127
/dev/md127:
Version :
Raid Level : raid0
Total Devices : 0
State : inactive
But individually all drive in mdadm ‹examine , are RAID 5 member .
Anyone for help ?
i¹ was on the way to perform a
mdadm --create --assume-clean ‹level=5 --raid-devices=7 --size=11720300544
/dev/md127 /dev/sdb /dev/sdc /dev/sdd /dev/sde /dev/sdf /dev/sdg /dev/sdh
Which looks a bit stupid before ask for help
Regards thomas
--
To unsubscribe from this list: send the line "unsubscribe linux-raid" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* mvsas panics and dies when attached to a port extender on newer kernels
From: Adam Talbot @ 2015-04-12 20:22 UTC (permalink / raw)
To: linux-raid
[-- Attachment #1: Type: text/plain, Size: 909 bytes --]
Not %100 sure what group this one should go to. As this is on my
Linux RAID system; I hope I got the right group. I think I found a
mvsas bug.
My Debian Jessie system was running great on a 3.18 kernel. Changed
cases to a newer supermicro case with a SAS expander backplane. That
was the only hardware change. When ever I boot, the system kernel
panics.
To make a very long debugging story short, I think there is an issues
with the mvsas driver. It works, with older kernels, and breaks on
newer kernels.
3.2.65-1+deb7u2 works
3.9.0 Gentoo CD works
3.16+ all fail
Attached are 3 kernel panics on 3.16+ kernels.
Hardware is a Supermicro X8SIE, with a "Marvell Technology Group Ltd.
88SE6440 SAS/SATA PCIe controller"
Is this a known bug?
At this point I have two options:
Stick with the old kernel
Buy a new card running a better supported chipset
Any help would be greatly appreciated
Thanks
Adam
[-- Attachment #2: 3.16.0-kernel-panic --]
[-- Type: application/octet-stream, Size: 9317 bytes --]
[ 5.192410] scsi0 : mvsas
[ 5.456795] general protection fault: 0000 [#1] SMP
[ 5.461913] Modules linked in: usb_storage hid_generic usbhid hid ata_generic mvsas libsas ata_piix ehci_pci libata ehci_hcd scsi_transport_sas e1000e ptp crc32c_intel pps_core usbcore scsi_mod usb_common
[ 5.481174] CPU: 3 PID: 138 Comm: kworker/u16:5 Not tainted 3.16.0-4-amd64 #1 Debian 3.16.7-ckt7-1
[ 5.490194] Hardware name: Supermicro X8SIE/X8SIE, BIOS 1.2a 06/27/2012
[ 5.497378] Workqueue: events_unbound async_run_entry_fn
[ 5.502791] task: ffff880036f68010 ti: ffff880036f74000 task.ti: ffff880036f74000
[ 5.511515] RIP: 0010:[<ffffffffa00bd90f>] [<ffffffffa00bd90f>] mvs_task_prep+0x73f/0xd50 [mvsas]
[ 5.520596] RSP: 0018:ffff880036f77948 EFLAGS: 00010097
[ 5.525960] RAX: 000000000000002c RBX: cccccccccccccccc RCX: 0000000000000000
[ 5.533142] RDX: 0000000000000000 RSI: ffff880036ea55b8 RDI: ffff880036e80000
[ 5.540325] RBP: 0000000000000000 R08: ffff880036e836a0 R09: ffff880036e70000
[ 5.547508] R10: ffffc900018f0000 R11: 0000000000000005 R12: ffff880036e80000
[ 5.554692] R13: ffff88032f4bf600 R14: ffff880036e836a0 R15: ffff88032f123e40
[ 5.561875] FS: 0000000000000000(0000) GS:ffff88033fcc0000(0000) knlGS:0000000000000000
[ 5.570028] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
[ 5.575825] CR2: 00007ffb9332c000 CR3: 0000000001813000 CR4: 00000000000007e0
[ 5.583006] Stack:
[ 5.585070] ffff880036e81e38 ffff880036f779ec ffff880036ea55b8 ffff880300000001
[ 5.592737] ffff880036e82678 ffff880036ea55b0 ffff880036e836a0 ffff880036e70000
[ 5.600404] 0000000000012f00 ffff880036f68010 ffff880332584418 ffff880036f77a68
[ 5.608063] Call Trace:
[ 5.610562] [<ffffffffa00bdf77>] ? mvs_task_exec.isra.13+0x57/0xd0 [mvsas]
[ 5.617571] [<ffffffffa00bea71>] ? mvs_queue_command+0x301/0x310 [mvsas]
[ 5.624407] [<ffffffff8118e2bf>] ? kmem_cache_alloc+0x21f/0x450
[ 5.630464] [<ffffffffa011c3ac>] ? sas_ata_qc_issue+0x22c/0x270 [libsas]
[ 5.637303] [<ffffffffa026272c>] ? ata_qc_issue+0x16c/0x390 [libata]
[ 5.643789] [<ffffffffa0262be2>] ? ata_exec_internal_sg+0x292/0x570 [libata]
[ 5.650974] [<ffffffffa0262f18>] ? ata_exec_internal+0x58/0xa0 [libata]
[ 5.657721] [<ffffffffa011c0ae>] ? sas_ata_hard_reset+0x9e/0x140 [libsas]
[ 5.664647] [<ffffffffa0263325>] ? ata_dev_read_id+0x275/0x560 [libata]
[ 5.671395] [<ffffffffa026d5a4>] ? ata_eh_schedule_probe+0x104/0x150 [libata]
[ 5.678681] [<ffffffffa02707d4>] ? ata_eh_recover+0x814/0x1350 [libata]
[ 5.685432] [<ffffffffa0265d40>] ? ata_phys_link_offline+0x30/0x30 [libata]
[ 5.692526] [<ffffffffa011c010>] ? sas_ata_printk+0x80/0x80 [libsas]
[ 5.699018] [<ffffffffa0265bc0>] ? ata_phys_link_online+0x30/0x30 [libata]
[ 5.706025] [<ffffffff8150768a>] ? printk+0x54/0x56
[ 5.711042] [<ffffffffa0265d40>] ? ata_phys_link_offline+0x30/0x30 [libata]
[ 5.718140] [<ffffffffa011c010>] ? sas_ata_printk+0x80/0x80 [libsas]
[ 5.724628] [<ffffffffa0271de4>] ? ata_do_eh+0x44/0xb0 [libata]
[ 5.730685] [<ffffffffa0265bc0>] ? ata_phys_link_online+0x30/0x30 [libata]
[ 5.737694] [<ffffffffa0271908>] ? ata_scsi_port_error_handler+0x518/0x8f0 [libata]
[ 5.745500] [<ffffffffa011c786>] ? async_sas_ata_eh+0x46/0x70 [libsas]
[ 5.752162] [<ffffffff8108d872>] ? async_run_entry_fn+0x32/0x120
[ 5.758307] [<ffffffff810814e2>] ? process_one_work+0x172/0x420
[ 5.764362] [<ffffffff81081b73>] ? worker_thread+0x113/0x4f0
[ 5.770159] [<ffffffff8150c291>] ? __schedule+0x2b1/0x710
[ 5.775694] [<ffffffff81081a60>] ? rescuer_thread+0x2d0/0x2d0
[ 5.781578] [<ffffffff81087dfd>] ? kthread+0xbd/0xe0
[ 5.786682] [<ffffffff81087d40>] ? kthread_create_on_node+0x180/0x180
[ 5.793259] [<ffffffff8150febc>] ? ret_from_fork+0x7c/0xb0
[ 5.798883] [<ffffffff81087d40>] ? kthread_create_on_node+0x180/0x180
[ 5.805458] Code: 92 b8 02 00 00 41 80 b8 84 00 00 00 7f 48 8b 80 58 01 00 00 48 8b 1c d0 0f 84 9d 05 00 00 41 8b 44 24 58 48 8b 74 24 10 89 46 1c <8b> 8b 54 02 00 00 be 00 10 00 00 41 8b 54 24 58 49 8b 44 24 48
[ 5.827742] RIP [<ffffffffa00bd90f>] mvs_task_prep+0x73f/0xd50 [mvsas]
[ 5.834457] RSP <ffff880036f77948>
[ 5.837992] ---[ end trace ab01b27d6a351042 ]---
[ 5.842671] BUG: unable to handle kernel paging request at ffffffffffffffd8
[ 5.849783] IP: [<ffffffff8108838c>] kthread_data+0xc/0x20
[ 5.855374] PGD 1816067 PUD 1818067 PMD 0
[ 5.859679] Oops: 0000 [#2] SMP
[ 5.863058] Modules linked in: usb_storage hid_generic usbhid hid ata_generic mvsas libsas ata_piix ehci_pci libata ehci_hcd scsi_transport_sas e1000e ptp crc32c_intel pps_core usbcore scsi_mod usb_common
[ 5.882319] CPU: 3 PID: 138 Comm: kworker/u16:5 Tainted: G D 3.16.0-4-amd64 #1 Debian 3.16.7-ckt7-1
[ 5.892379] Hardware name: Supermicro X8SIE/X8SIE, BIOS 1.2a 06/27/2012
[ 5.899567] task: ffff880036f68010 ti: ffff880036f74000 task.ti: ffff880036f74000
[ 5.907109] RIP: 0010:[<ffffffff8108838c>] [<ffffffff8108838c>] kthread_data+0xc/0x20
[ 5.915140] RSP: 0018:ffff880036f77798 EFLAGS: 00010002
[ 5.920504] RAX: 0000000000000000 RBX: 0000000000000003 RCX: 000000000000000f
[ 5.927686] RDX: 0000000000000000 RSI: 0000000000000003 RDI: ffff880036f68010
[ 5.934869] RBP: ffff880036f68010 R08: 0000000000000000 R09: 0000000000000000
[ 5.942052] R10: ffffffff81adf150 R11: 0000000000000000 R12: ffff88033fcd2f00
[ 5.949235] R13: 0000000000000003 R14: 0000000000000000 R15: ffff880036f68010
[ 5.956417] FS: 0000000000000000(0000) GS:ffff88033fcc0000(0000) knlGS:0000000000000000
[ 5.964571] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
[ 5.970367] CR2: 0000000000000028 CR3: 0000000001813000 CR4: 00000000000007e0
[ 5.977550] Stack:
[ 5.979612] ffffffff81081fbd ffff880036f68470 ffffffff8150c43d 0000000000012f00
[ 5.987281] ffff880036f77fd8 0000000000012f00 ffff880036f68010 ffff880036f68690
[ 5.994948] ffff880036f68368 ffff880036f68000 ffff8803329752b0 ffff880036f68000
[ 6.002608] Call Trace:
[ 6.005104] [<ffffffff81081fbd>] ? wq_worker_sleeping+0xd/0x80
[ 6.011075] [<ffffffff8150c43d>] ? __schedule+0x45d/0x710
[ 6.016611] [<ffffffff81069f0f>] ? do_exit+0x6ef/0xa60
[ 6.021890] [<ffffffff810163b7>] ? oops_end+0x97/0xe0
[ 6.027078] [<ffffffff81511f48>] ? general_protection+0x28/0x30
[ 6.033134] [<ffffffffa00bd90f>] ? mvs_task_prep+0x73f/0xd50 [mvsas]
[ 6.039625] [<ffffffffa00bdec5>] ? mvs_task_prep+0xcf5/0xd50 [mvsas]
[ 6.046115] [<ffffffffa00bdf77>] ? mvs_task_exec.isra.13+0x57/0xd0 [mvsas]
[ 6.053124] [<ffffffffa00bea71>] ? mvs_queue_command+0x301/0x310 [mvsas]
[ 6.059960] [<ffffffff8118e2bf>] ? kmem_cache_alloc+0x21f/0x450
[ 6.066015] [<ffffffffa011c3ac>] ? sas_ata_qc_issue+0x22c/0x270 [libsas]
[ 6.072852] [<ffffffffa026272c>] ? ata_qc_issue+0x16c/0x390 [libata]
[ 6.079341] [<ffffffffa0262be2>] ? ata_exec_internal_sg+0x292/0x570 [libata]
[ 6.086526] [<ffffffffa0262f18>] ? ata_exec_internal+0x58/0xa0 [libata]
[ 6.093274] [<ffffffffa011c0ae>] ? sas_ata_hard_reset+0x9e/0x140 [libsas]
[ 6.100198] [<ffffffffa0263325>] ? ata_dev_read_id+0x275/0x560 [libata]
[ 6.106947] [<ffffffffa026d5a4>] ? ata_eh_schedule_probe+0x104/0x150 [libata]
[ 6.114233] [<ffffffffa02707d4>] ? ata_eh_recover+0x814/0x1350 [libata]
[ 6.118548] scsi 5:0:0:0: Direct-Access Generic USB EDC 1.00 PQ: 0 ANSI: 2
[ 6.129139] [<ffffffffa0265d40>] ? ata_phys_link_offline+0x30/0x30 [libata]
[ 6.136242] [<ffffffffa011c010>] ? sas_ata_printk+0x80/0x80 [libsas]
[ 6.142732] [<ffffffffa0265bc0>] ? ata_phys_link_online+0x30/0x30 [libata]
[ 6.149740] [<ffffffff8150768a>] ? printk+0x54/0x56
[ 6.154759] [<ffffffffa0265d40>] ? ata_phys_link_offline+0x30/0x30 [libata]
[ 6.161852] [<ffffffffa011c010>] ? sas_ata_printk+0x80/0x80 [libsas]
[ 6.168345] [<ffffffffa0271de4>] ? ata_do_eh+0x44/0xb0 [libata]
[ 6.174401] [<ffffffffa0265bc0>] ? ata_phys_link_online+0x30/0x30 [libata]
[ 6.181410] [<ffffffffa0271908>] ? ata_scsi_port_error_handler+0x518/0x8f0 [libata]
[ 6.189215] [<ffffffffa011c786>] ? async_sas_ata_eh+0x46/0x70 [libsas]
[ 6.195879] [<ffffffff8108d872>] ? async_run_entry_fn+0x32/0x120
[ 6.202022] [<ffffffff810814e2>] ? process_one_work+0x172/0x420
[ 6.208077] [<ffffffff81081b73>] ? worker_thread+0x113/0x4f0
[ 6.213874] [<ffffffff8150c291>] ? __schedule+0x2b1/0x710
[ 6.219410] [<ffffffff81081a60>] ? rescuer_thread+0x2d0/0x2d0
[ 6.225293] [<ffffffff81087dfd>] ? kthread+0xbd/0xe0
[ 6.230397] [<ffffffff81087d40>] ? kthread_create_on_node+0x180/0x180
[ 6.236973] [<ffffffff8150febc>] ? ret_from_fork+0x7c/0xb0
[ 6.242597] [<ffffffff81087d40>] ? kthread_create_on_node+0x180/0x180
[ 6.249172] Code: 08 04 00 00 48 8b 40 c8 48 c1 e8 02 83 e0 01 c3 66 66 66 66 66 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 48 8b 87 08 04 00 00 <48> 8b 40 d8 c3 66 66 66 66 66 66 2e 0f 1f 84 00 00 00 00 00 66
[ 6.271449] RIP [<ffffffff8108838c>] kthread_data+0xc/0x20
[ 6.277115] RSP <ffff880036f77798>
[ 6.280649] CR2: ffffffffffffffd8
[ 6.284012] ---[ end trace ab01b27d6a351043 ]---
[ 6.288681] Fixing recursive fault but reboot is needed!
[-- Attachment #3: 3.18.0-kernel-panic --]
[-- Type: application/octet-stream, Size: 10168 bytes --]
[ 5.295367] scsi host0: mvsas
[ 5.555699] BUG: unable to handle kernel paging request at 00000030747371bc
[ 5.562813] IP: [<ffffffffa00cd7ed>] mvs_task_prep+0x78d/0xe40 [mvsas]
[ 5.569440] PGD 0
[ 5.571555] Oops: 0000 [#1] SMP
[ 5.574934] Modules linked in: usb_storage hid_generic usbhid hid ata_generic mvsas libsas ata_piix ehci_pci ehci_hcd libata scsi_transport_sas e1000e ptp crc32c_intel usbcore pps_core usb_common scsi_mod
[ 5.594188] CPU: 3 PID: 78 Comm: kworker/u16:1 Not tainted 3.18.0-trunk-amd64 #1 Debian 3.18.5-1~exp1
[ 5.603467] Hardware name: Supermicro X8SIE/X8SIE, BIOS 1.2a 06/27/2012
[ 5.610651] Workqueue: events_unbound async_run_entry_fn
[ 5.616056] task: ffff88032fb6cc40 ti: ffff88032fb78000 task.ti: ffff88032fb78000
[ 5.624779] RIP: 0010:[<ffffffffa00cd7ed>] [<ffffffffa00cd7ed>] mvs_task_prep+0x78d/0xe40 [mvsas]
[ 5.633853] RSP: 0018:ffff88032fb7b888 EFLAGS: 00010097
[ 5.639217] RAX: 000000000000002c RBX: 0000003074736f68 RCX: 0000000000000000
[ 5.646399] RDX: ffff8800372c3618 RSI: 0000000000000000 RDI: 0000000000001000
[ 5.653580] RBP: 0000000000000000 R08: ffff88032f572400 R09: ffff8800372a8000
[ 5.660763] R10: ffffc900018f6000 R11: 0000000000010000 R12: 0000000000000000
[ 5.667947] R13: ffff8800372c3618 R14: ffff8800372c0000 R15: ffff88032f23ce40
[ 5.675129] FS: 0000000000000000(0000) GS:ffff88033fcc0000(0000) knlGS:0000000000000000
[ 5.683284] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
[ 5.689079] CR2: 00000030747371bc CR3: 0000000001813000 CR4: 00000000000007e0
[ 5.696261] Stack:
[ 5.698323] ffff88032fb7b934 ffff8800372c3618 0000000000000000 ffff8800372c2678
[ 5.705983] ffff8800372c1e38 ffff8800372a8000 ffff88032f572400 ffff88032fb7bfd8
[ 5.713642] 0000000000013740 ffff88032fb6cc40 0000000300000001 ffffffff8109904a
[ 5.721310] Call Trace:
[ 5.723810] [<ffffffff8109904a>] ? select_idle_sibling+0x10a/0x120
[ 5.730125] [<ffffffffa00cdef7>] ? mvs_task_exec.isra.13+0x57/0xd0 [mvsas]
[ 5.737135] [<ffffffffa00ce9f9>] ? mvs_queue_command+0x2e9/0x2f0 [mvsas]
[ 5.743971] [<ffffffffa017a7cc>] ? sas_ata_qc_issue+0x22c/0x270 [libsas]
[ 5.750808] [<ffffffffa028e845>] ? ata_qc_issue+0x1f5/0x380 [libata]
[ 5.757296] [<ffffffffa028ecbf>] ? ata_exec_internal_sg+0x2ef/0x5f0 [libata]
[ 5.764480] [<ffffffffa028f018>] ? ata_exec_internal+0x58/0xa0 [libata]
[ 5.771229] [<ffffffffa017a4cd>] ? sas_ata_hard_reset+0x9d/0x140 [libsas]
[ 5.778151] [<ffffffffa0291f35>] ? ata_phys_link_offline+0x15/0x30 [libata]
[ 5.785247] [<ffffffffa028f3b5>] ? ata_dev_read_id+0x205/0x5c0 [libata]
[ 5.791997] [<ffffffffa017a430>] ? sas_ata_printk+0x90/0x90 [libsas]
[ 5.798487] [<ffffffffa02996c1>] ? ata_eh_schedule_probe+0x141/0x180 [libata]
[ 5.805774] [<ffffffffa029c5da>] ? ata_eh_recover+0x7da/0x1370 [libata]
[ 5.812524] [<ffffffffa0291f50>] ? ata_phys_link_offline+0x30/0x30 [libata]
[ 5.819619] [<ffffffffa017a430>] ? sas_ata_printk+0x90/0x90 [libsas]
[ 5.826109] [<ffffffffa0291dc0>] ? ata_phys_link_online+0x30/0x30 [libata]
[ 5.833119] [<ffffffffa017a430>] ? sas_ata_printk+0x90/0x90 [libsas]
[ 5.839609] [<ffffffffa0291f50>] ? ata_phys_link_offline+0x30/0x30 [libata]
[ 5.846703] [<ffffffffa017a430>] ? sas_ata_printk+0x90/0x90 [libsas]
[ 5.853194] [<ffffffffa029db06>] ? ata_do_eh+0x46/0xb0 [libata]
[ 5.859251] [<ffffffffa0291dc0>] ? ata_phys_link_online+0x30/0x30 [libata]
[ 5.866262] [<ffffffff8101c206>] ? native_sched_clock+0x26/0x90
[ 5.872318] [<ffffffffa029d698>] ? ata_scsi_port_error_handler+0x448/0x7b0 [libata]
[ 5.880122] [<ffffffffa017abb6>] ? async_sas_ata_eh+0x46/0x70 [libsas]
[ 5.886786] [<ffffffff8108a973>] ? async_run_entry_fn+0x43/0x160
[ 5.892930] [<ffffffff81082a8e>] ? process_one_work+0x14e/0x3f0
[ 5.898985] [<ffffffff8108338b>] ? worker_thread+0x6b/0x4a0
[ 5.904693] [<ffffffff81083320>] ? rescuer_thread+0x290/0x290
[ 5.910578] [<ffffffff810880e3>] ? kthread+0xd3/0xf0
[ 5.915680] [<ffffffff81088010>] ? kthread_create_on_node+0x180/0x180
[ 5.922260] [<ffffffff8154543c>] ? ret_from_fork+0x7c/0xb0
[ 5.927880] [<ffffffff81088010>] ? kthread_create_on_node+0x180/0x180
[ 5.934455] Code: 8b 80 58 01 00 00 48 8b 1c c8 0f 84 29 06 00 00 41 8b 46 58 48 8d 4c ad 00 bf 00 10 00 00 48 8d 4c 4d 00 41 89 84 ce d4 55 02 00 <8b> 8b 54 02 00 00 49 8b 76 48 d3 e7 0f b6 8a 84 00 00 00 81 cf
[ 5.956722] RIP [<ffffffffa00cd7ed>] mvs_task_prep+0x78d/0xe40 [mvsas]
[ 5.963438] RSP <ffff88032fb7b888>
[ 5.966973] CR2: 00000030747371bc
[ 5.970337] ---[ end trace d307165136b3807d ]---
[ 5.987492] BUG: unable to handle kernel paging request at ffffffffffffffd8
[ 5.994605] IP: [<ffffffff810886fc>] kthread_data+0xc/0x20
[ 6.000183] PGD 1816067 PUD 1818067 PMD 0
[ 6.004483] Oops: 0000 [#2] SMP
[ 6.007871] Modules linked in: usb_storage hid_generic usbhid hid ata_generic mvsas libsas ata_piix ehci_pci ehci_hcd libata scsi_transport_sas e1000e ptp crc32c_intel usbcore pps_core usb_common scsi_mod
[ 6.027123] CPU: 3 PID: 78 Comm: kworker/u16:1 Tainted: G D 3.18.0-trunk-amd64 #1 Debian 3.18.5-1~exp1
[ 6.037529] Hardware name: Supermicro X8SIE/X8SIE, BIOS 1.2a 06/27/2012
[ 6.044715] task: ffff88032fb6cc40 ti: ffff88032fb78000 task.ti: ffff88032fb78000
[ 6.052258] RIP: 0010:[<ffffffff810886fc>] [<ffffffff810886fc>] kthread_data+0xc/0x20
[ 6.060288] RSP: 0018:ffff88032fb7b540 EFLAGS: 00010002
[ 6.065644] RAX: 0000000000000000 RBX: 0000000000000003 RCX: 0000000000000000
[ 6.072827] RDX: 000000000000000f RSI: 0000000000000003 RDI: ffff88032fb6cc40
[ 6.080009] RBP: ffff88032fb6cc40 R08: 0000000000000000 R09: 000000000000001f
[ 6.087192] R10: ffffffff81afdec0 R11: 000000000000001a R12: ffff88033fcd3740
[ 6.094374] R13: ffff88032fb6d0a8 R14: 0000000000000003 R15: 0000000000030001
[ 6.101560] FS: 0000000000000000(0000) GS:ffff88033fcc0000(0000) knlGS:0000000000000000
[ 6.109712] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
[ 6.115507] CR2: 0000000000000028 CR3: 0000000001813000 CR4: 00000000000007e0
[ 6.122690] Stack:
[ 6.124753] ffffffff8108383d 0000000000000000 ffffffff81540fbf 0000000000000000
[ 6.132422] ffffffff8106c24e 0000000000013740 ffff88032fb7bfd8 0000000000013740
[ 6.140090] ffff88032fb6cc40 0000000000000008 ffff88032fb6d2c8 ffff88032fb6cc30
[ 6.147749] Call Trace:
[ 6.150244] [<ffffffff8108383d>] ? wq_worker_sleeping+0xd/0xa0
[ 6.156215] [<ffffffff81540fbf>] ? __schedule+0x66f/0x7f0
[ 6.161751] [<ffffffff8106c24e>] ? release_task+0x36e/0x470
[ 6.167460] [<ffffffff8106dc67>] ? do_exit+0x857/0xad0
[ 6.172739] [<ffffffff810166b7>] ? oops_end+0x97/0xe0
[ 6.177928] [<ffffffff8105a0f6>] ? no_context+0x106/0x380
[ 6.183463] [<ffffffff8105a8c6>] ? __do_page_fault+0xb6/0x530
[ 6.189349] [<ffffffff812c7c20>] ? cpumask_next_and+0x30/0x50
[ 6.195230] [<ffffffff8109fca2>] ? load_balance+0x1f2/0x880
[ 6.200941] [<ffffffff8101c206>] ? native_sched_clock+0x26/0x90
[ 6.206997] [<ffffffff815474e8>] ? page_fault+0x28/0x30
[ 6.212361] [<ffffffffa00cd7ed>] ? mvs_task_prep+0x78d/0xe40 [mvsas]
[ 6.218850] [<ffffffffa00cde25>] ? mvs_task_prep+0xdc5/0xe40 [mvsas]
[ 6.225340] [<ffffffff8109904a>] ? select_idle_sibling+0x10a/0x120
[ 6.231657] [<ffffffffa00cdef7>] ? mvs_task_exec.isra.13+0x57/0xd0 [mvsas]
[ 6.238667] [<ffffffffa00ce9f9>] ? mvs_queue_command+0x2e9/0x2f0 [mvsas]
[ 6.245502] [<ffffffffa017a7cc>] ? sas_ata_qc_issue+0x22c/0x270 [libsas]
[ 6.252340] [<ffffffffa028e845>] ? ata_qc_issue+0x1f5/0x380 [libata]
[ 6.258830] [<ffffffffa028ecbf>] ? ata_exec_internal_sg+0x2ef/0x5f0 [libata]
[ 6.266013] [<ffffffffa028f018>] ? ata_exec_internal+0x58/0xa0 [libata]
[ 6.272759] [<ffffffffa017a4cd>] ? sas_ata_hard_reset+0x9d/0x140 [libsas]
[ 6.279685] [<ffffffffa0291f35>] ? ata_phys_link_offline+0x15/0x30 [libata]
[ 6.286788] [<ffffffffa028f3b5>] ? ata_dev_read_id+0x205/0x5c0 [libata]
[ 6.293537] [<ffffffffa017a430>] ? sas_ata_printk+0x90/0x90 [libsas]
[ 6.300028] [<ffffffffa02996c1>] ? ata_eh_schedule_probe+0x141/0x180 [libata]
[ 6.307315] [<ffffffffa029c5da>] ? ata_eh_recover+0x7da/0x1370 [libata]
[ 6.314065] [<ffffffffa0291f50>] ? ata_phys_link_offline+0x30/0x30 [libata]
[ 6.321161] [<ffffffffa017a430>] ? sas_ata_printk+0x90/0x90 [libsas]
[ 6.327651] [<ffffffffa0291dc0>] ? ata_phys_link_online+0x30/0x30 [libata]
[ 6.334659] [<ffffffffa017a430>] ? sas_ata_printk+0x90/0x90 [libsas]
[ 6.341150] [<ffffffffa0291f50>] ? ata_phys_link_offline+0x30/0x30 [libata]
[ 6.348246] [<ffffffffa017a430>] ? sas_ata_printk+0x90/0x90 [libsas]
[ 6.354736] [<ffffffffa029db06>] ? ata_do_eh+0x46/0xb0 [libata]
[ 6.360792] [<ffffffffa0291dc0>] ? ata_phys_link_online+0x30/0x30 [libata]
[ 6.367801] [<ffffffff8101c206>] ? native_sched_clock+0x26/0x90
[ 6.373859] [<ffffffffa029d698>] ? ata_scsi_port_error_handler+0x448/0x7b0 [libata]
[ 6.381665] [<ffffffffa017abb6>] ? async_sas_ata_eh+0x46/0x70 [libsas]
[ 6.388327] [<ffffffff8108a973>] ? async_run_entry_fn+0x43/0x160
[ 6.394468] [<ffffffff81082a8e>] ? process_one_work+0x14e/0x3f0
[ 6.400527] [<ffffffff8108338b>] ? worker_thread+0x6b/0x4a0
[ 6.406236] [<ffffffff81083320>] ? rescuer_thread+0x290/0x290
[ 6.412119] [<ffffffff810880e3>] ? kthread+0xd3/0xf0
[ 6.417221] [<ffffffff81088010>] ? kthread_create_on_node+0x180/0x180
[ 6.423798] [<ffffffff8154543c>] ? ret_from_fork+0x7c/0xb0
[ 6.429421] [<ffffffff81088010>] ? kthread_create_on_node+0x180/0x180
[ 6.435995] Code: 10 04 00 00 48 8b 40 c8 48 c1 e8 02 83 e0 01 c3 66 66 66 66 66 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 48 8b 87 10 04 00 00 <48> 8b 40 d8 c3 66 66 66 66 66 66 2e 0f 1f 84 00 00 00 00 00 66
[ 6.458265] RIP [<ffffffff810886fc>] kthread_data+0xc/0x20
[ 6.463931] RSP <ffff88032fb7b540>
[ 6.467466] CR2: ffffffffffffffd8
[ 6.470827] ---[ end trace d307165136b3807e ]---
[ 6.475490] Fixing recursive fault but reboot is needed!
[-- Attachment #4: 3.19.0-kernel-panic --]
[-- Type: application/octet-stream, Size: 28319 bytes --]
[ 5.022094] scsi host4: mvsas
[ 5.162509] floppy0: no floppy controllers found
[ 5.286606] BUG: unable to handle kernel paging request at 0000009a2484b963
[ 5.293727] IP: [<ffffffffa00f1877>] mvs_task_exec.isra.13+0x827/0xf10 [mvsas]
[ 5.301066] PGD 0
[ 5.303181] Oops: 0000 [#1] SMP
[ 5.306560] Modules linked in: usb_storage hid_generic usbhid hid ata_generic mvsas ehci_pci libsas ata_piix ehci_hcd libata e1000e scsi_transport_sas ptp crc32c_intel pps_core usbcore scsi_mod usb_common
[ 5.325822] CPU: 7 PID: 146 Comm: kworker/u16:6 Not tainted 3.19.0-trunk-amd64 #1 Debian 3.19.3-1~exp1
[ 5.335188] Hardware name: Supermicro X8SIE/X8SIE, BIOS 1.2a 06/27/2012
[ 5.342373] Workqueue: events_unbound async_run_entry_fn
[ 5.347778] task: ffff88033113e450 ti: ffff88032d868000 task.ti: ffff88032d868000
[ 5.356537] RIP: 0010:[<ffffffffa00f1877>] [<ffffffffa00f1877>] mvs_task_exec.isra.13+0x827/0xf10 [mvsas]
[ 5.366310] RSP: 0018:ffff88032d86b948 EFLAGS: 00010097
[ 5.371673] RAX: 000000000000002d RBX: ffff88032d8037b0 RCX: 0000009a2484b70f
[ 5.378855] RDX: 000000000000000b RSI: 0000000000001000 RDI: ffff88032d800000
[ 5.386040] RBP: 0000000000000001 R08: ffff88032d8037b0 R09: ffff88032e173200
[ 5.393221] R10: ffff8800ba230040 R11: 0000000000010000 R12: ffff88032dc55c00
[ 5.400404] R13: ffff88032d800000 R14: 0000000000000096 R15: ffff88032d800008
[ 5.407588] FS: 0000000000000000(0000) GS:ffff88033fdc0000(0000) knlGS:0000000000000000
[ 5.415740] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
[ 5.421537] CR2: 0000009a2484b963 CR3: 0000000001813000 CR4: 00000000000007e0
[ 5.428721] Stack:
[ 5.430782] 0000000003aa7d20 ffff88032d8037b0 ffff88032d801e38 ffff8800ba230040
[ 5.438442] 0000000000000001 ffff88032d802678 ffff88032e173200 0000009a2484b70f
[ 5.446100] ffff88032d8813f0 0000000000000286 ffff88032d8813f0 0000000000000286
[ 5.453760] Call Trace:
[ 5.456260] [<ffffffffa00d8c13>] ? sas_ata_qc_issue+0x1a3/0x250 [libsas]
[ 5.463104] [<ffffffffa0259875>] ? ata_qc_issue+0x1f5/0x380 [libata]
[ 5.469594] [<ffffffffa0259cef>] ? ata_exec_internal_sg+0x2ef/0x5f0 [libata]
[ 5.476784] [<ffffffffa025a048>] ? ata_exec_internal+0x58/0xa0 [libata]
[ 5.483533] [<ffffffffa00d89f2>] ? sas_ata_hard_reset+0xa2/0x120 [libsas]
[ 5.490456] [<ffffffffa025cfa5>] ? ata_phys_link_offline+0x15/0x30 [libata]
[ 5.497551] [<ffffffffa00d8950>] ? sas_ata_printk+0x90/0x90 [libsas]
[ 5.504042] [<ffffffffa025a525>] ? ata_dev_read_id+0x345/0x5d0 [libata]
[ 5.510792] [<ffffffffa00d8950>] ? sas_ata_printk+0x90/0x90 [libsas]
[ 5.517283] [<ffffffffa02647f1>] ? ata_eh_schedule_probe+0x141/0x180 [libata]
[ 5.524570] [<ffffffffa02676ef>] ? ata_eh_recover+0x7df/0x13b0 [libata]
[ 5.531318] [<ffffffffa025cfc0>] ? ata_phys_link_offline+0x30/0x30 [libata]
[ 5.538412] [<ffffffffa00d8950>] ? sas_ata_printk+0x90/0x90 [libsas]
[ 5.544904] [<ffffffffa025ce30>] ? ata_phys_link_online+0x30/0x30 [libata]
[ 5.551912] [<ffffffffa00d8950>] ? sas_ata_printk+0x90/0x90 [libsas]
[ 5.558404] [<ffffffffa025cfc0>] ? ata_phys_link_offline+0x30/0x30 [libata]
[ 5.565497] [<ffffffffa00d8950>] ? sas_ata_printk+0x90/0x90 [libsas]
[ 5.571990] [<ffffffffa0268c56>] ? ata_do_eh+0x46/0xb0 [libata]
[ 5.578046] [<ffffffffa025ce30>] ? ata_phys_link_online+0x30/0x30 [libata]
[ 5.585056] [<ffffffff8101d146>] ? native_sched_clock+0x26/0x90
[ 5.591112] [<ffffffffa02687e8>] ? ata_scsi_port_error_handler+0x448/0x7b0 [libata]
[ 5.598917] [<ffffffffa00d9046>] ? async_sas_ata_eh+0x46/0x70 [libsas]
[ 5.605579] [<ffffffff8108ca33>] ? async_run_entry_fn+0x43/0x160
[ 5.611725] [<ffffffff810849cd>] ? process_one_work+0x14d/0x3f0
[ 5.617778] [<ffffffff810853fb>] ? worker_thread+0x6b/0x4a0
[ 5.623488] [<ffffffff81085390>] ? rescuer_thread+0x310/0x310
[ 5.629371] [<ffffffff8108a1a3>] ? kthread+0xd3/0xf0
[ 5.634475] [<ffffffff8108a0d0>] ? kthread_create_on_node+0x180/0x180
[ 5.641054] [<ffffffff815509d8>] ? ret_from_fork+0x58/0x90
[ 5.646674] [<ffffffff8108a0d0>] ? kthread_create_on_node+0x180/0x180
[ 5.653251] Code: 8b 80 08 01 00 00 48 8b 0c d0 0f 84 e6 05 00 00 41 8b 45 58 48 8d 54 ad 00 be 00 10 00 00 48 8d 54 55 00 41 89 84 d5 d4 55 02 00 <8b> 89 54 02 00 00 49 8b 55 48 d3 e6 41 0f b6 88 84 00 00 00 81
[ 5.675544] RIP [<ffffffffa00f1877>] mvs_task_exec.isra.13+0x827/0xf10 [mvsas]
[ 5.682970] RSP <ffff88032d86b948>
[ 5.686505] CR2: 0000009a2484b963
[ 5.689867] ---[ end trace 50dd3aeb0a903cc2 ]---
[ 5.694551] BUG: unable to handle kernel paging request at ffffffffffffffd8
[ 5.701667] IP: [<ffffffff8108a7bc>] kthread_data+0xc/0x20
[ 5.707247] PGD 1816067 PUD 1818067 PMD 0
[ 5.711554] Oops: 0000 [#2] SMP
[ 5.714941] Modules linked in: usb_storage hid_generic usbhid hid ata_generic mvsas ehci_pci libsas ata_piix ehci_hcd libata e1000e scsi_transport_sas ptp crc32c_intel pps_core usbcore scsi_mod usb_common
[ 5.734195] CPU: 7 PID: 146 Comm: kworker/u16:6 Tainted: G D 3.19.0-trunk-amd64 #1 Debian 3.19.3-1~exp1
[ 5.744695] Hardware name: Supermicro X8SIE/X8SIE, BIOS 1.2a 06/27/2012
[ 5.751883] task: ffff88033113e450 ti: ffff88032d868000 task.ti: ffff88032d868000
[ 5.759424] RIP: 0010:[<ffffffff8108a7bc>] [<ffffffff8108a7bc>] kthread_data+0xc/0x20
[ 5.767455] RSP: 0018:ffff88032d86b600 EFLAGS: 00010002
[ 5.772820] RAX: 0000000000000000 RBX: 0000000000000007 RCX: 0000000000000000
[ 5.780003] RDX: 000000000000000f RSI: 0000000000000007 RDI: ffff88033113e450
[ 5.787185] RBP: ffff88033113e450 R08: 0000000000000000 R09: 000000000000001f
[ 5.794367] R10: ffffffff81af9330 R11: 000000000000001a R12: 0000000000000000
[ 5.801551] R13: ffff88033113e8b8 R14: 0000000000000007 R15: ffff880331601230
[ 5.808733] FS: 0000000000000000(0000) GS:ffff88033fdc0000(0000) knlGS:0000000000000000
[ 5.816886] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
[ 5.822684] CR2: 0000000000000028 CR3: 0000000001813000 CR4: 00000000000007e0
[ 5.829865] Stack:
[ 5.831928] ffffffff810858ad ffff88033fdd4140 ffffffff8154c701 0000000000000000
[ 5.839588] ffffffff8106e1dd 0000000000014140 ffff88032d86bfd8 0000000000014140
[ 5.847256] ffff88033113e450 0000000000000008 ffff88033113e450 ffff88033113ead8
[ 5.854924] Call Trace:
[ 5.857420] [<ffffffff810858ad>] ? wq_worker_sleeping+0xd/0xa0
[ 5.863391] [<ffffffff8154c701>] ? __schedule+0x651/0x800
[ 5.868927] [<ffffffff8106e1dd>] ? release_task+0x36d/0x460
[ 5.874635] [<ffffffff8106fab2>] ? do_exit+0x782/0xad0
[ 5.879913] [<ffffffff81017636>] ? oops_end+0x96/0xe0
[ 5.885105] [<ffffffff8105c1b6>] ? no_context+0x106/0x380
[ 5.890639] [<ffffffff8105c90e>] ? __do_page_fault+0xae/0x550
[ 5.896524] [<ffffffff810a3350>] ? pick_next_task_fair+0x1b0/0x850
[ 5.902840] [<ffffffff810125eb>] ? __switch_to+0x14b/0x5d0
[ 5.908464] [<ffffffff81552be8>] ? page_fault+0x28/0x30
[ 5.913826] [<ffffffffa00f1877>] ? mvs_task_exec.isra.13+0x827/0xf10 [mvsas]
[ 5.921010] [<ffffffffa00f1e71>] ? mvs_task_exec.isra.13+0xe21/0xf10 [mvsas]
[ 5.928192] [<ffffffffa00d8c13>] ? sas_ata_qc_issue+0x1a3/0x250 [libsas]
[ 5.935029] [<ffffffffa0259875>] ? ata_qc_issue+0x1f5/0x380 [libata]
[ 5.941518] [<ffffffffa0259cef>] ? ata_exec_internal_sg+0x2ef/0x5f0 [libata]
[ 5.948702] [<ffffffffa025a048>] ? ata_exec_internal+0x58/0xa0 [libata]
[ 5.955449] [<ffffffffa00d89f2>] ? sas_ata_hard_reset+0xa2/0x120 [libsas]
[ 5.962375] [<ffffffffa025cfa5>] ? ata_phys_link_offline+0x15/0x30 [libata]
[ 5.969477] [<ffffffffa00d8950>] ? sas_ata_printk+0x90/0x90 [libsas]
[ 5.975967] [<ffffffffa025a525>] ? ata_dev_read_id+0x345/0x5d0 [libata]
[ 5.982716] [<ffffffffa00d8950>] ? sas_ata_printk+0x90/0x90 [libsas]
[ 5.989208] [<ffffffffa02647f1>] ? ata_eh_schedule_probe+0x141/0x180 [libata]
[ 5.996496] [<ffffffffa02676ef>] ? ata_eh_recover+0x7df/0x13b0 [libata]
[ 6.003245] [<ffffffffa025cfc0>] ? ata_phys_link_offline+0x30/0x30 [libata]
[ 6.010338] [<ffffffffa00d8950>] ? sas_ata_printk+0x90/0x90 [libsas]
[ 6.016829] [<ffffffffa025ce30>] ? ata_phys_link_online+0x30/0x30 [libata]
[ 6.023838] [<ffffffffa00d8950>] ? sas_ata_printk+0x90/0x90 [libsas]
[ 6.030329] [<ffffffffa025cfc0>] ? ata_phys_link_offline+0x30/0x30 [libata]
[ 6.037425] [<ffffffffa00d8950>] ? sas_ata_printk+0x90/0x90 [libsas]
[ 6.043915] [<ffffffffa0268c56>] ? ata_do_eh+0x46/0xb0 [libata]
[ 6.049973] [<ffffffffa025ce30>] ? ata_phys_link_online+0x30/0x30 [libata]
[ 6.056980] [<ffffffff8101d146>] ? native_sched_clock+0x26/0x90
[ 6.063038] [<ffffffffa02687e8>] ? ata_scsi_port_error_handler+0x448/0x7b0 [libata]
[ 6.070843] [<ffffffffa00d9046>] ? async_sas_ata_eh+0x46/0x70 [libsas]
[ 6.077506] [<ffffffff8108ca33>] ? async_run_entry_fn+0x43/0x160
[ 6.083648] [<ffffffff810849cd>] ? process_one_work+0x14d/0x3f0
[ 6.089705] [<ffffffff810853fb>] ? worker_thread+0x6b/0x4a0
[ 6.095414] [<ffffffff81085390>] ? rescuer_thread+0x310/0x310
[ 6.101297] [<ffffffff8108a1a3>] ? kthread+0xd3/0xf0
[ 6.106402] [<ffffffff8108a0d0>] ? kthread_create_on_node+0x180/0x180
[ 6.112976] [<ffffffff815509d8>] ? ret_from_fork+0x58/0x90
[ 6.118601] [<ffffffff8108a0d0>] ? kthread_create_on_node+0x180/0x180
[ 6.125175] Code: 10 04 00 00 48 8b 40 c8 48 c1 e8 02 83 e0 01 c3 66 66 66 66 66 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 48 8b 87 10 04 00 00 <48> 8b 40 d8 c3 66 66 66 66 66 66 2e 0f 1f 84 00 00 00 00 00 66
[ 6.147461] RIP [<ffffffff8108a7bc>] kthread_data+0xc/0x20
[ 6.153128] RSP <ffff88032d86b600>
[ 6.156662] CR2: ffffffffffffffd8
[ 6.160024] ---[ end trace 50dd3aeb0a903cc3 ]---
[ 6.163260] scsi 5:0:0:0: Direct-Access Generic USB EDC 1.00 PQ: 0 ANSI: 2
[ 6.172847] Fixing recursive fault but reboot is needed!
[ 16.487514] random: nonblocking pool is initialized
[ 28.480381] INFO: rcu_sched detected stalls on CPUs/tasks: { 3 7} (detected by 2, t=5581 jiffies, g=-188, c=-189, q=50)
[ 28.491481] Task dump for CPU 3:
[ 28.494765] swapper/3 R running task 0 0 1 0x00000008
[ 28.501963] 000038ad1b4fe8cc ffffffffffffffff ffffffffffffff4e ffffffff8142366e
[ 28.509633] 0000000000000010 0000000000000217 ffff880331677e88 0000000000000018
[ 28.517301] ffffffff8142365e 0000000000000092 ffffffff818f60c0 ffffe8ffffcc6de0
[ 28.524969] Call Trace:
[ 28.527466] [<ffffffff8142366e>] ? cpuidle_enter_state+0x5e/0x160
[ 28.533693] [<ffffffff8142365e>] ? cpuidle_enter_state+0x4e/0x160
[ 28.539924] [<ffffffff810a936d>] ? cpu_startup_entry+0x34d/0x3f0
[ 28.546068] [<ffffffff810d9cc0>] ? tick_check_new_device+0xe0/0x110
[ 28.552472] [<ffffffff8104642e>] ? start_secondary+0x19e/0x1d0
[ 28.558438] Task dump for CPU 7:
[ 28.561715] kworker/u16:6 D ffff88032d800008 0 146 0 0x00000000
[ 28.568920] 0000000000000096 ffff88032d800000 ffff88032dc55c00 0000000000000001
[ 28.576582] ffff88032d8037b0 0000000000010000 ffff8800ba230040 ffff88032e173200
[ 28.584250] ffff88032d8037b0 000000000000002d 0000009a2484b70f 000000000000000b
[ 28.591919] Call Trace:
[ 28.594417] [<ffffffffa00f1877>] ? mvs_task_exec.isra.13+0x827/0xf10 [mvsas]
[ 28.601599] [<ffffffffa00f1e71>] ? mvs_task_exec.isra.13+0xe21/0xf10 [mvsas]
[ 28.608782] [<ffffffffa00d8c13>] ? sas_ata_qc_issue+0x1a3/0x250 [libsas]
[ 28.615618] [<ffffffffa0259875>] ? ata_qc_issue+0x1f5/0x380 [libata]
[ 28.622107] [<ffffffffa0259cef>] ? ata_exec_internal_sg+0x2ef/0x5f0 [libata]
[ 28.629290] [<ffffffffa025a048>] ? ata_exec_internal+0x58/0xa0 [libata]
[ 28.636040] [<ffffffffa00d89f2>] ? sas_ata_hard_reset+0xa2/0x120 [libsas]
[ 28.642962] [<ffffffffa025cfa5>] ? ata_phys_link_offline+0x15/0x30 [libata]
[ 28.651367] [<ffffffffa00d8950>] ? sas_ata_printk+0x90/0x90 [libsas]
[ 28.657858] [<ffffffffa025a525>] ? ata_dev_read_id+0x345/0x5d0 [libata]
[ 28.664607] [<ffffffffa00d8950>] ? sas_ata_printk+0x90/0x90 [libsas]
[ 28.671098] [<ffffffffa02647f1>] ? ata_eh_schedule_probe+0x141/0x180 [libata]
[ 28.678384] [<ffffffffa02676ef>] ? ata_eh_recover+0x7df/0x13b0 [libata]
[ 28.685134] [<ffffffffa025cfc0>] ? ata_phys_link_offline+0x30/0x30 [libata]
[ 28.692227] [<ffffffffa00d8950>] ? sas_ata_printk+0x90/0x90 [libsas]
[ 28.698719] [<ffffffffa025ce30>] ? ata_phys_link_online+0x30/0x30 [libata]
[ 28.705727] [<ffffffffa00d8950>] ? sas_ata_printk+0x90/0x90 [libsas]
[ 28.712217] [<ffffffffa025cfc0>] ? ata_phys_link_offline+0x30/0x30 [libata]
[ 28.719314] [<ffffffffa00d8950>] ? sas_ata_printk+0x90/0x90 [libsas]
[ 28.725803] [<ffffffffa0268c56>] ? ata_do_eh+0x46/0xb0 [libata]
[ 28.731859] [<ffffffffa025ce30>] ? ata_phys_link_online+0x30/0x30 [libata]
[ 28.738868] [<ffffffff8101d146>] ? native_sched_clock+0x26/0x90
[ 28.744928] [<ffffffffa02687e8>] ? ata_scsi_port_error_handler+0x448/0x7b0 [libata]
[ 28.752732] [<ffffffffa00d9046>] ? async_sas_ata_eh+0x46/0x70 [libsas]
[ 28.759394] [<ffffffff8108ca33>] ? async_run_entry_fn+0x43/0x160
[ 28.765537] [<ffffffff810849cd>] ? process_one_work+0x14d/0x3f0
[ 28.771593] [<ffffffff810853fb>] ? worker_thread+0x6b/0x4a0
[ 28.777304] [<ffffffff81085390>] ? rescuer_thread+0x310/0x310
[ 28.783186] [<ffffffff8108a1a3>] ? kthread+0xd3/0xf0
[ 28.788290] [<ffffffff8108a0d0>] ? kthread_create_on_node+0x180/0x180
[ 28.794867] [<ffffffff815509d8>] ? ret_from_fork+0x58/0x90
[ 28.800488] [<ffffffff8108a0d0>] ? kthread_create_on_node+0x180/0x180
[ 35.049176] ------------[ cut here ]------------
[ 35.053847] WARNING: CPU: 7 PID: 146 at /build/linux-gzeV2n/linux-3.19.3/kernel/watchdog.c:290 watchdog_overflow_callback+0x92/0xc0()
[ 35.065906] Watchdog detected hard LOCKUP on cpu 7
[ 35.070560] Modules linked in: usb_storage hid_generic usbhid hid ata_generic mvsas ehci_pci libsas ata_piix ehci_hcd libata e1000e scsi_transport_sas ptp crc32c_intel pps_core usbcore scsi_mod usb_common
[ 35.090107] CPU: 7 PID: 146 Comm: kworker/u16:6 Tainted: G D 3.19.0-trunk-amd64 #1 Debian 3.19.3-1~exp1
[ 35.100607] Hardware name: Supermicro X8SIE/X8SIE, BIOS 1.2a 06/27/2012
[ 35.107796] 0000000000000000 ffffffff81723ea8 ffffffff8154b001 ffff88033fdc5b60
[ 35.115483] ffffffff8106ced1 ffff880331146800[ 35.118656] ------------[ cut here ]------------
[ 35.118658] WARNING: CPU: 3 PID: 0 at /build/linux-gzeV2n/linux-3.19.3/kernel/watchdog.c:290 watchdog_overflow_callback+0x92/0xc0()
[ 35.118659] Watchdog detected hard LOCKUP on cpu 3Modules linked in: usb_storage hid_generic usbhid hid ata_generic mvsas ehci_pci libsas ata_piix ehci_hcd libata e1000e scsi_transport_sas ptp crc32c_intel pps_core usbcore scsi_mod usb_common
[ 35.159023] 0000000000000000 ffff88033fdc5c80
[ 35.163866] ffff88033fdc5ef8 0000000000000000 ffffffff8106cf4a ffffffff81723e80
[ 35.171536] Call Trace:
[ 35.174029] <NMI> [<ffffffff8154b001>] ? dump_stack+0x40/0x50
[ 35.180089] [<ffffffff8106ced1>] ? warn_slowpath_common+0x81/0xb0
[ 35.186316] [<ffffffff8106cf4a>] ? warn_slowpath_fmt+0x4a/0x50
[ 35.192287] [<ffffffff81109cd2>] ? watchdog_overflow_callback+0x92/0xc0
[ 35.199037] [<ffffffff81145e16>] ? __perf_event_overflow+0x86/0x230
[ 35.205442] [<ffffffff8102a689>] ? x86_perf_event_set_period+0xc9/0x170
[ 35.212190] [<ffffffff810319da>] ? intel_pmu_handle_irq+0x1ba/0x3a0
[ 35.218623] [<ffffffff8102962a>] ? perf_event_nmi_handler+0x2a/0x50
[ 35.225021] [<ffffffff8101d146>] ? native_sched_clock+0x26/0x90
[ 35.231078] [<ffffffff81017e16>] ? nmi_handle+0x86/0x120
[ 35.236527] [<ffffffff81018370>] ? default_do_nmi+0x40/0x110
[ 35.242325] [<ffffffff810184c8>] ? do_nmi+0x88/0xd0
[ 35.247341] [<ffffffff81552f61>] ? end_repeat_nmi+0x1e/0x2e
[ 35.253051] [<ffffffff8106fc06>] ? do_exit+0x8d6/0xad0
[ 35.258329] [<ffffffff81550512>] ? _raw_spin_lock_irq+0x32/0x40
[ 35.264383] [<ffffffff81550512>] ? _raw_spin_lock_irq+0x32/0x40
[ 35.270440] [<ffffffff81550512>] ? _raw_spin_lock_irq+0x32/0x40
[ 35.276495] <<EOE>> [<ffffffff8154c153>] ? __schedule+0xa3/0x800
[ 35.282812] [<ffffffff81549e3c>] ? printk+0x4d/0x52
[ 35.287831] [<ffffffff8106fc06>] ? do_exit+0x8d6/0xad0
[ 35.293107] [<ffffffff81017636>] ? oops_end+0x96/0xe0
[ 35.298297] [<ffffffff8105c1b6>] ? no_context+0x106/0x380
[ 35.303835] [<ffffffff8105c90e>] ? __do_page_fault+0xae/0x550
[ 35.309718] [<ffffffff812d7fa5>] ? vsnprintf+0x295/0x5d0
[ 35.315165] [<ffffffff812d80df>] ? vsnprintf+0x3cf/0x5d0
[ 35.320616] [<ffffffff812d92c3>] ? delay_tsc+0x33/0x60
[ 35.325892] [<ffffffff81552be8>] ? page_fault+0x28/0x30
[ 35.331257] [<ffffffff8108a7bc>] ? kthread_data+0xc/0x20
[ 35.336706] [<ffffffff810858ad>] ? wq_worker_sleeping+0xd/0xa0
[ 35.342675] [<ffffffff8154c701>] ? __schedule+0x651/0x800
[ 35.348211] [<ffffffff8106e1dd>] ? release_task+0x36d/0x460
[ 35.353923] [<ffffffff8106fab2>] ? do_exit+0x782/0xad0
[ 35.359199] [<ffffffff81017636>] ? oops_end+0x96/0xe0
[ 35.364390] [<ffffffff8105c1b6>] ? no_context+0x106/0x380
[ 35.369925] [<ffffffff8105c90e>] ? __do_page_fault+0xae/0x550
[ 35.375808] [<ffffffff810a3350>] ? pick_next_task_fair+0x1b0/0x850
[ 35.382126] [<ffffffff810125eb>] ? __switch_to+0x14b/0x5d0
[ 35.387749] [<ffffffff81552be8>] ? page_fault+0x28/0x30
[ 35.393114] [<ffffffffa00f1877>] ? mvs_task_exec.isra.13+0x827/0xf10 [mvsas]
[ 35.400296] [<ffffffffa00f1e71>] ? mvs_task_exec.isra.13+0xe21/0xf10 [mvsas]
[ 35.407478] [<ffffffffa00d8c13>] ? sas_ata_qc_issue+0x1a3/0x250 [libsas]
[ 35.414314] [<ffffffffa0259875>] ? ata_qc_issue+0x1f5/0x380 [libata]
[ 35.420805] [<ffffffffa0259cef>] ? ata_exec_internal_sg+0x2ef/0x5f0 [libata]
[ 35.427987] [<ffffffffa025a048>] ? ata_exec_internal+0x58/0xa0 [libata]
[ 35.434737] [<ffffffffa00d89f2>] ? sas_ata_hard_reset+0xa2/0x120 [libsas]
[ 35.441662] [<ffffffffa025cfa5>] ? ata_phys_link_offline+0x15/0x30 [libata]
[ 35.448764] [<ffffffffa00d8950>] ? sas_ata_printk+0x90/0x90 [libsas]
[ 35.455256] [<ffffffffa025a525>] ? ata_dev_read_id+0x345/0x5d0 [libata]
[ 35.462002] [<ffffffffa00d8950>] ? sas_ata_printk+0x90/0x90 [libsas]
[ 35.468494] [<ffffffffa02647f1>] ? ata_eh_schedule_probe+0x141/0x180 [libata]
[ 35.475781] [<ffffffffa02676ef>] ? ata_eh_recover+0x7df/0x13b0 [libata]
[ 35.482556] [<ffffffffa025cfc0>] ? ata_phys_link_offline+0x30/0x30 [libata]
[ 35.489652] [<ffffffffa00d8950>] ? sas_ata_printk+0x90/0x90 [libsas]
[ 35.496143] [<ffffffffa025ce30>] ? ata_phys_link_online+0x30/0x30 [libata]
[ 35.503149] [<ffffffffa00d8950>] ? sas_ata_printk+0x90/0x90 [libsas]
[ 35.509641] [<ffffffffa025cfc0>] ? ata_phys_link_offline+0x30/0x30 [libata]
[ 35.516736] [<ffffffffa00d8950>] ? sas_ata_printk+0x90/0x90 [libsas]
[ 35.523227] [<ffffffffa0268c56>] ? ata_do_eh+0x46/0xb0 [libata]
[ 35.529284] [<ffffffffa025ce30>] ? ata_phys_link_online+0x30/0x30 [libata]
[ 35.536290] [<ffffffff8101d146>] ? native_sched_clock+0x26/0x90
[ 35.542350] [<ffffffffa02687e8>] ? ata_scsi_port_error_handler+0x448/0x7b0 [libata]
[ 35.550155] [<ffffffffa00d9046>] ? async_sas_ata_eh+0x46/0x70 [libsas]
[ 35.556819] [<ffffffff8108ca33>] ? async_run_entry_fn+0x43/0x160
[ 35.562960] [<ffffffff810849cd>] ? process_one_work+0x14d/0x3f0
[ 35.569016] [<ffffffff810853fb>] ? worker_thread+0x6b/0x4a0
[ 35.574726] [<ffffffff81085390>] ? rescuer_thread+0x310/0x310
[ 35.580610] [<ffffffff8108a1a3>] ? kthread+0xd3/0xf0
[ 35.585714] [<ffffffff8108a0d0>] ? kthread_create_on_node+0x180/0x180
[ 35.592289] [<ffffffff815509d8>] ? ret_from_fork+0x58/0x90
[ 35.597912] [<ffffffff8108a0d0>] ? kthread_create_on_node+0x180/0x180
[ 35.604488] ---[ end trace 50dd3aeb0a903cc4 ]---
[ 35.604489] CPU: 3 PID: 0 Comm: swapper/3 Tainted: G D 3.19.0-trunk-amd64 #1 Debian 3.19.3-1~exp1
[ 35.604490] Hardware name: Supermicro X8SIE/X8SIE, BIOS 1.2a 06/27/2012
[ 35.604492] 0000000000000000 ffffffff81723ea8 ffffffff8154b001 ffff88033fcc5b60
[ 35.604493] ffffffff8106ced1 ffff880331003800 0000000000000000 ffff88033fcc5c80
[ 35.604494] ffff88033fcc5ef8 0000000000000000 ffffffff8106cf4a ffffffff81723e80
[ 35.604494] Call Trace:
[ 35.604496] <NMI> [<ffffffff8154b001>] ? dump_stack+0x40/0x50
[ 35.604498] [<ffffffff8106ced1>] ? warn_slowpath_common+0x81/0xb0
[ 35.604499] [<ffffffff8106cf4a>] ? warn_slowpath_fmt+0x4a/0x50
[ 35.604500] [<ffffffff81109cd2>] ? watchdog_overflow_callback+0x92/0xc0
[ 35.604502] [<ffffffff81145e16>] ? __perf_event_overflow+0x86/0x230
[ 35.604503] [<ffffffff8102a689>] ? x86_perf_event_set_period+0xc9/0x170
[ 35.604504] [<ffffffff810319da>] ? intel_pmu_handle_irq+0x1ba/0x3a0
[ 35.604506] [<ffffffff8102962a>] ? perf_event_nmi_handler+0x2a/0x50
[ 35.604508] [<ffffffff8101d146>] ? native_sched_clock+0x26/0x90
[ 35.604509] [<ffffffff81017e16>] ? nmi_handle+0x86/0x120
[ 35.604510] [<ffffffff81018370>] ? default_do_nmi+0x40/0x110
[ 35.604511] [<ffffffff810184c8>] ? do_nmi+0x88/0xd0
[ 35.604513] [<ffffffff81552f61>] ? end_repeat_nmi+0x1e/0x2e
[ 35.604514] [<ffffffff815506d2>] ? _raw_spin_lock+0x32/0x40
[ 35.604516] [<ffffffff815506d2>] ? _raw_spin_lock+0x32/0x40
[ 35.604517] [<ffffffff815506d2>] ? _raw_spin_lock+0x32/0x40
[ 35.604523] <<EOE>> <IRQ> [<ffffffffa00f5310>] ? mvs_64xx_isr+0x30/0x100 [mvsas]
[ 35.604527] [<ffffffffa00ef15f>] ? mvs_interrupt+0x6f/0xa0 [mvsas]
[ 35.604528] [<ffffffff810bcd35>] ? handle_irq_event_percpu+0x75/0x190
[ 35.604530] [<ffffffff810bce88>] ? handle_irq_event+0x38/0x50
[ 35.604531] [<ffffffff810bfa94>] ? handle_fasteoi_irq+0x84/0x140
[ 35.604533] [<ffffffff8101648d>] ? handle_irq+0x1d/0x30
[ 35.604534] [<ffffffff81553986>] ? do_IRQ+0x46/0xe0
[ 35.604536] [<ffffffff8155182d>] ? common_interrupt+0x6d/0x6d
[ 35.604538] <EOI> [<ffffffff8142366e>] ? cpuidle_enter_state+0x5e/0x160
[ 35.604539] [<ffffffff8142365e>] ? cpuidle_enter_state+0x4e/0x160
[ 35.604540] [<ffffffff810a936d>] ? cpu_startup_entry+0x34d/0x3f0
[ 35.604542] [<ffffffff810d9cc0>] ? tick_check_new_device+0xe0/0x110
[ 35.604544] [<ffffffff8104642e>] ? start_secondary+0x19e/0x1d0
[ 35.604545] ---[ end trace 50dd3aeb0a903cc5 ]---
[ 38.645536] ------------[ cut here ]------------
[ 38.650204] WARNING: CPU: 1 PID: 0 at /build/linux-gzeV2n/linux-3.19.3/kernel/watchdog.c:290 watchdog_overflow_callback+0x92/0xc0()
[ 38.662090] Watchdog detected hard LOCKUP on cpu 1
[ 38.666742] Modules linked in: usb_storage hid_generic usbhid hid ata_generic mvsas ehci_pci libsas ata_piix ehci_hcd libata e1000e scsi_transport_sas ptp crc32c_intel pps_core usbcore scsi_mod usb_common
[ 38.686238] CPU: 1 PID: 0 Comm: swapper/1 Tainted: G D W 3.19.0-trunk-amd64 #1 Debian 3.19.3-1~exp1
[ 38.696210] Hardware name: Supermicro X8SIE/X8SIE, BIOS 1.2a 06/27/2012
[ 38.703392] 0000000000000000 ffffffff81723ea8 ffffffff8154b001 ffff88033fc45b60
[ 38.711052] ffffffff8106ced1 ffff880331631800 0000000000000001 ffff88033fc45c80
[ 38.718719] ffff88033fc45ef8 0000000000000000 ffffffff8106cf4a ffffffff81723e80
[ 38.727549] Call Trace:
[ 38.730045] <NMI> [<ffffffff8154b001>] ? dump_stack+0x40/0x50
[ 38.736103] [<ffffffff8106ced1>] ? warn_slowpath_common+0x81/0xb0
[ 38.742331] [<ffffffff8106cf4a>] ? warn_slowpath_fmt+0x4a/0x50
[ 38.748301] [<ffffffff81109cd2>] ? watchdog_overflow_callback+0x92/0xc0
[ 38.755051] [<ffffffff81145e16>] ? __perf_event_overflow+0x86/0x230
[ 38.761454] [<ffffffff8102a689>] ? x86_perf_event_set_period+0xc9/0x170
[ 38.768204] [<ffffffff810319da>] ? intel_pmu_handle_irq+0x1ba/0x3a0
[ 38.774608] [<ffffffff8102962a>] ? perf_event_nmi_handler+0x2a/0x50
[ 38.781009] [<ffffffff8101d146>] ? native_sched_clock+0x26/0x90
[ 38.787067] [<ffffffff81017e16>] ? nmi_handle+0x86/0x120
[ 38.792516] [<ffffffff81018370>] ? default_do_nmi+0x40/0x110
[ 38.798313] [<ffffffff810184c8>] ? do_nmi+0x88/0xd0
[ 38.803330] [<ffffffff81552f61>] ? end_repeat_nmi+0x1e/0x2e
[ 38.809038] [<ffffffff815506d7>] ? _raw_spin_lock+0x37/0x40
[ 38.814749] [<ffffffff815506d7>] ? _raw_spin_lock+0x37/0x40
[ 38.820457] [<ffffffff815506d7>] ? _raw_spin_lock+0x37/0x40
[ 38.826168] <<EOE>> <IRQ> [<ffffffff810a5272>] ? sched_rt_period_timer+0xe2/0x2e0
[ 38.834124] [<ffffffff810a5190>] ? put_prev_task_rt+0x50/0x50
[ 38.840007] [<ffffffff810cd50b>] ? __run_hrtimer+0x6b/0x1c0
[ 38.845714] [<ffffffff8101cc25>] ? read_tsc+0x5/0x10
[ 38.850810] [<ffffffff810cd919>] ? hrtimer_interrupt+0xf9/0x230
[ 38.856868] [<ffffffff81054506>] ? hpet_interrupt_handler+0x16/0x40
[ 38.863270] [<ffffffff810bcd35>] ? handle_irq_event_percpu+0x75/0x190
[ 38.869845] [<ffffffff810bce88>] ? handle_irq_event+0x38/0x50
[ 38.875730] [<ffffffff810bfd81>] ? handle_edge_irq+0x71/0x120
[ 38.881613] [<ffffffff8101648d>] ? handle_irq+0x1d/0x30
[ 38.887011] [<ffffffff81553986>] ? do_IRQ+0x46/0xe0
[ 38.892027] [<ffffffff8155182d>] ? common_interrupt+0x6d/0x6d
[ 38.897910] <EOI> [<ffffffff8142366e>] ? cpuidle_enter_state+0x5e/0x160
[ 38.904832] [<ffffffff8142365e>] ? cpuidle_enter_state+0x4e/0x160
[ 38.911064] [<ffffffff810a936d>] ? cpu_startup_entry+0x34d/0x3f0
[ 38.917207] [<ffffffff8104642e>] ? start_secondary+0x19e/0x1d0
[ 38.923175] ---[ end trace 50dd3aeb0a903cc6 ]---
[ 44.432170] NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [systemd-udevd:101]
[ 44.440098] Modules linked in: usb_storage hid_generic usbhid hid ata_generic mvsas ehci_pci libsas ata_piix ehci_hcd libata e1000e scsi_transport_sas ptp crc32c_intel pps_core usbcore scsi_mod usb_common
[ 44.459360] CPU: 0 PID: 101 Comm: systemd-udevd Tainted: G D W 3.19.0-trunk-amd64 #1 Debian 3.19.3-1~exp1
[ 44.469853] Hardware name: Supermicro X8SIE/X8SIE, BIOS 1.2a 06/27/2012
[ 44.477036] task: ffff8803312e60d0 ti: ffff88032e1f8000 task.ti: ffff88032e1f8000
[ 44.484582] RIP: 0010:[<ffffffff810e0ca2>] [<ffffffff810e0ca2>] smp_call_function_many+0x212/0x270
[ 44.493739] RSP: 0018:ffff88032e1fbaf8 EFLAGS: 00000202
[ 44.499102] RAX: ffff88033fcd7918 RBX: 00000000000000fe RCX: 0000000000000003
[ 44.506287] RDX: ffff88033fcd7918 RSI: 0000000000000200 RDI: 0000000000000000
[ 44.513468] RBP: ffff88033fc14f40 R08: ffff88033fc14f48 R09: 0000000000000000
[ 44.520651] R10: 0000000000000008 R11: 000000000000a44a R12: 0000000000000002
[ 44.527834] R13: 0000000000000000 R14: 0000000000000293 R15: ffff88033fff3d80
[ 44.535018] FS: 00007f89f9312880(0000) GS:ffff88033fc00000(0000) knlGS:0000000000000000
[ 44.543171] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 44.548966] CR2: 00007f89f9319000 CR3: 000000032e061000 CR4: 00000000000007f0
[ 44.556149] Stack:
[ 44.558211] 0000000200000141 0000000100000000 0000004000000040 ffff88032e1fbc40
[ 44.565871] ffffffff81062fa0 ffff88032e1fbb70 ffff88033fdd0428 ffff88032e1fbc48
[ 44.573530] ffff88032e1fbb18 ffffffff810e0e68 ffff88032e1fbc40 ffff88033fdd04c0
[ 44.581190] Call Trace:
[ 44.583686] [<ffffffff81062fa0>] ? rbt_memtype_copy_nth_element+0xa0/0xa0
[ 44.590609] [<ffffffff810e0e68>] ? on_each_cpu+0x28/0x50
[ 44.596060] [<ffffffff81063710>] ? flush_tlb_kernel_range+0x60/0x70
[ 44.602463] [<ffffffff8118a553>] ? __purge_vmap_area_lazy+0x3c3/0x430
[ 44.609040] [<ffffffff810e1f30>] ? mod_find_symname+0x90/0x90
[ 44.614921] [<ffffffff8118a72c>] ? vm_unmap_aliases+0x16c/0x180
[ 44.620981] [<ffffffff8105f5ad>] ? change_page_attr_set_clr+0xdd/0x450
[ 44.627641] [<ffffffff8105fdcd>] ? set_memory_ro+0x2d/0x40
[ 44.633263] [<ffffffff810e31da>] ? set_section_ro_nx+0x7a/0x90
[ 44.639235] [<ffffffff810e6430>] ? load_module+0x1a80/0x2740
[ 44.645032] [<ffffffff810e725d>] ? SyS_finit_module+0x7d/0xa0
[ 44.650914] [<ffffffff81550a8d>] ? system_call_fast_compare_end+0xc/0x11
[ 44.657749] Code: 81 00 89 c1 0f 8d 7c fe ff ff 48 98 48 8b 55 00 48 03 14 c5 40 4a 8f 81 f6 42 18 01 48 89 d0 74 c8 0f 1f 84 00 00 00 00 00 f3 90 <f6> 40 18 01 75 f8 eb b6 0f b6 4c 24 0c 48 83 c4 18 4c 89 ea 5b
^ permalink raw reply
* Re: [f2fs-dev] [PATCH 2/2][v2] blk-plug: don't flush nested plug lists
From: nick @ 2015-04-11 4:11 UTC (permalink / raw)
To: Jeff Moyer, Dave Chinner
Cc: Vladimir Davydov, linux-aio, Miklos Szeredi, Mike Snitzer,
Ming Lei, Ming Lei, Trond Myklebust, Jianyu Zhan,
Nicholas A. Bellinger, linux-kernel, Sagi Grimberg, Chris Mason,
dm-devel, target-devel, Andreas Dilger, Mikulas Patocka,
Mark Rustad, Christoph Hellwig, Alasdair Kergon, Matthew Wilcox,
linux-scsi, Namjae Jeon, linux-raid, cluster-devel, Mel Gorman
In-Reply-To: <x498udzlkkx.fsf@segfault.boston.devel.redhat.com>
On 2015-04-10 05:50 PM, Jeff Moyer wrote:
> Dave Chinner <david@fromorbit.com> writes:
>
>> On Tue, Apr 07, 2015 at 02:55:13PM -0400, Jeff Moyer wrote:
>>> The way the on-stack plugging currently works, each nesting level
>>> flushes its own list of I/Os. This can be less than optimal (read
>>> awful) for certain workloads. For example, consider an application
>>> that issues asynchronous O_DIRECT I/Os. It can send down a bunch of
>>> I/Os together in a single io_submit call, only to have each of them
>>> dispatched individually down in the bowels of the dirct I/O code.
>>> The reason is that there are blk_plug-s instantiated both at the upper
>>> call site in do_io_submit and down in do_direct_IO. The latter will
>>> submit as little as 1 I/O at a time (if you have a small enough I/O
>>> size) instead of performing the batching that the plugging
>>> infrastructure is supposed to provide.
>>
>> I'm wondering what impact this will have on filesystem metadata IO
>> that needs to be issued immediately. e.g. we are doing writeback, so
>> there is a high level plug in place and we need to page in btree
>> blocks to do extent allocation. We do readahead at this point,
>> but it looks like this change will prevent the readahead from being
>> issued by the unplug in xfs_buf_iosubmit().
>
> I'm not ignoring you, Dave, I'm just doing some more investigation and
> testing. It's taking longer than I had hoped.
>
> -Jeff
>
Jeff,
Would you mind sending your test reports to the list so we can see what workloads
and tests your running your patch under. This is due to me and the others perhaps
being able to give input into the other major benchmarks or workloads we need to
test too in order to see if there are any regressions with your patch.
Thanks,
Nick
> ------------------------------------------------------------------------------
> BPM Camp - Free Virtual Workshop May 6th at 10am PDT/1PM EDT
> Develop your own process in accordance with the BPMN 2 standard
> Learn Process modeling best practices with Bonita BPM through live exercises
> http://www.bonitasoft.com/be-part-of-it/events/bpm-camp-virtual- event?utm_
> source=Sourceforge_BPM_Camp_5_6_15&utm_medium=email&utm_campaign=VA_SF
> _______________________________________________
> Linux-f2fs-devel mailing list
> Linux-f2fs-devel@lists.sourceforge.net
> https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
>
--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to majordomo@kvack.org. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"aart@kvack.org">aart@kvack.org</a>
^ permalink raw reply
* Re: [PATCH 2/2][v2] blk-plug: don't flush nested plug lists
From: Jeff Moyer @ 2015-04-10 21:50 UTC (permalink / raw)
To: Dave Chinner
Cc: Jens Axboe, Ming Lei, Konrad Rzeszutek Wilk, Roger Pau Monn??,
Alasdair Kergon, Mike Snitzer, Neil Brown, Nicholas A. Bellinger,
Alexander Viro, Chris Mason, Josef Bacik, David Sterba,
Theodore Ts'o, Andreas Dilger, Jaegeuk Kim, Changman Lee,
Steven Whitehouse, Mikulas Patocka, Andrew Morton, Rik van Riel,
Johannes Weiner
In-Reply-To: <20150408230203.GG15810@dastard>
Dave Chinner <david@fromorbit.com> writes:
> On Tue, Apr 07, 2015 at 02:55:13PM -0400, Jeff Moyer wrote:
>> The way the on-stack plugging currently works, each nesting level
>> flushes its own list of I/Os. This can be less than optimal (read
>> awful) for certain workloads. For example, consider an application
>> that issues asynchronous O_DIRECT I/Os. It can send down a bunch of
>> I/Os together in a single io_submit call, only to have each of them
>> dispatched individually down in the bowels of the dirct I/O code.
>> The reason is that there are blk_plug-s instantiated both at the upper
>> call site in do_io_submit and down in do_direct_IO. The latter will
>> submit as little as 1 I/O at a time (if you have a small enough I/O
>> size) instead of performing the batching that the plugging
>> infrastructure is supposed to provide.
>
> I'm wondering what impact this will have on filesystem metadata IO
> that needs to be issued immediately. e.g. we are doing writeback, so
> there is a high level plug in place and we need to page in btree
> blocks to do extent allocation. We do readahead at this point,
> but it looks like this change will prevent the readahead from being
> issued by the unplug in xfs_buf_iosubmit().
I'm not ignoring you, Dave, I'm just doing some more investigation and
testing. It's taking longer than I had hoped.
-Jeff
--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to majordomo@kvack.org. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"aart@kvack.org">aart@kvack.org</a>
^ permalink raw reply
* Re: Breaks LSI RAID on C600 chipset
From: David F. @ 2015-04-10 20:07 UTC (permalink / raw)
To: NeilBrown; +Cc: John Stoffel, linux-raid@vger.kernel.org
In-Reply-To: <20150410105001.171d6683@notabene.brown>
Okay, and to confirm, this is happening on other C600 based systems.
Boot to Linux with MDADM raid support and the raid is gone after
reboot.
On Thu, Apr 9, 2015 at 5:50 PM, NeilBrown <neilb@suse.de> wrote:
> On Thu, 9 Apr 2015 14:13:05 -0700 "David F." <df7729@gmail.com> wrote:
>
>> Hello,
>>
>> I built a new system, installed new drives (no partitioning, just raw
>> new drives), configured RAID 1, boot to linux, created reports
>> attached. Rebooted and the system doesn't see any configured raid
>> drives (just the two raw drives).
>
> Thanks. I might take a look, but I would greatly prefer it if you kept the
> linux-raid list on the Cc.....
>
> NeilBrown
>
^ permalink raw reply
* Re: [PATCH 4/4] md-cluster: re-add
From: Goldwyn Rodrigues @ 2015-04-10 13:37 UTC (permalink / raw)
To: Guoqing Jiang; +Cc: neilb, linux-raid
In-Reply-To: <55279B26.1090402@suse.com>
On 04/10/2015 04:43 AM, Guoqing Jiang wrote:
> Hi Goldwyn,
>> diff --git a/drivers/md/md-cluster.c b/drivers/md/md-cluster.c
>> index d036c83..afffbee 100644
>> --- a/drivers/md/md-cluster.c
>> +++ b/drivers/md/md-cluster.c
>> @@ -50,6 +50,7 @@ struct md_cluster_info {
>> /* dlm lock space and resources for clustered raid. */
>> dlm_lockspace_t *lockspace;
>> int slot_number;
>> + int total_slots;
>>
> There are two "nodes" in bitmap_info and bitmap_super_t, is it possible
> to use
> the existed nodes? Instead of introduce a new member in md_cluster_info.
Yes, incorporated. Thanks.
--
Goldwyn
^ permalink raw reply
* Re: [PATCH] md-cluster: correct the num for comparison
From: Goldwyn Rodrigues @ 2015-04-10 13:35 UTC (permalink / raw)
To: gqjiang, neilb; +Cc: linux-raid
In-Reply-To: <1428653172-12678-1-git-send-email-gqjiang@suse.com>
On 04/10/2015 03:06 AM, gqjiang@suse.com wrote:
> From: Guoqing Jiang <gqjiang@suse.com>
>
> Since the node num of md-cluster is from zero, and
> cinfo->slot_number represents the slot num of dlm,
> here also need to deduct one to keep consistency
> with other codes.
This is a good find. Though I would prefer that the comparison be
(nodes < cinfo->slot_number) to decrease computation and looks better,
though the compiler may optimize what you have done as well.
>
> Signed-off-by: Guoqing Jiang <gqjiang@suse.com>
> ---
> drivers/md/md-cluster.c | 2 +-
> 1 file changed, 1 insertion(+), 1 deletion(-)
>
> diff --git a/drivers/md/md-cluster.c b/drivers/md/md-cluster.c
> index ae8bb54..531cc2b 100644
> --- a/drivers/md/md-cluster.c
> +++ b/drivers/md/md-cluster.c
> @@ -612,7 +612,7 @@ static int join(struct mddev *mddev, int nodes)
> if (ret)
> goto err;
> wait_for_completion(&cinfo->completion);
> - if (nodes <= cinfo->slot_number) {
> + if (nodes <= cinfo->slot_number - 1) {
> pr_err("md-cluster: Slot allotted(%d) greater than available slots(%d)", cinfo->slot_number - 1,
> nodes);
> ret = -ERANGE;
>
--
Goldwyn
^ permalink raw reply
* Re: [PATCH 4/4] md-cluster: re-add
From: Guoqing Jiang @ 2015-04-10 9:43 UTC (permalink / raw)
To: Goldwyn Rodrigues; +Cc: neilb, linux-raid
In-Reply-To: <20150408192414.GA9693@shrek.lan>
Hi Goldwyn,
> diff --git a/drivers/md/md-cluster.c b/drivers/md/md-cluster.c
> index d036c83..afffbee 100644
> --- a/drivers/md/md-cluster.c
> +++ b/drivers/md/md-cluster.c
> @@ -50,6 +50,7 @@ struct md_cluster_info {
> /* dlm lock space and resources for clustered raid. */
> dlm_lockspace_t *lockspace;
> int slot_number;
> + int total_slots;
>
There are two "nodes" in bitmap_info and bitmap_super_t, is it possible
to use
the existed nodes? Instead of introduce a new member in md_cluster_info.
drivers/md/bitmap.c:440: sb->nodes =
cpu_to_le32(bitmap->mddev->bitmap_info.nodes);
drivers/md/bitmap.c:552: int nodes = 0;
drivers/md/bitmap.c:610: nodes = le32_to_cpu(sb->nodes);
drivers/md/bitmap.c:688: bitmap->mddev->bitmap_info.nodes = nodes;
> @@ -653,6 +671,7 @@ static int join(struct mddev *mddev, int nodes)
> ret = -ERANGE;
> goto err;
> }
> + cinfo->total_slots = nodes;
>
The nodes is got from "nodes = le32_to_cpu(sb->nodes);", right?
> cinfo->sb_lock = lockres_init(mddev, "cmd-super",
> NULL, 0);
> if (!cinfo->sb_lock) {
> @@ -900,6 +919,34 @@ static int remove_disk(struct mddev *mddev, struct md_rdev *rdev)
> return __sendmsg(cinfo, &cmsg);
> }
>
> +static int gather_bitmaps(struct md_rdev *rdev)
> +{
> + int sn, err;
> + sector_t lo, hi;
> + struct cluster_msg cmsg;
> + struct mddev *mddev = rdev->mddev;
> + struct md_cluster_info *cinfo = mddev->cluster_info;
> + struct mdp_superblock_1 *sb = page_address(rdev->sb_page);
> + char *uuid = sb->device_uuid;
> +
> + for (sn = 0; sn < cinfo->total_slots; sn++) {
>
The last place which used total_slots.
Thanks,
Guoqing
^ permalink raw reply
* [PATCH] md-cluster: correct the num for comparison
From: gqjiang @ 2015-04-10 8:06 UTC (permalink / raw)
To: neilb; +Cc: linux-raid, rgoldwyn, Guoqing Jiang
From: Guoqing Jiang <gqjiang@suse.com>
Since the node num of md-cluster is from zero, and
cinfo->slot_number represents the slot num of dlm,
here also need to deduct one to keep consistency
with other codes.
Signed-off-by: Guoqing Jiang <gqjiang@suse.com>
---
drivers/md/md-cluster.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/drivers/md/md-cluster.c b/drivers/md/md-cluster.c
index ae8bb54..531cc2b 100644
--- a/drivers/md/md-cluster.c
+++ b/drivers/md/md-cluster.c
@@ -612,7 +612,7 @@ static int join(struct mddev *mddev, int nodes)
if (ret)
goto err;
wait_for_completion(&cinfo->completion);
- if (nodes <= cinfo->slot_number) {
+ if (nodes <= cinfo->slot_number - 1) {
pr_err("md-cluster: Slot allotted(%d) greater than available slots(%d)", cinfo->slot_number - 1,
nodes);
ret = -ERANGE;
--
1.7.12.4
^ permalink raw reply related
* Re.
From: Mrs. Zhang Xiao @ 2015-04-10 7:24 UTC (permalink / raw)
To: Recipients
Dear,
I seek for your sincerity and trust in a deal which involved a total sum of 60,000,000.00 United states dollars. I need to know you and know your location. This really matter to the success of this deal.If interested respond to me for more details.
Yours Sincerely.
Mrs. Zhang Xiao (Accounts book Keeper)
Angang Steel Company Limited
396 Nan Zhong Hua Lu, Tie Dong District Anshan, Liaoning 114021, China.
---
This email is free from viruses and malware because avast! Antivirus protection is active.
http://www.avast.com
^ permalink raw reply
* Re: interesting MD-xfs bug
From: Dave Chinner @ 2015-04-10 6:05 UTC (permalink / raw)
To: NeilBrown; +Cc: Joe Landman, linux-raid, xfs
In-Reply-To: <20150410132253.644e3660@notabene.brown>
On Fri, Apr 10, 2015 at 01:22:53PM +1000, NeilBrown wrote:
> On Fri, 10 Apr 2015 11:31:57 +1000 Dave Chinner <david@fromorbit.com> wrote:
>
> > On Fri, Apr 10, 2015 at 09:36:52AM +1000, NeilBrown wrote:
> > > On Fri, 10 Apr 2015 09:10:35 +1000 Dave Chinner <david@fromorbit.com> wrote:
> > >
> > > > On Fri, Apr 10, 2015 at 08:53:22AM +1000, Dave Chinner wrote:
> > > > > On Thu, Apr 09, 2015 at 06:20:26PM -0400, Joe Landman wrote:
> > > > > >
> > > > > >
> > > > > > On 04/09/2015 06:18 PM, Dave Chinner wrote:
> > > > > > >On Thu, Apr 09, 2015 at 05:02:33PM -0400, Joe Landman wrote:
> > > > > > >>If I build an MD raid0 with a non power of 2 chunk size, it appears
> > > > > > >>that I can mkfs.xfs a file system, but it doesn't show up in blkid
> > > > > > >>and is not mountable. Yet, using a power of 2 chunk size, this does
> > > > > > >>work correctly. This is kernel 3.18.9.
> > > > > > >>
> > > > > >
> > > > > > [...]
> > > > > >
> > > > > > >That looks more like a blkid or udev problem. try using blkid -p so
> > > > > > >that it doesn't look up the cache but directly probes devices for
> > > > > > >the signatures. strace might tell you a bit more, too. And if the
> > > > > > >filesystem mounts, then it definitely isn't an XFS problem ;)
> > > > > >
> > > > > > Thats the thing, it didn't mount, even when I used the device name
> > > > > > directly.
> > > > >
> > > > > Ok, that's interesting. Let me see if I can reproduce it locally. If
> > > > > you don't hear otherwise, tracing would still be useful. Thanks for
> > > > > the bug report, Joe.
> > > >
> > > > No luck - md doesn't allow the device to be activated on 4.0-rc7:
> > > >
> > > > $ sudo mdadm --version
> > > > mdadm - v3.3.2 - 21st August 2014
> > > > $ uname -a
> > > > Linux test4 4.0.0-rc7-dgc+ #882 SMP Fri Apr 10 08:50:52 AEST 2015 x86_64 GNU/Linux
> > > > $ sudo wipefs -a /dev/vd[ab]
> > > > /dev/vda: 4 bytes were erased at offset 0x00001000 (linux_raid_member): fc 4e 2b a9
> > > > /dev/vdb: 4 bytes were erased at offset 0x00001000 (linux_raid_member): fc 4e 2b a9
> > > > $ sudo mdadm --create /dev/md20 --level=0 --metadata=1.2 --chunk=1152 --auto=yes --raid-disks=2 /dev/vd[ab]
> > >
> > > Weird. Works for me.
> > > Any messages in 'dmesg' ??
> > > How big are /dev/vd[ab]??
> >
> > vda is 5GB, vdb is 20GB
> >
> > dmesg:
> >
> > [ 125.131340] md: bind<vda>
> > [ 125.134547] md: bind<vdb>
> > [ 125.139669] md: personality for level 0 is not loaded!
> > [ 125.141302] md: md20 stopped.
> > [ 125.141986] md: unbind<vdb>
> > [ 125.160100] md: export_rdev(vdb)
> > [ 125.161751] md: unbind<vda>
> > [ 125.180126] md: export_rdev(vda)
> >
> > Oh, curious. Going from 4.0-rc4 to 4.0-rc7, and make oldconfig
> > has resulted in:
> >
> > # CONFIG_MD_RAID0 is not set
> >
> > Ok, so with that fixed, it's still horribly broken.
> >
> > RAID 0 on different sized devices should result in a device that is
> > twice the size of the smallest devices:
> >
> > $ sudo mdadm --create /dev/md20 --level=raid0 --metadata=1.2 --chunk=1024 --auto=yes --raid-disks=2 /dev/vd[ab]
> > mdadm: array /dev/md20 started.
> > $ cat /proc/mdstat
> > Personalities : [linear] [raid0] [raid1] [raid10] [raid6] [raid5] [raid4]
> > md20 : active raid0 vdb[1] vda[0]
> > 26206208 blocks super 1.2 1024k chunks
> >
> > unused devices: <none>
> > $ grep "md\|vd[ab]" /proc/partitions
> > 253 0 5242880 vda
> > 253 16 20971520 vdb
> > 9 20 26206208 md20
> > $
> >
> > Oh, "RAID0" is not actually RAID 0 - that's the size I'd expect from
> > a linear mapping. Half way through writing that block device, the IO
> > stats change in an obvious way:
> >
> > Device: r/s w/s rMB/s wMB/s
> > vda 0.00 144.00 0.00 48.00
> > vdb 0.00 145.20 0.00 48.40
> > md20 0.00 290.40 0.00 96.80
> >
> > Device: r/s w/s rMB/s wMB/s
> > vda 0.00 56.40 0.00 18.80
> > vdb 0.00 229.20 0.00 76.40
> > md20 0.00 285.20 0.00 95.10
> >
> > Device: r/s w/s rMB/s wMB/s
> > vda 0.00 0.00 0.00 0.00
> > vdb 0.00 290.40 0.00 96.80
> > md20 0.00 290.80 0.00 96.90
> >
> > So it's actually a stripe for the first 10GB, then some kind of
> > concatenated mapping of the remainder of the single device. That's
> > not what I expected, but it's also clearly not the problem.
> >
> > Anyway, change the stripe size to 1152:
> >
> > sudo mdadm --stop /dev/md20
> > mdadm: stopped /dev/md20
> > $ sudo wipefs -a /dev/vd[ab]
> > /dev/vda: 4 bytes were erased at offset 0x00001000 (linux_raid_member): fc 4e 2b a9
> > /dev/vdb: 4 bytes were erased at offset 0x00001000 (linux_raid_member): fc 4e 2b a9
> > $ sudo mdadm --create /dev/md20 --level=raid0 --metadata=1.2 --chunk=1152 --auto=yes --raid-disks=2 /dev/vd[ab]
> > mdadm: array /dev/md20 started.
> > $ sudo xfs_io -fd -c "pwrite -b 4m 0 25g" /dev/md20
> > wrote 26831355904/26843545600 bytes at offset 0
> > 24.989 GiB, 6398 ops; 0:00:16.00 (1.530 GiB/sec and 391.8556 ops/sec)
> > $
> >
> > Wait, what? Neil, did you put a flux capacitor in MD? :P
> >
> > The underlying drive is only capable of 100MB/s - 25GB of sequential
> > direct IO does not complete in 16 seconds on such a drive. But
> > there's also a 1GB BBWC in front of the physical drives (HW RAID1),
> > but even so, this write rate could only occur if every write is
> > hitting the BBWC. And so it is:
> >
> > $ sudo xfs_io -fd -c "pwrite -b 4m 0 25g" /dev/md20 & iostat -d -m 1
> > ...
> > Device: tps MB_read/s MB_wrtn/s MB_read MB_wrtn
> > vda 4214.00 0.00 1516.99 0 1516
> > vdb 0.00 0.00 0.00 0 0
> > md20 4223.00 0.00 1520.00 0 1520
> >
> > Device: tps MB_read/s MB_wrtn/s MB_read MB_wrtn
> > vda 2986.00 0.00 1075.01 0 1075
> > vdb 1174.00 0.00 422.88 0 422
> > md20 4154.00 0.00 1496.00 0 1496
> >
> > Device: tps MB_read/s MB_wrtn/s MB_read MB_wrtn
> > vda 0.00 0.00 0.00 0 0
> > vdb 4376.00 0.00 1575.12 0 1575
> > md20 4378.00 0.00 1576.00 0 1576
> >
> > Device: tps MB_read/s MB_wrtn/s MB_read MB_wrtn
> > vda 2682.00 0.00 965.74 0 965
> > vdb 1650.00 0.00 594.00 0 594
> > md20 4334.00 0.00 1560.00 0 1560
> >
> > Device: tps MB_read/s MB_wrtn/s MB_read MB_wrtn
> > vda 4518.00 0.00 1626.26 0 1626
> > vdb 138.00 0.00 49.50 0 49
> > md20 4656.00 0.00 1676.00 0 1676
> >
> > Device: tps MB_read/s MB_wrtn/s MB_read MB_wrtn
> > vda 0.00 0.00 0.00 0 0
> > vdb 4214.00 0.00 1517.48 0 1517
> > md20 4210.00 0.00 1516.00 0 1516
> > .....
> >
> > Note how it is cycling from one drive to the other with about a 2s
> > period?
> >
> > Yup, blocktrace on /dev/vda shows it is, indeed, hitting the BBWC
> > because the block mapping is clearly broken:
> >
> > 253,0 4 1 0.000000000 6972 Q WS 8192 + 1008 [xfs_io]
> > 253,0 4 5 0.000068012 6972 Q WS 8192 + 1008 [xfs_io]
> > 253,0 4 9 0.000093266 6972 Q WS 8192 + 288 [xfs_io]
> > 253,0 4 13 0.000129722 6972 Q WS 8193 + 1008 [xfs_io]
> > 253,0 4 17 0.000176872 6972 Q WS 8193 + 1008 [xfs_io]
> > 253,0 4 21 0.000205566 6972 Q WS 8193 + 288 [xfs_io]
> > 253,0 4 25 0.000240846 6972 Q WS 8194 + 1008 [xfs_io]
> > 253,0 4 29 0.000284990 6972 Q WS 8194 + 1008 [xfs_io]
> > 253,0 4 33 0.000313276 6972 Q WS 8194 + 288 [xfs_io]
> > 253,0 4 37 0.000352330 6972 Q WS 8195 + 1008 [xfs_io]
> > 253,0 4 41 0.000374272 6972 Q WS 8195 + 272 [xfs_io]
> > 253,0 4 56 0.001215857 6972 Q WS 8195 + 1008 [xfs_io]
> > 253,0 4 60 0.001252697 6972 Q WS 8195 + 16 [xfs_io]
> > 253,0 4 64 0.001284517 6972 Q WS 8196 + 1008 [xfs_io]
> > 253,0 4 68 0.001326130 6972 Q WS 8196 + 1008 [xfs_io]
> > 253,0 4 72 0.001355050 6972 Q WS 8196 + 288 [xfs_io]
> > 253,0 4 76 0.001393777 6972 Q WS 8197 + 1008 [xfs_io]
> > 253,0 4 80 0.001439547 6972 Q WS 8197 + 1008 [xfs_io]
> > 253,0 4 84 0.001466097 6972 Q WS 8197 + 288 [xfs_io]
> > 253,0 4 88 0.001501267 6972 Q WS 8198 + 1008 [xfs_io]
> > 253,0 4 92 0.001545863 6972 Q WS 8198 + 1008 [xfs_io]
> > 253,0 4 96 0.001571500 6972 Q WS 8198 + 288 [xfs_io]
> > 253,0 4 100 0.001584620 6972 Q WS 8199 + 256 [xfs_io]
> > 253,0 4 116 0.002730034 6972 Q WS 8199 + 1008 [xfs_io]
> > 253,0 4 120 0.002792351 6972 Q WS 8199 + 1008 [xfs_io]
> > 253,0 4 124 0.002810937 6972 Q WS 8199 + 32 [xfs_io]
> > 253,0 4 128 0.002842047 6972 Q WS 8200 + 1008 [xfs_io]
> > 253,0 4 132 0.002889087 6972 Q WS 8200 + 1008 [xfs_io]
> > 253,0 4 136 0.002916894 6972 Q WS 8200 + 288 [xfs_io]
> > 253,0 4 140 0.002952334 6972 Q WS 8201 + 1008 [xfs_io]
> > 253,0 4 144 0.002996101 6972 Q WS 8201 + 1008 [xfs_io]
> > 253,0 4 148 0.003022401 6972 Q WS 8201 + 288 [xfs_io]
> >
> >
> > Multiple IOs to teh same sector, then the sector increments by 1 and
> > we get more IOs to the same sector offset. After about a second the
> > mapping shifts IO to the other block device as it slowly increments
> > the sector, and that's why we see that cycling behaviour.
> >
> > IOWs, something is going wrong with the MD block mapping when the
> > RAID chunk size is not a power of 2....
> >
> > Over to you, Neil....
>
> That's .... not good. Not good at all.
>
> This should help. It seems that non-power-of-2 chunksizes aren't widely used.
I haven't tested the patch, but if you want to make sure that you
get regular smoke testing on this sort of config, write a simple
test for xfstests and throw it in the generic group. e.g. create
multiple loop devices, then iterate over various MD configurations
running a basic data integrity tests on them. e.g. mkfs, mount,
write a 20MB pattened file, fsync, unmount, mount, md5sum it,
unlink, unmount, check filesystem.
Something like that will get run all the time by FS developers and
QE departments, so it's a good way of smoke testing configurations
that don't usually get tested without even having to think about
it...
Cheers,
Dave.
--
Dave Chinner
david@fromorbit.com
^ permalink raw reply
* Re: interesting MD-xfs bug
From: Roman Mamedov @ 2015-04-10 4:43 UTC (permalink / raw)
To: Dave Chinner; +Cc: NeilBrown, Joe Landman, linux-raid, xfs
In-Reply-To: <20150410013156.GH15810@dastard>
[-- Attachment #1: Type: text/plain, Size: 937 bytes --]
On Fri, 10 Apr 2015 11:31:57 +1000
Dave Chinner <david@fromorbit.com> wrote:
> RAID 0 on different sized devices should result in a device that is
> twice the size of the smallest devices
> Oh, "RAID0" is not actually RAID 0 - that's the size I'd expect from
> a linear mapping.
> it's actually a stripe for the first 10GB, then some kind of
> concatenated mapping of the remainder of the single device.
It might be not what you expected, but it's also not a bug of any kind, just
the regular behavior of mdadm RAID0 with different sized devices (man md):
If devices in the array are not all the same size, then once the small‐
est device has been exhausted, the RAID0 driver starts collecting
chunks into smaller stripes that only span the drives which still have
remaining space.
Once or twice this came VERY handy for me in real life usage.
--
With respect,
Roman
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 198 bytes --]
^ permalink raw reply
* Re: [PATCH 4/4] md-cluster: re-add
From: Goldwyn Rodrigues @ 2015-04-10 3:49 UTC (permalink / raw)
To: NeilBrown; +Cc: linux-raid, GQJiang
In-Reply-To: <20150409095501.536f6216@notabene.brown>
On 04/08/2015 06:55 PM, NeilBrown wrote:
> On Wed, 8 Apr 2015 14:24:14 -0500 Goldwyn Rodrigues <rgoldwyn@suse.de> wrote:
>
>> This extends the capabilites of re-adding a failed device
>> to the clustering environment.
>>
>> A new function gather_bitmaps gathers set bits from bitmaps of
>> all nodes, sends a message to all nodes to readd the disk
>> and then initiates the recovery process.
>>
>> Question: Do you see a race in sending a READD and then performing
>> the bitmap resync/recovery? Should the initiating node perform the
>> recovery before sending the READD message? The recovery will send a
>> METADATA_UPDATE anyways.
>
> The RE-ADD has to happen *before* the bitmaps are gathered.
> After the RE-ADD, all writes will go to the new device.
> Any write before that RE-ADD will be recorded in the bitmap.
> To ensure that the recovery handles all regions affected by writes, it needs
> to know about all writes that didn't go to the new device. So it needs to
> collect bitmaps only once new writes have started going to the new device.
>
> Is that clear? If not, I'll try again.
>
Yes, I understood your point. Performing the re-add later would miss on
the ones between the recovery and the re-add.
--
Goldwyn
^ permalink raw reply
* Re: interesting MD-xfs bug
From: NeilBrown @ 2015-04-10 3:22 UTC (permalink / raw)
To: Dave Chinner; +Cc: Joe Landman, linux-raid, xfs
In-Reply-To: <20150410013156.GH15810@dastard>
[-- Attachment #1: Type: text/plain, Size: 11947 bytes --]
On Fri, 10 Apr 2015 11:31:57 +1000 Dave Chinner <david@fromorbit.com> wrote:
> On Fri, Apr 10, 2015 at 09:36:52AM +1000, NeilBrown wrote:
> > On Fri, 10 Apr 2015 09:10:35 +1000 Dave Chinner <david@fromorbit.com> wrote:
> >
> > > On Fri, Apr 10, 2015 at 08:53:22AM +1000, Dave Chinner wrote:
> > > > On Thu, Apr 09, 2015 at 06:20:26PM -0400, Joe Landman wrote:
> > > > >
> > > > >
> > > > > On 04/09/2015 06:18 PM, Dave Chinner wrote:
> > > > > >On Thu, Apr 09, 2015 at 05:02:33PM -0400, Joe Landman wrote:
> > > > > >>If I build an MD raid0 with a non power of 2 chunk size, it appears
> > > > > >>that I can mkfs.xfs a file system, but it doesn't show up in blkid
> > > > > >>and is not mountable. Yet, using a power of 2 chunk size, this does
> > > > > >>work correctly. This is kernel 3.18.9.
> > > > > >>
> > > > >
> > > > > [...]
> > > > >
> > > > > >That looks more like a blkid or udev problem. try using blkid -p so
> > > > > >that it doesn't look up the cache but directly probes devices for
> > > > > >the signatures. strace might tell you a bit more, too. And if the
> > > > > >filesystem mounts, then it definitely isn't an XFS problem ;)
> > > > >
> > > > > Thats the thing, it didn't mount, even when I used the device name
> > > > > directly.
> > > >
> > > > Ok, that's interesting. Let me see if I can reproduce it locally. If
> > > > you don't hear otherwise, tracing would still be useful. Thanks for
> > > > the bug report, Joe.
> > >
> > > No luck - md doesn't allow the device to be activated on 4.0-rc7:
> > >
> > > $ sudo mdadm --version
> > > mdadm - v3.3.2 - 21st August 2014
> > > $ uname -a
> > > Linux test4 4.0.0-rc7-dgc+ #882 SMP Fri Apr 10 08:50:52 AEST 2015 x86_64 GNU/Linux
> > > $ sudo wipefs -a /dev/vd[ab]
> > > /dev/vda: 4 bytes were erased at offset 0x00001000 (linux_raid_member): fc 4e 2b a9
> > > /dev/vdb: 4 bytes were erased at offset 0x00001000 (linux_raid_member): fc 4e 2b a9
> > > $ sudo mdadm --create /dev/md20 --level=0 --metadata=1.2 --chunk=1152 --auto=yes --raid-disks=2 /dev/vd[ab]
> >
> > Weird. Works for me.
> > Any messages in 'dmesg' ??
> > How big are /dev/vd[ab]??
>
> vda is 5GB, vdb is 20GB
>
> dmesg:
>
> [ 125.131340] md: bind<vda>
> [ 125.134547] md: bind<vdb>
> [ 125.139669] md: personality for level 0 is not loaded!
> [ 125.141302] md: md20 stopped.
> [ 125.141986] md: unbind<vdb>
> [ 125.160100] md: export_rdev(vdb)
> [ 125.161751] md: unbind<vda>
> [ 125.180126] md: export_rdev(vda)
>
> Oh, curious. Going from 4.0-rc4 to 4.0-rc7, and make oldconfig
> has resulted in:
>
> # CONFIG_MD_RAID0 is not set
>
> Ok, so with that fixed, it's still horribly broken.
>
> RAID 0 on different sized devices should result in a device that is
> twice the size of the smallest devices:
>
> $ sudo mdadm --create /dev/md20 --level=raid0 --metadata=1.2 --chunk=1024 --auto=yes --raid-disks=2 /dev/vd[ab]
> mdadm: array /dev/md20 started.
> $ cat /proc/mdstat
> Personalities : [linear] [raid0] [raid1] [raid10] [raid6] [raid5] [raid4]
> md20 : active raid0 vdb[1] vda[0]
> 26206208 blocks super 1.2 1024k chunks
>
> unused devices: <none>
> $ grep "md\|vd[ab]" /proc/partitions
> 253 0 5242880 vda
> 253 16 20971520 vdb
> 9 20 26206208 md20
> $
>
> Oh, "RAID0" is not actually RAID 0 - that's the size I'd expect from
> a linear mapping. Half way through writing that block device, the IO
> stats change in an obvious way:
>
> Device: r/s w/s rMB/s wMB/s
> vda 0.00 144.00 0.00 48.00
> vdb 0.00 145.20 0.00 48.40
> md20 0.00 290.40 0.00 96.80
>
> Device: r/s w/s rMB/s wMB/s
> vda 0.00 56.40 0.00 18.80
> vdb 0.00 229.20 0.00 76.40
> md20 0.00 285.20 0.00 95.10
>
> Device: r/s w/s rMB/s wMB/s
> vda 0.00 0.00 0.00 0.00
> vdb 0.00 290.40 0.00 96.80
> md20 0.00 290.80 0.00 96.90
>
> So it's actually a stripe for the first 10GB, then some kind of
> concatenated mapping of the remainder of the single device. That's
> not what I expected, but it's also clearly not the problem.
>
> Anyway, change the stripe size to 1152:
>
> sudo mdadm --stop /dev/md20
> mdadm: stopped /dev/md20
> $ sudo wipefs -a /dev/vd[ab]
> /dev/vda: 4 bytes were erased at offset 0x00001000 (linux_raid_member): fc 4e 2b a9
> /dev/vdb: 4 bytes were erased at offset 0x00001000 (linux_raid_member): fc 4e 2b a9
> $ sudo mdadm --create /dev/md20 --level=raid0 --metadata=1.2 --chunk=1152 --auto=yes --raid-disks=2 /dev/vd[ab]
> mdadm: array /dev/md20 started.
> $ sudo xfs_io -fd -c "pwrite -b 4m 0 25g" /dev/md20
> wrote 26831355904/26843545600 bytes at offset 0
> 24.989 GiB, 6398 ops; 0:00:16.00 (1.530 GiB/sec and 391.8556 ops/sec)
> $
>
> Wait, what? Neil, did you put a flux capacitor in MD? :P
>
> The underlying drive is only capable of 100MB/s - 25GB of sequential
> direct IO does not complete in 16 seconds on such a drive. But
> there's also a 1GB BBWC in front of the physical drives (HW RAID1),
> but even so, this write rate could only occur if every write is
> hitting the BBWC. And so it is:
>
> $ sudo xfs_io -fd -c "pwrite -b 4m 0 25g" /dev/md20 & iostat -d -m 1
> ...
> Device: tps MB_read/s MB_wrtn/s MB_read MB_wrtn
> vda 4214.00 0.00 1516.99 0 1516
> vdb 0.00 0.00 0.00 0 0
> md20 4223.00 0.00 1520.00 0 1520
>
> Device: tps MB_read/s MB_wrtn/s MB_read MB_wrtn
> vda 2986.00 0.00 1075.01 0 1075
> vdb 1174.00 0.00 422.88 0 422
> md20 4154.00 0.00 1496.00 0 1496
>
> Device: tps MB_read/s MB_wrtn/s MB_read MB_wrtn
> vda 0.00 0.00 0.00 0 0
> vdb 4376.00 0.00 1575.12 0 1575
> md20 4378.00 0.00 1576.00 0 1576
>
> Device: tps MB_read/s MB_wrtn/s MB_read MB_wrtn
> vda 2682.00 0.00 965.74 0 965
> vdb 1650.00 0.00 594.00 0 594
> md20 4334.00 0.00 1560.00 0 1560
>
> Device: tps MB_read/s MB_wrtn/s MB_read MB_wrtn
> vda 4518.00 0.00 1626.26 0 1626
> vdb 138.00 0.00 49.50 0 49
> md20 4656.00 0.00 1676.00 0 1676
>
> Device: tps MB_read/s MB_wrtn/s MB_read MB_wrtn
> vda 0.00 0.00 0.00 0 0
> vdb 4214.00 0.00 1517.48 0 1517
> md20 4210.00 0.00 1516.00 0 1516
> .....
>
> Note how it is cycling from one drive to the other with about a 2s
> period?
>
> Yup, blocktrace on /dev/vda shows it is, indeed, hitting the BBWC
> because the block mapping is clearly broken:
>
> 253,0 4 1 0.000000000 6972 Q WS 8192 + 1008 [xfs_io]
> 253,0 4 5 0.000068012 6972 Q WS 8192 + 1008 [xfs_io]
> 253,0 4 9 0.000093266 6972 Q WS 8192 + 288 [xfs_io]
> 253,0 4 13 0.000129722 6972 Q WS 8193 + 1008 [xfs_io]
> 253,0 4 17 0.000176872 6972 Q WS 8193 + 1008 [xfs_io]
> 253,0 4 21 0.000205566 6972 Q WS 8193 + 288 [xfs_io]
> 253,0 4 25 0.000240846 6972 Q WS 8194 + 1008 [xfs_io]
> 253,0 4 29 0.000284990 6972 Q WS 8194 + 1008 [xfs_io]
> 253,0 4 33 0.000313276 6972 Q WS 8194 + 288 [xfs_io]
> 253,0 4 37 0.000352330 6972 Q WS 8195 + 1008 [xfs_io]
> 253,0 4 41 0.000374272 6972 Q WS 8195 + 272 [xfs_io]
> 253,0 4 56 0.001215857 6972 Q WS 8195 + 1008 [xfs_io]
> 253,0 4 60 0.001252697 6972 Q WS 8195 + 16 [xfs_io]
> 253,0 4 64 0.001284517 6972 Q WS 8196 + 1008 [xfs_io]
> 253,0 4 68 0.001326130 6972 Q WS 8196 + 1008 [xfs_io]
> 253,0 4 72 0.001355050 6972 Q WS 8196 + 288 [xfs_io]
> 253,0 4 76 0.001393777 6972 Q WS 8197 + 1008 [xfs_io]
> 253,0 4 80 0.001439547 6972 Q WS 8197 + 1008 [xfs_io]
> 253,0 4 84 0.001466097 6972 Q WS 8197 + 288 [xfs_io]
> 253,0 4 88 0.001501267 6972 Q WS 8198 + 1008 [xfs_io]
> 253,0 4 92 0.001545863 6972 Q WS 8198 + 1008 [xfs_io]
> 253,0 4 96 0.001571500 6972 Q WS 8198 + 288 [xfs_io]
> 253,0 4 100 0.001584620 6972 Q WS 8199 + 256 [xfs_io]
> 253,0 4 116 0.002730034 6972 Q WS 8199 + 1008 [xfs_io]
> 253,0 4 120 0.002792351 6972 Q WS 8199 + 1008 [xfs_io]
> 253,0 4 124 0.002810937 6972 Q WS 8199 + 32 [xfs_io]
> 253,0 4 128 0.002842047 6972 Q WS 8200 + 1008 [xfs_io]
> 253,0 4 132 0.002889087 6972 Q WS 8200 + 1008 [xfs_io]
> 253,0 4 136 0.002916894 6972 Q WS 8200 + 288 [xfs_io]
> 253,0 4 140 0.002952334 6972 Q WS 8201 + 1008 [xfs_io]
> 253,0 4 144 0.002996101 6972 Q WS 8201 + 1008 [xfs_io]
> 253,0 4 148 0.003022401 6972 Q WS 8201 + 288 [xfs_io]
>
>
> Multiple IOs to teh same sector, then the sector increments by 1 and
> we get more IOs to the same sector offset. After about a second the
> mapping shifts IO to the other block device as it slowly increments
> the sector, and that's why we see that cycling behaviour.
>
> IOWs, something is going wrong with the MD block mapping when the
> RAID chunk size is not a power of 2....
>
> Over to you, Neil....
That's .... not good. Not good at all.
This should help. It seems that non-power-of-2 chunksizes aren't widely used.
Thanks,
NeilBrown
From: NeilBrown <neilb@suse.de>
Date: Fri, 10 Apr 2015 13:19:04 +1000
Subject: [PATCH] md/raid0: fix bug with chunksize not a power of 2.
Since commit 20d0189b1012a37d2533a87fb451f7852f2418d1
in v3.14-rc1 RAID0 has performed incorrect calculations
when the chunksize is not a power of 2.
This happens because "sector_div()" modifies its first argument, but
this wasn't taken into account in the patch.
So restore that first arg before re-using the variable.
Reported-by: Joe Landman <joe.landman@gmail.com>
Reported-by: Dave Chinner <david@fromorbit.com>
Fixes: 20d0189b1012a37d2533a87fb451f7852f2418d1
Cc: stable@vger.kernel.org (3.14 and later).
Signed-off-by: NeilBrown <neilb@suse.de>
diff --git a/drivers/md/raid0.c b/drivers/md/raid0.c
index e074813da6c0..2cb59a641cd2 100644
--- a/drivers/md/raid0.c
+++ b/drivers/md/raid0.c
@@ -315,7 +315,7 @@ static struct strip_zone *find_zone(struct r0conf *conf,
/*
* remaps the bio to the target device. we separate two flows.
- * power 2 flow and a general flow for the sake of perfromance
+ * power 2 flow and a general flow for the sake of performance
*/
static struct md_rdev *map_sector(struct mddev *mddev, struct strip_zone *zone,
sector_t sector, sector_t *sector_offset)
@@ -530,6 +530,7 @@ static void raid0_make_request(struct mddev *mddev, struct bio *bio)
split = bio;
}
+ sector = bio->bi_iter.bi_sector;
zone = find_zone(mddev->private, §or);
tmp_dev = map_sector(mddev, zone, sector, §or);
split->bi_bdev = tmp_dev->bdev;
[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 811 bytes --]
^ permalink raw reply related
* Personal insight - Re: Can't mount partitions after "mdadm --zero-superblock"
From: Eduard Rozenberg @ 2015-04-10 2:53 UTC (permalink / raw)
To: NeilBrown; +Cc: linux-raid
In-Reply-To: <22789EEC-4F93-4B65-A93F-987DEE075C43@pobox.com>
I finally figured out why I panicked and why things eventually worked
thanks to this very helpful post:
http://unix.stackexchange.com/questions/64889/how-to-mount-recover-data-on-a-disk-that-was-part-of-a-mdadm-raid-1-on-another-m
In particular this paragraph:
"Linux mdraid has several metadata formats. Formats 0.9 and 1.0 put the metadata
at the end of the containing device, and the payload (the filesystem) starts at
the beginning of the device and can be accessed directly without going through
the raid layer. Formats 1.1 and 1.2 put the metadata at the middle and
beginning of the containing device respectively, so the payload is at an offset."
I knew about the various metadata versions but I’d been working under the false
assumption (based on my 0.9/1.0 metadata format mdadm experience) that
I would be able to directly mount an ext4 partition that had been previously part of
a RAID1 mdadm device.
According to the paragraph above this is clearly no longer true with metadata
v1.1/1.2, and mounting a previously raid1 partition requires either the decimal
offset to mount it directly or creating a new md device with the partition in question
(without of course formatting the new md device). Certainly not as convenient
as it used to be but I'm sure there were good reasons to change the location of
the metadata to the front of each partition.
Perhaps the mdadm wiki could include information to this effect, that mounting
formerly raid1 mdadm partitions with metadata 1.1/1.2 requires these extra
steps and not to panic :).
Regards,
—Ed
> On Apr 6, 2015, at 20:31, Eduard Rozenberg <eduardr@pobox.com> wrote:
>
> Hello Neil,
>
> Success! This is stronger voodoo magic than I’ve ever
> had to perform so really didn’t have the faith to continue
> without the extra encouragement :). My initial mistake
> had been to use the octal value with losetup instead of
> getting the decimal value.
>
> Documenting my steps below for anybody else who
> might come here later.
>
> In the examples below, we are using partition "/dev/sdac2"
> Replace this with the appropriate partition you’re recovering.
>
> Step 1: find the decimal value for the start of the partition
> ----------------------------------------------------------------------------
>
> Note:
> ext4 partitions have a “magic" octal value of “ef53"
> to indicate the start of the partition. Note that "ef53"
> may show more than once as you read further into
> the partition. We are interested in the location the
> very first occurrence of "ef53". Other types of
> partitions (ext2, ext3, etc) probably have other magic
> values to look for so this “ef53" may not apply there.
>
> The "od" hex viewer command to search for “ef53":
>
> od -x /dev/sdac2 | awk '$6 == "ef53"'
>
> The results will look something like:
>
> 4002060 f3fd 5521 0004 0025 ef53 0001 0001 0000
> 1004000060 64be 4ec9 0000 0025 ef53 0000 0001 0000
> 1042630400 17f8 a7dd bb6e ee40 ef53 000d 3cfb 9e22
>
> We are only interested in the first line. So we now have
> the octal address of the ef53 magic value: it’s the first
> long number on the line: "4002060" (octal value!)
>
> According to Neil’s instructions then we have to subtract
> the octal value “0002060" from this number we found.
> We then have to convert the octal result into decimal.
>
> Luckily an online calculator makes this easy:
>
> http://www.csgnetwork.com/octaddsubcalc.html
>
> "Enter a octal value" - Enter “4002060" here
> "Enter Second Octal Value" - Enter “0002060" here
>
> Then take the value from the line:
> "Calculated Decimal Subtraction" - 1048576
>
> This is the decimal value for the start of our partition.
>
>
> Step 2: use the decimal start value to mount partition
> ———————————————————————————————————
>
> First create a loop device loop0 as a handle to the
> partition. We tell losetup where the start of the partition is:
>
> losetup -o 1048576 /dev/loop0 /dev/sdac2
>
> Next, try to mount loop0 read-only (hopefully it will work!)
>
> mount -o ro /dev/loop0 /mnt
>
> If the partition is unclean and needs to be fscked:
>
> fsck.ext4 /dev/loop0
>
>
> Thanks again Neil! Maybe a few years from now I’ll
> understand why this worked when nothing else did
> ( linux tools still have some ways to go to being
> intelligent enough to do this kind of recovery).
>
> Regards,
> —Ed
>
>
>> On Apr 6, 2015, at 17:52, NeilBrown <neilb@suse.de> wrote:
>>
>> On Mon, 6 Apr 2015 16:45:58 -0700 Eduard Rozenberg <eduardr@pobox.com> wrote:
>>
>>> Hello folks,
>>>
>>> I previously had the following setup:
>>>
>>> sda & sdb partitioned w/ GPT, 7 partitions each (usr, opt, var etc...)
>>> 7 raid1’s with 2 devices for each pair of partitions (/dev/sda1 & /dev/sdab1, etc)
>>> They’d been created under Slackware 13.37.
>>>
>>> I was trying to clean out mdadm from those partitions but keep the data so I ran
>>> "mdadm --zero-superblock” on each of those previously RAID1 mdadm 1.2 ext4
>>> partitions.
>>
>> The "1.2" metadata is stored 4k from the start of the device. The actual
>> data is some megabytes further in. I don't suppose you still have the output
>> of "mdadm --examine" from before you destroyed the superblocks??
>>
>>>
>>> As a result I am now currently unable to mount any partition after the first one on either
>>> disk. The first partition does mount. The partition table is visible and looks fine in gdisk.
>>>
>>> mount -t ext4 /dev/sdac2 /mnt
>>> mount: wrong fs type, bad option, bad superblock on /dev/sdac2,
>>> missing codepage or helper program, or other error
>>> In some cases useful info is found in syslog - try
>>> dmesg | tail or so
>>>
>>> I did try superblock recovery with each backup superblock that ext4 normally creates,
>>> but none of the superblock locations worked.
>>>
>>> For example:
>>>
>>> fsck.ext4 -b 4096000 /dev/sdac2
>>> e2fsck 1.42.8 (20-Jun-2013)
>>> /sbin/e2fsck: Invalid argument while trying to open /dev/sdac2
>>>
>>> The superblock could not be read or does not describe a correct ext2
>>> filesystem. If the device is valid and it really contains an ext2
>>> filesystem (and not swap or ufs or something else), then the superblock
>>> is corrupt, and you might try running e2fsck with an alternate superblock:
>>> e2fsck -b 8193 <device>
>>>
>>>
>>> Would be grateful for any advice on anything else I can try.
>>
>> You need to find where the filesystem actually starts, then you need to
>> create some way to access it as a block device, then it should "just work".
>>
>> An ext4 filesystem superblock has 0xef53 at an offset of 0x38, and the
>> superblock is typically 1K from the start of the partition.
>>
>> So you could:
>> od -x /dev/sdac2 | awk '$6 == "ef53"'
>>
>> Then subtrace 0002060 (octal) from the leading number, and that might be the
>> start of the partition.
>>
>> Then
>> losetup -o "start in decimal" /dev/loop0 /dev/sdac2
>>
>> and try 'fsck' on /dev/loop0
>>
>> Good luck.
>>
>> NeilBrown
>>
>>
>>
>>>
>>> Regards,
>>> —Ed--
>>> To unsubscribe from this list: send the line "unsubscribe linux-raid" in
>>> the body of a message to majordomo@vger.kernel.org
>>> More majordomo info at http://vger.kernel.org/majordomo-info.html
>>
>
--
To unsubscribe from this list: send the line "unsubscribe linux-raid" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* RAID IO stuck under heavy load (writes)
From: Tejas Rao @ 2015-04-10 1:54 UTC (permalink / raw)
To: linux-raid
We are running GPFS over md raid devices.
The GPFS storage servers each have 60 4TB jbod drives. This is setup in
a 6 RAID6 md devices(8+2), with default chunk size 512K. We are running
RHEL 6.5 , kernel 2.6.32-431.23.3.el6.x86_64.
stripe_cache_size for each md device is set to maximum 32768. If I set
the stripe_cache_size to 16384 or lower, I see stuck IO even at lower
work loads.
Under heavy write load we see IO getting stuck for several minutes (GPFS
waiters), sometimes as long as 30 minutes, eventually they all complete.
I see stripe_cache_active on the stuck md device close to maximum and
not changing(stuck?).
This happens randomly on different md devices on different servers, so I
am sure this is not a hardware problem tied to a failing disk/SAS port etc.
How can troubleshoot this further to isolate the cause? I am reproduce
this problem 100%.
This is what I see in the /var/log/messages file. mmfslinux/mmfs26 is
the GPFS application.
Feb 5 12:24:10 host12 kernel: Not tainted 2.6.32-431.23.3.el6.x86_64 #1
Feb 5 12:24:10 host12 kernel: "echo 0 >
/proc/sys/kernel/hung_task_timeout_secs" disables this message.
Feb 5 12:24:10 host12 kernel: mmfsd D 0000000000000012 0 28987 28418
0x00000080
Feb 5 12:24:10 host12 kernel: ffff880c9f1ffbe8 0000000000000082
0000000000000000 ffffffffa02833e8
Feb 5 12:24:10 host12 kernel: ffff880c9f1ffc48 ffffffffa088c133
0000000000016840 ffff880872b07740
Feb 5 12:24:10 host12 kernel: ffff88007dc3faf8 ffff880c9f1fffd8
000000000000fbc8 ffff88007dc3faf8
Feb 5 12:24:10 host12 kernel: Call Trace:
Feb 5 12:24:10 host12 kernel: [<ffffffffa02833e8>] ?
raid5_unplug_queue+0x18/0x20 [raid456]
Feb 5 12:24:10 host12 kernel: [<ffffffffa088c133>] ?
cxiStartIO+0x2a3/0x6b0 [mmfslinux]
Feb 5 12:24:10 host12 kernel: [<ffffffffa0888b6c>] cxiWaitIO+0x13c/0x1a0
[mmfslinux]
Feb 5 12:24:10 host12 kernel: [<ffffffff8109afa0>] ?
autoremove_wake_function+0x0/0x40
Feb 5 12:24:10 host12 kernel: [<ffffffffa0913c8d>]
_ZN9DiskSched7localIOEPP15MBDoDiskIOParmsiiP15KernelOperation+0x49d/0x6d0 [mmfs26]
Feb 5 12:24:10 host12 kernel: [<ffffffffa09132d0>] ?
_Z22LinuxIODoneIntCallbackPvj+0x0/0x2a0 [mmfs26]
Feb 5 12:24:10 host12 kernel: [<ffffffffa0913f8d>] ?
kxLocalIO+0xcd/0x110 [mmfs26]
Feb 5 12:24:10 host12 kernel: [<ffffffff810129de>] ?
copy_user_generic+0xe/0x20
Feb 5 12:24:10 host12 kernel: [<ffffffffa09e0755>] ?
_Z8ss_ioctljm+0x345/0x1650 [mmfs26]
Feb 5 12:24:10 host12 kernel: [<ffffffff8100b9ce>] ?
common_interrupt+0xe/0x13
Feb 5 12:24:10 host12 kernel: [<ffffffff8100b9ce>] ?
common_interrupt+0xe/0x13
Feb 5 12:24:10 host12 kernel: [<ffffffffa089a199>] ?
ss_fs_unlocked_ioctl+0x89/0x3e0 [mmfslinux]
Feb 5 12:24:10 host12 kernel: [<ffffffff8100b9ce>] ?
common_interrupt+0xe/0x13
Feb 5 12:24:10 host12 kernel: [<ffffffff8119e532>] ? vfs_ioctl+0x22/0xa0
Feb 5 12:24:10 host12 kernel: [<ffffffff8119e6e7>] ? do_vfs_ioctl+0x97/0x580
Feb 5 12:24:10 host12 kernel: [<ffffffff8119e6d4>] ? do_vfs_ioctl+0x84/0x580
Feb 5 12:24:10 host12 kernel: [<ffffffff8119ec51>] ? sys_ioctl+0x81/0xa0
Feb 5 12:24:10 host12 kernel: [<ffffffff810e1cde>] ?
__audit_syscall_exit+0x25e/0x290
Feb 5 12:24:10 host12 kernel: [<ffffffff8100b072>] ?
system_call_fastpath+0x16/0x1b
^ permalink raw reply
* Re: interesting MD-xfs bug
From: Dave Chinner @ 2015-04-10 1:31 UTC (permalink / raw)
To: NeilBrown; +Cc: Joe Landman, linux-raid, xfs
In-Reply-To: <20150410093652.73204748@notabene.brown>
On Fri, Apr 10, 2015 at 09:36:52AM +1000, NeilBrown wrote:
> On Fri, 10 Apr 2015 09:10:35 +1000 Dave Chinner <david@fromorbit.com> wrote:
>
> > On Fri, Apr 10, 2015 at 08:53:22AM +1000, Dave Chinner wrote:
> > > On Thu, Apr 09, 2015 at 06:20:26PM -0400, Joe Landman wrote:
> > > >
> > > >
> > > > On 04/09/2015 06:18 PM, Dave Chinner wrote:
> > > > >On Thu, Apr 09, 2015 at 05:02:33PM -0400, Joe Landman wrote:
> > > > >>If I build an MD raid0 with a non power of 2 chunk size, it appears
> > > > >>that I can mkfs.xfs a file system, but it doesn't show up in blkid
> > > > >>and is not mountable. Yet, using a power of 2 chunk size, this does
> > > > >>work correctly. This is kernel 3.18.9.
> > > > >>
> > > >
> > > > [...]
> > > >
> > > > >That looks more like a blkid or udev problem. try using blkid -p so
> > > > >that it doesn't look up the cache but directly probes devices for
> > > > >the signatures. strace might tell you a bit more, too. And if the
> > > > >filesystem mounts, then it definitely isn't an XFS problem ;)
> > > >
> > > > Thats the thing, it didn't mount, even when I used the device name
> > > > directly.
> > >
> > > Ok, that's interesting. Let me see if I can reproduce it locally. If
> > > you don't hear otherwise, tracing would still be useful. Thanks for
> > > the bug report, Joe.
> >
> > No luck - md doesn't allow the device to be activated on 4.0-rc7:
> >
> > $ sudo mdadm --version
> > mdadm - v3.3.2 - 21st August 2014
> > $ uname -a
> > Linux test4 4.0.0-rc7-dgc+ #882 SMP Fri Apr 10 08:50:52 AEST 2015 x86_64 GNU/Linux
> > $ sudo wipefs -a /dev/vd[ab]
> > /dev/vda: 4 bytes were erased at offset 0x00001000 (linux_raid_member): fc 4e 2b a9
> > /dev/vdb: 4 bytes were erased at offset 0x00001000 (linux_raid_member): fc 4e 2b a9
> > $ sudo mdadm --create /dev/md20 --level=0 --metadata=1.2 --chunk=1152 --auto=yes --raid-disks=2 /dev/vd[ab]
>
> Weird. Works for me.
> Any messages in 'dmesg' ??
> How big are /dev/vd[ab]??
vda is 5GB, vdb is 20GB
dmesg:
[ 125.131340] md: bind<vda>
[ 125.134547] md: bind<vdb>
[ 125.139669] md: personality for level 0 is not loaded!
[ 125.141302] md: md20 stopped.
[ 125.141986] md: unbind<vdb>
[ 125.160100] md: export_rdev(vdb)
[ 125.161751] md: unbind<vda>
[ 125.180126] md: export_rdev(vda)
Oh, curious. Going from 4.0-rc4 to 4.0-rc7, and make oldconfig
has resulted in:
# CONFIG_MD_RAID0 is not set
Ok, so with that fixed, it's still horribly broken.
RAID 0 on different sized devices should result in a device that is
twice the size of the smallest devices:
$ sudo mdadm --create /dev/md20 --level=raid0 --metadata=1.2 --chunk=1024 --auto=yes --raid-disks=2 /dev/vd[ab]
mdadm: array /dev/md20 started.
$ cat /proc/mdstat
Personalities : [linear] [raid0] [raid1] [raid10] [raid6] [raid5] [raid4]
md20 : active raid0 vdb[1] vda[0]
26206208 blocks super 1.2 1024k chunks
unused devices: <none>
$ grep "md\|vd[ab]" /proc/partitions
253 0 5242880 vda
253 16 20971520 vdb
9 20 26206208 md20
$
Oh, "RAID0" is not actually RAID 0 - that's the size I'd expect from
a linear mapping. Half way through writing that block device, the IO
stats change in an obvious way:
Device: r/s w/s rMB/s wMB/s
vda 0.00 144.00 0.00 48.00
vdb 0.00 145.20 0.00 48.40
md20 0.00 290.40 0.00 96.80
Device: r/s w/s rMB/s wMB/s
vda 0.00 56.40 0.00 18.80
vdb 0.00 229.20 0.00 76.40
md20 0.00 285.20 0.00 95.10
Device: r/s w/s rMB/s wMB/s
vda 0.00 0.00 0.00 0.00
vdb 0.00 290.40 0.00 96.80
md20 0.00 290.80 0.00 96.90
So it's actually a stripe for the first 10GB, then some kind of
concatenated mapping of the remainder of the single device. That's
not what I expected, but it's also clearly not the problem.
Anyway, change the stripe size to 1152:
sudo mdadm --stop /dev/md20
mdadm: stopped /dev/md20
$ sudo wipefs -a /dev/vd[ab]
/dev/vda: 4 bytes were erased at offset 0x00001000 (linux_raid_member): fc 4e 2b a9
/dev/vdb: 4 bytes were erased at offset 0x00001000 (linux_raid_member): fc 4e 2b a9
$ sudo mdadm --create /dev/md20 --level=raid0 --metadata=1.2 --chunk=1152 --auto=yes --raid-disks=2 /dev/vd[ab]
mdadm: array /dev/md20 started.
$ sudo xfs_io -fd -c "pwrite -b 4m 0 25g" /dev/md20
wrote 26831355904/26843545600 bytes at offset 0
24.989 GiB, 6398 ops; 0:00:16.00 (1.530 GiB/sec and 391.8556 ops/sec)
$
Wait, what? Neil, did you put a flux capacitor in MD? :P
The underlying drive is only capable of 100MB/s - 25GB of sequential
direct IO does not complete in 16 seconds on such a drive. But
there's also a 1GB BBWC in front of the physical drives (HW RAID1),
but even so, this write rate could only occur if every write is
hitting the BBWC. And so it is:
$ sudo xfs_io -fd -c "pwrite -b 4m 0 25g" /dev/md20 & iostat -d -m 1
...
Device: tps MB_read/s MB_wrtn/s MB_read MB_wrtn
vda 4214.00 0.00 1516.99 0 1516
vdb 0.00 0.00 0.00 0 0
md20 4223.00 0.00 1520.00 0 1520
Device: tps MB_read/s MB_wrtn/s MB_read MB_wrtn
vda 2986.00 0.00 1075.01 0 1075
vdb 1174.00 0.00 422.88 0 422
md20 4154.00 0.00 1496.00 0 1496
Device: tps MB_read/s MB_wrtn/s MB_read MB_wrtn
vda 0.00 0.00 0.00 0 0
vdb 4376.00 0.00 1575.12 0 1575
md20 4378.00 0.00 1576.00 0 1576
Device: tps MB_read/s MB_wrtn/s MB_read MB_wrtn
vda 2682.00 0.00 965.74 0 965
vdb 1650.00 0.00 594.00 0 594
md20 4334.00 0.00 1560.00 0 1560
Device: tps MB_read/s MB_wrtn/s MB_read MB_wrtn
vda 4518.00 0.00 1626.26 0 1626
vdb 138.00 0.00 49.50 0 49
md20 4656.00 0.00 1676.00 0 1676
Device: tps MB_read/s MB_wrtn/s MB_read MB_wrtn
vda 0.00 0.00 0.00 0 0
vdb 4214.00 0.00 1517.48 0 1517
md20 4210.00 0.00 1516.00 0 1516
.....
Note how it is cycling from one drive to the other with about a 2s
period?
Yup, blocktrace on /dev/vda shows it is, indeed, hitting the BBWC
because the block mapping is clearly broken:
253,0 4 1 0.000000000 6972 Q WS 8192 + 1008 [xfs_io]
253,0 4 5 0.000068012 6972 Q WS 8192 + 1008 [xfs_io]
253,0 4 9 0.000093266 6972 Q WS 8192 + 288 [xfs_io]
253,0 4 13 0.000129722 6972 Q WS 8193 + 1008 [xfs_io]
253,0 4 17 0.000176872 6972 Q WS 8193 + 1008 [xfs_io]
253,0 4 21 0.000205566 6972 Q WS 8193 + 288 [xfs_io]
253,0 4 25 0.000240846 6972 Q WS 8194 + 1008 [xfs_io]
253,0 4 29 0.000284990 6972 Q WS 8194 + 1008 [xfs_io]
253,0 4 33 0.000313276 6972 Q WS 8194 + 288 [xfs_io]
253,0 4 37 0.000352330 6972 Q WS 8195 + 1008 [xfs_io]
253,0 4 41 0.000374272 6972 Q WS 8195 + 272 [xfs_io]
253,0 4 56 0.001215857 6972 Q WS 8195 + 1008 [xfs_io]
253,0 4 60 0.001252697 6972 Q WS 8195 + 16 [xfs_io]
253,0 4 64 0.001284517 6972 Q WS 8196 + 1008 [xfs_io]
253,0 4 68 0.001326130 6972 Q WS 8196 + 1008 [xfs_io]
253,0 4 72 0.001355050 6972 Q WS 8196 + 288 [xfs_io]
253,0 4 76 0.001393777 6972 Q WS 8197 + 1008 [xfs_io]
253,0 4 80 0.001439547 6972 Q WS 8197 + 1008 [xfs_io]
253,0 4 84 0.001466097 6972 Q WS 8197 + 288 [xfs_io]
253,0 4 88 0.001501267 6972 Q WS 8198 + 1008 [xfs_io]
253,0 4 92 0.001545863 6972 Q WS 8198 + 1008 [xfs_io]
253,0 4 96 0.001571500 6972 Q WS 8198 + 288 [xfs_io]
253,0 4 100 0.001584620 6972 Q WS 8199 + 256 [xfs_io]
253,0 4 116 0.002730034 6972 Q WS 8199 + 1008 [xfs_io]
253,0 4 120 0.002792351 6972 Q WS 8199 + 1008 [xfs_io]
253,0 4 124 0.002810937 6972 Q WS 8199 + 32 [xfs_io]
253,0 4 128 0.002842047 6972 Q WS 8200 + 1008 [xfs_io]
253,0 4 132 0.002889087 6972 Q WS 8200 + 1008 [xfs_io]
253,0 4 136 0.002916894 6972 Q WS 8200 + 288 [xfs_io]
253,0 4 140 0.002952334 6972 Q WS 8201 + 1008 [xfs_io]
253,0 4 144 0.002996101 6972 Q WS 8201 + 1008 [xfs_io]
253,0 4 148 0.003022401 6972 Q WS 8201 + 288 [xfs_io]
Multiple IOs to teh same sector, then the sector increments by 1 and
we get more IOs to the same sector offset. After about a second the
mapping shifts IO to the other block device as it slowly increments
the sector, and that's why we see that cycling behaviour.
IOWs, something is going wrong with the MD block mapping when the
RAID chunk size is not a power of 2....
Over to you, Neil....
Cheers,
Dave.
--
Dave Chinner
david@fromorbit.com
^ permalink raw reply
* Re: interesting MD-xfs bug
From: NeilBrown @ 2015-04-09 23:36 UTC (permalink / raw)
To: Dave Chinner; +Cc: Joe Landman, linux-raid, xfs
In-Reply-To: <20150409231035.GI13731@dastard>
[-- Attachment #1: Type: text/plain, Size: 2288 bytes --]
On Fri, 10 Apr 2015 09:10:35 +1000 Dave Chinner <david@fromorbit.com> wrote:
> On Fri, Apr 10, 2015 at 08:53:22AM +1000, Dave Chinner wrote:
> > On Thu, Apr 09, 2015 at 06:20:26PM -0400, Joe Landman wrote:
> > >
> > >
> > > On 04/09/2015 06:18 PM, Dave Chinner wrote:
> > > >On Thu, Apr 09, 2015 at 05:02:33PM -0400, Joe Landman wrote:
> > > >>If I build an MD raid0 with a non power of 2 chunk size, it appears
> > > >>that I can mkfs.xfs a file system, but it doesn't show up in blkid
> > > >>and is not mountable. Yet, using a power of 2 chunk size, this does
> > > >>work correctly. This is kernel 3.18.9.
> > > >>
> > >
> > > [...]
> > >
> > > >That looks more like a blkid or udev problem. try using blkid -p so
> > > >that it doesn't look up the cache but directly probes devices for
> > > >the signatures. strace might tell you a bit more, too. And if the
> > > >filesystem mounts, then it definitely isn't an XFS problem ;)
> > >
> > > Thats the thing, it didn't mount, even when I used the device name
> > > directly.
> >
> > Ok, that's interesting. Let me see if I can reproduce it locally. If
> > you don't hear otherwise, tracing would still be useful. Thanks for
> > the bug report, Joe.
>
> No luck - md doesn't allow the device to be activated on 4.0-rc7:
>
> $ sudo mdadm --version
> mdadm - v3.3.2 - 21st August 2014
> $ uname -a
> Linux test4 4.0.0-rc7-dgc+ #882 SMP Fri Apr 10 08:50:52 AEST 2015 x86_64 GNU/Linux
> $ sudo wipefs -a /dev/vd[ab]
> /dev/vda: 4 bytes were erased at offset 0x00001000 (linux_raid_member): fc 4e 2b a9
> /dev/vdb: 4 bytes were erased at offset 0x00001000 (linux_raid_member): fc 4e 2b a9
> $ sudo mdadm --create /dev/md20 --level=0 --metadata=1.2 --chunk=1152 --auto=yes --raid-disks=2 /dev/vd[ab]
Weird. Works for me.
Any messages in 'dmesg' ??
How big are /dev/vd[ab]??
NeilBrown
> mdadm: RUN_ARRAY failed: Invalid argument
> Problem may be that chunk size is not a power of 2
> $ cat /proc/mdstat
> Personalities : [raid1] [raid10] [raid6] [raid5] [raid4]
> unused devices: <none>
> $
>
> So I can't actually reproduce what you are seeing because MD doesn't
> allow the device to be activated and so mdadm tears it back down.
>
> Cheers,
>
> Dave.
[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 811 bytes --]
^ permalink raw reply
* Re: interesting MD-xfs bug
From: Dave Chinner @ 2015-04-09 23:10 UTC (permalink / raw)
To: Joe Landman; +Cc: linux-raid, xfs
In-Reply-To: <20150409225322.GH13731@dastard>
On Fri, Apr 10, 2015 at 08:53:22AM +1000, Dave Chinner wrote:
> On Thu, Apr 09, 2015 at 06:20:26PM -0400, Joe Landman wrote:
> >
> >
> > On 04/09/2015 06:18 PM, Dave Chinner wrote:
> > >On Thu, Apr 09, 2015 at 05:02:33PM -0400, Joe Landman wrote:
> > >>If I build an MD raid0 with a non power of 2 chunk size, it appears
> > >>that I can mkfs.xfs a file system, but it doesn't show up in blkid
> > >>and is not mountable. Yet, using a power of 2 chunk size, this does
> > >>work correctly. This is kernel 3.18.9.
> > >>
> >
> > [...]
> >
> > >That looks more like a blkid or udev problem. try using blkid -p so
> > >that it doesn't look up the cache but directly probes devices for
> > >the signatures. strace might tell you a bit more, too. And if the
> > >filesystem mounts, then it definitely isn't an XFS problem ;)
> >
> > Thats the thing, it didn't mount, even when I used the device name
> > directly.
>
> Ok, that's interesting. Let me see if I can reproduce it locally. If
> you don't hear otherwise, tracing would still be useful. Thanks for
> the bug report, Joe.
No luck - md doesn't allow the device to be activated on 4.0-rc7:
$ sudo mdadm --version
mdadm - v3.3.2 - 21st August 2014
$ uname -a
Linux test4 4.0.0-rc7-dgc+ #882 SMP Fri Apr 10 08:50:52 AEST 2015 x86_64 GNU/Linux
$ sudo wipefs -a /dev/vd[ab]
/dev/vda: 4 bytes were erased at offset 0x00001000 (linux_raid_member): fc 4e 2b a9
/dev/vdb: 4 bytes were erased at offset 0x00001000 (linux_raid_member): fc 4e 2b a9
$ sudo mdadm --create /dev/md20 --level=0 --metadata=1.2 --chunk=1152 --auto=yes --raid-disks=2 /dev/vd[ab]
mdadm: RUN_ARRAY failed: Invalid argument
Problem may be that chunk size is not a power of 2
$ cat /proc/mdstat
Personalities : [raid1] [raid10] [raid6] [raid5] [raid4]
unused devices: <none>
$
So I can't actually reproduce what you are seeing because MD doesn't
allow the device to be activated and so mdadm tears it back down.
Cheers,
Dave.
--
Dave Chinner
david@fromorbit.com
^ permalink raw reply
* Re: interesting MD-xfs bug
From: Dave Chinner @ 2015-04-09 22:53 UTC (permalink / raw)
To: Joe Landman; +Cc: xfs, linux-raid
In-Reply-To: <5526FB2A.8060704@gmail.com>
On Thu, Apr 09, 2015 at 06:20:26PM -0400, Joe Landman wrote:
>
>
> On 04/09/2015 06:18 PM, Dave Chinner wrote:
> >On Thu, Apr 09, 2015 at 05:02:33PM -0400, Joe Landman wrote:
> >>If I build an MD raid0 with a non power of 2 chunk size, it appears
> >>that I can mkfs.xfs a file system, but it doesn't show up in blkid
> >>and is not mountable. Yet, using a power of 2 chunk size, this does
> >>work correctly. This is kernel 3.18.9.
> >>
>
> [...]
>
> >That looks more like a blkid or udev problem. try using blkid -p so
> >that it doesn't look up the cache but directly probes devices for
> >the signatures. strace might tell you a bit more, too. And if the
> >filesystem mounts, then it definitely isn't an XFS problem ;)
>
> Thats the thing, it didn't mount, even when I used the device name
> directly.
Ok, that's interesting. Let me see if I can reproduce it locally. If
you don't hear otherwise, tracing would still be useful. Thanks for
the bug report, Joe.
Cheers,
Dave.
--
Dave Chinner
david@fromorbit.com
^ permalink raw reply
* Re: interesting MD-xfs bug
From: Joe Landman @ 2015-04-09 22:20 UTC (permalink / raw)
To: Dave Chinner; +Cc: xfs, linux-raid
In-Reply-To: <20150409221846.GG13731@dastard>
On 04/09/2015 06:18 PM, Dave Chinner wrote:
> On Thu, Apr 09, 2015 at 05:02:33PM -0400, Joe Landman wrote:
>> If I build an MD raid0 with a non power of 2 chunk size, it appears
>> that I can mkfs.xfs a file system, but it doesn't show up in blkid
>> and is not mountable. Yet, using a power of 2 chunk size, this does
>> work correctly. This is kernel 3.18.9.
>>
[...]
> That looks more like a blkid or udev problem. try using blkid -p so
> that it doesn't look up the cache but directly probes devices for
> the signatures. strace might tell you a bit more, too. And if the
> filesystem mounts, then it definitely isn't an XFS problem ;)
Thats the thing, it didn't mount, even when I used the device name
directly.
Good point on stracing though. I'll do that tomorrow and report back.
Thanks!
Joe
>
> Cheers,
>
> Dave.
--
Joe Landman
e: joe.landman@gmail.com
t: @sijoe
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox