* [PATCH 1/3] raid5-cache: use a bio_set
From: Christoph Hellwig @ 2015-12-17 22:09 UTC (permalink / raw)
To: neilb, shli; +Cc: linux-raid
In-Reply-To: <1450390197-19479-1-git-send-email-hch@lst.de>
This allows us to make guaranteed forward progress.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
drivers/md/raid5-cache.c | 16 +++++++++++++++-
1 file changed, 15 insertions(+), 1 deletion(-)
diff --git a/drivers/md/raid5-cache.c b/drivers/md/raid5-cache.c
index 668e973..0a64e97 100644
--- a/drivers/md/raid5-cache.c
+++ b/drivers/md/raid5-cache.c
@@ -34,6 +34,12 @@
#define RECLAIM_MAX_FREE_SPACE (10 * 1024 * 1024 * 2) /* sector */
#define RECLAIM_MAX_FREE_SPACE_SHIFT (2)
+/*
+ * We only need 2 bios per I/O unit to make progress, but ensure we
+ * have a few more available to not get too tight.
+ */
+#define R5L_POOL_SIZE 4
+
struct r5l_log {
struct md_rdev *rdev;
@@ -70,6 +76,7 @@ struct r5l_log {
struct bio flush_bio;
struct kmem_cache *io_kc;
+ struct bio_set *bs;
struct md_thread *reclaim_thread;
unsigned long reclaim_target; /* number of space that need to be
@@ -248,7 +255,7 @@ static void r5l_submit_current_io(struct r5l_log *log)
static struct bio *r5l_bio_alloc(struct r5l_log *log)
{
- struct bio *bio = bio_kmalloc(GFP_NOIO | __GFP_NOFAIL, BIO_MAX_PAGES);
+ struct bio *bio = bio_alloc_bioset(GFP_NOIO, BIO_MAX_PAGES, log->bs);
bio->bi_rw = WRITE;
bio->bi_bdev = log->rdev->bdev;
@@ -1153,6 +1160,10 @@ int r5l_init_log(struct r5conf *conf, struct md_rdev *rdev)
if (!log->io_kc)
goto io_kc;
+ log->bs = bioset_create(R5L_POOL_SIZE, 0);
+ if (!log->bs)
+ goto io_bs;
+
log->reclaim_thread = md_register_thread(r5l_reclaim_thread,
log->rdev->mddev, "reclaim");
if (!log->reclaim_thread)
@@ -1170,6 +1181,8 @@ int r5l_init_log(struct r5conf *conf, struct md_rdev *rdev)
error:
md_unregister_thread(&log->reclaim_thread);
reclaim_thread:
+ bioset_free(log->bs);
+io_bs:
kmem_cache_destroy(log->io_kc);
io_kc:
kfree(log);
@@ -1179,6 +1192,7 @@ io_kc:
void r5l_exit_log(struct r5l_log *log)
{
md_unregister_thread(&log->reclaim_thread);
+ bioset_free(log->bs);
kmem_cache_destroy(log->io_kc);
kfree(log);
}
--
1.9.1
^ permalink raw reply related
* [PATCH 2/3] raid5-cache: use a mempool for the metadata block
From: Christoph Hellwig @ 2015-12-17 22:09 UTC (permalink / raw)
To: neilb, shli; +Cc: linux-raid
In-Reply-To: <1450390197-19479-1-git-send-email-hch@lst.de>
We only have a limited number in flight, so use a page based mempool.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
drivers/md/raid5-cache.c | 14 ++++++++++++--
1 file changed, 12 insertions(+), 2 deletions(-)
diff --git a/drivers/md/raid5-cache.c b/drivers/md/raid5-cache.c
index 0a64e97..e0a605f 100644
--- a/drivers/md/raid5-cache.c
+++ b/drivers/md/raid5-cache.c
@@ -77,6 +77,7 @@ struct r5l_log {
struct kmem_cache *io_kc;
struct bio_set *bs;
+ mempool_t *meta_pool;
struct md_thread *reclaim_thread;
unsigned long reclaim_target; /* number of space that need to be
@@ -216,7 +217,7 @@ static void r5l_log_endio(struct bio *bio)
md_error(log->rdev->mddev, log->rdev);
bio_put(bio);
- __free_page(io->meta_page);
+ mempool_free(io->meta_page, log->meta_pool);
spin_lock_irqsave(&log->io_list_lock, flags);
__r5l_set_io_unit_state(io, IO_UNIT_IO_END);
@@ -293,8 +294,9 @@ static struct r5l_io_unit *r5l_new_meta(struct r5l_log *log)
INIT_LIST_HEAD(&io->stripe_list);
io->state = IO_UNIT_RUNNING;
- io->meta_page = alloc_page(GFP_NOIO | __GFP_NOFAIL | __GFP_ZERO);
+ io->meta_page = mempool_alloc(log->meta_pool, GFP_NOIO);
block = page_address(io->meta_page);
+ clear_page(block);
block->magic = cpu_to_le32(R5LOG_MAGIC);
block->version = R5LOG_VERSION;
block->seq = cpu_to_le64(log->seq);
@@ -1164,6 +1166,10 @@ int r5l_init_log(struct r5conf *conf, struct md_rdev *rdev)
if (!log->bs)
goto io_bs;
+ log->meta_pool = mempool_create_page_pool(R5L_POOL_SIZE, 0);
+ if (!log->meta_pool)
+ goto out_mempool;
+
log->reclaim_thread = md_register_thread(r5l_reclaim_thread,
log->rdev->mddev, "reclaim");
if (!log->reclaim_thread)
@@ -1178,9 +1184,12 @@ int r5l_init_log(struct r5conf *conf, struct md_rdev *rdev)
conf->log = log;
return 0;
+
error:
md_unregister_thread(&log->reclaim_thread);
reclaim_thread:
+ mempool_destroy(log->meta_pool);
+out_mempool:
bioset_free(log->bs);
io_bs:
kmem_cache_destroy(log->io_kc);
@@ -1192,6 +1201,7 @@ io_kc:
void r5l_exit_log(struct r5l_log *log)
{
md_unregister_thread(&log->reclaim_thread);
+ mempool_destroy(log->meta_pool);
bioset_free(log->bs);
kmem_cache_destroy(log->io_kc);
kfree(log);
--
1.9.1
^ permalink raw reply related
* [PATCH 3/3] raid5: allow r5l_io_unit allocations to fail
From: Christoph Hellwig @ 2015-12-17 22:09 UTC (permalink / raw)
To: neilb, shli; +Cc: linux-raid
In-Reply-To: <1450390197-19479-1-git-send-email-hch@lst.de>
And propagate the error up the stack so we can add the stripe
to no_stripes_list and retry our log operation later. This avoids
blocking raid5d due to reclaim, an it allows to get rid of the
deadlock-prone GFP_NOFAIL allocation.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
drivers/md/raid5-cache.c | 49 +++++++++++++++++++++++++++++++++---------------
1 file changed, 34 insertions(+), 15 deletions(-)
diff --git a/drivers/md/raid5-cache.c b/drivers/md/raid5-cache.c
index e0a605f..ddee884 100644
--- a/drivers/md/raid5-cache.c
+++ b/drivers/md/raid5-cache.c
@@ -287,8 +287,10 @@ static struct r5l_io_unit *r5l_new_meta(struct r5l_log *log)
struct r5l_io_unit *io;
struct r5l_meta_block *block;
- /* We can't handle memory allocate failure so far */
- io = kmem_cache_zalloc(log->io_kc, GFP_NOIO | __GFP_NOFAIL);
+ io = kmem_cache_zalloc(log->io_kc, GFP_ATOMIC);
+ if (!io)
+ return NULL;
+
io->log = log;
INIT_LIST_HEAD(&io->log_sibling);
INIT_LIST_HEAD(&io->stripe_list);
@@ -326,8 +328,12 @@ static int r5l_get_meta(struct r5l_log *log, unsigned int payload_size)
log->current_io->meta_offset + payload_size > PAGE_SIZE)
r5l_submit_current_io(log);
- if (!log->current_io)
+ if (!log->current_io) {
log->current_io = r5l_new_meta(log);
+ if (!log->current_io)
+ return -ENOMEM;
+ }
+
return 0;
}
@@ -372,11 +378,12 @@ static void r5l_append_payload_page(struct r5l_log *log, struct page *page)
r5_reserve_log_entry(log, io);
}
-static void r5l_log_stripe(struct r5l_log *log, struct stripe_head *sh,
+static int r5l_log_stripe(struct r5l_log *log, struct stripe_head *sh,
int data_pages, int parity_pages)
{
int i;
int meta_size;
+ int ret;
struct r5l_io_unit *io;
meta_size =
@@ -385,7 +392,10 @@ static void r5l_log_stripe(struct r5l_log *log, struct stripe_head *sh,
sizeof(struct r5l_payload_data_parity) +
sizeof(__le32) * parity_pages;
- r5l_get_meta(log, meta_size);
+ ret = r5l_get_meta(log, meta_size);
+ if (ret)
+ return ret;
+
io = log->current_io;
for (i = 0; i < sh->disks; i++) {
@@ -415,6 +425,8 @@ static void r5l_log_stripe(struct r5l_log *log, struct stripe_head *sh,
list_add_tail(&sh->log_list, &io->stripe_list);
atomic_inc(&io->pending_stripe);
sh->log_io = io;
+
+ return 0;
}
static void r5l_wake_reclaim(struct r5l_log *log, sector_t space);
@@ -429,6 +441,7 @@ int r5l_write_stripe(struct r5l_log *log, struct stripe_head *sh)
int meta_size;
int reserve;
int i;
+ int ret = 0;
if (!log)
return -EAGAIN;
@@ -477,18 +490,24 @@ int r5l_write_stripe(struct r5l_log *log, struct stripe_head *sh)
mutex_lock(&log->io_mutex);
/* meta + data */
reserve = (1 + write_disks) << (PAGE_SHIFT - 9);
- if (r5l_has_free_space(log, reserve))
- r5l_log_stripe(log, sh, data_pages, parity_pages);
- else {
- spin_lock(&log->no_space_stripes_lock);
- list_add_tail(&sh->log_list, &log->no_space_stripes);
- spin_unlock(&log->no_space_stripes_lock);
-
- r5l_wake_reclaim(log, reserve);
- }
- mutex_unlock(&log->io_mutex);
+ if (!r5l_has_free_space(log, reserve))
+ goto err_retry;
+ ret = r5l_log_stripe(log, sh, data_pages, parity_pages);
+ if (ret)
+ goto err_retry;
+
+out_unlock:
+ mutex_unlock(&log->io_mutex);
return 0;
+
+err_retry:
+ spin_lock(&log->no_space_stripes_lock);
+ list_add_tail(&sh->log_list, &log->no_space_stripes);
+ spin_unlock(&log->no_space_stripes_lock);
+
+ r5l_wake_reclaim(log, reserve);
+ goto out_unlock;
}
void r5l_write_stripe_run(struct r5l_log *log)
--
1.9.1
^ permalink raw reply related
* Re: best base / worst case RAID 5,6 write speeds
From: Phil Turmel @ 2015-12-17 22:40 UTC (permalink / raw)
To: Dallas Clement; +Cc: Mark Knecht, John Stoffel, Linux-RAID
In-Reply-To: <CAE9DZUQj2TNX4Wpk5xbaCVzEgPq2D6Co7ZDNaAy-TOaB4hwZeA@mail.gmail.com>
On 12/17/2015 04:08 PM, Dallas Clement wrote:
> I am still in the process of collecting a bunch of performance data.
> But so far, it is shocking to see the throughput difference when
> blocks written are stripe aligned.
Random writes unaligned has at least a 4x multiplier on raid5 and 6x on
raid6 per my earlier explanation. Why does this surprise you? It's
parity raid. This is why users with heavy random workloads are pointed
at raid1 and raid10. I like raid10,f3 for VM host images and databases.
> However, in the non-ideal world it
> is not always possible to ensure that clients are writing blocks of
> data which are stripe aligned.
Hardly possible at all, except for bulk writes of large media files, and
then only if you are writing one stream at a time to an otherwise idle
storage stack. Not very realistic in a general-purpose storage
appliance. "General purpose" just isn't very sequential.
> If the goal is to reduce the # of RMWs
> it seems like writing big blocks would also help for sequential
> workloads where large quantities of data are being written.
The goal is to be able to read later what you need to write now. Unless
you have unlimited $ to spend, you have to balance speed, redundancy,
and capacity. As they say, pick two.
Lots of spindles is generally good. Raid5 is great for capacity, good
for redundancy, and marginal for speed. Raid6 is great for capacity,
great for redundancy, and pitiful for speed. Raid10,f2 is great for
speed, poor for capacity, and good for redundancy. Raid10,f3 is great
for speed, pitiful for capacity, and great for redundancy.
> Can any
> of you think of anything else that can be tuned in the kernel to
> reduce # of RMWs in the case where blocks are not stripe aligned? Is
> it a bad idea to mess with the timing of the stripe cache?
You can't really hold those writes for long, as any serious application
is going to call fdatasync at short intervals, for algorithmic integrity
reasons. On random workloads, you simply have no choice but to do RMWs.
Your only out is to make complete chunk stripes smaller than your
application's typical write size. That raises the odds that any
particular write will be aligned or mostly aligned. Have you tried 4k
chunks?
Phil
^ permalink raw reply
* Re: best base / worst case RAID 5,6 write speeds
From: Dallas Clement @ 2015-12-17 23:28 UTC (permalink / raw)
To: Phil Turmel; +Cc: Mark Knecht, John Stoffel, Linux-RAID
In-Reply-To: <567339F0.9000209@turmel.org>
On Thu, Dec 17, 2015 at 4:40 PM, Phil Turmel <philip@turmel.org> wrote:
> On 12/17/2015 04:08 PM, Dallas Clement wrote:
>> I am still in the process of collecting a bunch of performance data.
>> But so far, it is shocking to see the throughput difference when
>> blocks written are stripe aligned.
>
> Random writes unaligned has at least a 4x multiplier on raid5 and 6x on
> raid6 per my earlier explanation. Why does this surprise you? It's
> parity raid. This is why users with heavy random workloads are pointed
> at raid1 and raid10. I like raid10,f3 for VM host images and databases.
>
>> However, in the non-ideal world it
>> is not always possible to ensure that clients are writing blocks of
>> data which are stripe aligned.
>
> Hardly possible at all, except for bulk writes of large media files, and
> then only if you are writing one stream at a time to an otherwise idle
> storage stack. Not very realistic in a general-purpose storage
> appliance. "General purpose" just isn't very sequential.
>
>> If the goal is to reduce the # of RMWs
>> it seems like writing big blocks would also help for sequential
>> workloads where large quantities of data are being written.
>
> The goal is to be able to read later what you need to write now. Unless
> you have unlimited $ to spend, you have to balance speed, redundancy,
> and capacity. As they say, pick two.
>
> Lots of spindles is generally good. Raid5 is great for capacity, good
> for redundancy, and marginal for speed. Raid6 is great for capacity,
> great for redundancy, and pitiful for speed. Raid10,f2 is great for
> speed, poor for capacity, and good for redundancy. Raid10,f3 is great
> for speed, pitiful for capacity, and great for redundancy.
>
>> Can any
>> of you think of anything else that can be tuned in the kernel to
>> reduce # of RMWs in the case where blocks are not stripe aligned? Is
>> it a bad idea to mess with the timing of the stripe cache?
>
> You can't really hold those writes for long, as any serious application
> is going to call fdatasync at short intervals, for algorithmic integrity
> reasons. On random workloads, you simply have no choice but to do RMWs.
> Your only out is to make complete chunk stripes smaller than your
> application's typical write size. That raises the odds that any
> particular write will be aligned or mostly aligned. Have you tried 4k
> chunks?
>
> Phil
>
Hi Phil. Thanks for the explanation.
> Random writes unaligned has at least a 4x multiplier on raid5 and 6x on
> raid6 per my earlier explanation. Why does this surprise you? It's
> parity raid. This is why users with heavy random workloads are pointed
> at raid1 and raid10. I like raid10,f3 for VM host images and databases.
It really shouldn't surprise me. I should have said I am very HAPPY
to see such relatively good performance when the writes are stripe
aligned. :)
> Have you tried 4k chunks?
No not yet. I've been taking some measurements with 16k, 32k, 64k,
128k, 256k. So far it looks like 64k has the highest speeds for RAID
5 sequential writes.
^ permalink raw reply
* Re: raid5-cache: avoid GFP_NOFAIL allocation
From: NeilBrown @ 2015-12-17 23:31 UTC (permalink / raw)
To: Christoph Hellwig, shli; +Cc: linux-raid
In-Reply-To: <1450390197-19479-1-git-send-email-hch@lst.de>
[-- Attachment #1: Type: text/plain, Size: 156 bytes --]
On Fri, Dec 18 2015, Christoph Hellwig wrote:
> bio_sets, mempools and high level retries are our friends:
Yes they are!
all applied, Thanks.
NeilBrown
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 818 bytes --]
^ permalink raw reply
* Re: [PATCH 3/3] raid5: allow r5l_io_unit allocations to fail
From: Shaohua Li @ 2015-12-17 23:48 UTC (permalink / raw)
To: Christoph Hellwig; +Cc: neilb, linux-raid
In-Reply-To: <1450390197-19479-4-git-send-email-hch@lst.de>
On Thu, Dec 17, 2015 at 11:09:57PM +0100, Christoph Hellwig wrote:
> And propagate the error up the stack so we can add the stripe
> to no_stripes_list and retry our log operation later. This avoids
> blocking raid5d due to reclaim, an it allows to get rid of the
> deadlock-prone GFP_NOFAIL allocation.
>
> Signed-off-by: Christoph Hellwig <hch@lst.de>
> ---
> drivers/md/raid5-cache.c | 49 +++++++++++++++++++++++++++++++++---------------
> 1 file changed, 34 insertions(+), 15 deletions(-)
>
> diff --git a/drivers/md/raid5-cache.c b/drivers/md/raid5-cache.c
> index e0a605f..ddee884 100644
> --- a/drivers/md/raid5-cache.c
> +++ b/drivers/md/raid5-cache.c
> @@ -287,8 +287,10 @@ static struct r5l_io_unit *r5l_new_meta(struct r5l_log *log)
> struct r5l_io_unit *io;
> struct r5l_meta_block *block;
>
> - /* We can't handle memory allocate failure so far */
> - io = kmem_cache_zalloc(log->io_kc, GFP_NOIO | __GFP_NOFAIL);
> + io = kmem_cache_zalloc(log->io_kc, GFP_ATOMIC);
> + if (!io)
> + return NULL;
> +
> io->log = log;
> INIT_LIST_HEAD(&io->log_sibling);
> INIT_LIST_HEAD(&io->stripe_list);
> @@ -326,8 +328,12 @@ static int r5l_get_meta(struct r5l_log *log, unsigned int payload_size)
> log->current_io->meta_offset + payload_size > PAGE_SIZE)
> r5l_submit_current_io(log);
>
> - if (!log->current_io)
> + if (!log->current_io) {
> log->current_io = r5l_new_meta(log);
> + if (!log->current_io)
> + return -ENOMEM;
> + }
> +
> return 0;
> }
>
> @@ -372,11 +378,12 @@ static void r5l_append_payload_page(struct r5l_log *log, struct page *page)
> r5_reserve_log_entry(log, io);
> }
>
> -static void r5l_log_stripe(struct r5l_log *log, struct stripe_head *sh,
> +static int r5l_log_stripe(struct r5l_log *log, struct stripe_head *sh,
> int data_pages, int parity_pages)
> {
> int i;
> int meta_size;
> + int ret;
> struct r5l_io_unit *io;
>
> meta_size =
> @@ -385,7 +392,10 @@ static void r5l_log_stripe(struct r5l_log *log, struct stripe_head *sh,
> sizeof(struct r5l_payload_data_parity) +
> sizeof(__le32) * parity_pages;
>
> - r5l_get_meta(log, meta_size);
> + ret = r5l_get_meta(log, meta_size);
> + if (ret)
> + return ret;
> +
> io = log->current_io;
>
> for (i = 0; i < sh->disks; i++) {
> @@ -415,6 +425,8 @@ static void r5l_log_stripe(struct r5l_log *log, struct stripe_head *sh,
> list_add_tail(&sh->log_list, &io->stripe_list);
> atomic_inc(&io->pending_stripe);
> sh->log_io = io;
> +
> + return 0;
> }
>
> static void r5l_wake_reclaim(struct r5l_log *log, sector_t space);
> @@ -429,6 +441,7 @@ int r5l_write_stripe(struct r5l_log *log, struct stripe_head *sh)
> int meta_size;
> int reserve;
> int i;
> + int ret = 0;
>
> if (!log)
> return -EAGAIN;
> @@ -477,18 +490,24 @@ int r5l_write_stripe(struct r5l_log *log, struct stripe_head *sh)
> mutex_lock(&log->io_mutex);
> /* meta + data */
> reserve = (1 + write_disks) << (PAGE_SHIFT - 9);
> - if (r5l_has_free_space(log, reserve))
> - r5l_log_stripe(log, sh, data_pages, parity_pages);
> - else {
> - spin_lock(&log->no_space_stripes_lock);
> - list_add_tail(&sh->log_list, &log->no_space_stripes);
> - spin_unlock(&log->no_space_stripes_lock);
> -
> - r5l_wake_reclaim(log, reserve);
> - }
> - mutex_unlock(&log->io_mutex);
> + if (!r5l_has_free_space(log, reserve))
> + goto err_retry;
>
> + ret = r5l_log_stripe(log, sh, data_pages, parity_pages);
> + if (ret)
> + goto err_retry;
> +
> +out_unlock:
> + mutex_unlock(&log->io_mutex);
> return 0;
> +
> +err_retry:
> + spin_lock(&log->no_space_stripes_lock);
> + list_add_tail(&sh->log_list, &log->no_space_stripes);
> + spin_unlock(&log->no_space_stripes_lock);
> +
> + r5l_wake_reclaim(log, reserve);
> + goto out_unlock;
> }
if the reclaim thread doesn't have anything to reclaim,
r5l_run_no_space_stripes isn't called. we might miss the retry.
I'm a little worrying about the GFP_ATOMIC allocation. In the first try,
GFP_NOWAIT is better. And on the other hand, why sleep is bad here? We
could use GFP_NOIO | __GFP_NORETRY, there is no deadlock risk.
In the retry, GFP_NOIO looks better. No deadlock too, since it's not
called from raid5d (maybe we shouldn't call from reclaim thread if using
GFP_NOIO, a workqueue is better). Otherwise we could keep retring but do
nothing.
Thanks,
Shaohua
^ permalink raw reply
* Re: best base / worst case RAID 5,6 write speeds
From: Dallas Clement @ 2015-12-18 0:54 UTC (permalink / raw)
To: Phil Turmel; +Cc: Mark Knecht, John Stoffel, Linux-RAID
In-Reply-To: <CAE9DZUS+gmgTLr_4EUgLcPCPWknoqg640mCMZX46cxXeyT6RQA@mail.gmail.com>
>> Have you tried 4k chunks?
>
> No not yet. I've been taking some measurements with 16k, 32k, 64k,
> 128k, 256k. So far it looks like 64k has the highest speeds for RAID
> 5 sequential writes.
Correction. It looks like a 32k chunk size gives the best performance
overall for RAID 5 sequential and random access. I'm checking this on
RAID 0, 1, and 6 as well.
^ permalink raw reply
* Re: [PATCH] Fix remove_and_add_spares removes drive added as spare in slot_store
From: NeilBrown @ 2015-12-18 1:10 UTC (permalink / raw)
To: rgoldwyn, linux-raid; +Cc: pawel.baldysiak, Goldwyn Rodrigues
In-Reply-To: <1450359905-8042-1-git-send-email-rgoldwyn@suse.de>
[-- Attachment #1: Type: text/plain, Size: 1732 bytes --]
On Fri, Dec 18 2015, rgoldwyn@suse.de wrote:
> From: Goldwyn Rodrigues <rgoldwyn@suse.com>
>
> Commit 2910ff17d154baa5eb50e362a91104e831eb2bb6
> introduced a regression which would remove a recently added spare via
> slot_store. Revert part of the patch which touches slot_store() and add
> the disk directly using pers->hot_add_disk()
>
> Signed-off-by: Goldwyn Rodrigues <rgoldwyn@suse.com>
> Signed-off-by: Pawel Baldysiak <pawel.baldysiak@intel.com>
> ---
> drivers/md/md.c | 13 ++++++++++---
> 1 file changed, 10 insertions(+), 3 deletions(-)
>
> diff --git a/drivers/md/md.c b/drivers/md/md.c
> index a71b36f..0444afa 100644
> --- a/drivers/md/md.c
> +++ b/drivers/md/md.c
> @@ -2726,6 +2726,7 @@ slot_store(struct md_rdev *rdev, const char *buf, size_t len)
> /* Activating a spare .. or possibly reactivating
> * if we ever get bitmaps working here.
> */
> + int err;
>
> if (rdev->raid_disk != -1)
> return -EBUSY;
> @@ -2747,9 +2748,15 @@ slot_store(struct md_rdev *rdev, const char *buf, size_t len)
> rdev->saved_raid_disk = -1;
> clear_bit(In_sync, &rdev->flags);
> clear_bit(Bitmap_sync, &rdev->flags);
> - remove_and_add_spares(rdev->mddev, rdev);
> - if (rdev->raid_disk == -1)
> - return -EBUSY;
> + err = rdev->mddev->pers->
> + hot_add_disk(rdev->mddev, rdev);
> + if (err) {
> + rdev->raid_disk = -1;
> + return err;
> + } else
> + sysfs_notify_dirent_safe(rdev->sysfs_state);
> + if (sysfs_link_rdev(rdev->mddev, rdev))
> + /* failure here is OK */;
> /* don't wakeup anyone, leave that to userspace. */
> } else {
> if (slot >= rdev->mddev->raid_disks &&
> --
> 2.6.2
applied, thanks.
NeilBrown
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 818 bytes --]
^ permalink raw reply
* Re: [PATCH 3/3] raid5: allow r5l_io_unit allocations to fail
From: NeilBrown @ 2015-12-18 1:51 UTC (permalink / raw)
To: Shaohua Li, Christoph Hellwig; +Cc: linux-raid
In-Reply-To: <20151217234748.GA1860175@devbig084.prn1.facebook.com>
[-- Attachment #1: Type: text/plain, Size: 6011 bytes --]
On Fri, Dec 18 2015, Shaohua Li wrote:
> On Thu, Dec 17, 2015 at 11:09:57PM +0100, Christoph Hellwig wrote:
>> And propagate the error up the stack so we can add the stripe
>> to no_stripes_list and retry our log operation later. This avoids
>> blocking raid5d due to reclaim, an it allows to get rid of the
>> deadlock-prone GFP_NOFAIL allocation.
>>
>> Signed-off-by: Christoph Hellwig <hch@lst.de>
>> ---
>> drivers/md/raid5-cache.c | 49 +++++++++++++++++++++++++++++++++---------------
>> 1 file changed, 34 insertions(+), 15 deletions(-)
>>
>> diff --git a/drivers/md/raid5-cache.c b/drivers/md/raid5-cache.c
>> index e0a605f..ddee884 100644
>> --- a/drivers/md/raid5-cache.c
>> +++ b/drivers/md/raid5-cache.c
>> @@ -287,8 +287,10 @@ static struct r5l_io_unit *r5l_new_meta(struct r5l_log *log)
>> struct r5l_io_unit *io;
>> struct r5l_meta_block *block;
>>
>> - /* We can't handle memory allocate failure so far */
>> - io = kmem_cache_zalloc(log->io_kc, GFP_NOIO | __GFP_NOFAIL);
>> + io = kmem_cache_zalloc(log->io_kc, GFP_ATOMIC);
>> + if (!io)
>> + return NULL;
>> +
>> io->log = log;
>> INIT_LIST_HEAD(&io->log_sibling);
>> INIT_LIST_HEAD(&io->stripe_list);
>> @@ -326,8 +328,12 @@ static int r5l_get_meta(struct r5l_log *log, unsigned int payload_size)
>> log->current_io->meta_offset + payload_size > PAGE_SIZE)
>> r5l_submit_current_io(log);
>>
>> - if (!log->current_io)
>> + if (!log->current_io) {
>> log->current_io = r5l_new_meta(log);
>> + if (!log->current_io)
>> + return -ENOMEM;
>> + }
>> +
>> return 0;
>> }
>>
>> @@ -372,11 +378,12 @@ static void r5l_append_payload_page(struct r5l_log *log, struct page *page)
>> r5_reserve_log_entry(log, io);
>> }
>>
>> -static void r5l_log_stripe(struct r5l_log *log, struct stripe_head *sh,
>> +static int r5l_log_stripe(struct r5l_log *log, struct stripe_head *sh,
>> int data_pages, int parity_pages)
>> {
>> int i;
>> int meta_size;
>> + int ret;
>> struct r5l_io_unit *io;
>>
>> meta_size =
>> @@ -385,7 +392,10 @@ static void r5l_log_stripe(struct r5l_log *log, struct stripe_head *sh,
>> sizeof(struct r5l_payload_data_parity) +
>> sizeof(__le32) * parity_pages;
>>
>> - r5l_get_meta(log, meta_size);
>> + ret = r5l_get_meta(log, meta_size);
>> + if (ret)
>> + return ret;
>> +
>> io = log->current_io;
>>
>> for (i = 0; i < sh->disks; i++) {
>> @@ -415,6 +425,8 @@ static void r5l_log_stripe(struct r5l_log *log, struct stripe_head *sh,
>> list_add_tail(&sh->log_list, &io->stripe_list);
>> atomic_inc(&io->pending_stripe);
>> sh->log_io = io;
>> +
>> + return 0;
>> }
>>
>> static void r5l_wake_reclaim(struct r5l_log *log, sector_t space);
>> @@ -429,6 +441,7 @@ int r5l_write_stripe(struct r5l_log *log, struct stripe_head *sh)
>> int meta_size;
>> int reserve;
>> int i;
>> + int ret = 0;
>>
>> if (!log)
>> return -EAGAIN;
>> @@ -477,18 +490,24 @@ int r5l_write_stripe(struct r5l_log *log, struct stripe_head *sh)
>> mutex_lock(&log->io_mutex);
>> /* meta + data */
>> reserve = (1 + write_disks) << (PAGE_SHIFT - 9);
>> - if (r5l_has_free_space(log, reserve))
>> - r5l_log_stripe(log, sh, data_pages, parity_pages);
>> - else {
>> - spin_lock(&log->no_space_stripes_lock);
>> - list_add_tail(&sh->log_list, &log->no_space_stripes);
>> - spin_unlock(&log->no_space_stripes_lock);
>> -
>> - r5l_wake_reclaim(log, reserve);
>> - }
>> - mutex_unlock(&log->io_mutex);
>> + if (!r5l_has_free_space(log, reserve))
>> + goto err_retry;
>>
>> + ret = r5l_log_stripe(log, sh, data_pages, parity_pages);
>> + if (ret)
>> + goto err_retry;
>> +
>> +out_unlock:
>> + mutex_unlock(&log->io_mutex);
>> return 0;
>> +
>> +err_retry:
>> + spin_lock(&log->no_space_stripes_lock);
>> + list_add_tail(&sh->log_list, &log->no_space_stripes);
>> + spin_unlock(&log->no_space_stripes_lock);
>> +
>> + r5l_wake_reclaim(log, reserve);
>> + goto out_unlock;
>> }
>
> if the reclaim thread doesn't have anything to reclaim,
> r5l_run_no_space_stripes isn't called. we might miss the retry.
so something like this:
diff --git a/drivers/md/raid5-cache.c b/drivers/md/raid5-cache.c
index 18de1fc4a75b..b63878edf7e9 100644
--- a/drivers/md/raid5-cache.c
+++ b/drivers/md/raid5-cache.c
@@ -596,7 +596,8 @@ static void __r5l_stripe_write_finished(struct r5l_io_unit *io)
return;
}
- if (r5l_reclaimable_space(log) > log->max_free_space)
+ if (r5l_reclaimable_space(log) > log->max_free_space ||
+ !list_empty(&log->no_space_stripes))
r5l_wake_reclaim(log, 0);
spin_unlock_irqrestore(&log->io_list_lock, flags);
or is that too simplistic?
>
> I'm a little worrying about the GFP_ATOMIC allocation. In the first try,
> GFP_NOWAIT is better. And on the other hand, why sleep is bad here? We
> could use GFP_NOIO | __GFP_NORETRY, there is no deadlock risk.
>
> In the retry, GFP_NOIO looks better. No deadlock too, since it's not
> called from raid5d (maybe we shouldn't call from reclaim thread if using
> GFP_NOIO, a workqueue is better). Otherwise we could keep retring but do
> nothing.
I did wonder a little bit about that.
GFP_ATOMIC is (__GFP_HIGH)
GFP_NOIO | __GFP_NORETRY is (__GFP_WAIT | __GFP_NORETRY)
It isn't clear that we need 'HIGH', and WAIT with NORETRY should be OK.
It allows __alloc_pages_direct_reclaim, but only once and never waits
for other IO.
We probably should add __GFP_NOWARN too because we expect occasional
failure.
So
--- a/drivers/md/raid5-cache.c
+++ b/drivers/md/raid5-cache.c
@@ -287,7 +287,7 @@ static struct r5l_io_unit *r5l_new_meta(struct r5l_log *log)
struct r5l_io_unit *io;
struct r5l_meta_block *block;
- io = kmem_cache_zalloc(log->io_kc, GFP_ATOMIC);
+ io = kmem_cache_zalloc(log->io_kc, GFP_NOIO | __GFP_NORETRY | __GFP_NOWARN);
if (!io)
return NULL;
Thoughts?
Thanks,
NeilBrown
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 818 bytes --]
^ permalink raw reply related
* Re: [PATCH 3/3] raid5: allow r5l_io_unit allocations to fail
From: Shaohua Li @ 2015-12-18 1:58 UTC (permalink / raw)
To: NeilBrown; +Cc: Christoph Hellwig, linux-raid
In-Reply-To: <874mfg8qyc.fsf@notabene.neil.brown.name>
On Fri, Dec 18, 2015 at 12:51:07PM +1100, NeilBrown wrote:
> On Fri, Dec 18 2015, Shaohua Li wrote:
>
> > On Thu, Dec 17, 2015 at 11:09:57PM +0100, Christoph Hellwig wrote:
> >> And propagate the error up the stack so we can add the stripe
> >> to no_stripes_list and retry our log operation later. This avoids
> >> blocking raid5d due to reclaim, an it allows to get rid of the
> >> deadlock-prone GFP_NOFAIL allocation.
> >>
> >> Signed-off-by: Christoph Hellwig <hch@lst.de>
> >> ---
> >> drivers/md/raid5-cache.c | 49 +++++++++++++++++++++++++++++++++---------------
> >> 1 file changed, 34 insertions(+), 15 deletions(-)
> >>
> >> diff --git a/drivers/md/raid5-cache.c b/drivers/md/raid5-cache.c
> >> index e0a605f..ddee884 100644
> >> --- a/drivers/md/raid5-cache.c
> >> +++ b/drivers/md/raid5-cache.c
> >> @@ -287,8 +287,10 @@ static struct r5l_io_unit *r5l_new_meta(struct r5l_log *log)
> >> struct r5l_io_unit *io;
> >> struct r5l_meta_block *block;
> >>
> >> - /* We can't handle memory allocate failure so far */
> >> - io = kmem_cache_zalloc(log->io_kc, GFP_NOIO | __GFP_NOFAIL);
> >> + io = kmem_cache_zalloc(log->io_kc, GFP_ATOMIC);
> >> + if (!io)
> >> + return NULL;
> >> +
> >> io->log = log;
> >> INIT_LIST_HEAD(&io->log_sibling);
> >> INIT_LIST_HEAD(&io->stripe_list);
> >> @@ -326,8 +328,12 @@ static int r5l_get_meta(struct r5l_log *log, unsigned int payload_size)
> >> log->current_io->meta_offset + payload_size > PAGE_SIZE)
> >> r5l_submit_current_io(log);
> >>
> >> - if (!log->current_io)
> >> + if (!log->current_io) {
> >> log->current_io = r5l_new_meta(log);
> >> + if (!log->current_io)
> >> + return -ENOMEM;
> >> + }
> >> +
> >> return 0;
> >> }
> >>
> >> @@ -372,11 +378,12 @@ static void r5l_append_payload_page(struct r5l_log *log, struct page *page)
> >> r5_reserve_log_entry(log, io);
> >> }
> >>
> >> -static void r5l_log_stripe(struct r5l_log *log, struct stripe_head *sh,
> >> +static int r5l_log_stripe(struct r5l_log *log, struct stripe_head *sh,
> >> int data_pages, int parity_pages)
> >> {
> >> int i;
> >> int meta_size;
> >> + int ret;
> >> struct r5l_io_unit *io;
> >>
> >> meta_size =
> >> @@ -385,7 +392,10 @@ static void r5l_log_stripe(struct r5l_log *log, struct stripe_head *sh,
> >> sizeof(struct r5l_payload_data_parity) +
> >> sizeof(__le32) * parity_pages;
> >>
> >> - r5l_get_meta(log, meta_size);
> >> + ret = r5l_get_meta(log, meta_size);
> >> + if (ret)
> >> + return ret;
> >> +
> >> io = log->current_io;
> >>
> >> for (i = 0; i < sh->disks; i++) {
> >> @@ -415,6 +425,8 @@ static void r5l_log_stripe(struct r5l_log *log, struct stripe_head *sh,
> >> list_add_tail(&sh->log_list, &io->stripe_list);
> >> atomic_inc(&io->pending_stripe);
> >> sh->log_io = io;
> >> +
> >> + return 0;
> >> }
> >>
> >> static void r5l_wake_reclaim(struct r5l_log *log, sector_t space);
> >> @@ -429,6 +441,7 @@ int r5l_write_stripe(struct r5l_log *log, struct stripe_head *sh)
> >> int meta_size;
> >> int reserve;
> >> int i;
> >> + int ret = 0;
> >>
> >> if (!log)
> >> return -EAGAIN;
> >> @@ -477,18 +490,24 @@ int r5l_write_stripe(struct r5l_log *log, struct stripe_head *sh)
> >> mutex_lock(&log->io_mutex);
> >> /* meta + data */
> >> reserve = (1 + write_disks) << (PAGE_SHIFT - 9);
> >> - if (r5l_has_free_space(log, reserve))
> >> - r5l_log_stripe(log, sh, data_pages, parity_pages);
> >> - else {
> >> - spin_lock(&log->no_space_stripes_lock);
> >> - list_add_tail(&sh->log_list, &log->no_space_stripes);
> >> - spin_unlock(&log->no_space_stripes_lock);
> >> -
> >> - r5l_wake_reclaim(log, reserve);
> >> - }
> >> - mutex_unlock(&log->io_mutex);
> >> + if (!r5l_has_free_space(log, reserve))
> >> + goto err_retry;
> >>
> >> + ret = r5l_log_stripe(log, sh, data_pages, parity_pages);
> >> + if (ret)
> >> + goto err_retry;
> >> +
> >> +out_unlock:
> >> + mutex_unlock(&log->io_mutex);
> >> return 0;
> >> +
> >> +err_retry:
> >> + spin_lock(&log->no_space_stripes_lock);
> >> + list_add_tail(&sh->log_list, &log->no_space_stripes);
> >> + spin_unlock(&log->no_space_stripes_lock);
> >> +
> >> + r5l_wake_reclaim(log, reserve);
> >> + goto out_unlock;
> >> }
> >
> > if the reclaim thread doesn't have anything to reclaim,
> > r5l_run_no_space_stripes isn't called. we might miss the retry.
>
> so something like this:
> diff --git a/drivers/md/raid5-cache.c b/drivers/md/raid5-cache.c
> index 18de1fc4a75b..b63878edf7e9 100644
> --- a/drivers/md/raid5-cache.c
> +++ b/drivers/md/raid5-cache.c
> @@ -596,7 +596,8 @@ static void __r5l_stripe_write_finished(struct r5l_io_unit *io)
> return;
> }
>
> - if (r5l_reclaimable_space(log) > log->max_free_space)
> + if (r5l_reclaimable_space(log) > log->max_free_space ||
> + !list_empty(&log->no_space_stripes))
> r5l_wake_reclaim(log, 0);
>
> spin_unlock_irqrestore(&log->io_list_lock, flags);
>
> or is that too simplistic?
maybe add a new list and run the list at the begining of r5l_do_reclaim().
> >
> > I'm a little worrying about the GFP_ATOMIC allocation. In the first try,
> > GFP_NOWAIT is better. And on the other hand, why sleep is bad here? We
> > could use GFP_NOIO | __GFP_NORETRY, there is no deadlock risk.
> >
> > In the retry, GFP_NOIO looks better. No deadlock too, since it's not
> > called from raid5d (maybe we shouldn't call from reclaim thread if using
> > GFP_NOIO, a workqueue is better). Otherwise we could keep retring but do
> > nothing.
>
> I did wonder a little bit about that.
> GFP_ATOMIC is (__GFP_HIGH)
> GFP_NOIO | __GFP_NORETRY is (__GFP_WAIT | __GFP_NORETRY)
>
> It isn't clear that we need 'HIGH', and WAIT with NORETRY should be OK.
> It allows __alloc_pages_direct_reclaim, but only once and never waits
> for other IO.
>
> We probably should add __GFP_NOWARN too because we expect occasional
> failure.
>
> So
>
> --- a/drivers/md/raid5-cache.c
> +++ b/drivers/md/raid5-cache.c
> @@ -287,7 +287,7 @@ static struct r5l_io_unit *r5l_new_meta(struct r5l_log *log)
> struct r5l_io_unit *io;
> struct r5l_meta_block *block;
>
> - io = kmem_cache_zalloc(log->io_kc, GFP_ATOMIC);
> + io = kmem_cache_zalloc(log->io_kc, GFP_NOIO | __GFP_NORETRY | __GFP_NOWARN);
> if (!io)
> return NULL;
Looks good.
Thanks,
Shaohua
^ permalink raw reply
* Darlehen bieten
From: SKY, GROUP @ 2015-12-18 7:00 UTC (permalink / raw)
To: Recipients
Brauchen Sie jede Art von Darlehen? Haben Sie eine Weihnachts-Darlehen müssen bei 1% Wenn ja, füllen Sie bitte das Anmeldeformular below.Reply-To: skygroup.financia211@gmail.com
(1) Name:
(2) Der Kreditbetrag erforderlich:
(3) Laufzeit der Darlehen:
(4) Land:
(5) Telefon:
Freundliche Grüße
Herr Kelly Williams
--
To unsubscribe from this list: send the line "unsubscribe linux-raid" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* Re: [PATCH 3/3] raid5: allow r5l_io_unit allocations to fail
From: Christoph Hellwig @ 2015-12-18 11:23 UTC (permalink / raw)
To: NeilBrown; +Cc: Shaohua Li, Christoph Hellwig, linux-raid
In-Reply-To: <874mfg8qyc.fsf@notabene.neil.brown.name>
On Fri, Dec 18, 2015 at 12:51:07PM +1100, NeilBrown wrote:
> > if the reclaim thread doesn't have anything to reclaim,
> > r5l_run_no_space_stripes isn't called. we might miss the retry.
>
> so something like this:
that looks fine to me. I'll give a spin on my QA setup.
> > I'm a little worrying about the GFP_ATOMIC allocation. In the first try,
> > GFP_NOWAIT is better. And on the other hand, why sleep is bad here? We
> > could use GFP_NOIO | __GFP_NORETRY, there is no deadlock risk.
> >
> > In the retry, GFP_NOIO looks better. No deadlock too, since it's not
> > called from raid5d (maybe we shouldn't call from reclaim thread if using
> > GFP_NOIO, a workqueue is better). Otherwise we could keep retring but do
> > nothing.
>
> I did wonder a little bit about that.
> GFP_ATOMIC is (__GFP_HIGH)
> GFP_NOIO | __GFP_NORETRY is (__GFP_WAIT | __GFP_NORETRY)
>
> It isn't clear that we need 'HIGH', and WAIT with NORETRY should be OK.
> It allows __alloc_pages_direct_reclaim, but only once and never waits
> for other IO.
In general we go for HIGH on non-sleeping allocation to avoid having
the stalled. This is especially important in the I/O path.
WAIT means we will reclaim and wait for it, which looks a little dangerous
to me. In general I'd prefer not to use obscure gfp flag combination
unless there is a real need, and it's clearly documented.
^ permalink raw reply
* Re: [PATCH 3/3] raid5: allow r5l_io_unit allocations to fail
From: Christoph Hellwig @ 2015-12-18 11:25 UTC (permalink / raw)
To: Shaohua Li; +Cc: NeilBrown, Christoph Hellwig, linux-raid
In-Reply-To: <20151218015847.GA2146501@devbig084.prn1.facebook.com>
[can you trim your replies please? I had to trim almost 150 lines of
junk before getting to your reply!]
> > spin_unlock_irqrestore(&log->io_list_lock, flags);
> >
> > or is that too simplistic?
>
> maybe add a new list and run the list at the begining of r5l_do_reclaim().
What's the advantage of another list?
^ permalink raw reply
* clustered MD - beyond RAID1
From: Scott Sinno @ 2015-12-18 15:29 UTC (permalink / raw)
To: neilb, linux-raid; +Cc: Knister, Aaron S. (GSFC-606.2)[COMPUTER SCIENCE CORP]
Neil(or anyone well informed in mdadm development roadmaps),
Aaron and myself are engineers at NASA Goddard with strong interest in
MDADM. We currently host 6PB(raw) of live JBOD storage leveraging MDADM
exclusively for RAID functionality.
We're very interested in Clustered MDADM to improve data-availability
in the environment, but note that only RAID1 is currently supported.
Are there plans in the nearish-term(say over the next year) to expound
clustered bitmap functionality to RAID5/6, or anything else you can
divulge on that front? Thanks in advance for any guidance.
^ permalink raw reply
* RAID10 Write Performance
From: Marc Smith @ 2015-12-18 18:43 UTC (permalink / raw)
To: linux-raid
Hi,
I'm testing a (24) slot SSD array (Supermicro) with MD RAID. The setup
consists of the Supermicro chassis, (24) Pliant LB406M SAS SSD drives,
(3) Avago/LSI SAS3008 SAS HBAs, and (2) Intel Xeon E5-2660 2.60GHz
processors.
The (24) SSDs are directly connected (pass-through back-plane) to the
(3) SAS HBAs (eight drives per HBA) with no SAS expanders.
I'm planning to use RAID10 for this system. I started by playing with
some performance configurations, I'm specifically looking at random IO
performance.
The test commands I've been using with fio are the following:
4K 100% random, 100% READ: fio --bs=4k --direct=1 --rw=randread
--ioengine=libaio --iodepth=16 --numjobs=16 --name=/dev/md0
--runtime=60
4K 100% random, 100% WRITE: fio --bs=4k --direct=1 --rw=randwrite
--ioengine=libaio --iodepth=16 --numjobs=16 --name=/dev/md0
--runtime=60
As a benchmark, I initially tested all twenty-four drives using RAID0;
using a 8K chunk size and here are the numbers I got:
4K random read: 645,233 IOPS
4K random write: 309,879 IOPS
Not too shabby... obviously these are just for bench-marking, the plan
is to use RAID10 for production.
So, I won't go into the specifics of all the tests, but I've tried
quite a few different RAID10 configurations: Nested RAID 10 (1+0) -
RAID 0 (stripe) built with RAID 1 (mirror) arrays, Nested RAID 10
(0+1) - RAID 1 (mirror) built with RAID 0 (stripe) arrays, and
"Complex" RAID 10 - Near Layout / 2.
All of these yield very similar results using (12) of the disks spread
across the (3) HBAs. As an example:
Nested RAID 10 (0+1) - RAID 1 (mirror) built with RAID 0 (stripe) arrays
For the (2) stripe sets (2 disks per HBA, 6 total per set):
mdadm --create --verbose /dev/md0 --level=stripe --raid-devices=6
--chunk=64K /dev/sda1 /dev/sdb1 /dev/sdi1 /dev/sdj1 /dev/sdq1
/dev/sdr1
mdadm --create --verbose /dev/md1 --level=stripe --raid-devices=6
--chunk=64K /dev/sdc1 /dev/sdd1 /dev/sdk1 /dev/sdl1 /dev/sds1
/dev/sdt1
For the (1) mirror set (consisting of the 2 stripe sets):
mdadm --create --verbose /dev/md2 --level=mirror --raid-devices=2
/dev/md0 /dev/md1
Running the random 4K performance tests described above yields the
following results for the RAID10 array:
4K random read: 276,967 IOPS
4K random write: 643 IOPS
The read numbers seem in-line with what I expected, but the writes are
absolutely dismal. I expect them not be where the read numbers are,
but this is really, really low! I gotta have something configured
incorrectly, right?
I've experimented with different chunk sizes, and haven't gotten much
of a change in the write numbers. Again, I've tried several different
variations of a "RAID10" configuration (nested 1+0, nested 0+1,
complex using near/2) and all yield very similar results: Good read
performance, extremely poor write performance.
Even the throughput when doing a sequential test with the writes is
not where I'd expect it to be, so something definitely seems to be up
when mixing RAID levels 0 and 1. I didn't explore all the extremes of
the chunk sizes, so perhaps its as simple as that? I haven't tested
the "far" and "offset" layouts of RAID10 yet, but I'm not hopeful its
going to be any different.
Here is what I'm using:
Linux 3.14.57 (vanilla)
mdadm - v3.3.2 - 21st August 2014
fio-2.0.13
Any ideas or suggestions would be greatly appreciated. Just as a
simple test, I created a RAID5 volume using (4) of the SSDs and ran
the same random IO performance tests:
4K random read: 169,026 IOPS
4K random write: 12,682 IOPS
Not sure with the default RAID5 mdadm creation command that we get any
write cache, but we're getting ~ 12K IOPS with RAID5. Not great, but
when compared to the 643 IOPS with RAID10...
Thanks in advance!
--Marc
^ permalink raw reply
* Re: [PATCH 3/3] raid5: allow r5l_io_unit allocations to fail
From: Shaohua Li @ 2015-12-18 23:07 UTC (permalink / raw)
To: Christoph Hellwig; +Cc: NeilBrown, linux-raid
In-Reply-To: <20151218112535.GB28224@lst.de>
On Fri, Dec 18, 2015 at 12:25:35PM +0100, Christoph Hellwig wrote:
> [can you trim your replies please? I had to trim almost 150 lines of
> junk before getting to your reply!]
>
> > > spin_unlock_irqrestore(&log->io_list_lock, flags);
> > >
> > > or is that too simplistic?
> >
> > maybe add a new list and run the list at the begining of r5l_do_reclaim().
>
> What's the advantage of another list?
I mean simply waking up reclaim thread doesn't work as r5l_do_reclaim()
will return if reclaimable == 0. There is no guarantee reclaimable space
is available in the allocation failure. So we'd better move the retry at
the begining of r5l_do_reclaim(). If yes, we can't reuse the
no_space_stripes list.
Thanks,
Shaohua
^ permalink raw reply
* RAID6 - whole disk vs partitions
From: Steven Haigh @ 2015-12-19 14:34 UTC (permalink / raw)
To: linux-raid
[-- Attachment #1: Type: text/plain, Size: 1161 bytes --]
Hi all,
I'm revisiting this topic after a few years - so excuse me if I'm a
little behind the times on this questioning.
I'm just starting to replace some 5 year old disks in a RAID6 (5 disks)
due to one having started to throw uncorrectable / reallocated sectors.
As such, it seems a good time to go to 3Tb drives - which are only $20
more than 2Tb drives.
The array is created as follows:
md2 : active raid6 sdf[0] sde[6] sdg[7] sdd[5] sdc[8]
5860150272 blocks super 1.2 level 6, 128k chunk, algorithm 2 [5/5]
[UUUUU]
I've always used the whole disk for the RAID6 - as from what I
understand, this gives the best layout for various drives - no matter if
they're 512 or 4096 byte sectors.
Is this still the case?
I've noticed that my resync speeds on this RAID6 hover around 60Mb/sec -
which I was expecting to be a bit quicker (although I haven't tested
speeds on this array for years).
Currently using kernel 4.1.15 on an EL6 install.
Would be grateful for any input people may have on these topics.
--
Steven Haigh
Email: netwiz@crc.id.au
Web: https://www.crc.id.au
Phone: (03) 9001 6090 - 0412 935 897
[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 819 bytes --]
^ permalink raw reply
* Re: RAID6 - whole disk vs partitions
From: Phil Turmel @ 2015-12-19 20:36 UTC (permalink / raw)
To: Steven Haigh, linux-raid
In-Reply-To: <56756AF1.7050005@crc.id.au>
Hi Steven,
On 12/19/2015 09:34 AM, Steven Haigh wrote:
> I've always used the whole disk for the RAID6 - as from what I
> understand, this gives the best layout for various drives - no matter if
> they're 512 or 4096 byte sectors.
>
> Is this still the case?
There's no real advantage other than an itty bit of space. If my case
was too small to have a separate mirror for my OS, I'd make three
partitions on everything for boot, LVM root & etc (raid10), and bulk
storage (raid6). When I have room for the mirrored OS (boot & LVM), I
use whole disks for the bulk storage.
> I've noticed that my resync speeds on this RAID6 hover around 60Mb/sec -
> which I was expecting to be a bit quicker (although I haven't tested
> speeds on this array for years).
Consider using small chunk sizes for parity raid if there's even the
slightest random access usage. See the list archives for the past
couple weeks for a long thread on performance. There's many
optimization possibilities.
Phil
^ permalink raw reply
* ATTENTION!!!!!!
From: gh13 @ 2015-12-19 20:56 UTC (permalink / raw)
To: Recipients
Dear Beneficiary,
This is to officially inform you that we have written to you before without getting respond from you and we believe that our previous mail did not get to you therefore we write you again. We are contacting you concerning the release of your inheritance fund / Draft/Cheque /ATM Card which have been delayed for transfer by some officials who claim to be in position of your fund thereby extorting money from you in one way or the other.
Your Fund has finally been approved for transfer by the West Africa Fund Monitoring Unit. Your fund will be transfer to you via MasterCard ATM which is cashable in any ATM machine or Bank anywhere in the world.
We hereby inform you that the ATM card worth US$8.5, 000.000.00 has been credited in your favour as the first part payment of your inheritance fund which has been delayed by these officers who claim to be in position of your fund.
Therefore you are warned to stop any further communication with anybody concerning your inheritance fund.
Your fund to be released via MasterCard ATM in act to uphold the rule of law which we represent.You have to reconfirm the informations below for security reasons. The only money you are obliged to pay is the delivery charges only .
Contact the verification officer incharge of the delivery:
Name: Frank David
E-mail: frank.david849@yahoo.com
Telephone +2348140092560
Send them the following informations of yours for the conclusion of your ATM Card:
Full Name:__________
Delivery Address:____
Country:_______
Occupation:_____
Phone Number:____
Age:______
sex:______
Regards,
Frank David
^ permalink raw reply
* [GIT PULL REQUEST] md fixes for 4.4-rc
From: NeilBrown @ 2015-12-19 22:15 UTC (permalink / raw)
To: Linus Torvalds
Cc: LKML, linux-raid, Artur Paszkiewicz, Goldwyn Rodrigues,
Mikulas Patocka, Pawel Baldysiak, Shaohua Li
[-- Attachment #1: Type: text/plain, Size: 1157 bytes --]
The following changes since commit 9f9499ae8e6415cefc4fe0a96ad0e27864353c89:
Linux 4.4-rc5 (2015-12-13 17:42:58 -0800)
are available in the git repository at:
git://neil.brown.name/md tags/md/4.4-rc5-fixes
for you to fetch changes up to cb01c5496d2d9c0c862443561df16ff122db348f:
Fix remove_and_add_spares removes drive added as spare in slot_store (2015-12-18 15:19:16 +1100)
----------------------------------------------------------------
4 fixes for md in 4.4-rc5
- 2 recently introduced regressions fixed.
- one older bug in RAID10 - tagged for -stable since 4.2
- one minor sysfs api improvement.
----------------------------------------------------------------
Artur Paszkiewicz (1):
md/raid10: fix data corruption and crash during resync
Goldwyn Rodrigues (1):
Fix remove_and_add_spares removes drive added as spare in slot_store
Mikulas Patocka (1):
md: fix bug due to nested suspend
Shaohua Li (1):
MD: change journal disk role to disk 0
drivers/md/md.c | 22 +++++++++++++++-------
drivers/md/md.h | 8 ++++++--
drivers/md/raid10.c | 4 +++-
3 files changed, 24 insertions(+), 10 deletions(-)
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 818 bytes --]
^ permalink raw reply
* Re: RAID6 - whole disk vs partitions
From: Steven Haigh @ 2015-12-19 22:21 UTC (permalink / raw)
To: Phil Turmel, linux-raid
In-Reply-To: <5675BFBE.5060604@turmel.org>
[-- Attachment #1: Type: text/plain, Size: 1854 bytes --]
On 20/12/15 07:36, Phil Turmel wrote:
> Hi Steven,
>
> On 12/19/2015 09:34 AM, Steven Haigh wrote:
>
>> I've always used the whole disk for the RAID6 - as from what I
>> understand, this gives the best layout for various drives - no matter if
>> they're 512 or 4096 byte sectors.
>>
>> Is this still the case?
>
> There's no real advantage other than an itty bit of space. If my case
> was too small to have a separate mirror for my OS, I'd make three
> partitions on everything for boot, LVM root & etc (raid10), and bulk
> storage (raid6). When I have room for the mirrored OS (boot & LVM), I
> use whole disks for the bulk storage.
Yes, I have two separate disks in RAID1 as the boot / VM guests - the
RAID6 is used for bulk storage. The chassis has a total of 10 hot swap
bays, with 5 used by the RAID6, 2 by the RAID1 and 3 spare. I get resync
speeds of around 170Mb/sec on the RAID1.
>> I've noticed that my resync speeds on this RAID6 hover around 60Mb/sec -
>> which I was expecting to be a bit quicker (although I haven't tested
>> speeds on this array for years).
>
> Consider using small chunk sizes for parity raid if there's even the
> slightest random access usage. See the list archives for the past
> couple weeks for a long thread on performance. There's many
> optimization possibilities.
I have tried to keep up with those couple of threads - but I've found it
hard to follow across the length of time they've been going. A summary
of the findings in those threads with experimentation would be fantastic!
It seems that actual data beyond "increase --setra on /dev/sd[.] and
increase the stripe_cache_size is about all I find via Google - and that
data is years old...
--
Steven Haigh
Email: netwiz@crc.id.au
Web: https://www.crc.id.au
Phone: (03) 9001 6090 - 0412 935 897
[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 819 bytes --]
^ permalink raw reply
* JUST REPLY YES ONLY
From: Richard @ 2015-12-20 9:51 UTC (permalink / raw)
To: My-mail-box
Hello
My proposal will give us 2 million in seven days reply "YES" for details.
Regards,
Richard.
^ permalink raw reply
* Re: [PATCH 3/3] raid5: allow r5l_io_unit allocations to fail
From: NeilBrown @ 2015-12-20 22:51 UTC (permalink / raw)
Cc: Shaohua Li, Christoph Hellwig, linux-raid
In-Reply-To: <20151218112312.GA28224@lst.de>
[-- Attachment #1: Type: text/plain, Size: 2724 bytes --]
On Fri, Dec 18 2015, Christoph Hellwig wrote:
> On Fri, Dec 18, 2015 at 12:51:07PM +1100, NeilBrown wrote:
>> > if the reclaim thread doesn't have anything to reclaim,
>> > r5l_run_no_space_stripes isn't called. we might miss the retry.
>>
>> so something like this:
>
> that looks fine to me. I'll give a spin on my QA setup.
>
>> > I'm a little worrying about the GFP_ATOMIC allocation. In the first try,
>> > GFP_NOWAIT is better. And on the other hand, why sleep is bad here? We
>> > could use GFP_NOIO | __GFP_NORETRY, there is no deadlock risk.
>> >
>> > In the retry, GFP_NOIO looks better. No deadlock too, since it's not
>> > called from raid5d (maybe we shouldn't call from reclaim thread if using
>> > GFP_NOIO, a workqueue is better). Otherwise we could keep retring but do
>> > nothing.
>>
>> I did wonder a little bit about that.
>> GFP_ATOMIC is (__GFP_HIGH)
>> GFP_NOIO | __GFP_NORETRY is (__GFP_WAIT | __GFP_NORETRY)
>>
>> It isn't clear that we need 'HIGH', and WAIT with NORETRY should be OK.
>> It allows __alloc_pages_direct_reclaim, but only once and never waits
>> for other IO.
>
> In general we go for HIGH on non-sleeping allocation to avoid having
> the stalled. This is especially important in the I/O path.
>
> WAIT means we will reclaim and wait for it, which looks a little dangerous
> to me. In general I'd prefer not to use obscure gfp flag combination
> unless there is a real need, and it's clearly documented.
I don't think "will reclaim and wait for it" is accurate.
Certainly page_alloc will only try direct reclaim when WAIT is set, but
I don't think it actually waits for anything particular.
There are various waits in the direct reclaim path such as
throttle_vm_writeout(), but they seem to be throttling delays rather
than "wait for something particular" delays.
Also direct reclaim (e.g. in shrinkers) are allows to sleep (as long as
they don't violate NOFS or NOIO), so waiting can definitely happen for
non-throttling reasons.
I *think* (not 100% sure) that __GFP_WAIT means that the alloc code is
allowed to schedule(). Without that flag it mustn't.
So it makes sense to use WAIT when it is appropriate for a delay to be
introduce to throttle allocations. I don't think that is the case in
raid5d. raid5d is too low level - throttling it will not directly
affect higher level allocations.
So while my reasoning is a bit different, I agree that we don't really
want WAIT in an allocation in raid5d.
.... oh. I just noticed that __GFP_WAIT was renamed to __GFP_RECLAIM
last month. And there is a new __GFP_DIRECT_RECLAIM. I think I'm going
to have to learn how this stuff works again.
NeilBrown
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 818 bytes --]
^ permalink raw reply
* Re: [PATCH 3/3] raid5: allow r5l_io_unit allocations to fail
From: NeilBrown @ 2015-12-20 22:59 UTC (permalink / raw)
To: Shaohua Li, Christoph Hellwig; +Cc: linux-raid
In-Reply-To: <20151218230657.GA342953@devbig084.prn1.facebook.com>
[-- Attachment #1: Type: text/plain, Size: 1950 bytes --]
On Sat, Dec 19 2015, Shaohua Li wrote:
> On Fri, Dec 18, 2015 at 12:25:35PM +0100, Christoph Hellwig wrote:
>> [can you trim your replies please? I had to trim almost 150 lines of
>> junk before getting to your reply!]
>>
>> > > spin_unlock_irqrestore(&log->io_list_lock, flags);
>> > >
>> > > or is that too simplistic?
>> >
>> > maybe add a new list and run the list at the begining of r5l_do_reclaim().
>>
>> What's the advantage of another list?
>
> I mean simply waking up reclaim thread doesn't work as r5l_do_reclaim()
> will return if reclaimable == 0. There is no guarantee reclaimable space
> is available in the allocation failure. So we'd better move the retry at
> the begining of r5l_do_reclaim(). If yes, we can't reuse the
> no_space_stripes list.
They certainly are conceptually different lists. Whether the same
linked list can be used for both is an engineering question.
One list is for updates that cannot be handled yet because the log is
full. Once the head of the log is cleared and the start pointer
updated, those requests can be handled (or some of them can).
The other list is for updates that cannot be handled yet because a
kmalloc failed. There is no clear trigger for when to handle those
again so we would need to retry quite frequently. It would be easy to
miss retrying in rare circumstances...
I wonder if we should have a mempool for these io units too.
We would allocate with GFP_ATOMIC (or similar) so the allocation woult
fail instead of blocking, but we would then know that an allocation
could only fail if there was another request in flight. So the place
where we free an io_unit would be the obviously correct place to trigger
a retry of the delayed-due-to-mem-allocation-failure stripes.
So I think I would prefer two lists, another mempool, and very well
defined places to retry the two lists. Is that over-engineering?
Thanks,
NeilBrown
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 818 bytes --]
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox