From: "Javier González" <jg@lightnvm.io>
To: axboe@fb.com
Cc: linux-block@vger.kernel.org, linux-kernel@vger.kernel.org,
"Javier González" <javier@cnexlabs.com>
Subject: [PATCH 2/5] lightnvm: pblk: simplify work_queue mempool
Date: Thu, 14 Sep 2017 12:33:44 +0200 [thread overview]
Message-ID: <1505385227-28706-3-git-send-email-javier@cnexlabs.com> (raw)
In-Reply-To: <1505385227-28706-1-git-send-email-javier@cnexlabs.com>
In pblk, we have a mempool to allocate a generic structure that we
pass along workqueues. This is heavily used in the GC path in order
to have enough inflight reads and fully utilize the GC bandwidth.
However, the current GC path copies data to the host memory and puts it
back into the write buffer. This requires a vmalloc allocation for the
data and a memory copy. Thus, guaranteeing the allocation by using a
mempool for the structure in itself does not give us much. Until we
implement support for vector copy to avoid moving data through the host,
just allocate the workqueue structure using kmalloc.
This allows us to have a much smaller mempool.
Reported-by: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Javier González <javier@cnexlabs.com>
---
drivers/lightnvm/pblk-core.c | 13 +++++++------
drivers/lightnvm/pblk-gc.c | 32 ++++++++++++++++----------------
drivers/lightnvm/pblk-init.c | 32 ++++++++++++++++----------------
drivers/lightnvm/pblk-write.c | 4 ++--
drivers/lightnvm/pblk.h | 11 ++++++-----
5 files changed, 47 insertions(+), 45 deletions(-)
diff --git a/drivers/lightnvm/pblk-core.c b/drivers/lightnvm/pblk-core.c
index b114b57af0c3..b7a6b223b1a5 100644
--- a/drivers/lightnvm/pblk-core.c
+++ b/drivers/lightnvm/pblk-core.c
@@ -33,7 +33,8 @@ static void pblk_mark_bb(struct pblk *pblk, struct pblk_line *line,
pr_err("pblk: attempted to erase bb: line:%d, pos:%d\n",
line->id, pos);
- pblk_line_run_ws(pblk, NULL, ppa, pblk_line_mark_bb, pblk->bb_wq);
+ pblk_gen_run_ws(pblk, NULL, ppa, pblk_line_mark_bb,
+ GFP_ATOMIC, pblk->bb_wq);
}
static void __pblk_end_io_erase(struct pblk *pblk, struct nvm_rq *rqd)
@@ -1627,7 +1628,7 @@ void pblk_line_close_ws(struct work_struct *work)
struct pblk_line *line = line_ws->line;
pblk_line_close(pblk, line);
- mempool_free(line_ws, pblk->line_ws_pool);
+ mempool_free(line_ws, pblk->gen_ws_pool);
}
void pblk_line_mark_bb(struct work_struct *work)
@@ -1652,16 +1653,16 @@ void pblk_line_mark_bb(struct work_struct *work)
}
kfree(ppa);
- mempool_free(line_ws, pblk->line_ws_pool);
+ mempool_free(line_ws, pblk->gen_ws_pool);
}
-void pblk_line_run_ws(struct pblk *pblk, struct pblk_line *line, void *priv,
- void (*work)(struct work_struct *),
+void pblk_gen_run_ws(struct pblk *pblk, struct pblk_line *line, void *priv,
+ void (*work)(struct work_struct *), gfp_t gfp_mask,
struct workqueue_struct *wq)
{
struct pblk_line_ws *line_ws;
- line_ws = mempool_alloc(pblk->line_ws_pool, GFP_ATOMIC);
+ line_ws = mempool_alloc(pblk->gen_ws_pool, gfp_mask);
if (!line_ws)
return;
diff --git a/drivers/lightnvm/pblk-gc.c b/drivers/lightnvm/pblk-gc.c
index 6090d28f7995..f163829ecca8 100644
--- a/drivers/lightnvm/pblk-gc.c
+++ b/drivers/lightnvm/pblk-gc.c
@@ -136,12 +136,12 @@ static void pblk_put_line_back(struct pblk *pblk, struct pblk_line *line)
static void pblk_gc_line_ws(struct work_struct *work)
{
- struct pblk_line_ws *line_rq_ws = container_of(work,
+ struct pblk_line_ws *gc_rq_ws = container_of(work,
struct pblk_line_ws, ws);
- struct pblk *pblk = line_rq_ws->pblk;
+ struct pblk *pblk = gc_rq_ws->pblk;
struct pblk_gc *gc = &pblk->gc;
- struct pblk_line *line = line_rq_ws->line;
- struct pblk_gc_rq *gc_rq = line_rq_ws->priv;
+ struct pblk_line *line = gc_rq_ws->line;
+ struct pblk_gc_rq *gc_rq = gc_rq_ws->priv;
up(&gc->gc_sem);
@@ -151,7 +151,7 @@ static void pblk_gc_line_ws(struct work_struct *work)
gc_rq->nr_secs);
}
- mempool_free(line_rq_ws, pblk->line_ws_pool);
+ kfree(gc_rq_ws);
}
static void pblk_gc_line_prepare_ws(struct work_struct *work)
@@ -164,7 +164,7 @@ static void pblk_gc_line_prepare_ws(struct work_struct *work)
struct pblk_line_meta *lm = &pblk->lm;
struct pblk_gc *gc = &pblk->gc;
struct line_emeta *emeta_buf;
- struct pblk_line_ws *line_rq_ws;
+ struct pblk_line_ws *gc_rq_ws;
struct pblk_gc_rq *gc_rq;
__le64 *lba_list;
int sec_left, nr_secs, bit;
@@ -223,19 +223,19 @@ static void pblk_gc_line_prepare_ws(struct work_struct *work)
gc_rq->nr_secs = nr_secs;
gc_rq->line = line;
- line_rq_ws = mempool_alloc(pblk->line_ws_pool, GFP_KERNEL);
- if (!line_rq_ws)
+ gc_rq_ws = kmalloc(sizeof(struct pblk_line_ws), GFP_KERNEL);
+ if (!gc_rq_ws)
goto fail_free_gc_rq;
- line_rq_ws->pblk = pblk;
- line_rq_ws->line = line;
- line_rq_ws->priv = gc_rq;
+ gc_rq_ws->pblk = pblk;
+ gc_rq_ws->line = line;
+ gc_rq_ws->priv = gc_rq;
down(&gc->gc_sem);
kref_get(&line->ref);
- INIT_WORK(&line_rq_ws->ws, pblk_gc_line_ws);
- queue_work(gc->gc_line_reader_wq, &line_rq_ws->ws);
+ INIT_WORK(&gc_rq_ws->ws, pblk_gc_line_ws);
+ queue_work(gc->gc_line_reader_wq, &gc_rq_ws->ws);
sec_left -= nr_secs;
if (sec_left > 0)
@@ -243,7 +243,7 @@ static void pblk_gc_line_prepare_ws(struct work_struct *work)
out:
pblk_mfree(emeta_buf, l_mg->emeta_alloc_type);
- mempool_free(line_ws, pblk->line_ws_pool);
+ kfree(line_ws);
kref_put(&line->ref, pblk_line_put);
atomic_dec(&gc->inflight_gc);
@@ -256,7 +256,7 @@ static void pblk_gc_line_prepare_ws(struct work_struct *work)
pblk_mfree(emeta_buf, l_mg->emeta_alloc_type);
pblk_put_line_back(pblk, line);
kref_put(&line->ref, pblk_line_put);
- mempool_free(line_ws, pblk->line_ws_pool);
+ kfree(line_ws);
atomic_dec(&gc->inflight_gc);
pr_err("pblk: Failed to GC line %d\n", line->id);
@@ -269,7 +269,7 @@ static int pblk_gc_line(struct pblk *pblk, struct pblk_line *line)
pr_debug("pblk: line '%d' being reclaimed for GC\n", line->id);
- line_ws = mempool_alloc(pblk->line_ws_pool, GFP_KERNEL);
+ line_ws = kmalloc(sizeof(struct pblk_line_ws), GFP_KERNEL);
if (!line_ws)
return -ENOMEM;
diff --git a/drivers/lightnvm/pblk-init.c b/drivers/lightnvm/pblk-init.c
index 19cb1906a56a..0c65abb96dc6 100644
--- a/drivers/lightnvm/pblk-init.c
+++ b/drivers/lightnvm/pblk-init.c
@@ -20,7 +20,7 @@
#include "pblk.h"
-static struct kmem_cache *pblk_blk_ws_cache, *pblk_rec_cache, *pblk_g_rq_cache,
+static struct kmem_cache *pblk_ws_cache, *pblk_rec_cache, *pblk_g_rq_cache,
*pblk_w_rq_cache, *pblk_line_meta_cache;
static DECLARE_RWSEM(pblk_lock);
struct bio_set *pblk_bio_set;
@@ -184,9 +184,9 @@ static int pblk_init_global_caches(struct pblk *pblk)
char cache_name[PBLK_CACHE_NAME_LEN];
down_write(&pblk_lock);
- pblk_blk_ws_cache = kmem_cache_create("pblk_blk_ws",
+ pblk_ws_cache = kmem_cache_create("pblk_blk_ws",
sizeof(struct pblk_line_ws), 0, 0, NULL);
- if (!pblk_blk_ws_cache) {
+ if (!pblk_ws_cache) {
up_write(&pblk_lock);
return -ENOMEM;
}
@@ -194,7 +194,7 @@ static int pblk_init_global_caches(struct pblk *pblk)
pblk_rec_cache = kmem_cache_create("pblk_rec",
sizeof(struct pblk_rec_ctx), 0, 0, NULL);
if (!pblk_rec_cache) {
- kmem_cache_destroy(pblk_blk_ws_cache);
+ kmem_cache_destroy(pblk_ws_cache);
up_write(&pblk_lock);
return -ENOMEM;
}
@@ -202,7 +202,7 @@ static int pblk_init_global_caches(struct pblk *pblk)
pblk_g_rq_cache = kmem_cache_create("pblk_g_rq", pblk_g_rq_size,
0, 0, NULL);
if (!pblk_g_rq_cache) {
- kmem_cache_destroy(pblk_blk_ws_cache);
+ kmem_cache_destroy(pblk_ws_cache);
kmem_cache_destroy(pblk_rec_cache);
up_write(&pblk_lock);
return -ENOMEM;
@@ -211,7 +211,7 @@ static int pblk_init_global_caches(struct pblk *pblk)
pblk_w_rq_cache = kmem_cache_create("pblk_w_rq", pblk_w_rq_size,
0, 0, NULL);
if (!pblk_w_rq_cache) {
- kmem_cache_destroy(pblk_blk_ws_cache);
+ kmem_cache_destroy(pblk_ws_cache);
kmem_cache_destroy(pblk_rec_cache);
kmem_cache_destroy(pblk_g_rq_cache);
up_write(&pblk_lock);
@@ -223,7 +223,7 @@ static int pblk_init_global_caches(struct pblk *pblk)
pblk_line_meta_cache = kmem_cache_create(cache_name,
pblk->lm.sec_bitmap_len, 0, 0, NULL);
if (!pblk_line_meta_cache) {
- kmem_cache_destroy(pblk_blk_ws_cache);
+ kmem_cache_destroy(pblk_ws_cache);
kmem_cache_destroy(pblk_rec_cache);
kmem_cache_destroy(pblk_g_rq_cache);
kmem_cache_destroy(pblk_w_rq_cache);
@@ -246,20 +246,20 @@ static int pblk_core_init(struct pblk *pblk)
if (pblk_init_global_caches(pblk))
return -ENOMEM;
- /* internal bios can be at most the sectors signaled by the device. */
+ /* Internal bios can be at most the sectors signaled by the device. */
pblk->page_bio_pool = mempool_create_page_pool(nvm_max_phys_sects(dev),
0);
if (!pblk->page_bio_pool)
return -ENOMEM;
- pblk->line_ws_pool = mempool_create_slab_pool(PBLK_WS_POOL_SIZE,
- pblk_blk_ws_cache);
- if (!pblk->line_ws_pool)
+ pblk->gen_ws_pool = mempool_create_slab_pool(PBLK_GEN_WS_POOL_SIZE,
+ pblk_ws_cache);
+ if (!pblk->gen_ws_pool)
goto free_page_bio_pool;
pblk->rec_pool = mempool_create_slab_pool(geo->nr_luns, pblk_rec_cache);
if (!pblk->rec_pool)
- goto free_blk_ws_pool;
+ goto free_gen_ws_pool;
pblk->g_rq_pool = mempool_create_slab_pool(PBLK_READ_REQ_POOL_SIZE,
pblk_g_rq_cache);
@@ -308,8 +308,8 @@ static int pblk_core_init(struct pblk *pblk)
mempool_destroy(pblk->g_rq_pool);
free_rec_pool:
mempool_destroy(pblk->rec_pool);
-free_blk_ws_pool:
- mempool_destroy(pblk->line_ws_pool);
+free_gen_ws_pool:
+ mempool_destroy(pblk->gen_ws_pool);
free_page_bio_pool:
mempool_destroy(pblk->page_bio_pool);
return -ENOMEM;
@@ -324,13 +324,13 @@ static void pblk_core_free(struct pblk *pblk)
destroy_workqueue(pblk->bb_wq);
mempool_destroy(pblk->page_bio_pool);
- mempool_destroy(pblk->line_ws_pool);
+ mempool_destroy(pblk->gen_ws_pool);
mempool_destroy(pblk->rec_pool);
mempool_destroy(pblk->g_rq_pool);
mempool_destroy(pblk->w_rq_pool);
mempool_destroy(pblk->line_meta_pool);
- kmem_cache_destroy(pblk_blk_ws_cache);
+ kmem_cache_destroy(pblk_ws_cache);
kmem_cache_destroy(pblk_rec_cache);
kmem_cache_destroy(pblk_g_rq_cache);
kmem_cache_destroy(pblk_w_rq_cache);
diff --git a/drivers/lightnvm/pblk-write.c b/drivers/lightnvm/pblk-write.c
index b48d52b2ae38..81d1f1035fa3 100644
--- a/drivers/lightnvm/pblk-write.c
+++ b/drivers/lightnvm/pblk-write.c
@@ -201,8 +201,8 @@ static void pblk_end_io_write_meta(struct nvm_rq *rqd)
sync = atomic_add_return(rqd->nr_ppas, &emeta->sync);
if (sync == emeta->nr_entries)
- pblk_line_run_ws(pblk, line, NULL, pblk_line_close_ws,
- pblk->close_wq);
+ pblk_gen_run_ws(pblk, line, NULL, pblk_line_close_ws,
+ GFP_ATOMIC, pblk->close_wq);
bio_put(rqd->bio);
nvm_dev_dma_free(dev->parent, rqd->meta_list, rqd->dma_meta_list);
diff --git a/drivers/lightnvm/pblk.h b/drivers/lightnvm/pblk.h
index b2df4e707c36..dee8f66e6ce2 100644
--- a/drivers/lightnvm/pblk.h
+++ b/drivers/lightnvm/pblk.h
@@ -40,7 +40,6 @@
#define PBLK_MAX_REQ_ADDRS (64)
#define PBLK_MAX_REQ_ADDRS_PW (6)
-#define PBLK_WS_POOL_SIZE (128)
#define PBLK_META_POOL_SIZE (128)
#define PBLK_READ_REQ_POOL_SIZE (1024)
@@ -61,6 +60,8 @@
#define ERASE 2 /* READ = 0, WRITE = 1 */
+#define PBLK_GEN_WS_POOL_SIZE (2)
+
enum {
/* IO Types */
PBLK_IOTYPE_USER = 1 << 0,
@@ -620,7 +621,7 @@ struct pblk {
struct list_head compl_list;
mempool_t *page_bio_pool;
- mempool_t *line_ws_pool;
+ mempool_t *gen_ws_pool;
mempool_t *rec_pool;
mempool_t *g_rq_pool;
mempool_t *w_rq_pool;
@@ -724,9 +725,9 @@ void pblk_line_close_meta_sync(struct pblk *pblk);
void pblk_line_close_ws(struct work_struct *work);
void pblk_pipeline_stop(struct pblk *pblk);
void pblk_line_mark_bb(struct work_struct *work);
-void pblk_line_run_ws(struct pblk *pblk, struct pblk_line *line, void *priv,
- void (*work)(struct work_struct *),
- struct workqueue_struct *wq);
+void pblk_gen_run_ws(struct pblk *pblk, struct pblk_line *line, void *priv,
+ void (*work)(struct work_struct *), gfp_t gfp_mask,
+ struct workqueue_struct *wq);
u64 pblk_line_smeta_start(struct pblk *pblk, struct pblk_line *line);
int pblk_line_read_smeta(struct pblk *pblk, struct pblk_line *line);
int pblk_line_read_emeta(struct pblk *pblk, struct pblk_line *line,
--
2.7.4
next prev parent reply other threads:[~2017-09-14 10:34 UTC|newest]
Thread overview: 7+ messages / expand[flat|nested] mbox.gz Atom feed top
2017-09-14 10:33 [PATCH 0/5] lightnvm: pblk: audit mempool usage Javier González
2017-09-14 10:33 ` [PATCH 1/5] lightnvm: pblk: fix min size for page mempool Javier González
2017-09-14 10:33 ` Javier González [this message]
2017-09-14 10:33 ` [PATCH 3/5] lightnvm: pblk: decouple read/erase mempools Javier González
2017-09-14 10:33 ` [PATCH 4/5] lightnvm: pblk: do not use a mempool for line bitmaps Javier González
2017-09-14 10:33 ` [PATCH 5/5] lightnvm: pblk: remove checks on mempool alloc Javier González
2017-10-02 17:58 ` [PATCH 0/5] lightnvm: pblk: audit mempool usage Rakesh Pandit
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1505385227-28706-3-git-send-email-javier@cnexlabs.com \
--to=jg@lightnvm.io \
--cc=axboe@fb.com \
--cc=javier@cnexlabs.com \
--cc=linux-block@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).