* [PATCH V2 01/11] block-throttle: prepare support multiple limits
2016-09-15 16:23 [PATCH V2 00/11] block-throttle: add .high limit Shaohua Li
@ 2016-09-15 16:23 ` Shaohua Li
2016-09-15 16:23 ` [PATCH V2 02/11] block-throttle: add .high interface Shaohua Li
` (9 subsequent siblings)
10 siblings, 0 replies; 13+ messages in thread
From: Shaohua Li @ 2016-09-15 16:23 UTC (permalink / raw)
To: linux-block, linux-kernel; +Cc: axboe, Kernel-team, tj, jmoyer, vgoyal
We are going to support high/max limit, each cgroup will have 2 limits
after that. This patch prepares for the multiple limits change.
Signed-off-by: Shaohua Li <shli@fb.com>
---
block/blk-throttle.c | 109 ++++++++++++++++++++++++++++++++-------------------
1 file changed, 68 insertions(+), 41 deletions(-)
diff --git a/block/blk-throttle.c b/block/blk-throttle.c
index f1aba26..71ecee7 100644
--- a/block/blk-throttle.c
+++ b/block/blk-throttle.c
@@ -83,6 +83,11 @@ enum tg_state_flags {
#define rb_entry_tg(node) rb_entry((node), struct throtl_grp, rb_node)
+enum {
+ LIMIT_MAX = 0,
+ LIMIT_CNT = 1,
+};
+
struct throtl_grp {
/* must be the first member */
struct blkg_policy_data pd;
@@ -120,10 +125,10 @@ struct throtl_grp {
bool has_rules[2];
/* bytes per second rate limits */
- uint64_t bps[2];
+ uint64_t bps[2][LIMIT_CNT];
/* IOPS limits */
- unsigned int iops[2];
+ unsigned int iops[2][LIMIT_CNT];
/* Number of bytes disptached in current slice */
uint64_t bytes_disp[2];
@@ -147,6 +152,8 @@ struct throtl_data
/* Work for dispatching throttled bios */
struct work_struct dispatch_work;
+ unsigned int limit_index;
+ bool limit_valid[LIMIT_CNT];
};
static void throtl_pending_timer_fn(unsigned long arg);
@@ -198,6 +205,16 @@ static struct throtl_data *sq_to_td(struct throtl_service_queue *sq)
return container_of(sq, struct throtl_data, service_queue);
}
+static uint64_t tg_bps_limit(struct throtl_grp *tg, int rw)
+{
+ return tg->bps[rw][tg->td->limit_index];
+}
+
+static unsigned int tg_iops_limit(struct throtl_grp *tg, int rw)
+{
+ return tg->iops[rw][tg->td->limit_index];
+}
+
/**
* throtl_log - log debug message via blktrace
* @sq: the service_queue being reported
@@ -320,7 +337,7 @@ static void throtl_service_queue_init(struct throtl_service_queue *sq)
static struct blkg_policy_data *throtl_pd_alloc(gfp_t gfp, int node)
{
struct throtl_grp *tg;
- int rw;
+ int rw, index;
tg = kzalloc_node(sizeof(*tg), gfp, node);
if (!tg)
@@ -334,10 +351,12 @@ static struct blkg_policy_data *throtl_pd_alloc(gfp_t gfp, int node)
}
RB_CLEAR_NODE(&tg->rb_node);
- tg->bps[READ] = -1;
- tg->bps[WRITE] = -1;
- tg->iops[READ] = -1;
- tg->iops[WRITE] = -1;
+ for (rw = READ; rw <= WRITE; rw++) {
+ for (index = LIMIT_MAX; index < LIMIT_CNT; index++) {
+ tg->bps[rw][index] = -1;
+ tg->iops[rw][index] = -1;
+ }
+ }
return &tg->pd;
}
@@ -376,11 +395,14 @@ static void throtl_pd_init(struct blkg_policy_data *pd)
static void tg_update_has_rules(struct throtl_grp *tg)
{
struct throtl_grp *parent_tg = sq_to_tg(tg->service_queue.parent_sq);
+ struct throtl_data *td = tg->td;
int rw;
for (rw = READ; rw <= WRITE; rw++)
tg->has_rules[rw] = (parent_tg && parent_tg->has_rules[rw]) ||
- (tg->bps[rw] != -1 || tg->iops[rw] != -1);
+ (td->limit_valid[td->limit_index] &&
+ (tg_bps_limit(tg, rw) != -1 ||
+ tg_iops_limit(tg, rw) != -1));
}
static void throtl_pd_online(struct blkg_policy_data *pd)
@@ -632,11 +654,11 @@ static inline void throtl_trim_slice(struct throtl_grp *tg, bool rw)
if (!nr_slices)
return;
- tmp = tg->bps[rw] * throtl_slice * nr_slices;
+ tmp = tg_bps_limit(tg, rw) * throtl_slice * nr_slices;
do_div(tmp, HZ);
bytes_trim = tmp;
- io_trim = (tg->iops[rw] * throtl_slice * nr_slices)/HZ;
+ io_trim = (tg_iops_limit(tg, rw) * throtl_slice * nr_slices)/HZ;
if (!bytes_trim && !io_trim)
return;
@@ -682,7 +704,7 @@ static bool tg_with_in_iops_limit(struct throtl_grp *tg, struct bio *bio,
* have been trimmed.
*/
- tmp = (u64)tg->iops[rw] * jiffy_elapsed_rnd;
+ tmp = (u64)tg_iops_limit(tg, rw) * jiffy_elapsed_rnd;
do_div(tmp, HZ);
if (tmp > UINT_MAX)
@@ -697,7 +719,7 @@ static bool tg_with_in_iops_limit(struct throtl_grp *tg, struct bio *bio,
}
/* Calc approx time to dispatch */
- jiffy_wait = ((tg->io_disp[rw] + 1) * HZ)/tg->iops[rw] + 1;
+ jiffy_wait = ((tg->io_disp[rw] + 1) * HZ)/tg_iops_limit(tg, rw) + 1;
if (jiffy_wait > jiffy_elapsed)
jiffy_wait = jiffy_wait - jiffy_elapsed;
@@ -724,7 +746,7 @@ static bool tg_with_in_bps_limit(struct throtl_grp *tg, struct bio *bio,
jiffy_elapsed_rnd = roundup(jiffy_elapsed_rnd, throtl_slice);
- tmp = tg->bps[rw] * jiffy_elapsed_rnd;
+ tmp = tg_bps_limit(tg, rw) * jiffy_elapsed_rnd;
do_div(tmp, HZ);
bytes_allowed = tmp;
@@ -736,7 +758,7 @@ static bool tg_with_in_bps_limit(struct throtl_grp *tg, struct bio *bio,
/* Calc approx time to dispatch */
extra_bytes = tg->bytes_disp[rw] + bio->bi_iter.bi_size - bytes_allowed;
- jiffy_wait = div64_u64(extra_bytes * HZ, tg->bps[rw]);
+ jiffy_wait = div64_u64(extra_bytes * HZ, tg_bps_limit(tg, rw));
if (!jiffy_wait)
jiffy_wait = 1;
@@ -771,7 +793,7 @@ static bool tg_may_dispatch(struct throtl_grp *tg, struct bio *bio,
bio != throtl_peek_queued(&tg->service_queue.queued[rw]));
/* If tg->bps = -1, then BW is unlimited */
- if (tg->bps[rw] == -1 && tg->iops[rw] == -1) {
+ if (tg_bps_limit(tg, rw) == -1 && tg_iops_limit(tg, rw) == -1) {
if (wait)
*wait = 0;
return true;
@@ -1146,8 +1168,8 @@ static void tg_conf_updated(struct throtl_grp *tg)
throtl_log(&tg->service_queue,
"limit change rbps=%llu wbps=%llu riops=%u wiops=%u",
- tg->bps[READ], tg->bps[WRITE],
- tg->iops[READ], tg->iops[WRITE]);
+ tg_bps_limit(tg, READ), tg_bps_limit(tg, WRITE),
+ tg_iops_limit(tg, READ), tg_iops_limit(tg, WRITE));
/*
* Update has_rules[] flags for the updated tg's subtree. A tg is
@@ -1224,25 +1246,25 @@ static ssize_t tg_set_conf_uint(struct kernfs_open_file *of,
static struct cftype throtl_legacy_files[] = {
{
.name = "throttle.read_bps_device",
- .private = offsetof(struct throtl_grp, bps[READ]),
+ .private = offsetof(struct throtl_grp, bps[READ][LIMIT_MAX]),
.seq_show = tg_print_conf_u64,
.write = tg_set_conf_u64,
},
{
.name = "throttle.write_bps_device",
- .private = offsetof(struct throtl_grp, bps[WRITE]),
+ .private = offsetof(struct throtl_grp, bps[WRITE][LIMIT_MAX]),
.seq_show = tg_print_conf_u64,
.write = tg_set_conf_u64,
},
{
.name = "throttle.read_iops_device",
- .private = offsetof(struct throtl_grp, iops[READ]),
+ .private = offsetof(struct throtl_grp, iops[READ][LIMIT_MAX]),
.seq_show = tg_print_conf_uint,
.write = tg_set_conf_uint,
},
{
.name = "throttle.write_iops_device",
- .private = offsetof(struct throtl_grp, iops[WRITE]),
+ .private = offsetof(struct throtl_grp, iops[WRITE][LIMIT_MAX]),
.seq_show = tg_print_conf_uint,
.write = tg_set_conf_uint,
},
@@ -1268,18 +1290,22 @@ static u64 tg_prfill_max(struct seq_file *sf, struct blkg_policy_data *pd,
if (!dname)
return 0;
- if (tg->bps[READ] == -1 && tg->bps[WRITE] == -1 &&
- tg->iops[READ] == -1 && tg->iops[WRITE] == -1)
+ if (tg->bps[READ][LIMIT_MAX] == -1 && tg->bps[WRITE][LIMIT_MAX] == -1 &&
+ tg->iops[READ][LIMIT_MAX] == -1 && tg->iops[WRITE][LIMIT_MAX] == -1)
return 0;
- if (tg->bps[READ] != -1)
- snprintf(bufs[0], sizeof(bufs[0]), "%llu", tg->bps[READ]);
- if (tg->bps[WRITE] != -1)
- snprintf(bufs[1], sizeof(bufs[1]), "%llu", tg->bps[WRITE]);
- if (tg->iops[READ] != -1)
- snprintf(bufs[2], sizeof(bufs[2]), "%u", tg->iops[READ]);
- if (tg->iops[WRITE] != -1)
- snprintf(bufs[3], sizeof(bufs[3]), "%u", tg->iops[WRITE]);
+ if (tg->bps[READ][LIMIT_MAX] != -1)
+ snprintf(bufs[0], sizeof(bufs[0]), "%llu",
+ tg->bps[READ][LIMIT_MAX]);
+ if (tg->bps[WRITE][LIMIT_MAX] != -1)
+ snprintf(bufs[1], sizeof(bufs[1]), "%llu",
+ tg->bps[WRITE][LIMIT_MAX]);
+ if (tg->iops[READ][LIMIT_MAX] != -1)
+ snprintf(bufs[2], sizeof(bufs[2]), "%u",
+ tg->iops[READ][LIMIT_MAX]);
+ if (tg->iops[WRITE][LIMIT_MAX] != -1)
+ snprintf(bufs[3], sizeof(bufs[3]), "%u",
+ tg->iops[WRITE][LIMIT_MAX]);
seq_printf(sf, "%s rbps=%s wbps=%s riops=%s wiops=%s\n",
dname, bufs[0], bufs[1], bufs[2], bufs[3]);
@@ -1308,10 +1334,10 @@ static ssize_t tg_set_max(struct kernfs_open_file *of,
tg = blkg_to_tg(ctx.blkg);
- v[0] = tg->bps[READ];
- v[1] = tg->bps[WRITE];
- v[2] = tg->iops[READ];
- v[3] = tg->iops[WRITE];
+ v[0] = tg->bps[READ][LIMIT_MAX];
+ v[1] = tg->bps[WRITE][LIMIT_MAX];
+ v[2] = tg->iops[READ][LIMIT_MAX];
+ v[3] = tg->iops[WRITE][LIMIT_MAX];
while (true) {
char tok[27]; /* wiops=18446744073709551616 */
@@ -1348,10 +1374,10 @@ static ssize_t tg_set_max(struct kernfs_open_file *of,
goto out_finish;
}
- tg->bps[READ] = v[0];
- tg->bps[WRITE] = v[1];
- tg->iops[READ] = v[2];
- tg->iops[WRITE] = v[3];
+ tg->bps[READ][LIMIT_MAX] = v[0];
+ tg->bps[WRITE][LIMIT_MAX] = v[1];
+ tg->iops[READ][LIMIT_MAX] = v[2];
+ tg->iops[WRITE][LIMIT_MAX] = v[3];
tg_conf_updated(tg);
ret = 0;
@@ -1449,8 +1475,8 @@ bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg,
/* out-of-limit, queue to @tg */
throtl_log(sq, "[%c] bio. bdisp=%llu sz=%u bps=%llu iodisp=%u iops=%u queued=%d/%d",
rw == READ ? 'R' : 'W',
- tg->bytes_disp[rw], bio->bi_iter.bi_size, tg->bps[rw],
- tg->io_disp[rw], tg->iops[rw],
+ tg->bytes_disp[rw], bio->bi_iter.bi_size, tg_bps_limit(tg, rw),
+ tg->io_disp[rw], tg_iops_limit(tg, rw),
sq->nr_queued[READ], sq->nr_queued[WRITE]);
bio_associate_current(bio);
@@ -1561,6 +1587,7 @@ int blk_throtl_init(struct request_queue *q)
q->td = td;
td->queue = q;
+ td->limit_valid[LIMIT_MAX] = true;
/* activate policy */
ret = blkcg_activate_policy(q, &blkcg_policy_throtl);
if (ret)
--
2.8.0.rc2
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH V2 02/11] block-throttle: add .high interface
2016-09-15 16:23 [PATCH V2 00/11] block-throttle: add .high limit Shaohua Li
2016-09-15 16:23 ` [PATCH V2 01/11] block-throttle: prepare support multiple limits Shaohua Li
@ 2016-09-15 16:23 ` Shaohua Li
2016-09-15 16:23 ` [PATCH V2 03/11] block-throttle: configure bps/iops limit for cgroup in high limit Shaohua Li
` (8 subsequent siblings)
10 siblings, 0 replies; 13+ messages in thread
From: Shaohua Li @ 2016-09-15 16:23 UTC (permalink / raw)
To: linux-block, linux-kernel; +Cc: axboe, Kernel-team, tj, jmoyer, vgoyal
Add high limit for cgroup and corresponding cgroup interface.
Signed-off-by: Shaohua Li <shli@fb.com>
---
block/blk-throttle.c | 139 +++++++++++++++++++++++++++++++++++++++------------
1 file changed, 107 insertions(+), 32 deletions(-)
diff --git a/block/blk-throttle.c b/block/blk-throttle.c
index 71ecee7..02323fb 100644
--- a/block/blk-throttle.c
+++ b/block/blk-throttle.c
@@ -84,8 +84,9 @@ enum tg_state_flags {
#define rb_entry_tg(node) rb_entry((node), struct throtl_grp, rb_node)
enum {
- LIMIT_MAX = 0,
- LIMIT_CNT = 1,
+ LIMIT_HIGH = 0,
+ LIMIT_MAX = 1,
+ LIMIT_CNT = 2,
};
struct throtl_grp {
@@ -352,7 +353,7 @@ static struct blkg_policy_data *throtl_pd_alloc(gfp_t gfp, int node)
RB_CLEAR_NODE(&tg->rb_node);
for (rw = READ; rw <= WRITE; rw++) {
- for (index = LIMIT_MAX; index < LIMIT_CNT; index++) {
+ for (index = LIMIT_HIGH; index < LIMIT_CNT; index++) {
tg->bps[rw][index] = -1;
tg->iops[rw][index] = -1;
}
@@ -414,6 +415,46 @@ static void throtl_pd_online(struct blkg_policy_data *pd)
tg_update_has_rules(pd_to_tg(pd));
}
+static void blk_throtl_update_valid_limit(struct throtl_data *td)
+{
+ struct cgroup_subsys_state *pos_css;
+ struct blkcg_gq *blkg;
+ bool high_valid = false;
+
+ rcu_read_lock();
+ blkg_for_each_descendant_post(blkg, pos_css, td->queue->root_blkg) {
+ struct throtl_grp *tg = blkg_to_tg(blkg);
+
+ if (tg->bps[READ][LIMIT_HIGH] != -1 ||
+ tg->bps[WRITE][LIMIT_HIGH] != -1 ||
+ tg->iops[READ][LIMIT_HIGH] != -1 ||
+ tg->iops[WRITE][LIMIT_HIGH] != -1)
+ high_valid = true;
+ }
+ rcu_read_unlock();
+
+ if (high_valid)
+ td->limit_valid[LIMIT_HIGH] = true;
+ else
+ td->limit_valid[LIMIT_HIGH] = false;
+}
+
+static void throtl_pd_offline(struct blkg_policy_data *pd)
+{
+ struct throtl_grp *tg = pd_to_tg(pd);
+
+ tg->bps[READ][LIMIT_HIGH] = -1;
+ tg->bps[WRITE][LIMIT_HIGH] = -1;
+ tg->iops[READ][LIMIT_HIGH] = -1;
+ tg->iops[WRITE][LIMIT_HIGH] = -1;
+
+ blk_throtl_update_valid_limit(tg->td);
+
+ if (tg->td->limit_index == LIMIT_HIGH &&
+ !tg->td->limit_valid[LIMIT_HIGH])
+ tg->td->limit_index = LIMIT_MAX;
+}
+
static void throtl_pd_free(struct blkg_policy_data *pd)
{
struct throtl_grp *tg = pd_to_tg(pd);
@@ -1281,7 +1322,7 @@ static struct cftype throtl_legacy_files[] = {
{ } /* terminate */
};
-static u64 tg_prfill_max(struct seq_file *sf, struct blkg_policy_data *pd,
+static u64 tg_prfill_limit(struct seq_file *sf, struct blkg_policy_data *pd,
int off)
{
struct throtl_grp *tg = pd_to_tg(pd);
@@ -1290,36 +1331,32 @@ static u64 tg_prfill_max(struct seq_file *sf, struct blkg_policy_data *pd,
if (!dname)
return 0;
- if (tg->bps[READ][LIMIT_MAX] == -1 && tg->bps[WRITE][LIMIT_MAX] == -1 &&
- tg->iops[READ][LIMIT_MAX] == -1 && tg->iops[WRITE][LIMIT_MAX] == -1)
+ if (tg->bps[READ][off] == -1 && tg->bps[WRITE][off] == -1 &&
+ tg->iops[READ][off] == -1 && tg->iops[WRITE][off] == -1)
return 0;
- if (tg->bps[READ][LIMIT_MAX] != -1)
- snprintf(bufs[0], sizeof(bufs[0]), "%llu",
- tg->bps[READ][LIMIT_MAX]);
- if (tg->bps[WRITE][LIMIT_MAX] != -1)
- snprintf(bufs[1], sizeof(bufs[1]), "%llu",
- tg->bps[WRITE][LIMIT_MAX]);
- if (tg->iops[READ][LIMIT_MAX] != -1)
- snprintf(bufs[2], sizeof(bufs[2]), "%u",
- tg->iops[READ][LIMIT_MAX]);
- if (tg->iops[WRITE][LIMIT_MAX] != -1)
- snprintf(bufs[3], sizeof(bufs[3]), "%u",
- tg->iops[WRITE][LIMIT_MAX]);
+ if (tg->bps[READ][off] != -1)
+ snprintf(bufs[0], sizeof(bufs[0]), "%llu", tg->bps[READ][off]);
+ if (tg->bps[WRITE][off] != -1)
+ snprintf(bufs[1], sizeof(bufs[1]), "%llu", tg->bps[WRITE][off]);
+ if (tg->iops[READ][off] != -1)
+ snprintf(bufs[2], sizeof(bufs[2]), "%u", tg->iops[READ][off]);
+ if (tg->iops[WRITE][off] != -1)
+ snprintf(bufs[3], sizeof(bufs[3]), "%u", tg->iops[WRITE][off]);
seq_printf(sf, "%s rbps=%s wbps=%s riops=%s wiops=%s\n",
dname, bufs[0], bufs[1], bufs[2], bufs[3]);
return 0;
}
-static int tg_print_max(struct seq_file *sf, void *v)
+static int tg_print_limit(struct seq_file *sf, void *v)
{
- blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), tg_prfill_max,
+ blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), tg_prfill_limit,
&blkcg_policy_throtl, seq_cft(sf)->private, false);
return 0;
}
-static ssize_t tg_set_max(struct kernfs_open_file *of,
+static ssize_t tg_set_limit(struct kernfs_open_file *of,
char *buf, size_t nbytes, loff_t off)
{
struct blkcg *blkcg = css_to_blkcg(of_css(of));
@@ -1327,6 +1364,7 @@ static ssize_t tg_set_max(struct kernfs_open_file *of,
struct throtl_grp *tg;
u64 v[4];
int ret;
+ int index = of_cft(of)->private;
ret = blkg_conf_prep(blkcg, &blkcg_policy_throtl, buf, &ctx);
if (ret)
@@ -1334,10 +1372,10 @@ static ssize_t tg_set_max(struct kernfs_open_file *of,
tg = blkg_to_tg(ctx.blkg);
- v[0] = tg->bps[READ][LIMIT_MAX];
- v[1] = tg->bps[WRITE][LIMIT_MAX];
- v[2] = tg->iops[READ][LIMIT_MAX];
- v[3] = tg->iops[WRITE][LIMIT_MAX];
+ v[0] = tg->bps[READ][index];
+ v[1] = tg->bps[WRITE][index];
+ v[2] = tg->iops[READ][index];
+ v[3] = tg->iops[WRITE][index];
while (true) {
char tok[27]; /* wiops=18446744073709551616 */
@@ -1374,11 +1412,37 @@ static ssize_t tg_set_max(struct kernfs_open_file *of,
goto out_finish;
}
- tg->bps[READ][LIMIT_MAX] = v[0];
- tg->bps[WRITE][LIMIT_MAX] = v[1];
- tg->iops[READ][LIMIT_MAX] = v[2];
- tg->iops[WRITE][LIMIT_MAX] = v[3];
-
+ if (index == LIMIT_MAX) {
+ if ((v[0] < tg->bps[READ][LIMIT_HIGH] &&
+ tg->bps[READ][LIMIT_HIGH] != -1) ||
+ (v[1] < tg->bps[WRITE][LIMIT_HIGH] &&
+ tg->bps[WRITE][LIMIT_HIGH] != -1) ||
+ (v[2] < tg->iops[READ][LIMIT_HIGH] &&
+ tg->iops[READ][LIMIT_HIGH] != -1) ||
+ (v[3] < tg->iops[WRITE][LIMIT_HIGH] &&
+ tg->iops[WRITE][LIMIT_HIGH] != -1)) {
+ ret = -EINVAL;
+ goto out_finish;
+ }
+ } else if (index == LIMIT_HIGH) {
+ if ((v[0] > tg->bps[READ][LIMIT_MAX] && v[0] != -1) ||
+ (v[1] > tg->bps[WRITE][LIMIT_MAX] && v[1] != -1) ||
+ (v[2] > tg->iops[READ][LIMIT_MAX] && v[2] != -1) ||
+ (v[3] > tg->iops[WRITE][LIMIT_MAX] && v[3] != -1)) {
+ ret = -EINVAL;
+ goto out_finish;
+ }
+ }
+ tg->bps[READ][index] = v[0];
+ tg->bps[WRITE][index] = v[1];
+ tg->iops[READ][index] = v[2];
+ tg->iops[WRITE][index] = v[3];
+
+ if (index == LIMIT_HIGH) {
+ blk_throtl_update_valid_limit(tg->td);
+ if (tg->td->limit_valid[LIMIT_HIGH])
+ tg->td->limit_index = LIMIT_HIGH;
+ }
tg_conf_updated(tg);
ret = 0;
out_finish:
@@ -1388,10 +1452,18 @@ static ssize_t tg_set_max(struct kernfs_open_file *of,
static struct cftype throtl_files[] = {
{
+ .name = "high",
+ .flags = CFTYPE_NOT_ON_ROOT,
+ .seq_show = tg_print_limit,
+ .write = tg_set_limit,
+ .private = LIMIT_HIGH,
+ },
+ {
.name = "max",
.flags = CFTYPE_NOT_ON_ROOT,
- .seq_show = tg_print_max,
- .write = tg_set_max,
+ .seq_show = tg_print_limit,
+ .write = tg_set_limit,
+ .private = LIMIT_MAX,
},
{ } /* terminate */
};
@@ -1410,6 +1482,7 @@ static struct blkcg_policy blkcg_policy_throtl = {
.pd_alloc_fn = throtl_pd_alloc,
.pd_init_fn = throtl_pd_init,
.pd_online_fn = throtl_pd_online,
+ .pd_offline_fn = throtl_pd_offline,
.pd_free_fn = throtl_pd_free,
};
@@ -1587,7 +1660,9 @@ int blk_throtl_init(struct request_queue *q)
q->td = td;
td->queue = q;
+ td->limit_valid[LIMIT_HIGH] = false;
td->limit_valid[LIMIT_MAX] = true;
+ td->limit_index = LIMIT_MAX;
/* activate policy */
ret = blkcg_activate_policy(q, &blkcg_policy_throtl);
if (ret)
--
2.8.0.rc2
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH V2 03/11] block-throttle: configure bps/iops limit for cgroup in high limit
2016-09-15 16:23 [PATCH V2 00/11] block-throttle: add .high limit Shaohua Li
2016-09-15 16:23 ` [PATCH V2 01/11] block-throttle: prepare support multiple limits Shaohua Li
2016-09-15 16:23 ` [PATCH V2 02/11] block-throttle: add .high interface Shaohua Li
@ 2016-09-15 16:23 ` Shaohua Li
2016-09-15 16:23 ` [PATCH V2 04/11] block-throttle: add upgrade logic for LIMIT_HIGH state Shaohua Li
` (7 subsequent siblings)
10 siblings, 0 replies; 13+ messages in thread
From: Shaohua Li @ 2016-09-15 16:23 UTC (permalink / raw)
To: linux-block, linux-kernel; +Cc: axboe, Kernel-team, tj, jmoyer, vgoyal
each queue will have a state machine. Initially queue is in LIMIT_HIGH
state, which means all cgroups will be throttled according to their high
limit. After all cgroups with high limit cross the limit, the queue state
gets upgraded to LIMIT_MAX state.
cgroups without high limit will use max limit for their high limit.
Signed-off-by: Shaohua Li <shli@fb.com>
---
block/blk-throttle.c | 21 +++++++++++++++++++--
1 file changed, 19 insertions(+), 2 deletions(-)
diff --git a/block/blk-throttle.c b/block/blk-throttle.c
index 02323fb..6bae1b4 100644
--- a/block/blk-throttle.c
+++ b/block/blk-throttle.c
@@ -208,12 +208,29 @@ static struct throtl_data *sq_to_td(struct throtl_service_queue *sq)
static uint64_t tg_bps_limit(struct throtl_grp *tg, int rw)
{
- return tg->bps[rw][tg->td->limit_index];
+ struct blkcg_gq *blkg = tg_to_blkg(tg);
+ uint64_t ret;
+
+ if (cgroup_subsys_on_dfl(io_cgrp_subsys) && !blkg->parent)
+ return -1;
+ ret = tg->bps[rw][tg->td->limit_index];
+ if (ret == -1 && tg->td->limit_index == LIMIT_HIGH)
+ return tg->bps[rw][LIMIT_MAX];
+
+ return ret;
}
static unsigned int tg_iops_limit(struct throtl_grp *tg, int rw)
{
- return tg->iops[rw][tg->td->limit_index];
+ struct blkcg_gq *blkg = tg_to_blkg(tg);
+ unsigned int ret;
+
+ if (cgroup_subsys_on_dfl(io_cgrp_subsys) && !blkg->parent)
+ return -1;
+ ret = tg->iops[rw][tg->td->limit_index];
+ if (ret == -1 && tg->td->limit_index == LIMIT_HIGH)
+ return tg->iops[rw][LIMIT_MAX];
+ return ret;
}
/**
--
2.8.0.rc2
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH V2 04/11] block-throttle: add upgrade logic for LIMIT_HIGH state
2016-09-15 16:23 [PATCH V2 00/11] block-throttle: add .high limit Shaohua Li
` (2 preceding siblings ...)
2016-09-15 16:23 ` [PATCH V2 03/11] block-throttle: configure bps/iops limit for cgroup in high limit Shaohua Li
@ 2016-09-15 16:23 ` Shaohua Li
2016-09-15 16:23 ` [PATCH V2 05/11] block-throttle: add downgrade logic Shaohua Li
` (6 subsequent siblings)
10 siblings, 0 replies; 13+ messages in thread
From: Shaohua Li @ 2016-09-15 16:23 UTC (permalink / raw)
To: linux-block, linux-kernel; +Cc: axboe, Kernel-team, tj, jmoyer, vgoyal
When queue is in LIMIT_HIGH state and all cgroups with high limit cross
the bps/iops limitation, we will upgrade queue's state to
LIMIT_MAX
For a cgroup hierarchy, there are two cases. Children has lower high
limit than parent. Parent's high limit is meaningless. If children's
bps/iops cross high limit, we can upgrade queue state. The other case is
children has higher high limit than parent. Children's high limit is
meaningless. As long as parent's bps/iops cross high limit, we can
upgrade queue state.
Signed-off-by: Shaohua Li <shli@fb.com>
---
block/blk-throttle.c | 104 +++++++++++++++++++++++++++++++++++++++++++++++++--
1 file changed, 100 insertions(+), 4 deletions(-)
diff --git a/block/blk-throttle.c b/block/blk-throttle.c
index 6bae1b4..80b50cc 100644
--- a/block/blk-throttle.c
+++ b/block/blk-throttle.c
@@ -456,6 +456,7 @@ static void blk_throtl_update_valid_limit(struct throtl_data *td)
td->limit_valid[LIMIT_HIGH] = false;
}
+static void throtl_upgrade_state(struct throtl_data *td);
static void throtl_pd_offline(struct blkg_policy_data *pd)
{
struct throtl_grp *tg = pd_to_tg(pd);
@@ -467,9 +468,8 @@ static void throtl_pd_offline(struct blkg_policy_data *pd)
blk_throtl_update_valid_limit(tg->td);
- if (tg->td->limit_index == LIMIT_HIGH &&
- !tg->td->limit_valid[LIMIT_HIGH])
- tg->td->limit_index = LIMIT_MAX;
+ if (!tg->td->limit_valid[tg->td->limit_index])
+ throtl_upgrade_state(tg->td);
}
static void throtl_pd_free(struct blkg_policy_data *pd)
@@ -1075,6 +1075,8 @@ static int throtl_select_dispatch(struct throtl_service_queue *parent_sq)
return nr_disp;
}
+static bool throtl_can_upgrade(struct throtl_data *td,
+ struct throtl_grp *this_tg);
/**
* throtl_pending_timer_fn - timer function for service_queue->pending_timer
* @arg: the throtl_service_queue being serviced
@@ -1101,6 +1103,9 @@ static void throtl_pending_timer_fn(unsigned long arg)
int ret;
spin_lock_irq(q->queue_lock);
+ if (throtl_can_upgrade(td, NULL))
+ throtl_upgrade_state(td);
+
again:
parent_sq = sq->parent_sq;
dispatched = false;
@@ -1503,6 +1508,91 @@ static struct blkcg_policy blkcg_policy_throtl = {
.pd_free_fn = throtl_pd_free,
};
+static bool throtl_upgrade_check_one(struct throtl_grp *tg)
+{
+ struct throtl_service_queue *sq = &tg->service_queue;
+ bool read_limit, write_limit;
+
+ /* if cgroup reaches high/max limit, it's ok to next limit */
+ read_limit = tg->bps[READ][LIMIT_HIGH] != -1 ||
+ tg->iops[READ][LIMIT_HIGH] != -1 ||
+ tg->bps[READ][LIMIT_MAX] != -1 ||
+ tg->iops[READ][LIMIT_MAX] != -1;
+ write_limit = tg->bps[WRITE][LIMIT_HIGH] != -1 ||
+ tg->iops[WRITE][LIMIT_HIGH] != -1 ||
+ tg->bps[WRITE][LIMIT_MAX] != -1 ||
+ tg->iops[WRITE][LIMIT_MAX] != -1;
+ if (read_limit && sq->nr_queued[READ] &&
+ (!write_limit || sq->nr_queued[WRITE]))
+ return true;
+ if (write_limit && sq->nr_queued[WRITE] &&
+ (!read_limit || sq->nr_queued[READ]))
+ return true;
+ return false;
+}
+
+static bool throtl_upgrade_check_hierarchy(struct throtl_grp *tg)
+{
+ if (throtl_upgrade_check_one(tg))
+ return true;
+ while (true) {
+ if (!tg || (cgroup_subsys_on_dfl(io_cgrp_subsys) &&
+ !tg_to_blkg(tg)->parent))
+ return false;
+ if (throtl_upgrade_check_one(tg))
+ return true;
+ tg = sq_to_tg(tg->service_queue.parent_sq);
+ }
+ return false;
+}
+
+static bool throtl_can_upgrade(struct throtl_data *td,
+ struct throtl_grp *this_tg)
+{
+ struct cgroup_subsys_state *pos_css;
+ struct blkcg_gq *blkg;
+
+ if (td->limit_index != LIMIT_HIGH)
+ return false;
+
+ rcu_read_lock();
+ blkg_for_each_descendant_post(blkg, pos_css, td->queue->root_blkg) {
+ struct throtl_grp *tg = blkg_to_tg(blkg);
+
+ if (tg == this_tg)
+ continue;
+ if (!list_empty(&tg_to_blkg(tg)->blkcg->css.children))
+ continue;
+ if (!throtl_upgrade_check_hierarchy(tg)) {
+ rcu_read_unlock();
+ return false;
+ }
+ }
+ rcu_read_unlock();
+ return true;
+}
+
+static void throtl_upgrade_state(struct throtl_data *td)
+{
+ struct cgroup_subsys_state *pos_css;
+ struct blkcg_gq *blkg;
+
+ td->limit_index = LIMIT_MAX;
+ rcu_read_lock();
+ blkg_for_each_descendant_post(blkg, pos_css, td->queue->root_blkg) {
+ struct throtl_grp *tg = blkg_to_tg(blkg);
+ struct throtl_service_queue *sq = &tg->service_queue;
+
+ tg->disptime = jiffies - 1;
+ throtl_select_dispatch(sq);
+ throtl_schedule_next_dispatch(sq, false);
+ }
+ rcu_read_unlock();
+ throtl_select_dispatch(&td->service_queue);
+ throtl_schedule_next_dispatch(&td->service_queue, false);
+ queue_work(kthrotld_workqueue, &td->dispatch_work);
+}
+
bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg,
struct bio *bio)
{
@@ -1525,14 +1615,20 @@ bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg,
sq = &tg->service_queue;
+again:
while (true) {
/* throtl is FIFO - if bios are already queued, should queue */
if (sq->nr_queued[rw])
break;
/* if above limits, break to queue */
- if (!tg_may_dispatch(tg, bio, NULL))
+ if (!tg_may_dispatch(tg, bio, NULL)) {
+ if (throtl_can_upgrade(tg->td, tg)) {
+ throtl_upgrade_state(tg->td);
+ goto again;
+ }
break;
+ }
/* within limits, let's charge and dispatch directly */
throtl_charge_bio(tg, bio);
--
2.8.0.rc2
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH V2 05/11] block-throttle: add downgrade logic
2016-09-15 16:23 [PATCH V2 00/11] block-throttle: add .high limit Shaohua Li
` (3 preceding siblings ...)
2016-09-15 16:23 ` [PATCH V2 04/11] block-throttle: add upgrade logic for LIMIT_HIGH state Shaohua Li
@ 2016-09-15 16:23 ` Shaohua Li
2016-09-15 16:23 ` [PATCH V2 06/11] blk-throttle: make sure expire time isn't too big Shaohua Li
` (5 subsequent siblings)
10 siblings, 0 replies; 13+ messages in thread
From: Shaohua Li @ 2016-09-15 16:23 UTC (permalink / raw)
To: linux-block, linux-kernel; +Cc: axboe, Kernel-team, tj, jmoyer, vgoyal
When queue state machine is in LIMIT_MAX state, but a cgroup is below
its high limit for some time, the queue should be downgraded to lower
state as one cgroup's high limit isn't met.
Signed-off-by: Shaohua Li <shli@fb.com>
---
block/blk-throttle.c | 187 +++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 187 insertions(+)
diff --git a/block/blk-throttle.c b/block/blk-throttle.c
index 80b50cc..76988f0 100644
--- a/block/blk-throttle.c
+++ b/block/blk-throttle.c
@@ -136,6 +136,13 @@ struct throtl_grp {
/* Number of bio's dispatched in current slice */
unsigned int io_disp[2];
+ unsigned long last_high_overflow_time[2];
+
+ uint64_t last_bytes_disp[2];
+ unsigned int last_io_disp[2];
+
+ unsigned long last_check_time;
+
/* When did we start a new slice */
unsigned long slice_start[2];
unsigned long slice_end[2];
@@ -155,6 +162,9 @@ struct throtl_data
struct work_struct dispatch_work;
unsigned int limit_index;
bool limit_valid[LIMIT_CNT];
+
+ unsigned long high_upgrade_time;
+ unsigned long high_downgrade_time;
};
static void throtl_pending_timer_fn(unsigned long arg);
@@ -894,6 +904,8 @@ static void throtl_charge_bio(struct throtl_grp *tg, struct bio *bio)
/* Charge the bio to the group */
tg->bytes_disp[rw] += bio->bi_iter.bi_size;
tg->io_disp[rw]++;
+ tg->last_bytes_disp[rw] += bio->bi_iter.bi_size;
+ tg->last_io_disp[rw]++;
/*
* REQ_THROTTLED is used to prevent the same bio to be throttled
@@ -1508,6 +1520,64 @@ static struct blkcg_policy blkcg_policy_throtl = {
.pd_free_fn = throtl_pd_free,
};
+static unsigned long __tg_last_high_overflow_time(struct throtl_grp *tg)
+{
+ unsigned long rtime = -1, wtime = -1;
+ if (tg->bps[READ][LIMIT_HIGH] != -1 ||
+ tg->iops[READ][LIMIT_HIGH] != -1 ||
+ tg->bps[READ][LIMIT_MAX] != -1 ||
+ tg->iops[READ][LIMIT_MAX] != -1)
+ rtime = tg->last_high_overflow_time[READ];
+ if (tg->bps[WRITE][LIMIT_HIGH] != -1 ||
+ tg->iops[WRITE][LIMIT_HIGH] != -1 ||
+ tg->bps[WRITE][LIMIT_MAX] != -1 ||
+ tg->iops[WRITE][LIMIT_MAX] != -1)
+ wtime = tg->last_high_overflow_time[WRITE];
+ return min(rtime, wtime) == -1 ? 0 : min(rtime, wtime);
+}
+
+static unsigned long tg_last_high_overflow_time(struct throtl_grp *tg)
+{
+ struct throtl_service_queue *parent_sq;
+ struct throtl_grp *parent = tg;
+ unsigned long ret = __tg_last_high_overflow_time(tg);
+
+ while (true) {
+ parent_sq = parent->service_queue.parent_sq;
+ parent = sq_to_tg(parent_sq);
+ if (!parent)
+ break;
+ if (((parent->bps[READ][LIMIT_HIGH] != -1 &&
+ parent->bps[READ][LIMIT_HIGH] >=
+ tg->bps[READ][LIMIT_HIGH]) ||
+ (parent->bps[READ][LIMIT_HIGH] == -1 &&
+ parent->bps[READ][LIMIT_MAX] >=
+ tg->bps[READ][LIMIT_HIGH])) &&
+ ((parent->bps[WRITE][LIMIT_HIGH] != -1 &&
+ parent->bps[WRITE][LIMIT_HIGH] >=
+ tg->bps[WRITE][LIMIT_HIGH]) ||
+ (parent->bps[WRITE][LIMIT_HIGH] == -1 &&
+ parent->bps[WRITE][LIMIT_MAX] >=
+ tg->bps[WRITE][LIMIT_HIGH])) &&
+ ((parent->iops[READ][LIMIT_HIGH] != -1 &&
+ parent->iops[READ][LIMIT_HIGH] >=
+ tg->iops[READ][LIMIT_HIGH]) ||
+ (parent->iops[READ][LIMIT_HIGH] == -1 &&
+ parent->iops[READ][LIMIT_MAX] >=
+ tg->iops[READ][LIMIT_HIGH])) &&
+ ((parent->iops[WRITE][LIMIT_HIGH] != -1 &&
+ parent->iops[WRITE][LIMIT_HIGH] >=
+ tg->iops[WRITE][LIMIT_HIGH]) ||
+ (parent->iops[WRITE][LIMIT_HIGH] == -1 &&
+ parent->iops[WRITE][LIMIT_MAX] >=
+ tg->iops[WRITE][LIMIT_HIGH])))
+ break;
+ if (time_after(__tg_last_high_overflow_time(parent), ret))
+ ret = __tg_last_high_overflow_time(parent);
+ }
+ return ret;
+}
+
static bool throtl_upgrade_check_one(struct throtl_grp *tg)
{
struct throtl_service_queue *sq = &tg->service_queue;
@@ -1555,6 +1625,9 @@ static bool throtl_can_upgrade(struct throtl_data *td,
if (td->limit_index != LIMIT_HIGH)
return false;
+ if (time_before(jiffies, td->high_downgrade_time + throtl_slice))
+ return false;
+
rcu_read_lock();
blkg_for_each_descendant_post(blkg, pos_css, td->queue->root_blkg) {
struct throtl_grp *tg = blkg_to_tg(blkg);
@@ -1578,6 +1651,7 @@ static void throtl_upgrade_state(struct throtl_data *td)
struct blkcg_gq *blkg;
td->limit_index = LIMIT_MAX;
+ td->high_upgrade_time = jiffies;
rcu_read_lock();
blkg_for_each_descendant_post(blkg, pos_css, td->queue->root_blkg) {
struct throtl_grp *tg = blkg_to_tg(blkg);
@@ -1593,6 +1667,111 @@ static void throtl_upgrade_state(struct throtl_data *td)
queue_work(kthrotld_workqueue, &td->dispatch_work);
}
+static void throtl_downgrade_state(struct throtl_data *td, int new)
+{
+ td->limit_index = new;
+ td->high_downgrade_time = jiffies;
+}
+
+static bool throtl_downgrade_check_one(struct throtl_grp *tg)
+{
+ struct throtl_data *td = tg->td;
+ unsigned long now = jiffies;
+
+ /*
+ * If cgroup is below high limit, consider downgrade and throttle other
+ * cgroups
+ */
+ if (time_after_eq(now, td->high_upgrade_time + throtl_slice) &&
+ time_after_eq(now, tg_last_high_overflow_time(tg) + throtl_slice))
+ return true;
+ return false;
+}
+
+static bool throtl_downgrade_check_hierarchy(struct throtl_grp *tg)
+{
+ if (!throtl_downgrade_check_one(tg))
+ return false;
+ while (true) {
+ if (!tg || (cgroup_subsys_on_dfl(io_cgrp_subsys) &&
+ !tg_to_blkg(tg)->parent))
+ break;
+
+ if (!throtl_downgrade_check_one(tg))
+ return false;
+ tg = sq_to_tg(tg->service_queue.parent_sq);
+ }
+ return true;
+}
+
+static void throtl_downgrade_check(struct throtl_grp *tg)
+{
+ uint64_t bps;
+ unsigned int iops;
+ unsigned long elapsed_time;
+ unsigned long now = jiffies;
+
+ if (tg->td->limit_index != LIMIT_MAX ||
+ !tg->td->limit_valid[LIMIT_HIGH])
+ return;
+ if (!list_empty(&tg_to_blkg(tg)->blkcg->css.children))
+ return;
+ if (time_after(tg->last_check_time + throtl_slice, now))
+ return;
+
+ if (time_before(now, tg_last_high_overflow_time(tg) + throtl_slice))
+ return;
+
+ elapsed_time = now - tg->last_check_time;
+ tg->last_check_time = now;
+
+ if (tg->bps[READ][LIMIT_HIGH] != -1 ||
+ tg->bps[READ][LIMIT_MAX] != -1) {
+ bps = tg->last_bytes_disp[READ] * HZ;
+ do_div(bps, elapsed_time);
+ if (bps >= tg->bps[READ][LIMIT_HIGH] ||
+ bps >= tg->bps[READ][LIMIT_MAX])
+ tg->last_high_overflow_time[READ] = now;
+ }
+
+ if (tg->bps[WRITE][LIMIT_HIGH] != -1 ||
+ tg->bps[WRITE][LIMIT_MAX] != -1) {
+ bps = tg->last_bytes_disp[WRITE] * HZ;
+ do_div(bps, elapsed_time);
+ if (bps >= tg->bps[WRITE][LIMIT_HIGH] ||
+ bps >= tg->bps[WRITE][LIMIT_MAX])
+ tg->last_high_overflow_time[WRITE] = now;
+ }
+
+ if (tg->iops[READ][LIMIT_HIGH] != -1 ||
+ tg->iops[READ][LIMIT_MAX] != -1) {
+ iops = tg->last_io_disp[READ] * HZ / elapsed_time;
+ if (iops >= tg->iops[READ][LIMIT_HIGH] ||
+ iops >= tg->iops[READ][LIMIT_MAX])
+ tg->last_high_overflow_time[READ] = now;
+ }
+
+ if (tg->iops[WRITE][LIMIT_HIGH] != -1 ||
+ tg->iops[WRITE][LIMIT_MAX] != -1) {
+ iops = tg->last_io_disp[WRITE] * HZ / elapsed_time;
+ if (iops >= tg->iops[WRITE][LIMIT_HIGH] ||
+ iops >= tg->iops[WRITE][LIMIT_MAX])
+ tg->last_high_overflow_time[WRITE] = now;
+ }
+
+ /*
+ * If cgroup is below high limit, consider downgrade and throttle other
+ * cgroups
+ */
+ if (throtl_downgrade_check_hierarchy(tg))
+ throtl_downgrade_state(tg->td, LIMIT_HIGH);
+
+ tg->last_bytes_disp[READ] = 0;
+ tg->last_bytes_disp[WRITE] = 0;
+ tg->last_io_disp[READ] = 0;
+ tg->last_io_disp[WRITE] = 0;
+}
+
bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg,
struct bio *bio)
{
@@ -1617,12 +1796,16 @@ bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg,
again:
while (true) {
+ if (tg->last_high_overflow_time[rw] == 0)
+ tg->last_high_overflow_time[rw] = jiffies;
+ throtl_downgrade_check(tg);
/* throtl is FIFO - if bios are already queued, should queue */
if (sq->nr_queued[rw])
break;
/* if above limits, break to queue */
if (!tg_may_dispatch(tg, bio, NULL)) {
+ tg->last_high_overflow_time[rw] = jiffies;
if (throtl_can_upgrade(tg->td, tg)) {
throtl_upgrade_state(tg->td);
goto again;
@@ -1665,6 +1848,8 @@ bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg,
tg->io_disp[rw], tg_iops_limit(tg, rw),
sq->nr_queued[READ], sq->nr_queued[WRITE]);
+ tg->last_high_overflow_time[rw] = jiffies;
+
bio_associate_current(bio);
tg->td->nr_queued[rw]++;
throtl_add_bio_tg(bio, qn, tg);
@@ -1776,6 +1961,8 @@ int blk_throtl_init(struct request_queue *q)
td->limit_valid[LIMIT_HIGH] = false;
td->limit_valid[LIMIT_MAX] = true;
td->limit_index = LIMIT_MAX;
+ td->high_upgrade_time = jiffies;
+ td->high_downgrade_time = jiffies;
/* activate policy */
ret = blkcg_activate_policy(q, &blkcg_policy_throtl);
if (ret)
--
2.8.0.rc2
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH V2 06/11] blk-throttle: make sure expire time isn't too big
2016-09-15 16:23 [PATCH V2 00/11] block-throttle: add .high limit Shaohua Li
` (4 preceding siblings ...)
2016-09-15 16:23 ` [PATCH V2 05/11] block-throttle: add downgrade logic Shaohua Li
@ 2016-09-15 16:23 ` Shaohua Li
2016-09-15 16:23 ` [PATCH V2 07/11] blk-throttle: make throtl_slice tunable Shaohua Li
` (4 subsequent siblings)
10 siblings, 0 replies; 13+ messages in thread
From: Shaohua Li @ 2016-09-15 16:23 UTC (permalink / raw)
To: linux-block, linux-kernel; +Cc: axboe, Kernel-team, tj, jmoyer, vgoyal
cgroup could be throttled to a limit but when all cgroups cross high
limit, queue enters a higher state and so the group should be throttled
to a higher limit. It's possible the cgroup is sleeping because of
throttle and other cgroups don't dispatch IO any more. In this case,
nobody can trigger current downgrade/upgrade logic. To fix this issue,
we could either set up a timer to wakeup the cgroup if other cgroups are
idle or make sure this cgroup doesn't sleep too long. Setting up a timer
means we must change the timer very frequently. This patch chooses the
latter. Making cgroup sleep time not too big wouldn't change cgroup
bps/iops, but could make it wakeup more frequently, which isn't a big
issue because throtl_slice * 8 is already quite big.
Signed-off-by: Shaohua Li <shli@fb.com>
---
block/blk-throttle.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/block/blk-throttle.c b/block/blk-throttle.c
index 76988f0..2bd8333 100644
--- a/block/blk-throttle.c
+++ b/block/blk-throttle.c
@@ -589,6 +589,9 @@ static void throtl_dequeue_tg(struct throtl_grp *tg)
static void throtl_schedule_pending_timer(struct throtl_service_queue *sq,
unsigned long expires)
{
+ unsigned long max_expire = jiffies + 8 * throtl_slice;
+ if (time_after(expires, max_expire))
+ expires = max_expire;
mod_timer(&sq->pending_timer, expires);
throtl_log(sq, "schedule timer. delay=%lu jiffies=%lu",
expires - jiffies, jiffies);
--
2.8.0.rc2
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH V2 07/11] blk-throttle: make throtl_slice tunable
2016-09-15 16:23 [PATCH V2 00/11] block-throttle: add .high limit Shaohua Li
` (5 preceding siblings ...)
2016-09-15 16:23 ` [PATCH V2 06/11] blk-throttle: make sure expire time isn't too big Shaohua Li
@ 2016-09-15 16:23 ` Shaohua Li
2016-09-15 16:23 ` [PATCH V2 08/11] blk-throttle: detect completed idle cgroup Shaohua Li
` (3 subsequent siblings)
10 siblings, 0 replies; 13+ messages in thread
From: Shaohua Li @ 2016-09-15 16:23 UTC (permalink / raw)
To: linux-block, linux-kernel; +Cc: axboe, Kernel-team, tj, jmoyer, vgoyal
throtl_slice is important for blk-throttling. A lot of stuffes depend on
it, for example, throughput measurement. It has 100ms default value,
which is not appropriate for all disks. For example, for SSD we might
use a smaller value to make the throughput smoother. This patch makes it
tunable.
Signed-off-by: Shaohua Li <shli@fb.com>
---
block/blk-sysfs.c | 11 ++++++++
block/blk-throttle.c | 72 ++++++++++++++++++++++++++++++++++++----------------
block/blk.h | 3 +++
3 files changed, 64 insertions(+), 22 deletions(-)
diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c
index f87a7e7..610f08d 100644
--- a/block/blk-sysfs.c
+++ b/block/blk-sysfs.c
@@ -526,6 +526,14 @@ static struct queue_sysfs_entry queue_dax_entry = {
.show = queue_dax_show,
};
+#ifdef CONFIG_BLK_DEV_THROTTLING
+static struct queue_sysfs_entry throtl_slice_entry = {
+ .attr = {.name = "throttling_slice", .mode = S_IRUGO | S_IWUSR },
+ .show = blk_throtl_slice_show,
+ .store = blk_throtl_slice_store,
+};
+#endif
+
static struct attribute *default_attrs[] = {
&queue_requests_entry.attr,
&queue_ra_entry.attr,
@@ -553,6 +561,9 @@ static struct attribute *default_attrs[] = {
&queue_poll_entry.attr,
&queue_wc_entry.attr,
&queue_dax_entry.attr,
+#ifdef CONFIG_BLK_DEV_THROTTLING
+ &throtl_slice_entry.attr,
+#endif
NULL,
};
diff --git a/block/blk-throttle.c b/block/blk-throttle.c
index 2bd8333..bc94086 100644
--- a/block/blk-throttle.c
+++ b/block/blk-throttle.c
@@ -19,7 +19,8 @@ static int throtl_grp_quantum = 8;
static int throtl_quantum = 32;
/* Throttling is performed over 100ms slice and after that slice is renewed */
-static unsigned long throtl_slice = HZ/10; /* 100 ms */
+#define DFL_THROTL_SLICE (HZ / 10)
+#define MAX_THROTL_SLICE (HZ / 5)
static struct blkcg_policy blkcg_policy_throtl;
@@ -158,6 +159,8 @@ struct throtl_data
/* Total Number of queued bios on READ and WRITE lists */
unsigned int nr_queued[2];
+ unsigned int throtl_slice;
+
/* Work for dispatching throttled bios */
struct work_struct dispatch_work;
unsigned int limit_index;
@@ -589,7 +592,7 @@ static void throtl_dequeue_tg(struct throtl_grp *tg)
static void throtl_schedule_pending_timer(struct throtl_service_queue *sq,
unsigned long expires)
{
- unsigned long max_expire = jiffies + 8 * throtl_slice;
+ unsigned long max_expire = jiffies + 8 * sq_to_tg(sq)->td->throtl_slice;
if (time_after(expires, max_expire))
expires = max_expire;
mod_timer(&sq->pending_timer, expires);
@@ -649,7 +652,7 @@ static inline void throtl_start_new_slice_with_credit(struct throtl_grp *tg,
if (time_after_eq(start, tg->slice_start[rw]))
tg->slice_start[rw] = start;
- tg->slice_end[rw] = jiffies + throtl_slice;
+ tg->slice_end[rw] = jiffies + tg->td->throtl_slice;
throtl_log(&tg->service_queue,
"[%c] new slice with credit start=%lu end=%lu jiffies=%lu",
rw == READ ? 'R' : 'W', tg->slice_start[rw],
@@ -661,7 +664,7 @@ static inline void throtl_start_new_slice(struct throtl_grp *tg, bool rw)
tg->bytes_disp[rw] = 0;
tg->io_disp[rw] = 0;
tg->slice_start[rw] = jiffies;
- tg->slice_end[rw] = jiffies + throtl_slice;
+ tg->slice_end[rw] = jiffies + tg->td->throtl_slice;
throtl_log(&tg->service_queue,
"[%c] new slice start=%lu end=%lu jiffies=%lu",
rw == READ ? 'R' : 'W', tg->slice_start[rw],
@@ -671,13 +674,13 @@ static inline void throtl_start_new_slice(struct throtl_grp *tg, bool rw)
static inline void throtl_set_slice_end(struct throtl_grp *tg, bool rw,
unsigned long jiffy_end)
{
- tg->slice_end[rw] = roundup(jiffy_end, throtl_slice);
+ tg->slice_end[rw] = roundup(jiffy_end, tg->td->throtl_slice);
}
static inline void throtl_extend_slice(struct throtl_grp *tg, bool rw,
unsigned long jiffy_end)
{
- tg->slice_end[rw] = roundup(jiffy_end, throtl_slice);
+ tg->slice_end[rw] = roundup(jiffy_end, tg->td->throtl_slice);
throtl_log(&tg->service_queue,
"[%c] extend slice start=%lu end=%lu jiffies=%lu",
rw == READ ? 'R' : 'W', tg->slice_start[rw],
@@ -717,19 +720,19 @@ static inline void throtl_trim_slice(struct throtl_grp *tg, bool rw)
* is bad because it does not allow new slice to start.
*/
- throtl_set_slice_end(tg, rw, jiffies + throtl_slice);
+ throtl_set_slice_end(tg, rw, jiffies + tg->td->throtl_slice);
time_elapsed = jiffies - tg->slice_start[rw];
- nr_slices = time_elapsed / throtl_slice;
+ nr_slices = time_elapsed / tg->td->throtl_slice;
if (!nr_slices)
return;
- tmp = tg_bps_limit(tg, rw) * throtl_slice * nr_slices;
+ tmp = tg_bps_limit(tg, rw) * tg->td->throtl_slice * nr_slices;
do_div(tmp, HZ);
bytes_trim = tmp;
- io_trim = (tg_iops_limit(tg, rw) * throtl_slice * nr_slices)/HZ;
+ io_trim = (tg_iops_limit(tg, rw) * tg->td->throtl_slice * nr_slices)/HZ;
if (!bytes_trim && !io_trim)
return;
@@ -744,7 +747,7 @@ static inline void throtl_trim_slice(struct throtl_grp *tg, bool rw)
else
tg->io_disp[rw] = 0;
- tg->slice_start[rw] += nr_slices * throtl_slice;
+ tg->slice_start[rw] += nr_slices * tg->td->throtl_slice;
throtl_log(&tg->service_queue,
"[%c] trim slice nr=%lu bytes=%llu io=%lu start=%lu end=%lu jiffies=%lu",
@@ -764,9 +767,9 @@ static bool tg_with_in_iops_limit(struct throtl_grp *tg, struct bio *bio,
/* Slice has just started. Consider one slice interval */
if (!jiffy_elapsed)
- jiffy_elapsed_rnd = throtl_slice;
+ jiffy_elapsed_rnd = tg->td->throtl_slice;
- jiffy_elapsed_rnd = roundup(jiffy_elapsed_rnd, throtl_slice);
+ jiffy_elapsed_rnd = roundup(jiffy_elapsed_rnd, tg->td->throtl_slice);
/*
* jiffy_elapsed_rnd should not be a big value as minimum iops can be
@@ -813,9 +816,9 @@ static bool tg_with_in_bps_limit(struct throtl_grp *tg, struct bio *bio,
/* Slice has just started. Consider one slice interval */
if (!jiffy_elapsed)
- jiffy_elapsed_rnd = throtl_slice;
+ jiffy_elapsed_rnd = tg->td->throtl_slice;
- jiffy_elapsed_rnd = roundup(jiffy_elapsed_rnd, throtl_slice);
+ jiffy_elapsed_rnd = roundup(jiffy_elapsed_rnd, tg->td->throtl_slice);
tmp = tg_bps_limit(tg, rw) * jiffy_elapsed_rnd;
do_div(tmp, HZ);
@@ -878,8 +881,8 @@ static bool tg_may_dispatch(struct throtl_grp *tg, struct bio *bio,
if (throtl_slice_used(tg, rw))
throtl_start_new_slice(tg, rw);
else {
- if (time_before(tg->slice_end[rw], jiffies + throtl_slice))
- throtl_extend_slice(tg, rw, jiffies + throtl_slice);
+ if (time_before(tg->slice_end[rw], jiffies + tg->td->throtl_slice))
+ throtl_extend_slice(tg, rw, jiffies + tg->td->throtl_slice);
}
if (tg_with_in_bps_limit(tg, bio, &bps_wait) &&
@@ -1628,7 +1631,7 @@ static bool throtl_can_upgrade(struct throtl_data *td,
if (td->limit_index != LIMIT_HIGH)
return false;
- if (time_before(jiffies, td->high_downgrade_time + throtl_slice))
+ if (time_before(jiffies, td->high_downgrade_time + td->throtl_slice))
return false;
rcu_read_lock();
@@ -1685,8 +1688,8 @@ static bool throtl_downgrade_check_one(struct throtl_grp *tg)
* If cgroup is below high limit, consider downgrade and throttle other
* cgroups
*/
- if (time_after_eq(now, td->high_upgrade_time + throtl_slice) &&
- time_after_eq(now, tg_last_high_overflow_time(tg) + throtl_slice))
+ if (time_after_eq(now, td->high_upgrade_time + td->throtl_slice) &&
+ time_after_eq(now, tg_last_high_overflow_time(tg) + td->throtl_slice))
return true;
return false;
}
@@ -1719,10 +1722,10 @@ static void throtl_downgrade_check(struct throtl_grp *tg)
return;
if (!list_empty(&tg_to_blkg(tg)->blkcg->css.children))
return;
- if (time_after(tg->last_check_time + throtl_slice, now))
+ if (time_after(tg->last_check_time + tg->td->throtl_slice, now))
return;
- if (time_before(now, tg_last_high_overflow_time(tg) + throtl_slice))
+ if (time_before(now, tg_last_high_overflow_time(tg) + tg->td->throtl_slice))
return;
elapsed_time = now - tg->last_check_time;
@@ -1960,6 +1963,7 @@ int blk_throtl_init(struct request_queue *q)
q->td = td;
td->queue = q;
+ td->throtl_slice = DFL_THROTL_SLICE;
td->limit_valid[LIMIT_HIGH] = false;
td->limit_valid[LIMIT_MAX] = true;
@@ -1981,6 +1985,30 @@ void blk_throtl_exit(struct request_queue *q)
kfree(q->td);
}
+ssize_t blk_throtl_slice_show(struct request_queue *q, char *page)
+{
+ if (!q->td)
+ return -EINVAL;
+ return sprintf(page, "%ums\n", jiffies_to_msecs(q->td->throtl_slice));
+}
+
+ssize_t blk_throtl_slice_store(struct request_queue *q,
+ const char *page, size_t count)
+{
+ unsigned long v;
+ unsigned long t;
+
+ if (!q->td)
+ return -EINVAL;
+ if (kstrtoul(page, 10, &v))
+ return -EINVAL;
+ t = msecs_to_jiffies(v);
+ if (t == 0 || t > MAX_THROTL_SLICE)
+ return -EINVAL;
+ q->td->throtl_slice = t;
+ return count;
+}
+
static int __init throtl_init(void)
{
kthrotld_workqueue = alloc_workqueue("kthrotld", WQ_MEM_RECLAIM, 0);
diff --git a/block/blk.h b/block/blk.h
index c37492f..8ad6068 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -294,6 +294,9 @@ static inline struct io_context *create_io_context(gfp_t gfp_mask, int node)
extern void blk_throtl_drain(struct request_queue *q);
extern int blk_throtl_init(struct request_queue *q);
extern void blk_throtl_exit(struct request_queue *q);
+extern ssize_t blk_throtl_slice_show(struct request_queue *q, char *page);
+extern ssize_t blk_throtl_slice_store(struct request_queue *q,
+ const char *page, size_t count);
#else /* CONFIG_BLK_DEV_THROTTLING */
static inline void blk_throtl_drain(struct request_queue *q) { }
static inline int blk_throtl_init(struct request_queue *q) { return 0; }
--
2.8.0.rc2
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH V2 08/11] blk-throttle: detect completed idle cgroup
2016-09-15 16:23 [PATCH V2 00/11] block-throttle: add .high limit Shaohua Li
` (6 preceding siblings ...)
2016-09-15 16:23 ` [PATCH V2 07/11] blk-throttle: make throtl_slice tunable Shaohua Li
@ 2016-09-15 16:23 ` Shaohua Li
2016-09-15 16:23 ` [PATCH V2 09/11] block-throttle: make bandwidth change smooth Shaohua Li
` (2 subsequent siblings)
10 siblings, 0 replies; 13+ messages in thread
From: Shaohua Li @ 2016-09-15 16:23 UTC (permalink / raw)
To: linux-block, linux-kernel; +Cc: axboe, Kernel-team, tj, jmoyer, vgoyal
cgroup could be assigned a limit, but doesn't dispatch enough IO, eg the
cgroup is idle. When this happens, the cgroup doesn't hit its limit, so
we can't move the state machine to higher level and all cgroups will be
throttled to thier lower limit, so we waste bandwidth. Detecting idle
cgroup is hard. This patch handles a simple case, a cgroup doesn't
dispatch any IO. We ignore such cgroup's limit, so other cgroups can use
the bandwidth.
Signed-off-by: Shaohua Li <shli@fb.com>
---
block/blk-throttle.c | 17 ++++++++++++++++-
1 file changed, 16 insertions(+), 1 deletion(-)
diff --git a/block/blk-throttle.c b/block/blk-throttle.c
index bc94086..e4713b1 100644
--- a/block/blk-throttle.c
+++ b/block/blk-throttle.c
@@ -144,6 +144,8 @@ struct throtl_grp {
unsigned long last_check_time;
+ unsigned long last_dispatch_time[2];
+
/* When did we start a new slice */
unsigned long slice_start[2];
unsigned long slice_end[2];
@@ -438,11 +440,14 @@ static void tg_update_has_rules(struct throtl_grp *tg)
static void throtl_pd_online(struct blkg_policy_data *pd)
{
+ struct throtl_grp *tg = pd_to_tg(pd);
/*
* We don't want new groups to escape the limits of its ancestors.
* Update has_rules[] after a new group is brought online.
*/
- tg_update_has_rules(pd_to_tg(pd));
+ tg_update_has_rules(tg);
+ tg->last_dispatch_time[READ] = jiffies;
+ tg->last_dispatch_time[WRITE] = jiffies;
}
static void blk_throtl_update_valid_limit(struct throtl_data *td)
@@ -1604,6 +1609,12 @@ static bool throtl_upgrade_check_one(struct throtl_grp *tg)
if (write_limit && sq->nr_queued[WRITE] &&
(!read_limit || sq->nr_queued[READ]))
return true;
+
+ if (time_after_eq(jiffies,
+ tg->last_dispatch_time[READ] + tg->td->throtl_slice) &&
+ time_after_eq(jiffies,
+ tg->last_dispatch_time[WRITE] + tg->td->throtl_slice))
+ return true;
return false;
}
@@ -1684,6 +1695,9 @@ static bool throtl_downgrade_check_one(struct throtl_grp *tg)
struct throtl_data *td = tg->td;
unsigned long now = jiffies;
+ if (time_after_eq(now, tg->last_dispatch_time[READ] + td->throtl_slice) &&
+ time_after_eq(now, tg->last_dispatch_time[WRITE] + td->throtl_slice))
+ return false;
/*
* If cgroup is below high limit, consider downgrade and throttle other
* cgroups
@@ -1802,6 +1816,7 @@ bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg,
again:
while (true) {
+ tg->last_dispatch_time[rw] = jiffies;
if (tg->last_high_overflow_time[rw] == 0)
tg->last_high_overflow_time[rw] = jiffies;
throtl_downgrade_check(tg);
--
2.8.0.rc2
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH V2 09/11] block-throttle: make bandwidth change smooth
2016-09-15 16:23 [PATCH V2 00/11] block-throttle: add .high limit Shaohua Li
` (7 preceding siblings ...)
2016-09-15 16:23 ` [PATCH V2 08/11] blk-throttle: detect completed idle cgroup Shaohua Li
@ 2016-09-15 16:23 ` Shaohua Li
2016-09-15 16:23 ` [PATCH V2 10/11] block-throttle: add a simple idle detection Shaohua Li
2016-09-15 16:23 ` [PATCH V2 11/11] blk-throttle: ignore idle cgroup limit Shaohua Li
10 siblings, 0 replies; 13+ messages in thread
From: Shaohua Li @ 2016-09-15 16:23 UTC (permalink / raw)
To: linux-block, linux-kernel; +Cc: axboe, Kernel-team, tj, jmoyer, vgoyal
When cgroups all reach high limit, cgroups can dispatch more IO. This
could make some cgroups dispatch more IO but others not, and even some
cgroups could dispatch less IO than their high limit. For example, cg1
high limit 10MB/s, cg2 limit 80MB/s, assume disk maximum bandwidth is
120M/s for the workload. Their bps could something like this:
cg1/cg2 bps: T1: 10/80 -> T2: 60/60 -> T3: 10/80
At T1, all cgroups reach high limit, so they can dispatch more IO later.
Then cg1 dispatch more IO and cg2 has no room to dispatch enough IO. At
T2, cg2 only dispatches 60M/s. Since We detect cg2 dispatches less IO
than its high limit 80M/s, we downgrade the queue from LIMIT_MAX to
LIMIT_HIGH, then all cgroups are throttled to their high limit (T3). cg2
will have bandwidth below its high limit at most time.
The big problem here is we don't know the maximum bandwidth of the
workload, so we can't make smart decision to avoid the situation. This
patch makes cgroup bandwidth change smooth. After disk upgrades from
LIMIT_HIGH to LIMIT_MAX, we don't allow cgroups use all bandwidth upto
their max limit immediately. Their bandwidth limit will be increased
gradually to avoid above situation. So above example will became
something like:
cg1/cg2 bps: 10/80 -> 15/105 -> 20/100 -> 25/95 -> 30/90 -> 35/85 -> 40/80
-> 45/75 -> 10/80
In this way cgroups bandwidth will be above their limit in majority
time, this still doesn't fully utilize disk bandwidth, but that's
something we pay for sharing.
Note this doesn't completely avoid cgroup running under its high limit.
The best way to guarantee cgroup doesn't run under its limit is to set
max limit. For example, if we set cg1 max limit to 40, cg2 will never
run under its high limit.
Signed-off-by: Shaohua Li <shli@fb.com>
---
block/blk-throttle.c | 44 ++++++++++++++++++++++++++++++++++++++++----
1 file changed, 40 insertions(+), 4 deletions(-)
diff --git a/block/blk-throttle.c b/block/blk-throttle.c
index e4713b1..d956831 100644
--- a/block/blk-throttle.c
+++ b/block/blk-throttle.c
@@ -170,6 +170,8 @@ struct throtl_data
unsigned long high_upgrade_time;
unsigned long high_downgrade_time;
+
+ unsigned int scale;
};
static void throtl_pending_timer_fn(unsigned long arg);
@@ -224,12 +226,28 @@ static struct throtl_data *sq_to_td(struct throtl_service_queue *sq)
static uint64_t tg_bps_limit(struct throtl_grp *tg, int rw)
{
struct blkcg_gq *blkg = tg_to_blkg(tg);
+ struct throtl_data *td;
uint64_t ret;
if (cgroup_subsys_on_dfl(io_cgrp_subsys) && !blkg->parent)
return -1;
- ret = tg->bps[rw][tg->td->limit_index];
- if (ret == -1 && tg->td->limit_index == LIMIT_HIGH)
+ td = tg->td;
+ ret = tg->bps[rw][td->limit_index];
+ if (td->limit_index == LIMIT_MAX && tg->bps[rw][LIMIT_HIGH] != -1) {
+ uint64_t increase;
+
+ if (td->scale < 4096 && time_after_eq(jiffies,
+ td->high_upgrade_time + td->scale * td->throtl_slice)) {
+ uint64_t time = jiffies - td->high_upgrade_time;
+
+ do_div(time, td->throtl_slice);
+ td->scale = time;
+ }
+ increase = (tg->bps[rw][LIMIT_HIGH] >> 1) * td->scale;
+ ret = min(tg->bps[rw][LIMIT_MAX],
+ tg->bps[rw][LIMIT_HIGH] + increase);
+ }
+ if (ret == -1 && td->limit_index == LIMIT_HIGH)
return tg->bps[rw][LIMIT_MAX];
return ret;
@@ -238,12 +256,29 @@ static uint64_t tg_bps_limit(struct throtl_grp *tg, int rw)
static unsigned int tg_iops_limit(struct throtl_grp *tg, int rw)
{
struct blkcg_gq *blkg = tg_to_blkg(tg);
+ struct throtl_data *td;
unsigned int ret;
if (cgroup_subsys_on_dfl(io_cgrp_subsys) && !blkg->parent)
return -1;
- ret = tg->iops[rw][tg->td->limit_index];
- if (ret == -1 && tg->td->limit_index == LIMIT_HIGH)
+ td = tg->td;
+ ret = tg->iops[rw][td->limit_index];
+ if (td->limit_index == LIMIT_MAX && tg->iops[rw][LIMIT_HIGH] != -1) {
+ uint64_t increase;
+
+ if (td->scale < 4096 && time_after_eq(jiffies,
+ td->high_upgrade_time + td->scale * td->throtl_slice)) {
+ uint64_t time = jiffies - td->high_upgrade_time;
+
+ do_div(time, td->throtl_slice);
+ td->scale = time;
+ }
+
+ increase = (tg->iops[rw][LIMIT_HIGH] >> 1) * td->scale;
+ ret = min(tg->iops[rw][LIMIT_MAX],
+ tg->iops[rw][LIMIT_HIGH] + (unsigned int)increase);
+ }
+ if (ret == -1 && td->limit_index == LIMIT_HIGH)
return tg->iops[rw][LIMIT_MAX];
return ret;
}
@@ -1669,6 +1704,7 @@ static void throtl_upgrade_state(struct throtl_data *td)
td->limit_index = LIMIT_MAX;
td->high_upgrade_time = jiffies;
+ td->scale = 0;
rcu_read_lock();
blkg_for_each_descendant_post(blkg, pos_css, td->queue->root_blkg) {
struct throtl_grp *tg = blkg_to_tg(blkg);
--
2.8.0.rc2
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH V2 10/11] block-throttle: add a simple idle detection
2016-09-15 16:23 [PATCH V2 00/11] block-throttle: add .high limit Shaohua Li
` (8 preceding siblings ...)
2016-09-15 16:23 ` [PATCH V2 09/11] block-throttle: make bandwidth change smooth Shaohua Li
@ 2016-09-15 16:23 ` Shaohua Li
2016-09-16 1:14 ` kbuild test robot
2016-09-15 16:23 ` [PATCH V2 11/11] blk-throttle: ignore idle cgroup limit Shaohua Li
10 siblings, 1 reply; 13+ messages in thread
From: Shaohua Li @ 2016-09-15 16:23 UTC (permalink / raw)
To: linux-block, linux-kernel; +Cc: axboe, Kernel-team, tj, jmoyer, vgoyal
A cgroup gets assigned a high limit, but the cgroup could never dispatch
enough IO to cross the high limit. In such case, the queue state machine
will remain in LIMIT_HIGH state and all other cgroups will be throttled
according to high limit. This is unfair for other cgroups. We should
treat the cgroup idle and upgrade the state machine to higher state.
We also have a downgrade logic. If the state machine upgrades because of
cgroup idle (real idle), the state machine will downgrade soon as the
cgroup is below its high limit. This isn't what we want. A more
complicated case is cgroup isn't idle when queue is in LIMIT_HIGH. But
when queue gets upgraded to higher state, other cgroups could dispatch
more IO and this cgroup can't dispatch enough IO, so the cgroup is below
its high limit and looks like idle (fake idle). In this case, the queue
should downgrade soon. The key to determine if we should do downgrade is
to detect if cgroup is truely idle.
Unfortunately it's very hard to determine if a cgroup is real idle. This
patch uses the 'think time check' idea from CFQ for the purpose. Please
note, the idea doesn't work for all workloads. For example, a workload
with io depth 8 has disk utilization 100%, hence think time is 0, eg,
not idle. But the workload can run higher bandwidth with io depth 16.
Compared to io depth 16, the io depth 8 workload is idle. We use the
idea to roughly determine if a cgroup is idle.
We treat a cgroup idle if its think time is above a threshold (by
default 50us). 50us is choosen arbitrarily so far, but seems ok in
test and should allow the cpu does a lot of things before dispatch IO.
There is a knob to let user configure the threshold too.
Signed-off-by: Shaohua Li <shli@fb.com>
---
block/bio.c | 2 ++
block/blk-sysfs.c | 7 ++++
block/blk-throttle.c | 86 ++++++++++++++++++++++++++++++++++++++++++++++-
block/blk.h | 6 ++++
include/linux/blk_types.h | 1 +
5 files changed, 101 insertions(+), 1 deletion(-)
diff --git a/block/bio.c b/block/bio.c
index aa73540..06e414c 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -30,6 +30,7 @@
#include <linux/cgroup.h>
#include <trace/events/block.h>
+#include "blk.h"
/*
* Test patch to inline a certain number of bi_io_vec's inside the bio
@@ -1758,6 +1759,7 @@ void bio_endio(struct bio *bio)
goto again;
}
+ blk_throtl_bio_endio(bio);
if (bio->bi_end_io)
bio->bi_end_io(bio);
}
diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c
index 610f08d..209b67c 100644
--- a/block/blk-sysfs.c
+++ b/block/blk-sysfs.c
@@ -532,6 +532,12 @@ static struct queue_sysfs_entry throtl_slice_entry = {
.show = blk_throtl_slice_show,
.store = blk_throtl_slice_store,
};
+
+static struct queue_sysfs_entry throtl_idle_threshold_entry = {
+ .attr = {.name = "throttling_idle_threshold", .mode = S_IRUGO | S_IWUSR },
+ .show = blk_throtl_idle_threshold_show,
+ .store = blk_throtl_idle_threshold_store,
+};
#endif
static struct attribute *default_attrs[] = {
@@ -563,6 +569,7 @@ static struct attribute *default_attrs[] = {
&queue_dax_entry.attr,
#ifdef CONFIG_BLK_DEV_THROTTLING
&throtl_slice_entry.attr,
+ &throtl_idle_threshold_entry.attr,
#endif
NULL,
};
diff --git a/block/blk-throttle.c b/block/blk-throttle.c
index d956831..0810e1b 100644
--- a/block/blk-throttle.c
+++ b/block/blk-throttle.c
@@ -21,6 +21,7 @@ static int throtl_quantum = 32;
/* Throttling is performed over 100ms slice and after that slice is renewed */
#define DFL_THROTL_SLICE (HZ / 10)
#define MAX_THROTL_SLICE (HZ / 5)
+#define DFL_IDLE_THRESHOLD (50 * 1000)
static struct blkcg_policy blkcg_policy_throtl;
@@ -149,6 +150,10 @@ struct throtl_grp {
/* When did we start a new slice */
unsigned long slice_start[2];
unsigned long slice_end[2];
+
+ u64 last_finish_time;
+ u64 checked_last_finish_time;
+ u64 avg_ttime;
};
struct throtl_data
@@ -172,6 +177,8 @@ struct throtl_data
unsigned long high_downgrade_time;
unsigned int scale;
+
+ u64 idle_ttime_threshold;
};
static void throtl_pending_timer_fn(unsigned long arg);
@@ -1624,6 +1631,14 @@ static unsigned long tg_last_high_overflow_time(struct throtl_grp *tg)
return ret;
}
+static bool throtl_tg_is_idle(struct throtl_grp *tg)
+{
+ /* cgroup is idle if average think time is more than 50us */
+ return ktime_get_ns() - tg->last_finish_time >
+ 4 * tg->td->idle_ttime_threshold ||
+ tg->avg_ttime > tg->td->idle_ttime_threshold;
+}
+
static bool throtl_upgrade_check_one(struct throtl_grp *tg)
{
struct throtl_service_queue *sq = &tg->service_queue;
@@ -1828,6 +1843,19 @@ static void throtl_downgrade_check(struct throtl_grp *tg)
tg->last_io_disp[WRITE] = 0;
}
+static void blk_throtl_update_ttime(struct throtl_grp *tg)
+{
+ u64 now = ktime_get_ns();
+ u64 last_finish_time = tg->last_finish_time;
+
+ if (now <= last_finish_time || last_finish_time == 0 ||
+ last_finish_time == tg->checked_last_finish_time)
+ return;
+
+ tg->avg_ttime = (tg->avg_ttime * 31 + now - last_finish_time) >> 5;
+ tg->checked_last_finish_time = last_finish_time;
+}
+
bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg,
struct bio *bio)
{
@@ -1848,6 +1876,11 @@ bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg,
if (unlikely(blk_queue_bypass(q)))
goto out_unlock;
+ bio_associate_current(bio);
+ bio->bi_cg_private = q;
+
+ blk_throtl_update_ttime(tg);
+
sq = &tg->service_queue;
again:
@@ -1907,7 +1940,6 @@ bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg,
tg->last_high_overflow_time[rw] = jiffies;
- bio_associate_current(bio);
tg->td->nr_queued[rw]++;
throtl_add_bio_tg(bio, qn, tg);
throttled = true;
@@ -1936,6 +1968,34 @@ bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg,
return throttled;
}
+void blk_throtl_bio_endio(struct bio *bio)
+{
+ struct blkcg *blkcg;
+ struct blkcg_gq *blkg;
+ struct throtl_grp *tg;
+ struct request_queue *q;
+
+ q = bio->bi_cg_private;
+ if (!q)
+ return;
+ bio->bi_cg_private = NULL;
+
+ rcu_read_lock();
+ blkcg = bio_blkcg(bio);
+ if (!blkcg)
+ goto end;
+ blkg = blkg_lookup(blkcg, q);
+ if (!blkg)
+ goto end;
+
+ tg = blkg_to_tg(blkg ?: q->root_blkg);
+
+ tg->last_finish_time = ktime_get_ns();
+
+end:
+ rcu_read_unlock();
+}
+
/*
* Dispatch all bios from all children tg's queued on @parent_sq. On
* return, @parent_sq is guaranteed to not have any active children tg's
@@ -2021,6 +2081,8 @@ int blk_throtl_init(struct request_queue *q)
td->limit_index = LIMIT_MAX;
td->high_upgrade_time = jiffies;
td->high_downgrade_time = jiffies;
+
+ td->idle_ttime_threshold = DFL_IDLE_THRESHOLD;
/* activate policy */
ret = blkcg_activate_policy(q, &blkcg_policy_throtl);
if (ret)
@@ -2060,6 +2122,28 @@ ssize_t blk_throtl_slice_store(struct request_queue *q,
return count;
}
+ssize_t blk_throtl_idle_threshold_show(struct request_queue *q, char *page)
+{
+ if (!q->td)
+ return -EINVAL;
+ return sprintf(page, "%lluus\n", q->td->idle_ttime_threshold / 1000);
+}
+
+ssize_t blk_throtl_idle_threshold_store(struct request_queue *q,
+ const char *page, size_t count)
+{
+ unsigned long v;
+
+ if (!q->td)
+ return -EINVAL;
+ if (kstrtoul(page, 10, &v))
+ return -EINVAL;
+ if (v == 0)
+ return -EINVAL;
+ q->td->idle_ttime_threshold = v * 1000;
+ return count;
+}
+
static int __init throtl_init(void)
{
kthrotld_workqueue = alloc_workqueue("kthrotld", WQ_MEM_RECLAIM, 0);
diff --git a/block/blk.h b/block/blk.h
index 8ad6068..8e1aeca 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -297,10 +297,16 @@ extern void blk_throtl_exit(struct request_queue *q);
extern ssize_t blk_throtl_slice_show(struct request_queue *q, char *page);
extern ssize_t blk_throtl_slice_store(struct request_queue *q,
const char *page, size_t count);
+extern ssize_t blk_throtl_idle_threshold_show(struct request_queue *q,
+ char *page);
+extern ssize_t blk_throtl_idle_threshold_store(struct request_queue *q,
+ const char *page, size_t count);
+extern void blk_throtl_bio_endio(struct bio *bio);
#else /* CONFIG_BLK_DEV_THROTTLING */
static inline void blk_throtl_drain(struct request_queue *q) { }
static inline int blk_throtl_init(struct request_queue *q) { return 0; }
static inline void blk_throtl_exit(struct request_queue *q) { }
+static inline void blk_throtl_bio_endio(struct bio *bio) { }
#endif /* CONFIG_BLK_DEV_THROTTLING */
#endif /* BLK_INTERNAL_H */
diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h
index 436f43f..be9d10d 100644
--- a/include/linux/blk_types.h
+++ b/include/linux/blk_types.h
@@ -60,6 +60,7 @@ struct bio {
*/
struct io_context *bi_ioc;
struct cgroup_subsys_state *bi_css;
+ void *bi_cg_private;
#endif
union {
#if defined(CONFIG_BLK_DEV_INTEGRITY)
--
2.8.0.rc2
^ permalink raw reply related [flat|nested] 13+ messages in thread* Re: [PATCH V2 10/11] block-throttle: add a simple idle detection
2016-09-15 16:23 ` [PATCH V2 10/11] block-throttle: add a simple idle detection Shaohua Li
@ 2016-09-16 1:14 ` kbuild test robot
0 siblings, 0 replies; 13+ messages in thread
From: kbuild test robot @ 2016-09-16 1:14 UTC (permalink / raw)
To: Shaohua Li
Cc: kbuild-all, linux-block, linux-kernel, axboe, Kernel-team, tj,
jmoyer, vgoyal
[-- Attachment #1: Type: text/plain, Size: 1680 bytes --]
Hi Shaohua,
[auto build test ERROR on block/for-next]
[also build test ERROR on v4.8-rc6 next-20160915]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system]
[Suggest to use git(>=2.9.0) format-patch --base=<commit> (or --base=auto for convenience) to record what (public, well-known) commit your patch series was built on]
[Check https://git-scm.com/docs/git-format-patch for more information]
url: https://github.com/0day-ci/linux/commits/Shaohua-Li/block-throttle-add-high-limit/20160916-042818
base: https://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux-block.git for-next
config: arm-arm5 (attached as .config)
compiler: arm-linux-gnueabi-gcc (Debian 5.4.0-6) 5.4.0 20160609
reproduce:
wget https://git.kernel.org/cgit/linux/kernel/git/wfg/lkp-tests.git/plain/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
make.cross ARCH=arm
All errors (new ones prefixed by >>):
block/built-in.o: In function `blk_throtl_idle_threshold_show':
>> block/blk-throttle.c:2129: undefined reference to `__aeabi_uldivmod'
vim +2129 block/blk-throttle.c
2123 }
2124
2125 ssize_t blk_throtl_idle_threshold_show(struct request_queue *q, char *page)
2126 {
2127 if (!q->td)
2128 return -EINVAL;
> 2129 return sprintf(page, "%lluus\n", q->td->idle_ttime_threshold / 1000);
2130 }
2131
2132 ssize_t blk_throtl_idle_threshold_store(struct request_queue *q,
---
0-DAY kernel test infrastructure Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all Intel Corporation
[-- Attachment #2: .config.gz --]
[-- Type: application/octet-stream, Size: 27345 bytes --]
^ permalink raw reply [flat|nested] 13+ messages in thread
* [PATCH V2 11/11] blk-throttle: ignore idle cgroup limit
2016-09-15 16:23 [PATCH V2 00/11] block-throttle: add .high limit Shaohua Li
` (9 preceding siblings ...)
2016-09-15 16:23 ` [PATCH V2 10/11] block-throttle: add a simple idle detection Shaohua Li
@ 2016-09-15 16:23 ` Shaohua Li
10 siblings, 0 replies; 13+ messages in thread
From: Shaohua Li @ 2016-09-15 16:23 UTC (permalink / raw)
To: linux-block, linux-kernel; +Cc: axboe, Kernel-team, tj, jmoyer, vgoyal
Last patch introduces a way to detect idle cgroup. We use it to make
upgrade/downgrade decision.
Signed-off-by: Shaohua Li <shli@fb.com>
---
block/blk-throttle.c | 30 ++++++++++++++++++------------
1 file changed, 18 insertions(+), 12 deletions(-)
diff --git a/block/blk-throttle.c b/block/blk-throttle.c
index 0810e1b..276e71e 100644
--- a/block/blk-throttle.c
+++ b/block/blk-throttle.c
@@ -145,8 +145,6 @@ struct throtl_grp {
unsigned long last_check_time;
- unsigned long last_dispatch_time[2];
-
/* When did we start a new slice */
unsigned long slice_start[2];
unsigned long slice_end[2];
@@ -488,8 +486,6 @@ static void throtl_pd_online(struct blkg_policy_data *pd)
* Update has_rules[] after a new group is brought online.
*/
tg_update_has_rules(tg);
- tg->last_dispatch_time[READ] = jiffies;
- tg->last_dispatch_time[WRITE] = jiffies;
}
static void blk_throtl_update_valid_limit(struct throtl_data *td)
@@ -1661,9 +1657,8 @@ static bool throtl_upgrade_check_one(struct throtl_grp *tg)
return true;
if (time_after_eq(jiffies,
- tg->last_dispatch_time[READ] + tg->td->throtl_slice) &&
- time_after_eq(jiffies,
- tg->last_dispatch_time[WRITE] + tg->td->throtl_slice))
+ tg_last_high_overflow_time(tg) + tg->td->throtl_slice) &&
+ throtl_tg_is_idle(tg))
return true;
return false;
}
@@ -1712,6 +1707,19 @@ static bool throtl_can_upgrade(struct throtl_data *td,
return true;
}
+static void throtl_upgrade_check(struct throtl_grp *tg)
+{
+ if (tg->td->limit_index != LIMIT_HIGH)
+ return;
+
+ if (!time_after_eq(jiffies,
+ __tg_last_high_overflow_time(tg) + tg->td->throtl_slice))
+ return;
+
+ if (throtl_can_upgrade(tg->td, NULL))
+ throtl_upgrade_state(tg->td);
+}
+
static void throtl_upgrade_state(struct throtl_data *td)
{
struct cgroup_subsys_state *pos_css;
@@ -1746,15 +1754,13 @@ static bool throtl_downgrade_check_one(struct throtl_grp *tg)
struct throtl_data *td = tg->td;
unsigned long now = jiffies;
- if (time_after_eq(now, tg->last_dispatch_time[READ] + td->throtl_slice) &&
- time_after_eq(now, tg->last_dispatch_time[WRITE] + td->throtl_slice))
- return false;
/*
* If cgroup is below high limit, consider downgrade and throttle other
* cgroups
*/
if (time_after_eq(now, td->high_upgrade_time + td->throtl_slice) &&
- time_after_eq(now, tg_last_high_overflow_time(tg) + td->throtl_slice))
+ time_after_eq(now, tg_last_high_overflow_time(tg) + td->throtl_slice) &&
+ !throtl_tg_is_idle(tg))
return true;
return false;
}
@@ -1885,10 +1891,10 @@ bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg,
again:
while (true) {
- tg->last_dispatch_time[rw] = jiffies;
if (tg->last_high_overflow_time[rw] == 0)
tg->last_high_overflow_time[rw] = jiffies;
throtl_downgrade_check(tg);
+ throtl_upgrade_check(tg);
/* throtl is FIFO - if bios are already queued, should queue */
if (sq->nr_queued[rw])
break;
--
2.8.0.rc2
^ permalink raw reply related [flat|nested] 13+ messages in thread