Re: [PATCH V4 09/11] mmc: block: Add CQE support

Shawn Lin
2017-08-02T01:31:07Z

On 2017/8/1 18:06, Adrian Hunter wrote:
> On 01/08/17 11:57, Shawn Lin wrote:
>> Hi Adrian,
>>
>> On 2017/7/21 17:49, Adrian Hunter wrote:
>>> Add CQE support to the block driver, including:
>>> - optionally using DCMD for flush requests
>>> - manually issuing discard requests
>>> - issuing read / write requests to the CQE
>>> - supporting block-layer timeouts
>>> - handling recovery
>>> - supporting re-tuning
>>>
>>> Signed-off-by: Adrian Hunter <adrian.hunter@intel.com>
>>> ---
>>> drivers/mmc/core/block.c | 195 ++++++++++++++++++++++++++++++++-
>>> drivers/mmc/core/block.h | 7 ++
>>> drivers/mmc/core/queue.c | 273
>>> ++++++++++++++++++++++++++++++++++++++++++++++-
>>> drivers/mmc/core/queue.h | 42 +++++++-
>>> 4 files changed, 510 insertions(+), 7 deletions(-)
>>>
>>> diff --git a/drivers/mmc/core/block.c b/drivers/mmc/core/block.c
>>> index 915290c74363..2d25115637b7 100644
>>> --- a/drivers/mmc/core/block.c
>>> +++ b/drivers/mmc/core/block.c
>>> @@ -109,6 +109,7 @@ struct mmc_blk_data {
>>> #define MMC_BLK_WRITE BIT(1)
>>> #define MMC_BLK_DISCARD BIT(2)
>>> #define MMC_BLK_SECDISCARD BIT(3)
>>> +#define MMC_BLK_CQE_RECOVERY BIT(4)
>>> /*
>>> * Only set in main mmc_blk_data associated
>>> @@ -1612,6 +1613,198 @@ static void mmc_blk_data_prep(struct mmc_queue
>>> *mq, struct mmc_queue_req *mqrq,
>>> *do_data_tag_p = do_data_tag;
>>> }
>>> +#define MMC_CQE_RETRIES 2
>>> +
>>> +void mmc_blk_cqe_complete_rq(struct request *req)
>>> +{
>>> + struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
>>> + struct mmc_request *mrq = &mqrq->brq.mrq;
>>> + struct request_queue *q = req->q;
>>> + struct mmc_queue *mq = q->queuedata;
>>> + struct mmc_host *host = mq->card->host;
>>> + unsigned long flags;
>>> + bool put_card;
>>> + int err;
>>> +
>>> + mmc_cqe_post_req(host, mrq);
>>> +
>>> + spin_lock_irqsave(q->queue_lock, flags);
>>> +
>>> + mq->cqe_in_flight[mmc_cqe_issue_type(host, req)] -= 1;
>>> +
>>> + put_card = mmc_cqe_tot_in_flight(mq) == 0;
>>> +
>>> + if (mrq->cmd && mrq->cmd->error)
>>> + err = mrq->cmd->error;
>>> + else if (mrq->data && mrq->data->error)
>>> + err = mrq->data->error;
>>> + else
>>> + err = 0;
>>> +
>>> + if (err) {
>>> + if (mqrq->retries++ < MMC_CQE_RETRIES)
>>> + blk_requeue_request(q, req);
>>> + else
>>> + __blk_end_request_all(req, BLK_STS_IOERR);
>>> + } else if (mrq->data) {
>>> + if (__blk_end_request(req, BLK_STS_OK, mrq->data->bytes_xfered))
>>> + blk_requeue_request(q, req);
>>> + } else {
>>> + __blk_end_request_all(req, BLK_STS_OK);
>>> + }
>>> +
>>> + mmc_cqe_kick_queue(mq);
>>> +
>>> + spin_unlock_irqrestore(q->queue_lock, flags);
>>> +
>>> + if (put_card)
>>> + mmc_put_card(mq->card);
>>> +}
>>> +
>>> +void mmc_blk_cqe_recovery(struct mmc_queue *mq)
>>> +{
>>> + struct mmc_card *card = mq->card;
>>> + struct mmc_host *host = card->host;
>>> + int err;
>>> +
>>> + mmc_get_card(card);
>>> +
>>> + pr_debug("%s: CQE recovery start\n", mmc_hostname(host));
>>> +
>>> + mq->cqe_in_recovery = true;
>>> +
>>> + err = mmc_cqe_recovery(host);
>>> + if (err)
>>> + mmc_blk_reset(mq->blkdata, host, MMC_BLK_CQE_RECOVERY);
>>> + else
>>> + mmc_blk_reset_success(mq->blkdata, MMC_BLK_CQE_RECOVERY);
>>> +
>>> + mq->cqe_in_recovery = false;
>>> +
>>> + pr_debug("%s: CQE recovery done\n", mmc_hostname(host));
>>> +
>>> + mmc_put_card(card);
>>> +}
>>> +
>>> +static void mmc_blk_cqe_req_done(struct mmc_request *mrq)
>>> +{
>>> + struct mmc_queue_req *mqrq = container_of(mrq, struct mmc_queue_req,
>>> + brq.mrq);
>>> + struct request *req = mmc_queue_req_to_req(mqrq);
>>> + struct request_queue *q = req->q;
>>> + struct mmc_queue *mq = q->queuedata;
>>> +
>>> + /*
>>> + * Block layer timeouts race with completions which means the normal
>>> + * completion path cannot be used during recovery.
>>> + */
>>> + if (mq->cqe_in_recovery)
>>> + mmc_blk_cqe_complete_rq(req);
>>> + else
>>> + blk_complete_request(req);
>>> +}
>>> +
>>> +static int mmc_blk_cqe_start_req(struct mmc_host *host, struct
>>> mmc_request *mrq)
>>> +{
>>> + mrq->done = mmc_blk_cqe_req_done;
>>> + return mmc_cqe_start_req(host, mrq);
>>> +}
>>> +
>>> +static struct mmc_request *mmc_blk_cqe_prep_dcmd(struct mmc_queue_req *mqrq,
>>> + struct request *req)
>>> +{
>>> + struct mmc_blk_request *brq = &mqrq->brq;
>>> +
>>> + memset(brq, 0, sizeof(*brq));
>>> +
>>> + brq->mrq.cmd = &brq->cmd;
>>> + brq->mrq.tag = req->tag;
>>> +
>>> + return &brq->mrq;
>>> +}
>>> +
>>> +static int mmc_blk_cqe_issue_flush(struct mmc_queue *mq, struct request
>>> *req)
>>> +{
>>> + struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
>>> + struct mmc_request *mrq = mmc_blk_cqe_prep_dcmd(mqrq, req);
>>> +
>>> + mrq->cmd->opcode = MMC_SWITCH;
>>> + mrq->cmd->arg = (MMC_SWITCH_MODE_WRITE_BYTE << 24) |
>>> + (EXT_CSD_FLUSH_CACHE << 16) |
>>> + (1 << 8) |
>>> + EXT_CSD_CMD_SET_NORMAL;
>>> + mrq->cmd->flags = MMC_CMD_AC | MMC_RSP_R1B;
>>> +
>>> + return mmc_blk_cqe_start_req(mq->card->host, mrq);
>>> +}
>>> +
>>> +static int mmc_blk_cqe_issue_rw_rq(struct mmc_queue *mq, struct request
>>> *req)
>>> +{
>>> + struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
>>> +
>>> + mmc_blk_data_prep(mq, mqrq, 0, NULL, NULL);
>>> +
>>> + return mmc_blk_cqe_start_req(mq->card->host, &mqrq->brq.mrq);
>>> +}
>>> +
>>> +enum mmc_issued mmc_blk_cqe_issue_rq(struct mmc_queue *mq, struct request
>>> *req)
>>> +{
>>> + struct mmc_blk_data *md = mq->blkdata;
>>> + struct mmc_card *card = md->queue.card;
>>> + struct mmc_host *host = card->host;
>>> + int ret;
>>> +
>>> + ret = mmc_blk_part_switch(card, md);
>>> + if (ret)
>>> + return MMC_REQ_FAILED_TO_START;
>>> +
>>> + switch (mmc_cqe_issue_type(host, req)) {
>>> + case MMC_ISSUE_SYNC:
>>> + ret = host->cqe_ops->cqe_wait_for_idle(host);
>>> + if (ret)
>>> + return MMC_REQ_BUSY;
>>> + switch (req_op(req)) {
>>> + case REQ_OP_DRV_IN:
>>> + case REQ_OP_DRV_OUT:
>>> + mmc_blk_issue_drv_op(mq, req);
>>> + break;
>>> + case REQ_OP_DISCARD:
>>> + mmc_blk_issue_discard_rq(mq, req);
>>> + break;
>>> + case REQ_OP_SECURE_ERASE:
>>> + mmc_blk_issue_secdiscard_rq(mq, req);
>>> + break;
>>> + case REQ_OP_FLUSH:
>>> + mmc_blk_issue_flush(mq, req);
>>> + break;
>>> + default:
>>> + WARN_ON_ONCE(1);
>>> + return MMC_REQ_FAILED_TO_START;
>>> + }
>>> + return MMC_REQ_FINISHED;
>>> + case MMC_ISSUE_DCMD:
>>> + case MMC_ISSUE_ASYNC:
>>> + switch (req_op(req)) {
>>> + case REQ_OP_FLUSH:
>>> + ret = mmc_blk_cqe_issue_flush(mq, req);
>>> + break;
>>> + case REQ_OP_READ:
>>> + case REQ_OP_WRITE:
>>> + ret = mmc_blk_cqe_issue_rw_rq(mq, req);
>>> + break;
>>> + default:
>>> + WARN_ON_ONCE(1);
>>> + ret = -EINVAL;
>>> + }
>>> + if (!ret)
>>> + return MMC_REQ_STARTED;
>>> + return ret == -EBUSY ? MMC_REQ_BUSY : MMC_REQ_FAILED_TO_START;
>>> + default:
>>> + WARN_ON_ONCE(1);
>>> + return MMC_REQ_FAILED_TO_START;
>>> + }
>>> +}
>>> +
>>> static void mmc_blk_rw_rq_prep(struct mmc_queue_req *mqrq,
>>> struct mmc_card *card,
>>> int disable_multi,
>>> @@ -2035,7 +2228,7 @@ static struct mmc_blk_data *mmc_blk_alloc_req(struct
>>> mmc_card *card,
>>> INIT_LIST_HEAD(&md->part);
>>> md->usage = 1;
>>> - ret = mmc_init_queue(&md->queue, card, &md->lock, subname);
>>> + ret = mmc_init_queue(&md->queue, card, &md->lock, subname, area_type);
>>> if (ret)
>>> goto err_putdisk;
>>> diff --git a/drivers/mmc/core/block.h b/drivers/mmc/core/block.h
>>> index 860ca7c8df86..d7b3d7008b00 100644
>>> --- a/drivers/mmc/core/block.h
>>> +++ b/drivers/mmc/core/block.h
>>> @@ -6,4 +6,11 @@
>>> void mmc_blk_issue_rq(struct mmc_queue *mq, struct request *req);
>>> +enum mmc_issued;
>>> +
>>> +enum mmc_issued mmc_blk_cqe_issue_rq(struct mmc_queue *mq,
>>> + struct request *req);
>>> +void mmc_blk_cqe_complete_rq(struct request *rq);
>>> +void mmc_blk_cqe_recovery(struct mmc_queue *mq);
>>> +
>>> #endif
>>> diff --git a/drivers/mmc/core/queue.c b/drivers/mmc/core/queue.c
>>> index affa7370ba82..0cb7b0e8ee58 100644
>>> --- a/drivers/mmc/core/queue.c
>>> +++ b/drivers/mmc/core/queue.c
>>> @@ -36,10 +36,254 @@ static int mmc_prep_request(struct request_queue *q,
>>> struct request *req)
>>> return BLKPREP_KILL;
>>> req->rq_flags |= RQF_DONTPREP;
>>> + req_to_mmc_queue_req(req)->retries = 0;
>>> return BLKPREP_OK;
>>> }
>>> +static void mmc_cqe_request_fn(struct request_queue *q)
>>> +{
>>> + struct mmc_queue *mq = q->queuedata;
>>> + struct request *req;
>>> +
>>> + if (!mq) {
>>> + while ((req = blk_fetch_request(q)) != NULL) {
>>> + req->rq_flags |= RQF_QUIET;
>>> + __blk_end_request_all(req, BLK_STS_IOERR);
>>> + }
>>> + return;
>>> + }
>>> +
>>> + if (mq->asleep && !mq->cqe_busy)
>>> + wake_up_process(mq->thread);
>>> +}
>>> +
>>> +static inline bool mmc_cqe_dcmd_busy(struct mmc_queue *mq)
>>> +{
>>> + /* Allow only 1 DCMD at a time */
>>> + return mq->cqe_in_flight[MMC_ISSUE_DCMD];
>>> +}
>>> +
>>> +void mmc_cqe_kick_queue(struct mmc_queue *mq)
>>> +{
>>> + if ((mq->cqe_busy & MMC_CQE_DCMD_BUSY) && !mmc_cqe_dcmd_busy(mq))
>>> + mq->cqe_busy &= ~MMC_CQE_DCMD_BUSY;
>>> +
>>> + mq->cqe_busy &= ~MMC_CQE_QUEUE_FULL;
>>> +
>>> + if (mq->asleep && !mq->cqe_busy)
>>> + __blk_run_queue(mq->queue);
>>> +}
>>> +
>>> +static inline bool mmc_cqe_can_dcmd(struct mmc_host *host)
>>> +{
>>> + return host->caps2 & MMC_CAP2_CQE_DCMD;
>>> +}
>>> +
>>> +enum mmc_issue_type mmc_cqe_issue_type(struct mmc_host *host,
>>> + struct request *req)
>>> +{
>>> + switch (req_op(req)) {
>>> + case REQ_OP_DRV_IN:
>>> + case REQ_OP_DRV_OUT:
>>> + case REQ_OP_DISCARD:
>>> + case REQ_OP_SECURE_ERASE:
>>> + return MMC_ISSUE_SYNC;
>>> + case REQ_OP_FLUSH:
>>> + return mmc_cqe_can_dcmd(host) ? MMC_ISSUE_DCMD : MMC_ISSUE_SYNC;
>>> + default:
>>> + return MMC_ISSUE_ASYNC;
>>> + }
>>> +}
>>> +
>>> +static void __mmc_cqe_recovery_notifier(struct mmc_queue *mq)
>>> +{
>>> + if (!mq->cqe_recovery_needed) {
>>> + mq->cqe_recovery_needed = true;
>>> + wake_up_process(mq->thread);
>>> + }
>>> +}
>>> +
>>> +static void mmc_cqe_recovery_notifier(struct mmc_host *host,
>>> + struct mmc_request *mrq)
>>> +{
>>> + struct mmc_queue_req *mqrq = container_of(mrq, struct mmc_queue_req,
>>> + brq.mrq);
>>> + struct request *req = mmc_queue_req_to_req(mqrq);
>>> + struct request_queue *q = req->q;
>>> + struct mmc_queue *mq = q->queuedata;
>>> + unsigned long flags;
>>> +
>>> + spin_lock_irqsave(q->queue_lock, flags);
>>> + __mmc_cqe_recovery_notifier(mq);
>>> + spin_unlock_irqrestore(q->queue_lock, flags);
>>> +}
>>> +
>>> +static int mmc_cqe_thread(void *d)
>>> +{
>>> + struct mmc_queue *mq = d;
>>> + struct request_queue *q = mq->queue;
>>> + struct mmc_card *card = mq->card;
>>> + struct mmc_host *host = card->host;
>>> + unsigned long flags;
>>> + int get_put = 0;
>>> +
>>> + current->flags |= PF_MEMALLOC;
>>> +
>>> + down(&mq->thread_sem);
>>> + spin_lock_irqsave(q->queue_lock, flags);
>>> + while (1) {
>>> + struct request *req = NULL;
>>> + enum mmc_issue_type issue_type;
>>> + bool retune_ok = false;
>>> +
>>> + if (mq->cqe_recovery_needed) {
>>> + spin_unlock_irqrestore(q->queue_lock, flags);
>>> + mmc_blk_cqe_recovery(mq);
>>> + spin_lock_irqsave(q->queue_lock, flags);
>>> + mq->cqe_recovery_needed = false;
>>> + }
>>> +
>>> + set_current_state(TASK_INTERRUPTIBLE);
>>> +
>>> + if (!kthread_should_stop())
>>> + req = blk_peek_request(q);
>>> +
>>> + if (req) {
>>> + issue_type = mmc_cqe_issue_type(host, req);
>>> + switch (issue_type) {
>>> + case MMC_ISSUE_DCMD:
>>> + if (mmc_cqe_dcmd_busy(mq)) {
>>> + mq->cqe_busy |= MMC_CQE_DCMD_BUSY;
>>> + req = NULL;
>>> + break;
>>> + }
>>> + /* Fall through */
>>> + case MMC_ISSUE_ASYNC:
>>> + if (blk_queue_start_tag(q, req)) {
>>> + mq->cqe_busy |= MMC_CQE_QUEUE_FULL;
>>> + req = NULL;
>>> + }
>>> + break;
>>> + default:
>>> + /*
>>> + * Timeouts are handled by mmc core, so set a
>>> + * large value to avoid races.
>>> + */
>>> + req->timeout = 600 * HZ;
>>> + blk_start_request(req);
>>> + break;
>>> + }
>>> + if (req) {
>>> + mq->cqe_in_flight[issue_type] += 1;
>>> + if (mmc_cqe_tot_in_flight(mq) == 1)
>>> + get_put += 1;
>>> + if (mmc_cqe_qcnt(mq) == 1)
>>> + retune_ok = true;
>>> + }
>>> + }
>>> +
>>
>> Just a thought that mmc_cq_thread is a little heavy for manage in-flight
>> request, so could we kick the check back to blk layer like this in the
>> prepare and unprepare hook?
>
> I am not sure what you are aiming at. The prepare function is called by
> blk_peek_request() so the work is anyway being done by the thread. However
> the prepare function isn't always called, for example if the request has
> already been prepared and then later re-queued. So it looks to me as though
> it won't work that way.

yes, prepare is called when calling blk_peek_request. I was trying to
simplify the code the mmc_cqe_thread looks like. yes, I missed the
re-queued case. Thanks for the explaination. :)

>
>>
>>
>> --- a/drivers/mmc/core/queue.c
>> +++ b/drivers/mmc/core/queue.c
>> @@ -25,22 +25,59 @@
>>
>> #define MMC_QUEUE_BOUNCESZ 65536
>>
>> +static inline bool mmc_cqe_dcmd_busy(struct mmc_queue *mq);
>> +static inline bool mmc_cqe_can_dcmd(struct mmc_host *host);
>> +enum mmc_issue_type mmc_cqe_issue_type(struct mmc_host *host,
>> + struct request *req);
>> +
>> /*
>> * Prepare a MMC request. This just filters out odd stuff.
>> */
>> static int mmc_prep_request(struct request_queue *q, struct request *req)
>> {
>> struct mmc_queue *mq = q->queuedata;
>> + enum mmc_issue_type issue_type;
>>
>> if (mq && (mmc_card_removed(mq->card) || mmc_access_rpmb(mq)))
>> return BLKPREP_KILL;
>>
>> + if (mq->card->host->cqe_enabled) {
>> + issue_type = mmc_cqe_issue_type(mq->card->host, req);
>> + if ((issue_type == MMC_ISSUE_DCMD && mmc_cqe_dcmd_busy(mq))) {
>> + mq->cqe_busy |= MMC_CQE_DCMD_BUSY;
>> + return BLKPREP_DEFER;
>> + } else if (issue_type == MMC_ISSUE_ASYNC &&
>> + mq->cqe_in_flight[issue_type] >=
>> + mq->queue->queue_tags->max_depth) {
>> + mq->cqe_busy |= MMC_CQE_QUEUE_FULL;
>> + return BLKPREP_DEFER;
>> + }
>> + }
>> +
>> req->rq_flags |= RQF_DONTPREP;
>> req_to_mmc_queue_req(req)->retries = 0;
>>
>> return BLKPREP_OK;
>> }
>>
>> +static void mmc_unprep_request(struct request_queue *q, struct request *req)
>> +{
>> + struct mmc_queue *mq = q->queuedata;
>> + enum mmc_issue_type issue_type;
>> +
>> + if (!mq->card->host->cqe_enabled)
>> + return;
>> +
>> + issue_type = mmc_cqe_issue_type(mq->card->host, req);
>> + if (issue_type == MMC_ISSUE_DCMD &&
>> + (mq->cqe_busy & MMC_CQE_DCMD_BUSY) &&
>> + !mmc_cqe_dcmd_busy(mq))
>> + mq->cqe_busy &= ~MMC_CQE_DCMD_BUSY;
>> + else if (issue_type == MMC_ISSUE_ASYNC)
>> + mq->cqe_busy &= ~MMC_CQE_QUEUE_FULL;
>> +}
>> +
>> +
>> static void mmc_cqe_request_fn(struct request_queue *q)
>> {
>> struct mmc_queue *mq = q->queuedata;
>> @@ -66,11 +103,6 @@ static inline bool mmc_cqe_dcmd_busy(struct mmc_queue *mq)
>>
>> void mmc_cqe_kick_queue(struct mmc_queue *mq)
>> {
>> - if ((mq->cqe_busy & MMC_CQE_DCMD_BUSY) && !mmc_cqe_dcmd_busy(mq))
>> - mq->cqe_busy &= ~MMC_CQE_DCMD_BUSY;
>> -
>> - mq->cqe_busy &= ~MMC_CQE_QUEUE_FULL;
>> -
>> if (mq->asleep && !mq->cqe_busy)
>> __blk_run_queue(mq->queue);
>> }
>> @@ -151,21 +183,9 @@ static int mmc_cqe_thread(void *d)
>>
>> if (req) {
>> issue_type = mmc_cqe_issue_type(host, req);
>> - switch (issue_type) {
>> - case MMC_ISSUE_DCMD:
>> - if (mmc_cqe_dcmd_busy(mq)) {
>> - mq->cqe_busy |= MMC_CQE_DCMD_BUSY;
>> - req = NULL;
>> - break;
>> - }
>> - /* Fall through */
>> - case MMC_ISSUE_ASYNC:
>> - if (blk_queue_start_tag(q, req)) {
>> - mq->cqe_busy |= MMC_CQE_QUEUE_FULL;
>> - req = NULL;
>> - }
>> - break;
>> - default:
>> + if (issue_type == MMC_ISSUE_DCMD || issue_type ==
>> MMC_ISSUE_ASYNC) {
>> + blk_queue_start_tag(q, req);
>> + } else {
>> /*
>> * Timeouts are handled by mmc core, so set a
>> * large value to avoid races.
>> @@ -174,13 +194,12 @@ static int mmc_cqe_thread(void *d)
>> blk_start_request(req);
>> break;
>> }
>> - if (req) {
>> - mq->cqe_in_flight[issue_type] += 1;
>> - if (mmc_cqe_tot_in_flight(mq) == 1)
>> - get_put += 1;
>> - if (mmc_cqe_qcnt(mq) == 1)
>> - retune_ok = true;
>> - }
>> +
>> + mq->cqe_in_flight[issue_type] += 1;
>> + if (mmc_cqe_tot_in_flight(mq) == 1)
>> + get_put += 1;
>> + if (mmc_cqe_qcnt(mq) == 1)
>> + retune_ok = true;
>> }
>>
>> mq->asleep = !req;
>> @@ -523,6 +542,7 @@ int mmc_init_queue(struct mmc_queue *mq, struct mmc_card
>> *card,
>> }
>>
>> blk_queue_prep_rq(mq->queue, mmc_prep_request);
>> + blk_queue_unprep_rq(mq->queue, mmc_unprep_request);
>> queue_flag_set_unlocked(QUEUE_FLAG_NONROT, mq->queue);
>> queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, mq->queue);
>> if (mmc_can_erase(card))
>>
>>
>>
>>> + mq->asleep = !req;
>>> +
>>> + spin_unlock_irqrestore(q->queue_lock, flags);
>>> +
>>> + if (req) {
>>> + enum mmc_issued issued;
>>> +
>>> + set_current_state(TASK_RUNNING);
>>> +
>>> + if (get_put) {
>>> + get_put = 0;
>>> + mmc_get_card(card);
>>> + }
>>> +
>>> + if (host->need_retune && retune_ok &&
>>> + !host->hold_retune)
>>> + host->retune_now = true;
>>> + else
>>> + host->retune_now = false;
>>> +
>>> + issued = mmc_blk_cqe_issue_rq(mq, req);
>>> +
>>> + cond_resched();
>>> +
>>> + spin_lock_irqsave(q->queue_lock, flags);
>>> +
>>> + switch (issued) {
>>> + case MMC_REQ_STARTED:
>>> + break;
>>> + case MMC_REQ_BUSY:
>>> + blk_requeue_request(q, req);
>>> + goto finished;
>>> + case MMC_REQ_FAILED_TO_START:
>>> + __blk_end_request_all(req, BLK_STS_IOERR);
>>> + /* Fall through */
>>> + case MMC_REQ_FINISHED:
>>> +finished:
>>> + mq->cqe_in_flight[issue_type] -= 1;
>>> + if (mmc_cqe_tot_in_flight(mq) == 0)
>>> + get_put = -1;
>>> + }
>>> + } else {
>>> + if (get_put < 0) {
>>> + get_put = 0;
>>> + mmc_put_card(card);
>>> + }
>>> + /*
>>> + * Do not stop with requests in flight in case recovery
>>> + * is needed.
>>> + */
>>> + if (kthread_should_stop() &&
>>> + !mmc_cqe_tot_in_flight(mq)) {
>>> + set_current_state(TASK_RUNNING);
>>> + break;
>>> + }
>>> + up(&mq->thread_sem);
>>> + schedule();
>>> + down(&mq->thread_sem);
>>> + spin_lock_irqsave(q->queue_lock, flags);
>>> + }
>>> + } /* loop */
>>> + up(&mq->thread_sem);
>>> +
>>> + return 0;
>>> +}
>>> +
>>> +static enum blk_eh_timer_return __mmc_cqe_timed_out(struct request *req)
>>> +{
>>> + struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
>>> + struct mmc_request *mrq = &mqrq->brq.mrq;
>>> + struct mmc_queue *mq = req->q->queuedata;
>>> + struct mmc_host *host = mq->card->host;
>>> + enum mmc_issue_type issue_type = mmc_cqe_issue_type(host, req);
>>> + bool recovery_needed = false;
>>> +
>>> + switch (issue_type) {
>>> + case MMC_ISSUE_ASYNC:
>>> + case MMC_ISSUE_DCMD:
>>> + if (host->cqe_ops->cqe_timeout(host, mrq, &recovery_needed)) {
>>> + if (recovery_needed)
>>> + __mmc_cqe_recovery_notifier(mq);
>>> + return BLK_EH_RESET_TIMER;
>>> + }
>>> + /* No timeout */
>>> + return BLK_EH_HANDLED;
>>> + default:
>>> + /* Timeout is handled by mmc core */
>>> + return BLK_EH_RESET_TIMER;
>>> + }
>>> +}
>>> +
>>> +static enum blk_eh_timer_return mmc_cqe_timed_out(struct request *req)
>>> +{
>>> + struct mmc_queue *mq = req->q->queuedata;
>>> +
>>> + if (mq->cqe_recovery_needed)
>>> + return BLK_EH_RESET_TIMER;
>>> +
>>> + return __mmc_cqe_timed_out(req);
>>> +}
>>> +
>>> static int mmc_queue_thread(void *d)
>>> {
>>> struct mmc_queue *mq = d;
>>> @@ -233,11 +477,12 @@ static void mmc_exit_request(struct request_queue
>>> *q, struct request *req)
>>> * Initialise a MMC card request queue.
>>> */
>>> int mmc_init_queue(struct mmc_queue *mq, struct mmc_card *card,
>>> - spinlock_t *lock, const char *subname)
>>> + spinlock_t *lock, const char *subname, int area_type)
>>> {
>>> struct mmc_host *host = card->host;
>>> u64 limit = BLK_BOUNCE_HIGH;
>>> int ret = -ENOMEM;
>>> + bool use_cqe = host->cqe_enabled && area_type != MMC_BLK_DATA_AREA_RPMB;
>>> if (mmc_dev(host)->dma_mask && *mmc_dev(host)->dma_mask)
>>> limit = (u64)dma_max_pfn(mmc_dev(host)) << PAGE_SHIFT;
>>> @@ -247,7 +492,7 @@ int mmc_init_queue(struct mmc_queue *mq, struct
>>> mmc_card *card,
>>> if (!mq->queue)
>>> return -ENOMEM;
>>> mq->queue->queue_lock = lock;
>>> - mq->queue->request_fn = mmc_request_fn;
>>> + mq->queue->request_fn = use_cqe ? mmc_cqe_request_fn : mmc_request_fn;
>>> mq->queue->init_rq_fn = mmc_init_request;
>>> mq->queue->exit_rq_fn = mmc_exit_request;
>>> mq->queue->cmd_size = sizeof(struct mmc_queue_req);
>>> @@ -259,6 +504,24 @@ int mmc_init_queue(struct mmc_queue *mq, struct
>>> mmc_card *card,
>>> return ret;
>>> }
>>> + if (use_cqe) {
>>> + int q_depth = card->ext_csd.cmdq_depth;
>>> +
>>> + if (q_depth > host->cqe_qdepth)
>>> + q_depth = host->cqe_qdepth;
>>> +
>>> + ret = blk_queue_init_tags(mq->queue, q_depth, NULL,
>>> + BLK_TAG_ALLOC_FIFO);
>>> + if (ret)
>>> + goto cleanup_queue;
>>> +
>>> + blk_queue_softirq_done(mq->queue, mmc_blk_cqe_complete_rq);
>>> + blk_queue_rq_timed_out(mq->queue, mmc_cqe_timed_out);
>>> + blk_queue_rq_timeout(mq->queue, 60 * HZ);
>>> +
>>> + host->cqe_recovery_notifier = mmc_cqe_recovery_notifier;
>>> + }
>>> +
>>> blk_queue_prep_rq(mq->queue, mmc_prep_request);
>>> queue_flag_set_unlocked(QUEUE_FLAG_NONROT, mq->queue);
>>> queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, mq->queue);
>>> @@ -280,9 +543,9 @@ int mmc_init_queue(struct mmc_queue *mq, struct
>>> mmc_card *card,
>>> sema_init(&mq->thread_sem, 1);
>>> - mq->thread = kthread_run(mmc_queue_thread, mq, "mmcqd/%d%s",
>>> - host->index, subname ? subname : "");
>>> -
>>> + mq->thread = kthread_run(use_cqe ? mmc_cqe_thread : mmc_queue_thread,
>>> + mq, "mmcqd/%d%s", host->index,
>>> + subname ? subname : "");
>>> if (IS_ERR(mq->thread)) {
>>> ret = PTR_ERR(mq->thread);
>>> goto cleanup_queue;
>>> diff --git a/drivers/mmc/core/queue.h b/drivers/mmc/core/queue.h
>>> index 361b46408e0f..8e9273d977c0 100644
>>> --- a/drivers/mmc/core/queue.h
>>> +++ b/drivers/mmc/core/queue.h
>>> @@ -7,6 +7,20 @@
>>> #include <linux/mmc/core.h>
>>> #include <linux/mmc/host.h>
>>> +enum mmc_issued {
>>> + MMC_REQ_STARTED,
>>> + MMC_REQ_BUSY,
>>> + MMC_REQ_FAILED_TO_START,
>>> + MMC_REQ_FINISHED,
>>> +};
>>> +
>>> +enum mmc_issue_type {
>>> + MMC_ISSUE_SYNC,
>>> + MMC_ISSUE_DCMD,
>>> + MMC_ISSUE_ASYNC,
>>> + MMC_ISSUE_MAX,
>>> +};
>>> +
>>> static inline struct mmc_queue_req *req_to_mmc_queue_req(struct request
>>> *rq)
>>> {
>>> return blk_mq_rq_to_pdu(rq);
>>> @@ -53,6 +67,7 @@ struct mmc_queue_req {
>>> int drv_op_result;
>>> struct mmc_blk_ioc_data **idata;
>>> unsigned int ioc_count;
>>> + int retries;
>>> };
>>> struct mmc_queue {
>>> @@ -70,10 +85,17 @@ struct mmc_queue {
>>> * associated mmc_queue_req data.
>>> */
>>> int qcnt;
>>> + /* Following are defined for a Command Queue Engine */
>>> + int cqe_in_flight[MMC_ISSUE_MAX];
>>> + unsigned int cqe_busy;
>>> + bool cqe_recovery_needed;
>>> + bool cqe_in_recovery;
>>> +#define MMC_CQE_DCMD_BUSY BIT(0)
>>> +#define MMC_CQE_QUEUE_FULL BIT(1)
>>> };
>>> extern int mmc_init_queue(struct mmc_queue *, struct mmc_card *,
>>> spinlock_t *,
>>> - const char *);
>>> + const char *, int);
>>> extern void mmc_cleanup_queue(struct mmc_queue *);
>>> extern void mmc_queue_suspend(struct mmc_queue *);
>>> extern void mmc_queue_resume(struct mmc_queue *);
>>> @@ -85,4 +107,22 @@ extern unsigned int mmc_queue_map_sg(struct mmc_queue *,
>>> extern int mmc_access_rpmb(struct mmc_queue *);
>>> +void mmc_cqe_kick_queue(struct mmc_queue *mq);
>>> +
>>> +enum mmc_issue_type mmc_cqe_issue_type(struct mmc_host *host,
>>> + struct request *req);
>>> +
>>> +static inline int mmc_cqe_tot_in_flight(struct mmc_queue *mq)
>>> +{
>>> + return mq->cqe_in_flight[MMC_ISSUE_SYNC] +
>>> + mq->cqe_in_flight[MMC_ISSUE_DCMD] +
>>> + mq->cqe_in_flight[MMC_ISSUE_ASYNC];
>>> +}
>>> +
>>> +static inline int mmc_cqe_qcnt(struct mmc_queue *mq)
>>> +{
>>> + return mq->cqe_in_flight[MMC_ISSUE_DCMD] +
>>> + mq->cqe_in_flight[MMC_ISSUE_ASYNC];
>>> +}
>>> +
>>> #endif
>>>
>>
>>
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-mmc" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at http://vger.kernel.org/majordomo-info.html
>
>
>