[PATCH] hw/nvme: be more careful when deasserting IRQs

[PATCH] hw/nvme: be more careful when deasserting IRQs

[Jakub Jermář]

An IRQ vector used by a completion queue cannot be deasserted without
first checking if the same vector does not need to stay asserted for
some other completion queue.

Signed-off-by: Jakub Jermar <jakub.jermar@kernkonzept.com>
---
 hw/nvme/ctrl.c | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/hw/nvme/ctrl.c b/hw/nvme/ctrl.c
index 0bcaf7192f..c0980929eb 100644
--- a/hw/nvme/ctrl.c
+++ b/hw/nvme/ctrl.c
@@ -473,6 +473,21 @@ static void nvme_irq_deassert(NvmeCtrl *n, NvmeCQueue *cq)
     }
 }
 
+/*
+ * Check if the vector used by the cq can be deasserted, i.e. it needn't be
+ * asserted for some other cq.
+ */
+static bool nvme_irq_can_deassert(NvmeCtrl *n, NvmeCQueue *cq)
+{
+    for (unsigned qid = 0; qid < n->params.max_ioqpairs + 1; qid++) {
+        NvmeCQueue *q = n->cq[qid];
+
+        if (q && q->vector == cq->vector && q->head != q->tail)
+            return false;  /* some queue needs this to stay asserted */
+    }
+    return true;
+}
+
 static void nvme_req_clear(NvmeRequest *req)
 {
     req->ns = NULL;
@@ -4089,7 +4104,9 @@ static uint16_t nvme_del_cq(NvmeCtrl *n, NvmeRequest *req)
         trace_pci_nvme_err_invalid_del_cq_notempty(qid);
         return NVME_INVALID_QUEUE_DEL;
     }
-    nvme_irq_deassert(n, cq);
+    if (nvme_irq_can_deassert(n, cq)) {
+        nvme_irq_deassert(n, cq);
+    }
     trace_pci_nvme_del_cq(qid);
     nvme_free_cq(cq, n);
     return NVME_SUCCESS;
@@ -5757,7 +5774,7 @@ static void nvme_process_db(NvmeCtrl *n, hwaddr addr, int val)
             timer_mod(cq->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + 500);
         }
 
-        if (cq->tail == cq->head) {
+        if (nvme_irq_can_deassert(n, cq)) {
             nvme_irq_deassert(n, cq);
         }
     } else {
-- 
2.31.1

Re: [PATCH] hw/nvme: be more careful when deasserting IRQs

[Klaus Jensen]

[-- Attachment #1: Type: text/plain, Size: 2057 bytes --]

+cc qemu-block, maintainers

On Jun 10 13:46, Jakub Jermář wrote:
>An IRQ vector used by a completion queue cannot be deasserted without
>first checking if the same vector does not need to stay asserted for
>some other completion queue.
>
>Signed-off-by: Jakub Jermar <jakub.jermar@kernkonzept.com>
>---
> hw/nvme/ctrl.c | 21 +++++++++++++++++++--
> 1 file changed, 19 insertions(+), 2 deletions(-)
>
>diff --git a/hw/nvme/ctrl.c b/hw/nvme/ctrl.c
>index 0bcaf7192f..c0980929eb 100644
>--- a/hw/nvme/ctrl.c
>+++ b/hw/nvme/ctrl.c
>@@ -473,6 +473,21 @@ static void nvme_irq_deassert(NvmeCtrl *n, NvmeCQueue *cq)
>     }
> }
>
>+/*
>+ * Check if the vector used by the cq can be deasserted, i.e. it needn't be
>+ * asserted for some other cq.
>+ */
>+static bool nvme_irq_can_deassert(NvmeCtrl *n, NvmeCQueue *cq)
>+{
>+    for (unsigned qid = 0; qid < n->params.max_ioqpairs + 1; qid++) {
>+        NvmeCQueue *q = n->cq[qid];
>+
>+        if (q && q->vector == cq->vector && q->head != q->tail)
>+            return false;  /* some queue needs this to stay asserted */
>+    }
>+    return true;
>+}
>+
> static void nvme_req_clear(NvmeRequest *req)
> {
>     req->ns = NULL;
>@@ -4089,7 +4104,9 @@ static uint16_t nvme_del_cq(NvmeCtrl *n, NvmeRequest *req)
>         trace_pci_nvme_err_invalid_del_cq_notempty(qid);
>         return NVME_INVALID_QUEUE_DEL;
>     }
>-    nvme_irq_deassert(n, cq);
>+    if (nvme_irq_can_deassert(n, cq)) {
>+        nvme_irq_deassert(n, cq);
>+    }
>     trace_pci_nvme_del_cq(qid);
>     nvme_free_cq(cq, n);
>     return NVME_SUCCESS;
>@@ -5757,7 +5774,7 @@ static void nvme_process_db(NvmeCtrl *n, hwaddr addr, int val)
>             timer_mod(cq->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + 500);
>         }
>
>-        if (cq->tail == cq->head) {
>+        if (nvme_irq_can_deassert(n, cq)) {
>             nvme_irq_deassert(n, cq);
>         }
>     } else {
>-- 
>2.31.1
>
>

-- 
One of us - No more doubt, silence or taboo about mental illness.

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 488 bytes --]

Re: [PATCH] hw/nvme: be more careful when deasserting IRQs

[Klaus Jensen]

[-- Attachment #1: Type: text/plain, Size: 2763 bytes --]

On Jun 10 13:46, Jakub Jermář wrote:
>An IRQ vector used by a completion queue cannot be deasserted without
>first checking if the same vector does not need to stay asserted for
>some other completion queue.
>
>Signed-off-by: Jakub Jermar <jakub.jermar@kernkonzept.com>
>---
> hw/nvme/ctrl.c | 21 +++++++++++++++++++--
> 1 file changed, 19 insertions(+), 2 deletions(-)
>
>diff --git a/hw/nvme/ctrl.c b/hw/nvme/ctrl.c
>index 0bcaf7192f..c0980929eb 100644
>--- a/hw/nvme/ctrl.c
>+++ b/hw/nvme/ctrl.c
>@@ -473,6 +473,21 @@ static void nvme_irq_deassert(NvmeCtrl *n, NvmeCQueue *cq)
>     }
> }
>
>+/*
>+ * Check if the vector used by the cq can be deasserted, i.e. it needn't be
>+ * asserted for some other cq.
>+ */
>+static bool nvme_irq_can_deassert(NvmeCtrl *n, NvmeCQueue *cq)
>+{
>+    for (unsigned qid = 0; qid < n->params.max_ioqpairs + 1; qid++) {
>+        NvmeCQueue *q = n->cq[qid];
>+
>+        if (q && q->vector == cq->vector && q->head != q->tail)
>+            return false;  /* some queue needs this to stay asserted */
>+    }
>+    return true;
>+}
>+
> static void nvme_req_clear(NvmeRequest *req)
> {
>     req->ns = NULL;
>@@ -4089,7 +4104,9 @@ static uint16_t nvme_del_cq(NvmeCtrl *n, NvmeRequest *req)
>         trace_pci_nvme_err_invalid_del_cq_notempty(qid);
>         return NVME_INVALID_QUEUE_DEL;
>     }
>-    nvme_irq_deassert(n, cq);
>+    if (nvme_irq_can_deassert(n, cq)) {
>+        nvme_irq_deassert(n, cq);
>+    }
>     trace_pci_nvme_del_cq(qid);
>     nvme_free_cq(cq, n);
>     return NVME_SUCCESS;
>@@ -5757,7 +5774,7 @@ static void nvme_process_db(NvmeCtrl *n, hwaddr addr, int val)
>             timer_mod(cq->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + 500);
>         }
>
>-        if (cq->tail == cq->head) {
>+        if (nvme_irq_can_deassert(n, cq)) {
>             nvme_irq_deassert(n, cq);
>         }
>     } else {
>-- 
>2.31.1
>

This is actually an artifact of commit ca247d35098d3 ("hw/block/nvme: 
fix pin-based interrupt behavior") that I did a year ago. Prior to that 
fix, the completion queue id was used to index the internal IS register 
(irq_status), which, while wrong spec-wise, had the effect of... 
actually working.

Anyway, I agree that the logic is flawed right now, since we should only 
deassert when all outstanding cqe's have been acknowledged by the host.

nvme_irq_can_deassert should be guarded with a check on msix_enabled(), 
but in any case I am not happy about looping over all completion queues 
on each cq doorbell write. I think this can be ref counted? I.e. 
decrement when cq->tail == cq->head on the cq doorbell write and 
increment only when going from empty to non-empty in nvme_post_cqes().

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 488 bytes --]

Re: [PATCH] hw/nvme: be more careful when deasserting IRQs

[Jakub Jermář]

On 6/10/21 8:14 PM, Klaus Jensen wrote:
> On Jun 10 13:46, Jakub Jermář wrote:
>> An IRQ vector used by a completion queue cannot be deasserted without
>> first checking if the same vector does not need to stay asserted for
>> some other completion queue.
>>
>> Signed-off-by: Jakub Jermar <jakub.jermar@kernkonzept.com>
>> ---
>> hw/nvme/ctrl.c | 21 +++++++++++++++++++--
>> 1 file changed, 19 insertions(+), 2 deletions(-)
>>
>> diff --git a/hw/nvme/ctrl.c b/hw/nvme/ctrl.c
>> index 0bcaf7192f..c0980929eb 100644
>> --- a/hw/nvme/ctrl.c
>> +++ b/hw/nvme/ctrl.c
>> @@ -473,6 +473,21 @@ static void nvme_irq_deassert(NvmeCtrl *n, 
>> NvmeCQueue *cq)
>>     }
>> }
>>
>> +/*
>> + * Check if the vector used by the cq can be deasserted, i.e. it 
>> needn't be
>> + * asserted for some other cq.
>> + */
>> +static bool nvme_irq_can_deassert(NvmeCtrl *n, NvmeCQueue *cq)
>> +{
>> +    for (unsigned qid = 0; qid < n->params.max_ioqpairs + 1; qid++) {
>> +        NvmeCQueue *q = n->cq[qid];
>> +
>> +        if (q && q->vector == cq->vector && q->head != q->tail)
>> +            return false;  /* some queue needs this to stay asserted */
>> +    }
>> +    return true;
>> +}
>> +
>> static void nvme_req_clear(NvmeRequest *req)
>> {
>>     req->ns = NULL;
>> @@ -4089,7 +4104,9 @@ static uint16_t nvme_del_cq(NvmeCtrl *n, 
>> NvmeRequest *req)
>>         trace_pci_nvme_err_invalid_del_cq_notempty(qid);
>>         return NVME_INVALID_QUEUE_DEL;
>>     }
>> -    nvme_irq_deassert(n, cq);
>> +    if (nvme_irq_can_deassert(n, cq)) {
>> +        nvme_irq_deassert(n, cq);
>> +    }
>>     trace_pci_nvme_del_cq(qid);
>>     nvme_free_cq(cq, n);
>>     return NVME_SUCCESS;
>> @@ -5757,7 +5774,7 @@ static void nvme_process_db(NvmeCtrl *n, hwaddr 
>> addr, int val)
>>             timer_mod(cq->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) 
>> + 500);
>>         }
>>
>> -        if (cq->tail == cq->head) {
>> +        if (nvme_irq_can_deassert(n, cq)) {
>>             nvme_irq_deassert(n, cq);
>>         }
>>     } else {
>> -- 
>> 2.31.1
>>
> 
> This is actually an artifact of commit ca247d35098d3 ("hw/block/nvme: 
> fix pin-based interrupt behavior") that I did a year ago. Prior to that 
> fix, the completion queue id was used to index the internal IS register 
> (irq_status), which, while wrong spec-wise, had the effect of... 
> actually working.
> 
> Anyway, I agree that the logic is flawed right now, since we should only 
> deassert when all outstanding cqe's have been acknowledged by the host.
> 
> nvme_irq_can_deassert should be guarded with a check on msix_enabled(), 
> but in any case I am not happy about looping over all completion queues 
> on each cq doorbell write. I think this can be ref counted? I.e. 
> decrement when cq->tail == cq->head on the cq doorbell write and 
> increment only when going from empty to non-empty in nvme_post_cqes().

I reworked this to use reference counting in v2.

Jakub

Re: [PATCH] hw/nvme: be more careful when deasserting IRQs

[no-reply]

Patchew URL: https://patchew.org/QEMU/20210610114624.304681-1-jakub.jermar@kernkonzept.com/



Hi,

This series seems to have some coding style problems. See output below for
more information:

Type: series
Message-id: 20210610114624.304681-1-jakub.jermar@kernkonzept.com
Subject: [PATCH] hw/nvme: be more careful when deasserting IRQs

=== TEST SCRIPT BEGIN ===
#!/bin/bash
git rev-parse base > /dev/null || exit 0
git config --local diff.renamelimit 0
git config --local diff.renames True
git config --local diff.algorithm histogram
./scripts/checkpatch.pl --mailback base..
=== TEST SCRIPT END ===

Updating 3c8cf5a9c21ff8782164d1def7f44bd888713384
From https://github.com/patchew-project/qemu
 * [new tag]         patchew/20210610132505.5827-1-peter.maydell@linaro.org -> patchew/20210610132505.5827-1-peter.maydell@linaro.org
 * [new tag]         patchew/20210614135429.56475-1-jakub.jermar@kernkonzept.com -> patchew/20210614135429.56475-1-jakub.jermar@kernkonzept.com
Switched to a new branch 'test'
aacc285 hw/nvme: be more careful when deasserting IRQs

=== OUTPUT BEGIN ===
ERROR: braces {} are necessary for all arms of this statement
#34: FILE: hw/nvme/ctrl.c:485:
+        if (q && q->vector == cq->vector && q->head != q->tail)
[...]

total: 1 errors, 0 warnings, 39 lines checked

Commit aacc285402f0 (hw/nvme: be more careful when deasserting IRQs) has style problems, please review.  If any of these errors
are false positives report them to the maintainer, see
CHECKPATCH in MAINTAINERS.
=== OUTPUT END ===

Test command exited with code: 1


The full log is available at
http://patchew.org/logs/20210610114624.304681-1-jakub.jermar@kernkonzept.com/testing.checkpatch/?type=message.
---
Email generated automatically by Patchew [https://patchew.org/].
Please send your feedback to patchew-devel@redhat.com