Re: [PATCH v5 15/15] block/nvme: Use an array of EventNotifier

["Philippe Mathieu-Daudé" <philmd@redhat.com>]

On 8/21/20 12:29 PM, Stefano Garzarella wrote:
> On Thu, Aug 20, 2020 at 06:59:01PM +0200, Philippe Mathieu-Daudé wrote:
>> In preparation of using multiple IRQ (thus multiple eventfds)
>> make BDRVNVMeState::irq_notifier an array (for now of a single
>> element, the admin queue notifier).
>>
>> Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
>> Signed-off-by: Philippe Mathieu-Daudé <philmd@redhat.com>
>> ---
>>  block/nvme.c | 31 +++++++++++++++++++++----------
>>  1 file changed, 21 insertions(+), 10 deletions(-)
>>
>> diff --git a/block/nvme.c b/block/nvme.c
>> index a61e86a83eb..fe8a40b7ede 100644
>> --- a/block/nvme.c
>> +++ b/block/nvme.c
>> @@ -106,6 +106,12 @@ QEMU_BUILD_BUG_ON(offsetof(NVMeRegs, doorbells) != 0x1000);
>>  #define INDEX_ADMIN     0
>>  #define INDEX_IO(n)     (1 + n)
>>  
>> +/* This driver shares a single MSIX IRQ for the admin and I/O queues */
>> +enum {
>> +    MSIX_SHARED_IRQ_IDX = 0,
>> +    MSIX_IRQ_COUNT = 1
>> +};
>> +
>>  struct BDRVNVMeState {
>>      AioContext *aio_context;
>>      QEMUVFIOState *vfio;
>> @@ -120,7 +126,7 @@ struct BDRVNVMeState {
>>      /* How many uint32_t elements does each doorbell entry take. */
>>      size_t doorbell_scale;
>>      bool write_cache_supported;
>> -    EventNotifier irq_notifier;
>> +    EventNotifier irq_notifier[MSIX_IRQ_COUNT];
>>  
>>      uint64_t nsze; /* Namespace size reported by identify command */
>>      int nsid;      /* The namespace id to read/write data. */
>> @@ -631,7 +637,8 @@ static bool nvme_poll_queues(BDRVNVMeState *s)
>>  
>>  static void nvme_handle_event(EventNotifier *n)
>>  {
>> -    BDRVNVMeState *s = container_of(n, BDRVNVMeState, irq_notifier);
>> +    BDRVNVMeState *s = container_of(n, BDRVNVMeState,
>> +                                    irq_notifier[MSIX_SHARED_IRQ_IDX]);
>>  
>>      trace_nvme_handle_event(s);
>>      event_notifier_test_and_clear(n);
>> @@ -683,7 +690,8 @@ out_error:
>>  static bool nvme_poll_cb(void *opaque)
>>  {
>>      EventNotifier *e = opaque;
>> -    BDRVNVMeState *s = container_of(e, BDRVNVMeState, irq_notifier);
>> +    BDRVNVMeState *s = container_of(e, BDRVNVMeState,
>> +                                    irq_notifier[MSIX_SHARED_IRQ_IDX]);
>>  
>>      trace_nvme_poll_cb(s);
>>      return nvme_poll_queues(s);
>> @@ -705,7 +713,7 @@ static int nvme_init(BlockDriverState *bs, const char *device, int namespace,
>>      s->device = g_strdup(device);
>>      s->nsid = namespace;
>>      s->aio_context = bdrv_get_aio_context(bs);
>> -    ret = event_notifier_init(&s->irq_notifier, 0);
>> +    ret = event_notifier_init(&s->irq_notifier[MSIX_SHARED_IRQ_IDX], 0);
>>      if (ret) {
>>          error_setg(errp, "Failed to init event notifier");
>>          return ret;
>> @@ -784,12 +792,13 @@ static int nvme_init(BlockDriverState *bs, const char *device, int namespace,
>>          }
>>      }
>>  
>> -    ret = qemu_vfio_pci_init_irq(s->vfio, &s->irq_notifier,
>> +    ret = qemu_vfio_pci_init_irq(s->vfio, s->irq_notifier,
> 
> Maybe we can use '&s->irq_notifier[MSIX_SHARED_IRQ_IDX]' to match the other
> changes.

This makes the following patch in the next series (using multiple
queues) simpler, but if you prefer I don't mind using your suggestion
here, then adding another patch to directly use the array address
(instead of the address of the 1st element in that array). As you
wish :)

> 
>>                                   VFIO_PCI_MSIX_IRQ_INDEX, errp);
>>      if (ret) {
>>          goto out;
>>      }
>> -    aio_set_event_notifier(bdrv_get_aio_context(bs), &s->irq_notifier,
>> +    aio_set_event_notifier(bdrv_get_aio_context(bs),
>> +                           &s->irq_notifier[MSIX_SHARED_IRQ_IDX],
>>                             false, nvme_handle_event, nvme_poll_cb);
>>  
>>      nvme_identify(bs, namespace, &local_err);
>> @@ -872,9 +881,10 @@ static void nvme_close(BlockDriverState *bs)
>>          nvme_free_queue_pair(s->queues[i]);
>>      }
>>      g_free(s->queues);
>> -    aio_set_event_notifier(bdrv_get_aio_context(bs), &s->irq_notifier,
>> +    aio_set_event_notifier(bdrv_get_aio_context(bs),
>> +                           &s->irq_notifier[MSIX_SHARED_IRQ_IDX],
>>                             false, NULL, NULL);
>> -    event_notifier_cleanup(&s->irq_notifier);
>> +    event_notifier_cleanup(&s->irq_notifier[MSIX_SHARED_IRQ_IDX]);
>>      qemu_vfio_pci_unmap_bar(s->vfio, 0, (void *)s->regs, 0, NVME_BAR_SIZE);
>>      qemu_vfio_close(s->vfio);
>>  
>> @@ -1381,7 +1391,8 @@ static void nvme_detach_aio_context(BlockDriverState *bs)
>>          q->completion_bh = NULL;
>>      }
>>  
>> -    aio_set_event_notifier(bdrv_get_aio_context(bs), &s->irq_notifier,
>> +    aio_set_event_notifier(bdrv_get_aio_context(bs),
>> +                           &s->irq_notifier[MSIX_SHARED_IRQ_IDX],
>>                             false, NULL, NULL);
>>  }
>>  
>> @@ -1391,7 +1402,7 @@ static void nvme_attach_aio_context(BlockDriverState *bs,
>>      BDRVNVMeState *s = bs->opaque;
>>  
>>      s->aio_context = new_context;
>> -    aio_set_event_notifier(new_context, &s->irq_notifier,
>> +    aio_set_event_notifier(new_context, &s->irq_notifier[MSIX_SHARED_IRQ_IDX],
>>                             false, nvme_handle_event, nvme_poll_cb);
>>  
>>      for (int i = 0; i < s->nr_queues; i++) {
>> -- 
>> 2.26.2
>>
>>
>