Re: [Qemu-devel] [PATCH v17 03/14] block: Replace in_use with operation blocker

[Jeff Cody <jcody@redhat.com>]

On Sun, Apr 06, 2014 at 08:10:58PM -0400, Jeff Cody wrote:
> On Mon, Mar 10, 2014 at 03:25:59PM +0800, Fam Zheng wrote:
> > This drops BlockDriverState.in_use with op_blockers:
> > 
> >   - Call bdrv_op_block_all in place of bdrv_set_in_use(bs, 1).
> >   - Call bdrv_op_unblock_all in place of bdrv_set_in_use(bs, 0).
> >   - Check bdrv_op_is_blocked() in place of bdrv_in_use(bs).
> >     The specific types are used, e.g. in place of starting block backup,
> >     bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_BACKUP, ...).
> >   - Check bdrv_op_blocker_is_empty() in place of assert(!bs->in_use).
> > 
> > Note: there is only bdrv_op_block_all and bdrv_op_unblock_all callers at
> > this moment. So although the checks are specific to op types, this
> > changes can still be seen as identical logic with previously with
> > in_use. The difference is error message are improved because of blocker
> > error info.
> > 
> > Signed-off-by: Fam Zheng <famz@redhat.com>
> > Signed-off-by: Markus Armbruster <armbru@redhat.com>
> > ---
> >  block-migration.c               |  7 +++++--
> >  block.c                         | 24 +++++++-----------------
> >  blockdev.c                      | 19 +++++++++----------
> >  blockjob.c                      | 14 +++++++++-----
> >  hw/block/dataplane/virtio-blk.c | 19 ++++++++++++-------
> >  include/block/block.h           |  2 --
> >  include/block/block_int.h       |  1 -
> >  include/block/blockjob.h        |  3 +++
> >  8 files changed, 45 insertions(+), 44 deletions(-)
> > 
> > diff --git a/block-migration.c b/block-migration.c
> > index 897fdba..bf9a25f 100644
> > --- a/block-migration.c
> > +++ b/block-migration.c
> > @@ -59,6 +59,7 @@ typedef struct BlkMigDevState {
> >      unsigned long *aio_bitmap;
> >      int64_t completed_sectors;
> >      BdrvDirtyBitmap *dirty_bitmap;
> > +    Error *blocker;
> >  } BlkMigDevState;
> >  
> >  typedef struct BlkMigBlock {
> > @@ -346,7 +347,8 @@ static void init_blk_migration_it(void *opaque, BlockDriverState *bs)
> >          bmds->completed_sectors = 0;
> >          bmds->shared_base = block_mig_state.shared_base;
> >          alloc_aio_bitmap(bmds);
> > -        bdrv_set_in_use(bs, 1);
> > +        error_setg(&bmds->blocker, "block device is in use by migration");
> > +        bdrv_op_block_all(bs, bmds->blocker);
> >          bdrv_ref(bs);
> >  
> >          block_mig_state.total_sector_sum += sectors;
> > @@ -584,7 +586,8 @@ static void blk_mig_cleanup(void)
> >      blk_mig_lock();
> >      while ((bmds = QSIMPLEQ_FIRST(&block_mig_state.bmds_list)) != NULL) {
> >          QSIMPLEQ_REMOVE_HEAD(&block_mig_state.bmds_list, entry);
> > -        bdrv_set_in_use(bmds->bs, 0);
> > +        bdrv_op_unblock_all(bmds->bs, bmds->blocker);
> > +        error_free(bmds->blocker);
> >          bdrv_unref(bmds->bs);
> >          g_free(bmds->aio_bitmap);
> >          g_free(bmds);
> > diff --git a/block.c b/block.c
> > index 56a4433..c79d5dc 100644
> > --- a/block.c
> > +++ b/block.c
> > @@ -1850,7 +1850,6 @@ static void bdrv_move_feature_fields(BlockDriverState *bs_dest,
> >      bs_dest->refcnt             = bs_src->refcnt;
> >  
> >      /* job */
> > -    bs_dest->in_use             = bs_src->in_use;
> 
> [1] 
> 
> >      bs_dest->job                = bs_src->job;
> >  
> >      /* keep the same entry in bdrv_states */
> > @@ -1893,7 +1892,7 @@ void bdrv_swap(BlockDriverState *bs_new, BlockDriverState *bs_old)
> >      assert(QLIST_EMPTY(&bs_new->dirty_bitmaps));
> >      assert(bs_new->job == NULL);
> >      assert(bs_new->dev == NULL);
> > -    assert(bs_new->in_use == 0);
> > +    assert(bdrv_op_blocker_is_empty(bs_new));
> 
> Since we won't move the blocker via move_feature_fields() like we
> did with in_use, do you think we should also assert on bs_old blocker
> fields as well? I.e.:
>        assert(bdrv_op_blocker_is_empty(bs_old));
> 
> Or, alternatively, to keep the same behavior as in_use, preserve the
> overlay bs blocker via bdrv_move_feature_field at [1] (see above)?
> 
> >      assert(bs_new->io_limits_enabled == false);
> >      assert(!throttle_have_timer(&bs_new->throttle_state));
> >  
> > @@ -1912,7 +1911,7 @@ void bdrv_swap(BlockDriverState *bs_new, BlockDriverState *bs_old)
> >      /* Check a few fields that should remain attached to the device */
> >      assert(bs_new->dev == NULL);
> >      assert(bs_new->job == NULL);
> > -    assert(bs_new->in_use == 0);
> > +    assert(bdrv_op_blocker_is_empty(bs_new));


Ah, my comment above is not relevant, because of this assert, that I
missed.

Reviewed-by: Jeff Cody <jcody@redhat.com>


> >      assert(bs_new->io_limits_enabled == false);
> >      assert(!throttle_have_timer(&bs_new->throttle_state));
> >  
> > @@ -1957,7 +1956,7 @@ static void bdrv_delete(BlockDriverState *bs)
> >  {
> >      assert(!bs->dev);
> >      assert(!bs->job);
> > -    assert(!bs->in_use);
> > +    assert(bdrv_op_blocker_is_empty(bs));
> >      assert(!bs->refcnt);
> >      assert(QLIST_EMPTY(&bs->dirty_bitmaps));
> >  
> > @@ -2139,7 +2138,8 @@ int bdrv_commit(BlockDriverState *bs)
> >          return -ENOTSUP;
> >      }
> >  
> > -    if (bdrv_in_use(bs) || bdrv_in_use(bs->backing_hd)) {
> > +    if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_COMMIT, NULL) ||
> > +        bdrv_op_is_blocked(bs->backing_hd, BLOCK_OP_TYPE_COMMIT, NULL)) {
> >          return -EBUSY;
> >      }
> >  
> > @@ -3374,8 +3374,9 @@ int bdrv_truncate(BlockDriverState *bs, int64_t offset)
> >          return -ENOTSUP;
> >      if (bs->read_only)
> >          return -EACCES;
> > -    if (bdrv_in_use(bs))
> > +    if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_RESIZE, NULL)) {
> >          return -EBUSY;
> > +    }
> >      ret = drv->bdrv_truncate(bs, offset);
> >      if (ret == 0) {
> >          ret = refresh_total_sectors(bs, offset >> BDRV_SECTOR_BITS);
> > @@ -5226,17 +5227,6 @@ bool bdrv_op_blocker_is_empty(BlockDriverState *bs)
> >      return true;
> >  }
> >  
> > -void bdrv_set_in_use(BlockDriverState *bs, int in_use)
> > -{
> > -    assert(bs->in_use != in_use);
> > -    bs->in_use = in_use;
> > -}
> > -
> > -int bdrv_in_use(BlockDriverState *bs)
> > -{
> > -    return bs->in_use;
> > -}
> > -
> >  void bdrv_iostatus_enable(BlockDriverState *bs)
> >  {
> >      bs->iostatus_enabled = true;
> > diff --git a/blockdev.c b/blockdev.c
> > index c3422a1..593925e 100644
> > --- a/blockdev.c
> > +++ b/blockdev.c
> > @@ -1293,8 +1293,8 @@ static void external_snapshot_prepare(BlkTransactionState *common,
> >          return;
> >      }
> >  
> > -    if (bdrv_in_use(state->old_bs)) {
> > -        error_set(errp, QERR_DEVICE_IN_USE, device);
> > +    if (bdrv_op_is_blocked(state->old_bs,
> > +                           BLOCK_OP_TYPE_EXTERNAL_SNAPSHOT, errp)) {
> >          return;
> >      }
> >  
> > @@ -1516,8 +1516,7 @@ exit:
> >  
> >  static void eject_device(BlockDriverState *bs, int force, Error **errp)
> >  {
> > -    if (bdrv_in_use(bs)) {
> > -        error_set(errp, QERR_DEVICE_IN_USE, bdrv_get_device_name(bs));
> > +    if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_EJECT, errp)) {
> >          return;
> >      }
> >      if (!bdrv_dev_has_removable_media(bs)) {
> > @@ -1717,14 +1716,16 @@ int do_drive_del(Monitor *mon, const QDict *qdict, QObject **ret_data)
> >  {
> >      const char *id = qdict_get_str(qdict, "id");
> >      BlockDriverState *bs;
> > +    Error *local_err = NULL;
> >  
> >      bs = bdrv_find(id);
> >      if (!bs) {
> >          qerror_report(QERR_DEVICE_NOT_FOUND, id);
> >          return -1;
> >      }
> > -    if (bdrv_in_use(bs)) {
> > -        qerror_report(QERR_DEVICE_IN_USE, id);
> > +    if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_DRIVE_DEL, &local_err)) {
> > +        error_report("%s", error_get_pretty(local_err));
> > +        error_free(local_err);
> >          return -1;
> >      }
> >  
> > @@ -1976,8 +1977,7 @@ void qmp_drive_backup(const char *device, const char *target,
> >          }
> >      }
> >  
> > -    if (bdrv_in_use(bs)) {
> > -        error_set(errp, QERR_DEVICE_IN_USE, device);
> > +    if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_BACKUP_SOURCE, errp)) {
> >          return;
> >      }
> >  
> > @@ -2110,8 +2110,7 @@ void qmp_drive_mirror(const char *device, const char *target,
> >          }
> >      }
> >  
> > -    if (bdrv_in_use(bs)) {
> > -        error_set(errp, QERR_DEVICE_IN_USE, device);
> > +    if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_MIRROR, errp)) {
> >          return;
> >      }
> >  
> > diff --git a/blockjob.c b/blockjob.c
> > index b3ce14c..f643a78 100644
> > --- a/blockjob.c
> > +++ b/blockjob.c
> > @@ -41,14 +41,16 @@ void *block_job_create(const BlockJobDriver *driver, BlockDriverState *bs,
> >  {
> >      BlockJob *job;
> >  
> > -    if (bs->job || bdrv_in_use(bs)) {
> > +    if (bs->job || !bdrv_op_blocker_is_empty(bs)) {
> >          error_set(errp, QERR_DEVICE_IN_USE, bdrv_get_device_name(bs));
> >          return NULL;
> >      }
> >      bdrv_ref(bs);
> > -    bdrv_set_in_use(bs, 1);
> > -
> >      job = g_malloc0(driver->instance_size);
> > +    error_setg(&job->blocker, "block device is in use by block job: %s",
> > +               BlockJobType_lookup[driver->job_type]);
> > +    bdrv_op_block_all(bs, job->blocker);
> > +
> >      job->driver        = driver;
> >      job->bs            = bs;
> >      job->cb            = cb;
> > @@ -63,8 +65,9 @@ void *block_job_create(const BlockJobDriver *driver, BlockDriverState *bs,
> >          block_job_set_speed(job, speed, &local_err);
> >          if (local_err) {
> >              bs->job = NULL;
> > +            bdrv_op_unblock_all(bs, job->blocker);
> > +            error_free(job->blocker);
> >              g_free(job);
> > -            bdrv_set_in_use(bs, 0);
> >              error_propagate(errp, local_err);
> >              return NULL;
> >          }
> > @@ -79,8 +82,9 @@ void block_job_completed(BlockJob *job, int ret)
> >      assert(bs->job == job);
> >      job->cb(job->opaque, ret);
> >      bs->job = NULL;
> > +    bdrv_op_unblock_all(bs, job->blocker);
> > +    error_free(job->blocker);
> >      g_free(job);
> > -    bdrv_set_in_use(bs, 0);
> >  }
> >  
> >  void block_job_set_speed(BlockJob *job, int64_t speed, Error **errp)
> > diff --git a/hw/block/dataplane/virtio-blk.c b/hw/block/dataplane/virtio-blk.c
> > index 2237edb..cee0aa5 100644
> > --- a/hw/block/dataplane/virtio-blk.c
> > +++ b/hw/block/dataplane/virtio-blk.c
> > @@ -69,6 +69,9 @@ struct VirtIOBlockDataPlane {
> >                                               queue */
> >  
> >      unsigned int num_reqs;
> > +
> > +    /* Operation blocker on BDS */
> > +    Error *blocker;
> >  };
> >  
> >  /* Raise an interrupt to signal guest, if necessary */
> > @@ -368,6 +371,7 @@ void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *blk,
> >  {
> >      VirtIOBlockDataPlane *s;
> >      int fd;
> > +    Error *local_err = NULL;
> >  
> >      *dataplane = NULL;
> >  
> > @@ -390,9 +394,10 @@ void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *blk,
> >      /* If dataplane is (re-)enabled while the guest is running there could be
> >       * block jobs that can conflict.
> >       */
> > -    if (bdrv_in_use(blk->conf.bs)) {
> > -        error_setg(errp,
> > -                   "cannot start dataplane thread while device is in use");
> > +    if (bdrv_op_is_blocked(blk->conf.bs, BLOCK_OP_TYPE_DATAPLANE, &local_err)) {
> > +        error_report("cannot start dataplane thread: %s",
> > +                      error_get_pretty(local_err));
> > +        error_free(local_err);
> >          return;
> >      }
> >  
> > @@ -407,9 +412,8 @@ void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *blk,
> >      s->vdev = vdev;
> >      s->fd = fd;
> >      s->blk = blk;
> > -
> > -    /* Prevent block operations that conflict with data plane thread */
> > -    bdrv_set_in_use(blk->conf.bs, 1);
> > +    error_setg(&s->blocker, "block device is in use by data plane");
> > +    bdrv_op_block_all(blk->conf.bs, s->blocker);
> >  
> >      *dataplane = s;
> >  }
> > @@ -421,7 +425,8 @@ void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane *s)
> >      }
> >  
> >      virtio_blk_data_plane_stop(s);
> > -    bdrv_set_in_use(s->blk->conf.bs, 0);
> > +    bdrv_op_unblock_all(s->blk->conf.bs, s->blocker);
> > +    error_free(s->blocker);
> >      g_free(s);
> >  }
> >  
> > diff --git a/include/block/block.h b/include/block/block.h
> > index 4874e2a..a46f70a 100644
> > --- a/include/block/block.h
> > +++ b/include/block/block.h
> > @@ -471,8 +471,6 @@ void bdrv_disable_copy_on_read(BlockDriverState *bs);
> >  
> >  void bdrv_ref(BlockDriverState *bs);
> >  void bdrv_unref(BlockDriverState *bs);
> > -void bdrv_set_in_use(BlockDriverState *bs, int in_use);
> > -int bdrv_in_use(BlockDriverState *bs);
> >  
> >  bool bdrv_op_is_blocked(BlockDriverState *bs, BlockOpType op, Error **errp);
> >  void bdrv_op_block(BlockDriverState *bs, BlockOpType op, Error *reason);
> > diff --git a/include/block/block_int.h b/include/block/block_int.h
> > index 4e558d0..1d3f76f 100644
> > --- a/include/block/block_int.h
> > +++ b/include/block/block_int.h
> > @@ -359,7 +359,6 @@ struct BlockDriverState {
> >      QTAILQ_ENTRY(BlockDriverState) device_list;
> >      QLIST_HEAD(, BdrvDirtyBitmap) dirty_bitmaps;
> >      int refcnt;
> > -    int in_use; /* users other than guest access, eg. block migration */
> >  
> >      QLIST_HEAD(, BdrvTrackedRequest) tracked_requests;
> >  
> > diff --git a/include/block/blockjob.h b/include/block/blockjob.h
> > index d76de62..c0a7875 100644
> > --- a/include/block/blockjob.h
> > +++ b/include/block/blockjob.h
> > @@ -106,6 +106,9 @@ struct BlockJob {
> >      /** The completion function that will be called when the job completes.  */
> >      BlockDriverCompletionFunc *cb;
> >  
> > +    /** Block other operations when block job is running */
> > +    Error *blocker;
> > +
> >      /** The opaque value that is passed to the completion function.  */
> >      void *opaque;
> >  };
> > -- 
> > 1.9.0
> > 
> > 
>