Re: [Qemu-devel] [PATCH] block: Cleanup BMDS in bdrv_close_all

[Fam Zheng <famz@redhat.com>]

On Wed, 08/30 16:04, Juan Quintela wrote:
> Fam Zheng <famz@redhat.com> wrote:
> > On Wed, 08/30 13:49, Juan Quintela wrote:
> >> Fam Zheng <famz@redhat.com> wrote:
> >> > This fixes the assertion due to op blockers added by BMDS:
> >> >
> >> > block.c:3248: bdrv_delete: Assertion `bdrv_op_blocker_is_empty(bs)' failed.
> >> >
> >> > Reproducer: simply start block migration and quit QEMU before it ends.
> >> >
> >> > Cc: qemu-stable@nongnu.org
> >> > Signed-off-by: Fam Zheng <famz@redhat.com>
> >> 
> >> No need for one stub, see later.
> >> 
> >> 
> >> > ---
> >> >  block.c                 | 2 ++
> >> >  migration/block.c       | 2 +-
> >> >  migration/block.h       | 1 +
> >> >  stubs/Makefile.objs     | 1 +
> >> >  stubs/block-migration.c | 6 ++++++
> >> >  5 files changed, 11 insertions(+), 1 deletion(-)
> >> >  create mode 100644 stubs/block-migration.c
> >> >
> >> > diff --git a/block.c b/block.c
> >> > index 3308814bba..508a57274d 100644
> >> > --- a/block.c
> >> > +++ b/block.c
> >> > @@ -43,6 +43,7 @@
> >> >  #include "qemu/cutils.h"
> >> >  #include "qemu/id.h"
> >> >  #include "qapi/util.h"
> >> > +#include "migration/block.h"
> >> 
> >> this should be misc.h
> >> 
> >> >  
> >> >  #ifdef CONFIG_BSD
> >> >  #include <sys/ioctl.h>
> >> > @@ -3111,6 +3112,7 @@ static void bdrv_close(BlockDriverState *bs)
> >> >  
> >> >  void bdrv_close_all(void)
> >> >  {
> >> > +    block_migration_cleanup_bmds();
> >> >      block_job_cancel_sync_all();
> >> >      nbd_export_close_all();
> >> >  
> >> 
> >> > diff --git a/migration/block.h b/migration/block.h
> >> > index 22ebe94259..8bae1cf55a 100644
> >> > --- a/migration/block.h
> >> > +++ b/migration/block.h
> >> > @@ -42,4 +42,5 @@ static inline uint64_t blk_mig_bytes_total(void)
> >> >  #endif /* CONFIG_LIVE_BLOCK_MIGRATION */
> >> >  
> >> >  void migrate_set_block_enabled(bool value, Error **errp);
> >> > +void block_migration_cleanup_bmds(void);
> >> >  #endif /* MIGRATION_BLOCK_H */
> >> > diff --git a/stubs/Makefile.objs b/stubs/Makefile.objs
> >> > index e69c217aff..7540913767 100644
> >> > --- a/stubs/Makefile.objs
> >> > +++ b/stubs/Makefile.objs
> >> > @@ -19,6 +19,7 @@ stub-obj-y += is-daemonized.o
> >> >  stub-obj-$(CONFIG_LINUX_AIO) += linux-aio.o
> >> >  stub-obj-y += machine-init-done.o
> >> >  stub-obj-y += migr-blocker.o
> >> > +stub-obj-y += block-migration.o
> >> >  stub-obj-y += change-state-handler.o
> >> >  stub-obj-y += monitor.o
> >> >  stub-obj-y += notify-event.o
> >> > diff --git a/stubs/block-migration.c b/stubs/block-migration.c
> >> > new file mode 100644
> >> > index 0000000000..855f15c757
> >> > --- /dev/null
> >> > +++ b/stubs/block-migration.c
> >> > @@ -0,0 +1,6 @@
> >> > +#include "qemu/osdep.h"
> >> > +#include "migration/block.h"
> >> > +
> >> > +void block_migration_cleanup_bmds(void)
> >> > +{
> >> > +}
> >> 
> >> You can add this inside include/migration/misc.h
> >> 
> >> #ifdef CONFIG_LIVE_BLOCK_MIGRATION
> >> void blk_mig_init(void);
> >> #else
> >> static inline void blk_mig_init(void) {}
> >> 
> >> // And then you add the stub here?
> >
> > This doesn't work.  The function is not stubbed for !CONFIG_LIVE_BLOCK_MIGRATION
> > configs, but for tools that don't link to common-obj-y. For example with your
> > proposed change, I get:
> >
> >   LINK    qemu-nbd
> > block.o: In function `bdrv_close_all':
> > /home/fam/work/qemu/block.c:3115: undefined reference to
> > `block_migration_cleanup_bmds'
> > collect2: error: ld returned 1 exit status
> > make: *** [/home/fam/work/qemu/rules.mak:121: qemu-nbd] Error 1
> > make: Leaving directory '/home/fam/work/q/build'
> 
> 
> This works for me, for both CONFIG_LIVE_BLOCK_MIGRATION enabled and not.
> For qemu-system-x86_64 and qemu-nbd.  Could you test?

I get the same error:

  LINK    qemu-nbd
block.o: In function `bdrv_close_all':
/home/fam/work/qemu/block.c:3115: undefined reference to `block_migration_cleanup_bmds'
collect2: error: ld returned 1 exit status
make: *** [/home/fam/work/qemu/rules.mak:121: qemu-nbd] Error 1
make: Leaving directory '/home/fam/work/q/build'

(also applies to qemu-img etc.)

Fam

---

$ cat config.status 
#!/bin/sh
# Generated by configure.
# Run this file to recreate the current configuration.
# Compiler output produced by configure, useful for debugging
# configure, is in config.log if it exists.
exec '/home/fam/work/qemu/configure' '--prefix=/home/fam/work/q/install/bdrv_close_all-bmds' '--enable-debug' '--extra-cflags=-Wno-error=format-truncation' '--target-list=x86_64-softmmu' "$@"

$ grep CONFIG_LIVE_BLOCK_MIGRATION config-host.h
#define CONFIG_LIVE_BLOCK_MIGRATION 1

$ make qemu-nbd V=1
(cd /home/fam/work/qemu; printf '#define QEMU_PKGVERSION '; if test -n ""; then printf '""\n'; else if test -d .git; then printf '" ('; git describe --match 'v*' 2>/dev/null | tr -d '\n'; if ! git diff-index --quiet HEAD &>/dev/null; then printf -- '-dirty'; fi; printf ')"\n'; else printf '""\n'; fi; fi) > qemu-version.h.tmp
if ! cmp -s qemu-version.h qemu-version.h.tmp; then mv qemu-version.h.tmp qemu-version.h; else rm qemu-version.h.tmp; fi
c++ -I/usr/include/pixman-1   -Werror -DHAS_LIBSSH2_SFTP_FSYNC -pthread -I/usr/include/glib-2.0 -I/usr/lib64/glib-2.0/include  -DNCURSES_WIDECHAR -D_GNU_SOURCE -D_DEFAULT_SOURCE  -fPIE -DPIE -m64 -mcx16 -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -Wstrict-prototypes -Wredundant-decls -Wall -Wundef -Wwrite-strings -Wmissing-prototypes -fno-strict-aliasing -fno-common -fwrapv  -Wno-error=format-truncation -Wexpansion-to-defined -Wendif-labels -Wno-shift-negative-value -Wno-missing-include-dirs -Wempty-body -Wnested-externs -Wformat-security -Wformat-y2k -Winit-self -Wignored-qualifiers -Wold-style-declaration -Wold-style-definition -Wtype-limits -fstack-protector-strong  -I/usr/include/p11-kit-1    -I/usr/include/libpng16  -I/usr/include/spice-1 -I/usr/include/spice-server -I/usr/include/cacard -I/usr/include/glib-2.0 -I/usr/lib64/glib-2.0/include -I/usr/include/nss3 -I/usr/include/nspr4 -pthread -I/usr/include/pixman-1  -I/usr/include/cacard -I/usr/include/glib-2.0 -I/usr/lib64/glib-2.0/include -I/usr/include/nss3 -I/usr/include/nspr4  -I/usr/include/libusb-1.0   -I/home/fam/work/qemu/tests -g  -Wl,--warn-common -Wl,-z,relro -Wl,-z,now -pie -m64 -g  -o qemu-nbd qemu-nbd.o block.o blockjob.o qemu-io-cmds.o replication.o block/raw-format.o block/qcow.o block/vdi.o block/vmdk.o block/cloop.o block/bochs.o block/vpc.o block/vvfat.o block/dmg.o block/qcow2.o block/qcow2-refcount.o block/qcow2-cluster.o block/qcow2-snapshot.o block/qcow2-cache.o block/qcow2-bitmap.o block/qed.o block/qed-l2-cache.o block/qed-table.o block/qed-cluster.o block/qed-check.o block/vhdx.o block/vhdx-endian.o block/vhdx-log.o block/quorum.o block/parallels.o block/blkdebug.o block/blkverify.o block/blkreplay.o block/block-backend.o block/snapshot.o block/qapi.o block/file-posix.o block/linux-aio.o block/null.o block/mirror.o block/commit.o block/io.o block/throttle-groups.o block/nbd.o block/nbd-client.o block/sheepdog.o block/iscsi-opts.o block/accounting.o block/dirty-bitmap.o block/write-threshold.o block/backup.o block/replication.o block/crypto.o nbd/server.o nbd/client.o nbd/common.o block/iscsi.o block/curl.o block/rbd.o block/gluster.o block/ssh.o block/dmg-bz2.o crypto/init.o crypto/hash.o crypto/hash-nettle.o crypto/hmac.o crypto/hmac-nettle.o crypto/aes.o crypto/desrfb.o crypto/cipher.o crypto/tlscreds.o crypto/tlscredsanon.o crypto/tlscredsx509.o crypto/tlssession.o crypto/secret.o crypto/random-gnutls.o crypto/pbkdf.o crypto/pbkdf-nettle.o crypto/ivgen.o crypto/ivgen-essiv.o crypto/ivgen-plain.o crypto/ivgen-plain64.o crypto/afsplit.o crypto/xts.o crypto/block.o crypto/block-qcow.o crypto/block-luks.o io/channel.o io/channel-buffer.o io/channel-command.o io/channel-file.o io/channel-socket.o io/channel-tls.o io/channel-watch.o io/channel-websock.o io/channel-util.o io/dns-resolver.o io/task.o qom/object.o qom/container.o qom/qom-qobject.o qom/object_interfaces.o  libqemuutil.a libqemustub.a  -lz -laio -L/usr/lib64/iscsi -liscsi -lcurl -lrbd -lrados -lacl -lgfapi -lglusterfs -lgfrpc -lgfxdr -luuid -lssh2 -lbz2 -lm -lgthread-2.0 -pthread -lglib-2.0   -lz -lrt -lz -lnettle  -lgnutls   -lutil

$ nm migration/block.o | grep block_migration_cleanup_bmds
00000000000011e8 T block_migration_cleanup_bmds


> 
> gcommit 2888b96dfe5ea9c7901990f54e14b1a7ed3e46b9
> Author: Fam Zheng <famz@redhat.com>
> Date:   Wed Aug 30 18:06:05 2017 +0800
> 
>     block: Cleanup BMDS in bdrv_close_all
>     
>     This fixes the assertion due to op blockers added by BMDS:
>     
>     block.c:3248: bdrv_delete: Assertion `bdrv_op_blocker_is_empty(bs)' failed.
>     
>     Reproducer: simply start block migration and quit QEMU before it ends.
>     
>     Cc: qemu-stable@nongnu.org
>     Signed-off-by: Fam Zheng <famz@redhat.com>
>     
>     --
>     
>     Don't use stub
> 
> diff --git a/block.c b/block.c
> index 3615a6809e..4268f892da 100644
> --- a/block.c
> +++ b/block.c
> @@ -43,6 +43,7 @@
>  #include "qemu/cutils.h"
>  #include "qemu/id.h"
>  #include "qapi/util.h"
> +#include "migration/misc.h"
>  
>  #ifdef CONFIG_BSD
>  #include <sys/ioctl.h>
> @@ -3111,6 +3112,7 @@ static void bdrv_close(BlockDriverState *bs)
>  
>  void bdrv_close_all(void)
>  {
> +    block_migration_cleanup_bmds();
>      block_job_cancel_sync_all();
>      nbd_export_close_all();
>  
> diff --git a/include/migration/misc.h b/include/migration/misc.h
> index c079b7771b..6ecb7068d9 100644
> --- a/include/migration/misc.h
> +++ b/include/migration/misc.h
> @@ -24,8 +24,10 @@ void ram_mig_init(void);
>  
>  #ifdef CONFIG_LIVE_BLOCK_MIGRATION
>  void blk_mig_init(void);
> +void block_migration_cleanup_bmds(void);
>  #else
>  static inline void blk_mig_init(void) {}
> +static inline void block_migration_cleanup_bmds(void) {}
>  #endif
>  
>  #define SELF_ANNOUNCE_ROUNDS 5
> diff --git a/migration/block.c b/migration/block.c
> index 9171f60028..c059e48484 100644
> --- a/migration/block.c
> +++ b/migration/block.c
> @@ -673,7 +673,7 @@ static int64_t get_remaining_dirty(void)
>  
>  
>  /* Called with iothread lock taken.  */
> -static void block_migration_cleanup_bmds(void)
> +void block_migration_cleanup_bmds(void)
>  {
>      BlkMigDevState *bmds;
>      AioContext *ctx;
> 
>