kvm_log_clear_one_slot: Assertion `mem->dirty_bmap' failed

kvm_log_clear_one_slot: Assertion `mem->dirty_bmap' failed

[Pete Bridgman via]

[-- Attachment #1.1: Type: text/plain, Size: 2304 bytes --]

Good afternoon,

We're seeing a rare assertion failure in production on

    qemu-system-x86_64: ../../accel/kvm/kvm-all.c:909:
kvm_log_clear_one_slot: Assertion `mem->dirty_bmap' failed.

We have seen this very occasionally in logs from devices running QEMU 7.2
on Linux 5.19
with a Windows 10 guest, seemingly spontaneously; there are no migrations
involved which
afaict has been implicated when this assertion has been hit previously
(https://bugzilla.redhat.com/show_bug.cgi?id=1772774).

We would love to be able to reproduce this (ideally on a more recent QEMU
version) so that
we can determine whether a later QEMU version has resolved the issue, and
if not file a bug
/ work on a patch, but have thus far been unable to reproduce it on demand,
with only very
occasional logs from customer devices showing this issue.

Would anyone have any ideas about what conditions might make it more likely
to reproduce,
or any pointers on what might be going wrong here - or alternatively any
thoughts about
whether a later version might be expected to resolve this?

In the following bugzilla comment:
https://bugzilla.redhat.com/show_bug.cgi?id=1772774#c11
it was mentioned that perhaps this assertion failure could be downgraded to
an early exit as
it would likely be safe to continue operation in this case. Would anyone
have any thoughts on
this? Would we be unwise to make that change (even if in just our own fork?)

I have attached our QEMU command line for reference; many thanks in advance
for any insight
or guidance. I've CCd people who were involved in a bugfix for this
assertion failure previously
(9b3a31c7) on the advice of folks in IRC, hope you don't mind.

Kind regards

Pete

Peter Bridgman (he/him)

Co-Founder & CEO

+44 (0) 7917 448150
pb@fourieraudio.com
www.fourieraudio.com

Fourier Audio Ltd is registered in England and Wales (No. 13122782)
registered at 8 Denmark Street, London, WC2H 8LS. The content of this email
is confidential and intended for the recipient specified in the message. It
is strictly forbidden to share any part of this message with any third
party, without a written consent of the sender. If you received this
message by mistake, please reply to this message and follow with its
deletion, so that we can ensure such a mistake does not occur in the future.

[-- Attachment #1.2: Type: text/html, Size: 2816 bytes --]

[-- Attachment #2: dirty-bmap-cmdline.txt --]
[-- Type: text/plain, Size: 5807 bytes --]

/usr/bin/qemu-system-x86_64 \
-name guest=doppler-win-base,debug-threads=on \
-S \
-object '{"qom-type":"secret","id":"masterKey0","format":"raw","file":"/var/lib/libvirt/qemu/domain-4-doppler-win-base/master-key.aes"}' \
-machine pc-q35-5.2,usb=off,vmport=off,dump-guest-core=off,memory-backend=pc.ram \
-accel kvm \
-cpu host,migratable=off,hv-time=on,kvmclock=off,kvm-pv-eoi=on,kvm-pv-unhalt=on,hv-relaxed=on,hv-vapic=on,hv-spinlocks=0x1fff,hv-vpindex=on,hv-runtime=on,hv-synic=on,hv-stimer=on,hv-stimer-direct=on,hv-frequencies=on,hv-tlbflush=on,hv-ipi=on,hv-avic=on,hv-crash,kvm-hint-dedicated=on,kvm-poll-control=on,host-cache-info=on,l3-cache=off \
-m 24576 \
-object '{"qom-type":"memory-backend-memfd","id":"pc.ram","hugetlb":true,"hugetlbsize":1073741824,"x-use-canonical-path-for-ramblock-id":false,"prealloc":true,"size":25769803776}' \
-overcommit mem-lock=on \
-smp 10,sockets=1,dies=1,cores=10,threads=1 \
-object '{"qom-type":"iothread","id":"iothread1"}' \
-object '{"qom-type":"iothread","id":"iothread2"}' \
-object '{"qom-type":"iothread","id":"iothread3"}' \
-object '{"qom-type":"iothread","id":"iothread4"}' \
-uuid b5cc201d-58cb-4e3b-b2e7-d540e2541e85 \
-smbios 'type=1,manufacturer=Fourier Audio,product=Doppler,version=0,serial=TF-RD-0,sku=FTE-0' \
-smbios 'type=2,manufacturer=Fourier Audio' \
-smbios 'type=3,manufacturer=Fourier Audio' \
-no-user-config \
-nodefaults \
-chardev socket,id=charmonitor,fd=33,server=on,wait=off \
-mon chardev=charmonitor,id=monitor,mode=control \
-rtc base=utc \
-no-hpet \
-global kvm-pit.lost_tick_policy=delay \
-no-shutdown \
-global ICH9-LPC.disable_s3=1 \
-global ICH9-LPC.disable_s4=1 \
-boot menu=off,strict=on \
-device '{"driver":"pcie-root-port","port":16,"chassis":1,"id":"pci.1","bus":"pcie.0","multifunction":true,"addr":"0x2"}' \
-device '{"driver":"pcie-root-port","port":17,"chassis":2,"id":"pci.2","bus":"pcie.0","addr":"0x2.0x1"}' \
-device '{"driver":"pcie-root-port","port":18,"chassis":3,"id":"pci.3","bus":"pcie.0","addr":"0x2.0x2"}' \
-device '{"driver":"pcie-root-port","port":19,"chassis":4,"id":"pci.4","bus":"pcie.0","addr":"0x2.0x3"}' \
-device '{"driver":"pcie-root-port","port":20,"chassis":5,"id":"pci.5","bus":"pcie.0","addr":"0x2.0x4"}' \
-device '{"driver":"pcie-root-port","port":21,"chassis":6,"id":"pci.6","bus":"pcie.0","addr":"0x2.0x5"}' \
-device '{"driver":"pcie-pci-bridge","id":"pci.7","bus":"pci.6","addr":"0x0"}' \
-device '{"driver":"qemu-xhci","p2":15,"p3":15,"id":"usb","bus":"pci.2","addr":"0x0"}' \
-device '{"driver":"virtio-serial-pci","id":"virtio-serial0","bus":"pci.3","addr":"0x0"}' \
-blockdev '{"driver":"file","filename":"/opt/fa/win/image/base.qcow2","node-name":"libvirt-4-storage","auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-4-format","read-only":true,"driver":"qcow2","file":"libvirt-4-storage","backing":null}' \
-blockdev '{"driver":"file","filename":"/var/opt/fa/image/current.qcow2","node-name":"libvirt-3-storage","auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-3-format","read-only":true,"driver":"qcow2","file":"libvirt-3-storage","backing":"libvirt-4-format"}' \
-blockdev '{"driver":"file","filename":"/var/opt/fa/image/user.qcow2","node-name":"libvirt-2-storage","auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-2-format","read-only":false,"driver":"qcow2","file":"libvirt-2-storage","backing":"libvirt-3-format"}' \
-device '{"driver":"ide-hd","bus":"ide.0","drive":"libvirt-2-format","id":"sata0-0-0","bootindex":1}' \
-blockdev '{"driver":"file","filename":"/opt/fa/winbin/winbin-current.qcow2","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-1-format","read-only":false,"driver":"qcow2","file":"libvirt-1-storage","backing":null}' \
-device '{"driver":"ide-hd","bus":"ide.1","drive":"libvirt-1-format","id":"sata0-0-1"}' \
-netdev '{"type":"tap","fd":"34","vhost":true,"vhostfd":"38","id":"hostnet0"}' \
-device '{"driver":"virtio-net-pci","netdev":"hostnet0","id":"net0","mac":"52:54:00:7a:e8:17","bus":"pci.1","addr":"0x0"}' \
-chardev pty,id=charserial0 \
-device '{"driver":"isa-serial","chardev":"charserial0","id":"serial0","index":0}' \
-device '{"driver":"usb-tablet","id":"input0","bus":"usb.0","port":"1"}' \
-audiodev '{"id":"audio1","driver":"none"}' \
-vnc 0.0.0.0:1,websocket=5902,password=on,audiodev=audio1 \
-device '{"driver":"qxl-vga","id":"video0","max_outputs":1,"ram_size":67108864,"vram_size":67108864,"vram64_size_mb":0,"vgamem_mb":16,"bus":"pcie.0","addr":"0x1"}' \
-device '{"driver":"virtio-balloon-pci","id":"balloon0","bus":"pci.4","addr":"0x0"}' \
-object '{"qom-type":"rng-random","id":"objrng0","filename":"/dev/random"}' \
-device '{"driver":"virtio-rng-pci","rng":"objrng0","id":"rng0","max-bytes":1024,"period":10,"bus":"pci.5","addr":"0x0"}' \
-device qemu-xhci,id=xhci-external,addr=0xF,p2=15,p3=15 \
-device usb-host,bus=xhci-external.0,hostbus=1,hostport=12,vendorid=0x088e,productid=0x5036 \
-device usb-host,bus=xhci-external.0,hostbus=1,hostport=10.1 \
-device usb-host,bus=xhci-external.0,hostbus=1,hostport=10.2 \
-device usb-host,bus=xhci-external.0,hostbus=1,hostport=10.3 \
-device usb-host,bus=xhci-external.0,hostbus=1,hostport=10.4 \
-device usb-host,bus=xhci-external.0,hostbus=1,hostport=3 \
-device usb-host,bus=xhci-external.0,hostbus=1,hostport=4 \
-device usb-host,bus=xhci-external.0,hostbus=2,hostport=3 \
-device usb-host,bus=xhci-external.0,hostbus=2,hostport=4 \
-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \
-device '{"driver":"ivshmem-doorbell","id":"shmem0","chardev":"charshmem0","vectors":1,"ioeventfd":true,"bus":"pci.7","addr":"0x1"}' \
-chardev socket,id=charshmem0,path=/tmp/ivshmem_socket \
-msg timestamp=on

Re: kvm_log_clear_one_slot: Assertion `mem->dirty_bmap' failed

[Peter Xu]

On Mon, Aug 04, 2025 at 03:57:33PM +0100, Pete Bridgman wrote:
> Good afternoon,

Hi, Pete,

> 
> We're seeing a rare assertion failure in production on
> 
>     qemu-system-x86_64: ../../accel/kvm/kvm-all.c:909:
> kvm_log_clear_one_slot: Assertion `mem->dirty_bmap' failed.
> 
> We have seen this very occasionally in logs from devices running QEMU 7.2
> on Linux 5.19
> with a Windows 10 guest, seemingly spontaneously; there are no migrations
> involved which
> afaict has been implicated when this assertion has been hit previously
> (https://bugzilla.redhat.com/show_bug.cgi?id=1772774).

It is surprising indeed if this triggers without migration in progress.

Do you have any of the coredumps available?  Some dump from "thread apply
all bt" on a gdb session attached to the core dumps may provide some more
information.

Thanks,

> 
> We would love to be able to reproduce this (ideally on a more recent QEMU
> version) so that
> we can determine whether a later QEMU version has resolved the issue, and
> if not file a bug
> / work on a patch, but have thus far been unable to reproduce it on demand,
> with only very
> occasional logs from customer devices showing this issue.
> 
> Would anyone have any ideas about what conditions might make it more likely
> to reproduce,
> or any pointers on what might be going wrong here - or alternatively any
> thoughts about
> whether a later version might be expected to resolve this?
> 
> In the following bugzilla comment:
> https://bugzilla.redhat.com/show_bug.cgi?id=1772774#c11
> it was mentioned that perhaps this assertion failure could be downgraded to
> an early exit as
> it would likely be safe to continue operation in this case. Would anyone
> have any thoughts on
> this? Would we be unwise to make that change (even if in just our own fork?)
> 
> I have attached our QEMU command line for reference; many thanks in advance
> for any insight
> or guidance. I've CCd people who were involved in a bugfix for this
> assertion failure previously
> (9b3a31c7) on the advice of folks in IRC, hope you don't mind.
> 
> Kind regards
> 
> Pete
> 
> Peter Bridgman (he/him)
> 
> Co-Founder & CEO
> 
> +44 (0) 7917 448150
> pb@fourieraudio.com
> www.fourieraudio.com
> 
> Fourier Audio Ltd is registered in England and Wales (No. 13122782)
> registered at 8 Denmark Street, London, WC2H 8LS. The content of this email
> is confidential and intended for the recipient specified in the message. It
> is strictly forbidden to share any part of this message with any third
> party, without a written consent of the sender. If you received this
> message by mistake, please reply to this message and follow with its
> deletion, so that we can ensure such a mistake does not occur in the future.

> /usr/bin/qemu-system-x86_64 \
> -name guest=doppler-win-base,debug-threads=on \
> -S \
> -object '{"qom-type":"secret","id":"masterKey0","format":"raw","file":"/var/lib/libvirt/qemu/domain-4-doppler-win-base/master-key.aes"}' \
> -machine pc-q35-5.2,usb=off,vmport=off,dump-guest-core=off,memory-backend=pc.ram \
> -accel kvm \
> -cpu host,migratable=off,hv-time=on,kvmclock=off,kvm-pv-eoi=on,kvm-pv-unhalt=on,hv-relaxed=on,hv-vapic=on,hv-spinlocks=0x1fff,hv-vpindex=on,hv-runtime=on,hv-synic=on,hv-stimer=on,hv-stimer-direct=on,hv-frequencies=on,hv-tlbflush=on,hv-ipi=on,hv-avic=on,hv-crash,kvm-hint-dedicated=on,kvm-poll-control=on,host-cache-info=on,l3-cache=off \
> -m 24576 \
> -object '{"qom-type":"memory-backend-memfd","id":"pc.ram","hugetlb":true,"hugetlbsize":1073741824,"x-use-canonical-path-for-ramblock-id":false,"prealloc":true,"size":25769803776}' \
> -overcommit mem-lock=on \
> -smp 10,sockets=1,dies=1,cores=10,threads=1 \
> -object '{"qom-type":"iothread","id":"iothread1"}' \
> -object '{"qom-type":"iothread","id":"iothread2"}' \
> -object '{"qom-type":"iothread","id":"iothread3"}' \
> -object '{"qom-type":"iothread","id":"iothread4"}' \
> -uuid b5cc201d-58cb-4e3b-b2e7-d540e2541e85 \
> -smbios 'type=1,manufacturer=Fourier Audio,product=Doppler,version=0,serial=TF-RD-0,sku=FTE-0' \
> -smbios 'type=2,manufacturer=Fourier Audio' \
> -smbios 'type=3,manufacturer=Fourier Audio' \
> -no-user-config \
> -nodefaults \
> -chardev socket,id=charmonitor,fd=33,server=on,wait=off \
> -mon chardev=charmonitor,id=monitor,mode=control \
> -rtc base=utc \
> -no-hpet \
> -global kvm-pit.lost_tick_policy=delay \
> -no-shutdown \
> -global ICH9-LPC.disable_s3=1 \
> -global ICH9-LPC.disable_s4=1 \
> -boot menu=off,strict=on \
> -device '{"driver":"pcie-root-port","port":16,"chassis":1,"id":"pci.1","bus":"pcie.0","multifunction":true,"addr":"0x2"}' \
> -device '{"driver":"pcie-root-port","port":17,"chassis":2,"id":"pci.2","bus":"pcie.0","addr":"0x2.0x1"}' \
> -device '{"driver":"pcie-root-port","port":18,"chassis":3,"id":"pci.3","bus":"pcie.0","addr":"0x2.0x2"}' \
> -device '{"driver":"pcie-root-port","port":19,"chassis":4,"id":"pci.4","bus":"pcie.0","addr":"0x2.0x3"}' \
> -device '{"driver":"pcie-root-port","port":20,"chassis":5,"id":"pci.5","bus":"pcie.0","addr":"0x2.0x4"}' \
> -device '{"driver":"pcie-root-port","port":21,"chassis":6,"id":"pci.6","bus":"pcie.0","addr":"0x2.0x5"}' \
> -device '{"driver":"pcie-pci-bridge","id":"pci.7","bus":"pci.6","addr":"0x0"}' \
> -device '{"driver":"qemu-xhci","p2":15,"p3":15,"id":"usb","bus":"pci.2","addr":"0x0"}' \
> -device '{"driver":"virtio-serial-pci","id":"virtio-serial0","bus":"pci.3","addr":"0x0"}' \
> -blockdev '{"driver":"file","filename":"/opt/fa/win/image/base.qcow2","node-name":"libvirt-4-storage","auto-read-only":true,"discard":"unmap"}' \
> -blockdev '{"node-name":"libvirt-4-format","read-only":true,"driver":"qcow2","file":"libvirt-4-storage","backing":null}' \
> -blockdev '{"driver":"file","filename":"/var/opt/fa/image/current.qcow2","node-name":"libvirt-3-storage","auto-read-only":true,"discard":"unmap"}' \
> -blockdev '{"node-name":"libvirt-3-format","read-only":true,"driver":"qcow2","file":"libvirt-3-storage","backing":"libvirt-4-format"}' \
> -blockdev '{"driver":"file","filename":"/var/opt/fa/image/user.qcow2","node-name":"libvirt-2-storage","auto-read-only":true,"discard":"unmap"}' \
> -blockdev '{"node-name":"libvirt-2-format","read-only":false,"driver":"qcow2","file":"libvirt-2-storage","backing":"libvirt-3-format"}' \
> -device '{"driver":"ide-hd","bus":"ide.0","drive":"libvirt-2-format","id":"sata0-0-0","bootindex":1}' \
> -blockdev '{"driver":"file","filename":"/opt/fa/winbin/winbin-current.qcow2","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"}' \
> -blockdev '{"node-name":"libvirt-1-format","read-only":false,"driver":"qcow2","file":"libvirt-1-storage","backing":null}' \
> -device '{"driver":"ide-hd","bus":"ide.1","drive":"libvirt-1-format","id":"sata0-0-1"}' \
> -netdev '{"type":"tap","fd":"34","vhost":true,"vhostfd":"38","id":"hostnet0"}' \
> -device '{"driver":"virtio-net-pci","netdev":"hostnet0","id":"net0","mac":"52:54:00:7a:e8:17","bus":"pci.1","addr":"0x0"}' \
> -chardev pty,id=charserial0 \
> -device '{"driver":"isa-serial","chardev":"charserial0","id":"serial0","index":0}' \
> -device '{"driver":"usb-tablet","id":"input0","bus":"usb.0","port":"1"}' \
> -audiodev '{"id":"audio1","driver":"none"}' \
> -vnc 0.0.0.0:1,websocket=5902,password=on,audiodev=audio1 \
> -device '{"driver":"qxl-vga","id":"video0","max_outputs":1,"ram_size":67108864,"vram_size":67108864,"vram64_size_mb":0,"vgamem_mb":16,"bus":"pcie.0","addr":"0x1"}' \
> -device '{"driver":"virtio-balloon-pci","id":"balloon0","bus":"pci.4","addr":"0x0"}' \
> -object '{"qom-type":"rng-random","id":"objrng0","filename":"/dev/random"}' \
> -device '{"driver":"virtio-rng-pci","rng":"objrng0","id":"rng0","max-bytes":1024,"period":10,"bus":"pci.5","addr":"0x0"}' \
> -device qemu-xhci,id=xhci-external,addr=0xF,p2=15,p3=15 \
> -device usb-host,bus=xhci-external.0,hostbus=1,hostport=12,vendorid=0x088e,productid=0x5036 \
> -device usb-host,bus=xhci-external.0,hostbus=1,hostport=10.1 \
> -device usb-host,bus=xhci-external.0,hostbus=1,hostport=10.2 \
> -device usb-host,bus=xhci-external.0,hostbus=1,hostport=10.3 \
> -device usb-host,bus=xhci-external.0,hostbus=1,hostport=10.4 \
> -device usb-host,bus=xhci-external.0,hostbus=1,hostport=3 \
> -device usb-host,bus=xhci-external.0,hostbus=1,hostport=4 \
> -device usb-host,bus=xhci-external.0,hostbus=2,hostport=3 \
> -device usb-host,bus=xhci-external.0,hostbus=2,hostport=4 \
> -sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \
> -device '{"driver":"ivshmem-doorbell","id":"shmem0","chardev":"charshmem0","vectors":1,"ioeventfd":true,"bus":"pci.7","addr":"0x1"}' \
> -chardev socket,id=charshmem0,path=/tmp/ivshmem_socket \
> -msg timestamp=on


-- 
Peter Xu