Re: [PATCH v4 02/16] vfio/pci: Preserve vfio-pci device files across Live Update

[David Matlack <dmatlack@google.com>]

On 2026-05-11 04:47 PM, Vipin Sharma wrote:

> +static int vfio_pci_liveupdate_freeze(struct liveupdate_file_op_args *args)
> +{
> +	struct vfio_device *device = vfio_device_from_file(args->file);
> +	struct vfio_pci_core_device *vdev;
> +	struct pci_dev *pdev;
> +	int ret;
> +
> +	vdev = container_of(device, struct vfio_pci_core_device, vdev);
> +	pdev = vdev->pdev;
> +
> +	guard(mutex)(&device->dev_set->lock);
> +
> +	/*
> +	 * Userspace must disable interrupts on the device prior to freeze so
> +	 * that the device does not send any interrupts until new interrupt
> +	 * handlers have been established by the next kernel.
> +	 */
> +	if (vdev->irq_type != VFIO_PCI_NUM_IRQS) {
> +		pci_err(pdev, "Freeze failed! Interrupts are still enabled.\n");
> +		return -EINVAL;
> +	}
> +
> +	guard(rwsem_write)(&vdev->memory_lock);
> +
> +	/*
> +	 * Userspace must make sure device is not in the lower power state for
> +	 * live update. We may relax this in future.
> +	 */
> +	if (pdev->current_state != PCI_D0) {
> +		pci_err(pdev, "Freeze failed! Device not in D0 state.\n");
> +		return -EINVAL;
> +	}
> +
> +	/*
> +	 * Reset is a temporary measure to provide kernel after kexec a clean
> +	 * device while VFIO live update work is under development and not
> +	 * fully supported.  It will go away once continuous DMA support is
> +	 * added to device preservation.
> +	 */
> +	vfio_pci_zap_bars(vdev);
> +	ret = pci_load_saved_state(pdev, vdev->pci_saved_state);
> +	if (ret)
> +		return ret;
> +	pci_clear_master(pdev);
> +	vfio_pci_core_try_reset(vdev);

I am seeing the following lockdep splat get triggered by this reset when
testing with this commit using vfio_pci_liveupdate_kexec_test. It seems to be
related to taking memory_lock above.

[ 2710.299017][T75672] ======================================================
[ 2710.305908][T75672] WARNING: possible circular locking dependency detected
[ 2710.312797][T75672] 7.1.0-dbg-DEV #59 Tainted: G S
[ 2710.319077][T75672] ------------------------------------------------------
[ 2710.325967][T75672] kexec/75672 is trying to acquire lock:
[ 2710.331474][T75672] ff46fd4fdbaeef08 (&group->mutex){+.+.}-{4:4}, at: pci_dev_reset_iommu_prepare+0x6e/0x1a0
[ 2710.341336][T75672]
[ 2710.341336][T75672] but task is already holding lock:
[ 2710.348574][T75672] ff46fd501f9a19a8 (&vdev->memory_lock){++++}-{4:4}, at: vfio_pci_liveupdate_freeze+0x51/0x100
[ 2710.358764][T75672]
[ 2710.358764][T75672] which lock already depends on the new lock.
[ 2710.358764][T75672]
[ 2710.369031][T75672]
[ 2710.369031][T75672] the existing dependency chain (in reverse order) is:
[ 2710.377916][T75672]
[ 2710.377916][T75672] -> #4 (&vdev->memory_lock){++++}-{4:4}:
[ 2710.385675][T75672]        down_read+0x3d/0x150
[ 2710.390235][T75672]        vfio_pci_mmap_huge_fault+0xb9/0x160
[ 2710.396091][T75672]        __do_fault+0x46/0x140
[ 2710.400734][T75672]        do_pte_missing+0x4c3/0xff0
[ 2710.405803][T75672]        handle_mm_fault+0x7c4/0xb30
[ 2710.410961][T75672]        fixup_user_fault+0x115/0x270
[ 2710.416209][T75672]        vaddr_get_pfns+0x1a1/0x390
[ 2710.421286][T75672]        vfio_pin_pages_remote+0x148/0x4d0
[ 2710.426959][T75672]        vfio_pin_map_dma+0xcc/0x260
[ 2710.432116][T75672]        vfio_iommu_type1_ioctl+0xda4/0xec0
[ 2710.437884][T75672]        __se_sys_ioctl+0x71/0xc0
[ 2710.442790][T75672]        do_syscall_64+0x15f/0x710
[ 2710.447788][T75672]        entry_SYSCALL_64_after_hwframe+0x77/0x7f
[ 2710.454074][T75672]
[ 2710.454074][T75672] -> #3 (&mm->mmap_lock){++++}-{4:4}:
[ 2710.461489][T75672]        down_read_killable+0x48/0x180
[ 2710.466821][T75672]        mmap_read_lock_killable+0x12/0x50
[ 2710.472505][T75672]        lock_mm_and_find_vma+0x11d/0x130
[ 2710.478093][T75672]        do_user_addr_fault+0x3a0/0x6c0
[ 2710.483521][T75672]        exc_page_fault+0x68/0xa0
[ 2710.488423][T75672]        asm_exc_page_fault+0x26/0x30
[ 2710.493669][T75672]        filldir+0xe2/0x190
[ 2710.498047][T75672]        ext4_readdir+0xb47/0xcf0
[ 2710.502950][T75672]        iterate_dir+0x84/0x160
[ 2710.507677][T75672]        __se_sys_getdents+0x74/0x120
[ 2710.512929][T75672]        do_syscall_64+0x15f/0x710
[ 2710.517919][T75672]        entry_SYSCALL_64_after_hwframe+0x77/0x7f
[ 2710.524202][T75672]
[ 2710.524202][T75672] -> #2 (&type->i_mutex_dir_key#4){++++}-{4:4}:
[ 2710.532478][T75672]        down_read+0x3d/0x150
[ 2710.537030][T75672]        lookup_slow+0x26/0x50
[ 2710.541675][T75672]        link_path_walk+0x42c/0x580
[ 2710.546743][T75672]        path_openat+0xd1/0xde0
[ 2710.551466][T75672]        do_file_open_root+0x114/0x250
[ 2710.556798][T75672]        file_open_root+0x89/0xb0
[ 2710.561703][T75672]        kernel_read_file_from_path_initns+0xba/0x130
[ 2710.568342][T75672]        _request_firmware+0x4ab/0x8c0
[ 2710.573677][T75672]        request_firmware_direct+0x36/0x50
[ 2710.579356][T75672]        request_microcode_fw+0xf2/0x510
[ 2710.584869][T75672]        reload_store+0x197/0x230
[ 2710.589766][T75672]        kernfs_fop_write_iter+0x13f/0x1d0
[ 2710.595452][T75672]        vfs_write+0x2be/0x3b0
[ 2710.600097][T75672]        ksys_write+0x73/0x100
[ 2710.604735][T75672]        do_syscall_64+0x15f/0x710
[ 2710.609723][T75672]        entry_SYSCALL_64_after_hwframe+0x77/0x7f
[ 2710.616009][T75672]
[ 2710.616009][T75672] -> #1 (cpu_hotplug_lock){++++}-{0:0}:
[ 2710.623591][T75672]        cpus_read_lock+0x3b/0xd0
[ 2710.628499][T75672]        __cpuhp_state_add_instance+0x19/0x40
[ 2710.634443][T75672]        iova_domain_init_rcaches+0x1ef/0x230
[ 2710.640385][T75672]        iommu_setup_dma_ops+0x175/0x540
[ 2710.645891][T75672]        iommu_device_register+0x188/0x220
[ 2710.651564][T75672]        intel_iommu_init+0x35a/0x440
[ 2710.656811][T75672]        pci_iommu_init+0x16/0x40
[ 2710.661713][T75672]        do_one_initcall+0xf5/0x3a0
[ 2710.666786][T75672]        do_initcall_level+0x82/0xa0
[ 2710.671953][T75672]        do_initcalls+0x43/0x70
[ 2710.676672][T75672]        kernel_init_freeable+0x152/0x1d0
[ 2710.682266][T75672]        kernel_init+0x1a/0x130
[ 2710.686996][T75672]        ret_from_fork+0x16b/0x310
[ 2710.691991][T75672]        ret_from_fork_asm+0x1a/0x30
[ 2710.697151][T75672]
[ 2710.697151][T75672] -> #0 (&group->mutex){+.+.}-{4:4}:
[ 2710.704478][T75672]        __lock_acquire+0x14c6/0x2800
[ 2710.709729][T75672]        lock_acquire+0xd3/0x2c0
[ 2710.714542][T75672]        __mutex_lock+0x8f/0xcd0
[ 2710.719349][T75672]        pci_dev_reset_iommu_prepare+0x6e/0x1a0
[ 2710.725461][T75672]        pcie_flr+0x32/0xc0
[ 2710.729842][T75672]        __pci_reset_function_locked+0x84/0x120
[ 2710.735954][T75672]        vfio_pci_core_try_reset+0x96/0xe0
[ 2710.741630][T75672]        vfio_pci_liveupdate_freeze+0x89/0x100
[ 2710.747653][T75672]        luo_file_freeze+0xba/0x280
[ 2710.752725][T75672]        luo_session_serialize+0x69/0x190
[ 2710.758321][T75672]        liveupdate_reboot+0x19/0x30
[ 2710.763490][T75672]        kernel_kexec+0x2f/0xa0
[ 2710.768220][T75672]        __se_sys_reboot+0xfd/0x210
[ 2710.773301][T75672]        do_syscall_64+0x15f/0x710
[ 2710.778284][T75672]        entry_SYSCALL_64_after_hwframe+0x77/0x7f
[ 2710.784568][T75672]
[ 2710.784568][T75672] other info that might help us debug this:
[ 2710.784568][T75672]
[ 2710.794663][T75672] Chain exists of:
[ 2710.794663][T75672]   &group->mutex --> &mm->mmap_lock --> &vdev->memory_lock
[ 2710.794663][T75672]
[ 2710.807543][T75672]  Possible unsafe locking scenario:
[ 2710.807543][T75672]
[ 2710.814863][T75672]        CPU0                    CPU1
[ 2710.820106][T75672]        ----                    ----
[ 2710.825352][T75672]   lock(&vdev->memory_lock);
[ 2710.829904][T75672]                                lock(&mm->mmap_lock);
[ 2710.836620][T75672]                                lock(&vdev->memory_lock);
[ 2710.843682][T75672]   lock(&group->mutex);
[ 2710.847798][T75672]
[ 2710.847798][T75672]  *** DEADLOCK ***
[ 2710.847798][T75672]
[ 2710.855818][T75672] 7 locks held by kexec/75672:
[ 2710.860457][T75672]  #0: ffffffff90a81330 (system_transition_mutex){+.+.}-{4:4}, at: __se_sys_reboot+0xe4/0x210
[ 2710.870554][T75672]  #1: ffffffff90e1d0c0 (luo_session_global.outgoing.rwsem){+.+.}-{4:4}, at: luo_session_serialize+0x1f/0x190
[ 2710.882043][T75672]  #2: ff46fd50602b7ae0 (&session->mutex){+.+.}-{4:4}, at: luo_session_serialize+0x4f/0x190
[ 2710.891972][T75672]  #3: ff46fd500bec0788 (&luo_file->mutex){+.+.}-{4:4}, at: luo_file_freeze+0x65/0x280
[ 2710.901463][T75672]  #4: ff46fd509d8106a8 (&new_dev_set->lock){+.+.}-{4:4}, at: vfio_pci_liveupdate_freeze+0x36/0x100
[ 2710.912086][T75672]  #5: ff46fd501f9a19a8 (&vdev->memory_lock){++++}-{4:4}, at: vfio_pci_liveupdate_freeze+0x51/0x100
[ 2710.922701][T75672]  #6: ff46fd4fd416c1f0 (&dev->mutex){....}-{4:4}, at: pci_dev_trylock+0x25/0x60
[ 2710.931676][T75672]
[ 2710.931676][T75672] stack backtrace:
[ 2710.937439][T75672] CPU: 193 UID: 0 PID: 75672 Comm: kexec Tainted: G S                  7.1.0-dbg-DEV #59 PREEMPTLAZY
[ 2710.937442][T75672] Tainted: [S]=CPU_OUT_OF_SPEC
[ 2710.937442][T75672] Hardware name: Google Izumi-EMR/izumi, BIOS 0.20251023.0-0 10/23/2025
[ 2710.937443][T75672] Call Trace:
[ 2710.937446][T75672]  <TASK>
[ 2710.937448][T75672]  dump_stack_lvl+0x54/0x70
[ 2710.937453][T75672]  print_circular_bug+0x2e1/0x300
[ 2710.937455][T75672]  check_noncircular+0xf9/0x120
[ 2710.937456][T75672]  ? __bfs+0x129/0x200
[ 2710.937458][T75672]  __lock_acquire+0x14c6/0x2800
[ 2710.937460][T75672]  ? __lock_acquire+0x1240/0x2800
[ 2710.937463][T75672]  ? pci_dev_reset_iommu_prepare+0x6e/0x1a0
[ 2710.937465][T75672]  lock_acquire+0xd3/0x2c0
[ 2710.937466][T75672]  ? pci_dev_reset_iommu_prepare+0x6e/0x1a0
[ 2710.937468][T75672]  ? lock_is_held_type+0x76/0x100
[ 2710.937471][T75672]  ? pci_dev_reset_iommu_prepare+0x6e/0x1a0
[ 2710.937473][T75672]  __mutex_lock+0x8f/0xcd0
[ 2710.937473][T75672]  ? pci_dev_reset_iommu_prepare+0x6e/0x1a0
[ 2710.937475][T75672]  ? lockdep_hardirqs_on_prepare+0x151/0x210
[ 2710.937477][T75672]  ? _raw_spin_unlock_irqrestore+0x35/0x50
[ 2710.937482][T75672]  pci_dev_reset_iommu_prepare+0x6e/0x1a0
[ 2710.937484][T75672]  pcie_flr+0x32/0xc0
[ 2710.937485][T75672]  __pci_reset_function_locked+0x84/0x120
[ 2710.937487][T75672]  vfio_pci_core_try_reset+0x96/0xe0
[ 2710.937489][T75672]  vfio_pci_liveupdate_freeze+0x89/0x100
[ 2710.937490][T75672]  luo_file_freeze+0xba/0x280
[ 2710.937492][T75672]  luo_session_serialize+0x69/0x190
[ 2710.937493][T75672]  liveupdate_reboot+0x19/0x30
[ 2710.937495][T75672]  kernel_kexec+0x2f/0xa0
[ 2710.937496][T75672]  __se_sys_reboot+0xfd/0x210
[ 2710.937497][T75672]  ? check_object+0x1ee/0x390
[ 2710.937500][T75672]  ? lock_release+0xef/0x350
[ 2710.937501][T75672]  ? kmem_cache_free+0x1b5/0x520
[ 2710.937506][T75672]  ? _raw_spin_unlock_irqrestore+0x35/0x50
[ 2710.937508][T75672]  ? kmem_cache_free+0x1b5/0x520
[ 2710.937509][T75672]  ? __x64_sys_close+0x3d/0x80
[ 2710.937510][T75672]  ? entry_SYSCALL_64_after_hwframe+0x77/0x7f
[ 2710.937511][T75672]  ? entry_SYSCALL_64_after_hwframe+0x77/0x7f
[ 2710.937512][T75672]  do_syscall_64+0x15f/0x710
[ 2710.937514][T75672]  entry_SYSCALL_64_after_hwframe+0x77/0x7f
[ 2710.937515][T75672] RIP: 0033:0x7fa57e4f2513
[ 2710.937519][T75672] Code: cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc 89 fa b8 a9 00 00 00 bf ad de e1 fe be 69 19 12 28 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 f7 d8 48 8b 0d db 2c 07 00 64 89 01 48
[ 2710.937520][T75672] RSP: 002b:00007ffd16943748 EFLAGS: 00000246 ORIG_RAX: 00000000000000a9
[ 2710.937523][T75672] RAX: ffffffffffffffda RBX: 0000000000000001 RCX: 00007fa57e4f2513
[ 2710.937524][T75672] RDX: 0000000045584543 RSI: 0000000028121969 RDI: 00000000fee1dead
[ 2710.937526][T75672] RBP: 00007ffd16943a60 R08: 0000000000000009 R09: 00007fa57e5672e0
[ 2710.937527][T75672] R10: 0000000000000008 R11: 0000000000000246 R12: 00007ffd169438e0
[ 2710.937528][T75672] R13: 0000000000000000 R14: 00007ffd169438e0 R15: 0000000000000001
[ 2710.937532][T75672]  </TASK>

> +	pci_restore_state(pdev);
> +	return 0;
>  }