[PATCH v4 1/2] KVM: s390: pv: pull all vcpus out of SIE before converting to/from pv vcpu

[Michael Mueller <mimu@linux.ibm.com>]

That will allow to modify the control blocks without locking the vcpus.
A kvm-unit-test surfaced a possible DEADLOCK situation when guest cpus are
executing intercepted instructions at this time.

[  319.799638] ======================================================
[  319.799639] WARNING: possible circular locking dependency detected
[  319.799641] 5.17.0-rc5-08427-gfd14b6309198 #4661 Not tainted
[  319.799643] ------------------------------------------------------
[  319.799644] qemu-system-s39/14220 is trying to acquire lock:
[  319.799646] 00000000b30c0b50 (&kvm->lock){+.+.}-{3:3}, at: kvm_s390_set_tod_clock+0x36/0x250
[  319.799659]
               but task is already holding lock:
[  319.799660] 00000000b5beda60 (&vcpu->mutex){+.+.}-{3:3}, at: kvm_vcpu_ioctl+0x9a/0x958
[  319.799665]
               which lock already depends on the new lock.

[  319.799667]
               the existing dependency chain (in reverse order) is:
[  319.799668]
               -> #1 (&vcpu->mutex){+.+.}-{3:3}:
[  319.799671]        __mutex_lock+0x8a/0x798
[  319.799677]        mutex_lock_nested+0x32/0x40
[  319.799679]        kvm_arch_vm_ioctl+0x1902/0x2c58
[  319.799682]        kvm_vm_ioctl+0x5b0/0xa80
[  319.799685]        __s390x_sys_ioctl+0xbe/0x100
[  319.799688]        __do_syscall+0x1da/0x208
[  319.799689]        system_call+0x82/0xb0
[  319.799692]
               -> #0 (&kvm->lock){+.+.}-{3:3}:
[  319.799694]        __lock_acquire+0x1916/0x2e70
[  319.799699]        lock_acquire+0x164/0x388
[  319.799702]        __mutex_lock+0x8a/0x798
[  319.799757]        mutex_lock_nested+0x32/0x40
[  319.799759]        kvm_s390_set_tod_clock+0x36/0x250
[  319.799761]        kvm_s390_handle_b2+0x6cc/0x26f0
[  319.799764]        kvm_handle_sie_intercept+0x1fe/0xe98
[  319.799765]        kvm_arch_vcpu_ioctl_run+0xec8/0x1880
[  319.799768]        kvm_vcpu_ioctl+0x29e/0x958
[  319.799769]        __s390x_sys_ioctl+0xbe/0x100
[  319.799771]        __do_syscall+0x1da/0x208
[  319.799773]        system_call+0x82/0xb0
[  319.799774]
               other info that might help us debug this:

[  319.799776]  Possible unsafe locking scenario:

[  319.799777]        CPU0                    CPU1
[  319.799778]        ----                    ----
[  319.799779]   lock(&vcpu->mutex);
[  319.799780]                                lock(&kvm->lock);
[  319.799782]                                lock(&vcpu->mutex);
[  319.799783]   lock(&kvm->lock);
[  319.799784]
                *** DEADLOCK ***

[  319.799785] 2 locks held by qemu-system-s39/14220:
[  319.799787]  #0: 00000000b5beda60 (&vcpu->mutex){+.+.}-{3:3}, at: kvm_vcpu_ioctl+0x9a/0x958
[  319.799791]  #1: 00000000b30c4588 (&kvm->srcu){....}-{0:0}, at: kvm_arch_vcpu_ioctl_run+0x6f2/0x1880
[  319.799796]
               stack backtrace:
[  319.799798] CPU: 5 PID: 14220 Comm: qemu-system-s39 Not tainted 5.17.0-rc5-08427-gfd14b6309198 #4661
[  319.799801] Hardware name: IBM 8561 T01 701 (LPAR)
[  319.799802] Call Trace:
[  319.799803]  [<000000020d7410de>] dump_stack_lvl+0x76/0x98
[  319.799808]  [<000000020cbbd268>] check_noncircular+0x140/0x160
[  319.799811]  [<000000020cbc0efe>] __lock_acquire+0x1916/0x2e70
[  319.799813]  [<000000020cbc2dbc>] lock_acquire+0x164/0x388
[  319.799816]  [<000000020d75013a>] __mutex_lock+0x8a/0x798
[  319.799818]  [<000000020d75087a>] mutex_lock_nested+0x32/0x40
[  319.799820]  [<000000020cb029a6>] kvm_s390_set_tod_clock+0x36/0x250
[  319.799823]  [<000000020cb14d14>] kvm_s390_handle_b2+0x6cc/0x26f0
[  319.799825]  [<000000020cb09b6e>] kvm_handle_sie_intercept+0x1fe/0xe98
[  319.799827]  [<000000020cb06c28>] kvm_arch_vcpu_ioctl_run+0xec8/0x1880
[  319.799829]  [<000000020caeddc6>] kvm_vcpu_ioctl+0x29e/0x958
[  319.799831]  [<000000020ce4e82e>] __s390x_sys_ioctl+0xbe/0x100
[  319.799833]  [<000000020d744a72>] __do_syscall+0x1da/0x208
[  319.799835]  [<000000020d757322>] system_call+0x82/0xb0
[  319.799836] INFO: lockdep is turned off.

Signed-off-by: Michael Mueller <mimu@linux.ibm.com>
---
 arch/s390/kvm/kvm-s390.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
index 2296b1ff1e02..b75ea3e57172 100644
--- a/arch/s390/kvm/kvm-s390.c
+++ b/arch/s390/kvm/kvm-s390.c
@@ -2185,15 +2185,15 @@ static int kvm_s390_cpus_from_pv(struct kvm *kvm, u16 *rcp, u16 *rrcp)
 	 * behind.
 	 * We want to return the first failure rc and rrc, though.
 	 */
+	kvm_s390_vcpu_block_all(kvm);
 	kvm_for_each_vcpu(i, vcpu, kvm) {
-		mutex_lock(&vcpu->mutex);
 		if (kvm_s390_pv_destroy_cpu(vcpu, &rc, &rrc) && !ret) {
 			*rcp = rc;
 			*rrcp = rrc;
 			ret = -EIO;
 		}
-		mutex_unlock(&vcpu->mutex);
 	}
+	kvm_s390_vcpu_unblock_all(kvm);
 	return ret;
 }
 
@@ -2205,13 +2205,13 @@ static int kvm_s390_cpus_to_pv(struct kvm *kvm, u16 *rc, u16 *rrc)
 
 	struct kvm_vcpu *vcpu;
 
+	kvm_s390_vcpu_block_all(kvm);
 	kvm_for_each_vcpu(i, vcpu, kvm) {
-		mutex_lock(&vcpu->mutex);
 		r = kvm_s390_pv_create_cpu(vcpu, rc, rrc);
-		mutex_unlock(&vcpu->mutex);
 		if (r)
 			break;
 	}
+	kvm_s390_vcpu_unblock_all(kvm);
 	if (r)
 		kvm_s390_cpus_from_pv(kvm, &dummy, &dummy);
 	return r;
-- 
2.32.0