[PATCH v6 0/3] KVM: x86: Include host suspended time in steal time

[PATCH v6 0/3] KVM: x86: Include host suspended time in steal time

[Suleiman Souhlal]

This series makes it so that the time that the host is suspended is
included in guests' steal time.

When the host resumes from a suspend, the guest thinks any task
that was running during the suspend ran for a long time, even though
the effective run time was much shorter, which can end up having
negative effects with scheduling.

To mitigate this issue, include the time that the host was
suspended in steal time, if the guest requests it, which lets the
guest subtract the duration from the tasks' runtime. Add new ABI 
to make this behavior opt-in per-guest.

In addition, make the guest TSC behavior consistent whether the
host TSC went backwards or not.

v6:
- Use true/false for bools.
- Indentation.
- Remove superfluous flag. 
- Use atomic operations for accumulating suspend duration.
- Reuse generic vcpu block/kick infrastructure instead of rolling our own.
- Add ABI to make the behavior opt-in per-guest.
- Add command line parameter to make guest use this.
- Reword commit messages in imperative mood.

v5: https://lore.kernel.org/kvm/20250325041350.1728373-1-suleiman@google.com/
- Fix grammar mistakes in commit message.

v4: https://lore.kernel.org/kvm/20250221053927.486476-1-suleiman@google.com/
- Advance guest TSC on suspends where host TSC goes backwards.
- Block vCPUs from running until resume notifier.
- Move suspend duration accounting out of machine-independent kvm to
  x86.
- Merge code and documentation patches.
- Reworded documentation.

v3: https://lore.kernel.org/kvm/20250107042202.2554063-1-suleiman@google.com/
- Use PM notifier instead of syscore ops (kvm_suspend()/kvm_resume()),
  because the latter doesn't get called on shallow suspend.
- Don't call function under UACCESS.
- Whitespace.

v2: https://lore.kernel.org/kvm/20240820043543.837914-1-suleiman@google.com/
- Accumulate suspend time at machine-independent kvm layer and track per-VCPU
  instead of per-VM.
- Document changes.

v1: https://lore.kernel.org/kvm/20240710074410.770409-1-suleiman@google.com/

Suleiman Souhlal (3):
  KVM: x86: Advance guest TSC after deep suspend.
  KVM: x86: Include host suspended duration in steal time
  KVM: x86: Add "suspendsteal" cmdline to request host to add suspend
    duration in steal time

 .../admin-guide/kernel-parameters.txt         |   5 +
 Documentation/virt/kvm/x86/cpuid.rst          |   4 +
 Documentation/virt/kvm/x86/msr.rst            |  15 +++
 arch/x86/include/asm/kvm_host.h               |   4 +
 arch/x86/include/uapi/asm/kvm_para.h          |   2 +
 arch/x86/kernel/kvm.c                         |  15 +++
 arch/x86/kvm/cpuid.c                          |   4 +-
 arch/x86/kvm/x86.c                            | 108 +++++++++++++++++-
 8 files changed, 150 insertions(+), 7 deletions(-)

-- 
2.50.0.727.gbf7dc18ff4-goog

[PATCH v6 1/3] KVM: x86: Advance guest TSC after deep suspend.

[Suleiman Souhlal]

Advance guest TSC to current time after suspend when the host
TSCs went backwards.

This makes the behavior consistent between suspends where host TSC
resets and suspends where it doesn't, such as suspend-to-idle, where
in the former case if the host TSC resets, the guests' would
previously be "frozen" due to KVM's backwards TSC prevention, while
in the latter case they would advance.

Suggested-by: Sean Christopherson <seanjc@google.com>
Signed-off-by: Suleiman Souhlal <suleiman@google.com>
---
 arch/x86/include/asm/kvm_host.h |  1 +
 arch/x86/kvm/x86.c              | 28 +++++++++++++++++++++++++++-
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 639d9bcee8424d..5c465bdd6d088a 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -1414,6 +1414,7 @@ struct kvm_arch {
 	u64 cur_tsc_offset;
 	u64 cur_tsc_generation;
 	int nr_vcpus_matched_tsc;
+	bool host_was_suspended;
 
 	u32 default_tsc_khz;
 	bool user_set_tsc;
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index a9d992d5652fa0..e66bab1a1f56e2 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -5035,7 +5035,32 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 
 	/* Apply any externally detected TSC adjustments (due to suspend) */
 	if (unlikely(vcpu->arch.tsc_offset_adjustment)) {
-		adjust_tsc_offset_host(vcpu, vcpu->arch.tsc_offset_adjustment);
+		unsigned long flags;
+		struct kvm *kvm;
+		bool advance;
+		u64 kernel_ns, l1_tsc, offset, tsc_now;
+
+		kvm = vcpu->kvm;
+		advance = kvm_get_time_and_clockread(&kernel_ns, &tsc_now);
+		raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
+		/*
+		 * Advance the guest's TSC to current time instead of only
+		 * preventing it from going backwards, while making sure
+		 * all the vCPUs use the same offset.
+		 */
+		if (kvm->arch.host_was_suspended && advance) {
+			l1_tsc = nsec_to_cycles(vcpu,
+						kvm->arch.kvmclock_offset + kernel_ns);
+			offset = kvm_compute_l1_tsc_offset(vcpu, l1_tsc);
+			kvm->arch.cur_tsc_offset = offset;
+			kvm_vcpu_write_tsc_offset(vcpu, offset);
+		} else if (advance)
+			kvm_vcpu_write_tsc_offset(vcpu, kvm->arch.cur_tsc_offset);
+		} else {
+			adjust_tsc_offset_host(vcpu, vcpu->arch.tsc_offset_adjustment);
+		}
+		kvm->arch.host_was_suspended = false;
+		raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
 		vcpu->arch.tsc_offset_adjustment = 0;
 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
 	}
@@ -12729,6 +12754,7 @@ int kvm_arch_enable_virtualization_cpu(void)
 				kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
 			}
 
+			kvm->arch.host_was_suspended = true;
 			/*
 			 * We have to disable TSC offset matching.. if you were
 			 * booting a VM while issuing an S4 host suspend....
-- 
2.50.0.727.gbf7dc18ff4-goog

[PATCH v6 2/3] KVM: x86: Include host suspended duration in steal time

[Suleiman Souhlal]

Introduce MSR_KVM_SUSPEND_STEAL which controls whether or not a guest
wants the duration of host suspend to be included in steal time.

This lets guests subtract the duration during which the host was
suspended from the runtime of tasks that were running over the suspend,
in order to prevent cases where host suspend causes long runtimes in
guest tasks, even though their effective runtime was much shorter.

Signed-off-by: Suleiman Souhlal <suleiman@google.com>
---
 Documentation/virt/kvm/x86/cpuid.rst |  4 ++
 Documentation/virt/kvm/x86/msr.rst   | 15 ++++++
 arch/x86/include/asm/kvm_host.h      |  3 ++
 arch/x86/include/uapi/asm/kvm_para.h |  2 +
 arch/x86/kvm/cpuid.c                 |  4 +-
 arch/x86/kvm/x86.c                   | 80 ++++++++++++++++++++++++++--
 6 files changed, 102 insertions(+), 6 deletions(-)

diff --git a/Documentation/virt/kvm/x86/cpuid.rst b/Documentation/virt/kvm/x86/cpuid.rst
index bda3e3e737d71f..71b42b6499733b 100644
--- a/Documentation/virt/kvm/x86/cpuid.rst
+++ b/Documentation/virt/kvm/x86/cpuid.rst
@@ -103,6 +103,10 @@ KVM_FEATURE_HC_MAP_GPA_RANGE       16          guest checks this feature bit bef
 KVM_FEATURE_MIGRATION_CONTROL      17          guest checks this feature bit before
                                                using MSR_KVM_MIGRATION_CONTROL
 
+KVM_FEATURE_SUSPEND_STEAL          18          guest checks this feature bit
+                                               before using
+                                               MSR_KVM_SUSPEND_STEAL.
+
 KVM_FEATURE_CLOCKSOURCE_STABLE_BIT 24          host will warn if no guest-side
                                                per-cpu warps are expected in
                                                kvmclock
diff --git a/Documentation/virt/kvm/x86/msr.rst b/Documentation/virt/kvm/x86/msr.rst
index 3aecf2a70e7b43..8c31b3994e733d 100644
--- a/Documentation/virt/kvm/x86/msr.rst
+++ b/Documentation/virt/kvm/x86/msr.rst
@@ -296,6 +296,12 @@ data:
 		the amount of time in which this vCPU did not run, in
 		nanoseconds. Time during which the vcpu is idle, will not be
 		reported as steal time.
+		If the guest set the enable bit in MSR_KVM_SUSPEND_STEAL,
+		steal time includes the duration during which the host is
+		suspended. The case where the host suspends during a VM
+		migration might not be accounted if VCPUs aren't entered
+		post-resume. A workaround would be for the VMM to ensure that
+		the guest is entered with KVM_RUN after resuming from suspend.
 
 	preempted:
 		indicate the vCPU who owns this struct is running or
@@ -388,3 +394,12 @@ data:
         guest is communicating page encryption status to the host using the
         ``KVM_HC_MAP_GPA_RANGE`` hypercall, it can set bit 0 in this MSR to
         allow live migration of the guest.
+
+MSR_KVM_SUSPEND_STEAL:
+	0x4b564d09
+
+data:
+	This MSR is available if KVM_FEATURE_SUSPEND_STEAL is present in
+	CPUID. Bit 0 controls whether the host should include the duration it
+	has been suspended in steal time (1), or not (0).
+
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 5c465bdd6d088a..b35099260aa625 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -932,6 +932,8 @@ struct kvm_vcpu_arch {
 		u8 preempted;
 		u64 msr_val;
 		u64 last_steal;
+		u64 suspend_ts;
+		atomic64_t suspend_ns;
 		struct gfn_to_hva_cache cache;
 	} st;
 
@@ -1028,6 +1030,7 @@ struct kvm_vcpu_arch {
 	} pv_eoi;
 
 	u64 msr_kvm_poll_control;
+	u64 msr_kvm_suspend_steal;
 
 	/* pv related host specific info */
 	struct {
diff --git a/arch/x86/include/uapi/asm/kvm_para.h b/arch/x86/include/uapi/asm/kvm_para.h
index a1efa7907a0b10..678ebc3d7eeb55 100644
--- a/arch/x86/include/uapi/asm/kvm_para.h
+++ b/arch/x86/include/uapi/asm/kvm_para.h
@@ -36,6 +36,7 @@
 #define KVM_FEATURE_MSI_EXT_DEST_ID	15
 #define KVM_FEATURE_HC_MAP_GPA_RANGE	16
 #define KVM_FEATURE_MIGRATION_CONTROL	17
+#define KVM_FEATURE_SUSPEND_STEAL	18
 
 #define KVM_HINTS_REALTIME      0
 
@@ -58,6 +59,7 @@
 #define MSR_KVM_ASYNC_PF_INT	0x4b564d06
 #define MSR_KVM_ASYNC_PF_ACK	0x4b564d07
 #define MSR_KVM_MIGRATION_CONTROL	0x4b564d08
+#define MSR_KVM_SUSPEND_STEAL	0x4b564d09
 
 struct kvm_steal_time {
 	__u64 steal;
diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c
index b2d006756e02a6..983867f243cafc 100644
--- a/arch/x86/kvm/cpuid.c
+++ b/arch/x86/kvm/cpuid.c
@@ -1614,8 +1614,10 @@ static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function)
 			     (1 << KVM_FEATURE_PV_SCHED_YIELD) |
 			     (1 << KVM_FEATURE_ASYNC_PF_INT);
 
-		if (sched_info_on())
+		if (sched_info_on()) {
 			entry->eax |= (1 << KVM_FEATURE_STEAL_TIME);
+			entry->eax |= (1 << KVM_FEATURE_SUSPEND_STEAL);
+		}
 
 		entry->ebx = 0;
 		entry->ecx = 0;
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index e66bab1a1f56e2..691f46b5e50f27 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -3753,6 +3753,8 @@ static void record_steal_time(struct kvm_vcpu *vcpu)
 	steal += current->sched_info.run_delay -
 		vcpu->arch.st.last_steal;
 	vcpu->arch.st.last_steal = current->sched_info.run_delay;
+	if (unlikely(atomic64_read(&vcpu->arch.st.suspend_ns)))
+		steal += atomic64_xchg(&vcpu->arch.st.suspend_ns, 0);
 	unsafe_put_user(steal, &st->steal, out);
 
 	version += 1;
@@ -4058,6 +4060,17 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 		vcpu->arch.msr_kvm_poll_control = data;
 		break;
 
+	case MSR_KVM_SUSPEND_STEAL:
+		if (!guest_pv_has(vcpu, KVM_FEATURE_SUSPEND_STEAL) ||
+		    !guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME))
+			return 1;
+
+		if (!(data & KVM_MSR_ENABLED))
+			return 1;
+
+		vcpu->arch.msr_kvm_suspend_steal = data;
+		break;
+
 	case MSR_IA32_MCG_CTL:
 	case MSR_IA32_MCG_STATUS:
 	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
@@ -4404,6 +4417,11 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 
 		msr_info->data = vcpu->arch.msr_kvm_poll_control;
 		break;
+	case MSR_KVM_SUSPEND_STEAL:
+		if (!guest_pv_has(vcpu, KVM_FEATURE_SUSPEND_STEAL))
+			return 1;
+		msr_info->data = vcpu->arch.msr_kvm_suspend_steal;
+		break;
 	case MSR_IA32_P5_MC_ADDR:
 	case MSR_IA32_P5_MC_TYPE:
 	case MSR_IA32_MCG_CAP:
@@ -7006,13 +7024,52 @@ static int kvm_arch_suspend_notifier(struct kvm *kvm)
 {
 	struct kvm_vcpu *vcpu;
 	unsigned long i;
+	bool kick_vcpus = false;
 
-	/*
-	 * Ignore the return, marking the guest paused only "fails" if the vCPU
-	 * isn't using kvmclock; continuing on is correct and desirable.
-	 */
-	kvm_for_each_vcpu(i, vcpu, kvm)
+	kvm_for_each_vcpu(i, vcpu, kvm) {
+		if (vcpu->arch.msr_kvm_suspend_steal & KVM_MSR_ENABLED) {
+			kick_vcpus = true;
+			WRITE_ONCE(vcpu->arch.st.suspend_ts,
+				   ktime_get_boottime_ns());
+		}
+		/*
+		 * Ignore the return, marking the guest paused only "fails" if
+		 * the vCPU isn't using kvmclock; continuing on is correct and
+		 * desirable.
+		 */
 		(void)kvm_set_guest_paused(vcpu);
+	}
+
+	if (kick_vcpus)
+		kvm_make_all_cpus_request(kvm, KVM_REQ_OUTSIDE_GUEST_MODE);
+
+	return NOTIFY_DONE;
+}
+
+static int
+kvm_arch_resume_notifier(struct kvm *kvm)
+{
+	struct kvm_vcpu *vcpu;
+	unsigned long i;
+
+	kvm_for_each_vcpu(i, vcpu, kvm) {
+		u64 suspend_ns = ktime_get_boottime_ns() -
+				 vcpu->arch.st.suspend_ts;
+
+		WRITE_ONCE(vcpu->arch.st.suspend_ts, 0);
+
+		/*
+		 * Only accumulate the suspend time if suspend steal-time is
+		 * enabled, but always clear suspend_ts and kick the vCPU as
+		 * the vCPU could have disabled suspend steal-time after the
+		 * suspend notifier grabbed suspend_ts.
+		 */
+		if (vcpu->arch.msr_kvm_suspend_steal & KVM_MSR_ENABLED)
+			atomic64_add(suspend_ns, &vcpu->arch.st.suspend_ns);
+
+		kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
+		kvm_vcpu_kick(vcpu);
+	}
 
 	return NOTIFY_DONE;
 }
@@ -7023,6 +7080,9 @@ int kvm_arch_pm_notifier(struct kvm *kvm, unsigned long state)
 	case PM_HIBERNATION_PREPARE:
 	case PM_SUSPEND_PREPARE:
 		return kvm_arch_suspend_notifier(kvm);
+	case PM_POST_HIBERNATION:
+	case PM_POST_SUSPEND:
+		return kvm_arch_resume_notifier(kvm);
 	}
 
 	return NOTIFY_DONE;
@@ -11212,6 +11272,16 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
 
 static bool kvm_vcpu_running(struct kvm_vcpu *vcpu)
 {
+	/*
+	 * During host SUSPEND/RESUME tasks get frozen after SUSPEND notifiers
+	 * run, and thawed before RESUME notifiers, i.e. vCPUs can be actively
+	 * running when KVM sees the system as suspended.  Block the vCPU if
+	 * KVM sees the vCPU as suspended to ensure the suspend steal time is
+	 * accounted before the guest can run, and to the correct guest task.
+	 */
+	if (READ_ONCE(vcpu->arch.st.suspend_ts))
+		return false;
+
 	return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&
 		!vcpu->arch.apf.halted);
 }
-- 
2.50.0.727.gbf7dc18ff4-goog

[PATCH v6 3/3] KVM: x86: Add "suspendsteal" cmdline to request host to add suspend duration in steal time

[Suleiman Souhlal]

Introduce a new command line parameter, "suspendsteal", enabling the
guest to use MSR_KVM_SUSPEND_STEAL, which tells the host that it would
like host suspend duration to be included in steal time.

Signed-off-by: Suleiman Souhlal <suleiman@google.com>
---
 Documentation/admin-guide/kernel-parameters.txt |  5 +++++
 arch/x86/kernel/kvm.c                           | 15 +++++++++++++++
 2 files changed, 20 insertions(+)

diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index f1f2c0874da9dd..9f5758ca8fadd5 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -7074,6 +7074,11 @@
 			improve throughput, but will also increase the
 			amount of memory reserved for use by the client.
 
+	suspendsteal
+			[X86,PV_OPS]
+			Enable requesting the host to include the duration the
+			host was suspended in steal time. Disabled by default.
+
 	suspend.pm_test_delay=
 			[SUSPEND]
 			Sets the number of seconds to remain in a suspend test
diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
index 921c1c783bc187..35d1bb2283c2c0 100644
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -320,6 +320,18 @@ static void __init paravirt_ops_setup(void)
 #endif
 }
 
+static bool suspend_steal;
+
+static int __init suspendsteal_setup(char *s)
+{
+	if (kvm_para_has_feature(KVM_FEATURE_SUSPEND_STEAL))
+		suspend_steal = true;
+
+	return 0;
+}
+
+early_param("suspendsteal", suspendsteal_setup);
+
 static void kvm_register_steal_time(void)
 {
 	int cpu = smp_processor_id();
@@ -331,6 +343,9 @@ static void kvm_register_steal_time(void)
 	wrmsrq(MSR_KVM_STEAL_TIME, (slow_virt_to_phys(st) | KVM_MSR_ENABLED));
 	pr_debug("stealtime: cpu %d, msr %llx\n", cpu,
 		(unsigned long long) slow_virt_to_phys(st));
+
+	if (suspend_steal)
+		wrmsrl(MSR_KVM_SUSPEND_STEAL, KVM_MSR_ENABLED);
 }
 
 static DEFINE_PER_CPU_DECRYPTED(unsigned long, kvm_apic_eoi) = KVM_PV_EOI_DISABLED;
-- 
2.50.0.727.gbf7dc18ff4-goog

Re: [PATCH v6 1/3] KVM: x86: Advance guest TSC after deep suspend.

[kernel test robot]

Hi Suleiman,

kernel test robot noticed the following build warnings:

[auto build test WARNING on kvm/queue]
[also build test WARNING on kvm/next kvm/linux-next linus/master v6.16-rc5 next-20250709]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]

url:    https://github.com/intel-lab-lkp/linux/commits/Suleiman-Souhlal/KVM-x86-Advance-guest-TSC-after-deep-suspend/20250709-150751
base:   https://git.kernel.org/pub/scm/virt/kvm/kvm.git queue
patch link:    https://lore.kernel.org/r/20250709070450.473297-2-suleiman%40google.com
patch subject: [PATCH v6 1/3] KVM: x86: Advance guest TSC after deep suspend.
config: i386-buildonly-randconfig-002-20250710 (https://download.01.org/0day-ci/archive/20250710/202507100515.ZQd2P9F8-lkp@intel.com/config)
compiler: gcc-12 (Debian 12.2.0-14+deb12u1) 12.2.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20250710/202507100515.ZQd2P9F8-lkp@intel.com/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202507100515.ZQd2P9F8-lkp@intel.com/

All warnings (new ones prefixed by >>):

   arch/x86/kvm/x86.c: In function 'kvm_arch_vcpu_load':
   arch/x86/kvm/x86.c:5044:27: error: implicit declaration of function 'kvm_get_time_and_clockread'; did you mean 'kvm_get_monotonic_and_clockread'? [-Werror=implicit-function-declaration]
    5044 |                 advance = kvm_get_time_and_clockread(&kernel_ns, &tsc_now);
         |                           ^~~~~~~~~~~~~~~~~~~~~~~~~~
         |                           kvm_get_monotonic_and_clockread
   arch/x86/kvm/x86.c:5062:17: error: 'kvm' undeclared (first use in this function)
    5062 |                 kvm->arch.host_was_suspended = false;
         |                 ^~~
   arch/x86/kvm/x86.c:5062:17: note: each undeclared identifier is reported only once for each function it appears in
   In file included from include/linux/bitops.h:7,
                    from include/linux/kernel.h:23,
                    from include/linux/cpumask.h:11,
                    from include/linux/alloc_tag.h:13,
                    from include/linux/percpu.h:5,
                    from include/linux/context_tracking_state.h:5,
                    from include/linux/hardirq.h:5,
                    from include/linux/kvm_host.h:7,
                    from arch/x86/kvm/x86.c:20:
   arch/x86/kvm/x86.c:5063:71: error: 'flags' undeclared (first use in this function)
    5063 |                 raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
         |                                                                       ^~~~~
   include/linux/typecheck.h:11:16: note: in definition of macro 'typecheck'
      11 |         typeof(x) __dummy2; \
         |                ^
   arch/x86/kvm/x86.c:5063:17: note: in expansion of macro 'raw_spin_unlock_irqrestore'
    5063 |                 raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
         |                 ^~~~~~~~~~~~~~~~~~~~~~~~~~
>> include/linux/typecheck.h:12:25: warning: comparison of distinct pointer types lacks a cast
      12 |         (void)(&__dummy == &__dummy2); \
         |                         ^~
   include/linux/spinlock.h:281:17: note: in expansion of macro 'typecheck'
     281 |                 typecheck(unsigned long, flags);                \
         |                 ^~~~~~~~~
   arch/x86/kvm/x86.c:5063:17: note: in expansion of macro 'raw_spin_unlock_irqrestore'
    5063 |                 raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
         |                 ^~~~~~~~~~~~~~~~~~~~~~~~~~
   arch/x86/kvm/x86.c: At top level:
   arch/x86/kvm/x86.c:5068:9: error: expected identifier or '(' before 'if'
    5068 |         if (unlikely(vcpu->cpu != cpu) || kvm_check_tsc_unstable()) {
         |         ^~
   include/linux/kvm_host.h:182:39: error: expected declaration specifiers or '...' before '(' token
     182 | #define KVM_ARCH_REQ_FLAGS(nr, flags) ({ \
         |                                       ^
   include/linux/kvm_host.h:186:36: note: in expansion of macro 'KVM_ARCH_REQ_FLAGS'
     186 | #define KVM_ARCH_REQ(nr)           KVM_ARCH_REQ_FLAGS(nr, 0)
         |                                    ^~~~~~~~~~~~~~~~~~
   arch/x86/include/asm/kvm_host.h:94:41: note: in expansion of macro 'KVM_ARCH_REQ'
      94 | #define KVM_REQ_STEAL_UPDATE            KVM_ARCH_REQ(8)
         |                                         ^~~~~~~~~~~~
   arch/x86/kvm/x86.c:5096:26: note: in expansion of macro 'KVM_REQ_STEAL_UPDATE'
    5096 |         kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
         |                          ^~~~~~~~~~~~~~~~~~~~
   arch/x86/kvm/x86.c:5096:48: error: unknown type name 'vcpu'
    5096 |         kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
         |                                                ^~~~
   arch/x86/kvm/x86.c:5097:1: error: expected identifier or '(' before '}' token
    5097 | }
         | ^
   cc1: some warnings being treated as errors


vim +12 include/linux/typecheck.h

e0deaff470900a Andrew Morton 2008-07-25   4  
e0deaff470900a Andrew Morton 2008-07-25   5  /*
e0deaff470900a Andrew Morton 2008-07-25   6   * Check at compile time that something is of a particular type.
e0deaff470900a Andrew Morton 2008-07-25   7   * Always evaluates to 1 so you may use it easily in comparisons.
e0deaff470900a Andrew Morton 2008-07-25   8   */
e0deaff470900a Andrew Morton 2008-07-25   9  #define typecheck(type,x) \
e0deaff470900a Andrew Morton 2008-07-25  10  ({	type __dummy; \
e0deaff470900a Andrew Morton 2008-07-25  11  	typeof(x) __dummy2; \
e0deaff470900a Andrew Morton 2008-07-25 @12  	(void)(&__dummy == &__dummy2); \
e0deaff470900a Andrew Morton 2008-07-25  13  	1; \
e0deaff470900a Andrew Morton 2008-07-25  14  })
e0deaff470900a Andrew Morton 2008-07-25  15  

-- 
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki

Re: [PATCH v6 1/3] KVM: x86: Advance guest TSC after deep suspend.

[kernel test robot]

Hi Suleiman,

kernel test robot noticed the following build errors:

[auto build test ERROR on kvm/queue]
[also build test ERROR on kvm/next kvm/linux-next linus/master v6.16-rc5 next-20250709]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]

url:    https://github.com/intel-lab-lkp/linux/commits/Suleiman-Souhlal/KVM-x86-Advance-guest-TSC-after-deep-suspend/20250709-150751
base:   https://git.kernel.org/pub/scm/virt/kvm/kvm.git queue
patch link:    https://lore.kernel.org/r/20250709070450.473297-2-suleiman%40google.com
patch subject: [PATCH v6 1/3] KVM: x86: Advance guest TSC after deep suspend.
config: i386-buildonly-randconfig-002-20250710 (https://download.01.org/0day-ci/archive/20250710/202507100824.oV2rHgt9-lkp@intel.com/config)
compiler: gcc-12 (Debian 12.2.0-14+deb12u1) 12.2.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20250710/202507100824.oV2rHgt9-lkp@intel.com/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202507100824.oV2rHgt9-lkp@intel.com/

All errors (new ones prefixed by >>):

   arch/x86/kvm/x86.c: In function 'kvm_arch_vcpu_load':
>> arch/x86/kvm/x86.c:5044:27: error: implicit declaration of function 'kvm_get_time_and_clockread'; did you mean 'kvm_get_monotonic_and_clockread'? [-Werror=implicit-function-declaration]
    5044 |                 advance = kvm_get_time_and_clockread(&kernel_ns, &tsc_now);
         |                           ^~~~~~~~~~~~~~~~~~~~~~~~~~
         |                           kvm_get_monotonic_and_clockread
>> arch/x86/kvm/x86.c:5062:17: error: 'kvm' undeclared (first use in this function)
    5062 |                 kvm->arch.host_was_suspended = false;
         |                 ^~~
   arch/x86/kvm/x86.c:5062:17: note: each undeclared identifier is reported only once for each function it appears in
   In file included from include/linux/bitops.h:7,
                    from include/linux/kernel.h:23,
                    from include/linux/cpumask.h:11,
                    from include/linux/alloc_tag.h:13,
                    from include/linux/percpu.h:5,
                    from include/linux/context_tracking_state.h:5,
                    from include/linux/hardirq.h:5,
                    from include/linux/kvm_host.h:7,
                    from arch/x86/kvm/x86.c:20:
>> arch/x86/kvm/x86.c:5063:71: error: 'flags' undeclared (first use in this function)
    5063 |                 raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
         |                                                                       ^~~~~
   include/linux/typecheck.h:11:16: note: in definition of macro 'typecheck'
      11 |         typeof(x) __dummy2; \
         |                ^
   arch/x86/kvm/x86.c:5063:17: note: in expansion of macro 'raw_spin_unlock_irqrestore'
    5063 |                 raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
         |                 ^~~~~~~~~~~~~~~~~~~~~~~~~~
   include/linux/typecheck.h:12:25: warning: comparison of distinct pointer types lacks a cast
      12 |         (void)(&__dummy == &__dummy2); \
         |                         ^~
   include/linux/spinlock.h:281:17: note: in expansion of macro 'typecheck'
     281 |                 typecheck(unsigned long, flags);                \
         |                 ^~~~~~~~~
   arch/x86/kvm/x86.c:5063:17: note: in expansion of macro 'raw_spin_unlock_irqrestore'
    5063 |                 raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
         |                 ^~~~~~~~~~~~~~~~~~~~~~~~~~
   arch/x86/kvm/x86.c: At top level:
>> arch/x86/kvm/x86.c:5068:9: error: expected identifier or '(' before 'if'
    5068 |         if (unlikely(vcpu->cpu != cpu) || kvm_check_tsc_unstable()) {
         |         ^~
>> include/linux/kvm_host.h:182:39: error: expected declaration specifiers or '...' before '(' token
     182 | #define KVM_ARCH_REQ_FLAGS(nr, flags) ({ \
         |                                       ^
   include/linux/kvm_host.h:186:36: note: in expansion of macro 'KVM_ARCH_REQ_FLAGS'
     186 | #define KVM_ARCH_REQ(nr)           KVM_ARCH_REQ_FLAGS(nr, 0)
         |                                    ^~~~~~~~~~~~~~~~~~
   arch/x86/include/asm/kvm_host.h:94:41: note: in expansion of macro 'KVM_ARCH_REQ'
      94 | #define KVM_REQ_STEAL_UPDATE            KVM_ARCH_REQ(8)
         |                                         ^~~~~~~~~~~~
   arch/x86/kvm/x86.c:5096:26: note: in expansion of macro 'KVM_REQ_STEAL_UPDATE'
    5096 |         kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
         |                          ^~~~~~~~~~~~~~~~~~~~
>> arch/x86/kvm/x86.c:5096:48: error: unknown type name 'vcpu'
    5096 |         kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
         |                                                ^~~~
>> arch/x86/kvm/x86.c:5097:1: error: expected identifier or '(' before '}' token
    5097 | }
         | ^
   cc1: some warnings being treated as errors


vim +5044 arch/x86/kvm/x86.c

  4997	
  4998	void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
  4999	{
  5000		struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
  5001	
  5002		vcpu->arch.l1tf_flush_l1d = true;
  5003	
  5004		if (vcpu->scheduled_out && pmu->version && pmu->event_count) {
  5005			pmu->need_cleanup = true;
  5006			kvm_make_request(KVM_REQ_PMU, vcpu);
  5007		}
  5008	
  5009		/* Address WBINVD may be executed by guest */
  5010		if (need_emulate_wbinvd(vcpu)) {
  5011			if (kvm_x86_call(has_wbinvd_exit)())
  5012				cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
  5013			else if (vcpu->cpu != -1 && vcpu->cpu != cpu)
  5014				smp_call_function_single(vcpu->cpu,
  5015						wbinvd_ipi, NULL, 1);
  5016		}
  5017	
  5018		kvm_x86_call(vcpu_load)(vcpu, cpu);
  5019	
  5020		if (vcpu != per_cpu(last_vcpu, cpu)) {
  5021			/*
  5022			 * Flush the branch predictor when switching vCPUs on the same
  5023			 * physical CPU, as each vCPU needs its own branch prediction
  5024			 * domain.  No IBPB is needed when switching between L1 and L2
  5025			 * on the same vCPU unless IBRS is advertised to the vCPU; that
  5026			 * is handled on the nested VM-Exit path.
  5027			 */
  5028			if (static_branch_likely(&switch_vcpu_ibpb))
  5029				indirect_branch_prediction_barrier();
  5030			per_cpu(last_vcpu, cpu) = vcpu;
  5031		}
  5032	
  5033		/* Save host pkru register if supported */
  5034		vcpu->arch.host_pkru = read_pkru();
  5035	
  5036		/* Apply any externally detected TSC adjustments (due to suspend) */
  5037		if (unlikely(vcpu->arch.tsc_offset_adjustment)) {
  5038			unsigned long flags;
  5039			struct kvm *kvm;
  5040			bool advance;
  5041			u64 kernel_ns, l1_tsc, offset, tsc_now;
  5042	
  5043			kvm = vcpu->kvm;
> 5044			advance = kvm_get_time_and_clockread(&kernel_ns, &tsc_now);
  5045			raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
  5046			/*
  5047			 * Advance the guest's TSC to current time instead of only
  5048			 * preventing it from going backwards, while making sure
  5049			 * all the vCPUs use the same offset.
  5050			 */
  5051			if (kvm->arch.host_was_suspended && advance) {
  5052				l1_tsc = nsec_to_cycles(vcpu,
  5053							kvm->arch.kvmclock_offset + kernel_ns);
  5054				offset = kvm_compute_l1_tsc_offset(vcpu, l1_tsc);
  5055				kvm->arch.cur_tsc_offset = offset;
  5056				kvm_vcpu_write_tsc_offset(vcpu, offset);
  5057			} else if (advance)
  5058				kvm_vcpu_write_tsc_offset(vcpu, kvm->arch.cur_tsc_offset);
  5059			} else {
  5060				adjust_tsc_offset_host(vcpu, vcpu->arch.tsc_offset_adjustment);
  5061			}
> 5062			kvm->arch.host_was_suspended = false;
> 5063			raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
  5064			vcpu->arch.tsc_offset_adjustment = 0;
  5065			kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
  5066		}
  5067	
> 5068		if (unlikely(vcpu->cpu != cpu) || kvm_check_tsc_unstable()) {
  5069			s64 tsc_delta = !vcpu->arch.last_host_tsc ? 0 :
  5070					rdtsc() - vcpu->arch.last_host_tsc;
  5071			if (tsc_delta < 0)
  5072				mark_tsc_unstable("KVM discovered backwards TSC");
  5073	
  5074			if (kvm_check_tsc_unstable()) {
  5075				u64 offset = kvm_compute_l1_tsc_offset(vcpu,
  5076							vcpu->arch.last_guest_tsc);
  5077				kvm_vcpu_write_tsc_offset(vcpu, offset);
  5078				if (!vcpu->arch.guest_tsc_protected)
  5079					vcpu->arch.tsc_catchup = 1;
  5080			}
  5081	
  5082			if (kvm_lapic_hv_timer_in_use(vcpu))
  5083				kvm_lapic_restart_hv_timer(vcpu);
  5084	
  5085			/*
  5086			 * On a host with synchronized TSC, there is no need to update
  5087			 * kvmclock on vcpu->cpu migration
  5088			 */
  5089			if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1)
  5090				kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
  5091			if (vcpu->cpu != cpu)
  5092				kvm_make_request(KVM_REQ_MIGRATE_TIMER, vcpu);
  5093			vcpu->cpu = cpu;
  5094		}
  5095	
> 5096		kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
> 5097	}
  5098	

-- 
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki