[PATCH v4 0/5] Hyper-V VTL support

[PATCH v4 0/5] Hyper-V VTL support

[Saurabh Sengar]

This patch series introduces support for Virtual Trust Level (VTL)
in Hyper-V systems. It provide a foundation for the implementation
of Hyper-V VSM support in the Linux kernel, providing a secure
platform for the development and deployment of applications.

Virtual Secure Mode (VSM) is a critical aspect of the security
infrastructure in Hyper-V systems. It provides a set of hypervisor
capabilities and enlightenments that enable the creation and
management of new security boundaries within operating system
software. The VSM achieves and maintains isolation through Virtual
Trust Levels, which are hierarchical, with higher levels being more
privileged than lower levels. Please refer to this link for further
information: https://learn.microsoft.com/en-us/virtualization/hyper-v-on-windows/tlfs/vsm

This patch series adds the initialization of the x86 platform for VTL
systems. This also adds the VTL early bootup code for initializing
and bringing up secondary cpus to targeted VTL context. In VTL, AP
has to start directly in the 64-bit mode, bypassing the usual
16-bit -> 32-bit -> 64-bit mode transition sequence that occurs after
waking up an AP with SIPI whose vector points to the 16-bit AP
startup trampoline code.

Currently only VTL level supprted is '2'. This patch series is tested
extensively on VTL2 systems.

[V4]
- Move HYPERV_VTL_MODE definition from arch/x86/Kconfig to drivers/hv/Kconfig
- Move Kconfig changes before its getting used
- Replace initial_stack with current->thread.sp as per recent upstream changes

[V3]
 - Break in to 5 patches
 - hv_init_vp_context_t -> hv_init_vp_context
 - HYPERV_VTL -> HYPERV_VTL_MODE
 - Modify description of HYPERV_VTL_MODE
 - VTL 0 and VTL 2 -> VTL0 and VTL2
 - Remove casting for this_cpu_ptr pointer

[V2]
 - Remove the code for reserve 1 IRQ.
 - boot_cpu_has -> cpu_feature_enabled.
 - Improved commit message for 0002 patch.
 - Improved Kconfig flag description for HYPERV_VTL.
 - Removed hv_result as a wrapper around hv_do_hypercall().
 - The value of output[0] copied to a local variable before returning.

Saurabh Sengar (5):
  x86/init: Make get/set_rtc_noop() public
  x86/hyperv: Add VTL specific structs and hypercalls
  x86/hyperv: Make hv_get_nmi_reason public
  x86/Kconfig: Add HYPERV_VTL_MODE
  x86/hyperv: VTL support for Hyper-V

 arch/x86/hyperv/Makefile           |   1 +
 arch/x86/hyperv/hv_vtl.c           | 227 +++++++++++++++++++++++++++++
 arch/x86/include/asm/hyperv-tlfs.h |  75 ++++++++++
 arch/x86/include/asm/mshyperv.h    |  15 ++
 arch/x86/include/asm/x86_init.h    |   2 +
 arch/x86/kernel/cpu/mshyperv.c     |   6 +-
 arch/x86/kernel/x86_init.c         |   4 +-
 drivers/hv/Kconfig                 |  24 +++
 include/asm-generic/hyperv-tlfs.h  |   4 +
 9 files changed, 351 insertions(+), 7 deletions(-)
 create mode 100644 arch/x86/hyperv/hv_vtl.c

-- 
2.34.1

[PATCH v4 1/5] x86/init: Make get/set_rtc_noop() public

[Saurabh Sengar]

Make get/set_rtc_noop() to be public so that they can be used
in other modules as well.

Co-developed-by: Tianyu Lan <tiala@microsoft.com>
Signed-off-by: Tianyu Lan <tiala@microsoft.com>
Signed-off-by: Saurabh Sengar <ssengar@linux.microsoft.com>
Reviewed-by: Wei Liu <wei.liu@kernel.org>
Reviewed-by: Michael Kelley <mikelley@microsoft.com>
---
 arch/x86/include/asm/x86_init.h | 2 ++
 arch/x86/kernel/x86_init.c      | 4 ++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/arch/x86/include/asm/x86_init.h b/arch/x86/include/asm/x86_init.h
index acc20ae4079d..88085f369ff6 100644
--- a/arch/x86/include/asm/x86_init.h
+++ b/arch/x86/include/asm/x86_init.h
@@ -330,5 +330,7 @@ extern void x86_init_uint_noop(unsigned int unused);
 extern bool bool_x86_init_noop(void);
 extern void x86_op_int_noop(int cpu);
 extern bool x86_pnpbios_disabled(void);
+extern int set_rtc_noop(const struct timespec64 *now);
+extern void get_rtc_noop(struct timespec64 *now);
 
 #endif
diff --git a/arch/x86/kernel/x86_init.c b/arch/x86/kernel/x86_init.c
index 95be3831df73..d82f4fa2f1bf 100644
--- a/arch/x86/kernel/x86_init.c
+++ b/arch/x86/kernel/x86_init.c
@@ -33,8 +33,8 @@ static int __init iommu_init_noop(void) { return 0; }
 static void iommu_shutdown_noop(void) { }
 bool __init bool_x86_init_noop(void) { return false; }
 void x86_op_int_noop(int cpu) { }
-static __init int set_rtc_noop(const struct timespec64 *now) { return -EINVAL; }
-static __init void get_rtc_noop(struct timespec64 *now) { }
+int set_rtc_noop(const struct timespec64 *now) { return -EINVAL; }
+void get_rtc_noop(struct timespec64 *now) { }
 
 static __initconst const struct of_device_id of_cmos_match[] = {
 	{ .compatible = "motorola,mc146818" },
-- 
2.34.1

[PATCH v4 2/5] x86/hyperv: Add VTL specific structs and hypercalls

[Saurabh Sengar]

Add structs and hypercalls required to enable VTL support on x86.

Signed-off-by: Saurabh Sengar <ssengar@linux.microsoft.com>
Reviewed-by: Michael Kelley <mikelley@microsoft.com>
---
 arch/x86/include/asm/hyperv-tlfs.h | 75 ++++++++++++++++++++++++++++++
 include/asm-generic/hyperv-tlfs.h  |  4 ++
 2 files changed, 79 insertions(+)

diff --git a/arch/x86/include/asm/hyperv-tlfs.h b/arch/x86/include/asm/hyperv-tlfs.h
index 0b73a809e9e1..0b0b4e9a4318 100644
--- a/arch/x86/include/asm/hyperv-tlfs.h
+++ b/arch/x86/include/asm/hyperv-tlfs.h
@@ -713,6 +713,81 @@ union hv_msi_entry {
 	} __packed;
 };
 
+struct hv_x64_segment_register {
+	__u64 base;
+	__u32 limit;
+	__u16 selector;
+	union {
+		struct {
+			__u16 segment_type : 4;
+			__u16 non_system_segment : 1;
+			__u16 descriptor_privilege_level : 2;
+			__u16 present : 1;
+			__u16 reserved : 4;
+			__u16 available : 1;
+			__u16 _long : 1;
+			__u16 _default : 1;
+			__u16 granularity : 1;
+		} __packed;
+		__u16 attributes;
+	};
+} __packed;
+
+struct hv_x64_table_register {
+	__u16 pad[3];
+	__u16 limit;
+	__u64 base;
+} __packed;
+
+struct hv_init_vp_context {
+	u64 rip;
+	u64 rsp;
+	u64 rflags;
+
+	struct hv_x64_segment_register cs;
+	struct hv_x64_segment_register ds;
+	struct hv_x64_segment_register es;
+	struct hv_x64_segment_register fs;
+	struct hv_x64_segment_register gs;
+	struct hv_x64_segment_register ss;
+	struct hv_x64_segment_register tr;
+	struct hv_x64_segment_register ldtr;
+
+	struct hv_x64_table_register idtr;
+	struct hv_x64_table_register gdtr;
+
+	u64 efer;
+	u64 cr0;
+	u64 cr3;
+	u64 cr4;
+	u64 msr_cr_pat;
+} __packed;
+
+union hv_input_vtl {
+	u8 as_uint8;
+	struct {
+		u8 target_vtl: 4;
+		u8 use_target_vtl: 1;
+		u8 reserved_z: 3;
+	};
+} __packed;
+
+struct hv_enable_vp_vtl {
+	u64				partition_id;
+	u32				vp_index;
+	union hv_input_vtl		target_vtl;
+	u8				mbz0;
+	u16				mbz1;
+	struct hv_init_vp_context	vp_context;
+} __packed;
+
+struct hv_get_vp_from_apic_id_in {
+	u64 partition_id;
+	union hv_input_vtl target_vtl;
+	u8 res[7];
+	u32 apic_ids[];
+} __packed;
+
 #include <asm-generic/hyperv-tlfs.h>
 
 #endif
diff --git a/include/asm-generic/hyperv-tlfs.h b/include/asm-generic/hyperv-tlfs.h
index b870983596b9..87258341fd7c 100644
--- a/include/asm-generic/hyperv-tlfs.h
+++ b/include/asm-generic/hyperv-tlfs.h
@@ -146,6 +146,7 @@ union hv_reference_tsc_msr {
 /* Declare the various hypercall operations. */
 #define HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE	0x0002
 #define HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST	0x0003
+#define HVCALL_ENABLE_VP_VTL			0x000f
 #define HVCALL_NOTIFY_LONG_SPIN_WAIT		0x0008
 #define HVCALL_SEND_IPI				0x000b
 #define HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX	0x0013
@@ -165,6 +166,8 @@ union hv_reference_tsc_msr {
 #define HVCALL_MAP_DEVICE_INTERRUPT		0x007c
 #define HVCALL_UNMAP_DEVICE_INTERRUPT		0x007d
 #define HVCALL_RETARGET_INTERRUPT		0x007e
+#define HVCALL_START_VP				0x0099
+#define HVCALL_GET_VP_ID_FROM_APIC_ID		0x009a
 #define HVCALL_FLUSH_GUEST_PHYSICAL_ADDRESS_SPACE 0x00af
 #define HVCALL_FLUSH_GUEST_PHYSICAL_ADDRESS_LIST 0x00b0
 #define HVCALL_MODIFY_SPARSE_GPA_PAGE_HOST_VISIBILITY 0x00db
@@ -218,6 +221,7 @@ enum HV_GENERIC_SET_FORMAT {
 #define HV_STATUS_INVALID_PORT_ID		17
 #define HV_STATUS_INVALID_CONNECTION_ID		18
 #define HV_STATUS_INSUFFICIENT_BUFFERS		19
+#define HV_STATUS_VTL_ALREADY_ENABLED		134
 
 /*
  * The Hyper-V TimeRefCount register and the TSC
-- 
2.34.1

[PATCH v4 3/5] x86/hyperv: Make hv_get_nmi_reason public

[Saurabh Sengar]

Move hv_get_nmi_reason to .h file so it can be used in other
modules as well.

Signed-off-by: Saurabh Sengar <ssengar@linux.microsoft.com>
Reviewed-by: Michael Kelley <mikelley@microsoft.com>
---
 arch/x86/include/asm/mshyperv.h | 5 +++++
 arch/x86/kernel/cpu/mshyperv.c  | 5 -----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h
index e3cef98a0142..71ed240ef66d 100644
--- a/arch/x86/include/asm/mshyperv.h
+++ b/arch/x86/include/asm/mshyperv.h
@@ -189,6 +189,11 @@ static inline struct hv_vp_assist_page *hv_get_vp_assist_page(unsigned int cpu)
 	return hv_vp_assist_page[cpu];
 }
 
+static inline unsigned char hv_get_nmi_reason(void)
+{
+	return 0;
+}
+
 void __init hyperv_init(void);
 void hyperv_setup_mmu_ops(void);
 void set_hv_tscchange_cb(void (*cb)(void));
diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c
index 315fc358e584..5ee02af57dac 100644
--- a/arch/x86/kernel/cpu/mshyperv.c
+++ b/arch/x86/kernel/cpu/mshyperv.c
@@ -249,11 +249,6 @@ static uint32_t  __init ms_hyperv_platform(void)
 	return HYPERV_CPUID_VENDOR_AND_MAX_FUNCTIONS;
 }
 
-static unsigned char hv_get_nmi_reason(void)
-{
-	return 0;
-}
-
 #ifdef CONFIG_X86_LOCAL_APIC
 /*
  * Prior to WS2016 Debug-VM sends NMIs to all CPUs which makes
-- 
2.34.1

[PATCH v4 4/5] Drivers: hv: Kconfig: Add HYPERV_VTL_MODE

[Saurabh Sengar]

Add HYPERV_VTL_MODE Kconfig flag for VTL mode.

Signed-off-by: Saurabh Sengar <ssengar@linux.microsoft.com>
---
 drivers/hv/Kconfig | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/drivers/hv/Kconfig b/drivers/hv/Kconfig
index 0747a8f1fcee..511f2e012c59 100644
--- a/drivers/hv/Kconfig
+++ b/drivers/hv/Kconfig
@@ -13,6 +13,30 @@ config HYPERV
 	  Select this option to run Linux as a Hyper-V client operating
 	  system.
 
+config HYPERV_VTL_MODE
+	bool "Enable Linux to boot in VTL context"
+	depends on X86_64 && HYPERV
+	default n
+	help
+	  Virtual Secure Mode (VSM) is a set of hypervisor capabilities and
+	  enlightenments offered to host and guest partitions which enables
+	  the creation and management of new security boundaries within
+	  operating system software.
+
+	  VSM achieves and maintains isolation through Virtual Trust Levels
+	  (VTLs). Virtual Trust Levels are hierarchical, with higher levels
+	  being more privileged than lower levels. VTL0 is the least privileged
+	  level, and currently only other level supported is VTL2.
+
+	  Select this option to build a Linux kernel to run at a VTL other than
+	  the normal VTL0, which currently is only VTL2.  This option
+	  initializes the x86 platform for VTL2, and adds the ability to boot
+	  secondary CPUs directly into 64-bit context as required for VTLs other
+	  than 0.  A kernel built with this option must run at VTL2, and will
+	  not run as a normal guest.
+
+	  If unsure, say N
+
 config HYPERV_TIMER
 	def_bool HYPERV && X86
 
-- 
2.34.1

[PATCH v4 5/5] x86/hyperv: VTL support for Hyper-V

[Saurabh Sengar]

Virtual Trust Levels (VTL) helps enable Hyper-V Virtual Secure Mode (VSM)
feature. VSM is a set of hypervisor capabilities and enlightenments
offered to host and guest partitions which enable the creation and
management of new security boundaries within operating system software.
VSM achieves and maintains isolation through VTLs.

Add early initialization for Virtual Trust Levels (VTL). This includes
initializing the x86 platform for VTL and enabling boot support for
secondary CPUs to start in targeted VTL context. For now, only enable
the code for targeted VTL level as 2.

When starting an AP at a VTL other than VTL0, the AP must start directly
in 64-bit mode, bypassing the usual 16-bit -> 32-bit -> 64-bit mode
transition sequence that occurs after waking up an AP with SIPI whose
vector points to the 16-bit AP startup trampoline code.

Signed-off-by: Saurabh Sengar <ssengar@linux.microsoft.com>
---
[V4]
- replace initial_stack with current->thread.sp as per recent upstream changes

 arch/x86/hyperv/Makefile        |   1 +
 arch/x86/hyperv/hv_vtl.c        | 227 ++++++++++++++++++++++++++++++++
 arch/x86/include/asm/mshyperv.h |  10 ++
 arch/x86/kernel/cpu/mshyperv.c  |   1 +
 4 files changed, 239 insertions(+)
 create mode 100644 arch/x86/hyperv/hv_vtl.c

diff --git a/arch/x86/hyperv/Makefile b/arch/x86/hyperv/Makefile
index 5d2de10809ae..3a1548054b48 100644
--- a/arch/x86/hyperv/Makefile
+++ b/arch/x86/hyperv/Makefile
@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 obj-y			:= hv_init.o mmu.o nested.o irqdomain.o ivm.o
 obj-$(CONFIG_X86_64)	+= hv_apic.o hv_proc.o
+obj-$(CONFIG_HYPERV_VTL_MODE)	+= hv_vtl.o
 
 ifdef CONFIG_X86_64
 obj-$(CONFIG_PARAVIRT_SPINLOCKS)	+= hv_spinlock.o
diff --git a/arch/x86/hyperv/hv_vtl.c b/arch/x86/hyperv/hv_vtl.c
new file mode 100644
index 000000000000..1ba5d3b99b16
--- /dev/null
+++ b/arch/x86/hyperv/hv_vtl.c
@@ -0,0 +1,227 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2023, Microsoft Corporation.
+ *
+ * Author:
+ *   Saurabh Sengar <ssengar@microsoft.com>
+ */
+
+#include <asm/apic.h>
+#include <asm/boot.h>
+#include <asm/desc.h>
+#include <asm/i8259.h>
+#include <asm/mshyperv.h>
+#include <asm/realmode.h>
+
+extern struct boot_params boot_params;
+static struct real_mode_header hv_vtl_real_mode_header;
+
+void __init hv_vtl_init_platform(void)
+{
+	pr_info("Linux runs in Hyper-V Virtual Trust Level\n");
+
+	x86_init.irqs.pre_vector_init = x86_init_noop;
+	x86_init.timers.timer_init = x86_init_noop;
+
+	x86_platform.get_wallclock = get_rtc_noop;
+	x86_platform.set_wallclock = set_rtc_noop;
+	x86_platform.get_nmi_reason = hv_get_nmi_reason;
+
+	x86_platform.legacy.i8042 = X86_LEGACY_I8042_PLATFORM_ABSENT;
+	x86_platform.legacy.rtc = 0;
+	x86_platform.legacy.warm_reset = 0;
+	x86_platform.legacy.reserve_bios_regions = 0;
+	x86_platform.legacy.devices.pnpbios = 0;
+}
+
+static inline u64 hv_vtl_system_desc_base(struct ldttss_desc *desc)
+{
+	return ((u64)desc->base3 << 32) | ((u64)desc->base2 << 24) |
+		(desc->base1 << 16) | desc->base0;
+}
+
+static inline u32 hv_vtl_system_desc_limit(struct ldttss_desc *desc)
+{
+	return ((u32)desc->limit1 << 16) | (u32)desc->limit0;
+}
+
+typedef void (*secondary_startup_64_fn)(void*, void*);
+static void hv_vtl_ap_entry(void)
+{
+	((secondary_startup_64_fn)secondary_startup_64)(&boot_params, &boot_params);
+}
+
+static int hv_vtl_bringup_vcpu(u32 target_vp_index, u64 eip_ignored)
+{
+	u64 status;
+	int ret = 0;
+	struct hv_enable_vp_vtl *input;
+	unsigned long irq_flags;
+
+	struct desc_ptr gdt_ptr;
+	struct desc_ptr idt_ptr;
+
+	struct ldttss_desc *tss;
+	struct ldttss_desc *ldt;
+	struct desc_struct *gdt;
+
+	u64 rsp = current->thread.sp;
+	u64 rip = (u64)&hv_vtl_ap_entry;
+
+	native_store_gdt(&gdt_ptr);
+	store_idt(&idt_ptr);
+
+	gdt = (struct desc_struct *)((void *)(gdt_ptr.address));
+	tss = (struct ldttss_desc *)(gdt + GDT_ENTRY_TSS);
+	ldt = (struct ldttss_desc *)(gdt + GDT_ENTRY_LDT);
+
+	local_irq_save(irq_flags);
+
+	input = *this_cpu_ptr(hyperv_pcpu_input_arg);
+	memset(input, 0, sizeof(*input));
+
+	input->partition_id = HV_PARTITION_ID_SELF;
+	input->vp_index = target_vp_index;
+	input->target_vtl.target_vtl = HV_VTL_MGMT;
+
+	/*
+	 * The x86_64 Linux kernel follows the 16-bit -> 32-bit -> 64-bit
+	 * mode transition sequence after waking up an AP with SIPI whose
+	 * vector points to the 16-bit AP startup trampoline code. Here in
+	 * VTL2, we can't perform that sequence as the AP has to start in
+	 * the 64-bit mode.
+	 *
+	 * To make this happen, we tell the hypervisor to load a valid 64-bit
+	 * context (most of which is just magic numbers from the CPU manual)
+	 * so that AP jumps right to the 64-bit entry of the kernel, and the
+	 * control registers are loaded with values that let the AP fetch the
+	 * code and data and carry on with work it gets assigned.
+	 */
+
+	input->vp_context.rip = rip;
+	input->vp_context.rsp = rsp;
+	input->vp_context.rflags = 0x0000000000000002;
+	input->vp_context.efer = __rdmsr(MSR_EFER);
+	input->vp_context.cr0 = native_read_cr0();
+	input->vp_context.cr3 = __native_read_cr3();
+	input->vp_context.cr4 = native_read_cr4();
+	input->vp_context.msr_cr_pat = __rdmsr(MSR_IA32_CR_PAT);
+	input->vp_context.idtr.limit = idt_ptr.size;
+	input->vp_context.idtr.base = idt_ptr.address;
+	input->vp_context.gdtr.limit = gdt_ptr.size;
+	input->vp_context.gdtr.base = gdt_ptr.address;
+
+	/* Non-system desc (64bit), long, code, present */
+	input->vp_context.cs.selector = __KERNEL_CS;
+	input->vp_context.cs.base = 0;
+	input->vp_context.cs.limit = 0xffffffff;
+	input->vp_context.cs.attributes = 0xa09b;
+	/* Non-system desc (64bit), data, present, granularity, default */
+	input->vp_context.ss.selector = __KERNEL_DS;
+	input->vp_context.ss.base = 0;
+	input->vp_context.ss.limit = 0xffffffff;
+	input->vp_context.ss.attributes = 0xc093;
+
+	/* System desc (128bit), present, LDT */
+	input->vp_context.ldtr.selector = GDT_ENTRY_LDT * 8;
+	input->vp_context.ldtr.base = hv_vtl_system_desc_base(ldt);
+	input->vp_context.ldtr.limit = hv_vtl_system_desc_limit(ldt);
+	input->vp_context.ldtr.attributes = 0x82;
+
+	/* System desc (128bit), present, TSS, 0x8b - busy, 0x89 -- default */
+	input->vp_context.tr.selector = GDT_ENTRY_TSS * 8;
+	input->vp_context.tr.base = hv_vtl_system_desc_base(tss);
+	input->vp_context.tr.limit = hv_vtl_system_desc_limit(tss);
+	input->vp_context.tr.attributes = 0x8b;
+
+	status = hv_do_hypercall(HVCALL_ENABLE_VP_VTL, input, NULL);
+
+	if (!hv_result_success(status) &&
+	    hv_result(status) != HV_STATUS_VTL_ALREADY_ENABLED) {
+		pr_err("HVCALL_ENABLE_VP_VTL failed for VP : %d ! [Err: %#llx\n]",
+		       target_vp_index, status);
+		ret = -EINVAL;
+		goto free_lock;
+	}
+
+	status = hv_do_hypercall(HVCALL_START_VP, input, NULL);
+
+	if (!hv_result_success(status)) {
+		pr_err("HVCALL_START_VP failed for VP : %d ! [Err: %#llx]\n",
+		       target_vp_index, status);
+		ret = -EINVAL;
+	}
+
+free_lock:
+	local_irq_restore(irq_flags);
+
+	return ret;
+}
+
+static int hv_vtl_apicid_to_vp_id(u32 apic_id)
+{
+	u64 control;
+	u64 status;
+	unsigned long irq_flags;
+	struct hv_get_vp_from_apic_id_in *input;
+	u32 *output, ret;
+
+	local_irq_save(irq_flags);
+
+	input = *this_cpu_ptr(hyperv_pcpu_input_arg);
+	memset(input, 0, sizeof(*input));
+	input->partition_id = HV_PARTITION_ID_SELF;
+	input->apic_ids[0] = apic_id;
+
+	output = (u32 *)input;
+
+	control = HV_HYPERCALL_REP_COMP_1 | HVCALL_GET_VP_ID_FROM_APIC_ID;
+	status = hv_do_hypercall(control, input, output);
+	ret = output[0];
+
+	local_irq_restore(irq_flags);
+
+	if (!hv_result_success(status)) {
+		pr_err("failed to get vp id from apic id %d, status %#llx\n",
+		       apic_id, status);
+		return -EINVAL;
+	}
+
+	return ret;
+}
+
+static int hv_vtl_wakeup_secondary_cpu(int apicid, unsigned long start_eip)
+{
+	int vp_id;
+
+	pr_debug("Bringing up CPU with APIC ID %d in VTL2...\n", apicid);
+	vp_id = hv_vtl_apicid_to_vp_id(apicid);
+
+	if (vp_id < 0) {
+		pr_err("Couldn't find CPU with APIC ID %d\n", apicid);
+		return -EINVAL;
+	}
+	if (vp_id > ms_hyperv.max_vp_index) {
+		pr_err("Invalid CPU id %d for APIC ID %d\n", vp_id, apicid);
+		return -EINVAL;
+	}
+
+	return hv_vtl_bringup_vcpu(vp_id, start_eip);
+}
+
+static int __init hv_vtl_early_init(void)
+{
+	/*
+	 * `boot_cpu_has` returns the runtime feature support,
+	 * and here is the earliest it can be used.
+	 */
+	if (cpu_feature_enabled(X86_FEATURE_XSAVE))
+		panic("XSAVE has to be disabled as it is not supported by this module.\n"
+			  "Please add 'noxsave' to the kernel command line.\n");
+
+	real_mode_header = &hv_vtl_real_mode_header;
+	apic->wakeup_secondary_cpu_64 = hv_vtl_wakeup_secondary_cpu;
+
+	return 0;
+}
+early_initcall(hv_vtl_early_init);
diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h
index 71ed240ef66d..de4ad38f7d74 100644
--- a/arch/x86/include/asm/mshyperv.h
+++ b/arch/x86/include/asm/mshyperv.h
@@ -19,6 +19,10 @@
  */
 #define HV_IOAPIC_BASE_ADDRESS 0xfec00000
 
+#define HV_VTL_NORMAL 0x0
+#define HV_VTL_SECURE 0x1
+#define HV_VTL_MGMT   0x2
+
 union hv_ghcb;
 
 DECLARE_STATIC_KEY_FALSE(isolation_type_snp);
@@ -276,6 +280,12 @@ static inline u64 hv_get_non_nested_register(unsigned int reg) { return 0; }
 #endif /* CONFIG_HYPERV */
 
 
+#ifdef CONFIG_HYPERV_VTL_MODE
+void __init hv_vtl_init_platform(void);
+#else
+static inline void __init hv_vtl_init_platform(void) {}
+#endif
+
 #include <asm-generic/mshyperv.h>
 
 #endif
diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c
index 5ee02af57dac..5180e3c50184 100644
--- a/arch/x86/kernel/cpu/mshyperv.c
+++ b/arch/x86/kernel/cpu/mshyperv.c
@@ -519,6 +519,7 @@ static void __init ms_hyperv_init_platform(void)
 
 	/* Register Hyper-V specific clocksource */
 	hv_init_clocksource();
+	hv_vtl_init_platform();
 #endif
 	/*
 	 * TSC should be marked as unstable only after Hyper-V
-- 
2.34.1

Re: [PATCH v4 1/5] x86/init: Make get/set_rtc_noop() public

[Stanislav Kinsburskii]

On Tue, Apr 04, 2023 at 02:01:00AM -0700, Saurabh Sengar wrote:
> Make get/set_rtc_noop() to be public so that they can be used
> in other modules as well.
> 
> Co-developed-by: Tianyu Lan <tiala@microsoft.com>
> Signed-off-by: Tianyu Lan <tiala@microsoft.com>
> Signed-off-by: Saurabh Sengar <ssengar@linux.microsoft.com>
> Reviewed-by: Wei Liu <wei.liu@kernel.org>
> Reviewed-by: Michael Kelley <mikelley@microsoft.com>
> ---
>  arch/x86/include/asm/x86_init.h | 2 ++
>  arch/x86/kernel/x86_init.c      | 4 ++--
>  2 files changed, 4 insertions(+), 2 deletions(-)
> 
> diff --git a/arch/x86/include/asm/x86_init.h b/arch/x86/include/asm/x86_init.h
> index acc20ae4079d..88085f369ff6 100644
> --- a/arch/x86/include/asm/x86_init.h
> +++ b/arch/x86/include/asm/x86_init.h
> @@ -330,5 +330,7 @@ extern void x86_init_uint_noop(unsigned int unused);
>  extern bool bool_x86_init_noop(void);
>  extern void x86_op_int_noop(int cpu);
>  extern bool x86_pnpbios_disabled(void);
> +extern int set_rtc_noop(const struct timespec64 *now);
> +extern void get_rtc_noop(struct timespec64 *now);
>  
>  #endif
> diff --git a/arch/x86/kernel/x86_init.c b/arch/x86/kernel/x86_init.c
> index 95be3831df73..d82f4fa2f1bf 100644
> --- a/arch/x86/kernel/x86_init.c
> +++ b/arch/x86/kernel/x86_init.c
> @@ -33,8 +33,8 @@ static int __init iommu_init_noop(void) { return 0; }
>  static void iommu_shutdown_noop(void) { }
>  bool __init bool_x86_init_noop(void) { return false; }
>  void x86_op_int_noop(int cpu) { }
> -static __init int set_rtc_noop(const struct timespec64 *now) { return -EINVAL; }
> -static __init void get_rtc_noop(struct timespec64 *now) { }
> +int set_rtc_noop(const struct timespec64 *now) { return -EINVAL; }
> +void get_rtc_noop(struct timespec64 *now) { }
>  

These functions are just empty place holders.
What is the value in having then exported at all?

Also, if you do want to use exactly there functions, a better option
would be to move them to the header file as "static inline" ones.

>  static __initconst const struct of_device_id of_cmos_match[] = {
>  	{ .compatible = "motorola,mc146818" },
> -- 
> 2.34.1

Re: [PATCH v4 2/5] x86/hyperv: Add VTL specific structs and hypercalls

[Stanislav Kinsburskii]

On Tue, Apr 04, 2023 at 02:01:01AM -0700, Saurabh Sengar wrote:
> Add structs and hypercalls required to enable VTL support on x86.
> 
> Signed-off-by: Saurabh Sengar <ssengar@linux.microsoft.com>
> Reviewed-by: Michael Kelley <mikelley@microsoft.com>
> ---
>  arch/x86/include/asm/hyperv-tlfs.h | 75 ++++++++++++++++++++++++++++++
>  include/asm-generic/hyperv-tlfs.h  |  4 ++
>  2 files changed, 79 insertions(+)
> 
> diff --git a/arch/x86/include/asm/hyperv-tlfs.h b/arch/x86/include/asm/hyperv-tlfs.h
> index 0b73a809e9e1..0b0b4e9a4318 100644
> --- a/arch/x86/include/asm/hyperv-tlfs.h
> +++ b/arch/x86/include/asm/hyperv-tlfs.h
> @@ -713,6 +713,81 @@ union hv_msi_entry {
>  	} __packed;
>  };
>  
> +struct hv_x64_segment_register {
> +	__u64 base;

Ideally they arch-size types naming should be consistent: either with underscores or
without.
The majority of cases in this file are without underscores.

Reviewed-by: Stanislav Kinsburskii <stanislav.kinsburskii@gmail.com>

> +	__u32 limit;
> +	__u16 selector;
> +	union {
> +		struct {
> +			__u16 segment_type : 4;
> +			__u16 non_system_segment : 1;
> +			__u16 descriptor_privilege_level : 2;
> +			__u16 present : 1;
> +			__u16 reserved : 4;
> +			__u16 available : 1;
> +			__u16 _long : 1;
> +			__u16 _default : 1;
> +			__u16 granularity : 1;
> +		} __packed;
> +		__u16 attributes;
> +	};
> +} __packed;
> +
> +struct hv_x64_table_register {
> +	__u16 pad[3];
> +	__u16 limit;
> +	__u64 base;
> +} __packed;
> +
> +struct hv_init_vp_context {
> +	u64 rip;
> +	u64 rsp;
> +	u64 rflags;
> +
> +	struct hv_x64_segment_register cs;
> +	struct hv_x64_segment_register ds;
> +	struct hv_x64_segment_register es;
> +	struct hv_x64_segment_register fs;
> +	struct hv_x64_segment_register gs;
> +	struct hv_x64_segment_register ss;
> +	struct hv_x64_segment_register tr;
> +	struct hv_x64_segment_register ldtr;
> +
> +	struct hv_x64_table_register idtr;
> +	struct hv_x64_table_register gdtr;
> +
> +	u64 efer;
> +	u64 cr0;
> +	u64 cr3;
> +	u64 cr4;
> +	u64 msr_cr_pat;
> +} __packed;
> +
> +union hv_input_vtl {
> +	u8 as_uint8;
> +	struct {
> +		u8 target_vtl: 4;
> +		u8 use_target_vtl: 1;
> +		u8 reserved_z: 3;
> +	};
> +} __packed;
> +
> +struct hv_enable_vp_vtl {
> +	u64				partition_id;
> +	u32				vp_index;
> +	union hv_input_vtl		target_vtl;
> +	u8				mbz0;
> +	u16				mbz1;
> +	struct hv_init_vp_context	vp_context;
> +} __packed;
> +
> +struct hv_get_vp_from_apic_id_in {
> +	u64 partition_id;
> +	union hv_input_vtl target_vtl;
> +	u8 res[7];
> +	u32 apic_ids[];
> +} __packed;
> +
>  #include <asm-generic/hyperv-tlfs.h>
>  
>  #endif
> diff --git a/include/asm-generic/hyperv-tlfs.h b/include/asm-generic/hyperv-tlfs.h
> index b870983596b9..87258341fd7c 100644
> --- a/include/asm-generic/hyperv-tlfs.h
> +++ b/include/asm-generic/hyperv-tlfs.h
> @@ -146,6 +146,7 @@ union hv_reference_tsc_msr {
>  /* Declare the various hypercall operations. */
>  #define HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE	0x0002
>  #define HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST	0x0003
> +#define HVCALL_ENABLE_VP_VTL			0x000f
>  #define HVCALL_NOTIFY_LONG_SPIN_WAIT		0x0008
>  #define HVCALL_SEND_IPI				0x000b
>  #define HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX	0x0013
> @@ -165,6 +166,8 @@ union hv_reference_tsc_msr {
>  #define HVCALL_MAP_DEVICE_INTERRUPT		0x007c
>  #define HVCALL_UNMAP_DEVICE_INTERRUPT		0x007d
>  #define HVCALL_RETARGET_INTERRUPT		0x007e
> +#define HVCALL_START_VP				0x0099
> +#define HVCALL_GET_VP_ID_FROM_APIC_ID		0x009a
>  #define HVCALL_FLUSH_GUEST_PHYSICAL_ADDRESS_SPACE 0x00af
>  #define HVCALL_FLUSH_GUEST_PHYSICAL_ADDRESS_LIST 0x00b0
>  #define HVCALL_MODIFY_SPARSE_GPA_PAGE_HOST_VISIBILITY 0x00db
> @@ -218,6 +221,7 @@ enum HV_GENERIC_SET_FORMAT {
>  #define HV_STATUS_INVALID_PORT_ID		17
>  #define HV_STATUS_INVALID_CONNECTION_ID		18
>  #define HV_STATUS_INSUFFICIENT_BUFFERS		19
> +#define HV_STATUS_VTL_ALREADY_ENABLED		134
>  
>  /*
>   * The Hyper-V TimeRefCount register and the TSC
> -- 
> 2.34.1

Re: [PATCH v4 5/5] x86/hyperv: VTL support for Hyper-V

[Stanislav Kinsburskii]

On Tue, Apr 04, 2023 at 02:01:04AM -0700, Saurabh Sengar wrote:
> Virtual Trust Levels (VTL) helps enable Hyper-V Virtual Secure Mode (VSM)
> feature. VSM is a set of hypervisor capabilities and enlightenments
> offered to host and guest partitions which enable the creation and
> management of new security boundaries within operating system software.
> VSM achieves and maintains isolation through VTLs.
> 
> Add early initialization for Virtual Trust Levels (VTL). This includes
> initializing the x86 platform for VTL and enabling boot support for
> secondary CPUs to start in targeted VTL context. For now, only enable
> the code for targeted VTL level as 2.
> 
> When starting an AP at a VTL other than VTL0, the AP must start directly
> in 64-bit mode, bypassing the usual 16-bit -> 32-bit -> 64-bit mode
> transition sequence that occurs after waking up an AP with SIPI whose
> vector points to the 16-bit AP startup trampoline code.
> 
> Signed-off-by: Saurabh Sengar <ssengar@linux.microsoft.com>
> ---
> [V4]
> - replace initial_stack with current->thread.sp as per recent upstream changes
> 
>  arch/x86/hyperv/Makefile        |   1 +
>  arch/x86/hyperv/hv_vtl.c        | 227 ++++++++++++++++++++++++++++++++
>  arch/x86/include/asm/mshyperv.h |  10 ++
>  arch/x86/kernel/cpu/mshyperv.c  |   1 +
>  4 files changed, 239 insertions(+)
>  create mode 100644 arch/x86/hyperv/hv_vtl.c
> 
> diff --git a/arch/x86/hyperv/Makefile b/arch/x86/hyperv/Makefile
> index 5d2de10809ae..3a1548054b48 100644
> --- a/arch/x86/hyperv/Makefile
> +++ b/arch/x86/hyperv/Makefile
> @@ -1,6 +1,7 @@
>  # SPDX-License-Identifier: GPL-2.0-only
>  obj-y			:= hv_init.o mmu.o nested.o irqdomain.o ivm.o
>  obj-$(CONFIG_X86_64)	+= hv_apic.o hv_proc.o
> +obj-$(CONFIG_HYPERV_VTL_MODE)	+= hv_vtl.o
>  
>  ifdef CONFIG_X86_64
>  obj-$(CONFIG_PARAVIRT_SPINLOCKS)	+= hv_spinlock.o
> diff --git a/arch/x86/hyperv/hv_vtl.c b/arch/x86/hyperv/hv_vtl.c
> new file mode 100644
> index 000000000000..1ba5d3b99b16
> --- /dev/null
> +++ b/arch/x86/hyperv/hv_vtl.c
> @@ -0,0 +1,227 @@
> +// SPDX-License-Identifier: GPL-2.0
> +/*
> + * Copyright (c) 2023, Microsoft Corporation.
> + *
> + * Author:
> + *   Saurabh Sengar <ssengar@microsoft.com>
> + */
> +
> +#include <asm/apic.h>
> +#include <asm/boot.h>
> +#include <asm/desc.h>
> +#include <asm/i8259.h>
> +#include <asm/mshyperv.h>
> +#include <asm/realmode.h>
> +
> +extern struct boot_params boot_params;
> +static struct real_mode_header hv_vtl_real_mode_header;
> +
> +void __init hv_vtl_init_platform(void)
> +{
> +	pr_info("Linux runs in Hyper-V Virtual Trust Level\n");
> +
> +	x86_init.irqs.pre_vector_init = x86_init_noop;
> +	x86_init.timers.timer_init = x86_init_noop;
> +
> +	x86_platform.get_wallclock = get_rtc_noop;
> +	x86_platform.set_wallclock = set_rtc_noop;

Nit: this code is VTL feature and hypevisor specific.
Defining vtl_get_rtc_noop instead of exporting get_rtc_noop would allow to make
this series less intrusive to the rest of x86 generic code.

Reviewed-by: Stanislav Kinsburskii <stanislav.kinsburskii@gmail.com>

> +	x86_platform.get_nmi_reason = hv_get_nmi_reason;
> +
> +	x86_platform.legacy.i8042 = X86_LEGACY_I8042_PLATFORM_ABSENT;
> +	x86_platform.legacy.rtc = 0;
> +	x86_platform.legacy.warm_reset = 0;
> +	x86_platform.legacy.reserve_bios_regions = 0;
> +	x86_platform.legacy.devices.pnpbios = 0;
> +}
> +
> +static inline u64 hv_vtl_system_desc_base(struct ldttss_desc *desc)
> +{
> +	return ((u64)desc->base3 << 32) | ((u64)desc->base2 << 24) |
> +		(desc->base1 << 16) | desc->base0;
> +}
> +
> +static inline u32 hv_vtl_system_desc_limit(struct ldttss_desc *desc)
> +{
> +	return ((u32)desc->limit1 << 16) | (u32)desc->limit0;
> +}
> +
> +typedef void (*secondary_startup_64_fn)(void*, void*);
> +static void hv_vtl_ap_entry(void)
> +{
> +	((secondary_startup_64_fn)secondary_startup_64)(&boot_params, &boot_params);
> +}
> +
> +static int hv_vtl_bringup_vcpu(u32 target_vp_index, u64 eip_ignored)
> +{
> +	u64 status;
> +	int ret = 0;
> +	struct hv_enable_vp_vtl *input;
> +	unsigned long irq_flags;
> +
> +	struct desc_ptr gdt_ptr;
> +	struct desc_ptr idt_ptr;
> +
> +	struct ldttss_desc *tss;
> +	struct ldttss_desc *ldt;
> +	struct desc_struct *gdt;
> +
> +	u64 rsp = current->thread.sp;
> +	u64 rip = (u64)&hv_vtl_ap_entry;
> +
> +	native_store_gdt(&gdt_ptr);
> +	store_idt(&idt_ptr);
> +
> +	gdt = (struct desc_struct *)((void *)(gdt_ptr.address));
> +	tss = (struct ldttss_desc *)(gdt + GDT_ENTRY_TSS);
> +	ldt = (struct ldttss_desc *)(gdt + GDT_ENTRY_LDT);
> +
> +	local_irq_save(irq_flags);
> +
> +	input = *this_cpu_ptr(hyperv_pcpu_input_arg);
> +	memset(input, 0, sizeof(*input));
> +
> +	input->partition_id = HV_PARTITION_ID_SELF;
> +	input->vp_index = target_vp_index;
> +	input->target_vtl.target_vtl = HV_VTL_MGMT;
> +
> +	/*
> +	 * The x86_64 Linux kernel follows the 16-bit -> 32-bit -> 64-bit
> +	 * mode transition sequence after waking up an AP with SIPI whose
> +	 * vector points to the 16-bit AP startup trampoline code. Here in
> +	 * VTL2, we can't perform that sequence as the AP has to start in
> +	 * the 64-bit mode.
> +	 *
> +	 * To make this happen, we tell the hypervisor to load a valid 64-bit
> +	 * context (most of which is just magic numbers from the CPU manual)
> +	 * so that AP jumps right to the 64-bit entry of the kernel, and the
> +	 * control registers are loaded with values that let the AP fetch the
> +	 * code and data and carry on with work it gets assigned.
> +	 */
> +
> +	input->vp_context.rip = rip;
> +	input->vp_context.rsp = rsp;
> +	input->vp_context.rflags = 0x0000000000000002;
> +	input->vp_context.efer = __rdmsr(MSR_EFER);
> +	input->vp_context.cr0 = native_read_cr0();
> +	input->vp_context.cr3 = __native_read_cr3();
> +	input->vp_context.cr4 = native_read_cr4();
> +	input->vp_context.msr_cr_pat = __rdmsr(MSR_IA32_CR_PAT);
> +	input->vp_context.idtr.limit = idt_ptr.size;
> +	input->vp_context.idtr.base = idt_ptr.address;
> +	input->vp_context.gdtr.limit = gdt_ptr.size;
> +	input->vp_context.gdtr.base = gdt_ptr.address;
> +
> +	/* Non-system desc (64bit), long, code, present */
> +	input->vp_context.cs.selector = __KERNEL_CS;
> +	input->vp_context.cs.base = 0;
> +	input->vp_context.cs.limit = 0xffffffff;
> +	input->vp_context.cs.attributes = 0xa09b;
> +	/* Non-system desc (64bit), data, present, granularity, default */
> +	input->vp_context.ss.selector = __KERNEL_DS;
> +	input->vp_context.ss.base = 0;
> +	input->vp_context.ss.limit = 0xffffffff;
> +	input->vp_context.ss.attributes = 0xc093;
> +
> +	/* System desc (128bit), present, LDT */
> +	input->vp_context.ldtr.selector = GDT_ENTRY_LDT * 8;
> +	input->vp_context.ldtr.base = hv_vtl_system_desc_base(ldt);
> +	input->vp_context.ldtr.limit = hv_vtl_system_desc_limit(ldt);
> +	input->vp_context.ldtr.attributes = 0x82;
> +
> +	/* System desc (128bit), present, TSS, 0x8b - busy, 0x89 -- default */
> +	input->vp_context.tr.selector = GDT_ENTRY_TSS * 8;
> +	input->vp_context.tr.base = hv_vtl_system_desc_base(tss);
> +	input->vp_context.tr.limit = hv_vtl_system_desc_limit(tss);
> +	input->vp_context.tr.attributes = 0x8b;
> +
> +	status = hv_do_hypercall(HVCALL_ENABLE_VP_VTL, input, NULL);
> +
> +	if (!hv_result_success(status) &&
> +	    hv_result(status) != HV_STATUS_VTL_ALREADY_ENABLED) {
> +		pr_err("HVCALL_ENABLE_VP_VTL failed for VP : %d ! [Err: %#llx\n]",
> +		       target_vp_index, status);
> +		ret = -EINVAL;
> +		goto free_lock;
> +	}
> +
> +	status = hv_do_hypercall(HVCALL_START_VP, input, NULL);
> +
> +	if (!hv_result_success(status)) {
> +		pr_err("HVCALL_START_VP failed for VP : %d ! [Err: %#llx]\n",
> +		       target_vp_index, status);
> +		ret = -EINVAL;
> +	}
> +
> +free_lock:
> +	local_irq_restore(irq_flags);
> +
> +	return ret;
> +}
> +
> +static int hv_vtl_apicid_to_vp_id(u32 apic_id)
> +{
> +	u64 control;
> +	u64 status;
> +	unsigned long irq_flags;
> +	struct hv_get_vp_from_apic_id_in *input;
> +	u32 *output, ret;
> +
> +	local_irq_save(irq_flags);
> +
> +	input = *this_cpu_ptr(hyperv_pcpu_input_arg);
> +	memset(input, 0, sizeof(*input));
> +	input->partition_id = HV_PARTITION_ID_SELF;
> +	input->apic_ids[0] = apic_id;
> +
> +	output = (u32 *)input;
> +
> +	control = HV_HYPERCALL_REP_COMP_1 | HVCALL_GET_VP_ID_FROM_APIC_ID;
> +	status = hv_do_hypercall(control, input, output);
> +	ret = output[0];
> +
> +	local_irq_restore(irq_flags);
> +
> +	if (!hv_result_success(status)) {
> +		pr_err("failed to get vp id from apic id %d, status %#llx\n",
> +		       apic_id, status);
> +		return -EINVAL;
> +	}
> +
> +	return ret;
> +}
> +
> +static int hv_vtl_wakeup_secondary_cpu(int apicid, unsigned long start_eip)
> +{
> +	int vp_id;
> +
> +	pr_debug("Bringing up CPU with APIC ID %d in VTL2...\n", apicid);
> +	vp_id = hv_vtl_apicid_to_vp_id(apicid);
> +
> +	if (vp_id < 0) {
> +		pr_err("Couldn't find CPU with APIC ID %d\n", apicid);
> +		return -EINVAL;
> +	}
> +	if (vp_id > ms_hyperv.max_vp_index) {
> +		pr_err("Invalid CPU id %d for APIC ID %d\n", vp_id, apicid);
> +		return -EINVAL;
> +	}
> +
> +	return hv_vtl_bringup_vcpu(vp_id, start_eip);
> +}
> +
> +static int __init hv_vtl_early_init(void)
> +{
> +	/*
> +	 * `boot_cpu_has` returns the runtime feature support,
> +	 * and here is the earliest it can be used.
> +	 */
> +	if (cpu_feature_enabled(X86_FEATURE_XSAVE))
> +		panic("XSAVE has to be disabled as it is not supported by this module.\n"
> +			  "Please add 'noxsave' to the kernel command line.\n");
> +
> +	real_mode_header = &hv_vtl_real_mode_header;
> +	apic->wakeup_secondary_cpu_64 = hv_vtl_wakeup_secondary_cpu;
> +
> +	return 0;
> +}
> +early_initcall(hv_vtl_early_init);
> diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h
> index 71ed240ef66d..de4ad38f7d74 100644
> --- a/arch/x86/include/asm/mshyperv.h
> +++ b/arch/x86/include/asm/mshyperv.h
> @@ -19,6 +19,10 @@
>   */
>  #define HV_IOAPIC_BASE_ADDRESS 0xfec00000
>  
> +#define HV_VTL_NORMAL 0x0
> +#define HV_VTL_SECURE 0x1
> +#define HV_VTL_MGMT   0x2
> +
>  union hv_ghcb;
>  
>  DECLARE_STATIC_KEY_FALSE(isolation_type_snp);
> @@ -276,6 +280,12 @@ static inline u64 hv_get_non_nested_register(unsigned int reg) { return 0; }
>  #endif /* CONFIG_HYPERV */
>  
>  
> +#ifdef CONFIG_HYPERV_VTL_MODE
> +void __init hv_vtl_init_platform(void);
> +#else
> +static inline void __init hv_vtl_init_platform(void) {}
> +#endif
> +
>  #include <asm-generic/mshyperv.h>
>  
>  #endif
> diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c
> index 5ee02af57dac..5180e3c50184 100644
> --- a/arch/x86/kernel/cpu/mshyperv.c
> +++ b/arch/x86/kernel/cpu/mshyperv.c
> @@ -519,6 +519,7 @@ static void __init ms_hyperv_init_platform(void)
>  
>  	/* Register Hyper-V specific clocksource */
>  	hv_init_clocksource();
> +	hv_vtl_init_platform();
>  #endif
>  	/*
>  	 * TSC should be marked as unstable only after Hyper-V
> -- 
> 2.34.1

RE: [PATCH v4 4/5] Drivers: hv: Kconfig: Add HYPERV_VTL_MODE

[Michael Kelley (LINUX)]

From: Saurabh Sengar <ssengar@linux.microsoft.com> Sent: Tuesday, April 4, 2023 2:01 AM
> 
> Add HYPERV_VTL_MODE Kconfig flag for VTL mode.
> 
> Signed-off-by: Saurabh Sengar <ssengar@linux.microsoft.com>
> ---
>  drivers/hv/Kconfig | 24 ++++++++++++++++++++++++
>  1 file changed, 24 insertions(+)
> 
> diff --git a/drivers/hv/Kconfig b/drivers/hv/Kconfig
> index 0747a8f1fcee..511f2e012c59 100644
> --- a/drivers/hv/Kconfig
> +++ b/drivers/hv/Kconfig
> @@ -13,6 +13,30 @@ config HYPERV
>  	  Select this option to run Linux as a Hyper-V client operating
>  	  system.
> 
> +config HYPERV_VTL_MODE
> +	bool "Enable Linux to boot in VTL context"
> +	depends on X86_64 && HYPERV
> +	default n
> +	help
> +	  Virtual Secure Mode (VSM) is a set of hypervisor capabilities and
> +	  enlightenments offered to host and guest partitions which enables
> +	  the creation and management of new security boundaries within
> +	  operating system software.
> +
> +	  VSM achieves and maintains isolation through Virtual Trust Levels
> +	  (VTLs). Virtual Trust Levels are hierarchical, with higher levels
> +	  being more privileged than lower levels. VTL0 is the least privileged
> +	  level, and currently only other level supported is VTL2.
> +
> +	  Select this option to build a Linux kernel to run at a VTL other than
> +	  the normal VTL0, which currently is only VTL2.  This option
> +	  initializes the x86 platform for VTL2, and adds the ability to boot
> +	  secondary CPUs directly into 64-bit context as required for VTLs other
> +	  than 0.  A kernel built with this option must run at VTL2, and will
> +	  not run as a normal guest.
> +
> +	  If unsure, say N
> +
>  config HYPERV_TIMER
>  	def_bool HYPERV && X86
> 
> --
> 2.34.1

Reviewed-by: Michael Kelley <mikelley@microsoft.com>

RE: [PATCH v4 5/5] x86/hyperv: VTL support for Hyper-V

[Michael Kelley (LINUX)]

From: Saurabh Sengar <ssengar@linux.microsoft.com> Sent: Tuesday, April 4, 2023 2:01 AM
> 
> Virtual Trust Levels (VTL) helps enable Hyper-V Virtual Secure Mode (VSM)
> feature. VSM is a set of hypervisor capabilities and enlightenments
> offered to host and guest partitions which enable the creation and
> management of new security boundaries within operating system software.
> VSM achieves and maintains isolation through VTLs.
> 
> Add early initialization for Virtual Trust Levels (VTL). This includes
> initializing the x86 platform for VTL and enabling boot support for
> secondary CPUs to start in targeted VTL context. For now, only enable
> the code for targeted VTL level as 2.
> 
> When starting an AP at a VTL other than VTL0, the AP must start directly
> in 64-bit mode, bypassing the usual 16-bit -> 32-bit -> 64-bit mode
> transition sequence that occurs after waking up an AP with SIPI whose
> vector points to the 16-bit AP startup trampoline code.
> 
> Signed-off-by: Saurabh Sengar <ssengar@linux.microsoft.com>
> ---
> [V4]
> - replace initial_stack with current->thread.sp as per recent upstream changes
> 
>  arch/x86/hyperv/Makefile        |   1 +
>  arch/x86/hyperv/hv_vtl.c        | 227 ++++++++++++++++++++++++++++++++
>  arch/x86/include/asm/mshyperv.h |  10 ++
>  arch/x86/kernel/cpu/mshyperv.c  |   1 +
>  4 files changed, 239 insertions(+)
>  create mode 100644 arch/x86/hyperv/hv_vtl.c
> 

Reviewed-by: Michael Kelley <mikelley@microsoft.com>

Re: [PATCH v4 1/5] x86/init: Make get/set_rtc_noop() public

[Wei Liu]

On Tue, Apr 04, 2023 at 02:01:00AM -0700, Saurabh Sengar wrote:
> Make get/set_rtc_noop() to be public so that they can be used
> in other modules as well.
> 
> Co-developed-by: Tianyu Lan <tiala@microsoft.com>
> Signed-off-by: Tianyu Lan <tiala@microsoft.com>
> Signed-off-by: Saurabh Sengar <ssengar@linux.microsoft.com>
> Reviewed-by: Wei Liu <wei.liu@kernel.org>
> Reviewed-by: Michael Kelley <mikelley@microsoft.com>
> ---
>  arch/x86/include/asm/x86_init.h | 2 ++
>  arch/x86/kernel/x86_init.c      | 4 ++--
>  2 files changed, 4 insertions(+), 2 deletions(-)
> 
> diff --git a/arch/x86/include/asm/x86_init.h b/arch/x86/include/asm/x86_init.h
> index acc20ae4079d..88085f369ff6 100644
> --- a/arch/x86/include/asm/x86_init.h
> +++ b/arch/x86/include/asm/x86_init.h
> @@ -330,5 +330,7 @@ extern void x86_init_uint_noop(unsigned int unused);
>  extern bool bool_x86_init_noop(void);
>  extern void x86_op_int_noop(int cpu);
>  extern bool x86_pnpbios_disabled(void);
> +extern int set_rtc_noop(const struct timespec64 *now);
> +extern void get_rtc_noop(struct timespec64 *now);
>  
>  #endif
> diff --git a/arch/x86/kernel/x86_init.c b/arch/x86/kernel/x86_init.c
> index 95be3831df73..d82f4fa2f1bf 100644
> --- a/arch/x86/kernel/x86_init.c
> +++ b/arch/x86/kernel/x86_init.c
> @@ -33,8 +33,8 @@ static int __init iommu_init_noop(void) { return 0; }
>  static void iommu_shutdown_noop(void) { }
>  bool __init bool_x86_init_noop(void) { return false; }
>  void x86_op_int_noop(int cpu) { }
> -static __init int set_rtc_noop(const struct timespec64 *now) { return -EINVAL; }
> -static __init void get_rtc_noop(struct timespec64 *now) { }
> +int set_rtc_noop(const struct timespec64 *now) { return -EINVAL; }
> +void get_rtc_noop(struct timespec64 *now) { }

I just had a second thought on this -- do you really need to drop the
__init annotation for these two functions?

Thanks,
Wei.

>  
>  static __initconst const struct of_device_id of_cmos_match[] = {
>  	{ .compatible = "motorola,mc146818" },
> -- 
> 2.34.1
> 

RE: [PATCH v4 1/5] x86/init: Make get/set_rtc_noop() public

[Michael Kelley (LINUX)]

From: Wei Liu <wei.liu@kernel.org> Sent: Friday, April 7, 2023 10:26 AM
> 
> On Tue, Apr 04, 2023 at 02:01:00AM -0700, Saurabh Sengar wrote:
> > Make get/set_rtc_noop() to be public so that they can be used
> > in other modules as well.
> >
> > Co-developed-by: Tianyu Lan <tiala@microsoft.com>
> > Signed-off-by: Tianyu Lan <tiala@microsoft.com>
> > Signed-off-by: Saurabh Sengar <ssengar@linux.microsoft.com>
> > Reviewed-by: Wei Liu <wei.liu@kernel.org>
> > Reviewed-by: Michael Kelley <mikelley@microsoft.com>
> > ---
> >  arch/x86/include/asm/x86_init.h | 2 ++
> >  arch/x86/kernel/x86_init.c      | 4 ++--
> >  2 files changed, 4 insertions(+), 2 deletions(-)
> >
> > diff --git a/arch/x86/include/asm/x86_init.h b/arch/x86/include/asm/x86_init.h
> > index acc20ae4079d..88085f369ff6 100644
> > --- a/arch/x86/include/asm/x86_init.h
> > +++ b/arch/x86/include/asm/x86_init.h
> > @@ -330,5 +330,7 @@ extern void x86_init_uint_noop(unsigned int unused);
> >  extern bool bool_x86_init_noop(void);
> >  extern void x86_op_int_noop(int cpu);
> >  extern bool x86_pnpbios_disabled(void);
> > +extern int set_rtc_noop(const struct timespec64 *now);
> > +extern void get_rtc_noop(struct timespec64 *now);
> >
> >  #endif
> > diff --git a/arch/x86/kernel/x86_init.c b/arch/x86/kernel/x86_init.c
> > index 95be3831df73..d82f4fa2f1bf 100644
> > --- a/arch/x86/kernel/x86_init.c
> > +++ b/arch/x86/kernel/x86_init.c
> > @@ -33,8 +33,8 @@ static int __init iommu_init_noop(void) { return 0; }
> >  static void iommu_shutdown_noop(void) { }
> >  bool __init bool_x86_init_noop(void) { return false; }
> >  void x86_op_int_noop(int cpu) { }
> > -static __init int set_rtc_noop(const struct timespec64 *now) { return -EINVAL; }
> > -static __init void get_rtc_noop(struct timespec64 *now) { }
> > +int set_rtc_noop(const struct timespec64 *now) { return -EINVAL; }
> > +void get_rtc_noop(struct timespec64 *now) { }
> 
> I just had a second thought on this -- do you really need to drop the
> __init annotation for these two functions?
> 
> Thanks,
> Wei.

I think "yes".   In Patch 5 of the series, these are plugged into
x86_platform.get_wallclock() and set_wallclock().  The
x86_platform.get_wallclock() function can be called by
read_persistent_clock64(), which is not a __init function and
that may be called during resume from hibernation.

x86_platform.set_wallclock() is also called by a non __init function
update_persistent_clock64(), which is called by sync_hw_clock().

Michael

> 
> >
> >  static __initconst const struct of_device_id of_cmos_match[] = {
> >  	{ .compatible = "motorola,mc146818" },
> > --
> > 2.34.1
> >

RE: [PATCH v4 5/5] x86/hyperv: VTL support for Hyper-V

[Michael Kelley (LINUX)]

From: Stanislav Kinsburskii <skinsburskii@linux.microsoft.com> Sent: Thursday, April 6, 2023 7:08 AM
> 
> On Tue, Apr 04, 2023 at 02:01:04AM -0700, Saurabh Sengar wrote:

[snip]

> > --- /dev/null
> > +++ b/arch/x86/hyperv/hv_vtl.c
> > @@ -0,0 +1,227 @@
> > +// SPDX-License-Identifier: GPL-2.0
> > +/*
> > + * Copyright (c) 2023, Microsoft Corporation.
> > + *
> > + * Author:
> > + *   Saurabh Sengar <ssengar@microsoft.com>
> > + */
> > +
> > +#include <asm/apic.h>
> > +#include <asm/boot.h>
> > +#include <asm/desc.h>
> > +#include <asm/i8259.h>
> > +#include <asm/mshyperv.h>
> > +#include <asm/realmode.h>
> > +
> > +extern struct boot_params boot_params;
> > +static struct real_mode_header hv_vtl_real_mode_header;
> > +
> > +void __init hv_vtl_init_platform(void)
> > +{
> > +	pr_info("Linux runs in Hyper-V Virtual Trust Level\n");
> > +
> > +	x86_init.irqs.pre_vector_init = x86_init_noop;
> > +	x86_init.timers.timer_init = x86_init_noop;
> > +
> > +	x86_platform.get_wallclock = get_rtc_noop;
> > +	x86_platform.set_wallclock = set_rtc_noop;
> 
> Nit: this code is VTL feature and hypevisor specific.
> Defining vtl_get_rtc_noop instead of exporting get_rtc_noop would allow to make
> this series less intrusive to the rest of x86 generic code.
> 
> Reviewed-by: Stanislav Kinsburskii <stanislav.kinsburskii@gmail.com>
> 

Saurabh's initial version of the code did define its own version of
get/set_rtc_noop().  I had suggested that he use the existing functions
from x86 generic code, and KY had also commented about re-using
existing code wherever possible.  :-)   My suggestion was partly because
the VTL code is already re-using x86_init_noop(), and then just to avoid
code duplication.  Admittedly, there's not much code being duplicated
in these stub functions.  I slightly prefer re-using the existing functions,
but don't feel strongly about it.

Michael

Re: [PATCH v4 2/5] x86/hyperv: Add VTL specific structs and hypercalls

[Saurabh Singh Sengar]

On Thu, Apr 06, 2023 at 06:51:13AM -0700, Stanislav Kinsburskii wrote:
> On Tue, Apr 04, 2023 at 02:01:01AM -0700, Saurabh Sengar wrote:
> > Add structs and hypercalls required to enable VTL support on x86.
> > 
> > Signed-off-by: Saurabh Sengar <ssengar@linux.microsoft.com>
> > Reviewed-by: Michael Kelley <mikelley@microsoft.com>
> > ---
> >  arch/x86/include/asm/hyperv-tlfs.h | 75 ++++++++++++++++++++++++++++++
> >  include/asm-generic/hyperv-tlfs.h  |  4 ++
> >  2 files changed, 79 insertions(+)
> > 
> > diff --git a/arch/x86/include/asm/hyperv-tlfs.h b/arch/x86/include/asm/hyperv-tlfs.h
> > index 0b73a809e9e1..0b0b4e9a4318 100644
> > --- a/arch/x86/include/asm/hyperv-tlfs.h
> > +++ b/arch/x86/include/asm/hyperv-tlfs.h
> > @@ -713,6 +713,81 @@ union hv_msi_entry {
> >  	} __packed;
> >  };
> >  
> > +struct hv_x64_segment_register {
> > +	__u64 base;
> 
> Ideally they arch-size types naming should be consistent: either with underscores or
> without.
> The majority of cases in this file are without underscores.

Although I am fine either way, I think in a non-uapi file "without underscore" is prefered.
I can change this in next version.

Regards,
Saurabh


> 
> Reviewed-by: Stanislav Kinsburskii <stanislav.kinsburskii@gmail.com>
> 
> > +	__u32 limit;
> > +	__u16 selector;
> > +	union {
> > +		struct {
> > +			__u16 segment_type : 4;
> > +			__u16 non_system_segment : 1;
> > +			__u16 descriptor_privilege_level : 2;
> > +			__u16 present : 1;
> > +			__u16 reserved : 4;
> > +			__u16 available : 1;
> > +			__u16 _long : 1;
> > +			__u16 _default : 1;
> > +			__u16 granularity : 1;
> > +		} __packed;
> > +		__u16 attributes;
> > +	};
> > +} __packed;
> > +
> > +struct hv_x64_table_register {
> > +	__u16 pad[3];
> > +	__u16 limit;
> > +	__u64 base;
> > +} __packed;
> > +
> > +struct hv_init_vp_context {
> > +	u64 rip;
> > +	u64 rsp;
> > +	u64 rflags;
> > +
> > +	struct hv_x64_segment_register cs;
> > +	struct hv_x64_segment_register ds;
> > +	struct hv_x64_segment_register es;
> > +	struct hv_x64_segment_register fs;
> > +	struct hv_x64_segment_register gs;
> > +	struct hv_x64_segment_register ss;
> > +	struct hv_x64_segment_register tr;
> > +	struct hv_x64_segment_register ldtr;
> > +
> > +	struct hv_x64_table_register idtr;
> > +	struct hv_x64_table_register gdtr;
> > +
> > +	u64 efer;
> > +	u64 cr0;
> > +	u64 cr3;
> > +	u64 cr4;
> > +	u64 msr_cr_pat;
> > +} __packed;
> > +
> > +union hv_input_vtl {
> > +	u8 as_uint8;
> > +	struct {
> > +		u8 target_vtl: 4;
> > +		u8 use_target_vtl: 1;
> > +		u8 reserved_z: 3;
> > +	};
> > +} __packed;
> > +
> > +struct hv_enable_vp_vtl {
> > +	u64				partition_id;
> > +	u32				vp_index;
> > +	union hv_input_vtl		target_vtl;
> > +	u8				mbz0;
> > +	u16				mbz1;
> > +	struct hv_init_vp_context	vp_context;
> > +} __packed;
> > +
> > +struct hv_get_vp_from_apic_id_in {
> > +	u64 partition_id;
> > +	union hv_input_vtl target_vtl;
> > +	u8 res[7];
> > +	u32 apic_ids[];
> > +} __packed;
> > +
> >  #include <asm-generic/hyperv-tlfs.h>
> >  
> >  #endif
> > diff --git a/include/asm-generic/hyperv-tlfs.h b/include/asm-generic/hyperv-tlfs.h
> > index b870983596b9..87258341fd7c 100644
> > --- a/include/asm-generic/hyperv-tlfs.h
> > +++ b/include/asm-generic/hyperv-tlfs.h
> > @@ -146,6 +146,7 @@ union hv_reference_tsc_msr {
> >  /* Declare the various hypercall operations. */
> >  #define HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE	0x0002
> >  #define HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST	0x0003
> > +#define HVCALL_ENABLE_VP_VTL			0x000f
> >  #define HVCALL_NOTIFY_LONG_SPIN_WAIT		0x0008
> >  #define HVCALL_SEND_IPI				0x000b
> >  #define HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX	0x0013
> > @@ -165,6 +166,8 @@ union hv_reference_tsc_msr {
> >  #define HVCALL_MAP_DEVICE_INTERRUPT		0x007c
> >  #define HVCALL_UNMAP_DEVICE_INTERRUPT		0x007d
> >  #define HVCALL_RETARGET_INTERRUPT		0x007e
> > +#define HVCALL_START_VP				0x0099
> > +#define HVCALL_GET_VP_ID_FROM_APIC_ID		0x009a
> >  #define HVCALL_FLUSH_GUEST_PHYSICAL_ADDRESS_SPACE 0x00af
> >  #define HVCALL_FLUSH_GUEST_PHYSICAL_ADDRESS_LIST 0x00b0
> >  #define HVCALL_MODIFY_SPARSE_GPA_PAGE_HOST_VISIBILITY 0x00db
> > @@ -218,6 +221,7 @@ enum HV_GENERIC_SET_FORMAT {
> >  #define HV_STATUS_INVALID_PORT_ID		17
> >  #define HV_STATUS_INVALID_CONNECTION_ID		18
> >  #define HV_STATUS_INSUFFICIENT_BUFFERS		19
> > +#define HV_STATUS_VTL_ALREADY_ENABLED		134
> >  
> >  /*
> >   * The Hyper-V TimeRefCount register and the TSC
> > -- 
> > 2.34.1

Re: [PATCH v4 5/5] x86/hyperv: VTL support for Hyper-V

[Saurabh Singh Sengar]

On Fri, Apr 07, 2023 at 08:56:29PM +0000, Michael Kelley (LINUX) wrote:
> From: Stanislav Kinsburskii <skinsburskii@linux.microsoft.com> Sent: Thursday, April 6, 2023 7:08 AM
> > 
> > On Tue, Apr 04, 2023 at 02:01:04AM -0700, Saurabh Sengar wrote:
> 
> [snip]
> 
> > > --- /dev/null
> > > +++ b/arch/x86/hyperv/hv_vtl.c
> > > @@ -0,0 +1,227 @@
> > > +// SPDX-License-Identifier: GPL-2.0
> > > +/*
> > > + * Copyright (c) 2023, Microsoft Corporation.
> > > + *
> > > + * Author:
> > > + *   Saurabh Sengar <ssengar@microsoft.com>
> > > + */
> > > +
> > > +#include <asm/apic.h>
> > > +#include <asm/boot.h>
> > > +#include <asm/desc.h>
> > > +#include <asm/i8259.h>
> > > +#include <asm/mshyperv.h>
> > > +#include <asm/realmode.h>
> > > +
> > > +extern struct boot_params boot_params;
> > > +static struct real_mode_header hv_vtl_real_mode_header;
> > > +
> > > +void __init hv_vtl_init_platform(void)
> > > +{
> > > +	pr_info("Linux runs in Hyper-V Virtual Trust Level\n");
> > > +
> > > +	x86_init.irqs.pre_vector_init = x86_init_noop;
> > > +	x86_init.timers.timer_init = x86_init_noop;
> > > +
> > > +	x86_platform.get_wallclock = get_rtc_noop;
> > > +	x86_platform.set_wallclock = set_rtc_noop;
> > 
> > Nit: this code is VTL feature and hypevisor specific.
> > Defining vtl_get_rtc_noop instead of exporting get_rtc_noop would allow to make
> > this series less intrusive to the rest of x86 generic code.
> > 
> > Reviewed-by: Stanislav Kinsburskii <stanislav.kinsburskii@gmail.com>
> > 
> 
> Saurabh's initial version of the code did define its own version of
> get/set_rtc_noop().  I had suggested that he use the existing functions
> from x86 generic code, and KY had also commented about re-using
> existing code wherever possible.  :-)   My suggestion was partly because
> the VTL code is already re-using x86_init_noop(), and then just to avoid
> code duplication.  Admittedly, there's not much code being duplicated
> in these stub functions.  I slightly prefer re-using the existing functions,
> but don't feel strongly about it.

Thank you, Stanislav and Michael, for your comments.
Since the function is not doing anything significant, I think it's best to
minimize its effect on x86 generic code in this series.

I would like to redefine these functions in VTL code. Unless there are
compelling reasons against it, I plan to incorporate this change in the next
version.

- Saurabh

> Michael

RE: [EXTERNAL] Re: [PATCH v4 2/5] x86/hyperv: Add VTL specific structs and hypercalls

[Saurabh Singh Sengar]

> -----Original Message-----
> From: Saurabh Singh Sengar <ssengar@linux.microsoft.com>
> Sent: Saturday, April 8, 2023 9:58 AM
> To: Stanislav Kinsburskii <skinsburskii@linux.microsoft.com>
> Cc: tglx@linutronix.de; mingo@redhat.com; bp@alien8.de;
> dave.hansen@linux.intel.com; x86@kernel.org; hpa@zytor.com; KY
> Srinivasan <kys@microsoft.com>; Haiyang Zhang
> <haiyangz@microsoft.com>; wei.liu@kernel.org; Dexuan Cui
> <decui@microsoft.com>; arnd@arndb.de; Tianyu Lan
> <Tianyu.Lan@microsoft.com>; Michael Kelley (LINUX)
> <mikelley@microsoft.com>; linux-kernel@vger.kernel.org; linux-
> hyperv@vger.kernel.org; linux-arch@vger.kernel.org; jgross@suse.com;
> mat.jonczyk@o2.pl
> Subject: [EXTERNAL] Re: [PATCH v4 2/5] x86/hyperv: Add VTL specific structs
> and hypercalls
> 
> On Thu, Apr 06, 2023 at 06:51:13AM -0700, Stanislav Kinsburskii wrote:
> > On Tue, Apr 04, 2023 at 02:01:01AM -0700, Saurabh Sengar wrote:
> > > Add structs and hypercalls required to enable VTL support on x86.
> > >
> > > Signed-off-by: Saurabh Sengar <ssengar@linux.microsoft.com>
> > > Reviewed-by: Michael Kelley <mikelley@microsoft.com>
> > > ---
> > >  arch/x86/include/asm/hyperv-tlfs.h | 75
> > > ++++++++++++++++++++++++++++++  include/asm-generic/hyperv-tlfs.h  |
> > > 4 ++
> > >  2 files changed, 79 insertions(+)
> > >
> > > diff --git a/arch/x86/include/asm/hyperv-tlfs.h
> > > b/arch/x86/include/asm/hyperv-tlfs.h
> > > index 0b73a809e9e1..0b0b4e9a4318 100644
> > > --- a/arch/x86/include/asm/hyperv-tlfs.h
> > > +++ b/arch/x86/include/asm/hyperv-tlfs.h
> > > @@ -713,6 +713,81 @@ union hv_msi_entry {
> > >  	} __packed;
> > >  };
> > >
> > > +struct hv_x64_segment_register {
> > > +	__u64 base;
> >
> > Ideally they arch-size types naming should be consistent: either with
> > underscores or without.
> > The majority of cases in this file are without underscores.
> 
> Although I am fine either way, I think in a non-uapi file "without underscore"
> is prefered.
> I can change this in next version.

After reconsidering, I believe it would be wise to consult the "x86 maintainers" regarding this matter.
Hence, I intend to send the next version without this change.

- Saurabh

> 
> Regards,
> Saurabh
> 
> 
> >
> > Reviewed-by: Stanislav Kinsburskii <stanislav.kinsburskii@gmail.com>
> >
> > > +	__u32 limit;
> > > +	__u16 selector;
> > > +	union {
> > > +		struct {
> > > +			__u16 segment_type : 4;
> > > +			__u16 non_system_segment : 1;
> > > +			__u16 descriptor_privilege_level : 2;
> > > +			__u16 present : 1;
> > > +			__u16 reserved : 4;
> > > +			__u16 available : 1;
> > > +			__u16 _long : 1;
> > > +			__u16 _default : 1;
> > > +			__u16 granularity : 1;
> > > +		} __packed;
> > > +		__u16 attributes;
> > > +	};
> > > +} __packed;
> > > +
> > > +struct hv_x64_table_register {
> > > +	__u16 pad[3];
> > > +	__u16 limit;
> > > +	__u64 base;
> > > +} __packed;
> > > +
> > > +struct hv_init_vp_context {
> > > +	u64 rip;
> > > +	u64 rsp;
> > > +	u64 rflags;
> > > +
> > > +	struct hv_x64_segment_register cs;
> > > +	struct hv_x64_segment_register ds;
> > > +	struct hv_x64_segment_register es;
> > > +	struct hv_x64_segment_register fs;
> > > +	struct hv_x64_segment_register gs;
> > > +	struct hv_x64_segment_register ss;
> > > +	struct hv_x64_segment_register tr;
> > > +	struct hv_x64_segment_register ldtr;
> > > +
> > > +	struct hv_x64_table_register idtr;
> > > +	struct hv_x64_table_register gdtr;
> > > +
> > > +	u64 efer;
> > > +	u64 cr0;
> > > +	u64 cr3;
> > > +	u64 cr4;
> > > +	u64 msr_cr_pat;
> > > +} __packed;
> > > +
> > > +union hv_input_vtl {
> > > +	u8 as_uint8;
> > > +	struct {
> > > +		u8 target_vtl: 4;
> > > +		u8 use_target_vtl: 1;
> > > +		u8 reserved_z: 3;
> > > +	};
> > > +} __packed;
> > > +
> > > +struct hv_enable_vp_vtl {
> > > +	u64				partition_id;
> > > +	u32				vp_index;
> > > +	union hv_input_vtl		target_vtl;
> > > +	u8				mbz0;
> > > +	u16				mbz1;
> > > +	struct hv_init_vp_context	vp_context;
> > > +} __packed;
> > > +
> > > +struct hv_get_vp_from_apic_id_in {
> > > +	u64 partition_id;
> > > +	union hv_input_vtl target_vtl;
> > > +	u8 res[7];
> > > +	u32 apic_ids[];
> > > +} __packed;
> > > +
> > >  #include <asm-generic/hyperv-tlfs.h>
> > >
> > >  #endif
> > > diff --git a/include/asm-generic/hyperv-tlfs.h
> > > b/include/asm-generic/hyperv-tlfs.h
> > > index b870983596b9..87258341fd7c 100644
> > > --- a/include/asm-generic/hyperv-tlfs.h
> > > +++ b/include/asm-generic/hyperv-tlfs.h
> > > @@ -146,6 +146,7 @@ union hv_reference_tsc_msr {
> > >  /* Declare the various hypercall operations. */
> > >  #define HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE	0x0002
> > >  #define HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST	0x0003
> > > +#define HVCALL_ENABLE_VP_VTL			0x000f
> > >  #define HVCALL_NOTIFY_LONG_SPIN_WAIT		0x0008
> > >  #define HVCALL_SEND_IPI				0x000b
> > >  #define HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX	0x0013
> > > @@ -165,6 +166,8 @@ union hv_reference_tsc_msr {
> > >  #define HVCALL_MAP_DEVICE_INTERRUPT		0x007c
> > >  #define HVCALL_UNMAP_DEVICE_INTERRUPT		0x007d
> > >  #define HVCALL_RETARGET_INTERRUPT		0x007e
> > > +#define HVCALL_START_VP				0x0099
> > > +#define HVCALL_GET_VP_ID_FROM_APIC_ID		0x009a
> > >  #define HVCALL_FLUSH_GUEST_PHYSICAL_ADDRESS_SPACE 0x00af
> #define
> > > HVCALL_FLUSH_GUEST_PHYSICAL_ADDRESS_LIST 0x00b0  #define
> > > HVCALL_MODIFY_SPARSE_GPA_PAGE_HOST_VISIBILITY 0x00db @@ -
> 218,6
> > > +221,7 @@ enum HV_GENERIC_SET_FORMAT {
> > >  #define HV_STATUS_INVALID_PORT_ID		17
> > >  #define HV_STATUS_INVALID_CONNECTION_ID		18
> > >  #define HV_STATUS_INSUFFICIENT_BUFFERS		19
> > > +#define HV_STATUS_VTL_ALREADY_ENABLED		134
> > >
> > >  /*
> > >   * The Hyper-V TimeRefCount register and the TSC
> > > --
> > > 2.34.1