* [PATCH v4 1/5] Drivers: hv: Refactor and rename memory region handling functions
From: Stanislav Kinsburskii @ 2025-10-06 15:06 UTC (permalink / raw)
To: kys, haiyangz, wei.liu, decui; +Cc: linux-hyperv, linux-kernel
In-Reply-To: <175976284493.16834.4572937416426518745.stgit@skinsburskii-cloud-desktop.internal.cloudapp.net>
Simplify and unify memory region management to improve code clarity and
reliability. Consolidate pinning and invalidation logic, adopt consistent
naming, and remove redundant checks to reduce complexity.
Enhance documentation and update call sites for maintainability.
Signed-off-by: Stanislav Kinsburskii <skinsburskii@linux.microsoft.com>
---
drivers/hv/mshv_root_main.c | 80 +++++++++++++++++++------------------------
1 file changed, 36 insertions(+), 44 deletions(-)
diff --git a/drivers/hv/mshv_root_main.c b/drivers/hv/mshv_root_main.c
index fa42c40e1e02f..e923947d3c54d 100644
--- a/drivers/hv/mshv_root_main.c
+++ b/drivers/hv/mshv_root_main.c
@@ -1120,8 +1120,8 @@ mshv_region_map(struct mshv_mem_region *region)
}
static void
-mshv_region_evict_pages(struct mshv_mem_region *region,
- u64 page_offset, u64 page_count)
+mshv_region_invalidate_pages(struct mshv_mem_region *region,
+ u64 page_offset, u64 page_count)
{
if (region->flags.range_pinned)
unpin_user_pages(region->pages + page_offset, page_count);
@@ -1131,29 +1131,24 @@ mshv_region_evict_pages(struct mshv_mem_region *region,
}
static void
-mshv_region_evict(struct mshv_mem_region *region)
+mshv_region_invalidate(struct mshv_mem_region *region)
{
- mshv_region_evict_pages(region, 0, region->nr_pages);
+ mshv_region_invalidate_pages(region, 0, region->nr_pages);
}
static int
-mshv_region_populate_pages(struct mshv_mem_region *region,
- u64 page_offset, u64 page_count)
+mshv_region_pin(struct mshv_mem_region *region)
{
u64 done_count, nr_pages;
struct page **pages;
__u64 userspace_addr;
int ret;
- if (page_offset + page_count > region->nr_pages)
- return -EINVAL;
-
- for (done_count = 0; done_count < page_count; done_count += ret) {
- pages = region->pages + page_offset + done_count;
+ for (done_count = 0; done_count < region->nr_pages; done_count += ret) {
+ pages = region->pages + done_count;
userspace_addr = region->start_uaddr +
- (page_offset + done_count) *
- HV_HYP_PAGE_SIZE;
- nr_pages = min(page_count - done_count,
+ done_count * HV_HYP_PAGE_SIZE;
+ nr_pages = min(region->nr_pages - done_count,
MSHV_PIN_PAGES_BATCH_SIZE);
/*
@@ -1164,34 +1159,23 @@ mshv_region_populate_pages(struct mshv_mem_region *region,
* with the FOLL_LONGTERM flag does a large temporary
* allocation of contiguous memory.
*/
- if (region->flags.range_pinned)
- ret = pin_user_pages_fast(userspace_addr,
- nr_pages,
- FOLL_WRITE | FOLL_LONGTERM,
- pages);
- else
- ret = -EOPNOTSUPP;
-
+ ret = pin_user_pages_fast(userspace_addr, nr_pages,
+ FOLL_WRITE | FOLL_LONGTERM,
+ pages);
if (ret < 0)
goto release_pages;
}
- if (PageHuge(region->pages[page_offset]))
+ if (PageHuge(region->pages[0]))
region->flags.large_pages = true;
return 0;
release_pages:
- mshv_region_evict_pages(region, page_offset, done_count);
+ mshv_region_invalidate_pages(region, 0, done_count);
return ret;
}
-static int
-mshv_region_populate(struct mshv_mem_region *region)
-{
- return mshv_region_populate_pages(region, 0, region->nr_pages);
-}
-
static struct mshv_mem_region *
mshv_partition_region_by_gfn(struct mshv_partition *partition, u64 gfn)
{
@@ -1264,19 +1248,27 @@ static int mshv_partition_create_region(struct mshv_partition *partition,
return 0;
}
-/*
- * Map guest ram. if snp, make sure to release that from the host first
- * Side Effects: In case of failure, pages are unpinned when feasible.
+/**
+ * mshv_prepare_pinned_region - Pin and map memory regions
+ * @region: Pointer to the memory region structure
+ *
+ * This function processes memory regions that are explicitly marked as pinned.
+ * Pinned regions are preallocated, mapped upfront, and do not rely on fault-based
+ * population. The function ensures the region is properly populated, handles
+ * encryption requirements for SNP partitions if applicable, maps the region,
+ * and performs necessary sharing or eviction operations based on the mapping
+ * result.
+ *
+ * Return: 0 on success, negative error code on failure.
*/
-static int
-mshv_partition_mem_region_map(struct mshv_mem_region *region)
+static int mshv_prepare_pinned_region(struct mshv_mem_region *region)
{
struct mshv_partition *partition = region->partition;
int ret;
- ret = mshv_region_populate(region);
+ ret = mshv_region_pin(region);
if (ret) {
- pt_err(partition, "Failed to populate memory region: %d\n",
+ pt_err(partition, "Failed to pin memory region: %d\n",
ret);
goto err_out;
}
@@ -1294,7 +1286,7 @@ mshv_partition_mem_region_map(struct mshv_mem_region *region)
pt_err(partition,
"Failed to unshare memory region (guest_pfn: %llu): %d\n",
region->start_gfn, ret);
- goto evict_region;
+ goto invalidate_region;
}
}
@@ -1304,7 +1296,7 @@ mshv_partition_mem_region_map(struct mshv_mem_region *region)
shrc = mshv_partition_region_share(region);
if (!shrc)
- goto evict_region;
+ goto invalidate_region;
pt_err(partition,
"Failed to share memory region (guest_pfn: %llu): %d\n",
@@ -1318,8 +1310,8 @@ mshv_partition_mem_region_map(struct mshv_mem_region *region)
return 0;
-evict_region:
- mshv_region_evict(region);
+invalidate_region:
+ mshv_region_invalidate(region);
err_out:
return ret;
}
@@ -1368,7 +1360,7 @@ mshv_map_user_memory(struct mshv_partition *partition,
ret = hv_call_map_mmio_pages(partition->pt_id, mem.guest_pfn,
mmio_pfn, HVPFN_DOWN(mem.size));
else
- ret = mshv_partition_mem_region_map(region);
+ ret = mshv_prepare_pinned_region(region);
if (ret)
goto errout;
@@ -1413,7 +1405,7 @@ mshv_unmap_user_memory(struct mshv_partition *partition,
hv_call_unmap_gpa_pages(partition->pt_id, region->start_gfn,
region->nr_pages, unmap_flags);
- mshv_region_evict(region);
+ mshv_region_invalidate(region);
vfree(region);
return 0;
@@ -1827,7 +1819,7 @@ static void destroy_partition(struct mshv_partition *partition)
}
}
- mshv_region_evict(region);
+ mshv_region_invalidate(region);
vfree(region);
}
^ permalink raw reply related
* [PATCH v4 2/5] Drivers: hv: Centralize guest memory region destruction
From: Stanislav Kinsburskii @ 2025-10-06 15:06 UTC (permalink / raw)
To: kys, haiyangz, wei.liu, decui; +Cc: linux-hyperv, linux-kernel
In-Reply-To: <175976284493.16834.4572937416426518745.stgit@skinsburskii-cloud-desktop.internal.cloudapp.net>
Centralize guest memory region destruction to prevent resource leaks and
inconsistent cleanup across unmap and partition destruction paths.
Unify region removal, encrypted partition access recovery, and region
invalidation to improve maintainability and reliability. Reduce code
duplication and make future updates less error-prone by encapsulating
cleanup logic in a single helper.
Signed-off-by: Stanislav Kinsburskii <skinsburskii@linux.microsoft.com>
---
drivers/hv/mshv_root_main.c | 65 ++++++++++++++++++++++---------------------
1 file changed, 34 insertions(+), 31 deletions(-)
diff --git a/drivers/hv/mshv_root_main.c b/drivers/hv/mshv_root_main.c
index e923947d3c54d..97e322f3c6b5e 100644
--- a/drivers/hv/mshv_root_main.c
+++ b/drivers/hv/mshv_root_main.c
@@ -1375,13 +1375,42 @@ mshv_map_user_memory(struct mshv_partition *partition,
return ret;
}
+static void mshv_partition_destroy_region(struct mshv_mem_region *region)
+{
+ struct mshv_partition *partition = region->partition;
+ u32 unmap_flags = 0;
+ int ret;
+
+ hlist_del(®ion->hnode);
+
+ if (mshv_partition_encrypted(partition)) {
+ ret = mshv_partition_region_share(region);
+ if (ret) {
+ pt_err(partition,
+ "Failed to regain access to memory, unpinning user pages will fail and crash the host error: %d\n",
+ ret);
+ return;
+ }
+ }
+
+ if (region->flags.large_pages)
+ unmap_flags |= HV_UNMAP_GPA_LARGE_PAGE;
+
+ /* ignore unmap failures and continue as process may be exiting */
+ hv_call_unmap_gpa_pages(partition->pt_id, region->start_gfn,
+ region->nr_pages, unmap_flags);
+
+ mshv_region_invalidate(region);
+
+ vfree(region);
+}
+
/* Called for unmapping both the guest ram and the mmio space */
static long
mshv_unmap_user_memory(struct mshv_partition *partition,
struct mshv_user_mem_region mem)
{
struct mshv_mem_region *region;
- u32 unmap_flags = 0;
if (!(mem.flags & BIT(MSHV_SET_MEM_BIT_UNMAP)))
return -EINVAL;
@@ -1396,18 +1425,8 @@ mshv_unmap_user_memory(struct mshv_partition *partition,
region->nr_pages != HVPFN_DOWN(mem.size))
return -EINVAL;
- hlist_del(®ion->hnode);
+ mshv_partition_destroy_region(region);
- if (region->flags.large_pages)
- unmap_flags |= HV_UNMAP_GPA_LARGE_PAGE;
-
- /* ignore unmap failures and continue as process may be exiting */
- hv_call_unmap_gpa_pages(partition->pt_id, region->start_gfn,
- region->nr_pages, unmap_flags);
-
- mshv_region_invalidate(region);
-
- vfree(region);
return 0;
}
@@ -1743,8 +1762,8 @@ static void destroy_partition(struct mshv_partition *partition)
{
struct mshv_vp *vp;
struct mshv_mem_region *region;
- int i, ret;
struct hlist_node *n;
+ int i;
if (refcount_read(&partition->pt_ref_count)) {
pt_err(partition,
@@ -1804,25 +1823,9 @@ static void destroy_partition(struct mshv_partition *partition)
remove_partition(partition);
- /* Remove regions, regain access to the memory and unpin the pages */
hlist_for_each_entry_safe(region, n, &partition->pt_mem_regions,
- hnode) {
- hlist_del(®ion->hnode);
-
- if (mshv_partition_encrypted(partition)) {
- ret = mshv_partition_region_share(region);
- if (ret) {
- pt_err(partition,
- "Failed to regain access to memory, unpinning user pages will fail and crash the host error: %d\n",
- ret);
- return;
- }
- }
-
- mshv_region_invalidate(region);
-
- vfree(region);
- }
+ hnode)
+ mshv_partition_destroy_region(region);
/* Withdraw and free all pages we deposited */
hv_call_withdraw_memory(U64_MAX, NUMA_NO_NODE, partition->pt_id);
^ permalink raw reply related
* [PATCH v4 3/5] Drivers: hv: Batch GPA unmap operations to improve large region performance
From: Stanislav Kinsburskii @ 2025-10-06 15:06 UTC (permalink / raw)
To: kys, haiyangz, wei.liu, decui; +Cc: linux-hyperv, linux-kernel
In-Reply-To: <175976284493.16834.4572937416426518745.stgit@skinsburskii-cloud-desktop.internal.cloudapp.net>
Reduce overhead when unmapping large memory regions by batching GPA unmap
operations in 2MB-aligned chunks.
Use a dedicated constant for batch size to improve code clarity and
maintainability.
Signed-off-by: Stanislav Kinsburskii <skinsburskii@linux.microsoft.com>
---
drivers/hv/mshv_root.h | 2 ++
drivers/hv/mshv_root_hv_call.c | 2 +-
drivers/hv/mshv_root_main.c | 28 +++++++++++++++++++++++++---
3 files changed, 28 insertions(+), 4 deletions(-)
diff --git a/drivers/hv/mshv_root.h b/drivers/hv/mshv_root.h
index e3931b0f12693..97e64d5341b6e 100644
--- a/drivers/hv/mshv_root.h
+++ b/drivers/hv/mshv_root.h
@@ -32,6 +32,8 @@ static_assert(HV_HYP_PAGE_SIZE == MSHV_HV_PAGE_SIZE);
#define MSHV_PIN_PAGES_BATCH_SIZE (0x10000000ULL / HV_HYP_PAGE_SIZE)
+#define MSHV_MAX_UNMAP_GPA_PAGES 512
+
struct mshv_vp {
u32 vp_index;
struct mshv_partition *vp_partition;
diff --git a/drivers/hv/mshv_root_hv_call.c b/drivers/hv/mshv_root_hv_call.c
index c9c274f29c3c6..0696024ccfe31 100644
--- a/drivers/hv/mshv_root_hv_call.c
+++ b/drivers/hv/mshv_root_hv_call.c
@@ -17,7 +17,7 @@
/* Determined empirically */
#define HV_INIT_PARTITION_DEPOSIT_PAGES 208
#define HV_MAP_GPA_DEPOSIT_PAGES 256
-#define HV_UMAP_GPA_PAGES 512
+#define HV_UMAP_GPA_PAGES MSHV_MAX_UNMAP_GPA_PAGES
#define HV_PAGE_COUNT_2M_ALIGNED(pg_count) (!((pg_count) & (0x200 - 1)))
diff --git a/drivers/hv/mshv_root_main.c b/drivers/hv/mshv_root_main.c
index 97e322f3c6b5e..b61bef6b9c132 100644
--- a/drivers/hv/mshv_root_main.c
+++ b/drivers/hv/mshv_root_main.c
@@ -1378,6 +1378,7 @@ mshv_map_user_memory(struct mshv_partition *partition,
static void mshv_partition_destroy_region(struct mshv_mem_region *region)
{
struct mshv_partition *partition = region->partition;
+ u64 gfn, gfn_count, start_gfn, end_gfn;
u32 unmap_flags = 0;
int ret;
@@ -1396,9 +1397,30 @@ static void mshv_partition_destroy_region(struct mshv_mem_region *region)
if (region->flags.large_pages)
unmap_flags |= HV_UNMAP_GPA_LARGE_PAGE;
- /* ignore unmap failures and continue as process may be exiting */
- hv_call_unmap_gpa_pages(partition->pt_id, region->start_gfn,
- region->nr_pages, unmap_flags);
+ start_gfn = region->start_gfn;
+ end_gfn = region->start_gfn + region->nr_pages;
+
+ for (gfn = start_gfn; gfn < end_gfn; gfn += gfn_count) {
+ if (gfn % MSHV_MAX_UNMAP_GPA_PAGES)
+ gfn_count = ALIGN(gfn, MSHV_MAX_UNMAP_GPA_PAGES) - gfn;
+ else
+ gfn_count = MSHV_MAX_UNMAP_GPA_PAGES;
+
+ if (gfn + gfn_count > end_gfn)
+ gfn_count = end_gfn - gfn;
+
+ /* Skip if all pages in this range if none is mapped */
+ if (!memchr_inv(region->pages + (gfn - start_gfn), 0,
+ gfn_count * sizeof(struct page *)))
+ continue;
+
+ ret = hv_call_unmap_gpa_pages(partition->pt_id, gfn,
+ gfn_count, unmap_flags);
+ if (ret)
+ pt_err(partition,
+ "Failed to unmap GPA pages %#llx-%#llx: %d\n",
+ gfn, gfn + gfn_count - 1, ret);
+ }
mshv_region_invalidate(region);
^ permalink raw reply related
* [PATCH v4 4/5] Drivers: hv: Ensure large page GPA mapping is PMD-aligned
From: Stanislav Kinsburskii @ 2025-10-06 15:06 UTC (permalink / raw)
To: kys, haiyangz, wei.liu, decui; +Cc: linux-hyperv, linux-kernel
In-Reply-To: <175976284493.16834.4572937416426518745.stgit@skinsburskii-cloud-desktop.internal.cloudapp.net>
With the upcoming introduction of movable pages, a region doesn't guarantee
always having large pages mapped. Both mapping on fault and unmapping
during PTE invalidation may not be 2M-aligned, while the hypervisor
requires both the GFN and page count to be 2M-aligned to use the large page
flag.
Update the logic for large page mapping in mshv_region_remap_pages() to
require both page_offset and page_count to be PMD-aligned.
Signed-off-by: Stanislav Kinsburskii <skinsburskii@linux.microsoft.com>
---
drivers/hv/mshv_root_main.c | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/drivers/hv/mshv_root_main.c b/drivers/hv/mshv_root_main.c
index b61bef6b9c132..cb462495f34b5 100644
--- a/drivers/hv/mshv_root_main.c
+++ b/drivers/hv/mshv_root_main.c
@@ -34,6 +34,8 @@
#include "mshv.h"
#include "mshv_root.h"
+#define VALUE_PMD_ALIGNED(c) (!((c) & (PTRS_PER_PMD - 1)))
+
MODULE_AUTHOR("Microsoft");
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Microsoft Hyper-V root partition VMM interface /dev/mshv");
@@ -1100,7 +1102,9 @@ mshv_region_remap_pages(struct mshv_mem_region *region, u32 map_flags,
if (page_offset + page_count > region->nr_pages)
return -EINVAL;
- if (region->flags.large_pages)
+ if (region->flags.large_pages &&
+ VALUE_PMD_ALIGNED(page_offset) &&
+ VALUE_PMD_ALIGNED(page_count))
map_flags |= HV_MAP_GPA_LARGE_PAGE;
/* ask the hypervisor to map guest ram */
^ permalink raw reply related
* [PATCH v4 5/5] Drivers: hv: Add support for movable memory regions
From: Stanislav Kinsburskii @ 2025-10-06 15:06 UTC (permalink / raw)
To: kys, haiyangz, wei.liu, decui; +Cc: linux-hyperv, linux-kernel
In-Reply-To: <175976284493.16834.4572937416426518745.stgit@skinsburskii-cloud-desktop.internal.cloudapp.net>
Introduce support for movable memory regions in the Hyper-V root partition
driver, thus improving memory management flexibility and preparing the
driver for advanced use cases such as dynamic memory remapping.
Integrate mmu_interval_notifier for movable regions, implement functions to
handle HMM faults and memory invalidation, and update memory region mapping
logic to support movable regions.
While MMU notifiers are commonly used in virtualization drivers, this
implementation leverages HMM (Heterogeneous Memory Management) for its
tailored functionality. HMM provides a ready-made framework for mirroring,
invalidation, and fault handling, avoiding the need to reimplement these
mechanisms for a single callback. Although MMU notifiers are more generic,
using HMM reduces boilerplate and ensures maintainability by utilizing a
mechanism specifically designed for such use cases.
Signed-off-by: Anirudh Rayabharam <anrayabh@linux.microsoft.com>
Signed-off-by: Stanislav Kinsburskii <skinsburskii@linux.microsoft.com>
---
drivers/hv/Kconfig | 1
drivers/hv/mshv_root.h | 8 +
drivers/hv/mshv_root_main.c | 328 ++++++++++++++++++++++++++++++++++++++++++-
3 files changed, 327 insertions(+), 10 deletions(-)
diff --git a/drivers/hv/Kconfig b/drivers/hv/Kconfig
index e24f6299c3760..9d24a8c8c52e3 100644
--- a/drivers/hv/Kconfig
+++ b/drivers/hv/Kconfig
@@ -68,6 +68,7 @@ config MSHV_ROOT
depends on PAGE_SIZE_4KB
select EVENTFD
select VIRT_XFER_TO_GUEST_WORK
+ select HMM_MIRROR
default n
help
Select this option to enable support for booting and running as root
diff --git a/drivers/hv/mshv_root.h b/drivers/hv/mshv_root.h
index 97e64d5341b6e..13367c84497c0 100644
--- a/drivers/hv/mshv_root.h
+++ b/drivers/hv/mshv_root.h
@@ -15,6 +15,7 @@
#include <linux/hashtable.h>
#include <linux/dev_printk.h>
#include <linux/build_bug.h>
+#include <linux/mmu_notifier.h>
#include <uapi/linux/mshv.h>
/*
@@ -81,9 +82,14 @@ struct mshv_mem_region {
struct {
u64 large_pages: 1; /* 2MiB */
u64 range_pinned: 1;
- u64 reserved: 62;
+ u64 is_ram : 1; /* mem region can be ram or mmio */
+ u64 reserved: 61;
} flags;
struct mshv_partition *partition;
+#if defined(CONFIG_MMU_NOTIFIER)
+ struct mmu_interval_notifier mni;
+ struct mutex mutex; /* protects region pages remapping */
+#endif
struct page *pages[];
};
diff --git a/drivers/hv/mshv_root_main.c b/drivers/hv/mshv_root_main.c
index cb462495f34b5..bc0ea39bcd255 100644
--- a/drivers/hv/mshv_root_main.c
+++ b/drivers/hv/mshv_root_main.c
@@ -29,6 +29,7 @@
#include <linux/crash_dump.h>
#include <linux/panic_notifier.h>
#include <linux/vmalloc.h>
+#include <linux/hmm.h>
#include "mshv_eventfd.h"
#include "mshv.h"
@@ -36,6 +37,8 @@
#define VALUE_PMD_ALIGNED(c) (!((c) & (PTRS_PER_PMD - 1)))
+#define MSHV_MAP_FAULT_IN_PAGES HPAGE_PMD_NR
+
MODULE_AUTHOR("Microsoft");
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Microsoft Hyper-V root partition VMM interface /dev/mshv");
@@ -76,6 +79,11 @@ static int mshv_vp_mmap(struct file *file, struct vm_area_struct *vma);
static vm_fault_t mshv_vp_fault(struct vm_fault *vmf);
static int mshv_init_async_handler(struct mshv_partition *partition);
static void mshv_async_hvcall_handler(void *data, u64 *status);
+static struct mshv_mem_region
+ *mshv_partition_region_by_gfn(struct mshv_partition *pt, u64 gfn);
+static int mshv_region_remap_pages(struct mshv_mem_region *region,
+ u32 map_flags, u64 page_offset,
+ u64 page_count);
static const union hv_input_vtl input_vtl_zero;
static const union hv_input_vtl input_vtl_normal = {
@@ -602,14 +610,197 @@ static long mshv_run_vp_with_root_scheduler(struct mshv_vp *vp)
static_assert(sizeof(struct hv_message) <= MSHV_RUN_VP_BUF_SZ,
"sizeof(struct hv_message) must not exceed MSHV_RUN_VP_BUF_SZ");
+#ifdef CONFIG_X86_64
+
+#if defined(CONFIG_MMU_NOTIFIER)
+/**
+ * mshv_region_hmm_fault_and_lock - Handle HMM faults and lock the memory region
+ * @region: Pointer to the memory region structure
+ * @range: Pointer to the HMM range structure
+ *
+ * This function performs the following steps:
+ * 1. Reads the notifier sequence for the HMM range.
+ * 2. Acquires a read lock on the memory map.
+ * 3. Handles HMM faults for the specified range.
+ * 4. Releases the read lock on the memory map.
+ * 5. If successful, locks the memory region mutex.
+ * 6. Verifies if the notifier sequence has changed during the operation.
+ * If it has, releases the mutex and returns -EBUSY to match with
+ * hmm_range_fault() return code for repeating.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+static int mshv_region_hmm_fault_and_lock(struct mshv_mem_region *region,
+ struct hmm_range *range)
+{
+ int ret;
+
+ range->notifier_seq = mmu_interval_read_begin(range->notifier);
+ mmap_read_lock(region->mni.mm);
+ ret = hmm_range_fault(range);
+ mmap_read_unlock(region->mni.mm);
+ if (ret)
+ return ret;
+
+ mutex_lock(®ion->mutex);
+
+ if (mmu_interval_read_retry(range->notifier, range->notifier_seq)) {
+ mutex_unlock(®ion->mutex);
+ cond_resched();
+ return -EBUSY;
+ }
+
+ return 0;
+}
+
+/**
+ * mshv_region_range_fault - Handle memory range faults for a given region.
+ * @region: Pointer to the memory region structure.
+ * @page_offset: Offset of the page within the region.
+ * @page_count: Number of pages to handle.
+ *
+ * This function resolves memory faults for a specified range of pages
+ * within a memory region. It uses HMM (Heterogeneous Memory Management)
+ * to fault in the required pages and updates the region's page array.
+ *
+ * Return: 0 on success, negative error code on failure.
+ */
+static int mshv_region_range_fault(struct mshv_mem_region *region,
+ u64 page_offset, u64 page_count)
+{
+ struct hmm_range range = {
+ .notifier = ®ion->mni,
+ .default_flags = HMM_PFN_REQ_FAULT | HMM_PFN_REQ_WRITE,
+ };
+ unsigned long *pfns;
+ int ret;
+ u64 i;
+
+ pfns = kmalloc_array(page_count, sizeof(unsigned long), GFP_KERNEL);
+ if (!pfns)
+ return -ENOMEM;
+
+ range.hmm_pfns = pfns;
+ range.start = region->start_uaddr + page_offset * HV_HYP_PAGE_SIZE;
+ range.end = range.start + page_count * HV_HYP_PAGE_SIZE;
+
+ do {
+ ret = mshv_region_hmm_fault_and_lock(region, &range);
+ } while (ret == -EBUSY);
+
+ if (ret)
+ goto out;
+
+ for (i = 0; i < page_count; i++)
+ region->pages[page_offset + i] = hmm_pfn_to_page(pfns[i]);
+
+ if (PageHuge(region->pages[page_offset]))
+ region->flags.large_pages = true;
+
+ ret = mshv_region_remap_pages(region, region->hv_map_flags,
+ page_offset, page_count);
+
+ mutex_unlock(®ion->mutex);
+out:
+ kfree(pfns);
+ return ret;
+}
+#else /* CONFIG_MMU_NOTIFIER */
+static int mshv_region_range_fault(struct mshv_mem_region *region,
+ u64 page_offset, u64 page_count)
+{
+ return -ENODEV;
+}
+#endif /* CONFIG_MMU_NOTIFIER */
+
+static bool mshv_region_handle_gfn_fault(struct mshv_mem_region *region, u64 gfn)
+{
+ u64 page_offset, page_count;
+ int ret;
+
+ if (WARN_ON_ONCE(region->flags.range_pinned))
+ return false;
+
+ /* Align the page offset to the nearest MSHV_MAP_FAULT_IN_PAGES. */
+ page_offset = ALIGN_DOWN(gfn - region->start_gfn,
+ MSHV_MAP_FAULT_IN_PAGES);
+
+ /* Map more pages than requested to reduce the number of faults. */
+ page_count = min(region->nr_pages - page_offset,
+ MSHV_MAP_FAULT_IN_PAGES);
+
+ ret = mshv_region_range_fault(region, page_offset, page_count);
+
+ WARN_ONCE(ret,
+ "p%llu: GPA intercept failed: region %#llx-%#llx, gfn %#llx, page_offset %llu, page_count %llu\n",
+ region->partition->pt_id, region->start_uaddr,
+ region->start_uaddr + (region->nr_pages << HV_HYP_PAGE_SHIFT),
+ gfn, page_offset, page_count);
+
+ return !ret;
+}
+
+/**
+ * mshv_handle_gpa_intercept - Handle GPA (Guest Physical Address) intercepts.
+ * @vp: Pointer to the virtual processor structure.
+ *
+ * This function processes GPA intercepts by identifying the memory region
+ * corresponding to the intercepted GPA, aligning the page offset, and
+ * mapping the required pages. It ensures that the region is valid and
+ * handles faults efficiently by mapping multiple pages at once.
+ *
+ * Return: true if the intercept was handled successfully, false otherwise.
+ */
+static bool mshv_handle_gpa_intercept(struct mshv_vp *vp)
+{
+ struct mshv_partition *p = vp->vp_partition;
+ struct mshv_mem_region *region;
+ struct hv_x64_memory_intercept_message *msg;
+ u64 gfn;
+
+ msg = (struct hv_x64_memory_intercept_message *)
+ vp->vp_intercept_msg_page->u.payload;
+
+ gfn = HVPFN_DOWN(msg->guest_physical_address);
+
+ region = mshv_partition_region_by_gfn(p, gfn);
+ if (!region)
+ return false;
+
+ if (WARN_ON_ONCE(!region->flags.is_ram))
+ return false;
+
+ if (WARN_ON_ONCE(region->flags.range_pinned))
+ return false;
+
+ return mshv_region_handle_gfn_fault(region, gfn);
+}
+
+#else /* CONFIG_X86_64 */
+
+static bool mshv_handle_gpa_intercept(struct mshv_vp *vp) { return false; }
+
+#endif /* CONFIG_X86_64 */
+
+static bool mshv_vp_handle_intercept(struct mshv_vp *vp)
+{
+ switch (vp->vp_intercept_msg_page->header.message_type) {
+ case HVMSG_GPA_INTERCEPT:
+ return mshv_handle_gpa_intercept(vp);
+ }
+ return false;
+}
+
static long mshv_vp_ioctl_run_vp(struct mshv_vp *vp, void __user *ret_msg)
{
long rc;
- if (hv_scheduler_type == HV_SCHEDULER_TYPE_ROOT)
- rc = mshv_run_vp_with_root_scheduler(vp);
- else
- rc = mshv_run_vp_with_hyp_scheduler(vp);
+ do {
+ if (hv_scheduler_type == HV_SCHEDULER_TYPE_ROOT)
+ rc = mshv_run_vp_with_root_scheduler(vp);
+ else
+ rc = mshv_run_vp_with_hyp_scheduler(vp);
+ } while (rc == 0 && mshv_vp_handle_intercept(vp));
if (rc)
return rc;
@@ -1209,6 +1400,110 @@ mshv_partition_region_by_uaddr(struct mshv_partition *partition, u64 uaddr)
return NULL;
}
+#if defined(CONFIG_MMU_NOTIFIER)
+static void mshv_region_movable_fini(struct mshv_mem_region *region)
+{
+ if (region->flags.range_pinned)
+ return;
+
+ mmu_interval_notifier_remove(®ion->mni);
+}
+
+/**
+ * mshv_region_interval_invalidate - Invalidate a range of memory region
+ * @mni: Pointer to the mmu_interval_notifier structure
+ * @range: Pointer to the mmu_notifier_range structure
+ * @cur_seq: Current sequence number for the interval notifier
+ *
+ * This function invalidates a memory region by remapping its pages with
+ * no access permissions. It locks the region's mutex to ensure thread safety
+ * and updates the sequence number for the interval notifier. If the range
+ * is blockable, it uses a blocking lock; otherwise, it attempts a non-blocking
+ * lock and returns false if unsuccessful.
+ *
+ * NOTE: Failure to invalidate a region is a serious error, as the pages will
+ * be considered freed while they are still mapped by the hypervisor.
+ * Any attempt to access such pages will likely crash the system.
+ *
+ * Return: true if the region was successfully invalidated, false otherwise.
+ */
+static bool
+mshv_region_interval_invalidate(struct mmu_interval_notifier *mni,
+ const struct mmu_notifier_range *range,
+ unsigned long cur_seq)
+{
+ struct mshv_mem_region *region = container_of(mni,
+ struct mshv_mem_region,
+ mni);
+ u64 page_offset, page_count;
+ unsigned long mstart, mend;
+ int ret;
+
+ if (mmu_notifier_range_blockable(range))
+ mutex_lock(®ion->mutex);
+ else if (!mutex_trylock(®ion->mutex))
+ goto out_fail;
+
+ mmu_interval_set_seq(mni, cur_seq);
+
+ mstart = max(range->start, region->start_uaddr);
+ mend = min(range->end, region->start_uaddr +
+ (region->nr_pages << HV_HYP_PAGE_SHIFT));
+
+ page_offset = HVPFN_DOWN(mstart - region->start_uaddr);
+ page_count = HVPFN_DOWN(mend - mstart);
+
+ ret = mshv_region_remap_pages(region, HV_MAP_GPA_NO_ACCESS,
+ page_offset, page_count);
+ if (ret)
+ goto out_fail;
+
+ mshv_region_invalidate_pages(region, page_offset, page_count);
+
+ mutex_unlock(®ion->mutex);
+
+ return true;
+
+out_fail:
+ WARN_ONCE(ret,
+ "Failed to invalidate region %#llx-%#llx (range %#lx-%#lx, event: %u, pages %#llx-%#llx, mm: %#llx): %d\n",
+ region->start_uaddr,
+ region->start_uaddr + (region->nr_pages << HV_HYP_PAGE_SHIFT),
+ range->start, range->end, range->event,
+ page_offset, page_offset + page_count - 1, (u64)range->mm, ret);
+ return false;
+}
+
+static const struct mmu_interval_notifier_ops mshv_region_mni_ops = {
+ .invalidate = mshv_region_interval_invalidate,
+};
+
+static bool mshv_region_movable_init(struct mshv_mem_region *region)
+{
+ int ret;
+
+ ret = mmu_interval_notifier_insert(®ion->mni, current->mm,
+ region->start_uaddr,
+ region->nr_pages << HV_HYP_PAGE_SHIFT,
+ &mshv_region_mni_ops);
+ if (ret)
+ return false;
+
+ mutex_init(®ion->mutex);
+
+ return true;
+}
+#else
+static inline void mshv_region_movable_fini(struct mshv_mem_region *region)
+{
+}
+
+static inline bool mshv_region_movable_init(struct mshv_mem_region *region)
+{
+ return false;
+}
+#endif
+
/*
* NB: caller checks and makes sure mem->size is page aligned
* Returns: 0 with regionpp updated on success, or -errno
@@ -1241,9 +1536,14 @@ static int mshv_partition_create_region(struct mshv_partition *partition,
if (mem->flags & BIT(MSHV_SET_MEM_BIT_EXECUTABLE))
region->hv_map_flags |= HV_MAP_GPA_EXECUTABLE;
- /* Note: large_pages flag populated when we pin the pages */
- if (!is_mmio)
- region->flags.range_pinned = true;
+ /* Note: large_pages flag populated when pages are allocated. */
+ if (!is_mmio) {
+ region->flags.is_ram = true;
+
+ if (mshv_partition_encrypted(partition) ||
+ !mshv_region_movable_init(region))
+ region->flags.range_pinned = true;
+ }
region->partition = partition;
@@ -1363,9 +1663,16 @@ mshv_map_user_memory(struct mshv_partition *partition,
if (is_mmio)
ret = hv_call_map_mmio_pages(partition->pt_id, mem.guest_pfn,
mmio_pfn, HVPFN_DOWN(mem.size));
- else
+ else if (region->flags.range_pinned)
ret = mshv_prepare_pinned_region(region);
-
+ else
+ /*
+ * For non-pinned regions, remap with no access to let the
+ * hypervisor track dirty pages, enabling pre-copy live
+ * migration.
+ */
+ ret = mshv_region_remap_pages(region, HV_MAP_GPA_NO_ACCESS,
+ 0, region->nr_pages);
if (ret)
goto errout;
@@ -1388,6 +1695,9 @@ static void mshv_partition_destroy_region(struct mshv_mem_region *region)
hlist_del(®ion->hnode);
+ if (region->flags.is_ram)
+ mshv_region_movable_fini(region);
+
if (mshv_partition_encrypted(partition)) {
ret = mshv_partition_region_share(region);
if (ret) {
^ permalink raw reply related
* RE: [PATCH hyperv-next v6 00/17] Confidential VMBus
From: Michael Kelley @ 2025-10-06 16:55 UTC (permalink / raw)
To: Roman Kisel, arnd@arndb.de, bp@alien8.de, corbet@lwn.net,
dave.hansen@linux.intel.com, decui@microsoft.com,
haiyangz@microsoft.com, hpa@zytor.com, kys@microsoft.com,
mikelley@microsoft.com, mingo@redhat.com, tglx@linutronix.de,
Tianyu.Lan@microsoft.com, wei.liu@kernel.org, x86@kernel.org,
linux-hyperv@vger.kernel.org, linux-doc@vger.kernel.org,
linux-kernel@vger.kernel.org, linux-arch@vger.kernel.org
Cc: benhill@microsoft.com, bperkins@microsoft.com,
sunilmut@microsoft.com
In-Reply-To: <20251003222710.6257-1-romank@linux.microsoft.com>
From: Roman Kisel <romank@linux.microsoft.com> Sent: Friday, October 3, 2025 3:27 PM
>
> Greetings everyone,
>
> We've got to the 6th version of the patch series, and the full changelog
> is at the end of the cover letter. I addressed feedback from
> Michael and Wei on the previous version of the patch series.
>
> Since v5, the fallback mechanism for establishing the VMBus connection
> is no longer used as the availability of the Confidential VMBus is
> now indicated by a bit in the Virtualization Stack (VS) CPUID leaf.
> The v6 patch series breaks that out into a separate patch seizing
> the opportunity to refactor the code that uses the same leaf.
>
> That is obviously an x86_64 specific technique. On ARM64, the
> Confidential VMBus is expected to be required once support for ARM CCA is
> implemented. Despite that change, the functions for getting and setting
> registers via paravisor remain fallible.
This statement seems to contradict your description of the v6
changes further down in this cover letter:
- Gave another thought to the fallible routines for getting and setting
SynIC registers via paravisor introduced in the patch series, and after
Michael's feedback decided to make them infallible
Patches 4 and 12 of this series also implement "infallible".
> That provides a clearer root cause
> for failures instead of printing messages about unchecked MSR accesses.
> That might seem as not needed with the paravisors run in Azure (OpenHCL
> and the TrustedLauch aka HCL paravisor). However, if someone decides to
> implement their own or tweak the exisiting one, this will help with debugging.
>
> TLDR; is that these patches are for the Hyper-V guests, and the patches
> allow to keep data flowing from physical devices into the guests encrypted
> at the CPU level so that neither the root/host partition nor the hypervisor
> can access the data being processed (they only "see" the encrypted/garbled
> data) unless the guest decides to share it. The changes are backward compatible
> with older systems, and their full potential is realized on hardware that
> supports memory encryption.
>
> These features also require running a paravisor, such as
> OpenHCL (https://github.com/microsoft/openvmm) used in Azure. Another
> implementation of the functionality available in this patch set is
> available in the Hyper-V UEFI: https://github.com/microsoft/mu_msvm.
>
> A more detailed description of the patches follows.
>
> The guests running on Hyper-V can be confidential where the memory and the
> register content are encrypted, provided that the hardware supports that
> (currently support for AMD SEV-SNP and Intel TDX is implemented) and the guest
> is capable of using these features. The confidential guests cannot be
> introspected by the host nor the hypervisor without the guest sharing the
> memory contents upon doing which the memory is decrypted.
>
> In the confidential guests, neither the host nor the hypervisor need to be
> trusted, and the guests processing sensitive data can take advantage of that.
>
> Not trusting the host and the hypervisor (removing them from the Trusted
> Computing Base aka TCB) necessitates that the method of communication
> between the host and the guest be changed. Here is the data flow for a
> conventional and the confidential VMBus connections (`C` stands for the
> client or VSC, `S` for the server or VSP, the `DEVICE` is a physical one,
> might be with multiple virtual functions):
>
> 1. Without the paravisor the devices are connected to the host, and the
> host provides the device emulation or translation to the guest:
>
> +---- GUEST ----+ +----- DEVICE ----+ +----- HOST -----+
> | | | | | |
> | | | | | |
> | | | ========== |
> | | | | | |
> | | | | | |
> | | | | | |
> +----- C -------+ +-----------------+ +------- S ------+
> || ||
> || ||
> +------||------------------ VMBus --------------------------||------+
> | Interrupts, MMIO |
> +-------------------------------------------------------------------+
>
> 2. With the paravisor, the devices are connected to the paravisor, and
> the paravisor provides the device emulation or translation to the guest.
> The guest doesn't communicate with the host directly, and the guest
> communicates with the paravisor via the VMBus. The host is not trusted
> in this model, and the paravisor is trusted:
>
> +---- GUEST --------------- VTL0 ------+ +-- DEVICE --+
> | | | |
> | +- PARAVISOR --------- VTL2 -----+ | | |
> | | +-- VMBus Relay ------+ ====+================ |
> | | | Interrupts, MMIO | | | | |
> | | +-------- S ----------+ | | +------------+
> | | || | |
> | +---------+ || | |
> | | Linux | || OpenHCL | |
> | | kernel | || | |
> | +---- C --+-----||---------------+ |
> | || || |
> +-------++------- C -------------------+ +------------+
> || | HOST |
> || +---- S -----+
> +-------||----------------- VMBus ---------------------------||-----+
> | Interrupts, MMIO |
> +-------------------------------------------------------------------+
>
> Note that in the second case the guest doesn't need to share the memory
> with the host as it communicates only with the paravisor within their
> partition boundary. That is precisely the raison d'etre and the value
> proposition of this patch series: equip the confidential guest to use
> private (encrypted) memory and rely on the paravisor when this is
> available to be more secure.
>
> An implementation of the VMBus relay that offers the Confidential VMBus
> channels is available in the OpenVMM project as a part of the OpenHCL
> paravisor. Please refer to
>
> * https://openvmm.dev/guide/, and
> * https://github.com/microsoft/openvmm
>
> for more information about the OpenHCL paravisor. A VMBus client
> that can work with the Confidential VMBus is available in the
> open-source Hyper-V UEFI: https://github.com/microsoft/mu_msvm.
>
> I'd like to thank the following people for their help with this
> patch series:
>
> * Dexuan for help with validation and the fruitful discussions,
> * Easwar for reviewing the refactoring of the page allocating and
> freeing in `hv.c`,
> * John and Sven for the design,
> * Mike for helping to avoid pitfalls when dealing with the GFP flags,
> * Sven for blazing the trail and implementing the design in few
> codebases.
>
> I made sure to validate the patch series on
>
> {TrustedLaunch(x86_64), OpenHCL} x
> {SNP(x86_64), TDX(x86_64), No hardware isolation, No paravisor} x
> {VMBus 5.0, VMBus 6.0} x
> {arm64, x86_64}.
>
> [V6]
> - Rebased onto the latest hyperv-next tree.
>
> - Gave another thought to the fallible routines for getting and setting
> SynIC registers via paravisor introduced in the patch series, and after
> Michael's feedback decided to make them infallible as now we have the
> CPUID bit to indicate the availability of the Confidential VMBus. That
> simplifies the code and makes it clearer and more robust - a reflection
> of the improvements in the design throught the patch series iterations.
> - Removed the sentence discussing the fallback mechanism in the Documentation
> as it is no longer relevant.
> **Thank you, Michael!**
>
> - Avoided using the macro'es for (un)masking the proxy bit thanks to
> `union hv_synic_sint`.
> **Thank you, Wei!**
>
> [V5] https://lore.kernel.org/linux-hyperv/20250828010557.123869-1-romank@linux.microsoft.com/
> - Rebased onto the latest hyperv-next tree.
>
> - Fixed build issues with the configs provided by the kernel robot.
> **Thank you, kernel robot!**
>
> - Fixed the potential NULL deref in a failure path.
> **Thank you, Michael!**
>
> - Removed the added blurb from the vmbus_drv.c with taxonomy of Hyper-V VMs
> that was providing reasons for the trade-offs in the fallback code. That
> code is no longer needed.
>
> [V4] https://lore.kernel.org/linux-hyperv/20250714221545.5615-1-romank@linux.microsoft.com/
> - Rebased the patch series on top of the latest hyperv-next branch,
> applying changes as needed.
>
> - Fixed typos and clarifications all around the patch series.
> - Added clarifications in the patch 7 for `ms_hyperv.paravisor_present && !vmbus_is_confidential()`
> and using hypercalls vs SNP or TDX specific protocols.
> **Thank you, Alok!**
>
> - Trim the Documentation changes to 80 columns.
> **Thank you, Randy!**
>
> - Make sure adhere to the RST format, actually built the PDF docs
> and made sure the layout was correct.
> **Thank you, Jon!**
>
> - Better section order in Documentation.
> - Fixed the commit descriptions where suggested.
> - Moved EOI/EOM signaling for the confidential VMBus to the specialized function.
> - Removed the unused `cpu` parameters.
> - Clarified comments in the `hv_per_cpu_context` struct
> - Explicitly test for NULL and only call `iounmap()` if non-NULL instead of
> using `munmap()`.
> - Don't deallocate SynIC pages in the CPU online and offline paths.
> - Made sure the post page needs to be allocated for the future.
> - Added comments to describe trade-offs.
> **Thank you, Michael!**
>
> [V3] https://lore.kernel.org/linux-hyperv/20250604004341.7194-1-romank@linux.microsoft.com/
> - The patch series is rebased on top of the latest hyperv-next branch.
> - Reworked the "wiring" diagram in the cover letter, added links to the
> OpenVMM project and the OpenHCL paravisor.
>
> - More precise wording in the comments and clearer code.
> **Thank you, Alok!**
>
> - Reworked the documentation patch.
> - Split the patchset into much more granular patches.
> - Various fixes and improvements throughout the patch series.
> **Thank you, Michael!**
>
> [V2] https://lore.kernel.org/linux-hyperv/20250511230758.160674-1-romank@linux.microsoft.com/
> - The patch series is rebased on top of the latest hyperv-next branch.
>
> - Better wording in the commit messages and the Documentation.
> **Thank you, Alok and Wei!**
>
> - Removed the patches 5 and 6 concerning turning bounce buffering off from
> the previous version of the patch series as they were found to be
> architecturally unsound. The value proposition of the patch series is not
> diminished by this removal: these patches were an optimization and only for
> the storage (for the simplicity sake) but not for the network. These changes
> might be proposed in the future again after revolving the issues.
> ** Thanks you, Christoph, Dexuan, Dan, Michael, James, Robin! **
>
> [V1] https://lore.kernel.org/linux-hyperv/20250409000835.285105-1-romank@linux.microsoft.com/
>
> Roman Kisel (17):
> Documentation: hyperv: Confidential VMBus
> Drivers: hv: VMBus protocol version 6.0
> arch/x86: mshyperv: Discover Confidential VMBus availability
> arch: hyperv: Get/set SynIC synth.registers via paravisor
> arch/x86: mshyperv: Trap on access for some synthetic MSRs
> Drivers: hv: Rename fields for SynIC message and event pages
> Drivers: hv: Allocate the paravisor SynIC pages when required
> Drivers: hv: Post messages through the confidential VMBus if available
> Drivers: hv: remove stale comment
> Drivers: hv: Check message and event pages for non-NULL before
> iounmap()
> Drivers: hv: Rename the SynIC enable and disable routines
> Drivers: hv: Functions for setting up and tearing down the paravisor
> SynIC
> Drivers: hv: Allocate encrypted buffers when requested
> Drivers: hv: Free msginfo when the buffer fails to decrypt
> Drivers: hv: Support confidential VMBus channels
> Drivers: hv: Set the default VMBus version to 6.0
> Drivers: hv: Support establishing the confidential VMBus connection
>
> Documentation/virt/hyperv/coco.rst | 139 ++++++++++-
> arch/x86/kernel/cpu/mshyperv.c | 77 ++++--
> drivers/hv/channel.c | 73 ++++--
> drivers/hv/channel_mgmt.c | 27 ++-
> drivers/hv/connection.c | 6 +-
> drivers/hv/hv.c | 372 +++++++++++++++++++----------
> drivers/hv/hv_common.c | 16 ++
> drivers/hv/hyperv_vmbus.h | 75 +++++-
> drivers/hv/mshv_root.h | 2 +-
> drivers/hv/mshv_synic.c | 6 +-
> drivers/hv/ring_buffer.c | 5 +-
> drivers/hv/vmbus_drv.c | 186 ++++++++++-----
> include/asm-generic/mshyperv.h | 45 +---
> include/hyperv/hvgdk_mini.h | 1 +
> include/linux/hyperv.h | 69 ++++--
> 15 files changed, 793 insertions(+), 306 deletions(-)
>
Nice! The net lines of code added is now 487, vs. 591
lines added in v5 of this series.
Modulo the contradiction above in this cover letter, the two typos in
the documentation in Patch 1, and the simple fix for the error reported
by the kernel test robot for Patch 5, I'm happy with this entire series.
For the series,
Reviewed-by: Michael Kelley <mhklinux@outlook.com>
^ permalink raw reply
* RE: [PATCH hyperv-next v6 01/17] Documentation: hyperv: Confidential VMBus
From: Michael Kelley @ 2025-10-06 16:55 UTC (permalink / raw)
To: Roman Kisel, arnd@arndb.de, bp@alien8.de, corbet@lwn.net,
dave.hansen@linux.intel.com, decui@microsoft.com,
haiyangz@microsoft.com, hpa@zytor.com, kys@microsoft.com,
mikelley@microsoft.com, mingo@redhat.com, tglx@linutronix.de,
Tianyu.Lan@microsoft.com, wei.liu@kernel.org, x86@kernel.org,
linux-hyperv@vger.kernel.org, linux-doc@vger.kernel.org,
linux-kernel@vger.kernel.org, linux-arch@vger.kernel.org
Cc: benhill@microsoft.com, bperkins@microsoft.com,
sunilmut@microsoft.com
In-Reply-To: <20251003222710.6257-2-romank@linux.microsoft.com>
From: Roman Kisel <romank@linux.microsoft.com> Sent: Friday, October 3, 2025 3:27 PM
>
> Define what the confidential VMBus is and describe what advantages
> it offers on the capable hardware.
>
> Signed-off-by: Roman Kisel <romank@linux.microsoft.com>
> Reviewed-by: Alok Tiwari <alok.a.tiwari@oracle.com>
> Reviewed-by: Michael Kelley <mhklinux@outlook.com>
> ---
> Documentation/virt/hyperv/coco.rst | 139 ++++++++++++++++++++++++++++-
> 1 file changed, 138 insertions(+), 1 deletion(-)
>
> diff --git a/Documentation/virt/hyperv/coco.rst b/Documentation/virt/hyperv/coco.rst
> index c15d6fe34b4e..e00d94d9f88f 100644
> --- a/Documentation/virt/hyperv/coco.rst
> +++ b/Documentation/virt/hyperv/coco.rst
> @@ -178,7 +178,7 @@ These Hyper-V and VMBus memory pages are marked as decrypted:
>
> * VMBus monitor pages
>
> -* Synthetic interrupt controller (synic) related pages (unless supplied by
> +* Synthetic interrupt controller (SynIC) related pages (unless supplied by
> the paravisor)
>
> * Per-cpu hypercall input and output pages (unless running with a paravisor)
> @@ -232,6 +232,143 @@ with arguments explicitly describing the access. See
> _hv_pcifront_read_config() and _hv_pcifront_write_config() and the
> "use_calls" flag indicating to use hypercalls.
>
> +Confidential VMBus
> +------------------
> +The confidential VMBus enables the confidential guest not to interact with
> +the untrusted host partition and the untrusted hypervisor. Instead, the guest
> +relies on the trusted paravisor to communicate with the devices processing
> +sensitive data. The hardware (SNP or TDX) encrypts the guest memory and the
> +register state while measuring the paravisor image using the platform security
> +processor to ensure trusted and confidential computing.
> +
> +Confidential VMBus provides a secure communication channel between the guest
> +and the paravisor, ensuring that sensitive data is protected from hypervisor-
> +level access through memory encryption and register state isolation.
> +
> +Confidential VMBus is an extension of Confidential Computing (CoCo) VMs
> +(a.k.a. "Isolated" VMs in Hyper-V terminology). Without Confidential VMBus,
> +guest VMBus device drivers (the "VSC"s in VMBus terminology) communicate
> +with VMBus servers (the VSPs) running on the Hyper-V host. The
> +communication must be through memory that has been decrypted so the
> +host can access it. With Confidential VMBus, one or more of the VSPs reside
> +in the trusted paravisor layer in the guest VM. Since the paravisor layer also
> +operates in encrypted memory, the memory used for communication with
> +such VSPs does not need to be decrypted and thereby exposed to the
> +Hyper-V host. The paravisor is responsible for communicating securely
> +with the Hyper-V host as necessary.
> +
> +The data is transferred directly between the VM and a vPCI device (a.k.a.
> +a PCI pass-thru device, see :doc:`vpci`) that is directly assigned to VTL2
> +and that supports encrypted memory. In such a case, neither the host partition
> +nor the hypervisor has any access to the data. The guest needs to establish
> +a VMBus connection only with the paravisor for the channels that process
> +sensitive data, and the paravisor abstracts the details of communicating
> +with the specific devices away providing the guest with the well-established
> +VSP (Virtual Service Provider) interface that has had support in the Hyper-V
> +drivers for a decade.
> +
> +In the case the device does not support encrypted memory, the paravisor
> +provides bounce-buffering, and although the data is not encrypted, the backing
> +pages aren't mapped into the host partition through SLAT. While not impossible,
> +it becomes much more difficult for the host partition to exfiltrate the data
> +than it would be with a conventional VMBus connection where the host partition
> +has direct access to the memory used for communication.
> +
> +Here is the data flow for a conventional VMBus connection (`C` stands for the
> +client or VSC, `S` for the server or VSP, the `DEVICE` is a physical one, might
> +be with multiple virtual functions)::
> +
> + +---- GUEST ----+ +----- DEVICE ----+ +----- HOST -----+
> + | | | | | |
> + | | | | | |
> + | | | ========== |
> + | | | | | |
> + | | | | | |
> + | | | | | |
> + +----- C -------+ +-----------------+ +------- S ------+
> + || ||
> + || ||
> + +------||------------------ VMBus --------------------------||------+
> + | Interrupts, MMIO |
> + +-------------------------------------------------------------------+
> +
> +and the Confidential VMBus connection::
> +
> + +---- GUEST --------------- VTL0 ------+ +-- DEVICE --+
> + | | | |
> + | +- PARAVISOR --------- VTL2 -----+ | | |
> + | | +-- VMBus Relay ------+ ====+================ |
> + | | | Interrupts, MMIO | | | | |
> + | | +-------- S ----------+ | | +------------+
> + | | || | |
> + | +---------+ || | |
> + | | Linux | || OpenHCL | |
> + | | kernel | || | |
> + | +---- C --+-----||---------------+ |
> + | || || |
> + +-------++------- C -------------------+ +------------+
> + || | HOST |
> + || +---- S -----+
> + +-------||----------------- VMBus ---------------------------||-----+
> + | Interrupts, MMIO |
> + +-------------------------------------------------------------------+
> +
> +An implementation of the VMBus relay that offers the Confidential VMBus
> +channels is available in the OpenVMM project as a part of the OpenHCL
> +paravisor. Please refer to
> +
> + *
> https://openvmm.dev/
> %2F&data=05%7C02%7C%7Ceb6de4b7295c4e8a8ab908de02cc219c%7C84df9e7fe9
> f640afb435aaaaaaaaaaaa%7C1%7C0%7C638951272987274372%7CUnknown%7CT
> WFpbGZsb3d8eyJFbXB0eU1hcGkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIlAiOiJXaW4zMiIs
> IkFOIjoiTWFpbCIsIldUIjoyfQ%3D%3D%7C0%7C%7C%7C&sdata=9iTXf52zezdCCpMv4
> wv1S1AkWvnokRyXJD7hF3vU6h4%3D&reserved=0, and
> + *
> https://github.com/
> microsoft%2Fopenvmm&data=05%7C02%7C%7Ceb6de4b7295c4e8a8ab908de02cc2
> 19c%7C84df9e7fe9f640afb435aaaaaaaaaaaa%7C1%7C0%7C638951272987295766
> %7CUnknown%7CTWFpbGZsb3d8eyJFbXB0eU1hcGkiOnRydWUsIlYiOiIwLjAuMDAwM
> CIsIlAiOiJXaW4zMiIsIkFOIjoiTWFpbCIsIldUIjoyfQ%3D%3D%7C0%7C%7C%7C&sdata=r
> Dp0el9NwDfSpRLSTmdLQgIqXqgLxrFrWJFcZcgb3Zk%3D&reserved=0
> +
> +for more information about the OpenHCL paravisor.
> +
> +A guest that is running with a paravisor must determine at runtime if
> +Confidential VMBus is supported by the current paravisor.The x86_64-specific
Nit: Missing a space before "The".
> +approach relies on the CPUID Virtualization Stack leaf; the ARM64 implementation
> +is expected to support the Confidential VMBus unconditionally when running
> +the ARM CCA guests.
s/the ARM CCA/ARM CCA/
> +
> +Confidential VMBus is a characteristic of the VMBus connection as a whole,
> +and of each VMBus channel that is created. When a Confidential VMBus
> +connection is established, the paravisor provides the guest the message-passing
> +path that is used for VMBus device creation and deletion, and it provides a
> +per-CPU synthetic interrupt controller (SynIC) just like the SynIC that is
> +offered by the Hyper-V host. Each VMBus device that is offered to the guest
> +indicates the degree to which it participates in Confidential VMBus. The offer
> +indicates if the device uses encrypted ring buffers, and if the device uses
> +encrypted memory for DMA that is done outside the ring buffer. These settings
> +may be different for different devices using the same Confidential VMBus
> +connection.
> +
> +Although these settings are separate, in practice it'll always be encrypted
> +ring buffer only, or both encrypted ring buffer and external data. If a channel
> +is offered by the paravisor with confidential VMBus, the ring buffer can always
> +be encrypted since it's strictly for communication between the VTL2 paravisor
> +and the VTL0 guest. However, other memory regions are often used for e.g. DMA,
> +so they need to be accessible by the underlying hardware, and must be
> +unencrypted (unless the device supports encrypted memory). Currently, there are
> +not any VSPs in OpenHCL that support encrypted external memory, but future
> +versions are expected to enable this capability.
> +
> +Because some devices on a Confidential VMBus may require decrypted ring buffers
> +and DMA transfers, the guest must interact with two SynICs -- the one provided
> +by the paravisor and the one provided by the Hyper-V host when Confidential
> +VMBus is not offered. Interrupts are always signaled by the paravisor SynIC,
> +but the guest must check for messages and for channel interrupts on both SynICs.
> +
> +In the case of a confidential VMBus, regular SynIC access by the guest is
> +intercepted by the paravisor (this includes various MSRs such as the SIMP and
> +SIEFP, as well as hypercalls like HvPostMessage and HvSignalEvent). If the
> +guest actually wants to communicate with the hypervisor, it has to use special
> +mechanisms (GHCB page on SNP, or tdcall on TDX). Messages can be of either
> +kind: with confidential VMBus, messages use the paravisor SynIC, and if the
> +guest chose to communicate directly to the hypervisor, they use the hypervisor
> +SynIC. For interrupt signaling, some channels may be running on the host
> +(non-confidential, using the VMBus relay) and use the hypervisor SynIC, and
> +some on the paravisor and use its SynIC. The RelIDs are coordinated by the
> +OpenHCL VMBus server and are guaranteed to be unique regardless of whether
> +the channel originated on the host or the paravisor.
> +
> load_unaligned_zeropad()
> ------------------------
> When transitioning memory between encrypted and decrypted, the caller of
> --
> 2.43.0
>
^ permalink raw reply
* RE: [PATCH hyperv-next v6 05/17] arch/x86: mshyperv: Trap on access for some synthetic MSRs
From: Michael Kelley @ 2025-10-06 16:55 UTC (permalink / raw)
To: Roman Kisel, arnd@arndb.de, bp@alien8.de, corbet@lwn.net,
dave.hansen@linux.intel.com, decui@microsoft.com,
haiyangz@microsoft.com, hpa@zytor.com, kys@microsoft.com,
mikelley@microsoft.com, mingo@redhat.com, tglx@linutronix.de,
Tianyu.Lan@microsoft.com, wei.liu@kernel.org, x86@kernel.org,
linux-hyperv@vger.kernel.org, linux-doc@vger.kernel.org,
linux-kernel@vger.kernel.org, linux-arch@vger.kernel.org
Cc: benhill@microsoft.com, bperkins@microsoft.com,
sunilmut@microsoft.com
In-Reply-To: <20251003222710.6257-6-romank@linux.microsoft.com>
From: Roman Kisel <romank@linux.microsoft.com> Sent: Friday, October 3, 2025 3:27 PM
>
> hv_set_non_nested_msr() has special handling for SINT MSRs
> when a paravisor is present. In addition to updating the MSR on the
> host, the mirror MSR in the paravisor is updated, including with the
> proxy bit. But with Confidential VMBus, the proxy bit must not be
> used, so add a special case to skip it.
>
> Signed-off-by: Roman Kisel <romank@linux.microsoft.com>
> Reviewed-by: Alok Tiwari <alok.a.tiwari@oracle.com>
> Reviewed-by: Tianyu Lan <tiala@microsoft.com>
> ---
> arch/x86/kernel/cpu/mshyperv.c | 29 +++++++++++++++++++++++++----
> drivers/hv/hv_common.c | 5 +++++
> include/asm-generic/mshyperv.h | 1 +
> 3 files changed, 31 insertions(+), 4 deletions(-)
>
> diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c
> index af5a3bbbca9f..b410b930938a 100644
> --- a/arch/x86/kernel/cpu/mshyperv.c
> +++ b/arch/x86/kernel/cpu/mshyperv.c
> @@ -28,6 +28,7 @@
> #include <asm/apic.h>
> #include <asm/timer.h>
> #include <asm/reboot.h>
> +#include <asm/msr.h>
> #include <asm/nmi.h>
> #include <clocksource/hyperv_timer.h>
> #include <asm/msr.h>
> @@ -38,6 +39,12 @@
> bool hv_nested;
> struct ms_hyperv_info ms_hyperv;
>
> +/*
> + * When running with the paravisor, controls proxying the synthetic interrupts
> + * from the host
> + */
> +static bool hv_para_sint_proxy;
This needs to move down a few lines and be under the #if IS_ENABLED(CONFIG_HYPERV)
in order to eliminate the "unused variable" warning reported by the kernel test robot.
> +
> /* Used in modules via hv_do_hypercall(): see arch/x86/include/asm/mshyperv.h */
> bool hyperv_paravisor_present __ro_after_init;
> EXPORT_SYMBOL_GPL(hyperv_paravisor_present);
> @@ -79,17 +86,31 @@ EXPORT_SYMBOL_GPL(hv_get_non_nested_msr);
> void hv_set_non_nested_msr(unsigned int reg, u64 value)
> {
> if (hv_is_synic_msr(reg) && ms_hyperv.paravisor_present) {
> + /* The hypervisor will get the intercept. */
> hv_ivm_msr_write(reg, value);
>
> - /* Write proxy bit via wrmsl instruction */
> - if (hv_is_sint_msr(reg))
> - wrmsrq(reg, value | 1 << 20);
> + /* Using wrmsrq so the following goes to the paravisor. */
> + if (hv_is_sint_msr(reg)) {
> + union hv_synic_sint sint = { .as_uint64 = value };
> +
> + sint.proxy = hv_para_sint_proxy;
> + native_wrmsrq(reg, sint.as_uint64);
> + }
> } else {
> - wrmsrq(reg, value);
> + native_wrmsrq(reg, value);
> }
> }
> EXPORT_SYMBOL_GPL(hv_set_non_nested_msr);
>
> +/*
> + * Enable or disable proxying synthetic interrupts
> + * to the paravisor.
> + */
> +void hv_para_set_sint_proxy(bool enable)
> +{
> + hv_para_sint_proxy = enable;
> +}
> +
> /*
> * Get the SynIC register value from the paravisor.
> */
> diff --git a/drivers/hv/hv_common.c b/drivers/hv/hv_common.c
> index 8756ca834546..1a5c7a358971 100644
> --- a/drivers/hv/hv_common.c
> +++ b/drivers/hv/hv_common.c
> @@ -716,6 +716,11 @@ u64 __weak hv_tdx_hypercall(u64 control, u64 param1, u64
> param2)
> }
> EXPORT_SYMBOL_GPL(hv_tdx_hypercall);
>
> +void __weak hv_para_set_sint_proxy(bool enable)
> +{
> +}
> +EXPORT_SYMBOL_GPL(hv_para_set_sint_proxy);
> +
> u64 __weak hv_para_get_synic_register(unsigned int reg)
> {
> return ~0ULL;
> diff --git a/include/asm-generic/mshyperv.h b/include/asm-generic/mshyperv.h
> index c010059f1518..3955ba6d60b8 100644
> --- a/include/asm-generic/mshyperv.h
> +++ b/include/asm-generic/mshyperv.h
> @@ -298,6 +298,7 @@ bool hv_is_isolation_supported(void);
> bool hv_isolation_type_snp(void);
> u64 hv_ghcb_hypercall(u64 control, void *input, void *output, u32 input_size);
> u64 hv_tdx_hypercall(u64 control, u64 param1, u64 param2);
> +void hv_para_set_sint_proxy(bool enable);
> u64 hv_para_get_synic_register(unsigned int reg);
> void hv_para_set_synic_register(unsigned int reg, u64 val);
> void hyperv_cleanup(void);
> --
> 2.43.0
>
^ permalink raw reply
* RE: [PATCH v4 3/5] Drivers: hv: Batch GPA unmap operations to improve large region performance
From: Michael Kelley @ 2025-10-06 17:09 UTC (permalink / raw)
To: Stanislav Kinsburskii, kys@microsoft.com, haiyangz@microsoft.com,
wei.liu@kernel.org, decui@microsoft.com
Cc: linux-hyperv@vger.kernel.org, linux-kernel@vger.kernel.org
In-Reply-To: <175976318688.16834.16198650808431263017.stgit@skinsburskii-cloud-desktop.internal.cloudapp.net>
From: Stanislav Kinsburskii <skinsburskii@linux.microsoft.com> Sent: Monday, October 6, 2025 8:06 AM
>
> Reduce overhead when unmapping large memory regions by batching GPA unmap
> operations in 2MB-aligned chunks.
>
> Use a dedicated constant for batch size to improve code clarity and
> maintainability.
>
> Signed-off-by: Stanislav Kinsburskii <skinsburskii@linux.microsoft.com>
> ---
> drivers/hv/mshv_root.h | 2 ++
> drivers/hv/mshv_root_hv_call.c | 2 +-
> drivers/hv/mshv_root_main.c | 28 +++++++++++++++++++++++++---
> 3 files changed, 28 insertions(+), 4 deletions(-)
>
> diff --git a/drivers/hv/mshv_root.h b/drivers/hv/mshv_root.h
> index e3931b0f12693..97e64d5341b6e 100644
> --- a/drivers/hv/mshv_root.h
> +++ b/drivers/hv/mshv_root.h
> @@ -32,6 +32,8 @@ static_assert(HV_HYP_PAGE_SIZE == MSHV_HV_PAGE_SIZE);
>
> #define MSHV_PIN_PAGES_BATCH_SIZE (0x10000000ULL / HV_HYP_PAGE_SIZE)
>
> +#define MSHV_MAX_UNMAP_GPA_PAGES 512
> +
> struct mshv_vp {
> u32 vp_index;
> struct mshv_partition *vp_partition;
> diff --git a/drivers/hv/mshv_root_hv_call.c b/drivers/hv/mshv_root_hv_call.c
> index c9c274f29c3c6..0696024ccfe31 100644
> --- a/drivers/hv/mshv_root_hv_call.c
> +++ b/drivers/hv/mshv_root_hv_call.c
> @@ -17,7 +17,7 @@
> /* Determined empirically */
> #define HV_INIT_PARTITION_DEPOSIT_PAGES 208
> #define HV_MAP_GPA_DEPOSIT_PAGES 256
> -#define HV_UMAP_GPA_PAGES 512
> +#define HV_UMAP_GPA_PAGES MSHV_MAX_UNMAP_GPA_PAGES
>
> #define HV_PAGE_COUNT_2M_ALIGNED(pg_count) (!((pg_count) & (0x200 - 1)))
>
> diff --git a/drivers/hv/mshv_root_main.c b/drivers/hv/mshv_root_main.c
> index 97e322f3c6b5e..b61bef6b9c132 100644
> --- a/drivers/hv/mshv_root_main.c
> +++ b/drivers/hv/mshv_root_main.c
> @@ -1378,6 +1378,7 @@ mshv_map_user_memory(struct mshv_partition *partition,
> static void mshv_partition_destroy_region(struct mshv_mem_region *region)
> {
> struct mshv_partition *partition = region->partition;
> + u64 gfn, gfn_count, start_gfn, end_gfn;
> u32 unmap_flags = 0;
> int ret;
>
> @@ -1396,9 +1397,30 @@ static void mshv_partition_destroy_region(struct mshv_mem_region *region)
> if (region->flags.large_pages)
> unmap_flags |= HV_UNMAP_GPA_LARGE_PAGE;
>
> - /* ignore unmap failures and continue as process may be exiting */
> - hv_call_unmap_gpa_pages(partition->pt_id, region->start_gfn,
> - region->nr_pages, unmap_flags);
> + start_gfn = region->start_gfn;
> + end_gfn = region->start_gfn + region->nr_pages;
> +
> + for (gfn = start_gfn; gfn < end_gfn; gfn += gfn_count) {
> + if (gfn % MSHV_MAX_UNMAP_GPA_PAGES)
> + gfn_count = ALIGN(gfn, MSHV_MAX_UNMAP_GPA_PAGES) - gfn;
> + else
> + gfn_count = MSHV_MAX_UNMAP_GPA_PAGES;
You could do the entire if/else as:
gfn_count = ALIGN(gfn + 1, MSHV_MAX_UNMAP_GPA_PAGES) - gfn;
Using "gfn + 1" handles the case where gfn is already aligned. Arguably, this is a bit
more obscure, so it's just a suggestion.
> +
> + if (gfn + gfn_count > end_gfn)
> + gfn_count = end_gfn - gfn;
Or
gfn_count = min(gfn_count, end_gfn - gfn);
I usually prefer the "min" function instead of an "if" statement if logically
the intent is to compute the minimum. But again, just a suggestion.
> +
> + /* Skip if all pages in this range if none is mapped */
> + if (!memchr_inv(region->pages + (gfn - start_gfn), 0,
> + gfn_count * sizeof(struct page *)))
> + continue;
> +
> + ret = hv_call_unmap_gpa_pages(partition->pt_id, gfn,
> + gfn_count, unmap_flags);
> + if (ret)
> + pt_err(partition,
> + "Failed to unmap GPA pages %#llx-%#llx: %d\n",
> + gfn, gfn + gfn_count - 1, ret);
> + }
Overall, I think this algorithm looks good and handles all the edge cases.
Michael
>
> mshv_region_invalidate(region);
>
>
>
^ permalink raw reply
* Re: [PATCH hyperv-next v6 05/17] arch/x86: mshyperv: Trap on access for some synthetic MSRs
From: Roman Kisel @ 2025-10-06 18:30 UTC (permalink / raw)
To: Michael Kelley
Cc: benhill@microsoft.com, bperkins@microsoft.com,
sunilmut@microsoft.com, arnd@arndb.de, bp@alien8.de,
corbet@lwn.net, dave.hansen@linux.intel.com, decui@microsoft.com,
haiyangz@microsoft.com, hpa@zytor.com, kys@microsoft.com,
mikelley@microsoft.com, mingo@redhat.com, tglx@linutronix.de,
Tianyu.Lan@microsoft.com, wei.liu@kernel.org, x86@kernel.org,
linux-hyperv@vger.kernel.org, linux-doc@vger.kernel.org,
linux-kernel@vger.kernel.org, linux-arch@vger.kernel.org
In-Reply-To: <SN6PR02MB41571BD37714C5F0AB770CB5D4E3A@SN6PR02MB4157.namprd02.prod.outlook.com>
On 10/6/2025 9:55 AM, Michael Kelley wrote:
> From: Roman Kisel <romank@linux.microsoft.com> Sent: Friday, October 3, 2025 3:27 PM
[...]
>> +/*
>> + * When running with the paravisor, controls proxying the synthetic interrupts
>> + * from the host
>> + */
>> +static bool hv_para_sint_proxy;
>
> This needs to move down a few lines and be under the #if IS_ENABLED(CONFIG_HYPERV)
> in order to eliminate the "unused variable" warning reported by the kernel test robot.
Thanks, Michael, will do!
[...]
--
Thank you,
Roman
^ permalink raw reply
* Re: [PATCH hyperv-next v6 00/17] Confidential VMBus
From: Roman Kisel @ 2025-10-06 18:58 UTC (permalink / raw)
To: Michael Kelley, arnd@arndb.de, bp@alien8.de, corbet@lwn.net,
dave.hansen@linux.intel.com, decui@microsoft.com,
haiyangz@microsoft.com, hpa@zytor.com, kys@microsoft.com,
mikelley@microsoft.com, mingo@redhat.com, tglx@linutronix.de,
Tianyu.Lan@microsoft.com, wei.liu@kernel.org, x86@kernel.org,
linux-hyperv@vger.kernel.org, linux-doc@vger.kernel.org,
linux-kernel@vger.kernel.org, linux-arch@vger.kernel.org
Cc: benhill@microsoft.com, bperkins@microsoft.com,
sunilmut@microsoft.com
In-Reply-To: <SN6PR02MB415707D796045E8BD30396D8D4E3A@SN6PR02MB4157.namprd02.prod.outlook.com>
On 10/6/2025 9:55 AM, Michael Kelley wrote:
> From: Roman Kisel <romank@linux.microsoft.com> Sent: Friday, October 3, 2025 3:27 PM
[...]
>> include/linux/hyperv.h | 69 ++++--
>> 15 files changed, 793 insertions(+), 306 deletions(-)
>>
>
> Nice! The net lines of code added is now 487, vs. 591
> lines added in v5 of this series.
>
Thanks, I appreciate your help throughout the multiple versions very
much!!
> Modulo the contradiction above in this cover letter, the two typos in
> the documentation in Patch 1, and the simple fix for the error reported
> by the kernel test robot for Patch 5, I'm happy with this entire series.
I'll wait few days just in case and will send out the fixed series :)
> For the series,
>
> Reviewed-by: Michael Kelley <mhklinux@outlook.com>
--
Thank you,
Roman
^ permalink raw reply
* [PATCH v3 0/6] Hyper-V: Implement hypervisor core collection
From: Mukesh Rathor @ 2025-10-06 22:42 UTC (permalink / raw)
To: linux-hyperv, linux-kernel, linux-arch
Cc: kys, haiyangz, wei.liu, decui, tglx, mingo, bp, dave.hansen, x86,
hpa, arnd
This patch series implements hypervisor core collection when running
under Linux as root. By default initial hypervisor RAM is already mapped
into Linux as reserved. Further any RAM deposited comes from Linux memory
heap. The hypervisor locks all that RAM to protect it from root or any
other domains. At a high level, the methodology involes devirtualizing
the system on the fly upon either Linux crash or the hypervisor crash,
then collecting core as usual. This means hypervisor RAM is automatically
collected into the vmcore. Devirtualization is the process of disabling
the hypervisor and taking control of the system.
Hypervisor pages are then accessible via crash command (using raw mem
dump) or windbg which has the ability to read hypervisor pdb symbol
file.
V3:
o remove usage of the word "dom0" as asked by maintainer
o change hyp to hv in comment and ipi to IPI
o rebase to: hyperv-next: commit b595edcb2472
V2:
o change few comments and commit-messages
o add support for panic_timeout for better support if kdump kernel
is not loaded.
o some other minor changes, like change devirt_cr3arg to devirt_arg,
int to bool.
V1:
o Describe changes in imperative mood. Remove "This commit"
o Remove pr_emerg: causing unnecessary review noise
o Add missing kexec_crash_loaded()
o Remove leftover unnecessary memcpy in hv_crash_setup_trampdata
o Address objtool warnings via annotations
Mukesh Rathor (6):
x86/hyperv: Rename guest crash shutdown function
hyperv: Add two new hypercall numbers to guest ABI public header
hyperv: Add definitions for hypervisor crash dump support
x86/hyperv: Add trampoline asm code to transition from hypervisor
x86/hyperv: Implement hypervisor RAM collection into vmcore
x86/hyperv: Enable build of hypervisor crashdump collection files
arch/x86/hyperv/Makefile | 6 +
arch/x86/hyperv/hv_crash.c | 642 ++++++++++++++++++++++++++++++++
arch/x86/hyperv/hv_init.c | 1 +
arch/x86/hyperv/hv_trampoline.S | 101 +++++
arch/x86/include/asm/mshyperv.h | 13 +
arch/x86/kernel/cpu/mshyperv.c | 5 +-
include/hyperv/hvgdk_mini.h | 2 +
include/hyperv/hvhdk_mini.h | 55 +++
8 files changed, 823 insertions(+), 2 deletions(-)
create mode 100644 arch/x86/hyperv/hv_crash.c
create mode 100644 arch/x86/hyperv/hv_trampoline.S
--
2.36.1.vfs.0.0
^ permalink raw reply
* [PATCH v3 1/6] x86/hyperv: Rename guest crash shutdown function
From: Mukesh Rathor @ 2025-10-06 22:42 UTC (permalink / raw)
To: linux-hyperv, linux-kernel, linux-arch
Cc: kys, haiyangz, wei.liu, decui, tglx, mingo, bp, dave.hansen, x86,
hpa, arnd
In-Reply-To: <20251006224208.1060990-1-mrathor@linux.microsoft.com>
Rename hv_machine_crash_shutdown to more appropriate
hv_guest_crash_shutdown and make it applicable to guests only. This
in preparation for the subsequent hypervisor root crash support
patches.
Signed-off-by: Mukesh Rathor <mrathor@linux.microsoft.com>
---
arch/x86/kernel/cpu/mshyperv.c | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c
index 25773af116bc..1c6ec9b6107f 100644
--- a/arch/x86/kernel/cpu/mshyperv.c
+++ b/arch/x86/kernel/cpu/mshyperv.c
@@ -219,7 +219,7 @@ static void hv_machine_shutdown(void)
#endif /* CONFIG_KEXEC_CORE */
#ifdef CONFIG_CRASH_DUMP
-static void hv_machine_crash_shutdown(struct pt_regs *regs)
+static void hv_guest_crash_shutdown(struct pt_regs *regs)
{
if (hv_crash_handler)
hv_crash_handler(regs);
@@ -562,7 +562,8 @@ static void __init ms_hyperv_init_platform(void)
machine_ops.shutdown = hv_machine_shutdown;
#endif
#if defined(CONFIG_CRASH_DUMP)
- machine_ops.crash_shutdown = hv_machine_crash_shutdown;
+ if (!hv_root_partition())
+ machine_ops.crash_shutdown = hv_guest_crash_shutdown;
#endif
#endif
/*
--
2.36.1.vfs.0.0
^ permalink raw reply related
* [PATCH v3 2/6] hyperv: Add two new hypercall numbers to guest ABI public header
From: Mukesh Rathor @ 2025-10-06 22:42 UTC (permalink / raw)
To: linux-hyperv, linux-kernel, linux-arch
Cc: kys, haiyangz, wei.liu, decui, tglx, mingo, bp, dave.hansen, x86,
hpa, arnd
In-Reply-To: <20251006224208.1060990-1-mrathor@linux.microsoft.com>
In preparation for the subsequent crashdump patches, copy two hypercall
numbers to the guest ABI header published by Hyper-V. One to notify
hypervisor of an event that occurs in the root partition, other to ask
hypervisor to disable the hypervisor.
Signed-off-by: Mukesh Rathor <mrathor@linux.microsoft.com>
---
include/hyperv/hvgdk_mini.h | 2 ++
1 file changed, 2 insertions(+)
diff --git a/include/hyperv/hvgdk_mini.h b/include/hyperv/hvgdk_mini.h
index 77abddfc750e..bec54a103d62 100644
--- a/include/hyperv/hvgdk_mini.h
+++ b/include/hyperv/hvgdk_mini.h
@@ -469,6 +469,7 @@ union hv_vp_assist_msr_contents { /* HV_REGISTER_VP_ASSIST_PAGE */
#define HVCALL_MAP_DEVICE_INTERRUPT 0x007c
#define HVCALL_UNMAP_DEVICE_INTERRUPT 0x007d
#define HVCALL_RETARGET_INTERRUPT 0x007e
+#define HVCALL_NOTIFY_PARTITION_EVENT 0x0087
#define HVCALL_NOTIFY_PORT_RING_EMPTY 0x008b
#define HVCALL_REGISTER_INTERCEPT_RESULT 0x0091
#define HVCALL_ASSERT_VIRTUAL_INTERRUPT 0x0094
@@ -492,6 +493,7 @@ union hv_vp_assist_msr_contents { /* HV_REGISTER_VP_ASSIST_PAGE */
#define HVCALL_GET_VP_CPUID_VALUES 0x00f4
#define HVCALL_MMIO_READ 0x0106
#define HVCALL_MMIO_WRITE 0x0107
+#define HVCALL_DISABLE_HYP_EX 0x010f
/* HV_HYPERCALL_INPUT */
#define HV_HYPERCALL_RESULT_MASK GENMASK_ULL(15, 0)
--
2.36.1.vfs.0.0
^ permalink raw reply related
* [PATCH v3 3/6] hyperv: Add definitions for hypervisor crash dump support
From: Mukesh Rathor @ 2025-10-06 22:42 UTC (permalink / raw)
To: linux-hyperv, linux-kernel, linux-arch
Cc: kys, haiyangz, wei.liu, decui, tglx, mingo, bp, dave.hansen, x86,
hpa, arnd
In-Reply-To: <20251006224208.1060990-1-mrathor@linux.microsoft.com>
Add data structures for hypervisor crash dump support to the hypervisor
host ABI header file. Details of their usages are in subsequent commits.
Signed-off-by: Mukesh Rathor <mrathor@linux.microsoft.com>
---
include/hyperv/hvhdk_mini.h | 55 +++++++++++++++++++++++++++++++++++++
1 file changed, 55 insertions(+)
diff --git a/include/hyperv/hvhdk_mini.h b/include/hyperv/hvhdk_mini.h
index 858f6a3925b3..ad9a8048fb4e 100644
--- a/include/hyperv/hvhdk_mini.h
+++ b/include/hyperv/hvhdk_mini.h
@@ -116,6 +116,17 @@ enum hv_system_property {
/* Add more values when needed */
HV_SYSTEM_PROPERTY_SCHEDULER_TYPE = 15,
HV_DYNAMIC_PROCESSOR_FEATURE_PROPERTY = 21,
+ HV_SYSTEM_PROPERTY_CRASHDUMPAREA = 47,
+};
+
+#define HV_PFN_RANGE_PGBITS 24 /* HV_SPA_PAGE_RANGE_ADDITIONAL_PAGES_BITS */
+union hv_pfn_range { /* HV_SPA_PAGE_RANGE */
+ u64 as_uint64;
+ struct {
+ /* 39:0: base pfn. 63:40: additional pages */
+ u64 base_pfn : 64 - HV_PFN_RANGE_PGBITS;
+ u64 add_pfns : HV_PFN_RANGE_PGBITS;
+ } __packed;
};
enum hv_dynamic_processor_feature_property {
@@ -142,6 +153,8 @@ struct hv_output_get_system_property {
#if IS_ENABLED(CONFIG_X86)
u64 hv_processor_feature_value;
#endif
+ union hv_pfn_range hv_cda_info; /* CrashdumpAreaAddress */
+ u64 hv_tramp_pa; /* CrashdumpTrampolineAddress */
};
} __packed;
@@ -234,6 +247,48 @@ union hv_gpa_page_access_state {
u8 as_uint8;
} __packed;
+enum hv_crashdump_action {
+ HV_CRASHDUMP_NONE = 0,
+ HV_CRASHDUMP_SUSPEND_ALL_VPS,
+ HV_CRASHDUMP_PREPARE_FOR_STATE_SAVE,
+ HV_CRASHDUMP_STATE_SAVED,
+ HV_CRASHDUMP_ENTRY,
+};
+
+struct hv_partition_event_root_crashdump_input {
+ u32 crashdump_action; /* enum hv_crashdump_action */
+} __packed;
+
+struct hv_input_disable_hyp_ex { /* HV_X64_INPUT_DISABLE_HYPERVISOR_EX */
+ u64 rip;
+ u64 arg;
+} __packed;
+
+struct hv_crashdump_area { /* HV_CRASHDUMP_AREA */
+ u32 version;
+ union {
+ u32 flags_as_uint32;
+ struct {
+ u32 cda_valid : 1;
+ u32 cda_unused : 31;
+ } __packed;
+ };
+ /* more unused fields */
+} __packed;
+
+union hv_partition_event_input {
+ struct hv_partition_event_root_crashdump_input crashdump_input;
+};
+
+enum hv_partition_event {
+ HV_PARTITION_EVENT_ROOT_CRASHDUMP = 2,
+};
+
+struct hv_input_notify_partition_event {
+ u32 event; /* enum hv_partition_event */
+ union hv_partition_event_input input;
+} __packed;
+
struct hv_lp_startup_status {
u64 hv_status;
u64 substatus1;
--
2.36.1.vfs.0.0
^ permalink raw reply related
* [PATCH v3 4/6] x86/hyperv: Add trampoline asm code to transition from hypervisor
From: Mukesh Rathor @ 2025-10-06 22:42 UTC (permalink / raw)
To: linux-hyperv, linux-kernel, linux-arch
Cc: kys, haiyangz, wei.liu, decui, tglx, mingo, bp, dave.hansen, x86,
hpa, arnd
In-Reply-To: <20251006224208.1060990-1-mrathor@linux.microsoft.com>
Introduce a small asm stub to transition from the hypervisor to Linux
after devirtualization. Devirtualization means disabling hypervisor on
the fly, so after it is done, the code is running on physical processor
instead of virtual, and hypervisor is gone. This can be done by a
root vm only.
At a high level, during panic of either the hypervisor or the root,
the NMI handler asks hypervisor to devirtualize. As part of that,
the arguments include an entry point to return back to Linux. This asm
stub implements that entry point.
The stub is entered in protected mode, uses temporary gdt and page table
to enable long mode and get to kernel entry point which then restores full
kernel context to resume execution to kexec.
Signed-off-by: Mukesh Rathor <mrathor@linux.microsoft.com>
---
arch/x86/hyperv/hv_trampoline.S | 101 ++++++++++++++++++++++++++++++++
1 file changed, 101 insertions(+)
create mode 100644 arch/x86/hyperv/hv_trampoline.S
diff --git a/arch/x86/hyperv/hv_trampoline.S b/arch/x86/hyperv/hv_trampoline.S
new file mode 100644
index 000000000000..25f02ff12286
--- /dev/null
+++ b/arch/x86/hyperv/hv_trampoline.S
@@ -0,0 +1,101 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * X86 specific Hyper-V kdump/crash related code.
+ *
+ * Copyright (C) 2025, Microsoft, Inc.
+ *
+ */
+#include <linux/linkage.h>
+#include <asm/alternative.h>
+#include <asm/msr.h>
+#include <asm/processor-flags.h>
+#include <asm/nospec-branch.h>
+
+/*
+ * void noreturn hv_crash_asm32(arg1)
+ * arg1 == edi == 32bit PA of struct hv_crash_tramp_data
+ *
+ * The hypervisor jumps here upon devirtualization in protected mode. This
+ * code gets copied to a page in the low 4G ie, 32bit space so it can run
+ * in the protected mode. Hence we cannot use any compile/link time offsets or
+ * addresses. It restores long mode via temporary gdt and page tables and
+ * eventually jumps to kernel code entry at HV_CRASHDATA_OFFS_C_entry.
+ *
+ * PreCondition (ie, Hypervisor call back ABI):
+ * o CR0 is set to 0x0021: PE(prot mode) and NE are set, paging is disabled
+ * o CR4 is set to 0x0
+ * o IA32_EFER is set to 0x901 (SCE and NXE are set)
+ * o EDI is set to the Arg passed to HVCALL_DISABLE_HYP_EX.
+ * o CS, DS, ES, FS, GS are all initialized with a base of 0 and limit 0xFFFF
+ * o IDTR, TR and GDTR are initialized with a base of 0 and limit of 0xFFFF
+ * o LDTR is initialized as invalid (limit of 0)
+ * o MSR PAT is power on default.
+ * o Other state/registers are cleared. All TLBs flushed.
+ */
+
+#define HV_CRASHDATA_OFFS_TRAMPCR3 0x0 /* 0 */
+#define HV_CRASHDATA_OFFS_KERNCR3 0x8 /* 8 */
+#define HV_CRASHDATA_OFFS_GDTRLIMIT 0x12 /* 18 */
+#define HV_CRASHDATA_OFFS_CS_JMPTGT 0x28 /* 40 */
+#define HV_CRASHDATA_OFFS_C_entry 0x30 /* 48 */
+
+ .text
+ .code32
+
+SYM_CODE_START(hv_crash_asm32)
+ UNWIND_HINT_UNDEFINED
+ ENDBR
+ movl $X86_CR4_PAE, %ecx
+ movl %ecx, %cr4
+
+ movl %edi, %ebx
+ add $HV_CRASHDATA_OFFS_TRAMPCR3, %ebx
+ movl %cs:(%ebx), %eax
+ movl %eax, %cr3
+
+ /* Setup EFER for long mode now */
+ movl $MSR_EFER, %ecx
+ rdmsr
+ btsl $_EFER_LME, %eax
+ wrmsr
+
+ /* Turn paging on using the temp 32bit trampoline page table */
+ movl %cr0, %eax
+ orl $(X86_CR0_PG), %eax
+ movl %eax, %cr0
+
+ /* since kernel cr3 could be above 4G, we need to be in the long mode
+ * before we can load 64bits of the kernel cr3. We use a temp gdt for
+ * that with CS.L=1 and CS.D=0 */
+ mov %edi, %eax
+ add $HV_CRASHDATA_OFFS_GDTRLIMIT, %eax
+ lgdtl %cs:(%eax)
+
+ /* not done yet, restore CS now to switch to CS.L=1 */
+ mov %edi, %eax
+ add $HV_CRASHDATA_OFFS_CS_JMPTGT, %eax
+ ljmp %cs:*(%eax)
+SYM_CODE_END(hv_crash_asm32)
+
+ /* we now run in full 64bit IA32-e long mode, CS.L=1 and CS.D=0 */
+ .code64
+ .balign 8
+SYM_CODE_START(hv_crash_asm64)
+ UNWIND_HINT_UNDEFINED
+ ENDBR
+ /* restore kernel page tables so we can jump to kernel code */
+ mov %edi, %eax
+ add $HV_CRASHDATA_OFFS_KERNCR3, %eax
+ movq %cs:(%eax), %rbx
+ movq %rbx, %cr3
+
+ mov %edi, %eax
+ add $HV_CRASHDATA_OFFS_C_entry, %eax
+ movq %cs:(%eax), %rbx
+ ANNOTATE_RETPOLINE_SAFE
+ jmp *%rbx
+
+ int $3
+
+SYM_INNER_LABEL(hv_crash_asm_end, SYM_L_GLOBAL)
+SYM_CODE_END(hv_crash_asm64)
--
2.36.1.vfs.0.0
^ permalink raw reply related
* [PATCH v3 5/6] x86/hyperv: Implement hypervisor RAM collection into vmcore
From: Mukesh Rathor @ 2025-10-06 22:42 UTC (permalink / raw)
To: linux-hyperv, linux-kernel, linux-arch
Cc: kys, haiyangz, wei.liu, decui, tglx, mingo, bp, dave.hansen, x86,
hpa, arnd
In-Reply-To: <20251006224208.1060990-1-mrathor@linux.microsoft.com>
Introduce a new file to implement collection of hypervisor RAM into the
vmcore collected by linux. By default, the hypervisor RAM is locked, ie,
protected via hw page table. Hyper-V implements a disable hypercall which
essentially devirtualizes the system on the fly. This mechanism makes the
hypervisor RAM accessible to linux. Because the hypervisor RAM is already
mapped into linux address space (as reserved RAM), it is automatically
collected into the vmcore without extra work. More details of the
implementation are available in the file prologue.
Signed-off-by: Mukesh Rathor <mrathor@linux.microsoft.com>
---
arch/x86/hyperv/hv_crash.c | 642 +++++++++++++++++++++++++++++++++++++
1 file changed, 642 insertions(+)
create mode 100644 arch/x86/hyperv/hv_crash.c
diff --git a/arch/x86/hyperv/hv_crash.c b/arch/x86/hyperv/hv_crash.c
new file mode 100644
index 000000000000..c0e22921ace1
--- /dev/null
+++ b/arch/x86/hyperv/hv_crash.c
@@ -0,0 +1,642 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * X86 specific Hyper-V root partition kdump/crash support module
+ *
+ * Copyright (C) 2025, Microsoft, Inc.
+ *
+ * This module implements hypervisor RAM collection into vmcore for both
+ * cases of the hypervisor crash and Linux root crash. Hyper-V implements
+ * a disable hypercall with a 32bit protected mode ABI callback. This
+ * mechanism must be used to unlock hypervisor RAM. Since the hypervisor RAM
+ * is already mapped in Linux, it is automatically collected into Linux vmcore,
+ * and can be examined by the crash command (raw RAM dump) or windbg.
+ *
+ * At a high level:
+ *
+ * Hypervisor Crash:
+ * Upon crash, hypervisor goes into an emergency minimal dispatch loop, a
+ * restrictive mode with very limited hypercall and MSR support. Each cpu
+ * then injects NMIs into root vcpus. A shared page is used to check
+ * by Linux in the NMI handler if the hypervisor has crashed. This shared
+ * page is setup in hv_root_crash_init during boot.
+ *
+ * Linux Crash:
+ * In case of Linux crash, the callback hv_crash_stop_other_cpus will send
+ * NMIs to all cpus, then proceed to the crash_nmi_callback where it waits
+ * for all cpus to be in NMI.
+ *
+ * NMI Handler (upon quorum):
+ * Eventually, in both cases, all cpus will end up in the NMI handler.
+ * Hyper-V requires the disable hypervisor must be done from the BSP. So
+ * the BSP NMI handler saves current context, does some fixups and makes
+ * the hypercall to disable the hypervisor, ie, devirtualize. Hypervisor
+ * at that point will suspend all vcpus (except the BSP), unlock all its
+ * RAM, and return to Linux at the 32bit mode entry RIP.
+ *
+ * Linux 32bit entry trampoline will then restore long mode and call C
+ * function here to restore context and continue execution to crash kexec.
+ */
+
+#include <linux/delay.h>
+#include <linux/kexec.h>
+#include <linux/crash_dump.h>
+#include <linux/panic.h>
+#include <asm/apic.h>
+#include <asm/desc.h>
+#include <asm/page.h>
+#include <asm/pgalloc.h>
+#include <asm/mshyperv.h>
+#include <asm/nmi.h>
+#include <asm/idtentry.h>
+#include <asm/reboot.h>
+#include <asm/intel_pt.h>
+
+bool hv_crash_enabled;
+EXPORT_SYMBOL_GPL(hv_crash_enabled);
+
+struct hv_crash_ctxt {
+ ulong rsp;
+ ulong cr0;
+ ulong cr2;
+ ulong cr4;
+ ulong cr8;
+
+ u16 cs;
+ u16 ss;
+ u16 ds;
+ u16 es;
+ u16 fs;
+ u16 gs;
+
+ u16 gdt_fill;
+ struct desc_ptr gdtr;
+ char idt_fill[6];
+ struct desc_ptr idtr;
+
+ u64 gsbase;
+ u64 efer;
+ u64 pat;
+};
+static struct hv_crash_ctxt hv_crash_ctxt;
+
+/* Shared hypervisor page that contains crash dump area we peek into.
+ * NB: windbg looks for "hv_cda" symbol so don't change it.
+ */
+static struct hv_crashdump_area *hv_cda;
+
+static u32 trampoline_pa, devirt_arg;
+static atomic_t crash_cpus_wait;
+static void *hv_crash_ptpgs[4];
+static bool hv_has_crashed, lx_has_crashed;
+
+static void __noreturn hv_panic_timeout_reboot(void)
+{
+ #define PANIC_TIMER_STEP 100
+
+ if (panic_timeout > 0) {
+ int i;
+
+ for (i = 0; i < panic_timeout * 1000; i += PANIC_TIMER_STEP)
+ mdelay(PANIC_TIMER_STEP);
+ }
+
+ if (panic_timeout)
+ native_wrmsrq(HV_X64_MSR_RESET, 1); /* get hyp to reboot */
+
+ for (;;)
+ cpu_relax();
+}
+
+/* This cannot be inlined as it needs stack */
+static noinline __noclone void hv_crash_restore_tss(void)
+{
+ load_TR_desc();
+}
+
+/* This cannot be inlined as it needs stack */
+static noinline void hv_crash_clear_kernpt(void)
+{
+ pgd_t *pgd;
+ p4d_t *p4d;
+
+ /* Clear entry so it's not confusing to someone looking at the core */
+ pgd = pgd_offset_k(trampoline_pa);
+ p4d = p4d_offset(pgd, trampoline_pa);
+ native_p4d_clear(p4d);
+}
+
+/*
+ * This is the C entry point from the asm glue code after the disable hypercall.
+ * We enter here in IA32-e long mode, ie, full 64bit mode running on kernel
+ * page tables with our below 4G page identity mapped, but using a temporary
+ * GDT. ds/fs/gs/es are null. ss is not usable. bp is null. stack is not
+ * available. We restore kernel GDT, and rest of the context, and continue
+ * to kexec.
+ */
+static asmlinkage void __noreturn hv_crash_c_entry(void)
+{
+ struct hv_crash_ctxt *ctxt = &hv_crash_ctxt;
+
+ /* first thing, restore kernel gdt */
+ native_load_gdt(&ctxt->gdtr);
+
+ asm volatile("movw %%ax, %%ss" : : "a"(ctxt->ss));
+ asm volatile("movq %0, %%rsp" : : "m"(ctxt->rsp));
+
+ asm volatile("movw %%ax, %%ds" : : "a"(ctxt->ds));
+ asm volatile("movw %%ax, %%es" : : "a"(ctxt->es));
+ asm volatile("movw %%ax, %%fs" : : "a"(ctxt->fs));
+ asm volatile("movw %%ax, %%gs" : : "a"(ctxt->gs));
+
+ native_wrmsrq(MSR_IA32_CR_PAT, ctxt->pat);
+ asm volatile("movq %0, %%cr0" : : "r"(ctxt->cr0));
+
+ asm volatile("movq %0, %%cr8" : : "r"(ctxt->cr8));
+ asm volatile("movq %0, %%cr4" : : "r"(ctxt->cr4));
+ asm volatile("movq %0, %%cr2" : : "r"(ctxt->cr4));
+
+ native_load_idt(&ctxt->idtr);
+ native_wrmsrq(MSR_GS_BASE, ctxt->gsbase);
+ native_wrmsrq(MSR_EFER, ctxt->efer);
+
+ /* restore the original kernel CS now via far return */
+ asm volatile("movzwq %0, %%rax\n\t"
+ "pushq %%rax\n\t"
+ "pushq $1f\n\t"
+ "lretq\n\t"
+ "1:nop\n\t" : : "m"(ctxt->cs) : "rax");
+
+ /* We are in asmlinkage without stack frame, hence make C function
+ * calls which will buy stack frames.
+ */
+ hv_crash_restore_tss();
+ hv_crash_clear_kernpt();
+
+ /* we are now fully in devirtualized normal kernel mode */
+ __crash_kexec(NULL);
+
+ hv_panic_timeout_reboot();
+}
+/* Tell gcc we are using lretq long jump in the above function intentionally */
+STACK_FRAME_NON_STANDARD(hv_crash_c_entry);
+
+static void hv_mark_tss_not_busy(void)
+{
+ struct desc_struct *desc = get_current_gdt_rw();
+ tss_desc tss;
+
+ memcpy(&tss, &desc[GDT_ENTRY_TSS], sizeof(tss_desc));
+ tss.type = 0x9; /* available 64-bit TSS. 0xB is busy TSS */
+ write_gdt_entry(desc, GDT_ENTRY_TSS, &tss, DESC_TSS);
+}
+
+/* Save essential context */
+static void hv_hvcrash_ctxt_save(void)
+{
+ struct hv_crash_ctxt *ctxt = &hv_crash_ctxt;
+
+ asm volatile("movq %%rsp,%0" : "=m"(ctxt->rsp));
+
+ ctxt->cr0 = native_read_cr0();
+ ctxt->cr4 = native_read_cr4();
+
+ asm volatile("movq %%cr2, %0" : "=a"(ctxt->cr2));
+ asm volatile("movq %%cr8, %0" : "=a"(ctxt->cr8));
+
+ asm volatile("movl %%cs, %%eax" : "=a"(ctxt->cs));
+ asm volatile("movl %%ss, %%eax" : "=a"(ctxt->ss));
+ asm volatile("movl %%ds, %%eax" : "=a"(ctxt->ds));
+ asm volatile("movl %%es, %%eax" : "=a"(ctxt->es));
+ asm volatile("movl %%fs, %%eax" : "=a"(ctxt->fs));
+ asm volatile("movl %%gs, %%eax" : "=a"(ctxt->gs));
+
+ native_store_gdt(&ctxt->gdtr);
+ store_idt(&ctxt->idtr);
+
+ ctxt->gsbase = __rdmsr(MSR_GS_BASE);
+ ctxt->efer = __rdmsr(MSR_EFER);
+ ctxt->pat = __rdmsr(MSR_IA32_CR_PAT);
+}
+
+/* Add trampoline page to the kernel pagetable for transition to kernel PT */
+static void hv_crash_fixup_kernpt(void)
+{
+ pgd_t *pgd;
+ p4d_t *p4d;
+
+ pgd = pgd_offset_k(trampoline_pa);
+ p4d = p4d_offset(pgd, trampoline_pa);
+
+ /* trampoline_pa is below 4G, so no pre-existing entry to clobber */
+ p4d_populate(&init_mm, p4d, (pud_t *)hv_crash_ptpgs[1]);
+ p4d->p4d = p4d->p4d & ~(_PAGE_NX); /* enable execute */
+}
+
+/*
+ * Notify the hyp that Linux has crashed. This will cause the hyp to quiesce
+ * and suspend all guest VPs.
+ */
+static void hv_notify_prepare_hyp(void)
+{
+ u64 status;
+ struct hv_input_notify_partition_event *input;
+ struct hv_partition_event_root_crashdump_input *cda;
+
+ input = *this_cpu_ptr(hyperv_pcpu_input_arg);
+ cda = &input->input.crashdump_input;
+ memset(input, 0, sizeof(*input));
+ input->event = HV_PARTITION_EVENT_ROOT_CRASHDUMP;
+
+ cda->crashdump_action = HV_CRASHDUMP_ENTRY;
+ status = hv_do_hypercall(HVCALL_NOTIFY_PARTITION_EVENT, input, NULL);
+ if (!hv_result_success(status))
+ return;
+
+ cda->crashdump_action = HV_CRASHDUMP_SUSPEND_ALL_VPS;
+ hv_do_hypercall(HVCALL_NOTIFY_PARTITION_EVENT, input, NULL);
+}
+
+/*
+ * Common function for all cpus before devirtualization.
+ *
+ * Hypervisor crash: all cpus get here in NMI context.
+ * Linux crash: the panicing cpu gets here at base level, all others in NMI
+ * context. Note, panicing cpu may not be the BSP.
+ *
+ * The function is not inlined so it will show on the stack. It is named so
+ * because the crash cmd looks for certain well known function names on the
+ * stack before looking into the cpu saved note in the elf section, and
+ * that work is currently incomplete.
+ *
+ * Notes:
+ * Hypervisor crash:
+ * - the hypervisor is in a very restrictive mode at this point and any
+ * vmexit it cannot handle would result in reboot. So, no mumbo jumbo,
+ * just get to kexec as quickly as possible.
+ *
+ * Devirtualization is supported from the BSP only at present.
+ */
+static noinline __noclone void crash_nmi_callback(struct pt_regs *regs)
+{
+ struct hv_input_disable_hyp_ex *input;
+ u64 status;
+ int msecs = 1000, ccpu = smp_processor_id();
+
+ if (ccpu == 0) {
+ /* crash_save_cpu() will be done in the kexec path */
+ cpu_emergency_stop_pt(); /* disable performance trace */
+ atomic_inc(&crash_cpus_wait);
+ } else {
+ crash_save_cpu(regs, ccpu);
+ cpu_emergency_stop_pt(); /* disable performance trace */
+ atomic_inc(&crash_cpus_wait);
+ for (;;)
+ cpu_relax();
+ }
+
+ while (atomic_read(&crash_cpus_wait) < num_online_cpus() && msecs--)
+ mdelay(1);
+
+ stop_nmi();
+ if (!hv_has_crashed)
+ hv_notify_prepare_hyp();
+
+ if (crashing_cpu == -1)
+ crashing_cpu = ccpu; /* crash cmd uses this */
+
+ hv_hvcrash_ctxt_save();
+ hv_mark_tss_not_busy();
+ hv_crash_fixup_kernpt();
+
+ input = *this_cpu_ptr(hyperv_pcpu_input_arg);
+ memset(input, 0, sizeof(*input));
+ input->rip = trampoline_pa;
+ input->arg = devirt_arg;
+
+ status = hv_do_hypercall(HVCALL_DISABLE_HYP_EX, input, NULL);
+
+ hv_panic_timeout_reboot();
+}
+
+
+static DEFINE_SPINLOCK(hv_crash_reboot_lk);
+
+/*
+ * Generic NMI callback handler: could be called without any crash also.
+ * hv crash: hypervisor injects NMI's into all cpus
+ * lx crash: panicing cpu sends NMI to all but self via crash_stop_other_cpus
+ */
+static int hv_crash_nmi_local(unsigned int cmd, struct pt_regs *regs)
+{
+ if (!hv_has_crashed && hv_cda && hv_cda->cda_valid)
+ hv_has_crashed = true;
+
+ if (!hv_has_crashed && !lx_has_crashed)
+ return NMI_DONE; /* ignore the NMI */
+
+ if (hv_has_crashed && !kexec_crash_loaded()) {
+ if (spin_trylock(&hv_crash_reboot_lk))
+ hv_panic_timeout_reboot();
+ else
+ for (;;)
+ cpu_relax();
+ }
+
+ crash_nmi_callback(regs);
+
+ return NMI_DONE;
+}
+
+/*
+ * hv_crash_stop_other_cpus() == smp_ops.crash_stop_other_cpus
+ *
+ * On normal Linux panic, this is called twice: first from panic and then again
+ * from native_machine_crash_shutdown.
+ *
+ * In case of hyperv, 3 ways to get here:
+ * 1. hv crash (only BSP will get here):
+ * BSP : NMI callback -> DisableHv -> hv_crash_asm32 -> hv_crash_c_entry
+ * -> __crash_kexec -> native_machine_crash_shutdown
+ * -> crash_smp_send_stop -> smp_ops.crash_stop_other_cpus
+ * Linux panic:
+ * 2. panic cpu x: panic() -> crash_smp_send_stop
+ * -> smp_ops.crash_stop_other_cpus
+ * 3. BSP: native_machine_crash_shutdown -> crash_smp_send_stop
+ *
+ * NB: noclone and non standard stack because of call to crash_setup_regs().
+ */
+static void __noclone hv_crash_stop_other_cpus(void)
+{
+ static bool crash_stop_done;
+ struct pt_regs lregs;
+ int ccpu = smp_processor_id();
+
+ if (hv_has_crashed)
+ return; /* all cpus already in NMI handler path */
+
+ if (!kexec_crash_loaded()) {
+ hv_notify_prepare_hyp();
+ hv_panic_timeout_reboot(); /* no return */
+ }
+
+ /* If the hv crashes also, we could come here again before cpus_stopped
+ * is set in crash_smp_send_stop(). So use our own check.
+ */
+ if (crash_stop_done)
+ return;
+ crash_stop_done = true;
+
+ /* Linux has crashed: hv is healthy, we can IPI safely */
+ lx_has_crashed = true;
+ wmb(); /* NMI handlers look at lx_has_crashed */
+
+ apic->send_IPI_allbutself(NMI_VECTOR);
+
+ if (crashing_cpu == -1)
+ crashing_cpu = ccpu; /* crash cmd uses this */
+
+ /* crash_setup_regs() happens in kexec also, but for the kexec cpu which
+ * is the BSP. We could be here on non-BSP cpu, collect regs if so.
+ */
+ if (ccpu)
+ crash_setup_regs(&lregs, NULL);
+
+ crash_nmi_callback(&lregs);
+}
+STACK_FRAME_NON_STANDARD(hv_crash_stop_other_cpus);
+
+/* This GDT is accessed in IA32-e compat mode which uses 32bits addresses */
+struct hv_gdtreg_32 {
+ u16 fill;
+ u16 limit;
+ u32 address;
+} __packed;
+
+/* We need a CS with L bit to goto IA32-e long mode from 32bit compat mode */
+struct hv_crash_tramp_gdt {
+ u64 null; /* index 0, selector 0, null selector */
+ u64 cs64; /* index 1, selector 8, cs64 selector */
+} __packed;
+
+/* No stack, so jump via far ptr in memory to load the 64bit CS */
+struct hv_cs_jmptgt {
+ u32 address;
+ u16 csval;
+ u16 fill;
+} __packed;
+
+/* Linux use only, hypervisor doesn't look at this struct */
+struct hv_crash_tramp_data {
+ u64 tramp32_cr3;
+ u64 kernel_cr3;
+ struct hv_gdtreg_32 gdtr32;
+ struct hv_crash_tramp_gdt tramp_gdt;
+ struct hv_cs_jmptgt cs_jmptgt;
+ u64 c_entry_addr;
+} __packed;
+
+/*
+ * Setup a temporary gdt to allow the asm code to switch to the long mode.
+ * Since the asm code is relocated/copied to a below 4G page, it cannot use rip
+ * relative addressing, hence we must use trampoline_pa here. Also, save other
+ * info like jmp and C entry targets for same reasons.
+ *
+ * Returns: 0 on success, -1 on error
+ */
+static int hv_crash_setup_trampdata(u64 trampoline_va)
+{
+ int size, offs;
+ void *dest;
+ struct hv_crash_tramp_data *tramp;
+
+ /* These must match exactly the ones in the corresponding asm file */
+ BUILD_BUG_ON(offsetof(struct hv_crash_tramp_data, tramp32_cr3) != 0);
+ BUILD_BUG_ON(offsetof(struct hv_crash_tramp_data, kernel_cr3) != 8);
+ BUILD_BUG_ON(offsetof(struct hv_crash_tramp_data, gdtr32.limit) != 18);
+ BUILD_BUG_ON(offsetof(struct hv_crash_tramp_data,
+ cs_jmptgt.address) != 40);
+ BUILD_BUG_ON(offsetof(struct hv_crash_tramp_data, c_entry_addr) != 48);
+
+ /* hv_crash_asm_end is beyond last byte by 1 */
+ size = &hv_crash_asm_end - &hv_crash_asm32;
+ if (size + sizeof(struct hv_crash_tramp_data) > PAGE_SIZE) {
+ pr_err("%s: trampoline page overflow\n", __func__);
+ return -1;
+ }
+
+ dest = (void *)trampoline_va;
+ memcpy(dest, &hv_crash_asm32, size);
+
+ dest += size;
+ dest = (void *)round_up((ulong)dest, 16);
+ tramp = (struct hv_crash_tramp_data *)dest;
+
+ /* see MAX_ASID_AVAILABLE in tlb.c: "PCID 0 is reserved for use by
+ * non-PCID-aware users". Build cr3 with pcid 0
+ */
+ tramp->tramp32_cr3 = __sme_pa(hv_crash_ptpgs[0]);
+
+ /* Note, when restoring X86_CR4_PCIDE, cr3[11:0] must be zero */
+ tramp->kernel_cr3 = __sme_pa(init_mm.pgd);
+
+ tramp->gdtr32.limit = sizeof(struct hv_crash_tramp_gdt);
+ tramp->gdtr32.address = trampoline_pa +
+ (ulong)&tramp->tramp_gdt - trampoline_va;
+
+ /* base:0 limit:0xfffff type:b dpl:0 P:1 L:1 D:0 avl:0 G:1 */
+ tramp->tramp_gdt.cs64 = 0x00af9a000000ffff;
+
+ tramp->cs_jmptgt.csval = 0x8;
+ offs = (ulong)&hv_crash_asm64 - (ulong)&hv_crash_asm32;
+ tramp->cs_jmptgt.address = trampoline_pa + offs;
+
+ tramp->c_entry_addr = (u64)&hv_crash_c_entry;
+
+ devirt_arg = trampoline_pa + (ulong)dest - trampoline_va;
+
+ return 0;
+}
+
+/*
+ * Build 32bit trampoline page table for transition from protected mode
+ * non-paging to long-mode paging. This transition needs pagetables below 4G.
+ */
+static void hv_crash_build_tramp_pt(void)
+{
+ p4d_t *p4d;
+ pud_t *pud;
+ pmd_t *pmd;
+ pte_t *pte;
+ u64 pa, addr = trampoline_pa;
+
+ p4d = hv_crash_ptpgs[0] + pgd_index(addr) * sizeof(p4d);
+ pa = virt_to_phys(hv_crash_ptpgs[1]);
+ set_p4d(p4d, __p4d(_PAGE_TABLE | pa));
+ p4d->p4d &= ~(_PAGE_NX); /* enable execute */
+
+ pud = hv_crash_ptpgs[1] + pud_index(addr) * sizeof(pud);
+ pa = virt_to_phys(hv_crash_ptpgs[2]);
+ set_pud(pud, __pud(_PAGE_TABLE | pa));
+
+ pmd = hv_crash_ptpgs[2] + pmd_index(addr) * sizeof(pmd);
+ pa = virt_to_phys(hv_crash_ptpgs[3]);
+ set_pmd(pmd, __pmd(_PAGE_TABLE | pa));
+
+ pte = hv_crash_ptpgs[3] + pte_index(addr) * sizeof(pte);
+ set_pte(pte, pfn_pte(addr >> PAGE_SHIFT, PAGE_KERNEL_EXEC));
+}
+
+/*
+ * Setup trampoline for devirtualization:
+ * - a page below 4G, ie 32bit addr containing asm glue code that hyp jmps to
+ * in protected mode.
+ * - 4 pages for a temporary page table that asm code uses to turn paging on
+ * - a temporary gdt to use in the compat mode.
+ *
+ * Returns: 0 on success
+ */
+static int hv_crash_trampoline_setup(void)
+{
+ int i, rc, order;
+ struct page *page;
+ u64 trampoline_va;
+ gfp_t flags32 = GFP_KERNEL | GFP_DMA32 | __GFP_ZERO;
+
+ /* page for 32bit trampoline assembly code + hv_crash_tramp_data */
+ page = alloc_page(flags32);
+ if (page == NULL) {
+ pr_err("%s: failed to alloc asm stub page\n", __func__);
+ return -1;
+ }
+
+ trampoline_va = (u64)page_to_virt(page);
+ trampoline_pa = (u32)page_to_phys(page);
+
+ order = 2; /* alloc 2^2 pages */
+ page = alloc_pages(flags32, order);
+ if (page == NULL) {
+ pr_err("%s: failed to alloc pt pages\n", __func__);
+ free_page(trampoline_va);
+ return -1;
+ }
+
+ for (i = 0; i < 4; i++, page++)
+ hv_crash_ptpgs[i] = page_to_virt(page);
+
+ hv_crash_build_tramp_pt();
+
+ rc = hv_crash_setup_trampdata(trampoline_va);
+ if (rc)
+ goto errout;
+
+ return 0;
+
+errout:
+ free_page(trampoline_va);
+ free_pages((ulong)hv_crash_ptpgs[0], order);
+
+ return rc;
+}
+
+/* Setup for kdump kexec to collect hypervisor RAM when running as root */
+void hv_root_crash_init(void)
+{
+ int rc;
+ struct hv_input_get_system_property *input;
+ struct hv_output_get_system_property *output;
+ unsigned long flags;
+ u64 status;
+ union hv_pfn_range cda_info;
+
+ if (pgtable_l5_enabled()) {
+ pr_err("Hyper-V: crash dump not yet supported on 5level PTs\n");
+ return;
+ }
+
+ rc = register_nmi_handler(NMI_LOCAL, hv_crash_nmi_local, NMI_FLAG_FIRST,
+ "hv_crash_nmi");
+ if (rc) {
+ pr_err("Hyper-V: failed to register crash nmi handler\n");
+ return;
+ }
+
+ local_irq_save(flags);
+ input = *this_cpu_ptr(hyperv_pcpu_input_arg);
+ output = *this_cpu_ptr(hyperv_pcpu_output_arg);
+
+ memset(input, 0, sizeof(*input));
+ input->property_id = HV_SYSTEM_PROPERTY_CRASHDUMPAREA;
+
+ status = hv_do_hypercall(HVCALL_GET_SYSTEM_PROPERTY, input, output);
+ cda_info.as_uint64 = output->hv_cda_info.as_uint64;
+ local_irq_restore(flags);
+
+ if (!hv_result_success(status)) {
+ pr_err("Hyper-V: %s: property:%d %s\n", __func__,
+ input->property_id, hv_result_to_string(status));
+ goto err_out;
+ }
+
+ if (cda_info.base_pfn == 0) {
+ pr_err("Hyper-V: hypervisor crash dump area pfn is 0\n");
+ goto err_out;
+ }
+
+ hv_cda = phys_to_virt(cda_info.base_pfn << HV_HYP_PAGE_SHIFT);
+
+ rc = hv_crash_trampoline_setup();
+ if (rc)
+ goto err_out;
+
+ smp_ops.crash_stop_other_cpus = hv_crash_stop_other_cpus;
+
+ crash_kexec_post_notifiers = true;
+ hv_crash_enabled = true;
+ pr_info("Hyper-V: both linux and hypervisor kdump support enabled\n");
+
+ return;
+
+err_out:
+ unregister_nmi_handler(NMI_LOCAL, "hv_crash_nmi");
+ pr_err("Hyper-V: only linux root kdump support enabled\n");
+}
--
2.36.1.vfs.0.0
^ permalink raw reply related
* [PATCH v3 6/6] x86/hyperv: Enable build of hypervisor crashdump collection files
From: Mukesh Rathor @ 2025-10-06 22:42 UTC (permalink / raw)
To: linux-hyperv, linux-kernel, linux-arch
Cc: kys, haiyangz, wei.liu, decui, tglx, mingo, bp, dave.hansen, x86,
hpa, arnd
In-Reply-To: <20251006224208.1060990-1-mrathor@linux.microsoft.com>
Enable build of the new files introduced in the earlier commits and add
call to do the setup during boot.
Signed-off-by: Mukesh Rathor <mrathor@linux.microsoft.com>
---
arch/x86/hyperv/Makefile | 6 ++++++
arch/x86/hyperv/hv_init.c | 1 +
arch/x86/include/asm/mshyperv.h | 13 +++++++++++++
3 files changed, 20 insertions(+)
diff --git a/arch/x86/hyperv/Makefile b/arch/x86/hyperv/Makefile
index d55f494f471d..6f5d97cddd80 100644
--- a/arch/x86/hyperv/Makefile
+++ b/arch/x86/hyperv/Makefile
@@ -5,4 +5,10 @@ obj-$(CONFIG_HYPERV_VTL_MODE) += hv_vtl.o
ifdef CONFIG_X86_64
obj-$(CONFIG_PARAVIRT_SPINLOCKS) += hv_spinlock.o
+
+ ifdef CONFIG_MSHV_ROOT
+ CFLAGS_REMOVE_hv_trampoline.o += -pg
+ CFLAGS_hv_trampoline.o += -fno-stack-protector
+ obj-$(CONFIG_CRASH_DUMP) += hv_crash.o hv_trampoline.o
+ endif
endif
diff --git a/arch/x86/hyperv/hv_init.c b/arch/x86/hyperv/hv_init.c
index afdbda2dd7b7..577bbd143527 100644
--- a/arch/x86/hyperv/hv_init.c
+++ b/arch/x86/hyperv/hv_init.c
@@ -510,6 +510,7 @@ void __init hyperv_init(void)
memunmap(src);
hv_remap_tsc_clocksource();
+ hv_root_crash_init();
} else {
hypercall_msr.guest_physical_address = vmalloc_to_pfn(hv_hypercall_pg);
wrmsrq(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h
index abc4659f5809..207d953d7b90 100644
--- a/arch/x86/include/asm/mshyperv.h
+++ b/arch/x86/include/asm/mshyperv.h
@@ -292,6 +292,19 @@ static __always_inline u64 hv_raw_get_msr(unsigned int reg)
}
int hv_apicid_to_vp_index(u32 apic_id);
+#if IS_ENABLED(CONFIG_MSHV_ROOT)
+
+#ifdef CONFIG_CRASH_DUMP
+void hv_root_crash_init(void);
+void hv_crash_asm32(void);
+void hv_crash_asm64(void);
+void hv_crash_asm_end(void);
+#else /* CONFIG_CRASH_DUMP */
+static inline void hv_root_crash_init(void) {}
+#endif /* CONFIG_CRASH_DUMP */
+
+#endif /* CONFIG_MSHV_ROOT */
+
#else /* CONFIG_HYPERV */
static inline void hyperv_init(void) {}
static inline void hyperv_setup_mmu_ops(void) {}
--
2.36.1.vfs.0.0
^ permalink raw reply related
* [PATCH v2] Drivers: hv: Use better errno matches for HV_STATUS values
From: Easwar Hariharan @ 2025-10-06 23:08 UTC (permalink / raw)
To: K. Y. Srinivasan, Haiyang Zhang, Wei Liu, Dexuan Cui,
Stanislav Kinsburskii, Nuno Das Neves,
open list:Hyper-V/Azure CORE AND DRIVERS, open list
Cc: Easwar Hariharan
Use a better mapping of hypervisor status codes to errno values and
disambiguate the catch-all -EIO value. While here, remove the duplicate
INVALID_LP_INDEX and INVALID_REGISTER_VALUES hypervisor status entries.
Fixes: 3817854ba89201 ("hyperv: Log hypercall status codes as strings")
Signed-off-by: Easwar Hariharan <easwar.hariharan@linux.microsoft.com>
---
Changes in v2: Change more values, delete duplicated entries
v1: https://lore.kernel.org/all/20251002221347.402320-1-easwar.hariharan@linux.microsoft.com/
---
drivers/hv/hv_common.c | 22 ++++++++++------------
1 file changed, 10 insertions(+), 12 deletions(-)
diff --git a/drivers/hv/hv_common.c b/drivers/hv/hv_common.c
index 49898d10fafff..bb32471a53d68 100644
--- a/drivers/hv/hv_common.c
+++ b/drivers/hv/hv_common.c
@@ -758,32 +758,30 @@ static const struct hv_status_info hv_status_infos[] = {
_STATUS_INFO(HV_STATUS_SUCCESS, 0),
_STATUS_INFO(HV_STATUS_INVALID_HYPERCALL_CODE, -EINVAL),
_STATUS_INFO(HV_STATUS_INVALID_HYPERCALL_INPUT, -EINVAL),
- _STATUS_INFO(HV_STATUS_INVALID_ALIGNMENT, -EIO),
+ _STATUS_INFO(HV_STATUS_INVALID_ALIGNMENT, -EINVAL),
_STATUS_INFO(HV_STATUS_INVALID_PARAMETER, -EINVAL),
- _STATUS_INFO(HV_STATUS_ACCESS_DENIED, -EIO),
- _STATUS_INFO(HV_STATUS_INVALID_PARTITION_STATE, -EIO),
- _STATUS_INFO(HV_STATUS_OPERATION_DENIED, -EIO),
+ _STATUS_INFO(HV_STATUS_ACCESS_DENIED, -EACCES),
+ _STATUS_INFO(HV_STATUS_INVALID_PARTITION_STATE, -EINVAL),
+ _STATUS_INFO(HV_STATUS_OPERATION_DENIED, -EACCES),
_STATUS_INFO(HV_STATUS_UNKNOWN_PROPERTY, -EIO),
- _STATUS_INFO(HV_STATUS_PROPERTY_VALUE_OUT_OF_RANGE, -EIO),
+ _STATUS_INFO(HV_STATUS_PROPERTY_VALUE_OUT_OF_RANGE, -ERANGE),
_STATUS_INFO(HV_STATUS_INSUFFICIENT_MEMORY, -ENOMEM),
_STATUS_INFO(HV_STATUS_INVALID_PARTITION_ID, -EINVAL),
_STATUS_INFO(HV_STATUS_INVALID_VP_INDEX, -EINVAL),
_STATUS_INFO(HV_STATUS_NOT_FOUND, -EIO),
_STATUS_INFO(HV_STATUS_INVALID_PORT_ID, -EINVAL),
_STATUS_INFO(HV_STATUS_INVALID_CONNECTION_ID, -EINVAL),
- _STATUS_INFO(HV_STATUS_INSUFFICIENT_BUFFERS, -EIO),
- _STATUS_INFO(HV_STATUS_NOT_ACKNOWLEDGED, -EIO),
- _STATUS_INFO(HV_STATUS_INVALID_VP_STATE, -EIO),
+ _STATUS_INFO(HV_STATUS_INSUFFICIENT_BUFFERS, -ENOBUFS),
+ _STATUS_INFO(HV_STATUS_NOT_ACKNOWLEDGED, -EBUSY),
+ _STATUS_INFO(HV_STATUS_INVALID_VP_STATE, -EINVAL),
_STATUS_INFO(HV_STATUS_NO_RESOURCES, -EIO),
_STATUS_INFO(HV_STATUS_PROCESSOR_FEATURE_NOT_SUPPORTED, -EIO),
_STATUS_INFO(HV_STATUS_INVALID_LP_INDEX, -EINVAL),
_STATUS_INFO(HV_STATUS_INVALID_REGISTER_VALUE, -EINVAL),
- _STATUS_INFO(HV_STATUS_INVALID_LP_INDEX, -EIO),
- _STATUS_INFO(HV_STATUS_INVALID_REGISTER_VALUE, -EIO),
_STATUS_INFO(HV_STATUS_OPERATION_FAILED, -EIO),
- _STATUS_INFO(HV_STATUS_TIME_OUT, -EIO),
+ _STATUS_INFO(HV_STATUS_TIME_OUT, -ETIMEDOUT),
_STATUS_INFO(HV_STATUS_CALL_PENDING, -EIO),
- _STATUS_INFO(HV_STATUS_VTL_ALREADY_ENABLED, -EIO),
+ _STATUS_INFO(HV_STATUS_VTL_ALREADY_ENABLED, -EBUSY),
#undef _STATUS_INFO
};
--
2.43.0
^ permalink raw reply related
* Re: [PATCH v4 3/5] Drivers: hv: Batch GPA unmap operations to improve large region performance
From: Stanislav Kinsburskii @ 2025-10-06 23:32 UTC (permalink / raw)
To: Michael Kelley
Cc: kys@microsoft.com, haiyangz@microsoft.com, wei.liu@kernel.org,
decui@microsoft.com, linux-hyperv@vger.kernel.org,
linux-kernel@vger.kernel.org
In-Reply-To: <SN6PR02MB4157A4FBBF17A73E5D549DDFD4E3A@SN6PR02MB4157.namprd02.prod.outlook.com>
On Mon, Oct 06, 2025 at 05:09:07PM +0000, Michael Kelley wrote:
> From: Stanislav Kinsburskii <skinsburskii@linux.microsoft.com> Sent: Monday, October 6, 2025 8:06 AM
> >
> > Reduce overhead when unmapping large memory regions by batching GPA unmap
> > operations in 2MB-aligned chunks.
> >
> > Use a dedicated constant for batch size to improve code clarity and
> > maintainability.
> >
> > Signed-off-by: Stanislav Kinsburskii <skinsburskii@linux.microsoft.com>
> > ---
> > drivers/hv/mshv_root.h | 2 ++
> > drivers/hv/mshv_root_hv_call.c | 2 +-
> > drivers/hv/mshv_root_main.c | 28 +++++++++++++++++++++++++---
> > 3 files changed, 28 insertions(+), 4 deletions(-)
> >
> > diff --git a/drivers/hv/mshv_root.h b/drivers/hv/mshv_root.h
> > index e3931b0f12693..97e64d5341b6e 100644
> > --- a/drivers/hv/mshv_root.h
> > +++ b/drivers/hv/mshv_root.h
> > @@ -32,6 +32,8 @@ static_assert(HV_HYP_PAGE_SIZE == MSHV_HV_PAGE_SIZE);
> >
> > #define MSHV_PIN_PAGES_BATCH_SIZE (0x10000000ULL / HV_HYP_PAGE_SIZE)
> >
> > +#define MSHV_MAX_UNMAP_GPA_PAGES 512
> > +
> > struct mshv_vp {
> > u32 vp_index;
> > struct mshv_partition *vp_partition;
> > diff --git a/drivers/hv/mshv_root_hv_call.c b/drivers/hv/mshv_root_hv_call.c
> > index c9c274f29c3c6..0696024ccfe31 100644
> > --- a/drivers/hv/mshv_root_hv_call.c
> > +++ b/drivers/hv/mshv_root_hv_call.c
> > @@ -17,7 +17,7 @@
> > /* Determined empirically */
> > #define HV_INIT_PARTITION_DEPOSIT_PAGES 208
> > #define HV_MAP_GPA_DEPOSIT_PAGES 256
> > -#define HV_UMAP_GPA_PAGES 512
> > +#define HV_UMAP_GPA_PAGES MSHV_MAX_UNMAP_GPA_PAGES
> >
> > #define HV_PAGE_COUNT_2M_ALIGNED(pg_count) (!((pg_count) & (0x200 - 1)))
> >
> > diff --git a/drivers/hv/mshv_root_main.c b/drivers/hv/mshv_root_main.c
> > index 97e322f3c6b5e..b61bef6b9c132 100644
> > --- a/drivers/hv/mshv_root_main.c
> > +++ b/drivers/hv/mshv_root_main.c
> > @@ -1378,6 +1378,7 @@ mshv_map_user_memory(struct mshv_partition *partition,
> > static void mshv_partition_destroy_region(struct mshv_mem_region *region)
> > {
> > struct mshv_partition *partition = region->partition;
> > + u64 gfn, gfn_count, start_gfn, end_gfn;
> > u32 unmap_flags = 0;
> > int ret;
> >
> > @@ -1396,9 +1397,30 @@ static void mshv_partition_destroy_region(struct mshv_mem_region *region)
> > if (region->flags.large_pages)
> > unmap_flags |= HV_UNMAP_GPA_LARGE_PAGE;
> >
> > - /* ignore unmap failures and continue as process may be exiting */
> > - hv_call_unmap_gpa_pages(partition->pt_id, region->start_gfn,
> > - region->nr_pages, unmap_flags);
> > + start_gfn = region->start_gfn;
> > + end_gfn = region->start_gfn + region->nr_pages;
> > +
> > + for (gfn = start_gfn; gfn < end_gfn; gfn += gfn_count) {
> > + if (gfn % MSHV_MAX_UNMAP_GPA_PAGES)
> > + gfn_count = ALIGN(gfn, MSHV_MAX_UNMAP_GPA_PAGES) - gfn;
> > + else
> > + gfn_count = MSHV_MAX_UNMAP_GPA_PAGES;
>
> You could do the entire if/else as:
>
> gfn_count = ALIGN(gfn + 1, MSHV_MAX_UNMAP_GPA_PAGES) - gfn;
>
> Using "gfn + 1" handles the case where gfn is already aligned. Arguably, this is a bit
> more obscure, so it's just a suggestion.
>
> > +
> > + if (gfn + gfn_count > end_gfn)
> > + gfn_count = end_gfn - gfn;
>
> Or
> gfn_count = min(gfn_count, end_gfn - gfn);
>
> I usually prefer the "min" function instead of an "if" statement if logically
> the intent is to compute the minimum. But again, just a suggestion.
>
> > +
> > + /* Skip if all pages in this range if none is mapped */
> > + if (!memchr_inv(region->pages + (gfn - start_gfn), 0,
> > + gfn_count * sizeof(struct page *)))
> > + continue;
> > +
> > + ret = hv_call_unmap_gpa_pages(partition->pt_id, gfn,
> > + gfn_count, unmap_flags);
> > + if (ret)
> > + pt_err(partition,
> > + "Failed to unmap GPA pages %#llx-%#llx: %d\n",
> > + gfn, gfn + gfn_count - 1, ret);
> > + }
>
> Overall, I think this algorithm looks good and handles all the edge cases.
>
Thank you for your suggestions. I also generally prefer reducing the
code in a similar way, but in this case, I deliberately chose a more
elaborate approach to improve clarity.
So, if you don’t mind, I’d rather keep it as is, since this version is
easy to understand and self-documenting.
Thanks,
Stanislav
> Michael
>
> >
> > mshv_region_invalidate(region);
> >
> >
> >
>
^ permalink raw reply
* RE: [PATCH v4 3/5] Drivers: hv: Batch GPA unmap operations to improve large region performance
From: Michael Kelley @ 2025-10-07 0:02 UTC (permalink / raw)
To: Stanislav Kinsburskii
Cc: kys@microsoft.com, haiyangz@microsoft.com, wei.liu@kernel.org,
decui@microsoft.com, linux-hyperv@vger.kernel.org,
linux-kernel@vger.kernel.org
In-Reply-To: <aORRplcP1r17gave@skinsburskii.localdomain>
From: Stanislav Kinsburskii <skinsburskii@linux.microsoft.com> Sent: Monday, October 6, 2025 4:33 PM
>
> On Mon, Oct 06, 2025 at 05:09:07PM +0000, Michael Kelley wrote:
> > From: Stanislav Kinsburskii <skinsburskii@linux.microsoft.com> Sent: Monday, October 6, 2025 8:06 AM
> > >
> > > Reduce overhead when unmapping large memory regions by batching GPA unmap
> > > operations in 2MB-aligned chunks.
> > >
> > > Use a dedicated constant for batch size to improve code clarity and
> > > maintainability.
> > >
> > > Signed-off-by: Stanislav Kinsburskii <skinsburskii@linux.microsoft.com>
> > > ---
> > > drivers/hv/mshv_root.h | 2 ++
> > > drivers/hv/mshv_root_hv_call.c | 2 +-
> > > drivers/hv/mshv_root_main.c | 28 +++++++++++++++++++++++++---
> > > 3 files changed, 28 insertions(+), 4 deletions(-)
> > >
> > > diff --git a/drivers/hv/mshv_root.h b/drivers/hv/mshv_root.h
> > > index e3931b0f12693..97e64d5341b6e 100644
> > > --- a/drivers/hv/mshv_root.h
> > > +++ b/drivers/hv/mshv_root.h
> > > @@ -32,6 +32,8 @@ static_assert(HV_HYP_PAGE_SIZE == MSHV_HV_PAGE_SIZE);
> > >
> > > #define MSHV_PIN_PAGES_BATCH_SIZE (0x10000000ULL / HV_HYP_PAGE_SIZE)
> > >
> > > +#define MSHV_MAX_UNMAP_GPA_PAGES 512
> > > +
> > > struct mshv_vp {
> > > u32 vp_index;
> > > struct mshv_partition *vp_partition;
> > > diff --git a/drivers/hv/mshv_root_hv_call.c b/drivers/hv/mshv_root_hv_call.c
> > > index c9c274f29c3c6..0696024ccfe31 100644
> > > --- a/drivers/hv/mshv_root_hv_call.c
> > > +++ b/drivers/hv/mshv_root_hv_call.c
> > > @@ -17,7 +17,7 @@
> > > /* Determined empirically */
> > > #define HV_INIT_PARTITION_DEPOSIT_PAGES 208
> > > #define HV_MAP_GPA_DEPOSIT_PAGES 256
> > > -#define HV_UMAP_GPA_PAGES 512
> > > +#define HV_UMAP_GPA_PAGES MSHV_MAX_UNMAP_GPA_PAGES
> > >
> > > #define HV_PAGE_COUNT_2M_ALIGNED(pg_count) (!((pg_count) & (0x200 - 1)))
> > >
> > > diff --git a/drivers/hv/mshv_root_main.c b/drivers/hv/mshv_root_main.c
> > > index 97e322f3c6b5e..b61bef6b9c132 100644
> > > --- a/drivers/hv/mshv_root_main.c
> > > +++ b/drivers/hv/mshv_root_main.c
> > > @@ -1378,6 +1378,7 @@ mshv_map_user_memory(struct mshv_partition *partition,
> > > static void mshv_partition_destroy_region(struct mshv_mem_region *region)
> > > {
> > > struct mshv_partition *partition = region->partition;
> > > + u64 gfn, gfn_count, start_gfn, end_gfn;
> > > u32 unmap_flags = 0;
> > > int ret;
> > >
> > > @@ -1396,9 +1397,30 @@ static void mshv_partition_destroy_region(struct mshv_mem_region *region)
> > > if (region->flags.large_pages)
> > > unmap_flags |= HV_UNMAP_GPA_LARGE_PAGE;
> > >
> > > - /* ignore unmap failures and continue as process may be exiting */
> > > - hv_call_unmap_gpa_pages(partition->pt_id, region->start_gfn,
> > > - region->nr_pages, unmap_flags);
> > > + start_gfn = region->start_gfn;
> > > + end_gfn = region->start_gfn + region->nr_pages;
> > > +
> > > + for (gfn = start_gfn; gfn < end_gfn; gfn += gfn_count) {
> > > + if (gfn % MSHV_MAX_UNMAP_GPA_PAGES)
> > > + gfn_count = ALIGN(gfn, MSHV_MAX_UNMAP_GPA_PAGES) - gfn;
> > > + else
> > > + gfn_count = MSHV_MAX_UNMAP_GPA_PAGES;
> >
> > You could do the entire if/else as:
> >
> > gfn_count = ALIGN(gfn + 1, MSHV_MAX_UNMAP_GPA_PAGES) - gfn;
> >
> > Using "gfn + 1" handles the case where gfn is already aligned. Arguably, this is a bit
> > more obscure, so it's just a suggestion.
> >
> > > +
> > > + if (gfn + gfn_count > end_gfn)
> > > + gfn_count = end_gfn - gfn;
> >
> > Or
> > gfn_count = min(gfn_count, end_gfn - gfn);
> >
> > I usually prefer the "min" function instead of an "if" statement if logically
> > the intent is to compute the minimum. But again, just a suggestion.
> >
> > > +
> > > + /* Skip if all pages in this range if none is mapped */
> > > + if (!memchr_inv(region->pages + (gfn - start_gfn), 0,
> > > + gfn_count * sizeof(struct page *)))
> > > + continue;
> > > +
> > > + ret = hv_call_unmap_gpa_pages(partition->pt_id, gfn,
> > > + gfn_count, unmap_flags);
> > > + if (ret)
> > > + pt_err(partition,
> > > + "Failed to unmap GPA pages %#llx-%#llx: %d\n",
> > > + gfn, gfn + gfn_count - 1, ret);
> > > + }
> >
> > Overall, I think this algorithm looks good and handles all the edge cases.
> >
>
> Thank you for your suggestions. I also generally prefer reducing the
> code in a similar way, but in this case, I deliberately chose a more
> elaborate approach to improve clarity.
>
> So, if you don’t mind, I’d rather keep it as is, since this version is
> easy to understand and self-documenting.
>
Yes, that's fine with me.
Reviewed-by: Michael Kelley <mhklinux@outlook.com>
^ permalink raw reply
* Re: [PATCH] scsi: storvsc: Prefer returning channel with the same CPU as on the I/O issuing CPU
From: Martin K. Petersen @ 2025-10-07 2:05 UTC (permalink / raw)
To: longli
Cc: K. Y. Srinivasan, Haiyang Zhang, Wei Liu, Dexuan Cui,
James E.J. Bottomley, Martin K. Petersen, James Bottomley,
linux-hyperv, linux-scsi, linux-kernel, Long Li
In-Reply-To: <1759381530-7414-1-git-send-email-longli@linux.microsoft.com>
Long,
> When selecting an outgoing channel for I/O, storvsc tries to select a
> channel with a returning CPU that is not the same as issuing CPU. This
> worked well in the past, however it doesn't work well when the Hyper-V
> exposes a large number of channels (up to the number of all CPUs). Use
> a different CPU for returning channel is not efficient on Hyper-V.
Applied to 6.18/scsi-staging, thanks!
--
Martin K. Petersen
^ permalink raw reply
* Re: [PATCH hyperv-next v6 01/17] Documentation: hyperv: Confidential VMBus
From: Bagas Sanjaya @ 2025-10-07 2:23 UTC (permalink / raw)
To: Roman Kisel, arnd, bp, corbet, dave.hansen, decui, haiyangz, hpa,
kys, mikelley, mingo, tglx, Tianyu.Lan, wei.liu, x86,
linux-hyperv, linux-doc, linux-kernel, linux-arch
Cc: benhill, bperkins, sunilmut
In-Reply-To: <20251003222710.6257-2-romank@linux.microsoft.com>
[-- Attachment #1: Type: text/plain, Size: 442 bytes --]
On Fri, Oct 03, 2025 at 03:26:54PM -0700, Roman Kisel wrote:
> +The data is transferred directly between the VM and a vPCI device (a.k.a.
> +a PCI pass-thru device, see :doc:`vpci`) that is directly assigned to VTL2
> +and that supports encrypted memory. In such a case, neither the host partition
Nit: You can also write the cross-reference simply as vpci.rst.
Thanks.
--
An old man doll... just what I always wanted! - Clara
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 228 bytes --]
^ permalink raw reply
* [GIT PULL] Hyper-V patches for 6.18
From: Wei Liu @ 2025-10-07 5:55 UTC (permalink / raw)
To: Linus Torvalds
Cc: Wei Liu, Linux Kernel List, Linux on Hyper-V List, kys, haiyangz,
decui
Hi Linus,
The following changes since commit 8f5ae30d69d7543eee0d70083daf4de8fe15d585:
Linux 6.17-rc1 (2025-08-10 19:41:16 +0300)
are available in the Git repository at:
ssh://git@gitolite.kernel.org/pub/scm/linux/kernel/git/hyperv/linux.git tags/hyperv-next-signed-20251006
for you to fetch changes up to b595edcb24727e7f93e7962c3f6f971cc16dd29e:
hyperv: Remove the spurious null directive line (2025-10-02 21:21:24 +0000)
----------------------------------------------------------------
hyperv-next for v6.18
- Unify guest entry code for KVM and MSHV (Sean Christopherson)
- Switch Hyper-V MSI domain to use msi_create_parent_irq_domain() (Nam
Cao)
- Add CONFIG_HYPERV_VMBUS and limit the semantics of CONFIG_HYPERV (Mukesh
Rathor)
- Add kexec/kdump support on Azure CVMs (Vitaly Kuznetsov)
- Deprecate hyperv_fb in favor of Hyper-V DRM driver (Prasanna Kumar T S
M)
- Miscellaneous enhancements, fixes and cleanups (Abhishek Tiwari, Alok
Tiwari, Nuno Das Neves, Wei Liu, Roman Kisel, Michael Kelley))
----------------------------------------------------------------
Abhishek Tiwari (1):
Drivers: hv: util: Cosmetic changes for hv_utils_transport.c
Alok Tiwari (3):
Drivers: hv: vmbus: Clean up sscanf format specifier in target_cpu_store()
Drivers: hv: vmbus: Fix sysfs output format for ring buffer index
Drivers: hv: vmbus: Fix typos in vmbus_drv.c
Michael Kelley (1):
Drivers: hv: Simplify data structures for VMBus channel close message
Mukesh Rathor (2):
Drivers: hv: Add CONFIG_HYPERV_VMBUS option
Drivers: hv: Make CONFIG_HYPERV bool
Nam Cao (1):
x86/hyperv: Switch to msi_create_parent_irq_domain()
Nuno Das Neves (2):
hyperv: Add missing field to hv_output_map_device_interrupt
mshv: Add support for a new parent partition configuration
Prasanna Kumar T S M (2):
fbdev/hyperv_fb: deprecate this in favor of Hyper-V DRM driver
MAINTAINERS: Mark hyperv_fb driver Obsolete
Roman Kisel (1):
hyperv: Remove the spurious null directive line
Sean Christopherson (4):
mshv: Handle NEED_RESCHED_LAZY before transferring to guest
entry/kvm: KVM: Move KVM details related to signal/-EINTR into KVM proper
entry: Rename "kvm" entry code assets to "virt" to genericize APIs
mshv: Use common "entry virt" APIs to do work in root before running guest
Vitaly Kuznetsov (1):
x86/hyperv: Add kexec/kdump support on Azure CVMs
Wei Liu (1):
clocksource: hyper-v: Skip unnecessary checks for the root partition
MAINTAINERS | 13 +-
arch/arm64/kvm/Kconfig | 2 +-
arch/arm64/kvm/arm.c | 3 +-
arch/loongarch/kvm/Kconfig | 2 +-
arch/loongarch/kvm/vcpu.c | 3 +-
arch/riscv/kvm/Kconfig | 2 +-
arch/riscv/kvm/vcpu.c | 3 +-
arch/x86/hyperv/irqdomain.c | 111 ++++++++++-----
arch/x86/hyperv/ivm.c | 211 +++++++++++++++++++++++++++-
arch/x86/kernel/cpu/mshyperv.c | 11 +-
arch/x86/kvm/Kconfig | 2 +-
arch/x86/kvm/vmx/vmx.c | 1 -
arch/x86/kvm/x86.c | 3 +-
drivers/Makefile | 2 +-
drivers/clocksource/hyperv_timer.c | 10 +-
drivers/gpu/drm/Kconfig | 2 +-
drivers/hid/Kconfig | 2 +-
drivers/hv/Kconfig | 15 +-
drivers/hv/Makefile | 4 +-
drivers/hv/channel.c | 2 +-
drivers/hv/hv_common.c | 22 +--
drivers/hv/hv_utils_transport.c | 10 +-
drivers/hv/mshv.h | 2 -
drivers/hv/mshv_common.c | 22 ---
drivers/hv/mshv_root_main.c | 57 +++-----
drivers/hv/vmbus_drv.c | 10 +-
drivers/input/serio/Kconfig | 4 +-
drivers/net/hyperv/Kconfig | 2 +-
drivers/pci/Kconfig | 2 +-
drivers/scsi/Kconfig | 2 +-
drivers/uio/Kconfig | 2 +-
drivers/video/fbdev/Kconfig | 7 +-
drivers/video/fbdev/hyperv_fb.c | 2 +
include/asm-generic/mshyperv.h | 19 ++-
include/hyperv/hvgdk_mini.h | 2 -
include/hyperv/hvhdk_mini.h | 1 +
include/linux/{entry-kvm.h => entry-virt.h} | 19 +--
include/linux/hyperv.h | 7 +-
include/linux/kvm_host.h | 17 ++-
include/linux/rcupdate.h | 2 +-
kernel/entry/Makefile | 2 +-
kernel/entry/{kvm.c => virt.c} | 15 +-
kernel/rcu/tree.c | 6 +-
net/vmw_vsock/Kconfig | 2 +-
virt/kvm/Kconfig | 2 +-
45 files changed, 449 insertions(+), 193 deletions(-)
rename include/linux/{entry-kvm.h => entry-virt.h} (83%)
rename kernel/entry/{kvm.c => virt.c} (66%)
^ permalink raw reply
* RE: [PATCH] scsi: storvsc: Prefer returning channel with the same CPU as on the I/O issuing CPU
From: Michael Kelley @ 2025-10-07 15:41 UTC (permalink / raw)
To: longli@linux.microsoft.com, K. Y. Srinivasan, Haiyang Zhang,
Wei Liu, Dexuan Cui, James E.J. Bottomley, Martin K. Petersen,
James Bottomley, linux-hyperv@vger.kernel.org,
linux-scsi@vger.kernel.org, linux-kernel@vger.kernel.org
Cc: Long Li
In-Reply-To: <1759381530-7414-1-git-send-email-longli@linux.microsoft.com>
From: longli@linux.microsoft.com <longli@linux.microsoft.com> Sent: Wednesday, October 1, 2025 10:06 PM
>
> When selecting an outgoing channel for I/O, storvsc tries to select a
> channel with a returning CPU that is not the same as issuing CPU. This
> worked well in the past, however it doesn't work well when the Hyper-V
> exposes a large number of channels (up to the number of all CPUs). Use
> a different CPU for returning channel is not efficient on Hyper-V.
>
> Change this behavior by preferring to the channel with the same CPU
> as the current I/O issuing CPU whenever possible.
>
> Tests have shown improvements in newer Hyper-V/Azure environment, and
> no regression with older Hyper-V/Azure environments.
>
> Tested-by: Raheel Abdul Faizy <rabdulfaizy@microsoft.com>
> Signed-off-by: Long Li <longli@microsoft.com>
> ---
> drivers/scsi/storvsc_drv.c | 96 ++++++++++++++++++--------------------
> 1 file changed, 45 insertions(+), 51 deletions(-)
>
> diff --git a/drivers/scsi/storvsc_drv.c b/drivers/scsi/storvsc_drv.c
> index d9e59204a9c3..092939791ea0 100644
> --- a/drivers/scsi/storvsc_drv.c
> +++ b/drivers/scsi/storvsc_drv.c
> @@ -1406,14 +1406,19 @@ static struct vmbus_channel *get_og_chn(struct storvsc_device *stor_device,
> }
>
> /*
> - * Our channel array is sparsley populated and we
> + * Our channel array could be sparsley populated and we
> * initiated I/O on a processor/hw-q that does not
> * currently have a designated channel. Fix this.
> * The strategy is simple:
> - * I. Ensure NUMA locality
> - * II. Distribute evenly (best effort)
> + * I. Prefer the channel associated with the current CPU
> + * II. Ensure NUMA locality
> + * III. Distribute evenly (best effort)
> */
>
> + /* Prefer the channel on the I/O issuing processor/hw-q */
> + if (cpumask_test_cpu(q_num, &stor_device->alloced_cpus))
> + return stor_device->stor_chns[q_num];
> +
Hmmm. When get_og_chn() is called, we know that
stor_device->stor_chns[q_num] is NULL since storvsc_do_io() has
already handled the non-NULL case. And the checks are all done
with stor_device->lock held, so the stor_chns array can't change.
Hence the above code will return NULL, which will cause a NULL
reference when storvsc_do_io() sends out the VMBus packet.
My recollection is that get_og_chan() is called when there is no
channel that interrupts the current CPU (that's what it means
for stor_device->stor_chns[<current CPU>] to be NULL). So the
algorithm must pick a channel that interrupts some other CPU,
preferably a CPU in the current NUMA node. Adding code to prefer
the channel associated with the current CPU doesn't make sense in
get_og_chn(), as get_og_chn() is only called when it is already
known that there is no such channel.
Or is there a case that I'm missing? Regardless, the above code
seems problematic because it would return NULL.
Michael
> node_mask = cpumask_of_node(cpu_to_node(q_num));
>
> num_channels = 0;
> @@ -1469,59 +1474,48 @@ static int storvsc_do_io(struct hv_device *device,
> /* See storvsc_change_target_cpu(). */
> outgoing_channel = READ_ONCE(stor_device->stor_chns[q_num]);
> if (outgoing_channel != NULL) {
> - if (outgoing_channel->target_cpu == q_num) {
> - /*
> - * Ideally, we want to pick a different channel if
> - * available on the same NUMA node.
> - */
> - node_mask = cpumask_of_node(cpu_to_node(q_num));
> - for_each_cpu_wrap(tgt_cpu,
> - &stor_device->alloced_cpus, q_num + 1) {
> - if (!cpumask_test_cpu(tgt_cpu, node_mask))
> - continue;
> - if (tgt_cpu == q_num)
> - continue;
> - channel = READ_ONCE(
> - stor_device->stor_chns[tgt_cpu]);
> - if (channel == NULL)
> - continue;
> - if (hv_get_avail_to_write_percent(
> - &channel->outbound)
> - > ring_avail_percent_lowater) {
> - outgoing_channel = channel;
> - goto found_channel;
> - }
> - }
> + if (hv_get_avail_to_write_percent(&outgoing_channel->outbound)
> + > ring_avail_percent_lowater)
> + goto found_channel;
>
> - /*
> - * All the other channels on the same NUMA node are
> - * busy. Try to use the channel on the current CPU
> - */
> - if (hv_get_avail_to_write_percent(
> - &outgoing_channel->outbound)
> - > ring_avail_percent_lowater)
> + /*
> + * Channel is busy, try to find a channel on the same NUMA node
> + */
> + node_mask = cpumask_of_node(cpu_to_node(q_num));
> + for_each_cpu_wrap(tgt_cpu, &stor_device->alloced_cpus,
> + q_num + 1) {
> + if (!cpumask_test_cpu(tgt_cpu, node_mask))
> + continue;
> + channel = READ_ONCE(stor_device->stor_chns[tgt_cpu]);
> + if (!channel)
> + continue;
> + if (hv_get_avail_to_write_percent(&channel->outbound)
> + > ring_avail_percent_lowater) {
> + outgoing_channel = channel;
> goto found_channel;
> + }
> + }
>
> - /*
> - * If we reach here, all the channels on the current
> - * NUMA node are busy. Try to find a channel in
> - * other NUMA nodes
> - */
> - for_each_cpu(tgt_cpu, &stor_device->alloced_cpus) {
> - if (cpumask_test_cpu(tgt_cpu, node_mask))
> - continue;
> - channel = READ_ONCE(
> - stor_device->stor_chns[tgt_cpu]);
> - if (channel == NULL)
> - continue;
> - if (hv_get_avail_to_write_percent(
> - &channel->outbound)
> - > ring_avail_percent_lowater) {
> - outgoing_channel = channel;
> - goto found_channel;
> - }
> + /*
> + * If we reach here, all the channels on the current
> + * NUMA node are busy. Try to find a channel in
> + * all NUMA nodes
> + */
> + for_each_cpu_wrap(tgt_cpu, &stor_device->alloced_cpus,
> + q_num + 1) {
> + channel = READ_ONCE(stor_device->stor_chns[tgt_cpu]);
> + if (!channel)
> + continue;
> + if (hv_get_avail_to_write_percent(&channel->outbound)
> + > ring_avail_percent_lowater) {
> + outgoing_channel = channel;
> + goto found_channel;
> }
> }
> + /*
> + * If we reach here, all the channels are busy. Use the
> + * original channel found.
> + */
> } else {
> spin_lock_irqsave(&stor_device->lock, flags);
> outgoing_channel = stor_device->stor_chns[q_num];
> --
> 2.34.1
>
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox