[PATCH 2/3] x86/HVM: defer IOTLB flushes across a grant unmap batch

[Luca Di Maio <luca@edera.dev>]

gnttab_unmap_grant_ref() flushes the TLBs of a PV caller once per group
of GNTTAB_UNMAP_BATCH_SIZE unmaps. A translated caller instead gets an
IOTLB flush per page from the p2m code, so netback in a PVH dom0 issues
one for every page a domU transmits.

Open a batch for translated callers before each group, and end it next
to gnttab_flush_tlb(), so before unmap_common_complete() releases the
pages. Ending the batch flushes the smallest aligned power-of-two range
holding all the unmapped gfns, as VT-d flushes any other range as a
whole domain.

The batch lives in the vCPU, and iommu_dont_flush_iotlb isn't held
across the guest copies between the unmaps, as those can sleep on a
vm_event ring and change the p2m, e.g. by unsharing a page. If the
flush fails, the pages may still be reachable through the IOMMU, so
the group isn't completed and its pages leak.

PV callers, Arm, and other callers of replace_grant_host_mapping() are
unchanged.

Assisted-by: Claude Code:claude-opus-5-5
Signed-off-by: Luca Di Maio <luca@edera.dev>

---
 xen/arch/arm/include/asm/grant_table.h     |  3 ++
 xen/arch/x86/hvm/grant_table.c             | 47 +++++++++++++++++++++-
 xen/arch/x86/include/asm/grant_table.h     | 13 ++++++
 xen/arch/x86/include/asm/hvm/grant_table.h |  5 +++
 xen/arch/x86/include/asm/hvm/vcpu.h        |  9 +++++
 xen/common/grant_table.c                   |  6 +++
 6 files changed, 82 insertions(+), 1 deletion(-)

diff --git a/xen/arch/arm/include/asm/grant_table.h b/xen/arch/arm/include/asm/grant_table.h
index 7661a95341..66f90083db 100644
--- a/xen/arch/arm/include/asm/grant_table.h
+++ b/xen/arch/arm/include/asm/grant_table.h
@@ -40,6 +40,9 @@ int create_grant_host_mapping(uint64_t gpaddr, mfn_t frame,
 int replace_grant_host_mapping(uint64_t gpaddr, mfn_t frame,
                                uint64_t new_gpaddr, unsigned int flags);
 
+static inline void gnttab_unmap_batch_begin(void) {}
+static inline int gnttab_unmap_batch_end(void) { return 0; }
+
 /*
  * The region used by Xen on the memory will never be mapped in DOM0
  * memory layout. Therefore it can be used for the grant table.
diff --git a/xen/arch/x86/hvm/grant_table.c b/xen/arch/x86/hvm/grant_table.c
index afe449d888..1d7f9b19f9 100644
--- a/xen/arch/x86/hvm/grant_table.c
+++ b/xen/arch/x86/hvm/grant_table.c
@@ -41,6 +41,8 @@ int replace_grant_p2m_mapping(uint64_t addr, mfn_t frame,
     p2m_type_t type;
     mfn_t old_mfn;
     struct domain *d = current->domain;
+    struct hvm_grant_unmap_batch *batch = &current->arch.hvm.grant_unmap_batch;
+    int rc;
 
     if ( new_addr != 0 || (flags & GNTMAP_contains_pte) )
         return GNTST_general_error;
@@ -54,7 +56,25 @@ int replace_grant_p2m_mapping(uint64_t addr, mfn_t frame,
                  type, mfn_x(old_mfn), mfn_x(frame));
         return GNTST_general_error;
     }
-    if ( p2m_remove_page(d, _gfn(gfn), frame, PAGE_ORDER_4K) )
+
+    /* Not across the guest copies, which can change the p2m or sleep. */
+    if ( batch->open )
+    {
+        this_cpu(iommu_skipped_flush_flags) = 0;
+        this_cpu(iommu_dont_flush_iotlb) = true;
+    }
+
+    rc = p2m_remove_page(d, _gfn(gfn), frame, PAGE_ORDER_4K);
+
+    if ( batch->open )
+    {
+        this_cpu(iommu_dont_flush_iotlb) = false;
+        batch->iommu_flags |= this_cpu(iommu_skipped_flush_flags);
+        batch->gfn_lo = min(batch->gfn_lo, gfn);
+        batch->gfn_hi = max(batch->gfn_hi, gfn);
+    }
+
+    if ( rc )
     {
         put_gfn(d, gfn);
         return GNTST_general_error;
@@ -64,6 +84,31 @@ int replace_grant_p2m_mapping(uint64_t addr, mfn_t frame,
     return GNTST_okay;
 }
 
+void grant_p2m_unmap_batch_begin(void)
+{
+    struct hvm_grant_unmap_batch *batch = &current->arch.hvm.grant_unmap_batch;
+
+    ASSERT(!batch->open);
+    *batch = (struct hvm_grant_unmap_batch){ .open = true, .gfn_lo = ~0UL };
+}
+
+int grant_p2m_unmap_batch_end(void)
+{
+    struct domain *d = current->domain;
+    struct hvm_grant_unmap_batch *batch = &current->arch.hvm.grant_unmap_batch;
+    unsigned int order;
+
+    batch->open = false;
+
+    if ( !batch->iommu_flags )
+        return 0;
+
+    /* VT-d flushes any range but an aligned power of two as a whole domain. */
+    order = flsl(batch->gfn_lo ^ batch->gfn_hi);
+    return iommu_iotlb_flush(d, _dfn(ROUNDDOWN(batch->gfn_lo, 1UL << order)),
+                             1UL << order, batch->iommu_flags);
+}
+
 /*
  * Local variables:
  * mode: C
diff --git a/xen/arch/x86/include/asm/grant_table.h b/xen/arch/x86/include/asm/grant_table.h
index e939c9bacb..3aae47092f 100644
--- a/xen/arch/x86/include/asm/grant_table.h
+++ b/xen/arch/x86/include/asm/grant_table.h
@@ -32,6 +32,19 @@ static inline int replace_grant_host_mapping(uint64_t addr, mfn_t frame,
     return replace_grant_pv_mapping(addr, frame, new_addr, flags);
 }
 
+static inline void gnttab_unmap_batch_begin(void)
+{
+    if ( paging_mode_external(current->domain) )
+        grant_p2m_unmap_batch_begin();
+}
+
+static inline int __must_check gnttab_unmap_batch_end(void)
+{
+    if ( paging_mode_external(current->domain) )
+        return grant_p2m_unmap_batch_end();
+    return 0;
+}
+
 #define gnttab_get_frame_gfn(gt, st, idx) ({                             \
     mfn_t mfn_ = (st) ? gnttab_status_mfn(gt, idx)                       \
                       : gnttab_shared_mfn(gt, idx);                      \
diff --git a/xen/arch/x86/include/asm/hvm/grant_table.h b/xen/arch/x86/include/asm/hvm/grant_table.h
index 01e23f79b8..9877262f51 100644
--- a/xen/arch/x86/include/asm/hvm/grant_table.h
+++ b/xen/arch/x86/include/asm/hvm/grant_table.h
@@ -19,6 +19,8 @@ int create_grant_p2m_mapping(uint64_t addr, mfn_t frame,
                              unsigned int cache_flags);
 int replace_grant_p2m_mapping(uint64_t addr, mfn_t frame,
                               uint64_t new_addr, unsigned int flags);
+void grant_p2m_unmap_batch_begin(void);
+int grant_p2m_unmap_batch_end(void);
 
 #else
 
@@ -37,6 +39,9 @@ static inline int replace_grant_p2m_mapping(uint64_t addr, mfn_t frame,
     return GNTST_general_error;
 }
 
+static inline void grant_p2m_unmap_batch_begin(void) {}
+static inline int grant_p2m_unmap_batch_end(void) { return 0; }
+
 #endif
 
 #endif /* __X86_HVM_GRANT_TABLE_H__ */
diff --git a/xen/arch/x86/include/asm/hvm/vcpu.h b/xen/arch/x86/include/asm/hvm/vcpu.h
index 876c230784..12b7e9b707 100644
--- a/xen/arch/x86/include/asm/hvm/vcpu.h
+++ b/xen/arch/x86/include/asm/hvm/vcpu.h
@@ -108,6 +108,13 @@ struct altp2mvcpu {
 
 #define vcpu_altp2m(v) ((v)->arch.hvm.avcpu)
 
+/* Per vCPU, as the guest copies between the unmaps can sleep. */
+struct hvm_grant_unmap_batch {
+    bool          open;
+    unsigned int  iommu_flags;
+    unsigned long gfn_lo, gfn_hi;
+};
+
 struct hvm_vcpu {
     /* Guest control-register and EFER values, just as the guest sees them. */
     unsigned long       guest_cr[5];
@@ -172,6 +179,8 @@ struct hvm_vcpu {
     /* Pending hw/sw interrupt (.vector = -1 means nothing pending). */
     struct x86_event     inject_event;
 
+    struct hvm_grant_unmap_batch grant_unmap_batch;
+
 #ifdef CONFIG_VIRIDIAN
     struct viridian_vcpu *viridian;
 #endif
diff --git a/xen/common/grant_table.c b/xen/common/grant_table.c
index b15805f3f9..ade42b4cb4 100644
--- a/xen/common/grant_table.c
+++ b/xen/common/grant_table.c
@@ -1722,6 +1722,7 @@ gnttab_unmap_grant_ref(
         c = min(count, (unsigned int)GNTTAB_UNMAP_BATCH_SIZE);
         partial_done = 0;
 
+        gnttab_unmap_batch_begin();
         for ( i = 0; i < c; i++ )
         {
             if ( unlikely(__copy_from_guest(&op, uop, 1)) )
@@ -1734,6 +1735,9 @@ gnttab_unmap_grant_ref(
         }
 
         gnttab_flush_tlb(current->domain);
+        /* After a failed flush the pages may still be reachable: leak them. */
+        if ( gnttab_unmap_batch_end() )
+            partial_done = 0;
 
         for ( i = 0; i < partial_done; i++ )
             unmap_common_complete(&common[i]);
@@ -1749,6 +1753,8 @@ gnttab_unmap_grant_ref(
 
 fault:
     gnttab_flush_tlb(current->domain);
+    if ( gnttab_unmap_batch_end() )
+        partial_done = 0;
 
     for ( i = 0; i < partial_done; i++ )
         unmap_common_complete(&common[i]);
-- 
2.54.0