From: Aaron Lewis <aaronlewis@google.com>
To: kvm@vger.kernel.org
Cc: alex@shazbot.org, dmatlack@google.com, jgg@nvidia.com,
Aaron Lewis <aaronlewis@google.com>
Subject: [PATCH 2/3] vfio/type1: Track GUP vs PFNMAP in DMA map/unmap
Date: Thu, 19 Nov 2026 18:56:58 +0000 [thread overview]
Message-ID: <20261119185659.889654-3-aaronlewis@google.com> (raw)
In-Reply-To: <20261119185659.889654-1-aaronlewis@google.com>
Both vfio_pin_pages_remote() and vfio_unpin_pages_remote() use
PageReserved() as a heuristic to infer whether a page was PFNMAP'd or
GUP-pinned in vaddr_get_pfns(). That heuristic breaks down when the same
reserved struct pages can appear on both paths:
- DevDax pages mapped from /dev/dax* are reserved and GUP-pinned.
- Those same pages mapped from /dev/mem (PFNMAP) are also reserved,
but they are *not* GUP-pinned.
Because of this, when unpinning the DMA region, vfio_unpin_pages_remote()
doesn't know whether those reserved pages were GUP-pinned and skips
unpinning them altogether, *leaking page pins and refcounts*.
Fix this by replacing has_rsvd and has_non_rsvd in struct vfio_dma with
has_pfnmap and has_gup_pinned, tracking which path vaddr_get_pfns()
actually used, and clearing them when a DMA range is unpinned. Since
PFNMAP pages bypass struct vfio_batch, this also fixes locked_vm
accounting and batch error-unwind unpinning for reserved GUP-pinned
pages.
This isn't an exhaustive fix because mixed mappings are still allowed.
For example, if a single DMA region contains both GUP-pinned and
PFNMAP pages, vfio_unpin_pages_remote() can't tell them apart. Handle
this by preserving the existing put_pfn() fallback, and add
a comment explaining the situation.
Lastly, clear the new flags once a vfio_dma is fully unpinned. A vfio_dma
can outlive its pins and be pinned again later, after
vfio_iommu_unmap_unpin_reaccount() or a failed vfio_iommu_replay(), by
which time the user may have remapped the range. A stale has_rsvd only
selected the slower unpin path, but a stale has_pfnmap would send a
mapping that is now entirely GUP-pinned down the wrong unpin path.
Fixes: 3565fce3a659 ("mm, x86: get_user_pages() for dax mappings")
Signed-off-by: Aaron Lewis <aaronlewis@google.com>
---
drivers/vfio/vfio_iommu_type1.c | 103 ++++++++++++++++++--------------
1 file changed, 58 insertions(+), 45 deletions(-)
diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
index 7bad0508dd21..ee40ba942fd5 100644
--- a/drivers/vfio/vfio_iommu_type1.c
+++ b/drivers/vfio/vfio_iommu_type1.c
@@ -93,8 +93,8 @@ struct vfio_dma {
bool iommu_mapped;
bool lock_cap; /* capable(CAP_IPC_LOCK) */
bool vaddr_invalid;
- bool has_rsvd; /* has 1 or more rsvd pfns */
- bool has_non_rsvd; /* has 1 or more !rsvd pfns */
+ bool has_pfnmap; /* has 1 or more pfnmap pfns */
+ bool has_gup_pinned; /* has 1 or more GUP-pinned pfns */
struct task_struct *task;
struct rb_root pfn_list; /* Ex-user pinned pfn list */
unsigned long *bitmap;
@@ -416,6 +416,13 @@ static struct vfio_pfn *vfio_iova_get_vfio_pfn(struct vfio_dma *dma,
return vpfn;
}
+static inline void put_gup_pfns(unsigned long start_pfn, unsigned long npage,
+ int prot)
+{
+ unpin_user_page_range_dirty_lock(pfn_to_page(start_pfn), npage,
+ prot & IOMMU_WRITE);
+}
+
static int vfio_iova_put_vfio_pfn(struct vfio_dma *dma, struct vfio_pfn *vpfn)
{
int ret = 0;
@@ -523,12 +530,11 @@ static void vfio_batch_init_single(struct vfio_batch *batch)
static void vfio_batch_unpin(struct vfio_batch *batch, struct vfio_dma *dma)
{
- while (batch->size) {
- unsigned long pfn = page_to_pfn(batch->pages[batch->offset]);
-
- put_pfn(pfn, dma->prot);
- batch->offset++;
- batch->size--;
+ if (batch->size) {
+ unpin_user_pages_dirty_lock(&batch->pages[batch->offset],
+ batch->size,
+ dma->prot & IOMMU_WRITE);
+ batch->size = 0;
}
}
@@ -673,6 +679,15 @@ static long vpfn_pages(struct vfio_dma *dma,
return ret;
}
+/*
+ * After a successful vaddr_get_pfns(), GUP populates batch->pages while
+ * VM_PFNMAP leaves the batch empty.
+ */
+static bool vfio_batch_is_pfnmap(const struct vfio_batch *batch)
+{
+ return batch->size == 0;
+}
+
/*
* Attempt to pin pages. We really don't want to track all the pfns and
* the iommu can only map chunks of consecutive pfns anyway, so get the
@@ -685,7 +700,7 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
unsigned long pfn;
struct mm_struct *mm = current->mm;
long ret, pinned = 0, lock_acct = 0;
- bool rsvd;
+ bool pfnmap = false;
dma_addr_t iova = vaddr - dma->vaddr + dma->iova;
/* This code path is only user initiated */
@@ -696,7 +711,6 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
/* Leftover pages in batch from an earlier call. */
*pfn_base = page_to_pfn(batch->pages[batch->offset]);
pfn = *pfn_base;
- rsvd = is_invalid_reserved_pfn(*pfn_base);
} else {
*pfn_base = 0;
}
@@ -721,12 +735,18 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
if (!*pfn_base) {
*pfn_base = pfn;
- rsvd = is_invalid_reserved_pfn(*pfn_base);
+ pfnmap = vfio_batch_is_pfnmap(batch);
+ } else if (pfnmap != vfio_batch_is_pfnmap(batch)) {
+ /*
+ * Handle PFNMAP and GUP pages separately; any
+ * GUP pages remain in batch for the next call.
+ */
+ goto out;
}
/* Handle pfnmap */
- if (!batch->size) {
- if (pfn != *pfn_base + pinned || !rsvd)
+ if (pfnmap) {
+ if (pfn != *pfn_base + pinned)
goto out;
pinned += ret;
@@ -742,32 +762,20 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
* due to the fact that vaddr_get_pfns() needs to provide the
* initial pfn for pfnmaps. Therefore to reduce redundancy,
* the next pfn is fetched at the end of the loop.
- * A PageReserved() page could still qualify as page backed
- * and rsvd here, and therefore continues to use the batch.
*/
while (true) {
- long nr_pages, acct_pages = 0;
+ long nr_pages, acct_pages;
- if (pfn != *pfn_base + pinned ||
- rsvd != is_invalid_reserved_pfn(pfn))
+ if (pfn != *pfn_base + pinned)
goto out;
- /*
- * Using GUP with the FOLL_LONGTERM in
- * vaddr_get_pfns() will not return invalid
- * or reserved pages.
- */
nr_pages = num_pages_contiguous(
&batch->pages[batch->offset],
batch->size);
- if (!rsvd) {
- acct_pages = nr_pages;
- acct_pages -= vpfn_pages(dma, iova, nr_pages);
- }
+ acct_pages = nr_pages - vpfn_pages(dma, iova, nr_pages);
/*
- * Reserved pages aren't counted against the user,
- * externally pinned pages are already counted against
+ * Externally pinned pages are already counted against
* the user.
*/
if (acct_pages) {
@@ -796,29 +804,21 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
}
out:
- dma->has_rsvd |= rsvd;
- dma->has_non_rsvd |= !rsvd;
ret = vfio_lock_acct(dma, lock_acct, false);
unpin_out:
if (ret < 0) {
- if (pinned && !rsvd) {
- for (pfn = *pfn_base ; pinned ; pfn++, pinned--)
- put_pfn(pfn, dma->prot);
- }
+ if (pinned && !pfnmap)
+ put_gup_pfns(*pfn_base, pinned, dma->prot);
vfio_batch_unpin(batch, dma);
return ret;
}
- return pinned;
-}
+ dma->has_pfnmap |= pfnmap;
+ dma->has_gup_pinned |= !pfnmap;
-static inline void put_valid_unreserved_pfns(unsigned long start_pfn,
- unsigned long npage, int prot)
-{
- unpin_user_page_range_dirty_lock(pfn_to_page(start_pfn), npage,
- prot & IOMMU_WRITE);
+ return pinned;
}
static long vfio_unpin_pages_remote(struct vfio_dma *dma, dma_addr_t iova,
@@ -827,16 +827,23 @@ static long vfio_unpin_pages_remote(struct vfio_dma *dma, dma_addr_t iova,
{
long unlocked = 0, locked = vpfn_pages(dma, iova, npage);
- if (dma->has_rsvd) {
- if (dma->has_non_rsvd) {
+ if (dma->has_pfnmap) {
+ if (dma->has_gup_pinned) {
unsigned long i;
+ /*
+ * Because there is no way to tell if this DMA region
+ * was PFNMAP or GUP-pinned, fall back to the
+ * heuristic in put_pfn() of skipping invalid and
+ * reserved pfns, which fails to unpin reserved pages
+ * that were GUP-pinned.
+ */
for (i = 0; i < npage; i++)
if (put_pfn(pfn++, dma->prot))
unlocked++;
}
} else {
- put_valid_unreserved_pfns(pfn, npage, dma->prot);
+ put_gup_pfns(pfn, npage, dma->prot);
unlocked = npage;
}
if (do_accounting)
@@ -1232,6 +1239,9 @@ static long vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma,
&iotlb_gather);
}
+ dma->has_pfnmap = false;
+ dma->has_gup_pinned = false;
+
if (do_accounting) {
vfio_lock_acct(dma, -unlocked, true);
return 0;
@@ -1947,6 +1957,9 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu,
vfio_unpin_pages_remote(dma, iova, phys >> PAGE_SHIFT,
size >> PAGE_SHIFT, true);
}
+
+ dma->has_pfnmap = false;
+ dma->has_gup_pinned = false;
}
vfio_batch_fini(&batch);
--
2.56.1.708.g77570a6a1d-goog
next prev parent reply other threads:[~2026-11-19 18:57 UTC|newest]
Thread overview: 7+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-19 18:56 [PATCH 0/3] vfio/type1: Fix pin leaks for reserved GUP-pinned pages Aaron Lewis
2026-11-19 18:56 ` [PATCH 1/3] vfio/type1: Skip reserved PFNs in vpfn_pages() Aaron Lewis
2026-11-19 18:56 ` Aaron Lewis [this message]
2026-11-19 19:12 ` [PATCH 2/3] vfio/type1: Track GUP vs PFNMAP in DMA map/unmap sashiko-bot
2026-11-19 18:56 ` [PATCH 3/3] vfio/type1: Track GUP vs PFNMAP in external page pinning Aaron Lewis
2026-11-19 20:27 ` [PATCH 0/3] vfio/type1: Fix pin leaks for reserved GUP-pinned pages Jason Gunthorpe
2026-11-20 16:59 ` Aaron Lewis
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261119185659.889654-3-aaronlewis@google.com \
--to=aaronlewis@google.com \
--cc=alex@shazbot.org \
--cc=dmatlack@google.com \
--cc=jgg@nvidia.com \
--cc=kvm@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.