From: Aaron Lewis <aaronlewis@google.com>
To: kvm@vger.kernel.org
Cc: alex@shazbot.org, dmatlack@google.com, jgg@nvidia.com,
Aaron Lewis <aaronlewis@google.com>
Subject: [PATCH 2/3] vfio/type1: Track GUP vs PFNMAP in DMA map/unmap
Date: Thu, 19 Nov 2026 18:56:58 +0000 [thread overview]
Message-ID: <20261119185659.889654-3-aaronlewis@google.com> (raw)
In-Reply-To: <20261119185659.889654-1-aaronlewis@google.com>
Both vfio_pin_pages_remote() and vfio_unpin_pages_remote() use
PageReserved() as a heuristic to infer whether a page was PFNMAP'd or
GUP-pinned in vaddr_get_pfns(). That heuristic breaks down when the same
reserved struct pages can appear on both paths:
- DevDax pages mapped from /dev/dax* are reserved and GUP-pinned.
- Those same pages mapped from /dev/mem (PFNMAP) are also reserved,
but they are *not* GUP-pinned.
Because of this, when unpinning the DMA region, vfio_unpin_pages_remote()
doesn't know whether those reserved pages were GUP-pinned and skips
unpinning them altogether, *leaking page pins and refcounts*.
Fix this by replacing has_rsvd and has_non_rsvd in struct vfio_dma with
has_pfnmap and has_gup_pinned, tracking which path vaddr_get_pfns()
actually used, and clearing them when a DMA range is unpinned. Since
PFNMAP pages bypass struct vfio_batch, this also fixes locked_vm
accounting and batch error-unwind unpinning for reserved GUP-pinned
pages.
This isn't an exhaustive fix because mixed mappings are still allowed.
For example, if a single DMA region contains both GUP-pinned and
PFNMAP pages, vfio_unpin_pages_remote() can't tell them apart. Handle
this by preserving the existing put_pfn() fallback, and add
a comment explaining the situation.
Lastly, clear the new flags once a vfio_dma is fully unpinned. A vfio_dma
can outlive its pins and be pinned again later, after
vfio_iommu_unmap_unpin_reaccount() or a failed vfio_iommu_replay(), by
which time the user may have remapped the range. A stale has_rsvd only
selected the slower unpin path, but a stale has_pfnmap would send a
mapping that is now entirely GUP-pinned down the wrong unpin path.
Fixes: 3565fce3a659 ("mm, x86: get_user_pages() for dax mappings")
Signed-off-by: Aaron Lewis <aaronlewis@google.com>
---
drivers/vfio/vfio_iommu_type1.c | 103 ++++++++++++++++++--------------
1 file changed, 58 insertions(+), 45 deletions(-)
diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
index 7bad0508dd21..ee40ba942fd5 100644
--- a/drivers/vfio/vfio_iommu_type1.c
+++ b/drivers/vfio/vfio_iommu_type1.c
@@ -93,8 +93,8 @@ struct vfio_dma {
bool iommu_mapped;
bool lock_cap; /* capable(CAP_IPC_LOCK) */
bool vaddr_invalid;
- bool has_rsvd; /* has 1 or more rsvd pfns */
- bool has_non_rsvd; /* has 1 or more !rsvd pfns */
+ bool has_pfnmap; /* has 1 or more pfnmap pfns */
+ bool has_gup_pinned; /* has 1 or more GUP-pinned pfns */
struct task_struct *task;
struct rb_root pfn_list; /* Ex-user pinned pfn list */
unsigned long *bitmap;
@@ -416,6 +416,13 @@ static struct vfio_pfn *vfio_iova_get_vfio_pfn(struct vfio_dma *dma,
return vpfn;
}
+static inline void put_gup_pfns(unsigned long start_pfn, unsigned long npage,
+ int prot)
+{
+ unpin_user_page_range_dirty_lock(pfn_to_page(start_pfn), npage,
+ prot & IOMMU_WRITE);
+}
+
static int vfio_iova_put_vfio_pfn(struct vfio_dma *dma, struct vfio_pfn *vpfn)
{
int ret = 0;
@@ -523,12 +530,11 @@ static void vfio_batch_init_single(struct vfio_batch *batch)
static void vfio_batch_unpin(struct vfio_batch *batch, struct vfio_dma *dma)
{
- while (batch->size) {
- unsigned long pfn = page_to_pfn(batch->pages[batch->offset]);
-
- put_pfn(pfn, dma->prot);
- batch->offset++;
- batch->size--;
+ if (batch->size) {
+ unpin_user_pages_dirty_lock(&batch->pages[batch->offset],
+ batch->size,
+ dma->prot & IOMMU_WRITE);
+ batch->size = 0;
}
}
@@ -673,6 +679,15 @@ static long vpfn_pages(struct vfio_dma *dma,
return ret;
}
+/*
+ * After a successful vaddr_get_pfns(), GUP populates batch->pages while
+ * VM_PFNMAP leaves the batch empty.
+ */
+static bool vfio_batch_is_pfnmap(const struct vfio_batch *batch)
+{
+ return batch->size == 0;
+}
+
/*
* Attempt to pin pages. We really don't want to track all the pfns and
* the iommu can only map chunks of consecutive pfns anyway, so get the
@@ -685,7 +700,7 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
unsigned long pfn;
struct mm_struct *mm = current->mm;
long ret, pinned = 0, lock_acct = 0;
- bool rsvd;
+ bool pfnmap = false;
dma_addr_t iova = vaddr - dma->vaddr + dma->iova;
/* This code path is only user initiated */
@@ -696,7 +711,6 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
/* Leftover pages in batch from an earlier call. */
*pfn_base = page_to_pfn(batch->pages[batch->offset]);
pfn = *pfn_base;
- rsvd = is_invalid_reserved_pfn(*pfn_base);
} else {
*pfn_base = 0;
}
@@ -721,12 +735,18 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
if (!*pfn_base) {
*pfn_base = pfn;
- rsvd = is_invalid_reserved_pfn(*pfn_base);
+ pfnmap = vfio_batch_is_pfnmap(batch);
+ } else if (pfnmap != vfio_batch_is_pfnmap(batch)) {
+ /*
+ * Handle PFNMAP and GUP pages separately; any
+ * GUP pages remain in batch for the next call.
+ */
+ goto out;
}
/* Handle pfnmap */
- if (!batch->size) {
- if (pfn != *pfn_base + pinned || !rsvd)
+ if (pfnmap) {
+ if (pfn != *pfn_base + pinned)
goto out;
pinned += ret;
@@ -742,32 +762,20 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
* due to the fact that vaddr_get_pfns() needs to provide the
* initial pfn for pfnmaps. Therefore to reduce redundancy,
* the next pfn is fetched at the end of the loop.
- * A PageReserved() page could still qualify as page backed
- * and rsvd here, and therefore continues to use the batch.
*/
while (true) {
- long nr_pages, acct_pages = 0;
+ long nr_pages, acct_pages;
- if (pfn != *pfn_base + pinned ||
- rsvd != is_invalid_reserved_pfn(pfn))
+ if (pfn != *pfn_base + pinned)
goto out;
- /*
- * Using GUP with the FOLL_LONGTERM in
- * vaddr_get_pfns() will not return invalid
- * or reserved pages.
- */
nr_pages = num_pages_contiguous(
&batch->pages[batch->offset],
batch->size);
- if (!rsvd) {
- acct_pages = nr_pages;
- acct_pages -= vpfn_pages(dma, iova, nr_pages);
- }
+ acct_pages = nr_pages - vpfn_pages(dma, iova, nr_pages);
/*
- * Reserved pages aren't counted against the user,
- * externally pinned pages are already counted against
+ * Externally pinned pages are already counted against
* the user.
*/
if (acct_pages) {
@@ -796,29 +804,21 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
}
out:
- dma->has_rsvd |= rsvd;
- dma->has_non_rsvd |= !rsvd;
ret = vfio_lock_acct(dma, lock_acct, false);
unpin_out:
if (ret < 0) {
- if (pinned && !rsvd) {
- for (pfn = *pfn_base ; pinned ; pfn++, pinned--)
- put_pfn(pfn, dma->prot);
- }
+ if (pinned && !pfnmap)
+ put_gup_pfns(*pfn_base, pinned, dma->prot);
vfio_batch_unpin(batch, dma);
return ret;
}
- return pinned;
-}
+ dma->has_pfnmap |= pfnmap;
+ dma->has_gup_pinned |= !pfnmap;
-static inline void put_valid_unreserved_pfns(unsigned long start_pfn,
- unsigned long npage, int prot)
-{
- unpin_user_page_range_dirty_lock(pfn_to_page(start_pfn), npage,
- prot & IOMMU_WRITE);
+ return pinned;
}
static long vfio_unpin_pages_remote(struct vfio_dma *dma, dma_addr_t iova,
@@ -827,16 +827,23 @@ static long vfio_unpin_pages_remote(struct vfio_dma *dma, dma_addr_t iova,
{
long unlocked = 0, locked = vpfn_pages(dma, iova, npage);
- if (dma->has_rsvd) {
- if (dma->has_non_rsvd) {
+ if (dma->has_pfnmap) {
+ if (dma->has_gup_pinned) {
unsigned long i;
+ /*
+ * Because there is no way to tell if this DMA region
+ * was PFNMAP or GUP-pinned, fall back to the
+ * heuristic in put_pfn() of skipping invalid and
+ * reserved pfns, which fails to unpin reserved pages
+ * that were GUP-pinned.
+ */
for (i = 0; i < npage; i++)
if (put_pfn(pfn++, dma->prot))
unlocked++;
}
} else {
- put_valid_unreserved_pfns(pfn, npage, dma->prot);
+ put_gup_pfns(pfn, npage, dma->prot);
unlocked = npage;
}
if (do_accounting)
@@ -1232,6 +1239,9 @@ static long vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma,
&iotlb_gather);
}
+ dma->has_pfnmap = false;
+ dma->has_gup_pinned = false;
+
if (do_accounting) {
vfio_lock_acct(dma, -unlocked, true);
return 0;
@@ -1947,6 +1957,9 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu,
vfio_unpin_pages_remote(dma, iova, phys >> PAGE_SHIFT,
size >> PAGE_SHIFT, true);
}
+
+ dma->has_pfnmap = false;
+ dma->has_gup_pinned = false;
}
vfio_batch_fini(&batch);
--
2.56.1.708.g77570a6a1d-goog
next prev parent reply other threads:[~2026-11-19 18:57 UTC|newest]
Thread overview: 5+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-19 18:56 [PATCH 0/3] vfio/type1: Fix pin leaks for reserved GUP-pinned pages Aaron Lewis
2026-11-19 18:56 ` [PATCH 1/3] vfio/type1: Skip reserved PFNs in vpfn_pages() Aaron Lewis
2026-11-19 18:56 ` Aaron Lewis [this message]
2026-11-19 19:12 ` [PATCH 2/3] vfio/type1: Track GUP vs PFNMAP in DMA map/unmap sashiko-bot
2026-11-19 18:56 ` [PATCH 3/3] vfio/type1: Track GUP vs PFNMAP in external page pinning Aaron Lewis
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261119185659.889654-3-aaronlewis@google.com \
--to=aaronlewis@google.com \
--cc=alex@shazbot.org \
--cc=dmatlack@google.com \
--cc=jgg@nvidia.com \
--cc=kvm@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox