[RFC PATCH 06/12] virtio: add a device memory buffer region allocator

[Alexander Graf <graf@amazon.com>]

In preparation to support VIRTIO_F_DMB, create a mechanism to allocate
and map memory from the Device Memory Buffer (DMB). The DMB is a shared
memory region a device exposes and owns. A device that negotiates the
feature expects its virtqueues and all the buffers we hand it to live in
that region, and every address we publish to it is a byte offset into
the region.

Add virtio_dmb_init(), which locates the region by the shared memory id
the device reports and builds a page-granular allocator over it, and
virtio_dmb_destroy() to tear that down. Add virtio_dmb_map_ops, a struct
virtio_map_ops implementation that hands out allocations from that
allocator as region offsets: alloc() places a virtqueue area in the
region, map_page() copies a buffer that lives elsewhere into it and
copies it back on unmap. The map operations reach that allocator through
a new dmb member of union virtio_map.

The shared memory id is transport specific, so add a get_dmb_shm_id()
callback to struct virtio_config_ops for a transport to report it. A
transport that does not implement it must not accept VIRTIO_F_DMB. Add
CONFIG_VIRTIO_DMB to enable this support. It defaults to y, and a kernel
that will never meet such a device can turn it off to leave the
allocator and its bookkeeping out.

Link: https://lore.kernel.org/virtio-comment/20260804161202.38619-1-graf@amazon.com/
Assisted-by: Kiro:claude-opus-5 checkpatch sparse
Signed-off-by: Alexander Graf <graf@amazon.com>
---
 drivers/virtio/Kconfig        |   15 +
 drivers/virtio/Makefile       |    3 +-
 drivers/virtio/virtio_dmb.c   | 1317 +++++++++++++++++++++++++++++++++
 drivers/virtio/virtio_dmb.h   |   28 +
 include/linux/virtio.h        |    3 +
 include/linux/virtio_config.h |    8 +
 6 files changed, 1373 insertions(+), 1 deletion(-)
 create mode 100644 drivers/virtio/virtio_dmb.c
 create mode 100644 drivers/virtio/virtio_dmb.h

diff --git a/drivers/virtio/Kconfig b/drivers/virtio/Kconfig
index ce5bc0d9ea28..b6b5a36c3c21 100644
--- a/drivers/virtio/Kconfig
+++ b/drivers/virtio/Kconfig
@@ -188,6 +188,21 @@ config VIRTIO_DEBUG
 
 	  If unsure, say N.
 
+config VIRTIO_DMB
+	bool "Device Memory Buffer support"
+	depends on VIRTIO
+	default y
+	help
+	  Support devices that place their virtqueues and buffers in a shared
+	  memory region they own, rather than in memory the driver allocates.
+
+	  Enabling this adds a page allocator and per-page bookkeeping, both
+	  of which are set up only for a device that negotiates the feature.
+	  When disabled the feature is never accepted and such a device is
+	  driven as an ordinary one.
+
+	  If unsure, say Y.
+
 config VIRTIO_RTC
 	tristate "Virtio RTC driver"
 	depends on VIRTIO
diff --git a/drivers/virtio/Makefile b/drivers/virtio/Makefile
index eefcfe90d6b8..ba785ff44a16 100644
--- a/drivers/virtio/Makefile
+++ b/drivers/virtio/Makefile
@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0
-obj-$(CONFIG_VIRTIO) += virtio.o virtio_ring.o
+virtio-dmb-$(CONFIG_VIRTIO_DMB) := virtio_dmb.o
+obj-$(CONFIG_VIRTIO) += virtio.o virtio_ring.o $(virtio-dmb-y)
 obj-$(CONFIG_VIRTIO_ANCHOR) += virtio_anchor.o
 obj-$(CONFIG_VIRTIO_PCI_LIB) += virtio_pci_modern_dev.o
 obj-$(CONFIG_VIRTIO_PCI_LIB_LEGACY) += virtio_pci_legacy_dev.o
diff --git a/drivers/virtio/virtio_dmb.c b/drivers/virtio/virtio_dmb.c
new file mode 100644
index 000000000000..42126f928bc0
--- /dev/null
+++ b/drivers/virtio/virtio_dmb.c
@@ -0,0 +1,1317 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Device Memory Buffer support for virtio devices.
+ *
+ * A device that negotiates VIRTIO_F_DMB owns one shared memory region, the
+ * Device Memory Buffer, that holds its virtqueues and the buffers they
+ * reference.  Every address the driver publishes to such a device is a byte
+ * offset from the start of that region.
+ *
+ * This file provides an allocator over the region and the virtio_map_ops
+ * implementation that turns allocations into those offsets.  A mapping handle
+ * belonging to a DMB device is a region offset and nothing else: no code
+ * outside these operations may treat it as a DMA address.
+ *
+ * The region is shared with the device, which may read or write any of it at
+ * any time.  Nothing this file reads back from the region is used to compute
+ * a kernel address, a length or an index.  Handles and their sizes arrive
+ * from the ring's own bookkeeping in kernel memory, and are range-checked
+ * anyway so that a bug there cannot reach outside the arrays below.
+ */
+
+#include <linux/align.h>
+#include <linux/bitmap.h>
+#include <linux/bits.h>
+#include <linux/cache.h>
+#include <linux/cpumask.h>
+#include <linux/dma-mapping.h>
+#include <linux/export.h>
+#include <linux/highmem.h>
+#include <linux/io.h>
+#include <linux/ioport.h>
+#include <linux/limits.h>
+#include <linux/log2.h>
+#include <linux/math.h>
+#include <linux/minmax.h>
+#include <linux/module.h>
+#include <linux/overflow.h>
+#include <linux/slab.h>
+#include <linux/smp.h>
+#include <linux/spinlock.h>
+#include <linux/virtio.h>
+#include <linux/virtio_config.h>
+
+#include "virtio_dmb.h"
+
+/* No source recorded for this slot: it holds no bounced mapping. */
+#define DMB_SRC_NONE		((phys_addr_t)-1)
+
+/**
+ * struct virtio_dmb_slot - what one PAGE_SIZE slot of the pool records
+ * @src: physical address this slot bounces, DMB_SRC_NONE for a virtqueue area
+ * @end: one past the last slot of the allocation this slot belongs to, or
+ *	zero when the slot is free
+ * @tail: how many bytes of slot @end - 1 the allocation covers, PAGE_SIZE when
+ *	its length is a whole number of pages, zero when the slot is free
+ *
+ * Every slot of an allocation records the same @end and the same @tail, so a
+ * handle that points into the middle of one still yields both the allocation's
+ * extent in slots and its end in bytes from a single read.
+ *
+ * @tail is what makes the length a mapping is bounced against a byte count
+ * rather than a page count.  It is bounded by PAGE_SIZE, so u32 holds it on
+ * every configuration, which a whole length would not: virtio_dmb_op_alloc()
+ * bounds a request by the entire pool and a pool may exceed 4 GiB.
+ */
+struct virtio_dmb_slot {
+	phys_addr_t	src;
+	u32		end;
+	u32		tail;
+};
+
+/*
+ * Bounds on how many slots one area covers.
+ *
+ * The floor is one cacheline of bitmap, so that no two areas contend on the
+ * line their separate locks exist to keep apart, and at least 512 slots.  On a
+ * 64-byte line those coincide; a wider line raises the floor, which is
+ * correct.  Either way the floor is a power of two of at least BITS_PER_LONG,
+ * which is what makes an area own whole bitmap words and what makes the area
+ * of a slot a shift; virtio_dmb_init() asserts both rather than leaving them
+ * to inspection.
+ *
+ * The ceiling has no counterpart in kernel/dma/swiotlb.c, which this geometry
+ * otherwise follows, and it is needed because the search differs: swiotlb
+ * finds a run through a per-slot free-run list, while this sweeps a bitmap,
+ * so the cost of one search here is linear in the size of an area.  Without a
+ * ceiling that cost grows with the region, which is the thing being bounded.
+ */
+#define DMB_AREA_CACHELINE_SLOTS	((unsigned int)L1_CACHE_BYTES * BITS_PER_BYTE)
+#define DMB_AREA_MIN_SLOTS		(DMB_AREA_CACHELINE_SLOTS > 512u ? \
+					 DMB_AREA_CACHELINE_SLOTS : 512u)
+#define DMB_AREA_MAX_SLOTS		4096u
+
+/**
+ * struct virtio_dmb_area - one independently locked range of the pool
+ * @used: slots of this area that are allocated; exact under @lock
+ * @index: slot this area's next search starts from, relative to the area base
+ * @lock: covers this area's bits of the pool bitmap, @used and @index
+ *
+ * Cacheline-aligned where that means anything, so that two areas' locks do not
+ * share a line; the alignment compiles away on !SMP, where nothing contends.
+ */
+struct virtio_dmb_area {
+	unsigned int	used;
+	unsigned int	index;
+	/* Serialises this area's bits of the pool bitmap, @used and @index. */
+	spinlock_t	lock;
+} ____cacheline_aligned_in_smp;
+
+/**
+ * struct virtio_dmb - driver-side state for one Device Memory Buffer
+ * @vdev: the device that owns the region, for message context
+ * @map_va: what memremap() returned, for memunmap()
+ * @map_phys: physical base of the region, for release_mem_region()
+ * @map_len: length of the claimed and mapped part of the region
+ * @map_claimed: whether request_mem_region() succeeded for that range
+ * @prev_map: map operations the transport had installed, restored on teardown
+ * @prev_vmap: mapping token that went with @prev_map
+ * @base_va: kernel address the pool starts at, inside the mapping
+ * @base_off: pool start as an offset from the start of the region
+ * @nslots: pool size in PAGE_SIZE slots
+ * @bitmap: @nslots bits, set when the slot is allocated
+ * @slots: @nslots slot records
+ * @areas: the @nareas ranges the pool is divided into
+ * @nareas: how many areas the pool is divided into
+ * @area_slots: slots one area covers, a power of two; the last area covers
+ *	fewer when @nslots is not a multiple of it
+ * @area_shift: ilog2(@area_slots), so slot >> @area_shift names its area
+ * @shm_id: shared memory id the device reported for the region
+ */
+struct virtio_dmb {
+	struct virtio_device	*vdev;
+	void			*map_va;
+	phys_addr_t		 map_phys;
+	size_t			 map_len;
+	bool			 map_claimed;
+	const struct virtio_map_ops *prev_map;
+	union virtio_map	 prev_vmap;
+	void			*base_va;
+	u64			 base_off;
+	unsigned int		 nslots;
+	unsigned long		*bitmap;
+	struct virtio_dmb_slot	*slots;
+	struct virtio_dmb_area	*areas;
+	unsigned int		 nareas;
+	unsigned int		 area_slots;
+	unsigned int		 area_shift;
+	u16			 shm_id;
+};
+
+/* First slot of area @i. */
+static unsigned int virtio_dmb_area_base(const struct virtio_dmb *dmb,
+					 unsigned int i)
+{
+	return i << dmb->area_shift;
+}
+
+/* Slots area @i covers.  The last area is short unless nslots divides. */
+static unsigned int virtio_dmb_area_len(const struct virtio_dmb *dmb,
+					unsigned int i)
+{
+	return min(dmb->area_slots,
+		   dmb->nslots - virtio_dmb_area_base(dmb, i));
+}
+
+static unsigned int virtio_dmb_slots(size_t size)
+{
+	return DIV_ROUND_UP(size, PAGE_SIZE);
+}
+
+static size_t virtio_dmb_pool_size(const struct virtio_dmb *dmb)
+{
+	return (size_t)dmb->nslots << PAGE_SHIFT;
+}
+
+/*
+ * The largest buffer mapping this pool will serve: the smaller of an eighth of
+ * the pool and half of one area, but never less than one page.  That floor is
+ * what governs a region at the four-slot minimum, since an eighth of four
+ * pages rounds down to nothing.
+ *
+ * The eighth is a choice, not a derived value: it bounds the capacity one
+ * mapping can deny the rest of the device to seven eighths of the pool, so
+ * that a device with several virtqueues can still make forward progress while
+ * one large mapping is outstanding.
+ *
+ * The half-area is derived, and it is the reason this function has to be
+ * consulted rather than the eighth alone.  An allocation has to lie inside one
+ * area, because that is what lets one lock cover it and what lets the release
+ * path find that lock from the slot index.  Half rather than all of an area is
+ * headroom: a request the size of a whole area could only ever be satisfied by
+ * a completely empty one, and an area may be as small as
+ * DMB_AREA_MIN_SLOTS, so there is no expectation that one is empty.  The
+ * figure is half of the nominal area size, so a request at the cap can exceed
+ * the short last area outright; that costs a claim in one area out of nareas
+ * and the walk tries the others.
+ *
+ * The cap applies to map_page() only.  An alloc() is a virtqueue area, which
+ * is structural rather than in-flight: it lives for as long as the queue
+ * does and no back-pressure can defer it, so a cap on it could only shrink a
+ * queue on a region that is too small, or refuse one outright where the ring
+ * layout cannot be shrunk.  Sizing the region for the areas as well as the
+ * buffers is the device's obligation.
+ */
+static size_t virtio_dmb_max_mapping(const struct virtio_dmb *dmb)
+{
+	size_t eighth = ALIGN_DOWN(virtio_dmb_pool_size(dmb) / 8, PAGE_SIZE);
+	size_t half_area = ((size_t)dmb->area_slots / 2) << PAGE_SHIFT;
+
+	return max_t(size_t, min(eighth, half_area), PAGE_SIZE);
+}
+
+/*
+ * Claim @nr contiguous slots from area @i, or -ENOMEM when that one area
+ * cannot satisfy the request.  Takes and drops that area's lock and touches
+ * no other area's state, so no path ever holds two of these locks and there
+ * is no ordering between them to get right.
+ */
+static long virtio_dmb_area_claim(struct virtio_dmb *dmb, unsigned int i,
+				  unsigned int nr)
+{
+	struct virtio_dmb_area *area = &dmb->areas[i];
+	unsigned int base = virtio_dmb_area_base(dmb, i);
+	unsigned int end = base + virtio_dmb_area_len(dmb, i);
+	unsigned long flags, slot;
+
+	spin_lock_irqsave(&area->lock, flags);
+
+	/*
+	 * Exact, and inside the lock.  Written as a subtraction from the
+	 * area's own length rather than as len - used < nr, which underflows.
+	 */
+	if (nr > (end - base) - area->used)
+		goto not_found;
+
+	/*
+	 * Both sweeps are bounded at the area end, so an allocation cannot
+	 * span two areas and the release path can find one lock from the slot
+	 * index.  bitmap_find_next_zero_area() returns a value whose sum with
+	 * @nr exceeds the size it was given when it finds nothing, so that sum
+	 * is the test; the whole-pool "slot >= nslots" form does not transfer.
+	 */
+	slot = bitmap_find_next_zero_area(dmb->bitmap, end,
+					  base + area->index, nr, 0);
+	if (slot + nr > end && area->index)
+		slot = bitmap_find_next_zero_area(dmb->bitmap, end, base,
+						  nr, 0);
+	if (slot + nr > end)
+		goto not_found;
+
+	bitmap_set(dmb->bitmap, slot, nr);
+	area->used += nr;
+	area->index = slot + nr < end ? slot + nr - base : 0;
+
+	spin_unlock_irqrestore(&area->lock, flags);
+
+	return slot;
+
+not_found:
+	spin_unlock_irqrestore(&area->lock, flags);
+
+	return -ENOMEM;
+}
+
+/*
+ * Claim @nr contiguous slots.  Returns the first slot, or -ENOMEM when no
+ * area can satisfy the request.  Exhaustion is a routine condition: the
+ * region's length bounds how much virtqueue data can be in flight.  What a
+ * caller makes of it is the caller's, and it is not always back-pressure: a
+ * network receive fill has nothing to push back on when it cannot post a
+ * buffer, and repolls instead.  Reporting it at any level a working device
+ * would print would therefore be a log flood, and it is the only signal an
+ * undersized region produces at all, so it is reported through dynamic debug
+ * where it costs nothing until somebody asks for it.
+ *
+ * Next fit within one area, from a hint that advances past each claim and
+ * rewinds to each release, beginning in the area belonging to the running CPU
+ * and then trying each other area in turn.  So a multi-slot request can fail
+ * while the total free count would have satisfied it: a map_page() request is
+ * bounded by virtio_dmb_max_mapping(), which keeps it inside one area, but
+ * fragmentation within that area can still cost capacity and fail an
+ * individual mapping.  Nothing is moved to recover: handles are live in
+ * descriptors the device is reading.
+ *
+ * The walk visits every area and tests capacity inside that area's lock, so a
+ * refusal is a true statement about the pool rather than about one area.  It
+ * releases the lock and restores interrupts between areas, which is what
+ * bounds the interrupts-off window to a single area's sweep; the total work in
+ * the failing case is a whole-pool sweep either way.
+ */
+static long virtio_dmb_claim(struct virtio_dmb *dmb, unsigned int nr)
+{
+	unsigned int i, start;
+	long ret;
+
+	/*
+	 * raw_smp_processor_id() and not smp_processor_id(): the index is
+	 * computed before any lock is taken, so preemption or migration
+	 * between the read and the claim only changes which area is tried
+	 * first.  kernel/dma/swiotlb.c picks its home area on the same
+	 * reasoning.
+	 */
+	start = raw_smp_processor_id() % dmb->nareas;
+	i = start;
+	do {
+		ret = virtio_dmb_area_claim(dmb, i, nr);
+		if (ret >= 0)
+			return ret;
+
+		if (++i >= dmb->nareas)
+			i = 0;
+	} while (i != start);
+
+	/*
+	 * The geometry rather than a free count: there is no instant at which
+	 * a total free count is true under per-area locking, so printing one
+	 * would mean either a walk taking every lock or a torn read.
+	 */
+	dev_dbg_ratelimited(&dmb->vdev->dev,
+			    "device memory buffer has no run of %u pages in any of %u areas of %u pages\n",
+			    nr, dmb->nareas, dmb->area_slots);
+
+	return -ENOMEM;
+}
+
+static void virtio_dmb_release(struct virtio_dmb *dmb, unsigned int slot,
+			       unsigned int nr)
+{
+	struct virtio_dmb_area *area;
+	unsigned long flags;
+	unsigned int i, a;
+
+	if (dev_WARN_ONCE(&dmb->vdev->dev,
+			  !nr || slot >= dmb->nslots || nr > dmb->nslots - slot,
+			  "bad device memory buffer slot range %u+%u\n",
+			  slot, nr))
+		return;
+
+	/*
+	 * One area holds the whole allocation, so one lock covers it.  The
+	 * allocator establishes that by bounding both of its sweeps at an area
+	 * end; enforce it here rather than inherit it, because this is the path
+	 * that depends on it to pick a lock at all, and picking the wrong one
+	 * would clear bits and adjust a count under a lock that does not cover
+	 * either.
+	 */
+	a = slot >> dmb->area_shift;
+	if (dev_WARN_ONCE(&dmb->vdev->dev,
+			  ((slot + nr - 1) >> dmb->area_shift) != a,
+			  "device memory buffer allocation %u+%u spans two areas\n",
+			  slot, nr))
+		return;
+	area = &dmb->areas[a];
+
+	spin_lock_irqsave(&area->lock, flags);
+
+	/*
+	 * Releasing a range that is not wholly allocated would put slots that
+	 * a different allocation now owns back on the free list, which is what
+	 * a second release of one handle does.  The bitmap is the only record
+	 * that can answer whether that is happening, and the test costs less
+	 * than the bitmap_clear() it guards.
+	 */
+	if (dev_WARN_ONCE(&dmb->vdev->dev,
+			  find_next_zero_bit(dmb->bitmap, slot + nr, slot) <
+				slot + nr,
+			  "device memory buffer double release %u+%u\n",
+			  slot, nr))
+		goto out;
+
+	/*
+	 * Clear the records before the bits, so that a slot reachable from the
+	 * free list never carries an extent that virtio_dmb_resolve() would
+	 * trust.  Doing it here rather than in the callers covers every
+	 * release path with one copy of the invariant.
+	 *
+	 * WRITE_ONCE() because virtio_dmb_resolve() reads these three fields
+	 * without the lock, which is the pattern
+	 * tools/memory-model/Documentation/access-marking.txt calls
+	 * "Lock-Protected Writes With Lockless Reads" and asks to be marked on
+	 * both sides.
+	 */
+	for (i = slot; i < slot + nr; i++) {
+		WRITE_ONCE(dmb->slots[i].src, DMB_SRC_NONE);
+		WRITE_ONCE(dmb->slots[i].end, 0);
+		WRITE_ONCE(dmb->slots[i].tail, 0);
+	}
+
+	bitmap_clear(dmb->bitmap, slot, nr);
+	area->used -= nr;
+	area->index = slot - virtio_dmb_area_base(dmb, a);
+
+out:
+	spin_unlock_irqrestore(&area->lock, flags);
+}
+
+/**
+ * struct virtio_dmb_ref - a handle resolved against the pool
+ * @slot: the slot the handle lands in
+ * @nr: slots the allocation still holds from @slot on
+ * @src: the physical address @slot bounces, or DMB_SRC_NONE for a queue area
+ *
+ * @nr and @src are derived from the same read of the record that validated the
+ * handle, so the extent a caller releases and the pages the copy touches are
+ * the ones that were checked and not a later re-read of a field another CPU may
+ * meanwhile have cleared.
+ */
+struct virtio_dmb_ref {
+	unsigned int	slot;
+	unsigned int	nr;
+	phys_addr_t	src;
+};
+
+/* Extra conditions virtio_dmb_resolve() enforces for particular callers. */
+#define DMB_RESOLVE_BOUNCED	BIT(0)	/* must be a mapping, not a queue area */
+#define DMB_RESOLVE_WHOLE	BIT(1)	/* must be the allocation's first slot */
+
+/*
+ * Turn a handle and a length into a slot index and an extent, rejecting
+ * anything that does not lie wholly inside the pool.  The bound is exclusive,
+ * so a handle at the end of the pool and a zero length are both refused.  The
+ * arithmetic is done in u64 so that it cannot wrap where dma_addr_t is
+ * narrower.
+ *
+ * A handle may point into the middle of a mapping, because
+ * virtqueue_map_sync_single_range_for_cpu() and its counterpart pass
+ * handle + offset, so the extent is taken from the slot the handle lands in
+ * rather than from the first slot of the allocation.  That rejects a handle
+ * released twice, a length that runs past the end of its allocation, and a
+ * range that would continue into a neighbouring one.
+ *
+ * The length is checked in bytes, not in pages.  A page-granular check would
+ * pass a length ending anywhere inside the allocation's last slot, which is up
+ * to PAGE_SIZE - 1 bytes past what was mapped, and virtio_dmb_copy() would
+ * then touch the page after the source run.  swiotlb_bounce() keeps the same
+ * bound in alloc_size (kernel/dma/swiotlb.c:890) and clamps an over-long
+ * mapping to it, because it has callers it cannot refuse.  Every caller here
+ * passes a length the map side recorded, so an over-long one is a caller bug
+ * and the copy is refused outright rather than truncated.
+ *
+ * DMB_RESOLVE_BOUNCED additionally requires the allocation to be one this file
+ * bounced rather than a virtqueue area.  DMB_RESOLVE_WHOLE requires the handle
+ * to be the start of its allocation, which is what the callers that release it
+ * need: mid-extent tolerance exists for the sync ops alone, and a mid-extent
+ * release would free the tail of an allocation and leak its head for good.
+ *
+ * The resolved slot is read without the lock, which is legitimate for a caller
+ * that owns it: virtio_dmb_claim() published it exclusively to this caller, so
+ * nothing else writes it.  The DMB_RESOLVE_WHOLE test also reads the preceding
+ * slot, which the caller does not own, and that read rests on a different
+ * argument, stated where it is made.
+ */
+static bool virtio_dmb_resolve(struct virtio_dmb *dmb, dma_addr_t handle,
+			       size_t size, unsigned int rflags,
+			       struct virtio_dmb_ref *ref)
+{
+	u64 off, end, limit;
+	phys_addr_t src;
+	u32 slot_end, tail;
+
+	if (!size || (u64)handle < dmb->base_off)
+		goto bad_handle;
+
+	off = (u64)handle - dmb->base_off;
+	if (check_add_overflow(off, (u64)size, &end))
+		goto bad_handle;
+	if (end > (u64)virtio_dmb_pool_size(dmb))
+		goto bad_handle;
+
+	ref->slot = off >> PAGE_SHIFT;
+
+	/*
+	 * Snapshot the record here, once.  Every test below, the extent the
+	 * caller goes on to release and the pages virtio_dmb_copy() touches are
+	 * taken from these three reads rather than from a second look at the
+	 * entry, so what was validated is what gets used.  Reading the slot
+	 * without the lock is legitimate for a caller that owns it, as above,
+	 * but the callers these tests exist to catch are exactly the ones that
+	 * do not own it, and for those another CPU may be writing the entry.
+	 */
+	slot_end = READ_ONCE(dmb->slots[ref->slot].end);
+	tail = READ_ONCE(dmb->slots[ref->slot].tail);
+	src = READ_ONCE(dmb->slots[ref->slot].src);
+
+	if (dev_WARN_ONCE(&dmb->vdev->dev, !slot_end,
+			  "device memory buffer handle %pad holds no allocation\n",
+			  &handle))
+		return false;
+
+	/* The allocation's exclusive end in bytes, reachable from any slot. */
+	limit = ((u64)(slot_end - 1) << PAGE_SHIFT) + tail;
+
+	if (dev_WARN_ONCE(&dmb->vdev->dev, end > limit,
+			  "device memory buffer handle %pad length %zu leaves its allocation\n",
+			  &handle, size))
+		return false;
+
+	/*
+	 * Allocations are disjoint and every slot of one records the same end,
+	 * so the preceding slot shares that end if and only if this slot is not
+	 * the first of the allocation.
+	 *
+	 * That slot may belong to another caller, which is the read the
+	 * ownership argument above does not cover.  Disjointness covers it
+	 * instead: a neighbouring allocation ends at or before this slot and a
+	 * free slot records zero, so whichever of the two a concurrent
+	 * virtio_dmb_record() or virtio_dmb_release() leaves visible, neither
+	 * can equal slot_end.
+	 */
+	if ((rflags & DMB_RESOLVE_WHOLE) &&
+	    dev_WARN_ONCE(&dmb->vdev->dev,
+			  !IS_ALIGNED(off, PAGE_SIZE) ||
+			  (ref->slot &&
+			   READ_ONCE(dmb->slots[ref->slot - 1].end) == slot_end),
+			  "device memory buffer handle %pad is not the start of its allocation\n",
+			  &handle))
+		return false;
+
+	if ((rflags & DMB_RESOLVE_BOUNCED) &&
+	    dev_WARN_ONCE(&dmb->vdev->dev, src == DMB_SRC_NONE,
+			  "device memory buffer handle %pad holds no mapping\n",
+			  &handle))
+		return false;
+
+	ref->nr = slot_end - ref->slot;
+	ref->src = src;
+	return true;
+
+bad_handle:
+	dev_WARN_ONCE(&dmb->vdev->dev, 1,
+		      "device memory buffer handle %pad length %zu out of range\n",
+		      &handle, size);
+	return false;
+}
+
+/*
+ * Copy between the region and the pages a mapping bounces.  @ref is what
+ * virtio_dmb_resolve() validated for this handle, or what the map side has just
+ * recorded; every source address is derived from its snapshot rather than from
+ * dmb->slots[], which this function does not read at all, so validation and use
+ * cannot disagree about where the source is.  Walk the source one page at a
+ * time through kmap_local_page(): the source may be highmem, and the per-slot
+ * record is a physical address precisely so that no assumption about a linear
+ * kernel mapping across the mapping's pages is needed.  The region side needs
+ * no such split, being one contiguous mapping.
+ */
+static void virtio_dmb_copy(struct virtio_dmb *dmb,
+			    const struct virtio_dmb_ref *ref,
+			    dma_addr_t handle, size_t size, bool to_region)
+{
+	size_t off = (size_t)((u64)handle - dmb->base_off);
+	phys_addr_t base = ref->src - ((phys_addr_t)ref->slot << PAGE_SHIFT);
+	size_t done = 0;
+
+	while (done < size) {
+		size_t pos = off + done;
+		phys_addr_t src = base + pos;
+		unsigned int in_src = offset_in_page(src);
+		void *region = dmb->base_va + pos;
+		size_t n;
+		void *va;
+
+		n = min(size - done, (size_t)PAGE_SIZE - in_src);
+
+		va = kmap_local_page(pfn_to_page(PHYS_PFN(src)));
+		if (to_region)
+			memcpy(region, va + in_src, n);
+		else
+			memcpy(va + in_src, region, n);
+		kunmap_local(va);
+
+		done += n;
+	}
+}
+
+/*
+ * Record @nr slots from @slot as one allocation of @size bytes bouncing @src.
+ *
+ * Each slot carries the source address of its own page rather than only the
+ * first slot carrying the address of the mapping.  That is what makes
+ * virtio_dmb_copy() independent of which slot of the allocation a handle
+ * resolved to: taking the slot index back off the recorded address yields the
+ * same base from any slot of the mapping.  The sync ops do resolve to a slot
+ * in the middle of one, being handed handle + offset.
+ *
+ * The tail is recorded on every slot for the same reason, so that the byte end
+ * of the allocation is derivable from a mid-extent handle without also knowing
+ * which slot the allocation starts at.
+ */
+static void virtio_dmb_record(struct virtio_dmb *dmb, unsigned int slot,
+			      unsigned int nr, size_t size, phys_addr_t src)
+{
+	unsigned int i;
+	u32 tail;
+
+	/*
+	 * @nr is virtio_dmb_slots(size) at both call sites, so the last slot
+	 * carries between 1 and PAGE_SIZE bytes and the cast cannot truncate.
+	 */
+	tail = (u32)(size - ((size_t)(nr - 1) << PAGE_SHIFT));
+
+	for (i = 0; i < nr; i++) {
+		WRITE_ONCE(dmb->slots[slot + i].src,
+			   src == DMB_SRC_NONE ?
+			   DMB_SRC_NONE : src + ((phys_addr_t)i << PAGE_SHIFT));
+		WRITE_ONCE(dmb->slots[slot + i].end, slot + nr);
+		WRITE_ONCE(dmb->slots[slot + i].tail, tail);
+	}
+}
+
+static void *virtio_dmb_op_alloc(union virtio_map map, size_t size,
+				 dma_addr_t *map_handle, gfp_t gfp)
+{
+	struct virtio_dmb *dmb = map.dmb;
+	unsigned int nr, slot;
+	void *va;
+	long ret;
+
+	/*
+	 * The allocation-behaviour bits of gfp are ignored, because claiming
+	 * slots neither sleeps nor allocates; __GFP_NOWARN is honoured, for
+	 * the reason the failure path below gives.  The result is zeroed
+	 * because this stands in for dma_alloc_coherent(), whose callers rely
+	 * on that.
+	 */
+	if (!size)
+		return NULL;
+
+	/*
+	 * Bound the request before a slot count is derived from it, as
+	 * map_page() does.  The bound is the whole pool rather than the
+	 * fraction of it virtio_dmb_max_mapping() reports, for the reason
+	 * that function gives.  It is not the effective limit: every search
+	 * is bounded at one pool area, so an allocation larger than
+	 * area_slots pages fails even on an empty pool.
+	 */
+	if (size > virtio_dmb_pool_size(dmb))
+		goto no_room;
+
+	nr = virtio_dmb_slots(size);
+	ret = virtio_dmb_claim(dmb, nr);
+	if (ret < 0)
+		goto no_room;
+	slot = ret;
+
+	va = dmb->base_va + ((size_t)slot << PAGE_SHIFT);
+	memset(va, 0, (size_t)nr << PAGE_SHIFT);
+	virtio_dmb_record(dmb, slot, nr, size, DMB_SRC_NONE);
+
+	*map_handle = dmb->base_off + ((u64)slot << PAGE_SHIFT);
+	return va;
+
+no_room:
+	/*
+	 * A buffer that does not fit is back-pressure and stays quiet, but
+	 * this is a virtqueue area: no back-pressure can defer it, and a
+	 * region sized for the buffers but not for the areas otherwise fails
+	 * queue setup with nothing to tell it apart from every other reason
+	 * find_vqs() can fail, and it is the one of those a larger region
+	 * fixes.
+	 *
+	 * Which is why __GFP_NOWARN has to be honoured rather than ignored.
+	 * vring_alloc_queue_split() walks the queue size down from the size
+	 * the device asked for and marks every attempt but the last with the
+	 * flag, so warning regardless would print a line for every attempt
+	 * but the last, for a probe that then succeeds.  Dynamic debug still carries the
+	 * message, which is what a packed ring has to rely on: none of its
+	 * three areas can be made smaller and all three set the flag.
+	 */
+	if (gfp & __GFP_NOWARN)
+		dev_dbg(&dmb->vdev->dev,
+			"no room for a %zu-byte virtqueue area in %u pages\n",
+			size, dmb->nslots);
+	else
+		dev_warn(&dmb->vdev->dev,
+			 "no room for a %zu-byte virtqueue area in %u pages\n",
+			 size, dmb->nslots);
+	return NULL;
+}
+
+/*
+ * DMB_RESOLVE_WHOLE alone: there is deliberately no converse of
+ * DMB_RESOLVE_BOUNCED insisting that the allocation is a virtqueue area.  A
+ * caller reaching this with a mapping has called free() on something it got
+ * from map_page(), and would lose the copy-out that unmap_page() does; the
+ * bytes it loses are its own, and the only in-tree caller of the exported
+ * virtqueue_map_free_coherent() is vring_free_queue(), which frees an area.
+ * Refusing here would trade that for a leak of the slots, which is worse.
+ */
+static void virtio_dmb_op_free(union virtio_map map, size_t size, void *vaddr,
+			       dma_addr_t map_handle, unsigned long attrs)
+{
+	struct virtio_dmb *dmb = map.dmb;
+	struct virtio_dmb_ref ref;
+
+	if (!virtio_dmb_resolve(dmb, map_handle, size, DMB_RESOLVE_WHOLE, &ref))
+		return;
+
+	virtio_dmb_release(dmb, ref.slot, ref.nr);
+}
+
+static dma_addr_t virtio_dmb_op_map_page(union virtio_map map,
+					 struct page *page,
+					 unsigned long offset, size_t size,
+					 enum dma_data_direction dir,
+					 unsigned long attrs)
+{
+	struct virtio_dmb *dmb = map.dmb;
+	phys_addr_t src = page_to_phys(page) + offset;
+	struct virtio_dmb_ref ref;
+	unsigned int nr, slot;
+	dma_addr_t handle;
+	long ret;
+
+	if (!size || size > virtio_dmb_max_mapping(dmb))
+		return DMA_MAPPING_ERROR;
+
+	nr = virtio_dmb_slots(size);
+	ret = virtio_dmb_claim(dmb, nr);
+	if (ret < 0)
+		return DMA_MAPPING_ERROR;
+	slot = ret;
+
+	virtio_dmb_record(dmb, slot, nr, size, src);
+
+	handle = dmb->base_off + ((u64)slot << PAGE_SHIFT);
+
+	ref.slot = slot;
+	ref.nr = nr;
+	ref.src = src;
+
+	/*
+	 * Copy the caller's buffer in whatever the direction is, and without
+	 * honouring DMA_ATTR_SKIP_CPU_SYNC.  swiotlb_tbl_map_single() bounces
+	 * unconditionally for the same two reasons: a device that writes less
+	 * than the whole buffer must leave the rest of the caller's bytes
+	 * intact, and the mapped bytes must not reach the device as whatever
+	 * the slot held before.
+	 *
+	 * Those bytes and no others.  size need not be a multiple of
+	 * PAGE_SIZE, and [size, nr << PAGE_SHIFT) keeps what the slots held
+	 * before: a freed virtqueue area, an earlier mapping of this device,
+	 * or what the device left there itself.  Nothing outside a mapping's
+	 * own length is ever copied in, so the device reads nothing there
+	 * that the region did not already hold for it, and the descriptor
+	 * carries a length.  swiotlb leaves the remainder of its last slot
+	 * the same way.
+	 */
+	virtio_dmb_copy(dmb, &ref, handle, size, true);
+
+	return handle;
+}
+
+static void virtio_dmb_op_unmap_page(union virtio_map map,
+				     dma_addr_t map_handle, size_t size,
+				     enum dma_data_direction dir,
+				     unsigned long attrs)
+{
+	struct virtio_dmb *dmb = map.dmb;
+	struct virtio_dmb_ref ref;
+
+	if (!virtio_dmb_resolve(dmb, map_handle, size,
+				DMB_RESOLVE_BOUNCED | DMB_RESOLVE_WHOLE, &ref))
+		return;
+
+	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
+	    (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL))
+		virtio_dmb_copy(dmb, &ref, map_handle, size, false);
+
+	/*
+	 * The slot count comes from what map_page() recorded rather than from
+	 * the caller's size, so a mismatched size cannot release a different
+	 * number of slots than were claimed.
+	 */
+	virtio_dmb_release(dmb, ref.slot, ref.nr);
+}
+
+static void virtio_dmb_op_sync_single_for_cpu(union virtio_map map,
+					      dma_addr_t map_handle,
+					      size_t size,
+					      enum dma_data_direction dir)
+{
+	struct virtio_dmb *dmb = map.dmb;
+	struct virtio_dmb_ref ref;
+
+	/* A zero-length sync is a no-op and not a bad handle. */
+	if (!size)
+		return;
+
+	if (!virtio_dmb_resolve(dmb, map_handle, size, DMB_RESOLVE_BOUNCED,
+				&ref))
+		return;
+
+	if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
+		virtio_dmb_copy(dmb, &ref, map_handle, size, false);
+}
+
+static void virtio_dmb_op_sync_single_for_device(union virtio_map map,
+						 dma_addr_t map_handle,
+						 size_t size,
+						 enum dma_data_direction dir)
+{
+	struct virtio_dmb *dmb = map.dmb;
+	struct virtio_dmb_ref ref;
+
+	/* A zero-length sync is a no-op and not a bad handle. */
+	if (!size)
+		return;
+
+	if (!virtio_dmb_resolve(dmb, map_handle, size, DMB_RESOLVE_BOUNCED,
+				&ref))
+		return;
+
+	if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)
+		virtio_dmb_copy(dmb, &ref, map_handle, size, true);
+}
+
+static bool virtio_dmb_op_need_sync(union virtio_map map, dma_addr_t map_handle)
+{
+	/* Every mapping is a bounce, so every sync is a real copy. */
+	return true;
+}
+
+static int virtio_dmb_op_mapping_error(union virtio_map map,
+				       dma_addr_t map_handle)
+{
+	/*
+	 * DMA_MAPPING_ERROR is the value virtio_ring reserves.  Offset 0 is
+	 * reserved too, by the proposal and by every device implementation
+	 * that reads a queue address of zero as a queue that was never
+	 * programmed, so it is not an address this driver may publish either.
+	 * Nothing allocated here yields it, because the pool starts after the
+	 * first byte of the region for the reason virtio_dmb_init() gives, but
+	 * a premapped buffer carries an address its caller obtained and
+	 * vring_map_one_sg() asks this operation to judge that one.  Whether
+	 * such an address came from this map cannot be answered here, and a
+	 * containment test would answer a different question, but zero can be
+	 * answered: it is the one value the proposal rules out outright.
+	 */
+	if (map_handle == DMA_MAPPING_ERROR || !map_handle)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static size_t virtio_dmb_op_max_mapping_size(union virtio_map map)
+{
+	return virtio_dmb_max_mapping(map.dmb);
+}
+
+static const struct virtio_map_ops virtio_dmb_map_ops = {
+	.map_page		= virtio_dmb_op_map_page,
+	.unmap_page		= virtio_dmb_op_unmap_page,
+	.sync_single_for_cpu	= virtio_dmb_op_sync_single_for_cpu,
+	.sync_single_for_device	= virtio_dmb_op_sync_single_for_device,
+	.alloc			= virtio_dmb_op_alloc,
+	.free			= virtio_dmb_op_free,
+	.need_sync		= virtio_dmb_op_need_sync,
+	.mapping_error		= virtio_dmb_op_mapping_error,
+	.max_mapping_size	= virtio_dmb_op_max_mapping_size,
+};
+
+/*
+ * Whether the device still has virtqueues.  vqs_list_lock is what protects
+ * that list against a concurrent adder.  No caller here can race one, because
+ * every path that reaches this runs under the device lock and before or after
+ * the driver's find_vqs(), but the invariant is worth enforcing rather than
+ * inheriting from callers this file does not control.
+ */
+static bool virtio_dmb_vqs_live(struct virtio_device *vdev)
+{
+	bool live;
+
+	spin_lock(&vdev->vqs_list_lock);
+	live = !list_empty(&vdev->vqs);
+	spin_unlock(&vdev->vqs_list_lock);
+
+	return live;
+}
+
+/**
+ * virtio_dmb_destroy - release the Device Memory Buffer state of a device
+ * @vdev: the device
+ *
+ * Does nothing unless @vdev is currently using a Device Memory Buffer, and
+ * refuses if the device still has virtqueues: they would be left pointing
+ * into a region that is no longer mapped.  The mapping and the physical
+ * region claim then stay behind until something deletes those virtqueues and
+ * calls again, which unbinding the driver does: virtio_dev_remove() calls the
+ * driver's remove() before this.
+ */
+void virtio_dmb_destroy(struct virtio_device *vdev)
+{
+	struct virtio_dmb *dmb;
+
+	/*
+	 * vdev->map identifies which member of vdev->vmap is live, so it is
+	 * also the test for whether the union holds a Device Memory Buffer.
+	 */
+	if (vdev->map != &virtio_dmb_map_ops)
+		return;
+
+	/*
+	 * A virtqueue keeps the mapping token it was created with, while
+	 * vdev->map is consulted afresh on every dispatch.  Clearing vdev->map
+	 * therefore does not disarm a live virtqueue, it redirects that
+	 * virtqueue's copy of the token into the DMA API, where the pointer
+	 * this frees would be used as a struct device.  Refuse instead and
+	 * leak the mapping, which is unconditionally better than a
+	 * use-after-free.
+	 *
+	 * Reported rather than warned about, because a driver that left its
+	 * virtqueues in place is not the only way to get here.  A device that
+	 * fails to report its region on the way back from a suspend takes
+	 * virtio_device_restore() to its error path, which calls this, and a
+	 * driver with no freeze callback still has its virtqueues at that
+	 * point, correctly.  A condition a correct driver can satisfy must
+	 * not taint the kernel.
+	 */
+	if (virtio_dmb_vqs_live(vdev)) {
+		dev_warn(&vdev->dev,
+			 "device memory buffer not released, virtqueues are still live\n");
+		return;
+	}
+
+	dmb = vdev->vmap.dmb;
+
+	/* Put back exactly what the transport had installed. */
+	vdev->map = dmb->prev_map;
+	vdev->vmap = dmb->prev_vmap;
+
+	memunmap(dmb->map_va);
+	if (dmb->map_claimed)
+		release_mem_region(dmb->map_phys, dmb->map_len);
+	kvfree(dmb->slots);
+	kfree(dmb->areas);
+	bitmap_free(dmb->bitmap);
+	kfree(dmb);
+}
+EXPORT_SYMBOL_GPL(virtio_dmb_destroy);
+
+/**
+ * virtio_dmb_init - make a device's Device Memory Buffer state current
+ * @vdev: the device, with feature negotiation complete
+ *
+ * Reads the shared memory id the device reports, locates the region, builds
+ * an allocator over it and routes every mapping of the device through it.
+ * When the feature is not negotiated, releases any state a previous
+ * negotiation left behind.
+ *
+ * The operation is "make the state match what the device reports now", and it
+ * is reached again from resume and from reset completion.  A device that
+ * reports the region it reported last time keeps the state it already has, so
+ * handles held by a virtqueue that outlived the transition stay valid.  A
+ * device that reports a different region has the state rebuilt when no
+ * virtqueue is live, and is refused otherwise: a virtqueue holds kernel
+ * addresses inside the mapping and cannot be redirected into a new one.
+ *
+ * A caller that gets an error must set the FAILED device status bit, and must
+ * not touch the device for anything else before it does.  The device has
+ * already confirmed the feature by the time this runs, so it is entitled to
+ * assume the driver will address it through the region; the bit is what tells
+ * it the driver gave up instead.
+ *
+ * Return: 0 on success, or a negative errno.
+ */
+int virtio_dmb_init(struct virtio_device *vdev)
+{
+	struct virtio_shm_region region;
+	struct virtio_dmb *dmb;
+	unsigned int nslots, skew;
+	unsigned int area_slots, nareas, target, i;
+	u64 base_off, slots = 0;
+	size_t map_len;
+	u16 shm_id;
+	int err;
+
+	if (!virtio_has_feature(vdev, VIRTIO_F_DMB)) {
+		/*
+		 * The feature may have been withdrawn across re-negotiation.
+		 *
+		 * virtio_dmb_destroy() refuses under live virtqueues, and
+		 * returning 0 after a refusal would leave this file's map
+		 * operations installed for a device that has not negotiated the
+		 * feature, so every later mapping would resolve a handle
+		 * against a region the device no longer agrees it has.  Report
+		 * the refusal to the caller instead, which sets the FAILED
+		 * device status bit.  No path reaches this today: it needs map
+		 * operations an earlier negotiation installed, which unbinding
+		 * destroys, so only the restore path can find them, and that
+		 * path hands finalize_features() the word already accepted
+		 * rather than the offer, so a transport could drop the feature
+		 * there only in reaction to a device that changed what it
+		 * offers after the driver bound, and no reset does that.
+		 */
+		virtio_dmb_destroy(vdev);
+		if (vdev->map == &virtio_dmb_map_ops)
+			return -EBUSY;
+		return 0;
+	}
+
+	/*
+	 * Without both of these the region cannot be located at all, which is
+	 * what keeps a transport that does not implement them from offering
+	 * the feature in the first place.
+	 */
+	if (!vdev->config->get_dmb_shm_id || !vdev->config->get_shm_region) {
+		dev_warn(&vdev->dev,
+			 "transport cannot locate a device memory buffer\n");
+		return -EINVAL;
+	}
+
+	/* The feature is only defined together with VIRTIO_F_ACCESS_PLATFORM. */
+	if (!virtio_has_feature(vdev, VIRTIO_F_ACCESS_PLATFORM)) {
+		dev_warn(&vdev->dev,
+			 "device memory buffer without VIRTIO_F_ACCESS_PLATFORM\n");
+		return -EINVAL;
+	}
+
+	err = vdev->config->get_dmb_shm_id(vdev, &shm_id);
+	if (err)
+		return err;
+
+	/* A region is looked up by a u8 id. */
+	if (shm_id > U8_MAX) {
+		dev_warn(&vdev->dev,
+			 "device memory buffer id %u out of range\n", shm_id);
+		return -EINVAL;
+	}
+
+	if (!virtio_get_shm_region(vdev, &region, shm_id)) {
+		dev_warn(&vdev->dev,
+			 "cannot locate device memory buffer region %u\n",
+			 shm_id);
+		return -ENODEV;
+	}
+
+	/*
+	 * The region base carries no alignment guarantee, but every virtqueue
+	 * layout requires one of the areas placed in it.  Start the pool at a
+	 * PAGE_SIZE-aligned address and record the skew, so that page-granular
+	 * allocation makes every absolute address aligned.
+	 *
+	 * PAGE_SIZE - skew is the distance from the start of the region to the
+	 * first aligned address strictly after it, so the pool never begins at
+	 * the region's first byte and no handle is ever 0.  The proposal
+	 * reserves offset 0: it is not the address of any structure the driver
+	 * places in the region, and a device may treat it as an error.  A device
+	 * that predates the reservation reads a queue address of 0 as the queue
+	 * never having been programmed and ignores it, so the value is unusable
+	 * either way.  Keeping it out of the pool costs one page of an
+	 * already-aligned region and nothing at all of a misaligned one, whose
+	 * leading partial page was unusable regardless.
+	 */
+	skew = offset_in_page(region.addr);
+	base_off = PAGE_SIZE - skew;
+
+	if (region.len > base_off)
+		slots = (region.len - base_off) >> PAGE_SHIFT;
+
+	/*
+	 * The least a region could hold: one minimally-sized virtqueue plus
+	 * one buffer in flight against it.  A packed queue costs three
+	 * allocations, a descriptor ring and two event structures, and a split
+	 * queue up to two when the transport aligns its areas to PAGE_SIZE, so
+	 * four slots is the floor for either layout.
+	 *
+	 * Four slots is four pages of pool, which is five pages of region for
+	 * a region whose base is already aligned, since the page the pool
+	 * starts after is not part of it.
+	 *
+	 * That derivation counts one virtqueue.  A device that also offers an
+	 * administration virtqueue has its areas allocated from the same region
+	 * through the same path, and one administration command occupies
+	 * several slots more, so four pages is a floor such a device is
+	 * misconfigured to sit on rather than a size it can work at.
+	 *
+	 * This floor is enforced, but it is a floor and not a sufficiency
+	 * check.  How much a device actually needs depends on how many
+	 * virtqueues its driver creates and how deep they are, neither of
+	 * which is known here: this runs during feature negotiation, before
+	 * find_vqs().  A region above this floor but still too small fails
+	 * there instead, which for a split ring reduces the queue depth and
+	 * for a packed ring fails the queue.
+	 *
+	 * The upper bounds are what a slot index, a mapping length and a
+	 * published handle can each represent.  The mapping length bound is
+	 * exclusive because the length mapped is base_off larger than the
+	 * pool, and base_off is a whole page where the region base is
+	 * aligned: at the last representable slot count that sum would wrap
+	 * to zero on a 32-bit size_t.
+	 */
+	if (slots < 4 || slots > UINT_MAX ||
+	    slots >= (u64)(SIZE_MAX >> PAGE_SHIFT) ||
+	    base_off + (slots << PAGE_SHIFT) - 1 >
+			DMA_BIT_MASK(BITS_PER_TYPE(dma_addr_t))) {
+		dev_warn(&vdev->dev,
+			 "device memory buffer region holds %llu usable pages\n",
+			 slots);
+		return -EINVAL;
+	}
+	nslots = slots;
+
+	/* Nothing outside the pool and the bytes ahead of it is used. */
+	map_len = base_off + ((size_t)nslots << PAGE_SHIFT);
+
+	/*
+	 * The transport reports the region in 64 bits while a resource is
+	 * addressed in resource_size_t.  Refuse a region that does not fit
+	 * rather than claim and map a truncated one.
+	 */
+	if (region.addr > (u64)(resource_size_t)-1 - map_len) {
+		dev_warn(&vdev->dev,
+			 "device memory buffer region at 0x%llx is not addressable\n",
+			 region.addr);
+		return -EINVAL;
+	}
+
+	/*
+	 * Everything that identifies the region is known now and nothing has
+	 * been touched yet, so an unchanged region can be adopted instead of
+	 * being torn down and rebuilt identically.  That is what lets a
+	 * virtqueue which outlived a suspend or a reset keep handles that are
+	 * still valid, and it is why no separate freeze-time teardown is
+	 * needed.  A region that moved can be neither adopted nor replaced
+	 * under live virtqueues, so refuse without a warning: a device that
+	 * moves its region while its driver still has virtqueues is
+	 * misbehaving, which is not evidence of a kernel bug.
+	 */
+	if (vdev->map == &virtio_dmb_map_ops) {
+		dmb = vdev->vmap.dmb;
+
+		if (dmb->shm_id == shm_id && dmb->map_phys == region.addr &&
+		    dmb->map_len == map_len)
+			return 0;
+
+		if (virtio_dmb_vqs_live(vdev))
+			return -EBUSY;
+
+		virtio_dmb_destroy(vdev);
+	}
+
+	dmb = kzalloc(sizeof(*dmb), GFP_KERNEL);
+	if (!dmb)
+		return -ENOMEM;
+
+	dmb->vdev = vdev;
+	dmb->shm_id = shm_id;
+	dmb->base_off = base_off;
+	dmb->map_phys = region.addr;
+	dmb->map_len = map_len;
+
+	/*
+	 * A shared memory region need not lie in a BAR the transport already
+	 * claimed, so record a claim on the range here.  The claim is
+	 * advisory: where the region does lie in such a BAR the transport's
+	 * own claim already covers it, and that is not a conflict with
+	 * anything, so it must not fail the device.  A foreign driver cannot
+	 * own another device's BAR range either, so a refusal here is not
+	 * evidence that anything is wrong.
+	 */
+	dmb->map_claimed = request_mem_region(region.addr, map_len,
+					      "virtio-dmb") != NULL;
+	if (!dmb->map_claimed)
+		dev_dbg(&vdev->dev,
+			"device memory buffer region at 0x%llx already reserved\n",
+			region.addr);
+
+	/*
+	 * MEMREMAP_DEC because the region is memory shared with the device,
+	 * which is what the proposal requires of a driver wherever the platform
+	 * distinguishes that from memory private to the driver.  Without it
+	 * x86's arch_memremap_wb() applies the guest's own encryption to the
+	 * mapping, and the device would see ciphertext wherever a guest encrypts
+	 * its memory.  x86 is the only architecture that reads the flag; arm64
+	 * ignores it and reaches ioremap_prot() instead, where a realm guest's
+	 * hook finds the region is not protected memory and shares the mapping.
+	 * Anywhere else that draws the distinction, the proposal forbids the
+	 * device from offering the feature at all.
+	 */
+	dmb->map_va = memremap(region.addr, map_len,
+			       MEMREMAP_WB | MEMREMAP_DEC);
+	if (!dmb->map_va) {
+		/*
+		 * memremap() is silent, and every sibling failure in this
+		 * function names itself.  Without this the device is left with
+		 * the FAILED status bit set and nothing saying why.
+		 */
+		dev_warn(&vdev->dev,
+			 "cannot map device memory buffer region at 0x%llx\n",
+			 region.addr);
+		err = -ENOMEM;
+		goto err_unclaim;
+	}
+
+	/*
+	 * The skew was derived from the physical base, so the pool is aligned
+	 * in the mapping only if the mapping kept that page offset.  Check it
+	 * rather than assume it.
+	 */
+	if (offset_in_page(dmb->map_va) != skew) {
+		dev_warn(&vdev->dev,
+			 "device memory buffer mapping is not page-congruent\n");
+		err = -EINVAL;
+		goto err_unmap;
+	}
+
+	dmb->base_va = dmb->map_va + (size_t)base_off;
+
+	dmb->bitmap = bitmap_zalloc(nslots, GFP_KERNEL);
+	if (!dmb->bitmap) {
+		err = -ENOMEM;
+		goto err_unmap;
+	}
+
+	/* A zeroed record has end == 0, which is what marks a slot free. */
+	dmb->slots = kvcalloc(nslots, sizeof(*dmb->slots), GFP_KERNEL);
+	if (!dmb->slots) {
+		err = -ENOMEM;
+		goto err_free_bitmap;
+	}
+
+	/*
+	 * Divide the pool into independently locked areas, so that mappings on
+	 * different CPUs do not serialise on one lock and the interrupts-off
+	 * window of one search does not grow with the region.  This is the
+	 * structure kernel/dma/swiotlb.c adopted in commit 20347fca71a3
+	 * ("swiotlb: split up the global swiotlb lock"), for the same reason.
+	 *
+	 * area_slots is the power of two and nareas is derived from it, which
+	 * is the reverse of swiotlb.  swiotlb indexes areas with a mask and
+	 * rounds its pool size up to suit; a region's length is the device's
+	 * and cannot be rounded up, and dividing a power-of-two area count into
+	 * it would leave area_slots neither a power of two nor a multiple of
+	 * BITS_PER_LONG.  That matters for correctness rather than for tuning:
+	 * bitmap_set() and bitmap_clear() are non-atomic read-modify-write on
+	 * an unsigned long, so two areas sharing a bitmap word under separate
+	 * locks would lose updates.  A power-of-two area_slots at least
+	 * BITS_PER_LONG makes every area own whole words, and makes the area of
+	 * a slot a shift.  The cost is one division per claim where swiotlb has
+	 * a mask, which is a fraction of the two copies every mapping already
+	 * performs.
+	 *
+	 * num_possible_cpus() and not num_online_cpus(), so that the division
+	 * is sized for the CPUs that can run rather than for the ones running
+	 * when the region is installed, which on a guest that onlines the rest
+	 * later would divide the pool for one.  Nothing is allocated per CPU
+	 * and no area belongs to one, so a CPU going away strands no capacity
+	 * and there is no hotplug callback.  A pool too small to divide that
+	 * far yields fewer areas than CPUs, which then share.
+	 */
+	BUILD_BUG_ON(DMB_AREA_MIN_SLOTS > DMB_AREA_MAX_SLOTS);
+	BUILD_BUG_ON(DMB_AREA_MIN_SLOTS < BITS_PER_LONG);
+	BUILD_BUG_ON(!is_power_of_2(DMB_AREA_MIN_SLOTS));
+
+	target = nslots / roundup_pow_of_two(num_possible_cpus());
+	if (target < DMB_AREA_MIN_SLOTS)
+		/* rounddown_pow_of_two(0) is undefined. */
+		area_slots = DMB_AREA_MIN_SLOTS;
+	else
+		area_slots = clamp_t(unsigned int,
+				     rounddown_pow_of_two(target),
+				     DMB_AREA_MIN_SLOTS, DMB_AREA_MAX_SLOTS);
+
+	nareas = DIV_ROUND_UP(nslots, area_slots);
+
+	dmb->areas = kcalloc(nareas, sizeof(*dmb->areas), GFP_KERNEL);
+	if (!dmb->areas) {
+		err = -ENOMEM;
+		goto err_free_slots;
+	}
+
+	for (i = 0; i < nareas; i++)
+		spin_lock_init(&dmb->areas[i].lock);
+
+	dmb->nslots = nslots;
+	dmb->nareas = nareas;
+	dmb->area_slots = area_slots;
+	dmb->area_shift = ilog2(area_slots);
+
+	/* Published last: until now nothing routes a mapping here. */
+	dmb->prev_map = vdev->map;
+	dmb->prev_vmap = vdev->vmap;
+	vdev->vmap.dmb = dmb;
+	vdev->map = &virtio_dmb_map_ops;
+
+	/*
+	 * The feature moves every virtqueue of this device into a region and
+	 * changes what every address published to it means, and it activates
+	 * from a value the device supplies that nothing else records.  Report
+	 * the three facts about it that are recoverable nowhere else, on the
+	 * device that negotiated it, and the derived area geometry with them so
+	 * that it is visible without debugfs.
+	 */
+	dev_info(&vdev->dev,
+		 "device memory buffer %u at %pa, %u usable pages in %u areas of %u pages\n",
+		 shm_id, &dmb->map_phys, nslots, dmb->nareas,
+		 dmb->area_slots);
+
+	return 0;
+
+err_free_slots:
+	kvfree(dmb->slots);
+err_free_bitmap:
+	bitmap_free(dmb->bitmap);
+err_unmap:
+	memunmap(dmb->map_va);
+err_unclaim:
+	if (dmb->map_claimed)
+		release_mem_region(dmb->map_phys, dmb->map_len);
+	kfree(dmb);
+	return err;
+}
+EXPORT_SYMBOL_GPL(virtio_dmb_init);
+
+MODULE_DESCRIPTION("Virtio device memory buffer allocator");
+MODULE_LICENSE("GPL");
diff --git a/drivers/virtio/virtio_dmb.h b/drivers/virtio/virtio_dmb.h
new file mode 100644
index 000000000000..69fbcbb9c2c0
--- /dev/null
+++ b/drivers/virtio/virtio_dmb.h
@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Device Memory Buffer support for virtio devices.
+ */
+#ifndef _DRIVERS_VIRTIO_VIRTIO_DMB_H
+#define _DRIVERS_VIRTIO_VIRTIO_DMB_H
+
+struct virtio_device;
+
+#if IS_ENABLED(CONFIG_VIRTIO_DMB)
+
+int virtio_dmb_init(struct virtio_device *vdev);
+void virtio_dmb_destroy(struct virtio_device *vdev);
+
+#else
+
+static inline int virtio_dmb_init(struct virtio_device *vdev)
+{
+	return 0;
+}
+
+static inline void virtio_dmb_destroy(struct virtio_device *vdev)
+{
+}
+
+#endif /* CONFIG_VIRTIO_DMB */
+
+#endif /* _DRIVERS_VIRTIO_VIRTIO_DMB_H */
diff --git a/include/linux/virtio.h b/include/linux/virtio.h
index 93e573c56563..cd3fe82ff328 100644
--- a/include/linux/virtio.h
+++ b/include/linux/virtio.h
@@ -44,12 +44,15 @@ struct virtqueue {
 };
 
 struct vduse_vq_group;
+struct virtio_dmb;
 
 union virtio_map {
 	/* Device that performs DMA */
 	struct device *dma_dev;
 	/* VDUSE specific virtqueue group for doing map */
 	struct vduse_vq_group *group;
+	/* Device Memory Buffer holding the virtqueues and their buffers */
+	struct virtio_dmb *dmb;
 };
 
 int virtqueue_add_outbuf(struct virtqueue *vq,
diff --git a/include/linux/virtio_config.h b/include/linux/virtio_config.h
index 69f84ea85d71..a6780aa85966 100644
--- a/include/linux/virtio_config.h
+++ b/include/linux/virtio_config.h
@@ -96,6 +96,13 @@ struct virtqueue_info {
  * @set_vq_affinity: set the affinity for a virtqueue (optional).
  * @get_vq_affinity: get the affinity for a virtqueue (optional).
  * @get_shm_region: get a shared memory region based on the index.
+ * @get_dmb_shm_id: get the shared memory id of the Device Memory Buffer
+ *	(optional).
+ *	vdev: the device
+ *	id: where to store the shared memory id
+ *	Returns 0 on success or error status
+ *	Only valid once VIRTIO_F_DMB has been negotiated.  A transport that
+ *	does not implement this must not accept VIRTIO_F_DMB.
  * @disable_vq_and_reset: reset a queue individually (optional).
  *	vq: the virtqueue
  *	Returns 0 on success or error status
@@ -135,6 +142,7 @@ struct virtio_config_ops {
 						 int index);
 	bool (*get_shm_region)(struct virtio_device *vdev,
 			       struct virtio_shm_region *region, u8 id);
+	int (*get_dmb_shm_id)(struct virtio_device *vdev, u16 *id);
 	int (*disable_vq_and_reset)(struct virtqueue *vq);
 	int (*enable_vq_after_reset)(struct virtqueue *vq);
 };