[PATCH 15/16] gpu: nova-core: mm: Add BAR1 user interface

[Eliot Courtney <ecourtney@nvidia.com>]

From: Joel Fernandes <joelagnelf@nvidia.com>

Add the BAR1 user interface for CPU access to GPU virtual memory through
the BAR1 aperture.

Signed-off-by: Joel Fernandes <joelagnelf@nvidia.com>
[ecourtney: map BAR1 in NovaCore, borrow it in Gpu and BarUser, no Devres]
[ecourtney: update for the VramAddress raw API and gsp_resources chipset]
[ecourtney: drop the owned GpuMm, pass it mutably through map and release]
Signed-off-by: Eliot Courtney <ecourtney@nvidia.com>
---
 drivers/gpu/nova-core/driver.rs       |  45 +++++++--
 drivers/gpu/nova-core/gpu.rs          |  32 ++++++-
 drivers/gpu/nova-core/gsp/commands.rs |   1 -
 drivers/gpu/nova-core/mm.rs           |   1 +
 drivers/gpu/nova-core/mm/bar_user.rs  | 171 ++++++++++++++++++++++++++++++++++
 5 files changed, 240 insertions(+), 10 deletions(-)

diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs
index 5723ff8f71ea..0672a0707a71 100644
--- a/drivers/gpu/nova-core/driver.rs
+++ b/drivers/gpu/nova-core/driver.rs
@@ -2,7 +2,11 @@
 
 use kernel::{
     auxiliary,
-    device::Core,
+    device::{
+        Bound,
+        Core, //
+    },
+    io::resource,
     pci,
     pci::{
         Class,
@@ -28,6 +32,7 @@ pub(crate) struct NovaCore<'bound> {
     #[pin]
     pub(crate) gpu: Gpu<'bound>,
     bar: pci::Bar<'bound, BAR0_SIZE>,
+    bar1: Bar1<'bound>,
     #[allow(clippy::type_complexity)]
     _reg: auxiliary::Registration<'bound, CovariantForLt!(())>,
 }
@@ -38,9 +43,27 @@ pub(crate) struct NovaCore<'bound> {
 
 pub(crate) type Bar0<'a> = &'a pci::Bar<'a, BAR0_SIZE>;
 pub(crate) type NovaRegisters = kernel::io::Region<BAR0_SIZE>;
-#[expect(dead_code)]
 pub(crate) type Bar1<'a> = pci::Bar<'a>;
 
+/// Returns the Linux PCI resource index that holds BAR1 for an NVIDIA GPU.
+///
+/// On Maxwell through Ada, BAR0 is a 32-bit memory BAR occupying a single
+/// Linux PCI resource slot, so BAR1 lives at index 1. Starting with Blackwell
+/// (and on some Ampere GA100 / Hopper SKUs) BAR0 is a 64-bit memory BAR that
+/// consumes two consecutive resource slots: index 0 holds the low 32 bits and
+/// index 1 holds the high 32 bits (with no `flags` / or size of its own),
+/// shifting BAR1 to index 2.
+pub(crate) fn bar1_resource_index(pdev: &pci::Device<Bound>) -> Result<u32> {
+    // Probe the `IORESOURCE_MEM_64` flag of BAR0 as a robust way of exposing
+    // if BAR0 and hence BAR1 is 64-bit.
+    let flags0 = pdev.resource_flags(0)?;
+    if flags0.contains(resource::Flags::IORESOURCE_MEM_64) {
+        Ok(2)
+    } else {
+        Ok(1)
+    }
+}
+
 kernel::pci_device_table!(
     PCI_TABLE,
     <NovaCoreDriver as pci::Driver>::IdInfo,
@@ -82,12 +105,18 @@ fn probe<'bound>(
 
             Ok(try_pin_init!(NovaCore {
                 bar: pdev.iomap_region_sized::<BAR0_SIZE>(0, c"nova-core/bar0")?,
-                // TODO: Use `&bar` self-referential pin-init syntax once available.
-                //
-                // SAFETY: `bar` is initialized before this expression is evaluated
-                // (`try_pin_init!()` initializes fields in declaration order), lives at a pinned
-                // stable address, and is dropped after `gpu` (struct field drop order).
-                gpu <- Gpu::new(pdev, unsafe { &*core::ptr::from_ref(bar) }),
+                bar1: {
+                    let bar1_idx = bar1_resource_index(pdev)?;
+                    pdev.iomap_region(bar1_idx, c"nova-core/bar1")?
+                },
+                // TODO: Use self-referential pin-init syntax once available.
+                gpu <- Gpu::new(
+                    pdev,
+                    // SAFETY: `bar` is initialized above, pinned, and outlives `gpu`.
+                    unsafe { &*core::ptr::from_ref(bar) },
+                    // SAFETY: `bar1` is initialized above, pinned, and outlives `gpu`.
+                    unsafe { &*core::ptr::from_ref(bar1) },
+                ),
                 // Run optional GPU selftests.
                 #[cfg(CONFIG_NOVA_CORE_SELFTESTS)]
                 _: { gpu.run_selftests(pdev) },
diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs
index b797472279d3..f72a92e045e5 100644
--- a/drivers/gpu/nova-core/gpu.rs
+++ b/drivers/gpu/nova-core/gpu.rs
@@ -16,11 +16,15 @@
         SizeConstants,
         SZ_4K, //
     },
+    sync::Arc,
 };
 
 use crate::{
     bounded_enum,
-    driver::Bar0,
+    driver::{
+        Bar0,
+        Bar1, //
+    },
     falcon::{
         gsp::Gsp as GspFalcon,
         sec2::Sec2 as Sec2Falcon,
@@ -35,6 +39,8 @@
         GspBootContext, //
     },
     mm::{
+        bar_user::BarUser,
+        pagetable::MmuVersion,
         GpuMm,
         VramAddress, //
     },
@@ -148,6 +154,11 @@ pub(crate) const fn arch(self) -> Architecture {
     pub(crate) fn pci_config_mirror_range(self) -> Range<u32> {
         hal::gpu_hal(self).pci_config_mirror_range()
     }
+
+    /// Returns the MMU version for this chipset.
+    pub(crate) fn mmu_version(self) -> MmuVersion {
+        MmuVersion::from(self.arch())
+    }
 }
 
 // TODO
@@ -297,6 +308,8 @@ pub(crate) struct Gpu<'gpu> {
     /// Must be kept declared *before* `gsp_resources`, so that its components are dropped while
     /// the GSP is still operational.
     mm: GpuMm<'gpu>,
+    /// BAR1 user interface for CPU access to GPU virtual memory.
+    bar_user: Arc<BarUser<'gpu>>,
     /// GSP and its resources.
     #[pin]
     gsp_resources: GspResources<'gpu>,
@@ -340,6 +353,7 @@ impl<'gpu> Gpu<'gpu> {
     pub(crate) fn new<'a>(
         pdev: &'gpu pci::Device<device::Core<'a>>,
         bar: Bar0<'gpu>,
+        bar1: &'gpu Bar1<'gpu>,
     ) -> impl PinInit<Self, Error> + use<'gpu, 'a> {
         let dev = pdev.as_ref();
 
@@ -442,6 +456,22 @@ pub(crate) fn new<'a>(
                     VramAddress::from_raw(gsp_static_info.total_fb_end),
                 )?
             },
+
+            // Create BAR1 user interface for CPU access to GPU virtual memory.
+            bar_user: {
+                let pdb_addr = VramAddress::from_raw(gsp_static_info.bar1_pde_base);
+                let bar1_idx = crate::driver::bar1_resource_index(pdev)?;
+                let bar1_size = pdev.resource_len(bar1_idx)?;
+                Arc::pin_init(
+                    BarUser::new(
+                        pdb_addr,
+                        gsp_resources.spec.chipset,
+                        bar1_size,
+                        bar1,
+                    )?,
+                    GFP_KERNEL,
+                )?
+            },
         })
     }
 
diff --git a/drivers/gpu/nova-core/gsp/commands.rs b/drivers/gpu/nova-core/gsp/commands.rs
index 1c467718c679..663e1d124781 100644
--- a/drivers/gpu/nova-core/gsp/commands.rs
+++ b/drivers/gpu/nova-core/gsp/commands.rs
@@ -215,7 +215,6 @@ fn init(&self) -> impl Init<Self::Command, Self::InitError> {
 pub(crate) struct GetGspStaticInfoReply {
     gpu_name: [u8; 64],
     /// BAR1 Page Directory Entry base address.
-    #[expect(dead_code)]
     pub(crate) bar1_pde_base: u64,
     /// Usable FB (VRAM) regions for driver memory allocation.
     pub(crate) usable_fb_regions: KVec<Range<u64>>,
diff --git a/drivers/gpu/nova-core/mm.rs b/drivers/gpu/nova-core/mm.rs
index 2cf37254fbb9..e04497256519 100644
--- a/drivers/gpu/nova-core/mm.rs
+++ b/drivers/gpu/nova-core/mm.rs
@@ -60,6 +60,7 @@ macro_rules! impl_pfn_bounded {
 
 pub(crate) use tlb::Tlb;
 
+pub(crate) mod bar_user;
 mod hal;
 pub(super) mod pagetable;
 mod pramin;
diff --git a/drivers/gpu/nova-core/mm/bar_user.rs b/drivers/gpu/nova-core/mm/bar_user.rs
new file mode 100644
index 000000000000..ef1d8e6f8c9c
--- /dev/null
+++ b/drivers/gpu/nova-core/mm/bar_user.rs
@@ -0,0 +1,171 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! BAR1 user interface for CPU access to GPU virtual memory. Used for USERD
+//! for GPU work submission, and applications to access GPU buffers via mmap().
+
+use kernel::{
+    io::Io,
+    new_mutex,
+    prelude::*,
+    sync::{
+        Arc,
+        Mutex, //
+    },
+};
+
+use crate::{
+    driver::Bar1,
+    gpu::Chipset,
+    mm::{
+        vmm::{
+            MappedRange,
+            Vmm, //
+        },
+        GpuMm,
+        Pfn,
+        Vfn,
+        VirtualAddress,
+        VramAddress,
+        PAGE_SIZE, //
+    },
+    num::IntoSafeCast,
+};
+
+/// BAR1 user interface for virtual memory mappings.
+///
+/// Owns the [`Vmm`] for the BAR1 address space.
+#[pin_data]
+pub(crate) struct BarUser<'gpu> {
+    #[pin]
+    vmm: Mutex<Vmm>,
+    bar1: &'gpu Bar1<'gpu>,
+}
+
+impl<'gpu> BarUser<'gpu> {
+    /// Create a pin-initializer for [`BarUser`].
+    pub(crate) fn new(
+        pdb_addr: VramAddress,
+        chipset: Chipset,
+        va_size: u64,
+        bar1: &'gpu Bar1<'gpu>,
+    ) -> Result<impl PinInit<Self> + 'gpu> {
+        let vmm = Vmm::new(pdb_addr, chipset.mmu_version(), va_size)?;
+        Ok(pin_init!(Self {
+            vmm <- new_mutex!(vmm, "bar_user_vmm"),
+            bar1,
+        }))
+    }
+
+    /// Map physical pages to a contiguous BAR1 virtual range.
+    pub(crate) fn map(
+        self: &Arc<Self>,
+        mm: &mut GpuMm<'_>,
+        pfns: &[Pfn],
+        writable: bool,
+    ) -> Result<BarUserAccess<'gpu>> {
+        if pfns.is_empty() {
+            return Err(EINVAL);
+        }
+        let mut vmm = self.vmm.lock();
+        let mapped = vmm.map_pages(mm, pfns, None, writable)?;
+
+        Ok(BarUserAccess {
+            bar_user: self.clone(),
+            mapped: Some(mapped),
+        })
+    }
+}
+
+/// Access object for a mapped BAR1 region.
+pub(crate) struct BarUserAccess<'gpu> {
+    bar_user: Arc<BarUser<'gpu>>,
+    /// [`BarUserAccess::release`] [`Option::take`]s this; `Some` at
+    /// drop time means `release()` was never called.
+    mapped: Option<MappedRange>,
+}
+
+impl BarUserAccess<'_> {
+    /// Tear down the BAR1 mapping.
+    pub(crate) fn release(mut self, mm: &mut GpuMm<'_>) -> Result {
+        let mapped = self.mapped.take().ok_or(EINVAL)?;
+        let mut vmm = self.bar_user.vmm.lock();
+        vmm.unmap_pages(mm, mapped)?;
+        Ok(())
+    }
+
+    /// Returns the active mapping.
+    fn mapped(&self) -> &MappedRange {
+        // `mapped` is only `None` after `take()` in `release`; hence unwrap()
+        // cannot panic here.
+        self.mapped.as_ref().unwrap()
+    }
+
+    /// Get the base virtual address of this mapping.
+    pub(crate) fn base(&self) -> VirtualAddress {
+        VirtualAddress::from(self.mapped().vfn_start)
+    }
+
+    /// Get the total size of the mapped region in bytes.
+    pub(crate) fn size(&self) -> usize {
+        self.mapped().num_pages * PAGE_SIZE
+    }
+
+    /// Get the starting virtual frame number.
+    pub(crate) fn vfn_start(&self) -> Vfn {
+        self.mapped().vfn_start
+    }
+
+    /// Get the number of pages in this mapping.
+    pub(crate) fn num_pages(&self) -> usize {
+        self.mapped().num_pages
+    }
+
+    /// Translate an offset within this mapping to a BAR1 aperture offset.
+    fn bar_offset(&self, offset: usize) -> Result<usize> {
+        if offset >= self.size() {
+            return Err(EINVAL);
+        }
+
+        let base_vfn: usize = self.mapped().vfn_start.raw().into_safe_cast();
+        let base = base_vfn.checked_mul(PAGE_SIZE).ok_or(EOVERFLOW)?;
+        base.checked_add(offset).ok_or(EOVERFLOW)
+    }
+
+    // Fallible accessors with runtime bounds checking.
+
+    /// Read a 32-bit value at the given offset.
+    pub(crate) fn try_read32(&self, offset: usize) -> Result<u32> {
+        let off = self.bar_offset(offset)?;
+        self.bar_user.bar1.try_read32(off)
+    }
+
+    /// Write a 32-bit value at the given offset.
+    pub(crate) fn try_write32(&self, value: u32, offset: usize) -> Result {
+        let off = self.bar_offset(offset)?;
+        self.bar_user.bar1.try_write32(value, off)
+    }
+
+    /// Read a 64-bit value at the given offset.
+    pub(crate) fn try_read64(&self, offset: usize) -> Result<u64> {
+        let off = self.bar_offset(offset)?;
+        self.bar_user.bar1.try_read64(off)
+    }
+
+    /// Write a 64-bit value at the given offset.
+    pub(crate) fn try_write64(&self, value: u64, offset: usize) -> Result {
+        let off = self.bar_offset(offset)?;
+        self.bar_user.bar1.try_write64(value, off)
+    }
+}
+
+impl Drop for BarUserAccess<'_> {
+    fn drop(&mut self) {
+        if self.mapped.is_some() {
+            kernel::pr_warn!(
+                "BarUserAccess dropped without calling release(). BarUser address space will leak.\n"
+            );
+        }
+        // The inner `MappedRange`'s own `MustUnmapGuard` will also fire,
+        // identifying the leaked VA range.
+    }
+}

-- 
2.55.0