Re: [PATCH RFC v6 05/26] nova-core: mm: Add support to use PRAMIN windows to write to VRAM

[Zhi Wang <zhiw@nvidia.com>]

On Tue, 20 Jan 2026 15:42:42 -0500
Joel Fernandes <joelagnelf@nvidia.com> wrote:

> PRAMIN apertures are a crucial mechanism to direct read/write to VRAM.
> Add support for the same.
> 

I went through the code, this seems not designed for multiple users. As
this is used for writting PTEs for page tables, can you shed some light
about the plan of how we should handle the concurrency of writting multiple
page table PTEs, e.g. when two GPU memory mapping in two different GPU
page tables are procceding concurrently, this could happen when people
creating vGPUs concurrently. 

Z.

> Signed-off-by: Joel Fernandes <joelagnelf@nvidia.com>
> ---
>  drivers/gpu/nova-core/mm/mod.rs    |   5 +
>  drivers/gpu/nova-core/mm/pramin.rs | 244 +++++++++++++++++++++++++++++
>  drivers/gpu/nova-core/nova_core.rs |   1 +
>  drivers/gpu/nova-core/regs.rs      |   5 +
>  4 files changed, 255 insertions(+)
>  create mode 100644 drivers/gpu/nova-core/mm/mod.rs
>  create mode 100644 drivers/gpu/nova-core/mm/pramin.rs
> 
> diff --git a/drivers/gpu/nova-core/mm/mod.rs
> b/drivers/gpu/nova-core/mm/mod.rs new file mode 100644
> index 000000000000..7a5dd4220c67
> --- /dev/null
> +++ b/drivers/gpu/nova-core/mm/mod.rs
> @@ -0,0 +1,5 @@
> +// SPDX-License-Identifier: GPL-2.0
> +
> +//! Memory management subsystems for nova-core.
> +
> +pub(crate) mod pramin;
> diff --git a/drivers/gpu/nova-core/mm/pramin.rs
> b/drivers/gpu/nova-core/mm/pramin.rs new file mode 100644
> index 000000000000..6a7ea2dc7d77
> --- /dev/null
> +++ b/drivers/gpu/nova-core/mm/pramin.rs
> @@ -0,0 +1,244 @@
> +// SPDX-License-Identifier: GPL-2.0
> +
> +//! Direct VRAM access through the PRAMIN aperture.
> +//!
> +//! PRAMIN provides a 1MB sliding window into VRAM through BAR0,
> allowing the CPU to access +//! video memory directly. The [`Window`]
> type automatically repositions the window when +//! accessing different
> VRAM regions and restores the original position on drop. This allows
> +//! to reuse the same window for multiple accesses in the same window.
> +//! +//! The PRAMIN aperture is a 1MB region at BAR0 + 0x700000 for all
> GPUs. The window base is +//! controlled by the `NV_PBUS_BAR0_WINDOW`
> register and must be 64KB aligned. +//!
> +//! # Examples
> +//!
> +//! ## Basic read/write
> +//!
> +//! ```no_run
> +//! use crate::driver::Bar0;
> +//! use crate::mm::pramin;
> +//! use kernel::devres::Devres;
> +//! use kernel::sync::Arc;
> +//!
> +//! fn example(devres_bar: Arc<Devres<Bar0>>) -> Result<()> {
> +//!     let mut pram_win = pramin::Window::new(devres_bar)?;
> +//!
> +//!     // Write and read back.
> +//!     pram_win.try_write32(0x100, 0xDEADBEEF)?;
> +//!     let val = pram_win.try_read32(0x100)?;
> +//!     assert_eq!(val, 0xDEADBEEF);
> +//!
> +//!     Ok(())
> +//!     // Original window position restored on drop.
> +//! }
> +//! ```
> +//!
> +//! ## Auto-repositioning across VRAM regions
> +//!
> +//! ```no_run
> +//! use crate::driver::Bar0;
> +//! use crate::mm::pramin;
> +//! use kernel::devres::Devres;
> +//! use kernel::sync::Arc;
> +//!
> +//! fn example(devres_bar: Arc<Devres<Bar0>>) -> Result<()> {
> +//!     let mut pram_win = pramin::Window::new(devres_bar)?;
> +//!
> +//!     // Access first 1MB region.
> +//!     pram_win.try_write32(0x100, 0x11111111)?;
> +//!
> +//!     // Access at 2MB - window auto-repositions.
> +//!     pram_win.try_write32(0x200000, 0x22222222)?;
> +//!
> +//!     // Back to first region - window repositions again.
> +//!     let val = pram_win.try_read32(0x100)?;
> +//!     assert_eq!(val, 0x11111111);
> +//!
> +//!     Ok(())
> +//! }
> +//! ```
> +
> +#![allow(unused)]
> +
> +use crate::{
> +    driver::Bar0,
> +    regs, //
> +};
> +
> +use kernel::bits::genmask_u64;
> +use kernel::devres::Devres;
> +use kernel::prelude::*;
> +use kernel::ptr::{
> +    Alignable,
> +    Alignment, //
> +};
> +use kernel::sizes::{
> +    SZ_1M,
> +    SZ_64K, //
> +};
> +use kernel::sync::Arc;
> +
> +/// PRAMIN aperture base offset in BAR0.
> +const PRAMIN_BASE: usize = 0x700000;
> +
> +/// PRAMIN aperture size (1MB).
> +const PRAMIN_SIZE: usize = SZ_1M;
> +
> +/// 64KB alignment for window base.
> +const WINDOW_ALIGN: Alignment = Alignment::new::<SZ_64K>();
> +
> +/// Maximum addressable VRAM offset (40-bit address space).
> +///
> +/// The `NV_PBUS_BAR0_WINDOW` register has a 24-bit `window_base` field
> (bits 23:0) that stores +/// bits [39:16] of the target VRAM address.
> This limits the addressable space to 2^40 bytes. +///
> +/// CAST: On 64-bit systems, this fits in usize.
> +const MAX_VRAM_OFFSET: usize = genmask_u64(0..=39) as usize;
> +
> +/// Generate a PRAMIN read accessor.
> +macro_rules! define_pramin_read {
> +    ($name:ident, $ty:ty) => {
> +        #[doc = concat!("Read a `", stringify!($ty), "` from VRAM at
> the given offset.")]
> +        pub(crate) fn $name(&mut self, vram_offset: usize) ->
> Result<$ty> {
> +            // Compute window parameters without bar reference.
> +            let (bar_offset, new_base) =
> +                self.compute_window(vram_offset,
> ::core::mem::size_of::<$ty>())?; +
> +            // Update window base if needed and perform read.
> +            let bar = self.bar.try_access().ok_or(ENODEV)?;
> +            if let Some(base) = new_base {
> +                Self::write_window_base(&bar, base);
> +                self.current_base = base;
> +            }
> +            bar.$name(bar_offset)
> +        }
> +    };
> +}
> +
> +/// Generate a PRAMIN write accessor.
> +macro_rules! define_pramin_write {
> +    ($name:ident, $ty:ty) => {
> +        #[doc = concat!("Write a `", stringify!($ty), "` to VRAM at the
> given offset.")]
> +        pub(crate) fn $name(&mut self, vram_offset: usize, value: $ty)
> -> Result {
> +            // Compute window parameters without bar reference.
> +            let (bar_offset, new_base) =
> +                self.compute_window(vram_offset,
> ::core::mem::size_of::<$ty>())?; +
> +            // Update window base if needed and perform write.
> +            let bar = self.bar.try_access().ok_or(ENODEV)?;
> +            if let Some(base) = new_base {
> +                Self::write_window_base(&bar, base);
> +                self.current_base = base;
> +            }
> +            bar.$name(value, bar_offset)
> +        }
> +    };
> +}
> +
> +/// PRAMIN window for direct VRAM access.
> +///
> +/// The window auto-repositions when accessing VRAM offsets outside the
> current 1MB range. +/// Original window position is saved on creation
> and restored on drop. +pub(crate) struct Window {
> +    bar: Arc<Devres<Bar0>>,
> +    saved_base: usize,
> +    current_base: usize,
> +}
> +
> +impl Window {
> +    /// Create a new PRAMIN window accessor.
> +    ///
> +    /// Saves the current window position for restoration on drop.
> +    pub(crate) fn new(bar: Arc<Devres<Bar0>>) -> Result<Self> {
> +        let bar_access = bar.try_access().ok_or(ENODEV)?;
> +        let saved_base = Self::try_read_window_base(&bar_access)?;
> +
> +        Ok(Self {
> +            bar,
> +            saved_base,
> +            current_base: saved_base,
> +        })
> +    }
> +
> +    /// Read the current window base from the BAR0_WINDOW register.
> +    fn try_read_window_base(bar: &Bar0) -> Result<usize> {
> +        let reg = regs::NV_PBUS_BAR0_WINDOW::read(bar);
> +        let base = u64::from(reg.window_base());
> +        let shifted = base.checked_shl(16).ok_or(EOVERFLOW)?;
> +        shifted.try_into().map_err(|_| EOVERFLOW)
> +    }
> +
> +    /// Write a new window base to the BAR0_WINDOW register.
> +    fn write_window_base(bar: &Bar0, base: usize) {
> +        // CAST:
> +        // - We have guaranteed that the base is within the addressable
> range (40-bits).
> +        // - After >> 16, a 40-bit aligned base becomes 24 bits, which
> fits in u32.
> +        regs::NV_PBUS_BAR0_WINDOW::default()
> +            .set_window_base((base >> 16) as u32)
> +            .write(bar);
> +    }
> +
> +    /// Compute window parameters for a VRAM access.
> +    ///
> +    /// Returns (bar_offset, new_base) where:
> +    /// - bar_offset: The BAR0 offset to use for the access
> +    /// - new_base: Some(base) if window needs repositioning, None
> otherwise
> +    fn compute_window(
> +        &self,
> +        vram_offset: usize,
> +        access_size: usize,
> +    ) -> Result<(usize, Option<usize>)> {
> +        // Validate VRAM offset is within addressable range (40-bit
> address space).
> +        let end_offset =
> vram_offset.checked_add(access_size).ok_or(EINVAL)?;
> +        if end_offset > MAX_VRAM_OFFSET + 1 {
> +            return Err(EINVAL);
> +        }
> +
> +        // Calculate which 64KB-aligned base we need.
> +        let needed_base = vram_offset.align_down(WINDOW_ALIGN);
> +
> +        // Calculate offset within the window.
> +        let offset_in_window = vram_offset - needed_base;
> +
> +        // Check if access fits in 1MB window from this base.
> +        if offset_in_window + access_size > PRAMIN_SIZE {
> +            return Err(EINVAL);
> +        }
> +
> +        // Return bar offset and whether window needs repositioning.
> +        let new_base = if self.current_base != needed_base {
> +            Some(needed_base)
> +        } else {
> +            None
> +        };
> +
> +        Ok((PRAMIN_BASE + offset_in_window, new_base))
> +    }
> +
> +    define_pramin_read!(try_read8, u8);
> +    define_pramin_read!(try_read16, u16);
> +    define_pramin_read!(try_read32, u32);
> +    define_pramin_read!(try_read64, u64);
> +
> +    define_pramin_write!(try_write8, u8);
> +    define_pramin_write!(try_write16, u16);
> +    define_pramin_write!(try_write32, u32);
> +    define_pramin_write!(try_write64, u64);
> +}
> +
> +impl Drop for Window {
> +    fn drop(&mut self) {
> +        // Restore the original window base if it changed.
> +        if self.current_base != self.saved_base {
> +            if let Some(bar) = self.bar.try_access() {
> +                Self::write_window_base(&bar, self.saved_base);
> +            }
> +        }
> +    }
> +}
> +
> +// SAFETY: `Window` requires `&mut self` for all accessors.
> +unsafe impl Send for Window {}
> +
> +// SAFETY: `Window` requires `&mut self` for all accessors.
> +unsafe impl Sync for Window {}
> diff --git a/drivers/gpu/nova-core/nova_core.rs
> b/drivers/gpu/nova-core/nova_core.rs index c1121e7c64c5..3de00db3279e
> 100644 --- a/drivers/gpu/nova-core/nova_core.rs
> +++ b/drivers/gpu/nova-core/nova_core.rs
> @@ -13,6 +13,7 @@
>  mod gfw;
>  mod gpu;
>  mod gsp;
> +mod mm;
>  mod num;
>  mod regs;
>  mod sbuffer;
> diff --git a/drivers/gpu/nova-core/regs.rs
> b/drivers/gpu/nova-core/regs.rs index 82cc6c0790e5..c8b8fbdcf608 100644
> --- a/drivers/gpu/nova-core/regs.rs
> +++ b/drivers/gpu/nova-core/regs.rs
> @@ -96,6 +96,11 @@ fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) ->
> kernel::fmt::Result { 31:16   frts_err_code as u16;
>  });
>  
> +register!(NV_PBUS_BAR0_WINDOW @ 0x00001700, "BAR0 window control for
> PRAMIN access" {
> +    25:24   target as u8, "Target memory (0=VRAM, 1=SYS_MEM_COH,
> 2=SYS_MEM_NONCOH)";
> +    23:0    window_base as u32, "Window base address (bits 39:16 of FB
> addr)"; +});
> +
>  // PFB
>  
>  // The following two registers together hold the physical system memory
> address that is used by the