Re: [PATCH v13 6/9] gpu: nova-core: Hopper/Blackwell: add FSP Chain of Trust boot

["Eliot Courtney" <ecourtney@nvidia.com>]

On Wed Jun 3, 2026 at 4:30 PM JST, Alexandre Courbot wrote:
> From: John Hubbard <jhubbard@nvidia.com>
>
> Build and send the Chain of Trust message to FSP, bundling the
> DMA-coherent boot parameters that FSP reads at boot time.
>
> Signed-off-by: John Hubbard <jhubbard@nvidia.com>
> Co-developed-by: Alexandre Courbot <acourbot@nvidia.com>
> Signed-off-by: Alexandre Courbot <acourbot@nvidia.com>
> ---
>  drivers/gpu/nova-core/firmware/fsp.rs             |   2 -
>  drivers/gpu/nova-core/fsp.rs                      | 145 +++++++++++++++++++++-
>  drivers/gpu/nova-core/fsp/hal.rs                  |   1 -
>  drivers/gpu/nova-core/gsp.rs                      |   1 +
>  drivers/gpu/nova-core/gsp/fw.rs                   |  65 ++++++++++
>  drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs |  82 ++++++++++++
>  drivers/gpu/nova-core/gsp/hal/gh100.rs            |  23 +++-
>  drivers/gpu/nova-core/mctp.rs                     |   2 -
>  8 files changed, 308 insertions(+), 13 deletions(-)
>
> diff --git a/drivers/gpu/nova-core/firmware/fsp.rs b/drivers/gpu/nova-core/firmware/fsp.rs
> index 9b211426a75a..6eaf1c684b9d 100644
> --- a/drivers/gpu/nova-core/firmware/fsp.rs
> +++ b/drivers/gpu/nova-core/firmware/fsp.rs
> @@ -39,10 +39,8 @@ pub(crate) struct FmcSignatures {
>  
>  pub(crate) struct FspFirmware {
>      /// FMC firmware image data (only the "image" ELF section).
> -    #[expect(dead_code)]
>      pub(crate) fmc_image: Coherent<[u8]>,
>      /// FMC firmware signatures.
> -    #[expect(dead_code)]
>      pub(crate) fmc_sigs: KBox<FmcSignatures>,
>  }
>  
> diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs
> index cedfea173e50..883ac4f8b811 100644
> --- a/drivers/gpu/nova-core/fsp.rs
> +++ b/drivers/gpu/nova-core/fsp.rs
> @@ -9,8 +9,14 @@
>  
>  use kernel::{
>      device,
> +    dma::Coherent,
>      io::poll::read_poll_timeout,
>      prelude::*,
> +    ptr::{
> +        Alignable,
> +        Alignment, //
> +    },
> +    sizes::SZ_2M,
>      time::Delta,
>      transmute::{
>          AsBytes,
> @@ -24,13 +30,19 @@
>          fsp::Fsp as FspEngine,
>          Falcon, //
>      },
> -    firmware::fsp::FspFirmware,
> +    fb::FbLayout,
> +    firmware::fsp::{
> +        FmcSignatures,
> +        FspFirmware, //
> +    },
>      gpu::Chipset,
> +    gsp::GspFmcBootParams,
>      mctp::{
>          MctpHeader,
>          NvdmHeader,
>          NvdmType, //
>      },
> +    num,
>      regs, //
>  };
>  
> @@ -66,6 +78,114 @@ pub(crate) trait MessageToFsp: AsBytes {
>      const NVDM_TYPE: NvdmType;
>  }
>  
> +/// NVDM (NVIDIA Data Model) CoT (Chain of Trust) payload, the main
> +/// message body sent to FSP for Chain of Trust boot.
> +#[repr(C, packed)]
> +#[derive(Clone, Copy, Zeroable)]
> +struct NvdmPayloadCot {
> +    version: u16,
> +    size: u16,
> +    gsp_fmc_sysmem_offset: u64,
> +    frts_sysmem_offset: u64,
> +    frts_sysmem_size: u32,
> +    frts_vidmem_offset: u64,
> +    frts_vidmem_size: u32,
> +    sigs: FmcSignatures,
> +    gsp_boot_args_sysmem_offset: u64,
> +}
> +
> +/// Complete FSP message structure with MCTP and NVDM headers.
> +#[repr(C)]
> +#[derive(Clone, Copy)]
> +struct FspMessage {
> +    mctp_header: MctpHeader,
> +    nvdm_header: NvdmHeader,
> +    cot: NvdmPayloadCot,
> +}
> +
> +impl FspMessage {
> +    /// Returns an in-place initializer for [`FspMessage`].
> +    fn new<'a>(
> +        fb_layout: &FbLayout,
> +        fsp_fw: &'a FspFirmware,
> +        args: &'a FmcBootArgs,
> +    ) -> Result<impl Init<Self> + 'a> {
> +        // frts_offset is relative to FB end: FRTS_location = FB_END - frts_offset
> +        let frts_offset = if !args.resume {
> +            let frts_reserved_size = fb_layout.heap.len() + u64::from(fb_layout.pmu_reserved_size);
> +
> +            frts_reserved_size
> +                .align_up(Alignment::new::<SZ_2M>())
> +                .ok_or(EINVAL)?

Why is `frts_offset` calculated using heap and pmu_reserved_size rather
than looking at `fb_layout.frts`? If `fb_layout.frts` is the location we
are meant to have the frts in then why use a (potentially) different
location?

> +        } else {
> +            0
> +        };
> +
> +        let frts_size: u32 = if !args.resume {
> +            fb_layout.frts.len().try_into()?
> +        } else {
> +            0
> +        };
> +
> +        let version = hal::fsp_hal(args.chipset).ok_or(ENOTSUPP)?.cot_version();
> +        let size = num::usize_into_u16::<{ core::mem::size_of::<NvdmPayloadCot>() }>();
> +
> +        Ok(init!(Self {
> +            mctp_header: MctpHeader::single_packet(),
> +            nvdm_header: NvdmHeader::new(NvdmType::Cot),
> +            // The payload is packed, so we cannot use `init!`. Initialize it member-by-member using
> +            // `chain`.
> +            cot <- pin_init::init_zeroed(),
> +        })
> +        .chain(move |msg| {
> +            msg.cot.version = version;
> +            msg.cot.size = size;
> +            msg.cot.gsp_fmc_sysmem_offset = fsp_fw.fmc_image.dma_handle();
> +            msg.cot.frts_vidmem_offset = frts_offset;
> +            msg.cot.frts_vidmem_size = frts_size;
> +            msg.cot.gsp_boot_args_sysmem_offset = args.fmc_boot_params.dma_handle();
> +            msg.cot.sigs = *fsp_fw.fmc_sigs;

I guess we are leaving sysmem frts unset for now since we don't use it
for anything hey

> +
> +            Ok(())
> +        }))
> +    }
> +}
> +
> +// SAFETY: `FspMessage` is `#[repr(C)]` with no padding, so all of its
> +// bytes are initialized.
> +unsafe impl AsBytes for FspMessage {}
> +
> +impl MessageToFsp for FspMessage {
> +    const NVDM_TYPE: NvdmType = NvdmType::Cot;
> +}
> +
> +/// Bundled arguments for FMC boot via FSP Chain of Trust.
> +pub(crate) struct FmcBootArgs {
> +    chipset: Chipset,
> +    fmc_boot_params: Coherent<GspFmcBootParams>,
> +    resume: bool,
> +}
> +
> +impl FmcBootArgs {
> +    /// Builds FMC boot arguments, allocating the DMA-coherent boot parameter
> +    /// structure that FSP will read.
> +    pub(crate) fn new(
> +        dev: &device::Device<device::Bound>,
> +        chipset: Chipset,
> +        wpr_meta_addr: u64,
> +        libos_addr: u64,
> +        resume: bool,
> +    ) -> Result<Self> {
> +        let init = GspFmcBootParams::new(wpr_meta_addr, libos_addr);
> +
> +        Ok(Self {
> +            chipset,
> +            fmc_boot_params: Coherent::<GspFmcBootParams>::init(dev, GFP_KERNEL, init)?,
> +            resume,
> +        })
> +    }
> +}
> +
>  /// FSP interface for Hopper/Blackwell GPUs.
>  ///
>  /// An `Fsp` is produced by [`Fsp::wait_secure_boot`], which only returns once FSP secure boot
> @@ -73,7 +193,6 @@ pub(crate) trait MessageToFsp: AsBytes {
>  /// Chain of Trust boot.
>  pub(crate) struct Fsp {
>      falcon: Falcon<FspEngine>,
> -    #[expect(dead_code)]
>      fsp_fw: FspFirmware,
>  }
>  
> @@ -109,7 +228,6 @@ pub(crate) fn wait_secure_boot(
>      }
>  
>      /// Sends a message to FSP and waits for the response.
> -    #[expect(dead_code)]
>      fn send_sync_fsp<M>(&mut self, dev: &device::Device, bar: &Bar0, msg: &M) -> Result
>      where
>          M: MessageToFsp,
> @@ -170,4 +288,25 @@ fn send_sync_fsp<M>(&mut self, dev: &device::Device, bar: &Bar0, msg: &M) -> Res
>  
>          Ok(())
>      }
> +
> +    /// Boots GSP FMC via FSP Chain of Trust.
> +    ///
> +    /// Builds the CoT message from the pre-configured [`FmcBootArgs`], sends it
> +    /// to FSP, and waits for the response.
> +    pub(crate) fn boot_fmc(
> +        &mut self,
> +        dev: &device::Device<device::Bound>,
> +        bar: &Bar0,
> +        fb_layout: &FbLayout,
> +        args: &FmcBootArgs,
> +    ) -> Result {
> +        dev_dbg!(dev, "Starting FSP boot sequence for {}\n", args.chipset);
> +
> +        let msg = KBox::init(FspMessage::new(fb_layout, &self.fsp_fw, args)?, GFP_KERNEL)?;
> +
> +        self.send_sync_fsp(dev, bar, &*msg)?;
> +
> +        dev_dbg!(dev, "FSP Chain of Trust completed successfully\n");
> +        Ok(())
> +    }
>  }
> diff --git a/drivers/gpu/nova-core/fsp/hal.rs b/drivers/gpu/nova-core/fsp/hal.rs
> index 8aebe1800a64..86c595d70c8e 100644
> --- a/drivers/gpu/nova-core/fsp/hal.rs
> +++ b/drivers/gpu/nova-core/fsp/hal.rs
> @@ -18,7 +18,6 @@ pub(super) trait FspHal {
>      fn fsp_boot_status(&self, bar: &Bar0) -> u32;
>  
>      /// Returns the FSP Chain of Trust protocol version this chipset advertises.
> -    #[expect(dead_code)]
>      fn cot_version(&self) -> u16;
>  }
>  
> diff --git a/drivers/gpu/nova-core/gsp.rs b/drivers/gpu/nova-core/gsp.rs
> index 1885cfa5cb38..69175ca3315c 100644
> --- a/drivers/gpu/nova-core/gsp.rs
> +++ b/drivers/gpu/nova-core/gsp.rs
> @@ -25,6 +25,7 @@
>  mod sequencer;
>  
>  pub(crate) use fw::{
> +    GspFmcBootParams,
>      GspFwWprMeta,
>      LibosParams, //
>  };
> diff --git a/drivers/gpu/nova-core/gsp/fw.rs b/drivers/gpu/nova-core/gsp/fw.rs
> index 0c54e8bf4bb3..4db0cfa4dc4d 100644
> --- a/drivers/gpu/nova-core/gsp/fw.rs
> +++ b/drivers/gpu/nova-core/gsp/fw.rs
> @@ -934,3 +934,68 @@ fn new(cmdq: &Cmdq) -> impl Init<Self> + '_ {
>          })
>      }
>  }
> +
> +#[repr(u32)]
> +pub(crate) enum GspDmaTarget {
> +    #[expect(dead_code)]
> +    LocalFb = bindings::GSP_DMA_TARGET_GSP_DMA_TARGET_LOCAL_FB,
> +    CoherentSystem = bindings::GSP_DMA_TARGET_GSP_DMA_TARGET_COHERENT_SYSTEM,
> +    NoncoherentSystem = bindings::GSP_DMA_TARGET_GSP_DMA_TARGET_NONCOHERENT_SYSTEM,
> +}
> +
> +type GspAcrBootGspRmParams = bindings::GSP_ACR_BOOT_GSP_RM_PARAMS;
> +
> +impl GspAcrBootGspRmParams {
> +    fn new(target: GspDmaTarget, wpr_meta_addr: u64) -> impl Init<Self> {
> +        #[allow(non_snake_case)]
> +        let params = init!(Self {
> +            target: target as u32,
> +            gspRmDescSize: num::usize_into_u32::<{ size_of::<GspFwWprMeta>() }>(),
> +            gspRmDescOffset: wpr_meta_addr,
> +            bIsGspRmBoot: 1,
> +            wprCarveoutOffset: 0,
> +            wprCarveoutSize: 0,
> +            __bindgen_padding_0: Default::default(),
> +        });
> +
> +        params
> +    }
> +}
> +
> +type GspRmParams = bindings::GSP_RM_PARAMS;
> +
> +impl GspRmParams {
> +    fn new(target: GspDmaTarget, libos_addr: u64) -> impl Init<Self> {
> +        #[allow(non_snake_case)]
> +        let params = init!(Self {
> +            target: target as u32,
> +            bootArgsOffset: libos_addr,
> +            __bindgen_padding_0: Default::default(),
> +        });
> +
> +        params
> +    }
> +}
> +
> +pub(crate) type GspFmcBootParams = bindings::GSP_FMC_BOOT_PARAMS;
> +
> +// SAFETY: Padding is explicit and will not contain uninitialized data.
> +unsafe impl AsBytes for GspFmcBootParams {}
> +// SAFETY: This struct only contains integer types for which all bit patterns are valid.
> +unsafe impl FromBytes for GspFmcBootParams {}
> +
> +impl GspFmcBootParams {
> +    pub(crate) fn new(wpr_meta_addr: u64, libos_addr: u64) -> impl Init<Self> {
> +        #[allow(non_snake_case)]
> +        let init = init!(Self {
> +            // Blackwell FSP obtains WPR info from other sources, so
> +            // wprCarveoutOffset and wprCarveoutSize are left zero.
> +            bootGspRmParams <- GspAcrBootGspRmParams::new(GspDmaTarget::CoherentSystem,
> +                wpr_meta_addr),
> +            gspRmParams <- GspRmParams::new(GspDmaTarget::NoncoherentSystem, libos_addr),
> +            ..Zeroable::init_zeroed()
> +        });
> +
> +        init
> +    }
> +}
> diff --git a/drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs b/drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs
> index 1d592bd3f9ed..ea350f9b2cc4 100644
> --- a/drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs
> +++ b/drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs
> @@ -883,6 +883,88 @@ fn default() -> Self {
>          }
>      }
>  }
> +pub const GSP_DMA_TARGET_GSP_DMA_TARGET_LOCAL_FB: GSP_DMA_TARGET = 0;
> +pub const GSP_DMA_TARGET_GSP_DMA_TARGET_COHERENT_SYSTEM: GSP_DMA_TARGET = 1;
> +pub const GSP_DMA_TARGET_GSP_DMA_TARGET_NONCOHERENT_SYSTEM: GSP_DMA_TARGET = 2;
> +pub const GSP_DMA_TARGET_GSP_DMA_TARGET_COUNT: GSP_DMA_TARGET = 3;
> +pub type GSP_DMA_TARGET = ffi::c_uint;
> +#[repr(C)]
> +#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
> +pub struct GSP_FMC_INIT_PARAMS {
> +    pub regkeys: u32_,
> +}
> +#[repr(C)]
> +#[derive(Debug, Copy, Clone, MaybeZeroable)]
> +pub struct GSP_ACR_BOOT_GSP_RM_PARAMS {
> +    pub target: GSP_DMA_TARGET,
> +    pub gspRmDescSize: u32_,
> +    pub gspRmDescOffset: u64_,
> +    pub wprCarveoutOffset: u64_,
> +    pub wprCarveoutSize: u32_,
> +    pub bIsGspRmBoot: u8_,
> +    pub __bindgen_padding_0: [u8; 3usize],
> +}
> +impl Default for GSP_ACR_BOOT_GSP_RM_PARAMS {
> +    fn default() -> Self {
> +        let mut s = ::core::mem::MaybeUninit::<Self>::uninit();
> +        unsafe {
> +            ::core::ptr::write_bytes(s.as_mut_ptr(), 0, 1);
> +            s.assume_init()
> +        }
> +    }
> +}
> +#[repr(C)]
> +#[derive(Debug, Copy, Clone, MaybeZeroable)]
> +pub struct GSP_RM_PARAMS {
> +    pub target: GSP_DMA_TARGET,
> +    pub __bindgen_padding_0: [u8; 4usize],
> +    pub bootArgsOffset: u64_,
> +}
> +impl Default for GSP_RM_PARAMS {
> +    fn default() -> Self {
> +        let mut s = ::core::mem::MaybeUninit::<Self>::uninit();
> +        unsafe {
> +            ::core::ptr::write_bytes(s.as_mut_ptr(), 0, 1);
> +            s.assume_init()
> +        }
> +    }
> +}
> +#[repr(C)]
> +#[derive(Debug, Copy, Clone, MaybeZeroable)]
> +pub struct GSP_SPDM_PARAMS {
> +    pub target: GSP_DMA_TARGET,
> +    pub __bindgen_padding_0: [u8; 4usize],
> +    pub payloadBufferOffset: u64_,
> +    pub payloadBufferSize: u32_,
> +    pub __bindgen_padding_1: [u8; 4usize],
> +}
> +impl Default for GSP_SPDM_PARAMS {
> +    fn default() -> Self {
> +        let mut s = ::core::mem::MaybeUninit::<Self>::uninit();
> +        unsafe {
> +            ::core::ptr::write_bytes(s.as_mut_ptr(), 0, 1);
> +            s.assume_init()
> +        }
> +    }
> +}
> +#[repr(C)]
> +#[derive(Debug, Copy, Clone, MaybeZeroable)]
> +pub struct GSP_FMC_BOOT_PARAMS {
> +    pub initParams: GSP_FMC_INIT_PARAMS,
> +    pub __bindgen_padding_0: [u8; 4usize],
> +    pub bootGspRmParams: GSP_ACR_BOOT_GSP_RM_PARAMS,
> +    pub gspRmParams: GSP_RM_PARAMS,
> +    pub gspSpdmParams: GSP_SPDM_PARAMS,
> +}
> +impl Default for GSP_FMC_BOOT_PARAMS {
> +    fn default() -> Self {
> +        let mut s = ::core::mem::MaybeUninit::<Self>::uninit();
> +        unsafe {
> +            ::core::ptr::write_bytes(s.as_mut_ptr(), 0, 1);
> +            s.assume_init()
> +        }
> +    }
> +}
>  #[repr(C)]
>  #[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
>  pub struct rpc_unloading_guest_driver_v1F_07 {
> diff --git a/drivers/gpu/nova-core/gsp/hal/gh100.rs b/drivers/gpu/nova-core/gsp/hal/gh100.rs
> index b25970dd4561..f41f3fea15ff 100644
> --- a/drivers/gpu/nova-core/gsp/hal/gh100.rs
> +++ b/drivers/gpu/nova-core/gsp/hal/gh100.rs
> @@ -20,7 +20,10 @@
>          fsp::FspFirmware,
>          FIRMWARE_VERSION, //
>      },
> -    fsp::Fsp,
> +    fsp::{
> +        FmcBootArgs,
> +        Fsp, //
> +    },
>      gpu::Chipset,
>      gsp::{
>          boot::BootUnloadGuard,
> @@ -39,17 +42,27 @@ impl GspHal for Gh100 {
>      /// the GSP boot internally - no manual GSP reset/boot is needed.
>      fn boot<'a>(
>          &self,
> -        _gsp: &'a Gsp,
> +        gsp: &'a Gsp,
>          dev: &'a device::Device<device::Bound>,
>          bar: &'a Bar0,
>          chipset: Chipset,
> -        _fb_layout: &FbLayout,
> -        _wpr_meta: &Coherent<GspFwWprMeta>,
> +        fb_layout: &FbLayout,
> +        wpr_meta: &Coherent<GspFwWprMeta>,
>          _gsp_falcon: &'a Falcon<GspEngine>,
>          _sec2_falcon: &'a Falcon<Sec2>,
>      ) -> Result<BootUnloadGuard<'a>> {
>          let fsp_fw = FspFirmware::new(dev, chipset, FIRMWARE_VERSION)?;
> -        let _fsp = Fsp::wait_secure_boot(dev, bar, chipset, fsp_fw)?;
> +        let mut fsp = Fsp::wait_secure_boot(dev, bar, chipset, fsp_fw)?;
> +
> +        let args = FmcBootArgs::new(
> +            dev,
> +            chipset,
> +            wpr_meta.dma_handle(),
> +            gsp.libos.dma_handle(),
> +            false,
> +        )?;
> +
> +        fsp.boot_fmc(dev, bar, fb_layout, &args)?;
>  
>          Err(ENOTSUPP)
>      }
> diff --git a/drivers/gpu/nova-core/mctp.rs b/drivers/gpu/nova-core/mctp.rs
> index 90e289d4c3fe..482786e07bc7 100644
> --- a/drivers/gpu/nova-core/mctp.rs
> +++ b/drivers/gpu/nova-core/mctp.rs
> @@ -7,8 +7,6 @@
>  //! Data Model) messages between the kernel driver and GPU firmware processors
>  //! such as FSP and GSP.
>  
> -#![expect(dead_code)]
> -
>  use kernel::pci::Vendor;
>  
>  /// NVDM message type identifiers carried over MCTP.