Re: [PATCH 23/31] gpu: nova-core: Hopper/Blackwell: add FSP Chain of Trust boot

[Joel Fernandes <joelagnelf@nvidia.com>]

On Tue, Dec 02, 2025 at 09:59:15PM -0800, John Hubbard wrote:
> Add the boot functions that construct FMC boot parameters and send the
> Chain of Trust message to FSP. This completes the FSP communication
> infrastructure needed to boot GSP firmware on Hopper/Blackwell GPUs.
> 
> Signed-off-by: John Hubbard <jhubbard@nvidia.com>
> ---
>  drivers/gpu/nova-core/fsp.rs | 156 +++++++++++++++++++++++++++++++++++
>  drivers/gpu/nova-core/gpu.rs |   1 -
>  2 files changed, 156 insertions(+), 1 deletion(-)
> 
> diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs
> index bb1e19c03c30..5840ab78e79f 100644
> --- a/drivers/gpu/nova-core/fsp.rs
> +++ b/drivers/gpu/nova-core/fsp.rs
> @@ -13,6 +13,10 @@
>      device,
>      io::poll::read_poll_timeout,
>      prelude::*,
> +    ptr::{
> +        Alignable,
> +        Alignment, //
> +    },
>      time::Delta,
>      transmute::{
>          AsBytes,
> @@ -22,6 +26,10 @@
>  
>  use crate::regs::FSP_BOOT_COMPLETE_SUCCESS;
>  
> +/// FSP Chain of Trust (COT) version for Blackwell.
> +/// GB202 uses version 2 (not 1 like GH100)
> +const FSP_COT_VERSION: u16 = 2;

the comment says that Hopper needs this to be version 1 so how does this
series boot on Hopper? maybe we need to add a new function to return a
different version number for Hopper versus Blackwell?

edit: I just noticed in the cover letter that you mentioned this is tested
only on Blackwell. I think perhaps we should drop the claim that the series
works on hopper in the patches / cover letter.

thanks,

 - Joel




> +
>  /// FSP message timeout in milliseconds.
>  const FSP_MSG_TIMEOUT_MS: i64 = 2000;
>  
> @@ -364,6 +372,154 @@ pub(crate) fn extract_fmc_signatures_static(
>          Ok(signatures)
>      }
>  
> +    /// Creates FMC boot parameters structure for FSP.
> +    ///
> +    /// This structure tells FSP how to boot GSP-RM with the correct memory layout.
> +    pub(crate) fn create_fmc_boot_params(
> +        dev: &device::Device<device::Bound>,
> +        wpr_meta_addr: u64,
> +        wpr_meta_size: u32,
> +        libos_addr: u64,
> +    ) -> Result<kernel::dma::CoherentAllocation<GspFmcBootParams>> {
> +        use kernel::dma::CoherentAllocation;
> +
> +        const GSP_DMA_TARGET_COHERENT_SYSTEM: u32 = 1;
> +        const GSP_DMA_TARGET_NONCOHERENT_SYSTEM: u32 = 2;
> +
> +        let fmc_boot_params = CoherentAllocation::<GspFmcBootParams>::alloc_coherent(
> +            dev,
> +            1,
> +            GFP_KERNEL | __GFP_ZERO,
> +        )?;
> +
> +        // Configure ACR boot parameters (WPR metadata location) using dma_write! macro
> +        kernel::dma_write!(
> +            fmc_boot_params[0].boot_gsp_rm_params.target = GSP_DMA_TARGET_COHERENT_SYSTEM
> +        )?;
> +        kernel::dma_write!(
> +            fmc_boot_params[0].boot_gsp_rm_params.gsp_rm_desc_offset = wpr_meta_addr
> +        )?;
> +        kernel::dma_write!(fmc_boot_params[0].boot_gsp_rm_params.gsp_rm_desc_size = wpr_meta_size)?;
> +
> +        // Blackwell FSP expects wpr_carveout_offset and wpr_carveout_size to be zero;
> +        // it obtains WPR info from other sources.
> +
> +        kernel::dma_write!(fmc_boot_params[0].boot_gsp_rm_params.b_is_gsp_rm_boot = 1)?;
> +
> +        // Configure RM parameters (libos location) using dma_write! macro
> +        kernel::dma_write!(
> +            fmc_boot_params[0].gsp_rm_params.target = GSP_DMA_TARGET_NONCOHERENT_SYSTEM
> +        )?;
> +        kernel::dma_write!(fmc_boot_params[0].gsp_rm_params.boot_args_offset = libos_addr)?;
> +
> +        dev_dbg!(
> +            dev,
> +            "FMC Boot Params (addr={:#x}):\n  target={}\n  desc_size={:#x}\n  \
> +             desc_offset={:#x}\n  rm_target={}\n  boot_args_offset={:#x} \
> +             (libos_addr passed in: {:#x})\n",
> +            fmc_boot_params.dma_handle(),
> +            GSP_DMA_TARGET_COHERENT_SYSTEM,
> +            wpr_meta_size,
> +            wpr_meta_addr,
> +            GSP_DMA_TARGET_NONCOHERENT_SYSTEM,
> +            libos_addr,
> +            libos_addr
> +        );
> +
> +        Ok(fmc_boot_params)
> +    }
> +
> +    /// Boot GSP FMC with pre-extracted signatures.
> +    ///
> +    /// This version takes pre-extracted signatures and FMC image data.
> +    /// Used when signatures are extracted separately from the full ELF file.
> +    #[allow(clippy::too_many_arguments)]
> +    pub(crate) fn boot_gsp_fmc_with_signatures(
> +        dev: &device::Device<device::Bound>,
> +        bar: &crate::driver::Bar0,
> +        chipset: crate::gpu::Chipset,
> +        fmc_image_fw: &crate::dma::DmaObject, // Contains only the image section
> +        fmc_boot_params: &kernel::dma::CoherentAllocation<GspFmcBootParams>,
> +        total_reserved_size: u64,
> +        resume: bool,
> +        fsp_falcon: &crate::falcon::Falcon<crate::falcon::fsp::Fsp>,
> +        signatures: &FmcSignatures,
> +    ) -> Result<()> {
> +        dev_dbg!(dev, "Starting FSP boot sequence for {}\n", chipset);
> +
> +        // Build FSP Chain of Trust message
> +        let fmc_addr = fmc_image_fw.dma_handle(); // Now points to image data only
> +        let fmc_boot_params_addr = fmc_boot_params.dma_handle();
> +
> +        // frts_offset is relative to FB end: FRTS_location = FB_END - frts_offset
> +        let frts_offset = if !resume {
> +            let mut frts_reserved_size = if chipset.needs_large_reserved_mem() {
> +                0x220000 // heap_size_non_wpr for Hopper/Blackwell+
> +            } else {
> +                total_reserved_size
> +            };
> +
> +            // Add PMU reserved size
> +            frts_reserved_size += u64::from(crate::fb::PMU_RESERVED_SIZE);
> +
> +            frts_reserved_size
> +                .align_up(Alignment::new::<0x200000>())
> +                .unwrap_or(frts_reserved_size)
> +        } else {
> +            0
> +        };
> +        let frts_size = if !resume { 0x100000 } else { 0 }; // 1MB FRTS size
> +
> +        // Build the FSP message
> +        let msg = KBox::new(
> +            FspMessage {
> +                mctp_header: (mctp::HEADER_SOM << 31)
> +                    | (mctp::HEADER_EOM << 30)
> +                    | (mctp::HEADER_SEID << 16)
> +                    | (mctp::HEADER_SEQ << 28),
> +
> +                nvdm_header: (mctp::MSG_TYPE_VENDOR_PCI)
> +                    | (mctp::VENDOR_ID_NV << 8)
> +                    | (mctp::NVDM_TYPE_COT << 24),
> +
> +                cot: NvdmPayloadCot {
> +                    version: FSP_COT_VERSION,
> +                    size: core::mem::size_of::<NvdmPayloadCot>() as u16,
> +                    gsp_fmc_sysmem_offset: fmc_addr,
> +                    frts_sysmem_offset: 0,
> +                    frts_sysmem_size: 0,
> +                    frts_vidmem_offset: frts_offset,
> +                    frts_vidmem_size: frts_size,
> +                    hash384: signatures.hash384,
> +                    public_key: signatures.public_key,
> +                    signature: signatures.signature,
> +                    gsp_boot_args_sysmem_offset: fmc_boot_params_addr,
> +                },
> +            },
> +            GFP_KERNEL,
> +        )?;
> +
> +        // Convert message to bytes for sending
> +        let msg_bytes = msg.as_bytes();
> +
> +        dev_dbg!(
> +            dev,
> +            "FSP COT Message:\n  size={} bytes\n  fmc_addr={:#x}\n  boot_params={:#x}\n  \
> +             frts_offset={:#x}\n  frts_size={:#x}\n",
> +            msg_bytes.len(),
> +            fmc_addr,
> +            fmc_boot_params_addr,
> +            frts_offset,
> +            frts_size
> +        );
> +
> +        // Send COT message to FSP and wait for response
> +        Self::send_sync_fsp(dev, bar, fsp_falcon, mctp::NVDM_TYPE_COT, msg_bytes)?;
> +
> +        dev_dbg!(dev, "FSP Chain of Trust completed successfully\n");
> +        Ok(())
> +    }
> +
>      /// Send message to FSP and wait for response.
>      fn send_sync_fsp(
>          dev: &device::Device<device::Bound>,
> diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs
> index c0473ef8ac47..8fdce488612a 100644
> --- a/drivers/gpu/nova-core/gpu.rs
> +++ b/drivers/gpu/nova-core/gpu.rs
> @@ -124,7 +124,6 @@ pub(crate) const fn arch(&self) -> Architecture {
>          }
>      }
>  
> -    #[expect(dead_code)]
>      pub(crate) fn needs_large_reserved_mem(&self) -> bool {
>          matches!(self.arch(), Architecture::Hopper | Architecture::Blackwell)
>      }
> -- 
> 2.52.0
>