Re: [PATCH 29/31] gpu: nova-core: Hopper/Blackwell: add FSP Chain of Trust boot path

["Gary Guo" <gary@garyguo.net>]

On Wed Dec 3, 2025 at 5:59 AM GMT, John Hubbard wrote:
> Add the FSP-based boot path for Hopper and Blackwell GPUs. Unlike
> Turing/Ampere/Ada which use SEC2 to load the booter firmware, Hopper
> and Blackwell use FSP (Firmware System Processor) with FMC firmware
> to establish a Chain of Trust and boot GSP directly.
>
> The boot() function now dispatches to either run_booter() (SEC2 path)
> or run_fsp() (FSP path) based on the GPU architecture. The cmdq
> commands are moved to after GSP boot, and the GSP sequencer is only
> run for SEC2-based architectures.
>
> Signed-off-by: John Hubbard <jhubbard@nvidia.com>
> ---
>  drivers/gpu/nova-core/firmware/fsp.rs |   6 +-
>  drivers/gpu/nova-core/fsp.rs          |   6 +-
>  drivers/gpu/nova-core/gsp/boot.rs     | 159 ++++++++++++++++++++------
>  3 files changed, 126 insertions(+), 45 deletions(-)
>
> diff --git a/drivers/gpu/nova-core/firmware/fsp.rs b/drivers/gpu/nova-core/firmware/fsp.rs
> index 80401b964488..d88c7a91e2bc 100644
> --- a/drivers/gpu/nova-core/firmware/fsp.rs
> +++ b/drivers/gpu/nova-core/firmware/fsp.rs
> @@ -13,16 +13,14 @@
>      gpu::Chipset, //
>  };
>  
> -#[expect(unused)]
>  pub(crate) struct FspFirmware {
>      /// FMC firmware image data (only the .image section)
> -    fmc_image: DmaObject,
> +    pub(crate) fmc_image: DmaObject,
>      /// Full FMC ELF data (for signature extraction)
> -    fmc_full: DmaObject,
> +    pub(crate) fmc_full: DmaObject,
>  }
>  
>  impl FspFirmware {
> -    #[expect(unused)]
>      pub(crate) fn new(
>          dev: &device::Device<device::Bound>,
>          chipset: Chipset,
> diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs
> index 7d46fbcc7abd..9c11ceb6ab4d 100644
> --- a/drivers/gpu/nova-core/fsp.rs
> +++ b/drivers/gpu/nova-core/fsp.rs
> @@ -1,8 +1,5 @@
>  // SPDX-License-Identifier: GPL-2.0
>  
> -// TODO: remove this once the code is fully functional
> -#![expect(dead_code)]
> -
>  //! FSP (Firmware System Processor) interface for Hopper/Blackwell GPUs.
>  //!
>  //! Hopper/Blackwell use a simplified firmware boot sequence: FMC --> FSP --> GSP.
> @@ -11,6 +8,7 @@
>  
>  use kernel::{
>      device,
> +    dma::CoherentAllocation,
>      io::poll::read_poll_timeout,
>      prelude::*,
>      ptr::{
> @@ -381,8 +379,6 @@ pub(crate) fn create_fmc_boot_params(
>          wpr_meta_size: u32,
>          libos_addr: u64,
>      ) -> Result<kernel::dma::CoherentAllocation<GspFmcBootParams>> {
> -        use kernel::dma::CoherentAllocation;
> -
>          const GSP_DMA_TARGET_COHERENT_SYSTEM: u32 = 1;
>          const GSP_DMA_TARGET_NONCOHERENT_SYSTEM: u32 = 2;
>  
> diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs
> index 4d04135a700e..0fbaa73eb55c 100644
> --- a/drivers/gpu/nova-core/gsp/boot.rs
> +++ b/drivers/gpu/nova-core/gsp/boot.rs
> @@ -13,6 +13,7 @@
>  use crate::{
>      driver::Bar0,
>      falcon::{
> +        fsp::Fsp as FspEngine,
>          gsp::Gsp,
>          sec2::Sec2,
>          Falcon,
> @@ -24,6 +25,7 @@
>              BooterFirmware,
>              BooterKind, //
>          },
> +        fsp::FspFirmware,
>          fwsec::{
>              FwsecCommand,
>              FwsecFirmware, //
> @@ -31,9 +33,11 @@
>          gsp::GspFirmware,
>          FIRMWARE_VERSION, //
>      },
> -    gpu::Chipset,
> +    fsp::Fsp,
> +    gpu::{Architecture, Chipset},
>      gsp::{
>          commands,
> +        fw::LibosMemoryRegionInitArgument,
>          sequencer::{
>              GspSequencer,
>              GspSequencerParams, //
> @@ -155,6 +159,59 @@ fn run_booter(
>          Ok(())
>      }
>  
> +    fn run_fsp(
> +        dev: &device::Device<device::Bound>,
> +        bar: &Bar0,
> +        chipset: Chipset,
> +        gsp_falcon: &Falcon<Gsp>,
> +        wpr_meta: &CoherentAllocation<GspFwWprMeta>,
> +        libos: &CoherentAllocation<LibosMemoryRegionInitArgument>,
> +        fb_layout: &FbLayout,
> +    ) -> Result {
> +        let fsp_falcon = Falcon::<FspEngine>::new(dev, chipset)?;
> +
> +        Fsp::wait_secure_boot(dev, bar, chipset.arch())?;
> +
> +        let fsp_fw = FspFirmware::new(dev, chipset, FIRMWARE_VERSION)?;
> +
> +        // SAFETY: fmc_full is a valid DmaObject with a contiguous allocation of size() bytes
> +        // starting at start_ptr(). The slice is only used for signature extraction within this
> +        // function scope while fsp_fw remains valid.
> +        let fmc_full_data = unsafe {
> +            core::slice::from_raw_parts(fsp_fw.fmc_full.start_ptr(), fsp_fw.fmc_full.size())
> +        };

The justification is week because it does not mention about the non-race nature
of this, which need to be justified for a DMA allocation. If you use
`CoherentAllocation::as_slice`, then this requirement would be obvious.

For example:

    // SAFETY: the dma buffer is not yet submitted too hardware and we are the
    // unique owner at this point.
    let fmc_full_data = unsafe { fsp_fw.fmc_full.as_slice(0, fsp_fw.fmc_full.size()) };

Best,
Gary

> +        let signatures = Fsp::extract_fmc_signatures_static(dev, fmc_full_data)?;
> +
> +        // Create FMC boot parameters
> +        let fmc_boot_params = Fsp::create_fmc_boot_params(
> +            dev,
> +            wpr_meta.dma_handle(),
> +            core::mem::size_of::<GspFwWprMeta>() as u32,
> +            libos.dma_handle(),
> +        )?;
> +
> +        // Execute FSP Chain of Trust
> +        // NOTE: FSP Chain of Trust handles GSP boot internally - we do NOT reset or boot GSP
> +        Fsp::boot_gsp_fmc_with_signatures(
> +            dev,
> +            bar,
> +            chipset,
> +            &fsp_fw.fmc_image,
> +            &fmc_boot_params,
> +            u64::from(fb_layout.total_reserved_size),
> +            false, // not resuming
> +            &fsp_falcon,
> +            &signatures,
> +        )?;
> +
> +        // Wait for GSP lockdown to be released
> +        let fmc_boot_params_addr = fmc_boot_params.dma_handle();
> +        let _mbox0 =
> +            Self::wait_for_gsp_lockdown_release(dev, bar, gsp_falcon, fmc_boot_params_addr)?;
> +
> +        Ok(())
> +    }
> +
>      /// Check if GSP lockdown has been released after FSP Chain of Trust
>      fn gsp_lockdown_released(
>          dev: &device::Device,
> @@ -192,7 +249,6 @@ fn gsp_lockdown_released(
>      }
>  
>      /// Wait for GSP lockdown to be released after FSP Chain of Trust
> -    #[expect(dead_code)]
>      fn wait_for_gsp_lockdown_release(
>          dev: &device::Device,
>          bar: &Bar0,
> @@ -255,8 +311,6 @@ pub(crate) fn boot(
>      ) -> Result {
>          let dev = pdev.as_ref();
>  
> -        let bios = Vbios::new(dev, bar)?;
> -
>          let gsp_fw = KBox::pin_init(
>              GspFirmware::new(dev, chipset, FIRMWARE_VERSION)?,
>              GFP_KERNEL,
> @@ -265,36 +319,58 @@ pub(crate) fn boot(
>          let fb_layout = FbLayout::new(chipset, bar, &gsp_fw)?;
>          dev_dbg!(dev, "{:#x?}\n", fb_layout);
>  
> -        Self::run_fwsec_frts(dev, gsp_falcon, bar, &bios, &fb_layout)?;
> +        if matches!(
> +            chipset.arch(),
> +            Architecture::Turing | Architecture::Ampere | Architecture::Ada
> +        ) {
> +            let bios = Vbios::new(dev, bar)?;
> +            Self::run_fwsec_frts(dev, gsp_falcon, bar, &bios, &fb_layout)?;
> +        }
>  
>          let wpr_meta =
>              CoherentAllocation::<GspFwWprMeta>::alloc_coherent(dev, 1, GFP_KERNEL | __GFP_ZERO)?;
>          dma_write!(wpr_meta[0] = GspFwWprMeta::new(&gsp_fw, &fb_layout))?;
>  
> -        self.cmdq
> -            .send_command(bar, commands::SetSystemInfo::new(pdev))?;
> -        self.cmdq.send_command(bar, commands::SetRegistry::new())?;
> +        // For SEC2-based architectures, reset GSP and boot it before SEC2
> +        if matches!(
> +            chipset.arch(),
> +            Architecture::Turing | Architecture::Ampere | Architecture::Ada
> +        ) {
> +            gsp_falcon.reset(bar)?;
> +            let libos_handle = self.libos.dma_handle();
> +            let (mbox0, mbox1) = gsp_falcon.boot(
> +                bar,
> +                Some(libos_handle as u32),
> +                Some((libos_handle >> 32) as u32),
> +            )?;
> +            dev_dbg!(
> +                pdev.as_ref(),
> +                "GSP MBOX0: {:#x}, MBOX1: {:#x}\n",
> +                mbox0,
> +                mbox1
> +            );
>  
> -        gsp_falcon.reset(bar)?;
> -        let libos_handle = self.libos.dma_handle();
> -        let (mbox0, mbox1) = gsp_falcon.boot(
> -            bar,
> -            Some(libos_handle as u32),
> -            Some((libos_handle >> 32) as u32),
> -        )?;
> -        dev_dbg!(
> -            pdev.as_ref(),
> -            "GSP MBOX0: {:#x}, MBOX1: {:#x}\n",
> -            mbox0,
> -            mbox1
> -        );
> +            dev_dbg!(
> +                pdev.as_ref(),
> +                "Using SEC2 to load and run the booter_load firmware...\n"
> +            );
> +        }
>  
> -        dev_dbg!(
> -            pdev.as_ref(),
> -            "Using SEC2 to load and run the booter_load firmware...\n"
> -        );
> +        match chipset.arch() {
> +            Architecture::Turing | Architecture::Ampere | Architecture::Ada => {
> +                Self::run_booter(dev, bar, chipset, sec2_falcon, &wpr_meta)?
> +            }
>  
> -        Self::run_booter(dev, bar, chipset, sec2_falcon, &wpr_meta)?;
> +            Architecture::Hopper | Architecture::Blackwell => Self::run_fsp(
> +                dev,
> +                bar,
> +                chipset,
> +                gsp_falcon,
> +                &wpr_meta,
> +                &self.libos,
> +                &fb_layout,
> +            )?,
> +        }
>  
>          gsp_falcon.write_os_version(bar, gsp_fw.bootloader.app_version);
>  
> @@ -312,16 +388,27 @@ pub(crate) fn boot(
>              gsp_falcon.is_riscv_active(bar),
>          );
>  
> -        // Create and run the GSP sequencer.
> -        let seq_params = GspSequencerParams {
> -            bootloader_app_version: gsp_fw.bootloader.app_version,
> -            libos_dma_handle: libos_handle,
> -            gsp_falcon,
> -            sec2_falcon,
> -            dev: pdev.as_ref().into(),
> -            bar,
> -        };
> -        GspSequencer::run(&mut self.cmdq, seq_params)?;
> +        // Now that GSP is active, send system info and registry
> +        self.cmdq
> +            .send_command(bar, commands::SetSystemInfo::new(pdev))?;
> +        self.cmdq.send_command(bar, commands::SetRegistry::new())?;
> +
> +        if matches!(
> +            chipset.arch(),
> +            Architecture::Turing | Architecture::Ampere | Architecture::Ada
> +        ) {
> +            let libos_handle = self.libos.dma_handle();
> +            // Create and run the GSP sequencer.
> +            let seq_params = GspSequencerParams {
> +                bootloader_app_version: gsp_fw.bootloader.app_version,
> +                libos_dma_handle: libos_handle,
> +                gsp_falcon,
> +                sec2_falcon,
> +                dev: pdev.as_ref().into(),
> +                bar,
> +            };
> +            GspSequencer::run(&mut self.cmdq, seq_params)?;
> +        }
>  
>          // Wait until GSP is fully initialized.
>          commands::wait_gsp_init_done(&mut self.cmdq)?;