NVIDIA GPU driver infrastructure
 help / color / mirror / Atom feed
From: Zhi Wang <zhiw@nvidia.com>
To: <dakr@kernel.org>, <acourbot@nvidia.com>
Cc: <alex@shazbot.org>, <jgg@nvidia.com>, <yishaih@nvidia.com>,
	<skolothumtho@nvidia.com>, <kevin.tian@intel.com>,
	<airlied@gmail.com>, <simona@ffwll.ch>, <ojeda@kernel.org>,
	<alex.gaynor@gmail.com>, <boqun.feng@gmail.com>,
	<gary@garyguo.net>, <bjorn3_gh@protonmail.com>,
	<lossin@kernel.org>, <a.hindborg@kernel.org>,
	<aliceryhl@google.com>, <tmgross@umich.edu>,
	<jhubbard@nvidia.com>, <ecourtney@nvidia.com>, <cjia@nvidia.com>,
	<smitra@nvidia.com>, <kjaju@nvidia.com>, <alkumar@nvidia.com>,
	<ankita@nvidia.com>, <aniketa@nvidia.com>, <kwankhede@nvidia.com>,
	<targupta@nvidia.com>, <nova-gpu@lists.linux.dev>,
	<linux-kernel@vger.kernel.org>, <zhiwang@kernel.org>,
	Zhi Wang <zhiw@nvidia.com>
Subject: [PATCH 07/13] gpu: nova-core: vgpu: add vGPU bootload
Date: Sat, 5 Sep 2026 11:11:10 +0300	[thread overview]
Message-ID: <20260905081116.106613-8-zhiw@nvidia.com> (raw)
In-Reply-To: <20260905081116.106613-1-zhiw@nvidia.com>

Implement the GMCAPI VGPU_BOOTLOAD command that boots the GSP plugin
for a vGPU instance, together with the VGPU_SHUTDOWN and VGPU_CLEANUP
teardown sequence.

Encode the typed channel map, framebuffer, management heap and log
locations, then poll the PluginRpc BAR1 marker for boot completion.

Co-developed-by: Alok Kumar <alkumar@nvidia.com>
Signed-off-by: Alok Kumar <alkumar@nvidia.com>
Signed-off-by: Zhi Wang <zhiw@nvidia.com>
---
 drivers/gpu/nova-core/gsp/commands.rs    |   2 +
 drivers/gpu/nova-core/gsp/fw.rs          |  18 ++-
 drivers/gpu/nova-core/gsp/fw/commands.rs |  60 +++++++
 drivers/gpu/nova-core/mm/vram.rs         |   3 -
 drivers/gpu/nova-core/vgpu/bootload.rs   | 162 +++++++++++++++++++
 drivers/gpu/nova-core/vgpu/consts.rs     |   8 +
 drivers/gpu/nova-core/vgpu/fw.rs         | 193 +++++++++++++++++++++++
 drivers/gpu/nova-core/vgpu/instance.rs   | 100 ++++++++++--
 drivers/gpu/nova-core/vgpu/mod.rs        |   2 +
 drivers/gpu/nova-core/vgpu/plugin_rpc.rs |  65 ++++++++
 drivers/gpu/nova-core/vgpu/vram.rs       |   2 -
 11 files changed, 598 insertions(+), 17 deletions(-)
 create mode 100644 drivers/gpu/nova-core/vgpu/bootload.rs
 create mode 100644 drivers/gpu/nova-core/vgpu/plugin_rpc.rs

diff --git a/drivers/gpu/nova-core/gsp/commands.rs b/drivers/gpu/nova-core/gsp/commands.rs
index bdb358a19738..481ec8e221ce 100644
--- a/drivers/gpu/nova-core/gsp/commands.rs
+++ b/drivers/gpu/nova-core/gsp/commands.rs
@@ -54,6 +54,8 @@
 };
 
 pub(crate) use fw::commands::{
+    encode_vgpu_bootload,
+    ChannelMapEntry,
     Dbdf,
     VgpuProperties, //
 };
diff --git a/drivers/gpu/nova-core/gsp/fw.rs b/drivers/gpu/nova-core/gsp/fw.rs
index f8c6fc5ea8a7..d68b790533af 100644
--- a/drivers/gpu/nova-core/gsp/fw.rs
+++ b/drivers/gpu/nova-core/gsp/fw.rs
@@ -7,8 +7,24 @@
 /// Raw firmware declarations used by vGPU management.
 pub(crate) mod vgpu_bindings {
     pub(crate) use super::r000_00::{
+        GMCAPI_COMMANDS_GMCAPI_CMD_BOOTLOAD_GSP_VGPU_PLUGIN_TASK,
+        GMCAPI_COMMANDS_GMCAPI_CMD_CLEANUP_GSP_VGPU_PLUGIN_RESOURCES,
         GMCAPI_COMMANDS_GMCAPI_CMD_QUERY_ASSIGNED_VF_VGPU_TYPE,
-        GMCAPI_COMMANDS_GMCAPI_CMD_QUERY_VGPU_PROPERTIES, //
+        GMCAPI_COMMANDS_GMCAPI_CMD_QUERY_VGPU_PROPERTIES,
+        GMCAPI_COMMANDS_GMCAPI_CMD_SHUTDOWN_GSP_VGPU_PLUGIN_TASK,
+        GMCAPI_COMMANDS_GMCAPI_CMD_SHUTDOWN_GSP_VGPU_PLUGIN_TASK_COMPLETE,
+        GSP_PLUGIN_BOOTLOADED,
+        VGPU_CPU_GSP_COMMUNICATION_BUFF_TOTAL_SIZE,
+        VGPU_CPU_GSP_CTRL_BUFF_REGION,
+        VGPU_CPU_GSP_CTRL_BUFF_REGION_SIZE,
+        VGPU_CPU_GSP_ERROR_BUFF_REGION_SIZE,
+        VGPU_CPU_GSP_GUEST_RPC_TRACE_BUFF_REGION_SIZE,
+        VGPU_CPU_GSP_INIT_TASK_LOG_BUFF_REGION_SIZE,
+        VGPU_CPU_GSP_KERNEL_TASK_LOG_BUFF_REGION_SIZE,
+        VGPU_CPU_GSP_MESSAGE_BUFF_REGION_SIZE,
+        VGPU_CPU_GSP_MIGRATION_BUFF_REGION_SIZE,
+        VGPU_CPU_GSP_RESPONSE_BUFF_REGION_SIZE,
+        VGPU_CPU_GSP_VGPU_TASK_LOG_BUFF_REGION_SIZE, //
     };
 }
 
diff --git a/drivers/gpu/nova-core/gsp/fw/commands.rs b/drivers/gpu/nova-core/gsp/fw/commands.rs
index 0603fbde172f..fe4a7af88ec7 100644
--- a/drivers/gpu/nova-core/gsp/fw/commands.rs
+++ b/drivers/gpu/nova-core/gsp/fw/commands.rs
@@ -537,6 +537,13 @@ pub(crate) struct ChannelMapEntry(u64) {
 
 impl ChannelMapEntry {
     const KEY: KeyId = 0x1001;
+
+    pub(crate) fn new(engine_type: usize, index: u32, chid_offset: u32) -> Result<Self> {
+        Self::zeroed()
+            .try_with_engine_type(u64::try_from(engine_type).map_err(|_| EOVERFLOW)?)
+            .and_then(|entry| entry.try_with_index(u64::from(index)))
+            .and_then(|entry| entry.try_with_chid_offset(u64::from(chid_offset)))
+    }
 }
 
 impl Encodeable for KVVec<ChannelMapEntry> {
@@ -611,6 +618,59 @@ impl VgpuBootloadRequest {
     const MIG_RM_HEAP_LENGTH_KEY: KeyId = 0x100E;
 }
 
+/// Encodes a `VGPU_BOOTLOAD` request using the typed NVKV schema.
+#[expect(clippy::too_many_arguments)]
+pub(crate) fn encode_vgpu_bootload(
+    dbdf: Dbdf,
+    gfid: u32,
+    vgpu_type: u32,
+    vm_pid: u32,
+    num_channels: u32,
+    num_plugin_channels: u32,
+    channel_mapping: KVVec<ChannelMapEntry>,
+    guest_fb_address: u64,
+    guest_fb_length: u64,
+    plugin_heap_address: u64,
+    plugin_heap_length: u64,
+    ctrl_buffer_offset: u64,
+    init_log_address: u64,
+    init_log_size: u64,
+    vgpu_log_address: u64,
+    vgpu_log_size: u64,
+    kernel_log_address: u64,
+    kernel_log_size: u64,
+) -> Result<KVVec<u64>> {
+    let request = VgpuBootloadRequest {
+        dbdf: dbdf.into(),
+        gfid: gfid.into(),
+        vgpu_type: vgpu_type.into(),
+        vm_pid: vm_pid.into(),
+        swizz_id: SwizzId::WHOLE_GPU.into(),
+        num_channels: num_channels.into(),
+        num_plugin_channels: num_plugin_channels.into(),
+        guest_fb_segment_count: 1.into(),
+        options: VgpuBootloadOptions::zeroed().into(),
+        channel_mapping,
+        guest_fb_segment_phys_addr: Array::new(&[guest_fb_address])?,
+        guest_fb_segment_length: Array::new(&[guest_fb_length])?,
+        plugin_heap_phys_addr: plugin_heap_address.into(),
+        plugin_heap_length: plugin_heap_length.into(),
+        ctrl_buff_offset: ctrl_buffer_offset.into(),
+        init_task_log_offset: init_log_address.into(),
+        init_task_log_size: init_log_size.into(),
+        vgpu_task_log_offset: vgpu_log_address.into(),
+        vgpu_task_log_size: vgpu_log_size.into(),
+        kernel_log_offset: kernel_log_address.into(),
+        kernel_log_size: kernel_log_size.into(),
+        mig_rm_heap_phys_addr: 0.into(),
+        mig_rm_heap_length: 0.into(),
+    };
+
+    let mut encoder = Encoder::new();
+    request.encode(&mut encoder)?;
+    Ok(encoder.finish())
+}
+
 // VGPU_MGMT_QUERY_PROPERTIES
 
 nvkv_decode! {
diff --git a/drivers/gpu/nova-core/mm/vram.rs b/drivers/gpu/nova-core/mm/vram.rs
index 87b7ce7f2c93..4a4bd42c9f18 100644
--- a/drivers/gpu/nova-core/mm/vram.rs
+++ b/drivers/gpu/nova-core/mm/vram.rs
@@ -86,19 +86,16 @@ fn new(backing: Arc<VramBlock>, range: Range<u64>) -> Result<Self> {
     }
 
     /// Return the physical address of the first byte in this region.
-    #[expect(dead_code)]
     pub(crate) const fn address(&self) -> u64 {
         self.address
     }
 
     /// Return the region size in bytes.
-    #[expect(dead_code)]
     pub(crate) const fn size(&self) -> u64 {
         self.size
     }
 
     /// Return a checked subregion relative to this region.
-    #[expect(dead_code)]
     pub(crate) fn subregion(&self, range: Range<u64>) -> Result<Self> {
         let size = range
             .end
diff --git a/drivers/gpu/nova-core/vgpu/bootload.rs b/drivers/gpu/nova-core/vgpu/bootload.rs
new file mode 100644
index 000000000000..0d382af46158
--- /dev/null
+++ b/drivers/gpu/nova-core/vgpu/bootload.rs
@@ -0,0 +1,162 @@
+// SPDX-License-Identifier: GPL-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+use kernel::{
+    device,
+    prelude::*,
+    time::Delta,
+    transmute::AsBytes, //
+};
+
+use crate::{
+    driver::Bar0,
+    gsp::{
+        cmdq::Cmdq,
+        commands::{
+            encode_vgpu_bootload,
+            ChannelMapEntry,
+            FifoEngineList, //
+        },
+    },
+    vgpu::consts::gmc, //
+};
+
+use super::instance::{
+    Gfid,
+    VgpuInstance, //
+};
+
+/// Build the typed channel mapping from the GSP FIFO engine list.
+fn channel_mapping(
+    fifo_engine_list: &FifoEngineList,
+    chid_offset: u32,
+) -> Result<KVVec<ChannelMapEntry>> {
+    let mut mapping = KVVec::new();
+    for &gmc_id in &fifo_engine_list.gmc_ids[..fifo_engine_list.count] {
+        let engine_type = (gmc_id & 0xffff) as usize;
+        let index = gmc_id >> 16;
+        mapping.push(
+            ChannelMapEntry::new(engine_type, index, chid_offset)?,
+            GFP_KERNEL,
+        )?;
+    }
+    Ok(mapping)
+}
+
+/// Bootload the GSP vGPU plugin and wait for its BAR1 ready indication.
+pub(crate) fn bootload(
+    dev: &device::Device<device::Bound>,
+    cmdq: &Cmdq,
+    bar: Bar0<'_>,
+    instance: &VgpuInstance<'_>,
+    fifo_engine_list: &FifoEngineList,
+) -> Result {
+    let fb = &instance.vram_slot.fbmem;
+    let mgmt = &instance.vram_slot.mgmt_heap;
+    let logs = instance.plugin_rpc.plugin_logs()?;
+
+    let payload = encode_vgpu_bootload(
+        instance.dbdf,
+        instance.gfid.0,
+        instance.vgpu_type.vgpu_type_id(),
+        instance.vm_pid,
+        u32::try_from(instance.chids.len()).map_err(|_| EOVERFLOW)?,
+        instance.num_plugin_channels,
+        channel_mapping(
+            fifo_engine_list,
+            u32::try_from(instance.chids.start).map_err(|_| EOVERFLOW)?,
+        )?,
+        fb.address(),
+        fb.size(),
+        mgmt.address(),
+        mgmt.size(),
+        0,
+        logs.init().address(),
+        logs.init().size(),
+        logs.vgpu().address(),
+        logs.vgpu().size(),
+        logs.kernel().address(),
+        logs.kernel().size(),
+    )?;
+
+    dev_dbg!(
+        dev,
+        "bootload: gfid={} sending {} typed NVKV bytes\n",
+        instance.gfid.0,
+        payload.len() * size_of::<u64>(),
+    );
+
+    // BOOTLOAD completes synchronously. The receive path dispatches any RM RPC
+    // frames that arrive before matching the response by command and sequence.
+    let response = cmdq.send_gmc_and_receive_timeout(
+        bar,
+        gmc::BOOTLOAD,
+        AsBytes::as_bytes(payload.as_slice()),
+        0,
+        Delta::from_secs(10),
+    )?;
+    if response.status != 0 {
+        return Err(EIO);
+    }
+
+    instance.plugin_rpc.wait_plugin_ready(dev)?;
+
+    dev_dbg!(dev, "bootload: gfid={} plugin ready\n", instance.gfid.0);
+    Ok(())
+}
+
+/// Shut down a vGPU plugin task and wait for its completion event.
+pub(crate) fn shutdown(
+    dev: &device::Device<device::Bound>,
+    cmdq: &Cmdq,
+    bar: Bar0<'_>,
+    gfid: Gfid,
+) -> Result {
+    let payload = gfid.0.to_le_bytes();
+
+    cmdq.send_gmc_and_wait_event(
+        bar,
+        gmc::SHUTDOWN,
+        &payload,
+        Delta::from_secs(10),
+        |command_id, status, _sequence, payload_0, payload_1| {
+            if command_id != gmc::SHUTDOWN_COMPLETE
+                || !payload
+                    .iter()
+                    .copied()
+                    .eq(Iterator::chain(payload_0.iter(), payload_1.iter())
+                        .take(payload.len())
+                        .copied())
+            {
+                return Ok(false);
+            }
+            if status != 0 {
+                return Err(EIO);
+            }
+            Ok(true)
+        },
+        |command_id, status, _sequence, _payload_0, _payload_1| {
+            dev_dbg!(
+                dev,
+                "shutdown: ignoring unrelated event command={:#x} status={:#x}\n",
+                command_id,
+                status,
+            );
+            Ok(())
+        },
+    )?;
+    dev_dbg!(dev, "shutdown: gfid={} stopped\n", gfid.0);
+    Ok(())
+}
+
+/// Release firmware resources after a plugin task has stopped.
+pub(crate) fn cleanup(
+    dev: &device::Device<device::Bound>,
+    cmdq: &Cmdq,
+    bar: Bar0<'_>,
+    gfid: Gfid,
+) -> Result {
+    cmdq.send_gmc_and_check_status(bar, gmc::CLEANUP, &gfid.0.to_le_bytes())?;
+    dev_dbg!(dev, "cleanup: gfid={} done\n", gfid.0);
+    Ok(())
+}
diff --git a/drivers/gpu/nova-core/vgpu/consts.rs b/drivers/gpu/nova-core/vgpu/consts.rs
index 7ec577ec12f2..2ebbf2a0daa3 100644
--- a/drivers/gpu/nova-core/vgpu/consts.rs
+++ b/drivers/gpu/nova-core/vgpu/consts.rs
@@ -9,4 +9,12 @@ pub(crate) mod gmc {
         bindings::GMCAPI_COMMANDS_GMCAPI_CMD_QUERY_VGPU_PROPERTIES;
     pub(crate) const VGPU_MGMT_QUERY_ASSIGNED_VF: u32 =
         bindings::GMCAPI_COMMANDS_GMCAPI_CMD_QUERY_ASSIGNED_VF_VGPU_TYPE;
+    pub(crate) const BOOTLOAD: u32 =
+        bindings::GMCAPI_COMMANDS_GMCAPI_CMD_BOOTLOAD_GSP_VGPU_PLUGIN_TASK;
+    pub(crate) const SHUTDOWN: u32 =
+        bindings::GMCAPI_COMMANDS_GMCAPI_CMD_SHUTDOWN_GSP_VGPU_PLUGIN_TASK;
+    pub(crate) const SHUTDOWN_COMPLETE: u32 =
+        bindings::GMCAPI_COMMANDS_GMCAPI_CMD_SHUTDOWN_GSP_VGPU_PLUGIN_TASK_COMPLETE;
+    pub(crate) const CLEANUP: u32 =
+        bindings::GMCAPI_COMMANDS_GMCAPI_CMD_CLEANUP_GSP_VGPU_PLUGIN_RESOURCES;
 }
diff --git a/drivers/gpu/nova-core/vgpu/fw.rs b/drivers/gpu/nova-core/vgpu/fw.rs
index edfb0f984b6d..3cd77f0cd62e 100644
--- a/drivers/gpu/nova-core/vgpu/fw.rs
+++ b/drivers/gpu/nova-core/vgpu/fw.rs
@@ -1,2 +1,195 @@
 // SPDX-License-Identifier: GPL-2.0
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+use kernel::prelude::*;
+
+use crate::{
+    gsp::vgpu_bindings as bindings,
+    mm::{
+        bar_user::{
+            Bar1Map,
+            BarUser, //
+        },
+        vram::VramRegion,
+        GpuMm, //
+    },
+};
+
+type RawControlRegion = bindings::VGPU_CPU_GSP_CTRL_BUFF_REGION;
+
+/// Physical VRAM regions containing the vGPU plugin logs.
+pub(crate) struct PluginLogRegions {
+    init: VramRegion,
+    vgpu: VramRegion,
+    kernel: VramRegion,
+}
+
+impl PluginLogRegions {
+    /// Return the init-task log region.
+    pub(crate) const fn init(&self) -> &VramRegion {
+        &self.init
+    }
+
+    /// Return the vGPU-task log region.
+    pub(crate) const fn vgpu(&self) -> &VramRegion {
+        &self.vgpu
+    }
+
+    /// Return the kernel-task log region.
+    pub(crate) const fn kernel(&self) -> &VramRegion {
+        &self.kernel
+    }
+}
+
+/// Take the next firmware-defined subregion from a communication buffer.
+fn take_region(region: &VramRegion, cursor: &mut u64, size: u32) -> Result<VramRegion> {
+    let end = cursor.checked_add(u64::from(size)).ok_or(EOVERFLOW)?;
+    let subregion = region.subregion(*cursor..end)?;
+    *cursor = end;
+    Ok(subregion)
+}
+
+/// BAR1 mapping and semantic regions of a vGPU CPU-GSP communication buffer.
+///
+/// The host and GSP plugin exchange control, response, message, migration,
+/// error, and diagnostic data through firmware-defined subregions of the
+/// management heap. Firmware accesses that VRAM directly; the host accesses
+/// the same storage through the owned BAR1 mapping.
+///
+/// The firmware bindings define each subregion's size and order, but are used
+/// only to describe the layout. Field accesses must use the BAR1 I/O accessors
+/// because bindgen does not preserve C `volatile` semantics. Keep this object
+/// alive while the plugin or a host reader can use the buffer, then consume it
+/// with [`Self::destroy`] after those users have stopped.
+pub(crate) struct CommBufferRegion<'gpu> {
+    map: Bar1Map<'gpu>,
+    control: VramRegion,
+    init_log: VramRegion,
+    vgpu_log: VramRegion,
+    kernel_log: VramRegion,
+}
+
+impl<'gpu> CommBufferRegion<'gpu> {
+    /// Map the communication portion of a plugin management heap.
+    pub(crate) fn new(
+        bar_user: &BarUser<'gpu>,
+        mm: &mut GpuMm<'_>,
+        management_heap: &VramRegion,
+    ) -> Result<Self> {
+        let total_size = u64::from(bindings::VGPU_CPU_GSP_COMMUNICATION_BUFF_TOTAL_SIZE);
+        let region = management_heap.subregion(0..total_size)?;
+        let mut cursor = 0;
+
+        let control = take_region(
+            &region,
+            &mut cursor,
+            bindings::VGPU_CPU_GSP_CTRL_BUFF_REGION_SIZE,
+        )?;
+        take_region(
+            &region,
+            &mut cursor,
+            bindings::VGPU_CPU_GSP_RESPONSE_BUFF_REGION_SIZE,
+        )?;
+        take_region(
+            &region,
+            &mut cursor,
+            bindings::VGPU_CPU_GSP_MESSAGE_BUFF_REGION_SIZE,
+        )?;
+        take_region(
+            &region,
+            &mut cursor,
+            bindings::VGPU_CPU_GSP_MIGRATION_BUFF_REGION_SIZE,
+        )?;
+        take_region(
+            &region,
+            &mut cursor,
+            bindings::VGPU_CPU_GSP_ERROR_BUFF_REGION_SIZE,
+        )?;
+        let init_log = take_region(
+            &region,
+            &mut cursor,
+            bindings::VGPU_CPU_GSP_INIT_TASK_LOG_BUFF_REGION_SIZE,
+        )?;
+        let vgpu_log = take_region(
+            &region,
+            &mut cursor,
+            bindings::VGPU_CPU_GSP_VGPU_TASK_LOG_BUFF_REGION_SIZE,
+        )?;
+        let kernel_log = take_region(
+            &region,
+            &mut cursor,
+            bindings::VGPU_CPU_GSP_KERNEL_TASK_LOG_BUFF_REGION_SIZE,
+        )?;
+        take_region(
+            &region,
+            &mut cursor,
+            bindings::VGPU_CPU_GSP_GUEST_RPC_TRACE_BUFF_REGION_SIZE,
+        )?;
+
+        if cursor != total_size || control.size() != u64::try_from(size_of::<RawControlRegion>())? {
+            return Err(EINVAL);
+        }
+
+        let map = Bar1Map::new(bar_user, mm, region, true)?;
+
+        Ok(Self {
+            map,
+            control,
+            init_log,
+            vgpu_log,
+            kernel_log,
+        })
+    }
+
+    fn region_offset(&self, region: &VramRegion) -> Result<usize> {
+        let offset = region
+            .address()
+            .checked_sub(self.map.region().address())
+            .ok_or(EINVAL)?;
+        if offset.checked_add(region.size()).ok_or(EOVERFLOW)? > self.map.region().size() {
+            return Err(EINVAL);
+        }
+
+        usize::try_from(offset).map_err(|_| EOVERFLOW)
+    }
+
+    fn io_offset(&self, region: &VramRegion, field: usize, width: usize) -> Result<usize> {
+        let field_end = field.checked_add(width).ok_or(EOVERFLOW)?;
+        if u64::try_from(field_end).map_err(|_| EOVERFLOW)? > region.size() {
+            return Err(EINVAL);
+        }
+
+        self.region_offset(region)?
+            .checked_add(field)
+            .ok_or(EOVERFLOW)
+    }
+
+    fn read_u32(&self, region: &VramRegion, field: usize) -> Result<u32> {
+        self.map
+            .try_read32(self.io_offset(region, field, size_of::<u32>())?)
+    }
+
+    /// Return the physical regions occupied by the three plugin logs.
+    pub(crate) fn plugin_logs(&self) -> Result<PluginLogRegions> {
+        Ok(PluginLogRegions {
+            init: self.init_log.clone(),
+            vgpu: self.vgpu_log.clone(),
+            kernel: self.kernel_log.clone(),
+        })
+    }
+
+    /// Return whether firmware has published the plugin boot marker.
+    pub(crate) fn is_plugin_ready(&self) -> Result<bool> {
+        let value = self.read_u32(
+            &self.control,
+            core::mem::offset_of!(RawControlRegion, __bindgen_anon_1.message_seq_num),
+        )?;
+
+        Ok(value == bindings::GSP_PLUGIN_BOOTLOADED)
+    }
+
+    /// Invalidate the PTEs and release the communication mapping.
+    pub(crate) fn destroy(self, bar_user: &BarUser<'gpu>, mm: &mut GpuMm<'_>) -> Result {
+        self.map.destroy(bar_user, mm)
+    }
+}
diff --git a/drivers/gpu/nova-core/vgpu/instance.rs b/drivers/gpu/nova-core/vgpu/instance.rs
index ed304945330a..834b647e254a 100644
--- a/drivers/gpu/nova-core/vgpu/instance.rs
+++ b/drivers/gpu/nova-core/vgpu/instance.rs
@@ -4,6 +4,7 @@
 use core::num::NonZeroUsize;
 
 use kernel::{
+    device,
     prelude::*,
     ptr::Alignment,
     sizes::SizeConstants, //
@@ -17,12 +18,23 @@
         commands::{
             decode_vgpu_properties,
             Dbdf,
+            FifoEngineList,
             VgpuProperties, //
         },
     },
-    mm::GpuMm,
+    mm::{
+        bar_user::BarUser,
+        GpuMm, //
+    },
     vgpu::{
+        bootload::{
+            bootload,
+            cleanup,
+            shutdown, //
+        },
         consts::gmc,
+        fw::CommBufferRegion,
+        plugin_rpc::PluginRpc,
         vram::{
             VgpuVramLayout,
             VgpuVramSlot,
@@ -59,6 +71,10 @@ pub(crate) struct VgpuType {
 }
 
 impl VgpuType {
+    pub(crate) const fn vgpu_type_id(&self) -> u32 {
+        self.vgpu_type_id
+    }
+
     fn from_properties(properties: &VgpuProperties) -> Self {
         let mut name = [0; 64];
         let name_len = properties.name.len().min(name.len());
@@ -99,6 +115,24 @@ pub(crate) struct VgpuInstance<'gpu> {
     pub(crate) chids: ChannelIdReservation<'gpu>,
     pub(crate) num_plugin_channels: u32,
     pub(crate) vram_slot: VgpuVramSlot,
+    pub(crate) plugin_rpc: PluginRpc<'gpu>,
+}
+
+impl<'gpu> VgpuInstance<'gpu> {
+    /// Unmap the plugin communication buffer and return the slot release token.
+    fn unmap_and_take_slot(
+        self,
+        bar_user: &BarUser<'gpu>,
+        mm: &mut GpuMm<'_>,
+    ) -> Result<VgpuVramSlot> {
+        let Self {
+            plugin_rpc,
+            vram_slot,
+            ..
+        } = self;
+        plugin_rpc.destroy(bar_user, mm)?;
+        Ok(vram_slot)
+    }
 }
 
 /// Identity and firmware profile used to allocate an instance.
@@ -166,10 +200,13 @@ fn release_vram_slot(&mut self, slot: VgpuVramSlot) {
         allocator.release(slot);
     }
 
-    /// Allocate resources and register a new inactive vGPU instance.
+    /// Allocate resources, map the management communication region, and
+    /// register a new inactive vGPU instance.
     pub(crate) fn allocate_instance(
         &mut self,
-        mm: &GpuMm<'_>,
+        dev: &device::Device<device::Bound>,
+        bar_user: &BarUser<'gpu>,
+        mm: &mut GpuMm<'_>,
         vgpu: &VgpuManager<'gpu>,
         info: InstanceInfo,
     ) -> Result<Gfid> {
@@ -220,6 +257,21 @@ pub(crate) fn allocate_instance(
             fb_align: vgpu.vmmu_segment_size().ok_or(ENODEV)?,
         };
         let vram_slot = self.alloc_vram_slot(mm, layout)?;
+        let comm = match CommBufferRegion::new(bar_user, mm, &vram_slot.mgmt_heap) {
+            Ok(comm) => comm,
+            Err(error) => {
+                // A failed page-table update may have installed a partial mapping without
+                // returning a handle that can unmap it. Keep the slot reserved so its backing
+                // VRAM cannot be reused while stale BAR1 PTEs may still reference it.
+                dev_err!(
+                    dev,
+                    "allocate_instance: retaining slot {} after BAR1 map error {:?}\n",
+                    vram_slot.index(),
+                    error,
+                );
+                return Err(error);
+            }
+        };
 
         let instance = VgpuInstance {
             gfid,
@@ -229,26 +281,40 @@ pub(crate) fn allocate_instance(
             chids,
             num_plugin_channels: 3,
             vram_slot,
+            plugin_rpc: PluginRpc::new(comm),
         };
         match self.instances.push_within_capacity(instance) {
             Ok(()) => Ok(gfid),
-            Err(error) => {
-                let VgpuInstance { vram_slot, .. } = error.0;
-                self.release_vram_slot(vram_slot);
-                Err(EIO)
-            }
+            Err(error) => match error.0.unmap_and_take_slot(bar_user, mm) {
+                Ok(vram_slot) => {
+                    self.release_vram_slot(vram_slot);
+                    Err(EIO)
+                }
+                Err(error) => Err(error),
+            },
         }
     }
 
-    /// Remove an instance and release its channel and VRAM reservations.
-    pub(crate) fn destroy_instance(&mut self, gfid: Gfid) -> Result {
+    /// Shut down and remove an instance, then release its reservations.
+    pub(crate) fn destroy_instance(
+        &mut self,
+        dev: &device::Device<device::Bound>,
+        cmdq: &Cmdq,
+        bar: Bar0<'_>,
+        bar_user: &BarUser<'gpu>,
+        mm: &mut GpuMm<'_>,
+        gfid: Gfid,
+    ) -> Result {
         let index = self
             .instances
             .iter()
             .position(|instance| instance.gfid == gfid)
             .ok_or(ENOENT)?;
+
+        shutdown(dev, cmdq, bar, gfid)?;
+        cleanup(dev, cmdq, bar, gfid)?;
         let instance = self.instances.remove(index).map_err(|_| EIO)?;
-        let VgpuInstance { vram_slot, .. } = instance;
+        let vram_slot = instance.unmap_and_take_slot(bar_user, mm)?;
         self.release_vram_slot(vram_slot);
         Ok(())
     }
@@ -286,3 +352,15 @@ pub(crate) fn query_vgpu_type(cmdq: &Cmdq, bar: Bar0<'_>, type_id: u32) -> Resul
     }
     Ok(VgpuType::from_properties(&properties))
 }
+
+/// Bootload the GSP plugin for an allocated instance.
+#[expect(dead_code)]
+pub(crate) fn activate_instance(
+    dev: &device::Device<device::Bound>,
+    cmdq: &Cmdq,
+    bar: Bar0<'_>,
+    instance: &mut VgpuInstance<'_>,
+    fifo_engine_list: &FifoEngineList,
+) -> Result {
+    bootload(dev, cmdq, bar, instance, fifo_engine_list)
+}
diff --git a/drivers/gpu/nova-core/vgpu/mod.rs b/drivers/gpu/nova-core/vgpu/mod.rs
index 320230ddd1dd..1c67d7afbe56 100644
--- a/drivers/gpu/nova-core/vgpu/mod.rs
+++ b/drivers/gpu/nova-core/vgpu/mod.rs
@@ -3,8 +3,10 @@
 
 use core::num::NonZero;
 
+pub(crate) mod bootload;
 pub(crate) mod consts;
 pub(crate) mod instance;
+pub(crate) mod plugin_rpc;
 
 pub(crate) use self::instance::VgpuInstances;
 
diff --git a/drivers/gpu/nova-core/vgpu/plugin_rpc.rs b/drivers/gpu/nova-core/vgpu/plugin_rpc.rs
new file mode 100644
index 000000000000..d6077a468672
--- /dev/null
+++ b/drivers/gpu/nova-core/vgpu/plugin_rpc.rs
@@ -0,0 +1,65 @@
+// SPDX-License-Identifier: GPL-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+use kernel::{
+    device,
+    prelude::*,
+    time::{
+        delay::fsleep,
+        Delta,
+        Instant,
+        Monotonic, //
+    },
+};
+
+use crate::{
+    mm::{
+        bar_user::BarUser,
+        GpuMm, //
+    },
+    vgpu::fw::{
+        CommBufferRegion,
+        PluginLogRegions, //
+    },
+};
+
+/// Host-side ready limit from `vmiopd_negotiate_cpu_gsp_version()` in
+/// `vmiop-vgpu.c`, which polls the same boot marker for 10 seconds.
+const PLUGIN_READY_TIMEOUT: Delta = Delta::from_secs(10);
+
+/// BAR1-backed channel used to communicate with the vGPU plugin.
+pub(crate) struct PluginRpc<'gpu> {
+    comm: CommBufferRegion<'gpu>,
+}
+
+impl<'gpu> PluginRpc<'gpu> {
+    pub(crate) fn new(comm: CommBufferRegion<'gpu>) -> Self {
+        Self { comm }
+    }
+
+    /// Return the physical regions occupied by the plugin logs.
+    pub(crate) fn plugin_logs(&self) -> Result<PluginLogRegions> {
+        self.comm.plugin_logs()
+    }
+
+    /// Poll the control buffer until the plugin publishes its boot marker.
+    pub(crate) fn wait_plugin_ready(&self, dev: &device::Device<device::Bound>) -> Result {
+        let start = Instant::<Monotonic>::now();
+
+        loop {
+            if self.comm.is_plugin_ready()? {
+                dev_dbg!(dev, "vGPU plugin ready after {:?}\n", start.elapsed());
+                return Ok(());
+            }
+            if start.elapsed() >= PLUGIN_READY_TIMEOUT {
+                return Err(ETIMEDOUT);
+            }
+            fsleep(Delta::from_millis(1));
+        }
+    }
+
+    /// Release the BAR1 mapping.
+    pub(crate) fn destroy(self, bar_user: &BarUser<'gpu>, mm: &mut GpuMm<'_>) -> Result {
+        self.comm.destroy(bar_user, mm)
+    }
+}
diff --git a/drivers/gpu/nova-core/vgpu/vram.rs b/drivers/gpu/nova-core/vgpu/vram.rs
index c646511b6fd6..bc7283abba29 100644
--- a/drivers/gpu/nova-core/vgpu/vram.rs
+++ b/drivers/gpu/nova-core/vgpu/vram.rs
@@ -3,8 +3,6 @@
 
 //! VRAM slot allocation for vGPU instances.
 
-#![expect(dead_code)]
-
 use kernel::{
     bitmap::BitmapVec,
     prelude::*,
-- 
2.53.0


  parent reply	other threads:[~2026-09-05  8:12 UTC|newest]

Thread overview: 15+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-09-05  8:11 [PATCH 00/13] Introduce NVIDIA vGPU manager and VFIO variant driver Zhi Wang
2026-09-05  8:11 ` [PATCH 01/13] gpu: nova-core: vgpu: add post-GSP-boot vGPU initialization Zhi Wang
2026-09-05  8:11 ` [PATCH 02/13] gpu: nova-core: mm: add VramBlock and Bar1Map Zhi Wang
2026-09-05  8:11 ` [PATCH 03/13] gpu: nova-core: vgpu: add VRAM slot allocator Zhi Wang
2026-09-05  8:11 ` [PATCH 04/13] gpu: nova-core: vgpu: add r000 plugin bindings Zhi Wang
2026-09-05  8:11 ` [PATCH 05/13] gpu: nova-core: vgpu: add instance create/destroy Zhi Wang
2026-09-05  8:11 ` [PATCH 06/13] gpu: nova-core: gsp: add GMC transaction helpers Zhi Wang
2026-09-05  8:11 ` Zhi Wang [this message]
2026-09-05  8:11 ` [PATCH 08/13] gpu: nova-core: vgpu: implement PluginRpc channel and config params Zhi Wang
2026-09-05  8:11 ` [PATCH 09/13] gpu: nova-core: vgpu: scrub guest framebuffer memory with CeUtils Zhi Wang
2026-09-05  8:11 ` [PATCH 10/13] gpu: nova-core: vgpu: export plugin log buffers via debugfs Zhi Wang
2026-09-05  8:11 ` [PATCH 11/13] gpu: nova-core: vgpu: export lifecycle operations to VFIO Zhi Wang
2026-09-05  8:11 ` [PATCH 12/13] vfio/nvidia-vgpu: add the NVIDIA vGPU VFIO variant driver Zhi Wang
2026-09-09  3:00   ` Alex Williamson
2026-09-05  8:11 ` [PATCH 13/13] gpu: nova-core: reserve the 48-VM WPR2 heap Zhi Wang

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20260905081116.106613-8-zhiw@nvidia.com \
    --to=zhiw@nvidia.com \
    --cc=a.hindborg@kernel.org \
    --cc=acourbot@nvidia.com \
    --cc=airlied@gmail.com \
    --cc=alex.gaynor@gmail.com \
    --cc=alex@shazbot.org \
    --cc=aliceryhl@google.com \
    --cc=alkumar@nvidia.com \
    --cc=aniketa@nvidia.com \
    --cc=ankita@nvidia.com \
    --cc=bjorn3_gh@protonmail.com \
    --cc=boqun.feng@gmail.com \
    --cc=cjia@nvidia.com \
    --cc=dakr@kernel.org \
    --cc=ecourtney@nvidia.com \
    --cc=gary@garyguo.net \
    --cc=jgg@nvidia.com \
    --cc=jhubbard@nvidia.com \
    --cc=kevin.tian@intel.com \
    --cc=kjaju@nvidia.com \
    --cc=kwankhede@nvidia.com \
    --cc=linux-kernel@vger.kernel.org \
    --cc=lossin@kernel.org \
    --cc=nova-gpu@lists.linux.dev \
    --cc=ojeda@kernel.org \
    --cc=simona@ffwll.ch \
    --cc=skolothumtho@nvidia.com \
    --cc=smitra@nvidia.com \
    --cc=targupta@nvidia.com \
    --cc=tmgross@umich.edu \
    --cc=yishaih@nvidia.com \
    --cc=zhiwang@kernel.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox