[PATCH 0/3] gpu: nova-core: add formatted trace events

[PATCH 0/3] gpu: nova-core: add formatted trace events

[Zhi Wang]

Tracking driver and firmware activity is useful for debugging and bug
reports, e.g. GSP and FSP message sends and receives, vGPU states,
lifecycle events[1]. Trace events provide a low-overhead interface for
frequent diagnostics and allow each component stream to be enabled
independently.

This series adds a shared formatted event class and independent events for
general driver, FSP, GSP and vGPU messages. It replaces selected dev_dbg!()
calls with their corresponding trace events and adds FSP NVDM send and
receive records with the message type and length.

Use the module selector to enable the events before nova-core probes. With
tracefs mounted at /sys/kernel/tracing, run:

  cd /sys/kernel/tracing
  echo 'nova_core:*:mod:nova_core' > set_event
  echo 1 > tracing_on
  insmod /path/to/nova-core.ko
  cat trace

[1] https://github.com/zhiwang-nvidia/nova-core/tree/zhi/nova-gpu-wip-nova-gsp-20260803

Zhi Wang (3):
  gpu: nova-core: add formatted tracing
  gpu: nova-core: trace driver probe and firmware messages
  gpu: nova-core: trace vGPU state

 drivers/gpu/Makefile               |   4 +-
 drivers/gpu/nova-core/driver.rs    |   7 +-
 drivers/gpu/nova-core/fsp.rs       |  16 ++-
 drivers/gpu/nova-core/gsp/cmdq.rs  |  11 ++-
 drivers/gpu/nova-core/nova_core.rs |   1 +
 drivers/gpu/nova-core/trace.c      |   5 +
 drivers/gpu/nova-core/trace.h      |  52 ++++++++++
 drivers/gpu/nova-core/trace.rs     | 151 +++++++++++++++++++++++++++++
 drivers/gpu/nova-core/vgpu.rs      |   5 +-
 rust/bindings/bindings_helper.h    |   4 +
 10 files changed, 245 insertions(+), 11 deletions(-)
 create mode 100644 drivers/gpu/nova-core/trace.c
 create mode 100644 drivers/gpu/nova-core/trace.h
 create mode 100644 drivers/gpu/nova-core/trace.rs


base-commit: 44e7e7f7cffb10a93bb88e7cb59b7b8b3e2deb1c
-- 
2.53.0

[PATCH 1/3] gpu: nova-core: add formatted tracing

[Zhi Wang]

Provide formatted trace events for general driver, FSP, GSP and vGPU
messages. Add a shared event class and a formatting helper that avoids
formatting disabled events and uses a fixed-size stack buffer.

Signed-off-by: Zhi Wang <zhiw@nvidia.com>
---
 drivers/gpu/Makefile               |   4 +-
 drivers/gpu/nova-core/nova_core.rs |   1 +
 drivers/gpu/nova-core/trace.c      |   5 +
 drivers/gpu/nova-core/trace.h      |  52 +++++++++
 drivers/gpu/nova-core/trace.rs     | 162 +++++++++++++++++++++++++++++
 rust/bindings/bindings_helper.h    |   4 +
 6 files changed, 227 insertions(+), 1 deletion(-)
 create mode 100644 drivers/gpu/nova-core/trace.c
 create mode 100644 drivers/gpu/nova-core/trace.h
 create mode 100644 drivers/gpu/nova-core/trace.rs

diff --git a/drivers/gpu/Makefile b/drivers/gpu/Makefile
index e372fc02139f..d26e7dea9e66 100644
--- a/drivers/gpu/Makefile
+++ b/drivers/gpu/Makefile
@@ -14,7 +14,9 @@ obj-$(CONFIG_TRACE_GPU_MEM)		+= trace/
 # system supports cross-crate dependencies natively.
 
 obj-$(CONFIG_NOVA_CORE) += nova-core.o
-nova-core-y := nova-core/nova_core.o nova-core/nova_core_exports.o
+nova-core-y := nova-core/nova_core.o \
+	       nova-core/trace.o \
+	       nova-core/nova_core_exports.o
 
 obj-$(CONFIG_DRM_NOVA) += nova-drm.o
 nova-drm-y := drm/nova/nova.o
diff --git a/drivers/gpu/nova-core/nova_core.rs b/drivers/gpu/nova-core/nova_core.rs
index 35a8b1214b0e..85ee5067ef56 100644
--- a/drivers/gpu/nova-core/nova_core.rs
+++ b/drivers/gpu/nova-core/nova_core.rs
@@ -22,6 +22,7 @@
 mod num;
 mod regs;
 mod sbuffer;
+mod trace;
 mod vbios;
 mod vgpu;
 
diff --git a/drivers/gpu/nova-core/trace.c b/drivers/gpu/nova-core/trace.c
new file mode 100644
index 000000000000..c72cb0f65b47
--- /dev/null
+++ b/drivers/gpu/nova-core/trace.c
@@ -0,0 +1,5 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#define CREATE_TRACE_POINTS
+#define CREATE_RUST_TRACE_POINTS
+#include "trace.h"
diff --git a/drivers/gpu/nova-core/trace.h b/drivers/gpu/nova-core/trace.h
new file mode 100644
index 000000000000..a8e5181ba302
--- /dev/null
+++ b/drivers/gpu/nova-core/trace.h
@@ -0,0 +1,52 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM nova_core
+
+#if !defined(_NOVA_CORE_TRACE_H_) || defined(TRACE_HEADER_MULTI_READ)
+#define _NOVA_CORE_TRACE_H_
+
+#include <linux/tracepoint.h>
+
+DECLARE_EVENT_CLASS(nova_core_trace_class,
+	TP_PROTO(const char *dev, const char *message, size_t message_len),
+	TP_ARGS(dev, message, message_len),
+	TP_STRUCT__entry(
+		__string(dev, dev)
+		__string_len(message, message, message_len)
+	),
+	TP_fast_assign(
+		__assign_str(dev);
+		__assign_str(message);
+	),
+	TP_printk("%s %s", __get_str(dev), __get_str(message))
+);
+
+DEFINE_EVENT(nova_core_trace_class, nova_core_trace_driver,
+	TP_PROTO(const char *dev, const char *message, size_t message_len),
+	TP_ARGS(dev, message, message_len)
+);
+
+DEFINE_EVENT(nova_core_trace_class, nova_core_trace_fsp,
+	TP_PROTO(const char *dev, const char *message, size_t message_len),
+	TP_ARGS(dev, message, message_len)
+);
+
+DEFINE_EVENT(nova_core_trace_class, nova_core_trace_gsp,
+	TP_PROTO(const char *dev, const char *message, size_t message_len),
+	TP_ARGS(dev, message, message_len)
+);
+
+DEFINE_EVENT(nova_core_trace_class, nova_core_trace_vgpu,
+	TP_PROTO(const char *dev, const char *message, size_t message_len),
+	TP_ARGS(dev, message, message_len)
+);
+
+#endif /* _NOVA_CORE_TRACE_H_ */
+
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH ../../drivers/gpu/nova-core
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_FILE trace
+
+#include <trace/define_trace.h>
diff --git a/drivers/gpu/nova-core/trace.rs b/drivers/gpu/nova-core/trace.rs
new file mode 100644
index 000000000000..893479b3fcb5
--- /dev/null
+++ b/drivers/gpu/nova-core/trace.rs
@@ -0,0 +1,162 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Nova tracepoint helpers.
+
+use kernel::{
+    ffi::c_char,
+    fmt::{
+        self,
+        Write, //
+    },
+    str::{
+        CStr,
+        CStrExt,
+        Formatter, //
+    }, //
+};
+
+const MESSAGE_MAX: usize = 512;
+
+// To add another formatted Nova Core trace event:
+//
+// 1. Define it from `nova_core_trace_class` in `trace.h`.
+// 2. Declare its Rust entry point with `declare_nova_core_trace!` below.
+// 3. Add a public frontend macro that passes its event name to
+//    `nova_core_trace_impl!`, following the examples at the end of this file.
+pub(crate) mod raw {
+    use super::c_char;
+
+    macro_rules! declare_nova_core_trace {
+        ($event:ident) => {
+            kernel::tracepoint::declare_trace! {
+                /// # Safety
+                ///
+                /// `dev` must point to a valid NUL-terminated string, and
+                /// `message` must point to `message_len` readable bytes for
+                /// this call.
+                pub(crate) unsafe fn $event(
+                    dev: *const c_char,
+                    message: *const c_char,
+                    message_len: usize,
+                );
+            }
+        };
+    }
+
+    declare_nova_core_trace!(nova_core_trace_driver);
+    declare_nova_core_trace!(nova_core_trace_fsp);
+    declare_nova_core_trace!(nova_core_trace_gsp);
+    declare_nova_core_trace!(nova_core_trace_vgpu);
+}
+
+/// Formats and emits a Nova Core text trace event.
+///
+/// # Safety
+///
+/// `trace` must synchronously consume `dev`, `message`, and `message_len`
+/// according to the `nova_core_trace_class` event prototype.
+#[expect(dead_code)]
+pub(crate) unsafe fn nova_core_trace_fmt(
+    dev: &CStr,
+    args: fmt::Arguments<'_>,
+    trace: unsafe fn(*const c_char, *const c_char, usize),
+) {
+    let mut message = [0u8; MESSAGE_MAX];
+    let message_len = {
+        let mut formatter = Formatter::new(&mut message);
+
+        let _ = formatter.write_fmt(args);
+        formatter.bytes_written().min(MESSAGE_MAX)
+    };
+
+    // SAFETY: The caller guarantees that `trace` synchronously consumes its
+    // arguments. The device name is NUL-terminated, and `message` contains
+    // `message_len` initialized bytes.
+    unsafe {
+        trace(
+            dev.as_char_ptr(),
+            message.as_ptr().cast::<c_char>(),
+            message_len,
+        )
+    }
+}
+
+#[expect(unused_macros)]
+macro_rules! nova_core_trace_impl {
+    ($event:ident, $dev:expr, $($arg:tt)*) => {{
+        #[cfg(CONFIG_TRACEPOINTS)]
+        let should_trace = {
+            // SAFETY: `$event` names a real C tracepoint static key.
+            unsafe {
+                kernel::macros::paste! {
+                    kernel::jump_label::static_branch_unlikely!(
+                        kernel::bindings::[<__tracepoint_ $event>],
+                        kernel::bindings::tracepoint,
+                        key
+                    )
+                }
+            }
+        };
+
+        #[cfg(not(CONFIG_TRACEPOINTS))]
+        let should_trace = false;
+
+        if should_trace {
+            match ($dev, kernel::prelude::fmt!($($arg)*)) {
+                (dev, args) => {
+                    // SAFETY: `$event` has the event-class prototype required
+                    // by `nova_core_trace_fmt`.
+                    unsafe {
+                        $crate::trace::nova_core_trace_fmt(
+                            dev.as_ref().name(),
+                            args,
+                            $crate::trace::raw::$event,
+                        )
+                    }
+                }
+            }
+        }
+    }};
+}
+
+// Frontend macros expand in their caller's module and invoke this helper by
+// path, so the re-export must be visible from the parent module.
+#[expect(unused_imports)]
+pub(super) use nova_core_trace_impl;
+
+#[expect(unused_macros)]
+macro_rules! nova_core_trace_driver {
+    ($dev:expr, $($arg:tt)*) => {
+        $crate::trace::nova_core_trace_impl!(nova_core_trace_driver, $dev, $($arg)*)
+    };
+}
+
+#[expect(unused_macros)]
+macro_rules! nova_core_trace_fsp {
+    ($dev:expr, $($arg:tt)*) => {
+        $crate::trace::nova_core_trace_impl!(nova_core_trace_fsp, $dev, $($arg)*)
+    };
+}
+
+#[expect(unused_macros)]
+macro_rules! nova_core_trace_gsp {
+    ($dev:expr, $($arg:tt)*) => {
+        $crate::trace::nova_core_trace_impl!(nova_core_trace_gsp, $dev, $($arg)*)
+    };
+}
+
+#[expect(unused_macros)]
+macro_rules! nova_core_trace_vgpu {
+    ($dev:expr, $($arg:tt)*) => {
+        $crate::trace::nova_core_trace_impl!(nova_core_trace_vgpu, $dev, $($arg)*)
+    };
+}
+
+#[expect(unused_imports)]
+pub(crate) use nova_core_trace_driver;
+#[expect(unused_imports)]
+pub(crate) use nova_core_trace_fsp;
+#[expect(unused_imports)]
+pub(crate) use nova_core_trace_gsp;
+#[expect(unused_imports)]
+pub(crate) use nova_core_trace_vgpu;
diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h
index 1124785e210b..d1e0a60e370f 100644
--- a/rust/bindings/bindings_helper.h
+++ b/rust/bindings/bindings_helper.h
@@ -165,3 +165,7 @@ const unsigned long RUST_CONST_HELPER_GPU_BUDDY_TRIM_DISABLE = GPU_BUDDY_TRIM_DI
 #include "../../drivers/android/binder/rust_binder.h"
 #include "../../drivers/android/binder/rust_binder_events.h"
 #endif
+
+#if IS_ENABLED(CONFIG_NOVA_CORE)
+#include "../../drivers/gpu/nova-core/trace.h"
+#endif
-- 
2.53.0

[PATCH 2/3] gpu: nova-core: trace driver probe and firmware messages

[Zhi Wang]

Use independently selectable trace streams for driver, GSP and FSP
activity. Replace the probe and GSP dev_dbg() calls with formatted
traces, and add FSP NVDM send and receive traces with the message type
and length.

Signed-off-by: Zhi Wang <zhiw@nvidia.com>
---
 drivers/gpu/nova-core/driver.rs   |  7 +++++--
 drivers/gpu/nova-core/fsp.rs      | 16 +++++++++++++++-
 drivers/gpu/nova-core/gsp/cmdq.rs | 11 ++++++-----
 drivers/gpu/nova-core/trace.rs    |  9 ---------
 4 files changed, 26 insertions(+), 17 deletions(-)

diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs
index 5738d4ac521b..cb92ce9a594e 100644
--- a/drivers/gpu/nova-core/driver.rs
+++ b/drivers/gpu/nova-core/driver.rs
@@ -18,7 +18,10 @@
     types::ForLt,
 };
 
-use crate::gpu::Gpu;
+use crate::{
+    gpu::Gpu,
+    trace::nova_core_trace_driver, //
+};
 
 /// Counter for generating unique auxiliary device IDs.
 static AUXILIARY_ID_COUNTER: Atomic<u32> = Atomic::new(0);
@@ -73,7 +76,7 @@ fn probe<'bound>(
         _info: &'bound Self::IdInfo,
     ) -> impl PinInit<Self::Data<'bound>, Error> + 'bound {
         pin_init::pin_init_scope(move || {
-            dev_dbg!(pdev, "Probe Nova Core GPU driver.\n");
+            nova_core_trace_driver!(pdev, "Probe Nova Core GPU driver.");
 
             pdev.enable_device_mem()?;
             pdev.set_master();
diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs
index ba4544210e40..9dc919de2200 100644
--- a/drivers/gpu/nova-core/fsp.rs
+++ b/drivers/gpu/nova-core/fsp.rs
@@ -51,7 +51,8 @@
         NvdmType, //
     },
     num,
-    regs, //
+    regs,
+    trace::nova_core_trace_fsp, //
 };
 
 mod hal;
@@ -434,6 +435,12 @@ fn send_sync_fsp<M>(&mut self, dev: &device::Device, msg: &M) -> Result<KVec<u8>
     where
         M: MessageToFsp,
     {
+        nova_core_trace_fsp!(
+            dev,
+            "FSP NVDM: send: type={:?}, length=0x{:x}",
+            M::NVDM_TYPE,
+            msg.as_bytes().len(),
+        );
         self.falcon.send_msg(msg.as_bytes())?;
 
         let response_buf = self.falcon.recv_msg().inspect_err(|e| {
@@ -489,6 +496,13 @@ fn send_sync_fsp<M>(&mut self, dev: &device::Device, msg: &M) -> Result<KVec<u8>
             return Err(EIO);
         }
 
+        nova_core_trace_fsp!(
+            dev,
+            "FSP NVDM: receive: type={:?}, length=0x{:x}",
+            M::NVDM_TYPE,
+            response_buf.len(),
+        );
+
         Ok(response_buf)
     }
 
diff --git a/drivers/gpu/nova-core/gsp/cmdq.rs b/drivers/gpu/nova-core/gsp/cmdq.rs
index cd844fe48f05..6137c60b6231 100644
--- a/drivers/gpu/nova-core/gsp/cmdq.rs
+++ b/drivers/gpu/nova-core/gsp/cmdq.rs
@@ -58,7 +58,8 @@
         GSP_PAGE_SIZE, //
     },
     num,
-    sbuffer::SBufferIter, //
+    sbuffer::SBufferIter,
+    trace::nova_core_trace_gsp, //
 };
 
 use super::regs;
@@ -675,9 +676,9 @@ fn send_single_command<M>(&mut self, bar: Bar0<'_>, command: M) -> Result
                 dst.contents.1,
             ])));
 
-        dev_dbg!(
+        nova_core_trace_gsp!(
             &self.dev,
-            "GSP RPC: send: seq# {}, function={:?}, length=0x{:x}\n",
+            "GSP RPC: send: seq# {}, function={:?}, length=0x{:x}",
             self.seq,
             M::FUNCTION,
             dst.header.length(),
@@ -754,9 +755,9 @@ fn wait_for_msg(&self, timeout: Delta) -> Result<GspMessage<'_>> {
         // Extract the `GspMsgElement`.
         let (header, slice_1) = GspMsgElement::from_bytes_prefix(slice_1).ok_or(EIO)?;
 
-        dev_dbg!(
+        nova_core_trace_gsp!(
             &self.dev,
-            "GSP RPC: receive: seq# {}, function={:?}, length=0x{:x}\n",
+            "GSP RPC: receive: seq# {}, function={:?}, length=0x{:x}",
             header.sequence(),
             header.function(),
             header.length(),
diff --git a/drivers/gpu/nova-core/trace.rs b/drivers/gpu/nova-core/trace.rs
index 893479b3fcb5..db975804f855 100644
--- a/drivers/gpu/nova-core/trace.rs
+++ b/drivers/gpu/nova-core/trace.rs
@@ -55,7 +55,6 @@ pub(crate) unsafe fn $event(
 ///
 /// `trace` must synchronously consume `dev`, `message`, and `message_len`
 /// according to the `nova_core_trace_class` event prototype.
-#[expect(dead_code)]
 pub(crate) unsafe fn nova_core_trace_fmt(
     dev: &CStr,
     args: fmt::Arguments<'_>,
@@ -81,7 +80,6 @@ pub(crate) unsafe fn nova_core_trace_fmt(
     }
 }
 
-#[expect(unused_macros)]
 macro_rules! nova_core_trace_impl {
     ($event:ident, $dev:expr, $($arg:tt)*) => {{
         #[cfg(CONFIG_TRACEPOINTS)]
@@ -121,24 +119,20 @@ macro_rules! nova_core_trace_impl {
 
 // Frontend macros expand in their caller's module and invoke this helper by
 // path, so the re-export must be visible from the parent module.
-#[expect(unused_imports)]
 pub(super) use nova_core_trace_impl;
 
-#[expect(unused_macros)]
 macro_rules! nova_core_trace_driver {
     ($dev:expr, $($arg:tt)*) => {
         $crate::trace::nova_core_trace_impl!(nova_core_trace_driver, $dev, $($arg)*)
     };
 }
 
-#[expect(unused_macros)]
 macro_rules! nova_core_trace_fsp {
     ($dev:expr, $($arg:tt)*) => {
         $crate::trace::nova_core_trace_impl!(nova_core_trace_fsp, $dev, $($arg)*)
     };
 }
 
-#[expect(unused_macros)]
 macro_rules! nova_core_trace_gsp {
     ($dev:expr, $($arg:tt)*) => {
         $crate::trace::nova_core_trace_impl!(nova_core_trace_gsp, $dev, $($arg)*)
@@ -152,11 +146,8 @@ macro_rules! nova_core_trace_vgpu {
     };
 }
 
-#[expect(unused_imports)]
 pub(crate) use nova_core_trace_driver;
-#[expect(unused_imports)]
 pub(crate) use nova_core_trace_fsp;
-#[expect(unused_imports)]
 pub(crate) use nova_core_trace_gsp;
 #[expect(unused_imports)]
 pub(crate) use nova_core_trace_vgpu;
-- 
2.53.0

[PATCH 3/3] gpu: nova-core: trace vGPU state

[Zhi Wang]

Record the vGPU state in an independently selectable vGPU trace stream.
Replace the vGPU state dev_dbg() call with the formatted vGPU trace
event.

Signed-off-by: Zhi Wang <zhiw@nvidia.com>
---
 drivers/gpu/nova-core/trace.rs | 2 --
 drivers/gpu/nova-core/vgpu.rs  | 5 +++--
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/drivers/gpu/nova-core/trace.rs b/drivers/gpu/nova-core/trace.rs
index db975804f855..17967a686fa2 100644
--- a/drivers/gpu/nova-core/trace.rs
+++ b/drivers/gpu/nova-core/trace.rs
@@ -139,7 +139,6 @@ macro_rules! nova_core_trace_gsp {
     };
 }
 
-#[expect(unused_macros)]
 macro_rules! nova_core_trace_vgpu {
     ($dev:expr, $($arg:tt)*) => {
         $crate::trace::nova_core_trace_impl!(nova_core_trace_vgpu, $dev, $($arg)*)
@@ -149,5 +148,4 @@ macro_rules! nova_core_trace_vgpu {
 pub(crate) use nova_core_trace_driver;
 pub(crate) use nova_core_trace_fsp;
 pub(crate) use nova_core_trace_gsp;
-#[expect(unused_imports)]
 pub(crate) use nova_core_trace_vgpu;
diff --git a/drivers/gpu/nova-core/vgpu.rs b/drivers/gpu/nova-core/vgpu.rs
index 6b7e045acea8..e4fc7e101c93 100644
--- a/drivers/gpu/nova-core/vgpu.rs
+++ b/drivers/gpu/nova-core/vgpu.rs
@@ -13,7 +13,8 @@
         Fsp,
         VgpuMode, //
     },
-    gpu::Chipset, //
+    gpu::Chipset,
+    trace::nova_core_trace_vgpu, //
 };
 
 mod hal;
@@ -50,7 +51,7 @@ pub(crate) fn new(
             );
             VgpuState::Disabled
         });
-        dev_dbg!(pdev, "vGPU state: {:?}\n", state);
+        nova_core_trace_vgpu!(pdev, "state: {:?}", state);
 
         Self { state }
     }
-- 
2.53.0