diff --git a/drivers/gpu/nova-core/fb.rs b/drivers/gpu/nova-core/fb.rs index b3a6ab8b57a6..b18dc5447a73 100644 --- a/drivers/gpu/nova-core/fb.rs +++ b/drivers/gpu/nova-core/fb.rs @@ -236,7 +236,7 @@ impl FbRanges { let elf = { const ELF_DOWN_ALIGN: Alignment = Alignment::new::(); - let elf_size = u64::from_safe_cast(gsp_fw.size); + let elf_size = u64::from_safe_cast(gsp_fw.size()); let elf_addr = (boot.start - elf_size).align_down(ELF_DOWN_ALIGN); FbRange(elf_addr..elf_addr + elf_size) diff --git a/drivers/gpu/nova-core/firmware.rs b/drivers/gpu/nova-core/firmware.rs index c16fee6e2b2a..d8f6509a35d1 100644 --- a/drivers/gpu/nova-core/firmware.rs +++ b/drivers/gpu/nova-core/firmware.rs @@ -26,6 +26,7 @@ pub(crate) mod booter; pub(crate) mod fwsec; pub(crate) mod gsp; pub(crate) mod gsp_fmc; +pub(crate) mod radix3; pub(crate) mod riscv; pub(crate) mod tlv; diff --git a/drivers/gpu/nova-core/firmware/gsp.rs b/drivers/gpu/nova-core/firmware/gsp.rs index 22d1f9329c9f..49a9626166bb 100644 --- a/drivers/gpu/nova-core/firmware/gsp.rs +++ b/drivers/gpu/nova-core/firmware/gsp.rs @@ -4,21 +4,16 @@ use kernel::{ device, dma::{ Coherent, - CoherentBox, - DataDirection, DmaAddress, // }, firmware, prelude::*, - scatterlist::{ - Owned, - SGTable, // - }, str::CString, }; use crate::{ firmware::{ + radix3::Radix3, riscv::RiscvFirmware, // tlv::{ request_tlv, // @@ -26,38 +21,15 @@ use crate::{ }, }, gpu::Chipset, - gsp::GSP_PAGE_SIZE, num::FromSafeCast, }; -/// GSP firmware with 3-level radix page tables for the GSP bootloader. -/// -/// The bootloader expects firmware to be mapped starting at address 0 in GSP's virtual address -/// space: -/// -/// ```text -/// Level 0: 1 page, 1 entry -> points to first level 1 page -/// Level 1: Multiple pages/entries -> each entry points to a level 2 page -/// Level 2: Multiple pages/entries -> each entry points to a firmware page -/// ``` -/// -/// Each page is 4KB, each entry is 8 bytes (64-bit DMA address). -/// Also known as "Radix3" firmware. +/// The GSP firmware image, its signatures, and the GSP bootloader. #[pin_data] pub(crate) struct GspFirmware<'a> { - /// The GSP firmware inside a [`VVec`], device-mapped via a SG table. - #[pin] - fw: SGTable>>, - /// Level 2 page table whose entries contain DMA addresses of firmware pages. + /// The firmware image and the radix3 table that maps it. #[pin] - level2: SGTable>>, - /// Level 1 page table whose entries contain DMA addresses of level 2 pages. - #[pin] - level1: SGTable>>, - /// Level 0 page table (single 4KB page) with one entry: DMA address of first level 1 page. - level0: Coherent<'a, [u64]>, - /// Size in bytes of the firmware contained in [`Self::fw`]. - pub(crate) size: usize, + radix3: Radix3<'a>, /// Device-mapped GSP signatures matching the GPU's [`Chipset`]. pub(crate) signatures: Coherent<'a, [u8]>, /// GSP bootloader, verifies the GSP firmware before loading and running it. @@ -87,49 +59,7 @@ impl<'a> GspFirmware<'a> { let signatures = Coherent::from_slice(dev, tlv.get_bytes(b"SIGN")?, GFP_KERNEL)?; Ok(try_pin_init!(Self { - fw <- SGTable::new(dev, fw_vvec, DataDirection::ToDevice, GFP_KERNEL), - level2 <- { - // Allocate the level 2 page table, map the firmware onto it, and map it into - // the device address space. - VVec::::with_capacity( - fw.iter().count() * core::mem::size_of::(), - GFP_KERNEL, - ) - .map_err(|_| ENOMEM) - .and_then(|level2| map_into_lvl(&fw, level2)) - .map(|level2| SGTable::new(dev, level2, DataDirection::ToDevice, GFP_KERNEL))? - }, - level1 <- { - // Allocate the level 1 page table, map the level 2 page table onto it, and map - // it into the device address space. - VVec::::with_capacity( - level2.iter().count() * core::mem::size_of::(), - GFP_KERNEL, - ) - .map_err(|_| ENOMEM) - .and_then(|level1| map_into_lvl(&level2, level1)) - .map(|level1| SGTable::new(dev, level1, DataDirection::ToDevice, GFP_KERNEL))? - }, - level0: { - // Allocate the level 0 page table as a device-visible DMA object, and map the - // level 1 page table onto it. - - // Fill level 1 page entry. - let level1_entry = level1.iter().next().ok_or(EINVAL)?; - let level1_entry_addr = level1_entry.dma_address(); - - // Create level 0 page table data and fill its first entry with the level 1 - // table. - let mut level0 = CoherentBox::<'_, [u64]>::zeroed_slice( - dev, - GSP_PAGE_SIZE / size_of::(), - GFP_KERNEL - )?; - level0[0] = level1_entry_addr.to_le(); - - level0.into() - }, - size, + radix3 <- Radix3::new(dev, fw_vvec), signatures, bootloader: { let bl = request_tlv(dev, chipset, "gsp_bootloader")?; @@ -140,28 +70,13 @@ impl<'a> GspFirmware<'a> { }) } - /// Returns the DMA address of the radix3 level 0 page table. - pub(crate) fn radix3_dma_address(&self) -> DmaAddress { - self.level0.dma_address() + /// Returns the size of the firmware image, in bytes. + pub(crate) fn size(&self) -> usize { + self.radix3.size() } -} -/// Build a page table from a scatter-gather list. -/// -/// Takes each DMA-mapped region from `sg_table` and writes page table entries -/// for all 4KB pages within that region. For example, a 16KB SG entry becomes -/// 4 consecutive page table entries. -fn map_into_lvl(sg_table: &SGTable>>, mut dst: VVec) -> Result> { - for sg_entry in sg_table.iter() { - // Number of pages we need to map. - let num_pages = usize::from_safe_cast(sg_entry.dma_len()).div_ceil(GSP_PAGE_SIZE); - - for i in 0..num_pages { - let entry = sg_entry.dma_address() - + (u64::from_safe_cast(i) * u64::from_safe_cast(GSP_PAGE_SIZE)); - dst.extend_from_slice(&entry.to_le_bytes(), GFP_KERNEL)?; - } + /// Returns the DMA address of the radix3 level 0 page table. + pub(crate) fn radix3_dma_address(&self) -> DmaAddress { + self.radix3.dma_address() } - - Ok(dst) } diff --git a/drivers/gpu/nova-core/firmware/radix3.rs b/drivers/gpu/nova-core/firmware/radix3.rs new file mode 100644 index 000000000000..225a3dd1b4a9 --- /dev/null +++ b/drivers/gpu/nova-core/firmware/radix3.rs @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +use kernel::{ + device, + dma::{ + Coherent, + CoherentBox, + DataDirection, + DmaAddress, // + }, + prelude::*, + scatterlist::{ + Owned, + SGTable, // + }, +}; + +use crate::{ + gsp::GSP_PAGE_SIZE, + num::FromSafeCast, // +}; + +/// GSP firmware with 3-level radix page tables for the GSP bootloader. +/// +/// The bootloader expects firmware to be mapped starting at address 0 in GSP's virtual address +/// space: +/// +/// ```text +/// Level 0: 1 page, 1 entry -> points to first level 1 page +/// Level 1: Multiple pages/entries -> each entry points to a level 2 page +/// Level 2: Multiple pages/entries -> each entry points to a firmware page +/// ``` +/// +/// Each page is 4KB, each entry is 8 bytes (64-bit DMA address). +/// Also known as "Radix3" firmware. +#[pin_data] +pub(crate) struct Radix3<'a> { + /// The GSP firmware inside a [`VVec`], device-mapped via a SG table. + #[pin] + fw: SGTable>>, + /// Level 2 page table whose entries contain DMA addresses of firmware pages. + #[pin] + level2: SGTable>>, + /// Level 1 page table whose entries contain DMA addresses of level 2 pages. + #[pin] + level1: SGTable>>, + /// Level 0 page table (single 4KB page) with one entry: DMA address of first level 1 page. + level0: Coherent<'a, [u64]>, + /// Size in bytes of the firmware contained in [`Self::fw`]. + size: usize, +} + +impl<'a> Radix3<'a> { + /// Builds a radix3 page table over `fw_vvec`, mapped for `dev` to read. May sleep. + pub(crate) fn new( + dev: &'a device::Device, + fw_vvec: VVec, + ) -> impl PinInit + 'a { + let size = fw_vvec.len(); + + pin_init::pin_init_scope(move || { + Ok(try_pin_init!(Self { + fw <- SGTable::new(dev, fw_vvec, DataDirection::ToDevice, GFP_KERNEL), + level2 <- { + // Allocate the level 2 page table, map the firmware onto it, and map it into + // the device address space. + VVec::::with_capacity( + fw.iter().count() * core::mem::size_of::(), + GFP_KERNEL, + ) + .map_err(|_| ENOMEM) + .and_then(|level2| map_into_lvl(&fw, level2)) + .map(|level2| SGTable::new(dev, level2, DataDirection::ToDevice, GFP_KERNEL))? + }, + level1 <- { + // Allocate the level 1 page table, map the level 2 page table onto it, and map + // it into the device address space. + VVec::::with_capacity( + level2.iter().count() * core::mem::size_of::(), + GFP_KERNEL, + ) + .map_err(|_| ENOMEM) + .and_then(|level1| map_into_lvl(&level2, level1)) + .map(|level1| SGTable::new(dev, level1, DataDirection::ToDevice, GFP_KERNEL))? + }, + level0: { + // Allocate the level 0 page table as a device-visible DMA object, and map the + // level 1 page table onto it. + + // Fill level 1 page entry. + let level1_entry = level1.iter().next().ok_or(EINVAL)?; + let level1_entry_addr = level1_entry.dma_address(); + + // Create level 0 page table data and fill its first entry with the level 1 + // table. + let mut level0 = CoherentBox::<'_, [u64]>::zeroed_slice( + dev, + GSP_PAGE_SIZE / size_of::(), + GFP_KERNEL + )?; + level0[0] = level1_entry_addr.to_le(); + + level0.into() + }, + size, + })) + }) + } + + /// Returns the DMA address of the radix3 level 0 page table. + pub(crate) fn dma_address(&self) -> DmaAddress { + self.level0.dma_address() + } + + /// Returns the length of the mapped data, in bytes. + pub(crate) fn size(&self) -> usize { + self.size + } +} + +/// Build a page table from a scatter-gather list. +/// +/// Takes each DMA-mapped region from `sg_table` and writes page table entries +/// for all 4KB pages within that region. For example, a 16KB SG entry becomes +/// 4 consecutive page table entries. +fn map_into_lvl(sg_table: &SGTable>>, mut dst: VVec) -> Result> { + for sg_entry in sg_table.iter() { + // Number of pages we need to map. + let num_pages = usize::from_safe_cast(sg_entry.dma_len()).div_ceil(GSP_PAGE_SIZE); + + for i in 0..num_pages { + let entry = sg_entry.dma_address() + + (u64::from_safe_cast(i) * u64::from_safe_cast(GSP_PAGE_SIZE)); + dst.extend_from_slice(&entry.to_le_bytes(), GFP_KERNEL)?; + } + } + + Ok(dst) +} diff --git a/drivers/gpu/nova-core/gsp/fw.rs b/drivers/gpu/nova-core/gsp/fw.rs index 8778c4bf79c0..ae98da42410b 100644 --- a/drivers/gpu/nova-core/gsp/fw.rs +++ b/drivers/gpu/nova-core/gsp/fw.rs @@ -187,7 +187,7 @@ impl GspFwWprMeta { magic: bindings::GSP_FW_WPR_META_MAGIC as u64, revision: u64::from(bindings::GSP_FW_WPR_META_REVISION), sysmemAddrOfRadix3Elf: gsp_firmware.radix3_dma_address(), - sizeOfRadix3Elf: u64::from_safe_cast(gsp_firmware.size), + sizeOfRadix3Elf: u64::from_safe_cast(gsp_firmware.size()), sysmemAddrOfBootloader: gsp_firmware.bootloader.ucode.dma_address(), sizeOfBootloader: u64::from_safe_cast(gsp_firmware.bootloader.ucode.size()), bootloaderCodeOffset: u64::from(gsp_firmware.bootloader.code_offset), @@ -242,7 +242,7 @@ impl GspFwWprMeta { magic: bindings::GSP_FW_WPR_META_MAGIC as u64, revision: u64::from(bindings::GSP_FW_WPR_META_REVISION), sysmemAddrOfRadix3Elf: gsp_firmware.radix3_dma_address(), - sizeOfRadix3Elf: u64::from_safe_cast(gsp_firmware.size), + sizeOfRadix3Elf: u64::from_safe_cast(gsp_firmware.size()), sysmemAddrOfBootloader: gsp_firmware.bootloader.ucode.dma_address(), sizeOfBootloader: u64::from_safe_cast(gsp_firmware.bootloader.ucode.size()), bootloaderCodeOffset: u64::from(gsp_firmware.bootloader.code_offset),