Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145// SPDX-License-Identifier: GPL-2.0// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
//! The radix3 page table, through which firmware running on the GPU reads a buffer in system//! memory. LibOS, the operating system of the GSP, defines the format: three levels of//! [`GSP_PAGE_SIZE`] pages, each entry the little-endian DMA address of one page.//!//! ```text//! Level 0: one page, one entry -> the first level 1 page//! Level 1: pages of entries -> each entry a level 2 page//! Level 2: pages of entries -> each entry a page of the buffer//! ```
use kernel::{ device, dma::{ Coherent, CoherentBox, DataDirection, DmaAddress, // }, prelude::*, scatterlist::{ Owned, SGTable, // },};
use crate::{ gsp::GSP_PAGE_SIZE, num::FromSafeCast, //};
/// A radix3 page table and the data it maps.#[pin_data]pub(crate) struct Radix3<'a> { /// The mapped data. #[pin] data: SGTable<Owned<VVec<u8>>>, /// Level 2: one entry per page of `data`. #[pin] level2: SGTable<Owned<VVec<u8>>>, /// Level 1: one entry per page of `level2`. #[pin] level1: SGTable<Owned<VVec<u8>>>, /// Level 0: one page, whose single entry is the DMA address of the first `level1` page. level0: Coherent<'a, [u64]>, /// Length of `data`, in bytes. size: usize,}
impl<'a> Radix3<'a> { /// Builds a radix3 page table over `data`, mapped for `dev` to read. May sleep. pub(crate) fn new( dev: &'a device::Device<device::Bound>, data: VVec<u8>, ) -> impl PinInit<Self, Error> + 'a { let size = data.len();
pin_init::pin_init_scope(move || { Ok(try_pin_init!(Self { data <- SGTable::new(dev, data, DataDirection::ToDevice, GFP_KERNEL), level2 <- { build_lvl(&data) .map(|l2| SGTable::new(dev, l2, DataDirection::ToDevice, GFP_KERNEL))? }, level1 <- { build_lvl(&level2) .map(|l1| SGTable::new(dev, l1, DataDirection::ToDevice, GFP_KERNEL))? }, level0: { // Allocate the level 0 page table as a device-visible DMA object, and map the // level 1 page table onto it.
// Fill level 1 page entry. let level1_entry = level1.iter().next().ok_or(EINVAL)?; let level1_entry_addr = level1_entry.dma_address();
// Create level 0 page table data and fill its first entry with the level 1 // table. let mut level0 = CoherentBox::<'_, [u64]>::zeroed_slice( dev, GSP_PAGE_SIZE / size_of::<u64>(), GFP_KERNEL )?; level0[0] = level1_entry_addr.to_le();
level0.into() }, size, })) }) }
/// Returns the DMA address of the level 0 page, which is the address of the table. pub(crate) fn dma_address(&self) -> DmaAddress { self.level0.dma_address() }
/// Returns the length of the mapped data, in bytes. pub(crate) fn size(&self) -> usize { self.size }}
/// Returns the size of the level that maps `sg_table`: one `u64` entry per [`GSP_PAGE_SIZE`]/// page of each DMA-mapped region, rounded up to whole pages.fn lvl_size(sg_table: &SGTable<Owned<VVec<u8>>>) -> usize { let entries: usize = sg_table .iter() .map(|sg_entry| usize::from_safe_cast(sg_entry.dma_len()).div_ceil(GSP_PAGE_SIZE)) .sum();
(entries * size_of::<u64>()).next_multiple_of(GSP_PAGE_SIZE)}
/// Builds one level of the table over `sg_table`: one entry per [`GSP_PAGE_SIZE`] page of each/// DMA-mapped region, in region order. The level is a whole number of pages and every byte past/// the last entry is zero, because the booter reads a level a whole page at a time.////// # Errors////// - `ENOMEM` if the level cannot be allocated./// - `EINVAL` if `sg_table` yields more entries than [`lvl_size`] sized the level for.fn build_lvl(sg_table: &SGTable<Owned<VVec<u8>>>) -> Result<VVec<u8>> { let mut dst = VVec::<u8>::zeroed(lvl_size(sg_table), GFP_KERNEL).map_err(|_| ENOMEM)?; let mut entries = dst.chunks_exact_mut(size_of::<u64>());
for sg_entry in sg_table.iter() { let num_pages = usize::from_safe_cast(sg_entry.dma_len()).div_ceil(GSP_PAGE_SIZE);
for i in 0..num_pages { let entry = sg_entry.dma_address() + (u64::from_safe_cast(i) * u64::from_safe_cast(GSP_PAGE_SIZE));
entries .next() .ok_or(EINVAL)? .copy_from_slice(&entry.to_le_bytes()); } }
Ok(dst)}