Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329// SPDX-License-Identifier: GPL-2.0 OR MIT
#![cfg(CONFIG_RUST_DRM_GPUVM)]
//! DRM GPUVM in immediate mode//!//! Rust abstractions for using GPUVM in immediate mode. This is when the GPUVM state is updated//! during `run_job()`, i.e., in the DMA fence signalling critical path, to ensure that the GPUVM//! and the GPU's virtual address space has the same state at all times.//!//! C header: [`include/drm/drm_gpuvm.h`](srctree/include/drm/drm_gpuvm.h)
use kernel::{ alloc::{ AllocError, Flags as AllocFlags, // }, bindings, drm, drm::gem::IntoGEMObject, error::to_result, prelude::*, sync::aref::{ ARef, AlwaysRefCounted, // }, types::Opaque, //};
use core::{ cell::UnsafeCell, marker::PhantomData, mem::{ ManuallyDrop, MaybeUninit, // }, ops::{ Deref, DerefMut, Range, // }, ptr::{ self, NonNull, // }, //};
mod sm_ops;pub use self::sm_ops::*;
mod vm_bo;pub use self::vm_bo::*;
mod va;pub use self::va::*;
/// A DRM GPU VA manager.////// This object is refcounted, but the locations of mapped ranges may only be accessed or changed/// via the special unique handle [`UniqueRefGpuVm`].////// # Invariants////// * Stored in an allocation managed by the refcount in `self.vm`./// * Access to `data` and the gpuvm interval tree is controlled via the [`UniqueRefGpuVm`] type./// * Does not contain any sparse [`GpuVa<T>`] instances.#[pin_data]pub struct GpuVm<T: DriverGpuVm> { #[pin] vm: Opaque<bindings::drm_gpuvm>, /// Accessed only through the [`UniqueRefGpuVm`] reference. data: UnsafeCell<T>,}
// SAFETY: The GPUVM api does not assume that it is tied to a specific thread. The destructor will// drop the `data` field, which is okay because it is guaranteed `Send` by the `DriverGpuVm` trait.unsafe impl<T: DriverGpuVm> Send for GpuVm<T> {}// SAFETY: The GPUVM api is designed to allow &self methods to be called in parallel.unsafe impl<T: DriverGpuVm> Sync for GpuVm<T> {}
// SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`.unsafe impl<T: DriverGpuVm> AlwaysRefCounted for GpuVm<T> { fn inc_ref(&self) { // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`. unsafe { bindings::drm_gpuvm_get(self.vm.get()) }; }
unsafe fn dec_ref(obj: NonNull<Self>) { // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`. unsafe { bindings::drm_gpuvm_put((*obj.as_ptr()).vm.get()) }; }}
impl<T: DriverGpuVm> PartialEq for GpuVm<T> { #[inline] fn eq(&self, other: &Self) -> bool { core::ptr::eq(self.as_raw(), other.as_raw()) }}impl<T: DriverGpuVm> Eq for GpuVm<T> {}
impl<T: DriverGpuVm> GpuVm<T> { const fn vtable() -> &'static bindings::drm_gpuvm_ops { &bindings::drm_gpuvm_ops { vm_free: Some(Self::vm_free), op_alloc: None, op_free: None, vm_bo_alloc: GpuVmBo::<T>::ALLOC_FN, vm_bo_free: GpuVmBo::<T>::FREE_FN, vm_bo_validate: None, sm_step_map: Some(Self::sm_step_map), sm_step_unmap: Some(Self::sm_step_unmap), sm_step_remap: Some(Self::sm_step_remap), } }
/// Creates a GPUVM instance. #[expect(clippy::new_ret_no_self)] pub fn new<E>( name: &'static CStr, dev: &drm::Device<T::Driver>, r_obj: &T::Object, range: Range<u64>, reserve_range: Range<u64>, data: T, ) -> Result<UniqueRefGpuVm<T>, E> where E: From<AllocError>, E: From<core::convert::Infallible>, { let obj = KBox::try_pin_init::<E>( try_pin_init!(Self { data: UnsafeCell::new(data), vm <- Opaque::ffi_init(|vm| { // SAFETY: These arguments are valid. `vm` is valid until refcount drops to // zero. The `vm` is zeroed before calling this method by `__GFP_ZERO` flag // below. unsafe { bindings::drm_gpuvm_init( vm, name.as_char_ptr(), bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE | bindings::drm_gpuvm_flags_DRM_GPUVM_RESV_PROTECTED, dev.as_raw(), r_obj.as_raw(), range.start, range.end - range.start, reserve_range.start, reserve_range.end - reserve_range.start, const { Self::vtable() }, ) } }), }? E), GFP_KERNEL | __GFP_ZERO, )?; // SAFETY: This transfers the initial refcount to the ARef. let aref = unsafe { ARef::from_raw(NonNull::new_unchecked(KBox::into_raw( Pin::into_inner_unchecked(obj), ))) }; // INVARIANT: This reference is unique. Ok(UniqueRefGpuVm(aref)) }
/// Access this [`GpuVm`] from a raw pointer. /// /// # Safety /// /// The pointer must reference the `struct drm_gpuvm` in a valid [`GpuVm<T>`] that remains /// valid for at least `'a`. #[inline] pub unsafe fn from_raw<'a>(ptr: *mut bindings::drm_gpuvm) -> &'a Self { // SAFETY: Caller passes a pointer to the `drm_gpuvm` in a `GpuVm<T>`. Caller ensures the // pointer is valid for 'a. unsafe { &*kernel::container_of!(Opaque::cast_from(ptr), Self, vm) } }
/// Returns a raw pointer to the embedded `struct drm_gpuvm`. #[inline] pub fn as_raw(&self) -> *mut bindings::drm_gpuvm { self.vm.get() }
/// The start of the VA space. #[inline] pub fn va_start(&self) -> u64 { // SAFETY: The `mm_start` field is immutable. unsafe { (*self.as_raw()).mm_start } }
/// The length of the GPU's virtual address space. #[inline] pub fn va_length(&self) -> u64 { // SAFETY: The `mm_range` field is immutable. unsafe { (*self.as_raw()).mm_range } }
/// Returns the range of the GPU virtual address space. #[inline] pub fn va_range(&self) -> Range<u64> { let start = self.va_start(); // OVERFLOW: This reconstructs the Range<u64> passed to the constructor, so it won't fail. let end = start + self.va_length(); Range { start, end } }
/// Get or create the [`GpuVmBo`] for this gem object. #[inline] pub fn obtain( &self, obj: &T::Object, data: impl PinInit<T::VmBoData>, ) -> Result<ARef<GpuVmBo<T>>, AllocError> { Ok(GpuVmBoAlloc::new(self, obj, data)?.obtain()) }
/// Clean up buffer objects that are no longer used. #[inline] pub fn deferred_cleanup(&self) { // SAFETY: This GPUVM uses immediate mode. unsafe { bindings::drm_gpuvm_bo_deferred_cleanup(self.as_raw()) } }
/// Check if this GEM object is an external object for this GPUVM. #[inline] pub fn is_extobj(&self, obj: &T::Object) -> bool { // SAFETY: We may call this with any GPUVM and GEM object. unsafe { bindings::drm_gpuvm_is_extobj(self.as_raw(), obj.as_raw()) } }
/// Free this GPUVM. /// /// # Safety /// /// Called when refcount hits zero. unsafe extern "C" fn vm_free(me: *mut bindings::drm_gpuvm) { // SAFETY: Caller passes a pointer to the `drm_gpuvm` in a `GpuVm<T>`. let me = unsafe { kernel::container_of!(Opaque::cast_from(me), Self, vm).cast_mut() }; // SAFETY: By type invariants we can free it when refcount hits zero. drop(unsafe { KBox::from_raw(me) }) }
#[inline] fn raw_resv(&self) -> *mut bindings::dma_resv { // SAFETY: `r_obj` is immutable and valid for duration of GPUVM. unsafe { (*(*self.as_raw()).r_obj).resv } }}
/// The manager for a GPUVM.pub trait DriverGpuVm: Sized + Send { /// Parent `Driver` for this object. type Driver: drm::Driver<Object = Self::Object>;
/// The kind of GEM object stored in this GPUVM. type Object: IntoGEMObject;
/// Data stored with each [`struct drm_gpuva`](struct@GpuVa). type VaData;
/// Data stored with each [`struct drm_gpuvm_bo`](struct@GpuVmBo). type VmBoData;
/// The private data passed to callbacks. type SmContext<'ctx>;
/// Indicates that a new mapping should be created. fn sm_step_map<'op, 'ctx>( &mut self, op: OpMap<'op, Self>, context: &mut Self::SmContext<'ctx>, ) -> Result<OpMapped<'op, Self>, Error>;
/// Indicates that an existing mapping should be removed. fn sm_step_unmap<'op, 'ctx>( &mut self, op: OpUnmap<'op, Self>, context: &mut Self::SmContext<'ctx>, ) -> Result<OpUnmapped<'op, Self>, Error>;
/// Indicates that an existing mapping should be split up. fn sm_step_remap<'op, 'ctx>( &mut self, op: OpRemap<'op, Self>, context: &mut Self::SmContext<'ctx>, ) -> Result<OpRemapped<'op, Self>, Error>;}
/// The core of the DRM GPU VA manager.////// This object is a unique reference to the VM that can access the interval tree and the Rust/// `data` field.////// # Invariants////// Each `GpuVm` instance has at most one `UniqueRefGpuVm` reference.pub struct UniqueRefGpuVm<T: DriverGpuVm>(ARef<GpuVm<T>>);
// SAFETY: The GPUVM api is designed to allow &self methods to be called in parallel, and// concurrent access to `data` is safe due to the `T: Sync` requirement.unsafe impl<T: DriverGpuVm + Sync> Sync for UniqueRefGpuVm<T> {}
impl<T: DriverGpuVm> UniqueRefGpuVm<T> { /// Access the data owned by this `UniqueRefGpuVm` immutably. #[inline] pub fn data_ref(&self) -> &T { // SAFETY: By the type invariants we may access `data`. unsafe { &*self.0.data.get() } }
/// Access the data owned by this `UniqueRefGpuVm` mutably. #[inline] pub fn data(&mut self) -> &mut T { // SAFETY: By the type invariants we may access `data`. unsafe { &mut *self.0.data.get() } }}
impl<T: DriverGpuVm> Deref for UniqueRefGpuVm<T> { type Target = GpuVm<T>;
#[inline] fn deref(&self) -> &GpuVm<T> { &self.0 }}