Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297// SPDX-License-Identifier: MIT/* * Copyright © 2026 Intel Corporation */
#include <drm/drm_managed.h>#include <linux/mutex.h>
#include "xe_cpu_bind.h"#include "xe_device_types.h"#include "xe_exec_queue.h"#include "xe_pt.h"#include "xe_sched_job.h"#include "xe_trace_bo.h"#include "xe_vm.h"
/** * struct xe_cpu_bind - cpu_bind context. */struct xe_cpu_bind { /** @xe: Xe device */ struct xe_device *xe; /** @q: Default exec queue used for kernel binds */ struct xe_exec_queue *q; /** @job_mutex: Timeline mutex for @q. */ struct mutex job_mutex;};
static bool is_cpu_bind_queue(struct xe_cpu_bind *cpu_bind, struct xe_exec_queue *q){ return cpu_bind->q == q;}
static void xe_cpu_bind_fini(void *arg){ struct xe_cpu_bind *cpu_bind = arg;
mutex_destroy(&cpu_bind->job_mutex); xe_exec_queue_put(cpu_bind->q);}
/** * xe_cpu_bind_init() - Initialize a cpu_bind context * @xe: &struct xe_device * * Return: 0 if successful, negative error code on failure */int xe_cpu_bind_init(struct xe_device *xe){ struct xe_cpu_bind *cpu_bind = drmm_kzalloc(&xe->drm, sizeof(*cpu_bind), GFP_KERNEL); struct xe_exec_queue *q;
if (!cpu_bind) return -ENOMEM;
q = xe_exec_queue_create_bind(xe, xe_device_get_root_tile(xe), NULL, EXEC_QUEUE_FLAG_KERNEL | EXEC_QUEUE_FLAG_MIGRATE, 0); if (IS_ERR(q)) return PTR_ERR(q);
cpu_bind->xe = xe; cpu_bind->q = q; xe->cpu_bind = cpu_bind;
mutex_init(&cpu_bind->job_mutex);
fs_reclaim_acquire(GFP_KERNEL); might_lock(&cpu_bind->job_mutex); fs_reclaim_release(GFP_KERNEL);
return devm_add_action_or_reset(cpu_bind->xe->drm.dev, xe_cpu_bind_fini, cpu_bind);}
/** * xe_cpu_bind_queue() - Get the bind queue from cpu_bind context. * @cpu_bind: The cpu bind context. * * Return: Pointer to bind queue. */struct xe_exec_queue *xe_cpu_bind_queue(struct xe_cpu_bind *cpu_bind){ return cpu_bind->q;}
/** * xe_cpu_bind_update_pgtables_execute() - Update a VM's PTEs via the CPU * @vm: The VM being updated * @tile: The tile being updated * @ops: The CPU bind PT update ops * @pt_op: The VM PT update op * @num_ops: The number of The VM PT update ops * @force_clear: Force clear operation * * Execute the VM PT update ops array which results in a VM's PTEs being updated * via the CPU. */voidxe_cpu_bind_update_pgtables_execute(struct xe_vm *vm, struct xe_tile *tile, const struct xe_cpu_bind_pt_update_ops *ops, struct xe_vm_pgtable_update_op *pt_op, u32 num_ops, bool force_clear){ u32 j, i;
for (j = 0; j < num_ops; ++j, ++pt_op) { for (i = 0; i < pt_op->num_entries; i++) { const struct xe_vm_pgtable_update *update = &pt_op->entries[i];
xe_assert(vm->xe, !iosys_map_is_null(&update->pt_bo->vmap));
if (pt_op->bind && !force_clear) ops->populate(tile, &update->pt_bo->vmap, update); else ops->clear(vm, tile, &update->pt_bo->vmap, update); } }
trace_xe_vm_cpu_bind(vm); xe_device_wmb(vm->xe);}
static struct dma_fence *xe_cpu_bind_update_pgtables_no_job(struct xe_cpu_bind *cpu_bind, struct xe_cpu_bind_pt_update *pt_update){ const struct xe_cpu_bind_pt_update_ops *ops = pt_update->ops; struct xe_vm *vm = pt_update->vops->vm; struct xe_tile *tile; int err, id;
if (ops->pre_commit) { pt_update->job = NULL; err = ops->pre_commit(pt_update); if (err) return ERR_PTR(err); }
for_each_tile(tile, vm->xe, id) { struct xe_vm_pgtable_update_ops *pt_update_ops = &pt_update->vops->pt_update_ops[tile->id];
if (!pt_update_ops->pt_job_ops) continue;
xe_cpu_bind_update_pgtables_execute(vm, tile, ops, pt_update_ops->pt_job_ops->ops, pt_update_ops->pt_job_ops->current_op, false); }
return dma_fence_get_stub();}
static struct dma_fence *xe_cpu_bind_update_pgtables_job(struct xe_cpu_bind *cpu_bind, struct xe_cpu_bind_pt_update *pt_update){ const struct xe_cpu_bind_pt_update_ops *ops = pt_update->ops; struct xe_exec_queue *q = pt_update->vops->q; struct xe_device *xe = cpu_bind->xe; struct xe_sched_job *job; struct dma_fence *fence; struct xe_tile *tile; int err, id; bool is_cpu_bind = is_cpu_bind_queue(cpu_bind, q);
job = xe_sched_job_create(q, NULL); if (IS_ERR(job)) return ERR_CAST(job);
xe_assert(xe, job->is_pt_job);
if (ops->pre_commit) { pt_update->job = job; err = ops->pre_commit(pt_update); if (err) goto err_job; }
if (is_cpu_bind) mutex_lock(&cpu_bind->job_mutex);
job->pt_update[0].vm = pt_update->vops->vm; job->pt_update[0].ops = ops; for_each_tile(tile, xe, id) { struct xe_vm_pgtable_update_ops *pt_update_ops = &pt_update->vops->pt_update_ops[tile->id];
job->pt_update[0].pt_job_ops[tile->id] = xe_pt_job_ops_get(pt_update_ops->pt_job_ops); }
xe_sched_job_arm(job); fence = dma_fence_get(&job->drm.s_fence->finished); xe_sched_job_push(job);
if (is_cpu_bind) mutex_unlock(&cpu_bind->job_mutex);
return fence;
err_job: xe_sched_job_put(job); return ERR_PTR(err);}
/** * xe_cpu_bind_update_pgtables() - Pipelined page-table update * @cpu_bind: The cpu bind context. * @pt_update: PT update arguments * * Perform a pipelined page-table update. The update descriptors are typically * built under the same lock critical section as a call to this function. If * using the default engine for the updates, they will be performed in the * order they grab the job_mutex. If different engines are used, external * synchronization is needed for overlapping updates to maintain page-table * consistency. Note that the meaning of "overlapping" is that the updates * touch the same page-table, which might be a higher-level page-directory. * If no pipelining is needed, then updates may be performed by the cpu. * * Return: A dma_fence that, when signaled, indicates the update completion. */struct dma_fence *xe_cpu_bind_update_pgtables(struct xe_cpu_bind *cpu_bind, struct xe_cpu_bind_pt_update *pt_update){ struct dma_fence *fence;
fence = xe_cpu_bind_update_pgtables_no_job(cpu_bind, pt_update);
/* -ETIME indicates a job is needed, anything else is legit error */ if (!IS_ERR(fence) || PTR_ERR(fence) != -ETIME) return fence;
return xe_cpu_bind_update_pgtables_job(cpu_bind, pt_update);}
/** * xe_cpu_bind_job_lock() - Lock cpu_bind job lock * @cpu_bind: The cpu bind context. * @q: Queue associated with the operation which requires a lock * * Lock the cpu_bind job lock if the queue is a cpu bind queue, otherwise * assert the VM's dma-resv is held (user queue's have own locking). */void xe_cpu_bind_job_lock(struct xe_cpu_bind *cpu_bind, struct xe_exec_queue *q){ bool is_cpu_bind = is_cpu_bind_queue(cpu_bind, q);
if (is_cpu_bind) mutex_lock(&cpu_bind->job_mutex); else xe_vm_assert_held(q->user_vm); /* User queues VM's should be locked */}
/** * xe_cpu_bind_job_unlock() - Unlock cpu_bind job lock * @cpu_bind: The cpu bind context. * @q: Queue associated with the operation which requires a lock * * Unlock the cpu_bind job lock if the queue is a cpu bind queue, otherwise * assert the VM's dma-resv is held (user queue's have own locking). */void xe_cpu_bind_job_unlock(struct xe_cpu_bind *cpu_bind, struct xe_exec_queue *q){ bool is_cpu_bind = is_cpu_bind_queue(cpu_bind, q);
if (is_cpu_bind) mutex_unlock(&cpu_bind->job_mutex); else xe_vm_assert_held(q->user_vm); /* User queues VM's should be locked */}
#if IS_ENABLED(CONFIG_PROVE_LOCKING)/** * xe_cpu_bind_job_lock_assert() - Assert cpu_bind job lock held of queue * @q: cpu bind queue */void xe_cpu_bind_job_lock_assert(struct xe_exec_queue *q){ struct xe_device *xe = gt_to_xe(q->gt); struct xe_cpu_bind *cpu_bind = xe->cpu_bind;
xe_assert(xe, q == cpu_bind->q); lockdep_assert_held(&cpu_bind->job_mutex);}#endif