tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2023-2024 Intel Corporation
4 */
5
6#include <drm/drm_debugfs.h>
7#include <drm/drm_managed.h>
8
9#include "xe_assert.h"
10#include "xe_device.h"
11#include "xe_gt.h"
12#include "xe_gt_sriov_printk.h"
13#include "xe_gt_sriov_vf.h"
14#include "xe_guc.h"
15#include "xe_guc_ct.h"
16#include "xe_guc_submit.h"
17#include "xe_irq.h"
18#include "xe_lrc.h"
19#include "xe_pm.h"
20#include "xe_sriov.h"
21#include "xe_sriov_printk.h"
22#include "xe_sriov_vf.h"
23#include "xe_sriov_vf_ccs.h"
24#include "xe_tile_sriov_vf.h"
25
26/**
27 * DOC: VF restore procedure in PF KMD and VF KMD
28 *
29 * Restoring previously saved state of a VF is one of core features of
30 * SR-IOV. All major VM Management applications allow saving and restoring
31 * the VM state, and doing that to a VM which uses SRIOV VF as one of
32 * the accessible devices requires support from KMD on both PF and VF side.
33 * VMM initiates all required operations through VFIO module, which then
34 * translates them into PF KMD calls. This description will focus on these
35 * calls, leaving out the module which initiates these steps (VFIO).
36 *
37 * In order to start the restore procedure, GuC needs to keep the VF in
38 * proper state. The PF driver can ensure GuC set it to VF_READY state
39 * by provisioning the VF, which in turn can be done after Function Level
40 * Reset of said VF (or after it was freshly created - in that case FLR
41 * is not needed). The FLR procedure ends with GuC sending message
42 * `GUC_PF_NOTIFY_VF_FLR_DONE`, and then provisioning data is sent to GuC.
43 * After the provisioning is completed, the VF needs to be paused, and
44 * at that point the actual restore can begin.
45 *
46 * During VF Restore, state of several resources is restored. These may
47 * include local memory content (system memory is restored by VMM itself),
48 * values of MMIO registers, stateless compression metadata and others.
49 * The final resource which also needs restoring is state of the VF
50 * submission maintained within GuC. For that, `GUC_PF_OPCODE_VF_RESTORE`
51 * message is used, with reference to the state blob to be consumed by
52 * GuC.
53 *
54 * Next, when VFIO is asked to set the VM into running state, the PF driver
55 * sends `GUC_PF_TRIGGER_VF_RESUME` to GuC. When sent after restore, this
56 * changes VF state within GuC to `VF_RESFIX_BLOCKED` rather than the
57 * usual `VF_RUNNING`. At this point GuC triggers an interrupt to inform
58 * the VF KMD within the VM that it was migrated.
59 *
60 * As soon as Virtual GPU of the VM starts, the VF driver within receives
61 * the MIGRATED interrupt and schedules post-migration recovery worker.
62 * That worker queries GuC for new provisioning (using MMIO communication),
63 * and applies fixups to any non-virtualized resources used by the VF.
64 *
65 * When the VF driver is ready to continue operation on the newly connected
66 * hardware, it sends `VF2GUC_NOTIFY_RESFIX_DONE` which causes it to
67 * enter the long awaited `VF_RUNNING` state, and therefore start handling
68 * CTB messages and scheduling workloads from the VF::
69 *
70 * PF GuC VF
71 * [ ] | |
72 * [ ] PF2GUC_VF_CONTROL(pause) | |
73 * [ ]---------------------------> [ ] |
74 * [ ] [ ] GuC sets new VF state to |
75 * [ ] [ ]------- VF_READY_PAUSED |
76 * [ ] [ ] | |
77 * [ ] [ ] <----- |
78 * [ ] success [ ] |
79 * [ ] <---------------------------[ ] |
80 * [ ] | |
81 * [ ] PF loads resources from the | |
82 * [ ]------- saved image supplied | |
83 * [ ] | | |
84 * [ ] <----- | |
85 * [ ] | |
86 * [ ] GUC_PF_OPCODE_VF_RESTORE | |
87 * [ ]---------------------------> [ ] |
88 * [ ] [ ] GuC loads contexts and CTB |
89 * [ ] [ ]------- state from image |
90 * [ ] [ ] | |
91 * [ ] [ ] <----- |
92 * [ ] [ ] |
93 * [ ] [ ] GuC sets new VF state to |
94 * [ ] [ ]------- VF_RESFIX_PAUSED |
95 * [ ] [ ] | |
96 * [ ] success [ ] <----- |
97 * [ ] <---------------------------[ ] |
98 * [ ] | |
99 * [ ] GUC_PF_TRIGGER_VF_RESUME | |
100 * [ ]---------------------------> [ ] |
101 * [ ] [ ] GuC sets new VF state to |
102 * [ ] [ ]------- VF_RESFIX_BLOCKED |
103 * [ ] [ ] | |
104 * [ ] [ ] <----- |
105 * [ ] [ ] |
106 * [ ] [ ] GUC_INTR_SW_INT_0 |
107 * [ ] success [ ]---------------------------> [ ]
108 * [ ] <---------------------------[ ] [ ]
109 * | | VF2GUC_QUERY_SINGLE_KLV [ ]
110 * | [ ] <---------------------------[ ]
111 * | [ ] [ ]
112 * | [ ] new VF provisioning [ ]
113 * | [ ]---------------------------> [ ]
114 * | | [ ]
115 * | | VF driver applies post [ ]
116 * | | migration fixups -------[ ]
117 * | | | [ ]
118 * | | -----> [ ]
119 * | | [ ]
120 * | | VF2GUC_NOTIFY_RESFIX_DONE [ ]
121 * | [ ] <---------------------------[ ]
122 * | [ ] [ ]
123 * | [ ] GuC sets new VF state to [ ]
124 * | [ ]------- VF_RUNNING [ ]
125 * | [ ] | [ ]
126 * | [ ] <----- [ ]
127 * | [ ] success [ ]
128 * | [ ]---------------------------> [ ]
129 * | | |
130 * | | |
131 */
132
133/**
134 * xe_sriov_vf_migration_supported - Report whether SR-IOV VF migration is
135 * supported or not.
136 * @xe: the &xe_device to check
137 *
138 * Returns: true if VF migration is supported, false otherwise.
139 */
140bool xe_sriov_vf_migration_supported(struct xe_device *xe)
141{
142 xe_assert(xe, IS_SRIOV_VF(xe));
143 return xe->sriov.vf.migration.enabled;
144}
145
146static void vf_disable_migration(struct xe_device *xe, const char *fmt, ...)
147{
148 struct va_format vaf;
149 va_list va_args;
150
151 xe_assert(xe, IS_SRIOV_VF(xe));
152
153 va_start(va_args, fmt);
154 vaf.fmt = fmt;
155 vaf.va = &va_args;
156 xe_sriov_notice(xe, "migration disabled: %pV\n", &vaf);
157 va_end(va_args);
158
159 xe->sriov.vf.migration.enabled = false;
160}
161
162static void migration_worker_func(struct work_struct *w);
163
164static void vf_migration_init_early(struct xe_device *xe)
165{
166 /*
167 * TODO: Add conditions to allow specific platforms, when they're
168 * supported at production quality.
169 */
170 if (!IS_ENABLED(CONFIG_DRM_XE_DEBUG))
171 return vf_disable_migration(xe,
172 "experimental feature not available on production builds");
173
174 if (GRAPHICS_VER(xe) < 20)
175 return vf_disable_migration(xe, "requires gfx version >= 20, but only %u found",
176 GRAPHICS_VER(xe));
177
178 if (!IS_DGFX(xe)) {
179 struct xe_uc_fw_version guc_version;
180
181 xe_gt_sriov_vf_guc_versions(xe_device_get_gt(xe, 0), NULL, &guc_version);
182 if (MAKE_GUC_VER_STRUCT(guc_version) < MAKE_GUC_VER(1, 23, 0))
183 return vf_disable_migration(xe,
184 "CCS migration requires GuC ABI >= 1.23 but only %u.%u found",
185 guc_version.major, guc_version.minor);
186 }
187
188 INIT_WORK(&xe->sriov.vf.migration.worker, migration_worker_func);
189
190 xe->sriov.vf.migration.enabled = true;
191 xe_sriov_dbg(xe, "migration support enabled\n");
192}
193
194/**
195 * xe_sriov_vf_init_early - Initialize SR-IOV VF specific data.
196 * @xe: the &xe_device to initialize
197 */
198void xe_sriov_vf_init_early(struct xe_device *xe)
199{
200 vf_migration_init_early(xe);
201}
202
203/**
204 * vf_post_migration_shutdown - Stop the driver activities after VF migration.
205 * @xe: the &xe_device struct instance
206 *
207 * After this VM is migrated and assigned to a new VF, it is running on a new
208 * hardware, and therefore many hardware-dependent states and related structures
209 * require fixups. Without fixups, the hardware cannot do any work, and therefore
210 * all GPU pipelines are stalled.
211 * Stop some of kernel activities to make the fixup process faster.
212 */
213static void vf_post_migration_shutdown(struct xe_device *xe)
214{
215 struct xe_gt *gt;
216 unsigned int id;
217 int ret = 0;
218
219 for_each_gt(gt, xe, id) {
220 xe_guc_submit_pause(>->uc.guc);
221 ret |= xe_guc_submit_reset_block(>->uc.guc);
222 }
223
224 if (ret)
225 drm_info(&xe->drm, "migration recovery encountered ongoing reset\n");
226}
227
228/**
229 * vf_post_migration_kickstart - Re-start the driver activities under new hardware.
230 * @xe: the &xe_device struct instance
231 *
232 * After we have finished with all post-migration fixups, restart the driver
233 * activities to continue feeding the GPU with workloads.
234 */
235static void vf_post_migration_kickstart(struct xe_device *xe)
236{
237 struct xe_gt *gt;
238 unsigned int id;
239
240 /*
241 * Make sure interrupts on the new HW are properly set. The GuC IRQ
242 * must be working at this point, since the recovery did started,
243 * but the rest was not enabled using the procedure from spec.
244 */
245 xe_irq_resume(xe);
246
247 for_each_gt(gt, xe, id) {
248 xe_guc_submit_reset_unblock(>->uc.guc);
249 xe_guc_submit_unpause(>->uc.guc);
250 }
251}
252
253static bool gt_vf_post_migration_needed(struct xe_gt *gt)
254{
255 return test_bit(gt->info.id, >_to_xe(gt)->sriov.vf.migration.gt_flags);
256}
257
258/*
259 * Notify GuCs marked in flags about resource fixups apply finished.
260 * @xe: the &xe_device struct instance
261 * @gt_flags: flags marking to which GTs the notification shall be sent
262 */
263static int vf_post_migration_notify_resfix_done(struct xe_device *xe, unsigned long gt_flags)
264{
265 struct xe_gt *gt;
266 unsigned int id;
267 int err = 0;
268
269 for_each_gt(gt, xe, id) {
270 if (!test_bit(id, >_flags))
271 continue;
272 /* skip asking GuC for RESFIX exit if new recovery request arrived */
273 if (gt_vf_post_migration_needed(gt))
274 continue;
275 err = xe_gt_sriov_vf_notify_resfix_done(gt);
276 if (err)
277 break;
278 clear_bit(id, >_flags);
279 }
280
281 if (gt_flags && !err)
282 drm_dbg(&xe->drm, "another recovery imminent, skipped some notifications\n");
283 return err;
284}
285
286static int vf_get_next_migrated_gt_id(struct xe_device *xe)
287{
288 struct xe_gt *gt;
289 unsigned int id;
290
291 for_each_gt(gt, xe, id) {
292 if (test_and_clear_bit(id, &xe->sriov.vf.migration.gt_flags))
293 return id;
294 }
295 return -1;
296}
297
298static size_t post_migration_scratch_size(struct xe_device *xe)
299{
300 return max(xe_lrc_reg_size(xe), LRC_WA_BB_SIZE);
301}
302
303/**
304 * Perform post-migration fixups on a single GT.
305 *
306 * After migration, GuC needs to be re-queried for VF configuration to check
307 * if it matches previous provisioning. Most of VF provisioning shall be the
308 * same, except GGTT range, since GGTT is not virtualized per-VF. If GGTT
309 * range has changed, we have to perform fixups - shift all GGTT references
310 * used anywhere within the driver. After the fixups in this function succeed,
311 * it is allowed to ask the GuC bound to this GT to continue normal operation.
312 *
313 * Returns: 0 if the operation completed successfully, or a negative error
314 * code otherwise.
315 */
316static int gt_vf_post_migration_fixups(struct xe_gt *gt)
317{
318 s64 shift;
319 void *buf;
320 int err;
321
322 buf = kmalloc(post_migration_scratch_size(gt_to_xe(gt)), GFP_KERNEL);
323 if (!buf)
324 return -ENOMEM;
325
326 err = xe_gt_sriov_vf_query_config(gt);
327 if (err)
328 goto out;
329
330 shift = xe_gt_sriov_vf_ggtt_shift(gt);
331 if (shift) {
332 xe_tile_sriov_vf_fixup_ggtt_nodes(gt_to_tile(gt), shift);
333 xe_gt_sriov_vf_default_lrcs_hwsp_rebase(gt);
334 err = xe_guc_contexts_hwsp_rebase(>->uc.guc, buf);
335 if (err)
336 goto out;
337 xe_guc_jobs_ring_rebase(>->uc.guc);
338 xe_guc_ct_fixup_messages_with_ggtt(>->uc.guc.ct, shift);
339 }
340
341out:
342 kfree(buf);
343 return err;
344}
345
346static void vf_post_migration_recovery(struct xe_device *xe)
347{
348 unsigned long fixed_gts = 0;
349 int id, err;
350
351 drm_dbg(&xe->drm, "migration recovery in progress\n");
352 xe_pm_runtime_get(xe);
353 vf_post_migration_shutdown(xe);
354
355 if (!xe_sriov_vf_migration_supported(xe)) {
356 xe_sriov_err(xe, "migration is not supported\n");
357 err = -ENOTRECOVERABLE;
358 goto fail;
359 }
360
361 while (id = vf_get_next_migrated_gt_id(xe), id >= 0) {
362 struct xe_gt *gt = xe_device_get_gt(xe, id);
363
364 err = gt_vf_post_migration_fixups(gt);
365 if (err)
366 goto fail;
367
368 set_bit(id, &fixed_gts);
369 }
370
371 vf_post_migration_kickstart(xe);
372 err = vf_post_migration_notify_resfix_done(xe, fixed_gts);
373 if (err)
374 goto fail;
375
376 xe_pm_runtime_put(xe);
377 drm_notice(&xe->drm, "migration recovery ended\n");
378 return;
379fail:
380 xe_pm_runtime_put(xe);
381 drm_err(&xe->drm, "migration recovery failed (%pe)\n", ERR_PTR(err));
382 xe_device_declare_wedged(xe);
383}
384
385static void migration_worker_func(struct work_struct *w)
386{
387 struct xe_device *xe = container_of(w, struct xe_device,
388 sriov.vf.migration.worker);
389
390 vf_post_migration_recovery(xe);
391}
392
393/*
394 * Check if post-restore recovery is coming on any of GTs.
395 * @xe: the &xe_device struct instance
396 *
397 * Return: True if migration recovery worker will soon be running. Any worker currently
398 * executing does not affect the result.
399 */
400static bool vf_ready_to_recovery_on_any_gts(struct xe_device *xe)
401{
402 struct xe_gt *gt;
403 unsigned int id;
404
405 for_each_gt(gt, xe, id) {
406 if (test_bit(id, &xe->sriov.vf.migration.gt_flags))
407 return true;
408 }
409 return false;
410}
411
412/**
413 * xe_sriov_vf_start_migration_recovery - Start VF migration recovery.
414 * @xe: the &xe_device to start recovery on
415 *
416 * This function shall be called only by VF.
417 */
418void xe_sriov_vf_start_migration_recovery(struct xe_device *xe)
419{
420 bool started;
421
422 xe_assert(xe, IS_SRIOV_VF(xe));
423
424 if (!vf_ready_to_recovery_on_any_gts(xe))
425 return;
426
427 started = queue_work(xe->sriov.wq, &xe->sriov.vf.migration.worker);
428 drm_info(&xe->drm, "VF migration recovery %s\n", started ?
429 "scheduled" : "already in progress");
430}
431
432/**
433 * xe_sriov_vf_init_late() - SR-IOV VF late initialization functions.
434 * @xe: the &xe_device to initialize
435 *
436 * This function initializes code for CCS migration.
437 *
438 * Return: 0 on success or a negative error code on failure.
439 */
440int xe_sriov_vf_init_late(struct xe_device *xe)
441{
442 int err = 0;
443
444 if (xe_sriov_vf_migration_supported(xe))
445 err = xe_sriov_vf_ccs_init(xe);
446
447 return err;
448}
449
450static int sa_info_vf_ccs(struct seq_file *m, void *data)
451{
452 struct drm_info_node *node = m->private;
453 struct xe_device *xe = to_xe_device(node->minor->dev);
454 struct drm_printer p = drm_seq_file_printer(m);
455
456 xe_sriov_vf_ccs_print(xe, &p);
457 return 0;
458}
459
460static const struct drm_info_list debugfs_list[] = {
461 { .name = "sa_info_vf_ccs", .show = sa_info_vf_ccs },
462};
463
464/**
465 * xe_sriov_vf_debugfs_register - Register VF debugfs attributes.
466 * @xe: the &xe_device
467 * @root: the root &dentry
468 *
469 * Prepare debugfs attributes exposed by the VF.
470 */
471void xe_sriov_vf_debugfs_register(struct xe_device *xe, struct dentry *root)
472{
473 drm_debugfs_create_files(debugfs_list, ARRAY_SIZE(debugfs_list),
474 root, xe->drm.primary);
475}