tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2023-2024 Intel Corporation
4 */
5
6#include <drm/drm_managed.h>
7
8#include "regs/xe_guc_regs.h"
9#include "regs/xe_regs.h"
10
11#include "xe_gt.h"
12#include "xe_gt_sriov_pf.h"
13#include "xe_gt_sriov_pf_config.h"
14#include "xe_gt_sriov_pf_control.h"
15#include "xe_gt_sriov_pf_helpers.h"
16#include "xe_gt_sriov_pf_migration.h"
17#include "xe_gt_sriov_pf_service.h"
18#include "xe_gt_sriov_printk.h"
19#include "xe_guc_submit.h"
20#include "xe_mmio.h"
21#include "xe_pm.h"
22
23static void pf_worker_restart_func(struct work_struct *w);
24
25/*
26 * VF's metadata is maintained in the flexible array where:
27 * - entry [0] contains metadata for the PF (only if applicable),
28 * - entries [1..n] contain metadata for VF1..VFn::
29 *
30 * <--------------------------- 1 + total_vfs ----------->
31 * +-------+-------+-------+-----------------------+-------+
32 * | 0 | 1 | 2 | | n |
33 * +-------+-------+-------+-----------------------+-------+
34 * | PF | VF1 | VF2 | ... ... | VFn |
35 * +-------+-------+-------+-----------------------+-------+
36 */
37static int pf_alloc_metadata(struct xe_gt *gt)
38{
39 unsigned int num_vfs = xe_gt_sriov_pf_get_totalvfs(gt);
40
41 gt->sriov.pf.vfs = drmm_kcalloc(>_to_xe(gt)->drm, 1 + num_vfs,
42 sizeof(*gt->sriov.pf.vfs), GFP_KERNEL);
43 if (!gt->sriov.pf.vfs)
44 return -ENOMEM;
45
46 return 0;
47}
48
49static void pf_init_workers(struct xe_gt *gt)
50{
51 xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
52 INIT_WORK(>->sriov.pf.workers.restart, pf_worker_restart_func);
53}
54
55static void pf_fini_workers(struct xe_gt *gt)
56{
57 xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
58 disable_work_sync(>->sriov.pf.workers.restart);
59}
60
61/**
62 * xe_gt_sriov_pf_init_early - Prepare SR-IOV PF data structures on PF.
63 * @gt: the &xe_gt to initialize
64 *
65 * Early initialization of the PF data.
66 *
67 * Return: 0 on success or a negative error code on failure.
68 */
69int xe_gt_sriov_pf_init_early(struct xe_gt *gt)
70{
71 int err;
72
73 err = pf_alloc_metadata(gt);
74 if (err)
75 return err;
76
77 err = xe_gt_sriov_pf_service_init(gt);
78 if (err)
79 return err;
80
81 err = xe_gt_sriov_pf_control_init(gt);
82 if (err)
83 return err;
84
85 pf_init_workers(gt);
86
87 return 0;
88}
89
90static void pf_fini_action(void *arg)
91{
92 struct xe_gt *gt = arg;
93
94 pf_fini_workers(gt);
95}
96
97static int pf_init_late(struct xe_gt *gt)
98{
99 struct xe_device *xe = gt_to_xe(gt);
100
101 xe_gt_assert(gt, IS_SRIOV_PF(xe));
102 return devm_add_action_or_reset(xe->drm.dev, pf_fini_action, gt);
103}
104
105/**
106 * xe_gt_sriov_pf_init - Prepare SR-IOV PF data structures on PF.
107 * @gt: the &xe_gt to initialize
108 *
109 * Late one-time initialization of the PF data.
110 *
111 * Return: 0 on success or a negative error code on failure.
112 */
113int xe_gt_sriov_pf_init(struct xe_gt *gt)
114{
115 int err;
116
117 err = xe_gt_sriov_pf_config_init(gt);
118 if (err)
119 return err;
120
121 err = xe_gt_sriov_pf_migration_init(gt);
122 if (err)
123 return err;
124
125 err = pf_init_late(gt);
126 if (err)
127 return err;
128
129 return 0;
130}
131
132static bool pf_needs_enable_ggtt_guest_update(struct xe_device *xe)
133{
134 return GRAPHICS_VERx100(xe) == 1200;
135}
136
137static void pf_enable_ggtt_guest_update(struct xe_gt *gt)
138{
139 xe_mmio_write32(>->mmio, VIRTUAL_CTRL_REG, GUEST_GTT_UPDATE_EN);
140}
141
142/**
143 * xe_gt_sriov_pf_init_hw - Initialize SR-IOV hardware support.
144 * @gt: the &xe_gt to initialize
145 *
146 * On some platforms the PF must explicitly enable VF's access to the GGTT.
147 */
148void xe_gt_sriov_pf_init_hw(struct xe_gt *gt)
149{
150 if (pf_needs_enable_ggtt_guest_update(gt_to_xe(gt)))
151 pf_enable_ggtt_guest_update(gt);
152
153 xe_gt_sriov_pf_service_update(gt);
154}
155
156static u32 pf_get_vf_regs_stride(struct xe_device *xe)
157{
158 return GRAPHICS_VERx100(xe) > 1200 ? 0x400 : 0x1000;
159}
160
161static struct xe_reg xe_reg_vf_to_pf(struct xe_reg vf_reg, unsigned int vfid, u32 stride)
162{
163 struct xe_reg pf_reg = vf_reg;
164
165 pf_reg.vf = 0;
166 pf_reg.addr += stride * vfid;
167
168 return pf_reg;
169}
170
171static void pf_clear_vf_scratch_regs(struct xe_gt *gt, unsigned int vfid)
172{
173 u32 stride = pf_get_vf_regs_stride(gt_to_xe(gt));
174 struct xe_reg scratch;
175 int n, count;
176
177 if (xe_gt_is_media_type(gt)) {
178 count = MED_VF_SW_FLAG_COUNT;
179 for (n = 0; n < count; n++) {
180 scratch = xe_reg_vf_to_pf(MED_VF_SW_FLAG(n), vfid, stride);
181 xe_mmio_write32(>->mmio, scratch, 0);
182 }
183 } else {
184 count = VF_SW_FLAG_COUNT;
185 for (n = 0; n < count; n++) {
186 scratch = xe_reg_vf_to_pf(VF_SW_FLAG(n), vfid, stride);
187 xe_mmio_write32(>->mmio, scratch, 0);
188 }
189 }
190}
191
192/**
193 * xe_gt_sriov_pf_sanitize_hw() - Reset hardware state related to a VF.
194 * @gt: the &xe_gt
195 * @vfid: the VF identifier
196 *
197 * This function can only be called on PF.
198 */
199void xe_gt_sriov_pf_sanitize_hw(struct xe_gt *gt, unsigned int vfid)
200{
201 xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
202
203 pf_clear_vf_scratch_regs(gt, vfid);
204}
205
206static void pf_cancel_restart(struct xe_gt *gt)
207{
208 xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
209
210 if (cancel_work_sync(>->sriov.pf.workers.restart))
211 xe_gt_sriov_dbg_verbose(gt, "pending restart canceled!\n");
212}
213
214/**
215 * xe_gt_sriov_pf_stop_prepare() - Prepare to stop SR-IOV support.
216 * @gt: the &xe_gt
217 *
218 * This function can only be called on the PF.
219 */
220void xe_gt_sriov_pf_stop_prepare(struct xe_gt *gt)
221{
222 pf_cancel_restart(gt);
223}
224
225static void pf_restart(struct xe_gt *gt)
226{
227 struct xe_device *xe = gt_to_xe(gt);
228
229 xe_pm_runtime_get(xe);
230 xe_gt_sriov_pf_config_restart(gt);
231 xe_gt_sriov_pf_control_restart(gt);
232 xe_pm_runtime_put(xe);
233
234 xe_gt_sriov_dbg(gt, "restart completed\n");
235}
236
237static void pf_worker_restart_func(struct work_struct *w)
238{
239 struct xe_gt *gt = container_of(w, typeof(*gt), sriov.pf.workers.restart);
240
241 pf_restart(gt);
242}
243
244static void pf_queue_restart(struct xe_gt *gt)
245{
246 struct xe_device *xe = gt_to_xe(gt);
247
248 xe_gt_assert(gt, IS_SRIOV_PF(xe));
249
250 if (!queue_work(xe->sriov.wq, >->sriov.pf.workers.restart))
251 xe_gt_sriov_dbg(gt, "restart already in queue!\n");
252}
253
254/**
255 * xe_gt_sriov_pf_restart - Restart SR-IOV support after a GT reset.
256 * @gt: the &xe_gt
257 *
258 * This function can only be called on PF.
259 */
260void xe_gt_sriov_pf_restart(struct xe_gt *gt)
261{
262 pf_queue_restart(gt);
263}
264
265static void pf_flush_restart(struct xe_gt *gt)
266{
267 xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
268 flush_work(>->sriov.pf.workers.restart);
269}
270
271/**
272 * xe_gt_sriov_pf_wait_ready() - Wait until per-GT PF SR-IOV support is ready.
273 * @gt: the &xe_gt
274 *
275 * This function can only be called on PF.
276 *
277 * Return: 0 on success or a negative error code on failure.
278 */
279int xe_gt_sriov_pf_wait_ready(struct xe_gt *gt)
280{
281 /* don't wait if there is another ongoing reset */
282 if (xe_guc_read_stopped(>->uc.guc))
283 return -EBUSY;
284
285 pf_flush_restart(gt);
286 return 0;
287}