tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2020-2024 Intel Corporation
4 */
5
6#include <linux/circ_buf.h>
7#include <linux/highmem.h>
8
9#include "ivpu_drv.h"
10#include "ivpu_hw.h"
11#include "ivpu_hw_reg_io.h"
12#include "ivpu_mmu.h"
13#include "ivpu_mmu_context.h"
14#include "ivpu_pm.h"
15
16#define IVPU_MMU_REG_IDR0 0x00200000u
17#define IVPU_MMU_REG_IDR1 0x00200004u
18#define IVPU_MMU_REG_IDR3 0x0020000cu
19#define IVPU_MMU_REG_IDR5 0x00200014u
20#define IVPU_MMU_REG_CR0 0x00200020u
21#define IVPU_MMU_REG_CR0ACK 0x00200024u
22#define IVPU_MMU_REG_CR0ACK_VAL_MASK GENMASK(31, 0)
23#define IVPU_MMU_REG_CR0_ATSCHK_MASK BIT(4)
24#define IVPU_MMU_REG_CR0_CMDQEN_MASK BIT(3)
25#define IVPU_MMU_REG_CR0_EVTQEN_MASK BIT(2)
26#define IVPU_MMU_REG_CR0_PRIQEN_MASK BIT(1)
27#define IVPU_MMU_REG_CR0_SMMUEN_MASK BIT(0)
28
29#define IVPU_MMU_REG_CR1 0x00200028u
30#define IVPU_MMU_REG_CR2 0x0020002cu
31#define IVPU_MMU_REG_IRQ_CTRL 0x00200050u
32#define IVPU_MMU_REG_IRQ_CTRLACK 0x00200054u
33#define IVPU_MMU_REG_IRQ_CTRLACK_VAL_MASK GENMASK(31, 0)
34
35#define IVPU_MMU_REG_GERROR 0x00200060u
36#define IVPU_MMU_REG_GERROR_CMDQ_MASK BIT_MASK(0)
37#define IVPU_MMU_REG_GERROR_EVTQ_ABT_MASK BIT_MASK(2)
38#define IVPU_MMU_REG_GERROR_PRIQ_ABT_MASK BIT_MASK(3)
39#define IVPU_MMU_REG_GERROR_MSI_CMDQ_ABT_MASK BIT_MASK(4)
40#define IVPU_MMU_REG_GERROR_MSI_EVTQ_ABT_MASK BIT_MASK(5)
41#define IVPU_MMU_REG_GERROR_MSI_PRIQ_ABT_MASK BIT_MASK(6)
42#define IVPU_MMU_REG_GERROR_MSI_ABT_MASK BIT_MASK(7)
43
44#define IVPU_MMU_REG_GERRORN 0x00200064u
45
46#define IVPU_MMU_REG_STRTAB_BASE 0x00200080u
47#define IVPU_MMU_REG_STRTAB_BASE_CFG 0x00200088u
48#define IVPU_MMU_REG_CMDQ_BASE 0x00200090u
49#define IVPU_MMU_REG_CMDQ_PROD 0x00200098u
50#define IVPU_MMU_REG_CMDQ_CONS 0x0020009cu
51#define IVPU_MMU_REG_CMDQ_CONS_VAL_MASK GENMASK(23, 0)
52#define IVPU_MMU_REG_CMDQ_CONS_ERR_MASK GENMASK(30, 24)
53#define IVPU_MMU_REG_EVTQ_BASE 0x002000a0u
54#define IVPU_MMU_REG_EVTQ_PROD 0x002000a8u
55#define IVPU_MMU_REG_EVTQ_CONS 0x002000acu
56#define IVPU_MMU_REG_EVTQ_PROD_SEC (0x002000a8u + SZ_64K)
57#define IVPU_MMU_REG_EVTQ_CONS_SEC (0x002000acu + SZ_64K)
58
59#define IVPU_MMU_IDR0_REF 0x080f3e0f
60#define IVPU_MMU_IDR0_REF_SIMICS 0x080f3e1f
61#define IVPU_MMU_IDR1_REF 0x0e739d18
62#define IVPU_MMU_IDR3_REF 0x0000003c
63#define IVPU_MMU_IDR5_REF 0x00040070
64#define IVPU_MMU_IDR5_REF_SIMICS 0x00000075
65#define IVPU_MMU_IDR5_REF_FPGA 0x00800075
66
67#define IVPU_MMU_CDTAB_ENT_SIZE 64
68#define IVPU_MMU_CDTAB_ENT_COUNT_LOG2 8 /* 256 entries */
69#define IVPU_MMU_CDTAB_ENT_COUNT ((u32)1 << IVPU_MMU_CDTAB_ENT_COUNT_LOG2)
70
71#define IVPU_MMU_STREAM_ID0 0
72#define IVPU_MMU_STREAM_ID3 3
73
74#define IVPU_MMU_STRTAB_ENT_SIZE 64
75#define IVPU_MMU_STRTAB_ENT_COUNT 4
76#define IVPU_MMU_STRTAB_CFG_LOG2SIZE 2
77#define IVPU_MMU_STRTAB_CFG IVPU_MMU_STRTAB_CFG_LOG2SIZE
78
79#define IVPU_MMU_Q_COUNT_LOG2 4 /* 16 entries */
80#define IVPU_MMU_Q_COUNT ((u32)1 << IVPU_MMU_Q_COUNT_LOG2)
81#define IVPU_MMU_Q_WRAP_MASK GENMASK(IVPU_MMU_Q_COUNT_LOG2, 0)
82#define IVPU_MMU_Q_IDX_MASK (IVPU_MMU_Q_COUNT - 1)
83#define IVPU_MMU_Q_IDX(val) ((val) & IVPU_MMU_Q_IDX_MASK)
84#define IVPU_MMU_Q_WRP(val) ((val) & IVPU_MMU_Q_COUNT)
85
86#define IVPU_MMU_CMDQ_CMD_SIZE 16
87#define IVPU_MMU_CMDQ_SIZE (IVPU_MMU_Q_COUNT * IVPU_MMU_CMDQ_CMD_SIZE)
88
89#define IVPU_MMU_EVTQ_CMD_SIZE 32
90#define IVPU_MMU_EVTQ_SIZE (IVPU_MMU_Q_COUNT * IVPU_MMU_EVTQ_CMD_SIZE)
91
92#define IVPU_MMU_CMD_OPCODE GENMASK(7, 0)
93
94#define IVPU_MMU_CMD_SYNC_0_CS GENMASK(13, 12)
95#define IVPU_MMU_CMD_SYNC_0_MSH GENMASK(23, 22)
96#define IVPU_MMU_CMD_SYNC_0_MSI_ATTR GENMASK(27, 24)
97#define IVPU_MMU_CMD_SYNC_0_MSI_ATTR GENMASK(27, 24)
98#define IVPU_MMU_CMD_SYNC_0_MSI_DATA GENMASK(63, 32)
99
100#define IVPU_MMU_CMD_CFGI_0_SSEC BIT(10)
101#define IVPU_MMU_CMD_CFGI_0_SSV BIT(11)
102#define IVPU_MMU_CMD_CFGI_0_SSID GENMASK(31, 12)
103#define IVPU_MMU_CMD_CFGI_0_SID GENMASK(63, 32)
104#define IVPU_MMU_CMD_CFGI_1_RANGE GENMASK(4, 0)
105
106#define IVPU_MMU_CMD_TLBI_0_ASID GENMASK(63, 48)
107#define IVPU_MMU_CMD_TLBI_0_VMID GENMASK(47, 32)
108
109#define CMD_PREFETCH_CFG 0x1
110#define CMD_CFGI_STE 0x3
111#define CMD_CFGI_ALL 0x4
112#define CMD_CFGI_CD 0x5
113#define CMD_CFGI_CD_ALL 0x6
114#define CMD_TLBI_NH_ASID 0x11
115#define CMD_TLBI_EL2_ALL 0x20
116#define CMD_TLBI_NSNH_ALL 0x30
117#define CMD_SYNC 0x46
118
119#define IVPU_MMU_EVT_F_UUT 0x01
120#define IVPU_MMU_EVT_C_BAD_STREAMID 0x02
121#define IVPU_MMU_EVT_F_STE_FETCH 0x03
122#define IVPU_MMU_EVT_C_BAD_STE 0x04
123#define IVPU_MMU_EVT_F_BAD_ATS_TREQ 0x05
124#define IVPU_MMU_EVT_F_STREAM_DISABLED 0x06
125#define IVPU_MMU_EVT_F_TRANSL_FORBIDDEN 0x07
126#define IVPU_MMU_EVT_C_BAD_SUBSTREAMID 0x08
127#define IVPU_MMU_EVT_F_CD_FETCH 0x09
128#define IVPU_MMU_EVT_C_BAD_CD 0x0a
129#define IVPU_MMU_EVT_F_WALK_EABT 0x0b
130#define IVPU_MMU_EVT_F_TRANSLATION 0x10
131#define IVPU_MMU_EVT_F_ADDR_SIZE 0x11
132#define IVPU_MMU_EVT_F_ACCESS 0x12
133#define IVPU_MMU_EVT_F_PERMISSION 0x13
134#define IVPU_MMU_EVT_F_TLB_CONFLICT 0x20
135#define IVPU_MMU_EVT_F_CFG_CONFLICT 0x21
136#define IVPU_MMU_EVT_E_PAGE_REQUEST 0x24
137#define IVPU_MMU_EVT_F_VMS_FETCH 0x25
138
139#define IVPU_MMU_EVT_OP_MASK GENMASK_ULL(7, 0)
140#define IVPU_MMU_EVT_SSID_MASK GENMASK_ULL(31, 12)
141
142#define IVPU_MMU_Q_BASE_RWA BIT(62)
143#define IVPU_MMU_Q_BASE_ADDR_MASK GENMASK_ULL(51, 5)
144#define IVPU_MMU_STRTAB_BASE_RA BIT(62)
145#define IVPU_MMU_STRTAB_BASE_ADDR_MASK GENMASK_ULL(51, 6)
146
147#define IVPU_MMU_IRQ_EVTQ_EN BIT(2)
148#define IVPU_MMU_IRQ_GERROR_EN BIT(0)
149
150#define IVPU_MMU_CR1_TABLE_SH GENMASK(11, 10)
151#define IVPU_MMU_CR1_TABLE_OC GENMASK(9, 8)
152#define IVPU_MMU_CR1_TABLE_IC GENMASK(7, 6)
153#define IVPU_MMU_CR1_QUEUE_SH GENMASK(5, 4)
154#define IVPU_MMU_CR1_QUEUE_OC GENMASK(3, 2)
155#define IVPU_MMU_CR1_QUEUE_IC GENMASK(1, 0)
156#define IVPU_MMU_CACHE_NC 0
157#define IVPU_MMU_CACHE_WB 1
158#define IVPU_MMU_CACHE_WT 2
159#define IVPU_MMU_SH_NSH 0
160#define IVPU_MMU_SH_OSH 2
161#define IVPU_MMU_SH_ISH 3
162
163#define IVPU_MMU_CMDQ_OP GENMASK_ULL(7, 0)
164
165#define IVPU_MMU_CD_0_TCR_T0SZ GENMASK_ULL(5, 0)
166#define IVPU_MMU_CD_0_TCR_TG0 GENMASK_ULL(7, 6)
167#define IVPU_MMU_CD_0_TCR_IRGN0 GENMASK_ULL(9, 8)
168#define IVPU_MMU_CD_0_TCR_ORGN0 GENMASK_ULL(11, 10)
169#define IVPU_MMU_CD_0_TCR_SH0 GENMASK_ULL(13, 12)
170#define IVPU_MMU_CD_0_TCR_EPD0 BIT_ULL(14)
171#define IVPU_MMU_CD_0_TCR_EPD1 BIT_ULL(30)
172#define IVPU_MMU_CD_0_ENDI BIT(15)
173#define IVPU_MMU_CD_0_V BIT(31)
174#define IVPU_MMU_CD_0_TCR_IPS GENMASK_ULL(34, 32)
175#define IVPU_MMU_CD_0_TCR_TBI0 BIT_ULL(38)
176#define IVPU_MMU_CD_0_AA64 BIT(41)
177#define IVPU_MMU_CD_0_S BIT(44)
178#define IVPU_MMU_CD_0_R BIT(45)
179#define IVPU_MMU_CD_0_A BIT(46)
180#define IVPU_MMU_CD_0_ASET BIT(47)
181#define IVPU_MMU_CD_0_ASID GENMASK_ULL(63, 48)
182
183#define IVPU_MMU_T0SZ_48BIT 16
184#define IVPU_MMU_T0SZ_38BIT 26
185
186#define IVPU_MMU_IPS_48BIT 5
187#define IVPU_MMU_IPS_44BIT 4
188#define IVPU_MMU_IPS_42BIT 3
189#define IVPU_MMU_IPS_40BIT 2
190#define IVPU_MMU_IPS_36BIT 1
191#define IVPU_MMU_IPS_32BIT 0
192
193#define IVPU_MMU_CD_1_TTB0_MASK GENMASK_ULL(51, 4)
194
195#define IVPU_MMU_STE_0_S1CDMAX GENMASK_ULL(63, 59)
196#define IVPU_MMU_STE_0_S1FMT GENMASK_ULL(5, 4)
197#define IVPU_MMU_STE_0_S1FMT_LINEAR 0
198#define IVPU_MMU_STE_DWORDS 8
199#define IVPU_MMU_STE_0_CFG_S1_TRANS 5
200#define IVPU_MMU_STE_0_CFG GENMASK_ULL(3, 1)
201#define IVPU_MMU_STE_0_S1CTXPTR_MASK GENMASK_ULL(51, 6)
202#define IVPU_MMU_STE_0_V BIT(0)
203
204#define IVPU_MMU_STE_1_STRW_NSEL1 0ul
205#define IVPU_MMU_STE_1_CONT GENMASK_ULL(16, 13)
206#define IVPU_MMU_STE_1_STRW GENMASK_ULL(31, 30)
207#define IVPU_MMU_STE_1_PRIVCFG GENMASK_ULL(49, 48)
208#define IVPU_MMU_STE_1_PRIVCFG_UNPRIV 2ul
209#define IVPU_MMU_STE_1_INSTCFG GENMASK_ULL(51, 50)
210#define IVPU_MMU_STE_1_INSTCFG_DATA 2ul
211#define IVPU_MMU_STE_1_MEV BIT(19)
212#define IVPU_MMU_STE_1_S1STALLD BIT(27)
213#define IVPU_MMU_STE_1_S1C_CACHE_NC 0ul
214#define IVPU_MMU_STE_1_S1C_CACHE_WBRA 1ul
215#define IVPU_MMU_STE_1_S1C_CACHE_WT 2ul
216#define IVPU_MMU_STE_1_S1C_CACHE_WB 3ul
217#define IVPU_MMU_STE_1_S1CIR GENMASK_ULL(3, 2)
218#define IVPU_MMU_STE_1_S1COR GENMASK_ULL(5, 4)
219#define IVPU_MMU_STE_1_S1CSH GENMASK_ULL(7, 6)
220#define IVPU_MMU_STE_1_S1DSS GENMASK_ULL(1, 0)
221#define IVPU_MMU_STE_1_S1DSS_TERMINATE 0x0
222
223#define IVPU_MMU_REG_TIMEOUT_US (10 * USEC_PER_MSEC)
224#define IVPU_MMU_QUEUE_TIMEOUT_US (100 * USEC_PER_MSEC)
225
226#define IVPU_MMU_GERROR_ERR_MASK ((REG_FLD(IVPU_MMU_REG_GERROR, CMDQ)) | \
227 (REG_FLD(IVPU_MMU_REG_GERROR, EVTQ_ABT)) | \
228 (REG_FLD(IVPU_MMU_REG_GERROR, PRIQ_ABT)) | \
229 (REG_FLD(IVPU_MMU_REG_GERROR, MSI_CMDQ_ABT)) | \
230 (REG_FLD(IVPU_MMU_REG_GERROR, MSI_EVTQ_ABT)) | \
231 (REG_FLD(IVPU_MMU_REG_GERROR, MSI_PRIQ_ABT)) | \
232 (REG_FLD(IVPU_MMU_REG_GERROR, MSI_ABT)))
233
234#define IVPU_MMU_CERROR_NONE 0x0
235#define IVPU_MMU_CERROR_ILL 0x1
236#define IVPU_MMU_CERROR_ABT 0x2
237#define IVPU_MMU_CERROR_ATC_INV_SYNC 0x3
238
239static const char *ivpu_mmu_event_to_str(u32 cmd)
240{
241 switch (cmd) {
242 case IVPU_MMU_EVT_F_UUT:
243 return "Unsupported Upstream Transaction";
244 case IVPU_MMU_EVT_C_BAD_STREAMID:
245 return "Transaction StreamID out of range";
246 case IVPU_MMU_EVT_F_STE_FETCH:
247 return "Fetch of STE caused external abort";
248 case IVPU_MMU_EVT_C_BAD_STE:
249 return "Used STE invalid";
250 case IVPU_MMU_EVT_F_BAD_ATS_TREQ:
251 return "Address Request disallowed for a StreamID";
252 case IVPU_MMU_EVT_F_STREAM_DISABLED:
253 return "Transaction marks non-substream disabled";
254 case IVPU_MMU_EVT_F_TRANSL_FORBIDDEN:
255 return "MMU bypass is disallowed for this StreamID";
256 case IVPU_MMU_EVT_C_BAD_SUBSTREAMID:
257 return "Invalid StreamID";
258 case IVPU_MMU_EVT_F_CD_FETCH:
259 return "Fetch of CD caused external abort";
260 case IVPU_MMU_EVT_C_BAD_CD:
261 return "Fetched CD invalid";
262 case IVPU_MMU_EVT_F_WALK_EABT:
263 return " An external abort occurred fetching a TLB";
264 case IVPU_MMU_EVT_F_TRANSLATION:
265 return "Translation fault";
266 case IVPU_MMU_EVT_F_ADDR_SIZE:
267 return " Output address caused address size fault";
268 case IVPU_MMU_EVT_F_ACCESS:
269 return "Access flag fault";
270 case IVPU_MMU_EVT_F_PERMISSION:
271 return "Permission fault occurred on page access";
272 case IVPU_MMU_EVT_F_TLB_CONFLICT:
273 return "A TLB conflict";
274 case IVPU_MMU_EVT_F_CFG_CONFLICT:
275 return "A configuration cache conflict";
276 case IVPU_MMU_EVT_E_PAGE_REQUEST:
277 return "Page request hint from a client device";
278 case IVPU_MMU_EVT_F_VMS_FETCH:
279 return "Fetch of VMS caused external abort";
280 default:
281 return "Unknown event";
282 }
283}
284
285static const char *ivpu_mmu_cmdq_err_to_str(u32 err)
286{
287 switch (err) {
288 case IVPU_MMU_CERROR_NONE:
289 return "No error";
290 case IVPU_MMU_CERROR_ILL:
291 return "Illegal command";
292 case IVPU_MMU_CERROR_ABT:
293 return "External abort on command queue read";
294 case IVPU_MMU_CERROR_ATC_INV_SYNC:
295 return "Sync failed to complete ATS invalidation";
296 default:
297 return "Unknown error";
298 }
299}
300
301static void ivpu_mmu_config_check(struct ivpu_device *vdev)
302{
303 u32 val_ref;
304 u32 val;
305
306 if (ivpu_is_simics(vdev))
307 val_ref = IVPU_MMU_IDR0_REF_SIMICS;
308 else
309 val_ref = IVPU_MMU_IDR0_REF;
310
311 val = REGV_RD32(IVPU_MMU_REG_IDR0);
312 if (val != val_ref)
313 ivpu_dbg(vdev, MMU, "IDR0 0x%x != IDR0_REF 0x%x\n", val, val_ref);
314
315 val = REGV_RD32(IVPU_MMU_REG_IDR1);
316 if (val != IVPU_MMU_IDR1_REF)
317 ivpu_dbg(vdev, MMU, "IDR1 0x%x != IDR1_REF 0x%x\n", val, IVPU_MMU_IDR1_REF);
318
319 val = REGV_RD32(IVPU_MMU_REG_IDR3);
320 if (val != IVPU_MMU_IDR3_REF)
321 ivpu_dbg(vdev, MMU, "IDR3 0x%x != IDR3_REF 0x%x\n", val, IVPU_MMU_IDR3_REF);
322
323 if (ivpu_is_simics(vdev))
324 val_ref = IVPU_MMU_IDR5_REF_SIMICS;
325 else if (ivpu_is_fpga(vdev))
326 val_ref = IVPU_MMU_IDR5_REF_FPGA;
327 else
328 val_ref = IVPU_MMU_IDR5_REF;
329
330 val = REGV_RD32(IVPU_MMU_REG_IDR5);
331 if (val != val_ref)
332 ivpu_dbg(vdev, MMU, "IDR5 0x%x != IDR5_REF 0x%x\n", val, val_ref);
333}
334
335static int ivpu_mmu_cdtab_alloc(struct ivpu_device *vdev)
336{
337 struct ivpu_mmu_info *mmu = vdev->mmu;
338 struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab;
339 size_t size = IVPU_MMU_CDTAB_ENT_COUNT * IVPU_MMU_CDTAB_ENT_SIZE;
340
341 cdtab->base = dmam_alloc_coherent(vdev->drm.dev, size, &cdtab->dma, GFP_KERNEL);
342 if (!cdtab->base)
343 return -ENOMEM;
344
345 ivpu_dbg(vdev, MMU, "CDTAB alloc: dma=%pad size=%zu\n", &cdtab->dma, size);
346
347 return 0;
348}
349
350static int ivpu_mmu_strtab_alloc(struct ivpu_device *vdev)
351{
352 struct ivpu_mmu_info *mmu = vdev->mmu;
353 struct ivpu_mmu_strtab *strtab = &mmu->strtab;
354 size_t size = IVPU_MMU_STRTAB_ENT_COUNT * IVPU_MMU_STRTAB_ENT_SIZE;
355
356 strtab->base = dmam_alloc_coherent(vdev->drm.dev, size, &strtab->dma, GFP_KERNEL);
357 if (!strtab->base)
358 return -ENOMEM;
359
360 strtab->base_cfg = IVPU_MMU_STRTAB_CFG;
361 strtab->dma_q = IVPU_MMU_STRTAB_BASE_RA;
362 strtab->dma_q |= strtab->dma & IVPU_MMU_STRTAB_BASE_ADDR_MASK;
363
364 ivpu_dbg(vdev, MMU, "STRTAB alloc: dma=%pad dma_q=%pad size=%zu\n",
365 &strtab->dma, &strtab->dma_q, size);
366
367 return 0;
368}
369
370static int ivpu_mmu_cmdq_alloc(struct ivpu_device *vdev)
371{
372 struct ivpu_mmu_info *mmu = vdev->mmu;
373 struct ivpu_mmu_queue *q = &mmu->cmdq;
374
375 q->base = dmam_alloc_coherent(vdev->drm.dev, IVPU_MMU_CMDQ_SIZE, &q->dma, GFP_KERNEL);
376 if (!q->base)
377 return -ENOMEM;
378
379 q->dma_q = IVPU_MMU_Q_BASE_RWA;
380 q->dma_q |= q->dma & IVPU_MMU_Q_BASE_ADDR_MASK;
381 q->dma_q |= IVPU_MMU_Q_COUNT_LOG2;
382
383 ivpu_dbg(vdev, MMU, "CMDQ alloc: dma=%pad dma_q=%pad size=%u\n",
384 &q->dma, &q->dma_q, IVPU_MMU_CMDQ_SIZE);
385
386 return 0;
387}
388
389static int ivpu_mmu_evtq_alloc(struct ivpu_device *vdev)
390{
391 struct ivpu_mmu_info *mmu = vdev->mmu;
392 struct ivpu_mmu_queue *q = &mmu->evtq;
393
394 q->base = dmam_alloc_coherent(vdev->drm.dev, IVPU_MMU_EVTQ_SIZE, &q->dma, GFP_KERNEL);
395 if (!q->base)
396 return -ENOMEM;
397
398 q->dma_q = IVPU_MMU_Q_BASE_RWA;
399 q->dma_q |= q->dma & IVPU_MMU_Q_BASE_ADDR_MASK;
400 q->dma_q |= IVPU_MMU_Q_COUNT_LOG2;
401
402 ivpu_dbg(vdev, MMU, "EVTQ alloc: dma=%pad dma_q=%pad size=%u\n",
403 &q->dma, &q->dma_q, IVPU_MMU_EVTQ_SIZE);
404
405 return 0;
406}
407
408static int ivpu_mmu_structs_alloc(struct ivpu_device *vdev)
409{
410 int ret;
411
412 ret = ivpu_mmu_cdtab_alloc(vdev);
413 if (ret) {
414 ivpu_err(vdev, "Failed to allocate cdtab: %d\n", ret);
415 return ret;
416 }
417
418 ret = ivpu_mmu_strtab_alloc(vdev);
419 if (ret) {
420 ivpu_err(vdev, "Failed to allocate strtab: %d\n", ret);
421 return ret;
422 }
423
424 ret = ivpu_mmu_cmdq_alloc(vdev);
425 if (ret) {
426 ivpu_err(vdev, "Failed to allocate cmdq: %d\n", ret);
427 return ret;
428 }
429
430 ret = ivpu_mmu_evtq_alloc(vdev);
431 if (ret)
432 ivpu_err(vdev, "Failed to allocate evtq: %d\n", ret);
433
434 return ret;
435}
436
437static int ivpu_mmu_reg_write_cr0(struct ivpu_device *vdev, u32 val)
438{
439 REGV_WR32(IVPU_MMU_REG_CR0, val);
440
441 return REGV_POLL_FLD(IVPU_MMU_REG_CR0ACK, VAL, val, IVPU_MMU_REG_TIMEOUT_US);
442}
443
444static int ivpu_mmu_reg_write_irq_ctrl(struct ivpu_device *vdev, u32 val)
445{
446 REGV_WR32(IVPU_MMU_REG_IRQ_CTRL, val);
447
448 return REGV_POLL_FLD(IVPU_MMU_REG_IRQ_CTRLACK, VAL, val, IVPU_MMU_REG_TIMEOUT_US);
449}
450
451static int ivpu_mmu_irqs_setup(struct ivpu_device *vdev)
452{
453 u32 irq_ctrl = IVPU_MMU_IRQ_EVTQ_EN | IVPU_MMU_IRQ_GERROR_EN;
454 int ret;
455
456 ret = ivpu_mmu_reg_write_irq_ctrl(vdev, 0);
457 if (ret)
458 return ret;
459
460 return ivpu_mmu_reg_write_irq_ctrl(vdev, irq_ctrl);
461}
462
463static int ivpu_mmu_cmdq_wait_for_cons(struct ivpu_device *vdev)
464{
465 struct ivpu_mmu_queue *cmdq = &vdev->mmu->cmdq;
466 int ret;
467
468 ret = REGV_POLL_FLD(IVPU_MMU_REG_CMDQ_CONS, VAL, cmdq->prod,
469 IVPU_MMU_QUEUE_TIMEOUT_US);
470 if (ret)
471 return ret;
472
473 cmdq->cons = cmdq->prod;
474
475 return 0;
476}
477
478static bool ivpu_mmu_queue_is_full(struct ivpu_mmu_queue *q)
479{
480 return ((IVPU_MMU_Q_IDX(q->prod) == IVPU_MMU_Q_IDX(q->cons)) &&
481 (IVPU_MMU_Q_WRP(q->prod) != IVPU_MMU_Q_WRP(q->cons)));
482}
483
484static bool ivpu_mmu_queue_is_empty(struct ivpu_mmu_queue *q)
485{
486 return ((IVPU_MMU_Q_IDX(q->prod) == IVPU_MMU_Q_IDX(q->cons)) &&
487 (IVPU_MMU_Q_WRP(q->prod) == IVPU_MMU_Q_WRP(q->cons)));
488}
489
490static int ivpu_mmu_cmdq_cmd_write(struct ivpu_device *vdev, const char *name, u64 data0, u64 data1)
491{
492 struct ivpu_mmu_queue *cmdq = &vdev->mmu->cmdq;
493 u64 *queue_buffer = cmdq->base;
494 int idx = IVPU_MMU_Q_IDX(cmdq->prod) * (IVPU_MMU_CMDQ_CMD_SIZE / sizeof(*queue_buffer));
495
496 if (ivpu_mmu_queue_is_full(cmdq)) {
497 ivpu_err(vdev, "Failed to write MMU CMD %s\n", name);
498 return -EBUSY;
499 }
500
501 queue_buffer[idx] = data0;
502 queue_buffer[idx + 1] = data1;
503 cmdq->prod = (cmdq->prod + 1) & IVPU_MMU_Q_WRAP_MASK;
504
505 ivpu_dbg(vdev, MMU, "CMD write: %s data: 0x%llx 0x%llx\n", name, data0, data1);
506
507 return 0;
508}
509
510static int ivpu_mmu_cmdq_sync(struct ivpu_device *vdev)
511{
512 struct ivpu_mmu_queue *q = &vdev->mmu->cmdq;
513 u64 val;
514 int ret;
515
516 val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_SYNC);
517
518 ret = ivpu_mmu_cmdq_cmd_write(vdev, "SYNC", val, 0);
519 if (ret)
520 return ret;
521
522 if (!ivpu_is_force_snoop_enabled(vdev))
523 clflush_cache_range(q->base, IVPU_MMU_CMDQ_SIZE);
524 REGV_WR32(IVPU_MMU_REG_CMDQ_PROD, q->prod);
525
526 ret = ivpu_mmu_cmdq_wait_for_cons(vdev);
527 if (ret) {
528 u32 err;
529
530 val = REGV_RD32(IVPU_MMU_REG_CMDQ_CONS);
531 err = REG_GET_FLD(IVPU_MMU_REG_CMDQ_CONS, ERR, val);
532
533 ivpu_err(vdev, "Timed out waiting for MMU consumer: %d, error: %s\n", ret,
534 ivpu_mmu_cmdq_err_to_str(err));
535 ivpu_hw_diagnose_failure(vdev);
536 }
537
538 return ret;
539}
540
541static int ivpu_mmu_cmdq_write_cfgi_all(struct ivpu_device *vdev)
542{
543 u64 data0 = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_CFGI_ALL);
544 u64 data1 = FIELD_PREP(IVPU_MMU_CMD_CFGI_1_RANGE, 0x1f);
545
546 return ivpu_mmu_cmdq_cmd_write(vdev, "CFGI_ALL", data0, data1);
547}
548
549static int ivpu_mmu_cmdq_write_tlbi_nh_asid(struct ivpu_device *vdev, u16 ssid)
550{
551 u64 val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_TLBI_NH_ASID) |
552 FIELD_PREP(IVPU_MMU_CMD_TLBI_0_ASID, ssid);
553
554 return ivpu_mmu_cmdq_cmd_write(vdev, "TLBI_NH_ASID", val, 0);
555}
556
557static int ivpu_mmu_cmdq_write_tlbi_nsnh_all(struct ivpu_device *vdev)
558{
559 u64 val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_TLBI_NSNH_ALL);
560
561 return ivpu_mmu_cmdq_cmd_write(vdev, "TLBI_NSNH_ALL", val, 0);
562}
563
564static int ivpu_mmu_reset(struct ivpu_device *vdev)
565{
566 struct ivpu_mmu_info *mmu = vdev->mmu;
567 u32 val;
568 int ret;
569
570 memset(mmu->cmdq.base, 0, IVPU_MMU_CMDQ_SIZE);
571 if (!ivpu_is_force_snoop_enabled(vdev))
572 clflush_cache_range(mmu->cmdq.base, IVPU_MMU_CMDQ_SIZE);
573 mmu->cmdq.prod = 0;
574 mmu->cmdq.cons = 0;
575
576 memset(mmu->evtq.base, 0, IVPU_MMU_EVTQ_SIZE);
577 mmu->evtq.prod = 0;
578 mmu->evtq.cons = 0;
579
580 ret = ivpu_mmu_reg_write_cr0(vdev, 0);
581 if (ret)
582 return ret;
583
584 val = FIELD_PREP(IVPU_MMU_CR1_TABLE_SH, IVPU_MMU_SH_ISH) |
585 FIELD_PREP(IVPU_MMU_CR1_TABLE_OC, IVPU_MMU_CACHE_WB) |
586 FIELD_PREP(IVPU_MMU_CR1_TABLE_IC, IVPU_MMU_CACHE_WB) |
587 FIELD_PREP(IVPU_MMU_CR1_QUEUE_SH, IVPU_MMU_SH_ISH) |
588 FIELD_PREP(IVPU_MMU_CR1_QUEUE_OC, IVPU_MMU_CACHE_WB) |
589 FIELD_PREP(IVPU_MMU_CR1_QUEUE_IC, IVPU_MMU_CACHE_WB);
590 REGV_WR32(IVPU_MMU_REG_CR1, val);
591
592 REGV_WR64(IVPU_MMU_REG_STRTAB_BASE, mmu->strtab.dma_q);
593 REGV_WR32(IVPU_MMU_REG_STRTAB_BASE_CFG, mmu->strtab.base_cfg);
594
595 REGV_WR64(IVPU_MMU_REG_CMDQ_BASE, mmu->cmdq.dma_q);
596 REGV_WR32(IVPU_MMU_REG_CMDQ_PROD, 0);
597 REGV_WR32(IVPU_MMU_REG_CMDQ_CONS, 0);
598
599 val = REG_SET_FLD(IVPU_MMU_REG_CR0, CMDQEN, 0);
600 ret = ivpu_mmu_reg_write_cr0(vdev, val);
601 if (ret)
602 return ret;
603
604 ret = ivpu_mmu_cmdq_write_cfgi_all(vdev);
605 if (ret)
606 return ret;
607
608 ret = ivpu_mmu_cmdq_write_tlbi_nsnh_all(vdev);
609 if (ret)
610 return ret;
611
612 ret = ivpu_mmu_cmdq_sync(vdev);
613 if (ret)
614 return ret;
615
616 REGV_WR64(IVPU_MMU_REG_EVTQ_BASE, mmu->evtq.dma_q);
617 REGV_WR32(IVPU_MMU_REG_EVTQ_PROD_SEC, 0);
618 REGV_WR32(IVPU_MMU_REG_EVTQ_CONS_SEC, 0);
619
620 val = REG_SET_FLD(IVPU_MMU_REG_CR0, EVTQEN, val);
621 ret = ivpu_mmu_reg_write_cr0(vdev, val);
622 if (ret)
623 return ret;
624
625 val = REG_SET_FLD(IVPU_MMU_REG_CR0, ATSCHK, val);
626 ret = ivpu_mmu_reg_write_cr0(vdev, val);
627 if (ret)
628 return ret;
629
630 ret = ivpu_mmu_irqs_setup(vdev);
631 if (ret)
632 return ret;
633
634 val = REG_SET_FLD(IVPU_MMU_REG_CR0, SMMUEN, val);
635 return ivpu_mmu_reg_write_cr0(vdev, val);
636}
637
638static void ivpu_mmu_strtab_link_cd(struct ivpu_device *vdev, u32 sid)
639{
640 struct ivpu_mmu_info *mmu = vdev->mmu;
641 struct ivpu_mmu_strtab *strtab = &mmu->strtab;
642 struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab;
643 u64 *entry = strtab->base + (sid * IVPU_MMU_STRTAB_ENT_SIZE);
644 u64 str[2];
645
646 str[0] = FIELD_PREP(IVPU_MMU_STE_0_CFG, IVPU_MMU_STE_0_CFG_S1_TRANS) |
647 FIELD_PREP(IVPU_MMU_STE_0_S1CDMAX, IVPU_MMU_CDTAB_ENT_COUNT_LOG2) |
648 FIELD_PREP(IVPU_MMU_STE_0_S1FMT, IVPU_MMU_STE_0_S1FMT_LINEAR) |
649 IVPU_MMU_STE_0_V |
650 (cdtab->dma & IVPU_MMU_STE_0_S1CTXPTR_MASK);
651
652 str[1] = FIELD_PREP(IVPU_MMU_STE_1_S1DSS, IVPU_MMU_STE_1_S1DSS_TERMINATE) |
653 FIELD_PREP(IVPU_MMU_STE_1_S1CIR, IVPU_MMU_STE_1_S1C_CACHE_NC) |
654 FIELD_PREP(IVPU_MMU_STE_1_S1COR, IVPU_MMU_STE_1_S1C_CACHE_NC) |
655 FIELD_PREP(IVPU_MMU_STE_1_S1CSH, IVPU_MMU_SH_NSH) |
656 FIELD_PREP(IVPU_MMU_STE_1_PRIVCFG, IVPU_MMU_STE_1_PRIVCFG_UNPRIV) |
657 FIELD_PREP(IVPU_MMU_STE_1_INSTCFG, IVPU_MMU_STE_1_INSTCFG_DATA) |
658 FIELD_PREP(IVPU_MMU_STE_1_STRW, IVPU_MMU_STE_1_STRW_NSEL1) |
659 FIELD_PREP(IVPU_MMU_STE_1_CONT, IVPU_MMU_STRTAB_CFG_LOG2SIZE) |
660 IVPU_MMU_STE_1_MEV |
661 IVPU_MMU_STE_1_S1STALLD;
662
663 WRITE_ONCE(entry[1], str[1]);
664 WRITE_ONCE(entry[0], str[0]);
665
666 if (!ivpu_is_force_snoop_enabled(vdev))
667 clflush_cache_range(entry, IVPU_MMU_STRTAB_ENT_SIZE);
668
669 ivpu_dbg(vdev, MMU, "STRTAB write entry (SSID=%u): 0x%llx, 0x%llx\n", sid, str[0], str[1]);
670}
671
672static int ivpu_mmu_strtab_init(struct ivpu_device *vdev)
673{
674 ivpu_mmu_strtab_link_cd(vdev, IVPU_MMU_STREAM_ID0);
675 ivpu_mmu_strtab_link_cd(vdev, IVPU_MMU_STREAM_ID3);
676
677 return 0;
678}
679
680int ivpu_mmu_invalidate_tlb(struct ivpu_device *vdev, u16 ssid)
681{
682 struct ivpu_mmu_info *mmu = vdev->mmu;
683 int ret = 0;
684
685 mutex_lock(&mmu->lock);
686 if (!mmu->on)
687 goto unlock;
688
689 ret = ivpu_mmu_cmdq_write_tlbi_nh_asid(vdev, ssid);
690 if (ret)
691 goto unlock;
692
693 ret = ivpu_mmu_cmdq_sync(vdev);
694unlock:
695 mutex_unlock(&mmu->lock);
696 return ret;
697}
698
699static int ivpu_mmu_cdtab_entry_set(struct ivpu_device *vdev, u32 ssid, u64 cd_dma, bool valid)
700{
701 struct ivpu_mmu_info *mmu = vdev->mmu;
702 struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab;
703 u64 *entry;
704 u64 cd[4];
705 int ret = 0;
706
707 if (ssid > IVPU_MMU_CDTAB_ENT_COUNT)
708 return -EINVAL;
709
710 entry = cdtab->base + (ssid * IVPU_MMU_CDTAB_ENT_SIZE);
711 drm_WARN_ON(&vdev->drm, (entry[0] & IVPU_MMU_CD_0_V) == valid);
712
713 cd[0] = FIELD_PREP(IVPU_MMU_CD_0_TCR_T0SZ, IVPU_MMU_T0SZ_48BIT) |
714 FIELD_PREP(IVPU_MMU_CD_0_TCR_TG0, 0) |
715 FIELD_PREP(IVPU_MMU_CD_0_TCR_IRGN0, 0) |
716 FIELD_PREP(IVPU_MMU_CD_0_TCR_ORGN0, 0) |
717 FIELD_PREP(IVPU_MMU_CD_0_TCR_SH0, 0) |
718 FIELD_PREP(IVPU_MMU_CD_0_TCR_IPS, IVPU_MMU_IPS_48BIT) |
719 FIELD_PREP(IVPU_MMU_CD_0_ASID, ssid) |
720 IVPU_MMU_CD_0_TCR_EPD1 |
721 IVPU_MMU_CD_0_AA64 |
722 IVPU_MMU_CD_0_R |
723 IVPU_MMU_CD_0_ASET;
724 cd[1] = cd_dma & IVPU_MMU_CD_1_TTB0_MASK;
725 cd[2] = 0;
726 cd[3] = 0x0000000000007444;
727
728 /* For global and reserved contexts generate memory fault on VPU */
729 if (ssid == IVPU_GLOBAL_CONTEXT_MMU_SSID || ssid == IVPU_RESERVED_CONTEXT_MMU_SSID)
730 cd[0] |= IVPU_MMU_CD_0_A;
731
732 if (valid)
733 cd[0] |= IVPU_MMU_CD_0_V;
734
735 WRITE_ONCE(entry[1], cd[1]);
736 WRITE_ONCE(entry[2], cd[2]);
737 WRITE_ONCE(entry[3], cd[3]);
738 WRITE_ONCE(entry[0], cd[0]);
739
740 if (!ivpu_is_force_snoop_enabled(vdev))
741 clflush_cache_range(entry, IVPU_MMU_CDTAB_ENT_SIZE);
742
743 ivpu_dbg(vdev, MMU, "CDTAB set %s entry (SSID=%u, dma=%pad): 0x%llx, 0x%llx, 0x%llx, 0x%llx\n",
744 valid ? "valid" : "invalid", ssid, &cd_dma, cd[0], cd[1], cd[2], cd[3]);
745
746 mutex_lock(&mmu->lock);
747 if (!mmu->on)
748 goto unlock;
749
750 ret = ivpu_mmu_cmdq_write_cfgi_all(vdev);
751 if (ret)
752 goto err_invalidate;
753
754 ret = ivpu_mmu_cmdq_sync(vdev);
755 if (ret)
756 goto err_invalidate;
757unlock:
758 mutex_unlock(&mmu->lock);
759 return 0;
760
761err_invalidate:
762 WRITE_ONCE(entry[0], 0);
763 mutex_unlock(&mmu->lock);
764 return ret;
765}
766
767int ivpu_mmu_init(struct ivpu_device *vdev)
768{
769 struct ivpu_mmu_info *mmu = vdev->mmu;
770 int ret;
771
772 ivpu_dbg(vdev, MMU, "Init..\n");
773
774 ivpu_mmu_config_check(vdev);
775
776 ret = drmm_mutex_init(&vdev->drm, &mmu->lock);
777 if (ret)
778 return ret;
779
780 ret = ivpu_mmu_structs_alloc(vdev);
781 if (ret)
782 return ret;
783
784 ret = ivpu_mmu_strtab_init(vdev);
785 if (ret) {
786 ivpu_err(vdev, "Failed to initialize strtab: %d\n", ret);
787 return ret;
788 }
789
790 ret = ivpu_mmu_enable(vdev);
791 if (ret) {
792 ivpu_err(vdev, "Failed to resume MMU: %d\n", ret);
793 return ret;
794 }
795
796 ivpu_dbg(vdev, MMU, "Init done\n");
797
798 return 0;
799}
800
801int ivpu_mmu_enable(struct ivpu_device *vdev)
802{
803 struct ivpu_mmu_info *mmu = vdev->mmu;
804 int ret;
805
806 mutex_lock(&mmu->lock);
807
808 mmu->on = true;
809
810 ret = ivpu_mmu_reset(vdev);
811 if (ret) {
812 ivpu_err(vdev, "Failed to reset MMU: %d\n", ret);
813 goto err;
814 }
815
816 ret = ivpu_mmu_cmdq_write_cfgi_all(vdev);
817 if (ret)
818 goto err;
819
820 ret = ivpu_mmu_cmdq_write_tlbi_nsnh_all(vdev);
821 if (ret)
822 goto err;
823
824 ret = ivpu_mmu_cmdq_sync(vdev);
825 if (ret)
826 goto err;
827
828 mutex_unlock(&mmu->lock);
829
830 return 0;
831err:
832 mmu->on = false;
833 mutex_unlock(&mmu->lock);
834 return ret;
835}
836
837void ivpu_mmu_disable(struct ivpu_device *vdev)
838{
839 struct ivpu_mmu_info *mmu = vdev->mmu;
840
841 mutex_lock(&mmu->lock);
842 mmu->on = false;
843 mutex_unlock(&mmu->lock);
844}
845
846static void ivpu_mmu_dump_event(struct ivpu_device *vdev, u32 *event)
847{
848 u32 ssid = FIELD_GET(IVPU_MMU_EVT_SSID_MASK, event[0]);
849 u32 op = FIELD_GET(IVPU_MMU_EVT_OP_MASK, event[0]);
850 u64 fetch_addr = ((u64)event[7]) << 32 | event[6];
851 u64 in_addr = ((u64)event[5]) << 32 | event[4];
852 u32 sid = event[1];
853
854 ivpu_err_ratelimited(vdev, "MMU EVTQ: 0x%x (%s) SSID: %d SID: %d, e[2] %08x, e[3] %08x, in addr: 0x%llx, fetch addr: 0x%llx\n",
855 op, ivpu_mmu_event_to_str(op), ssid, sid,
856 event[2], event[3], in_addr, fetch_addr);
857}
858
859static u32 *ivpu_mmu_get_event(struct ivpu_device *vdev)
860{
861 struct ivpu_mmu_queue *evtq = &vdev->mmu->evtq;
862 u32 idx = IVPU_MMU_Q_IDX(evtq->cons);
863 u32 *evt = evtq->base + (idx * IVPU_MMU_EVTQ_CMD_SIZE);
864
865 evtq->prod = REGV_RD32(IVPU_MMU_REG_EVTQ_PROD_SEC);
866 if (ivpu_mmu_queue_is_empty(evtq))
867 return NULL;
868
869 evtq->cons = (evtq->cons + 1) & IVPU_MMU_Q_WRAP_MASK;
870 return evt;
871}
872
873static int ivpu_mmu_evtq_set(struct ivpu_device *vdev, bool enable)
874{
875 u32 val = REGV_RD32(IVPU_MMU_REG_CR0);
876
877 if (enable)
878 val = REG_SET_FLD(IVPU_MMU_REG_CR0, EVTQEN, val);
879 else
880 val = REG_CLR_FLD(IVPU_MMU_REG_CR0, EVTQEN, val);
881 REGV_WR32(IVPU_MMU_REG_CR0, val);
882
883 return REGV_POLL_FLD(IVPU_MMU_REG_CR0ACK, VAL, val, IVPU_MMU_REG_TIMEOUT_US);
884}
885
886static int ivpu_mmu_evtq_enable(struct ivpu_device *vdev)
887{
888 return ivpu_mmu_evtq_set(vdev, true);
889}
890
891static int ivpu_mmu_evtq_disable(struct ivpu_device *vdev)
892{
893 return ivpu_mmu_evtq_set(vdev, false);
894}
895
896void ivpu_mmu_discard_events(struct ivpu_device *vdev)
897{
898 struct ivpu_mmu_info *mmu = vdev->mmu;
899
900 mutex_lock(&mmu->lock);
901 /*
902 * Disable event queue (stop MMU from updating the producer)
903 * to allow synchronization of consumer and producer indexes
904 */
905 ivpu_mmu_evtq_disable(vdev);
906
907 vdev->mmu->evtq.cons = REGV_RD32(IVPU_MMU_REG_EVTQ_PROD_SEC);
908 REGV_WR32(IVPU_MMU_REG_EVTQ_CONS_SEC, vdev->mmu->evtq.cons);
909 vdev->mmu->evtq.prod = REGV_RD32(IVPU_MMU_REG_EVTQ_PROD_SEC);
910
911 ivpu_mmu_evtq_enable(vdev);
912
913 drm_WARN_ON_ONCE(&vdev->drm, vdev->mmu->evtq.cons != vdev->mmu->evtq.prod);
914
915 mutex_unlock(&mmu->lock);
916}
917
918int ivpu_mmu_disable_ssid_events(struct ivpu_device *vdev, u32 ssid)
919{
920 struct ivpu_mmu_info *mmu = vdev->mmu;
921 struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab;
922 u64 *entry;
923 u64 val;
924
925 if (ssid > IVPU_MMU_CDTAB_ENT_COUNT)
926 return -EINVAL;
927
928 mutex_lock(&mmu->lock);
929
930 entry = cdtab->base + (ssid * IVPU_MMU_CDTAB_ENT_SIZE);
931
932 val = READ_ONCE(entry[0]);
933 val &= ~IVPU_MMU_CD_0_R;
934 WRITE_ONCE(entry[0], val);
935
936 if (!ivpu_is_force_snoop_enabled(vdev))
937 clflush_cache_range(entry, IVPU_MMU_CDTAB_ENT_SIZE);
938
939 ivpu_mmu_cmdq_write_cfgi_all(vdev);
940 ivpu_mmu_cmdq_sync(vdev);
941
942 mutex_unlock(&mmu->lock);
943
944 return 0;
945}
946
947void ivpu_mmu_irq_evtq_handler(struct ivpu_device *vdev)
948{
949 struct ivpu_file_priv *file_priv;
950 u32 *event;
951 u32 ssid;
952
953 ivpu_dbg(vdev, IRQ, "MMU event queue\n");
954
955 while ((event = ivpu_mmu_get_event(vdev))) {
956 ssid = FIELD_GET(IVPU_MMU_EVT_SSID_MASK, *event);
957 if (ssid == IVPU_GLOBAL_CONTEXT_MMU_SSID ||
958 ssid == IVPU_RESERVED_CONTEXT_MMU_SSID) {
959 ivpu_mmu_dump_event(vdev, event);
960 ivpu_pm_trigger_recovery(vdev, "MMU event");
961 return;
962 }
963
964 file_priv = xa_load(&vdev->context_xa, ssid);
965 if (file_priv) {
966 if (!READ_ONCE(file_priv->has_mmu_faults)) {
967 ivpu_mmu_dump_event(vdev, event);
968 WRITE_ONCE(file_priv->has_mmu_faults, true);
969 }
970 }
971 }
972
973 queue_work(system_wq, &vdev->context_abort_work);
974}
975
976void ivpu_mmu_evtq_dump(struct ivpu_device *vdev)
977{
978 u32 *event;
979
980 while ((event = ivpu_mmu_get_event(vdev)) != NULL)
981 ivpu_mmu_dump_event(vdev, event);
982}
983
984void ivpu_mmu_irq_gerr_handler(struct ivpu_device *vdev)
985{
986 u32 gerror_val, gerrorn_val, active;
987
988 ivpu_dbg(vdev, IRQ, "MMU error\n");
989
990 gerror_val = REGV_RD32(IVPU_MMU_REG_GERROR);
991 gerrorn_val = REGV_RD32(IVPU_MMU_REG_GERRORN);
992
993 active = gerror_val ^ gerrorn_val;
994 if (!(active & IVPU_MMU_GERROR_ERR_MASK))
995 return;
996
997 if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, MSI_ABT, active))
998 ivpu_warn_ratelimited(vdev, "MMU MSI ABT write aborted\n");
999
1000 if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, MSI_PRIQ_ABT, active))
1001 ivpu_warn_ratelimited(vdev, "MMU PRIQ MSI ABT write aborted\n");
1002
1003 if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, MSI_EVTQ_ABT, active))
1004 ivpu_warn_ratelimited(vdev, "MMU EVTQ MSI ABT write aborted\n");
1005
1006 if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, MSI_CMDQ_ABT, active))
1007 ivpu_warn_ratelimited(vdev, "MMU CMDQ MSI ABT write aborted\n");
1008
1009 if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, PRIQ_ABT, active))
1010 ivpu_err_ratelimited(vdev, "MMU PRIQ write aborted\n");
1011
1012 if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, EVTQ_ABT, active))
1013 ivpu_err_ratelimited(vdev, "MMU EVTQ write aborted\n");
1014
1015 if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, CMDQ, active))
1016 ivpu_err_ratelimited(vdev, "MMU CMDQ write aborted\n");
1017
1018 REGV_WR32(IVPU_MMU_REG_GERRORN, gerror_val);
1019}
1020
1021int ivpu_mmu_cd_set(struct ivpu_device *vdev, int ssid, struct ivpu_mmu_pgtable *pgtable)
1022{
1023 return ivpu_mmu_cdtab_entry_set(vdev, ssid, pgtable->pgd_dma, true);
1024}
1025
1026void ivpu_mmu_cd_clear(struct ivpu_device *vdev, int ssid)
1027{
1028 ivpu_mmu_cdtab_entry_set(vdev, ssid, 0, false);
1029}