tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/*
2 * Copyright 2019 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24#include <linux/firmware.h>
25#include <drm/drm_drv.h>
26
27#include "amdgpu.h"
28#include "amdgpu_vcn.h"
29#include "amdgpu_pm.h"
30#include "soc15.h"
31#include "soc15d.h"
32#include "vcn_v2_0.h"
33#include "mmsch_v1_0.h"
34#include "vcn_v2_5.h"
35
36#include "vcn/vcn_2_5_offset.h"
37#include "vcn/vcn_2_5_sh_mask.h"
38#include "ivsrcid/vcn/irqsrcs_vcn_2_0.h"
39
40#define VCN_VID_SOC_ADDRESS_2_0 0x1fa00
41#define VCN1_VID_SOC_ADDRESS_3_0 0x48200
42
43#define mmUVD_CONTEXT_ID_INTERNAL_OFFSET 0x27
44#define mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET 0x0f
45#define mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET 0x10
46#define mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET 0x11
47#define mmUVD_NO_OP_INTERNAL_OFFSET 0x29
48#define mmUVD_GP_SCRATCH8_INTERNAL_OFFSET 0x66
49#define mmUVD_SCRATCH9_INTERNAL_OFFSET 0xc01d
50
51#define mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET 0x431
52#define mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET 0x3b4
53#define mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET 0x3b5
54#define mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET 0x25c
55
56#define VCN25_MAX_HW_INSTANCES_ARCTURUS 2
57
58static void vcn_v2_5_set_dec_ring_funcs(struct amdgpu_device *adev);
59static void vcn_v2_5_set_enc_ring_funcs(struct amdgpu_device *adev);
60static void vcn_v2_5_set_irq_funcs(struct amdgpu_device *adev);
61static int vcn_v2_5_set_powergating_state(void *handle,
62 enum amd_powergating_state state);
63static int vcn_v2_5_pause_dpg_mode(struct amdgpu_device *adev,
64 int inst_idx, struct dpg_pause_state *new_state);
65static int vcn_v2_5_sriov_start(struct amdgpu_device *adev);
66static void vcn_v2_5_set_ras_funcs(struct amdgpu_device *adev);
67
68static int amdgpu_ih_clientid_vcns[] = {
69 SOC15_IH_CLIENTID_VCN,
70 SOC15_IH_CLIENTID_VCN1
71};
72
73/**
74 * vcn_v2_5_early_init - set function pointers and load microcode
75 *
76 * @handle: amdgpu_device pointer
77 *
78 * Set ring and irq function pointers
79 * Load microcode from filesystem
80 */
81static int vcn_v2_5_early_init(void *handle)
82{
83 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
84
85 if (amdgpu_sriov_vf(adev)) {
86 adev->vcn.num_vcn_inst = 2;
87 adev->vcn.harvest_config = 0;
88 adev->vcn.num_enc_rings = 1;
89 } else {
90 u32 harvest;
91 int i;
92
93 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
94 harvest = RREG32_SOC15(VCN, i, mmCC_UVD_HARVESTING);
95 if (harvest & CC_UVD_HARVESTING__UVD_DISABLE_MASK)
96 adev->vcn.harvest_config |= 1 << i;
97 }
98 if (adev->vcn.harvest_config == (AMDGPU_VCN_HARVEST_VCN0 |
99 AMDGPU_VCN_HARVEST_VCN1))
100 /* both instances are harvested, disable the block */
101 return -ENOENT;
102
103 adev->vcn.num_enc_rings = 2;
104 }
105
106 vcn_v2_5_set_dec_ring_funcs(adev);
107 vcn_v2_5_set_enc_ring_funcs(adev);
108 vcn_v2_5_set_irq_funcs(adev);
109 vcn_v2_5_set_ras_funcs(adev);
110
111 return amdgpu_vcn_early_init(adev);
112}
113
114/**
115 * vcn_v2_5_sw_init - sw init for VCN block
116 *
117 * @handle: amdgpu_device pointer
118 *
119 * Load firmware and sw initialization
120 */
121static int vcn_v2_5_sw_init(void *handle)
122{
123 struct amdgpu_ring *ring;
124 int i, j, r;
125 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
126
127 for (j = 0; j < adev->vcn.num_vcn_inst; j++) {
128 if (adev->vcn.harvest_config & (1 << j))
129 continue;
130 /* VCN DEC TRAP */
131 r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[j],
132 VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT, &adev->vcn.inst[j].irq);
133 if (r)
134 return r;
135
136 /* VCN ENC TRAP */
137 for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
138 r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[j],
139 i + VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE, &adev->vcn.inst[j].irq);
140 if (r)
141 return r;
142 }
143
144 /* VCN POISON TRAP */
145 r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[j],
146 VCN_2_6__SRCID_UVD_POISON, &adev->vcn.inst[j].ras_poison_irq);
147 if (r)
148 return r;
149 }
150
151 r = amdgpu_vcn_sw_init(adev);
152 if (r)
153 return r;
154
155 amdgpu_vcn_setup_ucode(adev);
156
157 r = amdgpu_vcn_resume(adev);
158 if (r)
159 return r;
160
161 for (j = 0; j < adev->vcn.num_vcn_inst; j++) {
162 volatile struct amdgpu_fw_shared *fw_shared;
163
164 if (adev->vcn.harvest_config & (1 << j))
165 continue;
166 adev->vcn.internal.context_id = mmUVD_CONTEXT_ID_INTERNAL_OFFSET;
167 adev->vcn.internal.ib_vmid = mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET;
168 adev->vcn.internal.ib_bar_low = mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET;
169 adev->vcn.internal.ib_bar_high = mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET;
170 adev->vcn.internal.ib_size = mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET;
171 adev->vcn.internal.gp_scratch8 = mmUVD_GP_SCRATCH8_INTERNAL_OFFSET;
172
173 adev->vcn.internal.scratch9 = mmUVD_SCRATCH9_INTERNAL_OFFSET;
174 adev->vcn.inst[j].external.scratch9 = SOC15_REG_OFFSET(VCN, j, mmUVD_SCRATCH9);
175 adev->vcn.internal.data0 = mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET;
176 adev->vcn.inst[j].external.data0 = SOC15_REG_OFFSET(VCN, j, mmUVD_GPCOM_VCPU_DATA0);
177 adev->vcn.internal.data1 = mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET;
178 adev->vcn.inst[j].external.data1 = SOC15_REG_OFFSET(VCN, j, mmUVD_GPCOM_VCPU_DATA1);
179 adev->vcn.internal.cmd = mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET;
180 adev->vcn.inst[j].external.cmd = SOC15_REG_OFFSET(VCN, j, mmUVD_GPCOM_VCPU_CMD);
181 adev->vcn.internal.nop = mmUVD_NO_OP_INTERNAL_OFFSET;
182 adev->vcn.inst[j].external.nop = SOC15_REG_OFFSET(VCN, j, mmUVD_NO_OP);
183
184 ring = &adev->vcn.inst[j].ring_dec;
185 ring->use_doorbell = true;
186
187 ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
188 (amdgpu_sriov_vf(adev) ? 2*j : 8*j);
189
190 if (amdgpu_ip_version(adev, UVD_HWIP, 0) == IP_VERSION(2, 5, 0))
191 ring->vm_hub = AMDGPU_MMHUB1(0);
192 else
193 ring->vm_hub = AMDGPU_MMHUB0(0);
194
195 sprintf(ring->name, "vcn_dec_%d", j);
196 r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[j].irq,
197 0, AMDGPU_RING_PRIO_DEFAULT, NULL);
198 if (r)
199 return r;
200
201 for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
202 enum amdgpu_ring_priority_level hw_prio = amdgpu_vcn_get_enc_ring_prio(i);
203
204 ring = &adev->vcn.inst[j].ring_enc[i];
205 ring->use_doorbell = true;
206
207 ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
208 (amdgpu_sriov_vf(adev) ? (1 + i + 2*j) : (2 + i + 8*j));
209
210 if (amdgpu_ip_version(adev, UVD_HWIP, 0) ==
211 IP_VERSION(2, 5, 0))
212 ring->vm_hub = AMDGPU_MMHUB1(0);
213 else
214 ring->vm_hub = AMDGPU_MMHUB0(0);
215
216 sprintf(ring->name, "vcn_enc_%d.%d", j, i);
217 r = amdgpu_ring_init(adev, ring, 512,
218 &adev->vcn.inst[j].irq, 0,
219 hw_prio, NULL);
220 if (r)
221 return r;
222 }
223
224 fw_shared = adev->vcn.inst[j].fw_shared.cpu_addr;
225 fw_shared->present_flag_0 = cpu_to_le32(AMDGPU_VCN_MULTI_QUEUE_FLAG);
226
227 if (amdgpu_vcnfw_log)
228 amdgpu_vcn_fwlog_init(&adev->vcn.inst[i]);
229 }
230
231 if (amdgpu_sriov_vf(adev)) {
232 r = amdgpu_virt_alloc_mm_table(adev);
233 if (r)
234 return r;
235 }
236
237 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
238 adev->vcn.pause_dpg_mode = vcn_v2_5_pause_dpg_mode;
239
240 r = amdgpu_vcn_ras_sw_init(adev);
241 if (r)
242 return r;
243
244 return 0;
245}
246
247/**
248 * vcn_v2_5_sw_fini - sw fini for VCN block
249 *
250 * @handle: amdgpu_device pointer
251 *
252 * VCN suspend and free up sw allocation
253 */
254static int vcn_v2_5_sw_fini(void *handle)
255{
256 int i, r, idx;
257 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
258 volatile struct amdgpu_fw_shared *fw_shared;
259
260 if (drm_dev_enter(adev_to_drm(adev), &idx)) {
261 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
262 if (adev->vcn.harvest_config & (1 << i))
263 continue;
264 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
265 fw_shared->present_flag_0 = 0;
266 }
267 drm_dev_exit(idx);
268 }
269
270
271 if (amdgpu_sriov_vf(adev))
272 amdgpu_virt_free_mm_table(adev);
273
274 r = amdgpu_vcn_suspend(adev);
275 if (r)
276 return r;
277
278 r = amdgpu_vcn_sw_fini(adev);
279
280 return r;
281}
282
283/**
284 * vcn_v2_5_hw_init - start and test VCN block
285 *
286 * @handle: amdgpu_device pointer
287 *
288 * Initialize the hardware, boot up the VCPU and do some testing
289 */
290static int vcn_v2_5_hw_init(void *handle)
291{
292 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
293 struct amdgpu_ring *ring;
294 int i, j, r = 0;
295
296 if (amdgpu_sriov_vf(adev))
297 r = vcn_v2_5_sriov_start(adev);
298
299 for (j = 0; j < adev->vcn.num_vcn_inst; ++j) {
300 if (adev->vcn.harvest_config & (1 << j))
301 continue;
302
303 if (amdgpu_sriov_vf(adev)) {
304 adev->vcn.inst[j].ring_enc[0].sched.ready = true;
305 adev->vcn.inst[j].ring_enc[1].sched.ready = false;
306 adev->vcn.inst[j].ring_enc[2].sched.ready = false;
307 adev->vcn.inst[j].ring_dec.sched.ready = true;
308 } else {
309
310 ring = &adev->vcn.inst[j].ring_dec;
311
312 adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
313 ring->doorbell_index, j);
314
315 r = amdgpu_ring_test_helper(ring);
316 if (r)
317 return r;
318
319 for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
320 ring = &adev->vcn.inst[j].ring_enc[i];
321 r = amdgpu_ring_test_helper(ring);
322 if (r)
323 return r;
324 }
325 }
326 }
327
328 return r;
329}
330
331/**
332 * vcn_v2_5_hw_fini - stop the hardware block
333 *
334 * @handle: amdgpu_device pointer
335 *
336 * Stop the VCN block, mark ring as not ready any more
337 */
338static int vcn_v2_5_hw_fini(void *handle)
339{
340 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
341 int i;
342
343 cancel_delayed_work_sync(&adev->vcn.idle_work);
344
345 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
346 if (adev->vcn.harvest_config & (1 << i))
347 continue;
348
349 if ((adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) ||
350 (adev->vcn.cur_state != AMD_PG_STATE_GATE &&
351 RREG32_SOC15(VCN, i, mmUVD_STATUS)))
352 vcn_v2_5_set_powergating_state(adev, AMD_PG_STATE_GATE);
353
354 if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN))
355 amdgpu_irq_put(adev, &adev->vcn.inst[i].ras_poison_irq, 0);
356 }
357
358 return 0;
359}
360
361/**
362 * vcn_v2_5_suspend - suspend VCN block
363 *
364 * @handle: amdgpu_device pointer
365 *
366 * HW fini and suspend VCN block
367 */
368static int vcn_v2_5_suspend(void *handle)
369{
370 int r;
371 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
372
373 r = vcn_v2_5_hw_fini(adev);
374 if (r)
375 return r;
376
377 r = amdgpu_vcn_suspend(adev);
378
379 return r;
380}
381
382/**
383 * vcn_v2_5_resume - resume VCN block
384 *
385 * @handle: amdgpu_device pointer
386 *
387 * Resume firmware and hw init VCN block
388 */
389static int vcn_v2_5_resume(void *handle)
390{
391 int r;
392 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
393
394 r = amdgpu_vcn_resume(adev);
395 if (r)
396 return r;
397
398 r = vcn_v2_5_hw_init(adev);
399
400 return r;
401}
402
403/**
404 * vcn_v2_5_mc_resume - memory controller programming
405 *
406 * @adev: amdgpu_device pointer
407 *
408 * Let the VCN memory controller know it's offsets
409 */
410static void vcn_v2_5_mc_resume(struct amdgpu_device *adev)
411{
412 uint32_t size;
413 uint32_t offset;
414 int i;
415
416 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
417 if (adev->vcn.harvest_config & (1 << i))
418 continue;
419
420 size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw[i]->size + 4);
421 /* cache window 0: fw */
422 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
423 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
424 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_lo));
425 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
426 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_hi));
427 WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_OFFSET0, 0);
428 offset = 0;
429 } else {
430 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
431 lower_32_bits(adev->vcn.inst[i].gpu_addr));
432 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
433 upper_32_bits(adev->vcn.inst[i].gpu_addr));
434 offset = size;
435 WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_OFFSET0,
436 AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
437 }
438 WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_SIZE0, size);
439
440 /* cache window 1: stack */
441 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
442 lower_32_bits(adev->vcn.inst[i].gpu_addr + offset));
443 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
444 upper_32_bits(adev->vcn.inst[i].gpu_addr + offset));
445 WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_OFFSET1, 0);
446 WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE);
447
448 /* cache window 2: context */
449 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
450 lower_32_bits(adev->vcn.inst[i].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
451 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
452 upper_32_bits(adev->vcn.inst[i].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
453 WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_OFFSET2, 0);
454 WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE);
455
456 /* non-cache window */
457 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW,
458 lower_32_bits(adev->vcn.inst[i].fw_shared.gpu_addr));
459 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH,
460 upper_32_bits(adev->vcn.inst[i].fw_shared.gpu_addr));
461 WREG32_SOC15(VCN, i, mmUVD_VCPU_NONCACHE_OFFSET0, 0);
462 WREG32_SOC15(VCN, i, mmUVD_VCPU_NONCACHE_SIZE0,
463 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared)));
464 }
465}
466
467static void vcn_v2_5_mc_resume_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
468{
469 uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw[inst_idx]->size + 4);
470 uint32_t offset;
471
472 /* cache window 0: fw */
473 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
474 if (!indirect) {
475 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
476 VCN, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
477 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_lo), 0, indirect);
478 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
479 VCN, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
480 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_hi), 0, indirect);
481 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
482 VCN, 0, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
483 } else {
484 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
485 VCN, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect);
486 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
487 VCN, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect);
488 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
489 VCN, 0, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
490 }
491 offset = 0;
492 } else {
493 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
494 VCN, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
495 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
496 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
497 VCN, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
498 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
499 offset = size;
500 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
501 VCN, 0, mmUVD_VCPU_CACHE_OFFSET0),
502 AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect);
503 }
504
505 if (!indirect)
506 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
507 VCN, 0, mmUVD_VCPU_CACHE_SIZE0), size, 0, indirect);
508 else
509 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
510 VCN, 0, mmUVD_VCPU_CACHE_SIZE0), 0, 0, indirect);
511
512 /* cache window 1: stack */
513 if (!indirect) {
514 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
515 VCN, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
516 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
517 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
518 VCN, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
519 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
520 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
521 VCN, 0, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
522 } else {
523 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
524 VCN, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect);
525 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
526 VCN, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect);
527 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
528 VCN, 0, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
529 }
530 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
531 VCN, 0, mmUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect);
532
533 /* cache window 2: context */
534 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
535 VCN, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
536 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
537 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
538 VCN, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
539 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
540 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
541 VCN, 0, mmUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect);
542 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
543 VCN, 0, mmUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect);
544
545 /* non-cache window */
546 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
547 VCN, 0, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),
548 lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
549 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
550 VCN, 0, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),
551 upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
552 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
553 VCN, 0, mmUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect);
554 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
555 VCN, 0, mmUVD_VCPU_NONCACHE_SIZE0),
556 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared)), 0, indirect);
557
558 /* VCN global tiling registers */
559 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
560 VCN, 0, mmUVD_GFX8_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect);
561}
562
563/**
564 * vcn_v2_5_disable_clock_gating - disable VCN clock gating
565 *
566 * @adev: amdgpu_device pointer
567 *
568 * Disable clock gating for VCN block
569 */
570static void vcn_v2_5_disable_clock_gating(struct amdgpu_device *adev)
571{
572 uint32_t data;
573 int i;
574
575 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
576 if (adev->vcn.harvest_config & (1 << i))
577 continue;
578 /* UVD disable CGC */
579 data = RREG32_SOC15(VCN, i, mmUVD_CGC_CTRL);
580 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
581 data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
582 else
583 data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
584 data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
585 data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
586 WREG32_SOC15(VCN, i, mmUVD_CGC_CTRL, data);
587
588 data = RREG32_SOC15(VCN, i, mmUVD_CGC_GATE);
589 data &= ~(UVD_CGC_GATE__SYS_MASK
590 | UVD_CGC_GATE__UDEC_MASK
591 | UVD_CGC_GATE__MPEG2_MASK
592 | UVD_CGC_GATE__REGS_MASK
593 | UVD_CGC_GATE__RBC_MASK
594 | UVD_CGC_GATE__LMI_MC_MASK
595 | UVD_CGC_GATE__LMI_UMC_MASK
596 | UVD_CGC_GATE__IDCT_MASK
597 | UVD_CGC_GATE__MPRD_MASK
598 | UVD_CGC_GATE__MPC_MASK
599 | UVD_CGC_GATE__LBSI_MASK
600 | UVD_CGC_GATE__LRBBM_MASK
601 | UVD_CGC_GATE__UDEC_RE_MASK
602 | UVD_CGC_GATE__UDEC_CM_MASK
603 | UVD_CGC_GATE__UDEC_IT_MASK
604 | UVD_CGC_GATE__UDEC_DB_MASK
605 | UVD_CGC_GATE__UDEC_MP_MASK
606 | UVD_CGC_GATE__WCB_MASK
607 | UVD_CGC_GATE__VCPU_MASK
608 | UVD_CGC_GATE__MMSCH_MASK);
609
610 WREG32_SOC15(VCN, i, mmUVD_CGC_GATE, data);
611
612 SOC15_WAIT_ON_RREG(VCN, i, mmUVD_CGC_GATE, 0, 0xFFFFFFFF);
613
614 data = RREG32_SOC15(VCN, i, mmUVD_CGC_CTRL);
615 data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK
616 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
617 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
618 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
619 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
620 | UVD_CGC_CTRL__SYS_MODE_MASK
621 | UVD_CGC_CTRL__UDEC_MODE_MASK
622 | UVD_CGC_CTRL__MPEG2_MODE_MASK
623 | UVD_CGC_CTRL__REGS_MODE_MASK
624 | UVD_CGC_CTRL__RBC_MODE_MASK
625 | UVD_CGC_CTRL__LMI_MC_MODE_MASK
626 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
627 | UVD_CGC_CTRL__IDCT_MODE_MASK
628 | UVD_CGC_CTRL__MPRD_MODE_MASK
629 | UVD_CGC_CTRL__MPC_MODE_MASK
630 | UVD_CGC_CTRL__LBSI_MODE_MASK
631 | UVD_CGC_CTRL__LRBBM_MODE_MASK
632 | UVD_CGC_CTRL__WCB_MODE_MASK
633 | UVD_CGC_CTRL__VCPU_MODE_MASK
634 | UVD_CGC_CTRL__MMSCH_MODE_MASK);
635 WREG32_SOC15(VCN, i, mmUVD_CGC_CTRL, data);
636
637 /* turn on */
638 data = RREG32_SOC15(VCN, i, mmUVD_SUVD_CGC_GATE);
639 data |= (UVD_SUVD_CGC_GATE__SRE_MASK
640 | UVD_SUVD_CGC_GATE__SIT_MASK
641 | UVD_SUVD_CGC_GATE__SMP_MASK
642 | UVD_SUVD_CGC_GATE__SCM_MASK
643 | UVD_SUVD_CGC_GATE__SDB_MASK
644 | UVD_SUVD_CGC_GATE__SRE_H264_MASK
645 | UVD_SUVD_CGC_GATE__SRE_HEVC_MASK
646 | UVD_SUVD_CGC_GATE__SIT_H264_MASK
647 | UVD_SUVD_CGC_GATE__SIT_HEVC_MASK
648 | UVD_SUVD_CGC_GATE__SCM_H264_MASK
649 | UVD_SUVD_CGC_GATE__SCM_HEVC_MASK
650 | UVD_SUVD_CGC_GATE__SDB_H264_MASK
651 | UVD_SUVD_CGC_GATE__SDB_HEVC_MASK
652 | UVD_SUVD_CGC_GATE__SCLR_MASK
653 | UVD_SUVD_CGC_GATE__UVD_SC_MASK
654 | UVD_SUVD_CGC_GATE__ENT_MASK
655 | UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK
656 | UVD_SUVD_CGC_GATE__SIT_HEVC_ENC_MASK
657 | UVD_SUVD_CGC_GATE__SITE_MASK
658 | UVD_SUVD_CGC_GATE__SRE_VP9_MASK
659 | UVD_SUVD_CGC_GATE__SCM_VP9_MASK
660 | UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK
661 | UVD_SUVD_CGC_GATE__SDB_VP9_MASK
662 | UVD_SUVD_CGC_GATE__IME_HEVC_MASK);
663 WREG32_SOC15(VCN, i, mmUVD_SUVD_CGC_GATE, data);
664
665 data = RREG32_SOC15(VCN, i, mmUVD_SUVD_CGC_CTRL);
666 data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
667 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
668 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
669 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
670 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
671 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
672 | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
673 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
674 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
675 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
676 WREG32_SOC15(VCN, i, mmUVD_SUVD_CGC_CTRL, data);
677 }
678}
679
680static void vcn_v2_5_clock_gating_dpg_mode(struct amdgpu_device *adev,
681 uint8_t sram_sel, int inst_idx, uint8_t indirect)
682{
683 uint32_t reg_data = 0;
684
685 /* enable sw clock gating control */
686 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
687 reg_data = 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
688 else
689 reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
690 reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
691 reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
692 reg_data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK |
693 UVD_CGC_CTRL__UDEC_CM_MODE_MASK |
694 UVD_CGC_CTRL__UDEC_IT_MODE_MASK |
695 UVD_CGC_CTRL__UDEC_DB_MODE_MASK |
696 UVD_CGC_CTRL__UDEC_MP_MODE_MASK |
697 UVD_CGC_CTRL__SYS_MODE_MASK |
698 UVD_CGC_CTRL__UDEC_MODE_MASK |
699 UVD_CGC_CTRL__MPEG2_MODE_MASK |
700 UVD_CGC_CTRL__REGS_MODE_MASK |
701 UVD_CGC_CTRL__RBC_MODE_MASK |
702 UVD_CGC_CTRL__LMI_MC_MODE_MASK |
703 UVD_CGC_CTRL__LMI_UMC_MODE_MASK |
704 UVD_CGC_CTRL__IDCT_MODE_MASK |
705 UVD_CGC_CTRL__MPRD_MODE_MASK |
706 UVD_CGC_CTRL__MPC_MODE_MASK |
707 UVD_CGC_CTRL__LBSI_MODE_MASK |
708 UVD_CGC_CTRL__LRBBM_MODE_MASK |
709 UVD_CGC_CTRL__WCB_MODE_MASK |
710 UVD_CGC_CTRL__VCPU_MODE_MASK |
711 UVD_CGC_CTRL__MMSCH_MODE_MASK);
712 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
713 VCN, 0, mmUVD_CGC_CTRL), reg_data, sram_sel, indirect);
714
715 /* turn off clock gating */
716 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
717 VCN, 0, mmUVD_CGC_GATE), 0, sram_sel, indirect);
718
719 /* turn on SUVD clock gating */
720 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
721 VCN, 0, mmUVD_SUVD_CGC_GATE), 1, sram_sel, indirect);
722
723 /* turn on sw mode in UVD_SUVD_CGC_CTRL */
724 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
725 VCN, 0, mmUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect);
726}
727
728/**
729 * vcn_v2_5_enable_clock_gating - enable VCN clock gating
730 *
731 * @adev: amdgpu_device pointer
732 *
733 * Enable clock gating for VCN block
734 */
735static void vcn_v2_5_enable_clock_gating(struct amdgpu_device *adev)
736{
737 uint32_t data = 0;
738 int i;
739
740 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
741 if (adev->vcn.harvest_config & (1 << i))
742 continue;
743 /* enable UVD CGC */
744 data = RREG32_SOC15(VCN, i, mmUVD_CGC_CTRL);
745 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
746 data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
747 else
748 data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
749 data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
750 data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
751 WREG32_SOC15(VCN, i, mmUVD_CGC_CTRL, data);
752
753 data = RREG32_SOC15(VCN, i, mmUVD_CGC_CTRL);
754 data |= (UVD_CGC_CTRL__UDEC_RE_MODE_MASK
755 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
756 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
757 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
758 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
759 | UVD_CGC_CTRL__SYS_MODE_MASK
760 | UVD_CGC_CTRL__UDEC_MODE_MASK
761 | UVD_CGC_CTRL__MPEG2_MODE_MASK
762 | UVD_CGC_CTRL__REGS_MODE_MASK
763 | UVD_CGC_CTRL__RBC_MODE_MASK
764 | UVD_CGC_CTRL__LMI_MC_MODE_MASK
765 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
766 | UVD_CGC_CTRL__IDCT_MODE_MASK
767 | UVD_CGC_CTRL__MPRD_MODE_MASK
768 | UVD_CGC_CTRL__MPC_MODE_MASK
769 | UVD_CGC_CTRL__LBSI_MODE_MASK
770 | UVD_CGC_CTRL__LRBBM_MODE_MASK
771 | UVD_CGC_CTRL__WCB_MODE_MASK
772 | UVD_CGC_CTRL__VCPU_MODE_MASK);
773 WREG32_SOC15(VCN, i, mmUVD_CGC_CTRL, data);
774
775 data = RREG32_SOC15(VCN, i, mmUVD_SUVD_CGC_CTRL);
776 data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
777 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
778 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
779 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
780 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
781 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
782 | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
783 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
784 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
785 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
786 WREG32_SOC15(VCN, i, mmUVD_SUVD_CGC_CTRL, data);
787 }
788}
789
790static void vcn_v2_6_enable_ras(struct amdgpu_device *adev, int inst_idx,
791 bool indirect)
792{
793 uint32_t tmp;
794
795 if (amdgpu_ip_version(adev, UVD_HWIP, 0) != IP_VERSION(2, 6, 0))
796 return;
797
798 tmp = VCN_RAS_CNTL__VCPU_VCODEC_REARM_MASK |
799 VCN_RAS_CNTL__VCPU_VCODEC_IH_EN_MASK |
800 VCN_RAS_CNTL__VCPU_VCODEC_PMI_EN_MASK |
801 VCN_RAS_CNTL__VCPU_VCODEC_STALL_EN_MASK;
802 WREG32_SOC15_DPG_MODE(inst_idx,
803 SOC15_DPG_MODE_OFFSET(VCN, 0, mmVCN_RAS_CNTL),
804 tmp, 0, indirect);
805
806 tmp = UVD_VCPU_INT_EN__RASCNTL_VCPU_VCODEC_EN_MASK;
807 WREG32_SOC15_DPG_MODE(inst_idx,
808 SOC15_DPG_MODE_OFFSET(VCN, 0, mmUVD_VCPU_INT_EN),
809 tmp, 0, indirect);
810
811 tmp = UVD_SYS_INT_EN__RASCNTL_VCPU_VCODEC_EN_MASK;
812 WREG32_SOC15_DPG_MODE(inst_idx,
813 SOC15_DPG_MODE_OFFSET(VCN, 0, mmUVD_SYS_INT_EN),
814 tmp, 0, indirect);
815}
816
817static int vcn_v2_5_start_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
818{
819 volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;
820 struct amdgpu_ring *ring;
821 uint32_t rb_bufsz, tmp;
822
823 /* disable register anti-hang mechanism */
824 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS), 1,
825 ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
826 /* enable dynamic power gating mode */
827 tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_POWER_STATUS);
828 tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK;
829 tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK;
830 WREG32_SOC15(VCN, inst_idx, mmUVD_POWER_STATUS, tmp);
831
832 if (indirect)
833 adev->vcn.inst[inst_idx].dpg_sram_curr_addr = (uint32_t *)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr;
834
835 /* enable clock gating */
836 vcn_v2_5_clock_gating_dpg_mode(adev, 0, inst_idx, indirect);
837
838 /* enable VCPU clock */
839 tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
840 tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
841 tmp |= UVD_VCPU_CNTL__BLK_RST_MASK;
842 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
843 VCN, 0, mmUVD_VCPU_CNTL), tmp, 0, indirect);
844
845 /* disable master interupt */
846 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
847 VCN, 0, mmUVD_MASTINT_EN), 0, 0, indirect);
848
849 /* setup mmUVD_LMI_CTRL */
850 tmp = (0x8 | UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
851 UVD_LMI_CTRL__REQ_MODE_MASK |
852 UVD_LMI_CTRL__CRC_RESET_MASK |
853 UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
854 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
855 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
856 (8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
857 0x00100000L);
858 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
859 VCN, 0, mmUVD_LMI_CTRL), tmp, 0, indirect);
860
861 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
862 VCN, 0, mmUVD_MPC_CNTL),
863 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect);
864
865 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
866 VCN, 0, mmUVD_MPC_SET_MUXA0),
867 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
868 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
869 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
870 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect);
871
872 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
873 VCN, 0, mmUVD_MPC_SET_MUXB0),
874 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
875 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
876 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
877 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect);
878
879 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
880 VCN, 0, mmUVD_MPC_SET_MUX),
881 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
882 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
883 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect);
884
885 vcn_v2_5_mc_resume_dpg_mode(adev, inst_idx, indirect);
886
887 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
888 VCN, 0, mmUVD_REG_XX_MASK), 0x10, 0, indirect);
889 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
890 VCN, 0, mmUVD_RBC_XX_IB_REG_CHECK), 0x3, 0, indirect);
891
892 /* enable LMI MC and UMC channels */
893 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
894 VCN, 0, mmUVD_LMI_CTRL2), 0, 0, indirect);
895
896 vcn_v2_6_enable_ras(adev, inst_idx, indirect);
897
898 /* unblock VCPU register access */
899 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
900 VCN, 0, mmUVD_RB_ARB_CTRL), 0, 0, indirect);
901
902 tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
903 tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
904 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
905 VCN, 0, mmUVD_VCPU_CNTL), tmp, 0, indirect);
906
907 /* enable master interrupt */
908 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
909 VCN, 0, mmUVD_MASTINT_EN),
910 UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect);
911
912 if (indirect)
913 amdgpu_vcn_psp_update_sram(adev, inst_idx, 0);
914
915 ring = &adev->vcn.inst[inst_idx].ring_dec;
916 /* force RBC into idle state */
917 rb_bufsz = order_base_2(ring->ring_size);
918 tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
919 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
920 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
921 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
922 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
923 WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_CNTL, tmp);
924
925 /* Stall DPG before WPTR/RPTR reset */
926 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
927 UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK,
928 ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
929 fw_shared->multi_queue.decode_queue_mode |= FW_QUEUE_RING_RESET;
930
931 /* set the write pointer delay */
932 WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR_CNTL, 0);
933
934 /* set the wb address */
935 WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR_ADDR,
936 (upper_32_bits(ring->gpu_addr) >> 2));
937
938 /* program the RB_BASE for ring buffer */
939 WREG32_SOC15(VCN, inst_idx, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
940 lower_32_bits(ring->gpu_addr));
941 WREG32_SOC15(VCN, inst_idx, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
942 upper_32_bits(ring->gpu_addr));
943
944 /* Initialize the ring buffer's read and write pointers */
945 WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR, 0);
946
947 WREG32_SOC15(VCN, inst_idx, mmUVD_SCRATCH2, 0);
948
949 ring->wptr = RREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR);
950 WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR,
951 lower_32_bits(ring->wptr));
952
953 fw_shared->multi_queue.decode_queue_mode &= ~FW_QUEUE_RING_RESET;
954 /* Unstall DPG */
955 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
956 0, ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
957
958 return 0;
959}
960
961static int vcn_v2_5_start(struct amdgpu_device *adev)
962{
963 struct amdgpu_ring *ring;
964 uint32_t rb_bufsz, tmp;
965 int i, j, k, r;
966
967 if (adev->pm.dpm_enabled)
968 amdgpu_dpm_enable_uvd(adev, true);
969
970 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
971 if (adev->vcn.harvest_config & (1 << i))
972 continue;
973 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
974 r = vcn_v2_5_start_dpg_mode(adev, i, adev->vcn.indirect_sram);
975 continue;
976 }
977
978 /* disable register anti-hang mechanism */
979 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_POWER_STATUS), 0,
980 ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
981
982 /* set uvd status busy */
983 tmp = RREG32_SOC15(VCN, i, mmUVD_STATUS) | UVD_STATUS__UVD_BUSY;
984 WREG32_SOC15(VCN, i, mmUVD_STATUS, tmp);
985 }
986
987 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
988 return 0;
989
990 /*SW clock gating */
991 vcn_v2_5_disable_clock_gating(adev);
992
993 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
994 if (adev->vcn.harvest_config & (1 << i))
995 continue;
996 /* enable VCPU clock */
997 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),
998 UVD_VCPU_CNTL__CLK_EN_MASK, ~UVD_VCPU_CNTL__CLK_EN_MASK);
999
1000 /* disable master interrupt */
1001 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_MASTINT_EN), 0,
1002 ~UVD_MASTINT_EN__VCPU_EN_MASK);
1003
1004 /* setup mmUVD_LMI_CTRL */
1005 tmp = RREG32_SOC15(VCN, i, mmUVD_LMI_CTRL);
1006 tmp &= ~0xff;
1007 WREG32_SOC15(VCN, i, mmUVD_LMI_CTRL, tmp | 0x8|
1008 UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
1009 UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
1010 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
1011 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK);
1012
1013 /* setup mmUVD_MPC_CNTL */
1014 tmp = RREG32_SOC15(VCN, i, mmUVD_MPC_CNTL);
1015 tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK;
1016 tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT;
1017 WREG32_SOC15(VCN, i, mmUVD_MPC_CNTL, tmp);
1018
1019 /* setup UVD_MPC_SET_MUXA0 */
1020 WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUXA0,
1021 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
1022 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
1023 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
1024 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)));
1025
1026 /* setup UVD_MPC_SET_MUXB0 */
1027 WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUXB0,
1028 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
1029 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
1030 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
1031 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)));
1032
1033 /* setup mmUVD_MPC_SET_MUX */
1034 WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUX,
1035 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
1036 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
1037 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)));
1038 }
1039
1040 vcn_v2_5_mc_resume(adev);
1041
1042 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1043 volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
1044 if (adev->vcn.harvest_config & (1 << i))
1045 continue;
1046 /* VCN global tiling registers */
1047 WREG32_SOC15(VCN, i, mmUVD_GFX8_ADDR_CONFIG,
1048 adev->gfx.config.gb_addr_config);
1049 WREG32_SOC15(VCN, i, mmUVD_GFX8_ADDR_CONFIG,
1050 adev->gfx.config.gb_addr_config);
1051
1052 /* enable LMI MC and UMC channels */
1053 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_LMI_CTRL2), 0,
1054 ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
1055
1056 /* unblock VCPU register access */
1057 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_RB_ARB_CTRL), 0,
1058 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1059
1060 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,
1061 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1062
1063 for (k = 0; k < 10; ++k) {
1064 uint32_t status;
1065
1066 for (j = 0; j < 100; ++j) {
1067 status = RREG32_SOC15(VCN, i, mmUVD_STATUS);
1068 if (status & 2)
1069 break;
1070 if (amdgpu_emu_mode == 1)
1071 msleep(500);
1072 else
1073 mdelay(10);
1074 }
1075 r = 0;
1076 if (status & 2)
1077 break;
1078
1079 DRM_ERROR("VCN decode not responding, trying to reset the VCPU!!!\n");
1080 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),
1081 UVD_VCPU_CNTL__BLK_RST_MASK,
1082 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1083 mdelay(10);
1084 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,
1085 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1086
1087 mdelay(10);
1088 r = -1;
1089 }
1090
1091 if (r) {
1092 DRM_ERROR("VCN decode not responding, giving up!!!\n");
1093 return r;
1094 }
1095
1096 /* enable master interrupt */
1097 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_MASTINT_EN),
1098 UVD_MASTINT_EN__VCPU_EN_MASK,
1099 ~UVD_MASTINT_EN__VCPU_EN_MASK);
1100
1101 /* clear the busy bit of VCN_STATUS */
1102 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_STATUS), 0,
1103 ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
1104
1105 WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_VMID, 0);
1106
1107 ring = &adev->vcn.inst[i].ring_dec;
1108 /* force RBC into idle state */
1109 rb_bufsz = order_base_2(ring->ring_size);
1110 tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
1111 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1112 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1113 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1114 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1115 WREG32_SOC15(VCN, i, mmUVD_RBC_RB_CNTL, tmp);
1116
1117 fw_shared->multi_queue.decode_queue_mode |= FW_QUEUE_RING_RESET;
1118 /* program the RB_BASE for ring buffer */
1119 WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
1120 lower_32_bits(ring->gpu_addr));
1121 WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
1122 upper_32_bits(ring->gpu_addr));
1123
1124 /* Initialize the ring buffer's read and write pointers */
1125 WREG32_SOC15(VCN, i, mmUVD_RBC_RB_RPTR, 0);
1126
1127 ring->wptr = RREG32_SOC15(VCN, i, mmUVD_RBC_RB_RPTR);
1128 WREG32_SOC15(VCN, i, mmUVD_RBC_RB_WPTR,
1129 lower_32_bits(ring->wptr));
1130 fw_shared->multi_queue.decode_queue_mode &= ~FW_QUEUE_RING_RESET;
1131
1132 fw_shared->multi_queue.encode_generalpurpose_queue_mode |= FW_QUEUE_RING_RESET;
1133 ring = &adev->vcn.inst[i].ring_enc[0];
1134 WREG32_SOC15(VCN, i, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
1135 WREG32_SOC15(VCN, i, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1136 WREG32_SOC15(VCN, i, mmUVD_RB_BASE_LO, ring->gpu_addr);
1137 WREG32_SOC15(VCN, i, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1138 WREG32_SOC15(VCN, i, mmUVD_RB_SIZE, ring->ring_size / 4);
1139 fw_shared->multi_queue.encode_generalpurpose_queue_mode &= ~FW_QUEUE_RING_RESET;
1140
1141 fw_shared->multi_queue.encode_lowlatency_queue_mode |= FW_QUEUE_RING_RESET;
1142 ring = &adev->vcn.inst[i].ring_enc[1];
1143 WREG32_SOC15(VCN, i, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
1144 WREG32_SOC15(VCN, i, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1145 WREG32_SOC15(VCN, i, mmUVD_RB_BASE_LO2, ring->gpu_addr);
1146 WREG32_SOC15(VCN, i, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
1147 WREG32_SOC15(VCN, i, mmUVD_RB_SIZE2, ring->ring_size / 4);
1148 fw_shared->multi_queue.encode_lowlatency_queue_mode &= ~FW_QUEUE_RING_RESET;
1149 }
1150
1151 return 0;
1152}
1153
1154static int vcn_v2_5_mmsch_start(struct amdgpu_device *adev,
1155 struct amdgpu_mm_table *table)
1156{
1157 uint32_t data = 0, loop = 0, size = 0;
1158 uint64_t addr = table->gpu_addr;
1159 struct mmsch_v1_1_init_header *header = NULL;
1160
1161 header = (struct mmsch_v1_1_init_header *)table->cpu_addr;
1162 size = header->total_size;
1163
1164 /*
1165 * 1, write to vce_mmsch_vf_ctx_addr_lo/hi register with GPU mc addr of
1166 * memory descriptor location
1167 */
1168 WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_ADDR_LO, lower_32_bits(addr));
1169 WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_ADDR_HI, upper_32_bits(addr));
1170
1171 /* 2, update vmid of descriptor */
1172 data = RREG32_SOC15(VCN, 0, mmMMSCH_VF_VMID);
1173 data &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK;
1174 /* use domain0 for MM scheduler */
1175 data |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT);
1176 WREG32_SOC15(VCN, 0, mmMMSCH_VF_VMID, data);
1177
1178 /* 3, notify mmsch about the size of this descriptor */
1179 WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_SIZE, size);
1180
1181 /* 4, set resp to zero */
1182 WREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_RESP, 0);
1183
1184 /*
1185 * 5, kick off the initialization and wait until
1186 * VCE_MMSCH_VF_MAILBOX_RESP becomes non-zero
1187 */
1188 WREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_HOST, 0x10000001);
1189
1190 data = RREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_RESP);
1191 loop = 10;
1192 while ((data & 0x10000002) != 0x10000002) {
1193 udelay(100);
1194 data = RREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_RESP);
1195 loop--;
1196 if (!loop)
1197 break;
1198 }
1199
1200 if (!loop) {
1201 dev_err(adev->dev,
1202 "failed to init MMSCH, mmMMSCH_VF_MAILBOX_RESP = %x\n",
1203 data);
1204 return -EBUSY;
1205 }
1206
1207 return 0;
1208}
1209
1210static int vcn_v2_5_sriov_start(struct amdgpu_device *adev)
1211{
1212 struct amdgpu_ring *ring;
1213 uint32_t offset, size, tmp, i, rb_bufsz;
1214 uint32_t table_size = 0;
1215 struct mmsch_v1_0_cmd_direct_write direct_wt = { { 0 } };
1216 struct mmsch_v1_0_cmd_direct_read_modify_write direct_rd_mod_wt = { { 0 } };
1217 struct mmsch_v1_0_cmd_end end = { { 0 } };
1218 uint32_t *init_table = adev->virt.mm_table.cpu_addr;
1219 struct mmsch_v1_1_init_header *header = (struct mmsch_v1_1_init_header *)init_table;
1220
1221 direct_wt.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_WRITE;
1222 direct_rd_mod_wt.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE;
1223 end.cmd_header.command_type = MMSCH_COMMAND__END;
1224
1225 header->version = MMSCH_VERSION;
1226 header->total_size = sizeof(struct mmsch_v1_1_init_header) >> 2;
1227 init_table += header->total_size;
1228
1229 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1230 header->eng[i].table_offset = header->total_size;
1231 header->eng[i].init_status = 0;
1232 header->eng[i].table_size = 0;
1233
1234 table_size = 0;
1235
1236 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(
1237 SOC15_REG_OFFSET(VCN, i, mmUVD_STATUS),
1238 ~UVD_STATUS__UVD_BUSY, UVD_STATUS__UVD_BUSY);
1239
1240 size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw[i]->size + 4);
1241 /* mc resume*/
1242 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1243 MMSCH_V1_0_INSERT_DIRECT_WT(
1244 SOC15_REG_OFFSET(VCN, i,
1245 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1246 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_lo);
1247 MMSCH_V1_0_INSERT_DIRECT_WT(
1248 SOC15_REG_OFFSET(VCN, i,
1249 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1250 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_hi);
1251 offset = 0;
1252 MMSCH_V1_0_INSERT_DIRECT_WT(
1253 SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_OFFSET0), 0);
1254 } else {
1255 MMSCH_V1_0_INSERT_DIRECT_WT(
1256 SOC15_REG_OFFSET(VCN, i,
1257 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1258 lower_32_bits(adev->vcn.inst[i].gpu_addr));
1259 MMSCH_V1_0_INSERT_DIRECT_WT(
1260 SOC15_REG_OFFSET(VCN, i,
1261 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1262 upper_32_bits(adev->vcn.inst[i].gpu_addr));
1263 offset = size;
1264 MMSCH_V1_0_INSERT_DIRECT_WT(
1265 SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_OFFSET0),
1266 AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
1267 }
1268
1269 MMSCH_V1_0_INSERT_DIRECT_WT(
1270 SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_SIZE0),
1271 size);
1272 MMSCH_V1_0_INSERT_DIRECT_WT(
1273 SOC15_REG_OFFSET(VCN, i,
1274 mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
1275 lower_32_bits(adev->vcn.inst[i].gpu_addr + offset));
1276 MMSCH_V1_0_INSERT_DIRECT_WT(
1277 SOC15_REG_OFFSET(VCN, i,
1278 mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
1279 upper_32_bits(adev->vcn.inst[i].gpu_addr + offset));
1280 MMSCH_V1_0_INSERT_DIRECT_WT(
1281 SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_OFFSET1),
1282 0);
1283 MMSCH_V1_0_INSERT_DIRECT_WT(
1284 SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_SIZE1),
1285 AMDGPU_VCN_STACK_SIZE);
1286 MMSCH_V1_0_INSERT_DIRECT_WT(
1287 SOC15_REG_OFFSET(VCN, i,
1288 mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
1289 lower_32_bits(adev->vcn.inst[i].gpu_addr + offset +
1290 AMDGPU_VCN_STACK_SIZE));
1291 MMSCH_V1_0_INSERT_DIRECT_WT(
1292 SOC15_REG_OFFSET(VCN, i,
1293 mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
1294 upper_32_bits(adev->vcn.inst[i].gpu_addr + offset +
1295 AMDGPU_VCN_STACK_SIZE));
1296 MMSCH_V1_0_INSERT_DIRECT_WT(
1297 SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_OFFSET2),
1298 0);
1299 MMSCH_V1_0_INSERT_DIRECT_WT(
1300 SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_SIZE2),
1301 AMDGPU_VCN_CONTEXT_SIZE);
1302
1303 ring = &adev->vcn.inst[i].ring_enc[0];
1304 ring->wptr = 0;
1305
1306 MMSCH_V1_0_INSERT_DIRECT_WT(
1307 SOC15_REG_OFFSET(VCN, i, mmUVD_RB_BASE_LO),
1308 lower_32_bits(ring->gpu_addr));
1309 MMSCH_V1_0_INSERT_DIRECT_WT(
1310 SOC15_REG_OFFSET(VCN, i, mmUVD_RB_BASE_HI),
1311 upper_32_bits(ring->gpu_addr));
1312 MMSCH_V1_0_INSERT_DIRECT_WT(
1313 SOC15_REG_OFFSET(VCN, i, mmUVD_RB_SIZE),
1314 ring->ring_size / 4);
1315
1316 ring = &adev->vcn.inst[i].ring_dec;
1317 ring->wptr = 0;
1318 MMSCH_V1_0_INSERT_DIRECT_WT(
1319 SOC15_REG_OFFSET(VCN, i,
1320 mmUVD_LMI_RBC_RB_64BIT_BAR_LOW),
1321 lower_32_bits(ring->gpu_addr));
1322 MMSCH_V1_0_INSERT_DIRECT_WT(
1323 SOC15_REG_OFFSET(VCN, i,
1324 mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH),
1325 upper_32_bits(ring->gpu_addr));
1326
1327 /* force RBC into idle state */
1328 rb_bufsz = order_base_2(ring->ring_size);
1329 tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
1330 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1331 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1332 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1333 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1334 MMSCH_V1_0_INSERT_DIRECT_WT(
1335 SOC15_REG_OFFSET(VCN, i, mmUVD_RBC_RB_CNTL), tmp);
1336
1337 /* add end packet */
1338 memcpy((void *)init_table, &end, sizeof(struct mmsch_v1_0_cmd_end));
1339 table_size += sizeof(struct mmsch_v1_0_cmd_end) / 4;
1340 init_table += sizeof(struct mmsch_v1_0_cmd_end) / 4;
1341
1342 /* refine header */
1343 header->eng[i].table_size = table_size;
1344 header->total_size += table_size;
1345 }
1346
1347 return vcn_v2_5_mmsch_start(adev, &adev->virt.mm_table);
1348}
1349
1350static int vcn_v2_5_stop_dpg_mode(struct amdgpu_device *adev, int inst_idx)
1351{
1352 uint32_t tmp;
1353
1354 /* Wait for power status to be 1 */
1355 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 1,
1356 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1357
1358 /* wait for read ptr to be equal to write ptr */
1359 tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR);
1360 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RB_RPTR, tmp, 0xFFFFFFFF);
1361
1362 tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR2);
1363 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RB_RPTR2, tmp, 0xFFFFFFFF);
1364
1365 tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR) & 0x7FFFFFFF;
1366 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RBC_RB_RPTR, tmp, 0xFFFFFFFF);
1367
1368 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 1,
1369 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1370
1371 /* disable dynamic power gating mode */
1372 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS), 0,
1373 ~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
1374
1375 return 0;
1376}
1377
1378static int vcn_v2_5_stop(struct amdgpu_device *adev)
1379{
1380 uint32_t tmp;
1381 int i, r = 0;
1382
1383 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1384 if (adev->vcn.harvest_config & (1 << i))
1385 continue;
1386 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1387 r = vcn_v2_5_stop_dpg_mode(adev, i);
1388 continue;
1389 }
1390
1391 /* wait for vcn idle */
1392 r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_STATUS, UVD_STATUS__IDLE, 0x7);
1393 if (r)
1394 return r;
1395
1396 tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK |
1397 UVD_LMI_STATUS__READ_CLEAN_MASK |
1398 UVD_LMI_STATUS__WRITE_CLEAN_MASK |
1399 UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK;
1400 r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_LMI_STATUS, tmp, tmp);
1401 if (r)
1402 return r;
1403
1404 /* block LMI UMC channel */
1405 tmp = RREG32_SOC15(VCN, i, mmUVD_LMI_CTRL2);
1406 tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK;
1407 WREG32_SOC15(VCN, i, mmUVD_LMI_CTRL2, tmp);
1408
1409 tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK|
1410 UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK;
1411 r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_LMI_STATUS, tmp, tmp);
1412 if (r)
1413 return r;
1414
1415 /* block VCPU register access */
1416 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_RB_ARB_CTRL),
1417 UVD_RB_ARB_CTRL__VCPU_DIS_MASK,
1418 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1419
1420 /* reset VCPU */
1421 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),
1422 UVD_VCPU_CNTL__BLK_RST_MASK,
1423 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1424
1425 /* disable VCPU clock */
1426 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,
1427 ~(UVD_VCPU_CNTL__CLK_EN_MASK));
1428
1429 /* clear status */
1430 WREG32_SOC15(VCN, i, mmUVD_STATUS, 0);
1431
1432 vcn_v2_5_enable_clock_gating(adev);
1433
1434 /* enable register anti-hang mechanism */
1435 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_POWER_STATUS),
1436 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK,
1437 ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1438 }
1439
1440 if (adev->pm.dpm_enabled)
1441 amdgpu_dpm_enable_uvd(adev, false);
1442
1443 return 0;
1444}
1445
1446static int vcn_v2_5_pause_dpg_mode(struct amdgpu_device *adev,
1447 int inst_idx, struct dpg_pause_state *new_state)
1448{
1449 struct amdgpu_ring *ring;
1450 uint32_t reg_data = 0;
1451 int ret_code = 0;
1452
1453 /* pause/unpause if state is changed */
1454 if (adev->vcn.inst[inst_idx].pause_state.fw_based != new_state->fw_based) {
1455 DRM_DEBUG("dpg pause state changed %d -> %d",
1456 adev->vcn.inst[inst_idx].pause_state.fw_based, new_state->fw_based);
1457 reg_data = RREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE) &
1458 (~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1459
1460 if (new_state->fw_based == VCN_DPG_STATE__PAUSE) {
1461 ret_code = SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 0x1,
1462 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1463
1464 if (!ret_code) {
1465 volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;
1466
1467 /* pause DPG */
1468 reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1469 WREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE, reg_data);
1470
1471 /* wait for ACK */
1472 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_DPG_PAUSE,
1473 UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK,
1474 UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1475
1476 /* Stall DPG before WPTR/RPTR reset */
1477 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1478 UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK,
1479 ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1480
1481 /* Restore */
1482 fw_shared->multi_queue.encode_generalpurpose_queue_mode |= FW_QUEUE_RING_RESET;
1483 ring = &adev->vcn.inst[inst_idx].ring_enc[0];
1484 ring->wptr = 0;
1485 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_LO, ring->gpu_addr);
1486 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1487 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_SIZE, ring->ring_size / 4);
1488 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
1489 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1490 fw_shared->multi_queue.encode_generalpurpose_queue_mode &= ~FW_QUEUE_RING_RESET;
1491
1492 fw_shared->multi_queue.encode_lowlatency_queue_mode |= FW_QUEUE_RING_RESET;
1493 ring = &adev->vcn.inst[inst_idx].ring_enc[1];
1494 ring->wptr = 0;
1495 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_LO2, ring->gpu_addr);
1496 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
1497 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_SIZE2, ring->ring_size / 4);
1498 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
1499 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1500 fw_shared->multi_queue.encode_lowlatency_queue_mode &= ~FW_QUEUE_RING_RESET;
1501
1502 /* Unstall DPG */
1503 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1504 0, ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1505
1506 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS,
1507 UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON, UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1508 }
1509 } else {
1510 reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1511 WREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE, reg_data);
1512 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 0x1,
1513 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1514 }
1515 adev->vcn.inst[inst_idx].pause_state.fw_based = new_state->fw_based;
1516 }
1517
1518 return 0;
1519}
1520
1521/**
1522 * vcn_v2_5_dec_ring_get_rptr - get read pointer
1523 *
1524 * @ring: amdgpu_ring pointer
1525 *
1526 * Returns the current hardware read pointer
1527 */
1528static uint64_t vcn_v2_5_dec_ring_get_rptr(struct amdgpu_ring *ring)
1529{
1530 struct amdgpu_device *adev = ring->adev;
1531
1532 return RREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_RPTR);
1533}
1534
1535/**
1536 * vcn_v2_5_dec_ring_get_wptr - get write pointer
1537 *
1538 * @ring: amdgpu_ring pointer
1539 *
1540 * Returns the current hardware write pointer
1541 */
1542static uint64_t vcn_v2_5_dec_ring_get_wptr(struct amdgpu_ring *ring)
1543{
1544 struct amdgpu_device *adev = ring->adev;
1545
1546 if (ring->use_doorbell)
1547 return *ring->wptr_cpu_addr;
1548 else
1549 return RREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_WPTR);
1550}
1551
1552/**
1553 * vcn_v2_5_dec_ring_set_wptr - set write pointer
1554 *
1555 * @ring: amdgpu_ring pointer
1556 *
1557 * Commits the write pointer to the hardware
1558 */
1559static void vcn_v2_5_dec_ring_set_wptr(struct amdgpu_ring *ring)
1560{
1561 struct amdgpu_device *adev = ring->adev;
1562
1563 if (ring->use_doorbell) {
1564 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1565 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1566 } else {
1567 WREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_WPTR, lower_32_bits(ring->wptr));
1568 }
1569}
1570
1571static const struct amdgpu_ring_funcs vcn_v2_5_dec_ring_vm_funcs = {
1572 .type = AMDGPU_RING_TYPE_VCN_DEC,
1573 .align_mask = 0xf,
1574 .secure_submission_supported = true,
1575 .get_rptr = vcn_v2_5_dec_ring_get_rptr,
1576 .get_wptr = vcn_v2_5_dec_ring_get_wptr,
1577 .set_wptr = vcn_v2_5_dec_ring_set_wptr,
1578 .emit_frame_size =
1579 SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
1580 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +
1581 8 + /* vcn_v2_0_dec_ring_emit_vm_flush */
1582 14 + 14 + /* vcn_v2_0_dec_ring_emit_fence x2 vm fence */
1583 6,
1584 .emit_ib_size = 8, /* vcn_v2_0_dec_ring_emit_ib */
1585 .emit_ib = vcn_v2_0_dec_ring_emit_ib,
1586 .emit_fence = vcn_v2_0_dec_ring_emit_fence,
1587 .emit_vm_flush = vcn_v2_0_dec_ring_emit_vm_flush,
1588 .test_ring = vcn_v2_0_dec_ring_test_ring,
1589 .test_ib = amdgpu_vcn_dec_ring_test_ib,
1590 .insert_nop = vcn_v2_0_dec_ring_insert_nop,
1591 .insert_start = vcn_v2_0_dec_ring_insert_start,
1592 .insert_end = vcn_v2_0_dec_ring_insert_end,
1593 .pad_ib = amdgpu_ring_generic_pad_ib,
1594 .begin_use = amdgpu_vcn_ring_begin_use,
1595 .end_use = amdgpu_vcn_ring_end_use,
1596 .emit_wreg = vcn_v2_0_dec_ring_emit_wreg,
1597 .emit_reg_wait = vcn_v2_0_dec_ring_emit_reg_wait,
1598 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1599};
1600
1601/**
1602 * vcn_v2_5_enc_ring_get_rptr - get enc read pointer
1603 *
1604 * @ring: amdgpu_ring pointer
1605 *
1606 * Returns the current hardware enc read pointer
1607 */
1608static uint64_t vcn_v2_5_enc_ring_get_rptr(struct amdgpu_ring *ring)
1609{
1610 struct amdgpu_device *adev = ring->adev;
1611
1612 if (ring == &adev->vcn.inst[ring->me].ring_enc[0])
1613 return RREG32_SOC15(VCN, ring->me, mmUVD_RB_RPTR);
1614 else
1615 return RREG32_SOC15(VCN, ring->me, mmUVD_RB_RPTR2);
1616}
1617
1618/**
1619 * vcn_v2_5_enc_ring_get_wptr - get enc write pointer
1620 *
1621 * @ring: amdgpu_ring pointer
1622 *
1623 * Returns the current hardware enc write pointer
1624 */
1625static uint64_t vcn_v2_5_enc_ring_get_wptr(struct amdgpu_ring *ring)
1626{
1627 struct amdgpu_device *adev = ring->adev;
1628
1629 if (ring == &adev->vcn.inst[ring->me].ring_enc[0]) {
1630 if (ring->use_doorbell)
1631 return *ring->wptr_cpu_addr;
1632 else
1633 return RREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR);
1634 } else {
1635 if (ring->use_doorbell)
1636 return *ring->wptr_cpu_addr;
1637 else
1638 return RREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR2);
1639 }
1640}
1641
1642/**
1643 * vcn_v2_5_enc_ring_set_wptr - set enc write pointer
1644 *
1645 * @ring: amdgpu_ring pointer
1646 *
1647 * Commits the enc write pointer to the hardware
1648 */
1649static void vcn_v2_5_enc_ring_set_wptr(struct amdgpu_ring *ring)
1650{
1651 struct amdgpu_device *adev = ring->adev;
1652
1653 if (ring == &adev->vcn.inst[ring->me].ring_enc[0]) {
1654 if (ring->use_doorbell) {
1655 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1656 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1657 } else {
1658 WREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1659 }
1660 } else {
1661 if (ring->use_doorbell) {
1662 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1663 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1664 } else {
1665 WREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1666 }
1667 }
1668}
1669
1670static const struct amdgpu_ring_funcs vcn_v2_5_enc_ring_vm_funcs = {
1671 .type = AMDGPU_RING_TYPE_VCN_ENC,
1672 .align_mask = 0x3f,
1673 .nop = VCN_ENC_CMD_NO_OP,
1674 .get_rptr = vcn_v2_5_enc_ring_get_rptr,
1675 .get_wptr = vcn_v2_5_enc_ring_get_wptr,
1676 .set_wptr = vcn_v2_5_enc_ring_set_wptr,
1677 .emit_frame_size =
1678 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1679 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
1680 4 + /* vcn_v2_0_enc_ring_emit_vm_flush */
1681 5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */
1682 1, /* vcn_v2_0_enc_ring_insert_end */
1683 .emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */
1684 .emit_ib = vcn_v2_0_enc_ring_emit_ib,
1685 .emit_fence = vcn_v2_0_enc_ring_emit_fence,
1686 .emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush,
1687 .test_ring = amdgpu_vcn_enc_ring_test_ring,
1688 .test_ib = amdgpu_vcn_enc_ring_test_ib,
1689 .insert_nop = amdgpu_ring_insert_nop,
1690 .insert_end = vcn_v2_0_enc_ring_insert_end,
1691 .pad_ib = amdgpu_ring_generic_pad_ib,
1692 .begin_use = amdgpu_vcn_ring_begin_use,
1693 .end_use = amdgpu_vcn_ring_end_use,
1694 .emit_wreg = vcn_v2_0_enc_ring_emit_wreg,
1695 .emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait,
1696 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1697};
1698
1699static void vcn_v2_5_set_dec_ring_funcs(struct amdgpu_device *adev)
1700{
1701 int i;
1702
1703 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1704 if (adev->vcn.harvest_config & (1 << i))
1705 continue;
1706 adev->vcn.inst[i].ring_dec.funcs = &vcn_v2_5_dec_ring_vm_funcs;
1707 adev->vcn.inst[i].ring_dec.me = i;
1708 }
1709}
1710
1711static void vcn_v2_5_set_enc_ring_funcs(struct amdgpu_device *adev)
1712{
1713 int i, j;
1714
1715 for (j = 0; j < adev->vcn.num_vcn_inst; ++j) {
1716 if (adev->vcn.harvest_config & (1 << j))
1717 continue;
1718 for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
1719 adev->vcn.inst[j].ring_enc[i].funcs = &vcn_v2_5_enc_ring_vm_funcs;
1720 adev->vcn.inst[j].ring_enc[i].me = j;
1721 }
1722 }
1723}
1724
1725static bool vcn_v2_5_is_idle(void *handle)
1726{
1727 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1728 int i, ret = 1;
1729
1730 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1731 if (adev->vcn.harvest_config & (1 << i))
1732 continue;
1733 ret &= (RREG32_SOC15(VCN, i, mmUVD_STATUS) == UVD_STATUS__IDLE);
1734 }
1735
1736 return ret;
1737}
1738
1739static int vcn_v2_5_wait_for_idle(void *handle)
1740{
1741 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1742 int i, ret = 0;
1743
1744 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1745 if (adev->vcn.harvest_config & (1 << i))
1746 continue;
1747 ret = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_STATUS, UVD_STATUS__IDLE,
1748 UVD_STATUS__IDLE);
1749 if (ret)
1750 return ret;
1751 }
1752
1753 return ret;
1754}
1755
1756static int vcn_v2_5_set_clockgating_state(void *handle,
1757 enum amd_clockgating_state state)
1758{
1759 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1760 bool enable = (state == AMD_CG_STATE_GATE);
1761
1762 if (amdgpu_sriov_vf(adev))
1763 return 0;
1764
1765 if (enable) {
1766 if (!vcn_v2_5_is_idle(handle))
1767 return -EBUSY;
1768 vcn_v2_5_enable_clock_gating(adev);
1769 } else {
1770 vcn_v2_5_disable_clock_gating(adev);
1771 }
1772
1773 return 0;
1774}
1775
1776static int vcn_v2_5_set_powergating_state(void *handle,
1777 enum amd_powergating_state state)
1778{
1779 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1780 int ret;
1781
1782 if (amdgpu_sriov_vf(adev))
1783 return 0;
1784
1785 if(state == adev->vcn.cur_state)
1786 return 0;
1787
1788 if (state == AMD_PG_STATE_GATE)
1789 ret = vcn_v2_5_stop(adev);
1790 else
1791 ret = vcn_v2_5_start(adev);
1792
1793 if(!ret)
1794 adev->vcn.cur_state = state;
1795
1796 return ret;
1797}
1798
1799static int vcn_v2_5_set_interrupt_state(struct amdgpu_device *adev,
1800 struct amdgpu_irq_src *source,
1801 unsigned type,
1802 enum amdgpu_interrupt_state state)
1803{
1804 return 0;
1805}
1806
1807static int vcn_v2_6_set_ras_interrupt_state(struct amdgpu_device *adev,
1808 struct amdgpu_irq_src *source,
1809 unsigned int type,
1810 enum amdgpu_interrupt_state state)
1811{
1812 return 0;
1813}
1814
1815static int vcn_v2_5_process_interrupt(struct amdgpu_device *adev,
1816 struct amdgpu_irq_src *source,
1817 struct amdgpu_iv_entry *entry)
1818{
1819 uint32_t ip_instance;
1820
1821 switch (entry->client_id) {
1822 case SOC15_IH_CLIENTID_VCN:
1823 ip_instance = 0;
1824 break;
1825 case SOC15_IH_CLIENTID_VCN1:
1826 ip_instance = 1;
1827 break;
1828 default:
1829 DRM_ERROR("Unhandled client id: %d\n", entry->client_id);
1830 return 0;
1831 }
1832
1833 DRM_DEBUG("IH: VCN TRAP\n");
1834
1835 switch (entry->src_id) {
1836 case VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT:
1837 amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_dec);
1838 break;
1839 case VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE:
1840 amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[0]);
1841 break;
1842 case VCN_2_0__SRCID__UVD_ENC_LOW_LATENCY:
1843 amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[1]);
1844 break;
1845 default:
1846 DRM_ERROR("Unhandled interrupt: %d %d\n",
1847 entry->src_id, entry->src_data[0]);
1848 break;
1849 }
1850
1851 return 0;
1852}
1853
1854static const struct amdgpu_irq_src_funcs vcn_v2_5_irq_funcs = {
1855 .set = vcn_v2_5_set_interrupt_state,
1856 .process = vcn_v2_5_process_interrupt,
1857};
1858
1859static const struct amdgpu_irq_src_funcs vcn_v2_6_ras_irq_funcs = {
1860 .set = vcn_v2_6_set_ras_interrupt_state,
1861 .process = amdgpu_vcn_process_poison_irq,
1862};
1863
1864static void vcn_v2_5_set_irq_funcs(struct amdgpu_device *adev)
1865{
1866 int i;
1867
1868 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1869 if (adev->vcn.harvest_config & (1 << i))
1870 continue;
1871 adev->vcn.inst[i].irq.num_types = adev->vcn.num_enc_rings + 1;
1872 adev->vcn.inst[i].irq.funcs = &vcn_v2_5_irq_funcs;
1873
1874 adev->vcn.inst[i].ras_poison_irq.num_types = adev->vcn.num_enc_rings + 1;
1875 adev->vcn.inst[i].ras_poison_irq.funcs = &vcn_v2_6_ras_irq_funcs;
1876 }
1877}
1878
1879static const struct amd_ip_funcs vcn_v2_5_ip_funcs = {
1880 .name = "vcn_v2_5",
1881 .early_init = vcn_v2_5_early_init,
1882 .late_init = NULL,
1883 .sw_init = vcn_v2_5_sw_init,
1884 .sw_fini = vcn_v2_5_sw_fini,
1885 .hw_init = vcn_v2_5_hw_init,
1886 .hw_fini = vcn_v2_5_hw_fini,
1887 .suspend = vcn_v2_5_suspend,
1888 .resume = vcn_v2_5_resume,
1889 .is_idle = vcn_v2_5_is_idle,
1890 .wait_for_idle = vcn_v2_5_wait_for_idle,
1891 .check_soft_reset = NULL,
1892 .pre_soft_reset = NULL,
1893 .soft_reset = NULL,
1894 .post_soft_reset = NULL,
1895 .set_clockgating_state = vcn_v2_5_set_clockgating_state,
1896 .set_powergating_state = vcn_v2_5_set_powergating_state,
1897 .dump_ip_state = NULL,
1898 .print_ip_state = NULL,
1899};
1900
1901static const struct amd_ip_funcs vcn_v2_6_ip_funcs = {
1902 .name = "vcn_v2_6",
1903 .early_init = vcn_v2_5_early_init,
1904 .late_init = NULL,
1905 .sw_init = vcn_v2_5_sw_init,
1906 .sw_fini = vcn_v2_5_sw_fini,
1907 .hw_init = vcn_v2_5_hw_init,
1908 .hw_fini = vcn_v2_5_hw_fini,
1909 .suspend = vcn_v2_5_suspend,
1910 .resume = vcn_v2_5_resume,
1911 .is_idle = vcn_v2_5_is_idle,
1912 .wait_for_idle = vcn_v2_5_wait_for_idle,
1913 .check_soft_reset = NULL,
1914 .pre_soft_reset = NULL,
1915 .soft_reset = NULL,
1916 .post_soft_reset = NULL,
1917 .set_clockgating_state = vcn_v2_5_set_clockgating_state,
1918 .set_powergating_state = vcn_v2_5_set_powergating_state,
1919 .dump_ip_state = NULL,
1920 .print_ip_state = NULL,
1921};
1922
1923const struct amdgpu_ip_block_version vcn_v2_5_ip_block =
1924{
1925 .type = AMD_IP_BLOCK_TYPE_VCN,
1926 .major = 2,
1927 .minor = 5,
1928 .rev = 0,
1929 .funcs = &vcn_v2_5_ip_funcs,
1930};
1931
1932const struct amdgpu_ip_block_version vcn_v2_6_ip_block =
1933{
1934 .type = AMD_IP_BLOCK_TYPE_VCN,
1935 .major = 2,
1936 .minor = 6,
1937 .rev = 0,
1938 .funcs = &vcn_v2_6_ip_funcs,
1939};
1940
1941static uint32_t vcn_v2_6_query_poison_by_instance(struct amdgpu_device *adev,
1942 uint32_t instance, uint32_t sub_block)
1943{
1944 uint32_t poison_stat = 0, reg_value = 0;
1945
1946 switch (sub_block) {
1947 case AMDGPU_VCN_V2_6_VCPU_VCODEC:
1948 reg_value = RREG32_SOC15(VCN, instance, mmUVD_RAS_VCPU_VCODEC_STATUS);
1949 poison_stat = REG_GET_FIELD(reg_value, UVD_RAS_VCPU_VCODEC_STATUS, POISONED_PF);
1950 break;
1951 default:
1952 break;
1953 }
1954
1955 if (poison_stat)
1956 dev_info(adev->dev, "Poison detected in VCN%d, sub_block%d\n",
1957 instance, sub_block);
1958
1959 return poison_stat;
1960}
1961
1962static bool vcn_v2_6_query_poison_status(struct amdgpu_device *adev)
1963{
1964 uint32_t inst, sub;
1965 uint32_t poison_stat = 0;
1966
1967 for (inst = 0; inst < adev->vcn.num_vcn_inst; inst++)
1968 for (sub = 0; sub < AMDGPU_VCN_V2_6_MAX_SUB_BLOCK; sub++)
1969 poison_stat +=
1970 vcn_v2_6_query_poison_by_instance(adev, inst, sub);
1971
1972 return !!poison_stat;
1973}
1974
1975const struct amdgpu_ras_block_hw_ops vcn_v2_6_ras_hw_ops = {
1976 .query_poison_status = vcn_v2_6_query_poison_status,
1977};
1978
1979static struct amdgpu_vcn_ras vcn_v2_6_ras = {
1980 .ras_block = {
1981 .hw_ops = &vcn_v2_6_ras_hw_ops,
1982 .ras_late_init = amdgpu_vcn_ras_late_init,
1983 },
1984};
1985
1986static void vcn_v2_5_set_ras_funcs(struct amdgpu_device *adev)
1987{
1988 switch (amdgpu_ip_version(adev, VCN_HWIP, 0)) {
1989 case IP_VERSION(2, 6, 0):
1990 adev->vcn.ras = &vcn_v2_6_ras;
1991 break;
1992 default:
1993 break;
1994 }
1995}