tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/*
2 * Copyright 2018 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23#include "amdgpu.h"
24#include "df_v3_6.h"
25
26#include "df/df_3_6_default.h"
27#include "df/df_3_6_offset.h"
28#include "df/df_3_6_sh_mask.h"
29
30static u32 df_v3_6_channel_number[] = {1, 2, 0, 4, 0, 8, 0,
31 16, 32, 0, 0, 0, 2, 4, 8};
32
33/* init df format attrs */
34AMDGPU_PMU_ATTR(event, "config:0-7");
35AMDGPU_PMU_ATTR(instance, "config:8-15");
36AMDGPU_PMU_ATTR(umask, "config:16-23");
37
38/* df format attributes */
39static struct attribute *df_v3_6_format_attrs[] = {
40 &pmu_attr_event.attr,
41 &pmu_attr_instance.attr,
42 &pmu_attr_umask.attr,
43 NULL
44};
45
46/* df format attribute group */
47static struct attribute_group df_v3_6_format_attr_group = {
48 .name = "format",
49 .attrs = df_v3_6_format_attrs,
50};
51
52/* df event attrs */
53AMDGPU_PMU_ATTR(cake0_pcsout_txdata,
54 "event=0x7,instance=0x46,umask=0x2");
55AMDGPU_PMU_ATTR(cake1_pcsout_txdata,
56 "event=0x7,instance=0x47,umask=0x2");
57AMDGPU_PMU_ATTR(cake0_pcsout_txmeta,
58 "event=0x7,instance=0x46,umask=0x4");
59AMDGPU_PMU_ATTR(cake1_pcsout_txmeta,
60 "event=0x7,instance=0x47,umask=0x4");
61AMDGPU_PMU_ATTR(cake0_ftiinstat_reqalloc,
62 "event=0xb,instance=0x46,umask=0x4");
63AMDGPU_PMU_ATTR(cake1_ftiinstat_reqalloc,
64 "event=0xb,instance=0x47,umask=0x4");
65AMDGPU_PMU_ATTR(cake0_ftiinstat_rspalloc,
66 "event=0xb,instance=0x46,umask=0x8");
67AMDGPU_PMU_ATTR(cake1_ftiinstat_rspalloc,
68 "event=0xb,instance=0x47,umask=0x8");
69
70/* df event attributes */
71static struct attribute *df_v3_6_event_attrs[] = {
72 &pmu_attr_cake0_pcsout_txdata.attr,
73 &pmu_attr_cake1_pcsout_txdata.attr,
74 &pmu_attr_cake0_pcsout_txmeta.attr,
75 &pmu_attr_cake1_pcsout_txmeta.attr,
76 &pmu_attr_cake0_ftiinstat_reqalloc.attr,
77 &pmu_attr_cake1_ftiinstat_reqalloc.attr,
78 &pmu_attr_cake0_ftiinstat_rspalloc.attr,
79 &pmu_attr_cake1_ftiinstat_rspalloc.attr,
80 NULL
81};
82
83/* df event attribute group */
84static struct attribute_group df_v3_6_event_attr_group = {
85 .name = "events",
86 .attrs = df_v3_6_event_attrs
87};
88
89/* df event attr groups */
90const struct attribute_group *df_v3_6_attr_groups[] = {
91 &df_v3_6_format_attr_group,
92 &df_v3_6_event_attr_group,
93 NULL
94};
95
96static uint64_t df_v3_6_get_fica(struct amdgpu_device *adev,
97 uint32_t ficaa_val)
98{
99 unsigned long flags, address, data;
100 uint32_t ficadl_val, ficadh_val;
101
102 address = adev->nbio.funcs->get_pcie_index_offset(adev);
103 data = adev->nbio.funcs->get_pcie_data_offset(adev);
104
105 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
106 WREG32(address, smnDF_PIE_AON_FabricIndirectConfigAccessAddress3);
107 WREG32(data, ficaa_val);
108
109 WREG32(address, smnDF_PIE_AON_FabricIndirectConfigAccessDataLo3);
110 ficadl_val = RREG32(data);
111
112 WREG32(address, smnDF_PIE_AON_FabricIndirectConfigAccessDataHi3);
113 ficadh_val = RREG32(data);
114
115 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
116
117 return (((ficadh_val & 0xFFFFFFFFFFFFFFFF) << 32) | ficadl_val);
118}
119
120static void df_v3_6_set_fica(struct amdgpu_device *adev, uint32_t ficaa_val,
121 uint32_t ficadl_val, uint32_t ficadh_val)
122{
123 unsigned long flags, address, data;
124
125 address = adev->nbio.funcs->get_pcie_index_offset(adev);
126 data = adev->nbio.funcs->get_pcie_data_offset(adev);
127
128 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
129 WREG32(address, smnDF_PIE_AON_FabricIndirectConfigAccessAddress3);
130 WREG32(data, ficaa_val);
131
132 WREG32(address, smnDF_PIE_AON_FabricIndirectConfigAccessDataLo3);
133 WREG32(data, ficadl_val);
134
135 WREG32(address, smnDF_PIE_AON_FabricIndirectConfigAccessDataHi3);
136 WREG32(data, ficadh_val);
137
138 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
139}
140
141/*
142 * df_v3_6_perfmon_rreg - read perfmon lo and hi
143 *
144 * required to be atomic. no mmio method provided so subsequent reads for lo
145 * and hi require to preserve df finite state machine
146 */
147static void df_v3_6_perfmon_rreg(struct amdgpu_device *adev,
148 uint32_t lo_addr, uint32_t *lo_val,
149 uint32_t hi_addr, uint32_t *hi_val)
150{
151 unsigned long flags, address, data;
152
153 address = adev->nbio.funcs->get_pcie_index_offset(adev);
154 data = adev->nbio.funcs->get_pcie_data_offset(adev);
155
156 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
157 WREG32(address, lo_addr);
158 *lo_val = RREG32(data);
159 WREG32(address, hi_addr);
160 *hi_val = RREG32(data);
161 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
162}
163
164/*
165 * df_v3_6_perfmon_wreg - write to perfmon lo and hi
166 *
167 * required to be atomic. no mmio method provided so subsequent reads after
168 * data writes cannot occur to preserve data fabrics finite state machine.
169 */
170static void df_v3_6_perfmon_wreg(struct amdgpu_device *adev, uint32_t lo_addr,
171 uint32_t lo_val, uint32_t hi_addr, uint32_t hi_val)
172{
173 unsigned long flags, address, data;
174
175 address = adev->nbio.funcs->get_pcie_index_offset(adev);
176 data = adev->nbio.funcs->get_pcie_data_offset(adev);
177
178 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
179 WREG32(address, lo_addr);
180 WREG32(data, lo_val);
181 WREG32(address, hi_addr);
182 WREG32(data, hi_val);
183 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
184}
185
186/* get the number of df counters available */
187static ssize_t df_v3_6_get_df_cntr_avail(struct device *dev,
188 struct device_attribute *attr,
189 char *buf)
190{
191 struct amdgpu_device *adev;
192 struct drm_device *ddev;
193 int i, count;
194
195 ddev = dev_get_drvdata(dev);
196 adev = ddev->dev_private;
197 count = 0;
198
199 for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
200 if (adev->df_perfmon_config_assign_mask[i] == 0)
201 count++;
202 }
203
204 return snprintf(buf, PAGE_SIZE, "%i\n", count);
205}
206
207/* device attr for available perfmon counters */
208static DEVICE_ATTR(df_cntr_avail, S_IRUGO, df_v3_6_get_df_cntr_avail, NULL);
209
210/* init perfmons */
211static void df_v3_6_sw_init(struct amdgpu_device *adev)
212{
213 int i, ret;
214
215 ret = device_create_file(adev->dev, &dev_attr_df_cntr_avail);
216 if (ret)
217 DRM_ERROR("failed to create file for available df counters\n");
218
219 for (i = 0; i < AMDGPU_MAX_DF_PERFMONS; i++)
220 adev->df_perfmon_config_assign_mask[i] = 0;
221}
222
223static void df_v3_6_sw_fini(struct amdgpu_device *adev)
224{
225
226 device_remove_file(adev->dev, &dev_attr_df_cntr_avail);
227
228}
229
230static void df_v3_6_enable_broadcast_mode(struct amdgpu_device *adev,
231 bool enable)
232{
233 u32 tmp;
234
235 if (enable) {
236 tmp = RREG32_SOC15(DF, 0, mmFabricConfigAccessControl);
237 tmp &= ~FabricConfigAccessControl__CfgRegInstAccEn_MASK;
238 WREG32_SOC15(DF, 0, mmFabricConfigAccessControl, tmp);
239 } else
240 WREG32_SOC15(DF, 0, mmFabricConfigAccessControl,
241 mmFabricConfigAccessControl_DEFAULT);
242}
243
244static u32 df_v3_6_get_fb_channel_number(struct amdgpu_device *adev)
245{
246 u32 tmp;
247
248 tmp = RREG32_SOC15(DF, 0, mmDF_CS_UMC_AON0_DramBaseAddress0);
249 tmp &= DF_CS_UMC_AON0_DramBaseAddress0__IntLvNumChan_MASK;
250 tmp >>= DF_CS_UMC_AON0_DramBaseAddress0__IntLvNumChan__SHIFT;
251
252 return tmp;
253}
254
255static u32 df_v3_6_get_hbm_channel_number(struct amdgpu_device *adev)
256{
257 int fb_channel_number;
258
259 fb_channel_number = adev->df_funcs->get_fb_channel_number(adev);
260 if (fb_channel_number >= ARRAY_SIZE(df_v3_6_channel_number))
261 fb_channel_number = 0;
262
263 return df_v3_6_channel_number[fb_channel_number];
264}
265
266static void df_v3_6_update_medium_grain_clock_gating(struct amdgpu_device *adev,
267 bool enable)
268{
269 u32 tmp;
270
271 if (adev->cg_flags & AMD_CG_SUPPORT_DF_MGCG) {
272 /* Put DF on broadcast mode */
273 adev->df_funcs->enable_broadcast_mode(adev, true);
274
275 if (enable) {
276 tmp = RREG32_SOC15(DF, 0,
277 mmDF_PIE_AON0_DfGlobalClkGater);
278 tmp &= ~DF_PIE_AON0_DfGlobalClkGater__MGCGMode_MASK;
279 tmp |= DF_V3_6_MGCG_ENABLE_15_CYCLE_DELAY;
280 WREG32_SOC15(DF, 0,
281 mmDF_PIE_AON0_DfGlobalClkGater, tmp);
282 } else {
283 tmp = RREG32_SOC15(DF, 0,
284 mmDF_PIE_AON0_DfGlobalClkGater);
285 tmp &= ~DF_PIE_AON0_DfGlobalClkGater__MGCGMode_MASK;
286 tmp |= DF_V3_6_MGCG_DISABLE;
287 WREG32_SOC15(DF, 0,
288 mmDF_PIE_AON0_DfGlobalClkGater, tmp);
289 }
290
291 /* Exit broadcast mode */
292 adev->df_funcs->enable_broadcast_mode(adev, false);
293 }
294}
295
296static void df_v3_6_get_clockgating_state(struct amdgpu_device *adev,
297 u32 *flags)
298{
299 u32 tmp;
300
301 /* AMD_CG_SUPPORT_DF_MGCG */
302 tmp = RREG32_SOC15(DF, 0, mmDF_PIE_AON0_DfGlobalClkGater);
303 if (tmp & DF_V3_6_MGCG_ENABLE_15_CYCLE_DELAY)
304 *flags |= AMD_CG_SUPPORT_DF_MGCG;
305}
306
307/* get assigned df perfmon ctr as int */
308static int df_v3_6_pmc_config_2_cntr(struct amdgpu_device *adev,
309 uint64_t config)
310{
311 int i;
312
313 for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
314 if ((config & 0x0FFFFFFUL) ==
315 adev->df_perfmon_config_assign_mask[i])
316 return i;
317 }
318
319 return -EINVAL;
320}
321
322/* get address based on counter assignment */
323static void df_v3_6_pmc_get_addr(struct amdgpu_device *adev,
324 uint64_t config,
325 int is_ctrl,
326 uint32_t *lo_base_addr,
327 uint32_t *hi_base_addr)
328{
329 int target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
330
331 if (target_cntr < 0)
332 return;
333
334 switch (target_cntr) {
335
336 case 0:
337 *lo_base_addr = is_ctrl ? smnPerfMonCtlLo0 : smnPerfMonCtrLo0;
338 *hi_base_addr = is_ctrl ? smnPerfMonCtlHi0 : smnPerfMonCtrHi0;
339 break;
340 case 1:
341 *lo_base_addr = is_ctrl ? smnPerfMonCtlLo1 : smnPerfMonCtrLo1;
342 *hi_base_addr = is_ctrl ? smnPerfMonCtlHi1 : smnPerfMonCtrHi1;
343 break;
344 case 2:
345 *lo_base_addr = is_ctrl ? smnPerfMonCtlLo2 : smnPerfMonCtrLo2;
346 *hi_base_addr = is_ctrl ? smnPerfMonCtlHi2 : smnPerfMonCtrHi2;
347 break;
348 case 3:
349 *lo_base_addr = is_ctrl ? smnPerfMonCtlLo3 : smnPerfMonCtrLo3;
350 *hi_base_addr = is_ctrl ? smnPerfMonCtlHi3 : smnPerfMonCtrHi3;
351 break;
352
353 }
354
355}
356
357/* get read counter address */
358static void df_v3_6_pmc_get_read_settings(struct amdgpu_device *adev,
359 uint64_t config,
360 uint32_t *lo_base_addr,
361 uint32_t *hi_base_addr)
362{
363 df_v3_6_pmc_get_addr(adev, config, 0, lo_base_addr, hi_base_addr);
364}
365
366/* get control counter settings i.e. address and values to set */
367static int df_v3_6_pmc_get_ctrl_settings(struct amdgpu_device *adev,
368 uint64_t config,
369 uint32_t *lo_base_addr,
370 uint32_t *hi_base_addr,
371 uint32_t *lo_val,
372 uint32_t *hi_val)
373{
374
375 uint32_t eventsel, instance, unitmask;
376 uint32_t instance_10, instance_5432, instance_76;
377
378 df_v3_6_pmc_get_addr(adev, config, 1, lo_base_addr, hi_base_addr);
379
380 if ((*lo_base_addr == 0) || (*hi_base_addr == 0)) {
381 DRM_ERROR("[DF PMC] addressing not retrieved! Lo: %x, Hi: %x",
382 *lo_base_addr, *hi_base_addr);
383 return -ENXIO;
384 }
385
386 eventsel = DF_V3_6_GET_EVENT(config) & 0x3f;
387 unitmask = DF_V3_6_GET_UNITMASK(config) & 0xf;
388 instance = DF_V3_6_GET_INSTANCE(config);
389
390 instance_10 = instance & 0x3;
391 instance_5432 = (instance >> 2) & 0xf;
392 instance_76 = (instance >> 6) & 0x3;
393
394 *lo_val = (unitmask << 8) | (instance_10 << 6) | eventsel | (1 << 22);
395 *hi_val = (instance_76 << 29) | instance_5432;
396
397 DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x:%08x",
398 config, *lo_base_addr, *hi_base_addr, *lo_val, *hi_val);
399
400 return 0;
401}
402
403/* add df performance counters for read */
404static int df_v3_6_pmc_add_cntr(struct amdgpu_device *adev,
405 uint64_t config)
406{
407 int i, target_cntr;
408
409 target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
410
411 if (target_cntr >= 0)
412 return 0;
413
414 for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
415 if (adev->df_perfmon_config_assign_mask[i] == 0U) {
416 adev->df_perfmon_config_assign_mask[i] =
417 config & 0x0FFFFFFUL;
418 return 0;
419 }
420 }
421
422 return -ENOSPC;
423}
424
425/* release performance counter */
426static void df_v3_6_pmc_release_cntr(struct amdgpu_device *adev,
427 uint64_t config)
428{
429 int target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
430
431 if (target_cntr >= 0)
432 adev->df_perfmon_config_assign_mask[target_cntr] = 0ULL;
433}
434
435
436static void df_v3_6_reset_perfmon_cntr(struct amdgpu_device *adev,
437 uint64_t config)
438{
439 uint32_t lo_base_addr, hi_base_addr;
440
441 df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
442 &hi_base_addr);
443
444 if ((lo_base_addr == 0) || (hi_base_addr == 0))
445 return;
446
447 df_v3_6_perfmon_wreg(adev, lo_base_addr, 0, hi_base_addr, 0);
448}
449
450static int df_v3_6_pmc_start(struct amdgpu_device *adev, uint64_t config,
451 int is_enable)
452{
453 uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
454 int ret = 0;
455
456 switch (adev->asic_type) {
457 case CHIP_VEGA20:
458
459 df_v3_6_reset_perfmon_cntr(adev, config);
460
461 if (is_enable) {
462 ret = df_v3_6_pmc_add_cntr(adev, config);
463 } else {
464 ret = df_v3_6_pmc_get_ctrl_settings(adev,
465 config,
466 &lo_base_addr,
467 &hi_base_addr,
468 &lo_val,
469 &hi_val);
470
471 if (ret)
472 return ret;
473
474 df_v3_6_perfmon_wreg(adev, lo_base_addr, lo_val,
475 hi_base_addr, hi_val);
476 }
477
478 break;
479 default:
480 break;
481 }
482
483 return ret;
484}
485
486static int df_v3_6_pmc_stop(struct amdgpu_device *adev, uint64_t config,
487 int is_disable)
488{
489 uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
490 int ret = 0;
491
492 switch (adev->asic_type) {
493 case CHIP_VEGA20:
494 ret = df_v3_6_pmc_get_ctrl_settings(adev,
495 config,
496 &lo_base_addr,
497 &hi_base_addr,
498 &lo_val,
499 &hi_val);
500
501 if (ret)
502 return ret;
503
504 df_v3_6_perfmon_wreg(adev, lo_base_addr, 0, hi_base_addr, 0);
505
506 if (is_disable)
507 df_v3_6_pmc_release_cntr(adev, config);
508
509 break;
510 default:
511 break;
512 }
513
514 return ret;
515}
516
517static void df_v3_6_pmc_get_count(struct amdgpu_device *adev,
518 uint64_t config,
519 uint64_t *count)
520{
521 uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
522 *count = 0;
523
524 switch (adev->asic_type) {
525 case CHIP_VEGA20:
526
527 df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
528 &hi_base_addr);
529
530 if ((lo_base_addr == 0) || (hi_base_addr == 0))
531 return;
532
533 df_v3_6_perfmon_rreg(adev, lo_base_addr, &lo_val,
534 hi_base_addr, &hi_val);
535
536 *count = ((hi_val | 0ULL) << 32) | (lo_val | 0ULL);
537
538 if (*count >= DF_V3_6_PERFMON_OVERFLOW)
539 *count = 0;
540
541 DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x:%08x",
542 config, lo_base_addr, hi_base_addr, lo_val, hi_val);
543
544 break;
545
546 default:
547 break;
548 }
549}
550
551const struct amdgpu_df_funcs df_v3_6_funcs = {
552 .sw_init = df_v3_6_sw_init,
553 .sw_fini = df_v3_6_sw_fini,
554 .enable_broadcast_mode = df_v3_6_enable_broadcast_mode,
555 .get_fb_channel_number = df_v3_6_get_fb_channel_number,
556 .get_hbm_channel_number = df_v3_6_get_hbm_channel_number,
557 .update_medium_grain_clock_gating =
558 df_v3_6_update_medium_grain_clock_gating,
559 .get_clockgating_state = df_v3_6_get_clockgating_state,
560 .pmc_start = df_v3_6_pmc_start,
561 .pmc_stop = df_v3_6_pmc_stop,
562 .pmc_get_count = df_v3_6_pmc_get_count,
563 .get_fica = df_v3_6_get_fica,
564 .set_fica = df_v3_6_set_fica
565};