Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983// SPDX-License-Identifier: MIT/* * Copyright 2025 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */
#include <linux/delay.h>
#include "amdgpu.h"#include "amdgpu_reset.h"#include "amdgpu_trace.h"#include "amdgpu_virt.h"#include "amdgpu_reg_access.h"
#define AMDGPU_PCIE_INDEX_FALLBACK (0x38 >> 2)#define AMDGPU_PCIE_INDEX_HI_FALLBACK (0x44 >> 2)#define AMDGPU_PCIE_DATA_FALLBACK (0x3C >> 2)
void amdgpu_reg_access_init(struct amdgpu_device *adev){ spin_lock_init(&adev->reg.smc.lock); adev->reg.smc.rreg = NULL; adev->reg.smc.wreg = NULL;
spin_lock_init(&adev->reg.uvd_ctx.lock); adev->reg.uvd_ctx.rreg = NULL; adev->reg.uvd_ctx.wreg = NULL;
spin_lock_init(&adev->reg.didt.lock); adev->reg.didt.rreg = NULL; adev->reg.didt.wreg = NULL;
spin_lock_init(&adev->reg.gc_cac.lock); adev->reg.gc_cac.rreg = NULL; adev->reg.gc_cac.wreg = NULL;
spin_lock_init(&adev->reg.se_cac.lock); adev->reg.se_cac.rreg = NULL; adev->reg.se_cac.wreg = NULL;
spin_lock_init(&adev->reg.audio_endpt.lock); adev->reg.audio_endpt.rreg = NULL; adev->reg.audio_endpt.wreg = NULL;
spin_lock_init(&adev->reg.pcie.lock); adev->reg.pcie.rreg = NULL; adev->reg.pcie.wreg = NULL; adev->reg.pcie.rreg_ext = NULL; adev->reg.pcie.wreg_ext = NULL; adev->reg.pcie.rreg64 = NULL; adev->reg.pcie.wreg64 = NULL; adev->reg.pcie.rreg64_ext = NULL; adev->reg.pcie.wreg64_ext = NULL; adev->reg.pcie.port_rreg = NULL; adev->reg.pcie.port_wreg = NULL;}
uint32_t amdgpu_reg_smc_rd32(struct amdgpu_device *adev, uint32_t reg){ if (!adev->reg.smc.rreg) { dev_err_once(adev->dev, "SMC register read not supported\n"); return 0; } return adev->reg.smc.rreg(adev, reg);}
void amdgpu_reg_smc_wr32(struct amdgpu_device *adev, uint32_t reg, uint32_t v){ if (!adev->reg.smc.wreg) { dev_err_once(adev->dev, "SMC register write not supported\n"); return; } adev->reg.smc.wreg(adev, reg, v);}
uint32_t amdgpu_reg_uvd_ctx_rd32(struct amdgpu_device *adev, uint32_t reg){ if (!adev->reg.uvd_ctx.rreg) { dev_err_once(adev->dev, "UVD_CTX register read not supported\n"); return 0; } return adev->reg.uvd_ctx.rreg(adev, reg);}
void amdgpu_reg_uvd_ctx_wr32(struct amdgpu_device *adev, uint32_t reg, uint32_t v){ if (!adev->reg.uvd_ctx.wreg) { dev_err_once(adev->dev, "UVD_CTX register write not supported\n"); return; } adev->reg.uvd_ctx.wreg(adev, reg, v);}
uint32_t amdgpu_reg_didt_rd32(struct amdgpu_device *adev, uint32_t reg){ if (!adev->reg.didt.rreg) { dev_err_once(adev->dev, "DIDT register read not supported\n"); return 0; } return adev->reg.didt.rreg(adev, reg);}
void amdgpu_reg_didt_wr32(struct amdgpu_device *adev, uint32_t reg, uint32_t v){ if (!adev->reg.didt.wreg) { dev_err_once(adev->dev, "DIDT register write not supported\n"); return; } adev->reg.didt.wreg(adev, reg, v);}
uint32_t amdgpu_reg_gc_cac_rd32(struct amdgpu_device *adev, uint32_t reg){ if (!adev->reg.gc_cac.rreg) { dev_err_once(adev->dev, "GC_CAC register read not supported\n"); return 0; } return adev->reg.gc_cac.rreg(adev, reg);}
void amdgpu_reg_gc_cac_wr32(struct amdgpu_device *adev, uint32_t reg, uint32_t v){ if (!adev->reg.gc_cac.wreg) { dev_err_once(adev->dev, "GC_CAC register write not supported\n"); return; } adev->reg.gc_cac.wreg(adev, reg, v);}
uint32_t amdgpu_reg_se_cac_rd32(struct amdgpu_device *adev, uint32_t reg){ if (!adev->reg.se_cac.rreg) { dev_err_once(adev->dev, "SE_CAC register read not supported\n"); return 0; } return adev->reg.se_cac.rreg(adev, reg);}
void amdgpu_reg_se_cac_wr32(struct amdgpu_device *adev, uint32_t reg, uint32_t v){ if (!adev->reg.se_cac.wreg) { dev_err_once(adev->dev, "SE_CAC register write not supported\n"); return; } adev->reg.se_cac.wreg(adev, reg, v);}
uint32_t amdgpu_reg_audio_endpt_rd32(struct amdgpu_device *adev, uint32_t block, uint32_t reg){ if (!adev->reg.audio_endpt.rreg) { dev_err_once(adev->dev, "AUDIO_ENDPT register read not supported\n"); return 0; } return adev->reg.audio_endpt.rreg(adev, block, reg);}
void amdgpu_reg_audio_endpt_wr32(struct amdgpu_device *adev, uint32_t block, uint32_t reg, uint32_t v){ if (!adev->reg.audio_endpt.wreg) { dev_err_once(adev->dev, "AUDIO_ENDPT register write not supported\n"); return; } adev->reg.audio_endpt.wreg(adev, block, reg, v);}
uint32_t amdgpu_reg_pcie_rd32(struct amdgpu_device *adev, uint32_t reg){ if (!adev->reg.pcie.rreg) { dev_err_once(adev->dev, "PCIE register read not supported\n"); return 0; } return adev->reg.pcie.rreg(adev, reg);}
void amdgpu_reg_pcie_wr32(struct amdgpu_device *adev, uint32_t reg, uint32_t v){ if (!adev->reg.pcie.wreg) { dev_err_once(adev->dev, "PCIE register write not supported\n"); return; } adev->reg.pcie.wreg(adev, reg, v);}
uint32_t amdgpu_reg_pcie_ext_rd32(struct amdgpu_device *adev, uint64_t reg){ if (!adev->reg.pcie.rreg_ext) { dev_err_once(adev->dev, "PCIE EXT register read not supported\n"); return 0; } return adev->reg.pcie.rreg_ext(adev, reg);}
void amdgpu_reg_pcie_ext_wr32(struct amdgpu_device *adev, uint64_t reg, uint32_t v){ if (!adev->reg.pcie.wreg_ext) { dev_err_once(adev->dev, "PCIE EXT register write not supported\n"); return; } adev->reg.pcie.wreg_ext(adev, reg, v);}
uint64_t amdgpu_reg_pcie_rd64(struct amdgpu_device *adev, uint32_t reg){ if (!adev->reg.pcie.rreg64) { dev_err_once(adev->dev, "PCIE 64-bit register read not supported\n"); return 0; } return adev->reg.pcie.rreg64(adev, reg);}
void amdgpu_reg_pcie_wr64(struct amdgpu_device *adev, uint32_t reg, uint64_t v){ if (!adev->reg.pcie.wreg64) { dev_err_once(adev->dev, "PCIE 64-bit register write not supported\n"); return; } adev->reg.pcie.wreg64(adev, reg, v);}
uint64_t amdgpu_reg_pcie_ext_rd64(struct amdgpu_device *adev, uint64_t reg){ if (!adev->reg.pcie.rreg64_ext) { dev_err_once(adev->dev, "PCIE EXT 64-bit register read not supported\n"); return 0; } return adev->reg.pcie.rreg64_ext(adev, reg);}
void amdgpu_reg_pcie_ext_wr64(struct amdgpu_device *adev, uint64_t reg, uint64_t v){ if (!adev->reg.pcie.wreg64_ext) { dev_err_once(adev->dev, "PCIE EXT 64-bit register write not supported\n"); return; } adev->reg.pcie.wreg64_ext(adev, reg, v);}
uint32_t amdgpu_reg_pciep_rd32(struct amdgpu_device *adev, uint32_t reg){ if (!adev->reg.pcie.port_rreg) { dev_err_once(adev->dev, "PCIEP register read not supported\n"); return 0; } return adev->reg.pcie.port_rreg(adev, reg);}
void amdgpu_reg_pciep_wr32(struct amdgpu_device *adev, uint32_t reg, uint32_t v){ if (!adev->reg.pcie.port_wreg) { dev_err_once(adev->dev, "PCIEP register write not supported\n"); return; } adev->reg.pcie.port_wreg(adev, reg, v);}
static int amdgpu_reg_get_smn_base_version(struct amdgpu_device *adev){ struct pci_dev *pdev = adev->pdev; int id;
if (amdgpu_sriov_vf(adev)) return -EOPNOTSUPP;
id = (pdev->device >> 4) & 0xFFFF; if (id == 0x74A || id == 0x74B || id == 0x75A || id == 0x75B) return 1;
return -EOPNOTSUPP;}
uint64_t amdgpu_reg_get_smn_base64(struct amdgpu_device *adev, enum amd_hw_ip_block_type block, int die_inst){ if (!adev->reg.smn.get_smn_base) { int version = amdgpu_reg_get_smn_base_version(adev); switch (version) { case 1: return amdgpu_reg_smn_v1_0_get_base(adev, block, die_inst); default: dev_err_once( adev->dev, "SMN base address query not supported for this device\n"); return 0; } } return adev->reg.smn.get_smn_base(adev, block, die_inst);}
uint64_t amdgpu_reg_smn_v1_0_get_base(struct amdgpu_device *adev, enum amd_hw_ip_block_type block, int die_inst){ uint64_t smn_base;
if (die_inst == 0) return 0;
switch (block) { case XGMI_HWIP: case NBIO_HWIP: case MP0_HWIP: case UMC_HWIP: case DF_HWIP: smn_base = ((uint64_t)(die_inst & 0x3) << 32) | (1ULL << 34); break; default: dev_warn_once( adev->dev, "SMN base address query not supported for this block %d\n", block); smn_base = 0; break; }
return smn_base;}
/* * register access helper functions. */
/** * amdgpu_device_rreg - read a memory mapped IO or indirect register * * @adev: amdgpu_device pointer * @reg: dword aligned register offset * @acc_flags: access flags which require special behavior * * Returns the 32 bit value from the offset specified. */uint32_t amdgpu_device_rreg(struct amdgpu_device *adev, uint32_t reg, uint32_t acc_flags){ uint32_t ret;
if (amdgpu_device_skip_hw_access(adev)) return 0;
if ((reg * 4) < adev->rmmio_size) { if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev) && down_read_trylock(&adev->reset_domain->sem)) { ret = amdgpu_kiq_rreg(adev, reg, 0); up_read(&adev->reset_domain->sem); } else { ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); } } else { ret = amdgpu_reg_pcie_rd32(adev, reg * 4); }
trace_amdgpu_device_rreg(adev->pdev->device, reg, ret);
return ret;}
/* * MMIO register read with bytes helper functions * @offset:bytes offset from MMIO start */
/** * amdgpu_mm_rreg8 - read a memory mapped IO register * * @adev: amdgpu_device pointer * @offset: byte aligned register offset * * Returns the 8 bit value from the offset specified. */uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset){ if (amdgpu_device_skip_hw_access(adev)) return 0;
if (offset < adev->rmmio_size) return (readb(adev->rmmio + offset));
dev_err(adev->dev, "invalid MMIO read offset 0x%x (rmmio size 0x%x)\n", offset, (unsigned int)adev->rmmio_size); return 0;}
/** * amdgpu_device_xcc_rreg - read a memory mapped IO or indirect register with specific XCC * * @adev: amdgpu_device pointer * @reg: dword aligned register offset * @acc_flags: access flags which require special behavior * @xcc_id: xcc accelerated compute core id * * Returns the 32 bit value from the offset specified. */uint32_t amdgpu_device_xcc_rreg(struct amdgpu_device *adev, uint32_t reg, uint32_t acc_flags, uint32_t xcc_id){ uint32_t ret, rlcg_flag;
if (amdgpu_device_skip_hw_access(adev)) return 0;
if ((reg * 4) < adev->rmmio_size) { if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_runtime(adev) && adev->gfx.rlc.rlcg_reg_access_supported && amdgpu_virt_get_rlcg_reg_access_flag( adev, acc_flags, GC_HWIP, false, &rlcg_flag)) { ret = amdgpu_virt_rlcg_reg_rw(adev, reg, 0, rlcg_flag, GET_INST(GC, xcc_id)); } else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev) && down_read_trylock(&adev->reset_domain->sem)) { ret = amdgpu_kiq_rreg(adev, reg, xcc_id); up_read(&adev->reset_domain->sem); } else { ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); } } else { ret = amdgpu_reg_pcie_rd32(adev, reg * 4); }
return ret;}
/* * MMIO register write with bytes helper functions * @offset:bytes offset from MMIO start * @value: the value want to be written to the register */
/** * amdgpu_mm_wreg8 - read a memory mapped IO register * * @adev: amdgpu_device pointer * @offset: byte aligned register offset * @value: 8 bit value to write * * Writes the value specified to the offset specified. */void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value){ if (amdgpu_device_skip_hw_access(adev)) return;
if (offset < adev->rmmio_size) { writeb(value, adev->rmmio + offset); } else { dev_err(adev->dev, "invalid MMIO write offset 0x%x (rmmio size 0x%x)\n", offset, (unsigned int)adev->rmmio_size); return; }}
/** * amdgpu_device_wreg - write to a memory mapped IO or indirect register * * @adev: amdgpu_device pointer * @reg: dword aligned register offset * @v: 32 bit value to write to the register * @acc_flags: access flags which require special behavior * * Writes the value specified to the offset specified. */void amdgpu_device_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, uint32_t acc_flags){ if (amdgpu_device_skip_hw_access(adev)) return;
if ((reg * 4) < adev->rmmio_size) { if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev) && down_read_trylock(&adev->reset_domain->sem)) { amdgpu_kiq_wreg(adev, reg, v, 0); up_read(&adev->reset_domain->sem); } else { writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); } } else { amdgpu_reg_pcie_wr32(adev, reg * 4, v); }
trace_amdgpu_device_wreg(adev->pdev->device, reg, v);}
/** * amdgpu_mm_wreg_mmio_rlc - write register either with direct/indirect mmio or with RLC path if in range * * @adev: amdgpu_device pointer * @reg: mmio/rlc register * @v: value to write * @xcc_id: xcc accelerated compute core id * * this function is invoked only for the debugfs register access */void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev, uint32_t reg, uint32_t v, uint32_t xcc_id){ if (amdgpu_device_skip_hw_access(adev)) return;
if (amdgpu_sriov_fullaccess(adev) && adev->gfx.rlc.funcs && adev->gfx.rlc.funcs->is_rlcg_access_range) { if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg)) return amdgpu_sriov_wreg(adev, reg, v, 0, 0, xcc_id); } else if ((reg * 4) >= adev->rmmio_size) { amdgpu_reg_pcie_wr32(adev, reg * 4, v); } else { writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); }}
/** * amdgpu_device_xcc_wreg - write to a memory mapped IO or indirect register with specific XCC * * @adev: amdgpu_device pointer * @reg: dword aligned register offset * @v: 32 bit value to write to the register * @acc_flags: access flags which require special behavior * @xcc_id: xcc accelerated compute core id * * Writes the value specified to the offset specified. */void amdgpu_device_xcc_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, uint32_t acc_flags, uint32_t xcc_id){ uint32_t rlcg_flag;
if (amdgpu_device_skip_hw_access(adev)) return;
if ((reg * 4) < adev->rmmio_size) { if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_runtime(adev) && adev->gfx.rlc.rlcg_reg_access_supported && amdgpu_virt_get_rlcg_reg_access_flag( adev, acc_flags, GC_HWIP, true, &rlcg_flag)) { amdgpu_virt_rlcg_reg_rw(adev, reg, v, rlcg_flag, GET_INST(GC, xcc_id)); } else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev) && down_read_trylock(&adev->reset_domain->sem)) { amdgpu_kiq_wreg(adev, reg, v, xcc_id); up_read(&adev->reset_domain->sem); } else { writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); } } else { amdgpu_reg_pcie_wr32(adev, reg * 4, v); }}
/** * amdgpu_device_indirect_rreg - read an indirect register * * @adev: amdgpu_device pointer * @reg_addr: indirect register address to read from * * Returns the value of indirect register @reg_addr */u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev, u32 reg_addr){ unsigned long flags, pcie_index, pcie_data; void __iomem *pcie_index_offset; void __iomem *pcie_data_offset; u32 r;
pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
spin_lock_irqsave(&adev->reg.pcie.lock, flags); pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
writel(reg_addr, pcie_index_offset); readl(pcie_index_offset); r = readl(pcie_data_offset); spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
return r;}
u32 amdgpu_device_indirect_rreg_ext(struct amdgpu_device *adev, u64 reg_addr){ unsigned long flags, pcie_index, pcie_index_hi, pcie_data; u32 r; void __iomem *pcie_index_offset; void __iomem *pcie_index_hi_offset; void __iomem *pcie_data_offset;
if (unlikely(!adev->nbio.funcs)) { pcie_index = AMDGPU_PCIE_INDEX_FALLBACK; pcie_data = AMDGPU_PCIE_DATA_FALLBACK; } else { pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); }
if (reg_addr >> 32) { if (unlikely(!adev->nbio.funcs)) pcie_index_hi = AMDGPU_PCIE_INDEX_HI_FALLBACK; else pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset( adev); } else { pcie_index_hi = 0; }
spin_lock_irqsave(&adev->reg.pcie.lock, flags); pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; if (pcie_index_hi != 0) pcie_index_hi_offset = (void __iomem *)adev->rmmio + pcie_index_hi * 4;
writel(reg_addr, pcie_index_offset); readl(pcie_index_offset); if (pcie_index_hi != 0) { writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset); readl(pcie_index_hi_offset); } r = readl(pcie_data_offset);
/* clear the high bits */ if (pcie_index_hi != 0) { writel(0, pcie_index_hi_offset); readl(pcie_index_hi_offset); }
spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
return r;}
/** * amdgpu_device_indirect_rreg64 - read a 64bits indirect register * * @adev: amdgpu_device pointer * @reg_addr: indirect register address to read from * * Returns the value of indirect register @reg_addr */u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev, u32 reg_addr){ unsigned long flags, pcie_index, pcie_data; void __iomem *pcie_index_offset; void __iomem *pcie_data_offset; u64 r;
pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
spin_lock_irqsave(&adev->reg.pcie.lock, flags); pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
/* read low 32 bits */ writel(reg_addr, pcie_index_offset); readl(pcie_index_offset); r = readl(pcie_data_offset); /* read high 32 bits */ writel(reg_addr + 4, pcie_index_offset); readl(pcie_index_offset); r |= ((u64)readl(pcie_data_offset) << 32); spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
return r;}
u64 amdgpu_device_indirect_rreg64_ext(struct amdgpu_device *adev, u64 reg_addr){ unsigned long flags, pcie_index, pcie_data; unsigned long pcie_index_hi = 0; void __iomem *pcie_index_offset; void __iomem *pcie_index_hi_offset; void __iomem *pcie_data_offset; u64 r;
pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset)) pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
spin_lock_irqsave(&adev->reg.pcie.lock, flags); pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; if (pcie_index_hi != 0) pcie_index_hi_offset = (void __iomem *)adev->rmmio + pcie_index_hi * 4;
/* read low 32 bits */ writel(reg_addr, pcie_index_offset); readl(pcie_index_offset); if (pcie_index_hi != 0) { writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset); readl(pcie_index_hi_offset); } r = readl(pcie_data_offset); /* read high 32 bits */ writel(reg_addr + 4, pcie_index_offset); readl(pcie_index_offset); if (pcie_index_hi != 0) { writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset); readl(pcie_index_hi_offset); } r |= ((u64)readl(pcie_data_offset) << 32);
/* clear the high bits */ if (pcie_index_hi != 0) { writel(0, pcie_index_hi_offset); readl(pcie_index_hi_offset); }
spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
return r;}
/** * amdgpu_device_indirect_wreg - write an indirect register address * * @adev: amdgpu_device pointer * @reg_addr: indirect register offset * @reg_data: indirect register data * */void amdgpu_device_indirect_wreg(struct amdgpu_device *adev, u32 reg_addr, u32 reg_data){ unsigned long flags, pcie_index, pcie_data; void __iomem *pcie_index_offset; void __iomem *pcie_data_offset;
pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
spin_lock_irqsave(&adev->reg.pcie.lock, flags); pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
writel(reg_addr, pcie_index_offset); readl(pcie_index_offset); writel(reg_data, pcie_data_offset); readl(pcie_data_offset); spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);}
void amdgpu_device_indirect_wreg_ext(struct amdgpu_device *adev, u64 reg_addr, u32 reg_data){ unsigned long flags, pcie_index, pcie_index_hi, pcie_data; void __iomem *pcie_index_offset; void __iomem *pcie_index_hi_offset; void __iomem *pcie_data_offset;
pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset)) pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev); else pcie_index_hi = 0;
spin_lock_irqsave(&adev->reg.pcie.lock, flags); pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; if (pcie_index_hi != 0) pcie_index_hi_offset = (void __iomem *)adev->rmmio + pcie_index_hi * 4;
writel(reg_addr, pcie_index_offset); readl(pcie_index_offset); if (pcie_index_hi != 0) { writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset); readl(pcie_index_hi_offset); } writel(reg_data, pcie_data_offset); readl(pcie_data_offset);
/* clear the high bits */ if (pcie_index_hi != 0) { writel(0, pcie_index_hi_offset); readl(pcie_index_hi_offset); }
spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);}
/** * amdgpu_device_indirect_wreg64 - write a 64bits indirect register address * * @adev: amdgpu_device pointer * @reg_addr: indirect register offset * @reg_data: indirect register data * */void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev, u32 reg_addr, u64 reg_data){ unsigned long flags, pcie_index, pcie_data; void __iomem *pcie_index_offset; void __iomem *pcie_data_offset;
pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
spin_lock_irqsave(&adev->reg.pcie.lock, flags); pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
/* write low 32 bits */ writel(reg_addr, pcie_index_offset); readl(pcie_index_offset); writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset); readl(pcie_data_offset); /* write high 32 bits */ writel(reg_addr + 4, pcie_index_offset); readl(pcie_index_offset); writel((u32)(reg_data >> 32), pcie_data_offset); readl(pcie_data_offset); spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);}
void amdgpu_device_indirect_wreg64_ext(struct amdgpu_device *adev, u64 reg_addr, u64 reg_data){ unsigned long flags, pcie_index, pcie_data; unsigned long pcie_index_hi = 0; void __iomem *pcie_index_offset; void __iomem *pcie_index_hi_offset; void __iomem *pcie_data_offset;
pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset)) pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
spin_lock_irqsave(&adev->reg.pcie.lock, flags); pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; if (pcie_index_hi != 0) pcie_index_hi_offset = (void __iomem *)adev->rmmio + pcie_index_hi * 4;
/* write low 32 bits */ writel(reg_addr, pcie_index_offset); readl(pcie_index_offset); if (pcie_index_hi != 0) { writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset); readl(pcie_index_hi_offset); } writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset); readl(pcie_data_offset); /* write high 32 bits */ writel(reg_addr + 4, pcie_index_offset); readl(pcie_index_offset); if (pcie_index_hi != 0) { writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset); readl(pcie_index_hi_offset); } writel((u32)(reg_data >> 32), pcie_data_offset); readl(pcie_data_offset);
/* clear the high bits */ if (pcie_index_hi != 0) { writel(0, pcie_index_hi_offset); readl(pcie_index_hi_offset); }
spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);}
u32 amdgpu_device_pcie_port_rreg(struct amdgpu_device *adev, u32 reg){ unsigned long flags, address, data; u32 r;
address = adev->nbio.funcs->get_pcie_port_index_offset(adev); data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
spin_lock_irqsave(&adev->reg.pcie.lock, flags); WREG32(address, reg * 4); (void)RREG32(address); r = RREG32(data); spin_unlock_irqrestore(&adev->reg.pcie.lock, flags); return r;}
void amdgpu_device_pcie_port_wreg(struct amdgpu_device *adev, u32 reg, u32 v){ unsigned long flags, address, data;
address = adev->nbio.funcs->get_pcie_port_index_offset(adev); data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
spin_lock_irqsave(&adev->reg.pcie.lock, flags); WREG32(address, reg * 4); (void)RREG32(address); WREG32(data, v); (void)RREG32(data); spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);}
uint32_t amdgpu_device_wait_on_rreg(struct amdgpu_device *adev, uint32_t inst, uint32_t reg_addr, char reg_name[], uint32_t expected_value, uint32_t mask){ uint32_t ret = 0; uint32_t old_ = 0; uint32_t tmp_ = RREG32(reg_addr); uint32_t loop = adev->usec_timeout;
while ((tmp_ & (mask)) != (expected_value)) { if (old_ != tmp_) { loop = adev->usec_timeout; old_ = tmp_; } else udelay(1); tmp_ = RREG32(reg_addr); loop--; if (!loop) { dev_warn( adev->dev, "Register(%d) [%s] failed to reach value 0x%08x != 0x%08xn", inst, reg_name, (uint32_t)expected_value, (uint32_t)(tmp_ & (mask))); ret = -ETIMEDOUT; break; } } return ret;}
uint32_t amdgpu_read_indexed_register(struct amdgpu_device *adev, u32 se_num, u32 sh_num, u32 reg_offset){ uint32_t val;
mutex_lock(&adev->grbm_idx_mutex); if (se_num != 0xffffffff || sh_num != 0xffffffff) amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff, 0);
val = RREG32(reg_offset);
if (se_num != 0xffffffff || sh_num != 0xffffffff) amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); mutex_unlock(&adev->grbm_idx_mutex); return val;}