Merge tag 'amd-drm-next-6.20-2026-01-23' of https://gitlab.freedesktop.org/agd5f/linux into drm-next

amd-drm-next-6.20-2026-01-23:

amdgpu:
- GC 12 fix
- UserQ fixes
- Misc error path fixes
- IP discovery fixes
- Misc cleanups
- HDMI fixes
- Documentation update
- Panel replay fixes
- Panel type handling fixes
- DCN 3.1.x fixes
- DC analog display fix
- SMU 6 fixes
- VCN 4.0.3 queue reset fix
- VCN 5.0.1 queue reset fix
- GPUVM TLB flush fix
- RAS fixes
- DC idle optimization fix

amdkfd:
- MQD fix for GC 9.4.3 and 9.5
- GC 11 cooperative launch fix
- GC 12.1 trap handler fixes
- Misc cleanups

radeon:
- Use drm_* logging helpers for VCE

Signed-off-by: Dave Airlie <[e-mail]>

From: Alex Deucher <[e-mail]>
Link: https://patch.msgid.link/[e-mail]

Documentation/gpu/amdgpu/apu-asic-info-table.csv

···
 Ryzen AI 330 series, Krackan Point, 3.6.0, 11.5.3, 4.0.5, 6.1.3, 14.0.5, 14.0.5
 Ryzen AI 350 series, Krackan Point, 3.5.0, 11.5.2, 4.0.5, 6.1.2, 14.0.4, 14.0.4
 Ryzen AI Max 300 series, Strix Halo, 3.5.1, 11.5.1, 4.0.6, 6.1.1, 14.0.1, 14.0.1
+Ryzen AI 9 475 / 470 / 465, Gorgon Point, 3.5.0, 11.5.0, 4.0.5, 6.1.0, 14.0.0, 14.0.0
+Ryzen AI 7 450, Gorgon Point, 3.5.0, 11.5.2, 4.0.5, 6.1.2, 14.0.4, 14.0.4
+Ryzen AI 5 440 / 435, Gorgon Point, 3.6.0, 11.5.3, 4.0.5, 6.1.3, 14.0.5, 14.0.5

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c

···
 	bp.size = size;
 	bp.byte_align = PAGE_SIZE;
 	bp.domain = domain;
-	bp.flags = AMDGPU_GEM_CREATE_CPU_GTT_USWC;
+	bp.flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
+		   AMDGPU_GEM_CREATE_CPU_GTT_USWC;
 	bp.type = ttm_bo_type_kernel;
 	bp.resv = NULL;
 	bp.bo_ptr_size = sizeof(struct amdgpu_bo);

drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c

···
 	return checksum;
 }
 
-static inline bool amdgpu_discovery_verify_checksum(uint8_t *data, uint32_t size,
+static inline bool amdgpu_discovery_verify_checksum(struct amdgpu_device *adev,
+							uint8_t *data, uint32_t size,
 						    uint16_t expected)
 {
-	return !!(amdgpu_discovery_calculate_checksum(data, size) == expected);
+	uint16_t calculated;
+
+	calculated = amdgpu_discovery_calculate_checksum(data, size);
+
+	if (calculated != expected) {
+		dev_err(adev->dev, "Discovery checksum failed: calc 0x%04x != exp 0x%04x, size %u.\n",
+				calculated, expected, size);
+		return false;
+	}
+
+	return true;
 }
 
 static inline bool amdgpu_discovery_verify_binary_signature(uint8_t *binary)
···
 		return -EINVAL;
 	}
 
-	if (!amdgpu_discovery_verify_checksum(discovery_bin + offset,
+	if (!amdgpu_discovery_verify_checksum(adev, discovery_bin + offset,
 					      le32_to_cpu(nhdr->size_bytes),
 					      checksum)) {
 		dev_dbg(adev->dev, "invalid nps info data table checksum\n");
···
 	size = le16_to_cpu(bhdr->binary_size) - offset;
 	checksum = le16_to_cpu(bhdr->binary_checksum);
 
-	if (!amdgpu_discovery_verify_checksum(discovery_bin + offset, size,
+	if (!amdgpu_discovery_verify_checksum(adev, discovery_bin + offset, size,
 					      checksum)) {
 		dev_err(adev->dev, "invalid ip discovery binary checksum\n");
 		r = -EINVAL;
···
 			goto out;
 		}
 
-		if (!amdgpu_discovery_verify_checksum(discovery_bin + offset,
+		if (!amdgpu_discovery_verify_checksum(adev, discovery_bin + offset,
 						      le16_to_cpu(ihdr->size),
 						      checksum)) {
 			dev_err(adev->dev, "invalid ip discovery data table checksum\n");
···
 			goto out;
 		}
 
-		if (!amdgpu_discovery_verify_checksum(discovery_bin + offset,
+		if (!amdgpu_discovery_verify_checksum(adev, discovery_bin + offset,
 						      le32_to_cpu(ghdr->size),
 						      checksum)) {
 			dev_err(adev->dev, "invalid gc data table checksum\n");
···
 			goto out;
 		}
 
-		if (!amdgpu_discovery_verify_checksum(
+		if (!amdgpu_discovery_verify_checksum(adev,
 			    discovery_bin + offset,
 			    sizeof(struct harvest_table), checksum)) {
 			dev_err(adev->dev, "invalid harvest data table checksum\n");
···
 			goto out;
 		}
 
-		if (!amdgpu_discovery_verify_checksum(
+		if (!amdgpu_discovery_verify_checksum(adev,
 			    discovery_bin + offset,
 			    le32_to_cpu(vhdr->size_bytes), checksum)) {
 			dev_err(adev->dev, "invalid vcn data table checksum\n");
···
 			goto out;
 		}
 
-		if (!amdgpu_discovery_verify_checksum(
+		if (!amdgpu_discovery_verify_checksum(adev,
 			    discovery_bin + offset,
 			    le32_to_cpu(mhdr->size_bytes), checksum)) {
 			dev_err(adev->dev, "invalid mall data table checksum\n");
···
 				  sizeof(*nps_data), false);
 
 	nhdr = (struct nps_info_header *)(nps_data);
-	if (!amdgpu_discovery_verify_checksum((uint8_t *)nps_data,
+	if (!amdgpu_discovery_verify_checksum(adev, (uint8_t *)nps_data,
 					      le32_to_cpu(nhdr->size_bytes),
 					      checksum)) {
 		dev_err(adev->dev, "nps data refresh, checksum mismatch\n");

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c

···
  */
 
 /**
- * amdgpu_fence_driver_guilty_force_completion - force signal of specified sequence
+ * amdgpu_fence_driver_update_timedout_fence_state - Update fence state and set errors
  *
- * @af: fence of the ring to signal
+ * @af: fence of the ring to update
  *
  */
-void amdgpu_fence_driver_guilty_force_completion(struct amdgpu_fence *af)
+void amdgpu_fence_driver_update_timedout_fence_state(struct amdgpu_fence *af)
 {
 	struct dma_fence *unprocessed;
 	struct dma_fence __rcu **ptr;
···
 }
 
 static void amdgpu_ring_backup_unprocessed_command(struct amdgpu_ring *ring,
-						   u64 start_wptr, u32 end_wptr)
+						   u64 start_wptr, u64 end_wptr)
 {
 	unsigned int first_idx = start_wptr & ring->buf_mask;
 	unsigned int last_idx = end_wptr & ring->buf_mask;

drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c

···
 failed_unlock:
 	spin_unlock_irqrestore(&kiq->ring_lock, flags);
 failed_kiq_hdp_flush:
-	dev_err(adev->dev, "failed to flush HDP via KIQ\n");
+	if (!amdgpu_in_reset(adev))
+		dev_err(adev->dev, "failed to flush HDP via KIQ\n");
 	return r < 0 ? r : -EIO;
 }
 

drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c

···
 
 	if (!adev->gmc.flush_pasid_uses_kiq || !ring->sched.ready) {
 
-		if (!adev->gmc.gmc_funcs->flush_gpu_tlb_pasid)
-			return 0;
+		if (!adev->gmc.gmc_funcs->flush_gpu_tlb_pasid) {
+			r = 0;
+			goto error_unlock_reset;
+		}
 
 		if (adev->gmc.flush_tlb_needs_extra_type_2)
 			adev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid,

drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c

···
 		if (job && job->vmid)
 			amdgpu_vmid_reset(adev, ring->vm_hub, job->vmid);
 		amdgpu_ring_undo(ring);
-		return r;
+		goto free_fence;
 	}
 	*f = &af->base;
 	/* get a ref for the job */

drivers/gpu/drm/amd/amdgpu/amdgpu_job.c

···
 	if (!entity)
 		return 0;
 
-	return drm_sched_job_init(&(*job)->base, entity, 1, owner,
-				  drm_client_id);
+	r = drm_sched_job_init(&(*job)->base, entity, 1, owner, drm_client_id);
+	if (!r)
+		return 0;
+
+	kfree((*job)->hw_vm_fence);
 
 err_fence:
 	kfree((*job)->hw_fence);

drivers/gpu/drm/amd/amdgpu/amdgpu_job.h

···
 #define AMDGPU_KERNEL_JOB_ID_FLUSH_GPU_TLB          (18446744073709551604ULL)
 #define AMDGPU_KERNEL_JOB_ID_KFD_GART_MAP           (18446744073709551603ULL)
 #define AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST          (18446744073709551602ULL)
+#define AMDGPU_KERNEL_JOB_ID_GFX_RING_TEST          (18446744073709551601ULL)
+#define AMDGPU_KERNEL_JOB_ID_SDMA_RING_TEST         (18446744073709551600ULL)
+#define AMDGPU_KERNEL_JOB_ID_VPE_RING_TEST          (18446744073709551599ULL)
+#define AMDGPU_KERNEL_JOB_ID_RUN_SHADER             (18446744073709551598ULL)
 
 struct amdgpu_job {
 	struct drm_sched_job    base;

drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c

···
 			if (!data->bps[i].ts)
 				continue;
 
+			/* U64_MAX is used to mark the record as invalid */
+			if (data->bps[i].retired_page == U64_MAX)
+				continue;
+
 			bps[r].bp = data->bps[i].retired_page;
 			r++;
 			if (r >= count)
···
 
 		if (amdgpu_ras_check_bad_page_unlock(con,
 			bps[j].retired_page << AMDGPU_GPU_PAGE_SHIFT)) {
+			/* set to U64_MAX to mark it as invalid */
+			data->bps[data->count].retired_page = U64_MAX;
 			data->count++;
 			data->space_left--;
 			continue;

drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c

···
 	if (r)
 		return r;
 
-	/* signal the guilty fence and set an error on all fences from the context */
+	/* set an error on all fences from the context */
 	if (guilty_fence)
-		amdgpu_fence_driver_guilty_force_completion(guilty_fence);
+		amdgpu_fence_driver_update_timedout_fence_state(guilty_fence);
 	/* Re-emit the non-guilty commands */
 	if (ring->ring_backup_entries_to_copy) {
 		amdgpu_ring_alloc_reemit(ring, ring->ring_backup_entries_to_copy);

drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h

···
 
 void amdgpu_fence_driver_set_error(struct amdgpu_ring *ring, int error);
 void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring);
-void amdgpu_fence_driver_guilty_force_completion(struct amdgpu_fence *af);
+void amdgpu_fence_driver_update_timedout_fence_state(struct amdgpu_fence *af);
 void amdgpu_fence_save_wptr(struct amdgpu_fence *af);
 
 int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring);

drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c

···
 	return r;
 }
 
+static void amdgpu_userq_hang_detect_work(struct work_struct *work)
+{
+	struct amdgpu_usermode_queue *queue = container_of(work,
+							  struct amdgpu_usermode_queue,
+							  hang_detect_work.work);
+	struct dma_fence *fence;
+	struct amdgpu_userq_mgr *uq_mgr;
+
+	if (!queue || !queue->userq_mgr)
+		return;
+
+	uq_mgr = queue->userq_mgr;
+	fence = READ_ONCE(queue->hang_detect_fence);
+	/* Fence already signaled – no action needed */
+	if (!fence || dma_fence_is_signaled(fence))
+		return;
+
+	mutex_lock(&uq_mgr->userq_mutex);
+	amdgpu_userq_detect_and_reset_queues(uq_mgr);
+	mutex_unlock(&uq_mgr->userq_mutex);
+}
+
+/*
+ * Start hang detection for a user queue fence. A delayed work will be scheduled
+ * to check if the fence is still pending after the timeout period.
+*/
+void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue)
+{
+	struct amdgpu_device *adev;
+	unsigned long timeout_ms;
+
+	if (!queue || !queue->userq_mgr || !queue->userq_mgr->adev)
+		return;
+
+	adev = queue->userq_mgr->adev;
+	/* Determine timeout based on queue type */
+	switch (queue->queue_type) {
+	case AMDGPU_RING_TYPE_GFX:
+		timeout_ms = adev->gfx_timeout;
+		break;
+	case AMDGPU_RING_TYPE_COMPUTE:
+		timeout_ms = adev->compute_timeout;
+		break;
+	case AMDGPU_RING_TYPE_SDMA:
+		timeout_ms = adev->sdma_timeout;
+		break;
+	default:
+		timeout_ms = adev->gfx_timeout;
+		break;
+	}
+
+	/* Store the fence to monitor and schedule hang detection */
+	WRITE_ONCE(queue->hang_detect_fence, queue->last_fence);
+	schedule_delayed_work(&queue->hang_detect_work,
+		     msecs_to_jiffies(timeout_ms));
+}
+
+static void amdgpu_userq_init_hang_detect_work(struct amdgpu_usermode_queue *queue)
+{
+	INIT_DELAYED_WORK(&queue->hang_detect_work, amdgpu_userq_hang_detect_work);
+	queue->hang_detect_fence = NULL;
+}
+
 static int amdgpu_userq_buffer_va_list_add(struct amdgpu_usermode_queue *queue,
 					   struct amdgpu_bo_va_mapping *va_map, u64 addr)
 {
···
 
 	cancel_delayed_work_sync(&uq_mgr->resume_work);
 	mutex_lock(&uq_mgr->userq_mutex);
-
 	queue = amdgpu_userq_find(uq_mgr, queue_id);
 	if (!queue) {
 		drm_dbg_driver(adev_to_drm(uq_mgr->adev), "Invalid queue id to destroy\n");
···
 		return -EINVAL;
 	}
 	amdgpu_userq_wait_for_last_fence(queue);
+	/* Cancel any pending hang detection work and cleanup */
+	if (queue->hang_detect_fence) {
+		cancel_delayed_work_sync(&queue->hang_detect_work);
+		queue->hang_detect_fence = NULL;
+	}
 	r = amdgpu_bo_reserve(queue->db_obj.obj, true);
 	if (!r) {
 		amdgpu_bo_unpin(queue->db_obj.obj);
···
 	queue->debugfs_queue = debugfs_create_dir(queue_name, filp->debugfs_client);
 	debugfs_create_file("mqd_info", 0444, queue->debugfs_queue, queue, &amdgpu_mqd_info_fops);
 #endif
+	amdgpu_userq_init_hang_detect_work(queue);
 	kfree(queue_name);
 
 	args->out.queue_id = qid;

drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h

···
 	u32			xcp_id;
 	int			priority;
 	struct dentry		*debugfs_queue;
+	struct delayed_work hang_detect_work;
+	struct dma_fence *hang_detect_fence;
 
 	struct list_head	userq_va_list;
 };
···
 void amdgpu_userq_reset_work(struct work_struct *work);
 void amdgpu_userq_pre_reset(struct amdgpu_device *adev);
 int amdgpu_userq_post_reset(struct amdgpu_device *adev, bool vram_lost);
+void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue);
 
 int amdgpu_userq_input_va_validate(struct amdgpu_device *adev,
 				   struct amdgpu_usermode_queue *queue,

drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c

···
 
 	dma_fence_put(queue->last_fence);
 	queue->last_fence = dma_fence_get(fence);
+	amdgpu_userq_start_hang_detect_work(queue);
 	mutex_unlock(&userq_mgr->userq_mutex);
 
 	drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT,

drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c

···
 				   u32 sh_num, u32 instance, int xcc_id);
 static u32 gfx_v12_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev);
 
-static void gfx_v12_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, bool secure);
 static void gfx_v12_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
 				     uint32_t val);
 static int gfx_v12_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev);
···
 	return r;
 }
 
-static void gfx_v12_0_ring_emit_frame_cntl(struct amdgpu_ring *ring,
-					   bool start,
-					   bool secure)
-{
-	uint32_t v = secure ? FRAME_TMZ : 0;
-
-	amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
-	amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1));
-}
-
 static void gfx_v12_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
 				     uint32_t reg_val_offs)
 {
···
 	.emit_cntxcntl = gfx_v12_0_ring_emit_cntxcntl,
 	.init_cond_exec = gfx_v12_0_ring_emit_init_cond_exec,
 	.preempt_ib = gfx_v12_0_ring_preempt_ib,
-	.emit_frame_cntl = gfx_v12_0_ring_emit_frame_cntl,
 	.emit_wreg = gfx_v12_0_ring_emit_wreg,
 	.emit_reg_wait = gfx_v12_0_ring_emit_reg_wait,
 	.emit_reg_write_reg_wait = gfx_v12_0_ring_emit_reg_write_reg_wait,

drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c

···
 				  unsigned int vmid,
 				  struct amdgpu_fence *timedout_fence)
 {
-	if (amdgpu_sriov_vf(ring->adev))
-		return -EOPNOTSUPP;
+	struct amdgpu_device *adev = ring->adev;
+	struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[ring->me];
+	int r;
 
+	/* take the vcn reset mutex here because resetting VCN will reset jpeg as well */
+	mutex_lock(&vinst->engine_reset_mutex);
 	amdgpu_ring_reset_helper_begin(ring, timedout_fence);
 	jpeg_v4_0_3_core_stall_reset(ring);
 	jpeg_v4_0_3_start_jrbc(ring);
-	return amdgpu_ring_reset_helper_end(ring, timedout_fence);
+	r = amdgpu_ring_reset_helper_end(ring, timedout_fence);
+	mutex_unlock(&vinst->engine_reset_mutex);
+	return r;
 }
 
 static const struct amd_ip_funcs jpeg_v4_0_3_ip_funcs = {

drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.c

···
 				  unsigned int vmid,
 				  struct amdgpu_fence *timedout_fence)
 {
+	struct amdgpu_device *adev = ring->adev;
+	struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[ring->me];
+	int r;
+
+	/* take the vcn reset mutex here because resetting VCN will reset jpeg as well */
+	mutex_lock(&vinst->engine_reset_mutex);
+
 	amdgpu_ring_reset_helper_begin(ring, timedout_fence);
 	jpeg_v5_0_1_core_stall_reset(ring);
 	jpeg_v5_0_1_init_jrbc(ring);
-	return amdgpu_ring_reset_helper_end(ring, timedout_fence);
+	r = amdgpu_ring_reset_helper_end(ring, timedout_fence);
+	mutex_unlock(&vinst->engine_reset_mutex);
+	return r;
 }
 
 static const struct amd_ip_funcs jpeg_v5_0_1_ip_funcs = {

drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c

···
 	int inst_idx = vinst->inst;
 	struct amdgpu_vcn4_fw_shared *fw_shared =
 						adev->vcn.inst[inst_idx].fw_shared.cpu_addr;
+	struct dpg_pause_state state = {.fw_based = VCN_DPG_STATE__PAUSE};
 	struct amdgpu_ring *ring;
 	int vcn_inst, ret;
 	uint32_t tmp;
···
 	}
 
 	ring = &adev->vcn.inst[inst_idx].ring_enc[0];
+
+	/* Pause dpg */
+	vcn_v4_0_3_pause_dpg_mode(vinst, &state);
 
 	/* program the RB_BASE for ring buffer */
 	WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_LO,
···
 	int inst_idx = vinst->inst;
 	uint32_t tmp;
 	int vcn_inst;
+	struct dpg_pause_state state = {.fw_based = VCN_DPG_STATE__UNPAUSE};
 
 	vcn_inst = GET_INST(VCN, inst_idx);
+
+	/* Unpause dpg */
+	vcn_v4_0_3_pause_dpg_mode(vinst, &state);
 
 	/* Wait for power status to be 1 */
 	SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_POWER_STATUS, 1,
···
 static int vcn_v4_0_3_pause_dpg_mode(struct amdgpu_vcn_inst *vinst,
 				     struct dpg_pause_state *new_state)
 {
+	struct amdgpu_device *adev = vinst->adev;
+	int inst_idx = vinst->inst;
+	uint32_t reg_data = 0;
+	int ret_code;
+
+	/* pause/unpause if state is changed */
+	if (adev->vcn.inst[inst_idx].pause_state.fw_based != new_state->fw_based) {
+		DRM_DEV_DEBUG(adev->dev, "dpg pause state changed %d -> %d",
+			adev->vcn.inst[inst_idx].pause_state.fw_based,	new_state->fw_based);
+		reg_data = RREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE) &
+			(~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
+
+		if (new_state->fw_based == VCN_DPG_STATE__PAUSE) {
+			ret_code = SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS, 0x1,
+				UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
+
+			if (!ret_code) {
+				/* pause DPG */
+				reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
+				WREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE, reg_data);
+
+				/* wait for ACK */
+				SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_DPG_PAUSE,
+					UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK,
+					UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
+			}
+		} else {
+			/* unpause dpg, no need to wait */
+			reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
+			WREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE, reg_data);
+		}
+		adev->vcn.inst[inst_idx].pause_state.fw_based = new_state->fw_based;
+	}
 
 	return 0;
 }
···
 	}
 }
 
+static int vcn_v4_0_3_reset_jpeg_pre_helper(struct amdgpu_device *adev, int inst)
+{
+	struct amdgpu_ring *ring;
+	uint32_t wait_seq = 0;
+	int i;
+
+	for (i = 0; i < adev->jpeg.num_jpeg_rings; ++i) {
+		ring = &adev->jpeg.inst[inst].ring_dec[i];
+
+		drm_sched_wqueue_stop(&ring->sched);
+		/* Get the last emitted fence sequence */
+		wait_seq = atomic_read(&ring->fence_drv.last_seq);
+		if (wait_seq)
+			continue;
+
+		/* if Jobs are still pending after timeout,
+		 * We'll handle them in the bottom helper
+		 */
+		amdgpu_fence_wait_polling(ring, wait_seq, adev->video_timeout);
+       }
+
+	return 0;
+}
+
+static int vcn_v4_0_3_reset_jpeg_post_helper(struct amdgpu_device *adev, int inst)
+{
+	struct amdgpu_ring *ring;
+	int i, r = 0;
+
+	for (i = 0; i < adev->jpeg.num_jpeg_rings; ++i) {
+		ring = &adev->jpeg.inst[inst].ring_dec[i];
+		/* Force completion of any remaining jobs */
+		amdgpu_fence_driver_force_completion(ring);
+
+		if (ring->use_doorbell)
+			WREG32_SOC15_OFFSET(
+				VCN, GET_INST(VCN, inst),
+				regVCN_JPEG_DB_CTRL,
+				(ring->pipe ? (ring->pipe - 0x15) : 0),
+				ring->doorbell_index << VCN_JPEG_DB_CTRL__OFFSET__SHIFT |
+				VCN_JPEG_DB_CTRL__EN_MASK);
+
+		r = amdgpu_ring_test_helper(ring);
+		if (r)
+			return r;
+
+		drm_sched_wqueue_start(&ring->sched);
+
+		DRM_DEV_DEBUG(adev->dev, "JPEG ring %d (inst %d) restored and sched restarted\n",
+		      i, inst);
+	}
+	return 0;
+}
+
 static int vcn_v4_0_3_ring_reset(struct amdgpu_ring *ring,
 				 unsigned int vmid,
 				 struct amdgpu_fence *timedout_fence)
···
 	int vcn_inst;
 	struct amdgpu_device *adev = ring->adev;
 	struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[ring->me];
+	bool pg_state = false;
 
+	/* take the vcn reset mutex here because resetting VCN will reset jpeg as well */
+	mutex_lock(&vinst->engine_reset_mutex);
+	mutex_lock(&adev->jpeg.jpeg_pg_lock);
+	/* Ensure JPEG is powered on during reset if currently gated */
+	if (adev->jpeg.cur_state == AMD_PG_STATE_GATE) {
+		amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_JPEG,
+						       AMD_PG_STATE_UNGATE);
+		pg_state = true;
+	}
+
+	vcn_v4_0_3_reset_jpeg_pre_helper(adev, ring->me);
 	amdgpu_ring_reset_helper_begin(ring, timedout_fence);
 
 	vcn_inst = GET_INST(VCN, ring->me);
···
 
 	if (r) {
 		DRM_DEV_ERROR(adev->dev, "VCN reset fail : %d\n", r);
-		return r;
+		/* Restore JPEG power gating state if it was originally gated */
+		if (pg_state)
+			amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_JPEG,
+							       AMD_PG_STATE_GATE);
+		mutex_unlock(&adev->jpeg.jpeg_pg_lock);
+		goto unlock;
 	}
 
 	/* This flag is not set for VF, assumed to be disabled always */
···
 	vcn_v4_0_3_hw_init_inst(vinst);
 	vcn_v4_0_3_start_dpg_mode(vinst, adev->vcn.inst[ring->me].indirect_sram);
 
-	return amdgpu_ring_reset_helper_end(ring, timedout_fence);
+	r = amdgpu_ring_reset_helper_end(ring, timedout_fence);
+	if (r) {
+		if (pg_state)
+			amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_JPEG,
+							       AMD_PG_STATE_GATE);
+		mutex_unlock(&adev->jpeg.jpeg_pg_lock);
+		goto unlock;
+	}
+
+	r = vcn_v4_0_3_reset_jpeg_post_helper(adev, ring->me);
+	if (pg_state)
+		amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_JPEG,
+						       AMD_PG_STATE_GATE);
+	mutex_unlock(&adev->jpeg.jpeg_pg_lock);
+
+unlock:
+	mutex_unlock(&vinst->engine_reset_mutex);
+
+	return r;
 }
 
 static const struct amdgpu_ring_funcs vcn_v4_0_3_unified_ring_vm_funcs = {

drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c

···
 	}
 }
 
+static int vcn_v5_0_1_reset_jpeg_pre_helper(struct amdgpu_device *adev, int inst)
+{
+	struct amdgpu_ring *ring;
+	uint32_t wait_seq = 0;
+	int i;
+
+	for (i = 0; i < adev->jpeg.num_jpeg_rings; ++i) {
+		ring = &adev->jpeg.inst[inst].ring_dec[i];
+
+		drm_sched_wqueue_stop(&ring->sched);
+		wait_seq = atomic_read(&ring->fence_drv.last_seq);
+		if (wait_seq)
+			continue;
+
+		/* if Jobs are still pending after timeout,
+		 * We'll handle them in the bottom helper
+		 */
+		amdgpu_fence_wait_polling(ring, wait_seq, adev->video_timeout);
+       }
+
+	return 0;
+}
+
+static int vcn_v5_0_1_reset_jpeg_post_helper(struct amdgpu_device *adev, int inst)
+{
+	struct amdgpu_ring *ring;
+	int i, r = 0;
+
+	for (i = 0; i < adev->jpeg.num_jpeg_rings; ++i) {
+		ring = &adev->jpeg.inst[inst].ring_dec[i];
+		/* Force completion of any remaining jobs */
+		amdgpu_fence_driver_force_completion(ring);
+
+		if (ring->use_doorbell)
+			WREG32_SOC15_OFFSET(
+				VCN, GET_INST(VCN, inst),
+				regVCN_JPEG_DB_CTRL,
+				(ring->pipe ? (ring->pipe - 0x15) : 0),
+				ring->doorbell_index << VCN_JPEG_DB_CTRL__OFFSET__SHIFT |
+				VCN_JPEG_DB_CTRL__EN_MASK);
+
+		r = amdgpu_ring_test_helper(ring);
+		if (r)
+			return r;
+
+		drm_sched_wqueue_start(&ring->sched);
+
+		DRM_DEV_DEBUG(adev->dev, "JPEG ring %d (inst %d) restored and sched restarted\n",
+		      i, inst);
+	}
+	return 0;
+}
+
 static int vcn_v5_0_1_ring_reset(struct amdgpu_ring *ring,
 				 unsigned int vmid,
 				 struct amdgpu_fence *timedout_fence)
···
 	int vcn_inst;
 	struct amdgpu_device *adev = ring->adev;
 	struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[ring->me];
+	bool pg_state = false;
+
+	/* take the vcn reset mutex here because resetting VCN will reset jpeg as well */
+	mutex_lock(&vinst->engine_reset_mutex);
+	vcn_v5_0_1_reset_jpeg_pre_helper(adev, ring->me);
+	mutex_lock(&adev->jpeg.jpeg_pg_lock);
+	/* Ensure JPEG is powered on during reset if currently gated */
+	if (adev->jpeg.cur_state == AMD_PG_STATE_GATE) {
+		amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_JPEG,
+					       AMD_PG_STATE_UNGATE);
+		pg_state = true;
+	}
 
 	amdgpu_ring_reset_helper_begin(ring, timedout_fence);
 
···
 
 	if (r) {
 		DRM_DEV_ERROR(adev->dev, "VCN reset fail : %d\n", r);
-		return r;
+		/* Restore JPEG power gating state if it was originally gated */
+		if (pg_state)
+			amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_JPEG,
+							       AMD_PG_STATE_GATE);
+		mutex_unlock(&adev->jpeg.jpeg_pg_lock);
+		goto unlock;
 	}
 
 	vcn_v5_0_1_hw_init_inst(adev, ring->me);
 	vcn_v5_0_1_start_dpg_mode(vinst, vinst->indirect_sram);
 
-	return amdgpu_ring_reset_helper_end(ring, timedout_fence);
+	r = amdgpu_ring_reset_helper_end(ring, timedout_fence);
+	if (r) {
+		if (pg_state)
+			amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_JPEG,
+							       AMD_PG_STATE_GATE);
+		mutex_unlock(&adev->jpeg.jpeg_pg_lock);
+		goto unlock;
+	}
+
+	if (pg_state)
+		amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_JPEG,
+						       AMD_PG_STATE_GATE);
+	mutex_unlock(&adev->jpeg.jpeg_pg_lock);
+
+	r = vcn_v5_0_1_reset_jpeg_post_helper(adev, ring->me);
+
+unlock:
+	mutex_unlock(&vinst->engine_reset_mutex);
+
+	return r;
 }
 
 static const struct amdgpu_ring_funcs vcn_v5_0_1_unified_ring_vm_funcs = {

drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h

···
 };
 
 static const uint32_t cwsr_trap_gfx12_hex[] = {
-	0xbfa00001, 0xbfa002b2,
+	0xbfa00001, 0xbfa00239,
 	0xb0804009, 0xb8eef81a,
 	0xbf880000, 0xb980081a,
 	0x00000000, 0xb8f8f804,
···
 	0x807a817a, 0xbf0d997b,
 	0xbfa20002, 0x847a897a,
 	0xbfa00001, 0x847a8a7a,
-	0xb8fb1e06, 0x847b8a7b,
-	0x807a7b7a, 0x8b7bff7f,
-	0x0000ffff, 0x807aff7a,
-	0x00000200, 0x807a7e7a,
-	0x827b807b, 0xd7610000,
-	0x00010870, 0xd7610000,
-	0x00010a71, 0xd7610000,
-	0x00010c72, 0xd7610000,
-	0x00010e73, 0xd7610000,
-	0x00011074, 0xd7610000,
-	0x00011275, 0xd7610000,
-	0x00011476, 0xd7610000,
-	0x00011677, 0xd7610000,
-	0x00011a79, 0xd7610000,
-	0x00011c7e, 0xd7610000,
-	0x00011e7f, 0xd8500000,
-	0x00000000, 0xd8500000,
-	0x00000000, 0xd8500000,
-	0x00000000, 0xd8500000,
-	0x00000000, 0xd8500000,
-	0x00000000, 0xd8500000,
-	0x00000000, 0xd8500000,
-	0x00000000, 0xd8500000,
-	0x00000000, 0xbefe00ff,
-	0x00003fff, 0xbeff0080,
-	0xee0a407a, 0x000c0000,
-	0x00004000, 0xd760007a,
-	0x00011d00, 0xd760007b,
-	0x00011f00, 0xbefe007a,
-	0xbeff007b, 0xbef4007e,
-	0x8b75ff7f, 0x0000ffff,
-	0x8c75ff75, 0x00040000,
-	0xbef60080, 0xbef700ff,
-	0x10807fac, 0xbef1007d,
-	0xbef00080, 0xb8f30742,
-	0x84739973, 0xbefe00c1,
-	0x857d9973, 0x8b7d817d,
-	0xbf06817d, 0xbfa20002,
-	0xbeff0080, 0xbfa00002,
-	0xbeff00c1, 0xbfa0000c,
-	0xbef600ff, 0x01000000,
-	0xc4068070, 0x008ce801,
-	0x00008000, 0xc4068070,
-	0x008ce802, 0x00010000,
-	0xc4068070, 0x008ce803,
-	0x00018000, 0xbfa0000b,
-	0xbef600ff, 0x01000000,
-	0xc4068070, 0x008ce801,
-	0x00010000, 0xc4068070,
-	0x008ce802, 0x00020000,
-	0xc4068070, 0x008ce803,
-	0x00030000, 0xb8f03b05,
-	0x80708170, 0xbf0d9973,
-	0xbfa20002, 0x84708970,
-	0xbfa00001, 0x84708a70,
-	0xb8fa1e06, 0x847a8a7a,
-	0x80707a70, 0x8070ff70,
-	0x00000200, 0xbef600ff,
-	0x01000000, 0x7e000280,
-	0x7e020280, 0x7e040280,
-	0xbe804ec2, 0xbf94fffe,
-	0xb8faf804, 0x8b7a847a,
-	0x91788478, 0x8c787a78,
-	0x917aff6d, 0x80000000,
-	0xd7610002, 0x00010071,
-	0xd7610002, 0x0001026c,
-	0xd7610002, 0x0001047a,
-	0xd7610002, 0x0001066e,
-	0xd7610002, 0x0001086f,
-	0xd7610002, 0x00010a78,
-	0xd7610002, 0x00010e7b,
-	0xd8500000, 0x00000000,
-	0xd8500000, 0x00000000,
-	0xd8500000, 0x00000000,
-	0xd8500000, 0x00000000,
-	0xd8500000, 0x00000000,
-	0xd8500000, 0x00000000,
-	0xd8500000, 0x00000000,
-	0xd8500000, 0x00000000,
-	0xb8faf811, 0xd7610002,
-	0x00010c7a, 0xb8faf801,
-	0xd7610002, 0x0001107a,
-	0xb8faf814, 0xd7610002,
-	0x0001127a, 0xb8faf815,
-	0xd7610002, 0x0001147a,
-	0xb8faf812, 0xd7610002,
-	0x0001167a, 0xb8faf813,
-	0xd7610002, 0x0001187a,
-	0xb8faf802, 0xd7610002,
-	0x00011a7a, 0xbefa50c1,
-	0xbfc70000, 0xd7610002,
-	0x00011c7a, 0xd8500000,
-	0x00000000, 0xd8500000,
-	0x00000000, 0xd8500000,
-	0x00000000, 0xd8500000,
-	0x00000000, 0xd8500000,
-	0x00000000, 0xd8500000,
-	0x00000000, 0xd8500000,
-	0x00000000, 0xd8500000,
-	0x00000000, 0xbefe00ff,
-	0x0000ffff, 0xbeff0080,
-	0xc4068070, 0x008ce802,
-	0x00000000, 0xbefe00c1,
+	0x8b7bff7f, 0x0000ffff,
+	0x807aff7a, 0x00000240,
+	0x807a7e7a, 0x827b807b,
+	0xd7610000, 0x00010870,
+	0xd7610000, 0x00010a71,
+	0xd7610000, 0x00010c72,
+	0xd7610000, 0x00010e73,
+	0xd7610000, 0x00011074,
+	0xd7610000, 0x00011275,
+	0xd7610000, 0x00011476,
+	0xd7610000, 0x00011677,
+	0xd7610000, 0x00011a79,
+	0xd7610000, 0x00011c7e,
+	0xd7610000, 0x00011e7f,
+	0xbefe00ff, 0x00003fff,
+	0xbeff0080, 0xee0a407a,
+	0x000c0000, 0x00000000,
+	0xd760007a, 0x00011d00,
+	0xd760007b, 0x00011f00,
+	0xbefe007a, 0xbeff007b,
+	0xbef4007e, 0x8b75ff7f,
+	0x0000ffff, 0xbef1007d,
+	0xb8f30742, 0x84739973,
+	0xbefe00c1, 0x857d9973,
+	0x8b7d817d, 0xbf06817d,
+	0xbfa20002, 0xbeff0080,
+	0xbfa00002, 0xbeff00c1,
+	0xbfa0000a, 0xee0a4074,
+	0x008c0000, 0x00008000,
+	0xee0a4074, 0x010c0000,
+	0x00010000, 0xee0a4074,
+	0x018c0000, 0x00018000,
+	0xbfa00009, 0xee0a4074,
+	0x008c0000, 0x00010000,
+	0xee0a4074, 0x010c0000,
+	0x00020000, 0xee0a4074,
+	0x018c0000, 0x00030000,
 	0xb8f03b05, 0x80708170,
 	0xbf0d9973, 0xbfa20002,
 	0x84708970, 0xbfa00001,
-	0x84708a70, 0xb8fa1e06,
-	0x847a8a7a, 0x80707a70,
-	0xbef600ff, 0x01000000,
+	0x84708a70, 0x8070ff70,
+	0x00000200, 0x7e000280,
+	0x7e020280, 0x7e040280,
+	0xbefd0080, 0xd7610002,
+	0x0000fa71, 0x807d817d,
+	0xbe804ec2, 0xbf94fffe,
+	0xb8faf804, 0x8b7a847a,
+	0x91788478, 0x8c787a78,
+	0xd7610002, 0x0000fa6c,
+	0x807d817d, 0x8b7aff6d,
+	0x0000ffff, 0xd7610002,
+	0x0000fa7a, 0x807d817d,
+	0xd7610002, 0x0000fa6e,
+	0x807d817d, 0xd7610002,
+	0x0000fa6f, 0x807d817d,
+	0xd7610002, 0x0000fa78,
+	0x807d817d, 0xb8faf811,
+	0xd7610002, 0x0000fa7a,
+	0x807d817d, 0xbefa0080,
+	0xd7610002, 0x0000fa7a,
+	0x807d817d, 0xb8f1f801,
+	0xd7610002, 0x0000fa71,
+	0x807d817d, 0xb8f1f814,
+	0xd7610002, 0x0000fa71,
+	0x807d817d, 0xb8f1f815,
+	0xd7610002, 0x0000fa71,
+	0x807d817d, 0xb8f1f812,
+	0xd7610002, 0x0000fa71,
+	0x807d817d, 0xb8f1f813,
+	0xd7610002, 0x0000fa71,
+	0x807d817d, 0xb8faf802,
+	0xd7610002, 0x0000fa7a,
+	0x807d817d, 0xbefa50c1,
+	0xbfc70000, 0xd7610002,
+	0x0000fa7a, 0x807d817d,
+	0xbefe00ff, 0x0000ffff,
+	0xbeff0080, 0x80767074,
+	0x82778075, 0xee0a4076,
+	0x010c0000, 0x00000000,
+	0xbefe00c1, 0xb8f03b05,
+	0x80708170, 0xbf0d9973,
+	0xbfa20002, 0x84708970,
+	0xbfa00001, 0x84708a70,
 	0xbef90080, 0xbefd0080,
 	0xbf800000, 0xbe804100,
 	0xbe824102, 0xbe844104,
 	0xbe864106, 0xbe884108,
 	0xbe8a410a, 0xbe8c410c,
-	0xbe8e410e, 0xbf068079,
-	0xbfa10032, 0xd7610002,
-	0x00010000, 0xd7610002,
-	0x00010201, 0xd7610002,
-	0x00010402, 0xd7610002,
-	0x00010603, 0xd7610002,
-	0x00010804, 0xd7610002,
-	0x00010a05, 0xd7610002,
-	0x00010c06, 0xd7610002,
-	0x00010e07, 0xd7610002,
-	0x00011008, 0xd7610002,
-	0x00011209, 0xd7610002,
-	0x0001140a, 0xd7610002,
-	0x0001160b, 0xd7610002,
-	0x0001180c, 0xd7610002,
-	0x00011a0d, 0xd7610002,
-	0x00011c0e, 0xd7610002,
-	0x00011e0f, 0xd8500000,
-	0x00000000, 0xd8500000,
-	0x00000000, 0xd8500000,
-	0x00000000, 0xd8500000,
-	0x00000000, 0xd8500000,
-	0x00000000, 0xd8500000,
-	0x00000000, 0xd8500000,
-	0x00000000, 0xd8500000,
-	0x00000000, 0x80799079,
-	0xbfa00038, 0xd7610002,
-	0x00012000, 0xd7610002,
-	0x00012201, 0xd7610002,
-	0x00012402, 0xd7610002,
-	0x00012603, 0xd7610002,
-	0x00012804, 0xd7610002,
-	0x00012a05, 0xd7610002,
-	0x00012c06, 0xd7610002,
-	0x00012e07, 0xd7610002,
-	0x00013008, 0xd7610002,
-	0x00013209, 0xd7610002,
-	0x0001340a, 0xd7610002,
-	0x0001360b, 0xd7610002,
-	0x0001380c, 0xd7610002,
-	0x00013a0d, 0xd7610002,
-	0x00013c0e, 0xd7610002,
-	0x00013e0f, 0xd8500000,
-	0x00000000, 0xd8500000,
-	0x00000000, 0xd8500000,
-	0x00000000, 0xd8500000,
-	0x00000000, 0xd8500000,
-	0x00000000, 0xd8500000,
-	0x00000000, 0xd8500000,
-	0x00000000, 0xd8500000,
-	0x00000000, 0x80799079,
-	0xc4068070, 0x008ce802,
-	0x00000000, 0x8070ff70,
-	0x00000080, 0xbef90080,
-	0x7e040280, 0x807d907d,
-	0xbf0aff7d, 0x00000060,
-	0xbfa2ff88, 0xbe804100,
-	0xbe824102, 0xbe844104,
-	0xbe864106, 0xbe884108,
-	0xbe8a410a, 0xd7610002,
-	0x00010000, 0xd7610002,
-	0x00010201, 0xd7610002,
-	0x00010402, 0xd7610002,
-	0x00010603, 0xd7610002,
-	0x00010804, 0xd7610002,
-	0x00010a05, 0xd7610002,
-	0x00010c06, 0xd7610002,
-	0x00010e07, 0xd7610002,
-	0x00011008, 0xd7610002,
-	0x00011209, 0xd7610002,
-	0x0001140a, 0xd7610002,
-	0x0001160b, 0xd8500000,
-	0x00000000, 0xd8500000,
-	0x00000000, 0xd8500000,
-	0x00000000, 0xd8500000,
-	0x00000000, 0xd8500000,
-	0x00000000, 0xd8500000,
-	0x00000000, 0xd8500000,
-	0x00000000, 0xd8500000,
-	0x00000000, 0xc4068070,
-	0x008ce802, 0x00000000,
-	0xbefe00c1, 0x857d9973,
-	0x8b7d817d, 0xbf06817d,
-	0xbfa20002, 0xbeff0080,
-	0xbfa00001, 0xbeff00c1,
-	0xb8fb4306, 0x8b7bc17b,
-	0xbfa10044, 0x8b7aff6d,
-	0x80000000, 0xbfa10041,
-	0x847b897b, 0xbef6007b,
+	0xbe8e410e, 0xd7610002,
+	0x0000f200, 0x80798179,
+	0xd7610002, 0x0000f201,
+	0x80798179, 0xd7610002,
+	0x0000f202, 0x80798179,
+	0xd7610002, 0x0000f203,
+	0x80798179, 0xd7610002,
+	0x0000f204, 0x80798179,
+	0xd7610002, 0x0000f205,
+	0x80798179, 0xd7610002,
+	0x0000f206, 0x80798179,
+	0xd7610002, 0x0000f207,
+	0x80798179, 0xd7610002,
+	0x0000f208, 0x80798179,
+	0xd7610002, 0x0000f209,
+	0x80798179, 0xd7610002,
+	0x0000f20a, 0x80798179,
+	0xd7610002, 0x0000f20b,
+	0x80798179, 0xd7610002,
+	0x0000f20c, 0x80798179,
+	0xd7610002, 0x0000f20d,
+	0x80798179, 0xd7610002,
+	0x0000f20e, 0x80798179,
+	0xd7610002, 0x0000f20f,
+	0x80798179, 0xbf06a079,
+	0xbfa10009, 0x80767074,
+	0x82778075, 0xee0a4076,
+	0x010c0000, 0x00000000,
+	0x8070ff70, 0x00000080,
+	0xbef90080, 0x7e040280,
+	0x807d907d, 0xbf0aff7d,
+	0x00000060, 0xbfa2ffb9,
+	0xbe804100, 0xbe824102,
+	0xbe844104, 0xbe864106,
+	0xbe884108, 0xbe8a410a,
+	0xd7610002, 0x0000f200,
+	0x80798179, 0xd7610002,
+	0x0000f201, 0x80798179,
+	0xd7610002, 0x0000f202,
+	0x80798179, 0xd7610002,
+	0x0000f203, 0x80798179,
+	0xd7610002, 0x0000f204,
+	0x80798179, 0xd7610002,
+	0x0000f205, 0x80798179,
+	0xd7610002, 0x0000f206,
+	0x80798179, 0xd7610002,
+	0x0000f207, 0x80798179,
+	0xd7610002, 0x0000f208,
+	0x80798179, 0xd7610002,
+	0x0000f209, 0x80798179,
+	0xd7610002, 0x0000f20a,
+	0x80798179, 0xd7610002,
+	0x0000f20b, 0x80798179,
+	0x80767074, 0x82778075,
+	0xee0a4076, 0x010c0000,
+	0x00000000, 0xbefe00c1,
+	0x857d9973, 0x8b7d817d,
+	0xbf06817d, 0xbfa20002,
+	0xbeff0080, 0xbfa00001,
+	0xbeff00c1, 0xb8fb4306,
+	0x8b7bc17b, 0xbfa10042,
+	0x8b7aff6d, 0x80000000,
+	0xbfa1003f, 0x847b897b,
 	0xb8f03b05, 0x80708170,
 	0xbf0d9973, 0xbfa20002,
 	0x84708970, 0xbfa00001,
-	0x84708a70, 0xb8fa1e06,
-	0x847a8a7a, 0x80707a70,
-	0x8070ff70, 0x00000200,
-	0x8070ff70, 0x00000080,
-	0xbef600ff, 0x01000000,
-	0xd71f0000, 0x000100c1,
-	0xd7200000, 0x000200c1,
-	0x16000084, 0x857d9973,
-	0x8b7d817d, 0xbf06817d,
-	0xbefd0080, 0xbfa20013,
-	0xbe8300ff, 0x00000080,
+	0x84708a70, 0x8070ff70,
+	0x00000200, 0x8070ff70,
+	0x00000080, 0xd71f0000,
+	0x000100c1, 0xd7200000,
+	0x000200c1, 0x16000084,
+	0x857d9973, 0x8b7d817d,
+	0xbf06817d, 0xbefd0080,
+	0xbfa20015, 0xbe8300ff,
+	0x00000080, 0xbf800000,
+	0xbf800000, 0xbf800000,
+	0xd8d80000, 0x01000000,
+	0xbf8a0000, 0x80767074,
+	0x82778075, 0xee0a4076,
+	0x008c0000, 0x00000000,
+	0x807d037d, 0x80700370,
+	0xd5250000, 0x0001ff00,
+	0x00000080, 0xbf0a7b7d,
+	0xbfa2fff1, 0xbfa00014,
+	0xbe8300ff, 0x00000100,
 	0xbf800000, 0xbf800000,
 	0xbf800000, 0xd8d80000,
 	0x01000000, 0xbf8a0000,
-	0xc4068070, 0x008ce801,
+	0x80767074, 0x82778075,
+	0xee0a4076, 0x008c0000,
 	0x00000000, 0x807d037d,
 	0x80700370, 0xd5250000,
-	0x0001ff00, 0x00000080,
-	0xbf0a7b7d, 0xbfa2fff3,
-	0xbfa00012, 0xbe8300ff,
-	0x00000100, 0xbf800000,
-	0xbf800000, 0xbf800000,
-	0xd8d80000, 0x01000000,
-	0xbf8a0000, 0xc4068070,
-	0x008ce801, 0x00000000,
-	0x807d037d, 0x80700370,
-	0xd5250000, 0x0001ff00,
-	0x00000100, 0xbf0a7b7d,
-	0xbfa2fff3, 0xbefe00c1,
-	0x857d9973, 0x8b7d817d,
-	0xbf06817d, 0xbfa20004,
-	0xbef000ff, 0x00000200,
-	0xbeff0080, 0xbfa00003,
-	0xbef000ff, 0x00000400,
-	0xbeff00c1, 0xb8fb3b05,
-	0x807b817b, 0x847b827b,
-	0x857d9973, 0x8b7d817d,
-	0xbf06817d, 0xbfa2001b,
-	0xbef600ff, 0x01000000,
-	0xbefd0084, 0xbf0a7b7d,
-	0xbfa10040, 0x7e008700,
-	0x7e028701, 0x7e048702,
-	0x7e068703, 0xc4068070,
-	0x008ce800, 0x00000000,
-	0xc4068070, 0x008ce801,
-	0x00008000, 0xc4068070,
-	0x008ce802, 0x00010000,
-	0xc4068070, 0x008ce803,
-	0x00018000, 0x807d847d,
-	0x8070ff70, 0x00000200,
-	0xbf0a7b7d, 0xbfa2ffeb,
-	0xbfa0002a, 0xbef600ff,
-	0x01000000, 0xbefd0084,
-	0xbf0a7b7d, 0xbfa10015,
+	0x0001ff00, 0x00000100,
+	0xbf0a7b7d, 0xbfa2fff1,
+	0xbefe00c1, 0x857d9973,
+	0x8b7d817d, 0xbf06817d,
+	0xbfa20004, 0xbef000ff,
+	0x00000200, 0xbeff0080,
+	0xbfa00003, 0xbef000ff,
+	0x00000400, 0xbeff00c1,
+	0xb8fb3b05, 0x807b817b,
+	0x847b827b, 0x857d9973,
+	0x8b7d817d, 0xbf06817d,
+	0xbfa2001b, 0xbefd0084,
+	0xbf0a7b7d, 0xbfa10032,
 	0x7e008700, 0x7e028701,
 	0x7e048702, 0x7e068703,
-	0xc4068070, 0x008ce800,
-	0x00000000, 0xc4068070,
-	0x008ce801, 0x00010000,
-	0xc4068070, 0x008ce802,
-	0x00020000, 0xc4068070,
-	0x008ce803, 0x00030000,
+	0x80767074, 0x82778075,
+	0xee0a4076, 0x000c0000,
+	0x00000000, 0xee0a4076,
+	0x008c0000, 0x00008000,
+	0xee0a4076, 0x010c0000,
+	0x00010000, 0xee0a4076,
+	0x018c0000, 0x00018000,
 	0x807d847d, 0x8070ff70,
-	0x00000400, 0xbf0a7b7d,
-	0xbfa2ffeb, 0xb8fb1e06,
-	0x8b7bc17b, 0xbfa1000d,
-	0x847b837b, 0x807b7d7b,
-	0xbefe00c1, 0xbeff0080,
-	0x7e008700, 0xc4068070,
-	0x008ce800, 0x00000000,
-	0x807d817d, 0x8070ff70,
-	0x00000080, 0xbf0a7b7d,
-	0xbfa2fff7, 0xbfa00171,
-	0xbef4007e, 0x8b75ff7f,
-	0x0000ffff, 0x8c75ff75,
-	0x00040000, 0xbef60080,
-	0xbef700ff, 0x10807fac,
+	0x00000200, 0xbf0a7b7d,
+	0xbfa2ffe9, 0xbfa0001a,
+	0xbefd0084, 0xbf0a7b7d,
+	0xbfa10017, 0x7e008700,
+	0x7e028701, 0x7e048702,
+	0x7e068703, 0x80767074,
+	0x82778075, 0xee0a4076,
+	0x000c0000, 0x00000000,
+	0xee0a4076, 0x008c0000,
+	0x00010000, 0xee0a4076,
+	0x010c0000, 0x00020000,
+	0xee0a4076, 0x018c0000,
+	0x00030000, 0x807d847d,
+	0x8070ff70, 0x00000400,
+	0xbf0a7b7d, 0xbfa2ffe9,
+	0xbfa0014c, 0xbef4007e,
+	0x8b75ff7f, 0x0000ffff,
 	0xbef1007f, 0xb8f20742,
 	0x84729972, 0x8b6eff7f,
-	0x04000000, 0xbfa1003b,
+	0x04000000, 0xbfa10044,
 	0xbefe00c1, 0x857d9972,
 	0x8b7d817d, 0xbf06817d,
 	0xbfa20002, 0xbeff0080,
 	0xbfa00001, 0xbeff00c1,
 	0xb8ef4306, 0x8b6fc16f,
-	0xbfa10030, 0x846f896f,
-	0xbef6006f, 0xb8f83b05,
+	0xbfa10039, 0x846f896f,
+	0xb8f83b05, 0x80788178,
+	0xbf0d9972, 0xbfa20002,
+	0x84788978, 0xbfa00001,
+	0x84788a78, 0x8078ff78,
+	0x00000200, 0x8078ff78,
+	0x00000080, 0x857d9972,
+	0x8b7d817d, 0xbf06817d,
+	0xbefd0080, 0xd71f0001,
+	0x000100c1, 0xd7200001,
+	0x000202c1, 0x30020282,
+	0xbfa20012, 0x80767874,
+	0x82778075, 0xee0a0076,
+	0x000c0000, 0x00000000,
+	0xbf8a0000, 0xd8340000,
+	0x00000001, 0xd5250001,
+	0x0001ff01, 0x00000080,
+	0x807dff7d, 0x00000080,
+	0x8078ff78, 0x00000080,
+	0xbf0a6f7d, 0xbfa2ffef,
+	0xbfa00011, 0x80767874,
+	0x82778075, 0xee0a0076,
+	0x000c0000, 0x00000000,
+	0xbf8a0000, 0xd8340000,
+	0x00000001, 0xd5250001,
+	0x0001ff01, 0x00000100,
+	0x807dff7d, 0x00000100,
+	0x8078ff78, 0x00000100,
+	0xbf0a6f7d, 0xbfa2ffef,
+	0xbef80080, 0xbefe00c1,
+	0x857d9972, 0x8b7d817d,
+	0xbf06817d, 0xbfa20002,
+	0xbeff0080, 0xbfa00001,
+	0xbeff00c1, 0xb8ef3b05,
+	0x806f816f, 0x846f826f,
+	0x857d9972, 0x8b7d817d,
+	0xbf06817d, 0xbfa2002c,
+	0xbeee0078, 0x8078ff78,
+	0x00000200, 0xbefd0084,
+	0x80767874, 0x82778075,
+	0xee0a0076, 0x000c0000,
+	0x00000000, 0xee0a0076,
+	0x000c0001, 0x00008000,
+	0xee0a0076, 0x000c0002,
+	0x00010000, 0xee0a0076,
+	0x000c0003, 0x00018000,
+	0xbf8a0000, 0x7e008500,
+	0x7e028501, 0x7e048502,
+	0x7e068503, 0x807d847d,
+	0x8078ff78, 0x00000200,
+	0xbf0a6f7d, 0xbfa2ffe8,
+	0x80766e74, 0x82778075,
+	0xee0a0076, 0x000c0000,
+	0x00000000, 0xee0a0076,
+	0x000c0001, 0x00008000,
+	0xee0a0076, 0x000c0002,
+	0x00010000, 0xee0a0076,
+	0x000c0003, 0x00018000,
+	0xbf8a0000, 0xbfa0002d,
+	0xbeee0078, 0x8078ff78,
+	0x00000400, 0xbefd0084,
+	0xbf0a6f7d, 0xbfa10018,
+	0x80767874, 0x82778075,
+	0xee0a0076, 0x000c0000,
+	0x00000000, 0xee0a0076,
+	0x000c0001, 0x00010000,
+	0xee0a0076, 0x000c0002,
+	0x00020000, 0xee0a0076,
+	0x000c0003, 0x00030000,
+	0xbf8a0000, 0x7e008500,
+	0x7e028501, 0x7e048502,
+	0x7e068503, 0x807d847d,
+	0x8078ff78, 0x00000400,
+	0xbf0a6f7d, 0xbfa2ffe8,
+	0x80766e74, 0x82778075,
+	0xee0a0076, 0x000c0000,
+	0x00000000, 0xee0a0076,
+	0x000c0001, 0x00010000,
+	0xee0a0076, 0x000c0002,
+	0x00020000, 0xee0a0076,
+	0x000c0003, 0x00030000,
+	0xbf8a0000, 0xb8f83b05,
 	0x80788178, 0xbf0d9972,
 	0xbfa20002, 0x84788978,
 	0xbfa00001, 0x84788a78,
-	0xb8ee1e06, 0x846e8a6e,
-	0x80786e78, 0x8078ff78,
-	0x00000200, 0x8078ff78,
-	0x00000080, 0xbef600ff,
-	0x01000000, 0x857d9972,
-	0x8b7d817d, 0xbf06817d,
-	0xbefd0080, 0xbfa2000d,
-	0xc4050078, 0x0080e800,
-	0x00000000, 0xbf8a0000,
-	0xdac00000, 0x00000000,
-	0x807dff7d, 0x00000080,
-	0x8078ff78, 0x00000080,
-	0xbf0a6f7d, 0xbfa2fff4,
-	0xbfa0000c, 0xc4050078,
-	0x0080e800, 0x00000000,
-	0xbf8a0000, 0xdac00000,
-	0x00000000, 0x807dff7d,
-	0x00000100, 0x8078ff78,
-	0x00000100, 0xbf0a6f7d,
-	0xbfa2fff4, 0xbef80080,
-	0xbefe00c1, 0x857d9972,
-	0x8b7d817d, 0xbf06817d,
-	0xbfa20002, 0xbeff0080,
-	0xbfa00001, 0xbeff00c1,
-	0xb8ef3b05, 0x806f816f,
-	0x846f826f, 0x857d9972,
-	0x8b7d817d, 0xbf06817d,
-	0xbfa2002c, 0xbef600ff,
-	0x01000000, 0xbeee0078,
 	0x8078ff78, 0x00000200,
-	0xbefd0084, 0xbf0a6f7d,
-	0xbfa10061, 0xc4050078,
-	0x008ce800, 0x00000000,
-	0xc4050078, 0x008ce801,
-	0x00008000, 0xc4050078,
-	0x008ce802, 0x00010000,
-	0xc4050078, 0x008ce803,
-	0x00018000, 0xbf8a0000,
-	0x7e008500, 0x7e028501,
-	0x7e048502, 0x7e068503,
-	0x807d847d, 0x8078ff78,
-	0x00000200, 0xbf0a6f7d,
-	0xbfa2ffea, 0xc405006e,
-	0x008ce800, 0x00000000,
-	0xc405006e, 0x008ce801,
-	0x00008000, 0xc405006e,
-	0x008ce802, 0x00010000,
-	0xc405006e, 0x008ce803,
-	0x00018000, 0xbf8a0000,
-	0xbfa0003d, 0xbef600ff,
-	0x01000000, 0xbeee0078,
-	0x8078ff78, 0x00000400,
-	0xbefd0084, 0xbf0a6f7d,
-	0xbfa10016, 0xc4050078,
-	0x008ce800, 0x00000000,
-	0xc4050078, 0x008ce801,
-	0x00010000, 0xc4050078,
-	0x008ce802, 0x00020000,
-	0xc4050078, 0x008ce803,
-	0x00030000, 0xbf8a0000,
-	0x7e008500, 0x7e028501,
-	0x7e048502, 0x7e068503,
-	0x807d847d, 0x8078ff78,
-	0x00000400, 0xbf0a6f7d,
-	0xbfa2ffea, 0xb8ef1e06,
-	0x8b6fc16f, 0xbfa1000f,
-	0x846f836f, 0x806f7d6f,
-	0xbefe00c1, 0xbeff0080,
-	0xc4050078, 0x008ce800,
-	0x00000000, 0xbf8a0000,
-	0x7e008500, 0x807d817d,
-	0x8078ff78, 0x00000080,
-	0xbf0a6f7d, 0xbfa2fff6,
-	0xbeff00c1, 0xc405006e,
-	0x008ce800, 0x00000000,
-	0xc405006e, 0x008ce801,
-	0x00010000, 0xc405006e,
-	0x008ce802, 0x00020000,
-	0xc405006e, 0x008ce803,
-	0x00030000, 0xbf8a0000,
-	0xb8f83b05, 0x80788178,
-	0xbf0d9972, 0xbfa20002,
-	0x84788978, 0xbfa00001,
-	0x84788a78, 0xb8ee1e06,
-	0x846e8a6e, 0x80786e78,
-	0x8078ff78, 0x00000200,
-	0x80f8ff78, 0x00000050,
-	0xbef600ff, 0x01000000,
+	0x80f8ff78, 0x00000060,
+	0x80767874, 0x82778075,
 	0xbefd00ff, 0x0000006c,
-	0x80f89078, 0xf462403a,
-	0xf0000000, 0xbf8a0000,
-	0x80fd847d, 0xbf800000,
-	0xbe804300, 0xbe824302,
-	0x80f8a078, 0xf462603a,
-	0xf0000000, 0xbf8a0000,
+	0xf460403b, 0xf8000000,
+	0xbf8a0000, 0x80fd847d,
+	0xbf800000, 0xbe804300,
+	0xbe824302, 0x80f6a076,
+	0x82f78077, 0xf460603b,
+	0xf8000000, 0xbf8a0000,
 	0x80fd887d, 0xbf800000,
 	0xbe804300, 0xbe824302,
 	0xbe844304, 0xbe864306,
-	0x80f8c078, 0xf462803a,
-	0xf0000000, 0xbf8a0000,
-	0x80fd907d, 0xbf800000,
-	0xbe804300, 0xbe824302,
-	0xbe844304, 0xbe864306,
-	0xbe884308, 0xbe8a430a,
-	0xbe8c430c, 0xbe8e430e,
-	0xbf06807d, 0xbfa1fff0,
-	0xb980f801, 0x00000000,
-	0xb8f83b05, 0x80788178,
-	0xbf0d9972, 0xbfa20002,
-	0x84788978, 0xbfa00001,
-	0x84788a78, 0xb8ee1e06,
-	0x846e8a6e, 0x80786e78,
+	0x80f6c076, 0x82f78077,
+	0xf460803b, 0xf8000000,
+	0xbf8a0000, 0x80fd907d,
+	0xbf800000, 0xbe804300,
+	0xbe824302, 0xbe844304,
+	0xbe864306, 0xbe884308,
+	0xbe8a430a, 0xbe8c430c,
+	0xbe8e430e, 0xbf06807d,
+	0xbfa1ffef, 0xb980f801,
+	0x00000000, 0xb8f83b05,
+	0x80788178, 0xbf0d9972,
+	0xbfa20002, 0x84788978,
+	0xbfa00001, 0x84788a78,
 	0x8078ff78, 0x00000200,
-	0xbef600ff, 0x01000000,
-	0xbeff0071, 0xf4621bfa,
-	0xf0000000, 0x80788478,
-	0xf4621b3a, 0xf0000000,
-	0x80788478, 0xf4621b7a,
-	0xf0000000, 0x80788478,
-	0xf4621c3a, 0xf0000000,
-	0x80788478, 0xf4621c7a,
-	0xf0000000, 0x80788478,
-	0xf4621eba, 0xf0000000,
-	0x80788478, 0xf4621efa,
-	0xf0000000, 0x80788478,
-	0xf4621e7a, 0xf0000000,
-	0x80788478, 0xf4621cfa,
-	0xf0000000, 0x80788478,
-	0xf4621bba, 0xf0000000,
-	0x80788478, 0xbf8a0000,
-	0xb96ef814, 0xf4621bba,
-	0xf0000000, 0x80788478,
-	0xbf8a0000, 0xb96ef815,
-	0xf4621bba, 0xf0000000,
-	0x80788478, 0xbf8a0000,
-	0xb96ef812, 0xf4621bba,
-	0xf0000000, 0x80788478,
-	0xbf8a0000, 0xb96ef813,
-	0x8b6eff7f, 0x04000000,
-	0xbfa1000d, 0x80788478,
-	0xf4621bba, 0xf0000000,
-	0x80788478, 0xbf8a0000,
-	0xbf0d806e, 0xbfa10006,
-	0x856e906e, 0x8b6e6e6e,
-	0xbfa10003, 0xbe804ec1,
-	0x816ec16e, 0xbfa0fffb,
-	0xbefd006f, 0xbefe0070,
-	0xbeff0071, 0xb97b2011,
-	0x857b867b, 0xb97b0191,
-	0x857b827b, 0xb97bba11,
-	0xb973f801, 0xb8ee3b05,
-	0x806e816e, 0xbf0d9972,
-	0xbfa20002, 0x846e896e,
-	0xbfa00001, 0x846e8a6e,
-	0xb8ef1e06, 0x846f8a6f,
-	0x806e6f6e, 0x806eff6e,
-	0x00000200, 0x806e746e,
-	0x826f8075, 0x8b6fff6f,
-	0x0000ffff, 0xf4605c37,
-	0xf8000050, 0xf4605d37,
-	0xf8000060, 0xf4601e77,
-	0xf8000074, 0xbf8a0000,
+	0x80767874, 0x82778075,
+	0xbeff0071, 0xf4601bfb,
+	0xf8000000, 0xf4601b3b,
+	0xf8000004, 0xf4601b7b,
+	0xf8000008, 0xf4601c3b,
+	0xf800000c, 0xf4601c7b,
+	0xf8000010, 0xf4601ebb,
+	0xf8000014, 0xf4601efb,
+	0xf8000018, 0xf4601e7b,
+	0xf800001c, 0xf4601cfb,
+	0xf8000020, 0xf4601bbb,
+	0xf8000024, 0xbf8a0000,
+	0xb96ef814, 0xf4601bbb,
+	0xf8000028, 0xbf8a0000,
+	0xb96ef815, 0xf4601bbb,
+	0xf800002c, 0xbf8a0000,
+	0xb96ef812, 0xf4601bbb,
+	0xf8000030, 0xbf8a0000,
+	0xb96ef813, 0x8b6eff7f,
+	0x04000000, 0xbfa1000b,
+	0xf4601bbb, 0xf8000038,
+	0xbf8a0000, 0xbf0d806e,
+	0xbfa10006, 0x856e906e,
+	0x8b6e6e6e, 0xbfa10003,
+	0xbe804ec1, 0x816ec16e,
+	0xbfa0fffb, 0xbefd006f,
+	0xbefe0070, 0xbeff0071,
+	0xb97b2011, 0x857b867b,
+	0xb97b0191, 0x857b827b,
+	0xb97bba11, 0xb973f801,
+	0xb8ee3b05, 0x806e816e,
+	0xbf0d9972, 0xbfa20002,
+	0x846e896e, 0xbfa00001,
+	0x846e8a6e, 0x806eff6e,
+	0x00000240, 0x806e746e,
+	0x826f8075, 0xf4605c37,
+	0xf8000010, 0xf4605d37,
+	0xf8000020, 0xf4601e77,
+	0xf8000034, 0xbf8a0000,
 	0x8b6dff6d, 0x0000ffff,
 	0x8bfe7e7e, 0x8bea6a6a,
 	0x936eff77, 0x0002001a,
···
 };
 
 static const uint32_t cwsr_trap_gfx12_1_0_hex[] = {
-	0xbfa00001, 0xbfa003b7,
+	0xbfa00001, 0xbfa003ac,
 	0xb0804009, 0xb8f8f804,
 	0x9178ff78, 0x00008c00,
 	0xb8fbf811, 0x8b6eff78,
 	0x00004000, 0xbfa10008,
 	0x8b6eff7b, 0x00000080,
 	0xbfa20018, 0x8b6ea07b,
-	0xbfa200e1, 0xbf830010,
+	0xbfa200d1, 0xbf830010,
 	0xb8fbf811, 0xbfa0fffb,
 	0x8b6eff7b, 0x00000bd0,
 	0xbfa20010, 0xb8eef812,
···
 	0xf0000000, 0xbfa20005,
 	0x8b6fff6f, 0x00000200,
 	0xbfa20002, 0x8b6ea07b,
-	0xbfa200cb, 0x9177ff77,
+	0xbfa200bb, 0x9177ff77,
 	0x007fc000, 0xb8fa04a1,
 	0x847a967a, 0x8c777a77,
 	0xb8fa0421, 0x847a957a,
···
 	0x01ffffff, 0xb8fbf811,
 	0xbf0d847b, 0xbfa20078,
 	0xf4003eb6, 0xf8000000,
-	0xf4003bb6, 0xf8000008,
-	0xbfc70001, 0x8b76ff7a,
+	0xbfc70000, 0xf4003bb6,
+	0xf8000008, 0x8b76ff7a,
 	0x80000000, 0xbfa20027,
 	0x9376ff7a, 0x00060019,
 	0x81f9a376, 0xbf0b8179,
···
 	0x8c777a77, 0xb980f821,
 	0x00000000, 0xbf0d847b,
 	0xbfa20078, 0xf4003eb6,
-	0xf8000000, 0xf4003bb6,
-	0xf8000008, 0xbfc70001,
+	0xf8000000, 0xbfc70000,
+	0xf4003bb6, 0xf8000008,
 	0x8b76ff7a, 0x80000000,
 	0xbfa20027, 0x9376ff7a,
 	0x00060019, 0x81f9a376,
···
 	0x8070ff70, 0x00000200,
 	0x7e000280, 0x7e020280,
 	0x7e040280, 0xbefd0080,
-	0xb8faf802, 0xbf0c8b7a,
-	0xbfa20003, 0xbe804fc2,
-	0xbf94fffe, 0xbfa10001,
-	0xbe804ec4, 0xbf94fffc,
-	0xb8faf804, 0x8b7aff7a,
-	0x0001000c, 0x9178ff78,
-	0x0001000c, 0x8c787a78,
 	0xd7610002, 0x0000fa71,
-	0x807d817d, 0xd7610002,
+	0x807d817d, 0xb8faf802,
+	0xbf0c8b7a, 0xbfa20003,
+	0xbe804fc2, 0xbf94fffe,
+	0xbfa10001, 0xbe804ec4,
+	0xbf94fffc, 0xbefa4c88,
+	0xbfc70000, 0xbf0c807a,
+	0xbfa20006, 0x9371ff7a,
+	0x00070004, 0x937aff7a,
+	0x00070010, 0xbf06717a,
+	0xbfa2fff6, 0xb8faf804,
+	0x8b7aff7a, 0x0001000c,
+	0x9178ff78, 0x0001000c,
+	0x8c787a78, 0xd7610002,
 	0x0000fa6c, 0x807d817d,
-	0x917aff6d, 0x80000000,
+	0x8b7aff6d, 0x01ffffff,
 	0xd7610002, 0x0000fa7a,
 	0x807d817d, 0xd7610002,
 	0x0000fa6e, 0x807d817d,
···
 	0x807d817d, 0xbefa4c88,
 	0xbfc70000, 0xd7610002,
 	0x0000fa7a, 0x807d817d,
-	0xbefe00ff, 0x0000ffff,
-	0xbeff0080, 0x80767074,
-	0x82778075, 0xee0a4076,
-	0x010c0000, 0x00000000,
-	0xbefe00c1, 0x7e040280,
-	0xbefa5081, 0xbfc70000,
-	0xd7610002, 0x0001007a,
-	0xbefa5082, 0xbfc70000,
-	0xd7610002, 0x0001027a,
-	0xbefa5083, 0xbfc70000,
-	0xd7610002, 0x0001047a,
-	0xbefa5084, 0xbfc70000,
-	0xd7610002, 0x0001067a,
-	0xbefa5085, 0xbfc70000,
-	0xd7610002, 0x0001087a,
-	0xbefa5086, 0xbfc70000,
-	0xd7610002, 0x00010a7a,
-	0xbefa5087, 0xbfc70000,
-	0xd7610002, 0x00010c7a,
-	0xbefa5088, 0xbfc70000,
-	0xd7610002, 0x00010e7a,
-	0xbefa5089, 0xbfc70000,
-	0xd7610002, 0x0001107a,
-	0xbefa508a, 0xbfc70000,
-	0xd7610002, 0x0001127a,
-	0xbefa508b, 0xbfc70000,
-	0xd7610002, 0x0001147a,
-	0xbefa508c, 0xbfc70000,
-	0xd7610002, 0x0001167a,
-	0xbefa508d, 0xbfc70000,
-	0xd7610002, 0x0001187a,
-	0xbefa508e, 0xbfc70000,
-	0xd7610002, 0x00011a7a,
-	0xbefa508f, 0xbfc70000,
-	0xd7610002, 0x00011c7a,
-	0xbefa5090, 0xbfc70000,
-	0xd7610002, 0x00011e7a,
+	0xb8faf81a, 0xd7610002,
+	0x0000fa7a, 0x807d817d,
+	0xbefe00c1, 0xbeff0080,
+	0x80767074, 0x82778075,
 	0xee0a4076, 0x010c0000,
-	0x00008000, 0xb8f03b05,
-	0x80708170, 0xbf0d9973,
-	0xbfa20002, 0x84708970,
-	0xbfa00001, 0x84708a70,
-	0xbef90080, 0xbefd0080,
-	0xbf800000, 0xbe804100,
-	0xbe824102, 0xbe844104,
-	0xbe864106, 0xbe884108,
-	0xbe8a410a, 0xbe8c410c,
-	0xbe8e410e, 0xd7610002,
-	0x0000f200, 0x80798179,
-	0xd7610002, 0x0000f201,
-	0x80798179, 0xd7610002,
-	0x0000f202, 0x80798179,
-	0xd7610002, 0x0000f203,
-	0x80798179, 0xd7610002,
-	0x0000f204, 0x80798179,
-	0xd7610002, 0x0000f205,
-	0x80798179, 0xd7610002,
-	0x0000f206, 0x80798179,
-	0xd7610002, 0x0000f207,
-	0x80798179, 0xd7610002,
-	0x0000f208, 0x80798179,
-	0xd7610002, 0x0000f209,
-	0x80798179, 0xd7610002,
-	0x0000f20a, 0x80798179,
-	0xd7610002, 0x0000f20b,
-	0x80798179, 0xd7610002,
-	0x0000f20c, 0x80798179,
-	0xd7610002, 0x0000f20d,
-	0x80798179, 0xd7610002,
-	0x0000f20e, 0x80798179,
-	0xd7610002, 0x0000f20f,
-	0x80798179, 0xbf06a079,
-	0xbfa10009, 0x80767074,
-	0x82778075, 0xee0a4076,
-	0x010c0000, 0x00000000,
-	0x8070ff70, 0x00000080,
-	0xbef90080, 0x7e040280,
-	0x807d907d, 0xbf0aff7d,
-	0x00000060, 0xbfa2ffb9,
+	0x00000000, 0xbefe00c1,
+	0x7e040280, 0xbefa5081,
+	0xbfc70000, 0xd7610002,
+	0x0001007a, 0xbefa5082,
+	0xbfc70000, 0xd7610002,
+	0x0001027a, 0xbefa5083,
+	0xbfc70000, 0xd7610002,
+	0x0001047a, 0xbefa5084,
+	0xbfc70000, 0xd7610002,
+	0x0001067a, 0xbefa5085,
+	0xbfc70000, 0xd7610002,
+	0x0001087a, 0xbefa5086,
+	0xbfc70000, 0xd7610002,
+	0x00010a7a, 0xbefa5087,
+	0xbfc70000, 0xd7610002,
+	0x00010c7a, 0xbefa5088,
+	0xbfc70000, 0xd7610002,
+	0x00010e7a, 0xbefa5089,
+	0xbfc70000, 0xd7610002,
+	0x0001107a, 0xbefa508a,
+	0xbfc70000, 0xd7610002,
+	0x0001127a, 0xbefa508b,
+	0xbfc70000, 0xd7610002,
+	0x0001147a, 0xbefa508c,
+	0xbfc70000, 0xd7610002,
+	0x0001167a, 0xbefa508d,
+	0xbfc70000, 0xd7610002,
+	0x0001187a, 0xbefa508e,
+	0xbfc70000, 0xd7610002,
+	0x00011a7a, 0xbefa508f,
+	0xbfc70000, 0xd7610002,
+	0x00011c7a, 0xbefa5090,
+	0xbfc70000, 0xd7610002,
+	0x00011e7a, 0xee0a4076,
+	0x010c0000, 0x00008000,
+	0xb8f03b05, 0x80708170,
+	0xbf0d9973, 0xbfa20002,
+	0x84708970, 0xbfa00001,
+	0x84708a70, 0xbef90080,
+	0xbefd0080, 0xbf800000,
 	0xbe804100, 0xbe824102,
 	0xbe844104, 0xbe864106,
 	0xbe884108, 0xbe8a410a,
+	0xbe8c410c, 0xbe8e410e,
 	0xd7610002, 0x0000f200,
 	0x80798179, 0xd7610002,
 	0x0000f201, 0x80798179,
···
 	0xd7610002, 0x0000f20a,
 	0x80798179, 0xd7610002,
 	0x0000f20b, 0x80798179,
-	0xbefe00ff, 0x0000ffff,
+	0xd7610002, 0x0000f20c,
+	0x80798179, 0xd7610002,
+	0x0000f20d, 0x80798179,
+	0xd7610002, 0x0000f20e,
+	0x80798179, 0xd7610002,
+	0x0000f20f, 0x80798179,
+	0xbf06a079, 0xbfa10009,
 	0x80767074, 0x82778075,
 	0xee0a4076, 0x010c0000,
-	0x00000000, 0xbefe00c1,
-	0x857d9973, 0x8b7d817d,
-	0xbf06817d, 0xbfa20002,
-	0xbeff0080, 0xbfa00001,
-	0xbeff00c1, 0xb8fb4306,
-	0x8b7bc17b, 0xbfa10042,
-	0x8b7aff6d, 0x80000000,
-	0xbfa1003f, 0x847b8a7b,
-	0xb8f03b05, 0x80708170,
-	0xbf0d9973, 0xbfa20002,
-	0x84708970, 0xbfa00001,
-	0x84708a70, 0x8070ff70,
-	0x00000200, 0x8070ff70,
-	0x00000200, 0xd71f0000,
-	0x000100c1, 0xd7200000,
-	0x000200c1, 0x16000084,
-	0x857d9973, 0x8b7d817d,
-	0xbf06817d, 0xbefd0080,
-	0xbfa20015, 0xbe8300ff,
-	0x00000080, 0xbf800000,
-	0xbf800000, 0xbf800000,
-	0xd8d80000, 0x01000000,
-	0xbf8a0000, 0x80767074,
+	0x00000000, 0x8070ff70,
+	0x00000080, 0xbef90080,
+	0x7e040280, 0x807d907d,
+	0xbf0aff7d, 0x00000060,
+	0xbfa2ffb9, 0xbe804100,
+	0xbe824102, 0xbe844104,
+	0xbe864106, 0xbe884108,
+	0xbe8a410a, 0xd7610002,
+	0x0000f200, 0x80798179,
+	0xd7610002, 0x0000f201,
+	0x80798179, 0xd7610002,
+	0x0000f202, 0x80798179,
+	0xd7610002, 0x0000f203,
+	0x80798179, 0xd7610002,
+	0x0000f204, 0x80798179,
+	0xd7610002, 0x0000f205,
+	0x80798179, 0xd7610002,
+	0x0000f206, 0x80798179,
+	0xd7610002, 0x0000f207,
+	0x80798179, 0xd7610002,
+	0x0000f208, 0x80798179,
+	0xd7610002, 0x0000f209,
+	0x80798179, 0xd7610002,
+	0x0000f20a, 0x80798179,
+	0xd7610002, 0x0000f20b,
+	0x80798179, 0xbefe00ff,
+	0x0000ffff, 0x80767074,
 	0x82778075, 0xee0a4076,
-	0x008c0000, 0x00000000,
-	0x807d037d, 0x80700370,
-	0xd5250000, 0x0001ff00,
-	0x00000080, 0xbf0a7b7d,
-	0xbfa2fff1, 0xbfa00014,
-	0xbe8300ff, 0x00000100,
+	0x010c0000, 0x00000000,
+	0xbefe00c1, 0x857d9973,
+	0x8b7d817d, 0xbf06817d,
+	0xbfa20002, 0xbeff0080,
+	0xbfa00001, 0xbeff00c1,
+	0xb8fb4306, 0x8b7bc17b,
+	0xbfa10042, 0x8b7aff6d,
+	0x80000000, 0xbfa1003f,
+	0x847b8a7b, 0xb8f03b05,
+	0x80708170, 0xbf0d9973,
+	0xbfa20002, 0x84708970,
+	0xbfa00001, 0x84708a70,
+	0x8070ff70, 0x00000200,
+	0x8070ff70, 0x00000200,
+	0xd71f0000, 0x000100c1,
+	0xd7200000, 0x000200c1,
+	0x16000084, 0x857d9973,
+	0x8b7d817d, 0xbf06817d,
+	0xbefd0080, 0xbfa20015,
+	0xbe8300ff, 0x00000080,
 	0xbf800000, 0xbf800000,
 	0xbf800000, 0xd8d80000,
 	0x01000000, 0xbf8a0000,
···
 	0xee0a4076, 0x008c0000,
 	0x00000000, 0x807d037d,
 	0x80700370, 0xd5250000,
-	0x0001ff00, 0x00000100,
+	0x0001ff00, 0x00000080,
 	0xbf0a7b7d, 0xbfa2fff1,
-	0xbefe00c1, 0x857d9973,
-	0x8b7d817d, 0xbf06817d,
-	0xbfa20004, 0xbef000ff,
-	0x00000200, 0xbeff0080,
-	0xbfa00003, 0xbef000ff,
-	0x00000400, 0xbeff00c1,
-	0xb8fb3b05, 0x807b817b,
-	0x847b827b, 0x857d9973,
-	0x8b7d817d, 0xbf06817d,
-	0xbfa2001b, 0xbefd0084,
-	0xbf0a7b7d, 0xbfa10032,
-	0x7e008700, 0x7e028701,
-	0x7e048702, 0x7e068703,
-	0x80767074, 0x82778075,
-	0xee0a4076, 0x000c0000,
-	0x00000000, 0xee0a4076,
-	0x008c0000, 0x00008000,
-	0xee0a4076, 0x010c0000,
-	0x00010000, 0xee0a4076,
-	0x018c0000, 0x00018000,
-	0x807d847d, 0x8070ff70,
-	0x00000200, 0xbf0a7b7d,
-	0xbfa2ffe9, 0xbfa0001a,
+	0xbfa00014, 0xbe8300ff,
+	0x00000100, 0xbf800000,
+	0xbf800000, 0xbf800000,
+	0xd8d80000, 0x01000000,
+	0xbf8a0000, 0x80767074,
+	0x82778075, 0xee0a4076,
+	0x008c0000, 0x00000000,
+	0x807d037d, 0x80700370,
+	0xd5250000, 0x0001ff00,
+	0x00000100, 0xbf0a7b7d,
+	0xbfa2fff1, 0xbefe00c1,
+	0x857d9973, 0x8b7d817d,
+	0xbf06817d, 0xbfa20004,
+	0xbef000ff, 0x00000200,
+	0xbeff0080, 0xbfa00003,
+	0xbef000ff, 0x00000400,
+	0xbeff00c1, 0xb8fb3b05,
+	0x807b817b, 0x847b827b,
+	0x857d9973, 0x8b7d817d,
+	0xbf06817d, 0xbfa2001b,
 	0xbefd0084, 0xbf0a7b7d,
-	0xbfa10017, 0x7e008700,
+	0xbfa10032, 0x7e008700,
 	0x7e028701, 0x7e048702,
 	0x7e068703, 0x80767074,
 	0x82778075, 0xee0a4076,
 	0x000c0000, 0x00000000,
 	0xee0a4076, 0x008c0000,
-	0x00010000, 0xee0a4076,
-	0x010c0000, 0x00020000,
+	0x00008000, 0xee0a4076,
+	0x010c0000, 0x00010000,
 	0xee0a4076, 0x018c0000,
-	0x00030000, 0x807d847d,
-	0x8070ff70, 0x00000400,
+	0x00018000, 0x807d847d,
+	0x8070ff70, 0x00000200,
 	0xbf0a7b7d, 0xbfa2ffe9,
-	0xbfa00180, 0xbef4007e,
-	0x8b75ff7f, 0x01ffffff,
-	0xbef1007f, 0xb8f20742,
-	0x84729972, 0x8b6eff7f,
-	0x04000000, 0xbfa10044,
+	0xbfa0001a, 0xbefd0084,
+	0xbf0a7b7d, 0xbfa10017,
+	0x7e008700, 0x7e028701,
+	0x7e048702, 0x7e068703,
+	0x80767074, 0x82778075,
+	0xee0a4076, 0x000c0000,
+	0x00000000, 0xee0a4076,
+	0x008c0000, 0x00010000,
+	0xee0a4076, 0x010c0000,
+	0x00020000, 0xee0a4076,
+	0x018c0000, 0x00030000,
+	0x807d847d, 0x8070ff70,
+	0x00000400, 0xbf0a7b7d,
+	0xbfa2ffe9, 0xbfa00184,
+	0xbef4007e, 0x8b75ff7f,
+	0x01ffffff, 0xbef1007f,
+	0xb8f20742, 0x84729972,
+	0x8b6eff7f, 0x04000000,
+	0xbfa10044, 0xbefe00c1,
+	0x857d9972, 0x8b7d817d,
+	0xbf06817d, 0xbfa20002,
+	0xbeff0080, 0xbfa00001,
+	0xbeff00c1, 0xb8ef4306,
+	0x8b6fc16f, 0xbfa10039,
+	0x846f8a6f, 0xb8f83b05,
+	0x80788178, 0xbf0d9972,
+	0xbfa20002, 0x84788978,
+	0xbfa00001, 0x84788a78,
+	0x8078ff78, 0x00000200,
+	0x8078ff78, 0x00000200,
+	0x857d9972, 0x8b7d817d,
+	0xbf06817d, 0xbefd0080,
+	0xd71f0001, 0x000100c1,
+	0xd7200001, 0x000202c1,
+	0x30020282, 0xbfa20012,
+	0x80767874, 0x82778075,
+	0xee0a0076, 0x000c0000,
+	0x00000000, 0xbf8a0000,
+	0xd8340000, 0x00000001,
+	0xd5250001, 0x0001ff01,
+	0x00000080, 0x807dff7d,
+	0x00000080, 0x8078ff78,
+	0x00000080, 0xbf0a6f7d,
+	0xbfa2ffef, 0xbfa00011,
+	0x80767874, 0x82778075,
+	0xee0a0076, 0x000c0000,
+	0x00000000, 0xbf8a0000,
+	0xd8340000, 0x00000001,
+	0xd5250001, 0x0001ff01,
+	0x00000100, 0x807dff7d,
+	0x00000100, 0x8078ff78,
+	0x00000100, 0xbf0a6f7d,
+	0xbfa2ffef, 0xbef80080,
 	0xbefe00c1, 0x857d9972,
 	0x8b7d817d, 0xbf06817d,
 	0xbfa20002, 0xbeff0080,
 	0xbfa00001, 0xbeff00c1,
-	0xb8ef4306, 0x8b6fc16f,
-	0xbfa10039, 0x846f8a6f,
+	0xb8ef3b05, 0x806f816f,
+	0x846f826f, 0x857d9972,
+	0x8b7d817d, 0xbf06817d,
+	0xbfa2002c, 0xbeee0078,
+	0x8078ff78, 0x00000200,
+	0xbefd0084, 0x80767874,
+	0x82778075, 0xee0a0076,
+	0x000c0000, 0x00000000,
+	0xee0a0076, 0x000c0001,
+	0x00008000, 0xee0a0076,
+	0x000c0002, 0x00010000,
+	0xee0a0076, 0x000c0003,
+	0x00018000, 0xbf8a0000,
+	0x7e008500, 0x7e028501,
+	0x7e048502, 0x7e068503,
+	0x807d847d, 0x8078ff78,
+	0x00000200, 0xbf0a6f7d,
+	0xbfa2ffe8, 0x80766e74,
+	0x82778075, 0xee0a0076,
+	0x000c0000, 0x00000000,
+	0xee0a0076, 0x000c0001,
+	0x00008000, 0xee0a0076,
+	0x000c0002, 0x00010000,
+	0xee0a0076, 0x000c0003,
+	0x00018000, 0xbf8a0000,
+	0xbfa0002d, 0xbeee0078,
+	0x8078ff78, 0x00000400,
+	0xbefd0084, 0xbf0a6f7d,
+	0xbfa10018, 0x80767874,
+	0x82778075, 0xee0a0076,
+	0x000c0000, 0x00000000,
+	0xee0a0076, 0x000c0001,
+	0x00010000, 0xee0a0076,
+	0x000c0002, 0x00020000,
+	0xee0a0076, 0x000c0003,
+	0x00030000, 0xbf8a0000,
+	0x7e008500, 0x7e028501,
+	0x7e048502, 0x7e068503,
+	0x807d847d, 0x8078ff78,
+	0x00000400, 0xbf0a6f7d,
+	0xbfa2ffe8, 0x80766e74,
+	0x82778075, 0xee0a0076,
+	0x000c0000, 0x00000000,
+	0xee0a0076, 0x000c0001,
+	0x00010000, 0xee0a0076,
+	0x000c0002, 0x00020000,
+	0xee0a0076, 0x000c0003,
+	0x00030000, 0xbf8a0000,
 	0xb8f83b05, 0x80788178,
 	0xbf0d9972, 0xbfa20002,
 	0x84788978, 0xbfa00001,
 	0x84788a78, 0x8078ff78,
-	0x00000200, 0x8078ff78,
-	0x00000200, 0x857d9972,
-	0x8b7d817d, 0xbf06817d,
-	0xbefd0080, 0xd71f0001,
-	0x000100c1, 0xd7200001,
-	0x000202c1, 0x30020282,
-	0xbfa20012, 0x80767874,
-	0x82778075, 0xee0a0076,
-	0x000c0000, 0x00000000,
-	0xbf8a0000, 0xd8340000,
-	0x00000001, 0xd5250001,
-	0x0001ff01, 0x00000080,
-	0x807dff7d, 0x00000080,
-	0x8078ff78, 0x00000080,
-	0xbf0a6f7d, 0xbfa2ffef,
-	0xbfa00011, 0x80767874,
-	0x82778075, 0xee0a0076,
-	0x000c0000, 0x00000000,
-	0xbf8a0000, 0xd8340000,
-	0x00000001, 0xd5250001,
-	0x0001ff01, 0x00000100,
-	0x807dff7d, 0x00000100,
-	0x8078ff78, 0x00000100,
-	0xbf0a6f7d, 0xbfa2ffef,
-	0xbef80080, 0xbefe00c1,
-	0x857d9972, 0x8b7d817d,
-	0xbf06817d, 0xbfa20002,
-	0xbeff0080, 0xbfa00001,
-	0xbeff00c1, 0xb8ef3b05,
-	0x806f816f, 0x846f826f,
-	0x857d9972, 0x8b7d817d,
-	0xbf06817d, 0xbfa2002c,
-	0xbeee0078, 0x8078ff78,
-	0x00000200, 0xbefd0084,
-	0x80767874, 0x82778075,
-	0xee0a0076, 0x000c0000,
-	0x00000000, 0xee0a0076,
-	0x000c0001, 0x00008000,
-	0xee0a0076, 0x000c0002,
-	0x00010000, 0xee0a0076,
-	0x000c0003, 0x00018000,
-	0xbf8a0000, 0x7e008500,
-	0x7e028501, 0x7e048502,
-	0x7e068503, 0x807d847d,
-	0x8078ff78, 0x00000200,
-	0xbf0a6f7d, 0xbfa2ffe8,
-	0x80766e74, 0x82778075,
-	0xee0a0076, 0x000c0000,
-	0x00000000, 0xee0a0076,
-	0x000c0001, 0x00008000,
-	0xee0a0076, 0x000c0002,
-	0x00010000, 0xee0a0076,
-	0x000c0003, 0x00018000,
-	0xbf8a0000, 0xbfa0002d,
-	0xbeee0078, 0x8078ff78,
-	0x00000400, 0xbefd0084,
-	0xbf0a6f7d, 0xbfa10018,
-	0x80767874, 0x82778075,
-	0xee0a0076, 0x000c0000,
-	0x00000000, 0xee0a0076,
-	0x000c0001, 0x00010000,
-	0xee0a0076, 0x000c0002,
-	0x00020000, 0xee0a0076,
-	0x000c0003, 0x00030000,
-	0xbf8a0000, 0x7e008500,
-	0x7e028501, 0x7e048502,
-	0x7e068503, 0x807d847d,
-	0x8078ff78, 0x00000400,
-	0xbf0a6f7d, 0xbfa2ffe8,
-	0x80766e74, 0x82778075,
-	0xee0a0076, 0x000c0000,
-	0x00000000, 0xee0a0076,
-	0x000c0001, 0x00010000,
-	0xee0a0076, 0x000c0002,
-	0x00020000, 0xee0a0076,
-	0x000c0003, 0x00030000,
-	0xbf8a0000, 0xb8f83b05,
-	0x80788178, 0xbf0d9972,
-	0xbfa20002, 0x84788978,
-	0xbfa00001, 0x84788a78,
-	0x8078ff78, 0x00000200,
-	0x80f8ff78, 0x00000060,
-	0x80767874, 0x82778075,
-	0xbefd00ff, 0x0000006c,
-	0xf460403b, 0xf8000000,
-	0xbf8a0000, 0x80fd847d,
-	0xbf800000, 0xbe804300,
-	0xbe824302, 0x80f6a076,
-	0x82f78077, 0xf460603b,
+	0x00000200, 0x80f8ff78,
+	0x00000060, 0x80767874,
+	0x82778075, 0xbefd00ff,
+	0x0000006c, 0xf460403b,
 	0xf8000000, 0xbf8a0000,
-	0x80fd887d, 0xbf800000,
+	0x80fd847d, 0xbf800000,
 	0xbe804300, 0xbe824302,
-	0xbe844304, 0xbe864306,
-	0x80f6c076, 0x82f78077,
-	0xf460803b, 0xf8000000,
-	0xbf8a0000, 0x80fd907d,
+	0x80f6a076, 0x82f78077,
+	0xf460603b, 0xf8000000,
+	0xbf8a0000, 0x80fd887d,
 	0xbf800000, 0xbe804300,
 	0xbe824302, 0xbe844304,
-	0xbe864306, 0xbe884308,
-	0xbe8a430a, 0xbe8c430c,
-	0xbe8e430e, 0xbf06807d,
-	0xbfa1ffef, 0xb980f801,
-	0x00000000, 0xb8f83b05,
-	0x80788178, 0xbf0d9972,
-	0xbfa20002, 0x84788978,
-	0xbfa00001, 0x84788a78,
-	0x8078ff78, 0x00000200,
-	0x80767874, 0x82778075,
-	0xbeff0071, 0xf4601bfb,
-	0xf8000000, 0xf4601b3b,
-	0xf8000004, 0xf4601b7b,
-	0xf8000008, 0xf4601c3b,
-	0xf800000c, 0xf4601c7b,
-	0xf8000010, 0xf4601ebb,
-	0xf8000014, 0xf4601efb,
-	0xf8000018, 0xf4601e7b,
-	0xf800001c, 0xf4601cfb,
-	0xf8000020, 0xf4601bbb,
-	0xf8000024, 0xbf8a0000,
-	0xb96ef814, 0xf4601bbb,
-	0xf8000028, 0xbf8a0000,
-	0xb96ef815, 0xf4601bbb,
-	0xf800002c, 0xbf8a0000,
-	0xb96ef812, 0xf4601bbb,
-	0xf8000030, 0xbf8a0000,
-	0xb96ef813, 0x8b6eff7f,
-	0x04000000, 0xbfa10022,
-	0xf4601bbb, 0xf8000038,
-	0xbf8a0000, 0xbf0d806e,
-	0xbfa1001d, 0x856e906e,
-	0x8b6e6e6e, 0xbfa10003,
-	0xbe804ec1, 0x816ec16e,
-	0xbfa0fffb, 0xbef800ff,
-	0x00000080, 0xbefd0081,
-	0xf4601bbb, 0xf0000000,
-	0xbfc70000, 0x80788478,
-	0x937eff6e, 0x00070004,
-	0x847e907e, 0x8c7d7e7d,
-	0xbe80517d, 0x917dff7d,
-	0x007f0000, 0x856e906e,
-	0x8b6e6e6e, 0xbfa10003,
-	0xbe804e7d, 0x816ec16e,
-	0xbfa0fffb, 0x807d817d,
-	0xbf08907d, 0xbfa1ffec,
-	0xf4601bbb, 0xf800003c,
-	0xbfc70000, 0xbf0d806e,
-	0xbfa1000c, 0xbf0d9a7f,
-	0xbfa10002, 0xbf068180,
-	0xbe804fc4, 0xbf94fffc,
-	0xbfa10006, 0x856e906e,
-	0x8b6e6e6e, 0xbfa10003,
-	0xbe804ec3, 0x816ec16e,
-	0xbfa0fffb, 0xbefd006f,
-	0xbefe0070, 0xbeff0071,
-	0xb979f822, 0xb97b2011,
-	0x857b867b, 0xb97b0191,
-	0x857b827b, 0xb97bba11,
-	0xb973f801, 0xb8ee3b05,
-	0x806e816e, 0xbf0d9972,
-	0xbfa20002, 0x846e896e,
-	0xbfa00001, 0x846e8a6e,
-	0x806eff6e, 0x000001c0,
-	0x806e746e, 0x826f8075,
-	0xf4605c37, 0xf8000010,
-	0xf4605d37, 0xf8000020,
-	0xf4601e77, 0xf8000034,
-	0xbf8a0000, 0x856e9677,
-	0xb96e04a1, 0x856e9577,
-	0xb96e0421, 0x856e8e77,
-	0xb96e3021, 0x8b6dff6d,
-	0x01ffffff, 0x8bfe7e7e,
-	0x8bea6a6a, 0xb97af804,
+	0xbe864306, 0x80f6c076,
+	0x82f78077, 0xf460803b,
+	0xf8000000, 0xbf8a0000,
+	0x80fd907d, 0xbf800000,
+	0xbe804300, 0xbe824302,
+	0xbe844304, 0xbe864306,
+	0xbe884308, 0xbe8a430a,
+	0xbe8c430c, 0xbe8e430e,
+	0xbf06807d, 0xbfa1ffef,
+	0xb980f801, 0x00000000,
+	0xb8f83b05, 0x80788178,
+	0xbf0d9972, 0xbfa20002,
+	0x84788978, 0xbfa00001,
+	0x84788a78, 0x8078ff78,
+	0x00000200, 0x80767874,
+	0x82778075, 0xbeff0071,
+	0xf4601bfb, 0xf8000000,
+	0xf4601b3b, 0xf8000004,
+	0xf4601b7b, 0xf8000008,
+	0xf4601c3b, 0xf800000c,
+	0xf4601c7b, 0xf8000010,
+	0xf4601ebb, 0xf8000014,
+	0xf4601efb, 0xf8000018,
+	0xf4601e7b, 0xf800001c,
+	0xf4601cfb, 0xf8000020,
+	0xf4601bbb, 0xf8000024,
+	0xbf8a0000, 0xb96ef814,
+	0xf4601bbb, 0xf8000028,
+	0xbf8a0000, 0xb96ef815,
+	0xf4601bbb, 0xf800002c,
+	0xbf8a0000, 0xb96ef812,
+	0xf4601bbb, 0xf8000030,
+	0xbf8a0000, 0xb96ef813,
+	0x8b6eff7f, 0x04000000,
+	0xbfa10022, 0xf4601bbb,
+	0xf8000038, 0xbf8a0000,
+	0xbf0d806e, 0xbfa1001d,
+	0x856e906e, 0x8b6e6e6e,
+	0xbfa10003, 0xbe804ec1,
+	0x816ec16e, 0xbfa0fffb,
+	0xbef800ff, 0x00000080,
+	0xbefd0081, 0xf4601bbb,
+	0xf0000000, 0xbfc70000,
+	0x80788478, 0x937eff6e,
+	0x00070004, 0x847e907e,
+	0x8c7d7e7d, 0xbe80517d,
+	0x917dff7d, 0x007f0000,
+	0x856e906e, 0x8b6e6e6e,
+	0xbfa10003, 0xbe804e7d,
+	0x816ec16e, 0xbfa0fffb,
+	0x807d817d, 0xbf08907d,
+	0xbfa1ffec, 0xf4601bbb,
+	0xf800003c, 0xbfc70000,
+	0xbf0d806e, 0xbfa1000c,
+	0xbf0d9a7f, 0xbfa10002,
+	0xbf068180, 0xbe804fc4,
+	0xbf94fffc, 0xbfa10006,
+	0x856e906e, 0x8b6e6e6e,
+	0xbfa10003, 0xbe804ec3,
+	0x816ec16e, 0xbfa0fffb,
+	0xf4601bbb, 0xf8000040,
+	0xbfc70000, 0xb96ef81a,
+	0xbefd006f, 0xbefe0070,
+	0xbeff0071, 0xb979f822,
+	0xb97b2011, 0x857b867b,
+	0xb97b0191, 0x857b827b,
+	0xb97bba11, 0xb973f801,
+	0xb8ee3b05, 0x806e816e,
+	0xbf0d9972, 0xbfa20002,
+	0x846e896e, 0xbfa00001,
+	0x846e8a6e, 0x806eff6e,
+	0x000001c0, 0x806e746e,
+	0x826f8075, 0xf4605c37,
+	0xf8000010, 0xf4605d37,
+	0xf8000020, 0xf4601e77,
+	0xf8000034, 0xbf8a0000,
+	0x856e9677, 0xb96e04a1,
+	0x856e9577, 0xb96e0421,
+	0x856e8e77, 0xb96e3021,
+	0x8b6dff6d, 0x01ffffff,
+	0x8bfe7e7e, 0x8bea6a6a,
+	0xb97af804, 0xb8eef802,
+	0xbf0c8b6e, 0xbfa20003,
+	0xbe804fc2, 0xbf94fffe,
+	0xbfa10001, 0xbe804ec4,
+	0xbf94fffc, 0x857a897a,
+	0xb97a0244, 0xbe804a6c,
 	0xb8eef802, 0xbf0c8b6e,
 	0xbfa20003, 0xbe804fc2,
 	0xbf94fffe, 0xbfa10001,
 	0xbe804ec4, 0xbf94fffc,
-	0x857a897a, 0xb97a0244,
-	0xbe804a6c, 0xb8eef802,
-	0xbf0c8b6e, 0xbfa20003,
-	0xbe804fc2, 0xbf94fffe,
-	0xbfa10001, 0xbe804ec4,
-	0xbf94fffc, 0xbfb10000,
+	0xbfb10000, 0xbf9f0000,
 	0xbf9f0000, 0xbf9f0000,
 	0xbf9f0000, 0xbf9f0000,
-	0xbf9f0000, 0x00000000,
 };

drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx12.asm

···
 #define HAVE_BANKED_VGPRS (ASIC_FAMILY == CHIP_GC_12_0_3)
 #define NUM_NAMED_BARRIERS (ASIC_FAMILY == CHIP_GC_12_0_3 ? 0x10 : 0)
 #define HAVE_CLUSTER_BARRIER (ASIC_FAMILY == CHIP_GC_12_0_3)
+#define CLUSTER_BARRIER_SERIALIZE_WORKAROUND (ASIC_FAMILY == CHIP_GC_12_0_3)
+#define RELAXED_SCHEDULING_IN_TRAP (ASIC_FAMILY == CHIP_GFX12)
 
 #define SINGLE_STEP_MISSED_WORKAROUND 1	//workaround for lost TRAP_AFTER_INST exception when SAVECTX raised
 #define HAVE_VALU_SGPR_HAZARD (ASIC_FAMILY == CHIP_GFX12)
···
 var SQ_WAVE_SCHED_MODE_DEP_MODE_SIZE		= 2
 
 var BARRIER_STATE_SIGNAL_OFFSET			= 16
+var BARRIER_STATE_SIGNAL_SIZE			= 7
 var BARRIER_STATE_MEMBER_OFFSET			= 4
 var BARRIER_STATE_MEMBER_SIZE			= 7
 var BARRIER_STATE_VALID_OFFSET			= 0
+
+#if RELAXED_SCHEDULING_IN_TRAP
+var TTMP11_SCHED_MODE_SHIFT			= 26
+var TTMP11_SCHED_MODE_SIZE			= 2
+var TTMP11_SCHED_MODE_MASK			= 0xC000000
+#endif
 
 var NAMED_BARRIERS_SR_OFFSET_FROM_HWREG		= 0x80
 var S_BARRIER_INIT_MEMBERCNT_MASK		= 0x7F0000
···
 	s_branch	L_RESTORE
 
 L_SKIP_RESTORE:
+#if RELAXED_SCHEDULING_IN_TRAP
 	// Assume most relaxed scheduling mode is set. Save and revert to normal mode.
 	s_getreg_b32	ttmp2, hwreg(HW_REG_WAVE_SCHED_MODE)
 	s_wait_alu	0
 	s_setreg_imm32_b32	hwreg(HW_REG_WAVE_SCHED_MODE, \
 		SQ_WAVE_SCHED_MODE_DEP_MODE_SHIFT, SQ_WAVE_SCHED_MODE_DEP_MODE_SIZE), 0
+#endif
 
 	s_getreg_b32	s_save_state_priv, hwreg(HW_REG_WAVE_STATE_PRIV)	//save STATUS since we will change SCC
 
+#if RELAXED_SCHEDULING_IN_TRAP
 	// Save SCHED_MODE[1:0] into ttmp11[27:26].
 	s_andn2_b32	ttmp11, ttmp11, TTMP11_SCHED_MODE_MASK
 	s_lshl_b32	ttmp2, ttmp2, TTMP11_SCHED_MODE_SHIFT
 	s_or_b32	ttmp11, ttmp11, ttmp2
+#endif
 
 	// Clear SPI_PRIO: do not save with elevated priority.
 	// Clear ECC_ERR: prevents SQC store and triggers FATAL_HALT if setreg'd.
···
 	s_cbranch_scc0	L_NO_SIGN_EXTEND_TMA
 	s_or_b32	ttmp15, ttmp15, ~ADDRESS_HI32_MASK
 L_NO_SIGN_EXTEND_TMA:
-#if ASIC_FAMILY == CHIP_GFX12
+#if RELAXED_SCHEDULING_IN_TRAP
 	// Move SCHED_MODE[1:0] from ttmp11 to unused bits in ttmp1[27:26] (return PC_HI).
 	// The second-level trap will restore from ttmp1 for backwards compatibility.
 	s_and_b32	ttmp2, ttmp11, TTMP11_SCHED_MODE_MASK
···
 	// Only restore fields which the trap handler changes.
 	s_lshr_b32	s_save_state_priv, s_save_state_priv, SQ_WAVE_STATE_PRIV_SCC_SHIFT
 
+#if RELAXED_SCHEDULING_IN_TRAP
 	// Assume relaxed scheduling mode after this point.
 	restore_sched_mode(ttmp2)
+#endif
 
 	s_setreg_b32	hwreg(HW_REG_WAVE_STATE_PRIV, SQ_WAVE_STATE_PRIV_SCC_SHIFT, \
 		SQ_WAVE_STATE_PRIV_POISON_ERR_SHIFT - SQ_WAVE_STATE_PRIV_SCC_SHIFT + 1), s_save_state_priv
···
 	v_mov_b32	v2, 0x0							//Set of SGPRs for TCP store
 	s_mov_b32	m0, 0x0							//Next lane of v2 to write to
 
+	write_hwreg_to_v2(s_save_m0)
+
 	// Ensure no further changes to barrier or LDS state.
 	// STATE_PRIV.*BARRIER_COMPLETE may change up to this point.
-	wait_trap_barriers(s_save_tmp)
+	wait_trap_barriers(s_save_tmp, s_save_m0, 1)
 
 	// Re-read final state of *BARRIER_COMPLETE fields for save.
 	s_getreg_b32	s_save_tmp, hwreg(HW_REG_WAVE_STATE_PRIV)
···
 	s_andn2_b32	s_save_state_priv, s_save_state_priv, SQ_WAVE_STATE_PRIV_ALL_BARRIER_COMPLETE_MASK
 	s_or_b32	s_save_state_priv, s_save_state_priv, s_save_tmp
 
-	write_hwreg_to_v2(s_save_m0)
 	write_hwreg_to_v2(s_save_pc_lo)
-	s_andn2_b32	s_save_tmp, s_save_pc_hi, S_SAVE_PC_HI_FIRST_WAVE_MASK
+	s_and_b32       s_save_tmp, s_save_pc_hi, ADDRESS_HI32_MASK
 	write_hwreg_to_v2(s_save_tmp)
 	write_hwreg_to_v2(s_save_exec_lo)
 #if WAVE32_ONLY
···
 	write_hwreg_to_v2(s_save_tmp)
 #endif
 
+#if ASIC_FAMILY >= CHIP_GC_12_0_3
+	s_getreg_b32	s_save_tmp, hwreg(HW_REG_WAVE_SCHED_MODE)
+	write_hwreg_to_v2(s_save_tmp)
+#endif
+
+#if ! SAVE_TTMPS_IN_SGPR_BLOCK
 	// Write HWREGs with 16 VGPR lanes. TTMPs occupy space after this.
 	s_mov_b32       exec_lo, 0xFFFF
+#else
+	// All 128 bytes are available for HWREGs.
+	s_mov_b32       exec_lo, 0xFFFFFFFF
+#endif
 	s_mov_b32	exec_hi, 0x0
 	s_add_u32	s_save_addr_lo, s_save_base_addr_lo, s_save_mem_offset
 	s_addc_u32	s_save_addr_hi, s_save_base_addr_hi, 0x0
···
 L_SKIP_CLUSTER_BARRIER_RESTORE:
 #endif
 
+#if ASIC_FAMILY >= CHIP_GC_12_0_3
+	s_load_b32	s_restore_tmp, [s_restore_addr_lo, s_restore_addr_hi], null scope:SCOPE_SYS offset:0x40
+	s_wait_kmcnt	0
+	s_setreg_b32	hwreg(HW_REG_WAVE_SCHED_MODE), s_restore_tmp
+#endif
+
 	s_mov_b32	m0, s_restore_m0
 	s_mov_b32	exec_lo, s_restore_exec_lo
 	s_mov_b32	exec_hi, s_restore_exec_hi
···
 	s_and_b64	exec, exec, exec					// Restore STATUS.EXECZ, not writable by s_setreg_b32
 	s_and_b64	vcc, vcc, vcc						// Restore STATUS.VCCZ, not writable by s_setreg_b32
 
+#if RELAXED_SCHEDULING_IN_TRAP
 	// Assume relaxed scheduling mode after this point.
 	restore_sched_mode(s_restore_tmp)
+#endif
 
 	s_setreg_b32	hwreg(HW_REG_WAVE_STATE_PRIV), s_restore_state_priv	// SCC is included, which is changed by previous salu
 
 	// Make barrier and LDS state visible to all waves in the group/cluster.
 	// STATE_PRIV.*BARRIER_COMPLETE may change after this point.
-	wait_trap_barriers(s_restore_tmp)
+	wait_trap_barriers(s_restore_tmp, 0, 0)
 
 #if HAVE_CLUSTER_BARRIER
 	// SCC is changed by wait_trap_barriers, restore it separately.
···
 L_END_PGM:
 	// Make sure that no wave of the group/cluster can exit the trap handler
 	// before the group/cluster barrier state is saved.
-	wait_trap_barriers(s_restore_tmp)
+	wait_trap_barriers(s_restore_tmp, 0, 0)
 
 	s_endpgm_saved
 end
···
 end
 #endif
 
-function wait_trap_barriers(s_tmp)
+function wait_trap_barriers(s_tmp1, s_tmp2, serialize_wa)
 #if HAVE_CLUSTER_BARRIER
 	// If not in a WG then wave cannot use s_barrier_signal_isfirst.
-	s_getreg_b32	s_tmp, hwreg(HW_REG_WAVE_STATUS)
-	s_bitcmp0_b32	s_tmp, SQ_WAVE_STATUS_IN_WG_SHIFT
+	s_getreg_b32	s_tmp1, hwreg(HW_REG_WAVE_STATUS)
+	s_bitcmp0_b32	s_tmp1, SQ_WAVE_STATUS_IN_WG_SHIFT
 	s_cbranch_scc1	L_TRAP_CLUSTER_BARRIER_SIGNAL
 
 	s_barrier_signal_isfirst	-2
···
 
 L_SKIP_TRAP_CLUSTER_BARRIER_SIGNAL:
 	s_barrier_wait	-4
+
+#if CLUSTER_BARRIER_SERIALIZE_WORKAROUND
+if serialize_wa
+	// Trap cluster barrier may complete with a user cluster barrier in-flight.
+	// This is indicated if user cluster member count and signal count are equal.
+L_WAIT_USER_CLUSTER_BARRIER_COMPLETE:
+	s_sendmsg_rtn_b32	s_tmp1, sendmsg(MSG_RTN_GET_CLUSTER_BARRIER_STATE)
+	s_wait_kmcnt	0
+	s_bitcmp0_b32	s_tmp1, BARRIER_STATE_VALID_OFFSET
+	s_cbranch_scc1	L_NOT_IN_CLUSTER
+
+	s_bfe_u32	s_tmp2, s_tmp1, (BARRIER_STATE_MEMBER_OFFSET | (BARRIER_STATE_MEMBER_SIZE << 0x10))
+	s_bfe_u32	s_tmp1, s_tmp1, (BARRIER_STATE_SIGNAL_OFFSET | (BARRIER_STATE_SIGNAL_SIZE << 0x10))
+	s_cmp_eq_u32	s_tmp1, s_tmp2
+	s_cbranch_scc1	L_WAIT_USER_CLUSTER_BARRIER_COMPLETE
+end
+L_NOT_IN_CLUSTER:
+#endif
+
 #else
 	s_barrier_signal	-2
 	s_barrier_wait	-2
 #endif
 end
 
+#if RELAXED_SCHEDULING_IN_TRAP
 function restore_sched_mode(s_tmp)
 	s_bfe_u32	s_tmp, ttmp11, (TTMP11_SCHED_MODE_SHIFT | (TTMP11_SCHED_MODE_SIZE << 0x10))
 	s_setreg_b32	hwreg(HW_REG_WAVE_SCHED_MODE), s_tmp
+end
+#endif
 
 function restore_barrier_signal_count(barrier_id)
 	// extract the saved signal count from s_restore_tmp
···
 	// ttmp[0:1]: {7b'0} PC[56:0]
 	// ttmp2, 3, 10, 13, 14, 15: free
 	s_load_b64	[ttmp14, ttmp15], [ttmp0, ttmp1], 0 scope:SCOPE_CU	// Load the 2 instruction DW we are returning to
+	s_wait_kmcnt	0
 	s_load_b64	[ttmp2, ttmp3], [ttmp0, ttmp1], 8 scope:SCOPE_CU	// Load the next 2 instruction DW, just in case
-	s_wait_kmcnt	1
 	s_and_b32	ttmp10, ttmp14, 0x80000000				// Check bit 31 in the first DWORD
 										// SCC set if ttmp10 is != 0, i.e. if bit 31 == 1
 	s_cbranch_scc1	L_FIXUP_NOT_VOP12C					// If bit 31 is 1, we are not VOP1, VOP2, or VOP3C

drivers/gpu/drm/amd/amdkfd/kfd_debug.h

···
 			&& dev->kfd->mec2_fw_version < 0x1b6) ||
 		(KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 1)
 			&& dev->kfd->mec2_fw_version < 0x30) ||
-		(KFD_GC_VERSION(dev) >= IP_VERSION(11, 0, 0) &&
-			KFD_GC_VERSION(dev) < IP_VERSION(12, 0, 0)))
+		kfd_dbg_has_cwsr_workaround(dev))
 		return false;
 
 	/* Assume debugging and cooperative launch supported otherwise. */

drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c

···
 	m->cp_hqd_queue_priority = q->priority;
 }
 
+static bool mqd_on_vram(struct amdgpu_device *adev)
+{
+	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
+	case IP_VERSION(9, 4, 3):
+	case IP_VERSION(9, 5, 0):
+		return true;
+	default:
+		return false;
+	}
+}
+
 static struct kfd_mem_obj *allocate_mqd(struct kfd_node *node,
 		struct queue_properties *q)
 {
···
 			(ALIGN(q->ctl_stack_size, PAGE_SIZE) +
 			ALIGN(sizeof(struct v9_mqd), PAGE_SIZE)) *
 			NUM_XCC(node->xcc_mask),
-			AMDGPU_GEM_DOMAIN_GTT,
+			mqd_on_vram(node->adev) ? AMDGPU_GEM_DOMAIN_VRAM :
+						  AMDGPU_GEM_DOMAIN_GTT,
 			&(mqd_mem_obj->mem),
 			&(mqd_mem_obj->gpu_addr),
 			(void *)&(mqd_mem_obj->cpu_ptr), true);
···
 			*gart_addr = xcc_gart_addr;
 		}
 	}
+
+	if (mqd_on_vram(mm->dev->adev))
+		amdgpu_device_flush_hdp(mm->dev->adev, NULL);
 }
 
 static void update_mqd_v9_4_3(struct mqd_manager *mm, void *mqd,
···
 			m->pm4_target_xcc_in_xcp = q->pm4_target_xcc;
 		}
 	}
+
+	if (mqd_on_vram(mm->dev->adev))
+		amdgpu_device_flush_hdp(mm->dev->adev, NULL);
 }
 
 static void restore_mqd_v9_4_3(struct mqd_manager *mm, void **mqd,
···
 					(uint8_t *)ctl_stack_src + xcc *  mqd_ctl_stack_size,
 					mqd_ctl_stack_size);
 	}
+
+	if (mqd_on_vram(mm->dev->adev))
+		amdgpu_device_flush_hdp(mm->dev->adev, NULL);
 }
 static int destroy_mqd_v9_4_3(struct mqd_manager *mm, void *mqd,
 		   enum kfd_preempt_type type, unsigned int timeout,

drivers/gpu/drm/amd/amdkfd/kfd_svm.c

···
 svm_range_unmap_from_gpus(struct svm_range *prange, unsigned long start,
 			  unsigned long last, uint32_t trigger)
 {
-	DECLARE_BITMAP(bitmap, MAX_GPU_INSTANCE);
 	struct kfd_process_device *pdd;
 	struct dma_fence *fence = NULL;
 	struct kfd_process *p;
···
 		prange->mapped_to_gpu = false;
 	}
 
-	bitmap_or(bitmap, prange->bitmap_access, prange->bitmap_aip,
-		  MAX_GPU_INSTANCE);
 	p = container_of(prange->svms, struct kfd_process, svms);
 
-	for_each_set_bit(gpuidx, bitmap, MAX_GPU_INSTANCE) {
+	for_each_or_bit(gpuidx, prange->bitmap_access, prange->bitmap_aip, MAX_GPU_INSTANCE) {
 		pr_debug("unmap from gpu idx 0x%x\n", gpuidx);
 		pdd = kfd_process_device_from_gpuidx(p, gpuidx);
 		if (!pdd) {

drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c

···
 		drm_dp_mst_topology_mgr_destroy(&aconnector->mst_mgr);
 
 	/* Cancel and flush any pending HDMI HPD debounce work */
-	cancel_delayed_work_sync(&aconnector->hdmi_hpd_debounce_work);
-	if (aconnector->hdmi_prev_sink) {
-		dc_sink_release(aconnector->hdmi_prev_sink);
-		aconnector->hdmi_prev_sink = NULL;
+	if (aconnector->hdmi_hpd_debounce_delay_ms) {
+		cancel_delayed_work_sync(&aconnector->hdmi_hpd_debounce_work);
+		if (aconnector->hdmi_prev_sink) {
+			dc_sink_release(aconnector->hdmi_prev_sink);
+			aconnector->hdmi_prev_sink = NULL;
+		}
 	}
 
 	if (aconnector->bl_idx != -1) {
···
 
 		if (ieeeId == HDMI_AMD_VENDOR_SPECIFIC_DATA_BLOCK_IEEE_REGISTRATION_ID &&
 				amd_vsdb->version == HDMI_AMD_VENDOR_SPECIFIC_DATA_BLOCK_VERSION_3) {
+			u8 panel_type;
 			vsdb_info->replay_mode = (amd_vsdb->feature_caps & AMD_VSDB_VERSION_3_FEATURECAP_REPLAYMODE) ? true : false;
 			vsdb_info->amd_vsdb_version = HDMI_AMD_VENDOR_SPECIFIC_DATA_BLOCK_VERSION_3;
 			drm_dbg_kms(aconnector->base.dev, "Panel supports Replay Mode: %d\n", vsdb_info->replay_mode);
+			panel_type = (amd_vsdb->color_space_eotf_support & AMD_VDSB_VERSION_3_PANEL_TYPE_MASK) >> AMD_VDSB_VERSION_3_PANEL_TYPE_SHIFT;
+			switch (panel_type) {
+			case AMD_VSDB_PANEL_TYPE_OLED:
+				aconnector->dc_link->panel_type = PANEL_TYPE_OLED;
+				break;
+			case AMD_VSDB_PANEL_TYPE_MINILED:
+				aconnector->dc_link->panel_type = PANEL_TYPE_MINILED;
+				break;
+			default:
+				aconnector->dc_link->panel_type = PANEL_TYPE_NONE;
+				break;
+			}
+			drm_dbg_kms(aconnector->base.dev, "Panel type: %d\n",
+				    aconnector->dc_link->panel_type);
 
 			return true;
 		}

drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h

···
 
 #define HDMI_AMD_VENDOR_SPECIFIC_DATA_BLOCK_IEEE_REGISTRATION_ID 0x00001A
 #define AMD_VSDB_VERSION_3_FEATURECAP_REPLAYMODE 0x40
+#define AMD_VDSB_VERSION_3_PANEL_TYPE_MASK 0xC0
+#define AMD_VDSB_VERSION_3_PANEL_TYPE_SHIFT 6
 #define HDMI_AMD_VENDOR_SPECIFIC_DATA_BLOCK_VERSION_3 0x3
+
+enum amd_vsdb_panel_type {
+	AMD_VSDB_PANEL_TYPE_DEFAULT = 0,
+	AMD_VSDB_PANEL_TYPE_MINILED,
+	AMD_VSDB_PANEL_TYPE_OLED,
+	AMD_VSDB_PANEL_TYPE_RESERVED,
+};
 
 #define AMDGPU_HDR_MULT_DEFAULT (0x100000000LL)
 
···
 	unsigned char ieee_id[3];
 	unsigned char version;
 	unsigned char feature_caps;
+	unsigned char reserved[3];
+	unsigned char color_space_eotf_support;
 };
 
 struct common_irq_params {

drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c

···
 	struct vblank_control_work *vblank_work =
 		container_of(work, struct vblank_control_work, work);
 	struct amdgpu_display_manager *dm = vblank_work->dm;
-	struct amdgpu_device *adev = drm_to_adev(dm->ddev);
-	int r;
 
 	mutex_lock(&dm->dc_lock);
 
···
 
 	if (dm->active_vblank_irq_count == 0) {
 		dc_post_update_surfaces_to_stream(dm->dc);
-
-		r = amdgpu_dpm_pause_power_profile(adev, true);
-		if (r)
-			dev_warn(adev->dev, "failed to set default power profile mode\n");
-
 		dc_allow_idle_optimizations(dm->dc, true);
-
-		r = amdgpu_dpm_pause_power_profile(adev, false);
-		if (r)
-			dev_warn(adev->dev, "failed to restore the power profile mode\n");
 	}
 
 	mutex_unlock(&dm->dc_lock);

drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c

···
 }
 
 /*
+ * IPS residency information from DMUB service. Read only.
+ *
+ * For time-window (segment) measurement:
+ *	1) echo 1 > /sys/kernel/debug/dri/0/amdgpu_dm_ips_residency_cntl
+ *	2) sleep <seconds>
+ *	3) echo 0 > /sys/kernel/debug/dri/0/amdgpu_dm_ips_residency_cntl
+ *	4) cat /sys/kernel/debug/dri/0/amdgpu_dm_ips_residency
+ */
+static int ips_residency_show(struct seq_file *m, void *unused)
+{
+	struct amdgpu_device *adev = m->private;
+	struct dc *dc = adev->dm.dc;
+	uint8_t panel_inst = 0;
+	enum ips_residency_mode mode;
+	struct dmub_ips_residency_info info;
+
+	mutex_lock(&adev->dm.dc_lock);
+
+	mode = IPS_RESIDENCY__IPS1_RCG;
+	if (!dc_dmub_srv_ips_query_residency_info(dc->ctx, panel_inst, &info, mode)) {
+		seq_printf(m, "ISP query failed\n");
+	} else {
+		unsigned int pct, frac;
+		pct = info.residency_millipercent / 1000;
+		frac = info.residency_millipercent % 1000;
+
+		seq_printf(m, "IPS residency: %u.%03u%% \n", pct, frac);
+		seq_printf(m, "    entry_counter: %u\n", info.entry_counter);
+		seq_printf(m, "    total_time_us: %llu\n",
+			(unsigned long long)info.total_time_us);
+		seq_printf(m, "    total_inactive_time_us: %llu\n",
+			(unsigned long long)info.total_inactive_time_us);
+	}
+	mutex_unlock(&adev->dm.dc_lock);
+	return 0;
+}
+
+static int ips_residency_cntl_get(void *data, u64 *val)
+{
+	*val = 0;
+	return 0;
+}
+
+static int ips_residency_cntl_set(void *data, u64 val)
+{
+	struct amdgpu_device *adev = data;
+	struct dc *dc = adev->dm.dc;
+	uint8_t panel_inst = 0;
+	int ret = 0;
+
+	mutex_lock(&adev->dm.dc_lock);
+	if (!dc_dmub_srv_ips_residency_cntl(dc->ctx, panel_inst, !!val))
+		ret = -EIO;
+	mutex_unlock(&adev->dm.dc_lock);
+
+	return ret;
+}
+
+/*
  * Backlight at this moment.  Read only.
  * As written to display, taking ABM and backlight lut into account.
  * Ranges from 0x0 to 0x10000 (= 100% PWM)
···
 			disallow_edp_enter_psr_get,
 			disallow_edp_enter_psr_set, "%llu\n");
 
+DEFINE_DEBUGFS_ATTRIBUTE(ips_residency_cntl_fops, ips_residency_cntl_get,
+			   ips_residency_cntl_set, "%llu\n");
 DEFINE_SHOW_ATTRIBUTE(current_backlight);
 DEFINE_SHOW_ATTRIBUTE(target_backlight);
 DEFINE_SHOW_ATTRIBUTE(ips_status);
+DEFINE_SHOW_ATTRIBUTE(ips_residency);
 
 static const struct {
 	char *name;
···
 	debugfs_create_file_unsafe("amdgpu_dm_disable_hpd", 0644, root, adev,
 				   &disable_hpd_ops);
 
-	if (adev->dm.dc->caps.ips_support)
+	if (adev->dm.dc->caps.ips_support) {
 		debugfs_create_file_unsafe("amdgpu_dm_ips_status", 0644, root, adev,
 					   &ips_status_fops);
+
+		debugfs_create_file_unsafe("amdgpu_dm_ips_residency_cntl", 0644, root, adev,
+					   &ips_residency_cntl_fops);
+
+		debugfs_create_file_unsafe("amdgpu_dm_ips_residency", 0644, root, adev,
+					   &ips_residency_fops);
+	}
 }

drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c

···
 	/* TODO: something */
 }
 
+void dm_helpers_dmu_timeout(struct dc_context *ctx)
+{
+	// TODO:
+	//amdgpu_device_gpu_recover(dc_context->driver-context, NULL);
+}
+
 void dm_helpers_smu_timeout(struct dc_context *ctx, unsigned int msg_id, unsigned int param, unsigned int timeout_us)
 {
 	// TODO:

drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c

···
 		struct amdgpu_dm_connector *amdgpu_dm_connector;
 		const struct dc_link *dc_link;
 
-		use_polling |= connector->polled != DRM_CONNECTOR_POLL_HPD;
-
 		if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK)
 			continue;
 
 		amdgpu_dm_connector = to_amdgpu_dm_connector(connector);
+
+		/*
+		 * Analog connectors may be hot-plugged unlike other connector
+		 * types that don't support HPD. Only poll analog connectors.
+		 */
+		use_polling |=
+			amdgpu_dm_connector->dc_link &&
+			dc_connector_supports_analog(amdgpu_dm_connector->dc_link->link_id.id);
 
 		dc_link = amdgpu_dm_connector->dc_link;
 

drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c

···
 #undef DC_LOGGER
 #define DC_LOGGER \
 	clk_mgr->base.base.ctx->logger
-
 #define regCLK1_CLK_PLL_REQ			0x0237
 #define regCLK1_CLK_PLL_REQ_BASE_IDX		0
 
···
 #define CLK1_CLK_PLL_REQ__PllSpineDiv_MASK	0x0000F000L
 #define CLK1_CLK_PLL_REQ__FbMult_frac_MASK	0xFFFF0000L
 
-#define regCLK1_CLK0_DFS_CNTL				0x0269
-#define regCLK1_CLK0_DFS_CNTL_BASE_IDX		0
-#define regCLK1_CLK1_DFS_CNTL				0x026c
-#define regCLK1_CLK1_DFS_CNTL_BASE_IDX		0
-#define regCLK1_CLK2_DFS_CNTL				0x026f
-#define regCLK1_CLK2_DFS_CNTL_BASE_IDX		0
-#define regCLK1_CLK3_DFS_CNTL				0x0272
-#define regCLK1_CLK3_DFS_CNTL_BASE_IDX		0
-#define regCLK1_CLK4_DFS_CNTL				0x0275
-#define regCLK1_CLK4_DFS_CNTL_BASE_IDX		0
-#define regCLK1_CLK5_DFS_CNTL				0x0278
-#define regCLK1_CLK5_DFS_CNTL_BASE_IDX		0
-
-#define regCLK1_CLK0_CURRENT_CNT			0x02fb
-#define regCLK1_CLK0_CURRENT_CNT_BASE_IDX	0
-#define regCLK1_CLK1_CURRENT_CNT			0x02fc
-#define regCLK1_CLK1_CURRENT_CNT_BASE_IDX	0
-#define regCLK1_CLK2_CURRENT_CNT			0x02fd
-#define regCLK1_CLK2_CURRENT_CNT_BASE_IDX	0
-#define regCLK1_CLK3_CURRENT_CNT			0x02fe
-#define regCLK1_CLK3_CURRENT_CNT_BASE_IDX	0
-#define regCLK1_CLK4_CURRENT_CNT			0x02ff
-#define regCLK1_CLK4_CURRENT_CNT_BASE_IDX	0
-#define regCLK1_CLK5_CURRENT_CNT			0x0300
-#define regCLK1_CLK5_CURRENT_CNT_BASE_IDX	0
-
-#define regCLK1_CLK0_BYPASS_CNTL			0x028a
-#define regCLK1_CLK0_BYPASS_CNTL_BASE_IDX	0
-#define regCLK1_CLK1_BYPASS_CNTL			0x0293
-#define regCLK1_CLK1_BYPASS_CNTL_BASE_IDX	0
 #define regCLK1_CLK2_BYPASS_CNTL			0x029c
 #define regCLK1_CLK2_BYPASS_CNTL_BASE_IDX	0
-#define regCLK1_CLK3_BYPASS_CNTL			0x02a5
-#define regCLK1_CLK3_BYPASS_CNTL_BASE_IDX	0
-#define regCLK1_CLK4_BYPASS_CNTL			0x02ae
-#define regCLK1_CLK4_BYPASS_CNTL_BASE_IDX	0
-#define regCLK1_CLK5_BYPASS_CNTL			0x02b7
-#define regCLK1_CLK5_BYPASS_CNTL_BASE_IDX	0
-
-#define regCLK1_CLK0_DS_CNTL				0x0283
-#define regCLK1_CLK0_DS_CNTL_BASE_IDX		0
-#define regCLK1_CLK1_DS_CNTL				0x028c
-#define regCLK1_CLK1_DS_CNTL_BASE_IDX		0
-#define regCLK1_CLK2_DS_CNTL				0x0295
-#define regCLK1_CLK2_DS_CNTL_BASE_IDX		0
-#define regCLK1_CLK3_DS_CNTL				0x029e
-#define regCLK1_CLK3_DS_CNTL_BASE_IDX		0
-#define regCLK1_CLK4_DS_CNTL				0x02a7
-#define regCLK1_CLK4_DS_CNTL_BASE_IDX		0
-#define regCLK1_CLK5_DS_CNTL				0x02b0
-#define regCLK1_CLK5_DS_CNTL_BASE_IDX		0
-
-#define regCLK1_CLK0_ALLOW_DS				0x0284
-#define regCLK1_CLK0_ALLOW_DS_BASE_IDX		0
-#define regCLK1_CLK1_ALLOW_DS				0x028d
-#define regCLK1_CLK1_ALLOW_DS_BASE_IDX		0
-#define regCLK1_CLK2_ALLOW_DS				0x0296
-#define regCLK1_CLK2_ALLOW_DS_BASE_IDX		0
-#define regCLK1_CLK3_ALLOW_DS				0x029f
-#define regCLK1_CLK3_ALLOW_DS_BASE_IDX		0
-#define regCLK1_CLK4_ALLOW_DS				0x02a8
-#define regCLK1_CLK4_ALLOW_DS_BASE_IDX		0
-#define regCLK1_CLK5_ALLOW_DS				0x02b1
-#define regCLK1_CLK5_ALLOW_DS_BASE_IDX		0
 
 #define CLK1_CLK2_BYPASS_CNTL__CLK2_BYPASS_SEL__SHIFT	0x0
 #define CLK1_CLK2_BYPASS_CNTL__CLK2_BYPASS_DIV__SHIFT	0x10
···
 {
 	struct clk_mgr_internal *clk_mgr_int = TO_CLK_MGR_INTERNAL(clk_mgr);
 	uint32_t ref_dtbclk = clk_mgr->clks.ref_dtbclk_khz;
-	struct clk_mgr_dcn314 *clk_mgr_dcn314 = TO_CLK_MGR_DCN314(clk_mgr_int);
-	struct clk_log_info log_info = {0};
 
 	memset(&(clk_mgr->clks), 0, sizeof(struct dc_clocks));
 	// Assumption is that boot state always supports pstate
···
 			dce_adjust_dp_ref_freq_for_ss(clk_mgr_int, clk_mgr->dprefclk_khz);
 	else
 		clk_mgr->dp_dto_source_clock_in_khz = clk_mgr->dprefclk_khz;
-
-	dcn314_dump_clk_registers(&clk_mgr->boot_snapshot, &clk_mgr_dcn314->base.base, &log_info);
-	clk_mgr->clks.dispclk_khz =  clk_mgr->boot_snapshot.dispclk * 1000;
 }
 
 void dcn314_update_clocks(struct clk_mgr *clk_mgr_base,
···
 	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
 	struct dc_clocks *new_clocks = &context->bw_ctx.bw.dcn.clk;
 	struct dc *dc = clk_mgr_base->ctx->dc;
-	int display_count;
+	int display_count = 0;
 	bool update_dppclk = false;
 	bool update_dispclk = false;
 	bool dpp_clock_lowered = false;
···
 		return;
 
 	display_count = dcn314_get_active_display_cnt_wa(dc, context);
-
 	/*
 	 * if it is safe to lower, but we are already in the lower state, we don't have to do anything
 	 * also if safe to lower is false, we just go in the higher state
···
 	}
 
 	if (should_set_clock(safe_to_lower, new_clocks->dispclk_khz, clk_mgr_base->clks.dispclk_khz) &&
-	    (new_clocks->dispclk_khz > 0 || (safe_to_lower && display_count == 0))) {
+		(new_clocks->dispclk_khz > 0 || (safe_to_lower && display_count == 0))) {
 		int requested_dispclk_khz = new_clocks->dispclk_khz;
 
 		dcn314_disable_otg_wa(clk_mgr_base, context, safe_to_lower, true);
···
 
 		dcn314_smu_set_dispclk(clk_mgr, requested_dispclk_khz);
 		clk_mgr_base->clks.dispclk_khz = new_clocks->dispclk_khz;
-
 		dcn314_disable_otg_wa(clk_mgr_base, context, safe_to_lower, false);
 
 		update_dispclk = true;
···
 	return true;
 }
 
-
-static void dcn314_dump_clk_registers_internal(struct dcn35_clk_internal *internal, struct clk_mgr *clk_mgr_base)
-{
-	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
-
-	// read dtbclk
-	internal->CLK1_CLK4_CURRENT_CNT = REG_READ(CLK1_CLK4_CURRENT_CNT);
-	internal->CLK1_CLK4_BYPASS_CNTL = REG_READ(CLK1_CLK4_BYPASS_CNTL);
-
-	// read dcfclk
-	internal->CLK1_CLK3_CURRENT_CNT = REG_READ(CLK1_CLK3_CURRENT_CNT);
-	internal->CLK1_CLK3_BYPASS_CNTL = REG_READ(CLK1_CLK3_BYPASS_CNTL);
-
-	// read dcf deep sleep divider
-	internal->CLK1_CLK3_DS_CNTL = REG_READ(CLK1_CLK3_DS_CNTL);
-	internal->CLK1_CLK3_ALLOW_DS = REG_READ(CLK1_CLK3_ALLOW_DS);
-
-	// read dppclk
-	internal->CLK1_CLK1_CURRENT_CNT = REG_READ(CLK1_CLK1_CURRENT_CNT);
-	internal->CLK1_CLK1_BYPASS_CNTL = REG_READ(CLK1_CLK1_BYPASS_CNTL);
-
-	// read dprefclk
-	internal->CLK1_CLK2_CURRENT_CNT = REG_READ(CLK1_CLK2_CURRENT_CNT);
-	internal->CLK1_CLK2_BYPASS_CNTL = REG_READ(CLK1_CLK2_BYPASS_CNTL);
-
-	// read dispclk
-	internal->CLK1_CLK0_CURRENT_CNT = REG_READ(CLK1_CLK0_CURRENT_CNT);
-	internal->CLK1_CLK0_BYPASS_CNTL = REG_READ(CLK1_CLK0_BYPASS_CNTL);
-}
-
-void dcn314_dump_clk_registers(struct clk_state_registers_and_bypass *regs_and_bypass,
+static void dcn314_dump_clk_registers(struct clk_state_registers_and_bypass *regs_and_bypass,
 		struct clk_mgr *clk_mgr_base, struct clk_log_info *log_info)
 {
-
-	struct dcn35_clk_internal internal = {0};
-
-	dcn314_dump_clk_registers_internal(&internal, clk_mgr_base);
-
-	regs_and_bypass->dcfclk = internal.CLK1_CLK3_CURRENT_CNT / 10;
-	regs_and_bypass->dcf_deep_sleep_divider = internal.CLK1_CLK3_DS_CNTL / 10;
-	regs_and_bypass->dcf_deep_sleep_allow = internal.CLK1_CLK3_ALLOW_DS;
-	regs_and_bypass->dprefclk = internal.CLK1_CLK2_CURRENT_CNT / 10;
-	regs_and_bypass->dispclk = internal.CLK1_CLK0_CURRENT_CNT / 10;
-	regs_and_bypass->dppclk = internal.CLK1_CLK1_CURRENT_CNT / 10;
-	regs_and_bypass->dtbclk = internal.CLK1_CLK4_CURRENT_CNT / 10;
-
-	regs_and_bypass->dppclk_bypass = internal.CLK1_CLK1_BYPASS_CNTL & 0x0007;
-	if (regs_and_bypass->dppclk_bypass > 4)
-		regs_and_bypass->dppclk_bypass = 0;
-	regs_and_bypass->dcfclk_bypass = internal.CLK1_CLK3_BYPASS_CNTL & 0x0007;
-	if (regs_and_bypass->dcfclk_bypass > 4)
-		regs_and_bypass->dcfclk_bypass = 0;
-	regs_and_bypass->dispclk_bypass = internal.CLK1_CLK0_BYPASS_CNTL & 0x0007;
-	if (regs_and_bypass->dispclk_bypass > 4)
-		regs_and_bypass->dispclk_bypass = 0;
-	regs_and_bypass->dprefclk_bypass = internal.CLK1_CLK2_BYPASS_CNTL & 0x0007;
-	if (regs_and_bypass->dprefclk_bypass > 4)
-		regs_and_bypass->dprefclk_bypass = 0;
-
+	return;
 }
 
 static struct clk_bw_params dcn314_bw_params = {

drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.h

···
 
 void dcn314_clk_mgr_destroy(struct clk_mgr_internal *clk_mgr_int);
 
-
-void dcn314_dump_clk_registers(struct clk_state_registers_and_bypass *regs_and_bypass,
-		struct clk_mgr *clk_mgr_base, struct clk_log_info *log_info);
-
-
 #endif //__DCN314_CLK_MGR_H__

drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c

···
 #include "dm_helpers.h"
 
 #include "dc_dmub_srv.h"
-#include "reg_helper.h"
+
 #include "logger_types.h"
 #undef DC_LOGGER
 #define DC_LOGGER \
···
 
 #include "link_service.h"
 
-#define MAX_INSTANCE                                        7
-#define MAX_SEGMENT                                         8
-
-struct IP_BASE_INSTANCE {
-	unsigned int segment[MAX_SEGMENT];
-};
-
-struct IP_BASE {
-	struct IP_BASE_INSTANCE instance[MAX_INSTANCE];
-};
-
-static const struct IP_BASE CLK_BASE = { { { { 0x00016C00, 0x02401800, 0, 0, 0, 0, 0, 0 } },
-					{ { 0x00016E00, 0x02401C00, 0, 0, 0, 0, 0, 0 } },
-					{ { 0x00017000, 0x02402000, 0, 0, 0, 0, 0, 0 } },
-					{ { 0x00017200, 0x02402400, 0, 0, 0, 0, 0, 0 } },
-					{ { 0x0001B000, 0x0242D800, 0, 0, 0, 0, 0, 0 } },
-					{ { 0x0001B200, 0x0242DC00, 0, 0, 0, 0, 0, 0 } } } };
-
-#define regCLK1_CLK0_CURRENT_CNT			0x0314
-#define regCLK1_CLK0_CURRENT_CNT_BASE_IDX	0
-#define regCLK1_CLK1_CURRENT_CNT			0x0315
-#define regCLK1_CLK1_CURRENT_CNT_BASE_IDX	0
-#define regCLK1_CLK2_CURRENT_CNT			0x0316
-#define regCLK1_CLK2_CURRENT_CNT_BASE_IDX	0
-#define regCLK1_CLK3_CURRENT_CNT			0x0317
-#define regCLK1_CLK3_CURRENT_CNT_BASE_IDX	0
-#define regCLK1_CLK4_CURRENT_CNT			0x0318
-#define regCLK1_CLK4_CURRENT_CNT_BASE_IDX	0
-#define regCLK1_CLK5_CURRENT_CNT			0x0319
-#define regCLK1_CLK5_CURRENT_CNT_BASE_IDX	0
-
 #define TO_CLK_MGR_DCN315(clk_mgr)\
 	container_of(clk_mgr, struct clk_mgr_dcn315, base)
-
-#define REG(reg_name) \
-	(CLK_BASE.instance[0].segment[reg ## reg_name ## _BASE_IDX] + reg ## reg_name)
 
 #define UNSUPPORTED_DCFCLK 10000000
 #define MIN_DPP_DISP_CLK     100000
···
 	if (dc->work_arounds.skip_clock_update)
 		return;
 
-	clk_mgr_base->clks.zstate_support = new_clocks->zstate_support;
+	display_count = dcn315_get_active_display_cnt_wa(dc, context);
 	/*
 	 * if it is safe to lower, but we are already in the lower state, we don't have to do anything
 	 * also if safe to lower is false, we just go in the higher state
···
 		}
 		/* check that we're not already in lower */
 		if (clk_mgr_base->clks.pwr_state != DCN_PWR_STATE_LOW_POWER) {
-			display_count = dcn315_get_active_display_cnt_wa(dc, context);
 			/* if we can go lower, go lower */
 			if (display_count == 0) {
 				union display_idle_optimization_u idle_info = { 0 };
···
 	dc_wake_and_execute_dmub_cmd(dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
 }
 
-static void dcn315_dump_clk_registers_internal(struct dcn35_clk_internal *internal, struct clk_mgr *clk_mgr_base)
-{
-	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
-
-	// read dtbclk
-	internal->CLK1_CLK4_CURRENT_CNT = REG_READ(CLK1_CLK4_CURRENT_CNT);
-
-	// read dcfclk
-	internal->CLK1_CLK3_CURRENT_CNT = REG_READ(CLK1_CLK3_CURRENT_CNT);
-
-	// read dppclk
-	internal->CLK1_CLK1_CURRENT_CNT = REG_READ(CLK1_CLK1_CURRENT_CNT);
-
-	// read dprefclk
-	internal->CLK1_CLK2_CURRENT_CNT = REG_READ(CLK1_CLK2_CURRENT_CNT);
-
-	// read dispclk
-	internal->CLK1_CLK0_CURRENT_CNT = REG_READ(CLK1_CLK0_CURRENT_CNT);
-}
-
 static void dcn315_dump_clk_registers(struct clk_state_registers_and_bypass *regs_and_bypass,
 		struct clk_mgr *clk_mgr_base, struct clk_log_info *log_info)
 {
-	struct dcn35_clk_internal internal = {0};
-
-	dcn315_dump_clk_registers_internal(&internal, clk_mgr_base);
-
-	regs_and_bypass->dcfclk = internal.CLK1_CLK3_CURRENT_CNT / 10;
-	regs_and_bypass->dprefclk = internal.CLK1_CLK2_CURRENT_CNT / 10;
-	regs_and_bypass->dispclk = internal.CLK1_CLK0_CURRENT_CNT / 10;
-	regs_and_bypass->dppclk = internal.CLK1_CLK1_CURRENT_CNT / 10;
-	regs_and_bypass->dtbclk = internal.CLK1_CLK4_CURRENT_CNT / 10;
 	return;
 }
 
···
 	.get_dp_ref_clk_frequency = dce12_get_dp_ref_freq_khz,
 	.get_dtb_ref_clk_frequency = dcn31_get_dtb_ref_freq_khz,
 	.update_clocks = dcn315_update_clocks,
-	.init_clocks = dcn315_init_clocks,
+	.init_clocks = dcn31_init_clocks,
 	.enable_pme_wa = dcn315_enable_pme_wa,
 	.are_clock_states_equal = dcn31_are_clock_states_equal,
 	.notify_wm_ranges = dcn315_notify_wm_ranges
 };
 extern struct clk_mgr_funcs dcn3_fpga_funcs;
-
-void dcn315_init_clocks(struct clk_mgr *clk_mgr)
-{
-	struct clk_mgr_internal *clk_mgr_int = TO_CLK_MGR_INTERNAL(clk_mgr);
-	uint32_t ref_dtbclk = clk_mgr->clks.ref_dtbclk_khz;
-	struct clk_mgr_dcn315 *clk_mgr_dcn315 = TO_CLK_MGR_DCN315(clk_mgr_int);
-	struct clk_log_info log_info = {0};
-
-	memset(&(clk_mgr->clks), 0, sizeof(struct dc_clocks));
-	// Assumption is that boot state always supports pstate
-	clk_mgr->clks.ref_dtbclk_khz = ref_dtbclk;	// restore ref_dtbclk
-	clk_mgr->clks.p_state_change_support = true;
-	clk_mgr->clks.prev_p_state_change_support = true;
-	clk_mgr->clks.pwr_state = DCN_PWR_STATE_UNKNOWN;
-	clk_mgr->clks.zstate_support = DCN_ZSTATE_SUPPORT_UNKNOWN;
-
-	dcn315_dump_clk_registers(&clk_mgr->boot_snapshot, &clk_mgr_dcn315->base.base, &log_info);
-	clk_mgr->clks.dispclk_khz =  clk_mgr->boot_snapshot.dispclk * 1000;
-}
 
 void dcn315_clk_mgr_construct(
 		struct dc_context *ctx,
···
 	/* Saved clocks configured at boot for debug purposes */
 	dcn315_dump_clk_registers(&clk_mgr->base.base.boot_snapshot,
 				  &clk_mgr->base.base, &log_info);
-	clk_mgr->base.base.clks.dispclk_khz =  clk_mgr->base.base.boot_snapshot.dispclk * 1000;
 
 	clk_mgr->base.base.dprefclk_khz = 600000;
 	clk_mgr->base.base.dprefclk_khz = dcn315_smu_get_dpref_clk(&clk_mgr->base);

drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.h

···
 		struct pp_smu_funcs *pp_smu,
 		struct dccg *dccg);
 
-void dcn315_init_clocks(struct clk_mgr *clk_mgr);
 void dcn315_clk_mgr_destroy(struct clk_mgr_internal *clk_mgr_int);
 
 #endif //__DCN315_CLK_MGR_H__

drivers/gpu/drm/amd/display/dc/core/dc.c

···
 bool dc_get_qos_info(struct dc *dc, struct dc_qos_info *info)
 {
 	const struct dc_clocks *clk = &dc->current_state->bw_ctx.bw.dcn.clk;
+	struct memory_qos qos;
 
 	memset(info, 0, sizeof(*info));
 
-	// Check if all measurement functions are available
-	if (!dc->hwss.measure_peak_bw_mbps ||
-	    !dc->hwss.measure_avg_bw_mbps ||
-	    !dc->hwss.measure_max_latency_ns ||
-	    !dc->hwss.measure_avg_latency_ns) {
+	// Check if measurement function is available
+	if (!dc->hwss.measure_memory_qos) {
 		return false;
 	}
 
-	// Call measurement functions to get actual values
-	info->actual_peak_bw_in_mbps = dc->hwss.measure_peak_bw_mbps(dc);
-	info->actual_avg_bw_in_mbps = dc->hwss.measure_avg_bw_mbps(dc);
-	info->actual_max_latency_in_ns = dc->hwss.measure_max_latency_ns(dc);
-	info->actual_avg_latency_in_ns = dc->hwss.measure_avg_latency_ns(dc);
+	// Call unified measurement function
+	dc->hwss.measure_memory_qos(dc, &qos);
+
+	// Populate info from measured qos
+	info->actual_peak_bw_in_mbps = qos.peak_bw_mbps;
+	info->actual_avg_bw_in_mbps = qos.avg_bw_mbps;
+	info->actual_min_latency_in_ns = qos.min_latency_ns;
+	info->actual_max_latency_in_ns = qos.max_latency_ns;
+	info->actual_avg_latency_in_ns = qos.avg_latency_ns;
 	info->dcn_bandwidth_ub_in_mbps = (uint32_t)(clk->fclk_khz / 1000 * 64);
 
 	return true;

drivers/gpu/drm/amd/display/dc/dc.h

···
 struct dcn_optc_reg_state;
 struct dcn_dccg_reg_state;
 
-#define DC_VER "3.2.365"
+#define DC_VER "3.2.366"
 
 /**
  * MAX_SURFACES - representative of the upper bound of surfaces that can be piped to a single CRTC
···
 	uint32_t actual_avg_bw_in_mbps;
 	uint32_t calculated_avg_bw_in_mbps;
 	uint32_t actual_max_latency_in_ns;
+	uint32_t actual_min_latency_in_ns;
 	uint32_t qos_max_latency_ub_in_ns;
 	uint32_t actual_avg_latency_in_ns;
 	uint32_t qos_avg_latency_ub_in_ns;
···
 	uint32_t custom_psp_footer_size;
 	bool disable_deferred_minimal_transitions;
 	unsigned int num_fast_flips_to_steady_state_override;
+	bool enable_dmu_recovery;
 };
 
 
···
 	bool link_powered_externally;	// Used to bypass hardware sequencing delays when panel is powered down forcibly
 
 	struct dc_panel_config panel_config;
+	enum dc_panel_type panel_type;
 	struct phy_state phy_state;
 	uint32_t phy_transition_bitmask;
 	// BW ALLOCATON USB4 ONLY

drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c

···
 #define DC_LOGGER CTX->logger
 #define GPINT_RETRY_NUM 20
 
+#define MAX_WAIT_US 100000
+
 static void dc_dmub_srv_construct(struct dc_dmub_srv *dc_srv, struct dc *dc,
 				  struct dmub_srv *dmub)
 {
 	dc_srv->dmub = dmub;
 	dc_srv->ctx = dc->ctx;
+}
+
+static void dc_dmub_srv_handle_failure(struct dc_dmub_srv *dc_dmub_srv)
+{
+	dc_dmub_srv_log_diagnostic_data(dc_dmub_srv);
+	if (dc_dmub_srv->ctx->dc->debug.enable_dmu_recovery)
+		dm_helpers_dmu_timeout(dc_dmub_srv->ctx);
 }
 
 struct dc_dmub_srv *dc_dmub_srv_create(struct dc *dc, struct dmub_srv *dmub)
···
 	dmub = dc_dmub_srv->dmub;
 
 	do {
-		status = dmub_srv_wait_for_pending(dmub, 100000);
+		status = dmub_srv_wait_for_pending(dmub, MAX_WAIT_US);
 	} while (dc_dmub_srv->ctx->dc->debug.disable_timeout && status != DMUB_STATUS_OK);
 
 	if (status != DMUB_STATUS_OK) {
 		DC_ERROR("Error waiting for DMUB idle: status=%d\n", status);
-		dc_dmub_srv_log_diagnostic_data(dc_dmub_srv);
+		dc_dmub_srv_handle_failure(dc_dmub_srv);
 	}
 
 	return status == DMUB_STATUS_OK;
···
 	status = dmub_srv_clear_inbox0_ack(dmub);
 	if (status != DMUB_STATUS_OK) {
 		DC_ERROR("Error clearing INBOX0 ack: status=%d\n", status);
-		dc_dmub_srv_log_diagnostic_data(dc_dmub_srv);
+		dc_dmub_srv_handle_failure(dc_dmub_srv);
 	}
 }
 
···
 	struct dc_context *dc_ctx = dc_dmub_srv->ctx;
 	enum dmub_status status = DMUB_STATUS_OK;
 
-	status = dmub_srv_wait_for_inbox0_ack(dmub, 100000);
+	status = dmub_srv_wait_for_inbox0_ack(dmub, MAX_WAIT_US);
 	if (status != DMUB_STATUS_OK) {
 		DC_ERROR("Error waiting for INBOX0 HW Lock Ack\n");
-		dc_dmub_srv_log_diagnostic_data(dc_dmub_srv);
+		dc_dmub_srv_handle_failure(dc_dmub_srv);
 	}
 }
 
···
 	status = dmub_srv_send_inbox0_cmd(dmub, data);
 	if (status != DMUB_STATUS_OK) {
 		DC_ERROR("Error sending INBOX0 cmd\n");
-		dc_dmub_srv_log_diagnostic_data(dc_dmub_srv);
+		dc_dmub_srv_handle_failure(dc_dmub_srv);
 	}
 }
 
···
 	for (i = 0 ; i < count; i++) {
 		/* confirm no messages pending */
 		do {
-			status = dmub_srv_wait_for_idle(dmub, 100000);
+			status = dmub_srv_wait_for_idle(dmub, MAX_WAIT_US);
 		} while (dc_dmub_srv->ctx->dc->debug.disable_timeout && status != DMUB_STATUS_OK);
 
 		/* queue command */
···
 	if (status != DMUB_STATUS_OK) {
 		if (status != DMUB_STATUS_POWER_STATE_D3) {
 			DC_ERROR("Error starting DMUB execution: status=%d\n", status);
-			dc_dmub_srv_log_diagnostic_data(dc_dmub_srv);
+			dc_dmub_srv_handle_failure(dc_dmub_srv);
 		}
 		return false;
 	}
···
 				return false;
 
 			do {
-					status = dmub_srv_wait_for_inbox_free(dmub, 100000, count - i);
+					status = dmub_srv_wait_for_inbox_free(dmub, MAX_WAIT_US, count - i);
 			} while (dc_dmub_srv->ctx->dc->debug.disable_timeout && status != DMUB_STATUS_OK);
 
 			/* Requeue the command. */
···
 		if (status != DMUB_STATUS_OK) {
 			if (status != DMUB_STATUS_POWER_STATE_D3) {
 				DC_ERROR("Error queueing DMUB command: status=%d\n", status);
-				dc_dmub_srv_log_diagnostic_data(dc_dmub_srv);
+				dc_dmub_srv_handle_failure(dc_dmub_srv);
 			}
 			return false;
 		}
···
 	if (status != DMUB_STATUS_OK) {
 		if (status != DMUB_STATUS_POWER_STATE_D3) {
 			DC_ERROR("Error starting DMUB execution: status=%d\n", status);
-			dc_dmub_srv_log_diagnostic_data(dc_dmub_srv);
+			dc_dmub_srv_handle_failure(dc_dmub_srv);
 		}
 		return false;
 	}
···
 	// Wait for DMUB to process command
 	if (wait_type != DM_DMUB_WAIT_TYPE_NO_WAIT) {
 		do {
-			status = dmub_srv_wait_for_idle(dmub, 100000);
+			status = dmub_srv_wait_for_idle(dmub, MAX_WAIT_US);
 		} while (dc_dmub_srv->ctx->dc->debug.disable_timeout && status != DMUB_STATUS_OK);
 
 		if (status != DMUB_STATUS_OK) {
···
 					dmub->debug.timeout_info.timeout_cmd = *cmd_list;
 				dmub->debug.timeout_info.timestamp = dm_get_timestamp(dc_dmub_srv->ctx);
 			}
-			dc_dmub_srv_log_diagnostic_data(dc_dmub_srv);
+			dc_dmub_srv_handle_failure(dc_dmub_srv);
 			return false;
 		}
 

drivers/gpu/drm/amd/display/dc/dc_types.h

···
 	enum display_endpoint_type ep_type;
 };
 
+enum dc_panel_type {
+	PANEL_TYPE_NONE = 0, // UNKONWN, not determined yet
+	PANEL_TYPE_LCD = 1,
+	PANEL_TYPE_OLED = 2,
+	PANEL_TYPE_MINILED = 3,
+};
+
 enum backlight_control_type {
 	BACKLIGHT_CONTROL_PWM = 0,
 	BACKLIGHT_CONTROL_VESA_AUX = 1,

drivers/gpu/drm/amd/display/dc/dm_helpers.h

···
 
 bool dm_helpers_dmub_outbox_interrupt_control(struct dc_context *ctx, bool enable);
 
+void dm_helpers_dmu_timeout(struct dc_context *ctx);
 void dm_helpers_smu_timeout(struct dc_context *ctx, unsigned int msg_id, unsigned int param, unsigned int timeout_us);
 
 // 0x1 = Result_OK, 0xFE = Result_UnkmownCmd, 0x0 = Status_Busy

drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c

···
 	if (!is_dsc_possible)
 		goto done;
 
-	/* increase miniumum slice count to meet sink slice width limitations */
+	/* increase minimum slice count to meet sink slice width limitations */
 	min_slices_h = dc_fixpt_ceil(dc_fixpt_max(
 			dc_fixpt_div_int(dc_fixpt_from_int(pic_width), dsc_common_caps.max_slice_width), // sink min
 			dc_fixpt_from_int(min_slices_h))); // source min

drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c

···
 #include "dc_state_priv.h"
 #include "dpcd_defs.h"
 #include "dsc.h"
+#include "dc_dp_types.h"
 /* include DCE11 register header files */
 #include "dce/dce_11_0_d.h"
 #include "dce/dce_11_0_sh_mask.h"
···
 	int i;
 
 	for (i = 0; i < dc->link_count; i++) {
-		enum signal_type signal = dc->links[i]->connector_signal;
+		struct dc_link *link = dc->links[i];
+		struct link_encoder *link_enc = link->link_enc;
+		enum signal_type signal = link->connector_signal;
 
-		dc->link_srv->blank_dp_stream(dc->links[i], false);
-
+		dc->link_srv->blank_dp_stream(link, false);
 		if (signal != SIGNAL_TYPE_EDP)
 			signal = SIGNAL_TYPE_NONE;
 
-		if (dc->links[i]->ep_type == DISPLAY_ENDPOINT_PHY)
-			dc->links[i]->link_enc->funcs->disable_output(
-					dc->links[i]->link_enc, signal);
+		if (link->ep_type == DISPLAY_ENDPOINT_PHY)
+			link_enc->funcs->disable_output(link_enc, signal);
 
-		dc->links[i]->link_status.link_active = false;
-		memset(&dc->links[i]->cur_link_settings, 0,
-				sizeof(dc->links[i]->cur_link_settings));
+		if (link->fec_state == dc_link_fec_enabled) {
+			link_enc->funcs->fec_set_enable(link_enc, false);
+			link_enc->funcs->fec_set_ready(link_enc, false);
+			link->fec_state = dc_link_fec_not_ready;
+		}
+
+		link->link_status.link_active = false;
+		memset(&link->cur_link_settings, 0, sizeof(link->cur_link_settings));
 	}
 }
 

drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c

···
 	REG_WRITE(RBBMIF_TIMEOUT_DIS, 0xFFFFFFFF);
 	REG_WRITE(RBBMIF_TIMEOUT_DIS_2, 0xFFFFFFFF);
 
-	hws->funcs.dccg_init(hws);
-
 	REG_UPDATE(DCHUBBUB_GLOBAL_TIMER_CNTL, DCHUBBUB_GLOBAL_TIMER_REFDIV, 2);
 	REG_UPDATE(DCHUBBUB_GLOBAL_TIMER_CNTL, DCHUBBUB_GLOBAL_TIMER_ENABLE, 1);
 	if (REG(REFCLK_CNTL))

drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_init.c

···
 	.set_hdr_multiplier = dcn10_set_hdr_multiplier,
 	.verify_allow_pstate_change_high = dcn10_verify_allow_pstate_change_high,
 	.wait_for_blank_complete = dcn20_wait_for_blank_complete,
-	.dccg_init = dcn20_dccg_init,
 	.set_blend_lut = dcn30_set_blend_lut,
 	.set_shaper_3dlut = dcn20_set_shaper_3dlut,
 };

drivers/gpu/drm/amd/display/dc/hwss/dcn301/dcn301_init.c

···
 	.set_hdr_multiplier = dcn10_set_hdr_multiplier,
 	.verify_allow_pstate_change_high = dcn10_verify_allow_pstate_change_high,
 	.wait_for_blank_complete = dcn20_wait_for_blank_complete,
-	.dccg_init = dcn20_dccg_init,
 	.set_blend_lut = dcn30_set_blend_lut,
 	.set_shaper_3dlut = dcn20_set_shaper_3dlut,
 };

drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c

···
 	if (pipe_ctx->stream_res.tg->funcs->set_odm_bypass)
 		pipe_ctx->stream_res.tg->funcs->set_odm_bypass(
 				pipe_ctx->stream_res.tg, &pipe_ctx->stream->timing);
+	/*
+	 * TODO - convert symclk_ref_cnts for otg to a bit map to solve
+	 * the case where the same symclk is shared across multiple otg
+	 * instances
+	 */
 	if (dc_is_hdmi_tmds_signal(pipe_ctx->stream->signal))
-		pipe_ctx->stream->link->phy_state.symclk_ref_cnts.otg = 0;
+		link->phy_state.symclk_ref_cnts.otg = 0;
+
+	if (pipe_ctx->top_pipe == NULL) {
+		if (link->phy_state.symclk_state == SYMCLK_ON_TX_OFF) {
+			const struct link_hwss *link_hwss = get_link_hwss(link, &pipe_ctx->link_res);
+
+			link_hwss->disable_link_output(link, &pipe_ctx->link_res, pipe_ctx->stream->signal);
+			link->phy_state.symclk_state = SYMCLK_OFF_TX_OFF;
+		}
+	}
 
 	set_drr_and_clear_adjust_pending(pipe_ctx, pipe_ctx->stream, NULL);
 

drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_init.c

···
 	.set_hdr_multiplier = dcn10_set_hdr_multiplier,
 	.verify_allow_pstate_change_high = dcn10_verify_allow_pstate_change_high,
 	.wait_for_blank_complete = dcn20_wait_for_blank_complete,
-	.dccg_init = dcn20_dccg_init,
 	.set_blend_lut = dcn30_set_blend_lut,
 	.set_shaper_3dlut = dcn20_set_shaper_3dlut,
 	.setup_hpo_hw_control = dcn31_setup_hpo_hw_control,

drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_init.c

···
 	.set_hdr_multiplier = dcn10_set_hdr_multiplier,
 	.verify_allow_pstate_change_high = dcn10_verify_allow_pstate_change_high,
 	.wait_for_blank_complete = dcn20_wait_for_blank_complete,
-	.dccg_init = dcn20_dccg_init,
 	.set_blend_lut = dcn30_set_blend_lut,
 	.set_shaper_3dlut = dcn20_set_shaper_3dlut,
 	.setup_hpo_hw_control = dcn31_setup_hpo_hw_control,

drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_init.c

···
 	.set_hdr_multiplier = dcn10_set_hdr_multiplier,
 	.verify_allow_pstate_change_high = dcn10_verify_allow_pstate_change_high,
 	.wait_for_blank_complete = dcn20_wait_for_blank_complete,
-	.dccg_init = dcn20_dccg_init,
 	.set_mcm_luts = dcn32_set_mcm_luts,
 	.program_mall_pipe_config = dcn32_program_mall_pipe_config,
 	.update_force_pstate = dcn32_update_force_pstate,

drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_init.c

···
 	.set_hdr_multiplier = dcn10_set_hdr_multiplier,
 	.verify_allow_pstate_change_high = dcn10_verify_allow_pstate_change_high,
 	.wait_for_blank_complete = dcn20_wait_for_blank_complete,
-	.dccg_init = dcn20_dccg_init,
 	.set_mcm_luts = dcn32_set_mcm_luts,
 	.setup_hpo_hw_control = dcn35_setup_hpo_hw_control,
 	.calculate_dccg_k1_k2_values = dcn32_calculate_dccg_k1_k2_values,

drivers/gpu/drm/amd/display/dc/hwss/dcn351/dcn351_init.c

···
 	.set_hdr_multiplier = dcn10_set_hdr_multiplier,
 	.verify_allow_pstate_change_high = dcn10_verify_allow_pstate_change_high,
 	.wait_for_blank_complete = dcn20_wait_for_blank_complete,
-	.dccg_init = dcn20_dccg_init,
 	.set_mcm_luts = dcn32_set_mcm_luts,
 	.setup_hpo_hw_control = dcn35_setup_hpo_hw_control,
 	.calculate_dccg_k1_k2_values = dcn32_calculate_dccg_k1_k2_values,

drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c

···
 	const struct link_hwss *link_hwss = get_link_hwss(link, &pipe_ctx->link_res);
 	struct dc *dc = pipe_ctx->stream->ctx->dc;
 	struct dccg *dccg = dc->res_pool->dccg;
-	enum phyd32clk_clock_source phyd32clk;
+	enum phyd32clk_clock_source phyd32clk = PHYD32CLKA;
 	int dp_hpo_inst = 0;
 	unsigned int tmds_div = PIXEL_RATE_DIV_NA;
 	unsigned int unused_div = PIXEL_RATE_DIV_NA;

drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_init.c

···
 	.verify_allow_pstate_change_high = dcn10_verify_allow_pstate_change_high,
 	.verify_allow_pstate_change_high_sequence = dcn401_verify_allow_pstate_change_high_sequence,
 	.wait_for_blank_complete = dcn20_wait_for_blank_complete,
-	.dccg_init = dcn20_dccg_init,
 	.set_mcm_luts = dcn401_set_mcm_luts,
 	.program_mall_pipe_config = dcn32_program_mall_pipe_config,
 	.program_mall_pipe_config_sequence = dcn401_program_mall_pipe_config_sequence,

drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h

···
 struct dc_underflow_debug_data;
 struct dsc_optc_config;
 struct vm_system_aperture_param;
-
+struct memory_qos;
 struct subvp_pipe_control_lock_fast_params {
 	struct dc *dc;
 	bool lock;
···
 			struct dc_underflow_debug_data *out_data);
 
 	/**
-	 * measure_peak_bw_mbps - Measure actual peak bandwidth in Mbps
+	 * measure_memory_qos - Measure memory QoS metrics
 	 * @dc: DC structure
+	 * @qos: Pointer to memory_qos struct to populate with measured values
 	 *
-	 * Returns the measured peak bandwidth value in Mbps from hardware
-	 * performance counters or registers.
+	 * Populates the provided memory_qos struct with peak bandwidth, average bandwidth,
+	 * max latency, min latency, and average latency from hardware performance counters.
 	 */
-	uint32_t (*measure_peak_bw_mbps)(struct dc *dc);
-
-	/**
-	 * measure_avg_bw_mbps - Measure actual average bandwidth in Mbps
-	 * @dc: DC structure
-	 *
-	 * Returns the measured average bandwidth value in Mbps from hardware
-	 * performance counters or registers.
-	 */
-	uint32_t (*measure_avg_bw_mbps)(struct dc *dc);
-
-	/**
-	 * measure_max_latency_ns - Measure actual maximum latency in nanoseconds
-	 * @dc: DC structure
-	 *
-	 * Returns the measured maximum latency value in nanoseconds from hardware
-	 * performance counters or registers.
-	 */
-	uint32_t (*measure_max_latency_ns)(struct dc *dc);
-
-	/**
-	 * measure_avg_latency_ns - Measure actual average latency in nanoseconds
-	 * @dc: DC structure
-	 *
-	 * Returns the measured average latency value in nanoseconds from hardware
-	 * performance counters or registers.
-	 */
-	uint32_t (*measure_avg_latency_ns)(struct dc *dc);
+	void (*measure_memory_qos)(struct dc *dc, struct memory_qos *qos);
 
 };
 

drivers/gpu/drm/amd/display/dc/inc/core_types.h

···
 	int max_phyclk_mhz;
 };
 
+struct memory_qos {
+	uint32_t peak_bw_mbps;
+	uint32_t avg_bw_mbps;
+	uint32_t max_latency_ns;
+	uint32_t min_latency_ns;
+	uint32_t avg_latency_ns;
+};
+
 #endif /* _CORE_TYPES_H_ */

drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h

···
 	bool (*program_arbiter)(struct hubbub *hubbub, struct dml2_display_arb_regs *arb_regs, bool safe_to_lower);
 	void (*dchvm_init)(struct hubbub *hubbub);
 
+	/* Performance monitoring related functions */
 	struct hubbub_perfmon_funcs {
 		void (*reset)(struct hubbub *hubbub);
-		void (*start_measuring_max_memory_latency_ns)(
+		void (*start_measuring_memory_latencies)(
 				struct hubbub *hubbub);
-		uint32_t (*get_max_memory_latency_ns)(struct hubbub *hubbub,
-				uint32_t refclk_mhz, uint32_t *sample_count);
-		void (*start_measuring_average_memory_latency_ns)(
+		uint32_t (*get_memory_latencies_ns)(struct hubbub *hubbub,
+				uint32_t refclk_mhz, uint32_t *min_latency_ns,
+				uint32_t *max_latency_ns, uint32_t *avg_latency_ns);
+		void (*start_measuring_urgent_assertion_count)(
 				struct hubbub *hubbub);
-		uint32_t (*get_average_memory_latency_ns)(struct hubbub *hubbub,
-				uint32_t refclk_mhz, uint32_t *sample_count);
-		void (*start_measuring_urgent_ramp_latency_ns)(
+		bool (*get_urgent_assertion_count)(struct hubbub *hubbub,
+				uint32_t refclk_mhz,
+				uint32_t *assertion_count,
+				uint32_t *deassertion_count,
+				uint32_t *timestamp_us);
+		void (*start_measuring_urgent_ramp_latency)(
 				struct hubbub *hubbub,
 				const struct hubbub_urgent_latency_params *params);
 		uint32_t (*get_urgent_ramp_latency_ns)(struct hubbub *hubbub,
 				uint32_t refclk_mhz);
-		void (*start_measuring_unbounded_bandwidth_mbps)(
+		void (*start_measuring_unbounded_bandwidth)(
 				struct hubbub *hubbub);
 		uint32_t (*get_unbounded_bandwidth_mbps)(struct hubbub *hubbub,
 				uint32_t refclk_mhz, uint32_t *duration_ns);
-		void (*start_measuring_average_bandwidth_mbps)(
+		void (*start_measuring_in_order_bandwidth)(
 				struct hubbub *hubbub);
-		uint32_t (*get_average_bandwidth_mbps)(struct hubbub *hubbub,
+		uint32_t (*get_in_order_bandwidth_mbps)(struct hubbub *hubbub,
 				uint32_t refclk_mhz, uint32_t min_duration_ns,
 				uint32_t *duration_ns);
+		void (*start_measuring_prefetch_data_size)(
+				struct hubbub *hubbub);
+		uint32_t (*get_prefetch_data_size)(struct hubbub *hubbub);
 	} perfmon;
 
 	struct hubbub_qos_funcs {

drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_panel_replay.c

···
 		const struct dc_link *link,
 		unsigned int *inst_out)
 {
-	if (dc_is_embedded_signal(link->connector_signal)) {
-		/* TODO: just get edp link panel inst for now, fix it */
-		return dc_get_edp_link_panel_inst(dc, link, inst_out);
-	} else if (dc_is_dp_sst_signal(link->connector_signal)) {
-		/* TODO: just set to 1 for now, fix it */
-		*inst_out = 1;
-		return true;
+	if (!dc || !link || !inst_out)
+		return false;
+
+	if (!dc_is_dp_sst_signal(link->connector_signal)) /* only supoprt DP sst (eDP included) for now */
+		return false;
+
+	for (unsigned int i = 0; i < MAX_PIPES; i++) {
+		if (dc->current_state->res_ctx.pipe_ctx[i].stream &&
+			dc->current_state->res_ctx.pipe_ctx[i].stream->link == link) {
+			/* *inst_out is equal to otg number */
+			if (dc->current_state->res_ctx.pipe_ctx[i].stream_res.tg)
+				*inst_out = dc->current_state->res_ctx.pipe_ctx[i].stream_res.tg->inst;
+			else
+				*inst_out = 0;
+
+			return true;
+		}
 	}
 
 	return false;
···
 		return false;
 
 	memset(&cmd, 0, sizeof(cmd));
+	memcpy(&cmd.pr_update_state.data, update_state_data, sizeof(struct dmub_cmd_pr_update_state_data));
+
 	cmd.pr_update_state.header.type = DMUB_CMD__PR;
 	cmd.pr_update_state.header.sub_type = DMUB_CMD__PR_UPDATE_STATE;
 	cmd.pr_update_state.header.payload_bytes = sizeof(struct dmub_cmd_pr_update_state_data);
 	cmd.pr_update_state.data.panel_inst = panel_inst;
-
-	memcpy(&cmd.pr_update_state.data, update_state_data, sizeof(struct dmub_cmd_pr_update_state_data));
 
 	dc_wake_and_execute_dmub_cmd(dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
 	return true;
···
 		return false;
 
 	memset(&cmd, 0, sizeof(cmd));
+	memcpy(&cmd.pr_general_cmd.data, general_cmd_data, sizeof(struct dmub_cmd_pr_general_cmd_data));
+
 	cmd.pr_general_cmd.header.type = DMUB_CMD__PR;
 	cmd.pr_general_cmd.header.sub_type = DMUB_CMD__PR_GENERAL_CMD;
 	cmd.pr_general_cmd.header.payload_bytes = sizeof(struct dmub_cmd_pr_general_cmd_data);
 	cmd.pr_general_cmd.data.panel_inst = panel_inst;
-
-	memcpy(&cmd.pr_general_cmd.data, general_cmd_data, sizeof(struct dmub_cmd_pr_general_cmd_data));
 
 	dc_wake_and_execute_dmub_cmd(dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
 	return true;

drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c

···
 #include "dce/dmub_replay.h"
 #include "abm.h"
 #include "resource.h"
+#include "link_dp_panel_replay.h"
 #define DC_LOGGER \
 	link->ctx->logger
 #define DC_LOGGER_INIT(logger)
···
 	if (replay == NULL && force_static)
 		return false;
 
-	if (!dc_get_edp_link_panel_inst(dc, link, &panel_inst))
+	if (!dp_pr_get_panel_inst(dc, link, &panel_inst))
 		return false;
 
 	/* Set power optimization flag */
···
 	unsigned int panel_inst;
 	enum replay_state pr_state = REPLAY_STATE_0;
 
-	if (!dc_get_edp_link_panel_inst(dc, link, &panel_inst))
+	if (!dp_pr_get_panel_inst(dc, link, &panel_inst))
 		return false;
 
 	if (replay != NULL && link->replay_settings.replay_feature_enabled)
···
 	if (!replay)
 		return false;
 
-	if (!dc_get_edp_link_panel_inst(dc, link, &panel_inst))
+	if (!dp_pr_get_panel_inst(dc, link, &panel_inst))
 		return false;
 
 	replay_context.aux_inst = link->ddc->ddc_pin->hw_info.ddc_channel;
···
 
 	DC_LOGGER_INIT(link->ctx->logger);
 
-	if (dc_get_edp_link_panel_inst(dc, link, &panel_inst))
+	if (dp_pr_get_panel_inst(dc, link, &panel_inst))
 		cmd_data->panel_inst = panel_inst;
 	else {
 		DC_LOG_DC("%s(): get edp panel inst fail ", __func__);
···
 	if (!replay)
 		return false;
 
-	if (!dc_get_edp_link_panel_inst(dc, link, &panel_inst))
+	if (!dp_pr_get_panel_inst(dc, link, &panel_inst))
 		return false;
 
 	if (coasting_vtotal && (link->replay_settings.coasting_vtotal != coasting_vtotal ||
···
 	struct dmub_replay *replay = dc->res_pool->replay;
 	unsigned int panel_inst;
 
-	if (!dc_get_edp_link_panel_inst(dc, link, &panel_inst))
+	if (!dp_pr_get_panel_inst(dc, link, &panel_inst))
 		return false;
 
 	if (!residency)
···
 	struct dmub_replay *replay = dc->res_pool->replay;
 	unsigned int panel_inst;
 
-	if (!dc_get_edp_link_panel_inst(dc, link, &panel_inst))
+	if (!dp_pr_get_panel_inst(dc, link, &panel_inst))
 		return false;
 
 	/* Only both power and coasting vtotal changed, this func could return true */

drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h

···
 		uint32_t lock_dig: 1;
 		uint32_t triple_buffer_lock: 1;
 
-		uint32_t lock: 1;				/**< Lock */
+		uint32_t lock: 1;			/**< Lock */
 		uint32_t should_release: 1;		/**< Release */
-		uint32_t reserved: 7; 			/**< Reserved for extending more clients, HW, etc. */
+		uint32_t reserved: 7;			/**< Reserved for extending more clients, HW, etc. */
 	} bits;
 	uint32_t all;
 };
···
 
 union dmub_cmd_fams2_config {
 	struct dmub_cmd_fams2_global_config global;
-// coverity[cert_dcl37_c_violation:FALSE]  errno.h, stddef.h, stdint.h not included in atombios.h
 	struct dmub_fams2_stream_static_state stream; //v0
 	union {
 		struct dmub_fams2_cmd_stream_static_base_state base;
···
 	/**
 	 * @ rate_control_caps : Indicate FreeSync PSR Sink Capabilities
 	 */
-	uint8_t rate_control_caps ;
+	uint8_t rate_control_caps;
 	/*
 	 * Force PSRSU always doing full frame update
 	 */
···
 	 */
 	DMUB_CMD__LSDMA_LINEAR_COPY = 1,
 	/**
-	* LSDMA copies data from source to destination linearly in sub window
-	*/
+	 * LSDMA copies data from source to destination linearly in sub window
+	 */
 	DMUB_CMD__LSDMA_LINEAR_SUB_WINDOW_COPY = 2,
 	/**
 	 * Send the tiled-to-tiled copy command

drivers/gpu/drm/amd/display/modules/power/power_helpers.c

···
 	uint32_t *frame_skip_number_array = NULL;
 	uint32_t frame_skip_number = 0;
 
+	if (link == NULL)
+		return;
+
 	if (false == link->replay_settings.config.frame_skip_supported)
 		return;
 
-	if (link == NULL || flicker_free_refresh_rate_mhz == 0 || coasting_vtotal_refresh_rate_mhz == 0)
+	if (flicker_free_refresh_rate_mhz == 0 || coasting_vtotal_refresh_rate_mhz == 0)
 		return;
 
 	if (is_defer)

drivers/gpu/drm/amd/pm/amdgpu_dpm.c

···
 	int ret = 0;
 
 	if (!pp_funcs->get_power_limit)
-		return -ENODATA;
+		return -EOPNOTSUPP;
 
 	mutex_lock(&adev->pm.mutex);
 	ret = pp_funcs->get_power_limit(adev->powerplay.pp_handle,

drivers/gpu/drm/amd/pm/amdgpu_pm.c

···
 	     attr == &sensor_dev_attr_temp3_emergency.dev_attr.attr))
 		return 0;
 
-	/* only Vangogh has fast PPT limit and power labels */
+	/* only a few GPUs have fast PPT limit and power labels */
 	if ((attr == &sensor_dev_attr_power2_cap_max.dev_attr.attr ||
 	     attr == &sensor_dev_attr_power2_cap_min.dev_attr.attr ||
 	     attr == &sensor_dev_attr_power2_cap.dev_attr.attr ||

drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c

···
 		if (scaling_factor == 0)
 			return -EINVAL;
 
-		memset(smc_table, 0, sizeof(SISLANDS_SMC_STATETABLE));
-
 		ret = si_calculate_adjusted_tdp_limits(adev,
 						       false, /* ??? */
 						       adev->pm.dpm.tdp_adjustment,
···
 						       &near_tdp_limit);
 		if (ret)
 			return ret;
+
+		if (adev->pdev->device == 0x6611 && adev->pdev->revision == 0x87) {
+			/* Workaround buggy powertune on Radeon 430 and 520. */
+			tdp_limit = 32;
+			near_tdp_limit = 28;
+		}
 
 		smc_table->dpm2Params.TDPLimit =
 			cpu_to_be32(si_scale_power_for_smc(tdp_limit, scaling_factor) * 1000);
···
 
 	if (ni_pi->enable_power_containment) {
 		SISLANDS_SMC_STATETABLE *smc_table = &si_pi->smc_statetable;
-		u32 scaling_factor = si_get_smc_power_scaling_factor(adev);
 		int ret;
-
-		memset(smc_table, 0, sizeof(SISLANDS_SMC_STATETABLE));
-
-		smc_table->dpm2Params.NearTDPLimit =
-			cpu_to_be32(si_scale_power_for_smc(adev->pm.dpm.near_tdp_limit_adjusted, scaling_factor) * 1000);
-		smc_table->dpm2Params.SafePowerLimit =
-			cpu_to_be32(si_scale_power_for_smc((adev->pm.dpm.near_tdp_limit_adjusted * SISLANDS_DPM2_TDP_SAFE_LIMIT_PERCENT) / 100, scaling_factor) * 1000);
 
 		ret = amdgpu_si_copy_bytes_to_smc(adev,
 						  (si_pi->state_table_start +
···
 		    (adev->pdev->revision == 0x80) ||
 		    (adev->pdev->revision == 0x81) ||
 		    (adev->pdev->revision == 0x83) ||
-		    (adev->pdev->revision == 0x87) ||
+		    (adev->pdev->revision == 0x87 &&
+				adev->pdev->device != 0x6611) ||
 		    (adev->pdev->device == 0x6604) ||
 		    (adev->pdev->device == 0x6605)) {
 			max_sclk = 75000;
+		} else if (adev->pdev->revision == 0x87 &&
+				adev->pdev->device == 0x6611) {
+			/* Radeon 430 and 520 */
+			max_sclk = 78000;
 		}
 	}
 
···
 		case AMDGPU_IRQ_STATE_DISABLE:
 			cg_thermal_int = RREG32_SMC(mmCG_THERMAL_INT);
 			cg_thermal_int |= CG_THERMAL_INT__THERM_INT_MASK_HIGH_MASK;
-			WREG32_SMC(mmCG_THERMAL_INT, cg_thermal_int);
+			WREG32(mmCG_THERMAL_INT, cg_thermal_int);
 			break;
 		case AMDGPU_IRQ_STATE_ENABLE:
 			cg_thermal_int = RREG32_SMC(mmCG_THERMAL_INT);
 			cg_thermal_int &= ~CG_THERMAL_INT__THERM_INT_MASK_HIGH_MASK;
-			WREG32_SMC(mmCG_THERMAL_INT, cg_thermal_int);
+			WREG32(mmCG_THERMAL_INT, cg_thermal_int);
 			break;
 		default:
 			break;
···
 		case AMDGPU_IRQ_STATE_DISABLE:
 			cg_thermal_int = RREG32_SMC(mmCG_THERMAL_INT);
 			cg_thermal_int |= CG_THERMAL_INT__THERM_INT_MASK_LOW_MASK;
-			WREG32_SMC(mmCG_THERMAL_INT, cg_thermal_int);
+			WREG32(mmCG_THERMAL_INT, cg_thermal_int);
 			break;
 		case AMDGPU_IRQ_STATE_ENABLE:
 			cg_thermal_int = RREG32_SMC(mmCG_THERMAL_INT);
 			cg_thermal_int &= ~CG_THERMAL_INT__THERM_INT_MASK_LOW_MASK;
-			WREG32_SMC(mmCG_THERMAL_INT, cg_thermal_int);
+			WREG32(mmCG_THERMAL_INT, cg_thermal_int);
 			break;
 		default:
 			break;

drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_12_ppt.c

···
 			gpu_metrics->gfx_below_host_limit_total_acc
 				[i] = SMUQ10_ROUND(
 				metrics->GfxclkBelowHostLimitTotalAcc[inst]);
-		};
+		}
 	}
 
 	gpu_metrics->xgmi_link_width = metrics->XgmiWidth;

drivers/gpu/drm/radeon/radeon_vce.c

···
 	if (sscanf(c, "%2u]", &rdev->vce.fb_version) != 1)
 		return -EINVAL;
 
-	DRM_INFO("Found VCE firmware/feedback version %d.%d.%d / %d!\n",
-		 start, mid, end, rdev->vce.fb_version);
+	drm_info(&rdev->ddev,
+			"Found VCE firmware/feedback version %d.%d.%d / %d!\n",
+			start, mid, end, rdev->vce.fb_version);
 
 	rdev->vce.fw_version = (start << 24) | (mid << 16) | (end << 8);