tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0 OR MIT */
2/*
3 * Copyright 2014-2022 Advanced Micro Devices, Inc.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21 * OTHER DEALINGS IN THE SOFTWARE.
22 *
23 */
24
25#ifndef KFD_DEVICE_QUEUE_MANAGER_H_
26#define KFD_DEVICE_QUEUE_MANAGER_H_
27
28#include <linux/rwsem.h>
29#include <linux/list.h>
30#include <linux/mutex.h>
31#include <linux/sched/mm.h>
32#include "kfd_priv.h"
33#include "kfd_mqd_manager.h"
34
35
36#define VMID_NUM 16
37
38struct device_process_node {
39 struct qcm_process_device *qpd;
40 struct list_head list;
41};
42
43union SQ_CMD_BITS {
44 struct {
45 uint32_t cmd:3;
46 uint32_t:1;
47 uint32_t mode:3;
48 uint32_t check_vmid:1;
49 uint32_t trap_id:3;
50 uint32_t:5;
51 uint32_t wave_id:4;
52 uint32_t simd_id:2;
53 uint32_t:2;
54 uint32_t queue_id:3;
55 uint32_t:1;
56 uint32_t vm_id:4;
57 } bitfields, bits;
58 uint32_t u32All;
59 signed int i32All;
60 float f32All;
61};
62
63union GRBM_GFX_INDEX_BITS {
64 struct {
65 uint32_t instance_index:8;
66 uint32_t sh_index:8;
67 uint32_t se_index:8;
68 uint32_t:5;
69 uint32_t sh_broadcast_writes:1;
70 uint32_t instance_broadcast_writes:1;
71 uint32_t se_broadcast_writes:1;
72 } bitfields, bits;
73 uint32_t u32All;
74 signed int i32All;
75 float f32All;
76};
77
78/**
79 * struct device_queue_manager_ops
80 *
81 * @create_queue: Queue creation routine.
82 *
83 * @destroy_queue: Queue destruction routine.
84 *
85 * @update_queue: Queue update routine.
86 *
87 * @exeute_queues: Dispatches the queues list to the H/W.
88 *
89 * @register_process: This routine associates a specific process with device.
90 *
91 * @unregister_process: destroys the associations between process to device.
92 *
93 * @initialize: Initializes the pipelines and memory module for that device.
94 *
95 * @start: Initializes the resources/modules the device needs for queues
96 * execution. This function is called on device initialization and after the
97 * system woke up after suspension.
98 *
99 * @stop: This routine stops execution of all the active queue running on the
100 * H/W and basically this function called on system suspend.
101 *
102 * @uninitialize: Destroys all the device queue manager resources allocated in
103 * initialize routine.
104 *
105 * @create_kernel_queue: Creates kernel queue. Used for debug queue.
106 *
107 * @destroy_kernel_queue: Destroys kernel queue. Used for debug queue.
108 *
109 * @set_cache_memory_policy: Sets memory policy (cached/ non cached) for the
110 * memory apertures.
111 *
112 * @process_termination: Clears all process queues belongs to that device.
113 *
114 * @evict_process_queues: Evict all active queues of a process
115 *
116 * @restore_process_queues: Restore all evicted queues of a process
117 *
118 * @get_wave_state: Retrieves context save state and optionally copies the
119 * control stack, if kept in the MQD, to the given userspace address.
120 *
121 * @reset_queues: reset queues which consume RAS poison
122 * @get_queue_checkpoint_info: Retrieves queue size information for CRIU checkpoint.
123 *
124 * @checkpoint_mqd: checkpoint queue MQD contents for CRIU.
125 */
126
127struct device_queue_manager_ops {
128 int (*create_queue)(struct device_queue_manager *dqm,
129 struct queue *q,
130 struct qcm_process_device *qpd,
131 const struct kfd_criu_queue_priv_data *qd,
132 const void *restore_mqd,
133 const void *restore_ctl_stack);
134
135 int (*destroy_queue)(struct device_queue_manager *dqm,
136 struct qcm_process_device *qpd,
137 struct queue *q);
138
139 int (*update_queue)(struct device_queue_manager *dqm,
140 struct queue *q, struct mqd_update_info *minfo);
141
142 int (*register_process)(struct device_queue_manager *dqm,
143 struct qcm_process_device *qpd);
144
145 int (*unregister_process)(struct device_queue_manager *dqm,
146 struct qcm_process_device *qpd);
147
148 int (*initialize)(struct device_queue_manager *dqm);
149 int (*start)(struct device_queue_manager *dqm);
150 int (*stop)(struct device_queue_manager *dqm);
151 void (*pre_reset)(struct device_queue_manager *dqm);
152 void (*uninitialize)(struct device_queue_manager *dqm);
153 int (*create_kernel_queue)(struct device_queue_manager *dqm,
154 struct kernel_queue *kq,
155 struct qcm_process_device *qpd);
156
157 void (*destroy_kernel_queue)(struct device_queue_manager *dqm,
158 struct kernel_queue *kq,
159 struct qcm_process_device *qpd);
160
161 bool (*set_cache_memory_policy)(struct device_queue_manager *dqm,
162 struct qcm_process_device *qpd,
163 enum cache_policy default_policy,
164 enum cache_policy alternate_policy,
165 void __user *alternate_aperture_base,
166 uint64_t alternate_aperture_size);
167
168 int (*process_termination)(struct device_queue_manager *dqm,
169 struct qcm_process_device *qpd);
170
171 int (*evict_process_queues)(struct device_queue_manager *dqm,
172 struct qcm_process_device *qpd);
173 int (*restore_process_queues)(struct device_queue_manager *dqm,
174 struct qcm_process_device *qpd);
175
176 int (*get_wave_state)(struct device_queue_manager *dqm,
177 struct queue *q,
178 void __user *ctl_stack,
179 u32 *ctl_stack_used_size,
180 u32 *save_area_used_size);
181
182 int (*reset_queues)(struct device_queue_manager *dqm,
183 uint16_t pasid);
184 void (*get_queue_checkpoint_info)(struct device_queue_manager *dqm,
185 const struct queue *q, u32 *mqd_size,
186 u32 *ctl_stack_size);
187
188 int (*checkpoint_mqd)(struct device_queue_manager *dqm,
189 const struct queue *q,
190 void *mqd,
191 void *ctl_stack);
192};
193
194struct device_queue_manager_asic_ops {
195 int (*update_qpd)(struct device_queue_manager *dqm,
196 struct qcm_process_device *qpd);
197 bool (*set_cache_memory_policy)(struct device_queue_manager *dqm,
198 struct qcm_process_device *qpd,
199 enum cache_policy default_policy,
200 enum cache_policy alternate_policy,
201 void __user *alternate_aperture_base,
202 uint64_t alternate_aperture_size);
203 void (*init_sdma_vm)(struct device_queue_manager *dqm,
204 struct queue *q,
205 struct qcm_process_device *qpd);
206 struct mqd_manager * (*mqd_manager_init)(enum KFD_MQD_TYPE type,
207 struct kfd_dev *dev);
208};
209
210/**
211 * struct device_queue_manager
212 *
213 * This struct is a base class for the kfd queues scheduler in the
214 * device level. The device base class should expose the basic operations
215 * for queue creation and queue destruction. This base class hides the
216 * scheduling mode of the driver and the specific implementation of the
217 * concrete device. This class is the only class in the queues scheduler
218 * that configures the H/W.
219 *
220 */
221
222struct device_queue_manager {
223 struct device_queue_manager_ops ops;
224 struct device_queue_manager_asic_ops asic_ops;
225
226 struct mqd_manager *mqd_mgrs[KFD_MQD_TYPE_MAX];
227 struct packet_manager packet_mgr;
228 struct kfd_dev *dev;
229 struct mutex lock_hidden; /* use dqm_lock/unlock(dqm) */
230 struct list_head queues;
231 unsigned int saved_flags;
232 unsigned int processes_count;
233 unsigned int active_queue_count;
234 unsigned int active_cp_queue_count;
235 unsigned int gws_queue_count;
236 unsigned int total_queue_count;
237 unsigned int next_pipe_to_allocate;
238 unsigned int *allocated_queues;
239 uint64_t sdma_bitmap;
240 uint64_t xgmi_sdma_bitmap;
241 /* the pasid mapping for each kfd vmid */
242 uint16_t vmid_pasid[VMID_NUM];
243 uint64_t pipelines_addr;
244 uint64_t fence_gpu_addr;
245 uint64_t *fence_addr;
246 struct kfd_mem_obj *fence_mem;
247 bool active_runlist;
248 int sched_policy;
249
250 /* hw exception */
251 bool is_hws_hang;
252 bool is_resetting;
253 struct work_struct hw_exception_work;
254 struct kfd_mem_obj hiq_sdma_mqd;
255 bool sched_running;
256};
257
258void device_queue_manager_init_cik(
259 struct device_queue_manager_asic_ops *asic_ops);
260void device_queue_manager_init_cik_hawaii(
261 struct device_queue_manager_asic_ops *asic_ops);
262void device_queue_manager_init_vi(
263 struct device_queue_manager_asic_ops *asic_ops);
264void device_queue_manager_init_vi_tonga(
265 struct device_queue_manager_asic_ops *asic_ops);
266void device_queue_manager_init_v9(
267 struct device_queue_manager_asic_ops *asic_ops);
268void device_queue_manager_init_v10_navi10(
269 struct device_queue_manager_asic_ops *asic_ops);
270void program_sh_mem_settings(struct device_queue_manager *dqm,
271 struct qcm_process_device *qpd);
272unsigned int get_cp_queues_num(struct device_queue_manager *dqm);
273unsigned int get_queues_per_pipe(struct device_queue_manager *dqm);
274unsigned int get_pipes_per_mec(struct device_queue_manager *dqm);
275unsigned int get_num_sdma_queues(struct device_queue_manager *dqm);
276unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm);
277
278static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd)
279{
280 return (pdd->lds_base >> 16) & 0xFF;
281}
282
283static inline unsigned int
284get_sh_mem_bases_nybble_64(struct kfd_process_device *pdd)
285{
286 return (pdd->lds_base >> 60) & 0x0E;
287}
288
289/* The DQM lock can be taken in MMU notifiers. Make sure no reclaim-FS
290 * happens while holding this lock anywhere to prevent deadlocks when
291 * an MMU notifier runs in reclaim-FS context.
292 */
293static inline void dqm_lock(struct device_queue_manager *dqm)
294{
295 mutex_lock(&dqm->lock_hidden);
296 dqm->saved_flags = memalloc_noreclaim_save();
297}
298static inline void dqm_unlock(struct device_queue_manager *dqm)
299{
300 memalloc_noreclaim_restore(dqm->saved_flags);
301 mutex_unlock(&dqm->lock_hidden);
302}
303
304static inline int read_sdma_queue_counter(uint64_t __user *q_rptr, uint64_t *val)
305{
306 /* SDMA activity counter is stored at queue's RPTR + 0x8 location. */
307 return get_user(*val, q_rptr + 1);
308}
309#endif /* KFD_DEVICE_QUEUE_MANAGER_H_ */