]>
Commit | Line | Data |
---|---|---|
e60f8db5 AX |
1 | /* |
2 | * Copyright 2016 Advanced Micro Devices, Inc. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | * | |
22 | */ | |
f867723b | 23 | |
e60f8db5 | 24 | #include <linux/firmware.h> |
f867723b SR |
25 | #include <linux/pci.h> |
26 | ||
fd5fd480 | 27 | #include <drm/drm_cache.h> |
f867723b | 28 | |
e60f8db5 AX |
29 | #include "amdgpu.h" |
30 | #include "gmc_v9_0.h" | |
8d6a5230 | 31 | #include "amdgpu_atomfirmware.h" |
2cddc50e | 32 | #include "amdgpu_gem.h" |
e60f8db5 | 33 | |
cde5c34f | 34 | #include "gc/gc_9_0_sh_mask.h" |
135d4b10 FX |
35 | #include "dce/dce_12_0_offset.h" |
36 | #include "dce/dce_12_0_sh_mask.h" | |
fb960bd2 | 37 | #include "vega10_enum.h" |
65417d9f | 38 | #include "mmhub/mmhub_1_0_offset.h" |
ea930000 | 39 | #include "athub/athub_1_0_sh_mask.h" |
6ce68225 | 40 | #include "athub/athub_1_0_offset.h" |
250b4228 | 41 | #include "oss/osssys_4_0_offset.h" |
e60f8db5 | 42 | |
946a4d5b | 43 | #include "soc15.h" |
ea930000 | 44 | #include "soc15d.h" |
e60f8db5 | 45 | #include "soc15_common.h" |
90c7a935 | 46 | #include "umc/umc_6_0_sh_mask.h" |
e60f8db5 | 47 | |
e60f8db5 AX |
48 | #include "gfxhub_v1_0.h" |
49 | #include "mmhub_v1_0.h" | |
bee7b51a | 50 | #include "athub_v1_0.h" |
bf0a60b7 | 51 | #include "gfxhub_v1_1.h" |
51cce480 | 52 | #include "mmhub_v9_4.h" |
85e39550 | 53 | #include "mmhub_v1_7.h" |
5b6b35aa | 54 | #include "umc_v6_1.h" |
e7da754b | 55 | #include "umc_v6_0.h" |
e60f8db5 | 56 | |
44a99b65 AG |
57 | #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h" |
58 | ||
791c4769 | 59 | #include "amdgpu_ras.h" |
029fbd43 | 60 | #include "amdgpu_xgmi.h" |
791c4769 | 61 | |
ebdef28e AD |
62 | /* add these here since we already include dce12 headers and these are for DCN */ |
63 | #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION 0x055d | |
64 | #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX 2 | |
65 | #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH__SHIFT 0x0 | |
66 | #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT__SHIFT 0x10 | |
67 | #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH_MASK 0x00003FFFL | |
68 | #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT_MASK 0x3FFF0000L | |
f8646661 AD |
69 | #define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0 0x049d |
70 | #define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0_BASE_IDX 2 | |
71 | ||
ebdef28e | 72 | |
be99ecbf AD |
73 | static const char *gfxhub_client_ids[] = { |
74 | "CB", | |
75 | "DB", | |
76 | "IA", | |
77 | "WD", | |
78 | "CPF", | |
79 | "CPC", | |
80 | "CPG", | |
81 | "RLC", | |
82 | "TCP", | |
83 | "SQC (inst)", | |
84 | "SQC (data)", | |
85 | "SQG", | |
86 | "PA", | |
87 | }; | |
88 | ||
02f23f5f AD |
89 | static const char *mmhub_client_ids_raven[][2] = { |
90 | [0][0] = "MP1", | |
91 | [1][0] = "MP0", | |
92 | [2][0] = "VCN", | |
93 | [3][0] = "VCNU", | |
94 | [4][0] = "HDP", | |
95 | [5][0] = "DCE", | |
96 | [13][0] = "UTCL2", | |
97 | [19][0] = "TLS", | |
98 | [26][0] = "OSS", | |
99 | [27][0] = "SDMA0", | |
100 | [0][1] = "MP1", | |
101 | [1][1] = "MP0", | |
102 | [2][1] = "VCN", | |
103 | [3][1] = "VCNU", | |
104 | [4][1] = "HDP", | |
105 | [5][1] = "XDP", | |
106 | [6][1] = "DBGU0", | |
107 | [7][1] = "DCE", | |
108 | [8][1] = "DCEDWB0", | |
109 | [9][1] = "DCEDWB1", | |
110 | [26][1] = "OSS", | |
111 | [27][1] = "SDMA0", | |
112 | }; | |
113 | ||
114 | static const char *mmhub_client_ids_renoir[][2] = { | |
115 | [0][0] = "MP1", | |
116 | [1][0] = "MP0", | |
117 | [2][0] = "HDP", | |
118 | [4][0] = "DCEDMC", | |
119 | [5][0] = "DCEVGA", | |
120 | [13][0] = "UTCL2", | |
121 | [19][0] = "TLS", | |
122 | [26][0] = "OSS", | |
123 | [27][0] = "SDMA0", | |
124 | [28][0] = "VCN", | |
125 | [29][0] = "VCNU", | |
126 | [30][0] = "JPEG", | |
127 | [0][1] = "MP1", | |
128 | [1][1] = "MP0", | |
129 | [2][1] = "HDP", | |
130 | [3][1] = "XDP", | |
131 | [6][1] = "DBGU0", | |
132 | [7][1] = "DCEDMC", | |
133 | [8][1] = "DCEVGA", | |
134 | [9][1] = "DCEDWB", | |
135 | [26][1] = "OSS", | |
136 | [27][1] = "SDMA0", | |
137 | [28][1] = "VCN", | |
138 | [29][1] = "VCNU", | |
139 | [30][1] = "JPEG", | |
140 | }; | |
141 | ||
142 | static const char *mmhub_client_ids_vega10[][2] = { | |
143 | [0][0] = "MP0", | |
144 | [1][0] = "UVD", | |
145 | [2][0] = "UVDU", | |
146 | [3][0] = "HDP", | |
147 | [13][0] = "UTCL2", | |
148 | [14][0] = "OSS", | |
149 | [15][0] = "SDMA1", | |
150 | [32+0][0] = "VCE0", | |
151 | [32+1][0] = "VCE0U", | |
152 | [32+2][0] = "XDMA", | |
153 | [32+3][0] = "DCE", | |
154 | [32+4][0] = "MP1", | |
155 | [32+14][0] = "SDMA0", | |
156 | [0][1] = "MP0", | |
157 | [1][1] = "UVD", | |
158 | [2][1] = "UVDU", | |
159 | [3][1] = "DBGU0", | |
160 | [4][1] = "HDP", | |
161 | [5][1] = "XDP", | |
162 | [14][1] = "OSS", | |
163 | [15][1] = "SDMA0", | |
164 | [32+0][1] = "VCE0", | |
165 | [32+1][1] = "VCE0U", | |
166 | [32+2][1] = "XDMA", | |
167 | [32+3][1] = "DCE", | |
168 | [32+4][1] = "DCEDWB", | |
169 | [32+5][1] = "MP1", | |
170 | [32+6][1] = "DBGU1", | |
171 | [32+14][1] = "SDMA1", | |
172 | }; | |
173 | ||
174 | static const char *mmhub_client_ids_vega12[][2] = { | |
175 | [0][0] = "MP0", | |
176 | [1][0] = "VCE0", | |
177 | [2][0] = "VCE0U", | |
178 | [3][0] = "HDP", | |
179 | [13][0] = "UTCL2", | |
180 | [14][0] = "OSS", | |
181 | [15][0] = "SDMA1", | |
182 | [32+0][0] = "DCE", | |
183 | [32+1][0] = "XDMA", | |
184 | [32+2][0] = "UVD", | |
185 | [32+3][0] = "UVDU", | |
186 | [32+4][0] = "MP1", | |
187 | [32+15][0] = "SDMA0", | |
188 | [0][1] = "MP0", | |
189 | [1][1] = "VCE0", | |
190 | [2][1] = "VCE0U", | |
191 | [3][1] = "DBGU0", | |
192 | [4][1] = "HDP", | |
193 | [5][1] = "XDP", | |
194 | [14][1] = "OSS", | |
195 | [15][1] = "SDMA0", | |
196 | [32+0][1] = "DCE", | |
197 | [32+1][1] = "DCEDWB", | |
198 | [32+2][1] = "XDMA", | |
199 | [32+3][1] = "UVD", | |
200 | [32+4][1] = "UVDU", | |
201 | [32+5][1] = "MP1", | |
202 | [32+6][1] = "DBGU1", | |
203 | [32+15][1] = "SDMA1", | |
204 | }; | |
205 | ||
206 | static const char *mmhub_client_ids_vega20[][2] = { | |
207 | [0][0] = "XDMA", | |
208 | [1][0] = "DCE", | |
209 | [2][0] = "VCE0", | |
210 | [3][0] = "VCE0U", | |
211 | [4][0] = "UVD", | |
212 | [5][0] = "UVD1U", | |
213 | [13][0] = "OSS", | |
214 | [14][0] = "HDP", | |
215 | [15][0] = "SDMA0", | |
216 | [32+0][0] = "UVD", | |
217 | [32+1][0] = "UVDU", | |
218 | [32+2][0] = "MP1", | |
219 | [32+3][0] = "MP0", | |
220 | [32+12][0] = "UTCL2", | |
221 | [32+14][0] = "SDMA1", | |
222 | [0][1] = "XDMA", | |
223 | [1][1] = "DCE", | |
224 | [2][1] = "DCEDWB", | |
225 | [3][1] = "VCE0", | |
226 | [4][1] = "VCE0U", | |
227 | [5][1] = "UVD1", | |
228 | [6][1] = "UVD1U", | |
229 | [7][1] = "DBGU0", | |
230 | [8][1] = "XDP", | |
231 | [13][1] = "OSS", | |
232 | [14][1] = "HDP", | |
233 | [15][1] = "SDMA0", | |
234 | [32+0][1] = "UVD", | |
235 | [32+1][1] = "UVDU", | |
236 | [32+2][1] = "DBGU1", | |
237 | [32+3][1] = "MP1", | |
238 | [32+4][1] = "MP0", | |
239 | [32+14][1] = "SDMA1", | |
240 | }; | |
241 | ||
242 | static const char *mmhub_client_ids_arcturus[][2] = { | |
e83db774 AD |
243 | [0][0] = "DBGU1", |
244 | [1][0] = "XDP", | |
02f23f5f | 245 | [2][0] = "MP1", |
02f23f5f | 246 | [14][0] = "HDP", |
e83db774 AD |
247 | [171][0] = "JPEG", |
248 | [172][0] = "VCN", | |
249 | [173][0] = "VCNU", | |
250 | [203][0] = "JPEG1", | |
251 | [204][0] = "VCN1", | |
252 | [205][0] = "VCN1U", | |
253 | [256][0] = "SDMA0", | |
254 | [257][0] = "SDMA1", | |
255 | [258][0] = "SDMA2", | |
256 | [259][0] = "SDMA3", | |
257 | [260][0] = "SDMA4", | |
258 | [261][0] = "SDMA5", | |
259 | [262][0] = "SDMA6", | |
260 | [263][0] = "SDMA7", | |
261 | [384][0] = "OSS", | |
02f23f5f AD |
262 | [0][1] = "DBGU1", |
263 | [1][1] = "XDP", | |
264 | [2][1] = "MP1", | |
02f23f5f | 265 | [14][1] = "HDP", |
e83db774 AD |
266 | [171][1] = "JPEG", |
267 | [172][1] = "VCN", | |
268 | [173][1] = "VCNU", | |
269 | [203][1] = "JPEG1", | |
270 | [204][1] = "VCN1", | |
271 | [205][1] = "VCN1U", | |
272 | [256][1] = "SDMA0", | |
273 | [257][1] = "SDMA1", | |
274 | [258][1] = "SDMA2", | |
275 | [259][1] = "SDMA3", | |
276 | [260][1] = "SDMA4", | |
277 | [261][1] = "SDMA5", | |
278 | [262][1] = "SDMA6", | |
279 | [263][1] = "SDMA7", | |
280 | [384][1] = "OSS", | |
02f23f5f | 281 | }; |
ebdef28e | 282 | |
e844cd99 AD |
283 | static const char *mmhub_client_ids_aldebaran[][2] = { |
284 | [2][0] = "MP1", | |
285 | [3][0] = "MP0", | |
286 | [15][0] = "SDMA0", | |
287 | [32+0][0] = "UTCL2", | |
288 | [32+4][0] = "MPIO", | |
289 | [32+13][0] = "OSS", | |
290 | [32+15][0] = "SDMA1", | |
291 | [64+15][0] = "SDMA2", | |
292 | [96+11][0] = "JPEG0", | |
293 | [96+12][0] = "VCN0", | |
294 | [96+13][0] = "VCNU0", | |
295 | [96+15][0] = "SDMA3", | |
296 | [128+11][0] = "JPEG1", | |
297 | [128+12][0] = "VCN1", | |
298 | [128+13][0] = "VCNU1", | |
299 | [128+15][0] = "SDMA4", | |
300 | [160+14][0] = "HDP", | |
301 | [2][1] = "MP1", | |
302 | [3][1] = "MP0", | |
303 | [15][1] = "SDMA0", | |
304 | [32+1][1] = "DBGU_IO0", | |
305 | [32+2][1] = "DBGU_IO2", | |
306 | [32+4][1] = "MPIO", | |
307 | [32+13][1] = "OSS", | |
308 | [32+15][1] = "SDMA1", | |
309 | [64+15][1] = "SDMA2", | |
310 | [96+11][1] = "JPEG0", | |
311 | [96+12][1] = "VCN0", | |
312 | [96+13][1] = "VCNU0", | |
313 | [96+15][1] = "SDMA3", | |
314 | [128+11][1] = "JPEG1", | |
315 | [128+12][1] = "VCN1", | |
316 | [128+13][1] = "VCNU1", | |
317 | [128+15][1] = "SDMA4", | |
318 | [160+14][1] = "HDP", | |
319 | }; | |
320 | ||
946a4d5b | 321 | static const struct soc15_reg_golden golden_settings_mmhub_1_0_0[] = |
5c583018 | 322 | { |
946a4d5b SL |
323 | SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmDAGB1_WRCLI2, 0x00000007, 0xfe5fe0fa), |
324 | SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmMMEA1_DRAM_WR_CLI2GRP_MAP0, 0x00000030, 0x55555565) | |
5c583018 EQ |
325 | }; |
326 | ||
946a4d5b | 327 | static const struct soc15_reg_golden golden_settings_athub_1_0_0[] = |
5c583018 | 328 | { |
946a4d5b SL |
329 | SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL, 0x0000ff00, 0x00000800), |
330 | SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL2, 0x00ff00ff, 0x00080008) | |
5c583018 EQ |
331 | }; |
332 | ||
791c4769 | 333 | static const uint32_t ecc_umc_mcumc_ctrl_addrs[] = { |
334 | (0x000143c0 + 0x00000000), | |
335 | (0x000143c0 + 0x00000800), | |
336 | (0x000143c0 + 0x00001000), | |
337 | (0x000143c0 + 0x00001800), | |
338 | (0x000543c0 + 0x00000000), | |
339 | (0x000543c0 + 0x00000800), | |
340 | (0x000543c0 + 0x00001000), | |
341 | (0x000543c0 + 0x00001800), | |
342 | (0x000943c0 + 0x00000000), | |
343 | (0x000943c0 + 0x00000800), | |
344 | (0x000943c0 + 0x00001000), | |
345 | (0x000943c0 + 0x00001800), | |
346 | (0x000d43c0 + 0x00000000), | |
347 | (0x000d43c0 + 0x00000800), | |
348 | (0x000d43c0 + 0x00001000), | |
349 | (0x000d43c0 + 0x00001800), | |
350 | (0x001143c0 + 0x00000000), | |
351 | (0x001143c0 + 0x00000800), | |
352 | (0x001143c0 + 0x00001000), | |
353 | (0x001143c0 + 0x00001800), | |
354 | (0x001543c0 + 0x00000000), | |
355 | (0x001543c0 + 0x00000800), | |
356 | (0x001543c0 + 0x00001000), | |
357 | (0x001543c0 + 0x00001800), | |
358 | (0x001943c0 + 0x00000000), | |
359 | (0x001943c0 + 0x00000800), | |
360 | (0x001943c0 + 0x00001000), | |
361 | (0x001943c0 + 0x00001800), | |
362 | (0x001d43c0 + 0x00000000), | |
363 | (0x001d43c0 + 0x00000800), | |
364 | (0x001d43c0 + 0x00001000), | |
365 | (0x001d43c0 + 0x00001800), | |
02bab923 DP |
366 | }; |
367 | ||
791c4769 | 368 | static const uint32_t ecc_umc_mcumc_ctrl_mask_addrs[] = { |
369 | (0x000143e0 + 0x00000000), | |
370 | (0x000143e0 + 0x00000800), | |
371 | (0x000143e0 + 0x00001000), | |
372 | (0x000143e0 + 0x00001800), | |
373 | (0x000543e0 + 0x00000000), | |
374 | (0x000543e0 + 0x00000800), | |
375 | (0x000543e0 + 0x00001000), | |
376 | (0x000543e0 + 0x00001800), | |
377 | (0x000943e0 + 0x00000000), | |
378 | (0x000943e0 + 0x00000800), | |
379 | (0x000943e0 + 0x00001000), | |
380 | (0x000943e0 + 0x00001800), | |
381 | (0x000d43e0 + 0x00000000), | |
382 | (0x000d43e0 + 0x00000800), | |
383 | (0x000d43e0 + 0x00001000), | |
384 | (0x000d43e0 + 0x00001800), | |
385 | (0x001143e0 + 0x00000000), | |
386 | (0x001143e0 + 0x00000800), | |
387 | (0x001143e0 + 0x00001000), | |
388 | (0x001143e0 + 0x00001800), | |
389 | (0x001543e0 + 0x00000000), | |
390 | (0x001543e0 + 0x00000800), | |
391 | (0x001543e0 + 0x00001000), | |
392 | (0x001543e0 + 0x00001800), | |
393 | (0x001943e0 + 0x00000000), | |
394 | (0x001943e0 + 0x00000800), | |
395 | (0x001943e0 + 0x00001000), | |
396 | (0x001943e0 + 0x00001800), | |
397 | (0x001d43e0 + 0x00000000), | |
398 | (0x001d43e0 + 0x00000800), | |
399 | (0x001d43e0 + 0x00001000), | |
400 | (0x001d43e0 + 0x00001800), | |
02bab923 DP |
401 | }; |
402 | ||
791c4769 | 403 | static int gmc_v9_0_ecc_interrupt_state(struct amdgpu_device *adev, |
404 | struct amdgpu_irq_src *src, | |
405 | unsigned type, | |
406 | enum amdgpu_interrupt_state state) | |
407 | { | |
408 | u32 bits, i, tmp, reg; | |
1e2c6d55 JC |
409 | |
410 | /* Devices newer then VEGA10/12 shall have these programming | |
411 | sequences performed by PSP BL */ | |
412 | if (adev->asic_type >= CHIP_VEGA20) | |
413 | return 0; | |
791c4769 | 414 | |
415 | bits = 0x7f; | |
416 | ||
417 | switch (state) { | |
418 | case AMDGPU_IRQ_STATE_DISABLE: | |
419 | for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) { | |
420 | reg = ecc_umc_mcumc_ctrl_addrs[i]; | |
421 | tmp = RREG32(reg); | |
422 | tmp &= ~bits; | |
423 | WREG32(reg, tmp); | |
424 | } | |
425 | for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) { | |
426 | reg = ecc_umc_mcumc_ctrl_mask_addrs[i]; | |
427 | tmp = RREG32(reg); | |
428 | tmp &= ~bits; | |
429 | WREG32(reg, tmp); | |
430 | } | |
431 | break; | |
432 | case AMDGPU_IRQ_STATE_ENABLE: | |
433 | for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) { | |
434 | reg = ecc_umc_mcumc_ctrl_addrs[i]; | |
435 | tmp = RREG32(reg); | |
436 | tmp |= bits; | |
437 | WREG32(reg, tmp); | |
438 | } | |
439 | for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) { | |
440 | reg = ecc_umc_mcumc_ctrl_mask_addrs[i]; | |
441 | tmp = RREG32(reg); | |
442 | tmp |= bits; | |
443 | WREG32(reg, tmp); | |
444 | } | |
445 | break; | |
446 | default: | |
447 | break; | |
448 | } | |
449 | ||
450 | return 0; | |
451 | } | |
452 | ||
e60f8db5 AX |
453 | static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev, |
454 | struct amdgpu_irq_src *src, | |
455 | unsigned type, | |
456 | enum amdgpu_interrupt_state state) | |
457 | { | |
458 | struct amdgpu_vmhub *hub; | |
ae6d1416 | 459 | u32 tmp, reg, bits, i, j; |
e60f8db5 | 460 | |
11250164 CK |
461 | bits = VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | |
462 | VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | | |
463 | VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | | |
464 | VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | | |
465 | VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | | |
466 | VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | | |
467 | VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK; | |
468 | ||
e60f8db5 AX |
469 | switch (state) { |
470 | case AMDGPU_IRQ_STATE_DISABLE: | |
1daa2bfa | 471 | for (j = 0; j < adev->num_vmhubs; j++) { |
ae6d1416 TSD |
472 | hub = &adev->vmhub[j]; |
473 | for (i = 0; i < 16; i++) { | |
474 | reg = hub->vm_context0_cntl + i; | |
475 | tmp = RREG32(reg); | |
476 | tmp &= ~bits; | |
477 | WREG32(reg, tmp); | |
478 | } | |
e60f8db5 AX |
479 | } |
480 | break; | |
481 | case AMDGPU_IRQ_STATE_ENABLE: | |
1daa2bfa | 482 | for (j = 0; j < adev->num_vmhubs; j++) { |
ae6d1416 TSD |
483 | hub = &adev->vmhub[j]; |
484 | for (i = 0; i < 16; i++) { | |
485 | reg = hub->vm_context0_cntl + i; | |
486 | tmp = RREG32(reg); | |
487 | tmp |= bits; | |
488 | WREG32(reg, tmp); | |
489 | } | |
e60f8db5 | 490 | } |
9304ca4d | 491 | break; |
e60f8db5 AX |
492 | default: |
493 | break; | |
494 | } | |
495 | ||
496 | return 0; | |
497 | } | |
498 | ||
499 | static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev, | |
e3898719 CK |
500 | struct amdgpu_irq_src *source, |
501 | struct amdgpu_iv_entry *entry) | |
e60f8db5 | 502 | { |
c468f9e2 | 503 | bool retry_fault = !!(entry->src_data[1] & 0x80); |
02f23f5f | 504 | uint32_t status = 0, cid = 0, rw = 0; |
e3898719 CK |
505 | struct amdgpu_task_info task_info; |
506 | struct amdgpu_vmhub *hub; | |
02f23f5f | 507 | const char *mmhub_cid; |
e3898719 CK |
508 | const char *hub_name; |
509 | u64 addr; | |
e60f8db5 AX |
510 | |
511 | addr = (u64)entry->src_data[0] << 12; | |
512 | addr |= ((u64)entry->src_data[1] & 0xf) << 44; | |
513 | ||
0291150d CK |
514 | if (retry_fault) { |
515 | /* Returning 1 here also prevents sending the IV to the KFD */ | |
516 | ||
517 | /* Process it onyl if it's the first fault for this address */ | |
518 | if (entry->ih != &adev->irq.ih_soft && | |
519 | amdgpu_gmc_filter_faults(adev, addr, entry->pasid, | |
520 | entry->timestamp)) | |
521 | return 1; | |
522 | ||
523 | /* Delegate it to a different ring if the hardware hasn't | |
524 | * already done it. | |
525 | */ | |
526 | if (in_interrupt()) { | |
527 | amdgpu_irq_delegate(adev, entry, 8); | |
528 | return 1; | |
529 | } | |
530 | ||
531 | /* Try to handle the recoverable page faults by filling page | |
532 | * tables | |
533 | */ | |
534 | if (amdgpu_vm_handle_fault(adev, entry->pasid, addr)) | |
535 | return 1; | |
536 | } | |
e3898719 CK |
537 | |
538 | if (!printk_ratelimit()) | |
539 | return 0; | |
540 | ||
51c60898 | 541 | if (entry->client_id == SOC15_IH_CLIENTID_VMC) { |
e3898719 | 542 | hub_name = "mmhub0"; |
51c60898 LM |
543 | hub = &adev->vmhub[AMDGPU_MMHUB_0]; |
544 | } else if (entry->client_id == SOC15_IH_CLIENTID_VMC1) { | |
e3898719 | 545 | hub_name = "mmhub1"; |
51c60898 LM |
546 | hub = &adev->vmhub[AMDGPU_MMHUB_1]; |
547 | } else { | |
e3898719 | 548 | hub_name = "gfxhub0"; |
51c60898 LM |
549 | hub = &adev->vmhub[AMDGPU_GFXHUB_0]; |
550 | } | |
551 | ||
e3898719 CK |
552 | memset(&task_info, 0, sizeof(struct amdgpu_task_info)); |
553 | amdgpu_vm_get_task_info(adev, entry->pasid, &task_info); | |
ec671737 | 554 | |
e3898719 CK |
555 | dev_err(adev->dev, |
556 | "[%s] %s page fault (src_id:%u ring:%u vmid:%u " | |
557 | "pasid:%u, for process %s pid %d thread %s pid %d)\n", | |
558 | hub_name, retry_fault ? "retry" : "no-retry", | |
559 | entry->src_id, entry->ring_id, entry->vmid, | |
560 | entry->pasid, task_info.process_name, task_info.tgid, | |
561 | task_info.task_name, task_info.pid); | |
be14729a YZ |
562 | dev_err(adev->dev, " in page starting at address 0x%016llx from IH client 0x%x (%s)\n", |
563 | addr, entry->client_id, | |
564 | soc15_ih_clientid_name[entry->client_id]); | |
e60f8db5 | 565 | |
e3898719 CK |
566 | if (amdgpu_sriov_vf(adev)) |
567 | return 0; | |
568 | ||
569 | /* | |
570 | * Issue a dummy read to wait for the status register to | |
571 | * be updated to avoid reading an incorrect value due to | |
572 | * the new fast GRBM interface. | |
573 | */ | |
574 | if (entry->vmid_src == AMDGPU_GFXHUB_0) | |
575 | RREG32(hub->vm_l2_pro_fault_status); | |
576 | ||
577 | status = RREG32(hub->vm_l2_pro_fault_status); | |
578 | cid = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, CID); | |
579 | rw = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, RW); | |
580 | WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1); | |
581 | ||
582 | ||
583 | dev_err(adev->dev, | |
584 | "VM_L2_PROTECTION_FAULT_STATUS:0x%08X\n", | |
585 | status); | |
586 | if (hub == &adev->vmhub[AMDGPU_GFXHUB_0]) { | |
587 | dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n", | |
588 | cid >= ARRAY_SIZE(gfxhub_client_ids) ? "unknown" : | |
589 | gfxhub_client_ids[cid], | |
590 | cid); | |
591 | } else { | |
592 | switch (adev->asic_type) { | |
593 | case CHIP_VEGA10: | |
594 | mmhub_cid = mmhub_client_ids_vega10[cid][rw]; | |
595 | break; | |
596 | case CHIP_VEGA12: | |
597 | mmhub_cid = mmhub_client_ids_vega12[cid][rw]; | |
598 | break; | |
599 | case CHIP_VEGA20: | |
600 | mmhub_cid = mmhub_client_ids_vega20[cid][rw]; | |
601 | break; | |
602 | case CHIP_ARCTURUS: | |
603 | mmhub_cid = mmhub_client_ids_arcturus[cid][rw]; | |
604 | break; | |
605 | case CHIP_RAVEN: | |
606 | mmhub_cid = mmhub_client_ids_raven[cid][rw]; | |
607 | break; | |
608 | case CHIP_RENOIR: | |
609 | mmhub_cid = mmhub_client_ids_renoir[cid][rw]; | |
610 | break; | |
e844cd99 AD |
611 | case CHIP_ALDEBARAN: |
612 | mmhub_cid = mmhub_client_ids_aldebaran[cid][rw]; | |
613 | break; | |
e3898719 CK |
614 | default: |
615 | mmhub_cid = NULL; | |
616 | break; | |
5ddd4a9a | 617 | } |
e3898719 CK |
618 | dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n", |
619 | mmhub_cid ? mmhub_cid : "unknown", cid); | |
79a0c465 | 620 | } |
e3898719 CK |
621 | dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n", |
622 | REG_GET_FIELD(status, | |
623 | VM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS)); | |
624 | dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n", | |
625 | REG_GET_FIELD(status, | |
626 | VM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR)); | |
627 | dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n", | |
628 | REG_GET_FIELD(status, | |
629 | VM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS)); | |
630 | dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n", | |
631 | REG_GET_FIELD(status, | |
632 | VM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR)); | |
633 | dev_err(adev->dev, "\t RW: 0x%x\n", rw); | |
e60f8db5 AX |
634 | return 0; |
635 | } | |
636 | ||
637 | static const struct amdgpu_irq_src_funcs gmc_v9_0_irq_funcs = { | |
638 | .set = gmc_v9_0_vm_fault_interrupt_state, | |
639 | .process = gmc_v9_0_process_interrupt, | |
640 | }; | |
641 | ||
791c4769 | 642 | |
643 | static const struct amdgpu_irq_src_funcs gmc_v9_0_ecc_funcs = { | |
644 | .set = gmc_v9_0_ecc_interrupt_state, | |
34cc4fd9 | 645 | .process = amdgpu_umc_process_ecc_irq, |
791c4769 | 646 | }; |
647 | ||
e60f8db5 AX |
648 | static void gmc_v9_0_set_irq_funcs(struct amdgpu_device *adev) |
649 | { | |
770d13b1 CK |
650 | adev->gmc.vm_fault.num_types = 1; |
651 | adev->gmc.vm_fault.funcs = &gmc_v9_0_irq_funcs; | |
791c4769 | 652 | |
2ee9403e ZL |
653 | if (!amdgpu_sriov_vf(adev)) { |
654 | adev->gmc.ecc_irq.num_types = 1; | |
655 | adev->gmc.ecc_irq.funcs = &gmc_v9_0_ecc_funcs; | |
656 | } | |
e60f8db5 AX |
657 | } |
658 | ||
2a79d868 YZ |
659 | static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid, |
660 | uint32_t flush_type) | |
03f89feb CK |
661 | { |
662 | u32 req = 0; | |
663 | ||
03f89feb | 664 | req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, |
c4f46f22 | 665 | PER_VMID_INVALIDATE_REQ, 1 << vmid); |
2a79d868 | 666 | req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type); |
03f89feb CK |
667 | req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1); |
668 | req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1); | |
669 | req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1); | |
670 | req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1); | |
671 | req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1); | |
672 | req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, | |
673 | CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0); | |
674 | ||
675 | return req; | |
676 | } | |
677 | ||
90f6452c | 678 | /** |
679 | * gmc_v9_0_use_invalidate_semaphore - judge whether to use semaphore | |
680 | * | |
681 | * @adev: amdgpu_device pointer | |
682 | * @vmhub: vmhub type | |
683 | * | |
684 | */ | |
685 | static bool gmc_v9_0_use_invalidate_semaphore(struct amdgpu_device *adev, | |
686 | uint32_t vmhub) | |
687 | { | |
b7daed1b | 688 | if (adev->asic_type == CHIP_ALDEBARAN) |
d477c5aa | 689 | return false; |
d477c5aa | 690 | |
90f6452c | 691 | return ((vmhub == AMDGPU_MMHUB_0 || |
692 | vmhub == AMDGPU_MMHUB_1) && | |
693 | (!amdgpu_sriov_vf(adev)) && | |
54f78a76 AD |
694 | (!(!(adev->apu_flags & AMD_APU_IS_RAVEN2) && |
695 | (adev->apu_flags & AMD_APU_IS_PICASSO)))); | |
90f6452c | 696 | } |
697 | ||
ea930000 AS |
698 | static bool gmc_v9_0_get_atc_vmid_pasid_mapping_info(struct amdgpu_device *adev, |
699 | uint8_t vmid, uint16_t *p_pasid) | |
700 | { | |
701 | uint32_t value; | |
702 | ||
703 | value = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING) | |
704 | + vmid); | |
705 | *p_pasid = value & ATC_VMID0_PASID_MAPPING__PASID_MASK; | |
706 | ||
707 | return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK); | |
708 | } | |
709 | ||
e60f8db5 AX |
710 | /* |
711 | * GART | |
712 | * VMID 0 is the physical GPU addresses as used by the kernel. | |
713 | * VMIDs 1-15 are used for userspace clients and are handled | |
714 | * by the amdgpu vm/hsa code. | |
715 | */ | |
716 | ||
717 | /** | |
2a79d868 | 718 | * gmc_v9_0_flush_gpu_tlb - tlb flush with certain type |
e60f8db5 AX |
719 | * |
720 | * @adev: amdgpu_device pointer | |
721 | * @vmid: vm instance to flush | |
bf0df09c | 722 | * @vmhub: which hub to flush |
2a79d868 | 723 | * @flush_type: the flush type |
e60f8db5 | 724 | * |
2a79d868 | 725 | * Flush the TLB for the requested page table using certain type. |
e60f8db5 | 726 | */ |
3ff98548 OZ |
727 | static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, |
728 | uint32_t vmhub, uint32_t flush_type) | |
e60f8db5 | 729 | { |
90f6452c | 730 | bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(adev, vmhub); |
e60f8db5 | 731 | const unsigned eng = 17; |
b80cd524 | 732 | u32 j, inv_req, inv_req2, tmp; |
3ff98548 | 733 | struct amdgpu_vmhub *hub; |
e60f8db5 | 734 | |
3ff98548 | 735 | BUG_ON(vmhub >= adev->num_vmhubs); |
e60f8db5 | 736 | |
3ff98548 | 737 | hub = &adev->vmhub[vmhub]; |
b80cd524 FK |
738 | if (adev->gmc.xgmi.num_physical_nodes && |
739 | adev->asic_type == CHIP_VEGA20) { | |
740 | /* Vega20+XGMI caches PTEs in TC and TLB. Add a | |
741 | * heavy-weight TLB flush (type 2), which flushes | |
742 | * both. Due to a race condition with concurrent | |
743 | * memory accesses using the same TLB cache line, we | |
744 | * still need a second TLB flush after this. | |
745 | */ | |
746 | inv_req = gmc_v9_0_get_invalidate_req(vmid, 2); | |
747 | inv_req2 = gmc_v9_0_get_invalidate_req(vmid, flush_type); | |
748 | } else { | |
749 | inv_req = gmc_v9_0_get_invalidate_req(vmid, flush_type); | |
750 | inv_req2 = 0; | |
751 | } | |
3890d111 | 752 | |
3ff98548 OZ |
753 | /* This is necessary for a HW workaround under SRIOV as well |
754 | * as GFXOFF under bare metal | |
755 | */ | |
756 | if (adev->gfx.kiq.ring.sched.ready && | |
81202807 DL |
757 | (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev)) && |
758 | down_read_trylock(&adev->reset_sem)) { | |
148f597d HR |
759 | uint32_t req = hub->vm_inv_eng0_req + hub->eng_distance * eng; |
760 | uint32_t ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng; | |
3ff98548 | 761 | |
37c58ddf | 762 | amdgpu_virt_kiq_reg_write_reg_wait(adev, req, ack, inv_req, |
148f597d | 763 | 1 << vmid); |
81202807 | 764 | up_read(&adev->reset_sem); |
3ff98548 OZ |
765 | return; |
766 | } | |
396557b0 | 767 | |
3ff98548 | 768 | spin_lock(&adev->gmc.invalidate_lock); |
f920d1bb | 769 | |
770 | /* | |
771 | * It may lose gpuvm invalidate acknowldege state across power-gating | |
772 | * off cycle, add semaphore acquire before invalidation and semaphore | |
773 | * release after invalidation to avoid entering power gated state | |
774 | * to WA the Issue | |
775 | */ | |
776 | ||
777 | /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ | |
90f6452c | 778 | if (use_semaphore) { |
f920d1bb | 779 | for (j = 0; j < adev->usec_timeout; j++) { |
780 | /* a read return value of 1 means semaphore acuqire */ | |
148f597d HR |
781 | tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_sem + |
782 | hub->eng_distance * eng); | |
f920d1bb | 783 | if (tmp & 0x1) |
784 | break; | |
785 | udelay(1); | |
786 | } | |
787 | ||
788 | if (j >= adev->usec_timeout) | |
789 | DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n"); | |
790 | } | |
791 | ||
b80cd524 | 792 | do { |
148f597d HR |
793 | WREG32_NO_KIQ(hub->vm_inv_eng0_req + |
794 | hub->eng_distance * eng, inv_req); | |
53499173 | 795 | |
b80cd524 FK |
796 | /* |
797 | * Issue a dummy read to wait for the ACK register to | |
798 | * be cleared to avoid a false ACK due to the new fast | |
799 | * GRBM interface. | |
800 | */ | |
801 | if (vmhub == AMDGPU_GFXHUB_0) | |
148f597d HR |
802 | RREG32_NO_KIQ(hub->vm_inv_eng0_req + |
803 | hub->eng_distance * eng); | |
53499173 | 804 | |
b80cd524 | 805 | for (j = 0; j < adev->usec_timeout; j++) { |
148f597d HR |
806 | tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_ack + |
807 | hub->eng_distance * eng); | |
b80cd524 FK |
808 | if (tmp & (1 << vmid)) |
809 | break; | |
810 | udelay(1); | |
811 | } | |
812 | ||
813 | inv_req = inv_req2; | |
814 | inv_req2 = 0; | |
815 | } while (inv_req); | |
f920d1bb | 816 | |
817 | /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ | |
90f6452c | 818 | if (use_semaphore) |
f920d1bb | 819 | /* |
820 | * add semaphore release after invalidation, | |
821 | * write with 0 means semaphore release | |
822 | */ | |
148f597d HR |
823 | WREG32_NO_KIQ(hub->vm_inv_eng0_sem + |
824 | hub->eng_distance * eng, 0); | |
f920d1bb | 825 | |
3ff98548 | 826 | spin_unlock(&adev->gmc.invalidate_lock); |
f920d1bb | 827 | |
3ff98548 OZ |
828 | if (j < adev->usec_timeout) |
829 | return; | |
830 | ||
831 | DRM_ERROR("Timeout waiting for VM flush ACK!\n"); | |
e60f8db5 AX |
832 | } |
833 | ||
ea930000 AS |
834 | /** |
835 | * gmc_v9_0_flush_gpu_tlb_pasid - tlb flush via pasid | |
836 | * | |
837 | * @adev: amdgpu_device pointer | |
838 | * @pasid: pasid to be flush | |
bf0df09c LJ |
839 | * @flush_type: the flush type |
840 | * @all_hub: flush all hubs | |
ea930000 AS |
841 | * |
842 | * Flush the TLB for the requested pasid. | |
843 | */ | |
844 | static int gmc_v9_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev, | |
845 | uint16_t pasid, uint32_t flush_type, | |
846 | bool all_hub) | |
847 | { | |
848 | int vmid, i; | |
849 | signed long r; | |
850 | uint32_t seq; | |
851 | uint16_t queried_pasid; | |
852 | bool ret; | |
853 | struct amdgpu_ring *ring = &adev->gfx.kiq.ring; | |
854 | struct amdgpu_kiq *kiq = &adev->gfx.kiq; | |
855 | ||
53b3f8f4 | 856 | if (amdgpu_in_reset(adev)) |
ea930000 AS |
857 | return -EIO; |
858 | ||
81202807 | 859 | if (ring->sched.ready && down_read_trylock(&adev->reset_sem)) { |
b80cd524 FK |
860 | /* Vega20+XGMI caches PTEs in TC and TLB. Add a |
861 | * heavy-weight TLB flush (type 2), which flushes | |
862 | * both. Due to a race condition with concurrent | |
863 | * memory accesses using the same TLB cache line, we | |
864 | * still need a second TLB flush after this. | |
865 | */ | |
866 | bool vega20_xgmi_wa = (adev->gmc.xgmi.num_physical_nodes && | |
867 | adev->asic_type == CHIP_VEGA20); | |
868 | /* 2 dwords flush + 8 dwords fence */ | |
869 | unsigned int ndw = kiq->pmf->invalidate_tlbs_size + 8; | |
870 | ||
871 | if (vega20_xgmi_wa) | |
872 | ndw += kiq->pmf->invalidate_tlbs_size; | |
873 | ||
ea930000 | 874 | spin_lock(&adev->gfx.kiq.ring_lock); |
36a1707a | 875 | /* 2 dwords flush + 8 dwords fence */ |
b80cd524 FK |
876 | amdgpu_ring_alloc(ring, ndw); |
877 | if (vega20_xgmi_wa) | |
878 | kiq->pmf->kiq_invalidate_tlbs(ring, | |
879 | pasid, 2, all_hub); | |
ea930000 AS |
880 | kiq->pmf->kiq_invalidate_tlbs(ring, |
881 | pasid, flush_type, all_hub); | |
04e4e2e9 YT |
882 | r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); |
883 | if (r) { | |
884 | amdgpu_ring_undo(ring); | |
abb17b1e | 885 | spin_unlock(&adev->gfx.kiq.ring_lock); |
81202807 | 886 | up_read(&adev->reset_sem); |
04e4e2e9 YT |
887 | return -ETIME; |
888 | } | |
889 | ||
ea930000 AS |
890 | amdgpu_ring_commit(ring); |
891 | spin_unlock(&adev->gfx.kiq.ring_lock); | |
892 | r = amdgpu_fence_wait_polling(ring, seq, adev->usec_timeout); | |
893 | if (r < 1) { | |
aac89168 | 894 | dev_err(adev->dev, "wait for kiq fence error: %ld.\n", r); |
81202807 | 895 | up_read(&adev->reset_sem); |
ea930000 AS |
896 | return -ETIME; |
897 | } | |
81202807 | 898 | up_read(&adev->reset_sem); |
ea930000 AS |
899 | return 0; |
900 | } | |
901 | ||
902 | for (vmid = 1; vmid < 16; vmid++) { | |
903 | ||
904 | ret = gmc_v9_0_get_atc_vmid_pasid_mapping_info(adev, vmid, | |
905 | &queried_pasid); | |
906 | if (ret && queried_pasid == pasid) { | |
907 | if (all_hub) { | |
908 | for (i = 0; i < adev->num_vmhubs; i++) | |
909 | gmc_v9_0_flush_gpu_tlb(adev, vmid, | |
fa34edbe | 910 | i, flush_type); |
ea930000 AS |
911 | } else { |
912 | gmc_v9_0_flush_gpu_tlb(adev, vmid, | |
fa34edbe | 913 | AMDGPU_GFXHUB_0, flush_type); |
ea930000 AS |
914 | } |
915 | break; | |
916 | } | |
917 | } | |
918 | ||
919 | return 0; | |
920 | ||
921 | } | |
922 | ||
9096d6e5 | 923 | static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring, |
c633c00b | 924 | unsigned vmid, uint64_t pd_addr) |
9096d6e5 | 925 | { |
90f6452c | 926 | bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(ring->adev, ring->funcs->vmhub); |
250b4228 CK |
927 | struct amdgpu_device *adev = ring->adev; |
928 | struct amdgpu_vmhub *hub = &adev->vmhub[ring->funcs->vmhub]; | |
2a79d868 | 929 | uint32_t req = gmc_v9_0_get_invalidate_req(vmid, 0); |
9096d6e5 CK |
930 | unsigned eng = ring->vm_inv_eng; |
931 | ||
f920d1bb | 932 | /* |
933 | * It may lose gpuvm invalidate acknowldege state across power-gating | |
934 | * off cycle, add semaphore acquire before invalidation and semaphore | |
935 | * release after invalidation to avoid entering power gated state | |
936 | * to WA the Issue | |
937 | */ | |
938 | ||
939 | /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ | |
90f6452c | 940 | if (use_semaphore) |
f920d1bb | 941 | /* a read return value of 1 means semaphore acuqire */ |
942 | amdgpu_ring_emit_reg_wait(ring, | |
148f597d HR |
943 | hub->vm_inv_eng0_sem + |
944 | hub->eng_distance * eng, 0x1, 0x1); | |
f920d1bb | 945 | |
148f597d HR |
946 | amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 + |
947 | (hub->ctx_addr_distance * vmid), | |
9096d6e5 CK |
948 | lower_32_bits(pd_addr)); |
949 | ||
148f597d HR |
950 | amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 + |
951 | (hub->ctx_addr_distance * vmid), | |
9096d6e5 CK |
952 | upper_32_bits(pd_addr)); |
953 | ||
148f597d HR |
954 | amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req + |
955 | hub->eng_distance * eng, | |
956 | hub->vm_inv_eng0_ack + | |
957 | hub->eng_distance * eng, | |
f8bc9037 | 958 | req, 1 << vmid); |
f732b6b3 | 959 | |
f920d1bb | 960 | /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ |
90f6452c | 961 | if (use_semaphore) |
f920d1bb | 962 | /* |
963 | * add semaphore release after invalidation, | |
964 | * write with 0 means semaphore release | |
965 | */ | |
148f597d HR |
966 | amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem + |
967 | hub->eng_distance * eng, 0); | |
f920d1bb | 968 | |
9096d6e5 CK |
969 | return pd_addr; |
970 | } | |
971 | ||
c633c00b CK |
972 | static void gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid, |
973 | unsigned pasid) | |
974 | { | |
975 | struct amdgpu_device *adev = ring->adev; | |
976 | uint32_t reg; | |
977 | ||
f2d66571 LM |
978 | /* Do nothing because there's no lut register for mmhub1. */ |
979 | if (ring->funcs->vmhub == AMDGPU_MMHUB_1) | |
980 | return; | |
981 | ||
a2d15ed7 | 982 | if (ring->funcs->vmhub == AMDGPU_GFXHUB_0) |
c633c00b CK |
983 | reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid; |
984 | else | |
985 | reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid; | |
986 | ||
987 | amdgpu_ring_emit_wreg(ring, reg, pasid); | |
988 | } | |
989 | ||
6490bd76 YZ |
990 | /* |
991 | * PTE format on VEGA 10: | |
992 | * 63:59 reserved | |
993 | * 58:57 mtype | |
994 | * 56 F | |
995 | * 55 L | |
996 | * 54 P | |
997 | * 53 SW | |
998 | * 52 T | |
999 | * 50:48 reserved | |
1000 | * 47:12 4k physical page base address | |
1001 | * 11:7 fragment | |
1002 | * 6 write | |
1003 | * 5 read | |
1004 | * 4 exe | |
1005 | * 3 Z | |
1006 | * 2 snooped | |
1007 | * 1 system | |
1008 | * 0 valid | |
e60f8db5 | 1009 | * |
6490bd76 YZ |
1010 | * PDE format on VEGA 10: |
1011 | * 63:59 block fragment size | |
1012 | * 58:55 reserved | |
1013 | * 54 P | |
1014 | * 53:48 reserved | |
1015 | * 47:6 physical base address of PD or PTE | |
1016 | * 5:3 reserved | |
1017 | * 2 C | |
1018 | * 1 system | |
1019 | * 0 valid | |
e60f8db5 | 1020 | */ |
e60f8db5 | 1021 | |
71776b6d | 1022 | static uint64_t gmc_v9_0_map_mtype(struct amdgpu_device *adev, uint32_t flags) |
e60f8db5 AX |
1023 | |
1024 | { | |
71776b6d | 1025 | switch (flags) { |
e60f8db5 | 1026 | case AMDGPU_VM_MTYPE_DEFAULT: |
71776b6d | 1027 | return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC); |
e60f8db5 | 1028 | case AMDGPU_VM_MTYPE_NC: |
71776b6d | 1029 | return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC); |
e60f8db5 | 1030 | case AMDGPU_VM_MTYPE_WC: |
71776b6d | 1031 | return AMDGPU_PTE_MTYPE_VG10(MTYPE_WC); |
093e48c0 | 1032 | case AMDGPU_VM_MTYPE_RW: |
71776b6d | 1033 | return AMDGPU_PTE_MTYPE_VG10(MTYPE_RW); |
e60f8db5 | 1034 | case AMDGPU_VM_MTYPE_CC: |
71776b6d | 1035 | return AMDGPU_PTE_MTYPE_VG10(MTYPE_CC); |
e60f8db5 | 1036 | case AMDGPU_VM_MTYPE_UC: |
71776b6d | 1037 | return AMDGPU_PTE_MTYPE_VG10(MTYPE_UC); |
e60f8db5 | 1038 | default: |
71776b6d | 1039 | return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC); |
e60f8db5 | 1040 | } |
e60f8db5 AX |
1041 | } |
1042 | ||
3de676d8 CK |
1043 | static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level, |
1044 | uint64_t *addr, uint64_t *flags) | |
e60f8db5 | 1045 | { |
bbc9fb10 | 1046 | if (!(*flags & AMDGPU_PDE_PTE) && !(*flags & AMDGPU_PTE_SYSTEM)) |
3de676d8 | 1047 | *addr = adev->vm_manager.vram_base_offset + *addr - |
770d13b1 | 1048 | adev->gmc.vram_start; |
3de676d8 | 1049 | BUG_ON(*addr & 0xFFFF00000000003FULL); |
6a42fd6f | 1050 | |
770d13b1 | 1051 | if (!adev->gmc.translate_further) |
6a42fd6f CK |
1052 | return; |
1053 | ||
1054 | if (level == AMDGPU_VM_PDB1) { | |
1055 | /* Set the block fragment size */ | |
1056 | if (!(*flags & AMDGPU_PDE_PTE)) | |
1057 | *flags |= AMDGPU_PDE_BFS(0x9); | |
1058 | ||
1059 | } else if (level == AMDGPU_VM_PDB0) { | |
1060 | if (*flags & AMDGPU_PDE_PTE) | |
1061 | *flags &= ~AMDGPU_PDE_PTE; | |
1062 | else | |
1063 | *flags |= AMDGPU_PTE_TF; | |
1064 | } | |
e60f8db5 AX |
1065 | } |
1066 | ||
cbfae36c CK |
1067 | static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev, |
1068 | struct amdgpu_bo_va_mapping *mapping, | |
1069 | uint64_t *flags) | |
1070 | { | |
1071 | *flags &= ~AMDGPU_PTE_EXECUTABLE; | |
1072 | *flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE; | |
1073 | ||
1074 | *flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK; | |
1075 | *flags |= mapping->flags & AMDGPU_PTE_MTYPE_VG10_MASK; | |
1076 | ||
1077 | if (mapping->flags & AMDGPU_PTE_PRT) { | |
1078 | *flags |= AMDGPU_PTE_PRT; | |
1079 | *flags &= ~AMDGPU_PTE_VALID; | |
1080 | } | |
1081 | ||
7ffe7238 YZ |
1082 | if ((adev->asic_type == CHIP_ARCTURUS || |
1083 | adev->asic_type == CHIP_ALDEBARAN) && | |
cbfae36c CK |
1084 | !(*flags & AMDGPU_PTE_SYSTEM) && |
1085 | mapping->bo_va->is_xgmi) | |
1086 | *flags |= AMDGPU_PTE_SNOOPED; | |
72b4db0f EH |
1087 | |
1088 | if (adev->asic_type == CHIP_ALDEBARAN) | |
1089 | *flags |= mapping->flags & AMDGPU_PTE_SNOOPED; | |
cbfae36c CK |
1090 | } |
1091 | ||
7b885f0e AD |
1092 | static unsigned gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev) |
1093 | { | |
1094 | u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL); | |
1095 | unsigned size; | |
1096 | ||
1097 | if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) { | |
1098 | size = AMDGPU_VBIOS_VGA_ALLOCATION; | |
1099 | } else { | |
1100 | u32 viewport; | |
1101 | ||
1102 | switch (adev->asic_type) { | |
1103 | case CHIP_RAVEN: | |
1104 | case CHIP_RENOIR: | |
1105 | viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION); | |
1106 | size = (REG_GET_FIELD(viewport, | |
1107 | HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) * | |
1108 | REG_GET_FIELD(viewport, | |
1109 | HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) * | |
1110 | 4); | |
1111 | break; | |
1112 | case CHIP_VEGA10: | |
1113 | case CHIP_VEGA12: | |
1114 | case CHIP_VEGA20: | |
1115 | default: | |
1116 | viewport = RREG32_SOC15(DCE, 0, mmSCL0_VIEWPORT_SIZE); | |
1117 | size = (REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_HEIGHT) * | |
1118 | REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_WIDTH) * | |
1119 | 4); | |
1120 | break; | |
1121 | } | |
1122 | } | |
1123 | ||
1124 | return size; | |
1125 | } | |
1126 | ||
132f34e4 CK |
1127 | static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = { |
1128 | .flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb, | |
ea930000 | 1129 | .flush_gpu_tlb_pasid = gmc_v9_0_flush_gpu_tlb_pasid, |
9096d6e5 | 1130 | .emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb, |
c633c00b | 1131 | .emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping, |
71776b6d | 1132 | .map_mtype = gmc_v9_0_map_mtype, |
cbfae36c | 1133 | .get_vm_pde = gmc_v9_0_get_vm_pde, |
7b885f0e AD |
1134 | .get_vm_pte = gmc_v9_0_get_vm_pte, |
1135 | .get_vbios_fb_size = gmc_v9_0_get_vbios_fb_size, | |
e60f8db5 AX |
1136 | }; |
1137 | ||
132f34e4 | 1138 | static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev) |
e60f8db5 | 1139 | { |
f54b30d7 | 1140 | adev->gmc.gmc_funcs = &gmc_v9_0_gmc_funcs; |
e60f8db5 AX |
1141 | } |
1142 | ||
5b6b35aa HZ |
1143 | static void gmc_v9_0_set_umc_funcs(struct amdgpu_device *adev) |
1144 | { | |
1145 | switch (adev->asic_type) { | |
e7da754b ML |
1146 | case CHIP_VEGA10: |
1147 | adev->umc.funcs = &umc_v6_0_funcs; | |
1148 | break; | |
5b6b35aa | 1149 | case CHIP_VEGA20: |
3aacf4ea TZ |
1150 | adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM; |
1151 | adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM; | |
1152 | adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM; | |
4cf781c2 JC |
1153 | adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_VG20; |
1154 | adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0]; | |
1155 | adev->umc.funcs = &umc_v6_1_funcs; | |
1156 | break; | |
9e612c11 | 1157 | case CHIP_ARCTURUS: |
3aacf4ea TZ |
1158 | adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM; |
1159 | adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM; | |
1160 | adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM; | |
4cf781c2 | 1161 | adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_ARCT; |
3aacf4ea | 1162 | adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0]; |
045c0216 | 1163 | adev->umc.funcs = &umc_v6_1_funcs; |
5b6b35aa HZ |
1164 | break; |
1165 | default: | |
1166 | break; | |
1167 | } | |
1168 | } | |
1169 | ||
3d093da0 TZ |
1170 | static void gmc_v9_0_set_mmhub_funcs(struct amdgpu_device *adev) |
1171 | { | |
1172 | switch (adev->asic_type) { | |
f6c3623b DL |
1173 | case CHIP_ARCTURUS: |
1174 | adev->mmhub.funcs = &mmhub_v9_4_funcs; | |
1175 | break; | |
4da999cd OZ |
1176 | case CHIP_ALDEBARAN: |
1177 | adev->mmhub.funcs = &mmhub_v1_7_funcs; | |
1178 | break; | |
3d093da0 | 1179 | default: |
9fb1506e | 1180 | adev->mmhub.funcs = &mmhub_v1_0_funcs; |
3d093da0 TZ |
1181 | break; |
1182 | } | |
1183 | } | |
1184 | ||
8ffff9b4 OZ |
1185 | static void gmc_v9_0_set_gfxhub_funcs(struct amdgpu_device *adev) |
1186 | { | |
21470d97 | 1187 | adev->gfxhub.funcs = &gfxhub_v1_0_funcs; |
8ffff9b4 OZ |
1188 | } |
1189 | ||
e60f8db5 AX |
1190 | static int gmc_v9_0_early_init(void *handle) |
1191 | { | |
1192 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
1193 | ||
132f34e4 | 1194 | gmc_v9_0_set_gmc_funcs(adev); |
e60f8db5 | 1195 | gmc_v9_0_set_irq_funcs(adev); |
5b6b35aa | 1196 | gmc_v9_0_set_umc_funcs(adev); |
3d093da0 | 1197 | gmc_v9_0_set_mmhub_funcs(adev); |
8ffff9b4 | 1198 | gmc_v9_0_set_gfxhub_funcs(adev); |
e60f8db5 | 1199 | |
31691b8d RB |
1200 | if (adev->asic_type == CHIP_VEGA20 || |
1201 | adev->asic_type == CHIP_ARCTURUS) | |
1202 | adev->gmc.xgmi.supported = true; | |
1203 | ||
1204 | if (adev->asic_type == CHIP_ALDEBARAN) { | |
1205 | adev->gmc.xgmi.supported = true; | |
1206 | adev->gmc.xgmi.connected_to_cpu = | |
1207 | adev->smuio.funcs->is_host_gpu_xgmi_supported(adev); | |
1208 | } | |
1209 | ||
770d13b1 CK |
1210 | adev->gmc.shared_aperture_start = 0x2000000000000000ULL; |
1211 | adev->gmc.shared_aperture_end = | |
1212 | adev->gmc.shared_aperture_start + (4ULL << 30) - 1; | |
bfa8eea2 | 1213 | adev->gmc.private_aperture_start = 0x1000000000000000ULL; |
770d13b1 CK |
1214 | adev->gmc.private_aperture_end = |
1215 | adev->gmc.private_aperture_start + (4ULL << 30) - 1; | |
a7ea6548 | 1216 | |
e60f8db5 AX |
1217 | return 0; |
1218 | } | |
1219 | ||
c713a461 EQ |
1220 | static int gmc_v9_0_late_init(void *handle) |
1221 | { | |
1222 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
c5b6e585 | 1223 | int r; |
c713a461 | 1224 | |
bdbe90f0 | 1225 | r = amdgpu_gmc_allocate_vm_inv_eng(adev); |
c713a461 EQ |
1226 | if (r) |
1227 | return r; | |
4a20300b GC |
1228 | |
1229 | /* | |
1230 | * Workaround performance drop issue with VBIOS enables partial | |
1231 | * writes, while disables HBM ECC for vega10. | |
1232 | */ | |
88474cca | 1233 | if (!amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_VEGA10)) { |
4a20300b | 1234 | if (!(adev->ras_features & (1 << AMDGPU_RAS_BLOCK__UMC))) { |
88474cca GC |
1235 | if (adev->df.funcs->enable_ecc_force_par_wr_rmw) |
1236 | adev->df.funcs->enable_ecc_force_par_wr_rmw(adev, false); | |
4a20300b | 1237 | } |
02bab923 DP |
1238 | } |
1239 | ||
fe5211f1 HZ |
1240 | if (adev->mmhub.funcs && adev->mmhub.funcs->reset_ras_error_count) |
1241 | adev->mmhub.funcs->reset_ras_error_count(adev); | |
1242 | ||
ba083492 | 1243 | r = amdgpu_gmc_ras_late_init(adev); |
791c4769 | 1244 | if (r) |
1245 | return r; | |
1246 | ||
770d13b1 | 1247 | return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0); |
e60f8db5 AX |
1248 | } |
1249 | ||
1250 | static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev, | |
770d13b1 | 1251 | struct amdgpu_gmc *mc) |
e60f8db5 | 1252 | { |
eeb2487d | 1253 | u64 base = 0; |
9d4f837a | 1254 | |
9fb1506e OZ |
1255 | if (!amdgpu_sriov_vf(adev)) |
1256 | base = adev->mmhub.funcs->get_fb_location(adev); | |
9d4f837a | 1257 | |
6fdd68b1 AD |
1258 | /* add the xgmi offset of the physical node */ |
1259 | base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; | |
f527f310 OZ |
1260 | if (adev->gmc.xgmi.connected_to_cpu) { |
1261 | amdgpu_gmc_sysvm_location(adev, mc); | |
1262 | } else { | |
1263 | amdgpu_gmc_vram_location(adev, mc, base); | |
1264 | amdgpu_gmc_gart_location(adev, mc); | |
1265 | amdgpu_gmc_agp_location(adev, mc); | |
1266 | } | |
bc099ee9 | 1267 | /* base offset of vram pages */ |
8ffff9b4 | 1268 | adev->vm_manager.vram_base_offset = adev->gfxhub.funcs->get_mc_fb_offset(adev); |
6fdd68b1 AD |
1269 | |
1270 | /* XXX: add the xgmi offset of the physical node? */ | |
1271 | adev->vm_manager.vram_base_offset += | |
1272 | adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; | |
e60f8db5 AX |
1273 | } |
1274 | ||
1275 | /** | |
1276 | * gmc_v9_0_mc_init - initialize the memory controller driver params | |
1277 | * | |
1278 | * @adev: amdgpu_device pointer | |
1279 | * | |
1280 | * Look up the amount of vram, vram width, and decide how to place | |
1281 | * vram and gart within the GPU's physical address space. | |
1282 | * Returns 0 for success. | |
1283 | */ | |
1284 | static int gmc_v9_0_mc_init(struct amdgpu_device *adev) | |
1285 | { | |
d6895ad3 | 1286 | int r; |
e60f8db5 | 1287 | |
e60f8db5 | 1288 | /* size in MB on si */ |
770d13b1 | 1289 | adev->gmc.mc_vram_size = |
bebc0762 | 1290 | adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL; |
770d13b1 | 1291 | adev->gmc.real_vram_size = adev->gmc.mc_vram_size; |
d6895ad3 | 1292 | |
be566196 OZ |
1293 | if (!(adev->flags & AMD_IS_APU) && |
1294 | !adev->gmc.xgmi.connected_to_cpu) { | |
d6895ad3 CK |
1295 | r = amdgpu_device_resize_fb_bar(adev); |
1296 | if (r) | |
1297 | return r; | |
1298 | } | |
770d13b1 CK |
1299 | adev->gmc.aper_base = pci_resource_start(adev->pdev, 0); |
1300 | adev->gmc.aper_size = pci_resource_len(adev->pdev, 0); | |
e60f8db5 | 1301 | |
156a81be | 1302 | #ifdef CONFIG_X86_64 |
31691b8d RB |
1303 | /* |
1304 | * AMD Accelerated Processing Platform (APP) supporting GPU-HOST xgmi | |
1305 | * interface can use VRAM through here as it appears system reserved | |
1306 | * memory in host address space. | |
1307 | * | |
1308 | * For APUs, VRAM is just the stolen system memory and can be accessed | |
1309 | * directly. | |
1310 | * | |
1311 | * Otherwise, use the legacy Host Data Path (HDP) through PCIe BAR. | |
1312 | */ | |
1313 | ||
1314 | /* check whether both host-gpu and gpu-gpu xgmi links exist */ | |
3de60d96 HZ |
1315 | if ((adev->flags & AMD_IS_APU) || |
1316 | (adev->gmc.xgmi.supported && | |
1317 | adev->gmc.xgmi.connected_to_cpu)) { | |
1318 | adev->gmc.aper_base = | |
1319 | adev->gfxhub.funcs->get_mc_fb_offset(adev) + | |
1320 | adev->gmc.xgmi.physical_node_id * | |
31691b8d | 1321 | adev->gmc.xgmi.node_segment_size; |
156a81be CZ |
1322 | adev->gmc.aper_size = adev->gmc.real_vram_size; |
1323 | } | |
31691b8d | 1324 | |
156a81be | 1325 | #endif |
e60f8db5 | 1326 | /* In case the PCI BAR is larger than the actual amount of vram */ |
770d13b1 CK |
1327 | adev->gmc.visible_vram_size = adev->gmc.aper_size; |
1328 | if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size) | |
1329 | adev->gmc.visible_vram_size = adev->gmc.real_vram_size; | |
e60f8db5 | 1330 | |
c3db7b5a AD |
1331 | /* set the gart size */ |
1332 | if (amdgpu_gart_size == -1) { | |
1333 | switch (adev->asic_type) { | |
1334 | case CHIP_VEGA10: /* all engines support GPUVM */ | |
273a14cd | 1335 | case CHIP_VEGA12: /* all engines support GPUVM */ |
d96b428c | 1336 | case CHIP_VEGA20: |
3de2ff5d | 1337 | case CHIP_ARCTURUS: |
85e39550 | 1338 | case CHIP_ALDEBARAN: |
c3db7b5a | 1339 | default: |
fe19b862 | 1340 | adev->gmc.gart_size = 512ULL << 20; |
c3db7b5a AD |
1341 | break; |
1342 | case CHIP_RAVEN: /* DCE SG support */ | |
8787ee01 | 1343 | case CHIP_RENOIR: |
770d13b1 | 1344 | adev->gmc.gart_size = 1024ULL << 20; |
c3db7b5a AD |
1345 | break; |
1346 | } | |
1347 | } else { | |
770d13b1 | 1348 | adev->gmc.gart_size = (u64)amdgpu_gart_size << 20; |
c3db7b5a AD |
1349 | } |
1350 | ||
f1dc12ca OZ |
1351 | adev->gmc.gart_size += adev->pm.smu_prv_buffer_size; |
1352 | ||
770d13b1 | 1353 | gmc_v9_0_vram_gtt_location(adev, &adev->gmc); |
e60f8db5 AX |
1354 | |
1355 | return 0; | |
1356 | } | |
1357 | ||
1358 | static int gmc_v9_0_gart_init(struct amdgpu_device *adev) | |
1359 | { | |
1360 | int r; | |
1361 | ||
1123b989 | 1362 | if (adev->gart.bo) { |
e60f8db5 AX |
1363 | WARN(1, "VEGA10 PCIE GART already initialized\n"); |
1364 | return 0; | |
1365 | } | |
7b454b3a OZ |
1366 | |
1367 | if (adev->gmc.xgmi.connected_to_cpu) { | |
1368 | adev->gmc.vmid0_page_table_depth = 1; | |
1369 | adev->gmc.vmid0_page_table_block_size = 12; | |
1370 | } else { | |
1371 | adev->gmc.vmid0_page_table_depth = 0; | |
1372 | adev->gmc.vmid0_page_table_block_size = 0; | |
1373 | } | |
1374 | ||
e60f8db5 AX |
1375 | /* Initialize common gart structure */ |
1376 | r = amdgpu_gart_init(adev); | |
1377 | if (r) | |
1378 | return r; | |
1379 | adev->gart.table_size = adev->gart.num_gpu_pages * 8; | |
7596ab68 | 1380 | adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE_VG10(MTYPE_UC) | |
e60f8db5 | 1381 | AMDGPU_PTE_EXECUTABLE; |
522510a6 OZ |
1382 | |
1383 | r = amdgpu_gart_table_vram_alloc(adev); | |
1384 | if (r) | |
1385 | return r; | |
1386 | ||
1387 | if (adev->gmc.xgmi.connected_to_cpu) { | |
1388 | r = amdgpu_gmc_pdb0_alloc(adev); | |
1389 | } | |
1390 | ||
1391 | return r; | |
e60f8db5 AX |
1392 | } |
1393 | ||
b0a2db9b AD |
1394 | /** |
1395 | * gmc_v9_0_save_registers - saves regs | |
1396 | * | |
1397 | * @adev: amdgpu_device pointer | |
1398 | * | |
1399 | * This saves potential register values that should be | |
1400 | * restored upon resume | |
1401 | */ | |
1402 | static void gmc_v9_0_save_registers(struct amdgpu_device *adev) | |
ebdef28e | 1403 | { |
b0a2db9b AD |
1404 | if (adev->asic_type == CHIP_RAVEN) |
1405 | adev->gmc.sdpif_register = RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0); | |
ebdef28e AD |
1406 | } |
1407 | ||
e60f8db5 AX |
1408 | static int gmc_v9_0_sw_init(void *handle) |
1409 | { | |
ad02e08e | 1410 | int r, vram_width = 0, vram_type = 0, vram_vendor = 0; |
e60f8db5 AX |
1411 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
1412 | ||
8ffff9b4 | 1413 | adev->gfxhub.funcs->init(adev); |
9fb1506e OZ |
1414 | |
1415 | adev->mmhub.funcs->init(adev); | |
0c8c0847 | 1416 | |
770d13b1 | 1417 | spin_lock_init(&adev->gmc.invalidate_lock); |
e60f8db5 | 1418 | |
ad02e08e OM |
1419 | r = amdgpu_atomfirmware_get_vram_info(adev, |
1420 | &vram_width, &vram_type, &vram_vendor); | |
631cdbd2 AD |
1421 | if (amdgpu_sriov_vf(adev)) |
1422 | /* For Vega10 SR-IOV, vram_width can't be read from ATOM as RAVEN, | |
1423 | * and DF related registers is not readable, seems hardcord is the | |
1424 | * only way to set the correct vram_width | |
1425 | */ | |
1426 | adev->gmc.vram_width = 2048; | |
1427 | else if (amdgpu_emu_mode != 1) | |
1428 | adev->gmc.vram_width = vram_width; | |
1429 | ||
1430 | if (!adev->gmc.vram_width) { | |
1431 | int chansize, numchan; | |
1432 | ||
1433 | /* hbm memory channel size */ | |
1434 | if (adev->flags & AMD_IS_APU) | |
1435 | chansize = 64; | |
1436 | else | |
1437 | chansize = 128; | |
1438 | ||
bdf84a80 | 1439 | numchan = adev->df.funcs->get_hbm_channel_number(adev); |
631cdbd2 AD |
1440 | adev->gmc.vram_width = numchan * chansize; |
1441 | } | |
1442 | ||
1443 | adev->gmc.vram_type = vram_type; | |
ad02e08e | 1444 | adev->gmc.vram_vendor = vram_vendor; |
fd66560b HZ |
1445 | switch (adev->asic_type) { |
1446 | case CHIP_RAVEN: | |
1daa2bfa LM |
1447 | adev->num_vmhubs = 2; |
1448 | ||
6a42fd6f | 1449 | if (adev->rev_id == 0x0 || adev->rev_id == 0x1) { |
f3368128 | 1450 | amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); |
6a42fd6f CK |
1451 | } else { |
1452 | /* vm_size is 128TB + 512GB for legacy 3-level page support */ | |
1453 | amdgpu_vm_adjust_size(adev, 128 * 1024 + 512, 9, 2, 48); | |
770d13b1 | 1454 | adev->gmc.translate_further = |
6a42fd6f CK |
1455 | adev->vm_manager.num_level > 1; |
1456 | } | |
fd66560b HZ |
1457 | break; |
1458 | case CHIP_VEGA10: | |
273a14cd | 1459 | case CHIP_VEGA12: |
d96b428c | 1460 | case CHIP_VEGA20: |
8787ee01 | 1461 | case CHIP_RENOIR: |
85e39550 | 1462 | case CHIP_ALDEBARAN: |
1daa2bfa LM |
1463 | adev->num_vmhubs = 2; |
1464 | ||
8787ee01 | 1465 | |
36b32a68 ZJ |
1466 | /* |
1467 | * To fulfill 4-level page support, | |
1468 | * vm size is 256TB (48bit), maximum size of Vega10, | |
1469 | * block size 512 (9bit) | |
1470 | */ | |
cdba61da | 1471 | /* sriov restrict max_pfn below AMDGPU_GMC_HOLE */ |
1472 | if (amdgpu_sriov_vf(adev)) | |
1473 | amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 47); | |
1474 | else | |
1475 | amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); | |
fd66560b | 1476 | break; |
3de2ff5d | 1477 | case CHIP_ARCTURUS: |
c8a6e2a3 LM |
1478 | adev->num_vmhubs = 3; |
1479 | ||
3de2ff5d LM |
1480 | /* Keep the vm size same with Vega20 */ |
1481 | amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); | |
1482 | break; | |
fd66560b HZ |
1483 | default: |
1484 | break; | |
e60f8db5 AX |
1485 | } |
1486 | ||
1487 | /* This interrupt is VMC page fault.*/ | |
44a99b65 | 1488 | r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC, VMC_1_0__SRCID__VM_FAULT, |
770d13b1 | 1489 | &adev->gmc.vm_fault); |
30da7bb1 CK |
1490 | if (r) |
1491 | return r; | |
1492 | ||
7d19b15f LM |
1493 | if (adev->asic_type == CHIP_ARCTURUS) { |
1494 | r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC1, VMC_1_0__SRCID__VM_FAULT, | |
1495 | &adev->gmc.vm_fault); | |
1496 | if (r) | |
1497 | return r; | |
1498 | } | |
1499 | ||
44a99b65 | 1500 | r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2, UTCL2_1_0__SRCID__FAULT, |
770d13b1 | 1501 | &adev->gmc.vm_fault); |
e60f8db5 AX |
1502 | |
1503 | if (r) | |
1504 | return r; | |
1505 | ||
2ee9403e ZL |
1506 | if (!amdgpu_sriov_vf(adev)) { |
1507 | /* interrupt sent to DF. */ | |
1508 | r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DF, 0, | |
1509 | &adev->gmc.ecc_irq); | |
1510 | if (r) | |
1511 | return r; | |
1512 | } | |
791c4769 | 1513 | |
e60f8db5 AX |
1514 | /* Set the internal MC address mask |
1515 | * This is the max address of the GPU's | |
1516 | * internal address space. | |
1517 | */ | |
770d13b1 | 1518 | adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */ |
e60f8db5 | 1519 | |
244511f3 | 1520 | r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(44)); |
e60f8db5 | 1521 | if (r) { |
e60f8db5 | 1522 | printk(KERN_WARNING "amdgpu: No suitable DMA available.\n"); |
244511f3 | 1523 | return r; |
e60f8db5 | 1524 | } |
244511f3 | 1525 | adev->need_swiotlb = drm_need_swiotlb(44); |
e60f8db5 | 1526 | |
47622ba0 | 1527 | if (adev->gmc.xgmi.supported) { |
8ffff9b4 | 1528 | r = adev->gfxhub.funcs->get_xgmi_info(adev); |
bf0a60b7 AD |
1529 | if (r) |
1530 | return r; | |
1531 | } | |
1532 | ||
e60f8db5 AX |
1533 | r = gmc_v9_0_mc_init(adev); |
1534 | if (r) | |
1535 | return r; | |
1536 | ||
7b885f0e | 1537 | amdgpu_gmc_get_vbios_allocations(adev); |
ebdef28e | 1538 | |
e60f8db5 AX |
1539 | /* Memory manager */ |
1540 | r = amdgpu_bo_init(adev); | |
1541 | if (r) | |
1542 | return r; | |
1543 | ||
1544 | r = gmc_v9_0_gart_init(adev); | |
1545 | if (r) | |
1546 | return r; | |
1547 | ||
05ec3eda CK |
1548 | /* |
1549 | * number of VMs | |
1550 | * VMID 0 is reserved for System | |
81659b20 FK |
1551 | * amdgpu graphics/compute will use VMIDs 1..n-1 |
1552 | * amdkfd will use VMIDs n..15 | |
1553 | * | |
1554 | * The first KFD VMID is 8 for GPUs with graphics, 3 for | |
1555 | * compute-only GPUs. On compute-only GPUs that leaves 2 VMIDs | |
1556 | * for video processing. | |
05ec3eda | 1557 | */ |
81659b20 FK |
1558 | adev->vm_manager.first_kfd_vmid = |
1559 | adev->asic_type == CHIP_ARCTURUS ? 3 : 8; | |
05ec3eda | 1560 | |
05ec3eda CK |
1561 | amdgpu_vm_manager_init(adev); |
1562 | ||
b0a2db9b AD |
1563 | gmc_v9_0_save_registers(adev); |
1564 | ||
05ec3eda | 1565 | return 0; |
e60f8db5 AX |
1566 | } |
1567 | ||
e60f8db5 AX |
1568 | static int gmc_v9_0_sw_fini(void *handle) |
1569 | { | |
1570 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
1571 | ||
2adf1344 | 1572 | amdgpu_gmc_ras_fini(adev); |
f59548c8 | 1573 | amdgpu_gem_force_release(adev); |
05ec3eda | 1574 | amdgpu_vm_manager_fini(adev); |
a3d9103e | 1575 | amdgpu_gart_table_vram_free(adev); |
e60f8db5 | 1576 | amdgpu_bo_fini(adev); |
a3d9103e | 1577 | amdgpu_gart_fini(adev); |
522510a6 OZ |
1578 | if (adev->gmc.pdb0_bo) |
1579 | amdgpu_bo_unref(&adev->gmc.pdb0_bo); | |
e60f8db5 AX |
1580 | |
1581 | return 0; | |
1582 | } | |
1583 | ||
1584 | static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev) | |
1585 | { | |
946a4d5b | 1586 | |
e60f8db5 AX |
1587 | switch (adev->asic_type) { |
1588 | case CHIP_VEGA10: | |
4cd4c5c0 | 1589 | if (amdgpu_sriov_vf(adev)) |
98cad2de | 1590 | break; |
df561f66 | 1591 | fallthrough; |
d96b428c | 1592 | case CHIP_VEGA20: |
946a4d5b | 1593 | soc15_program_register_sequence(adev, |
5c583018 | 1594 | golden_settings_mmhub_1_0_0, |
c47b41a7 | 1595 | ARRAY_SIZE(golden_settings_mmhub_1_0_0)); |
946a4d5b | 1596 | soc15_program_register_sequence(adev, |
5c583018 | 1597 | golden_settings_athub_1_0_0, |
c47b41a7 | 1598 | ARRAY_SIZE(golden_settings_athub_1_0_0)); |
e60f8db5 | 1599 | break; |
273a14cd AD |
1600 | case CHIP_VEGA12: |
1601 | break; | |
e4f3abaa | 1602 | case CHIP_RAVEN: |
8787ee01 | 1603 | /* TODO for renoir */ |
946a4d5b | 1604 | soc15_program_register_sequence(adev, |
5c583018 | 1605 | golden_settings_athub_1_0_0, |
c47b41a7 | 1606 | ARRAY_SIZE(golden_settings_athub_1_0_0)); |
e4f3abaa | 1607 | break; |
e60f8db5 AX |
1608 | default: |
1609 | break; | |
1610 | } | |
1611 | } | |
1612 | ||
c2ecd79b S |
1613 | /** |
1614 | * gmc_v9_0_restore_registers - restores regs | |
1615 | * | |
1616 | * @adev: amdgpu_device pointer | |
1617 | * | |
1618 | * This restores register values, saved at suspend. | |
1619 | */ | |
b0a2db9b | 1620 | void gmc_v9_0_restore_registers(struct amdgpu_device *adev) |
c2ecd79b | 1621 | { |
0eaa8012 | 1622 | if (adev->asic_type == CHIP_RAVEN) { |
f8646661 | 1623 | WREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0, adev->gmc.sdpif_register); |
0eaa8012 S |
1624 | WARN_ON(adev->gmc.sdpif_register != |
1625 | RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0)); | |
1626 | } | |
c2ecd79b S |
1627 | } |
1628 | ||
e60f8db5 AX |
1629 | /** |
1630 | * gmc_v9_0_gart_enable - gart enable | |
1631 | * | |
1632 | * @adev: amdgpu_device pointer | |
1633 | */ | |
1634 | static int gmc_v9_0_gart_enable(struct amdgpu_device *adev) | |
1635 | { | |
cb1545f7 | 1636 | int r; |
e60f8db5 | 1637 | |
522510a6 OZ |
1638 | if (adev->gmc.xgmi.connected_to_cpu) |
1639 | amdgpu_gmc_init_pdb0(adev); | |
1640 | ||
1123b989 | 1641 | if (adev->gart.bo == NULL) { |
e60f8db5 AX |
1642 | dev_err(adev->dev, "No VRAM object for PCIE GART.\n"); |
1643 | return -EINVAL; | |
1644 | } | |
522510a6 | 1645 | |
ce1b1b66 ML |
1646 | r = amdgpu_gart_table_vram_pin(adev); |
1647 | if (r) | |
1648 | return r; | |
e60f8db5 | 1649 | |
8ffff9b4 | 1650 | r = adev->gfxhub.funcs->gart_enable(adev); |
e60f8db5 AX |
1651 | if (r) |
1652 | return r; | |
1653 | ||
9fb1506e | 1654 | r = adev->mmhub.funcs->gart_enable(adev); |
e60f8db5 AX |
1655 | if (r) |
1656 | return r; | |
1657 | ||
522510a6 OZ |
1658 | DRM_INFO("PCIE GART of %uM enabled.\n", |
1659 | (unsigned)(adev->gmc.gart_size >> 20)); | |
1660 | if (adev->gmc.pdb0_bo) | |
1661 | DRM_INFO("PDB0 located at 0x%016llX\n", | |
1662 | (unsigned long long)amdgpu_bo_gpu_offset(adev->gmc.pdb0_bo)); | |
1663 | DRM_INFO("PTB located at 0x%016llX\n", | |
1664 | (unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo)); | |
1665 | ||
cb1545f7 OZ |
1666 | adev->gart.ready = true; |
1667 | return 0; | |
1668 | } | |
1669 | ||
1670 | static int gmc_v9_0_hw_init(void *handle) | |
1671 | { | |
1672 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
1673 | bool value; | |
1674 | int r, i; | |
cb1545f7 OZ |
1675 | |
1676 | /* The sequence of these two function calls matters.*/ | |
1677 | gmc_v9_0_init_golden_registers(adev); | |
1678 | ||
1679 | if (adev->mode_info.num_crtc) { | |
d0f2f634 HZ |
1680 | /* Lockout access through VGA aperture*/ |
1681 | WREG32_FIELD15(DCE, 0, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1); | |
1682 | /* disable VGA render */ | |
1683 | WREG32_FIELD15(DCE, 0, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0); | |
cb1545f7 OZ |
1684 | } |
1685 | ||
9fb1506e OZ |
1686 | if (adev->mmhub.funcs->update_power_gating) |
1687 | adev->mmhub.funcs->update_power_gating(adev, true); | |
1688 | ||
455d40c9 | 1689 | adev->hdp.funcs->init_registers(adev); |
fe2b5323 | 1690 | |
1d4e0a8c | 1691 | /* After HDP is initialized, flush HDP.*/ |
455d40c9 | 1692 | adev->hdp.funcs->flush_hdp(adev, NULL); |
1d4e0a8c | 1693 | |
e60f8db5 AX |
1694 | if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) |
1695 | value = false; | |
1696 | else | |
1697 | value = true; | |
1698 | ||
20bf2f6f | 1699 | if (!amdgpu_sriov_vf(adev)) { |
8ffff9b4 | 1700 | adev->gfxhub.funcs->set_fault_enable_default(adev, value); |
9fb1506e | 1701 | adev->mmhub.funcs->set_fault_enable_default(adev, value); |
20bf2f6f | 1702 | } |
3ff98548 OZ |
1703 | for (i = 0; i < adev->num_vmhubs; ++i) |
1704 | gmc_v9_0_flush_gpu_tlb(adev, 0, i, 0); | |
e60f8db5 | 1705 | |
e7da754b ML |
1706 | if (adev->umc.funcs && adev->umc.funcs->init_registers) |
1707 | adev->umc.funcs->init_registers(adev); | |
1708 | ||
e60f8db5 AX |
1709 | r = gmc_v9_0_gart_enable(adev); |
1710 | ||
1711 | return r; | |
1712 | } | |
1713 | ||
1714 | /** | |
1715 | * gmc_v9_0_gart_disable - gart disable | |
1716 | * | |
1717 | * @adev: amdgpu_device pointer | |
1718 | * | |
1719 | * This disables all VM page table. | |
1720 | */ | |
1721 | static void gmc_v9_0_gart_disable(struct amdgpu_device *adev) | |
1722 | { | |
8ffff9b4 | 1723 | adev->gfxhub.funcs->gart_disable(adev); |
9fb1506e | 1724 | adev->mmhub.funcs->gart_disable(adev); |
ce1b1b66 | 1725 | amdgpu_gart_table_vram_unpin(adev); |
e60f8db5 AX |
1726 | } |
1727 | ||
1728 | static int gmc_v9_0_hw_fini(void *handle) | |
1729 | { | |
1730 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
1731 | ||
5dd696ae TH |
1732 | if (amdgpu_sriov_vf(adev)) { |
1733 | /* full access mode, so don't touch any GMC register */ | |
1734 | DRM_DEBUG("For SRIOV client, shouldn't do anything.\n"); | |
1735 | return 0; | |
1736 | } | |
1737 | ||
791c4769 | 1738 | amdgpu_irq_put(adev, &adev->gmc.ecc_irq, 0); |
770d13b1 | 1739 | amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0); |
e60f8db5 AX |
1740 | gmc_v9_0_gart_disable(adev); |
1741 | ||
1742 | return 0; | |
1743 | } | |
1744 | ||
1745 | static int gmc_v9_0_suspend(void *handle) | |
1746 | { | |
1747 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
1748 | ||
c24a3c05 | 1749 | return gmc_v9_0_hw_fini(adev); |
e60f8db5 AX |
1750 | } |
1751 | ||
1752 | static int gmc_v9_0_resume(void *handle) | |
1753 | { | |
1754 | int r; | |
1755 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
1756 | ||
1757 | r = gmc_v9_0_hw_init(adev); | |
1758 | if (r) | |
1759 | return r; | |
1760 | ||
620f774f | 1761 | amdgpu_vmid_reset_all(adev); |
e60f8db5 | 1762 | |
32601d48 | 1763 | return 0; |
e60f8db5 AX |
1764 | } |
1765 | ||
1766 | static bool gmc_v9_0_is_idle(void *handle) | |
1767 | { | |
1768 | /* MC is always ready in GMC v9.*/ | |
1769 | return true; | |
1770 | } | |
1771 | ||
1772 | static int gmc_v9_0_wait_for_idle(void *handle) | |
1773 | { | |
1774 | /* There is no need to wait for MC idle in GMC v9.*/ | |
1775 | return 0; | |
1776 | } | |
1777 | ||
1778 | static int gmc_v9_0_soft_reset(void *handle) | |
1779 | { | |
1780 | /* XXX for emulation.*/ | |
1781 | return 0; | |
1782 | } | |
1783 | ||
1784 | static int gmc_v9_0_set_clockgating_state(void *handle, | |
1785 | enum amd_clockgating_state state) | |
1786 | { | |
d5583d4f HR |
1787 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
1788 | ||
9fb1506e | 1789 | adev->mmhub.funcs->set_clockgating(adev, state); |
bee7b51a LM |
1790 | |
1791 | athub_v1_0_set_clockgating(adev, state); | |
1792 | ||
1793 | return 0; | |
e60f8db5 AX |
1794 | } |
1795 | ||
13052be5 HR |
1796 | static void gmc_v9_0_get_clockgating_state(void *handle, u32 *flags) |
1797 | { | |
1798 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
1799 | ||
9fb1506e | 1800 | adev->mmhub.funcs->get_clockgating(adev, flags); |
bee7b51a LM |
1801 | |
1802 | athub_v1_0_get_clockgating(adev, flags); | |
13052be5 HR |
1803 | } |
1804 | ||
e60f8db5 AX |
1805 | static int gmc_v9_0_set_powergating_state(void *handle, |
1806 | enum amd_powergating_state state) | |
1807 | { | |
1808 | return 0; | |
1809 | } | |
1810 | ||
1811 | const struct amd_ip_funcs gmc_v9_0_ip_funcs = { | |
1812 | .name = "gmc_v9_0", | |
1813 | .early_init = gmc_v9_0_early_init, | |
1814 | .late_init = gmc_v9_0_late_init, | |
1815 | .sw_init = gmc_v9_0_sw_init, | |
1816 | .sw_fini = gmc_v9_0_sw_fini, | |
1817 | .hw_init = gmc_v9_0_hw_init, | |
1818 | .hw_fini = gmc_v9_0_hw_fini, | |
1819 | .suspend = gmc_v9_0_suspend, | |
1820 | .resume = gmc_v9_0_resume, | |
1821 | .is_idle = gmc_v9_0_is_idle, | |
1822 | .wait_for_idle = gmc_v9_0_wait_for_idle, | |
1823 | .soft_reset = gmc_v9_0_soft_reset, | |
1824 | .set_clockgating_state = gmc_v9_0_set_clockgating_state, | |
1825 | .set_powergating_state = gmc_v9_0_set_powergating_state, | |
13052be5 | 1826 | .get_clockgating_state = gmc_v9_0_get_clockgating_state, |
e60f8db5 AX |
1827 | }; |
1828 | ||
1829 | const struct amdgpu_ip_block_version gmc_v9_0_ip_block = | |
1830 | { | |
1831 | .type = AMD_IP_BLOCK_TYPE_GMC, | |
1832 | .major = 9, | |
1833 | .minor = 0, | |
1834 | .rev = 0, | |
1835 | .funcs = &gmc_v9_0_ip_funcs, | |
1836 | }; |