]>
Commit | Line | Data |
---|---|---|
e60f8db5 AX |
1 | /* |
2 | * Copyright 2016 Advanced Micro Devices, Inc. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | * | |
22 | */ | |
f867723b | 23 | |
e60f8db5 | 24 | #include <linux/firmware.h> |
f867723b SR |
25 | #include <linux/pci.h> |
26 | ||
fd5fd480 | 27 | #include <drm/drm_cache.h> |
f867723b | 28 | |
e60f8db5 AX |
29 | #include "amdgpu.h" |
30 | #include "gmc_v9_0.h" | |
8d6a5230 | 31 | #include "amdgpu_atomfirmware.h" |
2cddc50e | 32 | #include "amdgpu_gem.h" |
e60f8db5 | 33 | |
cde5c34f | 34 | #include "gc/gc_9_0_sh_mask.h" |
135d4b10 FX |
35 | #include "dce/dce_12_0_offset.h" |
36 | #include "dce/dce_12_0_sh_mask.h" | |
fb960bd2 | 37 | #include "vega10_enum.h" |
65417d9f | 38 | #include "mmhub/mmhub_1_0_offset.h" |
ea930000 | 39 | #include "athub/athub_1_0_sh_mask.h" |
6ce68225 | 40 | #include "athub/athub_1_0_offset.h" |
250b4228 | 41 | #include "oss/osssys_4_0_offset.h" |
e60f8db5 | 42 | |
946a4d5b | 43 | #include "soc15.h" |
ea930000 | 44 | #include "soc15d.h" |
e60f8db5 | 45 | #include "soc15_common.h" |
90c7a935 | 46 | #include "umc/umc_6_0_sh_mask.h" |
e60f8db5 | 47 | |
e60f8db5 AX |
48 | #include "gfxhub_v1_0.h" |
49 | #include "mmhub_v1_0.h" | |
bee7b51a | 50 | #include "athub_v1_0.h" |
bf0a60b7 | 51 | #include "gfxhub_v1_1.h" |
51cce480 | 52 | #include "mmhub_v9_4.h" |
85e39550 | 53 | #include "mmhub_v1_7.h" |
5b6b35aa | 54 | #include "umc_v6_1.h" |
e7da754b | 55 | #include "umc_v6_0.h" |
186c8a85 | 56 | #include "umc_v6_7.h" |
6f12507f | 57 | #include "hdp_v4_0.h" |
3907c492 | 58 | #include "mca_v3_0.h" |
e60f8db5 | 59 | |
44a99b65 AG |
60 | #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h" |
61 | ||
791c4769 | 62 | #include "amdgpu_ras.h" |
029fbd43 | 63 | #include "amdgpu_xgmi.h" |
791c4769 | 64 | |
ebdef28e AD |
65 | /* add these here since we already include dce12 headers and these are for DCN */ |
66 | #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION 0x055d | |
67 | #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX 2 | |
68 | #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH__SHIFT 0x0 | |
69 | #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT__SHIFT 0x10 | |
70 | #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH_MASK 0x00003FFFL | |
71 | #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT_MASK 0x3FFF0000L | |
f8646661 AD |
72 | #define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0 0x049d |
73 | #define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0_BASE_IDX 2 | |
74 | ||
de44ce64 HW |
75 | #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_DCN2 0x05ea |
76 | #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_DCN2_BASE_IDX 2 | |
77 | ||
ebdef28e | 78 | |
be99ecbf AD |
79 | static const char *gfxhub_client_ids[] = { |
80 | "CB", | |
81 | "DB", | |
82 | "IA", | |
83 | "WD", | |
84 | "CPF", | |
85 | "CPC", | |
86 | "CPG", | |
87 | "RLC", | |
88 | "TCP", | |
89 | "SQC (inst)", | |
90 | "SQC (data)", | |
91 | "SQG", | |
92 | "PA", | |
93 | }; | |
94 | ||
02f23f5f AD |
95 | static const char *mmhub_client_ids_raven[][2] = { |
96 | [0][0] = "MP1", | |
97 | [1][0] = "MP0", | |
98 | [2][0] = "VCN", | |
99 | [3][0] = "VCNU", | |
100 | [4][0] = "HDP", | |
101 | [5][0] = "DCE", | |
102 | [13][0] = "UTCL2", | |
103 | [19][0] = "TLS", | |
104 | [26][0] = "OSS", | |
105 | [27][0] = "SDMA0", | |
106 | [0][1] = "MP1", | |
107 | [1][1] = "MP0", | |
108 | [2][1] = "VCN", | |
109 | [3][1] = "VCNU", | |
110 | [4][1] = "HDP", | |
111 | [5][1] = "XDP", | |
112 | [6][1] = "DBGU0", | |
113 | [7][1] = "DCE", | |
114 | [8][1] = "DCEDWB0", | |
115 | [9][1] = "DCEDWB1", | |
116 | [26][1] = "OSS", | |
117 | [27][1] = "SDMA0", | |
118 | }; | |
119 | ||
120 | static const char *mmhub_client_ids_renoir[][2] = { | |
121 | [0][0] = "MP1", | |
122 | [1][0] = "MP0", | |
123 | [2][0] = "HDP", | |
124 | [4][0] = "DCEDMC", | |
125 | [5][0] = "DCEVGA", | |
126 | [13][0] = "UTCL2", | |
127 | [19][0] = "TLS", | |
128 | [26][0] = "OSS", | |
129 | [27][0] = "SDMA0", | |
130 | [28][0] = "VCN", | |
131 | [29][0] = "VCNU", | |
132 | [30][0] = "JPEG", | |
133 | [0][1] = "MP1", | |
134 | [1][1] = "MP0", | |
135 | [2][1] = "HDP", | |
136 | [3][1] = "XDP", | |
137 | [6][1] = "DBGU0", | |
138 | [7][1] = "DCEDMC", | |
139 | [8][1] = "DCEVGA", | |
140 | [9][1] = "DCEDWB", | |
141 | [26][1] = "OSS", | |
142 | [27][1] = "SDMA0", | |
143 | [28][1] = "VCN", | |
144 | [29][1] = "VCNU", | |
145 | [30][1] = "JPEG", | |
146 | }; | |
147 | ||
148 | static const char *mmhub_client_ids_vega10[][2] = { | |
149 | [0][0] = "MP0", | |
150 | [1][0] = "UVD", | |
151 | [2][0] = "UVDU", | |
152 | [3][0] = "HDP", | |
153 | [13][0] = "UTCL2", | |
154 | [14][0] = "OSS", | |
155 | [15][0] = "SDMA1", | |
156 | [32+0][0] = "VCE0", | |
157 | [32+1][0] = "VCE0U", | |
158 | [32+2][0] = "XDMA", | |
159 | [32+3][0] = "DCE", | |
160 | [32+4][0] = "MP1", | |
161 | [32+14][0] = "SDMA0", | |
162 | [0][1] = "MP0", | |
163 | [1][1] = "UVD", | |
164 | [2][1] = "UVDU", | |
165 | [3][1] = "DBGU0", | |
166 | [4][1] = "HDP", | |
167 | [5][1] = "XDP", | |
168 | [14][1] = "OSS", | |
169 | [15][1] = "SDMA0", | |
170 | [32+0][1] = "VCE0", | |
171 | [32+1][1] = "VCE0U", | |
172 | [32+2][1] = "XDMA", | |
173 | [32+3][1] = "DCE", | |
174 | [32+4][1] = "DCEDWB", | |
175 | [32+5][1] = "MP1", | |
176 | [32+6][1] = "DBGU1", | |
177 | [32+14][1] = "SDMA1", | |
178 | }; | |
179 | ||
180 | static const char *mmhub_client_ids_vega12[][2] = { | |
181 | [0][0] = "MP0", | |
182 | [1][0] = "VCE0", | |
183 | [2][0] = "VCE0U", | |
184 | [3][0] = "HDP", | |
185 | [13][0] = "UTCL2", | |
186 | [14][0] = "OSS", | |
187 | [15][0] = "SDMA1", | |
188 | [32+0][0] = "DCE", | |
189 | [32+1][0] = "XDMA", | |
190 | [32+2][0] = "UVD", | |
191 | [32+3][0] = "UVDU", | |
192 | [32+4][0] = "MP1", | |
193 | [32+15][0] = "SDMA0", | |
194 | [0][1] = "MP0", | |
195 | [1][1] = "VCE0", | |
196 | [2][1] = "VCE0U", | |
197 | [3][1] = "DBGU0", | |
198 | [4][1] = "HDP", | |
199 | [5][1] = "XDP", | |
200 | [14][1] = "OSS", | |
201 | [15][1] = "SDMA0", | |
202 | [32+0][1] = "DCE", | |
203 | [32+1][1] = "DCEDWB", | |
204 | [32+2][1] = "XDMA", | |
205 | [32+3][1] = "UVD", | |
206 | [32+4][1] = "UVDU", | |
207 | [32+5][1] = "MP1", | |
208 | [32+6][1] = "DBGU1", | |
209 | [32+15][1] = "SDMA1", | |
210 | }; | |
211 | ||
212 | static const char *mmhub_client_ids_vega20[][2] = { | |
213 | [0][0] = "XDMA", | |
214 | [1][0] = "DCE", | |
215 | [2][0] = "VCE0", | |
216 | [3][0] = "VCE0U", | |
217 | [4][0] = "UVD", | |
218 | [5][0] = "UVD1U", | |
219 | [13][0] = "OSS", | |
220 | [14][0] = "HDP", | |
221 | [15][0] = "SDMA0", | |
222 | [32+0][0] = "UVD", | |
223 | [32+1][0] = "UVDU", | |
224 | [32+2][0] = "MP1", | |
225 | [32+3][0] = "MP0", | |
226 | [32+12][0] = "UTCL2", | |
227 | [32+14][0] = "SDMA1", | |
228 | [0][1] = "XDMA", | |
229 | [1][1] = "DCE", | |
230 | [2][1] = "DCEDWB", | |
231 | [3][1] = "VCE0", | |
232 | [4][1] = "VCE0U", | |
233 | [5][1] = "UVD1", | |
234 | [6][1] = "UVD1U", | |
235 | [7][1] = "DBGU0", | |
236 | [8][1] = "XDP", | |
237 | [13][1] = "OSS", | |
238 | [14][1] = "HDP", | |
239 | [15][1] = "SDMA0", | |
240 | [32+0][1] = "UVD", | |
241 | [32+1][1] = "UVDU", | |
242 | [32+2][1] = "DBGU1", | |
243 | [32+3][1] = "MP1", | |
244 | [32+4][1] = "MP0", | |
245 | [32+14][1] = "SDMA1", | |
246 | }; | |
247 | ||
248 | static const char *mmhub_client_ids_arcturus[][2] = { | |
e83db774 AD |
249 | [0][0] = "DBGU1", |
250 | [1][0] = "XDP", | |
02f23f5f | 251 | [2][0] = "MP1", |
02f23f5f | 252 | [14][0] = "HDP", |
e83db774 AD |
253 | [171][0] = "JPEG", |
254 | [172][0] = "VCN", | |
255 | [173][0] = "VCNU", | |
256 | [203][0] = "JPEG1", | |
257 | [204][0] = "VCN1", | |
258 | [205][0] = "VCN1U", | |
259 | [256][0] = "SDMA0", | |
260 | [257][0] = "SDMA1", | |
261 | [258][0] = "SDMA2", | |
262 | [259][0] = "SDMA3", | |
263 | [260][0] = "SDMA4", | |
264 | [261][0] = "SDMA5", | |
265 | [262][0] = "SDMA6", | |
266 | [263][0] = "SDMA7", | |
267 | [384][0] = "OSS", | |
02f23f5f AD |
268 | [0][1] = "DBGU1", |
269 | [1][1] = "XDP", | |
270 | [2][1] = "MP1", | |
02f23f5f | 271 | [14][1] = "HDP", |
e83db774 AD |
272 | [171][1] = "JPEG", |
273 | [172][1] = "VCN", | |
274 | [173][1] = "VCNU", | |
275 | [203][1] = "JPEG1", | |
276 | [204][1] = "VCN1", | |
277 | [205][1] = "VCN1U", | |
278 | [256][1] = "SDMA0", | |
279 | [257][1] = "SDMA1", | |
280 | [258][1] = "SDMA2", | |
281 | [259][1] = "SDMA3", | |
282 | [260][1] = "SDMA4", | |
283 | [261][1] = "SDMA5", | |
284 | [262][1] = "SDMA6", | |
285 | [263][1] = "SDMA7", | |
286 | [384][1] = "OSS", | |
02f23f5f | 287 | }; |
ebdef28e | 288 | |
e844cd99 AD |
289 | static const char *mmhub_client_ids_aldebaran[][2] = { |
290 | [2][0] = "MP1", | |
291 | [3][0] = "MP0", | |
f4ec3e50 AS |
292 | [32+1][0] = "DBGU_IO0", |
293 | [32+2][0] = "DBGU_IO2", | |
e844cd99 | 294 | [32+4][0] = "MPIO", |
e844cd99 AD |
295 | [96+11][0] = "JPEG0", |
296 | [96+12][0] = "VCN0", | |
297 | [96+13][0] = "VCNU0", | |
e844cd99 AD |
298 | [128+11][0] = "JPEG1", |
299 | [128+12][0] = "VCN1", | |
300 | [128+13][0] = "VCNU1", | |
f4ec3e50 | 301 | [160+1][0] = "XDP", |
e844cd99 | 302 | [160+14][0] = "HDP", |
f4ec3e50 AS |
303 | [256+0][0] = "SDMA0", |
304 | [256+1][0] = "SDMA1", | |
305 | [256+2][0] = "SDMA2", | |
306 | [256+3][0] = "SDMA3", | |
307 | [256+4][0] = "SDMA4", | |
308 | [384+0][0] = "OSS", | |
e844cd99 AD |
309 | [2][1] = "MP1", |
310 | [3][1] = "MP0", | |
e844cd99 AD |
311 | [32+1][1] = "DBGU_IO0", |
312 | [32+2][1] = "DBGU_IO2", | |
313 | [32+4][1] = "MPIO", | |
e844cd99 AD |
314 | [96+11][1] = "JPEG0", |
315 | [96+12][1] = "VCN0", | |
316 | [96+13][1] = "VCNU0", | |
e844cd99 AD |
317 | [128+11][1] = "JPEG1", |
318 | [128+12][1] = "VCN1", | |
319 | [128+13][1] = "VCNU1", | |
f4ec3e50 | 320 | [160+1][1] = "XDP", |
e844cd99 | 321 | [160+14][1] = "HDP", |
f4ec3e50 AS |
322 | [256+0][1] = "SDMA0", |
323 | [256+1][1] = "SDMA1", | |
324 | [256+2][1] = "SDMA2", | |
325 | [256+3][1] = "SDMA3", | |
326 | [256+4][1] = "SDMA4", | |
327 | [384+0][1] = "OSS", | |
e844cd99 AD |
328 | }; |
329 | ||
946a4d5b | 330 | static const struct soc15_reg_golden golden_settings_mmhub_1_0_0[] = |
5c583018 | 331 | { |
946a4d5b SL |
332 | SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmDAGB1_WRCLI2, 0x00000007, 0xfe5fe0fa), |
333 | SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmMMEA1_DRAM_WR_CLI2GRP_MAP0, 0x00000030, 0x55555565) | |
5c583018 EQ |
334 | }; |
335 | ||
946a4d5b | 336 | static const struct soc15_reg_golden golden_settings_athub_1_0_0[] = |
5c583018 | 337 | { |
946a4d5b SL |
338 | SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL, 0x0000ff00, 0x00000800), |
339 | SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL2, 0x00ff00ff, 0x00080008) | |
5c583018 EQ |
340 | }; |
341 | ||
791c4769 | 342 | static const uint32_t ecc_umc_mcumc_ctrl_addrs[] = { |
343 | (0x000143c0 + 0x00000000), | |
344 | (0x000143c0 + 0x00000800), | |
345 | (0x000143c0 + 0x00001000), | |
346 | (0x000143c0 + 0x00001800), | |
347 | (0x000543c0 + 0x00000000), | |
348 | (0x000543c0 + 0x00000800), | |
349 | (0x000543c0 + 0x00001000), | |
350 | (0x000543c0 + 0x00001800), | |
351 | (0x000943c0 + 0x00000000), | |
352 | (0x000943c0 + 0x00000800), | |
353 | (0x000943c0 + 0x00001000), | |
354 | (0x000943c0 + 0x00001800), | |
355 | (0x000d43c0 + 0x00000000), | |
356 | (0x000d43c0 + 0x00000800), | |
357 | (0x000d43c0 + 0x00001000), | |
358 | (0x000d43c0 + 0x00001800), | |
359 | (0x001143c0 + 0x00000000), | |
360 | (0x001143c0 + 0x00000800), | |
361 | (0x001143c0 + 0x00001000), | |
362 | (0x001143c0 + 0x00001800), | |
363 | (0x001543c0 + 0x00000000), | |
364 | (0x001543c0 + 0x00000800), | |
365 | (0x001543c0 + 0x00001000), | |
366 | (0x001543c0 + 0x00001800), | |
367 | (0x001943c0 + 0x00000000), | |
368 | (0x001943c0 + 0x00000800), | |
369 | (0x001943c0 + 0x00001000), | |
370 | (0x001943c0 + 0x00001800), | |
371 | (0x001d43c0 + 0x00000000), | |
372 | (0x001d43c0 + 0x00000800), | |
373 | (0x001d43c0 + 0x00001000), | |
374 | (0x001d43c0 + 0x00001800), | |
02bab923 DP |
375 | }; |
376 | ||
791c4769 | 377 | static const uint32_t ecc_umc_mcumc_ctrl_mask_addrs[] = { |
378 | (0x000143e0 + 0x00000000), | |
379 | (0x000143e0 + 0x00000800), | |
380 | (0x000143e0 + 0x00001000), | |
381 | (0x000143e0 + 0x00001800), | |
382 | (0x000543e0 + 0x00000000), | |
383 | (0x000543e0 + 0x00000800), | |
384 | (0x000543e0 + 0x00001000), | |
385 | (0x000543e0 + 0x00001800), | |
386 | (0x000943e0 + 0x00000000), | |
387 | (0x000943e0 + 0x00000800), | |
388 | (0x000943e0 + 0x00001000), | |
389 | (0x000943e0 + 0x00001800), | |
390 | (0x000d43e0 + 0x00000000), | |
391 | (0x000d43e0 + 0x00000800), | |
392 | (0x000d43e0 + 0x00001000), | |
393 | (0x000d43e0 + 0x00001800), | |
394 | (0x001143e0 + 0x00000000), | |
395 | (0x001143e0 + 0x00000800), | |
396 | (0x001143e0 + 0x00001000), | |
397 | (0x001143e0 + 0x00001800), | |
398 | (0x001543e0 + 0x00000000), | |
399 | (0x001543e0 + 0x00000800), | |
400 | (0x001543e0 + 0x00001000), | |
401 | (0x001543e0 + 0x00001800), | |
402 | (0x001943e0 + 0x00000000), | |
403 | (0x001943e0 + 0x00000800), | |
404 | (0x001943e0 + 0x00001000), | |
405 | (0x001943e0 + 0x00001800), | |
406 | (0x001d43e0 + 0x00000000), | |
407 | (0x001d43e0 + 0x00000800), | |
408 | (0x001d43e0 + 0x00001000), | |
409 | (0x001d43e0 + 0x00001800), | |
02bab923 DP |
410 | }; |
411 | ||
791c4769 | 412 | static int gmc_v9_0_ecc_interrupt_state(struct amdgpu_device *adev, |
413 | struct amdgpu_irq_src *src, | |
414 | unsigned type, | |
415 | enum amdgpu_interrupt_state state) | |
416 | { | |
417 | u32 bits, i, tmp, reg; | |
1e2c6d55 JC |
418 | |
419 | /* Devices newer then VEGA10/12 shall have these programming | |
420 | sequences performed by PSP BL */ | |
421 | if (adev->asic_type >= CHIP_VEGA20) | |
422 | return 0; | |
791c4769 | 423 | |
424 | bits = 0x7f; | |
425 | ||
426 | switch (state) { | |
427 | case AMDGPU_IRQ_STATE_DISABLE: | |
428 | for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) { | |
429 | reg = ecc_umc_mcumc_ctrl_addrs[i]; | |
430 | tmp = RREG32(reg); | |
431 | tmp &= ~bits; | |
432 | WREG32(reg, tmp); | |
433 | } | |
434 | for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) { | |
435 | reg = ecc_umc_mcumc_ctrl_mask_addrs[i]; | |
436 | tmp = RREG32(reg); | |
437 | tmp &= ~bits; | |
438 | WREG32(reg, tmp); | |
439 | } | |
440 | break; | |
441 | case AMDGPU_IRQ_STATE_ENABLE: | |
442 | for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) { | |
443 | reg = ecc_umc_mcumc_ctrl_addrs[i]; | |
444 | tmp = RREG32(reg); | |
445 | tmp |= bits; | |
446 | WREG32(reg, tmp); | |
447 | } | |
448 | for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) { | |
449 | reg = ecc_umc_mcumc_ctrl_mask_addrs[i]; | |
450 | tmp = RREG32(reg); | |
451 | tmp |= bits; | |
452 | WREG32(reg, tmp); | |
453 | } | |
454 | break; | |
455 | default: | |
456 | break; | |
457 | } | |
458 | ||
459 | return 0; | |
460 | } | |
461 | ||
e60f8db5 AX |
462 | static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev, |
463 | struct amdgpu_irq_src *src, | |
464 | unsigned type, | |
465 | enum amdgpu_interrupt_state state) | |
466 | { | |
467 | struct amdgpu_vmhub *hub; | |
ae6d1416 | 468 | u32 tmp, reg, bits, i, j; |
e60f8db5 | 469 | |
11250164 CK |
470 | bits = VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | |
471 | VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | | |
472 | VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | | |
473 | VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | | |
474 | VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | | |
475 | VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | | |
476 | VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK; | |
477 | ||
e60f8db5 AX |
478 | switch (state) { |
479 | case AMDGPU_IRQ_STATE_DISABLE: | |
1daa2bfa | 480 | for (j = 0; j < adev->num_vmhubs; j++) { |
ae6d1416 TSD |
481 | hub = &adev->vmhub[j]; |
482 | for (i = 0; i < 16; i++) { | |
483 | reg = hub->vm_context0_cntl + i; | |
484 | tmp = RREG32(reg); | |
485 | tmp &= ~bits; | |
486 | WREG32(reg, tmp); | |
487 | } | |
e60f8db5 AX |
488 | } |
489 | break; | |
490 | case AMDGPU_IRQ_STATE_ENABLE: | |
1daa2bfa | 491 | for (j = 0; j < adev->num_vmhubs; j++) { |
ae6d1416 TSD |
492 | hub = &adev->vmhub[j]; |
493 | for (i = 0; i < 16; i++) { | |
494 | reg = hub->vm_context0_cntl + i; | |
495 | tmp = RREG32(reg); | |
496 | tmp |= bits; | |
497 | WREG32(reg, tmp); | |
498 | } | |
e60f8db5 | 499 | } |
9304ca4d | 500 | break; |
e60f8db5 AX |
501 | default: |
502 | break; | |
503 | } | |
504 | ||
505 | return 0; | |
506 | } | |
507 | ||
508 | static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev, | |
e3898719 CK |
509 | struct amdgpu_irq_src *source, |
510 | struct amdgpu_iv_entry *entry) | |
e60f8db5 | 511 | { |
c468f9e2 | 512 | bool retry_fault = !!(entry->src_data[1] & 0x80); |
ff891a2e | 513 | bool write_fault = !!(entry->src_data[1] & 0x20); |
02f23f5f | 514 | uint32_t status = 0, cid = 0, rw = 0; |
e3898719 CK |
515 | struct amdgpu_task_info task_info; |
516 | struct amdgpu_vmhub *hub; | |
02f23f5f | 517 | const char *mmhub_cid; |
e3898719 CK |
518 | const char *hub_name; |
519 | u64 addr; | |
e60f8db5 AX |
520 | |
521 | addr = (u64)entry->src_data[0] << 12; | |
522 | addr |= ((u64)entry->src_data[1] & 0xf) << 44; | |
523 | ||
0291150d CK |
524 | if (retry_fault) { |
525 | /* Returning 1 here also prevents sending the IV to the KFD */ | |
526 | ||
527 | /* Process it onyl if it's the first fault for this address */ | |
528 | if (entry->ih != &adev->irq.ih_soft && | |
529 | amdgpu_gmc_filter_faults(adev, addr, entry->pasid, | |
530 | entry->timestamp)) | |
531 | return 1; | |
532 | ||
533 | /* Delegate it to a different ring if the hardware hasn't | |
534 | * already done it. | |
535 | */ | |
58df0d71 | 536 | if (entry->ih == &adev->irq.ih) { |
0291150d CK |
537 | amdgpu_irq_delegate(adev, entry, 8); |
538 | return 1; | |
539 | } | |
540 | ||
541 | /* Try to handle the recoverable page faults by filling page | |
542 | * tables | |
543 | */ | |
ff891a2e | 544 | if (amdgpu_vm_handle_fault(adev, entry->pasid, addr, write_fault)) |
0291150d CK |
545 | return 1; |
546 | } | |
e3898719 CK |
547 | |
548 | if (!printk_ratelimit()) | |
549 | return 0; | |
550 | ||
51c60898 | 551 | if (entry->client_id == SOC15_IH_CLIENTID_VMC) { |
e3898719 | 552 | hub_name = "mmhub0"; |
51c60898 LM |
553 | hub = &adev->vmhub[AMDGPU_MMHUB_0]; |
554 | } else if (entry->client_id == SOC15_IH_CLIENTID_VMC1) { | |
e3898719 | 555 | hub_name = "mmhub1"; |
51c60898 LM |
556 | hub = &adev->vmhub[AMDGPU_MMHUB_1]; |
557 | } else { | |
e3898719 | 558 | hub_name = "gfxhub0"; |
51c60898 LM |
559 | hub = &adev->vmhub[AMDGPU_GFXHUB_0]; |
560 | } | |
561 | ||
e3898719 CK |
562 | memset(&task_info, 0, sizeof(struct amdgpu_task_info)); |
563 | amdgpu_vm_get_task_info(adev, entry->pasid, &task_info); | |
ec671737 | 564 | |
e3898719 CK |
565 | dev_err(adev->dev, |
566 | "[%s] %s page fault (src_id:%u ring:%u vmid:%u " | |
567 | "pasid:%u, for process %s pid %d thread %s pid %d)\n", | |
568 | hub_name, retry_fault ? "retry" : "no-retry", | |
569 | entry->src_id, entry->ring_id, entry->vmid, | |
570 | entry->pasid, task_info.process_name, task_info.tgid, | |
571 | task_info.task_name, task_info.pid); | |
be14729a YZ |
572 | dev_err(adev->dev, " in page starting at address 0x%016llx from IH client 0x%x (%s)\n", |
573 | addr, entry->client_id, | |
574 | soc15_ih_clientid_name[entry->client_id]); | |
e60f8db5 | 575 | |
e3898719 CK |
576 | if (amdgpu_sriov_vf(adev)) |
577 | return 0; | |
578 | ||
579 | /* | |
580 | * Issue a dummy read to wait for the status register to | |
581 | * be updated to avoid reading an incorrect value due to | |
582 | * the new fast GRBM interface. | |
583 | */ | |
7845d80d AD |
584 | if ((entry->vmid_src == AMDGPU_GFXHUB_0) && |
585 | (adev->asic_type < CHIP_ALDEBARAN)) | |
e3898719 CK |
586 | RREG32(hub->vm_l2_pro_fault_status); |
587 | ||
588 | status = RREG32(hub->vm_l2_pro_fault_status); | |
589 | cid = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, CID); | |
590 | rw = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, RW); | |
591 | WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1); | |
592 | ||
593 | ||
594 | dev_err(adev->dev, | |
595 | "VM_L2_PROTECTION_FAULT_STATUS:0x%08X\n", | |
596 | status); | |
597 | if (hub == &adev->vmhub[AMDGPU_GFXHUB_0]) { | |
598 | dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n", | |
599 | cid >= ARRAY_SIZE(gfxhub_client_ids) ? "unknown" : | |
600 | gfxhub_client_ids[cid], | |
601 | cid); | |
602 | } else { | |
603 | switch (adev->asic_type) { | |
604 | case CHIP_VEGA10: | |
605 | mmhub_cid = mmhub_client_ids_vega10[cid][rw]; | |
606 | break; | |
607 | case CHIP_VEGA12: | |
608 | mmhub_cid = mmhub_client_ids_vega12[cid][rw]; | |
609 | break; | |
610 | case CHIP_VEGA20: | |
611 | mmhub_cid = mmhub_client_ids_vega20[cid][rw]; | |
612 | break; | |
613 | case CHIP_ARCTURUS: | |
614 | mmhub_cid = mmhub_client_ids_arcturus[cid][rw]; | |
615 | break; | |
616 | case CHIP_RAVEN: | |
617 | mmhub_cid = mmhub_client_ids_raven[cid][rw]; | |
618 | break; | |
619 | case CHIP_RENOIR: | |
620 | mmhub_cid = mmhub_client_ids_renoir[cid][rw]; | |
621 | break; | |
e844cd99 AD |
622 | case CHIP_ALDEBARAN: |
623 | mmhub_cid = mmhub_client_ids_aldebaran[cid][rw]; | |
624 | break; | |
e3898719 CK |
625 | default: |
626 | mmhub_cid = NULL; | |
627 | break; | |
5ddd4a9a | 628 | } |
e3898719 CK |
629 | dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n", |
630 | mmhub_cid ? mmhub_cid : "unknown", cid); | |
79a0c465 | 631 | } |
e3898719 CK |
632 | dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n", |
633 | REG_GET_FIELD(status, | |
634 | VM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS)); | |
635 | dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n", | |
636 | REG_GET_FIELD(status, | |
637 | VM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR)); | |
638 | dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n", | |
639 | REG_GET_FIELD(status, | |
640 | VM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS)); | |
641 | dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n", | |
642 | REG_GET_FIELD(status, | |
643 | VM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR)); | |
644 | dev_err(adev->dev, "\t RW: 0x%x\n", rw); | |
e60f8db5 AX |
645 | return 0; |
646 | } | |
647 | ||
648 | static const struct amdgpu_irq_src_funcs gmc_v9_0_irq_funcs = { | |
649 | .set = gmc_v9_0_vm_fault_interrupt_state, | |
650 | .process = gmc_v9_0_process_interrupt, | |
651 | }; | |
652 | ||
791c4769 | 653 | |
654 | static const struct amdgpu_irq_src_funcs gmc_v9_0_ecc_funcs = { | |
655 | .set = gmc_v9_0_ecc_interrupt_state, | |
34cc4fd9 | 656 | .process = amdgpu_umc_process_ecc_irq, |
791c4769 | 657 | }; |
658 | ||
e60f8db5 AX |
659 | static void gmc_v9_0_set_irq_funcs(struct amdgpu_device *adev) |
660 | { | |
770d13b1 CK |
661 | adev->gmc.vm_fault.num_types = 1; |
662 | adev->gmc.vm_fault.funcs = &gmc_v9_0_irq_funcs; | |
791c4769 | 663 | |
68d705dd HZ |
664 | if (!amdgpu_sriov_vf(adev) && |
665 | !adev->gmc.xgmi.connected_to_cpu) { | |
2ee9403e ZL |
666 | adev->gmc.ecc_irq.num_types = 1; |
667 | adev->gmc.ecc_irq.funcs = &gmc_v9_0_ecc_funcs; | |
668 | } | |
e60f8db5 AX |
669 | } |
670 | ||
2a79d868 YZ |
671 | static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid, |
672 | uint32_t flush_type) | |
03f89feb CK |
673 | { |
674 | u32 req = 0; | |
675 | ||
03f89feb | 676 | req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, |
c4f46f22 | 677 | PER_VMID_INVALIDATE_REQ, 1 << vmid); |
2a79d868 | 678 | req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type); |
03f89feb CK |
679 | req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1); |
680 | req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1); | |
681 | req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1); | |
682 | req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1); | |
683 | req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1); | |
684 | req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, | |
685 | CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0); | |
686 | ||
687 | return req; | |
688 | } | |
689 | ||
90f6452c | 690 | /** |
691 | * gmc_v9_0_use_invalidate_semaphore - judge whether to use semaphore | |
692 | * | |
693 | * @adev: amdgpu_device pointer | |
694 | * @vmhub: vmhub type | |
695 | * | |
696 | */ | |
697 | static bool gmc_v9_0_use_invalidate_semaphore(struct amdgpu_device *adev, | |
698 | uint32_t vmhub) | |
699 | { | |
b7daed1b | 700 | if (adev->asic_type == CHIP_ALDEBARAN) |
d477c5aa | 701 | return false; |
d477c5aa | 702 | |
90f6452c | 703 | return ((vmhub == AMDGPU_MMHUB_0 || |
704 | vmhub == AMDGPU_MMHUB_1) && | |
705 | (!amdgpu_sriov_vf(adev)) && | |
54f78a76 AD |
706 | (!(!(adev->apu_flags & AMD_APU_IS_RAVEN2) && |
707 | (adev->apu_flags & AMD_APU_IS_PICASSO)))); | |
90f6452c | 708 | } |
709 | ||
ea930000 AS |
710 | static bool gmc_v9_0_get_atc_vmid_pasid_mapping_info(struct amdgpu_device *adev, |
711 | uint8_t vmid, uint16_t *p_pasid) | |
712 | { | |
713 | uint32_t value; | |
714 | ||
715 | value = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING) | |
716 | + vmid); | |
717 | *p_pasid = value & ATC_VMID0_PASID_MAPPING__PASID_MASK; | |
718 | ||
719 | return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK); | |
720 | } | |
721 | ||
e60f8db5 AX |
722 | /* |
723 | * GART | |
724 | * VMID 0 is the physical GPU addresses as used by the kernel. | |
725 | * VMIDs 1-15 are used for userspace clients and are handled | |
726 | * by the amdgpu vm/hsa code. | |
727 | */ | |
728 | ||
729 | /** | |
2a79d868 | 730 | * gmc_v9_0_flush_gpu_tlb - tlb flush with certain type |
e60f8db5 AX |
731 | * |
732 | * @adev: amdgpu_device pointer | |
733 | * @vmid: vm instance to flush | |
bf0df09c | 734 | * @vmhub: which hub to flush |
2a79d868 | 735 | * @flush_type: the flush type |
e60f8db5 | 736 | * |
2a79d868 | 737 | * Flush the TLB for the requested page table using certain type. |
e60f8db5 | 738 | */ |
3ff98548 OZ |
739 | static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, |
740 | uint32_t vmhub, uint32_t flush_type) | |
e60f8db5 | 741 | { |
90f6452c | 742 | bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(adev, vmhub); |
e60f8db5 | 743 | const unsigned eng = 17; |
b80cd524 | 744 | u32 j, inv_req, inv_req2, tmp; |
3ff98548 | 745 | struct amdgpu_vmhub *hub; |
e60f8db5 | 746 | |
3ff98548 | 747 | BUG_ON(vmhub >= adev->num_vmhubs); |
e60f8db5 | 748 | |
3ff98548 | 749 | hub = &adev->vmhub[vmhub]; |
b80cd524 FK |
750 | if (adev->gmc.xgmi.num_physical_nodes && |
751 | adev->asic_type == CHIP_VEGA20) { | |
752 | /* Vega20+XGMI caches PTEs in TC and TLB. Add a | |
753 | * heavy-weight TLB flush (type 2), which flushes | |
754 | * both. Due to a race condition with concurrent | |
755 | * memory accesses using the same TLB cache line, we | |
756 | * still need a second TLB flush after this. | |
757 | */ | |
758 | inv_req = gmc_v9_0_get_invalidate_req(vmid, 2); | |
759 | inv_req2 = gmc_v9_0_get_invalidate_req(vmid, flush_type); | |
760 | } else { | |
761 | inv_req = gmc_v9_0_get_invalidate_req(vmid, flush_type); | |
762 | inv_req2 = 0; | |
763 | } | |
3890d111 | 764 | |
3ff98548 OZ |
765 | /* This is necessary for a HW workaround under SRIOV as well |
766 | * as GFXOFF under bare metal | |
767 | */ | |
768 | if (adev->gfx.kiq.ring.sched.ready && | |
81202807 DL |
769 | (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev)) && |
770 | down_read_trylock(&adev->reset_sem)) { | |
148f597d HR |
771 | uint32_t req = hub->vm_inv_eng0_req + hub->eng_distance * eng; |
772 | uint32_t ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng; | |
3ff98548 | 773 | |
37c58ddf | 774 | amdgpu_virt_kiq_reg_write_reg_wait(adev, req, ack, inv_req, |
148f597d | 775 | 1 << vmid); |
81202807 | 776 | up_read(&adev->reset_sem); |
3ff98548 OZ |
777 | return; |
778 | } | |
396557b0 | 779 | |
3ff98548 | 780 | spin_lock(&adev->gmc.invalidate_lock); |
f920d1bb | 781 | |
782 | /* | |
783 | * It may lose gpuvm invalidate acknowldege state across power-gating | |
784 | * off cycle, add semaphore acquire before invalidation and semaphore | |
785 | * release after invalidation to avoid entering power gated state | |
786 | * to WA the Issue | |
787 | */ | |
788 | ||
789 | /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ | |
90f6452c | 790 | if (use_semaphore) { |
f920d1bb | 791 | for (j = 0; j < adev->usec_timeout; j++) { |
792 | /* a read return value of 1 means semaphore acuqire */ | |
148f597d HR |
793 | tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_sem + |
794 | hub->eng_distance * eng); | |
f920d1bb | 795 | if (tmp & 0x1) |
796 | break; | |
797 | udelay(1); | |
798 | } | |
799 | ||
800 | if (j >= adev->usec_timeout) | |
801 | DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n"); | |
802 | } | |
803 | ||
b80cd524 | 804 | do { |
148f597d HR |
805 | WREG32_NO_KIQ(hub->vm_inv_eng0_req + |
806 | hub->eng_distance * eng, inv_req); | |
53499173 | 807 | |
b80cd524 FK |
808 | /* |
809 | * Issue a dummy read to wait for the ACK register to | |
810 | * be cleared to avoid a false ACK due to the new fast | |
811 | * GRBM interface. | |
812 | */ | |
7845d80d AD |
813 | if ((vmhub == AMDGPU_GFXHUB_0) && |
814 | (adev->asic_type < CHIP_ALDEBARAN)) | |
148f597d HR |
815 | RREG32_NO_KIQ(hub->vm_inv_eng0_req + |
816 | hub->eng_distance * eng); | |
53499173 | 817 | |
b80cd524 | 818 | for (j = 0; j < adev->usec_timeout; j++) { |
148f597d HR |
819 | tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_ack + |
820 | hub->eng_distance * eng); | |
b80cd524 FK |
821 | if (tmp & (1 << vmid)) |
822 | break; | |
823 | udelay(1); | |
824 | } | |
825 | ||
826 | inv_req = inv_req2; | |
827 | inv_req2 = 0; | |
828 | } while (inv_req); | |
f920d1bb | 829 | |
830 | /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ | |
90f6452c | 831 | if (use_semaphore) |
f920d1bb | 832 | /* |
833 | * add semaphore release after invalidation, | |
834 | * write with 0 means semaphore release | |
835 | */ | |
148f597d HR |
836 | WREG32_NO_KIQ(hub->vm_inv_eng0_sem + |
837 | hub->eng_distance * eng, 0); | |
f920d1bb | 838 | |
3ff98548 | 839 | spin_unlock(&adev->gmc.invalidate_lock); |
f920d1bb | 840 | |
3ff98548 OZ |
841 | if (j < adev->usec_timeout) |
842 | return; | |
843 | ||
844 | DRM_ERROR("Timeout waiting for VM flush ACK!\n"); | |
e60f8db5 AX |
845 | } |
846 | ||
ea930000 AS |
847 | /** |
848 | * gmc_v9_0_flush_gpu_tlb_pasid - tlb flush via pasid | |
849 | * | |
850 | * @adev: amdgpu_device pointer | |
851 | * @pasid: pasid to be flush | |
bf0df09c LJ |
852 | * @flush_type: the flush type |
853 | * @all_hub: flush all hubs | |
ea930000 AS |
854 | * |
855 | * Flush the TLB for the requested pasid. | |
856 | */ | |
857 | static int gmc_v9_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev, | |
858 | uint16_t pasid, uint32_t flush_type, | |
859 | bool all_hub) | |
860 | { | |
861 | int vmid, i; | |
862 | signed long r; | |
863 | uint32_t seq; | |
864 | uint16_t queried_pasid; | |
865 | bool ret; | |
866 | struct amdgpu_ring *ring = &adev->gfx.kiq.ring; | |
867 | struct amdgpu_kiq *kiq = &adev->gfx.kiq; | |
868 | ||
53b3f8f4 | 869 | if (amdgpu_in_reset(adev)) |
ea930000 AS |
870 | return -EIO; |
871 | ||
81202807 | 872 | if (ring->sched.ready && down_read_trylock(&adev->reset_sem)) { |
b80cd524 FK |
873 | /* Vega20+XGMI caches PTEs in TC and TLB. Add a |
874 | * heavy-weight TLB flush (type 2), which flushes | |
875 | * both. Due to a race condition with concurrent | |
876 | * memory accesses using the same TLB cache line, we | |
877 | * still need a second TLB flush after this. | |
878 | */ | |
879 | bool vega20_xgmi_wa = (adev->gmc.xgmi.num_physical_nodes && | |
880 | adev->asic_type == CHIP_VEGA20); | |
881 | /* 2 dwords flush + 8 dwords fence */ | |
882 | unsigned int ndw = kiq->pmf->invalidate_tlbs_size + 8; | |
883 | ||
884 | if (vega20_xgmi_wa) | |
885 | ndw += kiq->pmf->invalidate_tlbs_size; | |
886 | ||
ea930000 | 887 | spin_lock(&adev->gfx.kiq.ring_lock); |
36a1707a | 888 | /* 2 dwords flush + 8 dwords fence */ |
b80cd524 FK |
889 | amdgpu_ring_alloc(ring, ndw); |
890 | if (vega20_xgmi_wa) | |
891 | kiq->pmf->kiq_invalidate_tlbs(ring, | |
892 | pasid, 2, all_hub); | |
ea930000 AS |
893 | kiq->pmf->kiq_invalidate_tlbs(ring, |
894 | pasid, flush_type, all_hub); | |
04e4e2e9 YT |
895 | r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); |
896 | if (r) { | |
897 | amdgpu_ring_undo(ring); | |
abb17b1e | 898 | spin_unlock(&adev->gfx.kiq.ring_lock); |
81202807 | 899 | up_read(&adev->reset_sem); |
04e4e2e9 YT |
900 | return -ETIME; |
901 | } | |
902 | ||
ea930000 AS |
903 | amdgpu_ring_commit(ring); |
904 | spin_unlock(&adev->gfx.kiq.ring_lock); | |
905 | r = amdgpu_fence_wait_polling(ring, seq, adev->usec_timeout); | |
906 | if (r < 1) { | |
aac89168 | 907 | dev_err(adev->dev, "wait for kiq fence error: %ld.\n", r); |
81202807 | 908 | up_read(&adev->reset_sem); |
ea930000 AS |
909 | return -ETIME; |
910 | } | |
81202807 | 911 | up_read(&adev->reset_sem); |
ea930000 AS |
912 | return 0; |
913 | } | |
914 | ||
915 | for (vmid = 1; vmid < 16; vmid++) { | |
916 | ||
917 | ret = gmc_v9_0_get_atc_vmid_pasid_mapping_info(adev, vmid, | |
918 | &queried_pasid); | |
919 | if (ret && queried_pasid == pasid) { | |
920 | if (all_hub) { | |
921 | for (i = 0; i < adev->num_vmhubs; i++) | |
922 | gmc_v9_0_flush_gpu_tlb(adev, vmid, | |
fa34edbe | 923 | i, flush_type); |
ea930000 AS |
924 | } else { |
925 | gmc_v9_0_flush_gpu_tlb(adev, vmid, | |
fa34edbe | 926 | AMDGPU_GFXHUB_0, flush_type); |
ea930000 AS |
927 | } |
928 | break; | |
929 | } | |
930 | } | |
931 | ||
932 | return 0; | |
933 | ||
934 | } | |
935 | ||
9096d6e5 | 936 | static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring, |
c633c00b | 937 | unsigned vmid, uint64_t pd_addr) |
9096d6e5 | 938 | { |
90f6452c | 939 | bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(ring->adev, ring->funcs->vmhub); |
250b4228 CK |
940 | struct amdgpu_device *adev = ring->adev; |
941 | struct amdgpu_vmhub *hub = &adev->vmhub[ring->funcs->vmhub]; | |
2a79d868 | 942 | uint32_t req = gmc_v9_0_get_invalidate_req(vmid, 0); |
9096d6e5 CK |
943 | unsigned eng = ring->vm_inv_eng; |
944 | ||
f920d1bb | 945 | /* |
946 | * It may lose gpuvm invalidate acknowldege state across power-gating | |
947 | * off cycle, add semaphore acquire before invalidation and semaphore | |
948 | * release after invalidation to avoid entering power gated state | |
949 | * to WA the Issue | |
950 | */ | |
951 | ||
952 | /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ | |
90f6452c | 953 | if (use_semaphore) |
f920d1bb | 954 | /* a read return value of 1 means semaphore acuqire */ |
955 | amdgpu_ring_emit_reg_wait(ring, | |
148f597d HR |
956 | hub->vm_inv_eng0_sem + |
957 | hub->eng_distance * eng, 0x1, 0x1); | |
f920d1bb | 958 | |
148f597d HR |
959 | amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 + |
960 | (hub->ctx_addr_distance * vmid), | |
9096d6e5 CK |
961 | lower_32_bits(pd_addr)); |
962 | ||
148f597d HR |
963 | amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 + |
964 | (hub->ctx_addr_distance * vmid), | |
9096d6e5 CK |
965 | upper_32_bits(pd_addr)); |
966 | ||
148f597d HR |
967 | amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req + |
968 | hub->eng_distance * eng, | |
969 | hub->vm_inv_eng0_ack + | |
970 | hub->eng_distance * eng, | |
f8bc9037 | 971 | req, 1 << vmid); |
f732b6b3 | 972 | |
f920d1bb | 973 | /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ |
90f6452c | 974 | if (use_semaphore) |
f920d1bb | 975 | /* |
976 | * add semaphore release after invalidation, | |
977 | * write with 0 means semaphore release | |
978 | */ | |
148f597d HR |
979 | amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem + |
980 | hub->eng_distance * eng, 0); | |
f920d1bb | 981 | |
9096d6e5 CK |
982 | return pd_addr; |
983 | } | |
984 | ||
c633c00b CK |
985 | static void gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid, |
986 | unsigned pasid) | |
987 | { | |
988 | struct amdgpu_device *adev = ring->adev; | |
989 | uint32_t reg; | |
990 | ||
f2d66571 LM |
991 | /* Do nothing because there's no lut register for mmhub1. */ |
992 | if (ring->funcs->vmhub == AMDGPU_MMHUB_1) | |
993 | return; | |
994 | ||
a2d15ed7 | 995 | if (ring->funcs->vmhub == AMDGPU_GFXHUB_0) |
c633c00b CK |
996 | reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid; |
997 | else | |
998 | reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid; | |
999 | ||
1000 | amdgpu_ring_emit_wreg(ring, reg, pasid); | |
1001 | } | |
1002 | ||
6490bd76 YZ |
1003 | /* |
1004 | * PTE format on VEGA 10: | |
1005 | * 63:59 reserved | |
1006 | * 58:57 mtype | |
1007 | * 56 F | |
1008 | * 55 L | |
1009 | * 54 P | |
1010 | * 53 SW | |
1011 | * 52 T | |
1012 | * 50:48 reserved | |
1013 | * 47:12 4k physical page base address | |
1014 | * 11:7 fragment | |
1015 | * 6 write | |
1016 | * 5 read | |
1017 | * 4 exe | |
1018 | * 3 Z | |
1019 | * 2 snooped | |
1020 | * 1 system | |
1021 | * 0 valid | |
e60f8db5 | 1022 | * |
6490bd76 YZ |
1023 | * PDE format on VEGA 10: |
1024 | * 63:59 block fragment size | |
1025 | * 58:55 reserved | |
1026 | * 54 P | |
1027 | * 53:48 reserved | |
1028 | * 47:6 physical base address of PD or PTE | |
1029 | * 5:3 reserved | |
1030 | * 2 C | |
1031 | * 1 system | |
1032 | * 0 valid | |
e60f8db5 | 1033 | */ |
e60f8db5 | 1034 | |
71776b6d | 1035 | static uint64_t gmc_v9_0_map_mtype(struct amdgpu_device *adev, uint32_t flags) |
e60f8db5 AX |
1036 | |
1037 | { | |
71776b6d | 1038 | switch (flags) { |
e60f8db5 | 1039 | case AMDGPU_VM_MTYPE_DEFAULT: |
71776b6d | 1040 | return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC); |
e60f8db5 | 1041 | case AMDGPU_VM_MTYPE_NC: |
71776b6d | 1042 | return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC); |
e60f8db5 | 1043 | case AMDGPU_VM_MTYPE_WC: |
71776b6d | 1044 | return AMDGPU_PTE_MTYPE_VG10(MTYPE_WC); |
093e48c0 | 1045 | case AMDGPU_VM_MTYPE_RW: |
71776b6d | 1046 | return AMDGPU_PTE_MTYPE_VG10(MTYPE_RW); |
e60f8db5 | 1047 | case AMDGPU_VM_MTYPE_CC: |
71776b6d | 1048 | return AMDGPU_PTE_MTYPE_VG10(MTYPE_CC); |
e60f8db5 | 1049 | case AMDGPU_VM_MTYPE_UC: |
71776b6d | 1050 | return AMDGPU_PTE_MTYPE_VG10(MTYPE_UC); |
e60f8db5 | 1051 | default: |
71776b6d | 1052 | return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC); |
e60f8db5 | 1053 | } |
e60f8db5 AX |
1054 | } |
1055 | ||
3de676d8 CK |
1056 | static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level, |
1057 | uint64_t *addr, uint64_t *flags) | |
e60f8db5 | 1058 | { |
bbc9fb10 | 1059 | if (!(*flags & AMDGPU_PDE_PTE) && !(*flags & AMDGPU_PTE_SYSTEM)) |
0ca565ab | 1060 | *addr = amdgpu_gmc_vram_mc2pa(adev, *addr); |
3de676d8 | 1061 | BUG_ON(*addr & 0xFFFF00000000003FULL); |
6a42fd6f | 1062 | |
770d13b1 | 1063 | if (!adev->gmc.translate_further) |
6a42fd6f CK |
1064 | return; |
1065 | ||
1066 | if (level == AMDGPU_VM_PDB1) { | |
1067 | /* Set the block fragment size */ | |
1068 | if (!(*flags & AMDGPU_PDE_PTE)) | |
1069 | *flags |= AMDGPU_PDE_BFS(0x9); | |
1070 | ||
1071 | } else if (level == AMDGPU_VM_PDB0) { | |
1072 | if (*flags & AMDGPU_PDE_PTE) | |
1073 | *flags &= ~AMDGPU_PDE_PTE; | |
1074 | else | |
1075 | *flags |= AMDGPU_PTE_TF; | |
1076 | } | |
e60f8db5 AX |
1077 | } |
1078 | ||
cbfae36c CK |
1079 | static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev, |
1080 | struct amdgpu_bo_va_mapping *mapping, | |
1081 | uint64_t *flags) | |
1082 | { | |
1083 | *flags &= ~AMDGPU_PTE_EXECUTABLE; | |
1084 | *flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE; | |
1085 | ||
1086 | *flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK; | |
1087 | *flags |= mapping->flags & AMDGPU_PTE_MTYPE_VG10_MASK; | |
1088 | ||
1089 | if (mapping->flags & AMDGPU_PTE_PRT) { | |
1090 | *flags |= AMDGPU_PTE_PRT; | |
1091 | *flags &= ~AMDGPU_PTE_VALID; | |
1092 | } | |
1093 | ||
7ffe7238 YZ |
1094 | if ((adev->asic_type == CHIP_ARCTURUS || |
1095 | adev->asic_type == CHIP_ALDEBARAN) && | |
cbfae36c CK |
1096 | !(*flags & AMDGPU_PTE_SYSTEM) && |
1097 | mapping->bo_va->is_xgmi) | |
1098 | *flags |= AMDGPU_PTE_SNOOPED; | |
72b4db0f EH |
1099 | |
1100 | if (adev->asic_type == CHIP_ALDEBARAN) | |
1101 | *flags |= mapping->flags & AMDGPU_PTE_SNOOPED; | |
cbfae36c CK |
1102 | } |
1103 | ||
7b885f0e AD |
1104 | static unsigned gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev) |
1105 | { | |
1106 | u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL); | |
1107 | unsigned size; | |
1108 | ||
de44ce64 HW |
1109 | /* TODO move to DC so GMC doesn't need to hard-code DCN registers */ |
1110 | ||
7b885f0e AD |
1111 | if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) { |
1112 | size = AMDGPU_VBIOS_VGA_ALLOCATION; | |
1113 | } else { | |
1114 | u32 viewport; | |
1115 | ||
1116 | switch (adev->asic_type) { | |
1117 | case CHIP_RAVEN: | |
7b885f0e AD |
1118 | viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION); |
1119 | size = (REG_GET_FIELD(viewport, | |
1120 | HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) * | |
1121 | REG_GET_FIELD(viewport, | |
1122 | HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) * | |
1123 | 4); | |
1124 | break; | |
de44ce64 HW |
1125 | case CHIP_RENOIR: |
1126 | viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_DCN2); | |
1127 | size = (REG_GET_FIELD(viewport, | |
1128 | HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) * | |
1129 | REG_GET_FIELD(viewport, | |
1130 | HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) * | |
1131 | 4); | |
1132 | break; | |
7b885f0e AD |
1133 | case CHIP_VEGA10: |
1134 | case CHIP_VEGA12: | |
1135 | case CHIP_VEGA20: | |
1136 | default: | |
1137 | viewport = RREG32_SOC15(DCE, 0, mmSCL0_VIEWPORT_SIZE); | |
1138 | size = (REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_HEIGHT) * | |
1139 | REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_WIDTH) * | |
1140 | 4); | |
1141 | break; | |
1142 | } | |
1143 | } | |
1144 | ||
1145 | return size; | |
1146 | } | |
1147 | ||
132f34e4 CK |
1148 | static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = { |
1149 | .flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb, | |
ea930000 | 1150 | .flush_gpu_tlb_pasid = gmc_v9_0_flush_gpu_tlb_pasid, |
9096d6e5 | 1151 | .emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb, |
c633c00b | 1152 | .emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping, |
71776b6d | 1153 | .map_mtype = gmc_v9_0_map_mtype, |
cbfae36c | 1154 | .get_vm_pde = gmc_v9_0_get_vm_pde, |
7b885f0e AD |
1155 | .get_vm_pte = gmc_v9_0_get_vm_pte, |
1156 | .get_vbios_fb_size = gmc_v9_0_get_vbios_fb_size, | |
e60f8db5 AX |
1157 | }; |
1158 | ||
132f34e4 | 1159 | static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev) |
e60f8db5 | 1160 | { |
f54b30d7 | 1161 | adev->gmc.gmc_funcs = &gmc_v9_0_gmc_funcs; |
e60f8db5 AX |
1162 | } |
1163 | ||
5b6b35aa HZ |
1164 | static void gmc_v9_0_set_umc_funcs(struct amdgpu_device *adev) |
1165 | { | |
1166 | switch (adev->asic_type) { | |
e7da754b ML |
1167 | case CHIP_VEGA10: |
1168 | adev->umc.funcs = &umc_v6_0_funcs; | |
1169 | break; | |
5b6b35aa | 1170 | case CHIP_VEGA20: |
3aacf4ea TZ |
1171 | adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM; |
1172 | adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM; | |
1173 | adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM; | |
4cf781c2 JC |
1174 | adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_VG20; |
1175 | adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0]; | |
49070c4e | 1176 | adev->umc.ras_funcs = &umc_v6_1_ras_funcs; |
4cf781c2 | 1177 | break; |
9e612c11 | 1178 | case CHIP_ARCTURUS: |
3aacf4ea TZ |
1179 | adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM; |
1180 | adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM; | |
1181 | adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM; | |
4cf781c2 | 1182 | adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_ARCT; |
3aacf4ea | 1183 | adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0]; |
49070c4e | 1184 | adev->umc.ras_funcs = &umc_v6_1_ras_funcs; |
5b6b35aa | 1185 | break; |
186c8a85 JC |
1186 | case CHIP_ALDEBARAN: |
1187 | adev->umc.max_ras_err_cnt_per_query = UMC_V6_7_TOTAL_CHANNEL_NUM; | |
719e433e MJ |
1188 | adev->umc.channel_inst_num = UMC_V6_7_CHANNEL_INSTANCE_NUM; |
1189 | adev->umc.umc_inst_num = UMC_V6_7_UMC_INSTANCE_NUM; | |
186c8a85 JC |
1190 | adev->umc.channel_offs = UMC_V6_7_PER_CHANNEL_OFFSET; |
1191 | if (!adev->gmc.xgmi.connected_to_cpu) | |
1192 | adev->umc.ras_funcs = &umc_v6_7_ras_funcs; | |
1193 | if (1 & adev->smuio.funcs->get_die_id(adev)) | |
1194 | adev->umc.channel_idx_tbl = &umc_v6_7_channel_idx_tbl_first[0][0]; | |
1195 | else | |
1196 | adev->umc.channel_idx_tbl = &umc_v6_7_channel_idx_tbl_second[0][0]; | |
1197 | break; | |
5b6b35aa HZ |
1198 | default: |
1199 | break; | |
1200 | } | |
1201 | } | |
1202 | ||
3d093da0 TZ |
1203 | static void gmc_v9_0_set_mmhub_funcs(struct amdgpu_device *adev) |
1204 | { | |
1205 | switch (adev->asic_type) { | |
f6c3623b DL |
1206 | case CHIP_ARCTURUS: |
1207 | adev->mmhub.funcs = &mmhub_v9_4_funcs; | |
1208 | break; | |
4da999cd OZ |
1209 | case CHIP_ALDEBARAN: |
1210 | adev->mmhub.funcs = &mmhub_v1_7_funcs; | |
1211 | break; | |
3d093da0 | 1212 | default: |
9fb1506e | 1213 | adev->mmhub.funcs = &mmhub_v1_0_funcs; |
3d093da0 TZ |
1214 | break; |
1215 | } | |
1216 | } | |
1217 | ||
d844c6d7 HZ |
1218 | static void gmc_v9_0_set_mmhub_ras_funcs(struct amdgpu_device *adev) |
1219 | { | |
1220 | switch (adev->asic_type) { | |
1221 | case CHIP_VEGA20: | |
1222 | adev->mmhub.ras_funcs = &mmhub_v1_0_ras_funcs; | |
1223 | break; | |
1224 | case CHIP_ARCTURUS: | |
1225 | adev->mmhub.ras_funcs = &mmhub_v9_4_ras_funcs; | |
1226 | break; | |
1227 | case CHIP_ALDEBARAN: | |
1228 | adev->mmhub.ras_funcs = &mmhub_v1_7_ras_funcs; | |
1229 | break; | |
1230 | default: | |
1231 | /* mmhub ras is not available */ | |
1232 | break; | |
1233 | } | |
1234 | } | |
1235 | ||
8ffff9b4 OZ |
1236 | static void gmc_v9_0_set_gfxhub_funcs(struct amdgpu_device *adev) |
1237 | { | |
21470d97 | 1238 | adev->gfxhub.funcs = &gfxhub_v1_0_funcs; |
8ffff9b4 OZ |
1239 | } |
1240 | ||
6f12507f HZ |
1241 | static void gmc_v9_0_set_hdp_ras_funcs(struct amdgpu_device *adev) |
1242 | { | |
1243 | adev->hdp.ras_funcs = &hdp_v4_0_ras_funcs; | |
1244 | } | |
1245 | ||
3907c492 JC |
1246 | static void gmc_v9_0_set_mca_funcs(struct amdgpu_device *adev) |
1247 | { | |
1248 | switch (adev->asic_type) { | |
1249 | case CHIP_ALDEBARAN: | |
1250 | if (!adev->gmc.xgmi.connected_to_cpu) | |
1251 | adev->mca.funcs = &mca_v3_0_funcs; | |
1252 | break; | |
1253 | default: | |
1254 | break; | |
1255 | } | |
1256 | } | |
1257 | ||
e60f8db5 AX |
1258 | static int gmc_v9_0_early_init(void *handle) |
1259 | { | |
1260 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
1261 | ||
31691b8d RB |
1262 | if (adev->asic_type == CHIP_VEGA20 || |
1263 | adev->asic_type == CHIP_ARCTURUS) | |
1264 | adev->gmc.xgmi.supported = true; | |
1265 | ||
1266 | if (adev->asic_type == CHIP_ALDEBARAN) { | |
1267 | adev->gmc.xgmi.supported = true; | |
1268 | adev->gmc.xgmi.connected_to_cpu = | |
1269 | adev->smuio.funcs->is_host_gpu_xgmi_supported(adev); | |
52137ca8 | 1270 | } |
31691b8d | 1271 | |
49070c4e HZ |
1272 | gmc_v9_0_set_gmc_funcs(adev); |
1273 | gmc_v9_0_set_irq_funcs(adev); | |
1274 | gmc_v9_0_set_umc_funcs(adev); | |
1275 | gmc_v9_0_set_mmhub_funcs(adev); | |
d844c6d7 | 1276 | gmc_v9_0_set_mmhub_ras_funcs(adev); |
49070c4e | 1277 | gmc_v9_0_set_gfxhub_funcs(adev); |
6f12507f | 1278 | gmc_v9_0_set_hdp_ras_funcs(adev); |
3907c492 | 1279 | gmc_v9_0_set_mca_funcs(adev); |
49070c4e | 1280 | |
770d13b1 CK |
1281 | adev->gmc.shared_aperture_start = 0x2000000000000000ULL; |
1282 | adev->gmc.shared_aperture_end = | |
1283 | adev->gmc.shared_aperture_start + (4ULL << 30) - 1; | |
bfa8eea2 | 1284 | adev->gmc.private_aperture_start = 0x1000000000000000ULL; |
770d13b1 CK |
1285 | adev->gmc.private_aperture_end = |
1286 | adev->gmc.private_aperture_start + (4ULL << 30) - 1; | |
a7ea6548 | 1287 | |
e60f8db5 AX |
1288 | return 0; |
1289 | } | |
1290 | ||
c713a461 EQ |
1291 | static int gmc_v9_0_late_init(void *handle) |
1292 | { | |
1293 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
c5b6e585 | 1294 | int r; |
c713a461 | 1295 | |
bdbe90f0 | 1296 | r = amdgpu_gmc_allocate_vm_inv_eng(adev); |
c713a461 EQ |
1297 | if (r) |
1298 | return r; | |
4a20300b GC |
1299 | |
1300 | /* | |
1301 | * Workaround performance drop issue with VBIOS enables partial | |
1302 | * writes, while disables HBM ECC for vega10. | |
1303 | */ | |
88474cca | 1304 | if (!amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_VEGA10)) { |
8ab0d6f0 | 1305 | if (!(adev->ras_enabled & (1 << AMDGPU_RAS_BLOCK__UMC))) { |
88474cca GC |
1306 | if (adev->df.funcs->enable_ecc_force_par_wr_rmw) |
1307 | adev->df.funcs->enable_ecc_force_par_wr_rmw(adev, false); | |
4a20300b | 1308 | } |
02bab923 DP |
1309 | } |
1310 | ||
8f6368a9 JC |
1311 | if (!amdgpu_persistent_edc_harvesting_supported(adev)) { |
1312 | if (adev->mmhub.ras_funcs && | |
1313 | adev->mmhub.ras_funcs->reset_ras_error_count) | |
1314 | adev->mmhub.ras_funcs->reset_ras_error_count(adev); | |
1315 | ||
1316 | if (adev->hdp.ras_funcs && | |
1317 | adev->hdp.ras_funcs->reset_ras_error_count) | |
1318 | adev->hdp.ras_funcs->reset_ras_error_count(adev); | |
1319 | } | |
fe5211f1 | 1320 | |
ba083492 | 1321 | r = amdgpu_gmc_ras_late_init(adev); |
791c4769 | 1322 | if (r) |
1323 | return r; | |
1324 | ||
770d13b1 | 1325 | return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0); |
e60f8db5 AX |
1326 | } |
1327 | ||
1328 | static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev, | |
770d13b1 | 1329 | struct amdgpu_gmc *mc) |
e60f8db5 | 1330 | { |
adbe2e3d | 1331 | u64 base = adev->mmhub.funcs->get_fb_location(adev); |
9d4f837a | 1332 | |
6fdd68b1 AD |
1333 | /* add the xgmi offset of the physical node */ |
1334 | base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; | |
f527f310 OZ |
1335 | if (adev->gmc.xgmi.connected_to_cpu) { |
1336 | amdgpu_gmc_sysvm_location(adev, mc); | |
1337 | } else { | |
1338 | amdgpu_gmc_vram_location(adev, mc, base); | |
1339 | amdgpu_gmc_gart_location(adev, mc); | |
1340 | amdgpu_gmc_agp_location(adev, mc); | |
1341 | } | |
bc099ee9 | 1342 | /* base offset of vram pages */ |
8ffff9b4 | 1343 | adev->vm_manager.vram_base_offset = adev->gfxhub.funcs->get_mc_fb_offset(adev); |
6fdd68b1 AD |
1344 | |
1345 | /* XXX: add the xgmi offset of the physical node? */ | |
1346 | adev->vm_manager.vram_base_offset += | |
1347 | adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; | |
e60f8db5 AX |
1348 | } |
1349 | ||
1350 | /** | |
1351 | * gmc_v9_0_mc_init - initialize the memory controller driver params | |
1352 | * | |
1353 | * @adev: amdgpu_device pointer | |
1354 | * | |
1355 | * Look up the amount of vram, vram width, and decide how to place | |
1356 | * vram and gart within the GPU's physical address space. | |
1357 | * Returns 0 for success. | |
1358 | */ | |
1359 | static int gmc_v9_0_mc_init(struct amdgpu_device *adev) | |
1360 | { | |
d6895ad3 | 1361 | int r; |
e60f8db5 | 1362 | |
e60f8db5 | 1363 | /* size in MB on si */ |
770d13b1 | 1364 | adev->gmc.mc_vram_size = |
bebc0762 | 1365 | adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL; |
770d13b1 | 1366 | adev->gmc.real_vram_size = adev->gmc.mc_vram_size; |
d6895ad3 | 1367 | |
be566196 OZ |
1368 | if (!(adev->flags & AMD_IS_APU) && |
1369 | !adev->gmc.xgmi.connected_to_cpu) { | |
d6895ad3 CK |
1370 | r = amdgpu_device_resize_fb_bar(adev); |
1371 | if (r) | |
1372 | return r; | |
1373 | } | |
770d13b1 CK |
1374 | adev->gmc.aper_base = pci_resource_start(adev->pdev, 0); |
1375 | adev->gmc.aper_size = pci_resource_len(adev->pdev, 0); | |
e60f8db5 | 1376 | |
156a81be | 1377 | #ifdef CONFIG_X86_64 |
31691b8d RB |
1378 | /* |
1379 | * AMD Accelerated Processing Platform (APP) supporting GPU-HOST xgmi | |
1380 | * interface can use VRAM through here as it appears system reserved | |
1381 | * memory in host address space. | |
1382 | * | |
1383 | * For APUs, VRAM is just the stolen system memory and can be accessed | |
1384 | * directly. | |
1385 | * | |
1386 | * Otherwise, use the legacy Host Data Path (HDP) through PCIe BAR. | |
1387 | */ | |
1388 | ||
1389 | /* check whether both host-gpu and gpu-gpu xgmi links exist */ | |
3de60d96 HZ |
1390 | if ((adev->flags & AMD_IS_APU) || |
1391 | (adev->gmc.xgmi.supported && | |
1392 | adev->gmc.xgmi.connected_to_cpu)) { | |
1393 | adev->gmc.aper_base = | |
1394 | adev->gfxhub.funcs->get_mc_fb_offset(adev) + | |
1395 | adev->gmc.xgmi.physical_node_id * | |
31691b8d | 1396 | adev->gmc.xgmi.node_segment_size; |
156a81be CZ |
1397 | adev->gmc.aper_size = adev->gmc.real_vram_size; |
1398 | } | |
31691b8d | 1399 | |
156a81be | 1400 | #endif |
e60f8db5 | 1401 | /* In case the PCI BAR is larger than the actual amount of vram */ |
770d13b1 CK |
1402 | adev->gmc.visible_vram_size = adev->gmc.aper_size; |
1403 | if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size) | |
1404 | adev->gmc.visible_vram_size = adev->gmc.real_vram_size; | |
e60f8db5 | 1405 | |
c3db7b5a AD |
1406 | /* set the gart size */ |
1407 | if (amdgpu_gart_size == -1) { | |
1408 | switch (adev->asic_type) { | |
1409 | case CHIP_VEGA10: /* all engines support GPUVM */ | |
273a14cd | 1410 | case CHIP_VEGA12: /* all engines support GPUVM */ |
d96b428c | 1411 | case CHIP_VEGA20: |
3de2ff5d | 1412 | case CHIP_ARCTURUS: |
85e39550 | 1413 | case CHIP_ALDEBARAN: |
c3db7b5a | 1414 | default: |
fe19b862 | 1415 | adev->gmc.gart_size = 512ULL << 20; |
c3db7b5a AD |
1416 | break; |
1417 | case CHIP_RAVEN: /* DCE SG support */ | |
8787ee01 | 1418 | case CHIP_RENOIR: |
770d13b1 | 1419 | adev->gmc.gart_size = 1024ULL << 20; |
c3db7b5a AD |
1420 | break; |
1421 | } | |
1422 | } else { | |
770d13b1 | 1423 | adev->gmc.gart_size = (u64)amdgpu_gart_size << 20; |
c3db7b5a AD |
1424 | } |
1425 | ||
f1dc12ca OZ |
1426 | adev->gmc.gart_size += adev->pm.smu_prv_buffer_size; |
1427 | ||
770d13b1 | 1428 | gmc_v9_0_vram_gtt_location(adev, &adev->gmc); |
e60f8db5 AX |
1429 | |
1430 | return 0; | |
1431 | } | |
1432 | ||
1433 | static int gmc_v9_0_gart_init(struct amdgpu_device *adev) | |
1434 | { | |
1435 | int r; | |
1436 | ||
1123b989 | 1437 | if (adev->gart.bo) { |
e60f8db5 AX |
1438 | WARN(1, "VEGA10 PCIE GART already initialized\n"); |
1439 | return 0; | |
1440 | } | |
7b454b3a OZ |
1441 | |
1442 | if (adev->gmc.xgmi.connected_to_cpu) { | |
1443 | adev->gmc.vmid0_page_table_depth = 1; | |
1444 | adev->gmc.vmid0_page_table_block_size = 12; | |
1445 | } else { | |
1446 | adev->gmc.vmid0_page_table_depth = 0; | |
1447 | adev->gmc.vmid0_page_table_block_size = 0; | |
1448 | } | |
1449 | ||
e60f8db5 AX |
1450 | /* Initialize common gart structure */ |
1451 | r = amdgpu_gart_init(adev); | |
1452 | if (r) | |
1453 | return r; | |
1454 | adev->gart.table_size = adev->gart.num_gpu_pages * 8; | |
7596ab68 | 1455 | adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE_VG10(MTYPE_UC) | |
e60f8db5 | 1456 | AMDGPU_PTE_EXECUTABLE; |
522510a6 OZ |
1457 | |
1458 | r = amdgpu_gart_table_vram_alloc(adev); | |
1459 | if (r) | |
1460 | return r; | |
1461 | ||
1462 | if (adev->gmc.xgmi.connected_to_cpu) { | |
1463 | r = amdgpu_gmc_pdb0_alloc(adev); | |
1464 | } | |
1465 | ||
1466 | return r; | |
e60f8db5 AX |
1467 | } |
1468 | ||
b0a2db9b AD |
1469 | /** |
1470 | * gmc_v9_0_save_registers - saves regs | |
1471 | * | |
1472 | * @adev: amdgpu_device pointer | |
1473 | * | |
1474 | * This saves potential register values that should be | |
1475 | * restored upon resume | |
1476 | */ | |
1477 | static void gmc_v9_0_save_registers(struct amdgpu_device *adev) | |
ebdef28e | 1478 | { |
b0a2db9b AD |
1479 | if (adev->asic_type == CHIP_RAVEN) |
1480 | adev->gmc.sdpif_register = RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0); | |
ebdef28e AD |
1481 | } |
1482 | ||
e60f8db5 AX |
1483 | static int gmc_v9_0_sw_init(void *handle) |
1484 | { | |
ad02e08e | 1485 | int r, vram_width = 0, vram_type = 0, vram_vendor = 0; |
e60f8db5 AX |
1486 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
1487 | ||
8ffff9b4 | 1488 | adev->gfxhub.funcs->init(adev); |
9fb1506e OZ |
1489 | |
1490 | adev->mmhub.funcs->init(adev); | |
3907c492 JC |
1491 | if (adev->mca.funcs) |
1492 | adev->mca.funcs->init(adev); | |
0c8c0847 | 1493 | |
770d13b1 | 1494 | spin_lock_init(&adev->gmc.invalidate_lock); |
e60f8db5 | 1495 | |
ad02e08e OM |
1496 | r = amdgpu_atomfirmware_get_vram_info(adev, |
1497 | &vram_width, &vram_type, &vram_vendor); | |
631cdbd2 AD |
1498 | if (amdgpu_sriov_vf(adev)) |
1499 | /* For Vega10 SR-IOV, vram_width can't be read from ATOM as RAVEN, | |
1500 | * and DF related registers is not readable, seems hardcord is the | |
1501 | * only way to set the correct vram_width | |
1502 | */ | |
1503 | adev->gmc.vram_width = 2048; | |
1504 | else if (amdgpu_emu_mode != 1) | |
1505 | adev->gmc.vram_width = vram_width; | |
1506 | ||
1507 | if (!adev->gmc.vram_width) { | |
1508 | int chansize, numchan; | |
1509 | ||
1510 | /* hbm memory channel size */ | |
1511 | if (adev->flags & AMD_IS_APU) | |
1512 | chansize = 64; | |
1513 | else | |
1514 | chansize = 128; | |
1515 | ||
bdf84a80 | 1516 | numchan = adev->df.funcs->get_hbm_channel_number(adev); |
631cdbd2 AD |
1517 | adev->gmc.vram_width = numchan * chansize; |
1518 | } | |
1519 | ||
1520 | adev->gmc.vram_type = vram_type; | |
ad02e08e | 1521 | adev->gmc.vram_vendor = vram_vendor; |
fd66560b HZ |
1522 | switch (adev->asic_type) { |
1523 | case CHIP_RAVEN: | |
1daa2bfa LM |
1524 | adev->num_vmhubs = 2; |
1525 | ||
6a42fd6f | 1526 | if (adev->rev_id == 0x0 || adev->rev_id == 0x1) { |
f3368128 | 1527 | amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); |
6a42fd6f CK |
1528 | } else { |
1529 | /* vm_size is 128TB + 512GB for legacy 3-level page support */ | |
1530 | amdgpu_vm_adjust_size(adev, 128 * 1024 + 512, 9, 2, 48); | |
770d13b1 | 1531 | adev->gmc.translate_further = |
6a42fd6f CK |
1532 | adev->vm_manager.num_level > 1; |
1533 | } | |
fd66560b HZ |
1534 | break; |
1535 | case CHIP_VEGA10: | |
273a14cd | 1536 | case CHIP_VEGA12: |
d96b428c | 1537 | case CHIP_VEGA20: |
8787ee01 | 1538 | case CHIP_RENOIR: |
85e39550 | 1539 | case CHIP_ALDEBARAN: |
1daa2bfa LM |
1540 | adev->num_vmhubs = 2; |
1541 | ||
8787ee01 | 1542 | |
36b32a68 ZJ |
1543 | /* |
1544 | * To fulfill 4-level page support, | |
1545 | * vm size is 256TB (48bit), maximum size of Vega10, | |
1546 | * block size 512 (9bit) | |
1547 | */ | |
cdba61da | 1548 | /* sriov restrict max_pfn below AMDGPU_GMC_HOLE */ |
1549 | if (amdgpu_sriov_vf(adev)) | |
1550 | amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 47); | |
1551 | else | |
1552 | amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); | |
fd66560b | 1553 | break; |
3de2ff5d | 1554 | case CHIP_ARCTURUS: |
c8a6e2a3 LM |
1555 | adev->num_vmhubs = 3; |
1556 | ||
3de2ff5d LM |
1557 | /* Keep the vm size same with Vega20 */ |
1558 | amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); | |
1559 | break; | |
fd66560b HZ |
1560 | default: |
1561 | break; | |
e60f8db5 AX |
1562 | } |
1563 | ||
1564 | /* This interrupt is VMC page fault.*/ | |
44a99b65 | 1565 | r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC, VMC_1_0__SRCID__VM_FAULT, |
770d13b1 | 1566 | &adev->gmc.vm_fault); |
30da7bb1 CK |
1567 | if (r) |
1568 | return r; | |
1569 | ||
7d19b15f LM |
1570 | if (adev->asic_type == CHIP_ARCTURUS) { |
1571 | r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC1, VMC_1_0__SRCID__VM_FAULT, | |
1572 | &adev->gmc.vm_fault); | |
1573 | if (r) | |
1574 | return r; | |
1575 | } | |
1576 | ||
44a99b65 | 1577 | r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2, UTCL2_1_0__SRCID__FAULT, |
770d13b1 | 1578 | &adev->gmc.vm_fault); |
e60f8db5 AX |
1579 | |
1580 | if (r) | |
1581 | return r; | |
1582 | ||
68d705dd HZ |
1583 | if (!amdgpu_sriov_vf(adev) && |
1584 | !adev->gmc.xgmi.connected_to_cpu) { | |
2ee9403e ZL |
1585 | /* interrupt sent to DF. */ |
1586 | r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DF, 0, | |
1587 | &adev->gmc.ecc_irq); | |
1588 | if (r) | |
1589 | return r; | |
1590 | } | |
791c4769 | 1591 | |
e60f8db5 AX |
1592 | /* Set the internal MC address mask |
1593 | * This is the max address of the GPU's | |
1594 | * internal address space. | |
1595 | */ | |
770d13b1 | 1596 | adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */ |
e60f8db5 | 1597 | |
244511f3 | 1598 | r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(44)); |
e60f8db5 | 1599 | if (r) { |
e60f8db5 | 1600 | printk(KERN_WARNING "amdgpu: No suitable DMA available.\n"); |
244511f3 | 1601 | return r; |
e60f8db5 | 1602 | } |
244511f3 | 1603 | adev->need_swiotlb = drm_need_swiotlb(44); |
e60f8db5 | 1604 | |
47622ba0 | 1605 | if (adev->gmc.xgmi.supported) { |
8ffff9b4 | 1606 | r = adev->gfxhub.funcs->get_xgmi_info(adev); |
bf0a60b7 AD |
1607 | if (r) |
1608 | return r; | |
1609 | } | |
1610 | ||
e60f8db5 AX |
1611 | r = gmc_v9_0_mc_init(adev); |
1612 | if (r) | |
1613 | return r; | |
1614 | ||
7b885f0e | 1615 | amdgpu_gmc_get_vbios_allocations(adev); |
ebdef28e | 1616 | |
e60f8db5 AX |
1617 | /* Memory manager */ |
1618 | r = amdgpu_bo_init(adev); | |
1619 | if (r) | |
1620 | return r; | |
1621 | ||
1622 | r = gmc_v9_0_gart_init(adev); | |
1623 | if (r) | |
1624 | return r; | |
1625 | ||
05ec3eda CK |
1626 | /* |
1627 | * number of VMs | |
1628 | * VMID 0 is reserved for System | |
81659b20 FK |
1629 | * amdgpu graphics/compute will use VMIDs 1..n-1 |
1630 | * amdkfd will use VMIDs n..15 | |
1631 | * | |
1632 | * The first KFD VMID is 8 for GPUs with graphics, 3 for | |
1633 | * compute-only GPUs. On compute-only GPUs that leaves 2 VMIDs | |
1634 | * for video processing. | |
05ec3eda | 1635 | */ |
81659b20 | 1636 | adev->vm_manager.first_kfd_vmid = |
6dce50b1 FK |
1637 | (adev->asic_type == CHIP_ARCTURUS || |
1638 | adev->asic_type == CHIP_ALDEBARAN) ? 3 : 8; | |
05ec3eda | 1639 | |
05ec3eda CK |
1640 | amdgpu_vm_manager_init(adev); |
1641 | ||
b0a2db9b AD |
1642 | gmc_v9_0_save_registers(adev); |
1643 | ||
05ec3eda | 1644 | return 0; |
e60f8db5 AX |
1645 | } |
1646 | ||
e60f8db5 AX |
1647 | static int gmc_v9_0_sw_fini(void *handle) |
1648 | { | |
1649 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
1650 | ||
2adf1344 | 1651 | amdgpu_gmc_ras_fini(adev); |
f59548c8 | 1652 | amdgpu_gem_force_release(adev); |
05ec3eda | 1653 | amdgpu_vm_manager_fini(adev); |
a3d9103e | 1654 | amdgpu_gart_table_vram_free(adev); |
90dc05f7 | 1655 | amdgpu_bo_free_kernel(&adev->gmc.pdb0_bo, NULL, &adev->gmc.ptr_pdb0); |
e60f8db5 AX |
1656 | amdgpu_bo_fini(adev); |
1657 | ||
1658 | return 0; | |
1659 | } | |
1660 | ||
1661 | static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev) | |
1662 | { | |
946a4d5b | 1663 | |
e60f8db5 AX |
1664 | switch (adev->asic_type) { |
1665 | case CHIP_VEGA10: | |
4cd4c5c0 | 1666 | if (amdgpu_sriov_vf(adev)) |
98cad2de | 1667 | break; |
df561f66 | 1668 | fallthrough; |
d96b428c | 1669 | case CHIP_VEGA20: |
946a4d5b | 1670 | soc15_program_register_sequence(adev, |
5c583018 | 1671 | golden_settings_mmhub_1_0_0, |
c47b41a7 | 1672 | ARRAY_SIZE(golden_settings_mmhub_1_0_0)); |
946a4d5b | 1673 | soc15_program_register_sequence(adev, |
5c583018 | 1674 | golden_settings_athub_1_0_0, |
c47b41a7 | 1675 | ARRAY_SIZE(golden_settings_athub_1_0_0)); |
e60f8db5 | 1676 | break; |
273a14cd AD |
1677 | case CHIP_VEGA12: |
1678 | break; | |
e4f3abaa | 1679 | case CHIP_RAVEN: |
8787ee01 | 1680 | /* TODO for renoir */ |
946a4d5b | 1681 | soc15_program_register_sequence(adev, |
5c583018 | 1682 | golden_settings_athub_1_0_0, |
c47b41a7 | 1683 | ARRAY_SIZE(golden_settings_athub_1_0_0)); |
e4f3abaa | 1684 | break; |
e60f8db5 AX |
1685 | default: |
1686 | break; | |
1687 | } | |
1688 | } | |
1689 | ||
c2ecd79b S |
1690 | /** |
1691 | * gmc_v9_0_restore_registers - restores regs | |
1692 | * | |
1693 | * @adev: amdgpu_device pointer | |
1694 | * | |
1695 | * This restores register values, saved at suspend. | |
1696 | */ | |
b0a2db9b | 1697 | void gmc_v9_0_restore_registers(struct amdgpu_device *adev) |
c2ecd79b | 1698 | { |
0eaa8012 | 1699 | if (adev->asic_type == CHIP_RAVEN) { |
f8646661 | 1700 | WREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0, adev->gmc.sdpif_register); |
0eaa8012 S |
1701 | WARN_ON(adev->gmc.sdpif_register != |
1702 | RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0)); | |
1703 | } | |
c2ecd79b S |
1704 | } |
1705 | ||
e60f8db5 AX |
1706 | /** |
1707 | * gmc_v9_0_gart_enable - gart enable | |
1708 | * | |
1709 | * @adev: amdgpu_device pointer | |
1710 | */ | |
1711 | static int gmc_v9_0_gart_enable(struct amdgpu_device *adev) | |
1712 | { | |
cb1545f7 | 1713 | int r; |
e60f8db5 | 1714 | |
522510a6 OZ |
1715 | if (adev->gmc.xgmi.connected_to_cpu) |
1716 | amdgpu_gmc_init_pdb0(adev); | |
1717 | ||
1123b989 | 1718 | if (adev->gart.bo == NULL) { |
e60f8db5 AX |
1719 | dev_err(adev->dev, "No VRAM object for PCIE GART.\n"); |
1720 | return -EINVAL; | |
1721 | } | |
522510a6 | 1722 | |
e548ab19 JC |
1723 | if (amdgpu_sriov_vf(adev) && amdgpu_in_reset(adev)) |
1724 | goto skip_pin_bo; | |
1725 | ||
ce1b1b66 ML |
1726 | r = amdgpu_gart_table_vram_pin(adev); |
1727 | if (r) | |
1728 | return r; | |
e60f8db5 | 1729 | |
e548ab19 | 1730 | skip_pin_bo: |
8ffff9b4 | 1731 | r = adev->gfxhub.funcs->gart_enable(adev); |
e60f8db5 AX |
1732 | if (r) |
1733 | return r; | |
1734 | ||
9fb1506e | 1735 | r = adev->mmhub.funcs->gart_enable(adev); |
e60f8db5 AX |
1736 | if (r) |
1737 | return r; | |
1738 | ||
522510a6 OZ |
1739 | DRM_INFO("PCIE GART of %uM enabled.\n", |
1740 | (unsigned)(adev->gmc.gart_size >> 20)); | |
1741 | if (adev->gmc.pdb0_bo) | |
1742 | DRM_INFO("PDB0 located at 0x%016llX\n", | |
1743 | (unsigned long long)amdgpu_bo_gpu_offset(adev->gmc.pdb0_bo)); | |
1744 | DRM_INFO("PTB located at 0x%016llX\n", | |
1745 | (unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo)); | |
1746 | ||
cb1545f7 OZ |
1747 | adev->gart.ready = true; |
1748 | return 0; | |
1749 | } | |
1750 | ||
1751 | static int gmc_v9_0_hw_init(void *handle) | |
1752 | { | |
1753 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
1754 | bool value; | |
1755 | int r, i; | |
cb1545f7 OZ |
1756 | |
1757 | /* The sequence of these two function calls matters.*/ | |
1758 | gmc_v9_0_init_golden_registers(adev); | |
1759 | ||
1760 | if (adev->mode_info.num_crtc) { | |
d0f2f634 HZ |
1761 | /* Lockout access through VGA aperture*/ |
1762 | WREG32_FIELD15(DCE, 0, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1); | |
1763 | /* disable VGA render */ | |
1764 | WREG32_FIELD15(DCE, 0, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0); | |
cb1545f7 OZ |
1765 | } |
1766 | ||
9fb1506e OZ |
1767 | if (adev->mmhub.funcs->update_power_gating) |
1768 | adev->mmhub.funcs->update_power_gating(adev, true); | |
1769 | ||
455d40c9 | 1770 | adev->hdp.funcs->init_registers(adev); |
fe2b5323 | 1771 | |
1d4e0a8c | 1772 | /* After HDP is initialized, flush HDP.*/ |
455d40c9 | 1773 | adev->hdp.funcs->flush_hdp(adev, NULL); |
1d4e0a8c | 1774 | |
e60f8db5 AX |
1775 | if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) |
1776 | value = false; | |
1777 | else | |
1778 | value = true; | |
1779 | ||
20bf2f6f | 1780 | if (!amdgpu_sriov_vf(adev)) { |
8ffff9b4 | 1781 | adev->gfxhub.funcs->set_fault_enable_default(adev, value); |
9fb1506e | 1782 | adev->mmhub.funcs->set_fault_enable_default(adev, value); |
20bf2f6f | 1783 | } |
3ff98548 OZ |
1784 | for (i = 0; i < adev->num_vmhubs; ++i) |
1785 | gmc_v9_0_flush_gpu_tlb(adev, 0, i, 0); | |
e60f8db5 | 1786 | |
e7da754b ML |
1787 | if (adev->umc.funcs && adev->umc.funcs->init_registers) |
1788 | adev->umc.funcs->init_registers(adev); | |
1789 | ||
e60f8db5 AX |
1790 | r = gmc_v9_0_gart_enable(adev); |
1791 | ||
1792 | return r; | |
1793 | } | |
1794 | ||
1795 | /** | |
1796 | * gmc_v9_0_gart_disable - gart disable | |
1797 | * | |
1798 | * @adev: amdgpu_device pointer | |
1799 | * | |
1800 | * This disables all VM page table. | |
1801 | */ | |
1802 | static void gmc_v9_0_gart_disable(struct amdgpu_device *adev) | |
1803 | { | |
8ffff9b4 | 1804 | adev->gfxhub.funcs->gart_disable(adev); |
9fb1506e | 1805 | adev->mmhub.funcs->gart_disable(adev); |
ce1b1b66 | 1806 | amdgpu_gart_table_vram_unpin(adev); |
e60f8db5 AX |
1807 | } |
1808 | ||
1809 | static int gmc_v9_0_hw_fini(void *handle) | |
1810 | { | |
1811 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
1812 | ||
66805763 LS |
1813 | gmc_v9_0_gart_disable(adev); |
1814 | ||
5dd696ae TH |
1815 | if (amdgpu_sriov_vf(adev)) { |
1816 | /* full access mode, so don't touch any GMC register */ | |
1817 | DRM_DEBUG("For SRIOV client, shouldn't do anything.\n"); | |
1818 | return 0; | |
1819 | } | |
1820 | ||
791c4769 | 1821 | amdgpu_irq_put(adev, &adev->gmc.ecc_irq, 0); |
770d13b1 | 1822 | amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0); |
e60f8db5 AX |
1823 | |
1824 | return 0; | |
1825 | } | |
1826 | ||
1827 | static int gmc_v9_0_suspend(void *handle) | |
1828 | { | |
1829 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
1830 | ||
c24a3c05 | 1831 | return gmc_v9_0_hw_fini(adev); |
e60f8db5 AX |
1832 | } |
1833 | ||
1834 | static int gmc_v9_0_resume(void *handle) | |
1835 | { | |
1836 | int r; | |
1837 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
1838 | ||
1839 | r = gmc_v9_0_hw_init(adev); | |
1840 | if (r) | |
1841 | return r; | |
1842 | ||
620f774f | 1843 | amdgpu_vmid_reset_all(adev); |
e60f8db5 | 1844 | |
32601d48 | 1845 | return 0; |
e60f8db5 AX |
1846 | } |
1847 | ||
1848 | static bool gmc_v9_0_is_idle(void *handle) | |
1849 | { | |
1850 | /* MC is always ready in GMC v9.*/ | |
1851 | return true; | |
1852 | } | |
1853 | ||
1854 | static int gmc_v9_0_wait_for_idle(void *handle) | |
1855 | { | |
1856 | /* There is no need to wait for MC idle in GMC v9.*/ | |
1857 | return 0; | |
1858 | } | |
1859 | ||
1860 | static int gmc_v9_0_soft_reset(void *handle) | |
1861 | { | |
1862 | /* XXX for emulation.*/ | |
1863 | return 0; | |
1864 | } | |
1865 | ||
1866 | static int gmc_v9_0_set_clockgating_state(void *handle, | |
1867 | enum amd_clockgating_state state) | |
1868 | { | |
d5583d4f HR |
1869 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
1870 | ||
9fb1506e | 1871 | adev->mmhub.funcs->set_clockgating(adev, state); |
bee7b51a LM |
1872 | |
1873 | athub_v1_0_set_clockgating(adev, state); | |
1874 | ||
1875 | return 0; | |
e60f8db5 AX |
1876 | } |
1877 | ||
13052be5 HR |
1878 | static void gmc_v9_0_get_clockgating_state(void *handle, u32 *flags) |
1879 | { | |
1880 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
1881 | ||
9fb1506e | 1882 | adev->mmhub.funcs->get_clockgating(adev, flags); |
bee7b51a LM |
1883 | |
1884 | athub_v1_0_get_clockgating(adev, flags); | |
13052be5 HR |
1885 | } |
1886 | ||
e60f8db5 AX |
1887 | static int gmc_v9_0_set_powergating_state(void *handle, |
1888 | enum amd_powergating_state state) | |
1889 | { | |
1890 | return 0; | |
1891 | } | |
1892 | ||
1893 | const struct amd_ip_funcs gmc_v9_0_ip_funcs = { | |
1894 | .name = "gmc_v9_0", | |
1895 | .early_init = gmc_v9_0_early_init, | |
1896 | .late_init = gmc_v9_0_late_init, | |
1897 | .sw_init = gmc_v9_0_sw_init, | |
1898 | .sw_fini = gmc_v9_0_sw_fini, | |
1899 | .hw_init = gmc_v9_0_hw_init, | |
1900 | .hw_fini = gmc_v9_0_hw_fini, | |
1901 | .suspend = gmc_v9_0_suspend, | |
1902 | .resume = gmc_v9_0_resume, | |
1903 | .is_idle = gmc_v9_0_is_idle, | |
1904 | .wait_for_idle = gmc_v9_0_wait_for_idle, | |
1905 | .soft_reset = gmc_v9_0_soft_reset, | |
1906 | .set_clockgating_state = gmc_v9_0_set_clockgating_state, | |
1907 | .set_powergating_state = gmc_v9_0_set_powergating_state, | |
13052be5 | 1908 | .get_clockgating_state = gmc_v9_0_get_clockgating_state, |
e60f8db5 AX |
1909 | }; |
1910 | ||
1911 | const struct amdgpu_ip_block_version gmc_v9_0_ip_block = | |
1912 | { | |
1913 | .type = AMD_IP_BLOCK_TYPE_GMC, | |
1914 | .major = 9, | |
1915 | .minor = 0, | |
1916 | .rev = 0, | |
1917 | .funcs = &gmc_v9_0_ip_funcs, | |
1918 | }; |