]> git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blame - drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
projects / mirror_ubuntu-hirsute-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
UBUNTU: Ubuntu-5.11.0-22.23
[mirror_ubuntu-hirsute-kernel.git] / drivers / gpu / drm / amd / amdgpu / gmc_v9_0.c
CommitLineData
e60f8db5
AX		 1/*
2 * Copyright 2016 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
f867723b 23
e60f8db5 24#include <linux/firmware.h>
f867723b
SR		 25#include <linux/pci.h>
26
fd5fd480 27#include <drm/drm_cache.h>
f867723b 28
e60f8db5
AX		 29#include "amdgpu.h"
30#include "gmc_v9_0.h"
8d6a5230 31#include "amdgpu_atomfirmware.h"
2cddc50e 32#include "amdgpu_gem.h"
e60f8db5 33
75199b8c
FX		 34#include "hdp/hdp_4_0_offset.h"
35#include "hdp/hdp_4_0_sh_mask.h"
cde5c34f 36#include "gc/gc_9_0_sh_mask.h"
135d4b10
FX		 37#include "dce/dce_12_0_offset.h"
38#include "dce/dce_12_0_sh_mask.h"
fb960bd2 39#include "vega10_enum.h"
65417d9f 40#include "mmhub/mmhub_1_0_offset.h"
ea930000 41#include "athub/athub_1_0_sh_mask.h"
6ce68225 42#include "athub/athub_1_0_offset.h"
250b4228 43#include "oss/osssys_4_0_offset.h"
e60f8db5 44
946a4d5b 45#include "soc15.h"
ea930000 46#include "soc15d.h"
e60f8db5 47#include "soc15_common.h"
90c7a935 48#include "umc/umc_6_0_sh_mask.h"
e60f8db5 49
e60f8db5
AX		 50#include "gfxhub_v1_0.h"
51#include "mmhub_v1_0.h"
bee7b51a 52#include "athub_v1_0.h"
bf0a60b7 53#include "gfxhub_v1_1.h"
51cce480 54#include "mmhub_v9_4.h"
5b6b35aa 55#include "umc_v6_1.h"
e7da754b 56#include "umc_v6_0.h"
e60f8db5 57
44a99b65
AG		 58#include "ivsrcid/vmc/irqsrcs_vmc_1_0.h"
59
791c4769 60#include "amdgpu_ras.h"
029fbd43 61#include "amdgpu_xgmi.h"
791c4769 62
ebdef28e
AD		 63/* add these here since we already include dce12 headers and these are for DCN */
64#define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION 0x055d
65#define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX 2
66#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH__SHIFT 0x0
67#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT__SHIFT 0x10
68#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH_MASK 0x00003FFFL
69#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT_MASK 0x3FFF0000L
f8646661
AD		 70#define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0 0x049d
71#define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0_BASE_IDX 2
72
ebdef28e 73
be99ecbf
AD		 74static const char *gfxhub_client_ids[] = {
75 "CB",
76 "DB",
77 "IA",
78 "WD",
79 "CPF",
80 "CPC",
81 "CPG",
82 "RLC",
83 "TCP",
84 "SQC (inst)",
85 "SQC (data)",
86 "SQG",
87 "PA",
88};
89
02f23f5f
AD		 90static const char *mmhub_client_ids_raven[][2] = {
91 [0][0] = "MP1",
92 [1][0] = "MP0",
93 [2][0] = "VCN",
94 [3][0] = "VCNU",
95 [4][0] = "HDP",
96 [5][0] = "DCE",
97 [13][0] = "UTCL2",
98 [19][0] = "TLS",
99 [26][0] = "OSS",
100 [27][0] = "SDMA0",
101 [0][1] = "MP1",
102 [1][1] = "MP0",
103 [2][1] = "VCN",
104 [3][1] = "VCNU",
105 [4][1] = "HDP",
106 [5][1] = "XDP",
107 [6][1] = "DBGU0",
108 [7][1] = "DCE",
109 [8][1] = "DCEDWB0",
110 [9][1] = "DCEDWB1",
111 [26][1] = "OSS",
112 [27][1] = "SDMA0",
113};
114
115static const char *mmhub_client_ids_renoir[][2] = {
116 [0][0] = "MP1",
117 [1][0] = "MP0",
118 [2][0] = "HDP",
119 [4][0] = "DCEDMC",
120 [5][0] = "DCEVGA",
121 [13][0] = "UTCL2",
122 [19][0] = "TLS",
123 [26][0] = "OSS",
124 [27][0] = "SDMA0",
125 [28][0] = "VCN",
126 [29][0] = "VCNU",
127 [30][0] = "JPEG",
128 [0][1] = "MP1",
129 [1][1] = "MP0",
130 [2][1] = "HDP",
131 [3][1] = "XDP",
132 [6][1] = "DBGU0",
133 [7][1] = "DCEDMC",
134 [8][1] = "DCEVGA",
135 [9][1] = "DCEDWB",
136 [26][1] = "OSS",
137 [27][1] = "SDMA0",
138 [28][1] = "VCN",
139 [29][1] = "VCNU",
140 [30][1] = "JPEG",
141};
142
143static const char *mmhub_client_ids_vega10[][2] = {
144 [0][0] = "MP0",
145 [1][0] = "UVD",
146 [2][0] = "UVDU",
147 [3][0] = "HDP",
148 [13][0] = "UTCL2",
149 [14][0] = "OSS",
150 [15][0] = "SDMA1",
151 [32+0][0] = "VCE0",
152 [32+1][0] = "VCE0U",
153 [32+2][0] = "XDMA",
154 [32+3][0] = "DCE",
155 [32+4][0] = "MP1",
156 [32+14][0] = "SDMA0",
157 [0][1] = "MP0",
158 [1][1] = "UVD",
159 [2][1] = "UVDU",
160 [3][1] = "DBGU0",
161 [4][1] = "HDP",
162 [5][1] = "XDP",
163 [14][1] = "OSS",
164 [15][1] = "SDMA0",
165 [32+0][1] = "VCE0",
166 [32+1][1] = "VCE0U",
167 [32+2][1] = "XDMA",
168 [32+3][1] = "DCE",
169 [32+4][1] = "DCEDWB",
170 [32+5][1] = "MP1",
171 [32+6][1] = "DBGU1",
172 [32+14][1] = "SDMA1",
173};
174
175static const char *mmhub_client_ids_vega12[][2] = {
176 [0][0] = "MP0",
177 [1][0] = "VCE0",
178 [2][0] = "VCE0U",
179 [3][0] = "HDP",
180 [13][0] = "UTCL2",
181 [14][0] = "OSS",
182 [15][0] = "SDMA1",
183 [32+0][0] = "DCE",
184 [32+1][0] = "XDMA",
185 [32+2][0] = "UVD",
186 [32+3][0] = "UVDU",
187 [32+4][0] = "MP1",
188 [32+15][0] = "SDMA0",
189 [0][1] = "MP0",
190 [1][1] = "VCE0",
191 [2][1] = "VCE0U",
192 [3][1] = "DBGU0",
193 [4][1] = "HDP",
194 [5][1] = "XDP",
195 [14][1] = "OSS",
196 [15][1] = "SDMA0",
197 [32+0][1] = "DCE",
198 [32+1][1] = "DCEDWB",
199 [32+2][1] = "XDMA",
200 [32+3][1] = "UVD",
201 [32+4][1] = "UVDU",
202 [32+5][1] = "MP1",
203 [32+6][1] = "DBGU1",
204 [32+15][1] = "SDMA1",
205};
206
207static const char *mmhub_client_ids_vega20[][2] = {
208 [0][0] = "XDMA",
209 [1][0] = "DCE",
210 [2][0] = "VCE0",
211 [3][0] = "VCE0U",
212 [4][0] = "UVD",
213 [5][0] = "UVD1U",
214 [13][0] = "OSS",
215 [14][0] = "HDP",
216 [15][0] = "SDMA0",
217 [32+0][0] = "UVD",
218 [32+1][0] = "UVDU",
219 [32+2][0] = "MP1",
220 [32+3][0] = "MP0",
221 [32+12][0] = "UTCL2",
222 [32+14][0] = "SDMA1",
223 [0][1] = "XDMA",
224 [1][1] = "DCE",
225 [2][1] = "DCEDWB",
226 [3][1] = "VCE0",
227 [4][1] = "VCE0U",
228 [5][1] = "UVD1",
229 [6][1] = "UVD1U",
230 [7][1] = "DBGU0",
231 [8][1] = "XDP",
232 [13][1] = "OSS",
233 [14][1] = "HDP",
234 [15][1] = "SDMA0",
235 [32+0][1] = "UVD",
236 [32+1][1] = "UVDU",
237 [32+2][1] = "DBGU1",
238 [32+3][1] = "MP1",
239 [32+4][1] = "MP0",
240 [32+14][1] = "SDMA1",
241};
242
243static const char *mmhub_client_ids_arcturus[][2] = {
244 [2][0] = "MP1",
245 [3][0] = "MP0",
246 [10][0] = "UTCL2",
247 [13][0] = "OSS",
248 [14][0] = "HDP",
249 [15][0] = "SDMA0",
250 [32+15][0] = "SDMA1",
251 [64+15][0] = "SDMA2",
252 [96+15][0] = "SDMA3",
253 [128+15][0] = "SDMA4",
254 [160+11][0] = "JPEG",
255 [160+12][0] = "VCN",
256 [160+13][0] = "VCNU",
257 [160+15][0] = "SDMA5",
258 [192+10][0] = "UTCL2",
259 [192+11][0] = "JPEG1",
260 [192+12][0] = "VCN1",
261 [192+13][0] = "VCN1U",
262 [192+15][0] = "SDMA6",
263 [224+15][0] = "SDMA7",
264 [0][1] = "DBGU1",
265 [1][1] = "XDP",
266 [2][1] = "MP1",
267 [3][1] = "MP0",
268 [13][1] = "OSS",
269 [14][1] = "HDP",
270 [15][1] = "SDMA0",
271 [32+15][1] = "SDMA1",
02f23f5f
AD		 272 [64+15][1] = "SDMA2",
273 [96+15][1] = "SDMA3",
274 [128+15][1] = "SDMA4",
275 [160+11][1] = "JPEG",
276 [160+12][1] = "VCN",
277 [160+13][1] = "VCNU",
278 [160+15][1] = "SDMA5",
279 [192+11][1] = "JPEG1",
280 [192+12][1] = "VCN1",
281 [192+13][1] = "VCN1U",
282 [192+15][1] = "SDMA6",
283 [224+15][1] = "SDMA7",
284};
ebdef28e 285
e60f8db5
AX		 286static const u32 golden_settings_vega10_hdp[] =
287{
288 0xf64, 0x0fffffff, 0x00000000,
289 0xf65, 0x0fffffff, 0x00000000,
290 0xf66, 0x0fffffff, 0x00000000,
291 0xf67, 0x0fffffff, 0x00000000,
292 0xf68, 0x0fffffff, 0x00000000,
293 0xf6a, 0x0fffffff, 0x00000000,
294 0xf6b, 0x0fffffff, 0x00000000,
295 0xf6c, 0x0fffffff, 0x00000000,
296 0xf6d, 0x0fffffff, 0x00000000,
297 0xf6e, 0x0fffffff, 0x00000000,
298};
299
946a4d5b 300static const struct soc15_reg_golden golden_settings_mmhub_1_0_0[] =
5c583018 301{
946a4d5b
SL		 302 SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmDAGB1_WRCLI2, 0x00000007, 0xfe5fe0fa),
303 SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmMMEA1_DRAM_WR_CLI2GRP_MAP0, 0x00000030, 0x55555565)
5c583018
EQ		 304};
305
946a4d5b 306static const struct soc15_reg_golden golden_settings_athub_1_0_0[] =
5c583018 307{
946a4d5b
SL		 308 SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL, 0x0000ff00, 0x00000800),
309 SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL2, 0x00ff00ff, 0x00080008)
5c583018
EQ		 310};
311
791c4769 312static const uint32_t ecc_umc_mcumc_ctrl_addrs[] = {
313 (0x000143c0 + 0x00000000),
314 (0x000143c0 + 0x00000800),
315 (0x000143c0 + 0x00001000),
316 (0x000143c0 + 0x00001800),
317 (0x000543c0 + 0x00000000),
318 (0x000543c0 + 0x00000800),
319 (0x000543c0 + 0x00001000),
320 (0x000543c0 + 0x00001800),
321 (0x000943c0 + 0x00000000),
322 (0x000943c0 + 0x00000800),
323 (0x000943c0 + 0x00001000),
324 (0x000943c0 + 0x00001800),
325 (0x000d43c0 + 0x00000000),
326 (0x000d43c0 + 0x00000800),
327 (0x000d43c0 + 0x00001000),
328 (0x000d43c0 + 0x00001800),
329 (0x001143c0 + 0x00000000),
330 (0x001143c0 + 0x00000800),
331 (0x001143c0 + 0x00001000),
332 (0x001143c0 + 0x00001800),
333 (0x001543c0 + 0x00000000),
334 (0x001543c0 + 0x00000800),
335 (0x001543c0 + 0x00001000),
336 (0x001543c0 + 0x00001800),
337 (0x001943c0 + 0x00000000),
338 (0x001943c0 + 0x00000800),
339 (0x001943c0 + 0x00001000),
340 (0x001943c0 + 0x00001800),
341 (0x001d43c0 + 0x00000000),
342 (0x001d43c0 + 0x00000800),
343 (0x001d43c0 + 0x00001000),
344 (0x001d43c0 + 0x00001800),
02bab923
DP		 345};
346
791c4769 347static const uint32_t ecc_umc_mcumc_ctrl_mask_addrs[] = {
348 (0x000143e0 + 0x00000000),
349 (0x000143e0 + 0x00000800),
350 (0x000143e0 + 0x00001000),
351 (0x000143e0 + 0x00001800),
352 (0x000543e0 + 0x00000000),
353 (0x000543e0 + 0x00000800),
354 (0x000543e0 + 0x00001000),
355 (0x000543e0 + 0x00001800),
356 (0x000943e0 + 0x00000000),
357 (0x000943e0 + 0x00000800),
358 (0x000943e0 + 0x00001000),
359 (0x000943e0 + 0x00001800),
360 (0x000d43e0 + 0x00000000),
361 (0x000d43e0 + 0x00000800),
362 (0x000d43e0 + 0x00001000),
363 (0x000d43e0 + 0x00001800),
364 (0x001143e0 + 0x00000000),
365 (0x001143e0 + 0x00000800),
366 (0x001143e0 + 0x00001000),
367 (0x001143e0 + 0x00001800),
368 (0x001543e0 + 0x00000000),
369 (0x001543e0 + 0x00000800),
370 (0x001543e0 + 0x00001000),
371 (0x001543e0 + 0x00001800),
372 (0x001943e0 + 0x00000000),
373 (0x001943e0 + 0x00000800),
374 (0x001943e0 + 0x00001000),
375 (0x001943e0 + 0x00001800),
376 (0x001d43e0 + 0x00000000),
377 (0x001d43e0 + 0x00000800),
378 (0x001d43e0 + 0x00001000),
379 (0x001d43e0 + 0x00001800),
02bab923
DP		 380};
381
791c4769 382static int gmc_v9_0_ecc_interrupt_state(struct amdgpu_device *adev,
383 struct amdgpu_irq_src *src,
384 unsigned type,
385 enum amdgpu_interrupt_state state)
386{
387 u32 bits, i, tmp, reg;
1e2c6d55
JC		 388
389 /* Devices newer then VEGA10/12 shall have these programming
390 sequences performed by PSP BL */
391 if (adev->asic_type >= CHIP_VEGA20)
392 return 0;
791c4769 393
394 bits = 0x7f;
395
396 switch (state) {
397 case AMDGPU_IRQ_STATE_DISABLE:
398 for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) {
399 reg = ecc_umc_mcumc_ctrl_addrs[i];
400 tmp = RREG32(reg);
401 tmp &= ~bits;
402 WREG32(reg, tmp);
403 }
404 for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) {
405 reg = ecc_umc_mcumc_ctrl_mask_addrs[i];
406 tmp = RREG32(reg);
407 tmp &= ~bits;
408 WREG32(reg, tmp);
409 }
410 break;
411 case AMDGPU_IRQ_STATE_ENABLE:
412 for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) {
413 reg = ecc_umc_mcumc_ctrl_addrs[i];
414 tmp = RREG32(reg);
415 tmp |= bits;
416 WREG32(reg, tmp);
417 }
418 for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) {
419 reg = ecc_umc_mcumc_ctrl_mask_addrs[i];
420 tmp = RREG32(reg);
421 tmp |= bits;
422 WREG32(reg, tmp);
423 }
424 break;
425 default:
426 break;
427 }
428
429 return 0;
430}
431
e60f8db5
AX		 432static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
433 struct amdgpu_irq_src *src,
434 unsigned type,
435 enum amdgpu_interrupt_state state)
436{
437 struct amdgpu_vmhub *hub;
ae6d1416 438 u32 tmp, reg, bits, i, j;
e60f8db5 439
11250164
CK		 440 bits = VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
441 VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
442 VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
443 VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
444 VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
445 VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
446 VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK;
447
e60f8db5
AX		 448 switch (state) {
449 case AMDGPU_IRQ_STATE_DISABLE:
1daa2bfa 450 for (j = 0; j < adev->num_vmhubs; j++) {
ae6d1416
TSD		 451 hub = &adev->vmhub[j];
452 for (i = 0; i < 16; i++) {
453 reg = hub->vm_context0_cntl + i;
454 tmp = RREG32(reg);
455 tmp &= ~bits;
456 WREG32(reg, tmp);
457 }
e60f8db5
AX		 458 }
459 break;
460 case AMDGPU_IRQ_STATE_ENABLE:
1daa2bfa 461 for (j = 0; j < adev->num_vmhubs; j++) {
ae6d1416
TSD		 462 hub = &adev->vmhub[j];
463 for (i = 0; i < 16; i++) {
464 reg = hub->vm_context0_cntl + i;
465 tmp = RREG32(reg);
466 tmp |= bits;
467 WREG32(reg, tmp);
468 }
e60f8db5 469 }
9304ca4d 470 break;
e60f8db5
AX		 471 default:
472 break;
473 }
474
475 return 0;
476}
477
478static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev,
e3898719
CK		 479 struct amdgpu_irq_src *source,
480 struct amdgpu_iv_entry *entry)
e60f8db5 481{
c468f9e2 482 bool retry_fault = !!(entry->src_data[1] & 0x80);
02f23f5f 483 uint32_t status = 0, cid = 0, rw = 0;
e3898719
CK		 484 struct amdgpu_task_info task_info;
485 struct amdgpu_vmhub *hub;
02f23f5f 486 const char *mmhub_cid;
e3898719
CK		 487 const char *hub_name;
488 u64 addr;
e60f8db5
AX		 489
490 addr = (u64)entry->src_data[0] << 12;
491 addr |= ((u64)entry->src_data[1] & 0xf) << 44;
492
0291150d
CK		 493 if (retry_fault) {
494 /* Returning 1 here also prevents sending the IV to the KFD */
495
496 /* Process it onyl if it's the first fault for this address */
497 if (entry->ih != &adev->irq.ih_soft &&
498 amdgpu_gmc_filter_faults(adev, addr, entry->pasid,
499 entry->timestamp))
500 return 1;
501
502 /* Delegate it to a different ring if the hardware hasn't
503 * already done it.
504 */
505 if (in_interrupt()) {
506 amdgpu_irq_delegate(adev, entry, 8);
507 return 1;
508 }
509
510 /* Try to handle the recoverable page faults by filling page
511 * tables
512 */
513 if (amdgpu_vm_handle_fault(adev, entry->pasid, addr))
514 return 1;
515 }
e3898719
CK		 516
517 if (!printk_ratelimit())
518 return 0;
519
51c60898 520 if (entry->client_id == SOC15_IH_CLIENTID_VMC) {
e3898719 521 hub_name = "mmhub0";
51c60898
LM		 522 hub = &adev->vmhub[AMDGPU_MMHUB_0];
523 } else if (entry->client_id == SOC15_IH_CLIENTID_VMC1) {
e3898719 524 hub_name = "mmhub1";
51c60898
LM		 525 hub = &adev->vmhub[AMDGPU_MMHUB_1];
526 } else {
e3898719 527 hub_name = "gfxhub0";
51c60898
LM		 528 hub = &adev->vmhub[AMDGPU_GFXHUB_0];
529 }
530
e3898719
CK		 531 memset(&task_info, 0, sizeof(struct amdgpu_task_info));
532 amdgpu_vm_get_task_info(adev, entry->pasid, &task_info);
ec671737 533
e3898719
CK		 534 dev_err(adev->dev,
535 "[%s] %s page fault (src_id:%u ring:%u vmid:%u "
536 "pasid:%u, for process %s pid %d thread %s pid %d)\n",
537 hub_name, retry_fault ? "retry" : "no-retry",
538 entry->src_id, entry->ring_id, entry->vmid,
539 entry->pasid, task_info.process_name, task_info.tgid,
540 task_info.task_name, task_info.pid);
18e1a6c9 541 dev_err(adev->dev, " in page starting at address 0x%012llx from client %d\n",
e3898719 542 addr, entry->client_id);
e60f8db5 543
e3898719
CK		 544 if (amdgpu_sriov_vf(adev))
545 return 0;
546
547 /*
548 * Issue a dummy read to wait for the status register to
549 * be updated to avoid reading an incorrect value due to
550 * the new fast GRBM interface.
551 */
552 if (entry->vmid_src == AMDGPU_GFXHUB_0)
553 RREG32(hub->vm_l2_pro_fault_status);
554
555 status = RREG32(hub->vm_l2_pro_fault_status);
556 cid = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, CID);
557 rw = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, RW);
558 WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1);
559
560
561 dev_err(adev->dev,
562 "VM_L2_PROTECTION_FAULT_STATUS:0x%08X\n",
563 status);
564 if (hub == &adev->vmhub[AMDGPU_GFXHUB_0]) {
565 dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
566 cid >= ARRAY_SIZE(gfxhub_client_ids) ? "unknown" :
567 gfxhub_client_ids[cid],
568 cid);
569 } else {
570 switch (adev->asic_type) {
571 case CHIP_VEGA10:
572 mmhub_cid = mmhub_client_ids_vega10[cid][rw];
573 break;
574 case CHIP_VEGA12:
575 mmhub_cid = mmhub_client_ids_vega12[cid][rw];
576 break;
577 case CHIP_VEGA20:
578 mmhub_cid = mmhub_client_ids_vega20[cid][rw];
579 break;
580 case CHIP_ARCTURUS:
581 mmhub_cid = mmhub_client_ids_arcturus[cid][rw];
582 break;
583 case CHIP_RAVEN:
584 mmhub_cid = mmhub_client_ids_raven[cid][rw];
585 break;
586 case CHIP_RENOIR:
587 mmhub_cid = mmhub_client_ids_renoir[cid][rw];
588 break;
589 default:
590 mmhub_cid = NULL;
591 break;
5ddd4a9a 592 }
e3898719
CK		 593 dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
594 mmhub_cid ? mmhub_cid : "unknown", cid);
79a0c465 595 }
e3898719
CK		 596 dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n",
597 REG_GET_FIELD(status,
598 VM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS));
599 dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n",
600 REG_GET_FIELD(status,
601 VM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR));
602 dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n",
603 REG_GET_FIELD(status,
604 VM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS));
605 dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n",
606 REG_GET_FIELD(status,
607 VM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR));
608 dev_err(adev->dev, "\t RW: 0x%x\n", rw);
e60f8db5
AX		 609 return 0;
610}
611
612static const struct amdgpu_irq_src_funcs gmc_v9_0_irq_funcs = {
613 .set = gmc_v9_0_vm_fault_interrupt_state,
614 .process = gmc_v9_0_process_interrupt,
615};
616
791c4769 617
618static const struct amdgpu_irq_src_funcs gmc_v9_0_ecc_funcs = {
619 .set = gmc_v9_0_ecc_interrupt_state,
34cc4fd9 620 .process = amdgpu_umc_process_ecc_irq,
791c4769 621};
622
e60f8db5
AX		 623static void gmc_v9_0_set_irq_funcs(struct amdgpu_device *adev)
624{
770d13b1
CK		 625 adev->gmc.vm_fault.num_types = 1;
626 adev->gmc.vm_fault.funcs = &gmc_v9_0_irq_funcs;
791c4769 627
2ee9403e
ZL		 628 if (!amdgpu_sriov_vf(adev)) {
629 adev->gmc.ecc_irq.num_types = 1;
630 adev->gmc.ecc_irq.funcs = &gmc_v9_0_ecc_funcs;
631 }
e60f8db5
AX		 632}
633
2a79d868
YZ		 634static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid,
635 uint32_t flush_type)
03f89feb
CK		 636{
637 u32 req = 0;
638
03f89feb 639 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
c4f46f22 640 PER_VMID_INVALIDATE_REQ, 1 << vmid);
2a79d868 641 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type);
03f89feb
CK		 642 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1);
643 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1);
644 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1);
645 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1);
646 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1);
647 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
648 CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0);
649
650 return req;
651}
652
90f6452c 653/**
654 * gmc_v9_0_use_invalidate_semaphore - judge whether to use semaphore
655 *
656 * @adev: amdgpu_device pointer
657 * @vmhub: vmhub type
658 *
659 */
660static bool gmc_v9_0_use_invalidate_semaphore(struct amdgpu_device *adev,
661 uint32_t vmhub)
662{
663 return ((vmhub == AMDGPU_MMHUB_0 ||
664 vmhub == AMDGPU_MMHUB_1) &&
665 (!amdgpu_sriov_vf(adev)) &&
54f78a76
AD		 666 (!(!(adev->apu_flags & AMD_APU_IS_RAVEN2) &&
667 (adev->apu_flags & AMD_APU_IS_PICASSO))));
90f6452c 668}
669
ea930000
AS		 670static bool gmc_v9_0_get_atc_vmid_pasid_mapping_info(struct amdgpu_device *adev,
671 uint8_t vmid, uint16_t *p_pasid)
672{
673 uint32_t value;
674
675 value = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING)
676 + vmid);
677 *p_pasid = value & ATC_VMID0_PASID_MAPPING__PASID_MASK;
678
679 return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK);
680}
681
e60f8db5
AX		 682/*
683 * GART
684 * VMID 0 is the physical GPU addresses as used by the kernel.
685 * VMIDs 1-15 are used for userspace clients and are handled
686 * by the amdgpu vm/hsa code.
687 */
688
689/**
2a79d868 690 * gmc_v9_0_flush_gpu_tlb - tlb flush with certain type
e60f8db5
AX		 691 *
692 * @adev: amdgpu_device pointer
693 * @vmid: vm instance to flush
bf0df09c 694 * @vmhub: which hub to flush
2a79d868 695 * @flush_type: the flush type
e60f8db5 696 *
2a79d868 697 * Flush the TLB for the requested page table using certain type.
e60f8db5 698 */
3ff98548
OZ		 699static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
700 uint32_t vmhub, uint32_t flush_type)
e60f8db5 701{
90f6452c 702 bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(adev, vmhub);
e60f8db5 703 const unsigned eng = 17;
b80cd524 704 u32 j, inv_req, inv_req2, tmp;
3ff98548 705 struct amdgpu_vmhub *hub;
e60f8db5 706
3ff98548 707 BUG_ON(vmhub >= adev->num_vmhubs);
e60f8db5 708
3ff98548 709 hub = &adev->vmhub[vmhub];
b80cd524
FK		 710 if (adev->gmc.xgmi.num_physical_nodes &&
711 adev->asic_type == CHIP_VEGA20) {
712 /* Vega20+XGMI caches PTEs in TC and TLB. Add a
713 * heavy-weight TLB flush (type 2), which flushes
714 * both. Due to a race condition with concurrent
715 * memory accesses using the same TLB cache line, we
716 * still need a second TLB flush after this.
717 */
718 inv_req = gmc_v9_0_get_invalidate_req(vmid, 2);
719 inv_req2 = gmc_v9_0_get_invalidate_req(vmid, flush_type);
720 } else {
721 inv_req = gmc_v9_0_get_invalidate_req(vmid, flush_type);
722 inv_req2 = 0;
723 }
3890d111 724
3ff98548
OZ		 725 /* This is necessary for a HW workaround under SRIOV as well
726 * as GFXOFF under bare metal
727 */
728 if (adev->gfx.kiq.ring.sched.ready &&
81202807
DL		 729 (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev)) &&
730 down_read_trylock(&adev->reset_sem)) {
148f597d
HR		 731 uint32_t req = hub->vm_inv_eng0_req + hub->eng_distance * eng;
732 uint32_t ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng;
3ff98548 733
37c58ddf 734 amdgpu_virt_kiq_reg_write_reg_wait(adev, req, ack, inv_req,
148f597d 735 1 << vmid);
81202807 736 up_read(&adev->reset_sem);
3ff98548
OZ		 737 return;
738 }
396557b0 739
3ff98548 740 spin_lock(&adev->gmc.invalidate_lock);
f920d1bb 741
742 /*
743 * It may lose gpuvm invalidate acknowldege state across power-gating
744 * off cycle, add semaphore acquire before invalidation and semaphore
745 * release after invalidation to avoid entering power gated state
746 * to WA the Issue
747 */
748
749 /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
90f6452c 750 if (use_semaphore) {
f920d1bb 751 for (j = 0; j < adev->usec_timeout; j++) {
752 /* a read return value of 1 means semaphore acuqire */
148f597d
HR		 753 tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_sem +
754 hub->eng_distance * eng);
f920d1bb 755 if (tmp & 0x1)
756 break;
757 udelay(1);
758 }
759
760 if (j >= adev->usec_timeout)
761 DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n");
762 }
763
b80cd524 764 do {
148f597d
HR		 765 WREG32_NO_KIQ(hub->vm_inv_eng0_req +
766 hub->eng_distance * eng, inv_req);
53499173 767
b80cd524
FK		 768 /*
769 * Issue a dummy read to wait for the ACK register to
770 * be cleared to avoid a false ACK due to the new fast
771 * GRBM interface.
772 */
773 if (vmhub == AMDGPU_GFXHUB_0)
148f597d
HR		 774 RREG32_NO_KIQ(hub->vm_inv_eng0_req +
775 hub->eng_distance * eng);
53499173 776
b80cd524 777 for (j = 0; j < adev->usec_timeout; j++) {
148f597d
HR		 778 tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_ack +
779 hub->eng_distance * eng);
b80cd524
FK		 780 if (tmp & (1 << vmid))
781 break;
782 udelay(1);
783 }
784
785 inv_req = inv_req2;
786 inv_req2 = 0;
787 } while (inv_req);
f920d1bb 788
789 /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
90f6452c 790 if (use_semaphore)
f920d1bb 791 /*
792 * add semaphore release after invalidation,
793 * write with 0 means semaphore release
794 */
148f597d
HR		 795 WREG32_NO_KIQ(hub->vm_inv_eng0_sem +
796 hub->eng_distance * eng, 0);
f920d1bb 797
3ff98548 798 spin_unlock(&adev->gmc.invalidate_lock);
f920d1bb 799
3ff98548
OZ		 800 if (j < adev->usec_timeout)
801 return;
802
803 DRM_ERROR("Timeout waiting for VM flush ACK!\n");
e60f8db5
AX		 804}
805
ea930000
AS		 806/**
807 * gmc_v9_0_flush_gpu_tlb_pasid - tlb flush via pasid
808 *
809 * @adev: amdgpu_device pointer
810 * @pasid: pasid to be flush
bf0df09c
LJ		 811 * @flush_type: the flush type
812 * @all_hub: flush all hubs
ea930000
AS		 813 *
814 * Flush the TLB for the requested pasid.
815 */
816static int gmc_v9_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
817 uint16_t pasid, uint32_t flush_type,
818 bool all_hub)
819{
820 int vmid, i;
821 signed long r;
822 uint32_t seq;
823 uint16_t queried_pasid;
824 bool ret;
825 struct amdgpu_ring *ring = &adev->gfx.kiq.ring;
826 struct amdgpu_kiq *kiq = &adev->gfx.kiq;
827
53b3f8f4 828 if (amdgpu_in_reset(adev))
ea930000
AS		 829 return -EIO;
830
81202807 831 if (ring->sched.ready && down_read_trylock(&adev->reset_sem)) {
b80cd524
FK		 832 /* Vega20+XGMI caches PTEs in TC and TLB. Add a
833 * heavy-weight TLB flush (type 2), which flushes
834 * both. Due to a race condition with concurrent
835 * memory accesses using the same TLB cache line, we
836 * still need a second TLB flush after this.
837 */
838 bool vega20_xgmi_wa = (adev->gmc.xgmi.num_physical_nodes &&
839 adev->asic_type == CHIP_VEGA20);
840 /* 2 dwords flush + 8 dwords fence */
841 unsigned int ndw = kiq->pmf->invalidate_tlbs_size + 8;
842
843 if (vega20_xgmi_wa)
844 ndw += kiq->pmf->invalidate_tlbs_size;
845
ea930000 846 spin_lock(&adev->gfx.kiq.ring_lock);
36a1707a 847 /* 2 dwords flush + 8 dwords fence */
b80cd524
FK		 848 amdgpu_ring_alloc(ring, ndw);
849 if (vega20_xgmi_wa)
850 kiq->pmf->kiq_invalidate_tlbs(ring,
851 pasid, 2, all_hub);
ea930000
AS		 852 kiq->pmf->kiq_invalidate_tlbs(ring,
853 pasid, flush_type, all_hub);
04e4e2e9
YT		 854 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
855 if (r) {
856 amdgpu_ring_undo(ring);
abb17b1e 857 spin_unlock(&adev->gfx.kiq.ring_lock);
81202807 858 up_read(&adev->reset_sem);
04e4e2e9
YT		 859 return -ETIME;
860 }
861
ea930000
AS		 862 amdgpu_ring_commit(ring);
863 spin_unlock(&adev->gfx.kiq.ring_lock);
864 r = amdgpu_fence_wait_polling(ring, seq, adev->usec_timeout);
865 if (r < 1) {
aac89168 866 dev_err(adev->dev, "wait for kiq fence error: %ld.\n", r);
81202807 867 up_read(&adev->reset_sem);
ea930000
AS		 868 return -ETIME;
869 }
81202807 870 up_read(&adev->reset_sem);
ea930000
AS		 871 return 0;
872 }
873
874 for (vmid = 1; vmid < 16; vmid++) {
875
876 ret = gmc_v9_0_get_atc_vmid_pasid_mapping_info(adev, vmid,
877 &queried_pasid);
878 if (ret && queried_pasid == pasid) {
879 if (all_hub) {
880 for (i = 0; i < adev->num_vmhubs; i++)
881 gmc_v9_0_flush_gpu_tlb(adev, vmid,
fa34edbe 882 i, flush_type);
ea930000
AS		 883 } else {
884 gmc_v9_0_flush_gpu_tlb(adev, vmid,
fa34edbe 885 AMDGPU_GFXHUB_0, flush_type);
ea930000
AS		 886 }
887 break;
888 }
889 }
890
891 return 0;
892
893}
894
9096d6e5 895static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
c633c00b 896 unsigned vmid, uint64_t pd_addr)
9096d6e5 897{
90f6452c 898 bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(ring->adev, ring->funcs->vmhub);
250b4228
CK		 899 struct amdgpu_device *adev = ring->adev;
900 struct amdgpu_vmhub *hub = &adev->vmhub[ring->funcs->vmhub];
2a79d868 901 uint32_t req = gmc_v9_0_get_invalidate_req(vmid, 0);
9096d6e5
CK		 902 unsigned eng = ring->vm_inv_eng;
903
f920d1bb 904 /*
905 * It may lose gpuvm invalidate acknowldege state across power-gating
906 * off cycle, add semaphore acquire before invalidation and semaphore
907 * release after invalidation to avoid entering power gated state
908 * to WA the Issue
909 */
910
911 /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
90f6452c 912 if (use_semaphore)
f920d1bb 913 /* a read return value of 1 means semaphore acuqire */
914 amdgpu_ring_emit_reg_wait(ring,
148f597d
HR		 915 hub->vm_inv_eng0_sem +
916 hub->eng_distance * eng, 0x1, 0x1);
f920d1bb 917
148f597d
HR		 918 amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 +
919 (hub->ctx_addr_distance * vmid),
9096d6e5
CK		 920 lower_32_bits(pd_addr));
921
148f597d
HR		 922 amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 +
923 (hub->ctx_addr_distance * vmid),
9096d6e5
CK		 924 upper_32_bits(pd_addr));
925
148f597d
HR		 926 amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +
927 hub->eng_distance * eng,
928 hub->vm_inv_eng0_ack +
929 hub->eng_distance * eng,
f8bc9037 930 req, 1 << vmid);
f732b6b3 931
f920d1bb 932 /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
90f6452c 933 if (use_semaphore)
f920d1bb 934 /*
935 * add semaphore release after invalidation,
936 * write with 0 means semaphore release
937 */
148f597d
HR		 938 amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem +
939 hub->eng_distance * eng, 0);
f920d1bb 940
9096d6e5
CK		 941 return pd_addr;
942}
943
c633c00b
CK		 944static void gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid,
945 unsigned pasid)
946{
947 struct amdgpu_device *adev = ring->adev;
948 uint32_t reg;
949
f2d66571
LM		 950 /* Do nothing because there's no lut register for mmhub1. */
951 if (ring->funcs->vmhub == AMDGPU_MMHUB_1)
952 return;
953
a2d15ed7 954 if (ring->funcs->vmhub == AMDGPU_GFXHUB_0)
c633c00b
CK		 955 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid;
956 else
957 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid;
958
959 amdgpu_ring_emit_wreg(ring, reg, pasid);
960}
961
6490bd76
YZ		 962/*
963 * PTE format on VEGA 10:
964 * 63:59 reserved
965 * 58:57 mtype
966 * 56 F
967 * 55 L
968 * 54 P
969 * 53 SW
970 * 52 T
971 * 50:48 reserved
972 * 47:12 4k physical page base address
973 * 11:7 fragment
974 * 6 write
975 * 5 read
976 * 4 exe
977 * 3 Z
978 * 2 snooped
979 * 1 system
980 * 0 valid
e60f8db5 981 *
6490bd76
YZ		 982 * PDE format on VEGA 10:
983 * 63:59 block fragment size
984 * 58:55 reserved
985 * 54 P
986 * 53:48 reserved
987 * 47:6 physical base address of PD or PTE
988 * 5:3 reserved
989 * 2 C
990 * 1 system
991 * 0 valid
e60f8db5 992 */
e60f8db5 993
71776b6d 994static uint64_t gmc_v9_0_map_mtype(struct amdgpu_device *adev, uint32_t flags)
e60f8db5
AX		 995
996{
71776b6d 997 switch (flags) {
e60f8db5 998 case AMDGPU_VM_MTYPE_DEFAULT:
71776b6d 999 return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC);
e60f8db5 1000 case AMDGPU_VM_MTYPE_NC:
71776b6d 1001 return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC);
e60f8db5 1002 case AMDGPU_VM_MTYPE_WC:
71776b6d 1003 return AMDGPU_PTE_MTYPE_VG10(MTYPE_WC);
093e48c0 1004 case AMDGPU_VM_MTYPE_RW:
71776b6d 1005 return AMDGPU_PTE_MTYPE_VG10(MTYPE_RW);
e60f8db5 1006 case AMDGPU_VM_MTYPE_CC:
71776b6d 1007 return AMDGPU_PTE_MTYPE_VG10(MTYPE_CC);
e60f8db5 1008 case AMDGPU_VM_MTYPE_UC:
71776b6d 1009 return AMDGPU_PTE_MTYPE_VG10(MTYPE_UC);
e60f8db5 1010 default:
71776b6d 1011 return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC);
e60f8db5 1012 }
e60f8db5
AX		 1013}
1014
3de676d8
CK		 1015static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level,
1016 uint64_t *addr, uint64_t *flags)
e60f8db5 1017{
bbc9fb10 1018 if (!(*flags & AMDGPU_PDE_PTE) && !(*flags & AMDGPU_PTE_SYSTEM))
3de676d8 1019 *addr = adev->vm_manager.vram_base_offset + *addr -
770d13b1 1020 adev->gmc.vram_start;
3de676d8 1021 BUG_ON(*addr & 0xFFFF00000000003FULL);
6a42fd6f 1022
770d13b1 1023 if (!adev->gmc.translate_further)
6a42fd6f
CK		 1024 return;
1025
1026 if (level == AMDGPU_VM_PDB1) {
1027 /* Set the block fragment size */
1028 if (!(*flags & AMDGPU_PDE_PTE))
1029 *flags |= AMDGPU_PDE_BFS(0x9);
1030
1031 } else if (level == AMDGPU_VM_PDB0) {
1032 if (*flags & AMDGPU_PDE_PTE)
1033 *flags &= ~AMDGPU_PDE_PTE;
1034 else
1035 *flags |= AMDGPU_PTE_TF;
1036 }
e60f8db5
AX		 1037}
1038
cbfae36c
CK		 1039static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev,
1040 struct amdgpu_bo_va_mapping *mapping,
1041 uint64_t *flags)
1042{
1043 *flags &= ~AMDGPU_PTE_EXECUTABLE;
1044 *flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE;
1045
1046 *flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK;
1047 *flags |= mapping->flags & AMDGPU_PTE_MTYPE_VG10_MASK;
1048
1049 if (mapping->flags & AMDGPU_PTE_PRT) {
1050 *flags |= AMDGPU_PTE_PRT;
1051 *flags &= ~AMDGPU_PTE_VALID;
1052 }
1053
1054 if (adev->asic_type == CHIP_ARCTURUS &&
1055 !(*flags & AMDGPU_PTE_SYSTEM) &&
1056 mapping->bo_va->is_xgmi)
1057 *flags |= AMDGPU_PTE_SNOOPED;
1058}
1059
7b885f0e
AD		 1060static unsigned gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev)
1061{
1062 u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL);
1063 unsigned size;
1064
1065 if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) {
1066 size = AMDGPU_VBIOS_VGA_ALLOCATION;
1067 } else {
1068 u32 viewport;
1069
1070 switch (adev->asic_type) {
1071 case CHIP_RAVEN:
1072 case CHIP_RENOIR:
1073 viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION);
1074 size = (REG_GET_FIELD(viewport,
1075 HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) *
1076 REG_GET_FIELD(viewport,
1077 HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) *
1078 4);
1079 break;
1080 case CHIP_VEGA10:
1081 case CHIP_VEGA12:
1082 case CHIP_VEGA20:
1083 default:
1084 viewport = RREG32_SOC15(DCE, 0, mmSCL0_VIEWPORT_SIZE);
1085 size = (REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_HEIGHT) *
1086 REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_WIDTH) *
1087 4);
1088 break;
1089 }
1090 }
1091
1092 return size;
1093}
1094
132f34e4
CK		 1095static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = {
1096 .flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb,
ea930000 1097 .flush_gpu_tlb_pasid = gmc_v9_0_flush_gpu_tlb_pasid,
9096d6e5 1098 .emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb,
c633c00b 1099 .emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping,
71776b6d 1100 .map_mtype = gmc_v9_0_map_mtype,
cbfae36c 1101 .get_vm_pde = gmc_v9_0_get_vm_pde,
7b885f0e
AD		 1102 .get_vm_pte = gmc_v9_0_get_vm_pte,
1103 .get_vbios_fb_size = gmc_v9_0_get_vbios_fb_size,
e60f8db5
AX		 1104};
1105
132f34e4 1106static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev)
e60f8db5 1107{
f54b30d7 1108 adev->gmc.gmc_funcs = &gmc_v9_0_gmc_funcs;
e60f8db5
AX		 1109}
1110
5b6b35aa
HZ		 1111static void gmc_v9_0_set_umc_funcs(struct amdgpu_device *adev)
1112{
1113 switch (adev->asic_type) {
e7da754b
ML		 1114 case CHIP_VEGA10:
1115 adev->umc.funcs = &umc_v6_0_funcs;
1116 break;
5b6b35aa 1117 case CHIP_VEGA20:
3aacf4ea
TZ		 1118 adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM;
1119 adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM;
1120 adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM;
4cf781c2
JC		 1121 adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_VG20;
1122 adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0];
1123 adev->umc.funcs = &umc_v6_1_funcs;
1124 break;
9e612c11 1125 case CHIP_ARCTURUS:
3aacf4ea
TZ		 1126 adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM;
1127 adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM;
1128 adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM;
4cf781c2 1129 adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_ARCT;
3aacf4ea 1130 adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0];
045c0216 1131 adev->umc.funcs = &umc_v6_1_funcs;
5b6b35aa
HZ		 1132 break;
1133 default:
1134 break;
1135 }
1136}
1137
3d093da0
TZ		 1138static void gmc_v9_0_set_mmhub_funcs(struct amdgpu_device *adev)
1139{
1140 switch (adev->asic_type) {
f6c3623b
DL		 1141 case CHIP_ARCTURUS:
1142 adev->mmhub.funcs = &mmhub_v9_4_funcs;
1143 break;
3d093da0 1144 default:
9fb1506e 1145 adev->mmhub.funcs = &mmhub_v1_0_funcs;
3d093da0
TZ		 1146 break;
1147 }
1148}
1149
8ffff9b4
OZ		 1150static void gmc_v9_0_set_gfxhub_funcs(struct amdgpu_device *adev)
1151{
21470d97 1152 adev->gfxhub.funcs = &gfxhub_v1_0_funcs;
8ffff9b4
OZ		 1153}
1154
e60f8db5
AX		 1155static int gmc_v9_0_early_init(void *handle)
1156{
1157 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1158
132f34e4 1159 gmc_v9_0_set_gmc_funcs(adev);
e60f8db5 1160 gmc_v9_0_set_irq_funcs(adev);
5b6b35aa 1161 gmc_v9_0_set_umc_funcs(adev);
3d093da0 1162 gmc_v9_0_set_mmhub_funcs(adev);
8ffff9b4 1163 gmc_v9_0_set_gfxhub_funcs(adev);
e60f8db5 1164
770d13b1
CK		 1165 adev->gmc.shared_aperture_start = 0x2000000000000000ULL;
1166 adev->gmc.shared_aperture_end =
1167 adev->gmc.shared_aperture_start + (4ULL << 30) - 1;
bfa8eea2 1168 adev->gmc.private_aperture_start = 0x1000000000000000ULL;
770d13b1
CK		 1169 adev->gmc.private_aperture_end =
1170 adev->gmc.private_aperture_start + (4ULL << 30) - 1;
a7ea6548 1171
e60f8db5
AX		 1172 return 0;
1173}
1174
c713a461
EQ		 1175static int gmc_v9_0_late_init(void *handle)
1176{
1177 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
c5b6e585 1178 int r;
c713a461 1179
bdbe90f0 1180 r = amdgpu_gmc_allocate_vm_inv_eng(adev);
c713a461
EQ		 1181 if (r)
1182 return r;
4a20300b
GC		 1183
1184 /*
1185 * Workaround performance drop issue with VBIOS enables partial
1186 * writes, while disables HBM ECC for vega10.
1187 */
88474cca 1188 if (!amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_VEGA10)) {
4a20300b 1189 if (!(adev->ras_features & (1 << AMDGPU_RAS_BLOCK__UMC))) {
88474cca
GC		 1190 if (adev->df.funcs->enable_ecc_force_par_wr_rmw)
1191 adev->df.funcs->enable_ecc_force_par_wr_rmw(adev, false);
4a20300b 1192 }
02bab923
DP		 1193 }
1194
fe5211f1
HZ		 1195 if (adev->mmhub.funcs && adev->mmhub.funcs->reset_ras_error_count)
1196 adev->mmhub.funcs->reset_ras_error_count(adev);
1197
ba083492 1198 r = amdgpu_gmc_ras_late_init(adev);
791c4769 1199 if (r)
1200 return r;
1201
770d13b1 1202 return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0);
e60f8db5
AX		 1203}
1204
1205static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev,
770d13b1 1206 struct amdgpu_gmc *mc)
e60f8db5 1207{
eeb2487d 1208 u64 base = 0;
9d4f837a 1209
9fb1506e
OZ		 1210 if (!amdgpu_sriov_vf(adev))
1211 base = adev->mmhub.funcs->get_fb_location(adev);
9d4f837a 1212
6fdd68b1
AD		 1213 /* add the xgmi offset of the physical node */
1214 base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
83afe835 1215 amdgpu_gmc_vram_location(adev, mc, base);
961c75cf 1216 amdgpu_gmc_gart_location(adev, mc);
81c274c4 1217 amdgpu_gmc_agp_location(adev, mc);
bc099ee9 1218 /* base offset of vram pages */
8ffff9b4 1219 adev->vm_manager.vram_base_offset = adev->gfxhub.funcs->get_mc_fb_offset(adev);
6fdd68b1
AD		 1220
1221 /* XXX: add the xgmi offset of the physical node? */
1222 adev->vm_manager.vram_base_offset +=
1223 adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
e60f8db5
AX		 1224}
1225
1226/**
1227 * gmc_v9_0_mc_init - initialize the memory controller driver params
1228 *
1229 * @adev: amdgpu_device pointer
1230 *
1231 * Look up the amount of vram, vram width, and decide how to place
1232 * vram and gart within the GPU's physical address space.
1233 * Returns 0 for success.
1234 */
1235static int gmc_v9_0_mc_init(struct amdgpu_device *adev)
1236{
d6895ad3 1237 int r;
e60f8db5 1238
e60f8db5 1239 /* size in MB on si */
770d13b1 1240 adev->gmc.mc_vram_size =
bebc0762 1241 adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL;
770d13b1 1242 adev->gmc.real_vram_size = adev->gmc.mc_vram_size;
d6895ad3
CK		 1243
1244 if (!(adev->flags & AMD_IS_APU)) {
1245 r = amdgpu_device_resize_fb_bar(adev);
1246 if (r)
1247 return r;
1248 }
770d13b1
CK		 1249 adev->gmc.aper_base = pci_resource_start(adev->pdev, 0);
1250 adev->gmc.aper_size = pci_resource_len(adev->pdev, 0);
e60f8db5 1251
156a81be
CZ		 1252#ifdef CONFIG_X86_64
1253 if (adev->flags & AMD_IS_APU) {
8ffff9b4 1254 adev->gmc.aper_base = adev->gfxhub.funcs->get_mc_fb_offset(adev);
156a81be
CZ		 1255 adev->gmc.aper_size = adev->gmc.real_vram_size;
1256 }
1257#endif
e60f8db5 1258 /* In case the PCI BAR is larger than the actual amount of vram */
770d13b1
CK		 1259 adev->gmc.visible_vram_size = adev->gmc.aper_size;
1260 if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size)
1261 adev->gmc.visible_vram_size = adev->gmc.real_vram_size;
e60f8db5 1262
c3db7b5a
AD		 1263 /* set the gart size */
1264 if (amdgpu_gart_size == -1) {
1265 switch (adev->asic_type) {
1266 case CHIP_VEGA10: /* all engines support GPUVM */
273a14cd 1267 case CHIP_VEGA12: /* all engines support GPUVM */
d96b428c 1268 case CHIP_VEGA20:
3de2ff5d 1269 case CHIP_ARCTURUS:
c3db7b5a 1270 default:
fe19b862 1271 adev->gmc.gart_size = 512ULL << 20;
c3db7b5a
AD		 1272 break;
1273 case CHIP_RAVEN: /* DCE SG support */
8787ee01 1274 case CHIP_RENOIR:
770d13b1 1275 adev->gmc.gart_size = 1024ULL << 20;
c3db7b5a
AD		 1276 break;
1277 }
1278 } else {
770d13b1 1279 adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
c3db7b5a
AD		 1280 }
1281
770d13b1 1282 gmc_v9_0_vram_gtt_location(adev, &adev->gmc);
e60f8db5
AX		 1283
1284 return 0;
1285}
1286
1287static int gmc_v9_0_gart_init(struct amdgpu_device *adev)
1288{
1289 int r;
1290
1123b989 1291 if (adev->gart.bo) {
e60f8db5
AX		 1292 WARN(1, "VEGA10 PCIE GART already initialized\n");
1293 return 0;
1294 }
1295 /* Initialize common gart structure */
1296 r = amdgpu_gart_init(adev);
1297 if (r)
1298 return r;
1299 adev->gart.table_size = adev->gart.num_gpu_pages * 8;
7596ab68 1300 adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE_VG10(MTYPE_UC) |
e60f8db5
AX		 1301 AMDGPU_PTE_EXECUTABLE;
1302 return amdgpu_gart_table_vram_alloc(adev);
1303}
1304
b0a2db9b
AD		 1305/**
1306 * gmc_v9_0_save_registers - saves regs
1307 *
1308 * @adev: amdgpu_device pointer
1309 *
1310 * This saves potential register values that should be
1311 * restored upon resume
1312 */
1313static void gmc_v9_0_save_registers(struct amdgpu_device *adev)
ebdef28e 1314{
b0a2db9b
AD		 1315 if (adev->asic_type == CHIP_RAVEN)
1316 adev->gmc.sdpif_register = RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0);
ebdef28e
AD		 1317}
1318
e60f8db5
AX		 1319static int gmc_v9_0_sw_init(void *handle)
1320{
ad02e08e 1321 int r, vram_width = 0, vram_type = 0, vram_vendor = 0;
e60f8db5
AX		 1322 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1323
8ffff9b4 1324 adev->gfxhub.funcs->init(adev);
9fb1506e
OZ		 1325
1326 adev->mmhub.funcs->init(adev);
0c8c0847 1327
770d13b1 1328 spin_lock_init(&adev->gmc.invalidate_lock);
e60f8db5 1329
ad02e08e
OM		 1330 r = amdgpu_atomfirmware_get_vram_info(adev,
1331 &vram_width, &vram_type, &vram_vendor);
631cdbd2
AD		 1332 if (amdgpu_sriov_vf(adev))
1333 /* For Vega10 SR-IOV, vram_width can't be read from ATOM as RAVEN,
1334 * and DF related registers is not readable, seems hardcord is the
1335 * only way to set the correct vram_width
1336 */
1337 adev->gmc.vram_width = 2048;
1338 else if (amdgpu_emu_mode != 1)
1339 adev->gmc.vram_width = vram_width;
1340
1341 if (!adev->gmc.vram_width) {
1342 int chansize, numchan;
1343
1344 /* hbm memory channel size */
1345 if (adev->flags & AMD_IS_APU)
1346 chansize = 64;
1347 else
1348 chansize = 128;
1349
bdf84a80 1350 numchan = adev->df.funcs->get_hbm_channel_number(adev);
631cdbd2
AD		 1351 adev->gmc.vram_width = numchan * chansize;
1352 }
1353
1354 adev->gmc.vram_type = vram_type;
ad02e08e 1355 adev->gmc.vram_vendor = vram_vendor;
fd66560b
HZ		 1356 switch (adev->asic_type) {
1357 case CHIP_RAVEN:
1daa2bfa
LM		 1358 adev->num_vmhubs = 2;
1359
6a42fd6f 1360 if (adev->rev_id == 0x0 || adev->rev_id == 0x1) {
f3368128 1361 amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
6a42fd6f
CK		 1362 } else {
1363 /* vm_size is 128TB + 512GB for legacy 3-level page support */
1364 amdgpu_vm_adjust_size(adev, 128 * 1024 + 512, 9, 2, 48);
770d13b1 1365 adev->gmc.translate_further =
6a42fd6f
CK		 1366 adev->vm_manager.num_level > 1;
1367 }
fd66560b
HZ		 1368 break;
1369 case CHIP_VEGA10:
273a14cd 1370 case CHIP_VEGA12:
d96b428c 1371 case CHIP_VEGA20:
8787ee01 1372 case CHIP_RENOIR:
1daa2bfa
LM		 1373 adev->num_vmhubs = 2;
1374
8787ee01 1375
36b32a68
ZJ		 1376 /*
1377 * To fulfill 4-level page support,
1378 * vm size is 256TB (48bit), maximum size of Vega10,
1379 * block size 512 (9bit)
1380 */
cdba61da 1381 /* sriov restrict max_pfn below AMDGPU_GMC_HOLE */
1382 if (amdgpu_sriov_vf(adev))
1383 amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 47);
1384 else
1385 amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
fd66560b 1386 break;
3de2ff5d 1387 case CHIP_ARCTURUS:
c8a6e2a3
LM		 1388 adev->num_vmhubs = 3;
1389
3de2ff5d
LM		 1390 /* Keep the vm size same with Vega20 */
1391 amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
1392 break;
fd66560b
HZ		 1393 default:
1394 break;
e60f8db5
AX		 1395 }
1396
1397 /* This interrupt is VMC page fault.*/
44a99b65 1398 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC, VMC_1_0__SRCID__VM_FAULT,
770d13b1 1399 &adev->gmc.vm_fault);
30da7bb1
CK		 1400 if (r)
1401 return r;
1402
7d19b15f
LM		 1403 if (adev->asic_type == CHIP_ARCTURUS) {
1404 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC1, VMC_1_0__SRCID__VM_FAULT,
1405 &adev->gmc.vm_fault);
1406 if (r)
1407 return r;
1408 }
1409
44a99b65 1410 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2, UTCL2_1_0__SRCID__FAULT,
770d13b1 1411 &adev->gmc.vm_fault);
e60f8db5
AX		 1412
1413 if (r)
1414 return r;
1415
2ee9403e
ZL		 1416 if (!amdgpu_sriov_vf(adev)) {
1417 /* interrupt sent to DF. */
1418 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DF, 0,
1419 &adev->gmc.ecc_irq);
1420 if (r)
1421 return r;
1422 }
791c4769 1423
e60f8db5
AX		 1424 /* Set the internal MC address mask
1425 * This is the max address of the GPU's
1426 * internal address space.
1427 */
770d13b1 1428 adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */
e60f8db5 1429
244511f3 1430 r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(44));
e60f8db5 1431 if (r) {
e60f8db5 1432 printk(KERN_WARNING "amdgpu: No suitable DMA available.\n");
244511f3 1433 return r;
e60f8db5 1434 }
244511f3 1435 adev->need_swiotlb = drm_need_swiotlb(44);
e60f8db5 1436
47622ba0 1437 if (adev->gmc.xgmi.supported) {
8ffff9b4 1438 r = adev->gfxhub.funcs->get_xgmi_info(adev);
bf0a60b7
AD		 1439 if (r)
1440 return r;
1441 }
1442
e60f8db5
AX		 1443 r = gmc_v9_0_mc_init(adev);
1444 if (r)
1445 return r;
1446
7b885f0e 1447 amdgpu_gmc_get_vbios_allocations(adev);
ebdef28e 1448
e60f8db5
AX		 1449 /* Memory manager */
1450 r = amdgpu_bo_init(adev);
1451 if (r)
1452 return r;
1453
1454 r = gmc_v9_0_gart_init(adev);
1455 if (r)
1456 return r;
1457
05ec3eda
CK		 1458 /*
1459 * number of VMs
1460 * VMID 0 is reserved for System
81659b20
FK		 1461 * amdgpu graphics/compute will use VMIDs 1..n-1
1462 * amdkfd will use VMIDs n..15
1463 *
1464 * The first KFD VMID is 8 for GPUs with graphics, 3 for
1465 * compute-only GPUs. On compute-only GPUs that leaves 2 VMIDs
1466 * for video processing.
05ec3eda 1467 */
81659b20
FK		 1468 adev->vm_manager.first_kfd_vmid =
1469 adev->asic_type == CHIP_ARCTURUS ? 3 : 8;
05ec3eda 1470
05ec3eda
CK		 1471 amdgpu_vm_manager_init(adev);
1472
b0a2db9b
AD		 1473 gmc_v9_0_save_registers(adev);
1474
05ec3eda 1475 return 0;
e60f8db5
AX		 1476}
1477
e60f8db5
AX		 1478static int gmc_v9_0_sw_fini(void *handle)
1479{
1480 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1481
2adf1344 1482 amdgpu_gmc_ras_fini(adev);
f59548c8 1483 amdgpu_gem_force_release(adev);
05ec3eda 1484 amdgpu_vm_manager_fini(adev);
a3d9103e 1485 amdgpu_gart_table_vram_free(adev);
e60f8db5 1486 amdgpu_bo_fini(adev);
a3d9103e 1487 amdgpu_gart_fini(adev);
e60f8db5
AX		 1488
1489 return 0;
1490}
1491
1492static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev)
1493{
946a4d5b 1494
e60f8db5
AX		 1495 switch (adev->asic_type) {
1496 case CHIP_VEGA10:
4cd4c5c0 1497 if (amdgpu_sriov_vf(adev))
98cad2de 1498 break;
df561f66 1499 fallthrough;
d96b428c 1500 case CHIP_VEGA20:
946a4d5b 1501 soc15_program_register_sequence(adev,
5c583018 1502 golden_settings_mmhub_1_0_0,
c47b41a7 1503 ARRAY_SIZE(golden_settings_mmhub_1_0_0));
946a4d5b 1504 soc15_program_register_sequence(adev,
5c583018 1505 golden_settings_athub_1_0_0,
c47b41a7 1506 ARRAY_SIZE(golden_settings_athub_1_0_0));
e60f8db5 1507 break;
273a14cd
AD		 1508 case CHIP_VEGA12:
1509 break;
e4f3abaa 1510 case CHIP_RAVEN:
8787ee01 1511 /* TODO for renoir */
946a4d5b 1512 soc15_program_register_sequence(adev,
5c583018 1513 golden_settings_athub_1_0_0,
c47b41a7 1514 ARRAY_SIZE(golden_settings_athub_1_0_0));
e4f3abaa 1515 break;
e60f8db5
AX		 1516 default:
1517 break;
1518 }
1519}
1520
c2ecd79b
S		 1521/**
1522 * gmc_v9_0_restore_registers - restores regs
1523 *
1524 * @adev: amdgpu_device pointer
1525 *
1526 * This restores register values, saved at suspend.
1527 */
b0a2db9b 1528void gmc_v9_0_restore_registers(struct amdgpu_device *adev)
c2ecd79b 1529{
0eaa8012 1530 if (adev->asic_type == CHIP_RAVEN) {
f8646661 1531 WREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0, adev->gmc.sdpif_register);
0eaa8012
S		 1532 WARN_ON(adev->gmc.sdpif_register !=
1533 RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0));
1534 }
c2ecd79b
S		 1535}
1536
e60f8db5
AX		 1537/**
1538 * gmc_v9_0_gart_enable - gart enable
1539 *
1540 * @adev: amdgpu_device pointer
1541 */
1542static int gmc_v9_0_gart_enable(struct amdgpu_device *adev)
1543{
cb1545f7 1544 int r;
e60f8db5 1545
1123b989 1546 if (adev->gart.bo == NULL) {
e60f8db5
AX		 1547 dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
1548 return -EINVAL;
1549 }
ce1b1b66
ML		 1550 r = amdgpu_gart_table_vram_pin(adev);
1551 if (r)
1552 return r;
e60f8db5 1553
8ffff9b4 1554 r = adev->gfxhub.funcs->gart_enable(adev);
e60f8db5
AX		 1555 if (r)
1556 return r;
1557
9fb1506e 1558 r = adev->mmhub.funcs->gart_enable(adev);
e60f8db5
AX		 1559 if (r)
1560 return r;
1561
cb1545f7
OZ		 1562 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1563 (unsigned)(adev->gmc.gart_size >> 20),
1564 (unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo));
1565 adev->gart.ready = true;
1566 return 0;
1567}
1568
1569static int gmc_v9_0_hw_init(void *handle)
1570{
1571 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1572 bool value;
1573 int r, i;
1574 u32 tmp;
1575
1576 /* The sequence of these two function calls matters.*/
1577 gmc_v9_0_init_golden_registers(adev);
1578
1579 if (adev->mode_info.num_crtc) {
d0f2f634
HZ		 1580 /* Lockout access through VGA aperture*/
1581 WREG32_FIELD15(DCE, 0, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1);
1582 /* disable VGA render */
1583 WREG32_FIELD15(DCE, 0, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0);
cb1545f7
OZ		 1584 }
1585
1586 amdgpu_device_program_register_sequence(adev,
1587 golden_settings_vega10_hdp,
1588 ARRAY_SIZE(golden_settings_vega10_hdp));
1589
9fb1506e
OZ		 1590 if (adev->mmhub.funcs->update_power_gating)
1591 adev->mmhub.funcs->update_power_gating(adev, true);
1592
cb1545f7 1593 switch (adev->asic_type) {
f81b86a0
OZ		 1594 case CHIP_ARCTURUS:
1595 WREG32_FIELD15(HDP, 0, HDP_MMHUB_CNTL, HDP_MMHUB_GCC, 1);
1596 break;
cb1545f7
OZ		 1597 default:
1598 break;
1599 }
1600
846347c9 1601 WREG32_FIELD15(HDP, 0, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 1);
e60f8db5 1602
b9509c80
HR		 1603 tmp = RREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL);
1604 WREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL, tmp);
e60f8db5 1605
fe2b5323
TZ		 1606 WREG32_SOC15(HDP, 0, mmHDP_NONSURFACE_BASE, (adev->gmc.vram_start >> 8));
1607 WREG32_SOC15(HDP, 0, mmHDP_NONSURFACE_BASE_HI, (adev->gmc.vram_start >> 40));
1608
1d4e0a8c 1609 /* After HDP is initialized, flush HDP.*/
bebc0762 1610 adev->nbio.funcs->hdp_flush(adev, NULL);
1d4e0a8c 1611
e60f8db5
AX		 1612 if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
1613 value = false;
1614 else
1615 value = true;
1616
20bf2f6f 1617 if (!amdgpu_sriov_vf(adev)) {
8ffff9b4 1618 adev->gfxhub.funcs->set_fault_enable_default(adev, value);
9fb1506e 1619 adev->mmhub.funcs->set_fault_enable_default(adev, value);
20bf2f6f 1620 }
3ff98548
OZ		 1621 for (i = 0; i < adev->num_vmhubs; ++i)
1622 gmc_v9_0_flush_gpu_tlb(adev, 0, i, 0);
e60f8db5 1623
e7da754b
ML		 1624 if (adev->umc.funcs && adev->umc.funcs->init_registers)
1625 adev->umc.funcs->init_registers(adev);
1626
e60f8db5
AX		 1627 r = gmc_v9_0_gart_enable(adev);
1628
1629 return r;
1630}
1631
1632/**
1633 * gmc_v9_0_gart_disable - gart disable
1634 *
1635 * @adev: amdgpu_device pointer
1636 *
1637 * This disables all VM page table.
1638 */
1639static void gmc_v9_0_gart_disable(struct amdgpu_device *adev)
1640{
8ffff9b4 1641 adev->gfxhub.funcs->gart_disable(adev);
9fb1506e 1642 adev->mmhub.funcs->gart_disable(adev);
ce1b1b66 1643 amdgpu_gart_table_vram_unpin(adev);
e60f8db5
AX		 1644}
1645
1646static int gmc_v9_0_hw_fini(void *handle)
1647{
1648 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1649
5dd696ae
TH		 1650 if (amdgpu_sriov_vf(adev)) {
1651 /* full access mode, so don't touch any GMC register */
1652 DRM_DEBUG("For SRIOV client, shouldn't do anything.\n");
1653 return 0;
1654 }
1655
791c4769 1656 amdgpu_irq_put(adev, &adev->gmc.ecc_irq, 0);
770d13b1 1657 amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0);
e60f8db5
AX		 1658 gmc_v9_0_gart_disable(adev);
1659
1660 return 0;
1661}
1662
1663static int gmc_v9_0_suspend(void *handle)
1664{
1665 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1666
c24a3c05 1667 return gmc_v9_0_hw_fini(adev);
e60f8db5
AX		 1668}
1669
1670static int gmc_v9_0_resume(void *handle)
1671{
1672 int r;
1673 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1674
1675 r = gmc_v9_0_hw_init(adev);
1676 if (r)
1677 return r;
1678
620f774f 1679 amdgpu_vmid_reset_all(adev);
e60f8db5 1680
32601d48 1681 return 0;
e60f8db5
AX		 1682}
1683
1684static bool gmc_v9_0_is_idle(void *handle)
1685{
1686 /* MC is always ready in GMC v9.*/
1687 return true;
1688}
1689
1690static int gmc_v9_0_wait_for_idle(void *handle)
1691{
1692 /* There is no need to wait for MC idle in GMC v9.*/
1693 return 0;
1694}
1695
1696static int gmc_v9_0_soft_reset(void *handle)
1697{
1698 /* XXX for emulation.*/
1699 return 0;
1700}
1701
1702static int gmc_v9_0_set_clockgating_state(void *handle,
1703 enum amd_clockgating_state state)
1704{
d5583d4f
HR		 1705 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1706
9fb1506e 1707 adev->mmhub.funcs->set_clockgating(adev, state);
bee7b51a
LM		 1708
1709 athub_v1_0_set_clockgating(adev, state);
1710
1711 return 0;
e60f8db5
AX		 1712}
1713
13052be5
HR		 1714static void gmc_v9_0_get_clockgating_state(void *handle, u32 *flags)
1715{
1716 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1717
9fb1506e 1718 adev->mmhub.funcs->get_clockgating(adev, flags);
bee7b51a
LM		 1719
1720 athub_v1_0_get_clockgating(adev, flags);
13052be5
HR		 1721}
1722
e60f8db5
AX		 1723static int gmc_v9_0_set_powergating_state(void *handle,
1724 enum amd_powergating_state state)
1725{
1726 return 0;
1727}
1728
1729const struct amd_ip_funcs gmc_v9_0_ip_funcs = {
1730 .name = "gmc_v9_0",
1731 .early_init = gmc_v9_0_early_init,
1732 .late_init = gmc_v9_0_late_init,
1733 .sw_init = gmc_v9_0_sw_init,
1734 .sw_fini = gmc_v9_0_sw_fini,
1735 .hw_init = gmc_v9_0_hw_init,
1736 .hw_fini = gmc_v9_0_hw_fini,
1737 .suspend = gmc_v9_0_suspend,
1738 .resume = gmc_v9_0_resume,
1739 .is_idle = gmc_v9_0_is_idle,
1740 .wait_for_idle = gmc_v9_0_wait_for_idle,
1741 .soft_reset = gmc_v9_0_soft_reset,
1742 .set_clockgating_state = gmc_v9_0_set_clockgating_state,
1743 .set_powergating_state = gmc_v9_0_set_powergating_state,
13052be5 1744 .get_clockgating_state = gmc_v9_0_get_clockgating_state,
e60f8db5
AX		 1745};
1746
1747const struct amdgpu_ip_block_version gmc_v9_0_ip_block =
1748{
1749 .type = AMD_IP_BLOCK_TYPE_GMC,
1750 .major = 9,
1751 .minor = 0,
1752 .rev = 0,
1753 .funcs = &gmc_v9_0_ip_funcs,
1754};
Ubuntu Hirsute Linux kernel mirror