]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
projects / mirror_ubuntu-bionic-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge tag 'drm-misc-next-2017-09-20' of git://anongit.freedesktop.org/git/drm-misc...
[mirror_ubuntu-bionic-kernel.git] / drivers / gpu / drm / amd / amdgpu / gmc_v8_0.c
1 /*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23 #include <linux/firmware.h>
24 #include <drm/drmP.h>
25 #include "amdgpu.h"
26 #include "gmc_v8_0.h"
27 #include "amdgpu_ucode.h"
28
29 #include "gmc/gmc_8_1_d.h"
30 #include "gmc/gmc_8_1_sh_mask.h"
31
32 #include "bif/bif_5_0_d.h"
33 #include "bif/bif_5_0_sh_mask.h"
34
35 #include "oss/oss_3_0_d.h"
36 #include "oss/oss_3_0_sh_mask.h"
37
38 #include "dce/dce_10_0_d.h"
39 #include "dce/dce_10_0_sh_mask.h"
40
41 #include "vid.h"
42 #include "vi.h"
43
44 #include "amdgpu_atombios.h"
45
46
47 static void gmc_v8_0_set_gart_funcs(struct amdgpu_device *adev);
48 static void gmc_v8_0_set_irq_funcs(struct amdgpu_device *adev);
49 static int gmc_v8_0_wait_for_idle(void *handle);
50
51 MODULE_FIRMWARE("amdgpu/tonga_mc.bin");
52 MODULE_FIRMWARE("amdgpu/polaris11_mc.bin");
53 MODULE_FIRMWARE("amdgpu/polaris10_mc.bin");
54 MODULE_FIRMWARE("amdgpu/polaris12_mc.bin");
55
56 static const u32 golden_settings_tonga_a11[] =
57 {
58 mmMC_ARB_WTM_GRPWT_RD, 0x00000003, 0x00000000,
59 mmMC_HUB_RDREQ_DMIF_LIMIT, 0x0000007f, 0x00000028,
60 mmMC_HUB_WDP_UMC, 0x00007fb6, 0x00000991,
61 mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
62 mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
63 mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
64 mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
65 };
66
67 static const u32 tonga_mgcg_cgcg_init[] =
68 {
69 mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104
70 };
71
72 static const u32 golden_settings_fiji_a10[] =
73 {
74 mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
75 mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
76 mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
77 mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
78 };
79
80 static const u32 fiji_mgcg_cgcg_init[] =
81 {
82 mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104
83 };
84
85 static const u32 golden_settings_polaris11_a11[] =
86 {
87 mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
88 mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
89 mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
90 mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff
91 };
92
93 static const u32 golden_settings_polaris10_a11[] =
94 {
95 mmMC_ARB_WTM_GRPWT_RD, 0x00000003, 0x00000000,
96 mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
97 mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
98 mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
99 mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff
100 };
101
102 static const u32 cz_mgcg_cgcg_init[] =
103 {
104 mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104
105 };
106
107 static const u32 stoney_mgcg_cgcg_init[] =
108 {
109 mmATC_MISC_CG, 0xffffffff, 0x000c0200,
110 mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104
111 };
112
113 static const u32 golden_settings_stoney_common[] =
114 {
115 mmMC_HUB_RDREQ_UVD, MC_HUB_RDREQ_UVD__PRESCALE_MASK, 0x00000004,
116 mmMC_RD_GRP_OTH, MC_RD_GRP_OTH__UVD_MASK, 0x00600000
117 };
118
119 static void gmc_v8_0_init_golden_registers(struct amdgpu_device *adev)
120 {
121 switch (adev->asic_type) {
122 case CHIP_FIJI:
123 amdgpu_program_register_sequence(adev,
124 fiji_mgcg_cgcg_init,
125 (const u32)ARRAY_SIZE(fiji_mgcg_cgcg_init));
126 amdgpu_program_register_sequence(adev,
127 golden_settings_fiji_a10,
128 (const u32)ARRAY_SIZE(golden_settings_fiji_a10));
129 break;
130 case CHIP_TONGA:
131 amdgpu_program_register_sequence(adev,
132 tonga_mgcg_cgcg_init,
133 (const u32)ARRAY_SIZE(tonga_mgcg_cgcg_init));
134 amdgpu_program_register_sequence(adev,
135 golden_settings_tonga_a11,
136 (const u32)ARRAY_SIZE(golden_settings_tonga_a11));
137 break;
138 case CHIP_POLARIS11:
139 case CHIP_POLARIS12:
140 amdgpu_program_register_sequence(adev,
141 golden_settings_polaris11_a11,
142 (const u32)ARRAY_SIZE(golden_settings_polaris11_a11));
143 break;
144 case CHIP_POLARIS10:
145 amdgpu_program_register_sequence(adev,
146 golden_settings_polaris10_a11,
147 (const u32)ARRAY_SIZE(golden_settings_polaris10_a11));
148 break;
149 case CHIP_CARRIZO:
150 amdgpu_program_register_sequence(adev,
151 cz_mgcg_cgcg_init,
152 (const u32)ARRAY_SIZE(cz_mgcg_cgcg_init));
153 break;
154 case CHIP_STONEY:
155 amdgpu_program_register_sequence(adev,
156 stoney_mgcg_cgcg_init,
157 (const u32)ARRAY_SIZE(stoney_mgcg_cgcg_init));
158 amdgpu_program_register_sequence(adev,
159 golden_settings_stoney_common,
160 (const u32)ARRAY_SIZE(golden_settings_stoney_common));
161 break;
162 default:
163 break;
164 }
165 }
166
167 static void gmc_v8_0_mc_stop(struct amdgpu_device *adev)
168 {
169 u32 blackout;
170
171 gmc_v8_0_wait_for_idle(adev);
172
173 blackout = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
174 if (REG_GET_FIELD(blackout, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE) != 1) {
175 /* Block CPU access */
176 WREG32(mmBIF_FB_EN, 0);
177 /* blackout the MC */
178 blackout = REG_SET_FIELD(blackout,
179 MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 1);
180 WREG32(mmMC_SHARED_BLACKOUT_CNTL, blackout);
181 }
182 /* wait for the MC to settle */
183 udelay(100);
184 }
185
186 static void gmc_v8_0_mc_resume(struct amdgpu_device *adev)
187 {
188 u32 tmp;
189
190 /* unblackout the MC */
191 tmp = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
192 tmp = REG_SET_FIELD(tmp, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 0);
193 WREG32(mmMC_SHARED_BLACKOUT_CNTL, tmp);
194 /* allow CPU access */
195 tmp = REG_SET_FIELD(0, BIF_FB_EN, FB_READ_EN, 1);
196 tmp = REG_SET_FIELD(tmp, BIF_FB_EN, FB_WRITE_EN, 1);
197 WREG32(mmBIF_FB_EN, tmp);
198 }
199
200 /**
201 * gmc_v8_0_init_microcode - load ucode images from disk
202 *
203 * @adev: amdgpu_device pointer
204 *
205 * Use the firmware interface to load the ucode images into
206 * the driver (not loaded into hw).
207 * Returns 0 on success, error on failure.
208 */
209 static int gmc_v8_0_init_microcode(struct amdgpu_device *adev)
210 {
211 const char *chip_name;
212 char fw_name[30];
213 int err;
214
215 DRM_DEBUG("\n");
216
217 switch (adev->asic_type) {
218 case CHIP_TONGA:
219 chip_name = "tonga";
220 break;
221 case CHIP_POLARIS11:
222 chip_name = "polaris11";
223 break;
224 case CHIP_POLARIS10:
225 chip_name = "polaris10";
226 break;
227 case CHIP_POLARIS12:
228 chip_name = "polaris12";
229 break;
230 case CHIP_FIJI:
231 case CHIP_CARRIZO:
232 case CHIP_STONEY:
233 return 0;
234 default: BUG();
235 }
236
237 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mc.bin", chip_name);
238 err = request_firmware(&adev->mc.fw, fw_name, adev->dev);
239 if (err)
240 goto out;
241 err = amdgpu_ucode_validate(adev->mc.fw);
242
243 out:
244 if (err) {
245 pr_err("mc: Failed to load firmware \"%s\"\n", fw_name);
246 release_firmware(adev->mc.fw);
247 adev->mc.fw = NULL;
248 }
249 return err;
250 }
251
252 /**
253 * gmc_v8_0_tonga_mc_load_microcode - load tonga MC ucode into the hw
254 *
255 * @adev: amdgpu_device pointer
256 *
257 * Load the GDDR MC ucode into the hw (CIK).
258 * Returns 0 on success, error on failure.
259 */
260 static int gmc_v8_0_tonga_mc_load_microcode(struct amdgpu_device *adev)
261 {
262 const struct mc_firmware_header_v1_0 *hdr;
263 const __le32 *fw_data = NULL;
264 const __le32 *io_mc_regs = NULL;
265 u32 running;
266 int i, ucode_size, regs_size;
267
268 /* Skip MC ucode loading on SR-IOV capable boards.
269 * vbios does this for us in asic_init in that case.
270 * Skip MC ucode loading on VF, because hypervisor will do that
271 * for this adaptor.
272 */
273 if (amdgpu_sriov_bios(adev))
274 return 0;
275
276 if (!adev->mc.fw)
277 return -EINVAL;
278
279 hdr = (const struct mc_firmware_header_v1_0 *)adev->mc.fw->data;
280 amdgpu_ucode_print_mc_hdr(&hdr->header);
281
282 adev->mc.fw_version = le32_to_cpu(hdr->header.ucode_version);
283 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
284 io_mc_regs = (const __le32 *)
285 (adev->mc.fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
286 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
287 fw_data = (const __le32 *)
288 (adev->mc.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
289
290 running = REG_GET_FIELD(RREG32(mmMC_SEQ_SUP_CNTL), MC_SEQ_SUP_CNTL, RUN);
291
292 if (running == 0) {
293 /* reset the engine and set to writable */
294 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);
295 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000010);
296
297 /* load mc io regs */
298 for (i = 0; i < regs_size; i++) {
299 WREG32(mmMC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(io_mc_regs++));
300 WREG32(mmMC_SEQ_IO_DEBUG_DATA, le32_to_cpup(io_mc_regs++));
301 }
302 /* load the MC ucode */
303 for (i = 0; i < ucode_size; i++)
304 WREG32(mmMC_SEQ_SUP_PGM, le32_to_cpup(fw_data++));
305
306 /* put the engine back into the active state */
307 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);
308 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000004);
309 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000001);
310
311 /* wait for training to complete */
312 for (i = 0; i < adev->usec_timeout; i++) {
313 if (REG_GET_FIELD(RREG32(mmMC_SEQ_TRAIN_WAKEUP_CNTL),
314 MC_SEQ_TRAIN_WAKEUP_CNTL, TRAIN_DONE_D0))
315 break;
316 udelay(1);
317 }
318 for (i = 0; i < adev->usec_timeout; i++) {
319 if (REG_GET_FIELD(RREG32(mmMC_SEQ_TRAIN_WAKEUP_CNTL),
320 MC_SEQ_TRAIN_WAKEUP_CNTL, TRAIN_DONE_D1))
321 break;
322 udelay(1);
323 }
324 }
325
326 return 0;
327 }
328
329 static int gmc_v8_0_polaris_mc_load_microcode(struct amdgpu_device *adev)
330 {
331 const struct mc_firmware_header_v1_0 *hdr;
332 const __le32 *fw_data = NULL;
333 const __le32 *io_mc_regs = NULL;
334 u32 data, vbios_version;
335 int i, ucode_size, regs_size;
336
337 /* Skip MC ucode loading on SR-IOV capable boards.
338 * vbios does this for us in asic_init in that case.
339 * Skip MC ucode loading on VF, because hypervisor will do that
340 * for this adaptor.
341 */
342 if (amdgpu_sriov_bios(adev))
343 return 0;
344
345 WREG32(mmMC_SEQ_IO_DEBUG_INDEX, 0x9F);
346 data = RREG32(mmMC_SEQ_IO_DEBUG_DATA);
347 vbios_version = data & 0xf;
348
349 if (vbios_version == 0)
350 return 0;
351
352 if (!adev->mc.fw)
353 return -EINVAL;
354
355 hdr = (const struct mc_firmware_header_v1_0 *)adev->mc.fw->data;
356 amdgpu_ucode_print_mc_hdr(&hdr->header);
357
358 adev->mc.fw_version = le32_to_cpu(hdr->header.ucode_version);
359 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
360 io_mc_regs = (const __le32 *)
361 (adev->mc.fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
362 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
363 fw_data = (const __le32 *)
364 (adev->mc.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
365
366 data = RREG32(mmMC_SEQ_MISC0);
367 data &= ~(0x40);
368 WREG32(mmMC_SEQ_MISC0, data);
369
370 /* load mc io regs */
371 for (i = 0; i < regs_size; i++) {
372 WREG32(mmMC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(io_mc_regs++));
373 WREG32(mmMC_SEQ_IO_DEBUG_DATA, le32_to_cpup(io_mc_regs++));
374 }
375
376 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);
377 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000010);
378
379 /* load the MC ucode */
380 for (i = 0; i < ucode_size; i++)
381 WREG32(mmMC_SEQ_SUP_PGM, le32_to_cpup(fw_data++));
382
383 /* put the engine back into the active state */
384 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);
385 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000004);
386 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000001);
387
388 /* wait for training to complete */
389 for (i = 0; i < adev->usec_timeout; i++) {
390 data = RREG32(mmMC_SEQ_MISC0);
391 if (data & 0x80)
392 break;
393 udelay(1);
394 }
395
396 return 0;
397 }
398
399 static void gmc_v8_0_vram_gtt_location(struct amdgpu_device *adev,
400 struct amdgpu_mc *mc)
401 {
402 u64 base = 0;
403
404 if (!amdgpu_sriov_vf(adev))
405 base = RREG32(mmMC_VM_FB_LOCATION) & 0xFFFF;
406 base <<= 24;
407
408 if (mc->mc_vram_size > 0xFFC0000000ULL) {
409 /* leave room for at least 1024M GTT */
410 dev_warn(adev->dev, "limiting VRAM\n");
411 mc->real_vram_size = 0xFFC0000000ULL;
412 mc->mc_vram_size = 0xFFC0000000ULL;
413 }
414 amdgpu_vram_location(adev, &adev->mc, base);
415 amdgpu_gart_location(adev, mc);
416 }
417
418 /**
419 * gmc_v8_0_mc_program - program the GPU memory controller
420 *
421 * @adev: amdgpu_device pointer
422 *
423 * Set the location of vram, gart, and AGP in the GPU's
424 * physical address space (CIK).
425 */
426 static void gmc_v8_0_mc_program(struct amdgpu_device *adev)
427 {
428 u32 tmp;
429 int i, j;
430
431 /* Initialize HDP */
432 for (i = 0, j = 0; i < 32; i++, j += 0x6) {
433 WREG32((0xb05 + j), 0x00000000);
434 WREG32((0xb06 + j), 0x00000000);
435 WREG32((0xb07 + j), 0x00000000);
436 WREG32((0xb08 + j), 0x00000000);
437 WREG32((0xb09 + j), 0x00000000);
438 }
439 WREG32(mmHDP_REG_COHERENCY_FLUSH_CNTL, 0);
440
441 if (gmc_v8_0_wait_for_idle((void *)adev)) {
442 dev_warn(adev->dev, "Wait for MC idle timedout !\n");
443 }
444 if (adev->mode_info.num_crtc) {
445 /* Lockout access through VGA aperture*/
446 tmp = RREG32(mmVGA_HDP_CONTROL);
447 tmp = REG_SET_FIELD(tmp, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1);
448 WREG32(mmVGA_HDP_CONTROL, tmp);
449
450 /* disable VGA render */
451 tmp = RREG32(mmVGA_RENDER_CONTROL);
452 tmp = REG_SET_FIELD(tmp, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0);
453 WREG32(mmVGA_RENDER_CONTROL, tmp);
454 }
455 /* Update configuration */
456 WREG32(mmMC_VM_SYSTEM_APERTURE_LOW_ADDR,
457 adev->mc.vram_start >> 12);
458 WREG32(mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
459 adev->mc.vram_end >> 12);
460 WREG32(mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
461 adev->vram_scratch.gpu_addr >> 12);
462
463 if (amdgpu_sriov_vf(adev)) {
464 tmp = ((adev->mc.vram_end >> 24) & 0xFFFF) << 16;
465 tmp |= ((adev->mc.vram_start >> 24) & 0xFFFF);
466 WREG32(mmMC_VM_FB_LOCATION, tmp);
467 /* XXX double check these! */
468 WREG32(mmHDP_NONSURFACE_BASE, (adev->mc.vram_start >> 8));
469 WREG32(mmHDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
470 WREG32(mmHDP_NONSURFACE_SIZE, 0x3FFFFFFF);
471 }
472
473 WREG32(mmMC_VM_AGP_BASE, 0);
474 WREG32(mmMC_VM_AGP_TOP, 0x0FFFFFFF);
475 WREG32(mmMC_VM_AGP_BOT, 0x0FFFFFFF);
476 if (gmc_v8_0_wait_for_idle((void *)adev)) {
477 dev_warn(adev->dev, "Wait for MC idle timedout !\n");
478 }
479
480 WREG32(mmBIF_FB_EN, BIF_FB_EN__FB_READ_EN_MASK | BIF_FB_EN__FB_WRITE_EN_MASK);
481
482 tmp = RREG32(mmHDP_MISC_CNTL);
483 tmp = REG_SET_FIELD(tmp, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 0);
484 WREG32(mmHDP_MISC_CNTL, tmp);
485
486 tmp = RREG32(mmHDP_HOST_PATH_CNTL);
487 WREG32(mmHDP_HOST_PATH_CNTL, tmp);
488 }
489
490 /**
491 * gmc_v8_0_mc_init - initialize the memory controller driver params
492 *
493 * @adev: amdgpu_device pointer
494 *
495 * Look up the amount of vram, vram width, and decide how to place
496 * vram and gart within the GPU's physical address space (CIK).
497 * Returns 0 for success.
498 */
499 static int gmc_v8_0_mc_init(struct amdgpu_device *adev)
500 {
501 adev->mc.vram_width = amdgpu_atombios_get_vram_width(adev);
502 if (!adev->mc.vram_width) {
503 u32 tmp;
504 int chansize, numchan;
505
506 /* Get VRAM informations */
507 tmp = RREG32(mmMC_ARB_RAMCFG);
508 if (REG_GET_FIELD(tmp, MC_ARB_RAMCFG, CHANSIZE)) {
509 chansize = 64;
510 } else {
511 chansize = 32;
512 }
513 tmp = RREG32(mmMC_SHARED_CHMAP);
514 switch (REG_GET_FIELD(tmp, MC_SHARED_CHMAP, NOOFCHAN)) {
515 case 0:
516 default:
517 numchan = 1;
518 break;
519 case 1:
520 numchan = 2;
521 break;
522 case 2:
523 numchan = 4;
524 break;
525 case 3:
526 numchan = 8;
527 break;
528 case 4:
529 numchan = 3;
530 break;
531 case 5:
532 numchan = 6;
533 break;
534 case 6:
535 numchan = 10;
536 break;
537 case 7:
538 numchan = 12;
539 break;
540 case 8:
541 numchan = 16;
542 break;
543 }
544 adev->mc.vram_width = numchan * chansize;
545 }
546 /* Could aper size report 0 ? */
547 adev->mc.aper_base = pci_resource_start(adev->pdev, 0);
548 adev->mc.aper_size = pci_resource_len(adev->pdev, 0);
549 /* size in MB on si */
550 adev->mc.mc_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
551 adev->mc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
552
553 #ifdef CONFIG_X86_64
554 if (adev->flags & AMD_IS_APU) {
555 adev->mc.aper_base = ((u64)RREG32(mmMC_VM_FB_OFFSET)) << 22;
556 adev->mc.aper_size = adev->mc.real_vram_size;
557 }
558 #endif
559
560 /* In case the PCI BAR is larger than the actual amount of vram */
561 adev->mc.visible_vram_size = adev->mc.aper_size;
562 if (adev->mc.visible_vram_size > adev->mc.real_vram_size)
563 adev->mc.visible_vram_size = adev->mc.real_vram_size;
564
565 /* set the gart size */
566 if (amdgpu_gart_size == -1) {
567 switch (adev->asic_type) {
568 case CHIP_POLARIS11: /* all engines support GPUVM */
569 case CHIP_POLARIS10: /* all engines support GPUVM */
570 case CHIP_POLARIS12: /* all engines support GPUVM */
571 default:
572 adev->mc.gart_size = 256ULL << 20;
573 break;
574 case CHIP_TONGA: /* UVD, VCE do not support GPUVM */
575 case CHIP_FIJI: /* UVD, VCE do not support GPUVM */
576 case CHIP_CARRIZO: /* UVD, VCE do not support GPUVM, DCE SG support */
577 case CHIP_STONEY: /* UVD does not support GPUVM, DCE SG support */
578 adev->mc.gart_size = 1024ULL << 20;
579 break;
580 }
581 } else {
582 adev->mc.gart_size = (u64)amdgpu_gart_size << 20;
583 }
584
585 gmc_v8_0_vram_gtt_location(adev, &adev->mc);
586
587 return 0;
588 }
589
590 /*
591 * GART
592 * VMID 0 is the physical GPU addresses as used by the kernel.
593 * VMIDs 1-15 are used for userspace clients and are handled
594 * by the amdgpu vm/hsa code.
595 */
596
597 /**
598 * gmc_v8_0_gart_flush_gpu_tlb - gart tlb flush callback
599 *
600 * @adev: amdgpu_device pointer
601 * @vmid: vm instance to flush
602 *
603 * Flush the TLB for the requested page table (CIK).
604 */
605 static void gmc_v8_0_gart_flush_gpu_tlb(struct amdgpu_device *adev,
606 uint32_t vmid)
607 {
608 /* flush hdp cache */
609 WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 0);
610
611 /* bits 0-15 are the VM contexts0-15 */
612 WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid);
613 }
614
615 /**
616 * gmc_v8_0_gart_set_pte_pde - update the page tables using MMIO
617 *
618 * @adev: amdgpu_device pointer
619 * @cpu_pt_addr: cpu address of the page table
620 * @gpu_page_idx: entry in the page table to update
621 * @addr: dst addr to write into pte/pde
622 * @flags: access flags
623 *
624 * Update the page tables using the CPU.
625 */
626 static int gmc_v8_0_gart_set_pte_pde(struct amdgpu_device *adev,
627 void *cpu_pt_addr,
628 uint32_t gpu_page_idx,
629 uint64_t addr,
630 uint64_t flags)
631 {
632 void __iomem *ptr = (void *)cpu_pt_addr;
633 uint64_t value;
634
635 /*
636 * PTE format on VI:
637 * 63:40 reserved
638 * 39:12 4k physical page base address
639 * 11:7 fragment
640 * 6 write
641 * 5 read
642 * 4 exe
643 * 3 reserved
644 * 2 snooped
645 * 1 system
646 * 0 valid
647 *
648 * PDE format on VI:
649 * 63:59 block fragment size
650 * 58:40 reserved
651 * 39:1 physical base address of PTE
652 * bits 5:1 must be 0.
653 * 0 valid
654 */
655 value = addr & 0x000000FFFFFFF000ULL;
656 value |= flags;
657 writeq(value, ptr + (gpu_page_idx * 8));
658
659 return 0;
660 }
661
662 static uint64_t gmc_v8_0_get_vm_pte_flags(struct amdgpu_device *adev,
663 uint32_t flags)
664 {
665 uint64_t pte_flag = 0;
666
667 if (flags & AMDGPU_VM_PAGE_EXECUTABLE)
668 pte_flag |= AMDGPU_PTE_EXECUTABLE;
669 if (flags & AMDGPU_VM_PAGE_READABLE)
670 pte_flag |= AMDGPU_PTE_READABLE;
671 if (flags & AMDGPU_VM_PAGE_WRITEABLE)
672 pte_flag |= AMDGPU_PTE_WRITEABLE;
673 if (flags & AMDGPU_VM_PAGE_PRT)
674 pte_flag |= AMDGPU_PTE_PRT;
675
676 return pte_flag;
677 }
678
679 static uint64_t gmc_v8_0_get_vm_pde(struct amdgpu_device *adev, uint64_t addr)
680 {
681 BUG_ON(addr & 0xFFFFFF0000000FFFULL);
682 return addr;
683 }
684
685 /**
686 * gmc_v8_0_set_fault_enable_default - update VM fault handling
687 *
688 * @adev: amdgpu_device pointer
689 * @value: true redirects VM faults to the default page
690 */
691 static void gmc_v8_0_set_fault_enable_default(struct amdgpu_device *adev,
692 bool value)
693 {
694 u32 tmp;
695
696 tmp = RREG32(mmVM_CONTEXT1_CNTL);
697 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
698 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
699 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
700 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
701 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
702 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
703 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
704 VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
705 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
706 READ_PROTECTION_FAULT_ENABLE_DEFAULT, value);
707 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
708 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
709 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
710 EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
711 WREG32(mmVM_CONTEXT1_CNTL, tmp);
712 }
713
714 /**
715 * gmc_v8_0_set_prt - set PRT VM fault
716 *
717 * @adev: amdgpu_device pointer
718 * @enable: enable/disable VM fault handling for PRT
719 */
720 static void gmc_v8_0_set_prt(struct amdgpu_device *adev, bool enable)
721 {
722 u32 tmp;
723
724 if (enable && !adev->mc.prt_warning) {
725 dev_warn(adev->dev, "Disabling VM faults because of PRT request!\n");
726 adev->mc.prt_warning = true;
727 }
728
729 tmp = RREG32(mmVM_PRT_CNTL);
730 tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
731 CB_DISABLE_READ_FAULT_ON_UNMAPPED_ACCESS, enable);
732 tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
733 CB_DISABLE_WRITE_FAULT_ON_UNMAPPED_ACCESS, enable);
734 tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
735 TC_DISABLE_READ_FAULT_ON_UNMAPPED_ACCESS, enable);
736 tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
737 TC_DISABLE_WRITE_FAULT_ON_UNMAPPED_ACCESS, enable);
738 tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
739 L2_CACHE_STORE_INVALID_ENTRIES, enable);
740 tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
741 L1_TLB_STORE_INVALID_ENTRIES, enable);
742 tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
743 MASK_PDE0_FAULT, enable);
744 WREG32(mmVM_PRT_CNTL, tmp);
745
746 if (enable) {
747 uint32_t low = AMDGPU_VA_RESERVED_SIZE >> AMDGPU_GPU_PAGE_SHIFT;
748 uint32_t high = adev->vm_manager.max_pfn;
749
750 WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, low);
751 WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, low);
752 WREG32(mmVM_PRT_APERTURE2_LOW_ADDR, low);
753 WREG32(mmVM_PRT_APERTURE3_LOW_ADDR, low);
754 WREG32(mmVM_PRT_APERTURE0_HIGH_ADDR, high);
755 WREG32(mmVM_PRT_APERTURE1_HIGH_ADDR, high);
756 WREG32(mmVM_PRT_APERTURE2_HIGH_ADDR, high);
757 WREG32(mmVM_PRT_APERTURE3_HIGH_ADDR, high);
758 } else {
759 WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, 0xfffffff);
760 WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, 0xfffffff);
761 WREG32(mmVM_PRT_APERTURE2_LOW_ADDR, 0xfffffff);
762 WREG32(mmVM_PRT_APERTURE3_LOW_ADDR, 0xfffffff);
763 WREG32(mmVM_PRT_APERTURE0_HIGH_ADDR, 0x0);
764 WREG32(mmVM_PRT_APERTURE1_HIGH_ADDR, 0x0);
765 WREG32(mmVM_PRT_APERTURE2_HIGH_ADDR, 0x0);
766 WREG32(mmVM_PRT_APERTURE3_HIGH_ADDR, 0x0);
767 }
768 }
769
770 /**
771 * gmc_v8_0_gart_enable - gart enable
772 *
773 * @adev: amdgpu_device pointer
774 *
775 * This sets up the TLBs, programs the page tables for VMID0,
776 * sets up the hw for VMIDs 1-15 which are allocated on
777 * demand, and sets up the global locations for the LDS, GDS,
778 * and GPUVM for FSA64 clients (CIK).
779 * Returns 0 for success, errors for failure.
780 */
781 static int gmc_v8_0_gart_enable(struct amdgpu_device *adev)
782 {
783 int r, i;
784 u32 tmp, field;
785
786 if (adev->gart.robj == NULL) {
787 dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
788 return -EINVAL;
789 }
790 r = amdgpu_gart_table_vram_pin(adev);
791 if (r)
792 return r;
793 /* Setup TLB control */
794 tmp = RREG32(mmMC_VM_MX_L1_TLB_CNTL);
795 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1);
796 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING, 1);
797 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, 3);
798 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_ADVANCED_DRIVER_MODEL, 1);
799 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
800 WREG32(mmMC_VM_MX_L1_TLB_CNTL, tmp);
801 /* Setup L2 cache */
802 tmp = RREG32(mmVM_L2_CNTL);
803 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 1);
804 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 1);
805 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE, 1);
806 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE, 1);
807 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, EFFECTIVE_L2_QUEUE_SIZE, 7);
808 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1);
809 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY, 1);
810 WREG32(mmVM_L2_CNTL, tmp);
811 tmp = RREG32(mmVM_L2_CNTL2);
812 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1);
813 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_L2_CACHE, 1);
814 WREG32(mmVM_L2_CNTL2, tmp);
815
816 field = adev->vm_manager.fragment_size;
817 tmp = RREG32(mmVM_L2_CNTL3);
818 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY, 1);
819 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, field);
820 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_FRAGMENT_SIZE, field);
821 WREG32(mmVM_L2_CNTL3, tmp);
822 /* XXX: set to enable PTE/PDE in system memory */
823 tmp = RREG32(mmVM_L2_CNTL4);
824 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PDE_REQUEST_PHYSICAL, 0);
825 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PDE_REQUEST_SHARED, 0);
826 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PDE_REQUEST_SNOOP, 0);
827 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PTE_REQUEST_PHYSICAL, 0);
828 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PTE_REQUEST_SHARED, 0);
829 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PTE_REQUEST_SNOOP, 0);
830 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PDE_REQUEST_PHYSICAL, 0);
831 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PDE_REQUEST_SHARED, 0);
832 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PDE_REQUEST_SNOOP, 0);
833 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PTE_REQUEST_PHYSICAL, 0);
834 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PTE_REQUEST_SHARED, 0);
835 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PTE_REQUEST_SNOOP, 0);
836 WREG32(mmVM_L2_CNTL4, tmp);
837 /* setup context0 */
838 WREG32(mmVM_CONTEXT0_PAGE_TABLE_START_ADDR, adev->mc.gart_start >> 12);
839 WREG32(mmVM_CONTEXT0_PAGE_TABLE_END_ADDR, adev->mc.gart_end >> 12);
840 WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR, adev->gart.table_addr >> 12);
841 WREG32(mmVM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
842 (u32)(adev->dummy_page.addr >> 12));
843 WREG32(mmVM_CONTEXT0_CNTL2, 0);
844 tmp = RREG32(mmVM_CONTEXT0_CNTL);
845 tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1);
846 tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 0);
847 tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
848 WREG32(mmVM_CONTEXT0_CNTL, tmp);
849
850 WREG32(mmVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR, 0);
851 WREG32(mmVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR, 0);
852 WREG32(mmVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET, 0);
853
854 /* empty context1-15 */
855 /* FIXME start with 4G, once using 2 level pt switch to full
856 * vm size space
857 */
858 /* set vm size, must be a multiple of 4 */
859 WREG32(mmVM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
860 WREG32(mmVM_CONTEXT1_PAGE_TABLE_END_ADDR, adev->vm_manager.max_pfn - 1);
861 for (i = 1; i < 16; i++) {
862 if (i < 8)
863 WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + i,
864 adev->gart.table_addr >> 12);
865 else
866 WREG32(mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + i - 8,
867 adev->gart.table_addr >> 12);
868 }
869
870 /* enable context1-15 */
871 WREG32(mmVM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
872 (u32)(adev->dummy_page.addr >> 12));
873 WREG32(mmVM_CONTEXT1_CNTL2, 4);
874 tmp = RREG32(mmVM_CONTEXT1_CNTL);
875 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1);
876 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH, 1);
877 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
878 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
879 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
880 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
881 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
882 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
883 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
884 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_BLOCK_SIZE,
885 adev->vm_manager.block_size - 9);
886 WREG32(mmVM_CONTEXT1_CNTL, tmp);
887 if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
888 gmc_v8_0_set_fault_enable_default(adev, false);
889 else
890 gmc_v8_0_set_fault_enable_default(adev, true);
891
892 gmc_v8_0_gart_flush_gpu_tlb(adev, 0);
893 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
894 (unsigned)(adev->mc.gart_size >> 20),
895 (unsigned long long)adev->gart.table_addr);
896 adev->gart.ready = true;
897 return 0;
898 }
899
900 static int gmc_v8_0_gart_init(struct amdgpu_device *adev)
901 {
902 int r;
903
904 if (adev->gart.robj) {
905 WARN(1, "R600 PCIE GART already initialized\n");
906 return 0;
907 }
908 /* Initialize common gart structure */
909 r = amdgpu_gart_init(adev);
910 if (r)
911 return r;
912 adev->gart.table_size = adev->gart.num_gpu_pages * 8;
913 adev->gart.gart_pte_flags = AMDGPU_PTE_EXECUTABLE;
914 return amdgpu_gart_table_vram_alloc(adev);
915 }
916
917 /**
918 * gmc_v8_0_gart_disable - gart disable
919 *
920 * @adev: amdgpu_device pointer
921 *
922 * This disables all VM page table (CIK).
923 */
924 static void gmc_v8_0_gart_disable(struct amdgpu_device *adev)
925 {
926 u32 tmp;
927
928 /* Disable all tables */
929 WREG32(mmVM_CONTEXT0_CNTL, 0);
930 WREG32(mmVM_CONTEXT1_CNTL, 0);
931 /* Setup TLB control */
932 tmp = RREG32(mmMC_VM_MX_L1_TLB_CNTL);
933 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0);
934 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING, 0);
935 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_ADVANCED_DRIVER_MODEL, 0);
936 WREG32(mmMC_VM_MX_L1_TLB_CNTL, tmp);
937 /* Setup L2 cache */
938 tmp = RREG32(mmVM_L2_CNTL);
939 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 0);
940 WREG32(mmVM_L2_CNTL, tmp);
941 WREG32(mmVM_L2_CNTL2, 0);
942 amdgpu_gart_table_vram_unpin(adev);
943 }
944
945 /**
946 * gmc_v8_0_gart_fini - vm fini callback
947 *
948 * @adev: amdgpu_device pointer
949 *
950 * Tears down the driver GART/VM setup (CIK).
951 */
952 static void gmc_v8_0_gart_fini(struct amdgpu_device *adev)
953 {
954 amdgpu_gart_table_vram_free(adev);
955 amdgpu_gart_fini(adev);
956 }
957
958 /**
959 * gmc_v8_0_vm_decode_fault - print human readable fault info
960 *
961 * @adev: amdgpu_device pointer
962 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
963 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
964 *
965 * Print human readable fault information (CIK).
966 */
967 static void gmc_v8_0_vm_decode_fault(struct amdgpu_device *adev,
968 u32 status, u32 addr, u32 mc_client)
969 {
970 u32 mc_id;
971 u32 vmid = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, VMID);
972 u32 protections = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
973 PROTECTIONS);
974 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
975 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
976
977 mc_id = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
978 MEMORY_CLIENT_ID);
979
980 dev_err(adev->dev, "VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
981 protections, vmid, addr,
982 REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
983 MEMORY_CLIENT_RW) ?
984 "write" : "read", block, mc_client, mc_id);
985 }
986
987 static int gmc_v8_0_convert_vram_type(int mc_seq_vram_type)
988 {
989 switch (mc_seq_vram_type) {
990 case MC_SEQ_MISC0__MT__GDDR1:
991 return AMDGPU_VRAM_TYPE_GDDR1;
992 case MC_SEQ_MISC0__MT__DDR2:
993 return AMDGPU_VRAM_TYPE_DDR2;
994 case MC_SEQ_MISC0__MT__GDDR3:
995 return AMDGPU_VRAM_TYPE_GDDR3;
996 case MC_SEQ_MISC0__MT__GDDR4:
997 return AMDGPU_VRAM_TYPE_GDDR4;
998 case MC_SEQ_MISC0__MT__GDDR5:
999 return AMDGPU_VRAM_TYPE_GDDR5;
1000 case MC_SEQ_MISC0__MT__HBM:
1001 return AMDGPU_VRAM_TYPE_HBM;
1002 case MC_SEQ_MISC0__MT__DDR3:
1003 return AMDGPU_VRAM_TYPE_DDR3;
1004 default:
1005 return AMDGPU_VRAM_TYPE_UNKNOWN;
1006 }
1007 }
1008
1009 static int gmc_v8_0_early_init(void *handle)
1010 {
1011 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1012
1013 gmc_v8_0_set_gart_funcs(adev);
1014 gmc_v8_0_set_irq_funcs(adev);
1015
1016 adev->mc.shared_aperture_start = 0x2000000000000000ULL;
1017 adev->mc.shared_aperture_end =
1018 adev->mc.shared_aperture_start + (4ULL << 30) - 1;
1019 adev->mc.private_aperture_start =
1020 adev->mc.shared_aperture_end + 1;
1021 adev->mc.private_aperture_end =
1022 adev->mc.private_aperture_start + (4ULL << 30) - 1;
1023
1024 return 0;
1025 }
1026
1027 static int gmc_v8_0_late_init(void *handle)
1028 {
1029 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1030
1031 if (amdgpu_vm_fault_stop != AMDGPU_VM_FAULT_STOP_ALWAYS)
1032 return amdgpu_irq_get(adev, &adev->mc.vm_fault, 0);
1033 else
1034 return 0;
1035 }
1036
1037 #define mmMC_SEQ_MISC0_FIJI 0xA71
1038
1039 static int gmc_v8_0_sw_init(void *handle)
1040 {
1041 int r;
1042 int dma_bits;
1043 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1044
1045 if (adev->flags & AMD_IS_APU) {
1046 adev->mc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN;
1047 } else {
1048 u32 tmp;
1049
1050 if (adev->asic_type == CHIP_FIJI)
1051 tmp = RREG32(mmMC_SEQ_MISC0_FIJI);
1052 else
1053 tmp = RREG32(mmMC_SEQ_MISC0);
1054 tmp &= MC_SEQ_MISC0__MT__MASK;
1055 adev->mc.vram_type = gmc_v8_0_convert_vram_type(tmp);
1056 }
1057
1058 r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 146, &adev->mc.vm_fault);
1059 if (r)
1060 return r;
1061
1062 r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 147, &adev->mc.vm_fault);
1063 if (r)
1064 return r;
1065
1066 /* Adjust VM size here.
1067 * Currently set to 4GB ((1 << 20) 4k pages).
1068 * Max GPUVM size for cayman and SI is 40 bits.
1069 */
1070 amdgpu_vm_adjust_size(adev, 64, 4);
1071 adev->vm_manager.max_pfn = adev->vm_manager.vm_size << 18;
1072
1073 /* Set the internal MC address mask
1074 * This is the max address of the GPU's
1075 * internal address space.
1076 */
1077 adev->mc.mc_mask = 0xffffffffffULL; /* 40 bit MC */
1078
1079 adev->mc.stolen_size = 256 * 1024;
1080
1081 /* set DMA mask + need_dma32 flags.
1082 * PCIE - can handle 40-bits.
1083 * IGP - can handle 40-bits
1084 * PCI - dma32 for legacy pci gart, 40 bits on newer asics
1085 */
1086 adev->need_dma32 = false;
1087 dma_bits = adev->need_dma32 ? 32 : 40;
1088 r = pci_set_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
1089 if (r) {
1090 adev->need_dma32 = true;
1091 dma_bits = 32;
1092 pr_warn("amdgpu: No suitable DMA available\n");
1093 }
1094 r = pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
1095 if (r) {
1096 pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(32));
1097 pr_warn("amdgpu: No coherent DMA available\n");
1098 }
1099
1100 r = gmc_v8_0_init_microcode(adev);
1101 if (r) {
1102 DRM_ERROR("Failed to load mc firmware!\n");
1103 return r;
1104 }
1105
1106 r = gmc_v8_0_mc_init(adev);
1107 if (r)
1108 return r;
1109
1110 /* Memory manager */
1111 r = amdgpu_bo_init(adev);
1112 if (r)
1113 return r;
1114
1115 r = gmc_v8_0_gart_init(adev);
1116 if (r)
1117 return r;
1118
1119 /*
1120 * number of VMs
1121 * VMID 0 is reserved for System
1122 * amdgpu graphics/compute will use VMIDs 1-7
1123 * amdkfd will use VMIDs 8-15
1124 */
1125 adev->vm_manager.id_mgr[0].num_ids = AMDGPU_NUM_OF_VMIDS;
1126 adev->vm_manager.num_level = 1;
1127 amdgpu_vm_manager_init(adev);
1128
1129 /* base offset of vram pages */
1130 if (adev->flags & AMD_IS_APU) {
1131 u64 tmp = RREG32(mmMC_VM_FB_OFFSET);
1132
1133 tmp <<= 22;
1134 adev->vm_manager.vram_base_offset = tmp;
1135 } else {
1136 adev->vm_manager.vram_base_offset = 0;
1137 }
1138
1139 return 0;
1140 }
1141
1142 static int gmc_v8_0_sw_fini(void *handle)
1143 {
1144 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1145
1146 amdgpu_vm_manager_fini(adev);
1147 gmc_v8_0_gart_fini(adev);
1148 amdgpu_gem_force_release(adev);
1149 amdgpu_bo_fini(adev);
1150
1151 return 0;
1152 }
1153
1154 static int gmc_v8_0_hw_init(void *handle)
1155 {
1156 int r;
1157 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1158
1159 gmc_v8_0_init_golden_registers(adev);
1160
1161 gmc_v8_0_mc_program(adev);
1162
1163 if (adev->asic_type == CHIP_TONGA) {
1164 r = gmc_v8_0_tonga_mc_load_microcode(adev);
1165 if (r) {
1166 DRM_ERROR("Failed to load MC firmware!\n");
1167 return r;
1168 }
1169 } else if (adev->asic_type == CHIP_POLARIS11 ||
1170 adev->asic_type == CHIP_POLARIS10 ||
1171 adev->asic_type == CHIP_POLARIS12) {
1172 r = gmc_v8_0_polaris_mc_load_microcode(adev);
1173 if (r) {
1174 DRM_ERROR("Failed to load MC firmware!\n");
1175 return r;
1176 }
1177 }
1178
1179 r = gmc_v8_0_gart_enable(adev);
1180 if (r)
1181 return r;
1182
1183 return r;
1184 }
1185
1186 static int gmc_v8_0_hw_fini(void *handle)
1187 {
1188 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1189
1190 amdgpu_irq_put(adev, &adev->mc.vm_fault, 0);
1191 gmc_v8_0_gart_disable(adev);
1192
1193 return 0;
1194 }
1195
1196 static int gmc_v8_0_suspend(void *handle)
1197 {
1198 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1199
1200 gmc_v8_0_hw_fini(adev);
1201
1202 return 0;
1203 }
1204
1205 static int gmc_v8_0_resume(void *handle)
1206 {
1207 int r;
1208 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1209
1210 r = gmc_v8_0_hw_init(adev);
1211 if (r)
1212 return r;
1213
1214 amdgpu_vm_reset_all_ids(adev);
1215
1216 return 0;
1217 }
1218
1219 static bool gmc_v8_0_is_idle(void *handle)
1220 {
1221 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1222 u32 tmp = RREG32(mmSRBM_STATUS);
1223
1224 if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
1225 SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK | SRBM_STATUS__VMC_BUSY_MASK))
1226 return false;
1227
1228 return true;
1229 }
1230
1231 static int gmc_v8_0_wait_for_idle(void *handle)
1232 {
1233 unsigned i;
1234 u32 tmp;
1235 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1236
1237 for (i = 0; i < adev->usec_timeout; i++) {
1238 /* read MC_STATUS */
1239 tmp = RREG32(mmSRBM_STATUS) & (SRBM_STATUS__MCB_BUSY_MASK |
1240 SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
1241 SRBM_STATUS__MCC_BUSY_MASK |
1242 SRBM_STATUS__MCD_BUSY_MASK |
1243 SRBM_STATUS__VMC_BUSY_MASK |
1244 SRBM_STATUS__VMC1_BUSY_MASK);
1245 if (!tmp)
1246 return 0;
1247 udelay(1);
1248 }
1249 return -ETIMEDOUT;
1250
1251 }
1252
1253 static bool gmc_v8_0_check_soft_reset(void *handle)
1254 {
1255 u32 srbm_soft_reset = 0;
1256 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1257 u32 tmp = RREG32(mmSRBM_STATUS);
1258
1259 if (tmp & SRBM_STATUS__VMC_BUSY_MASK)
1260 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
1261 SRBM_SOFT_RESET, SOFT_RESET_VMC, 1);
1262
1263 if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
1264 SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK)) {
1265 if (!(adev->flags & AMD_IS_APU))
1266 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
1267 SRBM_SOFT_RESET, SOFT_RESET_MC, 1);
1268 }
1269 if (srbm_soft_reset) {
1270 adev->mc.srbm_soft_reset = srbm_soft_reset;
1271 return true;
1272 } else {
1273 adev->mc.srbm_soft_reset = 0;
1274 return false;
1275 }
1276 }
1277
1278 static int gmc_v8_0_pre_soft_reset(void *handle)
1279 {
1280 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1281
1282 if (!adev->mc.srbm_soft_reset)
1283 return 0;
1284
1285 gmc_v8_0_mc_stop(adev);
1286 if (gmc_v8_0_wait_for_idle(adev)) {
1287 dev_warn(adev->dev, "Wait for GMC idle timed out !\n");
1288 }
1289
1290 return 0;
1291 }
1292
1293 static int gmc_v8_0_soft_reset(void *handle)
1294 {
1295 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1296 u32 srbm_soft_reset;
1297
1298 if (!adev->mc.srbm_soft_reset)
1299 return 0;
1300 srbm_soft_reset = adev->mc.srbm_soft_reset;
1301
1302 if (srbm_soft_reset) {
1303 u32 tmp;
1304
1305 tmp = RREG32(mmSRBM_SOFT_RESET);
1306 tmp |= srbm_soft_reset;
1307 dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1308 WREG32(mmSRBM_SOFT_RESET, tmp);
1309 tmp = RREG32(mmSRBM_SOFT_RESET);
1310
1311 udelay(50);
1312
1313 tmp &= ~srbm_soft_reset;
1314 WREG32(mmSRBM_SOFT_RESET, tmp);
1315 tmp = RREG32(mmSRBM_SOFT_RESET);
1316
1317 /* Wait a little for things to settle down */
1318 udelay(50);
1319 }
1320
1321 return 0;
1322 }
1323
1324 static int gmc_v8_0_post_soft_reset(void *handle)
1325 {
1326 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1327
1328 if (!adev->mc.srbm_soft_reset)
1329 return 0;
1330
1331 gmc_v8_0_mc_resume(adev);
1332 return 0;
1333 }
1334
1335 static int gmc_v8_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
1336 struct amdgpu_irq_src *src,
1337 unsigned type,
1338 enum amdgpu_interrupt_state state)
1339 {
1340 u32 tmp;
1341 u32 bits = (VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1342 VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1343 VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1344 VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1345 VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1346 VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1347 VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK);
1348
1349 switch (state) {
1350 case AMDGPU_IRQ_STATE_DISABLE:
1351 /* system context */
1352 tmp = RREG32(mmVM_CONTEXT0_CNTL);
1353 tmp &= ~bits;
1354 WREG32(mmVM_CONTEXT0_CNTL, tmp);
1355 /* VMs */
1356 tmp = RREG32(mmVM_CONTEXT1_CNTL);
1357 tmp &= ~bits;
1358 WREG32(mmVM_CONTEXT1_CNTL, tmp);
1359 break;
1360 case AMDGPU_IRQ_STATE_ENABLE:
1361 /* system context */
1362 tmp = RREG32(mmVM_CONTEXT0_CNTL);
1363 tmp |= bits;
1364 WREG32(mmVM_CONTEXT0_CNTL, tmp);
1365 /* VMs */
1366 tmp = RREG32(mmVM_CONTEXT1_CNTL);
1367 tmp |= bits;
1368 WREG32(mmVM_CONTEXT1_CNTL, tmp);
1369 break;
1370 default:
1371 break;
1372 }
1373
1374 return 0;
1375 }
1376
1377 static int gmc_v8_0_process_interrupt(struct amdgpu_device *adev,
1378 struct amdgpu_irq_src *source,
1379 struct amdgpu_iv_entry *entry)
1380 {
1381 u32 addr, status, mc_client;
1382
1383 if (amdgpu_sriov_vf(adev)) {
1384 dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",
1385 entry->src_id, entry->src_data[0]);
1386 dev_err(adev->dev, " Can't decode VM fault info here on SRIOV VF\n");
1387 return 0;
1388 }
1389
1390 addr = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_ADDR);
1391 status = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_STATUS);
1392 mc_client = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
1393 /* reset addr and status */
1394 WREG32_P(mmVM_CONTEXT1_CNTL2, 1, ~1);
1395
1396 if (!addr && !status)
1397 return 0;
1398
1399 if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_FIRST)
1400 gmc_v8_0_set_fault_enable_default(adev, false);
1401
1402 if (printk_ratelimit()) {
1403 dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",
1404 entry->src_id, entry->src_data[0]);
1405 dev_err(adev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
1406 addr);
1407 dev_err(adev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1408 status);
1409 gmc_v8_0_vm_decode_fault(adev, status, addr, mc_client);
1410 }
1411
1412 return 0;
1413 }
1414
1415 static void fiji_update_mc_medium_grain_clock_gating(struct amdgpu_device *adev,
1416 bool enable)
1417 {
1418 uint32_t data;
1419
1420 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_MGCG)) {
1421 data = RREG32(mmMC_HUB_MISC_HUB_CG);
1422 data |= MC_HUB_MISC_HUB_CG__ENABLE_MASK;
1423 WREG32(mmMC_HUB_MISC_HUB_CG, data);
1424
1425 data = RREG32(mmMC_HUB_MISC_SIP_CG);
1426 data |= MC_HUB_MISC_SIP_CG__ENABLE_MASK;
1427 WREG32(mmMC_HUB_MISC_SIP_CG, data);
1428
1429 data = RREG32(mmMC_HUB_MISC_VM_CG);
1430 data |= MC_HUB_MISC_VM_CG__ENABLE_MASK;
1431 WREG32(mmMC_HUB_MISC_VM_CG, data);
1432
1433 data = RREG32(mmMC_XPB_CLK_GAT);
1434 data |= MC_XPB_CLK_GAT__ENABLE_MASK;
1435 WREG32(mmMC_XPB_CLK_GAT, data);
1436
1437 data = RREG32(mmATC_MISC_CG);
1438 data |= ATC_MISC_CG__ENABLE_MASK;
1439 WREG32(mmATC_MISC_CG, data);
1440
1441 data = RREG32(mmMC_CITF_MISC_WR_CG);
1442 data |= MC_CITF_MISC_WR_CG__ENABLE_MASK;
1443 WREG32(mmMC_CITF_MISC_WR_CG, data);
1444
1445 data = RREG32(mmMC_CITF_MISC_RD_CG);
1446 data |= MC_CITF_MISC_RD_CG__ENABLE_MASK;
1447 WREG32(mmMC_CITF_MISC_RD_CG, data);
1448
1449 data = RREG32(mmMC_CITF_MISC_VM_CG);
1450 data |= MC_CITF_MISC_VM_CG__ENABLE_MASK;
1451 WREG32(mmMC_CITF_MISC_VM_CG, data);
1452
1453 data = RREG32(mmVM_L2_CG);
1454 data |= VM_L2_CG__ENABLE_MASK;
1455 WREG32(mmVM_L2_CG, data);
1456 } else {
1457 data = RREG32(mmMC_HUB_MISC_HUB_CG);
1458 data &= ~MC_HUB_MISC_HUB_CG__ENABLE_MASK;
1459 WREG32(mmMC_HUB_MISC_HUB_CG, data);
1460
1461 data = RREG32(mmMC_HUB_MISC_SIP_CG);
1462 data &= ~MC_HUB_MISC_SIP_CG__ENABLE_MASK;
1463 WREG32(mmMC_HUB_MISC_SIP_CG, data);
1464
1465 data = RREG32(mmMC_HUB_MISC_VM_CG);
1466 data &= ~MC_HUB_MISC_VM_CG__ENABLE_MASK;
1467 WREG32(mmMC_HUB_MISC_VM_CG, data);
1468
1469 data = RREG32(mmMC_XPB_CLK_GAT);
1470 data &= ~MC_XPB_CLK_GAT__ENABLE_MASK;
1471 WREG32(mmMC_XPB_CLK_GAT, data);
1472
1473 data = RREG32(mmATC_MISC_CG);
1474 data &= ~ATC_MISC_CG__ENABLE_MASK;
1475 WREG32(mmATC_MISC_CG, data);
1476
1477 data = RREG32(mmMC_CITF_MISC_WR_CG);
1478 data &= ~MC_CITF_MISC_WR_CG__ENABLE_MASK;
1479 WREG32(mmMC_CITF_MISC_WR_CG, data);
1480
1481 data = RREG32(mmMC_CITF_MISC_RD_CG);
1482 data &= ~MC_CITF_MISC_RD_CG__ENABLE_MASK;
1483 WREG32(mmMC_CITF_MISC_RD_CG, data);
1484
1485 data = RREG32(mmMC_CITF_MISC_VM_CG);
1486 data &= ~MC_CITF_MISC_VM_CG__ENABLE_MASK;
1487 WREG32(mmMC_CITF_MISC_VM_CG, data);
1488
1489 data = RREG32(mmVM_L2_CG);
1490 data &= ~VM_L2_CG__ENABLE_MASK;
1491 WREG32(mmVM_L2_CG, data);
1492 }
1493 }
1494
1495 static void fiji_update_mc_light_sleep(struct amdgpu_device *adev,
1496 bool enable)
1497 {
1498 uint32_t data;
1499
1500 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_LS)) {
1501 data = RREG32(mmMC_HUB_MISC_HUB_CG);
1502 data |= MC_HUB_MISC_HUB_CG__MEM_LS_ENABLE_MASK;
1503 WREG32(mmMC_HUB_MISC_HUB_CG, data);
1504
1505 data = RREG32(mmMC_HUB_MISC_SIP_CG);
1506 data |= MC_HUB_MISC_SIP_CG__MEM_LS_ENABLE_MASK;
1507 WREG32(mmMC_HUB_MISC_SIP_CG, data);
1508
1509 data = RREG32(mmMC_HUB_MISC_VM_CG);
1510 data |= MC_HUB_MISC_VM_CG__MEM_LS_ENABLE_MASK;
1511 WREG32(mmMC_HUB_MISC_VM_CG, data);
1512
1513 data = RREG32(mmMC_XPB_CLK_GAT);
1514 data |= MC_XPB_CLK_GAT__MEM_LS_ENABLE_MASK;
1515 WREG32(mmMC_XPB_CLK_GAT, data);
1516
1517 data = RREG32(mmATC_MISC_CG);
1518 data |= ATC_MISC_CG__MEM_LS_ENABLE_MASK;
1519 WREG32(mmATC_MISC_CG, data);
1520
1521 data = RREG32(mmMC_CITF_MISC_WR_CG);
1522 data |= MC_CITF_MISC_WR_CG__MEM_LS_ENABLE_MASK;
1523 WREG32(mmMC_CITF_MISC_WR_CG, data);
1524
1525 data = RREG32(mmMC_CITF_MISC_RD_CG);
1526 data |= MC_CITF_MISC_RD_CG__MEM_LS_ENABLE_MASK;
1527 WREG32(mmMC_CITF_MISC_RD_CG, data);
1528
1529 data = RREG32(mmMC_CITF_MISC_VM_CG);
1530 data |= MC_CITF_MISC_VM_CG__MEM_LS_ENABLE_MASK;
1531 WREG32(mmMC_CITF_MISC_VM_CG, data);
1532
1533 data = RREG32(mmVM_L2_CG);
1534 data |= VM_L2_CG__MEM_LS_ENABLE_MASK;
1535 WREG32(mmVM_L2_CG, data);
1536 } else {
1537 data = RREG32(mmMC_HUB_MISC_HUB_CG);
1538 data &= ~MC_HUB_MISC_HUB_CG__MEM_LS_ENABLE_MASK;
1539 WREG32(mmMC_HUB_MISC_HUB_CG, data);
1540
1541 data = RREG32(mmMC_HUB_MISC_SIP_CG);
1542 data &= ~MC_HUB_MISC_SIP_CG__MEM_LS_ENABLE_MASK;
1543 WREG32(mmMC_HUB_MISC_SIP_CG, data);
1544
1545 data = RREG32(mmMC_HUB_MISC_VM_CG);
1546 data &= ~MC_HUB_MISC_VM_CG__MEM_LS_ENABLE_MASK;
1547 WREG32(mmMC_HUB_MISC_VM_CG, data);
1548
1549 data = RREG32(mmMC_XPB_CLK_GAT);
1550 data &= ~MC_XPB_CLK_GAT__MEM_LS_ENABLE_MASK;
1551 WREG32(mmMC_XPB_CLK_GAT, data);
1552
1553 data = RREG32(mmATC_MISC_CG);
1554 data &= ~ATC_MISC_CG__MEM_LS_ENABLE_MASK;
1555 WREG32(mmATC_MISC_CG, data);
1556
1557 data = RREG32(mmMC_CITF_MISC_WR_CG);
1558 data &= ~MC_CITF_MISC_WR_CG__MEM_LS_ENABLE_MASK;
1559 WREG32(mmMC_CITF_MISC_WR_CG, data);
1560
1561 data = RREG32(mmMC_CITF_MISC_RD_CG);
1562 data &= ~MC_CITF_MISC_RD_CG__MEM_LS_ENABLE_MASK;
1563 WREG32(mmMC_CITF_MISC_RD_CG, data);
1564
1565 data = RREG32(mmMC_CITF_MISC_VM_CG);
1566 data &= ~MC_CITF_MISC_VM_CG__MEM_LS_ENABLE_MASK;
1567 WREG32(mmMC_CITF_MISC_VM_CG, data);
1568
1569 data = RREG32(mmVM_L2_CG);
1570 data &= ~VM_L2_CG__MEM_LS_ENABLE_MASK;
1571 WREG32(mmVM_L2_CG, data);
1572 }
1573 }
1574
1575 static int gmc_v8_0_set_clockgating_state(void *handle,
1576 enum amd_clockgating_state state)
1577 {
1578 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1579
1580 if (amdgpu_sriov_vf(adev))
1581 return 0;
1582
1583 switch (adev->asic_type) {
1584 case CHIP_FIJI:
1585 fiji_update_mc_medium_grain_clock_gating(adev,
1586 state == AMD_CG_STATE_GATE);
1587 fiji_update_mc_light_sleep(adev,
1588 state == AMD_CG_STATE_GATE);
1589 break;
1590 default:
1591 break;
1592 }
1593 return 0;
1594 }
1595
1596 static int gmc_v8_0_set_powergating_state(void *handle,
1597 enum amd_powergating_state state)
1598 {
1599 return 0;
1600 }
1601
1602 static void gmc_v8_0_get_clockgating_state(void *handle, u32 *flags)
1603 {
1604 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1605 int data;
1606
1607 if (amdgpu_sriov_vf(adev))
1608 *flags = 0;
1609
1610 /* AMD_CG_SUPPORT_MC_MGCG */
1611 data = RREG32(mmMC_HUB_MISC_HUB_CG);
1612 if (data & MC_HUB_MISC_HUB_CG__ENABLE_MASK)
1613 *flags |= AMD_CG_SUPPORT_MC_MGCG;
1614
1615 /* AMD_CG_SUPPORT_MC_LS */
1616 if (data & MC_HUB_MISC_HUB_CG__MEM_LS_ENABLE_MASK)
1617 *flags |= AMD_CG_SUPPORT_MC_LS;
1618 }
1619
1620 static const struct amd_ip_funcs gmc_v8_0_ip_funcs = {
1621 .name = "gmc_v8_0",
1622 .early_init = gmc_v8_0_early_init,
1623 .late_init = gmc_v8_0_late_init,
1624 .sw_init = gmc_v8_0_sw_init,
1625 .sw_fini = gmc_v8_0_sw_fini,
1626 .hw_init = gmc_v8_0_hw_init,
1627 .hw_fini = gmc_v8_0_hw_fini,
1628 .suspend = gmc_v8_0_suspend,
1629 .resume = gmc_v8_0_resume,
1630 .is_idle = gmc_v8_0_is_idle,
1631 .wait_for_idle = gmc_v8_0_wait_for_idle,
1632 .check_soft_reset = gmc_v8_0_check_soft_reset,
1633 .pre_soft_reset = gmc_v8_0_pre_soft_reset,
1634 .soft_reset = gmc_v8_0_soft_reset,
1635 .post_soft_reset = gmc_v8_0_post_soft_reset,
1636 .set_clockgating_state = gmc_v8_0_set_clockgating_state,
1637 .set_powergating_state = gmc_v8_0_set_powergating_state,
1638 .get_clockgating_state = gmc_v8_0_get_clockgating_state,
1639 };
1640
1641 static const struct amdgpu_gart_funcs gmc_v8_0_gart_funcs = {
1642 .flush_gpu_tlb = gmc_v8_0_gart_flush_gpu_tlb,
1643 .set_pte_pde = gmc_v8_0_gart_set_pte_pde,
1644 .set_prt = gmc_v8_0_set_prt,
1645 .get_vm_pte_flags = gmc_v8_0_get_vm_pte_flags,
1646 .get_vm_pde = gmc_v8_0_get_vm_pde
1647 };
1648
1649 static const struct amdgpu_irq_src_funcs gmc_v8_0_irq_funcs = {
1650 .set = gmc_v8_0_vm_fault_interrupt_state,
1651 .process = gmc_v8_0_process_interrupt,
1652 };
1653
1654 static void gmc_v8_0_set_gart_funcs(struct amdgpu_device *adev)
1655 {
1656 if (adev->gart.gart_funcs == NULL)
1657 adev->gart.gart_funcs = &gmc_v8_0_gart_funcs;
1658 }
1659
1660 static void gmc_v8_0_set_irq_funcs(struct amdgpu_device *adev)
1661 {
1662 adev->mc.vm_fault.num_types = 1;
1663 adev->mc.vm_fault.funcs = &gmc_v8_0_irq_funcs;
1664 }
1665
1666 const struct amdgpu_ip_block_version gmc_v8_0_ip_block =
1667 {
1668 .type = AMD_IP_BLOCK_TYPE_GMC,
1669 .major = 8,
1670 .minor = 0,
1671 .rev = 0,
1672 .funcs = &gmc_v8_0_ip_funcs,
1673 };
1674
1675 const struct amdgpu_ip_block_version gmc_v8_1_ip_block =
1676 {
1677 .type = AMD_IP_BLOCK_TYPE_GMC,
1678 .major = 8,
1679 .minor = 1,
1680 .rev = 0,
1681 .funcs = &gmc_v8_0_ip_funcs,
1682 };
1683
1684 const struct amdgpu_ip_block_version gmc_v8_5_ip_block =
1685 {
1686 .type = AMD_IP_BLOCK_TYPE_GMC,
1687 .major = 8,
1688 .minor = 5,
1689 .rev = 0,
1690 .funcs = &gmc_v8_0_ip_funcs,
1691 };
ubuntu-bionic-kernel mirror