2 * Copyright 2014 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
23 #include <linux/firmware.h>
25 #include <drm/drm_cache.h>
28 #include "amdgpu_ucode.h"
30 #include "gmc/gmc_8_1_d.h"
31 #include "gmc/gmc_8_1_sh_mask.h"
33 #include "bif/bif_5_0_d.h"
34 #include "bif/bif_5_0_sh_mask.h"
36 #include "oss/oss_3_0_d.h"
37 #include "oss/oss_3_0_sh_mask.h"
39 #include "dce/dce_10_0_d.h"
40 #include "dce/dce_10_0_sh_mask.h"
45 #include "amdgpu_atombios.h"
48 static void gmc_v8_0_set_gmc_funcs(struct amdgpu_device
*adev
);
49 static void gmc_v8_0_set_irq_funcs(struct amdgpu_device
*adev
);
50 static int gmc_v8_0_wait_for_idle(void *handle
);
52 MODULE_FIRMWARE("amdgpu/tonga_mc.bin");
53 MODULE_FIRMWARE("amdgpu/polaris11_mc.bin");
54 MODULE_FIRMWARE("amdgpu/polaris10_mc.bin");
55 MODULE_FIRMWARE("amdgpu/polaris12_mc.bin");
57 static const u32 golden_settings_tonga_a11
[] =
59 mmMC_ARB_WTM_GRPWT_RD
, 0x00000003, 0x00000000,
60 mmMC_HUB_RDREQ_DMIF_LIMIT
, 0x0000007f, 0x00000028,
61 mmMC_HUB_WDP_UMC
, 0x00007fb6, 0x00000991,
62 mmVM_PRT_APERTURE0_LOW_ADDR
, 0x0fffffff, 0x0fffffff,
63 mmVM_PRT_APERTURE1_LOW_ADDR
, 0x0fffffff, 0x0fffffff,
64 mmVM_PRT_APERTURE2_LOW_ADDR
, 0x0fffffff, 0x0fffffff,
65 mmVM_PRT_APERTURE3_LOW_ADDR
, 0x0fffffff, 0x0fffffff,
68 static const u32 tonga_mgcg_cgcg_init
[] =
70 mmMC_MEM_POWER_LS
, 0xffffffff, 0x00000104
73 static const u32 golden_settings_fiji_a10
[] =
75 mmVM_PRT_APERTURE0_LOW_ADDR
, 0x0fffffff, 0x0fffffff,
76 mmVM_PRT_APERTURE1_LOW_ADDR
, 0x0fffffff, 0x0fffffff,
77 mmVM_PRT_APERTURE2_LOW_ADDR
, 0x0fffffff, 0x0fffffff,
78 mmVM_PRT_APERTURE3_LOW_ADDR
, 0x0fffffff, 0x0fffffff,
81 static const u32 fiji_mgcg_cgcg_init
[] =
83 mmMC_MEM_POWER_LS
, 0xffffffff, 0x00000104
86 static const u32 golden_settings_polaris11_a11
[] =
88 mmVM_PRT_APERTURE0_LOW_ADDR
, 0x0fffffff, 0x0fffffff,
89 mmVM_PRT_APERTURE1_LOW_ADDR
, 0x0fffffff, 0x0fffffff,
90 mmVM_PRT_APERTURE2_LOW_ADDR
, 0x0fffffff, 0x0fffffff,
91 mmVM_PRT_APERTURE3_LOW_ADDR
, 0x0fffffff, 0x0fffffff
94 static const u32 golden_settings_polaris10_a11
[] =
96 mmMC_ARB_WTM_GRPWT_RD
, 0x00000003, 0x00000000,
97 mmVM_PRT_APERTURE0_LOW_ADDR
, 0x0fffffff, 0x0fffffff,
98 mmVM_PRT_APERTURE1_LOW_ADDR
, 0x0fffffff, 0x0fffffff,
99 mmVM_PRT_APERTURE2_LOW_ADDR
, 0x0fffffff, 0x0fffffff,
100 mmVM_PRT_APERTURE3_LOW_ADDR
, 0x0fffffff, 0x0fffffff
103 static const u32 cz_mgcg_cgcg_init
[] =
105 mmMC_MEM_POWER_LS
, 0xffffffff, 0x00000104
108 static const u32 stoney_mgcg_cgcg_init
[] =
110 mmATC_MISC_CG
, 0xffffffff, 0x000c0200,
111 mmMC_MEM_POWER_LS
, 0xffffffff, 0x00000104
114 static const u32 golden_settings_stoney_common
[] =
116 mmMC_HUB_RDREQ_UVD
, MC_HUB_RDREQ_UVD__PRESCALE_MASK
, 0x00000004,
117 mmMC_RD_GRP_OTH
, MC_RD_GRP_OTH__UVD_MASK
, 0x00600000
120 static void gmc_v8_0_init_golden_registers(struct amdgpu_device
*adev
)
122 switch (adev
->asic_type
) {
124 amdgpu_device_program_register_sequence(adev
,
126 ARRAY_SIZE(fiji_mgcg_cgcg_init
));
127 amdgpu_device_program_register_sequence(adev
,
128 golden_settings_fiji_a10
,
129 ARRAY_SIZE(golden_settings_fiji_a10
));
132 amdgpu_device_program_register_sequence(adev
,
133 tonga_mgcg_cgcg_init
,
134 ARRAY_SIZE(tonga_mgcg_cgcg_init
));
135 amdgpu_device_program_register_sequence(adev
,
136 golden_settings_tonga_a11
,
137 ARRAY_SIZE(golden_settings_tonga_a11
));
142 amdgpu_device_program_register_sequence(adev
,
143 golden_settings_polaris11_a11
,
144 ARRAY_SIZE(golden_settings_polaris11_a11
));
147 amdgpu_device_program_register_sequence(adev
,
148 golden_settings_polaris10_a11
,
149 ARRAY_SIZE(golden_settings_polaris10_a11
));
152 amdgpu_device_program_register_sequence(adev
,
154 ARRAY_SIZE(cz_mgcg_cgcg_init
));
157 amdgpu_device_program_register_sequence(adev
,
158 stoney_mgcg_cgcg_init
,
159 ARRAY_SIZE(stoney_mgcg_cgcg_init
));
160 amdgpu_device_program_register_sequence(adev
,
161 golden_settings_stoney_common
,
162 ARRAY_SIZE(golden_settings_stoney_common
));
169 static void gmc_v8_0_mc_stop(struct amdgpu_device
*adev
)
173 gmc_v8_0_wait_for_idle(adev
);
175 blackout
= RREG32(mmMC_SHARED_BLACKOUT_CNTL
);
176 if (REG_GET_FIELD(blackout
, MC_SHARED_BLACKOUT_CNTL
, BLACKOUT_MODE
) != 1) {
177 /* Block CPU access */
178 WREG32(mmBIF_FB_EN
, 0);
179 /* blackout the MC */
180 blackout
= REG_SET_FIELD(blackout
,
181 MC_SHARED_BLACKOUT_CNTL
, BLACKOUT_MODE
, 1);
182 WREG32(mmMC_SHARED_BLACKOUT_CNTL
, blackout
);
184 /* wait for the MC to settle */
188 static void gmc_v8_0_mc_resume(struct amdgpu_device
*adev
)
192 /* unblackout the MC */
193 tmp
= RREG32(mmMC_SHARED_BLACKOUT_CNTL
);
194 tmp
= REG_SET_FIELD(tmp
, MC_SHARED_BLACKOUT_CNTL
, BLACKOUT_MODE
, 0);
195 WREG32(mmMC_SHARED_BLACKOUT_CNTL
, tmp
);
196 /* allow CPU access */
197 tmp
= REG_SET_FIELD(0, BIF_FB_EN
, FB_READ_EN
, 1);
198 tmp
= REG_SET_FIELD(tmp
, BIF_FB_EN
, FB_WRITE_EN
, 1);
199 WREG32(mmBIF_FB_EN
, tmp
);
203 * gmc_v8_0_init_microcode - load ucode images from disk
205 * @adev: amdgpu_device pointer
207 * Use the firmware interface to load the ucode images into
208 * the driver (not loaded into hw).
209 * Returns 0 on success, error on failure.
211 static int gmc_v8_0_init_microcode(struct amdgpu_device
*adev
)
213 const char *chip_name
;
219 switch (adev
->asic_type
) {
224 chip_name
= "polaris11";
227 chip_name
= "polaris10";
230 chip_name
= "polaris12";
240 snprintf(fw_name
, sizeof(fw_name
), "amdgpu/%s_mc.bin", chip_name
);
241 err
= request_firmware(&adev
->gmc
.fw
, fw_name
, adev
->dev
);
244 err
= amdgpu_ucode_validate(adev
->gmc
.fw
);
248 pr_err("mc: Failed to load firmware \"%s\"\n", fw_name
);
249 release_firmware(adev
->gmc
.fw
);
256 * gmc_v8_0_tonga_mc_load_microcode - load tonga MC ucode into the hw
258 * @adev: amdgpu_device pointer
260 * Load the GDDR MC ucode into the hw (CIK).
261 * Returns 0 on success, error on failure.
263 static int gmc_v8_0_tonga_mc_load_microcode(struct amdgpu_device
*adev
)
265 const struct mc_firmware_header_v1_0
*hdr
;
266 const __le32
*fw_data
= NULL
;
267 const __le32
*io_mc_regs
= NULL
;
269 int i
, ucode_size
, regs_size
;
271 /* Skip MC ucode loading on SR-IOV capable boards.
272 * vbios does this for us in asic_init in that case.
273 * Skip MC ucode loading on VF, because hypervisor will do that
276 if (amdgpu_sriov_bios(adev
))
282 hdr
= (const struct mc_firmware_header_v1_0
*)adev
->gmc
.fw
->data
;
283 amdgpu_ucode_print_mc_hdr(&hdr
->header
);
285 adev
->gmc
.fw_version
= le32_to_cpu(hdr
->header
.ucode_version
);
286 regs_size
= le32_to_cpu(hdr
->io_debug_size_bytes
) / (4 * 2);
287 io_mc_regs
= (const __le32
*)
288 (adev
->gmc
.fw
->data
+ le32_to_cpu(hdr
->io_debug_array_offset_bytes
));
289 ucode_size
= le32_to_cpu(hdr
->header
.ucode_size_bytes
) / 4;
290 fw_data
= (const __le32
*)
291 (adev
->gmc
.fw
->data
+ le32_to_cpu(hdr
->header
.ucode_array_offset_bytes
));
293 running
= REG_GET_FIELD(RREG32(mmMC_SEQ_SUP_CNTL
), MC_SEQ_SUP_CNTL
, RUN
);
296 /* reset the engine and set to writable */
297 WREG32(mmMC_SEQ_SUP_CNTL
, 0x00000008);
298 WREG32(mmMC_SEQ_SUP_CNTL
, 0x00000010);
300 /* load mc io regs */
301 for (i
= 0; i
< regs_size
; i
++) {
302 WREG32(mmMC_SEQ_IO_DEBUG_INDEX
, le32_to_cpup(io_mc_regs
++));
303 WREG32(mmMC_SEQ_IO_DEBUG_DATA
, le32_to_cpup(io_mc_regs
++));
305 /* load the MC ucode */
306 for (i
= 0; i
< ucode_size
; i
++)
307 WREG32(mmMC_SEQ_SUP_PGM
, le32_to_cpup(fw_data
++));
309 /* put the engine back into the active state */
310 WREG32(mmMC_SEQ_SUP_CNTL
, 0x00000008);
311 WREG32(mmMC_SEQ_SUP_CNTL
, 0x00000004);
312 WREG32(mmMC_SEQ_SUP_CNTL
, 0x00000001);
314 /* wait for training to complete */
315 for (i
= 0; i
< adev
->usec_timeout
; i
++) {
316 if (REG_GET_FIELD(RREG32(mmMC_SEQ_TRAIN_WAKEUP_CNTL
),
317 MC_SEQ_TRAIN_WAKEUP_CNTL
, TRAIN_DONE_D0
))
321 for (i
= 0; i
< adev
->usec_timeout
; i
++) {
322 if (REG_GET_FIELD(RREG32(mmMC_SEQ_TRAIN_WAKEUP_CNTL
),
323 MC_SEQ_TRAIN_WAKEUP_CNTL
, TRAIN_DONE_D1
))
332 static int gmc_v8_0_polaris_mc_load_microcode(struct amdgpu_device
*adev
)
334 const struct mc_firmware_header_v1_0
*hdr
;
335 const __le32
*fw_data
= NULL
;
336 const __le32
*io_mc_regs
= NULL
;
337 u32 data
, vbios_version
;
338 int i
, ucode_size
, regs_size
;
340 /* Skip MC ucode loading on SR-IOV capable boards.
341 * vbios does this for us in asic_init in that case.
342 * Skip MC ucode loading on VF, because hypervisor will do that
345 if (amdgpu_sriov_bios(adev
))
348 WREG32(mmMC_SEQ_IO_DEBUG_INDEX
, 0x9F);
349 data
= RREG32(mmMC_SEQ_IO_DEBUG_DATA
);
350 vbios_version
= data
& 0xf;
352 if (vbios_version
== 0)
358 hdr
= (const struct mc_firmware_header_v1_0
*)adev
->gmc
.fw
->data
;
359 amdgpu_ucode_print_mc_hdr(&hdr
->header
);
361 adev
->gmc
.fw_version
= le32_to_cpu(hdr
->header
.ucode_version
);
362 regs_size
= le32_to_cpu(hdr
->io_debug_size_bytes
) / (4 * 2);
363 io_mc_regs
= (const __le32
*)
364 (adev
->gmc
.fw
->data
+ le32_to_cpu(hdr
->io_debug_array_offset_bytes
));
365 ucode_size
= le32_to_cpu(hdr
->header
.ucode_size_bytes
) / 4;
366 fw_data
= (const __le32
*)
367 (adev
->gmc
.fw
->data
+ le32_to_cpu(hdr
->header
.ucode_array_offset_bytes
));
369 data
= RREG32(mmMC_SEQ_MISC0
);
371 WREG32(mmMC_SEQ_MISC0
, data
);
373 /* load mc io regs */
374 for (i
= 0; i
< regs_size
; i
++) {
375 WREG32(mmMC_SEQ_IO_DEBUG_INDEX
, le32_to_cpup(io_mc_regs
++));
376 WREG32(mmMC_SEQ_IO_DEBUG_DATA
, le32_to_cpup(io_mc_regs
++));
379 WREG32(mmMC_SEQ_SUP_CNTL
, 0x00000008);
380 WREG32(mmMC_SEQ_SUP_CNTL
, 0x00000010);
382 /* load the MC ucode */
383 for (i
= 0; i
< ucode_size
; i
++)
384 WREG32(mmMC_SEQ_SUP_PGM
, le32_to_cpup(fw_data
++));
386 /* put the engine back into the active state */
387 WREG32(mmMC_SEQ_SUP_CNTL
, 0x00000008);
388 WREG32(mmMC_SEQ_SUP_CNTL
, 0x00000004);
389 WREG32(mmMC_SEQ_SUP_CNTL
, 0x00000001);
391 /* wait for training to complete */
392 for (i
= 0; i
< adev
->usec_timeout
; i
++) {
393 data
= RREG32(mmMC_SEQ_MISC0
);
402 static void gmc_v8_0_vram_gtt_location(struct amdgpu_device
*adev
,
403 struct amdgpu_gmc
*mc
)
407 if (!amdgpu_sriov_vf(adev
))
408 base
= RREG32(mmMC_VM_FB_LOCATION
) & 0xFFFF;
411 amdgpu_device_vram_location(adev
, &adev
->gmc
, base
);
412 amdgpu_device_gart_location(adev
, mc
);
416 * gmc_v8_0_mc_program - program the GPU memory controller
418 * @adev: amdgpu_device pointer
420 * Set the location of vram, gart, and AGP in the GPU's
421 * physical address space (CIK).
423 static void gmc_v8_0_mc_program(struct amdgpu_device
*adev
)
429 for (i
= 0, j
= 0; i
< 32; i
++, j
+= 0x6) {
430 WREG32((0xb05 + j
), 0x00000000);
431 WREG32((0xb06 + j
), 0x00000000);
432 WREG32((0xb07 + j
), 0x00000000);
433 WREG32((0xb08 + j
), 0x00000000);
434 WREG32((0xb09 + j
), 0x00000000);
436 WREG32(mmHDP_REG_COHERENCY_FLUSH_CNTL
, 0);
438 if (gmc_v8_0_wait_for_idle((void *)adev
)) {
439 dev_warn(adev
->dev
, "Wait for MC idle timedout !\n");
441 if (adev
->mode_info
.num_crtc
) {
442 /* Lockout access through VGA aperture*/
443 tmp
= RREG32(mmVGA_HDP_CONTROL
);
444 tmp
= REG_SET_FIELD(tmp
, VGA_HDP_CONTROL
, VGA_MEMORY_DISABLE
, 1);
445 WREG32(mmVGA_HDP_CONTROL
, tmp
);
447 /* disable VGA render */
448 tmp
= RREG32(mmVGA_RENDER_CONTROL
);
449 tmp
= REG_SET_FIELD(tmp
, VGA_RENDER_CONTROL
, VGA_VSTATUS_CNTL
, 0);
450 WREG32(mmVGA_RENDER_CONTROL
, tmp
);
452 /* Update configuration */
453 WREG32(mmMC_VM_SYSTEM_APERTURE_LOW_ADDR
,
454 adev
->gmc
.vram_start
>> 12);
455 WREG32(mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR
,
456 adev
->gmc
.vram_end
>> 12);
457 WREG32(mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR
,
458 adev
->vram_scratch
.gpu_addr
>> 12);
460 if (amdgpu_sriov_vf(adev
)) {
461 tmp
= ((adev
->gmc
.vram_end
>> 24) & 0xFFFF) << 16;
462 tmp
|= ((adev
->gmc
.vram_start
>> 24) & 0xFFFF);
463 WREG32(mmMC_VM_FB_LOCATION
, tmp
);
464 /* XXX double check these! */
465 WREG32(mmHDP_NONSURFACE_BASE
, (adev
->gmc
.vram_start
>> 8));
466 WREG32(mmHDP_NONSURFACE_INFO
, (2 << 7) | (1 << 30));
467 WREG32(mmHDP_NONSURFACE_SIZE
, 0x3FFFFFFF);
470 WREG32(mmMC_VM_AGP_BASE
, 0);
471 WREG32(mmMC_VM_AGP_TOP
, 0x0FFFFFFF);
472 WREG32(mmMC_VM_AGP_BOT
, 0x0FFFFFFF);
473 if (gmc_v8_0_wait_for_idle((void *)adev
)) {
474 dev_warn(adev
->dev
, "Wait for MC idle timedout !\n");
477 WREG32(mmBIF_FB_EN
, BIF_FB_EN__FB_READ_EN_MASK
| BIF_FB_EN__FB_WRITE_EN_MASK
);
479 tmp
= RREG32(mmHDP_MISC_CNTL
);
480 tmp
= REG_SET_FIELD(tmp
, HDP_MISC_CNTL
, FLUSH_INVALIDATE_CACHE
, 0);
481 WREG32(mmHDP_MISC_CNTL
, tmp
);
483 tmp
= RREG32(mmHDP_HOST_PATH_CNTL
);
484 WREG32(mmHDP_HOST_PATH_CNTL
, tmp
);
488 * gmc_v8_0_mc_init - initialize the memory controller driver params
490 * @adev: amdgpu_device pointer
492 * Look up the amount of vram, vram width, and decide how to place
493 * vram and gart within the GPU's physical address space (CIK).
494 * Returns 0 for success.
496 static int gmc_v8_0_mc_init(struct amdgpu_device
*adev
)
500 adev
->gmc
.vram_width
= amdgpu_atombios_get_vram_width(adev
);
501 if (!adev
->gmc
.vram_width
) {
503 int chansize
, numchan
;
505 /* Get VRAM informations */
506 tmp
= RREG32(mmMC_ARB_RAMCFG
);
507 if (REG_GET_FIELD(tmp
, MC_ARB_RAMCFG
, CHANSIZE
)) {
512 tmp
= RREG32(mmMC_SHARED_CHMAP
);
513 switch (REG_GET_FIELD(tmp
, MC_SHARED_CHMAP
, NOOFCHAN
)) {
543 adev
->gmc
.vram_width
= numchan
* chansize
;
545 /* size in MB on si */
546 adev
->gmc
.mc_vram_size
= RREG32(mmCONFIG_MEMSIZE
) * 1024ULL * 1024ULL;
547 adev
->gmc
.real_vram_size
= RREG32(mmCONFIG_MEMSIZE
) * 1024ULL * 1024ULL;
549 if (!(adev
->flags
& AMD_IS_APU
)) {
550 r
= amdgpu_device_resize_fb_bar(adev
);
554 adev
->gmc
.aper_base
= pci_resource_start(adev
->pdev
, 0);
555 adev
->gmc
.aper_size
= pci_resource_len(adev
->pdev
, 0);
558 if (adev
->flags
& AMD_IS_APU
) {
559 adev
->gmc
.aper_base
= ((u64
)RREG32(mmMC_VM_FB_OFFSET
)) << 22;
560 adev
->gmc
.aper_size
= adev
->gmc
.real_vram_size
;
564 /* In case the PCI BAR is larger than the actual amount of vram */
565 adev
->gmc
.visible_vram_size
= adev
->gmc
.aper_size
;
566 if (adev
->gmc
.visible_vram_size
> adev
->gmc
.real_vram_size
)
567 adev
->gmc
.visible_vram_size
= adev
->gmc
.real_vram_size
;
569 /* set the gart size */
570 if (amdgpu_gart_size
== -1) {
571 switch (adev
->asic_type
) {
572 case CHIP_POLARIS10
: /* all engines support GPUVM */
573 case CHIP_POLARIS11
: /* all engines support GPUVM */
574 case CHIP_POLARIS12
: /* all engines support GPUVM */
575 case CHIP_VEGAM
: /* all engines support GPUVM */
577 adev
->gmc
.gart_size
= 256ULL << 20;
579 case CHIP_TONGA
: /* UVD, VCE do not support GPUVM */
580 case CHIP_FIJI
: /* UVD, VCE do not support GPUVM */
581 case CHIP_CARRIZO
: /* UVD, VCE do not support GPUVM, DCE SG support */
582 case CHIP_STONEY
: /* UVD does not support GPUVM, DCE SG support */
583 adev
->gmc
.gart_size
= 1024ULL << 20;
587 adev
->gmc
.gart_size
= (u64
)amdgpu_gart_size
<< 20;
590 gmc_v8_0_vram_gtt_location(adev
, &adev
->gmc
);
597 * VMID 0 is the physical GPU addresses as used by the kernel.
598 * VMIDs 1-15 are used for userspace clients and are handled
599 * by the amdgpu vm/hsa code.
603 * gmc_v8_0_flush_gpu_tlb - gart tlb flush callback
605 * @adev: amdgpu_device pointer
606 * @vmid: vm instance to flush
608 * Flush the TLB for the requested page table (CIK).
610 static void gmc_v8_0_flush_gpu_tlb(struct amdgpu_device
*adev
,
613 /* bits 0-15 are the VM contexts0-15 */
614 WREG32(mmVM_INVALIDATE_REQUEST
, 1 << vmid
);
617 static uint64_t gmc_v8_0_emit_flush_gpu_tlb(struct amdgpu_ring
*ring
,
618 unsigned vmid
, uint64_t pd_addr
)
623 reg
= mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR
+ vmid
;
625 reg
= mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR
+ vmid
- 8;
626 amdgpu_ring_emit_wreg(ring
, reg
, pd_addr
>> 12);
628 /* bits 0-15 are the VM contexts0-15 */
629 amdgpu_ring_emit_wreg(ring
, mmVM_INVALIDATE_REQUEST
, 1 << vmid
);
634 static void gmc_v8_0_emit_pasid_mapping(struct amdgpu_ring
*ring
, unsigned vmid
,
637 amdgpu_ring_emit_wreg(ring
, mmIH_VMID_0_LUT
+ vmid
, pasid
);
641 * gmc_v8_0_set_pte_pde - update the page tables using MMIO
643 * @adev: amdgpu_device pointer
644 * @cpu_pt_addr: cpu address of the page table
645 * @gpu_page_idx: entry in the page table to update
646 * @addr: dst addr to write into pte/pde
647 * @flags: access flags
649 * Update the page tables using the CPU.
651 static int gmc_v8_0_set_pte_pde(struct amdgpu_device
*adev
, void *cpu_pt_addr
,
652 uint32_t gpu_page_idx
, uint64_t addr
,
655 void __iomem
*ptr
= (void *)cpu_pt_addr
;
661 * 39:12 4k physical page base address
672 * 63:59 block fragment size
674 * 39:1 physical base address of PTE
675 * bits 5:1 must be 0.
678 value
= addr
& 0x000000FFFFFFF000ULL
;
680 writeq(value
, ptr
+ (gpu_page_idx
* 8));
685 static uint64_t gmc_v8_0_get_vm_pte_flags(struct amdgpu_device
*adev
,
688 uint64_t pte_flag
= 0;
690 if (flags
& AMDGPU_VM_PAGE_EXECUTABLE
)
691 pte_flag
|= AMDGPU_PTE_EXECUTABLE
;
692 if (flags
& AMDGPU_VM_PAGE_READABLE
)
693 pte_flag
|= AMDGPU_PTE_READABLE
;
694 if (flags
& AMDGPU_VM_PAGE_WRITEABLE
)
695 pte_flag
|= AMDGPU_PTE_WRITEABLE
;
696 if (flags
& AMDGPU_VM_PAGE_PRT
)
697 pte_flag
|= AMDGPU_PTE_PRT
;
702 static void gmc_v8_0_get_vm_pde(struct amdgpu_device
*adev
, int level
,
703 uint64_t *addr
, uint64_t *flags
)
705 BUG_ON(*addr
& 0xFFFFFF0000000FFFULL
);
709 * gmc_v8_0_set_fault_enable_default - update VM fault handling
711 * @adev: amdgpu_device pointer
712 * @value: true redirects VM faults to the default page
714 static void gmc_v8_0_set_fault_enable_default(struct amdgpu_device
*adev
,
719 tmp
= RREG32(mmVM_CONTEXT1_CNTL
);
720 tmp
= REG_SET_FIELD(tmp
, VM_CONTEXT1_CNTL
,
721 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT
, value
);
722 tmp
= REG_SET_FIELD(tmp
, VM_CONTEXT1_CNTL
,
723 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT
, value
);
724 tmp
= REG_SET_FIELD(tmp
, VM_CONTEXT1_CNTL
,
725 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT
, value
);
726 tmp
= REG_SET_FIELD(tmp
, VM_CONTEXT1_CNTL
,
727 VALID_PROTECTION_FAULT_ENABLE_DEFAULT
, value
);
728 tmp
= REG_SET_FIELD(tmp
, VM_CONTEXT1_CNTL
,
729 READ_PROTECTION_FAULT_ENABLE_DEFAULT
, value
);
730 tmp
= REG_SET_FIELD(tmp
, VM_CONTEXT1_CNTL
,
731 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT
, value
);
732 tmp
= REG_SET_FIELD(tmp
, VM_CONTEXT1_CNTL
,
733 EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT
, value
);
734 WREG32(mmVM_CONTEXT1_CNTL
, tmp
);
738 * gmc_v8_0_set_prt - set PRT VM fault
740 * @adev: amdgpu_device pointer
741 * @enable: enable/disable VM fault handling for PRT
743 static void gmc_v8_0_set_prt(struct amdgpu_device
*adev
, bool enable
)
747 if (enable
&& !adev
->gmc
.prt_warning
) {
748 dev_warn(adev
->dev
, "Disabling VM faults because of PRT request!\n");
749 adev
->gmc
.prt_warning
= true;
752 tmp
= RREG32(mmVM_PRT_CNTL
);
753 tmp
= REG_SET_FIELD(tmp
, VM_PRT_CNTL
,
754 CB_DISABLE_READ_FAULT_ON_UNMAPPED_ACCESS
, enable
);
755 tmp
= REG_SET_FIELD(tmp
, VM_PRT_CNTL
,
756 CB_DISABLE_WRITE_FAULT_ON_UNMAPPED_ACCESS
, enable
);
757 tmp
= REG_SET_FIELD(tmp
, VM_PRT_CNTL
,
758 TC_DISABLE_READ_FAULT_ON_UNMAPPED_ACCESS
, enable
);
759 tmp
= REG_SET_FIELD(tmp
, VM_PRT_CNTL
,
760 TC_DISABLE_WRITE_FAULT_ON_UNMAPPED_ACCESS
, enable
);
761 tmp
= REG_SET_FIELD(tmp
, VM_PRT_CNTL
,
762 L2_CACHE_STORE_INVALID_ENTRIES
, enable
);
763 tmp
= REG_SET_FIELD(tmp
, VM_PRT_CNTL
,
764 L1_TLB_STORE_INVALID_ENTRIES
, enable
);
765 tmp
= REG_SET_FIELD(tmp
, VM_PRT_CNTL
,
766 MASK_PDE0_FAULT
, enable
);
767 WREG32(mmVM_PRT_CNTL
, tmp
);
770 uint32_t low
= AMDGPU_VA_RESERVED_SIZE
>> AMDGPU_GPU_PAGE_SHIFT
;
771 uint32_t high
= adev
->vm_manager
.max_pfn
-
772 (AMDGPU_VA_RESERVED_SIZE
>> AMDGPU_GPU_PAGE_SHIFT
);
774 WREG32(mmVM_PRT_APERTURE0_LOW_ADDR
, low
);
775 WREG32(mmVM_PRT_APERTURE1_LOW_ADDR
, low
);
776 WREG32(mmVM_PRT_APERTURE2_LOW_ADDR
, low
);
777 WREG32(mmVM_PRT_APERTURE3_LOW_ADDR
, low
);
778 WREG32(mmVM_PRT_APERTURE0_HIGH_ADDR
, high
);
779 WREG32(mmVM_PRT_APERTURE1_HIGH_ADDR
, high
);
780 WREG32(mmVM_PRT_APERTURE2_HIGH_ADDR
, high
);
781 WREG32(mmVM_PRT_APERTURE3_HIGH_ADDR
, high
);
783 WREG32(mmVM_PRT_APERTURE0_LOW_ADDR
, 0xfffffff);
784 WREG32(mmVM_PRT_APERTURE1_LOW_ADDR
, 0xfffffff);
785 WREG32(mmVM_PRT_APERTURE2_LOW_ADDR
, 0xfffffff);
786 WREG32(mmVM_PRT_APERTURE3_LOW_ADDR
, 0xfffffff);
787 WREG32(mmVM_PRT_APERTURE0_HIGH_ADDR
, 0x0);
788 WREG32(mmVM_PRT_APERTURE1_HIGH_ADDR
, 0x0);
789 WREG32(mmVM_PRT_APERTURE2_HIGH_ADDR
, 0x0);
790 WREG32(mmVM_PRT_APERTURE3_HIGH_ADDR
, 0x0);
795 * gmc_v8_0_gart_enable - gart enable
797 * @adev: amdgpu_device pointer
799 * This sets up the TLBs, programs the page tables for VMID0,
800 * sets up the hw for VMIDs 1-15 which are allocated on
801 * demand, and sets up the global locations for the LDS, GDS,
802 * and GPUVM for FSA64 clients (CIK).
803 * Returns 0 for success, errors for failure.
805 static int gmc_v8_0_gart_enable(struct amdgpu_device
*adev
)
810 if (adev
->gart
.robj
== NULL
) {
811 dev_err(adev
->dev
, "No VRAM object for PCIE GART.\n");
814 r
= amdgpu_gart_table_vram_pin(adev
);
817 /* Setup TLB control */
818 tmp
= RREG32(mmMC_VM_MX_L1_TLB_CNTL
);
819 tmp
= REG_SET_FIELD(tmp
, MC_VM_MX_L1_TLB_CNTL
, ENABLE_L1_TLB
, 1);
820 tmp
= REG_SET_FIELD(tmp
, MC_VM_MX_L1_TLB_CNTL
, ENABLE_L1_FRAGMENT_PROCESSING
, 1);
821 tmp
= REG_SET_FIELD(tmp
, MC_VM_MX_L1_TLB_CNTL
, SYSTEM_ACCESS_MODE
, 3);
822 tmp
= REG_SET_FIELD(tmp
, MC_VM_MX_L1_TLB_CNTL
, ENABLE_ADVANCED_DRIVER_MODEL
, 1);
823 tmp
= REG_SET_FIELD(tmp
, MC_VM_MX_L1_TLB_CNTL
, SYSTEM_APERTURE_UNMAPPED_ACCESS
, 0);
824 WREG32(mmMC_VM_MX_L1_TLB_CNTL
, tmp
);
826 tmp
= RREG32(mmVM_L2_CNTL
);
827 tmp
= REG_SET_FIELD(tmp
, VM_L2_CNTL
, ENABLE_L2_CACHE
, 1);
828 tmp
= REG_SET_FIELD(tmp
, VM_L2_CNTL
, ENABLE_L2_FRAGMENT_PROCESSING
, 1);
829 tmp
= REG_SET_FIELD(tmp
, VM_L2_CNTL
, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE
, 1);
830 tmp
= REG_SET_FIELD(tmp
, VM_L2_CNTL
, ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE
, 1);
831 tmp
= REG_SET_FIELD(tmp
, VM_L2_CNTL
, EFFECTIVE_L2_QUEUE_SIZE
, 7);
832 tmp
= REG_SET_FIELD(tmp
, VM_L2_CNTL
, CONTEXT1_IDENTITY_ACCESS_MODE
, 1);
833 tmp
= REG_SET_FIELD(tmp
, VM_L2_CNTL
, ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY
, 1);
834 WREG32(mmVM_L2_CNTL
, tmp
);
835 tmp
= RREG32(mmVM_L2_CNTL2
);
836 tmp
= REG_SET_FIELD(tmp
, VM_L2_CNTL2
, INVALIDATE_ALL_L1_TLBS
, 1);
837 tmp
= REG_SET_FIELD(tmp
, VM_L2_CNTL2
, INVALIDATE_L2_CACHE
, 1);
838 WREG32(mmVM_L2_CNTL2
, tmp
);
840 field
= adev
->vm_manager
.fragment_size
;
841 tmp
= RREG32(mmVM_L2_CNTL3
);
842 tmp
= REG_SET_FIELD(tmp
, VM_L2_CNTL3
, L2_CACHE_BIGK_ASSOCIATIVITY
, 1);
843 tmp
= REG_SET_FIELD(tmp
, VM_L2_CNTL3
, BANK_SELECT
, field
);
844 tmp
= REG_SET_FIELD(tmp
, VM_L2_CNTL3
, L2_CACHE_BIGK_FRAGMENT_SIZE
, field
);
845 WREG32(mmVM_L2_CNTL3
, tmp
);
846 /* XXX: set to enable PTE/PDE in system memory */
847 tmp
= RREG32(mmVM_L2_CNTL4
);
848 tmp
= REG_SET_FIELD(tmp
, VM_L2_CNTL4
, VMC_TAP_CONTEXT0_PDE_REQUEST_PHYSICAL
, 0);
849 tmp
= REG_SET_FIELD(tmp
, VM_L2_CNTL4
, VMC_TAP_CONTEXT0_PDE_REQUEST_SHARED
, 0);
850 tmp
= REG_SET_FIELD(tmp
, VM_L2_CNTL4
, VMC_TAP_CONTEXT0_PDE_REQUEST_SNOOP
, 0);
851 tmp
= REG_SET_FIELD(tmp
, VM_L2_CNTL4
, VMC_TAP_CONTEXT0_PTE_REQUEST_PHYSICAL
, 0);
852 tmp
= REG_SET_FIELD(tmp
, VM_L2_CNTL4
, VMC_TAP_CONTEXT0_PTE_REQUEST_SHARED
, 0);
853 tmp
= REG_SET_FIELD(tmp
, VM_L2_CNTL4
, VMC_TAP_CONTEXT0_PTE_REQUEST_SNOOP
, 0);
854 tmp
= REG_SET_FIELD(tmp
, VM_L2_CNTL4
, VMC_TAP_CONTEXT1_PDE_REQUEST_PHYSICAL
, 0);
855 tmp
= REG_SET_FIELD(tmp
, VM_L2_CNTL4
, VMC_TAP_CONTEXT1_PDE_REQUEST_SHARED
, 0);
856 tmp
= REG_SET_FIELD(tmp
, VM_L2_CNTL4
, VMC_TAP_CONTEXT1_PDE_REQUEST_SNOOP
, 0);
857 tmp
= REG_SET_FIELD(tmp
, VM_L2_CNTL4
, VMC_TAP_CONTEXT1_PTE_REQUEST_PHYSICAL
, 0);
858 tmp
= REG_SET_FIELD(tmp
, VM_L2_CNTL4
, VMC_TAP_CONTEXT1_PTE_REQUEST_SHARED
, 0);
859 tmp
= REG_SET_FIELD(tmp
, VM_L2_CNTL4
, VMC_TAP_CONTEXT1_PTE_REQUEST_SNOOP
, 0);
860 WREG32(mmVM_L2_CNTL4
, tmp
);
862 WREG32(mmVM_CONTEXT0_PAGE_TABLE_START_ADDR
, adev
->gmc
.gart_start
>> 12);
863 WREG32(mmVM_CONTEXT0_PAGE_TABLE_END_ADDR
, adev
->gmc
.gart_end
>> 12);
864 WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR
, adev
->gart
.table_addr
>> 12);
865 WREG32(mmVM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR
,
866 (u32
)(adev
->dummy_page_addr
>> 12));
867 WREG32(mmVM_CONTEXT0_CNTL2
, 0);
868 tmp
= RREG32(mmVM_CONTEXT0_CNTL
);
869 tmp
= REG_SET_FIELD(tmp
, VM_CONTEXT0_CNTL
, ENABLE_CONTEXT
, 1);
870 tmp
= REG_SET_FIELD(tmp
, VM_CONTEXT0_CNTL
, PAGE_TABLE_DEPTH
, 0);
871 tmp
= REG_SET_FIELD(tmp
, VM_CONTEXT0_CNTL
, RANGE_PROTECTION_FAULT_ENABLE_DEFAULT
, 1);
872 WREG32(mmVM_CONTEXT0_CNTL
, tmp
);
874 WREG32(mmVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR
, 0);
875 WREG32(mmVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR
, 0);
876 WREG32(mmVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET
, 0);
878 /* empty context1-15 */
879 /* FIXME start with 4G, once using 2 level pt switch to full
882 /* set vm size, must be a multiple of 4 */
883 WREG32(mmVM_CONTEXT1_PAGE_TABLE_START_ADDR
, 0);
884 WREG32(mmVM_CONTEXT1_PAGE_TABLE_END_ADDR
, adev
->vm_manager
.max_pfn
- 1);
885 for (i
= 1; i
< 16; i
++) {
887 WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR
+ i
,
888 adev
->gart
.table_addr
>> 12);
890 WREG32(mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR
+ i
- 8,
891 adev
->gart
.table_addr
>> 12);
894 /* enable context1-15 */
895 WREG32(mmVM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR
,
896 (u32
)(adev
->dummy_page_addr
>> 12));
897 WREG32(mmVM_CONTEXT1_CNTL2
, 4);
898 tmp
= RREG32(mmVM_CONTEXT1_CNTL
);
899 tmp
= REG_SET_FIELD(tmp
, VM_CONTEXT1_CNTL
, ENABLE_CONTEXT
, 1);
900 tmp
= REG_SET_FIELD(tmp
, VM_CONTEXT1_CNTL
, PAGE_TABLE_DEPTH
, 1);
901 tmp
= REG_SET_FIELD(tmp
, VM_CONTEXT1_CNTL
, RANGE_PROTECTION_FAULT_ENABLE_DEFAULT
, 1);
902 tmp
= REG_SET_FIELD(tmp
, VM_CONTEXT1_CNTL
, DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT
, 1);
903 tmp
= REG_SET_FIELD(tmp
, VM_CONTEXT1_CNTL
, PDE0_PROTECTION_FAULT_ENABLE_DEFAULT
, 1);
904 tmp
= REG_SET_FIELD(tmp
, VM_CONTEXT1_CNTL
, VALID_PROTECTION_FAULT_ENABLE_DEFAULT
, 1);
905 tmp
= REG_SET_FIELD(tmp
, VM_CONTEXT1_CNTL
, READ_PROTECTION_FAULT_ENABLE_DEFAULT
, 1);
906 tmp
= REG_SET_FIELD(tmp
, VM_CONTEXT1_CNTL
, WRITE_PROTECTION_FAULT_ENABLE_DEFAULT
, 1);
907 tmp
= REG_SET_FIELD(tmp
, VM_CONTEXT1_CNTL
, EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT
, 1);
908 tmp
= REG_SET_FIELD(tmp
, VM_CONTEXT1_CNTL
, PAGE_TABLE_BLOCK_SIZE
,
909 adev
->vm_manager
.block_size
- 9);
910 WREG32(mmVM_CONTEXT1_CNTL
, tmp
);
911 if (amdgpu_vm_fault_stop
== AMDGPU_VM_FAULT_STOP_ALWAYS
)
912 gmc_v8_0_set_fault_enable_default(adev
, false);
914 gmc_v8_0_set_fault_enable_default(adev
, true);
916 gmc_v8_0_flush_gpu_tlb(adev
, 0);
917 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
918 (unsigned)(adev
->gmc
.gart_size
>> 20),
919 (unsigned long long)adev
->gart
.table_addr
);
920 adev
->gart
.ready
= true;
924 static int gmc_v8_0_gart_init(struct amdgpu_device
*adev
)
928 if (adev
->gart
.robj
) {
929 WARN(1, "R600 PCIE GART already initialized\n");
932 /* Initialize common gart structure */
933 r
= amdgpu_gart_init(adev
);
936 adev
->gart
.table_size
= adev
->gart
.num_gpu_pages
* 8;
937 adev
->gart
.gart_pte_flags
= AMDGPU_PTE_EXECUTABLE
;
938 return amdgpu_gart_table_vram_alloc(adev
);
942 * gmc_v8_0_gart_disable - gart disable
944 * @adev: amdgpu_device pointer
946 * This disables all VM page table (CIK).
948 static void gmc_v8_0_gart_disable(struct amdgpu_device
*adev
)
952 /* Disable all tables */
953 WREG32(mmVM_CONTEXT0_CNTL
, 0);
954 WREG32(mmVM_CONTEXT1_CNTL
, 0);
955 /* Setup TLB control */
956 tmp
= RREG32(mmMC_VM_MX_L1_TLB_CNTL
);
957 tmp
= REG_SET_FIELD(tmp
, MC_VM_MX_L1_TLB_CNTL
, ENABLE_L1_TLB
, 0);
958 tmp
= REG_SET_FIELD(tmp
, MC_VM_MX_L1_TLB_CNTL
, ENABLE_L1_FRAGMENT_PROCESSING
, 0);
959 tmp
= REG_SET_FIELD(tmp
, MC_VM_MX_L1_TLB_CNTL
, ENABLE_ADVANCED_DRIVER_MODEL
, 0);
960 WREG32(mmMC_VM_MX_L1_TLB_CNTL
, tmp
);
962 tmp
= RREG32(mmVM_L2_CNTL
);
963 tmp
= REG_SET_FIELD(tmp
, VM_L2_CNTL
, ENABLE_L2_CACHE
, 0);
964 WREG32(mmVM_L2_CNTL
, tmp
);
965 WREG32(mmVM_L2_CNTL2
, 0);
966 amdgpu_gart_table_vram_unpin(adev
);
970 * gmc_v8_0_gart_fini - vm fini callback
972 * @adev: amdgpu_device pointer
974 * Tears down the driver GART/VM setup (CIK).
976 static void gmc_v8_0_gart_fini(struct amdgpu_device
*adev
)
978 amdgpu_gart_table_vram_free(adev
);
979 amdgpu_gart_fini(adev
);
983 * gmc_v8_0_vm_decode_fault - print human readable fault info
985 * @adev: amdgpu_device pointer
986 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
987 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
989 * Print human readable fault information (CIK).
991 static void gmc_v8_0_vm_decode_fault(struct amdgpu_device
*adev
, u32 status
,
992 u32 addr
, u32 mc_client
, unsigned pasid
)
994 u32 vmid
= REG_GET_FIELD(status
, VM_CONTEXT1_PROTECTION_FAULT_STATUS
, VMID
);
995 u32 protections
= REG_GET_FIELD(status
, VM_CONTEXT1_PROTECTION_FAULT_STATUS
,
997 char block
[5] = { mc_client
>> 24, (mc_client
>> 16) & 0xff,
998 (mc_client
>> 8) & 0xff, mc_client
& 0xff, 0 };
1001 mc_id
= REG_GET_FIELD(status
, VM_CONTEXT1_PROTECTION_FAULT_STATUS
,
1004 dev_err(adev
->dev
, "VM fault (0x%02x, vmid %d, pasid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
1005 protections
, vmid
, pasid
, addr
,
1006 REG_GET_FIELD(status
, VM_CONTEXT1_PROTECTION_FAULT_STATUS
,
1008 "write" : "read", block
, mc_client
, mc_id
);
1011 static int gmc_v8_0_convert_vram_type(int mc_seq_vram_type
)
1013 switch (mc_seq_vram_type
) {
1014 case MC_SEQ_MISC0__MT__GDDR1
:
1015 return AMDGPU_VRAM_TYPE_GDDR1
;
1016 case MC_SEQ_MISC0__MT__DDR2
:
1017 return AMDGPU_VRAM_TYPE_DDR2
;
1018 case MC_SEQ_MISC0__MT__GDDR3
:
1019 return AMDGPU_VRAM_TYPE_GDDR3
;
1020 case MC_SEQ_MISC0__MT__GDDR4
:
1021 return AMDGPU_VRAM_TYPE_GDDR4
;
1022 case MC_SEQ_MISC0__MT__GDDR5
:
1023 return AMDGPU_VRAM_TYPE_GDDR5
;
1024 case MC_SEQ_MISC0__MT__HBM
:
1025 return AMDGPU_VRAM_TYPE_HBM
;
1026 case MC_SEQ_MISC0__MT__DDR3
:
1027 return AMDGPU_VRAM_TYPE_DDR3
;
1029 return AMDGPU_VRAM_TYPE_UNKNOWN
;
1033 static int gmc_v8_0_early_init(void *handle
)
1035 struct amdgpu_device
*adev
= (struct amdgpu_device
*)handle
;
1037 gmc_v8_0_set_gmc_funcs(adev
);
1038 gmc_v8_0_set_irq_funcs(adev
);
1040 adev
->gmc
.shared_aperture_start
= 0x2000000000000000ULL
;
1041 adev
->gmc
.shared_aperture_end
=
1042 adev
->gmc
.shared_aperture_start
+ (4ULL << 30) - 1;
1043 adev
->gmc
.private_aperture_start
=
1044 adev
->gmc
.shared_aperture_end
+ 1;
1045 adev
->gmc
.private_aperture_end
=
1046 adev
->gmc
.private_aperture_start
+ (4ULL << 30) - 1;
1051 static int gmc_v8_0_late_init(void *handle
)
1053 struct amdgpu_device
*adev
= (struct amdgpu_device
*)handle
;
1055 amdgpu_bo_late_init(adev
);
1057 if (amdgpu_vm_fault_stop
!= AMDGPU_VM_FAULT_STOP_ALWAYS
)
1058 return amdgpu_irq_get(adev
, &adev
->gmc
.vm_fault
, 0);
1063 static unsigned gmc_v8_0_get_vbios_fb_size(struct amdgpu_device
*adev
)
1065 u32 d1vga_control
= RREG32(mmD1VGA_CONTROL
);
1068 if (REG_GET_FIELD(d1vga_control
, D1VGA_CONTROL
, D1VGA_MODE_ENABLE
)) {
1069 size
= 9 * 1024 * 1024; /* reserve 8MB for vga emulator and 1 MB for FB */
1071 u32 viewport
= RREG32(mmVIEWPORT_SIZE
);
1072 size
= (REG_GET_FIELD(viewport
, VIEWPORT_SIZE
, VIEWPORT_HEIGHT
) *
1073 REG_GET_FIELD(viewport
, VIEWPORT_SIZE
, VIEWPORT_WIDTH
) *
1076 /* return 0 if the pre-OS buffer uses up most of vram */
1077 if ((adev
->gmc
.real_vram_size
- size
) < (8 * 1024 * 1024))
1082 #define mmMC_SEQ_MISC0_FIJI 0xA71
1084 static int gmc_v8_0_sw_init(void *handle
)
1088 struct amdgpu_device
*adev
= (struct amdgpu_device
*)handle
;
1090 if (adev
->flags
& AMD_IS_APU
) {
1091 adev
->gmc
.vram_type
= AMDGPU_VRAM_TYPE_UNKNOWN
;
1095 if ((adev
->asic_type
== CHIP_FIJI
) ||
1096 (adev
->asic_type
== CHIP_VEGAM
))
1097 tmp
= RREG32(mmMC_SEQ_MISC0_FIJI
);
1099 tmp
= RREG32(mmMC_SEQ_MISC0
);
1100 tmp
&= MC_SEQ_MISC0__MT__MASK
;
1101 adev
->gmc
.vram_type
= gmc_v8_0_convert_vram_type(tmp
);
1104 r
= amdgpu_irq_add_id(adev
, AMDGPU_IH_CLIENTID_LEGACY
, 146, &adev
->gmc
.vm_fault
);
1108 r
= amdgpu_irq_add_id(adev
, AMDGPU_IH_CLIENTID_LEGACY
, 147, &adev
->gmc
.vm_fault
);
1112 /* Adjust VM size here.
1113 * Currently set to 4GB ((1 << 20) 4k pages).
1114 * Max GPUVM size for cayman and SI is 40 bits.
1116 amdgpu_vm_adjust_size(adev
, 64, 9, 1, 40);
1118 /* Set the internal MC address mask
1119 * This is the max address of the GPU's
1120 * internal address space.
1122 adev
->gmc
.mc_mask
= 0xffffffffffULL
; /* 40 bit MC */
1124 /* set DMA mask + need_dma32 flags.
1125 * PCIE - can handle 40-bits.
1126 * IGP - can handle 40-bits
1127 * PCI - dma32 for legacy pci gart, 40 bits on newer asics
1129 adev
->need_dma32
= false;
1130 dma_bits
= adev
->need_dma32
? 32 : 40;
1131 r
= pci_set_dma_mask(adev
->pdev
, DMA_BIT_MASK(dma_bits
));
1133 adev
->need_dma32
= true;
1135 pr_warn("amdgpu: No suitable DMA available\n");
1137 r
= pci_set_consistent_dma_mask(adev
->pdev
, DMA_BIT_MASK(dma_bits
));
1139 pci_set_consistent_dma_mask(adev
->pdev
, DMA_BIT_MASK(32));
1140 pr_warn("amdgpu: No coherent DMA available\n");
1142 adev
->need_swiotlb
= drm_get_max_iomem() > ((u64
)1 << dma_bits
);
1144 r
= gmc_v8_0_init_microcode(adev
);
1146 DRM_ERROR("Failed to load mc firmware!\n");
1150 r
= gmc_v8_0_mc_init(adev
);
1154 adev
->gmc
.stolen_size
= gmc_v8_0_get_vbios_fb_size(adev
);
1156 /* Memory manager */
1157 r
= amdgpu_bo_init(adev
);
1161 r
= gmc_v8_0_gart_init(adev
);
1167 * VMID 0 is reserved for System
1168 * amdgpu graphics/compute will use VMIDs 1-7
1169 * amdkfd will use VMIDs 8-15
1171 adev
->vm_manager
.id_mgr
[0].num_ids
= AMDGPU_NUM_OF_VMIDS
;
1172 amdgpu_vm_manager_init(adev
);
1174 /* base offset of vram pages */
1175 if (adev
->flags
& AMD_IS_APU
) {
1176 u64 tmp
= RREG32(mmMC_VM_FB_OFFSET
);
1179 adev
->vm_manager
.vram_base_offset
= tmp
;
1181 adev
->vm_manager
.vram_base_offset
= 0;
1187 static int gmc_v8_0_sw_fini(void *handle
)
1189 struct amdgpu_device
*adev
= (struct amdgpu_device
*)handle
;
1191 amdgpu_gem_force_release(adev
);
1192 amdgpu_vm_manager_fini(adev
);
1193 gmc_v8_0_gart_fini(adev
);
1194 amdgpu_bo_fini(adev
);
1195 release_firmware(adev
->gmc
.fw
);
1196 adev
->gmc
.fw
= NULL
;
1201 static int gmc_v8_0_hw_init(void *handle
)
1204 struct amdgpu_device
*adev
= (struct amdgpu_device
*)handle
;
1206 gmc_v8_0_init_golden_registers(adev
);
1208 gmc_v8_0_mc_program(adev
);
1210 if (adev
->asic_type
== CHIP_TONGA
) {
1211 r
= gmc_v8_0_tonga_mc_load_microcode(adev
);
1213 DRM_ERROR("Failed to load MC firmware!\n");
1216 } else if (adev
->asic_type
== CHIP_POLARIS11
||
1217 adev
->asic_type
== CHIP_POLARIS10
||
1218 adev
->asic_type
== CHIP_POLARIS12
) {
1219 r
= gmc_v8_0_polaris_mc_load_microcode(adev
);
1221 DRM_ERROR("Failed to load MC firmware!\n");
1226 r
= gmc_v8_0_gart_enable(adev
);
1233 static int gmc_v8_0_hw_fini(void *handle
)
1235 struct amdgpu_device
*adev
= (struct amdgpu_device
*)handle
;
1237 amdgpu_irq_put(adev
, &adev
->gmc
.vm_fault
, 0);
1238 gmc_v8_0_gart_disable(adev
);
1243 static int gmc_v8_0_suspend(void *handle
)
1245 struct amdgpu_device
*adev
= (struct amdgpu_device
*)handle
;
1247 gmc_v8_0_hw_fini(adev
);
1252 static int gmc_v8_0_resume(void *handle
)
1255 struct amdgpu_device
*adev
= (struct amdgpu_device
*)handle
;
1257 r
= gmc_v8_0_hw_init(adev
);
1261 amdgpu_vmid_reset_all(adev
);
1266 static bool gmc_v8_0_is_idle(void *handle
)
1268 struct amdgpu_device
*adev
= (struct amdgpu_device
*)handle
;
1269 u32 tmp
= RREG32(mmSRBM_STATUS
);
1271 if (tmp
& (SRBM_STATUS__MCB_BUSY_MASK
| SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK
|
1272 SRBM_STATUS__MCC_BUSY_MASK
| SRBM_STATUS__MCD_BUSY_MASK
| SRBM_STATUS__VMC_BUSY_MASK
))
1278 static int gmc_v8_0_wait_for_idle(void *handle
)
1282 struct amdgpu_device
*adev
= (struct amdgpu_device
*)handle
;
1284 for (i
= 0; i
< adev
->usec_timeout
; i
++) {
1285 /* read MC_STATUS */
1286 tmp
= RREG32(mmSRBM_STATUS
) & (SRBM_STATUS__MCB_BUSY_MASK
|
1287 SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK
|
1288 SRBM_STATUS__MCC_BUSY_MASK
|
1289 SRBM_STATUS__MCD_BUSY_MASK
|
1290 SRBM_STATUS__VMC_BUSY_MASK
|
1291 SRBM_STATUS__VMC1_BUSY_MASK
);
1300 static bool gmc_v8_0_check_soft_reset(void *handle
)
1302 u32 srbm_soft_reset
= 0;
1303 struct amdgpu_device
*adev
= (struct amdgpu_device
*)handle
;
1304 u32 tmp
= RREG32(mmSRBM_STATUS
);
1306 if (tmp
& SRBM_STATUS__VMC_BUSY_MASK
)
1307 srbm_soft_reset
= REG_SET_FIELD(srbm_soft_reset
,
1308 SRBM_SOFT_RESET
, SOFT_RESET_VMC
, 1);
1310 if (tmp
& (SRBM_STATUS__MCB_BUSY_MASK
| SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK
|
1311 SRBM_STATUS__MCC_BUSY_MASK
| SRBM_STATUS__MCD_BUSY_MASK
)) {
1312 if (!(adev
->flags
& AMD_IS_APU
))
1313 srbm_soft_reset
= REG_SET_FIELD(srbm_soft_reset
,
1314 SRBM_SOFT_RESET
, SOFT_RESET_MC
, 1);
1316 if (srbm_soft_reset
) {
1317 adev
->gmc
.srbm_soft_reset
= srbm_soft_reset
;
1320 adev
->gmc
.srbm_soft_reset
= 0;
1325 static int gmc_v8_0_pre_soft_reset(void *handle
)
1327 struct amdgpu_device
*adev
= (struct amdgpu_device
*)handle
;
1329 if (!adev
->gmc
.srbm_soft_reset
)
1332 gmc_v8_0_mc_stop(adev
);
1333 if (gmc_v8_0_wait_for_idle(adev
)) {
1334 dev_warn(adev
->dev
, "Wait for GMC idle timed out !\n");
1340 static int gmc_v8_0_soft_reset(void *handle
)
1342 struct amdgpu_device
*adev
= (struct amdgpu_device
*)handle
;
1343 u32 srbm_soft_reset
;
1345 if (!adev
->gmc
.srbm_soft_reset
)
1347 srbm_soft_reset
= adev
->gmc
.srbm_soft_reset
;
1349 if (srbm_soft_reset
) {
1352 tmp
= RREG32(mmSRBM_SOFT_RESET
);
1353 tmp
|= srbm_soft_reset
;
1354 dev_info(adev
->dev
, "SRBM_SOFT_RESET=0x%08X\n", tmp
);
1355 WREG32(mmSRBM_SOFT_RESET
, tmp
);
1356 tmp
= RREG32(mmSRBM_SOFT_RESET
);
1360 tmp
&= ~srbm_soft_reset
;
1361 WREG32(mmSRBM_SOFT_RESET
, tmp
);
1362 tmp
= RREG32(mmSRBM_SOFT_RESET
);
1364 /* Wait a little for things to settle down */
1371 static int gmc_v8_0_post_soft_reset(void *handle
)
1373 struct amdgpu_device
*adev
= (struct amdgpu_device
*)handle
;
1375 if (!adev
->gmc
.srbm_soft_reset
)
1378 gmc_v8_0_mc_resume(adev
);
1382 static int gmc_v8_0_vm_fault_interrupt_state(struct amdgpu_device
*adev
,
1383 struct amdgpu_irq_src
*src
,
1385 enum amdgpu_interrupt_state state
)
1388 u32 bits
= (VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK
|
1389 VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK
|
1390 VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK
|
1391 VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK
|
1392 VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK
|
1393 VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK
|
1394 VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK
);
1397 case AMDGPU_IRQ_STATE_DISABLE
:
1398 /* system context */
1399 tmp
= RREG32(mmVM_CONTEXT0_CNTL
);
1401 WREG32(mmVM_CONTEXT0_CNTL
, tmp
);
1403 tmp
= RREG32(mmVM_CONTEXT1_CNTL
);
1405 WREG32(mmVM_CONTEXT1_CNTL
, tmp
);
1407 case AMDGPU_IRQ_STATE_ENABLE
:
1408 /* system context */
1409 tmp
= RREG32(mmVM_CONTEXT0_CNTL
);
1411 WREG32(mmVM_CONTEXT0_CNTL
, tmp
);
1413 tmp
= RREG32(mmVM_CONTEXT1_CNTL
);
1415 WREG32(mmVM_CONTEXT1_CNTL
, tmp
);
1424 static int gmc_v8_0_process_interrupt(struct amdgpu_device
*adev
,
1425 struct amdgpu_irq_src
*source
,
1426 struct amdgpu_iv_entry
*entry
)
1428 u32 addr
, status
, mc_client
;
1430 if (amdgpu_sriov_vf(adev
)) {
1431 dev_err(adev
->dev
, "GPU fault detected: %d 0x%08x\n",
1432 entry
->src_id
, entry
->src_data
[0]);
1433 dev_err(adev
->dev
, " Can't decode VM fault info here on SRIOV VF\n");
1437 addr
= RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_ADDR
);
1438 status
= RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_STATUS
);
1439 mc_client
= RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_MCCLIENT
);
1440 /* reset addr and status */
1441 WREG32_P(mmVM_CONTEXT1_CNTL2
, 1, ~1);
1443 if (!addr
&& !status
)
1446 if (amdgpu_vm_fault_stop
== AMDGPU_VM_FAULT_STOP_FIRST
)
1447 gmc_v8_0_set_fault_enable_default(adev
, false);
1449 if (printk_ratelimit()) {
1450 dev_err(adev
->dev
, "GPU fault detected: %d 0x%08x\n",
1451 entry
->src_id
, entry
->src_data
[0]);
1452 dev_err(adev
->dev
, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
1454 dev_err(adev
->dev
, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1456 gmc_v8_0_vm_decode_fault(adev
, status
, addr
, mc_client
,
1463 static void fiji_update_mc_medium_grain_clock_gating(struct amdgpu_device
*adev
,
1468 if (enable
&& (adev
->cg_flags
& AMD_CG_SUPPORT_MC_MGCG
)) {
1469 data
= RREG32(mmMC_HUB_MISC_HUB_CG
);
1470 data
|= MC_HUB_MISC_HUB_CG__ENABLE_MASK
;
1471 WREG32(mmMC_HUB_MISC_HUB_CG
, data
);
1473 data
= RREG32(mmMC_HUB_MISC_SIP_CG
);
1474 data
|= MC_HUB_MISC_SIP_CG__ENABLE_MASK
;
1475 WREG32(mmMC_HUB_MISC_SIP_CG
, data
);
1477 data
= RREG32(mmMC_HUB_MISC_VM_CG
);
1478 data
|= MC_HUB_MISC_VM_CG__ENABLE_MASK
;
1479 WREG32(mmMC_HUB_MISC_VM_CG
, data
);
1481 data
= RREG32(mmMC_XPB_CLK_GAT
);
1482 data
|= MC_XPB_CLK_GAT__ENABLE_MASK
;
1483 WREG32(mmMC_XPB_CLK_GAT
, data
);
1485 data
= RREG32(mmATC_MISC_CG
);
1486 data
|= ATC_MISC_CG__ENABLE_MASK
;
1487 WREG32(mmATC_MISC_CG
, data
);
1489 data
= RREG32(mmMC_CITF_MISC_WR_CG
);
1490 data
|= MC_CITF_MISC_WR_CG__ENABLE_MASK
;
1491 WREG32(mmMC_CITF_MISC_WR_CG
, data
);
1493 data
= RREG32(mmMC_CITF_MISC_RD_CG
);
1494 data
|= MC_CITF_MISC_RD_CG__ENABLE_MASK
;
1495 WREG32(mmMC_CITF_MISC_RD_CG
, data
);
1497 data
= RREG32(mmMC_CITF_MISC_VM_CG
);
1498 data
|= MC_CITF_MISC_VM_CG__ENABLE_MASK
;
1499 WREG32(mmMC_CITF_MISC_VM_CG
, data
);
1501 data
= RREG32(mmVM_L2_CG
);
1502 data
|= VM_L2_CG__ENABLE_MASK
;
1503 WREG32(mmVM_L2_CG
, data
);
1505 data
= RREG32(mmMC_HUB_MISC_HUB_CG
);
1506 data
&= ~MC_HUB_MISC_HUB_CG__ENABLE_MASK
;
1507 WREG32(mmMC_HUB_MISC_HUB_CG
, data
);
1509 data
= RREG32(mmMC_HUB_MISC_SIP_CG
);
1510 data
&= ~MC_HUB_MISC_SIP_CG__ENABLE_MASK
;
1511 WREG32(mmMC_HUB_MISC_SIP_CG
, data
);
1513 data
= RREG32(mmMC_HUB_MISC_VM_CG
);
1514 data
&= ~MC_HUB_MISC_VM_CG__ENABLE_MASK
;
1515 WREG32(mmMC_HUB_MISC_VM_CG
, data
);
1517 data
= RREG32(mmMC_XPB_CLK_GAT
);
1518 data
&= ~MC_XPB_CLK_GAT__ENABLE_MASK
;
1519 WREG32(mmMC_XPB_CLK_GAT
, data
);
1521 data
= RREG32(mmATC_MISC_CG
);
1522 data
&= ~ATC_MISC_CG__ENABLE_MASK
;
1523 WREG32(mmATC_MISC_CG
, data
);
1525 data
= RREG32(mmMC_CITF_MISC_WR_CG
);
1526 data
&= ~MC_CITF_MISC_WR_CG__ENABLE_MASK
;
1527 WREG32(mmMC_CITF_MISC_WR_CG
, data
);
1529 data
= RREG32(mmMC_CITF_MISC_RD_CG
);
1530 data
&= ~MC_CITF_MISC_RD_CG__ENABLE_MASK
;
1531 WREG32(mmMC_CITF_MISC_RD_CG
, data
);
1533 data
= RREG32(mmMC_CITF_MISC_VM_CG
);
1534 data
&= ~MC_CITF_MISC_VM_CG__ENABLE_MASK
;
1535 WREG32(mmMC_CITF_MISC_VM_CG
, data
);
1537 data
= RREG32(mmVM_L2_CG
);
1538 data
&= ~VM_L2_CG__ENABLE_MASK
;
1539 WREG32(mmVM_L2_CG
, data
);
1543 static void fiji_update_mc_light_sleep(struct amdgpu_device
*adev
,
1548 if (enable
&& (adev
->cg_flags
& AMD_CG_SUPPORT_MC_LS
)) {
1549 data
= RREG32(mmMC_HUB_MISC_HUB_CG
);
1550 data
|= MC_HUB_MISC_HUB_CG__MEM_LS_ENABLE_MASK
;
1551 WREG32(mmMC_HUB_MISC_HUB_CG
, data
);
1553 data
= RREG32(mmMC_HUB_MISC_SIP_CG
);
1554 data
|= MC_HUB_MISC_SIP_CG__MEM_LS_ENABLE_MASK
;
1555 WREG32(mmMC_HUB_MISC_SIP_CG
, data
);
1557 data
= RREG32(mmMC_HUB_MISC_VM_CG
);
1558 data
|= MC_HUB_MISC_VM_CG__MEM_LS_ENABLE_MASK
;
1559 WREG32(mmMC_HUB_MISC_VM_CG
, data
);
1561 data
= RREG32(mmMC_XPB_CLK_GAT
);
1562 data
|= MC_XPB_CLK_GAT__MEM_LS_ENABLE_MASK
;
1563 WREG32(mmMC_XPB_CLK_GAT
, data
);
1565 data
= RREG32(mmATC_MISC_CG
);
1566 data
|= ATC_MISC_CG__MEM_LS_ENABLE_MASK
;
1567 WREG32(mmATC_MISC_CG
, data
);
1569 data
= RREG32(mmMC_CITF_MISC_WR_CG
);
1570 data
|= MC_CITF_MISC_WR_CG__MEM_LS_ENABLE_MASK
;
1571 WREG32(mmMC_CITF_MISC_WR_CG
, data
);
1573 data
= RREG32(mmMC_CITF_MISC_RD_CG
);
1574 data
|= MC_CITF_MISC_RD_CG__MEM_LS_ENABLE_MASK
;
1575 WREG32(mmMC_CITF_MISC_RD_CG
, data
);
1577 data
= RREG32(mmMC_CITF_MISC_VM_CG
);
1578 data
|= MC_CITF_MISC_VM_CG__MEM_LS_ENABLE_MASK
;
1579 WREG32(mmMC_CITF_MISC_VM_CG
, data
);
1581 data
= RREG32(mmVM_L2_CG
);
1582 data
|= VM_L2_CG__MEM_LS_ENABLE_MASK
;
1583 WREG32(mmVM_L2_CG
, data
);
1585 data
= RREG32(mmMC_HUB_MISC_HUB_CG
);
1586 data
&= ~MC_HUB_MISC_HUB_CG__MEM_LS_ENABLE_MASK
;
1587 WREG32(mmMC_HUB_MISC_HUB_CG
, data
);
1589 data
= RREG32(mmMC_HUB_MISC_SIP_CG
);
1590 data
&= ~MC_HUB_MISC_SIP_CG__MEM_LS_ENABLE_MASK
;
1591 WREG32(mmMC_HUB_MISC_SIP_CG
, data
);
1593 data
= RREG32(mmMC_HUB_MISC_VM_CG
);
1594 data
&= ~MC_HUB_MISC_VM_CG__MEM_LS_ENABLE_MASK
;
1595 WREG32(mmMC_HUB_MISC_VM_CG
, data
);
1597 data
= RREG32(mmMC_XPB_CLK_GAT
);
1598 data
&= ~MC_XPB_CLK_GAT__MEM_LS_ENABLE_MASK
;
1599 WREG32(mmMC_XPB_CLK_GAT
, data
);
1601 data
= RREG32(mmATC_MISC_CG
);
1602 data
&= ~ATC_MISC_CG__MEM_LS_ENABLE_MASK
;
1603 WREG32(mmATC_MISC_CG
, data
);
1605 data
= RREG32(mmMC_CITF_MISC_WR_CG
);
1606 data
&= ~MC_CITF_MISC_WR_CG__MEM_LS_ENABLE_MASK
;
1607 WREG32(mmMC_CITF_MISC_WR_CG
, data
);
1609 data
= RREG32(mmMC_CITF_MISC_RD_CG
);
1610 data
&= ~MC_CITF_MISC_RD_CG__MEM_LS_ENABLE_MASK
;
1611 WREG32(mmMC_CITF_MISC_RD_CG
, data
);
1613 data
= RREG32(mmMC_CITF_MISC_VM_CG
);
1614 data
&= ~MC_CITF_MISC_VM_CG__MEM_LS_ENABLE_MASK
;
1615 WREG32(mmMC_CITF_MISC_VM_CG
, data
);
1617 data
= RREG32(mmVM_L2_CG
);
1618 data
&= ~VM_L2_CG__MEM_LS_ENABLE_MASK
;
1619 WREG32(mmVM_L2_CG
, data
);
1623 static int gmc_v8_0_set_clockgating_state(void *handle
,
1624 enum amd_clockgating_state state
)
1626 struct amdgpu_device
*adev
= (struct amdgpu_device
*)handle
;
1628 if (amdgpu_sriov_vf(adev
))
1631 switch (adev
->asic_type
) {
1633 fiji_update_mc_medium_grain_clock_gating(adev
,
1634 state
== AMD_CG_STATE_GATE
);
1635 fiji_update_mc_light_sleep(adev
,
1636 state
== AMD_CG_STATE_GATE
);
1644 static int gmc_v8_0_set_powergating_state(void *handle
,
1645 enum amd_powergating_state state
)
1650 static void gmc_v8_0_get_clockgating_state(void *handle
, u32
*flags
)
1652 struct amdgpu_device
*adev
= (struct amdgpu_device
*)handle
;
1655 if (amdgpu_sriov_vf(adev
))
1658 /* AMD_CG_SUPPORT_MC_MGCG */
1659 data
= RREG32(mmMC_HUB_MISC_HUB_CG
);
1660 if (data
& MC_HUB_MISC_HUB_CG__ENABLE_MASK
)
1661 *flags
|= AMD_CG_SUPPORT_MC_MGCG
;
1663 /* AMD_CG_SUPPORT_MC_LS */
1664 if (data
& MC_HUB_MISC_HUB_CG__MEM_LS_ENABLE_MASK
)
1665 *flags
|= AMD_CG_SUPPORT_MC_LS
;
1668 static const struct amd_ip_funcs gmc_v8_0_ip_funcs
= {
1670 .early_init
= gmc_v8_0_early_init
,
1671 .late_init
= gmc_v8_0_late_init
,
1672 .sw_init
= gmc_v8_0_sw_init
,
1673 .sw_fini
= gmc_v8_0_sw_fini
,
1674 .hw_init
= gmc_v8_0_hw_init
,
1675 .hw_fini
= gmc_v8_0_hw_fini
,
1676 .suspend
= gmc_v8_0_suspend
,
1677 .resume
= gmc_v8_0_resume
,
1678 .is_idle
= gmc_v8_0_is_idle
,
1679 .wait_for_idle
= gmc_v8_0_wait_for_idle
,
1680 .check_soft_reset
= gmc_v8_0_check_soft_reset
,
1681 .pre_soft_reset
= gmc_v8_0_pre_soft_reset
,
1682 .soft_reset
= gmc_v8_0_soft_reset
,
1683 .post_soft_reset
= gmc_v8_0_post_soft_reset
,
1684 .set_clockgating_state
= gmc_v8_0_set_clockgating_state
,
1685 .set_powergating_state
= gmc_v8_0_set_powergating_state
,
1686 .get_clockgating_state
= gmc_v8_0_get_clockgating_state
,
1689 static const struct amdgpu_gmc_funcs gmc_v8_0_gmc_funcs
= {
1690 .flush_gpu_tlb
= gmc_v8_0_flush_gpu_tlb
,
1691 .emit_flush_gpu_tlb
= gmc_v8_0_emit_flush_gpu_tlb
,
1692 .emit_pasid_mapping
= gmc_v8_0_emit_pasid_mapping
,
1693 .set_pte_pde
= gmc_v8_0_set_pte_pde
,
1694 .set_prt
= gmc_v8_0_set_prt
,
1695 .get_vm_pte_flags
= gmc_v8_0_get_vm_pte_flags
,
1696 .get_vm_pde
= gmc_v8_0_get_vm_pde
1699 static const struct amdgpu_irq_src_funcs gmc_v8_0_irq_funcs
= {
1700 .set
= gmc_v8_0_vm_fault_interrupt_state
,
1701 .process
= gmc_v8_0_process_interrupt
,
1704 static void gmc_v8_0_set_gmc_funcs(struct amdgpu_device
*adev
)
1706 if (adev
->gmc
.gmc_funcs
== NULL
)
1707 adev
->gmc
.gmc_funcs
= &gmc_v8_0_gmc_funcs
;
1710 static void gmc_v8_0_set_irq_funcs(struct amdgpu_device
*adev
)
1712 adev
->gmc
.vm_fault
.num_types
= 1;
1713 adev
->gmc
.vm_fault
.funcs
= &gmc_v8_0_irq_funcs
;
1716 const struct amdgpu_ip_block_version gmc_v8_0_ip_block
=
1718 .type
= AMD_IP_BLOCK_TYPE_GMC
,
1722 .funcs
= &gmc_v8_0_ip_funcs
,
1725 const struct amdgpu_ip_block_version gmc_v8_1_ip_block
=
1727 .type
= AMD_IP_BLOCK_TYPE_GMC
,
1731 .funcs
= &gmc_v8_0_ip_funcs
,
1734 const struct amdgpu_ip_block_version gmc_v8_5_ip_block
=
1736 .type
= AMD_IP_BLOCK_TYPE_GMC
,
1740 .funcs
= &gmc_v8_0_ip_funcs
,